Lines Matching +full:try +full:- +full:power +full:- +full:role
1 // SPDX-License-Identifier: GPL-2.0
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
57 if (lgr->is_smcd) { in smc_lgr_list_head()
58 *lgr_lock = &lgr->smcd->lgr_lock; in smc_lgr_list_head()
59 return &lgr->smcd->lgr_list; in smc_lgr_list_head()
70 * otherwise there is a risk of out-of-sync link groups. in smc_lgr_schedule_free_work()
72 if (!lgr->freeing) { in smc_lgr_schedule_free_work()
73 mod_delayed_work(system_wq, &lgr->free_work, in smc_lgr_schedule_free_work()
74 (!lgr->is_smcd && lgr->role == SMC_CLNT) ? in smc_lgr_schedule_free_work()
89 u32 token = conn->alert_token_local; in smc_lgr_add_alert_token()
91 link = &conn->lgr->conns_all.rb_node; in smc_lgr_add_alert_token()
97 if (cur->alert_token_local > token) in smc_lgr_add_alert_token()
98 link = &parent->rb_left; in smc_lgr_add_alert_token()
100 link = &parent->rb_right; in smc_lgr_add_alert_token()
103 rb_link_node(&conn->alert_node, parent, link); in smc_lgr_add_alert_token()
104 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); in smc_lgr_add_alert_token()
107 /* assign an SMC-R link to the connection */
116 struct smc_link *lnk = &conn->lgr->lnk[i]; in smcr_lgr_conn_assign_link()
118 if (lnk->state != expected || lnk->link_is_asym) in smcr_lgr_conn_assign_link()
120 if (conn->lgr->role == SMC_CLNT) { in smcr_lgr_conn_assign_link()
121 conn->lnk = lnk; /* temporary, SMC server assigns link*/ in smcr_lgr_conn_assign_link()
124 if (conn->lgr->conns_num % 2) { in smcr_lgr_conn_assign_link()
128 lnk2 = &conn->lgr->lnk[j]; in smcr_lgr_conn_assign_link()
129 if (lnk2->state == expected && in smcr_lgr_conn_assign_link()
130 !lnk2->link_is_asym) { in smcr_lgr_conn_assign_link()
131 conn->lnk = lnk2; in smcr_lgr_conn_assign_link()
136 if (!conn->lnk) in smcr_lgr_conn_assign_link()
137 conn->lnk = lnk; in smcr_lgr_conn_assign_link()
140 if (!conn->lnk) in smcr_lgr_conn_assign_link()
156 if (!conn->lgr->is_smcd) { in smc_lgr_register_conn()
164 sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */ in smc_lgr_register_conn()
165 while (!conn->alert_token_local) { in smc_lgr_register_conn()
166 conn->alert_token_local = atomic_inc_return(&nexttoken); in smc_lgr_register_conn()
167 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) in smc_lgr_register_conn()
168 conn->alert_token_local = 0; in smc_lgr_register_conn()
171 conn->lgr->conns_num++; in smc_lgr_register_conn()
180 struct smc_link_group *lgr = conn->lgr; in __smc_lgr_unregister_conn()
182 rb_erase(&conn->alert_node, &lgr->conns_all); in __smc_lgr_unregister_conn()
183 lgr->conns_num--; in __smc_lgr_unregister_conn()
184 conn->alert_token_local = 0; in __smc_lgr_unregister_conn()
185 sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */ in __smc_lgr_unregister_conn()
192 struct smc_link_group *lgr = conn->lgr; in smc_lgr_unregister_conn()
196 write_lock_bh(&lgr->conns_lock); in smc_lgr_unregister_conn()
197 if (conn->alert_token_local) { in smc_lgr_unregister_conn()
200 write_unlock_bh(&lgr->conns_lock); in smc_lgr_unregister_conn()
201 conn->lgr = NULL; in smc_lgr_unregister_conn()
206 struct smc_link_group *lgr = conn->lgr; in smc_lgr_cleanup_early()
228 struct smc_link *lnk = &lgr->lnk[i]; in smcr_lgr_link_deactivate_all()
231 lnk->state = SMC_LNK_INACTIVE; in smcr_lgr_link_deactivate_all()
233 wake_up_all(&lgr->llc_msg_waiter); in smcr_lgr_link_deactivate_all()
234 wake_up_all(&lgr->llc_flow_waiter); in smcr_lgr_link_deactivate_all()
249 if (lgr->freeing) { in smc_lgr_free_work()
253 read_lock_bh(&lgr->conns_lock); in smc_lgr_free_work()
254 conns = RB_EMPTY_ROOT(&lgr->conns_all); in smc_lgr_free_work()
255 read_unlock_bh(&lgr->conns_lock); in smc_lgr_free_work()
260 list_del_init(&lgr->list); /* remove from smc_lgr_list */ in smc_lgr_free_work()
261 lgr->freeing = 1; /* this instance does the freeing, no new schedule */ in smc_lgr_free_work()
263 cancel_delayed_work(&lgr->free_work); in smc_lgr_free_work()
265 if (!lgr->is_smcd && !lgr->terminating) in smc_lgr_free_work()
268 if (lgr->is_smcd && !lgr->terminating) in smc_lgr_free_work()
270 if (!lgr->is_smcd) in smc_lgr_free_work()
290 link_id = ++lgr->next_link_id; in smcr_next_link_id()
292 link_id = ++lgr->next_link_id; in smcr_next_link_id()
294 if (smc_link_usable(&lgr->lnk[i]) && in smcr_next_link_id()
295 lgr->lnk[i].link_id == link_id) in smcr_next_link_id()
309 get_device(&ini->ib_dev->ibdev->dev); in smcr_link_init()
310 atomic_inc(&ini->ib_dev->lnk_cnt); in smcr_link_init()
311 lnk->link_id = smcr_next_link_id(lgr); in smcr_link_init()
312 lnk->lgr = lgr; in smcr_link_init()
313 lnk->link_idx = link_idx; in smcr_link_init()
314 lnk->smcibdev = ini->ib_dev; in smcr_link_init()
315 lnk->ibport = ini->ib_port; in smcr_link_init()
316 lnk->path_mtu = ini->ib_dev->pattr[ini->ib_port - 1].active_mtu; in smcr_link_init()
318 INIT_WORK(&lnk->link_down_wrk, smc_link_down_work); in smcr_link_init()
319 if (!ini->ib_dev->initialized) { in smcr_link_init()
320 rc = (int)smc_ib_setup_per_ibdev(ini->ib_dev); in smcr_link_init()
325 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + in smcr_link_init()
327 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport, in smcr_link_init()
328 ini->vlan_id, lnk->gid, &lnk->sgid_index); in smcr_link_init()
346 lnk->state = SMC_LNK_ACTIVATING; in smcr_link_init()
358 put_device(&ini->ib_dev->ibdev->dev); in smcr_link_init()
360 lnk->state = SMC_LNK_UNUSED; in smcr_link_init()
361 if (!atomic_dec_return(&ini->ib_dev->lnk_cnt)) in smcr_link_init()
362 wake_up(&ini->ib_dev->lnks_deleted); in smcr_link_init()
377 if (ini->is_smcd && ini->vlan_id) { in smc_lgr_create()
378 if (smc_ism_get_vlan(ini->ism_dev[ini->ism_selected], in smc_lgr_create()
379 ini->vlan_id)) { in smc_lgr_create()
390 lgr->tx_wq = alloc_workqueue("smc_tx_wq-%*phN", 0, 0, in smc_lgr_create()
391 SMC_LGR_ID_SIZE, &lgr->id); in smc_lgr_create()
392 if (!lgr->tx_wq) { in smc_lgr_create()
393 rc = -ENOMEM; in smc_lgr_create()
396 lgr->is_smcd = ini->is_smcd; in smc_lgr_create()
397 lgr->sync_err = 0; in smc_lgr_create()
398 lgr->terminating = 0; in smc_lgr_create()
399 lgr->freeing = 0; in smc_lgr_create()
400 lgr->vlan_id = ini->vlan_id; in smc_lgr_create()
401 mutex_init(&lgr->sndbufs_lock); in smc_lgr_create()
402 mutex_init(&lgr->rmbs_lock); in smc_lgr_create()
403 rwlock_init(&lgr->conns_lock); in smc_lgr_create()
405 INIT_LIST_HEAD(&lgr->sndbufs[i]); in smc_lgr_create()
406 INIT_LIST_HEAD(&lgr->rmbs[i]); in smc_lgr_create()
408 lgr->next_link_id = 0; in smc_lgr_create()
410 memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE); in smc_lgr_create()
411 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); in smc_lgr_create()
412 INIT_WORK(&lgr->terminate_work, smc_lgr_terminate_work); in smc_lgr_create()
413 lgr->conns_all = RB_ROOT; in smc_lgr_create()
414 if (ini->is_smcd) { in smc_lgr_create()
415 /* SMC-D specific settings */ in smc_lgr_create()
416 get_device(&ini->ism_dev[ini->ism_selected]->dev); in smc_lgr_create()
417 lgr->peer_gid = ini->ism_peer_gid[ini->ism_selected]; in smc_lgr_create()
418 lgr->smcd = ini->ism_dev[ini->ism_selected]; in smc_lgr_create()
419 lgr_list = &ini->ism_dev[ini->ism_selected]->lgr_list; in smc_lgr_create()
420 lgr_lock = &lgr->smcd->lgr_lock; in smc_lgr_create()
421 lgr->smc_version = ini->smcd_version; in smc_lgr_create()
422 lgr->peer_shutdown = 0; in smc_lgr_create()
423 atomic_inc(&ini->ism_dev[ini->ism_selected]->lgr_cnt); in smc_lgr_create()
425 /* SMC-R specific settings */ in smc_lgr_create()
426 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; in smc_lgr_create()
427 memcpy(lgr->peer_systemid, ini->ib_lcl->id_for_peer, in smc_lgr_create()
429 memcpy(lgr->pnet_id, ini->ib_dev->pnetid[ini->ib_port - 1], in smc_lgr_create()
434 lnk = &lgr->lnk[link_idx]; in smc_lgr_create()
442 smc->conn.lgr = lgr; in smc_lgr_create()
444 list_add_tail(&lgr->list, lgr_list); in smc_lgr_create()
449 destroy_workqueue(lgr->tx_wq); in smc_lgr_create()
453 if (ini->is_smcd && ini->vlan_id) in smc_lgr_create()
454 smc_ism_put_vlan(ini->ism_dev[ini->ism_selected], ini->vlan_id); in smc_lgr_create()
457 if (rc == -ENOMEM) in smc_lgr_create()
467 int buffer_len = conn->peer_rmbe_size; in smc_write_space()
472 smc_curs_copy(&prod, &conn->local_tx_ctrl.prod, conn); in smc_write_space()
473 smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn); in smc_write_space()
475 space = buffer_len - smc_curs_diff(buffer_len, &cons, &prod); in smc_write_space()
482 struct smc_connection *conn = &smc->conn; in smc_switch_cursor()
487 smc_curs_copy(&conn->tx_curs_sent, &conn->tx_curs_fin, conn); in smc_switch_cursor()
488 smc_curs_copy(&fin, &conn->local_tx_ctrl_fin, conn); in smc_switch_cursor()
490 smc_curs_copy(&conn->local_tx_ctrl.prod, &fin, conn); in smc_switch_cursor()
491 smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn); in smc_switch_cursor()
493 if (smc_curs_comp(conn->peer_rmbe_size, &cons, &fin) < 0) { in smc_switch_cursor()
497 diff = smc_curs_diff(conn->peer_rmbe_size, &fin, &cons); in smc_switch_cursor()
498 smc_curs_add(conn->sndbuf_desc->len, in smc_switch_cursor()
499 &conn->tx_curs_sent, diff); in smc_switch_cursor()
500 smc_curs_add(conn->sndbuf_desc->len, in smc_switch_cursor()
501 &conn->tx_curs_fin, diff); in smc_switch_cursor()
504 atomic_add(diff, &conn->sndbuf_space); in smc_switch_cursor()
507 smc_curs_add(conn->peer_rmbe_size, in smc_switch_cursor()
508 &conn->local_tx_ctrl.prod, diff); in smc_switch_cursor()
509 smc_curs_add(conn->peer_rmbe_size, in smc_switch_cursor()
510 &conn->local_tx_ctrl_fin, diff); in smc_switch_cursor()
513 atomic_set(&smc->conn.peer_rmbe_space, smc_write_space(conn)); in smc_switch_cursor()
515 if (smc->sk.sk_state != SMC_INIT && in smc_switch_cursor()
516 smc->sk.sk_state != SMC_CLOSED) { in smc_switch_cursor()
519 queue_delayed_work(conn->lgr->tx_wq, &conn->tx_work, 0); in smc_switch_cursor()
520 smc->sk.sk_data_ready(&smc->sk); in smc_switch_cursor()
523 smc_wr_tx_put_slot(conn->lnk, in smc_switch_cursor()
544 if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx) in smc_switch_conns()
546 if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev && in smc_switch_conns()
547 from_lnk->ibport == lgr->lnk[i].ibport) { in smc_switch_conns()
550 to_lnk = &lgr->lnk[i]; in smc_switch_conns()
558 read_lock_bh(&lgr->conns_lock); in smc_switch_conns()
559 for (node = rb_first(&lgr->conns_all); node; node = rb_next(node)) { in smc_switch_conns()
561 if (conn->lnk != from_lnk) in smc_switch_conns()
564 /* conn->lnk not yet set in SMC_INIT state */ in smc_switch_conns()
565 if (smc->sk.sk_state == SMC_INIT) in smc_switch_conns()
567 if (smc->sk.sk_state == SMC_CLOSED || in smc_switch_conns()
568 smc->sk.sk_state == SMC_PEERCLOSEWAIT1 || in smc_switch_conns()
569 smc->sk.sk_state == SMC_PEERCLOSEWAIT2 || in smc_switch_conns()
570 smc->sk.sk_state == SMC_APPFINCLOSEWAIT || in smc_switch_conns()
571 smc->sk.sk_state == SMC_APPCLOSEWAIT1 || in smc_switch_conns()
572 smc->sk.sk_state == SMC_APPCLOSEWAIT2 || in smc_switch_conns()
573 smc->sk.sk_state == SMC_PEERFINCLOSEWAIT || in smc_switch_conns()
574 smc->sk.sk_state == SMC_PEERABORTWAIT || in smc_switch_conns()
575 smc->sk.sk_state == SMC_PROCESSABORT) { in smc_switch_conns()
576 spin_lock_bh(&conn->send_lock); in smc_switch_conns()
577 conn->lnk = to_lnk; in smc_switch_conns()
578 spin_unlock_bh(&conn->send_lock); in smc_switch_conns()
581 sock_hold(&smc->sk); in smc_switch_conns()
582 read_unlock_bh(&lgr->conns_lock); in smc_switch_conns()
583 /* pre-fetch buffer outside of send_lock, might sleep */ in smc_switch_conns()
590 spin_lock_bh(&conn->send_lock); in smc_switch_conns()
591 conn->lnk = to_lnk; in smc_switch_conns()
593 spin_unlock_bh(&conn->send_lock); in smc_switch_conns()
594 sock_put(&smc->sk); in smc_switch_conns()
601 read_unlock_bh(&lgr->conns_lock); in smc_switch_conns()
610 if (rmb_desc->is_conf_rkey && !list_empty(&lgr->list)) { in smcr_buf_unuse()
615 mutex_lock(&lgr->llc_conf_mutex); in smcr_buf_unuse()
617 rmb_desc->is_conf_rkey = false; in smcr_buf_unuse()
618 mutex_unlock(&lgr->llc_conf_mutex); in smcr_buf_unuse()
619 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); in smcr_buf_unuse()
623 if (rmb_desc->is_reg_err) { in smcr_buf_unuse()
625 mutex_lock(&lgr->rmbs_lock); in smcr_buf_unuse()
626 list_del(&rmb_desc->list); in smcr_buf_unuse()
627 mutex_unlock(&lgr->rmbs_lock); in smcr_buf_unuse()
631 rmb_desc->used = 0; in smcr_buf_unuse()
638 if (conn->sndbuf_desc) in smc_buf_unuse()
639 conn->sndbuf_desc->used = 0; in smc_buf_unuse()
640 if (conn->rmb_desc && lgr->is_smcd) in smc_buf_unuse()
641 conn->rmb_desc->used = 0; in smc_buf_unuse()
642 else if (conn->rmb_desc) in smc_buf_unuse()
643 smcr_buf_unuse(conn->rmb_desc, lgr); in smc_buf_unuse()
649 struct smc_link_group *lgr = conn->lgr; in smc_conn_free()
653 if (lgr->is_smcd) { in smc_conn_free()
654 if (!list_empty(&lgr->list)) in smc_conn_free()
656 tasklet_kill(&conn->rx_tsklet); in smc_conn_free()
659 if (current_work() != &conn->abort_work) in smc_conn_free()
660 cancel_work_sync(&conn->abort_work); in smc_conn_free()
662 if (!list_empty(&lgr->list)) { in smc_conn_free()
667 if (!lgr->conns_num) in smc_conn_free()
676 buf_desc->is_reg_mr[lnk->link_idx] = false; in smcr_buf_unmap_link()
677 if (!buf_desc->is_map_ib[lnk->link_idx]) in smcr_buf_unmap_link()
680 if (buf_desc->mr_rx[lnk->link_idx]) { in smcr_buf_unmap_link()
682 buf_desc->mr_rx[lnk->link_idx]); in smcr_buf_unmap_link()
683 buf_desc->mr_rx[lnk->link_idx] = NULL; in smcr_buf_unmap_link()
689 sg_free_table(&buf_desc->sgt[lnk->link_idx]); in smcr_buf_unmap_link()
690 buf_desc->is_map_ib[lnk->link_idx] = false; in smcr_buf_unmap_link()
696 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_unmap_lgr()
701 mutex_lock(&lgr->rmbs_lock); in smcr_buf_unmap_lgr()
702 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) in smcr_buf_unmap_lgr()
704 mutex_unlock(&lgr->rmbs_lock); in smcr_buf_unmap_lgr()
705 mutex_lock(&lgr->sndbufs_lock); in smcr_buf_unmap_lgr()
706 list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], in smcr_buf_unmap_lgr()
709 mutex_unlock(&lgr->sndbufs_lock); in smcr_buf_unmap_lgr()
715 struct smc_link_group *lgr = lnk->lgr; in smcr_rtoken_clear_link()
719 lgr->rtokens[i][lnk->link_idx].rkey = 0; in smcr_rtoken_clear_link()
720 lgr->rtokens[i][lnk->link_idx].dma_addr = 0; in smcr_rtoken_clear_link()
724 /* must be called under lgr->llc_conf_mutex lock */
729 if (!lnk->lgr || lnk->state == SMC_LNK_UNUSED) in smcr_link_clear()
731 lnk->peer_qpn = 0; in smcr_link_clear()
740 put_device(&lnk->smcibdev->ibdev->dev); in smcr_link_clear()
741 smcibdev = lnk->smcibdev; in smcr_link_clear()
743 lnk->state = SMC_LNK_UNUSED; in smcr_link_clear()
744 if (!atomic_dec_return(&smcibdev->lnk_cnt)) in smcr_link_clear()
745 wake_up(&smcibdev->lnks_deleted); in smcr_link_clear()
754 smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]); in smcr_buf_free()
756 if (buf_desc->pages) in smcr_buf_free()
757 __free_pages(buf_desc->pages, buf_desc->order); in smcr_buf_free()
766 buf_desc->len += sizeof(struct smcd_cdc_msg); in smcd_buf_free()
767 smc_ism_unregister_dmb(lgr->smcd, buf_desc); in smcd_buf_free()
769 kfree(buf_desc->cpu_addr); in smcd_buf_free()
777 if (lgr->is_smcd) in smc_buf_free()
791 buf_list = &lgr->rmbs[i]; in __smc_lgr_free_bufs()
793 buf_list = &lgr->sndbufs[i]; in __smc_lgr_free_bufs()
796 list_del(&buf_desc->list); in __smc_lgr_free_bufs()
815 if (!lgr->is_smcd) { in smc_lgr_free()
816 mutex_lock(&lgr->llc_conf_mutex); in smc_lgr_free()
818 if (lgr->lnk[i].state != SMC_LNK_UNUSED) in smc_lgr_free()
819 smcr_link_clear(&lgr->lnk[i], false); in smc_lgr_free()
821 mutex_unlock(&lgr->llc_conf_mutex); in smc_lgr_free()
826 destroy_workqueue(lgr->tx_wq); in smc_lgr_free()
827 if (lgr->is_smcd) { in smc_lgr_free()
828 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id); in smc_lgr_free()
829 put_device(&lgr->smcd->dev); in smc_lgr_free()
830 if (!atomic_dec_return(&lgr->smcd->lgr_cnt)) in smc_lgr_free()
831 wake_up(&lgr->smcd->lgrs_deleted); in smc_lgr_free()
846 list_for_each_entry(buf_desc, &lgr->rmbs[i], list) { in smcd_unregister_all_dmbs()
847 buf_desc->len += sizeof(struct smcd_cdc_msg); in smcd_unregister_all_dmbs()
848 smc_ism_unregister_dmb(lgr->smcd, buf_desc); in smcd_unregister_all_dmbs()
855 smc->sk.sk_write_space(&smc->sk); in smc_sk_wake_ups()
856 smc->sk.sk_data_ready(&smc->sk); in smc_sk_wake_ups()
857 smc->sk.sk_state_change(&smc->sk); in smc_sk_wake_ups()
865 if (conn->lgr->is_smcd && conn->lgr->peer_shutdown) in smc_conn_kill()
866 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; in smc_conn_kill()
869 conn->killed = 1; in smc_conn_kill()
870 smc->sk.sk_err = ECONNABORTED; in smc_conn_kill()
872 if (conn->lgr->is_smcd) { in smc_conn_kill()
875 tasklet_kill(&conn->rx_tsklet); in smc_conn_kill()
877 tasklet_unlock_wait(&conn->rx_tsklet); in smc_conn_kill()
887 if (lgr->is_smcd) { in smc_lgr_cleanup()
891 u32 rsn = lgr->llc_termination_rsn; in smc_lgr_cleanup()
910 if (lgr->terminating) in __smc_lgr_terminate()
912 /* cancel free_work sync, will terminate when lgr->freeing is set */ in __smc_lgr_terminate()
913 cancel_delayed_work_sync(&lgr->free_work); in __smc_lgr_terminate()
914 lgr->terminating = 1; in __smc_lgr_terminate()
917 read_lock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
918 node = rb_first(&lgr->conns_all); in __smc_lgr_terminate()
920 read_unlock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
923 sock_hold(&smc->sk); /* sock_put below */ in __smc_lgr_terminate()
924 lock_sock(&smc->sk); in __smc_lgr_terminate()
926 release_sock(&smc->sk); in __smc_lgr_terminate()
927 sock_put(&smc->sk); /* sock_hold above */ in __smc_lgr_terminate()
928 read_lock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
929 node = rb_first(&lgr->conns_all); in __smc_lgr_terminate()
931 read_unlock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
943 if (list_empty(&lgr->list) || lgr->terminating || lgr->freeing) { in smc_lgr_terminate_sched()
947 list_del_init(&lgr->list); in smc_lgr_terminate_sched()
948 lgr->freeing = 1; in smc_lgr_terminate_sched()
950 schedule_work(&lgr->terminate_work); in smc_lgr_terminate_sched()
960 spin_lock_bh(&dev->lgr_lock); in smc_smcd_terminate()
961 list_for_each_entry_safe(lgr, l, &dev->lgr_list, list) { in smc_smcd_terminate()
962 if ((!peer_gid || lgr->peer_gid == peer_gid) && in smc_smcd_terminate()
963 (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) { in smc_smcd_terminate()
965 lgr->peer_shutdown = 1; in smc_smcd_terminate()
966 list_move(&lgr->list, &lgr_free_list); in smc_smcd_terminate()
967 lgr->freeing = 1; in smc_smcd_terminate()
970 spin_unlock_bh(&dev->lgr_lock); in smc_smcd_terminate()
974 list_del_init(&lgr->list); in smc_smcd_terminate()
975 schedule_work(&lgr->terminate_work); in smc_smcd_terminate()
985 spin_lock_bh(&smcd->lgr_lock); in smc_smcd_terminate_all()
986 list_splice_init(&smcd->lgr_list, &lgr_free_list); in smc_smcd_terminate_all()
988 lgr->freeing = 1; in smc_smcd_terminate_all()
989 spin_unlock_bh(&smcd->lgr_lock); in smc_smcd_terminate_all()
992 list_del_init(&lgr->list); in smc_smcd_terminate_all()
996 if (atomic_read(&smcd->lgr_cnt)) in smc_smcd_terminate_all()
997 wait_event(smcd->lgrs_deleted, !atomic_read(&smcd->lgr_cnt)); in smc_smcd_terminate_all()
1014 lgr->freeing = 1; in smc_smcr_terminate_all()
1018 if (lgr->lnk[i].smcibdev == smcibdev) in smc_smcr_terminate_all()
1019 smcr_link_down_cond_sched(&lgr->lnk[i]); in smc_smcr_terminate_all()
1026 list_del_init(&lgr->list); in smc_smcr_terminate_all()
1032 if (atomic_read(&smcibdev->lnk_cnt)) in smc_smcr_terminate_all()
1033 wait_event(smcibdev->lnks_deleted, in smc_smcr_terminate_all()
1034 !atomic_read(&smcibdev->lnk_cnt)); in smc_smcr_terminate_all()
1048 if (smc_link_usable(&lgr->lnk[i])) in smcr_lgr_set_type()
1049 lgr->lnk[i].link_is_asym = false; in smcr_lgr_set_type()
1050 if (lgr->type == new_type) in smcr_lgr_set_type()
1052 lgr->type = new_type; in smcr_lgr_set_type()
1054 switch (lgr->type) { in smcr_lgr_set_type()
1071 pr_warn_ratelimited("smc: SMC-R lg %*phN state changed: " in smcr_lgr_set_type()
1072 "%s, pnetid %.16s\n", SMC_LGR_ID_SIZE, &lgr->id, in smcr_lgr_set_type()
1073 lgr_type, lgr->pnet_id); in smcr_lgr_set_type()
1081 lgr->lnk[asym_lnk_idx].link_is_asym = true; in smcr_lgr_set_type_asym()
1093 sock_put(&smc->sk); /* sock_hold done by schedulers of abort_work */ in smc_conn_abort_work()
1103 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, in smcr_port_add()
1105 lgr->type == SMC_LGR_SYMMETRIC || in smcr_port_add()
1106 lgr->type == SMC_LGR_ASYMMETRIC_PEER) in smcr_port_add()
1116 /* link is down - switch connections to alternate link,
1117 * must be called under lgr->llc_conf_mutex lock
1121 struct smc_link_group *lgr = lnk->lgr; in smcr_link_down()
1125 if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list)) in smcr_link_down()
1135 del_link_id = lnk->link_id; in smcr_link_down()
1137 if (lgr->role == SMC_SERV) { in smcr_link_down()
1141 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { in smcr_link_down()
1143 mutex_unlock(&lgr->llc_conf_mutex); in smcr_link_down()
1144 wait_event_timeout(lgr->llc_flow_waiter, in smcr_link_down()
1145 (list_empty(&lgr->list) || in smcr_link_down()
1146 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE), in smcr_link_down()
1148 mutex_lock(&lgr->llc_conf_mutex); in smcr_link_down()
1150 if (!list_empty(&lgr->list)) { in smcr_link_down()
1156 wake_up(&lgr->llc_flow_waiter); /* wake up next waiter */ in smcr_link_down()
1160 /* must be called under lgr->llc_conf_mutex lock */
1163 if (smc_link_downing(&lnk->state)) in smcr_link_down_cond()
1167 /* will get the lgr->llc_conf_mutex lock */
1170 if (smc_link_downing(&lnk->state)) in smcr_link_down_cond_sched()
1171 schedule_work(&lnk->link_down_wrk); in smcr_link_down_cond_sched()
1180 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, in smcr_port_err()
1183 if (list_empty(&lgr->list)) in smcr_port_err()
1186 struct smc_link *lnk = &lgr->lnk[i]; in smcr_port_err()
1189 lnk->smcibdev == smcibdev && lnk->ibport == ibport) in smcr_port_err()
1199 struct smc_link_group *lgr = link->lgr; in smc_link_down_work()
1201 if (list_empty(&lgr->list)) in smc_link_down_work()
1203 wake_up_all(&lgr->llc_msg_waiter); in smc_link_down_work()
1204 mutex_lock(&lgr->llc_conf_mutex); in smc_link_down_work()
1206 mutex_unlock(&lgr->llc_conf_mutex); in smc_link_down_work()
1214 struct dst_entry *dst = sk_dst_get(clcsock->sk); in smc_vlan_by_tcpsk()
1218 ini->vlan_id = 0; in smc_vlan_by_tcpsk()
1220 rc = -ENOTCONN; in smc_vlan_by_tcpsk()
1223 if (!dst->dev) { in smc_vlan_by_tcpsk()
1224 rc = -ENODEV; in smc_vlan_by_tcpsk()
1228 ndev = dst->dev; in smc_vlan_by_tcpsk()
1230 ini->vlan_id = vlan_dev_vlan_id(ndev); in smc_vlan_by_tcpsk()
1235 nest_lvl = ndev->lower_level; in smc_vlan_by_tcpsk()
1237 struct list_head *lower = &ndev->adj_list.lower; in smc_vlan_by_tcpsk()
1241 lower = lower->next; in smc_vlan_by_tcpsk()
1244 ini->vlan_id = vlan_dev_vlan_id(ndev); in smc_vlan_by_tcpsk()
1258 enum smc_lgr_role role, u32 clcqpn) in smcr_lgr_match() argument
1262 if (memcmp(lgr->peer_systemid, lcl->id_for_peer, SMC_SYSTEMID_LEN) || in smcr_lgr_match()
1263 lgr->role != role) in smcr_lgr_match()
1267 if (!smc_link_active(&lgr->lnk[i])) in smcr_lgr_match()
1269 if ((lgr->role == SMC_SERV || lgr->lnk[i].peer_qpn == clcqpn) && in smcr_lgr_match()
1270 !memcmp(lgr->lnk[i].peer_gid, &lcl->gid, SMC_GID_SIZE) && in smcr_lgr_match()
1271 !memcmp(lgr->lnk[i].peer_mac, lcl->mac, sizeof(lcl->mac))) in smcr_lgr_match()
1280 return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev; in smcd_lgr_match()
1286 struct smc_connection *conn = &smc->conn; in smc_conn_create()
1289 enum smc_lgr_role role; in smc_conn_create() local
1293 lgr_list = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_list : in smc_conn_create()
1295 lgr_lock = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_lock : in smc_conn_create()
1297 ini->first_contact_local = 1; in smc_conn_create()
1298 role = smc->listen_smc ? SMC_SERV : SMC_CLNT; in smc_conn_create()
1299 if (role == SMC_CLNT && ini->first_contact_peer) in smc_conn_create()
1306 write_lock_bh(&lgr->conns_lock); in smc_conn_create()
1307 if ((ini->is_smcd ? in smc_conn_create()
1308 smcd_lgr_match(lgr, ini->ism_dev[ini->ism_selected], in smc_conn_create()
1309 ini->ism_peer_gid[ini->ism_selected]) : in smc_conn_create()
1310 smcr_lgr_match(lgr, ini->ib_lcl, role, ini->ib_clcqpn)) && in smc_conn_create()
1311 !lgr->sync_err && in smc_conn_create()
1312 (ini->smcd_version == SMC_V2 || in smc_conn_create()
1313 lgr->vlan_id == ini->vlan_id) && in smc_conn_create()
1314 (role == SMC_CLNT || ini->is_smcd || in smc_conn_create()
1315 lgr->conns_num < SMC_RMBS_PER_LGR_MAX)) { in smc_conn_create()
1317 ini->first_contact_local = 0; in smc_conn_create()
1318 conn->lgr = lgr; in smc_conn_create()
1320 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
1321 if (!rc && delayed_work_pending(&lgr->free_work)) in smc_conn_create()
1322 cancel_delayed_work(&lgr->free_work); in smc_conn_create()
1325 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
1331 if (role == SMC_CLNT && !ini->first_contact_peer && in smc_conn_create()
1332 ini->first_contact_local) { in smc_conn_create()
1341 if (ini->first_contact_local) { in smc_conn_create()
1345 lgr = conn->lgr; in smc_conn_create()
1346 write_lock_bh(&lgr->conns_lock); in smc_conn_create()
1348 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
1352 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE; in smc_conn_create()
1353 conn->local_tx_ctrl.len = SMC_WR_TX_SIZE; in smc_conn_create()
1354 conn->urg_state = SMC_URG_READ; in smc_conn_create()
1355 INIT_WORK(&smc->conn.abort_work, smc_conn_abort_work); in smc_conn_create()
1356 if (ini->is_smcd) { in smc_conn_create()
1357 conn->rx_off = sizeof(struct smcd_cdc_msg); in smc_conn_create()
1360 conn->rx_off = 0; in smc_conn_create()
1363 spin_lock_init(&conn->acurs_lock); in smc_conn_create()
1370 /* convert the RMB size into the compressed notation - minimum 16K.
1371 * In contrast to plain ilog2, this rounds towards the next power of 2,
1381 size = (size - 1) >> 14; in smc_compress_bufsize()
1384 compressed = SMC_RMBE_SIZES - 1; in smc_compress_bufsize()
1397 /* try to reuse a sndbuf or rmb description slot for a certain
1408 if (cmpxchg(&buf_slot->used, 0, 1) == 0) { in smc_buf_get_slot()
1432 if (buf_desc->is_map_ib[lnk->link_idx]) in smcr_buf_map_link()
1435 rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], 1, GFP_KERNEL); in smcr_buf_map_link()
1438 sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl, in smcr_buf_map_link()
1439 buf_desc->cpu_addr, buf_desc->len); in smcr_buf_map_link()
1446 rc = -EAGAIN; in smcr_buf_map_link()
1452 rc = smc_ib_get_memory_region(lnk->roce_pd, in smcr_buf_map_link()
1455 buf_desc, lnk->link_idx); in smcr_buf_map_link()
1460 buf_desc->is_map_ib[lnk->link_idx] = true; in smcr_buf_map_link()
1467 sg_free_table(&buf_desc->sgt[lnk->link_idx]); in smcr_buf_map_link()
1472 * must be called under lgr->llc_conf_mutex lock
1476 if (list_empty(&link->lgr->list)) in smcr_link_reg_rmb()
1477 return -ENOLINK; in smcr_link_reg_rmb()
1478 if (!rmb_desc->is_reg_mr[link->link_idx]) { in smcr_link_reg_rmb()
1480 if (smc_wr_reg_send(link, rmb_desc->mr_rx[link->link_idx])) { in smcr_link_reg_rmb()
1481 rmb_desc->is_reg_err = true; in smcr_link_reg_rmb()
1482 return -EFAULT; in smcr_link_reg_rmb()
1484 rmb_desc->is_reg_mr[link->link_idx] = true; in smcr_link_reg_rmb()
1497 if (!buf_desc->used) in _smcr_buf_map_lgr()
1511 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_map_lgr()
1515 rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock, in smcr_buf_map_lgr()
1516 &lgr->rmbs[i], true); in smcr_buf_map_lgr()
1519 rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock, in smcr_buf_map_lgr()
1520 &lgr->sndbufs[i], false); in smcr_buf_map_lgr()
1528 * must be called under lgr->llc_conf_mutex lock
1532 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_reg_lgr()
1536 mutex_lock(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
1538 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) { in smcr_buf_reg_lgr()
1539 if (!buf_desc->used) in smcr_buf_reg_lgr()
1547 mutex_unlock(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
1556 /* try to alloc a new buffer */ in smcr_new_buf_create()
1559 return ERR_PTR(-ENOMEM); in smcr_new_buf_create()
1561 buf_desc->order = get_order(bufsize); in smcr_new_buf_create()
1562 buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN | in smcr_new_buf_create()
1565 buf_desc->order); in smcr_new_buf_create()
1566 if (!buf_desc->pages) { in smcr_new_buf_create()
1568 return ERR_PTR(-EAGAIN); in smcr_new_buf_create()
1570 buf_desc->cpu_addr = (void *)page_address(buf_desc->pages); in smcr_new_buf_create()
1571 buf_desc->len = bufsize; in smcr_new_buf_create()
1584 mutex_lock(&lgr->llc_conf_mutex); in smcr_buf_map_usable_links()
1586 struct smc_link *lnk = &lgr->lnk[i]; in smcr_buf_map_usable_links()
1591 rc = -ENOMEM; in smcr_buf_map_usable_links()
1596 mutex_unlock(&lgr->llc_conf_mutex); in smcr_buf_map_usable_links()
1600 #define SMCD_DMBE_SIZES 6 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */
1609 return ERR_PTR(-EAGAIN); in smcd_new_buf_create()
1611 /* try to alloc a new DMB */ in smcd_new_buf_create()
1614 return ERR_PTR(-ENOMEM); in smcd_new_buf_create()
1619 if (rc == -ENOMEM) in smcd_new_buf_create()
1620 return ERR_PTR(-EAGAIN); in smcd_new_buf_create()
1621 if (rc == -ENOSPC) in smcd_new_buf_create()
1622 return ERR_PTR(-ENOSPC); in smcd_new_buf_create()
1623 return ERR_PTR(-EIO); in smcd_new_buf_create()
1625 buf_desc->pages = virt_to_page(buf_desc->cpu_addr); in smcd_new_buf_create()
1627 buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg); in smcd_new_buf_create()
1629 buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL | in smcd_new_buf_create()
1632 if (!buf_desc->cpu_addr) { in smcd_new_buf_create()
1634 return ERR_PTR(-EAGAIN); in smcd_new_buf_create()
1636 buf_desc->len = bufsize; in smcd_new_buf_create()
1643 struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM); in __smc_buf_create()
1644 struct smc_connection *conn = &smc->conn; in __smc_buf_create()
1645 struct smc_link_group *lgr = conn->lgr; in __smc_buf_create()
1653 sk_buf_size = smc->sk.sk_rcvbuf / 2; in __smc_buf_create()
1656 sk_buf_size = smc->sk.sk_sndbuf / 2; in __smc_buf_create()
1659 bufsize_short >= 0; bufsize_short--) { in __smc_buf_create()
1662 lock = &lgr->rmbs_lock; in __smc_buf_create()
1663 buf_list = &lgr->rmbs[bufsize_short]; in __smc_buf_create()
1665 lock = &lgr->sndbufs_lock; in __smc_buf_create()
1666 buf_list = &lgr->sndbufs[bufsize_short]; in __smc_buf_create()
1675 memset(buf_desc->cpu_addr, 0, bufsize); in __smc_buf_create()
1684 if (PTR_ERR(buf_desc) == -ENOMEM) in __smc_buf_create()
1689 buf_desc->used = 1; in __smc_buf_create()
1691 list_add(&buf_desc->list, buf_list); in __smc_buf_create()
1702 return -ENOMEM; in __smc_buf_create()
1707 conn->rmb_desc = buf_desc; in __smc_buf_create()
1708 conn->rmbe_size_short = bufsize_short; in __smc_buf_create()
1709 smc->sk.sk_rcvbuf = bufsize * 2; in __smc_buf_create()
1710 atomic_set(&conn->bytes_to_rcv, 0); in __smc_buf_create()
1711 conn->rmbe_update_limit = in __smc_buf_create()
1712 smc_rmb_wnd_update_limit(buf_desc->len); in __smc_buf_create()
1716 conn->sndbuf_desc = buf_desc; in __smc_buf_create()
1717 smc->sk.sk_sndbuf = bufsize * 2; in __smc_buf_create()
1718 atomic_set(&conn->sndbuf_space, bufsize); in __smc_buf_create()
1725 if (!conn->lgr || conn->lgr->is_smcd || !smc_link_active(conn->lnk)) in smc_sndbuf_sync_sg_for_cpu()
1727 smc_ib_sync_sg_for_cpu(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE); in smc_sndbuf_sync_sg_for_cpu()
1732 if (!conn->lgr || conn->lgr->is_smcd || !smc_link_active(conn->lnk)) in smc_sndbuf_sync_sg_for_device()
1734 smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE); in smc_sndbuf_sync_sg_for_device()
1741 if (!conn->lgr || conn->lgr->is_smcd) in smc_rmb_sync_sg_for_cpu()
1744 if (!smc_link_active(&conn->lgr->lnk[i])) in smc_rmb_sync_sg_for_cpu()
1746 smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc, in smc_rmb_sync_sg_for_cpu()
1755 if (!conn->lgr || conn->lgr->is_smcd) in smc_rmb_sync_sg_for_device()
1758 if (!smc_link_active(&conn->lgr->lnk[i])) in smc_rmb_sync_sg_for_device()
1760 smc_ib_sync_sg_for_device(&conn->lgr->lnk[i], conn->rmb_desc, in smc_rmb_sync_sg_for_device()
1767 * (even though the SMC protocol allows more than one RMB-element per RMB,
1768 * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
1782 mutex_lock(&smc->conn.lgr->sndbufs_lock); in smc_buf_create()
1783 list_del(&smc->conn.sndbuf_desc->list); in smc_buf_create()
1784 mutex_unlock(&smc->conn.lgr->sndbufs_lock); in smc_buf_create()
1785 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc); in smc_buf_create()
1786 smc->conn.sndbuf_desc = NULL; in smc_buf_create()
1795 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { in smc_rmb_reserve_rtoken_idx()
1796 if (!test_and_set_bit(i, lgr->rtokens_used_mask)) in smc_rmb_reserve_rtoken_idx()
1799 return -ENOSPC; in smc_rmb_reserve_rtoken_idx()
1808 if (test_bit(i, lgr->rtokens_used_mask) && in smc_rtoken_find_by_link()
1809 lgr->rtokens[i][lnk_idx].rkey == rkey) in smc_rtoken_find_by_link()
1812 return -ENOENT; in smc_rtoken_find_by_link()
1822 if (rtok_idx == -ENOENT) in smc_rtoken_set()
1824 lgr->rtokens[rtok_idx][link_idx_new].rkey = ntohl(nw_rkey); in smc_rtoken_set()
1825 lgr->rtokens[rtok_idx][link_idx_new].dma_addr = be64_to_cpu(nw_vaddr); in smc_rtoken_set()
1838 if (lgr->lnk[link_idx].link_id == link_id) { in smc_rtoken_set2()
1845 lgr->rtokens[rtok_idx][link_idx].rkey = rkey; in smc_rtoken_set2()
1846 lgr->rtokens[rtok_idx][link_idx].dma_addr = dma_addr; in smc_rtoken_set2()
1858 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_add()
1859 lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr && in smc_rtoken_add()
1860 test_bit(i, lgr->rtokens_used_mask)) { in smc_rtoken_add()
1868 lgr->rtokens[i][lnk->link_idx].rkey = rkey; in smc_rtoken_add()
1869 lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr; in smc_rtoken_add()
1881 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_delete()
1882 test_bit(i, lgr->rtokens_used_mask)) { in smc_rtoken_delete()
1884 lgr->rtokens[i][j].rkey = 0; in smc_rtoken_delete()
1885 lgr->rtokens[i][j].dma_addr = 0; in smc_rtoken_delete()
1887 clear_bit(i, lgr->rtokens_used_mask); in smc_rtoken_delete()
1891 return -ENOENT; in smc_rtoken_delete()
1899 conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr, in smc_rmb_rtoken_handling()
1900 clc->r0.rmb_rkey); in smc_rmb_rtoken_handling()
1901 if (conn->rtoken_idx < 0) in smc_rmb_rtoken_handling()
1902 return conn->rtoken_idx; in smc_rmb_rtoken_handling()
1916 set_bit(i, smcibdev->ports_going_away); in smc_core_going_away()
1922 smcd->going_away = 1; in smc_core_going_away()