Lines Matching full:lnk
71 static void smc_ibdev_cnt_inc(struct smc_link *lnk) in smc_ibdev_cnt_inc() argument
73 atomic_inc(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]); in smc_ibdev_cnt_inc()
76 static void smc_ibdev_cnt_dec(struct smc_link *lnk) in smc_ibdev_cnt_dec() argument
78 atomic_dec(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]); in smc_ibdev_cnt_dec()
130 conn->lnk = NULL; /* reset conn->lnk first */ in smcr_lgr_conn_assign_link()
132 struct smc_link *lnk = &conn->lgr->lnk[i]; in smcr_lgr_conn_assign_link() local
134 if (lnk->state != expected || lnk->link_is_asym) in smcr_lgr_conn_assign_link()
137 conn->lnk = lnk; /* temporary, SMC server assigns link*/ in smcr_lgr_conn_assign_link()
144 lnk2 = &conn->lgr->lnk[j]; in smcr_lgr_conn_assign_link()
147 conn->lnk = lnk2; in smcr_lgr_conn_assign_link()
152 if (!conn->lnk) in smcr_lgr_conn_assign_link()
153 conn->lnk = lnk; in smcr_lgr_conn_assign_link()
156 if (!conn->lnk) in smcr_lgr_conn_assign_link()
158 atomic_inc(&conn->lnk->conn_cnt); in smcr_lgr_conn_assign_link()
202 if (conn->lnk) in __smc_lgr_unregister_conn()
203 atomic_dec(&conn->lnk->conn_cnt); in __smc_lgr_unregister_conn()
465 if (!smc_link_usable(&lgr->lnk[i])) in smc_nl_handle_lgr()
467 if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb)) in smc_nl_handle_lgr()
666 struct smc_link *lnk = &lgr->lnk[i]; in smcr_lgr_link_deactivate_all() local
668 if (smc_link_sendable(lnk)) in smcr_lgr_link_deactivate_all()
669 lnk->state = SMC_LNK_INACTIVE; in smcr_lgr_link_deactivate_all()
733 if (smc_link_usable(&lgr->lnk[i]) && in smcr_next_link_id()
734 lgr->lnk[i].link_id == link_id) in smcr_next_link_id()
751 int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk, in smcr_link_init() argument
759 lnk->smcibdev = ini->smcrv2.ib_dev_v2; in smcr_link_init()
760 lnk->ibport = ini->smcrv2.ib_port_v2; in smcr_link_init()
762 lnk->smcibdev = ini->ib_dev; in smcr_link_init()
763 lnk->ibport = ini->ib_port; in smcr_link_init()
765 get_device(&lnk->smcibdev->ibdev->dev); in smcr_link_init()
766 atomic_inc(&lnk->smcibdev->lnk_cnt); in smcr_link_init()
767 refcount_set(&lnk->refcnt, 1); /* link refcnt is set to 1 */ in smcr_link_init()
768 lnk->clearing = 0; in smcr_link_init()
769 lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu; in smcr_link_init()
770 lnk->link_id = smcr_next_link_id(lgr); in smcr_link_init()
771 lnk->lgr = lgr; in smcr_link_init()
773 lnk->link_idx = link_idx; in smcr_link_init()
774 lnk->wr_rx_id_compl = 0; in smcr_link_init()
775 smc_ibdev_cnt_inc(lnk); in smcr_link_init()
776 smcr_copy_dev_info_to_link(lnk); in smcr_link_init()
777 atomic_set(&lnk->conn_cnt, 0); in smcr_link_init()
778 smc_llc_link_set_uid(lnk); in smcr_link_init()
779 INIT_WORK(&lnk->link_down_wrk, smc_link_down_work); in smcr_link_init()
780 if (!lnk->smcibdev->initialized) { in smcr_link_init()
781 rc = (int)smc_ib_setup_per_ibdev(lnk->smcibdev); in smcr_link_init()
786 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + in smcr_link_init()
788 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport, in smcr_link_init()
789 ini->vlan_id, lnk->gid, &lnk->sgid_index, in smcr_link_init()
794 rc = smc_llc_link_init(lnk); in smcr_link_init()
797 rc = smc_wr_alloc_link_mem(lnk); in smcr_link_init()
800 rc = smc_ib_create_protection_domain(lnk); in smcr_link_init()
803 rc = smc_ib_create_queue_pair(lnk); in smcr_link_init()
806 rc = smc_wr_create_link(lnk); in smcr_link_init()
809 lnk->state = SMC_LNK_ACTIVATING; in smcr_link_init()
813 smc_ib_destroy_queue_pair(lnk); in smcr_link_init()
815 smc_ib_dealloc_protection_domain(lnk); in smcr_link_init()
817 smc_wr_free_link_mem(lnk); in smcr_link_init()
819 smc_llc_link_clear(lnk, false); in smcr_link_init()
821 smc_ibdev_cnt_dec(lnk); in smcr_link_init()
822 put_device(&lnk->smcibdev->ibdev->dev); in smcr_link_init()
823 smcibdev = lnk->smcibdev; in smcr_link_init()
824 memset(lnk, 0, sizeof(struct smc_link)); in smcr_link_init()
825 lnk->state = SMC_LNK_UNUSED; in smcr_link_init()
838 struct smc_link *lnk; in smc_lgr_create() local
928 lnk = &lgr->lnk[link_idx]; in smc_lgr_create()
929 rc = smcr_link_init(lgr, lnk, link_idx, ini); in smc_lgr_create()
934 lgr->net = smc_ib_net(lnk->smcibdev); in smc_lgr_create()
1021 smc_wr_tx_put_slot(conn->lnk, in smc_switch_cursor()
1030 atomic_dec(&conn->lnk->conn_cnt); in smc_switch_link_and_count()
1032 smcr_link_put(conn->lnk); in smc_switch_link_and_count()
1033 conn->lnk = to_lnk; in smc_switch_link_and_count()
1034 atomic_inc(&conn->lnk->conn_cnt); in smc_switch_link_and_count()
1036 smcr_link_hold(conn->lnk); in smc_switch_link_and_count()
1054 if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx) in smc_switch_conns()
1056 if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev && in smc_switch_conns()
1057 from_lnk->ibport == lgr->lnk[i].ibport) { in smc_switch_conns()
1060 to_lnk = &lgr->lnk[i]; in smc_switch_conns()
1071 if (conn->lnk != from_lnk) in smc_switch_conns()
1074 /* conn->lnk not yet set in SMC_INIT state */ in smc_switch_conns()
1210 smcr_link_put(conn->lnk); /* link_hold in smc_conn_create() */ in smc_conn_free()
1216 struct smc_link *lnk) in smcr_buf_unmap_link() argument
1219 buf_desc->is_reg_mr[lnk->link_idx] = false; in smcr_buf_unmap_link()
1220 if (!buf_desc->is_map_ib[lnk->link_idx]) in smcr_buf_unmap_link()
1224 buf_desc->mr[lnk->link_idx]) { in smcr_buf_unmap_link()
1225 smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]); in smcr_buf_unmap_link()
1226 buf_desc->mr[lnk->link_idx] = NULL; in smcr_buf_unmap_link()
1229 smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE); in smcr_buf_unmap_link()
1231 smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE); in smcr_buf_unmap_link()
1233 sg_free_table(&buf_desc->sgt[lnk->link_idx]); in smcr_buf_unmap_link()
1234 buf_desc->is_map_ib[lnk->link_idx] = false; in smcr_buf_unmap_link()
1238 static void smcr_buf_unmap_lgr(struct smc_link *lnk) in smcr_buf_unmap_lgr() argument
1240 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_unmap_lgr()
1247 smcr_buf_unmap_link(buf_desc, true, lnk); in smcr_buf_unmap_lgr()
1253 smcr_buf_unmap_link(buf_desc, false, lnk); in smcr_buf_unmap_lgr()
1258 static void smcr_rtoken_clear_link(struct smc_link *lnk) in smcr_rtoken_clear_link() argument
1260 struct smc_link_group *lgr = lnk->lgr; in smcr_rtoken_clear_link()
1264 lgr->rtokens[i][lnk->link_idx].rkey = 0; in smcr_rtoken_clear_link()
1265 lgr->rtokens[i][lnk->link_idx].dma_addr = 0; in smcr_rtoken_clear_link()
1269 static void __smcr_link_clear(struct smc_link *lnk) in __smcr_link_clear() argument
1271 struct smc_link_group *lgr = lnk->lgr; in __smcr_link_clear()
1274 smc_wr_free_link_mem(lnk); in __smcr_link_clear()
1275 smc_ibdev_cnt_dec(lnk); in __smcr_link_clear()
1276 put_device(&lnk->smcibdev->ibdev->dev); in __smcr_link_clear()
1277 smcibdev = lnk->smcibdev; in __smcr_link_clear()
1278 memset(lnk, 0, sizeof(struct smc_link)); in __smcr_link_clear()
1279 lnk->state = SMC_LNK_UNUSED; in __smcr_link_clear()
1286 void smcr_link_clear(struct smc_link *lnk, bool log) in smcr_link_clear() argument
1288 if (!lnk->lgr || lnk->clearing || in smcr_link_clear()
1289 lnk->state == SMC_LNK_UNUSED) in smcr_link_clear()
1291 lnk->clearing = 1; in smcr_link_clear()
1292 lnk->peer_qpn = 0; in smcr_link_clear()
1293 smc_llc_link_clear(lnk, log); in smcr_link_clear()
1294 smcr_buf_unmap_lgr(lnk); in smcr_link_clear()
1295 smcr_rtoken_clear_link(lnk); in smcr_link_clear()
1296 smc_ib_modify_qp_error(lnk); in smcr_link_clear()
1297 smc_wr_free_link(lnk); in smcr_link_clear()
1298 smc_ib_destroy_queue_pair(lnk); in smcr_link_clear()
1299 smc_ib_dealloc_protection_domain(lnk); in smcr_link_clear()
1300 smcr_link_put(lnk); /* theoretically last link_put */ in smcr_link_clear()
1303 void smcr_link_hold(struct smc_link *lnk) in smcr_link_hold() argument
1305 refcount_inc(&lnk->refcnt); in smcr_link_hold()
1308 void smcr_link_put(struct smc_link *lnk) in smcr_link_put() argument
1310 if (refcount_dec_and_test(&lnk->refcnt)) in smcr_link_put()
1311 __smcr_link_clear(lnk); in smcr_link_put()
1320 smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]); in smcr_buf_free()
1401 if (lgr->lnk[i].state != SMC_LNK_UNUSED) in smc_lgr_free()
1402 smcr_link_clear(&lgr->lnk[i], false); in smc_lgr_free()
1595 if (lgr->lnk[i].smcibdev == smcibdev) in smc_smcr_terminate_all()
1596 smcr_link_down_cond_sched(&lgr->lnk[i]); in smc_smcr_terminate_all()
1625 if (smc_link_usable(&lgr->lnk[i])) in smcr_lgr_set_type()
1626 lgr->lnk[i].link_is_asym = false; in smcr_lgr_set_type()
1658 lgr->lnk[asym_lnk_idx].link_is_asym = true; in smcr_lgr_set_type_asym()
1704 static void smcr_link_down(struct smc_link *lnk) in smcr_link_down() argument
1706 struct smc_link_group *lgr = lnk->lgr; in smcr_link_down()
1710 if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list)) in smcr_link_down()
1713 to_lnk = smc_switch_conns(lgr, lnk, true); in smcr_link_down()
1715 smcr_link_clear(lnk, true); in smcr_link_down()
1719 del_link_id = lnk->link_id; in smcr_link_down()
1738 smcr_link_clear(lnk, true); in smcr_link_down()
1745 void smcr_link_down_cond(struct smc_link *lnk) in smcr_link_down_cond() argument
1747 if (smc_link_downing(&lnk->state)) { in smcr_link_down_cond()
1748 trace_smcr_link_down(lnk, __builtin_return_address(0)); in smcr_link_down_cond()
1749 smcr_link_down(lnk); in smcr_link_down_cond()
1754 void smcr_link_down_cond_sched(struct smc_link *lnk) in smcr_link_down_cond_sched() argument
1756 if (smc_link_downing(&lnk->state)) { in smcr_link_down_cond_sched()
1757 trace_smcr_link_down(lnk, __builtin_return_address(0)); in smcr_link_down_cond_sched()
1758 schedule_work(&lnk->link_down_wrk); in smcr_link_down_cond_sched()
1774 struct smc_link *lnk = &lgr->lnk[i]; in smcr_port_err() local
1776 if (smc_link_usable(lnk) && in smcr_port_err()
1777 lnk->smcibdev == smcibdev && lnk->ibport == ibport) in smcr_port_err()
1778 smcr_link_down_cond_sched(lnk); in smcr_port_err()
1852 struct smc_link *lnk; in smcr_lgr_match() local
1860 lnk = &lgr->lnk[i]; in smcr_lgr_match()
1862 if (!smc_link_active(lnk)) in smcr_lgr_match()
1865 if (!rdma_dev_access_netns(lnk->smcibdev->ibdev, net)) in smcr_lgr_match()
1867 if ((lgr->role == SMC_SERV || lnk->peer_qpn == clcqpn) && in smcr_lgr_match()
1868 !memcmp(lnk->peer_gid, peer_gid, SMC_GID_SIZE) && in smcr_lgr_match()
1870 !memcmp(lnk->peer_mac, peer_mac_v1, ETH_ALEN))) in smcr_lgr_match()
1969 smcr_link_hold(conn->lnk); /* link_put in smc_conn_free() */ in smc_conn_create()
2057 struct smc_link *lnk) in smcr_buf_map_link() argument
2063 if (buf_desc->is_map_ib[lnk->link_idx]) in smcr_buf_map_link()
2075 rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL); in smcr_buf_map_link()
2081 for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) { in smcr_buf_map_link()
2090 sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl, in smcr_buf_map_link()
2095 rc = smc_ib_buf_map_sg(lnk, buf_desc, in smcr_buf_map_link()
2104 smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx; in smcr_buf_map_link()
2112 rc = smc_ib_get_memory_region(lnk->roce_pd, access_flags, in smcr_buf_map_link()
2113 buf_desc, lnk->link_idx); in smcr_buf_map_link()
2116 smc_ib_sync_sg_for_device(lnk, buf_desc, in smcr_buf_map_link()
2119 buf_desc->is_map_ib[lnk->link_idx] = true; in smcr_buf_map_link()
2123 smc_ib_buf_unmap_sg(lnk, buf_desc, in smcr_buf_map_link()
2126 sg_free_table(&buf_desc->sgt[lnk->link_idx]); in smcr_buf_map_link()
2151 static int _smcr_buf_map_lgr(struct smc_link *lnk, struct rw_semaphore *lock, in _smcr_buf_map_lgr() argument
2161 rc = smcr_buf_map_link(buf_desc, is_rmb, lnk); in _smcr_buf_map_lgr()
2171 int smcr_buf_map_lgr(struct smc_link *lnk) in smcr_buf_map_lgr() argument
2173 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_map_lgr()
2177 rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock, in smcr_buf_map_lgr()
2181 rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock, in smcr_buf_map_lgr()
2192 int smcr_buf_reg_lgr(struct smc_link *lnk) in smcr_buf_reg_lgr() argument
2194 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_reg_lgr()
2204 rc = smcr_link_reg_buf(lnk, buf_desc); in smcr_buf_reg_lgr()
2222 rc = smcr_link_reg_buf(lnk, buf_desc); in smcr_buf_reg_lgr()
2289 struct smc_link *lnk = &lgr->lnk[i]; in smcr_buf_map_usable_links() local
2291 if (!smc_link_usable(lnk)) in smcr_buf_map_usable_links()
2293 if (smcr_buf_map_link(buf_desc, is_rmb, lnk)) { in smcr_buf_map_usable_links()
2435 !smc_link_active(conn->lnk)) in smc_sndbuf_sync_sg_for_device()
2437 smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE); in smc_sndbuf_sync_sg_for_device()
2449 if (!smc_link_active(&conn->lgr->lnk[i])) in smc_rmb_sync_sg_for_cpu()
2451 smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc, in smc_rmb_sync_sg_for_cpu()
2529 if (lgr->lnk[link_idx].link_id == link_id) { in smc_rtoken_set2()
2541 int smc_rtoken_add(struct smc_link *lnk, __be64 nw_vaddr, __be32 nw_rkey) in smc_rtoken_add() argument
2543 struct smc_link_group *lgr = smc_get_lgr(lnk); in smc_rtoken_add()
2549 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_add()
2550 lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr && in smc_rtoken_add()
2559 lgr->rtokens[i][lnk->link_idx].rkey = rkey; in smc_rtoken_add()
2560 lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr; in smc_rtoken_add()
2565 int smc_rtoken_delete(struct smc_link *lnk, __be32 nw_rkey) in smc_rtoken_delete() argument
2567 struct smc_link_group *lgr = smc_get_lgr(lnk); in smc_rtoken_delete()
2572 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_delete()
2587 struct smc_link *lnk, in smc_rmb_rtoken_handling() argument
2590 conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr, in smc_rmb_rtoken_handling()