1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/ib_umem.h>
16 #include <rdma/uverbs_ioctl.h>
17
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21
assemble_qbuf_mtt_for_cmd(struct erdma_mem * mem,u32 * cfg,u64 * addr0,u64 * addr1)22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
23 u64 *addr0, u64 *addr1)
24 {
25 struct erdma_mtt *mtt = mem->mtt;
26
27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) {
28 *addr0 = mtt->buf_dma;
29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
30 ERDMA_MR_MTT_1LEVEL);
31 } else {
32 *addr0 = mtt->buf[0];
33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1));
34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
35 ERDMA_MR_MTT_0LEVEL);
36 }
37 }
38
create_qp_cmd(struct erdma_ucontext * uctx,struct erdma_qp * qp)39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
40 {
41 struct erdma_dev *dev = to_edev(qp->ibqp.device);
42 struct erdma_pd *pd = to_epd(qp->ibqp.pd);
43 struct erdma_cmdq_create_qp_req req;
44 struct erdma_uqp *user_qp;
45 u64 resp0, resp1;
46 int err;
47
48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
49 CMDQ_OPCODE_CREATE_QP);
50
51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
52 ilog2(qp->attrs.sq_size)) |
53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
55 ilog2(qp->attrs.rq_size)) |
56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
57
58 if (qp->ibqp.qp_type == IB_QPT_RC)
59 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
60 ERDMA_QPT_RC);
61 else
62 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
63 ERDMA_QPT_UD);
64
65 if (rdma_is_kernel_res(&qp->ibqp.res)) {
66 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
67
68 req.sq_cqn_mtt_cfg =
69 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
70 pgsz_range) |
71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
72 req.rq_cqn_mtt_cfg =
73 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
74 pgsz_range) |
75 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
76
77 req.sq_mtt_cfg =
78 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
79 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
80 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
81 ERDMA_MR_MTT_0LEVEL);
82 req.rq_mtt_cfg = req.sq_mtt_cfg;
83
84 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
85 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
86 req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
87 req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
88 } else {
89 user_qp = &qp->user_qp;
90 req.sq_cqn_mtt_cfg = FIELD_PREP(
91 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
92 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
93 req.sq_cqn_mtt_cfg |=
94 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
95
96 req.rq_cqn_mtt_cfg = FIELD_PREP(
97 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
98 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
99 req.rq_cqn_mtt_cfg |=
100 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
101
102 req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
103 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
104 user_qp->sq_mem.mtt_nents);
105
106 req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
107 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
108 user_qp->rq_mem.mtt_nents);
109
110 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
111 &req.sq_buf_addr, req.sq_mtt_entry);
112 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
113 &req.rq_buf_addr, req.rq_mtt_entry);
114
115 req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
116 req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
117
118 if (uctx->ext_db.enable) {
119 req.sq_cqn_mtt_cfg |=
120 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
121 req.db_cfg =
122 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
123 uctx->ext_db.sdb_off) |
124 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
125 uctx->ext_db.rdb_off);
126 }
127 }
128
129 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
130 true);
131 if (!err && erdma_device_iwarp(dev))
132 qp->attrs.iwarp.cookie =
133 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
134
135 return err;
136 }
137
regmr_cmd(struct erdma_dev * dev,struct erdma_mr * mr)138 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
139 {
140 struct erdma_pd *pd = to_epd(mr->ibmr.pd);
141 u32 mtt_level = ERDMA_MR_MTT_0LEVEL;
142 struct erdma_cmdq_reg_mr_req req;
143
144 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
145
146 if (mr->type == ERDMA_MR_TYPE_FRMR ||
147 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) {
148 if (mr->mem.mtt->continuous) {
149 req.phy_addr[0] = mr->mem.mtt->buf_dma;
150 mtt_level = ERDMA_MR_MTT_1LEVEL;
151 } else {
152 req.phy_addr[0] = sg_dma_address(mr->mem.mtt->sglist);
153 mtt_level = mr->mem.mtt->level;
154 }
155 } else if (mr->type != ERDMA_MR_TYPE_DMA) {
156 memcpy(req.phy_addr, mr->mem.mtt->buf,
157 MTT_SIZE(mr->mem.page_cnt));
158 }
159
160 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
161 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
162 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
163 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
164 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
165 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
166 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
167 ilog2(mr->mem.page_size)) |
168 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |
169 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
170
171 if (mr->type == ERDMA_MR_TYPE_DMA)
172 goto post_cmd;
173
174 if (mr->type == ERDMA_MR_TYPE_NORMAL) {
175 req.start_va = mr->mem.va;
176 req.size = mr->mem.len;
177 }
178
179 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) {
180 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);
181 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,
182 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);
183 req.size_h = upper_32_bits(mr->mem.len);
184 req.mtt_cnt_h = mr->mem.page_cnt >> 20;
185 }
186
187 post_cmd:
188 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
189 true);
190 }
191
create_cq_cmd(struct erdma_ucontext * uctx,struct erdma_cq * cq)192 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
193 {
194 struct erdma_dev *dev = to_edev(cq->ibcq.device);
195 struct erdma_cmdq_create_cq_req req;
196 struct erdma_mem *mem;
197 u32 page_size;
198
199 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
200 CMDQ_OPCODE_CREATE_CQ);
201
202 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
203 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
204 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
205
206 if (rdma_is_kernel_res(&cq->ibcq.res)) {
207 page_size = SZ_32M;
208 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
209 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
210 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
211 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
212
213 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
214 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
215 ERDMA_MR_MTT_0LEVEL);
216
217 req.first_page_offset = 0;
218 req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
219 } else {
220 mem = &cq->user_cq.qbuf_mem;
221 req.cfg0 |=
222 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
223 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
224 if (mem->mtt_nents == 1) {
225 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
226 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
227 req.cfg1 |=
228 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
229 ERDMA_MR_MTT_0LEVEL);
230 } else {
231 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
232 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
233 req.cfg1 |=
234 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
235 ERDMA_MR_MTT_1LEVEL);
236 }
237 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
238 mem->mtt_nents);
239
240 req.first_page_offset = mem->page_offset;
241 req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
242
243 if (uctx->ext_db.enable) {
244 req.cfg1 |= FIELD_PREP(
245 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
246 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
247 uctx->ext_db.cdb_off);
248 }
249 }
250
251 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
252 true);
253 }
254
erdma_alloc_idx(struct erdma_resource_cb * res_cb)255 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
256 {
257 int idx;
258 unsigned long flags;
259
260 spin_lock_irqsave(&res_cb->lock, flags);
261 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
262 res_cb->next_alloc_idx);
263 if (idx == res_cb->max_cap) {
264 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
265 if (idx == res_cb->max_cap) {
266 res_cb->next_alloc_idx = 1;
267 spin_unlock_irqrestore(&res_cb->lock, flags);
268 return -ENOSPC;
269 }
270 }
271
272 set_bit(idx, res_cb->bitmap);
273 res_cb->next_alloc_idx = idx + 1;
274 spin_unlock_irqrestore(&res_cb->lock, flags);
275
276 return idx;
277 }
278
erdma_free_idx(struct erdma_resource_cb * res_cb,u32 idx)279 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
280 {
281 unsigned long flags;
282 u32 used;
283
284 spin_lock_irqsave(&res_cb->lock, flags);
285 used = __test_and_clear_bit(idx, res_cb->bitmap);
286 spin_unlock_irqrestore(&res_cb->lock, flags);
287 WARN_ON(!used);
288 }
289
290 static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext * uctx,void * address,u32 size,u8 mmap_flag,u64 * mmap_offset)291 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
292 u32 size, u8 mmap_flag, u64 *mmap_offset)
293 {
294 struct erdma_user_mmap_entry *entry =
295 kzalloc(sizeof(*entry), GFP_KERNEL);
296 int ret;
297
298 if (!entry)
299 return NULL;
300
301 entry->address = (u64)address;
302 entry->mmap_flag = mmap_flag;
303
304 size = PAGE_ALIGN(size);
305
306 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
307 size);
308 if (ret) {
309 kfree(entry);
310 return NULL;
311 }
312
313 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
314
315 return &entry->rdma_entry;
316 }
317
erdma_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * unused)318 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
319 struct ib_udata *unused)
320 {
321 struct erdma_dev *dev = to_edev(ibdev);
322
323 memset(attr, 0, sizeof(*attr));
324
325 attr->max_mr_size = dev->attrs.max_mr_size;
326 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
327 attr->vendor_part_id = dev->pdev->device;
328 attr->hw_ver = dev->pdev->revision;
329 attr->max_qp = dev->attrs.max_qp - 1;
330 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
331 attr->max_qp_rd_atom = dev->attrs.max_ord;
332 attr->max_qp_init_rd_atom = dev->attrs.max_ird;
333 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
334 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
335 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
336 ibdev->local_dma_lkey = dev->attrs.local_dma_key;
337 attr->max_send_sge = dev->attrs.max_send_sge;
338 attr->max_recv_sge = dev->attrs.max_recv_sge;
339 attr->max_sge_rd = dev->attrs.max_sge_rd;
340 attr->max_cq = dev->attrs.max_cq - 1;
341 attr->max_cqe = dev->attrs.max_cqe;
342 attr->max_mr = dev->attrs.max_mr;
343 attr->max_pd = dev->attrs.max_pd;
344 attr->max_mw = dev->attrs.max_mw;
345 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
346 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
347
348 if (erdma_device_rocev2(dev)) {
349 attr->max_pkeys = ERDMA_MAX_PKEYS;
350 attr->max_ah = dev->attrs.max_ah;
351 }
352
353 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
354 attr->atomic_cap = IB_ATOMIC_GLOB;
355
356 attr->fw_ver = dev->attrs.fw_version;
357
358 if (dev->netdev)
359 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
360 dev->netdev->dev_addr);
361
362 return 0;
363 }
364
erdma_query_gid(struct ib_device * ibdev,u32 port,int idx,union ib_gid * gid)365 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
366 union ib_gid *gid)
367 {
368 struct erdma_dev *dev = to_edev(ibdev);
369
370 memset(gid, 0, sizeof(*gid));
371 ether_addr_copy(gid->raw, dev->attrs.peer_addr);
372
373 return 0;
374 }
375
erdma_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)376 int erdma_query_port(struct ib_device *ibdev, u32 port,
377 struct ib_port_attr *attr)
378 {
379 struct erdma_dev *dev = to_edev(ibdev);
380 struct net_device *ndev = dev->netdev;
381
382 memset(attr, 0, sizeof(*attr));
383
384 if (erdma_device_iwarp(dev)) {
385 attr->gid_tbl_len = 1;
386 } else {
387 attr->gid_tbl_len = dev->attrs.max_gid;
388 attr->ip_gids = true;
389 attr->pkey_tbl_len = ERDMA_MAX_PKEYS;
390 }
391
392 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
393 attr->max_msg_sz = -1;
394
395 if (!ndev)
396 goto out;
397
398 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
399 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
400 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
401 attr->state = ib_get_curr_port_state(ndev);
402
403 out:
404 if (attr->state == IB_PORT_ACTIVE)
405 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
406 else
407 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
408
409 return 0;
410 }
411
erdma_get_port_immutable(struct ib_device * ibdev,u32 port,struct ib_port_immutable * port_immutable)412 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
413 struct ib_port_immutable *port_immutable)
414 {
415 struct erdma_dev *dev = to_edev(ibdev);
416
417 if (erdma_device_iwarp(dev)) {
418 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
419 port_immutable->gid_tbl_len = 1;
420 } else {
421 port_immutable->core_cap_flags =
422 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
423 port_immutable->max_mad_size = IB_MGMT_MAD_SIZE;
424 port_immutable->gid_tbl_len = dev->attrs.max_gid;
425 port_immutable->pkey_tbl_len = ERDMA_MAX_PKEYS;
426 }
427
428 return 0;
429 }
430
erdma_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)431 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
432 {
433 struct erdma_pd *pd = to_epd(ibpd);
434 struct erdma_dev *dev = to_edev(ibpd->device);
435 int pdn;
436
437 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
438 if (pdn < 0)
439 return pdn;
440
441 pd->pdn = pdn;
442
443 return 0;
444 }
445
erdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)446 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
447 {
448 struct erdma_pd *pd = to_epd(ibpd);
449 struct erdma_dev *dev = to_edev(ibpd->device);
450
451 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
452
453 return 0;
454 }
455
erdma_flush_worker(struct work_struct * work)456 static void erdma_flush_worker(struct work_struct *work)
457 {
458 struct delayed_work *dwork = to_delayed_work(work);
459 struct erdma_qp *qp =
460 container_of(dwork, struct erdma_qp, reflush_dwork);
461 struct erdma_cmdq_reflush_req req;
462
463 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
464 CMDQ_OPCODE_REFLUSH);
465 req.qpn = QP_ID(qp);
466 req.sq_pi = qp->kern_qp.sq_pi;
467 req.rq_pi = qp->kern_qp.rq_pi;
468 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL,
469 true);
470 }
471
erdma_qp_validate_cap(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)472 static int erdma_qp_validate_cap(struct erdma_dev *dev,
473 struct ib_qp_init_attr *attrs)
474 {
475 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
476 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
477 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
478 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
479 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
480 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
481 return -EINVAL;
482 }
483
484 return 0;
485 }
486
erdma_qp_validate_attr(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)487 static int erdma_qp_validate_attr(struct erdma_dev *dev,
488 struct ib_qp_init_attr *attrs)
489 {
490 if (erdma_device_iwarp(dev) && attrs->qp_type != IB_QPT_RC)
491 return -EOPNOTSUPP;
492
493 if (erdma_device_rocev2(dev) && attrs->qp_type != IB_QPT_RC &&
494 attrs->qp_type != IB_QPT_UD && attrs->qp_type != IB_QPT_GSI)
495 return -EOPNOTSUPP;
496
497 if (attrs->srq)
498 return -EOPNOTSUPP;
499
500 if (!attrs->send_cq || !attrs->recv_cq)
501 return -EOPNOTSUPP;
502
503 return 0;
504 }
505
free_kernel_qp(struct erdma_qp * qp)506 static void free_kernel_qp(struct erdma_qp *qp)
507 {
508 struct erdma_dev *dev = qp->dev;
509
510 vfree(qp->kern_qp.swr_tbl);
511 vfree(qp->kern_qp.rwr_tbl);
512
513 if (qp->kern_qp.sq_buf)
514 dma_free_coherent(&dev->pdev->dev,
515 qp->attrs.sq_size << SQEBB_SHIFT,
516 qp->kern_qp.sq_buf,
517 qp->kern_qp.sq_buf_dma_addr);
518
519 if (qp->kern_qp.sq_dbrec)
520 dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
521 qp->kern_qp.sq_dbrec_dma);
522
523 if (qp->kern_qp.rq_buf)
524 dma_free_coherent(&dev->pdev->dev,
525 qp->attrs.rq_size << RQE_SHIFT,
526 qp->kern_qp.rq_buf,
527 qp->kern_qp.rq_buf_dma_addr);
528
529 if (qp->kern_qp.rq_dbrec)
530 dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
531 qp->kern_qp.rq_dbrec_dma);
532 }
533
init_kernel_qp(struct erdma_dev * dev,struct erdma_qp * qp,struct ib_qp_init_attr * attrs)534 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
535 struct ib_qp_init_attr *attrs)
536 {
537 struct erdma_kqp *kqp = &qp->kern_qp;
538 int size;
539
540 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
541 kqp->sig_all = 1;
542
543 kqp->sq_pi = 0;
544 kqp->sq_ci = 0;
545 kqp->rq_pi = 0;
546 kqp->rq_ci = 0;
547 kqp->hw_sq_db =
548 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
549 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
550
551 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
552 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
553 if (!kqp->swr_tbl || !kqp->rwr_tbl)
554 goto err_out;
555
556 size = qp->attrs.sq_size << SQEBB_SHIFT;
557 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
558 &kqp->sq_buf_dma_addr, GFP_KERNEL);
559 if (!kqp->sq_buf)
560 goto err_out;
561
562 kqp->sq_dbrec =
563 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
564 if (!kqp->sq_dbrec)
565 goto err_out;
566
567 size = qp->attrs.rq_size << RQE_SHIFT;
568 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
569 &kqp->rq_buf_dma_addr, GFP_KERNEL);
570 if (!kqp->rq_buf)
571 goto err_out;
572
573 kqp->rq_dbrec =
574 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
575 if (!kqp->rq_dbrec)
576 goto err_out;
577
578 return 0;
579
580 err_out:
581 free_kernel_qp(qp);
582 return -ENOMEM;
583 }
584
erdma_fill_bottom_mtt(struct erdma_dev * dev,struct erdma_mem * mem)585 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
586 {
587 struct erdma_mtt *mtt = mem->mtt;
588 struct ib_block_iter biter;
589 u32 idx = 0;
590
591 while (mtt->low_level)
592 mtt = mtt->low_level;
593
594 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
595 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
596 }
597
erdma_create_cont_mtt(struct erdma_dev * dev,size_t size)598 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
599 size_t size)
600 {
601 struct erdma_mtt *mtt;
602
603 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
604 if (!mtt)
605 return ERR_PTR(-ENOMEM);
606
607 mtt->size = size;
608 mtt->buf = kzalloc(mtt->size, GFP_KERNEL);
609 if (!mtt->buf)
610 goto err_free_mtt;
611
612 mtt->continuous = true;
613 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size,
614 DMA_TO_DEVICE);
615 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma))
616 goto err_free_mtt_buf;
617
618 return mtt;
619
620 err_free_mtt_buf:
621 kfree(mtt->buf);
622
623 err_free_mtt:
624 kfree(mtt);
625
626 return ERR_PTR(-ENOMEM);
627 }
628
erdma_destroy_mtt_buf_sg(struct erdma_dev * dev,struct erdma_mtt * mtt)629 static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev,
630 struct erdma_mtt *mtt)
631 {
632 dma_unmap_sg(&dev->pdev->dev, mtt->sglist,
633 DIV_ROUND_UP(mtt->size, PAGE_SIZE), DMA_TO_DEVICE);
634 vfree(mtt->sglist);
635 }
636
erdma_destroy_scatter_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)637 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,
638 struct erdma_mtt *mtt)
639 {
640 erdma_destroy_mtt_buf_sg(dev, mtt);
641 vfree(mtt->buf);
642 kfree(mtt);
643 }
644
erdma_init_middle_mtt(struct erdma_mtt * mtt,struct erdma_mtt * low_mtt)645 static void erdma_init_middle_mtt(struct erdma_mtt *mtt,
646 struct erdma_mtt *low_mtt)
647 {
648 struct scatterlist *sg;
649 u32 idx = 0, i;
650
651 for_each_sg(low_mtt->sglist, sg, low_mtt->nsg, i)
652 mtt->buf[idx++] = sg_dma_address(sg);
653 }
654
erdma_create_mtt_buf_sg(struct erdma_dev * dev,struct erdma_mtt * mtt)655 static int erdma_create_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt)
656 {
657 struct scatterlist *sglist;
658 void *buf = mtt->buf;
659 u32 npages, i, nsg;
660 struct page *pg;
661
662 /* Failed if buf is not page aligned */
663 if ((uintptr_t)buf & ~PAGE_MASK)
664 return -EINVAL;
665
666 npages = DIV_ROUND_UP(mtt->size, PAGE_SIZE);
667 sglist = vzalloc(npages * sizeof(*sglist));
668 if (!sglist)
669 return -ENOMEM;
670
671 sg_init_table(sglist, npages);
672 for (i = 0; i < npages; i++) {
673 pg = vmalloc_to_page(buf);
674 if (!pg)
675 goto err;
676 sg_set_page(&sglist[i], pg, PAGE_SIZE, 0);
677 buf += PAGE_SIZE;
678 }
679
680 nsg = dma_map_sg(&dev->pdev->dev, sglist, npages, DMA_TO_DEVICE);
681 if (!nsg)
682 goto err;
683
684 mtt->sglist = sglist;
685 mtt->nsg = nsg;
686
687 return 0;
688 err:
689 vfree(sglist);
690
691 return -ENOMEM;
692 }
693
erdma_create_scatter_mtt(struct erdma_dev * dev,size_t size)694 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,
695 size_t size)
696 {
697 struct erdma_mtt *mtt;
698 int ret = -ENOMEM;
699
700 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
701 if (!mtt)
702 return ERR_PTR(-ENOMEM);
703
704 mtt->size = ALIGN(size, PAGE_SIZE);
705 mtt->buf = vzalloc(mtt->size);
706 mtt->continuous = false;
707 if (!mtt->buf)
708 goto err_free_mtt;
709
710 ret = erdma_create_mtt_buf_sg(dev, mtt);
711 if (ret)
712 goto err_free_mtt_buf;
713
714 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, nsg:%u\n",
715 mtt->size, mtt->nsg);
716
717 return mtt;
718
719 err_free_mtt_buf:
720 vfree(mtt->buf);
721
722 err_free_mtt:
723 kfree(mtt);
724
725 return ERR_PTR(ret);
726 }
727
erdma_create_mtt(struct erdma_dev * dev,size_t size,bool force_continuous)728 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,
729 bool force_continuous)
730 {
731 struct erdma_mtt *mtt, *tmp_mtt;
732 int ret, level = 0;
733
734 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size,
735 force_continuous);
736
737 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA))
738 force_continuous = true;
739
740 if (force_continuous)
741 return erdma_create_cont_mtt(dev, size);
742
743 mtt = erdma_create_scatter_mtt(dev, size);
744 if (IS_ERR(mtt))
745 return mtt;
746 level = 1;
747
748 /* convergence the mtt table. */
749 while (mtt->nsg != 1 && level <= 3) {
750 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->nsg));
751 if (IS_ERR(tmp_mtt)) {
752 ret = PTR_ERR(tmp_mtt);
753 goto err_free_mtt;
754 }
755 erdma_init_middle_mtt(tmp_mtt, mtt);
756 tmp_mtt->low_level = mtt;
757 mtt = tmp_mtt;
758 level++;
759 }
760
761 if (level > 3) {
762 ret = -ENOMEM;
763 goto err_free_mtt;
764 }
765
766 mtt->level = level;
767 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",
768 mtt->level, mtt->sglist[0].dma_address);
769
770 return mtt;
771 err_free_mtt:
772 while (mtt) {
773 tmp_mtt = mtt->low_level;
774 erdma_destroy_scatter_mtt(dev, mtt);
775 mtt = tmp_mtt;
776 }
777
778 return ERR_PTR(ret);
779 }
780
erdma_destroy_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)781 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
782 {
783 struct erdma_mtt *tmp_mtt;
784
785 if (mtt->continuous) {
786 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size,
787 DMA_TO_DEVICE);
788 kfree(mtt->buf);
789 kfree(mtt);
790 } else {
791 while (mtt) {
792 tmp_mtt = mtt->low_level;
793 erdma_destroy_scatter_mtt(dev, mtt);
794 mtt = tmp_mtt;
795 }
796 }
797 }
798
get_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem,u64 start,u64 len,int access,u64 virt,unsigned long req_page_size,bool force_continuous)799 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
800 u64 start, u64 len, int access, u64 virt,
801 unsigned long req_page_size, bool force_continuous)
802 {
803 int ret = 0;
804
805 mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
806 if (IS_ERR(mem->umem)) {
807 ret = PTR_ERR(mem->umem);
808 mem->umem = NULL;
809 return ret;
810 }
811
812 mem->va = virt;
813 mem->len = len;
814 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
815 mem->page_offset = start & (mem->page_size - 1);
816 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
817 mem->page_cnt = mem->mtt_nents;
818 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
819 force_continuous);
820 if (IS_ERR(mem->mtt)) {
821 ret = PTR_ERR(mem->mtt);
822 goto error_ret;
823 }
824
825 erdma_fill_bottom_mtt(dev, mem);
826
827 return 0;
828
829 error_ret:
830 if (mem->umem) {
831 ib_umem_release(mem->umem);
832 mem->umem = NULL;
833 }
834
835 return ret;
836 }
837
put_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem)838 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
839 {
840 if (mem->mtt)
841 erdma_destroy_mtt(dev, mem->mtt);
842
843 if (mem->umem) {
844 ib_umem_release(mem->umem);
845 mem->umem = NULL;
846 }
847 }
848
erdma_map_user_dbrecords(struct erdma_ucontext * ctx,u64 dbrecords_va,struct erdma_user_dbrecords_page ** dbr_page,dma_addr_t * dma_addr)849 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
850 u64 dbrecords_va,
851 struct erdma_user_dbrecords_page **dbr_page,
852 dma_addr_t *dma_addr)
853 {
854 struct erdma_user_dbrecords_page *page = NULL;
855 int rv = 0;
856
857 mutex_lock(&ctx->dbrecords_page_mutex);
858
859 list_for_each_entry(page, &ctx->dbrecords_page_list, list)
860 if (page->va == (dbrecords_va & PAGE_MASK))
861 goto found;
862
863 page = kmalloc(sizeof(*page), GFP_KERNEL);
864 if (!page) {
865 rv = -ENOMEM;
866 goto out;
867 }
868
869 page->va = (dbrecords_va & PAGE_MASK);
870 page->refcnt = 0;
871
872 page->umem = ib_umem_get(ctx->ibucontext.device,
873 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
874 if (IS_ERR(page->umem)) {
875 rv = PTR_ERR(page->umem);
876 kfree(page);
877 goto out;
878 }
879
880 list_add(&page->list, &ctx->dbrecords_page_list);
881
882 found:
883 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
884 (dbrecords_va & ~PAGE_MASK);
885 *dbr_page = page;
886 page->refcnt++;
887
888 out:
889 mutex_unlock(&ctx->dbrecords_page_mutex);
890 return rv;
891 }
892
893 static void
erdma_unmap_user_dbrecords(struct erdma_ucontext * ctx,struct erdma_user_dbrecords_page ** dbr_page)894 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
895 struct erdma_user_dbrecords_page **dbr_page)
896 {
897 if (!ctx || !(*dbr_page))
898 return;
899
900 mutex_lock(&ctx->dbrecords_page_mutex);
901 if (--(*dbr_page)->refcnt == 0) {
902 list_del(&(*dbr_page)->list);
903 ib_umem_release((*dbr_page)->umem);
904 kfree(*dbr_page);
905 }
906
907 *dbr_page = NULL;
908 mutex_unlock(&ctx->dbrecords_page_mutex);
909 }
910
init_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx,u64 va,u32 len,u64 dbrec_va)911 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
912 u64 va, u32 len, u64 dbrec_va)
913 {
914 dma_addr_t dbrec_dma;
915 u32 rq_offset;
916 int ret;
917
918 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
919 qp->attrs.rq_size * RQE_SIZE))
920 return -EINVAL;
921
922 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
923 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
924 (SZ_1M - SZ_4K), true);
925 if (ret)
926 return ret;
927
928 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
929 qp->user_qp.rq_offset = rq_offset;
930
931 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
932 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
933 (SZ_1M - SZ_4K), true);
934 if (ret)
935 goto put_sq_mtt;
936
937 ret = erdma_map_user_dbrecords(uctx, dbrec_va,
938 &qp->user_qp.user_dbr_page,
939 &dbrec_dma);
940 if (ret)
941 goto put_rq_mtt;
942
943 qp->user_qp.sq_dbrec_dma = dbrec_dma;
944 qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
945
946 return 0;
947
948 put_rq_mtt:
949 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
950
951 put_sq_mtt:
952 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
953
954 return ret;
955 }
956
free_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx)957 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
958 {
959 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
960 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
961 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
962 }
963
erdma_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)964 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
965 struct ib_udata *udata)
966 {
967 struct erdma_qp *qp = to_eqp(ibqp);
968 struct erdma_dev *dev = to_edev(ibqp->device);
969 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
970 udata, struct erdma_ucontext, ibucontext);
971 struct erdma_ureq_create_qp ureq;
972 struct erdma_uresp_create_qp uresp;
973 void *old_entry;
974 int ret = 0;
975
976 ret = erdma_qp_validate_cap(dev, attrs);
977 if (ret)
978 goto err_out;
979
980 ret = erdma_qp_validate_attr(dev, attrs);
981 if (ret)
982 goto err_out;
983
984 qp->scq = to_ecq(attrs->send_cq);
985 qp->rcq = to_ecq(attrs->recv_cq);
986 qp->dev = dev;
987 qp->attrs.cc = dev->attrs.cc;
988
989 init_rwsem(&qp->state_lock);
990 kref_init(&qp->ref);
991 init_completion(&qp->safe_free);
992
993 if (qp->ibqp.qp_type == IB_QPT_GSI) {
994 old_entry = xa_store(&dev->qp_xa, 1, qp, GFP_KERNEL);
995 if (xa_is_err(old_entry))
996 ret = xa_err(old_entry);
997 } else {
998 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
999 XA_LIMIT(1, dev->attrs.max_qp - 1),
1000 &dev->next_alloc_qpn, GFP_KERNEL);
1001 }
1002
1003 if (ret < 0) {
1004 ret = -ENOMEM;
1005 goto err_out;
1006 }
1007
1008 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
1009 ERDMA_MAX_WQEBB_PER_SQE);
1010 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
1011
1012 if (uctx) {
1013 ret = ib_copy_from_udata(&ureq, udata,
1014 min(sizeof(ureq), udata->inlen));
1015 if (ret)
1016 goto err_out_xa;
1017
1018 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
1019 ureq.db_record_va);
1020 if (ret)
1021 goto err_out_xa;
1022
1023 memset(&uresp, 0, sizeof(uresp));
1024
1025 uresp.num_sqe = qp->attrs.sq_size;
1026 uresp.num_rqe = qp->attrs.rq_size;
1027 uresp.qp_id = QP_ID(qp);
1028 uresp.rq_offset = qp->user_qp.rq_offset;
1029
1030 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1031 if (ret)
1032 goto err_out_cmd;
1033 } else {
1034 init_kernel_qp(dev, qp, attrs);
1035 }
1036
1037 qp->attrs.max_send_sge = attrs->cap.max_send_sge;
1038 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
1039
1040 if (erdma_device_iwarp(qp->dev))
1041 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE;
1042 else
1043 qp->attrs.rocev2.state = ERDMA_QPS_ROCEV2_RESET;
1044
1045 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
1046
1047 ret = create_qp_cmd(uctx, qp);
1048 if (ret)
1049 goto err_out_cmd;
1050
1051 spin_lock_init(&qp->lock);
1052
1053 return 0;
1054
1055 err_out_cmd:
1056 if (uctx)
1057 free_user_qp(qp, uctx);
1058 else
1059 free_kernel_qp(qp);
1060 err_out_xa:
1061 xa_erase(&dev->qp_xa, QP_ID(qp));
1062 err_out:
1063 return ret;
1064 }
1065
erdma_create_stag(struct erdma_dev * dev,u32 * stag)1066 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
1067 {
1068 int stag_idx;
1069
1070 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
1071 if (stag_idx < 0)
1072 return stag_idx;
1073
1074 /* For now, we always let key field be zero. */
1075 *stag = (stag_idx << 8);
1076
1077 return 0;
1078 }
1079
erdma_get_dma_mr(struct ib_pd * ibpd,int acc)1080 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
1081 {
1082 struct erdma_dev *dev = to_edev(ibpd->device);
1083 struct erdma_mr *mr;
1084 u32 stag;
1085 int ret;
1086
1087 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1088 if (!mr)
1089 return ERR_PTR(-ENOMEM);
1090
1091 ret = erdma_create_stag(dev, &stag);
1092 if (ret)
1093 goto out_free;
1094
1095 mr->type = ERDMA_MR_TYPE_DMA;
1096
1097 mr->ibmr.lkey = stag;
1098 mr->ibmr.rkey = stag;
1099 mr->ibmr.pd = ibpd;
1100 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
1101 ret = regmr_cmd(dev, mr);
1102 if (ret)
1103 goto out_remove_stag;
1104
1105 return &mr->ibmr;
1106
1107 out_remove_stag:
1108 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1109 mr->ibmr.lkey >> 8);
1110
1111 out_free:
1112 kfree(mr);
1113
1114 return ERR_PTR(ret);
1115 }
1116
erdma_ib_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)1117 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
1118 u32 max_num_sg)
1119 {
1120 struct erdma_mr *mr;
1121 struct erdma_dev *dev = to_edev(ibpd->device);
1122 int ret;
1123 u32 stag;
1124
1125 if (mr_type != IB_MR_TYPE_MEM_REG)
1126 return ERR_PTR(-EOPNOTSUPP);
1127
1128 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
1129 return ERR_PTR(-EINVAL);
1130
1131 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1132 if (!mr)
1133 return ERR_PTR(-ENOMEM);
1134
1135 ret = erdma_create_stag(dev, &stag);
1136 if (ret)
1137 goto out_free;
1138
1139 mr->type = ERDMA_MR_TYPE_FRMR;
1140
1141 mr->ibmr.lkey = stag;
1142 mr->ibmr.rkey = stag;
1143 mr->ibmr.pd = ibpd;
1144 /* update it in FRMR. */
1145 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
1146 ERDMA_MR_ACC_RW;
1147
1148 mr->mem.page_size = PAGE_SIZE; /* update it later. */
1149 mr->mem.page_cnt = max_num_sg;
1150 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);
1151 if (IS_ERR(mr->mem.mtt)) {
1152 ret = PTR_ERR(mr->mem.mtt);
1153 goto out_remove_stag;
1154 }
1155
1156 ret = regmr_cmd(dev, mr);
1157 if (ret)
1158 goto out_destroy_mtt;
1159
1160 return &mr->ibmr;
1161
1162 out_destroy_mtt:
1163 erdma_destroy_mtt(dev, mr->mem.mtt);
1164
1165 out_remove_stag:
1166 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1167 mr->ibmr.lkey >> 8);
1168
1169 out_free:
1170 kfree(mr);
1171
1172 return ERR_PTR(ret);
1173 }
1174
erdma_set_page(struct ib_mr * ibmr,u64 addr)1175 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
1176 {
1177 struct erdma_mr *mr = to_emr(ibmr);
1178
1179 if (mr->mem.mtt_nents >= mr->mem.page_cnt)
1180 return -1;
1181
1182 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr;
1183 mr->mem.mtt_nents++;
1184
1185 return 0;
1186 }
1187
erdma_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)1188 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1189 unsigned int *sg_offset)
1190 {
1191 struct erdma_mr *mr = to_emr(ibmr);
1192 int num;
1193
1194 mr->mem.mtt_nents = 0;
1195
1196 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
1197 erdma_set_page);
1198
1199 return num;
1200 }
1201
erdma_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 virt,int access,struct ib_dmah * dmah,struct ib_udata * udata)1202 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1203 u64 virt, int access, struct ib_dmah *dmah,
1204 struct ib_udata *udata)
1205 {
1206 struct erdma_mr *mr = NULL;
1207 struct erdma_dev *dev = to_edev(ibpd->device);
1208 u32 stag;
1209 int ret;
1210
1211 if (dmah)
1212 return ERR_PTR(-EOPNOTSUPP);
1213
1214 if (!len || len > dev->attrs.max_mr_size)
1215 return ERR_PTR(-EINVAL);
1216
1217 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1218 if (!mr)
1219 return ERR_PTR(-ENOMEM);
1220
1221 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
1222 SZ_2G - SZ_4K, false);
1223 if (ret)
1224 goto err_out_free;
1225
1226 ret = erdma_create_stag(dev, &stag);
1227 if (ret)
1228 goto err_out_put_mtt;
1229
1230 mr->ibmr.lkey = mr->ibmr.rkey = stag;
1231 mr->ibmr.pd = ibpd;
1232 mr->mem.va = virt;
1233 mr->mem.len = len;
1234 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
1235 mr->valid = 1;
1236 mr->type = ERDMA_MR_TYPE_NORMAL;
1237
1238 ret = regmr_cmd(dev, mr);
1239 if (ret)
1240 goto err_out_mr;
1241
1242 return &mr->ibmr;
1243
1244 err_out_mr:
1245 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1246 mr->ibmr.lkey >> 8);
1247
1248 err_out_put_mtt:
1249 put_mtt_entries(dev, &mr->mem);
1250
1251 err_out_free:
1252 kfree(mr);
1253
1254 return ERR_PTR(ret);
1255 }
1256
erdma_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)1257 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1258 {
1259 struct erdma_mr *mr;
1260 struct erdma_dev *dev = to_edev(ibmr->device);
1261 struct erdma_cmdq_dereg_mr_req req;
1262 int ret;
1263
1264 mr = to_emr(ibmr);
1265
1266 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1267 CMDQ_OPCODE_DEREG_MR);
1268
1269 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
1270 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
1271
1272 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1273 true);
1274 if (ret)
1275 return ret;
1276
1277 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
1278
1279 put_mtt_entries(dev, &mr->mem);
1280
1281 kfree(mr);
1282 return 0;
1283 }
1284
erdma_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)1285 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1286 {
1287 struct erdma_cq *cq = to_ecq(ibcq);
1288 struct erdma_dev *dev = to_edev(ibcq->device);
1289 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1290 udata, struct erdma_ucontext, ibucontext);
1291 int err;
1292 struct erdma_cmdq_destroy_cq_req req;
1293
1294 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1295 CMDQ_OPCODE_DESTROY_CQ);
1296 req.cqn = cq->cqn;
1297
1298 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1299 true);
1300 if (err)
1301 return err;
1302
1303 if (rdma_is_kernel_res(&cq->ibcq.res)) {
1304 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1305 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1306 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1307 cq->kern_cq.dbrec_dma);
1308 } else {
1309 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1310 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1311 }
1312
1313 xa_erase(&dev->cq_xa, cq->cqn);
1314
1315 return 0;
1316 }
1317
erdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)1318 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1319 {
1320 struct erdma_qp *qp = to_eqp(ibqp);
1321 struct erdma_dev *dev = to_edev(ibqp->device);
1322 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1323 udata, struct erdma_ucontext, ibucontext);
1324 struct erdma_cmdq_destroy_qp_req req;
1325 union erdma_mod_qp_params params;
1326 int err;
1327
1328 down_write(&qp->state_lock);
1329 if (erdma_device_iwarp(dev)) {
1330 params.iwarp.state = ERDMA_QPS_IWARP_ERROR;
1331 erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1332 ERDMA_QPA_IWARP_STATE);
1333 } else {
1334 params.rocev2.state = ERDMA_QPS_ROCEV2_ERROR;
1335 erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1336 ERDMA_QPA_ROCEV2_STATE);
1337 }
1338 up_write(&qp->state_lock);
1339
1340 cancel_delayed_work_sync(&qp->reflush_dwork);
1341
1342 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1343 CMDQ_OPCODE_DESTROY_QP);
1344 req.qpn = QP_ID(qp);
1345
1346 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1347 true);
1348 if (err)
1349 return err;
1350
1351 erdma_qp_put(qp);
1352 wait_for_completion(&qp->safe_free);
1353
1354 if (rdma_is_kernel_res(&qp->ibqp.res)) {
1355 free_kernel_qp(qp);
1356 } else {
1357 put_mtt_entries(dev, &qp->user_qp.sq_mem);
1358 put_mtt_entries(dev, &qp->user_qp.rq_mem);
1359 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1360 }
1361
1362 if (qp->cep)
1363 erdma_cep_put(qp->cep);
1364 xa_erase(&dev->qp_xa, QP_ID(qp));
1365
1366 return 0;
1367 }
1368
erdma_qp_get_ref(struct ib_qp * ibqp)1369 void erdma_qp_get_ref(struct ib_qp *ibqp)
1370 {
1371 erdma_qp_get(to_eqp(ibqp));
1372 }
1373
erdma_qp_put_ref(struct ib_qp * ibqp)1374 void erdma_qp_put_ref(struct ib_qp *ibqp)
1375 {
1376 erdma_qp_put(to_eqp(ibqp));
1377 }
1378
erdma_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)1379 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1380 {
1381 struct rdma_user_mmap_entry *rdma_entry;
1382 struct erdma_user_mmap_entry *entry;
1383 pgprot_t prot;
1384 int err;
1385
1386 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1387 if (!rdma_entry)
1388 return -EINVAL;
1389
1390 entry = to_emmap(rdma_entry);
1391
1392 switch (entry->mmap_flag) {
1393 case ERDMA_MMAP_IO_NC:
1394 /* map doorbell. */
1395 prot = pgprot_device(vma->vm_page_prot);
1396 break;
1397 default:
1398 err = -EINVAL;
1399 goto put_entry;
1400 }
1401
1402 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1403 prot, rdma_entry);
1404
1405 put_entry:
1406 rdma_user_mmap_entry_put(rdma_entry);
1407 return err;
1408 }
1409
erdma_mmap_free(struct rdma_user_mmap_entry * rdma_entry)1410 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1411 {
1412 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1413
1414 kfree(entry);
1415 }
1416
alloc_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx,bool ext_db_en)1417 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
1418 bool ext_db_en)
1419 {
1420 struct erdma_cmdq_ext_db_req req = {};
1421 u64 val0, val1;
1422 int ret;
1423
1424 /*
1425 * CAP_SYS_RAWIO is required if hardware does not support extend
1426 * doorbell mechanism.
1427 */
1428 if (!ext_db_en && !capable(CAP_SYS_RAWIO))
1429 return -EPERM;
1430
1431 if (!ext_db_en) {
1432 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;
1433 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1434 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1435 return 0;
1436 }
1437
1438 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1439 CMDQ_OPCODE_ALLOC_DB);
1440
1441 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1442 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1443 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1444
1445 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1,
1446 true);
1447 if (ret)
1448 return ret;
1449
1450 ctx->ext_db.enable = true;
1451 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);
1452 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);
1453 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);
1454
1455 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);
1456 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);
1457 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);
1458
1459 return 0;
1460 }
1461
free_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx)1462 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
1463 {
1464 struct erdma_cmdq_ext_db_req req = {};
1465 int ret;
1466
1467 if (!ctx->ext_db.enable)
1468 return;
1469
1470 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1471 CMDQ_OPCODE_FREE_DB);
1472
1473 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1474 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1475 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1476
1477 req.sdb_off = ctx->ext_db.sdb_off;
1478 req.rdb_off = ctx->ext_db.rdb_off;
1479 req.cdb_off = ctx->ext_db.cdb_off;
1480
1481 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1482 true);
1483 if (ret)
1484 ibdev_err_ratelimited(&dev->ibdev,
1485 "free db resources failed %d", ret);
1486 }
1487
erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext * uctx)1488 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1489 {
1490 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1491 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1492 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1493 }
1494
erdma_alloc_ucontext(struct ib_ucontext * ibctx,struct ib_udata * udata)1495 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1496 {
1497 struct erdma_ucontext *ctx = to_ectx(ibctx);
1498 struct erdma_dev *dev = to_edev(ibctx->device);
1499 int ret;
1500 struct erdma_uresp_alloc_ctx uresp = {};
1501
1502 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1503 ret = -ENOMEM;
1504 goto err_out;
1505 }
1506
1507 if (udata->outlen < sizeof(uresp)) {
1508 ret = -EINVAL;
1509 goto err_out;
1510 }
1511
1512 INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1513 mutex_init(&ctx->dbrecords_page_mutex);
1514
1515 ret = alloc_db_resources(dev, ctx,
1516 !!(dev->attrs.cap_flags &
1517 ERDMA_DEV_CAP_FLAGS_EXTEND_DB));
1518 if (ret)
1519 goto err_out;
1520
1521 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1522 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1523 if (!ctx->sq_db_mmap_entry) {
1524 ret = -ENOMEM;
1525 goto err_free_ext_db;
1526 }
1527
1528 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1529 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1530 if (!ctx->rq_db_mmap_entry) {
1531 ret = -EINVAL;
1532 goto err_put_mmap_entries;
1533 }
1534
1535 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1536 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1537 if (!ctx->cq_db_mmap_entry) {
1538 ret = -EINVAL;
1539 goto err_put_mmap_entries;
1540 }
1541
1542 uresp.dev_id = dev->pdev->device;
1543
1544 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1545 if (ret)
1546 goto err_put_mmap_entries;
1547
1548 return 0;
1549
1550 err_put_mmap_entries:
1551 erdma_uctx_user_mmap_entries_remove(ctx);
1552
1553 err_free_ext_db:
1554 free_db_resources(dev, ctx);
1555
1556 err_out:
1557 atomic_dec(&dev->num_ctx);
1558 return ret;
1559 }
1560
erdma_dealloc_ucontext(struct ib_ucontext * ibctx)1561 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1562 {
1563 struct erdma_dev *dev = to_edev(ibctx->device);
1564 struct erdma_ucontext *ctx = to_ectx(ibctx);
1565
1566 erdma_uctx_user_mmap_entries_remove(ctx);
1567 free_db_resources(dev, ctx);
1568 atomic_dec(&dev->num_ctx);
1569 }
1570
erdma_attr_to_av(const struct rdma_ah_attr * ah_attr,struct erdma_av * av,u16 sport)1571 static void erdma_attr_to_av(const struct rdma_ah_attr *ah_attr,
1572 struct erdma_av *av, u16 sport)
1573 {
1574 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
1575
1576 av->port = rdma_ah_get_port_num(ah_attr);
1577 av->sgid_index = grh->sgid_index;
1578 av->hop_limit = grh->hop_limit;
1579 av->traffic_class = grh->traffic_class;
1580 av->sl = rdma_ah_get_sl(ah_attr);
1581
1582 av->flow_label = grh->flow_label;
1583 av->udp_sport = sport;
1584
1585 ether_addr_copy(av->dmac, ah_attr->roce.dmac);
1586 memcpy(av->dgid, grh->dgid.raw, ERDMA_ROCEV2_GID_SIZE);
1587
1588 if (ipv6_addr_v4mapped((struct in6_addr *)&grh->dgid))
1589 av->ntype = ERDMA_NETWORK_TYPE_IPV4;
1590 else
1591 av->ntype = ERDMA_NETWORK_TYPE_IPV6;
1592 }
1593
erdma_av_to_attr(struct erdma_av * av,struct rdma_ah_attr * ah_attr)1594 static void erdma_av_to_attr(struct erdma_av *av, struct rdma_ah_attr *ah_attr)
1595 {
1596 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE;
1597
1598 rdma_ah_set_sl(ah_attr, av->sl);
1599 rdma_ah_set_port_num(ah_attr, av->port);
1600 rdma_ah_set_ah_flags(ah_attr, IB_AH_GRH);
1601
1602 rdma_ah_set_grh(ah_attr, NULL, av->flow_label, av->sgid_index,
1603 av->hop_limit, av->traffic_class);
1604 rdma_ah_set_dgid_raw(ah_attr, av->dgid);
1605 }
1606
1607 static int ib_qps_to_erdma_qps[ERDMA_PROTO_COUNT][IB_QPS_ERR + 1] = {
1608 [ERDMA_PROTO_IWARP] = {
1609 [IB_QPS_RESET] = ERDMA_QPS_IWARP_IDLE,
1610 [IB_QPS_INIT] = ERDMA_QPS_IWARP_IDLE,
1611 [IB_QPS_RTR] = ERDMA_QPS_IWARP_RTR,
1612 [IB_QPS_RTS] = ERDMA_QPS_IWARP_RTS,
1613 [IB_QPS_SQD] = ERDMA_QPS_IWARP_CLOSING,
1614 [IB_QPS_SQE] = ERDMA_QPS_IWARP_TERMINATE,
1615 [IB_QPS_ERR] = ERDMA_QPS_IWARP_ERROR,
1616 },
1617 [ERDMA_PROTO_ROCEV2] = {
1618 [IB_QPS_RESET] = ERDMA_QPS_ROCEV2_RESET,
1619 [IB_QPS_INIT] = ERDMA_QPS_ROCEV2_INIT,
1620 [IB_QPS_RTR] = ERDMA_QPS_ROCEV2_RTR,
1621 [IB_QPS_RTS] = ERDMA_QPS_ROCEV2_RTS,
1622 [IB_QPS_SQD] = ERDMA_QPS_ROCEV2_SQD,
1623 [IB_QPS_SQE] = ERDMA_QPS_ROCEV2_SQE,
1624 [IB_QPS_ERR] = ERDMA_QPS_ROCEV2_ERROR,
1625 },
1626 };
1627
1628 static int erdma_qps_to_ib_qps[ERDMA_PROTO_COUNT][ERDMA_QPS_ROCEV2_COUNT] = {
1629 [ERDMA_PROTO_IWARP] = {
1630 [ERDMA_QPS_IWARP_IDLE] = IB_QPS_INIT,
1631 [ERDMA_QPS_IWARP_RTR] = IB_QPS_RTR,
1632 [ERDMA_QPS_IWARP_RTS] = IB_QPS_RTS,
1633 [ERDMA_QPS_IWARP_CLOSING] = IB_QPS_ERR,
1634 [ERDMA_QPS_IWARP_TERMINATE] = IB_QPS_ERR,
1635 [ERDMA_QPS_IWARP_ERROR] = IB_QPS_ERR,
1636 },
1637 [ERDMA_PROTO_ROCEV2] = {
1638 [ERDMA_QPS_ROCEV2_RESET] = IB_QPS_RESET,
1639 [ERDMA_QPS_ROCEV2_INIT] = IB_QPS_INIT,
1640 [ERDMA_QPS_ROCEV2_RTR] = IB_QPS_RTR,
1641 [ERDMA_QPS_ROCEV2_RTS] = IB_QPS_RTS,
1642 [ERDMA_QPS_ROCEV2_SQD] = IB_QPS_SQD,
1643 [ERDMA_QPS_ROCEV2_SQE] = IB_QPS_SQE,
1644 [ERDMA_QPS_ROCEV2_ERROR] = IB_QPS_ERR,
1645 },
1646 };
1647
ib_to_iwarp_qps(enum ib_qp_state state)1648 static inline enum erdma_qps_iwarp ib_to_iwarp_qps(enum ib_qp_state state)
1649 {
1650 return ib_qps_to_erdma_qps[ERDMA_PROTO_IWARP][state];
1651 }
1652
ib_to_rocev2_qps(enum ib_qp_state state)1653 static inline enum erdma_qps_rocev2 ib_to_rocev2_qps(enum ib_qp_state state)
1654 {
1655 return ib_qps_to_erdma_qps[ERDMA_PROTO_ROCEV2][state];
1656 }
1657
iwarp_to_ib_qps(enum erdma_qps_iwarp state)1658 static inline enum ib_qp_state iwarp_to_ib_qps(enum erdma_qps_iwarp state)
1659 {
1660 return erdma_qps_to_ib_qps[ERDMA_PROTO_IWARP][state];
1661 }
1662
rocev2_to_ib_qps(enum erdma_qps_rocev2 state)1663 static inline enum ib_qp_state rocev2_to_ib_qps(enum erdma_qps_rocev2 state)
1664 {
1665 return erdma_qps_to_ib_qps[ERDMA_PROTO_ROCEV2][state];
1666 }
1667
erdma_check_qp_attrs(struct erdma_qp * qp,struct ib_qp_attr * attr,int attr_mask)1668 static int erdma_check_qp_attrs(struct erdma_qp *qp, struct ib_qp_attr *attr,
1669 int attr_mask)
1670 {
1671 enum ib_qp_state cur_state, nxt_state;
1672 struct erdma_dev *dev = qp->dev;
1673 int ret = -EINVAL;
1674
1675 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) {
1676 ret = -EOPNOTSUPP;
1677 goto out;
1678 }
1679
1680 if ((attr_mask & IB_QP_PORT) &&
1681 !rdma_is_port_valid(&dev->ibdev, attr->port_num))
1682 goto out;
1683
1684 if (erdma_device_rocev2(dev)) {
1685 cur_state = (attr_mask & IB_QP_CUR_STATE) ?
1686 attr->cur_qp_state :
1687 rocev2_to_ib_qps(qp->attrs.rocev2.state);
1688
1689 nxt_state = (attr_mask & IB_QP_STATE) ? attr->qp_state :
1690 cur_state;
1691
1692 if (!ib_modify_qp_is_ok(cur_state, nxt_state, qp->ibqp.qp_type,
1693 attr_mask))
1694 goto out;
1695
1696 if ((attr_mask & IB_QP_AV) &&
1697 erdma_check_gid_attr(
1698 rdma_ah_read_grh(&attr->ah_attr)->sgid_attr))
1699 goto out;
1700
1701 if ((attr_mask & IB_QP_PKEY_INDEX) &&
1702 attr->pkey_index >= ERDMA_MAX_PKEYS)
1703 goto out;
1704 }
1705
1706 return 0;
1707
1708 out:
1709 return ret;
1710 }
1711
erdma_init_mod_qp_params_rocev2(struct erdma_qp * qp,struct erdma_mod_qp_params_rocev2 * params,int * erdma_attr_mask,struct ib_qp_attr * attr,int ib_attr_mask)1712 static void erdma_init_mod_qp_params_rocev2(
1713 struct erdma_qp *qp, struct erdma_mod_qp_params_rocev2 *params,
1714 int *erdma_attr_mask, struct ib_qp_attr *attr, int ib_attr_mask)
1715 {
1716 enum erdma_qpa_mask_rocev2 to_modify_attrs = 0;
1717 enum erdma_qps_rocev2 cur_state, nxt_state;
1718 u16 udp_sport;
1719
1720 if (ib_attr_mask & IB_QP_CUR_STATE)
1721 cur_state = ib_to_rocev2_qps(attr->cur_qp_state);
1722 else
1723 cur_state = qp->attrs.rocev2.state;
1724
1725 if (ib_attr_mask & IB_QP_STATE)
1726 nxt_state = ib_to_rocev2_qps(attr->qp_state);
1727 else
1728 nxt_state = cur_state;
1729
1730 to_modify_attrs |= ERDMA_QPA_ROCEV2_STATE;
1731 params->state = nxt_state;
1732
1733 if (ib_attr_mask & IB_QP_QKEY) {
1734 to_modify_attrs |= ERDMA_QPA_ROCEV2_QKEY;
1735 params->qkey = attr->qkey;
1736 }
1737
1738 if (ib_attr_mask & IB_QP_SQ_PSN) {
1739 to_modify_attrs |= ERDMA_QPA_ROCEV2_SQ_PSN;
1740 params->sq_psn = attr->sq_psn;
1741 }
1742
1743 if (ib_attr_mask & IB_QP_RQ_PSN) {
1744 to_modify_attrs |= ERDMA_QPA_ROCEV2_RQ_PSN;
1745 params->rq_psn = attr->rq_psn;
1746 }
1747
1748 if (ib_attr_mask & IB_QP_DEST_QPN) {
1749 to_modify_attrs |= ERDMA_QPA_ROCEV2_DST_QPN;
1750 params->dst_qpn = attr->dest_qp_num;
1751 }
1752
1753 if (ib_attr_mask & IB_QP_AV) {
1754 to_modify_attrs |= ERDMA_QPA_ROCEV2_AV;
1755 udp_sport = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
1756 QP_ID(qp), params->dst_qpn);
1757 erdma_attr_to_av(&attr->ah_attr, ¶ms->av, udp_sport);
1758 }
1759
1760 *erdma_attr_mask = to_modify_attrs;
1761 }
1762
erdma_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1763 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1764 struct ib_udata *udata)
1765 {
1766 struct erdma_qp *qp = to_eqp(ibqp);
1767 union erdma_mod_qp_params params;
1768 int ret = 0, erdma_attr_mask = 0;
1769
1770 down_write(&qp->state_lock);
1771
1772 ret = erdma_check_qp_attrs(qp, attr, attr_mask);
1773 if (ret)
1774 goto out;
1775
1776 if (erdma_device_iwarp(qp->dev)) {
1777 if (attr_mask & IB_QP_STATE) {
1778 erdma_attr_mask |= ERDMA_QPA_IWARP_STATE;
1779 params.iwarp.state = ib_to_iwarp_qps(attr->qp_state);
1780 }
1781
1782 ret = erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1783 erdma_attr_mask);
1784 } else {
1785 erdma_init_mod_qp_params_rocev2(
1786 qp, ¶ms.rocev2, &erdma_attr_mask, attr, attr_mask);
1787
1788 ret = erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1789 erdma_attr_mask);
1790 }
1791
1792 out:
1793 up_write(&qp->state_lock);
1794 return ret;
1795 }
1796
query_qp_state(struct erdma_qp * qp)1797 static enum ib_qp_state query_qp_state(struct erdma_qp *qp)
1798 {
1799 if (erdma_device_iwarp(qp->dev))
1800 return iwarp_to_ib_qps(qp->attrs.iwarp.state);
1801 else
1802 return rocev2_to_ib_qps(qp->attrs.rocev2.state);
1803 }
1804
erdma_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1805 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1806 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1807 {
1808 struct erdma_cmdq_query_qp_req_rocev2 req;
1809 struct erdma_dev *dev;
1810 struct erdma_qp *qp;
1811 u64 resp0, resp1;
1812 int ret;
1813
1814 if (ibqp && qp_attr && qp_init_attr) {
1815 qp = to_eqp(ibqp);
1816 dev = to_edev(ibqp->device);
1817 } else {
1818 return -EINVAL;
1819 }
1820
1821 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1822 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1823
1824 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1825 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1826 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1827 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1828
1829 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1830 qp_attr->max_rd_atomic = qp->attrs.irq_size;
1831 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1832
1833 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1834 IB_ACCESS_REMOTE_WRITE |
1835 IB_ACCESS_REMOTE_READ;
1836
1837 qp_init_attr->cap = qp_attr->cap;
1838
1839 if (erdma_device_rocev2(dev)) {
1840 /* Query hardware to get some attributes */
1841 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1842 CMDQ_OPCODE_QUERY_QP);
1843 req.qpn = QP_ID(qp);
1844
1845 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
1846 &resp1, true);
1847 if (ret)
1848 return ret;
1849
1850 qp_attr->sq_psn =
1851 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK, resp0);
1852 qp_attr->rq_psn =
1853 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK, resp0);
1854 qp_attr->qp_state = rocev2_to_ib_qps(FIELD_GET(
1855 ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK, resp0));
1856 qp_attr->cur_qp_state = qp_attr->qp_state;
1857 qp_attr->sq_draining = FIELD_GET(
1858 ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK, resp0);
1859
1860 qp_attr->pkey_index = 0;
1861 qp_attr->dest_qp_num = qp->attrs.rocev2.dst_qpn;
1862
1863 if (qp->ibqp.qp_type == IB_QPT_RC)
1864 erdma_av_to_attr(&qp->attrs.rocev2.av,
1865 &qp_attr->ah_attr);
1866 } else {
1867 qp_attr->qp_state = query_qp_state(qp);
1868 qp_attr->cur_qp_state = qp_attr->qp_state;
1869 }
1870
1871 return 0;
1872 }
1873
erdma_init_user_cq(struct erdma_ucontext * ctx,struct erdma_cq * cq,struct erdma_ureq_create_cq * ureq)1874 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1875 struct erdma_ureq_create_cq *ureq)
1876 {
1877 int ret;
1878 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1879
1880 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
1881 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1882 true);
1883 if (ret)
1884 return ret;
1885
1886 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1887 &cq->user_cq.user_dbr_page,
1888 &cq->user_cq.dbrec_dma);
1889 if (ret)
1890 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1891
1892 return ret;
1893 }
1894
erdma_init_kernel_cq(struct erdma_cq * cq)1895 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1896 {
1897 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1898
1899 cq->kern_cq.qbuf =
1900 dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1901 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1902 if (!cq->kern_cq.qbuf)
1903 return -ENOMEM;
1904
1905 cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
1906 &cq->kern_cq.dbrec_dma);
1907 if (!cq->kern_cq.dbrec)
1908 goto err_out;
1909
1910 spin_lock_init(&cq->kern_cq.lock);
1911 /* use default cqdb addr */
1912 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1913
1914 return 0;
1915
1916 err_out:
1917 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1918 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1919
1920 return -ENOMEM;
1921 }
1922
erdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)1923 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1924 struct uverbs_attr_bundle *attrs)
1925 {
1926 struct ib_udata *udata = &attrs->driver_udata;
1927 struct erdma_cq *cq = to_ecq(ibcq);
1928 struct erdma_dev *dev = to_edev(ibcq->device);
1929 unsigned int depth = attr->cqe;
1930 int ret;
1931 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1932 udata, struct erdma_ucontext, ibucontext);
1933
1934 if (depth > dev->attrs.max_cqe)
1935 return -EINVAL;
1936
1937 depth = roundup_pow_of_two(depth);
1938 cq->ibcq.cqe = depth;
1939 cq->depth = depth;
1940 cq->assoc_eqn = attr->comp_vector + 1;
1941
1942 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1943 XA_LIMIT(1, dev->attrs.max_cq - 1),
1944 &dev->next_alloc_cqn, GFP_KERNEL);
1945 if (ret < 0)
1946 return ret;
1947
1948 if (!rdma_is_kernel_res(&ibcq->res)) {
1949 struct erdma_ureq_create_cq ureq;
1950 struct erdma_uresp_create_cq uresp;
1951
1952 ret = ib_copy_from_udata(&ureq, udata,
1953 min(udata->inlen, sizeof(ureq)));
1954 if (ret)
1955 goto err_out_xa;
1956
1957 ret = erdma_init_user_cq(ctx, cq, &ureq);
1958 if (ret)
1959 goto err_out_xa;
1960
1961 uresp.cq_id = cq->cqn;
1962 uresp.num_cqe = depth;
1963
1964 ret = ib_copy_to_udata(udata, &uresp,
1965 min(sizeof(uresp), udata->outlen));
1966 if (ret)
1967 goto err_free_res;
1968 } else {
1969 ret = erdma_init_kernel_cq(cq);
1970 if (ret)
1971 goto err_out_xa;
1972 }
1973
1974 ret = create_cq_cmd(ctx, cq);
1975 if (ret)
1976 goto err_free_res;
1977
1978 return 0;
1979
1980 err_free_res:
1981 if (!rdma_is_kernel_res(&ibcq->res)) {
1982 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1983 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1984 } else {
1985 dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
1986 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1987 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1988 cq->kern_cq.dbrec_dma);
1989 }
1990
1991 err_out_xa:
1992 xa_erase(&dev->cq_xa, cq->cqn);
1993
1994 return ret;
1995 }
1996
erdma_disassociate_ucontext(struct ib_ucontext * ibcontext)1997 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)
1998 {
1999 }
2000
erdma_set_mtu(struct erdma_dev * dev,u32 mtu)2001 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
2002 {
2003 struct erdma_cmdq_config_mtu_req req;
2004
2005 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2006 CMDQ_OPCODE_CONF_MTU);
2007 req.mtu = mtu;
2008
2009 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true);
2010 }
2011
erdma_port_event(struct erdma_dev * dev,enum ib_event_type reason)2012 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
2013 {
2014 struct ib_event event;
2015
2016 event.device = &dev->ibdev;
2017 event.element.port_num = 1;
2018 event.event = reason;
2019
2020 ib_dispatch_event(&event);
2021 }
2022
2023 enum counters {
2024 ERDMA_STATS_TX_REQS_CNT,
2025 ERDMA_STATS_TX_PACKETS_CNT,
2026 ERDMA_STATS_TX_BYTES_CNT,
2027 ERDMA_STATS_TX_DISABLE_DROP_CNT,
2028 ERDMA_STATS_TX_BPS_METER_DROP_CNT,
2029 ERDMA_STATS_TX_PPS_METER_DROP_CNT,
2030
2031 ERDMA_STATS_RX_PACKETS_CNT,
2032 ERDMA_STATS_RX_BYTES_CNT,
2033 ERDMA_STATS_RX_DISABLE_DROP_CNT,
2034 ERDMA_STATS_RX_BPS_METER_DROP_CNT,
2035 ERDMA_STATS_RX_PPS_METER_DROP_CNT,
2036
2037 ERDMA_STATS_MAX
2038 };
2039
2040 static const struct rdma_stat_desc erdma_descs[] = {
2041 [ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt",
2042 [ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt",
2043 [ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt",
2044 [ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt",
2045 [ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt",
2046 [ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt",
2047 [ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt",
2048 [ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt",
2049 [ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt",
2050 [ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt",
2051 [ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt",
2052 };
2053
erdma_alloc_hw_port_stats(struct ib_device * device,u32 port_num)2054 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
2055 u32 port_num)
2056 {
2057 return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX,
2058 RDMA_HW_STATS_DEFAULT_LIFESPAN);
2059 }
2060
erdma_query_hw_stats(struct erdma_dev * dev,struct rdma_hw_stats * stats)2061 static int erdma_query_hw_stats(struct erdma_dev *dev,
2062 struct rdma_hw_stats *stats)
2063 {
2064 struct erdma_cmdq_query_stats_resp *resp;
2065 struct erdma_cmdq_query_req req;
2066 dma_addr_t dma_addr;
2067 int err;
2068
2069 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2070 CMDQ_OPCODE_GET_STATS);
2071
2072 resp = dma_pool_zalloc(dev->resp_pool, GFP_KERNEL, &dma_addr);
2073 if (!resp)
2074 return -ENOMEM;
2075
2076 req.target_addr = dma_addr;
2077 req.target_length = ERDMA_HW_RESP_SIZE;
2078
2079 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2080 true);
2081 if (err)
2082 goto out;
2083
2084 if (resp->hdr.magic != ERDMA_HW_RESP_MAGIC) {
2085 err = -EINVAL;
2086 goto out;
2087 }
2088
2089 memcpy(&stats->value[0], &resp->tx_req_cnt,
2090 sizeof(u64) * stats->num_counters);
2091
2092 out:
2093 dma_pool_free(dev->resp_pool, resp, dma_addr);
2094
2095 return err;
2096 }
2097
erdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port,int index)2098 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
2099 u32 port, int index)
2100 {
2101 struct erdma_dev *dev = to_edev(ibdev);
2102 int ret;
2103
2104 if (port == 0)
2105 return 0;
2106
2107 ret = erdma_query_hw_stats(dev, stats);
2108 if (ret)
2109 return ret;
2110
2111 return stats->num_counters;
2112 }
2113
erdma_get_link_layer(struct ib_device * ibdev,u32 port_num)2114 enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, u32 port_num)
2115 {
2116 return IB_LINK_LAYER_ETHERNET;
2117 }
2118
erdma_set_gid(struct erdma_dev * dev,u8 op,u32 idx,const union ib_gid * gid)2119 static int erdma_set_gid(struct erdma_dev *dev, u8 op, u32 idx,
2120 const union ib_gid *gid)
2121 {
2122 struct erdma_cmdq_set_gid_req req;
2123 u8 ntype;
2124
2125 req.cfg = FIELD_PREP(ERDMA_CMD_SET_GID_SGID_IDX_MASK, idx) |
2126 FIELD_PREP(ERDMA_CMD_SET_GID_OP_MASK, op);
2127
2128 if (op == ERDMA_SET_GID_OP_ADD) {
2129 if (ipv6_addr_v4mapped((struct in6_addr *)gid))
2130 ntype = ERDMA_NETWORK_TYPE_IPV4;
2131 else
2132 ntype = ERDMA_NETWORK_TYPE_IPV6;
2133
2134 req.cfg |= FIELD_PREP(ERDMA_CMD_SET_GID_NTYPE_MASK, ntype);
2135
2136 memcpy(&req.gid, gid, ERDMA_ROCEV2_GID_SIZE);
2137 }
2138
2139 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2140 CMDQ_OPCODE_SET_GID);
2141 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2142 true);
2143 }
2144
erdma_add_gid(const struct ib_gid_attr * attr,void ** context)2145 int erdma_add_gid(const struct ib_gid_attr *attr, void **context)
2146 {
2147 struct erdma_dev *dev = to_edev(attr->device);
2148 int ret;
2149
2150 ret = erdma_check_gid_attr(attr);
2151 if (ret)
2152 return ret;
2153
2154 return erdma_set_gid(dev, ERDMA_SET_GID_OP_ADD, attr->index,
2155 &attr->gid);
2156 }
2157
erdma_del_gid(const struct ib_gid_attr * attr,void ** context)2158 int erdma_del_gid(const struct ib_gid_attr *attr, void **context)
2159 {
2160 return erdma_set_gid(to_edev(attr->device), ERDMA_SET_GID_OP_DEL,
2161 attr->index, NULL);
2162 }
2163
erdma_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)2164 int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
2165 {
2166 if (index >= ERDMA_MAX_PKEYS)
2167 return -EINVAL;
2168
2169 *pkey = ERDMA_DEFAULT_PKEY;
2170 return 0;
2171 }
2172
erdma_set_av_cfg(struct erdma_av_cfg * av_cfg,struct erdma_av * av)2173 void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av)
2174 {
2175 av_cfg->cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_AV_FL_MASK, av->flow_label) |
2176 FIELD_PREP(ERDMA_CMD_CREATE_AV_NTYPE_MASK, av->ntype);
2177
2178 av_cfg->traffic_class = av->traffic_class;
2179 av_cfg->hop_limit = av->hop_limit;
2180 av_cfg->sl = av->sl;
2181
2182 av_cfg->udp_sport = av->udp_sport;
2183 av_cfg->sgid_index = av->sgid_index;
2184
2185 ether_addr_copy(av_cfg->dmac, av->dmac);
2186 memcpy(av_cfg->dgid, av->dgid, ERDMA_ROCEV2_GID_SIZE);
2187 }
2188
erdma_create_ah(struct ib_ah * ibah,struct rdma_ah_init_attr * init_attr,struct ib_udata * udata)2189 int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2190 struct ib_udata *udata)
2191 {
2192 const struct ib_global_route *grh =
2193 rdma_ah_read_grh(init_attr->ah_attr);
2194 struct erdma_dev *dev = to_edev(ibah->device);
2195 struct erdma_pd *pd = to_epd(ibah->pd);
2196 struct erdma_ah *ah = to_eah(ibah);
2197 struct erdma_cmdq_create_ah_req req;
2198 u32 udp_sport;
2199 int ret;
2200
2201 ret = erdma_check_gid_attr(grh->sgid_attr);
2202 if (ret)
2203 return ret;
2204
2205 ret = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_AH]);
2206 if (ret < 0)
2207 return ret;
2208
2209 ah->ahn = ret;
2210
2211 if (grh->flow_label)
2212 udp_sport = rdma_flow_label_to_udp_sport(grh->flow_label);
2213 else
2214 udp_sport =
2215 IB_ROCE_UDP_ENCAP_VALID_PORT_MIN + (ah->ahn & 0x3FFF);
2216
2217 erdma_attr_to_av(init_attr->ah_attr, &ah->av, udp_sport);
2218
2219 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2220 CMDQ_OPCODE_CREATE_AH);
2221
2222 req.pdn = pd->pdn;
2223 req.ahn = ah->ahn;
2224 erdma_set_av_cfg(&req.av_cfg, &ah->av);
2225
2226 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2227 init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
2228 if (ret) {
2229 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2230 return ret;
2231 }
2232
2233 return 0;
2234 }
2235
erdma_destroy_ah(struct ib_ah * ibah,u32 flags)2236 int erdma_destroy_ah(struct ib_ah *ibah, u32 flags)
2237 {
2238 struct erdma_dev *dev = to_edev(ibah->device);
2239 struct erdma_pd *pd = to_epd(ibah->pd);
2240 struct erdma_ah *ah = to_eah(ibah);
2241 struct erdma_cmdq_destroy_ah_req req;
2242 int ret;
2243
2244 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2245 CMDQ_OPCODE_DESTROY_AH);
2246
2247 req.pdn = pd->pdn;
2248 req.ahn = ah->ahn;
2249
2250 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2251 flags & RDMA_DESTROY_AH_SLEEPABLE);
2252 if (ret)
2253 return ret;
2254
2255 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2256
2257 return 0;
2258 }
2259
erdma_query_ah(struct ib_ah * ibah,struct rdma_ah_attr * ah_attr)2260 int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
2261 {
2262 struct erdma_ah *ah = to_eah(ibah);
2263
2264 memset(ah_attr, 0, sizeof(*ah_attr));
2265 erdma_av_to_attr(&ah->av, ah_attr);
2266
2267 return 0;
2268 }
2269