1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2018 - 2026 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "irdma_main.h"
36
37 #define IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
38
kc_rdma_flow_label_to_udp_sport(u32 fl)39 static u16 kc_rdma_flow_label_to_udp_sport(u32 fl) {
40 u32 fl_low = fl & 0x03FFF;
41 u32 fl_high = fl & 0xFC000;
42
43 fl_low ^= fl_high >> 14;
44
45 return (u16)(fl_low | IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN);
46 }
47
48 #define IRDMA_GRH_FLOWLABEL_MASK (0x000FFFFF)
49
kc_rdma_calc_flow_label(u32 lqpn,u32 rqpn)50 static u32 kc_rdma_calc_flow_label(u32 lqpn, u32 rqpn) {
51 u64 fl = (u64)lqpn * rqpn;
52
53 fl ^= fl >> 20;
54 fl ^= fl >> 40;
55
56 return (u32)(fl & IRDMA_GRH_FLOWLABEL_MASK);
57 }
58
59 u16
kc_rdma_get_udp_sport(u32 fl,u32 lqpn,u32 rqpn)60 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
61 {
62 if (!fl)
63 fl = kc_rdma_calc_flow_label(lqpn, rqpn);
64 return kc_rdma_flow_label_to_udp_sport(fl);
65 }
66
67 void
irdma_get_dev_fw_str(struct ib_device * dev,char * str,size_t str_len)68 irdma_get_dev_fw_str(struct ib_device *dev,
69 char *str,
70 size_t str_len)
71 {
72 struct irdma_device *iwdev = to_iwdev(dev);
73
74 snprintf(str, str_len, "%u.%u",
75 irdma_fw_major_ver(&iwdev->rf->sc_dev),
76 irdma_fw_minor_ver(&iwdev->rf->sc_dev));
77 }
78
79 int
irdma_add_gid(struct ib_device * device,u8 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr,void ** context)80 irdma_add_gid(struct ib_device *device,
81 u8 port_num,
82 unsigned int index,
83 const union ib_gid *gid,
84 const struct ib_gid_attr *attr,
85 void **context)
86 {
87 return 0;
88 }
89
90 int
irdma_del_gid(struct ib_device * device,u8 port_num,unsigned int index,void ** context)91 irdma_del_gid(struct ib_device *device,
92 u8 port_num,
93 unsigned int index,
94 void **context)
95 {
96 return 0;
97 }
98
99 /**
100 * irdma_alloc_mr - register stag for fast memory registration
101 * @pd: ibpd pointer
102 * @mr_type: memory for stag registrion
103 * @max_num_sg: man number of pages
104 * @udata: user data
105 */
106 struct ib_mr *
irdma_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_num_sg,struct ib_udata * udata)107 irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
108 u32 max_num_sg, struct ib_udata *udata)
109 {
110 struct irdma_device *iwdev = to_iwdev(pd->device);
111 struct irdma_pble_alloc *palloc;
112 struct irdma_pbl *iwpbl;
113 struct irdma_mr *iwmr;
114 int status;
115 u32 stag;
116 int err_code = -ENOMEM;
117
118 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
119 if (!iwmr)
120 return ERR_PTR(-ENOMEM);
121
122 stag = irdma_create_stag(iwdev);
123 if (!stag) {
124 err_code = -ENOMEM;
125 goto err;
126 }
127
128 iwmr->stag = stag;
129 iwmr->ibmr.rkey = stag;
130 iwmr->ibmr.lkey = stag;
131 iwmr->ibmr.pd = pd;
132 iwmr->ibmr.device = pd->device;
133 iwpbl = &iwmr->iwpbl;
134 iwpbl->iwmr = iwmr;
135 iwmr->type = IRDMA_MEMREG_TYPE_MEM;
136 palloc = &iwpbl->pble_alloc;
137 iwmr->page_cnt = max_num_sg;
138 /* Assume system PAGE_SIZE as the sg page sizes are unknown. */
139 iwmr->len = max_num_sg * PAGE_SIZE;
140 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
141 false);
142 if (status)
143 goto err_get_pble;
144
145 err_code = irdma_hw_alloc_stag(iwdev, iwmr);
146 if (err_code)
147 goto err_alloc_stag;
148
149 iwpbl->pbl_allocated = true;
150
151 return &iwmr->ibmr;
152 err_alloc_stag:
153 irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
154 err_get_pble:
155 irdma_free_stag(iwdev, stag);
156 err:
157 kfree(iwmr);
158
159 return ERR_PTR(err_code);
160 }
161
162 #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
163 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
164 /**
165 * irdma_alloc_ucontext - Allocate the user context data structure
166 * @uctx: context
167 * @udata: user data
168 *
169 * This keeps track of all objects associated with a particular
170 * user-mode client.
171 */
172 int
irdma_alloc_ucontext(struct ib_ucontext * uctx,struct ib_udata * udata)173 irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
174 {
175 struct ib_device *ibdev = uctx->device;
176 struct irdma_device *iwdev = to_iwdev(ibdev);
177 struct irdma_alloc_ucontext_req req = {0};
178 struct irdma_alloc_ucontext_resp uresp = {0};
179 struct irdma_ucontext *ucontext = to_ucontext(uctx);
180 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
181
182 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||
183 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN)
184 return -EINVAL;
185
186 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)))
187 return -EINVAL;
188
189 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER)
190 goto ver_error;
191
192 ucontext->iwdev = iwdev;
193 ucontext->abi_ver = req.userspace_ver;
194
195 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR)
196 ucontext->use_raw_attrs = true;
197
198 /* GEN_1 support for libi40iw */
199 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {
200 if (uk_attrs->hw_rev != IRDMA_GEN_1)
201 return -EOPNOTSUPP;
202
203 ucontext->legacy_mode = true;
204 uresp.max_qps = iwdev->rf->max_qp;
205 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
206 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
207 uresp.kernel_ver = req.userspace_ver;
208 if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)))
209 return -EFAULT;
210 } else {
211 u64 bar_off;
212
213 uresp.kernel_ver = IRDMA_ABI_VER;
214 uresp.feature_flags = uk_attrs->feature_flags;
215 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags;
216 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges;
217 uresp.max_hw_inline = uk_attrs->max_hw_inline;
218 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta;
219 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta;
220 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk;
221 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size;
222 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
223 uresp.hw_rev = uk_attrs->hw_rev;
224 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
225 uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
226 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
227
228 bar_off =
229 (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
230 ucontext->db_mmap_entry =
231 irdma_user_mmap_entry_insert(ucontext, bar_off,
232 IRDMA_MMAP_IO_NC,
233 &uresp.db_mmap_key);
234 if (!ucontext->db_mmap_entry) {
235 return -ENOMEM;
236 }
237
238 if (ib_copy_to_udata(udata, &uresp,
239 min(sizeof(uresp), udata->outlen))) {
240 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
241 return -EFAULT;
242 }
243 }
244
245 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
246 spin_lock_init(&ucontext->cq_reg_mem_list_lock);
247 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
248 spin_lock_init(&ucontext->qp_reg_mem_list_lock);
249 INIT_LIST_HEAD(&ucontext->vma_list);
250 mutex_init(&ucontext->vma_list_mutex);
251
252 return 0;
253
254 ver_error:
255 irdma_dev_err(&iwdev->ibdev,
256 "Invalid userspace driver version detected. Detected version %d, should be %d\n",
257 req.userspace_ver, IRDMA_ABI_VER);
258 return -EINVAL;
259 }
260
261
262 /**
263 * irdma_dealloc_ucontext - deallocate the user context data structure
264 * @context: user context created during alloc
265 */
266 void
irdma_dealloc_ucontext(struct ib_ucontext * context)267 irdma_dealloc_ucontext(struct ib_ucontext *context)
268 {
269 struct irdma_ucontext *ucontext = to_ucontext(context);
270
271 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
272
273 return;
274 }
275
276
277 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
278 /**
279 * irdma_alloc_pd - allocate protection domain
280 * @pd: protection domain
281 * @udata: user data
282 */
283 int
irdma_alloc_pd(struct ib_pd * pd,struct ib_udata * udata)284 irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
285 {
286 struct irdma_pd *iwpd = to_iwpd(pd);
287 struct irdma_device *iwdev = to_iwdev(pd->device);
288 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
289 struct irdma_pci_f *rf = iwdev->rf;
290 struct irdma_alloc_pd_resp uresp = {0};
291 struct irdma_sc_pd *sc_pd;
292 u32 pd_id = 0;
293 int err;
294
295 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN)
296 return -EINVAL;
297
298 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id,
299 &rf->next_pd);
300 if (err)
301 return err;
302
303 sc_pd = &iwpd->sc_pd;
304 if (udata) {
305 struct irdma_ucontext *ucontext =
306 rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
307
308 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
309 uresp.pd_id = pd_id;
310 if (ib_copy_to_udata(udata, &uresp,
311 min(sizeof(uresp), udata->outlen))) {
312 err = -EFAULT;
313 goto error;
314 }
315 } else {
316 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);
317 }
318
319 spin_lock_init(&iwpd->udqp_list_lock);
320 INIT_LIST_HEAD(&iwpd->udqp_list);
321
322 return 0;
323
324 error:
325
326 irdma_free_rsrc(rf, rf->allocated_pds, pd_id);
327
328 return err;
329 }
330
331
332 void
irdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)333 irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
334 {
335 struct irdma_pd *iwpd = to_iwpd(ibpd);
336 struct irdma_device *iwdev = to_iwdev(ibpd->device);
337
338 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
339 }
340
341
342
343 /**
344 * irdma_find_qp_update_qs - update QS handle for UD QPs
345 * @rf: RDMA PCI function
346 * @pd: protection domain object
347 * @user_pri: selected user priority
348 */
349 static void
irdma_find_qp_update_qs(struct irdma_pci_f * rf,struct irdma_pd * pd,u8 user_pri)350 irdma_find_qp_update_qs(struct irdma_pci_f *rf,
351 struct irdma_pd *pd, u8 user_pri)
352 {
353 struct irdma_qp *iwqp;
354 struct list_head *tmp_node, *list_node;
355 struct irdma_udqs_work *work;
356 unsigned long flags;
357 bool qs_change;
358
359 spin_lock_irqsave(&pd->udqp_list_lock, flags);
360 list_for_each_safe(list_node, tmp_node, &pd->udqp_list) {
361 qs_change = true;
362 iwqp = list_entry(list_node, struct irdma_qp, ud_list_elem);
363 irdma_qp_add_ref(&iwqp->ibqp);
364 /* check if qs_handle needs to be changed */
365 if (iwqp->sc_qp.qs_handle == iwqp->sc_qp.vsi->qos[user_pri].qs_handle) {
366 if (iwqp->ctx_info.user_pri == user_pri) {
367 /* qs_handle and user_pri don't change */
368 irdma_qp_rem_ref(&iwqp->ibqp);
369 continue;
370 }
371 qs_change = false;
372 }
373 /* perform qp qos change */
374 work = kzalloc(sizeof(*work), GFP_ATOMIC);
375 if (!work) {
376 irdma_qp_rem_ref(&iwqp->ibqp);
377 spin_unlock_irqrestore(&pd->udqp_list_lock, flags);
378 return;
379 }
380 work->iwqp = iwqp;
381 work->user_prio = user_pri;
382 work->qs_change = qs_change;
383 INIT_WORK(&work->work, irdma_udqp_qs_worker);
384 if (qs_change)
385 irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND);
386 queue_work(rf->iwdev->cleanup_wq, &work->work);
387 }
388 spin_unlock_irqrestore(&pd->udqp_list_lock, flags);
389 }
390
391 static void
irdma_fill_ah_info(struct vnet * vnet,struct irdma_ah_info * ah_info,const struct ib_gid_attr * sgid_attr,union irdma_sockaddr * sgid_addr,union irdma_sockaddr * dgid_addr,u8 * dmac,u8 net_type)392 irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info,
393 const struct ib_gid_attr *sgid_attr,
394 union irdma_sockaddr *sgid_addr,
395 union irdma_sockaddr *dgid_addr,
396 u8 *dmac, u8 net_type)
397 {
398 if (net_type == RDMA_NETWORK_IPV4) {
399 ah_info->ipv4_valid = true;
400 ah_info->dest_ip_addr[0] =
401 ntohl(dgid_addr->saddr_in.sin_addr.s_addr);
402 ah_info->src_ip_addr[0] =
403 ntohl(sgid_addr->saddr_in.sin_addr.s_addr);
404 CURVNET_SET_QUIET(vnet);
405 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0],
406 ah_info->dest_ip_addr[0]);
407 CURVNET_RESTORE();
408 if (ipv4_is_multicast(dgid_addr->saddr_in.sin_addr.s_addr)) {
409 irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac);
410 }
411 } else {
412 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
413 dgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
414 irdma_copy_ip_ntohl(ah_info->src_ip_addr,
415 sgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
416 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr,
417 ah_info->dest_ip_addr);
418 if (rdma_is_multicast_addr(&dgid_addr->saddr_in6.sin6_addr)) {
419 irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac);
420 }
421 }
422 }
423
irdma_roce_get_vlan_prio(if_t ndev,u8 prio)424 static inline u8 irdma_roce_get_vlan_prio(if_t ndev, u8 prio)
425 {
426 return prio;
427 }
428
429 static int
irdma_create_ah_vlan_tag(struct irdma_device * iwdev,struct irdma_pd * pd,struct irdma_ah_info * ah_info,const struct ib_gid_attr * sgid_attr,u8 * dmac)430 irdma_create_ah_vlan_tag(struct irdma_device *iwdev,
431 struct irdma_pd *pd,
432 struct irdma_ah_info *ah_info,
433 const struct ib_gid_attr *sgid_attr,
434 u8 *dmac)
435 {
436 u16 vlan_prio;
437
438 if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev))
439 ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev);
440 else
441 ah_info->vlan_tag = VLAN_N_VID;
442
443 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac);
444
445 if (ah_info->dst_arpindex == -1)
446 return -EINVAL;
447
448 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode)
449 ah_info->vlan_tag = 0;
450
451 if (ah_info->vlan_tag < VLAN_N_VID) {
452 ah_info->insert_vlan_tag = true;
453 vlan_prio = (u16)irdma_roce_get_vlan_prio(sgid_attr->ndev,
454 rt_tos2priority(ah_info->tc_tos));
455 ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT;
456 irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio);
457 }
458 if (iwdev->roce_dcqcn_en) {
459 ah_info->tc_tos &= ~ECN_CODE_PT_MASK;
460 ah_info->tc_tos |= ECN_CODE_PT_VAL;
461 }
462
463 return 0;
464 }
465
466 static int
irdma_create_ah_wait(struct irdma_pci_f * rf,struct irdma_sc_ah * sc_ah,bool sleep)467 irdma_create_ah_wait(struct irdma_pci_f *rf,
468 struct irdma_sc_ah *sc_ah, bool sleep)
469 {
470 int ret;
471
472 if (!sleep) {
473 bool timeout = false;
474 u64 start = get_jiffies_64();
475 u64 completed_ops = atomic64_read(&rf->sc_dev.cqp->completed_ops);
476 struct irdma_cqp_request *cqp_request =
477 sc_ah->ah_info.cqp_request;
478 const u64 timeout_jiffies =
479 msecs_to_jiffies(rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms *
480 CQP_TIMEOUT_THRESHOLD);
481
482 /*
483 * NOTE: irdma_check_cqp_progress is not used here because it relies on a notion of a cycle count, but
484 * we want to avoid unnecessary delays. We are in an atomic context here, so we might as well check in
485 * a tight loop.
486 */
487 while (!READ_ONCE(cqp_request->request_done)) {
488 u64 tmp;
489 u64 curr_jiffies;
490
491 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
492
493 curr_jiffies = get_jiffies_64();
494 tmp = atomic64_read(&rf->sc_dev.cqp->completed_ops);
495 if (tmp != completed_ops) {
496 /* CQP is progressing. Reset timer. */
497 completed_ops = tmp;
498 start = curr_jiffies;
499 }
500
501 if ((curr_jiffies - start) > timeout_jiffies) {
502 timeout = true;
503 break;
504 }
505 }
506
507 if (!timeout && !cqp_request->compl_info.op_ret_val) {
508 irdma_put_cqp_request(&rf->cqp, cqp_request);
509 sc_ah->ah_info.ah_valid = true;
510 } else {
511 ret = timeout ? -ETIMEDOUT : -EINVAL;
512 irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d",
513 ret, cqp_request->compl_info.op_ret_val);
514 irdma_put_cqp_request(&rf->cqp, cqp_request);
515 if (timeout && !rf->reset) {
516 rf->reset = true;
517 rf->gen_ops.request_reset(rf);
518 }
519 return ret;
520 }
521 }
522
523 return 0;
524 }
525
526 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
527
528 static int
irdma_create_sleepable_ah(struct ib_ah * ib_ah,struct ib_ah_attr * attr,u32 flags,struct ib_udata * udata)529 irdma_create_sleepable_ah(struct ib_ah *ib_ah,
530 struct ib_ah_attr *attr, u32 flags,
531 struct ib_udata *udata)
532 {
533 struct irdma_pd *pd = to_iwpd(ib_ah->pd);
534 struct irdma_ah *ah = container_of(ib_ah, struct irdma_ah, ibah);
535 struct irdma_device *iwdev = to_iwdev(ib_ah->pd->device);
536 union ib_gid sgid;
537 struct ib_gid_attr sgid_attr;
538 struct irdma_pci_f *rf = iwdev->rf;
539 struct irdma_sc_ah *sc_ah;
540 u32 ah_id = 0;
541 struct irdma_ah_info *ah_info;
542 struct irdma_create_ah_resp uresp = {};
543 union irdma_sockaddr sgid_addr, dgid_addr;
544 int err;
545 u8 dmac[ETHER_ADDR_LEN];
546 bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0;
547
548 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
549 return -EINVAL;
550
551 err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
552 rf->max_ah, &ah_id, &rf->next_ah);
553
554 if (err)
555 return err;
556
557 ah->pd = pd;
558 sc_ah = &ah->sc_ah;
559 sc_ah->ah_info.ah_idx = ah_id;
560 sc_ah->ah_info.vsi = &iwdev->vsi;
561 irdma_sc_init_ah(&rf->sc_dev, sc_ah);
562 ah->sgid_index = attr->grh.sgid_index;
563 memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid));
564 rcu_read_lock();
565 err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num,
566 attr->grh.sgid_index, &sgid, &sgid_attr);
567 rcu_read_unlock();
568 if (err) {
569 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
570 "GID lookup at idx=%d with port=%d failed\n",
571 attr->grh.sgid_index, attr->port_num);
572 err = -EINVAL;
573 goto err_gid_l2;
574 }
575 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid);
576 rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
577 ah->av.attrs = *attr;
578 ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
579
580 if (sgid_attr.ndev)
581 dev_put(sgid_attr.ndev);
582
583 ah_info = &sc_ah->ah_info;
584 ah_info->ah_idx = ah_id;
585 ah_info->pd_idx = pd->sc_pd.pd_id;
586 ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev));
587
588 if (attr->ah_flags & IB_AH_GRH) {
589 ah_info->flow_label = attr->grh.flow_label;
590 ah_info->hop_ttl = attr->grh.hop_limit;
591 ah_info->tc_tos = attr->grh.traffic_class;
592 }
593
594 ether_addr_copy(dmac, attr->dmac);
595
596 irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr,
597 dmac, ah->av.net_type);
598
599 err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac);
600 if (err)
601 goto err_gid_l2;
602
603 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
604 sleep, NULL, sc_ah);
605 if (err) {
606 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP-OP Create AH fail");
607 goto err_gid_l2;
608 }
609
610 err = irdma_create_ah_wait(rf, sc_ah, sleep);
611 if (err)
612 goto err_gid_l2;
613
614 if (udata) {
615 uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
616 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
617 if (err) {
618 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
619 IRDMA_OP_AH_DESTROY, false, NULL, ah);
620 goto err_gid_l2;
621 }
622 }
623
624 return 0;
625 err_gid_l2:
626 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
627
628 return err;
629 }
630
631 /**
632 * irdma_create_ah - create address handle
633 * @ib_ah: ptr to AH
634 * @attr: address handle attributes
635 * @flags: AH flags to wait
636 * @udata: user data
637 *
638 * returns 0 on success, error otherwise
639 */
640 int
irdma_create_ah(struct ib_ah * ib_ah,struct ib_ah_attr * attr,u32 flags,struct ib_udata * udata)641 irdma_create_ah(struct ib_ah *ib_ah,
642 struct ib_ah_attr *attr, u32 flags,
643 struct ib_udata *udata)
644 {
645 return irdma_create_sleepable_ah(ib_ah, attr, flags, udata);
646 }
647
648 void
irdma_ether_copy(u8 * dmac,struct ib_ah_attr * attr)649 irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr)
650 {
651 ether_addr_copy(dmac, attr->dmac);
652 }
653
654 int
irdma_create_ah_stub(struct ib_ah * ib_ah,struct ib_ah_attr * attr,u32 flags,struct ib_udata * udata)655 irdma_create_ah_stub(struct ib_ah *ib_ah,
656 struct ib_ah_attr *attr, u32 flags,
657 struct ib_udata *udata)
658 {
659 return -ENOSYS;
660 }
661
662 void
irdma_destroy_ah_stub(struct ib_ah * ibah,u32 flags)663 irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags)
664 {
665 return;
666 }
667
668
669 /**
670 * irdma_free_qp_rsrc - free up memory resources for qp
671 * @iwqp: qp ptr (user or kernel)
672 */
673 void
irdma_free_qp_rsrc(struct irdma_qp * iwqp)674 irdma_free_qp_rsrc(struct irdma_qp *iwqp)
675 {
676 struct irdma_device *iwdev = iwqp->iwdev;
677 struct irdma_pci_f *rf = iwdev->rf;
678 u32 qp_num = iwqp->ibqp.qp_num;
679
680 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
681 irdma_dealloc_push_page(rf, iwqp);
682 if (iwqp->sc_qp.vsi) {
683 irdma_qp_rem_qos(&iwqp->sc_qp);
684 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
685 iwqp->sc_qp.user_pri);
686 }
687
688 if (qp_num > 2)
689 irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
690 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem);
691 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem);
692 kfree(iwqp->kqp.sig_trk_mem);
693 iwqp->kqp.sig_trk_mem = NULL;
694 kfree(iwqp->kqp.sq_wrid_mem);
695 kfree(iwqp->kqp.rq_wrid_mem);
696 kfree(iwqp->sg_list);
697 kfree(iwqp);
698 }
699
700 /**
701 * irdma_create_qp - create qp
702 * @ibpd: ptr of pd
703 * @init_attr: attributes for qp
704 * @udata: user data for create qp
705 */
706 struct ib_qp *
irdma_create_qp(struct ib_pd * ibpd,struct ib_qp_init_attr * init_attr,struct ib_udata * udata)707 irdma_create_qp(struct ib_pd *ibpd,
708 struct ib_qp_init_attr *init_attr,
709 struct ib_udata *udata)
710 {
711 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx)
712 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd)
713 struct irdma_pd *iwpd = to_iwpd(ibpd);
714 struct irdma_device *iwdev = to_iwdev(ibpd->device);
715 struct irdma_pci_f *rf = iwdev->rf;
716 struct irdma_qp *iwqp;
717 struct irdma_create_qp_resp uresp = {0};
718 u32 qp_num = 0;
719 int ret;
720 int err_code;
721 struct irdma_sc_qp *qp;
722 struct irdma_sc_dev *dev = &rf->sc_dev;
723 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs;
724 struct irdma_qp_init_info init_info = {{0}};
725 struct irdma_qp_host_ctx_info *ctx_info;
726 u32 next_qp = 0;
727 unsigned long flags;
728
729 err_code = irdma_validate_qp_attrs(init_attr, iwdev);
730 if (err_code)
731 return ERR_PTR(err_code);
732
733 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN ||
734 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN))
735 return ERR_PTR(-EINVAL);
736
737 init_info.vsi = &iwdev->vsi;
738 init_info.qp_uk_init_info.uk_attrs = uk_attrs;
739 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr;
740 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr;
741 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
742 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
743 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
744
745 iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL);
746 if (!iwqp)
747 return ERR_PTR(-ENOMEM);
748
749 iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list),
750 GFP_KERNEL);
751 if (!iwqp->sg_list) {
752 kfree(iwqp);
753 return ERR_PTR(-ENOMEM);
754 }
755
756 qp = &iwqp->sc_qp;
757 qp->qp_uk.back_qp = iwqp;
758 qp->qp_uk.lock = &iwqp->lock;
759 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
760
761 iwqp->iwdev = iwdev;
762 iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE;
763 iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem,
764 iwqp->q2_ctx_mem.size,
765 256);
766 if (!iwqp->q2_ctx_mem.va) {
767 kfree(iwqp->sg_list);
768 kfree(iwqp);
769 return ERR_PTR(-ENOMEM);
770 }
771
772 init_info.q2 = iwqp->q2_ctx_mem.va;
773 init_info.q2_pa = iwqp->q2_ctx_mem.pa;
774 init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE);
775 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE;
776
777 if (init_attr->qp_type == IB_QPT_GSI)
778 qp_num = 1;
779 else if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2)
780 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp,
781 &qp_num, &next_qp);
782 else
783 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp,
784 &qp_num, &rf->next_qp);
785 if (err_code)
786 goto error;
787
788 iwqp->iwpd = iwpd;
789 iwqp->ibqp.qp_num = qp_num;
790 qp = &iwqp->sc_qp;
791 iwqp->iwscq = to_iwcq(init_attr->send_cq);
792 iwqp->iwrcq = to_iwcq(init_attr->recv_cq);
793 iwqp->host_ctx.va = init_info.host_ctx;
794 iwqp->host_ctx.pa = init_info.host_ctx_pa;
795 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE;
796
797 init_info.pd = &iwpd->sc_pd;
798 init_info.qp_uk_init_info.qp_id = qp_num;
799 if (!rdma_protocol_roce(&iwdev->ibdev, 1))
800 init_info.qp_uk_init_info.first_sq_wq = 1;
801 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
802 init_waitqueue_head(&iwqp->waitq);
803 init_waitqueue_head(&iwqp->mod_qp_waitq);
804
805 spin_lock_init(&iwqp->dwork_flush_lock);
806
807 if (udata) {
808 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_user_flush_worker);
809 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver;
810 err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr);
811 } else {
812 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_kern_flush_worker);
813 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER;
814 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr);
815 }
816
817 if (err_code) {
818 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "setup qp failed\n");
819 goto error;
820 }
821
822 if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
823 if (init_attr->qp_type == IB_QPT_RC) {
824 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC;
825 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
826 IRDMA_WRITE_WITH_IMM |
827 IRDMA_ROCE;
828 } else {
829 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD;
830 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
831 IRDMA_ROCE;
832 }
833 } else {
834 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP;
835 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM;
836 }
837
838 ret = irdma_sc_qp_init(qp, &init_info);
839 if (ret) {
840 err_code = -EPROTO;
841 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_init fail\n");
842 goto error;
843 }
844
845 ctx_info = &iwqp->ctx_info;
846 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
847 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
848
849 if (rdma_protocol_roce(&iwdev->ibdev, 1))
850 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info);
851 else
852 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info);
853
854 err_code = irdma_cqp_create_qp_cmd(iwqp);
855 if (err_code)
856 goto error;
857
858 atomic_set(&iwqp->refcnt, 1);
859 spin_lock_init(&iwqp->lock);
860 spin_lock_init(&iwqp->sc_qp.pfpdu.lock);
861 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
862 rf->qp_table[qp_num] = iwqp;
863
864 if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
865 if (dev->ws_add(&iwdev->vsi, 0)) {
866 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp);
867 err_code = -EINVAL;
868 goto error;
869 }
870
871 irdma_qp_add_qos(&iwqp->sc_qp);
872 spin_lock_irqsave(&iwpd->udqp_list_lock, flags);
873 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD)
874 list_add_tail(&iwqp->ud_list_elem, &iwpd->udqp_list);
875 spin_unlock_irqrestore(&iwpd->udqp_list_lock, flags);
876 }
877
878 if (udata) {
879 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
880 if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) {
881 uresp.lsmm = 1;
882 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;
883 } else {
884 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) {
885 uresp.lsmm = 1;
886 if (qp->qp_uk.start_wqe_idx) {
887 uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
888 uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx;
889 }
890 }
891 }
892 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
893 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
894 uresp.qp_id = qp_num;
895 uresp.qp_caps = qp->qp_uk.qp_caps;
896
897 err_code = ib_copy_to_udata(udata, &uresp,
898 min(sizeof(uresp), udata->outlen));
899 if (err_code) {
900 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n");
901 irdma_destroy_qp(&iwqp->ibqp, udata);
902 return ERR_PTR(err_code);
903 }
904 }
905
906 init_completion(&iwqp->free_qp);
907 return &iwqp->ibqp;
908
909 error:
910 irdma_free_qp_rsrc(iwqp);
911
912 return ERR_PTR(err_code);
913 }
914
915 /**
916 * irdma_destroy_qp - destroy qp
917 * @ibqp: qp's ib pointer also to get to device's qp address
918 * @udata: user data
919 */
920 int
irdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)921 irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
922 {
923 struct irdma_qp *iwqp = to_iwqp(ibqp);
924 struct irdma_device *iwdev = iwqp->iwdev;
925 unsigned long flags;
926
927 if (iwqp->sc_qp.qp_uk.destroy_pending)
928 goto free_rsrc;
929 iwqp->sc_qp.qp_uk.destroy_pending = true;
930
931 spin_lock_irqsave(&iwqp->iwpd->udqp_list_lock, flags);
932 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD)
933 list_del(&iwqp->ud_list_elem);
934 spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags);
935
936 if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE)
937 irdma_modify_qp_to_err(&iwqp->sc_qp);
938
939 if (!iwqp->user_mode) {
940 if (iwqp->iwscq) {
941 irdma_clean_cqes(iwqp, iwqp->iwscq);
942 if (iwqp->iwrcq != iwqp->iwscq)
943 irdma_clean_cqes(iwqp, iwqp->iwrcq);
944 }
945 }
946 irdma_qp_rem_ref(&iwqp->ibqp);
947 wait_for_completion(&iwqp->free_qp);
948 irdma_free_lsmm_rsrc(iwqp);
949 if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp))
950 return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE;
951 free_rsrc:
952 irdma_remove_push_mmap_entries(iwqp);
953 irdma_free_qp_rsrc(iwqp);
954
955 return 0;
956 }
957
958 /**
959 * irdma_create_cq - create cq
960 * @ibcq: CQ allocated
961 * @attr: attributes for cq
962 * @udata: user data
963 */
964 int
irdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)965 irdma_create_cq(struct ib_cq *ibcq,
966 const struct ib_cq_init_attr *attr,
967 struct ib_udata *udata)
968 {
969 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)
970 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)
971 struct ib_device *ibdev = ibcq->device;
972 struct irdma_device *iwdev = to_iwdev(ibdev);
973 struct irdma_pci_f *rf = iwdev->rf;
974 struct irdma_cq *iwcq = to_iwcq(ibcq);
975 u32 cq_num = 0;
976 struct irdma_sc_cq *cq;
977 struct irdma_sc_dev *dev = &rf->sc_dev;
978 struct irdma_cq_init_info info = {0};
979 int status;
980 struct irdma_cqp_request *cqp_request;
981 struct cqp_cmds_info *cqp_info;
982 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
983 unsigned long flags;
984 int err_code;
985 int entries = attr->cqe;
986
987 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
988 if (err_code)
989 return err_code;
990
991 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
992 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
993 return -EINVAL;
994 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num,
995 &rf->next_cq);
996 if (err_code)
997 return err_code;
998 cq = &iwcq->sc_cq;
999 cq->back_cq = iwcq;
1000 atomic_set(&iwcq->refcnt, 1);
1001 spin_lock_init(&iwcq->lock);
1002 INIT_LIST_HEAD(&iwcq->resize_list);
1003 INIT_LIST_HEAD(&iwcq->cmpl_generated);
1004 info.dev = dev;
1005 ukinfo->cq_size = max_t(int, entries, 4);
1006 ukinfo->cq_id = cq_num;
1007 iwcq->cq_num = cq_num;
1008 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
1009 atomic_set(&iwcq->armed, 0);
1010 if (attr->comp_vector < rf->ceqs_count)
1011 info.ceq_id = attr->comp_vector;
1012 info.ceq_id_valid = true;
1013 info.ceqe_mask = 1;
1014 info.type = IRDMA_CQ_TYPE_IWARP;
1015 info.vsi = &iwdev->vsi;
1016
1017 if (udata) {
1018 struct irdma_ucontext *ucontext;
1019 struct irdma_create_cq_req req = {0};
1020 struct irdma_cq_mr *cqmr;
1021 struct irdma_pbl *iwpbl;
1022 struct irdma_pbl *iwpbl_shadow;
1023 struct irdma_cq_mr *cqmr_shadow;
1024
1025 iwcq->user_mode = true;
1026 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
1027
1028 if (ib_copy_from_udata(&req, udata,
1029 min(sizeof(req), udata->inlen))) {
1030 err_code = -EFAULT;
1031 goto cq_free_rsrc;
1032 }
1033
1034 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1035 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
1036 &ucontext->cq_reg_mem_list);
1037 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1038 if (!iwpbl) {
1039 err_code = -EPROTO;
1040 goto cq_free_rsrc;
1041 }
1042 cqmr = &iwpbl->cq_mr;
1043
1044 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
1045 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) {
1046 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1047 iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area,
1048 &ucontext->cq_reg_mem_list);
1049 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1050
1051 if (!iwpbl_shadow) {
1052 err_code = -EPROTO;
1053 goto cq_free_rsrc;
1054 }
1055 cqmr_shadow = &iwpbl_shadow->cq_mr;
1056 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr;
1057 cqmr->split = true;
1058 } else {
1059 info.shadow_area_pa = cqmr->shadow;
1060 }
1061 if (iwpbl->pbl_allocated) {
1062 info.virtual_map = true;
1063 info.pbl_chunk_size = 1;
1064 info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
1065 } else {
1066 info.cq_base_pa = cqmr->cq_pbl.addr;
1067 }
1068 } else {
1069 /* Kmode allocations */
1070 int rsize;
1071
1072 if (entries < 1 || entries > rf->max_cqe) {
1073 err_code = -EINVAL;
1074 goto cq_free_rsrc;
1075 }
1076
1077 entries++;
1078 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2)
1079 entries *= 2;
1080 ukinfo->cq_size = entries;
1081
1082 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
1083 iwcq->kmem.size = round_up(rsize, IRDMA_HW_PAGE_SIZE);
1084 iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem,
1085 iwcq->kmem.size, IRDMA_HW_PAGE_SIZE);
1086 if (!iwcq->kmem.va) {
1087 err_code = -ENOMEM;
1088 goto cq_free_rsrc;
1089 }
1090
1091 iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3;
1092 iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw,
1093 &iwcq->kmem_shadow,
1094 iwcq->kmem_shadow.size,
1095 64);
1096
1097 if (!iwcq->kmem_shadow.va) {
1098 err_code = -ENOMEM;
1099 goto cq_kmem_free;
1100 }
1101 info.shadow_area_pa = iwcq->kmem_shadow.pa;
1102 ukinfo->shadow_area = iwcq->kmem_shadow.va;
1103 ukinfo->cq_base = iwcq->kmem.va;
1104 info.cq_base_pa = iwcq->kmem.pa;
1105 }
1106
1107 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,
1108 (u32)IRDMA_MAX_CQ_READ_THRESH);
1109 if (irdma_sc_cq_init(cq, &info)) {
1110 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n");
1111 err_code = -EPROTO;
1112 goto cq_kmem_free;
1113 }
1114
1115 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1116 if (!cqp_request) {
1117 err_code = -ENOMEM;
1118 goto cq_kmem_free;
1119 }
1120 cqp_info = &cqp_request->info;
1121 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1122 cqp_info->post_sq = 1;
1123 cqp_info->in.u.cq_create.cq = cq;
1124 cqp_info->in.u.cq_create.check_overflow = true;
1125 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1126 cqp_info->create = true;
1127 status = irdma_handle_cqp_op(rf, cqp_request);
1128 irdma_put_cqp_request(&rf->cqp, cqp_request);
1129 if (status) {
1130 err_code = -ENOMEM;
1131 goto cq_kmem_free;
1132 }
1133
1134 if (udata) {
1135 struct irdma_create_cq_resp resp = {0};
1136
1137 resp.cq_id = info.cq_uk_init_info.cq_id;
1138 resp.cq_size = info.cq_uk_init_info.cq_size;
1139 if (ib_copy_to_udata(udata, &resp,
1140 min(sizeof(resp), udata->outlen))) {
1141 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy to user data\n");
1142 err_code = -EPROTO;
1143 goto cq_destroy;
1144 }
1145 }
1146
1147 WRITE_ONCE(rf->cq_table[cq_num], iwcq);
1148 init_completion(&iwcq->free_cq);
1149
1150 return 0;
1151 cq_destroy:
1152 irdma_cq_wq_destroy(rf, cq);
1153 cq_kmem_free:
1154 if (!iwcq->user_mode) {
1155 irdma_free_dma_mem(dev->hw, &iwcq->kmem);
1156 irdma_free_dma_mem(dev->hw, &iwcq->kmem_shadow);
1157 }
1158 cq_free_rsrc:
1159 irdma_free_rsrc(rf, rf->allocated_cqs, cq_num);
1160 return err_code;
1161 }
1162
1163 /**
1164 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally
1165 * @iwmr: iwmr for IB's user page addresses
1166 * @pbl: ple pointer to save 1 level or 0 level pble
1167 * @level: indicated level 0, 1 or 2
1168 */
1169
1170 void
irdma_copy_user_pgaddrs(struct irdma_mr * iwmr,u64 * pbl,enum irdma_pble_level level)1171 irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
1172 enum irdma_pble_level level)
1173 {
1174 struct ib_umem *region = iwmr->region;
1175 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1176 int chunk_pages, entry, i;
1177 struct scatterlist *sg;
1178 u64 pg_addr = 0;
1179 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
1180 struct irdma_pble_info *pinfo;
1181 u32 idx = 0;
1182 u32 pbl_cnt = 0;
1183
1184 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf;
1185 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
1186 chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size);
1187 if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page)
1188 iwpbl->qp_mr.sq_page = sg_page(sg);
1189 for (i = 0; i < chunk_pages; i++) {
1190 pg_addr = sg_dma_address(sg) + (i * iwmr->page_size);
1191 if ((entry + i) == 0)
1192 *pbl = pg_addr & iwmr->page_msk;
1193 else if (!(pg_addr & ~iwmr->page_msk))
1194 *pbl = pg_addr;
1195 else
1196 continue;
1197 if (++pbl_cnt == palloc->total_cnt)
1198 break;
1199 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx);
1200 }
1201 }
1202 }
1203
1204 /**
1205 * irdma_destroy_ah - Destroy address handle
1206 * @ibah: pointer to address handle
1207 * @ah_flags: destroy flags
1208 */
1209
1210 void
irdma_destroy_ah(struct ib_ah * ibah,u32 ah_flags)1211 irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
1212 {
1213 struct irdma_device *iwdev = to_iwdev(ibah->device);
1214 struct irdma_ah *ah = to_iwah(ibah);
1215
1216 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,
1217 false, NULL, ah);
1218
1219 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
1220 ah->sc_ah.ah_info.ah_idx);
1221 }
1222
1223
1224 int
irdma_dereg_mr(struct ib_mr * ib_mr,struct ib_udata * udata)1225 irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
1226 {
1227 struct irdma_mr *iwmr = to_iwmr(ib_mr);
1228 struct irdma_device *iwdev = to_iwdev(ib_mr->device);
1229 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1230 int ret;
1231
1232 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
1233 if (iwmr->region) {
1234 struct irdma_ucontext *ucontext;
1235
1236 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
1237
1238 irdma_del_memlist(iwmr, ucontext);
1239 }
1240 goto done;
1241 }
1242
1243 ret = irdma_hwdereg_mr(ib_mr);
1244 if (ret)
1245 return ret;
1246
1247 irdma_free_stag(iwdev, iwmr->stag);
1248 done:
1249 if (iwpbl->pbl_allocated)
1250 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
1251
1252 if (iwmr->region)
1253 ib_umem_release(iwmr->region);
1254
1255 kfree(iwmr);
1256
1257 return 0;
1258 }
1259
1260 /**
1261 * irdma_reg_user_mr - Register a user memory region
1262 * @pd: ptr of pd
1263 * @start: virtual start address
1264 * @len: length of mr
1265 * @virt: virtual address
1266 * @access: access of mr
1267 * @udata: user data
1268 */
1269 struct ib_mr *
irdma_reg_user_mr(struct ib_pd * pd,u64 start,u64 len,u64 virt,int access,struct ib_udata * udata)1270 irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1271 u64 virt, int access,
1272 struct ib_udata *udata)
1273 {
1274 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
1275 struct irdma_device *iwdev = to_iwdev(pd->device);
1276 struct irdma_mem_reg_req req = {};
1277 struct ib_umem *region;
1278 struct irdma_mr *iwmr;
1279 int err;
1280
1281 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
1282 return ERR_PTR(-EINVAL);
1283
1284 if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
1285 return ERR_PTR(-EINVAL);
1286
1287 region = ib_umem_get(pd->uobject->context, start, len, access, 0);
1288
1289 if (IS_ERR(region)) {
1290 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
1291 "Failed to create ib_umem region err=%ld\n",
1292 PTR_ERR(region));
1293 return (struct ib_mr *)region;
1294 }
1295
1296 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) {
1297 ib_umem_release(region);
1298 return ERR_PTR(-EFAULT);
1299 }
1300
1301 iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type);
1302 if (IS_ERR(iwmr)) {
1303 ib_umem_release(region);
1304 return (struct ib_mr *)iwmr;
1305 }
1306
1307 switch (req.reg_type) {
1308 case IRDMA_MEMREG_TYPE_QP:
1309 err = irdma_reg_user_mr_type_qp(req, udata, iwmr);
1310 if (err)
1311 goto error;
1312
1313 break;
1314 case IRDMA_MEMREG_TYPE_CQ:
1315 err = irdma_reg_user_mr_type_cq(req, udata, iwmr);
1316 if (err)
1317 goto error;
1318
1319 break;
1320 case IRDMA_MEMREG_TYPE_MEM:
1321 err = irdma_reg_user_mr_type_mem(iwmr, access, true);
1322 if (err)
1323 goto error;
1324
1325 break;
1326 default:
1327 err = -EINVAL;
1328 goto error;
1329 }
1330
1331 return &iwmr->ibmr;
1332
1333 error:
1334 ib_umem_release(region);
1335 irdma_free_iwmr(iwmr);
1336
1337 return ERR_PTR(err);
1338 }
1339
1340 /*
1341 * irdma_rereg_user_mr - Re-Register a user memory region @ibmr: ib mem to access iwarp mr pointer @flags: bit mask to
1342 * indicate which of the attr's of MR modified @start: virtual start address @len: length of mr @virt: virtual address
1343 * @new access flags: bit mask of access flags @new_pd: ptr of pd @udata: user data
1344 */
1345 int
irdma_rereg_user_mr(struct ib_mr * ib_mr,int flags,u64 start,u64 len,u64 virt,int new_access,struct ib_pd * new_pd,struct ib_udata * udata)1346 irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len,
1347 u64 virt, int new_access, struct ib_pd *new_pd,
1348 struct ib_udata *udata)
1349 {
1350 struct irdma_device *iwdev = to_iwdev(ib_mr->device);
1351 struct irdma_mr *iwmr = to_iwmr(ib_mr);
1352 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1353 int ret;
1354
1355 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
1356 return -EINVAL;
1357
1358 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
1359 return -EOPNOTSUPP;
1360
1361 ret = irdma_hwdereg_mr(ib_mr);
1362 if (ret)
1363 return ret;
1364
1365 if (flags & IB_MR_REREG_ACCESS)
1366 iwmr->access = new_access;
1367
1368 if (flags & IB_MR_REREG_PD) {
1369 iwmr->ibmr.pd = new_pd;
1370 iwmr->ibmr.device = new_pd->device;
1371 }
1372
1373 if (flags & IB_MR_REREG_TRANS) {
1374 if (iwpbl->pbl_allocated) {
1375 irdma_free_pble(iwdev->rf->pble_rsrc,
1376 &iwpbl->pble_alloc);
1377 iwpbl->pbl_allocated = false;
1378 }
1379 if (iwmr->region) {
1380 ib_umem_release(iwmr->region);
1381 iwmr->region = NULL;
1382 }
1383
1384 ib_mr = irdma_rereg_mr_trans(iwmr, start, len, virt, udata);
1385 if (IS_ERR(ib_mr))
1386 return PTR_ERR(ib_mr);
1387
1388 } else {
1389 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
1390 if (ret)
1391 return ret;
1392 }
1393
1394 return 0;
1395 }
1396
1397 int
kc_irdma_set_roce_cm_info(struct irdma_qp * iwqp,struct ib_qp_attr * attr,u16 * vlan_id)1398 kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr,
1399 u16 *vlan_id)
1400 {
1401 int ret;
1402 union ib_gid sgid;
1403 struct ib_gid_attr sgid_attr;
1404 struct irdma_av *av = &iwqp->roce_ah.av;
1405
1406 ret = ib_get_cached_gid(iwqp->ibqp.device, attr->ah_attr.port_num,
1407 attr->ah_attr.grh.sgid_index, &sgid,
1408 &sgid_attr);
1409 if (ret)
1410 return ret;
1411
1412 if (sgid_attr.ndev) {
1413 *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev);
1414 ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, if_getlladdr(sgid_attr.ndev));
1415 }
1416
1417 av->net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
1418 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid);
1419 dev_put(sgid_attr.ndev);
1420 iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri;
1421
1422 return 0;
1423 }
1424
1425 /**
1426 * irdma_destroy_cq - destroy cq
1427 * @ib_cq: cq pointer
1428 * @udata: user data
1429 */
1430 void
irdma_destroy_cq(struct ib_cq * ib_cq,struct ib_udata * udata)1431 irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
1432 {
1433 struct irdma_device *iwdev = to_iwdev(ib_cq->device);
1434 struct irdma_cq *iwcq = to_iwcq(ib_cq);
1435 struct irdma_sc_cq *cq = &iwcq->sc_cq;
1436 struct irdma_sc_dev *dev = cq->dev;
1437 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id];
1438 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq);
1439 unsigned long flags;
1440
1441 spin_lock_irqsave(&iwcq->lock, flags);
1442 if (!list_empty(&iwcq->cmpl_generated))
1443 irdma_remove_cmpls_list(iwcq);
1444 if (!list_empty(&iwcq->resize_list))
1445 irdma_process_resize_list(iwcq, iwdev, NULL);
1446 spin_unlock_irqrestore(&iwcq->lock, flags);
1447
1448 irdma_cq_rem_ref(ib_cq);
1449 wait_for_completion(&iwcq->free_cq);
1450
1451 irdma_cq_wq_destroy(iwdev->rf, cq);
1452
1453 spin_lock_irqsave(&iwceq->ce_lock, flags);
1454 irdma_sc_cleanup_ceqes(cq, ceq);
1455 spin_unlock_irqrestore(&iwceq->ce_lock, flags);
1456 irdma_cq_free_rsrc(iwdev->rf, iwcq);
1457 }
1458
1459 /**
1460 * kc_set_loc_seq_num_mss - Set local seq number and mss
1461 * @cm_node: cm node info
1462 */
1463 void
kc_set_loc_seq_num_mss(struct irdma_cm_node * cm_node)1464 kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node)
1465 {
1466 struct timespec ts;
1467
1468 getnanotime(&ts);
1469 cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec;
1470 if (cm_node->iwdev->vsi.mtu > 1500 &&
1471 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd)
1472 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ?
1473 (1500 - IRDMA_MTU_TO_MSS_IPV4) :
1474 (1500 - IRDMA_MTU_TO_MSS_IPV6);
1475 else
1476 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ?
1477 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) :
1478 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6);
1479 }
1480
1481 /**
1482 * irdma_disassociate_ucontext - Disassociate user context
1483 * @context: ib user context
1484 */
1485 void
irdma_disassociate_ucontext(struct ib_ucontext * context)1486 irdma_disassociate_ucontext(struct ib_ucontext *context)
1487 {
1488 }
1489
1490 struct ib_device *
ib_device_get_by_netdev(if_t netdev,int driver_id)1491 ib_device_get_by_netdev(if_t netdev, int driver_id)
1492 {
1493 struct irdma_device *iwdev;
1494 struct irdma_handler *hdl;
1495 unsigned long flags;
1496
1497 spin_lock_irqsave(&irdma_handler_lock, flags);
1498 list_for_each_entry(hdl, &irdma_handlers, list) {
1499 iwdev = hdl->iwdev;
1500 if (netdev == iwdev->netdev) {
1501 spin_unlock_irqrestore(&irdma_handler_lock,
1502 flags);
1503 return &iwdev->ibdev;
1504 }
1505 }
1506 spin_unlock_irqrestore(&irdma_handler_lock, flags);
1507
1508 return NULL;
1509 }
1510
1511 void
ib_unregister_device_put(struct ib_device * device)1512 ib_unregister_device_put(struct ib_device *device)
1513 {
1514 ib_unregister_device(device);
1515 }
1516
1517 /**
1518 * irdma_query_gid_roce - Query port GID for Roce
1519 * @ibdev: device pointer from stack
1520 * @port: port number
1521 * @index: Entry index
1522 * @gid: Global ID
1523 */
1524 int
irdma_query_gid_roce(struct ib_device * ibdev,u8 port,int index,union ib_gid * gid)1525 irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index,
1526 union ib_gid *gid)
1527 {
1528 int ret;
1529
1530 ret = ib_get_cached_gid(ibdev, port, index, gid, NULL);
1531 if (ret == -EAGAIN) {
1532 memcpy(gid, &zgid, sizeof(*gid));
1533 return 0;
1534 }
1535
1536 return ret;
1537 }
1538
1539 /**
1540 * irdma_modify_port - modify port attributes
1541 * @ibdev: device pointer from stack
1542 * @port: port number for query
1543 * @mask: Property mask
1544 * @props: returning device attributes
1545 */
1546 int
irdma_modify_port(struct ib_device * ibdev,u8 port,int mask,struct ib_port_modify * props)1547 irdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
1548 struct ib_port_modify *props)
1549 {
1550 if (port > 1)
1551 return -EINVAL;
1552
1553 return 0;
1554 }
1555
1556 /**
1557 * irdma_query_pkey - Query partition key
1558 * @ibdev: device pointer from stack
1559 * @port: port number
1560 * @index: index of pkey
1561 * @pkey: pointer to store the pkey
1562 */
1563 int
irdma_query_pkey(struct ib_device * ibdev,u8 port,u16 index,u16 * pkey)1564 irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1565 u16 *pkey)
1566 {
1567 if (index >= IRDMA_PKEY_TBL_SZ)
1568 return -EINVAL;
1569
1570 *pkey = IRDMA_DEFAULT_PKEY;
1571 return 0;
1572 }
1573
1574 int
irdma_roce_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)1575 irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
1576 struct ib_port_immutable *immutable)
1577 {
1578 struct ib_port_attr attr;
1579 int err;
1580
1581 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
1582 err = ib_query_port(ibdev, port_num, &attr);
1583 if (err)
1584 return err;
1585
1586 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
1587 immutable->pkey_tbl_len = attr.pkey_tbl_len;
1588 immutable->gid_tbl_len = attr.gid_tbl_len;
1589
1590 return 0;
1591 }
1592
1593 int
irdma_iw_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)1594 irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num,
1595 struct ib_port_immutable *immutable)
1596 {
1597 struct ib_port_attr attr;
1598 int err;
1599
1600 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1601 err = ib_query_port(ibdev, port_num, &attr);
1602 if (err)
1603 return err;
1604 immutable->gid_tbl_len = 1;
1605
1606 return 0;
1607 }
1608
1609 /**
1610 * irdma_query_port - get port attributes
1611 * @ibdev: device pointer from stack
1612 * @port: port number for query
1613 * @props: returning device attributes
1614 */
1615 int
irdma_query_port(struct ib_device * ibdev,u8 port,struct ib_port_attr * props)1616 irdma_query_port(struct ib_device *ibdev, u8 port,
1617 struct ib_port_attr *props)
1618 {
1619 struct irdma_device *iwdev = to_iwdev(ibdev);
1620 if_t netdev = iwdev->netdev;
1621
1622 /* no need to zero out pros here. done by caller */
1623
1624 props->max_mtu = IB_MTU_4096;
1625 props->active_mtu = min(props->max_mtu, iboe_get_mtu(if_getmtu(netdev)));
1626 props->lid = 1;
1627 props->lmc = 0;
1628 props->sm_lid = 0;
1629 props->sm_sl = 0;
1630 if ((if_getlinkstate(netdev) == LINK_STATE_UP) && (if_getdrvflags(netdev) & IFF_DRV_RUNNING)) {
1631 props->state = IB_PORT_ACTIVE;
1632 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
1633 } else {
1634 props->state = IB_PORT_DOWN;
1635 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
1636 }
1637 ib_get_eth_speed(ibdev, port, (u16 *)& props->active_speed, &props->active_width);
1638
1639 if (rdma_protocol_roce(ibdev, 1)) {
1640 props->gid_tbl_len = 32;
1641 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
1642 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ;
1643 } else {
1644 props->gid_tbl_len = 1;
1645 }
1646 props->qkey_viol_cntr = 0;
1647 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP;
1648 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size;
1649
1650 return 0;
1651 }
1652
1653 static const char *const irdma_hw_stat_names[] = {
1654 /* gen1 - 32-bit */
1655 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
1656 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
1657 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
1658 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
1659 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
1660 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
1661 [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors",
1662 /* gen1 - 64-bit */
1663 [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets",
1664 [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts",
1665 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd",
1666 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts",
1667 [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets",
1668 [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts",
1669 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd",
1670 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts",
1671 [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets",
1672 [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts",
1673 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd",
1674 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts",
1675 [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets",
1676 [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts",
1677 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd",
1678 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts",
1679 [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "InRdmaReads",
1680 [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "InRdmaSends",
1681 [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "InRdmaWrites",
1682 [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "OutRdmaReads",
1683 [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "OutRdmaSends",
1684 [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "OutRdmaWrites",
1685 [IRDMA_HW_STAT_INDEX_RDMAVBND] = "RdmaBnd",
1686 [IRDMA_HW_STAT_INDEX_RDMAVINV] = "RdmaInv",
1687
1688 /* gen2 - 32-bit */
1689 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled",
1690 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored",
1691 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent",
1692 /* gen2 - 64-bit */
1693 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets",
1694 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets",
1695 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets",
1696 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets",
1697 [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP",
1698 [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP",
1699 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd",
1700 [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "RetransSegs",
1701 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "InOptErrors",
1702 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "InProtoErrors",
1703 [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "InSegs",
1704 [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "OutSegs",
1705 };
1706
1707 /**
1708 * irdma_alloc_hw_stats - Allocate a hw stats structure
1709 * @ibdev: device pointer from stack
1710 * @port_num: port number
1711 */
1712 struct rdma_hw_stats *
irdma_alloc_hw_stats(struct ib_device * ibdev,u8 port_num)1713 irdma_alloc_hw_stats(struct ib_device *ibdev,
1714 u8 port_num)
1715 {
1716 struct irdma_device *iwdev = to_iwdev(ibdev);
1717 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
1718
1719 int num_counters = dev->hw_attrs.max_stat_idx;
1720 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
1721
1722 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters,
1723 lifespan);
1724 }
1725
1726 /**
1727 * irdma_get_hw_stats - Populates the rdma_hw_stats structure
1728 * @ibdev: device pointer from stack
1729 * @stats: stats pointer from stack
1730 * @port_num: port number
1731 * @index: which hw counter the stack is requesting we update
1732 */
1733 int
irdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u8 port_num,int index)1734 irdma_get_hw_stats(struct ib_device *ibdev,
1735 struct rdma_hw_stats *stats, u8 port_num,
1736 int index)
1737 {
1738 struct irdma_device *iwdev = to_iwdev(ibdev);
1739 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats;
1740
1741 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)
1742 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true);
1743
1744 memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters);
1745
1746 return stats->num_counters;
1747 }
1748
1749 /**
1750 * irdma_query_gid - Query port GID
1751 * @ibdev: device pointer from stack
1752 * @port: port number
1753 * @index: Entry index
1754 * @gid: Global ID
1755 */
1756 int
irdma_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * gid)1757 irdma_query_gid(struct ib_device *ibdev, u8 port, int index,
1758 union ib_gid *gid)
1759 {
1760 struct irdma_device *iwdev = to_iwdev(ibdev);
1761
1762 memset(gid->raw, 0, sizeof(gid->raw));
1763 ether_addr_copy(gid->raw, if_getlladdr(iwdev->netdev));
1764
1765 return 0;
1766 }
1767
1768 enum rdma_link_layer
irdma_get_link_layer(struct ib_device * ibdev,u8 port_num)1769 irdma_get_link_layer(struct ib_device *ibdev,
1770 u8 port_num)
1771 {
1772 return IB_LINK_LAYER_ETHERNET;
1773 }
1774
1775 inline enum ib_mtu
ib_mtu_int_to_enum(int mtu)1776 ib_mtu_int_to_enum(int mtu)
1777 {
1778 if (mtu >= 4096)
1779 return IB_MTU_4096;
1780 else if (mtu >= 2048)
1781 return IB_MTU_2048;
1782 else if (mtu >= 1024)
1783 return IB_MTU_1024;
1784 else if (mtu >= 512)
1785 return IB_MTU_512;
1786 else
1787 return IB_MTU_256;
1788 }
1789
1790 inline void
kc_set_roce_uverbs_cmd_mask(struct irdma_device * iwdev)1791 kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev)
1792 {
1793 iwdev->ibdev.uverbs_cmd_mask |=
1794 BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) |
1795 BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) |
1796 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) |
1797 BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST);
1798 }
1799
1800 inline void
kc_set_rdma_uverbs_cmd_mask(struct irdma_device * iwdev)1801 kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev)
1802 {
1803 iwdev->ibdev.uverbs_cmd_mask =
1804 BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) |
1805 BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) |
1806 BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) |
1807 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) |
1808 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) |
1809 BIT_ULL(IB_USER_VERBS_CMD_REG_MR) |
1810 BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) |
1811 BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) |
1812 BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1813 BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) |
1814 BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) |
1815 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) |
1816 BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1817 BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) |
1818 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) |
1819 BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
1820 BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) |
1821 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
1822 BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) |
1823 BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);
1824 iwdev->ibdev.uverbs_ex_cmd_mask =
1825 BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) |
1826 BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE);
1827
1828 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)
1829 iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ);
1830 }
1831
1832 static void
ib_get_width_and_speed(u32 netdev_speed,u32 lanes,u16 * speed,u8 * width)1833 ib_get_width_and_speed(u32 netdev_speed, u32 lanes,
1834 u16 *speed, u8 *width)
1835 {
1836 if (!lanes) {
1837 if (netdev_speed <= SPEED_1000) {
1838 *width = IB_WIDTH_1X;
1839 *speed = IB_SPEED_SDR;
1840 } else if (netdev_speed <= SPEED_10000) {
1841 *width = IB_WIDTH_1X;
1842 *speed = IB_SPEED_FDR10;
1843 } else if (netdev_speed <= SPEED_20000) {
1844 *width = IB_WIDTH_4X;
1845 *speed = IB_SPEED_DDR;
1846 } else if (netdev_speed <= SPEED_25000) {
1847 *width = IB_WIDTH_1X;
1848 *speed = IB_SPEED_EDR;
1849 } else if (netdev_speed <= SPEED_40000) {
1850 *width = IB_WIDTH_4X;
1851 *speed = IB_SPEED_FDR10;
1852 } else if (netdev_speed <= SPEED_50000) {
1853 *width = IB_WIDTH_2X;
1854 *speed = IB_SPEED_EDR;
1855 } else if (netdev_speed <= SPEED_100000) {
1856 *width = IB_WIDTH_4X;
1857 *speed = IB_SPEED_EDR;
1858 } else if (netdev_speed <= SPEED_200000) {
1859 *width = IB_WIDTH_4X;
1860 *speed = IB_SPEED_HDR;
1861 } else {
1862 *width = IB_WIDTH_4X;
1863 *speed = IB_SPEED_NDR;
1864 }
1865
1866 return;
1867 }
1868
1869 switch (lanes) {
1870 case 1:
1871 *width = IB_WIDTH_1X;
1872 break;
1873 case 2:
1874 *width = IB_WIDTH_2X;
1875 break;
1876 case 4:
1877 *width = IB_WIDTH_4X;
1878 break;
1879 case 8:
1880 *width = IB_WIDTH_8X;
1881 break;
1882 case 12:
1883 *width = IB_WIDTH_12X;
1884 break;
1885 default:
1886 *width = IB_WIDTH_1X;
1887 }
1888
1889 switch (netdev_speed / lanes) {
1890 case SPEED_2500:
1891 *speed = IB_SPEED_SDR;
1892 break;
1893 case SPEED_5000:
1894 *speed = IB_SPEED_DDR;
1895 break;
1896 case SPEED_10000:
1897 *speed = IB_SPEED_FDR10;
1898 break;
1899 case SPEED_14000:
1900 *speed = IB_SPEED_FDR;
1901 break;
1902 case SPEED_25000:
1903 *speed = IB_SPEED_EDR;
1904 break;
1905 case SPEED_50000:
1906 *speed = IB_SPEED_HDR;
1907 break;
1908 case SPEED_100000:
1909 *speed = IB_SPEED_NDR;
1910 break;
1911 default:
1912 *speed = IB_SPEED_SDR;
1913 }
1914 }
1915
1916 int
ib_get_eth_speed(struct ib_device * ibdev,u32 port_num,u16 * speed,u8 * width)1917 ib_get_eth_speed(struct ib_device *ibdev, u32 port_num, u16 *speed, u8 *width)
1918 {
1919 if_t netdev = ibdev->get_netdev(ibdev, port_num);
1920 u32 netdev_speed, lanes;
1921
1922 if (!netdev)
1923 return -ENODEV;
1924
1925 netdev_speed = (u32)if_getbaudrate(netdev);
1926 dev_put(netdev);
1927 lanes = 0;
1928
1929 ib_get_width_and_speed(netdev_speed, lanes, speed, width);
1930
1931 return 0;
1932 }
1933
1934 u64
irdma_mac_to_u64(const u8 * eth_add)1935 irdma_mac_to_u64(const u8 *eth_add)
1936 {
1937 int idx;
1938 u64 u64_eth_add;
1939
1940 for (idx = 0, u64_eth_add = 0; idx < ETHER_ADDR_LEN; idx++)
1941 u64_eth_add = u64_eth_add << 8 | eth_add[idx];
1942
1943 return u64_eth_add;
1944 }
1945