1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2021 - 2026 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "osdep.h"
36 #include "ice_rdma.h"
37 #include "irdma_di_if.h"
38 #include "irdma_main.h"
39 #include "icrdma_hw.h"
40 #include <sys/gsb_crc32.h>
41 #include <netinet/in_fib.h>
42 #include <netinet6/in6_fib.h>
43 #include <net/route/nhop.h>
44 #include <net/if_llatbl.h>
45
46 /* additional QP debuging option. Keep false unless needed */
47 bool irdma_upload_context = false;
48 u8 irdma_sysctl_max_ord = ICRDMA_MAX_ORD_SIZE;
49 u8 irdma_sysctl_max_ird = ICRDMA_MAX_IRD_SIZE;
50 u8 irdma_rdpu_bw_tun = 0;
51
52 static void irdma_modify_rdpu_bw(struct irdma_pci_f *rf);
53
54 inline u32
irdma_rd32(struct irdma_dev_ctx * dev_ctx,u32 reg)55 irdma_rd32(struct irdma_dev_ctx *dev_ctx, u32 reg){
56
57 KASSERT(reg < dev_ctx->mem_bus_space_size,
58 ("irdma: register offset %#jx too large (max is %#jx)",
59 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
60
61 return (bus_space_read_4(dev_ctx->mem_bus_space_tag,
62 dev_ctx->mem_bus_space_handle, reg));
63 }
64
65 inline void
irdma_wr32(struct irdma_dev_ctx * dev_ctx,u32 reg,u32 value)66 irdma_wr32(struct irdma_dev_ctx *dev_ctx, u32 reg, u32 value)
67 {
68
69 KASSERT(reg < dev_ctx->mem_bus_space_size,
70 ("irdma: register offset %#jx too large (max is %#jx)",
71 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
72
73 bus_space_write_4(dev_ctx->mem_bus_space_tag,
74 dev_ctx->mem_bus_space_handle, reg, value);
75 }
76
77 inline u64
irdma_rd64(struct irdma_dev_ctx * dev_ctx,u32 reg)78 irdma_rd64(struct irdma_dev_ctx *dev_ctx, u32 reg){
79
80 KASSERT(reg < dev_ctx->mem_bus_space_size,
81 ("irdma: register offset %#jx too large (max is %#jx)",
82 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
83
84 return (bus_space_read_8(dev_ctx->mem_bus_space_tag,
85 dev_ctx->mem_bus_space_handle, reg));
86 }
87
88 inline void
irdma_wr64(struct irdma_dev_ctx * dev_ctx,u32 reg,u64 value)89 irdma_wr64(struct irdma_dev_ctx *dev_ctx, u32 reg, u64 value)
90 {
91
92 KASSERT(reg < dev_ctx->mem_bus_space_size,
93 ("irdma: register offset %#jx too large (max is %#jx)",
94 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
95
96 bus_space_write_8(dev_ctx->mem_bus_space_tag,
97 dev_ctx->mem_bus_space_handle, reg, value);
98
99 }
100
101 void
irdma_request_reset(struct irdma_pci_f * rf)102 irdma_request_reset(struct irdma_pci_f *rf)
103 {
104 struct ice_rdma_peer *peer = rf->peer_info;
105 struct ice_rdma_request req = {0};
106
107 req.type = ICE_RDMA_EVENT_RESET;
108
109 printf("%s:%d requesting pf-reset\n", __func__, __LINE__);
110 IRDMA_DI_REQ_HANDLER(peer, &req);
111 }
112
113 int
irdma_register_qset(struct irdma_sc_vsi * vsi,struct irdma_ws_node * tc_node)114 irdma_register_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node)
115 {
116 struct irdma_device *iwdev = vsi->back_vsi;
117 struct ice_rdma_peer *peer = iwdev->rf->peer_info;
118 struct ice_rdma_request req = {0};
119 struct ice_rdma_qset_update *res = &req.res;
120
121 req.type = ICE_RDMA_EVENT_QSET_REGISTER;
122 res->cnt_req = 1;
123 res->res_type = ICE_RDMA_QSET_ALLOC;
124 res->qsets.qs_handle = tc_node->qs_handle;
125 res->qsets.tc = tc_node->traffic_class;
126 res->qsets.vsi_id = vsi->vsi_idx;
127
128 IRDMA_DI_REQ_HANDLER(peer, &req);
129
130 tc_node->l2_sched_node_id = res->qsets.teid;
131 vsi->qos[tc_node->user_pri].l2_sched_node_id =
132 res->qsets.teid;
133
134 return 0;
135 }
136
137 void
irdma_unregister_qset(struct irdma_sc_vsi * vsi,struct irdma_ws_node * tc_node)138 irdma_unregister_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node)
139 {
140 struct irdma_device *iwdev = vsi->back_vsi;
141 struct ice_rdma_peer *peer = iwdev->rf->peer_info;
142 struct ice_rdma_request req = {0};
143 struct ice_rdma_qset_update *res = &req.res;
144
145 req.type = ICE_RDMA_EVENT_QSET_REGISTER;
146 res->res_allocated = 1;
147 res->res_type = ICE_RDMA_QSET_FREE;
148 res->qsets.vsi_id = vsi->vsi_idx;
149 res->qsets.teid = tc_node->l2_sched_node_id;
150 res->qsets.qs_handle = tc_node->qs_handle;
151
152 IRDMA_DI_REQ_HANDLER(peer, &req);
153 }
154
155 void *
hw_to_dev(struct irdma_hw * hw)156 hw_to_dev(struct irdma_hw *hw)
157 {
158 struct irdma_pci_f *rf;
159
160 rf = container_of(hw, struct irdma_pci_f, hw);
161 return rf->pcidev;
162 }
163
164 void
irdma_free_hash_desc(void * desc)165 irdma_free_hash_desc(void *desc)
166 {
167 return;
168 }
169
170 int
irdma_init_hash_desc(void ** desc)171 irdma_init_hash_desc(void **desc)
172 {
173 return 0;
174 }
175
176 int
irdma_ieq_check_mpacrc(void * desc,void * addr,u32 len,u32 val)177 irdma_ieq_check_mpacrc(void *desc,
178 void *addr, u32 len, u32 val)
179 {
180 u32 crc = calculate_crc32c(0xffffffff, addr, len) ^ 0xffffffff;
181 int ret_code = 0;
182
183 if (crc != val) {
184 irdma_pr_err("mpa crc check fail %x %x\n", crc, val);
185 ret_code = -EINVAL;
186 }
187 printf("%s: result crc=%x value=%x\n", __func__, crc, val);
188 return ret_code;
189 }
190
191 static u_int
irdma_add_ipv6_cb(void * arg,struct ifaddr * addr,u_int count __unused)192 irdma_add_ipv6_cb(void *arg, struct ifaddr *addr, u_int count __unused)
193 {
194 struct irdma_device *iwdev = arg;
195 struct sockaddr_in6 *sin6;
196 u32 local_ipaddr6[4] = {};
197 char ip6buf[INET6_ADDRSTRLEN];
198 u8 *mac_addr;
199
200 sin6 = (struct sockaddr_in6 *)addr->ifa_addr;
201
202 irdma_copy_ip_ntohl(local_ipaddr6, (u32 *)&sin6->sin6_addr);
203
204 mac_addr = if_getlladdr(addr->ifa_ifp);
205
206 printf("%s:%d IP=%s, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
207 __func__, __LINE__,
208 ip6_sprintf(ip6buf, &sin6->sin6_addr),
209 mac_addr[0], mac_addr[1], mac_addr[2],
210 mac_addr[3], mac_addr[4], mac_addr[5]);
211
212 irdma_manage_arp_cache(iwdev->rf, mac_addr, local_ipaddr6,
213 IRDMA_ARP_ADD);
214 return (0);
215 }
216
217 /**
218 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
219 * @iwdev: irdma device
220 * @ifp: interface network device pointer
221 */
222 static void
irdma_add_ipv6_addr(struct irdma_device * iwdev,struct ifnet * ifp)223 irdma_add_ipv6_addr(struct irdma_device *iwdev, struct ifnet *ifp)
224 {
225 if_addr_rlock(ifp);
226 if_foreach_addr_type(ifp, AF_INET6, irdma_add_ipv6_cb, iwdev);
227 if_addr_runlock(ifp);
228 }
229
230 static u_int
irdma_add_ipv4_cb(void * arg,struct ifaddr * addr,u_int count __unused)231 irdma_add_ipv4_cb(void *arg, struct ifaddr *addr, u_int count __unused)
232 {
233 struct irdma_device *iwdev = arg;
234 struct sockaddr_in *sin;
235 u32 ip_addr[4] = {};
236 uint8_t *mac_addr;
237
238 sin = (struct sockaddr_in *)addr->ifa_addr;
239
240 ip_addr[0] = ntohl(sin->sin_addr.s_addr);
241
242 mac_addr = if_getlladdr(addr->ifa_ifp);
243
244 printf("%s:%d IP=%d.%d.%d.%d, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
245 __func__, __LINE__,
246 ip_addr[0] >> 24,
247 (ip_addr[0] >> 16) & 0xFF,
248 (ip_addr[0] >> 8) & 0xFF,
249 ip_addr[0] & 0xFF,
250 mac_addr[0], mac_addr[1], mac_addr[2],
251 mac_addr[3], mac_addr[4], mac_addr[5]);
252
253 irdma_manage_arp_cache(iwdev->rf, mac_addr, ip_addr,
254 IRDMA_ARP_ADD);
255 return (0);
256 }
257
258 /**
259 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
260 * @iwdev: irdma device
261 * @ifp: interface network device pointer
262 */
263 static void
irdma_add_ipv4_addr(struct irdma_device * iwdev,struct ifnet * ifp)264 irdma_add_ipv4_addr(struct irdma_device *iwdev, struct ifnet *ifp)
265 {
266 if_addr_rlock(ifp);
267 if_foreach_addr_type(ifp, AF_INET, irdma_add_ipv4_cb, iwdev);
268 if_addr_runlock(ifp);
269 }
270
271 /**
272 * irdma_add_ip - add ip addresses
273 * @iwdev: irdma device
274 *
275 * Add ipv4/ipv6 addresses to the arp cache
276 */
277 void
irdma_add_ip(struct irdma_device * iwdev)278 irdma_add_ip(struct irdma_device *iwdev)
279 {
280 struct ifnet *ifp = iwdev->netdev;
281 struct ifnet *ifv;
282 struct epoch_tracker et;
283 int i;
284
285 irdma_add_ipv4_addr(iwdev, ifp);
286 irdma_add_ipv6_addr(iwdev, ifp);
287 for (i = 0; if_getvlantrunk(ifp) != NULL && i < VLAN_N_VID; ++i) {
288 NET_EPOCH_ENTER(et);
289 ifv = VLAN_DEVAT(ifp, i);
290 NET_EPOCH_EXIT(et);
291 if (!ifv)
292 continue;
293 irdma_add_ipv4_addr(iwdev, ifv);
294 irdma_add_ipv6_addr(iwdev, ifv);
295 }
296 }
297
298 static void
irdma_ifaddrevent_handler(void * arg,struct ifnet * ifp,struct ifaddr * ifa,int event)299 irdma_ifaddrevent_handler(void *arg, struct ifnet *ifp, struct ifaddr *ifa, int event)
300 {
301 struct irdma_pci_f *rf = arg;
302 struct ifnet *ifv = NULL;
303 struct sockaddr_in *sin;
304 struct epoch_tracker et;
305 int arp_index = 0, i = 0;
306 u32 ip[4] = {};
307
308 if (!ifa || !ifa->ifa_addr || !ifp)
309 return;
310 if (rf->iwdev->netdev != ifp) {
311 for (i = 0; if_getvlantrunk(rf->iwdev->netdev) != NULL && i < VLAN_N_VID; ++i) {
312 NET_EPOCH_ENTER(et);
313 ifv = VLAN_DEVAT(rf->iwdev->netdev, i);
314 NET_EPOCH_EXIT(et);
315 if (ifv == ifp)
316 break;
317 }
318 if (ifv != ifp)
319 return;
320 }
321 sin = (struct sockaddr_in *)ifa->ifa_addr;
322
323 switch (event) {
324 case IFADDR_EVENT_ADD:
325 if (sin->sin_family == AF_INET)
326 irdma_add_ipv4_addr(rf->iwdev, ifp);
327 else if (sin->sin_family == AF_INET6)
328 irdma_add_ipv6_addr(rf->iwdev, ifp);
329 break;
330 case IFADDR_EVENT_DEL:
331 if (sin->sin_family == AF_INET) {
332 ip[0] = ntohl(sin->sin_addr.s_addr);
333 } else if (sin->sin_family == AF_INET6) {
334 irdma_copy_ip_ntohl(ip, (u32 *)&((struct sockaddr_in6 *)sin)->sin6_addr);
335 } else {
336 break;
337 }
338 for_each_set_bit(arp_index, rf->allocated_arps, rf->arp_table_size) {
339 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) {
340 irdma_manage_arp_cache(rf, rf->arp_table[arp_index].mac_addr,
341 rf->arp_table[arp_index].ip_addr,
342 IRDMA_ARP_DELETE);
343 }
344 }
345 break;
346 default:
347 break;
348 }
349 }
350
351 void
irdma_reg_ipaddr_event_cb(struct irdma_pci_f * rf)352 irdma_reg_ipaddr_event_cb(struct irdma_pci_f *rf)
353 {
354 rf->irdma_ifaddr_event = EVENTHANDLER_REGISTER(ifaddr_event_ext,
355 irdma_ifaddrevent_handler,
356 rf,
357 EVENTHANDLER_PRI_ANY);
358 }
359
360 void
irdma_dereg_ipaddr_event_cb(struct irdma_pci_f * rf)361 irdma_dereg_ipaddr_event_cb(struct irdma_pci_f *rf)
362 {
363 EVENTHANDLER_DEREGISTER(ifaddr_event_ext, rf->irdma_ifaddr_event);
364 }
365
366 static int
irdma_get_route_ifp(struct sockaddr * dst_sin,struct ifnet * netdev,struct ifnet ** ifp,struct sockaddr ** nexthop,bool * gateway)367 irdma_get_route_ifp(struct sockaddr *dst_sin, struct ifnet *netdev,
368 struct ifnet **ifp, struct sockaddr **nexthop, bool *gateway)
369 {
370 struct nhop_object *nh;
371
372 if (dst_sin->sa_family == AF_INET6)
373 nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr,
374 ((struct sockaddr_in6 *)dst_sin)->sin6_scope_id, NHR_NONE, 0);
375 else
376 nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0);
377 if (!nh || (nh->nh_ifp != netdev &&
378 rdma_vlan_dev_real_dev(nh->nh_ifp) != netdev))
379 goto rt_not_found;
380 *gateway = (nh->nh_flags & NHF_GATEWAY) ? true : false;
381 *nexthop = (*gateway) ? &nh->gw_sa : dst_sin;
382 *ifp = nh->nh_ifp;
383
384 return 0;
385
386 rt_not_found:
387 pr_err("irdma: route not found\n");
388 return -ENETUNREACH;
389 }
390
391 /**
392 * irdma_get_dst_mac - get destination mac address
393 * @cm_node: connection's node
394 * @dst_sin: destination address information
395 * @dst_mac: mac address array to return
396 */
397 int
irdma_get_dst_mac(struct irdma_cm_node * cm_node,struct sockaddr * dst_sin,u8 * dst_mac)398 irdma_get_dst_mac(struct irdma_cm_node *cm_node, struct sockaddr *dst_sin, u8 *dst_mac)
399 {
400 struct ifnet *netdev = cm_node->iwdev->netdev;
401 #ifdef VIMAGE
402 struct vnet *vnet = irdma_cmid_to_vnet(cm_node->cm_id);
403 #endif
404 struct ifnet *ifp;
405 struct llentry *lle;
406 struct sockaddr *nexthop;
407 struct epoch_tracker et;
408 int err;
409 bool gateway;
410
411 NET_EPOCH_ENTER(et);
412 CURVNET_SET_QUIET(vnet);
413 err = irdma_get_route_ifp(dst_sin, netdev, &ifp, &nexthop, &gateway);
414 if (err)
415 goto get_route_fail;
416
417 if (dst_sin->sa_family == AF_INET) {
418 err = arpresolve(ifp, gateway, NULL, nexthop, dst_mac, NULL, &lle);
419 } else if (dst_sin->sa_family == AF_INET6) {
420 err = nd6_resolve(ifp, LLE_SF(AF_INET6, gateway), NULL, nexthop,
421 dst_mac, NULL, &lle);
422 } else {
423 err = -EPROTONOSUPPORT;
424 }
425
426 get_route_fail:
427 CURVNET_RESTORE();
428 NET_EPOCH_EXIT(et);
429 if (err) {
430 pr_err("failed to resolve neighbor address (err=%d)\n",
431 err);
432 return -ENETUNREACH;
433 }
434
435 return 0;
436 }
437
438 /**
439 * irdma_addr_resolve_neigh - resolve neighbor address
440 * @cm_node: connection's node
441 * @dst_ip: remote ip address
442 * @arpindex: if there is an arp entry
443 */
444 int
irdma_addr_resolve_neigh(struct irdma_cm_node * cm_node,u32 dst_ip,int arpindex)445 irdma_addr_resolve_neigh(struct irdma_cm_node *cm_node,
446 u32 dst_ip, int arpindex)
447 {
448 struct irdma_device *iwdev = cm_node->iwdev;
449 struct sockaddr_in dst_sin = {};
450 int err;
451 u32 ip[4] = {};
452 u8 dst_mac[MAX_ADDR_LEN];
453
454 dst_sin.sin_len = sizeof(dst_sin);
455 dst_sin.sin_family = AF_INET;
456 dst_sin.sin_port = 0;
457 dst_sin.sin_addr.s_addr = htonl(dst_ip);
458
459 err = irdma_get_dst_mac(cm_node, (struct sockaddr *)&dst_sin, dst_mac);
460 if (err)
461 return arpindex;
462
463 ip[0] = dst_ip;
464
465 return irdma_add_arp(iwdev->rf, ip, dst_mac);
466 }
467
468 /**
469 * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address
470 * @cm_node: connection's node
471 * @dest: remote ip address
472 * @arpindex: if there is an arp entry
473 */
474 int
irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node * cm_node,u32 * dest,int arpindex)475 irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node *cm_node,
476 u32 *dest, int arpindex)
477 {
478 struct irdma_device *iwdev = cm_node->iwdev;
479 struct sockaddr_in6 dst_addr = {};
480 int err;
481 u8 dst_mac[MAX_ADDR_LEN];
482
483 dst_addr.sin6_family = AF_INET6;
484 dst_addr.sin6_len = sizeof(dst_addr);
485 dst_addr.sin6_scope_id = if_getindex(iwdev->netdev);
486
487 irdma_copy_ip_htonl(dst_addr.sin6_addr.__u6_addr.__u6_addr32, dest);
488 err = irdma_get_dst_mac(cm_node, (struct sockaddr *)&dst_addr, dst_mac);
489 if (err)
490 return arpindex;
491
492 return irdma_add_arp(iwdev->rf, dest, dst_mac);
493 }
494
495 int
irdma_resolve_neigh_lpb_chk(struct irdma_device * iwdev,struct irdma_cm_node * cm_node,struct irdma_cm_info * cm_info)496 irdma_resolve_neigh_lpb_chk(struct irdma_device *iwdev, struct irdma_cm_node *cm_node,
497 struct irdma_cm_info *cm_info)
498 {
499 #ifdef VIMAGE
500 struct vnet *vnet = irdma_cmid_to_vnet(cm_node->cm_id);
501 #endif
502 int arpindex;
503 int oldarpindex;
504 bool is_lpb = false;
505
506 CURVNET_SET_QUIET(vnet);
507 is_lpb = cm_node->ipv4 ?
508 irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0]) :
509 irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr);
510 CURVNET_RESTORE();
511 if (is_lpb) {
512 cm_node->do_lpb = true;
513 arpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
514 NULL,
515 IRDMA_ARP_RESOLVE);
516 } else {
517 oldarpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
518 NULL,
519 IRDMA_ARP_RESOLVE);
520 if (cm_node->ipv4)
521 arpindex = irdma_addr_resolve_neigh(cm_node,
522 cm_info->rem_addr[0],
523 oldarpindex);
524 else
525 arpindex = irdma_addr_resolve_neigh_ipv6(cm_node,
526 cm_info->rem_addr,
527 oldarpindex);
528 }
529 return arpindex;
530 }
531
532 /**
533 * irdma_add_handler - add a handler to the list
534 * @hdl: handler to be added to the handler list
535 */
536 void
irdma_add_handler(struct irdma_handler * hdl)537 irdma_add_handler(struct irdma_handler *hdl)
538 {
539 unsigned long flags;
540
541 spin_lock_irqsave(&irdma_handler_lock, flags);
542 list_add(&hdl->list, &irdma_handlers);
543 spin_unlock_irqrestore(&irdma_handler_lock, flags);
544 }
545
546 /**
547 * irdma_del_handler - delete a handler from the list
548 * @hdl: handler to be deleted from the handler list
549 */
550 void
irdma_del_handler(struct irdma_handler * hdl)551 irdma_del_handler(struct irdma_handler *hdl)
552 {
553 unsigned long flags;
554
555 spin_lock_irqsave(&irdma_handler_lock, flags);
556 list_del(&hdl->list);
557 spin_unlock_irqrestore(&irdma_handler_lock, flags);
558 }
559
560 /**
561 * irdma_set_rf_user_cfg_params - apply user configurable settings
562 * @rf: RDMA PCI function
563 */
564 void
irdma_set_rf_user_cfg_params(struct irdma_pci_f * rf)565 irdma_set_rf_user_cfg_params(struct irdma_pci_f *rf)
566 {
567 int en_rem_endpoint_trk = 0;
568 int limits_sel = 4;
569
570 rf->en_rem_endpoint_trk = en_rem_endpoint_trk;
571 rf->limits_sel = limits_sel;
572 rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
573 /* Enable DCQCN algorithm by default */
574 rf->dcqcn_ena = true;
575
576 if (irdma_fw_major_ver(&rf->sc_dev) == 2 && irdma_rdpu_bw_tun)
577 irdma_modify_rdpu_bw(rf);
578 }
579
580 /**
581 * irdma_sysctl_dcqcn_update - handle dcqcn_ena sysctl update
582 * @arg1: pointer to rf
583 * @arg2: unused
584 * @oidp: sysctl oid structure
585 * @req: sysctl request pointer
586 */
587 static int
irdma_sysctl_dcqcn_update(SYSCTL_HANDLER_ARGS)588 irdma_sysctl_dcqcn_update(SYSCTL_HANDLER_ARGS)
589 {
590 struct irdma_pci_f *rf = (struct irdma_pci_f *)arg1;
591 int ret;
592 u8 dcqcn_ena = rf->dcqcn_ena;
593
594 ret = sysctl_handle_8(oidp, &dcqcn_ena, 0, req);
595 if ((ret) || (req->newptr == NULL))
596 return ret;
597 if (dcqcn_ena == 0)
598 rf->dcqcn_ena = false;
599 else
600 rf->dcqcn_ena = true;
601
602 return 0;
603 }
604
605 static void
irdma_modify_rdpu_bw(struct irdma_pci_f * rf)606 irdma_modify_rdpu_bw(struct irdma_pci_f *rf)
607 {
608 u32 val;
609 #define GL_RDPU_CNTRL 0x00052054
610
611 val = rd32(&rf->hw, GL_RDPU_CNTRL);
612 printf("pf%d Read GL_RDPU_CNTRL[%x] = 0x%08X",
613 if_getdunit(rf->peer_info->ifp), GL_RDPU_CNTRL, val);
614
615 /* Clear the load balancing bit */
616 val &= ~(0x1 << 2);
617 wr32(&rf->hw, GL_RDPU_CNTRL, val);
618 val = rd32(&rf->hw, GL_RDPU_CNTRL);
619 printf("pf%d Set GL_RDPU_CNTRL[%x] = 0x%08X",
620 if_getdunit(rf->peer_info->ifp), GL_RDPU_CNTRL, val);
621 }
622
623 enum irdma_qos_info {
624 IRDMA_QOS_DSCP_MAP = 1,
625 IRDMA_QOS_DSCP_MODE,
626 IRDMA_QOS_PRIO_TYPE,
627 IRDMA_QOS_QS_HANDLE,
628 IRDMA_QOS_REL_BW,
629 IRDMA_QOS_TC,
630 IRDMA_QOS_UP2TC
631 };
632
633 enum irdma_cqp_stats_info {
634 IRDMA_CQP_REQ_CMDS = 28,
635 IRDMA_CQP_CMPL_CMDS = 29
636 };
637
638 static int
irdma_sysctl_qos(SYSCTL_HANDLER_ARGS)639 irdma_sysctl_qos(SYSCTL_HANDLER_ARGS)
640 {
641 struct irdma_sc_vsi *vsi = (struct irdma_sc_vsi *)arg1;
642 char rslt[192] = "no vsi available yet";
643 int rslt_size = sizeof(rslt) - 1;
644 int option = (int)arg2;
645 int a;
646
647 if (!vsi) {
648 return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
649
650 }
651
652 snprintf(rslt, sizeof(rslt), "");
653 switch (option) {
654 case IRDMA_QOS_PRIO_TYPE:
655 for (a = 0; a < IRDMA_MAX_USER_PRIORITY; a++)
656 snprintf(rslt, rslt_size, "%s %02x", rslt, vsi->qos[a].prio_type);
657 break;
658 case IRDMA_QOS_REL_BW:
659 for (a = 0; a < IRDMA_MAX_USER_PRIORITY; a++)
660 snprintf(rslt, rslt_size, "%s %d", rslt, vsi->qos[a].rel_bw);
661 break;
662 case IRDMA_QOS_QS_HANDLE:
663 for (a = 0; a < IRDMA_MAX_USER_PRIORITY; a++)
664 snprintf(rslt, rslt_size, "%s %d", rslt, vsi->qos[a].qs_handle);
665 break;
666 case IRDMA_QOS_TC:
667 for (a = 0; a < IRDMA_MAX_USER_PRIORITY; a++)
668 snprintf(rslt, rslt_size, "%s %d", rslt, vsi->qos[a].traffic_class);
669 break;
670 case IRDMA_QOS_UP2TC:
671 for (a = 0; a < IRDMA_MAX_USER_PRIORITY; a++)
672 snprintf(rslt, rslt_size, "%s %d", rslt, vsi->cfg_check[a].traffic_class);
673 break;
674 case IRDMA_QOS_DSCP_MAP:
675 for (a = 0; a < sizeof(vsi->dscp_map); a++)
676 snprintf(rslt, rslt_size, "%s%02x", rslt, vsi->dscp_map[a]);
677 break;
678 case IRDMA_QOS_DSCP_MODE:
679 snprintf(rslt, rslt_size, "%d", vsi->dscp_mode);
680 }
681
682 return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
683 }
684
685 static int
irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS)686 irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS)
687 {
688 struct irdma_sc_cqp *cqp = (struct irdma_sc_cqp *)arg1;
689 char rslt[192] = "no cqp available yet";
690 int rslt_size = sizeof(rslt) - 1;
691 int option = (int)arg2;
692
693 if (!cqp) {
694 return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
695 }
696
697 snprintf(rslt, sizeof(rslt), "");
698 switch (option) {
699 case IRDMA_CQP_REQ_CMDS:
700 snprintf(rslt, rslt_size, "%lu", cqp->requested_ops);
701 break;
702 case IRDMA_CQP_CMPL_CMDS:
703 snprintf(rslt, rslt_size, "%lu", atomic64_read(&cqp->completed_ops));
704 break;
705 }
706
707 return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
708 }
709
710 struct irdma_sw_stats_tunable_info {
711 u8 op_type;
712 const char name[32];
713 const char desc[32];
714 uintptr_t value;
715 };
716
717 static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = {
718 {IRDMA_OP_CEQ_DESTROY, "ceq_destroy", "ceq_destroy", 0},
719 {IRDMA_OP_AEQ_DESTROY, "aeq_destroy", "aeq_destroy", 0},
720 {IRDMA_OP_DELETE_ARP_CACHE_ENTRY, "delete_arp_cache_entry",
721 "delete_arp_cache_entry", 0},
722 {IRDMA_OP_MANAGE_APBVT_ENTRY, "manage_apbvt_entry",
723 "manage_apbvt_entry", 0},
724 {IRDMA_OP_CEQ_CREATE, "ceq_create", "ceq_create", 0},
725 {IRDMA_OP_AEQ_CREATE, "aeq_create", "aeq_create", 0},
726 {IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY, "manage_qhash_table_entry",
727 "manage_qhash_table_entry", 0},
728 {IRDMA_OP_QP_MODIFY, "qp_modify", "qp_modify", 0},
729 {IRDMA_OP_QP_UPLOAD_CONTEXT, "qp_upload_context", "qp_upload_context",
730 0},
731 {IRDMA_OP_CQ_CREATE, "cq_create", "cq_create", 0},
732 {IRDMA_OP_CQ_DESTROY, "cq_destroy", "cq_destroy", 0},
733 {IRDMA_OP_QP_CREATE, "qp_create", "qp_create", 0},
734 {IRDMA_OP_QP_DESTROY, "qp_destroy", "qp_destroy", 0},
735 {IRDMA_OP_ALLOC_STAG, "alloc_stag", "alloc_stag", 0},
736 {IRDMA_OP_MR_REG_NON_SHARED, "mr_reg_non_shared", "mr_reg_non_shared",
737 0},
738 {IRDMA_OP_DEALLOC_STAG, "dealloc_stag", "dealloc_stag", 0},
739 {IRDMA_OP_MW_ALLOC, "mw_alloc", "mw_alloc", 0},
740 {IRDMA_OP_QP_FLUSH_WQES, "qp_flush_wqes", "qp_flush_wqes", 0},
741 {IRDMA_OP_ADD_ARP_CACHE_ENTRY, "add_arp_cache_entry",
742 "add_arp_cache_entry", 0},
743 {IRDMA_OP_MANAGE_PUSH_PAGE, "manage_push_page", "manage_push_page", 0},
744 {IRDMA_OP_UPDATE_PE_SDS, "update_pe_sds", "update_pe_sds", 0},
745 {IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE, "manage_hmc_pm_func_table",
746 "manage_hmc_pm_func_table", 0},
747 {IRDMA_OP_SUSPEND, "suspend", "suspend", 0},
748 {IRDMA_OP_RESUME, "resume", "resume", 0},
749 {25, "manage_vchnl_req_pble_bp", "manage_vchnl_req_pble_bp", 0},
750 {IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0},
751 {IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0},
752 {IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0},
753 {IRDMA_OP_AH_MODIFY, "ah_modify", "ah_modify", 0},
754 {IRDMA_OP_AH_DESTROY, "ah_destroy", "ah_destroy", 0},
755 {IRDMA_OP_MC_CREATE, "mc_create", "mc_create", 0},
756 {IRDMA_OP_MC_DESTROY, "mc_destroy", "mc_destroy", 0},
757 {IRDMA_OP_MC_MODIFY, "mc_modify", "mc_modify", 0},
758 {IRDMA_OP_STATS_ALLOCATE, "stats_allocate", "stats_allocate", 0},
759 {IRDMA_OP_STATS_FREE, "stats_free", "stats_free", 0},
760 {IRDMA_OP_STATS_GATHER, "stats_gather", "stats_gather", 0},
761 {IRDMA_OP_WS_ADD_NODE, "ws_add_node", "ws_add_node", 0},
762 {IRDMA_OP_WS_MODIFY_NODE, "ws_modify_node", "ws_modify_node", 0},
763 {IRDMA_OP_WS_DELETE_NODE, "ws_delete_node", "ws_delete_node", 0},
764 {IRDMA_OP_WS_FAILOVER_START, "ws_failover_start", "ws_failover_start",
765 0},
766 {IRDMA_OP_WS_FAILOVER_COMPLETE, "ws_failover_complete",
767 "ws_failover_complete", 0},
768 {IRDMA_OP_SET_UP_MAP, "set_up_map", "set_up_map", 0},
769 {IRDMA_OP_GEN_AE, "gen_ae", "gen_ae", 0},
770 {IRDMA_OP_QUERY_RDMA_FEATURES, "query_rdma_features",
771 "query_rdma_features", 0},
772 {IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY, "alloc_local_mac_entry",
773 "alloc_local_mac_entry", 0},
774 {IRDMA_OP_ADD_LOCAL_MAC_ENTRY, "add_local_mac_entry",
775 "add_local_mac_entry", 0},
776 {IRDMA_OP_DELETE_LOCAL_MAC_ENTRY, "delete_local_mac_entry",
777 "delete_local_mac_entry", 0},
778 {IRDMA_OP_CQ_MODIFY, "cq_modify", "cq_modify", 0}
779 };
780
781 static const struct irdma_sw_stats_tunable_info irdma_cmcs_list[] = {
782 {0, "cm_nodes_created", "cm_nodes_created",
783 offsetof(struct irdma_cm_core, stats_nodes_created)},
784 {0, "cm_nodes_destroyed", "cm_nodes_destroyed",
785 offsetof(struct irdma_cm_core, stats_nodes_destroyed)},
786 {0, "cm_listen_created", "cm_listen_created",
787 offsetof(struct irdma_cm_core, stats_listen_created)},
788 {0, "cm_listen_destroyed", "cm_listen_destroyed",
789 offsetof(struct irdma_cm_core, stats_listen_destroyed)},
790 {0, "cm_listen_nodes_created", "cm_listen_nodes_created",
791 offsetof(struct irdma_cm_core, stats_listen_nodes_created)},
792 {0, "cm_listen_nodes_destroyed", "cm_listen_nodes_destroyed",
793 offsetof(struct irdma_cm_core, stats_listen_nodes_destroyed)},
794 {0, "cm_lpbs", "cm_lpbs", offsetof(struct irdma_cm_core, stats_lpbs)},
795 {0, "cm_accepts", "cm_accepts", offsetof(struct irdma_cm_core,
796 stats_accepts)},
797 {0, "cm_rejects", "cm_rejects", offsetof(struct irdma_cm_core,
798 stats_rejects)},
799 {0, "cm_connect_errs", "cm_connect_errs",
800 offsetof(struct irdma_cm_core, stats_connect_errs)},
801 {0, "cm_passive_errs", "cm_passive_errs",
802 offsetof(struct irdma_cm_core, stats_passive_errs)},
803 {0, "cm_pkt_retrans", "cm_pkt_retrans", offsetof(struct irdma_cm_core,
804 stats_pkt_retrans)},
805 {0, "cm_backlog_drops", "cm_backlog_drops",
806 offsetof(struct irdma_cm_core, stats_backlog_drops)},
807 };
808
809 static const struct irdma_sw_stats_tunable_info irdma_ilqs32_list[] = {
810 {0, "ilq_avail_buf_count", "ilq_avail_buf_count",
811 offsetof(struct irdma_puda_rsrc, avail_buf_count)},
812 {0, "ilq_alloc_buf_count", "ilq_alloc_buf_count",
813 offsetof(struct irdma_puda_rsrc, alloc_buf_count)}
814 };
815
816 static const struct irdma_sw_stats_tunable_info irdma_ilqs_list[] = {
817 {0, "ilq_stats_buf_alloc_fail", "ilq_stats_buf_alloc_fail",
818 offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)},
819 {0, "ilq_stats_pkt_rcvd", "ilq_stats_pkt_rcvd",
820 offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)},
821 {0, "ilq_stats_pkt_sent", "ilq_stats_pkt_sent",
822 offsetof(struct irdma_puda_rsrc, stats_pkt_sent)},
823 {0, "ilq_stats_rcvd_pkt_err", "ilq_stats_rcvd_pkt_err",
824 offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)},
825 {0, "ilq_stats_sent_pkt_q", "ilq_stats_sent_pkt_q",
826 offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)}
827 };
828
829 static const struct irdma_sw_stats_tunable_info irdma_ieqs32_list[] = {
830 {0, "ieq_avail_buf_count", "ieq_avail_buf_count",
831 offsetof(struct irdma_puda_rsrc, avail_buf_count)},
832 {0, "ieq_alloc_buf_count", "ieq_alloc_buf_count",
833 offsetof(struct irdma_puda_rsrc, alloc_buf_count)}
834 };
835
836 static const struct irdma_sw_stats_tunable_info irdma_ieqs_list[] = {
837 {0, "ieq_stats_buf_alloc_fail", "ieq_stats_buf_alloc_fail",
838 offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)},
839 {0, "ieq_stats_pkt_rcvd", "ieq_stats_pkt_rcvd",
840 offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)},
841 {0, "ieq_stats_pkt_sent", "ieq_stats_pkt_sent",
842 offsetof(struct irdma_puda_rsrc, stats_pkt_sent)},
843 {0, "ieq_stats_rcvd_pkt_err", "ieq_stats_rcvd_pkt_err",
844 offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)},
845 {0, "ieq_stats_sent_pkt_q", "ieq_stats_sent_pkt_q",
846 offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)},
847 {0, "ieq_stats_bad_qp_id", "ieq_stats_bad_qp_id",
848 offsetof(struct irdma_puda_rsrc, stats_bad_qp_id)},
849 {0, "ieq_fpdu_processed", "ieq_fpdu_processed",
850 offsetof(struct irdma_puda_rsrc, fpdu_processed)},
851 {0, "ieq_bad_seq_num", "ieq_bad_seq_num",
852 offsetof(struct irdma_puda_rsrc, bad_seq_num)},
853 {0, "ieq_crc_err", "ieq_crc_err", offsetof(struct irdma_puda_rsrc,
854 crc_err)},
855 {0, "ieq_pmode_count", "ieq_pmode_count",
856 offsetof(struct irdma_puda_rsrc, pmode_count)},
857 {0, "ieq_partials_handled", "ieq_partials_handled",
858 offsetof(struct irdma_puda_rsrc, partials_handled)},
859 };
860
861 /**
862 * irdma_dcqcn_tunables_init - create tunables for dcqcn settings
863 * @rf: RDMA PCI function
864 *
865 * Create DCQCN related sysctls for the driver.
866 * dcqcn_ena is writeable settings and applicable to next QP creation or
867 * context setting.
868 * all other settings are of RDTUN type (read on driver load) and are
869 * applicable only to CQP creation.
870 */
871 void
irdma_dcqcn_tunables_init(struct irdma_pci_f * rf)872 irdma_dcqcn_tunables_init(struct irdma_pci_f *rf)
873 {
874 struct sysctl_oid_list *irdma_sysctl_oid_list;
875
876 irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree);
877
878 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
879 OID_AUTO, "dcqcn_enable", CTLFLAG_RW | CTLTYPE_U8, rf, 0,
880 irdma_sysctl_dcqcn_update, "A",
881 "enables DCQCN algorithm for RoCEv2 on all ports, default=true");
882
883 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
884 OID_AUTO, "dcqcn_cc_cfg_valid", CTLFLAG_RDTUN,
885 &rf->dcqcn_params.cc_cfg_valid, 0,
886 "set DCQCN parameters to be valid, default=false");
887
888 rf->dcqcn_params.min_dec_factor = 1;
889 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
890 OID_AUTO, "dcqcn_min_dec_factor", CTLFLAG_RDTUN,
891 &rf->dcqcn_params.min_dec_factor, 0,
892 "set minimum percentage factor by which tx rate can be changed for CNP, Range: 1-100, default=1");
893
894 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
895 OID_AUTO, "dcqcn_min_rate_MBps", CTLFLAG_RDTUN,
896 &rf->dcqcn_params.min_rate, 0,
897 "set minimum rate limit value, in MBits per second, default=0");
898
899 rf->dcqcn_params.dcqcn_f = 5;
900 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
901 OID_AUTO, "dcqcn_F", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_f, 0,
902 "set number of times to stay in each stage of bandwidth recovery, default=5");
903
904 rf->dcqcn_params.dcqcn_t = 0x37;
905 SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
906 OID_AUTO, "dcqcn_T", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_t, 0,
907 "number of us to elapse before increasing the CWND in DCQCN mode, default=0x37");
908
909 rf->dcqcn_params.dcqcn_b = 0x249f0;
910 SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
911 OID_AUTO, "dcqcn_B", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_b, 0,
912 "set number of MSS to add to the congestion window in additive increase mode, default=0x249f0");
913
914 rf->dcqcn_params.rai_factor = 1;
915 SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
916 OID_AUTO, "dcqcn_rai_factor", CTLFLAG_RDTUN,
917 &rf->dcqcn_params.rai_factor, 0,
918 "set number of MSS to add to the congestion window in additive increase mode, default=1");
919
920 rf->dcqcn_params.hai_factor = 5;
921 SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
922 OID_AUTO, "dcqcn_hai_factor", CTLFLAG_RDTUN,
923 &rf->dcqcn_params.hai_factor, 0,
924 "set number of MSS to add to the congestion window in hyperactive increase mode, default=5");
925
926 rf->dcqcn_params.rreduce_mperiod = 50;
927 SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
928 OID_AUTO, "dcqcn_rreduce_mperiod", CTLFLAG_RDTUN,
929 &rf->dcqcn_params.rreduce_mperiod, 0,
930 "set minimum time between 2 consecutive rate reductions for a single flow, default=50");
931 }
932
933 /**
934 * irdma_sysctl_settings - sysctl runtime settings init
935 * @rf: RDMA PCI function
936 */
937 void
irdma_sysctl_settings(struct irdma_pci_f * rf)938 irdma_sysctl_settings(struct irdma_pci_f *rf)
939 {
940 struct sysctl_oid_list *irdma_sysctl_oid_list;
941 u8 ird_ord_limit;
942
943 irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree);
944
945 SYSCTL_ADD_BOOL(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
946 OID_AUTO, "upload_context", CTLFLAG_RWTUN,
947 &irdma_upload_context, 0,
948 "allow for generating QP's upload context, default=0");
949
950 if (rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
951 return;
952
953 #define ICRDMA_HW_IRD_ORD_LIMIT 128
954 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
955 OID_AUTO, "ord_max_value", CTLFLAG_RDTUN,
956 &irdma_sysctl_max_ord, ICRDMA_MAX_ORD_SIZE,
957 "Limit Outbound RDMA Read Queue Depth, dflt=32, max=128");
958
959 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
960 OID_AUTO, "ird_max_value", CTLFLAG_RDTUN,
961 &irdma_sysctl_max_ird, ICRDMA_MAX_IRD_SIZE,
962 "Limit Inbound RDMA Read Queue Depth, dflt=32, max=128");
963 /*
964 * Ensure the ird/ord is equal and not more than ICRDMA_HW_IRD_ORD_LIMIT
965 */
966 ird_ord_limit = min(irdma_sysctl_max_ord, irdma_sysctl_max_ird);
967 if (ird_ord_limit > ICRDMA_HW_IRD_ORD_LIMIT)
968 ird_ord_limit = ICRDMA_HW_IRD_ORD_LIMIT;
969 irdma_sysctl_max_ird = ird_ord_limit;
970 irdma_sysctl_max_ord = ird_ord_limit;
971
972 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
973 OID_AUTO, "mod_rdpu_bw", CTLFLAG_RDTUN,
974 &irdma_rdpu_bw_tun, 0,
975 "Turn off RDPU BW balance, default=0");
976 }
977
978 /**
979 * irdma_qos_info_tunables_init - init tunables to read qos settings
980 * @rf: RDMA PCI function
981 */
982 void
irdma_qos_info_tunables_init(struct irdma_pci_f * rf)983 irdma_qos_info_tunables_init(struct irdma_pci_f *rf)
984 {
985 struct irdma_sc_vsi *vsi = &rf->iwdev->vsi;
986 struct sysctl_oid_list *qos_oid_list;
987
988 qos_oid_list = SYSCTL_CHILDREN(rf->tun_info.qos_sysctl_tree);
989 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list,
990 OID_AUTO, "vsi_rel_bw", CTLFLAG_RD,
991 &vsi->qos_rel_bw, 0,
992 "qos_rel_bw");
993 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list,
994 OID_AUTO, "vsi_prio_type", CTLFLAG_RD,
995 &vsi->qos_prio_type, 0, "vsi prio type");
996 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list,
997 OID_AUTO, "dscp_mode", CTLFLAG_RD | CTLTYPE_STRING,
998 vsi, IRDMA_QOS_DSCP_MODE, irdma_sysctl_qos, "A",
999 "show dscp_mode");
1000 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list, OID_AUTO,
1001 "dscp_map", CTLFLAG_RD | CTLTYPE_STRING, vsi,
1002 IRDMA_QOS_DSCP_MAP, irdma_sysctl_qos, "A",
1003 "show dscp map");
1004 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list, OID_AUTO,
1005 "up2tc", CTLFLAG_RD | CTLTYPE_STRING, vsi,
1006 IRDMA_QOS_UP2TC, irdma_sysctl_qos, "A",
1007 "up to tc mapping");
1008 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list, OID_AUTO,
1009 "qs", CTLFLAG_RD | CTLTYPE_STRING, vsi,
1010 IRDMA_QOS_QS_HANDLE, irdma_sysctl_qos, "A",
1011 "qs_handle");
1012 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list, OID_AUTO,
1013 "tc", CTLFLAG_RD | CTLTYPE_STRING, vsi, IRDMA_QOS_TC,
1014 irdma_sysctl_qos, "A", "tc list");
1015 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list, OID_AUTO,
1016 "rel_bw", CTLFLAG_RD | CTLTYPE_STRING, vsi,
1017 IRDMA_QOS_REL_BW, irdma_sysctl_qos, "A", "relative bw");
1018 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, qos_oid_list, OID_AUTO,
1019 "prio_type", CTLFLAG_RD | CTLTYPE_STRING, vsi,
1020 IRDMA_QOS_PRIO_TYPE, irdma_sysctl_qos, "A",
1021 "prio_type");
1022 }
1023
1024 void
irdma_sw_stats_tunables_init(struct irdma_pci_f * rf)1025 irdma_sw_stats_tunables_init(struct irdma_pci_f *rf)
1026 {
1027 struct sysctl_oid_list *sws_oid_list;
1028 struct sysctl_ctx_list *irdma_ctx = &rf->tun_info.irdma_sysctl_ctx;
1029 struct irdma_sc_dev *dev = &rf->sc_dev;
1030 struct irdma_cm_core *cm_core = &rf->iwdev->cm_core;
1031 struct irdma_puda_rsrc *ilq = rf->iwdev->vsi.ilq;
1032 struct irdma_puda_rsrc *ieq = rf->iwdev->vsi.ieq;
1033 u64 *ll_ptr;
1034 u32 *l_ptr;
1035 int cqp_stat_cnt = sizeof(irdma_sws_list) / sizeof(struct irdma_sw_stats_tunable_info);
1036 int cmcore_stat_cnt = sizeof(irdma_cmcs_list) / sizeof(struct irdma_sw_stats_tunable_info);
1037 int ilqs_stat_cnt = sizeof(irdma_ilqs_list) / sizeof(struct irdma_sw_stats_tunable_info);
1038 int ilqs32_stat_cnt = sizeof(irdma_ilqs32_list) / sizeof(struct irdma_sw_stats_tunable_info);
1039 int ieqs_stat_cnt = sizeof(irdma_ieqs_list) / sizeof(struct irdma_sw_stats_tunable_info);
1040 int ieqs32_stat_cnt = sizeof(irdma_ieqs32_list) / sizeof(struct irdma_sw_stats_tunable_info);
1041 int i;
1042
1043 sws_oid_list = SYSCTL_CHILDREN(rf->tun_info.sws_sysctl_tree);
1044
1045 for (i = 0; i < cqp_stat_cnt; ++i) {
1046 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
1047 irdma_sws_list[i].name, CTLFLAG_RD,
1048 &dev->cqp_cmd_stats[irdma_sws_list[i].op_type],
1049 0, irdma_sws_list[i].desc);
1050 }
1051 SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO,
1052 "req_cmds", CTLFLAG_RD | CTLTYPE_STRING,
1053 dev->cqp, IRDMA_CQP_REQ_CMDS, irdma_sysctl_cqp_stats, "A",
1054 "req_cmds");
1055 SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO,
1056 "cmpl_cmds", CTLFLAG_RD | CTLTYPE_STRING,
1057 dev->cqp, IRDMA_CQP_CMPL_CMDS, irdma_sysctl_cqp_stats, "A",
1058 "cmpl_cmds");
1059 for (i = 0; i < cmcore_stat_cnt; ++i) {
1060 ll_ptr = (u64 *)((uintptr_t)cm_core + irdma_cmcs_list[i].value);
1061 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
1062 irdma_cmcs_list[i].name, CTLFLAG_RD, ll_ptr,
1063 0, irdma_cmcs_list[i].desc);
1064 }
1065 for (i = 0; ilq && i < ilqs_stat_cnt; ++i) {
1066 ll_ptr = (u64 *)((uintptr_t)ilq + irdma_ilqs_list[i].value);
1067 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
1068 irdma_ilqs_list[i].name, CTLFLAG_RD, ll_ptr,
1069 0, irdma_ilqs_list[i].desc);
1070 }
1071 for (i = 0; ilq && i < ilqs32_stat_cnt; ++i) {
1072 l_ptr = (u32 *)((uintptr_t)ilq + irdma_ilqs32_list[i].value);
1073 SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO,
1074 irdma_ilqs32_list[i].name, CTLFLAG_RD, l_ptr,
1075 0, irdma_ilqs32_list[i].desc);
1076 }
1077 for (i = 0; ieq && i < ieqs_stat_cnt; ++i) {
1078 ll_ptr = (u64 *)((uintptr_t)ieq + irdma_ieqs_list[i].value);
1079 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
1080 irdma_ieqs_list[i].name, CTLFLAG_RD, ll_ptr,
1081 0, irdma_ieqs_list[i].desc);
1082 }
1083 for (i = 0; ieq && i < ieqs32_stat_cnt; ++i) {
1084 l_ptr = (u32 *)((uintptr_t)ieq + irdma_ieqs32_list[i].value);
1085 SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO,
1086 irdma_ieqs32_list[i].name, CTLFLAG_RD, l_ptr,
1087 0, irdma_ieqs32_list[i].desc);
1088 }
1089 }
1090
1091 /**
1092 * irdma_dmamap_cb - callback for bus_dmamap_load
1093 */
1094 static void
irdma_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)1095 irdma_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
1096 {
1097 if (error)
1098 return;
1099 *(bus_addr_t *) arg = segs->ds_addr;
1100 return;
1101 }
1102
1103 /**
1104 * irdma_allocate_dma_mem - allocate dma memory
1105 * @hw: pointer to hw structure
1106 * @mem: structure holding memory information
1107 * @size: requested size
1108 * @alignment: requested alignment
1109 */
1110 void *
irdma_allocate_dma_mem(struct irdma_hw * hw,struct irdma_dma_mem * mem,u64 size,u32 alignment)1111 irdma_allocate_dma_mem(struct irdma_hw *hw, struct irdma_dma_mem *mem,
1112 u64 size, u32 alignment)
1113 {
1114 struct irdma_dev_ctx *dev_ctx = (struct irdma_dev_ctx *)hw->dev_context;
1115 device_t dev = dev_ctx->dev;
1116 void *va;
1117 int ret;
1118
1119 ret = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1120 alignment, 0, /* alignment, bounds */
1121 BUS_SPACE_MAXADDR, /* lowaddr */
1122 BUS_SPACE_MAXADDR, /* highaddr */
1123 NULL, NULL, /* filter, filterarg */
1124 size, /* maxsize */
1125 1, /* nsegments */
1126 size, /* maxsegsize */
1127 BUS_DMA_ALLOCNOW, /* flags */
1128 NULL, /* lockfunc */
1129 NULL, /* lockfuncarg */
1130 &mem->tag);
1131 if (ret != 0) {
1132 device_printf(dev, "%s: bus_dma_tag_create failed, error %u\n",
1133 __func__, ret);
1134 goto fail_0;
1135 }
1136 ret = bus_dmamem_alloc(mem->tag, (void **)&va,
1137 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &mem->map);
1138 if (ret != 0) {
1139 device_printf(dev, "%s: bus_dmamem_alloc failed, error %u\n",
1140 __func__, ret);
1141 goto fail_1;
1142 }
1143 ret = bus_dmamap_load(mem->tag, mem->map, va, size,
1144 irdma_dmamap_cb, &mem->pa, BUS_DMA_NOWAIT);
1145 if (ret != 0) {
1146 device_printf(dev, "%s: bus_dmamap_load failed, error %u\n",
1147 __func__, ret);
1148 goto fail_2;
1149 }
1150 mem->nseg = 1;
1151 mem->size = size;
1152 bus_dmamap_sync(mem->tag, mem->map,
1153 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1154
1155 return va;
1156 fail_2:
1157 bus_dmamem_free(mem->tag, va, mem->map);
1158 fail_1:
1159 bus_dma_tag_destroy(mem->tag);
1160 fail_0:
1161 mem->map = NULL;
1162 mem->tag = NULL;
1163
1164 return NULL;
1165 }
1166
1167 /**
1168 * irdma_free_dma_mem - Memory free helper fn
1169 * @hw: pointer to hw structure
1170 * @mem: ptr to mem struct to free
1171 */
1172 int
irdma_free_dma_mem(struct irdma_hw * hw,struct irdma_dma_mem * mem)1173 irdma_free_dma_mem(struct irdma_hw *hw, struct irdma_dma_mem *mem)
1174 {
1175 if (!mem)
1176 return -EINVAL;
1177 bus_dmamap_sync(mem->tag, mem->map,
1178 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1179 bus_dmamap_unload(mem->tag, mem->map);
1180 if (!mem->va)
1181 return -ENOMEM;
1182 bus_dmamem_free(mem->tag, mem->va, mem->map);
1183 bus_dma_tag_destroy(mem->tag);
1184
1185 mem->va = NULL;
1186
1187 return 0;
1188 }
1189
1190 void
irdma_cleanup_dead_qps(struct irdma_sc_vsi * vsi)1191 irdma_cleanup_dead_qps(struct irdma_sc_vsi *vsi)
1192 {
1193 struct irdma_sc_qp *qp = NULL;
1194 struct irdma_qp *iwqp;
1195 struct irdma_pci_f *rf;
1196 u8 i;
1197
1198 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
1199 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
1200 while (qp) {
1201 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_UDA) {
1202 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
1203 continue;
1204 }
1205 iwqp = qp->qp_uk.back_qp;
1206 rf = iwqp->iwdev->rf;
1207 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem);
1208 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem);
1209
1210 kfree(iwqp->kqp.sq_wrid_mem);
1211 kfree(iwqp->kqp.rq_wrid_mem);
1212 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
1213 kfree(iwqp);
1214 }
1215 }
1216 }
1217