1 /*
2 * Copyright (c) 2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 */
29
30 #ifndef __BNXT_RE_MAIN_H__
31 #define __BNXT_RE_MAIN_H__
32
33 #include <sys/param.h>
34 #include <sys/queue.h>
35
36 #include <infiniband/driver.h>
37 #include <infiniband/endian.h>
38 #include <infiniband/udma_barrier.h>
39
40 #include <inttypes.h>
41 #include <math.h>
42 #include <pthread.h>
43 #include <stdatomic.h>
44 #include <stdbool.h>
45 #include <stddef.h>
46 #include <stdio.h>
47 #include <stdlib.h>
48
49 #include "abi.h"
50 #include "list.h"
51 #include "memory.h"
52
53 #define DEV "bnxt_re : "
54 #define BNXT_RE_UD_QP_STALL 0x400000
55
56 #define CHIP_NUM_57508 0x1750
57 #define CHIP_NUM_57504 0x1751
58 #define CHIP_NUM_57502 0x1752
59 #define CHIP_NUM_58818 0xd818
60 #define CHIP_NUM_57608 0x1760
61
62 #define BNXT_NSEC_PER_SEC 1000000000UL
63
64 struct bnxt_re_chip_ctx {
65 __u16 chip_num;
66 __u8 chip_rev;
67 __u8 chip_metal;
68 bool chip_is_gen_p5_thor2;
69 };
70
71 #define BNXT_RE_MAP_WC 0x1000
72 #define BNXT_RE_DBR_PAGE 0x2000
73 #define BNXT_RE_DB_RECOVERY_PAGE 0x3000
74
75 #define BNXT_RE_DB_REPLAY_YIELD_CNT 256
76 #define BNXT_RE_DB_KEY_INVALID -1
77 #define BNXT_RE_MAX_DO_PACING 0xFFFF
78 #define bnxt_re_wm_barrier() udma_to_device_barrier()
79 #define unlikely(x) __builtin_expect(!!(x), 0)
80 #define likely(x) __builtin_expect(!!(x), 1)
81
82 #define CNA(v, d) \
83 { .vendor = PCI_VENDOR_ID_##v, \
84 .device = d }
85 #define BNXT_RE_DEFINE_CNA_TABLE(_name) \
86 static const struct { \
87 unsigned vendor; \
88 unsigned device; \
89 } _name[]
90
91 struct bnxt_re_dpi {
92 __u32 dpindx;
93 __u32 wcdpi;
94 __u64 *dbpage;
95 __u64 *wcdbpg;
96 };
97
98 struct bnxt_re_pd {
99 struct ibv_pd ibvpd;
100 uint32_t pdid;
101 };
102
103 struct xorshift32_state {
104 uint32_t seed;
105 };
106
107 struct bnxt_re_cq {
108 struct ibv_cq ibvcq;
109 struct bnxt_re_list_head sfhead;
110 struct bnxt_re_list_head rfhead;
111 struct bnxt_re_list_head prev_cq_head;
112 struct bnxt_re_context *cntx;
113 struct bnxt_re_queue *cqq;
114 struct bnxt_re_dpi *udpi;
115 struct bnxt_re_mem *resize_mem;
116 struct bnxt_re_mem *mem;
117 struct bnxt_re_list_node dbnode;
118 uint64_t shadow_db_key;
119 uint32_t cqe_sz;
120 uint32_t cqid;
121 struct xorshift32_state rand;
122 int deferred_arm_flags;
123 bool first_arm;
124 bool deferred_arm;
125 bool phase;
126 uint8_t dbr_lock;
127 void *cq_page;
128 };
129
130 struct bnxt_re_push_buffer {
131 __u64 *pbuf; /*push wc buffer */
132 __u64 *wqe; /* hwqe addresses */
133 __u64 *ucdb;
134 uint32_t st_idx;
135 uint32_t qpid;
136 uint16_t wcdpi;
137 uint16_t nbit;
138 uint32_t tail;
139 };
140
141 enum bnxt_re_push_info_mask {
142 BNXT_RE_PUSH_SIZE_MASK = 0x1FUL,
143 BNXT_RE_PUSH_SIZE_SHIFT = 0x18UL
144 };
145
146 struct bnxt_re_db_ppp_hdr {
147 struct bnxt_re_db_hdr db_hdr;
148 __u64 rsv_psz_pidx;
149 };
150
151 struct bnxt_re_push_rec {
152 struct bnxt_re_dpi *udpi;
153 struct bnxt_re_push_buffer *pbuf;
154 __u32 pbmap; /* only 16 bits in use */
155 };
156
157 struct bnxt_re_wrid {
158 uint64_t wrid;
159 int next_idx;
160 uint32_t bytes;
161 uint8_t sig;
162 uint8_t slots;
163 uint8_t wc_opcd;
164 };
165
166 struct bnxt_re_qpcap {
167 uint32_t max_swr;
168 uint32_t max_rwr;
169 uint32_t max_ssge;
170 uint32_t max_rsge;
171 uint32_t max_inline;
172 uint8_t sqsig;
173 uint8_t is_atomic_cap;
174 };
175
176 struct bnxt_re_srq {
177 struct ibv_srq ibvsrq;
178 struct ibv_srq_attr cap;
179 uint32_t srqid;
180 struct bnxt_re_context *uctx;
181 struct bnxt_re_queue *srqq;
182 struct bnxt_re_wrid *srwrid;
183 struct bnxt_re_dpi *udpi;
184 struct bnxt_re_mem *mem;
185 int start_idx;
186 int last_idx;
187 struct bnxt_re_list_node dbnode;
188 uint64_t shadow_db_key;
189 struct xorshift32_state rand;
190 uint8_t dbr_lock;
191 bool arm_req;
192 void *srq_page;
193 };
194
195 struct bnxt_re_joint_queue {
196 struct bnxt_re_context *cntx;
197 struct bnxt_re_queue *hwque;
198 struct bnxt_re_wrid *swque;
199 uint32_t start_idx;
200 uint32_t last_idx;
201 };
202
203 struct bnxt_re_qp {
204 struct ibv_qp ibvqp;
205 struct bnxt_re_qpcap cap;
206 struct bnxt_re_context *cntx;
207 struct bnxt_re_chip_ctx *cctx;
208 struct bnxt_re_joint_queue *jsqq;
209 struct bnxt_re_joint_queue *jrqq;
210 struct bnxt_re_dpi *udpi;
211 uint64_t wqe_cnt;
212 uint16_t mtu;
213 uint16_t qpst;
214 uint8_t qptyp;
215 uint8_t qpmode;
216 uint8_t push_st_en;
217 uint8_t ppp_idx;
218 uint32_t sq_psn;
219 uint32_t sq_msn;
220 uint32_t qpid;
221 uint16_t max_push_sz;
222 uint8_t sq_dbr_lock;
223 uint8_t rq_dbr_lock;
224 struct xorshift32_state rand;
225 struct bnxt_re_list_node snode;
226 struct bnxt_re_list_node rnode;
227 struct bnxt_re_srq *srq;
228 struct bnxt_re_cq *rcq;
229 struct bnxt_re_cq *scq;
230 struct bnxt_re_mem *mem;/* at cl 6 */
231 struct bnxt_re_list_node dbnode;
232 uint64_t sq_shadow_db_key;
233 uint64_t rq_shadow_db_key;
234 };
235
236 struct bnxt_re_mr {
237 struct ibv_mr vmr;
238 };
239
240 struct bnxt_re_ah {
241 struct ibv_ah ibvah;
242 struct bnxt_re_pd *pd;
243 uint32_t avid;
244 };
245
246 struct bnxt_re_dev {
247 struct verbs_device vdev;
248 struct ibv_device_attr devattr;
249 uint32_t pg_size;
250 uint32_t cqe_size;
251 uint32_t max_cq_depth;
252 uint8_t abi_version;
253 };
254
255 struct bnxt_re_res_list {
256 struct bnxt_re_list_head head;
257 pthread_spinlock_t lock;
258 };
259
260 struct bnxt_re_context {
261 struct ibv_context ibvctx;
262 struct bnxt_re_dev *rdev;
263 struct bnxt_re_chip_ctx *cctx;
264 uint64_t comp_mask;
265 struct bnxt_re_dpi udpi;
266 uint32_t dev_id;
267 uint32_t max_qp;
268 uint32_t max_srq;
269 uint32_t modes;
270 void *shpg;
271 pthread_mutex_t shlock;
272 struct bnxt_re_push_rec *pbrec;
273 void *dbr_page;
274 void *bar_map;
275 struct bnxt_re_res_list qp_dbr_res;
276 struct bnxt_re_res_list cq_dbr_res;
277 struct bnxt_re_res_list srq_dbr_res;
278 void *db_recovery_page;
279 struct ibv_comp_channel *dbr_ev_chan;
280 struct ibv_cq *dbr_cq;
281 pthread_t dbr_thread;
282 uint64_t replay_cnt;
283 };
284
285 struct bnxt_re_pacing_data {
286 uint32_t do_pacing;
287 uint32_t pacing_th;
288 uint32_t dev_err_state;
289 uint32_t alarm_th;
290 };
291
292 /* Chip context related functions */
293 bool _is_chip_gen_p5(struct bnxt_re_chip_ctx *cctx);
294 bool _is_chip_a0(struct bnxt_re_chip_ctx *cctx);
295 bool _is_chip_thor2(struct bnxt_re_chip_ctx *cctx);
296 bool _is_chip_gen_p5_thor2(struct bnxt_re_chip_ctx *cctx);
297
298 /* DB ring functions used internally*/
299 void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp);
300 void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp);
301 void bnxt_re_ring_srq_arm(struct bnxt_re_srq *srq);
302 void bnxt_re_ring_srq_db(struct bnxt_re_srq *srq);
303 void bnxt_re_ring_cq_db(struct bnxt_re_cq *cq);
304 void bnxt_re_ring_cq_arm_db(struct bnxt_re_cq *cq, uint8_t aflag);
305
306 void bnxt_re_ring_pstart_db(struct bnxt_re_qp *qp,
307 struct bnxt_re_push_buffer *pbuf);
308 void bnxt_re_ring_pend_db(struct bnxt_re_qp *qp,
309 struct bnxt_re_push_buffer *pbuf);
310 void bnxt_re_fill_push_wcb(struct bnxt_re_qp *qp,
311 struct bnxt_re_push_buffer *pbuf,
312 uint32_t idx);
313
314 void bnxt_re_fill_ppp(struct bnxt_re_push_buffer *pbuf,
315 struct bnxt_re_qp *qp, uint8_t len, uint32_t idx);
316 int bnxt_re_init_pbuf_list(struct bnxt_re_context *cntx);
317 void bnxt_re_destroy_pbuf_list(struct bnxt_re_context *cntx);
318 struct bnxt_re_push_buffer *bnxt_re_get_pbuf(uint8_t *push_st_en,
319 uint8_t ppp_idx,
320 struct bnxt_re_context *cntx);
321 void bnxt_re_put_pbuf(struct bnxt_re_context *cntx,
322 struct bnxt_re_push_buffer *pbuf);
323
324 void bnxt_re_db_recovery(struct bnxt_re_context *cntx);
325 void *bnxt_re_dbr_thread(void *arg);
326 bool _is_db_drop_recovery_enable(struct bnxt_re_context *cntx);
327 int bnxt_re_poll_kernel_cq(struct bnxt_re_cq *cq);
328 extern int bnxt_single_threaded;
329 extern int bnxt_dyn_debug;
330
331 #define bnxt_re_trace(fmt, ...) \
332 { \
333 if (bnxt_dyn_debug) \
334 fprintf(stderr, fmt, ##__VA_ARGS__); \
335 }
336
337 /* pointer conversion functions*/
to_bnxt_re_dev(struct ibv_device * ibvdev)338 static inline struct bnxt_re_dev *to_bnxt_re_dev(struct ibv_device *ibvdev)
339 {
340 return container_of(ibvdev, struct bnxt_re_dev, vdev);
341 }
342
to_bnxt_re_context(struct ibv_context * ibvctx)343 static inline struct bnxt_re_context *to_bnxt_re_context(
344 struct ibv_context *ibvctx)
345 {
346 return container_of(ibvctx, struct bnxt_re_context, ibvctx);
347 }
348
to_bnxt_re_pd(struct ibv_pd * ibvpd)349 static inline struct bnxt_re_pd *to_bnxt_re_pd(struct ibv_pd *ibvpd)
350 {
351 return container_of(ibvpd, struct bnxt_re_pd, ibvpd);
352 }
353
to_bnxt_re_cq(struct ibv_cq * ibvcq)354 static inline struct bnxt_re_cq *to_bnxt_re_cq(struct ibv_cq *ibvcq)
355 {
356 return container_of(ibvcq, struct bnxt_re_cq, ibvcq);
357 }
358
to_bnxt_re_qp(struct ibv_qp * ibvqp)359 static inline struct bnxt_re_qp *to_bnxt_re_qp(struct ibv_qp *ibvqp)
360 {
361 return container_of(ibvqp, struct bnxt_re_qp, ibvqp);
362 }
363
to_bnxt_re_srq(struct ibv_srq * ibvsrq)364 static inline struct bnxt_re_srq *to_bnxt_re_srq(struct ibv_srq *ibvsrq)
365 {
366 return container_of(ibvsrq, struct bnxt_re_srq, ibvsrq);
367 }
368
to_bnxt_re_ah(struct ibv_ah * ibvah)369 static inline struct bnxt_re_ah *to_bnxt_re_ah(struct ibv_ah *ibvah)
370 {
371 return container_of(ibvah, struct bnxt_re_ah, ibvah);
372 }
373
374 /* CQE manipulations */
375 #define bnxt_re_get_cqe_sz() (sizeof(struct bnxt_re_req_cqe) + \
376 sizeof(struct bnxt_re_bcqe))
377 #define bnxt_re_get_sqe_hdr_sz() (sizeof(struct bnxt_re_bsqe) + \
378 sizeof(struct bnxt_re_send))
379 #define bnxt_re_get_srqe_hdr_sz() (sizeof(struct bnxt_re_brqe) + \
380 sizeof(struct bnxt_re_srqe))
381 #define bnxt_re_get_srqe_sz() (sizeof(struct bnxt_re_brqe) + \
382 sizeof(struct bnxt_re_srqe) + \
383 BNXT_RE_MAX_INLINE_SIZE)
384 #define bnxt_re_is_cqe_valid(valid, phase) \
385 (((valid) & BNXT_RE_BCQE_PH_MASK) == (phase))
386
bnxt_re_change_cq_phase(struct bnxt_re_cq * cq)387 static inline void bnxt_re_change_cq_phase(struct bnxt_re_cq *cq)
388 {
389 if (!cq->cqq->head)
390 cq->phase = !(cq->phase & BNXT_RE_BCQE_PH_MASK);
391 }
392
bnxt_re_get_swqe(struct bnxt_re_joint_queue * jqq,uint32_t * wqe_idx)393 static inline void *bnxt_re_get_swqe(struct bnxt_re_joint_queue *jqq,
394 uint32_t *wqe_idx)
395 {
396 if (wqe_idx)
397 *wqe_idx = jqq->start_idx;
398 return &jqq->swque[jqq->start_idx];
399 }
400
bnxt_re_jqq_mod_start(struct bnxt_re_joint_queue * jqq,uint32_t idx)401 static inline void bnxt_re_jqq_mod_start(struct bnxt_re_joint_queue *jqq,
402 uint32_t idx)
403 {
404 jqq->start_idx = jqq->swque[idx].next_idx;
405 }
406
bnxt_re_jqq_mod_last(struct bnxt_re_joint_queue * jqq,uint32_t idx)407 static inline void bnxt_re_jqq_mod_last(struct bnxt_re_joint_queue *jqq,
408 uint32_t idx)
409 {
410 jqq->last_idx = jqq->swque[idx].next_idx;
411 }
412
bnxt_re_init_depth(uint32_t ent,uint64_t cmask)413 static inline uint32_t bnxt_re_init_depth(uint32_t ent, uint64_t cmask)
414 {
415 return cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED ?
416 ent : roundup_pow_of_two(ent);
417 }
418
bnxt_re_get_diff(uint64_t cmask)419 static inline uint32_t bnxt_re_get_diff(uint64_t cmask)
420 {
421 return cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED ?
422 0 : BNXT_RE_FULL_FLAG_DELTA;
423 }
424
bnxt_re_calc_wqe_sz(int nsge)425 static inline int bnxt_re_calc_wqe_sz(int nsge)
426 {
427 /* This is used for both sq and rq. In case hdr size differs
428 * in future move to individual functions.
429 */
430 return sizeof(struct bnxt_re_sge) * nsge + bnxt_re_get_sqe_hdr_sz();
431 }
432
433 /* Helper function to copy to push buffers */
bnxt_re_copy_data_to_pb(struct bnxt_re_push_buffer * pbuf,uint8_t offset,uint32_t idx)434 static inline void bnxt_re_copy_data_to_pb(struct bnxt_re_push_buffer *pbuf,
435 uint8_t offset, uint32_t idx)
436 {
437 __u64 *src;
438 __u64 *dst;
439 int indx;
440
441 for (indx = 0; indx < idx; indx++) {
442 dst = (__u64 *)(pbuf->pbuf + 2 * (indx + offset));
443 src = (__u64 *)pbuf->wqe[indx];
444 iowrite64(dst, *src);
445
446 dst++;
447 src++;
448 iowrite64(dst, *src);
449 }
450 }
451
bnxt_re_dp_spin_init(struct bnxt_spinlock * lock,int pshared,int need_lock)452 static inline int bnxt_re_dp_spin_init(struct bnxt_spinlock *lock, int pshared, int need_lock)
453 {
454 lock->in_use = 0;
455 lock->need_lock = need_lock;
456 return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE);
457 }
458
bnxt_re_dp_spin_destroy(struct bnxt_spinlock * lock)459 static inline int bnxt_re_dp_spin_destroy(struct bnxt_spinlock *lock)
460 {
461 return pthread_spin_destroy(&lock->lock);
462 }
463
bnxt_spin_lock(struct bnxt_spinlock * lock)464 static inline int bnxt_spin_lock(struct bnxt_spinlock *lock)
465 {
466 if (lock->need_lock)
467 return pthread_spin_lock(&lock->lock);
468
469 if (unlikely(lock->in_use)) {
470 fprintf(stderr, "*** ERROR: multithreading violation ***\n"
471 "You are running a multithreaded application but\n"
472 "you set BNXT_SINGLE_THREADED=1. Please unset it.\n");
473 abort();
474 } else {
475 lock->in_use = 1;
476 /* This fence is not at all correct, but it increases the */
477 /* chance that in_use is detected by another thread without */
478 /* much runtime cost. */
479 atomic_thread_fence(memory_order_acq_rel);
480 }
481
482 return 0;
483 }
484
bnxt_spin_unlock(struct bnxt_spinlock * lock)485 static inline int bnxt_spin_unlock(struct bnxt_spinlock *lock)
486 {
487 if (lock->need_lock)
488 return pthread_spin_unlock(&lock->lock);
489
490 lock->in_use = 0;
491 return 0;
492 }
493
timespec_sub(const struct timespec * a,const struct timespec * b,struct timespec * res)494 static void timespec_sub(const struct timespec *a, const struct timespec *b,
495 struct timespec *res)
496 {
497 res->tv_sec = a->tv_sec - b->tv_sec;
498 res->tv_nsec = a->tv_nsec - b->tv_nsec;
499 if (res->tv_nsec < 0) {
500 res->tv_sec--;
501 res->tv_nsec += BNXT_NSEC_PER_SEC;
502 }
503 }
504
505 /*
506 * Function waits in a busy loop for a given nano seconds
507 * The maximum wait period allowed is less than one second
508 */
bnxt_re_sub_sec_busy_wait(uint32_t nsec)509 static inline void bnxt_re_sub_sec_busy_wait(uint32_t nsec)
510 {
511 struct timespec start, cur, res;
512
513 if (nsec >= BNXT_NSEC_PER_SEC)
514 return;
515
516 if (clock_gettime(CLOCK_REALTIME, &start)) {
517 fprintf(stderr, "%s: failed to get time : %d",
518 __func__, errno);
519 return;
520 }
521
522 while (1) {
523 if (clock_gettime(CLOCK_REALTIME, &cur)) {
524 fprintf(stderr, "%s: failed to get time : %d",
525 __func__, errno);
526 return;
527 }
528
529 timespec_sub(&cur, &start, &res);
530 if (res.tv_nsec >= nsec)
531 break;
532 }
533 }
534
535 #define BNXT_RE_HW_RETX(a) ((a)->comp_mask & BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED)
536 #define bnxt_re_dp_spin_lock(lock) bnxt_spin_lock(lock)
537 #define bnxt_re_dp_spin_unlock(lock) bnxt_spin_unlock(lock)
538
539 #endif
540