1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3
4 #ifndef _IONIC_IBDEV_H_
5 #define _IONIC_IBDEV_H_
6
7 #include <rdma/ib_umem.h>
8 #include <rdma/ib_verbs.h>
9 #include <rdma/ib_pack.h>
10 #include <rdma/uverbs_ioctl.h>
11
12 #include <rdma/ionic-abi.h>
13 #include <ionic_api.h>
14 #include <ionic_regs.h>
15
16 #include "ionic_fw.h"
17 #include "ionic_queue.h"
18 #include "ionic_res.h"
19
20 #include "ionic_lif_cfg.h"
21
22 /* Config knobs */
23 #define IONIC_EQ_DEPTH 511
24 #define IONIC_EQ_COUNT 32
25 #define IONIC_AQ_DEPTH 63
26 #define IONIC_AQ_COUNT 4
27 #define IONIC_EQ_ISR_BUDGET 10
28 #define IONIC_EQ_WORK_BUDGET 1000
29 #define IONIC_MAX_RD_ATOM 16
30 #define IONIC_PKEY_TBL_LEN 1
31 #define IONIC_GID_TBL_LEN 256
32
33 #define IONIC_MAX_QPID 0xffffff
34 #define IONIC_SPEC_HIGH 8
35 #define IONIC_MAX_PD 1024
36 #define IONIC_SPEC_HIGH 8
37 #define IONIC_SQCMB_ORDER 5
38 #define IONIC_RQCMB_ORDER 0
39
40 #define IONIC_META_LAST ((void *)1ul)
41 #define IONIC_META_POSTED ((void *)2ul)
42
43 #define IONIC_CQ_GRACE 100
44
45 #define IONIC_ROCE_UDP_SPORT 28272
46 #define IONIC_DMA_LKEY 0
47 #define IONIC_DMA_RKEY IONIC_DMA_LKEY
48
49 #define IONIC_CMB_SUPPORTED \
50 (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \
51 IONIC_CMB_WC | IONIC_CMB_UC)
52
53 /* resource is not reserved on the device, indicated in tbl_order */
54 #define IONIC_RES_INVALID -1
55
56 struct ionic_aq;
57 struct ionic_cq;
58 struct ionic_eq;
59 struct ionic_vcq;
60
61 enum ionic_admin_state {
62 IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */
63 IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */
64 IONIC_ADMIN_KILLED, /* not submitting, locally completed */
65 };
66
67 enum ionic_admin_flags {
68 IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */
69 IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */
70 IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */
71 };
72
73 enum ionic_mmap_flag {
74 IONIC_MMAP_WC = BIT(0),
75 };
76
77 struct ionic_mmap_entry {
78 struct rdma_user_mmap_entry rdma_entry;
79 unsigned long size;
80 unsigned long pfn;
81 u8 mmap_flags;
82 };
83
84 struct ionic_ibdev {
85 struct ib_device ibdev;
86
87 struct ionic_lif_cfg lif_cfg;
88
89 struct xarray qp_tbl;
90 struct xarray cq_tbl;
91
92 struct ionic_resid_bits inuse_dbid;
93 struct ionic_resid_bits inuse_pdid;
94 struct ionic_resid_bits inuse_ahid;
95 struct ionic_resid_bits inuse_mrid;
96 struct ionic_resid_bits inuse_qpid;
97 struct ionic_resid_bits inuse_cqid;
98
99 u8 half_cqid_udma_shift;
100 u8 half_qpid_udma_shift;
101 u8 next_qpid_udma_idx;
102 u8 next_mrkey;
103
104 struct work_struct reset_work;
105 bool reset_posted;
106 u32 reset_cnt;
107
108 struct delayed_work admin_dwork;
109 struct ionic_aq **aq_vec;
110 atomic_t admin_state;
111
112 struct ionic_eq **eq_vec;
113
114 struct ionic_v1_stat *hw_stats;
115 void *hw_stats_buf;
116 struct rdma_stat_desc *hw_stats_hdrs;
117 struct ionic_counter_stats *counter_stats;
118 int hw_stats_count;
119 };
120
121 struct ionic_eq {
122 struct ionic_ibdev *dev;
123
124 u32 eqid;
125 u32 intr;
126
127 struct ionic_queue q;
128
129 int armed;
130 bool enable;
131
132 struct work_struct work;
133
134 int irq;
135 char name[32];
136 };
137
138 struct ionic_admin_wr {
139 struct completion work;
140 struct list_head aq_ent;
141 struct ionic_v1_admin_wqe wqe;
142 struct ionic_v1_cqe cqe;
143 struct ionic_aq *aq;
144 int status;
145 };
146
147 struct ionic_admin_wr_q {
148 struct ionic_admin_wr *wr;
149 int wqe_strides;
150 };
151
152 struct ionic_aq {
153 struct ionic_ibdev *dev;
154 struct ionic_vcq *vcq;
155
156 struct work_struct work;
157
158 atomic_t admin_state;
159 unsigned long stamp;
160 bool armed;
161
162 u32 aqid;
163 u32 cqid;
164
165 spinlock_t lock; /* for posting */
166 struct ionic_queue q;
167 struct ionic_admin_wr_q *q_wr;
168 struct list_head wr_prod;
169 struct list_head wr_post;
170 };
171
172 struct ionic_ctx {
173 struct ib_ucontext ibctx;
174 u32 dbid;
175 struct rdma_user_mmap_entry *mmap_dbell;
176 };
177
178 struct ionic_tbl_buf {
179 u32 tbl_limit;
180 u32 tbl_pages;
181 size_t tbl_size;
182 __le64 *tbl_buf;
183 dma_addr_t tbl_dma;
184 u8 page_size_log2;
185 };
186
187 struct ionic_pd {
188 struct ib_pd ibpd;
189
190 u32 pdid;
191 u32 flags;
192 };
193
194 struct ionic_cq {
195 struct ionic_vcq *vcq;
196
197 u32 cqid;
198 u32 eqid;
199
200 spinlock_t lock; /* for polling */
201 struct list_head poll_sq;
202 bool flush;
203 struct list_head flush_sq;
204 struct list_head flush_rq;
205 struct list_head ibkill_flush_ent;
206
207 struct ionic_queue q;
208 bool color;
209 int credit;
210 u16 arm_any_prod;
211 u16 arm_sol_prod;
212
213 struct kref cq_kref;
214 struct completion cq_rel_comp;
215
216 /* infrequently accessed, keep at end */
217 struct ib_umem *umem;
218 };
219
220 struct ionic_vcq {
221 struct ib_cq ibcq;
222 struct ionic_cq cq[2];
223 u8 udma_mask;
224 u8 poll_idx;
225 };
226
227 struct ionic_sq_meta {
228 u64 wrid;
229 u32 len;
230 u16 seq;
231 u8 ibop;
232 u8 ibsts;
233 u8 remote:1;
234 u8 signal:1;
235 u8 local_comp:1;
236 };
237
238 struct ionic_rq_meta {
239 struct ionic_rq_meta *next;
240 u64 wrid;
241 };
242
243 struct ionic_qp {
244 struct ib_qp ibqp;
245 enum ib_qp_state state;
246
247 u32 qpid;
248 u32 ahid;
249 u32 sq_cqid;
250 u32 rq_cqid;
251 u8 udma_idx;
252 u8 has_ah:1;
253 u8 has_sq:1;
254 u8 has_rq:1;
255 u8 sig_all:1;
256
257 struct list_head qp_list_counter;
258
259 struct list_head cq_poll_sq;
260 struct list_head cq_flush_sq;
261 struct list_head cq_flush_rq;
262 struct list_head ibkill_flush_ent;
263
264 spinlock_t sq_lock; /* for posting and polling */
265 struct ionic_queue sq;
266 struct ionic_sq_meta *sq_meta;
267 u16 *sq_msn_idx;
268 int sq_spec;
269 u16 sq_old_prod;
270 u16 sq_msn_prod;
271 u16 sq_msn_cons;
272 u8 sq_cmb;
273 bool sq_flush;
274 bool sq_flush_rcvd;
275
276 spinlock_t rq_lock; /* for posting and polling */
277 struct ionic_queue rq;
278 struct ionic_rq_meta *rq_meta;
279 struct ionic_rq_meta *rq_meta_head;
280 int rq_spec;
281 u16 rq_old_prod;
282 u8 rq_cmb;
283 bool rq_flush;
284
285 struct kref qp_kref;
286 struct completion qp_rel_comp;
287
288 /* infrequently accessed, keep at end */
289 int sgid_index;
290 int sq_cmb_order;
291 u32 sq_cmb_pgid;
292 phys_addr_t sq_cmb_addr;
293 struct rdma_user_mmap_entry *mmap_sq_cmb;
294
295 struct ib_umem *sq_umem;
296
297 int rq_cmb_order;
298 u32 rq_cmb_pgid;
299 phys_addr_t rq_cmb_addr;
300 struct rdma_user_mmap_entry *mmap_rq_cmb;
301
302 struct ib_umem *rq_umem;
303
304 int dcqcn_profile;
305
306 struct ib_ud_header *hdr;
307 };
308
309 struct ionic_ah {
310 struct ib_ah ibah;
311 u32 ahid;
312 int sgid_index;
313 struct ib_ud_header hdr;
314 };
315
316 struct ionic_mr {
317 union {
318 struct ib_mr ibmr;
319 struct ib_mw ibmw;
320 };
321
322 u32 mrid;
323 int flags;
324
325 struct ib_umem *umem;
326 struct ionic_tbl_buf buf;
327 bool created;
328 };
329
330 struct ionic_counter_stats {
331 int queue_stats_count;
332 struct ionic_v1_stat *hdr;
333 struct rdma_stat_desc *stats_hdrs;
334 struct xarray xa_counters;
335 };
336
337 struct ionic_counter {
338 void *vals;
339 struct list_head qp_list;
340 };
341
to_ionic_ibdev(struct ib_device * ibdev)342 static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)
343 {
344 return container_of(ibdev, struct ionic_ibdev, ibdev);
345 }
346
to_ionic_ctx(struct ib_ucontext * ibctx)347 static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx)
348 {
349 return container_of(ibctx, struct ionic_ctx, ibctx);
350 }
351
to_ionic_ctx_uobj(struct ib_uobject * uobj)352 static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj)
353 {
354 if (!uobj)
355 return NULL;
356
357 if (!uobj->context)
358 return NULL;
359
360 return to_ionic_ctx(uobj->context);
361 }
362
to_ionic_pd(struct ib_pd * ibpd)363 static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd)
364 {
365 return container_of(ibpd, struct ionic_pd, ibpd);
366 }
367
to_ionic_mr(struct ib_mr * ibmr)368 static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr)
369 {
370 return container_of(ibmr, struct ionic_mr, ibmr);
371 }
372
to_ionic_mw(struct ib_mw * ibmw)373 static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw)
374 {
375 return container_of(ibmw, struct ionic_mr, ibmw);
376 }
377
to_ionic_vcq(struct ib_cq * ibcq)378 static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq)
379 {
380 return container_of(ibcq, struct ionic_vcq, ibcq);
381 }
382
to_ionic_vcq_cq(struct ib_cq * ibcq,uint8_t udma_idx)383 static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq,
384 uint8_t udma_idx)
385 {
386 return &to_ionic_vcq(ibcq)->cq[udma_idx];
387 }
388
to_ionic_qp(struct ib_qp * ibqp)389 static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)
390 {
391 return container_of(ibqp, struct ionic_qp, ibqp);
392 }
393
to_ionic_ah(struct ib_ah * ibah)394 static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)
395 {
396 return container_of(ibah, struct ionic_ah, ibah);
397 }
398
ionic_ctx_dbid(struct ionic_ibdev * dev,struct ionic_ctx * ctx)399 static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev,
400 struct ionic_ctx *ctx)
401 {
402 if (!ctx)
403 return dev->lif_cfg.dbid;
404
405 return ctx->dbid;
406 }
407
ionic_obj_dbid(struct ionic_ibdev * dev,struct ib_uobject * uobj)408 static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev,
409 struct ib_uobject *uobj)
410 {
411 return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));
412 }
413
ionic_ibop_is_local(enum ib_wr_opcode op)414 static inline bool ionic_ibop_is_local(enum ib_wr_opcode op)
415 {
416 return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;
417 }
418
ionic_qp_complete(struct kref * kref)419 static inline void ionic_qp_complete(struct kref *kref)
420 {
421 struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);
422
423 complete(&qp->qp_rel_comp);
424 }
425
ionic_cq_complete(struct kref * kref)426 static inline void ionic_cq_complete(struct kref *kref)
427 {
428 struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);
429
430 complete(&cq->cq_rel_comp);
431 }
432
433 /* ionic_admin.c */
434 extern struct workqueue_struct *ionic_evt_workq;
435 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
436 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,
437 enum ionic_admin_flags);
438
439 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);
440
441 int ionic_create_rdma_admin(struct ionic_ibdev *dev);
442 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);
443 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);
444
445 /* ionic_controlpath.c */
446 int ionic_create_cq_common(struct ionic_vcq *vcq,
447 struct ionic_tbl_buf *buf,
448 const struct ib_cq_init_attr *attr,
449 struct ionic_ctx *ctx,
450 struct ib_udata *udata,
451 struct ionic_qdesc *req_cq,
452 __u32 *resp_cqid,
453 int udma_idx);
454 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
455 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
456 void ionic_notify_flush_cq(struct ionic_cq *cq);
457
458 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);
459 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);
460 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);
461 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
462 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
463 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
464 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
465 struct ib_udata *udata);
466 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
467 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);
468 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);
469 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
470 u64 addr, int access, struct ib_dmah *dmah,
471 struct ib_udata *udata);
472 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
473 u64 length, u64 addr, int fd, int access,
474 struct ib_dmah *dmah,
475 struct uverbs_attr_bundle *attrs);
476 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
477 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
478 u32 max_sg);
479 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
480 unsigned int *sg_offset);
481 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
482 int ionic_dealloc_mw(struct ib_mw *ibmw);
483 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
484 struct uverbs_attr_bundle *attrs);
485 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
486 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
487 struct ib_udata *udata);
488 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
489 struct ib_udata *udata);
490 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
491 struct ib_qp_init_attr *init_attr);
492 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
493
494 /* ionic_datapath.c */
495 int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
496 const struct ib_send_wr **bad);
497 int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
498 const struct ib_recv_wr **bad);
499 int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);
500 int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
501
502 /* ionic_hw_stats.c */
503 void ionic_stats_init(struct ionic_ibdev *dev);
504 void ionic_stats_cleanup(struct ionic_ibdev *dev);
505
506 /* ionic_pgtbl.c */
507 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);
508 __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);
509 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);
510 int ionic_pgtbl_init(struct ionic_ibdev *dev,
511 struct ionic_tbl_buf *buf,
512 struct ib_umem *umem,
513 dma_addr_t dma,
514 int limit,
515 u64 page_size);
516 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);
517 #endif /* _IONIC_IBDEV_H_ */
518