1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (c) 2022 Microsoft Corporation. All rights reserved.
4  */
5 
6 #ifndef _MANA_IB_H_
7 #define _MANA_IB_H_
8 
9 #include <rdma/ib_verbs.h>
10 #include <rdma/ib_mad.h>
11 #include <rdma/ib_umem.h>
12 #include <rdma/mana-abi.h>
13 #include <rdma/uverbs_ioctl.h>
14 #include <linux/dmapool.h>
15 
16 #include <net/mana/mana.h>
17 #include "shadow_queue.h"
18 #include "counters.h"
19 
20 #define PAGE_SZ_BM                                                             \
21 	(SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K |        \
22 	 SZ_512K | SZ_1M | SZ_2M)
23 
24 /* MANA doesn't have any limit for MR size */
25 #define MANA_IB_MAX_MR_SIZE	U64_MAX
26 
27 /* Send queue ID mask */
28 #define MANA_SENDQ_MASK	BIT(31)
29 
30 /*
31  * The hardware limit of number of MRs is greater than maximum number of MRs
32  * that can possibly represent in 24 bits
33  */
34 #define MANA_IB_MAX_MR		0xFFFFFFu
35 
36 /*
37  * The CA timeout is approx. 260ms (4us * 2^(DELAY))
38  */
39 #define MANA_CA_ACK_DELAY	16
40 
41 /*
42  * The buffer used for writing AV
43  */
44 #define MANA_AV_BUFFER_SIZE	64
45 
46 struct mana_ib_adapter_caps {
47 	u32 max_sq_id;
48 	u32 max_rq_id;
49 	u32 max_cq_id;
50 	u32 max_qp_count;
51 	u32 max_cq_count;
52 	u32 max_mr_count;
53 	u32 max_pd_count;
54 	u32 max_inbound_read_limit;
55 	u32 max_outbound_read_limit;
56 	u32 mw_count;
57 	u32 max_srq_count;
58 	u32 max_qp_wr;
59 	u32 max_send_sge_count;
60 	u32 max_recv_sge_count;
61 	u32 max_inline_data_size;
62 	u64 feature_flags;
63 };
64 
65 struct mana_ib_queue {
66 	struct ib_umem *umem;
67 	struct gdma_queue *kmem;
68 	u64 gdma_region;
69 	u64 id;
70 };
71 
72 struct mana_ib_dev {
73 	struct ib_device ib_dev;
74 	struct gdma_dev *gdma_dev;
75 	mana_handle_t adapter_handle;
76 	struct gdma_queue *fatal_err_eq;
77 	struct gdma_queue **eqs;
78 	struct xarray qp_table_wq;
79 	struct mana_ib_adapter_caps adapter_caps;
80 	struct dma_pool *av_pool;
81 	netdevice_tracker dev_tracker;
82 	struct notifier_block nb;
83 };
84 
85 struct mana_ib_wq {
86 	struct ib_wq ibwq;
87 	struct mana_ib_queue queue;
88 	int wqe;
89 	u32 wq_buf_size;
90 	mana_handle_t rx_object;
91 };
92 
93 struct mana_ib_pd {
94 	struct ib_pd ibpd;
95 	u32 pdn;
96 	mana_handle_t pd_handle;
97 
98 	/* Mutex for sharing access to vport_use_count */
99 	struct mutex vport_mutex;
100 	int vport_use_count;
101 
102 	bool tx_shortform_allowed;
103 	u32 tx_vp_offset;
104 };
105 
106 struct mana_ib_av {
107 	u8 dest_ip[16];
108 	u8 dest_mac[ETH_ALEN];
109 	u16 udp_src_port;
110 	u8 src_ip[16];
111 	u32 hop_limit	: 8;
112 	u32 reserved1	: 12;
113 	u32 dscp	: 6;
114 	u32 reserved2	: 5;
115 	u32 is_ipv6	: 1;
116 	u32 reserved3	: 32;
117 };
118 
119 struct mana_ib_ah {
120 	struct ib_ah ibah;
121 	struct mana_ib_av *av;
122 	dma_addr_t dma_handle;
123 };
124 
125 struct mana_ib_mr {
126 	struct ib_mr ibmr;
127 	struct ib_umem *umem;
128 	mana_handle_t mr_handle;
129 };
130 
131 struct mana_ib_cq {
132 	struct ib_cq ibcq;
133 	struct mana_ib_queue queue;
134 	/* protects CQ polling */
135 	spinlock_t cq_lock;
136 	struct list_head list_send_qp;
137 	struct list_head list_recv_qp;
138 	int cqe;
139 	u32 comp_vector;
140 	mana_handle_t  cq_handle;
141 };
142 
143 enum mana_rc_queue_type {
144 	MANA_RC_SEND_QUEUE_REQUESTER = 0,
145 	MANA_RC_SEND_QUEUE_RESPONDER,
146 	MANA_RC_SEND_QUEUE_FMR,
147 	MANA_RC_RECV_QUEUE_REQUESTER,
148 	MANA_RC_RECV_QUEUE_RESPONDER,
149 	MANA_RC_QUEUE_TYPE_MAX,
150 };
151 
152 struct mana_ib_rc_qp {
153 	struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
154 };
155 
156 enum mana_ud_queue_type {
157 	MANA_UD_SEND_QUEUE = 0,
158 	MANA_UD_RECV_QUEUE,
159 	MANA_UD_QUEUE_TYPE_MAX,
160 };
161 
162 struct mana_ib_ud_qp {
163 	struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX];
164 	u32 sq_psn;
165 };
166 
167 struct mana_ib_qp {
168 	struct ib_qp ibqp;
169 
170 	mana_handle_t qp_handle;
171 	union {
172 		struct mana_ib_queue raw_sq;
173 		struct mana_ib_rc_qp rc_qp;
174 		struct mana_ib_ud_qp ud_qp;
175 	};
176 
177 	/* The port on the IB device, starting with 1 */
178 	u32 port;
179 
180 	struct list_head cq_send_list;
181 	struct list_head cq_recv_list;
182 	struct shadow_queue shadow_rq;
183 	struct shadow_queue shadow_sq;
184 
185 	refcount_t		refcount;
186 	struct completion	free;
187 };
188 
189 struct mana_ib_ucontext {
190 	struct ib_ucontext ibucontext;
191 	u32 doorbell;
192 };
193 
194 struct mana_ib_rwq_ind_table {
195 	struct ib_rwq_ind_table ib_ind_table;
196 };
197 
198 enum mana_ib_command_code {
199 	MANA_IB_GET_ADAPTER_CAP = 0x30001,
200 	MANA_IB_CREATE_ADAPTER  = 0x30002,
201 	MANA_IB_DESTROY_ADAPTER = 0x30003,
202 	MANA_IB_CONFIG_IP_ADDR	= 0x30004,
203 	MANA_IB_CONFIG_MAC_ADDR	= 0x30005,
204 	MANA_IB_CREATE_UD_QP	= 0x30006,
205 	MANA_IB_DESTROY_UD_QP	= 0x30007,
206 	MANA_IB_CREATE_CQ       = 0x30008,
207 	MANA_IB_DESTROY_CQ      = 0x30009,
208 	MANA_IB_CREATE_RC_QP    = 0x3000a,
209 	MANA_IB_DESTROY_RC_QP   = 0x3000b,
210 	MANA_IB_SET_QP_STATE	= 0x3000d,
211 	MANA_IB_QUERY_VF_COUNTERS = 0x30022,
212 };
213 
214 struct mana_ib_query_adapter_caps_req {
215 	struct gdma_req_hdr hdr;
216 }; /*HW Data */
217 
218 enum mana_ib_adapter_features {
219 	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
220 };
221 
222 struct mana_ib_query_adapter_caps_resp {
223 	struct gdma_resp_hdr hdr;
224 	u32 max_sq_id;
225 	u32 max_rq_id;
226 	u32 max_cq_id;
227 	u32 max_qp_count;
228 	u32 max_cq_count;
229 	u32 max_mr_count;
230 	u32 max_pd_count;
231 	u32 max_inbound_read_limit;
232 	u32 max_outbound_read_limit;
233 	u32 mw_count;
234 	u32 max_srq_count;
235 	u32 max_requester_sq_size;
236 	u32 max_responder_sq_size;
237 	u32 max_requester_rq_size;
238 	u32 max_responder_rq_size;
239 	u32 max_send_sge_count;
240 	u32 max_recv_sge_count;
241 	u32 max_inline_data_size;
242 	u64 feature_flags;
243 }; /* HW Data */
244 
245 enum mana_ib_adapter_features_request {
246 	MANA_IB_FEATURE_CLIENT_ERROR_CQE_REQUEST = BIT(1),
247 }; /*HW Data */
248 
249 struct mana_rnic_create_adapter_req {
250 	struct gdma_req_hdr hdr;
251 	u32 notify_eq_id;
252 	u32 reserved;
253 	u64 feature_flags;
254 }; /*HW Data */
255 
256 struct mana_rnic_create_adapter_resp {
257 	struct gdma_resp_hdr hdr;
258 	mana_handle_t adapter;
259 }; /* HW Data */
260 
261 struct mana_rnic_destroy_adapter_req {
262 	struct gdma_req_hdr hdr;
263 	mana_handle_t adapter;
264 }; /*HW Data */
265 
266 struct mana_rnic_destroy_adapter_resp {
267 	struct gdma_resp_hdr hdr;
268 }; /* HW Data */
269 
270 enum mana_ib_addr_op {
271 	ADDR_OP_ADD = 1,
272 	ADDR_OP_REMOVE = 2,
273 };
274 
275 enum sgid_entry_type {
276 	SGID_TYPE_IPV4 = 1,
277 	SGID_TYPE_IPV6 = 2,
278 };
279 
280 struct mana_rnic_config_addr_req {
281 	struct gdma_req_hdr hdr;
282 	mana_handle_t adapter;
283 	enum mana_ib_addr_op op;
284 	enum sgid_entry_type sgid_type;
285 	u8 ip_addr[16];
286 }; /* HW Data */
287 
288 struct mana_rnic_config_addr_resp {
289 	struct gdma_resp_hdr hdr;
290 }; /* HW Data */
291 
292 struct mana_rnic_config_mac_addr_req {
293 	struct gdma_req_hdr hdr;
294 	mana_handle_t adapter;
295 	enum mana_ib_addr_op op;
296 	u8 mac_addr[ETH_ALEN];
297 	u8 reserved[6];
298 }; /* HW Data */
299 
300 struct mana_rnic_config_mac_addr_resp {
301 	struct gdma_resp_hdr hdr;
302 }; /* HW Data */
303 
304 struct mana_rnic_create_cq_req {
305 	struct gdma_req_hdr hdr;
306 	mana_handle_t adapter;
307 	u64 gdma_region;
308 	u32 eq_id;
309 	u32 doorbell_page;
310 }; /* HW Data */
311 
312 struct mana_rnic_create_cq_resp {
313 	struct gdma_resp_hdr hdr;
314 	mana_handle_t cq_handle;
315 	u32 cq_id;
316 	u32 reserved;
317 }; /* HW Data */
318 
319 struct mana_rnic_destroy_cq_req {
320 	struct gdma_req_hdr hdr;
321 	mana_handle_t adapter;
322 	mana_handle_t cq_handle;
323 }; /* HW Data */
324 
325 struct mana_rnic_destroy_cq_resp {
326 	struct gdma_resp_hdr hdr;
327 }; /* HW Data */
328 
329 enum mana_rnic_create_rc_flags {
330 	MANA_RC_FLAG_NO_FMR = 2,
331 };
332 
333 struct mana_rnic_create_qp_req {
334 	struct gdma_req_hdr hdr;
335 	mana_handle_t adapter;
336 	mana_handle_t pd_handle;
337 	mana_handle_t send_cq_handle;
338 	mana_handle_t recv_cq_handle;
339 	u64 dma_region[MANA_RC_QUEUE_TYPE_MAX];
340 	u64 deprecated[2];
341 	u64 flags;
342 	u32 doorbell_page;
343 	u32 max_send_wr;
344 	u32 max_recv_wr;
345 	u32 max_send_sge;
346 	u32 max_recv_sge;
347 	u32 reserved;
348 }; /* HW Data */
349 
350 struct mana_rnic_create_qp_resp {
351 	struct gdma_resp_hdr hdr;
352 	mana_handle_t rc_qp_handle;
353 	u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX];
354 	u32 reserved;
355 }; /* HW Data*/
356 
357 struct mana_rnic_destroy_rc_qp_req {
358 	struct gdma_req_hdr hdr;
359 	mana_handle_t adapter;
360 	mana_handle_t rc_qp_handle;
361 }; /* HW Data */
362 
363 struct mana_rnic_destroy_rc_qp_resp {
364 	struct gdma_resp_hdr hdr;
365 }; /* HW Data */
366 
367 struct mana_rnic_create_udqp_req {
368 	struct gdma_req_hdr hdr;
369 	mana_handle_t adapter;
370 	mana_handle_t pd_handle;
371 	mana_handle_t send_cq_handle;
372 	mana_handle_t recv_cq_handle;
373 	u64 dma_region[MANA_UD_QUEUE_TYPE_MAX];
374 	u32 qp_type;
375 	u32 doorbell_page;
376 	u32 max_send_wr;
377 	u32 max_recv_wr;
378 	u32 max_send_sge;
379 	u32 max_recv_sge;
380 }; /* HW Data */
381 
382 struct mana_rnic_create_udqp_resp {
383 	struct gdma_resp_hdr hdr;
384 	mana_handle_t qp_handle;
385 	u32 queue_ids[MANA_UD_QUEUE_TYPE_MAX];
386 }; /* HW Data*/
387 
388 struct mana_rnic_destroy_udqp_req {
389 	struct gdma_req_hdr hdr;
390 	mana_handle_t adapter;
391 	mana_handle_t qp_handle;
392 }; /* HW Data */
393 
394 struct mana_rnic_destroy_udqp_resp {
395 	struct gdma_resp_hdr hdr;
396 }; /* HW Data */
397 
398 struct mana_ib_ah_attr {
399 	u8 src_addr[16];
400 	u8 dest_addr[16];
401 	u8 src_mac[ETH_ALEN];
402 	u8 dest_mac[ETH_ALEN];
403 	u8 src_addr_type;
404 	u8 dest_addr_type;
405 	u8 hop_limit;
406 	u8 traffic_class;
407 	u16 src_port;
408 	u16 dest_port;
409 	u32 reserved;
410 };
411 
412 struct mana_rnic_set_qp_state_req {
413 	struct gdma_req_hdr hdr;
414 	mana_handle_t adapter;
415 	mana_handle_t qp_handle;
416 	u64 attr_mask;
417 	u32 qp_state;
418 	u32 path_mtu;
419 	u32 rq_psn;
420 	u32 sq_psn;
421 	u32 dest_qpn;
422 	u32 max_dest_rd_atomic;
423 	u32 retry_cnt;
424 	u32 rnr_retry;
425 	u32 min_rnr_timer;
426 	u32 reserved;
427 	struct mana_ib_ah_attr ah_attr;
428 }; /* HW Data */
429 
430 struct mana_rnic_set_qp_state_resp {
431 	struct gdma_resp_hdr hdr;
432 }; /* HW Data */
433 
434 enum WQE_OPCODE_TYPES {
435 	WQE_TYPE_UD_SEND = 0,
436 	WQE_TYPE_UD_RECV = 8,
437 }; /* HW DATA */
438 
439 struct rdma_send_oob {
440 	u32 wqe_type	: 5;
441 	u32 fence	: 1;
442 	u32 signaled	: 1;
443 	u32 solicited	: 1;
444 	u32 psn		: 24;
445 
446 	u32 ssn_or_rqpn	: 24;
447 	u32 reserved1	: 8;
448 	union {
449 		struct {
450 			u32 remote_qkey;
451 			u32 immediate;
452 			u32 reserved1;
453 			u32 reserved2;
454 		} ud_send;
455 	};
456 }; /* HW DATA */
457 
458 struct mana_rdma_cqe {
459 	union {
460 		struct {
461 			u8 cqe_type;
462 			u8 data[GDMA_COMP_DATA_SIZE - 1];
463 		};
464 		struct {
465 			u32 cqe_type		: 8;
466 			u32 vendor_error	: 9;
467 			u32 reserved1		: 15;
468 			u32 sge_offset		: 5;
469 			u32 tx_wqe_offset	: 27;
470 		} ud_send;
471 		struct {
472 			u32 cqe_type		: 8;
473 			u32 reserved1		: 24;
474 			u32 msg_len;
475 			u32 src_qpn		: 24;
476 			u32 reserved2		: 8;
477 			u32 imm_data;
478 			u32 rx_wqe_offset;
479 		} ud_recv;
480 	};
481 }; /* HW DATA */
482 
483 struct mana_rnic_query_vf_cntrs_req {
484 	struct gdma_req_hdr hdr;
485 	mana_handle_t adapter;
486 }; /* HW Data */
487 
488 struct mana_rnic_query_vf_cntrs_resp {
489 	struct gdma_resp_hdr hdr;
490 	u64 requester_timeout;
491 	u64 requester_oos_nak;
492 	u64 requester_rnr_nak;
493 	u64 responder_rnr_nak;
494 	u64 responder_oos;
495 	u64 responder_dup_request;
496 	u64 requester_implicit_nak;
497 	u64 requester_readresp_psn_mismatch;
498 	u64 nak_inv_req;
499 	u64 nak_access_err;
500 	u64 nak_opp_err;
501 	u64 nak_inv_read;
502 	u64 responder_local_len_err;
503 	u64 requestor_local_prot_err;
504 	u64 responder_rem_access_err;
505 	u64 responder_local_qp_err;
506 	u64 responder_malformed_wqe;
507 	u64 general_hw_err;
508 	u64 requester_rnr_nak_retries_exceeded;
509 	u64 requester_retries_exceeded;
510 	u64 total_fatal_err;
511 	u64 received_cnps;
512 	u64 num_qps_congested;
513 	u64 rate_inc_events;
514 	u64 num_qps_recovered;
515 	u64 current_rate;
516 }; /* HW Data */
517 
mdev_to_gc(struct mana_ib_dev * mdev)518 static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
519 {
520 	return mdev->gdma_dev->gdma_context;
521 }
522 
mana_get_qp_ref(struct mana_ib_dev * mdev,u32 qid,bool is_sq)523 static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,
524 						 u32 qid, bool is_sq)
525 {
526 	struct mana_ib_qp *qp;
527 	unsigned long flag;
528 
529 	if (is_sq)
530 		qid |= MANA_SENDQ_MASK;
531 
532 	xa_lock_irqsave(&mdev->qp_table_wq, flag);
533 	qp = xa_load(&mdev->qp_table_wq, qid);
534 	if (qp)
535 		refcount_inc(&qp->refcount);
536 	xa_unlock_irqrestore(&mdev->qp_table_wq, flag);
537 	return qp;
538 }
539 
mana_put_qp_ref(struct mana_ib_qp * qp)540 static inline void mana_put_qp_ref(struct mana_ib_qp *qp)
541 {
542 	if (refcount_dec_and_test(&qp->refcount))
543 		complete(&qp->free);
544 }
545 
mana_ib_get_netdev(struct ib_device * ibdev,u32 port)546 static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
547 {
548 	struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
549 	struct gdma_context *gc = mdev_to_gc(mdev);
550 	struct mana_context *mc = gc->mana.driver_data;
551 
552 	if (port < 1 || port > mc->num_ports)
553 		return NULL;
554 	return mc->ports[port - 1];
555 }
556 
copy_in_reverse(u8 * dst,const u8 * src,u32 size)557 static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size)
558 {
559 	u32 i;
560 
561 	for (i = 0; i < size; i++)
562 		dst[size - 1 - i] = src[i];
563 }
564 
565 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
566 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
567 
568 int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
569 					  mana_handle_t *gdma_region);
570 
571 int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
572 			      mana_handle_t *gdma_region, u64 virt);
573 
574 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
575 				  mana_handle_t gdma_region);
576 
577 int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type,
578 				struct mana_ib_queue *queue);
579 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
580 			 struct mana_ib_queue *queue);
581 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
582 
583 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
584 				struct ib_wq_init_attr *init_attr,
585 				struct ib_udata *udata);
586 
587 int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
588 		      u32 wq_attr_mask, struct ib_udata *udata);
589 
590 int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata);
591 
592 int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
593 				 struct ib_rwq_ind_table_init_attr *init_attr,
594 				 struct ib_udata *udata);
595 
596 int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);
597 
598 struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags);
599 
600 struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
601 				  u64 iova, int access_flags,
602 				  struct ib_udata *udata);
603 
604 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
605 
606 int mana_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
607 		      struct ib_udata *udata);
608 
609 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
610 		      int attr_mask, struct ib_udata *udata);
611 
612 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
613 
614 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id,
615 		      struct mana_ib_pd *pd, u32 doorbell_id);
616 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
617 			 u32 port);
618 
619 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
620 		      struct uverbs_attr_bundle *attrs);
621 
622 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
623 
624 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
625 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
626 
627 int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
628 			   struct ib_udata *udata);
629 void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);
630 
631 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma);
632 
633 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
634 			       struct ib_port_immutable *immutable);
635 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
636 			 struct ib_udata *uhw);
637 int mana_ib_query_port(struct ib_device *ibdev, u32 port,
638 		       struct ib_port_attr *props);
639 int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
640 		      union ib_gid *gid);
641 
642 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);
643 
644 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev);
645 
646 int mana_ib_create_eqs(struct mana_ib_dev *mdev);
647 
648 void mana_ib_destroy_eqs(struct mana_ib_dev *mdev);
649 
650 int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev);
651 
652 int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev);
653 
654 int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);
655 
656 enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num);
657 
658 int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context);
659 
660 int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context);
661 
662 int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac);
663 
664 int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell);
665 
666 int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
667 
668 int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
669 			    struct ib_qp_init_attr *attr, u32 doorbell, u64 flags);
670 int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);
671 
672 int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
673 			    struct ib_qp_init_attr *attr, u32 doorbell, u32 type);
674 int mana_ib_gd_destroy_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);
675 
676 int mana_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
677 		      struct ib_udata *udata);
678 int mana_ib_destroy_ah(struct ib_ah *ah, u32 flags);
679 
680 int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
681 		      const struct ib_recv_wr **bad_wr);
682 int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
683 		      const struct ib_send_wr **bad_wr);
684 
685 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
686 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
687 
688 struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
689 					 u64 iova, int fd, int mr_access_flags,
690 					 struct uverbs_attr_bundle *attrs);
691 #endif
692