1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_VERBS_H
8 #define RXE_VERBS_H
9
10 #include <linux/interrupt.h>
11 #include <linux/workqueue.h>
12 #include "rxe_pool.h"
13 #include "rxe_task.h"
14 #include "rxe_hw_counters.h"
15
pkey_match(u16 key1,u16 key2)16 static inline int pkey_match(u16 key1, u16 key2)
17 {
18 return (((key1 & 0x7fff) != 0) &&
19 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
20 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
21 }
22
23 /* Return >0 if psn_a > psn_b
24 * 0 if psn_a == psn_b
25 * <0 if psn_a < psn_b
26 */
psn_compare(u32 psn_a,u32 psn_b)27 static inline int psn_compare(u32 psn_a, u32 psn_b)
28 {
29 s32 diff;
30
31 diff = (psn_a - psn_b) << 8;
32 return diff;
33 }
34
35 struct rxe_ucontext {
36 struct ib_ucontext ibuc;
37 struct rxe_pool_elem elem;
38 };
39
40 struct rxe_pd {
41 struct ib_pd ibpd;
42 struct rxe_pool_elem elem;
43 };
44
45 struct rxe_ah {
46 struct ib_ah ibah;
47 struct rxe_pool_elem elem;
48 struct rxe_av av;
49 bool is_user;
50 int ah_num;
51 };
52
53 struct rxe_cqe {
54 union {
55 struct ib_wc ibwc;
56 struct ib_uverbs_wc uibwc;
57 };
58 };
59
60 struct rxe_cq {
61 struct ib_cq ibcq;
62 struct rxe_pool_elem elem;
63 struct rxe_queue *queue;
64 spinlock_t cq_lock;
65 u8 notify;
66 bool is_user;
67 atomic_t num_wq;
68 };
69
70 enum wqe_state {
71 wqe_state_posted,
72 wqe_state_processing,
73 wqe_state_pending,
74 wqe_state_done,
75 wqe_state_error,
76 };
77
78 struct rxe_sq {
79 int max_wr;
80 int max_sge;
81 int max_inline;
82 spinlock_t sq_lock; /* guard queue */
83 struct rxe_queue *queue;
84 };
85
86 struct rxe_rq {
87 int max_wr;
88 int max_sge;
89 spinlock_t producer_lock; /* guard queue producer */
90 spinlock_t consumer_lock; /* guard queue consumer */
91 struct rxe_queue *queue;
92 };
93
94 struct rxe_srq {
95 struct ib_srq ibsrq;
96 struct rxe_pool_elem elem;
97 struct rxe_pd *pd;
98 struct rxe_rq rq;
99 u32 srq_num;
100
101 int limit;
102 int error;
103 };
104
105 struct rxe_req_info {
106 int wqe_index;
107 u32 psn;
108 int opcode;
109 atomic_t rd_atomic;
110 int wait_fence;
111 int need_rd_atomic;
112 int wait_psn;
113 int need_retry;
114 int wait_for_rnr_timer;
115 int noack_pkts;
116 int again;
117 };
118
119 struct rxe_comp_info {
120 u32 psn;
121 int opcode;
122 int timeout;
123 int timeout_retry;
124 int started_retry;
125 u32 retry_cnt;
126 u32 rnr_retry;
127 };
128
129 /* responder states */
130 enum resp_states {
131 RESPST_NONE,
132 RESPST_GET_REQ,
133 RESPST_CHK_PSN,
134 RESPST_CHK_OP_SEQ,
135 RESPST_CHK_OP_VALID,
136 RESPST_CHK_RESOURCE,
137 RESPST_CHK_LENGTH,
138 RESPST_CHK_RKEY,
139 RESPST_EXECUTE,
140 RESPST_READ_REPLY,
141 RESPST_ATOMIC_REPLY,
142 RESPST_ATOMIC_WRITE_REPLY,
143 RESPST_PROCESS_FLUSH,
144 RESPST_COMPLETE,
145 RESPST_ACKNOWLEDGE,
146 RESPST_CLEANUP,
147 RESPST_DUPLICATE_REQUEST,
148 RESPST_ERR_MALFORMED_WQE,
149 RESPST_ERR_UNSUPPORTED_OPCODE,
150 RESPST_ERR_MISALIGNED_ATOMIC,
151 RESPST_ERR_PSN_OUT_OF_SEQ,
152 RESPST_ERR_MISSING_OPCODE_FIRST,
153 RESPST_ERR_MISSING_OPCODE_LAST_C,
154 RESPST_ERR_MISSING_OPCODE_LAST_D1E,
155 RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
156 RESPST_ERR_RNR,
157 RESPST_ERR_RKEY_VIOLATION,
158 RESPST_ERR_INVALIDATE_RKEY,
159 RESPST_ERR_LENGTH,
160 RESPST_ERR_CQ_OVERFLOW,
161 RESPST_ERROR,
162 RESPST_DONE,
163 RESPST_EXIT,
164 };
165
166 enum rdatm_res_state {
167 rdatm_res_state_next,
168 rdatm_res_state_new,
169 rdatm_res_state_replay,
170 };
171
172 struct resp_res {
173 int type;
174 int replay;
175 u32 first_psn;
176 u32 last_psn;
177 u32 cur_psn;
178 enum rdatm_res_state state;
179
180 union {
181 struct {
182 u64 orig_val;
183 } atomic;
184 struct {
185 u64 va_org;
186 u32 rkey;
187 u32 length;
188 u64 va;
189 u32 resid;
190 } read;
191 struct {
192 u32 length;
193 u64 va;
194 u8 type;
195 u8 level;
196 } flush;
197 };
198 };
199
200 struct rxe_resp_info {
201 u32 msn;
202 u32 psn;
203 u32 ack_psn;
204 int opcode;
205 int drop_msg;
206 int goto_error;
207 int sent_psn_nak;
208 enum ib_wc_status status;
209 u8 aeth_syndrome;
210
211 /* Receive only */
212 struct rxe_recv_wqe *wqe;
213
214 /* RDMA read / atomic only */
215 u64 va;
216 u64 offset;
217 struct rxe_mr *mr;
218 u32 resid;
219 u32 rkey;
220 u32 length;
221
222 /* SRQ only */
223 struct {
224 struct rxe_recv_wqe wqe;
225 struct ib_sge sge[RXE_MAX_SGE];
226 } srq_wqe;
227
228 /* Responder resources. It's a circular list where the oldest
229 * resource is dropped first.
230 */
231 struct resp_res *resources;
232 unsigned int res_head;
233 unsigned int res_tail;
234 struct resp_res *res;
235 };
236
237 struct rxe_qp {
238 struct ib_qp ibqp;
239 struct rxe_pool_elem elem;
240 struct ib_qp_attr attr;
241 unsigned int valid;
242 unsigned int mtu;
243 bool is_user;
244
245 struct rxe_pd *pd;
246 struct rxe_srq *srq;
247 struct rxe_cq *scq;
248 struct rxe_cq *rcq;
249
250 enum ib_sig_type sq_sig_type;
251
252 struct rxe_sq sq;
253 struct rxe_rq rq;
254
255 struct socket *sk;
256 u32 dst_cookie;
257 u16 src_port;
258
259 struct rxe_av pri_av;
260 struct rxe_av alt_av;
261
262 atomic_t mcg_num;
263
264 struct sk_buff_head req_pkts;
265 struct sk_buff_head resp_pkts;
266
267 struct rxe_task send_task;
268 struct rxe_task recv_task;
269
270 struct rxe_req_info req;
271 struct rxe_comp_info comp;
272 struct rxe_resp_info resp;
273
274 atomic_t ssn;
275 atomic_t skb_out;
276 int need_req_skb;
277
278 /* Timer for retranmitting packet when ACKs have been lost. RC
279 * only. The requester sets it when it is not already
280 * started. The responder resets it whenever an ack is
281 * received.
282 */
283 struct timer_list retrans_timer;
284 u64 qp_timeout_jiffies;
285
286 /* Timer for handling RNR NAKS. */
287 struct timer_list rnr_nak_timer;
288
289 spinlock_t state_lock; /* guard requester and completer */
290
291 struct execute_work cleanup_work;
292 };
293
294 enum {
295 RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ
296 | IB_ACCESS_REMOTE_WRITE
297 | IB_ACCESS_REMOTE_ATOMIC,
298 RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE
299 | IB_ACCESS_LOCAL_WRITE
300 | IB_ACCESS_MW_BIND
301 | IB_ACCESS_ON_DEMAND
302 | IB_ACCESS_FLUSH_GLOBAL
303 | IB_ACCESS_FLUSH_PERSISTENT
304 | IB_ACCESS_OPTIONAL,
305 RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR,
306 RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR
307 | IB_ZERO_BASED,
308 };
309
310 enum rxe_mr_state {
311 RXE_MR_STATE_INVALID,
312 RXE_MR_STATE_FREE,
313 RXE_MR_STATE_VALID,
314 };
315
316 enum rxe_mr_copy_dir {
317 RXE_TO_MR_OBJ,
318 RXE_FROM_MR_OBJ,
319 };
320
321 enum rxe_mr_lookup_type {
322 RXE_LOOKUP_LOCAL,
323 RXE_LOOKUP_REMOTE,
324 };
325
326 enum rxe_rereg {
327 RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD
328 | IB_MR_REREG_ACCESS,
329 };
330
rkey_is_mw(u32 rkey)331 static inline int rkey_is_mw(u32 rkey)
332 {
333 u32 index = rkey >> 8;
334
335 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
336 }
337
338 struct rxe_mr_page {
339 struct page *page;
340 unsigned int offset; /* offset in system page */
341 };
342
343 struct rxe_mr {
344 struct rxe_pool_elem elem;
345 struct ib_mr ibmr;
346
347 struct ib_umem *umem;
348
349 u32 lkey;
350 u32 rkey;
351 enum rxe_mr_state state;
352 int access;
353 atomic_t num_mw;
354
355 unsigned int page_shift;
356 u64 page_mask;
357
358 /* size of page_info when mr allocated */
359 u32 num_buf;
360 /* real size of page_info */
361 u32 max_allowed_buf;
362 u32 nbuf;
363
364 struct rxe_mr_page *page_info;
365 };
366
mr_page_size(struct rxe_mr * mr)367 static inline unsigned int mr_page_size(struct rxe_mr *mr)
368 {
369 return mr ? mr->ibmr.page_size : PAGE_SIZE;
370 }
371
372 enum rxe_mw_state {
373 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
374 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
375 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,
376 };
377
378 struct rxe_mw {
379 struct ib_mw ibmw;
380 struct rxe_pool_elem elem;
381 spinlock_t lock;
382 enum rxe_mw_state state;
383 struct rxe_qp *qp; /* Type 2 only */
384 struct rxe_mr *mr;
385 u32 rkey;
386 int access;
387 u64 addr;
388 u64 length;
389 };
390
391 struct rxe_mcg {
392 struct rb_node node;
393 struct kref ref_cnt;
394 struct rxe_dev *rxe;
395 struct list_head qp_list;
396 union ib_gid mgid;
397 atomic_t qp_num;
398 u32 qkey;
399 u16 pkey;
400 };
401
402 struct rxe_mca {
403 struct list_head qp_list;
404 struct rxe_qp *qp;
405 };
406
407 struct rxe_port {
408 struct ib_port_attr attr;
409 __be64 port_guid;
410 __be64 subnet_prefix;
411 spinlock_t port_lock; /* guard port */
412 unsigned int mtu_cap;
413 /* special QPs */
414 u32 qp_gsi_index;
415 };
416
417 #define RXE_PORT 1
418 struct rxe_dev {
419 struct ib_device ib_dev;
420 struct ib_device_attr attr;
421 int max_ucontext;
422 int max_inline_data;
423 struct mutex usdev_lock;
424
425 char raw_gid[ETH_ALEN];
426
427 struct rxe_pool uc_pool;
428 struct rxe_pool pd_pool;
429 struct rxe_pool ah_pool;
430 struct rxe_pool srq_pool;
431 struct rxe_pool qp_pool;
432 struct rxe_pool cq_pool;
433 struct rxe_pool mr_pool;
434 struct rxe_pool mw_pool;
435
436 /* multicast support */
437 spinlock_t mcg_lock;
438 struct rb_root mcg_tree;
439 atomic_t mcg_num;
440 atomic_t mcg_attach;
441
442 spinlock_t pending_lock; /* guard pending_mmaps */
443 struct list_head pending_mmaps;
444
445 spinlock_t mmap_offset_lock; /* guard mmap_offset */
446 u64 mmap_offset;
447
448 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
449
450 struct rxe_port port;
451 };
452
rxe_ib_device_get_netdev(struct ib_device * dev)453 static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev)
454 {
455 return ib_device_get_netdev(dev, RXE_PORT);
456 }
457
rxe_counter_inc(struct rxe_dev * rxe,enum rxe_counters index)458 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
459 {
460 atomic64_inc(&rxe->stats_counters[index]);
461 }
462
to_rdev(struct ib_device * dev)463 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
464 {
465 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
466 }
467
to_ruc(struct ib_ucontext * uc)468 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
469 {
470 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
471 }
472
to_rpd(struct ib_pd * pd)473 static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
474 {
475 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
476 }
477
to_rah(struct ib_ah * ah)478 static inline struct rxe_ah *to_rah(struct ib_ah *ah)
479 {
480 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
481 }
482
to_rsrq(struct ib_srq * srq)483 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
484 {
485 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
486 }
487
to_rqp(struct ib_qp * qp)488 static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
489 {
490 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
491 }
492
to_rcq(struct ib_cq * cq)493 static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
494 {
495 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
496 }
497
to_rmr(struct ib_mr * mr)498 static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
499 {
500 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
501 }
502
to_rmw(struct ib_mw * mw)503 static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
504 {
505 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
506 }
507
rxe_ah_pd(struct rxe_ah * ah)508 static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
509 {
510 return to_rpd(ah->ibah.pd);
511 }
512
mr_pd(struct rxe_mr * mr)513 static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
514 {
515 return to_rpd(mr->ibmr.pd);
516 }
517
rxe_mw_pd(struct rxe_mw * mw)518 static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
519 {
520 return to_rpd(mw->ibmw.pd);
521 }
522
523 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name,
524 struct net_device *ndev);
525
526 #endif /* RXE_VERBS_H */
527