1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #ifndef __QEDR_H__
33 #define __QEDR_H__
34
35 #include <linux/pci.h>
36 #include <linux/xarray.h>
37 #include <rdma/ib_addr.h>
38 #include <linux/qed/qed_if.h>
39 #include <linux/qed/qed_chain.h>
40 #include <linux/qed/qed_rdma_if.h>
41 #include <linux/qed/qede_rdma.h>
42 #include <linux/qed/roce_common.h>
43 #include <linux/completion.h>
44 #include "qedr_hsi_rdma.h"
45
46 #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
47 #define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev)
48 #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP)
49 #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE)
50
51 #define DP_DEBUG(dev, module, fmt, ...) \
52 pr_debug("(%s) " module ": " fmt, \
53 DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
54
55 #define QEDR_MSG_INIT "INIT"
56 #define QEDR_MSG_CQ " CQ"
57 #define QEDR_MSG_MR " MR"
58 #define QEDR_MSG_QP " QP"
59 #define QEDR_MSG_SRQ " SRQ"
60 #define QEDR_MSG_GSI " GSI"
61 #define QEDR_MSG_IWARP " IW"
62
63 #define QEDR_CQ_MAGIC_NUMBER (0x11223344)
64
65 #define FW_PAGE_SHIFT (12)
66
67 struct qedr_dev;
68
69 struct qedr_cnq {
70 struct qedr_dev *dev;
71 struct qed_chain pbl;
72 struct qed_sb_info *sb;
73 char name[32];
74 u64 n_comp;
75 __le16 *hw_cons_ptr;
76 u8 index;
77 };
78
79 #define QEDR_MAX_SGID 128
80
81 struct qedr_device_attr {
82 u32 vendor_id;
83 u32 vendor_part_id;
84 u32 hw_ver;
85 u64 fw_ver;
86 u64 node_guid;
87 u64 sys_image_guid;
88 u8 max_cnq;
89 u8 max_sge;
90 u16 max_inline;
91 u32 max_sqe;
92 u32 max_rqe;
93 u8 max_qp_resp_rd_atomic_resc;
94 u8 max_qp_req_rd_atomic_resc;
95 u64 max_dev_resp_rd_atomic_resc;
96 u32 max_cq;
97 u32 max_qp;
98 u32 max_mr;
99 u64 max_mr_size;
100 u32 max_cqe;
101 u32 max_mw;
102 u32 max_mr_mw_fmr_pbl;
103 u64 max_mr_mw_fmr_size;
104 u32 max_pd;
105 u32 max_ah;
106 u8 max_pkey;
107 u32 max_srq;
108 u32 max_srq_wr;
109 u8 max_srq_sge;
110 u8 max_stats_queues;
111 u32 dev_caps;
112
113 u64 page_size_caps;
114 u8 dev_ack_delay;
115 u32 reserved_lkey;
116 u32 bad_pkey_counter;
117 struct qed_rdma_events events;
118 };
119
120 #define QEDR_ENET_STATE_BIT (0)
121
122 struct qedr_dev {
123 struct ib_device ibdev;
124 struct qed_dev *cdev;
125 struct pci_dev *pdev;
126 struct net_device *ndev;
127
128 enum ib_atomic_cap atomic_cap;
129
130 void *rdma_ctx;
131 struct qedr_device_attr attr;
132
133 const struct qed_rdma_ops *ops;
134 struct qed_int_info int_info;
135
136 struct qed_sb_info *sb_array;
137 struct qedr_cnq *cnq_array;
138 int num_cnq;
139 int sb_start;
140
141 void __iomem *db_addr;
142 u64 db_phys_addr;
143 u32 db_size;
144 u16 dpi;
145
146 union ib_gid *sgid_tbl;
147
148 /* Lock for sgid table */
149 spinlock_t sgid_lock;
150
151 u64 guid;
152
153 u32 dp_module;
154 u8 dp_level;
155 u8 num_hwfns;
156 #define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1)
157 u8 affin_hwfn_idx;
158 u8 gsi_ll2_handle;
159
160 uint wq_multiplier;
161 u8 gsi_ll2_mac_address[ETH_ALEN];
162 int gsi_qp_created;
163 struct qedr_cq *gsi_sqcq;
164 struct qedr_cq *gsi_rqcq;
165 struct qedr_qp *gsi_qp;
166 enum qed_rdma_type rdma_type;
167 struct xarray qps;
168 struct xarray srqs;
169 struct workqueue_struct *iwarp_wq;
170 u16 iwarp_max_mtu;
171
172 unsigned long enet_state;
173
174 u8 user_dpm_enabled;
175 };
176
177 #define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *))
178 #define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge))
179 #define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
180 QEDR_SQE_ELEMENT_SIZE)
181 #define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\
182 (RDMA_RING_PAGE_SIZE) / \
183 (QEDR_SQE_ELEMENT_SIZE) /\
184 (QEDR_MAX_SQE_ELEMENTS_PER_SQE))
185 /* RQ */
186 #define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *))
187 #define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge))
188 #define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE)
189 #define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\
190 (RDMA_RING_PAGE_SIZE) / \
191 (QEDR_RQE_ELEMENT_SIZE) /\
192 (QEDR_MAX_RQE_ELEMENTS_PER_RQE))
193
194 #define QEDR_CQE_SIZE (sizeof(union rdma_cqe))
195 #define QEDR_MAX_CQE_PBL_SIZE (512 * 1024)
196 #define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \
197 sizeof(u64)) - 1)
198 #define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \
199 (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE))
200
201 #define QEDR_ROCE_MAX_CNQ_SIZE (0x4000)
202
203 #define QEDR_PORT (1)
204
205 #define QEDR_ROCE_PKEY_TABLE_LEN 1
206 #define QEDR_ROCE_PKEY_DEFAULT 0xffff
207
208 struct qedr_pbl {
209 struct list_head list_entry;
210 void *va;
211 dma_addr_t pa;
212 };
213
214 struct qedr_ucontext {
215 struct ib_ucontext ibucontext;
216 struct qedr_dev *dev;
217 struct qedr_pd *pd;
218 void __iomem *dpi_addr;
219 struct rdma_user_mmap_entry *db_mmap_entry;
220 u64 dpi_phys_addr;
221 u32 dpi_size;
222 u16 dpi;
223 bool db_rec;
224 u8 edpm_mode;
225 };
226
227 union db_prod32 {
228 struct rdma_pwm_val16_data data;
229 u32 raw;
230 };
231
232 union db_prod64 {
233 struct rdma_pwm_val32_data data;
234 u64 raw;
235 };
236
237 enum qedr_cq_type {
238 QEDR_CQ_TYPE_GSI,
239 QEDR_CQ_TYPE_KERNEL,
240 QEDR_CQ_TYPE_USER,
241 };
242
243 struct qedr_pbl_info {
244 u32 num_pbls;
245 u32 num_pbes;
246 u32 pbl_size;
247 u32 pbe_size;
248 bool two_layered;
249 };
250
251 struct qedr_userq {
252 struct ib_umem *umem;
253 struct qedr_pbl_info pbl_info;
254 struct qedr_pbl *pbl_tbl;
255 u64 buf_addr;
256 size_t buf_len;
257
258 /* doorbell recovery */
259 void __iomem *db_addr;
260 struct qedr_user_db_rec *db_rec_data;
261 struct rdma_user_mmap_entry *db_mmap_entry;
262 void __iomem *db_rec_db2_addr;
263 union db_prod32 db_rec_db2_data;
264 };
265
266 struct qedr_cq {
267 struct ib_cq ibcq;
268
269 enum qedr_cq_type cq_type;
270 u32 sig;
271
272 u16 icid;
273
274 /* Lock to protect multiplem CQ's */
275 spinlock_t cq_lock;
276 u8 arm_flags;
277 struct qed_chain pbl;
278
279 void __iomem *db_addr;
280 union db_prod64 db;
281
282 u8 pbl_toggle;
283 union rdma_cqe *latest_cqe;
284 union rdma_cqe *toggle_cqe;
285
286 u32 cq_cons;
287
288 struct qedr_userq q;
289 u8 destroyed;
290 u16 cnq_notif;
291 };
292
293 struct qedr_pd {
294 struct ib_pd ibpd;
295 u32 pd_id;
296 struct qedr_ucontext *uctx;
297 };
298
299 struct qedr_xrcd {
300 struct ib_xrcd ibxrcd;
301 u16 xrcd_id;
302 };
303
304 struct qedr_qp_hwq_info {
305 /* WQE Elements */
306 struct qed_chain pbl;
307 u64 p_phys_addr_tbl;
308 u32 max_sges;
309
310 /* WQE */
311 u16 prod;
312 u16 cons;
313 u16 wqe_cons;
314 u16 gsi_cons;
315 u16 max_wr;
316
317 /* DB */
318 void __iomem *db;
319 union db_prod32 db_data;
320
321 void __iomem *iwarp_db2;
322 union db_prod32 iwarp_db2_data;
323 };
324
325 struct qedr_srq_hwq_info {
326 u32 max_sges;
327 u32 max_wr;
328 struct qed_chain pbl;
329 u64 p_phys_addr_tbl;
330 u32 wqe_prod;
331 u32 sge_prod;
332 u32 wr_prod_cnt;
333 atomic_t wr_cons_cnt;
334 u32 num_elems;
335
336 struct rdma_srq_producers *virt_prod_pair_addr;
337 dma_addr_t phy_prod_pair_addr;
338 };
339
340 struct qedr_srq {
341 struct ib_srq ibsrq;
342 struct qedr_dev *dev;
343
344 struct qedr_userq usrq;
345 struct qedr_srq_hwq_info hw_srq;
346 struct ib_umem *prod_umem;
347 u16 srq_id;
348 u32 srq_limit;
349 bool is_xrc;
350 /* lock to protect srq recv post */
351 spinlock_t lock;
352 };
353
354 enum qedr_qp_err_bitmap {
355 QEDR_QP_ERR_SQ_FULL = 1,
356 QEDR_QP_ERR_RQ_FULL = 2,
357 QEDR_QP_ERR_BAD_SR = 4,
358 QEDR_QP_ERR_BAD_RR = 8,
359 QEDR_QP_ERR_SQ_PBL_FULL = 16,
360 QEDR_QP_ERR_RQ_PBL_FULL = 32,
361 };
362
363 enum qedr_qp_create_type {
364 QEDR_QP_CREATE_NONE,
365 QEDR_QP_CREATE_USER,
366 QEDR_QP_CREATE_KERNEL,
367 };
368
369 enum qedr_iwarp_cm_flags {
370 QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0),
371 QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1),
372 };
373
374 struct qedr_qp {
375 struct ib_qp ibqp; /* must be first */
376 struct qedr_dev *dev;
377 struct qedr_qp_hwq_info sq;
378 struct qedr_qp_hwq_info rq;
379
380 u32 max_inline_data;
381
382 /* Lock for QP's */
383 spinlock_t q_lock;
384 struct qedr_cq *sq_cq;
385 struct qedr_cq *rq_cq;
386 struct qedr_srq *srq;
387 enum qed_roce_qp_state state;
388 u32 id;
389 struct qedr_pd *pd;
390 enum ib_qp_type qp_type;
391 enum qedr_qp_create_type create_type;
392 struct qed_rdma_qp *qed_qp;
393 u32 qp_id;
394 u16 icid;
395 u16 mtu;
396 int sgid_idx;
397 u32 rq_psn;
398 u32 sq_psn;
399 u32 qkey;
400 u32 dest_qp_num;
401 u8 timeout;
402
403 /* Relevant to qps created from kernel space only (ULPs) */
404 u8 prev_wqe_size;
405 u16 wqe_cons;
406 u32 err_bitmap;
407 bool signaled;
408
409 /* SQ shadow */
410 struct {
411 u64 wr_id;
412 enum ib_wc_opcode opcode;
413 u32 bytes_len;
414 u8 wqe_size;
415 bool signaled;
416 dma_addr_t icrc_mapping;
417 u32 *icrc;
418 struct qedr_mr *mr;
419 } *wqe_wr_id;
420
421 /* RQ shadow */
422 struct {
423 u64 wr_id;
424 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
425 u8 wqe_size;
426
427 u8 smac[ETH_ALEN];
428 u16 vlan;
429 int rc;
430 } *rqe_wr_id;
431
432 /* Relevant to qps created from user space only (applications) */
433 struct qedr_userq usq;
434 struct qedr_userq urq;
435
436 /* synchronization objects used with iwarp ep */
437 struct kref refcnt;
438 struct completion iwarp_cm_comp;
439 struct completion qp_rel_comp;
440 unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */
441 };
442
443 struct qedr_ah {
444 struct ib_ah ibah;
445 struct rdma_ah_attr attr;
446 };
447
448 enum qedr_mr_type {
449 QEDR_MR_USER,
450 QEDR_MR_KERNEL,
451 QEDR_MR_DMA,
452 QEDR_MR_FRMR,
453 };
454
455 struct mr_info {
456 struct qedr_pbl *pbl_table;
457 struct qedr_pbl_info pbl_info;
458 struct list_head free_pbl_list;
459 struct list_head inuse_pbl_list;
460 u32 completed;
461 u32 completed_handled;
462 };
463
464 struct qedr_mr {
465 struct ib_mr ibmr;
466 struct ib_umem *umem;
467
468 struct qed_rdma_register_tid_in_params hw_mr;
469 enum qedr_mr_type type;
470
471 struct qedr_dev *dev;
472 struct mr_info info;
473
474 u64 *pages;
475 u32 npages;
476 };
477
478 struct qedr_user_mmap_entry {
479 struct rdma_user_mmap_entry rdma_entry;
480 struct qedr_dev *dev;
481 union {
482 u64 io_address;
483 void *address;
484 };
485 size_t length;
486 u16 dpi;
487 u8 mmap_flag;
488 };
489
490 #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT)))
491
492 #define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
493 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
494 #define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
495 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
496 #define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
497 RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
498
qedr_inc_sw_cons(struct qedr_qp_hwq_info * info)499 static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info)
500 {
501 info->cons = (info->cons + 1) % info->max_wr;
502 info->wqe_cons++;
503 }
504
qedr_inc_sw_prod(struct qedr_qp_hwq_info * info)505 static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info)
506 {
507 info->prod = (info->prod + 1) % info->max_wr;
508 }
509
qedr_get_dmac(struct qedr_dev * dev,struct rdma_ah_attr * ah_attr,u8 * mac_addr)510 static inline int qedr_get_dmac(struct qedr_dev *dev,
511 struct rdma_ah_attr *ah_attr, u8 *mac_addr)
512 {
513 union ib_gid zero_sgid = { { 0 } };
514 struct in6_addr in6;
515 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
516 u8 *dmac;
517
518 if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) {
519 DP_ERR(dev, "Local port GID not supported\n");
520 eth_zero_addr(mac_addr);
521 return -EINVAL;
522 }
523
524 memcpy(&in6, grh->dgid.raw, sizeof(in6));
525 dmac = rdma_ah_retrieve_dmac(ah_attr);
526 if (!dmac)
527 return -EINVAL;
528 ether_addr_copy(mac_addr, dmac);
529
530 return 0;
531 }
532
533 struct qedr_iw_listener {
534 struct qedr_dev *dev;
535 struct iw_cm_id *cm_id;
536 int backlog;
537 void *qed_handle;
538 };
539
540 struct qedr_iw_ep {
541 struct qedr_dev *dev;
542 struct iw_cm_id *cm_id;
543 struct qedr_qp *qp;
544 void *qed_context;
545 struct kref refcnt;
546 };
547
548 static inline
get_qedr_ucontext(struct ib_ucontext * ibucontext)549 struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext)
550 {
551 return container_of(ibucontext, struct qedr_ucontext, ibucontext);
552 }
553
get_qedr_dev(struct ib_device * ibdev)554 static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
555 {
556 return container_of(ibdev, struct qedr_dev, ibdev);
557 }
558
get_qedr_pd(struct ib_pd * ibpd)559 static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
560 {
561 return container_of(ibpd, struct qedr_pd, ibpd);
562 }
563
get_qedr_xrcd(struct ib_xrcd * ibxrcd)564 static inline struct qedr_xrcd *get_qedr_xrcd(struct ib_xrcd *ibxrcd)
565 {
566 return container_of(ibxrcd, struct qedr_xrcd, ibxrcd);
567 }
568
get_qedr_cq(struct ib_cq * ibcq)569 static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
570 {
571 return container_of(ibcq, struct qedr_cq, ibcq);
572 }
573
get_qedr_qp(struct ib_qp * ibqp)574 static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp)
575 {
576 return container_of(ibqp, struct qedr_qp, ibqp);
577 }
578
get_qedr_ah(struct ib_ah * ibah)579 static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah)
580 {
581 return container_of(ibah, struct qedr_ah, ibah);
582 }
583
get_qedr_mr(struct ib_mr * ibmr)584 static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr)
585 {
586 return container_of(ibmr, struct qedr_mr, ibmr);
587 }
588
get_qedr_srq(struct ib_srq * ibsrq)589 static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq)
590 {
591 return container_of(ibsrq, struct qedr_srq, ibsrq);
592 }
593
qedr_qp_has_srq(struct qedr_qp * qp)594 static inline bool qedr_qp_has_srq(struct qedr_qp *qp)
595 {
596 return qp->srq;
597 }
598
qedr_qp_has_sq(struct qedr_qp * qp)599 static inline bool qedr_qp_has_sq(struct qedr_qp *qp)
600 {
601 if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_TGT)
602 return false;
603
604 return true;
605 }
606
qedr_qp_has_rq(struct qedr_qp * qp)607 static inline bool qedr_qp_has_rq(struct qedr_qp *qp)
608 {
609 if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_INI ||
610 qp->qp_type == IB_QPT_XRC_TGT || qedr_qp_has_srq(qp))
611 return false;
612
613 return true;
614 }
615
616 static inline struct qedr_user_mmap_entry *
get_qedr_mmap_entry(struct rdma_user_mmap_entry * rdma_entry)617 get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
618 {
619 return container_of(rdma_entry, struct qedr_user_mmap_entry,
620 rdma_entry);
621 }
622 #endif
623