1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #ifndef IRDMA_TYPE_H
4 #define IRDMA_TYPE_H
5 #include "osdep.h"
6 #include "irdma.h"
7 #include "user.h"
8 #include "hmc.h"
9 #include "uda.h"
10 #include "ws.h"
11 #define IRDMA_DEBUG_ERR		"ERR"
12 #define IRDMA_DEBUG_INIT	"INIT"
13 #define IRDMA_DEBUG_DEV		"DEV"
14 #define IRDMA_DEBUG_CM		"CM"
15 #define IRDMA_DEBUG_VERBS	"VERBS"
16 #define IRDMA_DEBUG_PUDA	"PUDA"
17 #define IRDMA_DEBUG_ILQ		"ILQ"
18 #define IRDMA_DEBUG_IEQ		"IEQ"
19 #define IRDMA_DEBUG_QP		"QP"
20 #define IRDMA_DEBUG_CQ		"CQ"
21 #define IRDMA_DEBUG_MR		"MR"
22 #define IRDMA_DEBUG_PBLE	"PBLE"
23 #define IRDMA_DEBUG_WQE		"WQE"
24 #define IRDMA_DEBUG_AEQ		"AEQ"
25 #define IRDMA_DEBUG_CQP		"CQP"
26 #define IRDMA_DEBUG_HMC		"HMC"
27 #define IRDMA_DEBUG_USER	"USER"
28 #define IRDMA_DEBUG_VIRT	"VIRT"
29 #define IRDMA_DEBUG_DCB		"DCB"
30 #define	IRDMA_DEBUG_CQE		"CQE"
31 #define IRDMA_DEBUG_CLNT	"CLNT"
32 #define IRDMA_DEBUG_WS		"WS"
33 #define IRDMA_DEBUG_STATS	"STATS"
34 
35 enum irdma_page_size {
36 	IRDMA_PAGE_SIZE_4K = 0,
37 	IRDMA_PAGE_SIZE_2M,
38 	IRDMA_PAGE_SIZE_1G,
39 };
40 
41 enum irdma_hdrct_flags {
42 	DDP_LEN_FLAG  = 0x80,
43 	DDP_HDR_FLAG  = 0x40,
44 	RDMA_HDR_FLAG = 0x20,
45 };
46 
47 enum irdma_term_layers {
48 	LAYER_RDMA = 0,
49 	LAYER_DDP  = 1,
50 	LAYER_MPA  = 2,
51 };
52 
53 enum irdma_term_error_types {
54 	RDMAP_REMOTE_PROT = 1,
55 	RDMAP_REMOTE_OP   = 2,
56 	DDP_CATASTROPHIC  = 0,
57 	DDP_TAGGED_BUF    = 1,
58 	DDP_UNTAGGED_BUF  = 2,
59 	DDP_LLP		  = 3,
60 };
61 
62 enum irdma_term_rdma_errors {
63 	RDMAP_INV_STAG		  = 0x00,
64 	RDMAP_INV_BOUNDS	  = 0x01,
65 	RDMAP_ACCESS		  = 0x02,
66 	RDMAP_UNASSOC_STAG	  = 0x03,
67 	RDMAP_TO_WRAP		  = 0x04,
68 	RDMAP_INV_RDMAP_VER       = 0x05,
69 	RDMAP_UNEXPECTED_OP       = 0x06,
70 	RDMAP_CATASTROPHIC_LOCAL  = 0x07,
71 	RDMAP_CATASTROPHIC_GLOBAL = 0x08,
72 	RDMAP_CANT_INV_STAG       = 0x09,
73 	RDMAP_UNSPECIFIED	  = 0xff,
74 };
75 
76 enum irdma_term_ddp_errors {
77 	DDP_CATASTROPHIC_LOCAL      = 0x00,
78 	DDP_TAGGED_INV_STAG	    = 0x00,
79 	DDP_TAGGED_BOUNDS	    = 0x01,
80 	DDP_TAGGED_UNASSOC_STAG     = 0x02,
81 	DDP_TAGGED_TO_WRAP	    = 0x03,
82 	DDP_TAGGED_INV_DDP_VER      = 0x04,
83 	DDP_UNTAGGED_INV_QN	    = 0x01,
84 	DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
85 	DDP_UNTAGGED_INV_MSN_RANGE  = 0x03,
86 	DDP_UNTAGGED_INV_MO	    = 0x04,
87 	DDP_UNTAGGED_INV_TOO_LONG   = 0x05,
88 	DDP_UNTAGGED_INV_DDP_VER    = 0x06,
89 };
90 
91 enum irdma_term_mpa_errors {
92 	MPA_CLOSED  = 0x01,
93 	MPA_CRC     = 0x02,
94 	MPA_MARKER  = 0x03,
95 	MPA_REQ_RSP = 0x04,
96 };
97 
98 enum irdma_qp_event_type {
99 	IRDMA_QP_EVENT_CATASTROPHIC,
100 	IRDMA_QP_EVENT_ACCESS_ERR,
101 	IRDMA_QP_EVENT_REQ_ERR,
102 };
103 
104 enum irdma_hw_stats_index {
105 	/* gen1 - 32-bit */
106 	IRDMA_HW_STAT_INDEX_IP4RXDISCARD	= 0,
107 	IRDMA_HW_STAT_INDEX_IP4RXTRUNC		= 1,
108 	IRDMA_HW_STAT_INDEX_IP4TXNOROUTE	= 2,
109 	IRDMA_HW_STAT_INDEX_IP6RXDISCARD	= 3,
110 	IRDMA_HW_STAT_INDEX_IP6RXTRUNC		= 4,
111 	IRDMA_HW_STAT_INDEX_IP6TXNOROUTE	= 5,
112 	IRDMA_HW_STAT_INDEX_TCPRTXSEG		= 6,
113 	IRDMA_HW_STAT_INDEX_TCPRXOPTERR		= 7,
114 	IRDMA_HW_STAT_INDEX_TCPRXPROTOERR	= 8,
115 	IRDMA_HW_STAT_INDEX_RXVLANERR		= 9,
116 		/* gen1 - 64-bit */
117 	IRDMA_HW_STAT_INDEX_IP4RXOCTS		= 10,
118 	IRDMA_HW_STAT_INDEX_IP4RXPKTS		= 11,
119 	IRDMA_HW_STAT_INDEX_IP4RXFRAGS		= 12,
120 	IRDMA_HW_STAT_INDEX_IP4RXMCPKTS		= 13,
121 	IRDMA_HW_STAT_INDEX_IP4TXOCTS		= 14,
122 	IRDMA_HW_STAT_INDEX_IP4TXPKTS		= 15,
123 	IRDMA_HW_STAT_INDEX_IP4TXFRAGS		= 16,
124 	IRDMA_HW_STAT_INDEX_IP4TXMCPKTS		= 17,
125 	IRDMA_HW_STAT_INDEX_IP6RXOCTS		= 18,
126 	IRDMA_HW_STAT_INDEX_IP6RXPKTS		= 19,
127 	IRDMA_HW_STAT_INDEX_IP6RXFRAGS		= 20,
128 	IRDMA_HW_STAT_INDEX_IP6RXMCPKTS		= 21,
129 	IRDMA_HW_STAT_INDEX_IP6TXOCTS		= 22,
130 	IRDMA_HW_STAT_INDEX_IP6TXPKTS		= 23,
131 	IRDMA_HW_STAT_INDEX_IP6TXFRAGS		= 24,
132 	IRDMA_HW_STAT_INDEX_IP6TXMCPKTS		= 25,
133 	IRDMA_HW_STAT_INDEX_TCPRXSEGS		= 26,
134 	IRDMA_HW_STAT_INDEX_TCPTXSEG		= 27,
135 	IRDMA_HW_STAT_INDEX_RDMARXRDS		= 28,
136 	IRDMA_HW_STAT_INDEX_RDMARXSNDS		= 29,
137 	IRDMA_HW_STAT_INDEX_RDMARXWRS		= 30,
138 	IRDMA_HW_STAT_INDEX_RDMATXRDS		= 31,
139 	IRDMA_HW_STAT_INDEX_RDMATXSNDS		= 32,
140 	IRDMA_HW_STAT_INDEX_RDMATXWRS		= 33,
141 	IRDMA_HW_STAT_INDEX_RDMAVBND		= 34,
142 	IRDMA_HW_STAT_INDEX_RDMAVINV		= 35,
143 	IRDMA_HW_STAT_INDEX_IP4RXMCOCTS         = 36,
144 	IRDMA_HW_STAT_INDEX_IP4TXMCOCTS         = 37,
145 	IRDMA_HW_STAT_INDEX_IP6RXMCOCTS         = 38,
146 	IRDMA_HW_STAT_INDEX_IP6TXMCOCTS         = 39,
147 	IRDMA_HW_STAT_INDEX_UDPRXPKTS           = 40,
148 	IRDMA_HW_STAT_INDEX_UDPTXPKTS           = 41,
149 	IRDMA_HW_STAT_INDEX_MAX_GEN_1           = 42, /* Must be same value as next entry */
150 	/* gen2 - 64-bit */
151 	IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS   = 42,
152 	/* gen2 - 32-bit */
153 	IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED      = 43,
154 	IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED      = 44,
155 	IRDMA_HW_STAT_INDEX_TXNPCNPSENT         = 45,
156 	IRDMA_HW_STAT_INDEX_MAX_GEN_2		= 46,
157 };
158 
159 enum irdma_feature_type {
160 	IRDMA_FEATURE_FW_INFO = 0,
161 	IRDMA_HW_VERSION_INFO = 1,
162 	IRDMA_QSETS_MAX       = 26,
163 	IRDMA_MAX_FEATURES, /* Must be last entry */
164 };
165 
166 enum irdma_sched_prio_type {
167 	IRDMA_PRIO_WEIGHTED_RR     = 1,
168 	IRDMA_PRIO_STRICT	   = 2,
169 	IRDMA_PRIO_WEIGHTED_STRICT = 3,
170 };
171 
172 enum irdma_vm_vf_type {
173 	IRDMA_VF_TYPE = 0,
174 	IRDMA_VM_TYPE,
175 	IRDMA_PF_TYPE,
176 };
177 
178 enum irdma_cqp_hmc_profile {
179 	IRDMA_HMC_PROFILE_DEFAULT  = 1,
180 	IRDMA_HMC_PROFILE_FAVOR_VF = 2,
181 	IRDMA_HMC_PROFILE_EQUAL    = 3,
182 };
183 
184 enum irdma_quad_entry_type {
185 	IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,
186 	IRDMA_QHASH_TYPE_TCP_SYN,
187 	IRDMA_QHASH_TYPE_UDP_UNICAST,
188 	IRDMA_QHASH_TYPE_UDP_MCAST,
189 	IRDMA_QHASH_TYPE_ROCE_MCAST,
190 	IRDMA_QHASH_TYPE_ROCEV2_HW,
191 };
192 
193 enum irdma_quad_hash_manage_type {
194 	IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,
195 	IRDMA_QHASH_MANAGE_TYPE_ADD,
196 	IRDMA_QHASH_MANAGE_TYPE_MODIFY,
197 };
198 
199 enum irdma_syn_rst_handling {
200 	IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,
201 	IRDMA_SYN_RST_HANDLING_HW_TCP,
202 	IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,
203 	IRDMA_SYN_RST_HANDLING_FW_TCP,
204 };
205 
206 enum irdma_queue_type {
207 	IRDMA_QUEUE_TYPE_SQ_RQ = 0,
208 	IRDMA_QUEUE_TYPE_CQP,
209 };
210 
211 struct irdma_sc_dev;
212 struct irdma_vsi_pestat;
213 
214 struct irdma_dcqcn_cc_params {
215 	u8 cc_cfg_valid;
216 	u8 min_dec_factor;
217 	u8 min_rate;
218 	u8 dcqcn_f;
219 	u16 rai_factor;
220 	u16 hai_factor;
221 	u16 dcqcn_t;
222 	u32 dcqcn_b;
223 	u32 rreduce_mperiod;
224 };
225 
226 struct irdma_cqp_init_info {
227 	u64 cqp_compl_ctx;
228 	u64 host_ctx_pa;
229 	u64 sq_pa;
230 	struct irdma_sc_dev *dev;
231 	struct irdma_cqp_quanta *sq;
232 	struct irdma_dcqcn_cc_params dcqcn_params;
233 	__le64 *host_ctx;
234 	u64 *scratch_array;
235 	u32 sq_size;
236 	u16 hw_maj_ver;
237 	u16 hw_min_ver;
238 	u8 struct_ver;
239 	u8 hmc_profile;
240 	u8 ena_vf_count;
241 	u8 ceqs_per_vf;
242 	bool en_datacenter_tcp:1;
243 	bool disable_packed:1;
244 	bool rocev2_rto_policy:1;
245 	enum irdma_protocol_used protocol_used;
246 };
247 
248 struct irdma_terminate_hdr {
249 	u8 layer_etype;
250 	u8 error_code;
251 	u8 hdrct;
252 	u8 rsvd;
253 };
254 
255 struct irdma_cqp_sq_wqe {
256 	__le64 buf[IRDMA_CQP_WQE_SIZE];
257 };
258 
259 struct irdma_sc_aeqe {
260 	__le64 buf[IRDMA_AEQE_SIZE];
261 };
262 
263 struct irdma_ceqe {
264 	__le64 buf[IRDMA_CEQE_SIZE];
265 };
266 
267 struct irdma_cqp_ctx {
268 	__le64 buf[IRDMA_CQP_CTX_SIZE];
269 };
270 
271 struct irdma_cq_shadow_area {
272 	__le64 buf[IRDMA_SHADOW_AREA_SIZE];
273 };
274 
275 struct irdma_dev_hw_stats_offsets {
276 	u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
277 };
278 
279 struct irdma_dev_hw_stats {
280 	u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
281 };
282 
283 struct irdma_gather_stats {
284 	u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
285 };
286 
287 struct irdma_hw_stat_map {
288 	u16 byteoff;
289 	u8 bitoff;
290 	u64 bitmask;
291 };
292 
293 struct irdma_stats_gather_info {
294 	bool use_hmc_fcn_index:1;
295 	bool use_stats_inst:1;
296 	u8 hmc_fcn_index;
297 	u8 stats_inst_index;
298 	struct irdma_dma_mem stats_buff_mem;
299 	void *gather_stats_va;
300 	void *last_gather_stats_va;
301 };
302 
303 struct irdma_vsi_pestat {
304 	struct irdma_hw *hw;
305 	struct irdma_dev_hw_stats hw_stats;
306 	struct irdma_stats_gather_info gather_info;
307 	struct timer_list stats_timer;
308 	struct irdma_sc_vsi *vsi;
309 	struct irdma_dev_hw_stats last_hw_stats;
310 	spinlock_t lock; /* rdma stats lock */
311 };
312 
313 struct irdma_hw {
314 	u8 __iomem *hw_addr;
315 	u8 __iomem *priv_hw_addr;
316 	struct device *device;
317 	struct irdma_hmc_info hmc;
318 };
319 
320 struct irdma_pfpdu {
321 	struct list_head rxlist;
322 	u32 rcv_nxt;
323 	u32 fps;
324 	u32 max_fpdu_data;
325 	u32 nextseqnum;
326 	u32 rcv_start_seq;
327 	bool mode:1;
328 	bool mpa_crc_err:1;
329 	u8  marker_len;
330 	u64 total_ieq_bufs;
331 	u64 fpdu_processed;
332 	u64 bad_seq_num;
333 	u64 crc_err;
334 	u64 no_tx_bufs;
335 	u64 tx_err;
336 	u64 out_of_order;
337 	u64 pmode_count;
338 	struct irdma_sc_ah *ah;
339 	struct irdma_puda_buf *ah_buf;
340 	spinlock_t lock; /* fpdu processing lock */
341 	struct irdma_puda_buf *lastrcv_buf;
342 };
343 
344 struct irdma_sc_pd {
345 	struct irdma_sc_dev *dev;
346 	u32 pd_id;
347 	int abi_ver;
348 };
349 
350 struct irdma_cqp_quanta {
351 	__le64 elem[IRDMA_CQP_WQE_SIZE];
352 };
353 
354 struct irdma_sc_cqp {
355 	u32 size;
356 	u64 sq_pa;
357 	u64 host_ctx_pa;
358 	void *back_cqp;
359 	struct irdma_sc_dev *dev;
360 	int (*process_cqp_sds)(struct irdma_sc_dev *dev,
361 			       struct irdma_update_sds_info *info);
362 	struct irdma_dma_mem sdbuf;
363 	struct irdma_ring sq_ring;
364 	struct irdma_cqp_quanta *sq_base;
365 	struct irdma_dcqcn_cc_params dcqcn_params;
366 	__le64 *host_ctx;
367 	u64 *scratch_array;
368 	u64 requested_ops;
369 	atomic64_t completed_ops;
370 	u32 cqp_id;
371 	u32 sq_size;
372 	u32 hw_sq_size;
373 	u16 hw_maj_ver;
374 	u16 hw_min_ver;
375 	u8 struct_ver;
376 	u8 polarity;
377 	u8 hmc_profile;
378 	u8 ena_vf_count;
379 	u8 timeout_count;
380 	u8 ceqs_per_vf;
381 	bool en_datacenter_tcp:1;
382 	bool disable_packed:1;
383 	bool rocev2_rto_policy:1;
384 	enum irdma_protocol_used protocol_used;
385 };
386 
387 struct irdma_sc_aeq {
388 	u32 size;
389 	u64 aeq_elem_pa;
390 	struct irdma_sc_dev *dev;
391 	struct irdma_sc_aeqe *aeqe_base;
392 	void *pbl_list;
393 	u32 elem_cnt;
394 	struct irdma_ring aeq_ring;
395 	u8 pbl_chunk_size;
396 	u32 first_pm_pbl_idx;
397 	u32 msix_idx;
398 	u8 polarity;
399 	bool virtual_map:1;
400 };
401 
402 struct irdma_sc_ceq {
403 	u32 size;
404 	u64 ceq_elem_pa;
405 	struct irdma_sc_dev *dev;
406 	struct irdma_ceqe *ceqe_base;
407 	void *pbl_list;
408 	u32 ceq_id;
409 	u32 elem_cnt;
410 	struct irdma_ring ceq_ring;
411 	u8 pbl_chunk_size;
412 	u8 tph_val;
413 	u32 first_pm_pbl_idx;
414 	u8 polarity;
415 	struct irdma_sc_vsi *vsi;
416 	struct irdma_sc_cq **reg_cq;
417 	u32 reg_cq_size;
418 	spinlock_t req_cq_lock; /* protect access to reg_cq array */
419 	bool virtual_map:1;
420 	bool tph_en:1;
421 	bool itr_no_expire:1;
422 };
423 
424 struct irdma_sc_cq {
425 	struct irdma_cq_uk cq_uk;
426 	u64 cq_pa;
427 	u64 shadow_area_pa;
428 	struct irdma_sc_dev *dev;
429 	struct irdma_sc_vsi *vsi;
430 	void *pbl_list;
431 	void *back_cq;
432 	u32 ceq_id;
433 	u32 shadow_read_threshold;
434 	u8 pbl_chunk_size;
435 	u8 cq_type;
436 	u8 tph_val;
437 	u32 first_pm_pbl_idx;
438 	bool ceqe_mask:1;
439 	bool virtual_map:1;
440 	bool check_overflow:1;
441 	bool ceq_id_valid:1;
442 	bool tph_en;
443 };
444 
445 struct irdma_sc_qp {
446 	struct irdma_qp_uk qp_uk;
447 	u64 sq_pa;
448 	u64 rq_pa;
449 	u64 hw_host_ctx_pa;
450 	u64 shadow_area_pa;
451 	u64 q2_pa;
452 	struct irdma_sc_dev *dev;
453 	struct irdma_sc_vsi *vsi;
454 	struct irdma_sc_pd *pd;
455 	__le64 *hw_host_ctx;
456 	void *llp_stream_handle;
457 	struct irdma_pfpdu pfpdu;
458 	u32 ieq_qp;
459 	u8 *q2_buf;
460 	u64 qp_compl_ctx;
461 	u32 push_idx;
462 	u16 qs_handle;
463 	u16 push_offset;
464 	u8 flush_wqes_count;
465 	u8 sq_tph_val;
466 	u8 rq_tph_val;
467 	u8 qp_state;
468 	u8 hw_sq_size;
469 	u8 hw_rq_size;
470 	u8 src_mac_addr_idx;
471 	bool on_qoslist:1;
472 	bool ieq_pass_thru:1;
473 	bool sq_tph_en:1;
474 	bool rq_tph_en:1;
475 	bool rcv_tph_en:1;
476 	bool xmit_tph_en:1;
477 	bool virtual_map:1;
478 	bool flush_sq:1;
479 	bool flush_rq:1;
480 	bool sq_flush_code:1;
481 	bool rq_flush_code:1;
482 	enum irdma_flush_opcode flush_code;
483 	enum irdma_qp_event_type event_type;
484 	u8 term_flags;
485 	u8 user_pri;
486 	struct list_head list;
487 };
488 
489 struct irdma_stats_inst_info {
490 	bool use_hmc_fcn_index;
491 	u8 hmc_fn_id;
492 	u8 stats_idx;
493 };
494 
495 struct irdma_up_info {
496 	u8 map[8];
497 	u8 cnp_up_override;
498 	u8 hmc_fcn_idx;
499 	bool use_vlan:1;
500 	bool use_cnp_up_override:1;
501 };
502 
503 #define IRDMA_MAX_WS_NODES	0x3FF
504 #define IRDMA_WS_NODE_INVALID	0xFFFF
505 
506 struct irdma_ws_node_info {
507 	u16 id;
508 	u16 vsi;
509 	u16 parent_id;
510 	u16 qs_handle;
511 	bool type_leaf:1;
512 	bool enable:1;
513 	u8 prio_type;
514 	u8 tc;
515 	u8 weight;
516 };
517 
518 struct irdma_hmc_fpm_misc {
519 	u32 max_ceqs;
520 	u32 max_sds;
521 	u32 xf_block_size;
522 	u32 q1_block_size;
523 	u32 ht_multiplier;
524 	u32 timer_bucket;
525 	u32 rrf_block_size;
526 	u32 ooiscf_block_size;
527 };
528 
529 #define IRDMA_LEAF_DEFAULT_REL_BW		64
530 #define IRDMA_PARENT_DEFAULT_REL_BW		1
531 
532 struct irdma_qos {
533 	struct list_head qplist;
534 	struct mutex qos_mutex; /* protect QoS attributes per QoS level */
535 	u64 lan_qos_handle;
536 	u32 l2_sched_node_id;
537 	u16 qs_handle;
538 	u8 traffic_class;
539 	u8 rel_bw;
540 	u8 prio_type;
541 	bool valid;
542 };
543 
544 #define IRDMA_INVALID_STATS_IDX 0xff
545 struct irdma_sc_vsi {
546 	u16 vsi_idx;
547 	struct irdma_sc_dev *dev;
548 	void *back_vsi;
549 	u32 ilq_count;
550 	struct irdma_virt_mem ilq_mem;
551 	struct irdma_puda_rsrc *ilq;
552 	u32 ieq_count;
553 	struct irdma_virt_mem ieq_mem;
554 	struct irdma_puda_rsrc *ieq;
555 	u32 exception_lan_q;
556 	u16 mtu;
557 	u16 vm_id;
558 	enum irdma_vm_vf_type vm_vf_type;
559 	bool stats_inst_alloc:1;
560 	bool tc_change_pending:1;
561 	struct irdma_vsi_pestat *pestat;
562 	atomic_t qp_suspend_reqs;
563 	int (*register_qset)(struct irdma_sc_vsi *vsi,
564 			     struct irdma_ws_node *tc_node);
565 	void (*unregister_qset)(struct irdma_sc_vsi *vsi,
566 				struct irdma_ws_node *tc_node);
567 	u8 qos_rel_bw;
568 	u8 qos_prio_type;
569 	u8 stats_idx;
570 	u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
571 	struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
572 	u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
573 	bool dscp_mode:1;
574 };
575 
576 struct irdma_sc_dev {
577 	struct list_head cqp_cmd_head; /* head of the CQP command list */
578 	spinlock_t cqp_lock; /* protect CQP list access */
579 	bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1];
580 	struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
581 	u64 fpm_query_buf_pa;
582 	u64 fpm_commit_buf_pa;
583 	__le64 *fpm_query_buf;
584 	__le64 *fpm_commit_buf;
585 	struct irdma_hw *hw;
586 	u8 __iomem *db_addr;
587 	u32 __iomem *wqe_alloc_db;
588 	u32 __iomem *cq_arm_db;
589 	u32 __iomem *aeq_alloc_db;
590 	u32 __iomem *cqp_db;
591 	u32 __iomem *cq_ack_db;
592 	u32 __iomem *ceq_itr_mask_db;
593 	u32 __iomem *aeq_itr_mask_db;
594 	u32 __iomem *hw_regs[IRDMA_MAX_REGS];
595 	u32 ceq_itr;   /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
596 	u64 hw_masks[IRDMA_MAX_MASKS];
597 	u64 hw_shifts[IRDMA_MAX_SHIFTS];
598 	const struct irdma_hw_stat_map *hw_stats_map;
599 	u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
600 	u64 feature_info[IRDMA_MAX_FEATURES];
601 	u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
602 	struct irdma_hw_attrs hw_attrs;
603 	struct irdma_hmc_info *hmc_info;
604 	struct irdma_sc_cqp *cqp;
605 	struct irdma_sc_aeq *aeq;
606 	struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];
607 	struct irdma_sc_cq *ccq;
608 	const struct irdma_irq_ops *irq_ops;
609 	struct irdma_hmc_fpm_misc hmc_fpm_misc;
610 	struct irdma_ws_node *ws_tree_root;
611 	struct mutex ws_mutex; /* ws tree mutex */
612 	u16 num_vfs;
613 	u8 hmc_fn_id;
614 	u8 vf_id;
615 	bool vchnl_up:1;
616 	bool ceq_valid:1;
617 	u8 pci_rev;
618 	int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
619 	void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
620 	void (*ws_reset)(struct irdma_sc_vsi *vsi);
621 };
622 
623 struct irdma_modify_cq_info {
624 	u64 cq_pa;
625 	struct irdma_cqe *cq_base;
626 	u32 cq_size;
627 	u32 shadow_read_threshold;
628 	u8 pbl_chunk_size;
629 	u32 first_pm_pbl_idx;
630 	bool virtual_map:1;
631 	bool check_overflow;
632 	bool cq_resize:1;
633 };
634 
635 struct irdma_create_qp_info {
636 	bool ord_valid:1;
637 	bool tcp_ctx_valid:1;
638 	bool cq_num_valid:1;
639 	bool arp_cache_idx_valid:1;
640 	bool mac_valid:1;
641 	bool force_lpb;
642 	u8 next_iwarp_state;
643 };
644 
645 struct irdma_modify_qp_info {
646 	u64 rx_win0;
647 	u64 rx_win1;
648 	u16 new_mss;
649 	u8 next_iwarp_state;
650 	u8 curr_iwarp_state;
651 	u8 termlen;
652 	bool ord_valid:1;
653 	bool tcp_ctx_valid:1;
654 	bool udp_ctx_valid:1;
655 	bool cq_num_valid:1;
656 	bool arp_cache_idx_valid:1;
657 	bool reset_tcp_conn:1;
658 	bool remove_hash_idx:1;
659 	bool dont_send_term:1;
660 	bool dont_send_fin:1;
661 	bool cached_var_valid:1;
662 	bool mss_change:1;
663 	bool force_lpb:1;
664 	bool mac_valid:1;
665 };
666 
667 struct irdma_ccq_cqe_info {
668 	struct irdma_sc_cqp *cqp;
669 	u64 scratch;
670 	u32 op_ret_val;
671 	u16 maj_err_code;
672 	u16 min_err_code;
673 	u8 op_code;
674 	bool error;
675 };
676 
677 struct irdma_dcb_app_info {
678 	u8 priority;
679 	u8 selector;
680 	u16 prot_id;
681 };
682 
683 struct irdma_qos_tc_info {
684 	u64 tc_ctx;
685 	u8 rel_bw;
686 	u8 prio_type;
687 	u8 egress_virt_up;
688 	u8 ingress_virt_up;
689 };
690 
691 struct irdma_l2params {
692 	struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];
693 	struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];
694 	u32 num_apps;
695 	u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
696 	u16 mtu;
697 	u8 up2tc[IRDMA_MAX_USER_PRIORITY];
698 	u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
699 	u8 num_tc;
700 	u8 vsi_rel_bw;
701 	u8 vsi_prio_type;
702 	bool mtu_changed:1;
703 	bool tc_changed:1;
704 	bool dscp_mode:1;
705 };
706 
707 struct irdma_vsi_init_info {
708 	struct irdma_sc_dev *dev;
709 	void *back_vsi;
710 	struct irdma_l2params *params;
711 	u16 exception_lan_q;
712 	u16 pf_data_vsi_num;
713 	enum irdma_vm_vf_type vm_vf_type;
714 	u16 vm_id;
715 	int (*register_qset)(struct irdma_sc_vsi *vsi,
716 			     struct irdma_ws_node *tc_node);
717 	void (*unregister_qset)(struct irdma_sc_vsi *vsi,
718 				struct irdma_ws_node *tc_node);
719 };
720 
721 struct irdma_vsi_stats_info {
722 	struct irdma_vsi_pestat *pestat;
723 	u8 fcn_id;
724 	bool alloc_stats_inst;
725 };
726 
727 struct irdma_device_init_info {
728 	u64 fpm_query_buf_pa;
729 	u64 fpm_commit_buf_pa;
730 	__le64 *fpm_query_buf;
731 	__le64 *fpm_commit_buf;
732 	struct irdma_hw *hw;
733 	void __iomem *bar0;
734 	u8 hmc_fn_id;
735 };
736 
737 struct irdma_ceq_init_info {
738 	u64 ceqe_pa;
739 	struct irdma_sc_dev *dev;
740 	u64 *ceqe_base;
741 	void *pbl_list;
742 	u32 elem_cnt;
743 	u32 ceq_id;
744 	bool virtual_map:1;
745 	bool tph_en:1;
746 	bool itr_no_expire:1;
747 	u8 pbl_chunk_size;
748 	u8 tph_val;
749 	u32 first_pm_pbl_idx;
750 	struct irdma_sc_vsi *vsi;
751 	struct irdma_sc_cq **reg_cq;
752 	u32 reg_cq_idx;
753 };
754 
755 struct irdma_aeq_init_info {
756 	u64 aeq_elem_pa;
757 	struct irdma_sc_dev *dev;
758 	u32 *aeqe_base;
759 	void *pbl_list;
760 	u32 elem_cnt;
761 	bool virtual_map;
762 	u8 pbl_chunk_size;
763 	u32 first_pm_pbl_idx;
764 	u32 msix_idx;
765 };
766 
767 struct irdma_ccq_init_info {
768 	u64 cq_pa;
769 	u64 shadow_area_pa;
770 	struct irdma_sc_dev *dev;
771 	struct irdma_cqe *cq_base;
772 	__le64 *shadow_area;
773 	void *pbl_list;
774 	u32 num_elem;
775 	u32 ceq_id;
776 	u32 shadow_read_threshold;
777 	bool ceqe_mask:1;
778 	bool ceq_id_valid:1;
779 	bool avoid_mem_cflct:1;
780 	bool virtual_map:1;
781 	bool tph_en:1;
782 	u8 tph_val;
783 	u8 pbl_chunk_size;
784 	u32 first_pm_pbl_idx;
785 	struct irdma_sc_vsi *vsi;
786 };
787 
788 struct irdma_udp_offload_info {
789 	bool ipv4:1;
790 	bool insert_vlan_tag:1;
791 	u8 ttl;
792 	u8 tos;
793 	u16 src_port;
794 	u16 dst_port;
795 	u32 dest_ip_addr[4];
796 	u32 snd_mss;
797 	u16 vlan_tag;
798 	u16 arp_idx;
799 	u32 flow_label;
800 	u8 udp_state;
801 	u32 psn_nxt;
802 	u32 lsn;
803 	u32 epsn;
804 	u32 psn_max;
805 	u32 psn_una;
806 	u32 local_ipaddr[4];
807 	u32 cwnd;
808 	u8 rexmit_thresh;
809 	u8 rnr_nak_thresh;
810 };
811 
812 struct irdma_roce_offload_info {
813 	u16 p_key;
814 	u16 err_rq_idx;
815 	u32 qkey;
816 	u32 dest_qp;
817 	u8 roce_tver;
818 	u8 ack_credits;
819 	u8 err_rq_idx_valid;
820 	u32 pd_id;
821 	u16 ord_size;
822 	u16 ird_size;
823 	bool is_qp1:1;
824 	bool udprivcq_en:1;
825 	bool dcqcn_en:1;
826 	bool rcv_no_icrc:1;
827 	bool wr_rdresp_en:1;
828 	bool bind_en:1;
829 	bool fast_reg_en:1;
830 	bool priv_mode_en:1;
831 	bool rd_en:1;
832 	bool timely_en:1;
833 	bool dctcp_en:1;
834 	bool fw_cc_enable:1;
835 	bool use_stats_inst:1;
836 	u16 t_high;
837 	u16 t_low;
838 	u8 last_byte_sent;
839 	u8 mac_addr[ETH_ALEN];
840 	u8 rtomin;
841 };
842 
843 struct irdma_iwarp_offload_info {
844 	u16 rcv_mark_offset;
845 	u16 snd_mark_offset;
846 	u8 ddp_ver;
847 	u8 rdmap_ver;
848 	u8 iwarp_mode;
849 	u16 err_rq_idx;
850 	u32 pd_id;
851 	u16 ord_size;
852 	u16 ird_size;
853 	bool ib_rd_en:1;
854 	bool align_hdrs:1;
855 	bool rcv_no_mpa_crc:1;
856 	bool err_rq_idx_valid:1;
857 	bool snd_mark_en:1;
858 	bool rcv_mark_en:1;
859 	bool wr_rdresp_en:1;
860 	bool bind_en:1;
861 	bool fast_reg_en:1;
862 	bool priv_mode_en:1;
863 	bool rd_en:1;
864 	bool timely_en:1;
865 	bool use_stats_inst:1;
866 	bool ecn_en:1;
867 	bool dctcp_en:1;
868 	u16 t_high;
869 	u16 t_low;
870 	u8 last_byte_sent;
871 	u8 mac_addr[ETH_ALEN];
872 	u8 rtomin;
873 };
874 
875 struct irdma_tcp_offload_info {
876 	bool ipv4:1;
877 	bool no_nagle:1;
878 	bool insert_vlan_tag:1;
879 	bool time_stamp:1;
880 	bool drop_ooo_seg:1;
881 	bool avoid_stretch_ack:1;
882 	bool wscale:1;
883 	bool ignore_tcp_opt:1;
884 	bool ignore_tcp_uns_opt:1;
885 	u8 cwnd_inc_limit;
886 	u8 dup_ack_thresh;
887 	u8 ttl;
888 	u8 src_mac_addr_idx;
889 	u8 tos;
890 	u16 src_port;
891 	u16 dst_port;
892 	u32 dest_ip_addr[4];
893 	//u32 dest_ip_addr0;
894 	//u32 dest_ip_addr1;
895 	//u32 dest_ip_addr2;
896 	//u32 dest_ip_addr3;
897 	u32 snd_mss;
898 	u16 syn_rst_handling;
899 	u16 vlan_tag;
900 	u16 arp_idx;
901 	u32 flow_label;
902 	u8 tcp_state;
903 	u8 snd_wscale;
904 	u8 rcv_wscale;
905 	u32 time_stamp_recent;
906 	u32 time_stamp_age;
907 	u32 snd_nxt;
908 	u32 snd_wnd;
909 	u32 rcv_nxt;
910 	u32 rcv_wnd;
911 	u32 snd_max;
912 	u32 snd_una;
913 	u32 srtt;
914 	u32 rtt_var;
915 	u32 ss_thresh;
916 	u32 cwnd;
917 	u32 snd_wl1;
918 	u32 snd_wl2;
919 	u32 max_snd_window;
920 	u8 rexmit_thresh;
921 	u32 local_ipaddr[4];
922 };
923 
924 struct irdma_qp_host_ctx_info {
925 	u64 qp_compl_ctx;
926 	union {
927 		struct irdma_tcp_offload_info *tcp_info;
928 		struct irdma_udp_offload_info *udp_info;
929 	};
930 	union {
931 		struct irdma_iwarp_offload_info *iwarp_info;
932 		struct irdma_roce_offload_info *roce_info;
933 	};
934 	u32 send_cq_num;
935 	u32 rcv_cq_num;
936 	u32 rem_endpoint_idx;
937 	u8 stats_idx;
938 	bool srq_valid:1;
939 	bool tcp_info_valid:1;
940 	bool iwarp_info_valid:1;
941 	bool stats_idx_valid:1;
942 	u8 user_pri;
943 };
944 
945 struct irdma_aeqe_info {
946 	u64 compl_ctx;
947 	u32 qp_cq_id;
948 	u16 ae_id;
949 	u16 wqe_idx;
950 	u8 tcp_state;
951 	u8 iwarp_state;
952 	bool qp:1;
953 	bool cq:1;
954 	bool sq:1;
955 	bool rq:1;
956 	bool in_rdrsp_wr:1;
957 	bool out_rdrsp:1;
958 	bool aeqe_overflow:1;
959 	u8 q2_data_written;
960 	u8 ae_src;
961 };
962 
963 struct irdma_allocate_stag_info {
964 	u64 total_len;
965 	u64 first_pm_pbl_idx;
966 	u32 chunk_size;
967 	u32 stag_idx;
968 	u32 page_size;
969 	u32 pd_id;
970 	u16 access_rights;
971 	bool remote_access:1;
972 	bool use_hmc_fcn_index:1;
973 	bool use_pf_rid:1;
974 	bool all_memory:1;
975 	u8 hmc_fcn_index;
976 };
977 
978 struct irdma_mw_alloc_info {
979 	u32 mw_stag_index;
980 	u32 page_size;
981 	u32 pd_id;
982 	bool remote_access:1;
983 	bool mw_wide:1;
984 	bool mw1_bind_dont_vldt_key:1;
985 };
986 
987 struct irdma_reg_ns_stag_info {
988 	u64 reg_addr_pa;
989 	u64 va;
990 	u64 total_len;
991 	u32 page_size;
992 	u32 chunk_size;
993 	u32 first_pm_pbl_index;
994 	enum irdma_addressing_type addr_type;
995 	irdma_stag_index stag_idx;
996 	u16 access_rights;
997 	u32 pd_id;
998 	irdma_stag_key stag_key;
999 	bool use_hmc_fcn_index:1;
1000 	u8 hmc_fcn_index;
1001 	bool use_pf_rid:1;
1002 	bool all_memory:1;
1003 };
1004 
1005 struct irdma_fast_reg_stag_info {
1006 	u64 wr_id;
1007 	u64 reg_addr_pa;
1008 	u64 fbo;
1009 	void *va;
1010 	u64 total_len;
1011 	u32 page_size;
1012 	u32 chunk_size;
1013 	u32 first_pm_pbl_index;
1014 	enum irdma_addressing_type addr_type;
1015 	irdma_stag_index stag_idx;
1016 	u16 access_rights;
1017 	u32 pd_id;
1018 	irdma_stag_key stag_key;
1019 	bool local_fence:1;
1020 	bool read_fence:1;
1021 	bool signaled:1;
1022 	bool use_hmc_fcn_index:1;
1023 	u8 hmc_fcn_index;
1024 	bool use_pf_rid:1;
1025 	bool defer_flag:1;
1026 };
1027 
1028 struct irdma_dealloc_stag_info {
1029 	u32 stag_idx;
1030 	u32 pd_id;
1031 	bool mr:1;
1032 	bool dealloc_pbl:1;
1033 };
1034 
1035 struct irdma_register_shared_stag {
1036 	u64 va;
1037 	enum irdma_addressing_type addr_type;
1038 	irdma_stag_index new_stag_idx;
1039 	irdma_stag_index parent_stag_idx;
1040 	u32 access_rights;
1041 	u32 pd_id;
1042 	u32 page_size;
1043 	irdma_stag_key new_stag_key;
1044 };
1045 
1046 struct irdma_qp_init_info {
1047 	struct irdma_qp_uk_init_info qp_uk_init_info;
1048 	struct irdma_sc_pd *pd;
1049 	struct irdma_sc_vsi *vsi;
1050 	__le64 *host_ctx;
1051 	u8 *q2;
1052 	u64 sq_pa;
1053 	u64 rq_pa;
1054 	u64 host_ctx_pa;
1055 	u64 q2_pa;
1056 	u64 shadow_area_pa;
1057 	u8 sq_tph_val;
1058 	u8 rq_tph_val;
1059 	bool sq_tph_en:1;
1060 	bool rq_tph_en:1;
1061 	bool rcv_tph_en:1;
1062 	bool xmit_tph_en:1;
1063 	bool virtual_map:1;
1064 };
1065 
1066 struct irdma_cq_init_info {
1067 	struct irdma_sc_dev *dev;
1068 	u64 cq_base_pa;
1069 	u64 shadow_area_pa;
1070 	u32 ceq_id;
1071 	u32 shadow_read_threshold;
1072 	u8 pbl_chunk_size;
1073 	u32 first_pm_pbl_idx;
1074 	bool virtual_map:1;
1075 	bool ceqe_mask:1;
1076 	bool ceq_id_valid:1;
1077 	bool tph_en:1;
1078 	u8 tph_val;
1079 	u8 type;
1080 	struct irdma_cq_uk_init_info cq_uk_init_info;
1081 	struct irdma_sc_vsi *vsi;
1082 };
1083 
1084 struct irdma_upload_context_info {
1085 	u64 buf_pa;
1086 	u32 qp_id;
1087 	u8 qp_type;
1088 	bool freeze_qp:1;
1089 	bool raw_format:1;
1090 };
1091 
1092 struct irdma_local_mac_entry_info {
1093 	u8 mac_addr[6];
1094 	u16 entry_idx;
1095 };
1096 
1097 struct irdma_add_arp_cache_entry_info {
1098 	u8 mac_addr[ETH_ALEN];
1099 	u32 reach_max;
1100 	u16 arp_index;
1101 	bool permanent;
1102 };
1103 
1104 struct irdma_apbvt_info {
1105 	u16 port;
1106 	bool add;
1107 };
1108 
1109 struct irdma_qhash_table_info {
1110 	struct irdma_sc_vsi *vsi;
1111 	enum irdma_quad_hash_manage_type manage;
1112 	enum irdma_quad_entry_type entry_type;
1113 	bool vlan_valid:1;
1114 	bool ipv4_valid:1;
1115 	u8 mac_addr[ETH_ALEN];
1116 	u16 vlan_id;
1117 	u8 user_pri;
1118 	u32 qp_num;
1119 	u32 dest_ip[4];
1120 	u32 src_ip[4];
1121 	u16 dest_port;
1122 	u16 src_port;
1123 };
1124 
1125 struct irdma_cqp_manage_push_page_info {
1126 	u32 push_idx;
1127 	u16 qs_handle;
1128 	u8 free_page;
1129 	u8 push_page_type;
1130 };
1131 
1132 struct irdma_qp_flush_info {
1133 	u16 sq_minor_code;
1134 	u16 sq_major_code;
1135 	u16 rq_minor_code;
1136 	u16 rq_major_code;
1137 	u16 ae_code;
1138 	u8 ae_src;
1139 	bool sq:1;
1140 	bool rq:1;
1141 	bool userflushcode:1;
1142 	bool generate_ae:1;
1143 };
1144 
1145 struct irdma_gen_ae_info {
1146 	u16 ae_code;
1147 	u8 ae_src;
1148 };
1149 
1150 struct irdma_cqp_timeout {
1151 	u64 compl_cqp_cmds;
1152 	u32 count;
1153 };
1154 
1155 struct irdma_irq_ops {
1156 	void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable);
1157 	void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
1158 			      bool enable);
1159 	void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);
1160 	void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);
1161 };
1162 
1163 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);
1164 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
1165 			bool check_overflow, bool post_sq);
1166 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
1167 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
1168 			      struct irdma_ccq_cqe_info *info);
1169 int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
1170 		      struct irdma_ccq_init_info *info);
1171 
1172 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
1173 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
1174 
1175 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
1176 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
1177 		      struct irdma_ceq_init_info *info);
1178 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);
1179 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
1180 
1181 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
1182 		      struct irdma_aeq_init_info *info);
1183 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
1184 			   struct irdma_aeqe_info *info);
1185 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);
1186 
1187 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
1188 		      int abi_ver);
1189 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);
1190 void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
1191 			      struct irdma_sc_dev *dev);
1192 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
1193 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
1194 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
1195 		      struct irdma_cqp_init_info *info);
1196 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
1197 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
1198 				  struct irdma_ccq_cqe_info *cmpl_info);
1199 int irdma_sc_fast_register(struct irdma_sc_qp *qp,
1200 			   struct irdma_fast_reg_stag_info *info, bool post_sq);
1201 int irdma_sc_qp_create(struct irdma_sc_qp *qp,
1202 		       struct irdma_create_qp_info *info, u64 scratch,
1203 		       bool post_sq);
1204 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
1205 			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
1206 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
1207 			   struct irdma_qp_flush_info *info, u64 scratch,
1208 			   bool post_sq);
1209 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
1210 int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
1211 		       struct irdma_modify_qp_info *info, u64 scratch,
1212 		       bool post_sq);
1213 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1214 			irdma_stag stag);
1215 
1216 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
1217 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1218 			struct irdma_qp_host_ctx_info *info);
1219 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1220 			     struct irdma_qp_host_ctx_info *info);
1221 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
1222 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
1223 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
1224 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
1225 					u8 hmc_fn_id, bool post_sq,
1226 					bool poll_registers);
1227 
1228 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
1229 struct cqp_info {
1230 	union {
1231 		struct {
1232 			struct irdma_sc_qp *qp;
1233 			struct irdma_create_qp_info info;
1234 			u64 scratch;
1235 		} qp_create;
1236 
1237 		struct {
1238 			struct irdma_sc_qp *qp;
1239 			struct irdma_modify_qp_info info;
1240 			u64 scratch;
1241 		} qp_modify;
1242 
1243 		struct {
1244 			struct irdma_sc_qp *qp;
1245 			u64 scratch;
1246 			bool remove_hash_idx;
1247 			bool ignore_mw_bnd;
1248 		} qp_destroy;
1249 
1250 		struct {
1251 			struct irdma_sc_cq *cq;
1252 			u64 scratch;
1253 			bool check_overflow;
1254 		} cq_create;
1255 
1256 		struct {
1257 			struct irdma_sc_cq *cq;
1258 			struct irdma_modify_cq_info info;
1259 			u64 scratch;
1260 		} cq_modify;
1261 
1262 		struct {
1263 			struct irdma_sc_cq *cq;
1264 			u64 scratch;
1265 		} cq_destroy;
1266 
1267 		struct {
1268 			struct irdma_sc_dev *dev;
1269 			struct irdma_allocate_stag_info info;
1270 			u64 scratch;
1271 		} alloc_stag;
1272 
1273 		struct {
1274 			struct irdma_sc_dev *dev;
1275 			struct irdma_mw_alloc_info info;
1276 			u64 scratch;
1277 		} mw_alloc;
1278 
1279 		struct {
1280 			struct irdma_sc_dev *dev;
1281 			struct irdma_reg_ns_stag_info info;
1282 			u64 scratch;
1283 		} mr_reg_non_shared;
1284 
1285 		struct {
1286 			struct irdma_sc_dev *dev;
1287 			struct irdma_dealloc_stag_info info;
1288 			u64 scratch;
1289 		} dealloc_stag;
1290 
1291 		struct {
1292 			struct irdma_sc_cqp *cqp;
1293 			struct irdma_add_arp_cache_entry_info info;
1294 			u64 scratch;
1295 		} add_arp_cache_entry;
1296 
1297 		struct {
1298 			struct irdma_sc_cqp *cqp;
1299 			u64 scratch;
1300 			u16 arp_index;
1301 		} del_arp_cache_entry;
1302 
1303 		struct {
1304 			struct irdma_sc_cqp *cqp;
1305 			struct irdma_local_mac_entry_info info;
1306 			u64 scratch;
1307 		} add_local_mac_entry;
1308 
1309 		struct {
1310 			struct irdma_sc_cqp *cqp;
1311 			u64 scratch;
1312 			u8 entry_idx;
1313 			u8 ignore_ref_count;
1314 		} del_local_mac_entry;
1315 
1316 		struct {
1317 			struct irdma_sc_cqp *cqp;
1318 			u64 scratch;
1319 		} alloc_local_mac_entry;
1320 
1321 		struct {
1322 			struct irdma_sc_cqp *cqp;
1323 			struct irdma_cqp_manage_push_page_info info;
1324 			u64 scratch;
1325 		} manage_push_page;
1326 
1327 		struct {
1328 			struct irdma_sc_dev *dev;
1329 			struct irdma_upload_context_info info;
1330 			u64 scratch;
1331 		} qp_upload_context;
1332 
1333 		struct {
1334 			struct irdma_sc_dev *dev;
1335 			struct irdma_hmc_fcn_info info;
1336 			u64 scratch;
1337 		} manage_hmc_pm;
1338 
1339 		struct {
1340 			struct irdma_sc_ceq *ceq;
1341 			u64 scratch;
1342 		} ceq_create;
1343 
1344 		struct {
1345 			struct irdma_sc_ceq *ceq;
1346 			u64 scratch;
1347 		} ceq_destroy;
1348 
1349 		struct {
1350 			struct irdma_sc_aeq *aeq;
1351 			u64 scratch;
1352 		} aeq_create;
1353 
1354 		struct {
1355 			struct irdma_sc_aeq *aeq;
1356 			u64 scratch;
1357 		} aeq_destroy;
1358 
1359 		struct {
1360 			struct irdma_sc_qp *qp;
1361 			struct irdma_qp_flush_info info;
1362 			u64 scratch;
1363 		} qp_flush_wqes;
1364 
1365 		struct {
1366 			struct irdma_sc_qp *qp;
1367 			struct irdma_gen_ae_info info;
1368 			u64 scratch;
1369 		} gen_ae;
1370 
1371 		struct {
1372 			struct irdma_sc_cqp *cqp;
1373 			void *fpm_val_va;
1374 			u64 fpm_val_pa;
1375 			u8 hmc_fn_id;
1376 			u64 scratch;
1377 		} query_fpm_val;
1378 
1379 		struct {
1380 			struct irdma_sc_cqp *cqp;
1381 			void *fpm_val_va;
1382 			u64 fpm_val_pa;
1383 			u8 hmc_fn_id;
1384 			u64 scratch;
1385 		} commit_fpm_val;
1386 
1387 		struct {
1388 			struct irdma_sc_cqp *cqp;
1389 			struct irdma_apbvt_info info;
1390 			u64 scratch;
1391 		} manage_apbvt_entry;
1392 
1393 		struct {
1394 			struct irdma_sc_cqp *cqp;
1395 			struct irdma_qhash_table_info info;
1396 			u64 scratch;
1397 		} manage_qhash_table_entry;
1398 
1399 		struct {
1400 			struct irdma_sc_dev *dev;
1401 			struct irdma_update_sds_info info;
1402 			u64 scratch;
1403 		} update_pe_sds;
1404 
1405 		struct {
1406 			struct irdma_sc_cqp *cqp;
1407 			struct irdma_sc_qp *qp;
1408 			u64 scratch;
1409 		} suspend_resume;
1410 
1411 		struct {
1412 			struct irdma_sc_cqp *cqp;
1413 			struct irdma_ah_info info;
1414 			u64 scratch;
1415 		} ah_create;
1416 
1417 		struct {
1418 			struct irdma_sc_cqp *cqp;
1419 			struct irdma_ah_info info;
1420 			u64 scratch;
1421 		} ah_destroy;
1422 
1423 		struct {
1424 			struct irdma_sc_cqp *cqp;
1425 			struct irdma_mcast_grp_info info;
1426 			u64 scratch;
1427 		} mc_create;
1428 
1429 		struct {
1430 			struct irdma_sc_cqp *cqp;
1431 			struct irdma_mcast_grp_info info;
1432 			u64 scratch;
1433 		} mc_destroy;
1434 
1435 		struct {
1436 			struct irdma_sc_cqp *cqp;
1437 			struct irdma_mcast_grp_info info;
1438 			u64 scratch;
1439 		} mc_modify;
1440 
1441 		struct {
1442 			struct irdma_sc_cqp *cqp;
1443 			struct irdma_stats_inst_info info;
1444 			u64 scratch;
1445 		} stats_manage;
1446 
1447 		struct {
1448 			struct irdma_sc_cqp *cqp;
1449 			struct irdma_stats_gather_info info;
1450 			u64 scratch;
1451 		} stats_gather;
1452 
1453 		struct {
1454 			struct irdma_sc_cqp *cqp;
1455 			struct irdma_ws_node_info info;
1456 			u64 scratch;
1457 		} ws_node;
1458 
1459 		struct {
1460 			struct irdma_sc_cqp *cqp;
1461 			struct irdma_up_info info;
1462 			u64 scratch;
1463 		} up_map;
1464 
1465 		struct {
1466 			struct irdma_sc_cqp *cqp;
1467 			struct irdma_dma_mem query_buff_mem;
1468 			u64 scratch;
1469 		} query_rdma;
1470 	} u;
1471 };
1472 
1473 struct cqp_cmds_info {
1474 	struct list_head cqp_cmd_entry;
1475 	u8 cqp_cmd;
1476 	u8 post_sq;
1477 	struct cqp_info in;
1478 };
1479 
1480 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
1481 					   u32 *wqe_idx);
1482 
1483 /**
1484  * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
1485  * @cqp: struct for cqp hw
1486  * @scratch: private data for CQP WQE
1487  */
irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp * cqp,u64 scratch)1488 static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)
1489 {
1490 	u32 wqe_idx;
1491 
1492 	return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
1493 }
1494 #endif /* IRDMA_TYPE_H */
1495