xref: /linux/drivers/crypto/hisilicon/qm.c (revision ab93e0dd72c37d378dd936f031ffb83ff2bd87ce)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <asm/page.h>
4 #include <linux/acpi.h>
5 #include <linux/bitmap.h>
6 #include <linux/dma-mapping.h>
7 #include <linux/idr.h>
8 #include <linux/io.h>
9 #include <linux/irqreturn.h>
10 #include <linux/log2.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/seq_file.h>
13 #include <linux/slab.h>
14 #include <linux/uacce.h>
15 #include <linux/uaccess.h>
16 #include <uapi/misc/uacce/hisi_qm.h>
17 #include <linux/hisi_acc_qm.h>
18 #include "qm_common.h"
19 
20 /* eq/aeq irq enable */
21 #define QM_VF_AEQ_INT_SOURCE		0x0
22 #define QM_VF_AEQ_INT_MASK		0x4
23 #define QM_VF_EQ_INT_SOURCE		0x8
24 #define QM_VF_EQ_INT_MASK		0xc
25 
26 #define QM_IRQ_VECTOR_MASK		GENMASK(15, 0)
27 #define QM_IRQ_TYPE_MASK		GENMASK(15, 0)
28 #define QM_IRQ_TYPE_SHIFT		16
29 #define QM_ABN_IRQ_TYPE_MASK		GENMASK(7, 0)
30 
31 /* mailbox */
32 #define QM_MB_PING_ALL_VFS		0xffff
33 #define QM_MB_STATUS_MASK		GENMASK(12, 9)
34 
35 /* sqc shift */
36 #define QM_SQ_HOP_NUM_SHIFT		0
37 #define QM_SQ_PAGE_SIZE_SHIFT		4
38 #define QM_SQ_BUF_SIZE_SHIFT		8
39 #define QM_SQ_SQE_SIZE_SHIFT		12
40 #define QM_SQ_PRIORITY_SHIFT		0
41 #define QM_SQ_ORDERS_SHIFT		4
42 #define QM_SQ_TYPE_SHIFT		8
43 #define QM_QC_PASID_ENABLE		0x1
44 #define QM_QC_PASID_ENABLE_SHIFT	7
45 
46 #define QM_SQ_TYPE_MASK			GENMASK(3, 0)
47 #define QM_SQ_TAIL_IDX(sqc)		((le16_to_cpu((sqc).w11) >> 6) & 0x1)
48 
49 /* cqc shift */
50 #define QM_CQ_HOP_NUM_SHIFT		0
51 #define QM_CQ_PAGE_SIZE_SHIFT		4
52 #define QM_CQ_BUF_SIZE_SHIFT		8
53 #define QM_CQ_CQE_SIZE_SHIFT		12
54 #define QM_CQ_PHASE_SHIFT		0
55 #define QM_CQ_FLAG_SHIFT		1
56 
57 #define QM_CQE_PHASE(cqe)		(le16_to_cpu((cqe)->w7) & 0x1)
58 #define QM_QC_CQE_SIZE			4
59 #define QM_CQ_TAIL_IDX(cqc)		((le16_to_cpu((cqc).w11) >> 6) & 0x1)
60 
61 /* eqc shift */
62 #define QM_EQE_AEQE_SIZE		(2UL << 12)
63 #define QM_EQC_PHASE_SHIFT		16
64 
65 #define QM_EQE_PHASE(eqe)		((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)
66 #define QM_EQE_CQN_MASK			GENMASK(15, 0)
67 
68 #define QM_AEQE_PHASE(aeqe)		((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
69 #define QM_AEQE_TYPE_SHIFT		17
70 #define QM_AEQE_TYPE_MASK		0xf
71 #define QM_AEQE_CQN_MASK		GENMASK(15, 0)
72 #define QM_CQ_OVERFLOW			0
73 #define QM_EQ_OVERFLOW			1
74 #define QM_CQE_ERROR			2
75 
76 #define QM_XQ_DEPTH_SHIFT		16
77 #define QM_XQ_DEPTH_MASK		GENMASK(15, 0)
78 
79 #define QM_DOORBELL_CMD_SQ		0
80 #define QM_DOORBELL_CMD_CQ		1
81 #define QM_DOORBELL_CMD_EQ		2
82 #define QM_DOORBELL_CMD_AEQ		3
83 
84 #define QM_DOORBELL_BASE_V1		0x340
85 #define QM_DB_CMD_SHIFT_V1		16
86 #define QM_DB_INDEX_SHIFT_V1		32
87 #define QM_DB_PRIORITY_SHIFT_V1		48
88 #define QM_PAGE_SIZE			0x0034
89 #define QM_QP_DB_INTERVAL		0x10000
90 #define QM_DB_TIMEOUT_CFG		0x100074
91 #define QM_DB_TIMEOUT_SET		0x1fffff
92 
93 #define QM_MEM_START_INIT		0x100040
94 #define QM_MEM_INIT_DONE		0x100044
95 #define QM_VFT_CFG_RDY			0x10006c
96 #define QM_VFT_CFG_OP_WR		0x100058
97 #define QM_VFT_CFG_TYPE			0x10005c
98 #define QM_VFT_CFG			0x100060
99 #define QM_VFT_CFG_OP_ENABLE		0x100054
100 #define QM_PM_CTRL			0x100148
101 #define QM_IDLE_DISABLE			BIT(9)
102 
103 #define QM_SUB_VERSION_ID		0x210
104 
105 #define QM_VFT_CFG_DATA_L		0x100064
106 #define QM_VFT_CFG_DATA_H		0x100068
107 #define QM_SQC_VFT_BUF_SIZE		(7ULL << 8)
108 #define QM_SQC_VFT_SQC_SIZE		(5ULL << 12)
109 #define QM_SQC_VFT_INDEX_NUMBER		(1ULL << 16)
110 #define QM_SQC_VFT_START_SQN_SHIFT	28
111 #define QM_SQC_VFT_VALID		(1ULL << 44)
112 #define QM_SQC_VFT_SQN_SHIFT		45
113 #define QM_CQC_VFT_BUF_SIZE		(7ULL << 8)
114 #define QM_CQC_VFT_SQC_SIZE		(5ULL << 12)
115 #define QM_CQC_VFT_INDEX_NUMBER		(1ULL << 16)
116 #define QM_CQC_VFT_VALID		(1ULL << 28)
117 
118 #define QM_SQC_VFT_BASE_SHIFT_V2	28
119 #define QM_SQC_VFT_BASE_MASK_V2		GENMASK(15, 0)
120 #define QM_SQC_VFT_NUM_SHIFT_V2		45
121 #define QM_SQC_VFT_NUM_MASK_V2		GENMASK(9, 0)
122 #define QM_MAX_QC_TYPE                  2
123 
124 #define QM_ABNORMAL_INT_SOURCE		0x100000
125 #define QM_ABNORMAL_INT_MASK		0x100004
126 #define QM_ABNORMAL_INT_MASK_VALUE	0x7fff
127 #define QM_ABNORMAL_INT_STATUS		0x100008
128 #define QM_ABNORMAL_INT_SET		0x10000c
129 #define QM_ABNORMAL_INF00		0x100010
130 #define QM_FIFO_OVERFLOW_TYPE		0xc0
131 #define QM_FIFO_OVERFLOW_TYPE_SHIFT	6
132 #define QM_FIFO_OVERFLOW_VF		0x3f
133 #define QM_FIFO_OVERFLOW_QP_SHIFT	16
134 #define QM_ABNORMAL_INF01		0x100014
135 #define QM_DB_TIMEOUT_TYPE		0xc0
136 #define QM_DB_TIMEOUT_TYPE_SHIFT	6
137 #define QM_DB_TIMEOUT_VF		0x3f
138 #define QM_DB_TIMEOUT_QP_SHIFT		16
139 #define QM_ABNORMAL_INF02		0x100018
140 #define QM_AXI_POISON_ERR		BIT(22)
141 #define QM_RAS_CE_ENABLE		0x1000ec
142 #define QM_RAS_FE_ENABLE		0x1000f0
143 #define QM_RAS_NFE_ENABLE		0x1000f4
144 #define QM_RAS_CE_THRESHOLD		0x1000f8
145 #define QM_RAS_CE_TIMES_PER_IRQ		1
146 #define QM_OOO_SHUTDOWN_SEL		0x1040f8
147 #define QM_AXI_RRESP_ERR		BIT(0)
148 #define QM_ECC_MBIT			BIT(2)
149 #define QM_DB_TIMEOUT			BIT(10)
150 #define QM_OF_FIFO_OF			BIT(11)
151 
152 #define QM_RESET_WAIT_TIMEOUT		400
153 #define QM_PEH_VENDOR_ID		0x1000d8
154 #define ACC_VENDOR_ID_VALUE		0x5a5a
155 #define QM_PEH_DFX_INFO0		0x1000fc
156 #define QM_PEH_DFX_INFO1		0x100100
157 #define QM_PEH_DFX_MASK			(BIT(0) | BIT(2))
158 #define QM_PEH_MSI_FINISH_MASK		GENMASK(19, 16)
159 #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT	3
160 #define ACC_PEH_MSI_DISABLE		GENMASK(31, 0)
161 #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN	0x1
162 #define ACC_MASTER_TRANS_RETURN_RW	3
163 #define ACC_MASTER_TRANS_RETURN		0x300150
164 #define ACC_MASTER_GLOBAL_CTRL		0x300000
165 #define ACC_AM_CFG_PORT_WR_EN		0x30001c
166 #define QM_RAS_NFE_MBIT_DISABLE		~QM_ECC_MBIT
167 #define ACC_AM_ROB_ECC_INT_STS		0x300104
168 #define ACC_ROB_ECC_ERR_MULTPL		BIT(1)
169 #define QM_MSI_CAP_ENABLE		BIT(16)
170 
171 /* interfunction communication */
172 #define QM_IFC_READY_STATUS		0x100128
173 #define QM_IFC_INT_SET_P		0x100130
174 #define QM_IFC_INT_CFG			0x100134
175 #define QM_IFC_INT_SOURCE_P		0x100138
176 #define QM_IFC_INT_SOURCE_V		0x0020
177 #define QM_IFC_INT_MASK			0x0024
178 #define QM_IFC_INT_STATUS		0x0028
179 #define QM_IFC_INT_SET_V		0x002C
180 #define QM_PF2VF_PF_W			0x104700
181 #define QM_VF2PF_PF_R			0x104800
182 #define QM_VF2PF_VF_W			0x320
183 #define QM_PF2VF_VF_R			0x380
184 #define QM_IFC_SEND_ALL_VFS		GENMASK(6, 0)
185 #define QM_IFC_INT_SOURCE_CLR		GENMASK(63, 0)
186 #define QM_IFC_INT_SOURCE_MASK		BIT(0)
187 #define QM_IFC_INT_DISABLE		BIT(0)
188 #define QM_IFC_INT_STATUS_MASK		BIT(0)
189 #define QM_IFC_INT_SET_MASK		BIT(0)
190 #define QM_WAIT_DST_ACK			10
191 #define QM_MAX_PF_WAIT_COUNT		10
192 #define QM_MAX_VF_WAIT_COUNT		40
193 #define QM_VF_RESET_WAIT_US		20000
194 #define QM_VF_RESET_WAIT_CNT		3000
195 #define QM_VF2PF_REG_SIZE		4
196 #define QM_IFC_CMD_MASK			GENMASK(31, 0)
197 #define QM_IFC_DATA_SHIFT		32
198 #define QM_VF_RESET_WAIT_TIMEOUT_US    \
199 	(QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT)
200 
201 #define POLL_PERIOD			10
202 #define POLL_TIMEOUT			1000
203 #define WAIT_PERIOD_US_MAX		200
204 #define WAIT_PERIOD_US_MIN		100
205 #define MAX_WAIT_COUNTS			1000
206 #define QM_CACHE_WB_START		0x204
207 #define QM_CACHE_WB_DONE		0x208
208 #define QM_FUNC_CAPS_REG		0x3100
209 #define QM_CAPBILITY_VERSION		GENMASK(7, 0)
210 
211 #define PCI_BAR_2			2
212 #define PCI_BAR_4			4
213 #define QMC_ALIGN(sz)			ALIGN(sz, 32)
214 
215 #define QM_DBG_READ_LEN		256
216 #define QM_PCI_COMMAND_INVALID		~0
217 #define QM_RESET_STOP_TX_OFFSET		1
218 #define QM_RESET_STOP_RX_OFFSET		2
219 
220 #define WAIT_PERIOD			20
221 #define REMOVE_WAIT_DELAY		10
222 
223 #define QM_QOS_PARAM_NUM		2
224 #define QM_QOS_MAX_VAL			1000
225 #define QM_QOS_RATE			100
226 #define QM_QOS_EXPAND_RATE		1000
227 #define QM_SHAPER_CIR_B_MASK		GENMASK(7, 0)
228 #define QM_SHAPER_CIR_U_MASK		GENMASK(10, 8)
229 #define QM_SHAPER_CIR_S_MASK		GENMASK(14, 11)
230 #define QM_SHAPER_FACTOR_CIR_U_SHIFT	8
231 #define QM_SHAPER_FACTOR_CIR_S_SHIFT	11
232 #define QM_SHAPER_FACTOR_CBS_B_SHIFT	15
233 #define QM_SHAPER_FACTOR_CBS_S_SHIFT	19
234 #define QM_SHAPER_CBS_B			1
235 #define QM_SHAPER_VFT_OFFSET		6
236 #define QM_QOS_MIN_ERROR_RATE		5
237 #define QM_SHAPER_MIN_CBS_S		8
238 #define QM_QOS_TICK			0x300U
239 #define QM_QOS_DIVISOR_CLK		0x1f40U
240 #define QM_QOS_MAX_CIR_B		200
241 #define QM_QOS_MIN_CIR_B		100
242 #define QM_QOS_MAX_CIR_U		6
243 #define QM_AUTOSUSPEND_DELAY		3000
244 
245  /* abnormal status value for stopping queue */
246 #define QM_STOP_QUEUE_FAIL		1
247 #define	QM_DUMP_SQC_FAIL		3
248 #define	QM_DUMP_CQC_FAIL		4
249 #define	QM_FINISH_WAIT			5
250 
251 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
252 	(((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \
253 	((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \
254 	((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \
255 	((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
256 
257 #define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \
258 	((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
259 
260 #define QM_MK_SQC_W13(priority, orders, alg_type) \
261 	(((priority) << QM_SQ_PRIORITY_SHIFT) | \
262 	((orders) << QM_SQ_ORDERS_SHIFT) | \
263 	(((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
264 
265 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
266 	(((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \
267 	((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \
268 	((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \
269 	((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
270 
271 #define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \
272 	((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
273 
274 enum vft_type {
275 	SQC_VFT = 0,
276 	CQC_VFT,
277 	SHAPER_VFT,
278 };
279 
280 enum qm_alg_type {
281 	ALG_TYPE_0,
282 	ALG_TYPE_1,
283 };
284 
285 enum qm_ifc_cmd {
286 	QM_PF_FLR_PREPARE = 0x01,
287 	QM_PF_SRST_PREPARE,
288 	QM_PF_RESET_DONE,
289 	QM_VF_PREPARE_DONE,
290 	QM_VF_PREPARE_FAIL,
291 	QM_VF_START_DONE,
292 	QM_VF_START_FAIL,
293 	QM_PF_SET_QOS,
294 	QM_VF_GET_QOS,
295 };
296 
297 enum qm_basic_type {
298 	QM_TOTAL_QP_NUM_CAP = 0x0,
299 	QM_FUNC_MAX_QP_CAP,
300 	QM_XEQ_DEPTH_CAP,
301 	QM_QP_DEPTH_CAP,
302 	QM_EQ_IRQ_TYPE_CAP,
303 	QM_AEQ_IRQ_TYPE_CAP,
304 	QM_ABN_IRQ_TYPE_CAP,
305 	QM_PF2VF_IRQ_TYPE_CAP,
306 	QM_PF_IRQ_NUM_CAP,
307 	QM_VF_IRQ_NUM_CAP,
308 };
309 
310 enum qm_cap_table_type {
311 	QM_CAP_VF  = 0x0,
312 	QM_AEQE_NUM,
313 	QM_SCQE_NUM,
314 	QM_EQ_IRQ,
315 	QM_AEQ_IRQ,
316 	QM_ABNORMAL_IRQ,
317 	QM_MB_IRQ,
318 	MAX_IRQ_NUM,
319 	EXT_BAR_INDEX,
320 };
321 
322 static const struct hisi_qm_cap_query_info qm_cap_query_info[] = {
323 	{QM_CAP_VF, "QM_CAP_VF                   ", 0x3100, 0x0, 0x0, 0x6F01},
324 	{QM_AEQE_NUM, "QM_AEQE_NUM                 ", 0x3104, 0x800, 0x4000800, 0x4000800},
325 	{QM_SCQE_NUM, "QM_SCQE_NUM                 ",
326 						0x3108, 0x4000400, 0x4000400, 0x4000400},
327 	{QM_EQ_IRQ, "QM_EQ_IRQ                   ", 0x310c, 0x10000, 0x10000, 0x10000},
328 	{QM_AEQ_IRQ, "QM_AEQ_IRQ                  ", 0x3110, 0x0, 0x10001, 0x10001},
329 	{QM_ABNORMAL_IRQ, "QM_ABNORMAL_IRQ             ", 0x3114, 0x0, 0x10003, 0x10003},
330 	{QM_MB_IRQ, "QM_MB_IRQ                   ", 0x3118, 0x0, 0x0, 0x10002},
331 	{MAX_IRQ_NUM, "MAX_IRQ_NUM                 ", 0x311c, 0x10001, 0x40002, 0x40003},
332 	{EXT_BAR_INDEX, "EXT_BAR_INDEX               ", 0x3120, 0x0, 0x0, 0x14},
333 };
334 
335 static const struct hisi_qm_cap_info qm_cap_info_comm[] = {
336 	{QM_SUPPORT_DB_ISOLATION, 0x30,   0, BIT(0),  0x0, 0x0, 0x0},
337 	{QM_SUPPORT_FUNC_QOS,     0x3100, 0, BIT(8),  0x0, 0x0, 0x1},
338 	{QM_SUPPORT_STOP_QP,      0x3100, 0, BIT(9),  0x0, 0x0, 0x1},
339 	{QM_SUPPORT_STOP_FUNC,     0x3100, 0, BIT(10), 0x0, 0x0, 0x1},
340 	{QM_SUPPORT_MB_COMMAND,   0x3100, 0, BIT(11), 0x0, 0x0, 0x1},
341 	{QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1},
342 	{QM_SUPPORT_DAE,          0x3100, 0, BIT(15), 0x0, 0x0, 0x0},
343 };
344 
345 static const struct hisi_qm_cap_info qm_cap_info_pf[] = {
346 	{QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1},
347 };
348 
349 static const struct hisi_qm_cap_info qm_cap_info_vf[] = {
350 	{QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0},
351 };
352 
353 static const struct hisi_qm_cap_info qm_basic_info[] = {
354 	{QM_TOTAL_QP_NUM_CAP,   0x100158, 0,  GENMASK(10, 0), 0x1000,    0x400,     0x400},
355 	{QM_FUNC_MAX_QP_CAP,    0x100158, 11, GENMASK(10, 0), 0x1000,    0x400,     0x400},
356 	{QM_XEQ_DEPTH_CAP,      0x3104,   0,  GENMASK(31, 0), 0x800,     0x4000800, 0x4000800},
357 	{QM_QP_DEPTH_CAP,       0x3108,   0,  GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400},
358 	{QM_EQ_IRQ_TYPE_CAP,    0x310c,   0,  GENMASK(31, 0), 0x10000,   0x10000,   0x10000},
359 	{QM_AEQ_IRQ_TYPE_CAP,   0x3110,   0,  GENMASK(31, 0), 0x0,       0x10001,   0x10001},
360 	{QM_ABN_IRQ_TYPE_CAP,   0x3114,   0,  GENMASK(31, 0), 0x0,       0x10003,   0x10003},
361 	{QM_PF2VF_IRQ_TYPE_CAP, 0x3118,   0,  GENMASK(31, 0), 0x0,       0x0,       0x10002},
362 	{QM_PF_IRQ_NUM_CAP,     0x311c,   16, GENMASK(15, 0), 0x1,       0x4,       0x4},
363 	{QM_VF_IRQ_NUM_CAP,     0x311c,   0,  GENMASK(15, 0), 0x1,       0x2,       0x3},
364 };
365 
366 struct qm_mailbox {
367 	__le16 w0;
368 	__le16 queue_num;
369 	__le32 base_l;
370 	__le32 base_h;
371 	__le32 rsvd;
372 };
373 
374 struct qm_doorbell {
375 	__le16 queue_num;
376 	__le16 cmd;
377 	__le16 index;
378 	__le16 priority;
379 };
380 
381 struct hisi_qm_resource {
382 	struct hisi_qm *qm;
383 	int distance;
384 	struct list_head list;
385 };
386 
387 /**
388  * struct qm_hw_err - Structure describing the device errors
389  * @list: hardware error list
390  * @timestamp: timestamp when the error occurred
391  */
392 struct qm_hw_err {
393 	struct list_head list;
394 	unsigned long long timestamp;
395 };
396 
397 struct hisi_qm_hw_ops {
398 	int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
399 	void (*qm_db)(struct hisi_qm *qm, u16 qn,
400 		      u8 cmd, u16 index, u8 priority);
401 	int (*debug_init)(struct hisi_qm *qm);
402 	void (*hw_error_init)(struct hisi_qm *qm);
403 	void (*hw_error_uninit)(struct hisi_qm *qm);
404 	enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
405 	int (*set_msi)(struct hisi_qm *qm, bool set);
406 
407 	/* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */
408 	int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num);
409 	void (*set_ifc_end)(struct hisi_qm *qm);
410 	int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num);
411 };
412 
413 struct hisi_qm_hw_error {
414 	u32 int_msk;
415 	const char *msg;
416 };
417 
418 static const struct hisi_qm_hw_error qm_hw_error[] = {
419 	{ .int_msk = BIT(0), .msg = "qm_axi_rresp" },
420 	{ .int_msk = BIT(1), .msg = "qm_axi_bresp" },
421 	{ .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
422 	{ .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
423 	{ .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
424 	{ .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
425 	{ .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
426 	{ .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
427 	{ .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
428 	{ .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
429 	{ .int_msk = BIT(10), .msg = "qm_db_timeout" },
430 	{ .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
431 	{ .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
432 	{ .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
433 	{ .int_msk = BIT(14), .msg = "qm_flr_timeout" },
434 };
435 
436 static const char * const qm_db_timeout[] = {
437 	"sq", "cq", "eq", "aeq",
438 };
439 
440 static const char * const qm_fifo_overflow[] = {
441 	"cq", "eq", "aeq",
442 };
443 
444 struct qm_typical_qos_table {
445 	u32 start;
446 	u32 end;
447 	u32 val;
448 };
449 
450 /* the qos step is 100 */
451 static struct qm_typical_qos_table shaper_cir_s[] = {
452 	{100, 100, 4},
453 	{200, 200, 3},
454 	{300, 500, 2},
455 	{600, 1000, 1},
456 	{1100, 100000, 0},
457 };
458 
459 static struct qm_typical_qos_table shaper_cbs_s[] = {
460 	{100, 200, 9},
461 	{300, 500, 11},
462 	{600, 1000, 12},
463 	{1100, 10000, 16},
464 	{10100, 25000, 17},
465 	{25100, 50000, 18},
466 	{50100, 100000, 19}
467 };
468 
469 static void qm_irqs_unregister(struct hisi_qm *qm);
470 static int qm_reset_device(struct hisi_qm *qm);
hisi_qm_q_num_set(const char * val,const struct kernel_param * kp,unsigned int device)471 int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp,
472 		      unsigned int device)
473 {
474 	struct pci_dev *pdev;
475 	u32 n, q_num;
476 	int ret;
477 
478 	if (!val)
479 		return -EINVAL;
480 
481 	pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL);
482 	if (!pdev) {
483 		q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2);
484 		pr_info("No device found currently, suppose queue number is %u\n",
485 			q_num);
486 	} else {
487 		if (pdev->revision == QM_HW_V1)
488 			q_num = QM_QNUM_V1;
489 		else
490 			q_num = QM_QNUM_V2;
491 
492 		pci_dev_put(pdev);
493 	}
494 
495 	ret = kstrtou32(val, 10, &n);
496 	if (ret || n < QM_MIN_QNUM || n > q_num)
497 		return -EINVAL;
498 
499 	return param_set_int(val, kp);
500 }
501 EXPORT_SYMBOL_GPL(hisi_qm_q_num_set);
502 
qm_get_hw_error_status(struct hisi_qm * qm)503 static u32 qm_get_hw_error_status(struct hisi_qm *qm)
504 {
505 	return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
506 }
507 
qm_get_dev_err_status(struct hisi_qm * qm)508 static u32 qm_get_dev_err_status(struct hisi_qm *qm)
509 {
510 	return qm->err_ini->get_dev_hw_err_status(qm);
511 }
512 
513 /* Check if the error causes the master ooo block */
qm_check_dev_error(struct hisi_qm * qm)514 static bool qm_check_dev_error(struct hisi_qm *qm)
515 {
516 	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
517 	u32 err_status;
518 
519 	if (pf_qm->fun_type == QM_HW_VF)
520 		return false;
521 
522 	err_status = qm_get_hw_error_status(pf_qm);
523 	if (err_status & pf_qm->err_info.qm_shutdown_mask)
524 		return true;
525 
526 	if (pf_qm->err_ini->dev_is_abnormal)
527 		return pf_qm->err_ini->dev_is_abnormal(pf_qm);
528 
529 	return false;
530 }
531 
qm_wait_reset_finish(struct hisi_qm * qm)532 static int qm_wait_reset_finish(struct hisi_qm *qm)
533 {
534 	int delay = 0;
535 
536 	/* All reset requests need to be queued for processing */
537 	while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
538 		msleep(++delay);
539 		if (delay > QM_RESET_WAIT_TIMEOUT)
540 			return -EBUSY;
541 	}
542 
543 	return 0;
544 }
545 
qm_reset_prepare_ready(struct hisi_qm * qm)546 static int qm_reset_prepare_ready(struct hisi_qm *qm)
547 {
548 	struct pci_dev *pdev = qm->pdev;
549 	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
550 
551 	/*
552 	 * PF and VF on host doesnot support resetting at the
553 	 * same time on Kunpeng920.
554 	 */
555 	if (qm->ver < QM_HW_V3)
556 		return qm_wait_reset_finish(pf_qm);
557 
558 	return qm_wait_reset_finish(qm);
559 }
560 
qm_reset_bit_clear(struct hisi_qm * qm)561 static void qm_reset_bit_clear(struct hisi_qm *qm)
562 {
563 	struct pci_dev *pdev = qm->pdev;
564 	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
565 
566 	if (qm->ver < QM_HW_V3)
567 		clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
568 
569 	clear_bit(QM_RESETTING, &qm->misc_ctl);
570 }
571 
qm_mb_pre_init(struct qm_mailbox * mailbox,u8 cmd,u64 base,u16 queue,bool op)572 static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,
573 			   u64 base, u16 queue, bool op)
574 {
575 	mailbox->w0 = cpu_to_le16((cmd) |
576 		((op) ? 0x1 << QM_MB_OP_SHIFT : 0) |
577 		(0x1 << QM_MB_BUSY_SHIFT));
578 	mailbox->queue_num = cpu_to_le16(queue);
579 	mailbox->base_l = cpu_to_le32(lower_32_bits(base));
580 	mailbox->base_h = cpu_to_le32(upper_32_bits(base));
581 	mailbox->rsvd = 0;
582 }
583 
584 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
hisi_qm_wait_mb_ready(struct hisi_qm * qm)585 int hisi_qm_wait_mb_ready(struct hisi_qm *qm)
586 {
587 	u32 val;
588 
589 	return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
590 					  val, !((val >> QM_MB_BUSY_SHIFT) &
591 					  0x1), POLL_PERIOD, POLL_TIMEOUT);
592 }
593 EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready);
594 
595 /* 128 bit should be written to hardware at one time to trigger a mailbox */
qm_mb_write(struct hisi_qm * qm,const void * src)596 static void qm_mb_write(struct hisi_qm *qm, const void *src)
597 {
598 	void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
599 
600 #if IS_ENABLED(CONFIG_ARM64)
601 	unsigned long tmp0 = 0, tmp1 = 0;
602 #endif
603 
604 	if (!IS_ENABLED(CONFIG_ARM64)) {
605 		memcpy_toio(fun_base, src, 16);
606 		dma_wmb();
607 		return;
608 	}
609 
610 #if IS_ENABLED(CONFIG_ARM64)
611 	asm volatile("ldp %0, %1, %3\n"
612 		     "stp %0, %1, %2\n"
613 		     "dmb oshst\n"
614 		     : "=&r" (tmp0),
615 		       "=&r" (tmp1),
616 		       "+Q" (*((char __iomem *)fun_base))
617 		     : "Q" (*((char *)src))
618 		     : "memory");
619 #endif
620 }
621 
qm_mb_nolock(struct hisi_qm * qm,struct qm_mailbox * mailbox)622 static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox)
623 {
624 	int ret;
625 	u32 val;
626 
627 	if (unlikely(hisi_qm_wait_mb_ready(qm))) {
628 		dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
629 		ret = -EBUSY;
630 		goto mb_busy;
631 	}
632 
633 	qm_mb_write(qm, mailbox);
634 
635 	if (unlikely(hisi_qm_wait_mb_ready(qm))) {
636 		dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
637 		ret = -ETIMEDOUT;
638 		goto mb_busy;
639 	}
640 
641 	val = readl(qm->io_base + QM_MB_CMD_SEND_BASE);
642 	if (val & QM_MB_STATUS_MASK) {
643 		dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n");
644 		ret = -EIO;
645 		goto mb_busy;
646 	}
647 
648 	return 0;
649 
650 mb_busy:
651 	atomic64_inc(&qm->debug.dfx.mb_err_cnt);
652 	return ret;
653 }
654 
hisi_qm_mb(struct hisi_qm * qm,u8 cmd,dma_addr_t dma_addr,u16 queue,bool op)655 int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
656 	       bool op)
657 {
658 	struct qm_mailbox mailbox;
659 	int ret;
660 
661 	qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op);
662 
663 	mutex_lock(&qm->mailbox_lock);
664 	ret = qm_mb_nolock(qm, &mailbox);
665 	mutex_unlock(&qm->mailbox_lock);
666 
667 	return ret;
668 }
669 EXPORT_SYMBOL_GPL(hisi_qm_mb);
670 
671 /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */
qm_set_and_get_xqc(struct hisi_qm * qm,u8 cmd,void * xqc,u32 qp_id,bool op)672 int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op)
673 {
674 	struct qm_mailbox mailbox;
675 	dma_addr_t xqc_dma;
676 	void *tmp_xqc;
677 	size_t size;
678 	int ret;
679 
680 	switch (cmd) {
681 	case QM_MB_CMD_SQC:
682 		size = sizeof(struct qm_sqc);
683 		tmp_xqc = qm->xqc_buf.sqc;
684 		xqc_dma = qm->xqc_buf.sqc_dma;
685 		break;
686 	case QM_MB_CMD_CQC:
687 		size = sizeof(struct qm_cqc);
688 		tmp_xqc = qm->xqc_buf.cqc;
689 		xqc_dma = qm->xqc_buf.cqc_dma;
690 		break;
691 	case QM_MB_CMD_EQC:
692 		size = sizeof(struct qm_eqc);
693 		tmp_xqc = qm->xqc_buf.eqc;
694 		xqc_dma = qm->xqc_buf.eqc_dma;
695 		break;
696 	case QM_MB_CMD_AEQC:
697 		size = sizeof(struct qm_aeqc);
698 		tmp_xqc = qm->xqc_buf.aeqc;
699 		xqc_dma = qm->xqc_buf.aeqc_dma;
700 		break;
701 	default:
702 		dev_err(&qm->pdev->dev, "unknown mailbox cmd %u\n", cmd);
703 		return -EINVAL;
704 	}
705 
706 	/* Setting xqc will fail if master OOO is blocked. */
707 	if (qm_check_dev_error(qm)) {
708 		dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n");
709 		return -EIO;
710 	}
711 
712 	mutex_lock(&qm->mailbox_lock);
713 	if (!op)
714 		memcpy(tmp_xqc, xqc, size);
715 
716 	qm_mb_pre_init(&mailbox, cmd, xqc_dma, qp_id, op);
717 	ret = qm_mb_nolock(qm, &mailbox);
718 	if (!ret && op)
719 		memcpy(xqc, tmp_xqc, size);
720 
721 	mutex_unlock(&qm->mailbox_lock);
722 
723 	return ret;
724 }
725 
qm_db_v1(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)726 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
727 {
728 	u64 doorbell;
729 
730 	doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
731 		   ((u64)index << QM_DB_INDEX_SHIFT_V1)  |
732 		   ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
733 
734 	writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
735 }
736 
qm_db_v2(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)737 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
738 {
739 	void __iomem *io_base = qm->io_base;
740 	u16 randata = 0;
741 	u64 doorbell;
742 
743 	if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
744 		io_base = qm->db_io_base + (u64)qn * qm->db_interval +
745 			  QM_DOORBELL_SQ_CQ_BASE_V2;
746 	else
747 		io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;
748 
749 	doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
750 		   ((u64)randata << QM_DB_RAND_SHIFT_V2) |
751 		   ((u64)index << QM_DB_INDEX_SHIFT_V2) |
752 		   ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
753 
754 	writeq(doorbell, io_base);
755 }
756 
qm_db(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)757 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
758 {
759 	dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
760 		qn, cmd, index);
761 
762 	qm->ops->qm_db(qm, qn, cmd, index, priority);
763 }
764 
qm_disable_clock_gate(struct hisi_qm * qm)765 static void qm_disable_clock_gate(struct hisi_qm *qm)
766 {
767 	u32 val;
768 
769 	/* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */
770 	if (qm->ver < QM_HW_V3)
771 		return;
772 
773 	val = readl(qm->io_base + QM_PM_CTRL);
774 	val |= QM_IDLE_DISABLE;
775 	writel(val, qm->io_base +  QM_PM_CTRL);
776 }
777 
qm_dev_mem_reset(struct hisi_qm * qm)778 static int qm_dev_mem_reset(struct hisi_qm *qm)
779 {
780 	u32 val;
781 
782 	writel(0x1, qm->io_base + QM_MEM_START_INIT);
783 	return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
784 					  val & BIT(0), POLL_PERIOD,
785 					  POLL_TIMEOUT);
786 }
787 
788 /**
789  * hisi_qm_get_hw_info() - Get device information.
790  * @qm: The qm which want to get information.
791  * @info_table: Array for storing device information.
792  * @index: Index in info_table.
793  * @is_read: Whether read from reg, 0: not support read from reg.
794  *
795  * This function returns device information the caller needs.
796  */
hisi_qm_get_hw_info(struct hisi_qm * qm,const struct hisi_qm_cap_info * info_table,u32 index,bool is_read)797 u32 hisi_qm_get_hw_info(struct hisi_qm *qm,
798 			const struct hisi_qm_cap_info *info_table,
799 			u32 index, bool is_read)
800 {
801 	u32 val;
802 
803 	switch (qm->ver) {
804 	case QM_HW_V1:
805 		return info_table[index].v1_val;
806 	case QM_HW_V2:
807 		return info_table[index].v2_val;
808 	default:
809 		if (!is_read)
810 			return info_table[index].v3_val;
811 
812 		val = readl(qm->io_base + info_table[index].offset);
813 		return (val >> info_table[index].shift) & info_table[index].mask;
814 	}
815 }
816 EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info);
817 
hisi_qm_get_cap_value(struct hisi_qm * qm,const struct hisi_qm_cap_query_info * info_table,u32 index,bool is_read)818 u32 hisi_qm_get_cap_value(struct hisi_qm *qm,
819 			const struct hisi_qm_cap_query_info *info_table,
820 			u32 index, bool is_read)
821 {
822 	u32 val;
823 
824 	switch (qm->ver) {
825 	case QM_HW_V1:
826 		return info_table[index].v1_val;
827 	case QM_HW_V2:
828 		return info_table[index].v2_val;
829 	default:
830 		if (!is_read)
831 			return info_table[index].v3_val;
832 
833 		val = readl(qm->io_base + info_table[index].offset);
834 		return val;
835 	}
836 }
837 EXPORT_SYMBOL_GPL(hisi_qm_get_cap_value);
838 
qm_get_xqc_depth(struct hisi_qm * qm,u16 * low_bits,u16 * high_bits,enum qm_basic_type type)839 static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits,
840 			     u16 *high_bits, enum qm_basic_type type)
841 {
842 	u32 depth;
843 
844 	depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver);
845 	*low_bits = depth & QM_XQ_DEPTH_MASK;
846 	*high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK;
847 }
848 
hisi_qm_set_algs(struct hisi_qm * qm,u64 alg_msk,const struct qm_dev_alg * dev_algs,u32 dev_algs_size)849 int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs,
850 		     u32 dev_algs_size)
851 {
852 	struct device *dev = &qm->pdev->dev;
853 	char *algs, *ptr;
854 	int i;
855 
856 	if (!qm->uacce)
857 		return 0;
858 
859 	if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) {
860 		dev_err(dev, "algs size %u is equal or larger than %d.\n",
861 			dev_algs_size, QM_DEV_ALG_MAX_LEN);
862 		return -EINVAL;
863 	}
864 
865 	algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL);
866 	if (!algs)
867 		return -ENOMEM;
868 
869 	for (i = 0; i < dev_algs_size; i++)
870 		if (alg_msk & dev_algs[i].alg_msk)
871 			strcat(algs, dev_algs[i].alg);
872 
873 	ptr = strrchr(algs, '\n');
874 	if (ptr)
875 		*ptr = '\0';
876 
877 	qm->uacce->algs = algs;
878 
879 	return 0;
880 }
881 EXPORT_SYMBOL_GPL(hisi_qm_set_algs);
882 
qm_get_irq_num(struct hisi_qm * qm)883 static u32 qm_get_irq_num(struct hisi_qm *qm)
884 {
885 	if (qm->fun_type == QM_HW_PF)
886 		return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver);
887 
888 	return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver);
889 }
890 
qm_pm_get_sync(struct hisi_qm * qm)891 static int qm_pm_get_sync(struct hisi_qm *qm)
892 {
893 	struct device *dev = &qm->pdev->dev;
894 	int ret;
895 
896 	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
897 		return 0;
898 
899 	ret = pm_runtime_resume_and_get(dev);
900 	if (ret < 0) {
901 		dev_err(dev, "failed to get_sync(%d).\n", ret);
902 		return ret;
903 	}
904 
905 	return 0;
906 }
907 
qm_pm_put_sync(struct hisi_qm * qm)908 static void qm_pm_put_sync(struct hisi_qm *qm)
909 {
910 	struct device *dev = &qm->pdev->dev;
911 
912 	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
913 		return;
914 
915 	pm_runtime_put_autosuspend(dev);
916 }
917 
qm_cq_head_update(struct hisi_qp * qp)918 static void qm_cq_head_update(struct hisi_qp *qp)
919 {
920 	if (qp->qp_status.cq_head == qp->cq_depth - 1) {
921 		qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
922 		qp->qp_status.cq_head = 0;
923 	} else {
924 		qp->qp_status.cq_head++;
925 	}
926 }
927 
qm_poll_req_cb(struct hisi_qp * qp)928 static void qm_poll_req_cb(struct hisi_qp *qp)
929 {
930 	struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
931 	struct hisi_qm *qm = qp->qm;
932 
933 	while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
934 		dma_rmb();
935 		qp->req_cb(qp, qp->sqe + qm->sqe_size *
936 			   le16_to_cpu(cqe->sq_head));
937 		qm_cq_head_update(qp);
938 		cqe = qp->cqe + qp->qp_status.cq_head;
939 		qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
940 		      qp->qp_status.cq_head, 0);
941 		atomic_dec(&qp->qp_status.used);
942 
943 		cond_resched();
944 	}
945 
946 	/* set c_flag */
947 	qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1);
948 }
949 
qm_work_process(struct work_struct * work)950 static void qm_work_process(struct work_struct *work)
951 {
952 	struct hisi_qm_poll_data *poll_data =
953 		container_of(work, struct hisi_qm_poll_data, work);
954 	struct hisi_qm *qm = poll_data->qm;
955 	u16 eqe_num = poll_data->eqe_num;
956 	struct hisi_qp *qp;
957 	int i;
958 
959 	for (i = eqe_num - 1; i >= 0; i--) {
960 		qp = &qm->qp_array[poll_data->qp_finish_id[i]];
961 		if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP))
962 			continue;
963 
964 		if (qp->event_cb) {
965 			qp->event_cb(qp);
966 			continue;
967 		}
968 
969 		if (likely(qp->req_cb))
970 			qm_poll_req_cb(qp);
971 	}
972 }
973 
qm_get_complete_eqe_num(struct hisi_qm * qm)974 static void qm_get_complete_eqe_num(struct hisi_qm *qm)
975 {
976 	struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
977 	struct hisi_qm_poll_data *poll_data = NULL;
978 	u16 eq_depth = qm->eq_depth;
979 	u16 cqn, eqe_num = 0;
980 
981 	if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) {
982 		atomic64_inc(&qm->debug.dfx.err_irq_cnt);
983 		qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
984 		return;
985 	}
986 
987 	cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
988 	if (unlikely(cqn >= qm->qp_num))
989 		return;
990 	poll_data = &qm->poll_data[cqn];
991 
992 	while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
993 		cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
994 		poll_data->qp_finish_id[eqe_num] = cqn;
995 		eqe_num++;
996 
997 		if (qm->status.eq_head == eq_depth - 1) {
998 			qm->status.eqc_phase = !qm->status.eqc_phase;
999 			eqe = qm->eqe;
1000 			qm->status.eq_head = 0;
1001 		} else {
1002 			eqe++;
1003 			qm->status.eq_head++;
1004 		}
1005 
1006 		if (eqe_num == (eq_depth >> 1) - 1)
1007 			break;
1008 	}
1009 
1010 	poll_data->eqe_num = eqe_num;
1011 	queue_work(qm->wq, &poll_data->work);
1012 	qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
1013 }
1014 
qm_eq_irq(int irq,void * data)1015 static irqreturn_t qm_eq_irq(int irq, void *data)
1016 {
1017 	struct hisi_qm *qm = data;
1018 
1019 	/* Get qp id of completed tasks and re-enable the interrupt */
1020 	qm_get_complete_eqe_num(qm);
1021 
1022 	return IRQ_HANDLED;
1023 }
1024 
qm_mb_cmd_irq(int irq,void * data)1025 static irqreturn_t qm_mb_cmd_irq(int irq, void *data)
1026 {
1027 	struct hisi_qm *qm = data;
1028 	u32 val;
1029 
1030 	val = readl(qm->io_base + QM_IFC_INT_STATUS);
1031 	val &= QM_IFC_INT_STATUS_MASK;
1032 	if (!val)
1033 		return IRQ_NONE;
1034 
1035 	if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) {
1036 		dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n");
1037 		return IRQ_HANDLED;
1038 	}
1039 
1040 	schedule_work(&qm->cmd_process);
1041 
1042 	return IRQ_HANDLED;
1043 }
1044 
qm_set_qp_disable(struct hisi_qp * qp,int offset)1045 static void qm_set_qp_disable(struct hisi_qp *qp, int offset)
1046 {
1047 	u32 *addr;
1048 
1049 	if (qp->is_in_kernel)
1050 		return;
1051 
1052 	addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset;
1053 	*addr = 1;
1054 
1055 	/* make sure setup is completed */
1056 	smp_wmb();
1057 }
1058 
qm_disable_qp(struct hisi_qm * qm,u32 qp_id)1059 static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id)
1060 {
1061 	struct hisi_qp *qp = &qm->qp_array[qp_id];
1062 
1063 	qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET);
1064 	hisi_qm_stop_qp(qp);
1065 	qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET);
1066 }
1067 
qm_reset_function(struct hisi_qm * qm)1068 static void qm_reset_function(struct hisi_qm *qm)
1069 {
1070 	struct device *dev = &qm->pdev->dev;
1071 	int ret;
1072 
1073 	if (qm_check_dev_error(qm))
1074 		return;
1075 
1076 	ret = qm_reset_prepare_ready(qm);
1077 	if (ret) {
1078 		dev_err(dev, "reset function not ready\n");
1079 		return;
1080 	}
1081 
1082 	ret = hisi_qm_stop(qm, QM_DOWN);
1083 	if (ret) {
1084 		dev_err(dev, "failed to stop qm when reset function\n");
1085 		goto clear_bit;
1086 	}
1087 
1088 	ret = hisi_qm_start(qm);
1089 	if (ret)
1090 		dev_err(dev, "failed to start qm when reset function\n");
1091 
1092 clear_bit:
1093 	qm_reset_bit_clear(qm);
1094 }
1095 
qm_aeq_thread(int irq,void * data)1096 static irqreturn_t qm_aeq_thread(int irq, void *data)
1097 {
1098 	struct hisi_qm *qm = data;
1099 	struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
1100 	u16 aeq_depth = qm->aeq_depth;
1101 	u32 type, qp_id;
1102 
1103 	atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
1104 
1105 	while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
1106 		type = (le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT) &
1107 			QM_AEQE_TYPE_MASK;
1108 		qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK;
1109 
1110 		switch (type) {
1111 		case QM_EQ_OVERFLOW:
1112 			dev_err(&qm->pdev->dev, "eq overflow, reset function\n");
1113 			qm_reset_function(qm);
1114 			return IRQ_HANDLED;
1115 		case QM_CQ_OVERFLOW:
1116 			dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n",
1117 				qp_id);
1118 			fallthrough;
1119 		case QM_CQE_ERROR:
1120 			qm_disable_qp(qm, qp_id);
1121 			break;
1122 		default:
1123 			dev_err(&qm->pdev->dev, "unknown error type %u\n",
1124 				type);
1125 			break;
1126 		}
1127 
1128 		if (qm->status.aeq_head == aeq_depth - 1) {
1129 			qm->status.aeqc_phase = !qm->status.aeqc_phase;
1130 			aeqe = qm->aeqe;
1131 			qm->status.aeq_head = 0;
1132 		} else {
1133 			aeqe++;
1134 			qm->status.aeq_head++;
1135 		}
1136 	}
1137 
1138 	qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
1139 
1140 	return IRQ_HANDLED;
1141 }
1142 
qm_init_qp_status(struct hisi_qp * qp)1143 static void qm_init_qp_status(struct hisi_qp *qp)
1144 {
1145 	struct hisi_qp_status *qp_status = &qp->qp_status;
1146 
1147 	qp_status->sq_tail = 0;
1148 	qp_status->cq_head = 0;
1149 	qp_status->cqc_phase = true;
1150 	atomic_set(&qp_status->used, 0);
1151 }
1152 
qm_init_prefetch(struct hisi_qm * qm)1153 static void qm_init_prefetch(struct hisi_qm *qm)
1154 {
1155 	struct device *dev = &qm->pdev->dev;
1156 	u32 page_type = 0x0;
1157 
1158 	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
1159 		return;
1160 
1161 	switch (PAGE_SIZE) {
1162 	case SZ_4K:
1163 		page_type = 0x0;
1164 		break;
1165 	case SZ_16K:
1166 		page_type = 0x1;
1167 		break;
1168 	case SZ_64K:
1169 		page_type = 0x2;
1170 		break;
1171 	default:
1172 		dev_err(dev, "system page size is not support: %lu, default set to 4KB",
1173 			PAGE_SIZE);
1174 	}
1175 
1176 	writel(page_type, qm->io_base + QM_PAGE_SIZE);
1177 }
1178 
1179 /*
1180  * acc_shaper_para_calc() Get the IR value by the qos formula, the return value
1181  * is the expected qos calculated.
1182  * the formula:
1183  * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps
1184  *
1185  *		IR_b * (2 ^ IR_u) * 8000
1186  * IR(Mbps) = -------------------------
1187  *		  Tick * (2 ^ IR_s)
1188  */
acc_shaper_para_calc(u64 cir_b,u64 cir_u,u64 cir_s)1189 static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
1190 {
1191 	return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) /
1192 					(QM_QOS_TICK * (1 << cir_s));
1193 }
1194 
acc_shaper_calc_cbs_s(u32 ir)1195 static u32 acc_shaper_calc_cbs_s(u32 ir)
1196 {
1197 	int table_size = ARRAY_SIZE(shaper_cbs_s);
1198 	int i;
1199 
1200 	for (i = 0; i < table_size; i++) {
1201 		if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end)
1202 			return shaper_cbs_s[i].val;
1203 	}
1204 
1205 	return QM_SHAPER_MIN_CBS_S;
1206 }
1207 
acc_shaper_calc_cir_s(u32 ir)1208 static u32 acc_shaper_calc_cir_s(u32 ir)
1209 {
1210 	int table_size = ARRAY_SIZE(shaper_cir_s);
1211 	int i;
1212 
1213 	for (i = 0; i < table_size; i++) {
1214 		if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end)
1215 			return shaper_cir_s[i].val;
1216 	}
1217 
1218 	return 0;
1219 }
1220 
qm_get_shaper_para(u32 ir,struct qm_shaper_factor * factor)1221 static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
1222 {
1223 	u32 cir_b, cir_u, cir_s, ir_calc;
1224 	u32 error_rate;
1225 
1226 	factor->cbs_s = acc_shaper_calc_cbs_s(ir);
1227 	cir_s = acc_shaper_calc_cir_s(ir);
1228 
1229 	for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {
1230 		for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {
1231 			ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
1232 
1233 			error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
1234 			if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
1235 				factor->cir_b = cir_b;
1236 				factor->cir_u = cir_u;
1237 				factor->cir_s = cir_s;
1238 				return 0;
1239 			}
1240 		}
1241 	}
1242 
1243 	return -EINVAL;
1244 }
1245 
qm_vft_data_cfg(struct hisi_qm * qm,enum vft_type type,u32 base,u32 number,struct qm_shaper_factor * factor)1246 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
1247 			    u32 number, struct qm_shaper_factor *factor)
1248 {
1249 	u64 tmp = 0;
1250 
1251 	if (number > 0) {
1252 		switch (type) {
1253 		case SQC_VFT:
1254 			if (qm->ver == QM_HW_V1) {
1255 				tmp = QM_SQC_VFT_BUF_SIZE	|
1256 				      QM_SQC_VFT_SQC_SIZE	|
1257 				      QM_SQC_VFT_INDEX_NUMBER	|
1258 				      QM_SQC_VFT_VALID		|
1259 				      (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
1260 			} else {
1261 				tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
1262 				      QM_SQC_VFT_VALID |
1263 				      (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
1264 			}
1265 			break;
1266 		case CQC_VFT:
1267 			if (qm->ver == QM_HW_V1) {
1268 				tmp = QM_CQC_VFT_BUF_SIZE	|
1269 				      QM_CQC_VFT_SQC_SIZE	|
1270 				      QM_CQC_VFT_INDEX_NUMBER	|
1271 				      QM_CQC_VFT_VALID;
1272 			} else {
1273 				tmp = QM_CQC_VFT_VALID;
1274 			}
1275 			break;
1276 		case SHAPER_VFT:
1277 			if (factor) {
1278 				tmp = factor->cir_b |
1279 				(factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) |
1280 				(factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) |
1281 				(QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) |
1282 				(factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT);
1283 			}
1284 			break;
1285 		}
1286 	}
1287 
1288 	writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
1289 	writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
1290 }
1291 
qm_set_vft_common(struct hisi_qm * qm,enum vft_type type,u32 fun_num,u32 base,u32 number)1292 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
1293 			     u32 fun_num, u32 base, u32 number)
1294 {
1295 	struct qm_shaper_factor *factor = NULL;
1296 	unsigned int val;
1297 	int ret;
1298 
1299 	if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
1300 		factor = &qm->factor[fun_num];
1301 
1302 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1303 					 val & BIT(0), POLL_PERIOD,
1304 					 POLL_TIMEOUT);
1305 	if (ret)
1306 		return ret;
1307 
1308 	writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
1309 	writel(type, qm->io_base + QM_VFT_CFG_TYPE);
1310 	if (type == SHAPER_VFT)
1311 		fun_num |= base << QM_SHAPER_VFT_OFFSET;
1312 
1313 	writel(fun_num, qm->io_base + QM_VFT_CFG);
1314 
1315 	qm_vft_data_cfg(qm, type, base, number, factor);
1316 
1317 	writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
1318 	writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
1319 
1320 	return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1321 					  val & BIT(0), POLL_PERIOD,
1322 					  POLL_TIMEOUT);
1323 }
1324 
qm_shaper_init_vft(struct hisi_qm * qm,u32 fun_num)1325 static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)
1326 {
1327 	u32 qos = qm->factor[fun_num].func_qos;
1328 	int ret, i;
1329 
1330 	ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]);
1331 	if (ret) {
1332 		dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");
1333 		return ret;
1334 	}
1335 	writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG);
1336 	for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
1337 		/* The base number of queue reuse for different alg type */
1338 		ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1);
1339 		if (ret)
1340 			return ret;
1341 	}
1342 
1343 	return 0;
1344 }
1345 
1346 /* The config should be conducted after qm_dev_mem_reset() */
qm_set_sqc_cqc_vft(struct hisi_qm * qm,u32 fun_num,u32 base,u32 number)1347 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
1348 			      u32 number)
1349 {
1350 	int ret, i;
1351 
1352 	for (i = SQC_VFT; i <= CQC_VFT; i++) {
1353 		ret = qm_set_vft_common(qm, i, fun_num, base, number);
1354 		if (ret)
1355 			return ret;
1356 	}
1357 
1358 	/* init default shaper qos val */
1359 	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {
1360 		ret = qm_shaper_init_vft(qm, fun_num);
1361 		if (ret)
1362 			goto back_sqc_cqc;
1363 	}
1364 
1365 	return 0;
1366 back_sqc_cqc:
1367 	for (i = SQC_VFT; i <= CQC_VFT; i++)
1368 		qm_set_vft_common(qm, i, fun_num, 0, 0);
1369 
1370 	return ret;
1371 }
1372 
qm_get_vft_v2(struct hisi_qm * qm,u32 * base,u32 * number)1373 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
1374 {
1375 	u64 sqc_vft;
1376 	int ret;
1377 
1378 	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
1379 	if (ret)
1380 		return ret;
1381 
1382 	sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
1383 		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
1384 	*base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
1385 	*number = (QM_SQC_VFT_NUM_MASK_V2 &
1386 		   (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
1387 
1388 	return 0;
1389 }
1390 
qm_hw_error_init_v1(struct hisi_qm * qm)1391 static void qm_hw_error_init_v1(struct hisi_qm *qm)
1392 {
1393 	writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1394 }
1395 
qm_hw_error_cfg(struct hisi_qm * qm)1396 static void qm_hw_error_cfg(struct hisi_qm *qm)
1397 {
1398 	struct hisi_qm_err_info *err_info = &qm->err_info;
1399 
1400 	qm->error_mask = err_info->nfe | err_info->ce | err_info->fe;
1401 	/* clear QM hw residual error source */
1402 	writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE);
1403 
1404 	/* configure error type */
1405 	writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE);
1406 	writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1407 	writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1408 	writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE);
1409 }
1410 
qm_hw_error_init_v2(struct hisi_qm * qm)1411 static void qm_hw_error_init_v2(struct hisi_qm *qm)
1412 {
1413 	u32 irq_unmask;
1414 
1415 	qm_hw_error_cfg(qm);
1416 
1417 	irq_unmask = ~qm->error_mask;
1418 	irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1419 	writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1420 }
1421 
qm_hw_error_uninit_v2(struct hisi_qm * qm)1422 static void qm_hw_error_uninit_v2(struct hisi_qm *qm)
1423 {
1424 	u32 irq_mask = qm->error_mask;
1425 
1426 	irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1427 	writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);
1428 }
1429 
qm_hw_error_init_v3(struct hisi_qm * qm)1430 static void qm_hw_error_init_v3(struct hisi_qm *qm)
1431 {
1432 	u32 irq_unmask;
1433 
1434 	qm_hw_error_cfg(qm);
1435 
1436 	/* enable close master ooo when hardware error happened */
1437 	writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL);
1438 
1439 	irq_unmask = ~qm->error_mask;
1440 	irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1441 	writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1442 }
1443 
qm_hw_error_uninit_v3(struct hisi_qm * qm)1444 static void qm_hw_error_uninit_v3(struct hisi_qm *qm)
1445 {
1446 	u32 irq_mask = qm->error_mask;
1447 
1448 	irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1449 	writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);
1450 
1451 	/* disable close master ooo when hardware error happened */
1452 	writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL);
1453 }
1454 
qm_log_hw_error(struct hisi_qm * qm,u32 error_status)1455 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
1456 {
1457 	const struct hisi_qm_hw_error *err;
1458 	struct device *dev = &qm->pdev->dev;
1459 	u32 reg_val, type, vf_num, qp_id;
1460 	int i;
1461 
1462 	for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
1463 		err = &qm_hw_error[i];
1464 		if (!(err->int_msk & error_status))
1465 			continue;
1466 
1467 		dev_err(dev, "%s [error status=0x%x] found\n",
1468 			err->msg, err->int_msk);
1469 
1470 		if (err->int_msk & QM_DB_TIMEOUT) {
1471 			reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);
1472 			type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
1473 			       QM_DB_TIMEOUT_TYPE_SHIFT;
1474 			vf_num = reg_val & QM_DB_TIMEOUT_VF;
1475 			qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT;
1476 			dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n",
1477 				qm_db_timeout[type], vf_num, qp_id);
1478 		} else if (err->int_msk & QM_OF_FIFO_OF) {
1479 			reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
1480 			type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
1481 			       QM_FIFO_OVERFLOW_TYPE_SHIFT;
1482 			vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
1483 			qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT;
1484 			if (type < ARRAY_SIZE(qm_fifo_overflow))
1485 				dev_err(dev, "qm %s fifo overflow in function %u qp %u\n",
1486 					qm_fifo_overflow[type], vf_num, qp_id);
1487 			else
1488 				dev_err(dev, "unknown error type\n");
1489 		} else if (err->int_msk & QM_AXI_RRESP_ERR) {
1490 			reg_val = readl(qm->io_base + QM_ABNORMAL_INF02);
1491 			if (reg_val & QM_AXI_POISON_ERR)
1492 				dev_err(dev, "qm axi poison error happened\n");
1493 		}
1494 	}
1495 }
1496 
qm_hw_error_handle_v2(struct hisi_qm * qm)1497 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
1498 {
1499 	u32 error_status;
1500 
1501 	error_status = qm_get_hw_error_status(qm);
1502 	if (error_status & qm->error_mask) {
1503 		if (error_status & QM_ECC_MBIT)
1504 			qm->err_status.is_qm_ecc_mbit = true;
1505 
1506 		qm_log_hw_error(qm, error_status);
1507 		if (error_status & qm->err_info.qm_reset_mask) {
1508 			/* Disable the same error reporting until device is recovered. */
1509 			writel(qm->err_info.nfe & (~error_status),
1510 			       qm->io_base + QM_RAS_NFE_ENABLE);
1511 			return ACC_ERR_NEED_RESET;
1512 		}
1513 
1514 		/* Clear error source if not need reset. */
1515 		writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);
1516 		writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1517 		writel(qm->err_info.ce, qm->io_base + QM_RAS_CE_ENABLE);
1518 	}
1519 
1520 	return ACC_ERR_RECOVERED;
1521 }
1522 
qm_get_mb_cmd(struct hisi_qm * qm,u64 * msg,u16 fun_num)1523 static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)
1524 {
1525 	struct qm_mailbox mailbox;
1526 	int ret;
1527 
1528 	qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0);
1529 	mutex_lock(&qm->mailbox_lock);
1530 	ret = qm_mb_nolock(qm, &mailbox);
1531 	if (ret)
1532 		goto err_unlock;
1533 
1534 	*msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
1535 		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
1536 
1537 err_unlock:
1538 	mutex_unlock(&qm->mailbox_lock);
1539 	return ret;
1540 }
1541 
qm_clear_cmd_interrupt(struct hisi_qm * qm,u64 vf_mask)1542 static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask)
1543 {
1544 	u32 val;
1545 
1546 	if (qm->fun_type == QM_HW_PF)
1547 		writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P);
1548 
1549 	val = readl(qm->io_base + QM_IFC_INT_SOURCE_V);
1550 	val |= QM_IFC_INT_SOURCE_MASK;
1551 	writel(val, qm->io_base + QM_IFC_INT_SOURCE_V);
1552 }
1553 
qm_handle_vf_msg(struct hisi_qm * qm,u32 vf_id)1554 static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id)
1555 {
1556 	struct device *dev = &qm->pdev->dev;
1557 	enum qm_ifc_cmd cmd;
1558 	int ret;
1559 
1560 	ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id);
1561 	if (ret) {
1562 		dev_err(dev, "failed to get command from VF(%u)!\n", vf_id);
1563 		return;
1564 	}
1565 
1566 	switch (cmd) {
1567 	case QM_VF_PREPARE_FAIL:
1568 		dev_err(dev, "failed to stop VF(%u)!\n", vf_id);
1569 		break;
1570 	case QM_VF_START_FAIL:
1571 		dev_err(dev, "failed to start VF(%u)!\n", vf_id);
1572 		break;
1573 	case QM_VF_PREPARE_DONE:
1574 	case QM_VF_START_DONE:
1575 		break;
1576 	default:
1577 		dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id);
1578 		break;
1579 	}
1580 }
1581 
qm_wait_vf_prepare_finish(struct hisi_qm * qm)1582 static int qm_wait_vf_prepare_finish(struct hisi_qm *qm)
1583 {
1584 	struct device *dev = &qm->pdev->dev;
1585 	u32 vfs_num = qm->vfs_num;
1586 	int cnt = 0;
1587 	int ret = 0;
1588 	u64 val;
1589 	u32 i;
1590 
1591 	if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
1592 		return 0;
1593 
1594 	while (true) {
1595 		val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
1596 		/* All VFs send command to PF, break */
1597 		if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1))
1598 			break;
1599 
1600 		if (++cnt > QM_MAX_PF_WAIT_COUNT) {
1601 			ret = -EBUSY;
1602 			break;
1603 		}
1604 
1605 		msleep(QM_WAIT_DST_ACK);
1606 	}
1607 
1608 	/* PF check VFs msg */
1609 	for (i = 1; i <= vfs_num; i++) {
1610 		if (val & BIT(i))
1611 			qm_handle_vf_msg(qm, i);
1612 		else
1613 			dev_err(dev, "VF(%u) not ping PF!\n", i);
1614 	}
1615 
1616 	/* PF clear interrupt to ack VFs */
1617 	qm_clear_cmd_interrupt(qm, val);
1618 
1619 	return ret;
1620 }
1621 
qm_trigger_vf_interrupt(struct hisi_qm * qm,u32 fun_num)1622 static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num)
1623 {
1624 	u32 val;
1625 
1626 	val = readl(qm->io_base + QM_IFC_INT_CFG);
1627 	val &= ~QM_IFC_SEND_ALL_VFS;
1628 	val |= fun_num;
1629 	writel(val, qm->io_base + QM_IFC_INT_CFG);
1630 
1631 	val = readl(qm->io_base + QM_IFC_INT_SET_P);
1632 	val |= QM_IFC_INT_SET_MASK;
1633 	writel(val, qm->io_base + QM_IFC_INT_SET_P);
1634 }
1635 
qm_trigger_pf_interrupt(struct hisi_qm * qm)1636 static void qm_trigger_pf_interrupt(struct hisi_qm *qm)
1637 {
1638 	u32 val;
1639 
1640 	val = readl(qm->io_base + QM_IFC_INT_SET_V);
1641 	val |= QM_IFC_INT_SET_MASK;
1642 	writel(val, qm->io_base + QM_IFC_INT_SET_V);
1643 }
1644 
qm_ping_single_vf(struct hisi_qm * qm,enum qm_ifc_cmd cmd,u32 data,u32 fun_num)1645 static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num)
1646 {
1647 	struct device *dev = &qm->pdev->dev;
1648 	int cnt = 0;
1649 	u64 val;
1650 	int ret;
1651 
1652 	ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num);
1653 	if (ret) {
1654 		dev_err(dev, "failed to send command to vf(%u)!\n", fun_num);
1655 		goto err_unlock;
1656 	}
1657 
1658 	qm_trigger_vf_interrupt(qm, fun_num);
1659 	while (true) {
1660 		msleep(QM_WAIT_DST_ACK);
1661 		val = readq(qm->io_base + QM_IFC_READY_STATUS);
1662 		/* if VF respond, PF notifies VF successfully. */
1663 		if (!(val & BIT(fun_num)))
1664 			goto err_unlock;
1665 
1666 		if (++cnt > QM_MAX_PF_WAIT_COUNT) {
1667 			dev_err(dev, "failed to get response from VF(%u)!\n", fun_num);
1668 			ret = -ETIMEDOUT;
1669 			break;
1670 		}
1671 	}
1672 
1673 err_unlock:
1674 	qm->ops->set_ifc_end(qm);
1675 	return ret;
1676 }
1677 
qm_ping_all_vfs(struct hisi_qm * qm,enum qm_ifc_cmd cmd)1678 static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd)
1679 {
1680 	struct device *dev = &qm->pdev->dev;
1681 	u32 vfs_num = qm->vfs_num;
1682 	u64 val = 0;
1683 	int cnt = 0;
1684 	int ret;
1685 	u32 i;
1686 
1687 	ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS);
1688 	if (ret) {
1689 		dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd);
1690 		qm->ops->set_ifc_end(qm);
1691 		return ret;
1692 	}
1693 
1694 	qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS);
1695 	while (true) {
1696 		msleep(QM_WAIT_DST_ACK);
1697 		val = readq(qm->io_base + QM_IFC_READY_STATUS);
1698 		/* If all VFs acked, PF notifies VFs successfully. */
1699 		if (!(val & GENMASK(vfs_num, 1))) {
1700 			qm->ops->set_ifc_end(qm);
1701 			return 0;
1702 		}
1703 
1704 		if (++cnt > QM_MAX_PF_WAIT_COUNT)
1705 			break;
1706 	}
1707 
1708 	qm->ops->set_ifc_end(qm);
1709 
1710 	/* Check which vf respond timeout. */
1711 	for (i = 1; i <= vfs_num; i++) {
1712 		if (val & BIT(i))
1713 			dev_err(dev, "failed to get response from VF(%u)!\n", i);
1714 	}
1715 
1716 	return -ETIMEDOUT;
1717 }
1718 
qm_ping_pf(struct hisi_qm * qm,enum qm_ifc_cmd cmd)1719 static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd)
1720 {
1721 	int cnt = 0;
1722 	u32 val;
1723 	int ret;
1724 
1725 	ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0);
1726 	if (ret) {
1727 		dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd);
1728 		goto unlock;
1729 	}
1730 
1731 	qm_trigger_pf_interrupt(qm);
1732 	/* Waiting for PF response */
1733 	while (true) {
1734 		msleep(QM_WAIT_DST_ACK);
1735 		val = readl(qm->io_base + QM_IFC_INT_SET_V);
1736 		if (!(val & QM_IFC_INT_STATUS_MASK))
1737 			break;
1738 
1739 		if (++cnt > QM_MAX_VF_WAIT_COUNT) {
1740 			ret = -ETIMEDOUT;
1741 			break;
1742 		}
1743 	}
1744 
1745 unlock:
1746 	qm->ops->set_ifc_end(qm);
1747 
1748 	return ret;
1749 }
1750 
qm_drain_qm(struct hisi_qm * qm)1751 static int qm_drain_qm(struct hisi_qm *qm)
1752 {
1753 	return hisi_qm_mb(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0);
1754 }
1755 
qm_stop_qp(struct hisi_qp * qp)1756 static int qm_stop_qp(struct hisi_qp *qp)
1757 {
1758 	return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);
1759 }
1760 
qm_set_msi(struct hisi_qm * qm,bool set)1761 static int qm_set_msi(struct hisi_qm *qm, bool set)
1762 {
1763 	struct pci_dev *pdev = qm->pdev;
1764 
1765 	if (set) {
1766 		pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
1767 				       0);
1768 	} else {
1769 		pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
1770 				       ACC_PEH_MSI_DISABLE);
1771 		if (qm->err_status.is_qm_ecc_mbit ||
1772 		    qm->err_status.is_dev_ecc_mbit)
1773 			return 0;
1774 
1775 		mdelay(1);
1776 		if (readl(qm->io_base + QM_PEH_DFX_INFO0))
1777 			return -EFAULT;
1778 	}
1779 
1780 	return 0;
1781 }
1782 
qm_wait_msi_finish(struct hisi_qm * qm)1783 static void qm_wait_msi_finish(struct hisi_qm *qm)
1784 {
1785 	struct pci_dev *pdev = qm->pdev;
1786 	u32 cmd = ~0;
1787 	int cnt = 0;
1788 	u32 val;
1789 	int ret;
1790 
1791 	while (true) {
1792 		pci_read_config_dword(pdev, pdev->msi_cap +
1793 				      PCI_MSI_PENDING_64, &cmd);
1794 		if (!cmd)
1795 			break;
1796 
1797 		if (++cnt > MAX_WAIT_COUNTS) {
1798 			pci_warn(pdev, "failed to empty MSI PENDING!\n");
1799 			break;
1800 		}
1801 
1802 		udelay(1);
1803 	}
1804 
1805 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0,
1806 					 val, !(val & QM_PEH_DFX_MASK),
1807 					 POLL_PERIOD, POLL_TIMEOUT);
1808 	if (ret)
1809 		pci_warn(pdev, "failed to empty PEH MSI!\n");
1810 
1811 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1,
1812 					 val, !(val & QM_PEH_MSI_FINISH_MASK),
1813 					 POLL_PERIOD, POLL_TIMEOUT);
1814 	if (ret)
1815 		pci_warn(pdev, "failed to finish MSI operation!\n");
1816 }
1817 
qm_set_msi_v3(struct hisi_qm * qm,bool set)1818 static int qm_set_msi_v3(struct hisi_qm *qm, bool set)
1819 {
1820 	struct pci_dev *pdev = qm->pdev;
1821 	int ret = -ETIMEDOUT;
1822 	u32 cmd, i;
1823 
1824 	pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
1825 	if (set)
1826 		cmd |= QM_MSI_CAP_ENABLE;
1827 	else
1828 		cmd &= ~QM_MSI_CAP_ENABLE;
1829 
1830 	pci_write_config_dword(pdev, pdev->msi_cap, cmd);
1831 	if (set) {
1832 		for (i = 0; i < MAX_WAIT_COUNTS; i++) {
1833 			pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
1834 			if (cmd & QM_MSI_CAP_ENABLE)
1835 				return 0;
1836 
1837 			udelay(1);
1838 		}
1839 	} else {
1840 		udelay(WAIT_PERIOD_US_MIN);
1841 		qm_wait_msi_finish(qm);
1842 		ret = 0;
1843 	}
1844 
1845 	return ret;
1846 }
1847 
qm_set_ifc_begin_v3(struct hisi_qm * qm,enum qm_ifc_cmd cmd,u32 data,u32 fun_num)1848 static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num)
1849 {
1850 	struct qm_mailbox mailbox;
1851 	u64 msg;
1852 
1853 	msg = cmd | (u64)data << QM_IFC_DATA_SHIFT;
1854 
1855 	qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0);
1856 	mutex_lock(&qm->mailbox_lock);
1857 	return qm_mb_nolock(qm, &mailbox);
1858 }
1859 
qm_set_ifc_end_v3(struct hisi_qm * qm)1860 static void qm_set_ifc_end_v3(struct hisi_qm *qm)
1861 {
1862 	mutex_unlock(&qm->mailbox_lock);
1863 }
1864 
qm_get_ifc_v3(struct hisi_qm * qm,enum qm_ifc_cmd * cmd,u32 * data,u32 fun_num)1865 static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num)
1866 {
1867 	u64 msg;
1868 	int ret;
1869 
1870 	ret = qm_get_mb_cmd(qm, &msg, fun_num);
1871 	if (ret)
1872 		return ret;
1873 
1874 	*cmd = msg & QM_IFC_CMD_MASK;
1875 
1876 	if (data)
1877 		*data = msg >> QM_IFC_DATA_SHIFT;
1878 
1879 	return 0;
1880 }
1881 
qm_set_ifc_begin_v4(struct hisi_qm * qm,enum qm_ifc_cmd cmd,u32 data,u32 fun_num)1882 static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num)
1883 {
1884 	uintptr_t offset;
1885 	u64 msg;
1886 
1887 	if (qm->fun_type == QM_HW_PF)
1888 		offset = QM_PF2VF_PF_W;
1889 	else
1890 		offset = QM_VF2PF_VF_W;
1891 
1892 	msg = cmd | (u64)data << QM_IFC_DATA_SHIFT;
1893 
1894 	mutex_lock(&qm->ifc_lock);
1895 	writeq(msg, qm->io_base + offset);
1896 
1897 	return 0;
1898 }
1899 
qm_set_ifc_end_v4(struct hisi_qm * qm)1900 static void qm_set_ifc_end_v4(struct hisi_qm *qm)
1901 {
1902 	mutex_unlock(&qm->ifc_lock);
1903 }
1904 
qm_get_ifc_pf(struct hisi_qm * qm,u32 fun_num)1905 static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num)
1906 {
1907 	uintptr_t offset;
1908 
1909 	offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num;
1910 
1911 	return (u64)readl(qm->io_base + offset);
1912 }
1913 
qm_get_ifc_vf(struct hisi_qm * qm)1914 static u64 qm_get_ifc_vf(struct hisi_qm *qm)
1915 {
1916 	return readq(qm->io_base + QM_PF2VF_VF_R);
1917 }
1918 
qm_get_ifc_v4(struct hisi_qm * qm,enum qm_ifc_cmd * cmd,u32 * data,u32 fun_num)1919 static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num)
1920 {
1921 	u64 msg;
1922 
1923 	if (qm->fun_type == QM_HW_PF)
1924 		msg = qm_get_ifc_pf(qm, fun_num);
1925 	else
1926 		msg = qm_get_ifc_vf(qm);
1927 
1928 	*cmd = msg & QM_IFC_CMD_MASK;
1929 
1930 	if (data)
1931 		*data = msg >> QM_IFC_DATA_SHIFT;
1932 
1933 	return 0;
1934 }
1935 
1936 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
1937 	.qm_db = qm_db_v1,
1938 	.hw_error_init = qm_hw_error_init_v1,
1939 	.set_msi = qm_set_msi,
1940 };
1941 
1942 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
1943 	.get_vft = qm_get_vft_v2,
1944 	.qm_db = qm_db_v2,
1945 	.hw_error_init = qm_hw_error_init_v2,
1946 	.hw_error_uninit = qm_hw_error_uninit_v2,
1947 	.hw_error_handle = qm_hw_error_handle_v2,
1948 	.set_msi = qm_set_msi,
1949 };
1950 
1951 static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {
1952 	.get_vft = qm_get_vft_v2,
1953 	.qm_db = qm_db_v2,
1954 	.hw_error_init = qm_hw_error_init_v3,
1955 	.hw_error_uninit = qm_hw_error_uninit_v3,
1956 	.hw_error_handle = qm_hw_error_handle_v2,
1957 	.set_msi = qm_set_msi_v3,
1958 	.set_ifc_begin = qm_set_ifc_begin_v3,
1959 	.set_ifc_end = qm_set_ifc_end_v3,
1960 	.get_ifc = qm_get_ifc_v3,
1961 };
1962 
1963 static const struct hisi_qm_hw_ops qm_hw_ops_v4 = {
1964 	.get_vft = qm_get_vft_v2,
1965 	.qm_db = qm_db_v2,
1966 	.hw_error_init = qm_hw_error_init_v3,
1967 	.hw_error_uninit = qm_hw_error_uninit_v3,
1968 	.hw_error_handle = qm_hw_error_handle_v2,
1969 	.set_msi = qm_set_msi_v3,
1970 	.set_ifc_begin = qm_set_ifc_begin_v4,
1971 	.set_ifc_end = qm_set_ifc_end_v4,
1972 	.get_ifc = qm_get_ifc_v4,
1973 };
1974 
qm_get_avail_sqe(struct hisi_qp * qp)1975 static void *qm_get_avail_sqe(struct hisi_qp *qp)
1976 {
1977 	struct hisi_qp_status *qp_status = &qp->qp_status;
1978 	u16 sq_tail = qp_status->sq_tail;
1979 
1980 	if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1))
1981 		return NULL;
1982 
1983 	return qp->sqe + sq_tail * qp->qm->sqe_size;
1984 }
1985 
hisi_qm_unset_hw_reset(struct hisi_qp * qp)1986 static void hisi_qm_unset_hw_reset(struct hisi_qp *qp)
1987 {
1988 	u64 *addr;
1989 
1990 	/* Use last 64 bits of DUS to reset status. */
1991 	addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET;
1992 	*addr = 0;
1993 }
1994 
qm_create_qp_nolock(struct hisi_qm * qm,u8 alg_type)1995 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
1996 {
1997 	struct device *dev = &qm->pdev->dev;
1998 	struct hisi_qp *qp;
1999 	int qp_id;
2000 
2001 	if (atomic_read(&qm->status.flags) == QM_STOP) {
2002 		dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n");
2003 		return ERR_PTR(-EPERM);
2004 	}
2005 
2006 	if (qm->qp_in_used == qm->qp_num) {
2007 		dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
2008 				     qm->qp_num);
2009 		atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
2010 		return ERR_PTR(-EBUSY);
2011 	}
2012 
2013 	qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);
2014 	if (qp_id < 0) {
2015 		dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
2016 				    qm->qp_num);
2017 		atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
2018 		return ERR_PTR(-EBUSY);
2019 	}
2020 
2021 	qp = &qm->qp_array[qp_id];
2022 	hisi_qm_unset_hw_reset(qp);
2023 	memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth);
2024 
2025 	qp->event_cb = NULL;
2026 	qp->req_cb = NULL;
2027 	qp->qp_id = qp_id;
2028 	qp->alg_type = alg_type;
2029 	qp->is_in_kernel = true;
2030 	qm->qp_in_used++;
2031 
2032 	return qp;
2033 }
2034 
2035 /**
2036  * hisi_qm_create_qp() - Create a queue pair from qm.
2037  * @qm: The qm we create a qp from.
2038  * @alg_type: Accelerator specific algorithm type in sqc.
2039  *
2040  * Return created qp, negative error code if failed.
2041  */
hisi_qm_create_qp(struct hisi_qm * qm,u8 alg_type)2042 static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
2043 {
2044 	struct hisi_qp *qp;
2045 	int ret;
2046 
2047 	ret = qm_pm_get_sync(qm);
2048 	if (ret)
2049 		return ERR_PTR(ret);
2050 
2051 	down_write(&qm->qps_lock);
2052 	qp = qm_create_qp_nolock(qm, alg_type);
2053 	up_write(&qm->qps_lock);
2054 
2055 	if (IS_ERR(qp))
2056 		qm_pm_put_sync(qm);
2057 
2058 	return qp;
2059 }
2060 
2061 /**
2062  * hisi_qm_release_qp() - Release a qp back to its qm.
2063  * @qp: The qp we want to release.
2064  *
2065  * This function releases the resource of a qp.
2066  */
hisi_qm_release_qp(struct hisi_qp * qp)2067 static void hisi_qm_release_qp(struct hisi_qp *qp)
2068 {
2069 	struct hisi_qm *qm = qp->qm;
2070 
2071 	down_write(&qm->qps_lock);
2072 
2073 	qm->qp_in_used--;
2074 	idr_remove(&qm->qp_idr, qp->qp_id);
2075 
2076 	up_write(&qm->qps_lock);
2077 
2078 	qm_pm_put_sync(qm);
2079 }
2080 
qm_sq_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2081 static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2082 {
2083 	struct hisi_qm *qm = qp->qm;
2084 	enum qm_hw_ver ver = qm->ver;
2085 	struct qm_sqc sqc = {0};
2086 
2087 	if (ver == QM_HW_V1) {
2088 		sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));
2089 		sqc.w8 = cpu_to_le16(qp->sq_depth - 1);
2090 	} else {
2091 		sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth));
2092 		sqc.w8 = 0; /* rand_qc */
2093 	}
2094 	sqc.w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));
2095 	sqc.base_l = cpu_to_le32(lower_32_bits(qp->sqe_dma));
2096 	sqc.base_h = cpu_to_le32(upper_32_bits(qp->sqe_dma));
2097 	sqc.cq_num = cpu_to_le16(qp_id);
2098 	sqc.pasid = cpu_to_le16(pasid);
2099 
2100 	if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2101 		sqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE <<
2102 				      QM_QC_PASID_ENABLE_SHIFT);
2103 
2104 	return qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 0);
2105 }
2106 
qm_cq_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2107 static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2108 {
2109 	struct hisi_qm *qm = qp->qm;
2110 	enum qm_hw_ver ver = qm->ver;
2111 	struct qm_cqc cqc = {0};
2112 
2113 	if (ver == QM_HW_V1) {
2114 		cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE));
2115 		cqc.w8 = cpu_to_le16(qp->cq_depth - 1);
2116 	} else {
2117 		cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth));
2118 		cqc.w8 = 0; /* rand_qc */
2119 	}
2120 	/*
2121 	 * Enable request finishing interrupts defaultly.
2122 	 * So, there will be some interrupts until disabling
2123 	 * this.
2124 	 */
2125 	cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
2126 	cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma));
2127 	cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma));
2128 	cqc.pasid = cpu_to_le16(pasid);
2129 
2130 	if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2131 		cqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE);
2132 
2133 	return qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 0);
2134 }
2135 
qm_qp_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2136 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2137 {
2138 	int ret;
2139 
2140 	qm_init_qp_status(qp);
2141 
2142 	ret = qm_sq_ctx_cfg(qp, qp_id, pasid);
2143 	if (ret)
2144 		return ret;
2145 
2146 	return qm_cq_ctx_cfg(qp, qp_id, pasid);
2147 }
2148 
qm_start_qp_nolock(struct hisi_qp * qp,unsigned long arg)2149 static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
2150 {
2151 	struct hisi_qm *qm = qp->qm;
2152 	struct device *dev = &qm->pdev->dev;
2153 	int qp_id = qp->qp_id;
2154 	u32 pasid = arg;
2155 	int ret;
2156 
2157 	if (atomic_read(&qm->status.flags) == QM_STOP) {
2158 		dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n");
2159 		return -EPERM;
2160 	}
2161 
2162 	ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
2163 	if (ret)
2164 		return ret;
2165 
2166 	atomic_set(&qp->qp_status.flags, QP_START);
2167 	dev_dbg(dev, "queue %d started\n", qp_id);
2168 
2169 	return 0;
2170 }
2171 
2172 /**
2173  * hisi_qm_start_qp() - Start a qp into running.
2174  * @qp: The qp we want to start to run.
2175  * @arg: Accelerator specific argument.
2176  *
2177  * After this function, qp can receive request from user. Return 0 if
2178  * successful, negative error code if failed.
2179  */
hisi_qm_start_qp(struct hisi_qp * qp,unsigned long arg)2180 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
2181 {
2182 	struct hisi_qm *qm = qp->qm;
2183 	int ret;
2184 
2185 	down_write(&qm->qps_lock);
2186 	ret = qm_start_qp_nolock(qp, arg);
2187 	up_write(&qm->qps_lock);
2188 
2189 	return ret;
2190 }
2191 EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
2192 
2193 /**
2194  * qp_stop_fail_cb() - call request cb.
2195  * @qp: stopped failed qp.
2196  *
2197  * Callback function should be called whether task completed or not.
2198  */
qp_stop_fail_cb(struct hisi_qp * qp)2199 static void qp_stop_fail_cb(struct hisi_qp *qp)
2200 {
2201 	int qp_used = atomic_read(&qp->qp_status.used);
2202 	u16 cur_tail = qp->qp_status.sq_tail;
2203 	u16 sq_depth = qp->sq_depth;
2204 	u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth;
2205 	struct hisi_qm *qm = qp->qm;
2206 	u16 pos;
2207 	int i;
2208 
2209 	for (i = 0; i < qp_used; i++) {
2210 		pos = (i + cur_head) % sq_depth;
2211 		qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));
2212 		atomic_dec(&qp->qp_status.used);
2213 	}
2214 }
2215 
qm_wait_qp_empty(struct hisi_qm * qm,u32 * state,u32 qp_id)2216 static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id)
2217 {
2218 	struct device *dev = &qm->pdev->dev;
2219 	struct qm_sqc sqc;
2220 	struct qm_cqc cqc;
2221 	int ret, i = 0;
2222 
2223 	while (++i) {
2224 		ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1);
2225 		if (ret) {
2226 			dev_err_ratelimited(dev, "Failed to dump sqc!\n");
2227 			*state = QM_DUMP_SQC_FAIL;
2228 			return ret;
2229 		}
2230 
2231 		ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1);
2232 		if (ret) {
2233 			dev_err_ratelimited(dev, "Failed to dump cqc!\n");
2234 			*state = QM_DUMP_CQC_FAIL;
2235 			return ret;
2236 		}
2237 
2238 		if ((sqc.tail == cqc.tail) &&
2239 		    (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))
2240 			break;
2241 
2242 		if (i == MAX_WAIT_COUNTS) {
2243 			dev_err(dev, "Fail to empty queue %u!\n", qp_id);
2244 			*state = QM_STOP_QUEUE_FAIL;
2245 			return -ETIMEDOUT;
2246 		}
2247 
2248 		usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);
2249 	}
2250 
2251 	return 0;
2252 }
2253 
2254 /**
2255  * qm_drain_qp() - Drain a qp.
2256  * @qp: The qp we want to drain.
2257  *
2258  * If the device does not support stopping queue by sending mailbox,
2259  * determine whether the queue is cleared by judging the tail pointers of
2260  * sq and cq.
2261  */
qm_drain_qp(struct hisi_qp * qp)2262 static int qm_drain_qp(struct hisi_qp *qp)
2263 {
2264 	struct hisi_qm *qm = qp->qm;
2265 	u32 state = 0;
2266 	int ret;
2267 
2268 	/* No need to judge if master OOO is blocked. */
2269 	if (qm_check_dev_error(qm))
2270 		return 0;
2271 
2272 	/* HW V3 supports drain qp by device */
2273 	if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) {
2274 		ret = qm_stop_qp(qp);
2275 		if (ret) {
2276 			dev_err(&qm->pdev->dev, "Failed to stop qp!\n");
2277 			state = QM_STOP_QUEUE_FAIL;
2278 			goto set_dev_state;
2279 		}
2280 		return ret;
2281 	}
2282 
2283 	ret = qm_wait_qp_empty(qm, &state, qp->qp_id);
2284 	if (ret)
2285 		goto set_dev_state;
2286 
2287 	return 0;
2288 
2289 set_dev_state:
2290 	if (qm->debug.dev_dfx.dev_timeout)
2291 		qm->debug.dev_dfx.dev_state = state;
2292 
2293 	return ret;
2294 }
2295 
qm_stop_qp_nolock(struct hisi_qp * qp)2296 static void qm_stop_qp_nolock(struct hisi_qp *qp)
2297 {
2298 	struct hisi_qm *qm = qp->qm;
2299 	struct device *dev = &qm->pdev->dev;
2300 	int ret;
2301 
2302 	/*
2303 	 * It is allowed to stop and release qp when reset, If the qp is
2304 	 * stopped when reset but still want to be released then, the
2305 	 * is_resetting flag should be set negative so that this qp will not
2306 	 * be restarted after reset.
2307 	 */
2308 	if (atomic_read(&qp->qp_status.flags) != QP_START) {
2309 		qp->is_resetting = false;
2310 		return;
2311 	}
2312 
2313 	atomic_set(&qp->qp_status.flags, QP_STOP);
2314 
2315 	/* V3 supports direct stop function when FLR prepare */
2316 	if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) {
2317 		ret = qm_drain_qp(qp);
2318 		if (ret)
2319 			dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", qp->qp_id);
2320 	}
2321 
2322 	flush_workqueue(qm->wq);
2323 	if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))
2324 		qp_stop_fail_cb(qp);
2325 
2326 	dev_dbg(dev, "stop queue %u!", qp->qp_id);
2327 }
2328 
2329 /**
2330  * hisi_qm_stop_qp() - Stop a qp in qm.
2331  * @qp: The qp we want to stop.
2332  *
2333  * This function is reverse of hisi_qm_start_qp.
2334  */
hisi_qm_stop_qp(struct hisi_qp * qp)2335 void hisi_qm_stop_qp(struct hisi_qp *qp)
2336 {
2337 	down_write(&qp->qm->qps_lock);
2338 	qm_stop_qp_nolock(qp);
2339 	up_write(&qp->qm->qps_lock);
2340 }
2341 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
2342 
2343 /**
2344  * hisi_qp_send() - Queue up a task in the hardware queue.
2345  * @qp: The qp in which to put the message.
2346  * @msg: The message.
2347  *
2348  * This function will return -EBUSY if qp is currently full, and -EAGAIN
2349  * if qp related qm is resetting.
2350  *
2351  * Note: This function may run with qm_irq_thread and ACC reset at same time.
2352  *       It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
2353  *       reset may happen, we have no lock here considering performance. This
2354  *       causes current qm_db sending fail or can not receive sended sqe. QM
2355  *       sync/async receive function should handle the error sqe. ACC reset
2356  *       done function should clear used sqe to 0.
2357  */
hisi_qp_send(struct hisi_qp * qp,const void * msg)2358 int hisi_qp_send(struct hisi_qp *qp, const void *msg)
2359 {
2360 	struct hisi_qp_status *qp_status = &qp->qp_status;
2361 	u16 sq_tail = qp_status->sq_tail;
2362 	u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth;
2363 	void *sqe = qm_get_avail_sqe(qp);
2364 
2365 	if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
2366 		     atomic_read(&qp->qm->status.flags) == QM_STOP ||
2367 		     qp->is_resetting)) {
2368 		dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
2369 		return -EAGAIN;
2370 	}
2371 
2372 	if (!sqe)
2373 		return -EBUSY;
2374 
2375 	memcpy(sqe, msg, qp->qm->sqe_size);
2376 
2377 	qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
2378 	atomic_inc(&qp->qp_status.used);
2379 	qp_status->sq_tail = sq_tail_next;
2380 
2381 	return 0;
2382 }
2383 EXPORT_SYMBOL_GPL(hisi_qp_send);
2384 
hisi_qm_cache_wb(struct hisi_qm * qm)2385 static void hisi_qm_cache_wb(struct hisi_qm *qm)
2386 {
2387 	unsigned int val;
2388 
2389 	if (qm->ver == QM_HW_V1)
2390 		return;
2391 
2392 	writel(0x1, qm->io_base + QM_CACHE_WB_START);
2393 	if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
2394 				       val, val & BIT(0), POLL_PERIOD,
2395 				       POLL_TIMEOUT))
2396 		dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
2397 }
2398 
qm_qp_event_notifier(struct hisi_qp * qp)2399 static void qm_qp_event_notifier(struct hisi_qp *qp)
2400 {
2401 	wake_up_interruptible(&qp->uacce_q->wait);
2402 }
2403 
2404  /* This function returns free number of qp in qm. */
hisi_qm_get_available_instances(struct uacce_device * uacce)2405 static int hisi_qm_get_available_instances(struct uacce_device *uacce)
2406 {
2407 	struct hisi_qm *qm = uacce->priv;
2408 	int ret;
2409 
2410 	down_read(&qm->qps_lock);
2411 	ret = qm->qp_num - qm->qp_in_used;
2412 	up_read(&qm->qps_lock);
2413 
2414 	return ret;
2415 }
2416 
hisi_qm_set_hw_reset(struct hisi_qm * qm,int offset)2417 static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset)
2418 {
2419 	int i;
2420 
2421 	for (i = 0; i < qm->qp_num; i++)
2422 		qm_set_qp_disable(&qm->qp_array[i], offset);
2423 }
2424 
hisi_qm_uacce_get_queue(struct uacce_device * uacce,unsigned long arg,struct uacce_queue * q)2425 static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,
2426 				   unsigned long arg,
2427 				   struct uacce_queue *q)
2428 {
2429 	struct hisi_qm *qm = uacce->priv;
2430 	struct hisi_qp *qp;
2431 	u8 alg_type = 0;
2432 
2433 	qp = hisi_qm_create_qp(qm, alg_type);
2434 	if (IS_ERR(qp))
2435 		return PTR_ERR(qp);
2436 
2437 	q->priv = qp;
2438 	q->uacce = uacce;
2439 	qp->uacce_q = q;
2440 	qp->event_cb = qm_qp_event_notifier;
2441 	qp->pasid = arg;
2442 	qp->is_in_kernel = false;
2443 
2444 	return 0;
2445 }
2446 
hisi_qm_uacce_put_queue(struct uacce_queue * q)2447 static void hisi_qm_uacce_put_queue(struct uacce_queue *q)
2448 {
2449 	struct hisi_qp *qp = q->priv;
2450 
2451 	hisi_qm_release_qp(qp);
2452 }
2453 
2454 /* map sq/cq/doorbell to user space */
hisi_qm_uacce_mmap(struct uacce_queue * q,struct vm_area_struct * vma,struct uacce_qfile_region * qfr)2455 static int hisi_qm_uacce_mmap(struct uacce_queue *q,
2456 			      struct vm_area_struct *vma,
2457 			      struct uacce_qfile_region *qfr)
2458 {
2459 	struct hisi_qp *qp = q->priv;
2460 	struct hisi_qm *qm = qp->qm;
2461 	resource_size_t phys_base = qm->db_phys_base +
2462 				    qp->qp_id * qm->db_interval;
2463 	size_t sz = vma->vm_end - vma->vm_start;
2464 	struct pci_dev *pdev = qm->pdev;
2465 	struct device *dev = &pdev->dev;
2466 	unsigned long vm_pgoff;
2467 	int ret;
2468 
2469 	switch (qfr->type) {
2470 	case UACCE_QFRT_MMIO:
2471 		if (qm->ver == QM_HW_V1) {
2472 			if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
2473 				return -EINVAL;
2474 		} else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {
2475 			if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
2476 			    QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
2477 				return -EINVAL;
2478 		} else {
2479 			if (sz > qm->db_interval)
2480 				return -EINVAL;
2481 		}
2482 
2483 		vm_flags_set(vma, VM_IO);
2484 
2485 		return remap_pfn_range(vma, vma->vm_start,
2486 				       phys_base >> PAGE_SHIFT,
2487 				       sz, pgprot_noncached(vma->vm_page_prot));
2488 	case UACCE_QFRT_DUS:
2489 		if (sz != qp->qdma.size)
2490 			return -EINVAL;
2491 
2492 		/*
2493 		 * dma_mmap_coherent() requires vm_pgoff as 0
2494 		 * restore vm_pfoff to initial value for mmap()
2495 		 */
2496 		vm_pgoff = vma->vm_pgoff;
2497 		vma->vm_pgoff = 0;
2498 		ret = dma_mmap_coherent(dev, vma, qp->qdma.va,
2499 					qp->qdma.dma, sz);
2500 		vma->vm_pgoff = vm_pgoff;
2501 		return ret;
2502 
2503 	default:
2504 		return -EINVAL;
2505 	}
2506 }
2507 
hisi_qm_uacce_start_queue(struct uacce_queue * q)2508 static int hisi_qm_uacce_start_queue(struct uacce_queue *q)
2509 {
2510 	struct hisi_qp *qp = q->priv;
2511 
2512 	return hisi_qm_start_qp(qp, qp->pasid);
2513 }
2514 
hisi_qm_uacce_stop_queue(struct uacce_queue * q)2515 static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)
2516 {
2517 	struct hisi_qp *qp = q->priv;
2518 	struct hisi_qm *qm = qp->qm;
2519 	struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx;
2520 	u32 i = 0;
2521 
2522 	hisi_qm_stop_qp(qp);
2523 
2524 	if (!dev_dfx->dev_timeout || !dev_dfx->dev_state)
2525 		return;
2526 
2527 	/*
2528 	 * After the queue fails to be stopped,
2529 	 * wait for a period of time before releasing the queue.
2530 	 */
2531 	while (++i) {
2532 		msleep(WAIT_PERIOD);
2533 
2534 		/* Since dev_timeout maybe modified, check i >= dev_timeout */
2535 		if (i >= dev_dfx->dev_timeout) {
2536 			dev_err(&qm->pdev->dev, "Stop q %u timeout, state %u\n",
2537 			       qp->qp_id, dev_dfx->dev_state);
2538 			dev_dfx->dev_state = QM_FINISH_WAIT;
2539 			break;
2540 		}
2541 	}
2542 }
2543 
hisi_qm_is_q_updated(struct uacce_queue * q)2544 static int hisi_qm_is_q_updated(struct uacce_queue *q)
2545 {
2546 	struct hisi_qp *qp = q->priv;
2547 	struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
2548 	int updated = 0;
2549 
2550 	while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
2551 		/* make sure to read data from memory */
2552 		dma_rmb();
2553 		qm_cq_head_update(qp);
2554 		cqe = qp->cqe + qp->qp_status.cq_head;
2555 		updated = 1;
2556 	}
2557 
2558 	return updated;
2559 }
2560 
qm_set_sqctype(struct uacce_queue * q,u16 type)2561 static void qm_set_sqctype(struct uacce_queue *q, u16 type)
2562 {
2563 	struct hisi_qm *qm = q->uacce->priv;
2564 	struct hisi_qp *qp = q->priv;
2565 
2566 	down_write(&qm->qps_lock);
2567 	qp->alg_type = type;
2568 	up_write(&qm->qps_lock);
2569 }
2570 
hisi_qm_uacce_ioctl(struct uacce_queue * q,unsigned int cmd,unsigned long arg)2571 static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,
2572 				unsigned long arg)
2573 {
2574 	struct hisi_qp *qp = q->priv;
2575 	struct hisi_qp_info qp_info;
2576 	struct hisi_qp_ctx qp_ctx;
2577 
2578 	if (cmd == UACCE_CMD_QM_SET_QP_CTX) {
2579 		if (copy_from_user(&qp_ctx, (void __user *)arg,
2580 				   sizeof(struct hisi_qp_ctx)))
2581 			return -EFAULT;
2582 
2583 		if (qp_ctx.qc_type > QM_MAX_QC_TYPE)
2584 			return -EINVAL;
2585 
2586 		qm_set_sqctype(q, qp_ctx.qc_type);
2587 		qp_ctx.id = qp->qp_id;
2588 
2589 		if (copy_to_user((void __user *)arg, &qp_ctx,
2590 				 sizeof(struct hisi_qp_ctx)))
2591 			return -EFAULT;
2592 
2593 		return 0;
2594 	} else if (cmd == UACCE_CMD_QM_SET_QP_INFO) {
2595 		if (copy_from_user(&qp_info, (void __user *)arg,
2596 				   sizeof(struct hisi_qp_info)))
2597 			return -EFAULT;
2598 
2599 		qp_info.sqe_size = qp->qm->sqe_size;
2600 		qp_info.sq_depth = qp->sq_depth;
2601 		qp_info.cq_depth = qp->cq_depth;
2602 
2603 		if (copy_to_user((void __user *)arg, &qp_info,
2604 				  sizeof(struct hisi_qp_info)))
2605 			return -EFAULT;
2606 
2607 		return 0;
2608 	}
2609 
2610 	return -EINVAL;
2611 }
2612 
2613 /**
2614  * qm_hw_err_isolate() - Try to set the isolation status of the uacce device
2615  * according to user's configuration of error threshold.
2616  * @qm: the uacce device
2617  */
qm_hw_err_isolate(struct hisi_qm * qm)2618 static int qm_hw_err_isolate(struct hisi_qm *qm)
2619 {
2620 	struct qm_hw_err *err, *tmp, *hw_err;
2621 	struct qm_err_isolate *isolate;
2622 	u32 count = 0;
2623 
2624 	isolate = &qm->isolate_data;
2625 
2626 #define SECONDS_PER_HOUR	3600
2627 
2628 	/* All the hw errs are processed by PF driver */
2629 	if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold)
2630 		return 0;
2631 
2632 	hw_err = kzalloc(sizeof(*hw_err), GFP_KERNEL);
2633 	if (!hw_err)
2634 		return -ENOMEM;
2635 
2636 	/*
2637 	 * Time-stamp every slot AER error. Then check the AER error log when the
2638 	 * next device AER error occurred. if the device slot AER error count exceeds
2639 	 * the setting error threshold in one hour, the isolated state will be set
2640 	 * to true. And the AER error logs that exceed one hour will be cleared.
2641 	 */
2642 	mutex_lock(&isolate->isolate_lock);
2643 	hw_err->timestamp = jiffies;
2644 	list_for_each_entry_safe(err, tmp, &isolate->qm_hw_errs, list) {
2645 		if ((hw_err->timestamp - err->timestamp) / HZ >
2646 		    SECONDS_PER_HOUR) {
2647 			list_del(&err->list);
2648 			kfree(err);
2649 		} else {
2650 			count++;
2651 		}
2652 	}
2653 	list_add(&hw_err->list, &isolate->qm_hw_errs);
2654 	mutex_unlock(&isolate->isolate_lock);
2655 
2656 	if (count >= isolate->err_threshold)
2657 		isolate->is_isolate = true;
2658 
2659 	return 0;
2660 }
2661 
qm_hw_err_destroy(struct hisi_qm * qm)2662 static void qm_hw_err_destroy(struct hisi_qm *qm)
2663 {
2664 	struct qm_hw_err *err, *tmp;
2665 
2666 	mutex_lock(&qm->isolate_data.isolate_lock);
2667 	list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) {
2668 		list_del(&err->list);
2669 		kfree(err);
2670 	}
2671 	mutex_unlock(&qm->isolate_data.isolate_lock);
2672 }
2673 
hisi_qm_get_isolate_state(struct uacce_device * uacce)2674 static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce)
2675 {
2676 	struct hisi_qm *qm = uacce->priv;
2677 	struct hisi_qm *pf_qm;
2678 
2679 	if (uacce->is_vf)
2680 		pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
2681 	else
2682 		pf_qm = qm;
2683 
2684 	return pf_qm->isolate_data.is_isolate ?
2685 			UACCE_DEV_ISOLATE : UACCE_DEV_NORMAL;
2686 }
2687 
hisi_qm_isolate_threshold_write(struct uacce_device * uacce,u32 num)2688 static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num)
2689 {
2690 	struct hisi_qm *qm = uacce->priv;
2691 
2692 	/* Must be set by PF */
2693 	if (uacce->is_vf)
2694 		return -EPERM;
2695 
2696 	if (qm->isolate_data.is_isolate)
2697 		return -EPERM;
2698 
2699 	qm->isolate_data.err_threshold = num;
2700 
2701 	/* After the policy is updated, need to reset the hardware err list */
2702 	qm_hw_err_destroy(qm);
2703 
2704 	return 0;
2705 }
2706 
hisi_qm_isolate_threshold_read(struct uacce_device * uacce)2707 static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce)
2708 {
2709 	struct hisi_qm *qm = uacce->priv;
2710 	struct hisi_qm *pf_qm;
2711 
2712 	if (uacce->is_vf) {
2713 		pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
2714 		return pf_qm->isolate_data.err_threshold;
2715 	}
2716 
2717 	return qm->isolate_data.err_threshold;
2718 }
2719 
2720 static const struct uacce_ops uacce_qm_ops = {
2721 	.get_available_instances = hisi_qm_get_available_instances,
2722 	.get_queue = hisi_qm_uacce_get_queue,
2723 	.put_queue = hisi_qm_uacce_put_queue,
2724 	.start_queue = hisi_qm_uacce_start_queue,
2725 	.stop_queue = hisi_qm_uacce_stop_queue,
2726 	.mmap = hisi_qm_uacce_mmap,
2727 	.ioctl = hisi_qm_uacce_ioctl,
2728 	.is_q_updated = hisi_qm_is_q_updated,
2729 	.get_isolate_state = hisi_qm_get_isolate_state,
2730 	.isolate_err_threshold_write = hisi_qm_isolate_threshold_write,
2731 	.isolate_err_threshold_read = hisi_qm_isolate_threshold_read,
2732 };
2733 
qm_remove_uacce(struct hisi_qm * qm)2734 static void qm_remove_uacce(struct hisi_qm *qm)
2735 {
2736 	struct uacce_device *uacce = qm->uacce;
2737 
2738 	if (qm->use_sva) {
2739 		qm_hw_err_destroy(qm);
2740 		uacce_remove(uacce);
2741 		qm->uacce = NULL;
2742 	}
2743 }
2744 
qm_alloc_uacce(struct hisi_qm * qm)2745 static int qm_alloc_uacce(struct hisi_qm *qm)
2746 {
2747 	struct pci_dev *pdev = qm->pdev;
2748 	struct uacce_device *uacce;
2749 	unsigned long mmio_page_nr;
2750 	unsigned long dus_page_nr;
2751 	u16 sq_depth, cq_depth;
2752 	struct uacce_interface interface = {
2753 		.flags = UACCE_DEV_SVA,
2754 		.ops = &uacce_qm_ops,
2755 	};
2756 	int ret;
2757 
2758 	ret = strscpy(interface.name, dev_driver_string(&pdev->dev),
2759 		      sizeof(interface.name));
2760 	if (ret < 0)
2761 		return -ENAMETOOLONG;
2762 
2763 	uacce = uacce_alloc(&pdev->dev, &interface);
2764 	if (IS_ERR(uacce))
2765 		return PTR_ERR(uacce);
2766 
2767 	if (uacce->flags & UACCE_DEV_SVA) {
2768 		qm->use_sva = true;
2769 	} else {
2770 		/* only consider sva case */
2771 		qm_remove_uacce(qm);
2772 		return -EINVAL;
2773 	}
2774 
2775 	uacce->is_vf = pdev->is_virtfn;
2776 	uacce->priv = qm;
2777 
2778 	if (qm->ver == QM_HW_V1)
2779 		uacce->api_ver = HISI_QM_API_VER_BASE;
2780 	else if (qm->ver == QM_HW_V2)
2781 		uacce->api_ver = HISI_QM_API_VER2_BASE;
2782 	else
2783 		uacce->api_ver = HISI_QM_API_VER3_BASE;
2784 
2785 	if (qm->ver == QM_HW_V1)
2786 		mmio_page_nr = QM_DOORBELL_PAGE_NR;
2787 	else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
2788 		mmio_page_nr = QM_DOORBELL_PAGE_NR +
2789 			QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
2790 	else
2791 		mmio_page_nr = qm->db_interval / PAGE_SIZE;
2792 
2793 	qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);
2794 
2795 	/* Add one more page for device or qp status */
2796 	dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth +
2797 		       sizeof(struct qm_cqe) * cq_depth  + PAGE_SIZE) >>
2798 					 PAGE_SHIFT;
2799 
2800 	uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;
2801 	uacce->qf_pg_num[UACCE_QFRT_DUS]  = dus_page_nr;
2802 
2803 	qm->uacce = uacce;
2804 	INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs);
2805 	mutex_init(&qm->isolate_data.isolate_lock);
2806 
2807 	return 0;
2808 }
2809 
2810 /**
2811  * qm_frozen() - Try to froze QM to cut continuous queue request. If
2812  * there is user on the QM, return failure without doing anything.
2813  * @qm: The qm needed to be fronzen.
2814  *
2815  * This function frozes QM, then we can do SRIOV disabling.
2816  */
qm_frozen(struct hisi_qm * qm)2817 static int qm_frozen(struct hisi_qm *qm)
2818 {
2819 	if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl))
2820 		return 0;
2821 
2822 	down_write(&qm->qps_lock);
2823 
2824 	if (!qm->qp_in_used) {
2825 		qm->qp_in_used = qm->qp_num;
2826 		up_write(&qm->qps_lock);
2827 		set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl);
2828 		return 0;
2829 	}
2830 
2831 	up_write(&qm->qps_lock);
2832 
2833 	return -EBUSY;
2834 }
2835 
qm_try_frozen_vfs(struct pci_dev * pdev,struct hisi_qm_list * qm_list)2836 static int qm_try_frozen_vfs(struct pci_dev *pdev,
2837 			     struct hisi_qm_list *qm_list)
2838 {
2839 	struct hisi_qm *qm, *vf_qm;
2840 	struct pci_dev *dev;
2841 	int ret = 0;
2842 
2843 	if (!qm_list || !pdev)
2844 		return -EINVAL;
2845 
2846 	/* Try to frozen all the VFs as disable SRIOV */
2847 	mutex_lock(&qm_list->lock);
2848 	list_for_each_entry(qm, &qm_list->list, list) {
2849 		dev = qm->pdev;
2850 		if (dev == pdev)
2851 			continue;
2852 		if (pci_physfn(dev) == pdev) {
2853 			vf_qm = pci_get_drvdata(dev);
2854 			ret = qm_frozen(vf_qm);
2855 			if (ret)
2856 				goto frozen_fail;
2857 		}
2858 	}
2859 
2860 frozen_fail:
2861 	mutex_unlock(&qm_list->lock);
2862 
2863 	return ret;
2864 }
2865 
2866 /**
2867  * hisi_qm_wait_task_finish() - Wait until the task is finished
2868  * when removing the driver.
2869  * @qm: The qm needed to wait for the task to finish.
2870  * @qm_list: The list of all available devices.
2871  */
hisi_qm_wait_task_finish(struct hisi_qm * qm,struct hisi_qm_list * qm_list)2872 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
2873 {
2874 	while (qm_frozen(qm) ||
2875 	       ((qm->fun_type == QM_HW_PF) &&
2876 	       qm_try_frozen_vfs(qm->pdev, qm_list))) {
2877 		msleep(WAIT_PERIOD);
2878 	}
2879 
2880 	while (test_bit(QM_RST_SCHED, &qm->misc_ctl) ||
2881 	       test_bit(QM_RESETTING, &qm->misc_ctl))
2882 		msleep(WAIT_PERIOD);
2883 
2884 	if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2885 		flush_work(&qm->cmd_process);
2886 
2887 	udelay(REMOVE_WAIT_DELAY);
2888 }
2889 EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
2890 
hisi_qp_memory_uninit(struct hisi_qm * qm,int num)2891 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
2892 {
2893 	struct device *dev = &qm->pdev->dev;
2894 	struct qm_dma *qdma;
2895 	int i;
2896 
2897 	for (i = num - 1; i >= 0; i--) {
2898 		qdma = &qm->qp_array[i].qdma;
2899 		dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
2900 		kfree(qm->poll_data[i].qp_finish_id);
2901 	}
2902 
2903 	kfree(qm->poll_data);
2904 	kfree(qm->qp_array);
2905 }
2906 
hisi_qp_memory_init(struct hisi_qm * qm,size_t dma_size,int id,u16 sq_depth,u16 cq_depth)2907 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id,
2908 			       u16 sq_depth, u16 cq_depth)
2909 {
2910 	struct device *dev = &qm->pdev->dev;
2911 	size_t off = qm->sqe_size * sq_depth;
2912 	struct hisi_qp *qp;
2913 	int ret = -ENOMEM;
2914 
2915 	qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16),
2916 						 GFP_KERNEL);
2917 	if (!qm->poll_data[id].qp_finish_id)
2918 		return -ENOMEM;
2919 
2920 	qp = &qm->qp_array[id];
2921 	qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
2922 					 GFP_KERNEL);
2923 	if (!qp->qdma.va)
2924 		goto err_free_qp_finish_id;
2925 
2926 	qp->sqe = qp->qdma.va;
2927 	qp->sqe_dma = qp->qdma.dma;
2928 	qp->cqe = qp->qdma.va + off;
2929 	qp->cqe_dma = qp->qdma.dma + off;
2930 	qp->qdma.size = dma_size;
2931 	qp->sq_depth = sq_depth;
2932 	qp->cq_depth = cq_depth;
2933 	qp->qm = qm;
2934 	qp->qp_id = id;
2935 
2936 	return 0;
2937 
2938 err_free_qp_finish_id:
2939 	kfree(qm->poll_data[id].qp_finish_id);
2940 	return ret;
2941 }
2942 
hisi_qm_pre_init(struct hisi_qm * qm)2943 static void hisi_qm_pre_init(struct hisi_qm *qm)
2944 {
2945 	struct pci_dev *pdev = qm->pdev;
2946 
2947 	if (qm->ver == QM_HW_V1)
2948 		qm->ops = &qm_hw_ops_v1;
2949 	else if (qm->ver == QM_HW_V2)
2950 		qm->ops = &qm_hw_ops_v2;
2951 	else if (qm->ver == QM_HW_V3)
2952 		qm->ops = &qm_hw_ops_v3;
2953 	else
2954 		qm->ops = &qm_hw_ops_v4;
2955 
2956 	pci_set_drvdata(pdev, qm);
2957 	mutex_init(&qm->mailbox_lock);
2958 	mutex_init(&qm->ifc_lock);
2959 	init_rwsem(&qm->qps_lock);
2960 	qm->qp_in_used = 0;
2961 	if (test_bit(QM_SUPPORT_RPM, &qm->caps)) {
2962 		if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev)))
2963 			dev_info(&pdev->dev, "_PS0 and _PR0 are not defined");
2964 	}
2965 }
2966 
qm_cmd_uninit(struct hisi_qm * qm)2967 static void qm_cmd_uninit(struct hisi_qm *qm)
2968 {
2969 	u32 val;
2970 
2971 	if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2972 		return;
2973 
2974 	val = readl(qm->io_base + QM_IFC_INT_MASK);
2975 	val |= QM_IFC_INT_DISABLE;
2976 	writel(val, qm->io_base + QM_IFC_INT_MASK);
2977 }
2978 
qm_cmd_init(struct hisi_qm * qm)2979 static void qm_cmd_init(struct hisi_qm *qm)
2980 {
2981 	u32 val;
2982 
2983 	if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2984 		return;
2985 
2986 	/* Clear communication interrupt source */
2987 	qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR);
2988 
2989 	/* Enable pf to vf communication reg. */
2990 	val = readl(qm->io_base + QM_IFC_INT_MASK);
2991 	val &= ~QM_IFC_INT_DISABLE;
2992 	writel(val, qm->io_base + QM_IFC_INT_MASK);
2993 }
2994 
qm_put_pci_res(struct hisi_qm * qm)2995 static void qm_put_pci_res(struct hisi_qm *qm)
2996 {
2997 	struct pci_dev *pdev = qm->pdev;
2998 
2999 	if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
3000 		iounmap(qm->db_io_base);
3001 
3002 	iounmap(qm->io_base);
3003 	pci_release_mem_regions(pdev);
3004 }
3005 
hisi_qm_pci_uninit(struct hisi_qm * qm)3006 static void hisi_qm_pci_uninit(struct hisi_qm *qm)
3007 {
3008 	struct pci_dev *pdev = qm->pdev;
3009 
3010 	pci_free_irq_vectors(pdev);
3011 	qm_put_pci_res(qm);
3012 	pci_disable_device(pdev);
3013 }
3014 
hisi_qm_set_state(struct hisi_qm * qm,u8 state)3015 static void hisi_qm_set_state(struct hisi_qm *qm, u8 state)
3016 {
3017 	if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF)
3018 		writel(state, qm->io_base + QM_VF_STATE);
3019 }
3020 
hisi_qm_unint_work(struct hisi_qm * qm)3021 static void hisi_qm_unint_work(struct hisi_qm *qm)
3022 {
3023 	destroy_workqueue(qm->wq);
3024 }
3025 
hisi_qm_free_rsv_buf(struct hisi_qm * qm)3026 static void hisi_qm_free_rsv_buf(struct hisi_qm *qm)
3027 {
3028 	struct qm_dma *xqc_dma = &qm->xqc_buf.qcdma;
3029 	struct device *dev = &qm->pdev->dev;
3030 
3031 	dma_free_coherent(dev, xqc_dma->size, xqc_dma->va, xqc_dma->dma);
3032 }
3033 
hisi_qm_memory_uninit(struct hisi_qm * qm)3034 static void hisi_qm_memory_uninit(struct hisi_qm *qm)
3035 {
3036 	struct device *dev = &qm->pdev->dev;
3037 
3038 	hisi_qp_memory_uninit(qm, qm->qp_num);
3039 	hisi_qm_free_rsv_buf(qm);
3040 	if (qm->qdma.va) {
3041 		hisi_qm_cache_wb(qm);
3042 		dma_free_coherent(dev, qm->qdma.size,
3043 				  qm->qdma.va, qm->qdma.dma);
3044 	}
3045 
3046 	idr_destroy(&qm->qp_idr);
3047 
3048 	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
3049 		kfree(qm->factor);
3050 }
3051 
3052 /**
3053  * hisi_qm_uninit() - Uninitialize qm.
3054  * @qm: The qm needed uninit.
3055  *
3056  * This function uninits qm related device resources.
3057  */
hisi_qm_uninit(struct hisi_qm * qm)3058 void hisi_qm_uninit(struct hisi_qm *qm)
3059 {
3060 	qm_cmd_uninit(qm);
3061 	hisi_qm_unint_work(qm);
3062 
3063 	down_write(&qm->qps_lock);
3064 	hisi_qm_memory_uninit(qm);
3065 	hisi_qm_set_state(qm, QM_NOT_READY);
3066 	up_write(&qm->qps_lock);
3067 
3068 	qm_remove_uacce(qm);
3069 	qm_irqs_unregister(qm);
3070 	hisi_qm_pci_uninit(qm);
3071 }
3072 EXPORT_SYMBOL_GPL(hisi_qm_uninit);
3073 
3074 /**
3075  * hisi_qm_get_vft() - Get vft from a qm.
3076  * @qm: The qm we want to get its vft.
3077  * @base: The base number of queue in vft.
3078  * @number: The number of queues in vft.
3079  *
3080  * We can allocate multiple queues to a qm by configuring virtual function
3081  * table. We get related configures by this function. Normally, we call this
3082  * function in VF driver to get the queue information.
3083  *
3084  * qm hw v1 does not support this interface.
3085  */
hisi_qm_get_vft(struct hisi_qm * qm,u32 * base,u32 * number)3086 static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
3087 {
3088 	if (!base || !number)
3089 		return -EINVAL;
3090 
3091 	if (!qm->ops->get_vft) {
3092 		dev_err(&qm->pdev->dev, "Don't support vft read!\n");
3093 		return -EINVAL;
3094 	}
3095 
3096 	return qm->ops->get_vft(qm, base, number);
3097 }
3098 
3099 /**
3100  * hisi_qm_set_vft() - Set vft to a qm.
3101  * @qm: The qm we want to set its vft.
3102  * @fun_num: The function number.
3103  * @base: The base number of queue in vft.
3104  * @number: The number of queues in vft.
3105  *
3106  * This function is alway called in PF driver, it is used to assign queues
3107  * among PF and VFs.
3108  *
3109  * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
3110  * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
3111  * (VF function number 0x2)
3112  */
hisi_qm_set_vft(struct hisi_qm * qm,u32 fun_num,u32 base,u32 number)3113 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
3114 		    u32 number)
3115 {
3116 	u32 max_q_num = qm->ctrl_qp_num;
3117 
3118 	if (base >= max_q_num || number > max_q_num ||
3119 	    (base + number) > max_q_num)
3120 		return -EINVAL;
3121 
3122 	return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
3123 }
3124 
qm_init_eq_aeq_status(struct hisi_qm * qm)3125 static void qm_init_eq_aeq_status(struct hisi_qm *qm)
3126 {
3127 	struct hisi_qm_status *status = &qm->status;
3128 
3129 	status->eq_head = 0;
3130 	status->aeq_head = 0;
3131 	status->eqc_phase = true;
3132 	status->aeqc_phase = true;
3133 }
3134 
qm_enable_eq_aeq_interrupts(struct hisi_qm * qm)3135 static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm)
3136 {
3137 	/* Clear eq/aeq interrupt source */
3138 	qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
3139 	qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
3140 
3141 	writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
3142 	writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
3143 }
3144 
qm_disable_eq_aeq_interrupts(struct hisi_qm * qm)3145 static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm)
3146 {
3147 	writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
3148 	writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
3149 }
3150 
qm_eq_ctx_cfg(struct hisi_qm * qm)3151 static int qm_eq_ctx_cfg(struct hisi_qm *qm)
3152 {
3153 	struct qm_eqc eqc = {0};
3154 
3155 	eqc.base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));
3156 	eqc.base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
3157 	if (qm->ver == QM_HW_V1)
3158 		eqc.dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
3159 	eqc.dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));
3160 
3161 	return qm_set_and_get_xqc(qm, QM_MB_CMD_EQC, &eqc, 0, 0);
3162 }
3163 
qm_aeq_ctx_cfg(struct hisi_qm * qm)3164 static int qm_aeq_ctx_cfg(struct hisi_qm *qm)
3165 {
3166 	struct qm_aeqc aeqc = {0};
3167 
3168 	aeqc.base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));
3169 	aeqc.base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));
3170 	aeqc.dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));
3171 
3172 	return qm_set_and_get_xqc(qm, QM_MB_CMD_AEQC, &aeqc, 0, 0);
3173 }
3174 
qm_eq_aeq_ctx_cfg(struct hisi_qm * qm)3175 static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)
3176 {
3177 	struct device *dev = &qm->pdev->dev;
3178 	int ret;
3179 
3180 	qm_init_eq_aeq_status(qm);
3181 
3182 	ret = qm_eq_ctx_cfg(qm);
3183 	if (ret) {
3184 		dev_err(dev, "Set eqc failed!\n");
3185 		return ret;
3186 	}
3187 
3188 	return qm_aeq_ctx_cfg(qm);
3189 }
3190 
__hisi_qm_start(struct hisi_qm * qm)3191 static int __hisi_qm_start(struct hisi_qm *qm)
3192 {
3193 	int ret;
3194 
3195 	WARN_ON(!qm->qdma.va);
3196 
3197 	if (qm->fun_type == QM_HW_PF) {
3198 		ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
3199 		if (ret)
3200 			return ret;
3201 	}
3202 
3203 	ret = qm_eq_aeq_ctx_cfg(qm);
3204 	if (ret)
3205 		return ret;
3206 
3207 	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
3208 	if (ret)
3209 		return ret;
3210 
3211 	ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
3212 	if (ret)
3213 		return ret;
3214 
3215 	qm_init_prefetch(qm);
3216 	qm_enable_eq_aeq_interrupts(qm);
3217 
3218 	return 0;
3219 }
3220 
3221 /**
3222  * hisi_qm_start() - start qm
3223  * @qm: The qm to be started.
3224  *
3225  * This function starts a qm, then we can allocate qp from this qm.
3226  */
hisi_qm_start(struct hisi_qm * qm)3227 int hisi_qm_start(struct hisi_qm *qm)
3228 {
3229 	struct device *dev = &qm->pdev->dev;
3230 	int ret = 0;
3231 
3232 	down_write(&qm->qps_lock);
3233 
3234 	dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);
3235 
3236 	if (!qm->qp_num) {
3237 		dev_err(dev, "qp_num should not be 0\n");
3238 		ret = -EINVAL;
3239 		goto err_unlock;
3240 	}
3241 
3242 	ret = __hisi_qm_start(qm);
3243 	if (ret)
3244 		goto err_unlock;
3245 
3246 	atomic_set(&qm->status.flags, QM_WORK);
3247 	hisi_qm_set_state(qm, QM_READY);
3248 
3249 err_unlock:
3250 	up_write(&qm->qps_lock);
3251 	return ret;
3252 }
3253 EXPORT_SYMBOL_GPL(hisi_qm_start);
3254 
qm_restart(struct hisi_qm * qm)3255 static int qm_restart(struct hisi_qm *qm)
3256 {
3257 	struct device *dev = &qm->pdev->dev;
3258 	struct hisi_qp *qp;
3259 	int ret, i;
3260 
3261 	ret = hisi_qm_start(qm);
3262 	if (ret < 0)
3263 		return ret;
3264 
3265 	down_write(&qm->qps_lock);
3266 	for (i = 0; i < qm->qp_num; i++) {
3267 		qp = &qm->qp_array[i];
3268 		if (atomic_read(&qp->qp_status.flags) == QP_STOP &&
3269 		    qp->is_resetting == true) {
3270 			ret = qm_start_qp_nolock(qp, 0);
3271 			if (ret < 0) {
3272 				dev_err(dev, "Failed to start qp%d!\n", i);
3273 
3274 				up_write(&qm->qps_lock);
3275 				return ret;
3276 			}
3277 			qp->is_resetting = false;
3278 		}
3279 	}
3280 	up_write(&qm->qps_lock);
3281 
3282 	return 0;
3283 }
3284 
3285 /* Stop started qps in reset flow */
qm_stop_started_qp(struct hisi_qm * qm)3286 static void qm_stop_started_qp(struct hisi_qm *qm)
3287 {
3288 	struct hisi_qp *qp;
3289 	int i;
3290 
3291 	for (i = 0; i < qm->qp_num; i++) {
3292 		qp = &qm->qp_array[i];
3293 		if (atomic_read(&qp->qp_status.flags) == QP_START) {
3294 			qp->is_resetting = true;
3295 			qm_stop_qp_nolock(qp);
3296 		}
3297 	}
3298 }
3299 
3300 /**
3301  * qm_clear_queues() - Clear all queues memory in a qm.
3302  * @qm: The qm in which the queues will be cleared.
3303  *
3304  * This function clears all queues memory in a qm. Reset of accelerator can
3305  * use this to clear queues.
3306  */
qm_clear_queues(struct hisi_qm * qm)3307 static void qm_clear_queues(struct hisi_qm *qm)
3308 {
3309 	struct hisi_qp *qp;
3310 	int i;
3311 
3312 	for (i = 0; i < qm->qp_num; i++) {
3313 		qp = &qm->qp_array[i];
3314 		if (qp->is_in_kernel && qp->is_resetting)
3315 			memset(qp->qdma.va, 0, qp->qdma.size);
3316 	}
3317 
3318 	memset(qm->qdma.va, 0, qm->qdma.size);
3319 }
3320 
3321 /**
3322  * hisi_qm_stop() - Stop a qm.
3323  * @qm: The qm which will be stopped.
3324  * @r: The reason to stop qm.
3325  *
3326  * This function stops qm and its qps, then qm can not accept request.
3327  * Related resources are not released at this state, we can use hisi_qm_start
3328  * to let qm start again.
3329  */
hisi_qm_stop(struct hisi_qm * qm,enum qm_stop_reason r)3330 int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
3331 {
3332 	struct device *dev = &qm->pdev->dev;
3333 	int ret = 0;
3334 
3335 	down_write(&qm->qps_lock);
3336 
3337 	if (atomic_read(&qm->status.flags) == QM_STOP)
3338 		goto err_unlock;
3339 
3340 	/* Stop all the request sending at first. */
3341 	atomic_set(&qm->status.flags, QM_STOP);
3342 	qm->status.stop_reason = r;
3343 
3344 	if (qm->status.stop_reason != QM_NORMAL) {
3345 		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
3346 		/*
3347 		 * When performing soft reset, the hardware will no longer
3348 		 * do tasks, and the tasks in the device will be flushed
3349 		 * out directly since the master ooo is closed.
3350 		 */
3351 		if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) &&
3352 		    r != QM_SOFT_RESET) {
3353 			ret = qm_drain_qm(qm);
3354 			if (ret) {
3355 				dev_err(dev, "failed to drain qm!\n");
3356 				goto err_unlock;
3357 			}
3358 		}
3359 
3360 		qm_stop_started_qp(qm);
3361 
3362 		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
3363 	}
3364 
3365 	qm_disable_eq_aeq_interrupts(qm);
3366 	if (qm->fun_type == QM_HW_PF) {
3367 		ret = hisi_qm_set_vft(qm, 0, 0, 0);
3368 		if (ret < 0) {
3369 			dev_err(dev, "Failed to set vft!\n");
3370 			ret = -EBUSY;
3371 			goto err_unlock;
3372 		}
3373 	}
3374 
3375 	qm_clear_queues(qm);
3376 	qm->status.stop_reason = QM_NORMAL;
3377 
3378 err_unlock:
3379 	up_write(&qm->qps_lock);
3380 	return ret;
3381 }
3382 EXPORT_SYMBOL_GPL(hisi_qm_stop);
3383 
qm_hw_error_init(struct hisi_qm * qm)3384 static void qm_hw_error_init(struct hisi_qm *qm)
3385 {
3386 	if (!qm->ops->hw_error_init) {
3387 		dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
3388 		return;
3389 	}
3390 
3391 	qm->ops->hw_error_init(qm);
3392 }
3393 
qm_hw_error_uninit(struct hisi_qm * qm)3394 static void qm_hw_error_uninit(struct hisi_qm *qm)
3395 {
3396 	if (!qm->ops->hw_error_uninit) {
3397 		dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");
3398 		return;
3399 	}
3400 
3401 	qm->ops->hw_error_uninit(qm);
3402 }
3403 
qm_hw_error_handle(struct hisi_qm * qm)3404 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)
3405 {
3406 	if (!qm->ops->hw_error_handle) {
3407 		dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");
3408 		return ACC_ERR_NONE;
3409 	}
3410 
3411 	return qm->ops->hw_error_handle(qm);
3412 }
3413 
3414 /**
3415  * hisi_qm_dev_err_init() - Initialize device error configuration.
3416  * @qm: The qm for which we want to do error initialization.
3417  *
3418  * Initialize QM and device error related configuration.
3419  */
hisi_qm_dev_err_init(struct hisi_qm * qm)3420 void hisi_qm_dev_err_init(struct hisi_qm *qm)
3421 {
3422 	if (qm->fun_type == QM_HW_VF)
3423 		return;
3424 
3425 	qm_hw_error_init(qm);
3426 
3427 	if (!qm->err_ini->hw_err_enable) {
3428 		dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");
3429 		return;
3430 	}
3431 	qm->err_ini->hw_err_enable(qm);
3432 }
3433 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);
3434 
3435 /**
3436  * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.
3437  * @qm: The qm for which we want to do error uninitialization.
3438  *
3439  * Uninitialize QM and device error related configuration.
3440  */
hisi_qm_dev_err_uninit(struct hisi_qm * qm)3441 void hisi_qm_dev_err_uninit(struct hisi_qm *qm)
3442 {
3443 	if (qm->fun_type == QM_HW_VF)
3444 		return;
3445 
3446 	qm_hw_error_uninit(qm);
3447 
3448 	if (!qm->err_ini->hw_err_disable) {
3449 		dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");
3450 		return;
3451 	}
3452 	qm->err_ini->hw_err_disable(qm);
3453 }
3454 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);
3455 
3456 /**
3457  * hisi_qm_free_qps() - free multiple queue pairs.
3458  * @qps: The queue pairs need to be freed.
3459  * @qp_num: The num of queue pairs.
3460  */
hisi_qm_free_qps(struct hisi_qp ** qps,int qp_num)3461 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)
3462 {
3463 	int i;
3464 
3465 	if (!qps || qp_num <= 0)
3466 		return;
3467 
3468 	for (i = qp_num - 1; i >= 0; i--)
3469 		hisi_qm_release_qp(qps[i]);
3470 }
3471 EXPORT_SYMBOL_GPL(hisi_qm_free_qps);
3472 
free_list(struct list_head * head)3473 static void free_list(struct list_head *head)
3474 {
3475 	struct hisi_qm_resource *res, *tmp;
3476 
3477 	list_for_each_entry_safe(res, tmp, head, list) {
3478 		list_del(&res->list);
3479 		kfree(res);
3480 	}
3481 }
3482 
hisi_qm_sort_devices(int node,struct list_head * head,struct hisi_qm_list * qm_list)3483 static int hisi_qm_sort_devices(int node, struct list_head *head,
3484 				struct hisi_qm_list *qm_list)
3485 {
3486 	struct hisi_qm_resource *res, *tmp;
3487 	struct hisi_qm *qm;
3488 	struct list_head *n;
3489 	struct device *dev;
3490 	int dev_node;
3491 
3492 	list_for_each_entry(qm, &qm_list->list, list) {
3493 		dev = &qm->pdev->dev;
3494 
3495 		dev_node = dev_to_node(dev);
3496 		if (dev_node < 0)
3497 			dev_node = 0;
3498 
3499 		res = kzalloc(sizeof(*res), GFP_KERNEL);
3500 		if (!res)
3501 			return -ENOMEM;
3502 
3503 		res->qm = qm;
3504 		res->distance = node_distance(dev_node, node);
3505 		n = head;
3506 		list_for_each_entry(tmp, head, list) {
3507 			if (res->distance < tmp->distance) {
3508 				n = &tmp->list;
3509 				break;
3510 			}
3511 		}
3512 		list_add_tail(&res->list, n);
3513 	}
3514 
3515 	return 0;
3516 }
3517 
3518 /**
3519  * hisi_qm_alloc_qps_node() - Create multiple queue pairs.
3520  * @qm_list: The list of all available devices.
3521  * @qp_num: The number of queue pairs need created.
3522  * @alg_type: The algorithm type.
3523  * @node: The numa node.
3524  * @qps: The queue pairs need created.
3525  *
3526  * This function will sort all available device according to numa distance.
3527  * Then try to create all queue pairs from one device, if all devices do
3528  * not meet the requirements will return error.
3529  */
hisi_qm_alloc_qps_node(struct hisi_qm_list * qm_list,int qp_num,u8 alg_type,int node,struct hisi_qp ** qps)3530 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
3531 			   u8 alg_type, int node, struct hisi_qp **qps)
3532 {
3533 	struct hisi_qm_resource *tmp;
3534 	int ret = -ENODEV;
3535 	LIST_HEAD(head);
3536 	int i;
3537 
3538 	if (!qps || !qm_list || qp_num <= 0)
3539 		return -EINVAL;
3540 
3541 	mutex_lock(&qm_list->lock);
3542 	if (hisi_qm_sort_devices(node, &head, qm_list)) {
3543 		mutex_unlock(&qm_list->lock);
3544 		goto err;
3545 	}
3546 
3547 	list_for_each_entry(tmp, &head, list) {
3548 		for (i = 0; i < qp_num; i++) {
3549 			qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);
3550 			if (IS_ERR(qps[i])) {
3551 				hisi_qm_free_qps(qps, i);
3552 				break;
3553 			}
3554 		}
3555 
3556 		if (i == qp_num) {
3557 			ret = 0;
3558 			break;
3559 		}
3560 	}
3561 
3562 	mutex_unlock(&qm_list->lock);
3563 	if (ret)
3564 		pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n",
3565 			node, alg_type, qp_num);
3566 
3567 err:
3568 	free_list(&head);
3569 	return ret;
3570 }
3571 EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
3572 
qm_vf_q_assign(struct hisi_qm * qm,u32 num_vfs)3573 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
3574 {
3575 	u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;
3576 	u32 max_qp_num = qm->max_qp_num;
3577 	u32 q_base = qm->qp_num;
3578 	int ret;
3579 
3580 	if (!num_vfs)
3581 		return -EINVAL;
3582 
3583 	vfs_q_num = qm->ctrl_qp_num - qm->qp_num;
3584 
3585 	/* If vfs_q_num is less than num_vfs, return error. */
3586 	if (vfs_q_num < num_vfs)
3587 		return -EINVAL;
3588 
3589 	q_num = vfs_q_num / num_vfs;
3590 	remain_q_num = vfs_q_num % num_vfs;
3591 
3592 	for (i = num_vfs; i > 0; i--) {
3593 		/*
3594 		 * if q_num + remain_q_num > max_qp_num in last vf, divide the
3595 		 * remaining queues equally.
3596 		 */
3597 		if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {
3598 			act_q_num = q_num + remain_q_num;
3599 			remain_q_num = 0;
3600 		} else if (remain_q_num > 0) {
3601 			act_q_num = q_num + 1;
3602 			remain_q_num--;
3603 		} else {
3604 			act_q_num = q_num;
3605 		}
3606 
3607 		act_q_num = min(act_q_num, max_qp_num);
3608 		ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);
3609 		if (ret) {
3610 			for (j = num_vfs; j > i; j--)
3611 				hisi_qm_set_vft(qm, j, 0, 0);
3612 			return ret;
3613 		}
3614 		q_base += act_q_num;
3615 	}
3616 
3617 	return 0;
3618 }
3619 
qm_clear_vft_config(struct hisi_qm * qm)3620 static int qm_clear_vft_config(struct hisi_qm *qm)
3621 {
3622 	int ret;
3623 	u32 i;
3624 
3625 	for (i = 1; i <= qm->vfs_num; i++) {
3626 		ret = hisi_qm_set_vft(qm, i, 0, 0);
3627 		if (ret)
3628 			return ret;
3629 	}
3630 	qm->vfs_num = 0;
3631 
3632 	return 0;
3633 }
3634 
qm_func_shaper_enable(struct hisi_qm * qm,u32 fun_index,u32 qos)3635 static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos)
3636 {
3637 	struct device *dev = &qm->pdev->dev;
3638 	u32 ir = qos * QM_QOS_RATE;
3639 	int ret, total_vfs, i;
3640 
3641 	total_vfs = pci_sriov_get_totalvfs(qm->pdev);
3642 	if (fun_index > total_vfs)
3643 		return -EINVAL;
3644 
3645 	qm->factor[fun_index].func_qos = qos;
3646 
3647 	ret = qm_get_shaper_para(ir, &qm->factor[fun_index]);
3648 	if (ret) {
3649 		dev_err(dev, "failed to calculate shaper parameter!\n");
3650 		return -EINVAL;
3651 	}
3652 
3653 	for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
3654 		/* The base number of queue reuse for different alg type */
3655 		ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1);
3656 		if (ret) {
3657 			dev_err(dev, "type: %d, failed to set shaper vft!\n", i);
3658 			return -EINVAL;
3659 		}
3660 	}
3661 
3662 	return 0;
3663 }
3664 
qm_get_shaper_vft_qos(struct hisi_qm * qm,u32 fun_index)3665 static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index)
3666 {
3667 	u64 cir_u = 0, cir_b = 0, cir_s = 0;
3668 	u64 shaper_vft, ir_calc, ir;
3669 	unsigned int val;
3670 	u32 error_rate;
3671 	int ret;
3672 
3673 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
3674 					 val & BIT(0), POLL_PERIOD,
3675 					 POLL_TIMEOUT);
3676 	if (ret)
3677 		return 0;
3678 
3679 	writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);
3680 	writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE);
3681 	writel(fun_index, qm->io_base + QM_VFT_CFG);
3682 
3683 	writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
3684 	writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
3685 
3686 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
3687 					 val & BIT(0), POLL_PERIOD,
3688 					 POLL_TIMEOUT);
3689 	if (ret)
3690 		return 0;
3691 
3692 	shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |
3693 		  ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32);
3694 
3695 	cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK;
3696 	cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK;
3697 	cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT;
3698 
3699 	cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK;
3700 	cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT;
3701 
3702 	ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
3703 
3704 	ir = qm->factor[fun_index].func_qos * QM_QOS_RATE;
3705 
3706 	error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
3707 	if (error_rate > QM_QOS_MIN_ERROR_RATE) {
3708 		pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate);
3709 		return 0;
3710 	}
3711 
3712 	return ir;
3713 }
3714 
qm_vf_get_qos(struct hisi_qm * qm,u32 fun_num)3715 static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num)
3716 {
3717 	struct device *dev = &qm->pdev->dev;
3718 	u32 qos;
3719 	int ret;
3720 
3721 	qos = qm_get_shaper_vft_qos(qm, fun_num);
3722 	if (!qos) {
3723 		dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num);
3724 		return;
3725 	}
3726 
3727 	ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num);
3728 	if (ret)
3729 		dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num);
3730 }
3731 
qm_vf_read_qos(struct hisi_qm * qm)3732 static int qm_vf_read_qos(struct hisi_qm *qm)
3733 {
3734 	int cnt = 0;
3735 	int ret = -EINVAL;
3736 
3737 	/* reset mailbox qos val */
3738 	qm->mb_qos = 0;
3739 
3740 	/* vf ping pf to get function qos */
3741 	ret = qm_ping_pf(qm, QM_VF_GET_QOS);
3742 	if (ret) {
3743 		pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n");
3744 		return ret;
3745 	}
3746 
3747 	while (true) {
3748 		msleep(QM_WAIT_DST_ACK);
3749 		if (qm->mb_qos)
3750 			break;
3751 
3752 		if (++cnt > QM_MAX_VF_WAIT_COUNT) {
3753 			pci_err(qm->pdev, "PF ping VF timeout!\n");
3754 			return  -ETIMEDOUT;
3755 		}
3756 	}
3757 
3758 	return ret;
3759 }
3760 
qm_algqos_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)3761 static ssize_t qm_algqos_read(struct file *filp, char __user *buf,
3762 			       size_t count, loff_t *pos)
3763 {
3764 	struct hisi_qm *qm = filp->private_data;
3765 	char tbuf[QM_DBG_READ_LEN];
3766 	u32 qos_val, ir;
3767 	int ret;
3768 
3769 	ret = hisi_qm_get_dfx_access(qm);
3770 	if (ret)
3771 		return ret;
3772 
3773 	/* Mailbox and reset cannot be operated at the same time */
3774 	if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
3775 		pci_err(qm->pdev, "dev resetting, read alg qos failed!\n");
3776 		ret = -EAGAIN;
3777 		goto err_put_dfx_access;
3778 	}
3779 
3780 	if (qm->fun_type == QM_HW_PF) {
3781 		ir = qm_get_shaper_vft_qos(qm, 0);
3782 	} else {
3783 		ret = qm_vf_read_qos(qm);
3784 		if (ret)
3785 			goto err_get_status;
3786 		ir = qm->mb_qos;
3787 	}
3788 
3789 	qos_val = ir / QM_QOS_RATE;
3790 	ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val);
3791 
3792 	ret = simple_read_from_buffer(buf, count, pos, tbuf, ret);
3793 
3794 err_get_status:
3795 	clear_bit(QM_RESETTING, &qm->misc_ctl);
3796 err_put_dfx_access:
3797 	hisi_qm_put_dfx_access(qm);
3798 	return ret;
3799 }
3800 
qm_get_qos_value(struct hisi_qm * qm,const char * buf,unsigned long * val,unsigned int * fun_index)3801 static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,
3802 			       unsigned long *val,
3803 			       unsigned int *fun_index)
3804 {
3805 	const struct bus_type *bus_type = qm->pdev->dev.bus;
3806 	char tbuf_bdf[QM_DBG_READ_LEN] = {0};
3807 	char val_buf[QM_DBG_READ_LEN] = {0};
3808 	struct pci_dev *pdev;
3809 	struct device *dev;
3810 	int ret;
3811 
3812 	ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf);
3813 	if (ret != QM_QOS_PARAM_NUM)
3814 		return -EINVAL;
3815 
3816 	ret = kstrtoul(val_buf, 10, val);
3817 	if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) {
3818 		pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
3819 		return -EINVAL;
3820 	}
3821 
3822 	dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf);
3823 	if (!dev) {
3824 		pci_err(qm->pdev, "input pci bdf number is error!\n");
3825 		return -ENODEV;
3826 	}
3827 
3828 	pdev = container_of(dev, struct pci_dev, dev);
3829 
3830 	*fun_index = pdev->devfn;
3831 
3832 	return 0;
3833 }
3834 
qm_algqos_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)3835 static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
3836 			       size_t count, loff_t *pos)
3837 {
3838 	struct hisi_qm *qm = filp->private_data;
3839 	char tbuf[QM_DBG_READ_LEN];
3840 	unsigned int fun_index;
3841 	unsigned long val;
3842 	int len, ret;
3843 
3844 	if (*pos != 0)
3845 		return 0;
3846 
3847 	if (count >= QM_DBG_READ_LEN)
3848 		return -ENOSPC;
3849 
3850 	len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
3851 	if (len < 0)
3852 		return len;
3853 
3854 	tbuf[len] = '\0';
3855 	ret = qm_get_qos_value(qm, tbuf, &val, &fun_index);
3856 	if (ret)
3857 		return ret;
3858 
3859 	/* Mailbox and reset cannot be operated at the same time */
3860 	if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
3861 		pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
3862 		return -EAGAIN;
3863 	}
3864 
3865 	ret = qm_pm_get_sync(qm);
3866 	if (ret) {
3867 		ret = -EINVAL;
3868 		goto err_get_status;
3869 	}
3870 
3871 	ret = qm_func_shaper_enable(qm, fun_index, val);
3872 	if (ret) {
3873 		pci_err(qm->pdev, "failed to enable function shaper!\n");
3874 		ret = -EINVAL;
3875 		goto err_put_sync;
3876 	}
3877 
3878 	pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n",
3879 		 fun_index, val);
3880 	ret = count;
3881 
3882 err_put_sync:
3883 	qm_pm_put_sync(qm);
3884 err_get_status:
3885 	clear_bit(QM_RESETTING, &qm->misc_ctl);
3886 	return ret;
3887 }
3888 
3889 static const struct file_operations qm_algqos_fops = {
3890 	.owner = THIS_MODULE,
3891 	.open = simple_open,
3892 	.read = qm_algqos_read,
3893 	.write = qm_algqos_write,
3894 };
3895 
3896 /**
3897  * hisi_qm_set_algqos_init() - Initialize function qos debugfs files.
3898  * @qm: The qm for which we want to add debugfs files.
3899  *
3900  * Create function qos debugfs files, VF ping PF to get function qos.
3901  */
hisi_qm_set_algqos_init(struct hisi_qm * qm)3902 void hisi_qm_set_algqos_init(struct hisi_qm *qm)
3903 {
3904 	if (qm->fun_type == QM_HW_PF)
3905 		debugfs_create_file("alg_qos", 0644, qm->debug.debug_root,
3906 				    qm, &qm_algqos_fops);
3907 	else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
3908 		debugfs_create_file("alg_qos", 0444, qm->debug.debug_root,
3909 				    qm, &qm_algqos_fops);
3910 }
3911 
hisi_qm_init_vf_qos(struct hisi_qm * qm,int total_func)3912 static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func)
3913 {
3914 	int i;
3915 
3916 	for (i = 1; i <= total_func; i++)
3917 		qm->factor[i].func_qos = QM_QOS_MAX_VAL;
3918 }
3919 
3920 /**
3921  * hisi_qm_sriov_enable() - enable virtual functions
3922  * @pdev: the PCIe device
3923  * @max_vfs: the number of virtual functions to enable
3924  *
3925  * Returns the number of enabled VFs. If there are VFs enabled already or
3926  * max_vfs is more than the total number of device can be enabled, returns
3927  * failure.
3928  */
hisi_qm_sriov_enable(struct pci_dev * pdev,int max_vfs)3929 int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)
3930 {
3931 	struct hisi_qm *qm = pci_get_drvdata(pdev);
3932 	int pre_existing_vfs, num_vfs, total_vfs, ret;
3933 
3934 	ret = qm_pm_get_sync(qm);
3935 	if (ret)
3936 		return ret;
3937 
3938 	total_vfs = pci_sriov_get_totalvfs(pdev);
3939 	pre_existing_vfs = pci_num_vf(pdev);
3940 	if (pre_existing_vfs) {
3941 		pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",
3942 			pre_existing_vfs);
3943 		goto err_put_sync;
3944 	}
3945 
3946 	if (max_vfs > total_vfs) {
3947 		pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs);
3948 		ret = -ERANGE;
3949 		goto err_put_sync;
3950 	}
3951 
3952 	num_vfs = max_vfs;
3953 
3954 	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
3955 		hisi_qm_init_vf_qos(qm, num_vfs);
3956 
3957 	ret = qm_vf_q_assign(qm, num_vfs);
3958 	if (ret) {
3959 		pci_err(pdev, "Can't assign queues for VF!\n");
3960 		goto err_put_sync;
3961 	}
3962 
3963 	ret = pci_enable_sriov(pdev, num_vfs);
3964 	if (ret) {
3965 		pci_err(pdev, "Can't enable VF!\n");
3966 		qm_clear_vft_config(qm);
3967 		goto err_put_sync;
3968 	}
3969 	qm->vfs_num = num_vfs;
3970 
3971 	pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);
3972 
3973 	return num_vfs;
3974 
3975 err_put_sync:
3976 	qm_pm_put_sync(qm);
3977 	return ret;
3978 }
3979 EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);
3980 
3981 /**
3982  * hisi_qm_sriov_disable - disable virtual functions
3983  * @pdev: the PCI device.
3984  * @is_frozen: true when all the VFs are frozen.
3985  *
3986  * Return failure if there are VFs assigned already or VF is in used.
3987  */
hisi_qm_sriov_disable(struct pci_dev * pdev,bool is_frozen)3988 int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)
3989 {
3990 	struct hisi_qm *qm = pci_get_drvdata(pdev);
3991 
3992 	if (pci_vfs_assigned(pdev)) {
3993 		pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");
3994 		return -EPERM;
3995 	}
3996 
3997 	/* While VF is in used, SRIOV cannot be disabled. */
3998 	if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {
3999 		pci_err(pdev, "Task is using its VF!\n");
4000 		return -EBUSY;
4001 	}
4002 
4003 	pci_disable_sriov(pdev);
4004 
4005 	qm->vfs_num = 0;
4006 	qm_pm_put_sync(qm);
4007 
4008 	return qm_clear_vft_config(qm);
4009 }
4010 EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);
4011 
4012 /**
4013  * hisi_qm_sriov_configure - configure the number of VFs
4014  * @pdev: The PCI device
4015  * @num_vfs: The number of VFs need enabled
4016  *
4017  * Enable SR-IOV according to num_vfs, 0 means disable.
4018  */
hisi_qm_sriov_configure(struct pci_dev * pdev,int num_vfs)4019 int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
4020 {
4021 	if (num_vfs == 0)
4022 		return hisi_qm_sriov_disable(pdev, false);
4023 	else
4024 		return hisi_qm_sriov_enable(pdev, num_vfs);
4025 }
4026 EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);
4027 
qm_dev_err_handle(struct hisi_qm * qm)4028 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
4029 {
4030 	if (!qm->err_ini->get_err_result) {
4031 		dev_err(&qm->pdev->dev, "Device doesn't support reset!\n");
4032 		return ACC_ERR_NONE;
4033 	}
4034 
4035 	return qm->err_ini->get_err_result(qm);
4036 }
4037 
qm_process_dev_error(struct hisi_qm * qm)4038 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)
4039 {
4040 	enum acc_err_result qm_ret, dev_ret;
4041 
4042 	/* log qm error */
4043 	qm_ret = qm_hw_error_handle(qm);
4044 
4045 	/* log device error */
4046 	dev_ret = qm_dev_err_handle(qm);
4047 
4048 	return (qm_ret == ACC_ERR_NEED_RESET ||
4049 		dev_ret == ACC_ERR_NEED_RESET) ?
4050 		ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
4051 }
4052 
4053 /**
4054  * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
4055  * @pdev: The PCI device which need report error.
4056  * @state: The connectivity between CPU and device.
4057  *
4058  * We register this function into PCIe AER handlers, It will report device or
4059  * qm hardware error status when error occur.
4060  */
hisi_qm_dev_err_detected(struct pci_dev * pdev,pci_channel_state_t state)4061 pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
4062 					  pci_channel_state_t state)
4063 {
4064 	struct hisi_qm *qm = pci_get_drvdata(pdev);
4065 	enum acc_err_result ret;
4066 
4067 	if (pdev->is_virtfn)
4068 		return PCI_ERS_RESULT_NONE;
4069 
4070 	pci_info(pdev, "PCI error detected, state(=%u)!!\n", state);
4071 	if (state == pci_channel_io_perm_failure)
4072 		return PCI_ERS_RESULT_DISCONNECT;
4073 
4074 	ret = qm_process_dev_error(qm);
4075 	if (ret == ACC_ERR_NEED_RESET)
4076 		return PCI_ERS_RESULT_NEED_RESET;
4077 
4078 	return PCI_ERS_RESULT_RECOVERED;
4079 }
4080 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);
4081 
qm_check_req_recv(struct hisi_qm * qm)4082 static int qm_check_req_recv(struct hisi_qm *qm)
4083 {
4084 	struct pci_dev *pdev = qm->pdev;
4085 	int ret;
4086 	u32 val;
4087 
4088 	if (qm->ver >= QM_HW_V3)
4089 		return 0;
4090 
4091 	writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);
4092 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4093 					 (val == ACC_VENDOR_ID_VALUE),
4094 					 POLL_PERIOD, POLL_TIMEOUT);
4095 	if (ret) {
4096 		dev_err(&pdev->dev, "Fails to read QM reg!\n");
4097 		return ret;
4098 	}
4099 
4100 	writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);
4101 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4102 					 (val == PCI_VENDOR_ID_HUAWEI),
4103 					 POLL_PERIOD, POLL_TIMEOUT);
4104 	if (ret)
4105 		dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");
4106 
4107 	return ret;
4108 }
4109 
qm_set_pf_mse(struct hisi_qm * qm,bool set)4110 static int qm_set_pf_mse(struct hisi_qm *qm, bool set)
4111 {
4112 	struct pci_dev *pdev = qm->pdev;
4113 	u16 cmd;
4114 	int i;
4115 
4116 	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4117 	if (set)
4118 		cmd |= PCI_COMMAND_MEMORY;
4119 	else
4120 		cmd &= ~PCI_COMMAND_MEMORY;
4121 
4122 	pci_write_config_word(pdev, PCI_COMMAND, cmd);
4123 	for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4124 		pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4125 		if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))
4126 			return 0;
4127 
4128 		udelay(1);
4129 	}
4130 
4131 	return -ETIMEDOUT;
4132 }
4133 
qm_set_vf_mse(struct hisi_qm * qm,bool set)4134 static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
4135 {
4136 	struct pci_dev *pdev = qm->pdev;
4137 	u16 sriov_ctrl;
4138 	int pos;
4139 	int i;
4140 
4141 	/*
4142 	 * Since function qm_set_vf_mse is called only after SRIOV is enabled,
4143 	 * pci_find_ext_capability cannot return 0, pos does not need to be
4144 	 * checked.
4145 	 */
4146 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
4147 	pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4148 	if (set)
4149 		sriov_ctrl |= PCI_SRIOV_CTRL_MSE;
4150 	else
4151 		sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;
4152 	pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);
4153 
4154 	for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4155 		pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4156 		if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>
4157 		    ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)
4158 			return 0;
4159 
4160 		udelay(1);
4161 	}
4162 
4163 	return -ETIMEDOUT;
4164 }
4165 
qm_dev_ecc_mbit_handle(struct hisi_qm * qm)4166 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)
4167 {
4168 	u32 nfe_enb = 0;
4169 
4170 	/* Kunpeng930 hardware automatically close master ooo when NFE occurs */
4171 	if (qm->ver >= QM_HW_V3)
4172 		return;
4173 
4174 	if (!qm->err_status.is_dev_ecc_mbit &&
4175 	    qm->err_status.is_qm_ecc_mbit &&
4176 	    qm->err_ini->close_axi_master_ooo) {
4177 		qm->err_ini->close_axi_master_ooo(qm);
4178 	} else if (qm->err_status.is_dev_ecc_mbit &&
4179 		   !qm->err_status.is_qm_ecc_mbit &&
4180 		   !qm->err_ini->close_axi_master_ooo) {
4181 		nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);
4182 		writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,
4183 		       qm->io_base + QM_RAS_NFE_ENABLE);
4184 		writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);
4185 	}
4186 }
4187 
qm_vf_reset_prepare(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4188 static int qm_vf_reset_prepare(struct hisi_qm *qm,
4189 			       enum qm_stop_reason stop_reason)
4190 {
4191 	struct hisi_qm_list *qm_list = qm->qm_list;
4192 	struct pci_dev *pdev = qm->pdev;
4193 	struct pci_dev *virtfn;
4194 	struct hisi_qm *vf_qm;
4195 	int ret = 0;
4196 
4197 	mutex_lock(&qm_list->lock);
4198 	list_for_each_entry(vf_qm, &qm_list->list, list) {
4199 		virtfn = vf_qm->pdev;
4200 		if (virtfn == pdev)
4201 			continue;
4202 
4203 		if (pci_physfn(virtfn) == pdev) {
4204 			/* save VFs PCIE BAR configuration */
4205 			pci_save_state(virtfn);
4206 
4207 			ret = hisi_qm_stop(vf_qm, stop_reason);
4208 			if (ret)
4209 				goto stop_fail;
4210 		}
4211 	}
4212 
4213 stop_fail:
4214 	mutex_unlock(&qm_list->lock);
4215 	return ret;
4216 }
4217 
qm_try_stop_vfs(struct hisi_qm * qm,enum qm_ifc_cmd cmd,enum qm_stop_reason stop_reason)4218 static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd,
4219 			   enum qm_stop_reason stop_reason)
4220 {
4221 	struct pci_dev *pdev = qm->pdev;
4222 	int ret;
4223 
4224 	if (!qm->vfs_num)
4225 		return 0;
4226 
4227 	/* Kunpeng930 supports to notify VFs to stop before PF reset */
4228 	if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {
4229 		ret = qm_ping_all_vfs(qm, cmd);
4230 		if (ret)
4231 			pci_err(pdev, "failed to send command to all VFs before PF reset!\n");
4232 	} else {
4233 		ret = qm_vf_reset_prepare(qm, stop_reason);
4234 		if (ret)
4235 			pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret);
4236 	}
4237 
4238 	return ret;
4239 }
4240 
qm_controller_reset_prepare(struct hisi_qm * qm)4241 static int qm_controller_reset_prepare(struct hisi_qm *qm)
4242 {
4243 	struct pci_dev *pdev = qm->pdev;
4244 	int ret;
4245 
4246 	if (qm->err_ini->set_priv_status) {
4247 		ret = qm->err_ini->set_priv_status(qm);
4248 		if (ret)
4249 			return ret;
4250 	}
4251 
4252 	ret = qm_reset_prepare_ready(qm);
4253 	if (ret) {
4254 		pci_err(pdev, "Controller reset not ready!\n");
4255 		return ret;
4256 	}
4257 
4258 	qm_dev_ecc_mbit_handle(qm);
4259 
4260 	/* PF obtains the information of VF by querying the register. */
4261 	qm_cmd_uninit(qm);
4262 
4263 	/* Whether VFs stop successfully, soft reset will continue. */
4264 	ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET);
4265 	if (ret)
4266 		pci_err(pdev, "failed to stop vfs by pf in soft reset.\n");
4267 
4268 	ret = hisi_qm_stop(qm, QM_SOFT_RESET);
4269 	if (ret) {
4270 		pci_err(pdev, "Fails to stop QM!\n");
4271 		qm_reset_bit_clear(qm);
4272 		return ret;
4273 	}
4274 
4275 	if (qm->use_sva) {
4276 		ret = qm_hw_err_isolate(qm);
4277 		if (ret)
4278 			pci_err(pdev, "failed to isolate hw err!\n");
4279 	}
4280 
4281 	ret = qm_wait_vf_prepare_finish(qm);
4282 	if (ret)
4283 		pci_err(pdev, "failed to stop by vfs in soft reset!\n");
4284 
4285 	clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4286 
4287 	return 0;
4288 }
4289 
qm_master_ooo_check(struct hisi_qm * qm)4290 static int qm_master_ooo_check(struct hisi_qm *qm)
4291 {
4292 	u32 val;
4293 	int ret;
4294 
4295 	/* Check the ooo register of the device before resetting the device. */
4296 	writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL);
4297 	ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
4298 					 val, (val == ACC_MASTER_TRANS_RETURN_RW),
4299 					 POLL_PERIOD, POLL_TIMEOUT);
4300 	if (ret)
4301 		pci_warn(qm->pdev, "Bus lock! Please reset system.\n");
4302 
4303 	return ret;
4304 }
4305 
qm_soft_reset_prepare(struct hisi_qm * qm)4306 static int qm_soft_reset_prepare(struct hisi_qm *qm)
4307 {
4308 	struct pci_dev *pdev = qm->pdev;
4309 	int ret;
4310 
4311 	/* Ensure all doorbells and mailboxes received by QM */
4312 	ret = qm_check_req_recv(qm);
4313 	if (ret)
4314 		return ret;
4315 
4316 	if (qm->vfs_num) {
4317 		ret = qm_set_vf_mse(qm, false);
4318 		if (ret) {
4319 			pci_err(pdev, "Fails to disable vf MSE bit.\n");
4320 			return ret;
4321 		}
4322 	}
4323 
4324 	ret = qm->ops->set_msi(qm, false);
4325 	if (ret) {
4326 		pci_err(pdev, "Fails to disable PEH MSI bit.\n");
4327 		return ret;
4328 	}
4329 
4330 	ret = qm_master_ooo_check(qm);
4331 	if (ret)
4332 		return ret;
4333 
4334 	if (qm->err_ini->close_sva_prefetch)
4335 		qm->err_ini->close_sva_prefetch(qm);
4336 
4337 	ret = qm_set_pf_mse(qm, false);
4338 	if (ret)
4339 		pci_err(pdev, "Fails to disable pf MSE bit.\n");
4340 
4341 	return ret;
4342 }
4343 
qm_reset_device(struct hisi_qm * qm)4344 static int qm_reset_device(struct hisi_qm *qm)
4345 {
4346 	struct pci_dev *pdev = qm->pdev;
4347 
4348 	/* The reset related sub-control registers are not in PCI BAR */
4349 	if (ACPI_HANDLE(&pdev->dev)) {
4350 		unsigned long long value = 0;
4351 		acpi_status s;
4352 
4353 		s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
4354 					  qm->err_info.acpi_rst,
4355 					  NULL, &value);
4356 		if (ACPI_FAILURE(s)) {
4357 			pci_err(pdev, "NO controller reset method!\n");
4358 			return -EIO;
4359 		}
4360 
4361 		if (value) {
4362 			pci_err(pdev, "Reset step %llu failed!\n", value);
4363 			return -EIO;
4364 		}
4365 
4366 		return 0;
4367 	}
4368 
4369 	pci_err(pdev, "No reset method!\n");
4370 	return -EINVAL;
4371 }
4372 
qm_soft_reset(struct hisi_qm * qm)4373 static int qm_soft_reset(struct hisi_qm *qm)
4374 {
4375 	int ret;
4376 
4377 	ret = qm_soft_reset_prepare(qm);
4378 	if (ret)
4379 		return ret;
4380 
4381 	return qm_reset_device(qm);
4382 }
4383 
qm_vf_reset_done(struct hisi_qm * qm)4384 static int qm_vf_reset_done(struct hisi_qm *qm)
4385 {
4386 	struct hisi_qm_list *qm_list = qm->qm_list;
4387 	struct pci_dev *pdev = qm->pdev;
4388 	struct pci_dev *virtfn;
4389 	struct hisi_qm *vf_qm;
4390 	int ret = 0;
4391 
4392 	mutex_lock(&qm_list->lock);
4393 	list_for_each_entry(vf_qm, &qm_list->list, list) {
4394 		virtfn = vf_qm->pdev;
4395 		if (virtfn == pdev)
4396 			continue;
4397 
4398 		if (pci_physfn(virtfn) == pdev) {
4399 			/* enable VFs PCIE BAR configuration */
4400 			pci_restore_state(virtfn);
4401 
4402 			ret = qm_restart(vf_qm);
4403 			if (ret)
4404 				goto restart_fail;
4405 		}
4406 	}
4407 
4408 restart_fail:
4409 	mutex_unlock(&qm_list->lock);
4410 	return ret;
4411 }
4412 
qm_try_start_vfs(struct hisi_qm * qm,enum qm_ifc_cmd cmd)4413 static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd)
4414 {
4415 	struct pci_dev *pdev = qm->pdev;
4416 	int ret;
4417 
4418 	if (!qm->vfs_num)
4419 		return 0;
4420 
4421 	ret = qm_vf_q_assign(qm, qm->vfs_num);
4422 	if (ret) {
4423 		pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret);
4424 		return ret;
4425 	}
4426 
4427 	/* Kunpeng930 supports to notify VFs to start after PF reset. */
4428 	if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {
4429 		ret = qm_ping_all_vfs(qm, cmd);
4430 		if (ret)
4431 			pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n");
4432 	} else {
4433 		ret = qm_vf_reset_done(qm);
4434 		if (ret)
4435 			pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret);
4436 	}
4437 
4438 	return ret;
4439 }
4440 
qm_dev_hw_init(struct hisi_qm * qm)4441 static int qm_dev_hw_init(struct hisi_qm *qm)
4442 {
4443 	return qm->err_ini->hw_init(qm);
4444 }
4445 
qm_restart_prepare(struct hisi_qm * qm)4446 static void qm_restart_prepare(struct hisi_qm *qm)
4447 {
4448 	u32 value;
4449 
4450 	if (qm->err_ini->open_sva_prefetch)
4451 		qm->err_ini->open_sva_prefetch(qm);
4452 
4453 	if (qm->ver >= QM_HW_V3)
4454 		return;
4455 
4456 	if (!qm->err_status.is_qm_ecc_mbit &&
4457 	    !qm->err_status.is_dev_ecc_mbit)
4458 		return;
4459 
4460 	/* temporarily close the OOO port used for PEH to write out MSI */
4461 	value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4462 	writel(value & ~qm->err_info.msi_wr_port,
4463 	       qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4464 
4465 	/* clear dev ecc 2bit error source if having */
4466 	value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;
4467 	if (value && qm->err_ini->clear_dev_hw_err_status)
4468 		qm->err_ini->clear_dev_hw_err_status(qm, value);
4469 
4470 	/* clear QM ecc mbit error source */
4471 	writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);
4472 
4473 	/* clear AM Reorder Buffer ecc mbit source */
4474 	writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);
4475 }
4476 
qm_restart_done(struct hisi_qm * qm)4477 static void qm_restart_done(struct hisi_qm *qm)
4478 {
4479 	u32 value;
4480 
4481 	if (qm->ver >= QM_HW_V3)
4482 		goto clear_flags;
4483 
4484 	if (!qm->err_status.is_qm_ecc_mbit &&
4485 	    !qm->err_status.is_dev_ecc_mbit)
4486 		return;
4487 
4488 	/* open the OOO port for PEH to write out MSI */
4489 	value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4490 	value |= qm->err_info.msi_wr_port;
4491 	writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4492 
4493 clear_flags:
4494 	qm->err_status.is_qm_ecc_mbit = false;
4495 	qm->err_status.is_dev_ecc_mbit = false;
4496 }
4497 
qm_controller_reset_done(struct hisi_qm * qm)4498 static int qm_controller_reset_done(struct hisi_qm *qm)
4499 {
4500 	struct pci_dev *pdev = qm->pdev;
4501 	int ret;
4502 
4503 	ret = qm->ops->set_msi(qm, true);
4504 	if (ret) {
4505 		pci_err(pdev, "Fails to enable PEH MSI bit!\n");
4506 		return ret;
4507 	}
4508 
4509 	ret = qm_set_pf_mse(qm, true);
4510 	if (ret) {
4511 		pci_err(pdev, "Fails to enable pf MSE bit!\n");
4512 		return ret;
4513 	}
4514 
4515 	if (qm->vfs_num) {
4516 		ret = qm_set_vf_mse(qm, true);
4517 		if (ret) {
4518 			pci_err(pdev, "Fails to enable vf MSE bit!\n");
4519 			return ret;
4520 		}
4521 	}
4522 
4523 	ret = qm_dev_hw_init(qm);
4524 	if (ret) {
4525 		pci_err(pdev, "Failed to init device\n");
4526 		return ret;
4527 	}
4528 
4529 	qm_restart_prepare(qm);
4530 	hisi_qm_dev_err_init(qm);
4531 	if (qm->err_ini->open_axi_master_ooo)
4532 		qm->err_ini->open_axi_master_ooo(qm);
4533 
4534 	ret = qm_dev_mem_reset(qm);
4535 	if (ret) {
4536 		pci_err(pdev, "failed to reset device memory\n");
4537 		return ret;
4538 	}
4539 
4540 	ret = qm_restart(qm);
4541 	if (ret) {
4542 		pci_err(pdev, "Failed to start QM!\n");
4543 		return ret;
4544 	}
4545 
4546 	ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
4547 	if (ret)
4548 		pci_err(pdev, "failed to start vfs by pf in soft reset.\n");
4549 
4550 	ret = qm_wait_vf_prepare_finish(qm);
4551 	if (ret)
4552 		pci_err(pdev, "failed to start by vfs in soft reset!\n");
4553 
4554 	qm_cmd_init(qm);
4555 	qm_restart_done(qm);
4556 
4557 	qm_reset_bit_clear(qm);
4558 
4559 	return 0;
4560 }
4561 
qm_controller_reset(struct hisi_qm * qm)4562 static int qm_controller_reset(struct hisi_qm *qm)
4563 {
4564 	struct pci_dev *pdev = qm->pdev;
4565 	int ret;
4566 
4567 	pci_info(pdev, "Controller resetting...\n");
4568 
4569 	ret = qm_controller_reset_prepare(qm);
4570 	if (ret) {
4571 		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4572 		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4573 		clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4574 		return ret;
4575 	}
4576 
4577 	hisi_qm_show_last_dfx_regs(qm);
4578 	if (qm->err_ini->show_last_dfx_regs)
4579 		qm->err_ini->show_last_dfx_regs(qm);
4580 
4581 	ret = qm_soft_reset(qm);
4582 	if (ret)
4583 		goto err_reset;
4584 
4585 	ret = qm_controller_reset_done(qm);
4586 	if (ret)
4587 		goto err_reset;
4588 
4589 	pci_info(pdev, "Controller reset complete\n");
4590 
4591 	return 0;
4592 
4593 err_reset:
4594 	pci_err(pdev, "Controller reset failed (%d)\n", ret);
4595 	qm_reset_bit_clear(qm);
4596 
4597 	/* if resetting fails, isolate the device */
4598 	if (qm->use_sva)
4599 		qm->isolate_data.is_isolate = true;
4600 	return ret;
4601 }
4602 
4603 /**
4604  * hisi_qm_dev_slot_reset() - slot reset
4605  * @pdev: the PCIe device
4606  *
4607  * This function offers QM relate PCIe device reset interface. Drivers which
4608  * use QM can use this function as slot_reset in its struct pci_error_handlers.
4609  */
hisi_qm_dev_slot_reset(struct pci_dev * pdev)4610 pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)
4611 {
4612 	struct hisi_qm *qm = pci_get_drvdata(pdev);
4613 	int ret;
4614 
4615 	if (pdev->is_virtfn)
4616 		return PCI_ERS_RESULT_RECOVERED;
4617 
4618 	/* reset pcie device controller */
4619 	ret = qm_controller_reset(qm);
4620 	if (ret) {
4621 		pci_err(pdev, "Controller reset failed (%d)\n", ret);
4622 		return PCI_ERS_RESULT_DISCONNECT;
4623 	}
4624 
4625 	return PCI_ERS_RESULT_RECOVERED;
4626 }
4627 EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);
4628 
hisi_qm_reset_prepare(struct pci_dev * pdev)4629 void hisi_qm_reset_prepare(struct pci_dev *pdev)
4630 {
4631 	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4632 	struct hisi_qm *qm = pci_get_drvdata(pdev);
4633 	u32 delay = 0;
4634 	int ret;
4635 
4636 	hisi_qm_dev_err_uninit(pf_qm);
4637 
4638 	/*
4639 	 * Check whether there is an ECC mbit error, If it occurs, need to
4640 	 * wait for soft reset to fix it.
4641 	 */
4642 	while (qm_check_dev_error(qm)) {
4643 		msleep(++delay);
4644 		if (delay > QM_RESET_WAIT_TIMEOUT)
4645 			return;
4646 	}
4647 
4648 	ret = qm_reset_prepare_ready(qm);
4649 	if (ret) {
4650 		pci_err(pdev, "FLR not ready!\n");
4651 		return;
4652 	}
4653 
4654 	/* PF obtains the information of VF by querying the register. */
4655 	if (qm->fun_type == QM_HW_PF)
4656 		qm_cmd_uninit(qm);
4657 
4658 	ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN);
4659 	if (ret)
4660 		pci_err(pdev, "failed to stop vfs by pf in FLR.\n");
4661 
4662 	ret = hisi_qm_stop(qm, QM_DOWN);
4663 	if (ret) {
4664 		pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
4665 		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4666 		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4667 		return;
4668 	}
4669 
4670 	ret = qm_wait_vf_prepare_finish(qm);
4671 	if (ret)
4672 		pci_err(pdev, "failed to stop by vfs in FLR!\n");
4673 
4674 	pci_info(pdev, "FLR resetting...\n");
4675 }
4676 EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);
4677 
qm_flr_reset_complete(struct pci_dev * pdev)4678 static bool qm_flr_reset_complete(struct pci_dev *pdev)
4679 {
4680 	struct pci_dev *pf_pdev = pci_physfn(pdev);
4681 	struct hisi_qm *qm = pci_get_drvdata(pf_pdev);
4682 	u32 id;
4683 
4684 	pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);
4685 	if (id == QM_PCI_COMMAND_INVALID) {
4686 		pci_err(pdev, "Device can not be used!\n");
4687 		return false;
4688 	}
4689 
4690 	return true;
4691 }
4692 
hisi_qm_reset_done(struct pci_dev * pdev)4693 void hisi_qm_reset_done(struct pci_dev *pdev)
4694 {
4695 	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4696 	struct hisi_qm *qm = pci_get_drvdata(pdev);
4697 	int ret;
4698 
4699 	if (qm->fun_type == QM_HW_PF) {
4700 		ret = qm_dev_hw_init(qm);
4701 		if (ret) {
4702 			pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);
4703 			goto flr_done;
4704 		}
4705 	}
4706 
4707 	hisi_qm_dev_err_init(pf_qm);
4708 
4709 	ret = qm_restart(qm);
4710 	if (ret) {
4711 		pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);
4712 		goto flr_done;
4713 	}
4714 
4715 	ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
4716 	if (ret)
4717 		pci_err(pdev, "failed to start vfs by pf in FLR.\n");
4718 
4719 	ret = qm_wait_vf_prepare_finish(qm);
4720 	if (ret)
4721 		pci_err(pdev, "failed to start by vfs in FLR!\n");
4722 
4723 flr_done:
4724 	if (qm->fun_type == QM_HW_PF)
4725 		qm_cmd_init(qm);
4726 
4727 	if (qm_flr_reset_complete(pdev))
4728 		pci_info(pdev, "FLR reset complete\n");
4729 
4730 	qm_reset_bit_clear(qm);
4731 }
4732 EXPORT_SYMBOL_GPL(hisi_qm_reset_done);
4733 
qm_abnormal_irq(int irq,void * data)4734 static irqreturn_t qm_abnormal_irq(int irq, void *data)
4735 {
4736 	struct hisi_qm *qm = data;
4737 	enum acc_err_result ret;
4738 
4739 	atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);
4740 	ret = qm_process_dev_error(qm);
4741 	if (ret == ACC_ERR_NEED_RESET &&
4742 	    !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) &&
4743 	    !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl))
4744 		schedule_work(&qm->rst_work);
4745 
4746 	return IRQ_HANDLED;
4747 }
4748 
4749 /**
4750  * hisi_qm_dev_shutdown() - Shutdown device.
4751  * @pdev: The device will be shutdown.
4752  *
4753  * This function will stop qm when OS shutdown or rebooting.
4754  */
hisi_qm_dev_shutdown(struct pci_dev * pdev)4755 void hisi_qm_dev_shutdown(struct pci_dev *pdev)
4756 {
4757 	struct hisi_qm *qm = pci_get_drvdata(pdev);
4758 	int ret;
4759 
4760 	ret = hisi_qm_stop(qm, QM_DOWN);
4761 	if (ret)
4762 		dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");
4763 
4764 	hisi_qm_cache_wb(qm);
4765 }
4766 EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);
4767 
hisi_qm_controller_reset(struct work_struct * rst_work)4768 static void hisi_qm_controller_reset(struct work_struct *rst_work)
4769 {
4770 	struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
4771 	int ret;
4772 
4773 	ret = qm_pm_get_sync(qm);
4774 	if (ret) {
4775 		clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4776 		return;
4777 	}
4778 
4779 	/* reset pcie device controller */
4780 	ret = qm_controller_reset(qm);
4781 	if (ret)
4782 		dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);
4783 
4784 	qm_pm_put_sync(qm);
4785 }
4786 
qm_pf_reset_vf_prepare(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4787 static void qm_pf_reset_vf_prepare(struct hisi_qm *qm,
4788 				   enum qm_stop_reason stop_reason)
4789 {
4790 	enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE;
4791 	struct pci_dev *pdev = qm->pdev;
4792 	int ret;
4793 
4794 	ret = qm_reset_prepare_ready(qm);
4795 	if (ret) {
4796 		dev_err(&pdev->dev, "reset prepare not ready!\n");
4797 		atomic_set(&qm->status.flags, QM_STOP);
4798 		cmd = QM_VF_PREPARE_FAIL;
4799 		goto err_prepare;
4800 	}
4801 
4802 	ret = hisi_qm_stop(qm, stop_reason);
4803 	if (ret) {
4804 		dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret);
4805 		atomic_set(&qm->status.flags, QM_STOP);
4806 		cmd = QM_VF_PREPARE_FAIL;
4807 		goto err_prepare;
4808 	} else {
4809 		goto out;
4810 	}
4811 
4812 err_prepare:
4813 	hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4814 	hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4815 out:
4816 	pci_save_state(pdev);
4817 	ret = qm_ping_pf(qm, cmd);
4818 	if (ret)
4819 		dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n");
4820 }
4821 
qm_pf_reset_vf_done(struct hisi_qm * qm)4822 static void qm_pf_reset_vf_done(struct hisi_qm *qm)
4823 {
4824 	enum qm_ifc_cmd cmd = QM_VF_START_DONE;
4825 	struct pci_dev *pdev = qm->pdev;
4826 	int ret;
4827 
4828 	pci_restore_state(pdev);
4829 	ret = hisi_qm_start(qm);
4830 	if (ret) {
4831 		dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret);
4832 		cmd = QM_VF_START_FAIL;
4833 	}
4834 
4835 	qm_cmd_init(qm);
4836 	ret = qm_ping_pf(qm, cmd);
4837 	if (ret)
4838 		dev_warn(&pdev->dev, "PF responds timeout in reset done!\n");
4839 
4840 	qm_reset_bit_clear(qm);
4841 }
4842 
qm_wait_pf_reset_finish(struct hisi_qm * qm)4843 static int qm_wait_pf_reset_finish(struct hisi_qm *qm)
4844 {
4845 	struct device *dev = &qm->pdev->dev;
4846 	u32 val, cmd;
4847 	int ret;
4848 
4849 	/* Wait for reset to finish */
4850 	ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val,
4851 					 val == BIT(0), QM_VF_RESET_WAIT_US,
4852 					 QM_VF_RESET_WAIT_TIMEOUT_US);
4853 	/* hardware completion status should be available by this time */
4854 	if (ret) {
4855 		dev_err(dev, "couldn't get reset done status from PF, timeout!\n");
4856 		return -ETIMEDOUT;
4857 	}
4858 
4859 	/*
4860 	 * Whether message is got successfully,
4861 	 * VF needs to ack PF by clearing the interrupt.
4862 	 */
4863 	ret = qm->ops->get_ifc(qm, &cmd, NULL, 0);
4864 	qm_clear_cmd_interrupt(qm, 0);
4865 	if (ret) {
4866 		dev_err(dev, "failed to get command from PF in reset done!\n");
4867 		return ret;
4868 	}
4869 
4870 	if (cmd != QM_PF_RESET_DONE) {
4871 		dev_err(dev, "the command(0x%x) is not reset done!\n", cmd);
4872 		ret = -EINVAL;
4873 	}
4874 
4875 	return ret;
4876 }
4877 
qm_pf_reset_vf_process(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4878 static void qm_pf_reset_vf_process(struct hisi_qm *qm,
4879 				   enum qm_stop_reason stop_reason)
4880 {
4881 	struct device *dev = &qm->pdev->dev;
4882 	int ret;
4883 
4884 	dev_info(dev, "device reset start...\n");
4885 
4886 	/* The message is obtained by querying the register during resetting */
4887 	qm_cmd_uninit(qm);
4888 	qm_pf_reset_vf_prepare(qm, stop_reason);
4889 
4890 	ret = qm_wait_pf_reset_finish(qm);
4891 	if (ret)
4892 		goto err_get_status;
4893 
4894 	qm_pf_reset_vf_done(qm);
4895 
4896 	dev_info(dev, "device reset done.\n");
4897 
4898 	return;
4899 
4900 err_get_status:
4901 	qm_cmd_init(qm);
4902 	qm_reset_bit_clear(qm);
4903 }
4904 
qm_handle_cmd_msg(struct hisi_qm * qm,u32 fun_num)4905 static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num)
4906 {
4907 	struct device *dev = &qm->pdev->dev;
4908 	enum qm_ifc_cmd cmd;
4909 	u32 data;
4910 	int ret;
4911 
4912 	/*
4913 	 * Get the msg from source by sending mailbox. Whether message is got
4914 	 * successfully, destination needs to ack source by clearing the interrupt.
4915 	 */
4916 	ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num);
4917 	qm_clear_cmd_interrupt(qm, BIT(fun_num));
4918 	if (ret) {
4919 		dev_err(dev, "failed to get command from source!\n");
4920 		return;
4921 	}
4922 
4923 	switch (cmd) {
4924 	case QM_PF_FLR_PREPARE:
4925 		qm_pf_reset_vf_process(qm, QM_DOWN);
4926 		break;
4927 	case QM_PF_SRST_PREPARE:
4928 		qm_pf_reset_vf_process(qm, QM_SOFT_RESET);
4929 		break;
4930 	case QM_VF_GET_QOS:
4931 		qm_vf_get_qos(qm, fun_num);
4932 		break;
4933 	case QM_PF_SET_QOS:
4934 		qm->mb_qos = data;
4935 		break;
4936 	default:
4937 		dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num);
4938 		break;
4939 	}
4940 }
4941 
qm_cmd_process(struct work_struct * cmd_process)4942 static void qm_cmd_process(struct work_struct *cmd_process)
4943 {
4944 	struct hisi_qm *qm = container_of(cmd_process,
4945 					struct hisi_qm, cmd_process);
4946 	u32 vfs_num = qm->vfs_num;
4947 	u64 val;
4948 	u32 i;
4949 
4950 	if (qm->fun_type == QM_HW_PF) {
4951 		val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
4952 		if (!val)
4953 			return;
4954 
4955 		for (i = 1; i <= vfs_num; i++) {
4956 			if (val & BIT(i))
4957 				qm_handle_cmd_msg(qm, i);
4958 		}
4959 
4960 		return;
4961 	}
4962 
4963 	qm_handle_cmd_msg(qm, 0);
4964 }
4965 
4966 /**
4967  * hisi_qm_alg_register() - Register alg to crypto.
4968  * @qm: The qm needs add.
4969  * @qm_list: The qm list.
4970  * @guard: Guard of qp_num.
4971  *
4972  * Register algorithm to crypto when the function is satisfy guard.
4973  */
hisi_qm_alg_register(struct hisi_qm * qm,struct hisi_qm_list * qm_list,int guard)4974 int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard)
4975 {
4976 	struct device *dev = &qm->pdev->dev;
4977 
4978 	if (qm->ver <= QM_HW_V2 && qm->use_sva) {
4979 		dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n");
4980 		return 0;
4981 	}
4982 
4983 	if (qm->qp_num < guard) {
4984 		dev_info(dev, "qp_num is less than task need.\n");
4985 		return 0;
4986 	}
4987 
4988 	return qm_list->register_to_crypto(qm);
4989 }
4990 EXPORT_SYMBOL_GPL(hisi_qm_alg_register);
4991 
4992 /**
4993  * hisi_qm_alg_unregister() - Unregister alg from crypto.
4994  * @qm: The qm needs delete.
4995  * @qm_list: The qm list.
4996  * @guard: Guard of qp_num.
4997  *
4998  * Unregister algorithm from crypto when the last function is satisfy guard.
4999  */
hisi_qm_alg_unregister(struct hisi_qm * qm,struct hisi_qm_list * qm_list,int guard)5000 void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard)
5001 {
5002 	if (qm->ver <= QM_HW_V2 && qm->use_sva)
5003 		return;
5004 
5005 	if (qm->qp_num < guard)
5006 		return;
5007 
5008 	qm_list->unregister_from_crypto(qm);
5009 }
5010 EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);
5011 
qm_unregister_abnormal_irq(struct hisi_qm * qm)5012 static void qm_unregister_abnormal_irq(struct hisi_qm *qm)
5013 {
5014 	struct pci_dev *pdev = qm->pdev;
5015 	u32 irq_vector, val;
5016 
5017 	if (qm->fun_type == QM_HW_VF)
5018 		return;
5019 
5020 	val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val;
5021 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
5022 		return;
5023 
5024 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5025 	free_irq(pci_irq_vector(pdev, irq_vector), qm);
5026 }
5027 
qm_register_abnormal_irq(struct hisi_qm * qm)5028 static int qm_register_abnormal_irq(struct hisi_qm *qm)
5029 {
5030 	struct pci_dev *pdev = qm->pdev;
5031 	u32 irq_vector, val;
5032 	int ret;
5033 
5034 	if (qm->fun_type == QM_HW_VF)
5035 		return 0;
5036 
5037 	val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val;
5038 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
5039 		return 0;
5040 
5041 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5042 	ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm);
5043 	if (ret)
5044 		dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret);
5045 
5046 	return ret;
5047 }
5048 
qm_unregister_mb_cmd_irq(struct hisi_qm * qm)5049 static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm)
5050 {
5051 	struct pci_dev *pdev = qm->pdev;
5052 	u32 irq_vector, val;
5053 
5054 	val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val;
5055 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5056 		return;
5057 
5058 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5059 	free_irq(pci_irq_vector(pdev, irq_vector), qm);
5060 }
5061 
qm_register_mb_cmd_irq(struct hisi_qm * qm)5062 static int qm_register_mb_cmd_irq(struct hisi_qm *qm)
5063 {
5064 	struct pci_dev *pdev = qm->pdev;
5065 	u32 irq_vector, val;
5066 	int ret;
5067 
5068 	val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val;
5069 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5070 		return 0;
5071 
5072 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5073 	ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm);
5074 	if (ret)
5075 		dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret);
5076 
5077 	return ret;
5078 }
5079 
qm_unregister_aeq_irq(struct hisi_qm * qm)5080 static void qm_unregister_aeq_irq(struct hisi_qm *qm)
5081 {
5082 	struct pci_dev *pdev = qm->pdev;
5083 	u32 irq_vector, val;
5084 
5085 	val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val;
5086 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5087 		return;
5088 
5089 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5090 	free_irq(pci_irq_vector(pdev, irq_vector), qm);
5091 }
5092 
qm_register_aeq_irq(struct hisi_qm * qm)5093 static int qm_register_aeq_irq(struct hisi_qm *qm)
5094 {
5095 	struct pci_dev *pdev = qm->pdev;
5096 	u32 irq_vector, val;
5097 	int ret;
5098 
5099 	val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val;
5100 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5101 		return 0;
5102 
5103 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5104 	ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL,
5105 						   qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm);
5106 	if (ret)
5107 		dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);
5108 
5109 	return ret;
5110 }
5111 
qm_unregister_eq_irq(struct hisi_qm * qm)5112 static void qm_unregister_eq_irq(struct hisi_qm *qm)
5113 {
5114 	struct pci_dev *pdev = qm->pdev;
5115 	u32 irq_vector, val;
5116 
5117 	val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val;
5118 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5119 		return;
5120 
5121 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5122 	free_irq(pci_irq_vector(pdev, irq_vector), qm);
5123 }
5124 
qm_register_eq_irq(struct hisi_qm * qm)5125 static int qm_register_eq_irq(struct hisi_qm *qm)
5126 {
5127 	struct pci_dev *pdev = qm->pdev;
5128 	u32 irq_vector, val;
5129 	int ret;
5130 
5131 	val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val;
5132 	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5133 		return 0;
5134 
5135 	irq_vector = val & QM_IRQ_VECTOR_MASK;
5136 	ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm);
5137 	if (ret)
5138 		dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);
5139 
5140 	return ret;
5141 }
5142 
qm_irqs_unregister(struct hisi_qm * qm)5143 static void qm_irqs_unregister(struct hisi_qm *qm)
5144 {
5145 	qm_unregister_mb_cmd_irq(qm);
5146 	qm_unregister_abnormal_irq(qm);
5147 	qm_unregister_aeq_irq(qm);
5148 	qm_unregister_eq_irq(qm);
5149 }
5150 
qm_irqs_register(struct hisi_qm * qm)5151 static int qm_irqs_register(struct hisi_qm *qm)
5152 {
5153 	int ret;
5154 
5155 	ret = qm_register_eq_irq(qm);
5156 	if (ret)
5157 		return ret;
5158 
5159 	ret = qm_register_aeq_irq(qm);
5160 	if (ret)
5161 		goto free_eq_irq;
5162 
5163 	ret = qm_register_abnormal_irq(qm);
5164 	if (ret)
5165 		goto free_aeq_irq;
5166 
5167 	ret = qm_register_mb_cmd_irq(qm);
5168 	if (ret)
5169 		goto free_abnormal_irq;
5170 
5171 	return 0;
5172 
5173 free_abnormal_irq:
5174 	qm_unregister_abnormal_irq(qm);
5175 free_aeq_irq:
5176 	qm_unregister_aeq_irq(qm);
5177 free_eq_irq:
5178 	qm_unregister_eq_irq(qm);
5179 	return ret;
5180 }
5181 
qm_get_qp_num(struct hisi_qm * qm)5182 static int qm_get_qp_num(struct hisi_qm *qm)
5183 {
5184 	struct device *dev = &qm->pdev->dev;
5185 	bool is_db_isolation;
5186 
5187 	/* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */
5188 	if (qm->fun_type == QM_HW_VF) {
5189 		if (qm->ver != QM_HW_V1)
5190 			/* v2 starts to support get vft by mailbox */
5191 			return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
5192 
5193 		return 0;
5194 	}
5195 
5196 	is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);
5197 	qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true);
5198 	qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info,
5199 					     QM_FUNC_MAX_QP_CAP, is_db_isolation);
5200 
5201 	if (qm->qp_num <= qm->max_qp_num)
5202 		return 0;
5203 
5204 	if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) {
5205 		/* Check whether the set qp number is valid */
5206 		dev_err(dev, "qp num(%u) is more than max qp num(%u)!\n",
5207 			qm->qp_num, qm->max_qp_num);
5208 		return -EINVAL;
5209 	}
5210 
5211 	dev_info(dev, "Default qp num(%u) is too big, reset it to Function's max qp num(%u)!\n",
5212 		 qm->qp_num, qm->max_qp_num);
5213 	qm->qp_num = qm->max_qp_num;
5214 	qm->debug.curr_qm_qp_num = qm->qp_num;
5215 
5216 	return 0;
5217 }
5218 
qm_pre_store_caps(struct hisi_qm * qm)5219 static int qm_pre_store_caps(struct hisi_qm *qm)
5220 {
5221 	struct hisi_qm_cap_record *qm_cap;
5222 	struct pci_dev *pdev = qm->pdev;
5223 	size_t i, size;
5224 
5225 	size = ARRAY_SIZE(qm_cap_query_info);
5226 	qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL);
5227 	if (!qm_cap)
5228 		return -ENOMEM;
5229 
5230 	for (i = 0; i < size; i++) {
5231 		qm_cap[i].type = qm_cap_query_info[i].type;
5232 		qm_cap[i].name = qm_cap_query_info[i].name;
5233 		qm_cap[i].cap_val = hisi_qm_get_cap_value(qm, qm_cap_query_info,
5234 							i, qm->cap_ver);
5235 	}
5236 
5237 	qm->cap_tables.qm_cap_table = qm_cap;
5238 	qm->cap_tables.qm_cap_size = size;
5239 
5240 	return 0;
5241 }
5242 
qm_get_hw_caps(struct hisi_qm * qm)5243 static int qm_get_hw_caps(struct hisi_qm *qm)
5244 {
5245 	const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ?
5246 						  qm_cap_info_pf : qm_cap_info_vf;
5247 	u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) :
5248 				   ARRAY_SIZE(qm_cap_info_vf);
5249 	u32 val, i;
5250 
5251 	/* Doorbell isolate register is a independent register. */
5252 	val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true);
5253 	if (val)
5254 		set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);
5255 
5256 	if (qm->ver >= QM_HW_V3) {
5257 		val = readl(qm->io_base + QM_FUNC_CAPS_REG);
5258 		qm->cap_ver = val & QM_CAPBILITY_VERSION;
5259 	}
5260 
5261 	/* Get PF/VF common capbility */
5262 	for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) {
5263 		val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver);
5264 		if (val)
5265 			set_bit(qm_cap_info_comm[i].type, &qm->caps);
5266 	}
5267 
5268 	/* Get PF/VF different capbility */
5269 	for (i = 0; i < size; i++) {
5270 		val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver);
5271 		if (val)
5272 			set_bit(cap_info[i].type, &qm->caps);
5273 	}
5274 
5275 	/* Fetch and save the value of qm capability registers */
5276 	return qm_pre_store_caps(qm);
5277 }
5278 
qm_get_version(struct hisi_qm * qm)5279 static void qm_get_version(struct hisi_qm *qm)
5280 {
5281 	struct pci_dev *pdev = qm->pdev;
5282 	u32 sub_version_id;
5283 
5284 	qm->ver = pdev->revision;
5285 
5286 	if (pdev->revision == QM_HW_V3) {
5287 		sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID);
5288 		if (sub_version_id)
5289 			qm->ver = sub_version_id;
5290 	}
5291 }
5292 
qm_get_pci_res(struct hisi_qm * qm)5293 static int qm_get_pci_res(struct hisi_qm *qm)
5294 {
5295 	struct pci_dev *pdev = qm->pdev;
5296 	struct device *dev = &pdev->dev;
5297 	int ret;
5298 
5299 	ret = pci_request_mem_regions(pdev, qm->dev_name);
5300 	if (ret < 0) {
5301 		dev_err(dev, "Failed to request mem regions!\n");
5302 		return ret;
5303 	}
5304 
5305 	qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);
5306 	qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2));
5307 	if (!qm->io_base) {
5308 		ret = -EIO;
5309 		goto err_request_mem_regions;
5310 	}
5311 
5312 	qm_get_version(qm);
5313 
5314 	ret = qm_get_hw_caps(qm);
5315 	if (ret)
5316 		goto err_ioremap;
5317 
5318 	if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {
5319 		qm->db_interval = QM_QP_DB_INTERVAL;
5320 		qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);
5321 		qm->db_io_base = ioremap(qm->db_phys_base,
5322 					 pci_resource_len(pdev, PCI_BAR_4));
5323 		if (!qm->db_io_base) {
5324 			ret = -EIO;
5325 			goto err_ioremap;
5326 		}
5327 	} else {
5328 		qm->db_phys_base = qm->phys_base;
5329 		qm->db_io_base = qm->io_base;
5330 		qm->db_interval = 0;
5331 	}
5332 
5333 	hisi_qm_pre_init(qm);
5334 	ret = qm_get_qp_num(qm);
5335 	if (ret)
5336 		goto err_db_ioremap;
5337 
5338 	return 0;
5339 
5340 err_db_ioremap:
5341 	if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
5342 		iounmap(qm->db_io_base);
5343 err_ioremap:
5344 	iounmap(qm->io_base);
5345 err_request_mem_regions:
5346 	pci_release_mem_regions(pdev);
5347 	return ret;
5348 }
5349 
qm_clear_device(struct hisi_qm * qm)5350 static int qm_clear_device(struct hisi_qm *qm)
5351 {
5352 	acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev);
5353 	int ret;
5354 
5355 	if (qm->fun_type == QM_HW_VF)
5356 		return 0;
5357 
5358 	/* Device does not support reset, return */
5359 	if (!qm->err_ini->err_info_init)
5360 		return 0;
5361 	qm->err_ini->err_info_init(qm);
5362 
5363 	if (!handle)
5364 		return 0;
5365 
5366 	/* No reset method, return */
5367 	if (!acpi_has_method(handle, qm->err_info.acpi_rst))
5368 		return 0;
5369 
5370 	ret = qm_master_ooo_check(qm);
5371 	if (ret) {
5372 		writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL);
5373 		return ret;
5374 	}
5375 
5376 	if (qm->err_ini->set_priv_status) {
5377 		ret = qm->err_ini->set_priv_status(qm);
5378 		if (ret) {
5379 			writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL);
5380 			return ret;
5381 		}
5382 	}
5383 
5384 	return qm_reset_device(qm);
5385 }
5386 
hisi_qm_pci_init(struct hisi_qm * qm)5387 static int hisi_qm_pci_init(struct hisi_qm *qm)
5388 {
5389 	struct pci_dev *pdev = qm->pdev;
5390 	struct device *dev = &pdev->dev;
5391 	unsigned int num_vec;
5392 	int ret;
5393 
5394 	ret = pci_enable_device_mem(pdev);
5395 	if (ret < 0) {
5396 		dev_err(dev, "Failed to enable device mem!\n");
5397 		return ret;
5398 	}
5399 
5400 	ret = qm_get_pci_res(qm);
5401 	if (ret)
5402 		goto err_disable_pcidev;
5403 
5404 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
5405 	if (ret < 0)
5406 		goto err_get_pci_res;
5407 	pci_set_master(pdev);
5408 
5409 	num_vec = qm_get_irq_num(qm);
5410 	ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
5411 	if (ret < 0) {
5412 		dev_err(dev, "Failed to enable MSI vectors!\n");
5413 		goto err_get_pci_res;
5414 	}
5415 
5416 	ret = qm_clear_device(qm);
5417 	if (ret)
5418 		goto err_free_vectors;
5419 
5420 	return 0;
5421 
5422 err_free_vectors:
5423 	pci_free_irq_vectors(pdev);
5424 err_get_pci_res:
5425 	qm_put_pci_res(qm);
5426 err_disable_pcidev:
5427 	pci_disable_device(pdev);
5428 	return ret;
5429 }
5430 
hisi_qm_init_work(struct hisi_qm * qm)5431 static int hisi_qm_init_work(struct hisi_qm *qm)
5432 {
5433 	int i;
5434 
5435 	for (i = 0; i < qm->qp_num; i++)
5436 		INIT_WORK(&qm->poll_data[i].work, qm_work_process);
5437 
5438 	if (qm->fun_type == QM_HW_PF)
5439 		INIT_WORK(&qm->rst_work, hisi_qm_controller_reset);
5440 
5441 	if (qm->ver > QM_HW_V2)
5442 		INIT_WORK(&qm->cmd_process, qm_cmd_process);
5443 
5444 	qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM |
5445 				 WQ_UNBOUND, num_online_cpus(),
5446 				 pci_name(qm->pdev));
5447 	if (!qm->wq) {
5448 		pci_err(qm->pdev, "failed to alloc workqueue!\n");
5449 		return -ENOMEM;
5450 	}
5451 
5452 	return 0;
5453 }
5454 
hisi_qp_alloc_memory(struct hisi_qm * qm)5455 static int hisi_qp_alloc_memory(struct hisi_qm *qm)
5456 {
5457 	struct device *dev = &qm->pdev->dev;
5458 	u16 sq_depth, cq_depth;
5459 	size_t qp_dma_size;
5460 	int i, ret;
5461 
5462 	qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);
5463 	if (!qm->qp_array)
5464 		return -ENOMEM;
5465 
5466 	qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL);
5467 	if (!qm->poll_data) {
5468 		kfree(qm->qp_array);
5469 		return -ENOMEM;
5470 	}
5471 
5472 	qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);
5473 
5474 	/* one more page for device or qp statuses */
5475 	qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth;
5476 	qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE;
5477 	for (i = 0; i < qm->qp_num; i++) {
5478 		qm->poll_data[i].qm = qm;
5479 		ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth);
5480 		if (ret)
5481 			goto err_init_qp_mem;
5482 
5483 		dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);
5484 	}
5485 
5486 	return 0;
5487 err_init_qp_mem:
5488 	hisi_qp_memory_uninit(qm, i);
5489 
5490 	return ret;
5491 }
5492 
hisi_qm_alloc_rsv_buf(struct hisi_qm * qm)5493 static int hisi_qm_alloc_rsv_buf(struct hisi_qm *qm)
5494 {
5495 	struct qm_rsv_buf *xqc_buf = &qm->xqc_buf;
5496 	struct qm_dma *xqc_dma = &xqc_buf->qcdma;
5497 	struct device *dev = &qm->pdev->dev;
5498 	size_t off = 0;
5499 
5500 #define QM_XQC_BUF_INIT(xqc_buf, type) do { \
5501 	(xqc_buf)->type = ((xqc_buf)->qcdma.va + (off)); \
5502 	(xqc_buf)->type##_dma = (xqc_buf)->qcdma.dma + (off); \
5503 	off += QMC_ALIGN(sizeof(struct qm_##type)); \
5504 } while (0)
5505 
5506 	xqc_dma->size = QMC_ALIGN(sizeof(struct qm_eqc)) +
5507 			QMC_ALIGN(sizeof(struct qm_aeqc)) +
5508 			QMC_ALIGN(sizeof(struct qm_sqc)) +
5509 			QMC_ALIGN(sizeof(struct qm_cqc));
5510 	xqc_dma->va = dma_alloc_coherent(dev, xqc_dma->size,
5511 					 &xqc_dma->dma, GFP_KERNEL);
5512 	if (!xqc_dma->va)
5513 		return -ENOMEM;
5514 
5515 	QM_XQC_BUF_INIT(xqc_buf, eqc);
5516 	QM_XQC_BUF_INIT(xqc_buf, aeqc);
5517 	QM_XQC_BUF_INIT(xqc_buf, sqc);
5518 	QM_XQC_BUF_INIT(xqc_buf, cqc);
5519 
5520 	return 0;
5521 }
5522 
hisi_qm_memory_init(struct hisi_qm * qm)5523 static int hisi_qm_memory_init(struct hisi_qm *qm)
5524 {
5525 	struct device *dev = &qm->pdev->dev;
5526 	int ret, total_func;
5527 	size_t off = 0;
5528 
5529 	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {
5530 		total_func = pci_sriov_get_totalvfs(qm->pdev) + 1;
5531 		qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL);
5532 		if (!qm->factor)
5533 			return -ENOMEM;
5534 
5535 		/* Only the PF value needs to be initialized */
5536 		qm->factor[0].func_qos = QM_QOS_MAX_VAL;
5537 	}
5538 
5539 #define QM_INIT_BUF(qm, type, num) do { \
5540 	(qm)->type = ((qm)->qdma.va + (off)); \
5541 	(qm)->type##_dma = (qm)->qdma.dma + (off); \
5542 	off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
5543 } while (0)
5544 
5545 	idr_init(&qm->qp_idr);
5546 	qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP);
5547 	qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) +
5548 			QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) +
5549 			QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
5550 			QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
5551 	qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,
5552 					 GFP_ATOMIC);
5553 	dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);
5554 	if (!qm->qdma.va) {
5555 		ret = -ENOMEM;
5556 		goto err_destroy_idr;
5557 	}
5558 
5559 	QM_INIT_BUF(qm, eqe, qm->eq_depth);
5560 	QM_INIT_BUF(qm, aeqe, qm->aeq_depth);
5561 	QM_INIT_BUF(qm, sqc, qm->qp_num);
5562 	QM_INIT_BUF(qm, cqc, qm->qp_num);
5563 
5564 	ret = hisi_qm_alloc_rsv_buf(qm);
5565 	if (ret)
5566 		goto err_free_qdma;
5567 
5568 	ret = hisi_qp_alloc_memory(qm);
5569 	if (ret)
5570 		goto err_free_reserve_buf;
5571 
5572 	return 0;
5573 
5574 err_free_reserve_buf:
5575 	hisi_qm_free_rsv_buf(qm);
5576 err_free_qdma:
5577 	dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
5578 err_destroy_idr:
5579 	idr_destroy(&qm->qp_idr);
5580 	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
5581 		kfree(qm->factor);
5582 
5583 	return ret;
5584 }
5585 
5586 /**
5587  * hisi_qm_init() - Initialize configures about qm.
5588  * @qm: The qm needing init.
5589  *
5590  * This function init qm, then we can call hisi_qm_start to put qm into work.
5591  */
hisi_qm_init(struct hisi_qm * qm)5592 int hisi_qm_init(struct hisi_qm *qm)
5593 {
5594 	struct pci_dev *pdev = qm->pdev;
5595 	struct device *dev = &pdev->dev;
5596 	int ret;
5597 
5598 	ret = hisi_qm_pci_init(qm);
5599 	if (ret)
5600 		return ret;
5601 
5602 	ret = qm_irqs_register(qm);
5603 	if (ret)
5604 		goto err_pci_init;
5605 
5606 	if (qm->fun_type == QM_HW_PF) {
5607 		/* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */
5608 		writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);
5609 		qm_disable_clock_gate(qm);
5610 		ret = qm_dev_mem_reset(qm);
5611 		if (ret) {
5612 			dev_err(dev, "failed to reset device memory\n");
5613 			goto err_irq_register;
5614 		}
5615 	}
5616 
5617 	if (qm->mode == UACCE_MODE_SVA) {
5618 		ret = qm_alloc_uacce(qm);
5619 		if (ret < 0)
5620 			dev_warn(dev, "fail to alloc uacce (%d)\n", ret);
5621 	}
5622 
5623 	ret = hisi_qm_memory_init(qm);
5624 	if (ret)
5625 		goto err_alloc_uacce;
5626 
5627 	ret = hisi_qm_init_work(qm);
5628 	if (ret)
5629 		goto err_free_qm_memory;
5630 
5631 	qm_cmd_init(qm);
5632 
5633 	return 0;
5634 
5635 err_free_qm_memory:
5636 	hisi_qm_memory_uninit(qm);
5637 err_alloc_uacce:
5638 	qm_remove_uacce(qm);
5639 err_irq_register:
5640 	qm_irqs_unregister(qm);
5641 err_pci_init:
5642 	hisi_qm_pci_uninit(qm);
5643 	return ret;
5644 }
5645 EXPORT_SYMBOL_GPL(hisi_qm_init);
5646 
5647 /**
5648  * hisi_qm_get_dfx_access() - Try to get dfx access.
5649  * @qm: pointer to accelerator device.
5650  *
5651  * Try to get dfx access, then user can get message.
5652  *
5653  * If device is in suspended, return failure, otherwise
5654  * bump up the runtime PM usage counter.
5655  */
hisi_qm_get_dfx_access(struct hisi_qm * qm)5656 int hisi_qm_get_dfx_access(struct hisi_qm *qm)
5657 {
5658 	struct device *dev = &qm->pdev->dev;
5659 
5660 	if (pm_runtime_suspended(dev)) {
5661 		dev_info(dev, "can not read/write - device in suspended.\n");
5662 		return -EAGAIN;
5663 	}
5664 
5665 	return qm_pm_get_sync(qm);
5666 }
5667 EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access);
5668 
5669 /**
5670  * hisi_qm_put_dfx_access() - Put dfx access.
5671  * @qm: pointer to accelerator device.
5672  *
5673  * Put dfx access, drop runtime PM usage counter.
5674  */
hisi_qm_put_dfx_access(struct hisi_qm * qm)5675 void hisi_qm_put_dfx_access(struct hisi_qm *qm)
5676 {
5677 	qm_pm_put_sync(qm);
5678 }
5679 EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access);
5680 
5681 /**
5682  * hisi_qm_pm_init() - Initialize qm runtime PM.
5683  * @qm: pointer to accelerator device.
5684  *
5685  * Function that initialize qm runtime PM.
5686  */
hisi_qm_pm_init(struct hisi_qm * qm)5687 void hisi_qm_pm_init(struct hisi_qm *qm)
5688 {
5689 	struct device *dev = &qm->pdev->dev;
5690 
5691 	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
5692 		return;
5693 
5694 	pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY);
5695 	pm_runtime_use_autosuspend(dev);
5696 	pm_runtime_put_noidle(dev);
5697 }
5698 EXPORT_SYMBOL_GPL(hisi_qm_pm_init);
5699 
5700 /**
5701  * hisi_qm_pm_uninit() - Uninitialize qm runtime PM.
5702  * @qm: pointer to accelerator device.
5703  *
5704  * Function that uninitialize qm runtime PM.
5705  */
hisi_qm_pm_uninit(struct hisi_qm * qm)5706 void hisi_qm_pm_uninit(struct hisi_qm *qm)
5707 {
5708 	struct device *dev = &qm->pdev->dev;
5709 
5710 	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
5711 		return;
5712 
5713 	pm_runtime_get_noresume(dev);
5714 	pm_runtime_dont_use_autosuspend(dev);
5715 }
5716 EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit);
5717 
qm_prepare_for_suspend(struct hisi_qm * qm)5718 static int qm_prepare_for_suspend(struct hisi_qm *qm)
5719 {
5720 	struct pci_dev *pdev = qm->pdev;
5721 	int ret;
5722 
5723 	ret = qm->ops->set_msi(qm, false);
5724 	if (ret) {
5725 		pci_err(pdev, "failed to disable MSI before suspending!\n");
5726 		return ret;
5727 	}
5728 
5729 	ret = qm_master_ooo_check(qm);
5730 	if (ret)
5731 		return ret;
5732 
5733 	if (qm->err_ini->set_priv_status) {
5734 		ret = qm->err_ini->set_priv_status(qm);
5735 		if (ret)
5736 			return ret;
5737 	}
5738 
5739 	ret = qm_set_pf_mse(qm, false);
5740 	if (ret)
5741 		pci_err(pdev, "failed to disable MSE before suspending!\n");
5742 
5743 	return ret;
5744 }
5745 
qm_rebuild_for_resume(struct hisi_qm * qm)5746 static int qm_rebuild_for_resume(struct hisi_qm *qm)
5747 {
5748 	struct pci_dev *pdev = qm->pdev;
5749 	int ret;
5750 
5751 	ret = qm_set_pf_mse(qm, true);
5752 	if (ret) {
5753 		pci_err(pdev, "failed to enable MSE after resuming!\n");
5754 		return ret;
5755 	}
5756 
5757 	ret = qm->ops->set_msi(qm, true);
5758 	if (ret) {
5759 		pci_err(pdev, "failed to enable MSI after resuming!\n");
5760 		return ret;
5761 	}
5762 
5763 	ret = qm_dev_hw_init(qm);
5764 	if (ret) {
5765 		pci_err(pdev, "failed to init device after resuming\n");
5766 		return ret;
5767 	}
5768 
5769 	qm_cmd_init(qm);
5770 	hisi_qm_dev_err_init(qm);
5771 	/* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */
5772 	writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);
5773 	qm_disable_clock_gate(qm);
5774 	ret = qm_dev_mem_reset(qm);
5775 	if (ret)
5776 		pci_err(pdev, "failed to reset device memory\n");
5777 
5778 	return ret;
5779 }
5780 
5781 /**
5782  * hisi_qm_suspend() - Runtime suspend of given device.
5783  * @dev: device to suspend.
5784  *
5785  * Function that suspend the device.
5786  */
hisi_qm_suspend(struct device * dev)5787 int hisi_qm_suspend(struct device *dev)
5788 {
5789 	struct pci_dev *pdev = to_pci_dev(dev);
5790 	struct hisi_qm *qm = pci_get_drvdata(pdev);
5791 	int ret;
5792 
5793 	pci_info(pdev, "entering suspended state\n");
5794 
5795 	ret = hisi_qm_stop(qm, QM_NORMAL);
5796 	if (ret) {
5797 		pci_err(pdev, "failed to stop qm(%d)\n", ret);
5798 		return ret;
5799 	}
5800 
5801 	ret = qm_prepare_for_suspend(qm);
5802 	if (ret)
5803 		pci_err(pdev, "failed to prepare suspended(%d)\n", ret);
5804 
5805 	return ret;
5806 }
5807 EXPORT_SYMBOL_GPL(hisi_qm_suspend);
5808 
5809 /**
5810  * hisi_qm_resume() - Runtime resume of given device.
5811  * @dev: device to resume.
5812  *
5813  * Function that resume the device.
5814  */
hisi_qm_resume(struct device * dev)5815 int hisi_qm_resume(struct device *dev)
5816 {
5817 	struct pci_dev *pdev = to_pci_dev(dev);
5818 	struct hisi_qm *qm = pci_get_drvdata(pdev);
5819 	int ret;
5820 
5821 	pci_info(pdev, "resuming from suspend state\n");
5822 
5823 	ret = qm_rebuild_for_resume(qm);
5824 	if (ret) {
5825 		pci_err(pdev, "failed to rebuild resume(%d)\n", ret);
5826 		return ret;
5827 	}
5828 
5829 	ret = hisi_qm_start(qm);
5830 	if (ret) {
5831 		if (qm_check_dev_error(qm)) {
5832 			pci_info(pdev, "failed to start qm due to device error, device will be reset!\n");
5833 			return 0;
5834 		}
5835 
5836 		pci_err(pdev, "failed to start qm(%d)!\n", ret);
5837 	}
5838 
5839 	return ret;
5840 }
5841 EXPORT_SYMBOL_GPL(hisi_qm_resume);
5842 
5843 MODULE_LICENSE("GPL v2");
5844 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
5845 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");
5846