1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <asm/page.h>
4 #include <linux/acpi.h>
5 #include <linux/bitmap.h>
6 #include <linux/dma-mapping.h>
7 #include <linux/idr.h>
8 #include <linux/io.h>
9 #include <linux/irqreturn.h>
10 #include <linux/log2.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/seq_file.h>
13 #include <linux/slab.h>
14 #include <linux/uacce.h>
15 #include <linux/uaccess.h>
16 #include <uapi/misc/uacce/hisi_qm.h>
17 #include <linux/hisi_acc_qm.h>
18 #include "qm_common.h"
19
20 /* eq/aeq irq enable */
21 #define QM_VF_AEQ_INT_SOURCE 0x0
22 #define QM_VF_AEQ_INT_MASK 0x4
23 #define QM_VF_EQ_INT_SOURCE 0x8
24 #define QM_VF_EQ_INT_MASK 0xc
25
26 #define QM_IRQ_VECTOR_MASK GENMASK(15, 0)
27 #define QM_IRQ_TYPE_MASK GENMASK(15, 0)
28 #define QM_IRQ_TYPE_SHIFT 16
29 #define QM_ABN_IRQ_TYPE_MASK GENMASK(7, 0)
30
31 /* mailbox */
32 #define QM_MB_PING_ALL_VFS 0xffff
33 #define QM_MB_STATUS_MASK GENMASK(12, 9)
34
35 /* sqc shift */
36 #define QM_SQ_HOP_NUM_SHIFT 0
37 #define QM_SQ_PAGE_SIZE_SHIFT 4
38 #define QM_SQ_BUF_SIZE_SHIFT 8
39 #define QM_SQ_SQE_SIZE_SHIFT 12
40 #define QM_SQ_PRIORITY_SHIFT 0
41 #define QM_SQ_ORDERS_SHIFT 4
42 #define QM_SQ_TYPE_SHIFT 8
43 #define QM_QC_PASID_ENABLE 0x1
44 #define QM_QC_PASID_ENABLE_SHIFT 7
45
46 #define QM_SQ_TYPE_MASK GENMASK(3, 0)
47 #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc).w11) >> 6) & 0x1)
48
49 /* cqc shift */
50 #define QM_CQ_HOP_NUM_SHIFT 0
51 #define QM_CQ_PAGE_SIZE_SHIFT 4
52 #define QM_CQ_BUF_SIZE_SHIFT 8
53 #define QM_CQ_CQE_SIZE_SHIFT 12
54 #define QM_CQ_PHASE_SHIFT 0
55 #define QM_CQ_FLAG_SHIFT 1
56
57 #define QM_CQE_PHASE(cqe) (le16_to_cpu((cqe)->w7) & 0x1)
58 #define QM_QC_CQE_SIZE 4
59 #define QM_CQ_TAIL_IDX(cqc) ((le16_to_cpu((cqc).w11) >> 6) & 0x1)
60
61 /* eqc shift */
62 #define QM_EQE_AEQE_SIZE (2UL << 12)
63 #define QM_EQC_PHASE_SHIFT 16
64
65 #define QM_EQE_PHASE(eqe) ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)
66 #define QM_EQE_CQN_MASK GENMASK(15, 0)
67
68 #define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
69 #define QM_AEQE_TYPE_SHIFT 17
70 #define QM_AEQE_TYPE_MASK 0xf
71 #define QM_AEQE_CQN_MASK GENMASK(15, 0)
72 #define QM_CQ_OVERFLOW 0
73 #define QM_EQ_OVERFLOW 1
74 #define QM_CQE_ERROR 2
75
76 #define QM_XQ_DEPTH_SHIFT 16
77 #define QM_XQ_DEPTH_MASK GENMASK(15, 0)
78
79 #define QM_DOORBELL_CMD_SQ 0
80 #define QM_DOORBELL_CMD_CQ 1
81 #define QM_DOORBELL_CMD_EQ 2
82 #define QM_DOORBELL_CMD_AEQ 3
83
84 #define QM_DOORBELL_BASE_V1 0x340
85 #define QM_DB_CMD_SHIFT_V1 16
86 #define QM_DB_INDEX_SHIFT_V1 32
87 #define QM_DB_PRIORITY_SHIFT_V1 48
88 #define QM_PAGE_SIZE 0x0034
89 #define QM_QP_DB_INTERVAL 0x10000
90 #define QM_DB_TIMEOUT_CFG 0x100074
91 #define QM_DB_TIMEOUT_SET 0x1fffff
92
93 #define QM_MEM_START_INIT 0x100040
94 #define QM_MEM_INIT_DONE 0x100044
95 #define QM_VFT_CFG_RDY 0x10006c
96 #define QM_VFT_CFG_OP_WR 0x100058
97 #define QM_VFT_CFG_TYPE 0x10005c
98 #define QM_VFT_CFG 0x100060
99 #define QM_VFT_CFG_OP_ENABLE 0x100054
100 #define QM_PM_CTRL 0x100148
101 #define QM_IDLE_DISABLE BIT(9)
102
103 #define QM_SUB_VERSION_ID 0x210
104
105 #define QM_VFT_CFG_DATA_L 0x100064
106 #define QM_VFT_CFG_DATA_H 0x100068
107 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8)
108 #define QM_SQC_VFT_SQC_SIZE (5ULL << 12)
109 #define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16)
110 #define QM_SQC_VFT_START_SQN_SHIFT 28
111 #define QM_SQC_VFT_VALID (1ULL << 44)
112 #define QM_SQC_VFT_SQN_SHIFT 45
113 #define QM_CQC_VFT_BUF_SIZE (7ULL << 8)
114 #define QM_CQC_VFT_SQC_SIZE (5ULL << 12)
115 #define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16)
116 #define QM_CQC_VFT_VALID (1ULL << 28)
117
118 #define QM_SQC_VFT_BASE_SHIFT_V2 28
119 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0)
120 #define QM_SQC_VFT_NUM_SHIFT_V2 45
121 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0)
122 #define QM_MAX_QC_TYPE 2
123
124 #define QM_ABNORMAL_INT_SOURCE 0x100000
125 #define QM_ABNORMAL_INT_MASK 0x100004
126 #define QM_ABNORMAL_INT_MASK_VALUE 0x7fff
127 #define QM_ABNORMAL_INT_STATUS 0x100008
128 #define QM_ABNORMAL_INT_SET 0x10000c
129 #define QM_ABNORMAL_INF00 0x100010
130 #define QM_FIFO_OVERFLOW_TYPE 0xc0
131 #define QM_FIFO_OVERFLOW_TYPE_SHIFT 6
132 #define QM_FIFO_OVERFLOW_VF 0x3f
133 #define QM_FIFO_OVERFLOW_QP_SHIFT 16
134 #define QM_ABNORMAL_INF01 0x100014
135 #define QM_DB_TIMEOUT_TYPE 0xc0
136 #define QM_DB_TIMEOUT_TYPE_SHIFT 6
137 #define QM_DB_TIMEOUT_VF 0x3f
138 #define QM_DB_TIMEOUT_QP_SHIFT 16
139 #define QM_ABNORMAL_INF02 0x100018
140 #define QM_AXI_POISON_ERR BIT(22)
141 #define QM_RAS_CE_ENABLE 0x1000ec
142 #define QM_RAS_FE_ENABLE 0x1000f0
143 #define QM_RAS_NFE_ENABLE 0x1000f4
144 #define QM_RAS_CE_THRESHOLD 0x1000f8
145 #define QM_RAS_CE_TIMES_PER_IRQ 1
146 #define QM_OOO_SHUTDOWN_SEL 0x1040f8
147 #define QM_AXI_RRESP_ERR BIT(0)
148 #define QM_ECC_MBIT BIT(2)
149 #define QM_DB_TIMEOUT BIT(10)
150 #define QM_OF_FIFO_OF BIT(11)
151
152 #define QM_RESET_WAIT_TIMEOUT 400
153 #define QM_PEH_VENDOR_ID 0x1000d8
154 #define ACC_VENDOR_ID_VALUE 0x5a5a
155 #define QM_PEH_DFX_INFO0 0x1000fc
156 #define QM_PEH_DFX_INFO1 0x100100
157 #define QM_PEH_DFX_MASK (BIT(0) | BIT(2))
158 #define QM_PEH_MSI_FINISH_MASK GENMASK(19, 16)
159 #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3
160 #define ACC_PEH_MSI_DISABLE GENMASK(31, 0)
161 #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1
162 #define ACC_MASTER_TRANS_RETURN_RW 3
163 #define ACC_MASTER_TRANS_RETURN 0x300150
164 #define ACC_MASTER_GLOBAL_CTRL 0x300000
165 #define ACC_AM_CFG_PORT_WR_EN 0x30001c
166 #define QM_RAS_NFE_MBIT_DISABLE ~QM_ECC_MBIT
167 #define ACC_AM_ROB_ECC_INT_STS 0x300104
168 #define ACC_ROB_ECC_ERR_MULTPL BIT(1)
169 #define QM_MSI_CAP_ENABLE BIT(16)
170
171 /* interfunction communication */
172 #define QM_IFC_READY_STATUS 0x100128
173 #define QM_IFC_INT_SET_P 0x100130
174 #define QM_IFC_INT_CFG 0x100134
175 #define QM_IFC_INT_SOURCE_P 0x100138
176 #define QM_IFC_INT_SOURCE_V 0x0020
177 #define QM_IFC_INT_MASK 0x0024
178 #define QM_IFC_INT_STATUS 0x0028
179 #define QM_IFC_INT_SET_V 0x002C
180 #define QM_PF2VF_PF_W 0x104700
181 #define QM_VF2PF_PF_R 0x104800
182 #define QM_VF2PF_VF_W 0x320
183 #define QM_PF2VF_VF_R 0x380
184 #define QM_IFC_SEND_ALL_VFS GENMASK(6, 0)
185 #define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0)
186 #define QM_IFC_INT_SOURCE_MASK BIT(0)
187 #define QM_IFC_INT_DISABLE BIT(0)
188 #define QM_IFC_INT_STATUS_MASK BIT(0)
189 #define QM_IFC_INT_SET_MASK BIT(0)
190 #define QM_WAIT_DST_ACK 10
191 #define QM_MAX_PF_WAIT_COUNT 10
192 #define QM_MAX_VF_WAIT_COUNT 40
193 #define QM_VF_RESET_WAIT_US 20000
194 #define QM_VF_RESET_WAIT_CNT 3000
195 #define QM_VF2PF_REG_SIZE 4
196 #define QM_IFC_CMD_MASK GENMASK(31, 0)
197 #define QM_IFC_DATA_SHIFT 32
198 #define QM_VF_RESET_WAIT_TIMEOUT_US \
199 (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT)
200
201 #define POLL_PERIOD 10
202 #define POLL_TIMEOUT 1000
203 #define WAIT_PERIOD_US_MAX 200
204 #define WAIT_PERIOD_US_MIN 100
205 #define MAX_WAIT_COUNTS 1000
206 #define QM_CACHE_WB_START 0x204
207 #define QM_CACHE_WB_DONE 0x208
208 #define QM_FUNC_CAPS_REG 0x3100
209 #define QM_CAPBILITY_VERSION GENMASK(7, 0)
210
211 #define PCI_BAR_2 2
212 #define PCI_BAR_4 4
213 #define QMC_ALIGN(sz) ALIGN(sz, 32)
214
215 #define QM_DBG_READ_LEN 256
216 #define QM_PCI_COMMAND_INVALID ~0
217 #define QM_RESET_STOP_TX_OFFSET 1
218 #define QM_RESET_STOP_RX_OFFSET 2
219
220 #define WAIT_PERIOD 20
221 #define REMOVE_WAIT_DELAY 10
222
223 #define QM_QOS_PARAM_NUM 2
224 #define QM_QOS_MAX_VAL 1000
225 #define QM_QOS_RATE 100
226 #define QM_QOS_EXPAND_RATE 1000
227 #define QM_SHAPER_CIR_B_MASK GENMASK(7, 0)
228 #define QM_SHAPER_CIR_U_MASK GENMASK(10, 8)
229 #define QM_SHAPER_CIR_S_MASK GENMASK(14, 11)
230 #define QM_SHAPER_FACTOR_CIR_U_SHIFT 8
231 #define QM_SHAPER_FACTOR_CIR_S_SHIFT 11
232 #define QM_SHAPER_FACTOR_CBS_B_SHIFT 15
233 #define QM_SHAPER_FACTOR_CBS_S_SHIFT 19
234 #define QM_SHAPER_CBS_B 1
235 #define QM_SHAPER_VFT_OFFSET 6
236 #define QM_QOS_MIN_ERROR_RATE 5
237 #define QM_SHAPER_MIN_CBS_S 8
238 #define QM_QOS_TICK 0x300U
239 #define QM_QOS_DIVISOR_CLK 0x1f40U
240 #define QM_QOS_MAX_CIR_B 200
241 #define QM_QOS_MIN_CIR_B 100
242 #define QM_QOS_MAX_CIR_U 6
243 #define QM_AUTOSUSPEND_DELAY 3000
244
245 /* abnormal status value for stopping queue */
246 #define QM_STOP_QUEUE_FAIL 1
247 #define QM_DUMP_SQC_FAIL 3
248 #define QM_DUMP_CQC_FAIL 4
249 #define QM_FINISH_WAIT 5
250
251 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
252 (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \
253 ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \
254 ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \
255 ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
256
257 #define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \
258 ((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
259
260 #define QM_MK_SQC_W13(priority, orders, alg_type) \
261 (((priority) << QM_SQ_PRIORITY_SHIFT) | \
262 ((orders) << QM_SQ_ORDERS_SHIFT) | \
263 (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
264
265 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
266 (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \
267 ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \
268 ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \
269 ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
270
271 #define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \
272 ((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
273
274 enum vft_type {
275 SQC_VFT = 0,
276 CQC_VFT,
277 SHAPER_VFT,
278 };
279
280 enum qm_alg_type {
281 ALG_TYPE_0,
282 ALG_TYPE_1,
283 };
284
285 enum qm_ifc_cmd {
286 QM_PF_FLR_PREPARE = 0x01,
287 QM_PF_SRST_PREPARE,
288 QM_PF_RESET_DONE,
289 QM_VF_PREPARE_DONE,
290 QM_VF_PREPARE_FAIL,
291 QM_VF_START_DONE,
292 QM_VF_START_FAIL,
293 QM_PF_SET_QOS,
294 QM_VF_GET_QOS,
295 };
296
297 enum qm_basic_type {
298 QM_TOTAL_QP_NUM_CAP = 0x0,
299 QM_FUNC_MAX_QP_CAP,
300 QM_XEQ_DEPTH_CAP,
301 QM_QP_DEPTH_CAP,
302 QM_EQ_IRQ_TYPE_CAP,
303 QM_AEQ_IRQ_TYPE_CAP,
304 QM_ABN_IRQ_TYPE_CAP,
305 QM_PF2VF_IRQ_TYPE_CAP,
306 QM_PF_IRQ_NUM_CAP,
307 QM_VF_IRQ_NUM_CAP,
308 };
309
310 enum qm_cap_table_type {
311 QM_CAP_VF = 0x0,
312 QM_AEQE_NUM,
313 QM_SCQE_NUM,
314 QM_EQ_IRQ,
315 QM_AEQ_IRQ,
316 QM_ABNORMAL_IRQ,
317 QM_MB_IRQ,
318 MAX_IRQ_NUM,
319 EXT_BAR_INDEX,
320 };
321
322 static const struct hisi_qm_cap_query_info qm_cap_query_info[] = {
323 {QM_CAP_VF, "QM_CAP_VF ", 0x3100, 0x0, 0x0, 0x6F01},
324 {QM_AEQE_NUM, "QM_AEQE_NUM ", 0x3104, 0x800, 0x4000800, 0x4000800},
325 {QM_SCQE_NUM, "QM_SCQE_NUM ",
326 0x3108, 0x4000400, 0x4000400, 0x4000400},
327 {QM_EQ_IRQ, "QM_EQ_IRQ ", 0x310c, 0x10000, 0x10000, 0x10000},
328 {QM_AEQ_IRQ, "QM_AEQ_IRQ ", 0x3110, 0x0, 0x10001, 0x10001},
329 {QM_ABNORMAL_IRQ, "QM_ABNORMAL_IRQ ", 0x3114, 0x0, 0x10003, 0x10003},
330 {QM_MB_IRQ, "QM_MB_IRQ ", 0x3118, 0x0, 0x0, 0x10002},
331 {MAX_IRQ_NUM, "MAX_IRQ_NUM ", 0x311c, 0x10001, 0x40002, 0x40003},
332 {EXT_BAR_INDEX, "EXT_BAR_INDEX ", 0x3120, 0x0, 0x0, 0x14},
333 };
334
335 static const struct hisi_qm_cap_info qm_cap_info_comm[] = {
336 {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0},
337 {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1},
338 {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1},
339 {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1},
340 {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1},
341 {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1},
342 {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0},
343 };
344
345 static const struct hisi_qm_cap_info qm_cap_info_pf[] = {
346 {QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1},
347 };
348
349 static const struct hisi_qm_cap_info qm_cap_info_vf[] = {
350 {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0},
351 };
352
353 static const struct hisi_qm_cap_info qm_basic_info[] = {
354 {QM_TOTAL_QP_NUM_CAP, 0x100158, 0, GENMASK(10, 0), 0x1000, 0x400, 0x400},
355 {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400},
356 {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(31, 0), 0x800, 0x4000800, 0x4000800},
357 {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400},
358 {QM_EQ_IRQ_TYPE_CAP, 0x310c, 0, GENMASK(31, 0), 0x10000, 0x10000, 0x10000},
359 {QM_AEQ_IRQ_TYPE_CAP, 0x3110, 0, GENMASK(31, 0), 0x0, 0x10001, 0x10001},
360 {QM_ABN_IRQ_TYPE_CAP, 0x3114, 0, GENMASK(31, 0), 0x0, 0x10003, 0x10003},
361 {QM_PF2VF_IRQ_TYPE_CAP, 0x3118, 0, GENMASK(31, 0), 0x0, 0x0, 0x10002},
362 {QM_PF_IRQ_NUM_CAP, 0x311c, 16, GENMASK(15, 0), 0x1, 0x4, 0x4},
363 {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3},
364 };
365
366 struct qm_mailbox {
367 __le16 w0;
368 __le16 queue_num;
369 __le32 base_l;
370 __le32 base_h;
371 __le32 rsvd;
372 };
373
374 struct qm_doorbell {
375 __le16 queue_num;
376 __le16 cmd;
377 __le16 index;
378 __le16 priority;
379 };
380
381 struct hisi_qm_resource {
382 struct hisi_qm *qm;
383 int distance;
384 struct list_head list;
385 };
386
387 /**
388 * struct qm_hw_err - Structure describing the device errors
389 * @list: hardware error list
390 * @timestamp: timestamp when the error occurred
391 */
392 struct qm_hw_err {
393 struct list_head list;
394 unsigned long long timestamp;
395 };
396
397 struct hisi_qm_hw_ops {
398 int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
399 void (*qm_db)(struct hisi_qm *qm, u16 qn,
400 u8 cmd, u16 index, u8 priority);
401 int (*debug_init)(struct hisi_qm *qm);
402 void (*hw_error_init)(struct hisi_qm *qm);
403 void (*hw_error_uninit)(struct hisi_qm *qm);
404 enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
405 int (*set_msi)(struct hisi_qm *qm, bool set);
406
407 /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */
408 int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num);
409 void (*set_ifc_end)(struct hisi_qm *qm);
410 int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num);
411 };
412
413 struct hisi_qm_hw_error {
414 u32 int_msk;
415 const char *msg;
416 };
417
418 static const struct hisi_qm_hw_error qm_hw_error[] = {
419 { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
420 { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
421 { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
422 { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
423 { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
424 { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
425 { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
426 { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
427 { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
428 { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
429 { .int_msk = BIT(10), .msg = "qm_db_timeout" },
430 { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
431 { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
432 { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
433 { .int_msk = BIT(14), .msg = "qm_flr_timeout" },
434 };
435
436 static const char * const qm_db_timeout[] = {
437 "sq", "cq", "eq", "aeq",
438 };
439
440 static const char * const qm_fifo_overflow[] = {
441 "cq", "eq", "aeq",
442 };
443
444 struct qm_typical_qos_table {
445 u32 start;
446 u32 end;
447 u32 val;
448 };
449
450 /* the qos step is 100 */
451 static struct qm_typical_qos_table shaper_cir_s[] = {
452 {100, 100, 4},
453 {200, 200, 3},
454 {300, 500, 2},
455 {600, 1000, 1},
456 {1100, 100000, 0},
457 };
458
459 static struct qm_typical_qos_table shaper_cbs_s[] = {
460 {100, 200, 9},
461 {300, 500, 11},
462 {600, 1000, 12},
463 {1100, 10000, 16},
464 {10100, 25000, 17},
465 {25100, 50000, 18},
466 {50100, 100000, 19}
467 };
468
469 static void qm_irqs_unregister(struct hisi_qm *qm);
470 static int qm_reset_device(struct hisi_qm *qm);
hisi_qm_q_num_set(const char * val,const struct kernel_param * kp,unsigned int device)471 int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp,
472 unsigned int device)
473 {
474 struct pci_dev *pdev;
475 u32 n, q_num;
476 int ret;
477
478 if (!val)
479 return -EINVAL;
480
481 pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL);
482 if (!pdev) {
483 q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2);
484 pr_info("No device found currently, suppose queue number is %u\n",
485 q_num);
486 } else {
487 if (pdev->revision == QM_HW_V1)
488 q_num = QM_QNUM_V1;
489 else
490 q_num = QM_QNUM_V2;
491
492 pci_dev_put(pdev);
493 }
494
495 ret = kstrtou32(val, 10, &n);
496 if (ret || n < QM_MIN_QNUM || n > q_num)
497 return -EINVAL;
498
499 return param_set_int(val, kp);
500 }
501 EXPORT_SYMBOL_GPL(hisi_qm_q_num_set);
502
qm_get_hw_error_status(struct hisi_qm * qm)503 static u32 qm_get_hw_error_status(struct hisi_qm *qm)
504 {
505 return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
506 }
507
qm_get_dev_err_status(struct hisi_qm * qm)508 static u32 qm_get_dev_err_status(struct hisi_qm *qm)
509 {
510 return qm->err_ini->get_dev_hw_err_status(qm);
511 }
512
513 /* Check if the error causes the master ooo block */
qm_check_dev_error(struct hisi_qm * qm)514 static bool qm_check_dev_error(struct hisi_qm *qm)
515 {
516 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
517 u32 err_status;
518
519 if (pf_qm->fun_type == QM_HW_VF)
520 return false;
521
522 err_status = qm_get_hw_error_status(pf_qm);
523 if (err_status & pf_qm->err_info.qm_shutdown_mask)
524 return true;
525
526 if (pf_qm->err_ini->dev_is_abnormal)
527 return pf_qm->err_ini->dev_is_abnormal(pf_qm);
528
529 return false;
530 }
531
qm_wait_reset_finish(struct hisi_qm * qm)532 static int qm_wait_reset_finish(struct hisi_qm *qm)
533 {
534 int delay = 0;
535
536 /* All reset requests need to be queued for processing */
537 while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
538 msleep(++delay);
539 if (delay > QM_RESET_WAIT_TIMEOUT)
540 return -EBUSY;
541 }
542
543 return 0;
544 }
545
qm_reset_prepare_ready(struct hisi_qm * qm)546 static int qm_reset_prepare_ready(struct hisi_qm *qm)
547 {
548 struct pci_dev *pdev = qm->pdev;
549 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
550
551 /*
552 * PF and VF on host doesnot support resetting at the
553 * same time on Kunpeng920.
554 */
555 if (qm->ver < QM_HW_V3)
556 return qm_wait_reset_finish(pf_qm);
557
558 return qm_wait_reset_finish(qm);
559 }
560
qm_reset_bit_clear(struct hisi_qm * qm)561 static void qm_reset_bit_clear(struct hisi_qm *qm)
562 {
563 struct pci_dev *pdev = qm->pdev;
564 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
565
566 if (qm->ver < QM_HW_V3)
567 clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
568
569 clear_bit(QM_RESETTING, &qm->misc_ctl);
570 }
571
qm_mb_pre_init(struct qm_mailbox * mailbox,u8 cmd,u64 base,u16 queue,bool op)572 static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,
573 u64 base, u16 queue, bool op)
574 {
575 mailbox->w0 = cpu_to_le16((cmd) |
576 ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) |
577 (0x1 << QM_MB_BUSY_SHIFT));
578 mailbox->queue_num = cpu_to_le16(queue);
579 mailbox->base_l = cpu_to_le32(lower_32_bits(base));
580 mailbox->base_h = cpu_to_le32(upper_32_bits(base));
581 mailbox->rsvd = 0;
582 }
583
584 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
hisi_qm_wait_mb_ready(struct hisi_qm * qm)585 int hisi_qm_wait_mb_ready(struct hisi_qm *qm)
586 {
587 u32 val;
588
589 return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
590 val, !((val >> QM_MB_BUSY_SHIFT) &
591 0x1), POLL_PERIOD, POLL_TIMEOUT);
592 }
593 EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready);
594
595 /* 128 bit should be written to hardware at one time to trigger a mailbox */
qm_mb_write(struct hisi_qm * qm,const void * src)596 static void qm_mb_write(struct hisi_qm *qm, const void *src)
597 {
598 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
599
600 #if IS_ENABLED(CONFIG_ARM64)
601 unsigned long tmp0 = 0, tmp1 = 0;
602 #endif
603
604 if (!IS_ENABLED(CONFIG_ARM64)) {
605 memcpy_toio(fun_base, src, 16);
606 dma_wmb();
607 return;
608 }
609
610 #if IS_ENABLED(CONFIG_ARM64)
611 asm volatile("ldp %0, %1, %3\n"
612 "stp %0, %1, %2\n"
613 "dmb oshst\n"
614 : "=&r" (tmp0),
615 "=&r" (tmp1),
616 "+Q" (*((char __iomem *)fun_base))
617 : "Q" (*((char *)src))
618 : "memory");
619 #endif
620 }
621
qm_mb_nolock(struct hisi_qm * qm,struct qm_mailbox * mailbox)622 static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox)
623 {
624 int ret;
625 u32 val;
626
627 if (unlikely(hisi_qm_wait_mb_ready(qm))) {
628 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
629 ret = -EBUSY;
630 goto mb_busy;
631 }
632
633 qm_mb_write(qm, mailbox);
634
635 if (unlikely(hisi_qm_wait_mb_ready(qm))) {
636 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
637 ret = -ETIMEDOUT;
638 goto mb_busy;
639 }
640
641 val = readl(qm->io_base + QM_MB_CMD_SEND_BASE);
642 if (val & QM_MB_STATUS_MASK) {
643 dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n");
644 ret = -EIO;
645 goto mb_busy;
646 }
647
648 return 0;
649
650 mb_busy:
651 atomic64_inc(&qm->debug.dfx.mb_err_cnt);
652 return ret;
653 }
654
hisi_qm_mb(struct hisi_qm * qm,u8 cmd,dma_addr_t dma_addr,u16 queue,bool op)655 int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
656 bool op)
657 {
658 struct qm_mailbox mailbox;
659 int ret;
660
661 qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op);
662
663 mutex_lock(&qm->mailbox_lock);
664 ret = qm_mb_nolock(qm, &mailbox);
665 mutex_unlock(&qm->mailbox_lock);
666
667 return ret;
668 }
669 EXPORT_SYMBOL_GPL(hisi_qm_mb);
670
671 /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */
qm_set_and_get_xqc(struct hisi_qm * qm,u8 cmd,void * xqc,u32 qp_id,bool op)672 int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op)
673 {
674 struct qm_mailbox mailbox;
675 dma_addr_t xqc_dma;
676 void *tmp_xqc;
677 size_t size;
678 int ret;
679
680 switch (cmd) {
681 case QM_MB_CMD_SQC:
682 size = sizeof(struct qm_sqc);
683 tmp_xqc = qm->xqc_buf.sqc;
684 xqc_dma = qm->xqc_buf.sqc_dma;
685 break;
686 case QM_MB_CMD_CQC:
687 size = sizeof(struct qm_cqc);
688 tmp_xqc = qm->xqc_buf.cqc;
689 xqc_dma = qm->xqc_buf.cqc_dma;
690 break;
691 case QM_MB_CMD_EQC:
692 size = sizeof(struct qm_eqc);
693 tmp_xqc = qm->xqc_buf.eqc;
694 xqc_dma = qm->xqc_buf.eqc_dma;
695 break;
696 case QM_MB_CMD_AEQC:
697 size = sizeof(struct qm_aeqc);
698 tmp_xqc = qm->xqc_buf.aeqc;
699 xqc_dma = qm->xqc_buf.aeqc_dma;
700 break;
701 default:
702 dev_err(&qm->pdev->dev, "unknown mailbox cmd %u\n", cmd);
703 return -EINVAL;
704 }
705
706 /* Setting xqc will fail if master OOO is blocked. */
707 if (qm_check_dev_error(qm)) {
708 dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n");
709 return -EIO;
710 }
711
712 mutex_lock(&qm->mailbox_lock);
713 if (!op)
714 memcpy(tmp_xqc, xqc, size);
715
716 qm_mb_pre_init(&mailbox, cmd, xqc_dma, qp_id, op);
717 ret = qm_mb_nolock(qm, &mailbox);
718 if (!ret && op)
719 memcpy(xqc, tmp_xqc, size);
720
721 mutex_unlock(&qm->mailbox_lock);
722
723 return ret;
724 }
725
qm_db_v1(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)726 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
727 {
728 u64 doorbell;
729
730 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
731 ((u64)index << QM_DB_INDEX_SHIFT_V1) |
732 ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
733
734 writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
735 }
736
qm_db_v2(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)737 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
738 {
739 void __iomem *io_base = qm->io_base;
740 u16 randata = 0;
741 u64 doorbell;
742
743 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
744 io_base = qm->db_io_base + (u64)qn * qm->db_interval +
745 QM_DOORBELL_SQ_CQ_BASE_V2;
746 else
747 io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;
748
749 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
750 ((u64)randata << QM_DB_RAND_SHIFT_V2) |
751 ((u64)index << QM_DB_INDEX_SHIFT_V2) |
752 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
753
754 writeq(doorbell, io_base);
755 }
756
qm_db(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)757 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
758 {
759 dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
760 qn, cmd, index);
761
762 qm->ops->qm_db(qm, qn, cmd, index, priority);
763 }
764
qm_disable_clock_gate(struct hisi_qm * qm)765 static void qm_disable_clock_gate(struct hisi_qm *qm)
766 {
767 u32 val;
768
769 /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */
770 if (qm->ver < QM_HW_V3)
771 return;
772
773 val = readl(qm->io_base + QM_PM_CTRL);
774 val |= QM_IDLE_DISABLE;
775 writel(val, qm->io_base + QM_PM_CTRL);
776 }
777
qm_dev_mem_reset(struct hisi_qm * qm)778 static int qm_dev_mem_reset(struct hisi_qm *qm)
779 {
780 u32 val;
781
782 writel(0x1, qm->io_base + QM_MEM_START_INIT);
783 return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
784 val & BIT(0), POLL_PERIOD,
785 POLL_TIMEOUT);
786 }
787
788 /**
789 * hisi_qm_get_hw_info() - Get device information.
790 * @qm: The qm which want to get information.
791 * @info_table: Array for storing device information.
792 * @index: Index in info_table.
793 * @is_read: Whether read from reg, 0: not support read from reg.
794 *
795 * This function returns device information the caller needs.
796 */
hisi_qm_get_hw_info(struct hisi_qm * qm,const struct hisi_qm_cap_info * info_table,u32 index,bool is_read)797 u32 hisi_qm_get_hw_info(struct hisi_qm *qm,
798 const struct hisi_qm_cap_info *info_table,
799 u32 index, bool is_read)
800 {
801 u32 val;
802
803 switch (qm->ver) {
804 case QM_HW_V1:
805 return info_table[index].v1_val;
806 case QM_HW_V2:
807 return info_table[index].v2_val;
808 default:
809 if (!is_read)
810 return info_table[index].v3_val;
811
812 val = readl(qm->io_base + info_table[index].offset);
813 return (val >> info_table[index].shift) & info_table[index].mask;
814 }
815 }
816 EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info);
817
hisi_qm_get_cap_value(struct hisi_qm * qm,const struct hisi_qm_cap_query_info * info_table,u32 index,bool is_read)818 u32 hisi_qm_get_cap_value(struct hisi_qm *qm,
819 const struct hisi_qm_cap_query_info *info_table,
820 u32 index, bool is_read)
821 {
822 u32 val;
823
824 switch (qm->ver) {
825 case QM_HW_V1:
826 return info_table[index].v1_val;
827 case QM_HW_V2:
828 return info_table[index].v2_val;
829 default:
830 if (!is_read)
831 return info_table[index].v3_val;
832
833 val = readl(qm->io_base + info_table[index].offset);
834 return val;
835 }
836 }
837 EXPORT_SYMBOL_GPL(hisi_qm_get_cap_value);
838
qm_get_xqc_depth(struct hisi_qm * qm,u16 * low_bits,u16 * high_bits,enum qm_basic_type type)839 static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits,
840 u16 *high_bits, enum qm_basic_type type)
841 {
842 u32 depth;
843
844 depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver);
845 *low_bits = depth & QM_XQ_DEPTH_MASK;
846 *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK;
847 }
848
hisi_qm_set_algs(struct hisi_qm * qm,u64 alg_msk,const struct qm_dev_alg * dev_algs,u32 dev_algs_size)849 int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs,
850 u32 dev_algs_size)
851 {
852 struct device *dev = &qm->pdev->dev;
853 char *algs, *ptr;
854 int i;
855
856 if (!qm->uacce)
857 return 0;
858
859 if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) {
860 dev_err(dev, "algs size %u is equal or larger than %d.\n",
861 dev_algs_size, QM_DEV_ALG_MAX_LEN);
862 return -EINVAL;
863 }
864
865 algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL);
866 if (!algs)
867 return -ENOMEM;
868
869 for (i = 0; i < dev_algs_size; i++)
870 if (alg_msk & dev_algs[i].alg_msk)
871 strcat(algs, dev_algs[i].alg);
872
873 ptr = strrchr(algs, '\n');
874 if (ptr)
875 *ptr = '\0';
876
877 qm->uacce->algs = algs;
878
879 return 0;
880 }
881 EXPORT_SYMBOL_GPL(hisi_qm_set_algs);
882
qm_get_irq_num(struct hisi_qm * qm)883 static u32 qm_get_irq_num(struct hisi_qm *qm)
884 {
885 if (qm->fun_type == QM_HW_PF)
886 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver);
887
888 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver);
889 }
890
qm_pm_get_sync(struct hisi_qm * qm)891 static int qm_pm_get_sync(struct hisi_qm *qm)
892 {
893 struct device *dev = &qm->pdev->dev;
894 int ret;
895
896 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
897 return 0;
898
899 ret = pm_runtime_resume_and_get(dev);
900 if (ret < 0) {
901 dev_err(dev, "failed to get_sync(%d).\n", ret);
902 return ret;
903 }
904
905 return 0;
906 }
907
qm_pm_put_sync(struct hisi_qm * qm)908 static void qm_pm_put_sync(struct hisi_qm *qm)
909 {
910 struct device *dev = &qm->pdev->dev;
911
912 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
913 return;
914
915 pm_runtime_put_autosuspend(dev);
916 }
917
qm_cq_head_update(struct hisi_qp * qp)918 static void qm_cq_head_update(struct hisi_qp *qp)
919 {
920 if (qp->qp_status.cq_head == qp->cq_depth - 1) {
921 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
922 qp->qp_status.cq_head = 0;
923 } else {
924 qp->qp_status.cq_head++;
925 }
926 }
927
qm_poll_req_cb(struct hisi_qp * qp)928 static void qm_poll_req_cb(struct hisi_qp *qp)
929 {
930 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
931 struct hisi_qm *qm = qp->qm;
932
933 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
934 dma_rmb();
935 qp->req_cb(qp, qp->sqe + qm->sqe_size *
936 le16_to_cpu(cqe->sq_head));
937 qm_cq_head_update(qp);
938 cqe = qp->cqe + qp->qp_status.cq_head;
939 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
940 qp->qp_status.cq_head, 0);
941 atomic_dec(&qp->qp_status.used);
942
943 cond_resched();
944 }
945
946 /* set c_flag */
947 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1);
948 }
949
qm_work_process(struct work_struct * work)950 static void qm_work_process(struct work_struct *work)
951 {
952 struct hisi_qm_poll_data *poll_data =
953 container_of(work, struct hisi_qm_poll_data, work);
954 struct hisi_qm *qm = poll_data->qm;
955 u16 eqe_num = poll_data->eqe_num;
956 struct hisi_qp *qp;
957 int i;
958
959 for (i = eqe_num - 1; i >= 0; i--) {
960 qp = &qm->qp_array[poll_data->qp_finish_id[i]];
961 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP))
962 continue;
963
964 if (qp->event_cb) {
965 qp->event_cb(qp);
966 continue;
967 }
968
969 if (likely(qp->req_cb))
970 qm_poll_req_cb(qp);
971 }
972 }
973
qm_get_complete_eqe_num(struct hisi_qm * qm)974 static void qm_get_complete_eqe_num(struct hisi_qm *qm)
975 {
976 struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
977 struct hisi_qm_poll_data *poll_data = NULL;
978 u16 eq_depth = qm->eq_depth;
979 u16 cqn, eqe_num = 0;
980
981 if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) {
982 atomic64_inc(&qm->debug.dfx.err_irq_cnt);
983 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
984 return;
985 }
986
987 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
988 if (unlikely(cqn >= qm->qp_num))
989 return;
990 poll_data = &qm->poll_data[cqn];
991
992 while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
993 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
994 poll_data->qp_finish_id[eqe_num] = cqn;
995 eqe_num++;
996
997 if (qm->status.eq_head == eq_depth - 1) {
998 qm->status.eqc_phase = !qm->status.eqc_phase;
999 eqe = qm->eqe;
1000 qm->status.eq_head = 0;
1001 } else {
1002 eqe++;
1003 qm->status.eq_head++;
1004 }
1005
1006 if (eqe_num == (eq_depth >> 1) - 1)
1007 break;
1008 }
1009
1010 poll_data->eqe_num = eqe_num;
1011 queue_work(qm->wq, &poll_data->work);
1012 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
1013 }
1014
qm_eq_irq(int irq,void * data)1015 static irqreturn_t qm_eq_irq(int irq, void *data)
1016 {
1017 struct hisi_qm *qm = data;
1018
1019 /* Get qp id of completed tasks and re-enable the interrupt */
1020 qm_get_complete_eqe_num(qm);
1021
1022 return IRQ_HANDLED;
1023 }
1024
qm_mb_cmd_irq(int irq,void * data)1025 static irqreturn_t qm_mb_cmd_irq(int irq, void *data)
1026 {
1027 struct hisi_qm *qm = data;
1028 u32 val;
1029
1030 val = readl(qm->io_base + QM_IFC_INT_STATUS);
1031 val &= QM_IFC_INT_STATUS_MASK;
1032 if (!val)
1033 return IRQ_NONE;
1034
1035 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) {
1036 dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n");
1037 return IRQ_HANDLED;
1038 }
1039
1040 schedule_work(&qm->cmd_process);
1041
1042 return IRQ_HANDLED;
1043 }
1044
qm_set_qp_disable(struct hisi_qp * qp,int offset)1045 static void qm_set_qp_disable(struct hisi_qp *qp, int offset)
1046 {
1047 u32 *addr;
1048
1049 if (qp->is_in_kernel)
1050 return;
1051
1052 addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset;
1053 *addr = 1;
1054
1055 /* make sure setup is completed */
1056 smp_wmb();
1057 }
1058
qm_disable_qp(struct hisi_qm * qm,u32 qp_id)1059 static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id)
1060 {
1061 struct hisi_qp *qp = &qm->qp_array[qp_id];
1062
1063 qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET);
1064 hisi_qm_stop_qp(qp);
1065 qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET);
1066 }
1067
qm_reset_function(struct hisi_qm * qm)1068 static void qm_reset_function(struct hisi_qm *qm)
1069 {
1070 struct device *dev = &qm->pdev->dev;
1071 int ret;
1072
1073 if (qm_check_dev_error(qm))
1074 return;
1075
1076 ret = qm_reset_prepare_ready(qm);
1077 if (ret) {
1078 dev_err(dev, "reset function not ready\n");
1079 return;
1080 }
1081
1082 ret = hisi_qm_stop(qm, QM_DOWN);
1083 if (ret) {
1084 dev_err(dev, "failed to stop qm when reset function\n");
1085 goto clear_bit;
1086 }
1087
1088 ret = hisi_qm_start(qm);
1089 if (ret)
1090 dev_err(dev, "failed to start qm when reset function\n");
1091
1092 clear_bit:
1093 qm_reset_bit_clear(qm);
1094 }
1095
qm_aeq_thread(int irq,void * data)1096 static irqreturn_t qm_aeq_thread(int irq, void *data)
1097 {
1098 struct hisi_qm *qm = data;
1099 struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
1100 u16 aeq_depth = qm->aeq_depth;
1101 u32 type, qp_id;
1102
1103 atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
1104
1105 while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
1106 type = (le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT) &
1107 QM_AEQE_TYPE_MASK;
1108 qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK;
1109
1110 switch (type) {
1111 case QM_EQ_OVERFLOW:
1112 dev_err(&qm->pdev->dev, "eq overflow, reset function\n");
1113 qm_reset_function(qm);
1114 return IRQ_HANDLED;
1115 case QM_CQ_OVERFLOW:
1116 dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n",
1117 qp_id);
1118 fallthrough;
1119 case QM_CQE_ERROR:
1120 qm_disable_qp(qm, qp_id);
1121 break;
1122 default:
1123 dev_err(&qm->pdev->dev, "unknown error type %u\n",
1124 type);
1125 break;
1126 }
1127
1128 if (qm->status.aeq_head == aeq_depth - 1) {
1129 qm->status.aeqc_phase = !qm->status.aeqc_phase;
1130 aeqe = qm->aeqe;
1131 qm->status.aeq_head = 0;
1132 } else {
1133 aeqe++;
1134 qm->status.aeq_head++;
1135 }
1136 }
1137
1138 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
1139
1140 return IRQ_HANDLED;
1141 }
1142
qm_init_qp_status(struct hisi_qp * qp)1143 static void qm_init_qp_status(struct hisi_qp *qp)
1144 {
1145 struct hisi_qp_status *qp_status = &qp->qp_status;
1146
1147 qp_status->sq_tail = 0;
1148 qp_status->cq_head = 0;
1149 qp_status->cqc_phase = true;
1150 atomic_set(&qp_status->used, 0);
1151 }
1152
qm_init_prefetch(struct hisi_qm * qm)1153 static void qm_init_prefetch(struct hisi_qm *qm)
1154 {
1155 struct device *dev = &qm->pdev->dev;
1156 u32 page_type = 0x0;
1157
1158 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
1159 return;
1160
1161 switch (PAGE_SIZE) {
1162 case SZ_4K:
1163 page_type = 0x0;
1164 break;
1165 case SZ_16K:
1166 page_type = 0x1;
1167 break;
1168 case SZ_64K:
1169 page_type = 0x2;
1170 break;
1171 default:
1172 dev_err(dev, "system page size is not support: %lu, default set to 4KB",
1173 PAGE_SIZE);
1174 }
1175
1176 writel(page_type, qm->io_base + QM_PAGE_SIZE);
1177 }
1178
1179 /*
1180 * acc_shaper_para_calc() Get the IR value by the qos formula, the return value
1181 * is the expected qos calculated.
1182 * the formula:
1183 * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps
1184 *
1185 * IR_b * (2 ^ IR_u) * 8000
1186 * IR(Mbps) = -------------------------
1187 * Tick * (2 ^ IR_s)
1188 */
acc_shaper_para_calc(u64 cir_b,u64 cir_u,u64 cir_s)1189 static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
1190 {
1191 return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) /
1192 (QM_QOS_TICK * (1 << cir_s));
1193 }
1194
acc_shaper_calc_cbs_s(u32 ir)1195 static u32 acc_shaper_calc_cbs_s(u32 ir)
1196 {
1197 int table_size = ARRAY_SIZE(shaper_cbs_s);
1198 int i;
1199
1200 for (i = 0; i < table_size; i++) {
1201 if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end)
1202 return shaper_cbs_s[i].val;
1203 }
1204
1205 return QM_SHAPER_MIN_CBS_S;
1206 }
1207
acc_shaper_calc_cir_s(u32 ir)1208 static u32 acc_shaper_calc_cir_s(u32 ir)
1209 {
1210 int table_size = ARRAY_SIZE(shaper_cir_s);
1211 int i;
1212
1213 for (i = 0; i < table_size; i++) {
1214 if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end)
1215 return shaper_cir_s[i].val;
1216 }
1217
1218 return 0;
1219 }
1220
qm_get_shaper_para(u32 ir,struct qm_shaper_factor * factor)1221 static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
1222 {
1223 u32 cir_b, cir_u, cir_s, ir_calc;
1224 u32 error_rate;
1225
1226 factor->cbs_s = acc_shaper_calc_cbs_s(ir);
1227 cir_s = acc_shaper_calc_cir_s(ir);
1228
1229 for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {
1230 for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {
1231 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
1232
1233 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
1234 if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
1235 factor->cir_b = cir_b;
1236 factor->cir_u = cir_u;
1237 factor->cir_s = cir_s;
1238 return 0;
1239 }
1240 }
1241 }
1242
1243 return -EINVAL;
1244 }
1245
qm_vft_data_cfg(struct hisi_qm * qm,enum vft_type type,u32 base,u32 number,struct qm_shaper_factor * factor)1246 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
1247 u32 number, struct qm_shaper_factor *factor)
1248 {
1249 u64 tmp = 0;
1250
1251 if (number > 0) {
1252 switch (type) {
1253 case SQC_VFT:
1254 if (qm->ver == QM_HW_V1) {
1255 tmp = QM_SQC_VFT_BUF_SIZE |
1256 QM_SQC_VFT_SQC_SIZE |
1257 QM_SQC_VFT_INDEX_NUMBER |
1258 QM_SQC_VFT_VALID |
1259 (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
1260 } else {
1261 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
1262 QM_SQC_VFT_VALID |
1263 (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
1264 }
1265 break;
1266 case CQC_VFT:
1267 if (qm->ver == QM_HW_V1) {
1268 tmp = QM_CQC_VFT_BUF_SIZE |
1269 QM_CQC_VFT_SQC_SIZE |
1270 QM_CQC_VFT_INDEX_NUMBER |
1271 QM_CQC_VFT_VALID;
1272 } else {
1273 tmp = QM_CQC_VFT_VALID;
1274 }
1275 break;
1276 case SHAPER_VFT:
1277 if (factor) {
1278 tmp = factor->cir_b |
1279 (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) |
1280 (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) |
1281 (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) |
1282 (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT);
1283 }
1284 break;
1285 }
1286 }
1287
1288 writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
1289 writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
1290 }
1291
qm_set_vft_common(struct hisi_qm * qm,enum vft_type type,u32 fun_num,u32 base,u32 number)1292 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
1293 u32 fun_num, u32 base, u32 number)
1294 {
1295 struct qm_shaper_factor *factor = NULL;
1296 unsigned int val;
1297 int ret;
1298
1299 if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
1300 factor = &qm->factor[fun_num];
1301
1302 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1303 val & BIT(0), POLL_PERIOD,
1304 POLL_TIMEOUT);
1305 if (ret)
1306 return ret;
1307
1308 writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
1309 writel(type, qm->io_base + QM_VFT_CFG_TYPE);
1310 if (type == SHAPER_VFT)
1311 fun_num |= base << QM_SHAPER_VFT_OFFSET;
1312
1313 writel(fun_num, qm->io_base + QM_VFT_CFG);
1314
1315 qm_vft_data_cfg(qm, type, base, number, factor);
1316
1317 writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
1318 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
1319
1320 return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1321 val & BIT(0), POLL_PERIOD,
1322 POLL_TIMEOUT);
1323 }
1324
qm_shaper_init_vft(struct hisi_qm * qm,u32 fun_num)1325 static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)
1326 {
1327 u32 qos = qm->factor[fun_num].func_qos;
1328 int ret, i;
1329
1330 ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]);
1331 if (ret) {
1332 dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");
1333 return ret;
1334 }
1335 writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG);
1336 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
1337 /* The base number of queue reuse for different alg type */
1338 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1);
1339 if (ret)
1340 return ret;
1341 }
1342
1343 return 0;
1344 }
1345
1346 /* The config should be conducted after qm_dev_mem_reset() */
qm_set_sqc_cqc_vft(struct hisi_qm * qm,u32 fun_num,u32 base,u32 number)1347 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
1348 u32 number)
1349 {
1350 int ret, i;
1351
1352 for (i = SQC_VFT; i <= CQC_VFT; i++) {
1353 ret = qm_set_vft_common(qm, i, fun_num, base, number);
1354 if (ret)
1355 return ret;
1356 }
1357
1358 /* init default shaper qos val */
1359 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {
1360 ret = qm_shaper_init_vft(qm, fun_num);
1361 if (ret)
1362 goto back_sqc_cqc;
1363 }
1364
1365 return 0;
1366 back_sqc_cqc:
1367 for (i = SQC_VFT; i <= CQC_VFT; i++)
1368 qm_set_vft_common(qm, i, fun_num, 0, 0);
1369
1370 return ret;
1371 }
1372
qm_get_vft_v2(struct hisi_qm * qm,u32 * base,u32 * number)1373 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
1374 {
1375 u64 sqc_vft;
1376 int ret;
1377
1378 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
1379 if (ret)
1380 return ret;
1381
1382 sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
1383 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
1384 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
1385 *number = (QM_SQC_VFT_NUM_MASK_V2 &
1386 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
1387
1388 return 0;
1389 }
1390
qm_hw_error_init_v1(struct hisi_qm * qm)1391 static void qm_hw_error_init_v1(struct hisi_qm *qm)
1392 {
1393 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1394 }
1395
qm_hw_error_cfg(struct hisi_qm * qm)1396 static void qm_hw_error_cfg(struct hisi_qm *qm)
1397 {
1398 struct hisi_qm_err_info *err_info = &qm->err_info;
1399
1400 qm->error_mask = err_info->nfe | err_info->ce | err_info->fe;
1401 /* clear QM hw residual error source */
1402 writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE);
1403
1404 /* configure error type */
1405 writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE);
1406 writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1407 writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1408 writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE);
1409 }
1410
qm_hw_error_init_v2(struct hisi_qm * qm)1411 static void qm_hw_error_init_v2(struct hisi_qm *qm)
1412 {
1413 u32 irq_unmask;
1414
1415 qm_hw_error_cfg(qm);
1416
1417 irq_unmask = ~qm->error_mask;
1418 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1419 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1420 }
1421
qm_hw_error_uninit_v2(struct hisi_qm * qm)1422 static void qm_hw_error_uninit_v2(struct hisi_qm *qm)
1423 {
1424 u32 irq_mask = qm->error_mask;
1425
1426 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1427 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);
1428 }
1429
qm_hw_error_init_v3(struct hisi_qm * qm)1430 static void qm_hw_error_init_v3(struct hisi_qm *qm)
1431 {
1432 u32 irq_unmask;
1433
1434 qm_hw_error_cfg(qm);
1435
1436 /* enable close master ooo when hardware error happened */
1437 writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL);
1438
1439 irq_unmask = ~qm->error_mask;
1440 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1441 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1442 }
1443
qm_hw_error_uninit_v3(struct hisi_qm * qm)1444 static void qm_hw_error_uninit_v3(struct hisi_qm *qm)
1445 {
1446 u32 irq_mask = qm->error_mask;
1447
1448 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1449 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);
1450
1451 /* disable close master ooo when hardware error happened */
1452 writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL);
1453 }
1454
qm_log_hw_error(struct hisi_qm * qm,u32 error_status)1455 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
1456 {
1457 const struct hisi_qm_hw_error *err;
1458 struct device *dev = &qm->pdev->dev;
1459 u32 reg_val, type, vf_num, qp_id;
1460 int i;
1461
1462 for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
1463 err = &qm_hw_error[i];
1464 if (!(err->int_msk & error_status))
1465 continue;
1466
1467 dev_err(dev, "%s [error status=0x%x] found\n",
1468 err->msg, err->int_msk);
1469
1470 if (err->int_msk & QM_DB_TIMEOUT) {
1471 reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);
1472 type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
1473 QM_DB_TIMEOUT_TYPE_SHIFT;
1474 vf_num = reg_val & QM_DB_TIMEOUT_VF;
1475 qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT;
1476 dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n",
1477 qm_db_timeout[type], vf_num, qp_id);
1478 } else if (err->int_msk & QM_OF_FIFO_OF) {
1479 reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
1480 type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
1481 QM_FIFO_OVERFLOW_TYPE_SHIFT;
1482 vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
1483 qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT;
1484 if (type < ARRAY_SIZE(qm_fifo_overflow))
1485 dev_err(dev, "qm %s fifo overflow in function %u qp %u\n",
1486 qm_fifo_overflow[type], vf_num, qp_id);
1487 else
1488 dev_err(dev, "unknown error type\n");
1489 } else if (err->int_msk & QM_AXI_RRESP_ERR) {
1490 reg_val = readl(qm->io_base + QM_ABNORMAL_INF02);
1491 if (reg_val & QM_AXI_POISON_ERR)
1492 dev_err(dev, "qm axi poison error happened\n");
1493 }
1494 }
1495 }
1496
qm_hw_error_handle_v2(struct hisi_qm * qm)1497 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
1498 {
1499 u32 error_status;
1500
1501 error_status = qm_get_hw_error_status(qm);
1502 if (error_status & qm->error_mask) {
1503 if (error_status & QM_ECC_MBIT)
1504 qm->err_status.is_qm_ecc_mbit = true;
1505
1506 qm_log_hw_error(qm, error_status);
1507 if (error_status & qm->err_info.qm_reset_mask) {
1508 /* Disable the same error reporting until device is recovered. */
1509 writel(qm->err_info.nfe & (~error_status),
1510 qm->io_base + QM_RAS_NFE_ENABLE);
1511 return ACC_ERR_NEED_RESET;
1512 }
1513
1514 /* Clear error source if not need reset. */
1515 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);
1516 writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1517 writel(qm->err_info.ce, qm->io_base + QM_RAS_CE_ENABLE);
1518 }
1519
1520 return ACC_ERR_RECOVERED;
1521 }
1522
qm_get_mb_cmd(struct hisi_qm * qm,u64 * msg,u16 fun_num)1523 static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)
1524 {
1525 struct qm_mailbox mailbox;
1526 int ret;
1527
1528 qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0);
1529 mutex_lock(&qm->mailbox_lock);
1530 ret = qm_mb_nolock(qm, &mailbox);
1531 if (ret)
1532 goto err_unlock;
1533
1534 *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
1535 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
1536
1537 err_unlock:
1538 mutex_unlock(&qm->mailbox_lock);
1539 return ret;
1540 }
1541
qm_clear_cmd_interrupt(struct hisi_qm * qm,u64 vf_mask)1542 static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask)
1543 {
1544 u32 val;
1545
1546 if (qm->fun_type == QM_HW_PF)
1547 writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P);
1548
1549 val = readl(qm->io_base + QM_IFC_INT_SOURCE_V);
1550 val |= QM_IFC_INT_SOURCE_MASK;
1551 writel(val, qm->io_base + QM_IFC_INT_SOURCE_V);
1552 }
1553
qm_handle_vf_msg(struct hisi_qm * qm,u32 vf_id)1554 static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id)
1555 {
1556 struct device *dev = &qm->pdev->dev;
1557 enum qm_ifc_cmd cmd;
1558 int ret;
1559
1560 ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id);
1561 if (ret) {
1562 dev_err(dev, "failed to get command from VF(%u)!\n", vf_id);
1563 return;
1564 }
1565
1566 switch (cmd) {
1567 case QM_VF_PREPARE_FAIL:
1568 dev_err(dev, "failed to stop VF(%u)!\n", vf_id);
1569 break;
1570 case QM_VF_START_FAIL:
1571 dev_err(dev, "failed to start VF(%u)!\n", vf_id);
1572 break;
1573 case QM_VF_PREPARE_DONE:
1574 case QM_VF_START_DONE:
1575 break;
1576 default:
1577 dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id);
1578 break;
1579 }
1580 }
1581
qm_wait_vf_prepare_finish(struct hisi_qm * qm)1582 static int qm_wait_vf_prepare_finish(struct hisi_qm *qm)
1583 {
1584 struct device *dev = &qm->pdev->dev;
1585 u32 vfs_num = qm->vfs_num;
1586 int cnt = 0;
1587 int ret = 0;
1588 u64 val;
1589 u32 i;
1590
1591 if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
1592 return 0;
1593
1594 while (true) {
1595 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
1596 /* All VFs send command to PF, break */
1597 if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1))
1598 break;
1599
1600 if (++cnt > QM_MAX_PF_WAIT_COUNT) {
1601 ret = -EBUSY;
1602 break;
1603 }
1604
1605 msleep(QM_WAIT_DST_ACK);
1606 }
1607
1608 /* PF check VFs msg */
1609 for (i = 1; i <= vfs_num; i++) {
1610 if (val & BIT(i))
1611 qm_handle_vf_msg(qm, i);
1612 else
1613 dev_err(dev, "VF(%u) not ping PF!\n", i);
1614 }
1615
1616 /* PF clear interrupt to ack VFs */
1617 qm_clear_cmd_interrupt(qm, val);
1618
1619 return ret;
1620 }
1621
qm_trigger_vf_interrupt(struct hisi_qm * qm,u32 fun_num)1622 static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num)
1623 {
1624 u32 val;
1625
1626 val = readl(qm->io_base + QM_IFC_INT_CFG);
1627 val &= ~QM_IFC_SEND_ALL_VFS;
1628 val |= fun_num;
1629 writel(val, qm->io_base + QM_IFC_INT_CFG);
1630
1631 val = readl(qm->io_base + QM_IFC_INT_SET_P);
1632 val |= QM_IFC_INT_SET_MASK;
1633 writel(val, qm->io_base + QM_IFC_INT_SET_P);
1634 }
1635
qm_trigger_pf_interrupt(struct hisi_qm * qm)1636 static void qm_trigger_pf_interrupt(struct hisi_qm *qm)
1637 {
1638 u32 val;
1639
1640 val = readl(qm->io_base + QM_IFC_INT_SET_V);
1641 val |= QM_IFC_INT_SET_MASK;
1642 writel(val, qm->io_base + QM_IFC_INT_SET_V);
1643 }
1644
qm_ping_single_vf(struct hisi_qm * qm,enum qm_ifc_cmd cmd,u32 data,u32 fun_num)1645 static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num)
1646 {
1647 struct device *dev = &qm->pdev->dev;
1648 int cnt = 0;
1649 u64 val;
1650 int ret;
1651
1652 ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num);
1653 if (ret) {
1654 dev_err(dev, "failed to send command to vf(%u)!\n", fun_num);
1655 goto err_unlock;
1656 }
1657
1658 qm_trigger_vf_interrupt(qm, fun_num);
1659 while (true) {
1660 msleep(QM_WAIT_DST_ACK);
1661 val = readq(qm->io_base + QM_IFC_READY_STATUS);
1662 /* if VF respond, PF notifies VF successfully. */
1663 if (!(val & BIT(fun_num)))
1664 goto err_unlock;
1665
1666 if (++cnt > QM_MAX_PF_WAIT_COUNT) {
1667 dev_err(dev, "failed to get response from VF(%u)!\n", fun_num);
1668 ret = -ETIMEDOUT;
1669 break;
1670 }
1671 }
1672
1673 err_unlock:
1674 qm->ops->set_ifc_end(qm);
1675 return ret;
1676 }
1677
qm_ping_all_vfs(struct hisi_qm * qm,enum qm_ifc_cmd cmd)1678 static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd)
1679 {
1680 struct device *dev = &qm->pdev->dev;
1681 u32 vfs_num = qm->vfs_num;
1682 u64 val = 0;
1683 int cnt = 0;
1684 int ret;
1685 u32 i;
1686
1687 ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS);
1688 if (ret) {
1689 dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd);
1690 qm->ops->set_ifc_end(qm);
1691 return ret;
1692 }
1693
1694 qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS);
1695 while (true) {
1696 msleep(QM_WAIT_DST_ACK);
1697 val = readq(qm->io_base + QM_IFC_READY_STATUS);
1698 /* If all VFs acked, PF notifies VFs successfully. */
1699 if (!(val & GENMASK(vfs_num, 1))) {
1700 qm->ops->set_ifc_end(qm);
1701 return 0;
1702 }
1703
1704 if (++cnt > QM_MAX_PF_WAIT_COUNT)
1705 break;
1706 }
1707
1708 qm->ops->set_ifc_end(qm);
1709
1710 /* Check which vf respond timeout. */
1711 for (i = 1; i <= vfs_num; i++) {
1712 if (val & BIT(i))
1713 dev_err(dev, "failed to get response from VF(%u)!\n", i);
1714 }
1715
1716 return -ETIMEDOUT;
1717 }
1718
qm_ping_pf(struct hisi_qm * qm,enum qm_ifc_cmd cmd)1719 static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd)
1720 {
1721 int cnt = 0;
1722 u32 val;
1723 int ret;
1724
1725 ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0);
1726 if (ret) {
1727 dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd);
1728 goto unlock;
1729 }
1730
1731 qm_trigger_pf_interrupt(qm);
1732 /* Waiting for PF response */
1733 while (true) {
1734 msleep(QM_WAIT_DST_ACK);
1735 val = readl(qm->io_base + QM_IFC_INT_SET_V);
1736 if (!(val & QM_IFC_INT_STATUS_MASK))
1737 break;
1738
1739 if (++cnt > QM_MAX_VF_WAIT_COUNT) {
1740 ret = -ETIMEDOUT;
1741 break;
1742 }
1743 }
1744
1745 unlock:
1746 qm->ops->set_ifc_end(qm);
1747
1748 return ret;
1749 }
1750
qm_drain_qm(struct hisi_qm * qm)1751 static int qm_drain_qm(struct hisi_qm *qm)
1752 {
1753 return hisi_qm_mb(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0);
1754 }
1755
qm_stop_qp(struct hisi_qp * qp)1756 static int qm_stop_qp(struct hisi_qp *qp)
1757 {
1758 return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);
1759 }
1760
qm_set_msi(struct hisi_qm * qm,bool set)1761 static int qm_set_msi(struct hisi_qm *qm, bool set)
1762 {
1763 struct pci_dev *pdev = qm->pdev;
1764
1765 if (set) {
1766 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
1767 0);
1768 } else {
1769 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
1770 ACC_PEH_MSI_DISABLE);
1771 if (qm->err_status.is_qm_ecc_mbit ||
1772 qm->err_status.is_dev_ecc_mbit)
1773 return 0;
1774
1775 mdelay(1);
1776 if (readl(qm->io_base + QM_PEH_DFX_INFO0))
1777 return -EFAULT;
1778 }
1779
1780 return 0;
1781 }
1782
qm_wait_msi_finish(struct hisi_qm * qm)1783 static void qm_wait_msi_finish(struct hisi_qm *qm)
1784 {
1785 struct pci_dev *pdev = qm->pdev;
1786 u32 cmd = ~0;
1787 int cnt = 0;
1788 u32 val;
1789 int ret;
1790
1791 while (true) {
1792 pci_read_config_dword(pdev, pdev->msi_cap +
1793 PCI_MSI_PENDING_64, &cmd);
1794 if (!cmd)
1795 break;
1796
1797 if (++cnt > MAX_WAIT_COUNTS) {
1798 pci_warn(pdev, "failed to empty MSI PENDING!\n");
1799 break;
1800 }
1801
1802 udelay(1);
1803 }
1804
1805 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0,
1806 val, !(val & QM_PEH_DFX_MASK),
1807 POLL_PERIOD, POLL_TIMEOUT);
1808 if (ret)
1809 pci_warn(pdev, "failed to empty PEH MSI!\n");
1810
1811 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1,
1812 val, !(val & QM_PEH_MSI_FINISH_MASK),
1813 POLL_PERIOD, POLL_TIMEOUT);
1814 if (ret)
1815 pci_warn(pdev, "failed to finish MSI operation!\n");
1816 }
1817
qm_set_msi_v3(struct hisi_qm * qm,bool set)1818 static int qm_set_msi_v3(struct hisi_qm *qm, bool set)
1819 {
1820 struct pci_dev *pdev = qm->pdev;
1821 int ret = -ETIMEDOUT;
1822 u32 cmd, i;
1823
1824 pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
1825 if (set)
1826 cmd |= QM_MSI_CAP_ENABLE;
1827 else
1828 cmd &= ~QM_MSI_CAP_ENABLE;
1829
1830 pci_write_config_dword(pdev, pdev->msi_cap, cmd);
1831 if (set) {
1832 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
1833 pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
1834 if (cmd & QM_MSI_CAP_ENABLE)
1835 return 0;
1836
1837 udelay(1);
1838 }
1839 } else {
1840 udelay(WAIT_PERIOD_US_MIN);
1841 qm_wait_msi_finish(qm);
1842 ret = 0;
1843 }
1844
1845 return ret;
1846 }
1847
qm_set_ifc_begin_v3(struct hisi_qm * qm,enum qm_ifc_cmd cmd,u32 data,u32 fun_num)1848 static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num)
1849 {
1850 struct qm_mailbox mailbox;
1851 u64 msg;
1852
1853 msg = cmd | (u64)data << QM_IFC_DATA_SHIFT;
1854
1855 qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0);
1856 mutex_lock(&qm->mailbox_lock);
1857 return qm_mb_nolock(qm, &mailbox);
1858 }
1859
qm_set_ifc_end_v3(struct hisi_qm * qm)1860 static void qm_set_ifc_end_v3(struct hisi_qm *qm)
1861 {
1862 mutex_unlock(&qm->mailbox_lock);
1863 }
1864
qm_get_ifc_v3(struct hisi_qm * qm,enum qm_ifc_cmd * cmd,u32 * data,u32 fun_num)1865 static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num)
1866 {
1867 u64 msg;
1868 int ret;
1869
1870 ret = qm_get_mb_cmd(qm, &msg, fun_num);
1871 if (ret)
1872 return ret;
1873
1874 *cmd = msg & QM_IFC_CMD_MASK;
1875
1876 if (data)
1877 *data = msg >> QM_IFC_DATA_SHIFT;
1878
1879 return 0;
1880 }
1881
qm_set_ifc_begin_v4(struct hisi_qm * qm,enum qm_ifc_cmd cmd,u32 data,u32 fun_num)1882 static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num)
1883 {
1884 uintptr_t offset;
1885 u64 msg;
1886
1887 if (qm->fun_type == QM_HW_PF)
1888 offset = QM_PF2VF_PF_W;
1889 else
1890 offset = QM_VF2PF_VF_W;
1891
1892 msg = cmd | (u64)data << QM_IFC_DATA_SHIFT;
1893
1894 mutex_lock(&qm->ifc_lock);
1895 writeq(msg, qm->io_base + offset);
1896
1897 return 0;
1898 }
1899
qm_set_ifc_end_v4(struct hisi_qm * qm)1900 static void qm_set_ifc_end_v4(struct hisi_qm *qm)
1901 {
1902 mutex_unlock(&qm->ifc_lock);
1903 }
1904
qm_get_ifc_pf(struct hisi_qm * qm,u32 fun_num)1905 static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num)
1906 {
1907 uintptr_t offset;
1908
1909 offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num;
1910
1911 return (u64)readl(qm->io_base + offset);
1912 }
1913
qm_get_ifc_vf(struct hisi_qm * qm)1914 static u64 qm_get_ifc_vf(struct hisi_qm *qm)
1915 {
1916 return readq(qm->io_base + QM_PF2VF_VF_R);
1917 }
1918
qm_get_ifc_v4(struct hisi_qm * qm,enum qm_ifc_cmd * cmd,u32 * data,u32 fun_num)1919 static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num)
1920 {
1921 u64 msg;
1922
1923 if (qm->fun_type == QM_HW_PF)
1924 msg = qm_get_ifc_pf(qm, fun_num);
1925 else
1926 msg = qm_get_ifc_vf(qm);
1927
1928 *cmd = msg & QM_IFC_CMD_MASK;
1929
1930 if (data)
1931 *data = msg >> QM_IFC_DATA_SHIFT;
1932
1933 return 0;
1934 }
1935
1936 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
1937 .qm_db = qm_db_v1,
1938 .hw_error_init = qm_hw_error_init_v1,
1939 .set_msi = qm_set_msi,
1940 };
1941
1942 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
1943 .get_vft = qm_get_vft_v2,
1944 .qm_db = qm_db_v2,
1945 .hw_error_init = qm_hw_error_init_v2,
1946 .hw_error_uninit = qm_hw_error_uninit_v2,
1947 .hw_error_handle = qm_hw_error_handle_v2,
1948 .set_msi = qm_set_msi,
1949 };
1950
1951 static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {
1952 .get_vft = qm_get_vft_v2,
1953 .qm_db = qm_db_v2,
1954 .hw_error_init = qm_hw_error_init_v3,
1955 .hw_error_uninit = qm_hw_error_uninit_v3,
1956 .hw_error_handle = qm_hw_error_handle_v2,
1957 .set_msi = qm_set_msi_v3,
1958 .set_ifc_begin = qm_set_ifc_begin_v3,
1959 .set_ifc_end = qm_set_ifc_end_v3,
1960 .get_ifc = qm_get_ifc_v3,
1961 };
1962
1963 static const struct hisi_qm_hw_ops qm_hw_ops_v4 = {
1964 .get_vft = qm_get_vft_v2,
1965 .qm_db = qm_db_v2,
1966 .hw_error_init = qm_hw_error_init_v3,
1967 .hw_error_uninit = qm_hw_error_uninit_v3,
1968 .hw_error_handle = qm_hw_error_handle_v2,
1969 .set_msi = qm_set_msi_v3,
1970 .set_ifc_begin = qm_set_ifc_begin_v4,
1971 .set_ifc_end = qm_set_ifc_end_v4,
1972 .get_ifc = qm_get_ifc_v4,
1973 };
1974
qm_get_avail_sqe(struct hisi_qp * qp)1975 static void *qm_get_avail_sqe(struct hisi_qp *qp)
1976 {
1977 struct hisi_qp_status *qp_status = &qp->qp_status;
1978 u16 sq_tail = qp_status->sq_tail;
1979
1980 if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1))
1981 return NULL;
1982
1983 return qp->sqe + sq_tail * qp->qm->sqe_size;
1984 }
1985
hisi_qm_unset_hw_reset(struct hisi_qp * qp)1986 static void hisi_qm_unset_hw_reset(struct hisi_qp *qp)
1987 {
1988 u64 *addr;
1989
1990 /* Use last 64 bits of DUS to reset status. */
1991 addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET;
1992 *addr = 0;
1993 }
1994
qm_create_qp_nolock(struct hisi_qm * qm,u8 alg_type)1995 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
1996 {
1997 struct device *dev = &qm->pdev->dev;
1998 struct hisi_qp *qp;
1999 int qp_id;
2000
2001 if (atomic_read(&qm->status.flags) == QM_STOP) {
2002 dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n");
2003 return ERR_PTR(-EPERM);
2004 }
2005
2006 if (qm->qp_in_used == qm->qp_num) {
2007 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
2008 qm->qp_num);
2009 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
2010 return ERR_PTR(-EBUSY);
2011 }
2012
2013 qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);
2014 if (qp_id < 0) {
2015 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
2016 qm->qp_num);
2017 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
2018 return ERR_PTR(-EBUSY);
2019 }
2020
2021 qp = &qm->qp_array[qp_id];
2022 hisi_qm_unset_hw_reset(qp);
2023 memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth);
2024
2025 qp->event_cb = NULL;
2026 qp->req_cb = NULL;
2027 qp->qp_id = qp_id;
2028 qp->alg_type = alg_type;
2029 qp->is_in_kernel = true;
2030 qm->qp_in_used++;
2031
2032 return qp;
2033 }
2034
2035 /**
2036 * hisi_qm_create_qp() - Create a queue pair from qm.
2037 * @qm: The qm we create a qp from.
2038 * @alg_type: Accelerator specific algorithm type in sqc.
2039 *
2040 * Return created qp, negative error code if failed.
2041 */
hisi_qm_create_qp(struct hisi_qm * qm,u8 alg_type)2042 static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
2043 {
2044 struct hisi_qp *qp;
2045 int ret;
2046
2047 ret = qm_pm_get_sync(qm);
2048 if (ret)
2049 return ERR_PTR(ret);
2050
2051 down_write(&qm->qps_lock);
2052 qp = qm_create_qp_nolock(qm, alg_type);
2053 up_write(&qm->qps_lock);
2054
2055 if (IS_ERR(qp))
2056 qm_pm_put_sync(qm);
2057
2058 return qp;
2059 }
2060
2061 /**
2062 * hisi_qm_release_qp() - Release a qp back to its qm.
2063 * @qp: The qp we want to release.
2064 *
2065 * This function releases the resource of a qp.
2066 */
hisi_qm_release_qp(struct hisi_qp * qp)2067 static void hisi_qm_release_qp(struct hisi_qp *qp)
2068 {
2069 struct hisi_qm *qm = qp->qm;
2070
2071 down_write(&qm->qps_lock);
2072
2073 qm->qp_in_used--;
2074 idr_remove(&qm->qp_idr, qp->qp_id);
2075
2076 up_write(&qm->qps_lock);
2077
2078 qm_pm_put_sync(qm);
2079 }
2080
qm_sq_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2081 static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2082 {
2083 struct hisi_qm *qm = qp->qm;
2084 enum qm_hw_ver ver = qm->ver;
2085 struct qm_sqc sqc = {0};
2086
2087 if (ver == QM_HW_V1) {
2088 sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));
2089 sqc.w8 = cpu_to_le16(qp->sq_depth - 1);
2090 } else {
2091 sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth));
2092 sqc.w8 = 0; /* rand_qc */
2093 }
2094 sqc.w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));
2095 sqc.base_l = cpu_to_le32(lower_32_bits(qp->sqe_dma));
2096 sqc.base_h = cpu_to_le32(upper_32_bits(qp->sqe_dma));
2097 sqc.cq_num = cpu_to_le16(qp_id);
2098 sqc.pasid = cpu_to_le16(pasid);
2099
2100 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2101 sqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE <<
2102 QM_QC_PASID_ENABLE_SHIFT);
2103
2104 return qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 0);
2105 }
2106
qm_cq_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2107 static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2108 {
2109 struct hisi_qm *qm = qp->qm;
2110 enum qm_hw_ver ver = qm->ver;
2111 struct qm_cqc cqc = {0};
2112
2113 if (ver == QM_HW_V1) {
2114 cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE));
2115 cqc.w8 = cpu_to_le16(qp->cq_depth - 1);
2116 } else {
2117 cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth));
2118 cqc.w8 = 0; /* rand_qc */
2119 }
2120 /*
2121 * Enable request finishing interrupts defaultly.
2122 * So, there will be some interrupts until disabling
2123 * this.
2124 */
2125 cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
2126 cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma));
2127 cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma));
2128 cqc.pasid = cpu_to_le16(pasid);
2129
2130 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2131 cqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE);
2132
2133 return qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 0);
2134 }
2135
qm_qp_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2136 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2137 {
2138 int ret;
2139
2140 qm_init_qp_status(qp);
2141
2142 ret = qm_sq_ctx_cfg(qp, qp_id, pasid);
2143 if (ret)
2144 return ret;
2145
2146 return qm_cq_ctx_cfg(qp, qp_id, pasid);
2147 }
2148
qm_start_qp_nolock(struct hisi_qp * qp,unsigned long arg)2149 static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
2150 {
2151 struct hisi_qm *qm = qp->qm;
2152 struct device *dev = &qm->pdev->dev;
2153 int qp_id = qp->qp_id;
2154 u32 pasid = arg;
2155 int ret;
2156
2157 if (atomic_read(&qm->status.flags) == QM_STOP) {
2158 dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n");
2159 return -EPERM;
2160 }
2161
2162 ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
2163 if (ret)
2164 return ret;
2165
2166 atomic_set(&qp->qp_status.flags, QP_START);
2167 dev_dbg(dev, "queue %d started\n", qp_id);
2168
2169 return 0;
2170 }
2171
2172 /**
2173 * hisi_qm_start_qp() - Start a qp into running.
2174 * @qp: The qp we want to start to run.
2175 * @arg: Accelerator specific argument.
2176 *
2177 * After this function, qp can receive request from user. Return 0 if
2178 * successful, negative error code if failed.
2179 */
hisi_qm_start_qp(struct hisi_qp * qp,unsigned long arg)2180 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
2181 {
2182 struct hisi_qm *qm = qp->qm;
2183 int ret;
2184
2185 down_write(&qm->qps_lock);
2186 ret = qm_start_qp_nolock(qp, arg);
2187 up_write(&qm->qps_lock);
2188
2189 return ret;
2190 }
2191 EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
2192
2193 /**
2194 * qp_stop_fail_cb() - call request cb.
2195 * @qp: stopped failed qp.
2196 *
2197 * Callback function should be called whether task completed or not.
2198 */
qp_stop_fail_cb(struct hisi_qp * qp)2199 static void qp_stop_fail_cb(struct hisi_qp *qp)
2200 {
2201 int qp_used = atomic_read(&qp->qp_status.used);
2202 u16 cur_tail = qp->qp_status.sq_tail;
2203 u16 sq_depth = qp->sq_depth;
2204 u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth;
2205 struct hisi_qm *qm = qp->qm;
2206 u16 pos;
2207 int i;
2208
2209 for (i = 0; i < qp_used; i++) {
2210 pos = (i + cur_head) % sq_depth;
2211 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));
2212 atomic_dec(&qp->qp_status.used);
2213 }
2214 }
2215
qm_wait_qp_empty(struct hisi_qm * qm,u32 * state,u32 qp_id)2216 static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id)
2217 {
2218 struct device *dev = &qm->pdev->dev;
2219 struct qm_sqc sqc;
2220 struct qm_cqc cqc;
2221 int ret, i = 0;
2222
2223 while (++i) {
2224 ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1);
2225 if (ret) {
2226 dev_err_ratelimited(dev, "Failed to dump sqc!\n");
2227 *state = QM_DUMP_SQC_FAIL;
2228 return ret;
2229 }
2230
2231 ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1);
2232 if (ret) {
2233 dev_err_ratelimited(dev, "Failed to dump cqc!\n");
2234 *state = QM_DUMP_CQC_FAIL;
2235 return ret;
2236 }
2237
2238 if ((sqc.tail == cqc.tail) &&
2239 (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))
2240 break;
2241
2242 if (i == MAX_WAIT_COUNTS) {
2243 dev_err(dev, "Fail to empty queue %u!\n", qp_id);
2244 *state = QM_STOP_QUEUE_FAIL;
2245 return -ETIMEDOUT;
2246 }
2247
2248 usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);
2249 }
2250
2251 return 0;
2252 }
2253
2254 /**
2255 * qm_drain_qp() - Drain a qp.
2256 * @qp: The qp we want to drain.
2257 *
2258 * If the device does not support stopping queue by sending mailbox,
2259 * determine whether the queue is cleared by judging the tail pointers of
2260 * sq and cq.
2261 */
qm_drain_qp(struct hisi_qp * qp)2262 static int qm_drain_qp(struct hisi_qp *qp)
2263 {
2264 struct hisi_qm *qm = qp->qm;
2265 u32 state = 0;
2266 int ret;
2267
2268 /* No need to judge if master OOO is blocked. */
2269 if (qm_check_dev_error(qm))
2270 return 0;
2271
2272 /* HW V3 supports drain qp by device */
2273 if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) {
2274 ret = qm_stop_qp(qp);
2275 if (ret) {
2276 dev_err(&qm->pdev->dev, "Failed to stop qp!\n");
2277 state = QM_STOP_QUEUE_FAIL;
2278 goto set_dev_state;
2279 }
2280 return ret;
2281 }
2282
2283 ret = qm_wait_qp_empty(qm, &state, qp->qp_id);
2284 if (ret)
2285 goto set_dev_state;
2286
2287 return 0;
2288
2289 set_dev_state:
2290 if (qm->debug.dev_dfx.dev_timeout)
2291 qm->debug.dev_dfx.dev_state = state;
2292
2293 return ret;
2294 }
2295
qm_stop_qp_nolock(struct hisi_qp * qp)2296 static void qm_stop_qp_nolock(struct hisi_qp *qp)
2297 {
2298 struct hisi_qm *qm = qp->qm;
2299 struct device *dev = &qm->pdev->dev;
2300 int ret;
2301
2302 /*
2303 * It is allowed to stop and release qp when reset, If the qp is
2304 * stopped when reset but still want to be released then, the
2305 * is_resetting flag should be set negative so that this qp will not
2306 * be restarted after reset.
2307 */
2308 if (atomic_read(&qp->qp_status.flags) != QP_START) {
2309 qp->is_resetting = false;
2310 return;
2311 }
2312
2313 atomic_set(&qp->qp_status.flags, QP_STOP);
2314
2315 /* V3 supports direct stop function when FLR prepare */
2316 if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) {
2317 ret = qm_drain_qp(qp);
2318 if (ret)
2319 dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", qp->qp_id);
2320 }
2321
2322 flush_workqueue(qm->wq);
2323 if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))
2324 qp_stop_fail_cb(qp);
2325
2326 dev_dbg(dev, "stop queue %u!", qp->qp_id);
2327 }
2328
2329 /**
2330 * hisi_qm_stop_qp() - Stop a qp in qm.
2331 * @qp: The qp we want to stop.
2332 *
2333 * This function is reverse of hisi_qm_start_qp.
2334 */
hisi_qm_stop_qp(struct hisi_qp * qp)2335 void hisi_qm_stop_qp(struct hisi_qp *qp)
2336 {
2337 down_write(&qp->qm->qps_lock);
2338 qm_stop_qp_nolock(qp);
2339 up_write(&qp->qm->qps_lock);
2340 }
2341 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
2342
2343 /**
2344 * hisi_qp_send() - Queue up a task in the hardware queue.
2345 * @qp: The qp in which to put the message.
2346 * @msg: The message.
2347 *
2348 * This function will return -EBUSY if qp is currently full, and -EAGAIN
2349 * if qp related qm is resetting.
2350 *
2351 * Note: This function may run with qm_irq_thread and ACC reset at same time.
2352 * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
2353 * reset may happen, we have no lock here considering performance. This
2354 * causes current qm_db sending fail or can not receive sended sqe. QM
2355 * sync/async receive function should handle the error sqe. ACC reset
2356 * done function should clear used sqe to 0.
2357 */
hisi_qp_send(struct hisi_qp * qp,const void * msg)2358 int hisi_qp_send(struct hisi_qp *qp, const void *msg)
2359 {
2360 struct hisi_qp_status *qp_status = &qp->qp_status;
2361 u16 sq_tail = qp_status->sq_tail;
2362 u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth;
2363 void *sqe = qm_get_avail_sqe(qp);
2364
2365 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
2366 atomic_read(&qp->qm->status.flags) == QM_STOP ||
2367 qp->is_resetting)) {
2368 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
2369 return -EAGAIN;
2370 }
2371
2372 if (!sqe)
2373 return -EBUSY;
2374
2375 memcpy(sqe, msg, qp->qm->sqe_size);
2376
2377 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
2378 atomic_inc(&qp->qp_status.used);
2379 qp_status->sq_tail = sq_tail_next;
2380
2381 return 0;
2382 }
2383 EXPORT_SYMBOL_GPL(hisi_qp_send);
2384
hisi_qm_cache_wb(struct hisi_qm * qm)2385 static void hisi_qm_cache_wb(struct hisi_qm *qm)
2386 {
2387 unsigned int val;
2388
2389 if (qm->ver == QM_HW_V1)
2390 return;
2391
2392 writel(0x1, qm->io_base + QM_CACHE_WB_START);
2393 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
2394 val, val & BIT(0), POLL_PERIOD,
2395 POLL_TIMEOUT))
2396 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
2397 }
2398
qm_qp_event_notifier(struct hisi_qp * qp)2399 static void qm_qp_event_notifier(struct hisi_qp *qp)
2400 {
2401 wake_up_interruptible(&qp->uacce_q->wait);
2402 }
2403
2404 /* This function returns free number of qp in qm. */
hisi_qm_get_available_instances(struct uacce_device * uacce)2405 static int hisi_qm_get_available_instances(struct uacce_device *uacce)
2406 {
2407 struct hisi_qm *qm = uacce->priv;
2408 int ret;
2409
2410 down_read(&qm->qps_lock);
2411 ret = qm->qp_num - qm->qp_in_used;
2412 up_read(&qm->qps_lock);
2413
2414 return ret;
2415 }
2416
hisi_qm_set_hw_reset(struct hisi_qm * qm,int offset)2417 static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset)
2418 {
2419 int i;
2420
2421 for (i = 0; i < qm->qp_num; i++)
2422 qm_set_qp_disable(&qm->qp_array[i], offset);
2423 }
2424
hisi_qm_uacce_get_queue(struct uacce_device * uacce,unsigned long arg,struct uacce_queue * q)2425 static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,
2426 unsigned long arg,
2427 struct uacce_queue *q)
2428 {
2429 struct hisi_qm *qm = uacce->priv;
2430 struct hisi_qp *qp;
2431 u8 alg_type = 0;
2432
2433 qp = hisi_qm_create_qp(qm, alg_type);
2434 if (IS_ERR(qp))
2435 return PTR_ERR(qp);
2436
2437 q->priv = qp;
2438 q->uacce = uacce;
2439 qp->uacce_q = q;
2440 qp->event_cb = qm_qp_event_notifier;
2441 qp->pasid = arg;
2442 qp->is_in_kernel = false;
2443
2444 return 0;
2445 }
2446
hisi_qm_uacce_put_queue(struct uacce_queue * q)2447 static void hisi_qm_uacce_put_queue(struct uacce_queue *q)
2448 {
2449 struct hisi_qp *qp = q->priv;
2450
2451 hisi_qm_release_qp(qp);
2452 }
2453
2454 /* map sq/cq/doorbell to user space */
hisi_qm_uacce_mmap(struct uacce_queue * q,struct vm_area_struct * vma,struct uacce_qfile_region * qfr)2455 static int hisi_qm_uacce_mmap(struct uacce_queue *q,
2456 struct vm_area_struct *vma,
2457 struct uacce_qfile_region *qfr)
2458 {
2459 struct hisi_qp *qp = q->priv;
2460 struct hisi_qm *qm = qp->qm;
2461 resource_size_t phys_base = qm->db_phys_base +
2462 qp->qp_id * qm->db_interval;
2463 size_t sz = vma->vm_end - vma->vm_start;
2464 struct pci_dev *pdev = qm->pdev;
2465 struct device *dev = &pdev->dev;
2466 unsigned long vm_pgoff;
2467 int ret;
2468
2469 switch (qfr->type) {
2470 case UACCE_QFRT_MMIO:
2471 if (qm->ver == QM_HW_V1) {
2472 if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
2473 return -EINVAL;
2474 } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {
2475 if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
2476 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
2477 return -EINVAL;
2478 } else {
2479 if (sz > qm->db_interval)
2480 return -EINVAL;
2481 }
2482
2483 vm_flags_set(vma, VM_IO);
2484
2485 return remap_pfn_range(vma, vma->vm_start,
2486 phys_base >> PAGE_SHIFT,
2487 sz, pgprot_noncached(vma->vm_page_prot));
2488 case UACCE_QFRT_DUS:
2489 if (sz != qp->qdma.size)
2490 return -EINVAL;
2491
2492 /*
2493 * dma_mmap_coherent() requires vm_pgoff as 0
2494 * restore vm_pfoff to initial value for mmap()
2495 */
2496 vm_pgoff = vma->vm_pgoff;
2497 vma->vm_pgoff = 0;
2498 ret = dma_mmap_coherent(dev, vma, qp->qdma.va,
2499 qp->qdma.dma, sz);
2500 vma->vm_pgoff = vm_pgoff;
2501 return ret;
2502
2503 default:
2504 return -EINVAL;
2505 }
2506 }
2507
hisi_qm_uacce_start_queue(struct uacce_queue * q)2508 static int hisi_qm_uacce_start_queue(struct uacce_queue *q)
2509 {
2510 struct hisi_qp *qp = q->priv;
2511
2512 return hisi_qm_start_qp(qp, qp->pasid);
2513 }
2514
hisi_qm_uacce_stop_queue(struct uacce_queue * q)2515 static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)
2516 {
2517 struct hisi_qp *qp = q->priv;
2518 struct hisi_qm *qm = qp->qm;
2519 struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx;
2520 u32 i = 0;
2521
2522 hisi_qm_stop_qp(qp);
2523
2524 if (!dev_dfx->dev_timeout || !dev_dfx->dev_state)
2525 return;
2526
2527 /*
2528 * After the queue fails to be stopped,
2529 * wait for a period of time before releasing the queue.
2530 */
2531 while (++i) {
2532 msleep(WAIT_PERIOD);
2533
2534 /* Since dev_timeout maybe modified, check i >= dev_timeout */
2535 if (i >= dev_dfx->dev_timeout) {
2536 dev_err(&qm->pdev->dev, "Stop q %u timeout, state %u\n",
2537 qp->qp_id, dev_dfx->dev_state);
2538 dev_dfx->dev_state = QM_FINISH_WAIT;
2539 break;
2540 }
2541 }
2542 }
2543
hisi_qm_is_q_updated(struct uacce_queue * q)2544 static int hisi_qm_is_q_updated(struct uacce_queue *q)
2545 {
2546 struct hisi_qp *qp = q->priv;
2547 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
2548 int updated = 0;
2549
2550 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
2551 /* make sure to read data from memory */
2552 dma_rmb();
2553 qm_cq_head_update(qp);
2554 cqe = qp->cqe + qp->qp_status.cq_head;
2555 updated = 1;
2556 }
2557
2558 return updated;
2559 }
2560
qm_set_sqctype(struct uacce_queue * q,u16 type)2561 static void qm_set_sqctype(struct uacce_queue *q, u16 type)
2562 {
2563 struct hisi_qm *qm = q->uacce->priv;
2564 struct hisi_qp *qp = q->priv;
2565
2566 down_write(&qm->qps_lock);
2567 qp->alg_type = type;
2568 up_write(&qm->qps_lock);
2569 }
2570
hisi_qm_uacce_ioctl(struct uacce_queue * q,unsigned int cmd,unsigned long arg)2571 static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,
2572 unsigned long arg)
2573 {
2574 struct hisi_qp *qp = q->priv;
2575 struct hisi_qp_info qp_info;
2576 struct hisi_qp_ctx qp_ctx;
2577
2578 if (cmd == UACCE_CMD_QM_SET_QP_CTX) {
2579 if (copy_from_user(&qp_ctx, (void __user *)arg,
2580 sizeof(struct hisi_qp_ctx)))
2581 return -EFAULT;
2582
2583 if (qp_ctx.qc_type > QM_MAX_QC_TYPE)
2584 return -EINVAL;
2585
2586 qm_set_sqctype(q, qp_ctx.qc_type);
2587 qp_ctx.id = qp->qp_id;
2588
2589 if (copy_to_user((void __user *)arg, &qp_ctx,
2590 sizeof(struct hisi_qp_ctx)))
2591 return -EFAULT;
2592
2593 return 0;
2594 } else if (cmd == UACCE_CMD_QM_SET_QP_INFO) {
2595 if (copy_from_user(&qp_info, (void __user *)arg,
2596 sizeof(struct hisi_qp_info)))
2597 return -EFAULT;
2598
2599 qp_info.sqe_size = qp->qm->sqe_size;
2600 qp_info.sq_depth = qp->sq_depth;
2601 qp_info.cq_depth = qp->cq_depth;
2602
2603 if (copy_to_user((void __user *)arg, &qp_info,
2604 sizeof(struct hisi_qp_info)))
2605 return -EFAULT;
2606
2607 return 0;
2608 }
2609
2610 return -EINVAL;
2611 }
2612
2613 /**
2614 * qm_hw_err_isolate() - Try to set the isolation status of the uacce device
2615 * according to user's configuration of error threshold.
2616 * @qm: the uacce device
2617 */
qm_hw_err_isolate(struct hisi_qm * qm)2618 static int qm_hw_err_isolate(struct hisi_qm *qm)
2619 {
2620 struct qm_hw_err *err, *tmp, *hw_err;
2621 struct qm_err_isolate *isolate;
2622 u32 count = 0;
2623
2624 isolate = &qm->isolate_data;
2625
2626 #define SECONDS_PER_HOUR 3600
2627
2628 /* All the hw errs are processed by PF driver */
2629 if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold)
2630 return 0;
2631
2632 hw_err = kzalloc(sizeof(*hw_err), GFP_KERNEL);
2633 if (!hw_err)
2634 return -ENOMEM;
2635
2636 /*
2637 * Time-stamp every slot AER error. Then check the AER error log when the
2638 * next device AER error occurred. if the device slot AER error count exceeds
2639 * the setting error threshold in one hour, the isolated state will be set
2640 * to true. And the AER error logs that exceed one hour will be cleared.
2641 */
2642 mutex_lock(&isolate->isolate_lock);
2643 hw_err->timestamp = jiffies;
2644 list_for_each_entry_safe(err, tmp, &isolate->qm_hw_errs, list) {
2645 if ((hw_err->timestamp - err->timestamp) / HZ >
2646 SECONDS_PER_HOUR) {
2647 list_del(&err->list);
2648 kfree(err);
2649 } else {
2650 count++;
2651 }
2652 }
2653 list_add(&hw_err->list, &isolate->qm_hw_errs);
2654 mutex_unlock(&isolate->isolate_lock);
2655
2656 if (count >= isolate->err_threshold)
2657 isolate->is_isolate = true;
2658
2659 return 0;
2660 }
2661
qm_hw_err_destroy(struct hisi_qm * qm)2662 static void qm_hw_err_destroy(struct hisi_qm *qm)
2663 {
2664 struct qm_hw_err *err, *tmp;
2665
2666 mutex_lock(&qm->isolate_data.isolate_lock);
2667 list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) {
2668 list_del(&err->list);
2669 kfree(err);
2670 }
2671 mutex_unlock(&qm->isolate_data.isolate_lock);
2672 }
2673
hisi_qm_get_isolate_state(struct uacce_device * uacce)2674 static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce)
2675 {
2676 struct hisi_qm *qm = uacce->priv;
2677 struct hisi_qm *pf_qm;
2678
2679 if (uacce->is_vf)
2680 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
2681 else
2682 pf_qm = qm;
2683
2684 return pf_qm->isolate_data.is_isolate ?
2685 UACCE_DEV_ISOLATE : UACCE_DEV_NORMAL;
2686 }
2687
hisi_qm_isolate_threshold_write(struct uacce_device * uacce,u32 num)2688 static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num)
2689 {
2690 struct hisi_qm *qm = uacce->priv;
2691
2692 /* Must be set by PF */
2693 if (uacce->is_vf)
2694 return -EPERM;
2695
2696 if (qm->isolate_data.is_isolate)
2697 return -EPERM;
2698
2699 qm->isolate_data.err_threshold = num;
2700
2701 /* After the policy is updated, need to reset the hardware err list */
2702 qm_hw_err_destroy(qm);
2703
2704 return 0;
2705 }
2706
hisi_qm_isolate_threshold_read(struct uacce_device * uacce)2707 static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce)
2708 {
2709 struct hisi_qm *qm = uacce->priv;
2710 struct hisi_qm *pf_qm;
2711
2712 if (uacce->is_vf) {
2713 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
2714 return pf_qm->isolate_data.err_threshold;
2715 }
2716
2717 return qm->isolate_data.err_threshold;
2718 }
2719
2720 static const struct uacce_ops uacce_qm_ops = {
2721 .get_available_instances = hisi_qm_get_available_instances,
2722 .get_queue = hisi_qm_uacce_get_queue,
2723 .put_queue = hisi_qm_uacce_put_queue,
2724 .start_queue = hisi_qm_uacce_start_queue,
2725 .stop_queue = hisi_qm_uacce_stop_queue,
2726 .mmap = hisi_qm_uacce_mmap,
2727 .ioctl = hisi_qm_uacce_ioctl,
2728 .is_q_updated = hisi_qm_is_q_updated,
2729 .get_isolate_state = hisi_qm_get_isolate_state,
2730 .isolate_err_threshold_write = hisi_qm_isolate_threshold_write,
2731 .isolate_err_threshold_read = hisi_qm_isolate_threshold_read,
2732 };
2733
qm_remove_uacce(struct hisi_qm * qm)2734 static void qm_remove_uacce(struct hisi_qm *qm)
2735 {
2736 struct uacce_device *uacce = qm->uacce;
2737
2738 if (qm->use_sva) {
2739 qm_hw_err_destroy(qm);
2740 uacce_remove(uacce);
2741 qm->uacce = NULL;
2742 }
2743 }
2744
qm_alloc_uacce(struct hisi_qm * qm)2745 static int qm_alloc_uacce(struct hisi_qm *qm)
2746 {
2747 struct pci_dev *pdev = qm->pdev;
2748 struct uacce_device *uacce;
2749 unsigned long mmio_page_nr;
2750 unsigned long dus_page_nr;
2751 u16 sq_depth, cq_depth;
2752 struct uacce_interface interface = {
2753 .flags = UACCE_DEV_SVA,
2754 .ops = &uacce_qm_ops,
2755 };
2756 int ret;
2757
2758 ret = strscpy(interface.name, dev_driver_string(&pdev->dev),
2759 sizeof(interface.name));
2760 if (ret < 0)
2761 return -ENAMETOOLONG;
2762
2763 uacce = uacce_alloc(&pdev->dev, &interface);
2764 if (IS_ERR(uacce))
2765 return PTR_ERR(uacce);
2766
2767 if (uacce->flags & UACCE_DEV_SVA) {
2768 qm->use_sva = true;
2769 } else {
2770 /* only consider sva case */
2771 qm_remove_uacce(qm);
2772 return -EINVAL;
2773 }
2774
2775 uacce->is_vf = pdev->is_virtfn;
2776 uacce->priv = qm;
2777
2778 if (qm->ver == QM_HW_V1)
2779 uacce->api_ver = HISI_QM_API_VER_BASE;
2780 else if (qm->ver == QM_HW_V2)
2781 uacce->api_ver = HISI_QM_API_VER2_BASE;
2782 else
2783 uacce->api_ver = HISI_QM_API_VER3_BASE;
2784
2785 if (qm->ver == QM_HW_V1)
2786 mmio_page_nr = QM_DOORBELL_PAGE_NR;
2787 else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
2788 mmio_page_nr = QM_DOORBELL_PAGE_NR +
2789 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
2790 else
2791 mmio_page_nr = qm->db_interval / PAGE_SIZE;
2792
2793 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);
2794
2795 /* Add one more page for device or qp status */
2796 dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth +
2797 sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >>
2798 PAGE_SHIFT;
2799
2800 uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;
2801 uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr;
2802
2803 qm->uacce = uacce;
2804 INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs);
2805 mutex_init(&qm->isolate_data.isolate_lock);
2806
2807 return 0;
2808 }
2809
2810 /**
2811 * qm_frozen() - Try to froze QM to cut continuous queue request. If
2812 * there is user on the QM, return failure without doing anything.
2813 * @qm: The qm needed to be fronzen.
2814 *
2815 * This function frozes QM, then we can do SRIOV disabling.
2816 */
qm_frozen(struct hisi_qm * qm)2817 static int qm_frozen(struct hisi_qm *qm)
2818 {
2819 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl))
2820 return 0;
2821
2822 down_write(&qm->qps_lock);
2823
2824 if (!qm->qp_in_used) {
2825 qm->qp_in_used = qm->qp_num;
2826 up_write(&qm->qps_lock);
2827 set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl);
2828 return 0;
2829 }
2830
2831 up_write(&qm->qps_lock);
2832
2833 return -EBUSY;
2834 }
2835
qm_try_frozen_vfs(struct pci_dev * pdev,struct hisi_qm_list * qm_list)2836 static int qm_try_frozen_vfs(struct pci_dev *pdev,
2837 struct hisi_qm_list *qm_list)
2838 {
2839 struct hisi_qm *qm, *vf_qm;
2840 struct pci_dev *dev;
2841 int ret = 0;
2842
2843 if (!qm_list || !pdev)
2844 return -EINVAL;
2845
2846 /* Try to frozen all the VFs as disable SRIOV */
2847 mutex_lock(&qm_list->lock);
2848 list_for_each_entry(qm, &qm_list->list, list) {
2849 dev = qm->pdev;
2850 if (dev == pdev)
2851 continue;
2852 if (pci_physfn(dev) == pdev) {
2853 vf_qm = pci_get_drvdata(dev);
2854 ret = qm_frozen(vf_qm);
2855 if (ret)
2856 goto frozen_fail;
2857 }
2858 }
2859
2860 frozen_fail:
2861 mutex_unlock(&qm_list->lock);
2862
2863 return ret;
2864 }
2865
2866 /**
2867 * hisi_qm_wait_task_finish() - Wait until the task is finished
2868 * when removing the driver.
2869 * @qm: The qm needed to wait for the task to finish.
2870 * @qm_list: The list of all available devices.
2871 */
hisi_qm_wait_task_finish(struct hisi_qm * qm,struct hisi_qm_list * qm_list)2872 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
2873 {
2874 while (qm_frozen(qm) ||
2875 ((qm->fun_type == QM_HW_PF) &&
2876 qm_try_frozen_vfs(qm->pdev, qm_list))) {
2877 msleep(WAIT_PERIOD);
2878 }
2879
2880 while (test_bit(QM_RST_SCHED, &qm->misc_ctl) ||
2881 test_bit(QM_RESETTING, &qm->misc_ctl))
2882 msleep(WAIT_PERIOD);
2883
2884 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2885 flush_work(&qm->cmd_process);
2886
2887 udelay(REMOVE_WAIT_DELAY);
2888 }
2889 EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
2890
hisi_qp_memory_uninit(struct hisi_qm * qm,int num)2891 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
2892 {
2893 struct device *dev = &qm->pdev->dev;
2894 struct qm_dma *qdma;
2895 int i;
2896
2897 for (i = num - 1; i >= 0; i--) {
2898 qdma = &qm->qp_array[i].qdma;
2899 dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
2900 kfree(qm->poll_data[i].qp_finish_id);
2901 }
2902
2903 kfree(qm->poll_data);
2904 kfree(qm->qp_array);
2905 }
2906
hisi_qp_memory_init(struct hisi_qm * qm,size_t dma_size,int id,u16 sq_depth,u16 cq_depth)2907 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id,
2908 u16 sq_depth, u16 cq_depth)
2909 {
2910 struct device *dev = &qm->pdev->dev;
2911 size_t off = qm->sqe_size * sq_depth;
2912 struct hisi_qp *qp;
2913 int ret = -ENOMEM;
2914
2915 qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16),
2916 GFP_KERNEL);
2917 if (!qm->poll_data[id].qp_finish_id)
2918 return -ENOMEM;
2919
2920 qp = &qm->qp_array[id];
2921 qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
2922 GFP_KERNEL);
2923 if (!qp->qdma.va)
2924 goto err_free_qp_finish_id;
2925
2926 qp->sqe = qp->qdma.va;
2927 qp->sqe_dma = qp->qdma.dma;
2928 qp->cqe = qp->qdma.va + off;
2929 qp->cqe_dma = qp->qdma.dma + off;
2930 qp->qdma.size = dma_size;
2931 qp->sq_depth = sq_depth;
2932 qp->cq_depth = cq_depth;
2933 qp->qm = qm;
2934 qp->qp_id = id;
2935
2936 return 0;
2937
2938 err_free_qp_finish_id:
2939 kfree(qm->poll_data[id].qp_finish_id);
2940 return ret;
2941 }
2942
hisi_qm_pre_init(struct hisi_qm * qm)2943 static void hisi_qm_pre_init(struct hisi_qm *qm)
2944 {
2945 struct pci_dev *pdev = qm->pdev;
2946
2947 if (qm->ver == QM_HW_V1)
2948 qm->ops = &qm_hw_ops_v1;
2949 else if (qm->ver == QM_HW_V2)
2950 qm->ops = &qm_hw_ops_v2;
2951 else if (qm->ver == QM_HW_V3)
2952 qm->ops = &qm_hw_ops_v3;
2953 else
2954 qm->ops = &qm_hw_ops_v4;
2955
2956 pci_set_drvdata(pdev, qm);
2957 mutex_init(&qm->mailbox_lock);
2958 mutex_init(&qm->ifc_lock);
2959 init_rwsem(&qm->qps_lock);
2960 qm->qp_in_used = 0;
2961 if (test_bit(QM_SUPPORT_RPM, &qm->caps)) {
2962 if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev)))
2963 dev_info(&pdev->dev, "_PS0 and _PR0 are not defined");
2964 }
2965 }
2966
qm_cmd_uninit(struct hisi_qm * qm)2967 static void qm_cmd_uninit(struct hisi_qm *qm)
2968 {
2969 u32 val;
2970
2971 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2972 return;
2973
2974 val = readl(qm->io_base + QM_IFC_INT_MASK);
2975 val |= QM_IFC_INT_DISABLE;
2976 writel(val, qm->io_base + QM_IFC_INT_MASK);
2977 }
2978
qm_cmd_init(struct hisi_qm * qm)2979 static void qm_cmd_init(struct hisi_qm *qm)
2980 {
2981 u32 val;
2982
2983 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2984 return;
2985
2986 /* Clear communication interrupt source */
2987 qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR);
2988
2989 /* Enable pf to vf communication reg. */
2990 val = readl(qm->io_base + QM_IFC_INT_MASK);
2991 val &= ~QM_IFC_INT_DISABLE;
2992 writel(val, qm->io_base + QM_IFC_INT_MASK);
2993 }
2994
qm_put_pci_res(struct hisi_qm * qm)2995 static void qm_put_pci_res(struct hisi_qm *qm)
2996 {
2997 struct pci_dev *pdev = qm->pdev;
2998
2999 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
3000 iounmap(qm->db_io_base);
3001
3002 iounmap(qm->io_base);
3003 pci_release_mem_regions(pdev);
3004 }
3005
hisi_qm_pci_uninit(struct hisi_qm * qm)3006 static void hisi_qm_pci_uninit(struct hisi_qm *qm)
3007 {
3008 struct pci_dev *pdev = qm->pdev;
3009
3010 pci_free_irq_vectors(pdev);
3011 qm_put_pci_res(qm);
3012 pci_disable_device(pdev);
3013 }
3014
hisi_qm_set_state(struct hisi_qm * qm,u8 state)3015 static void hisi_qm_set_state(struct hisi_qm *qm, u8 state)
3016 {
3017 if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF)
3018 writel(state, qm->io_base + QM_VF_STATE);
3019 }
3020
hisi_qm_unint_work(struct hisi_qm * qm)3021 static void hisi_qm_unint_work(struct hisi_qm *qm)
3022 {
3023 destroy_workqueue(qm->wq);
3024 }
3025
hisi_qm_free_rsv_buf(struct hisi_qm * qm)3026 static void hisi_qm_free_rsv_buf(struct hisi_qm *qm)
3027 {
3028 struct qm_dma *xqc_dma = &qm->xqc_buf.qcdma;
3029 struct device *dev = &qm->pdev->dev;
3030
3031 dma_free_coherent(dev, xqc_dma->size, xqc_dma->va, xqc_dma->dma);
3032 }
3033
hisi_qm_memory_uninit(struct hisi_qm * qm)3034 static void hisi_qm_memory_uninit(struct hisi_qm *qm)
3035 {
3036 struct device *dev = &qm->pdev->dev;
3037
3038 hisi_qp_memory_uninit(qm, qm->qp_num);
3039 hisi_qm_free_rsv_buf(qm);
3040 if (qm->qdma.va) {
3041 hisi_qm_cache_wb(qm);
3042 dma_free_coherent(dev, qm->qdma.size,
3043 qm->qdma.va, qm->qdma.dma);
3044 }
3045
3046 idr_destroy(&qm->qp_idr);
3047
3048 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
3049 kfree(qm->factor);
3050 }
3051
3052 /**
3053 * hisi_qm_uninit() - Uninitialize qm.
3054 * @qm: The qm needed uninit.
3055 *
3056 * This function uninits qm related device resources.
3057 */
hisi_qm_uninit(struct hisi_qm * qm)3058 void hisi_qm_uninit(struct hisi_qm *qm)
3059 {
3060 qm_cmd_uninit(qm);
3061 hisi_qm_unint_work(qm);
3062
3063 down_write(&qm->qps_lock);
3064 hisi_qm_memory_uninit(qm);
3065 hisi_qm_set_state(qm, QM_NOT_READY);
3066 up_write(&qm->qps_lock);
3067
3068 qm_remove_uacce(qm);
3069 qm_irqs_unregister(qm);
3070 hisi_qm_pci_uninit(qm);
3071 }
3072 EXPORT_SYMBOL_GPL(hisi_qm_uninit);
3073
3074 /**
3075 * hisi_qm_get_vft() - Get vft from a qm.
3076 * @qm: The qm we want to get its vft.
3077 * @base: The base number of queue in vft.
3078 * @number: The number of queues in vft.
3079 *
3080 * We can allocate multiple queues to a qm by configuring virtual function
3081 * table. We get related configures by this function. Normally, we call this
3082 * function in VF driver to get the queue information.
3083 *
3084 * qm hw v1 does not support this interface.
3085 */
hisi_qm_get_vft(struct hisi_qm * qm,u32 * base,u32 * number)3086 static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
3087 {
3088 if (!base || !number)
3089 return -EINVAL;
3090
3091 if (!qm->ops->get_vft) {
3092 dev_err(&qm->pdev->dev, "Don't support vft read!\n");
3093 return -EINVAL;
3094 }
3095
3096 return qm->ops->get_vft(qm, base, number);
3097 }
3098
3099 /**
3100 * hisi_qm_set_vft() - Set vft to a qm.
3101 * @qm: The qm we want to set its vft.
3102 * @fun_num: The function number.
3103 * @base: The base number of queue in vft.
3104 * @number: The number of queues in vft.
3105 *
3106 * This function is alway called in PF driver, it is used to assign queues
3107 * among PF and VFs.
3108 *
3109 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
3110 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
3111 * (VF function number 0x2)
3112 */
hisi_qm_set_vft(struct hisi_qm * qm,u32 fun_num,u32 base,u32 number)3113 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
3114 u32 number)
3115 {
3116 u32 max_q_num = qm->ctrl_qp_num;
3117
3118 if (base >= max_q_num || number > max_q_num ||
3119 (base + number) > max_q_num)
3120 return -EINVAL;
3121
3122 return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
3123 }
3124
qm_init_eq_aeq_status(struct hisi_qm * qm)3125 static void qm_init_eq_aeq_status(struct hisi_qm *qm)
3126 {
3127 struct hisi_qm_status *status = &qm->status;
3128
3129 status->eq_head = 0;
3130 status->aeq_head = 0;
3131 status->eqc_phase = true;
3132 status->aeqc_phase = true;
3133 }
3134
qm_enable_eq_aeq_interrupts(struct hisi_qm * qm)3135 static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm)
3136 {
3137 /* Clear eq/aeq interrupt source */
3138 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
3139 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
3140
3141 writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
3142 writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
3143 }
3144
qm_disable_eq_aeq_interrupts(struct hisi_qm * qm)3145 static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm)
3146 {
3147 writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
3148 writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
3149 }
3150
qm_eq_ctx_cfg(struct hisi_qm * qm)3151 static int qm_eq_ctx_cfg(struct hisi_qm *qm)
3152 {
3153 struct qm_eqc eqc = {0};
3154
3155 eqc.base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));
3156 eqc.base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
3157 if (qm->ver == QM_HW_V1)
3158 eqc.dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
3159 eqc.dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));
3160
3161 return qm_set_and_get_xqc(qm, QM_MB_CMD_EQC, &eqc, 0, 0);
3162 }
3163
qm_aeq_ctx_cfg(struct hisi_qm * qm)3164 static int qm_aeq_ctx_cfg(struct hisi_qm *qm)
3165 {
3166 struct qm_aeqc aeqc = {0};
3167
3168 aeqc.base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));
3169 aeqc.base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));
3170 aeqc.dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));
3171
3172 return qm_set_and_get_xqc(qm, QM_MB_CMD_AEQC, &aeqc, 0, 0);
3173 }
3174
qm_eq_aeq_ctx_cfg(struct hisi_qm * qm)3175 static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)
3176 {
3177 struct device *dev = &qm->pdev->dev;
3178 int ret;
3179
3180 qm_init_eq_aeq_status(qm);
3181
3182 ret = qm_eq_ctx_cfg(qm);
3183 if (ret) {
3184 dev_err(dev, "Set eqc failed!\n");
3185 return ret;
3186 }
3187
3188 return qm_aeq_ctx_cfg(qm);
3189 }
3190
__hisi_qm_start(struct hisi_qm * qm)3191 static int __hisi_qm_start(struct hisi_qm *qm)
3192 {
3193 int ret;
3194
3195 WARN_ON(!qm->qdma.va);
3196
3197 if (qm->fun_type == QM_HW_PF) {
3198 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
3199 if (ret)
3200 return ret;
3201 }
3202
3203 ret = qm_eq_aeq_ctx_cfg(qm);
3204 if (ret)
3205 return ret;
3206
3207 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
3208 if (ret)
3209 return ret;
3210
3211 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
3212 if (ret)
3213 return ret;
3214
3215 qm_init_prefetch(qm);
3216 qm_enable_eq_aeq_interrupts(qm);
3217
3218 return 0;
3219 }
3220
3221 /**
3222 * hisi_qm_start() - start qm
3223 * @qm: The qm to be started.
3224 *
3225 * This function starts a qm, then we can allocate qp from this qm.
3226 */
hisi_qm_start(struct hisi_qm * qm)3227 int hisi_qm_start(struct hisi_qm *qm)
3228 {
3229 struct device *dev = &qm->pdev->dev;
3230 int ret = 0;
3231
3232 down_write(&qm->qps_lock);
3233
3234 dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);
3235
3236 if (!qm->qp_num) {
3237 dev_err(dev, "qp_num should not be 0\n");
3238 ret = -EINVAL;
3239 goto err_unlock;
3240 }
3241
3242 ret = __hisi_qm_start(qm);
3243 if (ret)
3244 goto err_unlock;
3245
3246 atomic_set(&qm->status.flags, QM_WORK);
3247 hisi_qm_set_state(qm, QM_READY);
3248
3249 err_unlock:
3250 up_write(&qm->qps_lock);
3251 return ret;
3252 }
3253 EXPORT_SYMBOL_GPL(hisi_qm_start);
3254
qm_restart(struct hisi_qm * qm)3255 static int qm_restart(struct hisi_qm *qm)
3256 {
3257 struct device *dev = &qm->pdev->dev;
3258 struct hisi_qp *qp;
3259 int ret, i;
3260
3261 ret = hisi_qm_start(qm);
3262 if (ret < 0)
3263 return ret;
3264
3265 down_write(&qm->qps_lock);
3266 for (i = 0; i < qm->qp_num; i++) {
3267 qp = &qm->qp_array[i];
3268 if (atomic_read(&qp->qp_status.flags) == QP_STOP &&
3269 qp->is_resetting == true) {
3270 ret = qm_start_qp_nolock(qp, 0);
3271 if (ret < 0) {
3272 dev_err(dev, "Failed to start qp%d!\n", i);
3273
3274 up_write(&qm->qps_lock);
3275 return ret;
3276 }
3277 qp->is_resetting = false;
3278 }
3279 }
3280 up_write(&qm->qps_lock);
3281
3282 return 0;
3283 }
3284
3285 /* Stop started qps in reset flow */
qm_stop_started_qp(struct hisi_qm * qm)3286 static void qm_stop_started_qp(struct hisi_qm *qm)
3287 {
3288 struct hisi_qp *qp;
3289 int i;
3290
3291 for (i = 0; i < qm->qp_num; i++) {
3292 qp = &qm->qp_array[i];
3293 if (atomic_read(&qp->qp_status.flags) == QP_START) {
3294 qp->is_resetting = true;
3295 qm_stop_qp_nolock(qp);
3296 }
3297 }
3298 }
3299
3300 /**
3301 * qm_clear_queues() - Clear all queues memory in a qm.
3302 * @qm: The qm in which the queues will be cleared.
3303 *
3304 * This function clears all queues memory in a qm. Reset of accelerator can
3305 * use this to clear queues.
3306 */
qm_clear_queues(struct hisi_qm * qm)3307 static void qm_clear_queues(struct hisi_qm *qm)
3308 {
3309 struct hisi_qp *qp;
3310 int i;
3311
3312 for (i = 0; i < qm->qp_num; i++) {
3313 qp = &qm->qp_array[i];
3314 if (qp->is_in_kernel && qp->is_resetting)
3315 memset(qp->qdma.va, 0, qp->qdma.size);
3316 }
3317
3318 memset(qm->qdma.va, 0, qm->qdma.size);
3319 }
3320
3321 /**
3322 * hisi_qm_stop() - Stop a qm.
3323 * @qm: The qm which will be stopped.
3324 * @r: The reason to stop qm.
3325 *
3326 * This function stops qm and its qps, then qm can not accept request.
3327 * Related resources are not released at this state, we can use hisi_qm_start
3328 * to let qm start again.
3329 */
hisi_qm_stop(struct hisi_qm * qm,enum qm_stop_reason r)3330 int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
3331 {
3332 struct device *dev = &qm->pdev->dev;
3333 int ret = 0;
3334
3335 down_write(&qm->qps_lock);
3336
3337 if (atomic_read(&qm->status.flags) == QM_STOP)
3338 goto err_unlock;
3339
3340 /* Stop all the request sending at first. */
3341 atomic_set(&qm->status.flags, QM_STOP);
3342 qm->status.stop_reason = r;
3343
3344 if (qm->status.stop_reason != QM_NORMAL) {
3345 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
3346 /*
3347 * When performing soft reset, the hardware will no longer
3348 * do tasks, and the tasks in the device will be flushed
3349 * out directly since the master ooo is closed.
3350 */
3351 if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) &&
3352 r != QM_SOFT_RESET) {
3353 ret = qm_drain_qm(qm);
3354 if (ret) {
3355 dev_err(dev, "failed to drain qm!\n");
3356 goto err_unlock;
3357 }
3358 }
3359
3360 qm_stop_started_qp(qm);
3361
3362 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
3363 }
3364
3365 qm_disable_eq_aeq_interrupts(qm);
3366 if (qm->fun_type == QM_HW_PF) {
3367 ret = hisi_qm_set_vft(qm, 0, 0, 0);
3368 if (ret < 0) {
3369 dev_err(dev, "Failed to set vft!\n");
3370 ret = -EBUSY;
3371 goto err_unlock;
3372 }
3373 }
3374
3375 qm_clear_queues(qm);
3376 qm->status.stop_reason = QM_NORMAL;
3377
3378 err_unlock:
3379 up_write(&qm->qps_lock);
3380 return ret;
3381 }
3382 EXPORT_SYMBOL_GPL(hisi_qm_stop);
3383
qm_hw_error_init(struct hisi_qm * qm)3384 static void qm_hw_error_init(struct hisi_qm *qm)
3385 {
3386 if (!qm->ops->hw_error_init) {
3387 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
3388 return;
3389 }
3390
3391 qm->ops->hw_error_init(qm);
3392 }
3393
qm_hw_error_uninit(struct hisi_qm * qm)3394 static void qm_hw_error_uninit(struct hisi_qm *qm)
3395 {
3396 if (!qm->ops->hw_error_uninit) {
3397 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");
3398 return;
3399 }
3400
3401 qm->ops->hw_error_uninit(qm);
3402 }
3403
qm_hw_error_handle(struct hisi_qm * qm)3404 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)
3405 {
3406 if (!qm->ops->hw_error_handle) {
3407 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");
3408 return ACC_ERR_NONE;
3409 }
3410
3411 return qm->ops->hw_error_handle(qm);
3412 }
3413
3414 /**
3415 * hisi_qm_dev_err_init() - Initialize device error configuration.
3416 * @qm: The qm for which we want to do error initialization.
3417 *
3418 * Initialize QM and device error related configuration.
3419 */
hisi_qm_dev_err_init(struct hisi_qm * qm)3420 void hisi_qm_dev_err_init(struct hisi_qm *qm)
3421 {
3422 if (qm->fun_type == QM_HW_VF)
3423 return;
3424
3425 qm_hw_error_init(qm);
3426
3427 if (!qm->err_ini->hw_err_enable) {
3428 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");
3429 return;
3430 }
3431 qm->err_ini->hw_err_enable(qm);
3432 }
3433 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);
3434
3435 /**
3436 * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.
3437 * @qm: The qm for which we want to do error uninitialization.
3438 *
3439 * Uninitialize QM and device error related configuration.
3440 */
hisi_qm_dev_err_uninit(struct hisi_qm * qm)3441 void hisi_qm_dev_err_uninit(struct hisi_qm *qm)
3442 {
3443 if (qm->fun_type == QM_HW_VF)
3444 return;
3445
3446 qm_hw_error_uninit(qm);
3447
3448 if (!qm->err_ini->hw_err_disable) {
3449 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");
3450 return;
3451 }
3452 qm->err_ini->hw_err_disable(qm);
3453 }
3454 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);
3455
3456 /**
3457 * hisi_qm_free_qps() - free multiple queue pairs.
3458 * @qps: The queue pairs need to be freed.
3459 * @qp_num: The num of queue pairs.
3460 */
hisi_qm_free_qps(struct hisi_qp ** qps,int qp_num)3461 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)
3462 {
3463 int i;
3464
3465 if (!qps || qp_num <= 0)
3466 return;
3467
3468 for (i = qp_num - 1; i >= 0; i--)
3469 hisi_qm_release_qp(qps[i]);
3470 }
3471 EXPORT_SYMBOL_GPL(hisi_qm_free_qps);
3472
free_list(struct list_head * head)3473 static void free_list(struct list_head *head)
3474 {
3475 struct hisi_qm_resource *res, *tmp;
3476
3477 list_for_each_entry_safe(res, tmp, head, list) {
3478 list_del(&res->list);
3479 kfree(res);
3480 }
3481 }
3482
hisi_qm_sort_devices(int node,struct list_head * head,struct hisi_qm_list * qm_list)3483 static int hisi_qm_sort_devices(int node, struct list_head *head,
3484 struct hisi_qm_list *qm_list)
3485 {
3486 struct hisi_qm_resource *res, *tmp;
3487 struct hisi_qm *qm;
3488 struct list_head *n;
3489 struct device *dev;
3490 int dev_node;
3491
3492 list_for_each_entry(qm, &qm_list->list, list) {
3493 dev = &qm->pdev->dev;
3494
3495 dev_node = dev_to_node(dev);
3496 if (dev_node < 0)
3497 dev_node = 0;
3498
3499 res = kzalloc(sizeof(*res), GFP_KERNEL);
3500 if (!res)
3501 return -ENOMEM;
3502
3503 res->qm = qm;
3504 res->distance = node_distance(dev_node, node);
3505 n = head;
3506 list_for_each_entry(tmp, head, list) {
3507 if (res->distance < tmp->distance) {
3508 n = &tmp->list;
3509 break;
3510 }
3511 }
3512 list_add_tail(&res->list, n);
3513 }
3514
3515 return 0;
3516 }
3517
3518 /**
3519 * hisi_qm_alloc_qps_node() - Create multiple queue pairs.
3520 * @qm_list: The list of all available devices.
3521 * @qp_num: The number of queue pairs need created.
3522 * @alg_type: The algorithm type.
3523 * @node: The numa node.
3524 * @qps: The queue pairs need created.
3525 *
3526 * This function will sort all available device according to numa distance.
3527 * Then try to create all queue pairs from one device, if all devices do
3528 * not meet the requirements will return error.
3529 */
hisi_qm_alloc_qps_node(struct hisi_qm_list * qm_list,int qp_num,u8 alg_type,int node,struct hisi_qp ** qps)3530 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
3531 u8 alg_type, int node, struct hisi_qp **qps)
3532 {
3533 struct hisi_qm_resource *tmp;
3534 int ret = -ENODEV;
3535 LIST_HEAD(head);
3536 int i;
3537
3538 if (!qps || !qm_list || qp_num <= 0)
3539 return -EINVAL;
3540
3541 mutex_lock(&qm_list->lock);
3542 if (hisi_qm_sort_devices(node, &head, qm_list)) {
3543 mutex_unlock(&qm_list->lock);
3544 goto err;
3545 }
3546
3547 list_for_each_entry(tmp, &head, list) {
3548 for (i = 0; i < qp_num; i++) {
3549 qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);
3550 if (IS_ERR(qps[i])) {
3551 hisi_qm_free_qps(qps, i);
3552 break;
3553 }
3554 }
3555
3556 if (i == qp_num) {
3557 ret = 0;
3558 break;
3559 }
3560 }
3561
3562 mutex_unlock(&qm_list->lock);
3563 if (ret)
3564 pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n",
3565 node, alg_type, qp_num);
3566
3567 err:
3568 free_list(&head);
3569 return ret;
3570 }
3571 EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
3572
qm_vf_q_assign(struct hisi_qm * qm,u32 num_vfs)3573 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
3574 {
3575 u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;
3576 u32 max_qp_num = qm->max_qp_num;
3577 u32 q_base = qm->qp_num;
3578 int ret;
3579
3580 if (!num_vfs)
3581 return -EINVAL;
3582
3583 vfs_q_num = qm->ctrl_qp_num - qm->qp_num;
3584
3585 /* If vfs_q_num is less than num_vfs, return error. */
3586 if (vfs_q_num < num_vfs)
3587 return -EINVAL;
3588
3589 q_num = vfs_q_num / num_vfs;
3590 remain_q_num = vfs_q_num % num_vfs;
3591
3592 for (i = num_vfs; i > 0; i--) {
3593 /*
3594 * if q_num + remain_q_num > max_qp_num in last vf, divide the
3595 * remaining queues equally.
3596 */
3597 if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {
3598 act_q_num = q_num + remain_q_num;
3599 remain_q_num = 0;
3600 } else if (remain_q_num > 0) {
3601 act_q_num = q_num + 1;
3602 remain_q_num--;
3603 } else {
3604 act_q_num = q_num;
3605 }
3606
3607 act_q_num = min(act_q_num, max_qp_num);
3608 ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);
3609 if (ret) {
3610 for (j = num_vfs; j > i; j--)
3611 hisi_qm_set_vft(qm, j, 0, 0);
3612 return ret;
3613 }
3614 q_base += act_q_num;
3615 }
3616
3617 return 0;
3618 }
3619
qm_clear_vft_config(struct hisi_qm * qm)3620 static int qm_clear_vft_config(struct hisi_qm *qm)
3621 {
3622 int ret;
3623 u32 i;
3624
3625 for (i = 1; i <= qm->vfs_num; i++) {
3626 ret = hisi_qm_set_vft(qm, i, 0, 0);
3627 if (ret)
3628 return ret;
3629 }
3630 qm->vfs_num = 0;
3631
3632 return 0;
3633 }
3634
qm_func_shaper_enable(struct hisi_qm * qm,u32 fun_index,u32 qos)3635 static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos)
3636 {
3637 struct device *dev = &qm->pdev->dev;
3638 u32 ir = qos * QM_QOS_RATE;
3639 int ret, total_vfs, i;
3640
3641 total_vfs = pci_sriov_get_totalvfs(qm->pdev);
3642 if (fun_index > total_vfs)
3643 return -EINVAL;
3644
3645 qm->factor[fun_index].func_qos = qos;
3646
3647 ret = qm_get_shaper_para(ir, &qm->factor[fun_index]);
3648 if (ret) {
3649 dev_err(dev, "failed to calculate shaper parameter!\n");
3650 return -EINVAL;
3651 }
3652
3653 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
3654 /* The base number of queue reuse for different alg type */
3655 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1);
3656 if (ret) {
3657 dev_err(dev, "type: %d, failed to set shaper vft!\n", i);
3658 return -EINVAL;
3659 }
3660 }
3661
3662 return 0;
3663 }
3664
qm_get_shaper_vft_qos(struct hisi_qm * qm,u32 fun_index)3665 static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index)
3666 {
3667 u64 cir_u = 0, cir_b = 0, cir_s = 0;
3668 u64 shaper_vft, ir_calc, ir;
3669 unsigned int val;
3670 u32 error_rate;
3671 int ret;
3672
3673 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
3674 val & BIT(0), POLL_PERIOD,
3675 POLL_TIMEOUT);
3676 if (ret)
3677 return 0;
3678
3679 writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);
3680 writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE);
3681 writel(fun_index, qm->io_base + QM_VFT_CFG);
3682
3683 writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
3684 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
3685
3686 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
3687 val & BIT(0), POLL_PERIOD,
3688 POLL_TIMEOUT);
3689 if (ret)
3690 return 0;
3691
3692 shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |
3693 ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32);
3694
3695 cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK;
3696 cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK;
3697 cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT;
3698
3699 cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK;
3700 cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT;
3701
3702 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
3703
3704 ir = qm->factor[fun_index].func_qos * QM_QOS_RATE;
3705
3706 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
3707 if (error_rate > QM_QOS_MIN_ERROR_RATE) {
3708 pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate);
3709 return 0;
3710 }
3711
3712 return ir;
3713 }
3714
qm_vf_get_qos(struct hisi_qm * qm,u32 fun_num)3715 static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num)
3716 {
3717 struct device *dev = &qm->pdev->dev;
3718 u32 qos;
3719 int ret;
3720
3721 qos = qm_get_shaper_vft_qos(qm, fun_num);
3722 if (!qos) {
3723 dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num);
3724 return;
3725 }
3726
3727 ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num);
3728 if (ret)
3729 dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num);
3730 }
3731
qm_vf_read_qos(struct hisi_qm * qm)3732 static int qm_vf_read_qos(struct hisi_qm *qm)
3733 {
3734 int cnt = 0;
3735 int ret = -EINVAL;
3736
3737 /* reset mailbox qos val */
3738 qm->mb_qos = 0;
3739
3740 /* vf ping pf to get function qos */
3741 ret = qm_ping_pf(qm, QM_VF_GET_QOS);
3742 if (ret) {
3743 pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n");
3744 return ret;
3745 }
3746
3747 while (true) {
3748 msleep(QM_WAIT_DST_ACK);
3749 if (qm->mb_qos)
3750 break;
3751
3752 if (++cnt > QM_MAX_VF_WAIT_COUNT) {
3753 pci_err(qm->pdev, "PF ping VF timeout!\n");
3754 return -ETIMEDOUT;
3755 }
3756 }
3757
3758 return ret;
3759 }
3760
qm_algqos_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)3761 static ssize_t qm_algqos_read(struct file *filp, char __user *buf,
3762 size_t count, loff_t *pos)
3763 {
3764 struct hisi_qm *qm = filp->private_data;
3765 char tbuf[QM_DBG_READ_LEN];
3766 u32 qos_val, ir;
3767 int ret;
3768
3769 ret = hisi_qm_get_dfx_access(qm);
3770 if (ret)
3771 return ret;
3772
3773 /* Mailbox and reset cannot be operated at the same time */
3774 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
3775 pci_err(qm->pdev, "dev resetting, read alg qos failed!\n");
3776 ret = -EAGAIN;
3777 goto err_put_dfx_access;
3778 }
3779
3780 if (qm->fun_type == QM_HW_PF) {
3781 ir = qm_get_shaper_vft_qos(qm, 0);
3782 } else {
3783 ret = qm_vf_read_qos(qm);
3784 if (ret)
3785 goto err_get_status;
3786 ir = qm->mb_qos;
3787 }
3788
3789 qos_val = ir / QM_QOS_RATE;
3790 ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val);
3791
3792 ret = simple_read_from_buffer(buf, count, pos, tbuf, ret);
3793
3794 err_get_status:
3795 clear_bit(QM_RESETTING, &qm->misc_ctl);
3796 err_put_dfx_access:
3797 hisi_qm_put_dfx_access(qm);
3798 return ret;
3799 }
3800
qm_get_qos_value(struct hisi_qm * qm,const char * buf,unsigned long * val,unsigned int * fun_index)3801 static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,
3802 unsigned long *val,
3803 unsigned int *fun_index)
3804 {
3805 const struct bus_type *bus_type = qm->pdev->dev.bus;
3806 char tbuf_bdf[QM_DBG_READ_LEN] = {0};
3807 char val_buf[QM_DBG_READ_LEN] = {0};
3808 struct pci_dev *pdev;
3809 struct device *dev;
3810 int ret;
3811
3812 ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf);
3813 if (ret != QM_QOS_PARAM_NUM)
3814 return -EINVAL;
3815
3816 ret = kstrtoul(val_buf, 10, val);
3817 if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) {
3818 pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
3819 return -EINVAL;
3820 }
3821
3822 dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf);
3823 if (!dev) {
3824 pci_err(qm->pdev, "input pci bdf number is error!\n");
3825 return -ENODEV;
3826 }
3827
3828 pdev = container_of(dev, struct pci_dev, dev);
3829
3830 *fun_index = pdev->devfn;
3831
3832 return 0;
3833 }
3834
qm_algqos_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)3835 static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
3836 size_t count, loff_t *pos)
3837 {
3838 struct hisi_qm *qm = filp->private_data;
3839 char tbuf[QM_DBG_READ_LEN];
3840 unsigned int fun_index;
3841 unsigned long val;
3842 int len, ret;
3843
3844 if (*pos != 0)
3845 return 0;
3846
3847 if (count >= QM_DBG_READ_LEN)
3848 return -ENOSPC;
3849
3850 len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
3851 if (len < 0)
3852 return len;
3853
3854 tbuf[len] = '\0';
3855 ret = qm_get_qos_value(qm, tbuf, &val, &fun_index);
3856 if (ret)
3857 return ret;
3858
3859 /* Mailbox and reset cannot be operated at the same time */
3860 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
3861 pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
3862 return -EAGAIN;
3863 }
3864
3865 ret = qm_pm_get_sync(qm);
3866 if (ret) {
3867 ret = -EINVAL;
3868 goto err_get_status;
3869 }
3870
3871 ret = qm_func_shaper_enable(qm, fun_index, val);
3872 if (ret) {
3873 pci_err(qm->pdev, "failed to enable function shaper!\n");
3874 ret = -EINVAL;
3875 goto err_put_sync;
3876 }
3877
3878 pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n",
3879 fun_index, val);
3880 ret = count;
3881
3882 err_put_sync:
3883 qm_pm_put_sync(qm);
3884 err_get_status:
3885 clear_bit(QM_RESETTING, &qm->misc_ctl);
3886 return ret;
3887 }
3888
3889 static const struct file_operations qm_algqos_fops = {
3890 .owner = THIS_MODULE,
3891 .open = simple_open,
3892 .read = qm_algqos_read,
3893 .write = qm_algqos_write,
3894 };
3895
3896 /**
3897 * hisi_qm_set_algqos_init() - Initialize function qos debugfs files.
3898 * @qm: The qm for which we want to add debugfs files.
3899 *
3900 * Create function qos debugfs files, VF ping PF to get function qos.
3901 */
hisi_qm_set_algqos_init(struct hisi_qm * qm)3902 void hisi_qm_set_algqos_init(struct hisi_qm *qm)
3903 {
3904 if (qm->fun_type == QM_HW_PF)
3905 debugfs_create_file("alg_qos", 0644, qm->debug.debug_root,
3906 qm, &qm_algqos_fops);
3907 else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
3908 debugfs_create_file("alg_qos", 0444, qm->debug.debug_root,
3909 qm, &qm_algqos_fops);
3910 }
3911
hisi_qm_init_vf_qos(struct hisi_qm * qm,int total_func)3912 static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func)
3913 {
3914 int i;
3915
3916 for (i = 1; i <= total_func; i++)
3917 qm->factor[i].func_qos = QM_QOS_MAX_VAL;
3918 }
3919
3920 /**
3921 * hisi_qm_sriov_enable() - enable virtual functions
3922 * @pdev: the PCIe device
3923 * @max_vfs: the number of virtual functions to enable
3924 *
3925 * Returns the number of enabled VFs. If there are VFs enabled already or
3926 * max_vfs is more than the total number of device can be enabled, returns
3927 * failure.
3928 */
hisi_qm_sriov_enable(struct pci_dev * pdev,int max_vfs)3929 int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)
3930 {
3931 struct hisi_qm *qm = pci_get_drvdata(pdev);
3932 int pre_existing_vfs, num_vfs, total_vfs, ret;
3933
3934 ret = qm_pm_get_sync(qm);
3935 if (ret)
3936 return ret;
3937
3938 total_vfs = pci_sriov_get_totalvfs(pdev);
3939 pre_existing_vfs = pci_num_vf(pdev);
3940 if (pre_existing_vfs) {
3941 pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",
3942 pre_existing_vfs);
3943 goto err_put_sync;
3944 }
3945
3946 if (max_vfs > total_vfs) {
3947 pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs);
3948 ret = -ERANGE;
3949 goto err_put_sync;
3950 }
3951
3952 num_vfs = max_vfs;
3953
3954 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
3955 hisi_qm_init_vf_qos(qm, num_vfs);
3956
3957 ret = qm_vf_q_assign(qm, num_vfs);
3958 if (ret) {
3959 pci_err(pdev, "Can't assign queues for VF!\n");
3960 goto err_put_sync;
3961 }
3962
3963 ret = pci_enable_sriov(pdev, num_vfs);
3964 if (ret) {
3965 pci_err(pdev, "Can't enable VF!\n");
3966 qm_clear_vft_config(qm);
3967 goto err_put_sync;
3968 }
3969 qm->vfs_num = num_vfs;
3970
3971 pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);
3972
3973 return num_vfs;
3974
3975 err_put_sync:
3976 qm_pm_put_sync(qm);
3977 return ret;
3978 }
3979 EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);
3980
3981 /**
3982 * hisi_qm_sriov_disable - disable virtual functions
3983 * @pdev: the PCI device.
3984 * @is_frozen: true when all the VFs are frozen.
3985 *
3986 * Return failure if there are VFs assigned already or VF is in used.
3987 */
hisi_qm_sriov_disable(struct pci_dev * pdev,bool is_frozen)3988 int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)
3989 {
3990 struct hisi_qm *qm = pci_get_drvdata(pdev);
3991
3992 if (pci_vfs_assigned(pdev)) {
3993 pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");
3994 return -EPERM;
3995 }
3996
3997 /* While VF is in used, SRIOV cannot be disabled. */
3998 if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {
3999 pci_err(pdev, "Task is using its VF!\n");
4000 return -EBUSY;
4001 }
4002
4003 pci_disable_sriov(pdev);
4004
4005 qm->vfs_num = 0;
4006 qm_pm_put_sync(qm);
4007
4008 return qm_clear_vft_config(qm);
4009 }
4010 EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);
4011
4012 /**
4013 * hisi_qm_sriov_configure - configure the number of VFs
4014 * @pdev: The PCI device
4015 * @num_vfs: The number of VFs need enabled
4016 *
4017 * Enable SR-IOV according to num_vfs, 0 means disable.
4018 */
hisi_qm_sriov_configure(struct pci_dev * pdev,int num_vfs)4019 int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
4020 {
4021 if (num_vfs == 0)
4022 return hisi_qm_sriov_disable(pdev, false);
4023 else
4024 return hisi_qm_sriov_enable(pdev, num_vfs);
4025 }
4026 EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);
4027
qm_dev_err_handle(struct hisi_qm * qm)4028 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
4029 {
4030 if (!qm->err_ini->get_err_result) {
4031 dev_err(&qm->pdev->dev, "Device doesn't support reset!\n");
4032 return ACC_ERR_NONE;
4033 }
4034
4035 return qm->err_ini->get_err_result(qm);
4036 }
4037
qm_process_dev_error(struct hisi_qm * qm)4038 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)
4039 {
4040 enum acc_err_result qm_ret, dev_ret;
4041
4042 /* log qm error */
4043 qm_ret = qm_hw_error_handle(qm);
4044
4045 /* log device error */
4046 dev_ret = qm_dev_err_handle(qm);
4047
4048 return (qm_ret == ACC_ERR_NEED_RESET ||
4049 dev_ret == ACC_ERR_NEED_RESET) ?
4050 ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
4051 }
4052
4053 /**
4054 * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
4055 * @pdev: The PCI device which need report error.
4056 * @state: The connectivity between CPU and device.
4057 *
4058 * We register this function into PCIe AER handlers, It will report device or
4059 * qm hardware error status when error occur.
4060 */
hisi_qm_dev_err_detected(struct pci_dev * pdev,pci_channel_state_t state)4061 pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
4062 pci_channel_state_t state)
4063 {
4064 struct hisi_qm *qm = pci_get_drvdata(pdev);
4065 enum acc_err_result ret;
4066
4067 if (pdev->is_virtfn)
4068 return PCI_ERS_RESULT_NONE;
4069
4070 pci_info(pdev, "PCI error detected, state(=%u)!!\n", state);
4071 if (state == pci_channel_io_perm_failure)
4072 return PCI_ERS_RESULT_DISCONNECT;
4073
4074 ret = qm_process_dev_error(qm);
4075 if (ret == ACC_ERR_NEED_RESET)
4076 return PCI_ERS_RESULT_NEED_RESET;
4077
4078 return PCI_ERS_RESULT_RECOVERED;
4079 }
4080 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);
4081
qm_check_req_recv(struct hisi_qm * qm)4082 static int qm_check_req_recv(struct hisi_qm *qm)
4083 {
4084 struct pci_dev *pdev = qm->pdev;
4085 int ret;
4086 u32 val;
4087
4088 if (qm->ver >= QM_HW_V3)
4089 return 0;
4090
4091 writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);
4092 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4093 (val == ACC_VENDOR_ID_VALUE),
4094 POLL_PERIOD, POLL_TIMEOUT);
4095 if (ret) {
4096 dev_err(&pdev->dev, "Fails to read QM reg!\n");
4097 return ret;
4098 }
4099
4100 writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);
4101 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4102 (val == PCI_VENDOR_ID_HUAWEI),
4103 POLL_PERIOD, POLL_TIMEOUT);
4104 if (ret)
4105 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");
4106
4107 return ret;
4108 }
4109
qm_set_pf_mse(struct hisi_qm * qm,bool set)4110 static int qm_set_pf_mse(struct hisi_qm *qm, bool set)
4111 {
4112 struct pci_dev *pdev = qm->pdev;
4113 u16 cmd;
4114 int i;
4115
4116 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4117 if (set)
4118 cmd |= PCI_COMMAND_MEMORY;
4119 else
4120 cmd &= ~PCI_COMMAND_MEMORY;
4121
4122 pci_write_config_word(pdev, PCI_COMMAND, cmd);
4123 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4124 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4125 if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))
4126 return 0;
4127
4128 udelay(1);
4129 }
4130
4131 return -ETIMEDOUT;
4132 }
4133
qm_set_vf_mse(struct hisi_qm * qm,bool set)4134 static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
4135 {
4136 struct pci_dev *pdev = qm->pdev;
4137 u16 sriov_ctrl;
4138 int pos;
4139 int i;
4140
4141 /*
4142 * Since function qm_set_vf_mse is called only after SRIOV is enabled,
4143 * pci_find_ext_capability cannot return 0, pos does not need to be
4144 * checked.
4145 */
4146 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
4147 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4148 if (set)
4149 sriov_ctrl |= PCI_SRIOV_CTRL_MSE;
4150 else
4151 sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;
4152 pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);
4153
4154 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4155 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4156 if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>
4157 ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)
4158 return 0;
4159
4160 udelay(1);
4161 }
4162
4163 return -ETIMEDOUT;
4164 }
4165
qm_dev_ecc_mbit_handle(struct hisi_qm * qm)4166 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)
4167 {
4168 u32 nfe_enb = 0;
4169
4170 /* Kunpeng930 hardware automatically close master ooo when NFE occurs */
4171 if (qm->ver >= QM_HW_V3)
4172 return;
4173
4174 if (!qm->err_status.is_dev_ecc_mbit &&
4175 qm->err_status.is_qm_ecc_mbit &&
4176 qm->err_ini->close_axi_master_ooo) {
4177 qm->err_ini->close_axi_master_ooo(qm);
4178 } else if (qm->err_status.is_dev_ecc_mbit &&
4179 !qm->err_status.is_qm_ecc_mbit &&
4180 !qm->err_ini->close_axi_master_ooo) {
4181 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);
4182 writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,
4183 qm->io_base + QM_RAS_NFE_ENABLE);
4184 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);
4185 }
4186 }
4187
qm_vf_reset_prepare(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4188 static int qm_vf_reset_prepare(struct hisi_qm *qm,
4189 enum qm_stop_reason stop_reason)
4190 {
4191 struct hisi_qm_list *qm_list = qm->qm_list;
4192 struct pci_dev *pdev = qm->pdev;
4193 struct pci_dev *virtfn;
4194 struct hisi_qm *vf_qm;
4195 int ret = 0;
4196
4197 mutex_lock(&qm_list->lock);
4198 list_for_each_entry(vf_qm, &qm_list->list, list) {
4199 virtfn = vf_qm->pdev;
4200 if (virtfn == pdev)
4201 continue;
4202
4203 if (pci_physfn(virtfn) == pdev) {
4204 /* save VFs PCIE BAR configuration */
4205 pci_save_state(virtfn);
4206
4207 ret = hisi_qm_stop(vf_qm, stop_reason);
4208 if (ret)
4209 goto stop_fail;
4210 }
4211 }
4212
4213 stop_fail:
4214 mutex_unlock(&qm_list->lock);
4215 return ret;
4216 }
4217
qm_try_stop_vfs(struct hisi_qm * qm,enum qm_ifc_cmd cmd,enum qm_stop_reason stop_reason)4218 static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd,
4219 enum qm_stop_reason stop_reason)
4220 {
4221 struct pci_dev *pdev = qm->pdev;
4222 int ret;
4223
4224 if (!qm->vfs_num)
4225 return 0;
4226
4227 /* Kunpeng930 supports to notify VFs to stop before PF reset */
4228 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {
4229 ret = qm_ping_all_vfs(qm, cmd);
4230 if (ret)
4231 pci_err(pdev, "failed to send command to all VFs before PF reset!\n");
4232 } else {
4233 ret = qm_vf_reset_prepare(qm, stop_reason);
4234 if (ret)
4235 pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret);
4236 }
4237
4238 return ret;
4239 }
4240
qm_controller_reset_prepare(struct hisi_qm * qm)4241 static int qm_controller_reset_prepare(struct hisi_qm *qm)
4242 {
4243 struct pci_dev *pdev = qm->pdev;
4244 int ret;
4245
4246 if (qm->err_ini->set_priv_status) {
4247 ret = qm->err_ini->set_priv_status(qm);
4248 if (ret)
4249 return ret;
4250 }
4251
4252 ret = qm_reset_prepare_ready(qm);
4253 if (ret) {
4254 pci_err(pdev, "Controller reset not ready!\n");
4255 return ret;
4256 }
4257
4258 qm_dev_ecc_mbit_handle(qm);
4259
4260 /* PF obtains the information of VF by querying the register. */
4261 qm_cmd_uninit(qm);
4262
4263 /* Whether VFs stop successfully, soft reset will continue. */
4264 ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET);
4265 if (ret)
4266 pci_err(pdev, "failed to stop vfs by pf in soft reset.\n");
4267
4268 ret = hisi_qm_stop(qm, QM_SOFT_RESET);
4269 if (ret) {
4270 pci_err(pdev, "Fails to stop QM!\n");
4271 qm_reset_bit_clear(qm);
4272 return ret;
4273 }
4274
4275 if (qm->use_sva) {
4276 ret = qm_hw_err_isolate(qm);
4277 if (ret)
4278 pci_err(pdev, "failed to isolate hw err!\n");
4279 }
4280
4281 ret = qm_wait_vf_prepare_finish(qm);
4282 if (ret)
4283 pci_err(pdev, "failed to stop by vfs in soft reset!\n");
4284
4285 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4286
4287 return 0;
4288 }
4289
qm_master_ooo_check(struct hisi_qm * qm)4290 static int qm_master_ooo_check(struct hisi_qm *qm)
4291 {
4292 u32 val;
4293 int ret;
4294
4295 /* Check the ooo register of the device before resetting the device. */
4296 writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL);
4297 ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
4298 val, (val == ACC_MASTER_TRANS_RETURN_RW),
4299 POLL_PERIOD, POLL_TIMEOUT);
4300 if (ret)
4301 pci_warn(qm->pdev, "Bus lock! Please reset system.\n");
4302
4303 return ret;
4304 }
4305
qm_soft_reset_prepare(struct hisi_qm * qm)4306 static int qm_soft_reset_prepare(struct hisi_qm *qm)
4307 {
4308 struct pci_dev *pdev = qm->pdev;
4309 int ret;
4310
4311 /* Ensure all doorbells and mailboxes received by QM */
4312 ret = qm_check_req_recv(qm);
4313 if (ret)
4314 return ret;
4315
4316 if (qm->vfs_num) {
4317 ret = qm_set_vf_mse(qm, false);
4318 if (ret) {
4319 pci_err(pdev, "Fails to disable vf MSE bit.\n");
4320 return ret;
4321 }
4322 }
4323
4324 ret = qm->ops->set_msi(qm, false);
4325 if (ret) {
4326 pci_err(pdev, "Fails to disable PEH MSI bit.\n");
4327 return ret;
4328 }
4329
4330 ret = qm_master_ooo_check(qm);
4331 if (ret)
4332 return ret;
4333
4334 if (qm->err_ini->close_sva_prefetch)
4335 qm->err_ini->close_sva_prefetch(qm);
4336
4337 ret = qm_set_pf_mse(qm, false);
4338 if (ret)
4339 pci_err(pdev, "Fails to disable pf MSE bit.\n");
4340
4341 return ret;
4342 }
4343
qm_reset_device(struct hisi_qm * qm)4344 static int qm_reset_device(struct hisi_qm *qm)
4345 {
4346 struct pci_dev *pdev = qm->pdev;
4347
4348 /* The reset related sub-control registers are not in PCI BAR */
4349 if (ACPI_HANDLE(&pdev->dev)) {
4350 unsigned long long value = 0;
4351 acpi_status s;
4352
4353 s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
4354 qm->err_info.acpi_rst,
4355 NULL, &value);
4356 if (ACPI_FAILURE(s)) {
4357 pci_err(pdev, "NO controller reset method!\n");
4358 return -EIO;
4359 }
4360
4361 if (value) {
4362 pci_err(pdev, "Reset step %llu failed!\n", value);
4363 return -EIO;
4364 }
4365
4366 return 0;
4367 }
4368
4369 pci_err(pdev, "No reset method!\n");
4370 return -EINVAL;
4371 }
4372
qm_soft_reset(struct hisi_qm * qm)4373 static int qm_soft_reset(struct hisi_qm *qm)
4374 {
4375 int ret;
4376
4377 ret = qm_soft_reset_prepare(qm);
4378 if (ret)
4379 return ret;
4380
4381 return qm_reset_device(qm);
4382 }
4383
qm_vf_reset_done(struct hisi_qm * qm)4384 static int qm_vf_reset_done(struct hisi_qm *qm)
4385 {
4386 struct hisi_qm_list *qm_list = qm->qm_list;
4387 struct pci_dev *pdev = qm->pdev;
4388 struct pci_dev *virtfn;
4389 struct hisi_qm *vf_qm;
4390 int ret = 0;
4391
4392 mutex_lock(&qm_list->lock);
4393 list_for_each_entry(vf_qm, &qm_list->list, list) {
4394 virtfn = vf_qm->pdev;
4395 if (virtfn == pdev)
4396 continue;
4397
4398 if (pci_physfn(virtfn) == pdev) {
4399 /* enable VFs PCIE BAR configuration */
4400 pci_restore_state(virtfn);
4401
4402 ret = qm_restart(vf_qm);
4403 if (ret)
4404 goto restart_fail;
4405 }
4406 }
4407
4408 restart_fail:
4409 mutex_unlock(&qm_list->lock);
4410 return ret;
4411 }
4412
qm_try_start_vfs(struct hisi_qm * qm,enum qm_ifc_cmd cmd)4413 static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd)
4414 {
4415 struct pci_dev *pdev = qm->pdev;
4416 int ret;
4417
4418 if (!qm->vfs_num)
4419 return 0;
4420
4421 ret = qm_vf_q_assign(qm, qm->vfs_num);
4422 if (ret) {
4423 pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret);
4424 return ret;
4425 }
4426
4427 /* Kunpeng930 supports to notify VFs to start after PF reset. */
4428 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {
4429 ret = qm_ping_all_vfs(qm, cmd);
4430 if (ret)
4431 pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n");
4432 } else {
4433 ret = qm_vf_reset_done(qm);
4434 if (ret)
4435 pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret);
4436 }
4437
4438 return ret;
4439 }
4440
qm_dev_hw_init(struct hisi_qm * qm)4441 static int qm_dev_hw_init(struct hisi_qm *qm)
4442 {
4443 return qm->err_ini->hw_init(qm);
4444 }
4445
qm_restart_prepare(struct hisi_qm * qm)4446 static void qm_restart_prepare(struct hisi_qm *qm)
4447 {
4448 u32 value;
4449
4450 if (qm->err_ini->open_sva_prefetch)
4451 qm->err_ini->open_sva_prefetch(qm);
4452
4453 if (qm->ver >= QM_HW_V3)
4454 return;
4455
4456 if (!qm->err_status.is_qm_ecc_mbit &&
4457 !qm->err_status.is_dev_ecc_mbit)
4458 return;
4459
4460 /* temporarily close the OOO port used for PEH to write out MSI */
4461 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4462 writel(value & ~qm->err_info.msi_wr_port,
4463 qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4464
4465 /* clear dev ecc 2bit error source if having */
4466 value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;
4467 if (value && qm->err_ini->clear_dev_hw_err_status)
4468 qm->err_ini->clear_dev_hw_err_status(qm, value);
4469
4470 /* clear QM ecc mbit error source */
4471 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);
4472
4473 /* clear AM Reorder Buffer ecc mbit source */
4474 writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);
4475 }
4476
qm_restart_done(struct hisi_qm * qm)4477 static void qm_restart_done(struct hisi_qm *qm)
4478 {
4479 u32 value;
4480
4481 if (qm->ver >= QM_HW_V3)
4482 goto clear_flags;
4483
4484 if (!qm->err_status.is_qm_ecc_mbit &&
4485 !qm->err_status.is_dev_ecc_mbit)
4486 return;
4487
4488 /* open the OOO port for PEH to write out MSI */
4489 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4490 value |= qm->err_info.msi_wr_port;
4491 writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4492
4493 clear_flags:
4494 qm->err_status.is_qm_ecc_mbit = false;
4495 qm->err_status.is_dev_ecc_mbit = false;
4496 }
4497
qm_controller_reset_done(struct hisi_qm * qm)4498 static int qm_controller_reset_done(struct hisi_qm *qm)
4499 {
4500 struct pci_dev *pdev = qm->pdev;
4501 int ret;
4502
4503 ret = qm->ops->set_msi(qm, true);
4504 if (ret) {
4505 pci_err(pdev, "Fails to enable PEH MSI bit!\n");
4506 return ret;
4507 }
4508
4509 ret = qm_set_pf_mse(qm, true);
4510 if (ret) {
4511 pci_err(pdev, "Fails to enable pf MSE bit!\n");
4512 return ret;
4513 }
4514
4515 if (qm->vfs_num) {
4516 ret = qm_set_vf_mse(qm, true);
4517 if (ret) {
4518 pci_err(pdev, "Fails to enable vf MSE bit!\n");
4519 return ret;
4520 }
4521 }
4522
4523 ret = qm_dev_hw_init(qm);
4524 if (ret) {
4525 pci_err(pdev, "Failed to init device\n");
4526 return ret;
4527 }
4528
4529 qm_restart_prepare(qm);
4530 hisi_qm_dev_err_init(qm);
4531 if (qm->err_ini->open_axi_master_ooo)
4532 qm->err_ini->open_axi_master_ooo(qm);
4533
4534 ret = qm_dev_mem_reset(qm);
4535 if (ret) {
4536 pci_err(pdev, "failed to reset device memory\n");
4537 return ret;
4538 }
4539
4540 ret = qm_restart(qm);
4541 if (ret) {
4542 pci_err(pdev, "Failed to start QM!\n");
4543 return ret;
4544 }
4545
4546 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
4547 if (ret)
4548 pci_err(pdev, "failed to start vfs by pf in soft reset.\n");
4549
4550 ret = qm_wait_vf_prepare_finish(qm);
4551 if (ret)
4552 pci_err(pdev, "failed to start by vfs in soft reset!\n");
4553
4554 qm_cmd_init(qm);
4555 qm_restart_done(qm);
4556
4557 qm_reset_bit_clear(qm);
4558
4559 return 0;
4560 }
4561
qm_controller_reset(struct hisi_qm * qm)4562 static int qm_controller_reset(struct hisi_qm *qm)
4563 {
4564 struct pci_dev *pdev = qm->pdev;
4565 int ret;
4566
4567 pci_info(pdev, "Controller resetting...\n");
4568
4569 ret = qm_controller_reset_prepare(qm);
4570 if (ret) {
4571 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4572 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4573 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4574 return ret;
4575 }
4576
4577 hisi_qm_show_last_dfx_regs(qm);
4578 if (qm->err_ini->show_last_dfx_regs)
4579 qm->err_ini->show_last_dfx_regs(qm);
4580
4581 ret = qm_soft_reset(qm);
4582 if (ret)
4583 goto err_reset;
4584
4585 ret = qm_controller_reset_done(qm);
4586 if (ret)
4587 goto err_reset;
4588
4589 pci_info(pdev, "Controller reset complete\n");
4590
4591 return 0;
4592
4593 err_reset:
4594 pci_err(pdev, "Controller reset failed (%d)\n", ret);
4595 qm_reset_bit_clear(qm);
4596
4597 /* if resetting fails, isolate the device */
4598 if (qm->use_sva)
4599 qm->isolate_data.is_isolate = true;
4600 return ret;
4601 }
4602
4603 /**
4604 * hisi_qm_dev_slot_reset() - slot reset
4605 * @pdev: the PCIe device
4606 *
4607 * This function offers QM relate PCIe device reset interface. Drivers which
4608 * use QM can use this function as slot_reset in its struct pci_error_handlers.
4609 */
hisi_qm_dev_slot_reset(struct pci_dev * pdev)4610 pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)
4611 {
4612 struct hisi_qm *qm = pci_get_drvdata(pdev);
4613 int ret;
4614
4615 if (pdev->is_virtfn)
4616 return PCI_ERS_RESULT_RECOVERED;
4617
4618 /* reset pcie device controller */
4619 ret = qm_controller_reset(qm);
4620 if (ret) {
4621 pci_err(pdev, "Controller reset failed (%d)\n", ret);
4622 return PCI_ERS_RESULT_DISCONNECT;
4623 }
4624
4625 return PCI_ERS_RESULT_RECOVERED;
4626 }
4627 EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);
4628
hisi_qm_reset_prepare(struct pci_dev * pdev)4629 void hisi_qm_reset_prepare(struct pci_dev *pdev)
4630 {
4631 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4632 struct hisi_qm *qm = pci_get_drvdata(pdev);
4633 u32 delay = 0;
4634 int ret;
4635
4636 hisi_qm_dev_err_uninit(pf_qm);
4637
4638 /*
4639 * Check whether there is an ECC mbit error, If it occurs, need to
4640 * wait for soft reset to fix it.
4641 */
4642 while (qm_check_dev_error(qm)) {
4643 msleep(++delay);
4644 if (delay > QM_RESET_WAIT_TIMEOUT)
4645 return;
4646 }
4647
4648 ret = qm_reset_prepare_ready(qm);
4649 if (ret) {
4650 pci_err(pdev, "FLR not ready!\n");
4651 return;
4652 }
4653
4654 /* PF obtains the information of VF by querying the register. */
4655 if (qm->fun_type == QM_HW_PF)
4656 qm_cmd_uninit(qm);
4657
4658 ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN);
4659 if (ret)
4660 pci_err(pdev, "failed to stop vfs by pf in FLR.\n");
4661
4662 ret = hisi_qm_stop(qm, QM_DOWN);
4663 if (ret) {
4664 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
4665 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4666 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4667 return;
4668 }
4669
4670 ret = qm_wait_vf_prepare_finish(qm);
4671 if (ret)
4672 pci_err(pdev, "failed to stop by vfs in FLR!\n");
4673
4674 pci_info(pdev, "FLR resetting...\n");
4675 }
4676 EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);
4677
qm_flr_reset_complete(struct pci_dev * pdev)4678 static bool qm_flr_reset_complete(struct pci_dev *pdev)
4679 {
4680 struct pci_dev *pf_pdev = pci_physfn(pdev);
4681 struct hisi_qm *qm = pci_get_drvdata(pf_pdev);
4682 u32 id;
4683
4684 pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);
4685 if (id == QM_PCI_COMMAND_INVALID) {
4686 pci_err(pdev, "Device can not be used!\n");
4687 return false;
4688 }
4689
4690 return true;
4691 }
4692
hisi_qm_reset_done(struct pci_dev * pdev)4693 void hisi_qm_reset_done(struct pci_dev *pdev)
4694 {
4695 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4696 struct hisi_qm *qm = pci_get_drvdata(pdev);
4697 int ret;
4698
4699 if (qm->fun_type == QM_HW_PF) {
4700 ret = qm_dev_hw_init(qm);
4701 if (ret) {
4702 pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);
4703 goto flr_done;
4704 }
4705 }
4706
4707 hisi_qm_dev_err_init(pf_qm);
4708
4709 ret = qm_restart(qm);
4710 if (ret) {
4711 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);
4712 goto flr_done;
4713 }
4714
4715 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
4716 if (ret)
4717 pci_err(pdev, "failed to start vfs by pf in FLR.\n");
4718
4719 ret = qm_wait_vf_prepare_finish(qm);
4720 if (ret)
4721 pci_err(pdev, "failed to start by vfs in FLR!\n");
4722
4723 flr_done:
4724 if (qm->fun_type == QM_HW_PF)
4725 qm_cmd_init(qm);
4726
4727 if (qm_flr_reset_complete(pdev))
4728 pci_info(pdev, "FLR reset complete\n");
4729
4730 qm_reset_bit_clear(qm);
4731 }
4732 EXPORT_SYMBOL_GPL(hisi_qm_reset_done);
4733
qm_abnormal_irq(int irq,void * data)4734 static irqreturn_t qm_abnormal_irq(int irq, void *data)
4735 {
4736 struct hisi_qm *qm = data;
4737 enum acc_err_result ret;
4738
4739 atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);
4740 ret = qm_process_dev_error(qm);
4741 if (ret == ACC_ERR_NEED_RESET &&
4742 !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) &&
4743 !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl))
4744 schedule_work(&qm->rst_work);
4745
4746 return IRQ_HANDLED;
4747 }
4748
4749 /**
4750 * hisi_qm_dev_shutdown() - Shutdown device.
4751 * @pdev: The device will be shutdown.
4752 *
4753 * This function will stop qm when OS shutdown or rebooting.
4754 */
hisi_qm_dev_shutdown(struct pci_dev * pdev)4755 void hisi_qm_dev_shutdown(struct pci_dev *pdev)
4756 {
4757 struct hisi_qm *qm = pci_get_drvdata(pdev);
4758 int ret;
4759
4760 ret = hisi_qm_stop(qm, QM_DOWN);
4761 if (ret)
4762 dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");
4763
4764 hisi_qm_cache_wb(qm);
4765 }
4766 EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);
4767
hisi_qm_controller_reset(struct work_struct * rst_work)4768 static void hisi_qm_controller_reset(struct work_struct *rst_work)
4769 {
4770 struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
4771 int ret;
4772
4773 ret = qm_pm_get_sync(qm);
4774 if (ret) {
4775 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4776 return;
4777 }
4778
4779 /* reset pcie device controller */
4780 ret = qm_controller_reset(qm);
4781 if (ret)
4782 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);
4783
4784 qm_pm_put_sync(qm);
4785 }
4786
qm_pf_reset_vf_prepare(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4787 static void qm_pf_reset_vf_prepare(struct hisi_qm *qm,
4788 enum qm_stop_reason stop_reason)
4789 {
4790 enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE;
4791 struct pci_dev *pdev = qm->pdev;
4792 int ret;
4793
4794 ret = qm_reset_prepare_ready(qm);
4795 if (ret) {
4796 dev_err(&pdev->dev, "reset prepare not ready!\n");
4797 atomic_set(&qm->status.flags, QM_STOP);
4798 cmd = QM_VF_PREPARE_FAIL;
4799 goto err_prepare;
4800 }
4801
4802 ret = hisi_qm_stop(qm, stop_reason);
4803 if (ret) {
4804 dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret);
4805 atomic_set(&qm->status.flags, QM_STOP);
4806 cmd = QM_VF_PREPARE_FAIL;
4807 goto err_prepare;
4808 } else {
4809 goto out;
4810 }
4811
4812 err_prepare:
4813 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4814 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4815 out:
4816 pci_save_state(pdev);
4817 ret = qm_ping_pf(qm, cmd);
4818 if (ret)
4819 dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n");
4820 }
4821
qm_pf_reset_vf_done(struct hisi_qm * qm)4822 static void qm_pf_reset_vf_done(struct hisi_qm *qm)
4823 {
4824 enum qm_ifc_cmd cmd = QM_VF_START_DONE;
4825 struct pci_dev *pdev = qm->pdev;
4826 int ret;
4827
4828 pci_restore_state(pdev);
4829 ret = hisi_qm_start(qm);
4830 if (ret) {
4831 dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret);
4832 cmd = QM_VF_START_FAIL;
4833 }
4834
4835 qm_cmd_init(qm);
4836 ret = qm_ping_pf(qm, cmd);
4837 if (ret)
4838 dev_warn(&pdev->dev, "PF responds timeout in reset done!\n");
4839
4840 qm_reset_bit_clear(qm);
4841 }
4842
qm_wait_pf_reset_finish(struct hisi_qm * qm)4843 static int qm_wait_pf_reset_finish(struct hisi_qm *qm)
4844 {
4845 struct device *dev = &qm->pdev->dev;
4846 u32 val, cmd;
4847 int ret;
4848
4849 /* Wait for reset to finish */
4850 ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val,
4851 val == BIT(0), QM_VF_RESET_WAIT_US,
4852 QM_VF_RESET_WAIT_TIMEOUT_US);
4853 /* hardware completion status should be available by this time */
4854 if (ret) {
4855 dev_err(dev, "couldn't get reset done status from PF, timeout!\n");
4856 return -ETIMEDOUT;
4857 }
4858
4859 /*
4860 * Whether message is got successfully,
4861 * VF needs to ack PF by clearing the interrupt.
4862 */
4863 ret = qm->ops->get_ifc(qm, &cmd, NULL, 0);
4864 qm_clear_cmd_interrupt(qm, 0);
4865 if (ret) {
4866 dev_err(dev, "failed to get command from PF in reset done!\n");
4867 return ret;
4868 }
4869
4870 if (cmd != QM_PF_RESET_DONE) {
4871 dev_err(dev, "the command(0x%x) is not reset done!\n", cmd);
4872 ret = -EINVAL;
4873 }
4874
4875 return ret;
4876 }
4877
qm_pf_reset_vf_process(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4878 static void qm_pf_reset_vf_process(struct hisi_qm *qm,
4879 enum qm_stop_reason stop_reason)
4880 {
4881 struct device *dev = &qm->pdev->dev;
4882 int ret;
4883
4884 dev_info(dev, "device reset start...\n");
4885
4886 /* The message is obtained by querying the register during resetting */
4887 qm_cmd_uninit(qm);
4888 qm_pf_reset_vf_prepare(qm, stop_reason);
4889
4890 ret = qm_wait_pf_reset_finish(qm);
4891 if (ret)
4892 goto err_get_status;
4893
4894 qm_pf_reset_vf_done(qm);
4895
4896 dev_info(dev, "device reset done.\n");
4897
4898 return;
4899
4900 err_get_status:
4901 qm_cmd_init(qm);
4902 qm_reset_bit_clear(qm);
4903 }
4904
qm_handle_cmd_msg(struct hisi_qm * qm,u32 fun_num)4905 static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num)
4906 {
4907 struct device *dev = &qm->pdev->dev;
4908 enum qm_ifc_cmd cmd;
4909 u32 data;
4910 int ret;
4911
4912 /*
4913 * Get the msg from source by sending mailbox. Whether message is got
4914 * successfully, destination needs to ack source by clearing the interrupt.
4915 */
4916 ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num);
4917 qm_clear_cmd_interrupt(qm, BIT(fun_num));
4918 if (ret) {
4919 dev_err(dev, "failed to get command from source!\n");
4920 return;
4921 }
4922
4923 switch (cmd) {
4924 case QM_PF_FLR_PREPARE:
4925 qm_pf_reset_vf_process(qm, QM_DOWN);
4926 break;
4927 case QM_PF_SRST_PREPARE:
4928 qm_pf_reset_vf_process(qm, QM_SOFT_RESET);
4929 break;
4930 case QM_VF_GET_QOS:
4931 qm_vf_get_qos(qm, fun_num);
4932 break;
4933 case QM_PF_SET_QOS:
4934 qm->mb_qos = data;
4935 break;
4936 default:
4937 dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num);
4938 break;
4939 }
4940 }
4941
qm_cmd_process(struct work_struct * cmd_process)4942 static void qm_cmd_process(struct work_struct *cmd_process)
4943 {
4944 struct hisi_qm *qm = container_of(cmd_process,
4945 struct hisi_qm, cmd_process);
4946 u32 vfs_num = qm->vfs_num;
4947 u64 val;
4948 u32 i;
4949
4950 if (qm->fun_type == QM_HW_PF) {
4951 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
4952 if (!val)
4953 return;
4954
4955 for (i = 1; i <= vfs_num; i++) {
4956 if (val & BIT(i))
4957 qm_handle_cmd_msg(qm, i);
4958 }
4959
4960 return;
4961 }
4962
4963 qm_handle_cmd_msg(qm, 0);
4964 }
4965
4966 /**
4967 * hisi_qm_alg_register() - Register alg to crypto.
4968 * @qm: The qm needs add.
4969 * @qm_list: The qm list.
4970 * @guard: Guard of qp_num.
4971 *
4972 * Register algorithm to crypto when the function is satisfy guard.
4973 */
hisi_qm_alg_register(struct hisi_qm * qm,struct hisi_qm_list * qm_list,int guard)4974 int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard)
4975 {
4976 struct device *dev = &qm->pdev->dev;
4977
4978 if (qm->ver <= QM_HW_V2 && qm->use_sva) {
4979 dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n");
4980 return 0;
4981 }
4982
4983 if (qm->qp_num < guard) {
4984 dev_info(dev, "qp_num is less than task need.\n");
4985 return 0;
4986 }
4987
4988 return qm_list->register_to_crypto(qm);
4989 }
4990 EXPORT_SYMBOL_GPL(hisi_qm_alg_register);
4991
4992 /**
4993 * hisi_qm_alg_unregister() - Unregister alg from crypto.
4994 * @qm: The qm needs delete.
4995 * @qm_list: The qm list.
4996 * @guard: Guard of qp_num.
4997 *
4998 * Unregister algorithm from crypto when the last function is satisfy guard.
4999 */
hisi_qm_alg_unregister(struct hisi_qm * qm,struct hisi_qm_list * qm_list,int guard)5000 void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard)
5001 {
5002 if (qm->ver <= QM_HW_V2 && qm->use_sva)
5003 return;
5004
5005 if (qm->qp_num < guard)
5006 return;
5007
5008 qm_list->unregister_from_crypto(qm);
5009 }
5010 EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);
5011
qm_unregister_abnormal_irq(struct hisi_qm * qm)5012 static void qm_unregister_abnormal_irq(struct hisi_qm *qm)
5013 {
5014 struct pci_dev *pdev = qm->pdev;
5015 u32 irq_vector, val;
5016
5017 if (qm->fun_type == QM_HW_VF)
5018 return;
5019
5020 val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val;
5021 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
5022 return;
5023
5024 irq_vector = val & QM_IRQ_VECTOR_MASK;
5025 free_irq(pci_irq_vector(pdev, irq_vector), qm);
5026 }
5027
qm_register_abnormal_irq(struct hisi_qm * qm)5028 static int qm_register_abnormal_irq(struct hisi_qm *qm)
5029 {
5030 struct pci_dev *pdev = qm->pdev;
5031 u32 irq_vector, val;
5032 int ret;
5033
5034 if (qm->fun_type == QM_HW_VF)
5035 return 0;
5036
5037 val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val;
5038 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
5039 return 0;
5040
5041 irq_vector = val & QM_IRQ_VECTOR_MASK;
5042 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm);
5043 if (ret)
5044 dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret);
5045
5046 return ret;
5047 }
5048
qm_unregister_mb_cmd_irq(struct hisi_qm * qm)5049 static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm)
5050 {
5051 struct pci_dev *pdev = qm->pdev;
5052 u32 irq_vector, val;
5053
5054 val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val;
5055 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5056 return;
5057
5058 irq_vector = val & QM_IRQ_VECTOR_MASK;
5059 free_irq(pci_irq_vector(pdev, irq_vector), qm);
5060 }
5061
qm_register_mb_cmd_irq(struct hisi_qm * qm)5062 static int qm_register_mb_cmd_irq(struct hisi_qm *qm)
5063 {
5064 struct pci_dev *pdev = qm->pdev;
5065 u32 irq_vector, val;
5066 int ret;
5067
5068 val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val;
5069 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5070 return 0;
5071
5072 irq_vector = val & QM_IRQ_VECTOR_MASK;
5073 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm);
5074 if (ret)
5075 dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret);
5076
5077 return ret;
5078 }
5079
qm_unregister_aeq_irq(struct hisi_qm * qm)5080 static void qm_unregister_aeq_irq(struct hisi_qm *qm)
5081 {
5082 struct pci_dev *pdev = qm->pdev;
5083 u32 irq_vector, val;
5084
5085 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val;
5086 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5087 return;
5088
5089 irq_vector = val & QM_IRQ_VECTOR_MASK;
5090 free_irq(pci_irq_vector(pdev, irq_vector), qm);
5091 }
5092
qm_register_aeq_irq(struct hisi_qm * qm)5093 static int qm_register_aeq_irq(struct hisi_qm *qm)
5094 {
5095 struct pci_dev *pdev = qm->pdev;
5096 u32 irq_vector, val;
5097 int ret;
5098
5099 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val;
5100 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5101 return 0;
5102
5103 irq_vector = val & QM_IRQ_VECTOR_MASK;
5104 ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL,
5105 qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm);
5106 if (ret)
5107 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);
5108
5109 return ret;
5110 }
5111
qm_unregister_eq_irq(struct hisi_qm * qm)5112 static void qm_unregister_eq_irq(struct hisi_qm *qm)
5113 {
5114 struct pci_dev *pdev = qm->pdev;
5115 u32 irq_vector, val;
5116
5117 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val;
5118 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5119 return;
5120
5121 irq_vector = val & QM_IRQ_VECTOR_MASK;
5122 free_irq(pci_irq_vector(pdev, irq_vector), qm);
5123 }
5124
qm_register_eq_irq(struct hisi_qm * qm)5125 static int qm_register_eq_irq(struct hisi_qm *qm)
5126 {
5127 struct pci_dev *pdev = qm->pdev;
5128 u32 irq_vector, val;
5129 int ret;
5130
5131 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val;
5132 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5133 return 0;
5134
5135 irq_vector = val & QM_IRQ_VECTOR_MASK;
5136 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm);
5137 if (ret)
5138 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);
5139
5140 return ret;
5141 }
5142
qm_irqs_unregister(struct hisi_qm * qm)5143 static void qm_irqs_unregister(struct hisi_qm *qm)
5144 {
5145 qm_unregister_mb_cmd_irq(qm);
5146 qm_unregister_abnormal_irq(qm);
5147 qm_unregister_aeq_irq(qm);
5148 qm_unregister_eq_irq(qm);
5149 }
5150
qm_irqs_register(struct hisi_qm * qm)5151 static int qm_irqs_register(struct hisi_qm *qm)
5152 {
5153 int ret;
5154
5155 ret = qm_register_eq_irq(qm);
5156 if (ret)
5157 return ret;
5158
5159 ret = qm_register_aeq_irq(qm);
5160 if (ret)
5161 goto free_eq_irq;
5162
5163 ret = qm_register_abnormal_irq(qm);
5164 if (ret)
5165 goto free_aeq_irq;
5166
5167 ret = qm_register_mb_cmd_irq(qm);
5168 if (ret)
5169 goto free_abnormal_irq;
5170
5171 return 0;
5172
5173 free_abnormal_irq:
5174 qm_unregister_abnormal_irq(qm);
5175 free_aeq_irq:
5176 qm_unregister_aeq_irq(qm);
5177 free_eq_irq:
5178 qm_unregister_eq_irq(qm);
5179 return ret;
5180 }
5181
qm_get_qp_num(struct hisi_qm * qm)5182 static int qm_get_qp_num(struct hisi_qm *qm)
5183 {
5184 struct device *dev = &qm->pdev->dev;
5185 bool is_db_isolation;
5186
5187 /* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */
5188 if (qm->fun_type == QM_HW_VF) {
5189 if (qm->ver != QM_HW_V1)
5190 /* v2 starts to support get vft by mailbox */
5191 return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
5192
5193 return 0;
5194 }
5195
5196 is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);
5197 qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true);
5198 qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info,
5199 QM_FUNC_MAX_QP_CAP, is_db_isolation);
5200
5201 if (qm->qp_num <= qm->max_qp_num)
5202 return 0;
5203
5204 if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) {
5205 /* Check whether the set qp number is valid */
5206 dev_err(dev, "qp num(%u) is more than max qp num(%u)!\n",
5207 qm->qp_num, qm->max_qp_num);
5208 return -EINVAL;
5209 }
5210
5211 dev_info(dev, "Default qp num(%u) is too big, reset it to Function's max qp num(%u)!\n",
5212 qm->qp_num, qm->max_qp_num);
5213 qm->qp_num = qm->max_qp_num;
5214 qm->debug.curr_qm_qp_num = qm->qp_num;
5215
5216 return 0;
5217 }
5218
qm_pre_store_caps(struct hisi_qm * qm)5219 static int qm_pre_store_caps(struct hisi_qm *qm)
5220 {
5221 struct hisi_qm_cap_record *qm_cap;
5222 struct pci_dev *pdev = qm->pdev;
5223 size_t i, size;
5224
5225 size = ARRAY_SIZE(qm_cap_query_info);
5226 qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL);
5227 if (!qm_cap)
5228 return -ENOMEM;
5229
5230 for (i = 0; i < size; i++) {
5231 qm_cap[i].type = qm_cap_query_info[i].type;
5232 qm_cap[i].name = qm_cap_query_info[i].name;
5233 qm_cap[i].cap_val = hisi_qm_get_cap_value(qm, qm_cap_query_info,
5234 i, qm->cap_ver);
5235 }
5236
5237 qm->cap_tables.qm_cap_table = qm_cap;
5238 qm->cap_tables.qm_cap_size = size;
5239
5240 return 0;
5241 }
5242
qm_get_hw_caps(struct hisi_qm * qm)5243 static int qm_get_hw_caps(struct hisi_qm *qm)
5244 {
5245 const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ?
5246 qm_cap_info_pf : qm_cap_info_vf;
5247 u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) :
5248 ARRAY_SIZE(qm_cap_info_vf);
5249 u32 val, i;
5250
5251 /* Doorbell isolate register is a independent register. */
5252 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true);
5253 if (val)
5254 set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);
5255
5256 if (qm->ver >= QM_HW_V3) {
5257 val = readl(qm->io_base + QM_FUNC_CAPS_REG);
5258 qm->cap_ver = val & QM_CAPBILITY_VERSION;
5259 }
5260
5261 /* Get PF/VF common capbility */
5262 for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) {
5263 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver);
5264 if (val)
5265 set_bit(qm_cap_info_comm[i].type, &qm->caps);
5266 }
5267
5268 /* Get PF/VF different capbility */
5269 for (i = 0; i < size; i++) {
5270 val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver);
5271 if (val)
5272 set_bit(cap_info[i].type, &qm->caps);
5273 }
5274
5275 /* Fetch and save the value of qm capability registers */
5276 return qm_pre_store_caps(qm);
5277 }
5278
qm_get_version(struct hisi_qm * qm)5279 static void qm_get_version(struct hisi_qm *qm)
5280 {
5281 struct pci_dev *pdev = qm->pdev;
5282 u32 sub_version_id;
5283
5284 qm->ver = pdev->revision;
5285
5286 if (pdev->revision == QM_HW_V3) {
5287 sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID);
5288 if (sub_version_id)
5289 qm->ver = sub_version_id;
5290 }
5291 }
5292
qm_get_pci_res(struct hisi_qm * qm)5293 static int qm_get_pci_res(struct hisi_qm *qm)
5294 {
5295 struct pci_dev *pdev = qm->pdev;
5296 struct device *dev = &pdev->dev;
5297 int ret;
5298
5299 ret = pci_request_mem_regions(pdev, qm->dev_name);
5300 if (ret < 0) {
5301 dev_err(dev, "Failed to request mem regions!\n");
5302 return ret;
5303 }
5304
5305 qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);
5306 qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2));
5307 if (!qm->io_base) {
5308 ret = -EIO;
5309 goto err_request_mem_regions;
5310 }
5311
5312 qm_get_version(qm);
5313
5314 ret = qm_get_hw_caps(qm);
5315 if (ret)
5316 goto err_ioremap;
5317
5318 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {
5319 qm->db_interval = QM_QP_DB_INTERVAL;
5320 qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);
5321 qm->db_io_base = ioremap(qm->db_phys_base,
5322 pci_resource_len(pdev, PCI_BAR_4));
5323 if (!qm->db_io_base) {
5324 ret = -EIO;
5325 goto err_ioremap;
5326 }
5327 } else {
5328 qm->db_phys_base = qm->phys_base;
5329 qm->db_io_base = qm->io_base;
5330 qm->db_interval = 0;
5331 }
5332
5333 hisi_qm_pre_init(qm);
5334 ret = qm_get_qp_num(qm);
5335 if (ret)
5336 goto err_db_ioremap;
5337
5338 return 0;
5339
5340 err_db_ioremap:
5341 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
5342 iounmap(qm->db_io_base);
5343 err_ioremap:
5344 iounmap(qm->io_base);
5345 err_request_mem_regions:
5346 pci_release_mem_regions(pdev);
5347 return ret;
5348 }
5349
qm_clear_device(struct hisi_qm * qm)5350 static int qm_clear_device(struct hisi_qm *qm)
5351 {
5352 acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev);
5353 int ret;
5354
5355 if (qm->fun_type == QM_HW_VF)
5356 return 0;
5357
5358 /* Device does not support reset, return */
5359 if (!qm->err_ini->err_info_init)
5360 return 0;
5361 qm->err_ini->err_info_init(qm);
5362
5363 if (!handle)
5364 return 0;
5365
5366 /* No reset method, return */
5367 if (!acpi_has_method(handle, qm->err_info.acpi_rst))
5368 return 0;
5369
5370 ret = qm_master_ooo_check(qm);
5371 if (ret) {
5372 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL);
5373 return ret;
5374 }
5375
5376 if (qm->err_ini->set_priv_status) {
5377 ret = qm->err_ini->set_priv_status(qm);
5378 if (ret) {
5379 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL);
5380 return ret;
5381 }
5382 }
5383
5384 return qm_reset_device(qm);
5385 }
5386
hisi_qm_pci_init(struct hisi_qm * qm)5387 static int hisi_qm_pci_init(struct hisi_qm *qm)
5388 {
5389 struct pci_dev *pdev = qm->pdev;
5390 struct device *dev = &pdev->dev;
5391 unsigned int num_vec;
5392 int ret;
5393
5394 ret = pci_enable_device_mem(pdev);
5395 if (ret < 0) {
5396 dev_err(dev, "Failed to enable device mem!\n");
5397 return ret;
5398 }
5399
5400 ret = qm_get_pci_res(qm);
5401 if (ret)
5402 goto err_disable_pcidev;
5403
5404 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
5405 if (ret < 0)
5406 goto err_get_pci_res;
5407 pci_set_master(pdev);
5408
5409 num_vec = qm_get_irq_num(qm);
5410 ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
5411 if (ret < 0) {
5412 dev_err(dev, "Failed to enable MSI vectors!\n");
5413 goto err_get_pci_res;
5414 }
5415
5416 ret = qm_clear_device(qm);
5417 if (ret)
5418 goto err_free_vectors;
5419
5420 return 0;
5421
5422 err_free_vectors:
5423 pci_free_irq_vectors(pdev);
5424 err_get_pci_res:
5425 qm_put_pci_res(qm);
5426 err_disable_pcidev:
5427 pci_disable_device(pdev);
5428 return ret;
5429 }
5430
hisi_qm_init_work(struct hisi_qm * qm)5431 static int hisi_qm_init_work(struct hisi_qm *qm)
5432 {
5433 int i;
5434
5435 for (i = 0; i < qm->qp_num; i++)
5436 INIT_WORK(&qm->poll_data[i].work, qm_work_process);
5437
5438 if (qm->fun_type == QM_HW_PF)
5439 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset);
5440
5441 if (qm->ver > QM_HW_V2)
5442 INIT_WORK(&qm->cmd_process, qm_cmd_process);
5443
5444 qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM |
5445 WQ_UNBOUND, num_online_cpus(),
5446 pci_name(qm->pdev));
5447 if (!qm->wq) {
5448 pci_err(qm->pdev, "failed to alloc workqueue!\n");
5449 return -ENOMEM;
5450 }
5451
5452 return 0;
5453 }
5454
hisi_qp_alloc_memory(struct hisi_qm * qm)5455 static int hisi_qp_alloc_memory(struct hisi_qm *qm)
5456 {
5457 struct device *dev = &qm->pdev->dev;
5458 u16 sq_depth, cq_depth;
5459 size_t qp_dma_size;
5460 int i, ret;
5461
5462 qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);
5463 if (!qm->qp_array)
5464 return -ENOMEM;
5465
5466 qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL);
5467 if (!qm->poll_data) {
5468 kfree(qm->qp_array);
5469 return -ENOMEM;
5470 }
5471
5472 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);
5473
5474 /* one more page for device or qp statuses */
5475 qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth;
5476 qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE;
5477 for (i = 0; i < qm->qp_num; i++) {
5478 qm->poll_data[i].qm = qm;
5479 ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth);
5480 if (ret)
5481 goto err_init_qp_mem;
5482
5483 dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);
5484 }
5485
5486 return 0;
5487 err_init_qp_mem:
5488 hisi_qp_memory_uninit(qm, i);
5489
5490 return ret;
5491 }
5492
hisi_qm_alloc_rsv_buf(struct hisi_qm * qm)5493 static int hisi_qm_alloc_rsv_buf(struct hisi_qm *qm)
5494 {
5495 struct qm_rsv_buf *xqc_buf = &qm->xqc_buf;
5496 struct qm_dma *xqc_dma = &xqc_buf->qcdma;
5497 struct device *dev = &qm->pdev->dev;
5498 size_t off = 0;
5499
5500 #define QM_XQC_BUF_INIT(xqc_buf, type) do { \
5501 (xqc_buf)->type = ((xqc_buf)->qcdma.va + (off)); \
5502 (xqc_buf)->type##_dma = (xqc_buf)->qcdma.dma + (off); \
5503 off += QMC_ALIGN(sizeof(struct qm_##type)); \
5504 } while (0)
5505
5506 xqc_dma->size = QMC_ALIGN(sizeof(struct qm_eqc)) +
5507 QMC_ALIGN(sizeof(struct qm_aeqc)) +
5508 QMC_ALIGN(sizeof(struct qm_sqc)) +
5509 QMC_ALIGN(sizeof(struct qm_cqc));
5510 xqc_dma->va = dma_alloc_coherent(dev, xqc_dma->size,
5511 &xqc_dma->dma, GFP_KERNEL);
5512 if (!xqc_dma->va)
5513 return -ENOMEM;
5514
5515 QM_XQC_BUF_INIT(xqc_buf, eqc);
5516 QM_XQC_BUF_INIT(xqc_buf, aeqc);
5517 QM_XQC_BUF_INIT(xqc_buf, sqc);
5518 QM_XQC_BUF_INIT(xqc_buf, cqc);
5519
5520 return 0;
5521 }
5522
hisi_qm_memory_init(struct hisi_qm * qm)5523 static int hisi_qm_memory_init(struct hisi_qm *qm)
5524 {
5525 struct device *dev = &qm->pdev->dev;
5526 int ret, total_func;
5527 size_t off = 0;
5528
5529 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {
5530 total_func = pci_sriov_get_totalvfs(qm->pdev) + 1;
5531 qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL);
5532 if (!qm->factor)
5533 return -ENOMEM;
5534
5535 /* Only the PF value needs to be initialized */
5536 qm->factor[0].func_qos = QM_QOS_MAX_VAL;
5537 }
5538
5539 #define QM_INIT_BUF(qm, type, num) do { \
5540 (qm)->type = ((qm)->qdma.va + (off)); \
5541 (qm)->type##_dma = (qm)->qdma.dma + (off); \
5542 off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
5543 } while (0)
5544
5545 idr_init(&qm->qp_idr);
5546 qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP);
5547 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) +
5548 QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) +
5549 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
5550 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
5551 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,
5552 GFP_ATOMIC);
5553 dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);
5554 if (!qm->qdma.va) {
5555 ret = -ENOMEM;
5556 goto err_destroy_idr;
5557 }
5558
5559 QM_INIT_BUF(qm, eqe, qm->eq_depth);
5560 QM_INIT_BUF(qm, aeqe, qm->aeq_depth);
5561 QM_INIT_BUF(qm, sqc, qm->qp_num);
5562 QM_INIT_BUF(qm, cqc, qm->qp_num);
5563
5564 ret = hisi_qm_alloc_rsv_buf(qm);
5565 if (ret)
5566 goto err_free_qdma;
5567
5568 ret = hisi_qp_alloc_memory(qm);
5569 if (ret)
5570 goto err_free_reserve_buf;
5571
5572 return 0;
5573
5574 err_free_reserve_buf:
5575 hisi_qm_free_rsv_buf(qm);
5576 err_free_qdma:
5577 dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
5578 err_destroy_idr:
5579 idr_destroy(&qm->qp_idr);
5580 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
5581 kfree(qm->factor);
5582
5583 return ret;
5584 }
5585
5586 /**
5587 * hisi_qm_init() - Initialize configures about qm.
5588 * @qm: The qm needing init.
5589 *
5590 * This function init qm, then we can call hisi_qm_start to put qm into work.
5591 */
hisi_qm_init(struct hisi_qm * qm)5592 int hisi_qm_init(struct hisi_qm *qm)
5593 {
5594 struct pci_dev *pdev = qm->pdev;
5595 struct device *dev = &pdev->dev;
5596 int ret;
5597
5598 ret = hisi_qm_pci_init(qm);
5599 if (ret)
5600 return ret;
5601
5602 ret = qm_irqs_register(qm);
5603 if (ret)
5604 goto err_pci_init;
5605
5606 if (qm->fun_type == QM_HW_PF) {
5607 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */
5608 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);
5609 qm_disable_clock_gate(qm);
5610 ret = qm_dev_mem_reset(qm);
5611 if (ret) {
5612 dev_err(dev, "failed to reset device memory\n");
5613 goto err_irq_register;
5614 }
5615 }
5616
5617 if (qm->mode == UACCE_MODE_SVA) {
5618 ret = qm_alloc_uacce(qm);
5619 if (ret < 0)
5620 dev_warn(dev, "fail to alloc uacce (%d)\n", ret);
5621 }
5622
5623 ret = hisi_qm_memory_init(qm);
5624 if (ret)
5625 goto err_alloc_uacce;
5626
5627 ret = hisi_qm_init_work(qm);
5628 if (ret)
5629 goto err_free_qm_memory;
5630
5631 qm_cmd_init(qm);
5632
5633 return 0;
5634
5635 err_free_qm_memory:
5636 hisi_qm_memory_uninit(qm);
5637 err_alloc_uacce:
5638 qm_remove_uacce(qm);
5639 err_irq_register:
5640 qm_irqs_unregister(qm);
5641 err_pci_init:
5642 hisi_qm_pci_uninit(qm);
5643 return ret;
5644 }
5645 EXPORT_SYMBOL_GPL(hisi_qm_init);
5646
5647 /**
5648 * hisi_qm_get_dfx_access() - Try to get dfx access.
5649 * @qm: pointer to accelerator device.
5650 *
5651 * Try to get dfx access, then user can get message.
5652 *
5653 * If device is in suspended, return failure, otherwise
5654 * bump up the runtime PM usage counter.
5655 */
hisi_qm_get_dfx_access(struct hisi_qm * qm)5656 int hisi_qm_get_dfx_access(struct hisi_qm *qm)
5657 {
5658 struct device *dev = &qm->pdev->dev;
5659
5660 if (pm_runtime_suspended(dev)) {
5661 dev_info(dev, "can not read/write - device in suspended.\n");
5662 return -EAGAIN;
5663 }
5664
5665 return qm_pm_get_sync(qm);
5666 }
5667 EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access);
5668
5669 /**
5670 * hisi_qm_put_dfx_access() - Put dfx access.
5671 * @qm: pointer to accelerator device.
5672 *
5673 * Put dfx access, drop runtime PM usage counter.
5674 */
hisi_qm_put_dfx_access(struct hisi_qm * qm)5675 void hisi_qm_put_dfx_access(struct hisi_qm *qm)
5676 {
5677 qm_pm_put_sync(qm);
5678 }
5679 EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access);
5680
5681 /**
5682 * hisi_qm_pm_init() - Initialize qm runtime PM.
5683 * @qm: pointer to accelerator device.
5684 *
5685 * Function that initialize qm runtime PM.
5686 */
hisi_qm_pm_init(struct hisi_qm * qm)5687 void hisi_qm_pm_init(struct hisi_qm *qm)
5688 {
5689 struct device *dev = &qm->pdev->dev;
5690
5691 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
5692 return;
5693
5694 pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY);
5695 pm_runtime_use_autosuspend(dev);
5696 pm_runtime_put_noidle(dev);
5697 }
5698 EXPORT_SYMBOL_GPL(hisi_qm_pm_init);
5699
5700 /**
5701 * hisi_qm_pm_uninit() - Uninitialize qm runtime PM.
5702 * @qm: pointer to accelerator device.
5703 *
5704 * Function that uninitialize qm runtime PM.
5705 */
hisi_qm_pm_uninit(struct hisi_qm * qm)5706 void hisi_qm_pm_uninit(struct hisi_qm *qm)
5707 {
5708 struct device *dev = &qm->pdev->dev;
5709
5710 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
5711 return;
5712
5713 pm_runtime_get_noresume(dev);
5714 pm_runtime_dont_use_autosuspend(dev);
5715 }
5716 EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit);
5717
qm_prepare_for_suspend(struct hisi_qm * qm)5718 static int qm_prepare_for_suspend(struct hisi_qm *qm)
5719 {
5720 struct pci_dev *pdev = qm->pdev;
5721 int ret;
5722
5723 ret = qm->ops->set_msi(qm, false);
5724 if (ret) {
5725 pci_err(pdev, "failed to disable MSI before suspending!\n");
5726 return ret;
5727 }
5728
5729 ret = qm_master_ooo_check(qm);
5730 if (ret)
5731 return ret;
5732
5733 if (qm->err_ini->set_priv_status) {
5734 ret = qm->err_ini->set_priv_status(qm);
5735 if (ret)
5736 return ret;
5737 }
5738
5739 ret = qm_set_pf_mse(qm, false);
5740 if (ret)
5741 pci_err(pdev, "failed to disable MSE before suspending!\n");
5742
5743 return ret;
5744 }
5745
qm_rebuild_for_resume(struct hisi_qm * qm)5746 static int qm_rebuild_for_resume(struct hisi_qm *qm)
5747 {
5748 struct pci_dev *pdev = qm->pdev;
5749 int ret;
5750
5751 ret = qm_set_pf_mse(qm, true);
5752 if (ret) {
5753 pci_err(pdev, "failed to enable MSE after resuming!\n");
5754 return ret;
5755 }
5756
5757 ret = qm->ops->set_msi(qm, true);
5758 if (ret) {
5759 pci_err(pdev, "failed to enable MSI after resuming!\n");
5760 return ret;
5761 }
5762
5763 ret = qm_dev_hw_init(qm);
5764 if (ret) {
5765 pci_err(pdev, "failed to init device after resuming\n");
5766 return ret;
5767 }
5768
5769 qm_cmd_init(qm);
5770 hisi_qm_dev_err_init(qm);
5771 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */
5772 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);
5773 qm_disable_clock_gate(qm);
5774 ret = qm_dev_mem_reset(qm);
5775 if (ret)
5776 pci_err(pdev, "failed to reset device memory\n");
5777
5778 return ret;
5779 }
5780
5781 /**
5782 * hisi_qm_suspend() - Runtime suspend of given device.
5783 * @dev: device to suspend.
5784 *
5785 * Function that suspend the device.
5786 */
hisi_qm_suspend(struct device * dev)5787 int hisi_qm_suspend(struct device *dev)
5788 {
5789 struct pci_dev *pdev = to_pci_dev(dev);
5790 struct hisi_qm *qm = pci_get_drvdata(pdev);
5791 int ret;
5792
5793 pci_info(pdev, "entering suspended state\n");
5794
5795 ret = hisi_qm_stop(qm, QM_NORMAL);
5796 if (ret) {
5797 pci_err(pdev, "failed to stop qm(%d)\n", ret);
5798 return ret;
5799 }
5800
5801 ret = qm_prepare_for_suspend(qm);
5802 if (ret)
5803 pci_err(pdev, "failed to prepare suspended(%d)\n", ret);
5804
5805 return ret;
5806 }
5807 EXPORT_SYMBOL_GPL(hisi_qm_suspend);
5808
5809 /**
5810 * hisi_qm_resume() - Runtime resume of given device.
5811 * @dev: device to resume.
5812 *
5813 * Function that resume the device.
5814 */
hisi_qm_resume(struct device * dev)5815 int hisi_qm_resume(struct device *dev)
5816 {
5817 struct pci_dev *pdev = to_pci_dev(dev);
5818 struct hisi_qm *qm = pci_get_drvdata(pdev);
5819 int ret;
5820
5821 pci_info(pdev, "resuming from suspend state\n");
5822
5823 ret = qm_rebuild_for_resume(qm);
5824 if (ret) {
5825 pci_err(pdev, "failed to rebuild resume(%d)\n", ret);
5826 return ret;
5827 }
5828
5829 ret = hisi_qm_start(qm);
5830 if (ret) {
5831 if (qm_check_dev_error(qm)) {
5832 pci_info(pdev, "failed to start qm due to device error, device will be reset!\n");
5833 return 0;
5834 }
5835
5836 pci_err(pdev, "failed to start qm(%d)!\n", ret);
5837 }
5838
5839 return ret;
5840 }
5841 EXPORT_SYMBOL_GPL(hisi_qm_resume);
5842
5843 MODULE_LICENSE("GPL v2");
5844 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
5845 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");
5846