1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <linux/acpi.h>
4 #include <linux/bitops.h>
5 #include <linux/debugfs.h>
6 #include <linux/init.h>
7 #include <linux/io.h>
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/seq_file.h>
13 #include <linux/topology.h>
14 #include <linux/uacce.h>
15 #include "zip.h"
16
17 #define PCI_DEVICE_ID_HUAWEI_ZIP_PF 0xa250
18
19 #define HZIP_QUEUE_NUM_V1 4096
20
21 #define HZIP_CLOCK_GATE_CTRL 0x301004
22 #define HZIP_DECOMP_CHECK_ENABLE BIT(16)
23 #define HZIP_FSM_MAX_CNT 0x301008
24
25 #define HZIP_PORT_ARCA_CHE_0 0x301040
26 #define HZIP_PORT_ARCA_CHE_1 0x301044
27 #define HZIP_PORT_AWCA_CHE_0 0x301060
28 #define HZIP_PORT_AWCA_CHE_1 0x301064
29 #define HZIP_CACHE_ALL_EN 0xffffffff
30
31 #define HZIP_BD_RUSER_32_63 0x301110
32 #define HZIP_SGL_RUSER_32_63 0x30111c
33 #define HZIP_DATA_RUSER_32_63 0x301128
34 #define HZIP_DATA_WUSER_32_63 0x301134
35 #define HZIP_BD_WUSER_32_63 0x301140
36
37 #define HZIP_QM_IDEL_STATUS 0x3040e4
38
39 #define HZIP_CORE_DFX_BASE 0x301000
40 #define HZIP_CLOCK_GATED_CONTL 0X301004
41 #define HZIP_CORE_DFX_COMP_0 0x302000
42 #define HZIP_CORE_DFX_COMP_1 0x303000
43 #define HZIP_CORE_DFX_DECOMP_0 0x304000
44 #define HZIP_CORE_DFX_DECOMP_1 0x305000
45 #define HZIP_CORE_DFX_DECOMP_2 0x306000
46 #define HZIP_CORE_DFX_DECOMP_3 0x307000
47 #define HZIP_CORE_DFX_DECOMP_4 0x308000
48 #define HZIP_CORE_DFX_DECOMP_5 0x309000
49 #define HZIP_CORE_REGS_BASE_LEN 0xB0
50 #define HZIP_CORE_REGS_DFX_LEN 0x28
51
52 #define HZIP_CORE_INT_SOURCE 0x3010A0
53 #define HZIP_CORE_INT_MASK_REG 0x3010A4
54 #define HZIP_CORE_INT_SET 0x3010A8
55 #define HZIP_CORE_INT_STATUS 0x3010AC
56 #define HZIP_CORE_INT_STATUS_M_ECC BIT(1)
57 #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148
58 #define HZIP_CORE_INT_RAS_CE_ENB 0x301160
59 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164
60 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168
61 #define HZIP_CORE_INT_RAS_FE_ENB_MASK 0x0
62 #define HZIP_OOO_SHUTDOWN_SEL 0x30120C
63 #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16
64 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24
65 #define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0)
66 #define HZIP_SQE_SIZE 128
67 #define HZIP_PF_DEF_Q_NUM 64
68 #define HZIP_PF_DEF_Q_BASE 0
69 #define HZIP_CTX_Q_NUM_DEF 2
70
71 #define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000
72 #define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT BIT(0)
73 #define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C
74 #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14)
75 #define HZIP_WR_PORT BIT(11)
76
77 #define HZIP_ALG_ZLIB_BIT GENMASK(1, 0)
78 #define HZIP_ALG_GZIP_BIT GENMASK(3, 2)
79 #define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4)
80 #define HZIP_ALG_LZ77_BIT GENMASK(7, 6)
81
82 #define HZIP_BUF_SIZE 22
83 #define HZIP_SQE_MASK_OFFSET 64
84 #define HZIP_SQE_MASK_LEN 48
85
86 #define HZIP_CNT_CLR_CE_EN BIT(0)
87 #define HZIP_RO_CNT_CLR_CE_EN BIT(2)
88 #define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \
89 HZIP_RO_CNT_CLR_CE_EN)
90
91 #define HZIP_PREFETCH_CFG 0x3011B0
92 #define HZIP_SVA_TRANS 0x3011C4
93 #define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0)))
94 #define HZIP_SVA_PREFETCH_DISABLE BIT(26)
95 #define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30))
96 #define HZIP_SHAPER_RATE_COMPRESS 750
97 #define HZIP_SHAPER_RATE_DECOMPRESS 140
98 #define HZIP_DELAY_1_US 1
99 #define HZIP_POLL_TIMEOUT_US 1000
100
101 /* clock gating */
102 #define HZIP_PEH_CFG_AUTO_GATE 0x3011A8
103 #define HZIP_PEH_CFG_AUTO_GATE_EN BIT(0)
104 #define HZIP_CORE_GATED_EN GENMASK(15, 8)
105 #define HZIP_CORE_GATED_OOO_EN BIT(29)
106 #define HZIP_CLOCK_GATED_EN (HZIP_CORE_GATED_EN | \
107 HZIP_CORE_GATED_OOO_EN)
108
109 /* zip comp high performance */
110 #define HZIP_HIGH_PERF_OFFSET 0x301208
111
112 enum {
113 HZIP_HIGH_COMP_RATE,
114 HZIP_HIGH_COMP_PERF,
115 };
116
117 static const char hisi_zip_name[] = "hisi_zip";
118 static struct dentry *hzip_debugfs_root;
119
120 struct hisi_zip_hw_error {
121 u32 int_msk;
122 const char *msg;
123 };
124
125 struct zip_dfx_item {
126 const char *name;
127 u32 offset;
128 };
129
130 static const struct qm_dev_alg zip_dev_algs[] = { {
131 .alg_msk = HZIP_ALG_ZLIB_BIT,
132 .alg = "zlib\n",
133 }, {
134 .alg_msk = HZIP_ALG_GZIP_BIT,
135 .alg = "gzip\n",
136 }, {
137 .alg_msk = HZIP_ALG_DEFLATE_BIT,
138 .alg = "deflate\n",
139 }, {
140 .alg_msk = HZIP_ALG_LZ77_BIT,
141 .alg = "lz77_zstd\n",
142 },
143 };
144
145 static struct hisi_qm_list zip_devices = {
146 .register_to_crypto = hisi_zip_register_to_crypto,
147 .unregister_from_crypto = hisi_zip_unregister_from_crypto,
148 };
149
150 static struct zip_dfx_item zip_dfx_files[] = {
151 {"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)},
152 {"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)},
153 {"send_busy_cnt", offsetof(struct hisi_zip_dfx, send_busy_cnt)},
154 {"err_bd_cnt", offsetof(struct hisi_zip_dfx, err_bd_cnt)},
155 };
156
157 static const struct hisi_zip_hw_error zip_hw_error[] = {
158 { .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" },
159 { .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" },
160 { .int_msk = BIT(2), .msg = "zip_axi_rresp_err" },
161 { .int_msk = BIT(3), .msg = "zip_axi_bresp_err" },
162 { .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" },
163 { .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" },
164 { .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" },
165 { .int_msk = BIT(7), .msg = "zip_pre_in_data_err" },
166 { .int_msk = BIT(8), .msg = "zip_com_inf_err" },
167 { .int_msk = BIT(9), .msg = "zip_enc_inf_err" },
168 { .int_msk = BIT(10), .msg = "zip_pre_out_err" },
169 { .int_msk = BIT(11), .msg = "zip_axi_poison_err" },
170 { .int_msk = BIT(12), .msg = "zip_sva_err" },
171 { /* sentinel */ }
172 };
173
174 enum ctrl_debug_file_index {
175 HZIP_CLEAR_ENABLE,
176 HZIP_DEBUG_FILE_NUM,
177 };
178
179 static const char * const ctrl_debug_file_name[] = {
180 [HZIP_CLEAR_ENABLE] = "clear_enable",
181 };
182
183 struct ctrl_debug_file {
184 enum ctrl_debug_file_index index;
185 spinlock_t lock;
186 struct hisi_zip_ctrl *ctrl;
187 };
188
189 /*
190 * One ZIP controller has one PF and multiple VFs, some global configurations
191 * which PF has need this structure.
192 *
193 * Just relevant for PF.
194 */
195 struct hisi_zip_ctrl {
196 struct hisi_zip *hisi_zip;
197 struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM];
198 };
199
200 enum zip_cap_type {
201 ZIP_QM_NFE_MASK_CAP = 0x0,
202 ZIP_QM_RESET_MASK_CAP,
203 ZIP_QM_OOO_SHUTDOWN_MASK_CAP,
204 ZIP_QM_CE_MASK_CAP,
205 ZIP_NFE_MASK_CAP,
206 ZIP_RESET_MASK_CAP,
207 ZIP_OOO_SHUTDOWN_MASK_CAP,
208 ZIP_CE_MASK_CAP,
209 ZIP_CLUSTER_NUM_CAP,
210 ZIP_CORE_TYPE_NUM_CAP,
211 ZIP_CORE_NUM_CAP,
212 ZIP_CLUSTER_COMP_NUM_CAP,
213 ZIP_CLUSTER_DECOMP_NUM_CAP,
214 ZIP_DECOMP_ENABLE_BITMAP,
215 ZIP_COMP_ENABLE_BITMAP,
216 ZIP_DRV_ALG_BITMAP,
217 ZIP_DEV_ALG_BITMAP,
218 ZIP_CORE1_ALG_BITMAP,
219 ZIP_CORE2_ALG_BITMAP,
220 ZIP_CORE3_ALG_BITMAP,
221 ZIP_CORE4_ALG_BITMAP,
222 ZIP_CORE5_ALG_BITMAP,
223 ZIP_CAP_MAX
224 };
225
226 static struct hisi_qm_cap_info zip_basic_cap_info[] = {
227 {ZIP_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C57, 0x7C77},
228 {ZIP_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC57, 0x6C77},
229 {ZIP_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77},
230 {ZIP_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8},
231 {ZIP_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x1FFE},
232 {ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE},
233 {ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE},
234 {ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1},
235 {ZIP_CLUSTER_NUM_CAP, 0x313C, 28, GENMASK(3, 0), 0x1, 0x1, 0x1},
236 {ZIP_CORE_TYPE_NUM_CAP, 0x313C, 24, GENMASK(3, 0), 0x2, 0x2, 0x2},
237 {ZIP_CORE_NUM_CAP, 0x313C, 16, GENMASK(7, 0), 0x8, 0x8, 0x5},
238 {ZIP_CLUSTER_COMP_NUM_CAP, 0x313C, 8, GENMASK(7, 0), 0x2, 0x2, 0x2},
239 {ZIP_CLUSTER_DECOMP_NUM_CAP, 0x313C, 0, GENMASK(7, 0), 0x6, 0x6, 0x3},
240 {ZIP_DECOMP_ENABLE_BITMAP, 0x3140, 16, GENMASK(15, 0), 0xFC, 0xFC, 0x1C},
241 {ZIP_COMP_ENABLE_BITMAP, 0x3140, 0, GENMASK(15, 0), 0x3, 0x3, 0x3},
242 {ZIP_DRV_ALG_BITMAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x0, 0x30},
243 {ZIP_DEV_ALG_BITMAP, 0x3148, 0, GENMASK(31, 0), 0xF, 0xF, 0x3F},
244 {ZIP_CORE1_ALG_BITMAP, 0x314C, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},
245 {ZIP_CORE2_ALG_BITMAP, 0x3150, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},
246 {ZIP_CORE3_ALG_BITMAP, 0x3154, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
247 {ZIP_CORE4_ALG_BITMAP, 0x3158, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
248 {ZIP_CORE5_ALG_BITMAP, 0x315C, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
249 {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0}
250 };
251
252 enum zip_pre_store_cap_idx {
253 ZIP_CORE_NUM_CAP_IDX = 0x0,
254 ZIP_CLUSTER_COMP_NUM_CAP_IDX,
255 ZIP_CLUSTER_DECOMP_NUM_CAP_IDX,
256 ZIP_DECOMP_ENABLE_BITMAP_IDX,
257 ZIP_COMP_ENABLE_BITMAP_IDX,
258 ZIP_DRV_ALG_BITMAP_IDX,
259 ZIP_DEV_ALG_BITMAP_IDX,
260 };
261
262 static const u32 zip_pre_store_caps[] = {
263 ZIP_CORE_NUM_CAP,
264 ZIP_CLUSTER_COMP_NUM_CAP,
265 ZIP_CLUSTER_DECOMP_NUM_CAP,
266 ZIP_DECOMP_ENABLE_BITMAP,
267 ZIP_COMP_ENABLE_BITMAP,
268 ZIP_DRV_ALG_BITMAP,
269 ZIP_DEV_ALG_BITMAP,
270 };
271
272 enum {
273 HZIP_COMP_CORE0,
274 HZIP_COMP_CORE1,
275 HZIP_DECOMP_CORE0,
276 HZIP_DECOMP_CORE1,
277 HZIP_DECOMP_CORE2,
278 HZIP_DECOMP_CORE3,
279 HZIP_DECOMP_CORE4,
280 HZIP_DECOMP_CORE5,
281 };
282
283 static const u64 core_offsets[] = {
284 [HZIP_COMP_CORE0] = 0x302000,
285 [HZIP_COMP_CORE1] = 0x303000,
286 [HZIP_DECOMP_CORE0] = 0x304000,
287 [HZIP_DECOMP_CORE1] = 0x305000,
288 [HZIP_DECOMP_CORE2] = 0x306000,
289 [HZIP_DECOMP_CORE3] = 0x307000,
290 [HZIP_DECOMP_CORE4] = 0x308000,
291 [HZIP_DECOMP_CORE5] = 0x309000,
292 };
293
294 static const struct debugfs_reg32 hzip_dfx_regs[] = {
295 {"HZIP_GET_BD_NUM ", 0x00},
296 {"HZIP_GET_RIGHT_BD ", 0x04},
297 {"HZIP_GET_ERROR_BD ", 0x08},
298 {"HZIP_DONE_BD_NUM ", 0x0c},
299 {"HZIP_WORK_CYCLE ", 0x10},
300 {"HZIP_IDLE_CYCLE ", 0x18},
301 {"HZIP_MAX_DELAY ", 0x20},
302 {"HZIP_MIN_DELAY ", 0x24},
303 {"HZIP_AVG_DELAY ", 0x28},
304 {"HZIP_MEM_VISIBLE_DATA ", 0x30},
305 {"HZIP_MEM_VISIBLE_ADDR ", 0x34},
306 {"HZIP_CONSUMED_BYTE ", 0x38},
307 {"HZIP_PRODUCED_BYTE ", 0x40},
308 {"HZIP_COMP_INF ", 0x70},
309 {"HZIP_PRE_OUT ", 0x78},
310 {"HZIP_BD_RD ", 0x7c},
311 {"HZIP_BD_WR ", 0x80},
312 {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84},
313 {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88},
314 {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8c},
315 {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94},
316 {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9c},
317 };
318
319 static const struct debugfs_reg32 hzip_com_dfx_regs[] = {
320 {"HZIP_CLOCK_GATE_CTRL ", 0x301004},
321 {"HZIP_CORE_INT_RAS_CE_ENB ", 0x301160},
322 {"HZIP_CORE_INT_RAS_NFE_ENB ", 0x301164},
323 {"HZIP_CORE_INT_RAS_FE_ENB ", 0x301168},
324 {"HZIP_UNCOM_ERR_RAS_CTRL ", 0x30116C},
325 };
326
327 static const struct debugfs_reg32 hzip_dump_dfx_regs[] = {
328 {"HZIP_GET_BD_NUM ", 0x00},
329 {"HZIP_GET_RIGHT_BD ", 0x04},
330 {"HZIP_GET_ERROR_BD ", 0x08},
331 {"HZIP_DONE_BD_NUM ", 0x0c},
332 {"HZIP_MAX_DELAY ", 0x20},
333 };
334
335 /* define the ZIP's dfx regs region and region length */
336 static struct dfx_diff_registers hzip_diff_regs[] = {
337 {
338 .reg_offset = HZIP_CORE_DFX_BASE,
339 .reg_len = HZIP_CORE_REGS_BASE_LEN,
340 }, {
341 .reg_offset = HZIP_CORE_DFX_COMP_0,
342 .reg_len = HZIP_CORE_REGS_DFX_LEN,
343 }, {
344 .reg_offset = HZIP_CORE_DFX_COMP_1,
345 .reg_len = HZIP_CORE_REGS_DFX_LEN,
346 }, {
347 .reg_offset = HZIP_CORE_DFX_DECOMP_0,
348 .reg_len = HZIP_CORE_REGS_DFX_LEN,
349 }, {
350 .reg_offset = HZIP_CORE_DFX_DECOMP_1,
351 .reg_len = HZIP_CORE_REGS_DFX_LEN,
352 }, {
353 .reg_offset = HZIP_CORE_DFX_DECOMP_2,
354 .reg_len = HZIP_CORE_REGS_DFX_LEN,
355 }, {
356 .reg_offset = HZIP_CORE_DFX_DECOMP_3,
357 .reg_len = HZIP_CORE_REGS_DFX_LEN,
358 }, {
359 .reg_offset = HZIP_CORE_DFX_DECOMP_4,
360 .reg_len = HZIP_CORE_REGS_DFX_LEN,
361 }, {
362 .reg_offset = HZIP_CORE_DFX_DECOMP_5,
363 .reg_len = HZIP_CORE_REGS_DFX_LEN,
364 },
365 };
366
hzip_diff_regs_show(struct seq_file * s,void * unused)367 static int hzip_diff_regs_show(struct seq_file *s, void *unused)
368 {
369 struct hisi_qm *qm = s->private;
370
371 hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,
372 ARRAY_SIZE(hzip_diff_regs));
373
374 return 0;
375 }
376 DEFINE_SHOW_ATTRIBUTE(hzip_diff_regs);
377
perf_mode_set(const char * val,const struct kernel_param * kp)378 static int perf_mode_set(const char *val, const struct kernel_param *kp)
379 {
380 int ret;
381 u32 n;
382
383 if (!val)
384 return -EINVAL;
385
386 ret = kstrtou32(val, 10, &n);
387 if (ret != 0 || (n != HZIP_HIGH_COMP_PERF &&
388 n != HZIP_HIGH_COMP_RATE))
389 return -EINVAL;
390
391 return param_set_int(val, kp);
392 }
393
394 static const struct kernel_param_ops zip_com_perf_ops = {
395 .set = perf_mode_set,
396 .get = param_get_int,
397 };
398
399 /*
400 * perf_mode = 0 means enable high compression rate mode,
401 * perf_mode = 1 means enable high compression performance mode.
402 * These two modes only apply to the compression direction.
403 */
404 static u32 perf_mode = HZIP_HIGH_COMP_RATE;
405 module_param_cb(perf_mode, &zip_com_perf_ops, &perf_mode, 0444);
406 MODULE_PARM_DESC(perf_mode, "ZIP high perf mode 0(default), 1(enable)");
407
408 static const struct kernel_param_ops zip_uacce_mode_ops = {
409 .set = uacce_mode_set,
410 .get = param_get_int,
411 };
412
413 /*
414 * uacce_mode = 0 means zip only register to crypto,
415 * uacce_mode = 1 means zip both register to crypto and uacce.
416 */
417 static u32 uacce_mode = UACCE_MODE_NOUACCE;
418 module_param_cb(uacce_mode, &zip_uacce_mode_ops, &uacce_mode, 0444);
419 MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);
420
421 static bool pf_q_num_flag;
pf_q_num_set(const char * val,const struct kernel_param * kp)422 static int pf_q_num_set(const char *val, const struct kernel_param *kp)
423 {
424 pf_q_num_flag = true;
425
426 return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_ZIP_PF);
427 }
428
429 static const struct kernel_param_ops pf_q_num_ops = {
430 .set = pf_q_num_set,
431 .get = param_get_int,
432 };
433
434 static u32 pf_q_num = HZIP_PF_DEF_Q_NUM;
435 module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444);
436 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
437
438 static const struct kernel_param_ops vfs_num_ops = {
439 .set = vfs_num_set,
440 .get = param_get_int,
441 };
442
443 static u32 vfs_num;
444 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
445 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
446
447 static const struct pci_device_id hisi_zip_dev_ids[] = {
448 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_PF) },
449 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },
450 { 0, }
451 };
452 MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids);
453
zip_create_qps(struct hisi_qp ** qps,int qp_num,int node)454 int zip_create_qps(struct hisi_qp **qps, int qp_num, int node)
455 {
456 if (node == NUMA_NO_NODE)
457 node = cpu_to_node(smp_processor_id());
458
459 return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps);
460 }
461
hisi_zip_alg_support(struct hisi_qm * qm,u32 alg)462 bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg)
463 {
464 u32 cap_val;
465
466 cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_IDX].cap_val;
467 if ((alg & cap_val) == alg)
468 return true;
469
470 return false;
471 }
472
hisi_zip_set_high_perf(struct hisi_qm * qm)473 static int hisi_zip_set_high_perf(struct hisi_qm *qm)
474 {
475 u32 val;
476 int ret;
477
478 val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET);
479 if (perf_mode == HZIP_HIGH_COMP_PERF)
480 val |= HZIP_HIGH_COMP_PERF;
481 else
482 val &= ~HZIP_HIGH_COMP_PERF;
483
484 /* Set perf mode */
485 writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET);
486 ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_HIGH_PERF_OFFSET,
487 val, val == perf_mode, HZIP_DELAY_1_US,
488 HZIP_POLL_TIMEOUT_US);
489 if (ret)
490 pci_err(qm->pdev, "failed to set perf mode\n");
491
492 return ret;
493 }
494
hisi_zip_open_sva_prefetch(struct hisi_qm * qm)495 static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm)
496 {
497 u32 val;
498 int ret;
499
500 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
501 return;
502
503 /* Enable prefetch */
504 val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
505 val &= HZIP_PREFETCH_ENABLE;
506 writel(val, qm->io_base + HZIP_PREFETCH_CFG);
507
508 ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG,
509 val, !(val & HZIP_SVA_PREFETCH_DISABLE),
510 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
511 if (ret)
512 pci_err(qm->pdev, "failed to open sva prefetch\n");
513 }
514
hisi_zip_close_sva_prefetch(struct hisi_qm * qm)515 static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm)
516 {
517 u32 val;
518 int ret;
519
520 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
521 return;
522
523 val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
524 val |= HZIP_SVA_PREFETCH_DISABLE;
525 writel(val, qm->io_base + HZIP_PREFETCH_CFG);
526
527 ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS,
528 val, !(val & HZIP_SVA_DISABLE_READY),
529 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
530 if (ret)
531 pci_err(qm->pdev, "failed to close sva prefetch\n");
532 }
533
hisi_zip_enable_clock_gate(struct hisi_qm * qm)534 static void hisi_zip_enable_clock_gate(struct hisi_qm *qm)
535 {
536 u32 val;
537
538 if (qm->ver < QM_HW_V3)
539 return;
540
541 val = readl(qm->io_base + HZIP_CLOCK_GATE_CTRL);
542 val |= HZIP_CLOCK_GATED_EN;
543 writel(val, qm->io_base + HZIP_CLOCK_GATE_CTRL);
544
545 val = readl(qm->io_base + HZIP_PEH_CFG_AUTO_GATE);
546 val |= HZIP_PEH_CFG_AUTO_GATE_EN;
547 writel(val, qm->io_base + HZIP_PEH_CFG_AUTO_GATE);
548 }
549
hisi_zip_set_user_domain_and_cache(struct hisi_qm * qm)550 static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
551 {
552 void __iomem *base = qm->io_base;
553 u32 dcomp_bm, comp_bm;
554
555 /* qm user domain */
556 writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1);
557 writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE);
558 writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1);
559 writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE);
560 writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE);
561
562 /* qm cache */
563 writel(AXI_M_CFG, base + QM_AXI_M_CFG);
564 writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE);
565
566 /* disable FLR triggered by BME(bus master enable) */
567 writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG);
568 writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE);
569
570 /* cache */
571 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0);
572 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1);
573 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0);
574 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1);
575
576 /* user domain configurations */
577 writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
578 writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
579
580 if (qm->use_sva && qm->ver == QM_HW_V2) {
581 writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);
582 writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);
583 writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63);
584 } else {
585 writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);
586 writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);
587 writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
588 }
589
590 /* let's open all compression/decompression cores */
591 dcomp_bm = qm->cap_tables.dev_cap_table[ZIP_DECOMP_ENABLE_BITMAP_IDX].cap_val;
592 comp_bm = qm->cap_tables.dev_cap_table[ZIP_COMP_ENABLE_BITMAP_IDX].cap_val;
593 writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL);
594
595 /* enable sqc,cqc writeback */
596 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
597 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
598 FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL);
599
600 hisi_zip_enable_clock_gate(qm);
601
602 return 0;
603 }
604
hisi_zip_master_ooo_ctrl(struct hisi_qm * qm,bool enable)605 static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
606 {
607 u32 val1, val2;
608
609 val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
610 if (enable) {
611 val1 |= HZIP_AXI_SHUTDOWN_ENABLE;
612 val2 = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
613 ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
614 } else {
615 val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE;
616 val2 = 0x0;
617 }
618
619 if (qm->ver > QM_HW_V2)
620 writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL);
621
622 writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
623 }
624
hisi_zip_hw_error_enable(struct hisi_qm * qm)625 static void hisi_zip_hw_error_enable(struct hisi_qm *qm)
626 {
627 u32 nfe, ce;
628
629 if (qm->ver == QM_HW_V1) {
630 writel(HZIP_CORE_INT_MASK_ALL,
631 qm->io_base + HZIP_CORE_INT_MASK_REG);
632 dev_info(&qm->pdev->dev, "Does not support hw error handle\n");
633 return;
634 }
635
636 nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);
637 ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver);
638
639 /* clear ZIP hw error source if having */
640 writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_SOURCE);
641
642 /* configure error type */
643 writel(ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB);
644 writel(HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB);
645 writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
646
647 hisi_zip_master_ooo_ctrl(qm, true);
648
649 /* enable ZIP hw error interrupts */
650 writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG);
651 }
652
hisi_zip_hw_error_disable(struct hisi_qm * qm)653 static void hisi_zip_hw_error_disable(struct hisi_qm *qm)
654 {
655 u32 nfe, ce;
656
657 /* disable ZIP hw error interrupts */
658 nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);
659 ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver);
660 writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG);
661
662 hisi_zip_master_ooo_ctrl(qm, false);
663 }
664
file_to_qm(struct ctrl_debug_file * file)665 static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)
666 {
667 struct hisi_zip *hisi_zip = file->ctrl->hisi_zip;
668
669 return &hisi_zip->qm;
670 }
671
clear_enable_read(struct hisi_qm * qm)672 static u32 clear_enable_read(struct hisi_qm *qm)
673 {
674 return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
675 HZIP_SOFT_CTRL_CNT_CLR_CE_BIT;
676 }
677
clear_enable_write(struct hisi_qm * qm,u32 val)678 static int clear_enable_write(struct hisi_qm *qm, u32 val)
679 {
680 u32 tmp;
681
682 if (val != 1 && val != 0)
683 return -EINVAL;
684
685 tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
686 ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val;
687 writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
688
689 return 0;
690 }
691
hisi_zip_ctrl_debug_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)692 static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf,
693 size_t count, loff_t *pos)
694 {
695 struct ctrl_debug_file *file = filp->private_data;
696 struct hisi_qm *qm = file_to_qm(file);
697 char tbuf[HZIP_BUF_SIZE];
698 u32 val;
699 int ret;
700
701 ret = hisi_qm_get_dfx_access(qm);
702 if (ret)
703 return ret;
704
705 spin_lock_irq(&file->lock);
706 switch (file->index) {
707 case HZIP_CLEAR_ENABLE:
708 val = clear_enable_read(qm);
709 break;
710 default:
711 goto err_input;
712 }
713 spin_unlock_irq(&file->lock);
714
715 hisi_qm_put_dfx_access(qm);
716 ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val);
717 return simple_read_from_buffer(buf, count, pos, tbuf, ret);
718
719 err_input:
720 spin_unlock_irq(&file->lock);
721 hisi_qm_put_dfx_access(qm);
722 return -EINVAL;
723 }
724
hisi_zip_ctrl_debug_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)725 static ssize_t hisi_zip_ctrl_debug_write(struct file *filp,
726 const char __user *buf,
727 size_t count, loff_t *pos)
728 {
729 struct ctrl_debug_file *file = filp->private_data;
730 struct hisi_qm *qm = file_to_qm(file);
731 char tbuf[HZIP_BUF_SIZE];
732 unsigned long val;
733 int len, ret;
734
735 if (*pos != 0)
736 return 0;
737
738 if (count >= HZIP_BUF_SIZE)
739 return -ENOSPC;
740
741 len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count);
742 if (len < 0)
743 return len;
744
745 tbuf[len] = '\0';
746 ret = kstrtoul(tbuf, 0, &val);
747 if (ret)
748 return ret;
749
750 ret = hisi_qm_get_dfx_access(qm);
751 if (ret)
752 return ret;
753
754 spin_lock_irq(&file->lock);
755 switch (file->index) {
756 case HZIP_CLEAR_ENABLE:
757 ret = clear_enable_write(qm, val);
758 if (ret)
759 goto err_input;
760 break;
761 default:
762 ret = -EINVAL;
763 goto err_input;
764 }
765
766 ret = count;
767
768 err_input:
769 spin_unlock_irq(&file->lock);
770 hisi_qm_put_dfx_access(qm);
771 return ret;
772 }
773
774 static const struct file_operations ctrl_debug_fops = {
775 .owner = THIS_MODULE,
776 .open = simple_open,
777 .read = hisi_zip_ctrl_debug_read,
778 .write = hisi_zip_ctrl_debug_write,
779 };
780
zip_debugfs_atomic64_set(void * data,u64 val)781 static int zip_debugfs_atomic64_set(void *data, u64 val)
782 {
783 if (val)
784 return -EINVAL;
785
786 atomic64_set((atomic64_t *)data, 0);
787
788 return 0;
789 }
790
zip_debugfs_atomic64_get(void * data,u64 * val)791 static int zip_debugfs_atomic64_get(void *data, u64 *val)
792 {
793 *val = atomic64_read((atomic64_t *)data);
794
795 return 0;
796 }
797
798 DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get,
799 zip_debugfs_atomic64_set, "%llu\n");
800
hisi_zip_regs_show(struct seq_file * s,void * unused)801 static int hisi_zip_regs_show(struct seq_file *s, void *unused)
802 {
803 hisi_qm_regs_dump(s, s->private);
804
805 return 0;
806 }
807
808 DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs);
809
hisi_zip_core_debug_init(struct hisi_qm * qm)810 static int hisi_zip_core_debug_init(struct hisi_qm *qm)
811 {
812 u32 zip_core_num, zip_comp_core_num;
813 struct device *dev = &qm->pdev->dev;
814 struct debugfs_regset32 *regset;
815 struct dentry *tmp_d;
816 char buf[HZIP_BUF_SIZE];
817 int i;
818
819 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;
820 zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val;
821
822 for (i = 0; i < zip_core_num; i++) {
823 if (i < zip_comp_core_num)
824 scnprintf(buf, sizeof(buf), "comp_core%d", i);
825 else
826 scnprintf(buf, sizeof(buf), "decomp_core%d",
827 i - zip_comp_core_num);
828
829 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
830 if (!regset)
831 return -ENOENT;
832
833 regset->regs = hzip_dfx_regs;
834 regset->nregs = ARRAY_SIZE(hzip_dfx_regs);
835 regset->base = qm->io_base + core_offsets[i];
836 regset->dev = dev;
837
838 tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);
839 debugfs_create_file("regs", 0444, tmp_d, regset,
840 &hisi_zip_regs_fops);
841 }
842
843 return 0;
844 }
845
hisi_zip_dfx_debug_init(struct hisi_qm * qm)846 static void hisi_zip_dfx_debug_init(struct hisi_qm *qm)
847 {
848 struct dfx_diff_registers *hzip_regs = qm->debug.acc_diff_regs;
849 struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
850 struct hisi_zip_dfx *dfx = &zip->dfx;
851 struct dentry *tmp_dir;
852 void *data;
853 int i;
854
855 tmp_dir = debugfs_create_dir("zip_dfx", qm->debug.debug_root);
856 for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) {
857 data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset);
858 debugfs_create_file(zip_dfx_files[i].name,
859 0644, tmp_dir, data,
860 &zip_atomic64_ops);
861 }
862
863 if (qm->fun_type == QM_HW_PF && hzip_regs)
864 debugfs_create_file("diff_regs", 0444, tmp_dir,
865 qm, &hzip_diff_regs_fops);
866 }
867
hisi_zip_ctrl_debug_init(struct hisi_qm * qm)868 static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm)
869 {
870 struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
871 int i;
872
873 for (i = HZIP_CLEAR_ENABLE; i < HZIP_DEBUG_FILE_NUM; i++) {
874 spin_lock_init(&zip->ctrl->files[i].lock);
875 zip->ctrl->files[i].ctrl = zip->ctrl;
876 zip->ctrl->files[i].index = i;
877
878 debugfs_create_file(ctrl_debug_file_name[i], 0600,
879 qm->debug.debug_root,
880 zip->ctrl->files + i,
881 &ctrl_debug_fops);
882 }
883
884 return hisi_zip_core_debug_init(qm);
885 }
886
hisi_zip_debugfs_init(struct hisi_qm * qm)887 static int hisi_zip_debugfs_init(struct hisi_qm *qm)
888 {
889 struct device *dev = &qm->pdev->dev;
890 struct dentry *dev_d;
891 int ret;
892
893 dev_d = debugfs_create_dir(dev_name(dev), hzip_debugfs_root);
894
895 qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET;
896 qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN;
897 qm->debug.debug_root = dev_d;
898 ret = hisi_qm_regs_debugfs_init(qm, hzip_diff_regs, ARRAY_SIZE(hzip_diff_regs));
899 if (ret) {
900 dev_warn(dev, "Failed to init ZIP diff regs!\n");
901 goto debugfs_remove;
902 }
903
904 hisi_qm_debug_init(qm);
905
906 if (qm->fun_type == QM_HW_PF) {
907 ret = hisi_zip_ctrl_debug_init(qm);
908 if (ret)
909 goto failed_to_create;
910 }
911
912 hisi_zip_dfx_debug_init(qm);
913
914 return 0;
915
916 failed_to_create:
917 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));
918 debugfs_remove:
919 debugfs_remove_recursive(hzip_debugfs_root);
920 return ret;
921 }
922
923 /* hisi_zip_debug_regs_clear() - clear the zip debug regs */
hisi_zip_debug_regs_clear(struct hisi_qm * qm)924 static void hisi_zip_debug_regs_clear(struct hisi_qm *qm)
925 {
926 int i, j;
927
928 /* enable register read_clear bit */
929 writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
930 for (i = 0; i < ARRAY_SIZE(core_offsets); i++)
931 for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++)
932 readl(qm->io_base + core_offsets[i] +
933 hzip_dfx_regs[j].offset);
934
935 /* disable register read_clear bit */
936 writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
937
938 hisi_qm_debug_regs_clear(qm);
939 }
940
hisi_zip_debugfs_exit(struct hisi_qm * qm)941 static void hisi_zip_debugfs_exit(struct hisi_qm *qm)
942 {
943 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));
944
945 debugfs_remove_recursive(qm->debug.debug_root);
946
947 if (qm->fun_type == QM_HW_PF) {
948 hisi_zip_debug_regs_clear(qm);
949 qm->debug.curr_qm_qp_num = 0;
950 }
951 }
952
hisi_zip_show_last_regs_init(struct hisi_qm * qm)953 static int hisi_zip_show_last_regs_init(struct hisi_qm *qm)
954 {
955 int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs);
956 int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);
957 struct qm_debug *debug = &qm->debug;
958 void __iomem *io_base;
959 u32 zip_core_num;
960 int i, j, idx;
961
962 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;
963
964 debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num,
965 sizeof(unsigned int), GFP_KERNEL);
966 if (!debug->last_words)
967 return -ENOMEM;
968
969 for (i = 0; i < com_dfx_regs_num; i++) {
970 io_base = qm->io_base + hzip_com_dfx_regs[i].offset;
971 debug->last_words[i] = readl_relaxed(io_base);
972 }
973
974 for (i = 0; i < zip_core_num; i++) {
975 io_base = qm->io_base + core_offsets[i];
976 for (j = 0; j < core_dfx_regs_num; j++) {
977 idx = com_dfx_regs_num + i * core_dfx_regs_num + j;
978 debug->last_words[idx] = readl_relaxed(
979 io_base + hzip_dump_dfx_regs[j].offset);
980 }
981 }
982
983 return 0;
984 }
985
hisi_zip_show_last_regs_uninit(struct hisi_qm * qm)986 static void hisi_zip_show_last_regs_uninit(struct hisi_qm *qm)
987 {
988 struct qm_debug *debug = &qm->debug;
989
990 if (qm->fun_type == QM_HW_VF || !debug->last_words)
991 return;
992
993 kfree(debug->last_words);
994 debug->last_words = NULL;
995 }
996
hisi_zip_show_last_dfx_regs(struct hisi_qm * qm)997 static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm)
998 {
999 int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs);
1000 int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);
1001 u32 zip_core_num, zip_comp_core_num;
1002 struct qm_debug *debug = &qm->debug;
1003 char buf[HZIP_BUF_SIZE];
1004 void __iomem *base;
1005 int i, j, idx;
1006 u32 val;
1007
1008 if (qm->fun_type == QM_HW_VF || !debug->last_words)
1009 return;
1010
1011 for (i = 0; i < com_dfx_regs_num; i++) {
1012 val = readl_relaxed(qm->io_base + hzip_com_dfx_regs[i].offset);
1013 if (debug->last_words[i] != val)
1014 pci_info(qm->pdev, "com_dfx: %s \t= 0x%08x => 0x%08x\n",
1015 hzip_com_dfx_regs[i].name, debug->last_words[i], val);
1016 }
1017
1018 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;
1019 zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val;
1020
1021 for (i = 0; i < zip_core_num; i++) {
1022 if (i < zip_comp_core_num)
1023 scnprintf(buf, sizeof(buf), "Comp_core-%d", i);
1024 else
1025 scnprintf(buf, sizeof(buf), "Decomp_core-%d",
1026 i - zip_comp_core_num);
1027 base = qm->io_base + core_offsets[i];
1028
1029 pci_info(qm->pdev, "==>%s:\n", buf);
1030 /* dump last word for dfx regs during control resetting */
1031 for (j = 0; j < core_dfx_regs_num; j++) {
1032 idx = com_dfx_regs_num + i * core_dfx_regs_num + j;
1033 val = readl_relaxed(base + hzip_dump_dfx_regs[j].offset);
1034 if (debug->last_words[idx] != val)
1035 pci_info(qm->pdev, "%s \t= 0x%08x => 0x%08x\n",
1036 hzip_dump_dfx_regs[j].name,
1037 debug->last_words[idx], val);
1038 }
1039 }
1040 }
1041
hisi_zip_log_hw_error(struct hisi_qm * qm,u32 err_sts)1042 static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts)
1043 {
1044 const struct hisi_zip_hw_error *err = zip_hw_error;
1045 struct device *dev = &qm->pdev->dev;
1046 u32 err_val;
1047
1048 while (err->msg) {
1049 if (err->int_msk & err_sts) {
1050 dev_err(dev, "%s [error status=0x%x] found\n",
1051 err->msg, err->int_msk);
1052
1053 if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) {
1054 err_val = readl(qm->io_base +
1055 HZIP_CORE_SRAM_ECC_ERR_INFO);
1056 dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n",
1057 ((err_val >>
1058 HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF));
1059 }
1060 }
1061 err++;
1062 }
1063 }
1064
hisi_zip_get_hw_err_status(struct hisi_qm * qm)1065 static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm)
1066 {
1067 return readl(qm->io_base + HZIP_CORE_INT_STATUS);
1068 }
1069
hisi_zip_clear_hw_err_status(struct hisi_qm * qm,u32 err_sts)1070 static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
1071 {
1072 u32 nfe;
1073
1074 writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE);
1075 nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);
1076 writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1077 }
1078
hisi_zip_open_axi_master_ooo(struct hisi_qm * qm)1079 static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm)
1080 {
1081 u32 val;
1082
1083 val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1084
1085 writel(val & ~HZIP_AXI_SHUTDOWN_ENABLE,
1086 qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1087
1088 writel(val | HZIP_AXI_SHUTDOWN_ENABLE,
1089 qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1090 }
1091
hisi_zip_close_axi_master_ooo(struct hisi_qm * qm)1092 static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm)
1093 {
1094 u32 nfe_enb;
1095
1096 /* Disable ECC Mbit error report. */
1097 nfe_enb = readl(qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1098 writel(nfe_enb & ~HZIP_CORE_INT_STATUS_M_ECC,
1099 qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1100
1101 /* Inject zip ECC Mbit error to block master ooo. */
1102 writel(HZIP_CORE_INT_STATUS_M_ECC,
1103 qm->io_base + HZIP_CORE_INT_SET);
1104 }
1105
hisi_zip_err_info_init(struct hisi_qm * qm)1106 static void hisi_zip_err_info_init(struct hisi_qm *qm)
1107 {
1108 struct hisi_qm_err_info *err_info = &qm->err_info;
1109
1110 err_info->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK;
1111 err_info->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver);
1112 err_info->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1113 ZIP_QM_NFE_MASK_CAP, qm->cap_ver);
1114 err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC;
1115 err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1116 ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1117 err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1118 ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1119 err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1120 ZIP_QM_RESET_MASK_CAP, qm->cap_ver);
1121 err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1122 ZIP_RESET_MASK_CAP, qm->cap_ver);
1123 err_info->msi_wr_port = HZIP_WR_PORT;
1124 err_info->acpi_rst = "ZRST";
1125 }
1126
1127 static const struct hisi_qm_err_ini hisi_zip_err_ini = {
1128 .hw_init = hisi_zip_set_user_domain_and_cache,
1129 .hw_err_enable = hisi_zip_hw_error_enable,
1130 .hw_err_disable = hisi_zip_hw_error_disable,
1131 .get_dev_hw_err_status = hisi_zip_get_hw_err_status,
1132 .clear_dev_hw_err_status = hisi_zip_clear_hw_err_status,
1133 .log_dev_hw_err = hisi_zip_log_hw_error,
1134 .open_axi_master_ooo = hisi_zip_open_axi_master_ooo,
1135 .close_axi_master_ooo = hisi_zip_close_axi_master_ooo,
1136 .open_sva_prefetch = hisi_zip_open_sva_prefetch,
1137 .close_sva_prefetch = hisi_zip_close_sva_prefetch,
1138 .show_last_dfx_regs = hisi_zip_show_last_dfx_regs,
1139 .err_info_init = hisi_zip_err_info_init,
1140 };
1141
hisi_zip_pf_probe_init(struct hisi_zip * hisi_zip)1142 static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
1143 {
1144 struct hisi_qm *qm = &hisi_zip->qm;
1145 struct hisi_zip_ctrl *ctrl;
1146 int ret;
1147
1148 ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL);
1149 if (!ctrl)
1150 return -ENOMEM;
1151
1152 hisi_zip->ctrl = ctrl;
1153 ctrl->hisi_zip = hisi_zip;
1154 qm->err_ini = &hisi_zip_err_ini;
1155 qm->err_ini->err_info_init(qm);
1156
1157 ret = hisi_zip_set_user_domain_and_cache(qm);
1158 if (ret)
1159 return ret;
1160
1161 ret = hisi_zip_set_high_perf(qm);
1162 if (ret)
1163 return ret;
1164
1165 hisi_zip_open_sva_prefetch(qm);
1166 hisi_qm_dev_err_init(qm);
1167 hisi_zip_debug_regs_clear(qm);
1168
1169 ret = hisi_zip_show_last_regs_init(qm);
1170 if (ret)
1171 pci_err(qm->pdev, "Failed to init last word regs!\n");
1172
1173 return ret;
1174 }
1175
zip_pre_store_cap_reg(struct hisi_qm * qm)1176 static int zip_pre_store_cap_reg(struct hisi_qm *qm)
1177 {
1178 struct hisi_qm_cap_record *zip_cap;
1179 struct pci_dev *pdev = qm->pdev;
1180 size_t i, size;
1181
1182 size = ARRAY_SIZE(zip_pre_store_caps);
1183 zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL);
1184 if (!zip_cap)
1185 return -ENOMEM;
1186
1187 for (i = 0; i < size; i++) {
1188 zip_cap[i].type = zip_pre_store_caps[i];
1189 zip_cap[i].cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1190 zip_pre_store_caps[i], qm->cap_ver);
1191 }
1192
1193 qm->cap_tables.dev_cap_table = zip_cap;
1194
1195 return 0;
1196 }
1197
hisi_zip_qm_init(struct hisi_qm * qm,struct pci_dev * pdev)1198 static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
1199 {
1200 u64 alg_msk;
1201 int ret;
1202
1203 qm->pdev = pdev;
1204 qm->ver = pdev->revision;
1205 qm->mode = uacce_mode;
1206 qm->sqe_size = HZIP_SQE_SIZE;
1207 qm->dev_name = hisi_zip_name;
1208
1209 qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_ZIP_PF) ?
1210 QM_HW_PF : QM_HW_VF;
1211 if (qm->fun_type == QM_HW_PF) {
1212 qm->qp_base = HZIP_PF_DEF_Q_BASE;
1213 qm->qp_num = pf_q_num;
1214 qm->debug.curr_qm_qp_num = pf_q_num;
1215 qm->qm_list = &zip_devices;
1216 if (pf_q_num_flag)
1217 set_bit(QM_MODULE_PARAM, &qm->misc_ctl);
1218 } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
1219 /*
1220 * have no way to get qm configure in VM in v1 hardware,
1221 * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force
1222 * to trigger only one VF in v1 hardware.
1223 *
1224 * v2 hardware has no such problem.
1225 */
1226 qm->qp_base = HZIP_PF_DEF_Q_NUM;
1227 qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM;
1228 }
1229
1230 ret = hisi_qm_init(qm);
1231 if (ret) {
1232 pci_err(qm->pdev, "Failed to init zip qm configures!\n");
1233 return ret;
1234 }
1235
1236 /* Fetch and save the value of capability registers */
1237 ret = zip_pre_store_cap_reg(qm);
1238 if (ret) {
1239 pci_err(qm->pdev, "Failed to pre-store capability registers!\n");
1240 hisi_qm_uninit(qm);
1241 return ret;
1242 }
1243
1244 alg_msk = qm->cap_tables.dev_cap_table[ZIP_DEV_ALG_BITMAP_IDX].cap_val;
1245 ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs));
1246 if (ret) {
1247 pci_err(qm->pdev, "Failed to set zip algs!\n");
1248 hisi_qm_uninit(qm);
1249 }
1250
1251 return ret;
1252 }
1253
hisi_zip_qm_uninit(struct hisi_qm * qm)1254 static void hisi_zip_qm_uninit(struct hisi_qm *qm)
1255 {
1256 hisi_qm_uninit(qm);
1257 }
1258
hisi_zip_probe_init(struct hisi_zip * hisi_zip)1259 static int hisi_zip_probe_init(struct hisi_zip *hisi_zip)
1260 {
1261 u32 type_rate = HZIP_SHAPER_RATE_COMPRESS;
1262 struct hisi_qm *qm = &hisi_zip->qm;
1263 int ret;
1264
1265 if (qm->fun_type == QM_HW_PF) {
1266 ret = hisi_zip_pf_probe_init(hisi_zip);
1267 if (ret)
1268 return ret;
1269 /* enable shaper type 0 */
1270 if (qm->ver >= QM_HW_V3) {
1271 type_rate |= QM_SHAPER_ENABLE;
1272
1273 /* ZIP need to enable shaper type 1 */
1274 type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET;
1275 qm->type_rate = type_rate;
1276 }
1277 }
1278
1279 return 0;
1280 }
1281
hisi_zip_probe(struct pci_dev * pdev,const struct pci_device_id * id)1282 static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1283 {
1284 struct hisi_zip *hisi_zip;
1285 struct hisi_qm *qm;
1286 int ret;
1287
1288 hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);
1289 if (!hisi_zip)
1290 return -ENOMEM;
1291
1292 qm = &hisi_zip->qm;
1293
1294 ret = hisi_zip_qm_init(qm, pdev);
1295 if (ret) {
1296 pci_err(pdev, "Failed to init ZIP QM (%d)!\n", ret);
1297 return ret;
1298 }
1299
1300 ret = hisi_zip_probe_init(hisi_zip);
1301 if (ret) {
1302 pci_err(pdev, "Failed to probe (%d)!\n", ret);
1303 goto err_qm_uninit;
1304 }
1305
1306 ret = hisi_qm_start(qm);
1307 if (ret)
1308 goto err_dev_err_uninit;
1309
1310 ret = hisi_zip_debugfs_init(qm);
1311 if (ret)
1312 pci_err(pdev, "failed to init debugfs (%d)!\n", ret);
1313
1314 hisi_qm_add_list(qm, &zip_devices);
1315 ret = hisi_qm_alg_register(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);
1316 if (ret < 0) {
1317 pci_err(pdev, "failed to register driver to crypto!\n");
1318 goto err_qm_del_list;
1319 }
1320
1321 if (qm->uacce) {
1322 ret = uacce_register(qm->uacce);
1323 if (ret) {
1324 pci_err(pdev, "failed to register uacce (%d)!\n", ret);
1325 goto err_qm_alg_unregister;
1326 }
1327 }
1328
1329 if (qm->fun_type == QM_HW_PF && vfs_num > 0) {
1330 ret = hisi_qm_sriov_enable(pdev, vfs_num);
1331 if (ret < 0)
1332 goto err_qm_alg_unregister;
1333 }
1334
1335 hisi_qm_pm_init(qm);
1336
1337 return 0;
1338
1339 err_qm_alg_unregister:
1340 hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);
1341
1342 err_qm_del_list:
1343 hisi_qm_del_list(qm, &zip_devices);
1344 hisi_zip_debugfs_exit(qm);
1345 hisi_qm_stop(qm, QM_NORMAL);
1346
1347 err_dev_err_uninit:
1348 hisi_zip_show_last_regs_uninit(qm);
1349 hisi_qm_dev_err_uninit(qm);
1350
1351 err_qm_uninit:
1352 hisi_zip_qm_uninit(qm);
1353
1354 return ret;
1355 }
1356
hisi_zip_remove(struct pci_dev * pdev)1357 static void hisi_zip_remove(struct pci_dev *pdev)
1358 {
1359 struct hisi_qm *qm = pci_get_drvdata(pdev);
1360
1361 hisi_qm_pm_uninit(qm);
1362 hisi_qm_wait_task_finish(qm, &zip_devices);
1363 hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);
1364 hisi_qm_del_list(qm, &zip_devices);
1365
1366 if (qm->fun_type == QM_HW_PF && qm->vfs_num)
1367 hisi_qm_sriov_disable(pdev, true);
1368
1369 hisi_zip_debugfs_exit(qm);
1370 hisi_qm_stop(qm, QM_NORMAL);
1371 hisi_zip_show_last_regs_uninit(qm);
1372 hisi_qm_dev_err_uninit(qm);
1373 hisi_zip_qm_uninit(qm);
1374 }
1375
1376 static const struct dev_pm_ops hisi_zip_pm_ops = {
1377 SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL)
1378 };
1379
1380 static const struct pci_error_handlers hisi_zip_err_handler = {
1381 .error_detected = hisi_qm_dev_err_detected,
1382 .slot_reset = hisi_qm_dev_slot_reset,
1383 .reset_prepare = hisi_qm_reset_prepare,
1384 .reset_done = hisi_qm_reset_done,
1385 };
1386
1387 static struct pci_driver hisi_zip_pci_driver = {
1388 .name = "hisi_zip",
1389 .id_table = hisi_zip_dev_ids,
1390 .probe = hisi_zip_probe,
1391 .remove = hisi_zip_remove,
1392 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?
1393 hisi_qm_sriov_configure : NULL,
1394 .err_handler = &hisi_zip_err_handler,
1395 .shutdown = hisi_qm_dev_shutdown,
1396 .driver.pm = &hisi_zip_pm_ops,
1397 };
1398
hisi_zip_get_pf_driver(void)1399 struct pci_driver *hisi_zip_get_pf_driver(void)
1400 {
1401 return &hisi_zip_pci_driver;
1402 }
1403 EXPORT_SYMBOL_GPL(hisi_zip_get_pf_driver);
1404
hisi_zip_register_debugfs(void)1405 static void hisi_zip_register_debugfs(void)
1406 {
1407 if (!debugfs_initialized())
1408 return;
1409
1410 hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL);
1411 }
1412
hisi_zip_unregister_debugfs(void)1413 static void hisi_zip_unregister_debugfs(void)
1414 {
1415 debugfs_remove_recursive(hzip_debugfs_root);
1416 }
1417
hisi_zip_init(void)1418 static int __init hisi_zip_init(void)
1419 {
1420 int ret;
1421
1422 hisi_qm_init_list(&zip_devices);
1423 hisi_zip_register_debugfs();
1424
1425 ret = pci_register_driver(&hisi_zip_pci_driver);
1426 if (ret < 0) {
1427 hisi_zip_unregister_debugfs();
1428 pr_err("Failed to register pci driver.\n");
1429 }
1430
1431 return ret;
1432 }
1433
hisi_zip_exit(void)1434 static void __exit hisi_zip_exit(void)
1435 {
1436 pci_unregister_driver(&hisi_zip_pci_driver);
1437 hisi_zip_unregister_debugfs();
1438 }
1439
1440 module_init(hisi_zip_init);
1441 module_exit(hisi_zip_exit);
1442
1443 MODULE_LICENSE("GPL v2");
1444 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
1445 MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator");
1446