1 /* SPDX-License-Identifier: GPL-2.0-only
2 * Copyright (C) 2020 Marvell.
3 */
4 #ifndef __OTX2_CPTLF_H
5 #define __OTX2_CPTLF_H
6
7 #include <linux/soc/marvell/octeontx2/asm.h>
8 #include <linux/bitfield.h>
9 #include <mbox.h>
10 #include <rvu.h>
11 #include "otx2_cpt_common.h"
12 #include "otx2_cpt_reqmgr.h"
13
14 /*
15 * CPT instruction and pending queues user requested length in CPT_INST_S msgs
16 */
17 #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200
18
19 /*
20 * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S
21 * messages.
22 */
23 #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40)
24
25 /*
26 * CPT instruction and pending queues length in CPT_INST_S messages
27 */
28 #define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40)
29
30 /*
31 * LDWB is getting incorrectly used when IQB_LDWB = 1 and CPT instruction
32 * queue has less than 320 free entries. So, increase HW instruction queue
33 * size by 320 and give 320 entries less for SW/NIX RX as a workaround.
34 */
35 #define OTX2_CPT_INST_QLEN_EXTRA_BYTES (320 * OTX2_CPT_INST_SIZE)
36 #define OTX2_CPT_EXTRA_SIZE_DIV40 (320/40)
37
38 /* CPT instruction queue length in bytes */
39 #define OTX2_CPT_INST_QLEN_BYTES \
40 ((OTX2_CPT_SIZE_DIV40 * 40 * OTX2_CPT_INST_SIZE) + \
41 OTX2_CPT_INST_QLEN_EXTRA_BYTES)
42
43 /* CPT instruction group queue length in bytes */
44 #define OTX2_CPT_INST_GRP_QLEN_BYTES \
45 ((OTX2_CPT_SIZE_DIV40 + OTX2_CPT_EXTRA_SIZE_DIV40) * 16)
46
47 /* CPT FC length in bytes */
48 #define OTX2_CPT_Q_FC_LEN 128
49
50 /* CPT instruction queue alignment */
51 #define OTX2_CPT_INST_Q_ALIGNMENT 128
52
53 /* Mask which selects all engine groups */
54 #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF
55
56 /* Maximum LFs supported in OcteonTX2 for CPT */
57 #define OTX2_CPT_MAX_LFS_NUM 64
58
59 /* Queue priority */
60 #define OTX2_CPT_QUEUE_HI_PRIO 0x1
61 #define OTX2_CPT_QUEUE_LOW_PRIO 0x0
62
63 enum otx2_cptlf_state {
64 OTX2_CPTLF_IN_RESET,
65 OTX2_CPTLF_STARTED,
66 };
67
68 struct otx2_cpt_inst_queue {
69 u8 *vaddr;
70 u8 *real_vaddr;
71 dma_addr_t dma_addr;
72 dma_addr_t real_dma_addr;
73 u32 size;
74 };
75
76 struct otx2_cptlfs_info;
77 struct otx2_cptlf_wqe {
78 struct tasklet_struct work;
79 struct otx2_cptlfs_info *lfs;
80 u8 lf_num;
81 };
82
83 struct otx2_cptlf_info {
84 struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */
85 void __iomem *lmtline; /* Address of LMTLINE */
86 void __iomem *ioreg; /* LMTLINE send register */
87 int msix_offset; /* MSI-X interrupts offset */
88 cpumask_var_t affinity_mask; /* IRQs affinity mask */
89 u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */
90 u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */
91 u8 slot; /* Slot number of this LF */
92
93 struct otx2_cpt_inst_queue iqueue;/* Instruction queue */
94 struct otx2_cpt_pending_queue pqueue; /* Pending queue */
95 struct otx2_cptlf_wqe *wqe; /* Tasklet work info */
96 };
97
98 struct cpt_hw_ops {
99 void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num,
100 struct otx2_cptlf_info *lf);
101 u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result);
102 u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result);
103 struct otx2_cpt_inst_info *
104 (*cpt_sg_info_create)(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
105 gfp_t gfp);
106 };
107
108 #define LMTLINE_SIZE 128
109 #define LMTLINE_ALIGN 128
110 struct otx2_lmt_info {
111 void *base;
112 dma_addr_t iova;
113 u32 size;
114 u8 align;
115 };
116
117 struct otx2_cptlfs_info {
118 /* Registers start address of VF/PF LFs are attached to */
119 void __iomem *reg_base;
120 struct otx2_lmt_info lmt_info;
121 struct pci_dev *pdev; /* Device LFs are attached to */
122 struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM];
123 struct otx2_mbox *mbox;
124 struct cpt_hw_ops *ops;
125 u8 are_lfs_attached; /* Whether CPT LFs are attached */
126 u8 lfs_num; /* Number of CPT LFs */
127 u8 kcrypto_se_eng_grp_num; /* Crypto symmetric engine group number */
128 u8 kcrypto_ae_eng_grp_num; /* Crypto asymmetric engine group number */
129 u8 kvf_limits; /* Kernel crypto limits */
130 atomic_t state; /* LF's state. started/reset */
131 int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
132 int global_slot; /* Global slot across the blocks */
133 u8 ctx_ilen;
134 u8 ctx_ilen_ovrd;
135 };
136
otx2_cpt_free_instruction_queues(struct otx2_cptlfs_info * lfs)137 static inline void otx2_cpt_free_instruction_queues(
138 struct otx2_cptlfs_info *lfs)
139 {
140 struct otx2_cpt_inst_queue *iq;
141 int i;
142
143 for (i = 0; i < lfs->lfs_num; i++) {
144 iq = &lfs->lf[i].iqueue;
145 if (iq->real_vaddr)
146 dma_free_coherent(&lfs->pdev->dev,
147 iq->size,
148 iq->real_vaddr,
149 iq->real_dma_addr);
150 iq->real_vaddr = NULL;
151 iq->vaddr = NULL;
152 }
153 }
154
otx2_cpt_alloc_instruction_queues(struct otx2_cptlfs_info * lfs)155 static inline int otx2_cpt_alloc_instruction_queues(
156 struct otx2_cptlfs_info *lfs)
157 {
158 struct otx2_cpt_inst_queue *iq;
159 int ret = 0, i;
160
161 if (!lfs->lfs_num)
162 return -EINVAL;
163
164 for (i = 0; i < lfs->lfs_num; i++) {
165 iq = &lfs->lf[i].iqueue;
166 iq->size = OTX2_CPT_INST_QLEN_BYTES +
167 OTX2_CPT_Q_FC_LEN +
168 OTX2_CPT_INST_GRP_QLEN_BYTES +
169 OTX2_CPT_INST_Q_ALIGNMENT;
170 iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size,
171 &iq->real_dma_addr, GFP_KERNEL);
172 if (!iq->real_vaddr) {
173 ret = -ENOMEM;
174 goto error;
175 }
176 iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES;
177 iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES;
178
179 /* Align pointers */
180 iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT);
181 iq->dma_addr = PTR_ALIGN(iq->dma_addr,
182 OTX2_CPT_INST_Q_ALIGNMENT);
183 }
184 return 0;
185
186 error:
187 otx2_cpt_free_instruction_queues(lfs);
188 return ret;
189 }
190
otx2_cptlf_set_iqueues_base_addr(struct otx2_cptlfs_info * lfs)191 static inline void otx2_cptlf_set_iqueues_base_addr(
192 struct otx2_cptlfs_info *lfs)
193 {
194 union otx2_cptx_lf_q_base lf_q_base;
195 int slot;
196
197 for (slot = 0; slot < lfs->lfs_num; slot++) {
198 lf_q_base.u = lfs->lf[slot].iqueue.dma_addr;
199 otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot,
200 OTX2_CPT_LF_Q_BASE, lf_q_base.u);
201 }
202 }
203
otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info * lf)204 static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf)
205 {
206 union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 };
207
208 lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40 +
209 OTX2_CPT_EXTRA_SIZE_DIV40;
210 otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot,
211 OTX2_CPT_LF_Q_SIZE, lf_q_size.u);
212 }
213
otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info * lfs)214 static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs)
215 {
216 int slot;
217
218 for (slot = 0; slot < lfs->lfs_num; slot++)
219 otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]);
220 }
221
222 #define INFLIGHT GENMASK_ULL(8, 0)
223 #define GRB_CNT GENMASK_ULL(39, 32)
224 #define GWB_CNT GENMASK_ULL(47, 40)
225 #define XQ_XOR GENMASK_ULL(63, 63)
226 #define DQPTR GENMASK_ULL(19, 0)
227 #define NQPTR GENMASK_ULL(51, 32)
228
otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info * lf)229 static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf)
230 {
231 void __iomem *reg_base = lf->lfs->reg_base;
232 struct pci_dev *pdev = lf->lfs->pdev;
233 u8 blkaddr = lf->lfs->blkaddr;
234 int timeout = 1000000;
235 u64 inprog, inst_ptr;
236 u64 slot = lf->slot;
237 u64 qsize, pending;
238 int i = 0;
239
240 /* Disable instructions enqueuing */
241 otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_CTL, 0x0);
242
243 inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG);
244 inprog |= BIT_ULL(16);
245 otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG, inprog);
246
247 qsize = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_SIZE) & 0x7FFF;
248 do {
249 inst_ptr = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_INST_PTR);
250 pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) +
251 FIELD_GET(NQPTR, inst_ptr) - FIELD_GET(DQPTR, inst_ptr);
252 udelay(1);
253 timeout--;
254 } while ((pending != 0) && (timeout != 0));
255
256 if (timeout == 0)
257 dev_warn(&pdev->dev, "TIMEOUT: CPT poll on pending instructions\n");
258
259 timeout = 1000000;
260 /* Wait for CPT queue to become execution-quiescent */
261 do {
262 inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG);
263
264 if ((FIELD_GET(INFLIGHT, inprog) == 0) &&
265 (FIELD_GET(GRB_CNT, inprog) == 0)) {
266 i++;
267 } else {
268 i = 0;
269 timeout--;
270 }
271 } while ((timeout != 0) && (i < 10));
272
273 if (timeout == 0)
274 dev_warn(&pdev->dev, "TIMEOUT: CPT poll on inflight count\n");
275 /* Wait for 2 us to flush all queue writes to memory */
276 udelay(2);
277 }
278
otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info * lfs)279 static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs)
280 {
281 int slot;
282
283 for (slot = 0; slot < lfs->lfs_num; slot++) {
284 otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]);
285 otx2_cpt_lf_reset_msg(lfs, lfs->global_slot + slot);
286 }
287 }
288
otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info * lf,bool enable)289 static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf,
290 bool enable)
291 {
292 u8 blkaddr = lf->lfs->blkaddr;
293 union otx2_cptx_lf_ctl lf_ctl;
294
295 lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
296 OTX2_CPT_LF_CTL);
297
298 /* Set iqueue's enqueuing */
299 lf_ctl.s.ena = enable ? 0x1 : 0x0;
300 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
301 OTX2_CPT_LF_CTL, lf_ctl.u);
302 }
303
otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info * lf)304 static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf)
305 {
306 otx2_cptlf_set_iqueue_enq(lf, true);
307 }
308
otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info * lf,bool enable)309 static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf,
310 bool enable)
311 {
312 union otx2_cptx_lf_inprog lf_inprog;
313 u8 blkaddr = lf->lfs->blkaddr;
314
315 lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
316 OTX2_CPT_LF_INPROG);
317
318 /* Set iqueue's execution */
319 lf_inprog.s.eena = enable ? 0x1 : 0x0;
320 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
321 OTX2_CPT_LF_INPROG, lf_inprog.u);
322 }
323
otx2_cptlf_set_ctx_flr_flush(struct otx2_cptlf_info * lf)324 static inline void otx2_cptlf_set_ctx_flr_flush(struct otx2_cptlf_info *lf)
325 {
326 u8 blkaddr = lf->lfs->blkaddr;
327 u64 val;
328
329 val = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
330 OTX2_CPT_LF_CTX_CTL);
331 val |= BIT_ULL(0);
332
333 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
334 OTX2_CPT_LF_CTX_CTL, val);
335 }
336
otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info * lf)337 static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf)
338 {
339 otx2_cptlf_set_iqueue_exec(lf, true);
340 }
341
otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info * lf)342 static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf)
343 {
344 otx2_cptlf_set_iqueue_exec(lf, false);
345 }
346
otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info * lfs)347 static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs)
348 {
349 int slot;
350
351 for (slot = 0; slot < lfs->lfs_num; slot++) {
352 /* Enable flush on FLR for Errata */
353 if (is_dev_cn10kb(lfs->pdev))
354 otx2_cptlf_set_ctx_flr_flush(&lfs->lf[slot]);
355
356 otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]);
357 otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]);
358 }
359 }
360
otx2_cpt_fill_inst(union otx2_cpt_inst_s * cptinst,struct otx2_cpt_iq_command * iq_cmd,u64 comp_baddr)361 static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst,
362 struct otx2_cpt_iq_command *iq_cmd,
363 u64 comp_baddr)
364 {
365 cptinst->u[0] = 0x0;
366 cptinst->s.doneint = true;
367 cptinst->s.res_addr = comp_baddr;
368 cptinst->u[2] = 0x0;
369 cptinst->u[3] = 0x0;
370 cptinst->s.ei0 = iq_cmd->cmd.u;
371 cptinst->s.ei1 = iq_cmd->dptr;
372 cptinst->s.ei2 = iq_cmd->rptr;
373 cptinst->s.ei3 = iq_cmd->cptr.u;
374 }
375
376 /*
377 * On OcteonTX2 platform the parameter insts_num is used as a count of
378 * instructions to be enqueued. The valid values for insts_num are:
379 * 1 - 1 CPT instruction will be enqueued during LMTST operation
380 * 2 - 2 CPT instructions will be enqueued during LMTST operation
381 */
otx2_cpt_send_cmd(union otx2_cpt_inst_s * cptinst,u32 insts_num,struct otx2_cptlf_info * lf)382 static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst,
383 u32 insts_num, struct otx2_cptlf_info *lf)
384 {
385 void __iomem *lmtline = lf->lmtline;
386 long ret;
387
388 /*
389 * Make sure memory areas pointed in CPT_INST_S
390 * are flushed before the instruction is sent to CPT
391 */
392 dma_wmb();
393
394 do {
395 /* Copy CPT command to LMTLINE */
396 memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
397
398 /*
399 * LDEOR initiates atomic transfer to I/O device
400 * The following will cause the LMTST to fail (the LDEOR
401 * returns zero):
402 * - No stores have been performed to the LMTLINE since it was
403 * last invalidated.
404 * - The bytes which have been stored to LMTLINE since it was
405 * last invalidated form a pattern that is non-contiguous, does
406 * not start at byte 0, or does not end on a 8-byte boundary.
407 * (i.e.comprises a formation of other than 1–16 8-byte
408 * words.)
409 *
410 * These rules are designed such that an operating system
411 * context switch or hypervisor guest switch need have no
412 * knowledge of the LMTST operations; the switch code does not
413 * need to store to LMTCANCEL. Also note as LMTLINE data cannot
414 * be read, there is no information leakage between processes.
415 */
416 ret = otx2_lmt_flush(lf->ioreg);
417
418 } while (!ret);
419 }
420
otx2_cptlf_started(struct otx2_cptlfs_info * lfs)421 static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs)
422 {
423 return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED;
424 }
425
otx2_cptlf_set_dev_info(struct otx2_cptlfs_info * lfs,struct pci_dev * pdev,void __iomem * reg_base,struct otx2_mbox * mbox,int blkaddr)426 static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs,
427 struct pci_dev *pdev,
428 void __iomem *reg_base,
429 struct otx2_mbox *mbox,
430 int blkaddr)
431 {
432 lfs->pdev = pdev;
433 lfs->reg_base = reg_base;
434 lfs->mbox = mbox;
435 lfs->blkaddr = blkaddr;
436 }
437
438 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri,
439 int lfs_num);
440 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs);
441 int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs);
442 int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs);
443 void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs);
444 void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs);
445 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs);
446 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs);
447
448 #endif /* __OTX2_CPTLF_H */
449