1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * sun8i-ce-core.c - hardware cryptographic offloader for
4 * Allwinner H3/A64/H5/H2+/H6/R40 SoC
5 *
6 * Copyright (C) 2015-2019 Corentin Labbe <clabbe.montjoie@gmail.com>
7 *
8 * Core file which registers crypto algorithms supported by the CryptoEngine.
9 *
10 * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst
11 */
12
13 #include <crypto/engine.h>
14 #include <crypto/internal/hash.h>
15 #include <crypto/internal/rng.h>
16 #include <crypto/internal/skcipher.h>
17 #include <linux/clk.h>
18 #include <linux/delay.h>
19 #include <linux/dma-mapping.h>
20 #include <linux/err.h>
21 #include <linux/interrupt.h>
22 #include <linux/io.h>
23 #include <linux/irq.h>
24 #include <linux/kernel.h>
25 #include <linux/module.h>
26 #include <linux/of.h>
27 #include <linux/platform_device.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/reset.h>
30
31 #include "sun8i-ce.h"
32
33 /*
34 * mod clock is lower on H3 than other SoC due to some DMA timeout occurring
35 * with high value.
36 * If you want to tune mod clock, loading driver and passing selftest is
37 * insufficient, you need to test with some LUKS test (mount and write to it)
38 */
39 static const struct ce_variant ce_h3_variant = {
40 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
41 },
42 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
43 CE_ALG_SHA384, CE_ALG_SHA512
44 },
45 .op_mode = { CE_OP_ECB, CE_OP_CBC
46 },
47 .ce_clks = {
48 { "bus", 0, 200000000 },
49 { "mod", 50000000, 0 },
50 },
51 .esr = ESR_H3,
52 .prng = CE_ALG_PRNG,
53 .trng = CE_ID_NOTSUPP,
54 };
55
56 static const struct ce_variant ce_h5_variant = {
57 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
58 },
59 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
60 CE_ID_NOTSUPP, CE_ID_NOTSUPP
61 },
62 .op_mode = { CE_OP_ECB, CE_OP_CBC
63 },
64 .ce_clks = {
65 { "bus", 0, 200000000 },
66 { "mod", 300000000, 0 },
67 },
68 .esr = ESR_H5,
69 .prng = CE_ALG_PRNG,
70 .trng = CE_ID_NOTSUPP,
71 };
72
73 static const struct ce_variant ce_h6_variant = {
74 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
75 },
76 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
77 CE_ALG_SHA384, CE_ALG_SHA512
78 },
79 .op_mode = { CE_OP_ECB, CE_OP_CBC
80 },
81 .cipher_t_dlen_in_bytes = true,
82 .hash_t_dlen_in_bits = true,
83 .prng_t_dlen_in_bytes = true,
84 .trng_t_dlen_in_bytes = true,
85 .ce_clks = {
86 { "bus", 0, 200000000 },
87 { "mod", 300000000, 0 },
88 { "ram", 0, 400000000 },
89 },
90 .esr = ESR_H6,
91 .prng = CE_ALG_PRNG_V2,
92 .trng = CE_ALG_TRNG_V2,
93 };
94
95 static const struct ce_variant ce_a64_variant = {
96 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
97 },
98 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
99 CE_ID_NOTSUPP, CE_ID_NOTSUPP
100 },
101 .op_mode = { CE_OP_ECB, CE_OP_CBC
102 },
103 .ce_clks = {
104 { "bus", 0, 200000000 },
105 { "mod", 300000000, 0 },
106 },
107 .esr = ESR_A64,
108 .prng = CE_ALG_PRNG,
109 .trng = CE_ID_NOTSUPP,
110 };
111
112 static const struct ce_variant ce_d1_variant = {
113 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
114 },
115 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
116 CE_ALG_SHA384, CE_ALG_SHA512
117 },
118 .op_mode = { CE_OP_ECB, CE_OP_CBC
119 },
120 .ce_clks = {
121 { "bus", 0, 200000000 },
122 { "mod", 300000000, 0 },
123 { "ram", 0, 400000000 },
124 { "trng", 0, 0 },
125 },
126 .esr = ESR_D1,
127 .prng = CE_ALG_PRNG,
128 .trng = CE_ALG_TRNG,
129 };
130
131 static const struct ce_variant ce_r40_variant = {
132 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
133 },
134 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
135 CE_ID_NOTSUPP, CE_ID_NOTSUPP
136 },
137 .op_mode = { CE_OP_ECB, CE_OP_CBC
138 },
139 .ce_clks = {
140 { "bus", 0, 200000000 },
141 { "mod", 300000000, 0 },
142 },
143 .esr = ESR_R40,
144 .prng = CE_ALG_PRNG,
145 .trng = CE_ID_NOTSUPP,
146 };
147
148 /*
149 * sun8i_ce_get_engine_number() get the next channel slot
150 * This is a simple round-robin way of getting the next channel
151 * The flow 3 is reserve for xRNG operations
152 */
sun8i_ce_get_engine_number(struct sun8i_ce_dev * ce)153 int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce)
154 {
155 return atomic_inc_return(&ce->flow) % (MAXFLOW - 1);
156 }
157
sun8i_ce_run_task(struct sun8i_ce_dev * ce,int flow,const char * name)158 int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name)
159 {
160 u32 v;
161 int err = 0;
162 struct ce_task *cet = ce->chanlist[flow].tl;
163
164 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
165 ce->chanlist[flow].stat_req++;
166 #endif
167
168 mutex_lock(&ce->mlock);
169
170 v = readl(ce->base + CE_ICR);
171 v |= 1 << flow;
172 writel(v, ce->base + CE_ICR);
173
174 reinit_completion(&ce->chanlist[flow].complete);
175 writel(ce->chanlist[flow].t_phy, ce->base + CE_TDQ);
176
177 ce->chanlist[flow].status = 0;
178 /* Be sure all data is written before enabling the task */
179 wmb();
180
181 /* Only H6 needs to write a part of t_common_ctl along with "1", but since it is ignored
182 * on older SoCs, we have no reason to complicate things.
183 */
184 v = 1 | ((le32_to_cpu(ce->chanlist[flow].tl->t_common_ctl) & 0x7F) << 8);
185 writel(v, ce->base + CE_TLR);
186 mutex_unlock(&ce->mlock);
187
188 wait_for_completion_interruptible_timeout(&ce->chanlist[flow].complete,
189 msecs_to_jiffies(ce->chanlist[flow].timeout));
190
191 if (ce->chanlist[flow].status == 0) {
192 dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name,
193 ce->chanlist[flow].timeout, flow);
194 err = -EFAULT;
195 }
196 /* No need to lock for this read, the channel is locked so
197 * nothing could modify the error value for this channel
198 */
199 v = readl(ce->base + CE_ESR);
200 switch (ce->variant->esr) {
201 case ESR_H3:
202 /* Sadly, the error bit is not per flow */
203 if (v) {
204 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow);
205 err = -EFAULT;
206 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4,
207 cet, sizeof(struct ce_task), false);
208 }
209 if (v & CE_ERR_ALGO_NOTSUP)
210 dev_err(ce->dev, "CE ERROR: algorithm not supported\n");
211 if (v & CE_ERR_DATALEN)
212 dev_err(ce->dev, "CE ERROR: data length error\n");
213 if (v & CE_ERR_KEYSRAM)
214 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
215 break;
216 case ESR_A64:
217 case ESR_D1:
218 case ESR_H5:
219 case ESR_R40:
220 v >>= (flow * 4);
221 v &= 0xF;
222 if (v) {
223 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow);
224 err = -EFAULT;
225 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4,
226 cet, sizeof(struct ce_task), false);
227 }
228 if (v & CE_ERR_ALGO_NOTSUP)
229 dev_err(ce->dev, "CE ERROR: algorithm not supported\n");
230 if (v & CE_ERR_DATALEN)
231 dev_err(ce->dev, "CE ERROR: data length error\n");
232 if (v & CE_ERR_KEYSRAM)
233 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
234 break;
235 case ESR_H6:
236 v >>= (flow * 8);
237 v &= 0xFF;
238 if (v) {
239 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow);
240 err = -EFAULT;
241 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4,
242 cet, sizeof(struct ce_task), false);
243 }
244 if (v & CE_ERR_ALGO_NOTSUP)
245 dev_err(ce->dev, "CE ERROR: algorithm not supported\n");
246 if (v & CE_ERR_DATALEN)
247 dev_err(ce->dev, "CE ERROR: data length error\n");
248 if (v & CE_ERR_KEYSRAM)
249 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
250 if (v & CE_ERR_ADDR_INVALID)
251 dev_err(ce->dev, "CE ERROR: address invalid\n");
252 if (v & CE_ERR_KEYLADDER)
253 dev_err(ce->dev, "CE ERROR: key ladder configuration error\n");
254 break;
255 }
256
257 return err;
258 }
259
ce_irq_handler(int irq,void * data)260 static irqreturn_t ce_irq_handler(int irq, void *data)
261 {
262 struct sun8i_ce_dev *ce = (struct sun8i_ce_dev *)data;
263 int flow = 0;
264 u32 p;
265
266 p = readl(ce->base + CE_ISR);
267 for (flow = 0; flow < MAXFLOW; flow++) {
268 if (p & (BIT(flow))) {
269 writel(BIT(flow), ce->base + CE_ISR);
270 ce->chanlist[flow].status = 1;
271 complete(&ce->chanlist[flow].complete);
272 }
273 }
274
275 return IRQ_HANDLED;
276 }
277
278 static struct sun8i_ce_alg_template ce_algs[] = {
279 {
280 .type = CRYPTO_ALG_TYPE_SKCIPHER,
281 .ce_algo_id = CE_ID_CIPHER_AES,
282 .ce_blockmode = CE_ID_OP_CBC,
283 .alg.skcipher.base = {
284 .base = {
285 .cra_name = "cbc(aes)",
286 .cra_driver_name = "cbc-aes-sun8i-ce",
287 .cra_priority = 400,
288 .cra_blocksize = AES_BLOCK_SIZE,
289 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
290 CRYPTO_ALG_ASYNC |
291 CRYPTO_ALG_NEED_FALLBACK,
292 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx),
293 .cra_module = THIS_MODULE,
294 .cra_alignmask = 0xf,
295 .cra_init = sun8i_ce_cipher_init,
296 .cra_exit = sun8i_ce_cipher_exit,
297 },
298 .min_keysize = AES_MIN_KEY_SIZE,
299 .max_keysize = AES_MAX_KEY_SIZE,
300 .ivsize = AES_BLOCK_SIZE,
301 .setkey = sun8i_ce_aes_setkey,
302 .encrypt = sun8i_ce_skencrypt,
303 .decrypt = sun8i_ce_skdecrypt,
304 },
305 .alg.skcipher.op = {
306 .do_one_request = sun8i_ce_cipher_do_one,
307 },
308 },
309 {
310 .type = CRYPTO_ALG_TYPE_SKCIPHER,
311 .ce_algo_id = CE_ID_CIPHER_AES,
312 .ce_blockmode = CE_ID_OP_ECB,
313 .alg.skcipher.base = {
314 .base = {
315 .cra_name = "ecb(aes)",
316 .cra_driver_name = "ecb-aes-sun8i-ce",
317 .cra_priority = 400,
318 .cra_blocksize = AES_BLOCK_SIZE,
319 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
320 CRYPTO_ALG_ASYNC |
321 CRYPTO_ALG_NEED_FALLBACK,
322 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx),
323 .cra_module = THIS_MODULE,
324 .cra_alignmask = 0xf,
325 .cra_init = sun8i_ce_cipher_init,
326 .cra_exit = sun8i_ce_cipher_exit,
327 },
328 .min_keysize = AES_MIN_KEY_SIZE,
329 .max_keysize = AES_MAX_KEY_SIZE,
330 .setkey = sun8i_ce_aes_setkey,
331 .encrypt = sun8i_ce_skencrypt,
332 .decrypt = sun8i_ce_skdecrypt,
333 },
334 .alg.skcipher.op = {
335 .do_one_request = sun8i_ce_cipher_do_one,
336 },
337 },
338 {
339 .type = CRYPTO_ALG_TYPE_SKCIPHER,
340 .ce_algo_id = CE_ID_CIPHER_DES3,
341 .ce_blockmode = CE_ID_OP_CBC,
342 .alg.skcipher.base = {
343 .base = {
344 .cra_name = "cbc(des3_ede)",
345 .cra_driver_name = "cbc-des3-sun8i-ce",
346 .cra_priority = 400,
347 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
348 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
349 CRYPTO_ALG_ASYNC |
350 CRYPTO_ALG_NEED_FALLBACK,
351 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx),
352 .cra_module = THIS_MODULE,
353 .cra_alignmask = 0xf,
354 .cra_init = sun8i_ce_cipher_init,
355 .cra_exit = sun8i_ce_cipher_exit,
356 },
357 .min_keysize = DES3_EDE_KEY_SIZE,
358 .max_keysize = DES3_EDE_KEY_SIZE,
359 .ivsize = DES3_EDE_BLOCK_SIZE,
360 .setkey = sun8i_ce_des3_setkey,
361 .encrypt = sun8i_ce_skencrypt,
362 .decrypt = sun8i_ce_skdecrypt,
363 },
364 .alg.skcipher.op = {
365 .do_one_request = sun8i_ce_cipher_do_one,
366 },
367 },
368 {
369 .type = CRYPTO_ALG_TYPE_SKCIPHER,
370 .ce_algo_id = CE_ID_CIPHER_DES3,
371 .ce_blockmode = CE_ID_OP_ECB,
372 .alg.skcipher.base = {
373 .base = {
374 .cra_name = "ecb(des3_ede)",
375 .cra_driver_name = "ecb-des3-sun8i-ce",
376 .cra_priority = 400,
377 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
378 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
379 CRYPTO_ALG_ASYNC |
380 CRYPTO_ALG_NEED_FALLBACK,
381 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx),
382 .cra_module = THIS_MODULE,
383 .cra_alignmask = 0xf,
384 .cra_init = sun8i_ce_cipher_init,
385 .cra_exit = sun8i_ce_cipher_exit,
386 },
387 .min_keysize = DES3_EDE_KEY_SIZE,
388 .max_keysize = DES3_EDE_KEY_SIZE,
389 .setkey = sun8i_ce_des3_setkey,
390 .encrypt = sun8i_ce_skencrypt,
391 .decrypt = sun8i_ce_skdecrypt,
392 },
393 .alg.skcipher.op = {
394 .do_one_request = sun8i_ce_cipher_do_one,
395 },
396 },
397 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH
398 { .type = CRYPTO_ALG_TYPE_AHASH,
399 .ce_algo_id = CE_ID_HASH_MD5,
400 .alg.hash.base = {
401 .init = sun8i_ce_hash_init,
402 .update = sun8i_ce_hash_update,
403 .final = sun8i_ce_hash_final,
404 .finup = sun8i_ce_hash_finup,
405 .digest = sun8i_ce_hash_digest,
406 .export = sun8i_ce_hash_export,
407 .import = sun8i_ce_hash_import,
408 .init_tfm = sun8i_ce_hash_init_tfm,
409 .exit_tfm = sun8i_ce_hash_exit_tfm,
410 .halg = {
411 .digestsize = MD5_DIGEST_SIZE,
412 .statesize = sizeof(struct md5_state),
413 .base = {
414 .cra_name = "md5",
415 .cra_driver_name = "md5-sun8i-ce",
416 .cra_priority = 300,
417 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
418 CRYPTO_ALG_ASYNC |
419 CRYPTO_ALG_NEED_FALLBACK,
420 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
421 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
422 .cra_module = THIS_MODULE,
423 }
424 }
425 },
426 .alg.hash.op = {
427 .do_one_request = sun8i_ce_hash_run,
428 },
429
430 },
431 { .type = CRYPTO_ALG_TYPE_AHASH,
432 .ce_algo_id = CE_ID_HASH_SHA1,
433 .alg.hash.base = {
434 .init = sun8i_ce_hash_init,
435 .update = sun8i_ce_hash_update,
436 .final = sun8i_ce_hash_final,
437 .finup = sun8i_ce_hash_finup,
438 .digest = sun8i_ce_hash_digest,
439 .export = sun8i_ce_hash_export,
440 .import = sun8i_ce_hash_import,
441 .init_tfm = sun8i_ce_hash_init_tfm,
442 .exit_tfm = sun8i_ce_hash_exit_tfm,
443 .halg = {
444 .digestsize = SHA1_DIGEST_SIZE,
445 .statesize = sizeof(struct sha1_state),
446 .base = {
447 .cra_name = "sha1",
448 .cra_driver_name = "sha1-sun8i-ce",
449 .cra_priority = 300,
450 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
451 CRYPTO_ALG_ASYNC |
452 CRYPTO_ALG_NEED_FALLBACK,
453 .cra_blocksize = SHA1_BLOCK_SIZE,
454 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
455 .cra_module = THIS_MODULE,
456 }
457 }
458 },
459 .alg.hash.op = {
460 .do_one_request = sun8i_ce_hash_run,
461 },
462 },
463 { .type = CRYPTO_ALG_TYPE_AHASH,
464 .ce_algo_id = CE_ID_HASH_SHA224,
465 .alg.hash.base = {
466 .init = sun8i_ce_hash_init,
467 .update = sun8i_ce_hash_update,
468 .final = sun8i_ce_hash_final,
469 .finup = sun8i_ce_hash_finup,
470 .digest = sun8i_ce_hash_digest,
471 .export = sun8i_ce_hash_export,
472 .import = sun8i_ce_hash_import,
473 .init_tfm = sun8i_ce_hash_init_tfm,
474 .exit_tfm = sun8i_ce_hash_exit_tfm,
475 .halg = {
476 .digestsize = SHA224_DIGEST_SIZE,
477 .statesize = sizeof(struct sha256_state),
478 .base = {
479 .cra_name = "sha224",
480 .cra_driver_name = "sha224-sun8i-ce",
481 .cra_priority = 300,
482 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
483 CRYPTO_ALG_ASYNC |
484 CRYPTO_ALG_NEED_FALLBACK,
485 .cra_blocksize = SHA224_BLOCK_SIZE,
486 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
487 .cra_module = THIS_MODULE,
488 }
489 }
490 },
491 .alg.hash.op = {
492 .do_one_request = sun8i_ce_hash_run,
493 },
494 },
495 { .type = CRYPTO_ALG_TYPE_AHASH,
496 .ce_algo_id = CE_ID_HASH_SHA256,
497 .alg.hash.base = {
498 .init = sun8i_ce_hash_init,
499 .update = sun8i_ce_hash_update,
500 .final = sun8i_ce_hash_final,
501 .finup = sun8i_ce_hash_finup,
502 .digest = sun8i_ce_hash_digest,
503 .export = sun8i_ce_hash_export,
504 .import = sun8i_ce_hash_import,
505 .init_tfm = sun8i_ce_hash_init_tfm,
506 .exit_tfm = sun8i_ce_hash_exit_tfm,
507 .halg = {
508 .digestsize = SHA256_DIGEST_SIZE,
509 .statesize = sizeof(struct sha256_state),
510 .base = {
511 .cra_name = "sha256",
512 .cra_driver_name = "sha256-sun8i-ce",
513 .cra_priority = 300,
514 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
515 CRYPTO_ALG_ASYNC |
516 CRYPTO_ALG_NEED_FALLBACK,
517 .cra_blocksize = SHA256_BLOCK_SIZE,
518 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
519 .cra_module = THIS_MODULE,
520 }
521 }
522 },
523 .alg.hash.op = {
524 .do_one_request = sun8i_ce_hash_run,
525 },
526 },
527 { .type = CRYPTO_ALG_TYPE_AHASH,
528 .ce_algo_id = CE_ID_HASH_SHA384,
529 .alg.hash.base = {
530 .init = sun8i_ce_hash_init,
531 .update = sun8i_ce_hash_update,
532 .final = sun8i_ce_hash_final,
533 .finup = sun8i_ce_hash_finup,
534 .digest = sun8i_ce_hash_digest,
535 .export = sun8i_ce_hash_export,
536 .import = sun8i_ce_hash_import,
537 .init_tfm = sun8i_ce_hash_init_tfm,
538 .exit_tfm = sun8i_ce_hash_exit_tfm,
539 .halg = {
540 .digestsize = SHA384_DIGEST_SIZE,
541 .statesize = sizeof(struct sha512_state),
542 .base = {
543 .cra_name = "sha384",
544 .cra_driver_name = "sha384-sun8i-ce",
545 .cra_priority = 300,
546 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
547 CRYPTO_ALG_ASYNC |
548 CRYPTO_ALG_NEED_FALLBACK,
549 .cra_blocksize = SHA384_BLOCK_SIZE,
550 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
551 .cra_module = THIS_MODULE,
552 }
553 }
554 },
555 .alg.hash.op = {
556 .do_one_request = sun8i_ce_hash_run,
557 },
558 },
559 { .type = CRYPTO_ALG_TYPE_AHASH,
560 .ce_algo_id = CE_ID_HASH_SHA512,
561 .alg.hash.base = {
562 .init = sun8i_ce_hash_init,
563 .update = sun8i_ce_hash_update,
564 .final = sun8i_ce_hash_final,
565 .finup = sun8i_ce_hash_finup,
566 .digest = sun8i_ce_hash_digest,
567 .export = sun8i_ce_hash_export,
568 .import = sun8i_ce_hash_import,
569 .init_tfm = sun8i_ce_hash_init_tfm,
570 .exit_tfm = sun8i_ce_hash_exit_tfm,
571 .halg = {
572 .digestsize = SHA512_DIGEST_SIZE,
573 .statesize = sizeof(struct sha512_state),
574 .base = {
575 .cra_name = "sha512",
576 .cra_driver_name = "sha512-sun8i-ce",
577 .cra_priority = 300,
578 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
579 CRYPTO_ALG_ASYNC |
580 CRYPTO_ALG_NEED_FALLBACK,
581 .cra_blocksize = SHA512_BLOCK_SIZE,
582 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
583 .cra_module = THIS_MODULE,
584 }
585 }
586 },
587 .alg.hash.op = {
588 .do_one_request = sun8i_ce_hash_run,
589 },
590 },
591 #endif
592 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
593 {
594 .type = CRYPTO_ALG_TYPE_RNG,
595 .alg.rng = {
596 .base = {
597 .cra_name = "stdrng",
598 .cra_driver_name = "sun8i-ce-prng",
599 .cra_priority = 300,
600 .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx),
601 .cra_module = THIS_MODULE,
602 .cra_init = sun8i_ce_prng_init,
603 .cra_exit = sun8i_ce_prng_exit,
604 },
605 .generate = sun8i_ce_prng_generate,
606 .seed = sun8i_ce_prng_seed,
607 .seedsize = PRNG_SEED_SIZE,
608 }
609 },
610 #endif
611 };
612
sun8i_ce_debugfs_show(struct seq_file * seq,void * v)613 static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v)
614 {
615 struct sun8i_ce_dev *ce __maybe_unused = seq->private;
616 unsigned int i;
617
618 for (i = 0; i < MAXFLOW; i++)
619 seq_printf(seq, "Channel %d: nreq %lu\n", i,
620 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
621 ce->chanlist[i].stat_req);
622 #else
623 0ul);
624 #endif
625
626 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
627 if (!ce_algs[i].ce)
628 continue;
629 switch (ce_algs[i].type) {
630 case CRYPTO_ALG_TYPE_SKCIPHER:
631 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
632 ce_algs[i].alg.skcipher.base.base.cra_driver_name,
633 ce_algs[i].alg.skcipher.base.base.cra_name,
634 ce_algs[i].stat_req, ce_algs[i].stat_fb);
635 seq_printf(seq, "\tLast fallback is: %s\n",
636 ce_algs[i].fbname);
637 seq_printf(seq, "\tFallback due to 0 length: %lu\n",
638 ce_algs[i].stat_fb_len0);
639 seq_printf(seq, "\tFallback due to length !mod16: %lu\n",
640 ce_algs[i].stat_fb_mod16);
641 seq_printf(seq, "\tFallback due to length < IV: %lu\n",
642 ce_algs[i].stat_fb_leniv);
643 seq_printf(seq, "\tFallback due to source alignment: %lu\n",
644 ce_algs[i].stat_fb_srcali);
645 seq_printf(seq, "\tFallback due to dest alignment: %lu\n",
646 ce_algs[i].stat_fb_dstali);
647 seq_printf(seq, "\tFallback due to source length: %lu\n",
648 ce_algs[i].stat_fb_srclen);
649 seq_printf(seq, "\tFallback due to dest length: %lu\n",
650 ce_algs[i].stat_fb_dstlen);
651 seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
652 ce_algs[i].stat_fb_maxsg);
653 break;
654 case CRYPTO_ALG_TYPE_AHASH:
655 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
656 ce_algs[i].alg.hash.base.halg.base.cra_driver_name,
657 ce_algs[i].alg.hash.base.halg.base.cra_name,
658 ce_algs[i].stat_req, ce_algs[i].stat_fb);
659 seq_printf(seq, "\tLast fallback is: %s\n",
660 ce_algs[i].fbname);
661 seq_printf(seq, "\tFallback due to 0 length: %lu\n",
662 ce_algs[i].stat_fb_len0);
663 seq_printf(seq, "\tFallback due to length: %lu\n",
664 ce_algs[i].stat_fb_srclen);
665 seq_printf(seq, "\tFallback due to alignment: %lu\n",
666 ce_algs[i].stat_fb_srcali);
667 seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
668 ce_algs[i].stat_fb_maxsg);
669 break;
670 case CRYPTO_ALG_TYPE_RNG:
671 seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n",
672 ce_algs[i].alg.rng.base.cra_driver_name,
673 ce_algs[i].alg.rng.base.cra_name,
674 ce_algs[i].stat_req, ce_algs[i].stat_bytes);
675 break;
676 }
677 }
678 #if defined(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) && \
679 defined(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)
680 seq_printf(seq, "HWRNG %lu %lu\n",
681 ce->hwrng_stat_req, ce->hwrng_stat_bytes);
682 #endif
683 return 0;
684 }
685
686 DEFINE_SHOW_ATTRIBUTE(sun8i_ce_debugfs);
687
sun8i_ce_free_chanlist(struct sun8i_ce_dev * ce,int i)688 static void sun8i_ce_free_chanlist(struct sun8i_ce_dev *ce, int i)
689 {
690 while (i >= 0) {
691 crypto_engine_exit(ce->chanlist[i].engine);
692 if (ce->chanlist[i].tl)
693 dma_free_coherent(ce->dev, sizeof(struct ce_task),
694 ce->chanlist[i].tl,
695 ce->chanlist[i].t_phy);
696 i--;
697 }
698 }
699
700 /*
701 * Allocate the channel list structure
702 */
sun8i_ce_allocate_chanlist(struct sun8i_ce_dev * ce)703 static int sun8i_ce_allocate_chanlist(struct sun8i_ce_dev *ce)
704 {
705 int i, err;
706
707 ce->chanlist = devm_kcalloc(ce->dev, MAXFLOW,
708 sizeof(struct sun8i_ce_flow), GFP_KERNEL);
709 if (!ce->chanlist)
710 return -ENOMEM;
711
712 for (i = 0; i < MAXFLOW; i++) {
713 init_completion(&ce->chanlist[i].complete);
714
715 ce->chanlist[i].engine = crypto_engine_alloc_init(ce->dev, true);
716 if (!ce->chanlist[i].engine) {
717 dev_err(ce->dev, "Cannot allocate engine\n");
718 i--;
719 err = -ENOMEM;
720 goto error_engine;
721 }
722 err = crypto_engine_start(ce->chanlist[i].engine);
723 if (err) {
724 dev_err(ce->dev, "Cannot start engine\n");
725 goto error_engine;
726 }
727 ce->chanlist[i].tl = dma_alloc_coherent(ce->dev,
728 sizeof(struct ce_task),
729 &ce->chanlist[i].t_phy,
730 GFP_KERNEL);
731 if (!ce->chanlist[i].tl) {
732 dev_err(ce->dev, "Cannot get DMA memory for task %d\n",
733 i);
734 err = -ENOMEM;
735 goto error_engine;
736 }
737 ce->chanlist[i].bounce_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE,
738 GFP_KERNEL | GFP_DMA);
739 if (!ce->chanlist[i].bounce_iv) {
740 err = -ENOMEM;
741 goto error_engine;
742 }
743 ce->chanlist[i].backup_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE,
744 GFP_KERNEL);
745 if (!ce->chanlist[i].backup_iv) {
746 err = -ENOMEM;
747 goto error_engine;
748 }
749 }
750 return 0;
751 error_engine:
752 sun8i_ce_free_chanlist(ce, i);
753 return err;
754 }
755
756 /*
757 * Power management strategy: The device is suspended unless a TFM exists for
758 * one of the algorithms proposed by this driver.
759 */
sun8i_ce_pm_suspend(struct device * dev)760 static int sun8i_ce_pm_suspend(struct device *dev)
761 {
762 struct sun8i_ce_dev *ce = dev_get_drvdata(dev);
763 int i;
764
765 reset_control_assert(ce->reset);
766 for (i = 0; i < CE_MAX_CLOCKS; i++)
767 clk_disable_unprepare(ce->ceclks[i]);
768 return 0;
769 }
770
sun8i_ce_pm_resume(struct device * dev)771 static int sun8i_ce_pm_resume(struct device *dev)
772 {
773 struct sun8i_ce_dev *ce = dev_get_drvdata(dev);
774 int err, i;
775
776 for (i = 0; i < CE_MAX_CLOCKS; i++) {
777 if (!ce->variant->ce_clks[i].name)
778 continue;
779 err = clk_prepare_enable(ce->ceclks[i]);
780 if (err) {
781 dev_err(ce->dev, "Cannot prepare_enable %s\n",
782 ce->variant->ce_clks[i].name);
783 goto error;
784 }
785 }
786 err = reset_control_deassert(ce->reset);
787 if (err) {
788 dev_err(ce->dev, "Cannot deassert reset control\n");
789 goto error;
790 }
791 return 0;
792 error:
793 sun8i_ce_pm_suspend(dev);
794 return err;
795 }
796
797 static const struct dev_pm_ops sun8i_ce_pm_ops = {
798 SET_RUNTIME_PM_OPS(sun8i_ce_pm_suspend, sun8i_ce_pm_resume, NULL)
799 };
800
sun8i_ce_pm_init(struct sun8i_ce_dev * ce)801 static int sun8i_ce_pm_init(struct sun8i_ce_dev *ce)
802 {
803 int err;
804
805 pm_runtime_use_autosuspend(ce->dev);
806 pm_runtime_set_autosuspend_delay(ce->dev, 2000);
807
808 err = pm_runtime_set_suspended(ce->dev);
809 if (err)
810 return err;
811 pm_runtime_enable(ce->dev);
812 return err;
813 }
814
sun8i_ce_pm_exit(struct sun8i_ce_dev * ce)815 static void sun8i_ce_pm_exit(struct sun8i_ce_dev *ce)
816 {
817 pm_runtime_disable(ce->dev);
818 }
819
sun8i_ce_get_clks(struct sun8i_ce_dev * ce)820 static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce)
821 {
822 unsigned long cr;
823 int err, i;
824
825 for (i = 0; i < CE_MAX_CLOCKS; i++) {
826 if (!ce->variant->ce_clks[i].name)
827 continue;
828 ce->ceclks[i] = devm_clk_get(ce->dev, ce->variant->ce_clks[i].name);
829 if (IS_ERR(ce->ceclks[i])) {
830 err = PTR_ERR(ce->ceclks[i]);
831 dev_err(ce->dev, "Cannot get %s CE clock err=%d\n",
832 ce->variant->ce_clks[i].name, err);
833 return err;
834 }
835 cr = clk_get_rate(ce->ceclks[i]);
836 if (!cr)
837 return -EINVAL;
838 if (ce->variant->ce_clks[i].freq > 0 &&
839 cr != ce->variant->ce_clks[i].freq) {
840 dev_info(ce->dev, "Set %s clock to %lu (%lu Mhz) from %lu (%lu Mhz)\n",
841 ce->variant->ce_clks[i].name,
842 ce->variant->ce_clks[i].freq,
843 ce->variant->ce_clks[i].freq / 1000000,
844 cr, cr / 1000000);
845 err = clk_set_rate(ce->ceclks[i], ce->variant->ce_clks[i].freq);
846 if (err)
847 dev_err(ce->dev, "Fail to set %s clk speed to %lu hz\n",
848 ce->variant->ce_clks[i].name,
849 ce->variant->ce_clks[i].freq);
850 }
851 if (ce->variant->ce_clks[i].max_freq > 0 &&
852 cr > ce->variant->ce_clks[i].max_freq)
853 dev_warn(ce->dev, "Frequency for %s (%lu hz) is higher than datasheet's recommendation (%lu hz)",
854 ce->variant->ce_clks[i].name, cr,
855 ce->variant->ce_clks[i].max_freq);
856 }
857 return 0;
858 }
859
sun8i_ce_register_algs(struct sun8i_ce_dev * ce)860 static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce)
861 {
862 int ce_method, err, id;
863 unsigned int i;
864
865 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
866 ce_algs[i].ce = ce;
867 switch (ce_algs[i].type) {
868 case CRYPTO_ALG_TYPE_SKCIPHER:
869 id = ce_algs[i].ce_algo_id;
870 ce_method = ce->variant->alg_cipher[id];
871 if (ce_method == CE_ID_NOTSUPP) {
872 dev_dbg(ce->dev,
873 "DEBUG: Algo of %s not supported\n",
874 ce_algs[i].alg.skcipher.base.base.cra_name);
875 ce_algs[i].ce = NULL;
876 break;
877 }
878 id = ce_algs[i].ce_blockmode;
879 ce_method = ce->variant->op_mode[id];
880 if (ce_method == CE_ID_NOTSUPP) {
881 dev_dbg(ce->dev, "DEBUG: Blockmode of %s not supported\n",
882 ce_algs[i].alg.skcipher.base.base.cra_name);
883 ce_algs[i].ce = NULL;
884 break;
885 }
886 dev_info(ce->dev, "Register %s\n",
887 ce_algs[i].alg.skcipher.base.base.cra_name);
888 err = crypto_engine_register_skcipher(&ce_algs[i].alg.skcipher);
889 if (err) {
890 dev_err(ce->dev, "ERROR: Fail to register %s\n",
891 ce_algs[i].alg.skcipher.base.base.cra_name);
892 ce_algs[i].ce = NULL;
893 return err;
894 }
895 break;
896 case CRYPTO_ALG_TYPE_AHASH:
897 id = ce_algs[i].ce_algo_id;
898 ce_method = ce->variant->alg_hash[id];
899 if (ce_method == CE_ID_NOTSUPP) {
900 dev_info(ce->dev,
901 "DEBUG: Algo of %s not supported\n",
902 ce_algs[i].alg.hash.base.halg.base.cra_name);
903 ce_algs[i].ce = NULL;
904 break;
905 }
906 dev_info(ce->dev, "Register %s\n",
907 ce_algs[i].alg.hash.base.halg.base.cra_name);
908 err = crypto_engine_register_ahash(&ce_algs[i].alg.hash);
909 if (err) {
910 dev_err(ce->dev, "ERROR: Fail to register %s\n",
911 ce_algs[i].alg.hash.base.halg.base.cra_name);
912 ce_algs[i].ce = NULL;
913 return err;
914 }
915 break;
916 case CRYPTO_ALG_TYPE_RNG:
917 if (ce->variant->prng == CE_ID_NOTSUPP) {
918 dev_info(ce->dev,
919 "DEBUG: Algo of %s not supported\n",
920 ce_algs[i].alg.rng.base.cra_name);
921 ce_algs[i].ce = NULL;
922 break;
923 }
924 dev_info(ce->dev, "Register %s\n",
925 ce_algs[i].alg.rng.base.cra_name);
926 err = crypto_register_rng(&ce_algs[i].alg.rng);
927 if (err) {
928 dev_err(ce->dev, "Fail to register %s\n",
929 ce_algs[i].alg.rng.base.cra_name);
930 ce_algs[i].ce = NULL;
931 }
932 break;
933 default:
934 ce_algs[i].ce = NULL;
935 dev_err(ce->dev, "ERROR: tried to register an unknown algo\n");
936 }
937 }
938 return 0;
939 }
940
sun8i_ce_unregister_algs(struct sun8i_ce_dev * ce)941 static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce)
942 {
943 unsigned int i;
944
945 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
946 if (!ce_algs[i].ce)
947 continue;
948 switch (ce_algs[i].type) {
949 case CRYPTO_ALG_TYPE_SKCIPHER:
950 dev_info(ce->dev, "Unregister %d %s\n", i,
951 ce_algs[i].alg.skcipher.base.base.cra_name);
952 crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher);
953 break;
954 case CRYPTO_ALG_TYPE_AHASH:
955 dev_info(ce->dev, "Unregister %d %s\n", i,
956 ce_algs[i].alg.hash.base.halg.base.cra_name);
957 crypto_engine_unregister_ahash(&ce_algs[i].alg.hash);
958 break;
959 case CRYPTO_ALG_TYPE_RNG:
960 dev_info(ce->dev, "Unregister %d %s\n", i,
961 ce_algs[i].alg.rng.base.cra_name);
962 crypto_unregister_rng(&ce_algs[i].alg.rng);
963 break;
964 }
965 }
966 }
967
sun8i_ce_probe(struct platform_device * pdev)968 static int sun8i_ce_probe(struct platform_device *pdev)
969 {
970 struct sun8i_ce_dev *ce;
971 int err, irq;
972 u32 v;
973
974 ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL);
975 if (!ce)
976 return -ENOMEM;
977
978 ce->dev = &pdev->dev;
979 platform_set_drvdata(pdev, ce);
980
981 ce->variant = of_device_get_match_data(&pdev->dev);
982 if (!ce->variant) {
983 dev_err(&pdev->dev, "Missing Crypto Engine variant\n");
984 return -EINVAL;
985 }
986
987 ce->base = devm_platform_ioremap_resource(pdev, 0);
988 if (IS_ERR(ce->base))
989 return PTR_ERR(ce->base);
990
991 err = sun8i_ce_get_clks(ce);
992 if (err)
993 return err;
994
995 /* Get Non Secure IRQ */
996 irq = platform_get_irq(pdev, 0);
997 if (irq < 0)
998 return irq;
999
1000 ce->reset = devm_reset_control_get(&pdev->dev, NULL);
1001 if (IS_ERR(ce->reset))
1002 return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset),
1003 "No reset control found\n");
1004
1005 mutex_init(&ce->mlock);
1006 mutex_init(&ce->rnglock);
1007
1008 err = sun8i_ce_allocate_chanlist(ce);
1009 if (err)
1010 return err;
1011
1012 err = sun8i_ce_pm_init(ce);
1013 if (err)
1014 goto error_pm;
1015
1016 err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0,
1017 "sun8i-ce-ns", ce);
1018 if (err) {
1019 dev_err(ce->dev, "Cannot request CryptoEngine Non-secure IRQ (err=%d)\n", err);
1020 goto error_irq;
1021 }
1022
1023 err = sun8i_ce_register_algs(ce);
1024 if (err)
1025 goto error_alg;
1026
1027 err = pm_runtime_resume_and_get(ce->dev);
1028 if (err < 0)
1029 goto error_alg;
1030
1031 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG
1032 sun8i_ce_hwrng_register(ce);
1033 #endif
1034
1035 v = readl(ce->base + CE_CTR);
1036 v >>= CE_DIE_ID_SHIFT;
1037 v &= CE_DIE_ID_MASK;
1038 dev_info(&pdev->dev, "CryptoEngine Die ID %x\n", v);
1039
1040 pm_runtime_put_sync(ce->dev);
1041
1042 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {
1043 struct dentry *dbgfs_dir __maybe_unused;
1044 struct dentry *dbgfs_stats __maybe_unused;
1045
1046 /* Ignore error of debugfs */
1047 dbgfs_dir = debugfs_create_dir("sun8i-ce", NULL);
1048 dbgfs_stats = debugfs_create_file("stats", 0444,
1049 dbgfs_dir, ce,
1050 &sun8i_ce_debugfs_fops);
1051
1052 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
1053 ce->dbgfs_dir = dbgfs_dir;
1054 ce->dbgfs_stats = dbgfs_stats;
1055 #endif
1056 }
1057
1058 return 0;
1059 error_alg:
1060 sun8i_ce_unregister_algs(ce);
1061 error_irq:
1062 sun8i_ce_pm_exit(ce);
1063 error_pm:
1064 sun8i_ce_free_chanlist(ce, MAXFLOW - 1);
1065 return err;
1066 }
1067
sun8i_ce_remove(struct platform_device * pdev)1068 static void sun8i_ce_remove(struct platform_device *pdev)
1069 {
1070 struct sun8i_ce_dev *ce = platform_get_drvdata(pdev);
1071
1072 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG
1073 sun8i_ce_hwrng_unregister(ce);
1074 #endif
1075
1076 sun8i_ce_unregister_algs(ce);
1077
1078 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
1079 debugfs_remove_recursive(ce->dbgfs_dir);
1080 #endif
1081
1082 sun8i_ce_free_chanlist(ce, MAXFLOW - 1);
1083
1084 sun8i_ce_pm_exit(ce);
1085 }
1086
1087 static const struct of_device_id sun8i_ce_crypto_of_match_table[] = {
1088 { .compatible = "allwinner,sun8i-h3-crypto",
1089 .data = &ce_h3_variant },
1090 { .compatible = "allwinner,sun8i-r40-crypto",
1091 .data = &ce_r40_variant },
1092 { .compatible = "allwinner,sun20i-d1-crypto",
1093 .data = &ce_d1_variant },
1094 { .compatible = "allwinner,sun50i-a64-crypto",
1095 .data = &ce_a64_variant },
1096 { .compatible = "allwinner,sun50i-h5-crypto",
1097 .data = &ce_h5_variant },
1098 { .compatible = "allwinner,sun50i-h6-crypto",
1099 .data = &ce_h6_variant },
1100 {}
1101 };
1102 MODULE_DEVICE_TABLE(of, sun8i_ce_crypto_of_match_table);
1103
1104 static struct platform_driver sun8i_ce_driver = {
1105 .probe = sun8i_ce_probe,
1106 .remove_new = sun8i_ce_remove,
1107 .driver = {
1108 .name = "sun8i-ce",
1109 .pm = &sun8i_ce_pm_ops,
1110 .of_match_table = sun8i_ce_crypto_of_match_table,
1111 },
1112 };
1113
1114 module_platform_driver(sun8i_ce_driver);
1115
1116 MODULE_DESCRIPTION("Allwinner Crypto Engine cryptographic offloader");
1117 MODULE_LICENSE("GPL");
1118 MODULE_AUTHOR("Corentin Labbe <clabbe.montjoie@gmail.com>");
1119