1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * MIPS accelerated ChaCha and XChaCha stream ciphers,
4 * including ChaCha20 (RFC7539)
5 *
6 * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
7 */
8
9 #include <asm/byteorder.h>
10 #include <crypto/algapi.h>
11 #include <crypto/internal/chacha.h>
12 #include <crypto/internal/skcipher.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15
16 asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src,
17 unsigned int bytes, int nrounds);
18 EXPORT_SYMBOL(chacha_crypt_arch);
19
20 asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds);
21 EXPORT_SYMBOL(hchacha_block_arch);
22
chacha_mips_stream_xor(struct skcipher_request * req,const struct chacha_ctx * ctx,const u8 * iv)23 static int chacha_mips_stream_xor(struct skcipher_request *req,
24 const struct chacha_ctx *ctx, const u8 *iv)
25 {
26 struct skcipher_walk walk;
27 u32 state[16];
28 int err;
29
30 err = skcipher_walk_virt(&walk, req, false);
31
32 chacha_init(state, ctx->key, iv);
33
34 while (walk.nbytes > 0) {
35 unsigned int nbytes = walk.nbytes;
36
37 if (nbytes < walk.total)
38 nbytes = round_down(nbytes, walk.stride);
39
40 chacha_crypt(state, walk.dst.virt.addr, walk.src.virt.addr,
41 nbytes, ctx->nrounds);
42 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
43 }
44
45 return err;
46 }
47
chacha_mips(struct skcipher_request * req)48 static int chacha_mips(struct skcipher_request *req)
49 {
50 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
51 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
52
53 return chacha_mips_stream_xor(req, ctx, req->iv);
54 }
55
xchacha_mips(struct skcipher_request * req)56 static int xchacha_mips(struct skcipher_request *req)
57 {
58 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
59 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
60 struct chacha_ctx subctx;
61 u32 state[16];
62 u8 real_iv[16];
63
64 chacha_init(state, ctx->key, req->iv);
65
66 hchacha_block(state, subctx.key, ctx->nrounds);
67 subctx.nrounds = ctx->nrounds;
68
69 memcpy(&real_iv[0], req->iv + 24, 8);
70 memcpy(&real_iv[8], req->iv + 16, 8);
71 return chacha_mips_stream_xor(req, &subctx, real_iv);
72 }
73
74 static struct skcipher_alg algs[] = {
75 {
76 .base.cra_name = "chacha20",
77 .base.cra_driver_name = "chacha20-mips",
78 .base.cra_priority = 200,
79 .base.cra_blocksize = 1,
80 .base.cra_ctxsize = sizeof(struct chacha_ctx),
81 .base.cra_module = THIS_MODULE,
82
83 .min_keysize = CHACHA_KEY_SIZE,
84 .max_keysize = CHACHA_KEY_SIZE,
85 .ivsize = CHACHA_IV_SIZE,
86 .chunksize = CHACHA_BLOCK_SIZE,
87 .setkey = chacha20_setkey,
88 .encrypt = chacha_mips,
89 .decrypt = chacha_mips,
90 }, {
91 .base.cra_name = "xchacha20",
92 .base.cra_driver_name = "xchacha20-mips",
93 .base.cra_priority = 200,
94 .base.cra_blocksize = 1,
95 .base.cra_ctxsize = sizeof(struct chacha_ctx),
96 .base.cra_module = THIS_MODULE,
97
98 .min_keysize = CHACHA_KEY_SIZE,
99 .max_keysize = CHACHA_KEY_SIZE,
100 .ivsize = XCHACHA_IV_SIZE,
101 .chunksize = CHACHA_BLOCK_SIZE,
102 .setkey = chacha20_setkey,
103 .encrypt = xchacha_mips,
104 .decrypt = xchacha_mips,
105 }, {
106 .base.cra_name = "xchacha12",
107 .base.cra_driver_name = "xchacha12-mips",
108 .base.cra_priority = 200,
109 .base.cra_blocksize = 1,
110 .base.cra_ctxsize = sizeof(struct chacha_ctx),
111 .base.cra_module = THIS_MODULE,
112
113 .min_keysize = CHACHA_KEY_SIZE,
114 .max_keysize = CHACHA_KEY_SIZE,
115 .ivsize = XCHACHA_IV_SIZE,
116 .chunksize = CHACHA_BLOCK_SIZE,
117 .setkey = chacha12_setkey,
118 .encrypt = xchacha_mips,
119 .decrypt = xchacha_mips,
120 }
121 };
122
chacha_simd_mod_init(void)123 static int __init chacha_simd_mod_init(void)
124 {
125 return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ?
126 crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0;
127 }
128
chacha_simd_mod_fini(void)129 static void __exit chacha_simd_mod_fini(void)
130 {
131 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER))
132 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
133 }
134
135 module_init(chacha_simd_mod_init);
136 module_exit(chacha_simd_mod_fini);
137
138 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)");
139 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
140 MODULE_LICENSE("GPL v2");
141 MODULE_ALIAS_CRYPTO("chacha20");
142 MODULE_ALIAS_CRYPTO("chacha20-mips");
143 MODULE_ALIAS_CRYPTO("xchacha20");
144 MODULE_ALIAS_CRYPTO("xchacha20-mips");
145 MODULE_ALIAS_CRYPTO("xchacha12");
146 MODULE_ALIAS_CRYPTO("xchacha12-mips");
147