1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * PowerPC P10 (ppc64le) accelerated ChaCha and XChaCha stream ciphers,
4 * including ChaCha20 (RFC7539)
5 *
6 * Copyright 2023- IBM Corp. All rights reserved.
7 */
8
9 #include <crypto/algapi.h>
10 #include <crypto/internal/chacha.h>
11 #include <crypto/internal/simd.h>
12 #include <crypto/internal/skcipher.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/cpufeature.h>
16 #include <linux/sizes.h>
17 #include <asm/simd.h>
18 #include <asm/switch_to.h>
19
20 asmlinkage void chacha_p10le_8x(u32 *state, u8 *dst, const u8 *src,
21 unsigned int len, int nrounds);
22
23 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
24
vsx_begin(void)25 static void vsx_begin(void)
26 {
27 preempt_disable();
28 enable_kernel_vsx();
29 }
30
vsx_end(void)31 static void vsx_end(void)
32 {
33 disable_kernel_vsx();
34 preempt_enable();
35 }
36
chacha_p10_do_8x(u32 * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)37 static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src,
38 unsigned int bytes, int nrounds)
39 {
40 unsigned int l = bytes & ~0x0FF;
41
42 if (l > 0) {
43 chacha_p10le_8x(state, dst, src, l, nrounds);
44 bytes -= l;
45 src += l;
46 dst += l;
47 state[12] += l / CHACHA_BLOCK_SIZE;
48 }
49
50 if (bytes > 0)
51 chacha_crypt_generic(state, dst, src, bytes, nrounds);
52 }
53
hchacha_block_arch(const u32 * state,u32 * stream,int nrounds)54 void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
55 {
56 hchacha_block_generic(state, stream, nrounds);
57 }
58 EXPORT_SYMBOL(hchacha_block_arch);
59
chacha_crypt_arch(u32 * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)60 void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
61 int nrounds)
62 {
63 if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||
64 !crypto_simd_usable())
65 return chacha_crypt_generic(state, dst, src, bytes, nrounds);
66
67 do {
68 unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
69
70 vsx_begin();
71 chacha_p10_do_8x(state, dst, src, todo, nrounds);
72 vsx_end();
73
74 bytes -= todo;
75 src += todo;
76 dst += todo;
77 } while (bytes);
78 }
79 EXPORT_SYMBOL(chacha_crypt_arch);
80
chacha_p10_stream_xor(struct skcipher_request * req,const struct chacha_ctx * ctx,const u8 * iv)81 static int chacha_p10_stream_xor(struct skcipher_request *req,
82 const struct chacha_ctx *ctx, const u8 *iv)
83 {
84 struct skcipher_walk walk;
85 u32 state[16];
86 int err;
87
88 err = skcipher_walk_virt(&walk, req, false);
89 if (err)
90 return err;
91
92 chacha_init(state, ctx->key, iv);
93
94 while (walk.nbytes > 0) {
95 unsigned int nbytes = walk.nbytes;
96
97 if (nbytes < walk.total)
98 nbytes = rounddown(nbytes, walk.stride);
99
100 if (!crypto_simd_usable()) {
101 chacha_crypt_generic(state, walk.dst.virt.addr,
102 walk.src.virt.addr, nbytes,
103 ctx->nrounds);
104 } else {
105 vsx_begin();
106 chacha_p10_do_8x(state, walk.dst.virt.addr,
107 walk.src.virt.addr, nbytes, ctx->nrounds);
108 vsx_end();
109 }
110 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
111 if (err)
112 break;
113 }
114
115 return err;
116 }
117
chacha_p10(struct skcipher_request * req)118 static int chacha_p10(struct skcipher_request *req)
119 {
120 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
121 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
122
123 return chacha_p10_stream_xor(req, ctx, req->iv);
124 }
125
xchacha_p10(struct skcipher_request * req)126 static int xchacha_p10(struct skcipher_request *req)
127 {
128 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
129 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
130 struct chacha_ctx subctx;
131 u32 state[16];
132 u8 real_iv[16];
133
134 chacha_init(state, ctx->key, req->iv);
135 hchacha_block_arch(state, subctx.key, ctx->nrounds);
136 subctx.nrounds = ctx->nrounds;
137
138 memcpy(&real_iv[0], req->iv + 24, 8);
139 memcpy(&real_iv[8], req->iv + 16, 8);
140 return chacha_p10_stream_xor(req, &subctx, real_iv);
141 }
142
143 static struct skcipher_alg algs[] = {
144 {
145 .base.cra_name = "chacha20",
146 .base.cra_driver_name = "chacha20-p10",
147 .base.cra_priority = 300,
148 .base.cra_blocksize = 1,
149 .base.cra_ctxsize = sizeof(struct chacha_ctx),
150 .base.cra_module = THIS_MODULE,
151
152 .min_keysize = CHACHA_KEY_SIZE,
153 .max_keysize = CHACHA_KEY_SIZE,
154 .ivsize = CHACHA_IV_SIZE,
155 .chunksize = CHACHA_BLOCK_SIZE,
156 .setkey = chacha20_setkey,
157 .encrypt = chacha_p10,
158 .decrypt = chacha_p10,
159 }, {
160 .base.cra_name = "xchacha20",
161 .base.cra_driver_name = "xchacha20-p10",
162 .base.cra_priority = 300,
163 .base.cra_blocksize = 1,
164 .base.cra_ctxsize = sizeof(struct chacha_ctx),
165 .base.cra_module = THIS_MODULE,
166
167 .min_keysize = CHACHA_KEY_SIZE,
168 .max_keysize = CHACHA_KEY_SIZE,
169 .ivsize = XCHACHA_IV_SIZE,
170 .chunksize = CHACHA_BLOCK_SIZE,
171 .setkey = chacha20_setkey,
172 .encrypt = xchacha_p10,
173 .decrypt = xchacha_p10,
174 }, {
175 .base.cra_name = "xchacha12",
176 .base.cra_driver_name = "xchacha12-p10",
177 .base.cra_priority = 300,
178 .base.cra_blocksize = 1,
179 .base.cra_ctxsize = sizeof(struct chacha_ctx),
180 .base.cra_module = THIS_MODULE,
181
182 .min_keysize = CHACHA_KEY_SIZE,
183 .max_keysize = CHACHA_KEY_SIZE,
184 .ivsize = XCHACHA_IV_SIZE,
185 .chunksize = CHACHA_BLOCK_SIZE,
186 .setkey = chacha12_setkey,
187 .encrypt = xchacha_p10,
188 .decrypt = xchacha_p10,
189 }
190 };
191
chacha_p10_init(void)192 static int __init chacha_p10_init(void)
193 {
194 if (!cpu_has_feature(CPU_FTR_ARCH_31))
195 return 0;
196
197 static_branch_enable(&have_p10);
198
199 return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
200 }
201
chacha_p10_exit(void)202 static void __exit chacha_p10_exit(void)
203 {
204 if (!static_branch_likely(&have_p10))
205 return;
206
207 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
208 }
209
210 module_init(chacha_p10_init);
211 module_exit(chacha_p10_exit);
212
213 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (P10 accelerated)");
214 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");
215 MODULE_LICENSE("GPL v2");
216 MODULE_ALIAS_CRYPTO("chacha20");
217 MODULE_ALIAS_CRYPTO("chacha20-p10");
218 MODULE_ALIAS_CRYPTO("xchacha20");
219 MODULE_ALIAS_CRYPTO("xchacha20-p10");
220 MODULE_ALIAS_CRYPTO("xchacha12");
221 MODULE_ALIAS_CRYPTO("xchacha12-p10");
222