1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * aes-ce-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions
4 *
5 * Copyright (C) 2013 - 2017 Linaro Ltd.
6 * Copyright (C) 2024 Google LLC
7 *
8 * Author: Ard Biesheuvel <ardb@kernel.org>
9 */
10
11 #include <linux/unaligned.h>
12 #include <crypto/aes.h>
13 #include <crypto/scatterwalk.h>
14 #include <crypto/internal/aead.h>
15 #include <crypto/internal/skcipher.h>
16 #include <linux/module.h>
17
18 #include <asm/simd.h>
19
20 MODULE_IMPORT_NS("CRYPTO_INTERNAL");
21
num_rounds(struct crypto_aes_ctx * ctx)22 static int num_rounds(struct crypto_aes_ctx *ctx)
23 {
24 /*
25 * # of rounds specified by AES:
26 * 128 bit key 10 rounds
27 * 192 bit key 12 rounds
28 * 256 bit key 14 rounds
29 * => n byte key => 6 + (n/4) rounds
30 */
31 return 6 + ctx->key_length / 4;
32 }
33
34 asmlinkage u32 ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
35 int blocks, u8 dg[], int enc_before,
36 int enc_after);
37
38 asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
39 u32 const rk[], u32 rounds, u8 mac[],
40 u8 ctr[], u8 const final_iv[]);
41
42 asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
43 u32 const rk[], u32 rounds, u8 mac[],
44 u8 ctr[], u8 const final_iv[]);
45
ccm_setkey(struct crypto_aead * tfm,const u8 * in_key,unsigned int key_len)46 static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
47 unsigned int key_len)
48 {
49 struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
50
51 return ce_aes_expandkey(ctx, in_key, key_len);
52 }
53
ccm_setauthsize(struct crypto_aead * tfm,unsigned int authsize)54 static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
55 {
56 if ((authsize & 1) || authsize < 4)
57 return -EINVAL;
58 return 0;
59 }
60
ccm_init_mac(struct aead_request * req,u8 maciv[],u32 msglen)61 static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
62 {
63 struct crypto_aead *aead = crypto_aead_reqtfm(req);
64 __be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
65 u32 l = req->iv[0] + 1;
66
67 /* verify that CCM dimension 'L' is set correctly in the IV */
68 if (l < 2 || l > 8)
69 return -EINVAL;
70
71 /* verify that msglen can in fact be represented in L bytes */
72 if (l < 4 && msglen >> (8 * l))
73 return -EOVERFLOW;
74
75 /*
76 * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
77 * uses a u32 type to represent msglen so the top 4 bytes are always 0.
78 */
79 n[0] = 0;
80 n[1] = cpu_to_be32(msglen);
81
82 memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
83
84 /*
85 * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
86 * - bits 0..2 : max # of bytes required to represent msglen, minus 1
87 * (already set by caller)
88 * - bits 3..5 : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
89 * - bit 6 : indicates presence of authenticate-only data
90 */
91 maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
92 if (req->assoclen)
93 maciv[0] |= 0x40;
94
95 memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
96 return 0;
97 }
98
ce_aes_ccm_auth_data(u8 mac[],u8 const in[],u32 abytes,u32 macp,u32 const rk[],u32 rounds)99 static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes,
100 u32 macp, u32 const rk[], u32 rounds)
101 {
102 int enc_after = (macp + abytes) % AES_BLOCK_SIZE;
103
104 do {
105 u32 blocks = abytes / AES_BLOCK_SIZE;
106
107 if (macp == AES_BLOCK_SIZE || (!macp && blocks > 0)) {
108 u32 rem = ce_aes_mac_update(in, rk, rounds, blocks, mac,
109 macp, enc_after);
110 u32 adv = (blocks - rem) * AES_BLOCK_SIZE;
111
112 macp = enc_after ? 0 : AES_BLOCK_SIZE;
113 in += adv;
114 abytes -= adv;
115
116 if (unlikely(rem))
117 macp = 0;
118 } else {
119 u32 l = min(AES_BLOCK_SIZE - macp, abytes);
120
121 crypto_xor(&mac[macp], in, l);
122 in += l;
123 macp += l;
124 abytes -= l;
125 }
126 } while (abytes > 0);
127
128 return macp;
129 }
130
ccm_calculate_auth_mac(struct aead_request * req,u8 mac[])131 static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
132 {
133 struct crypto_aead *aead = crypto_aead_reqtfm(req);
134 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
135 struct __packed { __be16 l; __be32 h; u16 len; } ltag;
136 struct scatter_walk walk;
137 u32 len = req->assoclen;
138 u32 macp = AES_BLOCK_SIZE;
139
140 /* prepend the AAD with a length tag */
141 if (len < 0xff00) {
142 ltag.l = cpu_to_be16(len);
143 ltag.len = 2;
144 } else {
145 ltag.l = cpu_to_be16(0xfffe);
146 put_unaligned_be32(len, <ag.h);
147 ltag.len = 6;
148 }
149
150 macp = ce_aes_ccm_auth_data(mac, (u8 *)<ag, ltag.len, macp,
151 ctx->key_enc, num_rounds(ctx));
152 scatterwalk_start(&walk, req->src);
153
154 do {
155 unsigned int n;
156
157 n = scatterwalk_next(&walk, len);
158 macp = ce_aes_ccm_auth_data(mac, walk.addr, n, macp,
159 ctx->key_enc, num_rounds(ctx));
160 scatterwalk_done_src(&walk, n);
161 len -= n;
162 } while (len);
163 }
164
ccm_encrypt(struct aead_request * req)165 static int ccm_encrypt(struct aead_request *req)
166 {
167 struct crypto_aead *aead = crypto_aead_reqtfm(req);
168 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
169 struct skcipher_walk walk;
170 u8 __aligned(8) mac[AES_BLOCK_SIZE];
171 u8 orig_iv[AES_BLOCK_SIZE];
172 u32 len = req->cryptlen;
173 int err;
174
175 err = ccm_init_mac(req, mac, len);
176 if (err)
177 return err;
178
179 /* preserve the original iv for the final round */
180 memcpy(orig_iv, req->iv, AES_BLOCK_SIZE);
181
182 err = skcipher_walk_aead_encrypt(&walk, req, false);
183 if (unlikely(err))
184 return err;
185
186 scoped_ksimd() {
187 if (req->assoclen)
188 ccm_calculate_auth_mac(req, mac);
189
190 do {
191 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
192 const u8 *src = walk.src.virt.addr;
193 u8 *dst = walk.dst.virt.addr;
194 u8 buf[AES_BLOCK_SIZE];
195 u8 *final_iv = NULL;
196
197 if (walk.nbytes == walk.total) {
198 tail = 0;
199 final_iv = orig_iv;
200 }
201
202 if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
203 src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes],
204 src, walk.nbytes);
205
206 ce_aes_ccm_encrypt(dst, src, walk.nbytes - tail,
207 ctx->key_enc, num_rounds(ctx),
208 mac, walk.iv, final_iv);
209
210 if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
211 memcpy(walk.dst.virt.addr, dst, walk.nbytes);
212
213 if (walk.nbytes) {
214 err = skcipher_walk_done(&walk, tail);
215 }
216 } while (walk.nbytes);
217 }
218
219 if (unlikely(err))
220 return err;
221
222 /* copy authtag to end of dst */
223 scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
224 crypto_aead_authsize(aead), 1);
225
226 return 0;
227 }
228
ccm_decrypt(struct aead_request * req)229 static int ccm_decrypt(struct aead_request *req)
230 {
231 struct crypto_aead *aead = crypto_aead_reqtfm(req);
232 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
233 unsigned int authsize = crypto_aead_authsize(aead);
234 struct skcipher_walk walk;
235 u8 __aligned(8) mac[AES_BLOCK_SIZE];
236 u8 orig_iv[AES_BLOCK_SIZE];
237 u32 len = req->cryptlen - authsize;
238 int err;
239
240 err = ccm_init_mac(req, mac, len);
241 if (err)
242 return err;
243
244 /* preserve the original iv for the final round */
245 memcpy(orig_iv, req->iv, AES_BLOCK_SIZE);
246
247 err = skcipher_walk_aead_decrypt(&walk, req, false);
248 if (unlikely(err))
249 return err;
250
251 scoped_ksimd() {
252 if (req->assoclen)
253 ccm_calculate_auth_mac(req, mac);
254
255 do {
256 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
257 const u8 *src = walk.src.virt.addr;
258 u8 *dst = walk.dst.virt.addr;
259 u8 buf[AES_BLOCK_SIZE];
260 u8 *final_iv = NULL;
261
262 if (walk.nbytes == walk.total) {
263 tail = 0;
264 final_iv = orig_iv;
265 }
266
267 if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
268 src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes],
269 src, walk.nbytes);
270
271 ce_aes_ccm_decrypt(dst, src, walk.nbytes - tail,
272 ctx->key_enc, num_rounds(ctx),
273 mac, walk.iv, final_iv);
274
275 if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
276 memcpy(walk.dst.virt.addr, dst, walk.nbytes);
277
278 if (walk.nbytes) {
279 err = skcipher_walk_done(&walk, tail);
280 }
281 } while (walk.nbytes);
282 }
283
284 if (unlikely(err))
285 return err;
286
287 /* compare calculated auth tag with the stored one */
288 scatterwalk_map_and_copy(orig_iv, req->src,
289 req->assoclen + req->cryptlen - authsize,
290 authsize, 0);
291
292 if (crypto_memneq(mac, orig_iv, authsize))
293 return -EBADMSG;
294 return 0;
295 }
296
297 static struct aead_alg ccm_aes_alg = {
298 .base = {
299 .cra_name = "ccm(aes)",
300 .cra_driver_name = "ccm-aes-ce",
301 .cra_priority = 300,
302 .cra_blocksize = 1,
303 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
304 .cra_module = THIS_MODULE,
305 },
306 .ivsize = AES_BLOCK_SIZE,
307 .chunksize = AES_BLOCK_SIZE,
308 .maxauthsize = AES_BLOCK_SIZE,
309 .setkey = ccm_setkey,
310 .setauthsize = ccm_setauthsize,
311 .encrypt = ccm_encrypt,
312 .decrypt = ccm_decrypt,
313 };
314
aes_mod_init(void)315 static int __init aes_mod_init(void)
316 {
317 if (!cpu_have_named_feature(AES))
318 return -ENODEV;
319 return crypto_register_aead(&ccm_aes_alg);
320 }
321
aes_mod_exit(void)322 static void __exit aes_mod_exit(void)
323 {
324 crypto_unregister_aead(&ccm_aes_alg);
325 }
326
327 module_init(aes_mod_init);
328 module_exit(aes_mod_exit);
329
330 MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions");
331 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>");
332 MODULE_LICENSE("GPL v2");
333 MODULE_ALIAS_CRYPTO("ccm(aes)");
334