1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Support for AES-NI and VAES instructions. This file contains glue code.
4 * The real AES implementations are in aesni-intel_asm.S and other .S files.
5 *
6 * Copyright (C) 2008, Intel Corp.
7 * Author: Huang Ying <ying.huang@intel.com>
8 *
9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10 * interface for 64-bit kernels.
11 * Authors: Adrian Hoban <adrian.hoban@intel.com>
12 * Gabriele Paoloni <gabriele.paoloni@intel.com>
13 * Tadeusz Struk (tadeusz.struk@intel.com)
14 * Aidan O'Mahony (aidan.o.mahony@intel.com)
15 * Copyright (c) 2010, Intel Corporation.
16 *
17 * Copyright 2024 Google LLC
18 */
19
20 #include <linux/hardirq.h>
21 #include <linux/types.h>
22 #include <linux/module.h>
23 #include <linux/err.h>
24 #include <crypto/algapi.h>
25 #include <crypto/aes.h>
26 #include <crypto/b128ops.h>
27 #include <crypto/gcm.h>
28 #include <crypto/xts.h>
29 #include <asm/cpu_device_id.h>
30 #include <asm/simd.h>
31 #include <crypto/scatterwalk.h>
32 #include <crypto/internal/aead.h>
33 #include <crypto/internal/simd.h>
34 #include <crypto/internal/skcipher.h>
35 #include <linux/jump_label.h>
36 #include <linux/workqueue.h>
37 #include <linux/spinlock.h>
38 #include <linux/static_call.h>
39
40
41 #define AESNI_ALIGN 16
42 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
43 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
47
48 struct aesni_xts_ctx {
49 struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR;
50 struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR;
51 };
52
aes_align_addr(void * addr)53 static inline void *aes_align_addr(void *addr)
54 {
55 if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN)
56 return addr;
57 return PTR_ALIGN(addr, AESNI_ALIGN);
58 }
59
60 asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
61 unsigned int key_len);
62 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
63 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
64 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
65 const u8 *in, unsigned int len);
66 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
67 const u8 *in, unsigned int len);
68 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
69 const u8 *in, unsigned int len, u8 *iv);
70 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
71 const u8 *in, unsigned int len, u8 *iv);
72 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
73 const u8 *in, unsigned int len, u8 *iv);
74 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
75 const u8 *in, unsigned int len, u8 *iv);
76
77 asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out,
78 const u8 *in, unsigned int len, u8 *iv);
79
80 asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out,
81 const u8 *in, unsigned int len, u8 *iv);
82
83 #ifdef CONFIG_X86_64
84 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
85 const u8 *in, unsigned int len, u8 *iv);
86 #endif
87
aes_ctx(void * raw_ctx)88 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
89 {
90 return aes_align_addr(raw_ctx);
91 }
92
aes_xts_ctx(struct crypto_skcipher * tfm)93 static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
94 {
95 return aes_align_addr(crypto_skcipher_ctx(tfm));
96 }
97
aes_set_key_common(struct crypto_aes_ctx * ctx,const u8 * in_key,unsigned int key_len)98 static int aes_set_key_common(struct crypto_aes_ctx *ctx,
99 const u8 *in_key, unsigned int key_len)
100 {
101 int err;
102
103 if (!crypto_simd_usable())
104 return aes_expandkey(ctx, in_key, key_len);
105
106 err = aes_check_keylen(key_len);
107 if (err)
108 return err;
109
110 kernel_fpu_begin();
111 aesni_set_key(ctx, in_key, key_len);
112 kernel_fpu_end();
113 return 0;
114 }
115
aes_set_key(struct crypto_tfm * tfm,const u8 * in_key,unsigned int key_len)116 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
117 unsigned int key_len)
118 {
119 return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key,
120 key_len);
121 }
122
aesni_encrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)123 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
124 {
125 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
126
127 if (!crypto_simd_usable()) {
128 aes_encrypt(ctx, dst, src);
129 } else {
130 kernel_fpu_begin();
131 aesni_enc(ctx, dst, src);
132 kernel_fpu_end();
133 }
134 }
135
aesni_decrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)136 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
137 {
138 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
139
140 if (!crypto_simd_usable()) {
141 aes_decrypt(ctx, dst, src);
142 } else {
143 kernel_fpu_begin();
144 aesni_dec(ctx, dst, src);
145 kernel_fpu_end();
146 }
147 }
148
aesni_skcipher_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int len)149 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
150 unsigned int len)
151 {
152 return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
153 }
154
ecb_encrypt(struct skcipher_request * req)155 static int ecb_encrypt(struct skcipher_request *req)
156 {
157 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
158 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
159 struct skcipher_walk walk;
160 unsigned int nbytes;
161 int err;
162
163 err = skcipher_walk_virt(&walk, req, false);
164
165 while ((nbytes = walk.nbytes)) {
166 kernel_fpu_begin();
167 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
168 nbytes & AES_BLOCK_MASK);
169 kernel_fpu_end();
170 nbytes &= AES_BLOCK_SIZE - 1;
171 err = skcipher_walk_done(&walk, nbytes);
172 }
173
174 return err;
175 }
176
ecb_decrypt(struct skcipher_request * req)177 static int ecb_decrypt(struct skcipher_request *req)
178 {
179 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
180 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
181 struct skcipher_walk walk;
182 unsigned int nbytes;
183 int err;
184
185 err = skcipher_walk_virt(&walk, req, false);
186
187 while ((nbytes = walk.nbytes)) {
188 kernel_fpu_begin();
189 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
190 nbytes & AES_BLOCK_MASK);
191 kernel_fpu_end();
192 nbytes &= AES_BLOCK_SIZE - 1;
193 err = skcipher_walk_done(&walk, nbytes);
194 }
195
196 return err;
197 }
198
cbc_encrypt(struct skcipher_request * req)199 static int cbc_encrypt(struct skcipher_request *req)
200 {
201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
202 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
203 struct skcipher_walk walk;
204 unsigned int nbytes;
205 int err;
206
207 err = skcipher_walk_virt(&walk, req, false);
208
209 while ((nbytes = walk.nbytes)) {
210 kernel_fpu_begin();
211 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
212 nbytes & AES_BLOCK_MASK, walk.iv);
213 kernel_fpu_end();
214 nbytes &= AES_BLOCK_SIZE - 1;
215 err = skcipher_walk_done(&walk, nbytes);
216 }
217
218 return err;
219 }
220
cbc_decrypt(struct skcipher_request * req)221 static int cbc_decrypt(struct skcipher_request *req)
222 {
223 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
224 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
225 struct skcipher_walk walk;
226 unsigned int nbytes;
227 int err;
228
229 err = skcipher_walk_virt(&walk, req, false);
230
231 while ((nbytes = walk.nbytes)) {
232 kernel_fpu_begin();
233 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
234 nbytes & AES_BLOCK_MASK, walk.iv);
235 kernel_fpu_end();
236 nbytes &= AES_BLOCK_SIZE - 1;
237 err = skcipher_walk_done(&walk, nbytes);
238 }
239
240 return err;
241 }
242
cts_cbc_encrypt(struct skcipher_request * req)243 static int cts_cbc_encrypt(struct skcipher_request *req)
244 {
245 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
246 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
247 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
248 struct scatterlist *src = req->src, *dst = req->dst;
249 struct scatterlist sg_src[2], sg_dst[2];
250 struct skcipher_request subreq;
251 struct skcipher_walk walk;
252 int err;
253
254 skcipher_request_set_tfm(&subreq, tfm);
255 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
256 NULL, NULL);
257
258 if (req->cryptlen <= AES_BLOCK_SIZE) {
259 if (req->cryptlen < AES_BLOCK_SIZE)
260 return -EINVAL;
261 cbc_blocks = 1;
262 }
263
264 if (cbc_blocks > 0) {
265 skcipher_request_set_crypt(&subreq, req->src, req->dst,
266 cbc_blocks * AES_BLOCK_SIZE,
267 req->iv);
268
269 err = cbc_encrypt(&subreq);
270 if (err)
271 return err;
272
273 if (req->cryptlen == AES_BLOCK_SIZE)
274 return 0;
275
276 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
277 if (req->dst != req->src)
278 dst = scatterwalk_ffwd(sg_dst, req->dst,
279 subreq.cryptlen);
280 }
281
282 /* handle ciphertext stealing */
283 skcipher_request_set_crypt(&subreq, src, dst,
284 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
285 req->iv);
286
287 err = skcipher_walk_virt(&walk, &subreq, false);
288 if (err)
289 return err;
290
291 kernel_fpu_begin();
292 aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
293 walk.nbytes, walk.iv);
294 kernel_fpu_end();
295
296 return skcipher_walk_done(&walk, 0);
297 }
298
cts_cbc_decrypt(struct skcipher_request * req)299 static int cts_cbc_decrypt(struct skcipher_request *req)
300 {
301 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
302 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
303 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
304 struct scatterlist *src = req->src, *dst = req->dst;
305 struct scatterlist sg_src[2], sg_dst[2];
306 struct skcipher_request subreq;
307 struct skcipher_walk walk;
308 int err;
309
310 skcipher_request_set_tfm(&subreq, tfm);
311 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
312 NULL, NULL);
313
314 if (req->cryptlen <= AES_BLOCK_SIZE) {
315 if (req->cryptlen < AES_BLOCK_SIZE)
316 return -EINVAL;
317 cbc_blocks = 1;
318 }
319
320 if (cbc_blocks > 0) {
321 skcipher_request_set_crypt(&subreq, req->src, req->dst,
322 cbc_blocks * AES_BLOCK_SIZE,
323 req->iv);
324
325 err = cbc_decrypt(&subreq);
326 if (err)
327 return err;
328
329 if (req->cryptlen == AES_BLOCK_SIZE)
330 return 0;
331
332 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
333 if (req->dst != req->src)
334 dst = scatterwalk_ffwd(sg_dst, req->dst,
335 subreq.cryptlen);
336 }
337
338 /* handle ciphertext stealing */
339 skcipher_request_set_crypt(&subreq, src, dst,
340 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
341 req->iv);
342
343 err = skcipher_walk_virt(&walk, &subreq, false);
344 if (err)
345 return err;
346
347 kernel_fpu_begin();
348 aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
349 walk.nbytes, walk.iv);
350 kernel_fpu_end();
351
352 return skcipher_walk_done(&walk, 0);
353 }
354
355 #ifdef CONFIG_X86_64
356 /* This is the non-AVX version. */
ctr_crypt_aesni(struct skcipher_request * req)357 static int ctr_crypt_aesni(struct skcipher_request *req)
358 {
359 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
360 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
361 u8 keystream[AES_BLOCK_SIZE];
362 struct skcipher_walk walk;
363 unsigned int nbytes;
364 int err;
365
366 err = skcipher_walk_virt(&walk, req, false);
367
368 while ((nbytes = walk.nbytes) > 0) {
369 kernel_fpu_begin();
370 if (nbytes & AES_BLOCK_MASK)
371 aesni_ctr_enc(ctx, walk.dst.virt.addr,
372 walk.src.virt.addr,
373 nbytes & AES_BLOCK_MASK, walk.iv);
374 nbytes &= ~AES_BLOCK_MASK;
375
376 if (walk.nbytes == walk.total && nbytes > 0) {
377 aesni_enc(ctx, keystream, walk.iv);
378 crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
379 walk.src.virt.addr + walk.nbytes - nbytes,
380 keystream, nbytes);
381 crypto_inc(walk.iv, AES_BLOCK_SIZE);
382 nbytes = 0;
383 }
384 kernel_fpu_end();
385 err = skcipher_walk_done(&walk, nbytes);
386 }
387 return err;
388 }
389 #endif
390
xts_setkey_aesni(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)391 static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
392 unsigned int keylen)
393 {
394 struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
395 int err;
396
397 err = xts_verify_key(tfm, key, keylen);
398 if (err)
399 return err;
400
401 keylen /= 2;
402
403 /* first half of xts-key is for crypt */
404 err = aes_set_key_common(&ctx->crypt_ctx, key, keylen);
405 if (err)
406 return err;
407
408 /* second half of xts-key is for tweak */
409 return aes_set_key_common(&ctx->tweak_ctx, key + keylen, keylen);
410 }
411
412 typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key,
413 u8 iv[AES_BLOCK_SIZE]);
414 typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
415 const u8 *src, u8 *dst, int len,
416 u8 tweak[AES_BLOCK_SIZE]);
417
418 /* This handles cases where the source and/or destination span pages. */
419 static noinline int
xts_crypt_slowpath(struct skcipher_request * req,xts_crypt_func crypt_func)420 xts_crypt_slowpath(struct skcipher_request *req, xts_crypt_func crypt_func)
421 {
422 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
423 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
424 int tail = req->cryptlen % AES_BLOCK_SIZE;
425 struct scatterlist sg_src[2], sg_dst[2];
426 struct skcipher_request subreq;
427 struct skcipher_walk walk;
428 struct scatterlist *src, *dst;
429 int err;
430
431 /*
432 * If the message length isn't divisible by the AES block size, then
433 * separate off the last full block and the partial block. This ensures
434 * that they are processed in the same call to the assembly function,
435 * which is required for ciphertext stealing.
436 */
437 if (tail) {
438 skcipher_request_set_tfm(&subreq, tfm);
439 skcipher_request_set_callback(&subreq,
440 skcipher_request_flags(req),
441 NULL, NULL);
442 skcipher_request_set_crypt(&subreq, req->src, req->dst,
443 req->cryptlen - tail - AES_BLOCK_SIZE,
444 req->iv);
445 req = &subreq;
446 }
447
448 err = skcipher_walk_virt(&walk, req, false);
449
450 while (walk.nbytes) {
451 kernel_fpu_begin();
452 (*crypt_func)(&ctx->crypt_ctx,
453 walk.src.virt.addr, walk.dst.virt.addr,
454 walk.nbytes & ~(AES_BLOCK_SIZE - 1), req->iv);
455 kernel_fpu_end();
456 err = skcipher_walk_done(&walk,
457 walk.nbytes & (AES_BLOCK_SIZE - 1));
458 }
459
460 if (err || !tail)
461 return err;
462
463 /* Do ciphertext stealing with the last full block and partial block. */
464
465 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
466 if (req->dst != req->src)
467 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
468
469 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
470 req->iv);
471
472 err = skcipher_walk_virt(&walk, req, false);
473 if (err)
474 return err;
475
476 kernel_fpu_begin();
477 (*crypt_func)(&ctx->crypt_ctx, walk.src.virt.addr, walk.dst.virt.addr,
478 walk.nbytes, req->iv);
479 kernel_fpu_end();
480
481 return skcipher_walk_done(&walk, 0);
482 }
483
484 /* __always_inline to avoid indirect call in fastpath */
485 static __always_inline int
xts_crypt(struct skcipher_request * req,xts_encrypt_iv_func encrypt_iv,xts_crypt_func crypt_func)486 xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
487 xts_crypt_func crypt_func)
488 {
489 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
490 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
491
492 if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
493 return -EINVAL;
494
495 kernel_fpu_begin();
496 (*encrypt_iv)(&ctx->tweak_ctx, req->iv);
497
498 /*
499 * In practice, virtually all XTS plaintexts and ciphertexts are either
500 * 512 or 4096 bytes and do not use multiple scatterlist elements. To
501 * optimize the performance of these cases, the below fast-path handles
502 * single-scatterlist-element messages as efficiently as possible. The
503 * code is 64-bit specific, as it assumes no page mapping is needed.
504 */
505 if (IS_ENABLED(CONFIG_X86_64) &&
506 likely(req->src->length >= req->cryptlen &&
507 req->dst->length >= req->cryptlen)) {
508 (*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src),
509 sg_virt(req->dst), req->cryptlen, req->iv);
510 kernel_fpu_end();
511 return 0;
512 }
513 kernel_fpu_end();
514 return xts_crypt_slowpath(req, crypt_func);
515 }
516
aesni_xts_encrypt_iv(const struct crypto_aes_ctx * tweak_key,u8 iv[AES_BLOCK_SIZE])517 static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
518 u8 iv[AES_BLOCK_SIZE])
519 {
520 aesni_enc(tweak_key, iv, iv);
521 }
522
aesni_xts_encrypt(const struct crypto_aes_ctx * key,const u8 * src,u8 * dst,int len,u8 tweak[AES_BLOCK_SIZE])523 static void aesni_xts_encrypt(const struct crypto_aes_ctx *key,
524 const u8 *src, u8 *dst, int len,
525 u8 tweak[AES_BLOCK_SIZE])
526 {
527 aesni_xts_enc(key, dst, src, len, tweak);
528 }
529
aesni_xts_decrypt(const struct crypto_aes_ctx * key,const u8 * src,u8 * dst,int len,u8 tweak[AES_BLOCK_SIZE])530 static void aesni_xts_decrypt(const struct crypto_aes_ctx *key,
531 const u8 *src, u8 *dst, int len,
532 u8 tweak[AES_BLOCK_SIZE])
533 {
534 aesni_xts_dec(key, dst, src, len, tweak);
535 }
536
xts_encrypt_aesni(struct skcipher_request * req)537 static int xts_encrypt_aesni(struct skcipher_request *req)
538 {
539 return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_encrypt);
540 }
541
xts_decrypt_aesni(struct skcipher_request * req)542 static int xts_decrypt_aesni(struct skcipher_request *req)
543 {
544 return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_decrypt);
545 }
546
547 static struct crypto_alg aesni_cipher_alg = {
548 .cra_name = "aes",
549 .cra_driver_name = "aes-aesni",
550 .cra_priority = 300,
551 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
552 .cra_blocksize = AES_BLOCK_SIZE,
553 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
554 .cra_module = THIS_MODULE,
555 .cra_u = {
556 .cipher = {
557 .cia_min_keysize = AES_MIN_KEY_SIZE,
558 .cia_max_keysize = AES_MAX_KEY_SIZE,
559 .cia_setkey = aes_set_key,
560 .cia_encrypt = aesni_encrypt,
561 .cia_decrypt = aesni_decrypt
562 }
563 }
564 };
565
566 static struct skcipher_alg aesni_skciphers[] = {
567 {
568 .base = {
569 .cra_name = "ecb(aes)",
570 .cra_driver_name = "ecb-aes-aesni",
571 .cra_priority = 400,
572 .cra_blocksize = AES_BLOCK_SIZE,
573 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
574 .cra_module = THIS_MODULE,
575 },
576 .min_keysize = AES_MIN_KEY_SIZE,
577 .max_keysize = AES_MAX_KEY_SIZE,
578 .setkey = aesni_skcipher_setkey,
579 .encrypt = ecb_encrypt,
580 .decrypt = ecb_decrypt,
581 }, {
582 .base = {
583 .cra_name = "cbc(aes)",
584 .cra_driver_name = "cbc-aes-aesni",
585 .cra_priority = 400,
586 .cra_blocksize = AES_BLOCK_SIZE,
587 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
588 .cra_module = THIS_MODULE,
589 },
590 .min_keysize = AES_MIN_KEY_SIZE,
591 .max_keysize = AES_MAX_KEY_SIZE,
592 .ivsize = AES_BLOCK_SIZE,
593 .setkey = aesni_skcipher_setkey,
594 .encrypt = cbc_encrypt,
595 .decrypt = cbc_decrypt,
596 }, {
597 .base = {
598 .cra_name = "cts(cbc(aes))",
599 .cra_driver_name = "cts-cbc-aes-aesni",
600 .cra_priority = 400,
601 .cra_blocksize = AES_BLOCK_SIZE,
602 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
603 .cra_module = THIS_MODULE,
604 },
605 .min_keysize = AES_MIN_KEY_SIZE,
606 .max_keysize = AES_MAX_KEY_SIZE,
607 .ivsize = AES_BLOCK_SIZE,
608 .walksize = 2 * AES_BLOCK_SIZE,
609 .setkey = aesni_skcipher_setkey,
610 .encrypt = cts_cbc_encrypt,
611 .decrypt = cts_cbc_decrypt,
612 #ifdef CONFIG_X86_64
613 }, {
614 .base = {
615 .cra_name = "ctr(aes)",
616 .cra_driver_name = "ctr-aes-aesni",
617 .cra_priority = 400,
618 .cra_blocksize = 1,
619 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
620 .cra_module = THIS_MODULE,
621 },
622 .min_keysize = AES_MIN_KEY_SIZE,
623 .max_keysize = AES_MAX_KEY_SIZE,
624 .ivsize = AES_BLOCK_SIZE,
625 .chunksize = AES_BLOCK_SIZE,
626 .setkey = aesni_skcipher_setkey,
627 .encrypt = ctr_crypt_aesni,
628 .decrypt = ctr_crypt_aesni,
629 #endif
630 }, {
631 .base = {
632 .cra_name = "xts(aes)",
633 .cra_driver_name = "xts-aes-aesni",
634 .cra_priority = 401,
635 .cra_blocksize = AES_BLOCK_SIZE,
636 .cra_ctxsize = XTS_AES_CTX_SIZE,
637 .cra_module = THIS_MODULE,
638 },
639 .min_keysize = 2 * AES_MIN_KEY_SIZE,
640 .max_keysize = 2 * AES_MAX_KEY_SIZE,
641 .ivsize = AES_BLOCK_SIZE,
642 .walksize = 2 * AES_BLOCK_SIZE,
643 .setkey = xts_setkey_aesni,
644 .encrypt = xts_encrypt_aesni,
645 .decrypt = xts_decrypt_aesni,
646 }
647 };
648
649 #ifdef CONFIG_X86_64
650 asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
651 u8 iv[AES_BLOCK_SIZE]);
652
653 /* __always_inline to avoid indirect call */
654 static __always_inline int
ctr_crypt(struct skcipher_request * req,void (* ctr64_func)(const struct crypto_aes_ctx * key,const u8 * src,u8 * dst,int len,const u64 le_ctr[2]))655 ctr_crypt(struct skcipher_request *req,
656 void (*ctr64_func)(const struct crypto_aes_ctx *key,
657 const u8 *src, u8 *dst, int len,
658 const u64 le_ctr[2]))
659 {
660 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
661 const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
662 unsigned int nbytes, p1_nbytes, nblocks;
663 struct skcipher_walk walk;
664 u64 le_ctr[2];
665 u64 ctr64;
666 int err;
667
668 ctr64 = le_ctr[0] = get_unaligned_be64(&req->iv[8]);
669 le_ctr[1] = get_unaligned_be64(&req->iv[0]);
670
671 err = skcipher_walk_virt(&walk, req, false);
672
673 while ((nbytes = walk.nbytes) != 0) {
674 if (nbytes < walk.total) {
675 /* Not the end yet, so keep the length block-aligned. */
676 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
677 nblocks = nbytes / AES_BLOCK_SIZE;
678 } else {
679 /* It's the end, so include any final partial block. */
680 nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
681 }
682 ctr64 += nblocks;
683
684 kernel_fpu_begin();
685 if (likely(ctr64 >= nblocks)) {
686 /* The low 64 bits of the counter won't overflow. */
687 (*ctr64_func)(key, walk.src.virt.addr,
688 walk.dst.virt.addr, nbytes, le_ctr);
689 } else {
690 /*
691 * The low 64 bits of the counter will overflow. The
692 * assembly doesn't handle this case, so split the
693 * operation into two at the point where the overflow
694 * will occur. After the first part, add the carry bit.
695 */
696 p1_nbytes = min_t(unsigned int, nbytes,
697 (nblocks - ctr64) * AES_BLOCK_SIZE);
698 (*ctr64_func)(key, walk.src.virt.addr,
699 walk.dst.virt.addr, p1_nbytes, le_ctr);
700 le_ctr[0] = 0;
701 le_ctr[1]++;
702 (*ctr64_func)(key, walk.src.virt.addr + p1_nbytes,
703 walk.dst.virt.addr + p1_nbytes,
704 nbytes - p1_nbytes, le_ctr);
705 }
706 kernel_fpu_end();
707 le_ctr[0] = ctr64;
708
709 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
710 }
711
712 put_unaligned_be64(ctr64, &req->iv[8]);
713 put_unaligned_be64(le_ctr[1], &req->iv[0]);
714
715 return err;
716 }
717
718 /* __always_inline to avoid indirect call */
719 static __always_inline int
xctr_crypt(struct skcipher_request * req,void (* xctr_func)(const struct crypto_aes_ctx * key,const u8 * src,u8 * dst,int len,const u8 iv[AES_BLOCK_SIZE],u64 ctr))720 xctr_crypt(struct skcipher_request *req,
721 void (*xctr_func)(const struct crypto_aes_ctx *key,
722 const u8 *src, u8 *dst, int len,
723 const u8 iv[AES_BLOCK_SIZE], u64 ctr))
724 {
725 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
726 const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
727 struct skcipher_walk walk;
728 unsigned int nbytes;
729 u64 ctr = 1;
730 int err;
731
732 err = skcipher_walk_virt(&walk, req, false);
733 while ((nbytes = walk.nbytes) != 0) {
734 if (nbytes < walk.total)
735 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
736
737 kernel_fpu_begin();
738 (*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr,
739 nbytes, req->iv, ctr);
740 kernel_fpu_end();
741
742 ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
743 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
744 }
745 return err;
746 }
747
748 #define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority) \
749 \
750 asmlinkage void \
751 aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
752 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
753 asmlinkage void \
754 aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
755 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
756 \
757 static int xts_encrypt_##suffix(struct skcipher_request *req) \
758 { \
759 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_encrypt_##suffix); \
760 } \
761 \
762 static int xts_decrypt_##suffix(struct skcipher_request *req) \
763 { \
764 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix); \
765 } \
766 \
767 asmlinkage void \
768 aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key, \
769 const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\
770 \
771 static int ctr_crypt_##suffix(struct skcipher_request *req) \
772 { \
773 return ctr_crypt(req, aes_ctr64_crypt_##suffix); \
774 } \
775 \
776 asmlinkage void \
777 aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key, \
778 const u8 *src, u8 *dst, int len, \
779 const u8 iv[AES_BLOCK_SIZE], u64 ctr); \
780 \
781 static int xctr_crypt_##suffix(struct skcipher_request *req) \
782 { \
783 return xctr_crypt(req, aes_xctr_crypt_##suffix); \
784 } \
785 \
786 static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
787 .base.cra_name = "xts(aes)", \
788 .base.cra_driver_name = "xts-aes-" driver_name_suffix, \
789 .base.cra_priority = priority, \
790 .base.cra_blocksize = AES_BLOCK_SIZE, \
791 .base.cra_ctxsize = XTS_AES_CTX_SIZE, \
792 .base.cra_module = THIS_MODULE, \
793 .min_keysize = 2 * AES_MIN_KEY_SIZE, \
794 .max_keysize = 2 * AES_MAX_KEY_SIZE, \
795 .ivsize = AES_BLOCK_SIZE, \
796 .walksize = 2 * AES_BLOCK_SIZE, \
797 .setkey = xts_setkey_aesni, \
798 .encrypt = xts_encrypt_##suffix, \
799 .decrypt = xts_decrypt_##suffix, \
800 }, { \
801 .base.cra_name = "ctr(aes)", \
802 .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \
803 .base.cra_priority = priority, \
804 .base.cra_blocksize = 1, \
805 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
806 .base.cra_module = THIS_MODULE, \
807 .min_keysize = AES_MIN_KEY_SIZE, \
808 .max_keysize = AES_MAX_KEY_SIZE, \
809 .ivsize = AES_BLOCK_SIZE, \
810 .chunksize = AES_BLOCK_SIZE, \
811 .setkey = aesni_skcipher_setkey, \
812 .encrypt = ctr_crypt_##suffix, \
813 .decrypt = ctr_crypt_##suffix, \
814 }, { \
815 .base.cra_name = "xctr(aes)", \
816 .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \
817 .base.cra_priority = priority, \
818 .base.cra_blocksize = 1, \
819 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
820 .base.cra_module = THIS_MODULE, \
821 .min_keysize = AES_MIN_KEY_SIZE, \
822 .max_keysize = AES_MAX_KEY_SIZE, \
823 .ivsize = AES_BLOCK_SIZE, \
824 .chunksize = AES_BLOCK_SIZE, \
825 .setkey = aesni_skcipher_setkey, \
826 .encrypt = xctr_crypt_##suffix, \
827 .decrypt = xctr_crypt_##suffix, \
828 }}
829
830 DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
831 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
832 DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600);
833 DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800);
834 #endif
835
836 /* The common part of the x86_64 AES-GCM key struct */
837 struct aes_gcm_key {
838 /* Expanded AES key and the AES key length in bytes */
839 struct crypto_aes_ctx aes_key;
840
841 /* RFC4106 nonce (used only by the rfc4106 algorithms) */
842 u32 rfc4106_nonce;
843 };
844
845 /* Key struct used by the AES-NI implementations of AES-GCM */
846 struct aes_gcm_key_aesni {
847 /*
848 * Common part of the key. The assembly code requires 16-byte alignment
849 * for the round keys; we get this by them being located at the start of
850 * the struct and the whole struct being 16-byte aligned.
851 */
852 struct aes_gcm_key base;
853
854 /*
855 * Powers of the hash key H^8 through H^1. These are 128-bit values.
856 * They all have an extra factor of x^-1 and are byte-reversed. 16-byte
857 * alignment is required by the assembly code.
858 */
859 u64 h_powers[8][2] __aligned(16);
860
861 /*
862 * h_powers_xored[i] contains the two 64-bit halves of h_powers[i] XOR'd
863 * together. It's used for Karatsuba multiplication. 16-byte alignment
864 * is required by the assembly code.
865 */
866 u64 h_powers_xored[8] __aligned(16);
867
868 /*
869 * H^1 times x^64 (and also the usual extra factor of x^-1). 16-byte
870 * alignment is required by the assembly code.
871 */
872 u64 h_times_x64[2] __aligned(16);
873 };
874 #define AES_GCM_KEY_AESNI(key) \
875 container_of((key), struct aes_gcm_key_aesni, base)
876 #define AES_GCM_KEY_AESNI_SIZE \
877 (sizeof(struct aes_gcm_key_aesni) + (15 & ~(CRYPTO_MINALIGN - 1)))
878
879 /* Key struct used by the VAES + AVX10 implementations of AES-GCM */
880 struct aes_gcm_key_avx10 {
881 /*
882 * Common part of the key. The assembly code prefers 16-byte alignment
883 * for the round keys; we get this by them being located at the start of
884 * the struct and the whole struct being 64-byte aligned.
885 */
886 struct aes_gcm_key base;
887
888 /*
889 * Powers of the hash key H^16 through H^1. These are 128-bit values.
890 * They all have an extra factor of x^-1 and are byte-reversed. This
891 * array is aligned to a 64-byte boundary to make it naturally aligned
892 * for 512-bit loads, which can improve performance. (The assembly code
893 * doesn't *need* the alignment; this is just an optimization.)
894 */
895 u64 h_powers[16][2] __aligned(64);
896
897 /* Three padding blocks required by the assembly code */
898 u64 padding[3][2];
899 };
900 #define AES_GCM_KEY_AVX10(key) \
901 container_of((key), struct aes_gcm_key_avx10, base)
902 #define AES_GCM_KEY_AVX10_SIZE \
903 (sizeof(struct aes_gcm_key_avx10) + (63 & ~(CRYPTO_MINALIGN - 1)))
904
905 /*
906 * These flags are passed to the AES-GCM helper functions to specify the
907 * specific version of AES-GCM (RFC4106 or not), whether it's encryption or
908 * decryption, and which assembly functions should be called. Assembly
909 * functions are selected using flags instead of function pointers to avoid
910 * indirect calls (which are very expensive on x86) regardless of inlining.
911 */
912 #define FLAG_RFC4106 BIT(0)
913 #define FLAG_ENC BIT(1)
914 #define FLAG_AVX BIT(2)
915 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
916 # define FLAG_AVX10_256 BIT(3)
917 # define FLAG_AVX10_512 BIT(4)
918 #else
919 /*
920 * This should cause all calls to the AVX10 assembly functions to be
921 * optimized out, avoiding the need to ifdef each call individually.
922 */
923 # define FLAG_AVX10_256 0
924 # define FLAG_AVX10_512 0
925 #endif
926
927 static inline struct aes_gcm_key *
aes_gcm_key_get(struct crypto_aead * tfm,int flags)928 aes_gcm_key_get(struct crypto_aead *tfm, int flags)
929 {
930 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
931 return PTR_ALIGN(crypto_aead_ctx(tfm), 64);
932 else
933 return PTR_ALIGN(crypto_aead_ctx(tfm), 16);
934 }
935
936 asmlinkage void
937 aes_gcm_precompute_aesni(struct aes_gcm_key_aesni *key);
938 asmlinkage void
939 aes_gcm_precompute_aesni_avx(struct aes_gcm_key_aesni *key);
940 asmlinkage void
941 aes_gcm_precompute_vaes_avx10_256(struct aes_gcm_key_avx10 *key);
942 asmlinkage void
943 aes_gcm_precompute_vaes_avx10_512(struct aes_gcm_key_avx10 *key);
944
aes_gcm_precompute(struct aes_gcm_key * key,int flags)945 static void aes_gcm_precompute(struct aes_gcm_key *key, int flags)
946 {
947 /*
948 * To make things a bit easier on the assembly side, the AVX10
949 * implementations use the same key format. Therefore, a single
950 * function using 256-bit vectors would suffice here. However, it's
951 * straightforward to provide a 512-bit one because of how the assembly
952 * code is structured, and it works nicely because the total size of the
953 * key powers is a multiple of 512 bits. So we take advantage of that.
954 *
955 * A similar situation applies to the AES-NI implementations.
956 */
957 if (flags & FLAG_AVX10_512)
958 aes_gcm_precompute_vaes_avx10_512(AES_GCM_KEY_AVX10(key));
959 else if (flags & FLAG_AVX10_256)
960 aes_gcm_precompute_vaes_avx10_256(AES_GCM_KEY_AVX10(key));
961 else if (flags & FLAG_AVX)
962 aes_gcm_precompute_aesni_avx(AES_GCM_KEY_AESNI(key));
963 else
964 aes_gcm_precompute_aesni(AES_GCM_KEY_AESNI(key));
965 }
966
967 asmlinkage void
968 aes_gcm_aad_update_aesni(const struct aes_gcm_key_aesni *key,
969 u8 ghash_acc[16], const u8 *aad, int aadlen);
970 asmlinkage void
971 aes_gcm_aad_update_aesni_avx(const struct aes_gcm_key_aesni *key,
972 u8 ghash_acc[16], const u8 *aad, int aadlen);
973 asmlinkage void
974 aes_gcm_aad_update_vaes_avx10(const struct aes_gcm_key_avx10 *key,
975 u8 ghash_acc[16], const u8 *aad, int aadlen);
976
aes_gcm_aad_update(const struct aes_gcm_key * key,u8 ghash_acc[16],const u8 * aad,int aadlen,int flags)977 static void aes_gcm_aad_update(const struct aes_gcm_key *key, u8 ghash_acc[16],
978 const u8 *aad, int aadlen, int flags)
979 {
980 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
981 aes_gcm_aad_update_vaes_avx10(AES_GCM_KEY_AVX10(key), ghash_acc,
982 aad, aadlen);
983 else if (flags & FLAG_AVX)
984 aes_gcm_aad_update_aesni_avx(AES_GCM_KEY_AESNI(key), ghash_acc,
985 aad, aadlen);
986 else
987 aes_gcm_aad_update_aesni(AES_GCM_KEY_AESNI(key), ghash_acc,
988 aad, aadlen);
989 }
990
991 asmlinkage void
992 aes_gcm_enc_update_aesni(const struct aes_gcm_key_aesni *key,
993 const u32 le_ctr[4], u8 ghash_acc[16],
994 const u8 *src, u8 *dst, int datalen);
995 asmlinkage void
996 aes_gcm_enc_update_aesni_avx(const struct aes_gcm_key_aesni *key,
997 const u32 le_ctr[4], u8 ghash_acc[16],
998 const u8 *src, u8 *dst, int datalen);
999 asmlinkage void
1000 aes_gcm_enc_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key,
1001 const u32 le_ctr[4], u8 ghash_acc[16],
1002 const u8 *src, u8 *dst, int datalen);
1003 asmlinkage void
1004 aes_gcm_enc_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key,
1005 const u32 le_ctr[4], u8 ghash_acc[16],
1006 const u8 *src, u8 *dst, int datalen);
1007
1008 asmlinkage void
1009 aes_gcm_dec_update_aesni(const struct aes_gcm_key_aesni *key,
1010 const u32 le_ctr[4], u8 ghash_acc[16],
1011 const u8 *src, u8 *dst, int datalen);
1012 asmlinkage void
1013 aes_gcm_dec_update_aesni_avx(const struct aes_gcm_key_aesni *key,
1014 const u32 le_ctr[4], u8 ghash_acc[16],
1015 const u8 *src, u8 *dst, int datalen);
1016 asmlinkage void
1017 aes_gcm_dec_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key,
1018 const u32 le_ctr[4], u8 ghash_acc[16],
1019 const u8 *src, u8 *dst, int datalen);
1020 asmlinkage void
1021 aes_gcm_dec_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key,
1022 const u32 le_ctr[4], u8 ghash_acc[16],
1023 const u8 *src, u8 *dst, int datalen);
1024
1025 /* __always_inline to optimize out the branches based on @flags */
1026 static __always_inline void
aes_gcm_update(const struct aes_gcm_key * key,const u32 le_ctr[4],u8 ghash_acc[16],const u8 * src,u8 * dst,int datalen,int flags)1027 aes_gcm_update(const struct aes_gcm_key *key,
1028 const u32 le_ctr[4], u8 ghash_acc[16],
1029 const u8 *src, u8 *dst, int datalen, int flags)
1030 {
1031 if (flags & FLAG_ENC) {
1032 if (flags & FLAG_AVX10_512)
1033 aes_gcm_enc_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key),
1034 le_ctr, ghash_acc,
1035 src, dst, datalen);
1036 else if (flags & FLAG_AVX10_256)
1037 aes_gcm_enc_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key),
1038 le_ctr, ghash_acc,
1039 src, dst, datalen);
1040 else if (flags & FLAG_AVX)
1041 aes_gcm_enc_update_aesni_avx(AES_GCM_KEY_AESNI(key),
1042 le_ctr, ghash_acc,
1043 src, dst, datalen);
1044 else
1045 aes_gcm_enc_update_aesni(AES_GCM_KEY_AESNI(key), le_ctr,
1046 ghash_acc, src, dst, datalen);
1047 } else {
1048 if (flags & FLAG_AVX10_512)
1049 aes_gcm_dec_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key),
1050 le_ctr, ghash_acc,
1051 src, dst, datalen);
1052 else if (flags & FLAG_AVX10_256)
1053 aes_gcm_dec_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key),
1054 le_ctr, ghash_acc,
1055 src, dst, datalen);
1056 else if (flags & FLAG_AVX)
1057 aes_gcm_dec_update_aesni_avx(AES_GCM_KEY_AESNI(key),
1058 le_ctr, ghash_acc,
1059 src, dst, datalen);
1060 else
1061 aes_gcm_dec_update_aesni(AES_GCM_KEY_AESNI(key),
1062 le_ctr, ghash_acc,
1063 src, dst, datalen);
1064 }
1065 }
1066
1067 asmlinkage void
1068 aes_gcm_enc_final_aesni(const struct aes_gcm_key_aesni *key,
1069 const u32 le_ctr[4], u8 ghash_acc[16],
1070 u64 total_aadlen, u64 total_datalen);
1071 asmlinkage void
1072 aes_gcm_enc_final_aesni_avx(const struct aes_gcm_key_aesni *key,
1073 const u32 le_ctr[4], u8 ghash_acc[16],
1074 u64 total_aadlen, u64 total_datalen);
1075 asmlinkage void
1076 aes_gcm_enc_final_vaes_avx10(const struct aes_gcm_key_avx10 *key,
1077 const u32 le_ctr[4], u8 ghash_acc[16],
1078 u64 total_aadlen, u64 total_datalen);
1079
1080 /* __always_inline to optimize out the branches based on @flags */
1081 static __always_inline void
aes_gcm_enc_final(const struct aes_gcm_key * key,const u32 le_ctr[4],u8 ghash_acc[16],u64 total_aadlen,u64 total_datalen,int flags)1082 aes_gcm_enc_final(const struct aes_gcm_key *key,
1083 const u32 le_ctr[4], u8 ghash_acc[16],
1084 u64 total_aadlen, u64 total_datalen, int flags)
1085 {
1086 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
1087 aes_gcm_enc_final_vaes_avx10(AES_GCM_KEY_AVX10(key),
1088 le_ctr, ghash_acc,
1089 total_aadlen, total_datalen);
1090 else if (flags & FLAG_AVX)
1091 aes_gcm_enc_final_aesni_avx(AES_GCM_KEY_AESNI(key),
1092 le_ctr, ghash_acc,
1093 total_aadlen, total_datalen);
1094 else
1095 aes_gcm_enc_final_aesni(AES_GCM_KEY_AESNI(key),
1096 le_ctr, ghash_acc,
1097 total_aadlen, total_datalen);
1098 }
1099
1100 asmlinkage bool __must_check
1101 aes_gcm_dec_final_aesni(const struct aes_gcm_key_aesni *key,
1102 const u32 le_ctr[4], const u8 ghash_acc[16],
1103 u64 total_aadlen, u64 total_datalen,
1104 const u8 tag[16], int taglen);
1105 asmlinkage bool __must_check
1106 aes_gcm_dec_final_aesni_avx(const struct aes_gcm_key_aesni *key,
1107 const u32 le_ctr[4], const u8 ghash_acc[16],
1108 u64 total_aadlen, u64 total_datalen,
1109 const u8 tag[16], int taglen);
1110 asmlinkage bool __must_check
1111 aes_gcm_dec_final_vaes_avx10(const struct aes_gcm_key_avx10 *key,
1112 const u32 le_ctr[4], const u8 ghash_acc[16],
1113 u64 total_aadlen, u64 total_datalen,
1114 const u8 tag[16], int taglen);
1115
1116 /* __always_inline to optimize out the branches based on @flags */
1117 static __always_inline bool __must_check
aes_gcm_dec_final(const struct aes_gcm_key * key,const u32 le_ctr[4],u8 ghash_acc[16],u64 total_aadlen,u64 total_datalen,u8 tag[16],int taglen,int flags)1118 aes_gcm_dec_final(const struct aes_gcm_key *key, const u32 le_ctr[4],
1119 u8 ghash_acc[16], u64 total_aadlen, u64 total_datalen,
1120 u8 tag[16], int taglen, int flags)
1121 {
1122 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512))
1123 return aes_gcm_dec_final_vaes_avx10(AES_GCM_KEY_AVX10(key),
1124 le_ctr, ghash_acc,
1125 total_aadlen, total_datalen,
1126 tag, taglen);
1127 else if (flags & FLAG_AVX)
1128 return aes_gcm_dec_final_aesni_avx(AES_GCM_KEY_AESNI(key),
1129 le_ctr, ghash_acc,
1130 total_aadlen, total_datalen,
1131 tag, taglen);
1132 else
1133 return aes_gcm_dec_final_aesni(AES_GCM_KEY_AESNI(key),
1134 le_ctr, ghash_acc,
1135 total_aadlen, total_datalen,
1136 tag, taglen);
1137 }
1138
1139 /*
1140 * This is the Integrity Check Value (aka the authentication tag) length and can
1141 * be 8, 12 or 16 bytes long.
1142 */
common_rfc4106_set_authsize(struct crypto_aead * aead,unsigned int authsize)1143 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
1144 unsigned int authsize)
1145 {
1146 switch (authsize) {
1147 case 8:
1148 case 12:
1149 case 16:
1150 break;
1151 default:
1152 return -EINVAL;
1153 }
1154
1155 return 0;
1156 }
1157
generic_gcmaes_set_authsize(struct crypto_aead * tfm,unsigned int authsize)1158 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
1159 unsigned int authsize)
1160 {
1161 switch (authsize) {
1162 case 4:
1163 case 8:
1164 case 12:
1165 case 13:
1166 case 14:
1167 case 15:
1168 case 16:
1169 break;
1170 default:
1171 return -EINVAL;
1172 }
1173
1174 return 0;
1175 }
1176
1177 /*
1178 * This is the setkey function for the x86_64 implementations of AES-GCM. It
1179 * saves the RFC4106 nonce if applicable, expands the AES key, and precomputes
1180 * powers of the hash key.
1181 *
1182 * To comply with the crypto_aead API, this has to be usable in no-SIMD context.
1183 * For that reason, this function includes a portable C implementation of the
1184 * needed logic. However, the portable C implementation is very slow, taking
1185 * about the same time as encrypting 37 KB of data. To be ready for users that
1186 * may set a key even somewhat frequently, we therefore also include a SIMD
1187 * assembly implementation, expanding the AES key using AES-NI and precomputing
1188 * the hash key powers using PCLMULQDQ or VPCLMULQDQ.
1189 */
gcm_setkey(struct crypto_aead * tfm,const u8 * raw_key,unsigned int keylen,int flags)1190 static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
1191 unsigned int keylen, int flags)
1192 {
1193 struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags);
1194 int err;
1195
1196 if (flags & FLAG_RFC4106) {
1197 if (keylen < 4)
1198 return -EINVAL;
1199 keylen -= 4;
1200 key->rfc4106_nonce = get_unaligned_be32(raw_key + keylen);
1201 }
1202
1203 /* The assembly code assumes the following offsets. */
1204 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_enc) != 0);
1205 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_length) != 480);
1206 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496);
1207 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624);
1208 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688);
1209 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_enc) != 0);
1210 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_length) != 480);
1211 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, h_powers) != 512);
1212 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, padding) != 768);
1213
1214 if (likely(crypto_simd_usable())) {
1215 err = aes_check_keylen(keylen);
1216 if (err)
1217 return err;
1218 kernel_fpu_begin();
1219 aesni_set_key(&key->aes_key, raw_key, keylen);
1220 aes_gcm_precompute(key, flags);
1221 kernel_fpu_end();
1222 } else {
1223 static const u8 x_to_the_minus1[16] __aligned(__alignof__(be128)) = {
1224 [0] = 0xc2, [15] = 1
1225 };
1226 static const u8 x_to_the_63[16] __aligned(__alignof__(be128)) = {
1227 [7] = 1,
1228 };
1229 be128 h1 = {};
1230 be128 h;
1231 int i;
1232
1233 err = aes_expandkey(&key->aes_key, raw_key, keylen);
1234 if (err)
1235 return err;
1236
1237 /* Encrypt the all-zeroes block to get the hash key H^1 */
1238 aes_encrypt(&key->aes_key, (u8 *)&h1, (u8 *)&h1);
1239
1240 /* Compute H^1 * x^-1 */
1241 h = h1;
1242 gf128mul_lle(&h, (const be128 *)x_to_the_minus1);
1243
1244 /* Compute the needed key powers */
1245 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) {
1246 struct aes_gcm_key_avx10 *k = AES_GCM_KEY_AVX10(key);
1247
1248 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1249 k->h_powers[i][0] = be64_to_cpu(h.b);
1250 k->h_powers[i][1] = be64_to_cpu(h.a);
1251 gf128mul_lle(&h, &h1);
1252 }
1253 memset(k->padding, 0, sizeof(k->padding));
1254 } else {
1255 struct aes_gcm_key_aesni *k = AES_GCM_KEY_AESNI(key);
1256
1257 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1258 k->h_powers[i][0] = be64_to_cpu(h.b);
1259 k->h_powers[i][1] = be64_to_cpu(h.a);
1260 k->h_powers_xored[i] = k->h_powers[i][0] ^
1261 k->h_powers[i][1];
1262 gf128mul_lle(&h, &h1);
1263 }
1264 gf128mul_lle(&h1, (const be128 *)x_to_the_63);
1265 k->h_times_x64[0] = be64_to_cpu(h1.b);
1266 k->h_times_x64[1] = be64_to_cpu(h1.a);
1267 }
1268 }
1269 return 0;
1270 }
1271
1272 /*
1273 * Initialize @ghash_acc, then pass all @assoclen bytes of associated data
1274 * (a.k.a. additional authenticated data) from @sg_src through the GHASH update
1275 * assembly function. kernel_fpu_begin() must have already been called.
1276 */
gcm_process_assoc(const struct aes_gcm_key * key,u8 ghash_acc[16],struct scatterlist * sg_src,unsigned int assoclen,int flags)1277 static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
1278 struct scatterlist *sg_src, unsigned int assoclen,
1279 int flags)
1280 {
1281 struct scatter_walk walk;
1282 /*
1283 * The assembly function requires that the length of any non-last
1284 * segment of associated data be a multiple of 16 bytes, so this
1285 * function does the buffering needed to achieve that.
1286 */
1287 unsigned int pos = 0;
1288 u8 buf[16];
1289
1290 memset(ghash_acc, 0, 16);
1291 scatterwalk_start(&walk, sg_src);
1292
1293 while (assoclen) {
1294 unsigned int orig_len_this_step = scatterwalk_next(
1295 &walk, assoclen);
1296 unsigned int len_this_step = orig_len_this_step;
1297 unsigned int len;
1298 const u8 *src = walk.addr;
1299
1300 if (unlikely(pos)) {
1301 len = min(len_this_step, 16 - pos);
1302 memcpy(&buf[pos], src, len);
1303 pos += len;
1304 src += len;
1305 len_this_step -= len;
1306 if (pos < 16)
1307 goto next;
1308 aes_gcm_aad_update(key, ghash_acc, buf, 16, flags);
1309 pos = 0;
1310 }
1311 len = len_this_step;
1312 if (unlikely(assoclen)) /* Not the last segment yet? */
1313 len = round_down(len, 16);
1314 aes_gcm_aad_update(key, ghash_acc, src, len, flags);
1315 src += len;
1316 len_this_step -= len;
1317 if (unlikely(len_this_step)) {
1318 memcpy(buf, src, len_this_step);
1319 pos = len_this_step;
1320 }
1321 next:
1322 scatterwalk_done_src(&walk, orig_len_this_step);
1323 if (need_resched()) {
1324 kernel_fpu_end();
1325 kernel_fpu_begin();
1326 }
1327 assoclen -= orig_len_this_step;
1328 }
1329 if (unlikely(pos))
1330 aes_gcm_aad_update(key, ghash_acc, buf, pos, flags);
1331 }
1332
1333
1334 /* __always_inline to optimize out the branches based on @flags */
1335 static __always_inline int
gcm_crypt(struct aead_request * req,int flags)1336 gcm_crypt(struct aead_request *req, int flags)
1337 {
1338 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1339 const struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags);
1340 unsigned int assoclen = req->assoclen;
1341 struct skcipher_walk walk;
1342 unsigned int nbytes;
1343 u8 ghash_acc[16]; /* GHASH accumulator */
1344 u32 le_ctr[4]; /* Counter in little-endian format */
1345 int taglen;
1346 int err;
1347
1348 /* Initialize the counter and determine the associated data length. */
1349 le_ctr[0] = 2;
1350 if (flags & FLAG_RFC4106) {
1351 if (unlikely(assoclen != 16 && assoclen != 20))
1352 return -EINVAL;
1353 assoclen -= 8;
1354 le_ctr[1] = get_unaligned_be32(req->iv + 4);
1355 le_ctr[2] = get_unaligned_be32(req->iv + 0);
1356 le_ctr[3] = key->rfc4106_nonce; /* already byte-swapped */
1357 } else {
1358 le_ctr[1] = get_unaligned_be32(req->iv + 8);
1359 le_ctr[2] = get_unaligned_be32(req->iv + 4);
1360 le_ctr[3] = get_unaligned_be32(req->iv + 0);
1361 }
1362
1363 /* Begin walking through the plaintext or ciphertext. */
1364 if (flags & FLAG_ENC)
1365 err = skcipher_walk_aead_encrypt(&walk, req, false);
1366 else
1367 err = skcipher_walk_aead_decrypt(&walk, req, false);
1368 if (err)
1369 return err;
1370
1371 /*
1372 * Since the AES-GCM assembly code requires that at least three assembly
1373 * functions be called to process any message (this is needed to support
1374 * incremental updates cleanly), to reduce overhead we try to do all
1375 * three calls in the same kernel FPU section if possible. We close the
1376 * section and start a new one if there are multiple data segments or if
1377 * rescheduling is needed while processing the associated data.
1378 */
1379 kernel_fpu_begin();
1380
1381 /* Pass the associated data through GHASH. */
1382 gcm_process_assoc(key, ghash_acc, req->src, assoclen, flags);
1383
1384 /* En/decrypt the data and pass the ciphertext through GHASH. */
1385 while (unlikely((nbytes = walk.nbytes) < walk.total)) {
1386 /*
1387 * Non-last segment. In this case, the assembly function
1388 * requires that the length be a multiple of 16 (AES_BLOCK_SIZE)
1389 * bytes. The needed buffering of up to 16 bytes is handled by
1390 * the skcipher_walk. Here we just need to round down to a
1391 * multiple of 16.
1392 */
1393 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
1394 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr,
1395 walk.dst.virt.addr, nbytes, flags);
1396 le_ctr[0] += nbytes / AES_BLOCK_SIZE;
1397 kernel_fpu_end();
1398 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
1399 if (err)
1400 return err;
1401 kernel_fpu_begin();
1402 }
1403 /* Last segment: process all remaining data. */
1404 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr,
1405 walk.dst.virt.addr, nbytes, flags);
1406 /*
1407 * The low word of the counter isn't used by the finalize, so there's no
1408 * need to increment it here.
1409 */
1410
1411 /* Finalize */
1412 taglen = crypto_aead_authsize(tfm);
1413 if (flags & FLAG_ENC) {
1414 /* Finish computing the auth tag. */
1415 aes_gcm_enc_final(key, le_ctr, ghash_acc, assoclen,
1416 req->cryptlen, flags);
1417
1418 /* Store the computed auth tag in the dst scatterlist. */
1419 scatterwalk_map_and_copy(ghash_acc, req->dst, req->assoclen +
1420 req->cryptlen, taglen, 1);
1421 } else {
1422 unsigned int datalen = req->cryptlen - taglen;
1423 u8 tag[16];
1424
1425 /* Get the transmitted auth tag from the src scatterlist. */
1426 scatterwalk_map_and_copy(tag, req->src, req->assoclen + datalen,
1427 taglen, 0);
1428 /*
1429 * Finish computing the auth tag and compare it to the
1430 * transmitted one. The assembly function does the actual tag
1431 * comparison. Here, just check the boolean result.
1432 */
1433 if (!aes_gcm_dec_final(key, le_ctr, ghash_acc, assoclen,
1434 datalen, tag, taglen, flags))
1435 err = -EBADMSG;
1436 }
1437 kernel_fpu_end();
1438 if (nbytes)
1439 skcipher_walk_done(&walk, 0);
1440 return err;
1441 }
1442
1443 #define DEFINE_GCM_ALGS(suffix, flags, generic_driver_name, rfc_driver_name, \
1444 ctxsize, priority) \
1445 \
1446 static int gcm_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \
1447 unsigned int keylen) \
1448 { \
1449 return gcm_setkey(tfm, raw_key, keylen, (flags)); \
1450 } \
1451 \
1452 static int gcm_encrypt_##suffix(struct aead_request *req) \
1453 { \
1454 return gcm_crypt(req, (flags) | FLAG_ENC); \
1455 } \
1456 \
1457 static int gcm_decrypt_##suffix(struct aead_request *req) \
1458 { \
1459 return gcm_crypt(req, (flags)); \
1460 } \
1461 \
1462 static int rfc4106_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \
1463 unsigned int keylen) \
1464 { \
1465 return gcm_setkey(tfm, raw_key, keylen, (flags) | FLAG_RFC4106); \
1466 } \
1467 \
1468 static int rfc4106_encrypt_##suffix(struct aead_request *req) \
1469 { \
1470 return gcm_crypt(req, (flags) | FLAG_RFC4106 | FLAG_ENC); \
1471 } \
1472 \
1473 static int rfc4106_decrypt_##suffix(struct aead_request *req) \
1474 { \
1475 return gcm_crypt(req, (flags) | FLAG_RFC4106); \
1476 } \
1477 \
1478 static struct aead_alg aes_gcm_algs_##suffix[] = { { \
1479 .setkey = gcm_setkey_##suffix, \
1480 .setauthsize = generic_gcmaes_set_authsize, \
1481 .encrypt = gcm_encrypt_##suffix, \
1482 .decrypt = gcm_decrypt_##suffix, \
1483 .ivsize = GCM_AES_IV_SIZE, \
1484 .chunksize = AES_BLOCK_SIZE, \
1485 .maxauthsize = 16, \
1486 .base = { \
1487 .cra_name = "gcm(aes)", \
1488 .cra_driver_name = generic_driver_name, \
1489 .cra_priority = (priority), \
1490 .cra_blocksize = 1, \
1491 .cra_ctxsize = (ctxsize), \
1492 .cra_module = THIS_MODULE, \
1493 }, \
1494 }, { \
1495 .setkey = rfc4106_setkey_##suffix, \
1496 .setauthsize = common_rfc4106_set_authsize, \
1497 .encrypt = rfc4106_encrypt_##suffix, \
1498 .decrypt = rfc4106_decrypt_##suffix, \
1499 .ivsize = GCM_RFC4106_IV_SIZE, \
1500 .chunksize = AES_BLOCK_SIZE, \
1501 .maxauthsize = 16, \
1502 .base = { \
1503 .cra_name = "rfc4106(gcm(aes))", \
1504 .cra_driver_name = rfc_driver_name, \
1505 .cra_priority = (priority), \
1506 .cra_blocksize = 1, \
1507 .cra_ctxsize = (ctxsize), \
1508 .cra_module = THIS_MODULE, \
1509 }, \
1510 } }
1511
1512 /* aes_gcm_algs_aesni */
1513 DEFINE_GCM_ALGS(aesni, /* no flags */ 0,
1514 "generic-gcm-aesni", "rfc4106-gcm-aesni",
1515 AES_GCM_KEY_AESNI_SIZE, 400);
1516
1517 /* aes_gcm_algs_aesni_avx */
1518 DEFINE_GCM_ALGS(aesni_avx, FLAG_AVX,
1519 "generic-gcm-aesni-avx", "rfc4106-gcm-aesni-avx",
1520 AES_GCM_KEY_AESNI_SIZE, 500);
1521
1522 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
1523 /* aes_gcm_algs_vaes_avx10_256 */
1524 DEFINE_GCM_ALGS(vaes_avx10_256, FLAG_AVX10_256,
1525 "generic-gcm-vaes-avx10_256", "rfc4106-gcm-vaes-avx10_256",
1526 AES_GCM_KEY_AVX10_SIZE, 700);
1527
1528 /* aes_gcm_algs_vaes_avx10_512 */
1529 DEFINE_GCM_ALGS(vaes_avx10_512, FLAG_AVX10_512,
1530 "generic-gcm-vaes-avx10_512", "rfc4106-gcm-vaes-avx10_512",
1531 AES_GCM_KEY_AVX10_SIZE, 800);
1532 #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */
1533
register_avx_algs(void)1534 static int __init register_avx_algs(void)
1535 {
1536 int err;
1537
1538 if (!boot_cpu_has(X86_FEATURE_AVX))
1539 return 0;
1540 err = crypto_register_skciphers(skcipher_algs_aesni_avx,
1541 ARRAY_SIZE(skcipher_algs_aesni_avx));
1542 if (err)
1543 return err;
1544 err = crypto_register_aeads(aes_gcm_algs_aesni_avx,
1545 ARRAY_SIZE(aes_gcm_algs_aesni_avx));
1546 if (err)
1547 return err;
1548 /*
1549 * Note: not all the algorithms registered below actually require
1550 * VPCLMULQDQ. But in practice every CPU with VAES also has VPCLMULQDQ.
1551 * Similarly, the assembler support was added at about the same time.
1552 * For simplicity, just always check for VAES and VPCLMULQDQ together.
1553 */
1554 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
1555 if (!boot_cpu_has(X86_FEATURE_AVX2) ||
1556 !boot_cpu_has(X86_FEATURE_VAES) ||
1557 !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) ||
1558 !boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
1559 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
1560 return 0;
1561 err = crypto_register_skciphers(skcipher_algs_vaes_avx2,
1562 ARRAY_SIZE(skcipher_algs_vaes_avx2));
1563 if (err)
1564 return err;
1565
1566 if (!boot_cpu_has(X86_FEATURE_AVX512BW) ||
1567 !boot_cpu_has(X86_FEATURE_AVX512VL) ||
1568 !boot_cpu_has(X86_FEATURE_BMI2) ||
1569 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
1570 XFEATURE_MASK_AVX512, NULL))
1571 return 0;
1572
1573 err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_256,
1574 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256));
1575 if (err)
1576 return err;
1577
1578 if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
1579 int i;
1580
1581 for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++)
1582 skcipher_algs_vaes_avx512[i].base.cra_priority = 1;
1583 for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512); i++)
1584 aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1;
1585 }
1586
1587 err = crypto_register_skciphers(skcipher_algs_vaes_avx512,
1588 ARRAY_SIZE(skcipher_algs_vaes_avx512));
1589 if (err)
1590 return err;
1591 err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_512,
1592 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512));
1593 if (err)
1594 return err;
1595 #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */
1596 return 0;
1597 }
1598
1599 #define unregister_skciphers(A) \
1600 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
1601 crypto_unregister_skciphers((A), ARRAY_SIZE(A))
1602 #define unregister_aeads(A) \
1603 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
1604 crypto_unregister_aeads((A), ARRAY_SIZE(A))
1605
unregister_avx_algs(void)1606 static void unregister_avx_algs(void)
1607 {
1608 unregister_skciphers(skcipher_algs_aesni_avx);
1609 unregister_aeads(aes_gcm_algs_aesni_avx);
1610 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
1611 unregister_skciphers(skcipher_algs_vaes_avx2);
1612 unregister_skciphers(skcipher_algs_vaes_avx512);
1613 unregister_aeads(aes_gcm_algs_vaes_avx10_256);
1614 unregister_aeads(aes_gcm_algs_vaes_avx10_512);
1615 #endif
1616 }
1617 #else /* CONFIG_X86_64 */
1618 static struct aead_alg aes_gcm_algs_aesni[0];
1619
register_avx_algs(void)1620 static int __init register_avx_algs(void)
1621 {
1622 return 0;
1623 }
1624
unregister_avx_algs(void)1625 static void unregister_avx_algs(void)
1626 {
1627 }
1628 #endif /* !CONFIG_X86_64 */
1629
1630 static const struct x86_cpu_id aesni_cpu_id[] = {
1631 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1632 {}
1633 };
1634 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1635
aesni_init(void)1636 static int __init aesni_init(void)
1637 {
1638 int err;
1639
1640 if (!x86_match_cpu(aesni_cpu_id))
1641 return -ENODEV;
1642
1643 err = crypto_register_alg(&aesni_cipher_alg);
1644 if (err)
1645 return err;
1646
1647 err = crypto_register_skciphers(aesni_skciphers,
1648 ARRAY_SIZE(aesni_skciphers));
1649 if (err)
1650 goto unregister_cipher;
1651
1652 err = crypto_register_aeads(aes_gcm_algs_aesni,
1653 ARRAY_SIZE(aes_gcm_algs_aesni));
1654 if (err)
1655 goto unregister_skciphers;
1656
1657 err = register_avx_algs();
1658 if (err)
1659 goto unregister_avx;
1660
1661 return 0;
1662
1663 unregister_avx:
1664 unregister_avx_algs();
1665 crypto_unregister_aeads(aes_gcm_algs_aesni,
1666 ARRAY_SIZE(aes_gcm_algs_aesni));
1667 unregister_skciphers:
1668 crypto_unregister_skciphers(aesni_skciphers,
1669 ARRAY_SIZE(aesni_skciphers));
1670 unregister_cipher:
1671 crypto_unregister_alg(&aesni_cipher_alg);
1672 return err;
1673 }
1674
aesni_exit(void)1675 static void __exit aesni_exit(void)
1676 {
1677 crypto_unregister_aeads(aes_gcm_algs_aesni,
1678 ARRAY_SIZE(aes_gcm_algs_aesni));
1679 crypto_unregister_skciphers(aesni_skciphers,
1680 ARRAY_SIZE(aesni_skciphers));
1681 crypto_unregister_alg(&aesni_cipher_alg);
1682 unregister_avx_algs();
1683 }
1684
1685 module_init(aesni_init);
1686 module_exit(aesni_exit);
1687
1688 MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions");
1689 MODULE_LICENSE("GPL");
1690 MODULE_ALIAS_CRYPTO("aes");
1691