1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * AES block cipher, optimized for ARM64
4 *
5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 * Copyright 2026 Google LLC
7 */
8
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <linux/unaligned.h>
12 #include <linux/cpufeature.h>
13
14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
15
16 struct aes_block {
17 u8 b[AES_BLOCK_SIZE];
18 };
19
20 asmlinkage void __aes_arm64_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
21 const u8 in[AES_BLOCK_SIZE], int rounds);
22 asmlinkage void __aes_arm64_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
23 const u8 in[AES_BLOCK_SIZE], int rounds);
24 asmlinkage void __aes_ce_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
25 const u8 in[AES_BLOCK_SIZE], int rounds);
26 asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
27 const u8 in[AES_BLOCK_SIZE], int rounds);
28 asmlinkage u32 __aes_ce_sub(u32 l);
29 asmlinkage void __aes_ce_invert(struct aes_block *out,
30 const struct aes_block *in);
31
32 /*
33 * Expand an AES key using the crypto extensions if supported and usable or
34 * generic code otherwise. The expanded key format is compatible between the
35 * two cases. The outputs are @rndkeys (required) and @inv_rndkeys (optional).
36 */
aes_expandkey_arm64(u32 rndkeys[],u32 * inv_rndkeys,const u8 * in_key,int key_len,int nrounds)37 static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys,
38 const u8 *in_key, int key_len, int nrounds)
39 {
40 /*
41 * The AES key schedule round constants
42 */
43 static u8 const rcon[] = {
44 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
45 };
46
47 u32 kwords = key_len / sizeof(u32);
48 struct aes_block *key_enc, *key_dec;
49 int i, j;
50
51 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) ||
52 !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
53 aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
54 return;
55 }
56
57 for (i = 0; i < kwords; i++)
58 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]);
59
60 scoped_ksimd() {
61 for (i = 0; i < sizeof(rcon); i++) {
62 u32 *rki = &rndkeys[i * kwords];
63 u32 *rko = rki + kwords;
64
65 rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^
66 rcon[i] ^ rki[0];
67 rko[1] = rko[0] ^ rki[1];
68 rko[2] = rko[1] ^ rki[2];
69 rko[3] = rko[2] ^ rki[3];
70
71 if (key_len == AES_KEYSIZE_192) {
72 if (i >= 7)
73 break;
74 rko[4] = rko[3] ^ rki[4];
75 rko[5] = rko[4] ^ rki[5];
76 } else if (key_len == AES_KEYSIZE_256) {
77 if (i >= 6)
78 break;
79 rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
80 rko[5] = rko[4] ^ rki[5];
81 rko[6] = rko[5] ^ rki[6];
82 rko[7] = rko[6] ^ rki[7];
83 }
84 }
85
86 /*
87 * Generate the decryption keys for the Equivalent Inverse
88 * Cipher. This involves reversing the order of the round
89 * keys, and applying the Inverse Mix Columns transformation on
90 * all but the first and the last one.
91 */
92 if (inv_rndkeys) {
93 key_enc = (struct aes_block *)rndkeys;
94 key_dec = (struct aes_block *)inv_rndkeys;
95 j = nrounds;
96
97 key_dec[0] = key_enc[j];
98 for (i = 1, j--; j > 0; i++, j--)
99 __aes_ce_invert(key_dec + i, key_enc + j);
100 key_dec[i] = key_enc[0];
101 }
102 }
103 }
104
aes_preparekey_arch(union aes_enckey_arch * k,union aes_invkey_arch * inv_k,const u8 * in_key,int key_len,int nrounds)105 static void aes_preparekey_arch(union aes_enckey_arch *k,
106 union aes_invkey_arch *inv_k,
107 const u8 *in_key, int key_len, int nrounds)
108 {
109 aes_expandkey_arm64(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
110 in_key, key_len, nrounds);
111 }
112
113 /*
114 * This is here temporarily until the remaining AES mode implementations are
115 * migrated from arch/arm64/crypto/ to lib/crypto/arm64/.
116 */
ce_aes_expandkey(struct crypto_aes_ctx * ctx,const u8 * in_key,unsigned int key_len)117 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
118 unsigned int key_len)
119 {
120 if (aes_check_keylen(key_len) != 0)
121 return -EINVAL;
122 ctx->key_length = key_len;
123 aes_expandkey_arm64(ctx->key_enc, ctx->key_dec, in_key, key_len,
124 6 + key_len / 4);
125 return 0;
126 }
127 EXPORT_SYMBOL(ce_aes_expandkey);
128
aes_encrypt_arch(const struct aes_enckey * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])129 static void aes_encrypt_arch(const struct aes_enckey *key,
130 u8 out[AES_BLOCK_SIZE],
131 const u8 in[AES_BLOCK_SIZE])
132 {
133 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
134 static_branch_likely(&have_aes) && likely(may_use_simd())) {
135 scoped_ksimd()
136 __aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds);
137 } else {
138 __aes_arm64_encrypt(key->k.rndkeys, out, in, key->nrounds);
139 }
140 }
141
aes_decrypt_arch(const struct aes_key * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])142 static void aes_decrypt_arch(const struct aes_key *key,
143 u8 out[AES_BLOCK_SIZE],
144 const u8 in[AES_BLOCK_SIZE])
145 {
146 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
147 static_branch_likely(&have_aes) && likely(may_use_simd())) {
148 scoped_ksimd()
149 __aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in,
150 key->nrounds);
151 } else {
152 __aes_arm64_decrypt(key->inv_k.inv_rndkeys, out, in,
153 key->nrounds);
154 }
155 }
156
157 #ifdef CONFIG_KERNEL_MODE_NEON
158 #define aes_mod_init_arch aes_mod_init_arch
aes_mod_init_arch(void)159 static void aes_mod_init_arch(void)
160 {
161 if (cpu_have_named_feature(AES))
162 static_branch_enable(&have_aes);
163 }
164 #endif /* CONFIG_KERNEL_MODE_NEON */
165