1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
4 * Copyright (C) 2015 International Business Machines Inc.
5 * Copyright 2026 Google LLC
6 */
7 #include <asm/simd.h>
8 #include <asm/switch_to.h>
9 #include <linux/cpufeature.h>
10 #include <linux/jump_label.h>
11 #include <linux/preempt.h>
12 #include <linux/uaccess.h>
13
14 #ifdef CONFIG_SPE
15
16 EXPORT_SYMBOL_GPL(ppc_expand_key_128);
17 EXPORT_SYMBOL_GPL(ppc_expand_key_192);
18 EXPORT_SYMBOL_GPL(ppc_expand_key_256);
19 EXPORT_SYMBOL_GPL(ppc_generate_decrypt_key);
20 EXPORT_SYMBOL_GPL(ppc_encrypt_ecb);
21 EXPORT_SYMBOL_GPL(ppc_decrypt_ecb);
22 EXPORT_SYMBOL_GPL(ppc_encrypt_cbc);
23 EXPORT_SYMBOL_GPL(ppc_decrypt_cbc);
24 EXPORT_SYMBOL_GPL(ppc_crypt_ctr);
25 EXPORT_SYMBOL_GPL(ppc_encrypt_xts);
26 EXPORT_SYMBOL_GPL(ppc_decrypt_xts);
27
28 void ppc_encrypt_aes(u8 *out, const u8 *in, const u32 *key_enc, u32 rounds);
29 void ppc_decrypt_aes(u8 *out, const u8 *in, const u32 *key_dec, u32 rounds);
30
spe_begin(void)31 static void spe_begin(void)
32 {
33 /* disable preemption and save users SPE registers if required */
34 preempt_disable();
35 enable_kernel_spe();
36 }
37
spe_end(void)38 static void spe_end(void)
39 {
40 disable_kernel_spe();
41 /* reenable preemption */
42 preempt_enable();
43 }
44
aes_preparekey_arch(union aes_enckey_arch * k,union aes_invkey_arch * inv_k,const u8 * in_key,int key_len,int nrounds)45 static void aes_preparekey_arch(union aes_enckey_arch *k,
46 union aes_invkey_arch *inv_k,
47 const u8 *in_key, int key_len, int nrounds)
48 {
49 if (key_len == AES_KEYSIZE_128)
50 ppc_expand_key_128(k->spe_enc_key, in_key);
51 else if (key_len == AES_KEYSIZE_192)
52 ppc_expand_key_192(k->spe_enc_key, in_key);
53 else
54 ppc_expand_key_256(k->spe_enc_key, in_key);
55
56 if (inv_k)
57 ppc_generate_decrypt_key(inv_k->spe_dec_key, k->spe_enc_key,
58 key_len);
59 }
60
aes_encrypt_arch(const struct aes_enckey * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])61 static void aes_encrypt_arch(const struct aes_enckey *key,
62 u8 out[AES_BLOCK_SIZE],
63 const u8 in[AES_BLOCK_SIZE])
64 {
65 spe_begin();
66 ppc_encrypt_aes(out, in, key->k.spe_enc_key, key->nrounds / 2 - 1);
67 spe_end();
68 }
69
aes_decrypt_arch(const struct aes_key * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])70 static void aes_decrypt_arch(const struct aes_key *key,
71 u8 out[AES_BLOCK_SIZE],
72 const u8 in[AES_BLOCK_SIZE])
73 {
74 spe_begin();
75 ppc_decrypt_aes(out, in, key->inv_k.spe_dec_key, key->nrounds / 2 - 1);
76 spe_end();
77 }
78
79 #else /* CONFIG_SPE */
80
81 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
82
83 EXPORT_SYMBOL_GPL(aes_p8_set_encrypt_key);
84 EXPORT_SYMBOL_GPL(aes_p8_set_decrypt_key);
85 EXPORT_SYMBOL_GPL(aes_p8_encrypt);
86 EXPORT_SYMBOL_GPL(aes_p8_decrypt);
87 EXPORT_SYMBOL_GPL(aes_p8_cbc_encrypt);
88 EXPORT_SYMBOL_GPL(aes_p8_ctr32_encrypt_blocks);
89 EXPORT_SYMBOL_GPL(aes_p8_xts_encrypt);
90 EXPORT_SYMBOL_GPL(aes_p8_xts_decrypt);
91
is_vsx_format(const struct p8_aes_key * key)92 static inline bool is_vsx_format(const struct p8_aes_key *key)
93 {
94 return key->nrounds != 0;
95 }
96
97 /*
98 * Convert a round key from VSX to generic format by reflecting all 16 bytes (if
99 * little endian) or reflecting the bytes in each 4-byte word (if big endian),
100 * and (if apply_inv_mix=true) applying InvMixColumn to each column.
101 *
102 * It would be nice if the VSX and generic key formats would be compatible. But
103 * that's very difficult to do, with the assembly code having been borrowed from
104 * OpenSSL and also targeted to POWER8 rather than POWER9.
105 *
106 * Fortunately, this conversion should only be needed in extremely rare cases,
107 * possibly not at all in practice. It's just included for full correctness.
108 */
rndkey_from_vsx(u32 out[4],const u32 in[4],bool apply_inv_mix)109 static void rndkey_from_vsx(u32 out[4], const u32 in[4], bool apply_inv_mix)
110 {
111 const bool be = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
112 u32 k0 = swab32(in[0]);
113 u32 k1 = swab32(in[1]);
114 u32 k2 = swab32(in[2]);
115 u32 k3 = swab32(in[3]);
116
117 if (apply_inv_mix) {
118 k0 = inv_mix_columns(k0);
119 k1 = inv_mix_columns(k1);
120 k2 = inv_mix_columns(k2);
121 k3 = inv_mix_columns(k3);
122 }
123 out[0] = be ? k0 : k3;
124 out[1] = be ? k1 : k2;
125 out[2] = be ? k2 : k1;
126 out[3] = be ? k3 : k0;
127 }
128
aes_preparekey_arch(union aes_enckey_arch * k,union aes_invkey_arch * inv_k,const u8 * in_key,int key_len,int nrounds)129 static void aes_preparekey_arch(union aes_enckey_arch *k,
130 union aes_invkey_arch *inv_k,
131 const u8 *in_key, int key_len, int nrounds)
132 {
133 const int keybits = 8 * key_len;
134 int ret;
135
136 if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
137 preempt_disable();
138 pagefault_disable();
139 enable_kernel_vsx();
140 ret = aes_p8_set_encrypt_key(in_key, keybits, &k->p8);
141 /*
142 * aes_p8_set_encrypt_key() should never fail here, since the
143 * key length was already validated.
144 */
145 WARN_ON_ONCE(ret);
146 if (inv_k) {
147 ret = aes_p8_set_decrypt_key(in_key, keybits,
148 &inv_k->p8);
149 /* ... and likewise for aes_p8_set_decrypt_key(). */
150 WARN_ON_ONCE(ret);
151 }
152 disable_kernel_vsx();
153 pagefault_enable();
154 preempt_enable();
155 } else {
156 aes_expandkey_generic(k->rndkeys,
157 inv_k ? inv_k->inv_rndkeys : NULL,
158 in_key, key_len);
159 /* Mark the key as using the generic format. */
160 k->p8.nrounds = 0;
161 if (inv_k)
162 inv_k->p8.nrounds = 0;
163 }
164 }
165
aes_encrypt_arch(const struct aes_enckey * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])166 static void aes_encrypt_arch(const struct aes_enckey *key,
167 u8 out[AES_BLOCK_SIZE],
168 const u8 in[AES_BLOCK_SIZE])
169 {
170 if (static_branch_likely(&have_vec_crypto) &&
171 likely(is_vsx_format(&key->k.p8) && may_use_simd())) {
172 preempt_disable();
173 pagefault_disable();
174 enable_kernel_vsx();
175 aes_p8_encrypt(in, out, &key->k.p8);
176 disable_kernel_vsx();
177 pagefault_enable();
178 preempt_enable();
179 } else if (unlikely(is_vsx_format(&key->k.p8))) {
180 /*
181 * This handles (the hopefully extremely rare) case where a key
182 * was prepared using the VSX optimized format, then encryption
183 * is done in a context that cannot use VSX instructions.
184 */
185 u32 rndkeys[AES_MAX_KEYLENGTH_U32];
186
187 for (int i = 0; i < 4 * (key->nrounds + 1); i += 4)
188 rndkey_from_vsx(&rndkeys[i],
189 &key->k.p8.rndkeys[i], false);
190 aes_encrypt_generic(rndkeys, key->nrounds, out, in);
191 } else {
192 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
193 }
194 }
195
aes_decrypt_arch(const struct aes_key * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])196 static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
197 const u8 in[AES_BLOCK_SIZE])
198 {
199 if (static_branch_likely(&have_vec_crypto) &&
200 likely(is_vsx_format(&key->inv_k.p8) && may_use_simd())) {
201 preempt_disable();
202 pagefault_disable();
203 enable_kernel_vsx();
204 aes_p8_decrypt(in, out, &key->inv_k.p8);
205 disable_kernel_vsx();
206 pagefault_enable();
207 preempt_enable();
208 } else if (unlikely(is_vsx_format(&key->inv_k.p8))) {
209 /*
210 * This handles (the hopefully extremely rare) case where a key
211 * was prepared using the VSX optimized format, then decryption
212 * is done in a context that cannot use VSX instructions.
213 */
214 u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];
215 int i;
216
217 rndkey_from_vsx(&inv_rndkeys[0],
218 &key->inv_k.p8.rndkeys[0], false);
219 for (i = 4; i < 4 * key->nrounds; i += 4) {
220 rndkey_from_vsx(&inv_rndkeys[i],
221 &key->inv_k.p8.rndkeys[i], true);
222 }
223 rndkey_from_vsx(&inv_rndkeys[i],
224 &key->inv_k.p8.rndkeys[i], false);
225 aes_decrypt_generic(inv_rndkeys, key->nrounds, out, in);
226 } else {
227 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
228 out, in);
229 }
230 }
231
232 #define aes_mod_init_arch aes_mod_init_arch
aes_mod_init_arch(void)233 static void aes_mod_init_arch(void)
234 {
235 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
236 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
237 static_branch_enable(&have_vec_crypto);
238 }
239
240 #endif /* !CONFIG_SPE */
241