1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * AES accelerated using the sparc64 aes opcodes
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Copyright (c) 2010, Intel Corporation.
7 * Copyright 2026 Google LLC
8 */
9
10 #include <asm/fpumacro.h>
11 #include <asm/opcodes.h>
12 #include <asm/pstate.h>
13 #include <asm/elf.h>
14
15 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes_opcodes);
16
17 EXPORT_SYMBOL_GPL(aes_sparc64_key_expand);
18 EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_128);
19 EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_192);
20 EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_256);
21 EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_128);
22 EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_192);
23 EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_256);
24 EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_128);
25 EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_192);
26 EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_256);
27 EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_128);
28 EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_192);
29 EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_256);
30 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_128);
31 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_192);
32 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
33 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_128);
34 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_192);
35 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_256);
36 EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_128);
37 EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_192);
38 EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_256);
39
40 void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, u32 *output);
41 void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, u32 *output);
42 void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, u32 *output);
43 void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, u32 *output);
44 void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, u32 *output);
45 void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, u32 *output);
46
aes_preparekey_arch(union aes_enckey_arch * k,union aes_invkey_arch * inv_k,const u8 * in_key,int key_len,int nrounds)47 static void aes_preparekey_arch(union aes_enckey_arch *k,
48 union aes_invkey_arch *inv_k,
49 const u8 *in_key, int key_len, int nrounds)
50 {
51 if (static_branch_likely(&have_aes_opcodes)) {
52 u32 aligned_key[AES_MAX_KEY_SIZE / 4];
53
54 if (IS_ALIGNED((uintptr_t)in_key, 4)) {
55 aes_sparc64_key_expand((const u32 *)in_key,
56 k->sparc_rndkeys, key_len);
57 } else {
58 memcpy(aligned_key, in_key, key_len);
59 aes_sparc64_key_expand(aligned_key,
60 k->sparc_rndkeys, key_len);
61 memzero_explicit(aligned_key, key_len);
62 }
63 /*
64 * Note that nothing needs to be written to inv_k (if it's
65 * non-NULL) here, since the SPARC64 assembly code uses
66 * k->sparc_rndkeys for both encryption and decryption.
67 */
68 } else {
69 aes_expandkey_generic(k->rndkeys,
70 inv_k ? inv_k->inv_rndkeys : NULL,
71 in_key, key_len);
72 }
73 }
74
aes_sparc64_encrypt(const struct aes_enckey * key,const u32 * input,u32 * output)75 static void aes_sparc64_encrypt(const struct aes_enckey *key,
76 const u32 *input, u32 *output)
77 {
78 if (key->len == AES_KEYSIZE_128)
79 aes_sparc64_encrypt_128(key->k.sparc_rndkeys, input, output);
80 else if (key->len == AES_KEYSIZE_192)
81 aes_sparc64_encrypt_192(key->k.sparc_rndkeys, input, output);
82 else
83 aes_sparc64_encrypt_256(key->k.sparc_rndkeys, input, output);
84 }
85
aes_encrypt_arch(const struct aes_enckey * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])86 static void aes_encrypt_arch(const struct aes_enckey *key,
87 u8 out[AES_BLOCK_SIZE],
88 const u8 in[AES_BLOCK_SIZE])
89 {
90 u32 bounce_buf[AES_BLOCK_SIZE / 4];
91
92 if (static_branch_likely(&have_aes_opcodes)) {
93 if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) {
94 aes_sparc64_encrypt(key, (const u32 *)in, (u32 *)out);
95 } else {
96 memcpy(bounce_buf, in, AES_BLOCK_SIZE);
97 aes_sparc64_encrypt(key, bounce_buf, bounce_buf);
98 memcpy(out, bounce_buf, AES_BLOCK_SIZE);
99 }
100 } else {
101 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
102 }
103 }
104
aes_sparc64_decrypt(const struct aes_key * key,const u32 * input,u32 * output)105 static void aes_sparc64_decrypt(const struct aes_key *key,
106 const u32 *input, u32 *output)
107 {
108 if (key->len == AES_KEYSIZE_128)
109 aes_sparc64_decrypt_128(key->k.sparc_rndkeys, input, output);
110 else if (key->len == AES_KEYSIZE_192)
111 aes_sparc64_decrypt_192(key->k.sparc_rndkeys, input, output);
112 else
113 aes_sparc64_decrypt_256(key->k.sparc_rndkeys, input, output);
114 }
115
aes_decrypt_arch(const struct aes_key * key,u8 out[AES_BLOCK_SIZE],const u8 in[AES_BLOCK_SIZE])116 static void aes_decrypt_arch(const struct aes_key *key,
117 u8 out[AES_BLOCK_SIZE],
118 const u8 in[AES_BLOCK_SIZE])
119 {
120 u32 bounce_buf[AES_BLOCK_SIZE / 4];
121
122 if (static_branch_likely(&have_aes_opcodes)) {
123 if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) {
124 aes_sparc64_decrypt(key, (const u32 *)in, (u32 *)out);
125 } else {
126 memcpy(bounce_buf, in, AES_BLOCK_SIZE);
127 aes_sparc64_decrypt(key, bounce_buf, bounce_buf);
128 memcpy(out, bounce_buf, AES_BLOCK_SIZE);
129 }
130 } else {
131 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
132 out, in);
133 }
134 }
135
136 #define aes_mod_init_arch aes_mod_init_arch
aes_mod_init_arch(void)137 static void aes_mod_init_arch(void)
138 {
139 unsigned long cfr;
140
141 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
142 return;
143
144 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
145 if (!(cfr & CFR_AES))
146 return;
147
148 static_branch_enable(&have_aes_opcodes);
149 }
150