1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue code for accelerated AES-GCM stitched implementation for ppc64le. 4 * 5 * Copyright 2022- IBM Inc. All rights reserved 6 */ 7 8 #include <linux/unaligned.h> 9 #include <asm/simd.h> 10 #include <asm/switch_to.h> 11 #include <crypto/gcm.h> 12 #include <crypto/aes.h> 13 #include <crypto/algapi.h> 14 #include <crypto/b128ops.h> 15 #include <crypto/gf128mul.h> 16 #include <crypto/internal/simd.h> 17 #include <crypto/internal/aead.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/skcipher.h> 20 #include <crypto/scatterwalk.h> 21 #include <linux/cpufeature.h> 22 #include <linux/crypto.h> 23 #include <linux/module.h> 24 #include <linux/types.h> 25 26 #define PPC_ALIGN 16 27 #define GCM_IV_SIZE 12 28 #define RFC4106_NONCE_SIZE 4 29 30 MODULE_DESCRIPTION("PPC64le AES-GCM with Stitched implementation"); 31 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com"); 32 MODULE_LICENSE("GPL v2"); 33 MODULE_ALIAS_CRYPTO("aes"); 34 35 asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits, 36 void *key); 37 asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key); 38 asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, 39 void *rkey, u8 *iv, void *Xi); 40 asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len, 41 void *rkey, u8 *iv, void *Xi); 42 asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); 43 asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, 44 unsigned char *aad, unsigned int alen); 45 asmlinkage void gcm_update(u8 *iv, void *Xi); 46 47 struct aes_key { 48 u8 key[AES_MAX_KEYLENGTH]; 49 u64 rounds; 50 }; 51 52 struct gcm_ctx { 53 u8 iv[16]; 54 u8 ivtag[16]; 55 u8 aad_hash[16]; 56 u64 aadLen; 57 u64 Plen; /* offset 56 - used in aes_p10_gcm_{en/de}crypt */ 58 u8 pblock[16]; 59 }; 60 struct Hash_ctx { 61 u8 H[16]; /* subkey */ 62 u8 Htable[256]; /* Xi, Hash table(offset 32) */ 63 }; 64 65 struct p10_aes_gcm_ctx { 66 struct aes_key enc_key; 67 u8 nonce[RFC4106_NONCE_SIZE]; 68 }; 69 70 static void vsx_begin(void) 71 { 72 preempt_disable(); 73 pagefault_disable(); 74 enable_kernel_vsx(); 75 } 76 77 static void vsx_end(void) 78 { 79 disable_kernel_vsx(); 80 pagefault_enable(); 81 preempt_enable(); 82 } 83 84 static void set_subkey(unsigned char *hash) 85 { 86 *(u64 *)&hash[0] = be64_to_cpup((__be64 *)&hash[0]); 87 *(u64 *)&hash[8] = be64_to_cpup((__be64 *)&hash[8]); 88 } 89 90 /* 91 * Compute aad if any. 92 * - Hash aad and copy to Xi. 93 */ 94 static void set_aad(struct gcm_ctx *gctx, struct Hash_ctx *hash, 95 unsigned char *aad, int alen) 96 { 97 int i; 98 u8 nXi[16] = {0, }; 99 100 gctx->aadLen = alen; 101 i = alen & ~0xf; 102 if (i) { 103 gcm_ghash_p10(nXi, hash->Htable+32, aad, i); 104 aad += i; 105 alen -= i; 106 } 107 if (alen) { 108 for (i = 0; i < alen; i++) 109 nXi[i] ^= aad[i]; 110 111 memset(gctx->aad_hash, 0, 16); 112 gcm_ghash_p10(gctx->aad_hash, hash->Htable+32, nXi, 16); 113 } else { 114 memcpy(gctx->aad_hash, nXi, 16); 115 } 116 117 memcpy(hash->Htable, gctx->aad_hash, 16); 118 } 119 120 static void gcmp10_init(struct gcm_ctx *gctx, u8 *iv, unsigned char *rdkey, 121 struct Hash_ctx *hash, u8 *assoc, unsigned int assoclen) 122 { 123 __be32 counter = cpu_to_be32(1); 124 125 aes_p10_encrypt(hash->H, hash->H, rdkey); 126 set_subkey(hash->H); 127 gcm_init_htable(hash->Htable+32, hash->H); 128 129 *((__be32 *)(iv+12)) = counter; 130 131 gctx->Plen = 0; 132 133 /* 134 * Encrypt counter vector as iv tag and increment counter. 135 */ 136 aes_p10_encrypt(iv, gctx->ivtag, rdkey); 137 138 counter = cpu_to_be32(2); 139 *((__be32 *)(iv+12)) = counter; 140 memcpy(gctx->iv, iv, 16); 141 142 gctx->aadLen = assoclen; 143 memset(gctx->aad_hash, 0, 16); 144 if (assoclen) 145 set_aad(gctx, hash, assoc, assoclen); 146 } 147 148 static void finish_tag(struct gcm_ctx *gctx, struct Hash_ctx *hash, int len) 149 { 150 int i; 151 unsigned char len_ac[16 + PPC_ALIGN]; 152 unsigned char *aclen = PTR_ALIGN((void *)len_ac, PPC_ALIGN); 153 __be64 clen = cpu_to_be64(len << 3); 154 __be64 alen = cpu_to_be64(gctx->aadLen << 3); 155 156 if (len == 0 && gctx->aadLen == 0) { 157 memcpy(hash->Htable, gctx->ivtag, 16); 158 return; 159 } 160 161 /* 162 * Len is in bits. 163 */ 164 *((__be64 *)(aclen)) = alen; 165 *((__be64 *)(aclen+8)) = clen; 166 167 /* 168 * hash (AAD len and len) 169 */ 170 gcm_ghash_p10(hash->Htable, hash->Htable+32, aclen, 16); 171 172 for (i = 0; i < 16; i++) 173 hash->Htable[i] ^= gctx->ivtag[i]; 174 } 175 176 static int set_authsize(struct crypto_aead *tfm, unsigned int authsize) 177 { 178 switch (authsize) { 179 case 4: 180 case 8: 181 case 12: 182 case 13: 183 case 14: 184 case 15: 185 case 16: 186 break; 187 default: 188 return -EINVAL; 189 } 190 191 return 0; 192 } 193 194 static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, 195 unsigned int keylen) 196 { 197 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 198 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 199 int ret; 200 201 vsx_begin(); 202 ret = aes_p10_set_encrypt_key(key, keylen * 8, &ctx->enc_key); 203 vsx_end(); 204 205 return ret ? -EINVAL : 0; 206 } 207 208 static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, 209 int assoclen, int enc) 210 { 211 struct crypto_tfm *tfm = req->base.tfm; 212 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 213 u8 databuf[sizeof(struct gcm_ctx) + PPC_ALIGN]; 214 struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN); 215 u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN]; 216 struct Hash_ctx *hash = PTR_ALIGN((void *)hashbuf, PPC_ALIGN); 217 struct skcipher_walk walk; 218 u8 *assocmem = NULL; 219 u8 *assoc; 220 unsigned int cryptlen = req->cryptlen; 221 unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN]; 222 unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN); 223 int ret; 224 unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm)); 225 u8 otag[16]; 226 int total_processed = 0; 227 int nbytes; 228 229 memset(databuf, 0, sizeof(databuf)); 230 memset(hashbuf, 0, sizeof(hashbuf)); 231 memset(ivbuf, 0, sizeof(ivbuf)); 232 memcpy(iv, riv, GCM_IV_SIZE); 233 234 /* Linearize assoc, if not already linear */ 235 if (req->src->length >= assoclen && req->src->length) { 236 assoc = sg_virt(req->src); /* ppc64 is !HIGHMEM */ 237 } else { 238 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 239 GFP_KERNEL : GFP_ATOMIC; 240 241 /* assoc can be any length, so must be on heap */ 242 assocmem = kmalloc(assoclen, flags); 243 if (unlikely(!assocmem)) 244 return -ENOMEM; 245 assoc = assocmem; 246 247 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); 248 } 249 250 vsx_begin(); 251 gcmp10_init(gctx, iv, (unsigned char *) &ctx->enc_key, hash, assoc, assoclen); 252 vsx_end(); 253 254 kfree(assocmem); 255 256 if (enc) 257 ret = skcipher_walk_aead_encrypt(&walk, req, false); 258 else 259 ret = skcipher_walk_aead_decrypt(&walk, req, false); 260 if (ret) 261 return ret; 262 263 while ((nbytes = walk.nbytes) > 0 && ret == 0) { 264 u8 *src = walk.src.virt.addr; 265 u8 *dst = walk.dst.virt.addr; 266 u8 buf[AES_BLOCK_SIZE]; 267 268 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 269 src = dst = memcpy(buf, src, nbytes); 270 271 vsx_begin(); 272 if (enc) 273 aes_p10_gcm_encrypt(src, dst, nbytes, 274 &ctx->enc_key, gctx->iv, hash->Htable); 275 else 276 aes_p10_gcm_decrypt(src, dst, nbytes, 277 &ctx->enc_key, gctx->iv, hash->Htable); 278 279 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 280 memcpy(walk.dst.virt.addr, buf, nbytes); 281 282 vsx_end(); 283 284 total_processed += walk.nbytes; 285 ret = skcipher_walk_done(&walk, 0); 286 } 287 288 if (ret) 289 return ret; 290 291 /* Finalize hash */ 292 vsx_begin(); 293 gcm_update(gctx->iv, hash->Htable); 294 finish_tag(gctx, hash, total_processed); 295 vsx_end(); 296 297 /* copy Xi to end of dst */ 298 if (enc) 299 scatterwalk_map_and_copy(hash->Htable, req->dst, req->assoclen + cryptlen, 300 auth_tag_len, 1); 301 else { 302 scatterwalk_map_and_copy(otag, req->src, 303 req->assoclen + cryptlen - auth_tag_len, 304 auth_tag_len, 0); 305 306 if (crypto_memneq(otag, hash->Htable, auth_tag_len)) { 307 memzero_explicit(hash->Htable, 16); 308 return -EBADMSG; 309 } 310 } 311 312 return 0; 313 } 314 315 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 316 unsigned int keylen) 317 { 318 struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 319 int err; 320 321 keylen -= RFC4106_NONCE_SIZE; 322 err = p10_aes_gcm_setkey(tfm, inkey, keylen); 323 if (err) 324 return err; 325 326 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 327 return 0; 328 } 329 330 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 331 { 332 return crypto_rfc4106_check_authsize(authsize); 333 } 334 335 static int rfc4106_encrypt(struct aead_request *req) 336 { 337 struct crypto_aead *aead = crypto_aead_reqtfm(req); 338 struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); 339 u8 iv[AES_BLOCK_SIZE]; 340 341 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 342 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 343 344 return crypto_ipsec_check_assoclen(req->assoclen) ?: 345 p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 1); 346 } 347 348 static int rfc4106_decrypt(struct aead_request *req) 349 { 350 struct crypto_aead *aead = crypto_aead_reqtfm(req); 351 struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); 352 u8 iv[AES_BLOCK_SIZE]; 353 354 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 355 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 356 357 return crypto_ipsec_check_assoclen(req->assoclen) ?: 358 p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 0); 359 } 360 361 static int p10_aes_gcm_encrypt(struct aead_request *req) 362 { 363 return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 1); 364 } 365 366 static int p10_aes_gcm_decrypt(struct aead_request *req) 367 { 368 return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 0); 369 } 370 371 static struct aead_alg gcm_aes_algs[] = {{ 372 .ivsize = GCM_IV_SIZE, 373 .maxauthsize = 16, 374 375 .setauthsize = set_authsize, 376 .setkey = p10_aes_gcm_setkey, 377 .encrypt = p10_aes_gcm_encrypt, 378 .decrypt = p10_aes_gcm_decrypt, 379 380 .base.cra_name = "__gcm(aes)", 381 .base.cra_driver_name = "__aes_gcm_p10", 382 .base.cra_priority = 2100, 383 .base.cra_blocksize = 1, 384 .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx)+ 385 4 * sizeof(u64[2]), 386 .base.cra_module = THIS_MODULE, 387 .base.cra_flags = CRYPTO_ALG_INTERNAL, 388 }, { 389 .ivsize = GCM_RFC4106_IV_SIZE, 390 .maxauthsize = 16, 391 .setkey = rfc4106_setkey, 392 .setauthsize = rfc4106_setauthsize, 393 .encrypt = rfc4106_encrypt, 394 .decrypt = rfc4106_decrypt, 395 396 .base.cra_name = "__rfc4106(gcm(aes))", 397 .base.cra_driver_name = "__rfc4106_aes_gcm_p10", 398 .base.cra_priority = 2100, 399 .base.cra_blocksize = 1, 400 .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx) + 401 4 * sizeof(u64[2]), 402 .base.cra_module = THIS_MODULE, 403 .base.cra_flags = CRYPTO_ALG_INTERNAL, 404 }}; 405 406 static struct simd_aead_alg *p10_simd_aeads[ARRAY_SIZE(gcm_aes_algs)]; 407 408 static int __init p10_init(void) 409 { 410 int ret; 411 412 if (!cpu_has_feature(CPU_FTR_ARCH_31)) 413 return 0; 414 415 ret = simd_register_aeads_compat(gcm_aes_algs, 416 ARRAY_SIZE(gcm_aes_algs), 417 p10_simd_aeads); 418 if (ret) { 419 simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), 420 p10_simd_aeads); 421 return ret; 422 } 423 return 0; 424 } 425 426 static void __exit p10_exit(void) 427 { 428 simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), 429 p10_simd_aeads); 430 } 431 432 module_init(p10_init); 433 module_exit(p10_exit); 434