1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * The AEGIS-128 Authenticated-Encryption Algorithm 4 * 5 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com> 6 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. 7 */ 8 9 #include <crypto/algapi.h> 10 #include <crypto/internal/aead.h> 11 #include <crypto/internal/simd.h> 12 #include <crypto/internal/skcipher.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/err.h> 15 #include <linux/init.h> 16 #include <linux/jump_label.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/scatterlist.h> 20 21 #include <asm/simd.h> 22 23 #include "aegis.h" 24 25 #define AEGIS128_NONCE_SIZE 16 26 #define AEGIS128_STATE_BLOCKS 5 27 #define AEGIS128_KEY_SIZE 16 28 #define AEGIS128_MIN_AUTH_SIZE 8 29 #define AEGIS128_MAX_AUTH_SIZE 16 30 31 struct aegis_state { 32 union aegis_block blocks[AEGIS128_STATE_BLOCKS]; 33 }; 34 35 struct aegis_ctx { 36 union aegis_block key; 37 }; 38 39 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_simd); 40 41 static const union aegis_block crypto_aegis_const[2] = { 42 { .words64 = { 43 cpu_to_le64(U64_C(0x0d08050302010100)), 44 cpu_to_le64(U64_C(0x6279e99059372215)), 45 } }, 46 { .words64 = { 47 cpu_to_le64(U64_C(0xf12fc26d55183ddb)), 48 cpu_to_le64(U64_C(0xdd28b57342311120)), 49 } }, 50 }; 51 52 static bool aegis128_do_simd(void) 53 { 54 #ifdef CONFIG_CRYPTO_AEGIS128_SIMD 55 if (static_branch_likely(&have_simd)) 56 return crypto_simd_usable(); 57 #endif 58 return false; 59 } 60 61 static void crypto_aegis128_update(struct aegis_state *state) 62 { 63 union aegis_block tmp; 64 unsigned int i; 65 66 tmp = state->blocks[AEGIS128_STATE_BLOCKS - 1]; 67 for (i = AEGIS128_STATE_BLOCKS - 1; i > 0; i--) 68 crypto_aegis_aesenc(&state->blocks[i], &state->blocks[i - 1], 69 &state->blocks[i]); 70 crypto_aegis_aesenc(&state->blocks[0], &tmp, &state->blocks[0]); 71 } 72 73 static void crypto_aegis128_update_a(struct aegis_state *state, 74 const union aegis_block *msg, 75 bool do_simd) 76 { 77 if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) { 78 crypto_aegis128_update_simd(state, msg); 79 return; 80 } 81 82 crypto_aegis128_update(state); 83 crypto_aegis_block_xor(&state->blocks[0], msg); 84 } 85 86 static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg, 87 bool do_simd) 88 { 89 if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) { 90 crypto_aegis128_update_simd(state, msg); 91 return; 92 } 93 94 crypto_aegis128_update(state); 95 crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE); 96 } 97 98 static void crypto_aegis128_init(struct aegis_state *state, 99 const union aegis_block *key, 100 const u8 *iv) 101 { 102 union aegis_block key_iv; 103 unsigned int i; 104 105 key_iv = *key; 106 crypto_xor(key_iv.bytes, iv, AEGIS_BLOCK_SIZE); 107 108 state->blocks[0] = key_iv; 109 state->blocks[1] = crypto_aegis_const[1]; 110 state->blocks[2] = crypto_aegis_const[0]; 111 state->blocks[3] = *key; 112 state->blocks[4] = *key; 113 114 crypto_aegis_block_xor(&state->blocks[3], &crypto_aegis_const[0]); 115 crypto_aegis_block_xor(&state->blocks[4], &crypto_aegis_const[1]); 116 117 for (i = 0; i < 5; i++) { 118 crypto_aegis128_update_a(state, key, false); 119 crypto_aegis128_update_a(state, &key_iv, false); 120 } 121 } 122 123 static void crypto_aegis128_ad(struct aegis_state *state, 124 const u8 *src, unsigned int size, 125 bool do_simd) 126 { 127 if (AEGIS_ALIGNED(src)) { 128 const union aegis_block *src_blk = 129 (const union aegis_block *)src; 130 131 while (size >= AEGIS_BLOCK_SIZE) { 132 crypto_aegis128_update_a(state, src_blk, do_simd); 133 134 size -= AEGIS_BLOCK_SIZE; 135 src_blk++; 136 } 137 } else { 138 while (size >= AEGIS_BLOCK_SIZE) { 139 crypto_aegis128_update_u(state, src, do_simd); 140 141 size -= AEGIS_BLOCK_SIZE; 142 src += AEGIS_BLOCK_SIZE; 143 } 144 } 145 } 146 147 static void crypto_aegis128_wipe_chunk(struct aegis_state *state, u8 *dst, 148 const u8 *src, unsigned int size) 149 { 150 memzero_explicit(dst, size); 151 } 152 153 static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, 154 const u8 *src, unsigned int size) 155 { 156 union aegis_block tmp; 157 158 if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) { 159 while (size >= AEGIS_BLOCK_SIZE) { 160 union aegis_block *dst_blk = 161 (union aegis_block *)dst; 162 const union aegis_block *src_blk = 163 (const union aegis_block *)src; 164 165 tmp = state->blocks[2]; 166 crypto_aegis_block_and(&tmp, &state->blocks[3]); 167 crypto_aegis_block_xor(&tmp, &state->blocks[4]); 168 crypto_aegis_block_xor(&tmp, &state->blocks[1]); 169 crypto_aegis_block_xor(&tmp, src_blk); 170 171 crypto_aegis128_update_a(state, src_blk, false); 172 173 *dst_blk = tmp; 174 175 size -= AEGIS_BLOCK_SIZE; 176 src += AEGIS_BLOCK_SIZE; 177 dst += AEGIS_BLOCK_SIZE; 178 } 179 } else { 180 while (size >= AEGIS_BLOCK_SIZE) { 181 tmp = state->blocks[2]; 182 crypto_aegis_block_and(&tmp, &state->blocks[3]); 183 crypto_aegis_block_xor(&tmp, &state->blocks[4]); 184 crypto_aegis_block_xor(&tmp, &state->blocks[1]); 185 crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE); 186 187 crypto_aegis128_update_u(state, src, false); 188 189 memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE); 190 191 size -= AEGIS_BLOCK_SIZE; 192 src += AEGIS_BLOCK_SIZE; 193 dst += AEGIS_BLOCK_SIZE; 194 } 195 } 196 197 if (size > 0) { 198 union aegis_block msg = {}; 199 memcpy(msg.bytes, src, size); 200 201 tmp = state->blocks[2]; 202 crypto_aegis_block_and(&tmp, &state->blocks[3]); 203 crypto_aegis_block_xor(&tmp, &state->blocks[4]); 204 crypto_aegis_block_xor(&tmp, &state->blocks[1]); 205 206 crypto_aegis128_update_a(state, &msg, false); 207 208 crypto_aegis_block_xor(&msg, &tmp); 209 210 memcpy(dst, msg.bytes, size); 211 } 212 } 213 214 static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, 215 const u8 *src, unsigned int size) 216 { 217 union aegis_block tmp; 218 219 if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) { 220 while (size >= AEGIS_BLOCK_SIZE) { 221 union aegis_block *dst_blk = 222 (union aegis_block *)dst; 223 const union aegis_block *src_blk = 224 (const union aegis_block *)src; 225 226 tmp = state->blocks[2]; 227 crypto_aegis_block_and(&tmp, &state->blocks[3]); 228 crypto_aegis_block_xor(&tmp, &state->blocks[4]); 229 crypto_aegis_block_xor(&tmp, &state->blocks[1]); 230 crypto_aegis_block_xor(&tmp, src_blk); 231 232 crypto_aegis128_update_a(state, &tmp, false); 233 234 *dst_blk = tmp; 235 236 size -= AEGIS_BLOCK_SIZE; 237 src += AEGIS_BLOCK_SIZE; 238 dst += AEGIS_BLOCK_SIZE; 239 } 240 } else { 241 while (size >= AEGIS_BLOCK_SIZE) { 242 tmp = state->blocks[2]; 243 crypto_aegis_block_and(&tmp, &state->blocks[3]); 244 crypto_aegis_block_xor(&tmp, &state->blocks[4]); 245 crypto_aegis_block_xor(&tmp, &state->blocks[1]); 246 crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE); 247 248 crypto_aegis128_update_a(state, &tmp, false); 249 250 memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE); 251 252 size -= AEGIS_BLOCK_SIZE; 253 src += AEGIS_BLOCK_SIZE; 254 dst += AEGIS_BLOCK_SIZE; 255 } 256 } 257 258 if (size > 0) { 259 union aegis_block msg = {}; 260 memcpy(msg.bytes, src, size); 261 262 tmp = state->blocks[2]; 263 crypto_aegis_block_and(&tmp, &state->blocks[3]); 264 crypto_aegis_block_xor(&tmp, &state->blocks[4]); 265 crypto_aegis_block_xor(&tmp, &state->blocks[1]); 266 crypto_aegis_block_xor(&msg, &tmp); 267 268 memset(msg.bytes + size, 0, AEGIS_BLOCK_SIZE - size); 269 270 crypto_aegis128_update_a(state, &msg, false); 271 272 memcpy(dst, msg.bytes, size); 273 } 274 } 275 276 static void crypto_aegis128_process_ad(struct aegis_state *state, 277 struct scatterlist *sg_src, 278 unsigned int assoclen, 279 bool do_simd) 280 { 281 struct scatter_walk walk; 282 union aegis_block buf; 283 unsigned int pos = 0; 284 285 scatterwalk_start(&walk, sg_src); 286 while (assoclen != 0) { 287 unsigned int size = scatterwalk_next(&walk, assoclen); 288 const u8 *src = walk.addr; 289 unsigned int left = size; 290 291 if (pos + size >= AEGIS_BLOCK_SIZE) { 292 if (pos > 0) { 293 unsigned int fill = AEGIS_BLOCK_SIZE - pos; 294 memcpy(buf.bytes + pos, src, fill); 295 crypto_aegis128_update_a(state, &buf, do_simd); 296 pos = 0; 297 left -= fill; 298 src += fill; 299 } 300 301 crypto_aegis128_ad(state, src, left, do_simd); 302 src += left & ~(AEGIS_BLOCK_SIZE - 1); 303 left &= AEGIS_BLOCK_SIZE - 1; 304 } 305 306 memcpy(buf.bytes + pos, src, left); 307 308 pos += left; 309 assoclen -= size; 310 scatterwalk_done_src(&walk, size); 311 } 312 313 if (pos > 0) { 314 memset(buf.bytes + pos, 0, AEGIS_BLOCK_SIZE - pos); 315 crypto_aegis128_update_a(state, &buf, do_simd); 316 } 317 } 318 319 static __always_inline 320 int crypto_aegis128_process_crypt(struct aegis_state *state, 321 struct skcipher_walk *walk, 322 void (*crypt)(struct aegis_state *state, 323 u8 *dst, 324 const u8 *src, 325 unsigned int size)) 326 { 327 int err = 0; 328 329 while (walk->nbytes) { 330 unsigned int nbytes = walk->nbytes; 331 332 if (nbytes < walk->total) 333 nbytes = round_down(nbytes, walk->stride); 334 335 crypt(state, walk->dst.virt.addr, walk->src.virt.addr, nbytes); 336 337 err = skcipher_walk_done(walk, walk->nbytes - nbytes); 338 } 339 return err; 340 } 341 342 static void crypto_aegis128_final(struct aegis_state *state, 343 union aegis_block *tag_xor, 344 u64 assoclen, u64 cryptlen) 345 { 346 u64 assocbits = assoclen * 8; 347 u64 cryptbits = cryptlen * 8; 348 349 union aegis_block tmp; 350 unsigned int i; 351 352 tmp.words64[0] = cpu_to_le64(assocbits); 353 tmp.words64[1] = cpu_to_le64(cryptbits); 354 355 crypto_aegis_block_xor(&tmp, &state->blocks[3]); 356 357 for (i = 0; i < 7; i++) 358 crypto_aegis128_update_a(state, &tmp, false); 359 360 for (i = 0; i < AEGIS128_STATE_BLOCKS; i++) 361 crypto_aegis_block_xor(tag_xor, &state->blocks[i]); 362 } 363 364 static int crypto_aegis128_setkey(struct crypto_aead *aead, const u8 *key, 365 unsigned int keylen) 366 { 367 struct aegis_ctx *ctx = crypto_aead_ctx(aead); 368 369 if (keylen != AEGIS128_KEY_SIZE) 370 return -EINVAL; 371 372 memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE); 373 return 0; 374 } 375 376 static int crypto_aegis128_setauthsize(struct crypto_aead *tfm, 377 unsigned int authsize) 378 { 379 if (authsize > AEGIS128_MAX_AUTH_SIZE) 380 return -EINVAL; 381 if (authsize < AEGIS128_MIN_AUTH_SIZE) 382 return -EINVAL; 383 return 0; 384 } 385 386 static int crypto_aegis128_encrypt_generic(struct aead_request *req) 387 { 388 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 389 union aegis_block tag = {}; 390 unsigned int authsize = crypto_aead_authsize(tfm); 391 struct aegis_ctx *ctx = crypto_aead_ctx(tfm); 392 unsigned int cryptlen = req->cryptlen; 393 struct skcipher_walk walk; 394 struct aegis_state state; 395 396 skcipher_walk_aead_encrypt(&walk, req, false); 397 crypto_aegis128_init(&state, &ctx->key, req->iv); 398 crypto_aegis128_process_ad(&state, req->src, req->assoclen, false); 399 crypto_aegis128_process_crypt(&state, &walk, 400 crypto_aegis128_encrypt_chunk); 401 crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); 402 403 scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, 404 authsize, 1); 405 return 0; 406 } 407 408 static int crypto_aegis128_decrypt_generic(struct aead_request *req) 409 { 410 static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {}; 411 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 412 union aegis_block tag; 413 unsigned int authsize = crypto_aead_authsize(tfm); 414 unsigned int cryptlen = req->cryptlen - authsize; 415 struct aegis_ctx *ctx = crypto_aead_ctx(tfm); 416 struct skcipher_walk walk; 417 struct aegis_state state; 418 419 scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen, 420 authsize, 0); 421 422 skcipher_walk_aead_decrypt(&walk, req, false); 423 crypto_aegis128_init(&state, &ctx->key, req->iv); 424 crypto_aegis128_process_ad(&state, req->src, req->assoclen, false); 425 crypto_aegis128_process_crypt(&state, &walk, 426 crypto_aegis128_decrypt_chunk); 427 crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); 428 429 if (unlikely(crypto_memneq(tag.bytes, zeros, authsize))) { 430 /* 431 * From Chapter 4. 'Security Analysis' of the AEGIS spec [0] 432 * 433 * "3. If verification fails, the decrypted plaintext and the 434 * wrong authentication tag should not be given as output." 435 * 436 * [0] https://competitions.cr.yp.to/round3/aegisv11.pdf 437 */ 438 skcipher_walk_aead_decrypt(&walk, req, false); 439 crypto_aegis128_process_crypt(NULL, &walk, 440 crypto_aegis128_wipe_chunk); 441 memzero_explicit(&tag, sizeof(tag)); 442 return -EBADMSG; 443 } 444 return 0; 445 } 446 447 static int crypto_aegis128_encrypt_simd(struct aead_request *req) 448 { 449 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 450 union aegis_block tag = {}; 451 unsigned int authsize = crypto_aead_authsize(tfm); 452 struct aegis_ctx *ctx = crypto_aead_ctx(tfm); 453 unsigned int cryptlen = req->cryptlen; 454 struct skcipher_walk walk; 455 struct aegis_state state; 456 457 if (!aegis128_do_simd()) 458 return crypto_aegis128_encrypt_generic(req); 459 460 skcipher_walk_aead_encrypt(&walk, req, false); 461 crypto_aegis128_init_simd(&state, &ctx->key, req->iv); 462 crypto_aegis128_process_ad(&state, req->src, req->assoclen, true); 463 crypto_aegis128_process_crypt(&state, &walk, 464 crypto_aegis128_encrypt_chunk_simd); 465 crypto_aegis128_final_simd(&state, &tag, req->assoclen, cryptlen, 0); 466 467 scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, 468 authsize, 1); 469 return 0; 470 } 471 472 static int crypto_aegis128_decrypt_simd(struct aead_request *req) 473 { 474 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 475 union aegis_block tag; 476 unsigned int authsize = crypto_aead_authsize(tfm); 477 unsigned int cryptlen = req->cryptlen - authsize; 478 struct aegis_ctx *ctx = crypto_aead_ctx(tfm); 479 struct skcipher_walk walk; 480 struct aegis_state state; 481 482 if (!aegis128_do_simd()) 483 return crypto_aegis128_decrypt_generic(req); 484 485 scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen, 486 authsize, 0); 487 488 skcipher_walk_aead_decrypt(&walk, req, false); 489 crypto_aegis128_init_simd(&state, &ctx->key, req->iv); 490 crypto_aegis128_process_ad(&state, req->src, req->assoclen, true); 491 crypto_aegis128_process_crypt(&state, &walk, 492 crypto_aegis128_decrypt_chunk_simd); 493 494 if (unlikely(crypto_aegis128_final_simd(&state, &tag, req->assoclen, 495 cryptlen, authsize))) { 496 skcipher_walk_aead_decrypt(&walk, req, false); 497 crypto_aegis128_process_crypt(NULL, &walk, 498 crypto_aegis128_wipe_chunk); 499 return -EBADMSG; 500 } 501 return 0; 502 } 503 504 static struct aead_alg crypto_aegis128_alg_generic = { 505 .setkey = crypto_aegis128_setkey, 506 .setauthsize = crypto_aegis128_setauthsize, 507 .encrypt = crypto_aegis128_encrypt_generic, 508 .decrypt = crypto_aegis128_decrypt_generic, 509 510 .ivsize = AEGIS128_NONCE_SIZE, 511 .maxauthsize = AEGIS128_MAX_AUTH_SIZE, 512 .chunksize = AEGIS_BLOCK_SIZE, 513 514 .base.cra_blocksize = 1, 515 .base.cra_ctxsize = sizeof(struct aegis_ctx), 516 .base.cra_priority = 100, 517 .base.cra_name = "aegis128", 518 .base.cra_driver_name = "aegis128-generic", 519 .base.cra_module = THIS_MODULE, 520 }; 521 522 static struct aead_alg crypto_aegis128_alg_simd = { 523 .setkey = crypto_aegis128_setkey, 524 .setauthsize = crypto_aegis128_setauthsize, 525 .encrypt = crypto_aegis128_encrypt_simd, 526 .decrypt = crypto_aegis128_decrypt_simd, 527 528 .ivsize = AEGIS128_NONCE_SIZE, 529 .maxauthsize = AEGIS128_MAX_AUTH_SIZE, 530 .chunksize = AEGIS_BLOCK_SIZE, 531 532 .base.cra_blocksize = 1, 533 .base.cra_ctxsize = sizeof(struct aegis_ctx), 534 .base.cra_priority = 200, 535 .base.cra_name = "aegis128", 536 .base.cra_driver_name = "aegis128-simd", 537 .base.cra_module = THIS_MODULE, 538 }; 539 540 static int __init crypto_aegis128_module_init(void) 541 { 542 int ret; 543 544 ret = crypto_register_aead(&crypto_aegis128_alg_generic); 545 if (ret) 546 return ret; 547 548 if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && 549 crypto_aegis128_have_simd()) { 550 ret = crypto_register_aead(&crypto_aegis128_alg_simd); 551 if (ret) { 552 crypto_unregister_aead(&crypto_aegis128_alg_generic); 553 return ret; 554 } 555 static_branch_enable(&have_simd); 556 } 557 return 0; 558 } 559 560 static void __exit crypto_aegis128_module_exit(void) 561 { 562 if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && 563 crypto_aegis128_have_simd()) 564 crypto_unregister_aead(&crypto_aegis128_alg_simd); 565 566 crypto_unregister_aead(&crypto_aegis128_alg_generic); 567 } 568 569 subsys_initcall(crypto_aegis128_module_init); 570 module_exit(crypto_aegis128_module_exit); 571 572 MODULE_LICENSE("GPL"); 573 MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>"); 574 MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm"); 575 MODULE_ALIAS_CRYPTO("aegis128"); 576 MODULE_ALIAS_CRYPTO("aegis128-generic"); 577 MODULE_ALIAS_CRYPTO("aegis128-simd"); 578