1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions 4 * 5 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #include <asm/hwcap.h> 13 #include <asm/neon.h> 14 #include <asm/simd.h> 15 #include <crypto/internal/hash.h> 16 #include <crypto/sha3.h> 17 #include <linux/cpufeature.h> 18 #include <linux/kernel.h> 19 #include <linux/module.h> 20 #include <linux/string.h> 21 #include <linux/unaligned.h> 22 23 MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions"); 24 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 25 MODULE_LICENSE("GPL v2"); 26 MODULE_ALIAS_CRYPTO("sha3-224"); 27 MODULE_ALIAS_CRYPTO("sha3-256"); 28 MODULE_ALIAS_CRYPTO("sha3-384"); 29 MODULE_ALIAS_CRYPTO("sha3-512"); 30 31 asmlinkage int sha3_ce_transform(u64 *st, const u8 *data, int blocks, 32 int md_len); 33 34 static int sha3_update(struct shash_desc *desc, const u8 *data, 35 unsigned int len) 36 { 37 struct sha3_state *sctx = shash_desc_ctx(desc); 38 struct crypto_shash *tfm = desc->tfm; 39 unsigned int bs, ds; 40 int blocks; 41 42 ds = crypto_shash_digestsize(tfm); 43 bs = crypto_shash_blocksize(tfm); 44 blocks = len / bs; 45 len -= blocks * bs; 46 do { 47 int rem; 48 49 kernel_neon_begin(); 50 rem = sha3_ce_transform(sctx->st, data, blocks, ds); 51 kernel_neon_end(); 52 data += (blocks - rem) * bs; 53 blocks = rem; 54 } while (blocks); 55 return len; 56 } 57 58 static int sha3_finup(struct shash_desc *desc, const u8 *src, unsigned int len, 59 u8 *out) 60 { 61 struct sha3_state *sctx = shash_desc_ctx(desc); 62 struct crypto_shash *tfm = desc->tfm; 63 __le64 *digest = (__le64 *)out; 64 u8 block[SHA3_224_BLOCK_SIZE]; 65 unsigned int bs, ds; 66 int i; 67 68 ds = crypto_shash_digestsize(tfm); 69 bs = crypto_shash_blocksize(tfm); 70 memcpy(block, src, len); 71 72 block[len++] = 0x06; 73 memset(block + len, 0, bs - len); 74 block[bs - 1] |= 0x80; 75 76 kernel_neon_begin(); 77 sha3_ce_transform(sctx->st, block, 1, ds); 78 kernel_neon_end(); 79 memzero_explicit(block , sizeof(block)); 80 81 for (i = 0; i < ds / 8; i++) 82 put_unaligned_le64(sctx->st[i], digest++); 83 84 if (ds & 4) 85 put_unaligned_le32(sctx->st[i], (__le32 *)digest); 86 87 return 0; 88 } 89 90 static struct shash_alg algs[] = { { 91 .digestsize = SHA3_224_DIGEST_SIZE, 92 .init = crypto_sha3_init, 93 .update = sha3_update, 94 .finup = sha3_finup, 95 .descsize = SHA3_STATE_SIZE, 96 .base.cra_name = "sha3-224", 97 .base.cra_driver_name = "sha3-224-ce", 98 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 99 .base.cra_blocksize = SHA3_224_BLOCK_SIZE, 100 .base.cra_module = THIS_MODULE, 101 .base.cra_priority = 200, 102 }, { 103 .digestsize = SHA3_256_DIGEST_SIZE, 104 .init = crypto_sha3_init, 105 .update = sha3_update, 106 .finup = sha3_finup, 107 .descsize = SHA3_STATE_SIZE, 108 .base.cra_name = "sha3-256", 109 .base.cra_driver_name = "sha3-256-ce", 110 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 111 .base.cra_blocksize = SHA3_256_BLOCK_SIZE, 112 .base.cra_module = THIS_MODULE, 113 .base.cra_priority = 200, 114 }, { 115 .digestsize = SHA3_384_DIGEST_SIZE, 116 .init = crypto_sha3_init, 117 .update = sha3_update, 118 .finup = sha3_finup, 119 .descsize = SHA3_STATE_SIZE, 120 .base.cra_name = "sha3-384", 121 .base.cra_driver_name = "sha3-384-ce", 122 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 123 .base.cra_blocksize = SHA3_384_BLOCK_SIZE, 124 .base.cra_module = THIS_MODULE, 125 .base.cra_priority = 200, 126 }, { 127 .digestsize = SHA3_512_DIGEST_SIZE, 128 .init = crypto_sha3_init, 129 .update = sha3_update, 130 .finup = sha3_finup, 131 .descsize = SHA3_STATE_SIZE, 132 .base.cra_name = "sha3-512", 133 .base.cra_driver_name = "sha3-512-ce", 134 .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 135 .base.cra_blocksize = SHA3_512_BLOCK_SIZE, 136 .base.cra_module = THIS_MODULE, 137 .base.cra_priority = 200, 138 } }; 139 140 static int __init sha3_neon_mod_init(void) 141 { 142 return crypto_register_shashes(algs, ARRAY_SIZE(algs)); 143 } 144 145 static void __exit sha3_neon_mod_fini(void) 146 { 147 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 148 } 149 150 module_cpu_feature_match(SHA3, sha3_neon_mod_init); 151 module_exit(sha3_neon_mod_fini); 152