1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 */
9
10 #include <asm/simd.h>
11 #include <linux/cpufeature.h>
12
13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
14
15 asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data,
16 size_t nblocks, size_t block_size);
17
sha3_absorb_blocks(struct sha3_state * state,const u8 * data,size_t nblocks,size_t block_size)18 static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
19 size_t nblocks, size_t block_size)
20 {
21 if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
22 do {
23 size_t rem;
24
25 scoped_ksimd()
26 rem = sha3_ce_transform(state, data, nblocks,
27 block_size);
28 data += (nblocks - rem) * block_size;
29 nblocks = rem;
30 } while (nblocks);
31 } else {
32 sha3_absorb_blocks_generic(state, data, nblocks, block_size);
33 }
34 }
35
sha3_keccakf(struct sha3_state * state)36 static void sha3_keccakf(struct sha3_state *state)
37 {
38 if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
39 /*
40 * Passing zeroes into sha3_ce_transform() gives the plain
41 * Keccak-f permutation, which is what we want here. Any
42 * supported block size may be used. Use SHA3_512_BLOCK_SIZE
43 * since it's the shortest.
44 */
45 static const u8 zeroes[SHA3_512_BLOCK_SIZE];
46
47 scoped_ksimd()
48 sha3_ce_transform(state, zeroes, 1, sizeof(zeroes));
49 } else {
50 sha3_keccakf_generic(state);
51 }
52 }
53
54 #define sha3_mod_init_arch sha3_mod_init_arch
sha3_mod_init_arch(void)55 static void sha3_mod_init_arch(void)
56 {
57 if (cpu_have_named_feature(SHA3))
58 static_branch_enable(&have_sha3);
59 }
60