1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * GHASH, RISC-V optimized
4 *
5 * Copyright (C) 2023 VRULL GmbH
6 * Copyright (C) 2023 SiFive, Inc.
7 * Copyright 2026 Google LLC
8 */
9
10 #include <asm/simd.h>
11 #include <asm/vector.h>
12
13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg);
14
15 asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
16 const u8 key[GHASH_BLOCK_SIZE],
17 const u8 *data, size_t nblocks);
18
19 #define ghash_preparekey_arch ghash_preparekey_arch
ghash_preparekey_arch(struct ghash_key * key,const u8 raw_key[GHASH_BLOCK_SIZE])20 static void ghash_preparekey_arch(struct ghash_key *key,
21 const u8 raw_key[GHASH_BLOCK_SIZE])
22 {
23 /* Save key in POLYVAL format for fallback */
24 ghash_key_to_polyval(raw_key, &key->h);
25
26 /* Save key in GHASH format for zvkg */
27 memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
28 }
29
30 #define ghash_blocks_arch ghash_blocks_arch
ghash_blocks_arch(struct polyval_elem * acc,const struct ghash_key * key,const u8 * data,size_t nblocks)31 static void ghash_blocks_arch(struct polyval_elem *acc,
32 const struct ghash_key *key,
33 const u8 *data, size_t nblocks)
34 {
35 if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) {
36 u8 ghash_acc[GHASH_BLOCK_SIZE];
37
38 polyval_acc_to_ghash(acc, ghash_acc);
39
40 kernel_vector_begin();
41 ghash_zvkg(ghash_acc, key->h_raw, data, nblocks);
42 kernel_vector_end();
43
44 ghash_acc_to_polyval(ghash_acc, acc);
45 memzero_explicit(ghash_acc, sizeof(ghash_acc));
46 } else {
47 ghash_blocks_generic(acc, &key->h, data, nblocks);
48 }
49 }
50
51 #define gf128hash_mod_init_arch gf128hash_mod_init_arch
gf128hash_mod_init_arch(void)52 static void gf128hash_mod_init_arch(void)
53 {
54 if (riscv_isa_extension_available(NULL, ZVKG) &&
55 riscv_vector_vlen() >= 128)
56 static_branch_enable(&have_zvkg);
57 }
58