1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * GHASH, arm optimized
4 *
5 * Copyright 2026 Google LLC
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11
12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
13
14 void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
15 const u8 *src, const struct polyval_elem *h);
16
17 #define ghash_blocks_arch ghash_blocks_arch
ghash_blocks_arch(struct polyval_elem * acc,const struct ghash_key * key,const u8 * data,size_t nblocks)18 static void ghash_blocks_arch(struct polyval_elem *acc,
19 const struct ghash_key *key,
20 const u8 *data, size_t nblocks)
21 {
22 if (static_branch_likely(&have_neon) && may_use_simd()) {
23 do {
24 /* Allow rescheduling every 4 KiB. */
25 size_t n =
26 min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE);
27
28 scoped_ksimd()
29 pmull_ghash_update_p8(n, acc, data, &key->h);
30 data += n * GHASH_BLOCK_SIZE;
31 nblocks -= n;
32 } while (nblocks);
33 } else {
34 ghash_blocks_generic(acc, &key->h, data, nblocks);
35 }
36 }
37
38 #define gf128hash_mod_init_arch gf128hash_mod_init_arch
gf128hash_mod_init_arch(void)39 static void gf128hash_mod_init_arch(void)
40 {
41 if (elf_hwcap & HWCAP_NEON)
42 static_branch_enable(&have_neon);
43 }
44