1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * GHASH and POLYVAL, arm64 optimized
4 *
5 * Copyright 2025 Google LLC
6 */
7 #include <asm/simd.h>
8 #include <linux/cpufeature.h>
9
10 #define NUM_H_POWERS 8
11
12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
14
15 asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
16 const u8 *src,
17 const struct polyval_elem *h);
18 asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
19 const struct polyval_elem *b);
20 asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
21 const struct polyval_key *key,
22 const u8 *data, size_t nblocks);
23
24 #define polyval_preparekey_arch polyval_preparekey_arch
polyval_preparekey_arch(struct polyval_key * key,const u8 raw_key[POLYVAL_BLOCK_SIZE])25 static void polyval_preparekey_arch(struct polyval_key *key,
26 const u8 raw_key[POLYVAL_BLOCK_SIZE])
27 {
28 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
29 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
30 if (static_branch_likely(&have_pmull) && may_use_simd()) {
31 scoped_ksimd() {
32 for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
33 key->h_powers[i] = key->h_powers[i + 1];
34 polyval_mul_pmull(
35 &key->h_powers[i],
36 &key->h_powers[NUM_H_POWERS - 1]);
37 }
38 }
39 } else {
40 for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
41 key->h_powers[i] = key->h_powers[i + 1];
42 polyval_mul_generic(&key->h_powers[i],
43 &key->h_powers[NUM_H_POWERS - 1]);
44 }
45 }
46 }
47
polyval_mul_arm64(struct polyval_elem * a,const struct polyval_elem * b)48 static void polyval_mul_arm64(struct polyval_elem *a,
49 const struct polyval_elem *b)
50 {
51 if (static_branch_likely(&have_asimd) && may_use_simd()) {
52 static const u8 zeroes[GHASH_BLOCK_SIZE];
53
54 scoped_ksimd() {
55 if (static_branch_likely(&have_pmull)) {
56 polyval_mul_pmull(a, b);
57 } else {
58 /*
59 * Note that this is indeed equivalent to a
60 * POLYVAL multiplication, since it takes the
61 * accumulator and key in POLYVAL format, and
62 * byte-swapping a block of zeroes is a no-op.
63 */
64 pmull_ghash_update_p8(1, a, zeroes, b);
65 }
66 }
67 } else {
68 polyval_mul_generic(a, b);
69 }
70 }
71
72 #define ghash_mul_arch ghash_mul_arch
ghash_mul_arch(struct polyval_elem * acc,const struct ghash_key * key)73 static void ghash_mul_arch(struct polyval_elem *acc,
74 const struct ghash_key *key)
75 {
76 polyval_mul_arm64(acc, &key->h);
77 }
78
79 #define polyval_mul_arch polyval_mul_arch
polyval_mul_arch(struct polyval_elem * acc,const struct polyval_key * key)80 static void polyval_mul_arch(struct polyval_elem *acc,
81 const struct polyval_key *key)
82 {
83 polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]);
84 }
85
86 #define ghash_blocks_arch ghash_blocks_arch
ghash_blocks_arch(struct polyval_elem * acc,const struct ghash_key * key,const u8 * data,size_t nblocks)87 static void ghash_blocks_arch(struct polyval_elem *acc,
88 const struct ghash_key *key,
89 const u8 *data, size_t nblocks)
90 {
91 if (static_branch_likely(&have_asimd) && may_use_simd()) {
92 scoped_ksimd()
93 pmull_ghash_update_p8(nblocks, acc, data, &key->h);
94 } else {
95 ghash_blocks_generic(acc, &key->h, data, nblocks);
96 }
97 }
98
99 #define polyval_blocks_arch polyval_blocks_arch
polyval_blocks_arch(struct polyval_elem * acc,const struct polyval_key * key,const u8 * data,size_t nblocks)100 static void polyval_blocks_arch(struct polyval_elem *acc,
101 const struct polyval_key *key,
102 const u8 *data, size_t nblocks)
103 {
104 if (static_branch_likely(&have_pmull) && may_use_simd()) {
105 scoped_ksimd()
106 polyval_blocks_pmull(acc, key, data, nblocks);
107 } else {
108 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
109 data, nblocks);
110 }
111 }
112
113 #define gf128hash_mod_init_arch gf128hash_mod_init_arch
gf128hash_mod_init_arch(void)114 static void gf128hash_mod_init_arch(void)
115 {
116 if (cpu_have_named_feature(ASIMD)) {
117 static_branch_enable(&have_asimd);
118 if (cpu_have_named_feature(PMULL))
119 static_branch_enable(&have_pmull);
120 }
121 }
122