1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * GHASH and POLYVAL, x86_64 optimized
4 *
5 * Copyright 2025 Google LLC
6 */
7 #include <asm/fpu/api.h>
8 #include <linux/cpufeature.h>
9
10 #define NUM_H_POWERS 8
11
12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul);
13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx);
14
15 asmlinkage void polyval_mul_pclmul(struct polyval_elem *a,
16 const struct polyval_elem *b);
17 asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a,
18 const struct polyval_elem *b);
19
20 asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc,
21 const struct polyval_elem *key,
22 const u8 *data, size_t nblocks);
23 asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
24 const struct polyval_key *key,
25 const u8 *data, size_t nblocks);
26
27 #define polyval_preparekey_arch polyval_preparekey_arch
polyval_preparekey_arch(struct polyval_key * key,const u8 raw_key[POLYVAL_BLOCK_SIZE])28 static void polyval_preparekey_arch(struct polyval_key *key,
29 const u8 raw_key[POLYVAL_BLOCK_SIZE])
30 {
31 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
32 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
33 if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
34 kernel_fpu_begin();
35 for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
36 key->h_powers[i] = key->h_powers[i + 1];
37 polyval_mul_pclmul_avx(
38 &key->h_powers[i],
39 &key->h_powers[NUM_H_POWERS - 1]);
40 }
41 kernel_fpu_end();
42 } else {
43 for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
44 key->h_powers[i] = key->h_powers[i + 1];
45 polyval_mul_generic(&key->h_powers[i],
46 &key->h_powers[NUM_H_POWERS - 1]);
47 }
48 }
49 }
50
polyval_mul_x86(struct polyval_elem * a,const struct polyval_elem * b)51 static void polyval_mul_x86(struct polyval_elem *a,
52 const struct polyval_elem *b)
53 {
54 if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
55 kernel_fpu_begin();
56 if (static_branch_likely(&have_pclmul_avx))
57 polyval_mul_pclmul_avx(a, b);
58 else
59 polyval_mul_pclmul(a, b);
60 kernel_fpu_end();
61 } else {
62 polyval_mul_generic(a, b);
63 }
64 }
65
66 #define ghash_mul_arch ghash_mul_arch
ghash_mul_arch(struct polyval_elem * acc,const struct ghash_key * key)67 static void ghash_mul_arch(struct polyval_elem *acc,
68 const struct ghash_key *key)
69 {
70 polyval_mul_x86(acc, &key->h);
71 }
72
73 #define polyval_mul_arch polyval_mul_arch
polyval_mul_arch(struct polyval_elem * acc,const struct polyval_key * key)74 static void polyval_mul_arch(struct polyval_elem *acc,
75 const struct polyval_key *key)
76 {
77 polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]);
78 }
79
80 #define ghash_blocks_arch ghash_blocks_arch
ghash_blocks_arch(struct polyval_elem * acc,const struct ghash_key * key,const u8 * data,size_t nblocks)81 static void ghash_blocks_arch(struct polyval_elem *acc,
82 const struct ghash_key *key,
83 const u8 *data, size_t nblocks)
84 {
85 if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
86 do {
87 /* Allow rescheduling every 4 KiB. */
88 size_t n = min_t(size_t, nblocks,
89 4096 / GHASH_BLOCK_SIZE);
90
91 kernel_fpu_begin();
92 ghash_blocks_pclmul(acc, &key->h, data, n);
93 kernel_fpu_end();
94 data += n * GHASH_BLOCK_SIZE;
95 nblocks -= n;
96 } while (nblocks);
97 } else {
98 ghash_blocks_generic(acc, &key->h, data, nblocks);
99 }
100 }
101
102 #define polyval_blocks_arch polyval_blocks_arch
polyval_blocks_arch(struct polyval_elem * acc,const struct polyval_key * key,const u8 * data,size_t nblocks)103 static void polyval_blocks_arch(struct polyval_elem *acc,
104 const struct polyval_key *key,
105 const u8 *data, size_t nblocks)
106 {
107 if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
108 do {
109 /* Allow rescheduling every 4 KiB. */
110 size_t n = min_t(size_t, nblocks,
111 4096 / POLYVAL_BLOCK_SIZE);
112
113 kernel_fpu_begin();
114 polyval_blocks_pclmul_avx(acc, key, data, n);
115 kernel_fpu_end();
116 data += n * POLYVAL_BLOCK_SIZE;
117 nblocks -= n;
118 } while (nblocks);
119 } else {
120 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
121 data, nblocks);
122 }
123 }
124
125 #define gf128hash_mod_init_arch gf128hash_mod_init_arch
gf128hash_mod_init_arch(void)126 static void gf128hash_mod_init_arch(void)
127 {
128 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
129 static_branch_enable(&have_pclmul);
130 if (boot_cpu_has(X86_FEATURE_AVX))
131 static_branch_enable(&have_pclmul_avx);
132 }
133 }
134