1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF)
4 *
5 * Copyright 2026 Google LLC
6 */
7 #include <asm/cpacf.h>
8 #include <linux/cpufeature.h>
9
10 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash);
11
12 #define ghash_preparekey_arch ghash_preparekey_arch
ghash_preparekey_arch(struct ghash_key * key,const u8 raw_key[GHASH_BLOCK_SIZE])13 static void ghash_preparekey_arch(struct ghash_key *key,
14 const u8 raw_key[GHASH_BLOCK_SIZE])
15 {
16 /* Save key in POLYVAL format for fallback */
17 ghash_key_to_polyval(raw_key, &key->h);
18
19 /* Save key in GHASH format for CPACF_KIMD_GHASH */
20 memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
21 }
22
23 #define ghash_blocks_arch ghash_blocks_arch
ghash_blocks_arch(struct polyval_elem * acc,const struct ghash_key * key,const u8 * data,size_t nblocks)24 static void ghash_blocks_arch(struct polyval_elem *acc,
25 const struct ghash_key *key,
26 const u8 *data, size_t nblocks)
27 {
28 if (static_branch_likely(&have_cpacf_ghash)) {
29 /*
30 * CPACF_KIMD_GHASH requires the accumulator and key in a single
31 * buffer, each using the GHASH convention.
32 */
33 u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8);
34
35 polyval_acc_to_ghash(acc, ctx[0]);
36 memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE);
37
38 cpacf_kimd(CPACF_KIMD_GHASH, ctx, data,
39 nblocks * GHASH_BLOCK_SIZE);
40
41 ghash_acc_to_polyval(ctx[0], acc);
42 memzero_explicit(ctx, sizeof(ctx));
43 } else {
44 ghash_blocks_generic(acc, &key->h, data, nblocks);
45 }
46 }
47
48 #define gf128hash_mod_init_arch gf128hash_mod_init_arch
gf128hash_mod_init_arch(void)49 static void gf128hash_mod_init_arch(void)
50 {
51 if (cpu_have_feature(S390_CPU_FEATURE_MSA) &&
52 cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH))
53 static_branch_enable(&have_cpacf_ghash);
54 }
55