1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * GHASH routines supporting VMX instructions on the Power 8
4 *
5 * Copyright (C) 2015, 2019 International Business Machines Inc.
6 *
7 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
8 *
9 * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback
10 * mechanism. The new approach is based on arm64 code, which is:
11 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
12 */
13
14 #include "aesp8-ppc.h"
15 #include <asm/switch_to.h>
16 #include <crypto/aes.h>
17 #include <crypto/gf128mul.h>
18 #include <crypto/ghash.h>
19 #include <crypto/internal/hash.h>
20 #include <crypto/internal/simd.h>
21 #include <linux/err.h>
22 #include <linux/kernel.h>
23 #include <linux/module.h>
24 #include <linux/string.h>
25 #include <linux/uaccess.h>
26
27 void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
28 void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
29 void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
30 const u8 *in, size_t len);
31
32 struct p8_ghash_ctx {
33 /* key used by vector asm */
34 u128 htable[16];
35 /* key used by software fallback */
36 be128 key;
37 };
38
39 struct p8_ghash_desc_ctx {
40 u64 shash[2];
41 };
42
p8_ghash_init(struct shash_desc * desc)43 static int p8_ghash_init(struct shash_desc *desc)
44 {
45 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
46
47 memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
48 return 0;
49 }
50
p8_ghash_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int keylen)51 static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
52 unsigned int keylen)
53 {
54 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
55
56 if (keylen != GHASH_BLOCK_SIZE)
57 return -EINVAL;
58
59 preempt_disable();
60 pagefault_disable();
61 enable_kernel_vsx();
62 gcm_init_p8(ctx->htable, (const u64 *) key);
63 disable_kernel_vsx();
64 pagefault_enable();
65 preempt_enable();
66
67 memcpy(&ctx->key, key, GHASH_BLOCK_SIZE);
68
69 return 0;
70 }
71
__ghash_block(struct p8_ghash_ctx * ctx,struct p8_ghash_desc_ctx * dctx,const u8 * src)72 static inline void __ghash_block(struct p8_ghash_ctx *ctx,
73 struct p8_ghash_desc_ctx *dctx,
74 const u8 *src)
75 {
76 if (crypto_simd_usable()) {
77 preempt_disable();
78 pagefault_disable();
79 enable_kernel_vsx();
80 gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE);
81 disable_kernel_vsx();
82 pagefault_enable();
83 preempt_enable();
84 } else {
85 crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
86 gf128mul_lle((be128 *)dctx->shash, &ctx->key);
87 }
88 }
89
__ghash_blocks(struct p8_ghash_ctx * ctx,struct p8_ghash_desc_ctx * dctx,const u8 * src,unsigned int srclen)90 static inline int __ghash_blocks(struct p8_ghash_ctx *ctx,
91 struct p8_ghash_desc_ctx *dctx,
92 const u8 *src, unsigned int srclen)
93 {
94 int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE);
95
96 srclen -= remain;
97 if (crypto_simd_usable()) {
98 preempt_disable();
99 pagefault_disable();
100 enable_kernel_vsx();
101 gcm_ghash_p8(dctx->shash, ctx->htable,
102 src, srclen);
103 disable_kernel_vsx();
104 pagefault_enable();
105 preempt_enable();
106 } else {
107 do {
108 crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
109 gf128mul_lle((be128 *)dctx->shash, &ctx->key);
110 srclen -= GHASH_BLOCK_SIZE;
111 src += GHASH_BLOCK_SIZE;
112 } while (srclen);
113 }
114
115 return remain;
116 }
117
p8_ghash_update(struct shash_desc * desc,const u8 * src,unsigned int srclen)118 static int p8_ghash_update(struct shash_desc *desc,
119 const u8 *src, unsigned int srclen)
120 {
121 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
122 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
123
124 return __ghash_blocks(ctx, dctx, src, srclen);
125 }
126
p8_ghash_finup(struct shash_desc * desc,const u8 * src,unsigned int len,u8 * out)127 static int p8_ghash_finup(struct shash_desc *desc, const u8 *src,
128 unsigned int len, u8 *out)
129 {
130 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
131 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
132
133 if (len) {
134 u8 buf[GHASH_BLOCK_SIZE] = {};
135
136 memcpy(buf, src, len);
137 __ghash_block(ctx, dctx, buf);
138 memzero_explicit(buf, sizeof(buf));
139 }
140 memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
141 return 0;
142 }
143
144 struct shash_alg p8_ghash_alg = {
145 .digestsize = GHASH_DIGEST_SIZE,
146 .init = p8_ghash_init,
147 .update = p8_ghash_update,
148 .finup = p8_ghash_finup,
149 .setkey = p8_ghash_setkey,
150 .descsize = sizeof(struct p8_ghash_desc_ctx),
151 .base = {
152 .cra_name = "ghash",
153 .cra_driver_name = "p8_ghash",
154 .cra_priority = 1000,
155 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
156 .cra_blocksize = GHASH_BLOCK_SIZE,
157 .cra_ctxsize = sizeof(struct p8_ghash_ctx),
158 .cra_module = THIS_MODULE,
159 },
160 };
161