1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Glue code for POLYVAL using PCMULQDQ-NI 4 * 5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 6 * Copyright (c) 2009 Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * Copyright 2021 Google LLC 9 */ 10 11 /* 12 * Glue code based on ghash-clmulni-intel_glue.c. 13 * 14 * This implementation of POLYVAL uses montgomery multiplication 15 * accelerated by PCLMULQDQ-NI to implement the finite field 16 * operations. 17 */ 18 19 #include <asm/cpu_device_id.h> 20 #include <asm/fpu/api.h> 21 #include <crypto/internal/hash.h> 22 #include <crypto/polyval.h> 23 #include <crypto/utils.h> 24 #include <linux/errno.h> 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/string.h> 28 29 #define POLYVAL_ALIGN 16 30 #define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN) 31 #define POLYVAL_ALIGN_EXTRA ((POLYVAL_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 32 #define POLYVAL_CTX_SIZE (sizeof(struct polyval_tfm_ctx) + POLYVAL_ALIGN_EXTRA) 33 #define NUM_KEY_POWERS 8 34 35 struct polyval_tfm_ctx { 36 /* 37 * These powers must be in the order h^8, ..., h^1. 38 */ 39 u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE] POLYVAL_ALIGN_ATTR; 40 }; 41 42 struct polyval_desc_ctx { 43 u8 buffer[POLYVAL_BLOCK_SIZE]; 44 }; 45 46 asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys, 47 const u8 *in, size_t nblocks, u8 *accumulator); 48 asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2); 49 50 static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm) 51 { 52 return PTR_ALIGN(crypto_shash_ctx(tfm), POLYVAL_ALIGN); 53 } 54 55 static void internal_polyval_update(const struct polyval_tfm_ctx *keys, 56 const u8 *in, size_t nblocks, u8 *accumulator) 57 { 58 kernel_fpu_begin(); 59 clmul_polyval_update(keys, in, nblocks, accumulator); 60 kernel_fpu_end(); 61 } 62 63 static void internal_polyval_mul(u8 *op1, const u8 *op2) 64 { 65 kernel_fpu_begin(); 66 clmul_polyval_mul(op1, op2); 67 kernel_fpu_end(); 68 } 69 70 static int polyval_x86_setkey(struct crypto_shash *tfm, 71 const u8 *key, unsigned int keylen) 72 { 73 struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm); 74 int i; 75 76 if (keylen != POLYVAL_BLOCK_SIZE) 77 return -EINVAL; 78 79 memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE); 80 81 for (i = NUM_KEY_POWERS-2; i >= 0; i--) { 82 memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE); 83 internal_polyval_mul(tctx->key_powers[i], 84 tctx->key_powers[i+1]); 85 } 86 87 return 0; 88 } 89 90 static int polyval_x86_init(struct shash_desc *desc) 91 { 92 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); 93 94 memset(dctx, 0, sizeof(*dctx)); 95 96 return 0; 97 } 98 99 static int polyval_x86_update(struct shash_desc *desc, 100 const u8 *src, unsigned int srclen) 101 { 102 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); 103 const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm); 104 unsigned int nblocks; 105 106 do { 107 /* Allow rescheduling every 4K bytes. */ 108 nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE; 109 internal_polyval_update(tctx, src, nblocks, dctx->buffer); 110 srclen -= nblocks * POLYVAL_BLOCK_SIZE; 111 src += nblocks * POLYVAL_BLOCK_SIZE; 112 } while (srclen >= POLYVAL_BLOCK_SIZE); 113 114 return srclen; 115 } 116 117 static int polyval_x86_finup(struct shash_desc *desc, const u8 *src, 118 unsigned int len, u8 *dst) 119 { 120 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); 121 const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm); 122 123 if (len) { 124 crypto_xor(dctx->buffer, src, len); 125 internal_polyval_mul(dctx->buffer, 126 tctx->key_powers[NUM_KEY_POWERS-1]); 127 } 128 129 memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE); 130 131 return 0; 132 } 133 134 static struct shash_alg polyval_alg = { 135 .digestsize = POLYVAL_DIGEST_SIZE, 136 .init = polyval_x86_init, 137 .update = polyval_x86_update, 138 .finup = polyval_x86_finup, 139 .setkey = polyval_x86_setkey, 140 .descsize = sizeof(struct polyval_desc_ctx), 141 .base = { 142 .cra_name = "polyval", 143 .cra_driver_name = "polyval-clmulni", 144 .cra_priority = 200, 145 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 146 .cra_blocksize = POLYVAL_BLOCK_SIZE, 147 .cra_ctxsize = POLYVAL_CTX_SIZE, 148 .cra_module = THIS_MODULE, 149 }, 150 }; 151 152 __maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = { 153 X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), 154 {} 155 }; 156 MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); 157 158 static int __init polyval_clmulni_mod_init(void) 159 { 160 if (!x86_match_cpu(pcmul_cpu_id)) 161 return -ENODEV; 162 163 if (!boot_cpu_has(X86_FEATURE_AVX)) 164 return -ENODEV; 165 166 return crypto_register_shash(&polyval_alg); 167 } 168 169 static void __exit polyval_clmulni_mod_exit(void) 170 { 171 crypto_unregister_shash(&polyval_alg); 172 } 173 174 module_init(polyval_clmulni_mod_init); 175 module_exit(polyval_clmulni_mod_exit); 176 177 MODULE_LICENSE("GPL"); 178 MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI"); 179 MODULE_ALIAS_CRYPTO("polyval"); 180 MODULE_ALIAS_CRYPTO("polyval-clmulni"); 181