1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * CRC-32 implemented with the z/Architecture Vector Extension Facility.
4 *
5 * Copyright IBM Corp. 2015
6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7 */
8
9 #include <linux/cpufeature.h>
10 #include <asm/fpu.h>
11 #include "crc32-vx.h"
12
13 #define VX_MIN_LEN 64
14 #define VX_ALIGNMENT 16L
15 #define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
16
17 /*
18 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
19 *
20 * Creates a function to perform a particular CRC-32 computation. Depending
21 * on the message buffer, the hardware-accelerated or software implementation
22 * is used. Note that the message buffer is aligned to improve fetch
23 * operations of VECTOR LOAD MULTIPLE instructions.
24 */
25 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
26 static inline u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
27 { \
28 unsigned long prealign, aligned, remaining; \
29 DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
30 \
31 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx()) \
32 return ___crc32_sw(crc, data, datalen); \
33 \
34 if ((unsigned long)data & VX_ALIGN_MASK) { \
35 prealign = VX_ALIGNMENT - \
36 ((unsigned long)data & VX_ALIGN_MASK); \
37 datalen -= prealign; \
38 crc = ___crc32_sw(crc, data, prealign); \
39 data = (void *)((unsigned long)data + prealign); \
40 } \
41 \
42 aligned = datalen & ~VX_ALIGN_MASK; \
43 remaining = datalen & VX_ALIGN_MASK; \
44 \
45 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
46 crc = ___crc32_vx(crc, data, aligned); \
47 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
48 \
49 if (remaining) \
50 crc = ___crc32_sw(crc, data + aligned, remaining); \
51 \
52 return crc; \
53 }
54
DEFINE_CRC32_VX(crc32_le_arch,crc32_le_vgfm_16,crc32_le_base)55 DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
56 DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
57 DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base)
58
59 static inline u32 crc32_optimizations_arch(void)
60 {
61 if (cpu_has_vx()) {
62 return CRC32_LE_OPTIMIZATION |
63 CRC32_BE_OPTIMIZATION |
64 CRC32C_OPTIMIZATION;
65 }
66 return 0;
67 }
68