1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * x86-optimized CRC32 functions
4 *
5 * Copyright (C) 2008 Intel Corporation
6 * Copyright 2012 Xyratex Technology Limited
7 * Copyright 2024 Google LLC
8 */
9
10 #include <linux/crc32.h>
11 #include <linux/module.h>
12 #include "crc-pclmul-template.h"
13
14 static DEFINE_STATIC_KEY_FALSE(have_crc32);
15 static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
16
17 DECLARE_CRC_PCLMUL_FUNCS(crc32_lsb, u32);
18
crc32_le_arch(u32 crc,const u8 * p,size_t len)19 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
20 {
21 CRC_PCLMUL(crc, p, len, crc32_lsb, crc32_lsb_0xedb88320_consts,
22 have_pclmulqdq);
23 return crc32_le_base(crc, p, len);
24 }
25 EXPORT_SYMBOL(crc32_le_arch);
26
27 #ifdef CONFIG_X86_64
28 #define CRC32_INST "crc32q %1, %q0"
29 #else
30 #define CRC32_INST "crc32l %1, %0"
31 #endif
32
33 /*
34 * Use carryless multiply version of crc32c when buffer size is >= 512 to
35 * account for FPU state save/restore overhead.
36 */
37 #define CRC32C_PCLMUL_BREAKEVEN 512
38
39 asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
40
crc32c_arch(u32 crc,const u8 * p,size_t len)41 u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
42 {
43 size_t num_longs;
44
45 if (!static_branch_likely(&have_crc32))
46 return crc32c_base(crc, p, len);
47
48 if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN &&
49 static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
50 kernel_fpu_begin();
51 crc = crc32c_x86_3way(crc, p, len);
52 kernel_fpu_end();
53 return crc;
54 }
55
56 for (num_longs = len / sizeof(unsigned long);
57 num_longs != 0; num_longs--, p += sizeof(unsigned long))
58 asm(CRC32_INST : "+r" (crc) : ASM_INPUT_RM (*(unsigned long *)p));
59
60 if (sizeof(unsigned long) > 4 && (len & 4)) {
61 asm("crc32l %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u32 *)p));
62 p += 4;
63 }
64 if (len & 2) {
65 asm("crc32w %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u16 *)p));
66 p += 2;
67 }
68 if (len & 1)
69 asm("crc32b %1, %0" : "+r" (crc) : ASM_INPUT_RM (*p));
70
71 return crc;
72 }
73 EXPORT_SYMBOL(crc32c_arch);
74
crc32_be_arch(u32 crc,const u8 * p,size_t len)75 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
76 {
77 return crc32_be_base(crc, p, len);
78 }
79 EXPORT_SYMBOL(crc32_be_arch);
80
crc32_x86_init(void)81 static int __init crc32_x86_init(void)
82 {
83 if (boot_cpu_has(X86_FEATURE_XMM4_2))
84 static_branch_enable(&have_crc32);
85 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
86 static_branch_enable(&have_pclmulqdq);
87 INIT_CRC_PCLMUL(crc32_lsb);
88 }
89 return 0;
90 }
91 arch_initcall(crc32_x86_init);
92
crc32_x86_exit(void)93 static void __exit crc32_x86_exit(void)
94 {
95 }
96 module_exit(crc32_x86_exit);
97
crc32_optimizations(void)98 u32 crc32_optimizations(void)
99 {
100 u32 optimizations = 0;
101
102 if (static_key_enabled(&have_crc32))
103 optimizations |= CRC32C_OPTIMIZATION;
104 if (static_key_enabled(&have_pclmulqdq))
105 optimizations |= CRC32_LE_OPTIMIZATION;
106 return optimizations;
107 }
108 EXPORT_SYMBOL(crc32_optimizations);
109
110 MODULE_DESCRIPTION("x86-optimized CRC32 functions");
111 MODULE_LICENSE("GPL");
112