1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/crc32.h> 4 #include <linux/linkage.h> 5 #include <linux/module.h> 6 7 #include <asm/alternative.h> 8 #include <asm/cpufeature.h> 9 #include <asm/neon.h> 10 #include <asm/simd.h> 11 12 #include <crypto/internal/simd.h> 13 14 // The minimum input length to consider the 4-way interleaved code path 15 static const size_t min_len = 1024; 16 17 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); 18 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); 19 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); 20 21 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 22 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 23 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); 24 25 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 26 { 27 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 28 return crc32_le_base(crc, p, len); 29 30 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 31 kernel_neon_begin(); 32 crc = crc32_le_arm64_4way(crc, p, len); 33 kernel_neon_end(); 34 35 p += round_down(len, 64); 36 len %= 64; 37 38 if (!len) 39 return crc; 40 } 41 42 return crc32_le_arm64(crc, p, len); 43 } 44 EXPORT_SYMBOL(crc32_le_arch); 45 46 u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 47 { 48 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 49 return crc32c_base(crc, p, len); 50 51 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 52 kernel_neon_begin(); 53 crc = crc32c_le_arm64_4way(crc, p, len); 54 kernel_neon_end(); 55 56 p += round_down(len, 64); 57 len %= 64; 58 59 if (!len) 60 return crc; 61 } 62 63 return crc32c_le_arm64(crc, p, len); 64 } 65 EXPORT_SYMBOL(crc32c_arch); 66 67 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 68 { 69 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 70 return crc32_be_base(crc, p, len); 71 72 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 73 kernel_neon_begin(); 74 crc = crc32_be_arm64_4way(crc, p, len); 75 kernel_neon_end(); 76 77 p += round_down(len, 64); 78 len %= 64; 79 80 if (!len) 81 return crc; 82 } 83 84 return crc32_be_arm64(crc, p, len); 85 } 86 EXPORT_SYMBOL(crc32_be_arch); 87 88 u32 crc32_optimizations(void) 89 { 90 if (alternative_has_cap_likely(ARM64_HAS_CRC32)) 91 return CRC32_LE_OPTIMIZATION | 92 CRC32_BE_OPTIMIZATION | 93 CRC32C_OPTIMIZATION; 94 return 0; 95 } 96 EXPORT_SYMBOL(crc32_optimizations); 97 98 MODULE_LICENSE("GPL"); 99 MODULE_DESCRIPTION("arm64-optimized CRC32 functions"); 100