1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/crc-t10dif.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/string.h> 13 14 #include <crypto/internal/simd.h> 15 16 #include <asm/neon.h> 17 #include <asm/simd.h> 18 19 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 21 22 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 23 24 asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); 25 asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, 26 u8 out[16]); 27 28 u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 29 { 30 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 31 if (static_branch_likely(&have_pmull)) { 32 if (crypto_simd_usable()) { 33 kernel_neon_begin(); 34 crc = crc_t10dif_pmull64(crc, data, length); 35 kernel_neon_end(); 36 return crc; 37 } 38 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 39 static_branch_likely(&have_neon) && 40 crypto_simd_usable()) { 41 u8 buf[16] __aligned(16); 42 43 kernel_neon_begin(); 44 crc_t10dif_pmull8(crc, data, length, buf); 45 kernel_neon_end(); 46 47 return crc_t10dif_generic(0, buf, sizeof(buf)); 48 } 49 } 50 return crc_t10dif_generic(crc, data, length); 51 } 52 EXPORT_SYMBOL(crc_t10dif_arch); 53 54 static int __init crc_t10dif_arm_init(void) 55 { 56 if (elf_hwcap & HWCAP_NEON) { 57 static_branch_enable(&have_neon); 58 if (elf_hwcap2 & HWCAP2_PMULL) 59 static_branch_enable(&have_pmull); 60 } 61 return 0; 62 } 63 subsys_initcall(crc_t10dif_arm_init); 64 65 static void __exit crc_t10dif_arm_exit(void) 66 { 67 } 68 module_exit(crc_t10dif_arm_exit); 69 70 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 71 MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); 72 MODULE_LICENSE("GPL v2"); 73