1 // SPDX-License-Identifier: GPL-2.0-only
27 #include <asm/sbi.h>
30 #include "copy-unaligned.h"
32 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
37 #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)
44 /* Per-cpu ISA extensions. */
53  * riscv_isa_extension_base() - Get base extension word
69  * __riscv_isa_extension_available() - Check whether given extension
94 			pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n");  in riscv_isa_extension_check()
97 			pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n");  in riscv_isa_extension_check()
103 			pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n");  in riscv_isa_extension_check()
106 			pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");  in riscv_isa_extension_check()
206  * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V
219  * Ordinarily, for in-kernel data structures, this order is unimportant but
225  * 1. All multi-letter extensions must be separated from other extensions by an
229  *    single-letter extensions and before any higher-privileged extensions.
236  * 3. Standard supervisor-level extensions (starting with 'S') must be listed
237  *    after standard unprivileged extensions.  If multiple supervisor-level
240  * 4. Standard machine-level extensions (starting with 'Zxm') must be listed
241  *    after any lower-privileged, standard extensions.  If multiple
242  *    machine-level extensions are listed, they must be ordered
245  * 5. Non-standard extensions (starting with 'X') must be listed after all
246  *    standard extensions. If multiple non-standard extensions are listed, they
328 	if ((name_end - name == strlen(ext->name)) &&  in match_isa_ext()
329 	     !strncasecmp(name, ext->name, name_end - name)) {  in match_isa_ext()
334 		if (ext->subset_ext_size) {  in match_isa_ext()
335 			for (int i = 0; i < ext->subset_ext_size; i++) {  in match_isa_ext()
336 				if (riscv_isa_extension_check(ext->subset_ext_ids[i]))  in match_isa_ext()
337 					set_bit(ext->subset_ext_ids[i], isainfo->isa);  in match_isa_ext()
345 		if (riscv_isa_extension_check(ext->id))  in match_isa_ext()
346 			set_bit(ext->id, isainfo->isa);  in match_isa_ext()
369 			 * Workaround for invalid single-letter 's' & 'u' (QEMU).  in riscv_parse_isa_string()
372 			 * multi-letter extension in the ISA string begins with  in riscv_parse_isa_string()
375 			if (ext[-1] != '_' && ext[1] == 'u') {  in riscv_parse_isa_string()
388 			 * As multi-letter extensions must be split from other multi-letter  in riscv_parse_isa_string()
389 			 * extensions with an "_", the end of a multi-letter extension will  in riscv_parse_isa_string()
391 			 * multi-letter extension.  in riscv_parse_isa_string()
402 			 * A simple re-increment solves this problem.  in riscv_parse_isa_string()
413 			if (!isdigit(ext_end[-1]))  in riscv_parse_isa_string()
416 			while (isdigit(*--ext_end))  in riscv_parse_isa_string()
419 			if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) {  in riscv_parse_isa_string()
424 			while (isdigit(*--ext_end))  in riscv_parse_isa_string()
431 			 * Things are a little easier for single-letter extensions, as they  in riscv_parse_isa_string()
438 			 * If we are already on a non-digit, there is nothing to do. Either  in riscv_parse_isa_string()
439 			 * we have a multi-letter extension's _, or the start of an  in riscv_parse_isa_string()
463 				--isa;  in riscv_parse_isa_string()
476 		 * on meeting a non-alphanumeric character, an extra increment is needed  in riscv_parse_isa_string()
477 		 * where the succeeding extension is a multi-letter prefixed with an "_".  in riscv_parse_isa_string()
485 			int nr = tolower(*ext) - 'a';  in riscv_parse_isa_string()
489 				set_bit(nr, isainfo->isa);  in riscv_parse_isa_string()
524 			rc = of_property_read_string(node, "riscv,isa", &isa);  in riscv_fill_hwcap_from_isa_string()
527 				pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");  in riscv_fill_hwcap_from_isa_string()
533 				pr_warn("Unable to get ISA for the hart - %d\n", cpu);  in riscv_fill_hwcap_from_isa_string()
542 		 * port & dt-bindings were upstreamed, and so can be set  in riscv_fill_hwcap_from_isa_string()
543 		 * unconditionally where `i` is in riscv,isa on DT systems.  in riscv_fill_hwcap_from_isa_string()
546 			set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
547 			set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
548 			set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
549 			set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
554 		 * just the standard V-1.0 but vendors aren't well behaved.  in riscv_fill_hwcap_from_isa_string()
555 		 * Many vendors with T-Head CPU cores which implement the 0.7.1  in riscv_fill_hwcap_from_isa_string()
557 		 * CPU cores with the ratified spec will contain non-zero  in riscv_fill_hwcap_from_isa_string()
563 			clear_bit(RISCV_ISA_EXT_v, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
577 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_isa_string()
579 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_isa_string()
601 		if (!of_property_present(cpu_node, "riscv,isa-extensions")) {  in riscv_fill_hwcap_from_ext_list()
609 			if (of_property_match_string(cpu_node, "riscv,isa-extensions",  in riscv_fill_hwcap_from_ext_list()
610 						     ext->property) < 0)  in riscv_fill_hwcap_from_ext_list()
613 			if (ext->subset_ext_size) {  in riscv_fill_hwcap_from_ext_list()
614 				for (int j = 0; j < ext->subset_ext_size; j++) {  in riscv_fill_hwcap_from_ext_list()
615 					if (riscv_isa_extension_check(ext->subset_ext_ids[i]))  in riscv_fill_hwcap_from_ext_list()
616 						set_bit(ext->subset_ext_ids[j], isainfo->isa);  in riscv_fill_hwcap_from_ext_list()
620 			if (riscv_isa_extension_check(ext->id)) {  in riscv_fill_hwcap_from_ext_list()
621 				set_bit(ext->id, isainfo->isa);  in riscv_fill_hwcap_from_ext_list()
641 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_ext_list()
643 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_ext_list()
647 		return -ENOENT;  in riscv_fill_hwcap_from_ext_list()
670 	isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;  in riscv_fill_hwcap()
671 	isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;  in riscv_fill_hwcap()
672 	isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A;  in riscv_fill_hwcap()
673 	isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;  in riscv_fill_hwcap()
674 	isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;  in riscv_fill_hwcap()
675 	isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;  in riscv_fill_hwcap()
676 	isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V;  in riscv_fill_hwcap()
684 			pr_info("Falling back to deprecated \"riscv,isa\"\n");  in riscv_fill_hwcap()
713 	pr_info("riscv: base ISA extensions %s\n", print_str);  in riscv_fill_hwcap()
719 	pr_info("riscv: ELF capabilities %s\n", print_str);  in riscv_fill_hwcap()
726 	hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1));  in riscv_get_elf_hwcap()
755 	word_cycles = -1ULL;  in check_unaligned_access()
775 		if ((end_cycles - start_cycles) < word_cycles)  in check_unaligned_access()
776 			word_cycles = end_cycles - start_cycles;  in check_unaligned_access()
779 	byte_cycles = -1ULL;  in check_unaligned_access()
791 		if ((end_cycles - start_cycles) < byte_cycles)  in check_unaligned_access()
792 			byte_cycles = end_cycles - start_cycles;  in check_unaligned_access()
862 	modify_unaligned_access_branches(&fast_except_me, num_online_cpus() - 1);  in set_unaligned_access_static_branches_except_cpu()
905 		return -ENOMEM;  in riscv_online_cpu()
959 	cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",  in check_unaligned_access_all_cpus()
985  * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the
986  * patch site is for an erratum, identified by the 32-bit patch ID. When
988  * further break down patch ID into two 16-bit numbers. The lower 16 bits
992  * patching on a per-site basis will provide non-zero values and implement
1027 		if (alt->vendor_id != 0)  in riscv_cpufeature_patch_func()
1030 		id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);  in riscv_cpufeature_patch_func()
1040 		value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);  in riscv_cpufeature_patch_func()
1048 		patch_text_nosync(oldptr, altptr, alt->alt_len);  in riscv_cpufeature_patch_func()
1049 		riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);  in riscv_cpufeature_patch_func()