1 // SPDX-License-Identifier: GPL-2.0
6  *	- Rafael R. Reilova (moved everything from head.S),
8  *	- Channing Corn (tests & fixes),
9  *	- Andrew D. Balsa (code cleanup).
20 #include <asm/spec-ctrl.h>
24 #include <asm/processor-flags.h>
54 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
58 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
86  * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
97 	 * When KERNEL_IBRS this MSR is written on return-to-user, unless  in update_spec_ctrl_cond()
214 	 * current's TIF_SSBD thread flag.  in x86_virt_spec_ctrl()
219 		hostval = ssbd_tif_to_spec_ctrl(ti->flags);  in x86_virt_spec_ctrl()
248 /* Default mitigation for MDS-affected CPUs */
267 /* Default mitigation for TAA-affected CPUs */
321 		return -EINVAL;  in mds_cmdline()
411 		return -EINVAL;  in tsx_async_abort_parse_cmdline()
459 	 * mitigations, disable KVM-only mitigation in that case.  in mmio_select_mitigation()
467 	 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can  in mmio_select_mitigation()
499 		return -EINVAL;  in mmio_stale_data_parse_cmdline()
544 		return -EINVAL;  in rfds_parse_cmdline()
617 	 * As these mitigations are inter-related and rely on VERW instruction  in md_clear_select_mitigation()
712 		return -EINVAL;  in srbds_parse_cmdline()
876 		return -EINVAL;  in gds_parse_cmdline()
919 	 * Consider SMAP to be non-functional as a mitigation on these  in smap_works_speculatively()
938 		 * path of a conditional swapgs with a user-controlled GS  in spectre_v1_select_mitigation()
963 			 * Enable lfences in the kernel entry (non-swapgs)  in spectre_v1_select_mitigation()
1022 		return -EINVAL;  in retbleed_parse_cmdline()
1147 		 * software-based untraining so clear those in case some  in retbleed_select_mitigation()
1232 		return -EINVAL;  in its_parse_cmdline()
1280 		pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n");  in its_select_mitigation()
1362 	return spectre_v2_bad_module ? " - vulnerable module loaded" : "";  in spectre_v2_module_string()
1429 	[SPECTRE_V2_USER_STRICT_PREFERRED]	= "User space: Mitigation: STIBP always-on protection",
1461 	int ret, i;  in spectre_v2_parse_user_cmdline()  local
1480 	for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {  in spectre_v2_parse_user_cmdline()
1481 		if (match_option(arg, ret, v2_user_options[i].option)) {  in spectre_v2_parse_user_cmdline()
1482 			spec_v2_user_print_cond(v2_user_options[i].option,  in spectre_v2_parse_user_cmdline()
1483 						v2_user_options[i].secure);  in spectre_v2_parse_user_cmdline()
1484 			return v2_user_options[i].cmd;  in spectre_v2_parse_user_cmdline()
1550 			"always-on" : "conditional");  in spectre_v2_user_select_mitigation()
1557 	 * Intel's Enhanced IBRS also protects against cross-thread branch target  in spectre_v2_user_select_mitigation()
1558 	 * injection in user-mode as the IBRS bit remains always set which  in spectre_v2_user_select_mitigation()
1559 	 * implicitly enables cross-thread protections.  However, in legacy IBRS  in spectre_v2_user_select_mitigation()
1562 	 * These modes therefore disable the implicit cross-thread protection,  in spectre_v2_user_select_mitigation()
1573 	 * If STIBP support is not being forced, check if STIBP always-on  in spectre_v2_user_select_mitigation()
1584 			pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");  in spectre_v2_user_select_mitigation()
1632 	int ret, i;  in spectre_v2_parse_cmdline()  local
1643 	for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {  in spectre_v2_parse_cmdline()
1644 		if (!match_option(arg, ret, mitigation_options[i].option))  in spectre_v2_parse_cmdline()
1646 		cmd = mitigation_options[i].cmd;  in spectre_v2_parse_cmdline()
1650 	if (i >= ARRAY_SIZE(mitigation_options)) {  in spectre_v2_parse_cmdline()
1662 		       mitigation_options[i].option);  in spectre_v2_parse_cmdline()
1671 		       mitigation_options[i].option);  in spectre_v2_parse_cmdline()
1679 		       mitigation_options[i].option);  in spectre_v2_parse_cmdline()
1685 		       mitigation_options[i].option);  in spectre_v2_parse_cmdline()
1691 		       mitigation_options[i].option);  in spectre_v2_parse_cmdline()
1697 		       mitigation_options[i].option);  in spectre_v2_parse_cmdline()
1703 		       mitigation_options[i].option);  in spectre_v2_parse_cmdline()
1707 	spec_v2_print_cond(mitigation_options[i].option,  in spectre_v2_parse_cmdline()
1708 			   mitigation_options[i].secure);  in spectre_v2_parse_cmdline()
1724 /* Disable in-kernel use of non-RSB RET predictors */
1747 	 * code related to RSB-related mitigations.  Before doing so, carefully  in spectre_v2_select_rsb_mitigation()
1750 	 *   Documentation/admin-guide/hw-vuln/rsb.rst  in spectre_v2_select_rsb_mitigation()
1754 	 *   - User->user RSB attacks are conditionally mitigated during  in spectre_v2_select_rsb_mitigation()
1755 	 *     context switches by cond_mitigation -> write_ibpb().  in spectre_v2_select_rsb_mitigation()
1757 	 *   - User->kernel and guest->host attacks are mitigated by eIBRS or  in spectre_v2_select_rsb_mitigation()
1773 			pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");  in spectre_v2_select_rsb_mitigation()
1821 		return -EINVAL;  in spectre_bhi_parse_cmdline()
1970 	 * JMPs gets protection against BHI and Intramode-BTI, but RET  in spectre_v2_select_mitigation()
1971 	 * prediction from a non-RSB predictor is still a risk.  in spectre_v2_select_mitigation()
1994 	 * the CPU supports Enhanced IBRS, kernel might un-intentionally not  in spectre_v2_select_mitigation()
2034 		mask & SPEC_CTRL_STIBP ? "always-on" : "off");  in update_stibp_strict()
2060 	 * repartitioning leak would be a window dressing exercise.  in update_mds_branch_idle()
2073 …T "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/…
2074 …T "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/…
2075 …e Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/…
2172 	int ret, i;  in ssb_parse_cmdline()  local
2185 		for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {  in ssb_parse_cmdline()
2186 			if (!match_option(arg, ret, ssb_mitigation_options[i].option))  in ssb_parse_cmdline()
2189 			cmd = ssb_mitigation_options[i].cmd;  in ssb_parse_cmdline()
2193 		if (i >= ARRAY_SIZE(ssb_mitigation_options)) {  in ssb_parse_cmdline()
2240 	 *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.  in __ssb_select_mitigation()
2241 	 *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass  in __ssb_select_mitigation()
2242 	 *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation  in __ssb_select_mitigation()
2279 	 * Immediately update the speculation control MSRs for the current  in task_update_spec_tif()
2280 	 * task, but for a non-current task delay setting the CPU  in task_update_spec_tif()
2284 	 * always the current task.  in task_update_spec_tif()
2286 	if (tsk == current)  in task_update_spec_tif()
2294 		return -EPERM;  in l1d_flush_prctl_set()
2298 		set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);  in l1d_flush_prctl_set()
2301 		clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);  in l1d_flush_prctl_set()
2304 		return -ERANGE;  in l1d_flush_prctl_set()
2312 		return -ENXIO;  in ssb_prctl_set()
2318 			return -EPERM;  in ssb_prctl_set()
2336 			return -EPERM;  in ssb_prctl_set()
2342 		return -ERANGE;  in ssb_prctl_set()
2371 		 * updated, unless it was force-disabled by a previous prctl  in ib_prctl_set()
2380 			return -EPERM;  in ib_prctl_set()
2393 			return -EPERM;  in ib_prctl_set()
2402 		if (task == current)  in ib_prctl_set()
2406 		return -ERANGE;  in ib_prctl_set()
2422 		return -ENODEV;  in arch_prctl_spec_ctrl_set()
2442 	if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))  in l1d_flush_prctl_get()
2502 		return -ENODEV;  in arch_prctl_spec_ctrl_get()
2521 /* Default mitigation for L1TF-affected CPUs */
2546 	if (c->x86 != 6)  in override_cache_bits()
2549 	switch (c->x86_vfm) {  in override_cache_bits()
2563 		if (c->x86_cache_bits < 44)  in override_cache_bits()
2564 			c->x86_cache_bits = 44;  in override_cache_bits()
2604 			e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {  in l1tf_select_mitigation()
2609 …pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help y…  in l1tf_select_mitigation()
2622 		return -EINVAL;  in l1tf_cmdline()
2680 		return -EINVAL;  in srso_parse_cmdline()
2686 	else if (!strcmp(str, "safe-ret"))  in srso_parse_cmdline()
2690 	else if (!strcmp(str, "ibpb-vmexit"))  in srso_parse_cmdline()
2699 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html …
2730 		pr_warn("IBPB-extending microcode not applied!\n");  in srso_select_mitigation()
2782 				 * software-based untraining so clear those in case some  in srso_select_mitigation()
2969 		return "; STIBP: always-on";  in stibp_state()
2982 			return "; IBPB: always-on";  in ibpb_state()
2995 			return "; PBRSB-eIBRS: SW sequence";  in pbrsb_eibrs_state()
2997 			return "; PBRSB-eIBRS: Vulnerable";  in pbrsb_eibrs_state()
2999 		return "; PBRSB-eIBRS: Not affected";  in pbrsb_eibrs_state()
3056 			return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");  in retbleed_show_state()