1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Intel Transactional Synchronization Extensions (TSX) control. 4 * 5 * Copyright (C) 2019-2021 Intel Corporation 6 * 7 * Author: 8 * Pawan Gupta <pawan.kumar.gupta@linux.intel.com> 9 */ 10 11 #include <linux/cpufeature.h> 12 13 #include <asm/cmdline.h> 14 #include <asm/cpu.h> 15 #include <asm/msr.h> 16 17 #include "cpu.h" 18 19 #undef pr_fmt 20 #define pr_fmt(fmt) "tsx: " fmt 21 22 enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED; 23 24 static void tsx_disable(void) 25 { 26 u64 tsx; 27 28 rdmsrq(MSR_IA32_TSX_CTRL, tsx); 29 30 /* Force all transactions to immediately abort */ 31 tsx |= TSX_CTRL_RTM_DISABLE; 32 33 /* 34 * Ensure TSX support is not enumerated in CPUID. 35 * This is visible to userspace and will ensure they 36 * do not waste resources trying TSX transactions that 37 * will always abort. 38 */ 39 tsx |= TSX_CTRL_CPUID_CLEAR; 40 41 wrmsrq(MSR_IA32_TSX_CTRL, tsx); 42 } 43 44 static void tsx_enable(void) 45 { 46 u64 tsx; 47 48 rdmsrq(MSR_IA32_TSX_CTRL, tsx); 49 50 /* Enable the RTM feature in the cpu */ 51 tsx &= ~TSX_CTRL_RTM_DISABLE; 52 53 /* 54 * Ensure TSX support is enumerated in CPUID. 55 * This is visible to userspace and will ensure they 56 * can enumerate and use the TSX feature. 57 */ 58 tsx &= ~TSX_CTRL_CPUID_CLEAR; 59 60 wrmsrq(MSR_IA32_TSX_CTRL, tsx); 61 } 62 63 static enum tsx_ctrl_states x86_get_tsx_auto_mode(void) 64 { 65 if (boot_cpu_has_bug(X86_BUG_TAA)) 66 return TSX_CTRL_DISABLE; 67 68 return TSX_CTRL_ENABLE; 69 } 70 71 /* 72 * Disabling TSX is not a trivial business. 73 * 74 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT 75 * which says that TSX is practically disabled (all transactions are 76 * aborted by default). When that bit is set, the kernel unconditionally 77 * disables TSX. 78 * 79 * In order to do that, however, it needs to dance a bit: 80 * 81 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and 82 * the MSR is present only when *two* CPUID bits are set: 83 * 84 * - X86_FEATURE_RTM_ALWAYS_ABORT 85 * - X86_FEATURE_TSX_FORCE_ABORT 86 * 87 * 2. The second method is for CPUs which do not have the above-mentioned 88 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX 89 * through that one. Those CPUs can also have the initially mentioned 90 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy 91 * applies: TSX gets disabled unconditionally. 92 * 93 * When either of the two methods are present, the kernel disables TSX and 94 * clears the respective RTM and HLE feature flags. 95 * 96 * An additional twist in the whole thing presents late microcode loading 97 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID 98 * bit to be set after the update. 99 * 100 * A subsequent hotplug operation on any logical CPU except the BSP will 101 * cause for the supported CPUID feature bits to get re-detected and, if 102 * RTM and HLE get cleared all of a sudden, but, userspace did consult 103 * them before the update, then funny explosions will happen. Long story 104 * short: the kernel doesn't modify CPUID feature bits after booting. 105 * 106 * That's why, this function's call in init_intel() doesn't clear the 107 * feature flags. 108 */ 109 static void tsx_clear_cpuid(void) 110 { 111 u64 msr; 112 113 /* 114 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID 115 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present. 116 */ 117 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) && 118 boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { 119 rdmsrq(MSR_TSX_FORCE_ABORT, msr); 120 msr |= MSR_TFA_TSX_CPUID_CLEAR; 121 wrmsrq(MSR_TSX_FORCE_ABORT, msr); 122 } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { 123 rdmsrq(MSR_IA32_TSX_CTRL, msr); 124 msr |= TSX_CTRL_CPUID_CLEAR; 125 wrmsrq(MSR_IA32_TSX_CTRL, msr); 126 } 127 } 128 129 /* 130 * Disable TSX development mode 131 * 132 * When the microcode released in Feb 2022 is applied, TSX will be disabled by 133 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123 134 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is 135 * not recommended for production deployments. In particular, applying MD_CLEAR 136 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient 137 * execution attack may not be effective on these processors when Intel TSX is 138 * enabled with updated microcode. 139 */ 140 static void tsx_dev_mode_disable(void) 141 { 142 u64 mcu_opt_ctrl; 143 144 /* Check if RTM_ALLOW exists */ 145 if (!boot_cpu_has_bug(X86_BUG_TAA) || 146 !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) || 147 !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL)) 148 return; 149 150 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 151 152 if (mcu_opt_ctrl & RTM_ALLOW) { 153 mcu_opt_ctrl &= ~RTM_ALLOW; 154 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 155 setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT); 156 } 157 } 158 159 void __init tsx_init(void) 160 { 161 char arg[5] = {}; 162 int ret; 163 164 tsx_dev_mode_disable(); 165 166 /* 167 * Hardware will always abort a TSX transaction when the CPUID bit 168 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate 169 * CPUID.RTM and CPUID.HLE bits. Clear them here. 170 */ 171 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) { 172 tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT; 173 tsx_clear_cpuid(); 174 setup_clear_cpu_cap(X86_FEATURE_RTM); 175 setup_clear_cpu_cap(X86_FEATURE_HLE); 176 return; 177 } 178 179 /* 180 * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this 181 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES. 182 * 183 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a 184 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES 185 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get 186 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus, 187 * tsx= cmdline requests will do nothing on CPUs without 188 * MSR_IA32_TSX_CTRL support. 189 */ 190 if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) { 191 setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL); 192 } else { 193 tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED; 194 return; 195 } 196 197 ret = cmdline_find_option(boot_command_line, "tsx", arg, sizeof(arg)); 198 if (ret >= 0) { 199 if (!strcmp(arg, "on")) { 200 tsx_ctrl_state = TSX_CTRL_ENABLE; 201 } else if (!strcmp(arg, "off")) { 202 tsx_ctrl_state = TSX_CTRL_DISABLE; 203 } else if (!strcmp(arg, "auto")) { 204 tsx_ctrl_state = x86_get_tsx_auto_mode(); 205 } else { 206 tsx_ctrl_state = TSX_CTRL_DISABLE; 207 pr_err("invalid option, defaulting to off\n"); 208 } 209 } else { 210 /* tsx= not provided */ 211 if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO)) 212 tsx_ctrl_state = x86_get_tsx_auto_mode(); 213 else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF)) 214 tsx_ctrl_state = TSX_CTRL_DISABLE; 215 else 216 tsx_ctrl_state = TSX_CTRL_ENABLE; 217 } 218 219 if (tsx_ctrl_state == TSX_CTRL_DISABLE) { 220 tsx_disable(); 221 222 /* 223 * tsx_disable() will change the state of the RTM and HLE CPUID 224 * bits. Clear them here since they are now expected to be not 225 * set. 226 */ 227 setup_clear_cpu_cap(X86_FEATURE_RTM); 228 setup_clear_cpu_cap(X86_FEATURE_HLE); 229 } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) { 230 231 /* 232 * HW defaults TSX to be enabled at bootup. 233 * We may still need the TSX enable support 234 * during init for special cases like 235 * kexec after TSX is disabled. 236 */ 237 tsx_enable(); 238 239 /* 240 * tsx_enable() will change the state of the RTM and HLE CPUID 241 * bits. Force them here since they are now expected to be set. 242 */ 243 setup_force_cpu_cap(X86_FEATURE_RTM); 244 setup_force_cpu_cap(X86_FEATURE_HLE); 245 } 246 } 247 248 void tsx_ap_init(void) 249 { 250 tsx_dev_mode_disable(); 251 252 if (tsx_ctrl_state == TSX_CTRL_ENABLE) 253 tsx_enable(); 254 else if (tsx_ctrl_state == TSX_CTRL_DISABLE) 255 tsx_disable(); 256 else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT) 257 /* See comment over that function for more details. */ 258 tsx_clear_cpuid(); 259 } 260