1 // SPDX-License-Identifier: GPL-2.0
2 
3 #define pr_fmt(fmt) "x86/split lock detection: " fmt
4 
5 #include <linux/semaphore.h>
6 #include <linux/workqueue.h>
7 #include <linux/delay.h>
8 #include <linux/cpuhotplug.h>
9 #include <asm/cpu_device_id.h>
10 #include <asm/cmdline.h>
11 #include <asm/traps.h>
12 #include <asm/cpu.h>
13 
14 enum split_lock_detect_state {
15 	sld_off = 0,
16 	sld_warn,
17 	sld_fatal,
18 	sld_ratelimit,
19 };
20 
21 /*
22  * Default to sld_off because most systems do not support split lock detection.
23  * sld_state_setup() will switch this to sld_warn on systems that support
24  * split lock/bus lock detect, unless there is a command line override.
25  */
26 static enum split_lock_detect_state sld_state __ro_after_init = sld_off;
27 static u64 msr_test_ctrl_cache __ro_after_init;
28 
29 /*
30  * With a name like MSR_TEST_CTL it should go without saying, but don't touch
31  * MSR_TEST_CTL unless the CPU is one of the whitelisted models.  Writing it
32  * on CPUs that do not support SLD can cause fireworks, even when writing '0'.
33  */
34 static bool cpu_model_supports_sld __ro_after_init;
35 
36 static const struct {
37 	const char			*option;
38 	enum split_lock_detect_state	state;
39 } sld_options[] __initconst = {
40 	{ "off",	sld_off   },
41 	{ "warn",	sld_warn  },
42 	{ "fatal",	sld_fatal },
43 	{ "ratelimit:", sld_ratelimit },
44 };
45 
46 static struct ratelimit_state bld_ratelimit;
47 
48 static unsigned int sysctl_sld_mitigate = 1;
49 static DEFINE_SEMAPHORE(buslock_sem, 1);
50 
51 #ifdef CONFIG_PROC_SYSCTL
52 static const struct ctl_table sld_sysctls[] = {
53 	{
54 		.procname       = "split_lock_mitigate",
55 		.data           = &sysctl_sld_mitigate,
56 		.maxlen         = sizeof(unsigned int),
57 		.mode           = 0644,
58 		.proc_handler	= proc_douintvec_minmax,
59 		.extra1         = SYSCTL_ZERO,
60 		.extra2         = SYSCTL_ONE,
61 	},
62 };
63 
sld_mitigate_sysctl_init(void)64 static int __init sld_mitigate_sysctl_init(void)
65 {
66 	register_sysctl_init("kernel", sld_sysctls);
67 	return 0;
68 }
69 
70 late_initcall(sld_mitigate_sysctl_init);
71 #endif
72 
match_option(const char * arg,int arglen,const char * opt)73 static inline bool match_option(const char *arg, int arglen, const char *opt)
74 {
75 	int len = strlen(opt), ratelimit;
76 
77 	if (strncmp(arg, opt, len))
78 		return false;
79 
80 	/*
81 	 * Min ratelimit is 1 bus lock/sec.
82 	 * Max ratelimit is 1000 bus locks/sec.
83 	 */
84 	if (sscanf(arg, "ratelimit:%d", &ratelimit) == 1 &&
85 	    ratelimit > 0 && ratelimit <= 1000) {
86 		ratelimit_state_init(&bld_ratelimit, HZ, ratelimit);
87 		ratelimit_set_flags(&bld_ratelimit, RATELIMIT_MSG_ON_RELEASE);
88 		return true;
89 	}
90 
91 	return len == arglen;
92 }
93 
split_lock_verify_msr(bool on)94 static bool split_lock_verify_msr(bool on)
95 {
96 	u64 ctrl, tmp;
97 
98 	if (rdmsrl_safe(MSR_TEST_CTRL, &ctrl))
99 		return false;
100 	if (on)
101 		ctrl |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT;
102 	else
103 		ctrl &= ~MSR_TEST_CTRL_SPLIT_LOCK_DETECT;
104 	if (wrmsrl_safe(MSR_TEST_CTRL, ctrl))
105 		return false;
106 	rdmsrl(MSR_TEST_CTRL, tmp);
107 	return ctrl == tmp;
108 }
109 
sld_state_setup(void)110 static void __init sld_state_setup(void)
111 {
112 	enum split_lock_detect_state state = sld_warn;
113 	char arg[20];
114 	int i, ret;
115 
116 	if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT) &&
117 	    !boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT))
118 		return;
119 
120 	ret = cmdline_find_option(boot_command_line, "split_lock_detect",
121 				  arg, sizeof(arg));
122 	if (ret >= 0) {
123 		for (i = 0; i < ARRAY_SIZE(sld_options); i++) {
124 			if (match_option(arg, ret, sld_options[i].option)) {
125 				state = sld_options[i].state;
126 				break;
127 			}
128 		}
129 	}
130 	sld_state = state;
131 }
132 
__split_lock_setup(void)133 static void __init __split_lock_setup(void)
134 {
135 	if (!split_lock_verify_msr(false)) {
136 		pr_info("MSR access failed: Disabled\n");
137 		return;
138 	}
139 
140 	rdmsrl(MSR_TEST_CTRL, msr_test_ctrl_cache);
141 
142 	if (!split_lock_verify_msr(true)) {
143 		pr_info("MSR access failed: Disabled\n");
144 		return;
145 	}
146 
147 	/* Restore the MSR to its cached value. */
148 	wrmsrl(MSR_TEST_CTRL, msr_test_ctrl_cache);
149 
150 	setup_force_cpu_cap(X86_FEATURE_SPLIT_LOCK_DETECT);
151 }
152 
153 /*
154  * MSR_TEST_CTRL is per core, but we treat it like a per CPU MSR. Locking
155  * is not implemented as one thread could undo the setting of the other
156  * thread immediately after dropping the lock anyway.
157  */
sld_update_msr(bool on)158 static void sld_update_msr(bool on)
159 {
160 	u64 test_ctrl_val = msr_test_ctrl_cache;
161 
162 	if (on)
163 		test_ctrl_val |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT;
164 
165 	wrmsrl(MSR_TEST_CTRL, test_ctrl_val);
166 }
167 
split_lock_init(void)168 void split_lock_init(void)
169 {
170 	/*
171 	 * #DB for bus lock handles ratelimit and #AC for split lock is
172 	 * disabled.
173 	 */
174 	if (sld_state == sld_ratelimit) {
175 		split_lock_verify_msr(false);
176 		return;
177 	}
178 
179 	if (cpu_model_supports_sld)
180 		split_lock_verify_msr(sld_state != sld_off);
181 }
182 
__split_lock_reenable_unlock(struct work_struct * work)183 static void __split_lock_reenable_unlock(struct work_struct *work)
184 {
185 	sld_update_msr(true);
186 	up(&buslock_sem);
187 }
188 
189 static DECLARE_DELAYED_WORK(sl_reenable_unlock, __split_lock_reenable_unlock);
190 
__split_lock_reenable(struct work_struct * work)191 static void __split_lock_reenable(struct work_struct *work)
192 {
193 	sld_update_msr(true);
194 }
195 /*
196  * In order for each CPU to schedule its delayed work independently of the
197  * others, delayed work struct must be per-CPU. This is not required when
198  * sysctl_sld_mitigate is enabled because of the semaphore that limits
199  * the number of simultaneously scheduled delayed works to 1.
200  */
201 static DEFINE_PER_CPU(struct delayed_work, sl_reenable);
202 
203 /*
204  * Per-CPU delayed_work can't be statically initialized properly because
205  * the struct address is unknown. Thus per-CPU delayed_work structures
206  * have to be initialized during kernel initialization and after calling
207  * setup_per_cpu_areas().
208  */
setup_split_lock_delayed_work(void)209 static int __init setup_split_lock_delayed_work(void)
210 {
211 	unsigned int cpu;
212 
213 	for_each_possible_cpu(cpu) {
214 		struct delayed_work *work = per_cpu_ptr(&sl_reenable, cpu);
215 
216 		INIT_DELAYED_WORK(work, __split_lock_reenable);
217 	}
218 
219 	return 0;
220 }
221 pure_initcall(setup_split_lock_delayed_work);
222 
223 /*
224  * If a CPU goes offline with pending delayed work to re-enable split lock
225  * detection then the delayed work will be executed on some other CPU. That
226  * handles releasing the buslock_sem, but because it executes on a
227  * different CPU probably won't re-enable split lock detection. This is a
228  * problem on HT systems since the sibling CPU on the same core may then be
229  * left running with split lock detection disabled.
230  *
231  * Unconditionally re-enable detection here.
232  */
splitlock_cpu_offline(unsigned int cpu)233 static int splitlock_cpu_offline(unsigned int cpu)
234 {
235 	sld_update_msr(true);
236 
237 	return 0;
238 }
239 
split_lock_warn(unsigned long ip)240 static void split_lock_warn(unsigned long ip)
241 {
242 	struct delayed_work *work;
243 	int cpu;
244 	unsigned int saved_sld_mitigate = READ_ONCE(sysctl_sld_mitigate);
245 
246 	if (!current->reported_split_lock)
247 		pr_warn_ratelimited("#AC: %s/%d took a split_lock trap at address: 0x%lx\n",
248 				    current->comm, current->pid, ip);
249 	current->reported_split_lock = 1;
250 
251 	if (saved_sld_mitigate) {
252 		/*
253 		 * misery factor #1:
254 		 * sleep 10ms before trying to execute split lock.
255 		 */
256 		if (msleep_interruptible(10) > 0)
257 			return;
258 		/*
259 		 * Misery factor #2:
260 		 * only allow one buslocked disabled core at a time.
261 		 */
262 		if (down_interruptible(&buslock_sem) == -EINTR)
263 			return;
264 	}
265 
266 	cpu = get_cpu();
267 	work = saved_sld_mitigate ? &sl_reenable_unlock : per_cpu_ptr(&sl_reenable, cpu);
268 	schedule_delayed_work_on(cpu, work, 2);
269 
270 	/* Disable split lock detection on this CPU to make progress */
271 	sld_update_msr(false);
272 	put_cpu();
273 }
274 
handle_guest_split_lock(unsigned long ip)275 bool handle_guest_split_lock(unsigned long ip)
276 {
277 	if (sld_state == sld_warn) {
278 		split_lock_warn(ip);
279 		return true;
280 	}
281 
282 	pr_warn_once("#AC: %s/%d %s split_lock trap at address: 0x%lx\n",
283 		     current->comm, current->pid,
284 		     sld_state == sld_fatal ? "fatal" : "bogus", ip);
285 
286 	current->thread.error_code = 0;
287 	current->thread.trap_nr = X86_TRAP_AC;
288 	force_sig_fault(SIGBUS, BUS_ADRALN, NULL);
289 	return false;
290 }
291 EXPORT_SYMBOL_GPL(handle_guest_split_lock);
292 
bus_lock_init(void)293 void bus_lock_init(void)
294 {
295 	u64 val;
296 
297 	if (!boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT))
298 		return;
299 
300 	rdmsrl(MSR_IA32_DEBUGCTLMSR, val);
301 
302 	if ((boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT) &&
303 	    (sld_state == sld_warn || sld_state == sld_fatal)) ||
304 	    sld_state == sld_off) {
305 		/*
306 		 * Warn and fatal are handled by #AC for split lock if #AC for
307 		 * split lock is supported.
308 		 */
309 		val &= ~DEBUGCTLMSR_BUS_LOCK_DETECT;
310 	} else {
311 		val |= DEBUGCTLMSR_BUS_LOCK_DETECT;
312 	}
313 
314 	wrmsrl(MSR_IA32_DEBUGCTLMSR, val);
315 }
316 
handle_user_split_lock(struct pt_regs * regs,long error_code)317 bool handle_user_split_lock(struct pt_regs *regs, long error_code)
318 {
319 	if ((regs->flags & X86_EFLAGS_AC) || sld_state == sld_fatal)
320 		return false;
321 	split_lock_warn(regs->ip);
322 	return true;
323 }
324 
handle_bus_lock(struct pt_regs * regs)325 void handle_bus_lock(struct pt_regs *regs)
326 {
327 	switch (sld_state) {
328 	case sld_off:
329 		break;
330 	case sld_ratelimit:
331 		/* Enforce no more than bld_ratelimit bus locks/sec. */
332 		while (!__ratelimit(&bld_ratelimit))
333 			msleep(20);
334 		/* Warn on the bus lock. */
335 		fallthrough;
336 	case sld_warn:
337 		pr_warn_ratelimited("#DB: %s/%d took a bus_lock trap at address: 0x%lx\n",
338 				    current->comm, current->pid, regs->ip);
339 		break;
340 	case sld_fatal:
341 		force_sig_fault(SIGBUS, BUS_ADRALN, NULL);
342 		break;
343 	}
344 }
345 
346 /*
347  * CPU models that are known to have the per-core split-lock detection
348  * feature even though they do not enumerate IA32_CORE_CAPABILITIES.
349  */
350 static const struct x86_cpu_id split_lock_cpu_ids[] __initconst = {
351 	X86_MATCH_VFM(INTEL_ICELAKE_X,	0),
352 	X86_MATCH_VFM(INTEL_ICELAKE_L,	0),
353 	X86_MATCH_VFM(INTEL_ICELAKE_D,	0),
354 	{}
355 };
356 
split_lock_setup(struct cpuinfo_x86 * c)357 static void __init split_lock_setup(struct cpuinfo_x86 *c)
358 {
359 	const struct x86_cpu_id *m;
360 	u64 ia32_core_caps;
361 
362 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
363 		return;
364 
365 	/* Check for CPUs that have support but do not enumerate it: */
366 	m = x86_match_cpu(split_lock_cpu_ids);
367 	if (m)
368 		goto supported;
369 
370 	if (!cpu_has(c, X86_FEATURE_CORE_CAPABILITIES))
371 		return;
372 
373 	/*
374 	 * Not all bits in MSR_IA32_CORE_CAPS are architectural, but
375 	 * MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT is.  All CPUs that set
376 	 * it have split lock detection.
377 	 */
378 	rdmsrl(MSR_IA32_CORE_CAPS, ia32_core_caps);
379 	if (ia32_core_caps & MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT)
380 		goto supported;
381 
382 	/* CPU is not in the model list and does not have the MSR bit: */
383 	return;
384 
385 supported:
386 	cpu_model_supports_sld = true;
387 	__split_lock_setup();
388 }
389 
sld_state_show(void)390 static void sld_state_show(void)
391 {
392 	if (!boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) &&
393 	    !boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
394 		return;
395 
396 	switch (sld_state) {
397 	case sld_off:
398 		pr_info("disabled\n");
399 		break;
400 	case sld_warn:
401 		if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) {
402 			pr_info("#AC: crashing the kernel on kernel split_locks and warning on user-space split_locks\n");
403 			if (cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
404 					      "x86/splitlock", NULL, splitlock_cpu_offline) < 0)
405 				pr_warn("No splitlock CPU offline handler\n");
406 		} else if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT)) {
407 			pr_info("#DB: warning on user-space bus_locks\n");
408 		}
409 		break;
410 	case sld_fatal:
411 		if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) {
412 			pr_info("#AC: crashing the kernel on kernel split_locks and sending SIGBUS on user-space split_locks\n");
413 		} else if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT)) {
414 			pr_info("#DB: sending SIGBUS on user-space bus_locks%s\n",
415 				boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT) ?
416 				" from non-WB" : "");
417 		}
418 		break;
419 	case sld_ratelimit:
420 		if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT))
421 			pr_info("#DB: setting system wide bus lock rate limit to %u/sec\n", bld_ratelimit.burst);
422 		break;
423 	}
424 }
425 
sld_setup(struct cpuinfo_x86 * c)426 void __init sld_setup(struct cpuinfo_x86 *c)
427 {
428 	split_lock_setup(c);
429 	sld_state_setup();
430 	sld_state_show();
431 }
432