1 // SPDX-License-Identifier: GPL-2.0-only
2 /* cpu_feature_enabled() cannot be used this early */
3 #define USE_EARLY_PGTABLE_L5
4 
5 #include <linux/memblock.h>
6 #include <linux/linkage.h>
7 #include <linux/bitops.h>
8 #include <linux/kernel.h>
9 #include <linux/export.h>
10 #include <linux/percpu.h>
11 #include <linux/string.h>
12 #include <linux/ctype.h>
13 #include <linux/delay.h>
14 #include <linux/sched/mm.h>
15 #include <linux/sched/clock.h>
16 #include <linux/sched/task.h>
17 #include <linux/sched/smt.h>
18 #include <linux/init.h>
19 #include <linux/kprobes.h>
20 #include <linux/kgdb.h>
21 #include <linux/mem_encrypt.h>
22 #include <linux/smp.h>
23 #include <linux/cpu.h>
24 #include <linux/io.h>
25 #include <linux/syscore_ops.h>
26 #include <linux/pgtable.h>
27 #include <linux/stackprotector.h>
28 #include <linux/utsname.h>
29 
30 #include <asm/alternative.h>
31 #include <asm/cmdline.h>
32 #include <asm/cpuid.h>
33 #include <asm/perf_event.h>
34 #include <asm/mmu_context.h>
35 #include <asm/doublefault.h>
36 #include <asm/archrandom.h>
37 #include <asm/hypervisor.h>
38 #include <asm/processor.h>
39 #include <asm/tlbflush.h>
40 #include <asm/debugreg.h>
41 #include <asm/sections.h>
42 #include <asm/vsyscall.h>
43 #include <linux/topology.h>
44 #include <linux/cpumask.h>
45 #include <linux/atomic.h>
46 #include <asm/proto.h>
47 #include <asm/setup.h>
48 #include <asm/apic.h>
49 #include <asm/desc.h>
50 #include <asm/fpu/api.h>
51 #include <asm/mtrr.h>
52 #include <asm/hwcap2.h>
53 #include <linux/numa.h>
54 #include <asm/numa.h>
55 #include <asm/asm.h>
56 #include <asm/bugs.h>
57 #include <asm/cpu.h>
58 #include <asm/mce.h>
59 #include <asm/msr.h>
60 #include <asm/cacheinfo.h>
61 #include <asm/memtype.h>
62 #include <asm/microcode.h>
63 #include <asm/intel-family.h>
64 #include <asm/cpu_device_id.h>
65 #include <asm/fred.h>
66 #include <asm/uv/uv.h>
67 #include <asm/ia32.h>
68 #include <asm/set_memory.h>
69 #include <asm/traps.h>
70 #include <asm/sev.h>
71 #include <asm/tdx.h>
72 #include <asm/posted_intr.h>
73 #include <asm/runtime-const.h>
74 
75 #include "cpu.h"
76 
77 DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
78 EXPORT_PER_CPU_SYMBOL(cpu_info);
79 
80 u32 elf_hwcap2 __read_mostly;
81 
82 /* Number of siblings per CPU package */
83 unsigned int __max_threads_per_core __ro_after_init = 1;
84 EXPORT_SYMBOL(__max_threads_per_core);
85 
86 unsigned int __max_dies_per_package __ro_after_init = 1;
87 EXPORT_SYMBOL(__max_dies_per_package);
88 
89 unsigned int __max_logical_packages __ro_after_init = 1;
90 EXPORT_SYMBOL(__max_logical_packages);
91 
92 unsigned int __num_cores_per_package __ro_after_init = 1;
93 EXPORT_SYMBOL(__num_cores_per_package);
94 
95 unsigned int __num_threads_per_package __ro_after_init = 1;
96 EXPORT_SYMBOL(__num_threads_per_package);
97 
98 static struct ppin_info {
99 	int	feature;
100 	int	msr_ppin_ctl;
101 	int	msr_ppin;
102 } ppin_info[] = {
103 	[X86_VENDOR_INTEL] = {
104 		.feature = X86_FEATURE_INTEL_PPIN,
105 		.msr_ppin_ctl = MSR_PPIN_CTL,
106 		.msr_ppin = MSR_PPIN
107 	},
108 	[X86_VENDOR_AMD] = {
109 		.feature = X86_FEATURE_AMD_PPIN,
110 		.msr_ppin_ctl = MSR_AMD_PPIN_CTL,
111 		.msr_ppin = MSR_AMD_PPIN
112 	},
113 };
114 
115 static const struct x86_cpu_id ppin_cpuids[] = {
116 	X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
117 	X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
118 
119 	/* Legacy models without CPUID enumeration */
120 	X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
121 	X86_MATCH_VFM(INTEL_HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
122 	X86_MATCH_VFM(INTEL_BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
123 	X86_MATCH_VFM(INTEL_BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
124 	X86_MATCH_VFM(INTEL_SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
125 	X86_MATCH_VFM(INTEL_ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
126 	X86_MATCH_VFM(INTEL_ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
127 	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
128 	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
129 	X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
130 	X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
131 
132 	{}
133 };
134 
ppin_init(struct cpuinfo_x86 * c)135 static void ppin_init(struct cpuinfo_x86 *c)
136 {
137 	const struct x86_cpu_id *id;
138 	unsigned long long val;
139 	struct ppin_info *info;
140 
141 	id = x86_match_cpu(ppin_cpuids);
142 	if (!id)
143 		return;
144 
145 	/*
146 	 * Testing the presence of the MSR is not enough. Need to check
147 	 * that the PPIN_CTL allows reading of the PPIN.
148 	 */
149 	info = (struct ppin_info *)id->driver_data;
150 
151 	if (rdmsrl_safe(info->msr_ppin_ctl, &val))
152 		goto clear_ppin;
153 
154 	if ((val & 3UL) == 1UL) {
155 		/* PPIN locked in disabled mode */
156 		goto clear_ppin;
157 	}
158 
159 	/* If PPIN is disabled, try to enable */
160 	if (!(val & 2UL)) {
161 		wrmsrl_safe(info->msr_ppin_ctl,  val | 2UL);
162 		rdmsrl_safe(info->msr_ppin_ctl, &val);
163 	}
164 
165 	/* Is the enable bit set? */
166 	if (val & 2UL) {
167 		c->ppin = __rdmsr(info->msr_ppin);
168 		set_cpu_cap(c, info->feature);
169 		return;
170 	}
171 
172 clear_ppin:
173 	setup_clear_cpu_cap(info->feature);
174 }
175 
default_init(struct cpuinfo_x86 * c)176 static void default_init(struct cpuinfo_x86 *c)
177 {
178 #ifdef CONFIG_X86_64
179 	cpu_detect_cache_sizes(c);
180 #else
181 	/* Not much we can do here... */
182 	/* Check if at least it has cpuid */
183 	if (c->cpuid_level == -1) {
184 		/* No cpuid. It must be an ancient CPU */
185 		if (c->x86 == 4)
186 			strcpy(c->x86_model_id, "486");
187 		else if (c->x86 == 3)
188 			strcpy(c->x86_model_id, "386");
189 	}
190 #endif
191 }
192 
193 static const struct cpu_dev default_cpu = {
194 	.c_init		= default_init,
195 	.c_vendor	= "Unknown",
196 	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
197 };
198 
199 static const struct cpu_dev *this_cpu = &default_cpu;
200 
201 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
202 #ifdef CONFIG_X86_64
203 	/*
204 	 * We need valid kernel segments for data and code in long mode too
205 	 * IRET will check the segment types  kkeil 2000/10/28
206 	 * Also sysret mandates a special GDT layout
207 	 *
208 	 * TLS descriptors are currently at a different place compared to i386.
209 	 * Hopefully nobody expects them at a fixed place (Wine?)
210 	 */
211 	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
212 	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),
213 	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),
214 	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
215 	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),
216 	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),
217 #else
218 	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
219 	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
220 	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
221 	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),
222 	/*
223 	 * Segments used for calling PnP BIOS have byte granularity.
224 	 * They code segments and data segments have fixed 64k limits,
225 	 * the transfer segment sizes are set at run time.
226 	 */
227 	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
228 	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
229 	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),
230 	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
231 	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
232 	/*
233 	 * The APM segments have byte granularity and their bases
234 	 * are set at run time.  All have 64k limits.
235 	 */
236 	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
237 	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
238 	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff),
239 
240 	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
241 	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
242 #endif
243 } };
244 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
245 
246 #ifdef CONFIG_X86_64
x86_nopcid_setup(char * s)247 static int __init x86_nopcid_setup(char *s)
248 {
249 	/* nopcid doesn't accept parameters */
250 	if (s)
251 		return -EINVAL;
252 
253 	/* do not emit a message if the feature is not present */
254 	if (!boot_cpu_has(X86_FEATURE_PCID))
255 		return 0;
256 
257 	setup_clear_cpu_cap(X86_FEATURE_PCID);
258 	pr_info("nopcid: PCID feature disabled\n");
259 	return 0;
260 }
261 early_param("nopcid", x86_nopcid_setup);
262 #endif
263 
x86_noinvpcid_setup(char * s)264 static int __init x86_noinvpcid_setup(char *s)
265 {
266 	/* noinvpcid doesn't accept parameters */
267 	if (s)
268 		return -EINVAL;
269 
270 	/* do not emit a message if the feature is not present */
271 	if (!boot_cpu_has(X86_FEATURE_INVPCID))
272 		return 0;
273 
274 	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
275 	pr_info("noinvpcid: INVPCID feature disabled\n");
276 	return 0;
277 }
278 early_param("noinvpcid", x86_noinvpcid_setup);
279 
280 /* Standard macro to see if a specific flag is changeable */
flag_is_changeable_p(unsigned long flag)281 static inline bool flag_is_changeable_p(unsigned long flag)
282 {
283 	unsigned long f1, f2;
284 
285 	if (!IS_ENABLED(CONFIG_X86_32))
286 		return true;
287 
288 	/*
289 	 * Cyrix and IDT cpus allow disabling of CPUID
290 	 * so the code below may return different results
291 	 * when it is executed before and after enabling
292 	 * the CPUID. Add "volatile" to not allow gcc to
293 	 * optimize the subsequent calls to this function.
294 	 */
295 	asm volatile ("pushfl		\n\t"
296 		      "pushfl		\n\t"
297 		      "popl %0		\n\t"
298 		      "movl %0, %1	\n\t"
299 		      "xorl %2, %0	\n\t"
300 		      "pushl %0		\n\t"
301 		      "popfl		\n\t"
302 		      "pushfl		\n\t"
303 		      "popl %0		\n\t"
304 		      "popfl		\n\t"
305 
306 		      : "=&r" (f1), "=&r" (f2)
307 		      : "ir" (flag));
308 
309 	return (f1 ^ f2) & flag;
310 }
311 
312 #ifdef CONFIG_X86_32
313 static int cachesize_override = -1;
314 static int disable_x86_serial_nr = 1;
315 
cachesize_setup(char * str)316 static int __init cachesize_setup(char *str)
317 {
318 	get_option(&str, &cachesize_override);
319 	return 1;
320 }
321 __setup("cachesize=", cachesize_setup);
322 
323 /* Probe for the CPUID instruction */
have_cpuid_p(void)324 bool have_cpuid_p(void)
325 {
326 	return flag_is_changeable_p(X86_EFLAGS_ID);
327 }
328 
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)329 static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
330 {
331 	unsigned long lo, hi;
332 
333 	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
334 		return;
335 
336 	/* Disable processor serial number: */
337 
338 	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
339 	lo |= 0x200000;
340 	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
341 
342 	pr_notice("CPU serial number disabled.\n");
343 	clear_cpu_cap(c, X86_FEATURE_PN);
344 
345 	/* Disabling the serial number may affect the cpuid level */
346 	c->cpuid_level = cpuid_eax(0);
347 }
348 
x86_serial_nr_setup(char * s)349 static int __init x86_serial_nr_setup(char *s)
350 {
351 	disable_x86_serial_nr = 0;
352 	return 1;
353 }
354 __setup("serialnumber", x86_serial_nr_setup);
355 #else
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)356 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
357 {
358 }
359 #endif
360 
setup_smep(struct cpuinfo_x86 * c)361 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
362 {
363 	if (cpu_has(c, X86_FEATURE_SMEP))
364 		cr4_set_bits(X86_CR4_SMEP);
365 }
366 
setup_smap(struct cpuinfo_x86 * c)367 static __always_inline void setup_smap(struct cpuinfo_x86 *c)
368 {
369 	unsigned long eflags = native_save_fl();
370 
371 	/* This should have been cleared long ago */
372 	BUG_ON(eflags & X86_EFLAGS_AC);
373 
374 	if (cpu_has(c, X86_FEATURE_SMAP))
375 		cr4_set_bits(X86_CR4_SMAP);
376 }
377 
setup_umip(struct cpuinfo_x86 * c)378 static __always_inline void setup_umip(struct cpuinfo_x86 *c)
379 {
380 	/* Check the boot processor, plus build option for UMIP. */
381 	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
382 		goto out;
383 
384 	/* Check the current processor's cpuid bits. */
385 	if (!cpu_has(c, X86_FEATURE_UMIP))
386 		goto out;
387 
388 	cr4_set_bits(X86_CR4_UMIP);
389 
390 	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
391 
392 	return;
393 
394 out:
395 	/*
396 	 * Make sure UMIP is disabled in case it was enabled in a
397 	 * previous boot (e.g., via kexec).
398 	 */
399 	cr4_clear_bits(X86_CR4_UMIP);
400 }
401 
402 /* These bits should not change their value after CPU init is finished. */
403 static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
404 					     X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED;
405 static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
406 static unsigned long cr4_pinned_bits __ro_after_init;
407 
native_write_cr0(unsigned long val)408 void native_write_cr0(unsigned long val)
409 {
410 	unsigned long bits_missing = 0;
411 
412 set_register:
413 	asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
414 
415 	if (static_branch_likely(&cr_pinning)) {
416 		if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
417 			bits_missing = X86_CR0_WP;
418 			val |= bits_missing;
419 			goto set_register;
420 		}
421 		/* Warn after we've set the missing bits. */
422 		WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
423 	}
424 }
425 EXPORT_SYMBOL(native_write_cr0);
426 
native_write_cr4(unsigned long val)427 void __no_profile native_write_cr4(unsigned long val)
428 {
429 	unsigned long bits_changed = 0;
430 
431 set_register:
432 	asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
433 
434 	if (static_branch_likely(&cr_pinning)) {
435 		if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
436 			bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
437 			val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
438 			goto set_register;
439 		}
440 		/* Warn after we've corrected the changed bits. */
441 		WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
442 			  bits_changed);
443 	}
444 }
445 #if IS_MODULE(CONFIG_LKDTM)
446 EXPORT_SYMBOL_GPL(native_write_cr4);
447 #endif
448 
cr4_update_irqsoff(unsigned long set,unsigned long clear)449 void cr4_update_irqsoff(unsigned long set, unsigned long clear)
450 {
451 	unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
452 
453 	lockdep_assert_irqs_disabled();
454 
455 	newval = (cr4 & ~clear) | set;
456 	if (newval != cr4) {
457 		this_cpu_write(cpu_tlbstate.cr4, newval);
458 		__write_cr4(newval);
459 	}
460 }
461 EXPORT_SYMBOL(cr4_update_irqsoff);
462 
463 /* Read the CR4 shadow. */
cr4_read_shadow(void)464 unsigned long cr4_read_shadow(void)
465 {
466 	return this_cpu_read(cpu_tlbstate.cr4);
467 }
468 EXPORT_SYMBOL_GPL(cr4_read_shadow);
469 
cr4_init(void)470 void cr4_init(void)
471 {
472 	unsigned long cr4 = __read_cr4();
473 
474 	if (boot_cpu_has(X86_FEATURE_PCID))
475 		cr4 |= X86_CR4_PCIDE;
476 	if (static_branch_likely(&cr_pinning))
477 		cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
478 
479 	__write_cr4(cr4);
480 
481 	/* Initialize cr4 shadow for this CPU. */
482 	this_cpu_write(cpu_tlbstate.cr4, cr4);
483 }
484 
485 /*
486  * Once CPU feature detection is finished (and boot params have been
487  * parsed), record any of the sensitive CR bits that are set, and
488  * enable CR pinning.
489  */
setup_cr_pinning(void)490 static void __init setup_cr_pinning(void)
491 {
492 	cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
493 	static_key_enable(&cr_pinning.key);
494 }
495 
x86_nofsgsbase_setup(char * arg)496 static __init int x86_nofsgsbase_setup(char *arg)
497 {
498 	/* Require an exact match without trailing characters. */
499 	if (strlen(arg))
500 		return 0;
501 
502 	/* Do not emit a message if the feature is not present. */
503 	if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
504 		return 1;
505 
506 	setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
507 	pr_info("FSGSBASE disabled via kernel command line\n");
508 	return 1;
509 }
510 __setup("nofsgsbase", x86_nofsgsbase_setup);
511 
512 /*
513  * Protection Keys are not available in 32-bit mode.
514  */
515 static bool pku_disabled;
516 
setup_pku(struct cpuinfo_x86 * c)517 static __always_inline void setup_pku(struct cpuinfo_x86 *c)
518 {
519 	if (c == &boot_cpu_data) {
520 		if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
521 			return;
522 		/*
523 		 * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
524 		 * bit to be set.  Enforce it.
525 		 */
526 		setup_force_cpu_cap(X86_FEATURE_OSPKE);
527 
528 	} else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
529 		return;
530 	}
531 
532 	cr4_set_bits(X86_CR4_PKE);
533 	/* Load the default PKRU value */
534 	pkru_write_default();
535 }
536 
537 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
setup_disable_pku(char * arg)538 static __init int setup_disable_pku(char *arg)
539 {
540 	/*
541 	 * Do not clear the X86_FEATURE_PKU bit.  All of the
542 	 * runtime checks are against OSPKE so clearing the
543 	 * bit does nothing.
544 	 *
545 	 * This way, we will see "pku" in cpuinfo, but not
546 	 * "ospke", which is exactly what we want.  It shows
547 	 * that the CPU has PKU, but the OS has not enabled it.
548 	 * This happens to be exactly how a system would look
549 	 * if we disabled the config option.
550 	 */
551 	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
552 	pku_disabled = true;
553 	return 1;
554 }
555 __setup("nopku", setup_disable_pku);
556 #endif
557 
558 #ifdef CONFIG_X86_KERNEL_IBT
559 
ibt_save(bool disable)560 __noendbr u64 ibt_save(bool disable)
561 {
562 	u64 msr = 0;
563 
564 	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
565 		rdmsrl(MSR_IA32_S_CET, msr);
566 		if (disable)
567 			wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
568 	}
569 
570 	return msr;
571 }
572 
ibt_restore(u64 save)573 __noendbr void ibt_restore(u64 save)
574 {
575 	u64 msr;
576 
577 	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
578 		rdmsrl(MSR_IA32_S_CET, msr);
579 		msr &= ~CET_ENDBR_EN;
580 		msr |= (save & CET_ENDBR_EN);
581 		wrmsrl(MSR_IA32_S_CET, msr);
582 	}
583 }
584 
585 #endif
586 
setup_cet(struct cpuinfo_x86 * c)587 static __always_inline void setup_cet(struct cpuinfo_x86 *c)
588 {
589 	bool user_shstk, kernel_ibt;
590 
591 	if (!IS_ENABLED(CONFIG_X86_CET))
592 		return;
593 
594 	kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
595 	user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
596 		     IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
597 
598 	if (!kernel_ibt && !user_shstk)
599 		return;
600 
601 	if (user_shstk)
602 		set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
603 
604 	if (kernel_ibt)
605 		wrmsrl(MSR_IA32_S_CET, CET_ENDBR_EN);
606 	else
607 		wrmsrl(MSR_IA32_S_CET, 0);
608 
609 	cr4_set_bits(X86_CR4_CET);
610 
611 	if (kernel_ibt && ibt_selftest()) {
612 		pr_err("IBT selftest: Failed!\n");
613 		wrmsrl(MSR_IA32_S_CET, 0);
614 		setup_clear_cpu_cap(X86_FEATURE_IBT);
615 	}
616 }
617 
cet_disable(void)618 __noendbr void cet_disable(void)
619 {
620 	if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
621 	      cpu_feature_enabled(X86_FEATURE_SHSTK)))
622 		return;
623 
624 	wrmsrl(MSR_IA32_S_CET, 0);
625 	wrmsrl(MSR_IA32_U_CET, 0);
626 }
627 
628 /*
629  * Some CPU features depend on higher CPUID levels, which may not always
630  * be available due to CPUID level capping or broken virtualization
631  * software.  Add those features to this table to auto-disable them.
632  */
633 struct cpuid_dependent_feature {
634 	u32 feature;
635 	u32 level;
636 };
637 
638 static const struct cpuid_dependent_feature
639 cpuid_dependent_features[] = {
640 	{ X86_FEATURE_MWAIT,		CPUID_LEAF_MWAIT },
641 	{ X86_FEATURE_DCA,		CPUID_LEAF_DCA },
642 	{ X86_FEATURE_XSAVE,		CPUID_LEAF_XSTATE },
643 	{ 0, 0 }
644 };
645 
filter_cpuid_features(struct cpuinfo_x86 * c,bool warn)646 static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
647 {
648 	const struct cpuid_dependent_feature *df;
649 
650 	for (df = cpuid_dependent_features; df->feature; df++) {
651 
652 		if (!cpu_has(c, df->feature))
653 			continue;
654 		/*
655 		 * Note: cpuid_level is set to -1 if unavailable, but
656 		 * extended_extended_level is set to 0 if unavailable
657 		 * and the legitimate extended levels are all negative
658 		 * when signed; hence the weird messing around with
659 		 * signs here...
660 		 */
661 		if (!((s32)df->level < 0 ?
662 		     (u32)df->level > (u32)c->extended_cpuid_level :
663 		     (s32)df->level > (s32)c->cpuid_level))
664 			continue;
665 
666 		clear_cpu_cap(c, df->feature);
667 		if (!warn)
668 			continue;
669 
670 		pr_warn("CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
671 			x86_cap_flags[df->feature], df->level);
672 	}
673 }
674 
675 /*
676  * Naming convention should be: <Name> [(<Codename>)]
677  * This table only is used unless init_<vendor>() below doesn't set it;
678  * in particular, if CPUID levels 0x80000002..4 are supported, this
679  * isn't used
680  */
681 
682 /* Look up CPU names by table lookup. */
table_lookup_model(struct cpuinfo_x86 * c)683 static const char *table_lookup_model(struct cpuinfo_x86 *c)
684 {
685 #ifdef CONFIG_X86_32
686 	const struct legacy_cpu_model_info *info;
687 
688 	if (c->x86_model >= 16)
689 		return NULL;	/* Range check */
690 
691 	if (!this_cpu)
692 		return NULL;
693 
694 	info = this_cpu->legacy_models;
695 
696 	while (info->family) {
697 		if (info->family == c->x86)
698 			return info->model_names[c->x86_model];
699 		info++;
700 	}
701 #endif
702 	return NULL;		/* Not found */
703 }
704 
705 /* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
706 __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
707 __u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
708 
709 #ifdef CONFIG_X86_32
710 /* The 32-bit entry code needs to find cpu_entry_area. */
711 DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
712 #endif
713 
714 /* Load the original GDT from the per-cpu structure */
load_direct_gdt(int cpu)715 void load_direct_gdt(int cpu)
716 {
717 	struct desc_ptr gdt_descr;
718 
719 	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
720 	gdt_descr.size = GDT_SIZE - 1;
721 	load_gdt(&gdt_descr);
722 }
723 EXPORT_SYMBOL_GPL(load_direct_gdt);
724 
725 /* Load a fixmap remapping of the per-cpu GDT */
load_fixmap_gdt(int cpu)726 void load_fixmap_gdt(int cpu)
727 {
728 	struct desc_ptr gdt_descr;
729 
730 	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
731 	gdt_descr.size = GDT_SIZE - 1;
732 	load_gdt(&gdt_descr);
733 }
734 EXPORT_SYMBOL_GPL(load_fixmap_gdt);
735 
736 /**
737  * switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
738  * @cpu:	The CPU number for which this is invoked
739  *
740  * Invoked during early boot to switch from early GDT and early per CPU to
741  * the direct GDT and the runtime per CPU area. On 32-bit the percpu base
742  * switch is implicit by loading the direct GDT. On 64bit this requires
743  * to update GSBASE.
744  */
switch_gdt_and_percpu_base(int cpu)745 void __init switch_gdt_and_percpu_base(int cpu)
746 {
747 	load_direct_gdt(cpu);
748 
749 #ifdef CONFIG_X86_64
750 	/*
751 	 * No need to load %gs. It is already correct.
752 	 *
753 	 * Writing %gs on 64bit would zero GSBASE which would make any per
754 	 * CPU operation up to the point of the wrmsrl() fault.
755 	 *
756 	 * Set GSBASE to the new offset. Until the wrmsrl() happens the
757 	 * early mapping is still valid. That means the GSBASE update will
758 	 * lose any prior per CPU data which was not copied over in
759 	 * setup_per_cpu_areas().
760 	 *
761 	 * This works even with stackprotector enabled because the
762 	 * per CPU stack canary is 0 in both per CPU areas.
763 	 */
764 	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
765 #else
766 	/*
767 	 * %fs is already set to __KERNEL_PERCPU, but after switching GDT
768 	 * it is required to load FS again so that the 'hidden' part is
769 	 * updated from the new GDT. Up to this point the early per CPU
770 	 * translation is active. Any content of the early per CPU data
771 	 * which was not copied over in setup_per_cpu_areas() is lost.
772 	 */
773 	loadsegment(fs, __KERNEL_PERCPU);
774 #endif
775 }
776 
777 static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
778 
get_model_name(struct cpuinfo_x86 * c)779 static void get_model_name(struct cpuinfo_x86 *c)
780 {
781 	unsigned int *v;
782 	char *p, *q, *s;
783 
784 	if (c->extended_cpuid_level < 0x80000004)
785 		return;
786 
787 	v = (unsigned int *)c->x86_model_id;
788 	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
789 	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
790 	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
791 	c->x86_model_id[48] = 0;
792 
793 	/* Trim whitespace */
794 	p = q = s = &c->x86_model_id[0];
795 
796 	while (*p == ' ')
797 		p++;
798 
799 	while (*p) {
800 		/* Note the last non-whitespace index */
801 		if (!isspace(*p))
802 			s = q;
803 
804 		*q++ = *p++;
805 	}
806 
807 	*(s + 1) = '\0';
808 }
809 
cpu_detect_cache_sizes(struct cpuinfo_x86 * c)810 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
811 {
812 	unsigned int n, dummy, ebx, ecx, edx, l2size;
813 
814 	n = c->extended_cpuid_level;
815 
816 	if (n >= 0x80000005) {
817 		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
818 		c->x86_cache_size = (ecx>>24) + (edx>>24);
819 #ifdef CONFIG_X86_64
820 		/* On K8 L1 TLB is inclusive, so don't count it */
821 		c->x86_tlbsize = 0;
822 #endif
823 	}
824 
825 	if (n < 0x80000006)	/* Some chips just has a large L1. */
826 		return;
827 
828 	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
829 	l2size = ecx >> 16;
830 
831 #ifdef CONFIG_X86_64
832 	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
833 #else
834 	/* do processor-specific cache resizing */
835 	if (this_cpu->legacy_cache_size)
836 		l2size = this_cpu->legacy_cache_size(c, l2size);
837 
838 	/* Allow user to override all this if necessary. */
839 	if (cachesize_override != -1)
840 		l2size = cachesize_override;
841 
842 	if (l2size == 0)
843 		return;		/* Again, no L2 cache is possible */
844 #endif
845 
846 	c->x86_cache_size = l2size;
847 }
848 
849 u16 __read_mostly tlb_lli_4k;
850 u16 __read_mostly tlb_lli_2m;
851 u16 __read_mostly tlb_lli_4m;
852 u16 __read_mostly tlb_lld_4k;
853 u16 __read_mostly tlb_lld_2m;
854 u16 __read_mostly tlb_lld_4m;
855 u16 __read_mostly tlb_lld_1g;
856 
cpu_detect_tlb(struct cpuinfo_x86 * c)857 static void cpu_detect_tlb(struct cpuinfo_x86 *c)
858 {
859 	if (this_cpu->c_detect_tlb)
860 		this_cpu->c_detect_tlb(c);
861 
862 	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
863 		tlb_lli_4k, tlb_lli_2m, tlb_lli_4m);
864 
865 	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
866 		tlb_lld_4k, tlb_lld_2m, tlb_lld_4m, tlb_lld_1g);
867 }
868 
get_cpu_vendor(struct cpuinfo_x86 * c)869 void get_cpu_vendor(struct cpuinfo_x86 *c)
870 {
871 	char *v = c->x86_vendor_id;
872 	int i;
873 
874 	for (i = 0; i < X86_VENDOR_NUM; i++) {
875 		if (!cpu_devs[i])
876 			break;
877 
878 		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
879 		    (cpu_devs[i]->c_ident[1] &&
880 		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
881 
882 			this_cpu = cpu_devs[i];
883 			c->x86_vendor = this_cpu->c_x86_vendor;
884 			return;
885 		}
886 	}
887 
888 	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
889 		    "CPU: Your system may be unstable.\n", v);
890 
891 	c->x86_vendor = X86_VENDOR_UNKNOWN;
892 	this_cpu = &default_cpu;
893 }
894 
cpu_detect(struct cpuinfo_x86 * c)895 void cpu_detect(struct cpuinfo_x86 *c)
896 {
897 	/* Get vendor name */
898 	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
899 	      (unsigned int *)&c->x86_vendor_id[0],
900 	      (unsigned int *)&c->x86_vendor_id[8],
901 	      (unsigned int *)&c->x86_vendor_id[4]);
902 
903 	c->x86 = 4;
904 	/* Intel-defined flags: level 0x00000001 */
905 	if (c->cpuid_level >= 0x00000001) {
906 		u32 junk, tfms, cap0, misc;
907 
908 		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
909 		c->x86		= x86_family(tfms);
910 		c->x86_model	= x86_model(tfms);
911 		c->x86_stepping	= x86_stepping(tfms);
912 
913 		if (cap0 & (1<<19)) {
914 			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
915 			c->x86_cache_alignment = c->x86_clflush_size;
916 		}
917 	}
918 }
919 
apply_forced_caps(struct cpuinfo_x86 * c)920 static void apply_forced_caps(struct cpuinfo_x86 *c)
921 {
922 	int i;
923 
924 	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
925 		c->x86_capability[i] &= ~cpu_caps_cleared[i];
926 		c->x86_capability[i] |= cpu_caps_set[i];
927 	}
928 }
929 
init_speculation_control(struct cpuinfo_x86 * c)930 static void init_speculation_control(struct cpuinfo_x86 *c)
931 {
932 	/*
933 	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
934 	 * and they also have a different bit for STIBP support. Also,
935 	 * a hypervisor might have set the individual AMD bits even on
936 	 * Intel CPUs, for finer-grained selection of what's available.
937 	 */
938 	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
939 		set_cpu_cap(c, X86_FEATURE_IBRS);
940 		set_cpu_cap(c, X86_FEATURE_IBPB);
941 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
942 	}
943 
944 	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
945 		set_cpu_cap(c, X86_FEATURE_STIBP);
946 
947 	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
948 	    cpu_has(c, X86_FEATURE_VIRT_SSBD))
949 		set_cpu_cap(c, X86_FEATURE_SSBD);
950 
951 	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
952 		set_cpu_cap(c, X86_FEATURE_IBRS);
953 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
954 	}
955 
956 	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
957 		set_cpu_cap(c, X86_FEATURE_IBPB);
958 
959 	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
960 		set_cpu_cap(c, X86_FEATURE_STIBP);
961 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
962 	}
963 
964 	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
965 		set_cpu_cap(c, X86_FEATURE_SSBD);
966 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
967 		clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
968 	}
969 }
970 
get_cpu_cap(struct cpuinfo_x86 * c)971 void get_cpu_cap(struct cpuinfo_x86 *c)
972 {
973 	u32 eax, ebx, ecx, edx;
974 
975 	/* Intel-defined flags: level 0x00000001 */
976 	if (c->cpuid_level >= 0x00000001) {
977 		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
978 
979 		c->x86_capability[CPUID_1_ECX] = ecx;
980 		c->x86_capability[CPUID_1_EDX] = edx;
981 	}
982 
983 	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
984 	if (c->cpuid_level >= 0x00000006)
985 		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
986 
987 	/* Additional Intel-defined flags: level 0x00000007 */
988 	if (c->cpuid_level >= 0x00000007) {
989 		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
990 		c->x86_capability[CPUID_7_0_EBX] = ebx;
991 		c->x86_capability[CPUID_7_ECX] = ecx;
992 		c->x86_capability[CPUID_7_EDX] = edx;
993 
994 		/* Check valid sub-leaf index before accessing it */
995 		if (eax >= 1) {
996 			cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
997 			c->x86_capability[CPUID_7_1_EAX] = eax;
998 		}
999 	}
1000 
1001 	/* Extended state features: level 0x0000000d */
1002 	if (c->cpuid_level >= 0x0000000d) {
1003 		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
1004 
1005 		c->x86_capability[CPUID_D_1_EAX] = eax;
1006 	}
1007 
1008 	/* AMD-defined flags: level 0x80000001 */
1009 	eax = cpuid_eax(0x80000000);
1010 	c->extended_cpuid_level = eax;
1011 
1012 	if ((eax & 0xffff0000) == 0x80000000) {
1013 		if (eax >= 0x80000001) {
1014 			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
1015 
1016 			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1017 			c->x86_capability[CPUID_8000_0001_EDX] = edx;
1018 		}
1019 	}
1020 
1021 	if (c->extended_cpuid_level >= 0x80000007) {
1022 		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
1023 
1024 		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1025 		c->x86_power = edx;
1026 	}
1027 
1028 	if (c->extended_cpuid_level >= 0x80000008) {
1029 		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1030 		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1031 	}
1032 
1033 	if (c->extended_cpuid_level >= 0x8000000a)
1034 		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
1035 
1036 	if (c->extended_cpuid_level >= 0x8000001f)
1037 		c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
1038 
1039 	if (c->extended_cpuid_level >= 0x80000021)
1040 		c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
1041 
1042 	init_scattered_cpuid_features(c);
1043 	init_speculation_control(c);
1044 
1045 	/*
1046 	 * Clear/Set all flags overridden by options, after probe.
1047 	 * This needs to happen each time we re-probe, which may happen
1048 	 * several times during CPU initialization.
1049 	 */
1050 	apply_forced_caps(c);
1051 }
1052 
get_cpu_address_sizes(struct cpuinfo_x86 * c)1053 void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1054 {
1055 	u32 eax, ebx, ecx, edx;
1056 
1057 	if (!cpu_has(c, X86_FEATURE_CPUID) ||
1058 	    (c->extended_cpuid_level < 0x80000008)) {
1059 		if (IS_ENABLED(CONFIG_X86_64)) {
1060 			c->x86_clflush_size = 64;
1061 			c->x86_phys_bits = 36;
1062 			c->x86_virt_bits = 48;
1063 		} else {
1064 			c->x86_clflush_size = 32;
1065 			c->x86_virt_bits = 32;
1066 			c->x86_phys_bits = 32;
1067 
1068 			if (cpu_has(c, X86_FEATURE_PAE) ||
1069 			    cpu_has(c, X86_FEATURE_PSE36))
1070 				c->x86_phys_bits = 36;
1071 		}
1072 	} else {
1073 		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1074 
1075 		c->x86_virt_bits = (eax >> 8) & 0xff;
1076 		c->x86_phys_bits = eax & 0xff;
1077 
1078 		/* Provide a sane default if not enumerated: */
1079 		if (!c->x86_clflush_size)
1080 			c->x86_clflush_size = 32;
1081 	}
1082 
1083 	c->x86_cache_bits = c->x86_phys_bits;
1084 	c->x86_cache_alignment = c->x86_clflush_size;
1085 }
1086 
identify_cpu_without_cpuid(struct cpuinfo_x86 * c)1087 static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1088 {
1089 	int i;
1090 
1091 	/*
1092 	 * First of all, decide if this is a 486 or higher
1093 	 * It's a 486 if we can modify the AC flag
1094 	 */
1095 	if (flag_is_changeable_p(X86_EFLAGS_AC))
1096 		c->x86 = 4;
1097 	else
1098 		c->x86 = 3;
1099 
1100 	for (i = 0; i < X86_VENDOR_NUM; i++)
1101 		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1102 			c->x86_vendor_id[0] = 0;
1103 			cpu_devs[i]->c_identify(c);
1104 			if (c->x86_vendor_id[0]) {
1105 				get_cpu_vendor(c);
1106 				break;
1107 			}
1108 		}
1109 }
1110 
1111 #define NO_SPECULATION		BIT(0)
1112 #define NO_MELTDOWN		BIT(1)
1113 #define NO_SSB			BIT(2)
1114 #define NO_L1TF			BIT(3)
1115 #define NO_MDS			BIT(4)
1116 #define MSBDS_ONLY		BIT(5)
1117 #define NO_SWAPGS		BIT(6)
1118 #define NO_ITLB_MULTIHIT	BIT(7)
1119 #define NO_SPECTRE_V2		BIT(8)
1120 #define NO_MMIO			BIT(9)
1121 #define NO_EIBRS_PBRSB		BIT(10)
1122 #define NO_BHI			BIT(11)
1123 
1124 #define VULNWL(vendor, family, model, whitelist)	\
1125 	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1126 
1127 #define VULNWL_INTEL(vfm, whitelist)		\
1128 	X86_MATCH_VFM(vfm, whitelist)
1129 
1130 #define VULNWL_AMD(family, whitelist)		\
1131 	VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1132 
1133 #define VULNWL_HYGON(family, whitelist)		\
1134 	VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1135 
1136 static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1137 	VULNWL(ANY,	4, X86_MODEL_ANY,	NO_SPECULATION),
1138 	VULNWL(CENTAUR,	5, X86_MODEL_ANY,	NO_SPECULATION),
1139 	VULNWL(INTEL,	5, X86_MODEL_ANY,	NO_SPECULATION),
1140 	VULNWL(NSC,	5, X86_MODEL_ANY,	NO_SPECULATION),
1141 	VULNWL(VORTEX,	5, X86_MODEL_ANY,	NO_SPECULATION),
1142 	VULNWL(VORTEX,	6, X86_MODEL_ANY,	NO_SPECULATION),
1143 
1144 	/* Intel Family 6 */
1145 	VULNWL_INTEL(INTEL_TIGERLAKE,		NO_MMIO),
1146 	VULNWL_INTEL(INTEL_TIGERLAKE_L,		NO_MMIO),
1147 	VULNWL_INTEL(INTEL_ALDERLAKE,		NO_MMIO),
1148 	VULNWL_INTEL(INTEL_ALDERLAKE_L,		NO_MMIO),
1149 
1150 	VULNWL_INTEL(INTEL_ATOM_SALTWELL,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1151 	VULNWL_INTEL(INTEL_ATOM_SALTWELL_TABLET, NO_SPECULATION | NO_ITLB_MULTIHIT),
1152 	VULNWL_INTEL(INTEL_ATOM_SALTWELL_MID,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1153 	VULNWL_INTEL(INTEL_ATOM_BONNELL,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1154 	VULNWL_INTEL(INTEL_ATOM_BONNELL_MID,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1155 
1156 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1157 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_D,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1158 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1159 	VULNWL_INTEL(INTEL_ATOM_AIRMONT,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1160 	VULNWL_INTEL(INTEL_XEON_PHI_KNL,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1161 	VULNWL_INTEL(INTEL_XEON_PHI_KNM,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1162 
1163 	VULNWL_INTEL(INTEL_CORE_YONAH,		NO_SSB),
1164 
1165 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID2,NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | MSBDS_ONLY),
1166 	VULNWL_INTEL(INTEL_ATOM_AIRMONT_NP,	NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1167 
1168 	VULNWL_INTEL(INTEL_ATOM_GOLDMONT,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1169 	VULNWL_INTEL(INTEL_ATOM_GOLDMONT_D,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1170 	VULNWL_INTEL(INTEL_ATOM_GOLDMONT_PLUS,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1171 
1172 	/*
1173 	 * Technically, swapgs isn't serializing on AMD (despite it previously
1174 	 * being documented as such in the APM).  But according to AMD, %gs is
1175 	 * updated non-speculatively, and the issuing of %gs-relative memory
1176 	 * operands will be blocked until the %gs update completes, which is
1177 	 * good enough for our purposes.
1178 	 */
1179 
1180 	VULNWL_INTEL(INTEL_ATOM_TREMONT,	NO_EIBRS_PBRSB),
1181 	VULNWL_INTEL(INTEL_ATOM_TREMONT_L,	NO_EIBRS_PBRSB),
1182 	VULNWL_INTEL(INTEL_ATOM_TREMONT_D,	NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1183 
1184 	/* AMD Family 0xf - 0x12 */
1185 	VULNWL_AMD(0x0f,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1186 	VULNWL_AMD(0x10,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1187 	VULNWL_AMD(0x11,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1188 	VULNWL_AMD(0x12,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1189 
1190 	/* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1191 	VULNWL_AMD(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1192 	VULNWL_HYGON(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1193 
1194 	/* Zhaoxin Family 7 */
1195 	VULNWL(CENTAUR,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1196 	VULNWL(ZHAOXIN,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1197 	{}
1198 };
1199 
1200 #define VULNBL(vendor, family, model, blacklist)	\
1201 	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1202 
1203 #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues)		   \
1204 	X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues)
1205 
1206 #define VULNBL_INTEL_TYPE(vfm, cpu_type, issues)	\
1207 	X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues)
1208 
1209 #define VULNBL_AMD(family, blacklist)		\
1210 	VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1211 
1212 #define VULNBL_HYGON(family, blacklist)		\
1213 	VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1214 
1215 #define SRBDS		BIT(0)
1216 /* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1217 #define MMIO		BIT(1)
1218 /* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1219 #define MMIO_SBDS	BIT(2)
1220 /* CPU is affected by RETbleed, speculating where you would not expect it */
1221 #define RETBLEED	BIT(3)
1222 /* CPU is affected by SMT (cross-thread) return predictions */
1223 #define SMT_RSB		BIT(4)
1224 /* CPU is affected by SRSO */
1225 #define SRSO		BIT(5)
1226 /* CPU is affected by GDS */
1227 #define GDS		BIT(6)
1228 /* CPU is affected by Register File Data Sampling */
1229 #define RFDS		BIT(7)
1230 /* CPU is affected by Indirect Target Selection */
1231 #define ITS		BIT(8)
1232 /* CPU is affected by Indirect Target Selection, but guest-host isolation is not affected */
1233 #define ITS_NATIVE_ONLY	BIT(9)
1234 
1235 static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1236 	VULNBL_INTEL_STEPS(INTEL_IVYBRIDGE,	     X86_STEP_MAX,	SRBDS),
1237 	VULNBL_INTEL_STEPS(INTEL_HASWELL,	     X86_STEP_MAX,	SRBDS),
1238 	VULNBL_INTEL_STEPS(INTEL_HASWELL_L,	     X86_STEP_MAX,	SRBDS),
1239 	VULNBL_INTEL_STEPS(INTEL_HASWELL_G,	     X86_STEP_MAX,	SRBDS),
1240 	VULNBL_INTEL_STEPS(INTEL_HASWELL_X,	     X86_STEP_MAX,	MMIO),
1241 	VULNBL_INTEL_STEPS(INTEL_BROADWELL_D,	     X86_STEP_MAX,	MMIO),
1242 	VULNBL_INTEL_STEPS(INTEL_BROADWELL_G,	     X86_STEP_MAX,	SRBDS),
1243 	VULNBL_INTEL_STEPS(INTEL_BROADWELL_X,	     X86_STEP_MAX,	MMIO),
1244 	VULNBL_INTEL_STEPS(INTEL_BROADWELL,	     X86_STEP_MAX,	SRBDS),
1245 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X,		      0x5,	MMIO | RETBLEED | GDS),
1246 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | ITS),
1247 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_L,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS),
1248 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS),
1249 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L,		      0xb,	MMIO | RETBLEED | GDS | SRBDS),
1250 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS | ITS),
1251 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE,		      0xc,	MMIO | RETBLEED | GDS | SRBDS),
1252 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS | ITS),
1253 	VULNBL_INTEL_STEPS(INTEL_CANNONLAKE_L,	     X86_STEP_MAX,	RETBLEED),
1254 	VULNBL_INTEL_STEPS(INTEL_ICELAKE_L,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
1255 	VULNBL_INTEL_STEPS(INTEL_ICELAKE_D,	     X86_STEP_MAX,	MMIO | GDS | ITS | ITS_NATIVE_ONLY),
1256 	VULNBL_INTEL_STEPS(INTEL_ICELAKE_X,	     X86_STEP_MAX,	MMIO | GDS | ITS | ITS_NATIVE_ONLY),
1257 	VULNBL_INTEL_STEPS(INTEL_COMETLAKE,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED | GDS | ITS),
1258 	VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L,		      0x0,	MMIO | RETBLEED | ITS),
1259 	VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED | GDS | ITS),
1260 	VULNBL_INTEL_STEPS(INTEL_TIGERLAKE_L,	     X86_STEP_MAX,	GDS | ITS | ITS_NATIVE_ONLY),
1261 	VULNBL_INTEL_STEPS(INTEL_TIGERLAKE,	     X86_STEP_MAX,	GDS | ITS | ITS_NATIVE_ONLY),
1262 	VULNBL_INTEL_STEPS(INTEL_LAKEFIELD,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED),
1263 	VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
1264 	VULNBL_INTEL_TYPE(INTEL_ALDERLAKE,		     ATOM,	RFDS),
1265 	VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L,	     X86_STEP_MAX,	RFDS),
1266 	VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE,		     ATOM,	RFDS),
1267 	VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P,	     X86_STEP_MAX,	RFDS),
1268 	VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S,	     X86_STEP_MAX,	RFDS),
1269 	VULNBL_INTEL_STEPS(INTEL_ATOM_GRACEMONT,     X86_STEP_MAX,	RFDS),
1270 	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RFDS),
1271 	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_D,     X86_STEP_MAX,	MMIO | RFDS),
1272 	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_L,     X86_STEP_MAX,	MMIO | MMIO_SBDS | RFDS),
1273 	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT,      X86_STEP_MAX,	RFDS),
1274 	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_D,    X86_STEP_MAX,	RFDS),
1275 	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_PLUS, X86_STEP_MAX,	RFDS),
1276 
1277 	VULNBL_AMD(0x15, RETBLEED),
1278 	VULNBL_AMD(0x16, RETBLEED),
1279 	VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO),
1280 	VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO),
1281 	VULNBL_AMD(0x19, SRSO),
1282 	VULNBL_AMD(0x1a, SRSO),
1283 	{}
1284 };
1285 
cpu_matches(const struct x86_cpu_id * table,unsigned long which)1286 static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1287 {
1288 	const struct x86_cpu_id *m = x86_match_cpu(table);
1289 
1290 	return m && !!(m->driver_data & which);
1291 }
1292 
x86_read_arch_cap_msr(void)1293 u64 x86_read_arch_cap_msr(void)
1294 {
1295 	u64 x86_arch_cap_msr = 0;
1296 
1297 	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1298 		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr);
1299 
1300 	return x86_arch_cap_msr;
1301 }
1302 
arch_cap_mmio_immune(u64 x86_arch_cap_msr)1303 static bool arch_cap_mmio_immune(u64 x86_arch_cap_msr)
1304 {
1305 	return (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO &&
1306 		x86_arch_cap_msr & ARCH_CAP_PSDP_NO &&
1307 		x86_arch_cap_msr & ARCH_CAP_SBDR_SSDP_NO);
1308 }
1309 
vulnerable_to_rfds(u64 x86_arch_cap_msr)1310 static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr)
1311 {
1312 	/* The "immunity" bit trumps everything else: */
1313 	if (x86_arch_cap_msr & ARCH_CAP_RFDS_NO)
1314 		return false;
1315 
1316 	/*
1317 	 * VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to
1318 	 * indicate that mitigation is needed because guest is running on a
1319 	 * vulnerable hardware or may migrate to such hardware:
1320 	 */
1321 	if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
1322 		return true;
1323 
1324 	/* Only consult the blacklist when there is no enumeration: */
1325 	return cpu_matches(cpu_vuln_blacklist, RFDS);
1326 }
1327 
vulnerable_to_its(u64 x86_arch_cap_msr)1328 static bool __init vulnerable_to_its(u64 x86_arch_cap_msr)
1329 {
1330 	/* The "immunity" bit trumps everything else: */
1331 	if (x86_arch_cap_msr & ARCH_CAP_ITS_NO)
1332 		return false;
1333 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
1334 		return false;
1335 
1336 	/* None of the affected CPUs have BHI_CTRL */
1337 	if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
1338 		return false;
1339 
1340 	/*
1341 	 * If a VMM did not expose ITS_NO, assume that a guest could
1342 	 * be running on a vulnerable hardware or may migrate to such
1343 	 * hardware.
1344 	 */
1345 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1346 		return true;
1347 
1348 	if (cpu_matches(cpu_vuln_blacklist, ITS))
1349 		return true;
1350 
1351 	return false;
1352 }
1353 
cpu_set_bug_bits(struct cpuinfo_x86 * c)1354 static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1355 {
1356 	u64 x86_arch_cap_msr = x86_read_arch_cap_msr();
1357 
1358 	/* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1359 	if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1360 	    !(x86_arch_cap_msr & ARCH_CAP_PSCHANGE_MC_NO))
1361 		setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1362 
1363 	if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1364 		return;
1365 
1366 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1367 
1368 	if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2)) {
1369 		setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1370 		setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER);
1371 	}
1372 
1373 	if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1374 	    !(x86_arch_cap_msr & ARCH_CAP_SSB_NO) &&
1375 	   !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1376 		setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1377 
1378 	/*
1379 	 * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
1380 	 * flag and protect from vendor-specific bugs via the whitelist.
1381 	 *
1382 	 * Don't use AutoIBRS when SNP is enabled because it degrades host
1383 	 * userspace indirect branch performance.
1384 	 */
1385 	if ((x86_arch_cap_msr & ARCH_CAP_IBRS_ALL) ||
1386 	    (cpu_has(c, X86_FEATURE_AUTOIBRS) &&
1387 	     !cpu_feature_enabled(X86_FEATURE_SEV_SNP))) {
1388 		setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1389 		if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1390 		    !(x86_arch_cap_msr & ARCH_CAP_PBRSB_NO))
1391 			setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1392 	}
1393 
1394 	if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1395 	    !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)) {
1396 		setup_force_cpu_bug(X86_BUG_MDS);
1397 		if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1398 			setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1399 	}
1400 
1401 	if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1402 		setup_force_cpu_bug(X86_BUG_SWAPGS);
1403 
1404 	/*
1405 	 * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1406 	 *	- TSX is supported or
1407 	 *	- TSX_CTRL is present
1408 	 *
1409 	 * TSX_CTRL check is needed for cases when TSX could be disabled before
1410 	 * the kernel boot e.g. kexec.
1411 	 * TSX_CTRL check alone is not sufficient for cases when the microcode
1412 	 * update is not present or running as guest that don't get TSX_CTRL.
1413 	 */
1414 	if (!(x86_arch_cap_msr & ARCH_CAP_TAA_NO) &&
1415 	    (cpu_has(c, X86_FEATURE_RTM) ||
1416 	     (x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)))
1417 		setup_force_cpu_bug(X86_BUG_TAA);
1418 
1419 	/*
1420 	 * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1421 	 * in the vulnerability blacklist.
1422 	 *
1423 	 * Some of the implications and mitigation of Shared Buffers Data
1424 	 * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1425 	 * SRBDS.
1426 	 */
1427 	if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1428 	     cpu_has(c, X86_FEATURE_RDSEED)) &&
1429 	    cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1430 		    setup_force_cpu_bug(X86_BUG_SRBDS);
1431 
1432 	/*
1433 	 * Processor MMIO Stale Data bug enumeration
1434 	 *
1435 	 * Affected CPU list is generally enough to enumerate the vulnerability,
1436 	 * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1437 	 * not want the guest to enumerate the bug.
1438 	 *
1439 	 * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist,
1440 	 * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits.
1441 	 */
1442 	if (!arch_cap_mmio_immune(x86_arch_cap_msr)) {
1443 		if (cpu_matches(cpu_vuln_blacklist, MMIO))
1444 			setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1445 		else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO))
1446 			setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN);
1447 	}
1448 
1449 	if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1450 		if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (x86_arch_cap_msr & ARCH_CAP_RSBA))
1451 			setup_force_cpu_bug(X86_BUG_RETBLEED);
1452 	}
1453 
1454 	if (cpu_matches(cpu_vuln_blacklist, SMT_RSB))
1455 		setup_force_cpu_bug(X86_BUG_SMT_RSB);
1456 
1457 	if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
1458 		if (cpu_matches(cpu_vuln_blacklist, SRSO))
1459 			setup_force_cpu_bug(X86_BUG_SRSO);
1460 	}
1461 
1462 	/*
1463 	 * Check if CPU is vulnerable to GDS. If running in a virtual machine on
1464 	 * an affected processor, the VMM may have disabled the use of GATHER by
1465 	 * disabling AVX2. The only way to do this in HW is to clear XCR0[2],
1466 	 * which means that AVX will be disabled.
1467 	 */
1468 	if (cpu_matches(cpu_vuln_blacklist, GDS) && !(x86_arch_cap_msr & ARCH_CAP_GDS_NO) &&
1469 	    boot_cpu_has(X86_FEATURE_AVX))
1470 		setup_force_cpu_bug(X86_BUG_GDS);
1471 
1472 	if (vulnerable_to_rfds(x86_arch_cap_msr))
1473 		setup_force_cpu_bug(X86_BUG_RFDS);
1474 
1475 	/*
1476 	 * Intel parts with eIBRS are vulnerable to BHI attacks. Parts with
1477 	 * BHI_NO still need to use the BHI mitigation to prevent Intra-mode
1478 	 * attacks.  When virtualized, eIBRS could be hidden, assume vulnerable.
1479 	 */
1480 	if (!cpu_matches(cpu_vuln_whitelist, NO_BHI) &&
1481 	    (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) ||
1482 	     boot_cpu_has(X86_FEATURE_HYPERVISOR)))
1483 		setup_force_cpu_bug(X86_BUG_BHI);
1484 
1485 	if (cpu_has(c, X86_FEATURE_AMD_IBPB) && !cpu_has(c, X86_FEATURE_AMD_IBPB_RET))
1486 		setup_force_cpu_bug(X86_BUG_IBPB_NO_RET);
1487 
1488 	if (vulnerable_to_its(x86_arch_cap_msr)) {
1489 		setup_force_cpu_bug(X86_BUG_ITS);
1490 		if (cpu_matches(cpu_vuln_blacklist, ITS_NATIVE_ONLY))
1491 			setup_force_cpu_bug(X86_BUG_ITS_NATIVE_ONLY);
1492 	}
1493 
1494 	if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1495 		return;
1496 
1497 	/* Rogue Data Cache Load? No! */
1498 	if (x86_arch_cap_msr & ARCH_CAP_RDCL_NO)
1499 		return;
1500 
1501 	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1502 
1503 	if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1504 		return;
1505 
1506 	setup_force_cpu_bug(X86_BUG_L1TF);
1507 }
1508 
1509 /*
1510  * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1511  * unfortunately, that's not true in practice because of early VIA
1512  * chips and (more importantly) broken virtualizers that are not easy
1513  * to detect. In the latter case it doesn't even *fail* reliably, so
1514  * probing for it doesn't even work. Disable it completely on 32-bit
1515  * unless we can find a reliable way to detect all the broken cases.
1516  * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1517  */
detect_nopl(void)1518 static void detect_nopl(void)
1519 {
1520 #ifdef CONFIG_X86_32
1521 	setup_clear_cpu_cap(X86_FEATURE_NOPL);
1522 #else
1523 	setup_force_cpu_cap(X86_FEATURE_NOPL);
1524 #endif
1525 }
1526 
parse_set_clear_cpuid(char * arg,bool set)1527 static inline bool parse_set_clear_cpuid(char *arg, bool set)
1528 {
1529 	char *opt;
1530 	int taint = 0;
1531 
1532 	while (arg) {
1533 		bool found __maybe_unused = false;
1534 		unsigned int bit;
1535 
1536 		opt = strsep(&arg, ",");
1537 
1538 		/*
1539 		 * Handle naked numbers first for feature flags which don't
1540 		 * have names. It doesn't make sense for a bug not to have a
1541 		 * name so don't handle bug flags here.
1542 		 */
1543 		if (!kstrtouint(opt, 10, &bit)) {
1544 			if (bit < NCAPINTS * 32) {
1545 
1546 				if (set) {
1547 					pr_warn("setcpuid: force-enabling CPU feature flag:");
1548 					setup_force_cpu_cap(bit);
1549 				} else {
1550 					pr_warn("clearcpuid: force-disabling CPU feature flag:");
1551 					setup_clear_cpu_cap(bit);
1552 				}
1553 				/* empty-string, i.e., ""-defined feature flags */
1554 				if (!x86_cap_flags[bit])
1555 					pr_cont(" %d:%d\n", bit >> 5, bit & 31);
1556 				else
1557 					pr_cont(" %s\n", x86_cap_flags[bit]);
1558 
1559 				taint++;
1560 			}
1561 			/*
1562 			 * The assumption is that there are no feature names with only
1563 			 * numbers in the name thus go to the next argument.
1564 			 */
1565 			continue;
1566 		}
1567 
1568 		for (bit = 0; bit < 32 * (NCAPINTS + NBUGINTS); bit++) {
1569 			const char *flag;
1570 			const char *kind;
1571 
1572 			if (bit < 32 * NCAPINTS) {
1573 				flag = x86_cap_flags[bit];
1574 				kind = "feature";
1575 			} else {
1576 				kind = "bug";
1577 				flag = x86_bug_flags[bit - (32 * NCAPINTS)];
1578 			}
1579 
1580 			if (!flag)
1581 				continue;
1582 
1583 			if (strcmp(flag, opt))
1584 				continue;
1585 
1586 			if (set) {
1587 				pr_warn("setcpuid: force-enabling CPU %s flag: %s\n",
1588 					kind, flag);
1589 				setup_force_cpu_cap(bit);
1590 			} else {
1591 				pr_warn("clearcpuid: force-disabling CPU %s flag: %s\n",
1592 					kind, flag);
1593 				setup_clear_cpu_cap(bit);
1594 			}
1595 			taint++;
1596 			found = true;
1597 			break;
1598 		}
1599 
1600 		if (!found)
1601 			pr_warn("%s: unknown CPU flag: %s", set ? "setcpuid" : "clearcpuid", opt);
1602 	}
1603 
1604 	return taint;
1605 }
1606 
1607 
1608 /*
1609  * We parse cpu parameters early because fpu__init_system() is executed
1610  * before parse_early_param().
1611  */
cpu_parse_early_param(void)1612 static void __init cpu_parse_early_param(void)
1613 {
1614 	bool cpuid_taint = false;
1615 	char arg[128];
1616 	int arglen;
1617 
1618 #ifdef CONFIG_X86_32
1619 	if (cmdline_find_option_bool(boot_command_line, "no387"))
1620 #ifdef CONFIG_MATH_EMULATION
1621 		setup_clear_cpu_cap(X86_FEATURE_FPU);
1622 #else
1623 		pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1624 #endif
1625 
1626 	if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1627 		setup_clear_cpu_cap(X86_FEATURE_FXSR);
1628 #endif
1629 
1630 	if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1631 		setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1632 
1633 	if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1634 		setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1635 
1636 	if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1637 		setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1638 
1639 	if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
1640 		setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1641 
1642 	/* Minimize the gap between FRED is available and available but disabled. */
1643 	arglen = cmdline_find_option(boot_command_line, "fred", arg, sizeof(arg));
1644 	if (arglen != 2 || strncmp(arg, "on", 2))
1645 		setup_clear_cpu_cap(X86_FEATURE_FRED);
1646 
1647 	arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1648 	if (arglen > 0)
1649 		cpuid_taint |= parse_set_clear_cpuid(arg, false);
1650 
1651 	arglen = cmdline_find_option(boot_command_line, "setcpuid", arg, sizeof(arg));
1652 	if (arglen > 0)
1653 		cpuid_taint |= parse_set_clear_cpuid(arg, true);
1654 
1655 	if (cpuid_taint) {
1656 		pr_warn("!!! setcpuid=/clearcpuid= in use, this is for TESTING ONLY, may break things horribly. Tainting kernel.\n");
1657 		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1658 	}
1659 }
1660 
1661 /*
1662  * Do minimum CPU detection early.
1663  * Fields really needed: vendor, cpuid_level, family, model, mask,
1664  * cache alignment.
1665  * The others are not touched to avoid unwanted side effects.
1666  *
1667  * WARNING: this function is only called on the boot CPU.  Don't add code
1668  * here that is supposed to run on all CPUs.
1669  */
early_identify_cpu(struct cpuinfo_x86 * c)1670 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1671 {
1672 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1673 	c->extended_cpuid_level = 0;
1674 
1675 	if (!have_cpuid_p())
1676 		identify_cpu_without_cpuid(c);
1677 
1678 	/* cyrix could have cpuid enabled via c_identify()*/
1679 	if (have_cpuid_p()) {
1680 		cpu_detect(c);
1681 		get_cpu_vendor(c);
1682 		intel_unlock_cpuid_leafs(c);
1683 		get_cpu_cap(c);
1684 		setup_force_cpu_cap(X86_FEATURE_CPUID);
1685 		get_cpu_address_sizes(c);
1686 		cpu_parse_early_param();
1687 
1688 		cpu_init_topology(c);
1689 
1690 		if (this_cpu->c_early_init)
1691 			this_cpu->c_early_init(c);
1692 
1693 		c->cpu_index = 0;
1694 		filter_cpuid_features(c, false);
1695 		check_cpufeature_deps(c);
1696 
1697 		if (this_cpu->c_bsp_init)
1698 			this_cpu->c_bsp_init(c);
1699 	} else {
1700 		setup_clear_cpu_cap(X86_FEATURE_CPUID);
1701 		get_cpu_address_sizes(c);
1702 		cpu_init_topology(c);
1703 	}
1704 
1705 	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1706 
1707 	cpu_set_bug_bits(c);
1708 
1709 	sld_setup(c);
1710 
1711 #ifdef CONFIG_X86_32
1712 	/*
1713 	 * Regardless of whether PCID is enumerated, the SDM says
1714 	 * that it can't be enabled in 32-bit mode.
1715 	 */
1716 	setup_clear_cpu_cap(X86_FEATURE_PCID);
1717 #endif
1718 
1719 	/*
1720 	 * Later in the boot process pgtable_l5_enabled() relies on
1721 	 * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1722 	 * enabled by this point we need to clear the feature bit to avoid
1723 	 * false-positives at the later stage.
1724 	 *
1725 	 * pgtable_l5_enabled() can be false here for several reasons:
1726 	 *  - 5-level paging is disabled compile-time;
1727 	 *  - it's 32-bit kernel;
1728 	 *  - machine doesn't support 5-level paging;
1729 	 *  - user specified 'no5lvl' in kernel command line.
1730 	 */
1731 	if (!pgtable_l5_enabled())
1732 		setup_clear_cpu_cap(X86_FEATURE_LA57);
1733 
1734 	detect_nopl();
1735 }
1736 
init_cpu_devs(void)1737 void __init init_cpu_devs(void)
1738 {
1739 	const struct cpu_dev *const *cdev;
1740 	int count = 0;
1741 
1742 	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1743 		const struct cpu_dev *cpudev = *cdev;
1744 
1745 		if (count >= X86_VENDOR_NUM)
1746 			break;
1747 		cpu_devs[count] = cpudev;
1748 		count++;
1749 	}
1750 }
1751 
early_cpu_init(void)1752 void __init early_cpu_init(void)
1753 {
1754 #ifdef CONFIG_PROCESSOR_SELECT
1755 	unsigned int i, j;
1756 
1757 	pr_info("KERNEL supported cpus:\n");
1758 #endif
1759 
1760 	init_cpu_devs();
1761 
1762 #ifdef CONFIG_PROCESSOR_SELECT
1763 	for (i = 0; i < X86_VENDOR_NUM && cpu_devs[i]; i++) {
1764 		for (j = 0; j < 2; j++) {
1765 			if (!cpu_devs[i]->c_ident[j])
1766 				continue;
1767 			pr_info("  %s %s\n", cpu_devs[i]->c_vendor,
1768 				cpu_devs[i]->c_ident[j]);
1769 		}
1770 	}
1771 #endif
1772 
1773 	early_identify_cpu(&boot_cpu_data);
1774 }
1775 
detect_null_seg_behavior(void)1776 static bool detect_null_seg_behavior(void)
1777 {
1778 	/*
1779 	 * Empirically, writing zero to a segment selector on AMD does
1780 	 * not clear the base, whereas writing zero to a segment
1781 	 * selector on Intel does clear the base.  Intel's behavior
1782 	 * allows slightly faster context switches in the common case
1783 	 * where GS is unused by the prev and next threads.
1784 	 *
1785 	 * Since neither vendor documents this anywhere that I can see,
1786 	 * detect it directly instead of hard-coding the choice by
1787 	 * vendor.
1788 	 *
1789 	 * I've designated AMD's behavior as the "bug" because it's
1790 	 * counterintuitive and less friendly.
1791 	 */
1792 
1793 	unsigned long old_base, tmp;
1794 	rdmsrl(MSR_FS_BASE, old_base);
1795 	wrmsrl(MSR_FS_BASE, 1);
1796 	loadsegment(fs, 0);
1797 	rdmsrl(MSR_FS_BASE, tmp);
1798 	wrmsrl(MSR_FS_BASE, old_base);
1799 	return tmp == 0;
1800 }
1801 
check_null_seg_clears_base(struct cpuinfo_x86 * c)1802 void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1803 {
1804 	/* BUG_NULL_SEG is only relevant with 64bit userspace */
1805 	if (!IS_ENABLED(CONFIG_X86_64))
1806 		return;
1807 
1808 	if (cpu_has(c, X86_FEATURE_NULL_SEL_CLR_BASE))
1809 		return;
1810 
1811 	/*
1812 	 * CPUID bit above wasn't set. If this kernel is still running
1813 	 * as a HV guest, then the HV has decided not to advertize
1814 	 * that CPUID bit for whatever reason.	For example, one
1815 	 * member of the migration pool might be vulnerable.  Which
1816 	 * means, the bug is present: set the BUG flag and return.
1817 	 */
1818 	if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1819 		set_cpu_bug(c, X86_BUG_NULL_SEG);
1820 		return;
1821 	}
1822 
1823 	/*
1824 	 * Zen2 CPUs also have this behaviour, but no CPUID bit.
1825 	 * 0x18 is the respective family for Hygon.
1826 	 */
1827 	if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1828 	    detect_null_seg_behavior())
1829 		return;
1830 
1831 	/* All the remaining ones are affected */
1832 	set_cpu_bug(c, X86_BUG_NULL_SEG);
1833 }
1834 
generic_identify(struct cpuinfo_x86 * c)1835 static void generic_identify(struct cpuinfo_x86 *c)
1836 {
1837 	c->extended_cpuid_level = 0;
1838 
1839 	if (!have_cpuid_p())
1840 		identify_cpu_without_cpuid(c);
1841 
1842 	/* cyrix could have cpuid enabled via c_identify()*/
1843 	if (!have_cpuid_p())
1844 		return;
1845 
1846 	cpu_detect(c);
1847 
1848 	get_cpu_vendor(c);
1849 	intel_unlock_cpuid_leafs(c);
1850 	get_cpu_cap(c);
1851 
1852 	get_cpu_address_sizes(c);
1853 
1854 	get_model_name(c); /* Default name */
1855 
1856 	/*
1857 	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
1858 	 * systems that run Linux at CPL > 0 may or may not have the
1859 	 * issue, but, even if they have the issue, there's absolutely
1860 	 * nothing we can do about it because we can't use the real IRET
1861 	 * instruction.
1862 	 *
1863 	 * NB: For the time being, only 32-bit kernels support
1864 	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
1865 	 * whether to apply espfix using paravirt hooks.  If any
1866 	 * non-paravirt system ever shows up that does *not* have the
1867 	 * ESPFIX issue, we can change this.
1868 	 */
1869 #ifdef CONFIG_X86_32
1870 	set_cpu_bug(c, X86_BUG_ESPFIX);
1871 #endif
1872 }
1873 
1874 /*
1875  * This does the hard work of actually picking apart the CPU stuff...
1876  */
identify_cpu(struct cpuinfo_x86 * c)1877 static void identify_cpu(struct cpuinfo_x86 *c)
1878 {
1879 	int i;
1880 
1881 	c->loops_per_jiffy = loops_per_jiffy;
1882 	c->x86_cache_size = 0;
1883 	c->x86_vendor = X86_VENDOR_UNKNOWN;
1884 	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
1885 	c->x86_vendor_id[0] = '\0'; /* Unset */
1886 	c->x86_model_id[0] = '\0';  /* Unset */
1887 #ifdef CONFIG_X86_64
1888 	c->x86_clflush_size = 64;
1889 	c->x86_phys_bits = 36;
1890 	c->x86_virt_bits = 48;
1891 #else
1892 	c->cpuid_level = -1;	/* CPUID not detected */
1893 	c->x86_clflush_size = 32;
1894 	c->x86_phys_bits = 32;
1895 	c->x86_virt_bits = 32;
1896 #endif
1897 	c->x86_cache_alignment = c->x86_clflush_size;
1898 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1899 #ifdef CONFIG_X86_VMX_FEATURE_NAMES
1900 	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1901 #endif
1902 
1903 	generic_identify(c);
1904 
1905 	cpu_parse_topology(c);
1906 
1907 	if (this_cpu->c_identify)
1908 		this_cpu->c_identify(c);
1909 
1910 	/* Clear/Set all flags overridden by options, after probe */
1911 	apply_forced_caps(c);
1912 
1913 	/*
1914 	 * Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
1915 	 * Hygon will clear it in ->c_init() below.
1916 	 */
1917 	set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1918 
1919 	/*
1920 	 * Vendor-specific initialization.  In this section we
1921 	 * canonicalize the feature flags, meaning if there are
1922 	 * features a certain CPU supports which CPUID doesn't
1923 	 * tell us, CPUID claiming incorrect flags, or other bugs,
1924 	 * we handle them here.
1925 	 *
1926 	 * At the end of this section, c->x86_capability better
1927 	 * indicate the features this CPU genuinely supports!
1928 	 */
1929 	if (this_cpu->c_init)
1930 		this_cpu->c_init(c);
1931 
1932 	bus_lock_init();
1933 
1934 	/* Disable the PN if appropriate */
1935 	squash_the_stupid_serial_number(c);
1936 
1937 	/* Set up SMEP/SMAP/UMIP */
1938 	setup_smep(c);
1939 	setup_smap(c);
1940 	setup_umip(c);
1941 
1942 	/* Enable FSGSBASE instructions if available. */
1943 	if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1944 		cr4_set_bits(X86_CR4_FSGSBASE);
1945 		elf_hwcap2 |= HWCAP2_FSGSBASE;
1946 	}
1947 
1948 	/*
1949 	 * The vendor-specific functions might have changed features.
1950 	 * Now we do "generic changes."
1951 	 */
1952 
1953 	/* Filter out anything that depends on CPUID levels we don't have */
1954 	filter_cpuid_features(c, true);
1955 
1956 	/* Check for unmet dependencies based on the CPUID dependency table */
1957 	check_cpufeature_deps(c);
1958 
1959 	/* If the model name is still unset, do table lookup. */
1960 	if (!c->x86_model_id[0]) {
1961 		const char *p;
1962 		p = table_lookup_model(c);
1963 		if (p)
1964 			strcpy(c->x86_model_id, p);
1965 		else
1966 			/* Last resort... */
1967 			sprintf(c->x86_model_id, "%02x/%02x",
1968 				c->x86, c->x86_model);
1969 	}
1970 
1971 	x86_init_rdrand(c);
1972 	setup_pku(c);
1973 	setup_cet(c);
1974 
1975 	/*
1976 	 * Clear/Set all flags overridden by options, need do it
1977 	 * before following smp all cpus cap AND.
1978 	 */
1979 	apply_forced_caps(c);
1980 
1981 	/*
1982 	 * On SMP, boot_cpu_data holds the common feature set between
1983 	 * all CPUs; so make sure that we indicate which features are
1984 	 * common between the CPUs.  The first time this routine gets
1985 	 * executed, c == &boot_cpu_data.
1986 	 */
1987 	if (c != &boot_cpu_data) {
1988 		/* AND the already accumulated flags with these */
1989 		for (i = 0; i < NCAPINTS; i++)
1990 			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1991 
1992 		/* OR, i.e. replicate the bug flags */
1993 		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1994 			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1995 	}
1996 
1997 	ppin_init(c);
1998 
1999 	/* Init Machine Check Exception if available. */
2000 	mcheck_cpu_init(c);
2001 
2002 	numa_add_cpu(smp_processor_id());
2003 }
2004 
2005 /*
2006  * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
2007  * on 32-bit kernels:
2008  */
2009 #ifdef CONFIG_X86_32
enable_sep_cpu(void)2010 void enable_sep_cpu(void)
2011 {
2012 	struct tss_struct *tss;
2013 	int cpu;
2014 
2015 	if (!boot_cpu_has(X86_FEATURE_SEP))
2016 		return;
2017 
2018 	cpu = get_cpu();
2019 	tss = &per_cpu(cpu_tss_rw, cpu);
2020 
2021 	/*
2022 	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
2023 	 * see the big comment in struct x86_hw_tss's definition.
2024 	 */
2025 
2026 	tss->x86_tss.ss1 = __KERNEL_CS;
2027 	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
2028 	wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
2029 	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
2030 
2031 	put_cpu();
2032 }
2033 #endif
2034 
identify_boot_cpu(void)2035 static __init void identify_boot_cpu(void)
2036 {
2037 	identify_cpu(&boot_cpu_data);
2038 	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
2039 		pr_info("CET detected: Indirect Branch Tracking enabled\n");
2040 #ifdef CONFIG_X86_32
2041 	enable_sep_cpu();
2042 #endif
2043 	cpu_detect_tlb(&boot_cpu_data);
2044 	setup_cr_pinning();
2045 
2046 	tsx_init();
2047 	tdx_init();
2048 	lkgs_init();
2049 }
2050 
identify_secondary_cpu(unsigned int cpu)2051 void identify_secondary_cpu(unsigned int cpu)
2052 {
2053 	struct cpuinfo_x86 *c = &cpu_data(cpu);
2054 
2055 	/* Copy boot_cpu_data only on the first bringup */
2056 	if (!c->initialized)
2057 		*c = boot_cpu_data;
2058 	c->cpu_index = cpu;
2059 
2060 	identify_cpu(c);
2061 #ifdef CONFIG_X86_32
2062 	enable_sep_cpu();
2063 #endif
2064 	x86_spec_ctrl_setup_ap();
2065 	update_srbds_msr();
2066 	if (boot_cpu_has_bug(X86_BUG_GDS))
2067 		update_gds_msr();
2068 
2069 	tsx_ap_init();
2070 	c->initialized = true;
2071 }
2072 
print_cpu_info(struct cpuinfo_x86 * c)2073 void print_cpu_info(struct cpuinfo_x86 *c)
2074 {
2075 	const char *vendor = NULL;
2076 
2077 	if (c->x86_vendor < X86_VENDOR_NUM) {
2078 		vendor = this_cpu->c_vendor;
2079 	} else {
2080 		if (c->cpuid_level >= 0)
2081 			vendor = c->x86_vendor_id;
2082 	}
2083 
2084 	if (vendor && !strstr(c->x86_model_id, vendor))
2085 		pr_cont("%s ", vendor);
2086 
2087 	if (c->x86_model_id[0])
2088 		pr_cont("%s", c->x86_model_id);
2089 	else
2090 		pr_cont("%d86", c->x86);
2091 
2092 	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
2093 
2094 	if (c->x86_stepping || c->cpuid_level >= 0)
2095 		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
2096 	else
2097 		pr_cont(")\n");
2098 }
2099 
2100 /*
2101  * clearcpuid= and setcpuid= were already parsed in cpu_parse_early_param().
2102  * These dummy functions prevent them from becoming an environment variable for
2103  * init.
2104  */
2105 
setup_clearcpuid(char * arg)2106 static __init int setup_clearcpuid(char *arg)
2107 {
2108 	return 1;
2109 }
2110 __setup("clearcpuid=", setup_clearcpuid);
2111 
setup_setcpuid(char * arg)2112 static __init int setup_setcpuid(char *arg)
2113 {
2114 	return 1;
2115 }
2116 __setup("setcpuid=", setup_setcpuid);
2117 
2118 DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
2119 EXPORT_PER_CPU_SYMBOL(current_task);
2120 EXPORT_PER_CPU_SYMBOL(const_current_task);
2121 
2122 DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;
2123 EXPORT_PER_CPU_SYMBOL(__preempt_count);
2124 
2125 DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
2126 
2127 #ifdef CONFIG_X86_64
2128 /*
2129  * Note: Do not make this dependant on CONFIG_MITIGATION_CALL_DEPTH_TRACKING
2130  * so that this space is reserved in the hot cache section even when the
2131  * mitigation is disabled.
2132  */
2133 DEFINE_PER_CPU_CACHE_HOT(u64, __x86_call_depth);
2134 EXPORT_PER_CPU_SYMBOL(__x86_call_depth);
2135 
wrmsrl_cstar(unsigned long val)2136 static void wrmsrl_cstar(unsigned long val)
2137 {
2138 	/*
2139 	 * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2140 	 * is so far ignored by the CPU, but raises a #VE trap in a TDX
2141 	 * guest. Avoid the pointless write on all Intel CPUs.
2142 	 */
2143 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2144 		wrmsrl(MSR_CSTAR, val);
2145 }
2146 
idt_syscall_init(void)2147 static inline void idt_syscall_init(void)
2148 {
2149 	wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
2150 
2151 	if (ia32_enabled()) {
2152 		wrmsrl_cstar((unsigned long)entry_SYSCALL_compat);
2153 		/*
2154 		 * This only works on Intel CPUs.
2155 		 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2156 		 * This does not cause SYSENTER to jump to the wrong location, because
2157 		 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2158 		 */
2159 		wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
2160 		wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
2161 			    (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2162 		wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
2163 	} else {
2164 		wrmsrl_cstar((unsigned long)entry_SYSCALL32_ignore);
2165 		wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
2166 		wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
2167 		wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
2168 	}
2169 
2170 	/*
2171 	 * Flags to clear on syscall; clear as much as possible
2172 	 * to minimize user space-kernel interference.
2173 	 */
2174 	wrmsrl(MSR_SYSCALL_MASK,
2175 	       X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2176 	       X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2177 	       X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2178 	       X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2179 	       X86_EFLAGS_AC|X86_EFLAGS_ID);
2180 }
2181 
2182 /* May not be marked __init: used by software suspend */
syscall_init(void)2183 void syscall_init(void)
2184 {
2185 	/* The default user and kernel segments */
2186 	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
2187 
2188 	/*
2189 	 * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and
2190 	 * SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED
2191 	 * entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit
2192 	 * instruction to return to ring 3 (both sysexit and sysret cause
2193 	 * #UD when FRED is enabled).
2194 	 */
2195 	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2196 		idt_syscall_init();
2197 }
2198 #endif /* CONFIG_X86_64 */
2199 
2200 #ifdef CONFIG_STACKPROTECTOR
2201 DEFINE_PER_CPU_CACHE_HOT(unsigned long, __stack_chk_guard);
2202 #ifndef CONFIG_SMP
2203 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2204 #endif
2205 #endif
2206 
2207 /*
2208  * Clear all 6 debug registers:
2209  */
clear_all_debug_regs(void)2210 static void clear_all_debug_regs(void)
2211 {
2212 	int i;
2213 
2214 	for (i = 0; i < 8; i++) {
2215 		/* Ignore db4, db5 */
2216 		if ((i == 4) || (i == 5))
2217 			continue;
2218 
2219 		set_debugreg(0, i);
2220 	}
2221 }
2222 
2223 #ifdef CONFIG_KGDB
2224 /*
2225  * Restore debug regs if using kgdbwait and you have a kernel debugger
2226  * connection established.
2227  */
dbg_restore_debug_regs(void)2228 static void dbg_restore_debug_regs(void)
2229 {
2230 	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2231 		arch_kgdb_ops.correct_hw_break();
2232 }
2233 #else /* ! CONFIG_KGDB */
2234 #define dbg_restore_debug_regs()
2235 #endif /* ! CONFIG_KGDB */
2236 
setup_getcpu(int cpu)2237 static inline void setup_getcpu(int cpu)
2238 {
2239 	unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
2240 	struct desc_struct d = { };
2241 
2242 	if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2243 		wrmsr(MSR_TSC_AUX, cpudata, 0);
2244 
2245 	/* Store CPU and node number in limit. */
2246 	d.limit0 = cpudata;
2247 	d.limit1 = cpudata >> 16;
2248 
2249 	d.type = 5;		/* RO data, expand down, accessed */
2250 	d.dpl = 3;		/* Visible to user code */
2251 	d.s = 1;		/* Not a system segment */
2252 	d.p = 1;		/* Present */
2253 	d.d = 1;		/* 32-bit */
2254 
2255 	write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
2256 }
2257 
2258 #ifdef CONFIG_X86_64
tss_setup_ist(struct tss_struct * tss)2259 static inline void tss_setup_ist(struct tss_struct *tss)
2260 {
2261 	/* Set up the per-CPU TSS IST stacks */
2262 	tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2263 	tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2264 	tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2265 	tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2266 	/* Only mapped when SEV-ES is active */
2267 	tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2268 }
2269 #else /* CONFIG_X86_64 */
tss_setup_ist(struct tss_struct * tss)2270 static inline void tss_setup_ist(struct tss_struct *tss) { }
2271 #endif /* !CONFIG_X86_64 */
2272 
tss_setup_io_bitmap(struct tss_struct * tss)2273 static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2274 {
2275 	tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2276 
2277 #ifdef CONFIG_X86_IOPL_IOPERM
2278 	tss->io_bitmap.prev_max = 0;
2279 	tss->io_bitmap.prev_sequence = 0;
2280 	memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2281 	/*
2282 	 * Invalidate the extra array entry past the end of the all
2283 	 * permission bitmap as required by the hardware.
2284 	 */
2285 	tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2286 #endif
2287 }
2288 
2289 /*
2290  * Setup everything needed to handle exceptions from the IDT, including the IST
2291  * exceptions which use paranoid_entry().
2292  */
cpu_init_exception_handling(bool boot_cpu)2293 void cpu_init_exception_handling(bool boot_cpu)
2294 {
2295 	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2296 	int cpu = raw_smp_processor_id();
2297 
2298 	/* paranoid_entry() gets the CPU number from the GDT */
2299 	setup_getcpu(cpu);
2300 
2301 	/* For IDT mode, IST vectors need to be set in TSS. */
2302 	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2303 		tss_setup_ist(tss);
2304 	tss_setup_io_bitmap(tss);
2305 	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2306 
2307 	load_TR_desc();
2308 
2309 	/* GHCB needs to be setup to handle #VC. */
2310 	setup_ghcb();
2311 
2312 	if (cpu_feature_enabled(X86_FEATURE_FRED)) {
2313 		/* The boot CPU has enabled FRED during early boot */
2314 		if (!boot_cpu)
2315 			cpu_init_fred_exceptions();
2316 
2317 		cpu_init_fred_rsps();
2318 	} else {
2319 		load_current_idt();
2320 	}
2321 }
2322 
cpu_init_replace_early_idt(void)2323 void __init cpu_init_replace_early_idt(void)
2324 {
2325 	if (cpu_feature_enabled(X86_FEATURE_FRED))
2326 		cpu_init_fred_exceptions();
2327 	else
2328 		idt_setup_early_pf();
2329 }
2330 
2331 /*
2332  * cpu_init() initializes state that is per-CPU. Some data is already
2333  * initialized (naturally) in the bootstrap process, such as the GDT.  We
2334  * reload it nevertheless, this function acts as a 'CPU state barrier',
2335  * nothing should get across.
2336  */
cpu_init(void)2337 void cpu_init(void)
2338 {
2339 	struct task_struct *cur = current;
2340 	int cpu = raw_smp_processor_id();
2341 
2342 #ifdef CONFIG_NUMA
2343 	if (this_cpu_read(numa_node) == 0 &&
2344 	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
2345 		set_numa_node(early_cpu_to_node(cpu));
2346 #endif
2347 	pr_debug("Initializing CPU#%d\n", cpu);
2348 
2349 	if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2350 	    boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2351 		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2352 
2353 	if (IS_ENABLED(CONFIG_X86_64)) {
2354 		loadsegment(fs, 0);
2355 		memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2356 		syscall_init();
2357 
2358 		wrmsrl(MSR_FS_BASE, 0);
2359 		wrmsrl(MSR_KERNEL_GS_BASE, 0);
2360 		barrier();
2361 
2362 		x2apic_setup();
2363 
2364 		intel_posted_msi_init();
2365 	}
2366 
2367 	mmgrab(&init_mm);
2368 	cur->active_mm = &init_mm;
2369 	BUG_ON(cur->mm);
2370 	initialize_tlbstate_and_flush();
2371 	enter_lazy_tlb(&init_mm, cur);
2372 
2373 	/*
2374 	 * sp0 points to the entry trampoline stack regardless of what task
2375 	 * is running.
2376 	 */
2377 	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2378 
2379 	load_mm_ldt(&init_mm);
2380 
2381 	clear_all_debug_regs();
2382 	dbg_restore_debug_regs();
2383 
2384 	doublefault_init_cpu_tss();
2385 
2386 	if (is_uv_system())
2387 		uv_cpu_init();
2388 
2389 	load_fixmap_gdt(cpu);
2390 }
2391 
2392 #ifdef CONFIG_MICROCODE_LATE_LOADING
2393 /**
2394  * store_cpu_caps() - Store a snapshot of CPU capabilities
2395  * @curr_info: Pointer where to store it
2396  *
2397  * Returns: None
2398  */
store_cpu_caps(struct cpuinfo_x86 * curr_info)2399 void store_cpu_caps(struct cpuinfo_x86 *curr_info)
2400 {
2401 	/* Reload CPUID max function as it might've changed. */
2402 	curr_info->cpuid_level = cpuid_eax(0);
2403 
2404 	/* Copy all capability leafs and pick up the synthetic ones. */
2405 	memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
2406 	       sizeof(curr_info->x86_capability));
2407 
2408 	/* Get the hardware CPUID leafs */
2409 	get_cpu_cap(curr_info);
2410 }
2411 
2412 /**
2413  * microcode_check() - Check if any CPU capabilities changed after an update.
2414  * @prev_info:	CPU capabilities stored before an update.
2415  *
2416  * The microcode loader calls this upon late microcode load to recheck features,
2417  * only when microcode has been updated. Caller holds and CPU hotplug lock.
2418  *
2419  * Return: None
2420  */
microcode_check(struct cpuinfo_x86 * prev_info)2421 void microcode_check(struct cpuinfo_x86 *prev_info)
2422 {
2423 	struct cpuinfo_x86 curr_info;
2424 
2425 	perf_check_microcode();
2426 
2427 	amd_check_microcode();
2428 
2429 	store_cpu_caps(&curr_info);
2430 
2431 	if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability,
2432 		    sizeof(prev_info->x86_capability)))
2433 		return;
2434 
2435 	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2436 	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2437 }
2438 #endif
2439 
2440 /*
2441  * Invoked from core CPU hotplug code after hotplug operations
2442  */
arch_smt_update(void)2443 void arch_smt_update(void)
2444 {
2445 	/* Handle the speculative execution misfeatures */
2446 	cpu_bugs_smt_update();
2447 	/* Check whether IPI broadcasting can be enabled */
2448 	apic_smt_update();
2449 }
2450 
arch_cpu_finalize_init(void)2451 void __init arch_cpu_finalize_init(void)
2452 {
2453 	struct cpuinfo_x86 *c = this_cpu_ptr(&cpu_info);
2454 
2455 	identify_boot_cpu();
2456 
2457 	select_idle_routine();
2458 
2459 	/*
2460 	 * identify_boot_cpu() initialized SMT support information, let the
2461 	 * core code know.
2462 	 */
2463 	cpu_smt_set_num_threads(__max_threads_per_core, __max_threads_per_core);
2464 
2465 	if (!IS_ENABLED(CONFIG_SMP)) {
2466 		pr_info("CPU: ");
2467 		print_cpu_info(&boot_cpu_data);
2468 	}
2469 
2470 	cpu_select_mitigations();
2471 
2472 	arch_smt_update();
2473 
2474 	if (IS_ENABLED(CONFIG_X86_32)) {
2475 		/*
2476 		 * Check whether this is a real i386 which is not longer
2477 		 * supported and fixup the utsname.
2478 		 */
2479 		if (boot_cpu_data.x86 < 4)
2480 			panic("Kernel requires i486+ for 'invlpg' and other features");
2481 
2482 		init_utsname()->machine[1] =
2483 			'0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
2484 	}
2485 
2486 	/*
2487 	 * Must be before alternatives because it might set or clear
2488 	 * feature bits.
2489 	 */
2490 	fpu__init_system();
2491 	fpu__init_cpu();
2492 
2493 	/*
2494 	 * Ensure that access to the per CPU representation has the initial
2495 	 * boot CPU configuration.
2496 	 */
2497 	*c = boot_cpu_data;
2498 	c->initialized = true;
2499 
2500 	alternative_instructions();
2501 
2502 	if (IS_ENABLED(CONFIG_X86_64)) {
2503 		unsigned long USER_PTR_MAX = TASK_SIZE_MAX;
2504 
2505 		/*
2506 		 * Enable this when LAM is gated on LASS support
2507 		if (cpu_feature_enabled(X86_FEATURE_LAM))
2508 			USER_PTR_MAX = (1ul << 63) - PAGE_SIZE;
2509 		 */
2510 		runtime_const_init(ptr, USER_PTR_MAX);
2511 
2512 		/*
2513 		 * Make sure the first 2MB area is not mapped by huge pages
2514 		 * There are typically fixed size MTRRs in there and overlapping
2515 		 * MTRRs into large pages causes slow downs.
2516 		 *
2517 		 * Right now we don't do that with gbpages because there seems
2518 		 * very little benefit for that case.
2519 		 */
2520 		if (!direct_gbpages)
2521 			set_memory_4k((unsigned long)__va(0), 1);
2522 	} else {
2523 		fpu__init_check_bugs();
2524 	}
2525 
2526 	/*
2527 	 * This needs to be called before any devices perform DMA
2528 	 * operations that might use the SWIOTLB bounce buffers. It will
2529 	 * mark the bounce buffers as decrypted so that their usage will
2530 	 * not cause "plain-text" data to be decrypted when accessed. It
2531 	 * must be called after late_time_init() so that Hyper-V x86/x64
2532 	 * hypercalls work when the SWIOTLB bounce buffers are decrypted.
2533 	 */
2534 	mem_encrypt_init();
2535 }
2536