1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2 
3 #include <linux/kvm_host.h>
4 
5 #include "irq.h"
6 #include "mmu.h"
7 #include "kvm_cache_regs.h"
8 #include "x86.h"
9 #include "smm.h"
10 #include "cpuid.h"
11 #include "pmu.h"
12 
13 #include <linux/module.h>
14 #include <linux/mod_devicetable.h>
15 #include <linux/kernel.h>
16 #include <linux/vmalloc.h>
17 #include <linux/highmem.h>
18 #include <linux/amd-iommu.h>
19 #include <linux/sched.h>
20 #include <linux/trace_events.h>
21 #include <linux/slab.h>
22 #include <linux/hashtable.h>
23 #include <linux/objtool.h>
24 #include <linux/psp-sev.h>
25 #include <linux/file.h>
26 #include <linux/pagemap.h>
27 #include <linux/swap.h>
28 #include <linux/rwsem.h>
29 #include <linux/cc_platform.h>
30 #include <linux/smp.h>
31 #include <linux/string_choices.h>
32 
33 #include <asm/apic.h>
34 #include <asm/perf_event.h>
35 #include <asm/tlbflush.h>
36 #include <asm/desc.h>
37 #include <asm/debugreg.h>
38 #include <asm/kvm_para.h>
39 #include <asm/irq_remapping.h>
40 #include <asm/spec-ctrl.h>
41 #include <asm/cpu_device_id.h>
42 #include <asm/traps.h>
43 #include <asm/reboot.h>
44 #include <asm/fpu/api.h>
45 
46 #include <trace/events/ipi.h>
47 
48 #include "trace.h"
49 
50 #include "svm.h"
51 #include "svm_ops.h"
52 
53 #include "kvm_onhyperv.h"
54 #include "svm_onhyperv.h"
55 
56 MODULE_AUTHOR("Qumranet");
57 MODULE_DESCRIPTION("KVM support for SVM (AMD-V) extensions");
58 MODULE_LICENSE("GPL");
59 
60 #ifdef MODULE
61 static const struct x86_cpu_id svm_cpu_id[] = {
62 	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
63 	{}
64 };
65 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
66 #endif
67 
68 #define SEG_TYPE_LDT 2
69 #define SEG_TYPE_BUSY_TSS16 3
70 
71 static bool erratum_383_found __read_mostly;
72 
73 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
74 
75 /*
76  * Set osvw_len to higher value when updated Revision Guides
77  * are published and we know what the new status bits are
78  */
79 static uint64_t osvw_len = 4, osvw_status;
80 
81 static DEFINE_PER_CPU(u64, current_tsc_ratio);
82 
83 #define X2APIC_MSR(x)	(APIC_BASE_MSR + (x >> 4))
84 
85 static const struct svm_direct_access_msrs {
86 	u32 index;   /* Index of the MSR */
87 	bool always; /* True if intercept is initially cleared */
88 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
89 	{ .index = MSR_STAR,				.always = true  },
90 	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
91 	{ .index = MSR_IA32_SYSENTER_EIP,		.always = false },
92 	{ .index = MSR_IA32_SYSENTER_ESP,		.always = false },
93 #ifdef CONFIG_X86_64
94 	{ .index = MSR_GS_BASE,				.always = true  },
95 	{ .index = MSR_FS_BASE,				.always = true  },
96 	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
97 	{ .index = MSR_LSTAR,				.always = true  },
98 	{ .index = MSR_CSTAR,				.always = true  },
99 	{ .index = MSR_SYSCALL_MASK,			.always = true  },
100 #endif
101 	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
102 	{ .index = MSR_IA32_PRED_CMD,			.always = false },
103 	{ .index = MSR_IA32_FLUSH_CMD,			.always = false },
104 	{ .index = MSR_IA32_DEBUGCTLMSR,		.always = false },
105 	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
106 	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
107 	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
108 	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
109 	{ .index = MSR_IA32_XSS,			.always = false },
110 	{ .index = MSR_EFER,				.always = false },
111 	{ .index = MSR_IA32_CR_PAT,			.always = false },
112 	{ .index = MSR_AMD64_SEV_ES_GHCB,		.always = true  },
113 	{ .index = MSR_TSC_AUX,				.always = false },
114 	{ .index = X2APIC_MSR(APIC_ID),			.always = false },
115 	{ .index = X2APIC_MSR(APIC_LVR),		.always = false },
116 	{ .index = X2APIC_MSR(APIC_TASKPRI),		.always = false },
117 	{ .index = X2APIC_MSR(APIC_ARBPRI),		.always = false },
118 	{ .index = X2APIC_MSR(APIC_PROCPRI),		.always = false },
119 	{ .index = X2APIC_MSR(APIC_EOI),		.always = false },
120 	{ .index = X2APIC_MSR(APIC_RRR),		.always = false },
121 	{ .index = X2APIC_MSR(APIC_LDR),		.always = false },
122 	{ .index = X2APIC_MSR(APIC_DFR),		.always = false },
123 	{ .index = X2APIC_MSR(APIC_SPIV),		.always = false },
124 	{ .index = X2APIC_MSR(APIC_ISR),		.always = false },
125 	{ .index = X2APIC_MSR(APIC_TMR),		.always = false },
126 	{ .index = X2APIC_MSR(APIC_IRR),		.always = false },
127 	{ .index = X2APIC_MSR(APIC_ESR),		.always = false },
128 	{ .index = X2APIC_MSR(APIC_ICR),		.always = false },
129 	{ .index = X2APIC_MSR(APIC_ICR2),		.always = false },
130 
131 	/*
132 	 * Note:
133 	 * AMD does not virtualize APIC TSC-deadline timer mode, but it is
134 	 * emulated by KVM. When setting APIC LVTT (0x832) register bit 18,
135 	 * the AVIC hardware would generate GP fault. Therefore, always
136 	 * intercept the MSR 0x832, and do not setup direct_access_msr.
137 	 */
138 	{ .index = X2APIC_MSR(APIC_LVTTHMR),		.always = false },
139 	{ .index = X2APIC_MSR(APIC_LVTPC),		.always = false },
140 	{ .index = X2APIC_MSR(APIC_LVT0),		.always = false },
141 	{ .index = X2APIC_MSR(APIC_LVT1),		.always = false },
142 	{ .index = X2APIC_MSR(APIC_LVTERR),		.always = false },
143 	{ .index = X2APIC_MSR(APIC_TMICT),		.always = false },
144 	{ .index = X2APIC_MSR(APIC_TMCCT),		.always = false },
145 	{ .index = X2APIC_MSR(APIC_TDCR),		.always = false },
146 	{ .index = MSR_INVALID,				.always = false },
147 };
148 
149 /*
150  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
151  * pause_filter_count: On processors that support Pause filtering(indicated
152  *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
153  *	count value. On VMRUN this value is loaded into an internal counter.
154  *	Each time a pause instruction is executed, this counter is decremented
155  *	until it reaches zero at which time a #VMEXIT is generated if pause
156  *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
157  *	Intercept Filtering for more details.
158  *	This also indicate if ple logic enabled.
159  *
160  * pause_filter_thresh: In addition, some processor families support advanced
161  *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
162  *	the amount of time a guest is allowed to execute in a pause loop.
163  *	In this mode, a 16-bit pause filter threshold field is added in the
164  *	VMCB. The threshold value is a cycle count that is used to reset the
165  *	pause counter. As with simple pause filtering, VMRUN loads the pause
166  *	count value from VMCB into an internal counter. Then, on each pause
167  *	instruction the hardware checks the elapsed number of cycles since
168  *	the most recent pause instruction against the pause filter threshold.
169  *	If the elapsed cycle count is greater than the pause filter threshold,
170  *	then the internal pause count is reloaded from the VMCB and execution
171  *	continues. If the elapsed cycle count is less than the pause filter
172  *	threshold, then the internal pause count is decremented. If the count
173  *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
174  *	triggered. If advanced pause filtering is supported and pause filter
175  *	threshold field is set to zero, the filter will operate in the simpler,
176  *	count only mode.
177  */
178 
179 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
180 module_param(pause_filter_thresh, ushort, 0444);
181 
182 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
183 module_param(pause_filter_count, ushort, 0444);
184 
185 /* Default doubles per-vcpu window every exit. */
186 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
187 module_param(pause_filter_count_grow, ushort, 0444);
188 
189 /* Default resets per-vcpu window every exit to pause_filter_count. */
190 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
191 module_param(pause_filter_count_shrink, ushort, 0444);
192 
193 /* Default is to compute the maximum so we can never overflow. */
194 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
195 module_param(pause_filter_count_max, ushort, 0444);
196 
197 /*
198  * Use nested page tables by default.  Note, NPT may get forced off by
199  * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
200  */
201 bool npt_enabled = true;
202 module_param_named(npt, npt_enabled, bool, 0444);
203 
204 /* allow nested virtualization in KVM/SVM */
205 static int nested = true;
206 module_param(nested, int, 0444);
207 
208 /* enable/disable Next RIP Save */
209 int nrips = true;
210 module_param(nrips, int, 0444);
211 
212 /* enable/disable Virtual VMLOAD VMSAVE */
213 static int vls = true;
214 module_param(vls, int, 0444);
215 
216 /* enable/disable Virtual GIF */
217 int vgif = true;
218 module_param(vgif, int, 0444);
219 
220 /* enable/disable LBR virtualization */
221 int lbrv = true;
222 module_param(lbrv, int, 0444);
223 
224 static int tsc_scaling = true;
225 module_param(tsc_scaling, int, 0444);
226 
227 /*
228  * enable / disable AVIC.  Because the defaults differ for APICv
229  * support between VMX and SVM we cannot use module_param_named.
230  */
231 static bool avic;
232 module_param(avic, bool, 0444);
233 
234 bool __read_mostly dump_invalid_vmcb;
235 module_param(dump_invalid_vmcb, bool, 0644);
236 
237 
238 bool intercept_smi = true;
239 module_param(intercept_smi, bool, 0444);
240 
241 bool vnmi = true;
242 module_param(vnmi, bool, 0444);
243 
244 static bool svm_gp_erratum_intercept = true;
245 
246 static u8 rsm_ins_bytes[] = "\x0f\xaa";
247 
248 static unsigned long iopm_base;
249 
250 DEFINE_PER_CPU(struct svm_cpu_data, svm_data);
251 
252 /*
253  * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
254  * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
255  *
256  * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
257  * defer the restoration of TSC_AUX until the CPU returns to userspace.
258  */
259 static int tsc_aux_uret_slot __read_mostly = -1;
260 
261 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
262 
263 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
264 #define MSRS_RANGE_SIZE 2048
265 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
266 
svm_msrpm_offset(u32 msr)267 u32 svm_msrpm_offset(u32 msr)
268 {
269 	u32 offset;
270 	int i;
271 
272 	for (i = 0; i < NUM_MSR_MAPS; i++) {
273 		if (msr < msrpm_ranges[i] ||
274 		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
275 			continue;
276 
277 		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
278 		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
279 
280 		/* Now we have the u8 offset - but need the u32 offset */
281 		return offset / 4;
282 	}
283 
284 	/* MSR not in any range */
285 	return MSR_INVALID;
286 }
287 
get_npt_level(void)288 static int get_npt_level(void)
289 {
290 #ifdef CONFIG_X86_64
291 	return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
292 #else
293 	return PT32E_ROOT_LEVEL;
294 #endif
295 }
296 
svm_set_efer(struct kvm_vcpu * vcpu,u64 efer)297 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
298 {
299 	struct vcpu_svm *svm = to_svm(vcpu);
300 	u64 old_efer = vcpu->arch.efer;
301 	vcpu->arch.efer = efer;
302 
303 	if (!npt_enabled) {
304 		/* Shadow paging assumes NX to be available.  */
305 		efer |= EFER_NX;
306 
307 		if (!(efer & EFER_LMA))
308 			efer &= ~EFER_LME;
309 	}
310 
311 	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
312 		if (!(efer & EFER_SVME)) {
313 			svm_leave_nested(vcpu);
314 			svm_set_gif(svm, true);
315 			/* #GP intercept is still needed for vmware backdoor */
316 			if (!enable_vmware_backdoor)
317 				clr_exception_intercept(svm, GP_VECTOR);
318 
319 			/*
320 			 * Free the nested guest state, unless we are in SMM.
321 			 * In this case we will return to the nested guest
322 			 * as soon as we leave SMM.
323 			 */
324 			if (!is_smm(vcpu))
325 				svm_free_nested(svm);
326 
327 		} else {
328 			int ret = svm_allocate_nested(svm);
329 
330 			if (ret) {
331 				vcpu->arch.efer = old_efer;
332 				return ret;
333 			}
334 
335 			/*
336 			 * Never intercept #GP for SEV guests, KVM can't
337 			 * decrypt guest memory to workaround the erratum.
338 			 */
339 			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
340 				set_exception_intercept(svm, GP_VECTOR);
341 		}
342 	}
343 
344 	svm->vmcb->save.efer = efer | EFER_SVME;
345 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
346 	return 0;
347 }
348 
svm_get_interrupt_shadow(struct kvm_vcpu * vcpu)349 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
350 {
351 	struct vcpu_svm *svm = to_svm(vcpu);
352 	u32 ret = 0;
353 
354 	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
355 		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
356 	return ret;
357 }
358 
svm_set_interrupt_shadow(struct kvm_vcpu * vcpu,int mask)359 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
360 {
361 	struct vcpu_svm *svm = to_svm(vcpu);
362 
363 	if (mask == 0)
364 		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
365 	else
366 		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
367 
368 }
369 
__svm_skip_emulated_instruction(struct kvm_vcpu * vcpu,bool commit_side_effects)370 static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
371 					   bool commit_side_effects)
372 {
373 	struct vcpu_svm *svm = to_svm(vcpu);
374 	unsigned long old_rflags;
375 
376 	/*
377 	 * SEV-ES does not expose the next RIP. The RIP update is controlled by
378 	 * the type of exit and the #VC handler in the guest.
379 	 */
380 	if (sev_es_guest(vcpu->kvm))
381 		goto done;
382 
383 	if (nrips && svm->vmcb->control.next_rip != 0) {
384 		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
385 		svm->next_rip = svm->vmcb->control.next_rip;
386 	}
387 
388 	if (!svm->next_rip) {
389 		if (unlikely(!commit_side_effects))
390 			old_rflags = svm->vmcb->save.rflags;
391 
392 		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
393 			return 0;
394 
395 		if (unlikely(!commit_side_effects))
396 			svm->vmcb->save.rflags = old_rflags;
397 	} else {
398 		kvm_rip_write(vcpu, svm->next_rip);
399 	}
400 
401 done:
402 	if (likely(commit_side_effects))
403 		svm_set_interrupt_shadow(vcpu, 0);
404 
405 	return 1;
406 }
407 
svm_skip_emulated_instruction(struct kvm_vcpu * vcpu)408 static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
409 {
410 	return __svm_skip_emulated_instruction(vcpu, true);
411 }
412 
svm_update_soft_interrupt_rip(struct kvm_vcpu * vcpu)413 static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
414 {
415 	unsigned long rip, old_rip = kvm_rip_read(vcpu);
416 	struct vcpu_svm *svm = to_svm(vcpu);
417 
418 	/*
419 	 * Due to architectural shortcomings, the CPU doesn't always provide
420 	 * NextRIP, e.g. if KVM intercepted an exception that occurred while
421 	 * the CPU was vectoring an INTO/INT3 in the guest.  Temporarily skip
422 	 * the instruction even if NextRIP is supported to acquire the next
423 	 * RIP so that it can be shoved into the NextRIP field, otherwise
424 	 * hardware will fail to advance guest RIP during event injection.
425 	 * Drop the exception/interrupt if emulation fails and effectively
426 	 * retry the instruction, it's the least awful option.  If NRIPS is
427 	 * in use, the skip must not commit any side effects such as clearing
428 	 * the interrupt shadow or RFLAGS.RF.
429 	 */
430 	if (!__svm_skip_emulated_instruction(vcpu, !nrips))
431 		return -EIO;
432 
433 	rip = kvm_rip_read(vcpu);
434 
435 	/*
436 	 * Save the injection information, even when using next_rip, as the
437 	 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection
438 	 * doesn't complete due to a VM-Exit occurring while the CPU is
439 	 * vectoring the event.   Decoding the instruction isn't guaranteed to
440 	 * work as there may be no backing instruction, e.g. if the event is
441 	 * being injected by L1 for L2, or if the guest is patching INT3 into
442 	 * a different instruction.
443 	 */
444 	svm->soft_int_injected = true;
445 	svm->soft_int_csbase = svm->vmcb->save.cs.base;
446 	svm->soft_int_old_rip = old_rip;
447 	svm->soft_int_next_rip = rip;
448 
449 	if (nrips)
450 		kvm_rip_write(vcpu, old_rip);
451 
452 	if (static_cpu_has(X86_FEATURE_NRIPS))
453 		svm->vmcb->control.next_rip = rip;
454 
455 	return 0;
456 }
457 
svm_inject_exception(struct kvm_vcpu * vcpu)458 static void svm_inject_exception(struct kvm_vcpu *vcpu)
459 {
460 	struct kvm_queued_exception *ex = &vcpu->arch.exception;
461 	struct vcpu_svm *svm = to_svm(vcpu);
462 
463 	kvm_deliver_exception_payload(vcpu, ex);
464 
465 	if (kvm_exception_is_soft(ex->vector) &&
466 	    svm_update_soft_interrupt_rip(vcpu))
467 		return;
468 
469 	svm->vmcb->control.event_inj = ex->vector
470 		| SVM_EVTINJ_VALID
471 		| (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
472 		| SVM_EVTINJ_TYPE_EXEPT;
473 	svm->vmcb->control.event_inj_err = ex->error_code;
474 }
475 
svm_init_erratum_383(void)476 static void svm_init_erratum_383(void)
477 {
478 	u32 low, high;
479 	int err;
480 	u64 val;
481 
482 	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
483 		return;
484 
485 	/* Use _safe variants to not break nested virtualization */
486 	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
487 	if (err)
488 		return;
489 
490 	val |= (1ULL << 47);
491 
492 	low  = lower_32_bits(val);
493 	high = upper_32_bits(val);
494 
495 	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
496 
497 	erratum_383_found = true;
498 }
499 
svm_init_osvw(struct kvm_vcpu * vcpu)500 static void svm_init_osvw(struct kvm_vcpu *vcpu)
501 {
502 	/*
503 	 * Guests should see errata 400 and 415 as fixed (assuming that
504 	 * HLT and IO instructions are intercepted).
505 	 */
506 	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
507 	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
508 
509 	/*
510 	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
511 	 * all osvw.status bits inside that length, including bit 0 (which is
512 	 * reserved for erratum 298), are valid. However, if host processor's
513 	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
514 	 * be conservative here and therefore we tell the guest that erratum 298
515 	 * is present (because we really don't know).
516 	 */
517 	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
518 		vcpu->arch.osvw.status |= 1;
519 }
520 
__kvm_is_svm_supported(void)521 static bool __kvm_is_svm_supported(void)
522 {
523 	int cpu = smp_processor_id();
524 	struct cpuinfo_x86 *c = &cpu_data(cpu);
525 
526 	if (c->x86_vendor != X86_VENDOR_AMD &&
527 	    c->x86_vendor != X86_VENDOR_HYGON) {
528 		pr_err("CPU %d isn't AMD or Hygon\n", cpu);
529 		return false;
530 	}
531 
532 	if (!cpu_has(c, X86_FEATURE_SVM)) {
533 		pr_err("SVM not supported by CPU %d\n", cpu);
534 		return false;
535 	}
536 
537 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
538 		pr_info("KVM is unsupported when running as an SEV guest\n");
539 		return false;
540 	}
541 
542 	return true;
543 }
544 
kvm_is_svm_supported(void)545 static bool kvm_is_svm_supported(void)
546 {
547 	bool supported;
548 
549 	migrate_disable();
550 	supported = __kvm_is_svm_supported();
551 	migrate_enable();
552 
553 	return supported;
554 }
555 
svm_check_processor_compat(void)556 static int svm_check_processor_compat(void)
557 {
558 	if (!__kvm_is_svm_supported())
559 		return -EIO;
560 
561 	return 0;
562 }
563 
__svm_write_tsc_multiplier(u64 multiplier)564 static void __svm_write_tsc_multiplier(u64 multiplier)
565 {
566 	if (multiplier == __this_cpu_read(current_tsc_ratio))
567 		return;
568 
569 	wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
570 	__this_cpu_write(current_tsc_ratio, multiplier);
571 }
572 
sev_es_host_save_area(struct svm_cpu_data * sd)573 static __always_inline struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd)
574 {
575 	return &sd->save_area->host_sev_es_save;
576 }
577 
kvm_cpu_svm_disable(void)578 static inline void kvm_cpu_svm_disable(void)
579 {
580 	uint64_t efer;
581 
582 	wrmsrl(MSR_VM_HSAVE_PA, 0);
583 	rdmsrl(MSR_EFER, efer);
584 	if (efer & EFER_SVME) {
585 		/*
586 		 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
587 		 * NMI aren't blocked.
588 		 */
589 		stgi();
590 		wrmsrl(MSR_EFER, efer & ~EFER_SVME);
591 	}
592 }
593 
svm_emergency_disable_virtualization_cpu(void)594 static void svm_emergency_disable_virtualization_cpu(void)
595 {
596 	kvm_rebooting = true;
597 
598 	kvm_cpu_svm_disable();
599 }
600 
svm_disable_virtualization_cpu(void)601 static void svm_disable_virtualization_cpu(void)
602 {
603 	/* Make sure we clean up behind us */
604 	if (tsc_scaling)
605 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
606 
607 	kvm_cpu_svm_disable();
608 
609 	amd_pmu_disable_virt();
610 }
611 
svm_enable_virtualization_cpu(void)612 static int svm_enable_virtualization_cpu(void)
613 {
614 
615 	struct svm_cpu_data *sd;
616 	uint64_t efer;
617 	int me = raw_smp_processor_id();
618 
619 	rdmsrl(MSR_EFER, efer);
620 	if (efer & EFER_SVME)
621 		return -EBUSY;
622 
623 	sd = per_cpu_ptr(&svm_data, me);
624 	sd->asid_generation = 1;
625 	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
626 	sd->next_asid = sd->max_asid + 1;
627 	sd->min_asid = max_sev_asid + 1;
628 
629 	wrmsrl(MSR_EFER, efer | EFER_SVME);
630 
631 	wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa);
632 
633 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
634 		/*
635 		 * Set the default value, even if we don't use TSC scaling
636 		 * to avoid having stale value in the msr
637 		 */
638 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
639 	}
640 
641 
642 	/*
643 	 * Get OSVW bits.
644 	 *
645 	 * Note that it is possible to have a system with mixed processor
646 	 * revisions and therefore different OSVW bits. If bits are not the same
647 	 * on different processors then choose the worst case (i.e. if erratum
648 	 * is present on one processor and not on another then assume that the
649 	 * erratum is present everywhere).
650 	 */
651 	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
652 		uint64_t len, status = 0;
653 		int err;
654 
655 		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
656 		if (!err)
657 			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
658 						      &err);
659 
660 		if (err)
661 			osvw_status = osvw_len = 0;
662 		else {
663 			if (len < osvw_len)
664 				osvw_len = len;
665 			osvw_status |= status;
666 			osvw_status &= (1ULL << osvw_len) - 1;
667 		}
668 	} else
669 		osvw_status = osvw_len = 0;
670 
671 	svm_init_erratum_383();
672 
673 	amd_pmu_enable_virt();
674 
675 	/*
676 	 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type
677 	 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests.
678 	 * Since Linux does not change the value of TSC_AUX once set, prime the
679 	 * TSC_AUX field now to avoid a RDMSR on every vCPU run.
680 	 */
681 	if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
682 		u32 __maybe_unused msr_hi;
683 
684 		rdmsr(MSR_TSC_AUX, sev_es_host_save_area(sd)->tsc_aux, msr_hi);
685 	}
686 
687 	return 0;
688 }
689 
svm_cpu_uninit(int cpu)690 static void svm_cpu_uninit(int cpu)
691 {
692 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
693 
694 	if (!sd->save_area)
695 		return;
696 
697 	kfree(sd->sev_vmcbs);
698 	__free_page(__sme_pa_to_page(sd->save_area_pa));
699 	sd->save_area_pa = 0;
700 	sd->save_area = NULL;
701 }
702 
svm_cpu_init(int cpu)703 static int svm_cpu_init(int cpu)
704 {
705 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
706 	struct page *save_area_page;
707 	int ret = -ENOMEM;
708 
709 	memset(sd, 0, sizeof(struct svm_cpu_data));
710 	save_area_page = snp_safe_alloc_page_node(cpu_to_node(cpu), GFP_KERNEL);
711 	if (!save_area_page)
712 		return ret;
713 
714 	ret = sev_cpu_init(sd);
715 	if (ret)
716 		goto free_save_area;
717 
718 	sd->save_area = page_address(save_area_page);
719 	sd->save_area_pa = __sme_page_pa(save_area_page);
720 	return 0;
721 
722 free_save_area:
723 	__free_page(save_area_page);
724 	return ret;
725 
726 }
727 
set_dr_intercepts(struct vcpu_svm * svm)728 static void set_dr_intercepts(struct vcpu_svm *svm)
729 {
730 	struct vmcb *vmcb = svm->vmcb01.ptr;
731 
732 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ);
733 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ);
734 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ);
735 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ);
736 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ);
737 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ);
738 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ);
739 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE);
740 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE);
741 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE);
742 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE);
743 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE);
744 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE);
745 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE);
746 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
747 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
748 
749 	recalc_intercepts(svm);
750 }
751 
clr_dr_intercepts(struct vcpu_svm * svm)752 static void clr_dr_intercepts(struct vcpu_svm *svm)
753 {
754 	struct vmcb *vmcb = svm->vmcb01.ptr;
755 
756 	vmcb->control.intercepts[INTERCEPT_DR] = 0;
757 
758 	recalc_intercepts(svm);
759 }
760 
direct_access_msr_slot(u32 msr)761 static int direct_access_msr_slot(u32 msr)
762 {
763 	u32 i;
764 
765 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
766 		if (direct_access_msrs[i].index == msr)
767 			return i;
768 
769 	return -ENOENT;
770 }
771 
set_shadow_msr_intercept(struct kvm_vcpu * vcpu,u32 msr,int read,int write)772 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
773 				     int write)
774 {
775 	struct vcpu_svm *svm = to_svm(vcpu);
776 	int slot = direct_access_msr_slot(msr);
777 
778 	if (slot == -ENOENT)
779 		return;
780 
781 	/* Set the shadow bitmaps to the desired intercept states */
782 	if (read)
783 		set_bit(slot, svm->shadow_msr_intercept.read);
784 	else
785 		clear_bit(slot, svm->shadow_msr_intercept.read);
786 
787 	if (write)
788 		set_bit(slot, svm->shadow_msr_intercept.write);
789 	else
790 		clear_bit(slot, svm->shadow_msr_intercept.write);
791 }
792 
valid_msr_intercept(u32 index)793 static bool valid_msr_intercept(u32 index)
794 {
795 	return direct_access_msr_slot(index) != -ENOENT;
796 }
797 
msr_write_intercepted(struct kvm_vcpu * vcpu,u32 msr)798 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
799 {
800 	u8 bit_write;
801 	unsigned long tmp;
802 	u32 offset;
803 	u32 *msrpm;
804 
805 	/*
806 	 * For non-nested case:
807 	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
808 	 * save it.
809 	 *
810 	 * For nested case:
811 	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
812 	 * save it.
813 	 */
814 	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
815 				      to_svm(vcpu)->msrpm;
816 
817 	offset    = svm_msrpm_offset(msr);
818 	bit_write = 2 * (msr & 0x0f) + 1;
819 	tmp       = msrpm[offset];
820 
821 	BUG_ON(offset == MSR_INVALID);
822 
823 	return test_bit(bit_write, &tmp);
824 }
825 
set_msr_interception_bitmap(struct kvm_vcpu * vcpu,u32 * msrpm,u32 msr,int read,int write)826 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
827 					u32 msr, int read, int write)
828 {
829 	struct vcpu_svm *svm = to_svm(vcpu);
830 	u8 bit_read, bit_write;
831 	unsigned long tmp;
832 	u32 offset;
833 
834 	/*
835 	 * If this warning triggers extend the direct_access_msrs list at the
836 	 * beginning of the file
837 	 */
838 	WARN_ON(!valid_msr_intercept(msr));
839 
840 	/* Enforce non allowed MSRs to trap */
841 	if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
842 		read = 0;
843 
844 	if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
845 		write = 0;
846 
847 	offset    = svm_msrpm_offset(msr);
848 	bit_read  = 2 * (msr & 0x0f);
849 	bit_write = 2 * (msr & 0x0f) + 1;
850 	tmp       = msrpm[offset];
851 
852 	BUG_ON(offset == MSR_INVALID);
853 
854 	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
855 	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
856 
857 	msrpm[offset] = tmp;
858 
859 	svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
860 	svm->nested.force_msr_bitmap_recalc = true;
861 }
862 
set_msr_interception(struct kvm_vcpu * vcpu,u32 * msrpm,u32 msr,int read,int write)863 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
864 			  int read, int write)
865 {
866 	set_shadow_msr_intercept(vcpu, msr, read, write);
867 	set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
868 }
869 
svm_vcpu_alloc_msrpm(void)870 u32 *svm_vcpu_alloc_msrpm(void)
871 {
872 	unsigned int order = get_order(MSRPM_SIZE);
873 	struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
874 	u32 *msrpm;
875 
876 	if (!pages)
877 		return NULL;
878 
879 	msrpm = page_address(pages);
880 	memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
881 
882 	return msrpm;
883 }
884 
svm_vcpu_init_msrpm(struct kvm_vcpu * vcpu,u32 * msrpm)885 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
886 {
887 	int i;
888 
889 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
890 		if (!direct_access_msrs[i].always)
891 			continue;
892 		set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
893 	}
894 }
895 
svm_set_x2apic_msr_interception(struct vcpu_svm * svm,bool intercept)896 void svm_set_x2apic_msr_interception(struct vcpu_svm *svm, bool intercept)
897 {
898 	int i;
899 
900 	if (intercept == svm->x2avic_msrs_intercepted)
901 		return;
902 
903 	if (!x2avic_enabled)
904 		return;
905 
906 	for (i = 0; i < MAX_DIRECT_ACCESS_MSRS; i++) {
907 		int index = direct_access_msrs[i].index;
908 
909 		if ((index < APIC_BASE_MSR) ||
910 		    (index > APIC_BASE_MSR + 0xff))
911 			continue;
912 		set_msr_interception(&svm->vcpu, svm->msrpm, index,
913 				     !intercept, !intercept);
914 	}
915 
916 	svm->x2avic_msrs_intercepted = intercept;
917 }
918 
svm_vcpu_free_msrpm(u32 * msrpm)919 void svm_vcpu_free_msrpm(u32 *msrpm)
920 {
921 	__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
922 }
923 
svm_msr_filter_changed(struct kvm_vcpu * vcpu)924 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
925 {
926 	struct vcpu_svm *svm = to_svm(vcpu);
927 	u32 i;
928 
929 	/*
930 	 * Set intercept permissions for all direct access MSRs again. They
931 	 * will automatically get filtered through the MSR filter, so we are
932 	 * back in sync after this.
933 	 */
934 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
935 		u32 msr = direct_access_msrs[i].index;
936 		u32 read = test_bit(i, svm->shadow_msr_intercept.read);
937 		u32 write = test_bit(i, svm->shadow_msr_intercept.write);
938 
939 		set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
940 	}
941 }
942 
add_msr_offset(u32 offset)943 static void add_msr_offset(u32 offset)
944 {
945 	int i;
946 
947 	for (i = 0; i < MSRPM_OFFSETS; ++i) {
948 
949 		/* Offset already in list? */
950 		if (msrpm_offsets[i] == offset)
951 			return;
952 
953 		/* Slot used by another offset? */
954 		if (msrpm_offsets[i] != MSR_INVALID)
955 			continue;
956 
957 		/* Add offset to list */
958 		msrpm_offsets[i] = offset;
959 
960 		return;
961 	}
962 
963 	/*
964 	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
965 	 * increase MSRPM_OFFSETS in this case.
966 	 */
967 	BUG();
968 }
969 
init_msrpm_offsets(void)970 static void init_msrpm_offsets(void)
971 {
972 	int i;
973 
974 	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
975 
976 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
977 		u32 offset;
978 
979 		offset = svm_msrpm_offset(direct_access_msrs[i].index);
980 		BUG_ON(offset == MSR_INVALID);
981 
982 		add_msr_offset(offset);
983 	}
984 }
985 
svm_copy_lbrs(struct vmcb * to_vmcb,struct vmcb * from_vmcb)986 void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
987 {
988 	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
989 	to_vmcb->save.br_from		= from_vmcb->save.br_from;
990 	to_vmcb->save.br_to		= from_vmcb->save.br_to;
991 	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
992 	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
993 
994 	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
995 }
996 
svm_enable_lbrv(struct kvm_vcpu * vcpu)997 void svm_enable_lbrv(struct kvm_vcpu *vcpu)
998 {
999 	struct vcpu_svm *svm = to_svm(vcpu);
1000 
1001 	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1002 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1003 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1004 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1005 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1006 
1007 	if (sev_es_guest(vcpu->kvm))
1008 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1);
1009 
1010 	/* Move the LBR msrs to the vmcb02 so that the guest can see them. */
1011 	if (is_guest_mode(vcpu))
1012 		svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr);
1013 }
1014 
svm_disable_lbrv(struct kvm_vcpu * vcpu)1015 static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
1016 {
1017 	struct vcpu_svm *svm = to_svm(vcpu);
1018 
1019 	KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm);
1020 
1021 	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1022 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1023 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1024 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1025 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1026 
1027 	/*
1028 	 * Move the LBR msrs back to the vmcb01 to avoid copying them
1029 	 * on nested guest entries.
1030 	 */
1031 	if (is_guest_mode(vcpu))
1032 		svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb);
1033 }
1034 
svm_get_lbr_vmcb(struct vcpu_svm * svm)1035 static struct vmcb *svm_get_lbr_vmcb(struct vcpu_svm *svm)
1036 {
1037 	/*
1038 	 * If LBR virtualization is disabled, the LBR MSRs are always kept in
1039 	 * vmcb01.  If LBR virtualization is enabled and L1 is running VMs of
1040 	 * its own, the MSRs are moved between vmcb01 and vmcb02 as needed.
1041 	 */
1042 	return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb :
1043 								   svm->vmcb01.ptr;
1044 }
1045 
svm_update_lbrv(struct kvm_vcpu * vcpu)1046 void svm_update_lbrv(struct kvm_vcpu *vcpu)
1047 {
1048 	struct vcpu_svm *svm = to_svm(vcpu);
1049 	bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
1050 	bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) ||
1051 			    (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
1052 			    (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
1053 
1054 	if (enable_lbrv == current_enable_lbrv)
1055 		return;
1056 
1057 	if (enable_lbrv)
1058 		svm_enable_lbrv(vcpu);
1059 	else
1060 		svm_disable_lbrv(vcpu);
1061 }
1062 
disable_nmi_singlestep(struct vcpu_svm * svm)1063 void disable_nmi_singlestep(struct vcpu_svm *svm)
1064 {
1065 	svm->nmi_singlestep = false;
1066 
1067 	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1068 		/* Clear our flags if they were not set by the guest */
1069 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1070 			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1071 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1072 			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1073 	}
1074 }
1075 
grow_ple_window(struct kvm_vcpu * vcpu)1076 static void grow_ple_window(struct kvm_vcpu *vcpu)
1077 {
1078 	struct vcpu_svm *svm = to_svm(vcpu);
1079 	struct vmcb_control_area *control = &svm->vmcb->control;
1080 	int old = control->pause_filter_count;
1081 
1082 	if (kvm_pause_in_guest(vcpu->kvm))
1083 		return;
1084 
1085 	control->pause_filter_count = __grow_ple_window(old,
1086 							pause_filter_count,
1087 							pause_filter_count_grow,
1088 							pause_filter_count_max);
1089 
1090 	if (control->pause_filter_count != old) {
1091 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1092 		trace_kvm_ple_window_update(vcpu->vcpu_id,
1093 					    control->pause_filter_count, old);
1094 	}
1095 }
1096 
shrink_ple_window(struct kvm_vcpu * vcpu)1097 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1098 {
1099 	struct vcpu_svm *svm = to_svm(vcpu);
1100 	struct vmcb_control_area *control = &svm->vmcb->control;
1101 	int old = control->pause_filter_count;
1102 
1103 	if (kvm_pause_in_guest(vcpu->kvm))
1104 		return;
1105 
1106 	control->pause_filter_count =
1107 				__shrink_ple_window(old,
1108 						    pause_filter_count,
1109 						    pause_filter_count_shrink,
1110 						    pause_filter_count);
1111 	if (control->pause_filter_count != old) {
1112 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1113 		trace_kvm_ple_window_update(vcpu->vcpu_id,
1114 					    control->pause_filter_count, old);
1115 	}
1116 }
1117 
svm_hardware_unsetup(void)1118 static void svm_hardware_unsetup(void)
1119 {
1120 	int cpu;
1121 
1122 	sev_hardware_unsetup();
1123 
1124 	for_each_possible_cpu(cpu)
1125 		svm_cpu_uninit(cpu);
1126 
1127 	__free_pages(__sme_pa_to_page(iopm_base), get_order(IOPM_SIZE));
1128 	iopm_base = 0;
1129 }
1130 
init_seg(struct vmcb_seg * seg)1131 static void init_seg(struct vmcb_seg *seg)
1132 {
1133 	seg->selector = 0;
1134 	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1135 		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1136 	seg->limit = 0xffff;
1137 	seg->base = 0;
1138 }
1139 
init_sys_seg(struct vmcb_seg * seg,uint32_t type)1140 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1141 {
1142 	seg->selector = 0;
1143 	seg->attrib = SVM_SELECTOR_P_MASK | type;
1144 	seg->limit = 0xffff;
1145 	seg->base = 0;
1146 }
1147 
svm_get_l2_tsc_offset(struct kvm_vcpu * vcpu)1148 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1149 {
1150 	struct vcpu_svm *svm = to_svm(vcpu);
1151 
1152 	return svm->nested.ctl.tsc_offset;
1153 }
1154 
svm_get_l2_tsc_multiplier(struct kvm_vcpu * vcpu)1155 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1156 {
1157 	struct vcpu_svm *svm = to_svm(vcpu);
1158 
1159 	return svm->tsc_ratio_msr;
1160 }
1161 
svm_write_tsc_offset(struct kvm_vcpu * vcpu)1162 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu)
1163 {
1164 	struct vcpu_svm *svm = to_svm(vcpu);
1165 
1166 	svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1167 	svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset;
1168 	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1169 }
1170 
svm_write_tsc_multiplier(struct kvm_vcpu * vcpu)1171 void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu)
1172 {
1173 	preempt_disable();
1174 	if (to_svm(vcpu)->guest_state_loaded)
1175 		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1176 	preempt_enable();
1177 }
1178 
1179 /* Evaluate instruction intercepts that depend on guest CPUID features. */
svm_recalc_instruction_intercepts(struct kvm_vcpu * vcpu,struct vcpu_svm * svm)1180 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1181 					      struct vcpu_svm *svm)
1182 {
1183 	/*
1184 	 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
1185 	 * roots, or if INVPCID is disabled in the guest to inject #UD.
1186 	 */
1187 	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1188 		if (!npt_enabled ||
1189 		    !guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_INVPCID))
1190 			svm_set_intercept(svm, INTERCEPT_INVPCID);
1191 		else
1192 			svm_clr_intercept(svm, INTERCEPT_INVPCID);
1193 	}
1194 
1195 	if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1196 		if (guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP))
1197 			svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1198 		else
1199 			svm_set_intercept(svm, INTERCEPT_RDTSCP);
1200 	}
1201 }
1202 
init_vmcb_after_set_cpuid(struct kvm_vcpu * vcpu)1203 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu)
1204 {
1205 	struct vcpu_svm *svm = to_svm(vcpu);
1206 
1207 	if (guest_cpuid_is_intel_compatible(vcpu)) {
1208 		/*
1209 		 * We must intercept SYSENTER_EIP and SYSENTER_ESP
1210 		 * accesses because the processor only stores 32 bits.
1211 		 * For the same reason we cannot use virtual VMLOAD/VMSAVE.
1212 		 */
1213 		svm_set_intercept(svm, INTERCEPT_VMLOAD);
1214 		svm_set_intercept(svm, INTERCEPT_VMSAVE);
1215 		svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1216 
1217 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
1218 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
1219 	} else {
1220 		/*
1221 		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1222 		 * in VMCB and clear intercepts to avoid #VMEXIT.
1223 		 */
1224 		if (vls) {
1225 			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1226 			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1227 			svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1228 		}
1229 		/* No need to intercept these MSRs */
1230 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
1231 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
1232 	}
1233 }
1234 
init_vmcb(struct kvm_vcpu * vcpu)1235 static void init_vmcb(struct kvm_vcpu *vcpu)
1236 {
1237 	struct vcpu_svm *svm = to_svm(vcpu);
1238 	struct vmcb *vmcb = svm->vmcb01.ptr;
1239 	struct vmcb_control_area *control = &vmcb->control;
1240 	struct vmcb_save_area *save = &vmcb->save;
1241 
1242 	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1243 	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1244 	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1245 	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1246 	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1247 	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1248 	if (!kvm_vcpu_apicv_active(vcpu))
1249 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1250 
1251 	set_dr_intercepts(svm);
1252 
1253 	set_exception_intercept(svm, PF_VECTOR);
1254 	set_exception_intercept(svm, UD_VECTOR);
1255 	set_exception_intercept(svm, MC_VECTOR);
1256 	set_exception_intercept(svm, AC_VECTOR);
1257 	set_exception_intercept(svm, DB_VECTOR);
1258 	/*
1259 	 * Guest access to VMware backdoor ports could legitimately
1260 	 * trigger #GP because of TSS I/O permission bitmap.
1261 	 * We intercept those #GP and allow access to them anyway
1262 	 * as VMware does.
1263 	 */
1264 	if (enable_vmware_backdoor)
1265 		set_exception_intercept(svm, GP_VECTOR);
1266 
1267 	svm_set_intercept(svm, INTERCEPT_INTR);
1268 	svm_set_intercept(svm, INTERCEPT_NMI);
1269 
1270 	if (intercept_smi)
1271 		svm_set_intercept(svm, INTERCEPT_SMI);
1272 
1273 	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1274 	svm_set_intercept(svm, INTERCEPT_RDPMC);
1275 	svm_set_intercept(svm, INTERCEPT_CPUID);
1276 	svm_set_intercept(svm, INTERCEPT_INVD);
1277 	svm_set_intercept(svm, INTERCEPT_INVLPG);
1278 	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1279 	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1280 	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1281 	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1282 	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1283 	svm_set_intercept(svm, INTERCEPT_VMRUN);
1284 	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1285 	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1286 	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1287 	svm_set_intercept(svm, INTERCEPT_STGI);
1288 	svm_set_intercept(svm, INTERCEPT_CLGI);
1289 	svm_set_intercept(svm, INTERCEPT_SKINIT);
1290 	svm_set_intercept(svm, INTERCEPT_WBINVD);
1291 	svm_set_intercept(svm, INTERCEPT_XSETBV);
1292 	svm_set_intercept(svm, INTERCEPT_RDPRU);
1293 	svm_set_intercept(svm, INTERCEPT_RSM);
1294 
1295 	if (!kvm_mwait_in_guest(vcpu->kvm)) {
1296 		svm_set_intercept(svm, INTERCEPT_MONITOR);
1297 		svm_set_intercept(svm, INTERCEPT_MWAIT);
1298 	}
1299 
1300 	if (!kvm_hlt_in_guest(vcpu->kvm)) {
1301 		if (cpu_feature_enabled(X86_FEATURE_IDLE_HLT))
1302 			svm_set_intercept(svm, INTERCEPT_IDLE_HLT);
1303 		else
1304 			svm_set_intercept(svm, INTERCEPT_HLT);
1305 	}
1306 
1307 	control->iopm_base_pa = iopm_base;
1308 	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1309 	control->int_ctl = V_INTR_MASKING_MASK;
1310 
1311 	init_seg(&save->es);
1312 	init_seg(&save->ss);
1313 	init_seg(&save->ds);
1314 	init_seg(&save->fs);
1315 	init_seg(&save->gs);
1316 
1317 	save->cs.selector = 0xf000;
1318 	save->cs.base = 0xffff0000;
1319 	/* Executable/Readable Code Segment */
1320 	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1321 		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1322 	save->cs.limit = 0xffff;
1323 
1324 	save->gdtr.base = 0;
1325 	save->gdtr.limit = 0xffff;
1326 	save->idtr.base = 0;
1327 	save->idtr.limit = 0xffff;
1328 
1329 	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1330 	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1331 
1332 	if (npt_enabled) {
1333 		/* Setup VMCB for Nested Paging */
1334 		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1335 		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1336 		clr_exception_intercept(svm, PF_VECTOR);
1337 		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1338 		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1339 		save->g_pat = vcpu->arch.pat;
1340 		save->cr3 = 0;
1341 	}
1342 	svm->current_vmcb->asid_generation = 0;
1343 	svm->asid = 0;
1344 
1345 	svm->nested.vmcb12_gpa = INVALID_GPA;
1346 	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1347 
1348 	if (!kvm_pause_in_guest(vcpu->kvm)) {
1349 		control->pause_filter_count = pause_filter_count;
1350 		if (pause_filter_thresh)
1351 			control->pause_filter_thresh = pause_filter_thresh;
1352 		svm_set_intercept(svm, INTERCEPT_PAUSE);
1353 	} else {
1354 		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1355 	}
1356 
1357 	svm_recalc_instruction_intercepts(vcpu, svm);
1358 
1359 	/*
1360 	 * If the host supports V_SPEC_CTRL then disable the interception
1361 	 * of MSR_IA32_SPEC_CTRL.
1362 	 */
1363 	if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1364 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1365 
1366 	if (kvm_vcpu_apicv_active(vcpu))
1367 		avic_init_vmcb(svm, vmcb);
1368 
1369 	if (vnmi)
1370 		svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK;
1371 
1372 	if (vgif) {
1373 		svm_clr_intercept(svm, INTERCEPT_STGI);
1374 		svm_clr_intercept(svm, INTERCEPT_CLGI);
1375 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1376 	}
1377 
1378 	if (sev_guest(vcpu->kvm))
1379 		sev_init_vmcb(svm);
1380 
1381 	svm_hv_init_vmcb(vmcb);
1382 	init_vmcb_after_set_cpuid(vcpu);
1383 
1384 	vmcb_mark_all_dirty(vmcb);
1385 
1386 	enable_gif(svm);
1387 }
1388 
__svm_vcpu_reset(struct kvm_vcpu * vcpu)1389 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1390 {
1391 	struct vcpu_svm *svm = to_svm(vcpu);
1392 
1393 	svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1394 
1395 	svm_init_osvw(vcpu);
1396 
1397 	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS))
1398 		vcpu->arch.microcode_version = 0x01000065;
1399 	svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio;
1400 
1401 	svm->nmi_masked = false;
1402 	svm->awaiting_iret_completion = false;
1403 
1404 	if (sev_es_guest(vcpu->kvm))
1405 		sev_es_vcpu_reset(svm);
1406 }
1407 
svm_vcpu_reset(struct kvm_vcpu * vcpu,bool init_event)1408 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1409 {
1410 	struct vcpu_svm *svm = to_svm(vcpu);
1411 
1412 	svm->spec_ctrl = 0;
1413 	svm->virt_spec_ctrl = 0;
1414 
1415 	if (init_event)
1416 		sev_snp_init_protected_guest_state(vcpu);
1417 
1418 	init_vmcb(vcpu);
1419 
1420 	if (!init_event)
1421 		__svm_vcpu_reset(vcpu);
1422 }
1423 
svm_switch_vmcb(struct vcpu_svm * svm,struct kvm_vmcb_info * target_vmcb)1424 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1425 {
1426 	svm->current_vmcb = target_vmcb;
1427 	svm->vmcb = target_vmcb->ptr;
1428 }
1429 
svm_vcpu_create(struct kvm_vcpu * vcpu)1430 static int svm_vcpu_create(struct kvm_vcpu *vcpu)
1431 {
1432 	struct vcpu_svm *svm;
1433 	struct page *vmcb01_page;
1434 	struct page *vmsa_page = NULL;
1435 	int err;
1436 
1437 	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1438 	svm = to_svm(vcpu);
1439 
1440 	err = -ENOMEM;
1441 	vmcb01_page = snp_safe_alloc_page();
1442 	if (!vmcb01_page)
1443 		goto out;
1444 
1445 	if (sev_es_guest(vcpu->kvm)) {
1446 		/*
1447 		 * SEV-ES guests require a separate VMSA page used to contain
1448 		 * the encrypted register state of the guest.
1449 		 */
1450 		vmsa_page = snp_safe_alloc_page();
1451 		if (!vmsa_page)
1452 			goto error_free_vmcb_page;
1453 	}
1454 
1455 	err = avic_init_vcpu(svm);
1456 	if (err)
1457 		goto error_free_vmsa_page;
1458 
1459 	svm->msrpm = svm_vcpu_alloc_msrpm();
1460 	if (!svm->msrpm) {
1461 		err = -ENOMEM;
1462 		goto error_free_vmsa_page;
1463 	}
1464 
1465 	svm->x2avic_msrs_intercepted = true;
1466 
1467 	svm->vmcb01.ptr = page_address(vmcb01_page);
1468 	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1469 	svm_switch_vmcb(svm, &svm->vmcb01);
1470 
1471 	if (vmsa_page)
1472 		svm->sev_es.vmsa = page_address(vmsa_page);
1473 
1474 	svm->guest_state_loaded = false;
1475 
1476 	return 0;
1477 
1478 error_free_vmsa_page:
1479 	if (vmsa_page)
1480 		__free_page(vmsa_page);
1481 error_free_vmcb_page:
1482 	__free_page(vmcb01_page);
1483 out:
1484 	return err;
1485 }
1486 
svm_clear_current_vmcb(struct vmcb * vmcb)1487 static void svm_clear_current_vmcb(struct vmcb *vmcb)
1488 {
1489 	int i;
1490 
1491 	for_each_online_cpu(i)
1492 		cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL);
1493 }
1494 
svm_vcpu_free(struct kvm_vcpu * vcpu)1495 static void svm_vcpu_free(struct kvm_vcpu *vcpu)
1496 {
1497 	struct vcpu_svm *svm = to_svm(vcpu);
1498 
1499 	/*
1500 	 * The vmcb page can be recycled, causing a false negative in
1501 	 * svm_vcpu_load(). So, ensure that no logical CPU has this
1502 	 * vmcb page recorded as its current vmcb.
1503 	 */
1504 	svm_clear_current_vmcb(svm->vmcb);
1505 
1506 	svm_leave_nested(vcpu);
1507 	svm_free_nested(svm);
1508 
1509 	sev_free_vcpu(vcpu);
1510 
1511 	__free_page(__sme_pa_to_page(svm->vmcb01.pa));
1512 	__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1513 }
1514 
1515 #ifdef CONFIG_CPU_MITIGATIONS
1516 static DEFINE_SPINLOCK(srso_lock);
1517 static atomic_t srso_nr_vms;
1518 
svm_srso_clear_bp_spec_reduce(void * ign)1519 static void svm_srso_clear_bp_spec_reduce(void *ign)
1520 {
1521 	struct svm_cpu_data *sd = this_cpu_ptr(&svm_data);
1522 
1523 	if (!sd->bp_spec_reduce_set)
1524 		return;
1525 
1526 	msr_clear_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
1527 	sd->bp_spec_reduce_set = false;
1528 }
1529 
svm_srso_vm_destroy(void)1530 static void svm_srso_vm_destroy(void)
1531 {
1532 	if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
1533 		return;
1534 
1535 	if (atomic_dec_return(&srso_nr_vms))
1536 		return;
1537 
1538 	guard(spinlock)(&srso_lock);
1539 
1540 	/*
1541 	 * Verify a new VM didn't come along, acquire the lock, and increment
1542 	 * the count before this task acquired the lock.
1543 	 */
1544 	if (atomic_read(&srso_nr_vms))
1545 		return;
1546 
1547 	on_each_cpu(svm_srso_clear_bp_spec_reduce, NULL, 1);
1548 }
1549 
svm_srso_vm_init(void)1550 static void svm_srso_vm_init(void)
1551 {
1552 	if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
1553 		return;
1554 
1555 	/*
1556 	 * Acquire the lock on 0 => 1 transitions to ensure a potential 1 => 0
1557 	 * transition, i.e. destroying the last VM, is fully complete, e.g. so
1558 	 * that a delayed IPI doesn't clear BP_SPEC_REDUCE after a vCPU runs.
1559 	 */
1560 	if (atomic_inc_not_zero(&srso_nr_vms))
1561 		return;
1562 
1563 	guard(spinlock)(&srso_lock);
1564 
1565 	atomic_inc(&srso_nr_vms);
1566 }
1567 #else
svm_srso_vm_init(void)1568 static void svm_srso_vm_init(void) { }
svm_srso_vm_destroy(void)1569 static void svm_srso_vm_destroy(void) { }
1570 #endif
1571 
svm_prepare_switch_to_guest(struct kvm_vcpu * vcpu)1572 static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1573 {
1574 	struct vcpu_svm *svm = to_svm(vcpu);
1575 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
1576 
1577 	if (sev_es_guest(vcpu->kvm))
1578 		sev_es_unmap_ghcb(svm);
1579 
1580 	if (svm->guest_state_loaded)
1581 		return;
1582 
1583 	/*
1584 	 * Save additional host state that will be restored on VMEXIT (sev-es)
1585 	 * or subsequent vmload of host save area.
1586 	 */
1587 	vmsave(sd->save_area_pa);
1588 	if (sev_es_guest(vcpu->kvm))
1589 		sev_es_prepare_switch_to_guest(svm, sev_es_host_save_area(sd));
1590 
1591 	if (tsc_scaling)
1592 		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1593 
1594 	/*
1595 	 * TSC_AUX is always virtualized for SEV-ES guests when the feature is
1596 	 * available. The user return MSR support is not required in this case
1597 	 * because TSC_AUX is restored on #VMEXIT from the host save area
1598 	 * (which has been initialized in svm_enable_virtualization_cpu()).
1599 	 */
1600 	if (likely(tsc_aux_uret_slot >= 0) &&
1601 	    (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1602 		kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1603 
1604 	if (cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE) &&
1605 	    !sd->bp_spec_reduce_set) {
1606 		sd->bp_spec_reduce_set = true;
1607 		msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
1608 	}
1609 	svm->guest_state_loaded = true;
1610 }
1611 
svm_prepare_host_switch(struct kvm_vcpu * vcpu)1612 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1613 {
1614 	to_svm(vcpu)->guest_state_loaded = false;
1615 }
1616 
svm_vcpu_load(struct kvm_vcpu * vcpu,int cpu)1617 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1618 {
1619 	struct vcpu_svm *svm = to_svm(vcpu);
1620 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
1621 
1622 	if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm))
1623 		shrink_ple_window(vcpu);
1624 
1625 	if (sd->current_vmcb != svm->vmcb) {
1626 		sd->current_vmcb = svm->vmcb;
1627 
1628 		if (!cpu_feature_enabled(X86_FEATURE_IBPB_ON_VMEXIT) &&
1629 		    static_branch_likely(&switch_vcpu_ibpb))
1630 			indirect_branch_prediction_barrier();
1631 	}
1632 	if (kvm_vcpu_apicv_active(vcpu))
1633 		avic_vcpu_load(vcpu, cpu);
1634 }
1635 
svm_vcpu_put(struct kvm_vcpu * vcpu)1636 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1637 {
1638 	if (kvm_vcpu_apicv_active(vcpu))
1639 		avic_vcpu_put(vcpu);
1640 
1641 	svm_prepare_host_switch(vcpu);
1642 
1643 	++vcpu->stat.host_state_reload;
1644 }
1645 
svm_get_rflags(struct kvm_vcpu * vcpu)1646 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1647 {
1648 	struct vcpu_svm *svm = to_svm(vcpu);
1649 	unsigned long rflags = svm->vmcb->save.rflags;
1650 
1651 	if (svm->nmi_singlestep) {
1652 		/* Hide our flags if they were not set by the guest */
1653 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1654 			rflags &= ~X86_EFLAGS_TF;
1655 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1656 			rflags &= ~X86_EFLAGS_RF;
1657 	}
1658 	return rflags;
1659 }
1660 
svm_set_rflags(struct kvm_vcpu * vcpu,unsigned long rflags)1661 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1662 {
1663 	if (to_svm(vcpu)->nmi_singlestep)
1664 		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1665 
1666        /*
1667         * Any change of EFLAGS.VM is accompanied by a reload of SS
1668         * (caused by either a task switch or an inter-privilege IRET),
1669         * so we do not need to update the CPL here.
1670         */
1671 	to_svm(vcpu)->vmcb->save.rflags = rflags;
1672 }
1673 
svm_get_if_flag(struct kvm_vcpu * vcpu)1674 static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
1675 {
1676 	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1677 
1678 	return sev_es_guest(vcpu->kvm)
1679 		? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
1680 		: kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
1681 }
1682 
svm_cache_reg(struct kvm_vcpu * vcpu,enum kvm_reg reg)1683 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1684 {
1685 	kvm_register_mark_available(vcpu, reg);
1686 
1687 	switch (reg) {
1688 	case VCPU_EXREG_PDPTR:
1689 		/*
1690 		 * When !npt_enabled, mmu->pdptrs[] is already available since
1691 		 * it is always updated per SDM when moving to CRs.
1692 		 */
1693 		if (npt_enabled)
1694 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
1695 		break;
1696 	default:
1697 		KVM_BUG_ON(1, vcpu->kvm);
1698 	}
1699 }
1700 
svm_set_vintr(struct vcpu_svm * svm)1701 static void svm_set_vintr(struct vcpu_svm *svm)
1702 {
1703 	struct vmcb_control_area *control;
1704 
1705 	/*
1706 	 * The following fields are ignored when AVIC is enabled
1707 	 */
1708 	WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu));
1709 
1710 	svm_set_intercept(svm, INTERCEPT_VINTR);
1711 
1712 	/*
1713 	 * Recalculating intercepts may have cleared the VINTR intercept.  If
1714 	 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF
1715 	 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN.
1716 	 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as
1717 	 * interrupts will never be unblocked while L2 is running.
1718 	 */
1719 	if (!svm_is_intercept(svm, INTERCEPT_VINTR))
1720 		return;
1721 
1722 	/*
1723 	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1724 	 * Actual injection of virtual interrupts happens through EVENTINJ.
1725 	 */
1726 	control = &svm->vmcb->control;
1727 	control->int_vector = 0x0;
1728 	control->int_ctl &= ~V_INTR_PRIO_MASK;
1729 	control->int_ctl |= V_IRQ_MASK |
1730 		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1731 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1732 }
1733 
svm_clear_vintr(struct vcpu_svm * svm)1734 static void svm_clear_vintr(struct vcpu_svm *svm)
1735 {
1736 	svm_clr_intercept(svm, INTERCEPT_VINTR);
1737 
1738 	/* Drop int_ctl fields related to VINTR injection.  */
1739 	svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1740 	if (is_guest_mode(&svm->vcpu)) {
1741 		svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1742 
1743 		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1744 			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1745 
1746 		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1747 			V_IRQ_INJECTION_BITS_MASK;
1748 
1749 		svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1750 	}
1751 
1752 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1753 }
1754 
svm_seg(struct kvm_vcpu * vcpu,int seg)1755 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1756 {
1757 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1758 	struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1759 
1760 	switch (seg) {
1761 	case VCPU_SREG_CS: return &save->cs;
1762 	case VCPU_SREG_DS: return &save->ds;
1763 	case VCPU_SREG_ES: return &save->es;
1764 	case VCPU_SREG_FS: return &save01->fs;
1765 	case VCPU_SREG_GS: return &save01->gs;
1766 	case VCPU_SREG_SS: return &save->ss;
1767 	case VCPU_SREG_TR: return &save01->tr;
1768 	case VCPU_SREG_LDTR: return &save01->ldtr;
1769 	}
1770 	BUG();
1771 	return NULL;
1772 }
1773 
svm_get_segment_base(struct kvm_vcpu * vcpu,int seg)1774 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1775 {
1776 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1777 
1778 	return s->base;
1779 }
1780 
svm_get_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)1781 static void svm_get_segment(struct kvm_vcpu *vcpu,
1782 			    struct kvm_segment *var, int seg)
1783 {
1784 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1785 
1786 	var->base = s->base;
1787 	var->limit = s->limit;
1788 	var->selector = s->selector;
1789 	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1790 	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1791 	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1792 	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1793 	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1794 	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1795 	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1796 
1797 	/*
1798 	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1799 	 * However, the SVM spec states that the G bit is not observed by the
1800 	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1801 	 * So let's synthesize a legal G bit for all segments, this helps
1802 	 * running KVM nested. It also helps cross-vendor migration, because
1803 	 * Intel's vmentry has a check on the 'G' bit.
1804 	 */
1805 	var->g = s->limit > 0xfffff;
1806 
1807 	/*
1808 	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1809 	 * for cross vendor migration purposes by "not present"
1810 	 */
1811 	var->unusable = !var->present;
1812 
1813 	switch (seg) {
1814 	case VCPU_SREG_TR:
1815 		/*
1816 		 * Work around a bug where the busy flag in the tr selector
1817 		 * isn't exposed
1818 		 */
1819 		var->type |= 0x2;
1820 		break;
1821 	case VCPU_SREG_DS:
1822 	case VCPU_SREG_ES:
1823 	case VCPU_SREG_FS:
1824 	case VCPU_SREG_GS:
1825 		/*
1826 		 * The accessed bit must always be set in the segment
1827 		 * descriptor cache, although it can be cleared in the
1828 		 * descriptor, the cached bit always remains at 1. Since
1829 		 * Intel has a check on this, set it here to support
1830 		 * cross-vendor migration.
1831 		 */
1832 		if (!var->unusable)
1833 			var->type |= 0x1;
1834 		break;
1835 	case VCPU_SREG_SS:
1836 		/*
1837 		 * On AMD CPUs sometimes the DB bit in the segment
1838 		 * descriptor is left as 1, although the whole segment has
1839 		 * been made unusable. Clear it here to pass an Intel VMX
1840 		 * entry check when cross vendor migrating.
1841 		 */
1842 		if (var->unusable)
1843 			var->db = 0;
1844 		/* This is symmetric with svm_set_segment() */
1845 		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1846 		break;
1847 	}
1848 }
1849 
svm_get_cpl(struct kvm_vcpu * vcpu)1850 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1851 {
1852 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1853 
1854 	return save->cpl;
1855 }
1856 
svm_get_cs_db_l_bits(struct kvm_vcpu * vcpu,int * db,int * l)1857 static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1858 {
1859 	struct kvm_segment cs;
1860 
1861 	svm_get_segment(vcpu, &cs, VCPU_SREG_CS);
1862 	*db = cs.db;
1863 	*l = cs.l;
1864 }
1865 
svm_get_idt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1866 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1867 {
1868 	struct vcpu_svm *svm = to_svm(vcpu);
1869 
1870 	dt->size = svm->vmcb->save.idtr.limit;
1871 	dt->address = svm->vmcb->save.idtr.base;
1872 }
1873 
svm_set_idt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1874 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1875 {
1876 	struct vcpu_svm *svm = to_svm(vcpu);
1877 
1878 	svm->vmcb->save.idtr.limit = dt->size;
1879 	svm->vmcb->save.idtr.base = dt->address ;
1880 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1881 }
1882 
svm_get_gdt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1883 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1884 {
1885 	struct vcpu_svm *svm = to_svm(vcpu);
1886 
1887 	dt->size = svm->vmcb->save.gdtr.limit;
1888 	dt->address = svm->vmcb->save.gdtr.base;
1889 }
1890 
svm_set_gdt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1891 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1892 {
1893 	struct vcpu_svm *svm = to_svm(vcpu);
1894 
1895 	svm->vmcb->save.gdtr.limit = dt->size;
1896 	svm->vmcb->save.gdtr.base = dt->address ;
1897 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1898 }
1899 
sev_post_set_cr3(struct kvm_vcpu * vcpu,unsigned long cr3)1900 static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1901 {
1902 	struct vcpu_svm *svm = to_svm(vcpu);
1903 
1904 	/*
1905 	 * For guests that don't set guest_state_protected, the cr3 update is
1906 	 * handled via kvm_mmu_load() while entering the guest. For guests
1907 	 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1908 	 * VMCB save area now, since the save area will become the initial
1909 	 * contents of the VMSA, and future VMCB save area updates won't be
1910 	 * seen.
1911 	 */
1912 	if (sev_es_guest(vcpu->kvm)) {
1913 		svm->vmcb->save.cr3 = cr3;
1914 		vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1915 	}
1916 }
1917 
svm_is_valid_cr0(struct kvm_vcpu * vcpu,unsigned long cr0)1918 static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1919 {
1920 	return true;
1921 }
1922 
svm_set_cr0(struct kvm_vcpu * vcpu,unsigned long cr0)1923 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1924 {
1925 	struct vcpu_svm *svm = to_svm(vcpu);
1926 	u64 hcr0 = cr0;
1927 	bool old_paging = is_paging(vcpu);
1928 
1929 #ifdef CONFIG_X86_64
1930 	if (vcpu->arch.efer & EFER_LME) {
1931 		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1932 			vcpu->arch.efer |= EFER_LMA;
1933 			if (!vcpu->arch.guest_state_protected)
1934 				svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1935 		}
1936 
1937 		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1938 			vcpu->arch.efer &= ~EFER_LMA;
1939 			if (!vcpu->arch.guest_state_protected)
1940 				svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1941 		}
1942 	}
1943 #endif
1944 	vcpu->arch.cr0 = cr0;
1945 
1946 	if (!npt_enabled) {
1947 		hcr0 |= X86_CR0_PG | X86_CR0_WP;
1948 		if (old_paging != is_paging(vcpu))
1949 			svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
1950 	}
1951 
1952 	/*
1953 	 * re-enable caching here because the QEMU bios
1954 	 * does not do it - this results in some delay at
1955 	 * reboot
1956 	 */
1957 	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1958 		hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1959 
1960 	svm->vmcb->save.cr0 = hcr0;
1961 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1962 
1963 	/*
1964 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
1965 	 * tracking is done using the CR write traps.
1966 	 */
1967 	if (sev_es_guest(vcpu->kvm))
1968 		return;
1969 
1970 	if (hcr0 == cr0) {
1971 		/* Selective CR0 write remains on.  */
1972 		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1973 		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1974 	} else {
1975 		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1976 		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1977 	}
1978 }
1979 
svm_is_valid_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1980 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1981 {
1982 	return true;
1983 }
1984 
svm_set_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1985 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1986 {
1987 	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1988 	unsigned long old_cr4 = vcpu->arch.cr4;
1989 
1990 	vcpu->arch.cr4 = cr4;
1991 	if (!npt_enabled) {
1992 		cr4 |= X86_CR4_PAE;
1993 
1994 		if (!is_paging(vcpu))
1995 			cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
1996 	}
1997 	cr4 |= host_cr4_mce;
1998 	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1999 	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2000 
2001 	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
2002 		vcpu->arch.cpuid_dynamic_bits_dirty = true;
2003 }
2004 
svm_set_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)2005 static void svm_set_segment(struct kvm_vcpu *vcpu,
2006 			    struct kvm_segment *var, int seg)
2007 {
2008 	struct vcpu_svm *svm = to_svm(vcpu);
2009 	struct vmcb_seg *s = svm_seg(vcpu, seg);
2010 
2011 	s->base = var->base;
2012 	s->limit = var->limit;
2013 	s->selector = var->selector;
2014 	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2015 	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2016 	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2017 	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2018 	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2019 	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2020 	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2021 	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2022 
2023 	/*
2024 	 * This is always accurate, except if SYSRET returned to a segment
2025 	 * with SS.DPL != 3.  Intel does not have this quirk, and always
2026 	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2027 	 * would entail passing the CPL to userspace and back.
2028 	 */
2029 	if (seg == VCPU_SREG_SS)
2030 		/* This is symmetric with svm_get_segment() */
2031 		svm->vmcb->save.cpl = (var->dpl & 3);
2032 
2033 	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
2034 }
2035 
svm_update_exception_bitmap(struct kvm_vcpu * vcpu)2036 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
2037 {
2038 	struct vcpu_svm *svm = to_svm(vcpu);
2039 
2040 	clr_exception_intercept(svm, BP_VECTOR);
2041 
2042 	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2043 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2044 			set_exception_intercept(svm, BP_VECTOR);
2045 	}
2046 }
2047 
new_asid(struct vcpu_svm * svm,struct svm_cpu_data * sd)2048 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2049 {
2050 	if (sd->next_asid > sd->max_asid) {
2051 		++sd->asid_generation;
2052 		sd->next_asid = sd->min_asid;
2053 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2054 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
2055 	}
2056 
2057 	svm->current_vmcb->asid_generation = sd->asid_generation;
2058 	svm->asid = sd->next_asid++;
2059 }
2060 
svm_set_dr6(struct kvm_vcpu * vcpu,unsigned long value)2061 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2062 {
2063 	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
2064 
2065 	if (vcpu->arch.guest_state_protected)
2066 		return;
2067 
2068 	if (unlikely(value != vmcb->save.dr6)) {
2069 		vmcb->save.dr6 = value;
2070 		vmcb_mark_dirty(vmcb, VMCB_DR);
2071 	}
2072 }
2073 
svm_sync_dirty_debug_regs(struct kvm_vcpu * vcpu)2074 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2075 {
2076 	struct vcpu_svm *svm = to_svm(vcpu);
2077 
2078 	if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm)))
2079 		return;
2080 
2081 	get_debugreg(vcpu->arch.db[0], 0);
2082 	get_debugreg(vcpu->arch.db[1], 1);
2083 	get_debugreg(vcpu->arch.db[2], 2);
2084 	get_debugreg(vcpu->arch.db[3], 3);
2085 	/*
2086 	 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
2087 	 * because db_interception might need it.  We can do it before vmentry.
2088 	 */
2089 	vcpu->arch.dr6 = svm->vmcb->save.dr6;
2090 	vcpu->arch.dr7 = svm->vmcb->save.dr7;
2091 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2092 	set_dr_intercepts(svm);
2093 }
2094 
svm_set_dr7(struct kvm_vcpu * vcpu,unsigned long value)2095 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2096 {
2097 	struct vcpu_svm *svm = to_svm(vcpu);
2098 
2099 	if (vcpu->arch.guest_state_protected)
2100 		return;
2101 
2102 	svm->vmcb->save.dr7 = value;
2103 	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
2104 }
2105 
pf_interception(struct kvm_vcpu * vcpu)2106 static int pf_interception(struct kvm_vcpu *vcpu)
2107 {
2108 	struct vcpu_svm *svm = to_svm(vcpu);
2109 
2110 	u64 fault_address = svm->vmcb->control.exit_info_2;
2111 	u64 error_code = svm->vmcb->control.exit_info_1;
2112 
2113 	return kvm_handle_page_fault(vcpu, error_code, fault_address,
2114 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2115 			svm->vmcb->control.insn_bytes : NULL,
2116 			svm->vmcb->control.insn_len);
2117 }
2118 
npf_interception(struct kvm_vcpu * vcpu)2119 static int npf_interception(struct kvm_vcpu *vcpu)
2120 {
2121 	struct vcpu_svm *svm = to_svm(vcpu);
2122 	int rc;
2123 
2124 	u64 fault_address = svm->vmcb->control.exit_info_2;
2125 	u64 error_code = svm->vmcb->control.exit_info_1;
2126 
2127 	/*
2128 	 * WARN if hardware generates a fault with an error code that collides
2129 	 * with KVM-defined sythentic flags.  Clear the flags and continue on,
2130 	 * i.e. don't terminate the VM, as KVM can't possibly be relying on a
2131 	 * flag that KVM doesn't know about.
2132 	 */
2133 	if (WARN_ON_ONCE(error_code & PFERR_SYNTHETIC_MASK))
2134 		error_code &= ~PFERR_SYNTHETIC_MASK;
2135 
2136 	if (sev_snp_guest(vcpu->kvm) && (error_code & PFERR_GUEST_ENC_MASK))
2137 		error_code |= PFERR_PRIVATE_ACCESS;
2138 
2139 	trace_kvm_page_fault(vcpu, fault_address, error_code);
2140 	rc = kvm_mmu_page_fault(vcpu, fault_address, error_code,
2141 				static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2142 				svm->vmcb->control.insn_bytes : NULL,
2143 				svm->vmcb->control.insn_len);
2144 
2145 	if (rc > 0 && error_code & PFERR_GUEST_RMP_MASK)
2146 		sev_handle_rmp_fault(vcpu, fault_address, error_code);
2147 
2148 	return rc;
2149 }
2150 
db_interception(struct kvm_vcpu * vcpu)2151 static int db_interception(struct kvm_vcpu *vcpu)
2152 {
2153 	struct kvm_run *kvm_run = vcpu->run;
2154 	struct vcpu_svm *svm = to_svm(vcpu);
2155 
2156 	if (!(vcpu->guest_debug &
2157 	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2158 		!svm->nmi_singlestep) {
2159 		u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
2160 		kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
2161 		return 1;
2162 	}
2163 
2164 	if (svm->nmi_singlestep) {
2165 		disable_nmi_singlestep(svm);
2166 		/* Make sure we check for pending NMIs upon entry */
2167 		kvm_make_request(KVM_REQ_EVENT, vcpu);
2168 	}
2169 
2170 	if (vcpu->guest_debug &
2171 	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2172 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
2173 		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
2174 		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
2175 		kvm_run->debug.arch.pc =
2176 			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2177 		kvm_run->debug.arch.exception = DB_VECTOR;
2178 		return 0;
2179 	}
2180 
2181 	return 1;
2182 }
2183 
bp_interception(struct kvm_vcpu * vcpu)2184 static int bp_interception(struct kvm_vcpu *vcpu)
2185 {
2186 	struct vcpu_svm *svm = to_svm(vcpu);
2187 	struct kvm_run *kvm_run = vcpu->run;
2188 
2189 	kvm_run->exit_reason = KVM_EXIT_DEBUG;
2190 	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2191 	kvm_run->debug.arch.exception = BP_VECTOR;
2192 	return 0;
2193 }
2194 
ud_interception(struct kvm_vcpu * vcpu)2195 static int ud_interception(struct kvm_vcpu *vcpu)
2196 {
2197 	return handle_ud(vcpu);
2198 }
2199 
ac_interception(struct kvm_vcpu * vcpu)2200 static int ac_interception(struct kvm_vcpu *vcpu)
2201 {
2202 	kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
2203 	return 1;
2204 }
2205 
is_erratum_383(void)2206 static bool is_erratum_383(void)
2207 {
2208 	int err, i;
2209 	u64 value;
2210 
2211 	if (!erratum_383_found)
2212 		return false;
2213 
2214 	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2215 	if (err)
2216 		return false;
2217 
2218 	/* Bit 62 may or may not be set for this mce */
2219 	value &= ~(1ULL << 62);
2220 
2221 	if (value != 0xb600000000010015ULL)
2222 		return false;
2223 
2224 	/* Clear MCi_STATUS registers */
2225 	for (i = 0; i < 6; ++i)
2226 		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2227 
2228 	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2229 	if (!err) {
2230 		u32 low, high;
2231 
2232 		value &= ~(1ULL << 2);
2233 		low    = lower_32_bits(value);
2234 		high   = upper_32_bits(value);
2235 
2236 		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2237 	}
2238 
2239 	/* Flush tlb to evict multi-match entries */
2240 	__flush_tlb_all();
2241 
2242 	return true;
2243 }
2244 
svm_handle_mce(struct kvm_vcpu * vcpu)2245 static void svm_handle_mce(struct kvm_vcpu *vcpu)
2246 {
2247 	if (is_erratum_383()) {
2248 		/*
2249 		 * Erratum 383 triggered. Guest state is corrupt so kill the
2250 		 * guest.
2251 		 */
2252 		pr_err("Guest triggered AMD Erratum 383\n");
2253 
2254 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2255 
2256 		return;
2257 	}
2258 
2259 	/*
2260 	 * On an #MC intercept the MCE handler is not called automatically in
2261 	 * the host. So do it by hand here.
2262 	 */
2263 	kvm_machine_check();
2264 }
2265 
mc_interception(struct kvm_vcpu * vcpu)2266 static int mc_interception(struct kvm_vcpu *vcpu)
2267 {
2268 	return 1;
2269 }
2270 
shutdown_interception(struct kvm_vcpu * vcpu)2271 static int shutdown_interception(struct kvm_vcpu *vcpu)
2272 {
2273 	struct kvm_run *kvm_run = vcpu->run;
2274 	struct vcpu_svm *svm = to_svm(vcpu);
2275 
2276 
2277 	/*
2278 	 * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2279 	 * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2280 	 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2281 	 * userspace.  At a platform view, INIT is acceptable behavior as
2282 	 * there exist bare metal platforms that automatically INIT the CPU
2283 	 * in response to shutdown.
2284 	 *
2285 	 * The VM save area for SEV-ES guests has already been encrypted so it
2286 	 * cannot be reinitialized, i.e. synthesizing INIT is futile.
2287 	 */
2288 	if (!sev_es_guest(vcpu->kvm)) {
2289 		clear_page(svm->vmcb);
2290 #ifdef CONFIG_KVM_SMM
2291 		if (is_smm(vcpu))
2292 			kvm_smm_changed(vcpu, false);
2293 #endif
2294 		kvm_vcpu_reset(vcpu, true);
2295 	}
2296 
2297 	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2298 	return 0;
2299 }
2300 
io_interception(struct kvm_vcpu * vcpu)2301 static int io_interception(struct kvm_vcpu *vcpu)
2302 {
2303 	struct vcpu_svm *svm = to_svm(vcpu);
2304 	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2305 	int size, in, string;
2306 	unsigned port;
2307 
2308 	++vcpu->stat.io_exits;
2309 	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2310 	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2311 	port = io_info >> 16;
2312 	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2313 
2314 	if (string) {
2315 		if (sev_es_guest(vcpu->kvm))
2316 			return sev_es_string_io(svm, size, port, in);
2317 		else
2318 			return kvm_emulate_instruction(vcpu, 0);
2319 	}
2320 
2321 	svm->next_rip = svm->vmcb->control.exit_info_2;
2322 
2323 	return kvm_fast_pio(vcpu, size, port, in);
2324 }
2325 
nmi_interception(struct kvm_vcpu * vcpu)2326 static int nmi_interception(struct kvm_vcpu *vcpu)
2327 {
2328 	return 1;
2329 }
2330 
smi_interception(struct kvm_vcpu * vcpu)2331 static int smi_interception(struct kvm_vcpu *vcpu)
2332 {
2333 	return 1;
2334 }
2335 
intr_interception(struct kvm_vcpu * vcpu)2336 static int intr_interception(struct kvm_vcpu *vcpu)
2337 {
2338 	++vcpu->stat.irq_exits;
2339 	return 1;
2340 }
2341 
vmload_vmsave_interception(struct kvm_vcpu * vcpu,bool vmload)2342 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2343 {
2344 	struct vcpu_svm *svm = to_svm(vcpu);
2345 	struct vmcb *vmcb12;
2346 	struct kvm_host_map map;
2347 	int ret;
2348 
2349 	if (nested_svm_check_permissions(vcpu))
2350 		return 1;
2351 
2352 	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2353 	if (ret) {
2354 		if (ret == -EINVAL)
2355 			kvm_inject_gp(vcpu, 0);
2356 		return 1;
2357 	}
2358 
2359 	vmcb12 = map.hva;
2360 
2361 	ret = kvm_skip_emulated_instruction(vcpu);
2362 
2363 	if (vmload) {
2364 		svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2365 		svm->sysenter_eip_hi = 0;
2366 		svm->sysenter_esp_hi = 0;
2367 	} else {
2368 		svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2369 	}
2370 
2371 	kvm_vcpu_unmap(vcpu, &map);
2372 
2373 	return ret;
2374 }
2375 
vmload_interception(struct kvm_vcpu * vcpu)2376 static int vmload_interception(struct kvm_vcpu *vcpu)
2377 {
2378 	return vmload_vmsave_interception(vcpu, true);
2379 }
2380 
vmsave_interception(struct kvm_vcpu * vcpu)2381 static int vmsave_interception(struct kvm_vcpu *vcpu)
2382 {
2383 	return vmload_vmsave_interception(vcpu, false);
2384 }
2385 
vmrun_interception(struct kvm_vcpu * vcpu)2386 static int vmrun_interception(struct kvm_vcpu *vcpu)
2387 {
2388 	if (nested_svm_check_permissions(vcpu))
2389 		return 1;
2390 
2391 	return nested_svm_vmrun(vcpu);
2392 }
2393 
2394 enum {
2395 	NONE_SVM_INSTR,
2396 	SVM_INSTR_VMRUN,
2397 	SVM_INSTR_VMLOAD,
2398 	SVM_INSTR_VMSAVE,
2399 };
2400 
2401 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
svm_instr_opcode(struct kvm_vcpu * vcpu)2402 static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2403 {
2404 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2405 
2406 	if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2407 		return NONE_SVM_INSTR;
2408 
2409 	switch (ctxt->modrm) {
2410 	case 0xd8: /* VMRUN */
2411 		return SVM_INSTR_VMRUN;
2412 	case 0xda: /* VMLOAD */
2413 		return SVM_INSTR_VMLOAD;
2414 	case 0xdb: /* VMSAVE */
2415 		return SVM_INSTR_VMSAVE;
2416 	default:
2417 		break;
2418 	}
2419 
2420 	return NONE_SVM_INSTR;
2421 }
2422 
emulate_svm_instr(struct kvm_vcpu * vcpu,int opcode)2423 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2424 {
2425 	const int guest_mode_exit_codes[] = {
2426 		[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2427 		[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2428 		[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2429 	};
2430 	int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2431 		[SVM_INSTR_VMRUN] = vmrun_interception,
2432 		[SVM_INSTR_VMLOAD] = vmload_interception,
2433 		[SVM_INSTR_VMSAVE] = vmsave_interception,
2434 	};
2435 	struct vcpu_svm *svm = to_svm(vcpu);
2436 	int ret;
2437 
2438 	if (is_guest_mode(vcpu)) {
2439 		/* Returns '1' or -errno on failure, '0' on success. */
2440 		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2441 		if (ret)
2442 			return ret;
2443 		return 1;
2444 	}
2445 	return svm_instr_handlers[opcode](vcpu);
2446 }
2447 
2448 /*
2449  * #GP handling code. Note that #GP can be triggered under the following two
2450  * cases:
2451  *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2452  *      some AMD CPUs when EAX of these instructions are in the reserved memory
2453  *      regions (e.g. SMM memory on host).
2454  *   2) VMware backdoor
2455  */
gp_interception(struct kvm_vcpu * vcpu)2456 static int gp_interception(struct kvm_vcpu *vcpu)
2457 {
2458 	struct vcpu_svm *svm = to_svm(vcpu);
2459 	u32 error_code = svm->vmcb->control.exit_info_1;
2460 	int opcode;
2461 
2462 	/* Both #GP cases have zero error_code */
2463 	if (error_code)
2464 		goto reinject;
2465 
2466 	/* Decode the instruction for usage later */
2467 	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2468 		goto reinject;
2469 
2470 	opcode = svm_instr_opcode(vcpu);
2471 
2472 	if (opcode == NONE_SVM_INSTR) {
2473 		if (!enable_vmware_backdoor)
2474 			goto reinject;
2475 
2476 		/*
2477 		 * VMware backdoor emulation on #GP interception only handles
2478 		 * IN{S}, OUT{S}, and RDPMC.
2479 		 */
2480 		if (!is_guest_mode(vcpu))
2481 			return kvm_emulate_instruction(vcpu,
2482 				EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2483 	} else {
2484 		/* All SVM instructions expect page aligned RAX */
2485 		if (svm->vmcb->save.rax & ~PAGE_MASK)
2486 			goto reinject;
2487 
2488 		return emulate_svm_instr(vcpu, opcode);
2489 	}
2490 
2491 reinject:
2492 	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2493 	return 1;
2494 }
2495 
svm_set_gif(struct vcpu_svm * svm,bool value)2496 void svm_set_gif(struct vcpu_svm *svm, bool value)
2497 {
2498 	if (value) {
2499 		/*
2500 		 * If VGIF is enabled, the STGI intercept is only added to
2501 		 * detect the opening of the SMI/NMI window; remove it now.
2502 		 * Likewise, clear the VINTR intercept, we will set it
2503 		 * again while processing KVM_REQ_EVENT if needed.
2504 		 */
2505 		if (vgif)
2506 			svm_clr_intercept(svm, INTERCEPT_STGI);
2507 		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2508 			svm_clear_vintr(svm);
2509 
2510 		enable_gif(svm);
2511 		if (svm->vcpu.arch.smi_pending ||
2512 		    svm->vcpu.arch.nmi_pending ||
2513 		    kvm_cpu_has_injectable_intr(&svm->vcpu) ||
2514 		    kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
2515 			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2516 	} else {
2517 		disable_gif(svm);
2518 
2519 		/*
2520 		 * After a CLGI no interrupts should come.  But if vGIF is
2521 		 * in use, we still rely on the VINTR intercept (rather than
2522 		 * STGI) to detect an open interrupt window.
2523 		*/
2524 		if (!vgif)
2525 			svm_clear_vintr(svm);
2526 	}
2527 }
2528 
stgi_interception(struct kvm_vcpu * vcpu)2529 static int stgi_interception(struct kvm_vcpu *vcpu)
2530 {
2531 	int ret;
2532 
2533 	if (nested_svm_check_permissions(vcpu))
2534 		return 1;
2535 
2536 	ret = kvm_skip_emulated_instruction(vcpu);
2537 	svm_set_gif(to_svm(vcpu), true);
2538 	return ret;
2539 }
2540 
clgi_interception(struct kvm_vcpu * vcpu)2541 static int clgi_interception(struct kvm_vcpu *vcpu)
2542 {
2543 	int ret;
2544 
2545 	if (nested_svm_check_permissions(vcpu))
2546 		return 1;
2547 
2548 	ret = kvm_skip_emulated_instruction(vcpu);
2549 	svm_set_gif(to_svm(vcpu), false);
2550 	return ret;
2551 }
2552 
invlpga_interception(struct kvm_vcpu * vcpu)2553 static int invlpga_interception(struct kvm_vcpu *vcpu)
2554 {
2555 	gva_t gva = kvm_rax_read(vcpu);
2556 	u32 asid = kvm_rcx_read(vcpu);
2557 
2558 	/* FIXME: Handle an address size prefix. */
2559 	if (!is_long_mode(vcpu))
2560 		gva = (u32)gva;
2561 
2562 	trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2563 
2564 	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2565 	kvm_mmu_invlpg(vcpu, gva);
2566 
2567 	return kvm_skip_emulated_instruction(vcpu);
2568 }
2569 
skinit_interception(struct kvm_vcpu * vcpu)2570 static int skinit_interception(struct kvm_vcpu *vcpu)
2571 {
2572 	trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2573 
2574 	kvm_queue_exception(vcpu, UD_VECTOR);
2575 	return 1;
2576 }
2577 
task_switch_interception(struct kvm_vcpu * vcpu)2578 static int task_switch_interception(struct kvm_vcpu *vcpu)
2579 {
2580 	struct vcpu_svm *svm = to_svm(vcpu);
2581 	u16 tss_selector;
2582 	int reason;
2583 	int int_type = svm->vmcb->control.exit_int_info &
2584 		SVM_EXITINTINFO_TYPE_MASK;
2585 	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2586 	uint32_t type =
2587 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2588 	uint32_t idt_v =
2589 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2590 	bool has_error_code = false;
2591 	u32 error_code = 0;
2592 
2593 	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2594 
2595 	if (svm->vmcb->control.exit_info_2 &
2596 	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2597 		reason = TASK_SWITCH_IRET;
2598 	else if (svm->vmcb->control.exit_info_2 &
2599 		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2600 		reason = TASK_SWITCH_JMP;
2601 	else if (idt_v)
2602 		reason = TASK_SWITCH_GATE;
2603 	else
2604 		reason = TASK_SWITCH_CALL;
2605 
2606 	if (reason == TASK_SWITCH_GATE) {
2607 		switch (type) {
2608 		case SVM_EXITINTINFO_TYPE_NMI:
2609 			vcpu->arch.nmi_injected = false;
2610 			break;
2611 		case SVM_EXITINTINFO_TYPE_EXEPT:
2612 			if (svm->vmcb->control.exit_info_2 &
2613 			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2614 				has_error_code = true;
2615 				error_code =
2616 					(u32)svm->vmcb->control.exit_info_2;
2617 			}
2618 			kvm_clear_exception_queue(vcpu);
2619 			break;
2620 		case SVM_EXITINTINFO_TYPE_INTR:
2621 		case SVM_EXITINTINFO_TYPE_SOFT:
2622 			kvm_clear_interrupt_queue(vcpu);
2623 			break;
2624 		default:
2625 			break;
2626 		}
2627 	}
2628 
2629 	if (reason != TASK_SWITCH_GATE ||
2630 	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2631 	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2632 	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2633 		if (!svm_skip_emulated_instruction(vcpu))
2634 			return 0;
2635 	}
2636 
2637 	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2638 		int_vec = -1;
2639 
2640 	return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2641 			       has_error_code, error_code);
2642 }
2643 
svm_clr_iret_intercept(struct vcpu_svm * svm)2644 static void svm_clr_iret_intercept(struct vcpu_svm *svm)
2645 {
2646 	if (!sev_es_guest(svm->vcpu.kvm))
2647 		svm_clr_intercept(svm, INTERCEPT_IRET);
2648 }
2649 
svm_set_iret_intercept(struct vcpu_svm * svm)2650 static void svm_set_iret_intercept(struct vcpu_svm *svm)
2651 {
2652 	if (!sev_es_guest(svm->vcpu.kvm))
2653 		svm_set_intercept(svm, INTERCEPT_IRET);
2654 }
2655 
iret_interception(struct kvm_vcpu * vcpu)2656 static int iret_interception(struct kvm_vcpu *vcpu)
2657 {
2658 	struct vcpu_svm *svm = to_svm(vcpu);
2659 
2660 	WARN_ON_ONCE(sev_es_guest(vcpu->kvm));
2661 
2662 	++vcpu->stat.nmi_window_exits;
2663 	svm->awaiting_iret_completion = true;
2664 
2665 	svm_clr_iret_intercept(svm);
2666 	svm->nmi_iret_rip = kvm_rip_read(vcpu);
2667 
2668 	kvm_make_request(KVM_REQ_EVENT, vcpu);
2669 	return 1;
2670 }
2671 
invlpg_interception(struct kvm_vcpu * vcpu)2672 static int invlpg_interception(struct kvm_vcpu *vcpu)
2673 {
2674 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2675 		return kvm_emulate_instruction(vcpu, 0);
2676 
2677 	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2678 	return kvm_skip_emulated_instruction(vcpu);
2679 }
2680 
emulate_on_interception(struct kvm_vcpu * vcpu)2681 static int emulate_on_interception(struct kvm_vcpu *vcpu)
2682 {
2683 	return kvm_emulate_instruction(vcpu, 0);
2684 }
2685 
rsm_interception(struct kvm_vcpu * vcpu)2686 static int rsm_interception(struct kvm_vcpu *vcpu)
2687 {
2688 	return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2689 }
2690 
check_selective_cr0_intercepted(struct kvm_vcpu * vcpu,unsigned long val)2691 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2692 					    unsigned long val)
2693 {
2694 	struct vcpu_svm *svm = to_svm(vcpu);
2695 	unsigned long cr0 = vcpu->arch.cr0;
2696 	bool ret = false;
2697 
2698 	if (!is_guest_mode(vcpu) ||
2699 	    (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2700 		return false;
2701 
2702 	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2703 	val &= ~SVM_CR0_SELECTIVE_MASK;
2704 
2705 	if (cr0 ^ val) {
2706 		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2707 		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2708 	}
2709 
2710 	return ret;
2711 }
2712 
2713 #define CR_VALID (1ULL << 63)
2714 
cr_interception(struct kvm_vcpu * vcpu)2715 static int cr_interception(struct kvm_vcpu *vcpu)
2716 {
2717 	struct vcpu_svm *svm = to_svm(vcpu);
2718 	int reg, cr;
2719 	unsigned long val;
2720 	int err;
2721 
2722 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2723 		return emulate_on_interception(vcpu);
2724 
2725 	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2726 		return emulate_on_interception(vcpu);
2727 
2728 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2729 	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2730 		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2731 	else
2732 		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2733 
2734 	err = 0;
2735 	if (cr >= 16) { /* mov to cr */
2736 		cr -= 16;
2737 		val = kvm_register_read(vcpu, reg);
2738 		trace_kvm_cr_write(cr, val);
2739 		switch (cr) {
2740 		case 0:
2741 			if (!check_selective_cr0_intercepted(vcpu, val))
2742 				err = kvm_set_cr0(vcpu, val);
2743 			else
2744 				return 1;
2745 
2746 			break;
2747 		case 3:
2748 			err = kvm_set_cr3(vcpu, val);
2749 			break;
2750 		case 4:
2751 			err = kvm_set_cr4(vcpu, val);
2752 			break;
2753 		case 8:
2754 			err = kvm_set_cr8(vcpu, val);
2755 			break;
2756 		default:
2757 			WARN(1, "unhandled write to CR%d", cr);
2758 			kvm_queue_exception(vcpu, UD_VECTOR);
2759 			return 1;
2760 		}
2761 	} else { /* mov from cr */
2762 		switch (cr) {
2763 		case 0:
2764 			val = kvm_read_cr0(vcpu);
2765 			break;
2766 		case 2:
2767 			val = vcpu->arch.cr2;
2768 			break;
2769 		case 3:
2770 			val = kvm_read_cr3(vcpu);
2771 			break;
2772 		case 4:
2773 			val = kvm_read_cr4(vcpu);
2774 			break;
2775 		case 8:
2776 			val = kvm_get_cr8(vcpu);
2777 			break;
2778 		default:
2779 			WARN(1, "unhandled read from CR%d", cr);
2780 			kvm_queue_exception(vcpu, UD_VECTOR);
2781 			return 1;
2782 		}
2783 		kvm_register_write(vcpu, reg, val);
2784 		trace_kvm_cr_read(cr, val);
2785 	}
2786 	return kvm_complete_insn_gp(vcpu, err);
2787 }
2788 
cr_trap(struct kvm_vcpu * vcpu)2789 static int cr_trap(struct kvm_vcpu *vcpu)
2790 {
2791 	struct vcpu_svm *svm = to_svm(vcpu);
2792 	unsigned long old_value, new_value;
2793 	unsigned int cr;
2794 	int ret = 0;
2795 
2796 	new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2797 
2798 	cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2799 	switch (cr) {
2800 	case 0:
2801 		old_value = kvm_read_cr0(vcpu);
2802 		svm_set_cr0(vcpu, new_value);
2803 
2804 		kvm_post_set_cr0(vcpu, old_value, new_value);
2805 		break;
2806 	case 4:
2807 		old_value = kvm_read_cr4(vcpu);
2808 		svm_set_cr4(vcpu, new_value);
2809 
2810 		kvm_post_set_cr4(vcpu, old_value, new_value);
2811 		break;
2812 	case 8:
2813 		ret = kvm_set_cr8(vcpu, new_value);
2814 		break;
2815 	default:
2816 		WARN(1, "unhandled CR%d write trap", cr);
2817 		kvm_queue_exception(vcpu, UD_VECTOR);
2818 		return 1;
2819 	}
2820 
2821 	return kvm_complete_insn_gp(vcpu, ret);
2822 }
2823 
dr_interception(struct kvm_vcpu * vcpu)2824 static int dr_interception(struct kvm_vcpu *vcpu)
2825 {
2826 	struct vcpu_svm *svm = to_svm(vcpu);
2827 	int reg, dr;
2828 	int err = 0;
2829 
2830 	/*
2831 	 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT
2832 	 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early.
2833 	 */
2834 	if (sev_es_guest(vcpu->kvm))
2835 		return 1;
2836 
2837 	if (vcpu->guest_debug == 0) {
2838 		/*
2839 		 * No more DR vmexits; force a reload of the debug registers
2840 		 * and reenter on this instruction.  The next vmexit will
2841 		 * retrieve the full state of the debug registers.
2842 		 */
2843 		clr_dr_intercepts(svm);
2844 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2845 		return 1;
2846 	}
2847 
2848 	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2849 		return emulate_on_interception(vcpu);
2850 
2851 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2852 	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2853 	if (dr >= 16) { /* mov to DRn  */
2854 		dr -= 16;
2855 		err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg));
2856 	} else {
2857 		kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr));
2858 	}
2859 
2860 	return kvm_complete_insn_gp(vcpu, err);
2861 }
2862 
cr8_write_interception(struct kvm_vcpu * vcpu)2863 static int cr8_write_interception(struct kvm_vcpu *vcpu)
2864 {
2865 	int r;
2866 
2867 	u8 cr8_prev = kvm_get_cr8(vcpu);
2868 	/* instruction emulation calls kvm_set_cr8() */
2869 	r = cr_interception(vcpu);
2870 	if (lapic_in_kernel(vcpu))
2871 		return r;
2872 	if (cr8_prev <= kvm_get_cr8(vcpu))
2873 		return r;
2874 	vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2875 	return 0;
2876 }
2877 
efer_trap(struct kvm_vcpu * vcpu)2878 static int efer_trap(struct kvm_vcpu *vcpu)
2879 {
2880 	struct msr_data msr_info;
2881 	int ret;
2882 
2883 	/*
2884 	 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2885 	 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2886 	 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2887 	 * the guest doesn't have X86_FEATURE_SVM.
2888 	 */
2889 	msr_info.host_initiated = false;
2890 	msr_info.index = MSR_EFER;
2891 	msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2892 	ret = kvm_set_msr_common(vcpu, &msr_info);
2893 
2894 	return kvm_complete_insn_gp(vcpu, ret);
2895 }
2896 
svm_get_feature_msr(u32 msr,u64 * data)2897 static int svm_get_feature_msr(u32 msr, u64 *data)
2898 {
2899 	*data = 0;
2900 
2901 	switch (msr) {
2902 	case MSR_AMD64_DE_CFG:
2903 		if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
2904 			*data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE;
2905 		break;
2906 	default:
2907 		return KVM_MSR_RET_UNSUPPORTED;
2908 	}
2909 
2910 	return 0;
2911 }
2912 
2913 static bool
sev_es_prevent_msr_access(struct kvm_vcpu * vcpu,struct msr_data * msr_info)2914 sev_es_prevent_msr_access(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2915 {
2916 	return sev_es_guest(vcpu->kvm) &&
2917 	       vcpu->arch.guest_state_protected &&
2918 	       svm_msrpm_offset(msr_info->index) != MSR_INVALID &&
2919 	       !msr_write_intercepted(vcpu, msr_info->index);
2920 }
2921 
svm_get_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)2922 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2923 {
2924 	struct vcpu_svm *svm = to_svm(vcpu);
2925 
2926 	if (sev_es_prevent_msr_access(vcpu, msr_info)) {
2927 		msr_info->data = 0;
2928 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
2929 	}
2930 
2931 	switch (msr_info->index) {
2932 	case MSR_AMD64_TSC_RATIO:
2933 		if (!msr_info->host_initiated &&
2934 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR))
2935 			return 1;
2936 		msr_info->data = svm->tsc_ratio_msr;
2937 		break;
2938 	case MSR_STAR:
2939 		msr_info->data = svm->vmcb01.ptr->save.star;
2940 		break;
2941 #ifdef CONFIG_X86_64
2942 	case MSR_LSTAR:
2943 		msr_info->data = svm->vmcb01.ptr->save.lstar;
2944 		break;
2945 	case MSR_CSTAR:
2946 		msr_info->data = svm->vmcb01.ptr->save.cstar;
2947 		break;
2948 	case MSR_GS_BASE:
2949 		msr_info->data = svm->vmcb01.ptr->save.gs.base;
2950 		break;
2951 	case MSR_FS_BASE:
2952 		msr_info->data = svm->vmcb01.ptr->save.fs.base;
2953 		break;
2954 	case MSR_KERNEL_GS_BASE:
2955 		msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2956 		break;
2957 	case MSR_SYSCALL_MASK:
2958 		msr_info->data = svm->vmcb01.ptr->save.sfmask;
2959 		break;
2960 #endif
2961 	case MSR_IA32_SYSENTER_CS:
2962 		msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2963 		break;
2964 	case MSR_IA32_SYSENTER_EIP:
2965 		msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2966 		if (guest_cpuid_is_intel_compatible(vcpu))
2967 			msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2968 		break;
2969 	case MSR_IA32_SYSENTER_ESP:
2970 		msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2971 		if (guest_cpuid_is_intel_compatible(vcpu))
2972 			msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2973 		break;
2974 	case MSR_TSC_AUX:
2975 		msr_info->data = svm->tsc_aux;
2976 		break;
2977 	case MSR_IA32_DEBUGCTLMSR:
2978 		msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl;
2979 		break;
2980 	case MSR_IA32_LASTBRANCHFROMIP:
2981 		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from;
2982 		break;
2983 	case MSR_IA32_LASTBRANCHTOIP:
2984 		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to;
2985 		break;
2986 	case MSR_IA32_LASTINTFROMIP:
2987 		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from;
2988 		break;
2989 	case MSR_IA32_LASTINTTOIP:
2990 		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to;
2991 		break;
2992 	case MSR_VM_HSAVE_PA:
2993 		msr_info->data = svm->nested.hsave_msr;
2994 		break;
2995 	case MSR_VM_CR:
2996 		msr_info->data = svm->nested.vm_cr_msr;
2997 		break;
2998 	case MSR_IA32_SPEC_CTRL:
2999 		if (!msr_info->host_initiated &&
3000 		    !guest_has_spec_ctrl_msr(vcpu))
3001 			return 1;
3002 
3003 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3004 			msr_info->data = svm->vmcb->save.spec_ctrl;
3005 		else
3006 			msr_info->data = svm->spec_ctrl;
3007 		break;
3008 	case MSR_AMD64_VIRT_SPEC_CTRL:
3009 		if (!msr_info->host_initiated &&
3010 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_VIRT_SSBD))
3011 			return 1;
3012 
3013 		msr_info->data = svm->virt_spec_ctrl;
3014 		break;
3015 	case MSR_F15H_IC_CFG: {
3016 
3017 		int family, model;
3018 
3019 		family = guest_cpuid_family(vcpu);
3020 		model  = guest_cpuid_model(vcpu);
3021 
3022 		if (family < 0 || model < 0)
3023 			return kvm_get_msr_common(vcpu, msr_info);
3024 
3025 		msr_info->data = 0;
3026 
3027 		if (family == 0x15 &&
3028 		    (model >= 0x2 && model < 0x20))
3029 			msr_info->data = 0x1E;
3030 		}
3031 		break;
3032 	case MSR_AMD64_DE_CFG:
3033 		msr_info->data = svm->msr_decfg;
3034 		break;
3035 	default:
3036 		return kvm_get_msr_common(vcpu, msr_info);
3037 	}
3038 	return 0;
3039 }
3040 
svm_complete_emulated_msr(struct kvm_vcpu * vcpu,int err)3041 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
3042 {
3043 	struct vcpu_svm *svm = to_svm(vcpu);
3044 	if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
3045 		return kvm_complete_insn_gp(vcpu, err);
3046 
3047 	svm_vmgexit_inject_exception(svm, X86_TRAP_GP);
3048 	return 1;
3049 }
3050 
svm_set_vm_cr(struct kvm_vcpu * vcpu,u64 data)3051 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
3052 {
3053 	struct vcpu_svm *svm = to_svm(vcpu);
3054 	int svm_dis, chg_mask;
3055 
3056 	if (data & ~SVM_VM_CR_VALID_MASK)
3057 		return 1;
3058 
3059 	chg_mask = SVM_VM_CR_VALID_MASK;
3060 
3061 	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
3062 		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3063 
3064 	svm->nested.vm_cr_msr &= ~chg_mask;
3065 	svm->nested.vm_cr_msr |= (data & chg_mask);
3066 
3067 	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3068 
3069 	/* check for svm_disable while efer.svme is set */
3070 	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3071 		return 1;
3072 
3073 	return 0;
3074 }
3075 
svm_set_msr(struct kvm_vcpu * vcpu,struct msr_data * msr)3076 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3077 {
3078 	struct vcpu_svm *svm = to_svm(vcpu);
3079 	int ret = 0;
3080 
3081 	u32 ecx = msr->index;
3082 	u64 data = msr->data;
3083 
3084 	if (sev_es_prevent_msr_access(vcpu, msr))
3085 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
3086 
3087 	switch (ecx) {
3088 	case MSR_AMD64_TSC_RATIO:
3089 
3090 		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR)) {
3091 
3092 			if (!msr->host_initiated)
3093 				return 1;
3094 			/*
3095 			 * In case TSC scaling is not enabled, always
3096 			 * leave this MSR at the default value.
3097 			 *
3098 			 * Due to bug in qemu 6.2.0, it would try to set
3099 			 * this msr to 0 if tsc scaling is not enabled.
3100 			 * Ignore this value as well.
3101 			 */
3102 			if (data != 0 && data != svm->tsc_ratio_msr)
3103 				return 1;
3104 			break;
3105 		}
3106 
3107 		if (data & SVM_TSC_RATIO_RSVD)
3108 			return 1;
3109 
3110 		svm->tsc_ratio_msr = data;
3111 
3112 		if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) &&
3113 		    is_guest_mode(vcpu))
3114 			nested_svm_update_tsc_ratio_msr(vcpu);
3115 
3116 		break;
3117 	case MSR_IA32_CR_PAT:
3118 		ret = kvm_set_msr_common(vcpu, msr);
3119 		if (ret)
3120 			break;
3121 
3122 		svm->vmcb01.ptr->save.g_pat = data;
3123 		if (is_guest_mode(vcpu))
3124 			nested_vmcb02_compute_g_pat(svm);
3125 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3126 		break;
3127 	case MSR_IA32_SPEC_CTRL:
3128 		if (!msr->host_initiated &&
3129 		    !guest_has_spec_ctrl_msr(vcpu))
3130 			return 1;
3131 
3132 		if (kvm_spec_ctrl_test_value(data))
3133 			return 1;
3134 
3135 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3136 			svm->vmcb->save.spec_ctrl = data;
3137 		else
3138 			svm->spec_ctrl = data;
3139 		if (!data)
3140 			break;
3141 
3142 		/*
3143 		 * For non-nested:
3144 		 * When it's written (to non-zero) for the first time, pass
3145 		 * it through.
3146 		 *
3147 		 * For nested:
3148 		 * The handling of the MSR bitmap for L2 guests is done in
3149 		 * nested_svm_vmrun_msrpm.
3150 		 * We update the L1 MSR bit as well since it will end up
3151 		 * touching the MSR anyway now.
3152 		 */
3153 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
3154 		break;
3155 	case MSR_AMD64_VIRT_SPEC_CTRL:
3156 		if (!msr->host_initiated &&
3157 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_VIRT_SSBD))
3158 			return 1;
3159 
3160 		if (data & ~SPEC_CTRL_SSBD)
3161 			return 1;
3162 
3163 		svm->virt_spec_ctrl = data;
3164 		break;
3165 	case MSR_STAR:
3166 		svm->vmcb01.ptr->save.star = data;
3167 		break;
3168 #ifdef CONFIG_X86_64
3169 	case MSR_LSTAR:
3170 		svm->vmcb01.ptr->save.lstar = data;
3171 		break;
3172 	case MSR_CSTAR:
3173 		svm->vmcb01.ptr->save.cstar = data;
3174 		break;
3175 	case MSR_GS_BASE:
3176 		svm->vmcb01.ptr->save.gs.base = data;
3177 		break;
3178 	case MSR_FS_BASE:
3179 		svm->vmcb01.ptr->save.fs.base = data;
3180 		break;
3181 	case MSR_KERNEL_GS_BASE:
3182 		svm->vmcb01.ptr->save.kernel_gs_base = data;
3183 		break;
3184 	case MSR_SYSCALL_MASK:
3185 		svm->vmcb01.ptr->save.sfmask = data;
3186 		break;
3187 #endif
3188 	case MSR_IA32_SYSENTER_CS:
3189 		svm->vmcb01.ptr->save.sysenter_cs = data;
3190 		break;
3191 	case MSR_IA32_SYSENTER_EIP:
3192 		svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
3193 		/*
3194 		 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
3195 		 * when we spoof an Intel vendor ID (for cross vendor migration).
3196 		 * In this case we use this intercept to track the high
3197 		 * 32 bit part of these msrs to support Intel's
3198 		 * implementation of SYSENTER/SYSEXIT.
3199 		 */
3200 		svm->sysenter_eip_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0;
3201 		break;
3202 	case MSR_IA32_SYSENTER_ESP:
3203 		svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
3204 		svm->sysenter_esp_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0;
3205 		break;
3206 	case MSR_TSC_AUX:
3207 		/*
3208 		 * TSC_AUX is always virtualized for SEV-ES guests when the
3209 		 * feature is available. The user return MSR support is not
3210 		 * required in this case because TSC_AUX is restored on #VMEXIT
3211 		 * from the host save area (which has been initialized in
3212 		 * svm_enable_virtualization_cpu()).
3213 		 */
3214 		if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
3215 			break;
3216 
3217 		/*
3218 		 * TSC_AUX is usually changed only during boot and never read
3219 		 * directly.  Intercept TSC_AUX instead of exposing it to the
3220 		 * guest via direct_access_msrs, and switch it via user return.
3221 		 */
3222 		preempt_disable();
3223 		ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
3224 		preempt_enable();
3225 		if (ret)
3226 			break;
3227 
3228 		svm->tsc_aux = data;
3229 		break;
3230 	case MSR_IA32_DEBUGCTLMSR:
3231 		if (!lbrv) {
3232 			kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3233 			break;
3234 		}
3235 
3236 		/*
3237 		 * AMD changed the architectural behavior of bits 5:2.  On CPUs
3238 		 * without BusLockTrap, bits 5:2 control "external pins", but
3239 		 * on CPUs that support BusLockDetect, bit 2 enables BusLockTrap
3240 		 * and bits 5:3 are reserved-to-zero.  Sadly, old KVM allowed
3241 		 * the guest to set bits 5:2 despite not actually virtualizing
3242 		 * Performance-Monitoring/Breakpoint external pins.  Drop bits
3243 		 * 5:2 for backwards compatibility.
3244 		 */
3245 		data &= ~GENMASK(5, 2);
3246 
3247 		/*
3248 		 * Suppress BTF as KVM doesn't virtualize BTF, but there's no
3249 		 * way to communicate lack of support to the guest.
3250 		 */
3251 		if (data & DEBUGCTLMSR_BTF) {
3252 			kvm_pr_unimpl_wrmsr(vcpu, MSR_IA32_DEBUGCTLMSR, data);
3253 			data &= ~DEBUGCTLMSR_BTF;
3254 		}
3255 
3256 		if (data & DEBUGCTL_RESERVED_BITS)
3257 			return 1;
3258 
3259 		svm_get_lbr_vmcb(svm)->save.dbgctl = data;
3260 		svm_update_lbrv(vcpu);
3261 		break;
3262 	case MSR_VM_HSAVE_PA:
3263 		/*
3264 		 * Old kernels did not validate the value written to
3265 		 * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
3266 		 * value to allow live migrating buggy or malicious guests
3267 		 * originating from those kernels.
3268 		 */
3269 		if (!msr->host_initiated && !page_address_valid(vcpu, data))
3270 			return 1;
3271 
3272 		svm->nested.hsave_msr = data & PAGE_MASK;
3273 		break;
3274 	case MSR_VM_CR:
3275 		return svm_set_vm_cr(vcpu, data);
3276 	case MSR_VM_IGNNE:
3277 		kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3278 		break;
3279 	case MSR_AMD64_DE_CFG: {
3280 		u64 supported_de_cfg;
3281 
3282 		if (svm_get_feature_msr(ecx, &supported_de_cfg))
3283 			return 1;
3284 
3285 		if (data & ~supported_de_cfg)
3286 			return 1;
3287 
3288 		svm->msr_decfg = data;
3289 		break;
3290 	}
3291 	default:
3292 		return kvm_set_msr_common(vcpu, msr);
3293 	}
3294 	return ret;
3295 }
3296 
msr_interception(struct kvm_vcpu * vcpu)3297 static int msr_interception(struct kvm_vcpu *vcpu)
3298 {
3299 	if (to_svm(vcpu)->vmcb->control.exit_info_1)
3300 		return kvm_emulate_wrmsr(vcpu);
3301 	else
3302 		return kvm_emulate_rdmsr(vcpu);
3303 }
3304 
interrupt_window_interception(struct kvm_vcpu * vcpu)3305 static int interrupt_window_interception(struct kvm_vcpu *vcpu)
3306 {
3307 	kvm_make_request(KVM_REQ_EVENT, vcpu);
3308 	svm_clear_vintr(to_svm(vcpu));
3309 
3310 	/*
3311 	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
3312 	 * In this case AVIC was temporarily disabled for
3313 	 * requesting the IRQ window and we have to re-enable it.
3314 	 *
3315 	 * If running nested, still remove the VM wide AVIC inhibit to
3316 	 * support case in which the interrupt window was requested when the
3317 	 * vCPU was not running nested.
3318 
3319 	 * All vCPUs which run still run nested, will remain to have their
3320 	 * AVIC still inhibited due to per-cpu AVIC inhibition.
3321 	 */
3322 	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3323 
3324 	++vcpu->stat.irq_window_exits;
3325 	return 1;
3326 }
3327 
pause_interception(struct kvm_vcpu * vcpu)3328 static int pause_interception(struct kvm_vcpu *vcpu)
3329 {
3330 	bool in_kernel;
3331 	/*
3332 	 * CPL is not made available for an SEV-ES guest, therefore
3333 	 * vcpu->arch.preempted_in_kernel can never be true.  Just
3334 	 * set in_kernel to false as well.
3335 	 */
3336 	in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3337 
3338 	grow_ple_window(vcpu);
3339 
3340 	kvm_vcpu_on_spin(vcpu, in_kernel);
3341 	return kvm_skip_emulated_instruction(vcpu);
3342 }
3343 
invpcid_interception(struct kvm_vcpu * vcpu)3344 static int invpcid_interception(struct kvm_vcpu *vcpu)
3345 {
3346 	struct vcpu_svm *svm = to_svm(vcpu);
3347 	unsigned long type;
3348 	gva_t gva;
3349 
3350 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_INVPCID)) {
3351 		kvm_queue_exception(vcpu, UD_VECTOR);
3352 		return 1;
3353 	}
3354 
3355 	/*
3356 	 * For an INVPCID intercept:
3357 	 * EXITINFO1 provides the linear address of the memory operand.
3358 	 * EXITINFO2 provides the contents of the register operand.
3359 	 */
3360 	type = svm->vmcb->control.exit_info_2;
3361 	gva = svm->vmcb->control.exit_info_1;
3362 
3363 	/*
3364 	 * FIXME: Perform segment checks for 32-bit mode, and inject #SS if the
3365 	 *        stack segment is used.  The intercept takes priority over all
3366 	 *        #GP checks except CPL>0, but somehow still generates a linear
3367 	 *        address?  The APM is sorely lacking.
3368 	 */
3369 	if (is_noncanonical_address(gva, vcpu, 0)) {
3370 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
3371 		return 1;
3372 	}
3373 
3374 	return kvm_handle_invpcid(vcpu, type, gva);
3375 }
3376 
3377 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3378 	[SVM_EXIT_READ_CR0]			= cr_interception,
3379 	[SVM_EXIT_READ_CR3]			= cr_interception,
3380 	[SVM_EXIT_READ_CR4]			= cr_interception,
3381 	[SVM_EXIT_READ_CR8]			= cr_interception,
3382 	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
3383 	[SVM_EXIT_WRITE_CR0]			= cr_interception,
3384 	[SVM_EXIT_WRITE_CR3]			= cr_interception,
3385 	[SVM_EXIT_WRITE_CR4]			= cr_interception,
3386 	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
3387 	[SVM_EXIT_READ_DR0]			= dr_interception,
3388 	[SVM_EXIT_READ_DR1]			= dr_interception,
3389 	[SVM_EXIT_READ_DR2]			= dr_interception,
3390 	[SVM_EXIT_READ_DR3]			= dr_interception,
3391 	[SVM_EXIT_READ_DR4]			= dr_interception,
3392 	[SVM_EXIT_READ_DR5]			= dr_interception,
3393 	[SVM_EXIT_READ_DR6]			= dr_interception,
3394 	[SVM_EXIT_READ_DR7]			= dr_interception,
3395 	[SVM_EXIT_WRITE_DR0]			= dr_interception,
3396 	[SVM_EXIT_WRITE_DR1]			= dr_interception,
3397 	[SVM_EXIT_WRITE_DR2]			= dr_interception,
3398 	[SVM_EXIT_WRITE_DR3]			= dr_interception,
3399 	[SVM_EXIT_WRITE_DR4]			= dr_interception,
3400 	[SVM_EXIT_WRITE_DR5]			= dr_interception,
3401 	[SVM_EXIT_WRITE_DR6]			= dr_interception,
3402 	[SVM_EXIT_WRITE_DR7]			= dr_interception,
3403 	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
3404 	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
3405 	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
3406 	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
3407 	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
3408 	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
3409 	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
3410 	[SVM_EXIT_INTR]				= intr_interception,
3411 	[SVM_EXIT_NMI]				= nmi_interception,
3412 	[SVM_EXIT_SMI]				= smi_interception,
3413 	[SVM_EXIT_VINTR]			= interrupt_window_interception,
3414 	[SVM_EXIT_RDPMC]			= kvm_emulate_rdpmc,
3415 	[SVM_EXIT_CPUID]			= kvm_emulate_cpuid,
3416 	[SVM_EXIT_IRET]                         = iret_interception,
3417 	[SVM_EXIT_INVD]                         = kvm_emulate_invd,
3418 	[SVM_EXIT_PAUSE]			= pause_interception,
3419 	[SVM_EXIT_HLT]				= kvm_emulate_halt,
3420 	[SVM_EXIT_INVLPG]			= invlpg_interception,
3421 	[SVM_EXIT_INVLPGA]			= invlpga_interception,
3422 	[SVM_EXIT_IOIO]				= io_interception,
3423 	[SVM_EXIT_MSR]				= msr_interception,
3424 	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
3425 	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
3426 	[SVM_EXIT_VMRUN]			= vmrun_interception,
3427 	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
3428 	[SVM_EXIT_VMLOAD]			= vmload_interception,
3429 	[SVM_EXIT_VMSAVE]			= vmsave_interception,
3430 	[SVM_EXIT_STGI]				= stgi_interception,
3431 	[SVM_EXIT_CLGI]				= clgi_interception,
3432 	[SVM_EXIT_SKINIT]			= skinit_interception,
3433 	[SVM_EXIT_RDTSCP]			= kvm_handle_invalid_op,
3434 	[SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3435 	[SVM_EXIT_MONITOR]			= kvm_emulate_monitor,
3436 	[SVM_EXIT_MWAIT]			= kvm_emulate_mwait,
3437 	[SVM_EXIT_XSETBV]			= kvm_emulate_xsetbv,
3438 	[SVM_EXIT_RDPRU]			= kvm_handle_invalid_op,
3439 	[SVM_EXIT_EFER_WRITE_TRAP]		= efer_trap,
3440 	[SVM_EXIT_CR0_WRITE_TRAP]		= cr_trap,
3441 	[SVM_EXIT_CR4_WRITE_TRAP]		= cr_trap,
3442 	[SVM_EXIT_CR8_WRITE_TRAP]		= cr_trap,
3443 	[SVM_EXIT_INVPCID]                      = invpcid_interception,
3444 	[SVM_EXIT_IDLE_HLT]			= kvm_emulate_halt,
3445 	[SVM_EXIT_NPF]				= npf_interception,
3446 	[SVM_EXIT_RSM]                          = rsm_interception,
3447 	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
3448 	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
3449 #ifdef CONFIG_KVM_AMD_SEV
3450 	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
3451 #endif
3452 };
3453 
dump_vmcb(struct kvm_vcpu * vcpu)3454 static void dump_vmcb(struct kvm_vcpu *vcpu)
3455 {
3456 	struct vcpu_svm *svm = to_svm(vcpu);
3457 	struct vmcb_control_area *control = &svm->vmcb->control;
3458 	struct vmcb_save_area *save = &svm->vmcb->save;
3459 	struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3460 
3461 	if (!dump_invalid_vmcb) {
3462 		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3463 		return;
3464 	}
3465 
3466 	pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3467 	       svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3468 	pr_err("VMCB Control Area:\n");
3469 	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3470 	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3471 	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3472 	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3473 	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3474 	pr_err("%-20s%08x %08x\n", "intercepts:",
3475               control->intercepts[INTERCEPT_WORD3],
3476 	       control->intercepts[INTERCEPT_WORD4]);
3477 	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3478 	pr_err("%-20s%d\n", "pause filter threshold:",
3479 	       control->pause_filter_thresh);
3480 	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3481 	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3482 	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3483 	pr_err("%-20s%d\n", "asid:", control->asid);
3484 	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3485 	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3486 	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3487 	pr_err("%-20s%08x\n", "int_state:", control->int_state);
3488 	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3489 	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3490 	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3491 	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3492 	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3493 	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3494 	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3495 	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3496 	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3497 	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3498 	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3499 	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3500 	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3501 	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3502 	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3503 	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3504 	pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3505 	pr_err("VMCB State Save Area:\n");
3506 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3507 	       "es:",
3508 	       save->es.selector, save->es.attrib,
3509 	       save->es.limit, save->es.base);
3510 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3511 	       "cs:",
3512 	       save->cs.selector, save->cs.attrib,
3513 	       save->cs.limit, save->cs.base);
3514 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3515 	       "ss:",
3516 	       save->ss.selector, save->ss.attrib,
3517 	       save->ss.limit, save->ss.base);
3518 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3519 	       "ds:",
3520 	       save->ds.selector, save->ds.attrib,
3521 	       save->ds.limit, save->ds.base);
3522 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3523 	       "fs:",
3524 	       save01->fs.selector, save01->fs.attrib,
3525 	       save01->fs.limit, save01->fs.base);
3526 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3527 	       "gs:",
3528 	       save01->gs.selector, save01->gs.attrib,
3529 	       save01->gs.limit, save01->gs.base);
3530 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3531 	       "gdtr:",
3532 	       save->gdtr.selector, save->gdtr.attrib,
3533 	       save->gdtr.limit, save->gdtr.base);
3534 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3535 	       "ldtr:",
3536 	       save01->ldtr.selector, save01->ldtr.attrib,
3537 	       save01->ldtr.limit, save01->ldtr.base);
3538 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3539 	       "idtr:",
3540 	       save->idtr.selector, save->idtr.attrib,
3541 	       save->idtr.limit, save->idtr.base);
3542 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3543 	       "tr:",
3544 	       save01->tr.selector, save01->tr.attrib,
3545 	       save01->tr.limit, save01->tr.base);
3546 	pr_err("vmpl: %d   cpl:  %d               efer:          %016llx\n",
3547 	       save->vmpl, save->cpl, save->efer);
3548 	pr_err("%-15s %016llx %-13s %016llx\n",
3549 	       "cr0:", save->cr0, "cr2:", save->cr2);
3550 	pr_err("%-15s %016llx %-13s %016llx\n",
3551 	       "cr3:", save->cr3, "cr4:", save->cr4);
3552 	pr_err("%-15s %016llx %-13s %016llx\n",
3553 	       "dr6:", save->dr6, "dr7:", save->dr7);
3554 	pr_err("%-15s %016llx %-13s %016llx\n",
3555 	       "rip:", save->rip, "rflags:", save->rflags);
3556 	pr_err("%-15s %016llx %-13s %016llx\n",
3557 	       "rsp:", save->rsp, "rax:", save->rax);
3558 	pr_err("%-15s %016llx %-13s %016llx\n",
3559 	       "star:", save01->star, "lstar:", save01->lstar);
3560 	pr_err("%-15s %016llx %-13s %016llx\n",
3561 	       "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3562 	pr_err("%-15s %016llx %-13s %016llx\n",
3563 	       "kernel_gs_base:", save01->kernel_gs_base,
3564 	       "sysenter_cs:", save01->sysenter_cs);
3565 	pr_err("%-15s %016llx %-13s %016llx\n",
3566 	       "sysenter_esp:", save01->sysenter_esp,
3567 	       "sysenter_eip:", save01->sysenter_eip);
3568 	pr_err("%-15s %016llx %-13s %016llx\n",
3569 	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3570 	pr_err("%-15s %016llx %-13s %016llx\n",
3571 	       "br_from:", save->br_from, "br_to:", save->br_to);
3572 	pr_err("%-15s %016llx %-13s %016llx\n",
3573 	       "excp_from:", save->last_excp_from,
3574 	       "excp_to:", save->last_excp_to);
3575 }
3576 
svm_check_exit_valid(u64 exit_code)3577 static bool svm_check_exit_valid(u64 exit_code)
3578 {
3579 	return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3580 		svm_exit_handlers[exit_code]);
3581 }
3582 
svm_handle_invalid_exit(struct kvm_vcpu * vcpu,u64 exit_code)3583 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3584 {
3585 	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3586 	dump_vmcb(vcpu);
3587 	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3588 	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3589 	vcpu->run->internal.ndata = 2;
3590 	vcpu->run->internal.data[0] = exit_code;
3591 	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3592 	return 0;
3593 }
3594 
svm_invoke_exit_handler(struct kvm_vcpu * vcpu,u64 exit_code)3595 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3596 {
3597 	if (!svm_check_exit_valid(exit_code))
3598 		return svm_handle_invalid_exit(vcpu, exit_code);
3599 
3600 #ifdef CONFIG_MITIGATION_RETPOLINE
3601 	if (exit_code == SVM_EXIT_MSR)
3602 		return msr_interception(vcpu);
3603 	else if (exit_code == SVM_EXIT_VINTR)
3604 		return interrupt_window_interception(vcpu);
3605 	else if (exit_code == SVM_EXIT_INTR)
3606 		return intr_interception(vcpu);
3607 	else if (exit_code == SVM_EXIT_HLT || exit_code == SVM_EXIT_IDLE_HLT)
3608 		return kvm_emulate_halt(vcpu);
3609 	else if (exit_code == SVM_EXIT_NPF)
3610 		return npf_interception(vcpu);
3611 #endif
3612 	return svm_exit_handlers[exit_code](vcpu);
3613 }
3614 
svm_get_exit_info(struct kvm_vcpu * vcpu,u32 * reason,u64 * info1,u64 * info2,u32 * intr_info,u32 * error_code)3615 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3616 			      u64 *info1, u64 *info2,
3617 			      u32 *intr_info, u32 *error_code)
3618 {
3619 	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3620 
3621 	*reason = control->exit_code;
3622 	*info1 = control->exit_info_1;
3623 	*info2 = control->exit_info_2;
3624 	*intr_info = control->exit_int_info;
3625 	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3626 	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3627 		*error_code = control->exit_int_info_err;
3628 	else
3629 		*error_code = 0;
3630 }
3631 
svm_get_entry_info(struct kvm_vcpu * vcpu,u32 * intr_info,u32 * error_code)3632 static void svm_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info,
3633 			       u32 *error_code)
3634 {
3635 	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3636 
3637 	*intr_info = control->event_inj;
3638 
3639 	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3640 	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3641 		*error_code = control->event_inj_err;
3642 	else
3643 		*error_code = 0;
3644 
3645 }
3646 
svm_handle_exit(struct kvm_vcpu * vcpu,fastpath_t exit_fastpath)3647 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3648 {
3649 	struct vcpu_svm *svm = to_svm(vcpu);
3650 	struct kvm_run *kvm_run = vcpu->run;
3651 	u32 exit_code = svm->vmcb->control.exit_code;
3652 
3653 	/* SEV-ES guests must use the CR write traps to track CR registers. */
3654 	if (!sev_es_guest(vcpu->kvm)) {
3655 		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3656 			vcpu->arch.cr0 = svm->vmcb->save.cr0;
3657 		if (npt_enabled)
3658 			vcpu->arch.cr3 = svm->vmcb->save.cr3;
3659 	}
3660 
3661 	if (is_guest_mode(vcpu)) {
3662 		int vmexit;
3663 
3664 		trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3665 
3666 		vmexit = nested_svm_exit_special(svm);
3667 
3668 		if (vmexit == NESTED_EXIT_CONTINUE)
3669 			vmexit = nested_svm_exit_handled(svm);
3670 
3671 		if (vmexit == NESTED_EXIT_DONE)
3672 			return 1;
3673 	}
3674 
3675 	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3676 		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3677 		kvm_run->fail_entry.hardware_entry_failure_reason
3678 			= svm->vmcb->control.exit_code;
3679 		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3680 		dump_vmcb(vcpu);
3681 		return 0;
3682 	}
3683 
3684 	if (exit_fastpath != EXIT_FASTPATH_NONE)
3685 		return 1;
3686 
3687 	return svm_invoke_exit_handler(vcpu, exit_code);
3688 }
3689 
pre_svm_run(struct kvm_vcpu * vcpu)3690 static int pre_svm_run(struct kvm_vcpu *vcpu)
3691 {
3692 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
3693 	struct vcpu_svm *svm = to_svm(vcpu);
3694 
3695 	/*
3696 	 * If the previous vmrun of the vmcb occurred on a different physical
3697 	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3698 	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3699 	 */
3700 	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3701 		svm->current_vmcb->asid_generation = 0;
3702 		vmcb_mark_all_dirty(svm->vmcb);
3703 		svm->current_vmcb->cpu = vcpu->cpu;
3704         }
3705 
3706 	if (sev_guest(vcpu->kvm))
3707 		return pre_sev_run(svm, vcpu->cpu);
3708 
3709 	/* FIXME: handle wraparound of asid_generation */
3710 	if (svm->current_vmcb->asid_generation != sd->asid_generation)
3711 		new_asid(svm, sd);
3712 
3713 	return 0;
3714 }
3715 
svm_inject_nmi(struct kvm_vcpu * vcpu)3716 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3717 {
3718 	struct vcpu_svm *svm = to_svm(vcpu);
3719 
3720 	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3721 
3722 	if (svm->nmi_l1_to_l2)
3723 		return;
3724 
3725 	/*
3726 	 * No need to manually track NMI masking when vNMI is enabled, hardware
3727 	 * automatically sets V_NMI_BLOCKING_MASK as appropriate, including the
3728 	 * case where software directly injects an NMI.
3729 	 */
3730 	if (!is_vnmi_enabled(svm)) {
3731 		svm->nmi_masked = true;
3732 		svm_set_iret_intercept(svm);
3733 	}
3734 	++vcpu->stat.nmi_injections;
3735 }
3736 
svm_is_vnmi_pending(struct kvm_vcpu * vcpu)3737 static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu)
3738 {
3739 	struct vcpu_svm *svm = to_svm(vcpu);
3740 
3741 	if (!is_vnmi_enabled(svm))
3742 		return false;
3743 
3744 	return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK);
3745 }
3746 
svm_set_vnmi_pending(struct kvm_vcpu * vcpu)3747 static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu)
3748 {
3749 	struct vcpu_svm *svm = to_svm(vcpu);
3750 
3751 	if (!is_vnmi_enabled(svm))
3752 		return false;
3753 
3754 	if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK)
3755 		return false;
3756 
3757 	svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK;
3758 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
3759 
3760 	/*
3761 	 * Because the pending NMI is serviced by hardware, KVM can't know when
3762 	 * the NMI is "injected", but for all intents and purposes, passing the
3763 	 * NMI off to hardware counts as injection.
3764 	 */
3765 	++vcpu->stat.nmi_injections;
3766 
3767 	return true;
3768 }
3769 
svm_inject_irq(struct kvm_vcpu * vcpu,bool reinjected)3770 static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
3771 {
3772 	struct vcpu_svm *svm = to_svm(vcpu);
3773 	u32 type;
3774 
3775 	if (vcpu->arch.interrupt.soft) {
3776 		if (svm_update_soft_interrupt_rip(vcpu))
3777 			return;
3778 
3779 		type = SVM_EVTINJ_TYPE_SOFT;
3780 	} else {
3781 		type = SVM_EVTINJ_TYPE_INTR;
3782 	}
3783 
3784 	trace_kvm_inj_virq(vcpu->arch.interrupt.nr,
3785 			   vcpu->arch.interrupt.soft, reinjected);
3786 	++vcpu->stat.irq_injections;
3787 
3788 	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3789 				       SVM_EVTINJ_VALID | type;
3790 }
3791 
svm_complete_interrupt_delivery(struct kvm_vcpu * vcpu,int delivery_mode,int trig_mode,int vector)3792 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
3793 				     int trig_mode, int vector)
3794 {
3795 	/*
3796 	 * apic->apicv_active must be read after vcpu->mode.
3797 	 * Pairs with smp_store_release in vcpu_enter_guest.
3798 	 */
3799 	bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
3800 
3801 	/* Note, this is called iff the local APIC is in-kernel. */
3802 	if (!READ_ONCE(vcpu->arch.apic->apicv_active)) {
3803 		/* Process the interrupt via kvm_check_and_inject_events(). */
3804 		kvm_make_request(KVM_REQ_EVENT, vcpu);
3805 		kvm_vcpu_kick(vcpu);
3806 		return;
3807 	}
3808 
3809 	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
3810 	if (in_guest_mode) {
3811 		/*
3812 		 * Signal the doorbell to tell hardware to inject the IRQ.  If
3813 		 * the vCPU exits the guest before the doorbell chimes, hardware
3814 		 * will automatically process AVIC interrupts at the next VMRUN.
3815 		 */
3816 		avic_ring_doorbell(vcpu);
3817 	} else {
3818 		/*
3819 		 * Wake the vCPU if it was blocking.  KVM will then detect the
3820 		 * pending IRQ when checking if the vCPU has a wake event.
3821 		 */
3822 		kvm_vcpu_wake_up(vcpu);
3823 	}
3824 }
3825 
svm_deliver_interrupt(struct kvm_lapic * apic,int delivery_mode,int trig_mode,int vector)3826 static void svm_deliver_interrupt(struct kvm_lapic *apic,  int delivery_mode,
3827 				  int trig_mode, int vector)
3828 {
3829 	kvm_lapic_set_irr(vector, apic);
3830 
3831 	/*
3832 	 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in
3833 	 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before
3834 	 * the read of guest_mode.  This guarantees that either VMRUN will see
3835 	 * and process the new vIRR entry, or that svm_complete_interrupt_delivery
3836 	 * will signal the doorbell if the CPU has already entered the guest.
3837 	 */
3838 	smp_mb__after_atomic();
3839 	svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector);
3840 }
3841 
svm_update_cr8_intercept(struct kvm_vcpu * vcpu,int tpr,int irr)3842 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3843 {
3844 	struct vcpu_svm *svm = to_svm(vcpu);
3845 
3846 	/*
3847 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
3848 	 * tracking is done using the CR write traps.
3849 	 */
3850 	if (sev_es_guest(vcpu->kvm))
3851 		return;
3852 
3853 	if (nested_svm_virtualize_tpr(vcpu))
3854 		return;
3855 
3856 	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3857 
3858 	if (irr == -1)
3859 		return;
3860 
3861 	if (tpr >= irr)
3862 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3863 }
3864 
svm_get_nmi_mask(struct kvm_vcpu * vcpu)3865 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3866 {
3867 	struct vcpu_svm *svm = to_svm(vcpu);
3868 
3869 	if (is_vnmi_enabled(svm))
3870 		return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK;
3871 	else
3872 		return svm->nmi_masked;
3873 }
3874 
svm_set_nmi_mask(struct kvm_vcpu * vcpu,bool masked)3875 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3876 {
3877 	struct vcpu_svm *svm = to_svm(vcpu);
3878 
3879 	if (is_vnmi_enabled(svm)) {
3880 		if (masked)
3881 			svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK;
3882 		else
3883 			svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
3884 
3885 	} else {
3886 		svm->nmi_masked = masked;
3887 		if (masked)
3888 			svm_set_iret_intercept(svm);
3889 		else
3890 			svm_clr_iret_intercept(svm);
3891 	}
3892 }
3893 
svm_nmi_blocked(struct kvm_vcpu * vcpu)3894 bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3895 {
3896 	struct vcpu_svm *svm = to_svm(vcpu);
3897 	struct vmcb *vmcb = svm->vmcb;
3898 
3899 	if (!gif_set(svm))
3900 		return true;
3901 
3902 	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3903 		return false;
3904 
3905 	if (svm_get_nmi_mask(vcpu))
3906 		return true;
3907 
3908 	return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK;
3909 }
3910 
svm_nmi_allowed(struct kvm_vcpu * vcpu,bool for_injection)3911 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3912 {
3913 	struct vcpu_svm *svm = to_svm(vcpu);
3914 	if (svm->nested.nested_run_pending)
3915 		return -EBUSY;
3916 
3917 	if (svm_nmi_blocked(vcpu))
3918 		return 0;
3919 
3920 	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3921 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3922 		return -EBUSY;
3923 	return 1;
3924 }
3925 
svm_interrupt_blocked(struct kvm_vcpu * vcpu)3926 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3927 {
3928 	struct vcpu_svm *svm = to_svm(vcpu);
3929 	struct vmcb *vmcb = svm->vmcb;
3930 
3931 	if (!gif_set(svm))
3932 		return true;
3933 
3934 	if (is_guest_mode(vcpu)) {
3935 		/* As long as interrupts are being delivered...  */
3936 		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3937 		    ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3938 		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3939 			return true;
3940 
3941 		/* ... vmexits aren't blocked by the interrupt shadow  */
3942 		if (nested_exit_on_intr(svm))
3943 			return false;
3944 	} else {
3945 		if (!svm_get_if_flag(vcpu))
3946 			return true;
3947 	}
3948 
3949 	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3950 }
3951 
svm_interrupt_allowed(struct kvm_vcpu * vcpu,bool for_injection)3952 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3953 {
3954 	struct vcpu_svm *svm = to_svm(vcpu);
3955 
3956 	if (svm->nested.nested_run_pending)
3957 		return -EBUSY;
3958 
3959 	if (svm_interrupt_blocked(vcpu))
3960 		return 0;
3961 
3962 	/*
3963 	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3964 	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3965 	 */
3966 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3967 		return -EBUSY;
3968 
3969 	return 1;
3970 }
3971 
svm_enable_irq_window(struct kvm_vcpu * vcpu)3972 static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3973 {
3974 	struct vcpu_svm *svm = to_svm(vcpu);
3975 
3976 	/*
3977 	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3978 	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3979 	 * get that intercept, this function will be called again though and
3980 	 * we'll get the vintr intercept. However, if the vGIF feature is
3981 	 * enabled, the STGI interception will not occur. Enable the irq
3982 	 * window under the assumption that the hardware will set the GIF.
3983 	 */
3984 	if (vgif || gif_set(svm)) {
3985 		/*
3986 		 * IRQ window is not needed when AVIC is enabled,
3987 		 * unless we have pending ExtINT since it cannot be injected
3988 		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
3989 		 * and fallback to injecting IRQ via V_IRQ.
3990 		 *
3991 		 * If running nested, AVIC is already locally inhibited
3992 		 * on this vCPU, therefore there is no need to request
3993 		 * the VM wide AVIC inhibition.
3994 		 */
3995 		if (!is_guest_mode(vcpu))
3996 			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3997 
3998 		svm_set_vintr(svm);
3999 	}
4000 }
4001 
svm_enable_nmi_window(struct kvm_vcpu * vcpu)4002 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
4003 {
4004 	struct vcpu_svm *svm = to_svm(vcpu);
4005 
4006 	/*
4007 	 * If NMIs are outright masked, i.e. the vCPU is already handling an
4008 	 * NMI, and KVM has not yet intercepted an IRET, then there is nothing
4009 	 * more to do at this time as KVM has already enabled IRET intercepts.
4010 	 * If KVM has already intercepted IRET, then single-step over the IRET,
4011 	 * as NMIs aren't architecturally unmasked until the IRET completes.
4012 	 *
4013 	 * If vNMI is enabled, KVM should never request an NMI window if NMIs
4014 	 * are masked, as KVM allows at most one to-be-injected NMI and one
4015 	 * pending NMI.  If two NMIs arrive simultaneously, KVM will inject one
4016 	 * NMI and set V_NMI_PENDING for the other, but if and only if NMIs are
4017 	 * unmasked.  KVM _will_ request an NMI window in some situations, e.g.
4018 	 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately
4019 	 * inject the NMI.  In those situations, KVM needs to single-step over
4020 	 * the STI shadow or intercept STGI.
4021 	 */
4022 	if (svm_get_nmi_mask(vcpu)) {
4023 		WARN_ON_ONCE(is_vnmi_enabled(svm));
4024 
4025 		if (!svm->awaiting_iret_completion)
4026 			return; /* IRET will cause a vm exit */
4027 	}
4028 
4029 	/*
4030 	 * SEV-ES guests are responsible for signaling when a vCPU is ready to
4031 	 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e.
4032 	 * KVM can't intercept and single-step IRET to detect when NMIs are
4033 	 * unblocked (architecturally speaking).  See SVM_VMGEXIT_NMI_COMPLETE.
4034 	 *
4035 	 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware
4036 	 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not
4037 	 * supported NAEs in the GHCB protocol.
4038 	 */
4039 	if (sev_es_guest(vcpu->kvm))
4040 		return;
4041 
4042 	if (!gif_set(svm)) {
4043 		if (vgif)
4044 			svm_set_intercept(svm, INTERCEPT_STGI);
4045 		return; /* STGI will cause a vm exit */
4046 	}
4047 
4048 	/*
4049 	 * Something prevents NMI from been injected. Single step over possible
4050 	 * problem (IRET or exception injection or interrupt shadow)
4051 	 */
4052 	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
4053 	svm->nmi_singlestep = true;
4054 	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
4055 }
4056 
svm_flush_tlb_asid(struct kvm_vcpu * vcpu)4057 static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
4058 {
4059 	struct vcpu_svm *svm = to_svm(vcpu);
4060 
4061 	/*
4062 	 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries.
4063 	 * A TLB flush for the current ASID flushes both "host" and "guest" TLB
4064 	 * entries, and thus is a superset of Hyper-V's fine grained flushing.
4065 	 */
4066 	kvm_hv_vcpu_purge_flush_tlb(vcpu);
4067 
4068 	/*
4069 	 * Flush only the current ASID even if the TLB flush was invoked via
4070 	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
4071 	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
4072 	 * unconditionally does a TLB flush on both nested VM-Enter and nested
4073 	 * VM-Exit (via kvm_mmu_reset_context()).
4074 	 */
4075 	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
4076 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
4077 	else
4078 		svm->current_vmcb->asid_generation--;
4079 }
4080 
svm_flush_tlb_current(struct kvm_vcpu * vcpu)4081 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
4082 {
4083 	hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
4084 
4085 	/*
4086 	 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
4087 	 * flush the NPT mappings via hypercall as flushing the ASID only
4088 	 * affects virtual to physical mappings, it does not invalidate guest
4089 	 * physical to host physical mappings.
4090 	 */
4091 	if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
4092 		hyperv_flush_guest_mapping(root_tdp);
4093 
4094 	svm_flush_tlb_asid(vcpu);
4095 }
4096 
svm_flush_tlb_all(struct kvm_vcpu * vcpu)4097 static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
4098 {
4099 	/*
4100 	 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
4101 	 * flushes should be routed to hv_flush_remote_tlbs() without requesting
4102 	 * a "regular" remote flush.  Reaching this point means either there's
4103 	 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of
4104 	 * which might be fatal to the guest.  Yell, but try to recover.
4105 	 */
4106 	if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
4107 		hv_flush_remote_tlbs(vcpu->kvm);
4108 
4109 	svm_flush_tlb_asid(vcpu);
4110 }
4111 
svm_flush_tlb_gva(struct kvm_vcpu * vcpu,gva_t gva)4112 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
4113 {
4114 	struct vcpu_svm *svm = to_svm(vcpu);
4115 
4116 	invlpga(gva, svm->vmcb->control.asid);
4117 }
4118 
sync_cr8_to_lapic(struct kvm_vcpu * vcpu)4119 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
4120 {
4121 	struct vcpu_svm *svm = to_svm(vcpu);
4122 
4123 	if (nested_svm_virtualize_tpr(vcpu))
4124 		return;
4125 
4126 	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
4127 		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
4128 		kvm_set_cr8(vcpu, cr8);
4129 	}
4130 }
4131 
sync_lapic_to_cr8(struct kvm_vcpu * vcpu)4132 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
4133 {
4134 	struct vcpu_svm *svm = to_svm(vcpu);
4135 	u64 cr8;
4136 
4137 	if (nested_svm_virtualize_tpr(vcpu) ||
4138 	    kvm_vcpu_apicv_active(vcpu))
4139 		return;
4140 
4141 	cr8 = kvm_get_cr8(vcpu);
4142 	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
4143 	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
4144 }
4145 
svm_complete_soft_interrupt(struct kvm_vcpu * vcpu,u8 vector,int type)4146 static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector,
4147 					int type)
4148 {
4149 	bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT);
4150 	bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT);
4151 	struct vcpu_svm *svm = to_svm(vcpu);
4152 
4153 	/*
4154 	 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's
4155 	 * associated with the original soft exception/interrupt.  next_rip is
4156 	 * cleared on all exits that can occur while vectoring an event, so KVM
4157 	 * needs to manually set next_rip for re-injection.  Unlike the !nrips
4158 	 * case below, this needs to be done if and only if KVM is re-injecting
4159 	 * the same event, i.e. if the event is a soft exception/interrupt,
4160 	 * otherwise next_rip is unused on VMRUN.
4161 	 */
4162 	if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) &&
4163 	    kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase))
4164 		svm->vmcb->control.next_rip = svm->soft_int_next_rip;
4165 	/*
4166 	 * If NRIPS isn't enabled, KVM must manually advance RIP prior to
4167 	 * injecting the soft exception/interrupt.  That advancement needs to
4168 	 * be unwound if vectoring didn't complete.  Note, the new event may
4169 	 * not be the injected event, e.g. if KVM injected an INTn, the INTn
4170 	 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will
4171 	 * be the reported vectored event, but RIP still needs to be unwound.
4172 	 */
4173 	else if (!nrips && (is_soft || is_exception) &&
4174 		 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase))
4175 		kvm_rip_write(vcpu, svm->soft_int_old_rip);
4176 }
4177 
svm_complete_interrupts(struct kvm_vcpu * vcpu)4178 static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
4179 {
4180 	struct vcpu_svm *svm = to_svm(vcpu);
4181 	u8 vector;
4182 	int type;
4183 	u32 exitintinfo = svm->vmcb->control.exit_int_info;
4184 	bool nmi_l1_to_l2 = svm->nmi_l1_to_l2;
4185 	bool soft_int_injected = svm->soft_int_injected;
4186 
4187 	svm->nmi_l1_to_l2 = false;
4188 	svm->soft_int_injected = false;
4189 
4190 	/*
4191 	 * If we've made progress since setting awaiting_iret_completion, we've
4192 	 * executed an IRET and can allow NMI injection.
4193 	 */
4194 	if (svm->awaiting_iret_completion &&
4195 	    kvm_rip_read(vcpu) != svm->nmi_iret_rip) {
4196 		svm->awaiting_iret_completion = false;
4197 		svm->nmi_masked = false;
4198 		kvm_make_request(KVM_REQ_EVENT, vcpu);
4199 	}
4200 
4201 	vcpu->arch.nmi_injected = false;
4202 	kvm_clear_exception_queue(vcpu);
4203 	kvm_clear_interrupt_queue(vcpu);
4204 
4205 	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4206 		return;
4207 
4208 	kvm_make_request(KVM_REQ_EVENT, vcpu);
4209 
4210 	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4211 	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4212 
4213 	if (soft_int_injected)
4214 		svm_complete_soft_interrupt(vcpu, vector, type);
4215 
4216 	switch (type) {
4217 	case SVM_EXITINTINFO_TYPE_NMI:
4218 		vcpu->arch.nmi_injected = true;
4219 		svm->nmi_l1_to_l2 = nmi_l1_to_l2;
4220 		break;
4221 	case SVM_EXITINTINFO_TYPE_EXEPT: {
4222 		u32 error_code = 0;
4223 
4224 		/*
4225 		 * Never re-inject a #VC exception.
4226 		 */
4227 		if (vector == X86_TRAP_VC)
4228 			break;
4229 
4230 		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR)
4231 			error_code = svm->vmcb->control.exit_int_info_err;
4232 
4233 		kvm_requeue_exception(vcpu, vector,
4234 				      exitintinfo & SVM_EXITINTINFO_VALID_ERR,
4235 				      error_code);
4236 		break;
4237 	}
4238 	case SVM_EXITINTINFO_TYPE_INTR:
4239 		kvm_queue_interrupt(vcpu, vector, false);
4240 		break;
4241 	case SVM_EXITINTINFO_TYPE_SOFT:
4242 		kvm_queue_interrupt(vcpu, vector, true);
4243 		break;
4244 	default:
4245 		break;
4246 	}
4247 
4248 }
4249 
svm_cancel_injection(struct kvm_vcpu * vcpu)4250 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4251 {
4252 	struct vcpu_svm *svm = to_svm(vcpu);
4253 	struct vmcb_control_area *control = &svm->vmcb->control;
4254 
4255 	control->exit_int_info = control->event_inj;
4256 	control->exit_int_info_err = control->event_inj_err;
4257 	control->event_inj = 0;
4258 	svm_complete_interrupts(vcpu);
4259 }
4260 
svm_vcpu_pre_run(struct kvm_vcpu * vcpu)4261 static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu)
4262 {
4263 	if (to_kvm_sev_info(vcpu->kvm)->need_init)
4264 		return -EINVAL;
4265 
4266 	return 1;
4267 }
4268 
svm_exit_handlers_fastpath(struct kvm_vcpu * vcpu)4269 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
4270 {
4271 	struct vcpu_svm *svm = to_svm(vcpu);
4272 
4273 	if (is_guest_mode(vcpu))
4274 		return EXIT_FASTPATH_NONE;
4275 
4276 	switch (svm->vmcb->control.exit_code) {
4277 	case SVM_EXIT_MSR:
4278 		if (!svm->vmcb->control.exit_info_1)
4279 			break;
4280 		return handle_fastpath_set_msr_irqoff(vcpu);
4281 	case SVM_EXIT_HLT:
4282 		return handle_fastpath_hlt(vcpu);
4283 	default:
4284 		break;
4285 	}
4286 
4287 	return EXIT_FASTPATH_NONE;
4288 }
4289 
svm_vcpu_enter_exit(struct kvm_vcpu * vcpu,bool spec_ctrl_intercepted)4290 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
4291 {
4292 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
4293 	struct vcpu_svm *svm = to_svm(vcpu);
4294 
4295 	guest_state_enter_irqoff();
4296 
4297 	/*
4298 	 * Set RFLAGS.IF prior to VMRUN, as the host's RFLAGS.IF at the time of
4299 	 * VMRUN controls whether or not physical IRQs are masked (KVM always
4300 	 * runs with V_INTR_MASKING_MASK).  Toggle RFLAGS.IF here to avoid the
4301 	 * temptation to do STI+VMRUN+CLI, as AMD CPUs bleed the STI shadow
4302 	 * into guest state if delivery of an event during VMRUN triggers a
4303 	 * #VMEXIT, and the guest_state transitions already tell lockdep that
4304 	 * IRQs are being enabled/disabled.  Note!  GIF=0 for the entirety of
4305 	 * this path, so IRQs aren't actually unmasked while running host code.
4306 	 */
4307 	raw_local_irq_enable();
4308 
4309 	amd_clear_divider();
4310 
4311 	if (sev_es_guest(vcpu->kvm))
4312 		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,
4313 				      sev_es_host_save_area(sd));
4314 	else
4315 		__svm_vcpu_run(svm, spec_ctrl_intercepted);
4316 
4317 	raw_local_irq_disable();
4318 
4319 	guest_state_exit_irqoff();
4320 }
4321 
svm_vcpu_run(struct kvm_vcpu * vcpu,bool force_immediate_exit)4322 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu,
4323 					  bool force_immediate_exit)
4324 {
4325 	struct vcpu_svm *svm = to_svm(vcpu);
4326 	bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);
4327 
4328 	trace_kvm_entry(vcpu, force_immediate_exit);
4329 
4330 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4331 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4332 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4333 
4334 	/*
4335 	 * Disable singlestep if we're injecting an interrupt/exception.
4336 	 * We don't want our modified rflags to be pushed on the stack where
4337 	 * we might not be able to easily reset them if we disabled NMI
4338 	 * singlestep later.
4339 	 */
4340 	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
4341 		/*
4342 		 * Event injection happens before external interrupts cause a
4343 		 * vmexit and interrupts are disabled here, so smp_send_reschedule
4344 		 * is enough to force an immediate vmexit.
4345 		 */
4346 		disable_nmi_singlestep(svm);
4347 		force_immediate_exit = true;
4348 	}
4349 
4350 	if (force_immediate_exit)
4351 		smp_send_reschedule(vcpu->cpu);
4352 
4353 	if (pre_svm_run(vcpu)) {
4354 		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4355 		vcpu->run->fail_entry.hardware_entry_failure_reason = SVM_EXIT_ERR;
4356 		vcpu->run->fail_entry.cpu = vcpu->cpu;
4357 		return EXIT_FASTPATH_EXIT_USERSPACE;
4358 	}
4359 
4360 	sync_lapic_to_cr8(vcpu);
4361 
4362 	if (unlikely(svm->asid != svm->vmcb->control.asid)) {
4363 		svm->vmcb->control.asid = svm->asid;
4364 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
4365 	}
4366 	svm->vmcb->save.cr2 = vcpu->arch.cr2;
4367 
4368 	svm_hv_update_vp_id(svm->vmcb, vcpu);
4369 
4370 	/*
4371 	 * Run with all-zero DR6 unless needed, so that we can get the exact cause
4372 	 * of a #DB.
4373 	 */
4374 	if (likely(!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)))
4375 		svm_set_dr6(vcpu, DR6_ACTIVE_LOW);
4376 
4377 	clgi();
4378 	kvm_load_guest_xsave_state(vcpu);
4379 
4380 	/*
4381 	 * Hardware only context switches DEBUGCTL if LBR virtualization is
4382 	 * enabled.  Manually load DEBUGCTL if necessary (and restore it after
4383 	 * VM-Exit), as running with the host's DEBUGCTL can negatively affect
4384 	 * guest state and can even be fatal, e.g. due to Bus Lock Detect.
4385 	 */
4386 	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
4387 	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
4388 		update_debugctlmsr(svm->vmcb->save.dbgctl);
4389 
4390 	kvm_wait_lapic_expire(vcpu);
4391 
4392 	/*
4393 	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
4394 	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
4395 	 * is no need to worry about the conditional branch over the wrmsr
4396 	 * being speculatively taken.
4397 	 */
4398 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4399 		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
4400 
4401 	svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);
4402 
4403 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4404 		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
4405 
4406 	if (!sev_es_guest(vcpu->kvm)) {
4407 		vcpu->arch.cr2 = svm->vmcb->save.cr2;
4408 		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4409 		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4410 		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4411 	}
4412 	vcpu->arch.regs_dirty = 0;
4413 
4414 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4415 		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
4416 
4417 	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
4418 	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
4419 		update_debugctlmsr(vcpu->arch.host_debugctl);
4420 
4421 	kvm_load_host_xsave_state(vcpu);
4422 	stgi();
4423 
4424 	/* Any pending NMI will happen here */
4425 
4426 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4427 		kvm_after_interrupt(vcpu);
4428 
4429 	sync_cr8_to_lapic(vcpu);
4430 
4431 	svm->next_rip = 0;
4432 	if (is_guest_mode(vcpu)) {
4433 		nested_sync_control_from_vmcb02(svm);
4434 
4435 		/* Track VMRUNs that have made past consistency checking */
4436 		if (svm->nested.nested_run_pending &&
4437 		    svm->vmcb->control.exit_code != SVM_EXIT_ERR)
4438                         ++vcpu->stat.nested_run;
4439 
4440 		svm->nested.nested_run_pending = 0;
4441 	}
4442 
4443 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4444 	vmcb_mark_all_clean(svm->vmcb);
4445 
4446 	/* if exit due to PF check for async PF */
4447 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4448 		vcpu->arch.apf.host_apf_flags =
4449 			kvm_read_and_reset_apf_flags();
4450 
4451 	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
4452 
4453 	/*
4454 	 * We need to handle MC intercepts here before the vcpu has a chance to
4455 	 * change the physical cpu
4456 	 */
4457 	if (unlikely(svm->vmcb->control.exit_code ==
4458 		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
4459 		svm_handle_mce(vcpu);
4460 
4461 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
4462 
4463 	svm_complete_interrupts(vcpu);
4464 
4465 	return svm_exit_handlers_fastpath(vcpu);
4466 }
4467 
svm_load_mmu_pgd(struct kvm_vcpu * vcpu,hpa_t root_hpa,int root_level)4468 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
4469 			     int root_level)
4470 {
4471 	struct vcpu_svm *svm = to_svm(vcpu);
4472 	unsigned long cr3;
4473 
4474 	if (npt_enabled) {
4475 		svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
4476 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
4477 
4478 		hv_track_root_tdp(vcpu, root_hpa);
4479 
4480 		cr3 = vcpu->arch.cr3;
4481 	} else if (root_level >= PT64_ROOT_4LEVEL) {
4482 		cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
4483 	} else {
4484 		/* PCID in the guest should be impossible with a 32-bit MMU. */
4485 		WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
4486 		cr3 = root_hpa;
4487 	}
4488 
4489 	svm->vmcb->save.cr3 = cr3;
4490 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
4491 }
4492 
4493 static void
svm_patch_hypercall(struct kvm_vcpu * vcpu,unsigned char * hypercall)4494 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4495 {
4496 	/*
4497 	 * Patch in the VMMCALL instruction:
4498 	 */
4499 	hypercall[0] = 0x0f;
4500 	hypercall[1] = 0x01;
4501 	hypercall[2] = 0xd9;
4502 }
4503 
4504 /*
4505  * The kvm parameter can be NULL (module initialization, or invocation before
4506  * VM creation). Be sure to check the kvm parameter before using it.
4507  */
svm_has_emulated_msr(struct kvm * kvm,u32 index)4508 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4509 {
4510 	switch (index) {
4511 	case MSR_IA32_MCG_EXT_CTL:
4512 	case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
4513 		return false;
4514 	case MSR_IA32_SMBASE:
4515 		if (!IS_ENABLED(CONFIG_KVM_SMM))
4516 			return false;
4517 		/* SEV-ES guests do not support SMM, so report false */
4518 		if (kvm && sev_es_guest(kvm))
4519 			return false;
4520 		break;
4521 	default:
4522 		break;
4523 	}
4524 
4525 	return true;
4526 }
4527 
svm_vcpu_after_set_cpuid(struct kvm_vcpu * vcpu)4528 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4529 {
4530 	struct vcpu_svm *svm = to_svm(vcpu);
4531 
4532 	/*
4533 	 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM
4534 	 * can only disable all variants of by disallowing CR4.OSXSAVE from
4535 	 * being set.  As a result, if the host has XSAVE and XSAVES, and the
4536 	 * guest has XSAVE enabled, the guest can execute XSAVES without
4537 	 * faulting.  Treat XSAVES as enabled in this case regardless of
4538 	 * whether it's advertised to the guest so that KVM context switches
4539 	 * XSS on VM-Enter/VM-Exit.  Failure to do so would effectively give
4540 	 * the guest read/write access to the host's XSS.
4541 	 */
4542 	guest_cpu_cap_change(vcpu, X86_FEATURE_XSAVES,
4543 			     boot_cpu_has(X86_FEATURE_XSAVES) &&
4544 			     guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVE));
4545 
4546 	/*
4547 	 * Intercept VMLOAD if the vCPU model is Intel in order to emulate that
4548 	 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing
4549 	 * SVM on Intel is bonkers and extremely unlikely to work).
4550 	 */
4551 	if (guest_cpuid_is_intel_compatible(vcpu))
4552 		guest_cpu_cap_clear(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD);
4553 
4554 	svm_recalc_instruction_intercepts(vcpu, svm);
4555 
4556 	if (boot_cpu_has(X86_FEATURE_IBPB))
4557 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0,
4558 				     !!guest_has_pred_cmd_msr(vcpu));
4559 
4560 	if (boot_cpu_has(X86_FEATURE_FLUSH_L1D))
4561 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,
4562 				     !!guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D));
4563 
4564 	if (sev_guest(vcpu->kvm))
4565 		sev_vcpu_after_set_cpuid(svm);
4566 
4567 	init_vmcb_after_set_cpuid(vcpu);
4568 }
4569 
svm_has_wbinvd_exit(void)4570 static bool svm_has_wbinvd_exit(void)
4571 {
4572 	return true;
4573 }
4574 
4575 #define PRE_EX(exit)  { .exit_code = (exit), \
4576 			.stage = X86_ICPT_PRE_EXCEPT, }
4577 #define POST_EX(exit) { .exit_code = (exit), \
4578 			.stage = X86_ICPT_POST_EXCEPT, }
4579 #define POST_MEM(exit) { .exit_code = (exit), \
4580 			.stage = X86_ICPT_POST_MEMACCESS, }
4581 
4582 static const struct __x86_intercept {
4583 	u32 exit_code;
4584 	enum x86_intercept_stage stage;
4585 } x86_intercept_map[] = {
4586 	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
4587 	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
4588 	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
4589 	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
4590 	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
4591 	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
4592 	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
4593 	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
4594 	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
4595 	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
4596 	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
4597 	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
4598 	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
4599 	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
4600 	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
4601 	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
4602 	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
4603 	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
4604 	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
4605 	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
4606 	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
4607 	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
4608 	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
4609 	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
4610 	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
4611 	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
4612 	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
4613 	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
4614 	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
4615 	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
4616 	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
4617 	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
4618 	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
4619 	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
4620 	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
4621 	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
4622 	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
4623 	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
4624 	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
4625 	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
4626 	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
4627 	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
4628 	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
4629 	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
4630 	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
4631 	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
4632 	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
4633 };
4634 
4635 #undef PRE_EX
4636 #undef POST_EX
4637 #undef POST_MEM
4638 
svm_check_intercept(struct kvm_vcpu * vcpu,struct x86_instruction_info * info,enum x86_intercept_stage stage,struct x86_exception * exception)4639 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4640 			       struct x86_instruction_info *info,
4641 			       enum x86_intercept_stage stage,
4642 			       struct x86_exception *exception)
4643 {
4644 	struct vcpu_svm *svm = to_svm(vcpu);
4645 	int vmexit, ret = X86EMUL_CONTINUE;
4646 	struct __x86_intercept icpt_info;
4647 	struct vmcb *vmcb = svm->vmcb;
4648 
4649 	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4650 		goto out;
4651 
4652 	icpt_info = x86_intercept_map[info->intercept];
4653 
4654 	if (stage != icpt_info.stage)
4655 		goto out;
4656 
4657 	switch (icpt_info.exit_code) {
4658 	case SVM_EXIT_READ_CR0:
4659 		if (info->intercept == x86_intercept_cr_read)
4660 			icpt_info.exit_code += info->modrm_reg;
4661 		break;
4662 	case SVM_EXIT_WRITE_CR0: {
4663 		unsigned long cr0, val;
4664 
4665 		if (info->intercept == x86_intercept_cr_write)
4666 			icpt_info.exit_code += info->modrm_reg;
4667 
4668 		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4669 		    info->intercept == x86_intercept_clts)
4670 			break;
4671 
4672 		if (!(vmcb12_is_intercept(&svm->nested.ctl,
4673 					INTERCEPT_SELECTIVE_CR0)))
4674 			break;
4675 
4676 		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4677 		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4678 
4679 		if (info->intercept == x86_intercept_lmsw) {
4680 			cr0 &= 0xfUL;
4681 			val &= 0xfUL;
4682 			/* lmsw can't clear PE - catch this here */
4683 			if (cr0 & X86_CR0_PE)
4684 				val |= X86_CR0_PE;
4685 		}
4686 
4687 		if (cr0 ^ val)
4688 			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4689 
4690 		break;
4691 	}
4692 	case SVM_EXIT_READ_DR0:
4693 	case SVM_EXIT_WRITE_DR0:
4694 		icpt_info.exit_code += info->modrm_reg;
4695 		break;
4696 	case SVM_EXIT_MSR:
4697 		if (info->intercept == x86_intercept_wrmsr)
4698 			vmcb->control.exit_info_1 = 1;
4699 		else
4700 			vmcb->control.exit_info_1 = 0;
4701 		break;
4702 	case SVM_EXIT_PAUSE:
4703 		/*
4704 		 * We get this for NOP only, but pause
4705 		 * is rep not, check this here
4706 		 */
4707 		if (info->rep_prefix != REPE_PREFIX)
4708 			goto out;
4709 		break;
4710 	case SVM_EXIT_IOIO: {
4711 		u64 exit_info;
4712 		u32 bytes;
4713 
4714 		if (info->intercept == x86_intercept_in ||
4715 		    info->intercept == x86_intercept_ins) {
4716 			exit_info = ((info->src_val & 0xffff) << 16) |
4717 				SVM_IOIO_TYPE_MASK;
4718 			bytes = info->dst_bytes;
4719 		} else {
4720 			exit_info = (info->dst_val & 0xffff) << 16;
4721 			bytes = info->src_bytes;
4722 		}
4723 
4724 		if (info->intercept == x86_intercept_outs ||
4725 		    info->intercept == x86_intercept_ins)
4726 			exit_info |= SVM_IOIO_STR_MASK;
4727 
4728 		if (info->rep_prefix)
4729 			exit_info |= SVM_IOIO_REP_MASK;
4730 
4731 		bytes = min(bytes, 4u);
4732 
4733 		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4734 
4735 		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4736 
4737 		vmcb->control.exit_info_1 = exit_info;
4738 		vmcb->control.exit_info_2 = info->next_rip;
4739 
4740 		break;
4741 	}
4742 	default:
4743 		break;
4744 	}
4745 
4746 	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4747 	if (static_cpu_has(X86_FEATURE_NRIPS))
4748 		vmcb->control.next_rip  = info->next_rip;
4749 	vmcb->control.exit_code = icpt_info.exit_code;
4750 	vmexit = nested_svm_exit_handled(svm);
4751 
4752 	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4753 					   : X86EMUL_CONTINUE;
4754 
4755 out:
4756 	return ret;
4757 }
4758 
svm_handle_exit_irqoff(struct kvm_vcpu * vcpu)4759 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4760 {
4761 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR)
4762 		vcpu->arch.at_instruction_boundary = true;
4763 }
4764 
svm_setup_mce(struct kvm_vcpu * vcpu)4765 static void svm_setup_mce(struct kvm_vcpu *vcpu)
4766 {
4767 	/* [63:9] are reserved. */
4768 	vcpu->arch.mcg_cap &= 0x1ff;
4769 }
4770 
4771 #ifdef CONFIG_KVM_SMM
svm_smi_blocked(struct kvm_vcpu * vcpu)4772 bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4773 {
4774 	struct vcpu_svm *svm = to_svm(vcpu);
4775 
4776 	/* Per APM Vol.2 15.22.2 "Response to SMI" */
4777 	if (!gif_set(svm))
4778 		return true;
4779 
4780 	return is_smm(vcpu);
4781 }
4782 
svm_smi_allowed(struct kvm_vcpu * vcpu,bool for_injection)4783 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4784 {
4785 	struct vcpu_svm *svm = to_svm(vcpu);
4786 	if (svm->nested.nested_run_pending)
4787 		return -EBUSY;
4788 
4789 	if (svm_smi_blocked(vcpu))
4790 		return 0;
4791 
4792 	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4793 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4794 		return -EBUSY;
4795 
4796 	return 1;
4797 }
4798 
svm_enter_smm(struct kvm_vcpu * vcpu,union kvm_smram * smram)4799 static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
4800 {
4801 	struct vcpu_svm *svm = to_svm(vcpu);
4802 	struct kvm_host_map map_save;
4803 	int ret;
4804 
4805 	if (!is_guest_mode(vcpu))
4806 		return 0;
4807 
4808 	/*
4809 	 * 32-bit SMRAM format doesn't preserve EFER and SVM state.  Userspace is
4810 	 * responsible for ensuring nested SVM and SMIs are mutually exclusive.
4811 	 */
4812 
4813 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
4814 		return 1;
4815 
4816 	smram->smram64.svm_guest_flag = 1;
4817 	smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa;
4818 
4819 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4820 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4821 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4822 
4823 	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
4824 	if (ret)
4825 		return ret;
4826 
4827 	/*
4828 	 * KVM uses VMCB01 to store L1 host state while L2 runs but
4829 	 * VMCB01 is going to be used during SMM and thus the state will
4830 	 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4831 	 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4832 	 * format of the area is identical to guest save area offsetted
4833 	 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4834 	 * within 'struct vmcb'). Note: HSAVE area may also be used by
4835 	 * L1 hypervisor to save additional host context (e.g. KVM does
4836 	 * that, see svm_prepare_switch_to_guest()) which must be
4837 	 * preserved.
4838 	 */
4839 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4840 		return 1;
4841 
4842 	BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4843 
4844 	svm_copy_vmrun_state(map_save.hva + 0x400,
4845 			     &svm->vmcb01.ptr->save);
4846 
4847 	kvm_vcpu_unmap(vcpu, &map_save);
4848 	return 0;
4849 }
4850 
svm_leave_smm(struct kvm_vcpu * vcpu,const union kvm_smram * smram)4851 static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
4852 {
4853 	struct vcpu_svm *svm = to_svm(vcpu);
4854 	struct kvm_host_map map, map_save;
4855 	struct vmcb *vmcb12;
4856 	int ret;
4857 
4858 	const struct kvm_smram_state_64 *smram64 = &smram->smram64;
4859 
4860 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
4861 		return 0;
4862 
4863 	/* Non-zero if SMI arrived while vCPU was in guest mode. */
4864 	if (!smram64->svm_guest_flag)
4865 		return 0;
4866 
4867 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SVM))
4868 		return 1;
4869 
4870 	if (!(smram64->efer & EFER_SVME))
4871 		return 1;
4872 
4873 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map))
4874 		return 1;
4875 
4876 	ret = 1;
4877 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4878 		goto unmap_map;
4879 
4880 	if (svm_allocate_nested(svm))
4881 		goto unmap_save;
4882 
4883 	/*
4884 	 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4885 	 * used during SMM (see svm_enter_smm())
4886 	 */
4887 
4888 	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4889 
4890 	/*
4891 	 * Enter the nested guest now
4892 	 */
4893 
4894 	vmcb_mark_all_dirty(svm->vmcb01.ptr);
4895 
4896 	vmcb12 = map.hva;
4897 	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
4898 	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
4899 	ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false);
4900 
4901 	if (ret)
4902 		goto unmap_save;
4903 
4904 	svm->nested.nested_run_pending = 1;
4905 
4906 unmap_save:
4907 	kvm_vcpu_unmap(vcpu, &map_save);
4908 unmap_map:
4909 	kvm_vcpu_unmap(vcpu, &map);
4910 	return ret;
4911 }
4912 
svm_enable_smi_window(struct kvm_vcpu * vcpu)4913 static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4914 {
4915 	struct vcpu_svm *svm = to_svm(vcpu);
4916 
4917 	if (!gif_set(svm)) {
4918 		if (vgif)
4919 			svm_set_intercept(svm, INTERCEPT_STGI);
4920 		/* STGI will cause a vm exit */
4921 	} else {
4922 		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4923 	}
4924 }
4925 #endif
4926 
svm_check_emulate_instruction(struct kvm_vcpu * vcpu,int emul_type,void * insn,int insn_len)4927 static int svm_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
4928 					 void *insn, int insn_len)
4929 {
4930 	struct vcpu_svm *svm = to_svm(vcpu);
4931 	bool smep, smap, is_user;
4932 	u64 error_code;
4933 
4934 	/* Check that emulation is possible during event vectoring */
4935 	if ((svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK) &&
4936 	    !kvm_can_emulate_event_vectoring(emul_type))
4937 		return X86EMUL_UNHANDLEABLE_VECTORING;
4938 
4939 	/* Emulation is always possible when KVM has access to all guest state. */
4940 	if (!sev_guest(vcpu->kvm))
4941 		return X86EMUL_CONTINUE;
4942 
4943 	/* #UD and #GP should never be intercepted for SEV guests. */
4944 	WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD |
4945 				  EMULTYPE_TRAP_UD_FORCED |
4946 				  EMULTYPE_VMWARE_GP));
4947 
4948 	/*
4949 	 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4950 	 * to guest register state.
4951 	 */
4952 	if (sev_es_guest(vcpu->kvm))
4953 		return X86EMUL_RETRY_INSTR;
4954 
4955 	/*
4956 	 * Emulation is possible if the instruction is already decoded, e.g.
4957 	 * when completing I/O after returning from userspace.
4958 	 */
4959 	if (emul_type & EMULTYPE_NO_DECODE)
4960 		return X86EMUL_CONTINUE;
4961 
4962 	/*
4963 	 * Emulation is possible for SEV guests if and only if a prefilled
4964 	 * buffer containing the bytes of the intercepted instruction is
4965 	 * available. SEV guest memory is encrypted with a guest specific key
4966 	 * and cannot be decrypted by KVM, i.e. KVM would read ciphertext and
4967 	 * decode garbage.
4968 	 *
4969 	 * If KVM is NOT trying to simply skip an instruction, inject #UD if
4970 	 * KVM reached this point without an instruction buffer.  In practice,
4971 	 * this path should never be hit by a well-behaved guest, e.g. KVM
4972 	 * doesn't intercept #UD or #GP for SEV guests, but this path is still
4973 	 * theoretically reachable, e.g. via unaccelerated fault-like AVIC
4974 	 * access, and needs to be handled by KVM to avoid putting the guest
4975 	 * into an infinite loop.   Injecting #UD is somewhat arbitrary, but
4976 	 * its the least awful option given lack of insight into the guest.
4977 	 *
4978 	 * If KVM is trying to skip an instruction, simply resume the guest.
4979 	 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM
4980 	 * will attempt to re-inject the INT3/INTO and skip the instruction.
4981 	 * In that scenario, retrying the INT3/INTO and hoping the guest will
4982 	 * make forward progress is the only option that has a chance of
4983 	 * success (and in practice it will work the vast majority of the time).
4984 	 */
4985 	if (unlikely(!insn)) {
4986 		if (emul_type & EMULTYPE_SKIP)
4987 			return X86EMUL_UNHANDLEABLE;
4988 
4989 		kvm_queue_exception(vcpu, UD_VECTOR);
4990 		return X86EMUL_PROPAGATE_FAULT;
4991 	}
4992 
4993 	/*
4994 	 * Emulate for SEV guests if the insn buffer is not empty.  The buffer
4995 	 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4996 	 * the faulting instruction because the code fetch itself faulted, e.g.
4997 	 * the guest attempted to fetch from emulated MMIO or a guest page
4998 	 * table used to translate CS:RIP resides in emulated MMIO.
4999 	 */
5000 	if (likely(insn_len))
5001 		return X86EMUL_CONTINUE;
5002 
5003 	/*
5004 	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
5005 	 *
5006 	 * Errata:
5007 	 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
5008 	 * possible that CPU microcode implementing DecodeAssist will fail to
5009 	 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
5010 	 * be '0'.  This happens because microcode reads CS:RIP using a _data_
5011 	 * loap uop with CPL=0 privileges.  If the load hits a SMAP #PF, ucode
5012 	 * gives up and does not fill the instruction bytes buffer.
5013 	 *
5014 	 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
5015 	 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
5016 	 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
5017 	 * GuestIntrBytes field of the VMCB.
5018 	 *
5019 	 * This does _not_ mean that the erratum has been encountered, as the
5020 	 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
5021 	 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
5022 	 * encountered a reserved/not-present #PF.
5023 	 *
5024 	 * To hit the erratum, the following conditions must be true:
5025 	 *    1. CR4.SMAP=1 (obviously).
5026 	 *    2. CR4.SMEP=0 || CPL=3.  If SMEP=1 and CPL<3, the erratum cannot
5027 	 *       have been hit as the guest would have encountered a SMEP
5028 	 *       violation #PF, not a #NPF.
5029 	 *    3. The #NPF is not due to a code fetch, in which case failure to
5030 	 *       retrieve the instruction bytes is legitimate (see abvoe).
5031 	 *
5032 	 * In addition, don't apply the erratum workaround if the #NPF occurred
5033 	 * while translating guest page tables (see below).
5034 	 */
5035 	error_code = svm->vmcb->control.exit_info_1;
5036 	if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK))
5037 		goto resume_guest;
5038 
5039 	smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP);
5040 	smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP);
5041 	is_user = svm_get_cpl(vcpu) == 3;
5042 	if (smap && (!smep || is_user)) {
5043 		pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n");
5044 
5045 		/*
5046 		 * If the fault occurred in userspace, arbitrarily inject #GP
5047 		 * to avoid killing the guest and to hopefully avoid confusing
5048 		 * the guest kernel too much, e.g. injecting #PF would not be
5049 		 * coherent with respect to the guest's page tables.  Request
5050 		 * triple fault if the fault occurred in the kernel as there's
5051 		 * no fault that KVM can inject without confusing the guest.
5052 		 * In practice, the triple fault is moot as no sane SEV kernel
5053 		 * will execute from user memory while also running with SMAP=1.
5054 		 */
5055 		if (is_user)
5056 			kvm_inject_gp(vcpu, 0);
5057 		else
5058 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5059 		return X86EMUL_PROPAGATE_FAULT;
5060 	}
5061 
5062 resume_guest:
5063 	/*
5064 	 * If the erratum was not hit, simply resume the guest and let it fault
5065 	 * again.  While awful, e.g. the vCPU may get stuck in an infinite loop
5066 	 * if the fault is at CPL=0, it's the lesser of all evils.  Exiting to
5067 	 * userspace will kill the guest, and letting the emulator read garbage
5068 	 * will yield random behavior and potentially corrupt the guest.
5069 	 *
5070 	 * Simply resuming the guest is technically not a violation of the SEV
5071 	 * architecture.  AMD's APM states that all code fetches and page table
5072 	 * accesses for SEV guest are encrypted, regardless of the C-Bit.  The
5073 	 * APM also states that encrypted accesses to MMIO are "ignored", but
5074 	 * doesn't explicitly define "ignored", i.e. doing nothing and letting
5075 	 * the guest spin is technically "ignoring" the access.
5076 	 */
5077 	return X86EMUL_RETRY_INSTR;
5078 }
5079 
svm_apic_init_signal_blocked(struct kvm_vcpu * vcpu)5080 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
5081 {
5082 	struct vcpu_svm *svm = to_svm(vcpu);
5083 
5084 	return !gif_set(svm);
5085 }
5086 
svm_vcpu_deliver_sipi_vector(struct kvm_vcpu * vcpu,u8 vector)5087 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
5088 {
5089 	if (!sev_es_guest(vcpu->kvm))
5090 		return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
5091 
5092 	sev_vcpu_deliver_sipi_vector(vcpu, vector);
5093 }
5094 
svm_vm_destroy(struct kvm * kvm)5095 static void svm_vm_destroy(struct kvm *kvm)
5096 {
5097 	avic_vm_destroy(kvm);
5098 	sev_vm_destroy(kvm);
5099 
5100 	svm_srso_vm_destroy();
5101 }
5102 
svm_vm_init(struct kvm * kvm)5103 static int svm_vm_init(struct kvm *kvm)
5104 {
5105 	int type = kvm->arch.vm_type;
5106 
5107 	if (type != KVM_X86_DEFAULT_VM &&
5108 	    type != KVM_X86_SW_PROTECTED_VM) {
5109 		kvm->arch.has_protected_state =
5110 			(type == KVM_X86_SEV_ES_VM || type == KVM_X86_SNP_VM);
5111 		to_kvm_sev_info(kvm)->need_init = true;
5112 
5113 		kvm->arch.has_private_mem = (type == KVM_X86_SNP_VM);
5114 		kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem;
5115 	}
5116 
5117 	if (!pause_filter_count || !pause_filter_thresh)
5118 		kvm->arch.pause_in_guest = true;
5119 
5120 	if (enable_apicv) {
5121 		int ret = avic_vm_init(kvm);
5122 		if (ret)
5123 			return ret;
5124 	}
5125 
5126 	svm_srso_vm_init();
5127 	return 0;
5128 }
5129 
svm_alloc_apic_backing_page(struct kvm_vcpu * vcpu)5130 static void *svm_alloc_apic_backing_page(struct kvm_vcpu *vcpu)
5131 {
5132 	struct page *page = snp_safe_alloc_page();
5133 
5134 	if (!page)
5135 		return NULL;
5136 
5137 	return page_address(page);
5138 }
5139 
5140 static struct kvm_x86_ops svm_x86_ops __initdata = {
5141 	.name = KBUILD_MODNAME,
5142 
5143 	.check_processor_compatibility = svm_check_processor_compat,
5144 
5145 	.hardware_unsetup = svm_hardware_unsetup,
5146 	.enable_virtualization_cpu = svm_enable_virtualization_cpu,
5147 	.disable_virtualization_cpu = svm_disable_virtualization_cpu,
5148 	.emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu,
5149 	.has_emulated_msr = svm_has_emulated_msr,
5150 
5151 	.vcpu_create = svm_vcpu_create,
5152 	.vcpu_free = svm_vcpu_free,
5153 	.vcpu_reset = svm_vcpu_reset,
5154 
5155 	.vm_size = sizeof(struct kvm_svm),
5156 	.vm_init = svm_vm_init,
5157 	.vm_destroy = svm_vm_destroy,
5158 
5159 	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
5160 	.vcpu_load = svm_vcpu_load,
5161 	.vcpu_put = svm_vcpu_put,
5162 	.vcpu_blocking = avic_vcpu_blocking,
5163 	.vcpu_unblocking = avic_vcpu_unblocking,
5164 
5165 	.update_exception_bitmap = svm_update_exception_bitmap,
5166 	.get_feature_msr = svm_get_feature_msr,
5167 	.get_msr = svm_get_msr,
5168 	.set_msr = svm_set_msr,
5169 	.get_segment_base = svm_get_segment_base,
5170 	.get_segment = svm_get_segment,
5171 	.set_segment = svm_set_segment,
5172 	.get_cpl = svm_get_cpl,
5173 	.get_cpl_no_cache = svm_get_cpl,
5174 	.get_cs_db_l_bits = svm_get_cs_db_l_bits,
5175 	.is_valid_cr0 = svm_is_valid_cr0,
5176 	.set_cr0 = svm_set_cr0,
5177 	.post_set_cr3 = sev_post_set_cr3,
5178 	.is_valid_cr4 = svm_is_valid_cr4,
5179 	.set_cr4 = svm_set_cr4,
5180 	.set_efer = svm_set_efer,
5181 	.get_idt = svm_get_idt,
5182 	.set_idt = svm_set_idt,
5183 	.get_gdt = svm_get_gdt,
5184 	.set_gdt = svm_set_gdt,
5185 	.set_dr6 = svm_set_dr6,
5186 	.set_dr7 = svm_set_dr7,
5187 	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
5188 	.cache_reg = svm_cache_reg,
5189 	.get_rflags = svm_get_rflags,
5190 	.set_rflags = svm_set_rflags,
5191 	.get_if_flag = svm_get_if_flag,
5192 
5193 	.flush_tlb_all = svm_flush_tlb_all,
5194 	.flush_tlb_current = svm_flush_tlb_current,
5195 	.flush_tlb_gva = svm_flush_tlb_gva,
5196 	.flush_tlb_guest = svm_flush_tlb_asid,
5197 
5198 	.vcpu_pre_run = svm_vcpu_pre_run,
5199 	.vcpu_run = svm_vcpu_run,
5200 	.handle_exit = svm_handle_exit,
5201 	.skip_emulated_instruction = svm_skip_emulated_instruction,
5202 	.update_emulated_instruction = NULL,
5203 	.set_interrupt_shadow = svm_set_interrupt_shadow,
5204 	.get_interrupt_shadow = svm_get_interrupt_shadow,
5205 	.patch_hypercall = svm_patch_hypercall,
5206 	.inject_irq = svm_inject_irq,
5207 	.inject_nmi = svm_inject_nmi,
5208 	.is_vnmi_pending = svm_is_vnmi_pending,
5209 	.set_vnmi_pending = svm_set_vnmi_pending,
5210 	.inject_exception = svm_inject_exception,
5211 	.cancel_injection = svm_cancel_injection,
5212 	.interrupt_allowed = svm_interrupt_allowed,
5213 	.nmi_allowed = svm_nmi_allowed,
5214 	.get_nmi_mask = svm_get_nmi_mask,
5215 	.set_nmi_mask = svm_set_nmi_mask,
5216 	.enable_nmi_window = svm_enable_nmi_window,
5217 	.enable_irq_window = svm_enable_irq_window,
5218 	.update_cr8_intercept = svm_update_cr8_intercept,
5219 
5220 	.x2apic_icr_is_split = true,
5221 	.set_virtual_apic_mode = avic_refresh_virtual_apic_mode,
5222 	.refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl,
5223 	.apicv_post_state_restore = avic_apicv_post_state_restore,
5224 	.required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS,
5225 
5226 	.get_exit_info = svm_get_exit_info,
5227 	.get_entry_info = svm_get_entry_info,
5228 
5229 	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
5230 
5231 	.has_wbinvd_exit = svm_has_wbinvd_exit,
5232 
5233 	.get_l2_tsc_offset = svm_get_l2_tsc_offset,
5234 	.get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
5235 	.write_tsc_offset = svm_write_tsc_offset,
5236 	.write_tsc_multiplier = svm_write_tsc_multiplier,
5237 
5238 	.load_mmu_pgd = svm_load_mmu_pgd,
5239 
5240 	.check_intercept = svm_check_intercept,
5241 	.handle_exit_irqoff = svm_handle_exit_irqoff,
5242 
5243 	.nested_ops = &svm_nested_ops,
5244 
5245 	.deliver_interrupt = svm_deliver_interrupt,
5246 	.pi_update_irte = avic_pi_update_irte,
5247 	.setup_mce = svm_setup_mce,
5248 
5249 #ifdef CONFIG_KVM_SMM
5250 	.smi_allowed = svm_smi_allowed,
5251 	.enter_smm = svm_enter_smm,
5252 	.leave_smm = svm_leave_smm,
5253 	.enable_smi_window = svm_enable_smi_window,
5254 #endif
5255 
5256 #ifdef CONFIG_KVM_AMD_SEV
5257 	.dev_get_attr = sev_dev_get_attr,
5258 	.mem_enc_ioctl = sev_mem_enc_ioctl,
5259 	.mem_enc_register_region = sev_mem_enc_register_region,
5260 	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
5261 	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
5262 
5263 	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
5264 	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
5265 #endif
5266 	.check_emulate_instruction = svm_check_emulate_instruction,
5267 
5268 	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
5269 
5270 	.msr_filter_changed = svm_msr_filter_changed,
5271 	.complete_emulated_msr = svm_complete_emulated_msr,
5272 
5273 	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
5274 	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
5275 	.alloc_apic_backing_page = svm_alloc_apic_backing_page,
5276 
5277 	.gmem_prepare = sev_gmem_prepare,
5278 	.gmem_invalidate = sev_gmem_invalidate,
5279 	.private_max_mapping_level = sev_private_max_mapping_level,
5280 };
5281 
5282 /*
5283  * The default MMIO mask is a single bit (excluding the present bit),
5284  * which could conflict with the memory encryption bit. Check for
5285  * memory encryption support and override the default MMIO mask if
5286  * memory encryption is enabled.
5287  */
svm_adjust_mmio_mask(void)5288 static __init void svm_adjust_mmio_mask(void)
5289 {
5290 	unsigned int enc_bit, mask_bit;
5291 	u64 msr, mask;
5292 
5293 	/* If there is no memory encryption support, use existing mask */
5294 	if (cpuid_eax(0x80000000) < 0x8000001f)
5295 		return;
5296 
5297 	/* If memory encryption is not enabled, use existing mask */
5298 	rdmsrl(MSR_AMD64_SYSCFG, msr);
5299 	if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
5300 		return;
5301 
5302 	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
5303 	mask_bit = boot_cpu_data.x86_phys_bits;
5304 
5305 	/* Increment the mask bit if it is the same as the encryption bit */
5306 	if (enc_bit == mask_bit)
5307 		mask_bit++;
5308 
5309 	/*
5310 	 * If the mask bit location is below 52, then some bits above the
5311 	 * physical addressing limit will always be reserved, so use the
5312 	 * rsvd_bits() function to generate the mask. This mask, along with
5313 	 * the present bit, will be used to generate a page fault with
5314 	 * PFER.RSV = 1.
5315 	 *
5316 	 * If the mask bit location is 52 (or above), then clear the mask.
5317 	 */
5318 	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
5319 
5320 	kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
5321 }
5322 
svm_set_cpu_caps(void)5323 static __init void svm_set_cpu_caps(void)
5324 {
5325 	kvm_set_cpu_caps();
5326 
5327 	kvm_caps.supported_perf_cap = 0;
5328 	kvm_caps.supported_xss = 0;
5329 
5330 	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
5331 	if (nested) {
5332 		kvm_cpu_cap_set(X86_FEATURE_SVM);
5333 		kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN);
5334 
5335 		/*
5336 		 * KVM currently flushes TLBs on *every* nested SVM transition,
5337 		 * and so for all intents and purposes KVM supports flushing by
5338 		 * ASID, i.e. KVM is guaranteed to honor every L1 ASID flush.
5339 		 */
5340 		kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID);
5341 
5342 		if (nrips)
5343 			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
5344 
5345 		if (npt_enabled)
5346 			kvm_cpu_cap_set(X86_FEATURE_NPT);
5347 
5348 		if (tsc_scaling)
5349 			kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
5350 
5351 		if (vls)
5352 			kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD);
5353 		if (lbrv)
5354 			kvm_cpu_cap_set(X86_FEATURE_LBRV);
5355 
5356 		if (boot_cpu_has(X86_FEATURE_PAUSEFILTER))
5357 			kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER);
5358 
5359 		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
5360 			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
5361 
5362 		if (vgif)
5363 			kvm_cpu_cap_set(X86_FEATURE_VGIF);
5364 
5365 		if (vnmi)
5366 			kvm_cpu_cap_set(X86_FEATURE_VNMI);
5367 
5368 		/* Nested VM can receive #VMEXIT instead of triggering #GP */
5369 		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
5370 	}
5371 
5372 	/* CPUID 0x80000008 */
5373 	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5374 	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
5375 		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
5376 
5377 	if (enable_pmu) {
5378 		/*
5379 		 * Enumerate support for PERFCTR_CORE if and only if KVM has
5380 		 * access to enough counters to virtualize "core" support,
5381 		 * otherwise limit vPMU support to the legacy number of counters.
5382 		 */
5383 		if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE)
5384 			kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS,
5385 							  kvm_pmu_cap.num_counters_gp);
5386 		else
5387 			kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE);
5388 
5389 		if (kvm_pmu_cap.version != 2 ||
5390 		    !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
5391 			kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2);
5392 	}
5393 
5394 	/* CPUID 0x8000001F (SME/SEV features) */
5395 	sev_set_cpu_caps();
5396 
5397 	/* Don't advertise Bus Lock Detect to guest if SVM support is absent */
5398 	kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT);
5399 }
5400 
svm_hardware_setup(void)5401 static __init int svm_hardware_setup(void)
5402 {
5403 	int cpu;
5404 	struct page *iopm_pages;
5405 	void *iopm_va;
5406 	int r;
5407 	unsigned int order = get_order(IOPM_SIZE);
5408 
5409 	/*
5410 	 * NX is required for shadow paging and for NPT if the NX huge pages
5411 	 * mitigation is enabled.
5412 	 */
5413 	if (!boot_cpu_has(X86_FEATURE_NX)) {
5414 		pr_err_ratelimited("NX (Execute Disable) not supported\n");
5415 		return -EOPNOTSUPP;
5416 	}
5417 	kvm_enable_efer_bits(EFER_NX);
5418 
5419 	iopm_pages = alloc_pages(GFP_KERNEL, order);
5420 
5421 	if (!iopm_pages)
5422 		return -ENOMEM;
5423 
5424 	iopm_va = page_address(iopm_pages);
5425 	memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
5426 	iopm_base = __sme_page_pa(iopm_pages);
5427 
5428 	init_msrpm_offsets();
5429 
5430 	kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
5431 				     XFEATURE_MASK_BNDCSR);
5432 
5433 	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
5434 		kvm_enable_efer_bits(EFER_FFXSR);
5435 
5436 	if (tsc_scaling) {
5437 		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
5438 			tsc_scaling = false;
5439 		} else {
5440 			pr_info("TSC scaling supported\n");
5441 			kvm_caps.has_tsc_control = true;
5442 		}
5443 	}
5444 	kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX;
5445 	kvm_caps.tsc_scaling_ratio_frac_bits = 32;
5446 
5447 	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
5448 
5449 	if (boot_cpu_has(X86_FEATURE_AUTOIBRS))
5450 		kvm_enable_efer_bits(EFER_AUTOIBRS);
5451 
5452 	/* Check for pause filtering support */
5453 	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
5454 		pause_filter_count = 0;
5455 		pause_filter_thresh = 0;
5456 	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
5457 		pause_filter_thresh = 0;
5458 	}
5459 
5460 	if (nested) {
5461 		pr_info("Nested Virtualization enabled\n");
5462 		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
5463 	}
5464 
5465 	/*
5466 	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
5467 	 * NPT isn't supported if the host is using 2-level paging since host
5468 	 * CR4 is unchanged on VMRUN.
5469 	 */
5470 	if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
5471 		npt_enabled = false;
5472 
5473 	if (!boot_cpu_has(X86_FEATURE_NPT))
5474 		npt_enabled = false;
5475 
5476 	/* Force VM NPT level equal to the host's paging level */
5477 	kvm_configure_mmu(npt_enabled, get_npt_level(),
5478 			  get_npt_level(), PG_LEVEL_1G);
5479 	pr_info("Nested Paging %s\n", str_enabled_disabled(npt_enabled));
5480 
5481 	/* Setup shadow_me_value and shadow_me_mask */
5482 	kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask);
5483 
5484 	svm_adjust_mmio_mask();
5485 
5486 	nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
5487 
5488 	if (lbrv) {
5489 		if (!boot_cpu_has(X86_FEATURE_LBRV))
5490 			lbrv = false;
5491 		else
5492 			pr_info("LBR virtualization supported\n");
5493 	}
5494 	/*
5495 	 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which
5496 	 * may be modified by svm_adjust_mmio_mask()), as well as nrips.
5497 	 */
5498 	sev_hardware_setup();
5499 
5500 	svm_hv_hardware_setup();
5501 
5502 	for_each_possible_cpu(cpu) {
5503 		r = svm_cpu_init(cpu);
5504 		if (r)
5505 			goto err;
5506 	}
5507 
5508 	enable_apicv = avic = avic && avic_hardware_setup();
5509 
5510 	if (!enable_apicv) {
5511 		svm_x86_ops.vcpu_blocking = NULL;
5512 		svm_x86_ops.vcpu_unblocking = NULL;
5513 		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
5514 	} else if (!x2avic_enabled) {
5515 		svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true;
5516 	}
5517 
5518 	if (vls) {
5519 		if (!npt_enabled ||
5520 		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
5521 		    !IS_ENABLED(CONFIG_X86_64)) {
5522 			vls = false;
5523 		} else {
5524 			pr_info("Virtual VMLOAD VMSAVE supported\n");
5525 		}
5526 	}
5527 
5528 	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
5529 		svm_gp_erratum_intercept = false;
5530 
5531 	if (vgif) {
5532 		if (!boot_cpu_has(X86_FEATURE_VGIF))
5533 			vgif = false;
5534 		else
5535 			pr_info("Virtual GIF supported\n");
5536 	}
5537 
5538 	vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI);
5539 	if (vnmi)
5540 		pr_info("Virtual NMI enabled\n");
5541 
5542 	if (!vnmi) {
5543 		svm_x86_ops.is_vnmi_pending = NULL;
5544 		svm_x86_ops.set_vnmi_pending = NULL;
5545 	}
5546 
5547 	if (!enable_pmu)
5548 		pr_info("PMU virtualization is disabled\n");
5549 
5550 	svm_set_cpu_caps();
5551 
5552 	/*
5553 	 * It seems that on AMD processors PTE's accessed bit is
5554 	 * being set by the CPU hardware before the NPF vmexit.
5555 	 * This is not expected behaviour and our tests fail because
5556 	 * of it.
5557 	 * A workaround here is to disable support for
5558 	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
5559 	 * In this case userspace can know if there is support using
5560 	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
5561 	 * it
5562 	 * If future AMD CPU models change the behaviour described above,
5563 	 * this variable can be changed accordingly
5564 	 */
5565 	allow_smaller_maxphyaddr = !npt_enabled;
5566 
5567 	return 0;
5568 
5569 err:
5570 	svm_hardware_unsetup();
5571 	return r;
5572 }
5573 
5574 
5575 static struct kvm_x86_init_ops svm_init_ops __initdata = {
5576 	.hardware_setup = svm_hardware_setup,
5577 
5578 	.runtime_ops = &svm_x86_ops,
5579 	.pmu_ops = &amd_pmu_ops,
5580 };
5581 
__svm_exit(void)5582 static void __svm_exit(void)
5583 {
5584 	kvm_x86_vendor_exit();
5585 }
5586 
svm_init(void)5587 static int __init svm_init(void)
5588 {
5589 	int r;
5590 
5591 	__unused_size_checks();
5592 
5593 	if (!kvm_is_svm_supported())
5594 		return -EOPNOTSUPP;
5595 
5596 	r = kvm_x86_vendor_init(&svm_init_ops);
5597 	if (r)
5598 		return r;
5599 
5600 	/*
5601 	 * Common KVM initialization _must_ come last, after this, /dev/kvm is
5602 	 * exposed to userspace!
5603 	 */
5604 	r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm),
5605 		     THIS_MODULE);
5606 	if (r)
5607 		goto err_kvm_init;
5608 
5609 	return 0;
5610 
5611 err_kvm_init:
5612 	__svm_exit();
5613 	return r;
5614 }
5615 
svm_exit(void)5616 static void __exit svm_exit(void)
5617 {
5618 	kvm_exit();
5619 	__svm_exit();
5620 }
5621 
5622 module_init(svm_init)
5623 module_exit(svm_exit)
5624