xref: /kvm-unit-tests/x86/vmx.c (revision a299895b7abb54e7ba6bb4108f202acbb484ac65)
1 /*
2  * x86/vmx.c : Framework for testing nested virtualization
3  *	This is a framework to test nested VMX for KVM, which
4  * 	started as a project of GSoC 2013. All test cases should
5  *	be located in x86/vmx_tests.c and framework related
6  *	functions should be in this file.
7  *
8  * How to write test cases?
9  *	Add callbacks of test suite in variant "vmx_tests". You can
10  *	write:
11  *		1. init function used for initializing test suite
12  *		2. main function for codes running in L2 guest,
13  *		3. exit_handler to handle vmexit of L2 to L1
14  *		4. syscall handler to handle L2 syscall vmexit
15  *		5. vmenter fail handler to handle direct failure of vmenter
16  *		6. guest_regs is loaded when vmenter and saved when
17  *			vmexit, you can read and set it in exit_handler
18  *	If no special function is needed for a test suite, use
19  *	coressponding basic_* functions as callback. More handlers
20  *	can be added to "vmx_tests", see details of "struct vmx_test"
21  *	and function test_run().
22  *
23  * Currently, vmx test framework only set up one VCPU and one
24  * concurrent guest test environment with same paging for L2 and
25  * L1. For usage of EPT, only 1:1 mapped paging is used from VFN
26  * to PFN.
27  *
28  * Author : Arthur Chunqi Li <yzt356@gmail.com>
29  */
30 
31 #include "libcflat.h"
32 #include "processor.h"
33 #include "alloc_page.h"
34 #include "vm.h"
35 #include "desc.h"
36 #include "vmx.h"
37 #include "msr.h"
38 #include "smp.h"
39 #include "apic.h"
40 
41 u64 *bsp_vmxon_region;
42 struct vmcs *vmcs_root;
43 u32 vpid_cnt;
44 void *guest_stack, *guest_syscall_stack;
45 u32 ctrl_pin, ctrl_enter, ctrl_exit, ctrl_cpu[2];
46 struct regs regs;
47 
48 struct vmx_test *current;
49 
50 #define MAX_TEST_TEARDOWN_STEPS 10
51 
52 struct test_teardown_step {
53 	test_teardown_func func;
54 	void *data;
55 };
56 
57 static int teardown_count;
58 static struct test_teardown_step teardown_steps[MAX_TEST_TEARDOWN_STEPS];
59 
60 static test_guest_func v2_guest_main;
61 
62 u64 hypercall_field;
63 bool launched;
64 static int matched;
65 static int guest_finished;
66 static int in_guest;
67 
68 union vmx_basic basic;
69 union vmx_ctrl_msr ctrl_pin_rev;
70 union vmx_ctrl_msr ctrl_cpu_rev[2];
71 union vmx_ctrl_msr ctrl_exit_rev;
72 union vmx_ctrl_msr ctrl_enter_rev;
73 union vmx_ept_vpid  ept_vpid;
74 
75 extern struct descriptor_table_ptr gdt64_desc;
76 extern struct descriptor_table_ptr idt_descr;
77 extern struct descriptor_table_ptr tss_descr;
78 extern void *vmx_return;
79 extern void *entry_sysenter;
80 extern void *guest_entry;
81 
82 static volatile u32 stage;
83 
84 static jmp_buf abort_target;
85 
86 struct vmcs_field {
87 	u64 mask;
88 	u64 encoding;
89 };
90 
91 #define MASK(_bits) GENMASK_ULL((_bits) - 1, 0)
92 #define MASK_NATURAL MASK(sizeof(unsigned long) * 8)
93 
94 static struct vmcs_field vmcs_fields[] = {
95 	{ MASK(16), VPID },
96 	{ MASK(16), PINV },
97 	{ MASK(16), EPTP_IDX },
98 
99 	{ MASK(16), GUEST_SEL_ES },
100 	{ MASK(16), GUEST_SEL_CS },
101 	{ MASK(16), GUEST_SEL_SS },
102 	{ MASK(16), GUEST_SEL_DS },
103 	{ MASK(16), GUEST_SEL_FS },
104 	{ MASK(16), GUEST_SEL_GS },
105 	{ MASK(16), GUEST_SEL_LDTR },
106 	{ MASK(16), GUEST_SEL_TR },
107 	{ MASK(16), GUEST_INT_STATUS },
108 
109 	{ MASK(16), HOST_SEL_ES },
110 	{ MASK(16), HOST_SEL_CS },
111 	{ MASK(16), HOST_SEL_SS },
112 	{ MASK(16), HOST_SEL_DS },
113 	{ MASK(16), HOST_SEL_FS },
114 	{ MASK(16), HOST_SEL_GS },
115 	{ MASK(16), HOST_SEL_TR },
116 
117 	{ MASK(64), IO_BITMAP_A },
118 	{ MASK(64), IO_BITMAP_B },
119 	{ MASK(64), MSR_BITMAP },
120 	{ MASK(64), EXIT_MSR_ST_ADDR },
121 	{ MASK(64), EXIT_MSR_LD_ADDR },
122 	{ MASK(64), ENTER_MSR_LD_ADDR },
123 	{ MASK(64), VMCS_EXEC_PTR },
124 	{ MASK(64), TSC_OFFSET },
125 	{ MASK(64), APIC_VIRT_ADDR },
126 	{ MASK(64), APIC_ACCS_ADDR },
127 	{ MASK(64), EPTP },
128 
129 	{ MASK(64), INFO_PHYS_ADDR },
130 
131 	{ MASK(64), VMCS_LINK_PTR },
132 	{ MASK(64), GUEST_DEBUGCTL },
133 	{ MASK(64), GUEST_EFER },
134 	{ MASK(64), GUEST_PAT },
135 	{ MASK(64), GUEST_PERF_GLOBAL_CTRL },
136 	{ MASK(64), GUEST_PDPTE },
137 
138 	{ MASK(64), HOST_PAT },
139 	{ MASK(64), HOST_EFER },
140 	{ MASK(64), HOST_PERF_GLOBAL_CTRL },
141 
142 	{ MASK(32), PIN_CONTROLS },
143 	{ MASK(32), CPU_EXEC_CTRL0 },
144 	{ MASK(32), EXC_BITMAP },
145 	{ MASK(32), PF_ERROR_MASK },
146 	{ MASK(32), PF_ERROR_MATCH },
147 	{ MASK(32), CR3_TARGET_COUNT },
148 	{ MASK(32), EXI_CONTROLS },
149 	{ MASK(32), EXI_MSR_ST_CNT },
150 	{ MASK(32), EXI_MSR_LD_CNT },
151 	{ MASK(32), ENT_CONTROLS },
152 	{ MASK(32), ENT_MSR_LD_CNT },
153 	{ MASK(32), ENT_INTR_INFO },
154 	{ MASK(32), ENT_INTR_ERROR },
155 	{ MASK(32), ENT_INST_LEN },
156 	{ MASK(32), TPR_THRESHOLD },
157 	{ MASK(32), CPU_EXEC_CTRL1 },
158 
159 	{ MASK(32), VMX_INST_ERROR },
160 	{ MASK(32), EXI_REASON },
161 	{ MASK(32), EXI_INTR_INFO },
162 	{ MASK(32), EXI_INTR_ERROR },
163 	{ MASK(32), IDT_VECT_INFO },
164 	{ MASK(32), IDT_VECT_ERROR },
165 	{ MASK(32), EXI_INST_LEN },
166 	{ MASK(32), EXI_INST_INFO },
167 
168 	{ MASK(32), GUEST_LIMIT_ES },
169 	{ MASK(32), GUEST_LIMIT_CS },
170 	{ MASK(32), GUEST_LIMIT_SS },
171 	{ MASK(32), GUEST_LIMIT_DS },
172 	{ MASK(32), GUEST_LIMIT_FS },
173 	{ MASK(32), GUEST_LIMIT_GS },
174 	{ MASK(32), GUEST_LIMIT_LDTR },
175 	{ MASK(32), GUEST_LIMIT_TR },
176 	{ MASK(32), GUEST_LIMIT_GDTR },
177 	{ MASK(32), GUEST_LIMIT_IDTR },
178 	{ 0x1d0ff, GUEST_AR_ES },
179 	{ 0x1f0ff, GUEST_AR_CS },
180 	{ 0x1d0ff, GUEST_AR_SS },
181 	{ 0x1d0ff, GUEST_AR_DS },
182 	{ 0x1d0ff, GUEST_AR_FS },
183 	{ 0x1d0ff, GUEST_AR_GS },
184 	{ 0x1d0ff, GUEST_AR_LDTR },
185 	{ 0x1d0ff, GUEST_AR_TR },
186 	{ MASK(32), GUEST_INTR_STATE },
187 	{ MASK(32), GUEST_ACTV_STATE },
188 	{ MASK(32), GUEST_SMBASE },
189 	{ MASK(32), GUEST_SYSENTER_CS },
190 	{ MASK(32), PREEMPT_TIMER_VALUE },
191 
192 	{ MASK(32), HOST_SYSENTER_CS },
193 
194 	{ MASK_NATURAL, CR0_MASK },
195 	{ MASK_NATURAL, CR4_MASK },
196 	{ MASK_NATURAL, CR0_READ_SHADOW },
197 	{ MASK_NATURAL, CR4_READ_SHADOW },
198 	{ MASK_NATURAL, CR3_TARGET_0 },
199 	{ MASK_NATURAL, CR3_TARGET_1 },
200 	{ MASK_NATURAL, CR3_TARGET_2 },
201 	{ MASK_NATURAL, CR3_TARGET_3 },
202 
203 	{ MASK_NATURAL, EXI_QUALIFICATION },
204 	{ MASK_NATURAL, IO_RCX },
205 	{ MASK_NATURAL, IO_RSI },
206 	{ MASK_NATURAL, IO_RDI },
207 	{ MASK_NATURAL, IO_RIP },
208 	{ MASK_NATURAL, GUEST_LINEAR_ADDRESS },
209 
210 	{ MASK_NATURAL, GUEST_CR0 },
211 	{ MASK_NATURAL, GUEST_CR3 },
212 	{ MASK_NATURAL, GUEST_CR4 },
213 	{ MASK_NATURAL, GUEST_BASE_ES },
214 	{ MASK_NATURAL, GUEST_BASE_CS },
215 	{ MASK_NATURAL, GUEST_BASE_SS },
216 	{ MASK_NATURAL, GUEST_BASE_DS },
217 	{ MASK_NATURAL, GUEST_BASE_FS },
218 	{ MASK_NATURAL, GUEST_BASE_GS },
219 	{ MASK_NATURAL, GUEST_BASE_LDTR },
220 	{ MASK_NATURAL, GUEST_BASE_TR },
221 	{ MASK_NATURAL, GUEST_BASE_GDTR },
222 	{ MASK_NATURAL, GUEST_BASE_IDTR },
223 	{ MASK_NATURAL, GUEST_DR7 },
224 	{ MASK_NATURAL, GUEST_RSP },
225 	{ MASK_NATURAL, GUEST_RIP },
226 	{ MASK_NATURAL, GUEST_RFLAGS },
227 	{ MASK_NATURAL, GUEST_PENDING_DEBUG },
228 	{ MASK_NATURAL, GUEST_SYSENTER_ESP },
229 	{ MASK_NATURAL, GUEST_SYSENTER_EIP },
230 
231 	{ MASK_NATURAL, HOST_CR0 },
232 	{ MASK_NATURAL, HOST_CR3 },
233 	{ MASK_NATURAL, HOST_CR4 },
234 	{ MASK_NATURAL, HOST_BASE_FS },
235 	{ MASK_NATURAL, HOST_BASE_GS },
236 	{ MASK_NATURAL, HOST_BASE_TR },
237 	{ MASK_NATURAL, HOST_BASE_GDTR },
238 	{ MASK_NATURAL, HOST_BASE_IDTR },
239 	{ MASK_NATURAL, HOST_SYSENTER_ESP },
240 	{ MASK_NATURAL, HOST_SYSENTER_EIP },
241 	{ MASK_NATURAL, HOST_RSP },
242 	{ MASK_NATURAL, HOST_RIP },
243 };
244 
245 enum vmcs_field_type {
246 	VMCS_FIELD_TYPE_CONTROL = 0,
247 	VMCS_FIELD_TYPE_READ_ONLY_DATA = 1,
248 	VMCS_FIELD_TYPE_GUEST = 2,
249 	VMCS_FIELD_TYPE_HOST = 3,
250 	VMCS_FIELD_TYPES,
251 };
252 
253 static inline int vmcs_field_type(struct vmcs_field *f)
254 {
255 	return (f->encoding >> VMCS_FIELD_TYPE_SHIFT) & 0x3;
256 }
257 
258 static int vmcs_field_readonly(struct vmcs_field *f)
259 {
260 	u64 ia32_vmx_misc;
261 
262 	ia32_vmx_misc = rdmsr(MSR_IA32_VMX_MISC);
263 	return !(ia32_vmx_misc & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS) &&
264 		(vmcs_field_type(f) == VMCS_FIELD_TYPE_READ_ONLY_DATA);
265 }
266 
267 static inline u64 vmcs_field_value(struct vmcs_field *f, u8 cookie)
268 {
269 	u64 value;
270 
271 	/* Incorporate the cookie and the field encoding into the value. */
272 	value = cookie;
273 	value |= (f->encoding << 8);
274 	value |= 0xdeadbeefull << 32;
275 
276 	return value & f->mask;
277 }
278 
279 static void set_vmcs_field(struct vmcs_field *f, u8 cookie)
280 {
281 	vmcs_write(f->encoding, vmcs_field_value(f, cookie));
282 }
283 
284 static bool check_vmcs_field(struct vmcs_field *f, u8 cookie, u32 *max_index)
285 {
286 	u64 expected;
287 	u64 actual;
288 	u32 index;
289 	int ret;
290 
291 	if (f->encoding == VMX_INST_ERROR) {
292 		printf("Skipping volatile field %lx\n", f->encoding);
293 		return true;
294 	}
295 
296 	ret = vmcs_read_checking(f->encoding, &actual);
297 	assert(!(ret & X86_EFLAGS_CF));
298 	/* Skip VMCS fields that aren't recognized by the CPU */
299 	if (ret & X86_EFLAGS_ZF)
300 		return true;
301 
302 	if (max_index) {
303 		index = f->encoding & VMCS_FIELD_INDEX_MASK;
304 		if (index > *max_index)
305 			*max_index = index;
306 	}
307 
308 	if (vmcs_field_readonly(f)) {
309 		printf("Skipping read-only field %lx\n", f->encoding);
310 		return true;
311 	}
312 
313 	expected = vmcs_field_value(f, cookie);
314 	actual &= f->mask;
315 
316 	if (expected == actual)
317 		return true;
318 
319 	printf("FAIL: VMWRITE/VMREAD %lx (expected: %lx, actual: %lx)\n",
320 	       f->encoding, (unsigned long) expected, (unsigned long) actual);
321 
322 	return false;
323 }
324 
325 static void set_all_vmcs_fields(u8 cookie)
326 {
327 	int i;
328 
329 	for (i = 0; i < ARRAY_SIZE(vmcs_fields); i++)
330 		set_vmcs_field(&vmcs_fields[i], cookie);
331 }
332 
333 static bool __check_all_vmcs_fields(u8 cookie, u32 *max_index)
334 {
335 	bool pass = true;
336 	int i;
337 
338 	for (i = 0; i < ARRAY_SIZE(vmcs_fields); i++) {
339 		if (!check_vmcs_field(&vmcs_fields[i], cookie, max_index))
340 			pass = false;
341 	}
342 
343 	return pass;
344 }
345 
346 static bool check_all_vmcs_fields(u8 cookie)
347 {
348 	return __check_all_vmcs_fields(cookie, NULL);
349 }
350 
351 static void test_vmwrite_vmread(void)
352 {
353 	struct vmcs *vmcs = alloc_page();
354 	u32 vmcs_enum_max, max_index = 0;
355 
356 	vmcs->hdr.revision_id = basic.revision;
357 	assert(!vmcs_clear(vmcs));
358 	assert(!make_vmcs_current(vmcs));
359 
360 	set_all_vmcs_fields(0x42);
361 	report(__check_all_vmcs_fields(0x42, &max_index), "VMWRITE/VMREAD");
362 
363 	vmcs_enum_max = rdmsr(MSR_IA32_VMX_VMCS_ENUM) & VMCS_FIELD_INDEX_MASK;
364 	report(vmcs_enum_max >= max_index,
365 	       "VMX_VMCS_ENUM.MAX_INDEX expected at least: %x, actual: %x",
366 	       max_index, vmcs_enum_max);
367 
368 	assert(!vmcs_clear(vmcs));
369 	free_page(vmcs);
370 }
371 
372 static void test_vmcs_high(void)
373 {
374 	struct vmcs *vmcs = alloc_page();
375 
376 	vmcs->hdr.revision_id = basic.revision;
377 	assert(!vmcs_clear(vmcs));
378 	assert(!make_vmcs_current(vmcs));
379 
380 	vmcs_write(TSC_OFFSET, 0x0123456789ABCDEFull);
381 	report(vmcs_read(TSC_OFFSET) == 0x0123456789ABCDEFull,
382 	       "VMREAD TSC_OFFSET after VMWRITE TSC_OFFSET");
383 	report(vmcs_read(TSC_OFFSET_HI) == 0x01234567ull,
384 	       "VMREAD TSC_OFFSET_HI after VMWRITE TSC_OFFSET");
385 	vmcs_write(TSC_OFFSET_HI, 0x76543210ul);
386 	report(vmcs_read(TSC_OFFSET_HI) == 0x76543210ul,
387 	       "VMREAD TSC_OFFSET_HI after VMWRITE TSC_OFFSET_HI");
388 	report(vmcs_read(TSC_OFFSET) == 0x7654321089ABCDEFull,
389 	       "VMREAD TSC_OFFSET after VMWRITE TSC_OFFSET_HI");
390 
391 	assert(!vmcs_clear(vmcs));
392 	free_page(vmcs);
393 }
394 
395 static void test_vmcs_lifecycle(void)
396 {
397 	struct vmcs *vmcs[2] = {};
398 	int i;
399 
400 	for (i = 0; i < ARRAY_SIZE(vmcs); i++) {
401 		vmcs[i] = alloc_page();
402 		vmcs[i]->hdr.revision_id = basic.revision;
403 	}
404 
405 #define VMPTRLD(_i) do { \
406 	assert(_i < ARRAY_SIZE(vmcs)); \
407 	assert(!make_vmcs_current(vmcs[_i])); \
408 	printf("VMPTRLD VMCS%d\n", (_i)); \
409 } while (0)
410 
411 #define VMCLEAR(_i) do { \
412 	assert(_i < ARRAY_SIZE(vmcs)); \
413 	assert(!vmcs_clear(vmcs[_i])); \
414 	printf("VMCLEAR VMCS%d\n", (_i)); \
415 } while (0)
416 
417 	VMCLEAR(0);
418 	VMPTRLD(0);
419 	set_all_vmcs_fields(0);
420 	report(check_all_vmcs_fields(0), "current:VMCS0 active:[VMCS0]");
421 
422 	VMCLEAR(0);
423 	VMPTRLD(0);
424 	report(check_all_vmcs_fields(0), "current:VMCS0 active:[VMCS0]");
425 
426 	VMCLEAR(1);
427 	report(check_all_vmcs_fields(0), "current:VMCS0 active:[VMCS0]");
428 
429 	VMPTRLD(1);
430 	set_all_vmcs_fields(1);
431 	report(check_all_vmcs_fields(1), "current:VMCS1 active:[VMCS0,VCMS1]");
432 
433 	VMPTRLD(0);
434 	report(check_all_vmcs_fields(0), "current:VMCS0 active:[VMCS0,VCMS1]");
435 	VMPTRLD(1);
436 	report(check_all_vmcs_fields(1), "current:VMCS1 active:[VMCS0,VCMS1]");
437 	VMPTRLD(1);
438 	report(check_all_vmcs_fields(1), "current:VMCS1 active:[VMCS0,VCMS1]");
439 
440 	VMCLEAR(0);
441 	report(check_all_vmcs_fields(1), "current:VMCS1 active:[VCMS1]");
442 
443 	/* VMPTRLD should not erase VMWRITEs to the current VMCS */
444 	set_all_vmcs_fields(2);
445 	VMPTRLD(1);
446 	report(check_all_vmcs_fields(2), "current:VMCS1 active:[VCMS1]");
447 
448 	for (i = 0; i < ARRAY_SIZE(vmcs); i++) {
449 		VMCLEAR(i);
450 		free_page(vmcs[i]);
451 	}
452 
453 #undef VMPTRLD
454 #undef VMCLEAR
455 }
456 
457 void vmx_set_test_stage(u32 s)
458 {
459 	barrier();
460 	stage = s;
461 	barrier();
462 }
463 
464 u32 vmx_get_test_stage(void)
465 {
466 	u32 s;
467 
468 	barrier();
469 	s = stage;
470 	barrier();
471 	return s;
472 }
473 
474 void vmx_inc_test_stage(void)
475 {
476 	barrier();
477 	stage++;
478 	barrier();
479 }
480 
481 /* entry_sysenter */
482 asm(
483 	".align	4, 0x90\n\t"
484 	".globl	entry_sysenter\n\t"
485 	"entry_sysenter:\n\t"
486 	SAVE_GPR
487 	"	and	$0xf, %rax\n\t"
488 	"	mov	%rax, %rdi\n\t"
489 	"	call	syscall_handler\n\t"
490 	LOAD_GPR
491 	"	vmresume\n\t"
492 );
493 
494 static void __attribute__((__used__)) syscall_handler(u64 syscall_no)
495 {
496 	if (current->syscall_handler)
497 		current->syscall_handler(syscall_no);
498 }
499 
500 static const char * const exit_reason_descriptions[] = {
501 	[VMX_EXC_NMI]		= "VMX_EXC_NMI",
502 	[VMX_EXTINT]		= "VMX_EXTINT",
503 	[VMX_TRIPLE_FAULT]	= "VMX_TRIPLE_FAULT",
504 	[VMX_INIT]		= "VMX_INIT",
505 	[VMX_SIPI]		= "VMX_SIPI",
506 	[VMX_SMI_IO]		= "VMX_SMI_IO",
507 	[VMX_SMI_OTHER]		= "VMX_SMI_OTHER",
508 	[VMX_INTR_WINDOW]	= "VMX_INTR_WINDOW",
509 	[VMX_NMI_WINDOW]	= "VMX_NMI_WINDOW",
510 	[VMX_TASK_SWITCH]	= "VMX_TASK_SWITCH",
511 	[VMX_CPUID]		= "VMX_CPUID",
512 	[VMX_GETSEC]		= "VMX_GETSEC",
513 	[VMX_HLT]		= "VMX_HLT",
514 	[VMX_INVD]		= "VMX_INVD",
515 	[VMX_INVLPG]		= "VMX_INVLPG",
516 	[VMX_RDPMC]		= "VMX_RDPMC",
517 	[VMX_RDTSC]		= "VMX_RDTSC",
518 	[VMX_RSM]		= "VMX_RSM",
519 	[VMX_VMCALL]		= "VMX_VMCALL",
520 	[VMX_VMCLEAR]		= "VMX_VMCLEAR",
521 	[VMX_VMLAUNCH]		= "VMX_VMLAUNCH",
522 	[VMX_VMPTRLD]		= "VMX_VMPTRLD",
523 	[VMX_VMPTRST]		= "VMX_VMPTRST",
524 	[VMX_VMREAD]		= "VMX_VMREAD",
525 	[VMX_VMRESUME]		= "VMX_VMRESUME",
526 	[VMX_VMWRITE]		= "VMX_VMWRITE",
527 	[VMX_VMXOFF]		= "VMX_VMXOFF",
528 	[VMX_VMXON]		= "VMX_VMXON",
529 	[VMX_CR]		= "VMX_CR",
530 	[VMX_DR]		= "VMX_DR",
531 	[VMX_IO]		= "VMX_IO",
532 	[VMX_RDMSR]		= "VMX_RDMSR",
533 	[VMX_WRMSR]		= "VMX_WRMSR",
534 	[VMX_FAIL_STATE]	= "VMX_FAIL_STATE",
535 	[VMX_FAIL_MSR]		= "VMX_FAIL_MSR",
536 	[VMX_MWAIT]		= "VMX_MWAIT",
537 	[VMX_MTF]		= "VMX_MTF",
538 	[VMX_MONITOR]		= "VMX_MONITOR",
539 	[VMX_PAUSE]		= "VMX_PAUSE",
540 	[VMX_FAIL_MCHECK]	= "VMX_FAIL_MCHECK",
541 	[VMX_TPR_THRESHOLD]	= "VMX_TPR_THRESHOLD",
542 	[VMX_APIC_ACCESS]	= "VMX_APIC_ACCESS",
543 	[VMX_EOI_INDUCED]	= "VMX_EOI_INDUCED",
544 	[VMX_GDTR_IDTR]		= "VMX_GDTR_IDTR",
545 	[VMX_LDTR_TR]		= "VMX_LDTR_TR",
546 	[VMX_EPT_VIOLATION]	= "VMX_EPT_VIOLATION",
547 	[VMX_EPT_MISCONFIG]	= "VMX_EPT_MISCONFIG",
548 	[VMX_INVEPT]		= "VMX_INVEPT",
549 	[VMX_PREEMPT]		= "VMX_PREEMPT",
550 	[VMX_INVVPID]		= "VMX_INVVPID",
551 	[VMX_WBINVD]		= "VMX_WBINVD",
552 	[VMX_XSETBV]		= "VMX_XSETBV",
553 	[VMX_APIC_WRITE]	= "VMX_APIC_WRITE",
554 	[VMX_RDRAND]		= "VMX_RDRAND",
555 	[VMX_INVPCID]		= "VMX_INVPCID",
556 	[VMX_VMFUNC]		= "VMX_VMFUNC",
557 	[VMX_RDSEED]		= "VMX_RDSEED",
558 	[VMX_PML_FULL]		= "VMX_PML_FULL",
559 	[VMX_XSAVES]		= "VMX_XSAVES",
560 	[VMX_XRSTORS]		= "VMX_XRSTORS",
561 };
562 
563 const char *exit_reason_description(u64 reason)
564 {
565 	if (reason >= ARRAY_SIZE(exit_reason_descriptions))
566 		return "(unknown)";
567 	return exit_reason_descriptions[reason] ? : "(unused)";
568 }
569 
570 void print_vmexit_info()
571 {
572 	u64 guest_rip, guest_rsp;
573 	ulong reason = vmcs_read(EXI_REASON) & 0xff;
574 	ulong exit_qual = vmcs_read(EXI_QUALIFICATION);
575 	guest_rip = vmcs_read(GUEST_RIP);
576 	guest_rsp = vmcs_read(GUEST_RSP);
577 	printf("VMEXIT info:\n");
578 	printf("\tvmexit reason = %ld\n", reason);
579 	printf("\texit qualification = %#lx\n", exit_qual);
580 	printf("\tBit 31 of reason = %lx\n", (vmcs_read(EXI_REASON) >> 31) & 1);
581 	printf("\tguest_rip = %#lx\n", guest_rip);
582 	printf("\tRAX=%#lx    RBX=%#lx    RCX=%#lx    RDX=%#lx\n",
583 		regs.rax, regs.rbx, regs.rcx, regs.rdx);
584 	printf("\tRSP=%#lx    RBP=%#lx    RSI=%#lx    RDI=%#lx\n",
585 		guest_rsp, regs.rbp, regs.rsi, regs.rdi);
586 	printf("\tR8 =%#lx    R9 =%#lx    R10=%#lx    R11=%#lx\n",
587 		regs.r8, regs.r9, regs.r10, regs.r11);
588 	printf("\tR12=%#lx    R13=%#lx    R14=%#lx    R15=%#lx\n",
589 		regs.r12, regs.r13, regs.r14, regs.r15);
590 }
591 
592 void
593 print_vmentry_failure_info(struct vmentry_failure *failure) {
594 	if (failure->early) {
595 		printf("Early %s failure: ", failure->instr);
596 		switch (failure->flags & VMX_ENTRY_FLAGS) {
597 		case X86_EFLAGS_CF:
598 			printf("current-VMCS pointer is not valid.\n");
599 			break;
600 		case X86_EFLAGS_ZF:
601 			printf("error number is %ld. See Intel 30.4.\n",
602 			       vmcs_read(VMX_INST_ERROR));
603 			break;
604 		default:
605 			printf("unexpected flags %lx!\n", failure->flags);
606 		}
607 	} else {
608 		u64 reason = vmcs_read(EXI_REASON);
609 		u64 qual = vmcs_read(EXI_QUALIFICATION);
610 
611 		printf("Non-early %s failure (reason=%#lx, qual=%#lx): ",
612 			failure->instr, reason, qual);
613 
614 		switch (reason & 0xff) {
615 		case VMX_FAIL_STATE:
616 			printf("invalid guest state\n");
617 			break;
618 		case VMX_FAIL_MSR:
619 			printf("MSR loading\n");
620 			break;
621 		case VMX_FAIL_MCHECK:
622 			printf("machine-check event\n");
623 			break;
624 		default:
625 			printf("unexpected basic exit reason %ld\n",
626 			       reason & 0xff);
627 		}
628 
629 		if (!(reason & VMX_ENTRY_FAILURE))
630 			printf("\tVMX_ENTRY_FAILURE BIT NOT SET!\n");
631 
632 		if (reason & 0x7fff0000)
633 			printf("\tRESERVED BITS SET!\n");
634 	}
635 }
636 
637 /*
638  * VMCLEAR should ensures all VMCS state is flushed to the VMCS
639  * region in memory.
640  */
641 static void test_vmclear_flushing(void)
642 {
643 	struct vmcs *vmcs[3] = {};
644 	int i;
645 
646 	for (i = 0; i < ARRAY_SIZE(vmcs); i++) {
647 		vmcs[i] = alloc_page();
648 	}
649 
650 	vmcs[0]->hdr.revision_id = basic.revision;
651 	assert(!vmcs_clear(vmcs[0]));
652 	assert(!make_vmcs_current(vmcs[0]));
653 	set_all_vmcs_fields(0x86);
654 
655 	assert(!vmcs_clear(vmcs[0]));
656 	memcpy(vmcs[1], vmcs[0], basic.size);
657 	assert(!make_vmcs_current(vmcs[1]));
658 	report(check_all_vmcs_fields(0x86),
659 	       "test vmclear flush (current VMCS)");
660 
661 	set_all_vmcs_fields(0x87);
662 	assert(!make_vmcs_current(vmcs[0]));
663 	assert(!vmcs_clear(vmcs[1]));
664 	memcpy(vmcs[2], vmcs[1], basic.size);
665 	assert(!make_vmcs_current(vmcs[2]));
666 	report(check_all_vmcs_fields(0x87),
667 	       "test vmclear flush (!current VMCS)");
668 
669 	for (i = 0; i < ARRAY_SIZE(vmcs); i++) {
670 		assert(!vmcs_clear(vmcs[i]));
671 		free_page(vmcs[i]);
672 	}
673 }
674 
675 static void test_vmclear(void)
676 {
677 	struct vmcs *tmp_root;
678 	int width = cpuid_maxphyaddr();
679 
680 	/*
681 	 * Note- The tests below do not necessarily have a
682 	 * valid VMCS, but that's ok since the invalid vmcs
683 	 * is only used for a specific test and is discarded
684 	 * without touching its contents
685 	 */
686 
687 	/* Unaligned page access */
688 	tmp_root = (struct vmcs *)((intptr_t)vmcs_root + 1);
689 	report(vmcs_clear(tmp_root) == 1, "test vmclear with unaligned vmcs");
690 
691 	/* gpa bits beyond physical address width are set*/
692 	tmp_root = (struct vmcs *)((intptr_t)vmcs_root |
693 				   ((u64)1 << (width+1)));
694 	report(vmcs_clear(tmp_root) == 1,
695 	       "test vmclear with vmcs address bits set beyond physical address width");
696 
697 	/* Pass VMXON region */
698 	tmp_root = (struct vmcs *)bsp_vmxon_region;
699 	report(vmcs_clear(tmp_root) == 1, "test vmclear with vmxon region");
700 
701 	/* Valid VMCS */
702 	report(vmcs_clear(vmcs_root) == 0,
703 	       "test vmclear with valid vmcs region");
704 
705 	test_vmclear_flushing();
706 }
707 
708 static void __attribute__((__used__)) guest_main(void)
709 {
710 	if (current->v2)
711 		v2_guest_main();
712 	else
713 		current->guest_main();
714 }
715 
716 /* guest_entry */
717 asm(
718 	".align	4, 0x90\n\t"
719 	".globl	entry_guest\n\t"
720 	"guest_entry:\n\t"
721 	"	call guest_main\n\t"
722 	"	mov $1, %edi\n\t"
723 	"	call hypercall\n\t"
724 );
725 
726 /* EPT paging structure related functions */
727 /* split_large_ept_entry: Split a 2M/1G large page into 512 smaller PTEs.
728 		@ptep : large page table entry to split
729 		@level : level of ptep (2 or 3)
730  */
731 static void split_large_ept_entry(unsigned long *ptep, int level)
732 {
733 	unsigned long *new_pt;
734 	unsigned long gpa;
735 	unsigned long pte;
736 	unsigned long prototype;
737 	int i;
738 
739 	pte = *ptep;
740 	assert(pte & EPT_PRESENT);
741 	assert(pte & EPT_LARGE_PAGE);
742 	assert(level == 2 || level == 3);
743 
744 	new_pt = alloc_page();
745 	assert(new_pt);
746 
747 	prototype = pte & ~EPT_ADDR_MASK;
748 	if (level == 2)
749 		prototype &= ~EPT_LARGE_PAGE;
750 
751 	gpa = pte & EPT_ADDR_MASK;
752 	for (i = 0; i < EPT_PGDIR_ENTRIES; i++) {
753 		new_pt[i] = prototype | gpa;
754 		gpa += 1ul << EPT_LEVEL_SHIFT(level - 1);
755 	}
756 
757 	pte &= ~EPT_LARGE_PAGE;
758 	pte &= ~EPT_ADDR_MASK;
759 	pte |= virt_to_phys(new_pt);
760 
761 	*ptep = pte;
762 }
763 
764 /* install_ept_entry : Install a page to a given level in EPT
765 		@pml4 : addr of pml4 table
766 		@pte_level : level of PTE to set
767 		@guest_addr : physical address of guest
768 		@pte : pte value to set
769 		@pt_page : address of page table, NULL for a new page
770  */
771 void install_ept_entry(unsigned long *pml4,
772 		int pte_level,
773 		unsigned long guest_addr,
774 		unsigned long pte,
775 		unsigned long *pt_page)
776 {
777 	int level;
778 	unsigned long *pt = pml4;
779 	unsigned offset;
780 
781 	/* EPT only uses 48 bits of GPA. */
782 	assert(guest_addr < (1ul << 48));
783 
784 	for (level = EPT_PAGE_LEVEL; level > pte_level; --level) {
785 		offset = (guest_addr >> EPT_LEVEL_SHIFT(level))
786 				& EPT_PGDIR_MASK;
787 		if (!(pt[offset] & (EPT_PRESENT))) {
788 			unsigned long *new_pt = pt_page;
789 			if (!new_pt)
790 				new_pt = alloc_page();
791 			else
792 				pt_page = 0;
793 			memset(new_pt, 0, PAGE_SIZE);
794 			pt[offset] = virt_to_phys(new_pt)
795 					| EPT_RA | EPT_WA | EPT_EA;
796 		} else if (pt[offset] & EPT_LARGE_PAGE)
797 			split_large_ept_entry(&pt[offset], level);
798 		pt = phys_to_virt(pt[offset] & EPT_ADDR_MASK);
799 	}
800 	offset = (guest_addr >> EPT_LEVEL_SHIFT(level)) & EPT_PGDIR_MASK;
801 	pt[offset] = pte;
802 }
803 
804 /* Map a page, @perm is the permission of the page */
805 void install_ept(unsigned long *pml4,
806 		unsigned long phys,
807 		unsigned long guest_addr,
808 		u64 perm)
809 {
810 	install_ept_entry(pml4, 1, guest_addr, (phys & PAGE_MASK) | perm, 0);
811 }
812 
813 /* Map a 1G-size page */
814 void install_1g_ept(unsigned long *pml4,
815 		unsigned long phys,
816 		unsigned long guest_addr,
817 		u64 perm)
818 {
819 	install_ept_entry(pml4, 3, guest_addr,
820 			(phys & PAGE_MASK) | perm | EPT_LARGE_PAGE, 0);
821 }
822 
823 /* Map a 2M-size page */
824 void install_2m_ept(unsigned long *pml4,
825 		unsigned long phys,
826 		unsigned long guest_addr,
827 		u64 perm)
828 {
829 	install_ept_entry(pml4, 2, guest_addr,
830 			(phys & PAGE_MASK) | perm | EPT_LARGE_PAGE, 0);
831 }
832 
833 /* setup_ept_range : Setup a range of 1:1 mapped page to EPT paging structure.
834 		@start : start address of guest page
835 		@len : length of address to be mapped
836 		@map_1g : whether 1G page map is used
837 		@map_2m : whether 2M page map is used
838 		@perm : permission for every page
839  */
840 void setup_ept_range(unsigned long *pml4, unsigned long start,
841 		     unsigned long len, int map_1g, int map_2m, u64 perm)
842 {
843 	u64 phys = start;
844 	u64 max = (u64)len + (u64)start;
845 
846 	if (map_1g) {
847 		while (phys + PAGE_SIZE_1G <= max) {
848 			install_1g_ept(pml4, phys, phys, perm);
849 			phys += PAGE_SIZE_1G;
850 		}
851 	}
852 	if (map_2m) {
853 		while (phys + PAGE_SIZE_2M <= max) {
854 			install_2m_ept(pml4, phys, phys, perm);
855 			phys += PAGE_SIZE_2M;
856 		}
857 	}
858 	while (phys + PAGE_SIZE <= max) {
859 		install_ept(pml4, phys, phys, perm);
860 		phys += PAGE_SIZE;
861 	}
862 }
863 
864 /* get_ept_pte : Get the PTE of a given level in EPT,
865     @level == 1 means get the latest level*/
866 bool get_ept_pte(unsigned long *pml4, unsigned long guest_addr, int level,
867 		unsigned long *pte)
868 {
869 	int l;
870 	unsigned long *pt = pml4, iter_pte;
871 	unsigned offset;
872 
873 	assert(level >= 1 && level <= 4);
874 
875 	for (l = EPT_PAGE_LEVEL; ; --l) {
876 		offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
877 		iter_pte = pt[offset];
878 		if (l == level)
879 			break;
880 		if (l < 4 && (iter_pte & EPT_LARGE_PAGE))
881 			return false;
882 		if (!(iter_pte & (EPT_PRESENT)))
883 			return false;
884 		pt = (unsigned long *)(iter_pte & EPT_ADDR_MASK);
885 	}
886 	offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
887 	if (pte)
888 		*pte = pt[offset];
889 	return true;
890 }
891 
892 static void clear_ept_ad_pte(unsigned long *pml4, unsigned long guest_addr)
893 {
894 	int l;
895 	unsigned long *pt = pml4;
896 	u64 pte;
897 	unsigned offset;
898 
899 	for (l = EPT_PAGE_LEVEL; ; --l) {
900 		offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
901 		pt[offset] &= ~(EPT_ACCESS_FLAG|EPT_DIRTY_FLAG);
902 		pte = pt[offset];
903 		if (l == 1 || (l < 4 && (pte & EPT_LARGE_PAGE)))
904 			break;
905 		pt = (unsigned long *)(pte & EPT_ADDR_MASK);
906 	}
907 }
908 
909 /* clear_ept_ad : Clear EPT A/D bits for the page table walk and the
910    final GPA of a guest address.  */
911 void clear_ept_ad(unsigned long *pml4, u64 guest_cr3,
912 		  unsigned long guest_addr)
913 {
914 	int l;
915 	unsigned long *pt = (unsigned long *)guest_cr3, gpa;
916 	u64 pte, offset_in_page;
917 	unsigned offset;
918 
919 	for (l = EPT_PAGE_LEVEL; ; --l) {
920 		offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
921 
922 		clear_ept_ad_pte(pml4, (u64) &pt[offset]);
923 		pte = pt[offset];
924 		if (l == 1 || (l < 4 && (pte & PT_PAGE_SIZE_MASK)))
925 			break;
926 		if (!(pte & PT_PRESENT_MASK))
927 			return;
928 		pt = (unsigned long *)(pte & PT_ADDR_MASK);
929 	}
930 
931 	offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
932 	offset_in_page = guest_addr & ((1 << EPT_LEVEL_SHIFT(l)) - 1);
933 	gpa = (pt[offset] & PT_ADDR_MASK) | (guest_addr & offset_in_page);
934 	clear_ept_ad_pte(pml4, gpa);
935 }
936 
937 /* check_ept_ad : Check the content of EPT A/D bits for the page table
938    walk and the final GPA of a guest address.  */
939 void check_ept_ad(unsigned long *pml4, u64 guest_cr3,
940 		  unsigned long guest_addr, int expected_gpa_ad,
941 		  int expected_pt_ad)
942 {
943 	int l;
944 	unsigned long *pt = (unsigned long *)guest_cr3, gpa;
945 	u64 ept_pte, pte, offset_in_page;
946 	unsigned offset;
947 	bool bad_pt_ad = false;
948 
949 	for (l = EPT_PAGE_LEVEL; ; --l) {
950 		offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
951 
952 		if (!get_ept_pte(pml4, (u64) &pt[offset], 1, &ept_pte)) {
953 			printf("EPT - guest level %d page table is not mapped.\n", l);
954 			return;
955 		}
956 
957 		if (!bad_pt_ad) {
958 			bad_pt_ad |= (ept_pte & (EPT_ACCESS_FLAG|EPT_DIRTY_FLAG)) != expected_pt_ad;
959 			if (bad_pt_ad)
960 				report(false,
961 				       "EPT - guest level %d page table A=%d/D=%d",
962 				       l,
963 				       !!(expected_pt_ad & EPT_ACCESS_FLAG),
964 				       !!(expected_pt_ad & EPT_DIRTY_FLAG));
965 		}
966 
967 		pte = pt[offset];
968 		if (l == 1 || (l < 4 && (pte & PT_PAGE_SIZE_MASK)))
969 			break;
970 		if (!(pte & PT_PRESENT_MASK))
971 			return;
972 		pt = (unsigned long *)(pte & PT_ADDR_MASK);
973 	}
974 
975 	if (!bad_pt_ad)
976 		report(true, "EPT - guest page table structures A=%d/D=%d",
977 		       !!(expected_pt_ad & EPT_ACCESS_FLAG),
978 		       !!(expected_pt_ad & EPT_DIRTY_FLAG));
979 
980 	offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
981 	offset_in_page = guest_addr & ((1 << EPT_LEVEL_SHIFT(l)) - 1);
982 	gpa = (pt[offset] & PT_ADDR_MASK) | (guest_addr & offset_in_page);
983 
984 	if (!get_ept_pte(pml4, gpa, 1, &ept_pte)) {
985 		report(false, "EPT - guest physical address is not mapped");
986 		return;
987 	}
988 	report((ept_pte & (EPT_ACCESS_FLAG | EPT_DIRTY_FLAG)) == expected_gpa_ad,
989 	       "EPT - guest physical address A=%d/D=%d",
990 	       !!(expected_gpa_ad & EPT_ACCESS_FLAG),
991 	       !!(expected_gpa_ad & EPT_DIRTY_FLAG));
992 }
993 
994 
995 void ept_sync(int type, u64 eptp)
996 {
997 	switch (type) {
998 	case INVEPT_SINGLE:
999 		if (ept_vpid.val & EPT_CAP_INVEPT_SINGLE) {
1000 			invept(INVEPT_SINGLE, eptp);
1001 			break;
1002 		}
1003 		/* else fall through */
1004 	case INVEPT_GLOBAL:
1005 		if (ept_vpid.val & EPT_CAP_INVEPT_ALL) {
1006 			invept(INVEPT_GLOBAL, eptp);
1007 			break;
1008 		}
1009 		/* else fall through */
1010 	default:
1011 		printf("WARNING: invept is not supported!\n");
1012 	}
1013 }
1014 
1015 void set_ept_pte(unsigned long *pml4, unsigned long guest_addr,
1016 		 int level, u64 pte_val)
1017 {
1018 	int l;
1019 	unsigned long *pt = pml4;
1020 	unsigned offset;
1021 
1022 	assert(level >= 1 && level <= 4);
1023 
1024 	for (l = EPT_PAGE_LEVEL; ; --l) {
1025 		offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
1026 		if (l == level)
1027 			break;
1028 		assert(pt[offset] & EPT_PRESENT);
1029 		pt = (unsigned long *)(pt[offset] & EPT_ADDR_MASK);
1030 	}
1031 	offset = (guest_addr >> EPT_LEVEL_SHIFT(l)) & EPT_PGDIR_MASK;
1032 	pt[offset] = pte_val;
1033 }
1034 
1035 bool ept_2m_supported(void)
1036 {
1037 	return ept_vpid.val & EPT_CAP_2M_PAGE;
1038 }
1039 
1040 bool ept_1g_supported(void)
1041 {
1042 	return ept_vpid.val & EPT_CAP_1G_PAGE;
1043 }
1044 
1045 bool ept_huge_pages_supported(int level)
1046 {
1047 	if (level == 2)
1048 		return ept_2m_supported();
1049 	else if (level == 3)
1050 		return ept_1g_supported();
1051 	else
1052 		return false;
1053 }
1054 
1055 bool ept_execute_only_supported(void)
1056 {
1057 	return ept_vpid.val & EPT_CAP_WT;
1058 }
1059 
1060 bool ept_ad_bits_supported(void)
1061 {
1062 	return ept_vpid.val & EPT_CAP_AD_FLAG;
1063 }
1064 
1065 void vpid_sync(int type, u16 vpid)
1066 {
1067 	switch(type) {
1068 	case INVVPID_CONTEXT_GLOBAL:
1069 		if (ept_vpid.val & VPID_CAP_INVVPID_CXTGLB) {
1070 			invvpid(INVVPID_CONTEXT_GLOBAL, vpid, 0);
1071 			break;
1072 		}
1073 	case INVVPID_ALL:
1074 		if (ept_vpid.val & VPID_CAP_INVVPID_ALL) {
1075 			invvpid(INVVPID_ALL, vpid, 0);
1076 			break;
1077 		}
1078 	default:
1079 		printf("WARNING: invvpid is not supported\n");
1080 	}
1081 }
1082 
1083 static void init_vmcs_ctrl(void)
1084 {
1085 	/* 26.2 CHECKS ON VMX CONTROLS AND HOST-STATE AREA */
1086 	/* 26.2.1.1 */
1087 	vmcs_write(PIN_CONTROLS, ctrl_pin);
1088 	/* Disable VMEXIT of IO instruction */
1089 	vmcs_write(CPU_EXEC_CTRL0, ctrl_cpu[0]);
1090 	if (ctrl_cpu_rev[0].set & CPU_SECONDARY) {
1091 		ctrl_cpu[1] = (ctrl_cpu[1] | ctrl_cpu_rev[1].set) &
1092 			ctrl_cpu_rev[1].clr;
1093 		vmcs_write(CPU_EXEC_CTRL1, ctrl_cpu[1]);
1094 	}
1095 	vmcs_write(CR3_TARGET_COUNT, 0);
1096 	vmcs_write(VPID, ++vpid_cnt);
1097 }
1098 
1099 static void init_vmcs_host(void)
1100 {
1101 	/* 26.2 CHECKS ON VMX CONTROLS AND HOST-STATE AREA */
1102 	/* 26.2.1.2 */
1103 	vmcs_write(HOST_EFER, rdmsr(MSR_EFER));
1104 
1105 	/* 26.2.1.3 */
1106 	vmcs_write(ENT_CONTROLS, ctrl_enter);
1107 	vmcs_write(EXI_CONTROLS, ctrl_exit);
1108 
1109 	/* 26.2.2 */
1110 	vmcs_write(HOST_CR0, read_cr0());
1111 	vmcs_write(HOST_CR3, read_cr3());
1112 	vmcs_write(HOST_CR4, read_cr4());
1113 	vmcs_write(HOST_SYSENTER_EIP, (u64)(&entry_sysenter));
1114 	vmcs_write(HOST_SYSENTER_CS,  KERNEL_CS);
1115 
1116 	/* 26.2.3 */
1117 	vmcs_write(HOST_SEL_CS, KERNEL_CS);
1118 	vmcs_write(HOST_SEL_SS, KERNEL_DS);
1119 	vmcs_write(HOST_SEL_DS, KERNEL_DS);
1120 	vmcs_write(HOST_SEL_ES, KERNEL_DS);
1121 	vmcs_write(HOST_SEL_FS, KERNEL_DS);
1122 	vmcs_write(HOST_SEL_GS, KERNEL_DS);
1123 	vmcs_write(HOST_SEL_TR, TSS_MAIN);
1124 	vmcs_write(HOST_BASE_TR, tss_descr.base);
1125 	vmcs_write(HOST_BASE_GDTR, gdt64_desc.base);
1126 	vmcs_write(HOST_BASE_IDTR, idt_descr.base);
1127 	vmcs_write(HOST_BASE_FS, 0);
1128 	vmcs_write(HOST_BASE_GS, 0);
1129 
1130 	/* Set other vmcs area */
1131 	vmcs_write(PF_ERROR_MASK, 0);
1132 	vmcs_write(PF_ERROR_MATCH, 0);
1133 	vmcs_write(VMCS_LINK_PTR, ~0ul);
1134 	vmcs_write(VMCS_LINK_PTR_HI, ~0ul);
1135 	vmcs_write(HOST_RIP, (u64)(&vmx_return));
1136 }
1137 
1138 static void init_vmcs_guest(void)
1139 {
1140 	/* 26.3 CHECKING AND LOADING GUEST STATE */
1141 	ulong guest_cr0, guest_cr4, guest_cr3;
1142 	/* 26.3.1.1 */
1143 	guest_cr0 = read_cr0();
1144 	guest_cr4 = read_cr4();
1145 	guest_cr3 = read_cr3();
1146 	if (ctrl_enter & ENT_GUEST_64) {
1147 		guest_cr0 |= X86_CR0_PG;
1148 		guest_cr4 |= X86_CR4_PAE;
1149 	}
1150 	if ((ctrl_enter & ENT_GUEST_64) == 0)
1151 		guest_cr4 &= (~X86_CR4_PCIDE);
1152 	if (guest_cr0 & X86_CR0_PG)
1153 		guest_cr0 |= X86_CR0_PE;
1154 	vmcs_write(GUEST_CR0, guest_cr0);
1155 	vmcs_write(GUEST_CR3, guest_cr3);
1156 	vmcs_write(GUEST_CR4, guest_cr4);
1157 	vmcs_write(GUEST_SYSENTER_CS,  KERNEL_CS);
1158 	vmcs_write(GUEST_SYSENTER_ESP,
1159 		(u64)(guest_syscall_stack + PAGE_SIZE - 1));
1160 	vmcs_write(GUEST_SYSENTER_EIP, (u64)(&entry_sysenter));
1161 	vmcs_write(GUEST_DR7, 0);
1162 	vmcs_write(GUEST_EFER, rdmsr(MSR_EFER));
1163 
1164 	/* 26.3.1.2 */
1165 	vmcs_write(GUEST_SEL_CS, KERNEL_CS);
1166 	vmcs_write(GUEST_SEL_SS, KERNEL_DS);
1167 	vmcs_write(GUEST_SEL_DS, KERNEL_DS);
1168 	vmcs_write(GUEST_SEL_ES, KERNEL_DS);
1169 	vmcs_write(GUEST_SEL_FS, KERNEL_DS);
1170 	vmcs_write(GUEST_SEL_GS, KERNEL_DS);
1171 	vmcs_write(GUEST_SEL_TR, TSS_MAIN);
1172 	vmcs_write(GUEST_SEL_LDTR, 0);
1173 
1174 	vmcs_write(GUEST_BASE_CS, 0);
1175 	vmcs_write(GUEST_BASE_ES, 0);
1176 	vmcs_write(GUEST_BASE_SS, 0);
1177 	vmcs_write(GUEST_BASE_DS, 0);
1178 	vmcs_write(GUEST_BASE_FS, 0);
1179 	vmcs_write(GUEST_BASE_GS, 0);
1180 	vmcs_write(GUEST_BASE_TR, tss_descr.base);
1181 	vmcs_write(GUEST_BASE_LDTR, 0);
1182 
1183 	vmcs_write(GUEST_LIMIT_CS, 0xFFFFFFFF);
1184 	vmcs_write(GUEST_LIMIT_DS, 0xFFFFFFFF);
1185 	vmcs_write(GUEST_LIMIT_ES, 0xFFFFFFFF);
1186 	vmcs_write(GUEST_LIMIT_SS, 0xFFFFFFFF);
1187 	vmcs_write(GUEST_LIMIT_FS, 0xFFFFFFFF);
1188 	vmcs_write(GUEST_LIMIT_GS, 0xFFFFFFFF);
1189 	vmcs_write(GUEST_LIMIT_LDTR, 0xffff);
1190 	vmcs_write(GUEST_LIMIT_TR, tss_descr.limit);
1191 
1192 	vmcs_write(GUEST_AR_CS, 0xa09b);
1193 	vmcs_write(GUEST_AR_DS, 0xc093);
1194 	vmcs_write(GUEST_AR_ES, 0xc093);
1195 	vmcs_write(GUEST_AR_FS, 0xc093);
1196 	vmcs_write(GUEST_AR_GS, 0xc093);
1197 	vmcs_write(GUEST_AR_SS, 0xc093);
1198 	vmcs_write(GUEST_AR_LDTR, 0x82);
1199 	vmcs_write(GUEST_AR_TR, 0x8b);
1200 
1201 	/* 26.3.1.3 */
1202 	vmcs_write(GUEST_BASE_GDTR, gdt64_desc.base);
1203 	vmcs_write(GUEST_BASE_IDTR, idt_descr.base);
1204 	vmcs_write(GUEST_LIMIT_GDTR, gdt64_desc.limit);
1205 	vmcs_write(GUEST_LIMIT_IDTR, idt_descr.limit);
1206 
1207 	/* 26.3.1.4 */
1208 	vmcs_write(GUEST_RIP, (u64)(&guest_entry));
1209 	vmcs_write(GUEST_RSP, (u64)(guest_stack + PAGE_SIZE - 1));
1210 	vmcs_write(GUEST_RFLAGS, 0x2);
1211 
1212 	/* 26.3.1.5 */
1213 	vmcs_write(GUEST_ACTV_STATE, ACTV_ACTIVE);
1214 	vmcs_write(GUEST_INTR_STATE, 0);
1215 }
1216 
1217 static int init_vmcs(struct vmcs **vmcs)
1218 {
1219 	*vmcs = alloc_page();
1220 	(*vmcs)->hdr.revision_id = basic.revision;
1221 	/* vmclear first to init vmcs */
1222 	if (vmcs_clear(*vmcs)) {
1223 		printf("%s : vmcs_clear error\n", __func__);
1224 		return 1;
1225 	}
1226 
1227 	if (make_vmcs_current(*vmcs)) {
1228 		printf("%s : make_vmcs_current error\n", __func__);
1229 		return 1;
1230 	}
1231 
1232 	/* All settings to pin/exit/enter/cpu
1233 	   control fields should be placed here */
1234 	ctrl_pin |= PIN_EXTINT | PIN_NMI | PIN_VIRT_NMI;
1235 	ctrl_exit = EXI_LOAD_EFER | EXI_HOST_64;
1236 	ctrl_enter = (ENT_LOAD_EFER | ENT_GUEST_64);
1237 	/* DIsable IO instruction VMEXIT now */
1238 	ctrl_cpu[0] &= (~(CPU_IO | CPU_IO_BITMAP));
1239 	ctrl_cpu[1] = 0;
1240 
1241 	ctrl_pin = (ctrl_pin | ctrl_pin_rev.set) & ctrl_pin_rev.clr;
1242 	ctrl_enter = (ctrl_enter | ctrl_enter_rev.set) & ctrl_enter_rev.clr;
1243 	ctrl_exit = (ctrl_exit | ctrl_exit_rev.set) & ctrl_exit_rev.clr;
1244 	ctrl_cpu[0] = (ctrl_cpu[0] | ctrl_cpu_rev[0].set) & ctrl_cpu_rev[0].clr;
1245 
1246 	init_vmcs_ctrl();
1247 	init_vmcs_host();
1248 	init_vmcs_guest();
1249 	return 0;
1250 }
1251 
1252 void enable_vmx(void)
1253 {
1254 	bool vmx_enabled =
1255 		rdmsr(MSR_IA32_FEATURE_CONTROL) &
1256 		FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
1257 
1258 	if (!vmx_enabled) {
1259 		wrmsr(MSR_IA32_FEATURE_CONTROL,
1260 				FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX |
1261 				FEATURE_CONTROL_LOCKED);
1262 	}
1263 }
1264 
1265 static void init_vmx_caps(void)
1266 {
1267 	basic.val = rdmsr(MSR_IA32_VMX_BASIC);
1268 	ctrl_pin_rev.val = rdmsr(basic.ctrl ? MSR_IA32_VMX_TRUE_PIN
1269 			: MSR_IA32_VMX_PINBASED_CTLS);
1270 	ctrl_exit_rev.val = rdmsr(basic.ctrl ? MSR_IA32_VMX_TRUE_EXIT
1271 			: MSR_IA32_VMX_EXIT_CTLS);
1272 	ctrl_enter_rev.val = rdmsr(basic.ctrl ? MSR_IA32_VMX_TRUE_ENTRY
1273 			: MSR_IA32_VMX_ENTRY_CTLS);
1274 	ctrl_cpu_rev[0].val = rdmsr(basic.ctrl ? MSR_IA32_VMX_TRUE_PROC
1275 			: MSR_IA32_VMX_PROCBASED_CTLS);
1276 	if ((ctrl_cpu_rev[0].clr & CPU_SECONDARY) != 0)
1277 		ctrl_cpu_rev[1].val = rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2);
1278 	else
1279 		ctrl_cpu_rev[1].val = 0;
1280 	if ((ctrl_cpu_rev[1].clr & (CPU_EPT | CPU_VPID)) != 0)
1281 		ept_vpid.val = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
1282 	else
1283 		ept_vpid.val = 0;
1284 }
1285 
1286 void init_vmx(u64 *vmxon_region)
1287 {
1288 	ulong fix_cr0_set, fix_cr0_clr;
1289 	ulong fix_cr4_set, fix_cr4_clr;
1290 
1291 	fix_cr0_set =  rdmsr(MSR_IA32_VMX_CR0_FIXED0);
1292 	fix_cr0_clr =  rdmsr(MSR_IA32_VMX_CR0_FIXED1);
1293 	fix_cr4_set =  rdmsr(MSR_IA32_VMX_CR4_FIXED0);
1294 	fix_cr4_clr = rdmsr(MSR_IA32_VMX_CR4_FIXED1);
1295 
1296 	write_cr0((read_cr0() & fix_cr0_clr) | fix_cr0_set);
1297 	write_cr4((read_cr4() & fix_cr4_clr) | fix_cr4_set | X86_CR4_VMXE);
1298 
1299 	*vmxon_region = basic.revision;
1300 }
1301 
1302 static void alloc_bsp_vmx_pages(void)
1303 {
1304 	bsp_vmxon_region = alloc_page();
1305 	guest_stack = alloc_page();
1306 	guest_syscall_stack = alloc_page();
1307 	vmcs_root = alloc_page();
1308 }
1309 
1310 static void init_bsp_vmx(void)
1311 {
1312 	init_vmx_caps();
1313 	alloc_bsp_vmx_pages();
1314 	init_vmx(bsp_vmxon_region);
1315 }
1316 
1317 static void do_vmxon_off(void *data)
1318 {
1319 	vmx_on();
1320 	vmx_off();
1321 }
1322 
1323 static void do_write_feature_control(void *data)
1324 {
1325 	wrmsr(MSR_IA32_FEATURE_CONTROL, 0);
1326 }
1327 
1328 static int test_vmx_feature_control(void)
1329 {
1330 	u64 ia32_feature_control;
1331 	bool vmx_enabled;
1332 	bool feature_control_locked;
1333 
1334 	ia32_feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
1335 	vmx_enabled =
1336 		ia32_feature_control & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
1337 	feature_control_locked =
1338 		ia32_feature_control & FEATURE_CONTROL_LOCKED;
1339 
1340 	if (vmx_enabled && feature_control_locked) {
1341 		printf("VMX enabled and locked by BIOS\n");
1342 		return 0;
1343 	} else if (feature_control_locked) {
1344 		printf("ERROR: VMX locked out by BIOS!?\n");
1345 		return 1;
1346 	}
1347 
1348 	wrmsr(MSR_IA32_FEATURE_CONTROL, 0);
1349 	report(test_for_exception(GP_VECTOR, &do_vmxon_off, NULL),
1350 	       "test vmxon with FEATURE_CONTROL cleared");
1351 
1352 	wrmsr(MSR_IA32_FEATURE_CONTROL, FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX);
1353 	report(test_for_exception(GP_VECTOR, &do_vmxon_off, NULL),
1354 	       "test vmxon without FEATURE_CONTROL lock");
1355 
1356 	wrmsr(MSR_IA32_FEATURE_CONTROL,
1357 		  FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX |
1358 		  FEATURE_CONTROL_LOCKED);
1359 
1360 	ia32_feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
1361 	vmx_enabled =
1362 		ia32_feature_control & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
1363 	report(vmx_enabled, "test enable VMX in FEATURE_CONTROL");
1364 
1365 	report(test_for_exception(GP_VECTOR, &do_write_feature_control, NULL),
1366 	       "test FEATURE_CONTROL lock bit");
1367 
1368 	return !vmx_enabled;
1369 }
1370 
1371 static int test_vmxon(void)
1372 {
1373 	int ret, ret1;
1374 	u64 *vmxon_region;
1375 	int width = cpuid_maxphyaddr();
1376 
1377 	/* Unaligned page access */
1378 	vmxon_region = (u64 *)((intptr_t)bsp_vmxon_region + 1);
1379 	ret1 = _vmx_on(vmxon_region);
1380 	report(ret1, "test vmxon with unaligned vmxon region");
1381 	if (!ret1) {
1382 		ret = 1;
1383 		goto out;
1384 	}
1385 
1386 	/* gpa bits beyond physical address width are set*/
1387 	vmxon_region = (u64 *)((intptr_t)bsp_vmxon_region | ((u64)1 << (width+1)));
1388 	ret1 = _vmx_on(vmxon_region);
1389 	report(ret1, "test vmxon with bits set beyond physical address width");
1390 	if (!ret1) {
1391 		ret = 1;
1392 		goto out;
1393 	}
1394 
1395 	/* invalid revision indentifier */
1396 	*bsp_vmxon_region = 0xba9da9;
1397 	ret1 = vmx_on();
1398 	report(ret1, "test vmxon with invalid revision identifier");
1399 	if (!ret1) {
1400 		ret = 1;
1401 		goto out;
1402 	}
1403 
1404 	/* and finally a valid region */
1405 	*bsp_vmxon_region = basic.revision;
1406 	ret = vmx_on();
1407 	report(!ret, "test vmxon with valid vmxon region");
1408 
1409 out:
1410 	return ret;
1411 }
1412 
1413 static void test_vmptrld(void)
1414 {
1415 	struct vmcs *vmcs, *tmp_root;
1416 	int width = cpuid_maxphyaddr();
1417 
1418 	vmcs = alloc_page();
1419 	vmcs->hdr.revision_id = basic.revision;
1420 
1421 	/* Unaligned page access */
1422 	tmp_root = (struct vmcs *)((intptr_t)vmcs + 1);
1423 	report(make_vmcs_current(tmp_root) == 1,
1424 	       "test vmptrld with unaligned vmcs");
1425 
1426 	/* gpa bits beyond physical address width are set*/
1427 	tmp_root = (struct vmcs *)((intptr_t)vmcs |
1428 				   ((u64)1 << (width+1)));
1429 	report(make_vmcs_current(tmp_root) == 1,
1430 	       "test vmptrld with vmcs address bits set beyond physical address width");
1431 
1432 	/* Pass VMXON region */
1433 	assert(!vmcs_clear(vmcs));
1434 	assert(!make_vmcs_current(vmcs));
1435 	tmp_root = (struct vmcs *)bsp_vmxon_region;
1436 	report(make_vmcs_current(tmp_root) == 1,
1437 	       "test vmptrld with vmxon region");
1438 	report(vmcs_read(VMX_INST_ERROR) == VMXERR_VMPTRLD_VMXON_POINTER,
1439 	       "test vmptrld with vmxon region vm-instruction error");
1440 
1441 	report(make_vmcs_current(vmcs) == 0,
1442 	       "test vmptrld with valid vmcs region");
1443 }
1444 
1445 static void test_vmptrst(void)
1446 {
1447 	int ret;
1448 	struct vmcs *vmcs1, *vmcs2;
1449 
1450 	vmcs1 = alloc_page();
1451 	init_vmcs(&vmcs1);
1452 	ret = vmcs_save(&vmcs2);
1453 	report((!ret) && (vmcs1 == vmcs2), "test vmptrst");
1454 }
1455 
1456 struct vmx_ctl_msr {
1457 	const char *name;
1458 	u32 index, true_index;
1459 	u32 default1;
1460 } vmx_ctl_msr[] = {
1461 	{ "MSR_IA32_VMX_PINBASED_CTLS", MSR_IA32_VMX_PINBASED_CTLS,
1462 	  MSR_IA32_VMX_TRUE_PIN, 0x16 },
1463 	{ "MSR_IA32_VMX_PROCBASED_CTLS", MSR_IA32_VMX_PROCBASED_CTLS,
1464 	  MSR_IA32_VMX_TRUE_PROC, 0x401e172 },
1465 	{ "MSR_IA32_VMX_PROCBASED_CTLS2", MSR_IA32_VMX_PROCBASED_CTLS2,
1466 	  MSR_IA32_VMX_PROCBASED_CTLS2, 0 },
1467 	{ "MSR_IA32_VMX_EXIT_CTLS", MSR_IA32_VMX_EXIT_CTLS,
1468 	  MSR_IA32_VMX_TRUE_EXIT, 0x36dff },
1469 	{ "MSR_IA32_VMX_ENTRY_CTLS", MSR_IA32_VMX_ENTRY_CTLS,
1470 	  MSR_IA32_VMX_TRUE_ENTRY, 0x11ff },
1471 };
1472 
1473 static void test_vmx_caps(void)
1474 {
1475 	u64 val, default1, fixed0, fixed1;
1476 	union vmx_ctrl_msr ctrl, true_ctrl;
1477 	unsigned int n;
1478 	bool ok;
1479 
1480 	printf("\nTest suite: VMX capability reporting\n");
1481 
1482 	report((basic.revision & (1ul << 31)) == 0 &&
1483 	       basic.size > 0 && basic.size <= 4096 &&
1484 	       (basic.type == 0 || basic.type == 6) &&
1485 	       basic.reserved1 == 0 && basic.reserved2 == 0,
1486 	       "MSR_IA32_VMX_BASIC");
1487 
1488 	val = rdmsr(MSR_IA32_VMX_MISC);
1489 	report((!(ctrl_cpu_rev[1].clr & CPU_URG) || val & (1ul << 5)) &&
1490 	       ((val >> 16) & 0x1ff) <= 256 &&
1491 	       (val & 0x80007e00) == 0,
1492 	       "MSR_IA32_VMX_MISC");
1493 
1494 	for (n = 0; n < ARRAY_SIZE(vmx_ctl_msr); n++) {
1495 		ctrl.val = rdmsr(vmx_ctl_msr[n].index);
1496 		default1 = vmx_ctl_msr[n].default1;
1497 		ok = (ctrl.set & default1) == default1;
1498 		ok = ok && (ctrl.set & ~ctrl.clr) == 0;
1499 		if (ok && basic.ctrl) {
1500 			true_ctrl.val = rdmsr(vmx_ctl_msr[n].true_index);
1501 			ok = ctrl.clr == true_ctrl.clr;
1502 			ok = ok && ctrl.set == (true_ctrl.set | default1);
1503 		}
1504 		report(ok, "%s", vmx_ctl_msr[n].name);
1505 	}
1506 
1507 	fixed0 = rdmsr(MSR_IA32_VMX_CR0_FIXED0);
1508 	fixed1 = rdmsr(MSR_IA32_VMX_CR0_FIXED1);
1509 	report(((fixed0 ^ fixed1) & ~fixed1) == 0,
1510 	       "MSR_IA32_VMX_IA32_VMX_CR0_FIXED0/1");
1511 
1512 	fixed0 = rdmsr(MSR_IA32_VMX_CR4_FIXED0);
1513 	fixed1 = rdmsr(MSR_IA32_VMX_CR4_FIXED1);
1514 	report(((fixed0 ^ fixed1) & ~fixed1) == 0,
1515 	       "MSR_IA32_VMX_IA32_VMX_CR4_FIXED0/1");
1516 
1517 	val = rdmsr(MSR_IA32_VMX_VMCS_ENUM);
1518 	report((val & VMCS_FIELD_INDEX_MASK) >= 0x2a &&
1519 	       (val & 0xfffffffffffffc01Ull) == 0,
1520 	       "MSR_IA32_VMX_VMCS_ENUM");
1521 
1522 	val = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
1523 	report((val & 0xfffff07ef98cbebeUll) == 0,
1524 	       "MSR_IA32_VMX_EPT_VPID_CAP");
1525 }
1526 
1527 /* This function can only be called in guest */
1528 static void __attribute__((__used__)) hypercall(u32 hypercall_no)
1529 {
1530 	u64 val = 0;
1531 	val = (hypercall_no & HYPERCALL_MASK) | HYPERCALL_BIT;
1532 	hypercall_field = val;
1533 	asm volatile("vmcall\n\t");
1534 }
1535 
1536 static bool is_hypercall(void)
1537 {
1538 	ulong reason, hyper_bit;
1539 
1540 	reason = vmcs_read(EXI_REASON) & 0xff;
1541 	hyper_bit = hypercall_field & HYPERCALL_BIT;
1542 	if (reason == VMX_VMCALL && hyper_bit)
1543 		return true;
1544 	return false;
1545 }
1546 
1547 static int handle_hypercall(void)
1548 {
1549 	ulong hypercall_no;
1550 
1551 	hypercall_no = hypercall_field & HYPERCALL_MASK;
1552 	hypercall_field = 0;
1553 	switch (hypercall_no) {
1554 	case HYPERCALL_VMEXIT:
1555 		return VMX_TEST_VMEXIT;
1556 	case HYPERCALL_VMABORT:
1557 		return VMX_TEST_VMABORT;
1558 	case HYPERCALL_VMSKIP:
1559 		return VMX_TEST_VMSKIP;
1560 	default:
1561 		printf("ERROR : Invalid hypercall number : %ld\n", hypercall_no);
1562 	}
1563 	return VMX_TEST_EXIT;
1564 }
1565 
1566 static void continue_abort(void)
1567 {
1568 	assert(!in_guest);
1569 	printf("Host was here when guest aborted:\n");
1570 	dump_stack();
1571 	longjmp(abort_target, 1);
1572 	abort();
1573 }
1574 
1575 void __abort_test(void)
1576 {
1577 	if (in_guest)
1578 		hypercall(HYPERCALL_VMABORT);
1579 	else
1580 		longjmp(abort_target, 1);
1581 	abort();
1582 }
1583 
1584 static void continue_skip(void)
1585 {
1586 	assert(!in_guest);
1587 	longjmp(abort_target, 1);
1588 	abort();
1589 }
1590 
1591 void test_skip(const char *msg)
1592 {
1593 	printf("%s skipping test: %s\n", in_guest ? "Guest" : "Host", msg);
1594 	if (in_guest)
1595 		hypercall(HYPERCALL_VMABORT);
1596 	else
1597 		longjmp(abort_target, 1);
1598 	abort();
1599 }
1600 
1601 static int exit_handler(void)
1602 {
1603 	int ret;
1604 
1605 	current->exits++;
1606 	regs.rflags = vmcs_read(GUEST_RFLAGS);
1607 	if (is_hypercall())
1608 		ret = handle_hypercall();
1609 	else
1610 		ret = current->exit_handler();
1611 	vmcs_write(GUEST_RFLAGS, regs.rflags);
1612 
1613 	return ret;
1614 }
1615 
1616 /*
1617  * Called if vmlaunch or vmresume fails.
1618  *	@early    - failure due to "VMX controls and host-state area" (26.2)
1619  *	@vmlaunch - was this a vmlaunch or vmresume
1620  *	@rflags   - host rflags
1621  */
1622 static int
1623 entry_failure_handler(struct vmentry_failure *failure)
1624 {
1625 	if (current->entry_failure_handler)
1626 		return current->entry_failure_handler(failure);
1627 	else
1628 		return VMX_TEST_EXIT;
1629 }
1630 
1631 /*
1632  * Tries to enter the guest. Returns true if entry succeeded. Otherwise,
1633  * populates @failure.
1634  */
1635 static void vmx_enter_guest(struct vmentry_failure *failure)
1636 {
1637 	failure->early = 0;
1638 
1639 	in_guest = 1;
1640 	asm volatile (
1641 		"mov %[HOST_RSP], %%rdi\n\t"
1642 		"vmwrite %%rsp, %%rdi\n\t"
1643 		LOAD_GPR_C
1644 		"cmpb $0, %[launched]\n\t"
1645 		"jne 1f\n\t"
1646 		"vmlaunch\n\t"
1647 		"jmp 2f\n\t"
1648 		"1: "
1649 		"vmresume\n\t"
1650 		"2: "
1651 		SAVE_GPR_C
1652 		"pushf\n\t"
1653 		"pop %%rdi\n\t"
1654 		"mov %%rdi, %[failure_flags]\n\t"
1655 		"movl $1, %[failure_early]\n\t"
1656 		"jmp 3f\n\t"
1657 		"vmx_return:\n\t"
1658 		SAVE_GPR_C
1659 		"3: \n\t"
1660 		: [failure_early]"+m"(failure->early),
1661 		  [failure_flags]"=m"(failure->flags)
1662 		: [launched]"m"(launched), [HOST_RSP]"i"(HOST_RSP)
1663 		: "rdi", "memory", "cc"
1664 	);
1665 	in_guest = 0;
1666 
1667 	failure->vmlaunch = !launched;
1668 	failure->instr = launched ? "vmresume" : "vmlaunch";
1669 }
1670 
1671 static int vmx_run(void)
1672 {
1673 	while (1) {
1674 		u32 ret;
1675 		bool entered;
1676 		struct vmentry_failure failure;
1677 
1678 		vmx_enter_guest(&failure);
1679 		entered = !failure.early &&
1680 			  !(vmcs_read(EXI_REASON) & VMX_ENTRY_FAILURE);
1681 
1682 		if (entered) {
1683 			/*
1684 			 * VMCS isn't in "launched" state if there's been any
1685 			 * entry failure (early or otherwise).
1686 			 */
1687 			launched = 1;
1688 			ret = exit_handler();
1689 		} else {
1690 			ret = entry_failure_handler(&failure);
1691 		}
1692 
1693 		switch (ret) {
1694 		case VMX_TEST_RESUME:
1695 			continue;
1696 		case VMX_TEST_VMEXIT:
1697 			guest_finished = 1;
1698 			return 0;
1699 		case VMX_TEST_EXIT:
1700 			break;
1701 		default:
1702 			printf("ERROR : Invalid %s_handler return val %d.\n",
1703 			       entered ? "exit" : "entry_failure",
1704 			       ret);
1705 			break;
1706 		}
1707 
1708 		if (entered)
1709 			print_vmexit_info();
1710 		else
1711 			print_vmentry_failure_info(&failure);
1712 		abort();
1713 	}
1714 }
1715 
1716 static void run_teardown_step(struct test_teardown_step *step)
1717 {
1718 	step->func(step->data);
1719 }
1720 
1721 static int test_run(struct vmx_test *test)
1722 {
1723 	int r;
1724 
1725 	/* Validate V2 interface. */
1726 	if (test->v2) {
1727 		int ret = 0;
1728 		if (test->init || test->guest_main || test->exit_handler ||
1729 		    test->syscall_handler) {
1730 			report(0, "V2 test cannot specify V1 callbacks.");
1731 			ret = 1;
1732 		}
1733 		if (ret)
1734 			return ret;
1735 	}
1736 
1737 	if (test->name == NULL)
1738 		test->name = "(no name)";
1739 	if (vmx_on()) {
1740 		printf("%s : vmxon failed.\n", __func__);
1741 		return 1;
1742 	}
1743 
1744 	init_vmcs(&(test->vmcs));
1745 	/* Directly call test->init is ok here, init_vmcs has done
1746 	   vmcs init, vmclear and vmptrld*/
1747 	if (test->init && test->init(test->vmcs) != VMX_TEST_START)
1748 		goto out;
1749 	teardown_count = 0;
1750 	v2_guest_main = NULL;
1751 	test->exits = 0;
1752 	current = test;
1753 	regs = test->guest_regs;
1754 	vmcs_write(GUEST_RFLAGS, regs.rflags | 0x2);
1755 	launched = 0;
1756 	guest_finished = 0;
1757 	printf("\nTest suite: %s\n", test->name);
1758 
1759 	r = setjmp(abort_target);
1760 	if (r) {
1761 		assert(!in_guest);
1762 		goto out;
1763 	}
1764 
1765 
1766 	if (test->v2)
1767 		test->v2();
1768 	else
1769 		vmx_run();
1770 
1771 	while (teardown_count > 0)
1772 		run_teardown_step(&teardown_steps[--teardown_count]);
1773 
1774 	if (launched && !guest_finished)
1775 		report(0, "Guest didn't run to completion.");
1776 
1777 out:
1778 	if (vmx_off()) {
1779 		printf("%s : vmxoff failed.\n", __func__);
1780 		return 1;
1781 	}
1782 	return 0;
1783 }
1784 
1785 /*
1786  * Add a teardown step. Executed after the test's main function returns.
1787  * Teardown steps executed in reverse order.
1788  */
1789 void test_add_teardown(test_teardown_func func, void *data)
1790 {
1791 	struct test_teardown_step *step;
1792 
1793 	TEST_ASSERT_MSG(teardown_count < MAX_TEST_TEARDOWN_STEPS,
1794 			"There are already %d teardown steps.",
1795 			teardown_count);
1796 	step = &teardown_steps[teardown_count++];
1797 	step->func = func;
1798 	step->data = data;
1799 }
1800 
1801 /*
1802  * Set the target of the first enter_guest call. Can only be called once per
1803  * test. Must be called before first enter_guest call.
1804  */
1805 void test_set_guest(test_guest_func func)
1806 {
1807 	assert(current->v2);
1808 	TEST_ASSERT_MSG(!v2_guest_main, "Already set guest func.");
1809 	v2_guest_main = func;
1810 }
1811 
1812 static void check_for_guest_termination(void)
1813 {
1814 	if (is_hypercall()) {
1815 		int ret;
1816 
1817 		ret = handle_hypercall();
1818 		switch (ret) {
1819 		case VMX_TEST_VMEXIT:
1820 			guest_finished = 1;
1821 			break;
1822 		case VMX_TEST_VMABORT:
1823 			continue_abort();
1824 			break;
1825 		case VMX_TEST_VMSKIP:
1826 			continue_skip();
1827 			break;
1828 		default:
1829 			printf("ERROR : Invalid handle_hypercall return %d.\n",
1830 			       ret);
1831 			abort();
1832 		}
1833 	}
1834 }
1835 
1836 #define        ABORT_ON_EARLY_VMENTRY_FAIL     0x1
1837 #define        ABORT_ON_INVALID_GUEST_STATE    0x2
1838 
1839 /*
1840  * Enters the guest (or launches it for the first time). Error to call once the
1841  * guest has returned (i.e., run past the end of its guest() function).
1842  */
1843 static void __enter_guest(u8 abort_flag, struct vmentry_failure *failure)
1844 {
1845 	TEST_ASSERT_MSG(v2_guest_main,
1846 			"Never called test_set_guest_func!");
1847 
1848 	TEST_ASSERT_MSG(!guest_finished,
1849 			"Called enter_guest() after guest returned.");
1850 
1851 	vmx_enter_guest(failure);
1852 	if ((abort_flag & ABORT_ON_EARLY_VMENTRY_FAIL && failure->early) ||
1853 	    (abort_flag & ABORT_ON_INVALID_GUEST_STATE &&
1854 	    vmcs_read(EXI_REASON) & VMX_ENTRY_FAILURE)) {
1855 
1856 		print_vmentry_failure_info(failure);
1857 		abort();
1858 	}
1859 
1860 	if (!failure->early && !(vmcs_read(EXI_REASON) & VMX_ENTRY_FAILURE)) {
1861 		launched = 1;
1862 		check_for_guest_termination();
1863 	}
1864 }
1865 
1866 void enter_guest_with_bad_controls(void)
1867 {
1868 	struct vmentry_failure failure = {0};
1869 
1870 	TEST_ASSERT_MSG(v2_guest_main,
1871 			"Never called test_set_guest_func!");
1872 
1873 	TEST_ASSERT_MSG(!guest_finished,
1874 			"Called enter_guest() after guest returned.");
1875 
1876 	__enter_guest(ABORT_ON_INVALID_GUEST_STATE, &failure);
1877 	report(failure.early, "failure occurred early");
1878 	report((failure.flags & VMX_ENTRY_FLAGS) == X86_EFLAGS_ZF,
1879                "FLAGS set correctly");
1880 	report(vmcs_read(VMX_INST_ERROR) == VMXERR_ENTRY_INVALID_CONTROL_FIELD,
1881 	       "VM-Inst Error # is %d (VM entry with invalid control field(s))",
1882 	       VMXERR_ENTRY_INVALID_CONTROL_FIELD);
1883 
1884 	/*
1885 	 * This if statement shouldn't fire, as the entire premise of this
1886 	 * function is that VM entry is expected to fail, rather than succeed
1887 	 * and execute to termination. However, if the VM entry does
1888 	 * unexpectedly succeed, it's nice to check whether the guest has
1889 	 * terminated, to reduce the number of error messages.
1890 	 */
1891 	if (!failure.early)
1892 		check_for_guest_termination();
1893 }
1894 
1895 void enter_guest(void)
1896 {
1897 	struct vmentry_failure failure = {0};
1898 
1899 	__enter_guest(ABORT_ON_EARLY_VMENTRY_FAIL |
1900 		      ABORT_ON_INVALID_GUEST_STATE, &failure);
1901 }
1902 
1903 void enter_guest_with_invalid_guest_state(void)
1904 {
1905 	struct vmentry_failure failure = {0};
1906 
1907 	__enter_guest(ABORT_ON_EARLY_VMENTRY_FAIL, &failure);
1908 }
1909 
1910 extern struct vmx_test vmx_tests[];
1911 
1912 static bool
1913 test_wanted(const char *name, const char *filters[], int filter_count)
1914 {
1915 	int i;
1916 	bool positive = false;
1917 	bool match = false;
1918 	char clean_name[strlen(name) + 1];
1919 	char *c;
1920 	const char *n;
1921 
1922 	/* Replace spaces with underscores. */
1923 	n = name;
1924 	c = &clean_name[0];
1925 	do *c++ = (*n == ' ') ? '_' : *n;
1926 	while (*n++);
1927 
1928 	for (i = 0; i < filter_count; i++) {
1929 		const char *filter = filters[i];
1930 
1931 		if (filter[0] == '-') {
1932 			if (simple_glob(clean_name, filter + 1))
1933 				return false;
1934 		} else {
1935 			positive = true;
1936 			match |= simple_glob(clean_name, filter);
1937 		}
1938 	}
1939 
1940 	if (!positive || match) {
1941 		matched++;
1942 		return true;
1943 	} else {
1944 		return false;
1945 	}
1946 }
1947 
1948 int main(int argc, const char *argv[])
1949 {
1950 	int i = 0;
1951 
1952 	setup_vm();
1953 	smp_init();
1954 	hypercall_field = 0;
1955 
1956 	/* We want xAPIC mode to test MMIO passthrough from L1 (us) to L2.  */
1957 	reset_apic();
1958 
1959 	argv++;
1960 	argc--;
1961 
1962 	if (!this_cpu_has(X86_FEATURE_VMX)) {
1963 		printf("WARNING: vmx not supported, add '-cpu host'\n");
1964 		goto exit;
1965 	}
1966 	init_bsp_vmx();
1967 	if (test_wanted("test_vmx_feature_control", argv, argc)) {
1968 		/* Sets MSR_IA32_FEATURE_CONTROL to 0x5 */
1969 		if (test_vmx_feature_control() != 0)
1970 			goto exit;
1971 	} else {
1972 		enable_vmx();
1973 	}
1974 
1975 	if (test_wanted("test_vmxon", argv, argc)) {
1976 		/* Enables VMX */
1977 		if (test_vmxon() != 0)
1978 			goto exit;
1979 	} else {
1980 		if (vmx_on()) {
1981 			report(0, "vmxon");
1982 			goto exit;
1983 		}
1984 	}
1985 
1986 	if (test_wanted("test_vmptrld", argv, argc))
1987 		test_vmptrld();
1988 	if (test_wanted("test_vmclear", argv, argc))
1989 		test_vmclear();
1990 	if (test_wanted("test_vmptrst", argv, argc))
1991 		test_vmptrst();
1992 	if (test_wanted("test_vmwrite_vmread", argv, argc))
1993 		test_vmwrite_vmread();
1994 	if (test_wanted("test_vmcs_high", argv, argc))
1995 		test_vmcs_high();
1996 	if (test_wanted("test_vmcs_lifecycle", argv, argc))
1997 		test_vmcs_lifecycle();
1998 	if (test_wanted("test_vmx_caps", argv, argc))
1999 		test_vmx_caps();
2000 
2001 	/* Balance vmxon from test_vmxon. */
2002 	vmx_off();
2003 
2004 	for (; vmx_tests[i].name != NULL; i++) {
2005 		if (!test_wanted(vmx_tests[i].name, argv, argc))
2006 			continue;
2007 		if (test_run(&vmx_tests[i]))
2008 			goto exit;
2009 	}
2010 
2011 	if (!matched)
2012 		report(matched, "command line didn't match any tests!");
2013 
2014 exit:
2015 	return report_summary();
2016 }
2017