xref: /kvm-unit-tests/x86/vmx.h (revision 6163f75d09a0a96a5c3db82dd768b13f79629c00)
1 #ifndef __VMX_H
2 #define __VMX_H
3 
4 #include "libcflat.h"
5 #include "processor.h"
6 #include "bitops.h"
7 #include "asm/page.h"
8 #include "asm/io.h"
9 
10 struct vmcs_hdr {
11 	u32 revision_id:31;
12 	u32 shadow_vmcs:1;
13 };
14 
15 struct vmcs {
16 	struct vmcs_hdr hdr;
17 	u32 abort; /* VMX-abort indicator */
18 	/* VMCS data */
19 	char data[0];
20 };
21 
22 struct invvpid_operand {
23 	u64 vpid;
24 	u64 gla;
25 };
26 
27 struct regs {
28 	u64 rax;
29 	u64 rcx;
30 	u64 rdx;
31 	u64 rbx;
32 	u64 cr2;
33 	u64 rbp;
34 	u64 rsi;
35 	u64 rdi;
36 	u64 r8;
37 	u64 r9;
38 	u64 r10;
39 	u64 r11;
40 	u64 r12;
41 	u64 r13;
42 	u64 r14;
43 	u64 r15;
44 	u64 rflags;
45 };
46 
47 struct vmentry_failure {
48 	/* Did a vmlaunch or vmresume fail? */
49 	bool vmlaunch;
50 	/* Instruction mnemonic (for convenience). */
51 	const char *instr;
52 	/* Did the instruction return right away, or did we jump to HOST_RIP? */
53 	bool early;
54 	/* Contents of [re]flags after failed entry. */
55 	unsigned long flags;
56 };
57 
58 struct vmx_test {
59 	const char *name;
60 	int (*init)(struct vmcs *vmcs);
61 	void (*guest_main)(void);
62 	int (*exit_handler)(void);
63 	void (*syscall_handler)(u64 syscall_no);
64 	struct regs guest_regs;
65 	int (*entry_failure_handler)(struct vmentry_failure *failure);
66 	struct vmcs *vmcs;
67 	int exits;
68 	/* Alternative test interface. */
69 	void (*v2)(void);
70 };
71 
72 union vmx_basic {
73 	u64 val;
74 	struct {
75 		u32 revision;
76 		u32	size:13,
77 			reserved1: 3,
78 			width:1,
79 			dual:1,
80 			type:4,
81 			insouts:1,
82 			ctrl:1,
83 			reserved2:8;
84 	};
85 };
86 
87 union vmx_ctrl_msr {
88 	u64 val;
89 	struct {
90 		u32 set, clr;
91 	};
92 };
93 
94 union vmx_ept_vpid {
95 	u64 val;
96 	struct {
97 		u32:16,
98 			super:2,
99 			: 2,
100 			invept:1,
101 			: 11;
102 		u32	invvpid:1;
103 	};
104 };
105 
106 enum Encoding {
107 	/* 16-Bit Control Fields */
108 	VPID			= 0x0000ul,
109 	/* Posted-interrupt notification vector */
110 	PINV			= 0x0002ul,
111 	/* EPTP index */
112 	EPTP_IDX		= 0x0004ul,
113 
114 	/* 16-Bit Guest State Fields */
115 	GUEST_SEL_ES		= 0x0800ul,
116 	GUEST_SEL_CS		= 0x0802ul,
117 	GUEST_SEL_SS		= 0x0804ul,
118 	GUEST_SEL_DS		= 0x0806ul,
119 	GUEST_SEL_FS		= 0x0808ul,
120 	GUEST_SEL_GS		= 0x080aul,
121 	GUEST_SEL_LDTR		= 0x080cul,
122 	GUEST_SEL_TR		= 0x080eul,
123 	GUEST_INT_STATUS	= 0x0810ul,
124 	GUEST_PML_INDEX         = 0x0812ul,
125 
126 	/* 16-Bit Host State Fields */
127 	HOST_SEL_ES		= 0x0c00ul,
128 	HOST_SEL_CS		= 0x0c02ul,
129 	HOST_SEL_SS		= 0x0c04ul,
130 	HOST_SEL_DS		= 0x0c06ul,
131 	HOST_SEL_FS		= 0x0c08ul,
132 	HOST_SEL_GS		= 0x0c0aul,
133 	HOST_SEL_TR		= 0x0c0cul,
134 
135 	/* 64-Bit Control Fields */
136 	IO_BITMAP_A		= 0x2000ul,
137 	IO_BITMAP_B		= 0x2002ul,
138 	MSR_BITMAP		= 0x2004ul,
139 	EXIT_MSR_ST_ADDR	= 0x2006ul,
140 	EXIT_MSR_LD_ADDR	= 0x2008ul,
141 	ENTER_MSR_LD_ADDR	= 0x200aul,
142 	VMCS_EXEC_PTR		= 0x200cul,
143 	TSC_OFFSET		= 0x2010ul,
144 	TSC_OFFSET_HI		= 0x2011ul,
145 	APIC_VIRT_ADDR		= 0x2012ul,
146 	APIC_ACCS_ADDR		= 0x2014ul,
147 	POSTED_INTR_DESC_ADDR	= 0x2016ul,
148 	EPTP			= 0x201aul,
149 	EPTP_HI			= 0x201bul,
150 	VMREAD_BITMAP           = 0x2026ul,
151 	VMREAD_BITMAP_HI        = 0x2027ul,
152 	VMWRITE_BITMAP          = 0x2028ul,
153 	VMWRITE_BITMAP_HI       = 0x2029ul,
154 	EOI_EXIT_BITMAP0	= 0x201cul,
155 	EOI_EXIT_BITMAP1	= 0x201eul,
156 	EOI_EXIT_BITMAP2	= 0x2020ul,
157 	EOI_EXIT_BITMAP3	= 0x2022ul,
158 	PMLADDR                 = 0x200eul,
159 	PMLADDR_HI              = 0x200ful,
160 
161 
162 	/* 64-Bit Readonly Data Field */
163 	INFO_PHYS_ADDR		= 0x2400ul,
164 
165 	/* 64-Bit Guest State */
166 	VMCS_LINK_PTR		= 0x2800ul,
167 	VMCS_LINK_PTR_HI	= 0x2801ul,
168 	GUEST_DEBUGCTL		= 0x2802ul,
169 	GUEST_DEBUGCTL_HI	= 0x2803ul,
170 	GUEST_EFER		= 0x2806ul,
171 	GUEST_PAT		= 0x2804ul,
172 	GUEST_PERF_GLOBAL_CTRL	= 0x2808ul,
173 	GUEST_PDPTE		= 0x280aul,
174 
175 	/* 64-Bit Host State */
176 	HOST_PAT		= 0x2c00ul,
177 	HOST_EFER		= 0x2c02ul,
178 	HOST_PERF_GLOBAL_CTRL	= 0x2c04ul,
179 
180 	/* 32-Bit Control Fields */
181 	PIN_CONTROLS		= 0x4000ul,
182 	CPU_EXEC_CTRL0		= 0x4002ul,
183 	EXC_BITMAP		= 0x4004ul,
184 	PF_ERROR_MASK		= 0x4006ul,
185 	PF_ERROR_MATCH		= 0x4008ul,
186 	CR3_TARGET_COUNT	= 0x400aul,
187 	EXI_CONTROLS		= 0x400cul,
188 	EXI_MSR_ST_CNT		= 0x400eul,
189 	EXI_MSR_LD_CNT		= 0x4010ul,
190 	ENT_CONTROLS		= 0x4012ul,
191 	ENT_MSR_LD_CNT		= 0x4014ul,
192 	ENT_INTR_INFO		= 0x4016ul,
193 	ENT_INTR_ERROR		= 0x4018ul,
194 	ENT_INST_LEN		= 0x401aul,
195 	TPR_THRESHOLD		= 0x401cul,
196 	CPU_EXEC_CTRL1		= 0x401eul,
197 
198 	/* 32-Bit R/O Data Fields */
199 	VMX_INST_ERROR		= 0x4400ul,
200 	EXI_REASON		= 0x4402ul,
201 	EXI_INTR_INFO		= 0x4404ul,
202 	EXI_INTR_ERROR		= 0x4406ul,
203 	IDT_VECT_INFO		= 0x4408ul,
204 	IDT_VECT_ERROR		= 0x440aul,
205 	EXI_INST_LEN		= 0x440cul,
206 	EXI_INST_INFO		= 0x440eul,
207 
208 	/* 32-Bit Guest State Fields */
209 	GUEST_LIMIT_ES		= 0x4800ul,
210 	GUEST_LIMIT_CS		= 0x4802ul,
211 	GUEST_LIMIT_SS		= 0x4804ul,
212 	GUEST_LIMIT_DS		= 0x4806ul,
213 	GUEST_LIMIT_FS		= 0x4808ul,
214 	GUEST_LIMIT_GS		= 0x480aul,
215 	GUEST_LIMIT_LDTR	= 0x480cul,
216 	GUEST_LIMIT_TR		= 0x480eul,
217 	GUEST_LIMIT_GDTR	= 0x4810ul,
218 	GUEST_LIMIT_IDTR	= 0x4812ul,
219 	GUEST_AR_ES		= 0x4814ul,
220 	GUEST_AR_CS		= 0x4816ul,
221 	GUEST_AR_SS		= 0x4818ul,
222 	GUEST_AR_DS		= 0x481aul,
223 	GUEST_AR_FS		= 0x481cul,
224 	GUEST_AR_GS		= 0x481eul,
225 	GUEST_AR_LDTR		= 0x4820ul,
226 	GUEST_AR_TR		= 0x4822ul,
227 	GUEST_INTR_STATE	= 0x4824ul,
228 	GUEST_ACTV_STATE	= 0x4826ul,
229 	GUEST_SMBASE		= 0x4828ul,
230 	GUEST_SYSENTER_CS	= 0x482aul,
231 	PREEMPT_TIMER_VALUE	= 0x482eul,
232 
233 	/* 32-Bit Host State Fields */
234 	HOST_SYSENTER_CS	= 0x4c00ul,
235 
236 	/* Natural-Width Control Fields */
237 	CR0_MASK		= 0x6000ul,
238 	CR4_MASK		= 0x6002ul,
239 	CR0_READ_SHADOW		= 0x6004ul,
240 	CR4_READ_SHADOW		= 0x6006ul,
241 	CR3_TARGET_0		= 0x6008ul,
242 	CR3_TARGET_1		= 0x600aul,
243 	CR3_TARGET_2		= 0x600cul,
244 	CR3_TARGET_3		= 0x600eul,
245 
246 	/* Natural-Width R/O Data Fields */
247 	EXI_QUALIFICATION	= 0x6400ul,
248 	IO_RCX			= 0x6402ul,
249 	IO_RSI			= 0x6404ul,
250 	IO_RDI			= 0x6406ul,
251 	IO_RIP			= 0x6408ul,
252 	GUEST_LINEAR_ADDRESS	= 0x640aul,
253 
254 	/* Natural-Width Guest State Fields */
255 	GUEST_CR0		= 0x6800ul,
256 	GUEST_CR3		= 0x6802ul,
257 	GUEST_CR4		= 0x6804ul,
258 	GUEST_BASE_ES		= 0x6806ul,
259 	GUEST_BASE_CS		= 0x6808ul,
260 	GUEST_BASE_SS		= 0x680aul,
261 	GUEST_BASE_DS		= 0x680cul,
262 	GUEST_BASE_FS		= 0x680eul,
263 	GUEST_BASE_GS		= 0x6810ul,
264 	GUEST_BASE_LDTR		= 0x6812ul,
265 	GUEST_BASE_TR		= 0x6814ul,
266 	GUEST_BASE_GDTR		= 0x6816ul,
267 	GUEST_BASE_IDTR		= 0x6818ul,
268 	GUEST_DR7		= 0x681aul,
269 	GUEST_RSP		= 0x681cul,
270 	GUEST_RIP		= 0x681eul,
271 	GUEST_RFLAGS		= 0x6820ul,
272 	GUEST_PENDING_DEBUG	= 0x6822ul,
273 	GUEST_SYSENTER_ESP	= 0x6824ul,
274 	GUEST_SYSENTER_EIP	= 0x6826ul,
275 
276 	/* Natural-Width Host State Fields */
277 	HOST_CR0		= 0x6c00ul,
278 	HOST_CR3		= 0x6c02ul,
279 	HOST_CR4		= 0x6c04ul,
280 	HOST_BASE_FS		= 0x6c06ul,
281 	HOST_BASE_GS		= 0x6c08ul,
282 	HOST_BASE_TR		= 0x6c0aul,
283 	HOST_BASE_GDTR		= 0x6c0cul,
284 	HOST_BASE_IDTR		= 0x6c0eul,
285 	HOST_SYSENTER_ESP	= 0x6c10ul,
286 	HOST_SYSENTER_EIP	= 0x6c12ul,
287 	HOST_RSP		= 0x6c14ul,
288 	HOST_RIP		= 0x6c16ul
289 };
290 
291 #define VMX_ENTRY_FAILURE	(1ul << 31)
292 #define VMX_ENTRY_FLAGS		(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \
293 				 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)
294 
295 enum Reason {
296 	VMX_EXC_NMI		= 0,
297 	VMX_EXTINT		= 1,
298 	VMX_TRIPLE_FAULT	= 2,
299 	VMX_INIT		= 3,
300 	VMX_SIPI		= 4,
301 	VMX_SMI_IO		= 5,
302 	VMX_SMI_OTHER		= 6,
303 	VMX_INTR_WINDOW		= 7,
304 	VMX_NMI_WINDOW		= 8,
305 	VMX_TASK_SWITCH		= 9,
306 	VMX_CPUID		= 10,
307 	VMX_GETSEC		= 11,
308 	VMX_HLT			= 12,
309 	VMX_INVD		= 13,
310 	VMX_INVLPG		= 14,
311 	VMX_RDPMC		= 15,
312 	VMX_RDTSC		= 16,
313 	VMX_RSM			= 17,
314 	VMX_VMCALL		= 18,
315 	VMX_VMCLEAR		= 19,
316 	VMX_VMLAUNCH		= 20,
317 	VMX_VMPTRLD		= 21,
318 	VMX_VMPTRST		= 22,
319 	VMX_VMREAD		= 23,
320 	VMX_VMRESUME		= 24,
321 	VMX_VMWRITE		= 25,
322 	VMX_VMXOFF		= 26,
323 	VMX_VMXON		= 27,
324 	VMX_CR			= 28,
325 	VMX_DR			= 29,
326 	VMX_IO			= 30,
327 	VMX_RDMSR		= 31,
328 	VMX_WRMSR		= 32,
329 	VMX_FAIL_STATE		= 33,
330 	VMX_FAIL_MSR		= 34,
331 	VMX_MWAIT		= 36,
332 	VMX_MTF			= 37,
333 	VMX_MONITOR		= 39,
334 	VMX_PAUSE		= 40,
335 	VMX_FAIL_MCHECK		= 41,
336 	VMX_TPR_THRESHOLD	= 43,
337 	VMX_APIC_ACCESS		= 44,
338 	VMX_EOI_INDUCED		= 45,
339 	VMX_GDTR_IDTR		= 46,
340 	VMX_LDTR_TR		= 47,
341 	VMX_EPT_VIOLATION	= 48,
342 	VMX_EPT_MISCONFIG	= 49,
343 	VMX_INVEPT		= 50,
344 	VMX_PREEMPT		= 52,
345 	VMX_INVVPID		= 53,
346 	VMX_WBINVD		= 54,
347 	VMX_XSETBV		= 55,
348 	VMX_APIC_WRITE		= 56,
349 	VMX_RDRAND		= 57,
350 	VMX_INVPCID		= 58,
351 	VMX_VMFUNC		= 59,
352 	VMX_RDSEED		= 61,
353 	VMX_PML_FULL		= 62,
354 	VMX_XSAVES		= 63,
355 	VMX_XRSTORS		= 64,
356 };
357 
358 enum Ctrl_exi {
359 	EXI_SAVE_DBGCTLS	= 1UL << 2,
360 	EXI_HOST_64		= 1UL << 9,
361 	EXI_LOAD_PERF		= 1UL << 12,
362 	EXI_INTA		= 1UL << 15,
363 	EXI_SAVE_PAT		= 1UL << 18,
364 	EXI_LOAD_PAT		= 1UL << 19,
365 	EXI_SAVE_EFER		= 1UL << 20,
366 	EXI_LOAD_EFER		= 1UL << 21,
367 	EXI_SAVE_PREEMPT	= 1UL << 22,
368 };
369 
370 enum Ctrl_ent {
371 	ENT_LOAD_DBGCTLS	= 1UL << 2,
372 	ENT_GUEST_64		= 1UL << 9,
373 	ENT_LOAD_PERF		= 1UL << 13,
374 	ENT_LOAD_PAT		= 1UL << 14,
375 	ENT_LOAD_EFER		= 1UL << 15,
376 };
377 
378 enum Ctrl_pin {
379 	PIN_EXTINT		= 1ul << 0,
380 	PIN_NMI			= 1ul << 3,
381 	PIN_VIRT_NMI		= 1ul << 5,
382 	PIN_PREEMPT		= 1ul << 6,
383 	PIN_POST_INTR		= 1ul << 7,
384 };
385 
386 enum Ctrl0 {
387 	CPU_INTR_WINDOW		= 1ul << 2,
388 	CPU_USE_TSC_OFFSET	= 1ul << 3,
389 	CPU_HLT			= 1ul << 7,
390 	CPU_INVLPG		= 1ul << 9,
391 	CPU_MWAIT		= 1ul << 10,
392 	CPU_RDPMC		= 1ul << 11,
393 	CPU_RDTSC		= 1ul << 12,
394 	CPU_CR3_LOAD		= 1ul << 15,
395 	CPU_CR3_STORE		= 1ul << 16,
396 	CPU_CR8_LOAD		= 1ul << 19,
397 	CPU_CR8_STORE		= 1ul << 20,
398 	CPU_TPR_SHADOW		= 1ul << 21,
399 	CPU_NMI_WINDOW		= 1ul << 22,
400 	CPU_IO			= 1ul << 24,
401 	CPU_IO_BITMAP		= 1ul << 25,
402 	CPU_MSR_BITMAP		= 1ul << 28,
403 	CPU_MONITOR		= 1ul << 29,
404 	CPU_PAUSE		= 1ul << 30,
405 	CPU_SECONDARY		= 1ul << 31,
406 };
407 
408 enum Ctrl1 {
409 	CPU_VIRT_APIC_ACCESSES	= 1ul << 0,
410 	CPU_EPT			= 1ul << 1,
411 	CPU_DESC_TABLE		= 1ul << 2,
412 	CPU_RDTSCP		= 1ul << 3,
413 	CPU_VIRT_X2APIC		= 1ul << 4,
414 	CPU_VPID		= 1ul << 5,
415 	CPU_WBINVD		= 1ul << 6,
416 	CPU_URG			= 1ul << 7,
417 	CPU_APIC_REG_VIRT	= 1ul << 8,
418 	CPU_VINTD		= 1ul << 9,
419 	CPU_RDRAND		= 1ul << 11,
420 	CPU_SHADOW_VMCS		= 1ul << 14,
421 	CPU_RDSEED		= 1ul << 16,
422 	CPU_PML                 = 1ul << 17,
423 };
424 
425 enum Intr_type {
426 	VMX_INTR_TYPE_EXT_INTR = 0,
427 	VMX_INTR_TYPE_NMI_INTR = 2,
428 	VMX_INTR_TYPE_HARD_EXCEPTION = 3,
429 	VMX_INTR_TYPE_SOFT_INTR = 4,
430 	VMX_INTR_TYPE_SOFT_EXCEPTION = 6,
431 };
432 
433 /*
434  * Interruption-information format
435  */
436 #define INTR_INFO_VECTOR_MASK           0xff            /* 7:0 */
437 #define INTR_INFO_INTR_TYPE_MASK        0x700           /* 10:8 */
438 #define INTR_INFO_DELIVER_CODE_MASK     0x800           /* 11 */
439 #define INTR_INFO_UNBLOCK_NMI_MASK      0x1000          /* 12 */
440 #define INTR_INFO_VALID_MASK            0x80000000      /* 31 */
441 
442 #define INTR_INFO_INTR_TYPE_SHIFT       8
443 
444 #define INTR_TYPE_EXT_INTR              (0 << 8) /* external interrupt */
445 #define INTR_TYPE_RESERVED              (1 << 8) /* reserved */
446 #define INTR_TYPE_NMI_INTR		(2 << 8) /* NMI */
447 #define INTR_TYPE_HARD_EXCEPTION	(3 << 8) /* processor exception */
448 #define INTR_TYPE_SOFT_INTR             (4 << 8) /* software interrupt */
449 #define INTR_TYPE_PRIV_SW_EXCEPTION	(5 << 8) /* priv. software exception */
450 #define INTR_TYPE_SOFT_EXCEPTION	(6 << 8) /* software exception */
451 #define INTR_TYPE_OTHER_EVENT           (7 << 8) /* other event */
452 
453 /*
454  * Guest interruptibility state
455  */
456 #define GUEST_INTR_STATE_STI		(1 << 0)
457 #define GUEST_INTR_STATE_MOVSS		(1 << 1)
458 #define GUEST_INTR_STATE_SMI		(1 << 2)
459 #define GUEST_INTR_STATE_NMI		(1 << 3)
460 #define GUEST_INTR_STATE_ENCLAVE	(1 << 4)
461 
462 /*
463  * VM-instruction error numbers
464  */
465 enum vm_instruction_error_number {
466 	VMXERR_VMCALL_IN_VMX_ROOT_OPERATION = 1,
467 	VMXERR_VMCLEAR_INVALID_ADDRESS = 2,
468 	VMXERR_VMCLEAR_VMXON_POINTER = 3,
469 	VMXERR_VMLAUNCH_NONCLEAR_VMCS = 4,
470 	VMXERR_VMRESUME_NONLAUNCHED_VMCS = 5,
471 	VMXERR_VMRESUME_AFTER_VMXOFF = 6,
472 	VMXERR_ENTRY_INVALID_CONTROL_FIELD = 7,
473 	VMXERR_ENTRY_INVALID_HOST_STATE_FIELD = 8,
474 	VMXERR_VMPTRLD_INVALID_ADDRESS = 9,
475 	VMXERR_VMPTRLD_VMXON_POINTER = 10,
476 	VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID = 11,
477 	VMXERR_UNSUPPORTED_VMCS_COMPONENT = 12,
478 	VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT = 13,
479 	VMXERR_VMXON_IN_VMX_ROOT_OPERATION = 15,
480 	VMXERR_ENTRY_INVALID_EXECUTIVE_VMCS_POINTER = 16,
481 	VMXERR_ENTRY_NONLAUNCHED_EXECUTIVE_VMCS = 17,
482 	VMXERR_ENTRY_EXECUTIVE_VMCS_POINTER_NOT_VMXON_POINTER = 18,
483 	VMXERR_VMCALL_NONCLEAR_VMCS = 19,
484 	VMXERR_VMCALL_INVALID_VM_EXIT_CONTROL_FIELDS = 20,
485 	VMXERR_VMCALL_INCORRECT_MSEG_REVISION_ID = 22,
486 	VMXERR_VMXOFF_UNDER_DUAL_MONITOR_TREATMENT_OF_SMIS_AND_SMM = 23,
487 	VMXERR_VMCALL_INVALID_SMM_MONITOR_FEATURES = 24,
488 	VMXERR_ENTRY_INVALID_VM_EXECUTION_CONTROL_FIELDS_IN_EXECUTIVE_VMCS = 25,
489 	VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS = 26,
490 	VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28,
491 };
492 
493 #define SAVE_GPR				\
494 	"xchg %rax, regs\n\t"			\
495 	"xchg %rcx, regs+0x8\n\t"		\
496 	"xchg %rdx, regs+0x10\n\t"		\
497 	"xchg %rbx, regs+0x18\n\t"		\
498 	"xchg %rbp, regs+0x28\n\t"		\
499 	"xchg %rsi, regs+0x30\n\t"		\
500 	"xchg %rdi, regs+0x38\n\t"		\
501 	"xchg %r8, regs+0x40\n\t"		\
502 	"xchg %r9, regs+0x48\n\t"		\
503 	"xchg %r10, regs+0x50\n\t"		\
504 	"xchg %r11, regs+0x58\n\t"		\
505 	"xchg %r12, regs+0x60\n\t"		\
506 	"xchg %r13, regs+0x68\n\t"		\
507 	"xchg %r14, regs+0x70\n\t"		\
508 	"xchg %r15, regs+0x78\n\t"
509 
510 #define LOAD_GPR	SAVE_GPR
511 
512 #define SAVE_GPR_C				\
513 	"xchg %%rax, regs\n\t"			\
514 	"xchg %%rcx, regs+0x8\n\t"		\
515 	"xchg %%rdx, regs+0x10\n\t"		\
516 	"xchg %%rbx, regs+0x18\n\t"		\
517 	"xchg %%rbp, regs+0x28\n\t"		\
518 	"xchg %%rsi, regs+0x30\n\t"		\
519 	"xchg %%rdi, regs+0x38\n\t"		\
520 	"xchg %%r8, regs+0x40\n\t"		\
521 	"xchg %%r9, regs+0x48\n\t"		\
522 	"xchg %%r10, regs+0x50\n\t"		\
523 	"xchg %%r11, regs+0x58\n\t"		\
524 	"xchg %%r12, regs+0x60\n\t"		\
525 	"xchg %%r13, regs+0x68\n\t"		\
526 	"xchg %%r14, regs+0x70\n\t"		\
527 	"xchg %%r15, regs+0x78\n\t"
528 
529 #define LOAD_GPR_C	SAVE_GPR_C
530 
531 #define VMX_IO_SIZE_MASK	0x7
532 #define _VMX_IO_BYTE		0
533 #define _VMX_IO_WORD		1
534 #define _VMX_IO_LONG		3
535 #define VMX_IO_DIRECTION_MASK	(1ul << 3)
536 #define VMX_IO_IN		(1ul << 3)
537 #define VMX_IO_OUT		0
538 #define VMX_IO_STRING		(1ul << 4)
539 #define VMX_IO_REP		(1ul << 5)
540 #define VMX_IO_OPRAND_IMM	(1ul << 6)
541 #define VMX_IO_PORT_MASK	0xFFFF0000
542 #define VMX_IO_PORT_SHIFT	16
543 
544 #define VMX_TEST_START		0
545 #define VMX_TEST_VMEXIT		1
546 #define VMX_TEST_EXIT		2
547 #define VMX_TEST_RESUME		3
548 #define VMX_TEST_VMABORT	4
549 #define VMX_TEST_VMSKIP		5
550 
551 #define HYPERCALL_BIT		(1ul << 12)
552 #define HYPERCALL_MASK		0xFFF
553 #define HYPERCALL_VMEXIT	0x1
554 #define HYPERCALL_VMABORT	0x2
555 #define HYPERCALL_VMSKIP	0x3
556 
557 #define EPTP_PG_WALK_LEN_SHIFT	3ul
558 #define EPTP_PG_WALK_LEN_MASK	0x38ul
559 #define EPTP_RESERV_BITS_MASK	0x1ful
560 #define EPTP_RESERV_BITS_SHIFT	0x7ul
561 #define EPTP_AD_FLAG		(1ul << 6)
562 
563 #define EPT_MEM_TYPE_UC		0ul
564 #define EPT_MEM_TYPE_WC		1ul
565 #define EPT_MEM_TYPE_WT		4ul
566 #define EPT_MEM_TYPE_WP		5ul
567 #define EPT_MEM_TYPE_WB		6ul
568 
569 #define EPT_RA			1ul
570 #define EPT_WA			2ul
571 #define EPT_EA			4ul
572 #define EPT_PRESENT		(EPT_RA | EPT_WA | EPT_EA)
573 #define EPT_ACCESS_FLAG		(1ul << 8)
574 #define EPT_DIRTY_FLAG		(1ul << 9)
575 #define EPT_LARGE_PAGE		(1ul << 7)
576 #define EPT_MEM_TYPE_SHIFT	3ul
577 #define EPT_MEM_TYPE_MASK	0x7ul
578 #define EPT_IGNORE_PAT		(1ul << 6)
579 #define EPT_SUPPRESS_VE		(1ull << 63)
580 
581 #define EPT_CAP_WT		1ull
582 #define EPT_CAP_PWL4		(1ull << 6)
583 #define EPT_CAP_UC		(1ull << 8)
584 #define EPT_CAP_WB		(1ull << 14)
585 #define EPT_CAP_2M_PAGE		(1ull << 16)
586 #define EPT_CAP_1G_PAGE		(1ull << 17)
587 #define EPT_CAP_INVEPT		(1ull << 20)
588 #define EPT_CAP_INVEPT_SINGLE	(1ull << 25)
589 #define EPT_CAP_INVEPT_ALL	(1ull << 26)
590 #define EPT_CAP_AD_FLAG		(1ull << 21)
591 #define VPID_CAP_INVVPID	(1ull << 32)
592 #define VPID_CAP_INVVPID_ADDR   (1ull << 40)
593 #define VPID_CAP_INVVPID_CXTGLB (1ull << 41)
594 #define VPID_CAP_INVVPID_ALL    (1ull << 42)
595 #define VPID_CAP_INVVPID_CXTLOC	(1ull << 43)
596 
597 #define PAGE_SIZE_2M		(512 * PAGE_SIZE)
598 #define PAGE_SIZE_1G		(512 * PAGE_SIZE_2M)
599 #define EPT_PAGE_LEVEL		4
600 #define EPT_PGDIR_WIDTH		9
601 #define EPT_PGDIR_MASK		511
602 #define EPT_PGDIR_ENTRIES	(1 << EPT_PGDIR_WIDTH)
603 #define EPT_LEVEL_SHIFT(level)	(((level)-1) * EPT_PGDIR_WIDTH + 12)
604 #define EPT_ADDR_MASK		GENMASK_ULL(51, 12)
605 #define PAGE_MASK_2M		(~(PAGE_SIZE_2M-1))
606 
607 #define EPT_VLT_RD		(1ull << 0)
608 #define EPT_VLT_WR		(1ull << 1)
609 #define EPT_VLT_FETCH		(1ull << 2)
610 #define EPT_VLT_PERM_RD		(1ull << 3)
611 #define EPT_VLT_PERM_WR		(1ull << 4)
612 #define EPT_VLT_PERM_EX		(1ull << 5)
613 #define EPT_VLT_PERM_USER_EX	(1ull << 6)
614 #define EPT_VLT_PERMS		(EPT_VLT_PERM_RD | EPT_VLT_PERM_WR | \
615 				 EPT_VLT_PERM_EX)
616 #define EPT_VLT_LADDR_VLD	(1ull << 7)
617 #define EPT_VLT_PADDR		(1ull << 8)
618 #define EPT_VLT_GUEST_USER	(1ull << 9)
619 #define EPT_VLT_GUEST_RW	(1ull << 10)
620 #define EPT_VLT_GUEST_EX	(1ull << 11)
621 #define EPT_VLT_GUEST_MASK	(EPT_VLT_GUEST_USER | EPT_VLT_GUEST_RW | \
622 				 EPT_VLT_GUEST_EX)
623 
624 #define MAGIC_VAL_1		0x12345678ul
625 #define MAGIC_VAL_2		0x87654321ul
626 #define MAGIC_VAL_3		0xfffffffful
627 #define MAGIC_VAL_4		0xdeadbeeful
628 
629 #define INVEPT_SINGLE		1
630 #define INVEPT_GLOBAL		2
631 
632 #define INVVPID_ADDR            0
633 #define INVVPID_CONTEXT_GLOBAL	1
634 #define INVVPID_ALL		2
635 #define INVVPID_CONTEXT_LOCAL	3
636 
637 #define ACTV_ACTIVE		0
638 #define ACTV_HLT		1
639 
640 /*
641  * VMCS field encoding:
642  * Bit 0: High-access
643  * Bits 1-9: Index
644  * Bits 10-12: Type
645  * Bits 13-15: Width
646  * Bits 15-64: Reserved
647  */
648 #define VMCS_FIELD_HIGH_SHIFT		(0)
649 #define VMCS_FIELD_INDEX_SHIFT		(1)
650 #define VMCS_FIELD_INDEX_MASK		GENMASK(9, 1)
651 #define VMCS_FIELD_TYPE_SHIFT		(10)
652 #define VMCS_FIELD_WIDTH_SHIFT		(13)
653 #define VMCS_FIELD_RESERVED_SHIFT	(15)
654 #define VMCS_FIELD_BIT_SIZE		(BITS_PER_LONG)
655 
656 extern struct regs regs;
657 
658 extern union vmx_basic basic;
659 extern union vmx_ctrl_msr ctrl_pin_rev;
660 extern union vmx_ctrl_msr ctrl_cpu_rev[2];
661 extern union vmx_ctrl_msr ctrl_exit_rev;
662 extern union vmx_ctrl_msr ctrl_enter_rev;
663 extern union vmx_ept_vpid  ept_vpid;
664 
665 extern u64 *bsp_vmxon_region;
666 extern bool launched;
667 
668 void vmx_set_test_stage(u32 s);
669 u32 vmx_get_test_stage(void);
670 void vmx_inc_test_stage(void);
671 
672 static int _vmx_on(u64 *vmxon_region)
673 {
674 	bool ret;
675 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
676 	asm volatile ("push %1; popf; vmxon %2; setbe %0\n\t"
677 		      : "=q" (ret) : "q" (rflags), "m" (vmxon_region) : "cc");
678 	return ret;
679 }
680 
681 static int vmx_on(void)
682 {
683 	return _vmx_on(bsp_vmxon_region);
684 }
685 
686 static int vmx_off(void)
687 {
688 	bool ret;
689 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
690 
691 	asm volatile("push %1; popf; vmxoff; setbe %0\n\t"
692 		     : "=q"(ret) : "q" (rflags) : "cc");
693 	return ret;
694 }
695 
696 static inline int make_vmcs_current(struct vmcs *vmcs)
697 {
698 	bool ret;
699 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
700 
701 	asm volatile ("push %1; popf; vmptrld %2; setbe %0"
702 		      : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc");
703 	return ret;
704 }
705 
706 static inline int vmcs_clear(struct vmcs *vmcs)
707 {
708 	bool ret;
709 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
710 
711 	asm volatile ("push %1; popf; vmclear %2; setbe %0"
712 		      : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc");
713 	return ret;
714 }
715 
716 static inline u64 vmcs_read(enum Encoding enc)
717 {
718 	u64 val;
719 	asm volatile ("vmread %1, %0" : "=rm" (val) : "r" ((u64)enc) : "cc");
720 	return val;
721 }
722 
723 static inline int vmcs_read_checking(enum Encoding enc, u64 *value)
724 {
725 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
726 	u64 encoding = enc;
727 	u64 val;
728 
729 	asm volatile ("shl $8, %%rax;"
730 		      "sahf;"
731 		      "vmread %[encoding], %[val];"
732 		      "lahf;"
733 		      "shr $8, %%rax"
734 		      : /* output */ [val]"=rm"(val), "+a"(rflags)
735 		      : /* input */ [encoding]"r"(encoding)
736 		      : /* clobber */ "cc");
737 
738 	*value = val;
739 	return rflags & (X86_EFLAGS_CF | X86_EFLAGS_ZF);
740 }
741 
742 static inline int vmcs_write(enum Encoding enc, u64 val)
743 {
744 	bool ret;
745 	asm volatile ("vmwrite %1, %2; setbe %0"
746 		: "=q"(ret) : "rm" (val), "r" ((u64)enc) : "cc");
747 	return ret;
748 }
749 
750 static inline int vmcs_set_bits(enum Encoding enc, u64 val)
751 {
752 	return vmcs_write(enc, vmcs_read(enc) | val);
753 }
754 
755 static inline int vmcs_clear_bits(enum Encoding enc, u64 val)
756 {
757 	return vmcs_write(enc, vmcs_read(enc) & ~val);
758 }
759 
760 static inline int vmcs_save(struct vmcs **vmcs)
761 {
762 	bool ret;
763 	unsigned long pa;
764 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
765 
766 	asm volatile ("push %2; popf; vmptrst %1; setbe %0"
767 		      : "=q" (ret), "=m" (pa) : "r" (rflags) : "cc");
768 	*vmcs = (pa == -1ull) ? NULL : phys_to_virt(pa);
769 	return ret;
770 }
771 
772 static inline bool invept(unsigned long type, u64 eptp)
773 {
774 	bool ret;
775 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
776 
777 	struct {
778 		u64 eptp, gpa;
779 	} operand = {eptp, 0};
780 	asm volatile("push %1; popf; invept %2, %3; setbe %0"
781 		     : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc");
782 	return ret;
783 }
784 
785 static inline bool invvpid(unsigned long type, u64 vpid, u64 gla)
786 {
787 	bool ret;
788 	u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
789 
790 	struct invvpid_operand operand = {vpid, gla};
791 	asm volatile("push %1; popf; invvpid %2, %3; setbe %0"
792 		     : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc");
793 	return ret;
794 }
795 
796 void enable_vmx(void);
797 void init_vmx(u64 *vmxon_region);
798 
799 const char *exit_reason_description(u64 reason);
800 void print_vmexit_info(void);
801 void print_vmentry_failure_info(struct vmentry_failure *failure);
802 void ept_sync(int type, u64 eptp);
803 void vpid_sync(int type, u16 vpid);
804 void install_ept_entry(unsigned long *pml4, int pte_level,
805 		unsigned long guest_addr, unsigned long pte,
806 		unsigned long *pt_page);
807 void install_1g_ept(unsigned long *pml4, unsigned long phys,
808 		unsigned long guest_addr, u64 perm);
809 void install_2m_ept(unsigned long *pml4, unsigned long phys,
810 		unsigned long guest_addr, u64 perm);
811 void install_ept(unsigned long *pml4, unsigned long phys,
812 		unsigned long guest_addr, u64 perm);
813 void setup_ept_range(unsigned long *pml4, unsigned long start,
814 		     unsigned long len, int map_1g, int map_2m, u64 perm);
815 bool get_ept_pte(unsigned long *pml4, unsigned long guest_addr, int level,
816 		unsigned long *pte);
817 void set_ept_pte(unsigned long *pml4, unsigned long guest_addr,
818 		int level, u64 pte_val);
819 void check_ept_ad(unsigned long *pml4, u64 guest_cr3,
820 		  unsigned long guest_addr, int expected_gpa_ad,
821 		  int expected_pt_ad);
822 void clear_ept_ad(unsigned long *pml4, u64 guest_cr3,
823 		  unsigned long guest_addr);
824 
825 bool ept_2m_supported(void);
826 bool ept_1g_supported(void);
827 bool ept_huge_pages_supported(int level);
828 bool ept_execute_only_supported(void);
829 bool ept_ad_bits_supported(void);
830 
831 void enter_guest(void);
832 void enter_guest_with_bad_controls(void);
833 void enter_guest_with_invalid_guest_state(void);
834 
835 typedef void (*test_guest_func)(void);
836 typedef void (*test_teardown_func)(void *data);
837 void test_set_guest(test_guest_func func);
838 void test_add_teardown(test_teardown_func func, void *data);
839 void test_skip(const char *msg);
840 
841 void __abort_test(void);
842 
843 #define TEST_ASSERT(cond) \
844 do { \
845 	if (!(cond)) { \
846 		report("%s:%d: Assertion failed: %s", 0, \
847 		       __FILE__, __LINE__, #cond); \
848 		dump_stack(); \
849 		__abort_test(); \
850 	} \
851 	report_pass(); \
852 } while (0)
853 
854 #define TEST_ASSERT_MSG(cond, fmt, args...) \
855 do { \
856 	if (!(cond)) { \
857 		report("%s:%d: Assertion failed: %s\n" fmt, 0, \
858 		       __FILE__, __LINE__, #cond, ##args); \
859 		dump_stack(); \
860 		__abort_test(); \
861 	} \
862 	report_pass(); \
863 } while (0)
864 
865 #define __TEST_EQ(a, b, a_str, b_str, assertion, fmt, args...) \
866 do { \
867 	typeof(a) _a = a; \
868 	typeof(b) _b = b; \
869 	if (_a != _b) { \
870 		char _bin_a[BINSTR_SZ]; \
871 		char _bin_b[BINSTR_SZ]; \
872 		binstr(_a, _bin_a); \
873 		binstr(_b, _bin_b); \
874 		report("%s:%d: %s failed: (%s) == (%s)\n" \
875 		       "\tLHS: %#018lx - %s - %lu\n" \
876 		       "\tRHS: %#018lx - %s - %lu%s" fmt, 0, \
877 		       __FILE__, __LINE__, \
878 		       assertion ? "Assertion" : "Expectation", a_str, b_str, \
879 		       (unsigned long) _a, _bin_a, (unsigned long) _a, \
880 		       (unsigned long) _b, _bin_b, (unsigned long) _b, \
881 		       fmt[0] == '\0' ? "" : "\n", ## args); \
882 		dump_stack(); \
883 		if (assertion) \
884 			__abort_test(); \
885 	} \
886 	report_pass(); \
887 } while (0)
888 
889 #define TEST_ASSERT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 1, "")
890 #define TEST_ASSERT_EQ_MSG(a, b, fmt, args...) \
891 	__TEST_EQ(a, b, #a, #b, 1, fmt, ## args)
892 #define TEST_EXPECT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 0, "")
893 #define TEST_EXPECT_EQ_MSG(a, b, fmt, args...) \
894 	__TEST_EQ(a, b, #a, #b, 0, fmt, ## args)
895 
896 #endif
897