1 #ifndef __VMX_H 2 #define __VMX_H 3 4 #include "libcflat.h" 5 #include "processor.h" 6 #include "bitops.h" 7 #include "asm/page.h" 8 #include "asm/io.h" 9 10 struct vmcs { 11 u32 revision_id; /* vmcs revision identifier */ 12 u32 abort; /* VMX-abort indicator */ 13 /* VMCS data */ 14 char data[0]; 15 }; 16 17 struct invvpid_operand { 18 u64 vpid; 19 u64 gla; 20 }; 21 22 struct regs { 23 u64 rax; 24 u64 rcx; 25 u64 rdx; 26 u64 rbx; 27 u64 cr2; 28 u64 rbp; 29 u64 rsi; 30 u64 rdi; 31 u64 r8; 32 u64 r9; 33 u64 r10; 34 u64 r11; 35 u64 r12; 36 u64 r13; 37 u64 r14; 38 u64 r15; 39 u64 rflags; 40 }; 41 42 struct vmentry_failure { 43 /* Did a vmlaunch or vmresume fail? */ 44 bool vmlaunch; 45 /* Instruction mnemonic (for convenience). */ 46 const char *instr; 47 /* Did the instruction return right away, or did we jump to HOST_RIP? */ 48 bool early; 49 /* Contents of [re]flags after failed entry. */ 50 unsigned long flags; 51 }; 52 53 struct vmx_test { 54 const char *name; 55 int (*init)(struct vmcs *vmcs); 56 void (*guest_main)(void); 57 int (*exit_handler)(void); 58 void (*syscall_handler)(u64 syscall_no); 59 struct regs guest_regs; 60 int (*entry_failure_handler)(struct vmentry_failure *failure); 61 struct vmcs *vmcs; 62 int exits; 63 /* Alternative test interface. */ 64 void (*v2)(void); 65 }; 66 67 union vmx_basic { 68 u64 val; 69 struct { 70 u32 revision; 71 u32 size:13, 72 reserved1: 3, 73 width:1, 74 dual:1, 75 type:4, 76 insouts:1, 77 ctrl:1, 78 reserved2:8; 79 }; 80 }; 81 82 union vmx_ctrl_msr { 83 u64 val; 84 struct { 85 u32 set, clr; 86 }; 87 }; 88 89 union vmx_ept_vpid { 90 u64 val; 91 struct { 92 u32:16, 93 super:2, 94 : 2, 95 invept:1, 96 : 11; 97 u32 invvpid:1; 98 }; 99 }; 100 101 enum Encoding { 102 /* 16-Bit Control Fields */ 103 VPID = 0x0000ul, 104 /* Posted-interrupt notification vector */ 105 PINV = 0x0002ul, 106 /* EPTP index */ 107 EPTP_IDX = 0x0004ul, 108 109 /* 16-Bit Guest State Fields */ 110 GUEST_SEL_ES = 0x0800ul, 111 GUEST_SEL_CS = 0x0802ul, 112 GUEST_SEL_SS = 0x0804ul, 113 GUEST_SEL_DS = 0x0806ul, 114 GUEST_SEL_FS = 0x0808ul, 115 GUEST_SEL_GS = 0x080aul, 116 GUEST_SEL_LDTR = 0x080cul, 117 GUEST_SEL_TR = 0x080eul, 118 GUEST_INT_STATUS = 0x0810ul, 119 GUEST_PML_INDEX = 0x0812ul, 120 121 /* 16-Bit Host State Fields */ 122 HOST_SEL_ES = 0x0c00ul, 123 HOST_SEL_CS = 0x0c02ul, 124 HOST_SEL_SS = 0x0c04ul, 125 HOST_SEL_DS = 0x0c06ul, 126 HOST_SEL_FS = 0x0c08ul, 127 HOST_SEL_GS = 0x0c0aul, 128 HOST_SEL_TR = 0x0c0cul, 129 130 /* 64-Bit Control Fields */ 131 IO_BITMAP_A = 0x2000ul, 132 IO_BITMAP_B = 0x2002ul, 133 MSR_BITMAP = 0x2004ul, 134 EXIT_MSR_ST_ADDR = 0x2006ul, 135 EXIT_MSR_LD_ADDR = 0x2008ul, 136 ENTER_MSR_LD_ADDR = 0x200aul, 137 VMCS_EXEC_PTR = 0x200cul, 138 TSC_OFFSET = 0x2010ul, 139 TSC_OFFSET_HI = 0x2011ul, 140 APIC_VIRT_ADDR = 0x2012ul, 141 APIC_ACCS_ADDR = 0x2014ul, 142 EPTP = 0x201aul, 143 EPTP_HI = 0x201bul, 144 EOI_EXIT_BITMAP0 = 0x201cul, 145 EOI_EXIT_BITMAP1 = 0x201eul, 146 EOI_EXIT_BITMAP2 = 0x2020ul, 147 EOI_EXIT_BITMAP3 = 0x2022ul, 148 PMLADDR = 0x200eul, 149 PMLADDR_HI = 0x200ful, 150 151 152 /* 64-Bit Readonly Data Field */ 153 INFO_PHYS_ADDR = 0x2400ul, 154 155 /* 64-Bit Guest State */ 156 VMCS_LINK_PTR = 0x2800ul, 157 VMCS_LINK_PTR_HI = 0x2801ul, 158 GUEST_DEBUGCTL = 0x2802ul, 159 GUEST_DEBUGCTL_HI = 0x2803ul, 160 GUEST_EFER = 0x2806ul, 161 GUEST_PAT = 0x2804ul, 162 GUEST_PERF_GLOBAL_CTRL = 0x2808ul, 163 GUEST_PDPTE = 0x280aul, 164 165 /* 64-Bit Host State */ 166 HOST_PAT = 0x2c00ul, 167 HOST_EFER = 0x2c02ul, 168 HOST_PERF_GLOBAL_CTRL = 0x2c04ul, 169 170 /* 32-Bit Control Fields */ 171 PIN_CONTROLS = 0x4000ul, 172 CPU_EXEC_CTRL0 = 0x4002ul, 173 EXC_BITMAP = 0x4004ul, 174 PF_ERROR_MASK = 0x4006ul, 175 PF_ERROR_MATCH = 0x4008ul, 176 CR3_TARGET_COUNT = 0x400aul, 177 EXI_CONTROLS = 0x400cul, 178 EXI_MSR_ST_CNT = 0x400eul, 179 EXI_MSR_LD_CNT = 0x4010ul, 180 ENT_CONTROLS = 0x4012ul, 181 ENT_MSR_LD_CNT = 0x4014ul, 182 ENT_INTR_INFO = 0x4016ul, 183 ENT_INTR_ERROR = 0x4018ul, 184 ENT_INST_LEN = 0x401aul, 185 TPR_THRESHOLD = 0x401cul, 186 CPU_EXEC_CTRL1 = 0x401eul, 187 188 /* 32-Bit R/O Data Fields */ 189 VMX_INST_ERROR = 0x4400ul, 190 EXI_REASON = 0x4402ul, 191 EXI_INTR_INFO = 0x4404ul, 192 EXI_INTR_ERROR = 0x4406ul, 193 IDT_VECT_INFO = 0x4408ul, 194 IDT_VECT_ERROR = 0x440aul, 195 EXI_INST_LEN = 0x440cul, 196 EXI_INST_INFO = 0x440eul, 197 198 /* 32-Bit Guest State Fields */ 199 GUEST_LIMIT_ES = 0x4800ul, 200 GUEST_LIMIT_CS = 0x4802ul, 201 GUEST_LIMIT_SS = 0x4804ul, 202 GUEST_LIMIT_DS = 0x4806ul, 203 GUEST_LIMIT_FS = 0x4808ul, 204 GUEST_LIMIT_GS = 0x480aul, 205 GUEST_LIMIT_LDTR = 0x480cul, 206 GUEST_LIMIT_TR = 0x480eul, 207 GUEST_LIMIT_GDTR = 0x4810ul, 208 GUEST_LIMIT_IDTR = 0x4812ul, 209 GUEST_AR_ES = 0x4814ul, 210 GUEST_AR_CS = 0x4816ul, 211 GUEST_AR_SS = 0x4818ul, 212 GUEST_AR_DS = 0x481aul, 213 GUEST_AR_FS = 0x481cul, 214 GUEST_AR_GS = 0x481eul, 215 GUEST_AR_LDTR = 0x4820ul, 216 GUEST_AR_TR = 0x4822ul, 217 GUEST_INTR_STATE = 0x4824ul, 218 GUEST_ACTV_STATE = 0x4826ul, 219 GUEST_SMBASE = 0x4828ul, 220 GUEST_SYSENTER_CS = 0x482aul, 221 PREEMPT_TIMER_VALUE = 0x482eul, 222 223 /* 32-Bit Host State Fields */ 224 HOST_SYSENTER_CS = 0x4c00ul, 225 226 /* Natural-Width Control Fields */ 227 CR0_MASK = 0x6000ul, 228 CR4_MASK = 0x6002ul, 229 CR0_READ_SHADOW = 0x6004ul, 230 CR4_READ_SHADOW = 0x6006ul, 231 CR3_TARGET_0 = 0x6008ul, 232 CR3_TARGET_1 = 0x600aul, 233 CR3_TARGET_2 = 0x600cul, 234 CR3_TARGET_3 = 0x600eul, 235 236 /* Natural-Width R/O Data Fields */ 237 EXI_QUALIFICATION = 0x6400ul, 238 IO_RCX = 0x6402ul, 239 IO_RSI = 0x6404ul, 240 IO_RDI = 0x6406ul, 241 IO_RIP = 0x6408ul, 242 GUEST_LINEAR_ADDRESS = 0x640aul, 243 244 /* Natural-Width Guest State Fields */ 245 GUEST_CR0 = 0x6800ul, 246 GUEST_CR3 = 0x6802ul, 247 GUEST_CR4 = 0x6804ul, 248 GUEST_BASE_ES = 0x6806ul, 249 GUEST_BASE_CS = 0x6808ul, 250 GUEST_BASE_SS = 0x680aul, 251 GUEST_BASE_DS = 0x680cul, 252 GUEST_BASE_FS = 0x680eul, 253 GUEST_BASE_GS = 0x6810ul, 254 GUEST_BASE_LDTR = 0x6812ul, 255 GUEST_BASE_TR = 0x6814ul, 256 GUEST_BASE_GDTR = 0x6816ul, 257 GUEST_BASE_IDTR = 0x6818ul, 258 GUEST_DR7 = 0x681aul, 259 GUEST_RSP = 0x681cul, 260 GUEST_RIP = 0x681eul, 261 GUEST_RFLAGS = 0x6820ul, 262 GUEST_PENDING_DEBUG = 0x6822ul, 263 GUEST_SYSENTER_ESP = 0x6824ul, 264 GUEST_SYSENTER_EIP = 0x6826ul, 265 266 /* Natural-Width Host State Fields */ 267 HOST_CR0 = 0x6c00ul, 268 HOST_CR3 = 0x6c02ul, 269 HOST_CR4 = 0x6c04ul, 270 HOST_BASE_FS = 0x6c06ul, 271 HOST_BASE_GS = 0x6c08ul, 272 HOST_BASE_TR = 0x6c0aul, 273 HOST_BASE_GDTR = 0x6c0cul, 274 HOST_BASE_IDTR = 0x6c0eul, 275 HOST_SYSENTER_ESP = 0x6c10ul, 276 HOST_SYSENTER_EIP = 0x6c12ul, 277 HOST_RSP = 0x6c14ul, 278 HOST_RIP = 0x6c16ul 279 }; 280 281 #define VMX_ENTRY_FAILURE (1ul << 31) 282 #define VMX_ENTRY_FLAGS (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 283 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 284 285 enum Reason { 286 VMX_EXC_NMI = 0, 287 VMX_EXTINT = 1, 288 VMX_TRIPLE_FAULT = 2, 289 VMX_INIT = 3, 290 VMX_SIPI = 4, 291 VMX_SMI_IO = 5, 292 VMX_SMI_OTHER = 6, 293 VMX_INTR_WINDOW = 7, 294 VMX_NMI_WINDOW = 8, 295 VMX_TASK_SWITCH = 9, 296 VMX_CPUID = 10, 297 VMX_GETSEC = 11, 298 VMX_HLT = 12, 299 VMX_INVD = 13, 300 VMX_INVLPG = 14, 301 VMX_RDPMC = 15, 302 VMX_RDTSC = 16, 303 VMX_RSM = 17, 304 VMX_VMCALL = 18, 305 VMX_VMCLEAR = 19, 306 VMX_VMLAUNCH = 20, 307 VMX_VMPTRLD = 21, 308 VMX_VMPTRST = 22, 309 VMX_VMREAD = 23, 310 VMX_VMRESUME = 24, 311 VMX_VMWRITE = 25, 312 VMX_VMXOFF = 26, 313 VMX_VMXON = 27, 314 VMX_CR = 28, 315 VMX_DR = 29, 316 VMX_IO = 30, 317 VMX_RDMSR = 31, 318 VMX_WRMSR = 32, 319 VMX_FAIL_STATE = 33, 320 VMX_FAIL_MSR = 34, 321 VMX_MWAIT = 36, 322 VMX_MTF = 37, 323 VMX_MONITOR = 39, 324 VMX_PAUSE = 40, 325 VMX_FAIL_MCHECK = 41, 326 VMX_TPR_THRESHOLD = 43, 327 VMX_APIC_ACCESS = 44, 328 VMX_EOI_INDUCED = 45, 329 VMX_GDTR_IDTR = 46, 330 VMX_LDTR_TR = 47, 331 VMX_EPT_VIOLATION = 48, 332 VMX_EPT_MISCONFIG = 49, 333 VMX_INVEPT = 50, 334 VMX_PREEMPT = 52, 335 VMX_INVVPID = 53, 336 VMX_WBINVD = 54, 337 VMX_XSETBV = 55, 338 VMX_APIC_WRITE = 56, 339 VMX_RDRAND = 57, 340 VMX_INVPCID = 58, 341 VMX_VMFUNC = 59, 342 VMX_RDSEED = 61, 343 VMX_PML_FULL = 62, 344 VMX_XSAVES = 63, 345 VMX_XRSTORS = 64, 346 }; 347 348 enum Ctrl_exi { 349 EXI_SAVE_DBGCTLS = 1UL << 2, 350 EXI_HOST_64 = 1UL << 9, 351 EXI_LOAD_PERF = 1UL << 12, 352 EXI_INTA = 1UL << 15, 353 EXI_SAVE_PAT = 1UL << 18, 354 EXI_LOAD_PAT = 1UL << 19, 355 EXI_SAVE_EFER = 1UL << 20, 356 EXI_LOAD_EFER = 1UL << 21, 357 EXI_SAVE_PREEMPT = 1UL << 22, 358 }; 359 360 enum Ctrl_ent { 361 ENT_LOAD_DBGCTLS = 1UL << 2, 362 ENT_GUEST_64 = 1UL << 9, 363 ENT_LOAD_PAT = 1UL << 14, 364 ENT_LOAD_EFER = 1UL << 15, 365 }; 366 367 enum Ctrl_pin { 368 PIN_EXTINT = 1ul << 0, 369 PIN_NMI = 1ul << 3, 370 PIN_VIRT_NMI = 1ul << 5, 371 PIN_PREEMPT = 1ul << 6, 372 PIN_POST_INTR = 1ul << 7, 373 }; 374 375 enum Ctrl0 { 376 CPU_INTR_WINDOW = 1ul << 2, 377 CPU_HLT = 1ul << 7, 378 CPU_INVLPG = 1ul << 9, 379 CPU_MWAIT = 1ul << 10, 380 CPU_RDPMC = 1ul << 11, 381 CPU_RDTSC = 1ul << 12, 382 CPU_CR3_LOAD = 1ul << 15, 383 CPU_CR3_STORE = 1ul << 16, 384 CPU_CR8_LOAD = 1ul << 19, 385 CPU_CR8_STORE = 1ul << 20, 386 CPU_TPR_SHADOW = 1ul << 21, 387 CPU_NMI_WINDOW = 1ul << 22, 388 CPU_IO = 1ul << 24, 389 CPU_IO_BITMAP = 1ul << 25, 390 CPU_MSR_BITMAP = 1ul << 28, 391 CPU_MONITOR = 1ul << 29, 392 CPU_PAUSE = 1ul << 30, 393 CPU_SECONDARY = 1ul << 31, 394 }; 395 396 enum Ctrl1 { 397 CPU_VIRT_APIC_ACCESSES = 1ul << 0, 398 CPU_EPT = 1ul << 1, 399 CPU_DESC_TABLE = 1ul << 2, 400 CPU_RDTSCP = 1ul << 3, 401 CPU_VIRT_X2APIC = 1ul << 4, 402 CPU_VPID = 1ul << 5, 403 CPU_WBINVD = 1ul << 6, 404 CPU_URG = 1ul << 7, 405 CPU_APIC_REG_VIRT = 1ul << 8, 406 CPU_VINTD = 1ul << 9, 407 CPU_RDRAND = 1ul << 11, 408 CPU_RDSEED = 1ul << 16, 409 CPU_PML = 1ul << 17, 410 }; 411 412 enum Intr_type { 413 VMX_INTR_TYPE_EXT_INTR = 0, 414 VMX_INTR_TYPE_NMI_INTR = 2, 415 VMX_INTR_TYPE_HARD_EXCEPTION = 3, 416 VMX_INTR_TYPE_SOFT_INTR = 4, 417 VMX_INTR_TYPE_SOFT_EXCEPTION = 6, 418 }; 419 420 /* 421 * Interruption-information format 422 */ 423 #define INTR_INFO_VECTOR_MASK 0xff /* 7:0 */ 424 #define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */ 425 #define INTR_INFO_DELIVER_CODE_MASK 0x800 /* 11 */ 426 #define INTR_INFO_UNBLOCK_NMI_MASK 0x1000 /* 12 */ 427 #define INTR_INFO_VALID_MASK 0x80000000 /* 31 */ 428 429 #define INTR_INFO_INTR_TYPE_SHIFT 8 430 431 /* 432 * VM-instruction error numbers 433 */ 434 enum vm_instruction_error_number { 435 VMXERR_VMCALL_IN_VMX_ROOT_OPERATION = 1, 436 VMXERR_VMCLEAR_INVALID_ADDRESS = 2, 437 VMXERR_VMCLEAR_VMXON_POINTER = 3, 438 VMXERR_VMLAUNCH_NONCLEAR_VMCS = 4, 439 VMXERR_VMRESUME_NONLAUNCHED_VMCS = 5, 440 VMXERR_VMRESUME_AFTER_VMXOFF = 6, 441 VMXERR_ENTRY_INVALID_CONTROL_FIELD = 7, 442 VMXERR_ENTRY_INVALID_HOST_STATE_FIELD = 8, 443 VMXERR_VMPTRLD_INVALID_ADDRESS = 9, 444 VMXERR_VMPTRLD_VMXON_POINTER = 10, 445 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID = 11, 446 VMXERR_UNSUPPORTED_VMCS_COMPONENT = 12, 447 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT = 13, 448 VMXERR_VMXON_IN_VMX_ROOT_OPERATION = 15, 449 VMXERR_ENTRY_INVALID_EXECUTIVE_VMCS_POINTER = 16, 450 VMXERR_ENTRY_NONLAUNCHED_EXECUTIVE_VMCS = 17, 451 VMXERR_ENTRY_EXECUTIVE_VMCS_POINTER_NOT_VMXON_POINTER = 18, 452 VMXERR_VMCALL_NONCLEAR_VMCS = 19, 453 VMXERR_VMCALL_INVALID_VM_EXIT_CONTROL_FIELDS = 20, 454 VMXERR_VMCALL_INCORRECT_MSEG_REVISION_ID = 22, 455 VMXERR_VMXOFF_UNDER_DUAL_MONITOR_TREATMENT_OF_SMIS_AND_SMM = 23, 456 VMXERR_VMCALL_INVALID_SMM_MONITOR_FEATURES = 24, 457 VMXERR_ENTRY_INVALID_VM_EXECUTION_CONTROL_FIELDS_IN_EXECUTIVE_VMCS = 25, 458 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS = 26, 459 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28, 460 }; 461 462 #define SAVE_GPR \ 463 "xchg %rax, regs\n\t" \ 464 "xchg %rbx, regs+0x8\n\t" \ 465 "xchg %rcx, regs+0x10\n\t" \ 466 "xchg %rdx, regs+0x18\n\t" \ 467 "xchg %rbp, regs+0x28\n\t" \ 468 "xchg %rsi, regs+0x30\n\t" \ 469 "xchg %rdi, regs+0x38\n\t" \ 470 "xchg %r8, regs+0x40\n\t" \ 471 "xchg %r9, regs+0x48\n\t" \ 472 "xchg %r10, regs+0x50\n\t" \ 473 "xchg %r11, regs+0x58\n\t" \ 474 "xchg %r12, regs+0x60\n\t" \ 475 "xchg %r13, regs+0x68\n\t" \ 476 "xchg %r14, regs+0x70\n\t" \ 477 "xchg %r15, regs+0x78\n\t" 478 479 #define LOAD_GPR SAVE_GPR 480 481 #define SAVE_GPR_C \ 482 "xchg %%rax, regs\n\t" \ 483 "xchg %%rbx, regs+0x8\n\t" \ 484 "xchg %%rcx, regs+0x10\n\t" \ 485 "xchg %%rdx, regs+0x18\n\t" \ 486 "xchg %%rbp, regs+0x28\n\t" \ 487 "xchg %%rsi, regs+0x30\n\t" \ 488 "xchg %%rdi, regs+0x38\n\t" \ 489 "xchg %%r8, regs+0x40\n\t" \ 490 "xchg %%r9, regs+0x48\n\t" \ 491 "xchg %%r10, regs+0x50\n\t" \ 492 "xchg %%r11, regs+0x58\n\t" \ 493 "xchg %%r12, regs+0x60\n\t" \ 494 "xchg %%r13, regs+0x68\n\t" \ 495 "xchg %%r14, regs+0x70\n\t" \ 496 "xchg %%r15, regs+0x78\n\t" 497 498 #define LOAD_GPR_C SAVE_GPR_C 499 500 #define VMX_IO_SIZE_MASK 0x7 501 #define _VMX_IO_BYTE 0 502 #define _VMX_IO_WORD 1 503 #define _VMX_IO_LONG 3 504 #define VMX_IO_DIRECTION_MASK (1ul << 3) 505 #define VMX_IO_IN (1ul << 3) 506 #define VMX_IO_OUT 0 507 #define VMX_IO_STRING (1ul << 4) 508 #define VMX_IO_REP (1ul << 5) 509 #define VMX_IO_OPRAND_IMM (1ul << 6) 510 #define VMX_IO_PORT_MASK 0xFFFF0000 511 #define VMX_IO_PORT_SHIFT 16 512 513 #define VMX_TEST_START 0 514 #define VMX_TEST_VMEXIT 1 515 #define VMX_TEST_EXIT 2 516 #define VMX_TEST_RESUME 3 517 #define VMX_TEST_VMABORT 4 518 #define VMX_TEST_VMSKIP 5 519 520 #define HYPERCALL_BIT (1ul << 12) 521 #define HYPERCALL_MASK 0xFFF 522 #define HYPERCALL_VMEXIT 0x1 523 #define HYPERCALL_VMABORT 0x2 524 #define HYPERCALL_VMSKIP 0x3 525 526 #define EPTP_PG_WALK_LEN_SHIFT 3ul 527 #define EPTP_AD_FLAG (1ul << 6) 528 529 #define EPT_MEM_TYPE_UC 0ul 530 #define EPT_MEM_TYPE_WC 1ul 531 #define EPT_MEM_TYPE_WT 4ul 532 #define EPT_MEM_TYPE_WP 5ul 533 #define EPT_MEM_TYPE_WB 6ul 534 535 #define EPT_RA 1ul 536 #define EPT_WA 2ul 537 #define EPT_EA 4ul 538 #define EPT_PRESENT (EPT_RA | EPT_WA | EPT_EA) 539 #define EPT_ACCESS_FLAG (1ul << 8) 540 #define EPT_DIRTY_FLAG (1ul << 9) 541 #define EPT_LARGE_PAGE (1ul << 7) 542 #define EPT_MEM_TYPE_SHIFT 3ul 543 #define EPT_IGNORE_PAT (1ul << 6) 544 #define EPT_SUPPRESS_VE (1ull << 63) 545 546 #define EPT_CAP_WT 1ull 547 #define EPT_CAP_PWL4 (1ull << 6) 548 #define EPT_CAP_UC (1ull << 8) 549 #define EPT_CAP_WB (1ull << 14) 550 #define EPT_CAP_2M_PAGE (1ull << 16) 551 #define EPT_CAP_1G_PAGE (1ull << 17) 552 #define EPT_CAP_INVEPT (1ull << 20) 553 #define EPT_CAP_INVEPT_SINGLE (1ull << 25) 554 #define EPT_CAP_INVEPT_ALL (1ull << 26) 555 #define EPT_CAP_AD_FLAG (1ull << 21) 556 #define VPID_CAP_INVVPID (1ull << 32) 557 #define VPID_CAP_INVVPID_ADDR (1ull << 40) 558 #define VPID_CAP_INVVPID_CXTGLB (1ull << 41) 559 #define VPID_CAP_INVVPID_ALL (1ull << 42) 560 #define VPID_CAP_INVVPID_CXTLOC (1ull << 43) 561 562 #define PAGE_SIZE_2M (512 * PAGE_SIZE) 563 #define PAGE_SIZE_1G (512 * PAGE_SIZE_2M) 564 #define EPT_PAGE_LEVEL 4 565 #define EPT_PGDIR_WIDTH 9 566 #define EPT_PGDIR_MASK 511 567 #define EPT_PGDIR_ENTRIES (1 << EPT_PGDIR_WIDTH) 568 #define EPT_LEVEL_SHIFT(level) (((level)-1) * EPT_PGDIR_WIDTH + 12) 569 #define EPT_ADDR_MASK GENMASK_ULL(51, 12) 570 #define PAGE_MASK_2M (~(PAGE_SIZE_2M-1)) 571 572 #define EPT_VLT_RD 1 573 #define EPT_VLT_WR (1 << 1) 574 #define EPT_VLT_FETCH (1 << 2) 575 #define EPT_VLT_PERM_RD (1 << 3) 576 #define EPT_VLT_PERM_WR (1 << 4) 577 #define EPT_VLT_PERM_EX (1 << 5) 578 #define EPT_VLT_PERMS (EPT_VLT_PERM_RD | EPT_VLT_PERM_WR | \ 579 EPT_VLT_PERM_EX) 580 #define EPT_VLT_LADDR_VLD (1 << 7) 581 #define EPT_VLT_PADDR (1 << 8) 582 583 #define MAGIC_VAL_1 0x12345678ul 584 #define MAGIC_VAL_2 0x87654321ul 585 #define MAGIC_VAL_3 0xfffffffful 586 #define MAGIC_VAL_4 0xdeadbeeful 587 588 #define INVEPT_SINGLE 1 589 #define INVEPT_GLOBAL 2 590 591 #define INVVPID_ADDR 0 592 #define INVVPID_CONTEXT_GLOBAL 1 593 #define INVVPID_ALL 2 594 #define INVVPID_CONTEXT_LOCAL 3 595 596 #define ACTV_ACTIVE 0 597 #define ACTV_HLT 1 598 599 extern struct regs regs; 600 601 extern union vmx_basic basic; 602 extern union vmx_ctrl_msr ctrl_pin_rev; 603 extern union vmx_ctrl_msr ctrl_cpu_rev[2]; 604 extern union vmx_ctrl_msr ctrl_exit_rev; 605 extern union vmx_ctrl_msr ctrl_enter_rev; 606 extern union vmx_ept_vpid ept_vpid; 607 608 extern u64 *vmxon_region; 609 610 void vmx_set_test_stage(u32 s); 611 u32 vmx_get_test_stage(void); 612 void vmx_inc_test_stage(void); 613 614 static int vmx_on(void) 615 { 616 bool ret; 617 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 618 asm volatile ("push %1; popf; vmxon %2; setbe %0\n\t" 619 : "=q" (ret) : "q" (rflags), "m" (vmxon_region) : "cc"); 620 return ret; 621 } 622 623 static int vmx_off(void) 624 { 625 bool ret; 626 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 627 628 asm volatile("push %1; popf; vmxoff; setbe %0\n\t" 629 : "=q"(ret) : "q" (rflags) : "cc"); 630 return ret; 631 } 632 633 static inline int make_vmcs_current(struct vmcs *vmcs) 634 { 635 bool ret; 636 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 637 638 asm volatile ("push %1; popf; vmptrld %2; setbe %0" 639 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 640 return ret; 641 } 642 643 static inline int vmcs_clear(struct vmcs *vmcs) 644 { 645 bool ret; 646 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 647 648 asm volatile ("push %1; popf; vmclear %2; setbe %0" 649 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 650 return ret; 651 } 652 653 static inline u64 vmcs_read(enum Encoding enc) 654 { 655 u64 val; 656 asm volatile ("vmread %1, %0" : "=rm" (val) : "r" ((u64)enc) : "cc"); 657 return val; 658 } 659 660 static inline int vmcs_read_checking(enum Encoding enc, u64 *value) 661 { 662 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 663 u64 encoding = enc; 664 u64 val; 665 666 asm volatile ("shl $8, %%rax;" 667 "sahf;" 668 "vmread %[encoding], %[val];" 669 "lahf;" 670 "shr $8, %%rax" 671 : /* output */ [val]"=rm"(val), "+a"(rflags) 672 : /* input */ [encoding]"r"(encoding) 673 : /* clobber */ "cc"); 674 675 *value = val; 676 return rflags & (X86_EFLAGS_CF | X86_EFLAGS_ZF); 677 } 678 679 static inline int vmcs_write(enum Encoding enc, u64 val) 680 { 681 bool ret; 682 asm volatile ("vmwrite %1, %2; setbe %0" 683 : "=q"(ret) : "rm" (val), "r" ((u64)enc) : "cc"); 684 return ret; 685 } 686 687 static inline int vmcs_save(struct vmcs **vmcs) 688 { 689 bool ret; 690 unsigned long pa; 691 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 692 693 asm volatile ("push %2; popf; vmptrst %1; setbe %0" 694 : "=q" (ret), "=m" (pa) : "r" (rflags) : "cc"); 695 *vmcs = (pa == -1ull) ? NULL : phys_to_virt(pa); 696 return ret; 697 } 698 699 static inline bool invept(unsigned long type, u64 eptp) 700 { 701 bool ret; 702 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 703 704 struct { 705 u64 eptp, gpa; 706 } operand = {eptp, 0}; 707 asm volatile("push %1; popf; invept %2, %3; setbe %0" 708 : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 709 return ret; 710 } 711 712 static inline bool invvpid(unsigned long type, u64 vpid, u64 gla) 713 { 714 bool ret; 715 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 716 717 struct invvpid_operand operand = {vpid, gla}; 718 asm volatile("push %1; popf; invvpid %2, %3; setbe %0" 719 : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 720 return ret; 721 } 722 723 const char *exit_reason_description(u64 reason); 724 void print_vmexit_info(void); 725 void print_vmentry_failure_info(struct vmentry_failure *failure); 726 void ept_sync(int type, u64 eptp); 727 void vpid_sync(int type, u16 vpid); 728 void install_ept_entry(unsigned long *pml4, int pte_level, 729 unsigned long guest_addr, unsigned long pte, 730 unsigned long *pt_page); 731 void install_1g_ept(unsigned long *pml4, unsigned long phys, 732 unsigned long guest_addr, u64 perm); 733 void install_2m_ept(unsigned long *pml4, unsigned long phys, 734 unsigned long guest_addr, u64 perm); 735 void install_ept(unsigned long *pml4, unsigned long phys, 736 unsigned long guest_addr, u64 perm); 737 void setup_ept_range(unsigned long *pml4, unsigned long start, 738 unsigned long len, int map_1g, int map_2m, u64 perm); 739 bool get_ept_pte(unsigned long *pml4, unsigned long guest_addr, int level, 740 unsigned long *pte); 741 void set_ept_pte(unsigned long *pml4, unsigned long guest_addr, 742 int level, u64 pte_val); 743 void check_ept_ad(unsigned long *pml4, u64 guest_cr3, 744 unsigned long guest_addr, int expected_gpa_ad, 745 int expected_pt_ad); 746 void clear_ept_ad(unsigned long *pml4, u64 guest_cr3, 747 unsigned long guest_addr); 748 749 bool ept_2m_supported(void); 750 bool ept_1g_supported(void); 751 bool ept_huge_pages_supported(int level); 752 bool ept_execute_only_supported(void); 753 bool ept_ad_bits_supported(void); 754 755 void enter_guest(void); 756 757 typedef void (*test_guest_func)(void); 758 typedef void (*test_teardown_func)(void *data); 759 void test_set_guest(test_guest_func func); 760 void test_add_teardown(test_teardown_func func, void *data); 761 void test_skip(const char *msg); 762 763 void __abort_test(void); 764 765 #define TEST_ASSERT(cond) \ 766 do { \ 767 if (!(cond)) { \ 768 report("%s:%d: Assertion failed: %s", 0, \ 769 __FILE__, __LINE__, #cond); \ 770 dump_stack(); \ 771 __abort_test(); \ 772 } \ 773 report_pass(); \ 774 } while (0) 775 776 #define TEST_ASSERT_MSG(cond, fmt, args...) \ 777 do { \ 778 if (!(cond)) { \ 779 report("%s:%d: Assertion failed: %s\n" fmt, 0, \ 780 __FILE__, __LINE__, #cond, ##args); \ 781 dump_stack(); \ 782 __abort_test(); \ 783 } \ 784 report_pass(); \ 785 } while (0) 786 787 #define __TEST_EQ(a, b, a_str, b_str, assertion, fmt, args...) \ 788 do { \ 789 typeof(a) _a = a; \ 790 typeof(b) _b = b; \ 791 if (_a != _b) { \ 792 char _bin_a[BINSTR_SZ]; \ 793 char _bin_b[BINSTR_SZ]; \ 794 binstr(_a, _bin_a); \ 795 binstr(_b, _bin_b); \ 796 report("%s:%d: %s failed: (%s) == (%s)\n" \ 797 "\tLHS: %#018lx - %s - %lu\n" \ 798 "\tRHS: %#018lx - %s - %lu%s" fmt, 0, \ 799 __FILE__, __LINE__, \ 800 assertion ? "Assertion" : "Expectation", a_str, b_str, \ 801 (unsigned long) _a, _bin_a, (unsigned long) _a, \ 802 (unsigned long) _b, _bin_b, (unsigned long) _b, \ 803 fmt[0] == '\0' ? "" : "\n", ## args); \ 804 dump_stack(); \ 805 if (assertion) \ 806 __abort_test(); \ 807 } \ 808 report_pass(); \ 809 } while (0) 810 811 #define TEST_ASSERT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 1, "") 812 #define TEST_ASSERT_EQ_MSG(a, b, fmt, args...) \ 813 __TEST_EQ(a, b, #a, #b, 1, fmt, ## args) 814 #define TEST_EXPECT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 0, "") 815 #define TEST_EXPECT_EQ_MSG(a, b, fmt, args...) \ 816 __TEST_EQ(a, b, #a, #b, 0, fmt, ## args) 817 818 #endif 819