1 #ifndef X86_VMX_H 2 #define X86_VMX_H 3 4 #include "libcflat.h" 5 #include "processor.h" 6 #include "bitops.h" 7 #include "util.h" 8 #include "asm/page.h" 9 #include "asm/io.h" 10 11 void __abort_test(void); 12 13 #define __TEST_ASSERT(cond) \ 14 do { \ 15 if (!(cond)) { \ 16 report_fail("%s:%d: Assertion failed: %s", \ 17 __FILE__, __LINE__, #cond); \ 18 dump_stack(); \ 19 __abort_test(); \ 20 } \ 21 } while (0) 22 23 #define TEST_ASSERT(cond) \ 24 do { \ 25 __TEST_ASSERT(cond); \ 26 report_passed(); \ 27 } while (0) 28 29 #define TEST_ASSERT_MSG(cond, fmt, args...) \ 30 do { \ 31 if (!(cond)) { \ 32 report_fail("%s:%d: Assertion failed: %s\n" fmt,\ 33 __FILE__, __LINE__, #cond, ##args); \ 34 dump_stack(); \ 35 __abort_test(); \ 36 } \ 37 report_passed(); \ 38 } while (0) 39 40 #define TEST_ASSERT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 1, __abort_test, "") 41 #define TEST_ASSERT_EQ_MSG(a, b, fmt, args...) \ 42 __TEST_EQ(a, b, #a, #b, 1, __abort_test, fmt, ## args) 43 44 struct vmcs_hdr { 45 u32 revision_id:31; 46 u32 shadow_vmcs:1; 47 }; 48 49 struct vmcs { 50 struct vmcs_hdr hdr; 51 u32 abort; /* VMX-abort indicator */ 52 /* VMCS data */ 53 char data[0]; 54 }; 55 56 struct invvpid_operand { 57 u64 vpid; 58 u64 gla; 59 }; 60 61 struct regs { 62 u64 rax; 63 u64 rcx; 64 u64 rdx; 65 u64 rbx; 66 u64 cr2; 67 u64 rbp; 68 u64 rsi; 69 u64 rdi; 70 u64 r8; 71 u64 r9; 72 u64 r10; 73 u64 r11; 74 u64 r12; 75 u64 r13; 76 u64 r14; 77 u64 r15; 78 u64 rflags; 79 }; 80 81 union exit_reason { 82 struct { 83 u32 basic : 16; 84 u32 reserved16 : 1; 85 u32 reserved17 : 1; 86 u32 reserved18 : 1; 87 u32 reserved19 : 1; 88 u32 reserved20 : 1; 89 u32 reserved21 : 1; 90 u32 reserved22 : 1; 91 u32 reserved23 : 1; 92 u32 reserved24 : 1; 93 u32 reserved25 : 1; 94 u32 reserved26 : 1; 95 u32 enclave_mode : 1; 96 u32 smi_pending_mtf : 1; 97 u32 smi_from_vmx_root : 1; 98 u32 reserved30 : 1; 99 u32 failed_vmentry : 1; 100 }; 101 u32 full; 102 }; 103 104 struct vmentry_result { 105 /* Instruction mnemonic (for convenience). */ 106 const char *instr; 107 /* Did the test attempt vmlaunch or vmresume? */ 108 bool vmlaunch; 109 /* Did the instruction VM-Fail? */ 110 bool vm_fail; 111 /* Did the VM-Entry fully enter the guest? */ 112 bool entered; 113 /* VM-Exit reason, valid iff !vm_fail */ 114 union exit_reason exit_reason; 115 /* Contents of [re]flags after failed entry. */ 116 unsigned long flags; 117 }; 118 119 struct vmx_test { 120 const char *name; 121 int (*init)(struct vmcs *vmcs); 122 void (*guest_main)(void); 123 int (*exit_handler)(union exit_reason exit_reason); 124 void (*syscall_handler)(u64 syscall_no); 125 struct regs guest_regs; 126 int (*entry_failure_handler)(struct vmentry_result *result); 127 struct vmcs *vmcs; 128 int exits; 129 /* Alternative test interface. */ 130 void (*v2)(void); 131 }; 132 133 union vmx_basic_msr { 134 u64 val; 135 struct { 136 u32 revision; 137 u32 size:13, 138 reserved1: 3, 139 width:1, 140 dual:1, 141 type:4, 142 insouts:1, 143 ctrl:1, 144 no_hw_errcode_cc:1, 145 reserved2:7; 146 }; 147 }; 148 149 union vmx_ctrl_msr { 150 u64 val; 151 struct { 152 u32 set, clr; 153 }; 154 }; 155 156 union vmx_misc { 157 u64 val; 158 struct { 159 u32 pt_bit:5, 160 stores_lma:1, 161 act_hlt:1, 162 act_shutdown:1, 163 act_wfsipi:1, 164 :5, 165 vmx_pt:1, 166 smm_smbase:1, 167 cr3_targets:9, 168 msr_list_size:3, 169 smm_mon_ctl:1, 170 vmwrite_any:1, 171 inject_len0:1, 172 :1; 173 u32 mseg_revision; 174 }; 175 }; 176 177 union vmx_ept_vpid { 178 u64 val; 179 struct { 180 u32:16, 181 super:2, 182 : 2, 183 invept:1, 184 : 11; 185 u32 invvpid:1; 186 }; 187 }; 188 189 enum Encoding { 190 /* 16-Bit Control Fields */ 191 VPID = 0x0000ul, 192 /* Posted-interrupt notification vector */ 193 PINV = 0x0002ul, 194 /* EPTP index */ 195 EPTP_IDX = 0x0004ul, 196 197 /* 16-Bit Guest State Fields */ 198 GUEST_SEL_ES = 0x0800ul, 199 GUEST_SEL_CS = 0x0802ul, 200 GUEST_SEL_SS = 0x0804ul, 201 GUEST_SEL_DS = 0x0806ul, 202 GUEST_SEL_FS = 0x0808ul, 203 GUEST_SEL_GS = 0x080aul, 204 GUEST_SEL_LDTR = 0x080cul, 205 GUEST_SEL_TR = 0x080eul, 206 GUEST_INT_STATUS = 0x0810ul, 207 GUEST_PML_INDEX = 0x0812ul, 208 209 /* 16-Bit Host State Fields */ 210 HOST_SEL_ES = 0x0c00ul, 211 HOST_SEL_CS = 0x0c02ul, 212 HOST_SEL_SS = 0x0c04ul, 213 HOST_SEL_DS = 0x0c06ul, 214 HOST_SEL_FS = 0x0c08ul, 215 HOST_SEL_GS = 0x0c0aul, 216 HOST_SEL_TR = 0x0c0cul, 217 218 /* 64-Bit Control Fields */ 219 IO_BITMAP_A = 0x2000ul, 220 IO_BITMAP_B = 0x2002ul, 221 MSR_BITMAP = 0x2004ul, 222 EXIT_MSR_ST_ADDR = 0x2006ul, 223 EXIT_MSR_LD_ADDR = 0x2008ul, 224 ENTER_MSR_LD_ADDR = 0x200aul, 225 VMCS_EXEC_PTR = 0x200cul, 226 TSC_OFFSET = 0x2010ul, 227 TSC_OFFSET_HI = 0x2011ul, 228 APIC_VIRT_ADDR = 0x2012ul, 229 APIC_ACCS_ADDR = 0x2014ul, 230 POSTED_INTR_DESC_ADDR = 0x2016ul, 231 EPTP = 0x201aul, 232 EPTP_HI = 0x201bul, 233 VMREAD_BITMAP = 0x2026ul, 234 VMREAD_BITMAP_HI = 0x2027ul, 235 VMWRITE_BITMAP = 0x2028ul, 236 VMWRITE_BITMAP_HI = 0x2029ul, 237 EOI_EXIT_BITMAP0 = 0x201cul, 238 EOI_EXIT_BITMAP1 = 0x201eul, 239 EOI_EXIT_BITMAP2 = 0x2020ul, 240 EOI_EXIT_BITMAP3 = 0x2022ul, 241 PMLADDR = 0x200eul, 242 PMLADDR_HI = 0x200ful, 243 244 245 /* 64-Bit Readonly Data Field */ 246 INFO_PHYS_ADDR = 0x2400ul, 247 248 /* 64-Bit Guest State */ 249 VMCS_LINK_PTR = 0x2800ul, 250 VMCS_LINK_PTR_HI = 0x2801ul, 251 GUEST_DEBUGCTL = 0x2802ul, 252 GUEST_DEBUGCTL_HI = 0x2803ul, 253 GUEST_EFER = 0x2806ul, 254 GUEST_PAT = 0x2804ul, 255 GUEST_PERF_GLOBAL_CTRL = 0x2808ul, 256 GUEST_PDPTE = 0x280aul, 257 GUEST_BNDCFGS = 0x2812ul, 258 259 /* 64-Bit Host State */ 260 HOST_PAT = 0x2c00ul, 261 HOST_EFER = 0x2c02ul, 262 HOST_PERF_GLOBAL_CTRL = 0x2c04ul, 263 264 /* 32-Bit Control Fields */ 265 PIN_CONTROLS = 0x4000ul, 266 CPU_EXEC_CTRL0 = 0x4002ul, 267 EXC_BITMAP = 0x4004ul, 268 PF_ERROR_MASK = 0x4006ul, 269 PF_ERROR_MATCH = 0x4008ul, 270 CR3_TARGET_COUNT = 0x400aul, 271 EXI_CONTROLS = 0x400cul, 272 EXI_MSR_ST_CNT = 0x400eul, 273 EXI_MSR_LD_CNT = 0x4010ul, 274 ENT_CONTROLS = 0x4012ul, 275 ENT_MSR_LD_CNT = 0x4014ul, 276 ENT_INTR_INFO = 0x4016ul, 277 ENT_INTR_ERROR = 0x4018ul, 278 ENT_INST_LEN = 0x401aul, 279 TPR_THRESHOLD = 0x401cul, 280 CPU_EXEC_CTRL1 = 0x401eul, 281 282 /* 32-Bit R/O Data Fields */ 283 VMX_INST_ERROR = 0x4400ul, 284 EXI_REASON = 0x4402ul, 285 EXI_INTR_INFO = 0x4404ul, 286 EXI_INTR_ERROR = 0x4406ul, 287 IDT_VECT_INFO = 0x4408ul, 288 IDT_VECT_ERROR = 0x440aul, 289 EXI_INST_LEN = 0x440cul, 290 EXI_INST_INFO = 0x440eul, 291 292 /* 32-Bit Guest State Fields */ 293 GUEST_LIMIT_ES = 0x4800ul, 294 GUEST_LIMIT_CS = 0x4802ul, 295 GUEST_LIMIT_SS = 0x4804ul, 296 GUEST_LIMIT_DS = 0x4806ul, 297 GUEST_LIMIT_FS = 0x4808ul, 298 GUEST_LIMIT_GS = 0x480aul, 299 GUEST_LIMIT_LDTR = 0x480cul, 300 GUEST_LIMIT_TR = 0x480eul, 301 GUEST_LIMIT_GDTR = 0x4810ul, 302 GUEST_LIMIT_IDTR = 0x4812ul, 303 GUEST_AR_ES = 0x4814ul, 304 GUEST_AR_CS = 0x4816ul, 305 GUEST_AR_SS = 0x4818ul, 306 GUEST_AR_DS = 0x481aul, 307 GUEST_AR_FS = 0x481cul, 308 GUEST_AR_GS = 0x481eul, 309 GUEST_AR_LDTR = 0x4820ul, 310 GUEST_AR_TR = 0x4822ul, 311 GUEST_INTR_STATE = 0x4824ul, 312 GUEST_ACTV_STATE = 0x4826ul, 313 GUEST_SMBASE = 0x4828ul, 314 GUEST_SYSENTER_CS = 0x482aul, 315 PREEMPT_TIMER_VALUE = 0x482eul, 316 317 /* 32-Bit Host State Fields */ 318 HOST_SYSENTER_CS = 0x4c00ul, 319 320 /* Natural-Width Control Fields */ 321 CR0_MASK = 0x6000ul, 322 CR4_MASK = 0x6002ul, 323 CR0_READ_SHADOW = 0x6004ul, 324 CR4_READ_SHADOW = 0x6006ul, 325 CR3_TARGET_0 = 0x6008ul, 326 CR3_TARGET_1 = 0x600aul, 327 CR3_TARGET_2 = 0x600cul, 328 CR3_TARGET_3 = 0x600eul, 329 330 /* Natural-Width R/O Data Fields */ 331 EXI_QUALIFICATION = 0x6400ul, 332 IO_RCX = 0x6402ul, 333 IO_RSI = 0x6404ul, 334 IO_RDI = 0x6406ul, 335 IO_RIP = 0x6408ul, 336 GUEST_LINEAR_ADDRESS = 0x640aul, 337 338 /* Natural-Width Guest State Fields */ 339 GUEST_CR0 = 0x6800ul, 340 GUEST_CR3 = 0x6802ul, 341 GUEST_CR4 = 0x6804ul, 342 GUEST_BASE_ES = 0x6806ul, 343 GUEST_BASE_CS = 0x6808ul, 344 GUEST_BASE_SS = 0x680aul, 345 GUEST_BASE_DS = 0x680cul, 346 GUEST_BASE_FS = 0x680eul, 347 GUEST_BASE_GS = 0x6810ul, 348 GUEST_BASE_LDTR = 0x6812ul, 349 GUEST_BASE_TR = 0x6814ul, 350 GUEST_BASE_GDTR = 0x6816ul, 351 GUEST_BASE_IDTR = 0x6818ul, 352 GUEST_DR7 = 0x681aul, 353 GUEST_RSP = 0x681cul, 354 GUEST_RIP = 0x681eul, 355 GUEST_RFLAGS = 0x6820ul, 356 GUEST_PENDING_DEBUG = 0x6822ul, 357 GUEST_SYSENTER_ESP = 0x6824ul, 358 GUEST_SYSENTER_EIP = 0x6826ul, 359 360 /* Natural-Width Host State Fields */ 361 HOST_CR0 = 0x6c00ul, 362 HOST_CR3 = 0x6c02ul, 363 HOST_CR4 = 0x6c04ul, 364 HOST_BASE_FS = 0x6c06ul, 365 HOST_BASE_GS = 0x6c08ul, 366 HOST_BASE_TR = 0x6c0aul, 367 HOST_BASE_GDTR = 0x6c0cul, 368 HOST_BASE_IDTR = 0x6c0eul, 369 HOST_SYSENTER_ESP = 0x6c10ul, 370 HOST_SYSENTER_EIP = 0x6c12ul, 371 HOST_RSP = 0x6c14ul, 372 HOST_RIP = 0x6c16ul 373 }; 374 375 #define VMX_ENTRY_FAILURE (1ul << 31) 376 #define VMX_ENTRY_FLAGS (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 377 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 378 379 enum Reason { 380 VMX_EXC_NMI = 0, 381 VMX_EXTINT = 1, 382 VMX_TRIPLE_FAULT = 2, 383 VMX_INIT = 3, 384 VMX_SIPI = 4, 385 VMX_SMI_IO = 5, 386 VMX_SMI_OTHER = 6, 387 VMX_INTR_WINDOW = 7, 388 VMX_NMI_WINDOW = 8, 389 VMX_TASK_SWITCH = 9, 390 VMX_CPUID = 10, 391 VMX_GETSEC = 11, 392 VMX_HLT = 12, 393 VMX_INVD = 13, 394 VMX_INVLPG = 14, 395 VMX_RDPMC = 15, 396 VMX_RDTSC = 16, 397 VMX_RSM = 17, 398 VMX_VMCALL = 18, 399 VMX_VMCLEAR = 19, 400 VMX_VMLAUNCH = 20, 401 VMX_VMPTRLD = 21, 402 VMX_VMPTRST = 22, 403 VMX_VMREAD = 23, 404 VMX_VMRESUME = 24, 405 VMX_VMWRITE = 25, 406 VMX_VMXOFF = 26, 407 VMX_VMXON = 27, 408 VMX_CR = 28, 409 VMX_DR = 29, 410 VMX_IO = 30, 411 VMX_RDMSR = 31, 412 VMX_WRMSR = 32, 413 VMX_FAIL_STATE = 33, 414 VMX_FAIL_MSR = 34, 415 VMX_MWAIT = 36, 416 VMX_MTF = 37, 417 VMX_MONITOR = 39, 418 VMX_PAUSE = 40, 419 VMX_FAIL_MCHECK = 41, 420 VMX_TPR_THRESHOLD = 43, 421 VMX_APIC_ACCESS = 44, 422 VMX_EOI_INDUCED = 45, 423 VMX_GDTR_IDTR = 46, 424 VMX_LDTR_TR = 47, 425 VMX_EPT_VIOLATION = 48, 426 VMX_EPT_MISCONFIG = 49, 427 VMX_INVEPT = 50, 428 VMX_PREEMPT = 52, 429 VMX_INVVPID = 53, 430 VMX_WBINVD = 54, 431 VMX_XSETBV = 55, 432 VMX_APIC_WRITE = 56, 433 VMX_RDRAND = 57, 434 VMX_INVPCID = 58, 435 VMX_VMFUNC = 59, 436 VMX_RDSEED = 61, 437 VMX_PML_FULL = 62, 438 VMX_XSAVES = 63, 439 VMX_XRSTORS = 64, 440 }; 441 442 enum Ctrl_exi { 443 EXI_SAVE_DBGCTLS = 1UL << 2, 444 EXI_HOST_64 = 1UL << 9, 445 EXI_LOAD_PERF = 1UL << 12, 446 EXI_INTA = 1UL << 15, 447 EXI_SAVE_PAT = 1UL << 18, 448 EXI_LOAD_PAT = 1UL << 19, 449 EXI_SAVE_EFER = 1UL << 20, 450 EXI_LOAD_EFER = 1UL << 21, 451 EXI_SAVE_PREEMPT = 1UL << 22, 452 }; 453 454 enum Ctrl_ent { 455 ENT_LOAD_DBGCTLS = 1UL << 2, 456 ENT_GUEST_64 = 1UL << 9, 457 ENT_LOAD_PERF = 1UL << 13, 458 ENT_LOAD_PAT = 1UL << 14, 459 ENT_LOAD_EFER = 1UL << 15, 460 ENT_LOAD_BNDCFGS = 1UL << 16 461 }; 462 463 enum Ctrl_pin { 464 PIN_EXTINT = 1ul << 0, 465 PIN_NMI = 1ul << 3, 466 PIN_VIRT_NMI = 1ul << 5, 467 PIN_PREEMPT = 1ul << 6, 468 PIN_POST_INTR = 1ul << 7, 469 }; 470 471 enum Ctrl0 { 472 CPU_INTR_WINDOW = 1ul << 2, 473 CPU_USE_TSC_OFFSET = 1ul << 3, 474 CPU_HLT = 1ul << 7, 475 CPU_INVLPG = 1ul << 9, 476 CPU_MWAIT = 1ul << 10, 477 CPU_RDPMC = 1ul << 11, 478 CPU_RDTSC = 1ul << 12, 479 CPU_CR3_LOAD = 1ul << 15, 480 CPU_CR3_STORE = 1ul << 16, 481 CPU_CR8_LOAD = 1ul << 19, 482 CPU_CR8_STORE = 1ul << 20, 483 CPU_TPR_SHADOW = 1ul << 21, 484 CPU_NMI_WINDOW = 1ul << 22, 485 CPU_IO = 1ul << 24, 486 CPU_IO_BITMAP = 1ul << 25, 487 CPU_MTF = 1ul << 27, 488 CPU_MSR_BITMAP = 1ul << 28, 489 CPU_MONITOR = 1ul << 29, 490 CPU_PAUSE = 1ul << 30, 491 CPU_SECONDARY = 1ul << 31, 492 }; 493 494 enum Ctrl1 { 495 CPU_VIRT_APIC_ACCESSES = 1ul << 0, 496 CPU_EPT = 1ul << 1, 497 CPU_DESC_TABLE = 1ul << 2, 498 CPU_RDTSCP = 1ul << 3, 499 CPU_VIRT_X2APIC = 1ul << 4, 500 CPU_VPID = 1ul << 5, 501 CPU_WBINVD = 1ul << 6, 502 CPU_URG = 1ul << 7, 503 CPU_APIC_REG_VIRT = 1ul << 8, 504 CPU_VINTD = 1ul << 9, 505 CPU_RDRAND = 1ul << 11, 506 CPU_SHADOW_VMCS = 1ul << 14, 507 CPU_RDSEED = 1ul << 16, 508 CPU_PML = 1ul << 17, 509 CPU_USE_TSC_SCALING = 1ul << 25, 510 }; 511 512 enum Intr_type { 513 VMX_INTR_TYPE_EXT_INTR = 0, 514 VMX_INTR_TYPE_NMI_INTR = 2, 515 VMX_INTR_TYPE_HARD_EXCEPTION = 3, 516 VMX_INTR_TYPE_SOFT_INTR = 4, 517 VMX_INTR_TYPE_SOFT_EXCEPTION = 6, 518 }; 519 520 /* 521 * Interruption-information format 522 */ 523 #define INTR_INFO_VECTOR_MASK 0xff /* 7:0 */ 524 #define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */ 525 #define INTR_INFO_DELIVER_CODE_MASK 0x800 /* 11 */ 526 #define INTR_INFO_UNBLOCK_NMI_MASK 0x1000 /* 12 */ 527 #define INTR_INFO_VALID_MASK 0x80000000 /* 31 */ 528 529 #define INTR_INFO_INTR_TYPE_SHIFT 8 530 531 #define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ 532 #define INTR_TYPE_RESERVED (1 << 8) /* reserved */ 533 #define INTR_TYPE_NMI_INTR (2 << 8) /* NMI */ 534 #define INTR_TYPE_HARD_EXCEPTION (3 << 8) /* processor exception */ 535 #define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ 536 #define INTR_TYPE_PRIV_SW_EXCEPTION (5 << 8) /* priv. software exception */ 537 #define INTR_TYPE_SOFT_EXCEPTION (6 << 8) /* software exception */ 538 #define INTR_TYPE_OTHER_EVENT (7 << 8) /* other event */ 539 540 /* 541 * Guest interruptibility state 542 */ 543 #define GUEST_INTR_STATE_STI (1 << 0) 544 #define GUEST_INTR_STATE_MOVSS (1 << 1) 545 #define GUEST_INTR_STATE_SMI (1 << 2) 546 #define GUEST_INTR_STATE_NMI (1 << 3) 547 #define GUEST_INTR_STATE_ENCLAVE (1 << 4) 548 549 /* 550 * VM-instruction error numbers 551 */ 552 enum vm_instruction_error_number { 553 VMXERR_VMCALL_IN_VMX_ROOT_OPERATION = 1, 554 VMXERR_VMCLEAR_INVALID_ADDRESS = 2, 555 VMXERR_VMCLEAR_VMXON_POINTER = 3, 556 VMXERR_VMLAUNCH_NONCLEAR_VMCS = 4, 557 VMXERR_VMRESUME_NONLAUNCHED_VMCS = 5, 558 VMXERR_VMRESUME_AFTER_VMXOFF = 6, 559 VMXERR_ENTRY_INVALID_CONTROL_FIELD = 7, 560 VMXERR_ENTRY_INVALID_HOST_STATE_FIELD = 8, 561 VMXERR_VMPTRLD_INVALID_ADDRESS = 9, 562 VMXERR_VMPTRLD_VMXON_POINTER = 10, 563 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID = 11, 564 VMXERR_UNSUPPORTED_VMCS_COMPONENT = 12, 565 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT = 13, 566 VMXERR_VMXON_IN_VMX_ROOT_OPERATION = 15, 567 VMXERR_ENTRY_INVALID_EXECUTIVE_VMCS_POINTER = 16, 568 VMXERR_ENTRY_NONLAUNCHED_EXECUTIVE_VMCS = 17, 569 VMXERR_ENTRY_EXECUTIVE_VMCS_POINTER_NOT_VMXON_POINTER = 18, 570 VMXERR_VMCALL_NONCLEAR_VMCS = 19, 571 VMXERR_VMCALL_INVALID_VM_EXIT_CONTROL_FIELDS = 20, 572 VMXERR_VMCALL_INCORRECT_MSEG_REVISION_ID = 22, 573 VMXERR_VMXOFF_UNDER_DUAL_MONITOR_TREATMENT_OF_SMIS_AND_SMM = 23, 574 VMXERR_VMCALL_INVALID_SMM_MONITOR_FEATURES = 24, 575 VMXERR_ENTRY_INVALID_VM_EXECUTION_CONTROL_FIELDS_IN_EXECUTIVE_VMCS = 25, 576 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS = 26, 577 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28, 578 }; 579 580 enum vm_entry_failure_code { 581 ENTRY_FAIL_DEFAULT = 0, 582 ENTRY_FAIL_PDPTE = 2, 583 ENTRY_FAIL_NMI = 3, 584 ENTRY_FAIL_VMCS_LINK_PTR = 4, 585 }; 586 587 #define SAVE_GPR \ 588 "xchg %rax, regs\n\t" \ 589 "xchg %rcx, regs+0x8\n\t" \ 590 "xchg %rdx, regs+0x10\n\t" \ 591 "xchg %rbx, regs+0x18\n\t" \ 592 "xchg %rbp, regs+0x28\n\t" \ 593 "xchg %rsi, regs+0x30\n\t" \ 594 "xchg %rdi, regs+0x38\n\t" \ 595 "xchg %r8, regs+0x40\n\t" \ 596 "xchg %r9, regs+0x48\n\t" \ 597 "xchg %r10, regs+0x50\n\t" \ 598 "xchg %r11, regs+0x58\n\t" \ 599 "xchg %r12, regs+0x60\n\t" \ 600 "xchg %r13, regs+0x68\n\t" \ 601 "xchg %r14, regs+0x70\n\t" \ 602 "xchg %r15, regs+0x78\n\t" 603 604 #define LOAD_GPR SAVE_GPR 605 606 #define SAVE_GPR_C \ 607 "xchg %%rax, regs\n\t" \ 608 "xchg %%rcx, regs+0x8\n\t" \ 609 "xchg %%rdx, regs+0x10\n\t" \ 610 "xchg %%rbx, regs+0x18\n\t" \ 611 "xchg %%rbp, regs+0x28\n\t" \ 612 "xchg %%rsi, regs+0x30\n\t" \ 613 "xchg %%rdi, regs+0x38\n\t" \ 614 "xchg %%r8, regs+0x40\n\t" \ 615 "xchg %%r9, regs+0x48\n\t" \ 616 "xchg %%r10, regs+0x50\n\t" \ 617 "xchg %%r11, regs+0x58\n\t" \ 618 "xchg %%r12, regs+0x60\n\t" \ 619 "xchg %%r13, regs+0x68\n\t" \ 620 "xchg %%r14, regs+0x70\n\t" \ 621 "xchg %%r15, regs+0x78\n\t" 622 623 #define LOAD_GPR_C SAVE_GPR_C 624 625 #define VMX_IO_SIZE_MASK 0x7 626 #define _VMX_IO_BYTE 0 627 #define _VMX_IO_WORD 1 628 #define _VMX_IO_LONG 3 629 #define VMX_IO_DIRECTION_MASK (1ul << 3) 630 #define VMX_IO_IN (1ul << 3) 631 #define VMX_IO_OUT 0 632 #define VMX_IO_STRING (1ul << 4) 633 #define VMX_IO_REP (1ul << 5) 634 #define VMX_IO_OPRAND_IMM (1ul << 6) 635 #define VMX_IO_PORT_MASK 0xFFFF0000 636 #define VMX_IO_PORT_SHIFT 16 637 638 #define VMX_TEST_START 0 639 #define VMX_TEST_VMEXIT 1 640 #define VMX_TEST_EXIT 2 641 #define VMX_TEST_RESUME 3 642 #define VMX_TEST_VMABORT 4 643 #define VMX_TEST_VMSKIP 5 644 645 #define HYPERCALL_BIT (1ul << 12) 646 #define HYPERCALL_MASK 0xFFF 647 #define HYPERCALL_VMEXIT 0x1 648 #define HYPERCALL_VMABORT 0x2 649 #define HYPERCALL_VMSKIP 0x3 650 651 #define EPTP_PG_WALK_LEN_SHIFT 3ul 652 #define EPTP_PG_WALK_LEN_MASK 0x38ul 653 #define EPTP_RESERV_BITS_MASK 0x1ful 654 #define EPTP_RESERV_BITS_SHIFT 0x7ul 655 #define EPTP_AD_FLAG (1ul << 6) 656 657 #define EPT_MEM_TYPE_UC 0ul 658 #define EPT_MEM_TYPE_WC 1ul 659 #define EPT_MEM_TYPE_WT 4ul 660 #define EPT_MEM_TYPE_WP 5ul 661 #define EPT_MEM_TYPE_WB 6ul 662 663 #define EPT_RA 1ul 664 #define EPT_WA 2ul 665 #define EPT_EA 4ul 666 #define EPT_PRESENT (EPT_RA | EPT_WA | EPT_EA) 667 #define EPT_ACCESS_FLAG (1ul << 8) 668 #define EPT_DIRTY_FLAG (1ul << 9) 669 #define EPT_LARGE_PAGE (1ul << 7) 670 #define EPT_MEM_TYPE_SHIFT 3ul 671 #define EPT_MEM_TYPE_MASK 0x7ul 672 #define EPT_IGNORE_PAT (1ul << 6) 673 #define EPT_SUPPRESS_VE (1ull << 63) 674 675 #define EPT_CAP_EXEC_ONLY (1ull << 0) 676 #define EPT_CAP_PWL4 (1ull << 6) 677 #define EPT_CAP_PWL5 (1ull << 7) 678 #define EPT_CAP_UC (1ull << 8) 679 #define EPT_CAP_WB (1ull << 14) 680 #define EPT_CAP_2M_PAGE (1ull << 16) 681 #define EPT_CAP_1G_PAGE (1ull << 17) 682 #define EPT_CAP_INVEPT (1ull << 20) 683 #define EPT_CAP_AD_FLAG (1ull << 21) 684 #define EPT_CAP_ADV_EPT_INFO (1ull << 22) 685 #define EPT_CAP_INVEPT_SINGLE (1ull << 25) 686 #define EPT_CAP_INVEPT_ALL (1ull << 26) 687 #define VPID_CAP_INVVPID (1ull << 32) 688 #define VPID_CAP_INVVPID_ADDR (1ull << 40) 689 #define VPID_CAP_INVVPID_CXTGLB (1ull << 41) 690 #define VPID_CAP_INVVPID_ALL (1ull << 42) 691 #define VPID_CAP_INVVPID_CXTLOC (1ull << 43) 692 693 #define PAGE_SIZE_2M (512 * PAGE_SIZE) 694 #define PAGE_SIZE_1G (512 * PAGE_SIZE_2M) 695 #define EPT_PAGE_LEVEL 4 696 #define EPT_PGDIR_WIDTH 9 697 #define EPT_PGDIR_MASK 511 698 #define EPT_PGDIR_ENTRIES (1 << EPT_PGDIR_WIDTH) 699 #define EPT_LEVEL_SHIFT(level) (((level)-1) * EPT_PGDIR_WIDTH + 12) 700 #define EPT_ADDR_MASK GENMASK_ULL(51, 12) 701 #define PAGE_MASK_2M (~(PAGE_SIZE_2M-1)) 702 703 #define EPT_VLT_RD (1ull << 0) 704 #define EPT_VLT_WR (1ull << 1) 705 #define EPT_VLT_FETCH (1ull << 2) 706 #define EPT_VLT_PERM_RD (1ull << 3) 707 #define EPT_VLT_PERM_WR (1ull << 4) 708 #define EPT_VLT_PERM_EX (1ull << 5) 709 #define EPT_VLT_PERM_USER_EX (1ull << 6) 710 #define EPT_VLT_PERMS (EPT_VLT_PERM_RD | EPT_VLT_PERM_WR | \ 711 EPT_VLT_PERM_EX) 712 #define EPT_VLT_LADDR_VLD (1ull << 7) 713 #define EPT_VLT_PADDR (1ull << 8) 714 #define EPT_VLT_GUEST_USER (1ull << 9) 715 #define EPT_VLT_GUEST_RW (1ull << 10) 716 #define EPT_VLT_GUEST_EX (1ull << 11) 717 #define EPT_VLT_GUEST_MASK (EPT_VLT_GUEST_USER | EPT_VLT_GUEST_RW | \ 718 EPT_VLT_GUEST_EX) 719 720 #define MAGIC_VAL_1 0x12345678ul 721 #define MAGIC_VAL_2 0x87654321ul 722 #define MAGIC_VAL_3 0xfffffffful 723 #define MAGIC_VAL_4 0xdeadbeeful 724 725 #define INVEPT_SINGLE 1 726 #define INVEPT_GLOBAL 2 727 728 #define INVVPID_ADDR 0 729 #define INVVPID_CONTEXT_GLOBAL 1 730 #define INVVPID_ALL 2 731 #define INVVPID_CONTEXT_LOCAL 3 732 733 #define ACTV_ACTIVE 0 734 #define ACTV_HLT 1 735 #define ACTV_SHUTDOWN 2 736 #define ACTV_WAIT_SIPI 3 737 738 /* 739 * VMCS field encoding: 740 * Bit 0: High-access 741 * Bits 1-9: Index 742 * Bits 10-12: Type 743 * Bits 13-15: Width 744 * Bits 15-64: Reserved 745 */ 746 #define VMCS_FIELD_HIGH_SHIFT (0) 747 #define VMCS_FIELD_INDEX_SHIFT (1) 748 #define VMCS_FIELD_INDEX_MASK GENMASK(9, 1) 749 #define VMCS_FIELD_TYPE_SHIFT (10) 750 #define VMCS_FIELD_WIDTH_SHIFT (13) 751 #define VMCS_FIELD_RESERVED_SHIFT (15) 752 #define VMCS_FIELD_BIT_SIZE (BITS_PER_LONG) 753 754 extern struct regs regs; 755 756 extern union vmx_basic_msr basic_msr; 757 extern union vmx_ctrl_msr ctrl_pin_rev; 758 extern union vmx_ctrl_msr ctrl_cpu_rev[2]; 759 extern union vmx_ctrl_msr ctrl_exit_rev; 760 extern union vmx_ctrl_msr ctrl_enter_rev; 761 extern union vmx_ept_vpid ept_vpid; 762 763 static inline bool ept_2m_supported(void) 764 { 765 return ept_vpid.val & EPT_CAP_2M_PAGE; 766 } 767 768 static inline bool ept_1g_supported(void) 769 { 770 return ept_vpid.val & EPT_CAP_1G_PAGE; 771 } 772 773 static inline bool ept_huge_pages_supported(int level) 774 { 775 if (level == 2) 776 return ept_2m_supported(); 777 else if (level == 3) 778 return ept_1g_supported(); 779 else 780 return false; 781 } 782 783 static inline bool ept_execute_only_supported(void) 784 { 785 return ept_vpid.val & EPT_CAP_EXEC_ONLY; 786 } 787 788 static inline bool ept_ad_bits_supported(void) 789 { 790 return ept_vpid.val & EPT_CAP_AD_FLAG; 791 } 792 793 static inline bool is_4_level_ept_supported(void) 794 { 795 return ept_vpid.val & EPT_CAP_PWL4; 796 } 797 798 static inline bool is_5_level_ept_supported(void) 799 { 800 return ept_vpid.val & EPT_CAP_PWL5; 801 } 802 803 static inline bool is_ept_memtype_supported(int type) 804 { 805 if (type == EPT_MEM_TYPE_UC) 806 return ept_vpid.val & EPT_CAP_UC; 807 808 if (type == EPT_MEM_TYPE_WB) 809 return ept_vpid.val & EPT_CAP_WB; 810 811 return false; 812 } 813 814 static inline bool is_invept_type_supported(u64 type) 815 { 816 if (type < INVEPT_SINGLE || type > INVEPT_GLOBAL) 817 return false; 818 819 return ept_vpid.val & (EPT_CAP_INVEPT_SINGLE << (type - INVEPT_SINGLE)); 820 } 821 822 static inline bool is_vpid_supported(void) 823 { 824 return (ctrl_cpu_rev[0].clr & CPU_SECONDARY) && 825 (ctrl_cpu_rev[1].clr & CPU_VPID); 826 } 827 828 static inline bool is_invvpid_supported(void) 829 { 830 return ept_vpid.val & VPID_CAP_INVVPID; 831 } 832 833 static inline bool is_invvpid_type_supported(unsigned long type) 834 { 835 if (type < INVVPID_ADDR || type > INVVPID_CONTEXT_LOCAL) 836 return false; 837 838 return ept_vpid.val & (VPID_CAP_INVVPID_ADDR << (type - INVVPID_ADDR)); 839 } 840 841 extern u64 *bsp_vmxon_region; 842 extern bool launched; 843 844 void vmx_set_test_stage(u32 s); 845 u32 vmx_get_test_stage(void); 846 void vmx_inc_test_stage(void); 847 848 /* -1 on VM-Fail, 0 on success, >1 on fault */ 849 static int __vmxon_safe(u64 *vmxon_region) 850 { 851 bool vmfail; 852 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 853 854 asm volatile ("push %1\n\t" 855 "popf\n\t" 856 ASM_TRY("1f") "vmxon %2\n\t" 857 "setbe %0\n\t" 858 "jmp 2f\n\t" 859 "1: movb $0, %0\n\t" 860 "2:\n\t" 861 : "=q" (vmfail) : "q" (rflags), "m" (vmxon_region) : "cc"); 862 863 if (vmfail) 864 return -1; 865 866 return exception_vector(); 867 } 868 869 static int vmxon_safe(void) 870 { 871 return __vmxon_safe(bsp_vmxon_region); 872 } 873 874 static int vmx_on(void) 875 { 876 return vmxon_safe(); 877 } 878 879 static int vmx_off(void) 880 { 881 bool ret; 882 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 883 884 asm volatile("push %1; popf; vmxoff; setbe %0\n\t" 885 : "=q"(ret) : "q" (rflags) : "cc"); 886 return ret; 887 } 888 889 static inline int make_vmcs_current(struct vmcs *vmcs) 890 { 891 bool ret; 892 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 893 894 asm volatile ("push %1; popf; vmptrld %2; setbe %0" 895 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 896 return ret; 897 } 898 899 static inline int vmcs_clear(struct vmcs *vmcs) 900 { 901 bool ret; 902 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 903 904 asm volatile ("push %1; popf; vmclear %2; setbe %0" 905 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 906 return ret; 907 } 908 909 static inline u64 vmcs_read(enum Encoding enc) 910 { 911 u64 val; 912 asm volatile ("vmread %1, %0" : "=rm" (val) : "r" ((u64)enc) : "cc"); 913 return val; 914 } 915 916 /* 917 * VMREAD with a guaranteed memory operand, used to test KVM's MMU by forcing 918 * KVM to translate GVA->GPA. 919 */ 920 static inline u64 vmcs_readm(enum Encoding enc) 921 { 922 u64 val; 923 924 asm volatile ("vmread %1, %0" : "=m" (val) : "r" ((u64)enc) : "cc"); 925 return val; 926 } 927 928 static inline int vmcs_read_safe(enum Encoding enc, u64 *value) 929 { 930 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 931 u64 encoding = enc; 932 u64 val; 933 934 asm volatile ("shl $8, %%rax;" 935 "sahf;" 936 "vmread %[encoding], %[val];" 937 "lahf;" 938 "shr $8, %%rax" 939 : /* output */ [val]"=rm"(val), "+a"(rflags) 940 : /* input */ [encoding]"r"(encoding) 941 : /* clobber */ "cc"); 942 943 *value = val; 944 return rflags & (X86_EFLAGS_CF | X86_EFLAGS_ZF); 945 } 946 947 static inline int vmcs_write(enum Encoding enc, u64 val) 948 { 949 bool ret; 950 asm volatile ("vmwrite %1, %2; setbe %0" 951 : "=q"(ret) : "rm" (val), "r" ((u64)enc) : "cc"); 952 return ret; 953 } 954 955 static inline int vmcs_set_bits(enum Encoding enc, u64 val) 956 { 957 return vmcs_write(enc, vmcs_read(enc) | val); 958 } 959 960 static inline int vmcs_clear_bits(enum Encoding enc, u64 val) 961 { 962 return vmcs_write(enc, vmcs_read(enc) & ~val); 963 } 964 965 static inline int vmcs_save(struct vmcs **vmcs) 966 { 967 bool ret; 968 unsigned long pa; 969 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 970 971 asm volatile ("push %2; popf; vmptrst %1; setbe %0" 972 : "=q" (ret), "=m" (pa) : "r" (rflags) : "cc"); 973 *vmcs = (pa == -1ull) ? NULL : phys_to_virt(pa); 974 return ret; 975 } 976 977 static inline int __invept(unsigned long type, u64 eptp) 978 { 979 bool failed = false; 980 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 981 982 struct { 983 u64 eptp, gpa; 984 } operand = {eptp, 0}; 985 asm volatile("push %1; popf; invept %2, %3; setbe %0" 986 : "=q" (failed) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 987 return failed ? -1: 0; 988 } 989 990 static inline void invept(unsigned long type, u64 eptp) 991 { 992 __TEST_ASSERT(!__invept(type, eptp)); 993 } 994 995 static inline int __invvpid(unsigned long type, u64 vpid, u64 gla) 996 { 997 bool failed = false; 998 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 999 1000 struct invvpid_operand operand = {vpid, gla}; 1001 asm volatile("push %1; popf; invvpid %2, %3; setbe %0" 1002 : "=q" (failed) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 1003 return failed ? -1: 0; 1004 } 1005 1006 static inline void invvpid(unsigned long type, u64 vpid, u64 gla) 1007 { 1008 __TEST_ASSERT(!__invvpid(type, vpid, gla)); 1009 } 1010 1011 void enable_vmx(void); 1012 void init_vmx(u64 *vmxon_region); 1013 int init_vmcs(struct vmcs **vmcs); 1014 1015 const char *exit_reason_description(u64 reason); 1016 void print_vmexit_info(union exit_reason exit_reason); 1017 void print_vmentry_failure_info(struct vmentry_result *result); 1018 void install_ept_entry(unsigned long *pml4, int pte_level, 1019 unsigned long guest_addr, unsigned long pte, 1020 unsigned long *pt_page); 1021 void install_1g_ept(unsigned long *pml4, unsigned long phys, 1022 unsigned long guest_addr, u64 perm); 1023 void install_2m_ept(unsigned long *pml4, unsigned long phys, 1024 unsigned long guest_addr, u64 perm); 1025 void install_ept(unsigned long *pml4, unsigned long phys, 1026 unsigned long guest_addr, u64 perm); 1027 void setup_ept_range(unsigned long *pml4, unsigned long start, 1028 unsigned long len, int map_1g, int map_2m, u64 perm); 1029 bool get_ept_pte(unsigned long *pml4, unsigned long guest_addr, int level, 1030 unsigned long *pte); 1031 void set_ept_pte(unsigned long *pml4, unsigned long guest_addr, 1032 int level, u64 pte_val); 1033 void check_ept_ad(unsigned long *pml4, u64 guest_cr3, 1034 unsigned long guest_addr, int expected_gpa_ad, 1035 int expected_pt_ad); 1036 void clear_ept_ad(unsigned long *pml4, u64 guest_cr3, 1037 unsigned long guest_addr); 1038 1039 #define ABORT_ON_EARLY_VMENTRY_FAIL 0x1 1040 #define ABORT_ON_INVALID_GUEST_STATE 0x2 1041 1042 void __enter_guest(u8 abort_flag, struct vmentry_result *result); 1043 void enter_guest(void); 1044 void enter_guest_with_bad_controls(void); 1045 void hypercall(u32 hypercall_no); 1046 1047 typedef void (*test_guest_func)(void); 1048 typedef void (*test_teardown_func)(void *data); 1049 void test_set_guest(test_guest_func func); 1050 void test_override_guest(test_guest_func func); 1051 void test_add_teardown(test_teardown_func func, void *data); 1052 void test_skip(const char *msg); 1053 void test_set_guest_finished(void); 1054 1055 #endif 1056