1 #ifndef __VMX_H 2 #define __VMX_H 3 4 #include "libcflat.h" 5 #include "processor.h" 6 #include "bitops.h" 7 #include "asm/page.h" 8 #include "asm/io.h" 9 10 struct vmcs_hdr { 11 u32 revision_id:31; 12 u32 shadow_vmcs:1; 13 }; 14 15 struct vmcs { 16 struct vmcs_hdr hdr; 17 u32 abort; /* VMX-abort indicator */ 18 /* VMCS data */ 19 char data[0]; 20 }; 21 22 struct invvpid_operand { 23 u64 vpid; 24 u64 gla; 25 }; 26 27 struct regs { 28 u64 rax; 29 u64 rcx; 30 u64 rdx; 31 u64 rbx; 32 u64 cr2; 33 u64 rbp; 34 u64 rsi; 35 u64 rdi; 36 u64 r8; 37 u64 r9; 38 u64 r10; 39 u64 r11; 40 u64 r12; 41 u64 r13; 42 u64 r14; 43 u64 r15; 44 u64 rflags; 45 }; 46 47 union exit_reason { 48 struct { 49 u32 basic : 16; 50 u32 reserved16 : 1; 51 u32 reserved17 : 1; 52 u32 reserved18 : 1; 53 u32 reserved19 : 1; 54 u32 reserved20 : 1; 55 u32 reserved21 : 1; 56 u32 reserved22 : 1; 57 u32 reserved23 : 1; 58 u32 reserved24 : 1; 59 u32 reserved25 : 1; 60 u32 reserved26 : 1; 61 u32 enclave_mode : 1; 62 u32 smi_pending_mtf : 1; 63 u32 smi_from_vmx_root : 1; 64 u32 reserved30 : 1; 65 u32 failed_vmentry : 1; 66 }; 67 u32 full; 68 }; 69 70 struct vmentry_result { 71 /* Instruction mnemonic (for convenience). */ 72 const char *instr; 73 /* Did the test attempt vmlaunch or vmresume? */ 74 bool vmlaunch; 75 /* Did the instruction VM-Fail? */ 76 bool vm_fail; 77 /* Did the VM-Entry fully enter the guest? */ 78 bool entered; 79 /* VM-Exit reason, valid iff !vm_fail */ 80 union exit_reason exit_reason; 81 /* Contents of [re]flags after failed entry. */ 82 unsigned long flags; 83 }; 84 85 struct vmx_test { 86 const char *name; 87 int (*init)(struct vmcs *vmcs); 88 void (*guest_main)(void); 89 int (*exit_handler)(union exit_reason exit_reason); 90 void (*syscall_handler)(u64 syscall_no); 91 struct regs guest_regs; 92 int (*entry_failure_handler)(struct vmentry_result *result); 93 struct vmcs *vmcs; 94 int exits; 95 /* Alternative test interface. */ 96 void (*v2)(void); 97 }; 98 99 union vmx_basic { 100 u64 val; 101 struct { 102 u32 revision; 103 u32 size:13, 104 reserved1: 3, 105 width:1, 106 dual:1, 107 type:4, 108 insouts:1, 109 ctrl:1, 110 reserved2:8; 111 }; 112 }; 113 114 union vmx_ctrl_msr { 115 u64 val; 116 struct { 117 u32 set, clr; 118 }; 119 }; 120 121 union vmx_misc { 122 u64 val; 123 struct { 124 u32 pt_bit:5, 125 stores_lma:1, 126 act_hlt:1, 127 act_shutdown:1, 128 act_wfsipi:1, 129 :5, 130 vmx_pt:1, 131 smm_smbase:1, 132 cr3_targets:9, 133 msr_list_size:3, 134 smm_mon_ctl:1, 135 vmwrite_any:1, 136 inject_len0:1, 137 :1; 138 u32 mseg_revision; 139 }; 140 }; 141 142 union vmx_ept_vpid { 143 u64 val; 144 struct { 145 u32:16, 146 super:2, 147 : 2, 148 invept:1, 149 : 11; 150 u32 invvpid:1; 151 }; 152 }; 153 154 enum Encoding { 155 /* 16-Bit Control Fields */ 156 VPID = 0x0000ul, 157 /* Posted-interrupt notification vector */ 158 PINV = 0x0002ul, 159 /* EPTP index */ 160 EPTP_IDX = 0x0004ul, 161 162 /* 16-Bit Guest State Fields */ 163 GUEST_SEL_ES = 0x0800ul, 164 GUEST_SEL_CS = 0x0802ul, 165 GUEST_SEL_SS = 0x0804ul, 166 GUEST_SEL_DS = 0x0806ul, 167 GUEST_SEL_FS = 0x0808ul, 168 GUEST_SEL_GS = 0x080aul, 169 GUEST_SEL_LDTR = 0x080cul, 170 GUEST_SEL_TR = 0x080eul, 171 GUEST_INT_STATUS = 0x0810ul, 172 GUEST_PML_INDEX = 0x0812ul, 173 174 /* 16-Bit Host State Fields */ 175 HOST_SEL_ES = 0x0c00ul, 176 HOST_SEL_CS = 0x0c02ul, 177 HOST_SEL_SS = 0x0c04ul, 178 HOST_SEL_DS = 0x0c06ul, 179 HOST_SEL_FS = 0x0c08ul, 180 HOST_SEL_GS = 0x0c0aul, 181 HOST_SEL_TR = 0x0c0cul, 182 183 /* 64-Bit Control Fields */ 184 IO_BITMAP_A = 0x2000ul, 185 IO_BITMAP_B = 0x2002ul, 186 MSR_BITMAP = 0x2004ul, 187 EXIT_MSR_ST_ADDR = 0x2006ul, 188 EXIT_MSR_LD_ADDR = 0x2008ul, 189 ENTER_MSR_LD_ADDR = 0x200aul, 190 VMCS_EXEC_PTR = 0x200cul, 191 TSC_OFFSET = 0x2010ul, 192 TSC_OFFSET_HI = 0x2011ul, 193 APIC_VIRT_ADDR = 0x2012ul, 194 APIC_ACCS_ADDR = 0x2014ul, 195 POSTED_INTR_DESC_ADDR = 0x2016ul, 196 EPTP = 0x201aul, 197 EPTP_HI = 0x201bul, 198 VMREAD_BITMAP = 0x2026ul, 199 VMREAD_BITMAP_HI = 0x2027ul, 200 VMWRITE_BITMAP = 0x2028ul, 201 VMWRITE_BITMAP_HI = 0x2029ul, 202 EOI_EXIT_BITMAP0 = 0x201cul, 203 EOI_EXIT_BITMAP1 = 0x201eul, 204 EOI_EXIT_BITMAP2 = 0x2020ul, 205 EOI_EXIT_BITMAP3 = 0x2022ul, 206 PMLADDR = 0x200eul, 207 PMLADDR_HI = 0x200ful, 208 209 210 /* 64-Bit Readonly Data Field */ 211 INFO_PHYS_ADDR = 0x2400ul, 212 213 /* 64-Bit Guest State */ 214 VMCS_LINK_PTR = 0x2800ul, 215 VMCS_LINK_PTR_HI = 0x2801ul, 216 GUEST_DEBUGCTL = 0x2802ul, 217 GUEST_DEBUGCTL_HI = 0x2803ul, 218 GUEST_EFER = 0x2806ul, 219 GUEST_PAT = 0x2804ul, 220 GUEST_PERF_GLOBAL_CTRL = 0x2808ul, 221 GUEST_PDPTE = 0x280aul, 222 GUEST_BNDCFGS = 0x2812ul, 223 224 /* 64-Bit Host State */ 225 HOST_PAT = 0x2c00ul, 226 HOST_EFER = 0x2c02ul, 227 HOST_PERF_GLOBAL_CTRL = 0x2c04ul, 228 229 /* 32-Bit Control Fields */ 230 PIN_CONTROLS = 0x4000ul, 231 CPU_EXEC_CTRL0 = 0x4002ul, 232 EXC_BITMAP = 0x4004ul, 233 PF_ERROR_MASK = 0x4006ul, 234 PF_ERROR_MATCH = 0x4008ul, 235 CR3_TARGET_COUNT = 0x400aul, 236 EXI_CONTROLS = 0x400cul, 237 EXI_MSR_ST_CNT = 0x400eul, 238 EXI_MSR_LD_CNT = 0x4010ul, 239 ENT_CONTROLS = 0x4012ul, 240 ENT_MSR_LD_CNT = 0x4014ul, 241 ENT_INTR_INFO = 0x4016ul, 242 ENT_INTR_ERROR = 0x4018ul, 243 ENT_INST_LEN = 0x401aul, 244 TPR_THRESHOLD = 0x401cul, 245 CPU_EXEC_CTRL1 = 0x401eul, 246 247 /* 32-Bit R/O Data Fields */ 248 VMX_INST_ERROR = 0x4400ul, 249 EXI_REASON = 0x4402ul, 250 EXI_INTR_INFO = 0x4404ul, 251 EXI_INTR_ERROR = 0x4406ul, 252 IDT_VECT_INFO = 0x4408ul, 253 IDT_VECT_ERROR = 0x440aul, 254 EXI_INST_LEN = 0x440cul, 255 EXI_INST_INFO = 0x440eul, 256 257 /* 32-Bit Guest State Fields */ 258 GUEST_LIMIT_ES = 0x4800ul, 259 GUEST_LIMIT_CS = 0x4802ul, 260 GUEST_LIMIT_SS = 0x4804ul, 261 GUEST_LIMIT_DS = 0x4806ul, 262 GUEST_LIMIT_FS = 0x4808ul, 263 GUEST_LIMIT_GS = 0x480aul, 264 GUEST_LIMIT_LDTR = 0x480cul, 265 GUEST_LIMIT_TR = 0x480eul, 266 GUEST_LIMIT_GDTR = 0x4810ul, 267 GUEST_LIMIT_IDTR = 0x4812ul, 268 GUEST_AR_ES = 0x4814ul, 269 GUEST_AR_CS = 0x4816ul, 270 GUEST_AR_SS = 0x4818ul, 271 GUEST_AR_DS = 0x481aul, 272 GUEST_AR_FS = 0x481cul, 273 GUEST_AR_GS = 0x481eul, 274 GUEST_AR_LDTR = 0x4820ul, 275 GUEST_AR_TR = 0x4822ul, 276 GUEST_INTR_STATE = 0x4824ul, 277 GUEST_ACTV_STATE = 0x4826ul, 278 GUEST_SMBASE = 0x4828ul, 279 GUEST_SYSENTER_CS = 0x482aul, 280 PREEMPT_TIMER_VALUE = 0x482eul, 281 282 /* 32-Bit Host State Fields */ 283 HOST_SYSENTER_CS = 0x4c00ul, 284 285 /* Natural-Width Control Fields */ 286 CR0_MASK = 0x6000ul, 287 CR4_MASK = 0x6002ul, 288 CR0_READ_SHADOW = 0x6004ul, 289 CR4_READ_SHADOW = 0x6006ul, 290 CR3_TARGET_0 = 0x6008ul, 291 CR3_TARGET_1 = 0x600aul, 292 CR3_TARGET_2 = 0x600cul, 293 CR3_TARGET_3 = 0x600eul, 294 295 /* Natural-Width R/O Data Fields */ 296 EXI_QUALIFICATION = 0x6400ul, 297 IO_RCX = 0x6402ul, 298 IO_RSI = 0x6404ul, 299 IO_RDI = 0x6406ul, 300 IO_RIP = 0x6408ul, 301 GUEST_LINEAR_ADDRESS = 0x640aul, 302 303 /* Natural-Width Guest State Fields */ 304 GUEST_CR0 = 0x6800ul, 305 GUEST_CR3 = 0x6802ul, 306 GUEST_CR4 = 0x6804ul, 307 GUEST_BASE_ES = 0x6806ul, 308 GUEST_BASE_CS = 0x6808ul, 309 GUEST_BASE_SS = 0x680aul, 310 GUEST_BASE_DS = 0x680cul, 311 GUEST_BASE_FS = 0x680eul, 312 GUEST_BASE_GS = 0x6810ul, 313 GUEST_BASE_LDTR = 0x6812ul, 314 GUEST_BASE_TR = 0x6814ul, 315 GUEST_BASE_GDTR = 0x6816ul, 316 GUEST_BASE_IDTR = 0x6818ul, 317 GUEST_DR7 = 0x681aul, 318 GUEST_RSP = 0x681cul, 319 GUEST_RIP = 0x681eul, 320 GUEST_RFLAGS = 0x6820ul, 321 GUEST_PENDING_DEBUG = 0x6822ul, 322 GUEST_SYSENTER_ESP = 0x6824ul, 323 GUEST_SYSENTER_EIP = 0x6826ul, 324 325 /* Natural-Width Host State Fields */ 326 HOST_CR0 = 0x6c00ul, 327 HOST_CR3 = 0x6c02ul, 328 HOST_CR4 = 0x6c04ul, 329 HOST_BASE_FS = 0x6c06ul, 330 HOST_BASE_GS = 0x6c08ul, 331 HOST_BASE_TR = 0x6c0aul, 332 HOST_BASE_GDTR = 0x6c0cul, 333 HOST_BASE_IDTR = 0x6c0eul, 334 HOST_SYSENTER_ESP = 0x6c10ul, 335 HOST_SYSENTER_EIP = 0x6c12ul, 336 HOST_RSP = 0x6c14ul, 337 HOST_RIP = 0x6c16ul 338 }; 339 340 #define VMX_ENTRY_FAILURE (1ul << 31) 341 #define VMX_ENTRY_FLAGS (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 342 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 343 344 enum Reason { 345 VMX_EXC_NMI = 0, 346 VMX_EXTINT = 1, 347 VMX_TRIPLE_FAULT = 2, 348 VMX_INIT = 3, 349 VMX_SIPI = 4, 350 VMX_SMI_IO = 5, 351 VMX_SMI_OTHER = 6, 352 VMX_INTR_WINDOW = 7, 353 VMX_NMI_WINDOW = 8, 354 VMX_TASK_SWITCH = 9, 355 VMX_CPUID = 10, 356 VMX_GETSEC = 11, 357 VMX_HLT = 12, 358 VMX_INVD = 13, 359 VMX_INVLPG = 14, 360 VMX_RDPMC = 15, 361 VMX_RDTSC = 16, 362 VMX_RSM = 17, 363 VMX_VMCALL = 18, 364 VMX_VMCLEAR = 19, 365 VMX_VMLAUNCH = 20, 366 VMX_VMPTRLD = 21, 367 VMX_VMPTRST = 22, 368 VMX_VMREAD = 23, 369 VMX_VMRESUME = 24, 370 VMX_VMWRITE = 25, 371 VMX_VMXOFF = 26, 372 VMX_VMXON = 27, 373 VMX_CR = 28, 374 VMX_DR = 29, 375 VMX_IO = 30, 376 VMX_RDMSR = 31, 377 VMX_WRMSR = 32, 378 VMX_FAIL_STATE = 33, 379 VMX_FAIL_MSR = 34, 380 VMX_MWAIT = 36, 381 VMX_MTF = 37, 382 VMX_MONITOR = 39, 383 VMX_PAUSE = 40, 384 VMX_FAIL_MCHECK = 41, 385 VMX_TPR_THRESHOLD = 43, 386 VMX_APIC_ACCESS = 44, 387 VMX_EOI_INDUCED = 45, 388 VMX_GDTR_IDTR = 46, 389 VMX_LDTR_TR = 47, 390 VMX_EPT_VIOLATION = 48, 391 VMX_EPT_MISCONFIG = 49, 392 VMX_INVEPT = 50, 393 VMX_PREEMPT = 52, 394 VMX_INVVPID = 53, 395 VMX_WBINVD = 54, 396 VMX_XSETBV = 55, 397 VMX_APIC_WRITE = 56, 398 VMX_RDRAND = 57, 399 VMX_INVPCID = 58, 400 VMX_VMFUNC = 59, 401 VMX_RDSEED = 61, 402 VMX_PML_FULL = 62, 403 VMX_XSAVES = 63, 404 VMX_XRSTORS = 64, 405 }; 406 407 enum Ctrl_exi { 408 EXI_SAVE_DBGCTLS = 1UL << 2, 409 EXI_HOST_64 = 1UL << 9, 410 EXI_LOAD_PERF = 1UL << 12, 411 EXI_INTA = 1UL << 15, 412 EXI_SAVE_PAT = 1UL << 18, 413 EXI_LOAD_PAT = 1UL << 19, 414 EXI_SAVE_EFER = 1UL << 20, 415 EXI_LOAD_EFER = 1UL << 21, 416 EXI_SAVE_PREEMPT = 1UL << 22, 417 }; 418 419 enum Ctrl_ent { 420 ENT_LOAD_DBGCTLS = 1UL << 2, 421 ENT_GUEST_64 = 1UL << 9, 422 ENT_LOAD_PERF = 1UL << 13, 423 ENT_LOAD_PAT = 1UL << 14, 424 ENT_LOAD_EFER = 1UL << 15, 425 ENT_LOAD_BNDCFGS = 1UL << 16 426 }; 427 428 enum Ctrl_pin { 429 PIN_EXTINT = 1ul << 0, 430 PIN_NMI = 1ul << 3, 431 PIN_VIRT_NMI = 1ul << 5, 432 PIN_PREEMPT = 1ul << 6, 433 PIN_POST_INTR = 1ul << 7, 434 }; 435 436 enum Ctrl0 { 437 CPU_INTR_WINDOW = 1ul << 2, 438 CPU_USE_TSC_OFFSET = 1ul << 3, 439 CPU_HLT = 1ul << 7, 440 CPU_INVLPG = 1ul << 9, 441 CPU_MWAIT = 1ul << 10, 442 CPU_RDPMC = 1ul << 11, 443 CPU_RDTSC = 1ul << 12, 444 CPU_CR3_LOAD = 1ul << 15, 445 CPU_CR3_STORE = 1ul << 16, 446 CPU_CR8_LOAD = 1ul << 19, 447 CPU_CR8_STORE = 1ul << 20, 448 CPU_TPR_SHADOW = 1ul << 21, 449 CPU_NMI_WINDOW = 1ul << 22, 450 CPU_IO = 1ul << 24, 451 CPU_IO_BITMAP = 1ul << 25, 452 CPU_MTF = 1ul << 27, 453 CPU_MSR_BITMAP = 1ul << 28, 454 CPU_MONITOR = 1ul << 29, 455 CPU_PAUSE = 1ul << 30, 456 CPU_SECONDARY = 1ul << 31, 457 }; 458 459 enum Ctrl1 { 460 CPU_VIRT_APIC_ACCESSES = 1ul << 0, 461 CPU_EPT = 1ul << 1, 462 CPU_DESC_TABLE = 1ul << 2, 463 CPU_RDTSCP = 1ul << 3, 464 CPU_VIRT_X2APIC = 1ul << 4, 465 CPU_VPID = 1ul << 5, 466 CPU_WBINVD = 1ul << 6, 467 CPU_URG = 1ul << 7, 468 CPU_APIC_REG_VIRT = 1ul << 8, 469 CPU_VINTD = 1ul << 9, 470 CPU_RDRAND = 1ul << 11, 471 CPU_SHADOW_VMCS = 1ul << 14, 472 CPU_RDSEED = 1ul << 16, 473 CPU_PML = 1ul << 17, 474 CPU_USE_TSC_SCALING = 1ul << 25, 475 }; 476 477 enum Intr_type { 478 VMX_INTR_TYPE_EXT_INTR = 0, 479 VMX_INTR_TYPE_NMI_INTR = 2, 480 VMX_INTR_TYPE_HARD_EXCEPTION = 3, 481 VMX_INTR_TYPE_SOFT_INTR = 4, 482 VMX_INTR_TYPE_SOFT_EXCEPTION = 6, 483 }; 484 485 /* 486 * Interruption-information format 487 */ 488 #define INTR_INFO_VECTOR_MASK 0xff /* 7:0 */ 489 #define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */ 490 #define INTR_INFO_DELIVER_CODE_MASK 0x800 /* 11 */ 491 #define INTR_INFO_UNBLOCK_NMI_MASK 0x1000 /* 12 */ 492 #define INTR_INFO_VALID_MASK 0x80000000 /* 31 */ 493 494 #define INTR_INFO_INTR_TYPE_SHIFT 8 495 496 #define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ 497 #define INTR_TYPE_RESERVED (1 << 8) /* reserved */ 498 #define INTR_TYPE_NMI_INTR (2 << 8) /* NMI */ 499 #define INTR_TYPE_HARD_EXCEPTION (3 << 8) /* processor exception */ 500 #define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ 501 #define INTR_TYPE_PRIV_SW_EXCEPTION (5 << 8) /* priv. software exception */ 502 #define INTR_TYPE_SOFT_EXCEPTION (6 << 8) /* software exception */ 503 #define INTR_TYPE_OTHER_EVENT (7 << 8) /* other event */ 504 505 /* 506 * Guest interruptibility state 507 */ 508 #define GUEST_INTR_STATE_STI (1 << 0) 509 #define GUEST_INTR_STATE_MOVSS (1 << 1) 510 #define GUEST_INTR_STATE_SMI (1 << 2) 511 #define GUEST_INTR_STATE_NMI (1 << 3) 512 #define GUEST_INTR_STATE_ENCLAVE (1 << 4) 513 514 /* 515 * VM-instruction error numbers 516 */ 517 enum vm_instruction_error_number { 518 VMXERR_VMCALL_IN_VMX_ROOT_OPERATION = 1, 519 VMXERR_VMCLEAR_INVALID_ADDRESS = 2, 520 VMXERR_VMCLEAR_VMXON_POINTER = 3, 521 VMXERR_VMLAUNCH_NONCLEAR_VMCS = 4, 522 VMXERR_VMRESUME_NONLAUNCHED_VMCS = 5, 523 VMXERR_VMRESUME_AFTER_VMXOFF = 6, 524 VMXERR_ENTRY_INVALID_CONTROL_FIELD = 7, 525 VMXERR_ENTRY_INVALID_HOST_STATE_FIELD = 8, 526 VMXERR_VMPTRLD_INVALID_ADDRESS = 9, 527 VMXERR_VMPTRLD_VMXON_POINTER = 10, 528 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID = 11, 529 VMXERR_UNSUPPORTED_VMCS_COMPONENT = 12, 530 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT = 13, 531 VMXERR_VMXON_IN_VMX_ROOT_OPERATION = 15, 532 VMXERR_ENTRY_INVALID_EXECUTIVE_VMCS_POINTER = 16, 533 VMXERR_ENTRY_NONLAUNCHED_EXECUTIVE_VMCS = 17, 534 VMXERR_ENTRY_EXECUTIVE_VMCS_POINTER_NOT_VMXON_POINTER = 18, 535 VMXERR_VMCALL_NONCLEAR_VMCS = 19, 536 VMXERR_VMCALL_INVALID_VM_EXIT_CONTROL_FIELDS = 20, 537 VMXERR_VMCALL_INCORRECT_MSEG_REVISION_ID = 22, 538 VMXERR_VMXOFF_UNDER_DUAL_MONITOR_TREATMENT_OF_SMIS_AND_SMM = 23, 539 VMXERR_VMCALL_INVALID_SMM_MONITOR_FEATURES = 24, 540 VMXERR_ENTRY_INVALID_VM_EXECUTION_CONTROL_FIELDS_IN_EXECUTIVE_VMCS = 25, 541 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS = 26, 542 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28, 543 }; 544 545 enum vm_entry_failure_code { 546 ENTRY_FAIL_DEFAULT = 0, 547 ENTRY_FAIL_PDPTE = 2, 548 ENTRY_FAIL_NMI = 3, 549 ENTRY_FAIL_VMCS_LINK_PTR = 4, 550 }; 551 552 #define SAVE_GPR \ 553 "xchg %rax, regs\n\t" \ 554 "xchg %rcx, regs+0x8\n\t" \ 555 "xchg %rdx, regs+0x10\n\t" \ 556 "xchg %rbx, regs+0x18\n\t" \ 557 "xchg %rbp, regs+0x28\n\t" \ 558 "xchg %rsi, regs+0x30\n\t" \ 559 "xchg %rdi, regs+0x38\n\t" \ 560 "xchg %r8, regs+0x40\n\t" \ 561 "xchg %r9, regs+0x48\n\t" \ 562 "xchg %r10, regs+0x50\n\t" \ 563 "xchg %r11, regs+0x58\n\t" \ 564 "xchg %r12, regs+0x60\n\t" \ 565 "xchg %r13, regs+0x68\n\t" \ 566 "xchg %r14, regs+0x70\n\t" \ 567 "xchg %r15, regs+0x78\n\t" 568 569 #define LOAD_GPR SAVE_GPR 570 571 #define SAVE_GPR_C \ 572 "xchg %%rax, regs\n\t" \ 573 "xchg %%rcx, regs+0x8\n\t" \ 574 "xchg %%rdx, regs+0x10\n\t" \ 575 "xchg %%rbx, regs+0x18\n\t" \ 576 "xchg %%rbp, regs+0x28\n\t" \ 577 "xchg %%rsi, regs+0x30\n\t" \ 578 "xchg %%rdi, regs+0x38\n\t" \ 579 "xchg %%r8, regs+0x40\n\t" \ 580 "xchg %%r9, regs+0x48\n\t" \ 581 "xchg %%r10, regs+0x50\n\t" \ 582 "xchg %%r11, regs+0x58\n\t" \ 583 "xchg %%r12, regs+0x60\n\t" \ 584 "xchg %%r13, regs+0x68\n\t" \ 585 "xchg %%r14, regs+0x70\n\t" \ 586 "xchg %%r15, regs+0x78\n\t" 587 588 #define LOAD_GPR_C SAVE_GPR_C 589 590 #define VMX_IO_SIZE_MASK 0x7 591 #define _VMX_IO_BYTE 0 592 #define _VMX_IO_WORD 1 593 #define _VMX_IO_LONG 3 594 #define VMX_IO_DIRECTION_MASK (1ul << 3) 595 #define VMX_IO_IN (1ul << 3) 596 #define VMX_IO_OUT 0 597 #define VMX_IO_STRING (1ul << 4) 598 #define VMX_IO_REP (1ul << 5) 599 #define VMX_IO_OPRAND_IMM (1ul << 6) 600 #define VMX_IO_PORT_MASK 0xFFFF0000 601 #define VMX_IO_PORT_SHIFT 16 602 603 #define VMX_TEST_START 0 604 #define VMX_TEST_VMEXIT 1 605 #define VMX_TEST_EXIT 2 606 #define VMX_TEST_RESUME 3 607 #define VMX_TEST_VMABORT 4 608 #define VMX_TEST_VMSKIP 5 609 610 #define HYPERCALL_BIT (1ul << 12) 611 #define HYPERCALL_MASK 0xFFF 612 #define HYPERCALL_VMEXIT 0x1 613 #define HYPERCALL_VMABORT 0x2 614 #define HYPERCALL_VMSKIP 0x3 615 616 #define EPTP_PG_WALK_LEN_SHIFT 3ul 617 #define EPTP_PG_WALK_LEN_MASK 0x38ul 618 #define EPTP_RESERV_BITS_MASK 0x1ful 619 #define EPTP_RESERV_BITS_SHIFT 0x7ul 620 #define EPTP_AD_FLAG (1ul << 6) 621 622 #define EPT_MEM_TYPE_UC 0ul 623 #define EPT_MEM_TYPE_WC 1ul 624 #define EPT_MEM_TYPE_WT 4ul 625 #define EPT_MEM_TYPE_WP 5ul 626 #define EPT_MEM_TYPE_WB 6ul 627 628 #define EPT_RA 1ul 629 #define EPT_WA 2ul 630 #define EPT_EA 4ul 631 #define EPT_PRESENT (EPT_RA | EPT_WA | EPT_EA) 632 #define EPT_ACCESS_FLAG (1ul << 8) 633 #define EPT_DIRTY_FLAG (1ul << 9) 634 #define EPT_LARGE_PAGE (1ul << 7) 635 #define EPT_MEM_TYPE_SHIFT 3ul 636 #define EPT_MEM_TYPE_MASK 0x7ul 637 #define EPT_IGNORE_PAT (1ul << 6) 638 #define EPT_SUPPRESS_VE (1ull << 63) 639 640 #define EPT_CAP_WT 1ull 641 #define EPT_CAP_PWL4 (1ull << 6) 642 #define EPT_CAP_PWL5 (1ull << 7) 643 #define EPT_CAP_UC (1ull << 8) 644 #define EPT_CAP_WB (1ull << 14) 645 #define EPT_CAP_2M_PAGE (1ull << 16) 646 #define EPT_CAP_1G_PAGE (1ull << 17) 647 #define EPT_CAP_INVEPT (1ull << 20) 648 #define EPT_CAP_AD_FLAG (1ull << 21) 649 #define EPT_CAP_ADV_EPT_INFO (1ull << 22) 650 #define EPT_CAP_INVEPT_SINGLE (1ull << 25) 651 #define EPT_CAP_INVEPT_ALL (1ull << 26) 652 #define VPID_CAP_INVVPID (1ull << 32) 653 #define VPID_CAP_INVVPID_ADDR (1ull << 40) 654 #define VPID_CAP_INVVPID_CXTGLB (1ull << 41) 655 #define VPID_CAP_INVVPID_ALL (1ull << 42) 656 #define VPID_CAP_INVVPID_CXTLOC (1ull << 43) 657 658 #define PAGE_SIZE_2M (512 * PAGE_SIZE) 659 #define PAGE_SIZE_1G (512 * PAGE_SIZE_2M) 660 #define EPT_PAGE_LEVEL 4 661 #define EPT_PGDIR_WIDTH 9 662 #define EPT_PGDIR_MASK 511 663 #define EPT_PGDIR_ENTRIES (1 << EPT_PGDIR_WIDTH) 664 #define EPT_LEVEL_SHIFT(level) (((level)-1) * EPT_PGDIR_WIDTH + 12) 665 #define EPT_ADDR_MASK GENMASK_ULL(51, 12) 666 #define PAGE_MASK_2M (~(PAGE_SIZE_2M-1)) 667 668 #define EPT_VLT_RD (1ull << 0) 669 #define EPT_VLT_WR (1ull << 1) 670 #define EPT_VLT_FETCH (1ull << 2) 671 #define EPT_VLT_PERM_RD (1ull << 3) 672 #define EPT_VLT_PERM_WR (1ull << 4) 673 #define EPT_VLT_PERM_EX (1ull << 5) 674 #define EPT_VLT_PERM_USER_EX (1ull << 6) 675 #define EPT_VLT_PERMS (EPT_VLT_PERM_RD | EPT_VLT_PERM_WR | \ 676 EPT_VLT_PERM_EX) 677 #define EPT_VLT_LADDR_VLD (1ull << 7) 678 #define EPT_VLT_PADDR (1ull << 8) 679 #define EPT_VLT_GUEST_USER (1ull << 9) 680 #define EPT_VLT_GUEST_RW (1ull << 10) 681 #define EPT_VLT_GUEST_EX (1ull << 11) 682 #define EPT_VLT_GUEST_MASK (EPT_VLT_GUEST_USER | EPT_VLT_GUEST_RW | \ 683 EPT_VLT_GUEST_EX) 684 685 #define MAGIC_VAL_1 0x12345678ul 686 #define MAGIC_VAL_2 0x87654321ul 687 #define MAGIC_VAL_3 0xfffffffful 688 #define MAGIC_VAL_4 0xdeadbeeful 689 690 #define INVEPT_SINGLE 1 691 #define INVEPT_GLOBAL 2 692 693 #define INVVPID_ADDR 0 694 #define INVVPID_CONTEXT_GLOBAL 1 695 #define INVVPID_ALL 2 696 #define INVVPID_CONTEXT_LOCAL 3 697 698 #define ACTV_ACTIVE 0 699 #define ACTV_HLT 1 700 701 /* 702 * VMCS field encoding: 703 * Bit 0: High-access 704 * Bits 1-9: Index 705 * Bits 10-12: Type 706 * Bits 13-15: Width 707 * Bits 15-64: Reserved 708 */ 709 #define VMCS_FIELD_HIGH_SHIFT (0) 710 #define VMCS_FIELD_INDEX_SHIFT (1) 711 #define VMCS_FIELD_INDEX_MASK GENMASK(9, 1) 712 #define VMCS_FIELD_TYPE_SHIFT (10) 713 #define VMCS_FIELD_WIDTH_SHIFT (13) 714 #define VMCS_FIELD_RESERVED_SHIFT (15) 715 #define VMCS_FIELD_BIT_SIZE (BITS_PER_LONG) 716 717 extern struct regs regs; 718 719 extern union vmx_basic basic; 720 extern union vmx_ctrl_msr ctrl_pin_rev; 721 extern union vmx_ctrl_msr ctrl_cpu_rev[2]; 722 extern union vmx_ctrl_msr ctrl_exit_rev; 723 extern union vmx_ctrl_msr ctrl_enter_rev; 724 extern union vmx_ept_vpid ept_vpid; 725 726 extern u64 *bsp_vmxon_region; 727 extern bool launched; 728 729 void vmx_set_test_stage(u32 s); 730 u32 vmx_get_test_stage(void); 731 void vmx_inc_test_stage(void); 732 733 static int _vmx_on(u64 *vmxon_region) 734 { 735 bool ret; 736 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 737 asm volatile ("push %1; popf; vmxon %2; setbe %0\n\t" 738 : "=q" (ret) : "q" (rflags), "m" (vmxon_region) : "cc"); 739 return ret; 740 } 741 742 static int vmx_on(void) 743 { 744 return _vmx_on(bsp_vmxon_region); 745 } 746 747 static int vmx_off(void) 748 { 749 bool ret; 750 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 751 752 asm volatile("push %1; popf; vmxoff; setbe %0\n\t" 753 : "=q"(ret) : "q" (rflags) : "cc"); 754 return ret; 755 } 756 757 static inline int make_vmcs_current(struct vmcs *vmcs) 758 { 759 bool ret; 760 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 761 762 asm volatile ("push %1; popf; vmptrld %2; setbe %0" 763 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 764 return ret; 765 } 766 767 static inline int vmcs_clear(struct vmcs *vmcs) 768 { 769 bool ret; 770 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 771 772 asm volatile ("push %1; popf; vmclear %2; setbe %0" 773 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 774 return ret; 775 } 776 777 static inline u64 vmcs_read(enum Encoding enc) 778 { 779 u64 val; 780 asm volatile ("vmread %1, %0" : "=rm" (val) : "r" ((u64)enc) : "cc"); 781 return val; 782 } 783 784 static inline int vmcs_read_checking(enum Encoding enc, u64 *value) 785 { 786 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 787 u64 encoding = enc; 788 u64 val; 789 790 asm volatile ("shl $8, %%rax;" 791 "sahf;" 792 "vmread %[encoding], %[val];" 793 "lahf;" 794 "shr $8, %%rax" 795 : /* output */ [val]"=rm"(val), "+a"(rflags) 796 : /* input */ [encoding]"r"(encoding) 797 : /* clobber */ "cc"); 798 799 *value = val; 800 return rflags & (X86_EFLAGS_CF | X86_EFLAGS_ZF); 801 } 802 803 static inline int vmcs_write(enum Encoding enc, u64 val) 804 { 805 bool ret; 806 asm volatile ("vmwrite %1, %2; setbe %0" 807 : "=q"(ret) : "rm" (val), "r" ((u64)enc) : "cc"); 808 return ret; 809 } 810 811 static inline int vmcs_set_bits(enum Encoding enc, u64 val) 812 { 813 return vmcs_write(enc, vmcs_read(enc) | val); 814 } 815 816 static inline int vmcs_clear_bits(enum Encoding enc, u64 val) 817 { 818 return vmcs_write(enc, vmcs_read(enc) & ~val); 819 } 820 821 static inline int vmcs_save(struct vmcs **vmcs) 822 { 823 bool ret; 824 unsigned long pa; 825 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 826 827 asm volatile ("push %2; popf; vmptrst %1; setbe %0" 828 : "=q" (ret), "=m" (pa) : "r" (rflags) : "cc"); 829 *vmcs = (pa == -1ull) ? NULL : phys_to_virt(pa); 830 return ret; 831 } 832 833 static inline bool invept(unsigned long type, u64 eptp) 834 { 835 bool ret; 836 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 837 838 struct { 839 u64 eptp, gpa; 840 } operand = {eptp, 0}; 841 asm volatile("push %1; popf; invept %2, %3; setbe %0" 842 : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 843 return ret; 844 } 845 846 static inline bool invvpid(unsigned long type, u64 vpid, u64 gla) 847 { 848 bool ret; 849 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 850 851 struct invvpid_operand operand = {vpid, gla}; 852 asm volatile("push %1; popf; invvpid %2, %3; setbe %0" 853 : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 854 return ret; 855 } 856 857 void enable_vmx(void); 858 void init_vmx(u64 *vmxon_region); 859 860 const char *exit_reason_description(u64 reason); 861 void print_vmexit_info(union exit_reason exit_reason); 862 void print_vmentry_failure_info(struct vmentry_result *result); 863 void ept_sync(int type, u64 eptp); 864 void vpid_sync(int type, u16 vpid); 865 void install_ept_entry(unsigned long *pml4, int pte_level, 866 unsigned long guest_addr, unsigned long pte, 867 unsigned long *pt_page); 868 void install_1g_ept(unsigned long *pml4, unsigned long phys, 869 unsigned long guest_addr, u64 perm); 870 void install_2m_ept(unsigned long *pml4, unsigned long phys, 871 unsigned long guest_addr, u64 perm); 872 void install_ept(unsigned long *pml4, unsigned long phys, 873 unsigned long guest_addr, u64 perm); 874 void setup_ept_range(unsigned long *pml4, unsigned long start, 875 unsigned long len, int map_1g, int map_2m, u64 perm); 876 bool get_ept_pte(unsigned long *pml4, unsigned long guest_addr, int level, 877 unsigned long *pte); 878 void set_ept_pte(unsigned long *pml4, unsigned long guest_addr, 879 int level, u64 pte_val); 880 void check_ept_ad(unsigned long *pml4, u64 guest_cr3, 881 unsigned long guest_addr, int expected_gpa_ad, 882 int expected_pt_ad); 883 void clear_ept_ad(unsigned long *pml4, u64 guest_cr3, 884 unsigned long guest_addr); 885 886 bool ept_2m_supported(void); 887 bool ept_1g_supported(void); 888 bool ept_huge_pages_supported(int level); 889 bool ept_execute_only_supported(void); 890 bool ept_ad_bits_supported(void); 891 892 #define ABORT_ON_EARLY_VMENTRY_FAIL 0x1 893 #define ABORT_ON_INVALID_GUEST_STATE 0x2 894 895 void __enter_guest(u8 abort_flag, struct vmentry_result *result); 896 void enter_guest(void); 897 void enter_guest_with_bad_controls(void); 898 void hypercall(u32 hypercall_no); 899 900 typedef void (*test_guest_func)(void); 901 typedef void (*test_teardown_func)(void *data); 902 void test_set_guest(test_guest_func func); 903 void test_add_teardown(test_teardown_func func, void *data); 904 void test_skip(const char *msg); 905 906 void __abort_test(void); 907 908 #define TEST_ASSERT(cond) \ 909 do { \ 910 if (!(cond)) { \ 911 report(0, "%s:%d: Assertion failed: %s", \ 912 __FILE__, __LINE__, #cond); \ 913 dump_stack(); \ 914 __abort_test(); \ 915 } \ 916 report_pass(); \ 917 } while (0) 918 919 #define TEST_ASSERT_MSG(cond, fmt, args...) \ 920 do { \ 921 if (!(cond)) { \ 922 report(0, "%s:%d: Assertion failed: %s\n" fmt, \ 923 __FILE__, __LINE__, #cond, ##args); \ 924 dump_stack(); \ 925 __abort_test(); \ 926 } \ 927 report_pass(); \ 928 } while (0) 929 930 #define __TEST_EQ(a, b, a_str, b_str, assertion, fmt, args...) \ 931 do { \ 932 typeof(a) _a = a; \ 933 typeof(b) _b = b; \ 934 if (_a != _b) { \ 935 char _bin_a[BINSTR_SZ]; \ 936 char _bin_b[BINSTR_SZ]; \ 937 binstr(_a, _bin_a); \ 938 binstr(_b, _bin_b); \ 939 report(0, \ 940 "%s:%d: %s failed: (%s) == (%s)\n" \ 941 "\tLHS: %#018lx - %s - %lu\n" \ 942 "\tRHS: %#018lx - %s - %lu%s" fmt, \ 943 __FILE__, __LINE__, \ 944 assertion ? "Assertion" : "Expectation", a_str, b_str, \ 945 (unsigned long) _a, _bin_a, (unsigned long) _a, \ 946 (unsigned long) _b, _bin_b, (unsigned long) _b, \ 947 fmt[0] == '\0' ? "" : "\n", ## args); \ 948 dump_stack(); \ 949 if (assertion) \ 950 __abort_test(); \ 951 } \ 952 report_pass(); \ 953 } while (0) 954 955 #define TEST_ASSERT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 1, "") 956 #define TEST_ASSERT_EQ_MSG(a, b, fmt, args...) \ 957 __TEST_EQ(a, b, #a, #b, 1, fmt, ## args) 958 #define TEST_EXPECT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 0, "") 959 #define TEST_EXPECT_EQ_MSG(a, b, fmt, args...) \ 960 __TEST_EQ(a, b, #a, #b, 0, fmt, ## args) 961 962 #endif 963