1 #ifndef __VMX_H 2 #define __VMX_H 3 4 #include "libcflat.h" 5 #include "processor.h" 6 #include "bitops.h" 7 #include "asm/page.h" 8 #include "asm/io.h" 9 10 struct vmcs_hdr { 11 u32 revision_id:31; 12 u32 shadow_vmcs:1; 13 }; 14 15 struct vmcs { 16 struct vmcs_hdr hdr; 17 u32 abort; /* VMX-abort indicator */ 18 /* VMCS data */ 19 char data[0]; 20 }; 21 22 struct invvpid_operand { 23 u64 vpid; 24 u64 gla; 25 }; 26 27 struct regs { 28 u64 rax; 29 u64 rcx; 30 u64 rdx; 31 u64 rbx; 32 u64 cr2; 33 u64 rbp; 34 u64 rsi; 35 u64 rdi; 36 u64 r8; 37 u64 r9; 38 u64 r10; 39 u64 r11; 40 u64 r12; 41 u64 r13; 42 u64 r14; 43 u64 r15; 44 u64 rflags; 45 }; 46 47 struct vmentry_failure { 48 /* Did a vmlaunch or vmresume fail? */ 49 bool vmlaunch; 50 /* Instruction mnemonic (for convenience). */ 51 const char *instr; 52 /* Did the instruction return right away, or did we jump to HOST_RIP? */ 53 bool early; 54 /* Contents of [re]flags after failed entry. */ 55 unsigned long flags; 56 }; 57 58 struct vmx_test { 59 const char *name; 60 int (*init)(struct vmcs *vmcs); 61 void (*guest_main)(void); 62 int (*exit_handler)(void); 63 void (*syscall_handler)(u64 syscall_no); 64 struct regs guest_regs; 65 int (*entry_failure_handler)(struct vmentry_failure *failure); 66 struct vmcs *vmcs; 67 int exits; 68 /* Alternative test interface. */ 69 void (*v2)(void); 70 }; 71 72 union vmx_basic { 73 u64 val; 74 struct { 75 u32 revision; 76 u32 size:13, 77 reserved1: 3, 78 width:1, 79 dual:1, 80 type:4, 81 insouts:1, 82 ctrl:1, 83 reserved2:8; 84 }; 85 }; 86 87 union vmx_ctrl_msr { 88 u64 val; 89 struct { 90 u32 set, clr; 91 }; 92 }; 93 94 union vmx_ept_vpid { 95 u64 val; 96 struct { 97 u32:16, 98 super:2, 99 : 2, 100 invept:1, 101 : 11; 102 u32 invvpid:1; 103 }; 104 }; 105 106 enum Encoding { 107 /* 16-Bit Control Fields */ 108 VPID = 0x0000ul, 109 /* Posted-interrupt notification vector */ 110 PINV = 0x0002ul, 111 /* EPTP index */ 112 EPTP_IDX = 0x0004ul, 113 114 /* 16-Bit Guest State Fields */ 115 GUEST_SEL_ES = 0x0800ul, 116 GUEST_SEL_CS = 0x0802ul, 117 GUEST_SEL_SS = 0x0804ul, 118 GUEST_SEL_DS = 0x0806ul, 119 GUEST_SEL_FS = 0x0808ul, 120 GUEST_SEL_GS = 0x080aul, 121 GUEST_SEL_LDTR = 0x080cul, 122 GUEST_SEL_TR = 0x080eul, 123 GUEST_INT_STATUS = 0x0810ul, 124 GUEST_PML_INDEX = 0x0812ul, 125 126 /* 16-Bit Host State Fields */ 127 HOST_SEL_ES = 0x0c00ul, 128 HOST_SEL_CS = 0x0c02ul, 129 HOST_SEL_SS = 0x0c04ul, 130 HOST_SEL_DS = 0x0c06ul, 131 HOST_SEL_FS = 0x0c08ul, 132 HOST_SEL_GS = 0x0c0aul, 133 HOST_SEL_TR = 0x0c0cul, 134 135 /* 64-Bit Control Fields */ 136 IO_BITMAP_A = 0x2000ul, 137 IO_BITMAP_B = 0x2002ul, 138 MSR_BITMAP = 0x2004ul, 139 EXIT_MSR_ST_ADDR = 0x2006ul, 140 EXIT_MSR_LD_ADDR = 0x2008ul, 141 ENTER_MSR_LD_ADDR = 0x200aul, 142 VMCS_EXEC_PTR = 0x200cul, 143 TSC_OFFSET = 0x2010ul, 144 TSC_OFFSET_HI = 0x2011ul, 145 APIC_VIRT_ADDR = 0x2012ul, 146 APIC_ACCS_ADDR = 0x2014ul, 147 POSTED_INTR_DESC_ADDR = 0x2016ul, 148 EPTP = 0x201aul, 149 EPTP_HI = 0x201bul, 150 VMREAD_BITMAP = 0x2026ul, 151 VMREAD_BITMAP_HI = 0x2027ul, 152 VMWRITE_BITMAP = 0x2028ul, 153 VMWRITE_BITMAP_HI = 0x2029ul, 154 EOI_EXIT_BITMAP0 = 0x201cul, 155 EOI_EXIT_BITMAP1 = 0x201eul, 156 EOI_EXIT_BITMAP2 = 0x2020ul, 157 EOI_EXIT_BITMAP3 = 0x2022ul, 158 PMLADDR = 0x200eul, 159 PMLADDR_HI = 0x200ful, 160 161 162 /* 64-Bit Readonly Data Field */ 163 INFO_PHYS_ADDR = 0x2400ul, 164 165 /* 64-Bit Guest State */ 166 VMCS_LINK_PTR = 0x2800ul, 167 VMCS_LINK_PTR_HI = 0x2801ul, 168 GUEST_DEBUGCTL = 0x2802ul, 169 GUEST_DEBUGCTL_HI = 0x2803ul, 170 GUEST_EFER = 0x2806ul, 171 GUEST_PAT = 0x2804ul, 172 GUEST_PERF_GLOBAL_CTRL = 0x2808ul, 173 GUEST_PDPTE = 0x280aul, 174 175 /* 64-Bit Host State */ 176 HOST_PAT = 0x2c00ul, 177 HOST_EFER = 0x2c02ul, 178 HOST_PERF_GLOBAL_CTRL = 0x2c04ul, 179 180 /* 32-Bit Control Fields */ 181 PIN_CONTROLS = 0x4000ul, 182 CPU_EXEC_CTRL0 = 0x4002ul, 183 EXC_BITMAP = 0x4004ul, 184 PF_ERROR_MASK = 0x4006ul, 185 PF_ERROR_MATCH = 0x4008ul, 186 CR3_TARGET_COUNT = 0x400aul, 187 EXI_CONTROLS = 0x400cul, 188 EXI_MSR_ST_CNT = 0x400eul, 189 EXI_MSR_LD_CNT = 0x4010ul, 190 ENT_CONTROLS = 0x4012ul, 191 ENT_MSR_LD_CNT = 0x4014ul, 192 ENT_INTR_INFO = 0x4016ul, 193 ENT_INTR_ERROR = 0x4018ul, 194 ENT_INST_LEN = 0x401aul, 195 TPR_THRESHOLD = 0x401cul, 196 CPU_EXEC_CTRL1 = 0x401eul, 197 198 /* 32-Bit R/O Data Fields */ 199 VMX_INST_ERROR = 0x4400ul, 200 EXI_REASON = 0x4402ul, 201 EXI_INTR_INFO = 0x4404ul, 202 EXI_INTR_ERROR = 0x4406ul, 203 IDT_VECT_INFO = 0x4408ul, 204 IDT_VECT_ERROR = 0x440aul, 205 EXI_INST_LEN = 0x440cul, 206 EXI_INST_INFO = 0x440eul, 207 208 /* 32-Bit Guest State Fields */ 209 GUEST_LIMIT_ES = 0x4800ul, 210 GUEST_LIMIT_CS = 0x4802ul, 211 GUEST_LIMIT_SS = 0x4804ul, 212 GUEST_LIMIT_DS = 0x4806ul, 213 GUEST_LIMIT_FS = 0x4808ul, 214 GUEST_LIMIT_GS = 0x480aul, 215 GUEST_LIMIT_LDTR = 0x480cul, 216 GUEST_LIMIT_TR = 0x480eul, 217 GUEST_LIMIT_GDTR = 0x4810ul, 218 GUEST_LIMIT_IDTR = 0x4812ul, 219 GUEST_AR_ES = 0x4814ul, 220 GUEST_AR_CS = 0x4816ul, 221 GUEST_AR_SS = 0x4818ul, 222 GUEST_AR_DS = 0x481aul, 223 GUEST_AR_FS = 0x481cul, 224 GUEST_AR_GS = 0x481eul, 225 GUEST_AR_LDTR = 0x4820ul, 226 GUEST_AR_TR = 0x4822ul, 227 GUEST_INTR_STATE = 0x4824ul, 228 GUEST_ACTV_STATE = 0x4826ul, 229 GUEST_SMBASE = 0x4828ul, 230 GUEST_SYSENTER_CS = 0x482aul, 231 PREEMPT_TIMER_VALUE = 0x482eul, 232 233 /* 32-Bit Host State Fields */ 234 HOST_SYSENTER_CS = 0x4c00ul, 235 236 /* Natural-Width Control Fields */ 237 CR0_MASK = 0x6000ul, 238 CR4_MASK = 0x6002ul, 239 CR0_READ_SHADOW = 0x6004ul, 240 CR4_READ_SHADOW = 0x6006ul, 241 CR3_TARGET_0 = 0x6008ul, 242 CR3_TARGET_1 = 0x600aul, 243 CR3_TARGET_2 = 0x600cul, 244 CR3_TARGET_3 = 0x600eul, 245 246 /* Natural-Width R/O Data Fields */ 247 EXI_QUALIFICATION = 0x6400ul, 248 IO_RCX = 0x6402ul, 249 IO_RSI = 0x6404ul, 250 IO_RDI = 0x6406ul, 251 IO_RIP = 0x6408ul, 252 GUEST_LINEAR_ADDRESS = 0x640aul, 253 254 /* Natural-Width Guest State Fields */ 255 GUEST_CR0 = 0x6800ul, 256 GUEST_CR3 = 0x6802ul, 257 GUEST_CR4 = 0x6804ul, 258 GUEST_BASE_ES = 0x6806ul, 259 GUEST_BASE_CS = 0x6808ul, 260 GUEST_BASE_SS = 0x680aul, 261 GUEST_BASE_DS = 0x680cul, 262 GUEST_BASE_FS = 0x680eul, 263 GUEST_BASE_GS = 0x6810ul, 264 GUEST_BASE_LDTR = 0x6812ul, 265 GUEST_BASE_TR = 0x6814ul, 266 GUEST_BASE_GDTR = 0x6816ul, 267 GUEST_BASE_IDTR = 0x6818ul, 268 GUEST_DR7 = 0x681aul, 269 GUEST_RSP = 0x681cul, 270 GUEST_RIP = 0x681eul, 271 GUEST_RFLAGS = 0x6820ul, 272 GUEST_PENDING_DEBUG = 0x6822ul, 273 GUEST_SYSENTER_ESP = 0x6824ul, 274 GUEST_SYSENTER_EIP = 0x6826ul, 275 276 /* Natural-Width Host State Fields */ 277 HOST_CR0 = 0x6c00ul, 278 HOST_CR3 = 0x6c02ul, 279 HOST_CR4 = 0x6c04ul, 280 HOST_BASE_FS = 0x6c06ul, 281 HOST_BASE_GS = 0x6c08ul, 282 HOST_BASE_TR = 0x6c0aul, 283 HOST_BASE_GDTR = 0x6c0cul, 284 HOST_BASE_IDTR = 0x6c0eul, 285 HOST_SYSENTER_ESP = 0x6c10ul, 286 HOST_SYSENTER_EIP = 0x6c12ul, 287 HOST_RSP = 0x6c14ul, 288 HOST_RIP = 0x6c16ul 289 }; 290 291 #define VMX_ENTRY_FAILURE (1ul << 31) 292 #define VMX_ENTRY_FLAGS (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 293 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 294 295 enum Reason { 296 VMX_EXC_NMI = 0, 297 VMX_EXTINT = 1, 298 VMX_TRIPLE_FAULT = 2, 299 VMX_INIT = 3, 300 VMX_SIPI = 4, 301 VMX_SMI_IO = 5, 302 VMX_SMI_OTHER = 6, 303 VMX_INTR_WINDOW = 7, 304 VMX_NMI_WINDOW = 8, 305 VMX_TASK_SWITCH = 9, 306 VMX_CPUID = 10, 307 VMX_GETSEC = 11, 308 VMX_HLT = 12, 309 VMX_INVD = 13, 310 VMX_INVLPG = 14, 311 VMX_RDPMC = 15, 312 VMX_RDTSC = 16, 313 VMX_RSM = 17, 314 VMX_VMCALL = 18, 315 VMX_VMCLEAR = 19, 316 VMX_VMLAUNCH = 20, 317 VMX_VMPTRLD = 21, 318 VMX_VMPTRST = 22, 319 VMX_VMREAD = 23, 320 VMX_VMRESUME = 24, 321 VMX_VMWRITE = 25, 322 VMX_VMXOFF = 26, 323 VMX_VMXON = 27, 324 VMX_CR = 28, 325 VMX_DR = 29, 326 VMX_IO = 30, 327 VMX_RDMSR = 31, 328 VMX_WRMSR = 32, 329 VMX_FAIL_STATE = 33, 330 VMX_FAIL_MSR = 34, 331 VMX_MWAIT = 36, 332 VMX_MTF = 37, 333 VMX_MONITOR = 39, 334 VMX_PAUSE = 40, 335 VMX_FAIL_MCHECK = 41, 336 VMX_TPR_THRESHOLD = 43, 337 VMX_APIC_ACCESS = 44, 338 VMX_EOI_INDUCED = 45, 339 VMX_GDTR_IDTR = 46, 340 VMX_LDTR_TR = 47, 341 VMX_EPT_VIOLATION = 48, 342 VMX_EPT_MISCONFIG = 49, 343 VMX_INVEPT = 50, 344 VMX_PREEMPT = 52, 345 VMX_INVVPID = 53, 346 VMX_WBINVD = 54, 347 VMX_XSETBV = 55, 348 VMX_APIC_WRITE = 56, 349 VMX_RDRAND = 57, 350 VMX_INVPCID = 58, 351 VMX_VMFUNC = 59, 352 VMX_RDSEED = 61, 353 VMX_PML_FULL = 62, 354 VMX_XSAVES = 63, 355 VMX_XRSTORS = 64, 356 }; 357 358 enum Ctrl_exi { 359 EXI_SAVE_DBGCTLS = 1UL << 2, 360 EXI_HOST_64 = 1UL << 9, 361 EXI_LOAD_PERF = 1UL << 12, 362 EXI_INTA = 1UL << 15, 363 EXI_SAVE_PAT = 1UL << 18, 364 EXI_LOAD_PAT = 1UL << 19, 365 EXI_SAVE_EFER = 1UL << 20, 366 EXI_LOAD_EFER = 1UL << 21, 367 EXI_SAVE_PREEMPT = 1UL << 22, 368 }; 369 370 enum Ctrl_ent { 371 ENT_LOAD_DBGCTLS = 1UL << 2, 372 ENT_GUEST_64 = 1UL << 9, 373 ENT_LOAD_PAT = 1UL << 14, 374 ENT_LOAD_EFER = 1UL << 15, 375 }; 376 377 enum Ctrl_pin { 378 PIN_EXTINT = 1ul << 0, 379 PIN_NMI = 1ul << 3, 380 PIN_VIRT_NMI = 1ul << 5, 381 PIN_PREEMPT = 1ul << 6, 382 PIN_POST_INTR = 1ul << 7, 383 }; 384 385 enum Ctrl0 { 386 CPU_INTR_WINDOW = 1ul << 2, 387 CPU_USE_TSC_OFFSET = 1ul << 3, 388 CPU_HLT = 1ul << 7, 389 CPU_INVLPG = 1ul << 9, 390 CPU_MWAIT = 1ul << 10, 391 CPU_RDPMC = 1ul << 11, 392 CPU_RDTSC = 1ul << 12, 393 CPU_CR3_LOAD = 1ul << 15, 394 CPU_CR3_STORE = 1ul << 16, 395 CPU_CR8_LOAD = 1ul << 19, 396 CPU_CR8_STORE = 1ul << 20, 397 CPU_TPR_SHADOW = 1ul << 21, 398 CPU_NMI_WINDOW = 1ul << 22, 399 CPU_IO = 1ul << 24, 400 CPU_IO_BITMAP = 1ul << 25, 401 CPU_MSR_BITMAP = 1ul << 28, 402 CPU_MONITOR = 1ul << 29, 403 CPU_PAUSE = 1ul << 30, 404 CPU_SECONDARY = 1ul << 31, 405 }; 406 407 enum Ctrl1 { 408 CPU_VIRT_APIC_ACCESSES = 1ul << 0, 409 CPU_EPT = 1ul << 1, 410 CPU_DESC_TABLE = 1ul << 2, 411 CPU_RDTSCP = 1ul << 3, 412 CPU_VIRT_X2APIC = 1ul << 4, 413 CPU_VPID = 1ul << 5, 414 CPU_WBINVD = 1ul << 6, 415 CPU_URG = 1ul << 7, 416 CPU_APIC_REG_VIRT = 1ul << 8, 417 CPU_VINTD = 1ul << 9, 418 CPU_RDRAND = 1ul << 11, 419 CPU_SHADOW_VMCS = 1ul << 14, 420 CPU_RDSEED = 1ul << 16, 421 CPU_PML = 1ul << 17, 422 }; 423 424 enum Intr_type { 425 VMX_INTR_TYPE_EXT_INTR = 0, 426 VMX_INTR_TYPE_NMI_INTR = 2, 427 VMX_INTR_TYPE_HARD_EXCEPTION = 3, 428 VMX_INTR_TYPE_SOFT_INTR = 4, 429 VMX_INTR_TYPE_SOFT_EXCEPTION = 6, 430 }; 431 432 /* 433 * Interruption-information format 434 */ 435 #define INTR_INFO_VECTOR_MASK 0xff /* 7:0 */ 436 #define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */ 437 #define INTR_INFO_DELIVER_CODE_MASK 0x800 /* 11 */ 438 #define INTR_INFO_UNBLOCK_NMI_MASK 0x1000 /* 12 */ 439 #define INTR_INFO_VALID_MASK 0x80000000 /* 31 */ 440 441 #define INTR_INFO_INTR_TYPE_SHIFT 8 442 443 #define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ 444 #define INTR_TYPE_RESERVED (1 << 8) /* reserved */ 445 #define INTR_TYPE_NMI_INTR (2 << 8) /* NMI */ 446 #define INTR_TYPE_HARD_EXCEPTION (3 << 8) /* processor exception */ 447 #define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ 448 #define INTR_TYPE_PRIV_SW_EXCEPTION (5 << 8) /* priv. software exception */ 449 #define INTR_TYPE_SOFT_EXCEPTION (6 << 8) /* software exception */ 450 #define INTR_TYPE_OTHER_EVENT (7 << 8) /* other event */ 451 452 /* 453 * Guest interruptibility state 454 */ 455 #define GUEST_INTR_STATE_STI (1 << 0) 456 #define GUEST_INTR_STATE_MOVSS (1 << 1) 457 #define GUEST_INTR_STATE_SMI (1 << 2) 458 #define GUEST_INTR_STATE_NMI (1 << 3) 459 #define GUEST_INTR_STATE_ENCLAVE (1 << 4) 460 461 /* 462 * VM-instruction error numbers 463 */ 464 enum vm_instruction_error_number { 465 VMXERR_VMCALL_IN_VMX_ROOT_OPERATION = 1, 466 VMXERR_VMCLEAR_INVALID_ADDRESS = 2, 467 VMXERR_VMCLEAR_VMXON_POINTER = 3, 468 VMXERR_VMLAUNCH_NONCLEAR_VMCS = 4, 469 VMXERR_VMRESUME_NONLAUNCHED_VMCS = 5, 470 VMXERR_VMRESUME_AFTER_VMXOFF = 6, 471 VMXERR_ENTRY_INVALID_CONTROL_FIELD = 7, 472 VMXERR_ENTRY_INVALID_HOST_STATE_FIELD = 8, 473 VMXERR_VMPTRLD_INVALID_ADDRESS = 9, 474 VMXERR_VMPTRLD_VMXON_POINTER = 10, 475 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID = 11, 476 VMXERR_UNSUPPORTED_VMCS_COMPONENT = 12, 477 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT = 13, 478 VMXERR_VMXON_IN_VMX_ROOT_OPERATION = 15, 479 VMXERR_ENTRY_INVALID_EXECUTIVE_VMCS_POINTER = 16, 480 VMXERR_ENTRY_NONLAUNCHED_EXECUTIVE_VMCS = 17, 481 VMXERR_ENTRY_EXECUTIVE_VMCS_POINTER_NOT_VMXON_POINTER = 18, 482 VMXERR_VMCALL_NONCLEAR_VMCS = 19, 483 VMXERR_VMCALL_INVALID_VM_EXIT_CONTROL_FIELDS = 20, 484 VMXERR_VMCALL_INCORRECT_MSEG_REVISION_ID = 22, 485 VMXERR_VMXOFF_UNDER_DUAL_MONITOR_TREATMENT_OF_SMIS_AND_SMM = 23, 486 VMXERR_VMCALL_INVALID_SMM_MONITOR_FEATURES = 24, 487 VMXERR_ENTRY_INVALID_VM_EXECUTION_CONTROL_FIELDS_IN_EXECUTIVE_VMCS = 25, 488 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS = 26, 489 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28, 490 }; 491 492 #define SAVE_GPR \ 493 "xchg %rax, regs\n\t" \ 494 "xchg %rbx, regs+0x8\n\t" \ 495 "xchg %rcx, regs+0x10\n\t" \ 496 "xchg %rdx, regs+0x18\n\t" \ 497 "xchg %rbp, regs+0x28\n\t" \ 498 "xchg %rsi, regs+0x30\n\t" \ 499 "xchg %rdi, regs+0x38\n\t" \ 500 "xchg %r8, regs+0x40\n\t" \ 501 "xchg %r9, regs+0x48\n\t" \ 502 "xchg %r10, regs+0x50\n\t" \ 503 "xchg %r11, regs+0x58\n\t" \ 504 "xchg %r12, regs+0x60\n\t" \ 505 "xchg %r13, regs+0x68\n\t" \ 506 "xchg %r14, regs+0x70\n\t" \ 507 "xchg %r15, regs+0x78\n\t" 508 509 #define LOAD_GPR SAVE_GPR 510 511 #define SAVE_GPR_C \ 512 "xchg %%rax, regs\n\t" \ 513 "xchg %%rbx, regs+0x8\n\t" \ 514 "xchg %%rcx, regs+0x10\n\t" \ 515 "xchg %%rdx, regs+0x18\n\t" \ 516 "xchg %%rbp, regs+0x28\n\t" \ 517 "xchg %%rsi, regs+0x30\n\t" \ 518 "xchg %%rdi, regs+0x38\n\t" \ 519 "xchg %%r8, regs+0x40\n\t" \ 520 "xchg %%r9, regs+0x48\n\t" \ 521 "xchg %%r10, regs+0x50\n\t" \ 522 "xchg %%r11, regs+0x58\n\t" \ 523 "xchg %%r12, regs+0x60\n\t" \ 524 "xchg %%r13, regs+0x68\n\t" \ 525 "xchg %%r14, regs+0x70\n\t" \ 526 "xchg %%r15, regs+0x78\n\t" 527 528 #define LOAD_GPR_C SAVE_GPR_C 529 530 #define VMX_IO_SIZE_MASK 0x7 531 #define _VMX_IO_BYTE 0 532 #define _VMX_IO_WORD 1 533 #define _VMX_IO_LONG 3 534 #define VMX_IO_DIRECTION_MASK (1ul << 3) 535 #define VMX_IO_IN (1ul << 3) 536 #define VMX_IO_OUT 0 537 #define VMX_IO_STRING (1ul << 4) 538 #define VMX_IO_REP (1ul << 5) 539 #define VMX_IO_OPRAND_IMM (1ul << 6) 540 #define VMX_IO_PORT_MASK 0xFFFF0000 541 #define VMX_IO_PORT_SHIFT 16 542 543 #define VMX_TEST_START 0 544 #define VMX_TEST_VMEXIT 1 545 #define VMX_TEST_EXIT 2 546 #define VMX_TEST_RESUME 3 547 #define VMX_TEST_VMABORT 4 548 #define VMX_TEST_VMSKIP 5 549 550 #define HYPERCALL_BIT (1ul << 12) 551 #define HYPERCALL_MASK 0xFFF 552 #define HYPERCALL_VMEXIT 0x1 553 #define HYPERCALL_VMABORT 0x2 554 #define HYPERCALL_VMSKIP 0x3 555 556 #define EPTP_PG_WALK_LEN_SHIFT 3ul 557 #define EPTP_PG_WALK_LEN_MASK 0x38ul 558 #define EPTP_RESERV_BITS_MASK 0x1ful 559 #define EPTP_RESERV_BITS_SHIFT 0x7ul 560 #define EPTP_AD_FLAG (1ul << 6) 561 562 #define EPT_MEM_TYPE_UC 0ul 563 #define EPT_MEM_TYPE_WC 1ul 564 #define EPT_MEM_TYPE_WT 4ul 565 #define EPT_MEM_TYPE_WP 5ul 566 #define EPT_MEM_TYPE_WB 6ul 567 568 #define EPT_RA 1ul 569 #define EPT_WA 2ul 570 #define EPT_EA 4ul 571 #define EPT_PRESENT (EPT_RA | EPT_WA | EPT_EA) 572 #define EPT_ACCESS_FLAG (1ul << 8) 573 #define EPT_DIRTY_FLAG (1ul << 9) 574 #define EPT_LARGE_PAGE (1ul << 7) 575 #define EPT_MEM_TYPE_SHIFT 3ul 576 #define EPT_MEM_TYPE_MASK 0x7ul 577 #define EPT_IGNORE_PAT (1ul << 6) 578 #define EPT_SUPPRESS_VE (1ull << 63) 579 580 #define EPT_CAP_WT 1ull 581 #define EPT_CAP_PWL4 (1ull << 6) 582 #define EPT_CAP_UC (1ull << 8) 583 #define EPT_CAP_WB (1ull << 14) 584 #define EPT_CAP_2M_PAGE (1ull << 16) 585 #define EPT_CAP_1G_PAGE (1ull << 17) 586 #define EPT_CAP_INVEPT (1ull << 20) 587 #define EPT_CAP_INVEPT_SINGLE (1ull << 25) 588 #define EPT_CAP_INVEPT_ALL (1ull << 26) 589 #define EPT_CAP_AD_FLAG (1ull << 21) 590 #define VPID_CAP_INVVPID (1ull << 32) 591 #define VPID_CAP_INVVPID_ADDR (1ull << 40) 592 #define VPID_CAP_INVVPID_CXTGLB (1ull << 41) 593 #define VPID_CAP_INVVPID_ALL (1ull << 42) 594 #define VPID_CAP_INVVPID_CXTLOC (1ull << 43) 595 596 #define PAGE_SIZE_2M (512 * PAGE_SIZE) 597 #define PAGE_SIZE_1G (512 * PAGE_SIZE_2M) 598 #define EPT_PAGE_LEVEL 4 599 #define EPT_PGDIR_WIDTH 9 600 #define EPT_PGDIR_MASK 511 601 #define EPT_PGDIR_ENTRIES (1 << EPT_PGDIR_WIDTH) 602 #define EPT_LEVEL_SHIFT(level) (((level)-1) * EPT_PGDIR_WIDTH + 12) 603 #define EPT_ADDR_MASK GENMASK_ULL(51, 12) 604 #define PAGE_MASK_2M (~(PAGE_SIZE_2M-1)) 605 606 #define EPT_VLT_RD 1 607 #define EPT_VLT_WR (1 << 1) 608 #define EPT_VLT_FETCH (1 << 2) 609 #define EPT_VLT_PERM_RD (1 << 3) 610 #define EPT_VLT_PERM_WR (1 << 4) 611 #define EPT_VLT_PERM_EX (1 << 5) 612 #define EPT_VLT_PERMS (EPT_VLT_PERM_RD | EPT_VLT_PERM_WR | \ 613 EPT_VLT_PERM_EX) 614 #define EPT_VLT_LADDR_VLD (1 << 7) 615 #define EPT_VLT_PADDR (1 << 8) 616 617 #define MAGIC_VAL_1 0x12345678ul 618 #define MAGIC_VAL_2 0x87654321ul 619 #define MAGIC_VAL_3 0xfffffffful 620 #define MAGIC_VAL_4 0xdeadbeeful 621 622 #define INVEPT_SINGLE 1 623 #define INVEPT_GLOBAL 2 624 625 #define INVVPID_ADDR 0 626 #define INVVPID_CONTEXT_GLOBAL 1 627 #define INVVPID_ALL 2 628 #define INVVPID_CONTEXT_LOCAL 3 629 630 #define ACTV_ACTIVE 0 631 #define ACTV_HLT 1 632 633 /* 634 * VMCS field encoding: 635 * Bit 0: High-access 636 * Bits 1-9: Index 637 * Bits 10-12: Type 638 * Bits 13-15: Width 639 * Bits 15-64: Reserved 640 */ 641 #define VMCS_FIELD_HIGH_SHIFT (0) 642 #define VMCS_FIELD_INDEX_SHIFT (1) 643 #define VMCS_FIELD_TYPE_SHIFT (10) 644 #define VMCS_FIELD_WIDTH_SHIFT (13) 645 #define VMCS_FIELD_RESERVED_SHIFT (15) 646 #define VMCS_FIELD_BIT_SIZE (BITS_PER_LONG) 647 648 extern struct regs regs; 649 650 extern union vmx_basic basic; 651 extern union vmx_ctrl_msr ctrl_pin_rev; 652 extern union vmx_ctrl_msr ctrl_cpu_rev[2]; 653 extern union vmx_ctrl_msr ctrl_exit_rev; 654 extern union vmx_ctrl_msr ctrl_enter_rev; 655 extern union vmx_ept_vpid ept_vpid; 656 657 extern u64 *vmxon_region; 658 659 void vmx_set_test_stage(u32 s); 660 u32 vmx_get_test_stage(void); 661 void vmx_inc_test_stage(void); 662 663 static int vmx_on(void) 664 { 665 bool ret; 666 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 667 asm volatile ("push %1; popf; vmxon %2; setbe %0\n\t" 668 : "=q" (ret) : "q" (rflags), "m" (vmxon_region) : "cc"); 669 return ret; 670 } 671 672 static int vmx_off(void) 673 { 674 bool ret; 675 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 676 677 asm volatile("push %1; popf; vmxoff; setbe %0\n\t" 678 : "=q"(ret) : "q" (rflags) : "cc"); 679 return ret; 680 } 681 682 static inline int make_vmcs_current(struct vmcs *vmcs) 683 { 684 bool ret; 685 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 686 687 asm volatile ("push %1; popf; vmptrld %2; setbe %0" 688 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 689 return ret; 690 } 691 692 static inline int vmcs_clear(struct vmcs *vmcs) 693 { 694 bool ret; 695 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 696 697 asm volatile ("push %1; popf; vmclear %2; setbe %0" 698 : "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc"); 699 return ret; 700 } 701 702 static inline u64 vmcs_read(enum Encoding enc) 703 { 704 u64 val; 705 asm volatile ("vmread %1, %0" : "=rm" (val) : "r" ((u64)enc) : "cc"); 706 return val; 707 } 708 709 static inline int vmcs_read_checking(enum Encoding enc, u64 *value) 710 { 711 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 712 u64 encoding = enc; 713 u64 val; 714 715 asm volatile ("shl $8, %%rax;" 716 "sahf;" 717 "vmread %[encoding], %[val];" 718 "lahf;" 719 "shr $8, %%rax" 720 : /* output */ [val]"=rm"(val), "+a"(rflags) 721 : /* input */ [encoding]"r"(encoding) 722 : /* clobber */ "cc"); 723 724 *value = val; 725 return rflags & (X86_EFLAGS_CF | X86_EFLAGS_ZF); 726 } 727 728 static inline int vmcs_write(enum Encoding enc, u64 val) 729 { 730 bool ret; 731 asm volatile ("vmwrite %1, %2; setbe %0" 732 : "=q"(ret) : "rm" (val), "r" ((u64)enc) : "cc"); 733 return ret; 734 } 735 736 static inline int vmcs_set_bits(enum Encoding enc, u64 val) 737 { 738 return vmcs_write(enc, vmcs_read(enc) | val); 739 } 740 741 static inline int vmcs_clear_bits(enum Encoding enc, u64 val) 742 { 743 return vmcs_write(enc, vmcs_read(enc) & ~val); 744 } 745 746 static inline int vmcs_save(struct vmcs **vmcs) 747 { 748 bool ret; 749 unsigned long pa; 750 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 751 752 asm volatile ("push %2; popf; vmptrst %1; setbe %0" 753 : "=q" (ret), "=m" (pa) : "r" (rflags) : "cc"); 754 *vmcs = (pa == -1ull) ? NULL : phys_to_virt(pa); 755 return ret; 756 } 757 758 static inline bool invept(unsigned long type, u64 eptp) 759 { 760 bool ret; 761 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 762 763 struct { 764 u64 eptp, gpa; 765 } operand = {eptp, 0}; 766 asm volatile("push %1; popf; invept %2, %3; setbe %0" 767 : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 768 return ret; 769 } 770 771 static inline bool invvpid(unsigned long type, u64 vpid, u64 gla) 772 { 773 bool ret; 774 u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF; 775 776 struct invvpid_operand operand = {vpid, gla}; 777 asm volatile("push %1; popf; invvpid %2, %3; setbe %0" 778 : "=q" (ret) : "r" (rflags), "m"(operand),"r"(type) : "cc"); 779 return ret; 780 } 781 782 static inline int enable_unrestricted_guest(void) 783 { 784 if (!(ctrl_cpu_rev[0].clr & CPU_SECONDARY)) 785 return -1; 786 787 if (!(ctrl_cpu_rev[1].clr & CPU_URG)) 788 return -1; 789 790 vmcs_write(CPU_EXEC_CTRL0, vmcs_read(CPU_EXEC_CTRL0) | CPU_SECONDARY); 791 vmcs_write(CPU_EXEC_CTRL1, vmcs_read(CPU_EXEC_CTRL1) | CPU_URG); 792 return 0; 793 } 794 795 static inline void disable_unrestricted_guest(void) 796 { 797 vmcs_write(CPU_EXEC_CTRL1, vmcs_read(CPU_EXEC_CTRL1) & ~CPU_URG); 798 } 799 800 const char *exit_reason_description(u64 reason); 801 void print_vmexit_info(void); 802 void print_vmentry_failure_info(struct vmentry_failure *failure); 803 void ept_sync(int type, u64 eptp); 804 void vpid_sync(int type, u16 vpid); 805 void install_ept_entry(unsigned long *pml4, int pte_level, 806 unsigned long guest_addr, unsigned long pte, 807 unsigned long *pt_page); 808 void install_1g_ept(unsigned long *pml4, unsigned long phys, 809 unsigned long guest_addr, u64 perm); 810 void install_2m_ept(unsigned long *pml4, unsigned long phys, 811 unsigned long guest_addr, u64 perm); 812 void install_ept(unsigned long *pml4, unsigned long phys, 813 unsigned long guest_addr, u64 perm); 814 void setup_ept_range(unsigned long *pml4, unsigned long start, 815 unsigned long len, int map_1g, int map_2m, u64 perm); 816 bool get_ept_pte(unsigned long *pml4, unsigned long guest_addr, int level, 817 unsigned long *pte); 818 void set_ept_pte(unsigned long *pml4, unsigned long guest_addr, 819 int level, u64 pte_val); 820 void check_ept_ad(unsigned long *pml4, u64 guest_cr3, 821 unsigned long guest_addr, int expected_gpa_ad, 822 int expected_pt_ad); 823 void clear_ept_ad(unsigned long *pml4, u64 guest_cr3, 824 unsigned long guest_addr); 825 826 bool ept_2m_supported(void); 827 bool ept_1g_supported(void); 828 bool ept_huge_pages_supported(int level); 829 bool ept_execute_only_supported(void); 830 bool ept_ad_bits_supported(void); 831 832 void enter_guest(void); 833 834 typedef void (*test_guest_func)(void); 835 typedef void (*test_teardown_func)(void *data); 836 void test_set_guest(test_guest_func func); 837 void test_add_teardown(test_teardown_func func, void *data); 838 void test_skip(const char *msg); 839 840 void __abort_test(void); 841 842 #define TEST_ASSERT(cond) \ 843 do { \ 844 if (!(cond)) { \ 845 report("%s:%d: Assertion failed: %s", 0, \ 846 __FILE__, __LINE__, #cond); \ 847 dump_stack(); \ 848 __abort_test(); \ 849 } \ 850 report_pass(); \ 851 } while (0) 852 853 #define TEST_ASSERT_MSG(cond, fmt, args...) \ 854 do { \ 855 if (!(cond)) { \ 856 report("%s:%d: Assertion failed: %s\n" fmt, 0, \ 857 __FILE__, __LINE__, #cond, ##args); \ 858 dump_stack(); \ 859 __abort_test(); \ 860 } \ 861 report_pass(); \ 862 } while (0) 863 864 #define __TEST_EQ(a, b, a_str, b_str, assertion, fmt, args...) \ 865 do { \ 866 typeof(a) _a = a; \ 867 typeof(b) _b = b; \ 868 if (_a != _b) { \ 869 char _bin_a[BINSTR_SZ]; \ 870 char _bin_b[BINSTR_SZ]; \ 871 binstr(_a, _bin_a); \ 872 binstr(_b, _bin_b); \ 873 report("%s:%d: %s failed: (%s) == (%s)\n" \ 874 "\tLHS: %#018lx - %s - %lu\n" \ 875 "\tRHS: %#018lx - %s - %lu%s" fmt, 0, \ 876 __FILE__, __LINE__, \ 877 assertion ? "Assertion" : "Expectation", a_str, b_str, \ 878 (unsigned long) _a, _bin_a, (unsigned long) _a, \ 879 (unsigned long) _b, _bin_b, (unsigned long) _b, \ 880 fmt[0] == '\0' ? "" : "\n", ## args); \ 881 dump_stack(); \ 882 if (assertion) \ 883 __abort_test(); \ 884 } \ 885 report_pass(); \ 886 } while (0) 887 888 #define TEST_ASSERT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 1, "") 889 #define TEST_ASSERT_EQ_MSG(a, b, fmt, args...) \ 890 __TEST_EQ(a, b, #a, #b, 1, fmt, ## args) 891 #define TEST_EXPECT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 0, "") 892 #define TEST_EXPECT_EQ_MSG(a, b, fmt, args...) \ 893 __TEST_EQ(a, b, #a, #b, 0, fmt, ## args) 894 895 #endif 896