1 #include "svm.h" 2 #include "libcflat.h" 3 #include "processor.h" 4 #include "desc.h" 5 #include "msr.h" 6 #include "vm.h" 7 #include "smp.h" 8 #include "types.h" 9 #include "alloc_page.h" 10 11 #define SVM_EXIT_MAX_DR_INTERCEPT 0x3f 12 13 /* for the nested page table*/ 14 u64 *pml4e; 15 u64 *pdpe; 16 u64 *pde[4]; 17 u64 *pte[2048]; 18 void *scratch_page; 19 20 #define LATENCY_RUNS 1000000 21 22 u64 tsc_start; 23 u64 tsc_end; 24 25 u64 vmrun_sum, vmexit_sum; 26 u64 vmsave_sum, vmload_sum; 27 u64 stgi_sum, clgi_sum; 28 u64 latvmrun_max; 29 u64 latvmrun_min; 30 u64 latvmexit_max; 31 u64 latvmexit_min; 32 u64 latvmload_max; 33 u64 latvmload_min; 34 u64 latvmsave_max; 35 u64 latvmsave_min; 36 u64 latstgi_max; 37 u64 latstgi_min; 38 u64 latclgi_max; 39 u64 latclgi_min; 40 u64 runs; 41 42 u8 *io_bitmap; 43 u8 io_bitmap_area[16384]; 44 45 #define MSR_BITMAP_SIZE 8192 46 47 u8 *msr_bitmap; 48 u8 msr_bitmap_area[MSR_BITMAP_SIZE + PAGE_SIZE]; 49 50 static bool npt_supported(void) 51 { 52 return this_cpu_has(X86_FEATURE_NPT); 53 } 54 55 static void setup_svm(void) 56 { 57 void *hsave = alloc_page(); 58 u64 *page, address; 59 int i,j; 60 61 wrmsr(MSR_VM_HSAVE_PA, virt_to_phys(hsave)); 62 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_SVME); 63 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX); 64 65 scratch_page = alloc_page(); 66 67 io_bitmap = (void *) (((ulong)io_bitmap_area + 4095) & ~4095); 68 69 msr_bitmap = (void *) ALIGN((ulong)msr_bitmap_area, PAGE_SIZE); 70 71 if (!npt_supported()) 72 return; 73 74 printf("NPT detected - running all tests with NPT enabled\n"); 75 76 /* 77 * Nested paging supported - Build a nested page table 78 * Build the page-table bottom-up and map everything with 4k pages 79 * to get enough granularity for the NPT unit-tests. 80 */ 81 82 address = 0; 83 84 /* PTE level */ 85 for (i = 0; i < 2048; ++i) { 86 page = alloc_page(); 87 88 for (j = 0; j < 512; ++j, address += 4096) 89 page[j] = address | 0x067ULL; 90 91 pte[i] = page; 92 } 93 94 /* PDE level */ 95 for (i = 0; i < 4; ++i) { 96 page = alloc_page(); 97 98 for (j = 0; j < 512; ++j) 99 page[j] = (u64)pte[(i * 512) + j] | 0x027ULL; 100 101 pde[i] = page; 102 } 103 104 /* PDPe level */ 105 pdpe = alloc_page(); 106 for (i = 0; i < 4; ++i) 107 pdpe[i] = ((u64)(pde[i])) | 0x27; 108 109 /* PML4e level */ 110 pml4e = alloc_page(); 111 pml4e[0] = ((u64)pdpe) | 0x27; 112 } 113 114 static u64 *npt_get_pde(u64 address) 115 { 116 int i1, i2; 117 118 address >>= 21; 119 i1 = (address >> 9) & 0x3; 120 i2 = address & 0x1ff; 121 122 return &pde[i1][i2]; 123 } 124 125 static u64 *npt_get_pte(u64 address) 126 { 127 int i1, i2; 128 129 address >>= 12; 130 i1 = (address >> 9) & 0x7ff; 131 i2 = address & 0x1ff; 132 133 return &pte[i1][i2]; 134 } 135 136 static void vmcb_set_seg(struct vmcb_seg *seg, u16 selector, 137 u64 base, u32 limit, u32 attr) 138 { 139 seg->selector = selector; 140 seg->attrib = attr; 141 seg->limit = limit; 142 seg->base = base; 143 } 144 145 static void vmcb_ident(struct vmcb *vmcb) 146 { 147 u64 vmcb_phys = virt_to_phys(vmcb); 148 struct vmcb_save_area *save = &vmcb->save; 149 struct vmcb_control_area *ctrl = &vmcb->control; 150 u32 data_seg_attr = 3 | SVM_SELECTOR_S_MASK | SVM_SELECTOR_P_MASK 151 | SVM_SELECTOR_DB_MASK | SVM_SELECTOR_G_MASK; 152 u32 code_seg_attr = 9 | SVM_SELECTOR_S_MASK | SVM_SELECTOR_P_MASK 153 | SVM_SELECTOR_L_MASK | SVM_SELECTOR_G_MASK; 154 struct descriptor_table_ptr desc_table_ptr; 155 156 memset(vmcb, 0, sizeof(*vmcb)); 157 asm volatile ("vmsave" : : "a"(vmcb_phys) : "memory"); 158 vmcb_set_seg(&save->es, read_es(), 0, -1U, data_seg_attr); 159 vmcb_set_seg(&save->cs, read_cs(), 0, -1U, code_seg_attr); 160 vmcb_set_seg(&save->ss, read_ss(), 0, -1U, data_seg_attr); 161 vmcb_set_seg(&save->ds, read_ds(), 0, -1U, data_seg_attr); 162 sgdt(&desc_table_ptr); 163 vmcb_set_seg(&save->gdtr, 0, desc_table_ptr.base, desc_table_ptr.limit, 0); 164 sidt(&desc_table_ptr); 165 vmcb_set_seg(&save->idtr, 0, desc_table_ptr.base, desc_table_ptr.limit, 0); 166 ctrl->asid = 1; 167 save->cpl = 0; 168 save->efer = rdmsr(MSR_EFER); 169 save->cr4 = read_cr4(); 170 save->cr3 = read_cr3(); 171 save->cr0 = read_cr0(); 172 save->dr7 = read_dr7(); 173 save->dr6 = read_dr6(); 174 save->cr2 = read_cr2(); 175 save->g_pat = rdmsr(MSR_IA32_CR_PAT); 176 save->dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 177 ctrl->intercept = (1ULL << INTERCEPT_VMRUN) | (1ULL << INTERCEPT_VMMCALL); 178 ctrl->iopm_base_pa = virt_to_phys(io_bitmap); 179 ctrl->msrpm_base_pa = virt_to_phys(msr_bitmap); 180 181 if (npt_supported()) { 182 ctrl->nested_ctl = 1; 183 ctrl->nested_cr3 = (u64)pml4e; 184 } 185 } 186 187 struct test { 188 const char *name; 189 bool (*supported)(void); 190 void (*prepare)(struct test *test); 191 void (*guest_func)(struct test *test); 192 bool (*finished)(struct test *test); 193 bool (*succeeded)(struct test *test); 194 struct vmcb *vmcb; 195 int exits; 196 ulong scratch; 197 }; 198 199 static inline void vmmcall(void) 200 { 201 asm volatile ("vmmcall" : : : "memory"); 202 } 203 204 static void test_thunk(struct test *test) 205 { 206 test->guest_func(test); 207 vmmcall(); 208 } 209 210 struct regs { 211 u64 rax; 212 u64 rbx; 213 u64 rcx; 214 u64 rdx; 215 u64 cr2; 216 u64 rbp; 217 u64 rsi; 218 u64 rdi; 219 u64 r8; 220 u64 r9; 221 u64 r10; 222 u64 r11; 223 u64 r12; 224 u64 r13; 225 u64 r14; 226 u64 r15; 227 u64 rflags; 228 }; 229 230 struct regs regs; 231 232 // rax handled specially below 233 234 #define SAVE_GPR_C \ 235 "xchg %%rbx, regs+0x8\n\t" \ 236 "xchg %%rcx, regs+0x10\n\t" \ 237 "xchg %%rdx, regs+0x18\n\t" \ 238 "xchg %%rbp, regs+0x28\n\t" \ 239 "xchg %%rsi, regs+0x30\n\t" \ 240 "xchg %%rdi, regs+0x38\n\t" \ 241 "xchg %%r8, regs+0x40\n\t" \ 242 "xchg %%r9, regs+0x48\n\t" \ 243 "xchg %%r10, regs+0x50\n\t" \ 244 "xchg %%r11, regs+0x58\n\t" \ 245 "xchg %%r12, regs+0x60\n\t" \ 246 "xchg %%r13, regs+0x68\n\t" \ 247 "xchg %%r14, regs+0x70\n\t" \ 248 "xchg %%r15, regs+0x78\n\t" 249 250 #define LOAD_GPR_C SAVE_GPR_C 251 252 static void test_run(struct test *test, struct vmcb *vmcb) 253 { 254 u64 vmcb_phys = virt_to_phys(vmcb); 255 u64 guest_stack[10000]; 256 257 test->vmcb = vmcb; 258 test->prepare(test); 259 vmcb->save.rip = (ulong)test_thunk; 260 vmcb->save.rsp = (ulong)(guest_stack + ARRAY_SIZE(guest_stack)); 261 regs.rdi = (ulong)test; 262 do { 263 tsc_start = rdtsc(); 264 asm volatile ( 265 "clgi \n\t" 266 "vmload \n\t" 267 "mov regs+0x80, %%r15\n\t" // rflags 268 "mov %%r15, 0x170(%0)\n\t" 269 "mov regs, %%r15\n\t" // rax 270 "mov %%r15, 0x1f8(%0)\n\t" 271 LOAD_GPR_C 272 "vmrun \n\t" 273 SAVE_GPR_C 274 "mov 0x170(%0), %%r15\n\t" // rflags 275 "mov %%r15, regs+0x80\n\t" 276 "mov 0x1f8(%0), %%r15\n\t" // rax 277 "mov %%r15, regs\n\t" 278 "vmsave \n\t" 279 "stgi" 280 : : "a"(vmcb_phys) 281 : "rbx", "rcx", "rdx", "rsi", 282 "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15", 283 "memory"); 284 tsc_end = rdtsc(); 285 ++test->exits; 286 } while (!test->finished(test)); 287 288 report("%s", test->succeeded(test), test->name); 289 } 290 291 static bool smp_supported(void) 292 { 293 return cpu_count() > 1; 294 } 295 296 static bool default_supported(void) 297 { 298 return true; 299 } 300 301 static void default_prepare(struct test *test) 302 { 303 vmcb_ident(test->vmcb); 304 cli(); 305 } 306 307 static bool default_finished(struct test *test) 308 { 309 return true; /* one vmexit */ 310 } 311 312 static void null_test(struct test *test) 313 { 314 } 315 316 static bool null_check(struct test *test) 317 { 318 return test->vmcb->control.exit_code == SVM_EXIT_VMMCALL; 319 } 320 321 static void prepare_no_vmrun_int(struct test *test) 322 { 323 test->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMRUN); 324 } 325 326 static bool check_no_vmrun_int(struct test *test) 327 { 328 return test->vmcb->control.exit_code == SVM_EXIT_ERR; 329 } 330 331 static void test_vmrun(struct test *test) 332 { 333 asm volatile ("vmrun" : : "a"(virt_to_phys(test->vmcb))); 334 } 335 336 static bool check_vmrun(struct test *test) 337 { 338 return test->vmcb->control.exit_code == SVM_EXIT_VMRUN; 339 } 340 341 static void prepare_cr3_intercept(struct test *test) 342 { 343 default_prepare(test); 344 test->vmcb->control.intercept_cr_read |= 1 << 3; 345 } 346 347 static void test_cr3_intercept(struct test *test) 348 { 349 asm volatile ("mov %%cr3, %0" : "=r"(test->scratch) : : "memory"); 350 } 351 352 static bool check_cr3_intercept(struct test *test) 353 { 354 return test->vmcb->control.exit_code == SVM_EXIT_READ_CR3; 355 } 356 357 static bool check_cr3_nointercept(struct test *test) 358 { 359 return null_check(test) && test->scratch == read_cr3(); 360 } 361 362 static void corrupt_cr3_intercept_bypass(void *_test) 363 { 364 struct test *test = _test; 365 extern volatile u32 mmio_insn; 366 367 while (!__sync_bool_compare_and_swap(&test->scratch, 1, 2)) 368 pause(); 369 pause(); 370 pause(); 371 pause(); 372 mmio_insn = 0x90d8200f; // mov %cr3, %rax; nop 373 } 374 375 static void prepare_cr3_intercept_bypass(struct test *test) 376 { 377 default_prepare(test); 378 test->vmcb->control.intercept_cr_read |= 1 << 3; 379 on_cpu_async(1, corrupt_cr3_intercept_bypass, test); 380 } 381 382 static void test_cr3_intercept_bypass(struct test *test) 383 { 384 ulong a = 0xa0000; 385 386 test->scratch = 1; 387 while (test->scratch != 2) 388 barrier(); 389 390 asm volatile ("mmio_insn: mov %0, (%0); nop" 391 : "+a"(a) : : "memory"); 392 test->scratch = a; 393 } 394 395 static void prepare_dr_intercept(struct test *test) 396 { 397 default_prepare(test); 398 test->vmcb->control.intercept_dr_read = 0xff; 399 test->vmcb->control.intercept_dr_write = 0xff; 400 } 401 402 static void test_dr_intercept(struct test *test) 403 { 404 unsigned int i, failcnt = 0; 405 406 /* Loop testing debug register reads */ 407 for (i = 0; i < 8; i++) { 408 409 switch (i) { 410 case 0: 411 asm volatile ("mov %%dr0, %0" : "=r"(test->scratch) : : "memory"); 412 break; 413 case 1: 414 asm volatile ("mov %%dr1, %0" : "=r"(test->scratch) : : "memory"); 415 break; 416 case 2: 417 asm volatile ("mov %%dr2, %0" : "=r"(test->scratch) : : "memory"); 418 break; 419 case 3: 420 asm volatile ("mov %%dr3, %0" : "=r"(test->scratch) : : "memory"); 421 break; 422 case 4: 423 asm volatile ("mov %%dr4, %0" : "=r"(test->scratch) : : "memory"); 424 break; 425 case 5: 426 asm volatile ("mov %%dr5, %0" : "=r"(test->scratch) : : "memory"); 427 break; 428 case 6: 429 asm volatile ("mov %%dr6, %0" : "=r"(test->scratch) : : "memory"); 430 break; 431 case 7: 432 asm volatile ("mov %%dr7, %0" : "=r"(test->scratch) : : "memory"); 433 break; 434 } 435 436 if (test->scratch != i) { 437 report("dr%u read intercept", false, i); 438 failcnt++; 439 } 440 } 441 442 /* Loop testing debug register writes */ 443 for (i = 0; i < 8; i++) { 444 445 switch (i) { 446 case 0: 447 asm volatile ("mov %0, %%dr0" : : "r"(test->scratch) : "memory"); 448 break; 449 case 1: 450 asm volatile ("mov %0, %%dr1" : : "r"(test->scratch) : "memory"); 451 break; 452 case 2: 453 asm volatile ("mov %0, %%dr2" : : "r"(test->scratch) : "memory"); 454 break; 455 case 3: 456 asm volatile ("mov %0, %%dr3" : : "r"(test->scratch) : "memory"); 457 break; 458 case 4: 459 asm volatile ("mov %0, %%dr4" : : "r"(test->scratch) : "memory"); 460 break; 461 case 5: 462 asm volatile ("mov %0, %%dr5" : : "r"(test->scratch) : "memory"); 463 break; 464 case 6: 465 asm volatile ("mov %0, %%dr6" : : "r"(test->scratch) : "memory"); 466 break; 467 case 7: 468 asm volatile ("mov %0, %%dr7" : : "r"(test->scratch) : "memory"); 469 break; 470 } 471 472 if (test->scratch != i) { 473 report("dr%u write intercept", false, i); 474 failcnt++; 475 } 476 } 477 478 test->scratch = failcnt; 479 } 480 481 static bool dr_intercept_finished(struct test *test) 482 { 483 ulong n = (test->vmcb->control.exit_code - SVM_EXIT_READ_DR0); 484 485 /* Only expect DR intercepts */ 486 if (n > (SVM_EXIT_MAX_DR_INTERCEPT - SVM_EXIT_READ_DR0)) 487 return true; 488 489 /* 490 * Compute debug register number. 491 * Per Appendix C "SVM Intercept Exit Codes" of AMD64 Architecture 492 * Programmer's Manual Volume 2 - System Programming: 493 * http://support.amd.com/TechDocs/24593.pdf 494 * there are 16 VMEXIT codes each for DR read and write. 495 */ 496 test->scratch = (n % 16); 497 498 /* Jump over MOV instruction */ 499 test->vmcb->save.rip += 3; 500 501 return false; 502 } 503 504 static bool check_dr_intercept(struct test *test) 505 { 506 return !test->scratch; 507 } 508 509 static bool next_rip_supported(void) 510 { 511 return this_cpu_has(X86_FEATURE_NRIPS); 512 } 513 514 static void prepare_next_rip(struct test *test) 515 { 516 test->vmcb->control.intercept |= (1ULL << INTERCEPT_RDTSC); 517 } 518 519 520 static void test_next_rip(struct test *test) 521 { 522 asm volatile ("rdtsc\n\t" 523 ".globl exp_next_rip\n\t" 524 "exp_next_rip:\n\t" ::: "eax", "edx"); 525 } 526 527 static bool check_next_rip(struct test *test) 528 { 529 extern char exp_next_rip; 530 unsigned long address = (unsigned long)&exp_next_rip; 531 532 return address == test->vmcb->control.next_rip; 533 } 534 535 static void prepare_msr_intercept(struct test *test) 536 { 537 default_prepare(test); 538 test->vmcb->control.intercept |= (1ULL << INTERCEPT_MSR_PROT); 539 test->vmcb->control.intercept_exceptions |= (1ULL << GP_VECTOR); 540 memset(msr_bitmap, 0xff, MSR_BITMAP_SIZE); 541 } 542 543 static void test_msr_intercept(struct test *test) 544 { 545 unsigned long msr_value = 0xef8056791234abcd; /* Arbitrary value */ 546 unsigned long msr_index; 547 548 for (msr_index = 0; msr_index <= 0xc0011fff; msr_index++) { 549 if (msr_index == 0xC0010131 /* MSR_SEV_STATUS */) { 550 /* 551 * Per section 15.34.10 "SEV_STATUS MSR" of AMD64 Architecture 552 * Programmer's Manual volume 2 - System Programming: 553 * http://support.amd.com/TechDocs/24593.pdf 554 * SEV_STATUS MSR (C001_0131) is a non-interceptable MSR. 555 */ 556 continue; 557 } 558 559 /* Skips gaps between supported MSR ranges */ 560 if (msr_index == 0x2000) 561 msr_index = 0xc0000000; 562 else if (msr_index == 0xc0002000) 563 msr_index = 0xc0010000; 564 565 test->scratch = -1; 566 567 rdmsr(msr_index); 568 569 /* Check that a read intercept occurred for MSR at msr_index */ 570 if (test->scratch != msr_index) 571 report("MSR 0x%lx read intercept", false, msr_index); 572 573 /* 574 * Poor man approach to generate a value that 575 * seems arbitrary each time around the loop. 576 */ 577 msr_value += (msr_value << 1); 578 579 wrmsr(msr_index, msr_value); 580 581 /* Check that a write intercept occurred for MSR with msr_value */ 582 if (test->scratch != msr_value) 583 report("MSR 0x%lx write intercept", false, msr_index); 584 } 585 586 test->scratch = -2; 587 } 588 589 static bool msr_intercept_finished(struct test *test) 590 { 591 u32 exit_code = test->vmcb->control.exit_code; 592 u64 exit_info_1; 593 u8 *opcode; 594 595 if (exit_code == SVM_EXIT_MSR) { 596 exit_info_1 = test->vmcb->control.exit_info_1; 597 } else { 598 /* 599 * If #GP exception occurs instead, check that it was 600 * for RDMSR/WRMSR and set exit_info_1 accordingly. 601 */ 602 603 if (exit_code != (SVM_EXIT_EXCP_BASE + GP_VECTOR)) 604 return true; 605 606 opcode = (u8 *)test->vmcb->save.rip; 607 if (opcode[0] != 0x0f) 608 return true; 609 610 switch (opcode[1]) { 611 case 0x30: /* WRMSR */ 612 exit_info_1 = 1; 613 break; 614 case 0x32: /* RDMSR */ 615 exit_info_1 = 0; 616 break; 617 default: 618 return true; 619 } 620 621 /* 622 * Warn that #GP exception occured instead. 623 * RCX holds the MSR index. 624 */ 625 printf("%s 0x%lx #GP exception\n", 626 exit_info_1 ? "WRMSR" : "RDMSR", regs.rcx); 627 } 628 629 /* Jump over RDMSR/WRMSR instruction */ 630 test->vmcb->save.rip += 2; 631 632 /* 633 * Test whether the intercept was for RDMSR/WRMSR. 634 * For RDMSR, test->scratch is set to the MSR index; 635 * RCX holds the MSR index. 636 * For WRMSR, test->scratch is set to the MSR value; 637 * RDX holds the upper 32 bits of the MSR value, 638 * while RAX hold its lower 32 bits. 639 */ 640 if (exit_info_1) 641 test->scratch = 642 ((regs.rdx << 32) | (test->vmcb->save.rax & 0xffffffff)); 643 else 644 test->scratch = regs.rcx; 645 646 return false; 647 } 648 649 static bool check_msr_intercept(struct test *test) 650 { 651 memset(msr_bitmap, 0, MSR_BITMAP_SIZE); 652 return (test->scratch == -2); 653 } 654 655 static void prepare_mode_switch(struct test *test) 656 { 657 test->vmcb->control.intercept_exceptions |= (1ULL << GP_VECTOR) 658 | (1ULL << UD_VECTOR) 659 | (1ULL << DF_VECTOR) 660 | (1ULL << PF_VECTOR); 661 test->scratch = 0; 662 } 663 664 static void test_mode_switch(struct test *test) 665 { 666 asm volatile(" cli\n" 667 " ljmp *1f\n" /* jump to 32-bit code segment */ 668 "1:\n" 669 " .long 2f\n" 670 " .long " xstr(KERNEL_CS32) "\n" 671 ".code32\n" 672 "2:\n" 673 " movl %%cr0, %%eax\n" 674 " btcl $31, %%eax\n" /* clear PG */ 675 " movl %%eax, %%cr0\n" 676 " movl $0xc0000080, %%ecx\n" /* EFER */ 677 " rdmsr\n" 678 " btcl $8, %%eax\n" /* clear LME */ 679 " wrmsr\n" 680 " movl %%cr4, %%eax\n" 681 " btcl $5, %%eax\n" /* clear PAE */ 682 " movl %%eax, %%cr4\n" 683 " movw %[ds16], %%ax\n" 684 " movw %%ax, %%ds\n" 685 " ljmpl %[cs16], $3f\n" /* jump to 16 bit protected-mode */ 686 ".code16\n" 687 "3:\n" 688 " movl %%cr0, %%eax\n" 689 " btcl $0, %%eax\n" /* clear PE */ 690 " movl %%eax, %%cr0\n" 691 " ljmpl $0, $4f\n" /* jump to real-mode */ 692 "4:\n" 693 " vmmcall\n" 694 " movl %%cr0, %%eax\n" 695 " btsl $0, %%eax\n" /* set PE */ 696 " movl %%eax, %%cr0\n" 697 " ljmpl %[cs32], $5f\n" /* back to protected mode */ 698 ".code32\n" 699 "5:\n" 700 " movl %%cr4, %%eax\n" 701 " btsl $5, %%eax\n" /* set PAE */ 702 " movl %%eax, %%cr4\n" 703 " movl $0xc0000080, %%ecx\n" /* EFER */ 704 " rdmsr\n" 705 " btsl $8, %%eax\n" /* set LME */ 706 " wrmsr\n" 707 " movl %%cr0, %%eax\n" 708 " btsl $31, %%eax\n" /* set PG */ 709 " movl %%eax, %%cr0\n" 710 " ljmpl %[cs64], $6f\n" /* back to long mode */ 711 ".code64\n\t" 712 "6:\n" 713 " vmmcall\n" 714 :: [cs16] "i"(KERNEL_CS16), [ds16] "i"(KERNEL_DS16), 715 [cs32] "i"(KERNEL_CS32), [cs64] "i"(KERNEL_CS64) 716 : "rax", "rbx", "rcx", "rdx", "memory"); 717 } 718 719 static bool mode_switch_finished(struct test *test) 720 { 721 u64 cr0, cr4, efer; 722 723 cr0 = test->vmcb->save.cr0; 724 cr4 = test->vmcb->save.cr4; 725 efer = test->vmcb->save.efer; 726 727 /* Only expect VMMCALL intercepts */ 728 if (test->vmcb->control.exit_code != SVM_EXIT_VMMCALL) 729 return true; 730 731 /* Jump over VMMCALL instruction */ 732 test->vmcb->save.rip += 3; 733 734 /* Do sanity checks */ 735 switch (test->scratch) { 736 case 0: 737 /* Test should be in real mode now - check for this */ 738 if ((cr0 & 0x80000001) || /* CR0.PG, CR0.PE */ 739 (cr4 & 0x00000020) || /* CR4.PAE */ 740 (efer & 0x00000500)) /* EFER.LMA, EFER.LME */ 741 return true; 742 break; 743 case 2: 744 /* Test should be back in long-mode now - check for this */ 745 if (((cr0 & 0x80000001) != 0x80000001) || /* CR0.PG, CR0.PE */ 746 ((cr4 & 0x00000020) != 0x00000020) || /* CR4.PAE */ 747 ((efer & 0x00000500) != 0x00000500)) /* EFER.LMA, EFER.LME */ 748 return true; 749 break; 750 } 751 752 /* one step forward */ 753 test->scratch += 1; 754 755 return test->scratch == 2; 756 } 757 758 static bool check_mode_switch(struct test *test) 759 { 760 return test->scratch == 2; 761 } 762 763 static void prepare_ioio(struct test *test) 764 { 765 test->vmcb->control.intercept |= (1ULL << INTERCEPT_IOIO_PROT); 766 test->scratch = 0; 767 memset(io_bitmap, 0, 8192); 768 io_bitmap[8192] = 0xFF; 769 } 770 771 static int get_test_stage(struct test *test) 772 { 773 barrier(); 774 return test->scratch; 775 } 776 777 static void inc_test_stage(struct test *test) 778 { 779 barrier(); 780 test->scratch++; 781 barrier(); 782 } 783 784 static void test_ioio(struct test *test) 785 { 786 // stage 0, test IO pass 787 inb(0x5000); 788 outb(0x0, 0x5000); 789 if (get_test_stage(test) != 0) 790 goto fail; 791 792 // test IO width, in/out 793 io_bitmap[0] = 0xFF; 794 inc_test_stage(test); 795 inb(0x0); 796 if (get_test_stage(test) != 2) 797 goto fail; 798 799 outw(0x0, 0x0); 800 if (get_test_stage(test) != 3) 801 goto fail; 802 803 inl(0x0); 804 if (get_test_stage(test) != 4) 805 goto fail; 806 807 // test low/high IO port 808 io_bitmap[0x5000 / 8] = (1 << (0x5000 % 8)); 809 inb(0x5000); 810 if (get_test_stage(test) != 5) 811 goto fail; 812 813 io_bitmap[0x9000 / 8] = (1 << (0x9000 % 8)); 814 inw(0x9000); 815 if (get_test_stage(test) != 6) 816 goto fail; 817 818 // test partial pass 819 io_bitmap[0x5000 / 8] = (1 << (0x5000 % 8)); 820 inl(0x4FFF); 821 if (get_test_stage(test) != 7) 822 goto fail; 823 824 // test across pages 825 inc_test_stage(test); 826 inl(0x7FFF); 827 if (get_test_stage(test) != 8) 828 goto fail; 829 830 inc_test_stage(test); 831 io_bitmap[0x8000 / 8] = 1 << (0x8000 % 8); 832 inl(0x7FFF); 833 if (get_test_stage(test) != 10) 834 goto fail; 835 836 io_bitmap[0] = 0; 837 inl(0xFFFF); 838 if (get_test_stage(test) != 11) 839 goto fail; 840 841 io_bitmap[0] = 0xFF; 842 io_bitmap[8192] = 0; 843 inl(0xFFFF); 844 inc_test_stage(test); 845 if (get_test_stage(test) != 12) 846 goto fail; 847 848 return; 849 850 fail: 851 report("stage %d", false, get_test_stage(test)); 852 test->scratch = -1; 853 } 854 855 static bool ioio_finished(struct test *test) 856 { 857 unsigned port, size; 858 859 /* Only expect IOIO intercepts */ 860 if (test->vmcb->control.exit_code == SVM_EXIT_VMMCALL) 861 return true; 862 863 if (test->vmcb->control.exit_code != SVM_EXIT_IOIO) 864 return true; 865 866 /* one step forward */ 867 test->scratch += 1; 868 869 port = test->vmcb->control.exit_info_1 >> 16; 870 size = (test->vmcb->control.exit_info_1 >> SVM_IOIO_SIZE_SHIFT) & 7; 871 872 while (size--) { 873 io_bitmap[port / 8] &= ~(1 << (port & 7)); 874 port++; 875 } 876 877 return false; 878 } 879 880 static bool check_ioio(struct test *test) 881 { 882 memset(io_bitmap, 0, 8193); 883 return test->scratch != -1; 884 } 885 886 static void prepare_asid_zero(struct test *test) 887 { 888 test->vmcb->control.asid = 0; 889 } 890 891 static void test_asid_zero(struct test *test) 892 { 893 asm volatile ("vmmcall\n\t"); 894 } 895 896 static bool check_asid_zero(struct test *test) 897 { 898 return test->vmcb->control.exit_code == SVM_EXIT_ERR; 899 } 900 901 static void sel_cr0_bug_prepare(struct test *test) 902 { 903 vmcb_ident(test->vmcb); 904 test->vmcb->control.intercept |= (1ULL << INTERCEPT_SELECTIVE_CR0); 905 } 906 907 static bool sel_cr0_bug_finished(struct test *test) 908 { 909 return true; 910 } 911 912 static void sel_cr0_bug_test(struct test *test) 913 { 914 unsigned long cr0; 915 916 /* read cr0, clear CD, and write back */ 917 cr0 = read_cr0(); 918 cr0 |= (1UL << 30); 919 write_cr0(cr0); 920 921 /* 922 * If we are here the test failed, not sure what to do now because we 923 * are not in guest-mode anymore so we can't trigger an intercept. 924 * Trigger a tripple-fault for now. 925 */ 926 report("sel_cr0 test. Can not recover from this - exiting", false); 927 exit(report_summary()); 928 } 929 930 static bool sel_cr0_bug_check(struct test *test) 931 { 932 return test->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE; 933 } 934 935 static void npt_nx_prepare(struct test *test) 936 { 937 938 u64 *pte; 939 940 vmcb_ident(test->vmcb); 941 pte = npt_get_pte((u64)null_test); 942 943 *pte |= (1ULL << 63); 944 } 945 946 static bool npt_nx_check(struct test *test) 947 { 948 u64 *pte = npt_get_pte((u64)null_test); 949 950 *pte &= ~(1ULL << 63); 951 952 test->vmcb->save.efer |= (1 << 11); 953 954 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 955 && (test->vmcb->control.exit_info_1 == 0x100000015ULL); 956 } 957 958 static void npt_us_prepare(struct test *test) 959 { 960 u64 *pte; 961 962 vmcb_ident(test->vmcb); 963 pte = npt_get_pte((u64)scratch_page); 964 965 *pte &= ~(1ULL << 2); 966 } 967 968 static void npt_us_test(struct test *test) 969 { 970 (void) *(volatile u64 *)scratch_page; 971 } 972 973 static bool npt_us_check(struct test *test) 974 { 975 u64 *pte = npt_get_pte((u64)scratch_page); 976 977 *pte |= (1ULL << 2); 978 979 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 980 && (test->vmcb->control.exit_info_1 == 0x100000005ULL); 981 } 982 983 u64 save_pde; 984 985 static void npt_rsvd_prepare(struct test *test) 986 { 987 u64 *pde; 988 989 vmcb_ident(test->vmcb); 990 pde = npt_get_pde((u64) null_test); 991 992 save_pde = *pde; 993 *pde = (1ULL << 19) | (1ULL << 7) | 0x27; 994 } 995 996 static bool npt_rsvd_check(struct test *test) 997 { 998 u64 *pde = npt_get_pde((u64) null_test); 999 1000 *pde = save_pde; 1001 1002 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 1003 && (test->vmcb->control.exit_info_1 == 0x10000001dULL); 1004 } 1005 1006 static void npt_rw_prepare(struct test *test) 1007 { 1008 1009 u64 *pte; 1010 1011 vmcb_ident(test->vmcb); 1012 pte = npt_get_pte(0x80000); 1013 1014 *pte &= ~(1ULL << 1); 1015 } 1016 1017 static void npt_rw_test(struct test *test) 1018 { 1019 u64 *data = (void*)(0x80000); 1020 1021 *data = 0; 1022 } 1023 1024 static bool npt_rw_check(struct test *test) 1025 { 1026 u64 *pte = npt_get_pte(0x80000); 1027 1028 *pte |= (1ULL << 1); 1029 1030 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 1031 && (test->vmcb->control.exit_info_1 == 0x100000007ULL); 1032 } 1033 1034 static void npt_rw_pfwalk_prepare(struct test *test) 1035 { 1036 1037 u64 *pte; 1038 1039 vmcb_ident(test->vmcb); 1040 pte = npt_get_pte(read_cr3()); 1041 1042 *pte &= ~(1ULL << 1); 1043 } 1044 1045 static bool npt_rw_pfwalk_check(struct test *test) 1046 { 1047 u64 *pte = npt_get_pte(read_cr3()); 1048 1049 *pte |= (1ULL << 1); 1050 1051 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 1052 && (test->vmcb->control.exit_info_1 == 0x200000006ULL) 1053 && (test->vmcb->control.exit_info_2 == read_cr3()); 1054 } 1055 1056 static void npt_rsvd_pfwalk_prepare(struct test *test) 1057 { 1058 1059 vmcb_ident(test->vmcb); 1060 1061 pdpe[0] |= (1ULL << 8); 1062 } 1063 1064 static bool npt_rsvd_pfwalk_check(struct test *test) 1065 { 1066 pdpe[0] &= ~(1ULL << 8); 1067 1068 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 1069 && (test->vmcb->control.exit_info_1 == 0x200000006ULL); 1070 } 1071 1072 static void npt_l1mmio_prepare(struct test *test) 1073 { 1074 vmcb_ident(test->vmcb); 1075 } 1076 1077 u32 nested_apic_version1; 1078 u32 nested_apic_version2; 1079 1080 static void npt_l1mmio_test(struct test *test) 1081 { 1082 volatile u32 *data = (volatile void*)(0xfee00030UL); 1083 1084 nested_apic_version1 = *data; 1085 nested_apic_version2 = *data; 1086 } 1087 1088 static bool npt_l1mmio_check(struct test *test) 1089 { 1090 volatile u32 *data = (volatile void*)(0xfee00030); 1091 u32 lvr = *data; 1092 1093 return nested_apic_version1 == lvr && nested_apic_version2 == lvr; 1094 } 1095 1096 static void npt_rw_l1mmio_prepare(struct test *test) 1097 { 1098 1099 u64 *pte; 1100 1101 vmcb_ident(test->vmcb); 1102 pte = npt_get_pte(0xfee00080); 1103 1104 *pte &= ~(1ULL << 1); 1105 } 1106 1107 static void npt_rw_l1mmio_test(struct test *test) 1108 { 1109 volatile u32 *data = (volatile void*)(0xfee00080); 1110 1111 *data = *data; 1112 } 1113 1114 static bool npt_rw_l1mmio_check(struct test *test) 1115 { 1116 u64 *pte = npt_get_pte(0xfee00080); 1117 1118 *pte |= (1ULL << 1); 1119 1120 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 1121 && (test->vmcb->control.exit_info_1 == 0x100000007ULL); 1122 } 1123 1124 #define TSC_ADJUST_VALUE (1ll << 32) 1125 #define TSC_OFFSET_VALUE (-1ll << 48) 1126 static bool ok; 1127 1128 static void tsc_adjust_prepare(struct test *test) 1129 { 1130 default_prepare(test); 1131 test->vmcb->control.tsc_offset = TSC_OFFSET_VALUE; 1132 1133 wrmsr(MSR_IA32_TSC_ADJUST, -TSC_ADJUST_VALUE); 1134 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST); 1135 ok = adjust == -TSC_ADJUST_VALUE; 1136 } 1137 1138 static void tsc_adjust_test(struct test *test) 1139 { 1140 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST); 1141 ok &= adjust == -TSC_ADJUST_VALUE; 1142 1143 uint64_t l1_tsc = rdtsc() - TSC_OFFSET_VALUE; 1144 wrmsr(MSR_IA32_TSC, l1_tsc - TSC_ADJUST_VALUE); 1145 1146 adjust = rdmsr(MSR_IA32_TSC_ADJUST); 1147 ok &= adjust <= -2 * TSC_ADJUST_VALUE; 1148 1149 uint64_t l1_tsc_end = rdtsc() - TSC_OFFSET_VALUE; 1150 ok &= (l1_tsc_end + TSC_ADJUST_VALUE - l1_tsc) < TSC_ADJUST_VALUE; 1151 1152 uint64_t l1_tsc_msr = rdmsr(MSR_IA32_TSC) - TSC_OFFSET_VALUE; 1153 ok &= (l1_tsc_msr + TSC_ADJUST_VALUE - l1_tsc) < TSC_ADJUST_VALUE; 1154 } 1155 1156 static bool tsc_adjust_check(struct test *test) 1157 { 1158 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST); 1159 1160 wrmsr(MSR_IA32_TSC_ADJUST, 0); 1161 return ok && adjust <= -2 * TSC_ADJUST_VALUE; 1162 } 1163 1164 static void latency_prepare(struct test *test) 1165 { 1166 default_prepare(test); 1167 runs = LATENCY_RUNS; 1168 latvmrun_min = latvmexit_min = -1ULL; 1169 latvmrun_max = latvmexit_max = 0; 1170 vmrun_sum = vmexit_sum = 0; 1171 } 1172 1173 static void latency_test(struct test *test) 1174 { 1175 u64 cycles; 1176 1177 start: 1178 tsc_end = rdtsc(); 1179 1180 cycles = tsc_end - tsc_start; 1181 1182 if (cycles > latvmrun_max) 1183 latvmrun_max = cycles; 1184 1185 if (cycles < latvmrun_min) 1186 latvmrun_min = cycles; 1187 1188 vmrun_sum += cycles; 1189 1190 tsc_start = rdtsc(); 1191 1192 asm volatile ("vmmcall" : : : "memory"); 1193 goto start; 1194 } 1195 1196 static bool latency_finished(struct test *test) 1197 { 1198 u64 cycles; 1199 1200 tsc_end = rdtsc(); 1201 1202 cycles = tsc_end - tsc_start; 1203 1204 if (cycles > latvmexit_max) 1205 latvmexit_max = cycles; 1206 1207 if (cycles < latvmexit_min) 1208 latvmexit_min = cycles; 1209 1210 vmexit_sum += cycles; 1211 1212 test->vmcb->save.rip += 3; 1213 1214 runs -= 1; 1215 1216 return runs == 0; 1217 } 1218 1219 static bool latency_check(struct test *test) 1220 { 1221 printf(" Latency VMRUN : max: %ld min: %ld avg: %ld\n", latvmrun_max, 1222 latvmrun_min, vmrun_sum / LATENCY_RUNS); 1223 printf(" Latency VMEXIT: max: %ld min: %ld avg: %ld\n", latvmexit_max, 1224 latvmexit_min, vmexit_sum / LATENCY_RUNS); 1225 return true; 1226 } 1227 1228 static void lat_svm_insn_prepare(struct test *test) 1229 { 1230 default_prepare(test); 1231 runs = LATENCY_RUNS; 1232 latvmload_min = latvmsave_min = latstgi_min = latclgi_min = -1ULL; 1233 latvmload_max = latvmsave_max = latstgi_max = latclgi_max = 0; 1234 vmload_sum = vmsave_sum = stgi_sum = clgi_sum; 1235 } 1236 1237 static bool lat_svm_insn_finished(struct test *test) 1238 { 1239 u64 vmcb_phys = virt_to_phys(test->vmcb); 1240 u64 cycles; 1241 1242 for ( ; runs != 0; runs--) { 1243 tsc_start = rdtsc(); 1244 asm volatile("vmload\n\t" : : "a"(vmcb_phys) : "memory"); 1245 cycles = rdtsc() - tsc_start; 1246 if (cycles > latvmload_max) 1247 latvmload_max = cycles; 1248 if (cycles < latvmload_min) 1249 latvmload_min = cycles; 1250 vmload_sum += cycles; 1251 1252 tsc_start = rdtsc(); 1253 asm volatile("vmsave\n\t" : : "a"(vmcb_phys) : "memory"); 1254 cycles = rdtsc() - tsc_start; 1255 if (cycles > latvmsave_max) 1256 latvmsave_max = cycles; 1257 if (cycles < latvmsave_min) 1258 latvmsave_min = cycles; 1259 vmsave_sum += cycles; 1260 1261 tsc_start = rdtsc(); 1262 asm volatile("stgi\n\t"); 1263 cycles = rdtsc() - tsc_start; 1264 if (cycles > latstgi_max) 1265 latstgi_max = cycles; 1266 if (cycles < latstgi_min) 1267 latstgi_min = cycles; 1268 stgi_sum += cycles; 1269 1270 tsc_start = rdtsc(); 1271 asm volatile("clgi\n\t"); 1272 cycles = rdtsc() - tsc_start; 1273 if (cycles > latclgi_max) 1274 latclgi_max = cycles; 1275 if (cycles < latclgi_min) 1276 latclgi_min = cycles; 1277 clgi_sum += cycles; 1278 } 1279 1280 return true; 1281 } 1282 1283 static bool lat_svm_insn_check(struct test *test) 1284 { 1285 printf(" Latency VMLOAD: max: %ld min: %ld avg: %ld\n", latvmload_max, 1286 latvmload_min, vmload_sum / LATENCY_RUNS); 1287 printf(" Latency VMSAVE: max: %ld min: %ld avg: %ld\n", latvmsave_max, 1288 latvmsave_min, vmsave_sum / LATENCY_RUNS); 1289 printf(" Latency STGI: max: %ld min: %ld avg: %ld\n", latstgi_max, 1290 latstgi_min, stgi_sum / LATENCY_RUNS); 1291 printf(" Latency CLGI: max: %ld min: %ld avg: %ld\n", latclgi_max, 1292 latclgi_min, clgi_sum / LATENCY_RUNS); 1293 return true; 1294 } 1295 static struct test tests[] = { 1296 { "null", default_supported, default_prepare, null_test, 1297 default_finished, null_check }, 1298 { "vmrun", default_supported, default_prepare, test_vmrun, 1299 default_finished, check_vmrun }, 1300 { "ioio", default_supported, prepare_ioio, test_ioio, 1301 ioio_finished, check_ioio }, 1302 { "vmrun intercept check", default_supported, prepare_no_vmrun_int, 1303 null_test, default_finished, check_no_vmrun_int }, 1304 { "cr3 read intercept", default_supported, prepare_cr3_intercept, 1305 test_cr3_intercept, default_finished, check_cr3_intercept }, 1306 { "cr3 read nointercept", default_supported, default_prepare, 1307 test_cr3_intercept, default_finished, check_cr3_nointercept }, 1308 { "cr3 read intercept emulate", smp_supported, 1309 prepare_cr3_intercept_bypass, test_cr3_intercept_bypass, 1310 default_finished, check_cr3_intercept }, 1311 { "dr intercept check", default_supported, prepare_dr_intercept, 1312 test_dr_intercept, dr_intercept_finished, check_dr_intercept }, 1313 { "next_rip", next_rip_supported, prepare_next_rip, test_next_rip, 1314 default_finished, check_next_rip }, 1315 { "msr intercept check", default_supported, prepare_msr_intercept, 1316 test_msr_intercept, msr_intercept_finished, check_msr_intercept }, 1317 { "mode_switch", default_supported, prepare_mode_switch, test_mode_switch, 1318 mode_switch_finished, check_mode_switch }, 1319 { "asid_zero", default_supported, prepare_asid_zero, test_asid_zero, 1320 default_finished, check_asid_zero }, 1321 { "sel_cr0_bug", default_supported, sel_cr0_bug_prepare, sel_cr0_bug_test, 1322 sel_cr0_bug_finished, sel_cr0_bug_check }, 1323 { "npt_nx", npt_supported, npt_nx_prepare, null_test, 1324 default_finished, npt_nx_check }, 1325 { "npt_us", npt_supported, npt_us_prepare, npt_us_test, 1326 default_finished, npt_us_check }, 1327 { "npt_rsvd", npt_supported, npt_rsvd_prepare, null_test, 1328 default_finished, npt_rsvd_check }, 1329 { "npt_rw", npt_supported, npt_rw_prepare, npt_rw_test, 1330 default_finished, npt_rw_check }, 1331 { "npt_rsvd_pfwalk", npt_supported, npt_rsvd_pfwalk_prepare, null_test, 1332 default_finished, npt_rsvd_pfwalk_check }, 1333 { "npt_rw_pfwalk", npt_supported, npt_rw_pfwalk_prepare, null_test, 1334 default_finished, npt_rw_pfwalk_check }, 1335 { "npt_l1mmio", npt_supported, npt_l1mmio_prepare, npt_l1mmio_test, 1336 default_finished, npt_l1mmio_check }, 1337 { "npt_rw_l1mmio", npt_supported, npt_rw_l1mmio_prepare, npt_rw_l1mmio_test, 1338 default_finished, npt_rw_l1mmio_check }, 1339 { "tsc_adjust", default_supported, tsc_adjust_prepare, tsc_adjust_test, 1340 default_finished, tsc_adjust_check }, 1341 { "latency_run_exit", default_supported, latency_prepare, latency_test, 1342 latency_finished, latency_check }, 1343 { "latency_svm_insn", default_supported, lat_svm_insn_prepare, null_test, 1344 lat_svm_insn_finished, lat_svm_insn_check }, 1345 }; 1346 1347 int main(int ac, char **av) 1348 { 1349 int i, nr; 1350 struct vmcb *vmcb; 1351 1352 setup_vm(); 1353 smp_init(); 1354 1355 if (!this_cpu_has(X86_FEATURE_SVM)) { 1356 printf("SVM not availble\n"); 1357 return report_summary(); 1358 } 1359 1360 setup_svm(); 1361 1362 vmcb = alloc_page(); 1363 1364 nr = ARRAY_SIZE(tests); 1365 for (i = 0; i < nr; ++i) { 1366 if (!tests[i].supported()) 1367 continue; 1368 test_run(&tests[i], vmcb); 1369 } 1370 1371 return report_summary(); 1372 } 1373