1 #include "svm.h" 2 #include "libcflat.h" 3 #include "processor.h" 4 #include "msr.h" 5 #include "vm.h" 6 #include "smp.h" 7 #include "types.h" 8 9 /* for the nested page table*/ 10 u64 *pml4e; 11 u64 *pdpe; 12 u64 *pde[4]; 13 u64 *pte[2048]; 14 u64 *scratch_page; 15 16 #define LATENCY_RUNS 1000000 17 18 u64 tsc_start; 19 u64 tsc_end; 20 21 u64 vmrun_sum, vmexit_sum; 22 u64 latvmrun_max; 23 u64 latvmrun_min; 24 u64 latvmexit_max; 25 u64 latvmexit_min; 26 u64 runs; 27 28 static bool npt_supported(void) 29 { 30 return cpuid(0x8000000A).d & 1; 31 } 32 33 static void setup_svm(void) 34 { 35 void *hsave = alloc_page(); 36 u64 *page, address; 37 int i,j; 38 39 wrmsr(MSR_VM_HSAVE_PA, virt_to_phys(hsave)); 40 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_SVME); 41 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX); 42 43 scratch_page = alloc_page(); 44 45 if (!npt_supported()) 46 return; 47 48 printf("NPT detected - running all tests with NPT enabled\n"); 49 50 /* 51 * Nested paging supported - Build a nested page table 52 * Build the page-table bottom-up and map everything with 4k pages 53 * to get enough granularity for the NPT unit-tests. 54 */ 55 56 address = 0; 57 58 /* PTE level */ 59 for (i = 0; i < 2048; ++i) { 60 page = alloc_page(); 61 62 for (j = 0; j < 512; ++j, address += 4096) 63 page[j] = address | 0x067ULL; 64 65 pte[i] = page; 66 } 67 68 /* PDE level */ 69 for (i = 0; i < 4; ++i) { 70 page = alloc_page(); 71 72 for (j = 0; j < 512; ++j) 73 page[j] = (u64)pte[(i * 514) + j] | 0x027ULL; 74 75 pde[i] = page; 76 } 77 78 /* PDPe level */ 79 pdpe = alloc_page(); 80 for (i = 0; i < 4; ++i) 81 pdpe[i] = ((u64)(pde[i])) | 0x27; 82 83 /* PML4e level */ 84 pml4e = alloc_page(); 85 pml4e[0] = ((u64)pdpe) | 0x27; 86 } 87 88 static u64 *get_pte(u64 address) 89 { 90 int i1, i2; 91 92 address >>= 12; 93 i1 = (address >> 9) & 0x7ff; 94 i2 = address & 0x1ff; 95 96 return &pte[i1][i2]; 97 } 98 99 static void vmcb_set_seg(struct vmcb_seg *seg, u16 selector, 100 u64 base, u32 limit, u32 attr) 101 { 102 seg->selector = selector; 103 seg->attrib = attr; 104 seg->limit = limit; 105 seg->base = base; 106 } 107 108 static void vmcb_ident(struct vmcb *vmcb) 109 { 110 u64 vmcb_phys = virt_to_phys(vmcb); 111 struct vmcb_save_area *save = &vmcb->save; 112 struct vmcb_control_area *ctrl = &vmcb->control; 113 u32 data_seg_attr = 3 | SVM_SELECTOR_S_MASK | SVM_SELECTOR_P_MASK 114 | SVM_SELECTOR_DB_MASK | SVM_SELECTOR_G_MASK; 115 u32 code_seg_attr = 9 | SVM_SELECTOR_S_MASK | SVM_SELECTOR_P_MASK 116 | SVM_SELECTOR_L_MASK | SVM_SELECTOR_G_MASK; 117 struct descriptor_table_ptr desc_table_ptr; 118 119 memset(vmcb, 0, sizeof(*vmcb)); 120 asm volatile ("vmsave" : : "a"(vmcb_phys) : "memory"); 121 vmcb_set_seg(&save->es, read_es(), 0, -1U, data_seg_attr); 122 vmcb_set_seg(&save->cs, read_cs(), 0, -1U, code_seg_attr); 123 vmcb_set_seg(&save->ss, read_ss(), 0, -1U, data_seg_attr); 124 vmcb_set_seg(&save->ds, read_ds(), 0, -1U, data_seg_attr); 125 sgdt(&desc_table_ptr); 126 vmcb_set_seg(&save->gdtr, 0, desc_table_ptr.base, desc_table_ptr.limit, 0); 127 sidt(&desc_table_ptr); 128 vmcb_set_seg(&save->idtr, 0, desc_table_ptr.base, desc_table_ptr.limit, 0); 129 ctrl->asid = 1; 130 save->cpl = 0; 131 save->efer = rdmsr(MSR_EFER); 132 save->cr4 = read_cr4(); 133 save->cr3 = read_cr3(); 134 save->cr0 = read_cr0(); 135 save->dr7 = read_dr7(); 136 save->dr6 = read_dr6(); 137 save->cr2 = read_cr2(); 138 save->g_pat = rdmsr(MSR_IA32_CR_PAT); 139 save->dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 140 ctrl->intercept = (1ULL << INTERCEPT_VMRUN) | (1ULL << INTERCEPT_VMMCALL); 141 142 if (npt_supported()) { 143 ctrl->nested_ctl = 1; 144 ctrl->nested_cr3 = (u64)pml4e; 145 } 146 } 147 148 struct test { 149 const char *name; 150 bool (*supported)(void); 151 void (*prepare)(struct test *test); 152 void (*guest_func)(struct test *test); 153 bool (*finished)(struct test *test); 154 bool (*succeeded)(struct test *test); 155 struct vmcb *vmcb; 156 int exits; 157 ulong scratch; 158 }; 159 160 static void test_thunk(struct test *test) 161 { 162 test->guest_func(test); 163 asm volatile ("vmmcall" : : : "memory"); 164 } 165 166 static bool test_run(struct test *test, struct vmcb *vmcb) 167 { 168 u64 vmcb_phys = virt_to_phys(vmcb); 169 u64 guest_stack[10000]; 170 bool success; 171 172 test->vmcb = vmcb; 173 test->prepare(test); 174 vmcb->save.rip = (ulong)test_thunk; 175 vmcb->save.rsp = (ulong)(guest_stack + ARRAY_SIZE(guest_stack)); 176 do { 177 tsc_start = rdtsc(); 178 asm volatile ( 179 "clgi \n\t" 180 "vmload \n\t" 181 "push %%rbp \n\t" 182 "push %1 \n\t" 183 "vmrun \n\t" 184 "pop %1 \n\t" 185 "pop %%rbp \n\t" 186 "vmsave \n\t" 187 "stgi" 188 : : "a"(vmcb_phys), "D"(test) 189 : "rbx", "rcx", "rdx", "rsi", 190 "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15", 191 "memory"); 192 tsc_end = rdtsc(); 193 ++test->exits; 194 } while (!test->finished(test)); 195 196 197 success = test->succeeded(test); 198 199 printf("%s: %s\n", test->name, success ? "PASS" : "FAIL"); 200 201 return success; 202 } 203 204 static bool default_supported(void) 205 { 206 return true; 207 } 208 209 static void default_prepare(struct test *test) 210 { 211 vmcb_ident(test->vmcb); 212 cli(); 213 } 214 215 static bool default_finished(struct test *test) 216 { 217 return true; /* one vmexit */ 218 } 219 220 static void null_test(struct test *test) 221 { 222 } 223 224 static bool null_check(struct test *test) 225 { 226 return test->vmcb->control.exit_code == SVM_EXIT_VMMCALL; 227 } 228 229 static void prepare_no_vmrun_int(struct test *test) 230 { 231 test->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMRUN); 232 } 233 234 static bool check_no_vmrun_int(struct test *test) 235 { 236 return test->vmcb->control.exit_code == SVM_EXIT_ERR; 237 } 238 239 static void test_vmrun(struct test *test) 240 { 241 asm volatile ("vmrun" : : "a"(virt_to_phys(test->vmcb))); 242 } 243 244 static bool check_vmrun(struct test *test) 245 { 246 return test->vmcb->control.exit_code == SVM_EXIT_VMRUN; 247 } 248 249 static void prepare_cr3_intercept(struct test *test) 250 { 251 default_prepare(test); 252 test->vmcb->control.intercept_cr_read |= 1 << 3; 253 } 254 255 static void test_cr3_intercept(struct test *test) 256 { 257 asm volatile ("mov %%cr3, %0" : "=r"(test->scratch) : : "memory"); 258 } 259 260 static bool check_cr3_intercept(struct test *test) 261 { 262 return test->vmcb->control.exit_code == SVM_EXIT_READ_CR3; 263 } 264 265 static bool check_cr3_nointercept(struct test *test) 266 { 267 return null_check(test) && test->scratch == read_cr3(); 268 } 269 270 static void corrupt_cr3_intercept_bypass(void *_test) 271 { 272 struct test *test = _test; 273 extern volatile u32 mmio_insn; 274 275 while (!__sync_bool_compare_and_swap(&test->scratch, 1, 2)) 276 pause(); 277 pause(); 278 pause(); 279 pause(); 280 mmio_insn = 0x90d8200f; // mov %cr3, %rax; nop 281 } 282 283 static void prepare_cr3_intercept_bypass(struct test *test) 284 { 285 default_prepare(test); 286 test->vmcb->control.intercept_cr_read |= 1 << 3; 287 on_cpu_async(1, corrupt_cr3_intercept_bypass, test); 288 } 289 290 static void test_cr3_intercept_bypass(struct test *test) 291 { 292 ulong a = 0xa0000; 293 294 test->scratch = 1; 295 while (test->scratch != 2) 296 barrier(); 297 298 asm volatile ("mmio_insn: mov %0, (%0); nop" 299 : "+a"(a) : : "memory"); 300 test->scratch = a; 301 } 302 303 static bool next_rip_supported(void) 304 { 305 return (cpuid(SVM_CPUID_FUNC).d & 8); 306 } 307 308 static void prepare_next_rip(struct test *test) 309 { 310 test->vmcb->control.intercept |= (1ULL << INTERCEPT_RDTSC); 311 } 312 313 314 static void test_next_rip(struct test *test) 315 { 316 asm volatile ("rdtsc\n\t" 317 ".globl exp_next_rip\n\t" 318 "exp_next_rip:\n\t" ::: "eax", "edx"); 319 } 320 321 static bool check_next_rip(struct test *test) 322 { 323 extern char exp_next_rip; 324 unsigned long address = (unsigned long)&exp_next_rip; 325 326 return address == test->vmcb->control.next_rip; 327 } 328 329 static void prepare_mode_switch(struct test *test) 330 { 331 test->vmcb->control.intercept_exceptions |= (1ULL << GP_VECTOR) 332 | (1ULL << UD_VECTOR) 333 | (1ULL << DF_VECTOR) 334 | (1ULL << PF_VECTOR); 335 test->scratch = 0; 336 } 337 338 static void test_mode_switch(struct test *test) 339 { 340 asm volatile(" cli\n" 341 " ljmp *1f\n" /* jump to 32-bit code segment */ 342 "1:\n" 343 " .long 2f\n" 344 " .long 40\n" 345 ".code32\n" 346 "2:\n" 347 " movl %%cr0, %%eax\n" 348 " btcl $31, %%eax\n" /* clear PG */ 349 " movl %%eax, %%cr0\n" 350 " movl $0xc0000080, %%ecx\n" /* EFER */ 351 " rdmsr\n" 352 " btcl $8, %%eax\n" /* clear LME */ 353 " wrmsr\n" 354 " movl %%cr4, %%eax\n" 355 " btcl $5, %%eax\n" /* clear PAE */ 356 " movl %%eax, %%cr4\n" 357 " movw $64, %%ax\n" 358 " movw %%ax, %%ds\n" 359 " ljmpl $56, $3f\n" /* jump to 16 bit protected-mode */ 360 ".code16\n" 361 "3:\n" 362 " movl %%cr0, %%eax\n" 363 " btcl $0, %%eax\n" /* clear PE */ 364 " movl %%eax, %%cr0\n" 365 " ljmpl $0, $4f\n" /* jump to real-mode */ 366 "4:\n" 367 " vmmcall\n" 368 " movl %%cr0, %%eax\n" 369 " btsl $0, %%eax\n" /* set PE */ 370 " movl %%eax, %%cr0\n" 371 " ljmpl $40, $5f\n" /* back to protected mode */ 372 ".code32\n" 373 "5:\n" 374 " movl %%cr4, %%eax\n" 375 " btsl $5, %%eax\n" /* set PAE */ 376 " movl %%eax, %%cr4\n" 377 " movl $0xc0000080, %%ecx\n" /* EFER */ 378 " rdmsr\n" 379 " btsl $8, %%eax\n" /* set LME */ 380 " wrmsr\n" 381 " movl %%cr0, %%eax\n" 382 " btsl $31, %%eax\n" /* set PG */ 383 " movl %%eax, %%cr0\n" 384 " ljmpl $8, $6f\n" /* back to long mode */ 385 ".code64\n\t" 386 "6:\n" 387 " vmmcall\n" 388 ::: "rax", "rbx", "rcx", "rdx", "memory"); 389 } 390 391 static bool mode_switch_finished(struct test *test) 392 { 393 u64 cr0, cr4, efer; 394 395 cr0 = test->vmcb->save.cr0; 396 cr4 = test->vmcb->save.cr4; 397 efer = test->vmcb->save.efer; 398 399 /* Only expect VMMCALL intercepts */ 400 if (test->vmcb->control.exit_code != SVM_EXIT_VMMCALL) 401 return true; 402 403 /* Jump over VMMCALL instruction */ 404 test->vmcb->save.rip += 3; 405 406 /* Do sanity checks */ 407 switch (test->scratch) { 408 case 0: 409 /* Test should be in real mode now - check for this */ 410 if ((cr0 & 0x80000001) || /* CR0.PG, CR0.PE */ 411 (cr4 & 0x00000020) || /* CR4.PAE */ 412 (efer & 0x00000500)) /* EFER.LMA, EFER.LME */ 413 return true; 414 break; 415 case 2: 416 /* Test should be back in long-mode now - check for this */ 417 if (((cr0 & 0x80000001) != 0x80000001) || /* CR0.PG, CR0.PE */ 418 ((cr4 & 0x00000020) != 0x00000020) || /* CR4.PAE */ 419 ((efer & 0x00000500) != 0x00000500)) /* EFER.LMA, EFER.LME */ 420 return true; 421 break; 422 } 423 424 /* one step forward */ 425 test->scratch += 1; 426 427 return test->scratch == 2; 428 } 429 430 static bool check_mode_switch(struct test *test) 431 { 432 return test->scratch == 2; 433 } 434 435 static void prepare_asid_zero(struct test *test) 436 { 437 test->vmcb->control.asid = 0; 438 } 439 440 static void test_asid_zero(struct test *test) 441 { 442 asm volatile ("vmmcall\n\t"); 443 } 444 445 static bool check_asid_zero(struct test *test) 446 { 447 return test->vmcb->control.exit_code == SVM_EXIT_ERR; 448 } 449 450 static void sel_cr0_bug_prepare(struct test *test) 451 { 452 vmcb_ident(test->vmcb); 453 test->vmcb->control.intercept |= (1ULL << INTERCEPT_SELECTIVE_CR0); 454 } 455 456 static bool sel_cr0_bug_finished(struct test *test) 457 { 458 return true; 459 } 460 461 static void sel_cr0_bug_test(struct test *test) 462 { 463 unsigned long cr0; 464 465 /* read cr0, clear CD, and write back */ 466 cr0 = read_cr0(); 467 cr0 |= (1UL << 30); 468 write_cr0(cr0); 469 470 /* 471 * If we are here the test failed, not sure what to do now because we 472 * are not in guest-mode anymore so we can't trigger an intercept. 473 * Trigger a tripple-fault for now. 474 */ 475 printf("sel_cr0 test failed. Can not recover from this - exiting\n"); 476 exit(1); 477 } 478 479 static bool sel_cr0_bug_check(struct test *test) 480 { 481 return test->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE; 482 } 483 484 static void npt_nx_prepare(struct test *test) 485 { 486 487 u64 *pte; 488 489 vmcb_ident(test->vmcb); 490 pte = get_pte((u64)null_test); 491 492 *pte |= (1ULL << 63); 493 } 494 495 static bool npt_nx_check(struct test *test) 496 { 497 u64 *pte = get_pte((u64)null_test); 498 499 *pte &= ~(1ULL << 63); 500 501 test->vmcb->save.efer |= (1 << 11); 502 503 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 504 && (test->vmcb->control.exit_info_1 == 0x15); 505 } 506 507 static void npt_us_prepare(struct test *test) 508 { 509 u64 *pte; 510 511 vmcb_ident(test->vmcb); 512 pte = get_pte((u64)scratch_page); 513 514 *pte &= ~(1ULL << 2); 515 } 516 517 static void npt_us_test(struct test *test) 518 { 519 volatile u64 data; 520 521 data = *scratch_page; 522 } 523 524 static bool npt_us_check(struct test *test) 525 { 526 u64 *pte = get_pte((u64)scratch_page); 527 528 *pte |= (1ULL << 2); 529 530 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 531 && (test->vmcb->control.exit_info_1 == 0x05); 532 } 533 534 static void npt_rsvd_prepare(struct test *test) 535 { 536 537 vmcb_ident(test->vmcb); 538 539 pdpe[0] |= (1ULL << 8); 540 } 541 542 static bool npt_rsvd_check(struct test *test) 543 { 544 pdpe[0] &= ~(1ULL << 8); 545 546 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 547 && (test->vmcb->control.exit_info_1 == 0x0f); 548 } 549 550 static void npt_rw_prepare(struct test *test) 551 { 552 553 u64 *pte; 554 555 vmcb_ident(test->vmcb); 556 pte = get_pte(0x80000); 557 558 *pte &= ~(1ULL << 1); 559 } 560 561 static void npt_rw_test(struct test *test) 562 { 563 u64 *data = (void*)(0x80000); 564 565 *data = 0; 566 } 567 568 static bool npt_rw_check(struct test *test) 569 { 570 u64 *pte = get_pte(0x80000); 571 572 *pte |= (1ULL << 1); 573 574 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 575 && (test->vmcb->control.exit_info_1 == 0x07); 576 } 577 578 static void npt_pfwalk_prepare(struct test *test) 579 { 580 581 u64 *pte; 582 583 vmcb_ident(test->vmcb); 584 pte = get_pte(read_cr3()); 585 586 *pte &= ~(1ULL << 1); 587 } 588 589 static bool npt_pfwalk_check(struct test *test) 590 { 591 u64 *pte = get_pte(read_cr3()); 592 593 *pte |= (1ULL << 1); 594 595 return (test->vmcb->control.exit_code == SVM_EXIT_NPF) 596 && (test->vmcb->control.exit_info_1 == 0x7) 597 && (test->vmcb->control.exit_info_2 == read_cr3()); 598 } 599 600 static void latency_prepare(struct test *test) 601 { 602 default_prepare(test); 603 runs = LATENCY_RUNS; 604 latvmrun_min = latvmexit_min = -1ULL; 605 latvmrun_max = latvmexit_max = 0; 606 vmrun_sum = vmexit_sum = 0; 607 } 608 609 static void latency_test(struct test *test) 610 { 611 u64 cycles; 612 613 start: 614 tsc_end = rdtsc(); 615 616 cycles = tsc_end - tsc_start; 617 618 if (cycles > latvmrun_max) 619 latvmrun_max = cycles; 620 621 if (cycles < latvmrun_min) 622 latvmrun_min = cycles; 623 624 vmrun_sum += cycles; 625 626 tsc_start = rdtsc(); 627 628 asm volatile ("vmmcall" : : : "memory"); 629 goto start; 630 } 631 632 static bool latency_finished(struct test *test) 633 { 634 u64 cycles; 635 636 tsc_end = rdtsc(); 637 638 cycles = tsc_end - tsc_start; 639 640 if (cycles > latvmexit_max) 641 latvmexit_max = cycles; 642 643 if (cycles < latvmexit_min) 644 latvmexit_min = cycles; 645 646 vmexit_sum += cycles; 647 648 test->vmcb->save.rip += 3; 649 650 runs -= 1; 651 652 return runs == 0; 653 } 654 655 static bool latency_check(struct test *test) 656 { 657 printf(" Latency VMRUN : max: %d min: %d avg: %d\n", latvmrun_max, 658 latvmrun_min, vmrun_sum / LATENCY_RUNS); 659 printf(" Latency VMEXIT: max: %d min: %d avg: %d\n", latvmexit_max, 660 latvmexit_min, vmexit_sum / LATENCY_RUNS); 661 return true; 662 } 663 664 static struct test tests[] = { 665 { "null", default_supported, default_prepare, null_test, 666 default_finished, null_check }, 667 { "vmrun", default_supported, default_prepare, test_vmrun, 668 default_finished, check_vmrun }, 669 { "vmrun intercept check", default_supported, prepare_no_vmrun_int, 670 null_test, default_finished, check_no_vmrun_int }, 671 { "cr3 read intercept", default_supported, prepare_cr3_intercept, 672 test_cr3_intercept, default_finished, check_cr3_intercept }, 673 { "cr3 read nointercept", default_supported, default_prepare, 674 test_cr3_intercept, default_finished, check_cr3_nointercept }, 675 { "cr3 read intercept emulate", default_supported, 676 prepare_cr3_intercept_bypass, test_cr3_intercept_bypass, 677 default_finished, check_cr3_intercept }, 678 { "next_rip", next_rip_supported, prepare_next_rip, test_next_rip, 679 default_finished, check_next_rip }, 680 { "mode_switch", default_supported, prepare_mode_switch, test_mode_switch, 681 mode_switch_finished, check_mode_switch }, 682 { "asid_zero", default_supported, prepare_asid_zero, test_asid_zero, 683 default_finished, check_asid_zero }, 684 { "sel_cr0_bug", default_supported, sel_cr0_bug_prepare, sel_cr0_bug_test, 685 sel_cr0_bug_finished, sel_cr0_bug_check }, 686 { "npt_nx", npt_supported, npt_nx_prepare, null_test, 687 default_finished, npt_nx_check }, 688 { "npt_us", npt_supported, npt_us_prepare, npt_us_test, 689 default_finished, npt_us_check }, 690 { "npt_rsvd", npt_supported, npt_rsvd_prepare, null_test, 691 default_finished, npt_rsvd_check }, 692 { "npt_rw", npt_supported, npt_rw_prepare, npt_rw_test, 693 default_finished, npt_rw_check }, 694 { "npt_pfwalk", npt_supported, npt_pfwalk_prepare, null_test, 695 default_finished, npt_pfwalk_check }, 696 { "latency_run_exit", default_supported, latency_prepare, latency_test, 697 latency_finished, latency_check }, 698 }; 699 700 int main(int ac, char **av) 701 { 702 int i, nr, passed, done; 703 struct vmcb *vmcb; 704 705 setup_vm(); 706 smp_init(); 707 708 if (!(cpuid(0x80000001).c & 4)) { 709 printf("SVM not availble\n"); 710 return 0; 711 } 712 713 setup_svm(); 714 715 vmcb = alloc_page(); 716 717 nr = ARRAY_SIZE(tests); 718 passed = done = 0; 719 for (i = 0; i < nr; ++i) { 720 if (!tests[i].supported()) 721 continue; 722 done += 1; 723 passed += test_run(&tests[i], vmcb); 724 } 725 726 printf("\nSUMMARY: %d TESTS, %d FAILURES\n", done, (done - passed)); 727 return passed == done ? 0 : 1; 728 } 729