1 #include "svm.h" 2 #include "libcflat.h" 3 #include "processor.h" 4 #include "desc.h" 5 #include "msr.h" 6 #include "vm.h" 7 #include "smp.h" 8 #include "types.h" 9 #include "alloc_page.h" 10 #include "isr.h" 11 #include "apic.h" 12 #include "delay.h" 13 14 #define SVM_EXIT_MAX_DR_INTERCEPT 0x3f 15 16 static void *scratch_page; 17 18 #define LATENCY_RUNS 1000000 19 20 extern u16 cpu_online_count; 21 22 u64 tsc_start; 23 u64 tsc_end; 24 25 u64 vmrun_sum, vmexit_sum; 26 u64 vmsave_sum, vmload_sum; 27 u64 stgi_sum, clgi_sum; 28 u64 latvmrun_max; 29 u64 latvmrun_min; 30 u64 latvmexit_max; 31 u64 latvmexit_min; 32 u64 latvmload_max; 33 u64 latvmload_min; 34 u64 latvmsave_max; 35 u64 latvmsave_min; 36 u64 latstgi_max; 37 u64 latstgi_min; 38 u64 latclgi_max; 39 u64 latclgi_min; 40 u64 runs; 41 42 static void null_test(struct svm_test *test) 43 { 44 } 45 46 static bool null_check(struct svm_test *test) 47 { 48 return vmcb->control.exit_code == SVM_EXIT_VMMCALL; 49 } 50 51 static void prepare_no_vmrun_int(struct svm_test *test) 52 { 53 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMRUN); 54 } 55 56 static bool check_no_vmrun_int(struct svm_test *test) 57 { 58 return vmcb->control.exit_code == SVM_EXIT_ERR; 59 } 60 61 static void test_vmrun(struct svm_test *test) 62 { 63 asm volatile ("vmrun %0" : : "a"(virt_to_phys(vmcb))); 64 } 65 66 static bool check_vmrun(struct svm_test *test) 67 { 68 return vmcb->control.exit_code == SVM_EXIT_VMRUN; 69 } 70 71 static void prepare_rsm_intercept(struct svm_test *test) 72 { 73 default_prepare(test); 74 vmcb->control.intercept |= 1 << INTERCEPT_RSM; 75 vmcb->control.intercept_exceptions |= (1ULL << UD_VECTOR); 76 } 77 78 static void test_rsm_intercept(struct svm_test *test) 79 { 80 asm volatile ("rsm" : : : "memory"); 81 } 82 83 static bool check_rsm_intercept(struct svm_test *test) 84 { 85 return get_test_stage(test) == 2; 86 } 87 88 static bool finished_rsm_intercept(struct svm_test *test) 89 { 90 switch (get_test_stage(test)) { 91 case 0: 92 if (vmcb->control.exit_code != SVM_EXIT_RSM) { 93 report_fail("VMEXIT not due to rsm. Exit reason 0x%x", 94 vmcb->control.exit_code); 95 return true; 96 } 97 vmcb->control.intercept &= ~(1 << INTERCEPT_RSM); 98 inc_test_stage(test); 99 break; 100 101 case 1: 102 if (vmcb->control.exit_code != SVM_EXIT_EXCP_BASE + UD_VECTOR) { 103 report_fail("VMEXIT not due to #UD. Exit reason 0x%x", 104 vmcb->control.exit_code); 105 return true; 106 } 107 vmcb->save.rip += 2; 108 inc_test_stage(test); 109 break; 110 111 default: 112 return true; 113 } 114 return get_test_stage(test) == 2; 115 } 116 117 static void prepare_cr3_intercept(struct svm_test *test) 118 { 119 default_prepare(test); 120 vmcb->control.intercept_cr_read |= 1 << 3; 121 } 122 123 static void test_cr3_intercept(struct svm_test *test) 124 { 125 asm volatile ("mov %%cr3, %0" : "=r"(test->scratch) : : "memory"); 126 } 127 128 static bool check_cr3_intercept(struct svm_test *test) 129 { 130 return vmcb->control.exit_code == SVM_EXIT_READ_CR3; 131 } 132 133 static bool check_cr3_nointercept(struct svm_test *test) 134 { 135 return null_check(test) && test->scratch == read_cr3(); 136 } 137 138 static void corrupt_cr3_intercept_bypass(void *_test) 139 { 140 struct svm_test *test = _test; 141 extern volatile u32 mmio_insn; 142 143 while (!__sync_bool_compare_and_swap(&test->scratch, 1, 2)) 144 pause(); 145 pause(); 146 pause(); 147 pause(); 148 mmio_insn = 0x90d8200f; // mov %cr3, %rax; nop 149 } 150 151 static void prepare_cr3_intercept_bypass(struct svm_test *test) 152 { 153 default_prepare(test); 154 vmcb->control.intercept_cr_read |= 1 << 3; 155 on_cpu_async(1, corrupt_cr3_intercept_bypass, test); 156 } 157 158 static void test_cr3_intercept_bypass(struct svm_test *test) 159 { 160 ulong a = 0xa0000; 161 162 test->scratch = 1; 163 while (test->scratch != 2) 164 barrier(); 165 166 asm volatile ("mmio_insn: mov %0, (%0); nop" 167 : "+a"(a) : : "memory"); 168 test->scratch = a; 169 } 170 171 static void prepare_dr_intercept(struct svm_test *test) 172 { 173 default_prepare(test); 174 vmcb->control.intercept_dr_read = 0xff; 175 vmcb->control.intercept_dr_write = 0xff; 176 } 177 178 static void test_dr_intercept(struct svm_test *test) 179 { 180 unsigned int i, failcnt = 0; 181 182 /* Loop testing debug register reads */ 183 for (i = 0; i < 8; i++) { 184 185 switch (i) { 186 case 0: 187 asm volatile ("mov %%dr0, %0" : "=r"(test->scratch) : : "memory"); 188 break; 189 case 1: 190 asm volatile ("mov %%dr1, %0" : "=r"(test->scratch) : : "memory"); 191 break; 192 case 2: 193 asm volatile ("mov %%dr2, %0" : "=r"(test->scratch) : : "memory"); 194 break; 195 case 3: 196 asm volatile ("mov %%dr3, %0" : "=r"(test->scratch) : : "memory"); 197 break; 198 case 4: 199 asm volatile ("mov %%dr4, %0" : "=r"(test->scratch) : : "memory"); 200 break; 201 case 5: 202 asm volatile ("mov %%dr5, %0" : "=r"(test->scratch) : : "memory"); 203 break; 204 case 6: 205 asm volatile ("mov %%dr6, %0" : "=r"(test->scratch) : : "memory"); 206 break; 207 case 7: 208 asm volatile ("mov %%dr7, %0" : "=r"(test->scratch) : : "memory"); 209 break; 210 } 211 212 if (test->scratch != i) { 213 report_fail("dr%u read intercept", i); 214 failcnt++; 215 } 216 } 217 218 /* Loop testing debug register writes */ 219 for (i = 0; i < 8; i++) { 220 221 switch (i) { 222 case 0: 223 asm volatile ("mov %0, %%dr0" : : "r"(test->scratch) : "memory"); 224 break; 225 case 1: 226 asm volatile ("mov %0, %%dr1" : : "r"(test->scratch) : "memory"); 227 break; 228 case 2: 229 asm volatile ("mov %0, %%dr2" : : "r"(test->scratch) : "memory"); 230 break; 231 case 3: 232 asm volatile ("mov %0, %%dr3" : : "r"(test->scratch) : "memory"); 233 break; 234 case 4: 235 asm volatile ("mov %0, %%dr4" : : "r"(test->scratch) : "memory"); 236 break; 237 case 5: 238 asm volatile ("mov %0, %%dr5" : : "r"(test->scratch) : "memory"); 239 break; 240 case 6: 241 asm volatile ("mov %0, %%dr6" : : "r"(test->scratch) : "memory"); 242 break; 243 case 7: 244 asm volatile ("mov %0, %%dr7" : : "r"(test->scratch) : "memory"); 245 break; 246 } 247 248 if (test->scratch != i) { 249 report_fail("dr%u write intercept", i); 250 failcnt++; 251 } 252 } 253 254 test->scratch = failcnt; 255 } 256 257 static bool dr_intercept_finished(struct svm_test *test) 258 { 259 ulong n = (vmcb->control.exit_code - SVM_EXIT_READ_DR0); 260 261 /* Only expect DR intercepts */ 262 if (n > (SVM_EXIT_MAX_DR_INTERCEPT - SVM_EXIT_READ_DR0)) 263 return true; 264 265 /* 266 * Compute debug register number. 267 * Per Appendix C "SVM Intercept Exit Codes" of AMD64 Architecture 268 * Programmer's Manual Volume 2 - System Programming: 269 * http://support.amd.com/TechDocs/24593.pdf 270 * there are 16 VMEXIT codes each for DR read and write. 271 */ 272 test->scratch = (n % 16); 273 274 /* Jump over MOV instruction */ 275 vmcb->save.rip += 3; 276 277 return false; 278 } 279 280 static bool check_dr_intercept(struct svm_test *test) 281 { 282 return !test->scratch; 283 } 284 285 static bool next_rip_supported(void) 286 { 287 return this_cpu_has(X86_FEATURE_NRIPS); 288 } 289 290 static void prepare_next_rip(struct svm_test *test) 291 { 292 vmcb->control.intercept |= (1ULL << INTERCEPT_RDTSC); 293 } 294 295 296 static void test_next_rip(struct svm_test *test) 297 { 298 asm volatile ("rdtsc\n\t" 299 ".globl exp_next_rip\n\t" 300 "exp_next_rip:\n\t" ::: "eax", "edx"); 301 } 302 303 static bool check_next_rip(struct svm_test *test) 304 { 305 extern char exp_next_rip; 306 unsigned long address = (unsigned long)&exp_next_rip; 307 308 return address == vmcb->control.next_rip; 309 } 310 311 extern u8 *msr_bitmap; 312 313 static void prepare_msr_intercept(struct svm_test *test) 314 { 315 default_prepare(test); 316 vmcb->control.intercept |= (1ULL << INTERCEPT_MSR_PROT); 317 vmcb->control.intercept_exceptions |= (1ULL << GP_VECTOR); 318 memset(msr_bitmap, 0xff, MSR_BITMAP_SIZE); 319 } 320 321 static void test_msr_intercept(struct svm_test *test) 322 { 323 unsigned long msr_value = 0xef8056791234abcd; /* Arbitrary value */ 324 unsigned long msr_index; 325 326 for (msr_index = 0; msr_index <= 0xc0011fff; msr_index++) { 327 if (msr_index == 0xC0010131 /* MSR_SEV_STATUS */) { 328 /* 329 * Per section 15.34.10 "SEV_STATUS MSR" of AMD64 Architecture 330 * Programmer's Manual volume 2 - System Programming: 331 * http://support.amd.com/TechDocs/24593.pdf 332 * SEV_STATUS MSR (C001_0131) is a non-interceptable MSR. 333 */ 334 continue; 335 } 336 337 /* Skips gaps between supported MSR ranges */ 338 if (msr_index == 0x2000) 339 msr_index = 0xc0000000; 340 else if (msr_index == 0xc0002000) 341 msr_index = 0xc0010000; 342 343 test->scratch = -1; 344 345 rdmsr(msr_index); 346 347 /* Check that a read intercept occurred for MSR at msr_index */ 348 if (test->scratch != msr_index) 349 report_fail("MSR 0x%lx read intercept", msr_index); 350 351 /* 352 * Poor man approach to generate a value that 353 * seems arbitrary each time around the loop. 354 */ 355 msr_value += (msr_value << 1); 356 357 wrmsr(msr_index, msr_value); 358 359 /* Check that a write intercept occurred for MSR with msr_value */ 360 if (test->scratch != msr_value) 361 report_fail("MSR 0x%lx write intercept", msr_index); 362 } 363 364 test->scratch = -2; 365 } 366 367 static bool msr_intercept_finished(struct svm_test *test) 368 { 369 u32 exit_code = vmcb->control.exit_code; 370 u64 exit_info_1; 371 u8 *opcode; 372 373 if (exit_code == SVM_EXIT_MSR) { 374 exit_info_1 = vmcb->control.exit_info_1; 375 } else { 376 /* 377 * If #GP exception occurs instead, check that it was 378 * for RDMSR/WRMSR and set exit_info_1 accordingly. 379 */ 380 381 if (exit_code != (SVM_EXIT_EXCP_BASE + GP_VECTOR)) 382 return true; 383 384 opcode = (u8 *)vmcb->save.rip; 385 if (opcode[0] != 0x0f) 386 return true; 387 388 switch (opcode[1]) { 389 case 0x30: /* WRMSR */ 390 exit_info_1 = 1; 391 break; 392 case 0x32: /* RDMSR */ 393 exit_info_1 = 0; 394 break; 395 default: 396 return true; 397 } 398 399 /* 400 * Warn that #GP exception occured instead. 401 * RCX holds the MSR index. 402 */ 403 printf("%s 0x%lx #GP exception\n", 404 exit_info_1 ? "WRMSR" : "RDMSR", get_regs().rcx); 405 } 406 407 /* Jump over RDMSR/WRMSR instruction */ 408 vmcb->save.rip += 2; 409 410 /* 411 * Test whether the intercept was for RDMSR/WRMSR. 412 * For RDMSR, test->scratch is set to the MSR index; 413 * RCX holds the MSR index. 414 * For WRMSR, test->scratch is set to the MSR value; 415 * RDX holds the upper 32 bits of the MSR value, 416 * while RAX hold its lower 32 bits. 417 */ 418 if (exit_info_1) 419 test->scratch = 420 ((get_regs().rdx << 32) | (vmcb->save.rax & 0xffffffff)); 421 else 422 test->scratch = get_regs().rcx; 423 424 return false; 425 } 426 427 static bool check_msr_intercept(struct svm_test *test) 428 { 429 memset(msr_bitmap, 0, MSR_BITMAP_SIZE); 430 return (test->scratch == -2); 431 } 432 433 static void prepare_mode_switch(struct svm_test *test) 434 { 435 vmcb->control.intercept_exceptions |= (1ULL << GP_VECTOR) 436 | (1ULL << UD_VECTOR) 437 | (1ULL << DF_VECTOR) 438 | (1ULL << PF_VECTOR); 439 test->scratch = 0; 440 } 441 442 static void test_mode_switch(struct svm_test *test) 443 { 444 asm volatile(" cli\n" 445 " ljmp *1f\n" /* jump to 32-bit code segment */ 446 "1:\n" 447 " .long 2f\n" 448 " .long " xstr(KERNEL_CS32) "\n" 449 ".code32\n" 450 "2:\n" 451 " movl %%cr0, %%eax\n" 452 " btcl $31, %%eax\n" /* clear PG */ 453 " movl %%eax, %%cr0\n" 454 " movl $0xc0000080, %%ecx\n" /* EFER */ 455 " rdmsr\n" 456 " btcl $8, %%eax\n" /* clear LME */ 457 " wrmsr\n" 458 " movl %%cr4, %%eax\n" 459 " btcl $5, %%eax\n" /* clear PAE */ 460 " movl %%eax, %%cr4\n" 461 " movw %[ds16], %%ax\n" 462 " movw %%ax, %%ds\n" 463 " ljmpl %[cs16], $3f\n" /* jump to 16 bit protected-mode */ 464 ".code16\n" 465 "3:\n" 466 " movl %%cr0, %%eax\n" 467 " btcl $0, %%eax\n" /* clear PE */ 468 " movl %%eax, %%cr0\n" 469 " ljmpl $0, $4f\n" /* jump to real-mode */ 470 "4:\n" 471 " vmmcall\n" 472 " movl %%cr0, %%eax\n" 473 " btsl $0, %%eax\n" /* set PE */ 474 " movl %%eax, %%cr0\n" 475 " ljmpl %[cs32], $5f\n" /* back to protected mode */ 476 ".code32\n" 477 "5:\n" 478 " movl %%cr4, %%eax\n" 479 " btsl $5, %%eax\n" /* set PAE */ 480 " movl %%eax, %%cr4\n" 481 " movl $0xc0000080, %%ecx\n" /* EFER */ 482 " rdmsr\n" 483 " btsl $8, %%eax\n" /* set LME */ 484 " wrmsr\n" 485 " movl %%cr0, %%eax\n" 486 " btsl $31, %%eax\n" /* set PG */ 487 " movl %%eax, %%cr0\n" 488 " ljmpl %[cs64], $6f\n" /* back to long mode */ 489 ".code64\n\t" 490 "6:\n" 491 " vmmcall\n" 492 :: [cs16] "i"(KERNEL_CS16), [ds16] "i"(KERNEL_DS16), 493 [cs32] "i"(KERNEL_CS32), [cs64] "i"(KERNEL_CS64) 494 : "rax", "rbx", "rcx", "rdx", "memory"); 495 } 496 497 static bool mode_switch_finished(struct svm_test *test) 498 { 499 u64 cr0, cr4, efer; 500 501 cr0 = vmcb->save.cr0; 502 cr4 = vmcb->save.cr4; 503 efer = vmcb->save.efer; 504 505 /* Only expect VMMCALL intercepts */ 506 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) 507 return true; 508 509 /* Jump over VMMCALL instruction */ 510 vmcb->save.rip += 3; 511 512 /* Do sanity checks */ 513 switch (test->scratch) { 514 case 0: 515 /* Test should be in real mode now - check for this */ 516 if ((cr0 & 0x80000001) || /* CR0.PG, CR0.PE */ 517 (cr4 & 0x00000020) || /* CR4.PAE */ 518 (efer & 0x00000500)) /* EFER.LMA, EFER.LME */ 519 return true; 520 break; 521 case 2: 522 /* Test should be back in long-mode now - check for this */ 523 if (((cr0 & 0x80000001) != 0x80000001) || /* CR0.PG, CR0.PE */ 524 ((cr4 & 0x00000020) != 0x00000020) || /* CR4.PAE */ 525 ((efer & 0x00000500) != 0x00000500)) /* EFER.LMA, EFER.LME */ 526 return true; 527 break; 528 } 529 530 /* one step forward */ 531 test->scratch += 1; 532 533 return test->scratch == 2; 534 } 535 536 static bool check_mode_switch(struct svm_test *test) 537 { 538 return test->scratch == 2; 539 } 540 541 extern u8 *io_bitmap; 542 543 static void prepare_ioio(struct svm_test *test) 544 { 545 vmcb->control.intercept |= (1ULL << INTERCEPT_IOIO_PROT); 546 test->scratch = 0; 547 memset(io_bitmap, 0, 8192); 548 io_bitmap[8192] = 0xFF; 549 } 550 551 static void test_ioio(struct svm_test *test) 552 { 553 // stage 0, test IO pass 554 inb(0x5000); 555 outb(0x0, 0x5000); 556 if (get_test_stage(test) != 0) 557 goto fail; 558 559 // test IO width, in/out 560 io_bitmap[0] = 0xFF; 561 inc_test_stage(test); 562 inb(0x0); 563 if (get_test_stage(test) != 2) 564 goto fail; 565 566 outw(0x0, 0x0); 567 if (get_test_stage(test) != 3) 568 goto fail; 569 570 inl(0x0); 571 if (get_test_stage(test) != 4) 572 goto fail; 573 574 // test low/high IO port 575 io_bitmap[0x5000 / 8] = (1 << (0x5000 % 8)); 576 inb(0x5000); 577 if (get_test_stage(test) != 5) 578 goto fail; 579 580 io_bitmap[0x9000 / 8] = (1 << (0x9000 % 8)); 581 inw(0x9000); 582 if (get_test_stage(test) != 6) 583 goto fail; 584 585 // test partial pass 586 io_bitmap[0x5000 / 8] = (1 << (0x5000 % 8)); 587 inl(0x4FFF); 588 if (get_test_stage(test) != 7) 589 goto fail; 590 591 // test across pages 592 inc_test_stage(test); 593 inl(0x7FFF); 594 if (get_test_stage(test) != 8) 595 goto fail; 596 597 inc_test_stage(test); 598 io_bitmap[0x8000 / 8] = 1 << (0x8000 % 8); 599 inl(0x7FFF); 600 if (get_test_stage(test) != 10) 601 goto fail; 602 603 io_bitmap[0] = 0; 604 inl(0xFFFF); 605 if (get_test_stage(test) != 11) 606 goto fail; 607 608 io_bitmap[0] = 0xFF; 609 io_bitmap[8192] = 0; 610 inl(0xFFFF); 611 inc_test_stage(test); 612 if (get_test_stage(test) != 12) 613 goto fail; 614 615 return; 616 617 fail: 618 report_fail("stage %d", get_test_stage(test)); 619 test->scratch = -1; 620 } 621 622 static bool ioio_finished(struct svm_test *test) 623 { 624 unsigned port, size; 625 626 /* Only expect IOIO intercepts */ 627 if (vmcb->control.exit_code == SVM_EXIT_VMMCALL) 628 return true; 629 630 if (vmcb->control.exit_code != SVM_EXIT_IOIO) 631 return true; 632 633 /* one step forward */ 634 test->scratch += 1; 635 636 port = vmcb->control.exit_info_1 >> 16; 637 size = (vmcb->control.exit_info_1 >> SVM_IOIO_SIZE_SHIFT) & 7; 638 639 while (size--) { 640 io_bitmap[port / 8] &= ~(1 << (port & 7)); 641 port++; 642 } 643 644 return false; 645 } 646 647 static bool check_ioio(struct svm_test *test) 648 { 649 memset(io_bitmap, 0, 8193); 650 return test->scratch != -1; 651 } 652 653 static void prepare_asid_zero(struct svm_test *test) 654 { 655 vmcb->control.asid = 0; 656 } 657 658 static void test_asid_zero(struct svm_test *test) 659 { 660 asm volatile ("vmmcall\n\t"); 661 } 662 663 static bool check_asid_zero(struct svm_test *test) 664 { 665 return vmcb->control.exit_code == SVM_EXIT_ERR; 666 } 667 668 static void sel_cr0_bug_prepare(struct svm_test *test) 669 { 670 vmcb->control.intercept |= (1ULL << INTERCEPT_SELECTIVE_CR0); 671 } 672 673 static bool sel_cr0_bug_finished(struct svm_test *test) 674 { 675 return true; 676 } 677 678 static void sel_cr0_bug_test(struct svm_test *test) 679 { 680 unsigned long cr0; 681 682 /* read cr0, clear CD, and write back */ 683 cr0 = read_cr0(); 684 cr0 |= (1UL << 30); 685 write_cr0(cr0); 686 687 /* 688 * If we are here the test failed, not sure what to do now because we 689 * are not in guest-mode anymore so we can't trigger an intercept. 690 * Trigger a tripple-fault for now. 691 */ 692 report_fail("sel_cr0 test. Can not recover from this - exiting"); 693 exit(report_summary()); 694 } 695 696 static bool sel_cr0_bug_check(struct svm_test *test) 697 { 698 return vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE; 699 } 700 701 static void npt_nx_prepare(struct svm_test *test) 702 { 703 u64 *pte; 704 705 test->scratch = rdmsr(MSR_EFER); 706 wrmsr(MSR_EFER, test->scratch | EFER_NX); 707 708 /* Clear the guest's EFER.NX, it should not affect NPT behavior. */ 709 vmcb->save.efer &= ~EFER_NX; 710 711 pte = npt_get_pte((u64)null_test); 712 713 *pte |= PT64_NX_MASK; 714 } 715 716 static bool npt_nx_check(struct svm_test *test) 717 { 718 u64 *pte = npt_get_pte((u64)null_test); 719 720 wrmsr(MSR_EFER, test->scratch); 721 722 *pte &= ~PT64_NX_MASK; 723 724 return (vmcb->control.exit_code == SVM_EXIT_NPF) 725 && (vmcb->control.exit_info_1 == 0x100000015ULL); 726 } 727 728 static void npt_np_prepare(struct svm_test *test) 729 { 730 u64 *pte; 731 732 scratch_page = alloc_page(); 733 pte = npt_get_pte((u64)scratch_page); 734 735 *pte &= ~1ULL; 736 } 737 738 static void npt_np_test(struct svm_test *test) 739 { 740 (void) *(volatile u64 *)scratch_page; 741 } 742 743 static bool npt_np_check(struct svm_test *test) 744 { 745 u64 *pte = npt_get_pte((u64)scratch_page); 746 747 *pte |= 1ULL; 748 749 return (vmcb->control.exit_code == SVM_EXIT_NPF) 750 && (vmcb->control.exit_info_1 == 0x100000004ULL); 751 } 752 753 static void npt_us_prepare(struct svm_test *test) 754 { 755 u64 *pte; 756 757 scratch_page = alloc_page(); 758 pte = npt_get_pte((u64)scratch_page); 759 760 *pte &= ~(1ULL << 2); 761 } 762 763 static void npt_us_test(struct svm_test *test) 764 { 765 (void) *(volatile u64 *)scratch_page; 766 } 767 768 static bool npt_us_check(struct svm_test *test) 769 { 770 u64 *pte = npt_get_pte((u64)scratch_page); 771 772 *pte |= (1ULL << 2); 773 774 return (vmcb->control.exit_code == SVM_EXIT_NPF) 775 && (vmcb->control.exit_info_1 == 0x100000005ULL); 776 } 777 778 static void npt_rw_prepare(struct svm_test *test) 779 { 780 781 u64 *pte; 782 783 pte = npt_get_pte(0x80000); 784 785 *pte &= ~(1ULL << 1); 786 } 787 788 static void npt_rw_test(struct svm_test *test) 789 { 790 u64 *data = (void*)(0x80000); 791 792 *data = 0; 793 } 794 795 static bool npt_rw_check(struct svm_test *test) 796 { 797 u64 *pte = npt_get_pte(0x80000); 798 799 *pte |= (1ULL << 1); 800 801 return (vmcb->control.exit_code == SVM_EXIT_NPF) 802 && (vmcb->control.exit_info_1 == 0x100000007ULL); 803 } 804 805 static void npt_rw_pfwalk_prepare(struct svm_test *test) 806 { 807 808 u64 *pte; 809 810 pte = npt_get_pte(read_cr3()); 811 812 *pte &= ~(1ULL << 1); 813 } 814 815 static bool npt_rw_pfwalk_check(struct svm_test *test) 816 { 817 u64 *pte = npt_get_pte(read_cr3()); 818 819 *pte |= (1ULL << 1); 820 821 return (vmcb->control.exit_code == SVM_EXIT_NPF) 822 && (vmcb->control.exit_info_1 == 0x200000007ULL) 823 && (vmcb->control.exit_info_2 == read_cr3()); 824 } 825 826 static void npt_l1mmio_prepare(struct svm_test *test) 827 { 828 } 829 830 u32 nested_apic_version1; 831 u32 nested_apic_version2; 832 833 static void npt_l1mmio_test(struct svm_test *test) 834 { 835 volatile u32 *data = (volatile void*)(0xfee00030UL); 836 837 nested_apic_version1 = *data; 838 nested_apic_version2 = *data; 839 } 840 841 static bool npt_l1mmio_check(struct svm_test *test) 842 { 843 volatile u32 *data = (volatile void*)(0xfee00030); 844 u32 lvr = *data; 845 846 return nested_apic_version1 == lvr && nested_apic_version2 == lvr; 847 } 848 849 static void npt_rw_l1mmio_prepare(struct svm_test *test) 850 { 851 852 u64 *pte; 853 854 pte = npt_get_pte(0xfee00080); 855 856 *pte &= ~(1ULL << 1); 857 } 858 859 static void npt_rw_l1mmio_test(struct svm_test *test) 860 { 861 volatile u32 *data = (volatile void*)(0xfee00080); 862 863 *data = *data; 864 } 865 866 static bool npt_rw_l1mmio_check(struct svm_test *test) 867 { 868 u64 *pte = npt_get_pte(0xfee00080); 869 870 *pte |= (1ULL << 1); 871 872 return (vmcb->control.exit_code == SVM_EXIT_NPF) 873 && (vmcb->control.exit_info_1 == 0x100000007ULL); 874 } 875 876 #define TSC_ADJUST_VALUE (1ll << 32) 877 #define TSC_OFFSET_VALUE (~0ull << 48) 878 static bool ok; 879 880 static bool tsc_adjust_supported(void) 881 { 882 return this_cpu_has(X86_FEATURE_TSC_ADJUST); 883 } 884 885 static void tsc_adjust_prepare(struct svm_test *test) 886 { 887 default_prepare(test); 888 vmcb->control.tsc_offset = TSC_OFFSET_VALUE; 889 890 wrmsr(MSR_IA32_TSC_ADJUST, -TSC_ADJUST_VALUE); 891 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST); 892 ok = adjust == -TSC_ADJUST_VALUE; 893 } 894 895 static void tsc_adjust_test(struct svm_test *test) 896 { 897 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST); 898 ok &= adjust == -TSC_ADJUST_VALUE; 899 900 uint64_t l1_tsc = rdtsc() - TSC_OFFSET_VALUE; 901 wrmsr(MSR_IA32_TSC, l1_tsc - TSC_ADJUST_VALUE); 902 903 adjust = rdmsr(MSR_IA32_TSC_ADJUST); 904 ok &= adjust <= -2 * TSC_ADJUST_VALUE; 905 906 uint64_t l1_tsc_end = rdtsc() - TSC_OFFSET_VALUE; 907 ok &= (l1_tsc_end + TSC_ADJUST_VALUE - l1_tsc) < TSC_ADJUST_VALUE; 908 909 uint64_t l1_tsc_msr = rdmsr(MSR_IA32_TSC) - TSC_OFFSET_VALUE; 910 ok &= (l1_tsc_msr + TSC_ADJUST_VALUE - l1_tsc) < TSC_ADJUST_VALUE; 911 } 912 913 static bool tsc_adjust_check(struct svm_test *test) 914 { 915 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST); 916 917 wrmsr(MSR_IA32_TSC_ADJUST, 0); 918 return ok && adjust <= -2 * TSC_ADJUST_VALUE; 919 } 920 921 922 static u64 guest_tsc_delay_value; 923 /* number of bits to shift tsc right for stable result */ 924 #define TSC_SHIFT 24 925 #define TSC_SCALE_ITERATIONS 10 926 927 static void svm_tsc_scale_guest(struct svm_test *test) 928 { 929 u64 start_tsc = rdtsc(); 930 931 while (rdtsc() - start_tsc < guest_tsc_delay_value) 932 cpu_relax(); 933 } 934 935 static void svm_tsc_scale_run_testcase(u64 duration, 936 double tsc_scale, u64 tsc_offset) 937 { 938 u64 start_tsc, actual_duration; 939 940 guest_tsc_delay_value = (duration << TSC_SHIFT) * tsc_scale; 941 942 test_set_guest(svm_tsc_scale_guest); 943 vmcb->control.tsc_offset = tsc_offset; 944 wrmsr(MSR_AMD64_TSC_RATIO, (u64)(tsc_scale * (1ULL << 32))); 945 946 start_tsc = rdtsc(); 947 948 if (svm_vmrun() != SVM_EXIT_VMMCALL) 949 report_fail("unexpected vm exit code 0x%x", vmcb->control.exit_code); 950 951 actual_duration = (rdtsc() - start_tsc) >> TSC_SHIFT; 952 953 report(duration == actual_duration, "tsc delay (expected: %lu, actual: %lu)", 954 duration, actual_duration); 955 } 956 957 static void svm_tsc_scale_test(void) 958 { 959 int i; 960 961 if (!tsc_scale_supported()) { 962 report_skip("TSC scale not supported in the guest"); 963 return; 964 } 965 966 report(rdmsr(MSR_AMD64_TSC_RATIO) == TSC_RATIO_DEFAULT, 967 "initial TSC scale ratio"); 968 969 for (i = 0 ; i < TSC_SCALE_ITERATIONS; i++) { 970 971 double tsc_scale = (double)(rdrand() % 100 + 1) / 10; 972 int duration = rdrand() % 50 + 1; 973 u64 tsc_offset = rdrand(); 974 975 report_info("duration=%d, tsc_scale=%d, tsc_offset=%ld", 976 duration, (int)(tsc_scale * 100), tsc_offset); 977 978 svm_tsc_scale_run_testcase(duration, tsc_scale, tsc_offset); 979 } 980 981 svm_tsc_scale_run_testcase(50, 255, rdrand()); 982 svm_tsc_scale_run_testcase(50, 0.0001, rdrand()); 983 } 984 985 static void latency_prepare(struct svm_test *test) 986 { 987 default_prepare(test); 988 runs = LATENCY_RUNS; 989 latvmrun_min = latvmexit_min = -1ULL; 990 latvmrun_max = latvmexit_max = 0; 991 vmrun_sum = vmexit_sum = 0; 992 tsc_start = rdtsc(); 993 } 994 995 static void latency_test(struct svm_test *test) 996 { 997 u64 cycles; 998 999 start: 1000 tsc_end = rdtsc(); 1001 1002 cycles = tsc_end - tsc_start; 1003 1004 if (cycles > latvmrun_max) 1005 latvmrun_max = cycles; 1006 1007 if (cycles < latvmrun_min) 1008 latvmrun_min = cycles; 1009 1010 vmrun_sum += cycles; 1011 1012 tsc_start = rdtsc(); 1013 1014 asm volatile ("vmmcall" : : : "memory"); 1015 goto start; 1016 } 1017 1018 static bool latency_finished(struct svm_test *test) 1019 { 1020 u64 cycles; 1021 1022 tsc_end = rdtsc(); 1023 1024 cycles = tsc_end - tsc_start; 1025 1026 if (cycles > latvmexit_max) 1027 latvmexit_max = cycles; 1028 1029 if (cycles < latvmexit_min) 1030 latvmexit_min = cycles; 1031 1032 vmexit_sum += cycles; 1033 1034 vmcb->save.rip += 3; 1035 1036 runs -= 1; 1037 1038 tsc_end = rdtsc(); 1039 1040 return runs == 0; 1041 } 1042 1043 static bool latency_finished_clean(struct svm_test *test) 1044 { 1045 vmcb->control.clean = VMCB_CLEAN_ALL; 1046 return latency_finished(test); 1047 } 1048 1049 static bool latency_check(struct svm_test *test) 1050 { 1051 printf(" Latency VMRUN : max: %ld min: %ld avg: %ld\n", latvmrun_max, 1052 latvmrun_min, vmrun_sum / LATENCY_RUNS); 1053 printf(" Latency VMEXIT: max: %ld min: %ld avg: %ld\n", latvmexit_max, 1054 latvmexit_min, vmexit_sum / LATENCY_RUNS); 1055 return true; 1056 } 1057 1058 static void lat_svm_insn_prepare(struct svm_test *test) 1059 { 1060 default_prepare(test); 1061 runs = LATENCY_RUNS; 1062 latvmload_min = latvmsave_min = latstgi_min = latclgi_min = -1ULL; 1063 latvmload_max = latvmsave_max = latstgi_max = latclgi_max = 0; 1064 vmload_sum = vmsave_sum = stgi_sum = clgi_sum; 1065 } 1066 1067 static bool lat_svm_insn_finished(struct svm_test *test) 1068 { 1069 u64 vmcb_phys = virt_to_phys(vmcb); 1070 u64 cycles; 1071 1072 for ( ; runs != 0; runs--) { 1073 tsc_start = rdtsc(); 1074 asm volatile("vmload %0\n\t" : : "a"(vmcb_phys) : "memory"); 1075 cycles = rdtsc() - tsc_start; 1076 if (cycles > latvmload_max) 1077 latvmload_max = cycles; 1078 if (cycles < latvmload_min) 1079 latvmload_min = cycles; 1080 vmload_sum += cycles; 1081 1082 tsc_start = rdtsc(); 1083 asm volatile("vmsave %0\n\t" : : "a"(vmcb_phys) : "memory"); 1084 cycles = rdtsc() - tsc_start; 1085 if (cycles > latvmsave_max) 1086 latvmsave_max = cycles; 1087 if (cycles < latvmsave_min) 1088 latvmsave_min = cycles; 1089 vmsave_sum += cycles; 1090 1091 tsc_start = rdtsc(); 1092 asm volatile("stgi\n\t"); 1093 cycles = rdtsc() - tsc_start; 1094 if (cycles > latstgi_max) 1095 latstgi_max = cycles; 1096 if (cycles < latstgi_min) 1097 latstgi_min = cycles; 1098 stgi_sum += cycles; 1099 1100 tsc_start = rdtsc(); 1101 asm volatile("clgi\n\t"); 1102 cycles = rdtsc() - tsc_start; 1103 if (cycles > latclgi_max) 1104 latclgi_max = cycles; 1105 if (cycles < latclgi_min) 1106 latclgi_min = cycles; 1107 clgi_sum += cycles; 1108 } 1109 1110 tsc_end = rdtsc(); 1111 1112 return true; 1113 } 1114 1115 static bool lat_svm_insn_check(struct svm_test *test) 1116 { 1117 printf(" Latency VMLOAD: max: %ld min: %ld avg: %ld\n", latvmload_max, 1118 latvmload_min, vmload_sum / LATENCY_RUNS); 1119 printf(" Latency VMSAVE: max: %ld min: %ld avg: %ld\n", latvmsave_max, 1120 latvmsave_min, vmsave_sum / LATENCY_RUNS); 1121 printf(" Latency STGI: max: %ld min: %ld avg: %ld\n", latstgi_max, 1122 latstgi_min, stgi_sum / LATENCY_RUNS); 1123 printf(" Latency CLGI: max: %ld min: %ld avg: %ld\n", latclgi_max, 1124 latclgi_min, clgi_sum / LATENCY_RUNS); 1125 return true; 1126 } 1127 1128 bool pending_event_ipi_fired; 1129 bool pending_event_guest_run; 1130 1131 static void pending_event_ipi_isr(isr_regs_t *regs) 1132 { 1133 pending_event_ipi_fired = true; 1134 eoi(); 1135 } 1136 1137 static void pending_event_prepare(struct svm_test *test) 1138 { 1139 int ipi_vector = 0xf1; 1140 1141 default_prepare(test); 1142 1143 pending_event_ipi_fired = false; 1144 1145 handle_irq(ipi_vector, pending_event_ipi_isr); 1146 1147 pending_event_guest_run = false; 1148 1149 vmcb->control.intercept |= (1ULL << INTERCEPT_INTR); 1150 vmcb->control.int_ctl |= V_INTR_MASKING_MASK; 1151 1152 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | 1153 APIC_DM_FIXED | ipi_vector, 0); 1154 1155 set_test_stage(test, 0); 1156 } 1157 1158 static void pending_event_test(struct svm_test *test) 1159 { 1160 pending_event_guest_run = true; 1161 } 1162 1163 static bool pending_event_finished(struct svm_test *test) 1164 { 1165 switch (get_test_stage(test)) { 1166 case 0: 1167 if (vmcb->control.exit_code != SVM_EXIT_INTR) { 1168 report_fail("VMEXIT not due to pending interrupt. Exit reason 0x%x", 1169 vmcb->control.exit_code); 1170 return true; 1171 } 1172 1173 vmcb->control.intercept &= ~(1ULL << INTERCEPT_INTR); 1174 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; 1175 1176 if (pending_event_guest_run) { 1177 report_fail("Guest ran before host received IPI\n"); 1178 return true; 1179 } 1180 1181 irq_enable(); 1182 asm volatile ("nop"); 1183 irq_disable(); 1184 1185 if (!pending_event_ipi_fired) { 1186 report_fail("Pending interrupt not dispatched after IRQ enabled\n"); 1187 return true; 1188 } 1189 break; 1190 1191 case 1: 1192 if (!pending_event_guest_run) { 1193 report_fail("Guest did not resume when no interrupt\n"); 1194 return true; 1195 } 1196 break; 1197 } 1198 1199 inc_test_stage(test); 1200 1201 return get_test_stage(test) == 2; 1202 } 1203 1204 static bool pending_event_check(struct svm_test *test) 1205 { 1206 return get_test_stage(test) == 2; 1207 } 1208 1209 static void pending_event_cli_prepare(struct svm_test *test) 1210 { 1211 default_prepare(test); 1212 1213 pending_event_ipi_fired = false; 1214 1215 handle_irq(0xf1, pending_event_ipi_isr); 1216 1217 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | 1218 APIC_DM_FIXED | 0xf1, 0); 1219 1220 set_test_stage(test, 0); 1221 } 1222 1223 static void pending_event_cli_prepare_gif_clear(struct svm_test *test) 1224 { 1225 asm("cli"); 1226 } 1227 1228 static void pending_event_cli_test(struct svm_test *test) 1229 { 1230 if (pending_event_ipi_fired == true) { 1231 set_test_stage(test, -1); 1232 report_fail("Interrupt preceeded guest"); 1233 vmmcall(); 1234 } 1235 1236 /* VINTR_MASKING is zero. This should cause the IPI to fire. */ 1237 irq_enable(); 1238 asm volatile ("nop"); 1239 irq_disable(); 1240 1241 if (pending_event_ipi_fired != true) { 1242 set_test_stage(test, -1); 1243 report_fail("Interrupt not triggered by guest"); 1244 } 1245 1246 vmmcall(); 1247 1248 /* 1249 * Now VINTR_MASKING=1, but no interrupt is pending so 1250 * the VINTR interception should be clear in VMCB02. Check 1251 * that L0 did not leave a stale VINTR in the VMCB. 1252 */ 1253 irq_enable(); 1254 asm volatile ("nop"); 1255 irq_disable(); 1256 } 1257 1258 static bool pending_event_cli_finished(struct svm_test *test) 1259 { 1260 if ( vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1261 report_fail("VM_EXIT return to host is not EXIT_VMMCALL exit reason 0x%x", 1262 vmcb->control.exit_code); 1263 return true; 1264 } 1265 1266 switch (get_test_stage(test)) { 1267 case 0: 1268 vmcb->save.rip += 3; 1269 1270 pending_event_ipi_fired = false; 1271 1272 vmcb->control.int_ctl |= V_INTR_MASKING_MASK; 1273 1274 /* Now entering again with VINTR_MASKING=1. */ 1275 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | 1276 APIC_DM_FIXED | 0xf1, 0); 1277 1278 break; 1279 1280 case 1: 1281 if (pending_event_ipi_fired == true) { 1282 report_fail("Interrupt triggered by guest"); 1283 return true; 1284 } 1285 1286 irq_enable(); 1287 asm volatile ("nop"); 1288 irq_disable(); 1289 1290 if (pending_event_ipi_fired != true) { 1291 report_fail("Interrupt not triggered by host"); 1292 return true; 1293 } 1294 1295 break; 1296 1297 default: 1298 return true; 1299 } 1300 1301 inc_test_stage(test); 1302 1303 return get_test_stage(test) == 2; 1304 } 1305 1306 static bool pending_event_cli_check(struct svm_test *test) 1307 { 1308 return get_test_stage(test) == 2; 1309 } 1310 1311 #define TIMER_VECTOR 222 1312 1313 static volatile bool timer_fired; 1314 1315 static void timer_isr(isr_regs_t *regs) 1316 { 1317 timer_fired = true; 1318 apic_write(APIC_EOI, 0); 1319 } 1320 1321 static void interrupt_prepare(struct svm_test *test) 1322 { 1323 default_prepare(test); 1324 handle_irq(TIMER_VECTOR, timer_isr); 1325 timer_fired = false; 1326 set_test_stage(test, 0); 1327 } 1328 1329 static void interrupt_test(struct svm_test *test) 1330 { 1331 long long start, loops; 1332 1333 apic_write(APIC_LVTT, TIMER_VECTOR); 1334 irq_enable(); 1335 apic_write(APIC_TMICT, 1); //Timer Initial Count Register 0x380 one-shot 1336 for (loops = 0; loops < 10000000 && !timer_fired; loops++) 1337 asm volatile ("nop"); 1338 1339 report(timer_fired, "direct interrupt while running guest"); 1340 1341 if (!timer_fired) { 1342 set_test_stage(test, -1); 1343 vmmcall(); 1344 } 1345 1346 apic_write(APIC_TMICT, 0); 1347 irq_disable(); 1348 vmmcall(); 1349 1350 timer_fired = false; 1351 apic_write(APIC_TMICT, 1); 1352 for (loops = 0; loops < 10000000 && !timer_fired; loops++) 1353 asm volatile ("nop"); 1354 1355 report(timer_fired, "intercepted interrupt while running guest"); 1356 1357 if (!timer_fired) { 1358 set_test_stage(test, -1); 1359 vmmcall(); 1360 } 1361 1362 irq_enable(); 1363 apic_write(APIC_TMICT, 0); 1364 irq_disable(); 1365 1366 timer_fired = false; 1367 start = rdtsc(); 1368 apic_write(APIC_TMICT, 1000000); 1369 safe_halt(); 1370 1371 report(rdtsc() - start > 10000 && timer_fired, 1372 "direct interrupt + hlt"); 1373 1374 if (!timer_fired) { 1375 set_test_stage(test, -1); 1376 vmmcall(); 1377 } 1378 1379 apic_write(APIC_TMICT, 0); 1380 irq_disable(); 1381 vmmcall(); 1382 1383 timer_fired = false; 1384 start = rdtsc(); 1385 apic_write(APIC_TMICT, 1000000); 1386 asm volatile ("hlt"); 1387 1388 report(rdtsc() - start > 10000 && timer_fired, 1389 "intercepted interrupt + hlt"); 1390 1391 if (!timer_fired) { 1392 set_test_stage(test, -1); 1393 vmmcall(); 1394 } 1395 1396 apic_write(APIC_TMICT, 0); 1397 irq_disable(); 1398 } 1399 1400 static bool interrupt_finished(struct svm_test *test) 1401 { 1402 switch (get_test_stage(test)) { 1403 case 0: 1404 case 2: 1405 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1406 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1407 vmcb->control.exit_code); 1408 return true; 1409 } 1410 vmcb->save.rip += 3; 1411 1412 vmcb->control.intercept |= (1ULL << INTERCEPT_INTR); 1413 vmcb->control.int_ctl |= V_INTR_MASKING_MASK; 1414 break; 1415 1416 case 1: 1417 case 3: 1418 if (vmcb->control.exit_code != SVM_EXIT_INTR) { 1419 report_fail("VMEXIT not due to intr intercept. Exit reason 0x%x", 1420 vmcb->control.exit_code); 1421 return true; 1422 } 1423 1424 irq_enable(); 1425 asm volatile ("nop"); 1426 irq_disable(); 1427 1428 vmcb->control.intercept &= ~(1ULL << INTERCEPT_INTR); 1429 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; 1430 break; 1431 1432 case 4: 1433 break; 1434 1435 default: 1436 return true; 1437 } 1438 1439 inc_test_stage(test); 1440 1441 return get_test_stage(test) == 5; 1442 } 1443 1444 static bool interrupt_check(struct svm_test *test) 1445 { 1446 return get_test_stage(test) == 5; 1447 } 1448 1449 static volatile bool nmi_fired; 1450 1451 static void nmi_handler(struct ex_regs *regs) 1452 { 1453 nmi_fired = true; 1454 } 1455 1456 static void nmi_prepare(struct svm_test *test) 1457 { 1458 default_prepare(test); 1459 nmi_fired = false; 1460 handle_exception(NMI_VECTOR, nmi_handler); 1461 set_test_stage(test, 0); 1462 } 1463 1464 static void nmi_test(struct svm_test *test) 1465 { 1466 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, 0); 1467 1468 report(nmi_fired, "direct NMI while running guest"); 1469 1470 if (!nmi_fired) 1471 set_test_stage(test, -1); 1472 1473 vmmcall(); 1474 1475 nmi_fired = false; 1476 1477 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, 0); 1478 1479 if (!nmi_fired) { 1480 report(nmi_fired, "intercepted pending NMI not dispatched"); 1481 set_test_stage(test, -1); 1482 } 1483 1484 } 1485 1486 static bool nmi_finished(struct svm_test *test) 1487 { 1488 switch (get_test_stage(test)) { 1489 case 0: 1490 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1491 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1492 vmcb->control.exit_code); 1493 return true; 1494 } 1495 vmcb->save.rip += 3; 1496 1497 vmcb->control.intercept |= (1ULL << INTERCEPT_NMI); 1498 break; 1499 1500 case 1: 1501 if (vmcb->control.exit_code != SVM_EXIT_NMI) { 1502 report_fail("VMEXIT not due to NMI intercept. Exit reason 0x%x", 1503 vmcb->control.exit_code); 1504 return true; 1505 } 1506 1507 report_pass("NMI intercept while running guest"); 1508 break; 1509 1510 case 2: 1511 break; 1512 1513 default: 1514 return true; 1515 } 1516 1517 inc_test_stage(test); 1518 1519 return get_test_stage(test) == 3; 1520 } 1521 1522 static bool nmi_check(struct svm_test *test) 1523 { 1524 return get_test_stage(test) == 3; 1525 } 1526 1527 #define NMI_DELAY 100000000ULL 1528 1529 static void nmi_message_thread(void *_test) 1530 { 1531 struct svm_test *test = _test; 1532 1533 while (get_test_stage(test) != 1) 1534 pause(); 1535 1536 delay(NMI_DELAY); 1537 1538 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]); 1539 1540 while (get_test_stage(test) != 2) 1541 pause(); 1542 1543 delay(NMI_DELAY); 1544 1545 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]); 1546 } 1547 1548 static void nmi_hlt_test(struct svm_test *test) 1549 { 1550 long long start; 1551 1552 on_cpu_async(1, nmi_message_thread, test); 1553 1554 start = rdtsc(); 1555 1556 set_test_stage(test, 1); 1557 1558 asm volatile ("hlt"); 1559 1560 report((rdtsc() - start > NMI_DELAY) && nmi_fired, 1561 "direct NMI + hlt"); 1562 1563 if (!nmi_fired) 1564 set_test_stage(test, -1); 1565 1566 nmi_fired = false; 1567 1568 vmmcall(); 1569 1570 start = rdtsc(); 1571 1572 set_test_stage(test, 2); 1573 1574 asm volatile ("hlt"); 1575 1576 report((rdtsc() - start > NMI_DELAY) && nmi_fired, 1577 "intercepted NMI + hlt"); 1578 1579 if (!nmi_fired) { 1580 report(nmi_fired, "intercepted pending NMI not dispatched"); 1581 set_test_stage(test, -1); 1582 vmmcall(); 1583 } 1584 1585 set_test_stage(test, 3); 1586 } 1587 1588 static bool nmi_hlt_finished(struct svm_test *test) 1589 { 1590 switch (get_test_stage(test)) { 1591 case 1: 1592 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1593 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1594 vmcb->control.exit_code); 1595 return true; 1596 } 1597 vmcb->save.rip += 3; 1598 1599 vmcb->control.intercept |= (1ULL << INTERCEPT_NMI); 1600 break; 1601 1602 case 2: 1603 if (vmcb->control.exit_code != SVM_EXIT_NMI) { 1604 report_fail("VMEXIT not due to NMI intercept. Exit reason 0x%x", 1605 vmcb->control.exit_code); 1606 return true; 1607 } 1608 1609 report_pass("NMI intercept while running guest"); 1610 break; 1611 1612 case 3: 1613 break; 1614 1615 default: 1616 return true; 1617 } 1618 1619 return get_test_stage(test) == 3; 1620 } 1621 1622 static bool nmi_hlt_check(struct svm_test *test) 1623 { 1624 return get_test_stage(test) == 3; 1625 } 1626 1627 static volatile int count_exc = 0; 1628 1629 static void my_isr(struct ex_regs *r) 1630 { 1631 count_exc++; 1632 } 1633 1634 static void exc_inject_prepare(struct svm_test *test) 1635 { 1636 default_prepare(test); 1637 handle_exception(DE_VECTOR, my_isr); 1638 handle_exception(NMI_VECTOR, my_isr); 1639 } 1640 1641 1642 static void exc_inject_test(struct svm_test *test) 1643 { 1644 asm volatile ("vmmcall\n\tvmmcall\n\t"); 1645 } 1646 1647 static bool exc_inject_finished(struct svm_test *test) 1648 { 1649 switch (get_test_stage(test)) { 1650 case 0: 1651 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1652 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1653 vmcb->control.exit_code); 1654 return true; 1655 } 1656 vmcb->save.rip += 3; 1657 vmcb->control.event_inj = NMI_VECTOR | SVM_EVTINJ_TYPE_EXEPT | SVM_EVTINJ_VALID; 1658 break; 1659 1660 case 1: 1661 if (vmcb->control.exit_code != SVM_EXIT_ERR) { 1662 report_fail("VMEXIT not due to error. Exit reason 0x%x", 1663 vmcb->control.exit_code); 1664 return true; 1665 } 1666 report(count_exc == 0, "exception with vector 2 not injected"); 1667 vmcb->control.event_inj = DE_VECTOR | SVM_EVTINJ_TYPE_EXEPT | SVM_EVTINJ_VALID; 1668 break; 1669 1670 case 2: 1671 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1672 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1673 vmcb->control.exit_code); 1674 return true; 1675 } 1676 vmcb->save.rip += 3; 1677 report(count_exc == 1, "divide overflow exception injected"); 1678 report(!(vmcb->control.event_inj & SVM_EVTINJ_VALID), "eventinj.VALID cleared"); 1679 break; 1680 1681 default: 1682 return true; 1683 } 1684 1685 inc_test_stage(test); 1686 1687 return get_test_stage(test) == 3; 1688 } 1689 1690 static bool exc_inject_check(struct svm_test *test) 1691 { 1692 return count_exc == 1 && get_test_stage(test) == 3; 1693 } 1694 1695 static volatile bool virq_fired; 1696 1697 static void virq_isr(isr_regs_t *regs) 1698 { 1699 virq_fired = true; 1700 } 1701 1702 static void virq_inject_prepare(struct svm_test *test) 1703 { 1704 handle_irq(0xf1, virq_isr); 1705 default_prepare(test); 1706 vmcb->control.int_ctl = V_INTR_MASKING_MASK | V_IRQ_MASK | 1707 (0x0f << V_INTR_PRIO_SHIFT); // Set to the highest priority 1708 vmcb->control.int_vector = 0xf1; 1709 virq_fired = false; 1710 set_test_stage(test, 0); 1711 } 1712 1713 static void virq_inject_test(struct svm_test *test) 1714 { 1715 if (virq_fired) { 1716 report_fail("virtual interrupt fired before L2 sti"); 1717 set_test_stage(test, -1); 1718 vmmcall(); 1719 } 1720 1721 irq_enable(); 1722 asm volatile ("nop"); 1723 irq_disable(); 1724 1725 if (!virq_fired) { 1726 report_fail("virtual interrupt not fired after L2 sti"); 1727 set_test_stage(test, -1); 1728 } 1729 1730 vmmcall(); 1731 1732 if (virq_fired) { 1733 report_fail("virtual interrupt fired before L2 sti after VINTR intercept"); 1734 set_test_stage(test, -1); 1735 vmmcall(); 1736 } 1737 1738 irq_enable(); 1739 asm volatile ("nop"); 1740 irq_disable(); 1741 1742 if (!virq_fired) { 1743 report_fail("virtual interrupt not fired after return from VINTR intercept"); 1744 set_test_stage(test, -1); 1745 } 1746 1747 vmmcall(); 1748 1749 irq_enable(); 1750 asm volatile ("nop"); 1751 irq_disable(); 1752 1753 if (virq_fired) { 1754 report_fail("virtual interrupt fired when V_IRQ_PRIO less than V_TPR"); 1755 set_test_stage(test, -1); 1756 } 1757 1758 vmmcall(); 1759 vmmcall(); 1760 } 1761 1762 static bool virq_inject_finished(struct svm_test *test) 1763 { 1764 vmcb->save.rip += 3; 1765 1766 switch (get_test_stage(test)) { 1767 case 0: 1768 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1769 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1770 vmcb->control.exit_code); 1771 return true; 1772 } 1773 if (vmcb->control.int_ctl & V_IRQ_MASK) { 1774 report_fail("V_IRQ not cleared on VMEXIT after firing"); 1775 return true; 1776 } 1777 virq_fired = false; 1778 vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR); 1779 vmcb->control.int_ctl = V_INTR_MASKING_MASK | V_IRQ_MASK | 1780 (0x0f << V_INTR_PRIO_SHIFT); 1781 break; 1782 1783 case 1: 1784 if (vmcb->control.exit_code != SVM_EXIT_VINTR) { 1785 report_fail("VMEXIT not due to vintr. Exit reason 0x%x", 1786 vmcb->control.exit_code); 1787 return true; 1788 } 1789 if (virq_fired) { 1790 report_fail("V_IRQ fired before SVM_EXIT_VINTR"); 1791 return true; 1792 } 1793 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); 1794 break; 1795 1796 case 2: 1797 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1798 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1799 vmcb->control.exit_code); 1800 return true; 1801 } 1802 virq_fired = false; 1803 // Set irq to lower priority 1804 vmcb->control.int_ctl = V_INTR_MASKING_MASK | V_IRQ_MASK | 1805 (0x08 << V_INTR_PRIO_SHIFT); 1806 // Raise guest TPR 1807 vmcb->control.int_ctl |= 0x0a & V_TPR_MASK; 1808 break; 1809 1810 case 3: 1811 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1812 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1813 vmcb->control.exit_code); 1814 return true; 1815 } 1816 vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR); 1817 break; 1818 1819 case 4: 1820 // INTERCEPT_VINTR should be ignored because V_INTR_PRIO < V_TPR 1821 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 1822 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x", 1823 vmcb->control.exit_code); 1824 return true; 1825 } 1826 break; 1827 1828 default: 1829 return true; 1830 } 1831 1832 inc_test_stage(test); 1833 1834 return get_test_stage(test) == 5; 1835 } 1836 1837 static bool virq_inject_check(struct svm_test *test) 1838 { 1839 return get_test_stage(test) == 5; 1840 } 1841 1842 /* 1843 * Detect nested guest RIP corruption as explained in kernel commit 1844 * b6162e82aef19fee9c32cb3fe9ac30d9116a8c73 1845 * 1846 * In the assembly loop below 'ins' is executed while IO instructions 1847 * are not intercepted; the instruction is emulated by L0. 1848 * 1849 * At the same time we are getting interrupts from the local APIC timer, 1850 * and we do intercept them in L1 1851 * 1852 * If the interrupt happens on the insb instruction, L0 will VMexit, emulate 1853 * the insb instruction and then it will inject the interrupt to L1 through 1854 * a nested VMexit. Due to a bug, it would leave pre-emulation values of RIP, 1855 * RAX and RSP in the VMCB. 1856 * 1857 * In our intercept handler we detect the bug by checking that RIP is that of 1858 * the insb instruction, but its memory operand has already been written. 1859 * This means that insb was already executed. 1860 */ 1861 1862 static volatile int isr_cnt = 0; 1863 static volatile uint8_t io_port_var = 0xAA; 1864 extern const char insb_instruction_label[]; 1865 1866 static void reg_corruption_isr(isr_regs_t *regs) 1867 { 1868 isr_cnt++; 1869 apic_write(APIC_EOI, 0); 1870 } 1871 1872 static void reg_corruption_prepare(struct svm_test *test) 1873 { 1874 default_prepare(test); 1875 set_test_stage(test, 0); 1876 1877 vmcb->control.int_ctl = V_INTR_MASKING_MASK; 1878 vmcb->control.intercept |= (1ULL << INTERCEPT_INTR); 1879 1880 handle_irq(TIMER_VECTOR, reg_corruption_isr); 1881 1882 /* set local APIC to inject external interrupts */ 1883 apic_write(APIC_TMICT, 0); 1884 apic_write(APIC_TDCR, 0); 1885 apic_write(APIC_LVTT, TIMER_VECTOR | APIC_LVT_TIMER_PERIODIC); 1886 apic_write(APIC_TMICT, 1000); 1887 } 1888 1889 static void reg_corruption_test(struct svm_test *test) 1890 { 1891 /* this is endless loop, which is interrupted by the timer interrupt */ 1892 asm volatile ( 1893 "1:\n\t" 1894 "movw $0x4d0, %%dx\n\t" // IO port 1895 "lea %[io_port_var], %%rdi\n\t" 1896 "movb $0xAA, %[io_port_var]\n\t" 1897 "insb_instruction_label:\n\t" 1898 "insb\n\t" 1899 "jmp 1b\n\t" 1900 1901 : [io_port_var] "=m" (io_port_var) 1902 : /* no inputs*/ 1903 : "rdx", "rdi" 1904 ); 1905 } 1906 1907 static bool reg_corruption_finished(struct svm_test *test) 1908 { 1909 if (isr_cnt == 10000) { 1910 report_pass("No RIP corruption detected after %d timer interrupts", 1911 isr_cnt); 1912 set_test_stage(test, 1); 1913 goto cleanup; 1914 } 1915 1916 if (vmcb->control.exit_code == SVM_EXIT_INTR) { 1917 1918 void* guest_rip = (void*)vmcb->save.rip; 1919 1920 irq_enable(); 1921 asm volatile ("nop"); 1922 irq_disable(); 1923 1924 if (guest_rip == insb_instruction_label && io_port_var != 0xAA) { 1925 report_fail("RIP corruption detected after %d timer interrupts", 1926 isr_cnt); 1927 goto cleanup; 1928 } 1929 1930 } 1931 return false; 1932 cleanup: 1933 apic_write(APIC_LVTT, APIC_LVT_TIMER_MASK); 1934 apic_write(APIC_TMICT, 0); 1935 return true; 1936 1937 } 1938 1939 static bool reg_corruption_check(struct svm_test *test) 1940 { 1941 return get_test_stage(test) == 1; 1942 } 1943 1944 static void get_tss_entry(void *data) 1945 { 1946 *((gdt_entry_t **)data) = get_tss_descr(); 1947 } 1948 1949 static int orig_cpu_count; 1950 1951 static void init_startup_prepare(struct svm_test *test) 1952 { 1953 gdt_entry_t *tss_entry; 1954 int i; 1955 1956 on_cpu(1, get_tss_entry, &tss_entry); 1957 1958 orig_cpu_count = cpu_online_count; 1959 1960 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_INIT | APIC_INT_ASSERT, 1961 id_map[1]); 1962 1963 delay(100000000ULL); 1964 1965 --cpu_online_count; 1966 1967 tss_entry->type &= ~DESC_BUSY; 1968 1969 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_STARTUP, id_map[1]); 1970 1971 for (i = 0; i < 5 && cpu_online_count < orig_cpu_count; i++) 1972 delay(100000000ULL); 1973 } 1974 1975 static bool init_startup_finished(struct svm_test *test) 1976 { 1977 return true; 1978 } 1979 1980 static bool init_startup_check(struct svm_test *test) 1981 { 1982 return cpu_online_count == orig_cpu_count; 1983 } 1984 1985 static volatile bool init_intercept; 1986 1987 static void init_intercept_prepare(struct svm_test *test) 1988 { 1989 init_intercept = false; 1990 vmcb->control.intercept |= (1ULL << INTERCEPT_INIT); 1991 } 1992 1993 static void init_intercept_test(struct svm_test *test) 1994 { 1995 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_INIT | APIC_INT_ASSERT, 0); 1996 } 1997 1998 static bool init_intercept_finished(struct svm_test *test) 1999 { 2000 vmcb->save.rip += 3; 2001 2002 if (vmcb->control.exit_code != SVM_EXIT_INIT) { 2003 report_fail("VMEXIT not due to init intercept. Exit reason 0x%x", 2004 vmcb->control.exit_code); 2005 2006 return true; 2007 } 2008 2009 init_intercept = true; 2010 2011 report_pass("INIT to vcpu intercepted"); 2012 2013 return true; 2014 } 2015 2016 static bool init_intercept_check(struct svm_test *test) 2017 { 2018 return init_intercept; 2019 } 2020 2021 /* 2022 * Setting host EFLAGS.TF causes a #DB trap after the VMRUN completes on the 2023 * host side (i.e., after the #VMEXIT from the guest). 2024 * 2025 * Setting host EFLAGS.RF suppresses any potential instruction breakpoint 2026 * match on the VMRUN and completion of the VMRUN instruction clears the 2027 * host EFLAGS.RF bit. 2028 * 2029 * [AMD APM] 2030 */ 2031 static volatile u8 host_rflags_guest_main_flag = 0; 2032 static volatile u8 host_rflags_db_handler_flag = 0; 2033 static volatile bool host_rflags_ss_on_vmrun = false; 2034 static volatile bool host_rflags_vmrun_reached = false; 2035 static volatile bool host_rflags_set_tf = false; 2036 static volatile bool host_rflags_set_rf = false; 2037 static u64 rip_detected; 2038 2039 extern u64 *vmrun_rip; 2040 2041 static void host_rflags_db_handler(struct ex_regs *r) 2042 { 2043 if (host_rflags_ss_on_vmrun) { 2044 if (host_rflags_vmrun_reached) { 2045 if (!host_rflags_set_rf) { 2046 r->rflags &= ~X86_EFLAGS_TF; 2047 rip_detected = r->rip; 2048 } else { 2049 r->rflags |= X86_EFLAGS_RF; 2050 ++host_rflags_db_handler_flag; 2051 } 2052 } else { 2053 if (r->rip == (u64)&vmrun_rip) { 2054 host_rflags_vmrun_reached = true; 2055 2056 if (host_rflags_set_rf) { 2057 host_rflags_guest_main_flag = 0; 2058 rip_detected = r->rip; 2059 r->rflags &= ~X86_EFLAGS_TF; 2060 2061 /* Trigger #DB via debug registers */ 2062 write_dr0((void *)&vmrun_rip); 2063 write_dr7(0x403); 2064 } 2065 } 2066 } 2067 } else { 2068 r->rflags &= ~X86_EFLAGS_TF; 2069 } 2070 } 2071 2072 static void host_rflags_prepare(struct svm_test *test) 2073 { 2074 default_prepare(test); 2075 handle_exception(DB_VECTOR, host_rflags_db_handler); 2076 set_test_stage(test, 0); 2077 } 2078 2079 static void host_rflags_prepare_gif_clear(struct svm_test *test) 2080 { 2081 if (host_rflags_set_tf) 2082 write_rflags(read_rflags() | X86_EFLAGS_TF); 2083 } 2084 2085 static void host_rflags_test(struct svm_test *test) 2086 { 2087 while (1) { 2088 if (get_test_stage(test) > 0) { 2089 if ((host_rflags_set_tf && !host_rflags_ss_on_vmrun && !host_rflags_db_handler_flag) || 2090 (host_rflags_set_rf && host_rflags_db_handler_flag == 1)) 2091 host_rflags_guest_main_flag = 1; 2092 } 2093 2094 if (get_test_stage(test) == 4) 2095 break; 2096 vmmcall(); 2097 } 2098 } 2099 2100 static bool host_rflags_finished(struct svm_test *test) 2101 { 2102 switch (get_test_stage(test)) { 2103 case 0: 2104 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 2105 report_fail("Unexpected VMEXIT. Exit reason 0x%x", 2106 vmcb->control.exit_code); 2107 return true; 2108 } 2109 vmcb->save.rip += 3; 2110 /* 2111 * Setting host EFLAGS.TF not immediately before VMRUN, causes 2112 * #DB trap before first guest instruction is executed 2113 */ 2114 host_rflags_set_tf = true; 2115 break; 2116 case 1: 2117 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL || 2118 host_rflags_guest_main_flag != 1) { 2119 report_fail("Unexpected VMEXIT or #DB handler" 2120 " invoked before guest main. Exit reason 0x%x", 2121 vmcb->control.exit_code); 2122 return true; 2123 } 2124 vmcb->save.rip += 3; 2125 /* 2126 * Setting host EFLAGS.TF immediately before VMRUN, causes #DB 2127 * trap after VMRUN completes on the host side (i.e., after 2128 * VMEXIT from guest). 2129 */ 2130 host_rflags_ss_on_vmrun = true; 2131 break; 2132 case 2: 2133 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL || 2134 rip_detected != (u64)&vmrun_rip + 3) { 2135 report_fail("Unexpected VMEXIT or RIP mismatch." 2136 " Exit reason 0x%x, RIP actual: %lx, RIP expected: " 2137 "%lx", vmcb->control.exit_code, 2138 (u64)&vmrun_rip + 3, rip_detected); 2139 return true; 2140 } 2141 host_rflags_set_rf = true; 2142 host_rflags_guest_main_flag = 0; 2143 host_rflags_vmrun_reached = false; 2144 vmcb->save.rip += 3; 2145 break; 2146 case 3: 2147 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL || 2148 rip_detected != (u64)&vmrun_rip || 2149 host_rflags_guest_main_flag != 1 || 2150 host_rflags_db_handler_flag > 1 || 2151 read_rflags() & X86_EFLAGS_RF) { 2152 report_fail("Unexpected VMEXIT or RIP mismatch or " 2153 "EFLAGS.RF not cleared." 2154 " Exit reason 0x%x, RIP actual: %lx, RIP expected: " 2155 "%lx", vmcb->control.exit_code, 2156 (u64)&vmrun_rip, rip_detected); 2157 return true; 2158 } 2159 host_rflags_set_tf = false; 2160 host_rflags_set_rf = false; 2161 vmcb->save.rip += 3; 2162 break; 2163 default: 2164 return true; 2165 } 2166 inc_test_stage(test); 2167 return get_test_stage(test) == 5; 2168 } 2169 2170 static bool host_rflags_check(struct svm_test *test) 2171 { 2172 return get_test_stage(test) == 4; 2173 } 2174 2175 #define TEST(name) { #name, .v2 = name } 2176 2177 /* 2178 * v2 tests 2179 */ 2180 2181 /* 2182 * Ensure that kvm recalculates the L1 guest's CPUID.01H:ECX.OSXSAVE 2183 * after VM-exit from an L2 guest that sets CR4.OSXSAVE to a different 2184 * value than in L1. 2185 */ 2186 2187 static void svm_cr4_osxsave_test_guest(struct svm_test *test) 2188 { 2189 write_cr4(read_cr4() & ~X86_CR4_OSXSAVE); 2190 } 2191 2192 static void svm_cr4_osxsave_test(void) 2193 { 2194 if (!this_cpu_has(X86_FEATURE_XSAVE)) { 2195 report_skip("XSAVE not detected"); 2196 return; 2197 } 2198 2199 if (!(read_cr4() & X86_CR4_OSXSAVE)) { 2200 unsigned long cr4 = read_cr4() | X86_CR4_OSXSAVE; 2201 2202 write_cr4(cr4); 2203 vmcb->save.cr4 = cr4; 2204 } 2205 2206 report(cpuid_osxsave(), "CPUID.01H:ECX.XSAVE set before VMRUN"); 2207 2208 test_set_guest(svm_cr4_osxsave_test_guest); 2209 report(svm_vmrun() == SVM_EXIT_VMMCALL, 2210 "svm_cr4_osxsave_test_guest finished with VMMCALL"); 2211 2212 report(cpuid_osxsave(), "CPUID.01H:ECX.XSAVE set after VMRUN"); 2213 } 2214 2215 static void basic_guest_main(struct svm_test *test) 2216 { 2217 } 2218 2219 2220 #define SVM_TEST_REG_RESERVED_BITS(start, end, inc, str_name, reg, val, \ 2221 resv_mask) \ 2222 { \ 2223 u64 tmp, mask; \ 2224 int i; \ 2225 \ 2226 for (i = start; i <= end; i = i + inc) { \ 2227 mask = 1ull << i; \ 2228 if (!(mask & resv_mask)) \ 2229 continue; \ 2230 tmp = val | mask; \ 2231 reg = tmp; \ 2232 report(svm_vmrun() == SVM_EXIT_ERR, "Test %s %d:%d: %lx",\ 2233 str_name, end, start, tmp); \ 2234 } \ 2235 } 2236 2237 #define SVM_TEST_CR_RESERVED_BITS(start, end, inc, cr, val, resv_mask, \ 2238 exit_code, test_name) \ 2239 { \ 2240 u64 tmp, mask; \ 2241 u32 r; \ 2242 int i; \ 2243 \ 2244 for (i = start; i <= end; i = i + inc) { \ 2245 mask = 1ull << i; \ 2246 if (!(mask & resv_mask)) \ 2247 continue; \ 2248 tmp = val | mask; \ 2249 switch (cr) { \ 2250 case 0: \ 2251 vmcb->save.cr0 = tmp; \ 2252 break; \ 2253 case 3: \ 2254 vmcb->save.cr3 = tmp; \ 2255 break; \ 2256 case 4: \ 2257 vmcb->save.cr4 = tmp; \ 2258 } \ 2259 r = svm_vmrun(); \ 2260 report(r == exit_code, "Test CR%d %s%d:%d: %lx, wanted exit 0x%x, got 0x%x",\ 2261 cr, test_name, end, start, tmp, exit_code, r); \ 2262 } \ 2263 } 2264 2265 static void test_efer(void) 2266 { 2267 /* 2268 * Un-setting EFER.SVME is illegal 2269 */ 2270 u64 efer_saved = vmcb->save.efer; 2271 u64 efer = efer_saved; 2272 2273 report (svm_vmrun() == SVM_EXIT_VMMCALL, "EFER.SVME: %lx", efer); 2274 efer &= ~EFER_SVME; 2275 vmcb->save.efer = efer; 2276 report (svm_vmrun() == SVM_EXIT_ERR, "EFER.SVME: %lx", efer); 2277 vmcb->save.efer = efer_saved; 2278 2279 /* 2280 * EFER MBZ bits: 63:16, 9 2281 */ 2282 efer_saved = vmcb->save.efer; 2283 2284 SVM_TEST_REG_RESERVED_BITS(8, 9, 1, "EFER", vmcb->save.efer, 2285 efer_saved, SVM_EFER_RESERVED_MASK); 2286 SVM_TEST_REG_RESERVED_BITS(16, 63, 4, "EFER", vmcb->save.efer, 2287 efer_saved, SVM_EFER_RESERVED_MASK); 2288 2289 /* 2290 * EFER.LME and CR0.PG are both set and CR4.PAE is zero. 2291 */ 2292 u64 cr0_saved = vmcb->save.cr0; 2293 u64 cr0; 2294 u64 cr4_saved = vmcb->save.cr4; 2295 u64 cr4; 2296 2297 efer = efer_saved | EFER_LME; 2298 vmcb->save.efer = efer; 2299 cr0 = cr0_saved | X86_CR0_PG | X86_CR0_PE; 2300 vmcb->save.cr0 = cr0; 2301 cr4 = cr4_saved & ~X86_CR4_PAE; 2302 vmcb->save.cr4 = cr4; 2303 report(svm_vmrun() == SVM_EXIT_ERR, "EFER.LME=1 (%lx), " 2304 "CR0.PG=1 (%lx) and CR4.PAE=0 (%lx)", efer, cr0, cr4); 2305 2306 /* 2307 * EFER.LME and CR0.PG are both set and CR0.PE is zero. 2308 * CR4.PAE needs to be set as we otherwise cannot 2309 * determine if CR4.PAE=0 or CR0.PE=0 triggered the 2310 * SVM_EXIT_ERR. 2311 */ 2312 cr4 = cr4_saved | X86_CR4_PAE; 2313 vmcb->save.cr4 = cr4; 2314 cr0 &= ~X86_CR0_PE; 2315 vmcb->save.cr0 = cr0; 2316 report(svm_vmrun() == SVM_EXIT_ERR, "EFER.LME=1 (%lx), " 2317 "CR0.PG=1 and CR0.PE=0 (%lx)", efer, cr0); 2318 2319 /* 2320 * EFER.LME, CR0.PG, CR4.PAE, CS.L, and CS.D are all non-zero. 2321 */ 2322 u32 cs_attrib_saved = vmcb->save.cs.attrib; 2323 u32 cs_attrib; 2324 2325 cr0 |= X86_CR0_PE; 2326 vmcb->save.cr0 = cr0; 2327 cs_attrib = cs_attrib_saved | SVM_SELECTOR_L_MASK | 2328 SVM_SELECTOR_DB_MASK; 2329 vmcb->save.cs.attrib = cs_attrib; 2330 report(svm_vmrun() == SVM_EXIT_ERR, "EFER.LME=1 (%lx), " 2331 "CR0.PG=1 (%lx), CR4.PAE=1 (%lx), CS.L=1 and CS.D=1 (%x)", 2332 efer, cr0, cr4, cs_attrib); 2333 2334 vmcb->save.cr0 = cr0_saved; 2335 vmcb->save.cr4 = cr4_saved; 2336 vmcb->save.efer = efer_saved; 2337 vmcb->save.cs.attrib = cs_attrib_saved; 2338 } 2339 2340 static void test_cr0(void) 2341 { 2342 /* 2343 * Un-setting CR0.CD and setting CR0.NW is illegal combination 2344 */ 2345 u64 cr0_saved = vmcb->save.cr0; 2346 u64 cr0 = cr0_saved; 2347 2348 cr0 |= X86_CR0_CD; 2349 cr0 &= ~X86_CR0_NW; 2350 vmcb->save.cr0 = cr0; 2351 report (svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR0 CD=1,NW=0: %lx", 2352 cr0); 2353 cr0 |= X86_CR0_NW; 2354 vmcb->save.cr0 = cr0; 2355 report (svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR0 CD=1,NW=1: %lx", 2356 cr0); 2357 cr0 &= ~X86_CR0_NW; 2358 cr0 &= ~X86_CR0_CD; 2359 vmcb->save.cr0 = cr0; 2360 report (svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR0 CD=0,NW=0: %lx", 2361 cr0); 2362 cr0 |= X86_CR0_NW; 2363 vmcb->save.cr0 = cr0; 2364 report (svm_vmrun() == SVM_EXIT_ERR, "Test CR0 CD=0,NW=1: %lx", 2365 cr0); 2366 vmcb->save.cr0 = cr0_saved; 2367 2368 /* 2369 * CR0[63:32] are not zero 2370 */ 2371 cr0 = cr0_saved; 2372 2373 SVM_TEST_REG_RESERVED_BITS(32, 63, 4, "CR0", vmcb->save.cr0, cr0_saved, 2374 SVM_CR0_RESERVED_MASK); 2375 vmcb->save.cr0 = cr0_saved; 2376 } 2377 2378 static void test_cr3(void) 2379 { 2380 /* 2381 * CR3 MBZ bits based on different modes: 2382 * [63:52] - long mode 2383 */ 2384 u64 cr3_saved = vmcb->save.cr3; 2385 2386 SVM_TEST_CR_RESERVED_BITS(0, 63, 1, 3, cr3_saved, 2387 SVM_CR3_LONG_MBZ_MASK, SVM_EXIT_ERR, ""); 2388 2389 vmcb->save.cr3 = cr3_saved & ~SVM_CR3_LONG_MBZ_MASK; 2390 report(svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR3 63:0: %lx", 2391 vmcb->save.cr3); 2392 2393 /* 2394 * CR3 non-MBZ reserved bits based on different modes: 2395 * [11:5] [2:0] - long mode (PCIDE=0) 2396 * [2:0] - PAE legacy mode 2397 */ 2398 u64 cr4_saved = vmcb->save.cr4; 2399 u64 *pdpe = npt_get_pml4e(); 2400 2401 /* 2402 * Long mode 2403 */ 2404 if (this_cpu_has(X86_FEATURE_PCID)) { 2405 vmcb->save.cr4 = cr4_saved | X86_CR4_PCIDE; 2406 SVM_TEST_CR_RESERVED_BITS(0, 11, 1, 3, cr3_saved, 2407 SVM_CR3_LONG_RESERVED_MASK, SVM_EXIT_VMMCALL, "(PCIDE=1) "); 2408 2409 vmcb->save.cr3 = cr3_saved & ~SVM_CR3_LONG_RESERVED_MASK; 2410 report(svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR3 63:0: %lx", 2411 vmcb->save.cr3); 2412 } 2413 2414 vmcb->save.cr4 = cr4_saved & ~X86_CR4_PCIDE; 2415 2416 if (!npt_supported()) 2417 goto skip_npt_only; 2418 2419 /* Clear P (Present) bit in NPT in order to trigger #NPF */ 2420 pdpe[0] &= ~1ULL; 2421 2422 SVM_TEST_CR_RESERVED_BITS(0, 11, 1, 3, cr3_saved, 2423 SVM_CR3_LONG_RESERVED_MASK, SVM_EXIT_NPF, "(PCIDE=0) "); 2424 2425 pdpe[0] |= 1ULL; 2426 vmcb->save.cr3 = cr3_saved; 2427 2428 /* 2429 * PAE legacy 2430 */ 2431 pdpe[0] &= ~1ULL; 2432 vmcb->save.cr4 = cr4_saved | X86_CR4_PAE; 2433 SVM_TEST_CR_RESERVED_BITS(0, 2, 1, 3, cr3_saved, 2434 SVM_CR3_PAE_LEGACY_RESERVED_MASK, SVM_EXIT_NPF, "(PAE) "); 2435 2436 pdpe[0] |= 1ULL; 2437 2438 skip_npt_only: 2439 vmcb->save.cr3 = cr3_saved; 2440 vmcb->save.cr4 = cr4_saved; 2441 } 2442 2443 /* Test CR4 MBZ bits based on legacy or long modes */ 2444 static void test_cr4(void) 2445 { 2446 u64 cr4_saved = vmcb->save.cr4; 2447 u64 efer_saved = vmcb->save.efer; 2448 u64 efer = efer_saved; 2449 2450 efer &= ~EFER_LME; 2451 vmcb->save.efer = efer; 2452 SVM_TEST_CR_RESERVED_BITS(12, 31, 1, 4, cr4_saved, 2453 SVM_CR4_LEGACY_RESERVED_MASK, SVM_EXIT_ERR, ""); 2454 2455 efer |= EFER_LME; 2456 vmcb->save.efer = efer; 2457 SVM_TEST_CR_RESERVED_BITS(12, 31, 1, 4, cr4_saved, 2458 SVM_CR4_RESERVED_MASK, SVM_EXIT_ERR, ""); 2459 SVM_TEST_CR_RESERVED_BITS(32, 63, 4, 4, cr4_saved, 2460 SVM_CR4_RESERVED_MASK, SVM_EXIT_ERR, ""); 2461 2462 vmcb->save.cr4 = cr4_saved; 2463 vmcb->save.efer = efer_saved; 2464 } 2465 2466 static void test_dr(void) 2467 { 2468 /* 2469 * DR6[63:32] and DR7[63:32] are MBZ 2470 */ 2471 u64 dr_saved = vmcb->save.dr6; 2472 2473 SVM_TEST_REG_RESERVED_BITS(32, 63, 4, "DR6", vmcb->save.dr6, dr_saved, 2474 SVM_DR6_RESERVED_MASK); 2475 vmcb->save.dr6 = dr_saved; 2476 2477 dr_saved = vmcb->save.dr7; 2478 SVM_TEST_REG_RESERVED_BITS(32, 63, 4, "DR7", vmcb->save.dr7, dr_saved, 2479 SVM_DR7_RESERVED_MASK); 2480 2481 vmcb->save.dr7 = dr_saved; 2482 } 2483 2484 /* TODO: verify if high 32-bits are sign- or zero-extended on bare metal */ 2485 #define TEST_BITMAP_ADDR(save_intercept, type, addr, exit_code, \ 2486 msg) { \ 2487 vmcb->control.intercept = saved_intercept | 1ULL << type; \ 2488 if (type == INTERCEPT_MSR_PROT) \ 2489 vmcb->control.msrpm_base_pa = addr; \ 2490 else \ 2491 vmcb->control.iopm_base_pa = addr; \ 2492 report(svm_vmrun() == exit_code, \ 2493 "Test %s address: %lx", msg, addr); \ 2494 } 2495 2496 /* 2497 * If the MSR or IOIO intercept table extends to a physical address that 2498 * is greater than or equal to the maximum supported physical address, the 2499 * guest state is illegal. 2500 * 2501 * The VMRUN instruction ignores the lower 12 bits of the address specified 2502 * in the VMCB. 2503 * 2504 * MSRPM spans 2 contiguous 4KB pages while IOPM spans 2 contiguous 4KB 2505 * pages + 1 byte. 2506 * 2507 * [APM vol 2] 2508 * 2509 * Note: Unallocated MSRPM addresses conforming to consistency checks, generate 2510 * #NPF. 2511 */ 2512 static void test_msrpm_iopm_bitmap_addrs(void) 2513 { 2514 u64 saved_intercept = vmcb->control.intercept; 2515 u64 addr_beyond_limit = 1ull << cpuid_maxphyaddr(); 2516 u64 addr = virt_to_phys(msr_bitmap) & (~((1ull << 12) - 1)); 2517 2518 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT, 2519 addr_beyond_limit - 2 * PAGE_SIZE, SVM_EXIT_ERR, 2520 "MSRPM"); 2521 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT, 2522 addr_beyond_limit - 2 * PAGE_SIZE + 1, SVM_EXIT_ERR, 2523 "MSRPM"); 2524 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT, 2525 addr_beyond_limit - PAGE_SIZE, SVM_EXIT_ERR, 2526 "MSRPM"); 2527 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT, addr, 2528 SVM_EXIT_VMMCALL, "MSRPM"); 2529 addr |= (1ull << 12) - 1; 2530 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT, addr, 2531 SVM_EXIT_VMMCALL, "MSRPM"); 2532 2533 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, 2534 addr_beyond_limit - 4 * PAGE_SIZE, SVM_EXIT_VMMCALL, 2535 "IOPM"); 2536 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, 2537 addr_beyond_limit - 3 * PAGE_SIZE, SVM_EXIT_VMMCALL, 2538 "IOPM"); 2539 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, 2540 addr_beyond_limit - 2 * PAGE_SIZE - 2, SVM_EXIT_VMMCALL, 2541 "IOPM"); 2542 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, 2543 addr_beyond_limit - 2 * PAGE_SIZE, SVM_EXIT_ERR, 2544 "IOPM"); 2545 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, 2546 addr_beyond_limit - PAGE_SIZE, SVM_EXIT_ERR, 2547 "IOPM"); 2548 addr = virt_to_phys(io_bitmap) & (~((1ull << 11) - 1)); 2549 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, addr, 2550 SVM_EXIT_VMMCALL, "IOPM"); 2551 addr |= (1ull << 12) - 1; 2552 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, addr, 2553 SVM_EXIT_VMMCALL, "IOPM"); 2554 2555 vmcb->control.intercept = saved_intercept; 2556 } 2557 2558 /* 2559 * Unlike VMSAVE, VMRUN seems not to update the value of noncanonical 2560 * segment bases in the VMCB. However, VMENTRY succeeds as documented. 2561 */ 2562 #define TEST_CANONICAL_VMRUN(seg_base, msg) \ 2563 saved_addr = seg_base; \ 2564 seg_base = (seg_base & ((1ul << addr_limit) - 1)) | noncanonical_mask; \ 2565 return_value = svm_vmrun(); \ 2566 report(return_value == SVM_EXIT_VMMCALL, \ 2567 "Successful VMRUN with noncanonical %s.base", msg); \ 2568 seg_base = saved_addr; 2569 2570 2571 #define TEST_CANONICAL_VMLOAD(seg_base, msg) \ 2572 saved_addr = seg_base; \ 2573 seg_base = (seg_base & ((1ul << addr_limit) - 1)) | noncanonical_mask; \ 2574 asm volatile ("vmload %0" : : "a"(vmcb_phys) : "memory"); \ 2575 asm volatile ("vmsave %0" : : "a"(vmcb_phys) : "memory"); \ 2576 report(is_canonical(seg_base), \ 2577 "Test %s.base for canonical form: %lx", msg, seg_base); \ 2578 seg_base = saved_addr; 2579 2580 static void test_canonicalization(void) 2581 { 2582 u64 saved_addr; 2583 u64 return_value; 2584 u64 addr_limit; 2585 u64 vmcb_phys = virt_to_phys(vmcb); 2586 2587 addr_limit = (this_cpu_has(X86_FEATURE_LA57)) ? 57 : 48; 2588 u64 noncanonical_mask = NONCANONICAL & ~((1ul << addr_limit) - 1); 2589 2590 TEST_CANONICAL_VMLOAD(vmcb->save.fs.base, "FS"); 2591 TEST_CANONICAL_VMLOAD(vmcb->save.gs.base, "GS"); 2592 TEST_CANONICAL_VMLOAD(vmcb->save.ldtr.base, "LDTR"); 2593 TEST_CANONICAL_VMLOAD(vmcb->save.tr.base, "TR"); 2594 TEST_CANONICAL_VMLOAD(vmcb->save.kernel_gs_base, "KERNEL GS"); 2595 TEST_CANONICAL_VMRUN(vmcb->save.es.base, "ES"); 2596 TEST_CANONICAL_VMRUN(vmcb->save.cs.base, "CS"); 2597 TEST_CANONICAL_VMRUN(vmcb->save.ss.base, "SS"); 2598 TEST_CANONICAL_VMRUN(vmcb->save.ds.base, "DS"); 2599 TEST_CANONICAL_VMRUN(vmcb->save.gdtr.base, "GDTR"); 2600 TEST_CANONICAL_VMRUN(vmcb->save.idtr.base, "IDTR"); 2601 } 2602 2603 /* 2604 * When VMRUN loads a guest value of 1 in EFLAGS.TF, that value does not 2605 * cause a trace trap between the VMRUN and the first guest instruction, but 2606 * rather after completion of the first guest instruction. 2607 * 2608 * [APM vol 2] 2609 */ 2610 u64 guest_rflags_test_trap_rip; 2611 2612 static void guest_rflags_test_db_handler(struct ex_regs *r) 2613 { 2614 guest_rflags_test_trap_rip = r->rip; 2615 r->rflags &= ~X86_EFLAGS_TF; 2616 } 2617 2618 static void svm_guest_state_test(void) 2619 { 2620 test_set_guest(basic_guest_main); 2621 test_efer(); 2622 test_cr0(); 2623 test_cr3(); 2624 test_cr4(); 2625 test_dr(); 2626 test_msrpm_iopm_bitmap_addrs(); 2627 test_canonicalization(); 2628 } 2629 2630 extern void guest_rflags_test_guest(struct svm_test *test); 2631 extern u64 *insn2; 2632 extern u64 *guest_end; 2633 2634 asm("guest_rflags_test_guest:\n\t" 2635 "push %rbp\n\t" 2636 ".global insn2\n\t" 2637 "insn2:\n\t" 2638 "mov %rsp,%rbp\n\t" 2639 "vmmcall\n\t" 2640 "vmmcall\n\t" 2641 ".global guest_end\n\t" 2642 "guest_end:\n\t" 2643 "vmmcall\n\t" 2644 "pop %rbp\n\t" 2645 "ret"); 2646 2647 static void svm_test_singlestep(void) 2648 { 2649 handle_exception(DB_VECTOR, guest_rflags_test_db_handler); 2650 2651 /* 2652 * Trap expected after completion of first guest instruction 2653 */ 2654 vmcb->save.rflags |= X86_EFLAGS_TF; 2655 report (__svm_vmrun((u64)guest_rflags_test_guest) == SVM_EXIT_VMMCALL && 2656 guest_rflags_test_trap_rip == (u64)&insn2, 2657 "Test EFLAGS.TF on VMRUN: trap expected after completion of first guest instruction"); 2658 /* 2659 * No trap expected 2660 */ 2661 guest_rflags_test_trap_rip = 0; 2662 vmcb->save.rip += 3; 2663 vmcb->save.rflags |= X86_EFLAGS_TF; 2664 report (__svm_vmrun(vmcb->save.rip) == SVM_EXIT_VMMCALL && 2665 guest_rflags_test_trap_rip == 0, "Test EFLAGS.TF on VMRUN: trap not expected"); 2666 2667 /* 2668 * Let guest finish execution 2669 */ 2670 vmcb->save.rip += 3; 2671 report (__svm_vmrun(vmcb->save.rip) == SVM_EXIT_VMMCALL && 2672 vmcb->save.rip == (u64)&guest_end, "Test EFLAGS.TF on VMRUN: guest execution completion"); 2673 } 2674 2675 static void __svm_npt_rsvd_bits_test(u64 *pxe, u64 rsvd_bits, u64 efer, 2676 ulong cr4, u64 guest_efer, ulong guest_cr4) 2677 { 2678 u64 pxe_orig = *pxe; 2679 int exit_reason; 2680 u64 pfec; 2681 2682 wrmsr(MSR_EFER, efer); 2683 write_cr4(cr4); 2684 2685 vmcb->save.efer = guest_efer; 2686 vmcb->save.cr4 = guest_cr4; 2687 2688 *pxe |= rsvd_bits; 2689 2690 exit_reason = svm_vmrun(); 2691 2692 report(exit_reason == SVM_EXIT_NPF, 2693 "Wanted #NPF on rsvd bits = 0x%lx, got exit = 0x%x", rsvd_bits, exit_reason); 2694 2695 if (pxe == npt_get_pdpe() || pxe == npt_get_pml4e()) { 2696 /* 2697 * The guest's page tables will blow up on a bad PDPE/PML4E, 2698 * before starting the final walk of the guest page. 2699 */ 2700 pfec = 0x20000000full; 2701 } else { 2702 /* RSVD #NPF on final walk of guest page. */ 2703 pfec = 0x10000000dULL; 2704 2705 /* PFEC.FETCH=1 if NX=1 *or* SMEP=1. */ 2706 if ((cr4 & X86_CR4_SMEP) || (efer & EFER_NX)) 2707 pfec |= 0x10; 2708 2709 } 2710 2711 report(vmcb->control.exit_info_1 == pfec, 2712 "Wanted PFEC = 0x%lx, got PFEC = %lx, PxE = 0x%lx. " 2713 "host.NX = %u, host.SMEP = %u, guest.NX = %u, guest.SMEP = %u", 2714 pfec, vmcb->control.exit_info_1, *pxe, 2715 !!(efer & EFER_NX), !!(cr4 & X86_CR4_SMEP), 2716 !!(guest_efer & EFER_NX), !!(guest_cr4 & X86_CR4_SMEP)); 2717 2718 *pxe = pxe_orig; 2719 } 2720 2721 static void _svm_npt_rsvd_bits_test(u64 *pxe, u64 pxe_rsvd_bits, u64 efer, 2722 ulong cr4, u64 guest_efer, ulong guest_cr4) 2723 { 2724 u64 rsvd_bits; 2725 int i; 2726 2727 /* 2728 * RDTSC or RDRAND can sometimes fail to generate a valid reserved bits 2729 */ 2730 if (!pxe_rsvd_bits) { 2731 report_skip("svm_npt_rsvd_bits_test: Reserved bits are not valid"); 2732 return; 2733 } 2734 2735 /* 2736 * Test all combinations of guest/host EFER.NX and CR4.SMEP. If host 2737 * EFER.NX=0, use NX as the reserved bit, otherwise use the passed in 2738 * @pxe_rsvd_bits. 2739 */ 2740 for (i = 0; i < 16; i++) { 2741 if (i & 1) { 2742 rsvd_bits = pxe_rsvd_bits; 2743 efer |= EFER_NX; 2744 } else { 2745 rsvd_bits = PT64_NX_MASK; 2746 efer &= ~EFER_NX; 2747 } 2748 if (i & 2) 2749 cr4 |= X86_CR4_SMEP; 2750 else 2751 cr4 &= ~X86_CR4_SMEP; 2752 if (i & 4) 2753 guest_efer |= EFER_NX; 2754 else 2755 guest_efer &= ~EFER_NX; 2756 if (i & 8) 2757 guest_cr4 |= X86_CR4_SMEP; 2758 else 2759 guest_cr4 &= ~X86_CR4_SMEP; 2760 2761 __svm_npt_rsvd_bits_test(pxe, rsvd_bits, efer, cr4, 2762 guest_efer, guest_cr4); 2763 } 2764 } 2765 2766 static u64 get_random_bits(u64 hi, u64 low) 2767 { 2768 unsigned retry = 5; 2769 u64 rsvd_bits = 0; 2770 2771 if (this_cpu_has(X86_FEATURE_RDRAND)) { 2772 do { 2773 rsvd_bits = (rdrand() << low) & GENMASK_ULL(hi, low); 2774 retry--; 2775 } while (!rsvd_bits && retry); 2776 } 2777 2778 if (!rsvd_bits) { 2779 retry = 5; 2780 do { 2781 rsvd_bits = (rdtsc() << low) & GENMASK_ULL(hi, low); 2782 retry--; 2783 } while (!rsvd_bits && retry); 2784 } 2785 2786 return rsvd_bits; 2787 } 2788 2789 2790 static void svm_npt_rsvd_bits_test(void) 2791 { 2792 u64 saved_efer, host_efer, sg_efer, guest_efer; 2793 ulong saved_cr4, host_cr4, sg_cr4, guest_cr4; 2794 2795 if (!npt_supported()) { 2796 report_skip("NPT not supported"); 2797 return; 2798 } 2799 2800 saved_efer = host_efer = rdmsr(MSR_EFER); 2801 saved_cr4 = host_cr4 = read_cr4(); 2802 sg_efer = guest_efer = vmcb->save.efer; 2803 sg_cr4 = guest_cr4 = vmcb->save.cr4; 2804 2805 test_set_guest(basic_guest_main); 2806 2807 /* 2808 * 4k PTEs don't have reserved bits if MAXPHYADDR >= 52, just skip the 2809 * sub-test. The NX test is still valid, but the extra bit of coverage 2810 * isn't worth the extra complexity. 2811 */ 2812 if (cpuid_maxphyaddr() >= 52) 2813 goto skip_pte_test; 2814 2815 _svm_npt_rsvd_bits_test(npt_get_pte((u64)basic_guest_main), 2816 get_random_bits(51, cpuid_maxphyaddr()), 2817 host_efer, host_cr4, guest_efer, guest_cr4); 2818 2819 skip_pte_test: 2820 _svm_npt_rsvd_bits_test(npt_get_pde((u64)basic_guest_main), 2821 get_random_bits(20, 13) | PT_PAGE_SIZE_MASK, 2822 host_efer, host_cr4, guest_efer, guest_cr4); 2823 2824 _svm_npt_rsvd_bits_test(npt_get_pdpe(), 2825 PT_PAGE_SIZE_MASK | 2826 (this_cpu_has(X86_FEATURE_GBPAGES) ? get_random_bits(29, 13) : 0), 2827 host_efer, host_cr4, guest_efer, guest_cr4); 2828 2829 _svm_npt_rsvd_bits_test(npt_get_pml4e(), BIT_ULL(8), 2830 host_efer, host_cr4, guest_efer, guest_cr4); 2831 2832 wrmsr(MSR_EFER, saved_efer); 2833 write_cr4(saved_cr4); 2834 vmcb->save.efer = sg_efer; 2835 vmcb->save.cr4 = sg_cr4; 2836 } 2837 2838 static bool volatile svm_errata_reproduced = false; 2839 static unsigned long volatile physical = 0; 2840 2841 2842 /* 2843 * 2844 * Test the following errata: 2845 * If the VMRUN/VMSAVE/VMLOAD are attempted by the nested guest, 2846 * the CPU would first check the EAX against host reserved memory 2847 * regions (so far only SMM_ADDR/SMM_MASK are known to cause it), 2848 * and only then signal #VMexit 2849 * 2850 * Try to reproduce this by trying vmsave on each possible 4K aligned memory 2851 * address in the low 4G where the SMM area has to reside. 2852 */ 2853 2854 static void gp_isr(struct ex_regs *r) 2855 { 2856 svm_errata_reproduced = true; 2857 /* skip over the vmsave instruction*/ 2858 r->rip += 3; 2859 } 2860 2861 static void svm_vmrun_errata_test(void) 2862 { 2863 unsigned long *last_page = NULL; 2864 2865 handle_exception(GP_VECTOR, gp_isr); 2866 2867 while (!svm_errata_reproduced) { 2868 2869 unsigned long *page = alloc_pages(1); 2870 2871 if (!page) { 2872 report_pass("All guest memory tested, no bug found"); 2873 break; 2874 } 2875 2876 physical = virt_to_phys(page); 2877 2878 asm volatile ( 2879 "mov %[_physical], %%rax\n\t" 2880 "vmsave %%rax\n\t" 2881 2882 : [_physical] "=m" (physical) 2883 : /* no inputs*/ 2884 : "rax" /*clobbers*/ 2885 ); 2886 2887 if (svm_errata_reproduced) { 2888 report_fail("Got #GP exception - svm errata reproduced at 0x%lx", 2889 physical); 2890 break; 2891 } 2892 2893 *page = (unsigned long)last_page; 2894 last_page = page; 2895 } 2896 2897 while (last_page) { 2898 unsigned long *page = last_page; 2899 last_page = (unsigned long *)*last_page; 2900 free_pages_by_order(page, 1); 2901 } 2902 } 2903 2904 static void vmload_vmsave_guest_main(struct svm_test *test) 2905 { 2906 u64 vmcb_phys = virt_to_phys(vmcb); 2907 2908 asm volatile ("vmload %0" : : "a"(vmcb_phys)); 2909 asm volatile ("vmsave %0" : : "a"(vmcb_phys)); 2910 } 2911 2912 static void svm_vmload_vmsave(void) 2913 { 2914 u32 intercept_saved = vmcb->control.intercept; 2915 2916 test_set_guest(vmload_vmsave_guest_main); 2917 2918 /* 2919 * Disabling intercept for VMLOAD and VMSAVE doesn't cause 2920 * respective #VMEXIT to host 2921 */ 2922 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMLOAD); 2923 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMSAVE); 2924 svm_vmrun(); 2925 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test " 2926 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT"); 2927 2928 /* 2929 * Enabling intercept for VMLOAD and VMSAVE causes respective 2930 * #VMEXIT to host 2931 */ 2932 vmcb->control.intercept |= (1ULL << INTERCEPT_VMLOAD); 2933 svm_vmrun(); 2934 report(vmcb->control.exit_code == SVM_EXIT_VMLOAD, "Test " 2935 "VMLOAD/VMSAVE intercept: Expected VMLOAD #VMEXIT"); 2936 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMLOAD); 2937 vmcb->control.intercept |= (1ULL << INTERCEPT_VMSAVE); 2938 svm_vmrun(); 2939 report(vmcb->control.exit_code == SVM_EXIT_VMSAVE, "Test " 2940 "VMLOAD/VMSAVE intercept: Expected VMSAVE #VMEXIT"); 2941 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMSAVE); 2942 svm_vmrun(); 2943 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test " 2944 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT"); 2945 2946 vmcb->control.intercept |= (1ULL << INTERCEPT_VMLOAD); 2947 svm_vmrun(); 2948 report(vmcb->control.exit_code == SVM_EXIT_VMLOAD, "Test " 2949 "VMLOAD/VMSAVE intercept: Expected VMLOAD #VMEXIT"); 2950 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMLOAD); 2951 svm_vmrun(); 2952 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test " 2953 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT"); 2954 2955 vmcb->control.intercept |= (1ULL << INTERCEPT_VMSAVE); 2956 svm_vmrun(); 2957 report(vmcb->control.exit_code == SVM_EXIT_VMSAVE, "Test " 2958 "VMLOAD/VMSAVE intercept: Expected VMSAVE #VMEXIT"); 2959 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMSAVE); 2960 svm_vmrun(); 2961 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test " 2962 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT"); 2963 2964 vmcb->control.intercept = intercept_saved; 2965 } 2966 2967 static void prepare_vgif_enabled(struct svm_test *test) 2968 { 2969 default_prepare(test); 2970 } 2971 2972 static void test_vgif(struct svm_test *test) 2973 { 2974 asm volatile ("vmmcall\n\tstgi\n\tvmmcall\n\tclgi\n\tvmmcall\n\t"); 2975 2976 } 2977 2978 static bool vgif_finished(struct svm_test *test) 2979 { 2980 switch (get_test_stage(test)) 2981 { 2982 case 0: 2983 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 2984 report_fail("VMEXIT not due to vmmcall."); 2985 return true; 2986 } 2987 vmcb->control.int_ctl |= V_GIF_ENABLED_MASK; 2988 vmcb->save.rip += 3; 2989 inc_test_stage(test); 2990 break; 2991 case 1: 2992 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 2993 report_fail("VMEXIT not due to vmmcall."); 2994 return true; 2995 } 2996 if (!(vmcb->control.int_ctl & V_GIF_MASK)) { 2997 report_fail("Failed to set VGIF when executing STGI."); 2998 vmcb->control.int_ctl &= ~V_GIF_ENABLED_MASK; 2999 return true; 3000 } 3001 report_pass("STGI set VGIF bit."); 3002 vmcb->save.rip += 3; 3003 inc_test_stage(test); 3004 break; 3005 case 2: 3006 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 3007 report_fail("VMEXIT not due to vmmcall."); 3008 return true; 3009 } 3010 if (vmcb->control.int_ctl & V_GIF_MASK) { 3011 report_fail("Failed to clear VGIF when executing CLGI."); 3012 vmcb->control.int_ctl &= ~V_GIF_ENABLED_MASK; 3013 return true; 3014 } 3015 report_pass("CLGI cleared VGIF bit."); 3016 vmcb->save.rip += 3; 3017 inc_test_stage(test); 3018 vmcb->control.int_ctl &= ~V_GIF_ENABLED_MASK; 3019 break; 3020 default: 3021 return true; 3022 break; 3023 } 3024 3025 return get_test_stage(test) == 3; 3026 } 3027 3028 static bool vgif_check(struct svm_test *test) 3029 { 3030 return get_test_stage(test) == 3; 3031 } 3032 3033 3034 static int pause_test_counter; 3035 static int wait_counter; 3036 3037 static void pause_filter_test_guest_main(struct svm_test *test) 3038 { 3039 int i; 3040 for (i = 0 ; i < pause_test_counter ; i++) 3041 pause(); 3042 3043 if (!wait_counter) 3044 return; 3045 3046 for (i = 0; i < wait_counter; i++) 3047 ; 3048 3049 for (i = 0 ; i < pause_test_counter ; i++) 3050 pause(); 3051 3052 } 3053 3054 static void pause_filter_run_test(int pause_iterations, int filter_value, int wait_iterations, int threshold) 3055 { 3056 test_set_guest(pause_filter_test_guest_main); 3057 3058 pause_test_counter = pause_iterations; 3059 wait_counter = wait_iterations; 3060 3061 vmcb->control.pause_filter_count = filter_value; 3062 vmcb->control.pause_filter_thresh = threshold; 3063 svm_vmrun(); 3064 3065 if (filter_value <= pause_iterations || wait_iterations < threshold) 3066 report(vmcb->control.exit_code == SVM_EXIT_PAUSE, "expected PAUSE vmexit"); 3067 else 3068 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "no expected PAUSE vmexit"); 3069 } 3070 3071 static void pause_filter_test(void) 3072 { 3073 if (!pause_filter_supported()) { 3074 report_skip("PAUSE filter not supported in the guest"); 3075 return; 3076 } 3077 3078 vmcb->control.intercept |= (1 << INTERCEPT_PAUSE); 3079 3080 // filter count more that pause count - no VMexit 3081 pause_filter_run_test(10, 9, 0, 0); 3082 3083 // filter count smaller pause count - no VMexit 3084 pause_filter_run_test(20, 21, 0, 0); 3085 3086 3087 if (pause_threshold_supported()) { 3088 // filter count smaller pause count - no VMexit + large enough threshold 3089 // so that filter counter resets 3090 pause_filter_run_test(20, 21, 1000, 10); 3091 3092 // filter count smaller pause count - no VMexit + small threshold 3093 // so that filter doesn't reset 3094 pause_filter_run_test(20, 21, 10, 1000); 3095 } else { 3096 report_skip("PAUSE threshold not supported in the guest"); 3097 return; 3098 } 3099 } 3100 3101 3102 static int of_test_counter; 3103 3104 static void guest_test_of_handler(struct ex_regs *r) 3105 { 3106 of_test_counter++; 3107 } 3108 3109 static void svm_of_test_guest(struct svm_test *test) 3110 { 3111 struct far_pointer32 fp = { 3112 .offset = (uintptr_t)&&into, 3113 .selector = KERNEL_CS32, 3114 }; 3115 uintptr_t rsp; 3116 3117 asm volatile ("mov %%rsp, %0" : "=r"(rsp)); 3118 3119 if (fp.offset != (uintptr_t)&&into) { 3120 printf("Codee address too high.\n"); 3121 return; 3122 } 3123 3124 if ((u32)rsp != rsp) { 3125 printf("Stack address too high.\n"); 3126 } 3127 3128 asm goto("lcall *%0" : : "m" (fp) : "rax" : into); 3129 return; 3130 into: 3131 3132 asm volatile (".code32;" 3133 "movl $0x7fffffff, %eax;" 3134 "addl %eax, %eax;" 3135 "into;" 3136 "lret;" 3137 ".code64"); 3138 __builtin_unreachable(); 3139 } 3140 3141 static void svm_into_test(void) 3142 { 3143 handle_exception(OF_VECTOR, guest_test_of_handler); 3144 test_set_guest(svm_of_test_guest); 3145 report(svm_vmrun() == SVM_EXIT_VMMCALL && of_test_counter == 1, 3146 "#OF is generated in L2 exception handler0"); 3147 } 3148 3149 static int bp_test_counter; 3150 3151 static void guest_test_bp_handler(struct ex_regs *r) 3152 { 3153 bp_test_counter++; 3154 } 3155 3156 static void svm_bp_test_guest(struct svm_test *test) 3157 { 3158 asm volatile("int3"); 3159 } 3160 3161 static void svm_int3_test(void) 3162 { 3163 handle_exception(BP_VECTOR, guest_test_bp_handler); 3164 test_set_guest(svm_bp_test_guest); 3165 report(svm_vmrun() == SVM_EXIT_VMMCALL && bp_test_counter == 1, 3166 "#BP is handled in L2 exception handler"); 3167 } 3168 3169 static int nm_test_counter; 3170 3171 static void guest_test_nm_handler(struct ex_regs *r) 3172 { 3173 nm_test_counter++; 3174 write_cr0(read_cr0() & ~X86_CR0_TS); 3175 write_cr0(read_cr0() & ~X86_CR0_EM); 3176 } 3177 3178 static void svm_nm_test_guest(struct svm_test *test) 3179 { 3180 asm volatile("fnop"); 3181 } 3182 3183 /* This test checks that: 3184 * 3185 * (a) If CR0.TS is set in L2, #NM is handled by L2 when 3186 * just an L2 handler is registered. 3187 * 3188 * (b) If CR0.TS is cleared and CR0.EM is set, #NM is handled 3189 * by L2 when just an l2 handler is registered. 3190 * 3191 * (c) If CR0.TS and CR0.EM are cleared in L2, no exception 3192 * is generated. 3193 */ 3194 3195 static void svm_nm_test(void) 3196 { 3197 handle_exception(NM_VECTOR, guest_test_nm_handler); 3198 write_cr0(read_cr0() & ~X86_CR0_TS); 3199 test_set_guest(svm_nm_test_guest); 3200 3201 vmcb->save.cr0 = vmcb->save.cr0 | X86_CR0_TS; 3202 report(svm_vmrun() == SVM_EXIT_VMMCALL && nm_test_counter == 1, 3203 "fnop with CR0.TS set in L2, #NM is triggered"); 3204 3205 vmcb->save.cr0 = (vmcb->save.cr0 & ~X86_CR0_TS) | X86_CR0_EM; 3206 report(svm_vmrun() == SVM_EXIT_VMMCALL && nm_test_counter == 2, 3207 "fnop with CR0.EM set in L2, #NM is triggered"); 3208 3209 vmcb->save.cr0 = vmcb->save.cr0 & ~(X86_CR0_TS | X86_CR0_EM); 3210 report(svm_vmrun() == SVM_EXIT_VMMCALL && nm_test_counter == 2, 3211 "fnop with CR0.TS and CR0.EM unset no #NM excpetion"); 3212 } 3213 3214 3215 static bool check_lbr(u64 *from_excepted, u64 *to_expected) 3216 { 3217 u64 from = rdmsr(MSR_IA32_LASTBRANCHFROMIP); 3218 u64 to = rdmsr(MSR_IA32_LASTBRANCHTOIP); 3219 3220 if ((u64)from_excepted != from) { 3221 report(false, "MSR_IA32_LASTBRANCHFROMIP, expected=0x%lx, actual=0x%lx", 3222 (u64)from_excepted, from); 3223 return false; 3224 } 3225 3226 if ((u64)to_expected != to) { 3227 report(false, "MSR_IA32_LASTBRANCHFROMIP, expected=0x%lx, actual=0x%lx", 3228 (u64)from_excepted, from); 3229 return false; 3230 } 3231 3232 return true; 3233 } 3234 3235 static bool check_dbgctl(u64 dbgctl, u64 dbgctl_expected) 3236 { 3237 if (dbgctl != dbgctl_expected) { 3238 report(false, "Unexpected MSR_IA32_DEBUGCTLMSR value 0x%lx", dbgctl); 3239 return false; 3240 } 3241 return true; 3242 } 3243 3244 3245 #define DO_BRANCH(branch_name) \ 3246 asm volatile ( \ 3247 # branch_name "_from:" \ 3248 "jmp " # branch_name "_to\n" \ 3249 "nop\n" \ 3250 "nop\n" \ 3251 # branch_name "_to:" \ 3252 "nop\n" \ 3253 ) 3254 3255 3256 extern u64 guest_branch0_from, guest_branch0_to; 3257 extern u64 guest_branch2_from, guest_branch2_to; 3258 3259 extern u64 host_branch0_from, host_branch0_to; 3260 extern u64 host_branch2_from, host_branch2_to; 3261 extern u64 host_branch3_from, host_branch3_to; 3262 extern u64 host_branch4_from, host_branch4_to; 3263 3264 u64 dbgctl; 3265 3266 static void svm_lbrv_test_guest1(void) 3267 { 3268 /* 3269 * This guest expects the LBR to be already enabled when it starts, 3270 * it does a branch, and then disables the LBR and then checks. 3271 */ 3272 3273 DO_BRANCH(guest_branch0); 3274 3275 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3276 wrmsr(MSR_IA32_DEBUGCTLMSR, 0); 3277 3278 if (dbgctl != DEBUGCTLMSR_LBR) 3279 asm volatile("ud2\n"); 3280 if (rdmsr(MSR_IA32_DEBUGCTLMSR) != 0) 3281 asm volatile("ud2\n"); 3282 if (rdmsr(MSR_IA32_LASTBRANCHFROMIP) != (u64)&guest_branch0_from) 3283 asm volatile("ud2\n"); 3284 if (rdmsr(MSR_IA32_LASTBRANCHTOIP) != (u64)&guest_branch0_to) 3285 asm volatile("ud2\n"); 3286 3287 asm volatile ("vmmcall\n"); 3288 } 3289 3290 static void svm_lbrv_test_guest2(void) 3291 { 3292 /* 3293 * This guest expects the LBR to be disabled when it starts, 3294 * enables it, does a branch, disables it and then checks. 3295 */ 3296 3297 DO_BRANCH(guest_branch1); 3298 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3299 3300 if (dbgctl != 0) 3301 asm volatile("ud2\n"); 3302 3303 if (rdmsr(MSR_IA32_LASTBRANCHFROMIP) != (u64)&host_branch2_from) 3304 asm volatile("ud2\n"); 3305 if (rdmsr(MSR_IA32_LASTBRANCHTOIP) != (u64)&host_branch2_to) 3306 asm volatile("ud2\n"); 3307 3308 3309 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR); 3310 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3311 DO_BRANCH(guest_branch2); 3312 wrmsr(MSR_IA32_DEBUGCTLMSR, 0); 3313 3314 if (dbgctl != DEBUGCTLMSR_LBR) 3315 asm volatile("ud2\n"); 3316 if (rdmsr(MSR_IA32_LASTBRANCHFROMIP) != (u64)&guest_branch2_from) 3317 asm volatile("ud2\n"); 3318 if (rdmsr(MSR_IA32_LASTBRANCHTOIP) != (u64)&guest_branch2_to) 3319 asm volatile("ud2\n"); 3320 3321 asm volatile ("vmmcall\n"); 3322 } 3323 3324 static void svm_lbrv_test0(void) 3325 { 3326 report(true, "Basic LBR test"); 3327 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR); 3328 DO_BRANCH(host_branch0); 3329 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3330 wrmsr(MSR_IA32_DEBUGCTLMSR, 0); 3331 3332 check_dbgctl(dbgctl, DEBUGCTLMSR_LBR); 3333 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3334 check_dbgctl(dbgctl, 0); 3335 3336 check_lbr(&host_branch0_from, &host_branch0_to); 3337 } 3338 3339 static void svm_lbrv_test1(void) 3340 { 3341 report(true, "Test that without LBRV enabled, guest LBR state does 'leak' to the host(1)"); 3342 3343 vmcb->save.rip = (ulong)svm_lbrv_test_guest1; 3344 vmcb->control.virt_ext = 0; 3345 3346 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR); 3347 DO_BRANCH(host_branch1); 3348 SVM_BARE_VMRUN; 3349 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3350 3351 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 3352 report(false, "VMEXIT not due to vmmcall. Exit reason 0x%x", 3353 vmcb->control.exit_code); 3354 return; 3355 } 3356 3357 check_dbgctl(dbgctl, 0); 3358 check_lbr(&guest_branch0_from, &guest_branch0_to); 3359 } 3360 3361 static void svm_lbrv_test2(void) 3362 { 3363 report(true, "Test that without LBRV enabled, guest LBR state does 'leak' to the host(2)"); 3364 3365 vmcb->save.rip = (ulong)svm_lbrv_test_guest2; 3366 vmcb->control.virt_ext = 0; 3367 3368 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR); 3369 DO_BRANCH(host_branch2); 3370 wrmsr(MSR_IA32_DEBUGCTLMSR, 0); 3371 SVM_BARE_VMRUN; 3372 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3373 wrmsr(MSR_IA32_DEBUGCTLMSR, 0); 3374 3375 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 3376 report(false, "VMEXIT not due to vmmcall. Exit reason 0x%x", 3377 vmcb->control.exit_code); 3378 return; 3379 } 3380 3381 check_dbgctl(dbgctl, 0); 3382 check_lbr(&guest_branch2_from, &guest_branch2_to); 3383 } 3384 3385 static void svm_lbrv_nested_test1(void) 3386 { 3387 if (!lbrv_supported()) { 3388 report_skip("LBRV not supported in the guest"); 3389 return; 3390 } 3391 3392 report(true, "Test that with LBRV enabled, guest LBR state doesn't leak (1)"); 3393 vmcb->save.rip = (ulong)svm_lbrv_test_guest1; 3394 vmcb->control.virt_ext = LBR_CTL_ENABLE_MASK; 3395 vmcb->save.dbgctl = DEBUGCTLMSR_LBR; 3396 3397 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR); 3398 DO_BRANCH(host_branch3); 3399 SVM_BARE_VMRUN; 3400 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3401 wrmsr(MSR_IA32_DEBUGCTLMSR, 0); 3402 3403 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 3404 report(false, "VMEXIT not due to vmmcall. Exit reason 0x%x", 3405 vmcb->control.exit_code); 3406 return; 3407 } 3408 3409 if (vmcb->save.dbgctl != 0) { 3410 report(false, "unexpected virtual guest MSR_IA32_DEBUGCTLMSR value 0x%lx", vmcb->save.dbgctl); 3411 return; 3412 } 3413 3414 check_dbgctl(dbgctl, DEBUGCTLMSR_LBR); 3415 check_lbr(&host_branch3_from, &host_branch3_to); 3416 } 3417 static void svm_lbrv_nested_test2(void) 3418 { 3419 if (!lbrv_supported()) { 3420 report_skip("LBRV not supported in the guest"); 3421 return; 3422 } 3423 3424 report(true, "Test that with LBRV enabled, guest LBR state doesn't leak (2)"); 3425 vmcb->save.rip = (ulong)svm_lbrv_test_guest2; 3426 vmcb->control.virt_ext = LBR_CTL_ENABLE_MASK; 3427 3428 vmcb->save.dbgctl = 0; 3429 vmcb->save.br_from = (u64)&host_branch2_from; 3430 vmcb->save.br_to = (u64)&host_branch2_to; 3431 3432 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR); 3433 DO_BRANCH(host_branch4); 3434 SVM_BARE_VMRUN; 3435 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR); 3436 wrmsr(MSR_IA32_DEBUGCTLMSR, 0); 3437 3438 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) { 3439 report(false, "VMEXIT not due to vmmcall. Exit reason 0x%x", 3440 vmcb->control.exit_code); 3441 return; 3442 } 3443 3444 check_dbgctl(dbgctl, DEBUGCTLMSR_LBR); 3445 check_lbr(&host_branch4_from, &host_branch4_to); 3446 } 3447 3448 3449 // test that a nested guest which does enable INTR interception 3450 // but doesn't enable virtual interrupt masking works 3451 3452 static volatile int dummy_isr_recevied; 3453 static void dummy_isr(isr_regs_t *regs) 3454 { 3455 dummy_isr_recevied++; 3456 eoi(); 3457 } 3458 3459 3460 static volatile int nmi_recevied; 3461 static void dummy_nmi_handler(struct ex_regs *regs) 3462 { 3463 nmi_recevied++; 3464 } 3465 3466 3467 static void svm_intr_intercept_mix_run_guest(volatile int *counter, int expected_vmexit) 3468 { 3469 if (counter) 3470 *counter = 0; 3471 3472 sti(); // host IF value should not matter 3473 clgi(); // vmrun will set back GI to 1 3474 3475 svm_vmrun(); 3476 3477 if (counter) 3478 report(!*counter, "No interrupt expected"); 3479 3480 stgi(); 3481 3482 if (counter) 3483 report(*counter == 1, "Interrupt is expected"); 3484 3485 report (vmcb->control.exit_code == expected_vmexit, "Test expected VM exit"); 3486 report(vmcb->save.rflags & X86_EFLAGS_IF, "Guest should have EFLAGS.IF set now"); 3487 cli(); 3488 } 3489 3490 3491 // subtest: test that enabling EFLAGS.IF is enought to trigger an interrupt 3492 static void svm_intr_intercept_mix_if_guest(struct svm_test *test) 3493 { 3494 asm volatile("nop;nop;nop;nop"); 3495 report(!dummy_isr_recevied, "No interrupt expected"); 3496 sti(); 3497 asm volatile("nop"); 3498 report(0, "must not reach here"); 3499 } 3500 3501 static void svm_intr_intercept_mix_if(void) 3502 { 3503 // make a physical interrupt to be pending 3504 handle_irq(0x55, dummy_isr); 3505 3506 vmcb->control.intercept |= (1 << INTERCEPT_INTR); 3507 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; 3508 vmcb->save.rflags &= ~X86_EFLAGS_IF; 3509 3510 test_set_guest(svm_intr_intercept_mix_if_guest); 3511 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | 0x55, 0); 3512 svm_intr_intercept_mix_run_guest(&dummy_isr_recevied, SVM_EXIT_INTR); 3513 } 3514 3515 3516 // subtest: test that a clever guest can trigger an interrupt by setting GIF 3517 // if GIF is not intercepted 3518 static void svm_intr_intercept_mix_gif_guest(struct svm_test *test) 3519 { 3520 3521 asm volatile("nop;nop;nop;nop"); 3522 report(!dummy_isr_recevied, "No interrupt expected"); 3523 3524 // clear GIF and enable IF 3525 // that should still not cause VM exit 3526 clgi(); 3527 sti(); 3528 asm volatile("nop"); 3529 report(!dummy_isr_recevied, "No interrupt expected"); 3530 3531 stgi(); 3532 asm volatile("nop"); 3533 report(0, "must not reach here"); 3534 } 3535 3536 static void svm_intr_intercept_mix_gif(void) 3537 { 3538 handle_irq(0x55, dummy_isr); 3539 3540 vmcb->control.intercept |= (1 << INTERCEPT_INTR); 3541 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; 3542 vmcb->save.rflags &= ~X86_EFLAGS_IF; 3543 3544 test_set_guest(svm_intr_intercept_mix_gif_guest); 3545 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | 0x55, 0); 3546 svm_intr_intercept_mix_run_guest(&dummy_isr_recevied, SVM_EXIT_INTR); 3547 } 3548 3549 3550 3551 // subtest: test that a clever guest can trigger an interrupt by setting GIF 3552 // if GIF is not intercepted and interrupt comes after guest 3553 // started running 3554 static void svm_intr_intercept_mix_gif_guest2(struct svm_test *test) 3555 { 3556 asm volatile("nop;nop;nop;nop"); 3557 report(!dummy_isr_recevied, "No interrupt expected"); 3558 3559 clgi(); 3560 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | 0x55, 0); 3561 report(!dummy_isr_recevied, "No interrupt expected"); 3562 3563 stgi(); 3564 asm volatile("nop"); 3565 report(0, "must not reach here"); 3566 } 3567 3568 static void svm_intr_intercept_mix_gif2(void) 3569 { 3570 handle_irq(0x55, dummy_isr); 3571 3572 vmcb->control.intercept |= (1 << INTERCEPT_INTR); 3573 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; 3574 vmcb->save.rflags |= X86_EFLAGS_IF; 3575 3576 test_set_guest(svm_intr_intercept_mix_gif_guest2); 3577 svm_intr_intercept_mix_run_guest(&dummy_isr_recevied, SVM_EXIT_INTR); 3578 } 3579 3580 3581 // subtest: test that pending NMI will be handled when guest enables GIF 3582 static void svm_intr_intercept_mix_nmi_guest(struct svm_test *test) 3583 { 3584 asm volatile("nop;nop;nop;nop"); 3585 report(!nmi_recevied, "No NMI expected"); 3586 cli(); // should have no effect 3587 3588 clgi(); 3589 asm volatile("nop"); 3590 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_NMI, 0); 3591 sti(); // should have no effect 3592 asm volatile("nop"); 3593 report(!nmi_recevied, "No NMI expected"); 3594 3595 stgi(); 3596 asm volatile("nop"); 3597 report(0, "must not reach here"); 3598 } 3599 3600 static void svm_intr_intercept_mix_nmi(void) 3601 { 3602 handle_exception(2, dummy_nmi_handler); 3603 3604 vmcb->control.intercept |= (1 << INTERCEPT_NMI); 3605 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; 3606 vmcb->save.rflags |= X86_EFLAGS_IF; 3607 3608 test_set_guest(svm_intr_intercept_mix_nmi_guest); 3609 svm_intr_intercept_mix_run_guest(&nmi_recevied, SVM_EXIT_NMI); 3610 } 3611 3612 // test that pending SMI will be handled when guest enables GIF 3613 // TODO: can't really count #SMIs so just test that guest doesn't hang 3614 // and VMexits on SMI 3615 static void svm_intr_intercept_mix_smi_guest(struct svm_test *test) 3616 { 3617 asm volatile("nop;nop;nop;nop"); 3618 3619 clgi(); 3620 asm volatile("nop"); 3621 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_SMI, 0); 3622 sti(); // should have no effect 3623 asm volatile("nop"); 3624 stgi(); 3625 asm volatile("nop"); 3626 report(0, "must not reach here"); 3627 } 3628 3629 static void svm_intr_intercept_mix_smi(void) 3630 { 3631 vmcb->control.intercept |= (1 << INTERCEPT_SMI); 3632 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; 3633 test_set_guest(svm_intr_intercept_mix_smi_guest); 3634 svm_intr_intercept_mix_run_guest(NULL, SVM_EXIT_SMI); 3635 } 3636 3637 struct svm_test svm_tests[] = { 3638 { "null", default_supported, default_prepare, 3639 default_prepare_gif_clear, null_test, 3640 default_finished, null_check }, 3641 { "vmrun", default_supported, default_prepare, 3642 default_prepare_gif_clear, test_vmrun, 3643 default_finished, check_vmrun }, 3644 { "ioio", default_supported, prepare_ioio, 3645 default_prepare_gif_clear, test_ioio, 3646 ioio_finished, check_ioio }, 3647 { "vmrun intercept check", default_supported, prepare_no_vmrun_int, 3648 default_prepare_gif_clear, null_test, default_finished, 3649 check_no_vmrun_int }, 3650 { "rsm", default_supported, 3651 prepare_rsm_intercept, default_prepare_gif_clear, 3652 test_rsm_intercept, finished_rsm_intercept, check_rsm_intercept }, 3653 { "cr3 read intercept", default_supported, 3654 prepare_cr3_intercept, default_prepare_gif_clear, 3655 test_cr3_intercept, default_finished, check_cr3_intercept }, 3656 { "cr3 read nointercept", default_supported, default_prepare, 3657 default_prepare_gif_clear, test_cr3_intercept, default_finished, 3658 check_cr3_nointercept }, 3659 { "cr3 read intercept emulate", smp_supported, 3660 prepare_cr3_intercept_bypass, default_prepare_gif_clear, 3661 test_cr3_intercept_bypass, default_finished, check_cr3_intercept }, 3662 { "dr intercept check", default_supported, prepare_dr_intercept, 3663 default_prepare_gif_clear, test_dr_intercept, dr_intercept_finished, 3664 check_dr_intercept }, 3665 { "next_rip", next_rip_supported, prepare_next_rip, 3666 default_prepare_gif_clear, test_next_rip, 3667 default_finished, check_next_rip }, 3668 { "msr intercept check", default_supported, prepare_msr_intercept, 3669 default_prepare_gif_clear, test_msr_intercept, 3670 msr_intercept_finished, check_msr_intercept }, 3671 { "mode_switch", default_supported, prepare_mode_switch, 3672 default_prepare_gif_clear, test_mode_switch, 3673 mode_switch_finished, check_mode_switch }, 3674 { "asid_zero", default_supported, prepare_asid_zero, 3675 default_prepare_gif_clear, test_asid_zero, 3676 default_finished, check_asid_zero }, 3677 { "sel_cr0_bug", default_supported, sel_cr0_bug_prepare, 3678 default_prepare_gif_clear, sel_cr0_bug_test, 3679 sel_cr0_bug_finished, sel_cr0_bug_check }, 3680 { "npt_nx", npt_supported, npt_nx_prepare, 3681 default_prepare_gif_clear, null_test, 3682 default_finished, npt_nx_check }, 3683 { "npt_np", npt_supported, npt_np_prepare, 3684 default_prepare_gif_clear, npt_np_test, 3685 default_finished, npt_np_check }, 3686 { "npt_us", npt_supported, npt_us_prepare, 3687 default_prepare_gif_clear, npt_us_test, 3688 default_finished, npt_us_check }, 3689 { "npt_rw", npt_supported, npt_rw_prepare, 3690 default_prepare_gif_clear, npt_rw_test, 3691 default_finished, npt_rw_check }, 3692 { "npt_rw_pfwalk", npt_supported, npt_rw_pfwalk_prepare, 3693 default_prepare_gif_clear, null_test, 3694 default_finished, npt_rw_pfwalk_check }, 3695 { "npt_l1mmio", npt_supported, npt_l1mmio_prepare, 3696 default_prepare_gif_clear, npt_l1mmio_test, 3697 default_finished, npt_l1mmio_check }, 3698 { "npt_rw_l1mmio", npt_supported, npt_rw_l1mmio_prepare, 3699 default_prepare_gif_clear, npt_rw_l1mmio_test, 3700 default_finished, npt_rw_l1mmio_check }, 3701 { "tsc_adjust", tsc_adjust_supported, tsc_adjust_prepare, 3702 default_prepare_gif_clear, tsc_adjust_test, 3703 default_finished, tsc_adjust_check }, 3704 { "latency_run_exit", default_supported, latency_prepare, 3705 default_prepare_gif_clear, latency_test, 3706 latency_finished, latency_check }, 3707 { "latency_run_exit_clean", default_supported, latency_prepare, 3708 default_prepare_gif_clear, latency_test, 3709 latency_finished_clean, latency_check }, 3710 { "latency_svm_insn", default_supported, lat_svm_insn_prepare, 3711 default_prepare_gif_clear, null_test, 3712 lat_svm_insn_finished, lat_svm_insn_check }, 3713 { "exc_inject", default_supported, exc_inject_prepare, 3714 default_prepare_gif_clear, exc_inject_test, 3715 exc_inject_finished, exc_inject_check }, 3716 { "pending_event", default_supported, pending_event_prepare, 3717 default_prepare_gif_clear, 3718 pending_event_test, pending_event_finished, pending_event_check }, 3719 { "pending_event_cli", default_supported, pending_event_cli_prepare, 3720 pending_event_cli_prepare_gif_clear, 3721 pending_event_cli_test, pending_event_cli_finished, 3722 pending_event_cli_check }, 3723 { "interrupt", default_supported, interrupt_prepare, 3724 default_prepare_gif_clear, interrupt_test, 3725 interrupt_finished, interrupt_check }, 3726 { "nmi", default_supported, nmi_prepare, 3727 default_prepare_gif_clear, nmi_test, 3728 nmi_finished, nmi_check }, 3729 { "nmi_hlt", smp_supported, nmi_prepare, 3730 default_prepare_gif_clear, nmi_hlt_test, 3731 nmi_hlt_finished, nmi_hlt_check }, 3732 { "virq_inject", default_supported, virq_inject_prepare, 3733 default_prepare_gif_clear, virq_inject_test, 3734 virq_inject_finished, virq_inject_check }, 3735 { "reg_corruption", default_supported, reg_corruption_prepare, 3736 default_prepare_gif_clear, reg_corruption_test, 3737 reg_corruption_finished, reg_corruption_check }, 3738 { "svm_init_startup_test", smp_supported, init_startup_prepare, 3739 default_prepare_gif_clear, null_test, 3740 init_startup_finished, init_startup_check }, 3741 { "svm_init_intercept_test", smp_supported, init_intercept_prepare, 3742 default_prepare_gif_clear, init_intercept_test, 3743 init_intercept_finished, init_intercept_check, .on_vcpu = 2 }, 3744 { "host_rflags", default_supported, host_rflags_prepare, 3745 host_rflags_prepare_gif_clear, host_rflags_test, 3746 host_rflags_finished, host_rflags_check }, 3747 { "vgif", vgif_supported, prepare_vgif_enabled, 3748 default_prepare_gif_clear, test_vgif, vgif_finished, 3749 vgif_check }, 3750 TEST(svm_cr4_osxsave_test), 3751 TEST(svm_guest_state_test), 3752 TEST(svm_npt_rsvd_bits_test), 3753 TEST(svm_vmrun_errata_test), 3754 TEST(svm_vmload_vmsave), 3755 TEST(svm_test_singlestep), 3756 TEST(svm_nm_test), 3757 TEST(svm_int3_test), 3758 TEST(svm_into_test), 3759 TEST(svm_lbrv_test0), 3760 TEST(svm_lbrv_test1), 3761 TEST(svm_lbrv_test2), 3762 TEST(svm_lbrv_nested_test1), 3763 TEST(svm_lbrv_nested_test2), 3764 TEST(svm_intr_intercept_mix_if), 3765 TEST(svm_intr_intercept_mix_gif), 3766 TEST(svm_intr_intercept_mix_gif2), 3767 TEST(svm_intr_intercept_mix_nmi), 3768 TEST(svm_intr_intercept_mix_smi), 3769 TEST(svm_tsc_scale_test), 3770 TEST(pause_filter_test), 3771 { NULL, NULL, NULL, NULL, NULL, NULL, NULL } 3772 }; 3773