1 #include "svm.h"
2 #include "libcflat.h"
3 #include "processor.h"
4 #include "desc.h"
5 #include "msr.h"
6 #include "vm.h"
7 #include "smp.h"
8 #include "alloc_page.h"
9 #include "isr.h"
10 #include "apic.h"
11 #include "delay.h"
12 #include "util.h"
13 #include "x86/usermode.h"
14 #include "vmalloc.h"
15
16 #define SVM_EXIT_MAX_DR_INTERCEPT 0x3f
17
18 #define LATENCY_RUNS 1000000
19
20 u64 tsc_start;
21 u64 tsc_end;
22
23 u64 vmrun_sum, vmexit_sum;
24 u64 vmsave_sum, vmload_sum;
25 u64 stgi_sum, clgi_sum;
26 u64 latvmrun_max;
27 u64 latvmrun_min;
28 u64 latvmexit_max;
29 u64 latvmexit_min;
30 u64 latvmload_max;
31 u64 latvmload_min;
32 u64 latvmsave_max;
33 u64 latvmsave_min;
34 u64 latstgi_max;
35 u64 latstgi_min;
36 u64 latclgi_max;
37 u64 latclgi_min;
38 u64 runs;
39
null_test(struct svm_test * test)40 static void null_test(struct svm_test *test)
41 {
42 }
43
null_check(struct svm_test * test)44 static bool null_check(struct svm_test *test)
45 {
46 return vmcb->control.exit_code == SVM_EXIT_VMMCALL;
47 }
48
prepare_no_vmrun_int(struct svm_test * test)49 static void prepare_no_vmrun_int(struct svm_test *test)
50 {
51 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMRUN);
52 }
53
check_no_vmrun_int(struct svm_test * test)54 static bool check_no_vmrun_int(struct svm_test *test)
55 {
56 return vmcb->control.exit_code == SVM_EXIT_ERR;
57 }
58
test_vmrun(struct svm_test * test)59 static void test_vmrun(struct svm_test *test)
60 {
61 asm volatile ("vmrun %0" : : "a"(virt_to_phys(vmcb)));
62 }
63
check_vmrun(struct svm_test * test)64 static bool check_vmrun(struct svm_test *test)
65 {
66 return vmcb->control.exit_code == SVM_EXIT_VMRUN;
67 }
68
prepare_rsm_intercept(struct svm_test * test)69 static void prepare_rsm_intercept(struct svm_test *test)
70 {
71 default_prepare(test);
72 vmcb->control.intercept |= 1 << INTERCEPT_RSM;
73 vmcb->control.intercept_exceptions |= (1ULL << UD_VECTOR);
74 }
75
test_rsm_intercept(struct svm_test * test)76 static void test_rsm_intercept(struct svm_test *test)
77 {
78 asm volatile ("rsm" : : : "memory");
79 }
80
check_rsm_intercept(struct svm_test * test)81 static bool check_rsm_intercept(struct svm_test *test)
82 {
83 return get_test_stage(test) == 2;
84 }
85
finished_rsm_intercept(struct svm_test * test)86 static bool finished_rsm_intercept(struct svm_test *test)
87 {
88 switch (get_test_stage(test)) {
89 case 0:
90 if (vmcb->control.exit_code != SVM_EXIT_RSM) {
91 report_fail("VMEXIT not due to rsm. Exit reason 0x%x",
92 vmcb->control.exit_code);
93 return true;
94 }
95 vmcb->control.intercept &= ~(1 << INTERCEPT_RSM);
96 inc_test_stage(test);
97 break;
98
99 case 1:
100 if (vmcb->control.exit_code != SVM_EXIT_EXCP_BASE + UD_VECTOR) {
101 report_fail("VMEXIT not due to #UD. Exit reason 0x%x",
102 vmcb->control.exit_code);
103 return true;
104 }
105 vmcb->save.rip += 2;
106 inc_test_stage(test);
107 break;
108
109 default:
110 return true;
111 }
112 return get_test_stage(test) == 2;
113 }
114
prepare_cr3_intercept(struct svm_test * test)115 static void prepare_cr3_intercept(struct svm_test *test)
116 {
117 default_prepare(test);
118 vmcb->control.intercept_cr_read |= 1 << 3;
119 }
120
test_cr3_intercept(struct svm_test * test)121 static void test_cr3_intercept(struct svm_test *test)
122 {
123 asm volatile ("mov %%cr3, %0" : "=r"(test->scratch) : : "memory");
124 }
125
check_cr3_intercept(struct svm_test * test)126 static bool check_cr3_intercept(struct svm_test *test)
127 {
128 return vmcb->control.exit_code == SVM_EXIT_READ_CR3;
129 }
130
check_cr3_nointercept(struct svm_test * test)131 static bool check_cr3_nointercept(struct svm_test *test)
132 {
133 return null_check(test) && test->scratch == read_cr3();
134 }
135
corrupt_cr3_intercept_bypass(void * _test)136 static void corrupt_cr3_intercept_bypass(void *_test)
137 {
138 struct svm_test *test = _test;
139 extern volatile u32 mmio_insn;
140
141 while (!__sync_bool_compare_and_swap(&test->scratch, 1, 2))
142 pause();
143 pause();
144 pause();
145 pause();
146 mmio_insn = 0x90d8200f; // mov %cr3, %rax; nop
147 }
148
prepare_cr3_intercept_bypass(struct svm_test * test)149 static void prepare_cr3_intercept_bypass(struct svm_test *test)
150 {
151 default_prepare(test);
152 vmcb->control.intercept_cr_read |= 1 << 3;
153 on_cpu_async(1, corrupt_cr3_intercept_bypass, test);
154 }
155
test_cr3_intercept_bypass(struct svm_test * test)156 static void test_cr3_intercept_bypass(struct svm_test *test)
157 {
158 ulong a = 0xa0000;
159
160 test->scratch = 1;
161 while (test->scratch != 2)
162 barrier();
163
164 asm volatile ("mmio_insn: mov %0, (%0); nop"
165 : "+a"(a) : : "memory");
166 test->scratch = a;
167 }
168
prepare_dr_intercept(struct svm_test * test)169 static void prepare_dr_intercept(struct svm_test *test)
170 {
171 default_prepare(test);
172 vmcb->control.intercept_dr_read = 0xff;
173 vmcb->control.intercept_dr_write = 0xff;
174 }
175
test_dr_intercept(struct svm_test * test)176 static void test_dr_intercept(struct svm_test *test)
177 {
178 unsigned int i, failcnt = 0;
179
180 /* Loop testing debug register reads */
181 for (i = 0; i < 8; i++) {
182
183 switch (i) {
184 case 0:
185 asm volatile ("mov %%dr0, %0" : "=r"(test->scratch) : : "memory");
186 break;
187 case 1:
188 asm volatile ("mov %%dr1, %0" : "=r"(test->scratch) : : "memory");
189 break;
190 case 2:
191 asm volatile ("mov %%dr2, %0" : "=r"(test->scratch) : : "memory");
192 break;
193 case 3:
194 asm volatile ("mov %%dr3, %0" : "=r"(test->scratch) : : "memory");
195 break;
196 case 4:
197 asm volatile ("mov %%dr4, %0" : "=r"(test->scratch) : : "memory");
198 break;
199 case 5:
200 asm volatile ("mov %%dr5, %0" : "=r"(test->scratch) : : "memory");
201 break;
202 case 6:
203 asm volatile ("mov %%dr6, %0" : "=r"(test->scratch) : : "memory");
204 break;
205 case 7:
206 asm volatile ("mov %%dr7, %0" : "=r"(test->scratch) : : "memory");
207 break;
208 }
209
210 if (test->scratch != i) {
211 report_fail("dr%u read intercept", i);
212 failcnt++;
213 }
214 }
215
216 /* Loop testing debug register writes */
217 for (i = 0; i < 8; i++) {
218
219 switch (i) {
220 case 0:
221 asm volatile ("mov %0, %%dr0" : : "r"(test->scratch) : "memory");
222 break;
223 case 1:
224 asm volatile ("mov %0, %%dr1" : : "r"(test->scratch) : "memory");
225 break;
226 case 2:
227 asm volatile ("mov %0, %%dr2" : : "r"(test->scratch) : "memory");
228 break;
229 case 3:
230 asm volatile ("mov %0, %%dr3" : : "r"(test->scratch) : "memory");
231 break;
232 case 4:
233 asm volatile ("mov %0, %%dr4" : : "r"(test->scratch) : "memory");
234 break;
235 case 5:
236 asm volatile ("mov %0, %%dr5" : : "r"(test->scratch) : "memory");
237 break;
238 case 6:
239 asm volatile ("mov %0, %%dr6" : : "r"(test->scratch) : "memory");
240 break;
241 case 7:
242 asm volatile ("mov %0, %%dr7" : : "r"(test->scratch) : "memory");
243 break;
244 }
245
246 if (test->scratch != i) {
247 report_fail("dr%u write intercept", i);
248 failcnt++;
249 }
250 }
251
252 test->scratch = failcnt;
253 }
254
dr_intercept_finished(struct svm_test * test)255 static bool dr_intercept_finished(struct svm_test *test)
256 {
257 ulong n = (vmcb->control.exit_code - SVM_EXIT_READ_DR0);
258
259 /* Only expect DR intercepts */
260 if (n > (SVM_EXIT_MAX_DR_INTERCEPT - SVM_EXIT_READ_DR0))
261 return true;
262
263 /*
264 * Compute debug register number.
265 * Per Appendix C "SVM Intercept Exit Codes" of AMD64 Architecture
266 * Programmer's Manual Volume 2 - System Programming:
267 * http://support.amd.com/TechDocs/24593.pdf
268 * there are 16 VMEXIT codes each for DR read and write.
269 */
270 test->scratch = (n % 16);
271
272 /* Jump over MOV instruction */
273 vmcb->save.rip += 3;
274
275 return false;
276 }
277
check_dr_intercept(struct svm_test * test)278 static bool check_dr_intercept(struct svm_test *test)
279 {
280 return !test->scratch;
281 }
282
next_rip_supported(void)283 static bool next_rip_supported(void)
284 {
285 return this_cpu_has(X86_FEATURE_NRIPS);
286 }
287
prepare_next_rip(struct svm_test * test)288 static void prepare_next_rip(struct svm_test *test)
289 {
290 vmcb->control.intercept |= (1ULL << INTERCEPT_RDTSC);
291 }
292
293
test_next_rip(struct svm_test * test)294 static void test_next_rip(struct svm_test *test)
295 {
296 asm volatile ("rdtsc\n\t"
297 ".globl exp_next_rip\n\t"
298 "exp_next_rip:\n\t" ::: "eax", "edx");
299 }
300
check_next_rip(struct svm_test * test)301 static bool check_next_rip(struct svm_test *test)
302 {
303 extern char exp_next_rip;
304 unsigned long address = (unsigned long)&exp_next_rip;
305
306 return address == vmcb->control.next_rip;
307 }
308
309 extern u8 *msr_bitmap;
310
prepare_msr_intercept(struct svm_test * test)311 static void prepare_msr_intercept(struct svm_test *test)
312 {
313 default_prepare(test);
314 vmcb->control.intercept |= (1ULL << INTERCEPT_MSR_PROT);
315 vmcb->control.intercept_exceptions |= (1ULL << GP_VECTOR);
316 memset(msr_bitmap, 0xff, MSR_BITMAP_SIZE);
317 }
318
test_msr_intercept(struct svm_test * test)319 static void test_msr_intercept(struct svm_test *test)
320 {
321 unsigned long msr_value = 0xef8056791234abcd; /* Arbitrary value */
322 unsigned long msr_index;
323
324 for (msr_index = 0; msr_index <= 0xc0011fff; msr_index++) {
325 if (msr_index == 0xC0010131 /* MSR_SEV_STATUS */) {
326 /*
327 * Per section 15.34.10 "SEV_STATUS MSR" of AMD64 Architecture
328 * Programmer's Manual volume 2 - System Programming:
329 * http://support.amd.com/TechDocs/24593.pdf
330 * SEV_STATUS MSR (C001_0131) is a non-interceptable MSR.
331 */
332 continue;
333 }
334
335 /* Skips gaps between supported MSR ranges */
336 if (msr_index == 0x2000)
337 msr_index = 0xc0000000;
338 else if (msr_index == 0xc0002000)
339 msr_index = 0xc0010000;
340
341 test->scratch = -1;
342
343 rdmsr(msr_index);
344
345 /* Check that a read intercept occurred for MSR at msr_index */
346 if (test->scratch != msr_index)
347 report_fail("MSR 0x%lx read intercept", msr_index);
348
349 /*
350 * Poor man approach to generate a value that
351 * seems arbitrary each time around the loop.
352 */
353 msr_value += (msr_value << 1);
354
355 wrmsr(msr_index, msr_value);
356
357 /* Check that a write intercept occurred for MSR with msr_value */
358 if (test->scratch != msr_value)
359 report_fail("MSR 0x%lx write intercept", msr_index);
360 }
361
362 test->scratch = -2;
363 }
364
msr_intercept_finished(struct svm_test * test)365 static bool msr_intercept_finished(struct svm_test *test)
366 {
367 u32 exit_code = vmcb->control.exit_code;
368 u64 exit_info_1;
369 u8 *opcode;
370
371 if (exit_code == SVM_EXIT_MSR) {
372 exit_info_1 = vmcb->control.exit_info_1;
373 } else {
374 /*
375 * If #GP exception occurs instead, check that it was
376 * for RDMSR/WRMSR and set exit_info_1 accordingly.
377 */
378
379 if (exit_code != (SVM_EXIT_EXCP_BASE + GP_VECTOR))
380 return true;
381
382 opcode = (u8 *)vmcb->save.rip;
383 if (opcode[0] != 0x0f)
384 return true;
385
386 switch (opcode[1]) {
387 case 0x30: /* WRMSR */
388 exit_info_1 = 1;
389 break;
390 case 0x32: /* RDMSR */
391 exit_info_1 = 0;
392 break;
393 default:
394 return true;
395 }
396
397 /*
398 * Warn that #GP exception occurred instead.
399 * RCX holds the MSR index.
400 */
401 printf("%s 0x%lx #GP exception\n",
402 exit_info_1 ? "WRMSR" : "RDMSR", get_regs().rcx);
403 }
404
405 /* Jump over RDMSR/WRMSR instruction */
406 vmcb->save.rip += 2;
407
408 /*
409 * Test whether the intercept was for RDMSR/WRMSR.
410 * For RDMSR, test->scratch is set to the MSR index;
411 * RCX holds the MSR index.
412 * For WRMSR, test->scratch is set to the MSR value;
413 * RDX holds the upper 32 bits of the MSR value,
414 * while RAX hold its lower 32 bits.
415 */
416 if (exit_info_1)
417 test->scratch =
418 ((get_regs().rdx << 32) | (vmcb->save.rax & 0xffffffff));
419 else
420 test->scratch = get_regs().rcx;
421
422 return false;
423 }
424
check_msr_intercept(struct svm_test * test)425 static bool check_msr_intercept(struct svm_test *test)
426 {
427 memset(msr_bitmap, 0, MSR_BITMAP_SIZE);
428 return (test->scratch == -2);
429 }
430
prepare_mode_switch(struct svm_test * test)431 static void prepare_mode_switch(struct svm_test *test)
432 {
433 vmcb->control.intercept_exceptions |= (1ULL << GP_VECTOR)
434 | (1ULL << UD_VECTOR)
435 | (1ULL << DF_VECTOR)
436 | (1ULL << PF_VECTOR);
437 test->scratch = 0;
438 }
439
test_mode_switch(struct svm_test * test)440 static void test_mode_switch(struct svm_test *test)
441 {
442 asm volatile(" cli\n"
443 " ljmp *1f\n" /* jump to 32-bit code segment */
444 "1:\n"
445 " .long 2f\n"
446 " .long " xstr(KERNEL_CS32) "\n"
447 ".code32\n"
448 "2:\n"
449 " movl %%cr0, %%eax\n"
450 " btcl $31, %%eax\n" /* clear PG */
451 " movl %%eax, %%cr0\n"
452 " movl $0xc0000080, %%ecx\n" /* EFER */
453 " rdmsr\n"
454 " btcl $8, %%eax\n" /* clear LME */
455 " wrmsr\n"
456 " movl %%cr4, %%eax\n"
457 " btcl $5, %%eax\n" /* clear PAE */
458 " movl %%eax, %%cr4\n"
459 " movw %[ds16], %%ax\n"
460 " movw %%ax, %%ds\n"
461 " ljmpl %[cs16], $3f\n" /* jump to 16 bit protected-mode */
462 ".code16\n"
463 "3:\n"
464 " movl %%cr0, %%eax\n"
465 " btcl $0, %%eax\n" /* clear PE */
466 " movl %%eax, %%cr0\n"
467 " ljmpl $0, $4f\n" /* jump to real-mode */
468 "4:\n"
469 " vmmcall\n"
470 " movl %%cr0, %%eax\n"
471 " btsl $0, %%eax\n" /* set PE */
472 " movl %%eax, %%cr0\n"
473 " ljmpl %[cs32], $5f\n" /* back to protected mode */
474 ".code32\n"
475 "5:\n"
476 " movl %%cr4, %%eax\n"
477 " btsl $5, %%eax\n" /* set PAE */
478 " movl %%eax, %%cr4\n"
479 " movl $0xc0000080, %%ecx\n" /* EFER */
480 " rdmsr\n"
481 " btsl $8, %%eax\n" /* set LME */
482 " wrmsr\n"
483 " movl %%cr0, %%eax\n"
484 " btsl $31, %%eax\n" /* set PG */
485 " movl %%eax, %%cr0\n"
486 " ljmpl %[cs64], $6f\n" /* back to long mode */
487 ".code64\n\t"
488 "6:\n"
489 " vmmcall\n"
490 :: [cs16] "i"(KERNEL_CS16), [ds16] "i"(KERNEL_DS16),
491 [cs32] "i"(KERNEL_CS32), [cs64] "i"(KERNEL_CS64)
492 : "rax", "rbx", "rcx", "rdx", "memory");
493 }
494
mode_switch_finished(struct svm_test * test)495 static bool mode_switch_finished(struct svm_test *test)
496 {
497 u64 cr0, cr4, efer;
498
499 cr0 = vmcb->save.cr0;
500 cr4 = vmcb->save.cr4;
501 efer = vmcb->save.efer;
502
503 /* Only expect VMMCALL intercepts */
504 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL)
505 return true;
506
507 /* Jump over VMMCALL instruction */
508 vmcb->save.rip += 3;
509
510 /* Do sanity checks */
511 switch (test->scratch) {
512 case 0:
513 /* Test should be in real mode now - check for this */
514 if ((cr0 & 0x80000001) || /* CR0.PG, CR0.PE */
515 (cr4 & 0x00000020) || /* CR4.PAE */
516 (efer & 0x00000500)) /* EFER.LMA, EFER.LME */
517 return true;
518 break;
519 case 2:
520 /* Test should be back in long-mode now - check for this */
521 if (((cr0 & 0x80000001) != 0x80000001) || /* CR0.PG, CR0.PE */
522 ((cr4 & 0x00000020) != 0x00000020) || /* CR4.PAE */
523 ((efer & 0x00000500) != 0x00000500)) /* EFER.LMA, EFER.LME */
524 return true;
525 break;
526 }
527
528 /* one step forward */
529 test->scratch += 1;
530
531 return test->scratch == 2;
532 }
533
check_mode_switch(struct svm_test * test)534 static bool check_mode_switch(struct svm_test *test)
535 {
536 return test->scratch == 2;
537 }
538
539 extern u8 *io_bitmap;
540
prepare_ioio(struct svm_test * test)541 static void prepare_ioio(struct svm_test *test)
542 {
543 vmcb->control.intercept |= (1ULL << INTERCEPT_IOIO_PROT);
544 test->scratch = 0;
545 memset(io_bitmap, 0, 8192);
546 io_bitmap[8192] = 0xFF;
547 }
548
test_ioio(struct svm_test * test)549 static void test_ioio(struct svm_test *test)
550 {
551 // stage 0, test IO pass
552 inb(0x5000);
553 outb(0x0, 0x5000);
554 if (get_test_stage(test) != 0)
555 goto fail;
556
557 // test IO width, in/out
558 io_bitmap[0] = 0xFF;
559 inc_test_stage(test);
560 inb(0x0);
561 if (get_test_stage(test) != 2)
562 goto fail;
563
564 outw(0x0, 0x0);
565 if (get_test_stage(test) != 3)
566 goto fail;
567
568 inl(0x0);
569 if (get_test_stage(test) != 4)
570 goto fail;
571
572 // test low/high IO port
573 io_bitmap[0x5000 / 8] = (1 << (0x5000 % 8));
574 inb(0x5000);
575 if (get_test_stage(test) != 5)
576 goto fail;
577
578 io_bitmap[0x9000 / 8] = (1 << (0x9000 % 8));
579 inw(0x9000);
580 if (get_test_stage(test) != 6)
581 goto fail;
582
583 // test partial pass
584 io_bitmap[0x5000 / 8] = (1 << (0x5000 % 8));
585 inl(0x4FFF);
586 if (get_test_stage(test) != 7)
587 goto fail;
588
589 // test across pages
590 inc_test_stage(test);
591 inl(0x7FFF);
592 if (get_test_stage(test) != 8)
593 goto fail;
594
595 inc_test_stage(test);
596 io_bitmap[0x8000 / 8] = 1 << (0x8000 % 8);
597 inl(0x7FFF);
598 if (get_test_stage(test) != 10)
599 goto fail;
600
601 io_bitmap[0] = 0;
602 inl(0xFFFF);
603 if (get_test_stage(test) != 11)
604 goto fail;
605
606 io_bitmap[0] = 0xFF;
607 io_bitmap[8192] = 0;
608 inl(0xFFFF);
609 inc_test_stage(test);
610 if (get_test_stage(test) != 12)
611 goto fail;
612
613 return;
614
615 fail:
616 report_fail("stage %d", get_test_stage(test));
617 test->scratch = -1;
618 }
619
ioio_finished(struct svm_test * test)620 static bool ioio_finished(struct svm_test *test)
621 {
622 unsigned port, size;
623
624 /* Only expect IOIO intercepts */
625 if (vmcb->control.exit_code == SVM_EXIT_VMMCALL)
626 return true;
627
628 if (vmcb->control.exit_code != SVM_EXIT_IOIO)
629 return true;
630
631 /* one step forward */
632 test->scratch += 1;
633
634 port = vmcb->control.exit_info_1 >> 16;
635 size = (vmcb->control.exit_info_1 >> SVM_IOIO_SIZE_SHIFT) & 7;
636
637 while (size--) {
638 io_bitmap[port / 8] &= ~(1 << (port & 7));
639 port++;
640 }
641
642 return false;
643 }
644
check_ioio(struct svm_test * test)645 static bool check_ioio(struct svm_test *test)
646 {
647 memset(io_bitmap, 0, 8193);
648 return test->scratch != -1;
649 }
650
prepare_asid_zero(struct svm_test * test)651 static void prepare_asid_zero(struct svm_test *test)
652 {
653 vmcb->control.asid = 0;
654 }
655
test_asid_zero(struct svm_test * test)656 static void test_asid_zero(struct svm_test *test)
657 {
658 asm volatile ("vmmcall\n\t");
659 }
660
check_asid_zero(struct svm_test * test)661 static bool check_asid_zero(struct svm_test *test)
662 {
663 return vmcb->control.exit_code == SVM_EXIT_ERR;
664 }
665
sel_cr0_bug_prepare(struct svm_test * test)666 static void sel_cr0_bug_prepare(struct svm_test *test)
667 {
668 vmcb->control.intercept |= (1ULL << INTERCEPT_SELECTIVE_CR0);
669 }
670
sel_cr0_bug_finished(struct svm_test * test)671 static bool sel_cr0_bug_finished(struct svm_test *test)
672 {
673 return true;
674 }
675
sel_cr0_bug_test(struct svm_test * test)676 static void sel_cr0_bug_test(struct svm_test *test)
677 {
678 unsigned long cr0;
679
680 /* read cr0, clear CD, and write back */
681 cr0 = read_cr0();
682 cr0 |= (1UL << 30);
683 write_cr0(cr0);
684
685 /*
686 * If we are here the test failed, not sure what to do now because we
687 * are not in guest-mode anymore so we can't trigger an intercept.
688 * Trigger a tripple-fault for now.
689 */
690 report_fail("sel_cr0 test. Can not recover from this - exiting");
691 exit(report_summary());
692 }
693
sel_cr0_bug_check(struct svm_test * test)694 static bool sel_cr0_bug_check(struct svm_test *test)
695 {
696 return vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE;
697 }
698
699 #define TSC_ADJUST_VALUE (1ll << 32)
700 #define TSC_OFFSET_VALUE (~0ull << 48)
701 static bool ok;
702
tsc_adjust_supported(void)703 static bool tsc_adjust_supported(void)
704 {
705 return this_cpu_has(X86_FEATURE_TSC_ADJUST);
706 }
707
tsc_adjust_prepare(struct svm_test * test)708 static void tsc_adjust_prepare(struct svm_test *test)
709 {
710 default_prepare(test);
711 vmcb->control.tsc_offset = TSC_OFFSET_VALUE;
712
713 wrmsr(MSR_IA32_TSC_ADJUST, -TSC_ADJUST_VALUE);
714 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST);
715 ok = adjust == -TSC_ADJUST_VALUE;
716 }
717
tsc_adjust_test(struct svm_test * test)718 static void tsc_adjust_test(struct svm_test *test)
719 {
720 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST);
721 ok &= adjust == -TSC_ADJUST_VALUE;
722
723 uint64_t l1_tsc = rdtsc() - TSC_OFFSET_VALUE;
724 wrmsr(MSR_IA32_TSC, l1_tsc - TSC_ADJUST_VALUE);
725
726 adjust = rdmsr(MSR_IA32_TSC_ADJUST);
727 ok &= adjust <= -2 * TSC_ADJUST_VALUE;
728
729 uint64_t l1_tsc_end = rdtsc() - TSC_OFFSET_VALUE;
730 ok &= (l1_tsc_end + TSC_ADJUST_VALUE - l1_tsc) < TSC_ADJUST_VALUE;
731
732 uint64_t l1_tsc_msr = rdmsr(MSR_IA32_TSC) - TSC_OFFSET_VALUE;
733 ok &= (l1_tsc_msr + TSC_ADJUST_VALUE - l1_tsc) < TSC_ADJUST_VALUE;
734 }
735
tsc_adjust_check(struct svm_test * test)736 static bool tsc_adjust_check(struct svm_test *test)
737 {
738 int64_t adjust = rdmsr(MSR_IA32_TSC_ADJUST);
739
740 wrmsr(MSR_IA32_TSC_ADJUST, 0);
741 return ok && adjust <= -2 * TSC_ADJUST_VALUE;
742 }
743
744
745 static u64 guest_tsc_delay_value;
746 /* number of bits to shift tsc right for stable result */
747 #define TSC_SHIFT 24
748 #define TSC_SCALE_ITERATIONS 10
749
svm_tsc_scale_guest(struct svm_test * test)750 static void svm_tsc_scale_guest(struct svm_test *test)
751 {
752 u64 start_tsc = rdtsc();
753
754 while (rdtsc() - start_tsc < guest_tsc_delay_value)
755 cpu_relax();
756 }
757
svm_tsc_scale_run_testcase(u64 duration,double tsc_scale,u64 tsc_offset)758 static void svm_tsc_scale_run_testcase(u64 duration,
759 double tsc_scale, u64 tsc_offset)
760 {
761 u64 start_tsc, actual_duration;
762
763 guest_tsc_delay_value = (duration << TSC_SHIFT) * tsc_scale;
764
765 test_set_guest(svm_tsc_scale_guest);
766 vmcb->control.tsc_offset = tsc_offset;
767 wrmsr(MSR_AMD64_TSC_RATIO, (u64)(tsc_scale * (1ULL << 32)));
768
769 start_tsc = rdtsc();
770
771 if (svm_vmrun() != SVM_EXIT_VMMCALL)
772 report_fail("unexpected vm exit code 0x%x", vmcb->control.exit_code);
773
774 actual_duration = (rdtsc() - start_tsc) >> TSC_SHIFT;
775
776 report(duration == actual_duration, "tsc delay (expected: %lu, actual: %lu)",
777 duration, actual_duration);
778 }
779
svm_tsc_scale_test(void)780 static void svm_tsc_scale_test(void)
781 {
782 int i;
783
784 if (!tsc_scale_supported()) {
785 report_skip("TSC scale not supported in the guest");
786 return;
787 }
788
789 report(rdmsr(MSR_AMD64_TSC_RATIO) == TSC_RATIO_DEFAULT,
790 "initial TSC scale ratio");
791
792 for (i = 0 ; i < TSC_SCALE_ITERATIONS; i++) {
793
794 double tsc_scale = (double)(rdrand() % 100 + 1) / 10;
795 int duration = rdrand() % 50 + 1;
796 u64 tsc_offset = rdrand();
797
798 report_info("duration=%d, tsc_scale=%d, tsc_offset=%ld",
799 duration, (int)(tsc_scale * 100), tsc_offset);
800
801 svm_tsc_scale_run_testcase(duration, tsc_scale, tsc_offset);
802 }
803
804 svm_tsc_scale_run_testcase(50, 255, rdrand());
805 svm_tsc_scale_run_testcase(50, 0.0001, rdrand());
806 }
807
latency_prepare(struct svm_test * test)808 static void latency_prepare(struct svm_test *test)
809 {
810 default_prepare(test);
811 runs = LATENCY_RUNS;
812 latvmrun_min = latvmexit_min = -1ULL;
813 latvmrun_max = latvmexit_max = 0;
814 vmrun_sum = vmexit_sum = 0;
815 tsc_start = rdtsc();
816 }
817
latency_test(struct svm_test * test)818 static void latency_test(struct svm_test *test)
819 {
820 u64 cycles;
821
822 start:
823 tsc_end = rdtsc();
824
825 cycles = tsc_end - tsc_start;
826
827 if (cycles > latvmrun_max)
828 latvmrun_max = cycles;
829
830 if (cycles < latvmrun_min)
831 latvmrun_min = cycles;
832
833 vmrun_sum += cycles;
834
835 tsc_start = rdtsc();
836
837 asm volatile ("vmmcall" : : : "memory");
838 goto start;
839 }
840
latency_finished(struct svm_test * test)841 static bool latency_finished(struct svm_test *test)
842 {
843 u64 cycles;
844
845 tsc_end = rdtsc();
846
847 cycles = tsc_end - tsc_start;
848
849 if (cycles > latvmexit_max)
850 latvmexit_max = cycles;
851
852 if (cycles < latvmexit_min)
853 latvmexit_min = cycles;
854
855 vmexit_sum += cycles;
856
857 vmcb->save.rip += 3;
858
859 runs -= 1;
860
861 tsc_end = rdtsc();
862
863 return runs == 0;
864 }
865
latency_finished_clean(struct svm_test * test)866 static bool latency_finished_clean(struct svm_test *test)
867 {
868 vmcb->control.clean = VMCB_CLEAN_ALL;
869 return latency_finished(test);
870 }
871
latency_check(struct svm_test * test)872 static bool latency_check(struct svm_test *test)
873 {
874 printf(" Latency VMRUN : max: %ld min: %ld avg: %ld\n", latvmrun_max,
875 latvmrun_min, vmrun_sum / LATENCY_RUNS);
876 printf(" Latency VMEXIT: max: %ld min: %ld avg: %ld\n", latvmexit_max,
877 latvmexit_min, vmexit_sum / LATENCY_RUNS);
878 return true;
879 }
880
lat_svm_insn_prepare(struct svm_test * test)881 static void lat_svm_insn_prepare(struct svm_test *test)
882 {
883 default_prepare(test);
884 runs = LATENCY_RUNS;
885 latvmload_min = latvmsave_min = latstgi_min = latclgi_min = -1ULL;
886 latvmload_max = latvmsave_max = latstgi_max = latclgi_max = 0;
887 vmload_sum = vmsave_sum = stgi_sum = clgi_sum;
888 }
889
lat_svm_insn_finished(struct svm_test * test)890 static bool lat_svm_insn_finished(struct svm_test *test)
891 {
892 u64 vmcb_phys = virt_to_phys(vmcb);
893 u64 cycles;
894
895 for ( ; runs != 0; runs--) {
896 tsc_start = rdtsc();
897 asm volatile("vmload %0\n\t" : : "a"(vmcb_phys) : "memory");
898 cycles = rdtsc() - tsc_start;
899 if (cycles > latvmload_max)
900 latvmload_max = cycles;
901 if (cycles < latvmload_min)
902 latvmload_min = cycles;
903 vmload_sum += cycles;
904
905 tsc_start = rdtsc();
906 asm volatile("vmsave %0\n\t" : : "a"(vmcb_phys) : "memory");
907 cycles = rdtsc() - tsc_start;
908 if (cycles > latvmsave_max)
909 latvmsave_max = cycles;
910 if (cycles < latvmsave_min)
911 latvmsave_min = cycles;
912 vmsave_sum += cycles;
913
914 tsc_start = rdtsc();
915 asm volatile("stgi\n\t");
916 cycles = rdtsc() - tsc_start;
917 if (cycles > latstgi_max)
918 latstgi_max = cycles;
919 if (cycles < latstgi_min)
920 latstgi_min = cycles;
921 stgi_sum += cycles;
922
923 tsc_start = rdtsc();
924 asm volatile("clgi\n\t");
925 cycles = rdtsc() - tsc_start;
926 if (cycles > latclgi_max)
927 latclgi_max = cycles;
928 if (cycles < latclgi_min)
929 latclgi_min = cycles;
930 clgi_sum += cycles;
931 }
932
933 tsc_end = rdtsc();
934
935 return true;
936 }
937
lat_svm_insn_check(struct svm_test * test)938 static bool lat_svm_insn_check(struct svm_test *test)
939 {
940 printf(" Latency VMLOAD: max: %ld min: %ld avg: %ld\n", latvmload_max,
941 latvmload_min, vmload_sum / LATENCY_RUNS);
942 printf(" Latency VMSAVE: max: %ld min: %ld avg: %ld\n", latvmsave_max,
943 latvmsave_min, vmsave_sum / LATENCY_RUNS);
944 printf(" Latency STGI: max: %ld min: %ld avg: %ld\n", latstgi_max,
945 latstgi_min, stgi_sum / LATENCY_RUNS);
946 printf(" Latency CLGI: max: %ld min: %ld avg: %ld\n", latclgi_max,
947 latclgi_min, clgi_sum / LATENCY_RUNS);
948 return true;
949 }
950
951 /*
952 * Report failures from SVM guest code, and on failure, set the stage to -1 and
953 * do VMMCALL to terminate the test (host side must treat -1 as "finished").
954 * TODO: fix the tests that don't play nice with a straight report, e.g. the
955 * V_TPR test fails if report() is invoked.
956 */
957 #define report_svm_guest(cond, test, fmt, args...) \
958 do { \
959 if (!(cond)) { \
960 report_fail(fmt, ##args); \
961 set_test_stage(test, -1); \
962 vmmcall(); \
963 } \
964 } while (0)
965
966 bool pending_event_ipi_fired;
967 bool pending_event_guest_run;
968
pending_event_ipi_isr(isr_regs_t * regs)969 static void pending_event_ipi_isr(isr_regs_t *regs)
970 {
971 pending_event_ipi_fired = true;
972 eoi();
973 }
974
pending_event_prepare(struct svm_test * test)975 static void pending_event_prepare(struct svm_test *test)
976 {
977 int ipi_vector = 0xf1;
978
979 default_prepare(test);
980
981 pending_event_ipi_fired = false;
982
983 handle_irq(ipi_vector, pending_event_ipi_isr);
984
985 pending_event_guest_run = false;
986
987 vmcb->control.intercept |= (1ULL << INTERCEPT_INTR);
988 vmcb->control.int_ctl |= V_INTR_MASKING_MASK;
989
990 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL |
991 APIC_DM_FIXED | ipi_vector, 0);
992
993 set_test_stage(test, 0);
994 }
995
pending_event_test(struct svm_test * test)996 static void pending_event_test(struct svm_test *test)
997 {
998 pending_event_guest_run = true;
999 }
1000
pending_event_finished(struct svm_test * test)1001 static bool pending_event_finished(struct svm_test *test)
1002 {
1003 switch (get_test_stage(test)) {
1004 case 0:
1005 if (vmcb->control.exit_code != SVM_EXIT_INTR) {
1006 report_fail("VMEXIT not due to pending interrupt. Exit reason 0x%x",
1007 vmcb->control.exit_code);
1008 return true;
1009 }
1010
1011 vmcb->control.intercept &= ~(1ULL << INTERCEPT_INTR);
1012 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
1013
1014 if (pending_event_guest_run) {
1015 report_fail("Guest ran before host received IPI\n");
1016 return true;
1017 }
1018
1019 sti_nop_cli();
1020
1021 if (!pending_event_ipi_fired) {
1022 report_fail("Pending interrupt not dispatched after IRQ enabled\n");
1023 return true;
1024 }
1025 break;
1026
1027 case 1:
1028 if (!pending_event_guest_run) {
1029 report_fail("Guest did not resume when no interrupt\n");
1030 return true;
1031 }
1032 break;
1033 }
1034
1035 inc_test_stage(test);
1036
1037 return get_test_stage(test) == 2;
1038 }
1039
pending_event_check(struct svm_test * test)1040 static bool pending_event_check(struct svm_test *test)
1041 {
1042 return get_test_stage(test) == 2;
1043 }
1044
pending_event_cli_prepare(struct svm_test * test)1045 static void pending_event_cli_prepare(struct svm_test *test)
1046 {
1047 default_prepare(test);
1048
1049 pending_event_ipi_fired = false;
1050
1051 handle_irq(0xf1, pending_event_ipi_isr);
1052
1053 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL |
1054 APIC_DM_FIXED | 0xf1, 0);
1055
1056 set_test_stage(test, 0);
1057 }
1058
pending_event_cli_prepare_gif_clear(struct svm_test * test)1059 static void pending_event_cli_prepare_gif_clear(struct svm_test *test)
1060 {
1061 asm("cli");
1062 }
1063
pending_event_cli_test(struct svm_test * test)1064 static void pending_event_cli_test(struct svm_test *test)
1065 {
1066 report_svm_guest(!pending_event_ipi_fired, test,
1067 "IRQ should NOT be delivered while IRQs disabled");
1068
1069 /* VINTR_MASKING is zero. This should cause the IPI to fire. */
1070 sti_nop_cli();
1071
1072 report_svm_guest(pending_event_ipi_fired, test,
1073 "IRQ should be delivered after enabling IRQs");
1074 vmmcall();
1075
1076 /*
1077 * Now VINTR_MASKING=1, but no interrupt is pending so
1078 * the VINTR interception should be clear in VMCB02. Check
1079 * that L0 did not leave a stale VINTR in the VMCB.
1080 */
1081 sti_nop_cli();
1082 }
1083
pending_event_cli_finished(struct svm_test * test)1084 static bool pending_event_cli_finished(struct svm_test *test)
1085 {
1086 report_svm_guest(vmcb->control.exit_code == SVM_EXIT_VMMCALL, test,
1087 "Wanted VMMCALL VM-Exit, got exit reason 0x%x",
1088 vmcb->control.exit_code);
1089
1090 switch (get_test_stage(test)) {
1091 case 0:
1092 vmcb->save.rip += 3;
1093
1094 pending_event_ipi_fired = false;
1095
1096 vmcb->control.int_ctl |= V_INTR_MASKING_MASK;
1097
1098 /* Now entering again with VINTR_MASKING=1. */
1099 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL |
1100 APIC_DM_FIXED | 0xf1, 0);
1101
1102 break;
1103
1104 case 1:
1105 if (pending_event_ipi_fired == true) {
1106 report_fail("Interrupt triggered by guest");
1107 return true;
1108 }
1109
1110 sti_nop_cli();
1111
1112 if (pending_event_ipi_fired != true) {
1113 report_fail("Interrupt not triggered by host");
1114 return true;
1115 }
1116
1117 break;
1118
1119 default:
1120 return true;
1121 }
1122
1123 inc_test_stage(test);
1124
1125 return get_test_stage(test) == 2;
1126 }
1127
pending_event_cli_check(struct svm_test * test)1128 static bool pending_event_cli_check(struct svm_test *test)
1129 {
1130 return get_test_stage(test) == 2;
1131 }
1132
1133 #define TIMER_VECTOR 222
1134
1135 static volatile bool timer_fired;
1136
timer_isr(isr_regs_t * regs)1137 static void timer_isr(isr_regs_t *regs)
1138 {
1139 timer_fired = true;
1140 apic_write(APIC_EOI, 0);
1141 }
1142
interrupt_prepare(struct svm_test * test)1143 static void interrupt_prepare(struct svm_test *test)
1144 {
1145 default_prepare(test);
1146 handle_irq(TIMER_VECTOR, timer_isr);
1147 timer_fired = false;
1148 set_test_stage(test, 0);
1149 }
1150
interrupt_test(struct svm_test * test)1151 static void interrupt_test(struct svm_test *test)
1152 {
1153 long long start, loops;
1154
1155 apic_setup_timer(TIMER_VECTOR, APIC_LVT_TIMER_PERIODIC);
1156 sti();
1157 apic_start_timer(1000);
1158
1159 for (loops = 0; loops < 10000000 && !timer_fired; loops++)
1160 asm volatile ("nop");
1161
1162 report_svm_guest(timer_fired, test,
1163 "direct interrupt while running guest");
1164
1165 apic_stop_timer();
1166 cli();
1167 vmmcall();
1168
1169 timer_fired = false;
1170 apic_start_timer(1000);
1171 for (loops = 0; loops < 10000000 && !timer_fired; loops++)
1172 asm volatile ("nop");
1173
1174 report_svm_guest(timer_fired, test,
1175 "intercepted interrupt while running guest");
1176
1177 sti();
1178 apic_stop_timer();
1179 cli();
1180
1181 timer_fired = false;
1182 start = rdtsc();
1183 apic_start_timer(1000000);
1184 safe_halt();
1185
1186 report_svm_guest(timer_fired, test, "direct interrupt + hlt");
1187 report(rdtsc() - start > 10000, "IRQ arrived after expected delay");
1188
1189 apic_stop_timer();
1190 cli();
1191 vmmcall();
1192
1193 timer_fired = false;
1194 start = rdtsc();
1195 apic_start_timer(1000000);
1196 asm volatile ("hlt");
1197
1198 report_svm_guest(timer_fired, test, "intercepted interrupt + hlt");
1199 report(rdtsc() - start > 10000, "IRQ arrived after expected delay");
1200
1201 apic_cleanup_timer();
1202 }
1203
interrupt_finished(struct svm_test * test)1204 static bool interrupt_finished(struct svm_test *test)
1205 {
1206 switch (get_test_stage(test)) {
1207 case 0:
1208 case 2:
1209 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1210 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1211 vmcb->control.exit_code);
1212 return true;
1213 }
1214 vmcb->save.rip += 3;
1215
1216 vmcb->control.intercept |= (1ULL << INTERCEPT_INTR);
1217 vmcb->control.int_ctl |= V_INTR_MASKING_MASK;
1218 break;
1219
1220 case 1:
1221 case 3:
1222 if (vmcb->control.exit_code != SVM_EXIT_INTR) {
1223 report_fail("VMEXIT not due to intr intercept. Exit reason 0x%x",
1224 vmcb->control.exit_code);
1225 return true;
1226 }
1227
1228 sti_nop_cli();
1229
1230 vmcb->control.intercept &= ~(1ULL << INTERCEPT_INTR);
1231 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
1232 break;
1233
1234 case 4:
1235 break;
1236
1237 default:
1238 return true;
1239 }
1240
1241 inc_test_stage(test);
1242
1243 return get_test_stage(test) == 5;
1244 }
1245
interrupt_check(struct svm_test * test)1246 static bool interrupt_check(struct svm_test *test)
1247 {
1248 return get_test_stage(test) == 5;
1249 }
1250
1251 static volatile bool nmi_fired;
1252
nmi_handler(struct ex_regs * regs)1253 static void nmi_handler(struct ex_regs *regs)
1254 {
1255 nmi_fired = true;
1256 }
1257
nmi_prepare(struct svm_test * test)1258 static void nmi_prepare(struct svm_test *test)
1259 {
1260 default_prepare(test);
1261 nmi_fired = false;
1262 handle_exception(NMI_VECTOR, nmi_handler);
1263 set_test_stage(test, 0);
1264 }
1265
nmi_test(struct svm_test * test)1266 static void nmi_test(struct svm_test *test)
1267 {
1268 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, 0);
1269
1270 report_svm_guest(nmi_fired, test, "direct NMI while running guest");
1271
1272 vmmcall();
1273
1274 nmi_fired = false;
1275
1276 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, 0);
1277
1278 report_svm_guest(nmi_fired, test, "intercepted pending NMI delivered to guest");
1279 }
1280
nmi_finished(struct svm_test * test)1281 static bool nmi_finished(struct svm_test *test)
1282 {
1283 switch (get_test_stage(test)) {
1284 case 0:
1285 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1286 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1287 vmcb->control.exit_code);
1288 return true;
1289 }
1290 vmcb->save.rip += 3;
1291
1292 vmcb->control.intercept |= (1ULL << INTERCEPT_NMI);
1293 break;
1294
1295 case 1:
1296 if (vmcb->control.exit_code != SVM_EXIT_NMI) {
1297 report_fail("VMEXIT not due to NMI intercept. Exit reason 0x%x",
1298 vmcb->control.exit_code);
1299 return true;
1300 }
1301
1302 report_pass("NMI intercept while running guest");
1303 break;
1304
1305 case 2:
1306 break;
1307
1308 default:
1309 return true;
1310 }
1311
1312 inc_test_stage(test);
1313
1314 return get_test_stage(test) == 3;
1315 }
1316
nmi_check(struct svm_test * test)1317 static bool nmi_check(struct svm_test *test)
1318 {
1319 return get_test_stage(test) == 3;
1320 }
1321
1322 #define NMI_DELAY 100000000ULL
1323
nmi_message_thread(void * _test)1324 static void nmi_message_thread(void *_test)
1325 {
1326 struct svm_test *test = _test;
1327
1328 while (get_test_stage(test) != 1)
1329 pause();
1330
1331 delay(NMI_DELAY);
1332
1333 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]);
1334
1335 while (get_test_stage(test) != 2)
1336 pause();
1337
1338 delay(NMI_DELAY);
1339
1340 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]);
1341 }
1342
nmi_hlt_test(struct svm_test * test)1343 static void nmi_hlt_test(struct svm_test *test)
1344 {
1345 long long start;
1346
1347 on_cpu_async(1, nmi_message_thread, test);
1348
1349 start = rdtsc();
1350
1351 set_test_stage(test, 1);
1352
1353 asm volatile ("hlt");
1354
1355 report_svm_guest(nmi_fired, test, "direct NMI + hlt");
1356 report(rdtsc() - start > NMI_DELAY, "direct NMI after expected delay");
1357
1358 nmi_fired = false;
1359
1360 vmmcall();
1361
1362 start = rdtsc();
1363
1364 set_test_stage(test, 2);
1365
1366 asm volatile ("hlt");
1367
1368 report_svm_guest(nmi_fired, test, "intercepted NMI + hlt");
1369 report(rdtsc() - start > NMI_DELAY, "intercepted NMI after expected delay");
1370
1371 set_test_stage(test, 3);
1372 }
1373
nmi_hlt_finished(struct svm_test * test)1374 static bool nmi_hlt_finished(struct svm_test *test)
1375 {
1376 switch (get_test_stage(test)) {
1377 case 1:
1378 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1379 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1380 vmcb->control.exit_code);
1381 return true;
1382 }
1383 vmcb->save.rip += 3;
1384
1385 vmcb->control.intercept |= (1ULL << INTERCEPT_NMI);
1386 break;
1387
1388 case 2:
1389 if (vmcb->control.exit_code != SVM_EXIT_NMI) {
1390 report_fail("VMEXIT not due to NMI intercept. Exit reason 0x%x",
1391 vmcb->control.exit_code);
1392 return true;
1393 }
1394
1395 report_pass("NMI intercept while running guest");
1396 break;
1397
1398 case 3:
1399 break;
1400
1401 default:
1402 return true;
1403 }
1404
1405 return get_test_stage(test) == 3;
1406 }
1407
nmi_hlt_check(struct svm_test * test)1408 static bool nmi_hlt_check(struct svm_test *test)
1409 {
1410 return get_test_stage(test) == 3;
1411 }
1412
vnmi_prepare(struct svm_test * test)1413 static void vnmi_prepare(struct svm_test *test)
1414 {
1415 nmi_prepare(test);
1416
1417 /*
1418 * Disable NMI interception to start. Enabling vNMI without
1419 * intercepting "real" NMIs should result in an ERR VM-Exit.
1420 */
1421 vmcb->control.intercept &= ~(1ULL << INTERCEPT_NMI);
1422 vmcb->control.int_ctl = V_NMI_ENABLE_MASK;
1423 vmcb->control.int_vector = NMI_VECTOR;
1424 }
1425
vnmi_test(struct svm_test * test)1426 static void vnmi_test(struct svm_test *test)
1427 {
1428 report_svm_guest(!nmi_fired, test, "No vNMI before injection");
1429 vmmcall();
1430
1431 report_svm_guest(nmi_fired, test, "vNMI delivered after injection");
1432 vmmcall();
1433 }
1434
vnmi_finished(struct svm_test * test)1435 static bool vnmi_finished(struct svm_test *test)
1436 {
1437 switch (get_test_stage(test)) {
1438 case 0:
1439 if (vmcb->control.exit_code != SVM_EXIT_ERR) {
1440 report_fail("Wanted ERR VM-Exit, got 0x%x",
1441 vmcb->control.exit_code);
1442 return true;
1443 }
1444 report(!nmi_fired, "vNMI enabled but NMI_INTERCEPT unset!");
1445 vmcb->control.intercept |= (1ULL << INTERCEPT_NMI);
1446 vmcb->save.rip += 3;
1447 break;
1448
1449 case 1:
1450 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1451 report_fail("Wanted VMMCALL VM-Exit, got 0x%x",
1452 vmcb->control.exit_code);
1453 return true;
1454 }
1455 report(!nmi_fired, "vNMI with vector 2 not injected");
1456 vmcb->control.int_ctl |= V_NMI_PENDING_MASK;
1457 vmcb->save.rip += 3;
1458 break;
1459
1460 case 2:
1461 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1462 report_fail("Wanted VMMCALL VM-Exit, got 0x%x",
1463 vmcb->control.exit_code);
1464 return true;
1465 }
1466 if (vmcb->control.int_ctl & V_NMI_BLOCKING_MASK) {
1467 report_fail("V_NMI_BLOCKING_MASK not cleared on VMEXIT");
1468 return true;
1469 }
1470 report_pass("VNMI serviced");
1471 vmcb->save.rip += 3;
1472 break;
1473
1474 default:
1475 return true;
1476 }
1477
1478 inc_test_stage(test);
1479
1480 return get_test_stage(test) == 3;
1481 }
1482
vnmi_check(struct svm_test * test)1483 static bool vnmi_check(struct svm_test *test)
1484 {
1485 return get_test_stage(test) == 3;
1486 }
1487
1488 static volatile int count_exc = 0;
1489
my_isr(struct ex_regs * r)1490 static void my_isr(struct ex_regs *r)
1491 {
1492 count_exc++;
1493 }
1494
exc_inject_prepare(struct svm_test * test)1495 static void exc_inject_prepare(struct svm_test *test)
1496 {
1497 default_prepare(test);
1498 handle_exception(DE_VECTOR, my_isr);
1499 handle_exception(NMI_VECTOR, my_isr);
1500 }
1501
1502
exc_inject_test(struct svm_test * test)1503 static void exc_inject_test(struct svm_test *test)
1504 {
1505 asm volatile ("vmmcall\n\tvmmcall\n\t");
1506 }
1507
exc_inject_finished(struct svm_test * test)1508 static bool exc_inject_finished(struct svm_test *test)
1509 {
1510 switch (get_test_stage(test)) {
1511 case 0:
1512 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1513 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1514 vmcb->control.exit_code);
1515 return true;
1516 }
1517 vmcb->save.rip += 3;
1518 vmcb->control.event_inj = NMI_VECTOR | SVM_EVTINJ_TYPE_EXEPT | SVM_EVTINJ_VALID;
1519 break;
1520
1521 case 1:
1522 if (vmcb->control.exit_code != SVM_EXIT_ERR) {
1523 report_fail("VMEXIT not due to error. Exit reason 0x%x",
1524 vmcb->control.exit_code);
1525 return true;
1526 }
1527 report(count_exc == 0, "exception with vector 2 not injected");
1528 vmcb->control.event_inj = DE_VECTOR | SVM_EVTINJ_TYPE_EXEPT | SVM_EVTINJ_VALID;
1529 break;
1530
1531 case 2:
1532 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1533 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1534 vmcb->control.exit_code);
1535 return true;
1536 }
1537 vmcb->save.rip += 3;
1538 report(count_exc == 1, "divide overflow exception injected");
1539 report(!(vmcb->control.event_inj & SVM_EVTINJ_VALID), "eventinj.VALID cleared");
1540 break;
1541
1542 default:
1543 return true;
1544 }
1545
1546 inc_test_stage(test);
1547
1548 return get_test_stage(test) == 3;
1549 }
1550
exc_inject_check(struct svm_test * test)1551 static bool exc_inject_check(struct svm_test *test)
1552 {
1553 return count_exc == 1 && get_test_stage(test) == 3;
1554 }
1555
1556 static volatile bool virq_fired;
1557 static volatile unsigned long virq_rip;
1558
virq_isr(isr_regs_t * regs)1559 static void virq_isr(isr_regs_t *regs)
1560 {
1561 virq_fired = true;
1562 virq_rip = regs->rip;
1563 }
1564
virq_inject_prepare(struct svm_test * test)1565 static void virq_inject_prepare(struct svm_test *test)
1566 {
1567 handle_irq(0xf1, virq_isr);
1568 default_prepare(test);
1569 vmcb->control.int_ctl = V_INTR_MASKING_MASK | V_IRQ_MASK |
1570 (0x0f << V_INTR_PRIO_SHIFT); // Set to the highest priority
1571 vmcb->control.int_vector = 0xf1;
1572 virq_fired = false;
1573 virq_rip = -1;
1574 set_test_stage(test, 0);
1575 }
1576
virq_inject_test(struct svm_test * test)1577 static void virq_inject_test(struct svm_test *test)
1578 {
1579 report_svm_guest(!virq_fired, test, "virtual IRQ blocked after L2 cli");
1580
1581 sti_nop_cli();
1582
1583 report_svm_guest(virq_fired, test, "virtual IRQ fired after L2 sti");
1584
1585 vmmcall();
1586
1587 report_svm_guest(!virq_fired, test, "intercepted VINTR blocked after L2 cli");
1588
1589 sti_nop_cli();
1590
1591 report_svm_guest(virq_fired, test, "intercepted VINTR fired after L2 sti");
1592
1593 vmmcall();
1594
1595 sti_nop_cli();
1596
1597 report_svm_guest(!virq_fired, test,
1598 "virtual IRQ blocked V_IRQ_PRIO less than V_TPR");
1599
1600 vmmcall();
1601 vmmcall();
1602 }
1603
virq_inject_finished(struct svm_test * test)1604 static bool virq_inject_finished(struct svm_test *test)
1605 {
1606 vmcb->save.rip += 3;
1607
1608 switch (get_test_stage(test)) {
1609 case 0:
1610 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1611 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1612 vmcb->control.exit_code);
1613 return true;
1614 }
1615 if (vmcb->control.int_ctl & V_IRQ_MASK) {
1616 report_fail("V_IRQ not cleared on VMEXIT after firing");
1617 return true;
1618 }
1619 virq_fired = false;
1620 vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR);
1621 vmcb->control.int_ctl = V_INTR_MASKING_MASK | V_IRQ_MASK |
1622 (0x0f << V_INTR_PRIO_SHIFT);
1623 break;
1624
1625 case 1:
1626 if (vmcb->control.exit_code != SVM_EXIT_VINTR) {
1627 report_fail("VMEXIT not due to vintr. Exit reason 0x%x",
1628 vmcb->control.exit_code);
1629 return true;
1630 }
1631 if (virq_fired) {
1632 report_fail("V_IRQ fired before SVM_EXIT_VINTR");
1633 return true;
1634 }
1635 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
1636 break;
1637
1638 case 2:
1639 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1640 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1641 vmcb->control.exit_code);
1642 return true;
1643 }
1644 virq_fired = false;
1645 // Set irq to lower priority
1646 vmcb->control.int_ctl = V_INTR_MASKING_MASK | V_IRQ_MASK |
1647 (0x08 << V_INTR_PRIO_SHIFT);
1648 // Raise guest TPR
1649 vmcb->control.int_ctl |= 0x0a & V_TPR_MASK;
1650 break;
1651
1652 case 3:
1653 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1654 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1655 vmcb->control.exit_code);
1656 return true;
1657 }
1658 vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR);
1659 break;
1660
1661 case 4:
1662 // INTERCEPT_VINTR should be ignored because V_INTR_PRIO < V_TPR
1663 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1664 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1665 vmcb->control.exit_code);
1666 return true;
1667 }
1668 break;
1669
1670 default:
1671 return true;
1672 }
1673
1674 inc_test_stage(test);
1675
1676 return get_test_stage(test) == 5;
1677 }
1678
virq_inject_check(struct svm_test * test)1679 static bool virq_inject_check(struct svm_test *test)
1680 {
1681 return get_test_stage(test) == 5;
1682 }
1683
virq_inject_within_shadow_prepare(struct svm_test * test)1684 static void virq_inject_within_shadow_prepare(struct svm_test *test)
1685 {
1686 virq_inject_prepare(test);
1687 vmcb->control.int_state = SVM_INTERRUPT_SHADOW_MASK;
1688 vmcb->save.rflags |= X86_EFLAGS_IF;
1689 }
1690
1691 extern void virq_inject_within_shadow_test(struct svm_test *test);
1692 asm("virq_inject_within_shadow_test: nop; nop; vmmcall");
1693
virq_inject_within_shadow_prepare_gif_clear(struct svm_test * test)1694 static void virq_inject_within_shadow_prepare_gif_clear(struct svm_test *test)
1695 {
1696 vmcb->save.rip = (unsigned long) test->guest_func;
1697 }
1698
virq_inject_within_shadow_finished(struct svm_test * test)1699 static bool virq_inject_within_shadow_finished(struct svm_test *test)
1700 {
1701 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL)
1702 report_fail("VMEXIT not due to vmmcall. Exit reason 0x%x",
1703 vmcb->control.exit_code);
1704 if (!virq_fired)
1705 report_fail("V_IRQ did not fire");
1706 else if (virq_rip != (unsigned long) virq_inject_within_shadow_test + 1)
1707 report_fail("Unexpected RIP for interrupt handler");
1708 else if (vmcb->control.int_ctl & V_IRQ_MASK)
1709 report_fail("V_IRQ not cleared on VMEXIT after firing");
1710 else if (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
1711 report_fail("Interrupt shadow not cleared");
1712 else
1713 inc_test_stage(test);
1714
1715 return true;
1716 }
1717
virq_inject_within_shadow_check(struct svm_test * test)1718 static bool virq_inject_within_shadow_check(struct svm_test *test)
1719 {
1720 return get_test_stage(test) == 1;
1721 }
1722
1723 /*
1724 * Detect nested guest RIP corruption as explained in kernel commit
1725 * b6162e82aef19fee9c32cb3fe9ac30d9116a8c73
1726 *
1727 * In the assembly loop below 'ins' is executed while IO instructions
1728 * are not intercepted; the instruction is emulated by L0.
1729 *
1730 * At the same time we are getting interrupts from the local APIC timer,
1731 * and we do intercept them in L1
1732 *
1733 * If the interrupt happens on the insb instruction, L0 will VMexit, emulate
1734 * the insb instruction and then it will inject the interrupt to L1 through
1735 * a nested VMexit. Due to a bug, it would leave pre-emulation values of RIP,
1736 * RAX and RSP in the VMCB.
1737 *
1738 * In our intercept handler we detect the bug by checking that RIP is that of
1739 * the insb instruction, but its memory operand has already been written.
1740 * This means that insb was already executed.
1741 */
1742
1743 static volatile int isr_cnt = 0;
1744 static volatile uint8_t io_port_var = 0xAA;
1745 extern const char insb_instruction_label[];
1746
reg_corruption_isr(isr_regs_t * regs)1747 static void reg_corruption_isr(isr_regs_t *regs)
1748 {
1749 isr_cnt++;
1750 apic_write(APIC_EOI, 0);
1751 }
1752
reg_corruption_prepare(struct svm_test * test)1753 static void reg_corruption_prepare(struct svm_test *test)
1754 {
1755 default_prepare(test);
1756 set_test_stage(test, 0);
1757
1758 vmcb->control.int_ctl = V_INTR_MASKING_MASK;
1759 vmcb->control.intercept |= (1ULL << INTERCEPT_INTR);
1760
1761 handle_irq(TIMER_VECTOR, reg_corruption_isr);
1762
1763 /* set local APIC to inject external interrupts */
1764 apic_setup_timer(TIMER_VECTOR, APIC_LVT_TIMER_PERIODIC);
1765 apic_start_timer(1000);
1766 }
1767
reg_corruption_test(struct svm_test * test)1768 static void reg_corruption_test(struct svm_test *test)
1769 {
1770 /* this is endless loop, which is interrupted by the timer interrupt */
1771 asm volatile (
1772 "1:\n\t"
1773 "movw $0x4d0, %%dx\n\t" // IO port
1774 "lea %[io_port_var], %%rdi\n\t"
1775 "movb $0xAA, %[io_port_var]\n\t"
1776 "insb_instruction_label:\n\t"
1777 "insb\n\t"
1778 "jmp 1b\n\t"
1779
1780 : [io_port_var] "=m" (io_port_var)
1781 : /* no inputs*/
1782 : "rdx", "rdi"
1783 );
1784 }
1785
reg_corruption_finished(struct svm_test * test)1786 static bool reg_corruption_finished(struct svm_test *test)
1787 {
1788 if (isr_cnt == 10000) {
1789 report_pass("No RIP corruption detected after %d timer interrupts",
1790 isr_cnt);
1791 set_test_stage(test, 1);
1792 goto cleanup;
1793 }
1794
1795 if (vmcb->control.exit_code == SVM_EXIT_INTR) {
1796
1797 void* guest_rip = (void*)vmcb->save.rip;
1798
1799 sti_nop_cli();
1800
1801 if (guest_rip == insb_instruction_label && io_port_var != 0xAA) {
1802 report_fail("RIP corruption detected after %d timer interrupts",
1803 isr_cnt);
1804 goto cleanup;
1805 }
1806
1807 }
1808 return false;
1809 cleanup:
1810 apic_cleanup_timer();
1811 return true;
1812
1813 }
1814
reg_corruption_check(struct svm_test * test)1815 static bool reg_corruption_check(struct svm_test *test)
1816 {
1817 return get_test_stage(test) == 1;
1818 }
1819
get_tss_entry(void * data)1820 static void get_tss_entry(void *data)
1821 {
1822 *((gdt_entry_t **)data) = get_tss_descr();
1823 }
1824
1825 static int orig_cpu_count;
1826
init_startup_prepare(struct svm_test * test)1827 static void init_startup_prepare(struct svm_test *test)
1828 {
1829 gdt_entry_t *tss_entry;
1830 int i;
1831
1832 on_cpu(1, get_tss_entry, &tss_entry);
1833
1834 orig_cpu_count = atomic_read(&cpu_online_count);
1835
1836 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_INIT | APIC_INT_ASSERT,
1837 id_map[1]);
1838
1839 delay(100000000ULL);
1840
1841 atomic_dec(&cpu_online_count);
1842
1843 tss_entry->type &= ~DESC_BUSY;
1844
1845 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_STARTUP, id_map[1]);
1846
1847 for (i = 0; i < 5 && atomic_read(&cpu_online_count) < orig_cpu_count; i++)
1848 delay(100000000ULL);
1849 }
1850
init_startup_finished(struct svm_test * test)1851 static bool init_startup_finished(struct svm_test *test)
1852 {
1853 return true;
1854 }
1855
init_startup_check(struct svm_test * test)1856 static bool init_startup_check(struct svm_test *test)
1857 {
1858 return atomic_read(&cpu_online_count) == orig_cpu_count;
1859 }
1860
1861 static volatile bool init_intercept;
1862
init_intercept_prepare(struct svm_test * test)1863 static void init_intercept_prepare(struct svm_test *test)
1864 {
1865 init_intercept = false;
1866 vmcb->control.intercept |= (1ULL << INTERCEPT_INIT);
1867 }
1868
init_intercept_test(struct svm_test * test)1869 static void init_intercept_test(struct svm_test *test)
1870 {
1871 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_INIT | APIC_INT_ASSERT, 0);
1872 }
1873
init_intercept_finished(struct svm_test * test)1874 static bool init_intercept_finished(struct svm_test *test)
1875 {
1876 vmcb->save.rip += 3;
1877
1878 if (vmcb->control.exit_code != SVM_EXIT_INIT) {
1879 report_fail("VMEXIT not due to init intercept. Exit reason 0x%x",
1880 vmcb->control.exit_code);
1881
1882 return true;
1883 }
1884
1885 init_intercept = true;
1886
1887 report_pass("INIT to vcpu intercepted");
1888
1889 return true;
1890 }
1891
init_intercept_check(struct svm_test * test)1892 static bool init_intercept_check(struct svm_test *test)
1893 {
1894 return init_intercept;
1895 }
1896
1897 /*
1898 * Setting host EFLAGS.TF causes a #DB trap after the VMRUN completes on the
1899 * host side (i.e., after the #VMEXIT from the guest).
1900 *
1901 * Setting host EFLAGS.RF suppresses any potential instruction breakpoint
1902 * match on the VMRUN and completion of the VMRUN instruction clears the
1903 * host EFLAGS.RF bit.
1904 *
1905 * [AMD APM]
1906 */
1907 static volatile u8 host_rflags_guest_main_flag = 0;
1908 static volatile u8 host_rflags_db_handler_flag = 0;
1909 static volatile bool host_rflags_ss_on_vmrun = false;
1910 static volatile bool host_rflags_vmrun_reached = false;
1911 static volatile bool host_rflags_set_tf = false;
1912 static volatile bool host_rflags_set_rf = false;
1913 static u64 rip_detected;
1914
1915 extern u64 *vmrun_rip;
1916
host_rflags_db_handler(struct ex_regs * r)1917 static void host_rflags_db_handler(struct ex_regs *r)
1918 {
1919 if (host_rflags_ss_on_vmrun) {
1920 if (host_rflags_vmrun_reached) {
1921 if (!host_rflags_set_rf) {
1922 r->rflags &= ~X86_EFLAGS_TF;
1923 rip_detected = r->rip;
1924 } else {
1925 r->rflags |= X86_EFLAGS_RF;
1926 ++host_rflags_db_handler_flag;
1927 }
1928 } else {
1929 if (r->rip == (u64)&vmrun_rip) {
1930 host_rflags_vmrun_reached = true;
1931
1932 if (host_rflags_set_rf) {
1933 host_rflags_guest_main_flag = 0;
1934 rip_detected = r->rip;
1935 r->rflags &= ~X86_EFLAGS_TF;
1936
1937 /* Trigger #DB via debug registers */
1938 write_dr0((void *)&vmrun_rip);
1939 write_dr7(0x403);
1940 }
1941 }
1942 }
1943 } else {
1944 r->rflags &= ~X86_EFLAGS_TF;
1945 }
1946 }
1947
host_rflags_prepare(struct svm_test * test)1948 static void host_rflags_prepare(struct svm_test *test)
1949 {
1950 default_prepare(test);
1951 handle_exception(DB_VECTOR, host_rflags_db_handler);
1952 set_test_stage(test, 0);
1953 }
1954
host_rflags_prepare_gif_clear(struct svm_test * test)1955 static void host_rflags_prepare_gif_clear(struct svm_test *test)
1956 {
1957 if (host_rflags_set_tf)
1958 write_rflags(read_rflags() | X86_EFLAGS_TF);
1959 }
1960
host_rflags_test(struct svm_test * test)1961 static void host_rflags_test(struct svm_test *test)
1962 {
1963 while (1) {
1964 if (get_test_stage(test) > 0) {
1965 if ((host_rflags_set_tf && !host_rflags_ss_on_vmrun && !host_rflags_db_handler_flag) ||
1966 (host_rflags_set_rf && host_rflags_db_handler_flag == 1))
1967 host_rflags_guest_main_flag = 1;
1968 }
1969
1970 if (get_test_stage(test) == 4)
1971 break;
1972 vmmcall();
1973 }
1974 }
1975
host_rflags_finished(struct svm_test * test)1976 static bool host_rflags_finished(struct svm_test *test)
1977 {
1978 switch (get_test_stage(test)) {
1979 case 0:
1980 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
1981 report_fail("Unexpected VMEXIT. Exit reason 0x%x",
1982 vmcb->control.exit_code);
1983 return true;
1984 }
1985 vmcb->save.rip += 3;
1986 /*
1987 * Setting host EFLAGS.TF not immediately before VMRUN, causes
1988 * #DB trap before first guest instruction is executed
1989 */
1990 host_rflags_set_tf = true;
1991 break;
1992 case 1:
1993 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL ||
1994 host_rflags_guest_main_flag != 1) {
1995 report_fail("Unexpected VMEXIT or #DB handler"
1996 " invoked before guest main. Exit reason 0x%x",
1997 vmcb->control.exit_code);
1998 return true;
1999 }
2000 vmcb->save.rip += 3;
2001 /*
2002 * Setting host EFLAGS.TF immediately before VMRUN, causes #DB
2003 * trap after VMRUN completes on the host side (i.e., after
2004 * VMEXIT from guest).
2005 */
2006 host_rflags_ss_on_vmrun = true;
2007 break;
2008 case 2:
2009 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL ||
2010 rip_detected != (u64)&vmrun_rip + 3) {
2011 report_fail("Unexpected VMEXIT or RIP mismatch."
2012 " Exit reason 0x%x, RIP actual: %lx, RIP expected: "
2013 "%lx", vmcb->control.exit_code,
2014 (u64)&vmrun_rip + 3, rip_detected);
2015 return true;
2016 }
2017 host_rflags_set_rf = true;
2018 host_rflags_guest_main_flag = 0;
2019 host_rflags_vmrun_reached = false;
2020 vmcb->save.rip += 3;
2021 break;
2022 case 3:
2023 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL ||
2024 rip_detected != (u64)&vmrun_rip ||
2025 host_rflags_guest_main_flag != 1 ||
2026 host_rflags_db_handler_flag > 1 ||
2027 read_rflags() & X86_EFLAGS_RF) {
2028 report_fail("Unexpected VMEXIT or RIP mismatch or "
2029 "EFLAGS.RF not cleared."
2030 " Exit reason 0x%x, RIP actual: %lx, RIP expected: "
2031 "%lx", vmcb->control.exit_code,
2032 (u64)&vmrun_rip, rip_detected);
2033 return true;
2034 }
2035 host_rflags_set_tf = false;
2036 host_rflags_set_rf = false;
2037 vmcb->save.rip += 3;
2038 break;
2039 default:
2040 return true;
2041 }
2042 inc_test_stage(test);
2043 return get_test_stage(test) == 5;
2044 }
2045
host_rflags_check(struct svm_test * test)2046 static bool host_rflags_check(struct svm_test *test)
2047 {
2048 return get_test_stage(test) == 4;
2049 }
2050
2051 #define TEST(name) { #name, .v2 = name }
2052
2053 /*
2054 * v2 tests
2055 */
2056
2057 /*
2058 * Ensure that kvm recalculates the L1 guest's CPUID.01H:ECX.OSXSAVE
2059 * after VM-exit from an L2 guest that sets CR4.OSXSAVE to a different
2060 * value than in L1.
2061 */
2062
svm_cr4_osxsave_test_guest(struct svm_test * test)2063 static void svm_cr4_osxsave_test_guest(struct svm_test *test)
2064 {
2065 write_cr4(read_cr4() & ~X86_CR4_OSXSAVE);
2066 }
2067
svm_cr4_osxsave_test(void)2068 static void svm_cr4_osxsave_test(void)
2069 {
2070 if (!this_cpu_has(X86_FEATURE_XSAVE)) {
2071 report_skip("XSAVE not detected");
2072 return;
2073 }
2074
2075 if (!(read_cr4() & X86_CR4_OSXSAVE)) {
2076 unsigned long cr4 = read_cr4() | X86_CR4_OSXSAVE;
2077
2078 write_cr4(cr4);
2079 vmcb->save.cr4 = cr4;
2080 }
2081
2082 report(this_cpu_has(X86_FEATURE_OSXSAVE), "CPUID.01H:ECX.XSAVE set before VMRUN");
2083
2084 test_set_guest(svm_cr4_osxsave_test_guest);
2085 report(svm_vmrun() == SVM_EXIT_VMMCALL,
2086 "svm_cr4_osxsave_test_guest finished with VMMCALL");
2087
2088 report(this_cpu_has(X86_FEATURE_OSXSAVE), "CPUID.01H:ECX.XSAVE set after VMRUN");
2089 }
2090
basic_guest_main(struct svm_test * test)2091 static void basic_guest_main(struct svm_test *test)
2092 {
2093 }
2094
2095
2096 #define SVM_TEST_REG_RESERVED_BITS(start, end, inc, str_name, reg, val, \
2097 resv_mask) \
2098 { \
2099 u64 tmp, mask; \
2100 int i; \
2101 \
2102 for (i = start; i <= end; i = i + inc) { \
2103 mask = 1ull << i; \
2104 if (!(mask & resv_mask)) \
2105 continue; \
2106 tmp = val | mask; \
2107 reg = tmp; \
2108 report(svm_vmrun() == SVM_EXIT_ERR, "Test %s %d:%d: %lx", \
2109 str_name, end, start, tmp); \
2110 } \
2111 }
2112
2113 #define SVM_TEST_CR_RESERVED_BITS(start, end, inc, cr, val, resv_mask, \
2114 exit_code, test_name) \
2115 { \
2116 u64 tmp, mask; \
2117 u32 r; \
2118 int i; \
2119 \
2120 for (i = start; i <= end; i = i + inc) { \
2121 mask = 1ull << i; \
2122 if (!(mask & resv_mask)) \
2123 continue; \
2124 tmp = val | mask; \
2125 switch (cr) { \
2126 case 0: \
2127 vmcb->save.cr0 = tmp; \
2128 break; \
2129 case 3: \
2130 vmcb->save.cr3 = tmp; \
2131 break; \
2132 case 4: \
2133 vmcb->save.cr4 = tmp; \
2134 } \
2135 r = svm_vmrun(); \
2136 report(r == exit_code, "Test CR%d %s%d:%d: %lx, wanted exit 0x%x, got 0x%x", \
2137 cr, test_name, end, start, tmp, exit_code, r); \
2138 } \
2139 }
2140
test_efer(void)2141 static void test_efer(void)
2142 {
2143 /*
2144 * Un-setting EFER.SVME is illegal
2145 */
2146 u64 efer_saved = vmcb->save.efer;
2147 u64 efer = efer_saved;
2148
2149 report (svm_vmrun() == SVM_EXIT_VMMCALL, "EFER.SVME: %lx", efer);
2150 efer &= ~EFER_SVME;
2151 vmcb->save.efer = efer;
2152 report (svm_vmrun() == SVM_EXIT_ERR, "EFER.SVME: %lx", efer);
2153 vmcb->save.efer = efer_saved;
2154
2155 /*
2156 * EFER MBZ bits: 63:16, 9
2157 */
2158 efer_saved = vmcb->save.efer;
2159
2160 SVM_TEST_REG_RESERVED_BITS(8, 9, 1, "EFER", vmcb->save.efer,
2161 efer_saved, SVM_EFER_RESERVED_MASK);
2162 SVM_TEST_REG_RESERVED_BITS(16, 63, 4, "EFER", vmcb->save.efer,
2163 efer_saved, SVM_EFER_RESERVED_MASK);
2164
2165 /*
2166 * EFER.LME and CR0.PG are both set and CR4.PAE is zero.
2167 */
2168 u64 cr0_saved = vmcb->save.cr0;
2169 u64 cr0;
2170 u64 cr4_saved = vmcb->save.cr4;
2171 u64 cr4;
2172
2173 efer = efer_saved | EFER_LME;
2174 vmcb->save.efer = efer;
2175 cr0 = cr0_saved | X86_CR0_PG | X86_CR0_PE;
2176 vmcb->save.cr0 = cr0;
2177 cr4 = cr4_saved & ~X86_CR4_PAE;
2178 vmcb->save.cr4 = cr4;
2179 report(svm_vmrun() == SVM_EXIT_ERR, "EFER.LME=1 (%lx), "
2180 "CR0.PG=1 (%lx) and CR4.PAE=0 (%lx)", efer, cr0, cr4);
2181
2182 /*
2183 * EFER.LME and CR0.PG are both set and CR0.PE is zero.
2184 * CR4.PAE needs to be set as we otherwise cannot
2185 * determine if CR4.PAE=0 or CR0.PE=0 triggered the
2186 * SVM_EXIT_ERR.
2187 */
2188 cr4 = cr4_saved | X86_CR4_PAE;
2189 vmcb->save.cr4 = cr4;
2190 cr0 &= ~X86_CR0_PE;
2191 vmcb->save.cr0 = cr0;
2192 report(svm_vmrun() == SVM_EXIT_ERR, "EFER.LME=1 (%lx), "
2193 "CR0.PG=1 and CR0.PE=0 (%lx)", efer, cr0);
2194
2195 /*
2196 * EFER.LME, CR0.PG, CR4.PAE, CS.L, and CS.D are all non-zero.
2197 */
2198 u32 cs_attrib_saved = vmcb->save.cs.attrib;
2199 u32 cs_attrib;
2200
2201 cr0 |= X86_CR0_PE;
2202 vmcb->save.cr0 = cr0;
2203 cs_attrib = cs_attrib_saved | SVM_SELECTOR_L_MASK |
2204 SVM_SELECTOR_DB_MASK;
2205 vmcb->save.cs.attrib = cs_attrib;
2206 report(svm_vmrun() == SVM_EXIT_ERR, "EFER.LME=1 (%lx), "
2207 "CR0.PG=1 (%lx), CR4.PAE=1 (%lx), CS.L=1 and CS.D=1 (%x)",
2208 efer, cr0, cr4, cs_attrib);
2209
2210 vmcb->save.cr0 = cr0_saved;
2211 vmcb->save.cr4 = cr4_saved;
2212 vmcb->save.efer = efer_saved;
2213 vmcb->save.cs.attrib = cs_attrib_saved;
2214 }
2215
test_cr0(void)2216 static void test_cr0(void)
2217 {
2218 /*
2219 * Un-setting CR0.CD and setting CR0.NW is illegal combination
2220 */
2221 u64 cr0_saved = vmcb->save.cr0;
2222 u64 cr0 = cr0_saved;
2223
2224 cr0 |= X86_CR0_CD;
2225 cr0 &= ~X86_CR0_NW;
2226 vmcb->save.cr0 = cr0;
2227 report (svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR0 CD=1,NW=0: %lx",
2228 cr0);
2229 cr0 |= X86_CR0_NW;
2230 vmcb->save.cr0 = cr0;
2231 report (svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR0 CD=1,NW=1: %lx",
2232 cr0);
2233 cr0 &= ~X86_CR0_NW;
2234 cr0 &= ~X86_CR0_CD;
2235 vmcb->save.cr0 = cr0;
2236 report (svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR0 CD=0,NW=0: %lx",
2237 cr0);
2238 cr0 |= X86_CR0_NW;
2239 vmcb->save.cr0 = cr0;
2240 report (svm_vmrun() == SVM_EXIT_ERR, "Test CR0 CD=0,NW=1: %lx",
2241 cr0);
2242 vmcb->save.cr0 = cr0_saved;
2243
2244 /*
2245 * CR0[63:32] are not zero
2246 */
2247 cr0 = cr0_saved;
2248
2249 SVM_TEST_REG_RESERVED_BITS(32, 63, 4, "CR0", vmcb->save.cr0, cr0_saved,
2250 SVM_CR0_RESERVED_MASK);
2251 vmcb->save.cr0 = cr0_saved;
2252 }
2253
test_cr3(void)2254 static void test_cr3(void)
2255 {
2256 /*
2257 * CR3 MBZ bits based on different modes:
2258 * [63:52] - long mode
2259 */
2260 u64 cr3_saved = vmcb->save.cr3;
2261
2262 SVM_TEST_CR_RESERVED_BITS(0, 63, 1, 3, cr3_saved,
2263 SVM_CR3_LONG_MBZ_MASK, SVM_EXIT_ERR, "");
2264
2265 vmcb->save.cr3 = cr3_saved & ~SVM_CR3_LONG_MBZ_MASK;
2266 report(svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR3 63:0: %lx",
2267 vmcb->save.cr3);
2268
2269 /*
2270 * CR3 non-MBZ reserved bits based on different modes:
2271 * [11:5] [2:0] - long mode (PCIDE=0)
2272 * [2:0] - PAE legacy mode
2273 */
2274 u64 cr4_saved = vmcb->save.cr4;
2275 u64 *pdpe = npt_get_pml4e();
2276
2277 /*
2278 * Long mode
2279 */
2280 if (this_cpu_has(X86_FEATURE_PCID)) {
2281 vmcb->save.cr4 = cr4_saved | X86_CR4_PCIDE;
2282 SVM_TEST_CR_RESERVED_BITS(0, 11, 1, 3, cr3_saved,
2283 SVM_CR3_LONG_RESERVED_MASK, SVM_EXIT_VMMCALL, "(PCIDE=1) ");
2284
2285 vmcb->save.cr3 = cr3_saved & ~SVM_CR3_LONG_RESERVED_MASK;
2286 report(svm_vmrun() == SVM_EXIT_VMMCALL, "Test CR3 63:0: %lx",
2287 vmcb->save.cr3);
2288 }
2289
2290 vmcb->save.cr4 = cr4_saved & ~X86_CR4_PCIDE;
2291
2292 if (!npt_supported())
2293 goto skip_npt_only;
2294
2295 /* Clear P (Present) bit in NPT in order to trigger #NPF */
2296 pdpe[0] &= ~1ULL;
2297
2298 SVM_TEST_CR_RESERVED_BITS(0, 11, 1, 3, cr3_saved,
2299 SVM_CR3_LONG_RESERVED_MASK, SVM_EXIT_NPF, "(PCIDE=0) ");
2300
2301 pdpe[0] |= 1ULL;
2302 vmcb->save.cr3 = cr3_saved;
2303
2304 /*
2305 * PAE legacy
2306 */
2307 pdpe[0] &= ~1ULL;
2308 vmcb->save.cr4 = cr4_saved | X86_CR4_PAE;
2309 SVM_TEST_CR_RESERVED_BITS(0, 2, 1, 3, cr3_saved,
2310 SVM_CR3_PAE_LEGACY_RESERVED_MASK, SVM_EXIT_NPF, "(PAE) ");
2311
2312 pdpe[0] |= 1ULL;
2313
2314 skip_npt_only:
2315 vmcb->save.cr3 = cr3_saved;
2316 vmcb->save.cr4 = cr4_saved;
2317 }
2318
2319 /* Test CR4 MBZ bits based on legacy or long modes */
test_cr4(void)2320 static void test_cr4(void)
2321 {
2322 u64 cr4_saved = vmcb->save.cr4;
2323 u64 efer_saved = vmcb->save.efer;
2324 u64 efer = efer_saved;
2325
2326 efer &= ~EFER_LME;
2327 vmcb->save.efer = efer;
2328 SVM_TEST_CR_RESERVED_BITS(12, 31, 1, 4, cr4_saved,
2329 SVM_CR4_LEGACY_RESERVED_MASK, SVM_EXIT_ERR, "");
2330
2331 efer |= EFER_LME;
2332 vmcb->save.efer = efer;
2333 SVM_TEST_CR_RESERVED_BITS(12, 31, 1, 4, cr4_saved,
2334 SVM_CR4_RESERVED_MASK, SVM_EXIT_ERR, "");
2335 SVM_TEST_CR_RESERVED_BITS(32, 63, 4, 4, cr4_saved,
2336 SVM_CR4_RESERVED_MASK, SVM_EXIT_ERR, "");
2337
2338 vmcb->save.cr4 = cr4_saved;
2339 vmcb->save.efer = efer_saved;
2340 }
2341
test_dr(void)2342 static void test_dr(void)
2343 {
2344 /*
2345 * DR6[63:32] and DR7[63:32] are MBZ
2346 */
2347 u64 dr_saved = vmcb->save.dr6;
2348
2349 SVM_TEST_REG_RESERVED_BITS(32, 63, 4, "DR6", vmcb->save.dr6, dr_saved,
2350 SVM_DR6_RESERVED_MASK);
2351 vmcb->save.dr6 = dr_saved;
2352
2353 dr_saved = vmcb->save.dr7;
2354 SVM_TEST_REG_RESERVED_BITS(32, 63, 4, "DR7", vmcb->save.dr7, dr_saved,
2355 SVM_DR7_RESERVED_MASK);
2356
2357 vmcb->save.dr7 = dr_saved;
2358 }
2359
2360 /* TODO: verify if high 32-bits are sign- or zero-extended on bare metal */
2361 #define TEST_BITMAP_ADDR(save_intercept, type, addr, exit_code, \
2362 msg) { \
2363 vmcb->control.intercept = saved_intercept | 1ULL << type; \
2364 if (type == INTERCEPT_MSR_PROT) \
2365 vmcb->control.msrpm_base_pa = addr; \
2366 else \
2367 vmcb->control.iopm_base_pa = addr; \
2368 report(svm_vmrun() == exit_code, \
2369 "Test %s address: %lx", msg, addr); \
2370 }
2371
2372 /*
2373 * If the MSR or IOIO intercept table extends to a physical address that
2374 * is greater than or equal to the maximum supported physical address, the
2375 * guest state is illegal.
2376 *
2377 * The VMRUN instruction ignores the lower 12 bits of the address specified
2378 * in the VMCB.
2379 *
2380 * MSRPM spans 2 contiguous 4KB pages while IOPM spans 2 contiguous 4KB
2381 * pages + 1 byte.
2382 *
2383 * [APM vol 2]
2384 *
2385 * Note: Unallocated MSRPM addresses conforming to consistency checks, generate
2386 * #NPF.
2387 */
test_msrpm_iopm_bitmap_addrs(void)2388 static void test_msrpm_iopm_bitmap_addrs(void)
2389 {
2390 u64 saved_intercept = vmcb->control.intercept;
2391 u64 addr_beyond_limit = 1ull << cpuid_maxphyaddr();
2392 u64 addr = virt_to_phys(msr_bitmap) & (~((1ull << 12) - 1));
2393
2394 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT,
2395 addr_beyond_limit - 2 * PAGE_SIZE, SVM_EXIT_ERR,
2396 "MSRPM");
2397 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT,
2398 addr_beyond_limit - 2 * PAGE_SIZE + 1, SVM_EXIT_ERR,
2399 "MSRPM");
2400 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT,
2401 addr_beyond_limit - PAGE_SIZE, SVM_EXIT_ERR,
2402 "MSRPM");
2403 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT, addr,
2404 SVM_EXIT_VMMCALL, "MSRPM");
2405 addr |= (1ull << 12) - 1;
2406 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_MSR_PROT, addr,
2407 SVM_EXIT_VMMCALL, "MSRPM");
2408
2409 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT,
2410 addr_beyond_limit - 4 * PAGE_SIZE, SVM_EXIT_VMMCALL,
2411 "IOPM");
2412 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT,
2413 addr_beyond_limit - 3 * PAGE_SIZE, SVM_EXIT_VMMCALL,
2414 "IOPM");
2415 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT,
2416 addr_beyond_limit - 2 * PAGE_SIZE - 2, SVM_EXIT_VMMCALL,
2417 "IOPM");
2418 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT,
2419 addr_beyond_limit - 2 * PAGE_SIZE, SVM_EXIT_ERR,
2420 "IOPM");
2421 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT,
2422 addr_beyond_limit - PAGE_SIZE, SVM_EXIT_ERR,
2423 "IOPM");
2424 addr = virt_to_phys(io_bitmap) & (~((1ull << 11) - 1));
2425 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, addr,
2426 SVM_EXIT_VMMCALL, "IOPM");
2427 addr |= (1ull << 12) - 1;
2428 TEST_BITMAP_ADDR(saved_intercept, INTERCEPT_IOIO_PROT, addr,
2429 SVM_EXIT_VMMCALL, "IOPM");
2430
2431 vmcb->control.intercept = saved_intercept;
2432 }
2433
2434 /*
2435 * Unlike VMSAVE, VMRUN seems not to update the value of noncanonical
2436 * segment bases in the VMCB. However, VMENTRY succeeds as documented.
2437 */
2438 #define TEST_CANONICAL_VMRUN(seg_base, msg) \
2439 saved_addr = seg_base; \
2440 seg_base = (seg_base & ((1ul << addr_limit) - 1)) | noncanonical_mask; \
2441 return_value = svm_vmrun(); \
2442 report(return_value == SVM_EXIT_VMMCALL, \
2443 "Successful VMRUN with noncanonical %s.base", msg); \
2444 seg_base = saved_addr;
2445
2446
2447 #define TEST_CANONICAL_VMLOAD(seg_base, msg) \
2448 saved_addr = seg_base; \
2449 seg_base = (seg_base & ((1ul << addr_limit) - 1)) | noncanonical_mask; \
2450 asm volatile ("vmload %0" : : "a"(vmcb_phys) : "memory"); \
2451 asm volatile ("vmsave %0" : : "a"(vmcb_phys) : "memory"); \
2452 report(is_canonical(seg_base), \
2453 "Test %s.base for canonical form: %lx", msg, seg_base); \
2454 seg_base = saved_addr;
2455
test_canonicalization(void)2456 static void test_canonicalization(void)
2457 {
2458 u64 saved_addr;
2459 u64 return_value;
2460 u64 addr_limit;
2461 u64 vmcb_phys = virt_to_phys(vmcb);
2462
2463 addr_limit = (this_cpu_has(X86_FEATURE_LA57)) ? 57 : 48;
2464 u64 noncanonical_mask = NONCANONICAL & ~((1ul << addr_limit) - 1);
2465
2466 TEST_CANONICAL_VMLOAD(vmcb->save.fs.base, "FS");
2467 TEST_CANONICAL_VMLOAD(vmcb->save.gs.base, "GS");
2468 TEST_CANONICAL_VMLOAD(vmcb->save.ldtr.base, "LDTR");
2469 TEST_CANONICAL_VMLOAD(vmcb->save.tr.base, "TR");
2470 TEST_CANONICAL_VMLOAD(vmcb->save.kernel_gs_base, "KERNEL GS");
2471 TEST_CANONICAL_VMRUN(vmcb->save.es.base, "ES");
2472 TEST_CANONICAL_VMRUN(vmcb->save.cs.base, "CS");
2473 TEST_CANONICAL_VMRUN(vmcb->save.ss.base, "SS");
2474 TEST_CANONICAL_VMRUN(vmcb->save.ds.base, "DS");
2475 TEST_CANONICAL_VMRUN(vmcb->save.gdtr.base, "GDTR");
2476 TEST_CANONICAL_VMRUN(vmcb->save.idtr.base, "IDTR");
2477 }
2478
2479 /*
2480 * When VMRUN loads a guest value of 1 in EFLAGS.TF, that value does not
2481 * cause a trace trap between the VMRUN and the first guest instruction, but
2482 * rather after completion of the first guest instruction.
2483 *
2484 * [APM vol 2]
2485 */
2486 u64 guest_rflags_test_trap_rip;
2487
guest_rflags_test_db_handler(struct ex_regs * r)2488 static void guest_rflags_test_db_handler(struct ex_regs *r)
2489 {
2490 guest_rflags_test_trap_rip = r->rip;
2491 r->rflags &= ~X86_EFLAGS_TF;
2492 }
2493
svm_guest_state_test(void)2494 static void svm_guest_state_test(void)
2495 {
2496 test_set_guest(basic_guest_main);
2497 test_efer();
2498 test_cr0();
2499 test_cr3();
2500 test_cr4();
2501 test_dr();
2502 test_msrpm_iopm_bitmap_addrs();
2503 test_canonicalization();
2504 }
2505
2506 extern void guest_rflags_test_guest(struct svm_test *test);
2507 extern u64 *insn2;
2508 extern u64 *guest_end;
2509
2510 asm("guest_rflags_test_guest:\n\t"
2511 "push %rbp\n\t"
2512 ".global insn2\n\t"
2513 "insn2:\n\t"
2514 "mov %rsp,%rbp\n\t"
2515 "vmmcall\n\t"
2516 "vmmcall\n\t"
2517 ".global guest_end\n\t"
2518 "guest_end:\n\t"
2519 "vmmcall\n\t"
2520 "pop %rbp\n\t"
2521 "ret");
2522
svm_test_singlestep(void)2523 static void svm_test_singlestep(void)
2524 {
2525 handle_exception(DB_VECTOR, guest_rflags_test_db_handler);
2526
2527 /*
2528 * Trap expected after completion of first guest instruction
2529 */
2530 vmcb->save.rflags |= X86_EFLAGS_TF;
2531 report (__svm_vmrun((u64)guest_rflags_test_guest) == SVM_EXIT_VMMCALL &&
2532 guest_rflags_test_trap_rip == (u64)&insn2,
2533 "Test EFLAGS.TF on VMRUN: trap expected after completion of first guest instruction");
2534 /*
2535 * No trap expected
2536 */
2537 guest_rflags_test_trap_rip = 0;
2538 vmcb->save.rip += 3;
2539 vmcb->save.rflags |= X86_EFLAGS_TF;
2540 report (__svm_vmrun(vmcb->save.rip) == SVM_EXIT_VMMCALL &&
2541 guest_rflags_test_trap_rip == 0, "Test EFLAGS.TF on VMRUN: trap not expected");
2542
2543 /*
2544 * Let guest finish execution
2545 */
2546 vmcb->save.rip += 3;
2547 report (__svm_vmrun(vmcb->save.rip) == SVM_EXIT_VMMCALL &&
2548 vmcb->save.rip == (u64)&guest_end, "Test EFLAGS.TF on VMRUN: guest execution completion");
2549 }
2550
2551 static bool volatile svm_errata_reproduced = false;
2552 static unsigned long volatile physical = 0;
2553
2554
2555 /*
2556 *
2557 * Test the following errata:
2558 * If the VMRUN/VMSAVE/VMLOAD are attempted by the nested guest,
2559 * the CPU would first check the EAX against host reserved memory
2560 * regions (so far only SMM_ADDR/SMM_MASK are known to cause it),
2561 * and only then signal #VMexit
2562 *
2563 * Try to reproduce this by trying vmsave on each possible 4K aligned memory
2564 * address in the low 4G where the SMM area has to reside.
2565 */
2566
gp_isr(struct ex_regs * r)2567 static void gp_isr(struct ex_regs *r)
2568 {
2569 svm_errata_reproduced = true;
2570 /* skip over the vmsave instruction*/
2571 r->rip += 3;
2572 }
2573
svm_vmrun_errata_test(void)2574 static void svm_vmrun_errata_test(void)
2575 {
2576 unsigned long *last_page = NULL;
2577
2578 handle_exception(GP_VECTOR, gp_isr);
2579
2580 while (!svm_errata_reproduced) {
2581
2582 unsigned long *page = alloc_pages(1);
2583
2584 if (!page) {
2585 report_pass("All guest memory tested, no bug found");
2586 break;
2587 }
2588
2589 physical = virt_to_phys(page);
2590
2591 asm volatile (
2592 "mov %[_physical], %%rax\n\t"
2593 "vmsave %%rax\n\t"
2594
2595 : [_physical] "=m" (physical)
2596 : /* no inputs*/
2597 : "rax" /*clobbers*/
2598 );
2599
2600 if (svm_errata_reproduced) {
2601 report_fail("Got #GP exception - svm errata reproduced at 0x%lx",
2602 physical);
2603 break;
2604 }
2605
2606 *page = (unsigned long)last_page;
2607 last_page = page;
2608 }
2609
2610 while (last_page) {
2611 unsigned long *page = last_page;
2612 last_page = (unsigned long *)*last_page;
2613 free_pages_by_order(page, 1);
2614 }
2615 }
2616
vmload_vmsave_guest_main(struct svm_test * test)2617 static void vmload_vmsave_guest_main(struct svm_test *test)
2618 {
2619 u64 vmcb_phys = virt_to_phys(vmcb);
2620
2621 asm volatile ("vmload %0" : : "a"(vmcb_phys));
2622 asm volatile ("vmsave %0" : : "a"(vmcb_phys));
2623 }
2624
svm_vmload_vmsave(void)2625 static void svm_vmload_vmsave(void)
2626 {
2627 u32 intercept_saved = vmcb->control.intercept;
2628
2629 test_set_guest(vmload_vmsave_guest_main);
2630
2631 /*
2632 * Disabling intercept for VMLOAD and VMSAVE doesn't cause
2633 * respective #VMEXIT to host
2634 */
2635 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMLOAD);
2636 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMSAVE);
2637 svm_vmrun();
2638 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test "
2639 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT");
2640
2641 /*
2642 * Enabling intercept for VMLOAD and VMSAVE causes respective
2643 * #VMEXIT to host
2644 */
2645 vmcb->control.intercept |= (1ULL << INTERCEPT_VMLOAD);
2646 svm_vmrun();
2647 report(vmcb->control.exit_code == SVM_EXIT_VMLOAD, "Test "
2648 "VMLOAD/VMSAVE intercept: Expected VMLOAD #VMEXIT");
2649 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMLOAD);
2650 vmcb->control.intercept |= (1ULL << INTERCEPT_VMSAVE);
2651 svm_vmrun();
2652 report(vmcb->control.exit_code == SVM_EXIT_VMSAVE, "Test "
2653 "VMLOAD/VMSAVE intercept: Expected VMSAVE #VMEXIT");
2654 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMSAVE);
2655 svm_vmrun();
2656 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test "
2657 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT");
2658
2659 vmcb->control.intercept |= (1ULL << INTERCEPT_VMLOAD);
2660 svm_vmrun();
2661 report(vmcb->control.exit_code == SVM_EXIT_VMLOAD, "Test "
2662 "VMLOAD/VMSAVE intercept: Expected VMLOAD #VMEXIT");
2663 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMLOAD);
2664 svm_vmrun();
2665 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test "
2666 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT");
2667
2668 vmcb->control.intercept |= (1ULL << INTERCEPT_VMSAVE);
2669 svm_vmrun();
2670 report(vmcb->control.exit_code == SVM_EXIT_VMSAVE, "Test "
2671 "VMLOAD/VMSAVE intercept: Expected VMSAVE #VMEXIT");
2672 vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMSAVE);
2673 svm_vmrun();
2674 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "Test "
2675 "VMLOAD/VMSAVE intercept: Expected VMMCALL #VMEXIT");
2676
2677 vmcb->control.intercept = intercept_saved;
2678 }
2679
prepare_vgif_enabled(struct svm_test * test)2680 static void prepare_vgif_enabled(struct svm_test *test)
2681 {
2682 default_prepare(test);
2683 }
2684
test_vgif(struct svm_test * test)2685 static void test_vgif(struct svm_test *test)
2686 {
2687 asm volatile ("vmmcall\n\tstgi\n\tvmmcall\n\tclgi\n\tvmmcall\n\t");
2688 }
2689
vgif_finished(struct svm_test * test)2690 static bool vgif_finished(struct svm_test *test)
2691 {
2692 switch (get_test_stage(test))
2693 {
2694 case 0:
2695 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
2696 report_fail("VMEXIT not due to vmmcall.");
2697 return true;
2698 }
2699 vmcb->control.int_ctl |= V_GIF_ENABLED_MASK;
2700 vmcb->save.rip += 3;
2701 inc_test_stage(test);
2702 break;
2703 case 1:
2704 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
2705 report_fail("VMEXIT not due to vmmcall.");
2706 return true;
2707 }
2708 if (!(vmcb->control.int_ctl & V_GIF_MASK)) {
2709 report_fail("Failed to set VGIF when executing STGI.");
2710 vmcb->control.int_ctl &= ~V_GIF_ENABLED_MASK;
2711 return true;
2712 }
2713 report_pass("STGI set VGIF bit.");
2714 vmcb->save.rip += 3;
2715 inc_test_stage(test);
2716 break;
2717 case 2:
2718 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
2719 report_fail("VMEXIT not due to vmmcall.");
2720 return true;
2721 }
2722 if (vmcb->control.int_ctl & V_GIF_MASK) {
2723 report_fail("Failed to clear VGIF when executing CLGI.");
2724 vmcb->control.int_ctl &= ~V_GIF_ENABLED_MASK;
2725 return true;
2726 }
2727 report_pass("CLGI cleared VGIF bit.");
2728 vmcb->save.rip += 3;
2729 inc_test_stage(test);
2730 vmcb->control.int_ctl &= ~V_GIF_ENABLED_MASK;
2731 break;
2732 default:
2733 return true;
2734 break;
2735 }
2736
2737 return get_test_stage(test) == 3;
2738 }
2739
vgif_check(struct svm_test * test)2740 static bool vgif_check(struct svm_test *test)
2741 {
2742 return get_test_stage(test) == 3;
2743 }
2744
2745
2746 static int pause_test_counter;
2747 static int wait_counter;
2748
pause_filter_test_guest_main(struct svm_test * test)2749 static void pause_filter_test_guest_main(struct svm_test *test)
2750 {
2751 int i;
2752 for (i = 0 ; i < pause_test_counter ; i++)
2753 pause();
2754
2755 if (!wait_counter)
2756 return;
2757
2758 for (i = 0; i < wait_counter; i++)
2759 ;
2760
2761 for (i = 0 ; i < pause_test_counter ; i++)
2762 pause();
2763
2764 }
2765
pause_filter_run_test(int pause_iterations,int filter_value,int wait_iterations,int threshold)2766 static void pause_filter_run_test(int pause_iterations, int filter_value, int wait_iterations, int threshold)
2767 {
2768 test_set_guest(pause_filter_test_guest_main);
2769
2770 pause_test_counter = pause_iterations;
2771 wait_counter = wait_iterations;
2772
2773 vmcb->control.pause_filter_count = filter_value;
2774 vmcb->control.pause_filter_thresh = threshold;
2775 svm_vmrun();
2776
2777 if (filter_value <= pause_iterations || wait_iterations < threshold)
2778 report(vmcb->control.exit_code == SVM_EXIT_PAUSE, "expected PAUSE vmexit");
2779 else
2780 report(vmcb->control.exit_code == SVM_EXIT_VMMCALL, "no expected PAUSE vmexit");
2781 }
2782
pause_filter_test(void)2783 static void pause_filter_test(void)
2784 {
2785 if (!pause_filter_supported()) {
2786 report_skip("PAUSE filter not supported in the guest");
2787 return;
2788 }
2789
2790 vmcb->control.intercept |= (1 << INTERCEPT_PAUSE);
2791
2792 // filter count more that pause count - no VMexit
2793 pause_filter_run_test(10, 9, 0, 0);
2794
2795 // filter count smaller pause count - no VMexit
2796 pause_filter_run_test(20, 21, 0, 0);
2797
2798
2799 if (pause_threshold_supported()) {
2800 // filter count smaller pause count - no VMexit + large enough threshold
2801 // so that filter counter resets
2802 pause_filter_run_test(20, 21, 1000, 10);
2803
2804 // filter count smaller pause count - no VMexit + small threshold
2805 // so that filter doesn't reset
2806 pause_filter_run_test(20, 21, 10, 1000);
2807 } else {
2808 report_skip("PAUSE threshold not supported in the guest");
2809 return;
2810 }
2811 }
2812
2813 /* If CR0.TS and CR0.EM are cleared in L2, no #NM is generated. */
svm_no_nm_test(void)2814 static void svm_no_nm_test(void)
2815 {
2816 write_cr0(read_cr0() & ~X86_CR0_TS);
2817 test_set_guest((test_guest_func)fnop);
2818
2819 vmcb->save.cr0 = vmcb->save.cr0 & ~(X86_CR0_TS | X86_CR0_EM);
2820 report(svm_vmrun() == SVM_EXIT_VMMCALL,
2821 "fnop with CR0.TS and CR0.EM unset no #NM exception");
2822 }
2823
amd_get_lbr_rip(u32 msr)2824 static u64 amd_get_lbr_rip(u32 msr)
2825 {
2826 return rdmsr(msr) & ~AMD_LBR_RECORD_MISPREDICT;
2827 }
2828
2829 #define HOST_CHECK_LBR(from_expected, to_expected) \
2830 do { \
2831 TEST_EXPECT_EQ((u64)from_expected, amd_get_lbr_rip(MSR_IA32_LASTBRANCHFROMIP)); \
2832 TEST_EXPECT_EQ((u64)to_expected, amd_get_lbr_rip(MSR_IA32_LASTBRANCHTOIP)); \
2833 } while (0)
2834
2835 /*
2836 * FIXME: Do something other than generate an exception to communicate failure.
2837 * Debugging without expected vs. actual is an absolute nightmare.
2838 */
2839 #define GUEST_CHECK_LBR(from_expected, to_expected) \
2840 do { \
2841 if ((u64)(from_expected) != amd_get_lbr_rip(MSR_IA32_LASTBRANCHFROMIP)) \
2842 asm volatile("ud2"); \
2843 if ((u64)(to_expected) != amd_get_lbr_rip(MSR_IA32_LASTBRANCHTOIP)) \
2844 asm volatile("ud2"); \
2845 } while (0)
2846
2847 #define REPORT_GUEST_LBR_ERROR(vmcb) \
2848 report(false, "LBR guest test failed. Exit reason 0x%x, RIP = %lx, from = %lx, to = %lx, ex from = %lx, ex to = %lx", \
2849 vmcb->control.exit_code, vmcb->save.rip, \
2850 vmcb->save.br_from, vmcb->save.br_to, \
2851 vmcb->save.last_excp_from, vmcb->save.last_excp_to)
2852
2853 #define DO_BRANCH(branch_name) \
2854 asm volatile ( \
2855 # branch_name "_from:" \
2856 "jmp " # branch_name "_to\n" \
2857 "nop\n" \
2858 "nop\n" \
2859 # branch_name "_to:" \
2860 "nop\n" \
2861 )
2862
2863
2864 extern u64 guest_branch0_from, guest_branch0_to;
2865 extern u64 guest_branch2_from, guest_branch2_to;
2866
2867 extern u64 host_branch0_from, host_branch0_to;
2868 extern u64 host_branch2_from, host_branch2_to;
2869 extern u64 host_branch3_from, host_branch3_to;
2870 extern u64 host_branch4_from, host_branch4_to;
2871
2872 u64 dbgctl;
2873
svm_lbrv_test_guest1(void)2874 static void svm_lbrv_test_guest1(void)
2875 {
2876 /*
2877 * This guest expects the LBR to be already enabled when it starts,
2878 * it does a branch, and then disables the LBR and then checks.
2879 */
2880
2881 DO_BRANCH(guest_branch0);
2882
2883 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2884 wrmsr(MSR_IA32_DEBUGCTLMSR, 0);
2885
2886 if (dbgctl != DEBUGCTLMSR_LBR)
2887 asm volatile("ud2\n");
2888 if (rdmsr(MSR_IA32_DEBUGCTLMSR) != 0)
2889 asm volatile("ud2\n");
2890
2891 GUEST_CHECK_LBR(&guest_branch0_from, &guest_branch0_to);
2892 asm volatile ("vmmcall\n");
2893 }
2894
svm_lbrv_test_guest2(void)2895 static void svm_lbrv_test_guest2(void)
2896 {
2897 /*
2898 * This guest expects the LBR to be disabled when it starts,
2899 * enables it, does a branch, disables it and then checks.
2900 */
2901
2902 DO_BRANCH(guest_branch1);
2903 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2904
2905 if (dbgctl != 0)
2906 asm volatile("ud2\n");
2907
2908 GUEST_CHECK_LBR(&host_branch2_from, &host_branch2_to);
2909
2910 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR);
2911 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2912 DO_BRANCH(guest_branch2);
2913 wrmsr(MSR_IA32_DEBUGCTLMSR, 0);
2914
2915 if (dbgctl != DEBUGCTLMSR_LBR)
2916 asm volatile("ud2\n");
2917 GUEST_CHECK_LBR(&guest_branch2_from, &guest_branch2_to);
2918
2919 asm volatile ("vmmcall\n");
2920 }
2921
svm_lbrv_test0(void)2922 static void svm_lbrv_test0(void)
2923 {
2924 report(true, "Basic LBR test");
2925 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR);
2926 DO_BRANCH(host_branch0);
2927 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2928 wrmsr(MSR_IA32_DEBUGCTLMSR, 0);
2929
2930 TEST_EXPECT_EQ(dbgctl, DEBUGCTLMSR_LBR);
2931 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2932 TEST_EXPECT_EQ(dbgctl, 0);
2933
2934 HOST_CHECK_LBR(&host_branch0_from, &host_branch0_to);
2935 }
2936
svm_lbrv_test1(void)2937 static void svm_lbrv_test1(void)
2938 {
2939 report(true, "Test that without LBRV enabled, guest LBR state does 'leak' to the host(1)");
2940
2941 svm_setup_vmrun((u64)svm_lbrv_test_guest1);
2942 vmcb->control.virt_ext = 0;
2943
2944 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR);
2945 DO_BRANCH(host_branch1);
2946 SVM_BARE_VMRUN;
2947 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2948
2949 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
2950 REPORT_GUEST_LBR_ERROR(vmcb);
2951 return;
2952 }
2953
2954 TEST_EXPECT_EQ(dbgctl, 0);
2955 HOST_CHECK_LBR(&guest_branch0_from, &guest_branch0_to);
2956 }
2957
svm_lbrv_test2(void)2958 static void svm_lbrv_test2(void)
2959 {
2960 report(true, "Test that without LBRV enabled, guest LBR state does 'leak' to the host(2)");
2961
2962 svm_setup_vmrun((u64)svm_lbrv_test_guest2);
2963 vmcb->control.virt_ext = 0;
2964
2965 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR);
2966 DO_BRANCH(host_branch2);
2967 wrmsr(MSR_IA32_DEBUGCTLMSR, 0);
2968 SVM_BARE_VMRUN;
2969 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2970 wrmsr(MSR_IA32_DEBUGCTLMSR, 0);
2971
2972 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
2973 REPORT_GUEST_LBR_ERROR(vmcb);
2974 return;
2975 }
2976
2977 TEST_EXPECT_EQ(dbgctl, 0);
2978 HOST_CHECK_LBR(&guest_branch2_from, &guest_branch2_to);
2979 }
2980
svm_lbrv_nested_test1(void)2981 static void svm_lbrv_nested_test1(void)
2982 {
2983 if (!lbrv_supported()) {
2984 report_skip("LBRV not supported in the guest");
2985 return;
2986 }
2987
2988 report(true, "Test that with LBRV enabled, guest LBR state doesn't leak (1)");
2989 svm_setup_vmrun((u64)svm_lbrv_test_guest1);
2990 vmcb->control.virt_ext = LBR_CTL_ENABLE_MASK;
2991 vmcb->save.dbgctl = DEBUGCTLMSR_LBR;
2992
2993 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR);
2994 DO_BRANCH(host_branch3);
2995 SVM_BARE_VMRUN;
2996 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
2997 wrmsr(MSR_IA32_DEBUGCTLMSR, 0);
2998
2999 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
3000 REPORT_GUEST_LBR_ERROR(vmcb);
3001 return;
3002 }
3003
3004 if (vmcb->save.dbgctl != 0) {
3005 report(false, "unexpected virtual guest MSR_IA32_DEBUGCTLMSR value 0x%lx", vmcb->save.dbgctl);
3006 return;
3007 }
3008
3009 TEST_EXPECT_EQ(dbgctl, DEBUGCTLMSR_LBR);
3010 HOST_CHECK_LBR(&host_branch3_from, &host_branch3_to);
3011 }
3012
svm_lbrv_nested_test2(void)3013 static void svm_lbrv_nested_test2(void)
3014 {
3015 if (!lbrv_supported()) {
3016 report_skip("LBRV not supported in the guest");
3017 return;
3018 }
3019
3020 report(true, "Test that with LBRV enabled, guest LBR state doesn't leak (2)");
3021 svm_setup_vmrun((u64)svm_lbrv_test_guest2);
3022 vmcb->control.virt_ext = LBR_CTL_ENABLE_MASK;
3023
3024 vmcb->save.dbgctl = 0;
3025 vmcb->save.br_from = (u64)&host_branch2_from;
3026 vmcb->save.br_to = (u64)&host_branch2_to;
3027
3028 wrmsr(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR);
3029 DO_BRANCH(host_branch4);
3030 SVM_BARE_VMRUN;
3031 dbgctl = rdmsr(MSR_IA32_DEBUGCTLMSR);
3032 wrmsr(MSR_IA32_DEBUGCTLMSR, 0);
3033
3034 if (vmcb->control.exit_code != SVM_EXIT_VMMCALL) {
3035 REPORT_GUEST_LBR_ERROR(vmcb);
3036 return;
3037 }
3038
3039 TEST_EXPECT_EQ(dbgctl, DEBUGCTLMSR_LBR);
3040 HOST_CHECK_LBR(&host_branch4_from, &host_branch4_to);
3041 }
3042
3043
3044 // test that a nested guest which does enable INTR interception
3045 // but doesn't enable virtual interrupt masking works
3046
3047 static volatile int dummy_isr_recevied;
dummy_isr(isr_regs_t * regs)3048 static void dummy_isr(isr_regs_t *regs)
3049 {
3050 dummy_isr_recevied++;
3051 eoi();
3052 }
3053
3054
3055 static volatile int nmi_recevied;
dummy_nmi_handler(struct ex_regs * regs)3056 static void dummy_nmi_handler(struct ex_regs *regs)
3057 {
3058 nmi_recevied++;
3059 }
3060
3061
svm_intr_intercept_mix_run_guest(volatile int * counter,int expected_vmexit)3062 static void svm_intr_intercept_mix_run_guest(volatile int *counter, int expected_vmexit)
3063 {
3064 if (counter)
3065 *counter = 0;
3066
3067 sti(); // host IF value should not matter
3068 clgi(); // vmrun will set back GI to 1
3069
3070 svm_vmrun();
3071
3072 if (counter)
3073 report(!*counter, "No interrupt expected");
3074
3075 stgi();
3076
3077 if (counter)
3078 report(*counter == 1, "Interrupt is expected");
3079
3080 report (vmcb->control.exit_code == expected_vmexit, "Test expected VM exit");
3081 report(vmcb->save.rflags & X86_EFLAGS_IF, "Guest should have EFLAGS.IF set now");
3082 cli();
3083 }
3084
3085
3086 // subtest: test that enabling EFLAGS.IF is enough to trigger an interrupt
svm_intr_intercept_mix_if_guest(struct svm_test * test)3087 static void svm_intr_intercept_mix_if_guest(struct svm_test *test)
3088 {
3089 asm volatile("nop;nop;nop;nop");
3090 report(!dummy_isr_recevied, "No interrupt expected");
3091 sti_nop();
3092 report(0, "must not reach here");
3093 }
3094
svm_intr_intercept_mix_if(void)3095 static void svm_intr_intercept_mix_if(void)
3096 {
3097 // make a physical interrupt to be pending
3098 handle_irq(0x55, dummy_isr);
3099
3100 vmcb->control.intercept |= (1 << INTERCEPT_INTR);
3101 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3102 vmcb->save.rflags &= ~X86_EFLAGS_IF;
3103
3104 test_set_guest(svm_intr_intercept_mix_if_guest);
3105 cli();
3106 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | 0x55, 0);
3107 svm_intr_intercept_mix_run_guest(&dummy_isr_recevied, SVM_EXIT_INTR);
3108 }
3109
3110
3111 // subtest: test that a clever guest can trigger an interrupt by setting GIF
3112 // if GIF is not intercepted
svm_intr_intercept_mix_gif_guest(struct svm_test * test)3113 static void svm_intr_intercept_mix_gif_guest(struct svm_test *test)
3114 {
3115
3116 asm volatile("nop;nop;nop;nop");
3117 report(!dummy_isr_recevied, "No interrupt expected");
3118
3119 // clear GIF and enable IF
3120 // that should still not cause VM exit
3121 clgi();
3122 sti_nop();
3123 report(!dummy_isr_recevied, "No interrupt expected");
3124
3125 stgi();
3126 report(0, "must not reach here");
3127 }
3128
svm_intr_intercept_mix_gif(void)3129 static void svm_intr_intercept_mix_gif(void)
3130 {
3131 handle_irq(0x55, dummy_isr);
3132
3133 vmcb->control.intercept |= (1 << INTERCEPT_INTR);
3134 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3135 vmcb->save.rflags &= ~X86_EFLAGS_IF;
3136
3137 test_set_guest(svm_intr_intercept_mix_gif_guest);
3138 cli();
3139 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | 0x55, 0);
3140 svm_intr_intercept_mix_run_guest(&dummy_isr_recevied, SVM_EXIT_INTR);
3141 }
3142
3143 // subtest: test that a clever guest can trigger an interrupt by setting GIF
3144 // if GIF is not intercepted and interrupt comes after guest
3145 // started running
svm_intr_intercept_mix_gif_guest2(struct svm_test * test)3146 static void svm_intr_intercept_mix_gif_guest2(struct svm_test *test)
3147 {
3148 asm volatile("nop;nop;nop;nop");
3149 report(!dummy_isr_recevied, "No interrupt expected");
3150
3151 clgi();
3152 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | 0x55, 0);
3153 report(!dummy_isr_recevied, "No interrupt expected");
3154
3155 stgi();
3156 report(0, "must not reach here");
3157 }
3158
svm_intr_intercept_mix_gif2(void)3159 static void svm_intr_intercept_mix_gif2(void)
3160 {
3161 handle_irq(0x55, dummy_isr);
3162
3163 vmcb->control.intercept |= (1 << INTERCEPT_INTR);
3164 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3165 vmcb->save.rflags |= X86_EFLAGS_IF;
3166
3167 test_set_guest(svm_intr_intercept_mix_gif_guest2);
3168 svm_intr_intercept_mix_run_guest(&dummy_isr_recevied, SVM_EXIT_INTR);
3169 }
3170
3171
3172 // subtest: test that pending NMI will be handled when guest enables GIF
svm_intr_intercept_mix_nmi_guest(struct svm_test * test)3173 static void svm_intr_intercept_mix_nmi_guest(struct svm_test *test)
3174 {
3175 asm volatile("nop;nop;nop;nop");
3176 report(!nmi_recevied, "No NMI expected");
3177 cli(); // should have no effect
3178
3179 clgi();
3180 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_NMI, 0);
3181 sti_nop(); // should have no effect
3182 report(!nmi_recevied, "No NMI expected");
3183
3184 stgi();
3185 report(0, "must not reach here");
3186 }
3187
svm_intr_intercept_mix_nmi(void)3188 static void svm_intr_intercept_mix_nmi(void)
3189 {
3190 handle_exception(2, dummy_nmi_handler);
3191
3192 vmcb->control.intercept |= (1 << INTERCEPT_NMI);
3193 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3194 vmcb->save.rflags |= X86_EFLAGS_IF;
3195
3196 test_set_guest(svm_intr_intercept_mix_nmi_guest);
3197 svm_intr_intercept_mix_run_guest(&nmi_recevied, SVM_EXIT_NMI);
3198 }
3199
3200 // test that pending SMI will be handled when guest enables GIF
3201 // TODO: can't really count #SMIs so just test that guest doesn't hang
3202 // and VMexits on SMI
svm_intr_intercept_mix_smi_guest(struct svm_test * test)3203 static void svm_intr_intercept_mix_smi_guest(struct svm_test *test)
3204 {
3205 asm volatile("nop;nop;nop;nop");
3206
3207 clgi();
3208 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_SMI, 0);
3209 sti_nop(); // should have no effect
3210 stgi();
3211 report(0, "must not reach here");
3212 }
3213
svm_intr_intercept_mix_smi(void)3214 static void svm_intr_intercept_mix_smi(void)
3215 {
3216 vmcb->control.intercept |= (1 << INTERCEPT_SMI);
3217 vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3218 test_set_guest(svm_intr_intercept_mix_smi_guest);
3219 svm_intr_intercept_mix_run_guest(NULL, SVM_EXIT_SMI);
3220 }
3221
svm_l2_ac_test(void)3222 static void svm_l2_ac_test(void)
3223 {
3224 bool hit_ac = false;
3225
3226 write_cr0(read_cr0() | X86_CR0_AM);
3227 write_rflags(read_rflags() | X86_EFLAGS_AC);
3228
3229 run_in_user(generate_usermode_ac, AC_VECTOR, 0, 0, 0, 0, &hit_ac);
3230 report(hit_ac, "Usermode #AC handled in L2");
3231 vmmcall();
3232 }
3233
3234 struct svm_exception_test {
3235 u8 vector;
3236 void (*guest_code)(void);
3237 };
3238
3239 struct svm_exception_test svm_exception_tests[] = {
3240 { GP_VECTOR, generate_non_canonical_gp },
3241 { UD_VECTOR, generate_ud },
3242 { DE_VECTOR, generate_de },
3243 { DB_VECTOR, generate_single_step_db },
3244 { BP_VECTOR, generate_bp },
3245 { AC_VECTOR, svm_l2_ac_test },
3246 { OF_VECTOR, generate_of },
3247 { NM_VECTOR, generate_cr0_ts_nm },
3248 { NM_VECTOR, generate_cr0_em_nm },
3249 };
3250
3251 static u8 svm_exception_test_vector;
3252
svm_exception_handler(struct ex_regs * regs)3253 static void svm_exception_handler(struct ex_regs *regs)
3254 {
3255 report(regs->vector == svm_exception_test_vector,
3256 "Handling %s in L2's exception handler",
3257 exception_mnemonic(svm_exception_test_vector));
3258 vmmcall();
3259 }
3260
handle_exception_in_l2(u8 vector)3261 static void handle_exception_in_l2(u8 vector)
3262 {
3263 handler old_handler = handle_exception(vector, svm_exception_handler);
3264 svm_exception_test_vector = vector;
3265
3266 report(svm_vmrun() == SVM_EXIT_VMMCALL,
3267 "%s handled by L2", exception_mnemonic(vector));
3268
3269 handle_exception(vector, old_handler);
3270 }
3271
handle_exception_in_l1(u32 vector)3272 static void handle_exception_in_l1(u32 vector)
3273 {
3274 u32 old_ie = vmcb->control.intercept_exceptions;
3275
3276 vmcb->control.intercept_exceptions |= (1ULL << vector);
3277
3278 report(svm_vmrun() == (SVM_EXIT_EXCP_BASE + vector),
3279 "%s handled by L1", exception_mnemonic(vector));
3280
3281 vmcb->control.intercept_exceptions = old_ie;
3282 }
3283
svm_exception_test(void)3284 static void svm_exception_test(void)
3285 {
3286 struct svm_exception_test *t;
3287 int i;
3288
3289 for (i = 0; i < ARRAY_SIZE(svm_exception_tests); i++) {
3290 t = &svm_exception_tests[i];
3291 test_set_guest((test_guest_func)t->guest_code);
3292
3293 handle_exception_in_l2(t->vector);
3294 vmcb_ident(vmcb);
3295
3296 handle_exception_in_l1(t->vector);
3297 vmcb_ident(vmcb);
3298 }
3299 }
3300
shutdown_intercept_test_guest(struct svm_test * test)3301 static void shutdown_intercept_test_guest(struct svm_test *test)
3302 {
3303 asm volatile ("ud2");
3304 report_fail("should not reach here\n");
3305
3306 }
3307
svm_shutdown_intercept_test(void)3308 static void svm_shutdown_intercept_test(void)
3309 {
3310 test_set_guest(shutdown_intercept_test_guest);
3311 vmcb->save.idtr.base = (u64)alloc_vpage();
3312 vmcb->control.intercept |= (1ULL << INTERCEPT_SHUTDOWN);
3313 svm_vmrun();
3314 report(vmcb->control.exit_code == SVM_EXIT_SHUTDOWN, "shutdown test passed");
3315 }
3316
3317 struct svm_test svm_tests[] = {
3318 { "null", default_supported, default_prepare,
3319 default_prepare_gif_clear, null_test,
3320 default_finished, null_check },
3321 { "vmrun", default_supported, default_prepare,
3322 default_prepare_gif_clear, test_vmrun,
3323 default_finished, check_vmrun },
3324 { "ioio", default_supported, prepare_ioio,
3325 default_prepare_gif_clear, test_ioio,
3326 ioio_finished, check_ioio },
3327 { "vmrun intercept check", default_supported, prepare_no_vmrun_int,
3328 default_prepare_gif_clear, null_test, default_finished,
3329 check_no_vmrun_int },
3330 { "rsm", default_supported,
3331 prepare_rsm_intercept, default_prepare_gif_clear,
3332 test_rsm_intercept, finished_rsm_intercept, check_rsm_intercept },
3333 { "cr3 read intercept", default_supported,
3334 prepare_cr3_intercept, default_prepare_gif_clear,
3335 test_cr3_intercept, default_finished, check_cr3_intercept },
3336 { "cr3 read nointercept", default_supported, default_prepare,
3337 default_prepare_gif_clear, test_cr3_intercept, default_finished,
3338 check_cr3_nointercept },
3339 { "cr3 read intercept emulate", smp_supported,
3340 prepare_cr3_intercept_bypass, default_prepare_gif_clear,
3341 test_cr3_intercept_bypass, default_finished, check_cr3_intercept },
3342 { "dr intercept check", default_supported, prepare_dr_intercept,
3343 default_prepare_gif_clear, test_dr_intercept, dr_intercept_finished,
3344 check_dr_intercept },
3345 { "next_rip", next_rip_supported, prepare_next_rip,
3346 default_prepare_gif_clear, test_next_rip,
3347 default_finished, check_next_rip },
3348 { "msr intercept check", default_supported, prepare_msr_intercept,
3349 default_prepare_gif_clear, test_msr_intercept,
3350 msr_intercept_finished, check_msr_intercept },
3351 { "mode_switch", default_supported, prepare_mode_switch,
3352 default_prepare_gif_clear, test_mode_switch,
3353 mode_switch_finished, check_mode_switch },
3354 { "asid_zero", default_supported, prepare_asid_zero,
3355 default_prepare_gif_clear, test_asid_zero,
3356 default_finished, check_asid_zero },
3357 { "sel_cr0_bug", default_supported, sel_cr0_bug_prepare,
3358 default_prepare_gif_clear, sel_cr0_bug_test,
3359 sel_cr0_bug_finished, sel_cr0_bug_check },
3360 { "tsc_adjust", tsc_adjust_supported, tsc_adjust_prepare,
3361 default_prepare_gif_clear, tsc_adjust_test,
3362 default_finished, tsc_adjust_check },
3363 { "latency_run_exit", default_supported, latency_prepare,
3364 default_prepare_gif_clear, latency_test,
3365 latency_finished, latency_check },
3366 { "latency_run_exit_clean", default_supported, latency_prepare,
3367 default_prepare_gif_clear, latency_test,
3368 latency_finished_clean, latency_check },
3369 { "latency_svm_insn", default_supported, lat_svm_insn_prepare,
3370 default_prepare_gif_clear, null_test,
3371 lat_svm_insn_finished, lat_svm_insn_check },
3372 { "exc_inject", default_supported, exc_inject_prepare,
3373 default_prepare_gif_clear, exc_inject_test,
3374 exc_inject_finished, exc_inject_check },
3375 { "pending_event", default_supported, pending_event_prepare,
3376 default_prepare_gif_clear,
3377 pending_event_test, pending_event_finished, pending_event_check },
3378 { "pending_event_cli", default_supported, pending_event_cli_prepare,
3379 pending_event_cli_prepare_gif_clear,
3380 pending_event_cli_test, pending_event_cli_finished,
3381 pending_event_cli_check },
3382 { "interrupt", default_supported, interrupt_prepare,
3383 default_prepare_gif_clear, interrupt_test,
3384 interrupt_finished, interrupt_check },
3385 { "nmi", default_supported, nmi_prepare,
3386 default_prepare_gif_clear, nmi_test,
3387 nmi_finished, nmi_check },
3388 { "nmi_hlt", smp_supported, nmi_prepare,
3389 default_prepare_gif_clear, nmi_hlt_test,
3390 nmi_hlt_finished, nmi_hlt_check },
3391 { "vnmi", vnmi_supported, vnmi_prepare,
3392 default_prepare_gif_clear, vnmi_test,
3393 vnmi_finished, vnmi_check },
3394 { "virq_inject", default_supported, virq_inject_prepare,
3395 default_prepare_gif_clear, virq_inject_test,
3396 virq_inject_finished, virq_inject_check },
3397 { "virq_inject_within_shadow", default_supported, virq_inject_within_shadow_prepare,
3398 virq_inject_within_shadow_prepare_gif_clear, virq_inject_within_shadow_test,
3399 virq_inject_within_shadow_finished, virq_inject_within_shadow_check },
3400 { "reg_corruption", default_supported, reg_corruption_prepare,
3401 default_prepare_gif_clear, reg_corruption_test,
3402 reg_corruption_finished, reg_corruption_check },
3403 { "svm_init_startup_test", smp_supported, init_startup_prepare,
3404 default_prepare_gif_clear, null_test,
3405 init_startup_finished, init_startup_check },
3406 { "svm_init_intercept_test", smp_supported, init_intercept_prepare,
3407 default_prepare_gif_clear, init_intercept_test,
3408 init_intercept_finished, init_intercept_check, .on_vcpu = 2 },
3409 { "host_rflags", default_supported, host_rflags_prepare,
3410 host_rflags_prepare_gif_clear, host_rflags_test,
3411 host_rflags_finished, host_rflags_check },
3412 { "vgif", vgif_supported, prepare_vgif_enabled,
3413 default_prepare_gif_clear, test_vgif, vgif_finished,
3414 vgif_check },
3415 TEST(svm_cr4_osxsave_test),
3416 TEST(svm_guest_state_test),
3417 TEST(svm_vmrun_errata_test),
3418 TEST(svm_vmload_vmsave),
3419 TEST(svm_test_singlestep),
3420 TEST(svm_no_nm_test),
3421 TEST(svm_exception_test),
3422 TEST(svm_lbrv_test0),
3423 TEST(svm_lbrv_test1),
3424 TEST(svm_lbrv_test2),
3425 TEST(svm_lbrv_nested_test1),
3426 TEST(svm_lbrv_nested_test2),
3427 TEST(svm_intr_intercept_mix_if),
3428 TEST(svm_intr_intercept_mix_gif),
3429 TEST(svm_intr_intercept_mix_gif2),
3430 TEST(svm_intr_intercept_mix_nmi),
3431 TEST(svm_intr_intercept_mix_smi),
3432 TEST(svm_tsc_scale_test),
3433 TEST(pause_filter_test),
3434 TEST(svm_shutdown_intercept_test),
3435 { NULL, NULL, NULL, NULL, NULL, NULL, NULL }
3436 };
3437
main(int ac,char ** av)3438 int main(int ac, char **av)
3439 {
3440 setup_vm();
3441 return run_svm_tests(ac, av, svm_tests);
3442 }
3443