xref: /kvm-unit-tests/x86/vmexit.c (revision 40ff1913cac3f9831d0ee4a9a675d8a6bb4a5e19)
1 #include "libcflat.h"
2 #include "smp.h"
3 #include "processor.h"
4 #include "atomic.h"
5 #include "pci.h"
6 #include "x86/vm.h"
7 #include "x86/desc.h"
8 #include "x86/acpi.h"
9 #include "x86/apic.h"
10 #include "x86/isr.h"
11 
12 #define IPI_TEST_VECTOR	0xb0
13 
14 struct test {
15 	void (*func)(void);
16 	const char *name;
17 	int (*valid)(void);
18 	int parallel;
19 	bool (*next)(struct test *);
20 };
21 
22 #define GOAL (1ull << 30)
23 
24 static int nr_cpus;
25 
26 static void cpuid_test(void)
27 {
28 	asm volatile ("push %%"R "bx; cpuid; pop %%"R "bx"
29 		      : : : "eax", "ecx", "edx");
30 }
31 
32 static void vmcall(void)
33 {
34 	unsigned long a = 0, b, c, d;
35 
36 	asm volatile ("vmcall" : "+a"(a), "=b"(b), "=c"(c), "=d"(d));
37 }
38 
39 #define MSR_EFER 0xc0000080
40 #define EFER_NX_MASK            (1ull << 11)
41 
42 #ifdef __x86_64__
43 static void mov_from_cr8(void)
44 {
45 	unsigned long cr8;
46 
47 	asm volatile ("mov %%cr8, %0" : "=r"(cr8));
48 }
49 
50 static void mov_to_cr8(void)
51 {
52 	unsigned long cr8 = 0;
53 
54 	asm volatile ("mov %0, %%cr8" : : "r"(cr8));
55 }
56 #endif
57 
58 static int is_smp(void)
59 {
60 	return cpu_count() > 1;
61 }
62 
63 static void nop(void *junk)
64 {
65 }
66 
67 volatile int x = 0;
68 
69 static void self_ipi_isr(isr_regs_t *regs)
70 {
71 	x++;
72 	eoi();
73 }
74 
75 static void x2apic_self_ipi(int vec)
76 {
77 	wrmsr(0x83f, vec);
78 }
79 
80 static void apic_self_ipi(int vec)
81 {
82         apic_icr_write(APIC_INT_ASSERT | APIC_DEST_SELF | APIC_DEST_PHYSICAL |
83 		       APIC_DM_FIXED | IPI_TEST_VECTOR, vec);
84 }
85 
86 static void self_ipi_sti_nop(void)
87 {
88 	x = 0;
89 	irq_disable();
90 	apic_self_ipi(IPI_TEST_VECTOR);
91 	asm volatile("sti; nop");
92 	if (x != 1) printf("%d", x);
93 }
94 
95 static void self_ipi_sti_hlt(void)
96 {
97 	x = 0;
98 	irq_disable();
99 	apic_self_ipi(IPI_TEST_VECTOR);
100 	asm volatile("sti; hlt");
101 	if (x != 1) printf("%d", x);
102 }
103 
104 static void self_ipi_tpr(void)
105 {
106 	x = 0;
107 	apic_set_tpr(0x0f);
108 	apic_self_ipi(IPI_TEST_VECTOR);
109 	apic_set_tpr(0x00);
110 	asm volatile("nop");
111 	if (x != 1) printf("%d", x);
112 }
113 
114 static void self_ipi_tpr_sti_nop(void)
115 {
116 	x = 0;
117 	irq_disable();
118 	apic_set_tpr(0x0f);
119 	apic_self_ipi(IPI_TEST_VECTOR);
120 	apic_set_tpr(0x00);
121 	asm volatile("sti; nop");
122 	if (x != 1) printf("%d", x);
123 }
124 
125 static void self_ipi_tpr_sti_hlt(void)
126 {
127 	x = 0;
128 	irq_disable();
129 	apic_set_tpr(0x0f);
130 	apic_self_ipi(IPI_TEST_VECTOR);
131 	apic_set_tpr(0x00);
132 	asm volatile("sti; hlt");
133 	if (x != 1) printf("%d", x);
134 }
135 
136 static int is_x2apic(void)
137 {
138     return rdmsr(MSR_IA32_APICBASE) & APIC_EXTD;
139 }
140 
141 static void x2apic_self_ipi_sti_nop(void)
142 {
143 	irq_disable();
144 	x2apic_self_ipi(IPI_TEST_VECTOR);
145 	asm volatile("sti; nop");
146 }
147 
148 static void x2apic_self_ipi_sti_hlt(void)
149 {
150 	irq_disable();
151 	x2apic_self_ipi(IPI_TEST_VECTOR);
152 	asm volatile("sti; hlt");
153 }
154 
155 static void x2apic_self_ipi_tpr(void)
156 {
157 	apic_set_tpr(0x0f);
158 	x2apic_self_ipi(IPI_TEST_VECTOR);
159 	apic_set_tpr(0x00);
160 	asm volatile("nop");
161 }
162 
163 static void x2apic_self_ipi_tpr_sti_nop(void)
164 {
165 	irq_disable();
166 	apic_set_tpr(0x0f);
167 	x2apic_self_ipi(IPI_TEST_VECTOR);
168 	apic_set_tpr(0x00);
169 	asm volatile("sti; nop");
170 }
171 
172 static void x2apic_self_ipi_tpr_sti_hlt(void)
173 {
174 	irq_disable();
175 	apic_set_tpr(0x0f);
176 	x2apic_self_ipi(IPI_TEST_VECTOR);
177 	apic_set_tpr(0x00);
178 	asm volatile("sti; hlt");
179 }
180 
181 static void ipi(void)
182 {
183 	on_cpu(1, nop, 0);
184 }
185 
186 static void ipi_halt(void)
187 {
188 	unsigned long long t;
189 
190 	on_cpu(1, nop, 0);
191 	t = rdtsc() + 2000;
192 	while (rdtsc() < t)
193 		;
194 }
195 
196 int pm_tmr_blk;
197 static void inl_pmtimer(void)
198 {
199     if (!pm_tmr_blk) {
200 	struct fadt_descriptor_rev1 *fadt;
201 
202 	fadt = find_acpi_table_addr(FACP_SIGNATURE);
203 	pm_tmr_blk = fadt->pm_tmr_blk;
204 	printf("PM timer port is %x\n", pm_tmr_blk);
205     }
206     inl(pm_tmr_blk);
207 }
208 
209 static void inl_nop_qemu(void)
210 {
211     inl(0x1234);
212 }
213 
214 static void inl_nop_kernel(void)
215 {
216     inb(0x4d0);
217 }
218 
219 static void outl_elcr_kernel(void)
220 {
221     outb(0, 0x4d0);
222 }
223 
224 static void mov_dr(void)
225 {
226     asm volatile("mov %0, %%dr7" : : "r" (0x400L));
227 }
228 
229 static void ple_round_robin(void)
230 {
231 	struct counter {
232 		volatile int n1;
233 		int n2;
234 	} __attribute__((aligned(64)));
235 	static struct counter counters[64] = { { -1, 0 } };
236 	int me = smp_id();
237 	int you;
238 	volatile struct counter *p = &counters[me];
239 
240 	while (p->n1 == p->n2)
241 		asm volatile ("pause");
242 
243 	p->n2 = p->n1;
244 	you = me + 1;
245 	if (you == nr_cpus)
246 		you = 0;
247 	++counters[you].n1;
248 }
249 
250 static void rd_tsc_adjust_msr(void)
251 {
252 	rdmsr(MSR_IA32_TSC_ADJUST);
253 }
254 
255 static void wr_tsc_adjust_msr(void)
256 {
257 	wrmsr(MSR_IA32_TSC_ADJUST, 0x0);
258 }
259 
260 static void wr_kernel_gs_base(void)
261 {
262 	wrmsr(MSR_KERNEL_GS_BASE, 0x0);
263 }
264 
265 static struct pci_test {
266 	unsigned iobar;
267 	unsigned ioport;
268 	volatile void *memaddr;
269 	volatile void *mem;
270 	int test_idx;
271 	uint32_t data;
272 	uint32_t offset;
273 } pci_test = {
274 	.test_idx = -1
275 };
276 
277 static void pci_mem_testb(void)
278 {
279 	*(volatile uint8_t *)pci_test.mem = pci_test.data;
280 }
281 
282 static void pci_mem_testw(void)
283 {
284 	*(volatile uint16_t *)pci_test.mem = pci_test.data;
285 }
286 
287 static void pci_mem_testl(void)
288 {
289 	*(volatile uint32_t *)pci_test.mem = pci_test.data;
290 }
291 
292 static void pci_io_testb(void)
293 {
294 	outb(pci_test.data, pci_test.ioport);
295 }
296 
297 static void pci_io_testw(void)
298 {
299 	outw(pci_test.data, pci_test.ioport);
300 }
301 
302 static void pci_io_testl(void)
303 {
304 	outl(pci_test.data, pci_test.ioport);
305 }
306 
307 static uint8_t ioreadb(unsigned long addr, bool io)
308 {
309 	if (io) {
310 		return inb(addr);
311 	} else {
312 		return *(volatile uint8_t *)addr;
313 	}
314 }
315 
316 static uint32_t ioreadl(unsigned long addr, bool io)
317 {
318 	/* Note: assumes little endian */
319 	if (io) {
320 		return inl(addr);
321 	} else {
322 		return *(volatile uint32_t *)addr;
323 	}
324 }
325 
326 static void iowriteb(unsigned long addr, uint8_t data, bool io)
327 {
328 	if (io) {
329 		outb(data, addr);
330 	} else {
331 		*(volatile uint8_t *)addr = data;
332 	}
333 }
334 
335 static bool pci_next(struct test *test, unsigned long addr, bool io)
336 {
337 	int i;
338 	uint8_t width;
339 
340 	if (!pci_test.memaddr) {
341 		test->func = NULL;
342 		return true;
343 	}
344 	pci_test.test_idx++;
345 	iowriteb(addr + offsetof(struct pci_test_dev_hdr, test),
346 		 pci_test.test_idx, io);
347 	width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width),
348 			io);
349 	switch (width) {
350 		case 1:
351 			test->func = io ? pci_io_testb : pci_mem_testb;
352 			break;
353 		case 2:
354 			test->func = io ? pci_io_testw : pci_mem_testw;
355 			break;
356 		case 4:
357 			test->func = io ? pci_io_testl : pci_mem_testl;
358 			break;
359 		default:
360 			/* Reset index for purposes of the next test */
361 			pci_test.test_idx = -1;
362 			test->func = NULL;
363 			return false;
364 	}
365 	pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data),
366 				io);
367 	pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr,
368 						  offset), io);
369 	for (i = 0; i < pci_test.offset; ++i) {
370 		char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr,
371 						 name) + i, io);
372 		if (!c) {
373 			break;
374 		}
375 		printf("%c",c);
376 	}
377 	printf(":");
378 	return true;
379 }
380 
381 static bool pci_mem_next(struct test *test)
382 {
383 	bool ret;
384 	ret = pci_next(test, ((unsigned long)pci_test.memaddr), false);
385 	if (ret) {
386 		pci_test.mem = pci_test.memaddr + pci_test.offset;
387 	}
388 	return ret;
389 }
390 
391 static bool pci_io_next(struct test *test)
392 {
393 	bool ret;
394 	ret = pci_next(test, ((unsigned long)pci_test.iobar), true);
395 	if (ret) {
396 		pci_test.ioport = pci_test.iobar + pci_test.offset;
397 	}
398 	return ret;
399 }
400 
401 static int has_tscdeadline(void)
402 {
403     uint32_t lvtt;
404 
405     if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
406         lvtt = APIC_LVT_TIMER_TSCDEADLINE | IPI_TEST_VECTOR;
407         apic_write(APIC_LVTT, lvtt);
408         return 1;
409     } else {
410         return 0;
411     }
412 }
413 
414 static void tscdeadline_immed(void)
415 {
416 	wrmsr(MSR_IA32_TSCDEADLINE, rdtsc());
417 	asm volatile("nop");
418 }
419 
420 static void tscdeadline(void)
421 {
422 	x = 0;
423 	wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()+3000);
424 	while (x == 0) barrier();
425 }
426 
427 static void wr_ibrs_msr(void)
428 {
429 	wrmsr(MSR_IA32_SPEC_CTRL, 1);
430 	wrmsr(MSR_IA32_SPEC_CTRL, 0);
431 }
432 
433 static int has_ibpb(void)
434 {
435     return has_spec_ctrl() || !!(this_cpu_has(X86_FEATURE_AMD_IBPB));
436 }
437 
438 static void wr_ibpb_msr(void)
439 {
440 	wrmsr(MSR_IA32_PRED_CMD, 1);
441 }
442 
443 static struct test tests[] = {
444 	{ cpuid_test, "cpuid", .parallel = 1,  },
445 	{ vmcall, "vmcall", .parallel = 1, },
446 #ifdef __x86_64__
447 	{ mov_from_cr8, "mov_from_cr8", .parallel = 1, },
448 	{ mov_to_cr8, "mov_to_cr8" , .parallel = 1, },
449 #endif
450 	{ inl_pmtimer, "inl_from_pmtimer", .parallel = 1, },
451 	{ inl_nop_qemu, "inl_from_qemu", .parallel = 1 },
452 	{ inl_nop_kernel, "inl_from_kernel", .parallel = 1 },
453 	{ outl_elcr_kernel, "outl_to_kernel", .parallel = 1 },
454 	{ mov_dr, "mov_dr", .parallel = 1 },
455 	{ tscdeadline_immed, "tscdeadline_immed", has_tscdeadline, .parallel = 1, },
456 	{ tscdeadline, "tscdeadline", has_tscdeadline, .parallel = 1, },
457 	{ self_ipi_sti_nop, "self_ipi_sti_nop", .parallel = 0, },
458 	{ self_ipi_sti_hlt, "self_ipi_sti_hlt", .parallel = 0, },
459 	{ self_ipi_tpr, "self_ipi_tpr", .parallel = 0, },
460 	{ self_ipi_tpr_sti_nop, "self_ipi_tpr_sti_nop", .parallel = 0, },
461 	{ self_ipi_tpr_sti_hlt, "self_ipi_tpr_sti_hlt", .parallel = 0, },
462 	{ x2apic_self_ipi_sti_nop, "x2apic_self_ipi_sti_nop", is_x2apic, .parallel = 0, },
463 	{ x2apic_self_ipi_sti_hlt, "x2apic_self_ipi_sti_hlt", is_x2apic, .parallel = 0, },
464 	{ x2apic_self_ipi_tpr, "x2apic_self_ipi_tpr", is_x2apic, .parallel = 0, },
465 	{ x2apic_self_ipi_tpr_sti_nop, "x2apic_self_ipi_tpr_sti_nop", is_x2apic, .parallel = 0, },
466 	{ x2apic_self_ipi_tpr_sti_hlt, "x2apic_self_ipi_tpr_sti_hlt", is_x2apic, .parallel = 0, },
467 	{ ipi, "ipi", is_smp, .parallel = 0, },
468 	{ ipi_halt, "ipi_halt", is_smp, .parallel = 0, },
469 	{ ple_round_robin, "ple_round_robin", .parallel = 1 },
470 	{ wr_kernel_gs_base, "wr_kernel_gs_base", .parallel = 1 },
471 	{ wr_ibrs_msr, "wr_ibrs_msr", has_spec_ctrl, .parallel = 1 },
472 	{ wr_ibpb_msr, "wr_ibpb_msr", has_ibpb, .parallel = 1 },
473 	{ wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 },
474 	{ rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 },
475 	{ NULL, "pci-mem", .parallel = 0, .next = pci_mem_next },
476 	{ NULL, "pci-io", .parallel = 0, .next = pci_io_next },
477 };
478 
479 unsigned iterations;
480 
481 static void run_test(void *_func)
482 {
483     int i;
484     void (*func)(void) = _func;
485 
486     for (i = 0; i < iterations; ++i)
487         func();
488 }
489 
490 static bool do_test(struct test *test)
491 {
492 	int i;
493 	unsigned long long t1, t2;
494         void (*func)(void);
495 
496         iterations = 32;
497 
498         if (test->valid && !test->valid()) {
499 		printf("%s (skipped)\n", test->name);
500 		return false;
501 	}
502 
503 	if (test->next && !test->next(test)) {
504 		return false;
505 	}
506 
507 	func = test->func;
508         if (!func) {
509 		printf("%s (skipped)\n", test->name);
510 		return false;
511 	}
512 
513 	do {
514 		iterations *= 2;
515 		t1 = rdtsc();
516 
517 		if (!test->parallel) {
518 			for (i = 0; i < iterations; ++i)
519 				func();
520 		} else {
521 			on_cpus(run_test, func);
522 		}
523 		t2 = rdtsc();
524 	} while ((t2 - t1) < GOAL);
525 	printf("%s %d\n", test->name, (int)((t2 - t1) / iterations));
526 	return test->next;
527 }
528 
529 static void enable_nx(void *junk)
530 {
531 	if (this_cpu_has(X86_FEATURE_NX))
532 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK);
533 }
534 
535 static bool test_wanted(struct test *test, char *wanted[], int nwanted)
536 {
537 	int i;
538 
539 	if (!nwanted)
540 		return true;
541 
542 	for (i = 0; i < nwanted; ++i)
543 		if (strcmp(wanted[i], test->name) == 0)
544 			return true;
545 
546 	return false;
547 }
548 
549 int main(int ac, char **av)
550 {
551 	int i;
552 	unsigned long membar = 0;
553 	struct pci_dev pcidev;
554 	int ret;
555 
556 	smp_init();
557 	setup_vm();
558 	handle_irq(IPI_TEST_VECTOR, self_ipi_isr);
559 	nr_cpus = cpu_count();
560 
561 	irq_enable();
562 	on_cpus(enable_nx, NULL);
563 
564 	ret = pci_find_dev(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_TEST);
565 	if (ret != PCIDEVADDR_INVALID) {
566 		pci_dev_init(&pcidev, ret);
567 		assert(pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_MEM));
568 		assert(!pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_IO));
569 		membar = pcidev.resource[PCI_TESTDEV_BAR_MEM];
570 		pci_test.memaddr = ioremap(membar, PAGE_SIZE);
571 		pci_test.iobar = pcidev.resource[PCI_TESTDEV_BAR_IO];
572 		printf("pci-testdev at %#x membar %lx iobar %x\n",
573 		       pcidev.bdf, membar, pci_test.iobar);
574 	}
575 
576 	for (i = 0; i < ARRAY_SIZE(tests); ++i)
577 		if (test_wanted(&tests[i], av + 1, ac - 1))
578 			while (do_test(&tests[i])) {}
579 
580 	return 0;
581 }
582