1 #include "libcflat.h" 2 #include "acpi.h" 3 #include "smp.h" 4 #include "vmalloc.h" 5 #include "pci.h" 6 #include "x86/vm.h" 7 #include "x86/desc.h" 8 #include "x86/apic.h" 9 #include "x86/isr.h" 10 11 #define IPI_TEST_VECTOR 0xb0 12 13 struct test { 14 void (*func)(void); 15 const char *name; 16 int (*valid)(void); 17 int parallel; 18 bool (*next)(struct test *); 19 }; 20 21 #define GOAL (1ull << 30) 22 23 static int nr_cpus; 24 static u64 cr4_shadow; 25 26 static void cpuid_test(void) 27 { 28 asm volatile ("push %%"R "bx; cpuid; pop %%"R "bx" 29 : : : "eax", "ecx", "edx"); 30 } 31 32 static void vmcall(void) 33 { 34 unsigned long a = 0, b, c, d; 35 36 asm volatile ("vmcall" : "+a"(a), "=b"(b), "=c"(c), "=d"(d)); 37 } 38 39 #define MSR_EFER 0xc0000080 40 #define EFER_NX_MASK (1ull << 11) 41 42 #ifdef __x86_64__ 43 static void mov_from_cr8(void) 44 { 45 unsigned long cr8; 46 47 asm volatile ("mov %%cr8, %0" : "=r"(cr8)); 48 } 49 50 static void mov_to_cr8(void) 51 { 52 unsigned long cr8 = 0; 53 54 asm volatile ("mov %0, %%cr8" : : "r"(cr8)); 55 } 56 #endif 57 58 static int is_smp(void) 59 { 60 return cpu_count() > 1; 61 } 62 63 static void nop(void *junk) 64 { 65 } 66 67 volatile int x = 0; 68 volatile uint64_t tsc_eoi = 0; 69 volatile uint64_t tsc_ipi = 0; 70 71 static void self_ipi_isr(isr_regs_t *regs) 72 { 73 x++; 74 uint64_t start = rdtsc(); 75 eoi(); 76 tsc_eoi += rdtsc() - start; 77 } 78 79 static void x2apic_self_ipi(int vec) 80 { 81 uint64_t start = rdtsc(); 82 wrmsr(0x83f, vec); 83 tsc_ipi += rdtsc() - start; 84 } 85 86 static void apic_self_ipi(int vec) 87 { 88 uint64_t start = rdtsc(); 89 apic_icr_write(APIC_INT_ASSERT | APIC_DEST_SELF | APIC_DEST_PHYSICAL | 90 APIC_DM_FIXED | IPI_TEST_VECTOR, vec); 91 tsc_ipi += rdtsc() - start; 92 } 93 94 static void self_ipi_sti_nop(void) 95 { 96 x = 0; 97 cli(); 98 apic_self_ipi(IPI_TEST_VECTOR); 99 asm volatile("sti; nop"); 100 if (x != 1) printf("%d", x); 101 } 102 103 static void self_ipi_sti_hlt(void) 104 { 105 x = 0; 106 cli(); 107 apic_self_ipi(IPI_TEST_VECTOR); 108 safe_halt(); 109 if (x != 1) printf("%d", x); 110 } 111 112 static void self_ipi_tpr(void) 113 { 114 x = 0; 115 apic_set_tpr(0x0f); 116 apic_self_ipi(IPI_TEST_VECTOR); 117 apic_set_tpr(0x00); 118 asm volatile("nop"); 119 if (x != 1) printf("%d", x); 120 } 121 122 static void self_ipi_tpr_sti_nop(void) 123 { 124 x = 0; 125 cli(); 126 apic_set_tpr(0x0f); 127 apic_self_ipi(IPI_TEST_VECTOR); 128 apic_set_tpr(0x00); 129 asm volatile("sti; nop"); 130 if (x != 1) printf("%d", x); 131 } 132 133 static void self_ipi_tpr_sti_hlt(void) 134 { 135 x = 0; 136 cli(); 137 apic_set_tpr(0x0f); 138 apic_self_ipi(IPI_TEST_VECTOR); 139 apic_set_tpr(0x00); 140 safe_halt(); 141 if (x != 1) printf("%d", x); 142 } 143 144 static int is_x2apic(void) 145 { 146 return rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 147 } 148 149 static void x2apic_self_ipi_sti_nop(void) 150 { 151 cli(); 152 x2apic_self_ipi(IPI_TEST_VECTOR); 153 asm volatile("sti; nop"); 154 } 155 156 static void x2apic_self_ipi_sti_hlt(void) 157 { 158 cli(); 159 x2apic_self_ipi(IPI_TEST_VECTOR); 160 safe_halt(); 161 } 162 163 static void x2apic_self_ipi_tpr(void) 164 { 165 apic_set_tpr(0x0f); 166 x2apic_self_ipi(IPI_TEST_VECTOR); 167 apic_set_tpr(0x00); 168 asm volatile("nop"); 169 } 170 171 static void x2apic_self_ipi_tpr_sti_nop(void) 172 { 173 cli(); 174 apic_set_tpr(0x0f); 175 x2apic_self_ipi(IPI_TEST_VECTOR); 176 apic_set_tpr(0x00); 177 asm volatile("sti; nop"); 178 } 179 180 static void x2apic_self_ipi_tpr_sti_hlt(void) 181 { 182 cli(); 183 apic_set_tpr(0x0f); 184 x2apic_self_ipi(IPI_TEST_VECTOR); 185 apic_set_tpr(0x00); 186 safe_halt(); 187 } 188 189 static void ipi(void) 190 { 191 uint64_t start = rdtsc(); 192 on_cpu(1, nop, 0); 193 tsc_ipi += rdtsc() - start; 194 } 195 196 static void ipi_halt(void) 197 { 198 unsigned long long t; 199 200 on_cpu(1, nop, 0); 201 t = rdtsc() + 2000; 202 while (rdtsc() < t) 203 ; 204 } 205 206 int pm_tmr_blk; 207 static void inl_pmtimer(void) 208 { 209 if (!pm_tmr_blk) { 210 struct acpi_table_fadt *fadt; 211 212 fadt = find_acpi_table_addr(FACP_SIGNATURE); 213 pm_tmr_blk = fadt->pm_tmr_blk; 214 printf("PM timer port is %x\n", pm_tmr_blk); 215 } 216 inl(pm_tmr_blk); 217 } 218 219 static void inl_nop_qemu(void) 220 { 221 inl(0x1234); 222 } 223 224 static void inl_nop_kernel(void) 225 { 226 inb(0x4d0); 227 } 228 229 static void outl_elcr_kernel(void) 230 { 231 outb(0, 0x4d0); 232 } 233 234 static void mov_dr(void) 235 { 236 asm volatile("mov %0, %%dr7" : : "r" (0x400L)); 237 } 238 239 static void ple_round_robin(void) 240 { 241 struct counter { 242 volatile int n1; 243 int n2; 244 } __attribute__((aligned(64))); 245 static struct counter counters[64] = { { -1, 0 } }; 246 int me = smp_id(); 247 int you; 248 volatile struct counter *p = &counters[me]; 249 250 while (p->n1 == p->n2) 251 asm volatile ("pause"); 252 253 p->n2 = p->n1; 254 you = me + 1; 255 if (you == nr_cpus) 256 you = 0; 257 ++counters[you].n1; 258 } 259 260 static void rd_tsc_adjust_msr(void) 261 { 262 rdmsr(MSR_IA32_TSC_ADJUST); 263 } 264 265 static void wr_tsc_adjust_msr(void) 266 { 267 wrmsr(MSR_IA32_TSC_ADJUST, 0x0); 268 } 269 270 static void wr_kernel_gs_base(void) 271 { 272 wrmsr(MSR_KERNEL_GS_BASE, 0x0); 273 } 274 275 static struct pci_test { 276 unsigned iobar; 277 unsigned ioport; 278 volatile void *memaddr; 279 volatile void *mem; 280 int test_idx; 281 uint32_t data; 282 uint32_t offset; 283 } pci_test = { 284 .test_idx = -1 285 }; 286 287 static void pci_mem_testb(void) 288 { 289 *(volatile uint8_t *)pci_test.mem = pci_test.data; 290 } 291 292 static void pci_mem_testw(void) 293 { 294 *(volatile uint16_t *)pci_test.mem = pci_test.data; 295 } 296 297 static void pci_mem_testl(void) 298 { 299 *(volatile uint32_t *)pci_test.mem = pci_test.data; 300 } 301 302 static void pci_io_testb(void) 303 { 304 outb(pci_test.data, pci_test.ioport); 305 } 306 307 static void pci_io_testw(void) 308 { 309 outw(pci_test.data, pci_test.ioport); 310 } 311 312 static void pci_io_testl(void) 313 { 314 outl(pci_test.data, pci_test.ioport); 315 } 316 317 static uint8_t ioreadb(unsigned long addr, bool io) 318 { 319 if (io) { 320 return inb(addr); 321 } else { 322 return *(volatile uint8_t *)addr; 323 } 324 } 325 326 static uint32_t ioreadl(unsigned long addr, bool io) 327 { 328 /* Note: assumes little endian */ 329 if (io) { 330 return inl(addr); 331 } else { 332 return *(volatile uint32_t *)addr; 333 } 334 } 335 336 static void iowriteb(unsigned long addr, uint8_t data, bool io) 337 { 338 if (io) { 339 outb(data, addr); 340 } else { 341 *(volatile uint8_t *)addr = data; 342 } 343 } 344 345 static bool pci_next(struct test *test, unsigned long addr, bool io) 346 { 347 int i; 348 uint8_t width; 349 350 if (!pci_test.memaddr) { 351 test->func = NULL; 352 return true; 353 } 354 pci_test.test_idx++; 355 iowriteb(addr + offsetof(struct pci_test_dev_hdr, test), 356 pci_test.test_idx, io); 357 width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width), 358 io); 359 switch (width) { 360 case 1: 361 test->func = io ? pci_io_testb : pci_mem_testb; 362 break; 363 case 2: 364 test->func = io ? pci_io_testw : pci_mem_testw; 365 break; 366 case 4: 367 test->func = io ? pci_io_testl : pci_mem_testl; 368 break; 369 default: 370 /* Reset index for purposes of the next test */ 371 pci_test.test_idx = -1; 372 test->func = NULL; 373 return false; 374 } 375 pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data), 376 io); 377 pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr, 378 offset), io); 379 for (i = 0; i < pci_test.offset; ++i) { 380 char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr, 381 name) + i, io); 382 if (!c) { 383 break; 384 } 385 printf("%c",c); 386 } 387 printf(":"); 388 return true; 389 } 390 391 static bool pci_mem_next(struct test *test) 392 { 393 bool ret; 394 ret = pci_next(test, ((unsigned long)pci_test.memaddr), false); 395 if (ret) { 396 pci_test.mem = pci_test.memaddr + pci_test.offset; 397 } 398 return ret; 399 } 400 401 static bool pci_io_next(struct test *test) 402 { 403 bool ret; 404 ret = pci_next(test, ((unsigned long)pci_test.iobar), true); 405 if (ret) { 406 pci_test.ioport = pci_test.iobar + pci_test.offset; 407 } 408 return ret; 409 } 410 411 static int has_tscdeadline(void) 412 { 413 uint32_t lvtt; 414 415 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 416 lvtt = APIC_LVT_TIMER_TSCDEADLINE | IPI_TEST_VECTOR; 417 apic_write(APIC_LVTT, lvtt); 418 return 1; 419 } else { 420 return 0; 421 } 422 } 423 424 static void tscdeadline_immed(void) 425 { 426 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()); 427 asm volatile("nop"); 428 } 429 430 static void tscdeadline(void) 431 { 432 x = 0; 433 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()+3000); 434 while (x == 0) barrier(); 435 } 436 437 static void wr_tsx_ctrl_msr(void) 438 { 439 wrmsr(MSR_IA32_TSX_CTRL, 0); 440 } 441 442 static int has_tsx_ctrl(void) 443 { 444 return this_cpu_has(X86_FEATURE_ARCH_CAPABILITIES) && 445 (rdmsr(MSR_IA32_ARCH_CAPABILITIES) & ARCH_CAP_TSX_CTRL_MSR); 446 } 447 448 static void wr_ibrs_msr(void) 449 { 450 wrmsr(MSR_IA32_SPEC_CTRL, 1); 451 wrmsr(MSR_IA32_SPEC_CTRL, 0); 452 } 453 454 static int has_ibpb(void) 455 { 456 return this_cpu_has(X86_FEATURE_SPEC_CTRL) || 457 this_cpu_has(X86_FEATURE_AMD_IBPB); 458 } 459 460 static int has_spec_ctrl(void) 461 { 462 return this_cpu_has(X86_FEATURE_SPEC_CTRL); 463 } 464 465 static void wr_ibpb_msr(void) 466 { 467 wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); 468 } 469 470 static void toggle_cr0_wp(void) 471 { 472 write_cr0(X86_CR0_PE|X86_CR0_PG); 473 write_cr0(X86_CR0_PE|X86_CR0_WP|X86_CR0_PG); 474 } 475 476 static void toggle_cr4_pge(void) 477 { 478 write_cr4(cr4_shadow ^ X86_CR4_PGE); 479 write_cr4(cr4_shadow); 480 } 481 482 static struct test tests[] = { 483 { cpuid_test, "cpuid", .parallel = 1, }, 484 { vmcall, "vmcall", .parallel = 1, }, 485 #ifdef __x86_64__ 486 { mov_from_cr8, "mov_from_cr8", .parallel = 1, }, 487 { mov_to_cr8, "mov_to_cr8" , .parallel = 1, }, 488 #endif 489 { inl_pmtimer, "inl_from_pmtimer", .parallel = 1, }, 490 { inl_nop_qemu, "inl_from_qemu", .parallel = 1 }, 491 { inl_nop_kernel, "inl_from_kernel", .parallel = 1 }, 492 { outl_elcr_kernel, "outl_to_kernel", .parallel = 1 }, 493 { mov_dr, "mov_dr", .parallel = 1 }, 494 { tscdeadline_immed, "tscdeadline_immed", has_tscdeadline, .parallel = 1, }, 495 { tscdeadline, "tscdeadline", has_tscdeadline, .parallel = 1, }, 496 { self_ipi_sti_nop, "self_ipi_sti_nop", .parallel = 0, }, 497 { self_ipi_sti_hlt, "self_ipi_sti_hlt", .parallel = 0, }, 498 { self_ipi_tpr, "self_ipi_tpr", .parallel = 0, }, 499 { self_ipi_tpr_sti_nop, "self_ipi_tpr_sti_nop", .parallel = 0, }, 500 { self_ipi_tpr_sti_hlt, "self_ipi_tpr_sti_hlt", .parallel = 0, }, 501 { x2apic_self_ipi_sti_nop, "x2apic_self_ipi_sti_nop", is_x2apic, .parallel = 0, }, 502 { x2apic_self_ipi_sti_hlt, "x2apic_self_ipi_sti_hlt", is_x2apic, .parallel = 0, }, 503 { x2apic_self_ipi_tpr, "x2apic_self_ipi_tpr", is_x2apic, .parallel = 0, }, 504 { x2apic_self_ipi_tpr_sti_nop, "x2apic_self_ipi_tpr_sti_nop", is_x2apic, .parallel = 0, }, 505 { x2apic_self_ipi_tpr_sti_hlt, "x2apic_self_ipi_tpr_sti_hlt", is_x2apic, .parallel = 0, }, 506 { ipi, "ipi", is_smp, .parallel = 0, }, 507 { ipi_halt, "ipi_halt", is_smp, .parallel = 0, }, 508 { ple_round_robin, "ple_round_robin", .parallel = 1 }, 509 { wr_kernel_gs_base, "wr_kernel_gs_base", .parallel = 1 }, 510 { wr_tsx_ctrl_msr, "wr_tsx_ctrl_msr", has_tsx_ctrl, .parallel = 1, }, 511 { wr_ibrs_msr, "wr_ibrs_msr", has_spec_ctrl, .parallel = 1 }, 512 { wr_ibpb_msr, "wr_ibpb_msr", has_ibpb, .parallel = 1 }, 513 { wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 }, 514 { rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 }, 515 { toggle_cr0_wp, "toggle_cr0_wp" , .parallel = 1, }, 516 { toggle_cr4_pge, "toggle_cr4_pge" , .parallel = 1, }, 517 { NULL, "pci-mem", .parallel = 0, .next = pci_mem_next }, 518 { NULL, "pci-io", .parallel = 0, .next = pci_io_next }, 519 }; 520 521 unsigned iterations; 522 523 static void run_test(void *_func) 524 { 525 int i; 526 void (*func)(void) = _func; 527 528 for (i = 0; i < iterations; ++i) 529 func(); 530 } 531 532 static bool do_test(struct test *test) 533 { 534 int i; 535 unsigned long long t1, t2; 536 void (*func)(void); 537 538 iterations = 32; 539 540 if (test->valid && !test->valid()) { 541 printf("%s (skipped)\n", test->name); 542 return false; 543 } 544 545 if (test->next && !test->next(test)) { 546 return false; 547 } 548 549 func = test->func; 550 if (!func) { 551 printf("%s (skipped)\n", test->name); 552 return false; 553 } 554 555 do { 556 tsc_eoi = tsc_ipi = 0; 557 iterations *= 2; 558 t1 = rdtsc(); 559 560 if (!test->parallel) { 561 for (i = 0; i < iterations; ++i) 562 func(); 563 } else { 564 on_cpus(run_test, func); 565 } 566 t2 = rdtsc(); 567 } while ((t2 - t1) < GOAL); 568 printf("%s %d\n", test->name, (int)((t2 - t1) / iterations)); 569 if (tsc_ipi) 570 printf(" ipi %s %d\n", test->name, (int)(tsc_ipi / iterations)); 571 if (tsc_eoi) 572 printf(" eoi %s %d\n", test->name, (int)(tsc_eoi / iterations)); 573 574 return test->next; 575 } 576 577 static void enable_nx(void *junk) 578 { 579 if (this_cpu_has(X86_FEATURE_NX)) 580 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK); 581 } 582 583 static bool test_wanted(struct test *test, char *wanted[], int nwanted) 584 { 585 int i; 586 587 if (!nwanted) 588 return true; 589 590 for (i = 0; i < nwanted; ++i) 591 if (strcmp(wanted[i], test->name) == 0) 592 return true; 593 594 return false; 595 } 596 597 int main(int ac, char **av) 598 { 599 int i; 600 unsigned long membar = 0; 601 struct pci_dev pcidev; 602 int ret; 603 604 setup_vm(); 605 cr4_shadow = read_cr4(); 606 handle_irq(IPI_TEST_VECTOR, self_ipi_isr); 607 nr_cpus = cpu_count(); 608 609 sti(); 610 on_cpus(enable_nx, NULL); 611 612 ret = pci_find_dev(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_TEST); 613 if (ret != PCIDEVADDR_INVALID) { 614 pci_dev_init(&pcidev, ret); 615 assert(pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_MEM)); 616 assert(!pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_IO)); 617 membar = pcidev.resource[PCI_TESTDEV_BAR_MEM]; 618 pci_test.memaddr = ioremap(membar, PAGE_SIZE); 619 pci_test.iobar = pcidev.resource[PCI_TESTDEV_BAR_IO]; 620 printf("pci-testdev at %#x membar %lx iobar %x\n", 621 pcidev.bdf, membar, pci_test.iobar); 622 } 623 624 for (i = 0; i < ARRAY_SIZE(tests); ++i) 625 if (test_wanted(&tests[i], av + 1, ac - 1)) 626 while (do_test(&tests[i])) {} 627 628 return 0; 629 } 630