1 #include "libcflat.h" 2 #include "smp.h" 3 #include "processor.h" 4 #include "atomic.h" 5 #include "pci.h" 6 #include "x86/vm.h" 7 #include "x86/desc.h" 8 #include "x86/acpi.h" 9 #include "x86/apic.h" 10 #include "x86/isr.h" 11 12 #define IPI_TEST_VECTOR 0xb0 13 14 struct test { 15 void (*func)(void); 16 const char *name; 17 int (*valid)(void); 18 int parallel; 19 bool (*next)(struct test *); 20 }; 21 22 #define GOAL (1ull << 30) 23 24 static int nr_cpus; 25 26 static void cpuid_test(void) 27 { 28 asm volatile ("push %%"R "bx; cpuid; pop %%"R "bx" 29 : : : "eax", "ecx", "edx"); 30 } 31 32 static void vmcall(void) 33 { 34 unsigned long a = 0, b, c, d; 35 36 asm volatile ("vmcall" : "+a"(a), "=b"(b), "=c"(c), "=d"(d)); 37 } 38 39 #define MSR_EFER 0xc0000080 40 #define EFER_NX_MASK (1ull << 11) 41 42 #ifdef __x86_64__ 43 static void mov_from_cr8(void) 44 { 45 unsigned long cr8; 46 47 asm volatile ("mov %%cr8, %0" : "=r"(cr8)); 48 } 49 50 static void mov_to_cr8(void) 51 { 52 unsigned long cr8 = 0; 53 54 asm volatile ("mov %0, %%cr8" : : "r"(cr8)); 55 } 56 #endif 57 58 static int is_smp(void) 59 { 60 return cpu_count() > 1; 61 } 62 63 static void nop(void *junk) 64 { 65 } 66 67 volatile int x = 0; 68 volatile uint64_t tsc_eoi = 0; 69 volatile uint64_t tsc_ipi = 0; 70 71 static void self_ipi_isr(isr_regs_t *regs) 72 { 73 x++; 74 uint64_t start = rdtsc(); 75 eoi(); 76 tsc_eoi += rdtsc() - start; 77 } 78 79 static void x2apic_self_ipi(int vec) 80 { 81 uint64_t start = rdtsc(); 82 wrmsr(0x83f, vec); 83 tsc_ipi += rdtsc() - start; 84 } 85 86 static void apic_self_ipi(int vec) 87 { 88 uint64_t start = rdtsc(); 89 apic_icr_write(APIC_INT_ASSERT | APIC_DEST_SELF | APIC_DEST_PHYSICAL | 90 APIC_DM_FIXED | IPI_TEST_VECTOR, vec); 91 tsc_ipi += rdtsc() - start; 92 } 93 94 static void self_ipi_sti_nop(void) 95 { 96 x = 0; 97 irq_disable(); 98 apic_self_ipi(IPI_TEST_VECTOR); 99 asm volatile("sti; nop"); 100 if (x != 1) printf("%d", x); 101 } 102 103 static void self_ipi_sti_hlt(void) 104 { 105 x = 0; 106 irq_disable(); 107 apic_self_ipi(IPI_TEST_VECTOR); 108 asm volatile("sti; hlt"); 109 if (x != 1) printf("%d", x); 110 } 111 112 static void self_ipi_tpr(void) 113 { 114 x = 0; 115 apic_set_tpr(0x0f); 116 apic_self_ipi(IPI_TEST_VECTOR); 117 apic_set_tpr(0x00); 118 asm volatile("nop"); 119 if (x != 1) printf("%d", x); 120 } 121 122 static void self_ipi_tpr_sti_nop(void) 123 { 124 x = 0; 125 irq_disable(); 126 apic_set_tpr(0x0f); 127 apic_self_ipi(IPI_TEST_VECTOR); 128 apic_set_tpr(0x00); 129 asm volatile("sti; nop"); 130 if (x != 1) printf("%d", x); 131 } 132 133 static void self_ipi_tpr_sti_hlt(void) 134 { 135 x = 0; 136 irq_disable(); 137 apic_set_tpr(0x0f); 138 apic_self_ipi(IPI_TEST_VECTOR); 139 apic_set_tpr(0x00); 140 asm volatile("sti; hlt"); 141 if (x != 1) printf("%d", x); 142 } 143 144 static int is_x2apic(void) 145 { 146 return rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 147 } 148 149 static void x2apic_self_ipi_sti_nop(void) 150 { 151 irq_disable(); 152 x2apic_self_ipi(IPI_TEST_VECTOR); 153 asm volatile("sti; nop"); 154 } 155 156 static void x2apic_self_ipi_sti_hlt(void) 157 { 158 irq_disable(); 159 x2apic_self_ipi(IPI_TEST_VECTOR); 160 asm volatile("sti; hlt"); 161 } 162 163 static void x2apic_self_ipi_tpr(void) 164 { 165 apic_set_tpr(0x0f); 166 x2apic_self_ipi(IPI_TEST_VECTOR); 167 apic_set_tpr(0x00); 168 asm volatile("nop"); 169 } 170 171 static void x2apic_self_ipi_tpr_sti_nop(void) 172 { 173 irq_disable(); 174 apic_set_tpr(0x0f); 175 x2apic_self_ipi(IPI_TEST_VECTOR); 176 apic_set_tpr(0x00); 177 asm volatile("sti; nop"); 178 } 179 180 static void x2apic_self_ipi_tpr_sti_hlt(void) 181 { 182 irq_disable(); 183 apic_set_tpr(0x0f); 184 x2apic_self_ipi(IPI_TEST_VECTOR); 185 apic_set_tpr(0x00); 186 asm volatile("sti; hlt"); 187 } 188 189 static void ipi(void) 190 { 191 uint64_t start = rdtsc(); 192 on_cpu(1, nop, 0); 193 tsc_ipi += rdtsc() - start; 194 } 195 196 static void ipi_halt(void) 197 { 198 unsigned long long t; 199 200 on_cpu(1, nop, 0); 201 t = rdtsc() + 2000; 202 while (rdtsc() < t) 203 ; 204 } 205 206 int pm_tmr_blk; 207 static void inl_pmtimer(void) 208 { 209 if (!pm_tmr_blk) { 210 struct fadt_descriptor_rev1 *fadt; 211 212 fadt = find_acpi_table_addr(FACP_SIGNATURE); 213 pm_tmr_blk = fadt->pm_tmr_blk; 214 printf("PM timer port is %x\n", pm_tmr_blk); 215 } 216 inl(pm_tmr_blk); 217 } 218 219 static void inl_nop_qemu(void) 220 { 221 inl(0x1234); 222 } 223 224 static void inl_nop_kernel(void) 225 { 226 inb(0x4d0); 227 } 228 229 static void outl_elcr_kernel(void) 230 { 231 outb(0, 0x4d0); 232 } 233 234 static void mov_dr(void) 235 { 236 asm volatile("mov %0, %%dr7" : : "r" (0x400L)); 237 } 238 239 static void ple_round_robin(void) 240 { 241 struct counter { 242 volatile int n1; 243 int n2; 244 } __attribute__((aligned(64))); 245 static struct counter counters[64] = { { -1, 0 } }; 246 int me = smp_id(); 247 int you; 248 volatile struct counter *p = &counters[me]; 249 250 while (p->n1 == p->n2) 251 asm volatile ("pause"); 252 253 p->n2 = p->n1; 254 you = me + 1; 255 if (you == nr_cpus) 256 you = 0; 257 ++counters[you].n1; 258 } 259 260 static void rd_tsc_adjust_msr(void) 261 { 262 rdmsr(MSR_IA32_TSC_ADJUST); 263 } 264 265 static void wr_tsc_adjust_msr(void) 266 { 267 wrmsr(MSR_IA32_TSC_ADJUST, 0x0); 268 } 269 270 static void wr_kernel_gs_base(void) 271 { 272 wrmsr(MSR_KERNEL_GS_BASE, 0x0); 273 } 274 275 static struct pci_test { 276 unsigned iobar; 277 unsigned ioport; 278 volatile void *memaddr; 279 volatile void *mem; 280 int test_idx; 281 uint32_t data; 282 uint32_t offset; 283 } pci_test = { 284 .test_idx = -1 285 }; 286 287 static void pci_mem_testb(void) 288 { 289 *(volatile uint8_t *)pci_test.mem = pci_test.data; 290 } 291 292 static void pci_mem_testw(void) 293 { 294 *(volatile uint16_t *)pci_test.mem = pci_test.data; 295 } 296 297 static void pci_mem_testl(void) 298 { 299 *(volatile uint32_t *)pci_test.mem = pci_test.data; 300 } 301 302 static void pci_io_testb(void) 303 { 304 outb(pci_test.data, pci_test.ioport); 305 } 306 307 static void pci_io_testw(void) 308 { 309 outw(pci_test.data, pci_test.ioport); 310 } 311 312 static void pci_io_testl(void) 313 { 314 outl(pci_test.data, pci_test.ioport); 315 } 316 317 static uint8_t ioreadb(unsigned long addr, bool io) 318 { 319 if (io) { 320 return inb(addr); 321 } else { 322 return *(volatile uint8_t *)addr; 323 } 324 } 325 326 static uint32_t ioreadl(unsigned long addr, bool io) 327 { 328 /* Note: assumes little endian */ 329 if (io) { 330 return inl(addr); 331 } else { 332 return *(volatile uint32_t *)addr; 333 } 334 } 335 336 static void iowriteb(unsigned long addr, uint8_t data, bool io) 337 { 338 if (io) { 339 outb(data, addr); 340 } else { 341 *(volatile uint8_t *)addr = data; 342 } 343 } 344 345 static bool pci_next(struct test *test, unsigned long addr, bool io) 346 { 347 int i; 348 uint8_t width; 349 350 if (!pci_test.memaddr) { 351 test->func = NULL; 352 return true; 353 } 354 pci_test.test_idx++; 355 iowriteb(addr + offsetof(struct pci_test_dev_hdr, test), 356 pci_test.test_idx, io); 357 width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width), 358 io); 359 switch (width) { 360 case 1: 361 test->func = io ? pci_io_testb : pci_mem_testb; 362 break; 363 case 2: 364 test->func = io ? pci_io_testw : pci_mem_testw; 365 break; 366 case 4: 367 test->func = io ? pci_io_testl : pci_mem_testl; 368 break; 369 default: 370 /* Reset index for purposes of the next test */ 371 pci_test.test_idx = -1; 372 test->func = NULL; 373 return false; 374 } 375 pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data), 376 io); 377 pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr, 378 offset), io); 379 for (i = 0; i < pci_test.offset; ++i) { 380 char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr, 381 name) + i, io); 382 if (!c) { 383 break; 384 } 385 printf("%c",c); 386 } 387 printf(":"); 388 return true; 389 } 390 391 static bool pci_mem_next(struct test *test) 392 { 393 bool ret; 394 ret = pci_next(test, ((unsigned long)pci_test.memaddr), false); 395 if (ret) { 396 pci_test.mem = pci_test.memaddr + pci_test.offset; 397 } 398 return ret; 399 } 400 401 static bool pci_io_next(struct test *test) 402 { 403 bool ret; 404 ret = pci_next(test, ((unsigned long)pci_test.iobar), true); 405 if (ret) { 406 pci_test.ioport = pci_test.iobar + pci_test.offset; 407 } 408 return ret; 409 } 410 411 static int has_tscdeadline(void) 412 { 413 uint32_t lvtt; 414 415 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 416 lvtt = APIC_LVT_TIMER_TSCDEADLINE | IPI_TEST_VECTOR; 417 apic_write(APIC_LVTT, lvtt); 418 return 1; 419 } else { 420 return 0; 421 } 422 } 423 424 static void tscdeadline_immed(void) 425 { 426 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()); 427 asm volatile("nop"); 428 } 429 430 static void tscdeadline(void) 431 { 432 x = 0; 433 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()+3000); 434 while (x == 0) barrier(); 435 } 436 437 static void wr_ibrs_msr(void) 438 { 439 wrmsr(MSR_IA32_SPEC_CTRL, 1); 440 wrmsr(MSR_IA32_SPEC_CTRL, 0); 441 } 442 443 static int has_ibpb(void) 444 { 445 return has_spec_ctrl() || !!(this_cpu_has(X86_FEATURE_AMD_IBPB)); 446 } 447 448 static void wr_ibpb_msr(void) 449 { 450 wrmsr(MSR_IA32_PRED_CMD, 1); 451 } 452 453 static struct test tests[] = { 454 { cpuid_test, "cpuid", .parallel = 1, }, 455 { vmcall, "vmcall", .parallel = 1, }, 456 #ifdef __x86_64__ 457 { mov_from_cr8, "mov_from_cr8", .parallel = 1, }, 458 { mov_to_cr8, "mov_to_cr8" , .parallel = 1, }, 459 #endif 460 { inl_pmtimer, "inl_from_pmtimer", .parallel = 1, }, 461 { inl_nop_qemu, "inl_from_qemu", .parallel = 1 }, 462 { inl_nop_kernel, "inl_from_kernel", .parallel = 1 }, 463 { outl_elcr_kernel, "outl_to_kernel", .parallel = 1 }, 464 { mov_dr, "mov_dr", .parallel = 1 }, 465 { tscdeadline_immed, "tscdeadline_immed", has_tscdeadline, .parallel = 1, }, 466 { tscdeadline, "tscdeadline", has_tscdeadline, .parallel = 1, }, 467 { self_ipi_sti_nop, "self_ipi_sti_nop", .parallel = 0, }, 468 { self_ipi_sti_hlt, "self_ipi_sti_hlt", .parallel = 0, }, 469 { self_ipi_tpr, "self_ipi_tpr", .parallel = 0, }, 470 { self_ipi_tpr_sti_nop, "self_ipi_tpr_sti_nop", .parallel = 0, }, 471 { self_ipi_tpr_sti_hlt, "self_ipi_tpr_sti_hlt", .parallel = 0, }, 472 { x2apic_self_ipi_sti_nop, "x2apic_self_ipi_sti_nop", is_x2apic, .parallel = 0, }, 473 { x2apic_self_ipi_sti_hlt, "x2apic_self_ipi_sti_hlt", is_x2apic, .parallel = 0, }, 474 { x2apic_self_ipi_tpr, "x2apic_self_ipi_tpr", is_x2apic, .parallel = 0, }, 475 { x2apic_self_ipi_tpr_sti_nop, "x2apic_self_ipi_tpr_sti_nop", is_x2apic, .parallel = 0, }, 476 { x2apic_self_ipi_tpr_sti_hlt, "x2apic_self_ipi_tpr_sti_hlt", is_x2apic, .parallel = 0, }, 477 { ipi, "ipi", is_smp, .parallel = 0, }, 478 { ipi_halt, "ipi_halt", is_smp, .parallel = 0, }, 479 { ple_round_robin, "ple_round_robin", .parallel = 1 }, 480 { wr_kernel_gs_base, "wr_kernel_gs_base", .parallel = 1 }, 481 { wr_ibrs_msr, "wr_ibrs_msr", has_spec_ctrl, .parallel = 1 }, 482 { wr_ibpb_msr, "wr_ibpb_msr", has_ibpb, .parallel = 1 }, 483 { wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 }, 484 { rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 }, 485 { NULL, "pci-mem", .parallel = 0, .next = pci_mem_next }, 486 { NULL, "pci-io", .parallel = 0, .next = pci_io_next }, 487 }; 488 489 unsigned iterations; 490 491 static void run_test(void *_func) 492 { 493 int i; 494 void (*func)(void) = _func; 495 496 for (i = 0; i < iterations; ++i) 497 func(); 498 } 499 500 static bool do_test(struct test *test) 501 { 502 int i; 503 unsigned long long t1, t2; 504 void (*func)(void); 505 506 iterations = 32; 507 508 if (test->valid && !test->valid()) { 509 printf("%s (skipped)\n", test->name); 510 return false; 511 } 512 513 if (test->next && !test->next(test)) { 514 return false; 515 } 516 517 func = test->func; 518 if (!func) { 519 printf("%s (skipped)\n", test->name); 520 return false; 521 } 522 523 do { 524 tsc_eoi = tsc_ipi = 0; 525 iterations *= 2; 526 t1 = rdtsc(); 527 528 if (!test->parallel) { 529 for (i = 0; i < iterations; ++i) 530 func(); 531 } else { 532 on_cpus(run_test, func); 533 } 534 t2 = rdtsc(); 535 } while ((t2 - t1) < GOAL); 536 printf("%s %d\n", test->name, (int)((t2 - t1) / iterations)); 537 if (tsc_ipi) 538 printf(" ipi %s %d\n", test->name, (int)(tsc_ipi / iterations)); 539 if (tsc_eoi) 540 printf(" eoi %s %d\n", test->name, (int)(tsc_eoi / iterations)); 541 542 return test->next; 543 } 544 545 static void enable_nx(void *junk) 546 { 547 if (this_cpu_has(X86_FEATURE_NX)) 548 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK); 549 } 550 551 static bool test_wanted(struct test *test, char *wanted[], int nwanted) 552 { 553 int i; 554 555 if (!nwanted) 556 return true; 557 558 for (i = 0; i < nwanted; ++i) 559 if (strcmp(wanted[i], test->name) == 0) 560 return true; 561 562 return false; 563 } 564 565 int main(int ac, char **av) 566 { 567 int i; 568 unsigned long membar = 0; 569 struct pci_dev pcidev; 570 int ret; 571 572 smp_init(); 573 setup_vm(); 574 handle_irq(IPI_TEST_VECTOR, self_ipi_isr); 575 nr_cpus = cpu_count(); 576 577 irq_enable(); 578 on_cpus(enable_nx, NULL); 579 580 ret = pci_find_dev(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_TEST); 581 if (ret != PCIDEVADDR_INVALID) { 582 pci_dev_init(&pcidev, ret); 583 assert(pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_MEM)); 584 assert(!pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_IO)); 585 membar = pcidev.resource[PCI_TESTDEV_BAR_MEM]; 586 pci_test.memaddr = ioremap(membar, PAGE_SIZE); 587 pci_test.iobar = pcidev.resource[PCI_TESTDEV_BAR_IO]; 588 printf("pci-testdev at %#x membar %lx iobar %x\n", 589 pcidev.bdf, membar, pci_test.iobar); 590 } 591 592 for (i = 0; i < ARRAY_SIZE(tests); ++i) 593 if (test_wanted(&tests[i], av + 1, ac - 1)) 594 while (do_test(&tests[i])) {} 595 596 return 0; 597 } 598