1 #include "libcflat.h" 2 #include "smp.h" 3 #include "processor.h" 4 #include "atomic.h" 5 #include "pci.h" 6 #include "x86/vm.h" 7 #include "x86/desc.h" 8 #include "x86/acpi.h" 9 #include "x86/apic.h" 10 #include "x86/isr.h" 11 12 #define IPI_TEST_VECTOR 0xb0 13 14 struct test { 15 void (*func)(void); 16 const char *name; 17 int (*valid)(void); 18 int parallel; 19 bool (*next)(struct test *); 20 }; 21 22 #define GOAL (1ull << 30) 23 24 static int nr_cpus; 25 26 static void cpuid_test(void) 27 { 28 asm volatile ("push %%"R "bx; cpuid; pop %%"R "bx" 29 : : : "eax", "ecx", "edx"); 30 } 31 32 static void vmcall(void) 33 { 34 unsigned long a = 0, b, c, d; 35 36 asm volatile ("vmcall" : "+a"(a), "=b"(b), "=c"(c), "=d"(d)); 37 } 38 39 #define MSR_EFER 0xc0000080 40 #define EFER_NX_MASK (1ull << 11) 41 42 #ifdef __x86_64__ 43 static void mov_from_cr8(void) 44 { 45 unsigned long cr8; 46 47 asm volatile ("mov %%cr8, %0" : "=r"(cr8)); 48 } 49 50 static void mov_to_cr8(void) 51 { 52 unsigned long cr8 = 0; 53 54 asm volatile ("mov %0, %%cr8" : : "r"(cr8)); 55 } 56 #endif 57 58 static int is_smp(void) 59 { 60 return cpu_count() > 1; 61 } 62 63 static void nop(void *junk) 64 { 65 } 66 67 volatile int x = 0; 68 69 static void self_ipi_isr(isr_regs_t *regs) 70 { 71 x++; 72 eoi(); 73 } 74 75 static void x2apic_self_ipi(int vec) 76 { 77 wrmsr(0x83f, vec); 78 } 79 80 static void apic_self_ipi(int vec) 81 { 82 apic_icr_write(APIC_INT_ASSERT | APIC_DEST_SELF | APIC_DEST_PHYSICAL | 83 APIC_DM_FIXED | IPI_TEST_VECTOR, vec); 84 } 85 86 static void self_ipi_sti_nop(void) 87 { 88 x = 0; 89 irq_disable(); 90 apic_self_ipi(IPI_TEST_VECTOR); 91 asm volatile("sti; nop"); 92 if (x != 1) printf("%d", x); 93 } 94 95 static void self_ipi_sti_hlt(void) 96 { 97 x = 0; 98 irq_disable(); 99 apic_self_ipi(IPI_TEST_VECTOR); 100 asm volatile("sti; hlt"); 101 if (x != 1) printf("%d", x); 102 } 103 104 static void self_ipi_tpr(void) 105 { 106 x = 0; 107 apic_set_tpr(0x0f); 108 apic_self_ipi(IPI_TEST_VECTOR); 109 apic_set_tpr(0x00); 110 asm volatile("nop"); 111 if (x != 1) printf("%d", x); 112 } 113 114 static void self_ipi_tpr_sti_nop(void) 115 { 116 x = 0; 117 irq_disable(); 118 apic_set_tpr(0x0f); 119 apic_self_ipi(IPI_TEST_VECTOR); 120 apic_set_tpr(0x00); 121 asm volatile("sti; nop"); 122 if (x != 1) printf("%d", x); 123 } 124 125 static void self_ipi_tpr_sti_hlt(void) 126 { 127 x = 0; 128 irq_disable(); 129 apic_set_tpr(0x0f); 130 apic_self_ipi(IPI_TEST_VECTOR); 131 apic_set_tpr(0x00); 132 asm volatile("sti; hlt"); 133 if (x != 1) printf("%d", x); 134 } 135 136 static int is_x2apic(void) 137 { 138 return rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 139 } 140 141 static void x2apic_self_ipi_sti_nop(void) 142 { 143 irq_disable(); 144 x2apic_self_ipi(IPI_TEST_VECTOR); 145 asm volatile("sti; nop"); 146 } 147 148 static void x2apic_self_ipi_sti_hlt(void) 149 { 150 irq_disable(); 151 x2apic_self_ipi(IPI_TEST_VECTOR); 152 asm volatile("sti; hlt"); 153 } 154 155 static void x2apic_self_ipi_tpr(void) 156 { 157 apic_set_tpr(0x0f); 158 x2apic_self_ipi(IPI_TEST_VECTOR); 159 apic_set_tpr(0x00); 160 asm volatile("nop"); 161 } 162 163 static void x2apic_self_ipi_tpr_sti_nop(void) 164 { 165 irq_disable(); 166 apic_set_tpr(0x0f); 167 x2apic_self_ipi(IPI_TEST_VECTOR); 168 apic_set_tpr(0x00); 169 asm volatile("sti; nop"); 170 } 171 172 static void x2apic_self_ipi_tpr_sti_hlt(void) 173 { 174 irq_disable(); 175 apic_set_tpr(0x0f); 176 x2apic_self_ipi(IPI_TEST_VECTOR); 177 apic_set_tpr(0x00); 178 asm volatile("sti; hlt"); 179 } 180 181 static void ipi(void) 182 { 183 on_cpu(1, nop, 0); 184 } 185 186 static void ipi_halt(void) 187 { 188 unsigned long long t; 189 190 on_cpu(1, nop, 0); 191 t = rdtsc() + 2000; 192 while (rdtsc() < t) 193 ; 194 } 195 196 int pm_tmr_blk; 197 static void inl_pmtimer(void) 198 { 199 inl(pm_tmr_blk); 200 } 201 202 static void inl_nop_qemu(void) 203 { 204 inl(0x1234); 205 } 206 207 static void inl_nop_kernel(void) 208 { 209 inb(0x4d0); 210 } 211 212 static void outl_elcr_kernel(void) 213 { 214 outb(0, 0x4d0); 215 } 216 217 static void mov_dr(void) 218 { 219 asm volatile("mov %0, %%dr7" : : "r" (0x400L)); 220 } 221 222 static void ple_round_robin(void) 223 { 224 struct counter { 225 volatile int n1; 226 int n2; 227 } __attribute__((aligned(64))); 228 static struct counter counters[64] = { { -1, 0 } }; 229 int me = smp_id(); 230 int you; 231 volatile struct counter *p = &counters[me]; 232 233 while (p->n1 == p->n2) 234 asm volatile ("pause"); 235 236 p->n2 = p->n1; 237 you = me + 1; 238 if (you == nr_cpus) 239 you = 0; 240 ++counters[you].n1; 241 } 242 243 static void rd_tsc_adjust_msr(void) 244 { 245 rdmsr(MSR_IA32_TSC_ADJUST); 246 } 247 248 static void wr_tsc_adjust_msr(void) 249 { 250 wrmsr(MSR_IA32_TSC_ADJUST, 0x0); 251 } 252 253 static void wr_kernel_gs_base(void) 254 { 255 wrmsr(MSR_KERNEL_GS_BASE, 0x0); 256 } 257 258 static struct pci_test { 259 unsigned iobar; 260 unsigned ioport; 261 volatile void *memaddr; 262 volatile void *mem; 263 int test_idx; 264 uint32_t data; 265 uint32_t offset; 266 } pci_test = { 267 .test_idx = -1 268 }; 269 270 static void pci_mem_testb(void) 271 { 272 *(volatile uint8_t *)pci_test.mem = pci_test.data; 273 } 274 275 static void pci_mem_testw(void) 276 { 277 *(volatile uint16_t *)pci_test.mem = pci_test.data; 278 } 279 280 static void pci_mem_testl(void) 281 { 282 *(volatile uint32_t *)pci_test.mem = pci_test.data; 283 } 284 285 static void pci_io_testb(void) 286 { 287 outb(pci_test.data, pci_test.ioport); 288 } 289 290 static void pci_io_testw(void) 291 { 292 outw(pci_test.data, pci_test.ioport); 293 } 294 295 static void pci_io_testl(void) 296 { 297 outl(pci_test.data, pci_test.ioport); 298 } 299 300 static uint8_t ioreadb(unsigned long addr, bool io) 301 { 302 if (io) { 303 return inb(addr); 304 } else { 305 return *(volatile uint8_t *)addr; 306 } 307 } 308 309 static uint32_t ioreadl(unsigned long addr, bool io) 310 { 311 /* Note: assumes little endian */ 312 if (io) { 313 return inl(addr); 314 } else { 315 return *(volatile uint32_t *)addr; 316 } 317 } 318 319 static void iowriteb(unsigned long addr, uint8_t data, bool io) 320 { 321 if (io) { 322 outb(data, addr); 323 } else { 324 *(volatile uint8_t *)addr = data; 325 } 326 } 327 328 static bool pci_next(struct test *test, unsigned long addr, bool io) 329 { 330 int i; 331 uint8_t width; 332 333 if (!pci_test.memaddr) { 334 test->func = NULL; 335 return true; 336 } 337 pci_test.test_idx++; 338 iowriteb(addr + offsetof(struct pci_test_dev_hdr, test), 339 pci_test.test_idx, io); 340 width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width), 341 io); 342 switch (width) { 343 case 1: 344 test->func = io ? pci_io_testb : pci_mem_testb; 345 break; 346 case 2: 347 test->func = io ? pci_io_testw : pci_mem_testw; 348 break; 349 case 4: 350 test->func = io ? pci_io_testl : pci_mem_testl; 351 break; 352 default: 353 /* Reset index for purposes of the next test */ 354 pci_test.test_idx = -1; 355 test->func = NULL; 356 return false; 357 } 358 pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data), 359 io); 360 pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr, 361 offset), io); 362 for (i = 0; i < pci_test.offset; ++i) { 363 char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr, 364 name) + i, io); 365 if (!c) { 366 break; 367 } 368 printf("%c",c); 369 } 370 printf(":"); 371 return true; 372 } 373 374 static bool pci_mem_next(struct test *test) 375 { 376 bool ret; 377 ret = pci_next(test, ((unsigned long)pci_test.memaddr), false); 378 if (ret) { 379 pci_test.mem = pci_test.memaddr + pci_test.offset; 380 } 381 return ret; 382 } 383 384 static bool pci_io_next(struct test *test) 385 { 386 bool ret; 387 ret = pci_next(test, ((unsigned long)pci_test.iobar), true); 388 if (ret) { 389 pci_test.ioport = pci_test.iobar + pci_test.offset; 390 } 391 return ret; 392 } 393 394 static int has_tscdeadline(void) 395 { 396 uint32_t lvtt; 397 398 if (cpuid(1).c & (1 << 24)) { 399 lvtt = APIC_LVT_TIMER_TSCDEADLINE | IPI_TEST_VECTOR; 400 apic_write(APIC_LVTT, lvtt); 401 return 1; 402 } else { 403 return 0; 404 } 405 } 406 407 static void tscdeadline_immed(void) 408 { 409 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()); 410 asm volatile("nop"); 411 } 412 413 static void tscdeadline(void) 414 { 415 x = 0; 416 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()+3000); 417 while (x == 0) barrier(); 418 } 419 420 static int has_spec_ctrl(void) 421 { 422 return !!(cpuid_indexed(7,0).d & (1 << 26)); 423 } 424 425 static void wr_ibrs_msr(void) 426 { 427 wrmsr(MSR_IA32_SPEC_CTRL, 1); 428 wrmsr(MSR_IA32_SPEC_CTRL, 0); 429 } 430 431 static int has_ibpb(void) 432 { 433 return has_spec_ctrl() || !!(cpuid(0x80000008).b & (1 << 12)); 434 } 435 436 static void wr_ibpb_msr(void) 437 { 438 wrmsr(MSR_IA32_PRED_CMD, 1); 439 } 440 441 static struct test tests[] = { 442 { cpuid_test, "cpuid", .parallel = 1, }, 443 { vmcall, "vmcall", .parallel = 1, }, 444 #ifdef __x86_64__ 445 { mov_from_cr8, "mov_from_cr8", .parallel = 1, }, 446 { mov_to_cr8, "mov_to_cr8" , .parallel = 1, }, 447 #endif 448 { inl_pmtimer, "inl_from_pmtimer", .parallel = 1, }, 449 { inl_nop_qemu, "inl_from_qemu", .parallel = 1 }, 450 { inl_nop_kernel, "inl_from_kernel", .parallel = 1 }, 451 { outl_elcr_kernel, "outl_to_kernel", .parallel = 1 }, 452 { mov_dr, "mov_dr", .parallel = 1 }, 453 { tscdeadline_immed, "tscdeadline_immed", has_tscdeadline, .parallel = 1, }, 454 { tscdeadline, "tscdeadline", has_tscdeadline, .parallel = 1, }, 455 { self_ipi_sti_nop, "self_ipi_sti_nop", .parallel = 0, }, 456 { self_ipi_sti_hlt, "self_ipi_sti_hlt", .parallel = 0, }, 457 { self_ipi_tpr, "self_ipi_tpr", .parallel = 0, }, 458 { self_ipi_tpr_sti_nop, "self_ipi_tpr_sti_nop", .parallel = 0, }, 459 { self_ipi_tpr_sti_hlt, "self_ipi_tpr_sti_hlt", .parallel = 0, }, 460 { x2apic_self_ipi_sti_nop, "x2apic_self_ipi_sti_nop", is_x2apic, .parallel = 0, }, 461 { x2apic_self_ipi_sti_hlt, "x2apic_self_ipi_sti_hlt", is_x2apic, .parallel = 0, }, 462 { x2apic_self_ipi_tpr, "x2apic_self_ipi_tpr", is_x2apic, .parallel = 0, }, 463 { x2apic_self_ipi_tpr_sti_nop, "x2apic_self_ipi_tpr_sti_nop", is_x2apic, .parallel = 0, }, 464 { x2apic_self_ipi_tpr_sti_hlt, "x2apic_self_ipi_tpr_sti_hlt", is_x2apic, .parallel = 0, }, 465 { ipi, "ipi", is_smp, .parallel = 0, }, 466 { ipi_halt, "ipi_halt", is_smp, .parallel = 0, }, 467 { ple_round_robin, "ple_round_robin", .parallel = 1 }, 468 { wr_kernel_gs_base, "wr_kernel_gs_base", .parallel = 1 }, 469 { wr_ibrs_msr, "wr_ibrs_msr", has_spec_ctrl, .parallel = 1 }, 470 { wr_ibpb_msr, "wr_ibpb_msr", has_ibpb, .parallel = 1 }, 471 { wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 }, 472 { rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 }, 473 { NULL, "pci-mem", .parallel = 0, .next = pci_mem_next }, 474 { NULL, "pci-io", .parallel = 0, .next = pci_io_next }, 475 }; 476 477 unsigned iterations; 478 479 static void run_test(void *_func) 480 { 481 int i; 482 void (*func)(void) = _func; 483 484 for (i = 0; i < iterations; ++i) 485 func(); 486 } 487 488 static bool do_test(struct test *test) 489 { 490 int i; 491 unsigned long long t1, t2; 492 void (*func)(void); 493 494 iterations = 32; 495 496 if (test->valid && !test->valid()) { 497 printf("%s (skipped)\n", test->name); 498 return false; 499 } 500 501 if (test->next && !test->next(test)) { 502 return false; 503 } 504 505 func = test->func; 506 if (!func) { 507 printf("%s (skipped)\n", test->name); 508 return false; 509 } 510 511 do { 512 iterations *= 2; 513 t1 = rdtsc(); 514 515 if (!test->parallel) { 516 for (i = 0; i < iterations; ++i) 517 func(); 518 } else { 519 on_cpus(run_test, func); 520 } 521 t2 = rdtsc(); 522 } while ((t2 - t1) < GOAL); 523 printf("%s %d\n", test->name, (int)((t2 - t1) / iterations)); 524 return test->next; 525 } 526 527 static void enable_nx(void *junk) 528 { 529 if (cpuid(0x80000001).d & (1 << 20)) 530 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK); 531 } 532 533 bool test_wanted(struct test *test, char *wanted[], int nwanted) 534 { 535 int i; 536 537 if (!nwanted) 538 return true; 539 540 for (i = 0; i < nwanted; ++i) 541 if (strcmp(wanted[i], test->name) == 0) 542 return true; 543 544 return false; 545 } 546 547 int main(int ac, char **av) 548 { 549 struct fadt_descriptor_rev1 *fadt; 550 int i; 551 unsigned long membar = 0; 552 struct pci_dev pcidev; 553 int ret; 554 555 smp_init(); 556 setup_vm(); 557 handle_irq(IPI_TEST_VECTOR, self_ipi_isr); 558 nr_cpus = cpu_count(); 559 560 irq_enable(); 561 on_cpus(enable_nx, NULL); 562 563 fadt = find_acpi_table_addr(FACP_SIGNATURE); 564 pm_tmr_blk = fadt->pm_tmr_blk; 565 printf("PM timer port is %x\n", pm_tmr_blk); 566 567 ret = pci_find_dev(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_TEST); 568 if (ret != PCIDEVADDR_INVALID) { 569 pci_dev_init(&pcidev, ret); 570 assert(pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_MEM)); 571 assert(!pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_IO)); 572 membar = pcidev.resource[PCI_TESTDEV_BAR_MEM]; 573 pci_test.memaddr = ioremap(membar, PAGE_SIZE); 574 pci_test.iobar = pcidev.resource[PCI_TESTDEV_BAR_IO]; 575 printf("pci-testdev at %#x membar %lx iobar %x\n", 576 pcidev.bdf, membar, pci_test.iobar); 577 } 578 579 for (i = 0; i < ARRAY_SIZE(tests); ++i) 580 if (test_wanted(&tests[i], av + 1, ac - 1)) 581 while (do_test(&tests[i])) {} 582 583 return 0; 584 } 585