1 #include "libcflat.h" 2 #include "smp.h" 3 #include "processor.h" 4 #include "atomic.h" 5 #include "pci.h" 6 #include "x86/vm.h" 7 #include "x86/desc.h" 8 #include "x86/acpi.h" 9 #include "x86/apic.h" 10 #include "x86/isr.h" 11 12 #define IPI_TEST_VECTOR 0xb0 13 14 struct test { 15 void (*func)(void); 16 const char *name; 17 int (*valid)(void); 18 int parallel; 19 bool (*next)(struct test *); 20 }; 21 22 #define GOAL (1ull << 30) 23 24 static int nr_cpus; 25 26 static void cpuid_test(void) 27 { 28 asm volatile ("push %%"R "bx; cpuid; pop %%"R "bx" 29 : : : "eax", "ecx", "edx"); 30 } 31 32 static void vmcall(void) 33 { 34 unsigned long a = 0, b, c, d; 35 36 asm volatile ("vmcall" : "+a"(a), "=b"(b), "=c"(c), "=d"(d)); 37 } 38 39 #define MSR_EFER 0xc0000080 40 #define EFER_NX_MASK (1ull << 11) 41 42 #ifdef __x86_64__ 43 static void mov_from_cr8(void) 44 { 45 unsigned long cr8; 46 47 asm volatile ("mov %%cr8, %0" : "=r"(cr8)); 48 } 49 50 static void mov_to_cr8(void) 51 { 52 unsigned long cr8 = 0; 53 54 asm volatile ("mov %0, %%cr8" : : "r"(cr8)); 55 } 56 #endif 57 58 static int is_smp(void) 59 { 60 return cpu_count() > 1; 61 } 62 63 static void nop(void *junk) 64 { 65 } 66 67 volatile int x = 0; 68 69 static void self_ipi_isr(isr_regs_t *regs) 70 { 71 x++; 72 eoi(); 73 } 74 75 static void x2apic_self_ipi(int vec) 76 { 77 wrmsr(0x83f, vec); 78 } 79 80 static void apic_self_ipi(int vec) 81 { 82 apic_icr_write(APIC_INT_ASSERT | APIC_DEST_SELF | APIC_DEST_PHYSICAL | 83 APIC_DM_FIXED | IPI_TEST_VECTOR, vec); 84 } 85 86 static void self_ipi_sti_nop(void) 87 { 88 x = 0; 89 irq_disable(); 90 apic_self_ipi(IPI_TEST_VECTOR); 91 asm volatile("sti; nop"); 92 if (x != 1) printf("%d", x); 93 } 94 95 static void self_ipi_sti_hlt(void) 96 { 97 x = 0; 98 irq_disable(); 99 apic_self_ipi(IPI_TEST_VECTOR); 100 asm volatile("sti; hlt"); 101 if (x != 1) printf("%d", x); 102 } 103 104 static void self_ipi_tpr(void) 105 { 106 x = 0; 107 apic_set_tpr(0x0f); 108 apic_self_ipi(IPI_TEST_VECTOR); 109 apic_set_tpr(0x00); 110 asm volatile("nop"); 111 if (x != 1) printf("%d", x); 112 } 113 114 static void self_ipi_tpr_sti_nop(void) 115 { 116 x = 0; 117 irq_disable(); 118 apic_set_tpr(0x0f); 119 apic_self_ipi(IPI_TEST_VECTOR); 120 apic_set_tpr(0x00); 121 asm volatile("sti; nop"); 122 if (x != 1) printf("%d", x); 123 } 124 125 static void self_ipi_tpr_sti_hlt(void) 126 { 127 x = 0; 128 irq_disable(); 129 apic_set_tpr(0x0f); 130 apic_self_ipi(IPI_TEST_VECTOR); 131 apic_set_tpr(0x00); 132 asm volatile("sti; hlt"); 133 if (x != 1) printf("%d", x); 134 } 135 136 static int is_x2apic(void) 137 { 138 return rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 139 } 140 141 static void x2apic_self_ipi_sti_nop(void) 142 { 143 irq_disable(); 144 x2apic_self_ipi(IPI_TEST_VECTOR); 145 asm volatile("sti; nop"); 146 } 147 148 static void x2apic_self_ipi_sti_hlt(void) 149 { 150 irq_disable(); 151 x2apic_self_ipi(IPI_TEST_VECTOR); 152 asm volatile("sti; hlt"); 153 } 154 155 static void x2apic_self_ipi_tpr(void) 156 { 157 apic_set_tpr(0x0f); 158 x2apic_self_ipi(IPI_TEST_VECTOR); 159 apic_set_tpr(0x00); 160 asm volatile("nop"); 161 } 162 163 static void x2apic_self_ipi_tpr_sti_nop(void) 164 { 165 irq_disable(); 166 apic_set_tpr(0x0f); 167 x2apic_self_ipi(IPI_TEST_VECTOR); 168 apic_set_tpr(0x00); 169 asm volatile("sti; nop"); 170 } 171 172 static void x2apic_self_ipi_tpr_sti_hlt(void) 173 { 174 irq_disable(); 175 apic_set_tpr(0x0f); 176 x2apic_self_ipi(IPI_TEST_VECTOR); 177 apic_set_tpr(0x00); 178 asm volatile("sti; hlt"); 179 } 180 181 static void ipi(void) 182 { 183 on_cpu(1, nop, 0); 184 } 185 186 static void ipi_halt(void) 187 { 188 unsigned long long t; 189 190 on_cpu(1, nop, 0); 191 t = rdtsc() + 2000; 192 while (rdtsc() < t) 193 ; 194 } 195 196 int pm_tmr_blk; 197 static void inl_pmtimer(void) 198 { 199 if (!pm_tmr_blk) { 200 struct fadt_descriptor_rev1 *fadt; 201 202 fadt = find_acpi_table_addr(FACP_SIGNATURE); 203 pm_tmr_blk = fadt->pm_tmr_blk; 204 printf("PM timer port is %x\n", pm_tmr_blk); 205 } 206 inl(pm_tmr_blk); 207 } 208 209 static void inl_nop_qemu(void) 210 { 211 inl(0x1234); 212 } 213 214 static void inl_nop_kernel(void) 215 { 216 inb(0x4d0); 217 } 218 219 static void outl_elcr_kernel(void) 220 { 221 outb(0, 0x4d0); 222 } 223 224 static void mov_dr(void) 225 { 226 asm volatile("mov %0, %%dr7" : : "r" (0x400L)); 227 } 228 229 static void ple_round_robin(void) 230 { 231 struct counter { 232 volatile int n1; 233 int n2; 234 } __attribute__((aligned(64))); 235 static struct counter counters[64] = { { -1, 0 } }; 236 int me = smp_id(); 237 int you; 238 volatile struct counter *p = &counters[me]; 239 240 while (p->n1 == p->n2) 241 asm volatile ("pause"); 242 243 p->n2 = p->n1; 244 you = me + 1; 245 if (you == nr_cpus) 246 you = 0; 247 ++counters[you].n1; 248 } 249 250 static void rd_tsc_adjust_msr(void) 251 { 252 rdmsr(MSR_IA32_TSC_ADJUST); 253 } 254 255 static void wr_tsc_adjust_msr(void) 256 { 257 wrmsr(MSR_IA32_TSC_ADJUST, 0x0); 258 } 259 260 static void wr_kernel_gs_base(void) 261 { 262 wrmsr(MSR_KERNEL_GS_BASE, 0x0); 263 } 264 265 static struct pci_test { 266 unsigned iobar; 267 unsigned ioport; 268 volatile void *memaddr; 269 volatile void *mem; 270 int test_idx; 271 uint32_t data; 272 uint32_t offset; 273 } pci_test = { 274 .test_idx = -1 275 }; 276 277 static void pci_mem_testb(void) 278 { 279 *(volatile uint8_t *)pci_test.mem = pci_test.data; 280 } 281 282 static void pci_mem_testw(void) 283 { 284 *(volatile uint16_t *)pci_test.mem = pci_test.data; 285 } 286 287 static void pci_mem_testl(void) 288 { 289 *(volatile uint32_t *)pci_test.mem = pci_test.data; 290 } 291 292 static void pci_io_testb(void) 293 { 294 outb(pci_test.data, pci_test.ioport); 295 } 296 297 static void pci_io_testw(void) 298 { 299 outw(pci_test.data, pci_test.ioport); 300 } 301 302 static void pci_io_testl(void) 303 { 304 outl(pci_test.data, pci_test.ioport); 305 } 306 307 static uint8_t ioreadb(unsigned long addr, bool io) 308 { 309 if (io) { 310 return inb(addr); 311 } else { 312 return *(volatile uint8_t *)addr; 313 } 314 } 315 316 static uint32_t ioreadl(unsigned long addr, bool io) 317 { 318 /* Note: assumes little endian */ 319 if (io) { 320 return inl(addr); 321 } else { 322 return *(volatile uint32_t *)addr; 323 } 324 } 325 326 static void iowriteb(unsigned long addr, uint8_t data, bool io) 327 { 328 if (io) { 329 outb(data, addr); 330 } else { 331 *(volatile uint8_t *)addr = data; 332 } 333 } 334 335 static bool pci_next(struct test *test, unsigned long addr, bool io) 336 { 337 int i; 338 uint8_t width; 339 340 if (!pci_test.memaddr) { 341 test->func = NULL; 342 return true; 343 } 344 pci_test.test_idx++; 345 iowriteb(addr + offsetof(struct pci_test_dev_hdr, test), 346 pci_test.test_idx, io); 347 width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width), 348 io); 349 switch (width) { 350 case 1: 351 test->func = io ? pci_io_testb : pci_mem_testb; 352 break; 353 case 2: 354 test->func = io ? pci_io_testw : pci_mem_testw; 355 break; 356 case 4: 357 test->func = io ? pci_io_testl : pci_mem_testl; 358 break; 359 default: 360 /* Reset index for purposes of the next test */ 361 pci_test.test_idx = -1; 362 test->func = NULL; 363 return false; 364 } 365 pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data), 366 io); 367 pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr, 368 offset), io); 369 for (i = 0; i < pci_test.offset; ++i) { 370 char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr, 371 name) + i, io); 372 if (!c) { 373 break; 374 } 375 printf("%c",c); 376 } 377 printf(":"); 378 return true; 379 } 380 381 static bool pci_mem_next(struct test *test) 382 { 383 bool ret; 384 ret = pci_next(test, ((unsigned long)pci_test.memaddr), false); 385 if (ret) { 386 pci_test.mem = pci_test.memaddr + pci_test.offset; 387 } 388 return ret; 389 } 390 391 static bool pci_io_next(struct test *test) 392 { 393 bool ret; 394 ret = pci_next(test, ((unsigned long)pci_test.iobar), true); 395 if (ret) { 396 pci_test.ioport = pci_test.iobar + pci_test.offset; 397 } 398 return ret; 399 } 400 401 static int has_tscdeadline(void) 402 { 403 uint32_t lvtt; 404 405 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 406 lvtt = APIC_LVT_TIMER_TSCDEADLINE | IPI_TEST_VECTOR; 407 apic_write(APIC_LVTT, lvtt); 408 return 1; 409 } else { 410 return 0; 411 } 412 } 413 414 static void tscdeadline_immed(void) 415 { 416 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()); 417 asm volatile("nop"); 418 } 419 420 static void tscdeadline(void) 421 { 422 x = 0; 423 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()+3000); 424 while (x == 0) barrier(); 425 } 426 427 static void wr_ibrs_msr(void) 428 { 429 wrmsr(MSR_IA32_SPEC_CTRL, 1); 430 wrmsr(MSR_IA32_SPEC_CTRL, 0); 431 } 432 433 static int has_ibpb(void) 434 { 435 return has_spec_ctrl() || !!(this_cpu_has(X86_FEATURE_AMD_IBPB)); 436 } 437 438 static void wr_ibpb_msr(void) 439 { 440 wrmsr(MSR_IA32_PRED_CMD, 1); 441 } 442 443 static struct test tests[] = { 444 { cpuid_test, "cpuid", .parallel = 1, }, 445 { vmcall, "vmcall", .parallel = 1, }, 446 #ifdef __x86_64__ 447 { mov_from_cr8, "mov_from_cr8", .parallel = 1, }, 448 { mov_to_cr8, "mov_to_cr8" , .parallel = 1, }, 449 #endif 450 { inl_pmtimer, "inl_from_pmtimer", .parallel = 1, }, 451 { inl_nop_qemu, "inl_from_qemu", .parallel = 1 }, 452 { inl_nop_kernel, "inl_from_kernel", .parallel = 1 }, 453 { outl_elcr_kernel, "outl_to_kernel", .parallel = 1 }, 454 { mov_dr, "mov_dr", .parallel = 1 }, 455 { tscdeadline_immed, "tscdeadline_immed", has_tscdeadline, .parallel = 1, }, 456 { tscdeadline, "tscdeadline", has_tscdeadline, .parallel = 1, }, 457 { self_ipi_sti_nop, "self_ipi_sti_nop", .parallel = 0, }, 458 { self_ipi_sti_hlt, "self_ipi_sti_hlt", .parallel = 0, }, 459 { self_ipi_tpr, "self_ipi_tpr", .parallel = 0, }, 460 { self_ipi_tpr_sti_nop, "self_ipi_tpr_sti_nop", .parallel = 0, }, 461 { self_ipi_tpr_sti_hlt, "self_ipi_tpr_sti_hlt", .parallel = 0, }, 462 { x2apic_self_ipi_sti_nop, "x2apic_self_ipi_sti_nop", is_x2apic, .parallel = 0, }, 463 { x2apic_self_ipi_sti_hlt, "x2apic_self_ipi_sti_hlt", is_x2apic, .parallel = 0, }, 464 { x2apic_self_ipi_tpr, "x2apic_self_ipi_tpr", is_x2apic, .parallel = 0, }, 465 { x2apic_self_ipi_tpr_sti_nop, "x2apic_self_ipi_tpr_sti_nop", is_x2apic, .parallel = 0, }, 466 { x2apic_self_ipi_tpr_sti_hlt, "x2apic_self_ipi_tpr_sti_hlt", is_x2apic, .parallel = 0, }, 467 { ipi, "ipi", is_smp, .parallel = 0, }, 468 { ipi_halt, "ipi_halt", is_smp, .parallel = 0, }, 469 { ple_round_robin, "ple_round_robin", .parallel = 1 }, 470 { wr_kernel_gs_base, "wr_kernel_gs_base", .parallel = 1 }, 471 { wr_ibrs_msr, "wr_ibrs_msr", has_spec_ctrl, .parallel = 1 }, 472 { wr_ibpb_msr, "wr_ibpb_msr", has_ibpb, .parallel = 1 }, 473 { wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 }, 474 { rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 }, 475 { NULL, "pci-mem", .parallel = 0, .next = pci_mem_next }, 476 { NULL, "pci-io", .parallel = 0, .next = pci_io_next }, 477 }; 478 479 unsigned iterations; 480 481 static void run_test(void *_func) 482 { 483 int i; 484 void (*func)(void) = _func; 485 486 for (i = 0; i < iterations; ++i) 487 func(); 488 } 489 490 static bool do_test(struct test *test) 491 { 492 int i; 493 unsigned long long t1, t2; 494 void (*func)(void); 495 496 iterations = 32; 497 498 if (test->valid && !test->valid()) { 499 printf("%s (skipped)\n", test->name); 500 return false; 501 } 502 503 if (test->next && !test->next(test)) { 504 return false; 505 } 506 507 func = test->func; 508 if (!func) { 509 printf("%s (skipped)\n", test->name); 510 return false; 511 } 512 513 do { 514 iterations *= 2; 515 t1 = rdtsc(); 516 517 if (!test->parallel) { 518 for (i = 0; i < iterations; ++i) 519 func(); 520 } else { 521 on_cpus(run_test, func); 522 } 523 t2 = rdtsc(); 524 } while ((t2 - t1) < GOAL); 525 printf("%s %d\n", test->name, (int)((t2 - t1) / iterations)); 526 return test->next; 527 } 528 529 static void enable_nx(void *junk) 530 { 531 if (this_cpu_has(X86_FEATURE_NX)) 532 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK); 533 } 534 535 static bool test_wanted(struct test *test, char *wanted[], int nwanted) 536 { 537 int i; 538 539 if (!nwanted) 540 return true; 541 542 for (i = 0; i < nwanted; ++i) 543 if (strcmp(wanted[i], test->name) == 0) 544 return true; 545 546 return false; 547 } 548 549 int main(int ac, char **av) 550 { 551 int i; 552 unsigned long membar = 0; 553 struct pci_dev pcidev; 554 int ret; 555 556 smp_init(); 557 setup_vm(); 558 handle_irq(IPI_TEST_VECTOR, self_ipi_isr); 559 nr_cpus = cpu_count(); 560 561 irq_enable(); 562 on_cpus(enable_nx, NULL); 563 564 ret = pci_find_dev(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_TEST); 565 if (ret != PCIDEVADDR_INVALID) { 566 pci_dev_init(&pcidev, ret); 567 assert(pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_MEM)); 568 assert(!pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_IO)); 569 membar = pcidev.resource[PCI_TESTDEV_BAR_MEM]; 570 pci_test.memaddr = ioremap(membar, PAGE_SIZE); 571 pci_test.iobar = pcidev.resource[PCI_TESTDEV_BAR_IO]; 572 printf("pci-testdev at %#x membar %lx iobar %x\n", 573 pcidev.bdf, membar, pci_test.iobar); 574 } 575 576 for (i = 0; i < ARRAY_SIZE(tests); ++i) 577 if (test_wanted(&tests[i], av + 1, ac - 1)) 578 while (do_test(&tests[i])) {} 579 580 return 0; 581 } 582