1 #include "libcflat.h" 2 #include "smp.h" 3 #include "processor.h" 4 #include "atomic.h" 5 #include "pci.h" 6 #include "x86/vm.h" 7 #include "x86/desc.h" 8 #include "x86/acpi.h" 9 #include "x86/apic.h" 10 #include "x86/isr.h" 11 12 #define IPI_TEST_VECTOR 0xb0 13 14 struct test { 15 void (*func)(void); 16 const char *name; 17 int (*valid)(void); 18 int parallel; 19 bool (*next)(struct test *); 20 }; 21 22 #define GOAL (1ull << 30) 23 24 static int nr_cpus; 25 26 static void cpuid_test(void) 27 { 28 asm volatile ("push %%"R "bx; cpuid; pop %%"R "bx" 29 : : : "eax", "ecx", "edx"); 30 } 31 32 static void vmcall(void) 33 { 34 unsigned long a = 0, b, c, d; 35 36 asm volatile ("vmcall" : "+a"(a), "=b"(b), "=c"(c), "=d"(d)); 37 } 38 39 #define MSR_TSC_ADJUST 0x3b 40 #define MSR_EFER 0xc0000080 41 #define EFER_NX_MASK (1ull << 11) 42 43 #ifdef __x86_64__ 44 static void mov_from_cr8(void) 45 { 46 unsigned long cr8; 47 48 asm volatile ("mov %%cr8, %0" : "=r"(cr8)); 49 } 50 51 static void mov_to_cr8(void) 52 { 53 unsigned long cr8 = 0; 54 55 asm volatile ("mov %0, %%cr8" : : "r"(cr8)); 56 } 57 #endif 58 59 static int is_smp(void) 60 { 61 return cpu_count() > 1; 62 } 63 64 static void nop(void *junk) 65 { 66 } 67 68 volatile int x = 0; 69 70 static void self_ipi_isr(isr_regs_t *regs) 71 { 72 x++; 73 eoi(); 74 } 75 76 static void x2apic_self_ipi(int vec) 77 { 78 wrmsr(0x83f, vec); 79 } 80 81 static void apic_self_ipi(int vec) 82 { 83 apic_icr_write(APIC_INT_ASSERT | APIC_DEST_SELF | APIC_DEST_PHYSICAL | 84 APIC_DM_FIXED | IPI_TEST_VECTOR, vec); 85 } 86 87 static void self_ipi_sti_nop(void) 88 { 89 x = 0; 90 irq_disable(); 91 apic_self_ipi(IPI_TEST_VECTOR); 92 asm volatile("sti; nop"); 93 if (x != 1) printf("%d", x); 94 } 95 96 static void self_ipi_sti_hlt(void) 97 { 98 x = 0; 99 irq_disable(); 100 apic_self_ipi(IPI_TEST_VECTOR); 101 asm volatile("sti; hlt"); 102 if (x != 1) printf("%d", x); 103 } 104 105 static void self_ipi_tpr(void) 106 { 107 x = 0; 108 apic_set_tpr(0x0f); 109 apic_self_ipi(IPI_TEST_VECTOR); 110 apic_set_tpr(0x00); 111 asm volatile("nop"); 112 if (x != 1) printf("%d", x); 113 } 114 115 static void self_ipi_tpr_sti_nop(void) 116 { 117 x = 0; 118 irq_disable(); 119 apic_set_tpr(0x0f); 120 apic_self_ipi(IPI_TEST_VECTOR); 121 apic_set_tpr(0x00); 122 asm volatile("sti; nop"); 123 if (x != 1) printf("%d", x); 124 } 125 126 static void self_ipi_tpr_sti_hlt(void) 127 { 128 x = 0; 129 irq_disable(); 130 apic_set_tpr(0x0f); 131 apic_self_ipi(IPI_TEST_VECTOR); 132 apic_set_tpr(0x00); 133 asm volatile("sti; hlt"); 134 if (x != 1) printf("%d", x); 135 } 136 137 static int is_x2apic(void) 138 { 139 return rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 140 } 141 142 static void x2apic_self_ipi_sti_nop(void) 143 { 144 irq_disable(); 145 x2apic_self_ipi(IPI_TEST_VECTOR); 146 asm volatile("sti; nop"); 147 } 148 149 static void x2apic_self_ipi_sti_hlt(void) 150 { 151 irq_disable(); 152 x2apic_self_ipi(IPI_TEST_VECTOR); 153 asm volatile("sti; hlt"); 154 } 155 156 static void x2apic_self_ipi_tpr(void) 157 { 158 apic_set_tpr(0x0f); 159 x2apic_self_ipi(IPI_TEST_VECTOR); 160 apic_set_tpr(0x00); 161 asm volatile("nop"); 162 } 163 164 static void x2apic_self_ipi_tpr_sti_nop(void) 165 { 166 irq_disable(); 167 apic_set_tpr(0x0f); 168 x2apic_self_ipi(IPI_TEST_VECTOR); 169 apic_set_tpr(0x00); 170 asm volatile("sti; nop"); 171 } 172 173 static void x2apic_self_ipi_tpr_sti_hlt(void) 174 { 175 irq_disable(); 176 apic_set_tpr(0x0f); 177 x2apic_self_ipi(IPI_TEST_VECTOR); 178 apic_set_tpr(0x00); 179 asm volatile("sti; hlt"); 180 } 181 182 static void ipi(void) 183 { 184 on_cpu(1, nop, 0); 185 } 186 187 static void ipi_halt(void) 188 { 189 unsigned long long t; 190 191 on_cpu(1, nop, 0); 192 t = rdtsc() + 2000; 193 while (rdtsc() < t) 194 ; 195 } 196 197 int pm_tmr_blk; 198 static void inl_pmtimer(void) 199 { 200 inl(pm_tmr_blk); 201 } 202 203 static void inl_nop_qemu(void) 204 { 205 inl(0x1234); 206 } 207 208 static void inl_nop_kernel(void) 209 { 210 inb(0x4d0); 211 } 212 213 static void outl_elcr_kernel(void) 214 { 215 outb(0, 0x4d0); 216 } 217 218 static void mov_dr(void) 219 { 220 asm volatile("mov %0, %%dr7" : : "r" (0x400L)); 221 } 222 223 static void ple_round_robin(void) 224 { 225 struct counter { 226 volatile int n1; 227 int n2; 228 } __attribute__((aligned(64))); 229 static struct counter counters[64] = { { -1, 0 } }; 230 int me = smp_id(); 231 int you; 232 volatile struct counter *p = &counters[me]; 233 234 while (p->n1 == p->n2) 235 asm volatile ("pause"); 236 237 p->n2 = p->n1; 238 you = me + 1; 239 if (you == nr_cpus) 240 you = 0; 241 ++counters[you].n1; 242 } 243 244 static void rd_tsc_adjust_msr(void) 245 { 246 rdmsr(MSR_TSC_ADJUST); 247 } 248 249 static void wr_tsc_adjust_msr(void) 250 { 251 wrmsr(MSR_TSC_ADJUST, 0x0); 252 } 253 254 static void wr_kernel_gs_base(void) 255 { 256 wrmsr(MSR_KERNEL_GS_BASE, 0x0); 257 } 258 259 static struct pci_test { 260 unsigned iobar; 261 unsigned ioport; 262 volatile void *memaddr; 263 volatile void *mem; 264 int test_idx; 265 uint32_t data; 266 uint32_t offset; 267 } pci_test = { 268 .test_idx = -1 269 }; 270 271 static void pci_mem_testb(void) 272 { 273 *(volatile uint8_t *)pci_test.mem = pci_test.data; 274 } 275 276 static void pci_mem_testw(void) 277 { 278 *(volatile uint16_t *)pci_test.mem = pci_test.data; 279 } 280 281 static void pci_mem_testl(void) 282 { 283 *(volatile uint32_t *)pci_test.mem = pci_test.data; 284 } 285 286 static void pci_io_testb(void) 287 { 288 outb(pci_test.data, pci_test.ioport); 289 } 290 291 static void pci_io_testw(void) 292 { 293 outw(pci_test.data, pci_test.ioport); 294 } 295 296 static void pci_io_testl(void) 297 { 298 outl(pci_test.data, pci_test.ioport); 299 } 300 301 static uint8_t ioreadb(unsigned long addr, bool io) 302 { 303 if (io) { 304 return inb(addr); 305 } else { 306 return *(volatile uint8_t *)addr; 307 } 308 } 309 310 static uint32_t ioreadl(unsigned long addr, bool io) 311 { 312 /* Note: assumes little endian */ 313 if (io) { 314 return inl(addr); 315 } else { 316 return *(volatile uint32_t *)addr; 317 } 318 } 319 320 static void iowriteb(unsigned long addr, uint8_t data, bool io) 321 { 322 if (io) { 323 outb(data, addr); 324 } else { 325 *(volatile uint8_t *)addr = data; 326 } 327 } 328 329 static bool pci_next(struct test *test, unsigned long addr, bool io) 330 { 331 int i; 332 uint8_t width; 333 334 if (!pci_test.memaddr) { 335 test->func = NULL; 336 return true; 337 } 338 pci_test.test_idx++; 339 iowriteb(addr + offsetof(struct pci_test_dev_hdr, test), 340 pci_test.test_idx, io); 341 width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width), 342 io); 343 switch (width) { 344 case 1: 345 test->func = io ? pci_io_testb : pci_mem_testb; 346 break; 347 case 2: 348 test->func = io ? pci_io_testw : pci_mem_testw; 349 break; 350 case 4: 351 test->func = io ? pci_io_testl : pci_mem_testl; 352 break; 353 default: 354 /* Reset index for purposes of the next test */ 355 pci_test.test_idx = -1; 356 test->func = NULL; 357 return false; 358 } 359 pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data), 360 io); 361 pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr, 362 offset), io); 363 for (i = 0; i < pci_test.offset; ++i) { 364 char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr, 365 name) + i, io); 366 if (!c) { 367 break; 368 } 369 printf("%c",c); 370 } 371 printf(":"); 372 return true; 373 } 374 375 static bool pci_mem_next(struct test *test) 376 { 377 bool ret; 378 ret = pci_next(test, ((unsigned long)pci_test.memaddr), false); 379 if (ret) { 380 pci_test.mem = pci_test.memaddr + pci_test.offset; 381 } 382 return ret; 383 } 384 385 static bool pci_io_next(struct test *test) 386 { 387 bool ret; 388 ret = pci_next(test, ((unsigned long)pci_test.iobar), true); 389 if (ret) { 390 pci_test.ioport = pci_test.iobar + pci_test.offset; 391 } 392 return ret; 393 } 394 395 static int has_tscdeadline(void) 396 { 397 uint32_t lvtt; 398 399 if (cpuid(1).c & (1 << 24)) { 400 lvtt = APIC_LVT_TIMER_TSCDEADLINE | IPI_TEST_VECTOR; 401 apic_write(APIC_LVTT, lvtt); 402 return 1; 403 } else { 404 return 0; 405 } 406 } 407 408 static void tscdeadline_immed(void) 409 { 410 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()); 411 asm volatile("nop"); 412 } 413 414 static void tscdeadline(void) 415 { 416 x = 0; 417 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()+3000); 418 while (x == 0) barrier(); 419 } 420 421 static struct test tests[] = { 422 { cpuid_test, "cpuid", .parallel = 1, }, 423 { vmcall, "vmcall", .parallel = 1, }, 424 #ifdef __x86_64__ 425 { mov_from_cr8, "mov_from_cr8", .parallel = 1, }, 426 { mov_to_cr8, "mov_to_cr8" , .parallel = 1, }, 427 #endif 428 { inl_pmtimer, "inl_from_pmtimer", .parallel = 1, }, 429 { inl_nop_qemu, "inl_from_qemu", .parallel = 1 }, 430 { inl_nop_kernel, "inl_from_kernel", .parallel = 1 }, 431 { outl_elcr_kernel, "outl_to_kernel", .parallel = 1 }, 432 { mov_dr, "mov_dr", .parallel = 1 }, 433 { tscdeadline_immed, "tscdeadline_immed", has_tscdeadline, .parallel = 1, }, 434 { tscdeadline, "tscdeadline", has_tscdeadline, .parallel = 1, }, 435 { self_ipi_sti_nop, "self_ipi_sti_nop", .parallel = 0, }, 436 { self_ipi_sti_hlt, "self_ipi_sti_hlt", .parallel = 0, }, 437 { self_ipi_tpr, "self_ipi_tpr", .parallel = 0, }, 438 { self_ipi_tpr_sti_nop, "self_ipi_tpr_sti_nop", .parallel = 0, }, 439 { self_ipi_tpr_sti_hlt, "self_ipi_tpr_sti_hlt", .parallel = 0, }, 440 { x2apic_self_ipi_sti_nop, "x2apic_self_ipi_sti_nop", is_x2apic, .parallel = 0, }, 441 { x2apic_self_ipi_sti_hlt, "x2apic_self_ipi_sti_hlt", is_x2apic, .parallel = 0, }, 442 { x2apic_self_ipi_tpr, "x2apic_self_ipi_tpr", is_x2apic, .parallel = 0, }, 443 { x2apic_self_ipi_tpr_sti_nop, "x2apic_self_ipi_tpr_sti_nop", is_x2apic, .parallel = 0, }, 444 { x2apic_self_ipi_tpr_sti_hlt, "x2apic_self_ipi_tpr_sti_hlt", is_x2apic, .parallel = 0, }, 445 { ipi, "ipi", is_smp, .parallel = 0, }, 446 { ipi_halt, "ipi_halt", is_smp, .parallel = 0, }, 447 { ple_round_robin, "ple_round_robin", .parallel = 1 }, 448 { wr_kernel_gs_base, "wr_kernel_gs_base", .parallel = 1 }, 449 { wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 }, 450 { rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 }, 451 { NULL, "pci-mem", .parallel = 0, .next = pci_mem_next }, 452 { NULL, "pci-io", .parallel = 0, .next = pci_io_next }, 453 }; 454 455 unsigned iterations; 456 457 static void run_test(void *_func) 458 { 459 int i; 460 void (*func)(void) = _func; 461 462 for (i = 0; i < iterations; ++i) 463 func(); 464 } 465 466 static bool do_test(struct test *test) 467 { 468 int i; 469 unsigned long long t1, t2; 470 void (*func)(void); 471 472 iterations = 32; 473 474 if (test->valid && !test->valid()) { 475 printf("%s (skipped)\n", test->name); 476 return false; 477 } 478 479 if (test->next && !test->next(test)) { 480 return false; 481 } 482 483 func = test->func; 484 if (!func) { 485 printf("%s (skipped)\n", test->name); 486 return false; 487 } 488 489 do { 490 iterations *= 2; 491 t1 = rdtsc(); 492 493 if (!test->parallel) { 494 for (i = 0; i < iterations; ++i) 495 func(); 496 } else { 497 on_cpus(run_test, func); 498 } 499 t2 = rdtsc(); 500 } while ((t2 - t1) < GOAL); 501 printf("%s %d\n", test->name, (int)((t2 - t1) / iterations)); 502 return test->next; 503 } 504 505 static void enable_nx(void *junk) 506 { 507 if (cpuid(0x80000001).d & (1 << 20)) 508 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK); 509 } 510 511 bool test_wanted(struct test *test, char *wanted[], int nwanted) 512 { 513 int i; 514 515 if (!nwanted) 516 return true; 517 518 for (i = 0; i < nwanted; ++i) 519 if (strcmp(wanted[i], test->name) == 0) 520 return true; 521 522 return false; 523 } 524 525 int main(int ac, char **av) 526 { 527 struct fadt_descriptor_rev1 *fadt; 528 int i; 529 unsigned long membar = 0; 530 struct pci_dev pcidev; 531 int ret; 532 533 smp_init(); 534 setup_vm(); 535 handle_irq(IPI_TEST_VECTOR, self_ipi_isr); 536 nr_cpus = cpu_count(); 537 538 irq_enable(); 539 on_cpus(enable_nx, NULL); 540 541 fadt = find_acpi_table_addr(FACP_SIGNATURE); 542 pm_tmr_blk = fadt->pm_tmr_blk; 543 printf("PM timer port is %x\n", pm_tmr_blk); 544 545 ret = pci_find_dev(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_TEST); 546 if (ret != PCIDEVADDR_INVALID) { 547 pci_dev_init(&pcidev, ret); 548 assert(pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_MEM)); 549 assert(!pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_IO)); 550 membar = pcidev.resource[PCI_TESTDEV_BAR_MEM]; 551 pci_test.memaddr = ioremap(membar, PAGE_SIZE); 552 pci_test.iobar = pcidev.resource[PCI_TESTDEV_BAR_IO]; 553 printf("pci-testdev at %#x membar %lx iobar %x\n", 554 pcidev.bdf, membar, pci_test.iobar); 555 } 556 557 for (i = 0; i < ARRAY_SIZE(tests); ++i) 558 if (test_wanted(&tests[i], av + 1, ac - 1)) 559 while (do_test(&tests[i])) {} 560 561 return 0; 562 } 563