1 #include "libcflat.h" 2 #include "smp.h" 3 #include "processor.h" 4 #include "atomic.h" 5 #include "pci.h" 6 #include "x86/vm.h" 7 #include "x86/desc.h" 8 #include "x86/acpi.h" 9 #include "x86/apic.h" 10 #include "x86/isr.h" 11 12 #define IPI_TEST_VECTOR 0xb0 13 14 struct test { 15 void (*func)(void); 16 const char *name; 17 int (*valid)(void); 18 int parallel; 19 bool (*next)(struct test *); 20 }; 21 22 #define GOAL (1ull << 30) 23 24 static int nr_cpus; 25 26 static void cpuid_test(void) 27 { 28 asm volatile ("push %%"R "bx; cpuid; pop %%"R "bx" 29 : : : "eax", "ecx", "edx"); 30 } 31 32 static void vmcall(void) 33 { 34 unsigned long a = 0, b, c, d; 35 36 asm volatile ("vmcall" : "+a"(a), "=b"(b), "=c"(c), "=d"(d)); 37 } 38 39 #define MSR_TSC_ADJUST 0x3b 40 #define MSR_EFER 0xc0000080 41 #define EFER_NX_MASK (1ull << 11) 42 43 #ifdef __x86_64__ 44 static void mov_from_cr8(void) 45 { 46 unsigned long cr8; 47 48 asm volatile ("mov %%cr8, %0" : "=r"(cr8)); 49 } 50 51 static void mov_to_cr8(void) 52 { 53 unsigned long cr8 = 0; 54 55 asm volatile ("mov %0, %%cr8" : : "r"(cr8)); 56 } 57 #endif 58 59 static int is_smp(void) 60 { 61 return cpu_count() > 1; 62 } 63 64 static void nop(void *junk) 65 { 66 } 67 68 volatile int x = 0; 69 70 static void self_ipi_isr(isr_regs_t *regs) 71 { 72 x++; 73 eoi(); 74 } 75 76 static void x2apic_self_ipi(int vec) 77 { 78 wrmsr(0x83f, vec); 79 } 80 81 static void apic_self_ipi(int vec) 82 { 83 apic_icr_write(APIC_INT_ASSERT | APIC_DEST_SELF | APIC_DEST_PHYSICAL | 84 APIC_DM_FIXED | IPI_TEST_VECTOR, vec); 85 } 86 87 static void self_ipi_sti_nop(void) 88 { 89 x = 0; 90 irq_disable(); 91 apic_self_ipi(IPI_TEST_VECTOR); 92 asm volatile("sti; nop"); 93 if (x != 1) printf("%d", x); 94 } 95 96 static void self_ipi_sti_hlt(void) 97 { 98 x = 0; 99 irq_disable(); 100 apic_self_ipi(IPI_TEST_VECTOR); 101 asm volatile("sti; hlt"); 102 if (x != 1) printf("%d", x); 103 } 104 105 static void self_ipi_tpr(void) 106 { 107 x = 0; 108 apic_set_tpr(0x0f); 109 apic_self_ipi(IPI_TEST_VECTOR); 110 apic_set_tpr(0x00); 111 asm volatile("nop"); 112 if (x != 1) printf("%d", x); 113 } 114 115 static void self_ipi_tpr_sti_nop(void) 116 { 117 x = 0; 118 irq_disable(); 119 apic_set_tpr(0x0f); 120 apic_self_ipi(IPI_TEST_VECTOR); 121 apic_set_tpr(0x00); 122 asm volatile("sti; nop"); 123 if (x != 1) printf("%d", x); 124 } 125 126 static void self_ipi_tpr_sti_hlt(void) 127 { 128 x = 0; 129 irq_disable(); 130 apic_set_tpr(0x0f); 131 apic_self_ipi(IPI_TEST_VECTOR); 132 apic_set_tpr(0x00); 133 asm volatile("sti; hlt"); 134 if (x != 1) printf("%d", x); 135 } 136 137 static int is_x2apic(void) 138 { 139 return rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 140 } 141 142 static void x2apic_self_ipi_sti_nop(void) 143 { 144 irq_disable(); 145 x2apic_self_ipi(IPI_TEST_VECTOR); 146 asm volatile("sti; nop"); 147 } 148 149 static void x2apic_self_ipi_sti_hlt(void) 150 { 151 irq_disable(); 152 x2apic_self_ipi(IPI_TEST_VECTOR); 153 asm volatile("sti; hlt"); 154 } 155 156 static void x2apic_self_ipi_tpr(void) 157 { 158 apic_set_tpr(0x0f); 159 x2apic_self_ipi(IPI_TEST_VECTOR); 160 apic_set_tpr(0x00); 161 asm volatile("nop"); 162 } 163 164 static void x2apic_self_ipi_tpr_sti_nop(void) 165 { 166 irq_disable(); 167 apic_set_tpr(0x0f); 168 x2apic_self_ipi(IPI_TEST_VECTOR); 169 apic_set_tpr(0x00); 170 asm volatile("sti; nop"); 171 } 172 173 static void x2apic_self_ipi_tpr_sti_hlt(void) 174 { 175 irq_disable(); 176 apic_set_tpr(0x0f); 177 x2apic_self_ipi(IPI_TEST_VECTOR); 178 apic_set_tpr(0x00); 179 asm volatile("sti; hlt"); 180 } 181 182 static void ipi(void) 183 { 184 on_cpu(1, nop, 0); 185 } 186 187 static void ipi_halt(void) 188 { 189 unsigned long long t; 190 191 on_cpu(1, nop, 0); 192 t = rdtsc() + 2000; 193 while (rdtsc() < t) 194 ; 195 } 196 197 int pm_tmr_blk; 198 static void inl_pmtimer(void) 199 { 200 inl(pm_tmr_blk); 201 } 202 203 static void inl_nop_qemu(void) 204 { 205 inl(0x1234); 206 } 207 208 static void inl_nop_kernel(void) 209 { 210 inb(0x4d0); 211 } 212 213 static void outl_elcr_kernel(void) 214 { 215 outb(0, 0x4d0); 216 } 217 218 static void mov_dr(void) 219 { 220 asm volatile("mov %0, %%dr7" : : "r" (0x400L)); 221 } 222 223 static void ple_round_robin(void) 224 { 225 struct counter { 226 volatile int n1; 227 int n2; 228 } __attribute__((aligned(64))); 229 static struct counter counters[64] = { { -1, 0 } }; 230 int me = smp_id(); 231 int you; 232 volatile struct counter *p = &counters[me]; 233 234 while (p->n1 == p->n2) 235 asm volatile ("pause"); 236 237 p->n2 = p->n1; 238 you = me + 1; 239 if (you == nr_cpus) 240 you = 0; 241 ++counters[you].n1; 242 } 243 244 static void rd_tsc_adjust_msr(void) 245 { 246 rdmsr(MSR_TSC_ADJUST); 247 } 248 249 static void wr_tsc_adjust_msr(void) 250 { 251 wrmsr(MSR_TSC_ADJUST, 0x0); 252 } 253 254 static struct pci_test { 255 unsigned iobar; 256 unsigned ioport; 257 volatile void *memaddr; 258 volatile void *mem; 259 int test_idx; 260 uint32_t data; 261 uint32_t offset; 262 } pci_test = { 263 .test_idx = -1 264 }; 265 266 static void pci_mem_testb(void) 267 { 268 *(volatile uint8_t *)pci_test.mem = pci_test.data; 269 } 270 271 static void pci_mem_testw(void) 272 { 273 *(volatile uint16_t *)pci_test.mem = pci_test.data; 274 } 275 276 static void pci_mem_testl(void) 277 { 278 *(volatile uint32_t *)pci_test.mem = pci_test.data; 279 } 280 281 static void pci_io_testb(void) 282 { 283 outb(pci_test.data, pci_test.ioport); 284 } 285 286 static void pci_io_testw(void) 287 { 288 outw(pci_test.data, pci_test.ioport); 289 } 290 291 static void pci_io_testl(void) 292 { 293 outl(pci_test.data, pci_test.ioport); 294 } 295 296 static uint8_t ioreadb(unsigned long addr, bool io) 297 { 298 if (io) { 299 return inb(addr); 300 } else { 301 return *(volatile uint8_t *)addr; 302 } 303 } 304 305 static uint32_t ioreadl(unsigned long addr, bool io) 306 { 307 /* Note: assumes little endian */ 308 if (io) { 309 return inl(addr); 310 } else { 311 return *(volatile uint32_t *)addr; 312 } 313 } 314 315 static void iowriteb(unsigned long addr, uint8_t data, bool io) 316 { 317 if (io) { 318 outb(data, addr); 319 } else { 320 *(volatile uint8_t *)addr = data; 321 } 322 } 323 324 static bool pci_next(struct test *test, unsigned long addr, bool io) 325 { 326 int i; 327 uint8_t width; 328 329 if (!pci_test.memaddr) { 330 test->func = NULL; 331 return true; 332 } 333 pci_test.test_idx++; 334 iowriteb(addr + offsetof(struct pci_test_dev_hdr, test), 335 pci_test.test_idx, io); 336 width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width), 337 io); 338 switch (width) { 339 case 1: 340 test->func = io ? pci_io_testb : pci_mem_testb; 341 break; 342 case 2: 343 test->func = io ? pci_io_testw : pci_mem_testw; 344 break; 345 case 4: 346 test->func = io ? pci_io_testl : pci_mem_testl; 347 break; 348 default: 349 /* Reset index for purposes of the next test */ 350 pci_test.test_idx = -1; 351 test->func = NULL; 352 return false; 353 } 354 pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data), 355 io); 356 pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr, 357 offset), io); 358 for (i = 0; i < pci_test.offset; ++i) { 359 char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr, 360 name) + i, io); 361 if (!c) { 362 break; 363 } 364 printf("%c",c); 365 } 366 printf(":"); 367 return true; 368 } 369 370 static bool pci_mem_next(struct test *test) 371 { 372 bool ret; 373 ret = pci_next(test, ((unsigned long)pci_test.memaddr), false); 374 if (ret) { 375 pci_test.mem = pci_test.memaddr + pci_test.offset; 376 } 377 return ret; 378 } 379 380 static bool pci_io_next(struct test *test) 381 { 382 bool ret; 383 ret = pci_next(test, ((unsigned long)pci_test.iobar), true); 384 if (ret) { 385 pci_test.ioport = pci_test.iobar + pci_test.offset; 386 } 387 return ret; 388 } 389 390 static int has_tscdeadline(void) 391 { 392 uint32_t lvtt; 393 394 if (cpuid(1).c & (1 << 24)) { 395 lvtt = APIC_LVT_TIMER_TSCDEADLINE | IPI_TEST_VECTOR; 396 apic_write(APIC_LVTT, lvtt); 397 return 1; 398 } else { 399 return 0; 400 } 401 } 402 403 static void tscdeadline_immed(void) 404 { 405 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()); 406 asm volatile("nop"); 407 } 408 409 static void tscdeadline(void) 410 { 411 x = 0; 412 wrmsr(MSR_IA32_TSCDEADLINE, rdtsc()+3000); 413 while (x == 0) barrier(); 414 } 415 416 static struct test tests[] = { 417 { cpuid_test, "cpuid", .parallel = 1, }, 418 { vmcall, "vmcall", .parallel = 1, }, 419 #ifdef __x86_64__ 420 { mov_from_cr8, "mov_from_cr8", .parallel = 1, }, 421 { mov_to_cr8, "mov_to_cr8" , .parallel = 1, }, 422 #endif 423 { inl_pmtimer, "inl_from_pmtimer", .parallel = 1, }, 424 { inl_nop_qemu, "inl_from_qemu", .parallel = 1 }, 425 { inl_nop_kernel, "inl_from_kernel", .parallel = 1 }, 426 { outl_elcr_kernel, "outl_to_kernel", .parallel = 1 }, 427 { mov_dr, "mov_dr", .parallel = 1 }, 428 { tscdeadline_immed, "tscdeadline_immed", has_tscdeadline, .parallel = 1, }, 429 { tscdeadline, "tscdeadline", has_tscdeadline, .parallel = 1, }, 430 { self_ipi_sti_nop, "self_ipi_sti_nop", .parallel = 0, }, 431 { self_ipi_sti_hlt, "self_ipi_sti_hlt", .parallel = 0, }, 432 { self_ipi_tpr, "self_ipi_tpr", .parallel = 0, }, 433 { self_ipi_tpr_sti_nop, "self_ipi_tpr_sti_nop", .parallel = 0, }, 434 { self_ipi_tpr_sti_hlt, "self_ipi_tpr_sti_hlt", .parallel = 0, }, 435 { x2apic_self_ipi_sti_nop, "x2apic_self_ipi_sti_nop", is_x2apic, .parallel = 0, }, 436 { x2apic_self_ipi_sti_hlt, "x2apic_self_ipi_sti_hlt", is_x2apic, .parallel = 0, }, 437 { x2apic_self_ipi_tpr, "x2apic_self_ipi_tpr", is_x2apic, .parallel = 0, }, 438 { x2apic_self_ipi_tpr_sti_nop, "x2apic_self_ipi_tpr_sti_nop", is_x2apic, .parallel = 0, }, 439 { x2apic_self_ipi_tpr_sti_hlt, "x2apic_self_ipi_tpr_sti_hlt", is_x2apic, .parallel = 0, }, 440 { ipi, "ipi", is_smp, .parallel = 0, }, 441 { ipi_halt, "ipi+halt", is_smp, .parallel = 0, }, 442 { ple_round_robin, "ple-round-robin", .parallel = 1 }, 443 { wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 }, 444 { rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 }, 445 { NULL, "pci-mem", .parallel = 0, .next = pci_mem_next }, 446 { NULL, "pci-io", .parallel = 0, .next = pci_io_next }, 447 }; 448 449 unsigned iterations; 450 451 static void run_test(void *_func) 452 { 453 int i; 454 void (*func)(void) = _func; 455 456 for (i = 0; i < iterations; ++i) 457 func(); 458 } 459 460 static bool do_test(struct test *test) 461 { 462 int i; 463 unsigned long long t1, t2; 464 void (*func)(void); 465 466 iterations = 32; 467 468 if (test->valid && !test->valid()) { 469 printf("%s (skipped)\n", test->name); 470 return false; 471 } 472 473 if (test->next && !test->next(test)) { 474 return false; 475 } 476 477 func = test->func; 478 if (!func) { 479 printf("%s (skipped)\n", test->name); 480 return false; 481 } 482 483 do { 484 iterations *= 2; 485 t1 = rdtsc(); 486 487 if (!test->parallel) { 488 for (i = 0; i < iterations; ++i) 489 func(); 490 } else { 491 on_cpus(run_test, func); 492 } 493 t2 = rdtsc(); 494 } while ((t2 - t1) < GOAL); 495 printf("%s %d\n", test->name, (int)((t2 - t1) / iterations)); 496 return test->next; 497 } 498 499 static void enable_nx(void *junk) 500 { 501 if (cpuid(0x80000001).d & (1 << 20)) 502 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK); 503 } 504 505 bool test_wanted(struct test *test, char *wanted[], int nwanted) 506 { 507 int i; 508 509 if (!nwanted) 510 return true; 511 512 for (i = 0; i < nwanted; ++i) 513 if (strcmp(wanted[i], test->name) == 0) 514 return true; 515 516 return false; 517 } 518 519 int main(int ac, char **av) 520 { 521 struct fadt_descriptor_rev1 *fadt; 522 int i; 523 unsigned long membar = 0; 524 struct pci_dev pcidev; 525 int ret; 526 527 smp_init(); 528 setup_vm(); 529 handle_irq(IPI_TEST_VECTOR, self_ipi_isr); 530 nr_cpus = cpu_count(); 531 532 irq_enable(); 533 on_cpus(enable_nx, NULL); 534 535 fadt = find_acpi_table_addr(FACP_SIGNATURE); 536 pm_tmr_blk = fadt->pm_tmr_blk; 537 printf("PM timer port is %x\n", pm_tmr_blk); 538 539 ret = pci_find_dev(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_TEST); 540 if (ret != PCIDEVADDR_INVALID) { 541 pci_dev_init(&pcidev, ret); 542 assert(pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_MEM)); 543 assert(!pci_bar_is_memory(&pcidev, PCI_TESTDEV_BAR_IO)); 544 membar = pcidev.resource[PCI_TESTDEV_BAR_MEM]; 545 pci_test.memaddr = ioremap(membar, PAGE_SIZE); 546 pci_test.iobar = pcidev.resource[PCI_TESTDEV_BAR_IO]; 547 printf("pci-testdev at %#x membar %lx iobar %x\n", 548 pcidev.bdf, membar, pci_test.iobar); 549 } 550 551 for (i = 0; i < ARRAY_SIZE(tests); ++i) 552 if (test_wanted(&tests[i], av + 1, ac - 1)) 553 while (do_test(&tests[i])) {} 554 555 return 0; 556 } 557