1 #include "libcflat.h" 2 #include "apic.h" 3 #include "vm.h" 4 #include "smp.h" 5 #include "desc.h" 6 #include "isr.h" 7 #include "msr.h" 8 #include "atomic.h" 9 #include "fwcfg.h" 10 11 #define MAX_TPR 0xf 12 13 static void test_lapic_existence(void) 14 { 15 u8 version; 16 17 version = (u8)apic_read(APIC_LVR); 18 printf("apic version: %x\n", version); 19 report(version >= 0x10 && version <= 0x15, "apic existence"); 20 } 21 22 #define TSC_DEADLINE_TIMER_VECTOR 0xef 23 #define BROADCAST_VECTOR 0xcf 24 25 static int tdt_count; 26 27 static void tsc_deadline_timer_isr(isr_regs_t *regs) 28 { 29 ++tdt_count; 30 eoi(); 31 } 32 33 static void __test_tsc_deadline_timer(void) 34 { 35 handle_irq(TSC_DEADLINE_TIMER_VECTOR, tsc_deadline_timer_isr); 36 irq_enable(); 37 38 wrmsr(MSR_IA32_TSCDEADLINE, rdmsr(MSR_IA32_TSC)); 39 asm volatile ("nop"); 40 report(tdt_count == 1, "tsc deadline timer"); 41 report(rdmsr(MSR_IA32_TSCDEADLINE) == 0, "tsc deadline timer clearing"); 42 } 43 44 static int enable_tsc_deadline_timer(void) 45 { 46 uint32_t lvtt; 47 48 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 49 lvtt = APIC_LVT_TIMER_TSCDEADLINE | TSC_DEADLINE_TIMER_VECTOR; 50 apic_write(APIC_LVTT, lvtt); 51 return 1; 52 } else { 53 return 0; 54 } 55 } 56 57 static void test_tsc_deadline_timer(void) 58 { 59 if(enable_tsc_deadline_timer()) { 60 __test_tsc_deadline_timer(); 61 } else { 62 report_skip("tsc deadline timer not detected"); 63 } 64 } 65 66 static void do_write_apicbase(void *data) 67 { 68 wrmsr(MSR_IA32_APICBASE, *(u64 *)data); 69 } 70 71 static bool test_write_apicbase_exception(u64 data) 72 { 73 return test_for_exception(GP_VECTOR, do_write_apicbase, &data); 74 } 75 76 static void test_enable_x2apic(void) 77 { 78 u64 orig_apicbase = rdmsr(MSR_IA32_APICBASE); 79 u64 apicbase; 80 81 if (enable_x2apic()) { 82 printf("x2apic enabled\n"); 83 84 apicbase = orig_apicbase & ~(APIC_EN | APIC_EXTD); 85 report(test_write_apicbase_exception(apicbase | APIC_EXTD), 86 "x2apic enabled to invalid state"); 87 report(test_write_apicbase_exception(apicbase | APIC_EN), 88 "x2apic enabled to apic enabled"); 89 90 report(!test_write_apicbase_exception(apicbase | 0), 91 "x2apic enabled to disabled state"); 92 report(test_write_apicbase_exception(apicbase | APIC_EXTD), 93 "disabled to invalid state"); 94 report(test_write_apicbase_exception(apicbase | APIC_EN | APIC_EXTD), 95 "disabled to x2apic enabled"); 96 97 report(!test_write_apicbase_exception(apicbase | APIC_EN), 98 "apic disabled to apic enabled"); 99 report(test_write_apicbase_exception(apicbase | APIC_EXTD), 100 "apic enabled to invalid state"); 101 102 if (orig_apicbase & APIC_EXTD) 103 enable_x2apic(); 104 else 105 reset_apic(); 106 107 /* 108 * Disabling the APIC resets various APIC registers, restore them to 109 * their desired values. 110 */ 111 apic_write(APIC_SPIV, 0x1ff); 112 } else { 113 printf("x2apic not detected\n"); 114 115 report(test_write_apicbase_exception(APIC_EN | APIC_EXTD), 116 "enable unsupported x2apic"); 117 } 118 } 119 120 static void verify_disabled_apic_mmio(void) 121 { 122 volatile u32 *lvr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_LVR); 123 volatile u32 *tpr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_TASKPRI); 124 u32 cr8 = read_cr8(); 125 126 memset((void *)APIC_DEFAULT_PHYS_BASE, 0xff, PAGE_SIZE); 127 report(*lvr == ~0, "*0xfee00030: %x", *lvr); 128 report(read_cr8() == cr8, "CR8: %lx", read_cr8()); 129 write_cr8(cr8 ^ MAX_TPR); 130 report(read_cr8() == (cr8 ^ MAX_TPR), "CR8: %lx", read_cr8()); 131 report(*tpr == ~0, "*0xfee00080: %x", *tpr); 132 write_cr8(cr8); 133 } 134 135 static void test_apic_disable(void) 136 { 137 volatile u32 *lvr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_LVR); 138 volatile u32 *tpr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_TASKPRI); 139 u64 orig_apicbase = rdmsr(MSR_IA32_APICBASE); 140 u32 apic_version = apic_read(APIC_LVR); 141 u32 cr8 = read_cr8(); 142 143 report_prefix_push("apic_disable"); 144 assert_msg(orig_apicbase & APIC_EN, "APIC not enabled."); 145 146 disable_apic(); 147 report(!(rdmsr(MSR_IA32_APICBASE) & APIC_EN), "Local apic disabled"); 148 report(!this_cpu_has(X86_FEATURE_APIC), 149 "CPUID.1H:EDX.APIC[bit 9] is clear"); 150 verify_disabled_apic_mmio(); 151 152 reset_apic(); 153 report((rdmsr(MSR_IA32_APICBASE) & (APIC_EN | APIC_EXTD)) == APIC_EN, 154 "Local apic enabled in xAPIC mode"); 155 report(this_cpu_has(X86_FEATURE_APIC), "CPUID.1H:EDX.APIC[bit 9] is set"); 156 report(*lvr == apic_version, "*0xfee00030: %x", *lvr); 157 report(*tpr == cr8, "*0xfee00080: %x", *tpr); 158 write_cr8(cr8 ^ MAX_TPR); 159 report(*tpr == (cr8 ^ MAX_TPR) << 4, "*0xfee00080: %x", *tpr); 160 write_cr8(cr8); 161 162 if (enable_x2apic()) { 163 apic_write(APIC_SPIV, 0x1ff); 164 report((rdmsr(MSR_IA32_APICBASE) & (APIC_EN | APIC_EXTD)) == (APIC_EN | APIC_EXTD), 165 "Local apic enabled in x2APIC mode"); 166 report(this_cpu_has(X86_FEATURE_APIC), 167 "CPUID.1H:EDX.APIC[bit 9] is set"); 168 verify_disabled_apic_mmio(); 169 if (!(orig_apicbase & APIC_EXTD)) 170 reset_apic(); 171 } 172 report_prefix_pop(); 173 } 174 175 #define ALTERNATE_APIC_BASE 0xfed40000 176 177 static void test_apicbase(void) 178 { 179 u64 orig_apicbase = rdmsr(MSR_IA32_APICBASE); 180 u32 lvr = apic_read(APIC_LVR); 181 u64 value; 182 183 wrmsr(MSR_IA32_APICBASE, orig_apicbase & ~(APIC_EN | APIC_EXTD)); 184 wrmsr(MSR_IA32_APICBASE, ALTERNATE_APIC_BASE | APIC_BSP | APIC_EN); 185 186 report_prefix_push("apicbase"); 187 188 report(*(volatile u32 *)(ALTERNATE_APIC_BASE + APIC_LVR) == lvr, 189 "relocate apic"); 190 191 value = orig_apicbase | (1UL << cpuid_maxphyaddr()); 192 report(test_for_exception(GP_VECTOR, do_write_apicbase, &value), 193 "reserved physaddr bits"); 194 195 value = orig_apicbase | 1; 196 report(test_for_exception(GP_VECTOR, do_write_apicbase, &value), 197 "reserved low bits"); 198 199 wrmsr(MSR_IA32_APICBASE, orig_apicbase); 200 apic_write(APIC_SPIV, 0x1ff); 201 202 report_prefix_pop(); 203 } 204 205 static void do_write_apic_id(void *id) 206 { 207 apic_write(APIC_ID, *(u32 *)id); 208 } 209 210 static void __test_apic_id(void * unused) 211 { 212 u32 id, newid; 213 u8 initial_xapic_id = cpuid(1).b >> 24; 214 u32 initial_x2apic_id = cpuid(0xb).d; 215 bool x2apic_mode = rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 216 217 if (x2apic_mode) 218 reset_apic(); 219 220 id = apic_id(); 221 report(initial_xapic_id == id, "xapic id matches cpuid"); 222 223 newid = (id + 1) << 24; 224 report(!test_for_exception(GP_VECTOR, do_write_apic_id, &newid) && 225 (id == apic_id() || id + 1 == apic_id()), 226 "writeable xapic id"); 227 228 if (!enable_x2apic()) 229 goto out; 230 231 report(test_for_exception(GP_VECTOR, do_write_apic_id, &newid), 232 "non-writeable x2apic id"); 233 report(initial_xapic_id == (apic_id() & 0xff), "sane x2apic id"); 234 235 /* old QEMUs do not set initial x2APIC ID */ 236 report(initial_xapic_id == (initial_x2apic_id & 0xff) && 237 initial_x2apic_id == apic_id(), 238 "x2apic id matches cpuid"); 239 240 out: 241 reset_apic(); 242 243 report(initial_xapic_id == apic_id(), "correct xapic id after reset"); 244 245 /* old KVMs do not reset xAPIC ID */ 246 if (id != apic_id()) 247 apic_write(APIC_ID, id << 24); 248 249 if (x2apic_mode) 250 enable_x2apic(); 251 } 252 253 static void test_apic_id(void) 254 { 255 if (cpu_count() < 2) 256 return; 257 258 on_cpu(1, __test_apic_id, NULL); 259 } 260 261 static int ipi_count; 262 263 static void self_ipi_isr(isr_regs_t *regs) 264 { 265 ++ipi_count; 266 eoi(); 267 } 268 269 static void test_self_ipi(void) 270 { 271 u64 start = rdtsc(); 272 int vec = 0xf1; 273 274 handle_irq(vec, self_ipi_isr); 275 irq_enable(); 276 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | vec, 277 id_map[0]); 278 279 do { 280 pause(); 281 } while (rdtsc() - start < 1000000000 && ipi_count == 0); 282 283 report(ipi_count == 1, "self ipi"); 284 } 285 286 volatile int nmi_counter_private, nmi_counter, nmi_hlt_counter, sti_loop_active; 287 288 static void sti_nop(char *p) 289 { 290 asm volatile ( 291 ".globl post_sti \n\t" 292 "sti \n" 293 /* 294 * vmx won't exit on external interrupt if blocked-by-sti, 295 * so give it a reason to exit by accessing an unmapped page. 296 */ 297 "post_sti: testb $0, %0 \n\t" 298 "nop \n\t" 299 "cli" 300 : : "m"(*p) 301 ); 302 nmi_counter = nmi_counter_private; 303 } 304 305 static void sti_loop(void *ignore) 306 { 307 unsigned k = 0; 308 309 while (sti_loop_active) { 310 sti_nop((char *)(ulong)((k++ * 4096) % (128 * 1024 * 1024))); 311 } 312 } 313 314 static void nmi_handler(isr_regs_t *regs) 315 { 316 extern void post_sti(void); 317 ++nmi_counter_private; 318 nmi_hlt_counter += regs->rip == (ulong)post_sti; 319 } 320 321 static void update_cr3(void *cr3) 322 { 323 write_cr3((ulong)cr3); 324 } 325 326 static void test_sti_nmi(void) 327 { 328 unsigned old_counter; 329 330 if (cpu_count() < 2) { 331 return; 332 } 333 334 handle_irq(2, nmi_handler); 335 on_cpu(1, update_cr3, (void *)read_cr3()); 336 337 sti_loop_active = 1; 338 on_cpu_async(1, sti_loop, 0); 339 while (nmi_counter < 30000) { 340 old_counter = nmi_counter; 341 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[1]); 342 while (nmi_counter == old_counter) { 343 ; 344 } 345 } 346 sti_loop_active = 0; 347 report(nmi_hlt_counter == 0, "nmi-after-sti"); 348 } 349 350 static volatile bool nmi_done, nmi_flushed; 351 static volatile int nmi_received; 352 static volatile int cpu0_nmi_ctr1, cpu1_nmi_ctr1; 353 static volatile int cpu0_nmi_ctr2, cpu1_nmi_ctr2; 354 355 static void multiple_nmi_handler(isr_regs_t *regs) 356 { 357 ++nmi_received; 358 } 359 360 static void kick_me_nmi(void *blah) 361 { 362 while (!nmi_done) { 363 ++cpu1_nmi_ctr1; 364 while (cpu1_nmi_ctr1 != cpu0_nmi_ctr1 && !nmi_done) { 365 pause(); 366 } 367 if (nmi_done) { 368 return; 369 } 370 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]); 371 /* make sure the NMI has arrived by sending an IPI after it */ 372 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_FIXED | APIC_INT_ASSERT 373 | 0x44, id_map[0]); 374 ++cpu1_nmi_ctr2; 375 while (cpu1_nmi_ctr2 != cpu0_nmi_ctr2 && !nmi_done) { 376 pause(); 377 } 378 } 379 } 380 381 static void flush_nmi(isr_regs_t *regs) 382 { 383 nmi_flushed = true; 384 apic_write(APIC_EOI, 0); 385 } 386 387 static void test_multiple_nmi(void) 388 { 389 int i; 390 bool ok = true; 391 392 if (cpu_count() < 2) { 393 return; 394 } 395 396 sti(); 397 handle_irq(2, multiple_nmi_handler); 398 handle_irq(0x44, flush_nmi); 399 on_cpu_async(1, kick_me_nmi, 0); 400 for (i = 0; i < 1000000; ++i) { 401 nmi_flushed = false; 402 nmi_received = 0; 403 ++cpu0_nmi_ctr1; 404 while (cpu1_nmi_ctr1 != cpu0_nmi_ctr1) { 405 pause(); 406 } 407 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]); 408 while (!nmi_flushed) { 409 pause(); 410 } 411 if (nmi_received != 2) { 412 ok = false; 413 break; 414 } 415 ++cpu0_nmi_ctr2; 416 while (cpu1_nmi_ctr2 != cpu0_nmi_ctr2) { 417 pause(); 418 } 419 } 420 nmi_done = true; 421 report(ok, "multiple nmi"); 422 } 423 424 static void pending_nmi_handler(isr_regs_t *regs) 425 { 426 int i; 427 428 if (++nmi_received == 1) { 429 for (i = 0; i < 10; ++i) 430 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI, 0); 431 } 432 } 433 434 static void test_pending_nmi(void) 435 { 436 int i; 437 438 handle_irq(2, pending_nmi_handler); 439 for (i = 0; i < 100000; ++i) { 440 nmi_received = 0; 441 442 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI, 0); 443 while (nmi_received < 2) 444 pause(); 445 446 if (nmi_received != 2) 447 break; 448 } 449 report(nmi_received == 2, "pending nmi"); 450 } 451 452 static volatile int lvtt_counter = 0; 453 454 static void lvtt_handler(isr_regs_t *regs) 455 { 456 lvtt_counter++; 457 eoi(); 458 } 459 460 static void test_apic_timer_one_shot(void) 461 { 462 uint64_t tsc1, tsc2; 463 static const uint32_t interval = 0x10000; 464 465 #define APIC_LVT_TIMER_VECTOR (0xee) 466 467 handle_irq(APIC_LVT_TIMER_VECTOR, lvtt_handler); 468 irq_enable(); 469 470 /* One shot mode */ 471 apic_write(APIC_LVTT, APIC_LVT_TIMER_ONESHOT | 472 APIC_LVT_TIMER_VECTOR); 473 /* Divider == 1 */ 474 apic_write(APIC_TDCR, 0x0000000b); 475 476 tsc1 = rdtsc(); 477 /* Set "Initial Counter Register", which starts the timer */ 478 apic_write(APIC_TMICT, interval); 479 while (!lvtt_counter); 480 tsc2 = rdtsc(); 481 482 /* 483 * For LVT Timer clock, SDM vol 3 10.5.4 says it should be 484 * derived from processor's bus clock (IIUC which is the same 485 * as TSC), however QEMU seems to be using nanosecond. In all 486 * cases, the following should satisfy on all modern 487 * processors. 488 */ 489 report((lvtt_counter == 1) && (tsc2 - tsc1 >= interval), 490 "APIC LVT timer one shot"); 491 } 492 493 static atomic_t broadcast_counter; 494 495 static void broadcast_handler(isr_regs_t *regs) 496 { 497 atomic_inc(&broadcast_counter); 498 eoi(); 499 } 500 501 static bool broadcast_received(unsigned ncpus) 502 { 503 unsigned counter; 504 u64 start = rdtsc(); 505 506 do { 507 counter = atomic_read(&broadcast_counter); 508 if (counter >= ncpus) 509 break; 510 pause(); 511 } while (rdtsc() - start < 1000000000); 512 513 atomic_set(&broadcast_counter, 0); 514 515 return counter == ncpus; 516 } 517 518 static void test_physical_broadcast(void) 519 { 520 unsigned ncpus = cpu_count(); 521 unsigned long cr3 = read_cr3(); 522 u32 broadcast_address = enable_x2apic() ? 0xffffffff : 0xff; 523 524 handle_irq(BROADCAST_VECTOR, broadcast_handler); 525 for (int c = 1; c < ncpus; c++) 526 on_cpu(c, update_cr3, (void *)cr3); 527 528 printf("starting broadcast (%s)\n", enable_x2apic() ? "x2apic" : "xapic"); 529 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_FIXED | APIC_INT_ASSERT | 530 BROADCAST_VECTOR, broadcast_address); 531 report(broadcast_received(ncpus), "APIC physical broadcast address"); 532 533 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_FIXED | APIC_INT_ASSERT | 534 BROADCAST_VECTOR | APIC_DEST_ALLINC, 0); 535 report(broadcast_received(ncpus), "APIC physical broadcast shorthand"); 536 } 537 538 static void wait_until_tmcct_common(uint32_t initial_count, bool stop_when_half, bool should_wrap_around) 539 { 540 uint32_t tmcct = apic_read(APIC_TMCCT); 541 542 if (tmcct) { 543 while (tmcct > (initial_count / 2)) 544 tmcct = apic_read(APIC_TMCCT); 545 546 if ( stop_when_half ) 547 return; 548 549 /* Wait until the counter reach 0 or wrap-around */ 550 while ( tmcct <= (initial_count / 2) && tmcct > 0 ) 551 tmcct = apic_read(APIC_TMCCT); 552 553 /* Wait specifically for wrap around to skip 0 TMCCR if we were asked to */ 554 while (should_wrap_around && !tmcct) 555 tmcct = apic_read(APIC_TMCCT); 556 } 557 } 558 559 static void wait_until_tmcct_is_zero(uint32_t initial_count, bool stop_when_half) 560 { 561 return wait_until_tmcct_common(initial_count, stop_when_half, false); 562 } 563 564 static void wait_until_tmcct_wrap_around(uint32_t initial_count, bool stop_when_half) 565 { 566 return wait_until_tmcct_common(initial_count, stop_when_half, true); 567 } 568 569 static inline void apic_change_mode(unsigned long new_mode) 570 { 571 uint32_t lvtt; 572 573 lvtt = apic_read(APIC_LVTT); 574 apic_write(APIC_LVTT, (lvtt & ~APIC_LVT_TIMER_MASK) | new_mode); 575 } 576 577 static void test_apic_change_mode(void) 578 { 579 uint32_t tmict = 0x999999; 580 581 printf("starting apic change mode\n"); 582 583 apic_write(APIC_TMICT, tmict); 584 585 apic_change_mode(APIC_LVT_TIMER_PERIODIC); 586 587 report(apic_read(APIC_TMICT) == tmict, "TMICT value reset"); 588 589 /* Testing one-shot */ 590 apic_change_mode(APIC_LVT_TIMER_ONESHOT); 591 apic_write(APIC_TMICT, tmict); 592 report(apic_read(APIC_TMCCT), "TMCCT should have a non-zero value"); 593 594 wait_until_tmcct_is_zero(tmict, false); 595 report(!apic_read(APIC_TMCCT), "TMCCT should have reached 0"); 596 597 /* 598 * Write TMICT before changing mode from one-shot to periodic TMCCT should 599 * be reset to TMICT periodicly 600 */ 601 apic_write(APIC_TMICT, tmict); 602 wait_until_tmcct_is_zero(tmict, true); 603 apic_change_mode(APIC_LVT_TIMER_PERIODIC); 604 report(apic_read(APIC_TMCCT), "TMCCT should have a non-zero value"); 605 606 /* 607 * After the change of mode, the counter should not be reset and continue 608 * counting down from where it was 609 */ 610 report(apic_read(APIC_TMCCT) < (tmict / 2), 611 "TMCCT should not be reset to TMICT value"); 612 /* 613 * Specifically wait for timer wrap around and skip 0. 614 * Under KVM lapic there is a possibility that a small amount of consecutive 615 * TMCCR reads return 0 while hrtimer is reset in an async callback 616 */ 617 wait_until_tmcct_wrap_around(tmict, false); 618 report(apic_read(APIC_TMCCT) > (tmict / 2), 619 "TMCCT should be reset to the initial-count"); 620 621 wait_until_tmcct_is_zero(tmict, true); 622 /* 623 * Keep the same TMICT and change timer mode to one-shot 624 * TMCCT should be > 0 and count-down to 0 625 */ 626 apic_change_mode(APIC_LVT_TIMER_ONESHOT); 627 report(apic_read(APIC_TMCCT) < (tmict / 2), 628 "TMCCT should not be reset to init"); 629 wait_until_tmcct_is_zero(tmict, false); 630 report(!apic_read(APIC_TMCCT), "TMCCT should have reach zero"); 631 632 /* now tmcct == 0 and tmict != 0 */ 633 apic_change_mode(APIC_LVT_TIMER_PERIODIC); 634 report(!apic_read(APIC_TMCCT), "TMCCT should stay at zero"); 635 } 636 637 #define KVM_HC_SEND_IPI 10 638 639 static void test_pv_ipi(void) 640 { 641 int ret; 642 unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0; 643 644 asm volatile("vmcall" : "=a"(ret) :"a"(KVM_HC_SEND_IPI), "b"(a0), "c"(a1), "d"(a2), "S"(a3)); 645 report(!ret, "PV IPIs testing"); 646 } 647 648 int main(void) 649 { 650 setup_vm(); 651 652 test_lapic_existence(); 653 654 mask_pic_interrupts(); 655 test_apic_id(); 656 test_apic_disable(); 657 test_enable_x2apic(); 658 test_apicbase(); 659 660 test_self_ipi(); 661 test_physical_broadcast(); 662 if (test_device_enabled()) 663 test_pv_ipi(); 664 665 test_sti_nmi(); 666 test_multiple_nmi(); 667 test_pending_nmi(); 668 669 test_apic_timer_one_shot(); 670 test_apic_change_mode(); 671 test_tsc_deadline_timer(); 672 673 return report_summary(); 674 } 675