1 #include "libcflat.h" 2 #include "apic.h" 3 #include "vm.h" 4 #include "smp.h" 5 #include "desc.h" 6 #include "isr.h" 7 #include "msr.h" 8 #include "atomic.h" 9 #include "fwcfg.h" 10 11 #define MAX_TPR 0xf 12 13 static void test_lapic_existence(void) 14 { 15 u8 version; 16 17 version = (u8)apic_read(APIC_LVR); 18 printf("apic version: %x\n", version); 19 report(version >= 0x10 && version <= 0x15, "apic existence"); 20 } 21 22 #define TSC_DEADLINE_TIMER_VECTOR 0xef 23 #define BROADCAST_VECTOR 0xcf 24 25 static int tdt_count; 26 27 static void tsc_deadline_timer_isr(isr_regs_t *regs) 28 { 29 ++tdt_count; 30 eoi(); 31 } 32 33 static void __test_tsc_deadline_timer(void) 34 { 35 handle_irq(TSC_DEADLINE_TIMER_VECTOR, tsc_deadline_timer_isr); 36 irq_enable(); 37 38 wrmsr(MSR_IA32_TSCDEADLINE, rdmsr(MSR_IA32_TSC)); 39 asm volatile ("nop"); 40 report(tdt_count == 1, "tsc deadline timer"); 41 report(rdmsr(MSR_IA32_TSCDEADLINE) == 0, "tsc deadline timer clearing"); 42 } 43 44 static int enable_tsc_deadline_timer(void) 45 { 46 uint32_t lvtt; 47 48 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 49 lvtt = APIC_LVT_TIMER_TSCDEADLINE | TSC_DEADLINE_TIMER_VECTOR; 50 apic_write(APIC_LVTT, lvtt); 51 return 1; 52 } else { 53 return 0; 54 } 55 } 56 57 static void test_tsc_deadline_timer(void) 58 { 59 if(enable_tsc_deadline_timer()) { 60 __test_tsc_deadline_timer(); 61 } else { 62 report_skip("tsc deadline timer not detected"); 63 } 64 } 65 66 static void do_write_apicbase(void *data) 67 { 68 wrmsr(MSR_IA32_APICBASE, *(u64 *)data); 69 } 70 71 static bool test_write_apicbase_exception(u64 data) 72 { 73 return test_for_exception(GP_VECTOR, do_write_apicbase, &data); 74 } 75 76 static void test_enable_x2apic(void) 77 { 78 u64 orig_apicbase = rdmsr(MSR_IA32_APICBASE); 79 u64 apicbase; 80 81 if (enable_x2apic()) { 82 printf("x2apic enabled\n"); 83 84 apicbase = orig_apicbase & ~(APIC_EN | APIC_EXTD); 85 report(test_write_apicbase_exception(apicbase | APIC_EXTD), 86 "x2apic enabled to invalid state"); 87 report(test_write_apicbase_exception(apicbase | APIC_EN), 88 "x2apic enabled to apic enabled"); 89 90 report(!test_write_apicbase_exception(apicbase | 0), 91 "x2apic enabled to disabled state"); 92 report(test_write_apicbase_exception(apicbase | APIC_EXTD), 93 "disabled to invalid state"); 94 report(test_write_apicbase_exception(apicbase | APIC_EN | APIC_EXTD), 95 "disabled to x2apic enabled"); 96 97 report(!test_write_apicbase_exception(apicbase | APIC_EN), 98 "apic disabled to apic enabled"); 99 report(test_write_apicbase_exception(apicbase | APIC_EXTD), 100 "apic enabled to invalid state"); 101 102 if (orig_apicbase & APIC_EXTD) 103 enable_x2apic(); 104 else 105 reset_apic(); 106 107 /* 108 * Disabling the APIC resets various APIC registers, restore them to 109 * their desired values. 110 */ 111 apic_write(APIC_SPIV, 0x1ff); 112 } else { 113 printf("x2apic not detected\n"); 114 115 report(test_write_apicbase_exception(APIC_EN | APIC_EXTD), 116 "enable unsupported x2apic"); 117 } 118 } 119 120 static void verify_disabled_apic_mmio(void) 121 { 122 volatile u32 *lvr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_LVR); 123 volatile u32 *tpr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_TASKPRI); 124 u32 cr8 = read_cr8(); 125 126 memset((void *)APIC_DEFAULT_PHYS_BASE, 0xff, PAGE_SIZE); 127 report(*lvr == ~0, "*0xfee00030: %x", *lvr); 128 report(read_cr8() == cr8, "CR8: %lx", read_cr8()); 129 write_cr8(cr8 ^ MAX_TPR); 130 report(read_cr8() == (cr8 ^ MAX_TPR), "CR8: %lx", read_cr8()); 131 report(*tpr == ~0, "*0xfee00080: %x", *tpr); 132 write_cr8(cr8); 133 } 134 135 static void test_apic_disable(void) 136 { 137 volatile u32 *lvr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_LVR); 138 volatile u32 *tpr = (volatile u32 *)(APIC_DEFAULT_PHYS_BASE + APIC_TASKPRI); 139 u64 orig_apicbase = rdmsr(MSR_IA32_APICBASE); 140 u32 apic_version = apic_read(APIC_LVR); 141 u32 cr8 = read_cr8(); 142 143 report_prefix_push("apic_disable"); 144 assert_msg(orig_apicbase & APIC_EN, "APIC not enabled."); 145 146 disable_apic(); 147 report(!(rdmsr(MSR_IA32_APICBASE) & APIC_EN), "Local apic disabled"); 148 report(!this_cpu_has(X86_FEATURE_APIC), 149 "CPUID.1H:EDX.APIC[bit 9] is clear"); 150 verify_disabled_apic_mmio(); 151 152 reset_apic(); 153 report((rdmsr(MSR_IA32_APICBASE) & (APIC_EN | APIC_EXTD)) == APIC_EN, 154 "Local apic enabled in xAPIC mode"); 155 report(this_cpu_has(X86_FEATURE_APIC), "CPUID.1H:EDX.APIC[bit 9] is set"); 156 report(*lvr == apic_version, "*0xfee00030: %x", *lvr); 157 report(*tpr == cr8, "*0xfee00080: %x", *tpr); 158 write_cr8(cr8 ^ MAX_TPR); 159 report(*tpr == (cr8 ^ MAX_TPR) << 4, "*0xfee00080: %x", *tpr); 160 write_cr8(cr8); 161 162 if (enable_x2apic()) { 163 apic_write(APIC_SPIV, 0x1ff); 164 report((rdmsr(MSR_IA32_APICBASE) & (APIC_EN | APIC_EXTD)) == (APIC_EN | APIC_EXTD), 165 "Local apic enabled in x2APIC mode"); 166 report(this_cpu_has(X86_FEATURE_APIC), 167 "CPUID.1H:EDX.APIC[bit 9] is set"); 168 verify_disabled_apic_mmio(); 169 if (!(orig_apicbase & APIC_EXTD)) 170 reset_apic(); 171 } 172 report_prefix_pop(); 173 } 174 175 #define ALTERNATE_APIC_BASE 0xfed40000 176 177 static void test_apicbase(void) 178 { 179 u64 orig_apicbase = rdmsr(MSR_IA32_APICBASE); 180 u32 lvr = apic_read(APIC_LVR); 181 u64 value; 182 183 wrmsr(MSR_IA32_APICBASE, orig_apicbase & ~(APIC_EN | APIC_EXTD)); 184 wrmsr(MSR_IA32_APICBASE, ALTERNATE_APIC_BASE | APIC_BSP | APIC_EN); 185 186 report_prefix_push("apicbase"); 187 188 report(*(volatile u32 *)(ALTERNATE_APIC_BASE + APIC_LVR) == lvr, 189 "relocate apic"); 190 191 value = orig_apicbase | (1UL << cpuid_maxphyaddr()); 192 report(test_for_exception(GP_VECTOR, do_write_apicbase, &value), 193 "reserved physaddr bits"); 194 195 value = orig_apicbase | 1; 196 report(test_for_exception(GP_VECTOR, do_write_apicbase, &value), 197 "reserved low bits"); 198 199 wrmsr(MSR_IA32_APICBASE, orig_apicbase); 200 apic_write(APIC_SPIV, 0x1ff); 201 202 report_prefix_pop(); 203 } 204 205 static void do_write_apic_id(void *id) 206 { 207 apic_write(APIC_ID, *(u32 *)id); 208 } 209 210 static void __test_apic_id(void * unused) 211 { 212 u32 id, newid; 213 u8 initial_xapic_id = cpuid(1).b >> 24; 214 u32 initial_x2apic_id = cpuid(0xb).d; 215 bool x2apic_mode = rdmsr(MSR_IA32_APICBASE) & APIC_EXTD; 216 217 if (x2apic_mode) 218 reset_apic(); 219 220 id = apic_id(); 221 report(initial_xapic_id == id, "xapic id matches cpuid"); 222 223 newid = (id + 1) << 24; 224 report(!test_for_exception(GP_VECTOR, do_write_apic_id, &newid) && 225 (id == apic_id() || id + 1 == apic_id()), 226 "writeable xapic id"); 227 228 if (!enable_x2apic()) 229 goto out; 230 231 report(test_for_exception(GP_VECTOR, do_write_apic_id, &newid), 232 "non-writeable x2apic id"); 233 report(initial_xapic_id == (apic_id() & 0xff), "sane x2apic id"); 234 235 /* old QEMUs do not set initial x2APIC ID */ 236 report(initial_xapic_id == (initial_x2apic_id & 0xff) && 237 initial_x2apic_id == apic_id(), 238 "x2apic id matches cpuid"); 239 240 out: 241 reset_apic(); 242 243 report(initial_xapic_id == apic_id(), "correct xapic id after reset"); 244 245 /* old KVMs do not reset xAPIC ID */ 246 if (id != apic_id()) 247 apic_write(APIC_ID, id << 24); 248 249 if (x2apic_mode) 250 enable_x2apic(); 251 } 252 253 static void test_apic_id(void) 254 { 255 if (cpu_count() < 2) 256 return; 257 258 on_cpu(1, __test_apic_id, NULL); 259 } 260 261 static int ipi_count; 262 263 static void self_ipi_isr(isr_regs_t *regs) 264 { 265 ++ipi_count; 266 eoi(); 267 } 268 269 static void test_self_ipi(void) 270 { 271 u64 start = rdtsc(); 272 int vec = 0xf1; 273 274 handle_irq(vec, self_ipi_isr); 275 irq_enable(); 276 apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED | vec, 277 id_map[0]); 278 279 do { 280 pause(); 281 } while (rdtsc() - start < 1000000000 && ipi_count == 0); 282 283 report(ipi_count == 1, "self ipi"); 284 } 285 286 volatile int nmi_counter_private, nmi_counter, nmi_hlt_counter, sti_loop_active; 287 288 static void sti_nop(char *p) 289 { 290 asm volatile ( 291 ".globl post_sti \n\t" 292 "sti \n" 293 /* 294 * vmx won't exit on external interrupt if blocked-by-sti, 295 * so give it a reason to exit by accessing an unmapped page. 296 */ 297 "post_sti: testb $0, %0 \n\t" 298 "nop \n\t" 299 "cli" 300 : : "m"(*p) 301 ); 302 nmi_counter = nmi_counter_private; 303 } 304 305 static void sti_loop(void *ignore) 306 { 307 unsigned k = 0; 308 309 while (sti_loop_active) { 310 sti_nop((char *)(ulong)((k++ * 4096) % (128 * 1024 * 1024))); 311 } 312 } 313 314 static void nmi_handler(isr_regs_t *regs) 315 { 316 extern void post_sti(void); 317 ++nmi_counter_private; 318 nmi_hlt_counter += regs->rip == (ulong)post_sti; 319 } 320 321 static void test_sti_nmi(void) 322 { 323 unsigned old_counter; 324 325 if (cpu_count() < 2) { 326 return; 327 } 328 329 handle_irq(2, nmi_handler); 330 on_cpu(1, update_cr3, (void *)read_cr3()); 331 332 sti_loop_active = 1; 333 on_cpu_async(1, sti_loop, 0); 334 while (nmi_counter < 30000) { 335 old_counter = nmi_counter; 336 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[1]); 337 while (nmi_counter == old_counter) { 338 ; 339 } 340 } 341 sti_loop_active = 0; 342 report(nmi_hlt_counter == 0, "nmi-after-sti"); 343 } 344 345 static volatile bool nmi_done, nmi_flushed; 346 static volatile int nmi_received; 347 static volatile int cpu0_nmi_ctr1, cpu1_nmi_ctr1; 348 static volatile int cpu0_nmi_ctr2, cpu1_nmi_ctr2; 349 350 static void multiple_nmi_handler(isr_regs_t *regs) 351 { 352 ++nmi_received; 353 } 354 355 static void kick_me_nmi(void *blah) 356 { 357 while (!nmi_done) { 358 ++cpu1_nmi_ctr1; 359 while (cpu1_nmi_ctr1 != cpu0_nmi_ctr1 && !nmi_done) { 360 pause(); 361 } 362 if (nmi_done) { 363 return; 364 } 365 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]); 366 /* make sure the NMI has arrived by sending an IPI after it */ 367 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_FIXED | APIC_INT_ASSERT 368 | 0x44, id_map[0]); 369 ++cpu1_nmi_ctr2; 370 while (cpu1_nmi_ctr2 != cpu0_nmi_ctr2 && !nmi_done) { 371 pause(); 372 } 373 } 374 } 375 376 static void flush_nmi(isr_regs_t *regs) 377 { 378 nmi_flushed = true; 379 apic_write(APIC_EOI, 0); 380 } 381 382 static void test_multiple_nmi(void) 383 { 384 int i; 385 bool ok = true; 386 387 if (cpu_count() < 2) { 388 return; 389 } 390 391 sti(); 392 handle_irq(2, multiple_nmi_handler); 393 handle_irq(0x44, flush_nmi); 394 on_cpu_async(1, kick_me_nmi, 0); 395 for (i = 0; i < 1000000; ++i) { 396 nmi_flushed = false; 397 nmi_received = 0; 398 ++cpu0_nmi_ctr1; 399 while (cpu1_nmi_ctr1 != cpu0_nmi_ctr1) { 400 pause(); 401 } 402 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI | APIC_INT_ASSERT, id_map[0]); 403 while (!nmi_flushed) { 404 pause(); 405 } 406 if (nmi_received != 2) { 407 ok = false; 408 break; 409 } 410 ++cpu0_nmi_ctr2; 411 while (cpu1_nmi_ctr2 != cpu0_nmi_ctr2) { 412 pause(); 413 } 414 } 415 nmi_done = true; 416 report(ok, "multiple nmi"); 417 } 418 419 static void pending_nmi_handler(isr_regs_t *regs) 420 { 421 int i; 422 423 if (++nmi_received == 1) { 424 for (i = 0; i < 10; ++i) 425 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI, 0); 426 } 427 } 428 429 static void test_pending_nmi(void) 430 { 431 int i; 432 433 handle_irq(2, pending_nmi_handler); 434 for (i = 0; i < 100000; ++i) { 435 nmi_received = 0; 436 437 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_NMI, 0); 438 while (nmi_received < 2) 439 pause(); 440 441 if (nmi_received != 2) 442 break; 443 } 444 report(nmi_received == 2, "pending nmi"); 445 } 446 447 static volatile int lvtt_counter = 0; 448 449 static void lvtt_handler(isr_regs_t *regs) 450 { 451 lvtt_counter++; 452 eoi(); 453 } 454 455 static void test_apic_timer_one_shot(void) 456 { 457 uint64_t tsc1, tsc2; 458 static const uint32_t interval = 0x10000; 459 460 #define APIC_LVT_TIMER_VECTOR (0xee) 461 462 handle_irq(APIC_LVT_TIMER_VECTOR, lvtt_handler); 463 irq_enable(); 464 465 /* One shot mode */ 466 apic_write(APIC_LVTT, APIC_LVT_TIMER_ONESHOT | 467 APIC_LVT_TIMER_VECTOR); 468 /* Divider == 1 */ 469 apic_write(APIC_TDCR, 0x0000000b); 470 471 tsc1 = rdtsc(); 472 /* Set "Initial Counter Register", which starts the timer */ 473 apic_write(APIC_TMICT, interval); 474 while (!lvtt_counter); 475 tsc2 = rdtsc(); 476 477 /* 478 * For LVT Timer clock, SDM vol 3 10.5.4 says it should be 479 * derived from processor's bus clock (IIUC which is the same 480 * as TSC), however QEMU seems to be using nanosecond. In all 481 * cases, the following should satisfy on all modern 482 * processors. 483 */ 484 report((lvtt_counter == 1) && (tsc2 - tsc1 >= interval), 485 "APIC LVT timer one shot"); 486 } 487 488 static atomic_t broadcast_counter; 489 490 static void broadcast_handler(isr_regs_t *regs) 491 { 492 atomic_inc(&broadcast_counter); 493 eoi(); 494 } 495 496 static bool broadcast_received(unsigned ncpus) 497 { 498 unsigned counter; 499 u64 start = rdtsc(); 500 501 do { 502 counter = atomic_read(&broadcast_counter); 503 if (counter >= ncpus) 504 break; 505 pause(); 506 } while (rdtsc() - start < 1000000000); 507 508 atomic_set(&broadcast_counter, 0); 509 510 return counter == ncpus; 511 } 512 513 static void test_physical_broadcast(void) 514 { 515 unsigned ncpus = cpu_count(); 516 unsigned long cr3 = read_cr3(); 517 u32 broadcast_address = enable_x2apic() ? 0xffffffff : 0xff; 518 519 handle_irq(BROADCAST_VECTOR, broadcast_handler); 520 for (int c = 1; c < ncpus; c++) 521 on_cpu(c, update_cr3, (void *)cr3); 522 523 printf("starting broadcast (%s)\n", enable_x2apic() ? "x2apic" : "xapic"); 524 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_FIXED | APIC_INT_ASSERT | 525 BROADCAST_VECTOR, broadcast_address); 526 report(broadcast_received(ncpus), "APIC physical broadcast address"); 527 528 apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_FIXED | APIC_INT_ASSERT | 529 BROADCAST_VECTOR | APIC_DEST_ALLINC, 0); 530 report(broadcast_received(ncpus), "APIC physical broadcast shorthand"); 531 } 532 533 static void wait_until_tmcct_common(uint32_t initial_count, bool stop_when_half, bool should_wrap_around) 534 { 535 uint32_t tmcct = apic_read(APIC_TMCCT); 536 537 if (tmcct) { 538 while (tmcct > (initial_count / 2)) 539 tmcct = apic_read(APIC_TMCCT); 540 541 if ( stop_when_half ) 542 return; 543 544 /* Wait until the counter reach 0 or wrap-around */ 545 while ( tmcct <= (initial_count / 2) && tmcct > 0 ) 546 tmcct = apic_read(APIC_TMCCT); 547 548 /* Wait specifically for wrap around to skip 0 TMCCR if we were asked to */ 549 while (should_wrap_around && !tmcct) 550 tmcct = apic_read(APIC_TMCCT); 551 } 552 } 553 554 static void wait_until_tmcct_is_zero(uint32_t initial_count, bool stop_when_half) 555 { 556 return wait_until_tmcct_common(initial_count, stop_when_half, false); 557 } 558 559 static void wait_until_tmcct_wrap_around(uint32_t initial_count, bool stop_when_half) 560 { 561 return wait_until_tmcct_common(initial_count, stop_when_half, true); 562 } 563 564 static inline void apic_change_mode(unsigned long new_mode) 565 { 566 uint32_t lvtt; 567 568 lvtt = apic_read(APIC_LVTT); 569 apic_write(APIC_LVTT, (lvtt & ~APIC_LVT_TIMER_MASK) | new_mode); 570 } 571 572 static void test_apic_change_mode(void) 573 { 574 uint32_t tmict = 0x999999; 575 576 printf("starting apic change mode\n"); 577 578 apic_write(APIC_TMICT, tmict); 579 580 apic_change_mode(APIC_LVT_TIMER_PERIODIC); 581 582 report(apic_read(APIC_TMICT) == tmict, "TMICT value reset"); 583 584 /* Testing one-shot */ 585 apic_change_mode(APIC_LVT_TIMER_ONESHOT); 586 apic_write(APIC_TMICT, tmict); 587 report(apic_read(APIC_TMCCT), "TMCCT should have a non-zero value"); 588 589 wait_until_tmcct_is_zero(tmict, false); 590 report(!apic_read(APIC_TMCCT), "TMCCT should have reached 0"); 591 592 /* 593 * Write TMICT before changing mode from one-shot to periodic TMCCT should 594 * be reset to TMICT periodicly 595 */ 596 apic_write(APIC_TMICT, tmict); 597 wait_until_tmcct_is_zero(tmict, true); 598 apic_change_mode(APIC_LVT_TIMER_PERIODIC); 599 report(apic_read(APIC_TMCCT), "TMCCT should have a non-zero value"); 600 601 /* 602 * After the change of mode, the counter should not be reset and continue 603 * counting down from where it was 604 */ 605 report(apic_read(APIC_TMCCT) < (tmict / 2), 606 "TMCCT should not be reset to TMICT value"); 607 /* 608 * Specifically wait for timer wrap around and skip 0. 609 * Under KVM lapic there is a possibility that a small amount of consecutive 610 * TMCCR reads return 0 while hrtimer is reset in an async callback 611 */ 612 wait_until_tmcct_wrap_around(tmict, false); 613 report(apic_read(APIC_TMCCT) > (tmict / 2), 614 "TMCCT should be reset to the initial-count"); 615 616 wait_until_tmcct_is_zero(tmict, true); 617 /* 618 * Keep the same TMICT and change timer mode to one-shot 619 * TMCCT should be > 0 and count-down to 0 620 */ 621 apic_change_mode(APIC_LVT_TIMER_ONESHOT); 622 report(apic_read(APIC_TMCCT) < (tmict / 2), 623 "TMCCT should not be reset to init"); 624 wait_until_tmcct_is_zero(tmict, false); 625 report(!apic_read(APIC_TMCCT), "TMCCT should have reach zero"); 626 627 /* now tmcct == 0 and tmict != 0 */ 628 apic_change_mode(APIC_LVT_TIMER_PERIODIC); 629 report(!apic_read(APIC_TMCCT), "TMCCT should stay at zero"); 630 } 631 632 #define KVM_HC_SEND_IPI 10 633 634 static void test_pv_ipi(void) 635 { 636 int ret; 637 unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0; 638 639 asm volatile("vmcall" : "=a"(ret) :"a"(KVM_HC_SEND_IPI), "b"(a0), "c"(a1), "d"(a2), "S"(a3)); 640 report(!ret, "PV IPIs testing"); 641 } 642 643 int main(void) 644 { 645 setup_vm(); 646 647 test_lapic_existence(); 648 649 mask_pic_interrupts(); 650 test_apic_id(); 651 test_apic_disable(); 652 test_enable_x2apic(); 653 test_apicbase(); 654 655 test_self_ipi(); 656 test_physical_broadcast(); 657 if (test_device_enabled()) 658 test_pv_ipi(); 659 660 test_sti_nmi(); 661 test_multiple_nmi(); 662 test_pending_nmi(); 663 664 test_apic_timer_one_shot(); 665 test_apic_change_mode(); 666 test_tsc_deadline_timer(); 667 668 return report_summary(); 669 } 670