1 #include "libcflat.h"
2 #include "apic.h"
3 #include "vm.h"
4 #include "smp.h"
5 #include "desc.h"
6 #include "isr.h"
7 #include "delay.h"
8
poll_remote_irr(unsigned line)9 static void poll_remote_irr(unsigned line)
10 {
11 while (ioapic_read_redir(line).remote_irr)
12 cpu_relax();
13 }
14
toggle_irq_line(unsigned line)15 static void toggle_irq_line(unsigned line)
16 {
17 set_irq_line(line, 1);
18 set_irq_line(line, 0);
19 }
20
ioapic_reg_version(void)21 static void ioapic_reg_version(void)
22 {
23 u8 version_offset;
24 uint32_t data_read, data_write;
25
26 version_offset = 0x01;
27 data_read = ioapic_read_reg(version_offset);
28 data_write = data_read ^ 0xffffffff;
29
30 ioapic_write_reg(version_offset, data_write);
31 report(data_read == ioapic_read_reg(version_offset),
32 "version register read only test");
33 }
34
ioapic_reg_id(void)35 static void ioapic_reg_id(void)
36 {
37 u8 id_offset;
38 uint32_t data_read, data_write, diff;
39
40 id_offset = 0x0;
41 data_read = ioapic_read_reg(id_offset);
42 data_write = data_read ^ 0xffffffff;
43
44 ioapic_write_reg(id_offset, data_write);
45
46 diff = data_read ^ ioapic_read_reg(id_offset);
47 report(diff == 0x0f000000, "id register only bits [24:27] writable");
48 }
49
ioapic_arbitration_id(void)50 static void ioapic_arbitration_id(void)
51 {
52 u8 id_offset, arb_offset;
53 uint32_t write;
54
55 id_offset = 0x0;
56 arb_offset = 0x2;
57 write = 0x0f000000;
58
59 ioapic_write_reg(id_offset, write);
60 report(ioapic_read_reg(arb_offset) == write,
61 "arbitration register set by id");
62
63 ioapic_write_reg(arb_offset, 0x0);
64 report(ioapic_read_reg(arb_offset) == write,
65 "arbtration register read only");
66 }
67
68 static volatile int g_isr_76;
69
ioapic_isr_76(isr_regs_t * regs)70 static void ioapic_isr_76(isr_regs_t *regs)
71 {
72 ++g_isr_76;
73 eoi();
74 }
75
test_ioapic_edge_intr(void)76 static void test_ioapic_edge_intr(void)
77 {
78 handle_irq(0x76, ioapic_isr_76);
79 ioapic_set_redir(0x0e, 0x76, TRIGGER_EDGE);
80 toggle_irq_line(0x0e);
81 asm volatile ("nop");
82 report(g_isr_76 == 1, "edge triggered intr");
83 }
84
85 static volatile int g_isr_77;
86
ioapic_isr_77(isr_regs_t * regs)87 static void ioapic_isr_77(isr_regs_t *regs)
88 {
89 ++g_isr_77;
90 set_irq_line(0x0e, 0);
91 eoi();
92 }
93
test_ioapic_level_intr(void)94 static void test_ioapic_level_intr(void)
95 {
96 handle_irq(0x77, ioapic_isr_77);
97 ioapic_set_redir(0x0e, 0x77, TRIGGER_LEVEL);
98 set_irq_line(0x0e, 1);
99 asm volatile ("nop");
100 report(g_isr_77 == 1, "level triggered intr");
101 }
102
103 static int g_78, g_66, g_66_after_78;
104 static ulong g_66_rip, g_78_rip;
105
ioapic_isr_78(isr_regs_t * regs)106 static void ioapic_isr_78(isr_regs_t *regs)
107 {
108 ++g_78;
109 g_78_rip = regs->rip;
110 eoi();
111 }
112
ioapic_isr_66(isr_regs_t * regs)113 static void ioapic_isr_66(isr_regs_t *regs)
114 {
115 ++g_66;
116 if (g_78)
117 ++g_66_after_78;
118 g_66_rip = regs->rip;
119 eoi();
120 }
121
test_ioapic_simultaneous(void)122 static void test_ioapic_simultaneous(void)
123 {
124 handle_irq(0x78, ioapic_isr_78);
125 handle_irq(0x66, ioapic_isr_66);
126 ioapic_set_redir(0x0e, 0x78, TRIGGER_EDGE);
127 ioapic_set_redir(0x0f, 0x66, TRIGGER_EDGE);
128 cli();
129 toggle_irq_line(0x0f);
130 toggle_irq_line(0x0e);
131 sti_nop();
132 report(g_66 && g_78 && g_66_after_78 && g_66_rip == g_78_rip,
133 "ioapic simultaneous edge interrupts");
134 }
135
136 static volatile int g_tmr_79 = -1;
137
ioapic_isr_79(isr_regs_t * regs)138 static void ioapic_isr_79(isr_regs_t *regs)
139 {
140 g_tmr_79 = apic_read_bit(APIC_TMR, 0x79);
141 set_irq_line(0x0e, 0);
142 eoi();
143 }
144
test_ioapic_edge_tmr(bool expected_tmr_before)145 static void test_ioapic_edge_tmr(bool expected_tmr_before)
146 {
147 int tmr_before;
148
149 handle_irq(0x79, ioapic_isr_79);
150 ioapic_set_redir(0x0e, 0x79, TRIGGER_EDGE);
151 tmr_before = apic_read_bit(APIC_TMR, 0x79);
152 toggle_irq_line(0x0e);
153 asm volatile ("nop");
154 report(tmr_before == expected_tmr_before && !g_tmr_79,
155 "TMR for ioapic edge interrupts (expected %s)",
156 expected_tmr_before ? "true" : "false");
157 }
158
test_ioapic_level_tmr(bool expected_tmr_before)159 static void test_ioapic_level_tmr(bool expected_tmr_before)
160 {
161 int tmr_before;
162
163 handle_irq(0x79, ioapic_isr_79);
164 ioapic_set_redir(0x0e, 0x79, TRIGGER_LEVEL);
165 tmr_before = apic_read_bit(APIC_TMR, 0x79);
166 set_irq_line(0x0e, 1);
167 asm volatile ("nop");
168 report(tmr_before == expected_tmr_before && g_tmr_79,
169 "TMR for ioapic level interrupts (expected %s)",
170 expected_tmr_before ? "true" : "false");
171 }
172
toggle_irq_line_0x0e(void * data)173 static void toggle_irq_line_0x0e(void *data)
174 {
175 cli();
176 delay(IPI_DELAY);
177 toggle_irq_line(0x0e);
178 sti();
179 }
180
test_ioapic_edge_tmr_smp(bool expected_tmr_before)181 static void test_ioapic_edge_tmr_smp(bool expected_tmr_before)
182 {
183 int tmr_before;
184 int i;
185
186 g_tmr_79 = -1;
187 handle_irq(0x79, ioapic_isr_79);
188 ioapic_set_redir(0x0e, 0x79, TRIGGER_EDGE);
189 tmr_before = apic_read_bit(APIC_TMR, 0x79);
190 on_cpu_async(1, toggle_irq_line_0x0e, 0);
191 i = 0;
192 while(g_tmr_79 == -1) i++;
193 printf("%d iterations before interrupt received\n", i);
194 report(tmr_before == expected_tmr_before && !g_tmr_79,
195 "TMR for ioapic edge interrupts (expected %s)",
196 expected_tmr_before ? "true" : "false");
197 }
198
set_irq_line_0x0e(void * data)199 static void set_irq_line_0x0e(void *data)
200 {
201 cli();
202 delay(IPI_DELAY);
203 set_irq_line(0x0e, 1);
204 sti();
205 }
206
test_ioapic_level_tmr_smp(bool expected_tmr_before)207 static void test_ioapic_level_tmr_smp(bool expected_tmr_before)
208 {
209 int i, tmr_before;
210
211 g_tmr_79 = -1;
212 handle_irq(0x79, ioapic_isr_79);
213 ioapic_set_redir(0x0e, 0x79, TRIGGER_LEVEL);
214 tmr_before = apic_read_bit(APIC_TMR, 0x79);
215 on_cpu_async(1, set_irq_line_0x0e, 0);
216 i = 0;
217 while(g_tmr_79 == -1) i++;
218 printf("%d iterations before interrupt received\n", i);
219 report(tmr_before == expected_tmr_before && g_tmr_79,
220 "TMR for ioapic level interrupts (expected %s)",
221 expected_tmr_before ? "true" : "false");
222 poll_remote_irr(0xe);
223 }
224
225 static int g_isr_98;
226
ioapic_isr_98(isr_regs_t * regs)227 static void ioapic_isr_98(isr_regs_t *regs)
228 {
229 ++g_isr_98;
230 if (g_isr_98 == 1) {
231 set_irq_line(0x0e, 0);
232 set_irq_line(0x0e, 1);
233 }
234 set_irq_line(0x0e, 0);
235 eoi();
236 }
237
test_ioapic_level_coalesce(void)238 static void test_ioapic_level_coalesce(void)
239 {
240 handle_irq(0x98, ioapic_isr_98);
241 ioapic_set_redir(0x0e, 0x98, TRIGGER_LEVEL);
242 set_irq_line(0x0e, 1);
243 asm volatile ("nop");
244 report(g_isr_98 == 1, "coalesce simultaneous level interrupts");
245 }
246
247 static int g_isr_99;
248
ioapic_isr_99(isr_regs_t * regs)249 static void ioapic_isr_99(isr_regs_t *regs)
250 {
251 ++g_isr_99;
252 set_irq_line(0x0e, 0);
253 eoi();
254 }
255
test_ioapic_level_sequential(void)256 static void test_ioapic_level_sequential(void)
257 {
258 handle_irq(0x99, ioapic_isr_99);
259 ioapic_set_redir(0x0e, 0x99, TRIGGER_LEVEL);
260 set_irq_line(0x0e, 1);
261 set_irq_line(0x0e, 1);
262 asm volatile ("nop");
263 report(g_isr_99 == 2, "sequential level interrupts");
264 }
265
266 static volatile int g_isr_9a;
267
ioapic_isr_9a(isr_regs_t * regs)268 static void ioapic_isr_9a(isr_regs_t *regs)
269 {
270 ++g_isr_9a;
271 if (g_isr_9a == 2)
272 set_irq_line(0x0e, 0);
273 eoi();
274 }
275
test_ioapic_level_retrigger(void)276 static void test_ioapic_level_retrigger(void)
277 {
278 int i;
279
280 handle_irq(0x9a, ioapic_isr_9a);
281 ioapic_set_redir(0x0e, 0x9a, TRIGGER_LEVEL);
282
283 asm volatile ("cli");
284 set_irq_line(0x0e, 1);
285
286 for (i = 0; i < 10; i++) {
287 if (g_isr_9a == 2)
288 break;
289
290 asm volatile ("sti; hlt; cli");
291 }
292
293 asm volatile ("sti");
294
295 report(g_isr_9a == 2, "retriggered level interrupts without masking");
296 }
297
298 static volatile int g_isr_81;
299
ioapic_isr_81(isr_regs_t * regs)300 static void ioapic_isr_81(isr_regs_t *regs)
301 {
302 ++g_isr_81;
303 set_irq_line(0x0e, 0);
304 eoi();
305 }
306
test_ioapic_edge_mask(void)307 static void test_ioapic_edge_mask(void)
308 {
309 handle_irq(0x81, ioapic_isr_81);
310 ioapic_set_redir(0x0e, 0x81, TRIGGER_EDGE);
311
312 set_mask(0x0e, true);
313 set_irq_line(0x0e, 1);
314 set_irq_line(0x0e, 0);
315
316 asm volatile ("nop");
317 report(g_isr_81 == 0, "masked level interrupt");
318
319 set_mask(0x0e, false);
320 set_irq_line(0x0e, 1);
321
322 asm volatile ("nop");
323 report(g_isr_81 == 1, "unmasked level interrupt");
324 }
325
326 static volatile int g_isr_82;
327
ioapic_isr_82(isr_regs_t * regs)328 static void ioapic_isr_82(isr_regs_t *regs)
329 {
330 ++g_isr_82;
331 set_irq_line(0x0e, 0);
332 eoi();
333 }
334
test_ioapic_level_mask(void)335 static void test_ioapic_level_mask(void)
336 {
337 handle_irq(0x82, ioapic_isr_82);
338 ioapic_set_redir(0x0e, 0x82, TRIGGER_LEVEL);
339
340 set_mask(0x0e, true);
341 set_irq_line(0x0e, 1);
342
343 asm volatile ("nop");
344 report(g_isr_82 == 0, "masked level interrupt");
345
346 set_mask(0x0e, false);
347
348 asm volatile ("nop");
349 report(g_isr_82 == 1, "unmasked level interrupt");
350 }
351
352 static volatile int g_isr_83;
353
ioapic_isr_83(isr_regs_t * regs)354 static void ioapic_isr_83(isr_regs_t *regs)
355 {
356 ++g_isr_83;
357 set_mask(0x0e, true);
358 eoi();
359 }
360
test_ioapic_level_retrigger_mask(void)361 static void test_ioapic_level_retrigger_mask(void)
362 {
363 handle_irq(0x83, ioapic_isr_83);
364 ioapic_set_redir(0x0e, 0x83, TRIGGER_LEVEL);
365
366 set_irq_line(0x0e, 1);
367 asm volatile ("nop");
368 set_mask(0x0e, false);
369 asm volatile ("nop");
370 report(g_isr_83 == 2, "retriggered level interrupts with mask");
371
372 set_irq_line(0x0e, 0);
373 set_mask(0x0e, false);
374 }
375
376 static volatile int g_isr_64, g_isr_84;
377
ioapic_reconfigure_dest(int line)378 static void ioapic_reconfigure_dest(int line)
379 {
380 ioapic_redir_entry_t e;
381
382 set_irq_line(line, 0);
383
384 e = ioapic_read_redir(line);
385 report(e.remote_irr == 1, "Reconfigure Remote IRR set");
386
387 e.dest_id = 1;
388
389 // Update only upper part of the register because we only change the
390 // destination, which resides in the upper part
391 ioapic_write_reg(0x10 + line * 2 + 1, ((u32 *)&e)[1]);
392 }
393
ioapic_isr_64(isr_regs_t * regs)394 static void ioapic_isr_64(isr_regs_t *regs)
395 {
396 /*
397 * Raise the IRQ line for the higher priority interrupt, *before*
398 * reconfiguring the I/O APIC routing. KVM should intercept EOI for
399 * both the in-service vector (line 0xd, vector 0x64) and the requested
400 * vector (line 0xe, vector 0x84).
401 */
402 set_irq_line(0x0e, 1);
403
404 ioapic_reconfigure_dest(0xe);
405 ioapic_reconfigure_dest(0xd);
406 eoi();
407
408 ++g_isr_64;
409 }
410
ioapic_isr_84(isr_regs_t * regs)411 static void ioapic_isr_84(isr_regs_t *regs)
412 {
413 report(g_isr_64, "Higher priority IRQ should be blocked until IRET restores RFLAGS.IF");
414
415 ++g_isr_84;
416 eoi();
417 }
418
test_ioapic_self_reconfigure(void)419 static void test_ioapic_self_reconfigure(void)
420 {
421 ioapic_redir_entry_t d = {
422 .vector = 0x64,
423 .delivery_mode = 0,
424 .dest_mode = 0,
425 .dest_id = 0,
426 .trig_mode = TRIGGER_LEVEL,
427 };
428 ioapic_redir_entry_t e = {
429 .vector = 0x84,
430 .delivery_mode = 0,
431 .dest_mode = 0,
432 .dest_id = 0,
433 .trig_mode = TRIGGER_LEVEL,
434 };
435
436 handle_irq(0x64, ioapic_isr_64);
437 ioapic_write_redir(0xd, d);
438
439 handle_irq(0x84, ioapic_isr_84);
440 ioapic_write_redir(0xe, e);
441
442 set_irq_line(0x0d, 1);
443
444 e = ioapic_read_redir(0xe);
445 report(g_isr_84 == 1 && e.remote_irr == 0, "Reconfigure self highest priority");
446
447 d = ioapic_read_redir(0xd);
448 report(g_isr_64 == 1 && d.remote_irr == 0, "Reconfigure self lower priority");
449
450 poll_remote_irr(0xd);
451 poll_remote_irr(0xe);
452 }
453
454 static volatile int g_isr_85;
455
ioapic_isr_85(isr_regs_t * regs)456 static void ioapic_isr_85(isr_regs_t *regs)
457 {
458 ++g_isr_85;
459 set_irq_line(0x0e, 0);
460 eoi();
461 }
462
test_ioapic_physical_destination_mode(void)463 static void test_ioapic_physical_destination_mode(void)
464 {
465 ioapic_redir_entry_t e = {
466 .vector = 0x85,
467 .delivery_mode = 0,
468 .dest_mode = 0,
469 .dest_id = 0x1,
470 .trig_mode = TRIGGER_LEVEL,
471 };
472 handle_irq(0x85, ioapic_isr_85);
473 ioapic_write_redir(0xe, e);
474 set_irq_line(0x0e, 1);
475 do {
476 pause();
477 } while(g_isr_85 != 1);
478 report(g_isr_85 == 1, "ioapic physical destination mode");
479 poll_remote_irr(0xe);
480 }
481
482 static volatile int g_isr_86;
483 struct spinlock ioapic_lock;
484
ioapic_isr_86(isr_regs_t * regs)485 static void ioapic_isr_86(isr_regs_t *regs)
486 {
487 spin_lock(&ioapic_lock);
488 ++g_isr_86;
489 spin_unlock(&ioapic_lock);
490 set_irq_line(0x0e, 0);
491 eoi();
492 }
493
test_ioapic_logical_destination_mode(void)494 static void test_ioapic_logical_destination_mode(void)
495 {
496 /* Number of vcpus which are configured/set in dest_id */
497 int nr_vcpus = 3;
498 ioapic_redir_entry_t e = {
499 .vector = 0x86,
500 .delivery_mode = 0,
501 .dest_mode = 1,
502 .dest_id = 0xd,
503 .trig_mode = TRIGGER_LEVEL,
504 };
505 handle_irq(0x86, ioapic_isr_86);
506 ioapic_write_redir(0xe, e);
507 set_irq_line(0x0e, 1);
508 do {
509 pause();
510 } while(g_isr_86 < nr_vcpus);
511 report(g_isr_86 == nr_vcpus, "ioapic logical destination mode");
512 poll_remote_irr(0xe);
513 }
514
main(void)515 int main(void)
516 {
517 setup_vm();
518
519 on_cpus(update_cr3, (void *)read_cr3());
520 mask_pic_interrupts();
521
522 if (enable_x2apic())
523 printf("x2apic enabled\n");
524 else
525 printf("x2apic not detected\n");
526
527 sti();
528
529 ioapic_reg_version();
530 ioapic_reg_id();
531 ioapic_arbitration_id();
532
533 test_ioapic_edge_intr();
534 test_ioapic_level_intr();
535 test_ioapic_simultaneous();
536
537 test_ioapic_level_coalesce();
538 test_ioapic_level_sequential();
539 test_ioapic_level_retrigger();
540
541 test_ioapic_edge_mask();
542 test_ioapic_level_mask();
543 test_ioapic_level_retrigger_mask();
544
545 test_ioapic_edge_tmr(false);
546 test_ioapic_level_tmr(false);
547 test_ioapic_level_tmr(true);
548 test_ioapic_edge_tmr(true);
549
550 if (cpu_count() > 1)
551 test_ioapic_physical_destination_mode();
552 if (cpu_count() > 3)
553 test_ioapic_logical_destination_mode();
554
555 if (cpu_count() > 1) {
556 test_ioapic_edge_tmr_smp(false);
557 test_ioapic_level_tmr_smp(false);
558 test_ioapic_level_tmr_smp(true);
559 test_ioapic_edge_tmr_smp(true);
560
561 test_ioapic_self_reconfigure();
562 }
563
564 return report_summary();
565 }
566