1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1996, by Steve Passe
5 * All rights reserved.
6 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. The name of the developer may NOT be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 * 3. Neither the name of the author nor the names of any co-contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * Local APIC support on Pentium and later processors.
34 */
35
36 #include <sys/cdefs.h>
37 #include "opt_atpic.h"
38
39 #include "opt_ddb.h"
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/asan.h>
44 #include <sys/bus.h>
45 #include <sys/kernel.h>
46 #include <sys/lock.h>
47 #include <sys/malloc.h>
48 #include <sys/msan.h>
49 #include <sys/mutex.h>
50 #include <sys/pcpu.h>
51 #include <sys/proc.h>
52 #include <sys/refcount.h>
53 #include <sys/sched.h>
54 #include <sys/smp.h>
55 #include <sys/sysctl.h>
56 #include <sys/timeet.h>
57 #include <sys/timetc.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61
62 #include <x86/apicreg.h>
63 #include <machine/clock.h>
64 #include <machine/cpufunc.h>
65 #include <machine/cputypes.h>
66 #include <machine/fpu.h>
67 #include <machine/frame.h>
68 #include <machine/intr_machdep.h>
69 #include <x86/apicvar.h>
70 #include <x86/mca.h>
71 #include <machine/md_var.h>
72 #include <machine/smp.h>
73 #include <machine/specialreg.h>
74 #include <x86/init.h>
75
76 #ifdef DDB
77 #include <sys/interrupt.h>
78 #include <ddb/ddb.h>
79 #endif
80
81 #ifdef __amd64__
82 #define SDT_APIC SDT_SYSIGT
83 #define GSEL_APIC 0
84 #else
85 #define SDT_APIC SDT_SYS386IGT
86 #define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL)
87 #endif
88
89 static MALLOC_DEFINE(M_LAPIC, "local_apic", "Local APIC items");
90
91 /* Sanity checks on IDT vectors. */
92 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
93 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
94 CTASSERT(APIC_LOCAL_INTS == 240);
95 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
96
97 /*
98 * I/O interrupts use non-negative IRQ values. These values are used
99 * to mark unused IDT entries or IDT entries reserved for a non-I/O
100 * interrupt.
101 */
102 #define IRQ_FREE -1
103 #define IRQ_TIMER -2
104 #define IRQ_SYSCALL -3
105 #define IRQ_DTRACE_RET -4
106 #define IRQ_EVTCHN -5
107
108 enum lat_timer_mode {
109 LAT_MODE_UNDEF = 0,
110 LAT_MODE_PERIODIC = 1,
111 LAT_MODE_ONESHOT = 2,
112 LAT_MODE_DEADLINE = 3,
113 };
114
115 /*
116 * Support for local APICs. Local APICs manage interrupts on each
117 * individual processor as opposed to I/O APICs which receive interrupts
118 * from I/O devices and then forward them on to the local APICs.
119 *
120 * Local APICs can also send interrupts to each other thus providing the
121 * mechanism for IPIs.
122 */
123
124 struct lvt {
125 u_int lvt_edgetrigger:1;
126 u_int lvt_activehi:1;
127 u_int lvt_masked:1;
128 u_int lvt_active:1;
129 u_int lvt_mode:16;
130 u_int lvt_vector:8;
131 u_int lvt_reg;
132 const char *lvt_desc;
133 };
134
135 struct lapic {
136 struct lvt la_lvts[APIC_LVT_MAX + 1];
137 struct lvt la_elvts[APIC_ELVT_MAX + 1];
138 u_int la_id:8;
139 u_int la_cluster:4;
140 u_int la_cluster_id:2;
141 u_int la_present:1;
142 u_long *la_timer_count;
143 uint64_t la_timer_period;
144 enum lat_timer_mode la_timer_mode;
145 uint32_t lvt_timer_base;
146 uint32_t lvt_timer_last;
147 /* Include IDT_SYSCALL to make indexing easier. */
148 int la_ioint_irqs[APIC_NUM_IOINTS + 1];
149 } static *lapics;
150
151 /* Global defaults for local APIC LVT entries. */
152 static struct lvt lvts[] = {
153 /* LINT0: masked ExtINT */
154 [APIC_LVT_LINT0] = {
155 .lvt_edgetrigger = 1,
156 .lvt_activehi = 1,
157 .lvt_masked = 1,
158 .lvt_active = 1,
159 .lvt_mode = APIC_LVT_DM_EXTINT,
160 .lvt_vector = 0,
161 .lvt_reg = LAPIC_LVT_LINT0,
162 .lvt_desc = "LINT0",
163 },
164 /* LINT1: NMI */
165 [APIC_LVT_LINT1] = {
166 .lvt_edgetrigger = 1,
167 .lvt_activehi = 1,
168 .lvt_masked = 0,
169 .lvt_active = 1,
170 .lvt_mode = APIC_LVT_DM_NMI,
171 .lvt_vector = 0,
172 .lvt_reg = LAPIC_LVT_LINT1,
173 .lvt_desc = "LINT1",
174 },
175 [APIC_LVT_TIMER] = {
176 .lvt_edgetrigger = 1,
177 .lvt_activehi = 1,
178 .lvt_masked = 1,
179 .lvt_active = 1,
180 .lvt_mode = APIC_LVT_DM_FIXED,
181 .lvt_vector = APIC_TIMER_INT,
182 .lvt_reg = LAPIC_LVT_TIMER,
183 .lvt_desc = "TIMER",
184 },
185 [APIC_LVT_ERROR] = {
186 .lvt_edgetrigger = 1,
187 .lvt_activehi = 1,
188 .lvt_masked = 0,
189 .lvt_active = 1,
190 .lvt_mode = APIC_LVT_DM_FIXED,
191 .lvt_vector = APIC_ERROR_INT,
192 .lvt_reg = LAPIC_LVT_ERROR,
193 .lvt_desc = "ERROR",
194 },
195 [APIC_LVT_PMC] = {
196 .lvt_edgetrigger = 1,
197 .lvt_activehi = 1,
198 .lvt_masked = 1,
199 .lvt_active = 1,
200 .lvt_mode = APIC_LVT_DM_NMI,
201 .lvt_vector = 0,
202 .lvt_reg = LAPIC_LVT_PCINT,
203 .lvt_desc = "PMC",
204 },
205 [APIC_LVT_THERMAL] = {
206 .lvt_edgetrigger = 1,
207 .lvt_activehi = 1,
208 .lvt_masked = 1,
209 .lvt_active = 1,
210 .lvt_mode = APIC_LVT_DM_FIXED,
211 .lvt_vector = APIC_THERMAL_INT,
212 .lvt_reg = LAPIC_LVT_THERMAL,
213 .lvt_desc = "THERM",
214 },
215 [APIC_LVT_CMCI] = {
216 .lvt_edgetrigger = 1,
217 .lvt_activehi = 1,
218 .lvt_masked = 1,
219 .lvt_active = 1,
220 .lvt_mode = APIC_LVT_DM_FIXED,
221 .lvt_vector = APIC_CMC_INT,
222 .lvt_reg = LAPIC_LVT_CMCI,
223 .lvt_desc = "CMCI",
224 },
225 };
226
227 /* Global defaults for AMD local APIC ELVT entries. */
228 static struct lvt elvts[] = {
229 [APIC_ELVT_IBS] = {
230 .lvt_edgetrigger = 1,
231 .lvt_activehi = 1,
232 .lvt_masked = 1,
233 .lvt_active = 1,
234 .lvt_mode = APIC_LVT_DM_NMI,
235 .lvt_vector = 0,
236 .lvt_reg = LAPIC_EXT_LVT0,
237 .lvt_desc = "IBS",
238 },
239 [APIC_ELVT_MCA] = {
240 .lvt_edgetrigger = 1,
241 .lvt_activehi = 1,
242 .lvt_masked = 1,
243 .lvt_active = 0,
244 .lvt_mode = APIC_LVT_DM_FIXED,
245 .lvt_vector = APIC_CMC_INT,
246 .lvt_reg = LAPIC_EXT_LVT1,
247 .lvt_desc = "MCA",
248 },
249 [APIC_ELVT_DEI] = {
250 .lvt_edgetrigger = 1,
251 .lvt_activehi = 1,
252 .lvt_masked = 1,
253 .lvt_active = 0,
254 .lvt_mode = APIC_LVT_DM_FIXED,
255 .lvt_vector = 0,
256 .lvt_reg = LAPIC_EXT_LVT2,
257 .lvt_desc = "ELVT2",
258 },
259 [APIC_ELVT_SBI] = {
260 .lvt_edgetrigger = 1,
261 .lvt_activehi = 1,
262 .lvt_masked = 1,
263 .lvt_active = 0,
264 .lvt_mode = APIC_LVT_DM_FIXED,
265 .lvt_vector = 0,
266 .lvt_reg = LAPIC_EXT_LVT3,
267 .lvt_desc = "ELVT3",
268 },
269 };
270
271 static inthand_t *ioint_handlers[] = {
272 NULL, /* 0 - 31 */
273 IDTVEC(apic_isr1), /* 32 - 63 */
274 IDTVEC(apic_isr2), /* 64 - 95 */
275 IDTVEC(apic_isr3), /* 96 - 127 */
276 IDTVEC(apic_isr4), /* 128 - 159 */
277 IDTVEC(apic_isr5), /* 160 - 191 */
278 IDTVEC(apic_isr6), /* 192 - 223 */
279 IDTVEC(apic_isr7), /* 224 - 255 */
280 };
281
282 static inthand_t *ioint_pti_handlers[] = {
283 NULL, /* 0 - 31 */
284 IDTVEC(apic_isr1_pti), /* 32 - 63 */
285 IDTVEC(apic_isr2_pti), /* 64 - 95 */
286 IDTVEC(apic_isr3_pti), /* 96 - 127 */
287 IDTVEC(apic_isr4_pti), /* 128 - 159 */
288 IDTVEC(apic_isr5_pti), /* 160 - 191 */
289 IDTVEC(apic_isr6_pti), /* 192 - 223 */
290 IDTVEC(apic_isr7_pti), /* 224 - 255 */
291 };
292
293 static u_int32_t lapic_timer_divisors[] = {
294 APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
295 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
296 };
297
298 extern inthand_t IDTVEC(rsvd_pti), IDTVEC(rsvd);
299
300 volatile char *lapic_map;
301 vm_paddr_t lapic_paddr = DEFAULT_APIC_BASE;
302 int x2apic_mode;
303 int lapic_eoi_suppression;
304 static int lapic_timer_tsc_deadline;
305 static u_long lapic_timer_divisor, count_freq;
306 static struct eventtimer lapic_et;
307 #ifdef SMP
308 static uint64_t lapic_ipi_wait_mult;
309 static int __read_mostly lapic_ds_idle_timeout = 1000000;
310 #endif
311 unsigned int max_apic_id;
312 static int pcint_refcnt = 0;
313
314 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
315 "APIC options");
316 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
317 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
318 &lapic_eoi_suppression, 0, "");
319 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
320 &lapic_timer_tsc_deadline, 0, "");
321 #ifdef SMP
322 SYSCTL_INT(_hw_apic, OID_AUTO, ds_idle_timeout, CTLFLAG_RWTUN,
323 &lapic_ds_idle_timeout, 0,
324 "timeout (in us) for APIC Delivery Status to become Idle (xAPIC only)");
325 #endif
326
327 static void lapic_calibrate_initcount(struct lapic *la);
328
329 /*
330 * Calculate the max index of the present LVT entry from the value of
331 * the LAPIC version register.
332 */
333 static int
lapic_maxlvt(uint32_t version)334 lapic_maxlvt(uint32_t version)
335 {
336 return ((version & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
337 }
338
339 /*
340 * Use __nosanitizethread to exempt the LAPIC I/O accessors from KCSan
341 * instrumentation. Otherwise, if x2APIC is not available, use of the global
342 * lapic_map will generate a KCSan false positive. While the mapping is
343 * shared among all CPUs, the physical access will always take place on the
344 * local CPU's APIC, so there isn't in fact a race here. Furthermore, the
345 * KCSan warning printf can cause a panic if issued during LAPIC access,
346 * due to attempted recursive use of event timer resources.
347 */
348
349 static uint32_t __nosanitizethread
lapic_read32(enum LAPIC_REGISTERS reg)350 lapic_read32(enum LAPIC_REGISTERS reg)
351 {
352 uint32_t res;
353
354 if (x2apic_mode) {
355 res = rdmsr32(MSR_APIC_000 + reg);
356 } else {
357 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
358 }
359 return (res);
360 }
361
362 static void __nosanitizethread
lapic_write32(enum LAPIC_REGISTERS reg,uint32_t val)363 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
364 {
365
366 if (x2apic_mode) {
367 mfence();
368 lfence();
369 wrmsr(MSR_APIC_000 + reg, val);
370 } else {
371 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
372 }
373 }
374
375 static void __nosanitizethread
lapic_write32_nofence(enum LAPIC_REGISTERS reg,uint32_t val)376 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
377 {
378
379 if (x2apic_mode) {
380 wrmsr(MSR_APIC_000 + reg, val);
381 } else {
382 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
383 }
384 }
385
386 #ifdef SMP
387 static uint64_t
lapic_read_icr_lo(void)388 lapic_read_icr_lo(void)
389 {
390
391 return (lapic_read32(LAPIC_ICR_LO));
392 }
393
394 static void
lapic_write_icr(uint32_t vhi,uint32_t vlo)395 lapic_write_icr(uint32_t vhi, uint32_t vlo)
396 {
397 register_t saveintr;
398 uint64_t v;
399
400 if (x2apic_mode) {
401 v = ((uint64_t)vhi << 32) | vlo;
402 mfence();
403 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
404 } else {
405 saveintr = intr_disable();
406 lapic_write32(LAPIC_ICR_HI, vhi);
407 lapic_write32(LAPIC_ICR_LO, vlo);
408 intr_restore(saveintr);
409 }
410 }
411
412 static void
lapic_write_icr_lo(uint32_t vlo)413 lapic_write_icr_lo(uint32_t vlo)
414 {
415
416 if (x2apic_mode) {
417 mfence();
418 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, vlo);
419 } else {
420 lapic_write32(LAPIC_ICR_LO, vlo);
421 }
422 }
423
424 static void
lapic_write_self_ipi(uint32_t vector)425 lapic_write_self_ipi(uint32_t vector)
426 {
427
428 KASSERT(x2apic_mode, ("SELF IPI write in xAPIC mode"));
429 wrmsr(MSR_APIC_000 + LAPIC_SELF_IPI, vector);
430 }
431 #endif /* SMP */
432
433 static void
lapic_enable_x2apic(void)434 lapic_enable_x2apic(void)
435 {
436 uint64_t apic_base;
437
438 apic_base = rdmsr(MSR_APICBASE);
439 apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
440 wrmsr(MSR_APICBASE, apic_base);
441 }
442
443 bool
lapic_is_x2apic(void)444 lapic_is_x2apic(void)
445 {
446 uint64_t apic_base;
447
448 apic_base = rdmsr(MSR_APICBASE);
449 return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
450 (APICBASE_X2APIC | APICBASE_ENABLED));
451 }
452
453 static void lapic_early_mask_vecs(void);
454 static void lapic_enable(void);
455 static void lapic_resume(struct pic *pic, bool suspend_cancelled);
456 static void lapic_timer_oneshot(struct lapic *);
457 static void lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
458 static void lapic_timer_periodic(struct lapic *);
459 static void lapic_timer_deadline(struct lapic *);
460 static void lapic_timer_stop(struct lapic *);
461 static void lapic_timer_set_divisor(u_int divisor);
462 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
463 static int lapic_et_start(struct eventtimer *et,
464 sbintime_t first, sbintime_t period);
465 static int lapic_et_stop(struct eventtimer *et);
466 static u_int apic_idt_to_irq(u_int apic_id, u_int vector);
467 static void lapic_set_tpr(u_int vector);
468
469 struct pic lapic_pic = { .pic_resume = lapic_resume };
470
471 static uint32_t
lvt_mode_impl(struct lapic * la,struct lvt * lvt,u_int pin,uint32_t value)472 lvt_mode_impl(struct lapic *la, struct lvt *lvt, u_int pin, uint32_t value)
473 {
474
475 value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
476 APIC_LVT_VECTOR);
477 if (lvt->lvt_edgetrigger == 0)
478 value |= APIC_LVT_TM;
479 if (lvt->lvt_activehi == 0)
480 value |= APIC_LVT_IIPP_INTALO;
481 if (lvt->lvt_masked)
482 value |= APIC_LVT_M;
483 value |= lvt->lvt_mode;
484 switch (lvt->lvt_mode) {
485 case APIC_LVT_DM_NMI:
486 case APIC_LVT_DM_SMI:
487 case APIC_LVT_DM_INIT:
488 case APIC_LVT_DM_EXTINT:
489 if (!lvt->lvt_edgetrigger) {
490 if (bootverbose) {
491 printf(
492 "lapic%u: Forcing LINT%u to edge trigger\n",
493 la->la_id, pin);
494 }
495 value &= ~APIC_LVT_TM;
496 }
497 /* Use a vector of 0. */
498 break;
499 case APIC_LVT_DM_FIXED:
500 value |= lvt->lvt_vector;
501 break;
502 default:
503 panic("bad APIC LVT delivery mode: %#x\n", value);
504 }
505 return (value);
506 }
507
508 static uint32_t
lvt_mode(struct lapic * la,u_int pin,uint32_t value)509 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
510 {
511 struct lvt *lvt;
512
513 KASSERT(pin <= APIC_LVT_MAX,
514 ("%s: pin %u out of range", __func__, pin));
515 if (la->la_lvts[pin].lvt_active)
516 lvt = &la->la_lvts[pin];
517 else
518 lvt = &lvts[pin];
519
520 return (lvt_mode_impl(la, lvt, pin, value));
521 }
522
523 static uint32_t
elvt_mode(struct lapic * la,u_int idx,uint32_t value)524 elvt_mode(struct lapic *la, u_int idx, uint32_t value)
525 {
526 struct lvt *elvt;
527
528 KASSERT(idx <= APIC_ELVT_MAX,
529 ("%s: idx %u out of range", __func__, idx));
530
531 if (la->la_elvts[idx].lvt_active)
532 elvt = &la->la_elvts[idx];
533 else
534 elvt = &elvts[idx];
535 KASSERT(elvt->lvt_active, ("%s: ELVT%u is not active", __func__, idx));
536 KASSERT(elvt->lvt_edgetrigger,
537 ("%s: ELVT%u is not edge triggered", __func__, idx));
538 KASSERT(elvt->lvt_activehi,
539 ("%s: ELVT%u is not active high", __func__, idx));
540 return (lvt_mode_impl(la, elvt, idx, value));
541 }
542
543 /*
544 * Map the local APIC and setup necessary interrupt vectors.
545 */
546 void
lapic_init(vm_paddr_t addr)547 lapic_init(vm_paddr_t addr)
548 {
549 #ifdef SMP
550 uint64_t r, r1, r2, rx;
551 #endif
552 uint32_t ver;
553 int i;
554 bool arat;
555
556 TSENTER();
557
558 /*
559 * Enable x2APIC mode if possible. Map the local APIC
560 * registers page.
561 *
562 * Keep the LAPIC registers page mapped uncached for x2APIC
563 * mode too, to have direct map page attribute set to
564 * uncached. This is needed to work around CPU errata present
565 * on all Intel processors.
566 */
567 KASSERT(trunc_page(addr) == addr,
568 ("local APIC not aligned on a page boundary"));
569 lapic_paddr = addr;
570 lapic_map = pmap_mapdev(addr, PAGE_SIZE);
571 if (x2apic_mode) {
572 lapic_enable_x2apic();
573 lapic_map = NULL;
574 }
575
576 /* Setup the spurious interrupt handler. */
577 setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
578 GSEL_APIC);
579
580 /* Perform basic initialization of the BSP's local APIC. */
581 lapic_enable();
582 lapic_early_mask_vecs();
583
584 /* Set BSP's per-CPU local APIC ID. */
585 PCPU_SET(apic_id, lapic_id());
586
587 /* Local APIC timer interrupt. */
588 setidt(APIC_TIMER_INT, pti ? IDTVEC(timerint_pti) : IDTVEC(timerint),
589 SDT_APIC, SEL_KPL, GSEL_APIC);
590
591 /* Local APIC error interrupt. */
592 setidt(APIC_ERROR_INT, pti ? IDTVEC(errorint_pti) : IDTVEC(errorint),
593 SDT_APIC, SEL_KPL, GSEL_APIC);
594
595 /* XXX: Thermal interrupt */
596
597 /* Local APIC CMCI. */
598 setidt(APIC_CMC_INT, pti ? IDTVEC(cmcint_pti) : IDTVEC(cmcint),
599 SDT_APIC, SEL_KPL, GSEL_APIC);
600
601 if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
602 /* Set if APIC timer runs in C3. */
603 arat = (cpu_power_eax & CPUTPM1_ARAT);
604
605 bzero(&lapic_et, sizeof(lapic_et));
606 lapic_et.et_name = "LAPIC";
607 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
608 ET_FLAGS_PERCPU;
609 lapic_et.et_quality = 600;
610 if (!arat) {
611 lapic_et.et_flags |= ET_FLAGS_C3STOP;
612 lapic_et.et_quality = 100;
613 }
614 if ((cpu_feature & CPUID_TSC) != 0 &&
615 (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
616 tsc_is_invariant && tsc_freq != 0) {
617 lapic_timer_tsc_deadline = 1;
618 TUNABLE_INT_FETCH("hw.apic.timer_tsc_deadline",
619 &lapic_timer_tsc_deadline);
620 }
621
622 lapic_et.et_frequency = 0;
623 /* We don't know frequency yet, so trying to guess. */
624 lapic_et.et_min_period = 0x00001000LL;
625 lapic_et.et_max_period = SBT_1S;
626 lapic_et.et_start = lapic_et_start;
627 lapic_et.et_stop = lapic_et_stop;
628 lapic_et.et_priv = NULL;
629 et_register(&lapic_et);
630 }
631
632 /*
633 * Set lapic_eoi_suppression after lapic_enable(), to not
634 * enable suppression in the hardware prematurely. Note that
635 * we by default enable suppression even when system only has
636 * one IO-APIC, since EOI is broadcasted to all APIC agents,
637 * including CPUs, otherwise.
638 *
639 * It seems that at least some KVM versions report
640 * EOI_SUPPRESSION bit, but auto-EOI does not work.
641 */
642 ver = lapic_read32(LAPIC_VERSION);
643 if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
644 lapic_eoi_suppression = 1;
645 if (vm_guest == VM_GUEST_KVM) {
646 if (bootverbose)
647 printf(
648 "KVM -- disabling lapic eoi suppression\n");
649 lapic_eoi_suppression = 0;
650 }
651 TUNABLE_INT_FETCH("hw.apic.eoi_suppression",
652 &lapic_eoi_suppression);
653 }
654
655 #ifdef SMP
656 #define LOOPS 1000
657 /*
658 * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
659 * lapic_ipi_wait_mult contains the number of iterations which
660 * approximately delay execution for 1 microsecond (the
661 * argument to lapic_ipi_wait() is in microseconds).
662 *
663 * We assume that TSC is present and already measured.
664 * Possible TSC frequency jumps are irrelevant to the
665 * calibration loop below, the CPU clock management code is
666 * not yet started, and we do not enter sleep states.
667 */
668 KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
669 ("TSC not initialized"));
670 if (!x2apic_mode) {
671 r = rdtsc();
672 for (rx = 0; rx < LOOPS; rx++) {
673 (void)lapic_read_icr_lo();
674 ia32_pause();
675 }
676 r = rdtsc() - r;
677 r1 = tsc_freq * LOOPS;
678 r2 = r * 1000000;
679 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
680 if (bootverbose) {
681 printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
682 "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
683 (uintmax_t)r, (uintmax_t)tsc_freq);
684 }
685 }
686 #undef LOOPS
687 #endif /* SMP */
688
689 TSEXIT();
690 }
691
692 /*
693 * Create a local APIC instance.
694 */
695 void
lapic_create(u_int apic_id,int boot_cpu)696 lapic_create(u_int apic_id, int boot_cpu)
697 {
698 int i;
699
700 if (apic_id > max_apic_id) {
701 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
702 if (boot_cpu)
703 panic("Can't ignore BSP");
704 return;
705 }
706 KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
707 apic_id));
708
709 /*
710 * Assume no local LVT overrides and a cluster of 0 and
711 * intra-cluster ID of 0.
712 */
713 lapics[apic_id].la_present = 1;
714 lapics[apic_id].la_id = apic_id;
715 for (i = 0; i <= APIC_LVT_MAX; i++) {
716 lapics[apic_id].la_lvts[i] = lvts[i];
717 lapics[apic_id].la_lvts[i].lvt_active = 0;
718 }
719 for (i = 0; i <= APIC_ELVT_MAX; i++) {
720 lapics[apic_id].la_elvts[i] = elvts[i];
721 lapics[apic_id].la_elvts[i].lvt_active = 0;
722 }
723 for (i = 0; i <= APIC_NUM_IOINTS; i++)
724 lapics[apic_id].la_ioint_irqs[i] = IRQ_FREE;
725 lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
726 lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
727 IRQ_TIMER;
728 #ifdef KDTRACE_HOOKS
729 lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
730 IRQ_DTRACE_RET;
731 #endif
732 #ifdef XENHVM
733 lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
734 #endif
735
736 #ifdef SMP
737 cpu_add(apic_id, boot_cpu);
738 #endif
739 }
740
741 static inline uint32_t
amd_read_ext_features(void)742 amd_read_ext_features(void)
743 {
744 uint32_t version;
745
746 if (cpu_vendor_id != CPU_VENDOR_AMD &&
747 cpu_vendor_id != CPU_VENDOR_HYGON)
748 return (0);
749 version = lapic_read32(LAPIC_VERSION);
750 if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
751 return (lapic_read32(LAPIC_EXT_FEATURES));
752 else
753 return (0);
754 }
755
756 static inline uint32_t
amd_read_elvt_count(void)757 amd_read_elvt_count(void)
758 {
759 uint32_t extf;
760 uint32_t count;
761
762 extf = amd_read_ext_features();
763 count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
764 count = min(count, APIC_ELVT_MAX + 1);
765 return (count);
766 }
767
768 /*
769 * Dump contents of local APIC registers
770 */
771 void
lapic_dump(const char * str)772 lapic_dump(const char* str)
773 {
774 uint32_t version;
775 uint32_t maxlvt;
776 uint32_t extf;
777 int elvt_count;
778 int i;
779
780 version = lapic_read32(LAPIC_VERSION);
781 maxlvt = lapic_maxlvt(version);
782 printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
783 printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
784 lapic_read32(LAPIC_ID), version,
785 lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
786 if ((cpu_feature2 & CPUID2_X2APIC) != 0)
787 printf(" x2APIC: %d", x2apic_mode);
788 printf("\n lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
789 lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
790 lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
791 printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x",
792 lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
793 lapic_read32(LAPIC_LVT_ERROR));
794 if (maxlvt >= APIC_LVT_PMC)
795 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
796 printf("\n");
797 if (maxlvt >= APIC_LVT_CMCI)
798 printf(" cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
799 extf = amd_read_ext_features();
800 if (extf != 0) {
801 printf(" AMD ext features: 0x%08x", extf);
802 elvt_count = amd_read_elvt_count();
803 for (i = 0; i < elvt_count; i++)
804 printf("%s elvt%d: 0x%08x", (i % 4) ? "" : "\n ", i,
805 lapic_read32(LAPIC_EXT_LVT0 + i));
806 printf("\n");
807 }
808 }
809
810 void
lapic_xapic_mode(void)811 lapic_xapic_mode(void)
812 {
813 register_t saveintr;
814
815 saveintr = intr_disable();
816 if (x2apic_mode)
817 lapic_enable_x2apic();
818 intr_restore(saveintr);
819 }
820
821 static void
lapic_early_mask_vec(const struct lvt * l)822 lapic_early_mask_vec(const struct lvt *l)
823 {
824 uint32_t v;
825
826 if (l->lvt_masked != 0) {
827 v = lapic_read32(l->lvt_reg);
828 v |= APIC_LVT_M;
829 lapic_write32(l->lvt_reg, v);
830 }
831 }
832
833 /* Done on BSP only */
834 static void
lapic_early_mask_vecs(void)835 lapic_early_mask_vecs(void)
836 {
837 int elvt_count, lvts_count, i;
838 uint32_t version;
839
840 version = lapic_read32(LAPIC_VERSION);
841 lvts_count = min(nitems(lvts), lapic_maxlvt(version) + 1);
842 for (i = 0; i < lvts_count; i++)
843 lapic_early_mask_vec(&lvts[i]);
844
845 elvt_count = amd_read_elvt_count();
846 for (i = 0; i < elvt_count; i++)
847 lapic_early_mask_vec(&elvts[i]);
848 }
849
850 void
lapic_setup(int boot)851 lapic_setup(int boot)
852 {
853 struct lapic *la;
854 uint32_t version;
855 uint32_t maxlvt;
856 register_t saveintr;
857 int elvt_count;
858 int i;
859
860 saveintr = intr_disable();
861
862 la = &lapics[lapic_id()];
863 KASSERT(la->la_present, ("missing APIC structure"));
864 version = lapic_read32(LAPIC_VERSION);
865 maxlvt = lapic_maxlvt(version);
866
867 /* Initialize the TPR to allow all interrupts. */
868 lapic_set_tpr(0);
869
870 /* Setup spurious vector and enable the local APIC. */
871 lapic_enable();
872
873 /* Program LINT[01] LVT entries. */
874 lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
875 lapic_read32(LAPIC_LVT_LINT0)));
876 lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
877 lapic_read32(LAPIC_LVT_LINT1)));
878
879 /* Program the PMC LVT entry if present. */
880 if (maxlvt >= APIC_LVT_PMC) {
881 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
882 LAPIC_LVT_PCINT));
883 }
884
885 /*
886 * Program the timer LVT. Calibration is deferred until it is certain
887 * that we have a reliable timecounter.
888 */
889 la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
890 lapic_read32(LAPIC_LVT_TIMER));
891 la->lvt_timer_last = la->lvt_timer_base;
892 lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
893
894 if (boot)
895 la->la_timer_mode = LAT_MODE_UNDEF;
896 else if (la->la_timer_mode != LAT_MODE_UNDEF) {
897 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
898 lapic_id()));
899 switch (la->la_timer_mode) {
900 case LAT_MODE_PERIODIC:
901 lapic_timer_set_divisor(lapic_timer_divisor);
902 lapic_timer_periodic(la);
903 break;
904 case LAT_MODE_ONESHOT:
905 lapic_timer_set_divisor(lapic_timer_divisor);
906 lapic_timer_oneshot(la);
907 break;
908 case LAT_MODE_DEADLINE:
909 lapic_timer_deadline(la);
910 break;
911 default:
912 panic("corrupted la_timer_mode %p %d", la,
913 la->la_timer_mode);
914 }
915 }
916
917 /* Program error LVT and clear any existing errors. */
918 lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
919 lapic_read32(LAPIC_LVT_ERROR)));
920 lapic_write32(LAPIC_ESR, 0);
921
922 /* XXX: Thermal LVT */
923
924 /* Program the CMCI LVT entry if present. */
925 if (maxlvt >= APIC_LVT_CMCI) {
926 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
927 lapic_read32(LAPIC_LVT_CMCI)));
928 }
929
930 elvt_count = amd_read_elvt_count();
931 for (i = 0; i < elvt_count; i++) {
932 if (la->la_elvts[i].lvt_active)
933 lapic_write32(LAPIC_EXT_LVT0 + i,
934 elvt_mode(la, i, lapic_read32(LAPIC_EXT_LVT0 + i)));
935 }
936
937 intr_restore(saveintr);
938 }
939
940 static void
lapic_intrcnt(void * dummy __unused)941 lapic_intrcnt(void *dummy __unused)
942 {
943 struct pcpu *pc;
944 struct lapic *la;
945 char buf[MAXCOMLEN + 1];
946
947 /* If there are no APICs, skip this function. */
948 if (lapics == NULL)
949 return;
950
951 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
952 la = &lapics[pc->pc_apic_id];
953 if (!la->la_present)
954 continue;
955
956 snprintf(buf, sizeof(buf), "cpu%d:timer", pc->pc_cpuid);
957 intrcnt_add(buf, &la->la_timer_count);
958 }
959 }
960 SYSINIT(lapic_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, lapic_intrcnt, NULL);
961
962 void
lapic_reenable_pcint(void)963 lapic_reenable_pcint(void)
964 {
965 uint32_t value;
966
967 if (refcount_load(&pcint_refcnt) == 0)
968 return;
969
970 value = lapic_read32(LAPIC_LVT_PCINT);
971 value &= ~APIC_LVT_M;
972 lapic_write32(LAPIC_LVT_PCINT, value);
973
974 if ((amd_feature2 & AMDID2_IBS) != 0) {
975 value = lapic_read32(LAPIC_EXT_LVT0);
976 value &= ~APIC_LVT_M;
977 lapic_write32(LAPIC_EXT_LVT0, value);
978 }
979 }
980
981 static void
lapic_update_pcint(void * dummy)982 lapic_update_pcint(void *dummy)
983 {
984 struct lapic *la;
985
986 la = &lapics[lapic_id()];
987 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
988 lapic_read32(LAPIC_LVT_PCINT)));
989
990 if ((amd_feature2 & AMDID2_IBS) != 0) {
991 lapic_write32(LAPIC_EXT_LVT0, elvt_mode(la, APIC_ELVT_IBS,
992 lapic_read32(LAPIC_EXT_LVT0)));
993 }
994 }
995
996 void
lapic_calibrate_timer(void)997 lapic_calibrate_timer(void)
998 {
999 struct lapic *la;
1000 register_t intr;
1001
1002 #ifdef DEV_ATPIC
1003 /* Fail if the local APIC is not present. */
1004 if (!x2apic_mode && lapic_map == NULL)
1005 return;
1006 #endif
1007
1008 intr = intr_disable();
1009 la = &lapics[lapic_id()];
1010
1011 lapic_calibrate_initcount(la);
1012
1013 intr_restore(intr);
1014
1015 if (lapic_timer_tsc_deadline && bootverbose) {
1016 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
1017 (uintmax_t)tsc_freq);
1018 }
1019 }
1020
1021 int
lapic_enable_pcint(void)1022 lapic_enable_pcint(void)
1023 {
1024 u_int32_t maxlvt;
1025
1026 #ifdef DEV_ATPIC
1027 /* Fail if the local APIC is not present. */
1028 if (!x2apic_mode && lapic_map == NULL)
1029 return (0);
1030 #endif
1031
1032 /* Fail if the PMC LVT is not present. */
1033 maxlvt = lapic_maxlvt(lapic_read32(LAPIC_VERSION));
1034 if (maxlvt < APIC_LVT_PMC)
1035 return (0);
1036 if (refcount_acquire(&pcint_refcnt) > 0)
1037 return (1);
1038 lvts[APIC_LVT_PMC].lvt_masked = 0;
1039
1040 if ((amd_feature2 & AMDID2_IBS) != 0)
1041 elvts[APIC_ELVT_IBS].lvt_masked = 0;
1042
1043 MPASS(mp_ncpus == 1 || smp_started);
1044 smp_rendezvous(NULL, lapic_update_pcint, NULL, NULL);
1045 return (1);
1046 }
1047
1048 void
lapic_disable_pcint(void)1049 lapic_disable_pcint(void)
1050 {
1051 u_int32_t maxlvt;
1052
1053 #ifdef DEV_ATPIC
1054 /* Fail if the local APIC is not present. */
1055 if (!x2apic_mode && lapic_map == NULL)
1056 return;
1057 #endif
1058
1059 /* Fail if the PMC LVT is not present. */
1060 maxlvt = lapic_maxlvt(lapic_read32(LAPIC_VERSION));
1061 if (maxlvt < APIC_LVT_PMC)
1062 return;
1063 if (!refcount_release(&pcint_refcnt))
1064 return;
1065 lvts[APIC_LVT_PMC].lvt_masked = 1;
1066 elvts[APIC_ELVT_IBS].lvt_masked = 1;
1067
1068 #ifdef SMP
1069 /* The APs should always be started when hwpmc is unloaded. */
1070 KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
1071 #endif
1072 smp_rendezvous(NULL, lapic_update_pcint, NULL, NULL);
1073 }
1074
1075 static int
lapic_calibrate_initcount_cpuid_vm(void)1076 lapic_calibrate_initcount_cpuid_vm(void)
1077 {
1078 u_int regs[4];
1079 uint64_t freq;
1080
1081 /* Get value from CPUID leaf if possible. */
1082 if (vm_guest == VM_GUEST_NO)
1083 return (false);
1084 if (hv_high < 0x40000010)
1085 return (false);
1086 do_cpuid(0x40000010, regs);
1087 freq = (uint64_t)(regs[1]) * 1000;
1088
1089 /* Pick timer divisor. */
1090 lapic_timer_divisor = 2;
1091 do {
1092 if (freq / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
1093 break;
1094 lapic_timer_divisor <<= 1;
1095 } while (lapic_timer_divisor <= 128);
1096 if (lapic_timer_divisor > 128)
1097 return (false);
1098
1099 /* Record divided frequency. */
1100 count_freq = freq / lapic_timer_divisor;
1101 return (count_freq != 0);
1102 }
1103
1104 static uint64_t
cb_lapic_getcount(void)1105 cb_lapic_getcount(void)
1106 {
1107
1108 return (APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER));
1109 }
1110
1111 static void
lapic_calibrate_initcount(struct lapic * la)1112 lapic_calibrate_initcount(struct lapic *la)
1113 {
1114 uint64_t freq;
1115
1116 if (lapic_calibrate_initcount_cpuid_vm())
1117 goto done;
1118
1119 /* Calibrate the APIC timer frequency. */
1120 lapic_timer_set_divisor(2);
1121 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
1122 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
1123 freq = clockcalib(cb_lapic_getcount, "lapic");
1124 fpu_kern_leave(curthread, NULL);
1125
1126 /* Pick a different divisor if necessary. */
1127 lapic_timer_divisor = 2;
1128 do {
1129 if (freq * 2 / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
1130 break;
1131 lapic_timer_divisor <<= 1;
1132 } while (lapic_timer_divisor <= 128);
1133 if (lapic_timer_divisor > 128)
1134 panic("lapic: Divisor too big");
1135 count_freq = freq * 2 / lapic_timer_divisor;
1136 done:
1137 if (bootverbose) {
1138 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
1139 lapic_timer_divisor, count_freq);
1140 }
1141 }
1142
1143 static void
lapic_change_mode(struct eventtimer * et,struct lapic * la,enum lat_timer_mode newmode)1144 lapic_change_mode(struct eventtimer *et, struct lapic *la,
1145 enum lat_timer_mode newmode)
1146 {
1147 if (la->la_timer_mode == newmode)
1148 return;
1149 switch (newmode) {
1150 case LAT_MODE_PERIODIC:
1151 lapic_timer_set_divisor(lapic_timer_divisor);
1152 et->et_frequency = count_freq;
1153 break;
1154 case LAT_MODE_DEADLINE:
1155 et->et_frequency = tsc_freq;
1156 break;
1157 case LAT_MODE_ONESHOT:
1158 lapic_timer_set_divisor(lapic_timer_divisor);
1159 et->et_frequency = count_freq;
1160 break;
1161 default:
1162 panic("lapic_change_mode %d", newmode);
1163 }
1164 la->la_timer_mode = newmode;
1165 et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
1166 et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
1167 }
1168
1169 static int
lapic_et_start(struct eventtimer * et,sbintime_t first,sbintime_t period)1170 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
1171 {
1172 struct lapic *la;
1173
1174 la = &lapics[PCPU_GET(apic_id)];
1175 if (period != 0) {
1176 lapic_change_mode(et, la, LAT_MODE_PERIODIC);
1177 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
1178 32;
1179 lapic_timer_periodic(la);
1180 } else if (lapic_timer_tsc_deadline) {
1181 lapic_change_mode(et, la, LAT_MODE_DEADLINE);
1182 la->la_timer_period = (et->et_frequency * first) >> 32;
1183 lapic_timer_deadline(la);
1184 } else {
1185 lapic_change_mode(et, la, LAT_MODE_ONESHOT);
1186 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
1187 32;
1188 lapic_timer_oneshot(la);
1189 }
1190 return (0);
1191 }
1192
1193 static int
lapic_et_stop(struct eventtimer * et)1194 lapic_et_stop(struct eventtimer *et)
1195 {
1196 struct lapic *la;
1197
1198 la = &lapics[PCPU_GET(apic_id)];
1199 lapic_timer_stop(la);
1200 la->la_timer_mode = LAT_MODE_UNDEF;
1201 return (0);
1202 }
1203
1204 void
lapic_disable(void)1205 lapic_disable(void)
1206 {
1207 uint32_t value;
1208
1209 /* Software disable the local APIC. */
1210 value = lapic_read32(LAPIC_SVR);
1211 value &= ~APIC_SVR_SWEN;
1212 lapic_write32(LAPIC_SVR, value);
1213 }
1214
1215 static void
lapic_enable(void)1216 lapic_enable(void)
1217 {
1218 uint32_t value;
1219
1220 /* Program the spurious vector to enable the local APIC. */
1221 value = lapic_read32(LAPIC_SVR);
1222 value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
1223 value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
1224 if (lapic_eoi_suppression)
1225 value |= APIC_SVR_EOI_SUPPRESSION;
1226 lapic_write32(LAPIC_SVR, value);
1227 }
1228
1229 /* Reset the local APIC on the BSP during resume. */
1230 static void
lapic_resume(struct pic * pic,bool suspend_cancelled)1231 lapic_resume(struct pic *pic, bool suspend_cancelled)
1232 {
1233
1234 lapic_setup(0);
1235 }
1236
1237 int
lapic_id(void)1238 lapic_id(void)
1239 {
1240 uint32_t v;
1241
1242 KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
1243 v = lapic_read32(LAPIC_ID);
1244 if (!x2apic_mode)
1245 v >>= APIC_ID_SHIFT;
1246 return (v);
1247 }
1248
1249 int
lapic_intr_pending(u_int vector)1250 lapic_intr_pending(u_int vector)
1251 {
1252 uint32_t irr;
1253
1254 /*
1255 * The IRR registers are an array of registers each of which
1256 * only describes 32 interrupts in the low 32 bits. Thus, we
1257 * divide the vector by 32 to get the register index.
1258 * Finally, we modulus the vector by 32 to determine the
1259 * individual bit to test.
1260 */
1261 irr = lapic_read32(LAPIC_IRR0 + vector / 32);
1262 return (irr & 1 << (vector % 32));
1263 }
1264
1265 void
lapic_set_logical_id(u_int apic_id,u_int cluster,u_int cluster_id)1266 lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
1267 {
1268 struct lapic *la;
1269
1270 KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
1271 __func__, apic_id));
1272 KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
1273 __func__, cluster));
1274 KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
1275 ("%s: intra cluster id %u too big", __func__, cluster_id));
1276 la = &lapics[apic_id];
1277 la->la_cluster = cluster;
1278 la->la_cluster_id = cluster_id;
1279 }
1280
1281 int
lapic_set_lvt_mask(u_int apic_id,u_int pin,u_char masked)1282 lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
1283 {
1284
1285 if (pin > APIC_LVT_MAX)
1286 return (EINVAL);
1287 if (apic_id == APIC_ID_ALL) {
1288 lvts[pin].lvt_masked = masked;
1289 if (bootverbose)
1290 printf("lapic:");
1291 } else {
1292 KASSERT(lapics[apic_id].la_present,
1293 ("%s: missing APIC %u", __func__, apic_id));
1294 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
1295 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1296 if (bootverbose)
1297 printf("lapic%u:", apic_id);
1298 }
1299 if (bootverbose)
1300 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
1301 return (0);
1302 }
1303
1304 int
lapic_set_lvt_mode(u_int apic_id,u_int pin,u_int32_t mode)1305 lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
1306 {
1307 struct lvt *lvt;
1308
1309 if (pin > APIC_LVT_MAX)
1310 return (EINVAL);
1311 if (apic_id == APIC_ID_ALL) {
1312 lvt = &lvts[pin];
1313 if (bootverbose)
1314 printf("lapic:");
1315 } else {
1316 KASSERT(lapics[apic_id].la_present,
1317 ("%s: missing APIC %u", __func__, apic_id));
1318 lvt = &lapics[apic_id].la_lvts[pin];
1319 lvt->lvt_active = 1;
1320 if (bootverbose)
1321 printf("lapic%u:", apic_id);
1322 }
1323 lvt->lvt_mode = mode;
1324 switch (mode) {
1325 case APIC_LVT_DM_NMI:
1326 case APIC_LVT_DM_SMI:
1327 case APIC_LVT_DM_INIT:
1328 case APIC_LVT_DM_EXTINT:
1329 lvt->lvt_edgetrigger = 1;
1330 lvt->lvt_activehi = 1;
1331 if (mode == APIC_LVT_DM_EXTINT)
1332 lvt->lvt_masked = 1;
1333 else
1334 lvt->lvt_masked = 0;
1335 break;
1336 default:
1337 panic("Unsupported delivery mode: 0x%x\n", mode);
1338 }
1339 if (bootverbose) {
1340 printf(" Routing ");
1341 switch (mode) {
1342 case APIC_LVT_DM_NMI:
1343 printf("NMI");
1344 break;
1345 case APIC_LVT_DM_SMI:
1346 printf("SMI");
1347 break;
1348 case APIC_LVT_DM_INIT:
1349 printf("INIT");
1350 break;
1351 case APIC_LVT_DM_EXTINT:
1352 printf("ExtINT");
1353 break;
1354 }
1355 printf(" -> LINT%u\n", pin);
1356 }
1357 return (0);
1358 }
1359
1360 int
lapic_set_lvt_polarity(u_int apic_id,u_int pin,enum intr_polarity pol)1361 lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1362 {
1363
1364 if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1365 return (EINVAL);
1366 if (apic_id == APIC_ID_ALL) {
1367 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1368 if (bootverbose)
1369 printf("lapic:");
1370 } else {
1371 KASSERT(lapics[apic_id].la_present,
1372 ("%s: missing APIC %u", __func__, apic_id));
1373 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1374 lapics[apic_id].la_lvts[pin].lvt_activehi =
1375 (pol == INTR_POLARITY_HIGH);
1376 if (bootverbose)
1377 printf("lapic%u:", apic_id);
1378 }
1379 if (bootverbose)
1380 printf(" LINT%u polarity: %s\n", pin,
1381 pol == INTR_POLARITY_HIGH ? "high" : "low");
1382 return (0);
1383 }
1384
1385 int
lapic_set_lvt_triggermode(u_int apic_id,u_int pin,enum intr_trigger trigger)1386 lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1387 enum intr_trigger trigger)
1388 {
1389
1390 if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1391 return (EINVAL);
1392 if (apic_id == APIC_ID_ALL) {
1393 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1394 if (bootverbose)
1395 printf("lapic:");
1396 } else {
1397 KASSERT(lapics[apic_id].la_present,
1398 ("%s: missing APIC %u", __func__, apic_id));
1399 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1400 (trigger == INTR_TRIGGER_EDGE);
1401 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1402 if (bootverbose)
1403 printf("lapic%u:", apic_id);
1404 }
1405 if (bootverbose)
1406 printf(" LINT%u trigger: %s\n", pin,
1407 trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1408 return (0);
1409 }
1410
1411 /*
1412 * Adjust the TPR of the current CPU so that it blocks all interrupts below
1413 * the passed in vector.
1414 */
1415 static void
lapic_set_tpr(u_int vector)1416 lapic_set_tpr(u_int vector)
1417 {
1418 #ifdef CHEAP_TPR
1419 lapic_write32(LAPIC_TPR, vector);
1420 #else
1421 uint32_t tpr;
1422
1423 tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1424 tpr |= vector;
1425 lapic_write32(LAPIC_TPR, tpr);
1426 #endif
1427 }
1428
1429 void
lapic_eoi(void)1430 lapic_eoi(void)
1431 {
1432
1433 lapic_write32_nofence(LAPIC_EOI, 0);
1434 }
1435
1436 void
lapic_handle_intr(int vector,struct trapframe * frame)1437 lapic_handle_intr(int vector, struct trapframe *frame)
1438 {
1439 struct intsrc *isrc;
1440
1441 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
1442 kmsan_mark(&vector, sizeof(vector), KMSAN_STATE_INITED);
1443 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1444 trap_check_kstack();
1445
1446 isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1447 vector));
1448 intr_execute_handlers(isrc, frame);
1449 }
1450
1451 void
lapic_handle_timer(struct trapframe * frame)1452 lapic_handle_timer(struct trapframe *frame)
1453 {
1454 struct lapic *la;
1455 struct trapframe *oldframe;
1456 struct thread *td;
1457
1458 /* Send EOI first thing. */
1459 lapic_eoi();
1460
1461 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
1462 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1463 trap_check_kstack();
1464
1465 if (!sched_do_timer_accounting())
1466 return;
1467
1468 /* Look up our local APIC structure for the tick counters. */
1469 la = &lapics[PCPU_GET(apic_id)];
1470 (*la->la_timer_count)++;
1471 critical_enter();
1472 if (lapic_et.et_active) {
1473 td = curthread;
1474 td->td_intr_nesting_level++;
1475 oldframe = td->td_intr_frame;
1476 td->td_intr_frame = frame;
1477 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1478 td->td_intr_frame = oldframe;
1479 td->td_intr_nesting_level--;
1480 }
1481 critical_exit();
1482 }
1483
1484 static void
lapic_timer_set_divisor(u_int divisor)1485 lapic_timer_set_divisor(u_int divisor)
1486 {
1487
1488 KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1489 KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors),
1490 ("lapic: invalid divisor %u", divisor));
1491 lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1492 }
1493
1494 static void
lapic_timer_oneshot(struct lapic * la)1495 lapic_timer_oneshot(struct lapic *la)
1496 {
1497 uint32_t value;
1498
1499 value = la->lvt_timer_base;
1500 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1501 value |= APIC_LVTT_TM_ONE_SHOT;
1502 la->lvt_timer_last = value;
1503 lapic_write32(LAPIC_LVT_TIMER, value);
1504 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1505 }
1506
1507 static void
lapic_timer_oneshot_nointr(struct lapic * la,uint32_t count)1508 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1509 {
1510 uint32_t value;
1511
1512 value = la->lvt_timer_base;
1513 value &= ~APIC_LVTT_TM;
1514 value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1515 la->lvt_timer_last = value;
1516 lapic_write32(LAPIC_LVT_TIMER, value);
1517 lapic_write32(LAPIC_ICR_TIMER, count);
1518 }
1519
1520 static void
lapic_timer_periodic(struct lapic * la)1521 lapic_timer_periodic(struct lapic *la)
1522 {
1523 uint32_t value;
1524
1525 value = la->lvt_timer_base;
1526 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1527 value |= APIC_LVTT_TM_PERIODIC;
1528 la->lvt_timer_last = value;
1529 lapic_write32(LAPIC_LVT_TIMER, value);
1530 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1531 }
1532
1533 static void
lapic_timer_deadline(struct lapic * la)1534 lapic_timer_deadline(struct lapic *la)
1535 {
1536 uint32_t value;
1537
1538 value = la->lvt_timer_base;
1539 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1540 value |= APIC_LVTT_TM_TSCDLT;
1541 if (value != la->lvt_timer_last) {
1542 la->lvt_timer_last = value;
1543 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1544 if (!x2apic_mode)
1545 mfence();
1546 }
1547 wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1548 }
1549
1550 static void
lapic_timer_stop(struct lapic * la)1551 lapic_timer_stop(struct lapic *la)
1552 {
1553 uint32_t value;
1554
1555 if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1556 wrmsr(MSR_TSC_DEADLINE, 0);
1557 mfence();
1558 } else {
1559 value = la->lvt_timer_base;
1560 value &= ~APIC_LVTT_TM;
1561 value |= APIC_LVT_M;
1562 la->lvt_timer_last = value;
1563 lapic_write32(LAPIC_LVT_TIMER, value);
1564 }
1565 }
1566
1567 void
lapic_handle_cmc(void)1568 lapic_handle_cmc(void)
1569 {
1570 trap_check_kstack();
1571
1572 lapic_eoi();
1573 cmc_intr();
1574 }
1575
1576 /*
1577 * Called from the mca_init() to activate the CMC interrupt if this CPU is
1578 * responsible for monitoring any MC banks for CMC events. Since mca_init()
1579 * is called prior to lapic_setup() during boot, this just needs to unmask
1580 * this CPU's LVT_CMCI entry.
1581 */
1582 void
lapic_enable_cmc(void)1583 lapic_enable_cmc(void)
1584 {
1585 u_int apic_id;
1586
1587 #ifdef DEV_ATPIC
1588 if (!x2apic_mode && lapic_map == NULL)
1589 return;
1590 #endif
1591 apic_id = PCPU_GET(apic_id);
1592 KASSERT(lapics[apic_id].la_present,
1593 ("%s: missing APIC %u", __func__, apic_id));
1594 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1595 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1596 }
1597
1598 int
lapic_enable_mca_elvt(void)1599 lapic_enable_mca_elvt(void)
1600 {
1601 u_int apic_id;
1602 uint32_t value;
1603 int elvt_count;
1604
1605 #ifdef DEV_ATPIC
1606 if (lapic_map == NULL)
1607 return (-1);
1608 #endif
1609
1610 apic_id = PCPU_GET(apic_id);
1611 KASSERT(lapics[apic_id].la_present,
1612 ("%s: missing APIC %u", __func__, apic_id));
1613 elvt_count = amd_read_elvt_count();
1614 if (elvt_count <= APIC_ELVT_MCA)
1615 return (-1);
1616
1617 value = lapic_read32(LAPIC_EXT_LVT0 + APIC_ELVT_MCA);
1618 if ((value & APIC_LVT_M) == 0) {
1619 if (bootverbose)
1620 printf("AMD MCE Thresholding Extended LVT is already active\n");
1621 return (APIC_ELVT_MCA);
1622 }
1623 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_masked = 0;
1624 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_active = 1;
1625 return (APIC_ELVT_MCA);
1626 }
1627
1628 void
lapic_handle_error(void)1629 lapic_handle_error(void)
1630 {
1631 uint32_t esr;
1632
1633 trap_check_kstack();
1634
1635 /*
1636 * Read the contents of the error status register. Write to
1637 * the register first before reading from it to force the APIC
1638 * to update its value to indicate any errors that have
1639 * occurred since the previous write to the register.
1640 */
1641 lapic_write32(LAPIC_ESR, 0);
1642 esr = lapic_read32(LAPIC_ESR);
1643
1644 printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1645 lapic_eoi();
1646 }
1647
1648 u_int
apic_cpuid(u_int apic_id)1649 apic_cpuid(u_int apic_id)
1650 {
1651 #ifdef SMP
1652 return apic_cpuids[apic_id];
1653 #else
1654 return 0;
1655 #endif
1656 }
1657
1658 /* Request a free IDT vector to be used by the specified IRQ. */
1659 u_int
apic_alloc_vector(u_int apic_id,u_int irq)1660 apic_alloc_vector(u_int apic_id, u_int irq)
1661 {
1662 u_int vector;
1663
1664 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq));
1665
1666 /*
1667 * Search for a free vector. Currently we just use a very simple
1668 * algorithm to find the first free vector.
1669 */
1670 mtx_lock_spin(&icu_lock);
1671 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1672 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE)
1673 continue;
1674 lapics[apic_id].la_ioint_irqs[vector] = irq;
1675 mtx_unlock_spin(&icu_lock);
1676 return (vector + APIC_IO_INTS);
1677 }
1678 mtx_unlock_spin(&icu_lock);
1679 return (0);
1680 }
1681
1682 /*
1683 * Request 'count' free contiguous IDT vectors to be used by 'count'
1684 * IRQs. 'count' must be a power of two and the vectors will be
1685 * aligned on a boundary of 'align'. If the request cannot be
1686 * satisfied, 0 is returned.
1687 */
1688 u_int
apic_alloc_vectors(u_int apic_id,u_int * irqs,u_int count,u_int align)1689 apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1690 {
1691 u_int first, run, vector;
1692
1693 KASSERT(powerof2(count), ("bad count"));
1694 KASSERT(powerof2(align), ("bad align"));
1695 KASSERT(align >= count, ("align < count"));
1696 #ifdef INVARIANTS
1697 for (run = 0; run < count; run++)
1698 KASSERT(irqs[run] < num_io_irqs, ("Invalid IRQ %u at index %u",
1699 irqs[run], run));
1700 #endif
1701
1702 /*
1703 * Search for 'count' free vectors. As with apic_alloc_vector(),
1704 * this just uses a simple first fit algorithm.
1705 */
1706 run = 0;
1707 first = 0;
1708 mtx_lock_spin(&icu_lock);
1709 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1710 /* Vector is in use, end run. */
1711 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE) {
1712 run = 0;
1713 first = 0;
1714 continue;
1715 }
1716
1717 /* Start a new run if run == 0 and vector is aligned. */
1718 if (run == 0) {
1719 if (((vector + APIC_IO_INTS) & (align - 1)) != 0)
1720 continue;
1721 first = vector;
1722 }
1723 run++;
1724
1725 /* Keep looping if the run isn't long enough yet. */
1726 if (run < count)
1727 continue;
1728
1729 /* Found a run, assign IRQs and return the first vector. */
1730 for (vector = 0; vector < count; vector++)
1731 lapics[apic_id].la_ioint_irqs[first + vector] =
1732 irqs[vector];
1733 mtx_unlock_spin(&icu_lock);
1734 return (first + APIC_IO_INTS);
1735 }
1736 mtx_unlock_spin(&icu_lock);
1737 printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1738 return (0);
1739 }
1740
1741 /*
1742 * Enable a vector for a particular apic_id. Since all lapics share idt
1743 * entries and ioint_handlers this enables the vector on all lapics. lapics
1744 * which do not have the vector configured would report spurious interrupts
1745 * should it fire.
1746 */
1747 void
apic_enable_vector(u_int apic_id,u_int vector)1748 apic_enable_vector(u_int apic_id, u_int vector)
1749 {
1750
1751 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1752 KASSERT(ioint_handlers[vector / 32] != NULL,
1753 ("No ISR handler for vector %u", vector));
1754 #ifdef KDTRACE_HOOKS
1755 KASSERT(vector != IDT_DTRACE_RET,
1756 ("Attempt to overwrite DTrace entry"));
1757 #endif
1758 setidt(vector, (pti ? ioint_pti_handlers : ioint_handlers)[vector / 32],
1759 SDT_APIC, SEL_KPL, GSEL_APIC);
1760 }
1761
1762 void
apic_disable_vector(u_int apic_id,u_int vector)1763 apic_disable_vector(u_int apic_id, u_int vector)
1764 {
1765
1766 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1767 #ifdef KDTRACE_HOOKS
1768 KASSERT(vector != IDT_DTRACE_RET,
1769 ("Attempt to overwrite DTrace entry"));
1770 #endif
1771 KASSERT(ioint_handlers[vector / 32] != NULL,
1772 ("No ISR handler for vector %u", vector));
1773 #ifdef notyet
1774 /*
1775 * We can not currently clear the idt entry because other cpus
1776 * may have a valid vector at this offset.
1777 */
1778 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC,
1779 SEL_KPL, GSEL_APIC);
1780 #endif
1781 }
1782
1783 /* Release an APIC vector when it's no longer in use. */
1784 void
apic_free_vector(u_int apic_id,u_int vector,u_int irq)1785 apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1786 {
1787 struct thread *td;
1788
1789 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1790 vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1791 ("Vector %u does not map to an IRQ line", vector));
1792 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq));
1793 KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1794 irq, ("IRQ mismatch"));
1795 #ifdef KDTRACE_HOOKS
1796 KASSERT(vector != IDT_DTRACE_RET,
1797 ("Attempt to overwrite DTrace entry"));
1798 #endif
1799
1800 /*
1801 * Bind us to the cpu that owned the vector before freeing it so
1802 * we don't lose an interrupt delivery race.
1803 */
1804 td = curthread;
1805 if (!rebooting) {
1806 thread_lock(td);
1807 if (sched_is_bound(td))
1808 panic("apic_free_vector: Thread already bound.\n");
1809 sched_bind(td, apic_cpuid(apic_id));
1810 thread_unlock(td);
1811 }
1812 mtx_lock_spin(&icu_lock);
1813 lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = IRQ_FREE;
1814 mtx_unlock_spin(&icu_lock);
1815 if (!rebooting) {
1816 thread_lock(td);
1817 sched_unbind(td);
1818 thread_unlock(td);
1819 }
1820 }
1821
1822 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1823 static u_int
apic_idt_to_irq(u_int apic_id,u_int vector)1824 apic_idt_to_irq(u_int apic_id, u_int vector)
1825 {
1826 int irq;
1827
1828 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1829 vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1830 ("Vector %u does not map to an IRQ line", vector));
1831 #ifdef KDTRACE_HOOKS
1832 KASSERT(vector != IDT_DTRACE_RET,
1833 ("Attempt to overwrite DTrace entry"));
1834 #endif
1835 irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1836 if (irq < 0)
1837 irq = 0;
1838 return (irq);
1839 }
1840
1841 #ifdef DDB
1842 /*
1843 * Dump data about APIC IDT vector mappings.
1844 */
DB_SHOW_COMMAND_FLAGS(apic,db_show_apic,DB_CMD_MEMSAFE)1845 DB_SHOW_COMMAND_FLAGS(apic, db_show_apic, DB_CMD_MEMSAFE)
1846 {
1847 struct intsrc *isrc;
1848 int i, verbose;
1849 u_int apic_id;
1850 u_int irq;
1851
1852 if (strcmp(modif, "vv") == 0)
1853 verbose = 2;
1854 else if (strcmp(modif, "v") == 0)
1855 verbose = 1;
1856 else
1857 verbose = 0;
1858 for (apic_id = 0; apic_id <= max_apic_id; apic_id++) {
1859 if (lapics[apic_id].la_present == 0)
1860 continue;
1861 db_printf("Interrupts bound to lapic %u\n", apic_id);
1862 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1863 irq = lapics[apic_id].la_ioint_irqs[i];
1864 if (irq == IRQ_FREE || irq == IRQ_SYSCALL)
1865 continue;
1866 #ifdef KDTRACE_HOOKS
1867 if (irq == IRQ_DTRACE_RET)
1868 continue;
1869 #endif
1870 #ifdef XENHVM
1871 if (irq == IRQ_EVTCHN)
1872 continue;
1873 #endif
1874 db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1875 if (irq == IRQ_TIMER)
1876 db_printf("lapic timer\n");
1877 else if (irq < num_io_irqs) {
1878 isrc = intr_lookup_source(irq);
1879 if (isrc == NULL || verbose == 0)
1880 db_printf("IRQ %u\n", irq);
1881 else
1882 db_dump_intr_event(isrc->is_event,
1883 verbose == 2);
1884 } else
1885 db_printf("IRQ %u ???\n", irq);
1886 }
1887 }
1888 }
1889
1890 static void
dump_mask(const char * prefix,uint32_t v,int base)1891 dump_mask(const char *prefix, uint32_t v, int base)
1892 {
1893 int i, first;
1894
1895 first = 1;
1896 for (i = 0; i < 32; i++)
1897 if (v & (1 << i)) {
1898 if (first) {
1899 db_printf("%s:", prefix);
1900 first = 0;
1901 }
1902 db_printf(" %02x", base + i);
1903 }
1904 if (!first)
1905 db_printf("\n");
1906 }
1907
1908 /* Show info from the lapic regs for this CPU. */
DB_SHOW_COMMAND_FLAGS(lapic,db_show_lapic,DB_CMD_MEMSAFE)1909 DB_SHOW_COMMAND_FLAGS(lapic, db_show_lapic, DB_CMD_MEMSAFE)
1910 {
1911 const struct lvt *l;
1912 int elvt_count, lvts_count, i;
1913 uint32_t v, vr;
1914
1915 db_printf("lapic ID = %d\n", lapic_id());
1916 v = lapic_read32(LAPIC_VERSION);
1917 db_printf("version = %d.%d (%#x) \n", (v & APIC_VER_VERSION) >> 4,
1918 v & 0xf, v);
1919 db_printf("max LVT = %d\n", lapic_maxlvt(v));
1920 vr = lapic_read32(LAPIC_SVR);
1921 db_printf("SVR = %02x (%s)\n", vr & APIC_SVR_VECTOR,
1922 vr & APIC_SVR_ENABLE ? "enabled" : "disabled");
1923 db_printf("TPR = %02x\n", lapic_read32(LAPIC_TPR));
1924
1925 lvts_count = min(nitems(lvts), lapic_maxlvt(v) + 1);
1926 for (i = 0; i < lvts_count; i++) {
1927 l = &lvts[i];
1928 db_printf("LVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg,
1929 l->lvt_desc, lapic_read32(l->lvt_reg));
1930 }
1931
1932 elvt_count = amd_read_elvt_count();
1933 for (i = 0; i < elvt_count; i++) {
1934 l = &elvts[i];
1935 db_printf("ELVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg,
1936 l->lvt_desc, lapic_read32(l->lvt_reg));
1937 }
1938
1939 #define dump_field(prefix, regn, index) \
1940 dump_mask(__XSTRING(prefix ## index), \
1941 lapic_read32(LAPIC_ ## regn ## index), \
1942 index * 32)
1943
1944 db_printf("In-service Interrupts:\n");
1945 dump_field(isr, ISR, 0);
1946 dump_field(isr, ISR, 1);
1947 dump_field(isr, ISR, 2);
1948 dump_field(isr, ISR, 3);
1949 dump_field(isr, ISR, 4);
1950 dump_field(isr, ISR, 5);
1951 dump_field(isr, ISR, 6);
1952 dump_field(isr, ISR, 7);
1953
1954 db_printf("TMR Interrupts:\n");
1955 dump_field(tmr, TMR, 0);
1956 dump_field(tmr, TMR, 1);
1957 dump_field(tmr, TMR, 2);
1958 dump_field(tmr, TMR, 3);
1959 dump_field(tmr, TMR, 4);
1960 dump_field(tmr, TMR, 5);
1961 dump_field(tmr, TMR, 6);
1962 dump_field(tmr, TMR, 7);
1963
1964 db_printf("IRR Interrupts:\n");
1965 dump_field(irr, IRR, 0);
1966 dump_field(irr, IRR, 1);
1967 dump_field(irr, IRR, 2);
1968 dump_field(irr, IRR, 3);
1969 dump_field(irr, IRR, 4);
1970 dump_field(irr, IRR, 5);
1971 dump_field(irr, IRR, 6);
1972 dump_field(irr, IRR, 7);
1973
1974 #undef dump_field
1975 }
1976 #endif
1977
1978 /*
1979 * APIC probing support code. This includes code to manage enumerators.
1980 */
1981
1982 static SLIST_HEAD(, apic_enumerator) enumerators =
1983 SLIST_HEAD_INITIALIZER(enumerators);
1984 static struct apic_enumerator *best_enum;
1985
1986 void
apic_register_enumerator(struct apic_enumerator * enumerator)1987 apic_register_enumerator(struct apic_enumerator *enumerator)
1988 {
1989 #ifdef INVARIANTS
1990 struct apic_enumerator *apic_enum;
1991
1992 SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
1993 if (apic_enum == enumerator)
1994 panic("%s: Duplicate register of %s", __func__,
1995 enumerator->apic_name);
1996 }
1997 #endif
1998 SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
1999 }
2000
2001 /*
2002 * We have to look for CPU's very, very early because certain subsystems
2003 * want to know how many CPU's we have extremely early on in the boot
2004 * process.
2005 */
2006 static void
apic_init(void * dummy __unused)2007 apic_init(void *dummy __unused)
2008 {
2009 struct apic_enumerator *enumerator;
2010 int retval, best;
2011
2012 /* We only support built in local APICs. */
2013 if (!(cpu_feature & CPUID_APIC))
2014 return;
2015
2016 /* Don't probe if APIC mode is disabled. */
2017 if (resource_disabled("apic", 0))
2018 return;
2019
2020 /* Probe all the enumerators to find the best match. */
2021 best_enum = NULL;
2022 best = 0;
2023 SLIST_FOREACH(enumerator, &enumerators, apic_next) {
2024 retval = enumerator->apic_probe();
2025 if (retval > 0)
2026 continue;
2027 if (best_enum == NULL || best < retval) {
2028 best_enum = enumerator;
2029 best = retval;
2030 }
2031 }
2032 if (best_enum == NULL) {
2033 if (bootverbose)
2034 printf("APIC: Could not find any APICs.\n");
2035 #ifndef DEV_ATPIC
2036 panic("running without device atpic requires a local APIC");
2037 #endif
2038 return;
2039 }
2040
2041 if (bootverbose)
2042 printf("APIC: Using the %s enumerator.\n",
2043 best_enum->apic_name);
2044
2045 #ifdef I686_CPU
2046 /*
2047 * To work around an errata, we disable the local APIC on some
2048 * CPUs during early startup. We need to turn the local APIC back
2049 * on on such CPUs now.
2050 */
2051 ppro_reenable_apic();
2052 #endif
2053
2054 /* Probe the CPU's in the system. */
2055 retval = best_enum->apic_probe_cpus();
2056 if (retval != 0)
2057 printf("%s: Failed to probe CPUs: returned %d\n",
2058 best_enum->apic_name, retval);
2059
2060 }
2061 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL);
2062
2063 /*
2064 * Setup the local APIC. We have to do this prior to starting up the APs
2065 * in the SMP case.
2066 */
2067 static void
apic_setup_local(void * dummy __unused)2068 apic_setup_local(void *dummy __unused)
2069 {
2070 int retval;
2071
2072 if (best_enum == NULL)
2073 return;
2074
2075 lapics = malloc(sizeof(*lapics) * (max_apic_id + 1), M_LAPIC,
2076 M_WAITOK | M_ZERO);
2077
2078 /* Initialize the local APIC. */
2079 retval = best_enum->apic_setup_local();
2080 if (retval != 0)
2081 printf("%s: Failed to setup the local APIC: returned %d\n",
2082 best_enum->apic_name, retval);
2083 }
2084 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
2085
2086 /*
2087 * Setup the I/O APICs.
2088 */
2089 static void
apic_setup_io(void * dummy __unused)2090 apic_setup_io(void *dummy __unused)
2091 {
2092 int retval;
2093
2094 if (best_enum == NULL)
2095 return;
2096
2097 /*
2098 * Local APIC must be registered before other PICs and pseudo PICs
2099 * for proper suspend/resume order.
2100 */
2101 intr_register_pic(&lapic_pic);
2102
2103 retval = best_enum->apic_setup_io();
2104 if (retval != 0)
2105 printf("%s: Failed to setup I/O APICs: returned %d\n",
2106 best_enum->apic_name, retval);
2107
2108 /*
2109 * Finish setting up the local APIC on the BSP once we know
2110 * how to properly program the LINT pins. In particular, this
2111 * enables the EOI suppression mode, if LAPIC supports it and
2112 * user did not disable the mode.
2113 */
2114 lapic_setup(1);
2115 if (bootverbose)
2116 lapic_dump("BSP");
2117
2118 /* Enable the MSI "pic". */
2119 msi_init();
2120
2121 #ifdef XENHVM
2122 xen_intr_alloc_irqs();
2123 #endif
2124 }
2125 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
2126
2127 #ifdef SMP
2128 /*
2129 * Inter Processor Interrupt functions. The lapic_ipi_*() functions are
2130 * private to the MD code. The public interface for the rest of the
2131 * kernel is defined in mp_machdep.c.
2132 */
2133
2134 /*
2135 * Wait delay microseconds for IPI to be sent. If delay is -1, we
2136 * wait forever.
2137 */
2138 int
lapic_ipi_wait(int delay)2139 lapic_ipi_wait(int delay)
2140 {
2141 uint64_t rx;
2142
2143 /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
2144 if (x2apic_mode)
2145 return (1);
2146
2147 for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
2148 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
2149 APIC_DELSTAT_IDLE)
2150 return (1);
2151 ia32_pause();
2152 }
2153 return (0);
2154 }
2155
2156 void
lapic_ipi_raw(register_t icrlo,u_int dest)2157 lapic_ipi_raw(register_t icrlo, u_int dest)
2158 {
2159 uint32_t icrhi;
2160
2161 /* XXX: Need more sanity checking of icrlo? */
2162 KASSERT(x2apic_mode || lapic_map != NULL,
2163 ("%s called too early", __func__));
2164 KASSERT(x2apic_mode ||
2165 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2166 ("%s: invalid dest field", __func__));
2167 KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
2168 ("%s: reserved bits set in ICR LO register", __func__));
2169
2170 if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
2171 if (x2apic_mode)
2172 icrhi = dest;
2173 else
2174 icrhi = dest << APIC_ID_SHIFT;
2175 lapic_write_icr(icrhi, icrlo);
2176 } else {
2177 lapic_write_icr_lo(icrlo);
2178 }
2179 }
2180
2181 #ifdef DETECT_DEADLOCK
2182 #define AFTER_SPIN 50
2183 #endif
2184
2185 static void
native_lapic_ipi_vectored(u_int vector,int dest)2186 native_lapic_ipi_vectored(u_int vector, int dest)
2187 {
2188 register_t icrlo, destfield;
2189
2190 KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
2191 ("%s: invalid vector %d", __func__, vector));
2192
2193 destfield = 0;
2194 switch (dest) {
2195 case APIC_IPI_DEST_SELF:
2196 if (x2apic_mode && vector < IPI_NMI_FIRST) {
2197 lapic_write_self_ipi(vector);
2198 return;
2199 }
2200 icrlo = APIC_DEST_SELF;
2201 break;
2202 case APIC_IPI_DEST_ALL:
2203 icrlo = APIC_DEST_ALLISELF;
2204 break;
2205 case APIC_IPI_DEST_OTHERS:
2206 icrlo = APIC_DEST_ALLESELF;
2207 break;
2208 default:
2209 icrlo = 0;
2210 KASSERT(x2apic_mode ||
2211 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2212 ("%s: invalid destination 0x%x", __func__, dest));
2213 destfield = dest;
2214 }
2215
2216 /*
2217 * NMI IPIs are just fake vectors used to send a NMI. Use special rules
2218 * regarding NMIs if passed, otherwise specify the vector.
2219 */
2220 if (vector >= IPI_NMI_FIRST)
2221 icrlo |= APIC_DELMODE_NMI;
2222 else
2223 icrlo |= vector | APIC_DELMODE_FIXED;
2224 icrlo |= APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
2225
2226 /* Wait for an earlier IPI to finish. */
2227 if (!lapic_ipi_wait(lapic_ds_idle_timeout)) {
2228 if (KERNEL_PANICKED())
2229 return;
2230 else
2231 panic("APIC: Previous IPI is stuck");
2232 }
2233
2234 lapic_ipi_raw(icrlo, destfield);
2235
2236 #ifdef DETECT_DEADLOCK
2237 /* Wait for IPI to be delivered. */
2238 if (!lapic_ipi_wait(AFTER_SPIN)) {
2239 #ifdef needsattention
2240 /*
2241 * XXX FIXME:
2242 *
2243 * The above function waits for the message to actually be
2244 * delivered. It breaks out after an arbitrary timeout
2245 * since the message should eventually be delivered (at
2246 * least in theory) and that if it wasn't we would catch
2247 * the failure with the check above when the next IPI is
2248 * sent.
2249 *
2250 * We could skip this wait entirely, EXCEPT it probably
2251 * protects us from other routines that assume that the
2252 * message was delivered and acted upon when this function
2253 * returns.
2254 */
2255 printf("APIC: IPI might be stuck\n");
2256 #else /* !needsattention */
2257 /* Wait until mesage is sent without a timeout. */
2258 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
2259 ia32_pause();
2260 #endif /* needsattention */
2261 }
2262 #endif /* DETECT_DEADLOCK */
2263 }
2264
2265 void (*ipi_vectored)(u_int, int) = &native_lapic_ipi_vectored;
2266 #endif /* SMP */
2267
2268 /*
2269 * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
2270 * visible.
2271 *
2272 * Consider the case where an IPI is generated immediately after allocation:
2273 * vector = lapic_ipi_alloc(ipifunc);
2274 * ipi_selected(other_cpus, vector);
2275 *
2276 * In xAPIC mode a write to ICR_LO has serializing semantics because the
2277 * APIC page is mapped as an uncached region. In x2APIC mode there is an
2278 * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
2279 * the IDT slot update is globally visible before the IPI is delivered.
2280 */
2281 int
lapic_ipi_alloc(inthand_t * ipifunc)2282 lapic_ipi_alloc(inthand_t *ipifunc)
2283 {
2284 struct gate_descriptor *ip;
2285 long func;
2286 int idx, vector;
2287
2288 KASSERT(ipifunc != &IDTVEC(rsvd) && ipifunc != &IDTVEC(rsvd_pti),
2289 ("invalid ipifunc %p", ipifunc));
2290
2291 vector = -1;
2292 mtx_lock_spin(&icu_lock);
2293 for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
2294 ip = &idt[idx];
2295 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2296 #ifdef __i386__
2297 func -= setidt_disp;
2298 #endif
2299 if ((!pti && func == (uintptr_t)&IDTVEC(rsvd)) ||
2300 (pti && func == (uintptr_t)&IDTVEC(rsvd_pti))) {
2301 vector = idx;
2302 setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
2303 break;
2304 }
2305 }
2306 mtx_unlock_spin(&icu_lock);
2307 return (vector);
2308 }
2309
2310 void
lapic_ipi_free(int vector)2311 lapic_ipi_free(int vector)
2312 {
2313 struct gate_descriptor *ip;
2314 long func __diagused;
2315
2316 KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
2317 ("%s: invalid vector %d", __func__, vector));
2318
2319 mtx_lock_spin(&icu_lock);
2320 ip = &idt[vector];
2321 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2322 #ifdef __i386__
2323 func -= setidt_disp;
2324 #endif
2325 KASSERT(func != (uintptr_t)&IDTVEC(rsvd) &&
2326 func != (uintptr_t)&IDTVEC(rsvd_pti),
2327 ("invalid idtfunc %#lx", func));
2328 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC,
2329 SEL_KPL, GSEL_APIC);
2330 mtx_unlock_spin(&icu_lock);
2331 }
2332