1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 2014-2018 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by 9 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 10 * the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: vector.s, 386BSD 0.1 unknown origin 37 */ 38 39/* 40 * Interrupt entry points for external interrupts triggered by I/O APICs 41 * as well as IPI handlers. 42 */ 43 44#include "opt_smp.h" 45 46#include "assym.inc" 47 48#include <machine/asmacros.h> 49#include <machine/specialreg.h> 50#include <x86/apicreg.h> 51 52 .text 53 SUPERALIGN_TEXT 54 /* End Of Interrupt to APIC */ 55as_lapic_eoi: 56 cmpl $0,x2apic_mode 57 jne 1f 58 movq lapic_map,%rax 59 movl $0,LA_EOI(%rax) 60 ret 611: 62 movl $MSR_APIC_EOI,%ecx 63 xorl %eax,%eax 64 xorl %edx,%edx 65 wrmsr 66 ret 67 68/* 69 * I/O Interrupt Entry Point. Rather than having one entry point for 70 * each interrupt source, we use one entry point for each 32-bit word 71 * in the ISR. The handler determines the highest bit set in the ISR, 72 * translates that into a vector, and passes the vector to the 73 * lapic_handle_intr() function. 74 */ 75 .macro ISR_VEC index, vec_name 76 INTR_HANDLER \vec_name 77 KMSAN_ENTER 78 cmpl $0,x2apic_mode 79 je 1f 80 movl $(MSR_APIC_ISR0 + \index),%ecx 81 rdmsr 82 jmp 2f 831: 84 movq lapic_map, %rdx /* pointer to local APIC */ 85 movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */ 862: 87 bsrl %eax, %eax /* index of highest set bit in ISR */ 88 jz 3f 89 addl $(32 * \index),%eax 90 movq %rsp, %rsi 91 movl %eax, %edi /* pass the IRQ */ 92 call lapic_handle_intr 933: 94 KMSAN_LEAVE 95 jmp doreti 96 .endm 97 98/* 99 * Handle "spurious INTerrupts". 100 * Notes: 101 * This is different than the "spurious INTerrupt" generated by an 102 * 8259 PIC for missing INTs. See the APIC documentation for details. 103 * This routine should NOT do an 'EOI' cycle. 104 */ 105 .text 106 SUPERALIGN_TEXT 107IDTVEC(spuriousint) 108 /* No EOI cycle used here */ 109 jmp doreti_iret 110 111 ISR_VEC 1, apic_isr1 112 ISR_VEC 2, apic_isr2 113 ISR_VEC 3, apic_isr3 114 ISR_VEC 4, apic_isr4 115 ISR_VEC 5, apic_isr5 116 ISR_VEC 6, apic_isr6 117 ISR_VEC 7, apic_isr7 118 119/* 120 * Local APIC periodic timer handler. 121 */ 122 INTR_HANDLER timerint 123 KMSAN_ENTER 124 movq %rsp, %rdi 125 call lapic_handle_timer 126 KMSAN_LEAVE 127 jmp doreti 128 129/* 130 * Local APIC CMCI handler. 131 */ 132 INTR_HANDLER cmcint 133 KMSAN_ENTER 134 call lapic_handle_cmc 135 KMSAN_LEAVE 136 jmp doreti 137 138/* 139 * Local APIC error interrupt handler. 140 */ 141 INTR_HANDLER errorint 142 KMSAN_ENTER 143 call lapic_handle_error 144 KMSAN_LEAVE 145 jmp doreti 146 147#ifdef XENHVM 148/* 149 * Xen event channel upcall interrupt handler. 150 * Only used when the hypervisor supports direct vector callbacks. 151 */ 152 INTR_HANDLER xen_intr_upcall 153 KMSAN_ENTER 154 movq %rsp, %rdi 155 call xen_arch_intr_handle_upcall 156 KMSAN_LEAVE 157 jmp doreti 158#endif 159 160/* 161 * Global address space TLB shootdown. 162 */ 163 .text 164 165 SUPERALIGN_TEXT 166/* 167 * IPI handler for cache and TLB shootdown 168 */ 169 INTR_HANDLER invlop 170 KMSAN_ENTER 171 call invlop_handler 172 call as_lapic_eoi 173 KMSAN_LEAVE 174 jmp ld_regs 175 176/* 177 * Handler for IPIs sent via the per-cpu IPI bitmap. 178 */ 179 INTR_HANDLER ipi_intr_bitmap_handler 180 call as_lapic_eoi 181 KMSAN_ENTER 182 movq %rsp,%rdi 183 call ipi_bitmap_handler 184 KMSAN_LEAVE 185 jmp doreti 186 187/* 188 * Executed by a CPU when it receives an IPI_STOP from another CPU. 189 */ 190 INTR_HANDLER cpustop 191 call as_lapic_eoi 192 KMSAN_ENTER 193 call cpustop_handler 194 KMSAN_LEAVE 195 jmp doreti 196 197/* 198 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 199 */ 200 INTR_HANDLER cpususpend 201 KMSAN_ENTER 202 call cpususpend_handler 203 call as_lapic_eoi 204 KMSAN_LEAVE 205 jmp doreti 206 207/* 208 * Executed by a CPU when it receives an IPI_OFF from another CPU. 209 * Should never return 210 */ 211 INTR_HANDLER cpuoff 212 KMSAN_ENTER 213 call cpuoff_handler 214 call as_lapic_eoi 215 KMSAN_LEAVE 216 jmp doreti 217 218/* 219 * Executed by a CPU when it receives an IPI_SWI. 220 */ 221 INTR_HANDLER ipi_swi 222 call as_lapic_eoi 223 KMSAN_ENTER 224 movq %rsp,%rdi 225 call ipi_swi_handler 226 KMSAN_LEAVE 227 jmp doreti 228 229/* 230 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 231 * 232 * - Calls the generic rendezvous action function. 233 */ 234 INTR_HANDLER rendezvous 235#ifdef COUNT_IPIS 236 movl PCPU(CPUID), %eax 237 movq ipi_rendezvous_counts(,%rax,8), %rax 238 incq (%rax) 239#endif 240 KMSAN_ENTER 241 call smp_rendezvous_action 242 call as_lapic_eoi 243 KMSAN_LEAVE 244 jmp doreti 245 246/* 247 * IPI handler whose purpose is to interrupt the CPU with minimum overhead. 248 * This is used by bhyve to force a host cpu executing in guest context to 249 * trap into the hypervisor. 250 * 251 * This handler is different from other IPI handlers in the following aspects: 252 * 253 * 1. It doesn't push a trapframe on the stack. 254 * 255 * This implies that a DDB backtrace involving 'justreturn' will skip the 256 * function that was interrupted by this handler. 257 * 258 * 2. It doesn't 'swapgs' when userspace is interrupted. 259 * 260 * The 'justreturn' handler does not access any pcpu data so it is not an 261 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI 262 * whose handler already doesn't trust GS.base when kernel code is interrupted. 263 */ 264 .text 265 SUPERALIGN_TEXT 266IDTVEC(justreturn) 267 pushq %rax 268 pushq %rcx 269 pushq %rdx 270 call as_lapic_eoi 271 popq %rdx 272 popq %rcx 273 popq %rax 274 jmp doreti_iret 275 276 INTR_HANDLER justreturn1 277 call as_lapic_eoi 278 jmp doreti 279