1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include "opt_bhyve_snapshot.h"
32
33 #include <sys/param.h>
34 #include <sys/queue.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40
41 #include <x86/apicreg.h>
42 #include <machine/vmm.h>
43 #include <machine/vmm_snapshot.h>
44
45 #include <dev/vmm/vmm_ktr.h>
46 #include <dev/vmm/vmm_vm.h>
47
48 #include "vmm_lapic.h"
49 #include "vlapic.h"
50 #include "vioapic.h"
51
52 #define IOREGSEL 0x00
53 #define IOWIN 0x10
54
55 #define REDIR_ENTRIES 32
56 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
57
58 struct vioapic {
59 struct vm *vm;
60 struct mtx mtx;
61 uint32_t id;
62 uint32_t ioregsel;
63 struct {
64 uint64_t reg;
65 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */
66 } rtbl[REDIR_ENTRIES];
67 };
68
69 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx))
70 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx))
71 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx))
72
73 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
74
75 #define VIOAPIC_CTR1(vioapic, fmt, a1) \
76 VM_CTR1((vioapic)->vm, fmt, a1)
77
78 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \
79 VM_CTR2((vioapic)->vm, fmt, a1, a2)
80
81 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \
82 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
83
84 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \
85 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
86
87 #ifdef KTR
88 static const char *
pinstate_str(bool asserted)89 pinstate_str(bool asserted)
90 {
91
92 if (asserted)
93 return ("asserted");
94 else
95 return ("deasserted");
96 }
97 #endif
98
99 static void
vioapic_send_intr(struct vioapic * vioapic,int pin)100 vioapic_send_intr(struct vioapic *vioapic, int pin)
101 {
102 int vector, delmode;
103 uint32_t low, high, dest;
104 bool level, phys;
105
106 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
107 ("vioapic_set_pinstate: invalid pin number %d", pin));
108
109 KASSERT(VIOAPIC_LOCKED(vioapic),
110 ("vioapic_set_pinstate: vioapic is not locked"));
111
112 low = vioapic->rtbl[pin].reg;
113 high = vioapic->rtbl[pin].reg >> 32;
114
115 if ((low & IOART_INTMASK) == IOART_INTMSET) {
116 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
117 return;
118 }
119
120 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
121 delmode = low & IOART_DELMOD;
122 level = low & IOART_TRGRLVL ? true : false;
123 if (level) {
124 if ((low & IOART_REM_IRR) != 0) {
125 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending",
126 pin);
127 return;
128 }
129 vioapic->rtbl[pin].reg |= IOART_REM_IRR;
130 }
131
132 vector = low & IOART_INTVEC;
133 dest = high >> APIC_ID_SHIFT;
134 /*
135 * Ideally we'd just call lapic_intr_msi() here with the
136 * constructed MSI instead of interpreting it for ourselves.
137 * But until/unless we support emulated IOMMUs with interrupt
138 * remapping, interpretation is simple. We just need to mask
139 * in the Extended Destination ID bits for the 15-bit
140 * enlightenment (http://david.woodhou.se/ExtDestId.pdf)
141 */
142 dest |= ((high & APIC_EXT_ID_MASK) >> APIC_EXT_ID_SHIFT) << 8;
143 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
144 }
145
146 static void
vioapic_set_pinstate(struct vioapic * vioapic,int pin,bool newstate)147 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
148 {
149 int oldcnt, newcnt;
150 bool needintr;
151
152 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
153 ("vioapic_set_pinstate: invalid pin number %d", pin));
154
155 KASSERT(VIOAPIC_LOCKED(vioapic),
156 ("vioapic_set_pinstate: vioapic is not locked"));
157
158 oldcnt = vioapic->rtbl[pin].acnt;
159 if (newstate)
160 vioapic->rtbl[pin].acnt++;
161 else
162 vioapic->rtbl[pin].acnt--;
163 newcnt = vioapic->rtbl[pin].acnt;
164
165 if (newcnt < 0) {
166 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
167 pin, newcnt);
168 }
169
170 needintr = false;
171 if (oldcnt == 0 && newcnt == 1) {
172 needintr = true;
173 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
174 } else if (oldcnt == 1 && newcnt == 0) {
175 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
176 } else {
177 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
178 pin, pinstate_str(newstate), newcnt);
179 }
180
181 if (needintr)
182 vioapic_send_intr(vioapic, pin);
183 }
184
185 enum irqstate {
186 IRQSTATE_ASSERT,
187 IRQSTATE_DEASSERT,
188 IRQSTATE_PULSE
189 };
190
191 static int
vioapic_set_irqstate(struct vm * vm,int irq,enum irqstate irqstate)192 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
193 {
194 struct vioapic *vioapic;
195
196 if (irq < 0 || irq >= REDIR_ENTRIES)
197 return (EINVAL);
198
199 vioapic = vm_ioapic(vm);
200
201 VIOAPIC_LOCK(vioapic);
202 switch (irqstate) {
203 case IRQSTATE_ASSERT:
204 vioapic_set_pinstate(vioapic, irq, true);
205 break;
206 case IRQSTATE_DEASSERT:
207 vioapic_set_pinstate(vioapic, irq, false);
208 break;
209 case IRQSTATE_PULSE:
210 vioapic_set_pinstate(vioapic, irq, true);
211 vioapic_set_pinstate(vioapic, irq, false);
212 break;
213 default:
214 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
215 }
216 VIOAPIC_UNLOCK(vioapic);
217
218 return (0);
219 }
220
221 int
vioapic_assert_irq(struct vm * vm,int irq)222 vioapic_assert_irq(struct vm *vm, int irq)
223 {
224
225 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
226 }
227
228 int
vioapic_deassert_irq(struct vm * vm,int irq)229 vioapic_deassert_irq(struct vm *vm, int irq)
230 {
231
232 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
233 }
234
235 int
vioapic_pulse_irq(struct vm * vm,int irq)236 vioapic_pulse_irq(struct vm *vm, int irq)
237 {
238
239 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
240 }
241
242 /*
243 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
244 * configuration.
245 */
246 static void
vioapic_update_tmr(struct vcpu * vcpu,void * arg)247 vioapic_update_tmr(struct vcpu *vcpu, void *arg)
248 {
249 struct vioapic *vioapic;
250 struct vlapic *vlapic;
251 uint32_t low, high, dest;
252 int delmode, pin, vector;
253 bool level, phys;
254
255 vlapic = vm_lapic(vcpu);
256 vioapic = vm_ioapic(vcpu_vm(vcpu));
257
258 VIOAPIC_LOCK(vioapic);
259 /*
260 * Reset all vectors to be edge-triggered.
261 */
262 vlapic_reset_tmr(vlapic);
263 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
264 low = vioapic->rtbl[pin].reg;
265 high = vioapic->rtbl[pin].reg >> 32;
266
267 level = low & IOART_TRGRLVL ? true : false;
268 if (!level)
269 continue;
270
271 /*
272 * For a level-triggered 'pin' let the vlapic figure out if
273 * an assertion on this 'pin' would result in an interrupt
274 * being delivered to it. If yes, then it will modify the
275 * TMR bit associated with this vector to level-triggered.
276 */
277 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
278 delmode = low & IOART_DELMOD;
279 vector = low & IOART_INTVEC;
280 dest = high >> APIC_ID_SHIFT;
281 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
282 }
283 VIOAPIC_UNLOCK(vioapic);
284 }
285
286 static uint32_t
vioapic_read(struct vioapic * vioapic,struct vcpu * vcpu,uint32_t addr)287 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr)
288 {
289 int regnum, pin, rshift;
290
291 regnum = addr & 0xff;
292 switch (regnum) {
293 case IOAPIC_ID:
294 return (vioapic->id);
295 break;
296 case IOAPIC_VER:
297 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
298 break;
299 case IOAPIC_ARB:
300 return (vioapic->id);
301 break;
302 default:
303 break;
304 }
305
306 /* redirection table entries */
307 if (regnum >= IOAPIC_REDTBL &&
308 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
309 pin = (regnum - IOAPIC_REDTBL) / 2;
310 if ((regnum - IOAPIC_REDTBL) % 2)
311 rshift = 32;
312 else
313 rshift = 0;
314
315 return (vioapic->rtbl[pin].reg >> rshift);
316 }
317
318 return (0);
319 }
320
321 static void
vioapic_write(struct vioapic * vioapic,struct vcpu * vcpu,uint32_t addr,uint32_t data)322 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr,
323 uint32_t data)
324 {
325 uint64_t data64, mask64;
326 uint64_t last, changed;
327 int regnum, pin, lshift;
328 cpuset_t allvcpus;
329
330 regnum = addr & 0xff;
331 switch (regnum) {
332 case IOAPIC_ID:
333 vioapic->id = data & APIC_ID_MASK;
334 break;
335 case IOAPIC_VER:
336 case IOAPIC_ARB:
337 /* readonly */
338 break;
339 default:
340 break;
341 }
342
343 /* redirection table entries */
344 if (regnum >= IOAPIC_REDTBL &&
345 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
346 pin = (regnum - IOAPIC_REDTBL) / 2;
347 if ((regnum - IOAPIC_REDTBL) % 2)
348 lshift = 32;
349 else
350 lshift = 0;
351
352 last = vioapic->rtbl[pin].reg;
353
354 data64 = (uint64_t)data << lshift;
355 mask64 = (uint64_t)0xffffffff << lshift;
356 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
357 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
358
359 /*
360 * Switching from level to edge triggering will clear the IRR
361 * bit. This is what FreeBSD will do in order to EOI an
362 * interrupt when the IO-APIC doesn't support targeted EOI (see
363 * _ioapic_eoi_source).
364 */
365 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
366 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
367 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
368
369 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
370 pin, vioapic->rtbl[pin].reg);
371
372 /*
373 * If any fields in the redirection table entry (except mask
374 * or polarity) have changed then rendezvous all the vcpus
375 * to update their vlapic trigger-mode registers.
376 */
377 changed = last ^ vioapic->rtbl[pin].reg;
378 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
379 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
380 "vlapic trigger-mode register", pin);
381 VIOAPIC_UNLOCK(vioapic);
382 allvcpus = vm_active_cpus(vioapic->vm);
383 (void)vm_smp_rendezvous(vcpu, allvcpus,
384 vioapic_update_tmr, NULL);
385 VIOAPIC_LOCK(vioapic);
386 }
387
388 /*
389 * Generate an interrupt if the following conditions are met:
390 * - pin trigger mode is level
391 * - pin level is asserted
392 */
393 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
394 (vioapic->rtbl[pin].acnt > 0)) {
395 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
396 "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
397 vioapic_send_intr(vioapic, pin);
398 }
399 }
400 }
401
402 static int
vioapic_mmio_rw(struct vioapic * vioapic,struct vcpu * vcpu,uint64_t gpa,uint64_t * data,int size,bool doread)403 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa,
404 uint64_t *data, int size, bool doread)
405 {
406 uint64_t offset;
407
408 offset = gpa - VIOAPIC_BASE;
409
410 /*
411 * The IOAPIC specification allows 32-bit wide accesses to the
412 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
413 */
414 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
415 if (doread)
416 *data = 0;
417 return (0);
418 }
419
420 VIOAPIC_LOCK(vioapic);
421 if (offset == IOREGSEL) {
422 if (doread)
423 *data = vioapic->ioregsel;
424 else
425 vioapic->ioregsel = *data;
426 } else {
427 if (doread) {
428 *data = vioapic_read(vioapic, vcpu,
429 vioapic->ioregsel);
430 } else {
431 vioapic_write(vioapic, vcpu, vioapic->ioregsel,
432 *data);
433 }
434 }
435 VIOAPIC_UNLOCK(vioapic);
436
437 return (0);
438 }
439
440 int
vioapic_mmio_read(struct vcpu * vcpu,uint64_t gpa,uint64_t * rval,int size,void * arg)441 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval,
442 int size, void *arg)
443 {
444 int error;
445 struct vioapic *vioapic;
446
447 vioapic = vm_ioapic(vcpu_vm(vcpu));
448 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true);
449 return (error);
450 }
451
452 int
vioapic_mmio_write(struct vcpu * vcpu,uint64_t gpa,uint64_t wval,int size,void * arg)453 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval,
454 int size, void *arg)
455 {
456 int error;
457 struct vioapic *vioapic;
458
459 vioapic = vm_ioapic(vcpu_vm(vcpu));
460 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false);
461 return (error);
462 }
463
464 void
vioapic_process_eoi(struct vm * vm,int vector)465 vioapic_process_eoi(struct vm *vm, int vector)
466 {
467 struct vioapic *vioapic;
468 int pin;
469
470 KASSERT(vector >= 0 && vector < 256,
471 ("vioapic_process_eoi: invalid vector %d", vector));
472
473 vioapic = vm_ioapic(vm);
474 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
475
476 /*
477 * XXX keep track of the pins associated with this vector instead
478 * of iterating on every single pin each time.
479 */
480 VIOAPIC_LOCK(vioapic);
481 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
482 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
483 continue;
484 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
485 continue;
486 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
487 if (vioapic->rtbl[pin].acnt > 0) {
488 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
489 "acnt %d", pin, vioapic->rtbl[pin].acnt);
490 vioapic_send_intr(vioapic, pin);
491 }
492 }
493 VIOAPIC_UNLOCK(vioapic);
494 }
495
496 struct vioapic *
vioapic_init(struct vm * vm)497 vioapic_init(struct vm *vm)
498 {
499 int i;
500 struct vioapic *vioapic;
501
502 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
503
504 vioapic->vm = vm;
505 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
506
507 /* Initialize all redirection entries to mask all interrupts */
508 for (i = 0; i < REDIR_ENTRIES; i++)
509 vioapic->rtbl[i].reg = 0x0001000000010000UL;
510
511 return (vioapic);
512 }
513
514 void
vioapic_cleanup(struct vioapic * vioapic)515 vioapic_cleanup(struct vioapic *vioapic)
516 {
517
518 mtx_destroy(&vioapic->mtx);
519 free(vioapic, M_VIOAPIC);
520 }
521
522 int
vioapic_pincount(struct vm * vm)523 vioapic_pincount(struct vm *vm)
524 {
525
526 return (REDIR_ENTRIES);
527 }
528
529 #ifdef BHYVE_SNAPSHOT
530 int
vioapic_snapshot(struct vioapic * vioapic,struct vm_snapshot_meta * meta)531 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta)
532 {
533 int ret;
534 int i;
535
536 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done);
537
538 for (i = 0; i < nitems(vioapic->rtbl); i++) {
539 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done);
540 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done);
541 }
542
543 done:
544 return (ret);
545 }
546 #endif
547