xref: /linux/arch/x86/kvm/irq.c (revision ab93e0dd72c37d378dd936f031ffb83ff2bd87ce)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * irq.c: API for in kernel interrupt controller
4  * Copyright (c) 2007, Intel Corporation.
5  * Copyright 2009 Red Hat, Inc. and/or its affiliates.
6  *
7  * Authors:
8  *   Yaozu (Eddie) Dong <Eddie.dong@intel.com>
9  */
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 
12 #include <linux/export.h>
13 #include <linux/kvm_host.h>
14 #include <linux/kvm_irqfd.h>
15 
16 #include "hyperv.h"
17 #include "ioapic.h"
18 #include "irq.h"
19 #include "trace.h"
20 #include "x86.h"
21 #include "xen.h"
22 
23 /*
24  * check if there are pending timer events
25  * to be processed.
26  */
kvm_cpu_has_pending_timer(struct kvm_vcpu * vcpu)27 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
28 {
29 	int r = 0;
30 
31 	if (lapic_in_kernel(vcpu))
32 		r = apic_has_pending_timer(vcpu);
33 	if (kvm_xen_timer_enabled(vcpu))
34 		r += kvm_xen_has_pending_timer(vcpu);
35 
36 	return r;
37 }
38 
39 /*
40  * check if there is a pending userspace external interrupt
41  */
pending_userspace_extint(struct kvm_vcpu * v)42 static int pending_userspace_extint(struct kvm_vcpu *v)
43 {
44 	return v->arch.pending_external_vector != -1;
45 }
46 
get_userspace_extint(struct kvm_vcpu * vcpu)47 static int get_userspace_extint(struct kvm_vcpu *vcpu)
48 {
49 	int vector = vcpu->arch.pending_external_vector;
50 
51 	vcpu->arch.pending_external_vector = -1;
52 	return vector;
53 }
54 
55 /*
56  * check if there is pending interrupt from
57  * non-APIC source without intack.
58  */
kvm_cpu_has_extint(struct kvm_vcpu * v)59 int kvm_cpu_has_extint(struct kvm_vcpu *v)
60 {
61 	/*
62 	 * FIXME: interrupt.injected represents an interrupt whose
63 	 * side-effects have already been applied (e.g. bit from IRR
64 	 * already moved to ISR). Therefore, it is incorrect to rely
65 	 * on interrupt.injected to know if there is a pending
66 	 * interrupt in the user-mode LAPIC.
67 	 * This leads to nVMX/nSVM not be able to distinguish
68 	 * if it should exit from L2 to L1 on EXTERNAL_INTERRUPT on
69 	 * pending interrupt or should re-inject an injected
70 	 * interrupt.
71 	 */
72 	if (!lapic_in_kernel(v))
73 		return v->arch.interrupt.injected;
74 
75 	if (kvm_xen_has_interrupt(v))
76 		return 1;
77 
78 	if (!kvm_apic_accept_pic_intr(v))
79 		return 0;
80 
81 #ifdef CONFIG_KVM_IOAPIC
82 	if (pic_in_kernel(v->kvm))
83 		return v->kvm->arch.vpic->output;
84 #endif
85 
86 	WARN_ON_ONCE(!irqchip_split(v->kvm));
87 	return pending_userspace_extint(v);
88 }
89 
90 /*
91  * check if there is injectable interrupt:
92  * when virtual interrupt delivery enabled,
93  * interrupt from apic will handled by hardware,
94  * we don't need to check it here.
95  */
kvm_cpu_has_injectable_intr(struct kvm_vcpu * v)96 int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v)
97 {
98 	if (kvm_cpu_has_extint(v))
99 		return 1;
100 
101 	if (!is_guest_mode(v) && kvm_vcpu_apicv_active(v))
102 		return 0;
103 
104 	return kvm_apic_has_interrupt(v) != -1; /* LAPIC */
105 }
106 EXPORT_SYMBOL_GPL(kvm_cpu_has_injectable_intr);
107 
108 /*
109  * check if there is pending interrupt without
110  * intack.
111  */
kvm_cpu_has_interrupt(struct kvm_vcpu * v)112 int kvm_cpu_has_interrupt(struct kvm_vcpu *v)
113 {
114 	if (kvm_cpu_has_extint(v))
115 		return 1;
116 
117 	if (lapic_in_kernel(v) && v->arch.apic->guest_apic_protected)
118 		return kvm_x86_call(protected_apic_has_interrupt)(v);
119 
120 	return kvm_apic_has_interrupt(v) != -1;	/* LAPIC */
121 }
122 EXPORT_SYMBOL_GPL(kvm_cpu_has_interrupt);
123 
124 /*
125  * Read pending interrupt(from non-APIC source)
126  * vector and intack.
127  */
kvm_cpu_get_extint(struct kvm_vcpu * v)128 int kvm_cpu_get_extint(struct kvm_vcpu *v)
129 {
130 	if (!kvm_cpu_has_extint(v)) {
131 		WARN_ON(!lapic_in_kernel(v));
132 		return -1;
133 	}
134 
135 	if (!lapic_in_kernel(v))
136 		return v->arch.interrupt.nr;
137 
138 #ifdef CONFIG_KVM_XEN
139 	if (kvm_xen_has_interrupt(v))
140 		return v->kvm->arch.xen.upcall_vector;
141 #endif
142 
143 #ifdef CONFIG_KVM_IOAPIC
144 	if (pic_in_kernel(v->kvm))
145 		return kvm_pic_read_irq(v->kvm); /* PIC */
146 #endif
147 
148 	WARN_ON_ONCE(!irqchip_split(v->kvm));
149 	return get_userspace_extint(v);
150 }
151 EXPORT_SYMBOL_GPL(kvm_cpu_get_extint);
152 
153 /*
154  * Read pending interrupt vector and intack.
155  */
kvm_cpu_get_interrupt(struct kvm_vcpu * v)156 int kvm_cpu_get_interrupt(struct kvm_vcpu *v)
157 {
158 	int vector = kvm_cpu_get_extint(v);
159 	if (vector != -1)
160 		return vector;			/* PIC */
161 
162 	vector = kvm_apic_has_interrupt(v);	/* APIC */
163 	if (vector != -1)
164 		kvm_apic_ack_interrupt(v, vector);
165 
166 	return vector;
167 }
168 
kvm_inject_pending_timer_irqs(struct kvm_vcpu * vcpu)169 void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu)
170 {
171 	if (lapic_in_kernel(vcpu))
172 		kvm_inject_apic_timer_irqs(vcpu);
173 	if (kvm_xen_timer_enabled(vcpu))
174 		kvm_xen_inject_timer_irqs(vcpu);
175 }
176 
__kvm_migrate_timers(struct kvm_vcpu * vcpu)177 void __kvm_migrate_timers(struct kvm_vcpu *vcpu)
178 {
179 	__kvm_migrate_apic_timer(vcpu);
180 #ifdef CONFIG_KVM_IOAPIC
181 	__kvm_migrate_pit_timer(vcpu);
182 #endif
183 	kvm_x86_call(migrate_timers)(vcpu);
184 }
185 
kvm_arch_irqfd_allowed(struct kvm * kvm,struct kvm_irqfd * args)186 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)
187 {
188 	bool resample = args->flags & KVM_IRQFD_FLAG_RESAMPLE;
189 
190 	return resample ? irqchip_full(kvm) : irqchip_in_kernel(kvm);
191 }
192 
kvm_arch_irqchip_in_kernel(struct kvm * kvm)193 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
194 {
195 	return irqchip_in_kernel(kvm);
196 }
197 
kvm_irq_delivery_to_apic(struct kvm * kvm,struct kvm_lapic * src,struct kvm_lapic_irq * irq,struct dest_map * dest_map)198 int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
199 			     struct kvm_lapic_irq *irq, struct dest_map *dest_map)
200 {
201 	int r = -1;
202 	struct kvm_vcpu *vcpu, *lowest = NULL;
203 	unsigned long i, dest_vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)];
204 	unsigned int dest_vcpus = 0;
205 
206 	if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map))
207 		return r;
208 
209 	if (irq->dest_mode == APIC_DEST_PHYSICAL &&
210 	    irq->dest_id == 0xff && kvm_lowest_prio_delivery(irq)) {
211 		pr_info("apic: phys broadcast and lowest prio\n");
212 		irq->delivery_mode = APIC_DM_FIXED;
213 	}
214 
215 	memset(dest_vcpu_bitmap, 0, sizeof(dest_vcpu_bitmap));
216 
217 	kvm_for_each_vcpu(i, vcpu, kvm) {
218 		if (!kvm_apic_present(vcpu))
219 			continue;
220 
221 		if (!kvm_apic_match_dest(vcpu, src, irq->shorthand,
222 					irq->dest_id, irq->dest_mode))
223 			continue;
224 
225 		if (!kvm_lowest_prio_delivery(irq)) {
226 			if (r < 0)
227 				r = 0;
228 			r += kvm_apic_set_irq(vcpu, irq, dest_map);
229 		} else if (kvm_apic_sw_enabled(vcpu->arch.apic)) {
230 			if (!kvm_vector_hashing_enabled()) {
231 				if (!lowest)
232 					lowest = vcpu;
233 				else if (kvm_apic_compare_prio(vcpu, lowest) < 0)
234 					lowest = vcpu;
235 			} else {
236 				__set_bit(i, dest_vcpu_bitmap);
237 				dest_vcpus++;
238 			}
239 		}
240 	}
241 
242 	if (dest_vcpus != 0) {
243 		int idx = kvm_vector_to_index(irq->vector, dest_vcpus,
244 					dest_vcpu_bitmap, KVM_MAX_VCPUS);
245 
246 		lowest = kvm_get_vcpu(kvm, idx);
247 	}
248 
249 	if (lowest)
250 		r = kvm_apic_set_irq(lowest, irq, dest_map);
251 
252 	return r;
253 }
254 
kvm_msi_to_lapic_irq(struct kvm * kvm,struct kvm_kernel_irq_routing_entry * e,struct kvm_lapic_irq * irq)255 static void kvm_msi_to_lapic_irq(struct kvm *kvm,
256 				 struct kvm_kernel_irq_routing_entry *e,
257 				 struct kvm_lapic_irq *irq)
258 {
259 	struct msi_msg msg = { .address_lo = e->msi.address_lo,
260 			       .address_hi = e->msi.address_hi,
261 			       .data = e->msi.data };
262 
263 	trace_kvm_msi_set_irq(msg.address_lo | (kvm->arch.x2apic_format ?
264 			      (u64)msg.address_hi << 32 : 0), msg.data);
265 
266 	irq->dest_id = x86_msi_msg_get_destid(&msg, kvm->arch.x2apic_format);
267 	irq->vector = msg.arch_data.vector;
268 	irq->dest_mode = kvm_lapic_irq_dest_mode(msg.arch_addr_lo.dest_mode_logical);
269 	irq->trig_mode = msg.arch_data.is_level;
270 	irq->delivery_mode = msg.arch_data.delivery_mode << 8;
271 	irq->msi_redir_hint = msg.arch_addr_lo.redirect_hint;
272 	irq->level = 1;
273 	irq->shorthand = APIC_DEST_NOSHORT;
274 }
275 
kvm_msi_route_invalid(struct kvm * kvm,struct kvm_kernel_irq_routing_entry * e)276 static inline bool kvm_msi_route_invalid(struct kvm *kvm,
277 		struct kvm_kernel_irq_routing_entry *e)
278 {
279 	return kvm->arch.x2apic_format && (e->msi.address_hi & 0xff);
280 }
281 
kvm_set_msi(struct kvm_kernel_irq_routing_entry * e,struct kvm * kvm,int irq_source_id,int level,bool line_status)282 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
283 		struct kvm *kvm, int irq_source_id, int level, bool line_status)
284 {
285 	struct kvm_lapic_irq irq;
286 
287 	if (kvm_msi_route_invalid(kvm, e))
288 		return -EINVAL;
289 
290 	if (!level)
291 		return -1;
292 
293 	kvm_msi_to_lapic_irq(kvm, e, &irq);
294 
295 	return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL);
296 }
297 
kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry * e,struct kvm * kvm,int irq_source_id,int level,bool line_status)298 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
299 			      struct kvm *kvm, int irq_source_id, int level,
300 			      bool line_status)
301 {
302 	struct kvm_lapic_irq irq;
303 	int r;
304 
305 	switch (e->type) {
306 #ifdef CONFIG_KVM_HYPERV
307 	case KVM_IRQ_ROUTING_HV_SINT:
308 		return kvm_hv_synic_set_irq(e, kvm, irq_source_id, level,
309 					    line_status);
310 #endif
311 
312 	case KVM_IRQ_ROUTING_MSI:
313 		if (kvm_msi_route_invalid(kvm, e))
314 			return -EINVAL;
315 
316 		kvm_msi_to_lapic_irq(kvm, e, &irq);
317 
318 		if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL))
319 			return r;
320 		break;
321 
322 #ifdef CONFIG_KVM_XEN
323 	case KVM_IRQ_ROUTING_XEN_EVTCHN:
324 		if (!level)
325 			return -1;
326 
327 		return kvm_xen_set_evtchn_fast(&e->xen_evtchn, kvm);
328 #endif
329 	default:
330 		break;
331 	}
332 
333 	return -EWOULDBLOCK;
334 }
335 
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_event,bool line_status)336 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
337 			bool line_status)
338 {
339 	if (!irqchip_in_kernel(kvm))
340 		return -ENXIO;
341 
342 	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
343 					irq_event->irq, irq_event->level,
344 					line_status);
345 	return 0;
346 }
347 
kvm_arch_can_set_irq_routing(struct kvm * kvm)348 bool kvm_arch_can_set_irq_routing(struct kvm *kvm)
349 {
350 	return irqchip_in_kernel(kvm);
351 }
352 
kvm_set_routing_entry(struct kvm * kvm,struct kvm_kernel_irq_routing_entry * e,const struct kvm_irq_routing_entry * ue)353 int kvm_set_routing_entry(struct kvm *kvm,
354 			  struct kvm_kernel_irq_routing_entry *e,
355 			  const struct kvm_irq_routing_entry *ue)
356 {
357 	/* We can't check irqchip_in_kernel() here as some callers are
358 	 * currently initializing the irqchip. Other callers should therefore
359 	 * check kvm_arch_can_set_irq_routing() before calling this function.
360 	 */
361 	switch (ue->type) {
362 #ifdef CONFIG_KVM_IOAPIC
363 	case KVM_IRQ_ROUTING_IRQCHIP:
364 		if (irqchip_split(kvm))
365 			return -EINVAL;
366 		e->irqchip.pin = ue->u.irqchip.pin;
367 		switch (ue->u.irqchip.irqchip) {
368 		case KVM_IRQCHIP_PIC_SLAVE:
369 			e->irqchip.pin += PIC_NUM_PINS / 2;
370 			fallthrough;
371 		case KVM_IRQCHIP_PIC_MASTER:
372 			if (ue->u.irqchip.pin >= PIC_NUM_PINS / 2)
373 				return -EINVAL;
374 			e->set = kvm_pic_set_irq;
375 			break;
376 		case KVM_IRQCHIP_IOAPIC:
377 			if (ue->u.irqchip.pin >= KVM_IOAPIC_NUM_PINS)
378 				return -EINVAL;
379 			e->set = kvm_ioapic_set_irq;
380 			break;
381 		default:
382 			return -EINVAL;
383 		}
384 		e->irqchip.irqchip = ue->u.irqchip.irqchip;
385 		break;
386 #endif
387 	case KVM_IRQ_ROUTING_MSI:
388 		e->set = kvm_set_msi;
389 		e->msi.address_lo = ue->u.msi.address_lo;
390 		e->msi.address_hi = ue->u.msi.address_hi;
391 		e->msi.data = ue->u.msi.data;
392 
393 		if (kvm_msi_route_invalid(kvm, e))
394 			return -EINVAL;
395 		break;
396 #ifdef CONFIG_KVM_HYPERV
397 	case KVM_IRQ_ROUTING_HV_SINT:
398 		e->set = kvm_hv_synic_set_irq;
399 		e->hv_sint.vcpu = ue->u.hv_sint.vcpu;
400 		e->hv_sint.sint = ue->u.hv_sint.sint;
401 		break;
402 #endif
403 #ifdef CONFIG_KVM_XEN
404 	case KVM_IRQ_ROUTING_XEN_EVTCHN:
405 		return kvm_xen_setup_evtchn(kvm, e, ue);
406 #endif
407 	default:
408 		return -EINVAL;
409 	}
410 
411 	return 0;
412 }
413 
kvm_intr_is_single_vcpu(struct kvm * kvm,struct kvm_lapic_irq * irq,struct kvm_vcpu ** dest_vcpu)414 bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
415 			     struct kvm_vcpu **dest_vcpu)
416 {
417 	int r = 0;
418 	unsigned long i;
419 	struct kvm_vcpu *vcpu;
420 
421 	if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu))
422 		return true;
423 
424 	kvm_for_each_vcpu(i, vcpu, kvm) {
425 		if (!kvm_apic_present(vcpu))
426 			continue;
427 
428 		if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand,
429 					irq->dest_id, irq->dest_mode))
430 			continue;
431 
432 		if (++r == 2)
433 			return false;
434 
435 		*dest_vcpu = vcpu;
436 	}
437 
438 	return r == 1;
439 }
440 EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu);
441 
kvm_scan_ioapic_irq(struct kvm_vcpu * vcpu,u32 dest_id,u16 dest_mode,u8 vector,unsigned long * ioapic_handled_vectors)442 void kvm_scan_ioapic_irq(struct kvm_vcpu *vcpu, u32 dest_id, u16 dest_mode,
443 			 u8 vector, unsigned long *ioapic_handled_vectors)
444 {
445 	/*
446 	 * Intercept EOI if the vCPU is the target of the new IRQ routing, or
447 	 * the vCPU has a pending IRQ from the old routing, i.e. if the vCPU
448 	 * may receive a level-triggered IRQ in the future, or already received
449 	 * level-triggered IRQ.  The EOI needs to be intercepted and forwarded
450 	 * to I/O APIC emulation so that the IRQ can be de-asserted.
451 	 */
452 	if (kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT, dest_id, dest_mode)) {
453 		__set_bit(vector, ioapic_handled_vectors);
454 	} else if (kvm_apic_pending_eoi(vcpu, vector)) {
455 		__set_bit(vector, ioapic_handled_vectors);
456 
457 		/*
458 		 * Track the highest pending EOI for which the vCPU is NOT the
459 		 * target in the new routing.  Only the EOI for the IRQ that is
460 		 * in-flight (for the old routing) needs to be intercepted, any
461 		 * future IRQs that arrive on this vCPU will be coincidental to
462 		 * the level-triggered routing and don't need to be intercepted.
463 		 */
464 		if ((int)vector > vcpu->arch.highest_stale_pending_ioapic_eoi)
465 			vcpu->arch.highest_stale_pending_ioapic_eoi = vector;
466 	}
467 }
468 
kvm_scan_ioapic_routes(struct kvm_vcpu * vcpu,ulong * ioapic_handled_vectors)469 void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu,
470 			    ulong *ioapic_handled_vectors)
471 {
472 	struct kvm *kvm = vcpu->kvm;
473 	struct kvm_kernel_irq_routing_entry *entry;
474 	struct kvm_irq_routing_table *table;
475 	u32 i, nr_ioapic_pins;
476 	int idx;
477 
478 	idx = srcu_read_lock(&kvm->irq_srcu);
479 	table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
480 	nr_ioapic_pins = min_t(u32, table->nr_rt_entries,
481 			       kvm->arch.nr_reserved_ioapic_pins);
482 	for (i = 0; i < nr_ioapic_pins; ++i) {
483 		hlist_for_each_entry(entry, &table->map[i], link) {
484 			struct kvm_lapic_irq irq;
485 
486 			if (entry->type != KVM_IRQ_ROUTING_MSI)
487 				continue;
488 
489 			kvm_msi_to_lapic_irq(vcpu->kvm, entry, &irq);
490 
491 			if (!irq.trig_mode)
492 				continue;
493 
494 			kvm_scan_ioapic_irq(vcpu, irq.dest_id, irq.dest_mode,
495 					    irq.vector, ioapic_handled_vectors);
496 		}
497 	}
498 	srcu_read_unlock(&kvm->irq_srcu, idx);
499 }
500 
kvm_arch_irq_routing_update(struct kvm * kvm)501 void kvm_arch_irq_routing_update(struct kvm *kvm)
502 {
503 #ifdef CONFIG_KVM_HYPERV
504 	kvm_hv_irq_routing_update(kvm);
505 #endif
506 
507 	if (irqchip_split(kvm))
508 		kvm_make_scan_ioapic_request(kvm);
509 }
510 
kvm_pi_update_irte(struct kvm_kernel_irqfd * irqfd,struct kvm_kernel_irq_routing_entry * entry)511 static int kvm_pi_update_irte(struct kvm_kernel_irqfd *irqfd,
512 			      struct kvm_kernel_irq_routing_entry *entry)
513 {
514 	unsigned int host_irq = irqfd->producer->irq;
515 	struct kvm *kvm = irqfd->kvm;
516 	struct kvm_vcpu *vcpu = NULL;
517 	struct kvm_lapic_irq irq;
518 	int r;
519 
520 	if (WARN_ON_ONCE(!irqchip_in_kernel(kvm) || !kvm_arch_has_irq_bypass()))
521 		return -EINVAL;
522 
523 	if (entry && entry->type == KVM_IRQ_ROUTING_MSI) {
524 		kvm_msi_to_lapic_irq(kvm, entry, &irq);
525 
526 		/*
527 		 * Force remapped mode if hardware doesn't support posting the
528 		 * virtual interrupt to a vCPU.  Only IRQs are postable (NMIs,
529 		 * SMIs, etc. are not), and neither AMD nor Intel IOMMUs support
530 		 * posting multicast/broadcast IRQs.  If the interrupt can't be
531 		 * posted, the device MSI needs to be routed to the host so that
532 		 * the guest's desired interrupt can be synthesized by KVM.
533 		 *
534 		 * This means that KVM can only post lowest-priority interrupts
535 		 * if they have a single CPU as the destination, e.g. only if
536 		 * the guest has affined the interrupt to a single vCPU.
537 		 */
538 		if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
539 		    !kvm_irq_is_postable(&irq))
540 			vcpu = NULL;
541 	}
542 
543 	if (!irqfd->irq_bypass_vcpu && !vcpu)
544 		return 0;
545 
546 	r = kvm_x86_call(pi_update_irte)(irqfd, irqfd->kvm, host_irq, irqfd->gsi,
547 					 vcpu, irq.vector);
548 	if (r) {
549 		WARN_ON_ONCE(irqfd->irq_bypass_vcpu && !vcpu);
550 		irqfd->irq_bypass_vcpu = NULL;
551 		return r;
552 	}
553 
554 	irqfd->irq_bypass_vcpu = vcpu;
555 
556 	trace_kvm_pi_irte_update(host_irq, vcpu, irqfd->gsi, irq.vector, !!vcpu);
557 	return 0;
558 }
559 
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)560 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
561 				      struct irq_bypass_producer *prod)
562 {
563 	struct kvm_kernel_irqfd *irqfd =
564 		container_of(cons, struct kvm_kernel_irqfd, consumer);
565 	struct kvm *kvm = irqfd->kvm;
566 	int ret = 0;
567 
568 	spin_lock_irq(&kvm->irqfds.lock);
569 	irqfd->producer = prod;
570 
571 	if (!kvm->arch.nr_possible_bypass_irqs++)
572 		kvm_x86_call(pi_start_bypass)(kvm);
573 
574 	if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) {
575 		ret = kvm_pi_update_irte(irqfd, &irqfd->irq_entry);
576 		if (ret)
577 			kvm->arch.nr_possible_bypass_irqs--;
578 	}
579 	spin_unlock_irq(&kvm->irqfds.lock);
580 
581 	return ret;
582 }
583 
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)584 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
585 				      struct irq_bypass_producer *prod)
586 {
587 	struct kvm_kernel_irqfd *irqfd =
588 		container_of(cons, struct kvm_kernel_irqfd, consumer);
589 	struct kvm *kvm = irqfd->kvm;
590 	int ret;
591 
592 	WARN_ON(irqfd->producer != prod);
593 
594 	/*
595 	 * If the producer of an IRQ that is currently being posted to a vCPU
596 	 * is unregistered, change the associated IRTE back to remapped mode as
597 	 * the IRQ has been released (or repurposed) by the device driver, i.e.
598 	 * KVM must relinquish control of the IRTE.
599 	 */
600 	spin_lock_irq(&kvm->irqfds.lock);
601 
602 	if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) {
603 		ret = kvm_pi_update_irte(irqfd, NULL);
604 		if (ret)
605 			pr_info("irq bypass consumer (eventfd %p) unregistration fails: %d\n",
606 				irqfd->consumer.eventfd, ret);
607 	}
608 	irqfd->producer = NULL;
609 
610 	kvm->arch.nr_possible_bypass_irqs--;
611 
612 	spin_unlock_irq(&kvm->irqfds.lock);
613 }
614 
kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd * irqfd,struct kvm_kernel_irq_routing_entry * old,struct kvm_kernel_irq_routing_entry * new)615 void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd,
616 				   struct kvm_kernel_irq_routing_entry *old,
617 				   struct kvm_kernel_irq_routing_entry *new)
618 {
619 	if (new->type != KVM_IRQ_ROUTING_MSI &&
620 	    old->type != KVM_IRQ_ROUTING_MSI)
621 		return;
622 
623 	if (old->type == KVM_IRQ_ROUTING_MSI &&
624 	    new->type == KVM_IRQ_ROUTING_MSI &&
625 	    !memcmp(&old->msi, &new->msi, sizeof(new->msi)))
626 		return;
627 
628 	kvm_pi_update_irte(irqfd, new);
629 }
630 
631 #ifdef CONFIG_KVM_IOAPIC
632 #define IOAPIC_ROUTING_ENTRY(irq) \
633 	{ .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP,	\
634 	  .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } }
635 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq)
636 
637 #define PIC_ROUTING_ENTRY(irq) \
638 	{ .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP,	\
639 	  .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } }
640 #define ROUTING_ENTRY2(irq) \
641 	IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq)
642 
643 static const struct kvm_irq_routing_entry default_routing[] = {
644 	ROUTING_ENTRY2(0), ROUTING_ENTRY2(1),
645 	ROUTING_ENTRY2(2), ROUTING_ENTRY2(3),
646 	ROUTING_ENTRY2(4), ROUTING_ENTRY2(5),
647 	ROUTING_ENTRY2(6), ROUTING_ENTRY2(7),
648 	ROUTING_ENTRY2(8), ROUTING_ENTRY2(9),
649 	ROUTING_ENTRY2(10), ROUTING_ENTRY2(11),
650 	ROUTING_ENTRY2(12), ROUTING_ENTRY2(13),
651 	ROUTING_ENTRY2(14), ROUTING_ENTRY2(15),
652 	ROUTING_ENTRY1(16), ROUTING_ENTRY1(17),
653 	ROUTING_ENTRY1(18), ROUTING_ENTRY1(19),
654 	ROUTING_ENTRY1(20), ROUTING_ENTRY1(21),
655 	ROUTING_ENTRY1(22), ROUTING_ENTRY1(23),
656 };
657 
kvm_setup_default_ioapic_and_pic_routing(struct kvm * kvm)658 int kvm_setup_default_ioapic_and_pic_routing(struct kvm *kvm)
659 {
660 	return kvm_set_irq_routing(kvm, default_routing,
661 				   ARRAY_SIZE(default_routing), 0);
662 }
663 
kvm_vm_ioctl_get_irqchip(struct kvm * kvm,struct kvm_irqchip * chip)664 int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
665 {
666 	struct kvm_pic *pic = kvm->arch.vpic;
667 	int r;
668 
669 	r = 0;
670 	switch (chip->chip_id) {
671 	case KVM_IRQCHIP_PIC_MASTER:
672 		memcpy(&chip->chip.pic, &pic->pics[0],
673 			sizeof(struct kvm_pic_state));
674 		break;
675 	case KVM_IRQCHIP_PIC_SLAVE:
676 		memcpy(&chip->chip.pic, &pic->pics[1],
677 			sizeof(struct kvm_pic_state));
678 		break;
679 	case KVM_IRQCHIP_IOAPIC:
680 		kvm_get_ioapic(kvm, &chip->chip.ioapic);
681 		break;
682 	default:
683 		r = -EINVAL;
684 		break;
685 	}
686 	return r;
687 }
688 
kvm_vm_ioctl_set_irqchip(struct kvm * kvm,struct kvm_irqchip * chip)689 int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
690 {
691 	struct kvm_pic *pic = kvm->arch.vpic;
692 	int r;
693 
694 	r = 0;
695 	switch (chip->chip_id) {
696 	case KVM_IRQCHIP_PIC_MASTER:
697 		spin_lock(&pic->lock);
698 		memcpy(&pic->pics[0], &chip->chip.pic,
699 			sizeof(struct kvm_pic_state));
700 		spin_unlock(&pic->lock);
701 		break;
702 	case KVM_IRQCHIP_PIC_SLAVE:
703 		spin_lock(&pic->lock);
704 		memcpy(&pic->pics[1], &chip->chip.pic,
705 			sizeof(struct kvm_pic_state));
706 		spin_unlock(&pic->lock);
707 		break;
708 	case KVM_IRQCHIP_IOAPIC:
709 		kvm_set_ioapic(kvm, &chip->chip.ioapic);
710 		break;
711 	default:
712 		r = -EINVAL;
713 		break;
714 	}
715 	kvm_pic_update_irq(pic);
716 	return r;
717 }
718 #endif
719