1 // SPDX-License-Identifier: GPL-2.0-only
16  * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
26  * Protect the per-CPU list with a per-CPU spinlock to handle task migration.
28  * ->sched_in() path will need to take the vCPU off the list of the _previous_
38 	return &(to_vmx(vcpu)->pi_desc);  in vcpu_to_pi_desc()
49 	if (!try_cmpxchg64(&pi_desc->control, pold, new))  in pi_try_set_control()
50 		return -EBUSY;  in pi_try_set_control()
64 	 * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and  in vmx_vcpu_pi_load()
65 	 * PI.SN up-to-date even if there is no assigned device or if APICv is  in vmx_vcpu_pi_load()
66 	 * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC.  in vmx_vcpu_pi_load()
76 	if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) {  in vmx_vcpu_pi_load()
93 	if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) {  in vmx_vcpu_pi_load()
94 		raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu);  in vmx_vcpu_pi_load()
104 		spin_acquire(&spinlock->dep_map, PI_LOCK_SCHED_OUT, 0, _RET_IP_);  in vmx_vcpu_pi_load()
105 		list_del(&vmx->pi_wakeup_list);  in vmx_vcpu_pi_load()
106 		spin_release(&spinlock->dep_map, _RET_IP_);  in vmx_vcpu_pi_load()
114 	old.control = READ_ONCE(pi_desc->control);  in vmx_vcpu_pi_load()
120 		 * handle task migration (@cpu != vcpu->cpu).  in vmx_vcpu_pi_load()
137 	 * Clear SN before reading the bitmap.  The VT-d firmware  in vmx_vcpu_pi_load()
170 	 * various per-CPU scheduler locks.  When the wakeup handler runs, it  in pi_enable_wakeup_handler()
179 	raw_spin_lock_nested(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu),  in pi_enable_wakeup_handler()
181 	list_add_tail(&vmx->pi_wakeup_list,  in pi_enable_wakeup_handler()
182 		      &per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu));  in pi_enable_wakeup_handler()
183 	raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));  in pi_enable_wakeup_handler()
187 	old.control = READ_ONCE(pi_desc->control);  in pi_enable_wakeup_handler()
195 	 * Send a wakeup IPI to this CPU if an interrupt may have been posted  in pi_enable_wakeup_handler()
197 	 * will arrive on the non-wakeup vector.  An IPI is needed as calling  in pi_enable_wakeup_handler()
198 	 * try_to_wake_up() from ->sched_out() isn't allowed (IRQs are not  in pi_enable_wakeup_handler()
212 	 * using either IPI virtualization or VT-d PI, so that the  in vmx_needs_pi_wakeup()
216 	return vmx_can_use_ipiv(vcpu) || vmx_can_use_vtd_pi(vcpu->kvm);  in vmx_needs_pi_wakeup()
234 	if (vcpu->preempted)  in vmx_vcpu_pi_put()
251 		if (pi_test_on(&vmx->pi_desc))  in pi_wakeup_handler()
252 			kvm_vcpu_wake_up(&vmx->vcpu);  in pi_wakeup_handler()
287  * vmx_pi_update_irte - set IRTE for Posted-Interrupts
309 	idx = srcu_read_lock(&kvm->irq_srcu);  in vmx_pi_update_irte()
310 	irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);  in vmx_pi_update_irte()
311 	if (guest_irq >= irq_rt->nr_rt_entries ||  in vmx_pi_update_irte()
312 	    hlist_empty(&irq_rt->map[guest_irq])) {  in vmx_pi_update_irte()
314 			     guest_irq, irq_rt->nr_rt_entries);  in vmx_pi_update_irte()
318 	hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {  in vmx_pi_update_irte()
319 		if (e->type != KVM_IRQ_ROUTING_MSI)  in vmx_pi_update_irte()
322 		 * VT-d PI cannot support posting multicast/broadcast  in vmx_pi_update_irte()
326 		 * For lowest-priority interrupts, we only support  in vmx_pi_update_irte()
329 		 * irqbalance to make the interrupts single-CPU.  in vmx_pi_update_irte()
331 		 * We will support full lowest-priority interrupt later.  in vmx_pi_update_irte()
345 		trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,  in vmx_pi_update_irte()
366 	srcu_read_unlock(&kvm->irq_srcu, idx);  in vmx_pi_update_irte()