1 // SPDX-License-Identifier: GPL-2.0-only
3  * kvm eventfd support - use eventfd objects to signal various KVM events
17 #include <linux/wait.h>
46 	struct kvm *kvm = irqfd->kvm;  in irqfd_inject()
48 	if (!irqfd->resampler) {  in irqfd_inject()
49 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,  in irqfd_inject()
51 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,  in irqfd_inject()
55 			    irqfd->gsi, 1, false);  in irqfd_inject()
62 	list_for_each_entry_srcu(irqfd, &resampler->list, resampler_link,  in irqfd_resampler_notify()
63 				 srcu_read_lock_held(&resampler->kvm->irq_srcu))  in irqfd_resampler_notify()
64 		eventfd_signal(irqfd->resamplefd);  in irqfd_resampler_notify()
68  * Since resampler irqfds share an IRQ source ID, we de-assert once
70  * do multiple de-asserts or we risk racing with incoming re-asserts.
81 	kvm = resampler->kvm;  in irqfd_resampler_ack()
84 		    resampler->notifier.gsi, 0, false);  in irqfd_resampler_ack()
86 	idx = srcu_read_lock(&kvm->irq_srcu);  in irqfd_resampler_ack()
88 	srcu_read_unlock(&kvm->irq_srcu, idx);  in irqfd_resampler_ack()
94 	struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;  in irqfd_resampler_shutdown()
95 	struct kvm *kvm = resampler->kvm;  in irqfd_resampler_shutdown()
97 	mutex_lock(&kvm->irqfds.resampler_lock);  in irqfd_resampler_shutdown()
99 	list_del_rcu(&irqfd->resampler_link);  in irqfd_resampler_shutdown()
100 	synchronize_srcu(&kvm->irq_srcu);  in irqfd_resampler_shutdown()
102 	if (list_empty(&resampler->list)) {  in irqfd_resampler_shutdown()
103 		list_del_rcu(&resampler->link);  in irqfd_resampler_shutdown()
104 		kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);  in irqfd_resampler_shutdown()
106 		 * synchronize_srcu(&kvm->irq_srcu) already called  in irqfd_resampler_shutdown()
110 			    resampler->notifier.gsi, 0, false);  in irqfd_resampler_shutdown()
114 	mutex_unlock(&kvm->irqfds.resampler_lock);  in irqfd_resampler_shutdown()
118  * Race-free decouple logic (ordering is critical)
125 	struct kvm *kvm = irqfd->kvm;  in irqfd_shutdown()
129 	synchronize_srcu(&kvm->irq_srcu);  in irqfd_shutdown()
132 	 * Synchronize with the wait-queue and unhook ourselves to prevent  in irqfd_shutdown()
135 	eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);  in irqfd_shutdown()
141 	flush_work(&irqfd->inject);  in irqfd_shutdown()
143 	if (irqfd->resampler) {  in irqfd_shutdown()
145 		eventfd_ctx_put(irqfd->resamplefd);  in irqfd_shutdown()
152 	irq_bypass_unregister_consumer(&irqfd->consumer);  in irqfd_shutdown()
154 	eventfd_ctx_put(irqfd->eventfd);  in irqfd_shutdown()
159 /* assumes kvm->irqfds.lock is held */
163 	return list_empty(&irqfd->list) ? false : true;  in irqfd_is_active()
169  * assumes kvm->irqfds.lock is held
176 	list_del_init(&irqfd->list);  in irqfd_deactivate()
178 	queue_work(irqfd_cleanup_wq, &irqfd->shutdown);  in irqfd_deactivate()
187 	return -EWOULDBLOCK;  in kvm_arch_set_irq_inatomic()
191  * Called with wqh->lock held and interrupts disabled
194 irqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)  in irqfd_wakeup()  argument
197 		container_of(wait, struct kvm_kernel_irqfd, wait);  in irqfd_wakeup()
200 	struct kvm *kvm = irqfd->kvm;  in irqfd_wakeup()
207 		eventfd_ctx_do_read(irqfd->eventfd, &cnt);  in irqfd_wakeup()
209 		idx = srcu_read_lock(&kvm->irq_srcu);  in irqfd_wakeup()
211 			seq = read_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_wakeup()
212 			irq = irqfd->irq_entry;  in irqfd_wakeup()
213 		} while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));  in irqfd_wakeup()
217 					      false) == -EWOULDBLOCK)  in irqfd_wakeup()
218 			schedule_work(&irqfd->inject);  in irqfd_wakeup()
219 		srcu_read_unlock(&kvm->irq_srcu, idx);  in irqfd_wakeup()
227 		spin_lock_irqsave(&kvm->irqfds.lock, iflags);  in irqfd_wakeup()
233 		 * the wait-queue.  If it is already deactivated, we can  in irqfd_wakeup()
236 		 * other side is required to acquire wqh->lock, which we hold  in irqfd_wakeup()
241 		spin_unlock_irqrestore(&kvm->irqfds.lock, iflags);  in irqfd_wakeup()
253 	add_wait_queue_priority(wqh, &irqfd->wait);  in irqfd_ptable_queue_proc()
263 	n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);  in irqfd_update()
265 	write_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_update()
269 		irqfd->irq_entry = *e;  in irqfd_update()
271 		irqfd->irq_entry.type = 0;  in irqfd_update()
273 	write_seqcount_end(&irqfd->irq_entry_sc);  in irqfd_update()
313 		return -EAGAIN;  in kvm_irqfd_assign()
316 		return -EINVAL;  in kvm_irqfd_assign()
320 		return -ENOMEM;  in kvm_irqfd_assign()
322 	irqfd->kvm = kvm;  in kvm_irqfd_assign()
323 	irqfd->gsi = args->gsi;  in kvm_irqfd_assign()
324 	INIT_LIST_HEAD(&irqfd->list);  in kvm_irqfd_assign()
325 	INIT_WORK(&irqfd->inject, irqfd_inject);  in kvm_irqfd_assign()
326 	INIT_WORK(&irqfd->shutdown, irqfd_shutdown);  in kvm_irqfd_assign()
327 	seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock);  in kvm_irqfd_assign()
329 	f = fdget(args->fd);  in kvm_irqfd_assign()
331 		ret = -EBADF;  in kvm_irqfd_assign()
341 	irqfd->eventfd = eventfd;  in kvm_irqfd_assign()
343 	if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) {  in kvm_irqfd_assign()
346 		resamplefd = eventfd_ctx_fdget(args->resamplefd);  in kvm_irqfd_assign()
352 		irqfd->resamplefd = resamplefd;  in kvm_irqfd_assign()
353 		INIT_LIST_HEAD(&irqfd->resampler_link);  in kvm_irqfd_assign()
355 		mutex_lock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
358 				    &kvm->irqfds.resampler_list, link) {  in kvm_irqfd_assign()
359 			if (resampler->notifier.gsi == irqfd->gsi) {  in kvm_irqfd_assign()
360 				irqfd->resampler = resampler;  in kvm_irqfd_assign()
365 		if (!irqfd->resampler) {  in kvm_irqfd_assign()
369 				ret = -ENOMEM;  in kvm_irqfd_assign()
370 				mutex_unlock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
374 			resampler->kvm = kvm;  in kvm_irqfd_assign()
375 			INIT_LIST_HEAD(&resampler->list);  in kvm_irqfd_assign()
376 			resampler->notifier.gsi = irqfd->gsi;  in kvm_irqfd_assign()
377 			resampler->notifier.irq_acked = irqfd_resampler_ack;  in kvm_irqfd_assign()
378 			INIT_LIST_HEAD(&resampler->link);  in kvm_irqfd_assign()
380 			list_add_rcu(&resampler->link, &kvm->irqfds.resampler_list);  in kvm_irqfd_assign()
382 						      &resampler->notifier);  in kvm_irqfd_assign()
383 			irqfd->resampler = resampler;  in kvm_irqfd_assign()
386 		list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);  in kvm_irqfd_assign()
387 		synchronize_srcu(&kvm->irq_srcu);  in kvm_irqfd_assign()
389 		mutex_unlock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
393 	 * Install our own custom wake-up handling so we are notified via  in kvm_irqfd_assign()
396 	init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);  in kvm_irqfd_assign()
397 	init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);  in kvm_irqfd_assign()
399 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
402 	list_for_each_entry(tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_assign()
403 		if (irqfd->eventfd != tmp->eventfd)  in kvm_irqfd_assign()
406 		ret = -EBUSY;  in kvm_irqfd_assign()
407 		spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
411 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_irqfd_assign()
414 	list_add_tail(&irqfd->list, &kvm->irqfds.items);  in kvm_irqfd_assign()
416 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
422 	events = vfs_poll(f.file, &irqfd->pt);  in kvm_irqfd_assign()
425 		schedule_work(&irqfd->inject);  in kvm_irqfd_assign()
429 		irqfd->consumer.token = (void *)irqfd->eventfd;  in kvm_irqfd_assign()
430 		irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;  in kvm_irqfd_assign()
431 		irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;  in kvm_irqfd_assign()
432 		irqfd->consumer.stop = kvm_arch_irq_bypass_stop;  in kvm_irqfd_assign()
433 		irqfd->consumer.start = kvm_arch_irq_bypass_start;  in kvm_irqfd_assign()
434 		ret = irq_bypass_register_consumer(&irqfd->consumer);  in kvm_irqfd_assign()
437 				irqfd->consumer.token, ret);  in kvm_irqfd_assign()
441 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irqfd_assign()
451 	if (irqfd->resampler)  in kvm_irqfd_assign()
467 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_irq_has_notifier()  argument
472 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_irq_has_notifier()
473 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_irq_has_notifier()
474 	if (gsi != -1)  in kvm_irq_has_notifier()
475 		hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,  in kvm_irq_has_notifier()
476 					  link, srcu_read_lock_held(&kvm->irq_srcu))  in kvm_irq_has_notifier()
477 			if (kian->gsi == gsi) {  in kvm_irq_has_notifier()
478 				srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irq_has_notifier()
482 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irq_has_notifier()
492 	hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,  in kvm_notify_acked_gsi()
493 				  link, srcu_read_lock_held(&kvm->irq_srcu))  in kvm_notify_acked_gsi()
494 		if (kian->gsi == gsi)  in kvm_notify_acked_gsi()
495 			kian->irq_acked(kian);  in kvm_notify_acked_gsi()
498 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_notify_acked_irq()  argument
502 	trace_kvm_ack_irq(irqchip, pin);  in kvm_notify_acked_irq()
504 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_notify_acked_irq()
505 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_notify_acked_irq()
506 	if (gsi != -1)  in kvm_notify_acked_irq()
508 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_acked_irq()
514 	mutex_lock(&kvm->irq_lock);  in kvm_register_irq_ack_notifier()
515 	hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);  in kvm_register_irq_ack_notifier()
516 	mutex_unlock(&kvm->irq_lock);  in kvm_register_irq_ack_notifier()
523 	mutex_lock(&kvm->irq_lock);  in kvm_unregister_irq_ack_notifier()
524 	hlist_del_init_rcu(&kian->link);  in kvm_unregister_irq_ack_notifier()
525 	mutex_unlock(&kvm->irq_lock);  in kvm_unregister_irq_ack_notifier()
526 	synchronize_srcu(&kvm->irq_srcu);  in kvm_unregister_irq_ack_notifier()
539 	eventfd = eventfd_ctx_fdget(args->fd);  in kvm_irqfd_deassign()
543 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_deassign()
545 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_deassign()
546 		if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {  in kvm_irqfd_deassign()
553 			write_seqcount_begin(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
554 			irqfd->irq_entry.type = 0;  in kvm_irqfd_deassign()
555 			write_seqcount_end(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
560 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_deassign()
576 	if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE))  in kvm_irqfd()
577 		return -EINVAL;  in kvm_irqfd()
579 	if (args->flags & KVM_IRQFD_FLAG_DEASSIGN)  in kvm_irqfd()
594 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_release()
596 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)  in kvm_irqfd_release()
599 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_release()
611  * Caller must invoke synchronize_srcu(&kvm->irq_srcu) afterwards.
617 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irq_routing_update()
619 	list_for_each_entry(irqfd, &kvm->irqfds.items, list) {  in kvm_irq_routing_update()
622 		struct kvm_kernel_irq_routing_entry old = irqfd->irq_entry;  in kvm_irq_routing_update()
628 		if (irqfd->producer &&  in kvm_irq_routing_update()
629 		    kvm_arch_irqfd_route_changed(&old, &irqfd->irq_entry)) {  in kvm_irq_routing_update()
631 					irqfd->kvm, irqfd->producer->irq,  in kvm_irq_routing_update()
632 					irqfd->gsi, 1);  in kvm_irq_routing_update()
638 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irq_routing_update()
643 				unsigned int pin)  in kvm_notify_irqfd_resampler()  argument
648 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_notify_irqfd_resampler()
649 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_notify_irqfd_resampler()
650 	if (gsi != -1) {  in kvm_notify_irqfd_resampler()
652 					 &kvm->irqfds.resampler_list, link,  in kvm_notify_irqfd_resampler()
653 					 srcu_read_lock_held(&kvm->irq_srcu)) {  in kvm_notify_irqfd_resampler()
654 			if (resampler->notifier.gsi == gsi) {  in kvm_notify_irqfd_resampler()
656 				srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_irqfd_resampler()
661 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_irqfd_resampler()
667  * create a host-wide workqueue for issuing deferred shutdown requests
673 	irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", 0, 0);  in kvm_irqfd_init()
675 		return -ENOMEM;  in kvm_irqfd_init()
687  * --------------------------------------------------------------------
692  * --------------------------------------------------------------------
715 	eventfd_ctx_put(p->eventfd);  in ioeventfd_release()
716 	list_del(&p->list);  in ioeventfd_release()
725 	if (addr != p->addr)  in ioeventfd_in_range()
729 	if (!p->length)  in ioeventfd_in_range()
733 	if (len != p->length)  in ioeventfd_in_range()
734 		/* address-range must be precise for a hit */  in ioeventfd_in_range()
737 	if (p->wildcard)  in ioeventfd_in_range()
762 	return _val == p->datamatch;  in ioeventfd_in_range()
773 		return -EOPNOTSUPP;  in ioeventfd_write()
775 	eventfd_signal(p->eventfd);  in ioeventfd_write()
796 /* assumes kvm->slots_lock held */
802 	list_for_each_entry(_p, &kvm->ioeventfds, list)  in ioeventfd_check_collision()
803 		if (_p->bus_idx == p->bus_idx &&  in ioeventfd_check_collision()
804 		    _p->addr == p->addr &&  in ioeventfd_check_collision()
805 		    (!_p->length || !p->length ||  in ioeventfd_check_collision()
806 		     (_p->length == p->length &&  in ioeventfd_check_collision()
807 		      (_p->wildcard || p->wildcard ||  in ioeventfd_check_collision()
808 		       _p->datamatch == p->datamatch))))  in ioeventfd_check_collision()
832 	eventfd = eventfd_ctx_fdget(args->fd);  in kvm_assign_ioeventfd_idx()
838 		ret = -ENOMEM;  in kvm_assign_ioeventfd_idx()
842 	INIT_LIST_HEAD(&p->list);  in kvm_assign_ioeventfd_idx()
843 	p->addr    = args->addr;  in kvm_assign_ioeventfd_idx()
844 	p->bus_idx = bus_idx;  in kvm_assign_ioeventfd_idx()
845 	p->length  = args->len;  in kvm_assign_ioeventfd_idx()
846 	p->eventfd = eventfd;  in kvm_assign_ioeventfd_idx()
849 	if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH)  in kvm_assign_ioeventfd_idx()
850 		p->datamatch = args->datamatch;  in kvm_assign_ioeventfd_idx()
852 		p->wildcard = true;  in kvm_assign_ioeventfd_idx()
854 	mutex_lock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
858 		ret = -EEXIST;  in kvm_assign_ioeventfd_idx()
862 	kvm_iodevice_init(&p->dev, &ioeventfd_ops);  in kvm_assign_ioeventfd_idx()
864 	ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length,  in kvm_assign_ioeventfd_idx()
865 				      &p->dev);  in kvm_assign_ioeventfd_idx()
869 	kvm_get_bus(kvm, bus_idx)->ioeventfd_count++;  in kvm_assign_ioeventfd_idx()
870 	list_add_tail(&p->list, &kvm->ioeventfds);  in kvm_assign_ioeventfd_idx()
872 	mutex_unlock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
877 	mutex_unlock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
893 	int                       ret = -ENOENT;  in kvm_deassign_ioeventfd_idx()
896 	eventfd = eventfd_ctx_fdget(args->fd);  in kvm_deassign_ioeventfd_idx()
900 	wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH);  in kvm_deassign_ioeventfd_idx()
902 	mutex_lock(&kvm->slots_lock);  in kvm_deassign_ioeventfd_idx()
904 	list_for_each_entry(p, &kvm->ioeventfds, list) {  in kvm_deassign_ioeventfd_idx()
905 		if (p->bus_idx != bus_idx ||  in kvm_deassign_ioeventfd_idx()
906 		    p->eventfd != eventfd  ||  in kvm_deassign_ioeventfd_idx()
907 		    p->addr != args->addr  ||  in kvm_deassign_ioeventfd_idx()
908 		    p->length != args->len ||  in kvm_deassign_ioeventfd_idx()
909 		    p->wildcard != wildcard)  in kvm_deassign_ioeventfd_idx()
912 		if (!p->wildcard && p->datamatch != args->datamatch)  in kvm_deassign_ioeventfd_idx()
915 		kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);  in kvm_deassign_ioeventfd_idx()
918 			bus->ioeventfd_count--;  in kvm_deassign_ioeventfd_idx()
923 	mutex_unlock(&kvm->slots_lock);  in kvm_deassign_ioeventfd_idx()
932 	enum kvm_bus bus_idx = ioeventfd_bus_from_flags(args->flags);  in kvm_deassign_ioeventfd()
935 	if (!args->len && bus_idx == KVM_MMIO_BUS)  in kvm_deassign_ioeventfd()
947 	bus_idx = ioeventfd_bus_from_flags(args->flags);  in kvm_assign_ioeventfd()
948 	/* must be natural-word sized, or 0 to ignore length */  in kvm_assign_ioeventfd()
949 	switch (args->len) {  in kvm_assign_ioeventfd()
957 		return -EINVAL;  in kvm_assign_ioeventfd()
961 	if (args->addr + args->len < args->addr)  in kvm_assign_ioeventfd()
962 		return -EINVAL;  in kvm_assign_ioeventfd()
965 	if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)  in kvm_assign_ioeventfd()
966 		return -EINVAL;  in kvm_assign_ioeventfd()
969 	if (!args->len && (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH))  in kvm_assign_ioeventfd()
970 		return -EINVAL;  in kvm_assign_ioeventfd()
979 	if (!args->len && bus_idx == KVM_MMIO_BUS) {  in kvm_assign_ioeventfd()
996 	if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN)  in kvm_ioeventfd()
1006 	spin_lock_init(&kvm->irqfds.lock);  in kvm_eventfd_init()
1007 	INIT_LIST_HEAD(&kvm->irqfds.items);  in kvm_eventfd_init()
1008 	INIT_LIST_HEAD(&kvm->irqfds.resampler_list);  in kvm_eventfd_init()
1009 	mutex_init(&kvm->irqfds.resampler_lock);  in kvm_eventfd_init()
1011 	INIT_LIST_HEAD(&kvm->ioeventfds);  in kvm_eventfd_init()