xref: /kvmtool/virtio/mmio.c (revision 8f160708e68f4322cd698052d17ade8708d0982c)
121ff329dSWill Deacon #include "kvm/devices.h"
260273720SAsias He #include "kvm/virtio-mmio.h"
360273720SAsias He #include "kvm/ioeventfd.h"
460273720SAsias He #include "kvm/ioport.h"
560273720SAsias He #include "kvm/virtio.h"
660273720SAsias He #include "kvm/kvm.h"
719d98215SMarc Zyngier #include "kvm/kvm-cpu.h"
860273720SAsias He #include "kvm/irq.h"
92454c7dcSWill Deacon #include "kvm/fdt.h"
1060273720SAsias He 
1160273720SAsias He #include <linux/virtio_mmio.h>
1260273720SAsias He #include <string.h>
1360273720SAsias He 
1460273720SAsias He static u32 virtio_mmio_io_space_blocks = KVM_VIRTIO_MMIO_AREA;
1560273720SAsias He 
1660273720SAsias He static u32 virtio_mmio_get_io_space_block(u32 size)
1760273720SAsias He {
1860273720SAsias He 	u32 block = virtio_mmio_io_space_blocks;
1960273720SAsias He 	virtio_mmio_io_space_blocks += size;
2060273720SAsias He 
2160273720SAsias He 	return block;
2260273720SAsias He }
2360273720SAsias He 
2460273720SAsias He static void virtio_mmio_ioevent_callback(struct kvm *kvm, void *param)
2560273720SAsias He {
2660273720SAsias He 	struct virtio_mmio_ioevent_param *ioeventfd = param;
2760273720SAsias He 	struct virtio_mmio *vmmio = ioeventfd->vdev->virtio;
2860273720SAsias He 
2960273720SAsias He 	ioeventfd->vdev->ops->notify_vq(kvm, vmmio->dev, ioeventfd->vq);
3060273720SAsias He }
3160273720SAsias He 
3260273720SAsias He static int virtio_mmio_init_ioeventfd(struct kvm *kvm,
3360273720SAsias He 				      struct virtio_device *vdev, u32 vq)
3460273720SAsias He {
3560273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
3660273720SAsias He 	struct ioevent ioevent;
3760273720SAsias He 	int err;
3860273720SAsias He 
3960273720SAsias He 	vmmio->ioeventfds[vq] = (struct virtio_mmio_ioevent_param) {
4060273720SAsias He 		.vdev		= vdev,
4160273720SAsias He 		.vq		= vq,
4260273720SAsias He 	};
4360273720SAsias He 
4460273720SAsias He 	ioevent = (struct ioevent) {
4560273720SAsias He 		.io_addr	= vmmio->addr + VIRTIO_MMIO_QUEUE_NOTIFY,
4660273720SAsias He 		.io_len		= sizeof(u32),
4760273720SAsias He 		.fn		= virtio_mmio_ioevent_callback,
4860273720SAsias He 		.fn_ptr		= &vmmio->ioeventfds[vq],
4960273720SAsias He 		.datamatch	= vq,
5060273720SAsias He 		.fn_kvm		= kvm,
5160273720SAsias He 		.fd		= eventfd(0, 0),
5260273720SAsias He 	};
5360273720SAsias He 
54627d6874SAsias He 	if (vdev->use_vhost)
55627d6874SAsias He 		/*
56627d6874SAsias He 		 * Vhost will poll the eventfd in host kernel side,
57627d6874SAsias He 		 * no need to poll in userspace.
58627d6874SAsias He 		 */
5927347f76SWill Deacon 		err = ioeventfd__add_event(&ioevent, 0);
60627d6874SAsias He 	else
61627d6874SAsias He 		/* Need to poll in userspace. */
6227347f76SWill Deacon 		err = ioeventfd__add_event(&ioevent, IOEVENTFD_FLAG_USER_POLL);
6360273720SAsias He 	if (err)
6460273720SAsias He 		return err;
6560273720SAsias He 
6660273720SAsias He 	if (vdev->ops->notify_vq_eventfd)
6760273720SAsias He 		vdev->ops->notify_vq_eventfd(kvm, vmmio->dev, vq, ioevent.fd);
6860273720SAsias He 
6960273720SAsias He 	return 0;
7060273720SAsias He }
7160273720SAsias He 
7260273720SAsias He int virtio_mmio_signal_vq(struct kvm *kvm, struct virtio_device *vdev, u32 vq)
7360273720SAsias He {
7460273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
7560273720SAsias He 
7660273720SAsias He 	vmmio->hdr.interrupt_state |= VIRTIO_MMIO_INT_VRING;
7760273720SAsias He 	kvm__irq_trigger(vmmio->kvm, vmmio->irq);
7860273720SAsias He 
7960273720SAsias He 	return 0;
8060273720SAsias He }
8160273720SAsias He 
82ad346c2eSJean-Philippe Brucker static void virtio_mmio_exit_vq(struct kvm *kvm, struct virtio_device *vdev,
83ad346c2eSJean-Philippe Brucker 				int vq)
84ad346c2eSJean-Philippe Brucker {
85ad346c2eSJean-Philippe Brucker 	struct virtio_mmio *vmmio = vdev->virtio;
86ad346c2eSJean-Philippe Brucker 
87ad346c2eSJean-Philippe Brucker 	ioeventfd__del_event(vmmio->addr + VIRTIO_MMIO_QUEUE_NOTIFY, vq);
88ad346c2eSJean-Philippe Brucker 	virtio_exit_vq(kvm, vdev, vmmio->dev, vq);
89ad346c2eSJean-Philippe Brucker }
90ad346c2eSJean-Philippe Brucker 
9160273720SAsias He int virtio_mmio_signal_config(struct kvm *kvm, struct virtio_device *vdev)
9260273720SAsias He {
9360273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
9460273720SAsias He 
9560273720SAsias He 	vmmio->hdr.interrupt_state |= VIRTIO_MMIO_INT_CONFIG;
9660273720SAsias He 	kvm__irq_trigger(vmmio->kvm, vmmio->irq);
9760273720SAsias He 
9860273720SAsias He 	return 0;
9960273720SAsias He }
10060273720SAsias He 
1019b735910SMarc Zyngier static void virtio_mmio_device_specific(struct kvm_cpu *vcpu,
1029b735910SMarc Zyngier 					u64 addr, u8 *data, u32 len,
10360273720SAsias He 					u8 is_write, struct virtio_device *vdev)
10460273720SAsias He {
10560273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
10660273720SAsias He 	u32 i;
10760273720SAsias He 
10860273720SAsias He 	for (i = 0; i < len; i++) {
10960273720SAsias He 		if (is_write)
110c5ae742bSSasha Levin 			vdev->ops->get_config(vmmio->kvm, vmmio->dev)[addr + i] =
111c5ae742bSSasha Levin 					      *(u8 *)data + i;
11260273720SAsias He 		else
11360273720SAsias He 			data[i] = vdev->ops->get_config(vmmio->kvm,
114c5ae742bSSasha Levin 							vmmio->dev)[addr + i];
11560273720SAsias He 	}
11660273720SAsias He }
11760273720SAsias He 
1189b735910SMarc Zyngier static void virtio_mmio_config_in(struct kvm_cpu *vcpu,
1199b735910SMarc Zyngier 				  u64 addr, void *data, u32 len,
12060273720SAsias He 				  struct virtio_device *vdev)
12160273720SAsias He {
12260273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
12353fbb17bSJean-Philippe Brucker 	struct virt_queue *vq;
12460273720SAsias He 	u32 val = 0;
12560273720SAsias He 
12660273720SAsias He 	switch (addr) {
12760273720SAsias He 	case VIRTIO_MMIO_MAGIC_VALUE:
12860273720SAsias He 	case VIRTIO_MMIO_VERSION:
12960273720SAsias He 	case VIRTIO_MMIO_DEVICE_ID:
13060273720SAsias He 	case VIRTIO_MMIO_VENDOR_ID:
13160273720SAsias He 	case VIRTIO_MMIO_STATUS:
13260273720SAsias He 	case VIRTIO_MMIO_INTERRUPT_STATUS:
13360273720SAsias He 		ioport__write32(data, *(u32 *)(((void *)&vmmio->hdr) + addr));
13460273720SAsias He 		break;
13560273720SAsias He 	case VIRTIO_MMIO_HOST_FEATURES:
13660273720SAsias He 		if (vmmio->hdr.host_features_sel == 0)
13760273720SAsias He 			val = vdev->ops->get_host_features(vmmio->kvm,
13860273720SAsias He 							   vmmio->dev);
13960273720SAsias He 		ioport__write32(data, val);
14060273720SAsias He 		break;
14160273720SAsias He 	case VIRTIO_MMIO_QUEUE_PFN:
14253fbb17bSJean-Philippe Brucker 		vq = vdev->ops->get_vq(vmmio->kvm, vmmio->dev,
14360273720SAsias He 				       vmmio->hdr.queue_sel);
14453fbb17bSJean-Philippe Brucker 		ioport__write32(data, vq->pfn);
14560273720SAsias He 		break;
14660273720SAsias He 	case VIRTIO_MMIO_QUEUE_NUM_MAX:
14760273720SAsias He 		val = vdev->ops->get_size_vq(vmmio->kvm, vmmio->dev,
14860273720SAsias He 					     vmmio->hdr.queue_sel);
14960273720SAsias He 		ioport__write32(data, val);
15060273720SAsias He 		break;
15160273720SAsias He 	default:
15260273720SAsias He 		break;
15360273720SAsias He 	}
15460273720SAsias He }
15560273720SAsias He 
1569b735910SMarc Zyngier static void virtio_mmio_config_out(struct kvm_cpu *vcpu,
1579b735910SMarc Zyngier 				   u64 addr, void *data, u32 len,
15860273720SAsias He 				   struct virtio_device *vdev)
15960273720SAsias He {
16060273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
161e2b98125SMarc Zyngier 	struct kvm *kvm = vmmio->kvm;
16260273720SAsias He 	u32 val = 0;
16360273720SAsias He 
16460273720SAsias He 	switch (addr) {
16560273720SAsias He 	case VIRTIO_MMIO_HOST_FEATURES_SEL:
16660273720SAsias He 	case VIRTIO_MMIO_GUEST_FEATURES_SEL:
16760273720SAsias He 	case VIRTIO_MMIO_QUEUE_SEL:
16860273720SAsias He 		val = ioport__read32(data);
16960273720SAsias He 		*(u32 *)(((void *)&vmmio->hdr) + addr) = val;
17060273720SAsias He 		break;
171e2b98125SMarc Zyngier 	case VIRTIO_MMIO_STATUS:
172e2b98125SMarc Zyngier 		vmmio->hdr.status = ioport__read32(data);
17319d98215SMarc Zyngier 		if (!vmmio->hdr.status) /* Sample endianness on reset */
17419d98215SMarc Zyngier 			vdev->endian = kvm_cpu__get_endianness(vcpu);
17595242e44SJean-Philippe Brucker 		virtio_notify_status(kvm, vdev, vmmio->dev, vmmio->hdr.status);
176e2b98125SMarc Zyngier 		break;
17760273720SAsias He 	case VIRTIO_MMIO_GUEST_FEATURES:
17860273720SAsias He 		if (vmmio->hdr.guest_features_sel == 0) {
17960273720SAsias He 			val = ioport__read32(data);
18056a16c90SJean-Philippe Brucker 			virtio_set_guest_features(vmmio->kvm, vdev,
18160273720SAsias He 						  vmmio->dev, val);
18260273720SAsias He 		}
18360273720SAsias He 		break;
18460273720SAsias He 	case VIRTIO_MMIO_GUEST_PAGE_SIZE:
18560273720SAsias He 		val = ioport__read32(data);
18660273720SAsias He 		vmmio->hdr.guest_page_size = val;
18760273720SAsias He 		break;
18860273720SAsias He 	case VIRTIO_MMIO_QUEUE_NUM:
18960273720SAsias He 		val = ioport__read32(data);
19060273720SAsias He 		vmmio->hdr.queue_num = val;
19160273720SAsias He 		vdev->ops->set_size_vq(vmmio->kvm, vmmio->dev,
19260273720SAsias He 				       vmmio->hdr.queue_sel, val);
19360273720SAsias He 		break;
19460273720SAsias He 	case VIRTIO_MMIO_QUEUE_ALIGN:
19560273720SAsias He 		val = ioport__read32(data);
19660273720SAsias He 		vmmio->hdr.queue_align = val;
19760273720SAsias He 		break;
19860273720SAsias He 	case VIRTIO_MMIO_QUEUE_PFN:
19960273720SAsias He 		val = ioport__read32(data);
200ad346c2eSJean-Philippe Brucker 		if (val) {
201ad346c2eSJean-Philippe Brucker 			virtio_mmio_init_ioeventfd(vmmio->kvm, vdev,
202ad346c2eSJean-Philippe Brucker 						   vmmio->hdr.queue_sel);
20360273720SAsias He 			vdev->ops->init_vq(vmmio->kvm, vmmio->dev,
204c59ba304SWill Deacon 					   vmmio->hdr.queue_sel,
205c59ba304SWill Deacon 					   vmmio->hdr.guest_page_size,
206c59ba304SWill Deacon 					   vmmio->hdr.queue_align,
207c59ba304SWill Deacon 					   val);
208ad346c2eSJean-Philippe Brucker 		} else {
209ad346c2eSJean-Philippe Brucker 			virtio_mmio_exit_vq(kvm, vdev, vmmio->hdr.queue_sel);
210ad346c2eSJean-Philippe Brucker 		}
21160273720SAsias He 		break;
21260273720SAsias He 	case VIRTIO_MMIO_QUEUE_NOTIFY:
21360273720SAsias He 		val = ioport__read32(data);
21460273720SAsias He 		vdev->ops->notify_vq(vmmio->kvm, vmmio->dev, val);
21560273720SAsias He 		break;
21660273720SAsias He 	case VIRTIO_MMIO_INTERRUPT_ACK:
21760273720SAsias He 		val = ioport__read32(data);
21860273720SAsias He 		vmmio->hdr.interrupt_state &= ~val;
21960273720SAsias He 		break;
22060273720SAsias He 	default:
22160273720SAsias He 		break;
22260273720SAsias He 	};
22360273720SAsias He }
22460273720SAsias He 
2259b735910SMarc Zyngier static void virtio_mmio_mmio_callback(struct kvm_cpu *vcpu,
2269b735910SMarc Zyngier 				      u64 addr, u8 *data, u32 len,
22760273720SAsias He 				      u8 is_write, void *ptr)
22860273720SAsias He {
22960273720SAsias He 	struct virtio_device *vdev = ptr;
23060273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
23160273720SAsias He 	u32 offset = addr - vmmio->addr;
23260273720SAsias He 
23360273720SAsias He 	if (offset >= VIRTIO_MMIO_CONFIG) {
23460273720SAsias He 		offset -= VIRTIO_MMIO_CONFIG;
2359b735910SMarc Zyngier 		virtio_mmio_device_specific(vcpu, offset, data, len, is_write, ptr);
23660273720SAsias He 		return;
23760273720SAsias He 	}
23860273720SAsias He 
23960273720SAsias He 	if (is_write)
2409b735910SMarc Zyngier 		virtio_mmio_config_out(vcpu, offset, data, len, ptr);
24160273720SAsias He 	else
2429b735910SMarc Zyngier 		virtio_mmio_config_in(vcpu, offset, data, len, ptr);
24360273720SAsias He }
24460273720SAsias He 
2452454c7dcSWill Deacon #ifdef CONFIG_HAS_LIBFDT
2462454c7dcSWill Deacon #define DEVICE_NAME_MAX_LEN 32
2472bfd9ac3SAndre Przywara static
2482bfd9ac3SAndre Przywara void generate_virtio_mmio_fdt_node(void *fdt,
2492454c7dcSWill Deacon 				   struct device_header *dev_hdr,
2502454c7dcSWill Deacon 				   void (*generate_irq_prop)(void *fdt,
2512bfd9ac3SAndre Przywara 							     u8 irq,
2522bfd9ac3SAndre Przywara 							     enum irq_type))
2532454c7dcSWill Deacon {
2542454c7dcSWill Deacon 	char dev_name[DEVICE_NAME_MAX_LEN];
2552454c7dcSWill Deacon 	struct virtio_mmio *vmmio = container_of(dev_hdr,
2562454c7dcSWill Deacon 						 struct virtio_mmio,
2572454c7dcSWill Deacon 						 dev_hdr);
2582454c7dcSWill Deacon 	u64 addr = vmmio->addr;
2592454c7dcSWill Deacon 	u64 reg_prop[] = {
2602454c7dcSWill Deacon 		cpu_to_fdt64(addr),
2612454c7dcSWill Deacon 		cpu_to_fdt64(VIRTIO_MMIO_IO_SIZE),
2622454c7dcSWill Deacon 	};
2632454c7dcSWill Deacon 
2642454c7dcSWill Deacon 	snprintf(dev_name, DEVICE_NAME_MAX_LEN, "virtio@%llx", addr);
2652454c7dcSWill Deacon 
2662454c7dcSWill Deacon 	_FDT(fdt_begin_node(fdt, dev_name));
2672454c7dcSWill Deacon 	_FDT(fdt_property_string(fdt, "compatible", "virtio,mmio"));
2682454c7dcSWill Deacon 	_FDT(fdt_property(fdt, "reg", reg_prop, sizeof(reg_prop)));
2699a8af7e3SRobin Murphy 	_FDT(fdt_property(fdt, "dma-coherent", NULL, 0));
2702bfd9ac3SAndre Przywara 	generate_irq_prop(fdt, vmmio->irq, IRQ_TYPE_EDGE_RISING);
2712454c7dcSWill Deacon 	_FDT(fdt_end_node(fdt));
2722454c7dcSWill Deacon }
2732454c7dcSWill Deacon #else
2742454c7dcSWill Deacon static void generate_virtio_mmio_fdt_node(void *fdt,
2752454c7dcSWill Deacon 					  struct device_header *dev_hdr,
2762454c7dcSWill Deacon 					  void (*generate_irq_prop)(void *fdt,
2772454c7dcSWill Deacon 								    u8 irq))
2782454c7dcSWill Deacon {
2792454c7dcSWill Deacon 	die("Unable to generate device tree nodes without libfdt\n");
2802454c7dcSWill Deacon }
2812454c7dcSWill Deacon #endif
2822454c7dcSWill Deacon 
283b5981636SWill Deacon void virtio_mmio_assign_irq(struct device_header *dev_hdr)
284b5981636SWill Deacon {
285b5981636SWill Deacon 	struct virtio_mmio *vmmio = container_of(dev_hdr,
286b5981636SWill Deacon 						 struct virtio_mmio,
287b5981636SWill Deacon 						 dev_hdr);
288b5981636SWill Deacon 	vmmio->irq = irq__alloc_line();
289b5981636SWill Deacon }
290b5981636SWill Deacon 
29160273720SAsias He int virtio_mmio_init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
29260273720SAsias He 		     int device_id, int subsys_id, int class)
29360273720SAsias He {
29460273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
295*8f160708SAlexandru Elisei 	int r;
29660273720SAsias He 
29760273720SAsias He 	vmmio->addr	= virtio_mmio_get_io_space_block(VIRTIO_MMIO_IO_SIZE);
29860273720SAsias He 	vmmio->kvm	= kvm;
29960273720SAsias He 	vmmio->dev	= dev;
30060273720SAsias He 
301*8f160708SAlexandru Elisei 	r = kvm__register_mmio(kvm, vmmio->addr, VIRTIO_MMIO_IO_SIZE,
30260273720SAsias He 			       false, virtio_mmio_mmio_callback, vdev);
303*8f160708SAlexandru Elisei 	if (r < 0)
304*8f160708SAlexandru Elisei 		return r;
30560273720SAsias He 
30660273720SAsias He 	vmmio->hdr = (struct virtio_mmio_hdr) {
30760273720SAsias He 		.magic		= {'v', 'i', 'r', 't'},
30860273720SAsias He 		.version	= 1,
309449d5eb3SWill Deacon 		.device_id	= subsys_id,
31060273720SAsias He 		.vendor_id	= 0x4d564b4c , /* 'LKVM' */
31160273720SAsias He 		.queue_num_max	= 256,
31260273720SAsias He 	};
31360273720SAsias He 
31421ff329dSWill Deacon 	vmmio->dev_hdr = (struct device_header) {
31521ff329dSWill Deacon 		.bus_type	= DEVICE_BUS_MMIO,
3162454c7dcSWill Deacon 		.data		= generate_virtio_mmio_fdt_node,
31721ff329dSWill Deacon 	};
31821ff329dSWill Deacon 
319*8f160708SAlexandru Elisei 	r = device__register(&vmmio->dev_hdr);
320*8f160708SAlexandru Elisei 	if (r < 0) {
321*8f160708SAlexandru Elisei 		kvm__deregister_mmio(kvm, vmmio->addr);
322*8f160708SAlexandru Elisei 		return r;
323*8f160708SAlexandru Elisei 	}
32460273720SAsias He 
32560273720SAsias He 	/*
32660273720SAsias He 	 * Instantiate guest virtio-mmio devices using kernel command line
32760273720SAsias He 	 * (or module) parameter, e.g
32860273720SAsias He 	 *
32960273720SAsias He 	 * virtio_mmio.devices=0x200@0xd2000000:5,0x200@0xd2000200:6
33060273720SAsias He 	 */
331e1c7c62aSAndre Przywara 	pr_debug("virtio-mmio.devices=0x%x@0x%x:%d", VIRTIO_MMIO_IO_SIZE,
332e1c7c62aSAndre Przywara 		 vmmio->addr, vmmio->irq);
33360273720SAsias He 
33460273720SAsias He 	return 0;
33560273720SAsias He }
33660273720SAsias He 
337eb34a8c2SJean-Philippe Brucker int virtio_mmio_reset(struct kvm *kvm, struct virtio_device *vdev)
338eb34a8c2SJean-Philippe Brucker {
339eb34a8c2SJean-Philippe Brucker 	int vq;
340eb34a8c2SJean-Philippe Brucker 	struct virtio_mmio *vmmio = vdev->virtio;
341eb34a8c2SJean-Philippe Brucker 
342eb34a8c2SJean-Philippe Brucker 	for (vq = 0; vq < vdev->ops->get_vq_count(kvm, vmmio->dev); vq++)
343eb34a8c2SJean-Philippe Brucker 		virtio_mmio_exit_vq(kvm, vdev, vq);
344eb34a8c2SJean-Philippe Brucker 
345eb34a8c2SJean-Philippe Brucker 	return 0;
346eb34a8c2SJean-Philippe Brucker }
347eb34a8c2SJean-Philippe Brucker 
34860273720SAsias He int virtio_mmio_exit(struct kvm *kvm, struct virtio_device *vdev)
34960273720SAsias He {
35060273720SAsias He 	struct virtio_mmio *vmmio = vdev->virtio;
35160273720SAsias He 
352eb34a8c2SJean-Philippe Brucker 	virtio_mmio_reset(kvm, vdev);
35360273720SAsias He 	kvm__deregister_mmio(kvm, vmmio->addr);
35460273720SAsias He 
35560273720SAsias He 	return 0;
35660273720SAsias He }
357