121ff329dSWill Deacon #include "kvm/devices.h" 260273720SAsias He #include "kvm/virtio-mmio.h" 360273720SAsias He #include "kvm/ioeventfd.h" 460273720SAsias He #include "kvm/ioport.h" 560273720SAsias He #include "kvm/virtio.h" 660273720SAsias He #include "kvm/kvm.h" 719d98215SMarc Zyngier #include "kvm/kvm-cpu.h" 860273720SAsias He #include "kvm/irq.h" 92454c7dcSWill Deacon #include "kvm/fdt.h" 1060273720SAsias He 1160273720SAsias He #include <linux/virtio_mmio.h> 1260273720SAsias He #include <string.h> 1360273720SAsias He 1460273720SAsias He static u32 virtio_mmio_io_space_blocks = KVM_VIRTIO_MMIO_AREA; 1560273720SAsias He 1660273720SAsias He static u32 virtio_mmio_get_io_space_block(u32 size) 1760273720SAsias He { 1860273720SAsias He u32 block = virtio_mmio_io_space_blocks; 1960273720SAsias He virtio_mmio_io_space_blocks += size; 2060273720SAsias He 2160273720SAsias He return block; 2260273720SAsias He } 2360273720SAsias He 2460273720SAsias He static void virtio_mmio_ioevent_callback(struct kvm *kvm, void *param) 2560273720SAsias He { 2660273720SAsias He struct virtio_mmio_ioevent_param *ioeventfd = param; 2760273720SAsias He struct virtio_mmio *vmmio = ioeventfd->vdev->virtio; 2860273720SAsias He 2960273720SAsias He ioeventfd->vdev->ops->notify_vq(kvm, vmmio->dev, ioeventfd->vq); 3060273720SAsias He } 3160273720SAsias He 3260273720SAsias He static int virtio_mmio_init_ioeventfd(struct kvm *kvm, 3360273720SAsias He struct virtio_device *vdev, u32 vq) 3460273720SAsias He { 3560273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 3660273720SAsias He struct ioevent ioevent; 3760273720SAsias He int err; 3860273720SAsias He 3960273720SAsias He vmmio->ioeventfds[vq] = (struct virtio_mmio_ioevent_param) { 4060273720SAsias He .vdev = vdev, 4160273720SAsias He .vq = vq, 4260273720SAsias He }; 4360273720SAsias He 4460273720SAsias He ioevent = (struct ioevent) { 4560273720SAsias He .io_addr = vmmio->addr + VIRTIO_MMIO_QUEUE_NOTIFY, 4660273720SAsias He .io_len = sizeof(u32), 4760273720SAsias He .fn = virtio_mmio_ioevent_callback, 4860273720SAsias He .fn_ptr = &vmmio->ioeventfds[vq], 4960273720SAsias He .datamatch = vq, 5060273720SAsias He .fn_kvm = kvm, 5160273720SAsias He .fd = eventfd(0, 0), 5260273720SAsias He }; 5360273720SAsias He 54627d6874SAsias He if (vdev->use_vhost) 55627d6874SAsias He /* 56627d6874SAsias He * Vhost will poll the eventfd in host kernel side, 57627d6874SAsias He * no need to poll in userspace. 58627d6874SAsias He */ 5927347f76SWill Deacon err = ioeventfd__add_event(&ioevent, 0); 60627d6874SAsias He else 61627d6874SAsias He /* Need to poll in userspace. */ 6227347f76SWill Deacon err = ioeventfd__add_event(&ioevent, IOEVENTFD_FLAG_USER_POLL); 6360273720SAsias He if (err) 6460273720SAsias He return err; 6560273720SAsias He 6660273720SAsias He if (vdev->ops->notify_vq_eventfd) 6760273720SAsias He vdev->ops->notify_vq_eventfd(kvm, vmmio->dev, vq, ioevent.fd); 6860273720SAsias He 6960273720SAsias He return 0; 7060273720SAsias He } 7160273720SAsias He 7260273720SAsias He int virtio_mmio_signal_vq(struct kvm *kvm, struct virtio_device *vdev, u32 vq) 7360273720SAsias He { 7460273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 7560273720SAsias He 7660273720SAsias He vmmio->hdr.interrupt_state |= VIRTIO_MMIO_INT_VRING; 7760273720SAsias He kvm__irq_trigger(vmmio->kvm, vmmio->irq); 7860273720SAsias He 7960273720SAsias He return 0; 8060273720SAsias He } 8160273720SAsias He 82ad346c2eSJean-Philippe Brucker static void virtio_mmio_exit_vq(struct kvm *kvm, struct virtio_device *vdev, 83ad346c2eSJean-Philippe Brucker int vq) 84ad346c2eSJean-Philippe Brucker { 85ad346c2eSJean-Philippe Brucker struct virtio_mmio *vmmio = vdev->virtio; 86ad346c2eSJean-Philippe Brucker 87ad346c2eSJean-Philippe Brucker ioeventfd__del_event(vmmio->addr + VIRTIO_MMIO_QUEUE_NOTIFY, vq); 88ad346c2eSJean-Philippe Brucker virtio_exit_vq(kvm, vdev, vmmio->dev, vq); 89ad346c2eSJean-Philippe Brucker } 90ad346c2eSJean-Philippe Brucker 9160273720SAsias He int virtio_mmio_signal_config(struct kvm *kvm, struct virtio_device *vdev) 9260273720SAsias He { 9360273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 9460273720SAsias He 9560273720SAsias He vmmio->hdr.interrupt_state |= VIRTIO_MMIO_INT_CONFIG; 9660273720SAsias He kvm__irq_trigger(vmmio->kvm, vmmio->irq); 9760273720SAsias He 9860273720SAsias He return 0; 9960273720SAsias He } 10060273720SAsias He 1019b735910SMarc Zyngier static void virtio_mmio_device_specific(struct kvm_cpu *vcpu, 1029b735910SMarc Zyngier u64 addr, u8 *data, u32 len, 10360273720SAsias He u8 is_write, struct virtio_device *vdev) 10460273720SAsias He { 10560273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 10660273720SAsias He u32 i; 10760273720SAsias He 10860273720SAsias He for (i = 0; i < len; i++) { 10960273720SAsias He if (is_write) 110c5ae742bSSasha Levin vdev->ops->get_config(vmmio->kvm, vmmio->dev)[addr + i] = 111c5ae742bSSasha Levin *(u8 *)data + i; 11260273720SAsias He else 11360273720SAsias He data[i] = vdev->ops->get_config(vmmio->kvm, 114c5ae742bSSasha Levin vmmio->dev)[addr + i]; 11560273720SAsias He } 11660273720SAsias He } 11760273720SAsias He 1189b735910SMarc Zyngier static void virtio_mmio_config_in(struct kvm_cpu *vcpu, 1199b735910SMarc Zyngier u64 addr, void *data, u32 len, 12060273720SAsias He struct virtio_device *vdev) 12160273720SAsias He { 12260273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 12353fbb17bSJean-Philippe Brucker struct virt_queue *vq; 12460273720SAsias He u32 val = 0; 12560273720SAsias He 12660273720SAsias He switch (addr) { 12760273720SAsias He case VIRTIO_MMIO_MAGIC_VALUE: 12860273720SAsias He case VIRTIO_MMIO_VERSION: 12960273720SAsias He case VIRTIO_MMIO_DEVICE_ID: 13060273720SAsias He case VIRTIO_MMIO_VENDOR_ID: 13160273720SAsias He case VIRTIO_MMIO_STATUS: 13260273720SAsias He case VIRTIO_MMIO_INTERRUPT_STATUS: 13360273720SAsias He ioport__write32(data, *(u32 *)(((void *)&vmmio->hdr) + addr)); 13460273720SAsias He break; 13560273720SAsias He case VIRTIO_MMIO_HOST_FEATURES: 13660273720SAsias He if (vmmio->hdr.host_features_sel == 0) 13760273720SAsias He val = vdev->ops->get_host_features(vmmio->kvm, 13860273720SAsias He vmmio->dev); 13960273720SAsias He ioport__write32(data, val); 14060273720SAsias He break; 14160273720SAsias He case VIRTIO_MMIO_QUEUE_PFN: 14253fbb17bSJean-Philippe Brucker vq = vdev->ops->get_vq(vmmio->kvm, vmmio->dev, 14360273720SAsias He vmmio->hdr.queue_sel); 14453fbb17bSJean-Philippe Brucker ioport__write32(data, vq->pfn); 14560273720SAsias He break; 14660273720SAsias He case VIRTIO_MMIO_QUEUE_NUM_MAX: 14760273720SAsias He val = vdev->ops->get_size_vq(vmmio->kvm, vmmio->dev, 14860273720SAsias He vmmio->hdr.queue_sel); 14960273720SAsias He ioport__write32(data, val); 15060273720SAsias He break; 15160273720SAsias He default: 15260273720SAsias He break; 15360273720SAsias He } 15460273720SAsias He } 15560273720SAsias He 1569b735910SMarc Zyngier static void virtio_mmio_config_out(struct kvm_cpu *vcpu, 1579b735910SMarc Zyngier u64 addr, void *data, u32 len, 15860273720SAsias He struct virtio_device *vdev) 15960273720SAsias He { 16060273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 161e2b98125SMarc Zyngier struct kvm *kvm = vmmio->kvm; 16260273720SAsias He u32 val = 0; 16360273720SAsias He 16460273720SAsias He switch (addr) { 16560273720SAsias He case VIRTIO_MMIO_HOST_FEATURES_SEL: 16660273720SAsias He case VIRTIO_MMIO_GUEST_FEATURES_SEL: 16760273720SAsias He case VIRTIO_MMIO_QUEUE_SEL: 16860273720SAsias He val = ioport__read32(data); 16960273720SAsias He *(u32 *)(((void *)&vmmio->hdr) + addr) = val; 17060273720SAsias He break; 171e2b98125SMarc Zyngier case VIRTIO_MMIO_STATUS: 172e2b98125SMarc Zyngier vmmio->hdr.status = ioport__read32(data); 17319d98215SMarc Zyngier if (!vmmio->hdr.status) /* Sample endianness on reset */ 17419d98215SMarc Zyngier vdev->endian = kvm_cpu__get_endianness(vcpu); 17595242e44SJean-Philippe Brucker virtio_notify_status(kvm, vdev, vmmio->dev, vmmio->hdr.status); 176e2b98125SMarc Zyngier break; 17760273720SAsias He case VIRTIO_MMIO_GUEST_FEATURES: 17860273720SAsias He if (vmmio->hdr.guest_features_sel == 0) { 17960273720SAsias He val = ioport__read32(data); 18056a16c90SJean-Philippe Brucker virtio_set_guest_features(vmmio->kvm, vdev, 18160273720SAsias He vmmio->dev, val); 18260273720SAsias He } 18360273720SAsias He break; 18460273720SAsias He case VIRTIO_MMIO_GUEST_PAGE_SIZE: 18560273720SAsias He val = ioport__read32(data); 18660273720SAsias He vmmio->hdr.guest_page_size = val; 18760273720SAsias He break; 18860273720SAsias He case VIRTIO_MMIO_QUEUE_NUM: 18960273720SAsias He val = ioport__read32(data); 19060273720SAsias He vmmio->hdr.queue_num = val; 19160273720SAsias He vdev->ops->set_size_vq(vmmio->kvm, vmmio->dev, 19260273720SAsias He vmmio->hdr.queue_sel, val); 19360273720SAsias He break; 19460273720SAsias He case VIRTIO_MMIO_QUEUE_ALIGN: 19560273720SAsias He val = ioport__read32(data); 19660273720SAsias He vmmio->hdr.queue_align = val; 19760273720SAsias He break; 19860273720SAsias He case VIRTIO_MMIO_QUEUE_PFN: 19960273720SAsias He val = ioport__read32(data); 200ad346c2eSJean-Philippe Brucker if (val) { 201ad346c2eSJean-Philippe Brucker virtio_mmio_init_ioeventfd(vmmio->kvm, vdev, 202ad346c2eSJean-Philippe Brucker vmmio->hdr.queue_sel); 20360273720SAsias He vdev->ops->init_vq(vmmio->kvm, vmmio->dev, 204c59ba304SWill Deacon vmmio->hdr.queue_sel, 205c59ba304SWill Deacon vmmio->hdr.guest_page_size, 206c59ba304SWill Deacon vmmio->hdr.queue_align, 207c59ba304SWill Deacon val); 208ad346c2eSJean-Philippe Brucker } else { 209ad346c2eSJean-Philippe Brucker virtio_mmio_exit_vq(kvm, vdev, vmmio->hdr.queue_sel); 210ad346c2eSJean-Philippe Brucker } 21160273720SAsias He break; 21260273720SAsias He case VIRTIO_MMIO_QUEUE_NOTIFY: 21360273720SAsias He val = ioport__read32(data); 21460273720SAsias He vdev->ops->notify_vq(vmmio->kvm, vmmio->dev, val); 21560273720SAsias He break; 21660273720SAsias He case VIRTIO_MMIO_INTERRUPT_ACK: 21760273720SAsias He val = ioport__read32(data); 21860273720SAsias He vmmio->hdr.interrupt_state &= ~val; 21960273720SAsias He break; 22060273720SAsias He default: 22160273720SAsias He break; 22260273720SAsias He }; 22360273720SAsias He } 22460273720SAsias He 2259b735910SMarc Zyngier static void virtio_mmio_mmio_callback(struct kvm_cpu *vcpu, 2269b735910SMarc Zyngier u64 addr, u8 *data, u32 len, 22760273720SAsias He u8 is_write, void *ptr) 22860273720SAsias He { 22960273720SAsias He struct virtio_device *vdev = ptr; 23060273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 23160273720SAsias He u32 offset = addr - vmmio->addr; 23260273720SAsias He 23360273720SAsias He if (offset >= VIRTIO_MMIO_CONFIG) { 23460273720SAsias He offset -= VIRTIO_MMIO_CONFIG; 2359b735910SMarc Zyngier virtio_mmio_device_specific(vcpu, offset, data, len, is_write, ptr); 23660273720SAsias He return; 23760273720SAsias He } 23860273720SAsias He 23960273720SAsias He if (is_write) 2409b735910SMarc Zyngier virtio_mmio_config_out(vcpu, offset, data, len, ptr); 24160273720SAsias He else 2429b735910SMarc Zyngier virtio_mmio_config_in(vcpu, offset, data, len, ptr); 24360273720SAsias He } 24460273720SAsias He 2452454c7dcSWill Deacon #ifdef CONFIG_HAS_LIBFDT 2462454c7dcSWill Deacon #define DEVICE_NAME_MAX_LEN 32 2472bfd9ac3SAndre Przywara static 2482bfd9ac3SAndre Przywara void generate_virtio_mmio_fdt_node(void *fdt, 2492454c7dcSWill Deacon struct device_header *dev_hdr, 2502454c7dcSWill Deacon void (*generate_irq_prop)(void *fdt, 2512bfd9ac3SAndre Przywara u8 irq, 2522bfd9ac3SAndre Przywara enum irq_type)) 2532454c7dcSWill Deacon { 2542454c7dcSWill Deacon char dev_name[DEVICE_NAME_MAX_LEN]; 2552454c7dcSWill Deacon struct virtio_mmio *vmmio = container_of(dev_hdr, 2562454c7dcSWill Deacon struct virtio_mmio, 2572454c7dcSWill Deacon dev_hdr); 2582454c7dcSWill Deacon u64 addr = vmmio->addr; 2592454c7dcSWill Deacon u64 reg_prop[] = { 2602454c7dcSWill Deacon cpu_to_fdt64(addr), 2612454c7dcSWill Deacon cpu_to_fdt64(VIRTIO_MMIO_IO_SIZE), 2622454c7dcSWill Deacon }; 2632454c7dcSWill Deacon 2642454c7dcSWill Deacon snprintf(dev_name, DEVICE_NAME_MAX_LEN, "virtio@%llx", addr); 2652454c7dcSWill Deacon 2662454c7dcSWill Deacon _FDT(fdt_begin_node(fdt, dev_name)); 2672454c7dcSWill Deacon _FDT(fdt_property_string(fdt, "compatible", "virtio,mmio")); 2682454c7dcSWill Deacon _FDT(fdt_property(fdt, "reg", reg_prop, sizeof(reg_prop))); 2699a8af7e3SRobin Murphy _FDT(fdt_property(fdt, "dma-coherent", NULL, 0)); 2702bfd9ac3SAndre Przywara generate_irq_prop(fdt, vmmio->irq, IRQ_TYPE_EDGE_RISING); 2712454c7dcSWill Deacon _FDT(fdt_end_node(fdt)); 2722454c7dcSWill Deacon } 2732454c7dcSWill Deacon #else 2742454c7dcSWill Deacon static void generate_virtio_mmio_fdt_node(void *fdt, 2752454c7dcSWill Deacon struct device_header *dev_hdr, 2762454c7dcSWill Deacon void (*generate_irq_prop)(void *fdt, 2772454c7dcSWill Deacon u8 irq)) 2782454c7dcSWill Deacon { 2792454c7dcSWill Deacon die("Unable to generate device tree nodes without libfdt\n"); 2802454c7dcSWill Deacon } 2812454c7dcSWill Deacon #endif 2822454c7dcSWill Deacon 283b5981636SWill Deacon void virtio_mmio_assign_irq(struct device_header *dev_hdr) 284b5981636SWill Deacon { 285b5981636SWill Deacon struct virtio_mmio *vmmio = container_of(dev_hdr, 286b5981636SWill Deacon struct virtio_mmio, 287b5981636SWill Deacon dev_hdr); 288b5981636SWill Deacon vmmio->irq = irq__alloc_line(); 289b5981636SWill Deacon } 290b5981636SWill Deacon 29160273720SAsias He int virtio_mmio_init(struct kvm *kvm, void *dev, struct virtio_device *vdev, 29260273720SAsias He int device_id, int subsys_id, int class) 29360273720SAsias He { 29460273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 295*8f160708SAlexandru Elisei int r; 29660273720SAsias He 29760273720SAsias He vmmio->addr = virtio_mmio_get_io_space_block(VIRTIO_MMIO_IO_SIZE); 29860273720SAsias He vmmio->kvm = kvm; 29960273720SAsias He vmmio->dev = dev; 30060273720SAsias He 301*8f160708SAlexandru Elisei r = kvm__register_mmio(kvm, vmmio->addr, VIRTIO_MMIO_IO_SIZE, 30260273720SAsias He false, virtio_mmio_mmio_callback, vdev); 303*8f160708SAlexandru Elisei if (r < 0) 304*8f160708SAlexandru Elisei return r; 30560273720SAsias He 30660273720SAsias He vmmio->hdr = (struct virtio_mmio_hdr) { 30760273720SAsias He .magic = {'v', 'i', 'r', 't'}, 30860273720SAsias He .version = 1, 309449d5eb3SWill Deacon .device_id = subsys_id, 31060273720SAsias He .vendor_id = 0x4d564b4c , /* 'LKVM' */ 31160273720SAsias He .queue_num_max = 256, 31260273720SAsias He }; 31360273720SAsias He 31421ff329dSWill Deacon vmmio->dev_hdr = (struct device_header) { 31521ff329dSWill Deacon .bus_type = DEVICE_BUS_MMIO, 3162454c7dcSWill Deacon .data = generate_virtio_mmio_fdt_node, 31721ff329dSWill Deacon }; 31821ff329dSWill Deacon 319*8f160708SAlexandru Elisei r = device__register(&vmmio->dev_hdr); 320*8f160708SAlexandru Elisei if (r < 0) { 321*8f160708SAlexandru Elisei kvm__deregister_mmio(kvm, vmmio->addr); 322*8f160708SAlexandru Elisei return r; 323*8f160708SAlexandru Elisei } 32460273720SAsias He 32560273720SAsias He /* 32660273720SAsias He * Instantiate guest virtio-mmio devices using kernel command line 32760273720SAsias He * (or module) parameter, e.g 32860273720SAsias He * 32960273720SAsias He * virtio_mmio.devices=0x200@0xd2000000:5,0x200@0xd2000200:6 33060273720SAsias He */ 331e1c7c62aSAndre Przywara pr_debug("virtio-mmio.devices=0x%x@0x%x:%d", VIRTIO_MMIO_IO_SIZE, 332e1c7c62aSAndre Przywara vmmio->addr, vmmio->irq); 33360273720SAsias He 33460273720SAsias He return 0; 33560273720SAsias He } 33660273720SAsias He 337eb34a8c2SJean-Philippe Brucker int virtio_mmio_reset(struct kvm *kvm, struct virtio_device *vdev) 338eb34a8c2SJean-Philippe Brucker { 339eb34a8c2SJean-Philippe Brucker int vq; 340eb34a8c2SJean-Philippe Brucker struct virtio_mmio *vmmio = vdev->virtio; 341eb34a8c2SJean-Philippe Brucker 342eb34a8c2SJean-Philippe Brucker for (vq = 0; vq < vdev->ops->get_vq_count(kvm, vmmio->dev); vq++) 343eb34a8c2SJean-Philippe Brucker virtio_mmio_exit_vq(kvm, vdev, vq); 344eb34a8c2SJean-Philippe Brucker 345eb34a8c2SJean-Philippe Brucker return 0; 346eb34a8c2SJean-Philippe Brucker } 347eb34a8c2SJean-Philippe Brucker 34860273720SAsias He int virtio_mmio_exit(struct kvm *kvm, struct virtio_device *vdev) 34960273720SAsias He { 35060273720SAsias He struct virtio_mmio *vmmio = vdev->virtio; 35160273720SAsias He 352eb34a8c2SJean-Philippe Brucker virtio_mmio_reset(kvm, vdev); 35360273720SAsias He kvm__deregister_mmio(kvm, vmmio->addr); 35460273720SAsias He 35560273720SAsias He return 0; 35660273720SAsias He } 357