136f5dc91SSasha Levin #include "kvm/virtio-pci.h" 236f5dc91SSasha Levin 336f5dc91SSasha Levin #include "kvm/ioport.h" 436f5dc91SSasha Levin #include "kvm/kvm.h" 54123ca55SMarc Zyngier #include "kvm/kvm-cpu.h" 636f5dc91SSasha Levin #include "kvm/virtio-pci-dev.h" 736f5dc91SSasha Levin #include "kvm/irq.h" 836f5dc91SSasha Levin #include "kvm/virtio.h" 91599d724SSasha Levin #include "kvm/ioeventfd.h" 102e7380dbSMarc Zyngier #include "kvm/util.h" 1136f5dc91SSasha Levin 1243c81c74SSasha Levin #include <sys/ioctl.h> 1336f5dc91SSasha Levin #include <linux/virtio_pci.h> 14aa73be70SMatt Evans #include <linux/byteorder.h> 155a8e4f25SAlexandru Elisei #include <assert.h> 1636f5dc91SSasha Levin #include <string.h> 1736f5dc91SSasha Levin 182e7380dbSMarc Zyngier #define ALIGN_UP(x, s) ALIGN((x) + (s) - 1, (s)) 192e7380dbSMarc Zyngier #define VIRTIO_NR_MSIX (VIRTIO_PCI_MAX_VQ + VIRTIO_PCI_MAX_CONFIG) 202e7380dbSMarc Zyngier #define VIRTIO_MSIX_TABLE_SIZE (VIRTIO_NR_MSIX * 16) 212e7380dbSMarc Zyngier #define VIRTIO_MSIX_PBA_SIZE (ALIGN_UP(VIRTIO_MSIX_TABLE_SIZE, 64) / 8) 222e7380dbSMarc Zyngier #define VIRTIO_MSIX_BAR_SIZE (1UL << fls_long(VIRTIO_MSIX_TABLE_SIZE + \ 232e7380dbSMarc Zyngier VIRTIO_MSIX_PBA_SIZE)) 242e7380dbSMarc Zyngier 25e539f3e4SAlexandru Elisei static u16 virtio_pci__port_addr(struct virtio_pci *vpci) 26e539f3e4SAlexandru Elisei { 27e539f3e4SAlexandru Elisei return pci__bar_address(&vpci->pci_hdr, 0); 28e539f3e4SAlexandru Elisei } 29e539f3e4SAlexandru Elisei 30e539f3e4SAlexandru Elisei static u32 virtio_pci__mmio_addr(struct virtio_pci *vpci) 31e539f3e4SAlexandru Elisei { 32e539f3e4SAlexandru Elisei return pci__bar_address(&vpci->pci_hdr, 1); 33e539f3e4SAlexandru Elisei } 34e539f3e4SAlexandru Elisei 35e539f3e4SAlexandru Elisei static u32 virtio_pci__msix_io_addr(struct virtio_pci *vpci) 36e539f3e4SAlexandru Elisei { 37e539f3e4SAlexandru Elisei return pci__bar_address(&vpci->pci_hdr, 2); 38e539f3e4SAlexandru Elisei } 39e539f3e4SAlexandru Elisei 401599d724SSasha Levin static void virtio_pci__ioevent_callback(struct kvm *kvm, void *param) 411599d724SSasha Levin { 421599d724SSasha Levin struct virtio_pci_ioevent_param *ioeventfd = param; 4302eca50cSAsias He struct virtio_pci *vpci = ioeventfd->vdev->virtio; 441599d724SSasha Levin 4502eca50cSAsias He ioeventfd->vdev->ops->notify_vq(kvm, vpci->dev, ioeventfd->vq); 461599d724SSasha Levin } 471599d724SSasha Levin 4802eca50cSAsias He static int virtio_pci__init_ioeventfd(struct kvm *kvm, struct virtio_device *vdev, u32 vq) 491599d724SSasha Levin { 501599d724SSasha Levin struct ioevent ioevent; 5102eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 52e539f3e4SAlexandru Elisei u32 mmio_addr = virtio_pci__mmio_addr(vpci); 53e539f3e4SAlexandru Elisei u16 port_addr = virtio_pci__port_addr(vpci); 540e1882a4SJean-Philippe Brucker int r, flags = 0; 550e1882a4SJean-Philippe Brucker int fd; 561599d724SSasha Levin 571599d724SSasha Levin vpci->ioeventfds[vq] = (struct virtio_pci_ioevent_param) { 5802eca50cSAsias He .vdev = vdev, 591599d724SSasha Levin .vq = vq, 601599d724SSasha Levin }; 611599d724SSasha Levin 621599d724SSasha Levin ioevent = (struct ioevent) { 631599d724SSasha Levin .fn = virtio_pci__ioevent_callback, 641599d724SSasha Levin .fn_ptr = &vpci->ioeventfds[vq], 651599d724SSasha Levin .datamatch = vq, 661599d724SSasha Levin .fn_kvm = kvm, 671599d724SSasha Levin }; 681599d724SSasha Levin 69627d6874SAsias He /* 70a463650cSWill Deacon * Vhost will poll the eventfd in host kernel side, otherwise we 71a463650cSWill Deacon * need to poll in userspace. 72627d6874SAsias He */ 73a463650cSWill Deacon if (!vdev->use_vhost) 74a463650cSWill Deacon flags |= IOEVENTFD_FLAG_USER_POLL; 75a463650cSWill Deacon 76a463650cSWill Deacon /* ioport */ 77e539f3e4SAlexandru Elisei ioevent.io_addr = port_addr + VIRTIO_PCI_QUEUE_NOTIFY; 78a463650cSWill Deacon ioevent.io_len = sizeof(u16); 790e1882a4SJean-Philippe Brucker ioevent.fd = fd = eventfd(0, 0); 8071ca0facSAndre Przywara r = ioeventfd__add_event(&ioevent, flags | IOEVENTFD_FLAG_PIO); 81ea6eeb1cSSasha Levin if (r) 82ea6eeb1cSSasha Levin return r; 831599d724SSasha Levin 84a463650cSWill Deacon /* mmio */ 85e539f3e4SAlexandru Elisei ioevent.io_addr = mmio_addr + VIRTIO_PCI_QUEUE_NOTIFY; 86fe50bacbSAndreas Herrmann ioevent.io_len = sizeof(u16); 870e1882a4SJean-Philippe Brucker ioevent.fd = eventfd(0, 0); 88a463650cSWill Deacon r = ioeventfd__add_event(&ioevent, flags); 89a463650cSWill Deacon if (r) 90a463650cSWill Deacon goto free_ioport_evt; 91263b80e8SSasha Levin 92a463650cSWill Deacon if (vdev->ops->notify_vq_eventfd) 930e1882a4SJean-Philippe Brucker vdev->ops->notify_vq_eventfd(kvm, vpci->dev, vq, fd); 941599d724SSasha Levin return 0; 95a463650cSWill Deacon 96a463650cSWill Deacon free_ioport_evt: 97e539f3e4SAlexandru Elisei ioeventfd__del_event(port_addr + VIRTIO_PCI_QUEUE_NOTIFY, vq); 98a463650cSWill Deacon return r; 991599d724SSasha Levin } 1001599d724SSasha Levin 101ad346c2eSJean-Philippe Brucker static void virtio_pci_exit_vq(struct kvm *kvm, struct virtio_device *vdev, 102ad346c2eSJean-Philippe Brucker int vq) 103ad346c2eSJean-Philippe Brucker { 104ad346c2eSJean-Philippe Brucker struct virtio_pci *vpci = vdev->virtio; 105e539f3e4SAlexandru Elisei u32 mmio_addr = virtio_pci__mmio_addr(vpci); 106e539f3e4SAlexandru Elisei u16 port_addr = virtio_pci__port_addr(vpci); 107ad346c2eSJean-Philippe Brucker 108e539f3e4SAlexandru Elisei ioeventfd__del_event(mmio_addr + VIRTIO_PCI_QUEUE_NOTIFY, vq); 109e539f3e4SAlexandru Elisei ioeventfd__del_event(port_addr + VIRTIO_PCI_QUEUE_NOTIFY, vq); 110ad346c2eSJean-Philippe Brucker virtio_exit_vq(kvm, vdev, vpci->dev, vq); 111ad346c2eSJean-Philippe Brucker } 112ad346c2eSJean-Philippe Brucker 11306f48103SSasha Levin static inline bool virtio_pci__msix_enabled(struct virtio_pci *vpci) 11406f48103SSasha Levin { 115aa73be70SMatt Evans return vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_ENABLE); 11606f48103SSasha Levin } 11706f48103SSasha Levin 118e09b599aSJulien Thierry static bool virtio_pci__specific_data_in(struct kvm *kvm, struct virtio_device *vdev, 119e09b599aSJulien Thierry void *data, int size, unsigned long offset) 12036f5dc91SSasha Levin { 12136f5dc91SSasha Levin u32 config_offset; 12202eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 12306f48103SSasha Levin int type = virtio__get_dev_specific_field(offset - 20, 12406f48103SSasha Levin virtio_pci__msix_enabled(vpci), 1251382aba0SSasha Levin &config_offset); 12636f5dc91SSasha Levin if (type == VIRTIO_PCI_O_MSIX) { 12736f5dc91SSasha Levin switch (offset) { 12836f5dc91SSasha Levin case VIRTIO_MSI_CONFIG_VECTOR: 12936f5dc91SSasha Levin ioport__write16(data, vpci->config_vector); 13036f5dc91SSasha Levin break; 13136f5dc91SSasha Levin case VIRTIO_MSI_QUEUE_VECTOR: 13236f5dc91SSasha Levin ioport__write16(data, vpci->vq_vector[vpci->queue_selector]); 13336f5dc91SSasha Levin break; 13436f5dc91SSasha Levin }; 13536f5dc91SSasha Levin 13636f5dc91SSasha Levin return true; 13736f5dc91SSasha Levin } else if (type == VIRTIO_PCI_O_CONFIG) { 13836f5dc91SSasha Levin u8 cfg; 13936f5dc91SSasha Levin 140c5ae742bSSasha Levin cfg = vdev->ops->get_config(kvm, vpci->dev)[config_offset]; 14136f5dc91SSasha Levin ioport__write8(data, cfg); 14236f5dc91SSasha Levin return true; 14336f5dc91SSasha Levin } 14436f5dc91SSasha Levin 14536f5dc91SSasha Levin return false; 14636f5dc91SSasha Levin } 14736f5dc91SSasha Levin 148e09b599aSJulien Thierry static bool virtio_pci__data_in(struct kvm_cpu *vcpu, struct virtio_device *vdev, 149e09b599aSJulien Thierry unsigned long offset, void *data, int size) 15036f5dc91SSasha Levin { 15136f5dc91SSasha Levin bool ret = true; 15236f5dc91SSasha Levin struct virtio_pci *vpci; 15353fbb17bSJean-Philippe Brucker struct virt_queue *vq; 1544123ca55SMarc Zyngier struct kvm *kvm; 15536f5dc91SSasha Levin u32 val; 15636f5dc91SSasha Levin 1574123ca55SMarc Zyngier kvm = vcpu->kvm; 15802eca50cSAsias He vpci = vdev->virtio; 15936f5dc91SSasha Levin 16036f5dc91SSasha Levin switch (offset) { 16136f5dc91SSasha Levin case VIRTIO_PCI_HOST_FEATURES: 16202eca50cSAsias He val = vdev->ops->get_host_features(kvm, vpci->dev); 16336f5dc91SSasha Levin ioport__write32(data, val); 16436f5dc91SSasha Levin break; 16536f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_PFN: 16653fbb17bSJean-Philippe Brucker vq = vdev->ops->get_vq(kvm, vpci->dev, vpci->queue_selector); 16753fbb17bSJean-Philippe Brucker ioport__write32(data, vq->pfn); 16836f5dc91SSasha Levin break; 16936f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_NUM: 17002eca50cSAsias He val = vdev->ops->get_size_vq(kvm, vpci->dev, vpci->queue_selector); 171657ee18bSMatt Evans ioport__write16(data, val); 17236f5dc91SSasha Levin break; 17336f5dc91SSasha Levin case VIRTIO_PCI_STATUS: 17436f5dc91SSasha Levin ioport__write8(data, vpci->status); 17536f5dc91SSasha Levin break; 17636f5dc91SSasha Levin case VIRTIO_PCI_ISR: 17736f5dc91SSasha Levin ioport__write8(data, vpci->isr); 178e9922aafSAndre Przywara kvm__irq_line(kvm, vpci->legacy_irq_line, VIRTIO_IRQ_LOW); 17936f5dc91SSasha Levin vpci->isr = VIRTIO_IRQ_LOW; 18036f5dc91SSasha Levin break; 18136f5dc91SSasha Levin default: 182e09b599aSJulien Thierry ret = virtio_pci__specific_data_in(kvm, vdev, data, size, offset); 18336f5dc91SSasha Levin break; 18436f5dc91SSasha Levin }; 18536f5dc91SSasha Levin 18636f5dc91SSasha Levin return ret; 18736f5dc91SSasha Levin } 18836f5dc91SSasha Levin 1896518065aSAndre Przywara static void update_msix_map(struct virtio_pci *vpci, 1906518065aSAndre Przywara struct msix_table *msix_entry, u32 vecnum) 1916518065aSAndre Przywara { 1926518065aSAndre Przywara u32 gsi, i; 1936518065aSAndre Przywara 1946518065aSAndre Przywara /* Find the GSI number used for that vector */ 1956518065aSAndre Przywara if (vecnum == vpci->config_vector) { 1966518065aSAndre Przywara gsi = vpci->config_gsi; 1976518065aSAndre Przywara } else { 1986518065aSAndre Przywara for (i = 0; i < VIRTIO_PCI_MAX_VQ; i++) 1996518065aSAndre Przywara if (vpci->vq_vector[i] == vecnum) 2006518065aSAndre Przywara break; 2016518065aSAndre Przywara if (i == VIRTIO_PCI_MAX_VQ) 2026518065aSAndre Przywara return; 2036518065aSAndre Przywara gsi = vpci->gsis[i]; 2046518065aSAndre Przywara } 2056518065aSAndre Przywara 2066518065aSAndre Przywara if (gsi == 0) 2076518065aSAndre Przywara return; 2086518065aSAndre Przywara 2096518065aSAndre Przywara msix_entry = &msix_entry[vecnum]; 2106518065aSAndre Przywara irq__update_msix_route(vpci->kvm, gsi, &msix_entry->msg); 2116518065aSAndre Przywara } 2126518065aSAndre Przywara 213e09b599aSJulien Thierry static bool virtio_pci__specific_data_out(struct kvm *kvm, struct virtio_device *vdev, 214e09b599aSJulien Thierry void *data, int size, unsigned long offset) 21536f5dc91SSasha Levin { 21602eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 2178ccc8549SAndre Przywara u32 config_offset, vec; 2188ccc8549SAndre Przywara int gsi; 21906f48103SSasha Levin int type = virtio__get_dev_specific_field(offset - 20, virtio_pci__msix_enabled(vpci), 2201382aba0SSasha Levin &config_offset); 22136f5dc91SSasha Levin if (type == VIRTIO_PCI_O_MSIX) { 22236f5dc91SSasha Levin switch (offset) { 22336f5dc91SSasha Levin case VIRTIO_MSI_CONFIG_VECTOR: 22436f5dc91SSasha Levin vec = vpci->config_vector = ioport__read16(data); 225f8327b05SSasha Levin if (vec == VIRTIO_MSI_NO_VECTOR) 226f8327b05SSasha Levin break; 22736f5dc91SSasha Levin 2288ccc8549SAndre Przywara gsi = irq__add_msix_route(kvm, 229f9ef46f2SAndre Przywara &vpci->msix_table[vec].msg, 230f9ef46f2SAndre Przywara vpci->dev_hdr.dev_num << 3); 231928ab7acSAndre Przywara /* 232928ab7acSAndre Przywara * We don't need IRQ routing if we can use 233928ab7acSAndre Przywara * MSI injection via the KVM_SIGNAL_MSI ioctl. 234928ab7acSAndre Przywara */ 235928ab7acSAndre Przywara if (gsi == -ENXIO && 236928ab7acSAndre Przywara vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 237928ab7acSAndre Przywara break; 238928ab7acSAndre Przywara 239928ab7acSAndre Przywara if (gsi < 0) { 240928ab7acSAndre Przywara die("failed to configure MSIs"); 241928ab7acSAndre Przywara break; 242928ab7acSAndre Przywara } 243928ab7acSAndre Przywara 24436f5dc91SSasha Levin vpci->config_gsi = gsi; 24536f5dc91SSasha Levin break; 2463a60be06SSasha Levin case VIRTIO_MSI_QUEUE_VECTOR: 2478ccc8549SAndre Przywara vec = ioport__read16(data); 2488ccc8549SAndre Przywara vpci->vq_vector[vpci->queue_selector] = vec; 24936f5dc91SSasha Levin 250f8327b05SSasha Levin if (vec == VIRTIO_MSI_NO_VECTOR) 251f8327b05SSasha Levin break; 252f8327b05SSasha Levin 2538ccc8549SAndre Przywara gsi = irq__add_msix_route(kvm, 254f9ef46f2SAndre Przywara &vpci->msix_table[vec].msg, 255f9ef46f2SAndre Przywara vpci->dev_hdr.dev_num << 3); 256928ab7acSAndre Przywara /* 257928ab7acSAndre Przywara * We don't need IRQ routing if we can use 258928ab7acSAndre Przywara * MSI injection via the KVM_SIGNAL_MSI ioctl. 259928ab7acSAndre Przywara */ 260928ab7acSAndre Przywara if (gsi == -ENXIO && 261928ab7acSAndre Przywara vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 2628ccc8549SAndre Przywara break; 263928ab7acSAndre Przywara 264928ab7acSAndre Przywara if (gsi < 0) { 265928ab7acSAndre Przywara die("failed to configure MSIs"); 266928ab7acSAndre Przywara break; 267928ab7acSAndre Przywara } 268928ab7acSAndre Przywara 26936f5dc91SSasha Levin vpci->gsis[vpci->queue_selector] = gsi; 27002eca50cSAsias He if (vdev->ops->notify_vq_gsi) 27102eca50cSAsias He vdev->ops->notify_vq_gsi(kvm, vpci->dev, 2728ccc8549SAndre Przywara vpci->queue_selector, 2738ccc8549SAndre Przywara gsi); 27436f5dc91SSasha Levin break; 27536f5dc91SSasha Levin }; 27636f5dc91SSasha Levin 27736f5dc91SSasha Levin return true; 27836f5dc91SSasha Levin } else if (type == VIRTIO_PCI_O_CONFIG) { 279c5ae742bSSasha Levin vdev->ops->get_config(kvm, vpci->dev)[config_offset] = *(u8 *)data; 28036f5dc91SSasha Levin 28136f5dc91SSasha Levin return true; 28236f5dc91SSasha Levin } 28336f5dc91SSasha Levin 28436f5dc91SSasha Levin return false; 28536f5dc91SSasha Levin } 28636f5dc91SSasha Levin 287e09b599aSJulien Thierry static bool virtio_pci__data_out(struct kvm_cpu *vcpu, struct virtio_device *vdev, 288e09b599aSJulien Thierry unsigned long offset, void *data, int size) 28936f5dc91SSasha Levin { 29036f5dc91SSasha Levin bool ret = true; 29136f5dc91SSasha Levin struct virtio_pci *vpci; 2924123ca55SMarc Zyngier struct kvm *kvm; 29336f5dc91SSasha Levin u32 val; 29436f5dc91SSasha Levin 2954123ca55SMarc Zyngier kvm = vcpu->kvm; 29602eca50cSAsias He vpci = vdev->virtio; 29736f5dc91SSasha Levin 29836f5dc91SSasha Levin switch (offset) { 29936f5dc91SSasha Levin case VIRTIO_PCI_GUEST_FEATURES: 30036f5dc91SSasha Levin val = ioport__read32(data); 30156a16c90SJean-Philippe Brucker virtio_set_guest_features(kvm, vdev, vpci->dev, val); 30236f5dc91SSasha Levin break; 30336f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_PFN: 30436f5dc91SSasha Levin val = ioport__read32(data); 305ad346c2eSJean-Philippe Brucker if (val) { 306ad346c2eSJean-Philippe Brucker virtio_pci__init_ioeventfd(kvm, vdev, 307ad346c2eSJean-Philippe Brucker vpci->queue_selector); 308c59ba304SWill Deacon vdev->ops->init_vq(kvm, vpci->dev, vpci->queue_selector, 309c59ba304SWill Deacon 1 << VIRTIO_PCI_QUEUE_ADDR_SHIFT, 310c59ba304SWill Deacon VIRTIO_PCI_VRING_ALIGN, val); 311ad346c2eSJean-Philippe Brucker } else { 312ad346c2eSJean-Philippe Brucker virtio_pci_exit_vq(kvm, vdev, vpci->queue_selector); 313ad346c2eSJean-Philippe Brucker } 31436f5dc91SSasha Levin break; 31536f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_SEL: 31636f5dc91SSasha Levin vpci->queue_selector = ioport__read16(data); 31736f5dc91SSasha Levin break; 31836f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_NOTIFY: 31936f5dc91SSasha Levin val = ioport__read16(data); 32002eca50cSAsias He vdev->ops->notify_vq(kvm, vpci->dev, val); 32136f5dc91SSasha Levin break; 32236f5dc91SSasha Levin case VIRTIO_PCI_STATUS: 32336f5dc91SSasha Levin vpci->status = ioport__read8(data); 3244123ca55SMarc Zyngier if (!vpci->status) /* Sample endianness on reset */ 3254123ca55SMarc Zyngier vdev->endian = kvm_cpu__get_endianness(vcpu); 32695242e44SJean-Philippe Brucker virtio_notify_status(kvm, vdev, vpci->dev, vpci->status); 32736f5dc91SSasha Levin break; 32836f5dc91SSasha Levin default: 329e09b599aSJulien Thierry ret = virtio_pci__specific_data_out(kvm, vdev, data, size, offset); 33036f5dc91SSasha Levin break; 33136f5dc91SSasha Levin }; 33236f5dc91SSasha Levin 33336f5dc91SSasha Levin return ret; 33436f5dc91SSasha Levin } 33536f5dc91SSasha Levin 3369b735910SMarc Zyngier static void virtio_pci__msix_mmio_callback(struct kvm_cpu *vcpu, 3379b735910SMarc Zyngier u64 addr, u8 *data, u32 len, 338a463650cSWill Deacon u8 is_write, void *ptr) 33936f5dc91SSasha Levin { 340e09b599aSJulien Thierry struct virtio_device *vdev = ptr; 341e09b599aSJulien Thierry struct virtio_pci *vpci = vdev->virtio; 3426518065aSAndre Przywara struct msix_table *table; 343e539f3e4SAlexandru Elisei u32 msix_io_addr = virtio_pci__msix_io_addr(vpci); 3442e7380dbSMarc Zyngier u32 pba_offset; 3456518065aSAndre Przywara int vecnum; 3466518065aSAndre Przywara size_t offset; 34736f5dc91SSasha Levin 3482e7380dbSMarc Zyngier BUILD_BUG_ON(VIRTIO_NR_MSIX > (sizeof(vpci->msix_pba) * 8)); 3492e7380dbSMarc Zyngier 3502e7380dbSMarc Zyngier pba_offset = vpci->pci_hdr.msix.pba_offset & ~PCI_MSIX_TABLE_BIR; 3512e7380dbSMarc Zyngier if (addr >= msix_io_addr + pba_offset) { 3522e7380dbSMarc Zyngier /* Read access to PBA */ 3536518065aSAndre Przywara if (is_write) 3546518065aSAndre Przywara return; 3552e7380dbSMarc Zyngier offset = addr - (msix_io_addr + pba_offset); 3562e7380dbSMarc Zyngier if ((offset + len) > sizeof (vpci->msix_pba)) 3572e7380dbSMarc Zyngier return; 3582e7380dbSMarc Zyngier memcpy(data, (void *)&vpci->msix_pba + offset, len); 3592e7380dbSMarc Zyngier return; 3602e7380dbSMarc Zyngier } 3612e7380dbSMarc Zyngier 3626518065aSAndre Przywara table = vpci->msix_table; 363e539f3e4SAlexandru Elisei offset = addr - msix_io_addr; 3642e7380dbSMarc Zyngier 3656518065aSAndre Przywara vecnum = offset / sizeof(struct msix_table); 3666518065aSAndre Przywara offset = offset % sizeof(struct msix_table); 3676518065aSAndre Przywara 3686518065aSAndre Przywara if (!is_write) { 3696518065aSAndre Przywara memcpy(data, (void *)&table[vecnum] + offset, len); 3706518065aSAndre Przywara return; 37136f5dc91SSasha Levin } 37236f5dc91SSasha Levin 3736518065aSAndre Przywara memcpy((void *)&table[vecnum] + offset, data, len); 3746518065aSAndre Przywara 3756518065aSAndre Przywara /* Did we just update the address or payload? */ 3766518065aSAndre Przywara if (offset < offsetof(struct msix_table, ctrl)) 3776518065aSAndre Przywara update_msix_map(vpci, table, vecnum); 37806f48103SSasha Levin } 37906f48103SSasha Levin 380714ab9e6SAndre Przywara static void virtio_pci__signal_msi(struct kvm *kvm, struct virtio_pci *vpci, 381714ab9e6SAndre Przywara int vec) 38243c81c74SSasha Levin { 38343c81c74SSasha Levin struct kvm_msi msi = { 38443c81c74SSasha Levin .address_lo = vpci->msix_table[vec].msg.address_lo, 38543c81c74SSasha Levin .address_hi = vpci->msix_table[vec].msg.address_hi, 38643c81c74SSasha Levin .data = vpci->msix_table[vec].msg.data, 38743c81c74SSasha Levin }; 38843c81c74SSasha Levin 389714ab9e6SAndre Przywara if (kvm->msix_needs_devid) { 390714ab9e6SAndre Przywara msi.flags = KVM_MSI_VALID_DEVID; 391714ab9e6SAndre Przywara msi.devid = vpci->dev_hdr.dev_num << 3; 392714ab9e6SAndre Przywara } 393714ab9e6SAndre Przywara 394f6108d72SJean-Philippe Brucker irq__signal_msi(kvm, &msi); 39543c81c74SSasha Levin } 39643c81c74SSasha Levin 39702eca50cSAsias He int virtio_pci__signal_vq(struct kvm *kvm, struct virtio_device *vdev, u32 vq) 39836f5dc91SSasha Levin { 39902eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 40006f48103SSasha Levin int tbl = vpci->vq_vector[vq]; 40136f5dc91SSasha Levin 402f8327b05SSasha Levin if (virtio_pci__msix_enabled(vpci) && tbl != VIRTIO_MSI_NO_VECTOR) { 403aa73be70SMatt Evans if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) || 404aa73be70SMatt Evans vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 40506f48103SSasha Levin 40606f48103SSasha Levin vpci->msix_pba |= 1 << tbl; 40706f48103SSasha Levin return 0; 40806f48103SSasha Levin } 40906f48103SSasha Levin 41043c81c74SSasha Levin if (vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 41143c81c74SSasha Levin virtio_pci__signal_msi(kvm, vpci, vpci->vq_vector[vq]); 41243c81c74SSasha Levin else 41306f48103SSasha Levin kvm__irq_trigger(kvm, vpci->gsis[vq]); 41406f48103SSasha Levin } else { 415a36eca7bSSasha Levin vpci->isr = VIRTIO_IRQ_HIGH; 416*2108c86dSMarc Zyngier kvm__irq_line(kvm, vpci->legacy_irq_line, VIRTIO_IRQ_HIGH); 41706f48103SSasha Levin } 41836f5dc91SSasha Levin return 0; 41936f5dc91SSasha Levin } 42036f5dc91SSasha Levin 42102eca50cSAsias He int virtio_pci__signal_config(struct kvm *kvm, struct virtio_device *vdev) 42236f5dc91SSasha Levin { 42302eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 42406f48103SSasha Levin int tbl = vpci->config_vector; 42506f48103SSasha Levin 426f8327b05SSasha Levin if (virtio_pci__msix_enabled(vpci) && tbl != VIRTIO_MSI_NO_VECTOR) { 427aa73be70SMatt Evans if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) || 428aa73be70SMatt Evans vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 42906f48103SSasha Levin 43006f48103SSasha Levin vpci->msix_pba |= 1 << tbl; 43106f48103SSasha Levin return 0; 43206f48103SSasha Levin } 43306f48103SSasha Levin 43443c81c74SSasha Levin if (vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 435f8327b05SSasha Levin virtio_pci__signal_msi(kvm, vpci, tbl); 43643c81c74SSasha Levin else 43706f48103SSasha Levin kvm__irq_trigger(kvm, vpci->config_gsi); 43806f48103SSasha Levin } else { 43906f48103SSasha Levin vpci->isr = VIRTIO_PCI_ISR_CONFIG; 440e9922aafSAndre Przywara kvm__irq_trigger(kvm, vpci->legacy_irq_line); 44106f48103SSasha Levin } 44236f5dc91SSasha Levin 44336f5dc91SSasha Levin return 0; 44436f5dc91SSasha Levin } 44536f5dc91SSasha Levin 4469b735910SMarc Zyngier static void virtio_pci__io_mmio_callback(struct kvm_cpu *vcpu, 4479b735910SMarc Zyngier u64 addr, u8 *data, u32 len, 448a463650cSWill Deacon u8 is_write, void *ptr) 449a463650cSWill Deacon { 450e09b599aSJulien Thierry struct virtio_device *vdev = ptr; 451e09b599aSJulien Thierry struct virtio_pci *vpci = vdev->virtio; 452205eaa79SAndre Przywara u32 ioport_addr = virtio_pci__port_addr(vpci); 453205eaa79SAndre Przywara u32 base_addr; 454205eaa79SAndre Przywara 455205eaa79SAndre Przywara if (addr >= ioport_addr && 456205eaa79SAndre Przywara addr < ioport_addr + pci__bar_size(&vpci->pci_hdr, 0)) 457205eaa79SAndre Przywara base_addr = ioport_addr; 458205eaa79SAndre Przywara else 459205eaa79SAndre Przywara base_addr = virtio_pci__mmio_addr(vpci); 460a463650cSWill Deacon 461e09b599aSJulien Thierry if (!is_write) 462205eaa79SAndre Przywara virtio_pci__data_in(vcpu, vdev, addr - base_addr, data, len); 463e09b599aSJulien Thierry else 464205eaa79SAndre Przywara virtio_pci__data_out(vcpu, vdev, addr - base_addr, data, len); 465a463650cSWill Deacon } 466a463650cSWill Deacon 4675a8e4f25SAlexandru Elisei static int virtio_pci__bar_activate(struct kvm *kvm, 4685a8e4f25SAlexandru Elisei struct pci_device_header *pci_hdr, 4695a8e4f25SAlexandru Elisei int bar_num, void *data) 4705a8e4f25SAlexandru Elisei { 4715a8e4f25SAlexandru Elisei struct virtio_device *vdev = data; 4725a8e4f25SAlexandru Elisei u32 bar_addr, bar_size; 4735a8e4f25SAlexandru Elisei int r = -EINVAL; 4745a8e4f25SAlexandru Elisei 4755a8e4f25SAlexandru Elisei assert(bar_num <= 2); 4765a8e4f25SAlexandru Elisei 4775a8e4f25SAlexandru Elisei bar_addr = pci__bar_address(pci_hdr, bar_num); 4785a8e4f25SAlexandru Elisei bar_size = pci__bar_size(pci_hdr, bar_num); 4795a8e4f25SAlexandru Elisei 4805a8e4f25SAlexandru Elisei switch (bar_num) { 4815a8e4f25SAlexandru Elisei case 0: 482205eaa79SAndre Przywara r = kvm__register_pio(kvm, bar_addr, bar_size, 483205eaa79SAndre Przywara virtio_pci__io_mmio_callback, vdev); 4845a8e4f25SAlexandru Elisei break; 4855a8e4f25SAlexandru Elisei case 1: 4865a8e4f25SAlexandru Elisei r = kvm__register_mmio(kvm, bar_addr, bar_size, false, 4875a8e4f25SAlexandru Elisei virtio_pci__io_mmio_callback, vdev); 4885a8e4f25SAlexandru Elisei break; 4895a8e4f25SAlexandru Elisei case 2: 4905a8e4f25SAlexandru Elisei r = kvm__register_mmio(kvm, bar_addr, bar_size, false, 4915a8e4f25SAlexandru Elisei virtio_pci__msix_mmio_callback, vdev); 4925a8e4f25SAlexandru Elisei break; 4935a8e4f25SAlexandru Elisei } 4945a8e4f25SAlexandru Elisei 4955a8e4f25SAlexandru Elisei return r; 4965a8e4f25SAlexandru Elisei } 4975a8e4f25SAlexandru Elisei 4985a8e4f25SAlexandru Elisei static int virtio_pci__bar_deactivate(struct kvm *kvm, 4995a8e4f25SAlexandru Elisei struct pci_device_header *pci_hdr, 5005a8e4f25SAlexandru Elisei int bar_num, void *data) 5015a8e4f25SAlexandru Elisei { 5025a8e4f25SAlexandru Elisei u32 bar_addr; 5035a8e4f25SAlexandru Elisei bool success; 5045a8e4f25SAlexandru Elisei int r = -EINVAL; 5055a8e4f25SAlexandru Elisei 5065a8e4f25SAlexandru Elisei assert(bar_num <= 2); 5075a8e4f25SAlexandru Elisei 5085a8e4f25SAlexandru Elisei bar_addr = pci__bar_address(pci_hdr, bar_num); 5095a8e4f25SAlexandru Elisei 5105a8e4f25SAlexandru Elisei switch (bar_num) { 5115a8e4f25SAlexandru Elisei case 0: 512205eaa79SAndre Przywara r = kvm__deregister_pio(kvm, bar_addr); 5135a8e4f25SAlexandru Elisei break; 5145a8e4f25SAlexandru Elisei case 1: 5155a8e4f25SAlexandru Elisei case 2: 5165a8e4f25SAlexandru Elisei success = kvm__deregister_mmio(kvm, bar_addr); 5175a8e4f25SAlexandru Elisei /* kvm__deregister_mmio fails when the region is not found. */ 5185a8e4f25SAlexandru Elisei r = (success ? 0 : -ENOENT); 5195a8e4f25SAlexandru Elisei break; 5205a8e4f25SAlexandru Elisei } 5215a8e4f25SAlexandru Elisei 5225a8e4f25SAlexandru Elisei return r; 5235a8e4f25SAlexandru Elisei } 5245a8e4f25SAlexandru Elisei 52502eca50cSAsias He int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev, 526507e02d8SAsias He int device_id, int subsys_id, int class) 52736f5dc91SSasha Levin { 52802eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 529e539f3e4SAlexandru Elisei u32 mmio_addr, msix_io_block; 530e539f3e4SAlexandru Elisei u16 port_addr; 5317af40b91SSasha Levin int r; 53236f5dc91SSasha Levin 533a463650cSWill Deacon vpci->kvm = kvm; 53436f5dc91SSasha Levin vpci->dev = dev; 53536f5dc91SSasha Levin 536ce2fc8f5SAlexandru Elisei BUILD_BUG_ON(!is_power_of_two(PCI_IO_SIZE)); 537ce2fc8f5SAlexandru Elisei 538e539f3e4SAlexandru Elisei port_addr = pci_get_io_port_block(PCI_IO_SIZE); 539e539f3e4SAlexandru Elisei mmio_addr = pci_get_mmio_block(PCI_IO_SIZE); 5402e7380dbSMarc Zyngier msix_io_block = pci_get_mmio_block(VIRTIO_MSIX_BAR_SIZE); 541a463650cSWill Deacon 54236f5dc91SSasha Levin vpci->pci_hdr = (struct pci_device_header) { 543aa73be70SMatt Evans .vendor_id = cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET), 544aa73be70SMatt Evans .device_id = cpu_to_le16(device_id), 545ec7dd52fSSasha Levin .command = PCI_COMMAND_IO | PCI_COMMAND_MEMORY, 54636f5dc91SSasha Levin .header_type = PCI_HEADER_TYPE_NORMAL, 54736f5dc91SSasha Levin .revision_id = 0, 548aa73be70SMatt Evans .class[0] = class & 0xff, 549aa73be70SMatt Evans .class[1] = (class >> 8) & 0xff, 550aa73be70SMatt Evans .class[2] = (class >> 16) & 0xff, 551aa73be70SMatt Evans .subsys_vendor_id = cpu_to_le16(PCI_SUBSYSTEM_VENDOR_ID_REDHAT_QUMRANET), 552aa73be70SMatt Evans .subsys_id = cpu_to_le16(subsys_id), 553e539f3e4SAlexandru Elisei .bar[0] = cpu_to_le32(port_addr 5549c26dab4SSasha Levin | PCI_BASE_ADDRESS_SPACE_IO), 555e539f3e4SAlexandru Elisei .bar[1] = cpu_to_le32(mmio_addr 556a508ea95SJean-Philippe Brucker | PCI_BASE_ADDRESS_SPACE_MEMORY), 557e539f3e4SAlexandru Elisei .bar[2] = cpu_to_le32(msix_io_block 558b4dab816SSasha Levin | PCI_BASE_ADDRESS_SPACE_MEMORY), 559aa73be70SMatt Evans .status = cpu_to_le16(PCI_STATUS_CAP_LIST), 56036f5dc91SSasha Levin .capabilities = (void *)&vpci->pci_hdr.msix - (void *)&vpci->pci_hdr, 56148843d10SJulien Thierry .bar_size[0] = cpu_to_le32(PCI_IO_SIZE), 56248843d10SJulien Thierry .bar_size[1] = cpu_to_le32(PCI_IO_SIZE), 5632e7380dbSMarc Zyngier .bar_size[2] = cpu_to_le32(VIRTIO_MSIX_BAR_SIZE), 56436f5dc91SSasha Levin }; 56536f5dc91SSasha Levin 5665a8e4f25SAlexandru Elisei r = pci__register_bar_regions(kvm, &vpci->pci_hdr, 5675a8e4f25SAlexandru Elisei virtio_pci__bar_activate, 5685a8e4f25SAlexandru Elisei virtio_pci__bar_deactivate, vdev); 5695a8e4f25SAlexandru Elisei if (r < 0) 5705a8e4f25SAlexandru Elisei return r; 5715a8e4f25SAlexandru Elisei 57221ff329dSWill Deacon vpci->dev_hdr = (struct device_header) { 57321ff329dSWill Deacon .bus_type = DEVICE_BUS_PCI, 57421ff329dSWill Deacon .data = &vpci->pci_hdr, 57521ff329dSWill Deacon }; 57621ff329dSWill Deacon 57736f5dc91SSasha Levin vpci->pci_hdr.msix.cap = PCI_CAP_ID_MSIX; 57836f5dc91SSasha Levin vpci->pci_hdr.msix.next = 0; 57914bba8a0SAsias He /* 5802e7380dbSMarc Zyngier * We at most have VIRTIO_NR_MSIX entries (VIRTIO_PCI_MAX_VQ 5812e7380dbSMarc Zyngier * entries for virt queue, VIRTIO_PCI_MAX_CONFIG entries for 5822e7380dbSMarc Zyngier * config). 58314bba8a0SAsias He * 58414bba8a0SAsias He * To quote the PCI spec: 58514bba8a0SAsias He * 58614bba8a0SAsias He * System software reads this field to determine the 58714bba8a0SAsias He * MSI-X Table Size N, which is encoded as N-1. 58814bba8a0SAsias He * For example, a returned value of "00000000011" 58914bba8a0SAsias He * indicates a table size of 4. 59014bba8a0SAsias He */ 5912e7380dbSMarc Zyngier vpci->pci_hdr.msix.ctrl = cpu_to_le16(VIRTIO_NR_MSIX - 1); 59206f48103SSasha Levin 593a463650cSWill Deacon /* Both table and PBA are mapped to the same BAR (2) */ 594a463650cSWill Deacon vpci->pci_hdr.msix.table_offset = cpu_to_le32(2); 5952e7380dbSMarc Zyngier vpci->pci_hdr.msix.pba_offset = cpu_to_le32(2 | VIRTIO_MSIX_TABLE_SIZE); 59636f5dc91SSasha Levin vpci->config_vector = 0; 59736f5dc91SSasha Levin 598f6108d72SJean-Philippe Brucker if (irq__can_signal_msi(kvm)) 59943c81c74SSasha Levin vpci->features |= VIRTIO_PCI_F_SIGNAL_MSI; 60043c81c74SSasha Levin 601c0c45eedSAndre Przywara vpci->legacy_irq_line = pci__assign_irq(&vpci->pci_hdr); 602c0c45eedSAndre Przywara 60321ff329dSWill Deacon r = device__register(&vpci->dev_hdr); 604495fbd4eSSasha Levin if (r < 0) 6055a8e4f25SAlexandru Elisei return r; 606495fbd4eSSasha Levin 607495fbd4eSSasha Levin return 0; 608495fbd4eSSasha Levin } 609495fbd4eSSasha Levin 610eb34a8c2SJean-Philippe Brucker int virtio_pci__reset(struct kvm *kvm, struct virtio_device *vdev) 611eb34a8c2SJean-Philippe Brucker { 612eb34a8c2SJean-Philippe Brucker int vq; 613eb34a8c2SJean-Philippe Brucker struct virtio_pci *vpci = vdev->virtio; 614eb34a8c2SJean-Philippe Brucker 615eb34a8c2SJean-Philippe Brucker for (vq = 0; vq < vdev->ops->get_vq_count(kvm, vpci->dev); vq++) 616eb34a8c2SJean-Philippe Brucker virtio_pci_exit_vq(kvm, vdev, vq); 617eb34a8c2SJean-Philippe Brucker 618eb34a8c2SJean-Philippe Brucker return 0; 619eb34a8c2SJean-Philippe Brucker } 620eb34a8c2SJean-Philippe Brucker 62102eca50cSAsias He int virtio_pci__exit(struct kvm *kvm, struct virtio_device *vdev) 622495fbd4eSSasha Levin { 62302eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 624495fbd4eSSasha Levin 625eb34a8c2SJean-Philippe Brucker virtio_pci__reset(kvm, vdev); 626e539f3e4SAlexandru Elisei kvm__deregister_mmio(kvm, virtio_pci__mmio_addr(vpci)); 627e539f3e4SAlexandru Elisei kvm__deregister_mmio(kvm, virtio_pci__msix_io_addr(vpci)); 628205eaa79SAndre Przywara kvm__deregister_pio(kvm, virtio_pci__port_addr(vpci)); 629495fbd4eSSasha Levin 63036f5dc91SSasha Levin return 0; 63136f5dc91SSasha Levin } 632