136f5dc91SSasha Levin #include "kvm/virtio-pci.h" 236f5dc91SSasha Levin 336f5dc91SSasha Levin #include "kvm/ioport.h" 436f5dc91SSasha Levin #include "kvm/kvm.h" 54123ca55SMarc Zyngier #include "kvm/kvm-cpu.h" 636f5dc91SSasha Levin #include "kvm/virtio-pci-dev.h" 736f5dc91SSasha Levin #include "kvm/irq.h" 836f5dc91SSasha Levin #include "kvm/virtio.h" 91599d724SSasha Levin #include "kvm/ioeventfd.h" 1036f5dc91SSasha Levin 1143c81c74SSasha Levin #include <sys/ioctl.h> 1236f5dc91SSasha Levin #include <linux/virtio_pci.h> 13aa73be70SMatt Evans #include <linux/byteorder.h> 1436f5dc91SSasha Levin #include <string.h> 1536f5dc91SSasha Levin 161599d724SSasha Levin static void virtio_pci__ioevent_callback(struct kvm *kvm, void *param) 171599d724SSasha Levin { 181599d724SSasha Levin struct virtio_pci_ioevent_param *ioeventfd = param; 1902eca50cSAsias He struct virtio_pci *vpci = ioeventfd->vdev->virtio; 201599d724SSasha Levin 2102eca50cSAsias He ioeventfd->vdev->ops->notify_vq(kvm, vpci->dev, ioeventfd->vq); 221599d724SSasha Levin } 231599d724SSasha Levin 2402eca50cSAsias He static int virtio_pci__init_ioeventfd(struct kvm *kvm, struct virtio_device *vdev, u32 vq) 251599d724SSasha Levin { 261599d724SSasha Levin struct ioevent ioevent; 2702eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 28*0e1882a4SJean-Philippe Brucker int r, flags = 0; 29*0e1882a4SJean-Philippe Brucker int fd; 301599d724SSasha Levin 311599d724SSasha Levin vpci->ioeventfds[vq] = (struct virtio_pci_ioevent_param) { 3202eca50cSAsias He .vdev = vdev, 331599d724SSasha Levin .vq = vq, 341599d724SSasha Levin }; 351599d724SSasha Levin 361599d724SSasha Levin ioevent = (struct ioevent) { 371599d724SSasha Levin .fn = virtio_pci__ioevent_callback, 381599d724SSasha Levin .fn_ptr = &vpci->ioeventfds[vq], 391599d724SSasha Levin .datamatch = vq, 401599d724SSasha Levin .fn_kvm = kvm, 411599d724SSasha Levin }; 421599d724SSasha Levin 43627d6874SAsias He /* 44a463650cSWill Deacon * Vhost will poll the eventfd in host kernel side, otherwise we 45a463650cSWill Deacon * need to poll in userspace. 46627d6874SAsias He */ 47a463650cSWill Deacon if (!vdev->use_vhost) 48a463650cSWill Deacon flags |= IOEVENTFD_FLAG_USER_POLL; 49a463650cSWill Deacon 50a463650cSWill Deacon /* ioport */ 51a463650cSWill Deacon ioevent.io_addr = vpci->port_addr + VIRTIO_PCI_QUEUE_NOTIFY; 52a463650cSWill Deacon ioevent.io_len = sizeof(u16); 53*0e1882a4SJean-Philippe Brucker ioevent.fd = fd = eventfd(0, 0); 5471ca0facSAndre Przywara r = ioeventfd__add_event(&ioevent, flags | IOEVENTFD_FLAG_PIO); 55ea6eeb1cSSasha Levin if (r) 56ea6eeb1cSSasha Levin return r; 571599d724SSasha Levin 58a463650cSWill Deacon /* mmio */ 59a463650cSWill Deacon ioevent.io_addr = vpci->mmio_addr + VIRTIO_PCI_QUEUE_NOTIFY; 60fe50bacbSAndreas Herrmann ioevent.io_len = sizeof(u16); 61*0e1882a4SJean-Philippe Brucker ioevent.fd = eventfd(0, 0); 62a463650cSWill Deacon r = ioeventfd__add_event(&ioevent, flags); 63a463650cSWill Deacon if (r) 64a463650cSWill Deacon goto free_ioport_evt; 65263b80e8SSasha Levin 66a463650cSWill Deacon if (vdev->ops->notify_vq_eventfd) 67*0e1882a4SJean-Philippe Brucker vdev->ops->notify_vq_eventfd(kvm, vpci->dev, vq, fd); 681599d724SSasha Levin return 0; 69a463650cSWill Deacon 70a463650cSWill Deacon free_ioport_evt: 71a463650cSWill Deacon ioeventfd__del_event(vpci->port_addr + VIRTIO_PCI_QUEUE_NOTIFY, vq); 72a463650cSWill Deacon return r; 731599d724SSasha Levin } 741599d724SSasha Levin 7506f48103SSasha Levin static inline bool virtio_pci__msix_enabled(struct virtio_pci *vpci) 7606f48103SSasha Levin { 77aa73be70SMatt Evans return vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_ENABLE); 7806f48103SSasha Levin } 7906f48103SSasha Levin 8002eca50cSAsias He static bool virtio_pci__specific_io_in(struct kvm *kvm, struct virtio_device *vdev, u16 port, 8136f5dc91SSasha Levin void *data, int size, int offset) 8236f5dc91SSasha Levin { 8336f5dc91SSasha Levin u32 config_offset; 8402eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 8506f48103SSasha Levin int type = virtio__get_dev_specific_field(offset - 20, 8606f48103SSasha Levin virtio_pci__msix_enabled(vpci), 871382aba0SSasha Levin &config_offset); 8836f5dc91SSasha Levin if (type == VIRTIO_PCI_O_MSIX) { 8936f5dc91SSasha Levin switch (offset) { 9036f5dc91SSasha Levin case VIRTIO_MSI_CONFIG_VECTOR: 9136f5dc91SSasha Levin ioport__write16(data, vpci->config_vector); 9236f5dc91SSasha Levin break; 9336f5dc91SSasha Levin case VIRTIO_MSI_QUEUE_VECTOR: 9436f5dc91SSasha Levin ioport__write16(data, vpci->vq_vector[vpci->queue_selector]); 9536f5dc91SSasha Levin break; 9636f5dc91SSasha Levin }; 9736f5dc91SSasha Levin 9836f5dc91SSasha Levin return true; 9936f5dc91SSasha Levin } else if (type == VIRTIO_PCI_O_CONFIG) { 10036f5dc91SSasha Levin u8 cfg; 10136f5dc91SSasha Levin 102c5ae742bSSasha Levin cfg = vdev->ops->get_config(kvm, vpci->dev)[config_offset]; 10336f5dc91SSasha Levin ioport__write8(data, cfg); 10436f5dc91SSasha Levin return true; 10536f5dc91SSasha Levin } 10636f5dc91SSasha Levin 10736f5dc91SSasha Levin return false; 10836f5dc91SSasha Levin } 10936f5dc91SSasha Levin 1104123ca55SMarc Zyngier static bool virtio_pci__io_in(struct ioport *ioport, struct kvm_cpu *vcpu, u16 port, void *data, int size) 11136f5dc91SSasha Levin { 11236f5dc91SSasha Levin unsigned long offset; 11336f5dc91SSasha Levin bool ret = true; 11402eca50cSAsias He struct virtio_device *vdev; 11536f5dc91SSasha Levin struct virtio_pci *vpci; 1164123ca55SMarc Zyngier struct kvm *kvm; 11736f5dc91SSasha Levin u32 val; 11836f5dc91SSasha Levin 1194123ca55SMarc Zyngier kvm = vcpu->kvm; 12002eca50cSAsias He vdev = ioport->priv; 12102eca50cSAsias He vpci = vdev->virtio; 122a463650cSWill Deacon offset = port - vpci->port_addr; 12336f5dc91SSasha Levin 12436f5dc91SSasha Levin switch (offset) { 12536f5dc91SSasha Levin case VIRTIO_PCI_HOST_FEATURES: 12602eca50cSAsias He val = vdev->ops->get_host_features(kvm, vpci->dev); 12736f5dc91SSasha Levin ioport__write32(data, val); 12836f5dc91SSasha Levin break; 12936f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_PFN: 13002eca50cSAsias He val = vdev->ops->get_pfn_vq(kvm, vpci->dev, vpci->queue_selector); 13136f5dc91SSasha Levin ioport__write32(data, val); 13236f5dc91SSasha Levin break; 13336f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_NUM: 13402eca50cSAsias He val = vdev->ops->get_size_vq(kvm, vpci->dev, vpci->queue_selector); 135657ee18bSMatt Evans ioport__write16(data, val); 13636f5dc91SSasha Levin break; 13736f5dc91SSasha Levin case VIRTIO_PCI_STATUS: 13836f5dc91SSasha Levin ioport__write8(data, vpci->status); 13936f5dc91SSasha Levin break; 14036f5dc91SSasha Levin case VIRTIO_PCI_ISR: 14136f5dc91SSasha Levin ioport__write8(data, vpci->isr); 142e9922aafSAndre Przywara kvm__irq_line(kvm, vpci->legacy_irq_line, VIRTIO_IRQ_LOW); 14336f5dc91SSasha Levin vpci->isr = VIRTIO_IRQ_LOW; 14436f5dc91SSasha Levin break; 14536f5dc91SSasha Levin default: 14602eca50cSAsias He ret = virtio_pci__specific_io_in(kvm, vdev, port, data, size, offset); 14736f5dc91SSasha Levin break; 14836f5dc91SSasha Levin }; 14936f5dc91SSasha Levin 15036f5dc91SSasha Levin return ret; 15136f5dc91SSasha Levin } 15236f5dc91SSasha Levin 1536518065aSAndre Przywara static void update_msix_map(struct virtio_pci *vpci, 1546518065aSAndre Przywara struct msix_table *msix_entry, u32 vecnum) 1556518065aSAndre Przywara { 1566518065aSAndre Przywara u32 gsi, i; 1576518065aSAndre Przywara 1586518065aSAndre Przywara /* Find the GSI number used for that vector */ 1596518065aSAndre Przywara if (vecnum == vpci->config_vector) { 1606518065aSAndre Przywara gsi = vpci->config_gsi; 1616518065aSAndre Przywara } else { 1626518065aSAndre Przywara for (i = 0; i < VIRTIO_PCI_MAX_VQ; i++) 1636518065aSAndre Przywara if (vpci->vq_vector[i] == vecnum) 1646518065aSAndre Przywara break; 1656518065aSAndre Przywara if (i == VIRTIO_PCI_MAX_VQ) 1666518065aSAndre Przywara return; 1676518065aSAndre Przywara gsi = vpci->gsis[i]; 1686518065aSAndre Przywara } 1696518065aSAndre Przywara 1706518065aSAndre Przywara if (gsi == 0) 1716518065aSAndre Przywara return; 1726518065aSAndre Przywara 1736518065aSAndre Przywara msix_entry = &msix_entry[vecnum]; 1746518065aSAndre Przywara irq__update_msix_route(vpci->kvm, gsi, &msix_entry->msg); 1756518065aSAndre Przywara } 1766518065aSAndre Przywara 17702eca50cSAsias He static bool virtio_pci__specific_io_out(struct kvm *kvm, struct virtio_device *vdev, u16 port, 17836f5dc91SSasha Levin void *data, int size, int offset) 17936f5dc91SSasha Levin { 18002eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 1818ccc8549SAndre Przywara u32 config_offset, vec; 1828ccc8549SAndre Przywara int gsi; 18306f48103SSasha Levin int type = virtio__get_dev_specific_field(offset - 20, virtio_pci__msix_enabled(vpci), 1841382aba0SSasha Levin &config_offset); 18536f5dc91SSasha Levin if (type == VIRTIO_PCI_O_MSIX) { 18636f5dc91SSasha Levin switch (offset) { 18736f5dc91SSasha Levin case VIRTIO_MSI_CONFIG_VECTOR: 18836f5dc91SSasha Levin vec = vpci->config_vector = ioport__read16(data); 189f8327b05SSasha Levin if (vec == VIRTIO_MSI_NO_VECTOR) 190f8327b05SSasha Levin break; 19136f5dc91SSasha Levin 1928ccc8549SAndre Przywara gsi = irq__add_msix_route(kvm, 193f9ef46f2SAndre Przywara &vpci->msix_table[vec].msg, 194f9ef46f2SAndre Przywara vpci->dev_hdr.dev_num << 3); 195928ab7acSAndre Przywara /* 196928ab7acSAndre Przywara * We don't need IRQ routing if we can use 197928ab7acSAndre Przywara * MSI injection via the KVM_SIGNAL_MSI ioctl. 198928ab7acSAndre Przywara */ 199928ab7acSAndre Przywara if (gsi == -ENXIO && 200928ab7acSAndre Przywara vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 201928ab7acSAndre Przywara break; 202928ab7acSAndre Przywara 203928ab7acSAndre Przywara if (gsi < 0) { 204928ab7acSAndre Przywara die("failed to configure MSIs"); 205928ab7acSAndre Przywara break; 206928ab7acSAndre Przywara } 207928ab7acSAndre Przywara 20836f5dc91SSasha Levin vpci->config_gsi = gsi; 20936f5dc91SSasha Levin break; 2103a60be06SSasha Levin case VIRTIO_MSI_QUEUE_VECTOR: 2118ccc8549SAndre Przywara vec = ioport__read16(data); 2128ccc8549SAndre Przywara vpci->vq_vector[vpci->queue_selector] = vec; 21336f5dc91SSasha Levin 214f8327b05SSasha Levin if (vec == VIRTIO_MSI_NO_VECTOR) 215f8327b05SSasha Levin break; 216f8327b05SSasha Levin 2178ccc8549SAndre Przywara gsi = irq__add_msix_route(kvm, 218f9ef46f2SAndre Przywara &vpci->msix_table[vec].msg, 219f9ef46f2SAndre Przywara vpci->dev_hdr.dev_num << 3); 220928ab7acSAndre Przywara /* 221928ab7acSAndre Przywara * We don't need IRQ routing if we can use 222928ab7acSAndre Przywara * MSI injection via the KVM_SIGNAL_MSI ioctl. 223928ab7acSAndre Przywara */ 224928ab7acSAndre Przywara if (gsi == -ENXIO && 225928ab7acSAndre Przywara vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 2268ccc8549SAndre Przywara break; 227928ab7acSAndre Przywara 228928ab7acSAndre Przywara if (gsi < 0) { 229928ab7acSAndre Przywara die("failed to configure MSIs"); 230928ab7acSAndre Przywara break; 231928ab7acSAndre Przywara } 232928ab7acSAndre Przywara 23336f5dc91SSasha Levin vpci->gsis[vpci->queue_selector] = gsi; 23402eca50cSAsias He if (vdev->ops->notify_vq_gsi) 23502eca50cSAsias He vdev->ops->notify_vq_gsi(kvm, vpci->dev, 2368ccc8549SAndre Przywara vpci->queue_selector, 2378ccc8549SAndre Przywara gsi); 23836f5dc91SSasha Levin break; 23936f5dc91SSasha Levin }; 24036f5dc91SSasha Levin 24136f5dc91SSasha Levin return true; 24236f5dc91SSasha Levin } else if (type == VIRTIO_PCI_O_CONFIG) { 243c5ae742bSSasha Levin vdev->ops->get_config(kvm, vpci->dev)[config_offset] = *(u8 *)data; 24436f5dc91SSasha Levin 24536f5dc91SSasha Levin return true; 24636f5dc91SSasha Levin } 24736f5dc91SSasha Levin 24836f5dc91SSasha Levin return false; 24936f5dc91SSasha Levin } 25036f5dc91SSasha Levin 2514123ca55SMarc Zyngier static bool virtio_pci__io_out(struct ioport *ioport, struct kvm_cpu *vcpu, u16 port, void *data, int size) 25236f5dc91SSasha Levin { 25336f5dc91SSasha Levin unsigned long offset; 25436f5dc91SSasha Levin bool ret = true; 25502eca50cSAsias He struct virtio_device *vdev; 25636f5dc91SSasha Levin struct virtio_pci *vpci; 2574123ca55SMarc Zyngier struct kvm *kvm; 25836f5dc91SSasha Levin u32 val; 25936f5dc91SSasha Levin 2604123ca55SMarc Zyngier kvm = vcpu->kvm; 26102eca50cSAsias He vdev = ioport->priv; 26202eca50cSAsias He vpci = vdev->virtio; 263a463650cSWill Deacon offset = port - vpci->port_addr; 26436f5dc91SSasha Levin 26536f5dc91SSasha Levin switch (offset) { 26636f5dc91SSasha Levin case VIRTIO_PCI_GUEST_FEATURES: 26736f5dc91SSasha Levin val = ioport__read32(data); 26856a16c90SJean-Philippe Brucker virtio_set_guest_features(kvm, vdev, vpci->dev, val); 26936f5dc91SSasha Levin break; 27036f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_PFN: 27136f5dc91SSasha Levin val = ioport__read32(data); 27202eca50cSAsias He virtio_pci__init_ioeventfd(kvm, vdev, vpci->queue_selector); 273c59ba304SWill Deacon vdev->ops->init_vq(kvm, vpci->dev, vpci->queue_selector, 274c59ba304SWill Deacon 1 << VIRTIO_PCI_QUEUE_ADDR_SHIFT, 275c59ba304SWill Deacon VIRTIO_PCI_VRING_ALIGN, val); 27636f5dc91SSasha Levin break; 27736f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_SEL: 27836f5dc91SSasha Levin vpci->queue_selector = ioport__read16(data); 27936f5dc91SSasha Levin break; 28036f5dc91SSasha Levin case VIRTIO_PCI_QUEUE_NOTIFY: 28136f5dc91SSasha Levin val = ioport__read16(data); 28202eca50cSAsias He vdev->ops->notify_vq(kvm, vpci->dev, val); 28336f5dc91SSasha Levin break; 28436f5dc91SSasha Levin case VIRTIO_PCI_STATUS: 28536f5dc91SSasha Levin vpci->status = ioport__read8(data); 2864123ca55SMarc Zyngier if (!vpci->status) /* Sample endianness on reset */ 2874123ca55SMarc Zyngier vdev->endian = kvm_cpu__get_endianness(vcpu); 28804b53c16SSasha Levin if (vdev->ops->notify_status) 28904b53c16SSasha Levin vdev->ops->notify_status(kvm, vpci->dev, vpci->status); 29036f5dc91SSasha Levin break; 29136f5dc91SSasha Levin default: 29202eca50cSAsias He ret = virtio_pci__specific_io_out(kvm, vdev, port, data, size, offset); 29336f5dc91SSasha Levin break; 29436f5dc91SSasha Levin }; 29536f5dc91SSasha Levin 29636f5dc91SSasha Levin return ret; 29736f5dc91SSasha Levin } 29836f5dc91SSasha Levin 29936f5dc91SSasha Levin static struct ioport_operations virtio_pci__io_ops = { 30036f5dc91SSasha Levin .io_in = virtio_pci__io_in, 30136f5dc91SSasha Levin .io_out = virtio_pci__io_out, 30236f5dc91SSasha Levin }; 30336f5dc91SSasha Levin 3049b735910SMarc Zyngier static void virtio_pci__msix_mmio_callback(struct kvm_cpu *vcpu, 3059b735910SMarc Zyngier u64 addr, u8 *data, u32 len, 306a463650cSWill Deacon u8 is_write, void *ptr) 30736f5dc91SSasha Levin { 30836f5dc91SSasha Levin struct virtio_pci *vpci = ptr; 3096518065aSAndre Przywara struct msix_table *table; 3106518065aSAndre Przywara int vecnum; 3116518065aSAndre Przywara size_t offset; 31236f5dc91SSasha Levin 3139c26dab4SSasha Levin if (addr > vpci->msix_io_block + PCI_IO_SIZE) { 3146518065aSAndre Przywara if (is_write) 3156518065aSAndre Przywara return; 3166518065aSAndre Przywara table = (struct msix_table *)&vpci->msix_pba; 3176518065aSAndre Przywara offset = addr - (vpci->msix_io_block + PCI_IO_SIZE); 3189c26dab4SSasha Levin } else { 3196518065aSAndre Przywara table = vpci->msix_table; 3206518065aSAndre Przywara offset = addr - vpci->msix_io_block; 3216518065aSAndre Przywara } 3226518065aSAndre Przywara vecnum = offset / sizeof(struct msix_table); 3236518065aSAndre Przywara offset = offset % sizeof(struct msix_table); 3246518065aSAndre Przywara 3256518065aSAndre Przywara if (!is_write) { 3266518065aSAndre Przywara memcpy(data, (void *)&table[vecnum] + offset, len); 3276518065aSAndre Przywara return; 32836f5dc91SSasha Levin } 32936f5dc91SSasha Levin 3306518065aSAndre Przywara memcpy((void *)&table[vecnum] + offset, data, len); 3316518065aSAndre Przywara 3326518065aSAndre Przywara /* Did we just update the address or payload? */ 3336518065aSAndre Przywara if (offset < offsetof(struct msix_table, ctrl)) 3346518065aSAndre Przywara update_msix_map(vpci, table, vecnum); 33506f48103SSasha Levin } 33606f48103SSasha Levin 337714ab9e6SAndre Przywara static void virtio_pci__signal_msi(struct kvm *kvm, struct virtio_pci *vpci, 338714ab9e6SAndre Przywara int vec) 33943c81c74SSasha Levin { 34043c81c74SSasha Levin struct kvm_msi msi = { 34143c81c74SSasha Levin .address_lo = vpci->msix_table[vec].msg.address_lo, 34243c81c74SSasha Levin .address_hi = vpci->msix_table[vec].msg.address_hi, 34343c81c74SSasha Levin .data = vpci->msix_table[vec].msg.data, 34443c81c74SSasha Levin }; 34543c81c74SSasha Levin 346714ab9e6SAndre Przywara if (kvm->msix_needs_devid) { 347714ab9e6SAndre Przywara msi.flags = KVM_MSI_VALID_DEVID; 348714ab9e6SAndre Przywara msi.devid = vpci->dev_hdr.dev_num << 3; 349714ab9e6SAndre Przywara } 350714ab9e6SAndre Przywara 351f6108d72SJean-Philippe Brucker irq__signal_msi(kvm, &msi); 35243c81c74SSasha Levin } 35343c81c74SSasha Levin 35402eca50cSAsias He int virtio_pci__signal_vq(struct kvm *kvm, struct virtio_device *vdev, u32 vq) 35536f5dc91SSasha Levin { 35602eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 35706f48103SSasha Levin int tbl = vpci->vq_vector[vq]; 35836f5dc91SSasha Levin 359f8327b05SSasha Levin if (virtio_pci__msix_enabled(vpci) && tbl != VIRTIO_MSI_NO_VECTOR) { 360aa73be70SMatt Evans if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) || 361aa73be70SMatt Evans vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 36206f48103SSasha Levin 36306f48103SSasha Levin vpci->msix_pba |= 1 << tbl; 36406f48103SSasha Levin return 0; 36506f48103SSasha Levin } 36606f48103SSasha Levin 36743c81c74SSasha Levin if (vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 36843c81c74SSasha Levin virtio_pci__signal_msi(kvm, vpci, vpci->vq_vector[vq]); 36943c81c74SSasha Levin else 37006f48103SSasha Levin kvm__irq_trigger(kvm, vpci->gsis[vq]); 37106f48103SSasha Levin } else { 372a36eca7bSSasha Levin vpci->isr = VIRTIO_IRQ_HIGH; 373e9922aafSAndre Przywara kvm__irq_trigger(kvm, vpci->legacy_irq_line); 37406f48103SSasha Levin } 37536f5dc91SSasha Levin return 0; 37636f5dc91SSasha Levin } 37736f5dc91SSasha Levin 37802eca50cSAsias He int virtio_pci__signal_config(struct kvm *kvm, struct virtio_device *vdev) 37936f5dc91SSasha Levin { 38002eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 38106f48103SSasha Levin int tbl = vpci->config_vector; 38206f48103SSasha Levin 383f8327b05SSasha Levin if (virtio_pci__msix_enabled(vpci) && tbl != VIRTIO_MSI_NO_VECTOR) { 384aa73be70SMatt Evans if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) || 385aa73be70SMatt Evans vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 38606f48103SSasha Levin 38706f48103SSasha Levin vpci->msix_pba |= 1 << tbl; 38806f48103SSasha Levin return 0; 38906f48103SSasha Levin } 39006f48103SSasha Levin 39143c81c74SSasha Levin if (vpci->features & VIRTIO_PCI_F_SIGNAL_MSI) 392f8327b05SSasha Levin virtio_pci__signal_msi(kvm, vpci, tbl); 39343c81c74SSasha Levin else 39406f48103SSasha Levin kvm__irq_trigger(kvm, vpci->config_gsi); 39506f48103SSasha Levin } else { 39606f48103SSasha Levin vpci->isr = VIRTIO_PCI_ISR_CONFIG; 397e9922aafSAndre Przywara kvm__irq_trigger(kvm, vpci->legacy_irq_line); 39806f48103SSasha Levin } 39936f5dc91SSasha Levin 40036f5dc91SSasha Levin return 0; 40136f5dc91SSasha Levin } 40236f5dc91SSasha Levin 4039b735910SMarc Zyngier static void virtio_pci__io_mmio_callback(struct kvm_cpu *vcpu, 4049b735910SMarc Zyngier u64 addr, u8 *data, u32 len, 405a463650cSWill Deacon u8 is_write, void *ptr) 406a463650cSWill Deacon { 407a463650cSWill Deacon struct virtio_pci *vpci = ptr; 408a463650cSWill Deacon int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN; 409a463650cSWill Deacon u16 port = vpci->port_addr + (addr & (IOPORT_SIZE - 1)); 410a463650cSWill Deacon 4114123ca55SMarc Zyngier kvm__emulate_io(vcpu, port, data, direction, len, 1); 412a463650cSWill Deacon } 413a463650cSWill Deacon 41402eca50cSAsias He int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev, 415507e02d8SAsias He int device_id, int subsys_id, int class) 41636f5dc91SSasha Levin { 41702eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 4187af40b91SSasha Levin int r; 41936f5dc91SSasha Levin 420a463650cSWill Deacon vpci->kvm = kvm; 42136f5dc91SSasha Levin vpci->dev = dev; 42236f5dc91SSasha Levin 4234346fd8fSSasha Levin r = ioport__register(kvm, IOPORT_EMPTY, &virtio_pci__io_ops, IOPORT_SIZE, vdev); 4247af40b91SSasha Levin if (r < 0) 4257af40b91SSasha Levin return r; 426a463650cSWill Deacon vpci->port_addr = (u16)r; 4277af40b91SSasha Levin 428a463650cSWill Deacon vpci->mmio_addr = pci_get_io_space_block(IOPORT_SIZE); 429a463650cSWill Deacon r = kvm__register_mmio(kvm, vpci->mmio_addr, IOPORT_SIZE, false, 430a463650cSWill Deacon virtio_pci__io_mmio_callback, vpci); 431495fbd4eSSasha Levin if (r < 0) 432495fbd4eSSasha Levin goto free_ioport; 43336f5dc91SSasha Levin 434a463650cSWill Deacon vpci->msix_io_block = pci_get_io_space_block(PCI_IO_SIZE * 2); 435a463650cSWill Deacon r = kvm__register_mmio(kvm, vpci->msix_io_block, PCI_IO_SIZE * 2, false, 436a463650cSWill Deacon virtio_pci__msix_mmio_callback, vpci); 437a463650cSWill Deacon if (r < 0) 438a463650cSWill Deacon goto free_mmio; 439a463650cSWill Deacon 44036f5dc91SSasha Levin vpci->pci_hdr = (struct pci_device_header) { 441aa73be70SMatt Evans .vendor_id = cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET), 442aa73be70SMatt Evans .device_id = cpu_to_le16(device_id), 443ec7dd52fSSasha Levin .command = PCI_COMMAND_IO | PCI_COMMAND_MEMORY, 44436f5dc91SSasha Levin .header_type = PCI_HEADER_TYPE_NORMAL, 44536f5dc91SSasha Levin .revision_id = 0, 446aa73be70SMatt Evans .class[0] = class & 0xff, 447aa73be70SMatt Evans .class[1] = (class >> 8) & 0xff, 448aa73be70SMatt Evans .class[2] = (class >> 16) & 0xff, 449aa73be70SMatt Evans .subsys_vendor_id = cpu_to_le16(PCI_SUBSYSTEM_VENDOR_ID_REDHAT_QUMRANET), 450aa73be70SMatt Evans .subsys_id = cpu_to_le16(subsys_id), 451a508ea95SJean-Philippe Brucker .bar[0] = cpu_to_le32(vpci->port_addr 4529c26dab4SSasha Levin | PCI_BASE_ADDRESS_SPACE_IO), 453a508ea95SJean-Philippe Brucker .bar[1] = cpu_to_le32(vpci->mmio_addr 454a508ea95SJean-Philippe Brucker | PCI_BASE_ADDRESS_SPACE_MEMORY), 455a463650cSWill Deacon .bar[2] = cpu_to_le32(vpci->msix_io_block 456b4dab816SSasha Levin | PCI_BASE_ADDRESS_SPACE_MEMORY), 457aa73be70SMatt Evans .status = cpu_to_le16(PCI_STATUS_CAP_LIST), 45836f5dc91SSasha Levin .capabilities = (void *)&vpci->pci_hdr.msix - (void *)&vpci->pci_hdr, 459a0a7d66fSDavid Daney .bar_size[0] = cpu_to_le32(IOPORT_SIZE), 460a0a7d66fSDavid Daney .bar_size[1] = cpu_to_le32(IOPORT_SIZE), 461a0a7d66fSDavid Daney .bar_size[2] = cpu_to_le32(PCI_IO_SIZE*2), 46236f5dc91SSasha Levin }; 46336f5dc91SSasha Levin 46421ff329dSWill Deacon vpci->dev_hdr = (struct device_header) { 46521ff329dSWill Deacon .bus_type = DEVICE_BUS_PCI, 46621ff329dSWill Deacon .data = &vpci->pci_hdr, 46721ff329dSWill Deacon }; 46821ff329dSWill Deacon 46936f5dc91SSasha Levin vpci->pci_hdr.msix.cap = PCI_CAP_ID_MSIX; 47036f5dc91SSasha Levin vpci->pci_hdr.msix.next = 0; 47114bba8a0SAsias He /* 47214bba8a0SAsias He * We at most have VIRTIO_PCI_MAX_VQ entries for virt queue, 47314bba8a0SAsias He * VIRTIO_PCI_MAX_CONFIG entries for config. 47414bba8a0SAsias He * 47514bba8a0SAsias He * To quote the PCI spec: 47614bba8a0SAsias He * 47714bba8a0SAsias He * System software reads this field to determine the 47814bba8a0SAsias He * MSI-X Table Size N, which is encoded as N-1. 47914bba8a0SAsias He * For example, a returned value of "00000000011" 48014bba8a0SAsias He * indicates a table size of 4. 48114bba8a0SAsias He */ 482aa73be70SMatt Evans vpci->pci_hdr.msix.ctrl = cpu_to_le16(VIRTIO_PCI_MAX_VQ + VIRTIO_PCI_MAX_CONFIG - 1); 48306f48103SSasha Levin 484a463650cSWill Deacon /* Both table and PBA are mapped to the same BAR (2) */ 485a463650cSWill Deacon vpci->pci_hdr.msix.table_offset = cpu_to_le32(2); 486a463650cSWill Deacon vpci->pci_hdr.msix.pba_offset = cpu_to_le32(2 | PCI_IO_SIZE); 48736f5dc91SSasha Levin vpci->config_vector = 0; 48836f5dc91SSasha Levin 489f6108d72SJean-Philippe Brucker if (irq__can_signal_msi(kvm)) 49043c81c74SSasha Levin vpci->features |= VIRTIO_PCI_F_SIGNAL_MSI; 49143c81c74SSasha Levin 49221ff329dSWill Deacon r = device__register(&vpci->dev_hdr); 493495fbd4eSSasha Levin if (r < 0) 494a463650cSWill Deacon goto free_msix_mmio; 495495fbd4eSSasha Levin 496e9922aafSAndre Przywara /* save the IRQ that device__register() has allocated */ 497e9922aafSAndre Przywara vpci->legacy_irq_line = vpci->pci_hdr.irq_line; 498e9922aafSAndre Przywara 499495fbd4eSSasha Levin return 0; 500495fbd4eSSasha Levin 501a463650cSWill Deacon free_msix_mmio: 502495fbd4eSSasha Levin kvm__deregister_mmio(kvm, vpci->msix_io_block); 503a463650cSWill Deacon free_mmio: 504a463650cSWill Deacon kvm__deregister_mmio(kvm, vpci->mmio_addr); 505495fbd4eSSasha Levin free_ioport: 506a463650cSWill Deacon ioport__unregister(kvm, vpci->port_addr); 507495fbd4eSSasha Levin return r; 508495fbd4eSSasha Levin } 509495fbd4eSSasha Levin 51002eca50cSAsias He int virtio_pci__exit(struct kvm *kvm, struct virtio_device *vdev) 511495fbd4eSSasha Levin { 51202eca50cSAsias He struct virtio_pci *vpci = vdev->virtio; 513495fbd4eSSasha Levin int i; 514495fbd4eSSasha Levin 515a463650cSWill Deacon kvm__deregister_mmio(kvm, vpci->mmio_addr); 516495fbd4eSSasha Levin kvm__deregister_mmio(kvm, vpci->msix_io_block); 517a463650cSWill Deacon ioport__unregister(kvm, vpci->port_addr); 518495fbd4eSSasha Levin 519a463650cSWill Deacon for (i = 0; i < VIRTIO_PCI_MAX_VQ; i++) { 520a463650cSWill Deacon ioeventfd__del_event(vpci->port_addr + VIRTIO_PCI_QUEUE_NOTIFY, i); 521a463650cSWill Deacon ioeventfd__del_event(vpci->mmio_addr + VIRTIO_PCI_QUEUE_NOTIFY, i); 522a463650cSWill Deacon } 52336f5dc91SSasha Levin 52436f5dc91SSasha Levin return 0; 52536f5dc91SSasha Levin } 526