1 #include "kvm/virtio-pci.h" 2 3 #include "kvm/ioport.h" 4 #include "kvm/kvm.h" 5 #include "kvm/virtio-pci-dev.h" 6 #include "kvm/irq.h" 7 #include "kvm/virtio.h" 8 #include "kvm/ioeventfd.h" 9 10 #include <linux/virtio_pci.h> 11 #include <linux/byteorder.h> 12 #include <string.h> 13 14 static void virtio_pci__ioevent_callback(struct kvm *kvm, void *param) 15 { 16 struct virtio_pci_ioevent_param *ioeventfd = param; 17 struct virtio_pci *vpci = ioeventfd->vdev->virtio; 18 19 ioeventfd->vdev->ops->notify_vq(kvm, vpci->dev, ioeventfd->vq); 20 } 21 22 static int virtio_pci__init_ioeventfd(struct kvm *kvm, struct virtio_device *vdev, u32 vq) 23 { 24 struct ioevent ioevent; 25 struct virtio_pci *vpci = vdev->virtio; 26 int r; 27 28 vpci->ioeventfds[vq] = (struct virtio_pci_ioevent_param) { 29 .vdev = vdev, 30 .vq = vq, 31 }; 32 33 ioevent = (struct ioevent) { 34 .io_addr = vpci->base_addr + VIRTIO_PCI_QUEUE_NOTIFY, 35 .io_len = sizeof(u16), 36 .fn = virtio_pci__ioevent_callback, 37 .fn_ptr = &vpci->ioeventfds[vq], 38 .datamatch = vq, 39 .fn_kvm = kvm, 40 .fd = eventfd(0, 0), 41 }; 42 43 r = ioeventfd__add_event(&ioevent, true); 44 if (r) 45 return r; 46 47 if (vdev->ops->notify_vq_eventfd) 48 vdev->ops->notify_vq_eventfd(kvm, vpci->dev, vq, ioevent.fd); 49 50 return 0; 51 } 52 53 static inline bool virtio_pci__msix_enabled(struct virtio_pci *vpci) 54 { 55 return vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_ENABLE); 56 } 57 58 static bool virtio_pci__specific_io_in(struct kvm *kvm, struct virtio_device *vdev, u16 port, 59 void *data, int size, int offset) 60 { 61 u32 config_offset; 62 struct virtio_pci *vpci = vdev->virtio; 63 int type = virtio__get_dev_specific_field(offset - 20, 64 virtio_pci__msix_enabled(vpci), 65 &config_offset); 66 if (type == VIRTIO_PCI_O_MSIX) { 67 switch (offset) { 68 case VIRTIO_MSI_CONFIG_VECTOR: 69 ioport__write16(data, vpci->config_vector); 70 break; 71 case VIRTIO_MSI_QUEUE_VECTOR: 72 ioport__write16(data, vpci->vq_vector[vpci->queue_selector]); 73 break; 74 }; 75 76 return true; 77 } else if (type == VIRTIO_PCI_O_CONFIG) { 78 u8 cfg; 79 80 cfg = vdev->ops->get_config(kvm, vpci->dev, config_offset); 81 ioport__write8(data, cfg); 82 return true; 83 } 84 85 return false; 86 } 87 88 static bool virtio_pci__io_in(struct ioport *ioport, struct kvm *kvm, u16 port, void *data, int size) 89 { 90 unsigned long offset; 91 bool ret = true; 92 struct virtio_device *vdev; 93 struct virtio_pci *vpci; 94 u32 val; 95 96 vdev = ioport->priv; 97 vpci = vdev->virtio; 98 offset = port - vpci->base_addr; 99 100 switch (offset) { 101 case VIRTIO_PCI_HOST_FEATURES: 102 val = vdev->ops->get_host_features(kvm, vpci->dev); 103 ioport__write32(data, val); 104 break; 105 case VIRTIO_PCI_QUEUE_PFN: 106 val = vdev->ops->get_pfn_vq(kvm, vpci->dev, vpci->queue_selector); 107 ioport__write32(data, val); 108 break; 109 case VIRTIO_PCI_QUEUE_NUM: 110 val = vdev->ops->get_size_vq(kvm, vpci->dev, vpci->queue_selector); 111 ioport__write16(data, val); 112 break; 113 case VIRTIO_PCI_STATUS: 114 ioport__write8(data, vpci->status); 115 break; 116 case VIRTIO_PCI_ISR: 117 ioport__write8(data, vpci->isr); 118 kvm__irq_line(kvm, vpci->pci_hdr.irq_line, VIRTIO_IRQ_LOW); 119 vpci->isr = VIRTIO_IRQ_LOW; 120 break; 121 default: 122 ret = virtio_pci__specific_io_in(kvm, vdev, port, data, size, offset); 123 break; 124 }; 125 126 return ret; 127 } 128 129 static bool virtio_pci__specific_io_out(struct kvm *kvm, struct virtio_device *vdev, u16 port, 130 void *data, int size, int offset) 131 { 132 struct virtio_pci *vpci = vdev->virtio; 133 u32 config_offset, gsi, vec; 134 int type = virtio__get_dev_specific_field(offset - 20, virtio_pci__msix_enabled(vpci), 135 &config_offset); 136 if (type == VIRTIO_PCI_O_MSIX) { 137 switch (offset) { 138 case VIRTIO_MSI_CONFIG_VECTOR: 139 vec = vpci->config_vector = ioport__read16(data); 140 141 gsi = irq__add_msix_route(kvm, &vpci->msix_table[vec].msg); 142 143 vpci->config_gsi = gsi; 144 break; 145 case VIRTIO_MSI_QUEUE_VECTOR: 146 vec = vpci->vq_vector[vpci->queue_selector] = ioport__read16(data); 147 148 gsi = irq__add_msix_route(kvm, &vpci->msix_table[vec].msg); 149 vpci->gsis[vpci->queue_selector] = gsi; 150 if (vdev->ops->notify_vq_gsi) 151 vdev->ops->notify_vq_gsi(kvm, vpci->dev, 152 vpci->queue_selector, gsi); 153 break; 154 }; 155 156 return true; 157 } else if (type == VIRTIO_PCI_O_CONFIG) { 158 vdev->ops->set_config(kvm, vpci->dev, *(u8 *)data, config_offset); 159 160 return true; 161 } 162 163 return false; 164 } 165 166 static bool virtio_pci__io_out(struct ioport *ioport, struct kvm *kvm, u16 port, void *data, int size) 167 { 168 unsigned long offset; 169 bool ret = true; 170 struct virtio_device *vdev; 171 struct virtio_pci *vpci; 172 u32 val; 173 174 vdev = ioport->priv; 175 vpci = vdev->virtio; 176 offset = port - vpci->base_addr; 177 178 switch (offset) { 179 case VIRTIO_PCI_GUEST_FEATURES: 180 val = ioport__read32(data); 181 vdev->ops->set_guest_features(kvm, vpci->dev, val); 182 break; 183 case VIRTIO_PCI_QUEUE_PFN: 184 val = ioport__read32(data); 185 virtio_pci__init_ioeventfd(kvm, vdev, vpci->queue_selector); 186 vdev->ops->init_vq(kvm, vpci->dev, vpci->queue_selector, val); 187 break; 188 case VIRTIO_PCI_QUEUE_SEL: 189 vpci->queue_selector = ioport__read16(data); 190 break; 191 case VIRTIO_PCI_QUEUE_NOTIFY: 192 val = ioport__read16(data); 193 vdev->ops->notify_vq(kvm, vpci->dev, val); 194 break; 195 case VIRTIO_PCI_STATUS: 196 vpci->status = ioport__read8(data); 197 break; 198 default: 199 ret = virtio_pci__specific_io_out(kvm, vdev, port, data, size, offset); 200 break; 201 }; 202 203 return ret; 204 } 205 206 static struct ioport_operations virtio_pci__io_ops = { 207 .io_in = virtio_pci__io_in, 208 .io_out = virtio_pci__io_out, 209 }; 210 211 static void callback_mmio_table(u64 addr, u8 *data, u32 len, u8 is_write, void *ptr) 212 { 213 struct virtio_pci *vpci = ptr; 214 void *table; 215 u32 offset; 216 217 if (addr > vpci->msix_io_block + PCI_IO_SIZE) { 218 table = &vpci->msix_pba; 219 offset = vpci->msix_io_block + PCI_IO_SIZE; 220 } else { 221 table = &vpci->msix_table; 222 offset = vpci->msix_io_block; 223 } 224 225 if (is_write) 226 memcpy(table + addr - offset, data, len); 227 else 228 memcpy(data, table + addr - offset, len); 229 } 230 231 int virtio_pci__signal_vq(struct kvm *kvm, struct virtio_device *vdev, u32 vq) 232 { 233 struct virtio_pci *vpci = vdev->virtio; 234 int tbl = vpci->vq_vector[vq]; 235 236 if (virtio_pci__msix_enabled(vpci)) { 237 if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) || 238 vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 239 240 vpci->msix_pba |= 1 << tbl; 241 return 0; 242 } 243 244 kvm__irq_trigger(kvm, vpci->gsis[vq]); 245 } else { 246 vpci->isr = VIRTIO_IRQ_HIGH; 247 kvm__irq_trigger(kvm, vpci->pci_hdr.irq_line); 248 } 249 return 0; 250 } 251 252 int virtio_pci__signal_config(struct kvm *kvm, struct virtio_device *vdev) 253 { 254 struct virtio_pci *vpci = vdev->virtio; 255 int tbl = vpci->config_vector; 256 257 if (virtio_pci__msix_enabled(vpci)) { 258 if (vpci->pci_hdr.msix.ctrl & cpu_to_le16(PCI_MSIX_FLAGS_MASKALL) || 259 vpci->msix_table[tbl].ctrl & cpu_to_le16(PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 260 261 vpci->msix_pba |= 1 << tbl; 262 return 0; 263 } 264 265 kvm__irq_trigger(kvm, vpci->config_gsi); 266 } else { 267 vpci->isr = VIRTIO_PCI_ISR_CONFIG; 268 kvm__irq_trigger(kvm, vpci->pci_hdr.irq_line); 269 } 270 271 return 0; 272 } 273 274 int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev, 275 int device_id, int subsys_id, int class) 276 { 277 struct virtio_pci *vpci = vdev->virtio; 278 u8 pin, line, ndev; 279 int r; 280 281 vpci->dev = dev; 282 vpci->msix_io_block = pci_get_io_space_block(PCI_IO_SIZE * 2); 283 284 r = ioport__register(IOPORT_EMPTY, &virtio_pci__io_ops, IOPORT_SIZE, vdev); 285 if (r < 0) 286 return r; 287 288 vpci->base_addr = (u16)r; 289 r = kvm__register_mmio(kvm, vpci->msix_io_block, PCI_IO_SIZE, false, 290 callback_mmio_table, vpci); 291 if (r < 0) 292 goto free_ioport; 293 294 vpci->pci_hdr = (struct pci_device_header) { 295 .vendor_id = cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET), 296 .device_id = cpu_to_le16(device_id), 297 .header_type = PCI_HEADER_TYPE_NORMAL, 298 .revision_id = 0, 299 .class[0] = class & 0xff, 300 .class[1] = (class >> 8) & 0xff, 301 .class[2] = (class >> 16) & 0xff, 302 .subsys_vendor_id = cpu_to_le16(PCI_SUBSYSTEM_VENDOR_ID_REDHAT_QUMRANET), 303 .subsys_id = cpu_to_le16(subsys_id), 304 .bar[0] = cpu_to_le32(vpci->base_addr 305 | PCI_BASE_ADDRESS_SPACE_IO), 306 .bar[1] = cpu_to_le32(vpci->msix_io_block 307 | PCI_BASE_ADDRESS_SPACE_MEMORY), 308 .status = cpu_to_le16(PCI_STATUS_CAP_LIST), 309 .capabilities = (void *)&vpci->pci_hdr.msix - (void *)&vpci->pci_hdr, 310 .bar_size[0] = IOPORT_SIZE, 311 .bar_size[1] = PCI_IO_SIZE, 312 .bar_size[3] = PCI_IO_SIZE, 313 }; 314 315 vpci->pci_hdr.msix.cap = PCI_CAP_ID_MSIX; 316 vpci->pci_hdr.msix.next = 0; 317 /* 318 * We at most have VIRTIO_PCI_MAX_VQ entries for virt queue, 319 * VIRTIO_PCI_MAX_CONFIG entries for config. 320 * 321 * To quote the PCI spec: 322 * 323 * System software reads this field to determine the 324 * MSI-X Table Size N, which is encoded as N-1. 325 * For example, a returned value of "00000000011" 326 * indicates a table size of 4. 327 */ 328 vpci->pci_hdr.msix.ctrl = cpu_to_le16(VIRTIO_PCI_MAX_VQ + VIRTIO_PCI_MAX_CONFIG - 1); 329 330 /* 331 * Both table and PBA could be mapped on the same BAR, but for now 332 * we're not in short of BARs 333 */ 334 vpci->pci_hdr.msix.table_offset = cpu_to_le32(1); /* Use BAR 1 */ 335 vpci->pci_hdr.msix.pba_offset = cpu_to_le32(1 | PCI_IO_SIZE); /* Use BAR 3 */ 336 vpci->config_vector = 0; 337 338 r = irq__register_device(subsys_id, &ndev, &pin, &line); 339 if (r < 0) 340 goto free_mmio; 341 342 vpci->pci_hdr.irq_pin = pin; 343 vpci->pci_hdr.irq_line = line; 344 r = pci__register(&vpci->pci_hdr, ndev); 345 if (r < 0) 346 goto free_ioport; 347 348 return 0; 349 350 free_mmio: 351 kvm__deregister_mmio(kvm, vpci->msix_io_block); 352 free_ioport: 353 ioport__unregister(vpci->base_addr); 354 return r; 355 } 356 357 int virtio_pci__exit(struct kvm *kvm, struct virtio_device *vdev) 358 { 359 struct virtio_pci *vpci = vdev->virtio; 360 int i; 361 362 kvm__deregister_mmio(kvm, vpci->msix_io_block); 363 ioport__unregister(vpci->base_addr); 364 365 for (i = 0; i < VIRTIO_PCI_MAX_VQ; i++) 366 ioeventfd__del_event(vpci->base_addr + VIRTIO_PCI_QUEUE_NOTIFY, i); 367 368 return 0; 369 } 370