1 /* 2 * Virtio Support 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qapi/error.h" 16 #include "qemu-common.h" 17 #include "cpu.h" 18 #include "trace.h" 19 #include "exec/address-spaces.h" 20 #include "qemu/error-report.h" 21 #include "hw/virtio/virtio.h" 22 #include "qemu/atomic.h" 23 #include "hw/virtio/virtio-bus.h" 24 #include "migration/migration.h" 25 #include "hw/virtio/virtio-access.h" 26 #include "sysemu/dma.h" 27 28 /* 29 * The alignment to use between consumer and producer parts of vring. 30 * x86 pagesize again. This is the default, used by transports like PCI 31 * which don't provide a means for the guest to tell the host the alignment. 32 */ 33 #define VIRTIO_PCI_VRING_ALIGN 4096 34 35 typedef struct VRingDesc 36 { 37 uint64_t addr; 38 uint32_t len; 39 uint16_t flags; 40 uint16_t next; 41 } VRingDesc; 42 43 typedef struct VRingAvail 44 { 45 uint16_t flags; 46 uint16_t idx; 47 uint16_t ring[0]; 48 } VRingAvail; 49 50 typedef struct VRingUsedElem 51 { 52 uint32_t id; 53 uint32_t len; 54 } VRingUsedElem; 55 56 typedef struct VRingUsed 57 { 58 uint16_t flags; 59 uint16_t idx; 60 VRingUsedElem ring[0]; 61 } VRingUsed; 62 63 typedef struct VRingMemoryRegionCaches { 64 struct rcu_head rcu; 65 MemoryRegionCache desc; 66 MemoryRegionCache avail; 67 MemoryRegionCache used; 68 } VRingMemoryRegionCaches; 69 70 typedef struct VRing 71 { 72 unsigned int num; 73 unsigned int num_default; 74 unsigned int align; 75 hwaddr desc; 76 hwaddr avail; 77 hwaddr used; 78 VRingMemoryRegionCaches *caches; 79 } VRing; 80 81 struct VirtQueue 82 { 83 VRing vring; 84 85 /* Next head to pop */ 86 uint16_t last_avail_idx; 87 88 /* Last avail_idx read from VQ. */ 89 uint16_t shadow_avail_idx; 90 91 uint16_t used_idx; 92 93 /* Last used index value we have signalled on */ 94 uint16_t signalled_used; 95 96 /* Last used index value we have signalled on */ 97 bool signalled_used_valid; 98 99 /* Notification enabled? */ 100 bool notification; 101 102 uint16_t queue_index; 103 104 unsigned int inuse; 105 106 uint16_t vector; 107 VirtIOHandleOutput handle_output; 108 VirtIOHandleAIOOutput handle_aio_output; 109 VirtIODevice *vdev; 110 EventNotifier guest_notifier; 111 EventNotifier host_notifier; 112 QLIST_ENTRY(VirtQueue) node; 113 }; 114 115 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches) 116 { 117 if (!caches) { 118 return; 119 } 120 121 address_space_cache_destroy(&caches->desc); 122 address_space_cache_destroy(&caches->avail); 123 address_space_cache_destroy(&caches->used); 124 g_free(caches); 125 } 126 127 static void virtio_init_region_cache(VirtIODevice *vdev, int n) 128 { 129 VirtQueue *vq = &vdev->vq[n]; 130 VRingMemoryRegionCaches *old = vq->vring.caches; 131 VRingMemoryRegionCaches *new; 132 hwaddr addr, size; 133 int event_size; 134 135 event_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 136 137 addr = vq->vring.desc; 138 if (!addr) { 139 return; 140 } 141 new = g_new0(VRingMemoryRegionCaches, 1); 142 size = virtio_queue_get_desc_size(vdev, n); 143 address_space_cache_init(&new->desc, vdev->dma_as, 144 addr, size, false); 145 146 size = virtio_queue_get_used_size(vdev, n) + event_size; 147 address_space_cache_init(&new->used, vdev->dma_as, 148 vq->vring.used, size, true); 149 150 size = virtio_queue_get_avail_size(vdev, n) + event_size; 151 address_space_cache_init(&new->avail, vdev->dma_as, 152 vq->vring.avail, size, false); 153 154 atomic_rcu_set(&vq->vring.caches, new); 155 if (old) { 156 call_rcu(old, virtio_free_region_cache, rcu); 157 } 158 } 159 160 /* virt queue functions */ 161 void virtio_queue_update_rings(VirtIODevice *vdev, int n) 162 { 163 VRing *vring = &vdev->vq[n].vring; 164 165 if (!vring->desc) { 166 /* not yet setup -> nothing to do */ 167 return; 168 } 169 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); 170 vring->used = vring_align(vring->avail + 171 offsetof(VRingAvail, ring[vring->num]), 172 vring->align); 173 virtio_init_region_cache(vdev, n); 174 } 175 176 /* Called within rcu_read_lock(). */ 177 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc, 178 MemoryRegionCache *cache, int i) 179 { 180 address_space_read_cached(cache, i * sizeof(VRingDesc), 181 desc, sizeof(VRingDesc)); 182 virtio_tswap64s(vdev, &desc->addr); 183 virtio_tswap32s(vdev, &desc->len); 184 virtio_tswap16s(vdev, &desc->flags); 185 virtio_tswap16s(vdev, &desc->next); 186 } 187 188 /* Called within rcu_read_lock(). */ 189 static inline uint16_t vring_avail_flags(VirtQueue *vq) 190 { 191 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 192 hwaddr pa = offsetof(VRingAvail, flags); 193 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 194 } 195 196 /* Called within rcu_read_lock(). */ 197 static inline uint16_t vring_avail_idx(VirtQueue *vq) 198 { 199 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 200 hwaddr pa = offsetof(VRingAvail, idx); 201 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 202 return vq->shadow_avail_idx; 203 } 204 205 /* Called within rcu_read_lock(). */ 206 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) 207 { 208 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 209 hwaddr pa = offsetof(VRingAvail, ring[i]); 210 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); 211 } 212 213 /* Called within rcu_read_lock(). */ 214 static inline uint16_t vring_get_used_event(VirtQueue *vq) 215 { 216 return vring_avail_ring(vq, vq->vring.num); 217 } 218 219 /* Called within rcu_read_lock(). */ 220 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem, 221 int i) 222 { 223 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 224 hwaddr pa = offsetof(VRingUsed, ring[i]); 225 virtio_tswap32s(vq->vdev, &uelem->id); 226 virtio_tswap32s(vq->vdev, &uelem->len); 227 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); 228 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); 229 } 230 231 /* Called within rcu_read_lock(). */ 232 static uint16_t vring_used_idx(VirtQueue *vq) 233 { 234 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 235 hwaddr pa = offsetof(VRingUsed, idx); 236 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 237 } 238 239 /* Called within rcu_read_lock(). */ 240 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val) 241 { 242 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 243 hwaddr pa = offsetof(VRingUsed, idx); 244 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 245 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 246 vq->used_idx = val; 247 } 248 249 /* Called within rcu_read_lock(). */ 250 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) 251 { 252 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 253 VirtIODevice *vdev = vq->vdev; 254 hwaddr pa = offsetof(VRingUsed, flags); 255 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 256 257 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); 258 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 259 } 260 261 /* Called within rcu_read_lock(). */ 262 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) 263 { 264 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches); 265 VirtIODevice *vdev = vq->vdev; 266 hwaddr pa = offsetof(VRingUsed, flags); 267 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); 268 269 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); 270 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); 271 } 272 273 /* Called within rcu_read_lock(). */ 274 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val) 275 { 276 VRingMemoryRegionCaches *caches; 277 hwaddr pa; 278 if (!vq->notification) { 279 return; 280 } 281 282 caches = atomic_rcu_read(&vq->vring.caches); 283 pa = offsetof(VRingUsed, ring[vq->vring.num]); 284 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); 285 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); 286 } 287 288 void virtio_queue_set_notification(VirtQueue *vq, int enable) 289 { 290 vq->notification = enable; 291 292 if (!vq->vring.desc) { 293 return; 294 } 295 296 rcu_read_lock(); 297 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 298 vring_set_avail_event(vq, vring_avail_idx(vq)); 299 } else if (enable) { 300 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); 301 } else { 302 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); 303 } 304 if (enable) { 305 /* Expose avail event/used flags before caller checks the avail idx. */ 306 smp_mb(); 307 } 308 rcu_read_unlock(); 309 } 310 311 int virtio_queue_ready(VirtQueue *vq) 312 { 313 return vq->vring.avail != 0; 314 } 315 316 /* Fetch avail_idx from VQ memory only when we really need to know if 317 * guest has added some buffers. 318 * Called within rcu_read_lock(). */ 319 static int virtio_queue_empty_rcu(VirtQueue *vq) 320 { 321 if (vq->shadow_avail_idx != vq->last_avail_idx) { 322 return 0; 323 } 324 325 return vring_avail_idx(vq) == vq->last_avail_idx; 326 } 327 328 int virtio_queue_empty(VirtQueue *vq) 329 { 330 bool empty; 331 332 if (vq->shadow_avail_idx != vq->last_avail_idx) { 333 return 0; 334 } 335 336 rcu_read_lock(); 337 empty = vring_avail_idx(vq) == vq->last_avail_idx; 338 rcu_read_unlock(); 339 return empty; 340 } 341 342 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem, 343 unsigned int len) 344 { 345 AddressSpace *dma_as = vq->vdev->dma_as; 346 unsigned int offset; 347 int i; 348 349 offset = 0; 350 for (i = 0; i < elem->in_num; i++) { 351 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); 352 353 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, 354 elem->in_sg[i].iov_len, 355 DMA_DIRECTION_FROM_DEVICE, size); 356 357 offset += size; 358 } 359 360 for (i = 0; i < elem->out_num; i++) 361 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, 362 elem->out_sg[i].iov_len, 363 DMA_DIRECTION_TO_DEVICE, 364 elem->out_sg[i].iov_len); 365 } 366 367 /* virtqueue_detach_element: 368 * @vq: The #VirtQueue 369 * @elem: The #VirtQueueElement 370 * @len: number of bytes written 371 * 372 * Detach the element from the virtqueue. This function is suitable for device 373 * reset or other situations where a #VirtQueueElement is simply freed and will 374 * not be pushed or discarded. 375 */ 376 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem, 377 unsigned int len) 378 { 379 vq->inuse--; 380 virtqueue_unmap_sg(vq, elem, len); 381 } 382 383 /* virtqueue_unpop: 384 * @vq: The #VirtQueue 385 * @elem: The #VirtQueueElement 386 * @len: number of bytes written 387 * 388 * Pretend the most recent element wasn't popped from the virtqueue. The next 389 * call to virtqueue_pop() will refetch the element. 390 */ 391 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem, 392 unsigned int len) 393 { 394 vq->last_avail_idx--; 395 virtqueue_detach_element(vq, elem, len); 396 } 397 398 /* virtqueue_rewind: 399 * @vq: The #VirtQueue 400 * @num: Number of elements to push back 401 * 402 * Pretend that elements weren't popped from the virtqueue. The next 403 * virtqueue_pop() will refetch the oldest element. 404 * 405 * Use virtqueue_unpop() instead if you have a VirtQueueElement. 406 * 407 * Returns: true on success, false if @num is greater than the number of in use 408 * elements. 409 */ 410 bool virtqueue_rewind(VirtQueue *vq, unsigned int num) 411 { 412 if (num > vq->inuse) { 413 return false; 414 } 415 vq->last_avail_idx -= num; 416 vq->inuse -= num; 417 return true; 418 } 419 420 /* Called within rcu_read_lock(). */ 421 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, 422 unsigned int len, unsigned int idx) 423 { 424 VRingUsedElem uelem; 425 426 trace_virtqueue_fill(vq, elem, len, idx); 427 428 virtqueue_unmap_sg(vq, elem, len); 429 430 if (unlikely(vq->vdev->broken)) { 431 return; 432 } 433 434 idx = (idx + vq->used_idx) % vq->vring.num; 435 436 uelem.id = elem->index; 437 uelem.len = len; 438 vring_used_write(vq, &uelem, idx); 439 } 440 441 /* Called within rcu_read_lock(). */ 442 void virtqueue_flush(VirtQueue *vq, unsigned int count) 443 { 444 uint16_t old, new; 445 446 if (unlikely(vq->vdev->broken)) { 447 vq->inuse -= count; 448 return; 449 } 450 451 /* Make sure buffer is written before we update index. */ 452 smp_wmb(); 453 trace_virtqueue_flush(vq, count); 454 old = vq->used_idx; 455 new = old + count; 456 vring_used_idx_set(vq, new); 457 vq->inuse -= count; 458 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) 459 vq->signalled_used_valid = false; 460 } 461 462 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, 463 unsigned int len) 464 { 465 rcu_read_lock(); 466 virtqueue_fill(vq, elem, len, 0); 467 virtqueue_flush(vq, 1); 468 rcu_read_unlock(); 469 } 470 471 /* Called within rcu_read_lock(). */ 472 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) 473 { 474 uint16_t num_heads = vring_avail_idx(vq) - idx; 475 476 /* Check it isn't doing very strange things with descriptor numbers. */ 477 if (num_heads > vq->vring.num) { 478 virtio_error(vq->vdev, "Guest moved used index from %u to %u", 479 idx, vq->shadow_avail_idx); 480 return -EINVAL; 481 } 482 /* On success, callers read a descriptor at vq->last_avail_idx. 483 * Make sure descriptor read does not bypass avail index read. */ 484 if (num_heads) { 485 smp_rmb(); 486 } 487 488 return num_heads; 489 } 490 491 /* Called within rcu_read_lock(). */ 492 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx, 493 unsigned int *head) 494 { 495 /* Grab the next descriptor number they're advertising, and increment 496 * the index we've seen. */ 497 *head = vring_avail_ring(vq, idx % vq->vring.num); 498 499 /* If their number is silly, that's a fatal mistake. */ 500 if (*head >= vq->vring.num) { 501 virtio_error(vq->vdev, "Guest says index %u is available", *head); 502 return false; 503 } 504 505 return true; 506 } 507 508 enum { 509 VIRTQUEUE_READ_DESC_ERROR = -1, 510 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */ 511 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */ 512 }; 513 514 static int virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc, 515 MemoryRegionCache *desc_cache, unsigned int max, 516 unsigned int *next) 517 { 518 /* If this descriptor says it doesn't chain, we're done. */ 519 if (!(desc->flags & VRING_DESC_F_NEXT)) { 520 return VIRTQUEUE_READ_DESC_DONE; 521 } 522 523 /* Check they're not leading us off end of descriptors. */ 524 *next = desc->next; 525 /* Make sure compiler knows to grab that: we don't want it changing! */ 526 smp_wmb(); 527 528 if (*next >= max) { 529 virtio_error(vdev, "Desc next is %u", *next); 530 return VIRTQUEUE_READ_DESC_ERROR; 531 } 532 533 vring_desc_read(vdev, desc, desc_cache, *next); 534 return VIRTQUEUE_READ_DESC_MORE; 535 } 536 537 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes, 538 unsigned int *out_bytes, 539 unsigned max_in_bytes, unsigned max_out_bytes) 540 { 541 VirtIODevice *vdev = vq->vdev; 542 unsigned int max, idx; 543 unsigned int total_bufs, in_total, out_total; 544 VRingMemoryRegionCaches *caches; 545 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 546 int64_t len = 0; 547 int rc; 548 549 rcu_read_lock(); 550 idx = vq->last_avail_idx; 551 total_bufs = in_total = out_total = 0; 552 553 max = vq->vring.num; 554 caches = atomic_rcu_read(&vq->vring.caches); 555 if (caches->desc.len < max * sizeof(VRingDesc)) { 556 virtio_error(vdev, "Cannot map descriptor ring"); 557 goto err; 558 } 559 560 while ((rc = virtqueue_num_heads(vq, idx)) > 0) { 561 MemoryRegionCache *desc_cache = &caches->desc; 562 unsigned int num_bufs; 563 VRingDesc desc; 564 unsigned int i; 565 566 num_bufs = total_bufs; 567 568 if (!virtqueue_get_head(vq, idx++, &i)) { 569 goto err; 570 } 571 572 vring_desc_read(vdev, &desc, desc_cache, i); 573 574 if (desc.flags & VRING_DESC_F_INDIRECT) { 575 if (desc.len % sizeof(VRingDesc)) { 576 virtio_error(vdev, "Invalid size for indirect buffer table"); 577 goto err; 578 } 579 580 /* If we've got too many, that implies a descriptor loop. */ 581 if (num_bufs >= max) { 582 virtio_error(vdev, "Looped descriptor"); 583 goto err; 584 } 585 586 /* loop over the indirect descriptor table */ 587 len = address_space_cache_init(&indirect_desc_cache, 588 vdev->dma_as, 589 desc.addr, desc.len, false); 590 desc_cache = &indirect_desc_cache; 591 if (len < desc.len) { 592 virtio_error(vdev, "Cannot map indirect buffer"); 593 goto err; 594 } 595 596 max = desc.len / sizeof(VRingDesc); 597 num_bufs = i = 0; 598 vring_desc_read(vdev, &desc, desc_cache, i); 599 } 600 601 do { 602 /* If we've got too many, that implies a descriptor loop. */ 603 if (++num_bufs > max) { 604 virtio_error(vdev, "Looped descriptor"); 605 goto err; 606 } 607 608 if (desc.flags & VRING_DESC_F_WRITE) { 609 in_total += desc.len; 610 } else { 611 out_total += desc.len; 612 } 613 if (in_total >= max_in_bytes && out_total >= max_out_bytes) { 614 goto done; 615 } 616 617 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i); 618 } while (rc == VIRTQUEUE_READ_DESC_MORE); 619 620 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 621 goto err; 622 } 623 624 if (desc_cache == &indirect_desc_cache) { 625 address_space_cache_destroy(&indirect_desc_cache); 626 total_bufs++; 627 } else { 628 total_bufs = num_bufs; 629 } 630 } 631 632 if (rc < 0) { 633 goto err; 634 } 635 636 done: 637 address_space_cache_destroy(&indirect_desc_cache); 638 if (in_bytes) { 639 *in_bytes = in_total; 640 } 641 if (out_bytes) { 642 *out_bytes = out_total; 643 } 644 rcu_read_unlock(); 645 return; 646 647 err: 648 in_total = out_total = 0; 649 goto done; 650 } 651 652 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes, 653 unsigned int out_bytes) 654 { 655 unsigned int in_total, out_total; 656 657 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes); 658 return in_bytes <= in_total && out_bytes <= out_total; 659 } 660 661 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg, 662 hwaddr *addr, struct iovec *iov, 663 unsigned int max_num_sg, bool is_write, 664 hwaddr pa, size_t sz) 665 { 666 bool ok = false; 667 unsigned num_sg = *p_num_sg; 668 assert(num_sg <= max_num_sg); 669 670 if (!sz) { 671 virtio_error(vdev, "virtio: zero sized buffers are not allowed"); 672 goto out; 673 } 674 675 while (sz) { 676 hwaddr len = sz; 677 678 if (num_sg == max_num_sg) { 679 virtio_error(vdev, "virtio: too many write descriptors in " 680 "indirect table"); 681 goto out; 682 } 683 684 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, 685 is_write ? 686 DMA_DIRECTION_FROM_DEVICE : 687 DMA_DIRECTION_TO_DEVICE); 688 if (!iov[num_sg].iov_base) { 689 virtio_error(vdev, "virtio: bogus descriptor or out of resources"); 690 goto out; 691 } 692 693 iov[num_sg].iov_len = len; 694 addr[num_sg] = pa; 695 696 sz -= len; 697 pa += len; 698 num_sg++; 699 } 700 ok = true; 701 702 out: 703 *p_num_sg = num_sg; 704 return ok; 705 } 706 707 /* Only used by error code paths before we have a VirtQueueElement (therefore 708 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to 709 * yet. 710 */ 711 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num, 712 struct iovec *iov) 713 { 714 unsigned int i; 715 716 for (i = 0; i < out_num + in_num; i++) { 717 int is_write = i >= out_num; 718 719 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); 720 iov++; 721 } 722 } 723 724 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg, 725 hwaddr *addr, unsigned int *num_sg, 726 int is_write) 727 { 728 unsigned int i; 729 hwaddr len; 730 731 for (i = 0; i < *num_sg; i++) { 732 len = sg[i].iov_len; 733 sg[i].iov_base = dma_memory_map(vdev->dma_as, 734 addr[i], &len, is_write ? 735 DMA_DIRECTION_FROM_DEVICE : 736 DMA_DIRECTION_TO_DEVICE); 737 if (!sg[i].iov_base) { 738 error_report("virtio: error trying to map MMIO memory"); 739 exit(1); 740 } 741 if (len != sg[i].iov_len) { 742 error_report("virtio: unexpected memory split"); 743 exit(1); 744 } 745 } 746 } 747 748 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem) 749 { 750 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, &elem->in_num, 1); 751 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, &elem->out_num, 0); 752 } 753 754 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num) 755 { 756 VirtQueueElement *elem; 757 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); 758 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); 759 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); 760 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); 761 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); 762 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); 763 764 assert(sz >= sizeof(VirtQueueElement)); 765 elem = g_malloc(out_sg_end); 766 elem->out_num = out_num; 767 elem->in_num = in_num; 768 elem->in_addr = (void *)elem + in_addr_ofs; 769 elem->out_addr = (void *)elem + out_addr_ofs; 770 elem->in_sg = (void *)elem + in_sg_ofs; 771 elem->out_sg = (void *)elem + out_sg_ofs; 772 return elem; 773 } 774 775 void *virtqueue_pop(VirtQueue *vq, size_t sz) 776 { 777 unsigned int i, head, max; 778 VRingMemoryRegionCaches *caches; 779 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; 780 MemoryRegionCache *desc_cache; 781 int64_t len; 782 VirtIODevice *vdev = vq->vdev; 783 VirtQueueElement *elem = NULL; 784 unsigned out_num, in_num; 785 hwaddr addr[VIRTQUEUE_MAX_SIZE]; 786 struct iovec iov[VIRTQUEUE_MAX_SIZE]; 787 VRingDesc desc; 788 int rc; 789 790 if (unlikely(vdev->broken)) { 791 return NULL; 792 } 793 rcu_read_lock(); 794 if (virtio_queue_empty_rcu(vq)) { 795 goto done; 796 } 797 /* Needed after virtio_queue_empty(), see comment in 798 * virtqueue_num_heads(). */ 799 smp_rmb(); 800 801 /* When we start there are none of either input nor output. */ 802 out_num = in_num = 0; 803 804 max = vq->vring.num; 805 806 if (vq->inuse >= vq->vring.num) { 807 virtio_error(vdev, "Virtqueue size exceeded"); 808 goto done; 809 } 810 811 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { 812 goto done; 813 } 814 815 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 816 vring_set_avail_event(vq, vq->last_avail_idx); 817 } 818 819 i = head; 820 821 caches = atomic_rcu_read(&vq->vring.caches); 822 if (caches->desc.len < max * sizeof(VRingDesc)) { 823 virtio_error(vdev, "Cannot map descriptor ring"); 824 goto done; 825 } 826 827 desc_cache = &caches->desc; 828 vring_desc_read(vdev, &desc, desc_cache, i); 829 if (desc.flags & VRING_DESC_F_INDIRECT) { 830 if (desc.len % sizeof(VRingDesc)) { 831 virtio_error(vdev, "Invalid size for indirect buffer table"); 832 goto done; 833 } 834 835 /* loop over the indirect descriptor table */ 836 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, 837 desc.addr, desc.len, false); 838 desc_cache = &indirect_desc_cache; 839 if (len < desc.len) { 840 virtio_error(vdev, "Cannot map indirect buffer"); 841 goto done; 842 } 843 844 max = desc.len / sizeof(VRingDesc); 845 i = 0; 846 vring_desc_read(vdev, &desc, desc_cache, i); 847 } 848 849 /* Collect all the descriptors */ 850 do { 851 bool map_ok; 852 853 if (desc.flags & VRING_DESC_F_WRITE) { 854 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num, 855 iov + out_num, 856 VIRTQUEUE_MAX_SIZE - out_num, true, 857 desc.addr, desc.len); 858 } else { 859 if (in_num) { 860 virtio_error(vdev, "Incorrect order for descriptors"); 861 goto err_undo_map; 862 } 863 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov, 864 VIRTQUEUE_MAX_SIZE, false, 865 desc.addr, desc.len); 866 } 867 if (!map_ok) { 868 goto err_undo_map; 869 } 870 871 /* If we've got too many, that implies a descriptor loop. */ 872 if ((in_num + out_num) > max) { 873 virtio_error(vdev, "Looped descriptor"); 874 goto err_undo_map; 875 } 876 877 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i); 878 } while (rc == VIRTQUEUE_READ_DESC_MORE); 879 880 if (rc == VIRTQUEUE_READ_DESC_ERROR) { 881 goto err_undo_map; 882 } 883 884 /* Now copy what we have collected and mapped */ 885 elem = virtqueue_alloc_element(sz, out_num, in_num); 886 elem->index = head; 887 for (i = 0; i < out_num; i++) { 888 elem->out_addr[i] = addr[i]; 889 elem->out_sg[i] = iov[i]; 890 } 891 for (i = 0; i < in_num; i++) { 892 elem->in_addr[i] = addr[out_num + i]; 893 elem->in_sg[i] = iov[out_num + i]; 894 } 895 896 vq->inuse++; 897 898 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); 899 done: 900 address_space_cache_destroy(&indirect_desc_cache); 901 rcu_read_unlock(); 902 903 return elem; 904 905 err_undo_map: 906 virtqueue_undo_map_desc(out_num, in_num, iov); 907 goto done; 908 } 909 910 /* virtqueue_drop_all: 911 * @vq: The #VirtQueue 912 * Drops all queued buffers and indicates them to the guest 913 * as if they are done. Useful when buffers can not be 914 * processed but must be returned to the guest. 915 */ 916 unsigned int virtqueue_drop_all(VirtQueue *vq) 917 { 918 unsigned int dropped = 0; 919 VirtQueueElement elem = {}; 920 VirtIODevice *vdev = vq->vdev; 921 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 922 923 if (unlikely(vdev->broken)) { 924 return 0; 925 } 926 927 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { 928 /* works similar to virtqueue_pop but does not map buffers 929 * and does not allocate any memory */ 930 smp_rmb(); 931 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { 932 break; 933 } 934 vq->inuse++; 935 vq->last_avail_idx++; 936 if (fEventIdx) { 937 vring_set_avail_event(vq, vq->last_avail_idx); 938 } 939 /* immediately push the element, nothing to unmap 940 * as both in_num and out_num are set to 0 */ 941 virtqueue_push(vq, &elem, 0); 942 dropped++; 943 } 944 945 return dropped; 946 } 947 948 /* Reading and writing a structure directly to QEMUFile is *awful*, but 949 * it is what QEMU has always done by mistake. We can change it sooner 950 * or later by bumping the version number of the affected vm states. 951 * In the meanwhile, since the in-memory layout of VirtQueueElement 952 * has changed, we need to marshal to and from the layout that was 953 * used before the change. 954 */ 955 typedef struct VirtQueueElementOld { 956 unsigned int index; 957 unsigned int out_num; 958 unsigned int in_num; 959 hwaddr in_addr[VIRTQUEUE_MAX_SIZE]; 960 hwaddr out_addr[VIRTQUEUE_MAX_SIZE]; 961 struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; 962 struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; 963 } VirtQueueElementOld; 964 965 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz) 966 { 967 VirtQueueElement *elem; 968 VirtQueueElementOld data; 969 int i; 970 971 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 972 973 /* TODO: teach all callers that this can fail, and return failure instead 974 * of asserting here. 975 * When we do, we might be able to re-enable NDEBUG below. 976 */ 977 #ifdef NDEBUG 978 #error building with NDEBUG is not supported 979 #endif 980 assert(ARRAY_SIZE(data.in_addr) >= data.in_num); 981 assert(ARRAY_SIZE(data.out_addr) >= data.out_num); 982 983 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num); 984 elem->index = data.index; 985 986 for (i = 0; i < elem->in_num; i++) { 987 elem->in_addr[i] = data.in_addr[i]; 988 } 989 990 for (i = 0; i < elem->out_num; i++) { 991 elem->out_addr[i] = data.out_addr[i]; 992 } 993 994 for (i = 0; i < elem->in_num; i++) { 995 /* Base is overwritten by virtqueue_map. */ 996 elem->in_sg[i].iov_base = 0; 997 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; 998 } 999 1000 for (i = 0; i < elem->out_num; i++) { 1001 /* Base is overwritten by virtqueue_map. */ 1002 elem->out_sg[i].iov_base = 0; 1003 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; 1004 } 1005 1006 virtqueue_map(vdev, elem); 1007 return elem; 1008 } 1009 1010 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem) 1011 { 1012 VirtQueueElementOld data; 1013 int i; 1014 1015 memset(&data, 0, sizeof(data)); 1016 data.index = elem->index; 1017 data.in_num = elem->in_num; 1018 data.out_num = elem->out_num; 1019 1020 for (i = 0; i < elem->in_num; i++) { 1021 data.in_addr[i] = elem->in_addr[i]; 1022 } 1023 1024 for (i = 0; i < elem->out_num; i++) { 1025 data.out_addr[i] = elem->out_addr[i]; 1026 } 1027 1028 for (i = 0; i < elem->in_num; i++) { 1029 /* Base is overwritten by virtqueue_map when loading. Do not 1030 * save it, as it would leak the QEMU address space layout. */ 1031 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; 1032 } 1033 1034 for (i = 0; i < elem->out_num; i++) { 1035 /* Do not save iov_base as above. */ 1036 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; 1037 } 1038 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld)); 1039 } 1040 1041 /* virtio device */ 1042 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) 1043 { 1044 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1045 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1046 1047 if (unlikely(vdev->broken)) { 1048 return; 1049 } 1050 1051 if (k->notify) { 1052 k->notify(qbus->parent, vector); 1053 } 1054 } 1055 1056 void virtio_update_irq(VirtIODevice *vdev) 1057 { 1058 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 1059 } 1060 1061 static int virtio_validate_features(VirtIODevice *vdev) 1062 { 1063 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1064 1065 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) && 1066 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { 1067 return -EFAULT; 1068 } 1069 1070 if (k->validate_features) { 1071 return k->validate_features(vdev); 1072 } else { 1073 return 0; 1074 } 1075 } 1076 1077 int virtio_set_status(VirtIODevice *vdev, uint8_t val) 1078 { 1079 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1080 trace_virtio_set_status(vdev, val); 1081 1082 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1083 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && 1084 val & VIRTIO_CONFIG_S_FEATURES_OK) { 1085 int ret = virtio_validate_features(vdev); 1086 1087 if (ret) { 1088 return ret; 1089 } 1090 } 1091 } 1092 if (k->set_status) { 1093 k->set_status(vdev, val); 1094 } 1095 vdev->status = val; 1096 return 0; 1097 } 1098 1099 bool target_words_bigendian(void); 1100 static enum virtio_device_endian virtio_default_endian(void) 1101 { 1102 if (target_words_bigendian()) { 1103 return VIRTIO_DEVICE_ENDIAN_BIG; 1104 } else { 1105 return VIRTIO_DEVICE_ENDIAN_LITTLE; 1106 } 1107 } 1108 1109 static enum virtio_device_endian virtio_current_cpu_endian(void) 1110 { 1111 CPUClass *cc = CPU_GET_CLASS(current_cpu); 1112 1113 if (cc->virtio_is_big_endian(current_cpu)) { 1114 return VIRTIO_DEVICE_ENDIAN_BIG; 1115 } else { 1116 return VIRTIO_DEVICE_ENDIAN_LITTLE; 1117 } 1118 } 1119 1120 void virtio_reset(void *opaque) 1121 { 1122 VirtIODevice *vdev = opaque; 1123 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1124 int i; 1125 1126 virtio_set_status(vdev, 0); 1127 if (current_cpu) { 1128 /* Guest initiated reset */ 1129 vdev->device_endian = virtio_current_cpu_endian(); 1130 } else { 1131 /* System reset */ 1132 vdev->device_endian = virtio_default_endian(); 1133 } 1134 1135 if (k->reset) { 1136 k->reset(vdev); 1137 } 1138 1139 vdev->broken = false; 1140 vdev->guest_features = 0; 1141 vdev->queue_sel = 0; 1142 vdev->status = 0; 1143 atomic_set(&vdev->isr, 0); 1144 vdev->config_vector = VIRTIO_NO_VECTOR; 1145 virtio_notify_vector(vdev, vdev->config_vector); 1146 1147 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1148 vdev->vq[i].vring.desc = 0; 1149 vdev->vq[i].vring.avail = 0; 1150 vdev->vq[i].vring.used = 0; 1151 vdev->vq[i].last_avail_idx = 0; 1152 vdev->vq[i].shadow_avail_idx = 0; 1153 vdev->vq[i].used_idx = 0; 1154 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR); 1155 vdev->vq[i].signalled_used = 0; 1156 vdev->vq[i].signalled_used_valid = false; 1157 vdev->vq[i].notification = true; 1158 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; 1159 vdev->vq[i].inuse = 0; 1160 } 1161 } 1162 1163 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) 1164 { 1165 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1166 uint8_t val; 1167 1168 if (addr + sizeof(val) > vdev->config_len) { 1169 return (uint32_t)-1; 1170 } 1171 1172 k->get_config(vdev, vdev->config); 1173 1174 val = ldub_p(vdev->config + addr); 1175 return val; 1176 } 1177 1178 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) 1179 { 1180 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1181 uint16_t val; 1182 1183 if (addr + sizeof(val) > vdev->config_len) { 1184 return (uint32_t)-1; 1185 } 1186 1187 k->get_config(vdev, vdev->config); 1188 1189 val = lduw_p(vdev->config + addr); 1190 return val; 1191 } 1192 1193 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) 1194 { 1195 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1196 uint32_t val; 1197 1198 if (addr + sizeof(val) > vdev->config_len) { 1199 return (uint32_t)-1; 1200 } 1201 1202 k->get_config(vdev, vdev->config); 1203 1204 val = ldl_p(vdev->config + addr); 1205 return val; 1206 } 1207 1208 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) 1209 { 1210 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1211 uint8_t val = data; 1212 1213 if (addr + sizeof(val) > vdev->config_len) { 1214 return; 1215 } 1216 1217 stb_p(vdev->config + addr, val); 1218 1219 if (k->set_config) { 1220 k->set_config(vdev, vdev->config); 1221 } 1222 } 1223 1224 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) 1225 { 1226 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1227 uint16_t val = data; 1228 1229 if (addr + sizeof(val) > vdev->config_len) { 1230 return; 1231 } 1232 1233 stw_p(vdev->config + addr, val); 1234 1235 if (k->set_config) { 1236 k->set_config(vdev, vdev->config); 1237 } 1238 } 1239 1240 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) 1241 { 1242 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1243 uint32_t val = data; 1244 1245 if (addr + sizeof(val) > vdev->config_len) { 1246 return; 1247 } 1248 1249 stl_p(vdev->config + addr, val); 1250 1251 if (k->set_config) { 1252 k->set_config(vdev, vdev->config); 1253 } 1254 } 1255 1256 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr) 1257 { 1258 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1259 uint8_t val; 1260 1261 if (addr + sizeof(val) > vdev->config_len) { 1262 return (uint32_t)-1; 1263 } 1264 1265 k->get_config(vdev, vdev->config); 1266 1267 val = ldub_p(vdev->config + addr); 1268 return val; 1269 } 1270 1271 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr) 1272 { 1273 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1274 uint16_t val; 1275 1276 if (addr + sizeof(val) > vdev->config_len) { 1277 return (uint32_t)-1; 1278 } 1279 1280 k->get_config(vdev, vdev->config); 1281 1282 val = lduw_le_p(vdev->config + addr); 1283 return val; 1284 } 1285 1286 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr) 1287 { 1288 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1289 uint32_t val; 1290 1291 if (addr + sizeof(val) > vdev->config_len) { 1292 return (uint32_t)-1; 1293 } 1294 1295 k->get_config(vdev, vdev->config); 1296 1297 val = ldl_le_p(vdev->config + addr); 1298 return val; 1299 } 1300 1301 void virtio_config_modern_writeb(VirtIODevice *vdev, 1302 uint32_t addr, uint32_t data) 1303 { 1304 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1305 uint8_t val = data; 1306 1307 if (addr + sizeof(val) > vdev->config_len) { 1308 return; 1309 } 1310 1311 stb_p(vdev->config + addr, val); 1312 1313 if (k->set_config) { 1314 k->set_config(vdev, vdev->config); 1315 } 1316 } 1317 1318 void virtio_config_modern_writew(VirtIODevice *vdev, 1319 uint32_t addr, uint32_t data) 1320 { 1321 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1322 uint16_t val = data; 1323 1324 if (addr + sizeof(val) > vdev->config_len) { 1325 return; 1326 } 1327 1328 stw_le_p(vdev->config + addr, val); 1329 1330 if (k->set_config) { 1331 k->set_config(vdev, vdev->config); 1332 } 1333 } 1334 1335 void virtio_config_modern_writel(VirtIODevice *vdev, 1336 uint32_t addr, uint32_t data) 1337 { 1338 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1339 uint32_t val = data; 1340 1341 if (addr + sizeof(val) > vdev->config_len) { 1342 return; 1343 } 1344 1345 stl_le_p(vdev->config + addr, val); 1346 1347 if (k->set_config) { 1348 k->set_config(vdev, vdev->config); 1349 } 1350 } 1351 1352 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr) 1353 { 1354 vdev->vq[n].vring.desc = addr; 1355 virtio_queue_update_rings(vdev, n); 1356 } 1357 1358 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n) 1359 { 1360 return vdev->vq[n].vring.desc; 1361 } 1362 1363 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, 1364 hwaddr avail, hwaddr used) 1365 { 1366 vdev->vq[n].vring.desc = desc; 1367 vdev->vq[n].vring.avail = avail; 1368 vdev->vq[n].vring.used = used; 1369 virtio_init_region_cache(vdev, n); 1370 } 1371 1372 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) 1373 { 1374 /* Don't allow guest to flip queue between existent and 1375 * nonexistent states, or to set it to an invalid size. 1376 */ 1377 if (!!num != !!vdev->vq[n].vring.num || 1378 num > VIRTQUEUE_MAX_SIZE || 1379 num < 0) { 1380 return; 1381 } 1382 vdev->vq[n].vring.num = num; 1383 } 1384 1385 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector) 1386 { 1387 return QLIST_FIRST(&vdev->vector_queues[vector]); 1388 } 1389 1390 VirtQueue *virtio_vector_next_queue(VirtQueue *vq) 1391 { 1392 return QLIST_NEXT(vq, node); 1393 } 1394 1395 int virtio_queue_get_num(VirtIODevice *vdev, int n) 1396 { 1397 return vdev->vq[n].vring.num; 1398 } 1399 1400 int virtio_queue_get_max_num(VirtIODevice *vdev, int n) 1401 { 1402 return vdev->vq[n].vring.num_default; 1403 } 1404 1405 int virtio_get_num_queues(VirtIODevice *vdev) 1406 { 1407 int i; 1408 1409 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1410 if (!virtio_queue_get_num(vdev, i)) { 1411 break; 1412 } 1413 } 1414 1415 return i; 1416 } 1417 1418 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align) 1419 { 1420 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1421 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1422 1423 /* virtio-1 compliant devices cannot change the alignment */ 1424 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1425 error_report("tried to modify queue alignment for virtio-1 device"); 1426 return; 1427 } 1428 /* Check that the transport told us it was going to do this 1429 * (so a buggy transport will immediately assert rather than 1430 * silently failing to migrate this state) 1431 */ 1432 assert(k->has_variable_vring_alignment); 1433 1434 vdev->vq[n].vring.align = align; 1435 virtio_queue_update_rings(vdev, n); 1436 } 1437 1438 static bool virtio_queue_notify_aio_vq(VirtQueue *vq) 1439 { 1440 if (vq->vring.desc && vq->handle_aio_output) { 1441 VirtIODevice *vdev = vq->vdev; 1442 1443 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 1444 return vq->handle_aio_output(vdev, vq); 1445 } 1446 1447 return false; 1448 } 1449 1450 static void virtio_queue_notify_vq(VirtQueue *vq) 1451 { 1452 if (vq->vring.desc && vq->handle_output) { 1453 VirtIODevice *vdev = vq->vdev; 1454 1455 if (unlikely(vdev->broken)) { 1456 return; 1457 } 1458 1459 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); 1460 vq->handle_output(vdev, vq); 1461 } 1462 } 1463 1464 void virtio_queue_notify(VirtIODevice *vdev, int n) 1465 { 1466 virtio_queue_notify_vq(&vdev->vq[n]); 1467 } 1468 1469 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) 1470 { 1471 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : 1472 VIRTIO_NO_VECTOR; 1473 } 1474 1475 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) 1476 { 1477 VirtQueue *vq = &vdev->vq[n]; 1478 1479 if (n < VIRTIO_QUEUE_MAX) { 1480 if (vdev->vector_queues && 1481 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { 1482 QLIST_REMOVE(vq, node); 1483 } 1484 vdev->vq[n].vector = vector; 1485 if (vdev->vector_queues && 1486 vector != VIRTIO_NO_VECTOR) { 1487 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); 1488 } 1489 } 1490 } 1491 1492 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, 1493 VirtIOHandleOutput handle_output) 1494 { 1495 int i; 1496 1497 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1498 if (vdev->vq[i].vring.num == 0) 1499 break; 1500 } 1501 1502 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) 1503 abort(); 1504 1505 vdev->vq[i].vring.num = queue_size; 1506 vdev->vq[i].vring.num_default = queue_size; 1507 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; 1508 vdev->vq[i].handle_output = handle_output; 1509 vdev->vq[i].handle_aio_output = NULL; 1510 1511 return &vdev->vq[i]; 1512 } 1513 1514 void virtio_del_queue(VirtIODevice *vdev, int n) 1515 { 1516 if (n < 0 || n >= VIRTIO_QUEUE_MAX) { 1517 abort(); 1518 } 1519 1520 vdev->vq[n].vring.num = 0; 1521 vdev->vq[n].vring.num_default = 0; 1522 } 1523 1524 static void virtio_set_isr(VirtIODevice *vdev, int value) 1525 { 1526 uint8_t old = atomic_read(&vdev->isr); 1527 1528 /* Do not write ISR if it does not change, so that its cacheline remains 1529 * shared in the common case where the guest does not read it. 1530 */ 1531 if ((old & value) != value) { 1532 atomic_or(&vdev->isr, value); 1533 } 1534 } 1535 1536 /* Called within rcu_read_lock(). */ 1537 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq) 1538 { 1539 uint16_t old, new; 1540 bool v; 1541 /* We need to expose used array entries before checking used event. */ 1542 smp_mb(); 1543 /* Always notify when queue is empty (when feature acknowledge) */ 1544 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) && 1545 !vq->inuse && virtio_queue_empty(vq)) { 1546 return true; 1547 } 1548 1549 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 1550 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); 1551 } 1552 1553 v = vq->signalled_used_valid; 1554 vq->signalled_used_valid = true; 1555 old = vq->signalled_used; 1556 new = vq->signalled_used = vq->used_idx; 1557 return !v || vring_need_event(vring_get_used_event(vq), new, old); 1558 } 1559 1560 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq) 1561 { 1562 bool should_notify; 1563 rcu_read_lock(); 1564 should_notify = virtio_should_notify(vdev, vq); 1565 rcu_read_unlock(); 1566 1567 if (!should_notify) { 1568 return; 1569 } 1570 1571 trace_virtio_notify_irqfd(vdev, vq); 1572 1573 /* 1574 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but 1575 * windows drivers included in virtio-win 1.8.0 (circa 2015) are 1576 * incorrectly polling this bit during crashdump and hibernation 1577 * in MSI mode, causing a hang if this bit is never updated. 1578 * Recent releases of Windows do not really shut down, but rather 1579 * log out and hibernate to make the next startup faster. Hence, 1580 * this manifested as a more serious hang during shutdown with 1581 * 1582 * Next driver release from 2016 fixed this problem, so working around it 1583 * is not a must, but it's easy to do so let's do it here. 1584 * 1585 * Note: it's safe to update ISR from any thread as it was switched 1586 * to an atomic operation. 1587 */ 1588 virtio_set_isr(vq->vdev, 0x1); 1589 event_notifier_set(&vq->guest_notifier); 1590 } 1591 1592 static void virtio_irq(VirtQueue *vq) 1593 { 1594 virtio_set_isr(vq->vdev, 0x1); 1595 virtio_notify_vector(vq->vdev, vq->vector); 1596 } 1597 1598 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) 1599 { 1600 bool should_notify; 1601 rcu_read_lock(); 1602 should_notify = virtio_should_notify(vdev, vq); 1603 rcu_read_unlock(); 1604 1605 if (!should_notify) { 1606 return; 1607 } 1608 1609 trace_virtio_notify(vdev, vq); 1610 virtio_irq(vq); 1611 } 1612 1613 void virtio_notify_config(VirtIODevice *vdev) 1614 { 1615 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1616 return; 1617 1618 virtio_set_isr(vdev, 0x3); 1619 vdev->generation++; 1620 virtio_notify_vector(vdev, vdev->config_vector); 1621 } 1622 1623 static bool virtio_device_endian_needed(void *opaque) 1624 { 1625 VirtIODevice *vdev = opaque; 1626 1627 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); 1628 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 1629 return vdev->device_endian != virtio_default_endian(); 1630 } 1631 /* Devices conforming to VIRTIO 1.0 or later are always LE. */ 1632 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; 1633 } 1634 1635 static bool virtio_64bit_features_needed(void *opaque) 1636 { 1637 VirtIODevice *vdev = opaque; 1638 1639 return (vdev->host_features >> 32) != 0; 1640 } 1641 1642 static bool virtio_virtqueue_needed(void *opaque) 1643 { 1644 VirtIODevice *vdev = opaque; 1645 1646 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1); 1647 } 1648 1649 static bool virtio_ringsize_needed(void *opaque) 1650 { 1651 VirtIODevice *vdev = opaque; 1652 int i; 1653 1654 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1655 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { 1656 return true; 1657 } 1658 } 1659 return false; 1660 } 1661 1662 static bool virtio_extra_state_needed(void *opaque) 1663 { 1664 VirtIODevice *vdev = opaque; 1665 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1666 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1667 1668 return k->has_extra_state && 1669 k->has_extra_state(qbus->parent); 1670 } 1671 1672 static bool virtio_broken_needed(void *opaque) 1673 { 1674 VirtIODevice *vdev = opaque; 1675 1676 return vdev->broken; 1677 } 1678 1679 static const VMStateDescription vmstate_virtqueue = { 1680 .name = "virtqueue_state", 1681 .version_id = 1, 1682 .minimum_version_id = 1, 1683 .fields = (VMStateField[]) { 1684 VMSTATE_UINT64(vring.avail, struct VirtQueue), 1685 VMSTATE_UINT64(vring.used, struct VirtQueue), 1686 VMSTATE_END_OF_LIST() 1687 } 1688 }; 1689 1690 static const VMStateDescription vmstate_virtio_virtqueues = { 1691 .name = "virtio/virtqueues", 1692 .version_id = 1, 1693 .minimum_version_id = 1, 1694 .needed = &virtio_virtqueue_needed, 1695 .fields = (VMStateField[]) { 1696 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 1697 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue), 1698 VMSTATE_END_OF_LIST() 1699 } 1700 }; 1701 1702 static const VMStateDescription vmstate_ringsize = { 1703 .name = "ringsize_state", 1704 .version_id = 1, 1705 .minimum_version_id = 1, 1706 .fields = (VMStateField[]) { 1707 VMSTATE_UINT32(vring.num_default, struct VirtQueue), 1708 VMSTATE_END_OF_LIST() 1709 } 1710 }; 1711 1712 static const VMStateDescription vmstate_virtio_ringsize = { 1713 .name = "virtio/ringsize", 1714 .version_id = 1, 1715 .minimum_version_id = 1, 1716 .needed = &virtio_ringsize_needed, 1717 .fields = (VMStateField[]) { 1718 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice, 1719 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue), 1720 VMSTATE_END_OF_LIST() 1721 } 1722 }; 1723 1724 static int get_extra_state(QEMUFile *f, void *pv, size_t size, 1725 VMStateField *field) 1726 { 1727 VirtIODevice *vdev = pv; 1728 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1729 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1730 1731 if (!k->load_extra_state) { 1732 return -1; 1733 } else { 1734 return k->load_extra_state(qbus->parent, f); 1735 } 1736 } 1737 1738 static int put_extra_state(QEMUFile *f, void *pv, size_t size, 1739 VMStateField *field, QJSON *vmdesc) 1740 { 1741 VirtIODevice *vdev = pv; 1742 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1743 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1744 1745 k->save_extra_state(qbus->parent, f); 1746 return 0; 1747 } 1748 1749 static const VMStateInfo vmstate_info_extra_state = { 1750 .name = "virtqueue_extra_state", 1751 .get = get_extra_state, 1752 .put = put_extra_state, 1753 }; 1754 1755 static const VMStateDescription vmstate_virtio_extra_state = { 1756 .name = "virtio/extra_state", 1757 .version_id = 1, 1758 .minimum_version_id = 1, 1759 .needed = &virtio_extra_state_needed, 1760 .fields = (VMStateField[]) { 1761 { 1762 .name = "extra_state", 1763 .version_id = 0, 1764 .field_exists = NULL, 1765 .size = 0, 1766 .info = &vmstate_info_extra_state, 1767 .flags = VMS_SINGLE, 1768 .offset = 0, 1769 }, 1770 VMSTATE_END_OF_LIST() 1771 } 1772 }; 1773 1774 static const VMStateDescription vmstate_virtio_device_endian = { 1775 .name = "virtio/device_endian", 1776 .version_id = 1, 1777 .minimum_version_id = 1, 1778 .needed = &virtio_device_endian_needed, 1779 .fields = (VMStateField[]) { 1780 VMSTATE_UINT8(device_endian, VirtIODevice), 1781 VMSTATE_END_OF_LIST() 1782 } 1783 }; 1784 1785 static const VMStateDescription vmstate_virtio_64bit_features = { 1786 .name = "virtio/64bit_features", 1787 .version_id = 1, 1788 .minimum_version_id = 1, 1789 .needed = &virtio_64bit_features_needed, 1790 .fields = (VMStateField[]) { 1791 VMSTATE_UINT64(guest_features, VirtIODevice), 1792 VMSTATE_END_OF_LIST() 1793 } 1794 }; 1795 1796 static const VMStateDescription vmstate_virtio_broken = { 1797 .name = "virtio/broken", 1798 .version_id = 1, 1799 .minimum_version_id = 1, 1800 .needed = &virtio_broken_needed, 1801 .fields = (VMStateField[]) { 1802 VMSTATE_BOOL(broken, VirtIODevice), 1803 VMSTATE_END_OF_LIST() 1804 } 1805 }; 1806 1807 static const VMStateDescription vmstate_virtio = { 1808 .name = "virtio", 1809 .version_id = 1, 1810 .minimum_version_id = 1, 1811 .minimum_version_id_old = 1, 1812 .fields = (VMStateField[]) { 1813 VMSTATE_END_OF_LIST() 1814 }, 1815 .subsections = (const VMStateDescription*[]) { 1816 &vmstate_virtio_device_endian, 1817 &vmstate_virtio_64bit_features, 1818 &vmstate_virtio_virtqueues, 1819 &vmstate_virtio_ringsize, 1820 &vmstate_virtio_broken, 1821 &vmstate_virtio_extra_state, 1822 NULL 1823 } 1824 }; 1825 1826 void virtio_save(VirtIODevice *vdev, QEMUFile *f) 1827 { 1828 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1829 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1830 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1831 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); 1832 int i; 1833 1834 if (k->save_config) { 1835 k->save_config(qbus->parent, f); 1836 } 1837 1838 qemu_put_8s(f, &vdev->status); 1839 qemu_put_8s(f, &vdev->isr); 1840 qemu_put_be16s(f, &vdev->queue_sel); 1841 qemu_put_be32s(f, &guest_features_lo); 1842 qemu_put_be32(f, vdev->config_len); 1843 qemu_put_buffer(f, vdev->config, vdev->config_len); 1844 1845 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1846 if (vdev->vq[i].vring.num == 0) 1847 break; 1848 } 1849 1850 qemu_put_be32(f, i); 1851 1852 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1853 if (vdev->vq[i].vring.num == 0) 1854 break; 1855 1856 qemu_put_be32(f, vdev->vq[i].vring.num); 1857 if (k->has_variable_vring_alignment) { 1858 qemu_put_be32(f, vdev->vq[i].vring.align); 1859 } 1860 /* 1861 * Save desc now, the rest of the ring addresses are saved in 1862 * subsections for VIRTIO-1 devices. 1863 */ 1864 qemu_put_be64(f, vdev->vq[i].vring.desc); 1865 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); 1866 if (k->save_queue) { 1867 k->save_queue(qbus->parent, i, f); 1868 } 1869 } 1870 1871 if (vdc->save != NULL) { 1872 vdc->save(vdev, f); 1873 } 1874 1875 if (vdc->vmsd) { 1876 vmstate_save_state(f, vdc->vmsd, vdev, NULL); 1877 } 1878 1879 /* Subsections */ 1880 vmstate_save_state(f, &vmstate_virtio, vdev, NULL); 1881 } 1882 1883 /* A wrapper for use as a VMState .put function */ 1884 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, 1885 VMStateField *field, QJSON *vmdesc) 1886 { 1887 virtio_save(VIRTIO_DEVICE(opaque), f); 1888 1889 return 0; 1890 } 1891 1892 /* A wrapper for use as a VMState .get function */ 1893 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size, 1894 VMStateField *field) 1895 { 1896 VirtIODevice *vdev = VIRTIO_DEVICE(opaque); 1897 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev)); 1898 1899 return virtio_load(vdev, f, dc->vmsd->version_id); 1900 } 1901 1902 const VMStateInfo virtio_vmstate_info = { 1903 .name = "virtio", 1904 .get = virtio_device_get, 1905 .put = virtio_device_put, 1906 }; 1907 1908 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val) 1909 { 1910 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); 1911 bool bad = (val & ~(vdev->host_features)) != 0; 1912 1913 val &= vdev->host_features; 1914 if (k->set_features) { 1915 k->set_features(vdev, val); 1916 } 1917 vdev->guest_features = val; 1918 return bad ? -1 : 0; 1919 } 1920 1921 int virtio_set_features(VirtIODevice *vdev, uint64_t val) 1922 { 1923 /* 1924 * The driver must not attempt to set features after feature negotiation 1925 * has finished. 1926 */ 1927 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { 1928 return -EINVAL; 1929 } 1930 return virtio_set_features_nocheck(vdev, val); 1931 } 1932 1933 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id) 1934 { 1935 int i, ret; 1936 int32_t config_len; 1937 uint32_t num; 1938 uint32_t features; 1939 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 1940 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 1941 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); 1942 1943 /* 1944 * We poison the endianness to ensure it does not get used before 1945 * subsections have been loaded. 1946 */ 1947 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; 1948 1949 if (k->load_config) { 1950 ret = k->load_config(qbus->parent, f); 1951 if (ret) 1952 return ret; 1953 } 1954 1955 qemu_get_8s(f, &vdev->status); 1956 qemu_get_8s(f, &vdev->isr); 1957 qemu_get_be16s(f, &vdev->queue_sel); 1958 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { 1959 return -1; 1960 } 1961 qemu_get_be32s(f, &features); 1962 1963 /* 1964 * Temporarily set guest_features low bits - needed by 1965 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 1966 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ. 1967 * 1968 * Note: devices should always test host features in future - don't create 1969 * new dependencies like this. 1970 */ 1971 vdev->guest_features = features; 1972 1973 config_len = qemu_get_be32(f); 1974 1975 /* 1976 * There are cases where the incoming config can be bigger or smaller 1977 * than what we have; so load what we have space for, and skip 1978 * any excess that's in the stream. 1979 */ 1980 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); 1981 1982 while (config_len > vdev->config_len) { 1983 qemu_get_byte(f); 1984 config_len--; 1985 } 1986 1987 num = qemu_get_be32(f); 1988 1989 if (num > VIRTIO_QUEUE_MAX) { 1990 error_report("Invalid number of virtqueues: 0x%x", num); 1991 return -1; 1992 } 1993 1994 for (i = 0; i < num; i++) { 1995 vdev->vq[i].vring.num = qemu_get_be32(f); 1996 if (k->has_variable_vring_alignment) { 1997 vdev->vq[i].vring.align = qemu_get_be32(f); 1998 } 1999 vdev->vq[i].vring.desc = qemu_get_be64(f); 2000 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); 2001 vdev->vq[i].signalled_used_valid = false; 2002 vdev->vq[i].notification = true; 2003 2004 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { 2005 error_report("VQ %d address 0x0 " 2006 "inconsistent with Host index 0x%x", 2007 i, vdev->vq[i].last_avail_idx); 2008 return -1; 2009 } 2010 if (k->load_queue) { 2011 ret = k->load_queue(qbus->parent, i, f); 2012 if (ret) 2013 return ret; 2014 } 2015 } 2016 2017 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); 2018 2019 if (vdc->load != NULL) { 2020 ret = vdc->load(vdev, f, version_id); 2021 if (ret) { 2022 return ret; 2023 } 2024 } 2025 2026 if (vdc->vmsd) { 2027 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id); 2028 if (ret) { 2029 return ret; 2030 } 2031 } 2032 2033 /* Subsections */ 2034 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1); 2035 if (ret) { 2036 return ret; 2037 } 2038 2039 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { 2040 vdev->device_endian = virtio_default_endian(); 2041 } 2042 2043 if (virtio_64bit_features_needed(vdev)) { 2044 /* 2045 * Subsection load filled vdev->guest_features. Run them 2046 * through virtio_set_features to sanity-check them against 2047 * host_features. 2048 */ 2049 uint64_t features64 = vdev->guest_features; 2050 if (virtio_set_features_nocheck(vdev, features64) < 0) { 2051 error_report("Features 0x%" PRIx64 " unsupported. " 2052 "Allowed features: 0x%" PRIx64, 2053 features64, vdev->host_features); 2054 return -1; 2055 } 2056 } else { 2057 if (virtio_set_features_nocheck(vdev, features) < 0) { 2058 error_report("Features 0x%x unsupported. " 2059 "Allowed features: 0x%" PRIx64, 2060 features, vdev->host_features); 2061 return -1; 2062 } 2063 } 2064 2065 rcu_read_lock(); 2066 for (i = 0; i < num; i++) { 2067 if (vdev->vq[i].vring.desc) { 2068 uint16_t nheads; 2069 2070 /* 2071 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so 2072 * only the region cache needs to be set up. Legacy devices need 2073 * to calculate used and avail ring addresses based on the desc 2074 * address. 2075 */ 2076 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2077 virtio_init_region_cache(vdev, i); 2078 } else { 2079 virtio_queue_update_rings(vdev, i); 2080 } 2081 2082 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; 2083 /* Check it isn't doing strange things with descriptor numbers. */ 2084 if (nheads > vdev->vq[i].vring.num) { 2085 error_report("VQ %d size 0x%x Guest index 0x%x " 2086 "inconsistent with Host index 0x%x: delta 0x%x", 2087 i, vdev->vq[i].vring.num, 2088 vring_avail_idx(&vdev->vq[i]), 2089 vdev->vq[i].last_avail_idx, nheads); 2090 return -1; 2091 } 2092 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); 2093 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); 2094 2095 /* 2096 * Some devices migrate VirtQueueElements that have been popped 2097 * from the avail ring but not yet returned to the used ring. 2098 * Since max ring size < UINT16_MAX it's safe to use modulo 2099 * UINT16_MAX + 1 subtraction. 2100 */ 2101 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - 2102 vdev->vq[i].used_idx); 2103 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { 2104 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " 2105 "used_idx 0x%x", 2106 i, vdev->vq[i].vring.num, 2107 vdev->vq[i].last_avail_idx, 2108 vdev->vq[i].used_idx); 2109 return -1; 2110 } 2111 } 2112 } 2113 rcu_read_unlock(); 2114 2115 return 0; 2116 } 2117 2118 void virtio_cleanup(VirtIODevice *vdev) 2119 { 2120 qemu_del_vm_change_state_handler(vdev->vmstate); 2121 } 2122 2123 static void virtio_vmstate_change(void *opaque, int running, RunState state) 2124 { 2125 VirtIODevice *vdev = opaque; 2126 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2127 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2128 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK); 2129 vdev->vm_running = running; 2130 2131 if (backend_run) { 2132 virtio_set_status(vdev, vdev->status); 2133 } 2134 2135 if (k->vmstate_change) { 2136 k->vmstate_change(qbus->parent, backend_run); 2137 } 2138 2139 if (!backend_run) { 2140 virtio_set_status(vdev, vdev->status); 2141 } 2142 } 2143 2144 void virtio_instance_init_common(Object *proxy_obj, void *data, 2145 size_t vdev_size, const char *vdev_name) 2146 { 2147 DeviceState *vdev = data; 2148 2149 object_initialize(vdev, vdev_size, vdev_name); 2150 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL); 2151 object_unref(OBJECT(vdev)); 2152 qdev_alias_all_properties(vdev, proxy_obj); 2153 } 2154 2155 void virtio_init(VirtIODevice *vdev, const char *name, 2156 uint16_t device_id, size_t config_size) 2157 { 2158 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2159 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 2160 int i; 2161 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; 2162 2163 if (nvectors) { 2164 vdev->vector_queues = 2165 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); 2166 } 2167 2168 vdev->device_id = device_id; 2169 vdev->status = 0; 2170 atomic_set(&vdev->isr, 0); 2171 vdev->queue_sel = 0; 2172 vdev->config_vector = VIRTIO_NO_VECTOR; 2173 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX); 2174 vdev->vm_running = runstate_is_running(); 2175 vdev->broken = false; 2176 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2177 vdev->vq[i].vector = VIRTIO_NO_VECTOR; 2178 vdev->vq[i].vdev = vdev; 2179 vdev->vq[i].queue_index = i; 2180 } 2181 2182 vdev->name = name; 2183 vdev->config_len = config_size; 2184 if (vdev->config_len) { 2185 vdev->config = g_malloc0(config_size); 2186 } else { 2187 vdev->config = NULL; 2188 } 2189 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change, 2190 vdev); 2191 vdev->device_endian = virtio_default_endian(); 2192 vdev->use_guest_notifier_mask = true; 2193 } 2194 2195 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) 2196 { 2197 return vdev->vq[n].vring.desc; 2198 } 2199 2200 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) 2201 { 2202 return vdev->vq[n].vring.avail; 2203 } 2204 2205 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n) 2206 { 2207 return vdev->vq[n].vring.used; 2208 } 2209 2210 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) 2211 { 2212 return sizeof(VRingDesc) * vdev->vq[n].vring.num; 2213 } 2214 2215 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n) 2216 { 2217 return offsetof(VRingAvail, ring) + 2218 sizeof(uint16_t) * vdev->vq[n].vring.num; 2219 } 2220 2221 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n) 2222 { 2223 return offsetof(VRingUsed, ring) + 2224 sizeof(VRingUsedElem) * vdev->vq[n].vring.num; 2225 } 2226 2227 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) 2228 { 2229 return vdev->vq[n].last_avail_idx; 2230 } 2231 2232 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx) 2233 { 2234 vdev->vq[n].last_avail_idx = idx; 2235 vdev->vq[n].shadow_avail_idx = idx; 2236 } 2237 2238 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n) 2239 { 2240 rcu_read_lock(); 2241 if (vdev->vq[n].vring.desc) { 2242 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); 2243 } 2244 rcu_read_unlock(); 2245 } 2246 2247 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n) 2248 { 2249 vdev->vq[n].signalled_used_valid = false; 2250 } 2251 2252 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) 2253 { 2254 return vdev->vq + n; 2255 } 2256 2257 uint16_t virtio_get_queue_index(VirtQueue *vq) 2258 { 2259 return vq->queue_index; 2260 } 2261 2262 static void virtio_queue_guest_notifier_read(EventNotifier *n) 2263 { 2264 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier); 2265 if (event_notifier_test_and_clear(n)) { 2266 virtio_irq(vq); 2267 } 2268 } 2269 2270 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, 2271 bool with_irqfd) 2272 { 2273 if (assign && !with_irqfd) { 2274 event_notifier_set_handler(&vq->guest_notifier, 2275 virtio_queue_guest_notifier_read); 2276 } else { 2277 event_notifier_set_handler(&vq->guest_notifier, NULL); 2278 } 2279 if (!assign) { 2280 /* Test and clear notifier before closing it, 2281 * in case poll callback didn't have time to run. */ 2282 virtio_queue_guest_notifier_read(&vq->guest_notifier); 2283 } 2284 } 2285 2286 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) 2287 { 2288 return &vq->guest_notifier; 2289 } 2290 2291 static void virtio_queue_host_notifier_aio_read(EventNotifier *n) 2292 { 2293 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2294 if (event_notifier_test_and_clear(n)) { 2295 virtio_queue_notify_aio_vq(vq); 2296 } 2297 } 2298 2299 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n) 2300 { 2301 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2302 2303 virtio_queue_set_notification(vq, 0); 2304 } 2305 2306 static bool virtio_queue_host_notifier_aio_poll(void *opaque) 2307 { 2308 EventNotifier *n = opaque; 2309 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2310 bool progress; 2311 2312 if (!vq->vring.desc || virtio_queue_empty(vq)) { 2313 return false; 2314 } 2315 2316 progress = virtio_queue_notify_aio_vq(vq); 2317 2318 /* In case the handler function re-enabled notifications */ 2319 virtio_queue_set_notification(vq, 0); 2320 return progress; 2321 } 2322 2323 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) 2324 { 2325 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2326 2327 /* Caller polls once more after this to catch requests that race with us */ 2328 virtio_queue_set_notification(vq, 1); 2329 } 2330 2331 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx, 2332 VirtIOHandleAIOOutput handle_output) 2333 { 2334 if (handle_output) { 2335 vq->handle_aio_output = handle_output; 2336 aio_set_event_notifier(ctx, &vq->host_notifier, true, 2337 virtio_queue_host_notifier_aio_read, 2338 virtio_queue_host_notifier_aio_poll); 2339 aio_set_event_notifier_poll(ctx, &vq->host_notifier, 2340 virtio_queue_host_notifier_aio_poll_begin, 2341 virtio_queue_host_notifier_aio_poll_end); 2342 } else { 2343 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL); 2344 /* Test and clear notifier before after disabling event, 2345 * in case poll callback didn't have time to run. */ 2346 virtio_queue_host_notifier_aio_read(&vq->host_notifier); 2347 vq->handle_aio_output = NULL; 2348 } 2349 } 2350 2351 void virtio_queue_host_notifier_read(EventNotifier *n) 2352 { 2353 VirtQueue *vq = container_of(n, VirtQueue, host_notifier); 2354 if (event_notifier_test_and_clear(n)) { 2355 virtio_queue_notify_vq(vq); 2356 } 2357 } 2358 2359 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) 2360 { 2361 return &vq->host_notifier; 2362 } 2363 2364 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name) 2365 { 2366 g_free(vdev->bus_name); 2367 vdev->bus_name = g_strdup(bus_name); 2368 } 2369 2370 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...) 2371 { 2372 va_list ap; 2373 2374 va_start(ap, fmt); 2375 error_vreport(fmt, ap); 2376 va_end(ap); 2377 2378 vdev->broken = true; 2379 2380 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { 2381 virtio_set_status(vdev, vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET); 2382 virtio_notify_config(vdev); 2383 } 2384 } 2385 2386 static void virtio_memory_listener_commit(MemoryListener *listener) 2387 { 2388 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener); 2389 int i; 2390 2391 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2392 if (vdev->vq[i].vring.num == 0) { 2393 break; 2394 } 2395 virtio_init_region_cache(vdev, i); 2396 } 2397 } 2398 2399 static void virtio_device_realize(DeviceState *dev, Error **errp) 2400 { 2401 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 2402 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 2403 Error *err = NULL; 2404 2405 /* Devices should either use vmsd or the load/save methods */ 2406 assert(!vdc->vmsd || !vdc->load); 2407 2408 if (vdc->realize != NULL) { 2409 vdc->realize(dev, &err); 2410 if (err != NULL) { 2411 error_propagate(errp, err); 2412 return; 2413 } 2414 } 2415 2416 virtio_bus_device_plugged(vdev, &err); 2417 if (err != NULL) { 2418 error_propagate(errp, err); 2419 return; 2420 } 2421 2422 vdev->listener.commit = virtio_memory_listener_commit; 2423 memory_listener_register(&vdev->listener, vdev->dma_as); 2424 } 2425 2426 static void virtio_device_unrealize(DeviceState *dev, Error **errp) 2427 { 2428 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 2429 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 2430 Error *err = NULL; 2431 2432 virtio_bus_device_unplugged(vdev); 2433 2434 if (vdc->unrealize != NULL) { 2435 vdc->unrealize(dev, &err); 2436 if (err != NULL) { 2437 error_propagate(errp, err); 2438 return; 2439 } 2440 } 2441 2442 g_free(vdev->bus_name); 2443 vdev->bus_name = NULL; 2444 } 2445 2446 static void virtio_device_free_virtqueues(VirtIODevice *vdev) 2447 { 2448 int i; 2449 if (!vdev->vq) { 2450 return; 2451 } 2452 2453 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 2454 VRingMemoryRegionCaches *caches; 2455 if (vdev->vq[i].vring.num == 0) { 2456 break; 2457 } 2458 caches = atomic_read(&vdev->vq[i].vring.caches); 2459 atomic_set(&vdev->vq[i].vring.caches, NULL); 2460 virtio_free_region_cache(caches); 2461 } 2462 g_free(vdev->vq); 2463 } 2464 2465 static void virtio_device_instance_finalize(Object *obj) 2466 { 2467 VirtIODevice *vdev = VIRTIO_DEVICE(obj); 2468 2469 memory_listener_unregister(&vdev->listener); 2470 virtio_device_free_virtqueues(vdev); 2471 2472 g_free(vdev->config); 2473 g_free(vdev->vector_queues); 2474 } 2475 2476 static Property virtio_properties[] = { 2477 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), 2478 DEFINE_PROP_END_OF_LIST(), 2479 }; 2480 2481 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev) 2482 { 2483 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 2484 int n, r, err; 2485 2486 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 2487 VirtQueue *vq = &vdev->vq[n]; 2488 if (!virtio_queue_get_num(vdev, n)) { 2489 continue; 2490 } 2491 r = virtio_bus_set_host_notifier(qbus, n, true); 2492 if (r < 0) { 2493 err = r; 2494 goto assign_error; 2495 } 2496 event_notifier_set_handler(&vq->host_notifier, 2497 virtio_queue_host_notifier_read); 2498 } 2499 2500 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 2501 /* Kick right away to begin processing requests already in vring */ 2502 VirtQueue *vq = &vdev->vq[n]; 2503 if (!vq->vring.num) { 2504 continue; 2505 } 2506 event_notifier_set(&vq->host_notifier); 2507 } 2508 return 0; 2509 2510 assign_error: 2511 while (--n >= 0) { 2512 VirtQueue *vq = &vdev->vq[n]; 2513 if (!virtio_queue_get_num(vdev, n)) { 2514 continue; 2515 } 2516 2517 event_notifier_set_handler(&vq->host_notifier, NULL); 2518 r = virtio_bus_set_host_notifier(qbus, n, false); 2519 assert(r >= 0); 2520 } 2521 return err; 2522 } 2523 2524 int virtio_device_start_ioeventfd(VirtIODevice *vdev) 2525 { 2526 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2527 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2528 2529 return virtio_bus_start_ioeventfd(vbus); 2530 } 2531 2532 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev) 2533 { 2534 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev))); 2535 int n, r; 2536 2537 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) { 2538 VirtQueue *vq = &vdev->vq[n]; 2539 2540 if (!virtio_queue_get_num(vdev, n)) { 2541 continue; 2542 } 2543 event_notifier_set_handler(&vq->host_notifier, NULL); 2544 r = virtio_bus_set_host_notifier(qbus, n, false); 2545 assert(r >= 0); 2546 } 2547 } 2548 2549 void virtio_device_stop_ioeventfd(VirtIODevice *vdev) 2550 { 2551 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2552 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2553 2554 virtio_bus_stop_ioeventfd(vbus); 2555 } 2556 2557 int virtio_device_grab_ioeventfd(VirtIODevice *vdev) 2558 { 2559 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2560 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2561 2562 return virtio_bus_grab_ioeventfd(vbus); 2563 } 2564 2565 void virtio_device_release_ioeventfd(VirtIODevice *vdev) 2566 { 2567 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2568 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2569 2570 virtio_bus_release_ioeventfd(vbus); 2571 } 2572 2573 static void virtio_device_class_init(ObjectClass *klass, void *data) 2574 { 2575 /* Set the default value here. */ 2576 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 2577 DeviceClass *dc = DEVICE_CLASS(klass); 2578 2579 dc->realize = virtio_device_realize; 2580 dc->unrealize = virtio_device_unrealize; 2581 dc->bus_type = TYPE_VIRTIO_BUS; 2582 dc->props = virtio_properties; 2583 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; 2584 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; 2585 2586 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; 2587 } 2588 2589 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev) 2590 { 2591 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev)); 2592 VirtioBusState *vbus = VIRTIO_BUS(qbus); 2593 2594 return virtio_bus_ioeventfd_enabled(vbus); 2595 } 2596 2597 static const TypeInfo virtio_device_info = { 2598 .name = TYPE_VIRTIO_DEVICE, 2599 .parent = TYPE_DEVICE, 2600 .instance_size = sizeof(VirtIODevice), 2601 .class_init = virtio_device_class_init, 2602 .instance_finalize = virtio_device_instance_finalize, 2603 .abstract = true, 2604 .class_size = sizeof(VirtioDeviceClass), 2605 }; 2606 2607 static void virtio_register_types(void) 2608 { 2609 type_register_static(&virtio_device_info); 2610 } 2611 2612 type_init(virtio_register_types) 2613