120 struct VirtQueue  struct
155     QLIST_ENTRY(VirtQueue) node;  argument
227 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)  in virtio_virtqueue_reset_region_cache()
240     VirtQueue *vq = &vdev->vq[n];  in virtio_init_region_cache()
357 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)  in vring_get_region_caches()
363 static inline uint16_t vring_avail_flags(VirtQueue *vq)  in vring_avail_flags()
376 static inline uint16_t vring_avail_idx(VirtQueue *vq)  in vring_avail_idx()
390 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)  in vring_avail_ring()
403 static inline uint16_t vring_get_used_event(VirtQueue *vq)  in vring_get_used_event()
409 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,  in vring_used_write()
426 static inline uint16_t vring_used_flags(VirtQueue *vq)  in vring_used_flags()
439 static uint16_t vring_used_idx(VirtQueue *vq)  in vring_used_idx()
452 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)  in vring_used_idx_set()
466 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)  in vring_used_flags_set_bit()
483 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)  in vring_used_flags_unset_bit()
500 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)  in vring_set_avail_event()
518 static void virtio_queue_split_set_notification(VirtQueue *vq, int enable)  in virtio_queue_split_set_notification()
535 static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable)  in virtio_queue_packed_set_notification()
568 bool virtio_queue_get_notification(VirtQueue *vq)  in virtio_queue_get_notification()
573 void virtio_queue_set_notification(VirtQueue *vq, int enable)  in virtio_queue_set_notification()
588 int virtio_queue_ready(VirtQueue *vq)  in virtio_queue_ready()
682 static int virtio_queue_empty_rcu(VirtQueue *vq)  in virtio_queue_empty_rcu()
699 static int virtio_queue_split_empty(VirtQueue *vq)  in virtio_queue_split_empty()
721 static int virtio_queue_packed_empty_rcu(VirtQueue *vq)  in virtio_queue_packed_empty_rcu()
741 static int virtio_queue_packed_empty(VirtQueue *vq)  in virtio_queue_packed_empty()
747 int virtio_queue_empty(VirtQueue *vq)  in virtio_queue_empty()
756 static bool virtio_queue_split_poll(VirtQueue *vq, unsigned shadow_idx)  in virtio_queue_split_poll()
765 static bool virtio_queue_packed_poll(VirtQueue *vq, unsigned shadow_idx)  in virtio_queue_packed_poll()
785 static bool virtio_queue_poll(VirtQueue *vq, unsigned shadow_idx)  in virtio_queue_poll()
798 bool virtio_queue_enable_notification_and_check(VirtQueue *vq,  in virtio_queue_enable_notification_and_check()
810 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_unmap_sg()
844 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_detach_element()
851 static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num)  in virtqueue_split_rewind()
856 static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num)  in virtqueue_packed_rewind()
874 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_unpop()
899 bool virtqueue_rewind(VirtQueue *vq, unsigned int num)  in virtqueue_rewind()
914 static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_split_fill()
930 static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_packed_fill()
938 static void virtqueue_ordered_fill(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_ordered_fill()
978 static void virtqueue_packed_fill_desc(VirtQueue *vq,  in virtqueue_packed_fill_desc()
1017 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_fill()
1038 static void virtqueue_split_flush(VirtQueue *vq, unsigned int count)  in virtqueue_split_flush()
1057 static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count)  in virtqueue_packed_flush()
1089 static void virtqueue_ordered_flush(VirtQueue *vq)  in virtqueue_ordered_flush()
1156 void virtqueue_flush(VirtQueue *vq, unsigned int count)  in virtqueue_flush()
1172 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,  in virtqueue_push()
1181 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)  in virtqueue_num_heads()
1213 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,  in virtqueue_get_head()
1256 static void virtqueue_split_get_avail_bytes(VirtQueue *vq,  in virtqueue_split_get_avail_bytes()
1365 static int virtqueue_packed_read_next_desc(VirtQueue *vq,  in virtqueue_packed_read_next_desc()
1392 static void virtqueue_packed_get_avail_bytes(VirtQueue *vq,  in virtqueue_packed_get_avail_bytes()
1508 int virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,  in virtqueue_get_avail_bytes()
1555 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,  in virtqueue_avail_bytes()
1682 static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)  in virtqueue_split_pop()
1829 static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz)  in virtqueue_packed_pop()
1979 void *virtqueue_pop(VirtQueue *vq, size_t sz)  in virtqueue_pop()
1992 static unsigned int virtqueue_packed_drop_all(VirtQueue *vq)  in virtqueue_packed_drop_all()
2045 static unsigned int virtqueue_split_drop_all(VirtQueue *vq)  in virtqueue_split_drop_all()
2079 unsigned int virtqueue_drop_all(VirtQueue *vq)  in virtqueue_drop_all()
2373 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)  in virtio_vector_first_queue()
2378 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)  in virtio_vector_next_queue()
2428 void virtio_queue_set_shadow_avail_idx(VirtQueue *vq, uint16_t shadow_avail_idx)  in virtio_queue_set_shadow_avail_idx()
2446 static void virtio_queue_notify_vq(VirtQueue *vq)  in virtio_queue_notify_vq()
2466     VirtQueue *vq = &vdev->vq[n];  in virtio_queue_notify()
2492     VirtQueue *vq = &vdev->vq[n];  in virtio_queue_set_vector()
2507 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,  in virtio_add_queue()
2529 void virtio_delete_queue(VirtQueue *vq)  in virtio_delete_queue()
2561 static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq)  in virtio_split_should_notify()
2584 static bool vring_packed_need_event(VirtQueue *vq, bool wrap,  in vring_packed_need_event()
2598 static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq)  in virtio_packed_should_notify()
2628 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)  in virtio_should_notify()
2641     VirtQueue *vq = container_of(notifier, VirtQueue, guest_notifier);  in virtio_notify_irqfd_deferred_fn()
2647 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)  in virtio_notify_irqfd()
2676 static void virtio_irq(VirtQueue *vq)  in virtio_irq()
2682 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)  in virtio_notify()
2786         VMSTATE_UINT64(vring.avail, struct VirtQueue),
2787         VMSTATE_UINT64(vring.used, struct VirtQueue),
2797         VMSTATE_UINT16(last_avail_idx, struct VirtQueue),
2798         VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue),
2799         VMSTATE_UINT16(used_idx, struct VirtQueue),
2800         VMSTATE_BOOL(used_wrap_counter, struct VirtQueue),
2801         VMSTATE_UINT32(inuse, struct VirtQueue),
2813                       VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
2825                       VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue),
2835         VMSTATE_UINT32(vring.num_default, struct VirtQueue),
2847                       VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
3493     vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX);  in virtio_init()
3639     struct VirtQueue *vq = &vdev->vq[n];  in virtio_queue_packed_set_last_avail_idx()
3718 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)  in virtio_get_queue()
3723 uint16_t virtio_get_queue_index(VirtQueue *vq)  in virtio_get_queue_index()
3730     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);  in virtio_queue_guest_notifier_read()
3743 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,  in virtio_queue_set_guest_notifier_fd_handler()
3776 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)  in virtio_queue_get_guest_notifier()
3783     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);  in virtio_queue_host_notifier_aio_poll_begin()
3791     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);  in virtio_queue_host_notifier_aio_poll()
3798     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);  in virtio_queue_host_notifier_aio_poll_ready()
3805     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);  in virtio_queue_host_notifier_aio_poll_end()
3811 void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx)  in virtio_queue_aio_attach_host_notifier()
3846 void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx)  in virtio_queue_aio_attach_host_notifier_no_poll()
3866 void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx)  in virtio_queue_aio_detach_host_notifier()
3884     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);  in virtio_queue_host_notifier_read()
3890 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)  in virtio_queue_get_host_notifier()
3900 void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled)  in virtio_queue_set_host_notifier_enabled()
4051         VirtQueue *vq = &vdev->vq[n];  in virtio_device_start_ioeventfd_impl()
4066         VirtQueue *vq = &vdev->vq[n];  in virtio_device_start_ioeventfd_impl()
4078         VirtQueue *vq = &vdev->vq[n];  in virtio_device_start_ioeventfd_impl()
4121         VirtQueue *vq = &vdev->vq[n];  in virtio_device_stop_ioeventfd_impl()
4281     VirtQueue *vq;  in qmp_x_query_virtio_queue_element()