Lines Matching +full:- +full:- +full:disable +full:- +full:vhost +full:- +full:vsock
10 * the COPYING file in the top-level directory.
16 #include "qapi/qapi-commands-virtio.h"
18 #include "qemu/defer-call.h"
19 #include "qemu/error-report.h"
21 #include "qemu/main-loop.h"
27 #include "hw/virtio/vhost.h"
28 #include "migration/qemu-file-types.h"
30 #include "hw/virtio/virtio-bus.h"
31 #include "hw/qdev-properties.h"
32 #include "hw/virtio/virtio-access.h"
35 #include "virtio-qmp.h"
37 #include "standard-headers/linux/virtio_ids.h"
38 #include "standard-headers/linux/vhost_types.h"
39 #include "standard-headers/linux/virtio_blk.h"
40 #include "standard-headers/linux/virtio_console.h"
41 #include "standard-headers/linux/virtio_gpu.h"
42 #include "standard-headers/linux/virtio_net.h"
43 #include "standard-headers/linux/virtio_scsi.h"
44 #include "standard-headers/linux/virtio_i2c.h"
45 #include "standard-headers/linux/virtio_balloon.h"
46 #include "standard-headers/linux/virtio_iommu.h"
47 #include "standard-headers/linux/virtio_mem.h"
48 #include "standard-headers/linux/virtio_vsock.h"
159 [VIRTIO_ID_NET] = "virtio-net",
160 [VIRTIO_ID_BLOCK] = "virtio-blk",
161 [VIRTIO_ID_CONSOLE] = "virtio-serial",
162 [VIRTIO_ID_RNG] = "virtio-rng",
163 [VIRTIO_ID_BALLOON] = "virtio-balloon",
164 [VIRTIO_ID_IOMEM] = "virtio-iomem",
165 [VIRTIO_ID_RPMSG] = "virtio-rpmsg",
166 [VIRTIO_ID_SCSI] = "virtio-scsi",
167 [VIRTIO_ID_9P] = "virtio-9p",
168 [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan",
169 [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial",
170 [VIRTIO_ID_CAIF] = "virtio-caif",
171 [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon",
172 [VIRTIO_ID_GPU] = "virtio-gpu",
173 [VIRTIO_ID_CLOCK] = "virtio-clk",
174 [VIRTIO_ID_INPUT] = "virtio-input",
175 [VIRTIO_ID_VSOCK] = "vhost-vsock",
176 [VIRTIO_ID_CRYPTO] = "virtio-crypto",
177 [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal",
178 [VIRTIO_ID_PSTORE] = "virtio-pstore",
179 [VIRTIO_ID_IOMMU] = "virtio-iommu",
180 [VIRTIO_ID_MEM] = "virtio-mem",
181 [VIRTIO_ID_SOUND] = "virtio-sound",
182 [VIRTIO_ID_FS] = "virtio-user-fs",
183 [VIRTIO_ID_PMEM] = "virtio-pmem",
184 [VIRTIO_ID_RPMB] = "virtio-rpmb",
185 [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim",
186 [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder",
187 [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder",
188 [VIRTIO_ID_SCMI] = "virtio-scmi",
189 [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod",
190 [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c",
191 [VIRTIO_ID_WATCHDOG] = "virtio-watchdog",
192 [VIRTIO_ID_CAN] = "virtio-can",
193 [VIRTIO_ID_DMABUF] = "virtio-dmabuf",
194 [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv",
195 [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol",
196 [VIRTIO_ID_BT] = "virtio-bluetooth",
197 [VIRTIO_ID_GPIO] = "virtio-gpio"
213 vdev->name); in virtio_check_indirect_feature()
221 address_space_cache_destroy(&caches->desc); in virtio_free_region_cache()
222 address_space_cache_destroy(&caches->avail); in virtio_free_region_cache()
223 address_space_cache_destroy(&caches->used); in virtio_free_region_cache()
231 caches = qatomic_read(&vq->vring.caches); in virtio_virtqueue_reset_region_cache()
232 qatomic_rcu_set(&vq->vring.caches, NULL); in virtio_virtqueue_reset_region_cache()
240 VirtQueue *vq = &vdev->vq[n]; in virtio_init_region_cache()
241 VRingMemoryRegionCaches *old = vq->vring.caches; in virtio_init_region_cache()
248 addr = vq->vring.desc; in virtio_init_region_cache()
254 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? in virtio_init_region_cache()
256 len = address_space_cache_init(&new->desc, vdev->dma_as, in virtio_init_region_cache()
264 len = address_space_cache_init(&new->used, vdev->dma_as, in virtio_init_region_cache()
265 vq->vring.used, size, true); in virtio_init_region_cache()
272 len = address_space_cache_init(&new->avail, vdev->dma_as, in virtio_init_region_cache()
273 vq->vring.avail, size, false); in virtio_init_region_cache()
279 qatomic_rcu_set(&vq->vring.caches, new); in virtio_init_region_cache()
286 address_space_cache_destroy(&new->avail); in virtio_init_region_cache()
288 address_space_cache_destroy(&new->used); in virtio_init_region_cache()
290 address_space_cache_destroy(&new->desc); in virtio_init_region_cache()
299 VRing *vring = &vdev->vq[n].vring; in virtio_queue_update_rings()
301 if (!vring->num || !vring->desc || !vring->align) { in virtio_queue_update_rings()
302 /* not yet setup -> nothing to do */ in virtio_queue_update_rings()
305 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); in virtio_queue_update_rings()
306 vring->used = vring_align(vring->avail + in virtio_queue_update_rings()
307 offsetof(VRingAvail, ring[vring->num]), in virtio_queue_update_rings()
308 vring->align); in virtio_queue_update_rings()
318 virtio_tswap64s(vdev, &desc->addr); in vring_split_desc_read()
319 virtio_tswap32s(vdev, &desc->len); in vring_split_desc_read()
320 virtio_tswap16s(vdev, &desc->flags); in vring_split_desc_read()
321 virtio_tswap16s(vdev, &desc->next); in vring_split_desc_read()
331 e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags); in vring_packed_event_read()
334 e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off); in vring_packed_event_read()
359 return qatomic_rcu_read(&vq->vring.caches); in vring_get_region_caches()
372 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); in vring_avail_flags()
385 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); in vring_avail_idx()
386 return vq->shadow_avail_idx; in vring_avail_idx()
399 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); in vring_avail_ring()
405 return vring_avail_ring(vq, vq->vring.num); in vring_get_used_event()
419 virtio_tswap32s(vq->vdev, &uelem->id); in vring_used_write()
420 virtio_tswap32s(vq->vdev, &uelem->len); in vring_used_write()
421 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); in vring_used_write()
422 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); in vring_used_write()
435 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_flags()
448 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_idx()
458 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); in vring_used_idx_set()
459 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); in vring_used_idx_set()
462 vq->used_idx = val; in vring_used_idx_set()
469 VirtIODevice *vdev = vq->vdev; in vring_used_flags_set_bit()
477 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_flags_set_bit()
478 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); in vring_used_flags_set_bit()
479 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); in vring_used_flags_set_bit()
486 VirtIODevice *vdev = vq->vdev; in vring_used_flags_unset_bit()
494 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_flags_unset_bit()
495 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); in vring_used_flags_unset_bit()
496 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); in vring_used_flags_unset_bit()
504 if (!vq->notification) { in vring_set_avail_event()
513 pa = offsetof(VRingUsed, ring[vq->vring.num]); in vring_set_avail_event()
514 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); in vring_set_avail_event()
515 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); in vring_set_avail_event()
522 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in virtio_queue_split_set_notification()
547 vring_packed_event_read(vq->vdev, &caches->used, &e); in virtio_queue_packed_set_notification()
551 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in virtio_queue_packed_set_notification()
552 off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15; in virtio_queue_packed_set_notification()
553 vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap); in virtio_queue_packed_set_notification()
561 vring_packed_flags_write(vq->vdev, &caches->used, e.flags); in virtio_queue_packed_set_notification()
570 return vq->notification; in virtio_queue_get_notification()
575 vq->notification = enable; in virtio_queue_set_notification()
577 if (!vq->vring.desc) { in virtio_queue_set_notification()
581 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_set_notification()
590 return vq->vring.avail != 0; in virtio_queue_ready()
610 vring_packed_desc_read_flags(vdev, &desc->flags, cache, i); in vring_packed_desc_read()
618 &desc->addr, sizeof(desc->addr)); in vring_packed_desc_read()
620 &desc->id, sizeof(desc->id)); in vring_packed_desc_read()
622 &desc->len, sizeof(desc->len)); in vring_packed_desc_read()
623 virtio_tswap64s(vdev, &desc->addr); in vring_packed_desc_read()
624 virtio_tswap16s(vdev, &desc->id); in vring_packed_desc_read()
625 virtio_tswap32s(vdev, &desc->len); in vring_packed_desc_read()
638 virtio_tswap32s(vdev, &desc->len); in vring_packed_desc_write_data()
639 virtio_tswap16s(vdev, &desc->id); in vring_packed_desc_write_data()
640 address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id)); in vring_packed_desc_write_data()
641 address_space_cache_invalidate(cache, off_id, sizeof(desc->id)); in vring_packed_desc_write_data()
642 address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len)); in vring_packed_desc_write_data()
643 address_space_cache_invalidate(cache, off_len, sizeof(desc->len)); in vring_packed_desc_write_data()
653 virtio_stw_phys_cached(vdev, cache, off, desc->flags); in vring_packed_desc_write_flags()
654 address_space_cache_invalidate(cache, off, sizeof(desc->flags)); in vring_packed_desc_write_flags()
684 if (virtio_device_disabled(vq->vdev)) { in virtio_queue_empty_rcu()
688 if (unlikely(!vq->vring.avail)) { in virtio_queue_empty_rcu()
692 if (vq->shadow_avail_idx != vq->last_avail_idx) { in virtio_queue_empty_rcu()
696 return vring_avail_idx(vq) == vq->last_avail_idx; in virtio_queue_empty_rcu()
703 if (virtio_device_disabled(vq->vdev)) { in virtio_queue_split_empty()
707 if (unlikely(!vq->vring.avail)) { in virtio_queue_split_empty()
711 if (vq->shadow_avail_idx != vq->last_avail_idx) { in virtio_queue_split_empty()
716 empty = vring_avail_idx(vq) == vq->last_avail_idx; in virtio_queue_split_empty()
726 if (unlikely(!vq->vring.desc)) { in virtio_queue_packed_empty_rcu()
735 vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc, in virtio_queue_packed_empty_rcu()
736 vq->last_avail_idx); in virtio_queue_packed_empty_rcu()
738 return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter); in virtio_queue_packed_empty_rcu()
749 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_empty()
758 if (unlikely(!vq->vring.avail)) { in virtio_queue_split_poll()
770 if (unlikely(!vq->vring.desc)) { in virtio_queue_packed_poll()
779 vring_packed_desc_read(vq->vdev, &desc, &caches->desc, in virtio_queue_packed_poll()
782 return is_desc_avail(desc.flags, vq->shadow_avail_wrap_counter); in virtio_queue_packed_poll()
787 if (virtio_device_disabled(vq->vdev)) { in virtio_queue_poll()
791 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_poll()
813 AddressSpace *dma_as = vq->vdev->dma_as; in virtqueue_unmap_sg()
818 for (i = 0; i < elem->in_num; i++) { in virtqueue_unmap_sg()
819 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); in virtqueue_unmap_sg()
821 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, in virtqueue_unmap_sg()
822 elem->in_sg[i].iov_len, in virtqueue_unmap_sg()
828 for (i = 0; i < elem->out_num; i++) in virtqueue_unmap_sg()
829 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, in virtqueue_unmap_sg()
830 elem->out_sg[i].iov_len, in virtqueue_unmap_sg()
832 elem->out_sg[i].iov_len); in virtqueue_unmap_sg()
847 vq->inuse -= elem->ndescs; in virtqueue_detach_element()
853 vq->last_avail_idx -= num; in virtqueue_split_rewind()
858 if (vq->last_avail_idx < num) { in virtqueue_packed_rewind()
859 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num; in virtqueue_packed_rewind()
860 vq->last_avail_wrap_counter ^= 1; in virtqueue_packed_rewind()
862 vq->last_avail_idx -= num; in virtqueue_packed_rewind()
878 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_unpop()
901 if (num > vq->inuse) { in virtqueue_rewind()
905 vq->inuse -= num; in virtqueue_rewind()
906 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_rewind()
919 if (unlikely(!vq->vring.used)) { in virtqueue_split_fill()
923 idx = (idx + vq->used_idx) % vq->vring.num; in virtqueue_split_fill()
925 uelem.id = elem->index; in virtqueue_split_fill()
933 vq->used_elems[idx].index = elem->index; in virtqueue_packed_fill()
934 vq->used_elems[idx].len = len; in virtqueue_packed_fill()
935 vq->used_elems[idx].ndescs = elem->ndescs; in virtqueue_packed_fill()
943 i = vq->used_idx % vq->vring.num; in virtqueue_ordered_fill()
949 max_steps = (vq->last_avail_idx - vq->used_idx) % vq->vring.num; in virtqueue_ordered_fill()
951 /* Search for element in vq->used_elems */ in virtqueue_ordered_fill()
954 if (vq->used_elems[i].index == elem->index) { in virtqueue_ordered_fill()
955 vq->used_elems[i].len = len; in virtqueue_ordered_fill()
956 vq->used_elems[i].in_order_filled = true; in virtqueue_ordered_fill()
960 i += vq->used_elems[i].ndescs; in virtqueue_ordered_fill()
961 steps += vq->used_elems[i].ndescs; in virtqueue_ordered_fill()
963 if (i >= vq->vring.num) { in virtqueue_ordered_fill()
964 i -= vq->vring.num; in virtqueue_ordered_fill()
974 __func__, vq->vdev->name, elem->index); in virtqueue_ordered_fill()
986 .id = elem->index, in virtqueue_packed_fill_desc()
987 .len = elem->len, in virtqueue_packed_fill_desc()
989 bool wrap_counter = vq->used_wrap_counter; in virtqueue_packed_fill_desc()
991 if (unlikely(!vq->vring.desc)) { in virtqueue_packed_fill_desc()
995 head = vq->used_idx + idx; in virtqueue_packed_fill_desc()
996 if (head >= vq->vring.num) { in virtqueue_packed_fill_desc()
997 head -= vq->vring.num; in virtqueue_packed_fill_desc()
1013 vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order); in virtqueue_packed_fill_desc()
1024 if (virtio_device_disabled(vq->vdev)) { in virtqueue_fill()
1028 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) { in virtqueue_fill()
1030 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_fill()
1042 if (unlikely(!vq->vring.used)) { in virtqueue_split_flush()
1049 old = vq->used_idx; in virtqueue_split_flush()
1052 vq->inuse -= count; in virtqueue_split_flush()
1053 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) in virtqueue_split_flush()
1054 vq->signalled_used_valid = false; in virtqueue_split_flush()
1061 if (unlikely(!vq->vring.desc)) { in virtqueue_packed_flush()
1071 * the value of 'vq->used_idx' plus the 'ndescs'. in virtqueue_packed_flush()
1073 ndescs += vq->used_elems[0].ndescs; in virtqueue_packed_flush()
1075 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false); in virtqueue_packed_flush()
1076 ndescs += vq->used_elems[i].ndescs; in virtqueue_packed_flush()
1078 virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true); in virtqueue_packed_flush()
1080 vq->inuse -= ndescs; in virtqueue_packed_flush()
1081 vq->used_idx += ndescs; in virtqueue_packed_flush()
1082 if (vq->used_idx >= vq->vring.num) { in virtqueue_packed_flush()
1083 vq->used_idx -= vq->vring.num; in virtqueue_packed_flush()
1084 vq->used_wrap_counter ^= 1; in virtqueue_packed_flush()
1085 vq->signalled_used_valid = false; in virtqueue_packed_flush()
1091 unsigned int i = vq->used_idx % vq->vring.num; in virtqueue_ordered_flush()
1093 uint16_t old = vq->used_idx; in virtqueue_ordered_flush()
1098 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED); in virtqueue_ordered_flush()
1101 if (unlikely(!vq->vring.desc)) { in virtqueue_ordered_flush()
1104 } else if (unlikely(!vq->vring.used)) { in virtqueue_ordered_flush()
1108 /* First expected in-order element isn't ready, nothing to do */ in virtqueue_ordered_flush()
1109 if (!vq->used_elems[i].in_order_filled) { in virtqueue_ordered_flush()
1113 /* Search for filled elements in-order */ in virtqueue_ordered_flush()
1114 while (vq->used_elems[i].in_order_filled) { in virtqueue_ordered_flush()
1119 if (packed && i != vq->used_idx) { in virtqueue_ordered_flush()
1120 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false); in virtqueue_ordered_flush()
1122 uelem.id = vq->used_elems[i].index; in virtqueue_ordered_flush()
1123 uelem.len = vq->used_elems[i].len; in virtqueue_ordered_flush()
1127 vq->used_elems[i].in_order_filled = false; in virtqueue_ordered_flush()
1128 ndescs += vq->used_elems[i].ndescs; in virtqueue_ordered_flush()
1129 i += vq->used_elems[i].ndescs; in virtqueue_ordered_flush()
1130 if (i >= vq->vring.num) { in virtqueue_ordered_flush()
1131 i -= vq->vring.num; in virtqueue_ordered_flush()
1136 virtqueue_packed_fill_desc(vq, &vq->used_elems[vq->used_idx], 0, true); in virtqueue_ordered_flush()
1137 vq->used_idx += ndescs; in virtqueue_ordered_flush()
1138 if (vq->used_idx >= vq->vring.num) { in virtqueue_ordered_flush()
1139 vq->used_idx -= vq->vring.num; in virtqueue_ordered_flush()
1140 vq->used_wrap_counter ^= 1; in virtqueue_ordered_flush()
1141 vq->signalled_used_valid = false; in virtqueue_ordered_flush()
1148 if (unlikely((int16_t)(new - vq->signalled_used) < in virtqueue_ordered_flush()
1149 (uint16_t)(new - old))) { in virtqueue_ordered_flush()
1150 vq->signalled_used_valid = false; in virtqueue_ordered_flush()
1153 vq->inuse -= ndescs; in virtqueue_ordered_flush()
1158 if (virtio_device_disabled(vq->vdev)) { in virtqueue_flush()
1159 vq->inuse -= count; in virtqueue_flush()
1163 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) { in virtqueue_flush()
1165 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_flush()
1186 avail_idx = (vq->shadow_avail_idx != idx) ? vq->shadow_avail_idx in virtqueue_num_heads()
1188 num_heads = avail_idx - idx; in virtqueue_num_heads()
1191 if (num_heads > vq->vring.num) { in virtqueue_num_heads()
1192 virtio_error(vq->vdev, "Guest moved used index from %u to %u", in virtqueue_num_heads()
1193 idx, vq->shadow_avail_idx); in virtqueue_num_heads()
1194 return -EINVAL; in virtqueue_num_heads()
1197 * On success, callers read a descriptor at vq->last_avail_idx. in virtqueue_num_heads()
1218 *head = vring_avail_ring(vq, idx % vq->vring.num); in virtqueue_get_head()
1221 if (*head >= vq->vring.num) { in virtqueue_get_head()
1222 virtio_error(vq->vdev, "Guest says index %u is available", *head); in virtqueue_get_head()
1230 VIRTQUEUE_READ_DESC_ERROR = -1,
1235 /* Reads the 'desc->next' descriptor into '*desc'. */
1241 if (!(desc->flags & VRING_DESC_F_NEXT)) { in virtqueue_split_read_next_desc()
1246 if (desc->next >= max) { in virtqueue_split_read_next_desc()
1247 virtio_error(vdev, "Desc next is %u", desc->next); in virtqueue_split_read_next_desc()
1251 vring_split_desc_read(vdev, desc, desc_cache, desc->next); in virtqueue_split_read_next_desc()
1261 VirtIODevice *vdev = vq->vdev; in virtqueue_split_get_avail_bytes()
1270 idx = vq->last_avail_idx; in virtqueue_split_get_avail_bytes()
1274 MemoryRegionCache *desc_cache = &caches->desc; in virtqueue_split_get_avail_bytes()
1278 unsigned int max = vq->vring.num; in virtqueue_split_get_avail_bytes()
1302 vdev->dma_as, in virtqueue_split_get_avail_bytes()
1374 if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) { in virtqueue_packed_read_next_desc()
1383 (*next) -= vq->vring.num; in virtqueue_packed_read_next_desc()
1387 vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false); in virtqueue_packed_read_next_desc()
1399 VirtIODevice *vdev = vq->vdev; in virtqueue_packed_get_avail_bytes()
1410 idx = vq->last_avail_idx; in virtqueue_packed_get_avail_bytes()
1411 wrap_counter = vq->last_avail_wrap_counter; in virtqueue_packed_get_avail_bytes()
1418 unsigned int max = vq->vring.num; in virtqueue_packed_get_avail_bytes()
1420 desc_cache = &caches->desc; in virtqueue_packed_get_avail_bytes()
1441 vdev->dma_as, in virtqueue_packed_get_avail_bytes()
1480 idx += num_bufs - total_bufs; in virtqueue_packed_get_avail_bytes()
1484 if (idx >= vq->vring.num) { in virtqueue_packed_get_avail_bytes()
1485 idx -= vq->vring.num; in virtqueue_packed_get_avail_bytes()
1491 vq->shadow_avail_idx = idx; in virtqueue_packed_get_avail_bytes()
1492 vq->shadow_avail_wrap_counter = wrap_counter; in virtqueue_packed_get_avail_bytes()
1517 if (unlikely(!vq->vring.desc)) { in virtqueue_get_avail_bytes()
1526 desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? in virtqueue_get_avail_bytes()
1528 if (caches->desc.len < vq->vring.num * desc_size) { in virtqueue_get_avail_bytes()
1529 virtio_error(vq->vdev, "Cannot map descriptor ring"); in virtqueue_get_avail_bytes()
1533 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_get_avail_bytes()
1543 return (int)vq->shadow_avail_idx; in virtqueue_get_avail_bytes()
1552 return -1; in virtqueue_get_avail_bytes()
1587 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, in virtqueue_map_desc()
1600 sz -= len; in virtqueue_map_desc()
1623 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); in virtqueue_undo_map_desc()
1637 sg[i].iov_base = dma_memory_map(vdev->dma_as, in virtqueue_map_iovec()
1655 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true); in virtqueue_map()
1656 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num, in virtqueue_map()
1663 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); in virtqueue_alloc_element()
1664 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); in virtqueue_alloc_element()
1665 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); in virtqueue_alloc_element()
1666 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); in virtqueue_alloc_element()
1667 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); in virtqueue_alloc_element()
1668 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); in virtqueue_alloc_element()
1673 elem->out_num = out_num; in virtqueue_alloc_element()
1674 elem->in_num = in_num; in virtqueue_alloc_element()
1675 elem->in_addr = (void *)elem + in_addr_ofs; in virtqueue_alloc_element()
1676 elem->out_addr = (void *)elem + out_addr_ofs; in virtqueue_alloc_element()
1677 elem->in_sg = (void *)elem + in_sg_ofs; in virtqueue_alloc_element()
1678 elem->out_sg = (void *)elem + out_sg_ofs; in virtqueue_alloc_element()
1689 VirtIODevice *vdev = vq->vdev; in virtqueue_split_pop()
1710 max = vq->vring.num; in virtqueue_split_pop()
1712 if (vq->inuse >= vq->vring.num) { in virtqueue_split_pop()
1717 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { in virtqueue_split_pop()
1722 vring_set_avail_event(vq, vq->last_avail_idx); in virtqueue_split_pop()
1733 if (caches->desc.len < max * sizeof(VRingDesc)) { in virtqueue_split_pop()
1738 desc_cache = &caches->desc; in virtqueue_split_pop()
1748 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, in virtqueue_split_pop()
1768 VIRTQUEUE_MAX_SIZE - out_num, true, in virtqueue_split_pop()
1798 elem->index = head; in virtqueue_split_pop()
1799 elem->ndescs = 1; in virtqueue_split_pop()
1801 elem->out_addr[i] = addr[i]; in virtqueue_split_pop()
1802 elem->out_sg[i] = iov[i]; in virtqueue_split_pop()
1805 elem->in_addr[i] = addr[out_num + i]; in virtqueue_split_pop()
1806 elem->in_sg[i] = iov[out_num + i]; in virtqueue_split_pop()
1810 idx = (vq->last_avail_idx - 1) % vq->vring.num; in virtqueue_split_pop()
1811 vq->used_elems[idx].index = elem->index; in virtqueue_split_pop()
1812 vq->used_elems[idx].len = elem->len; in virtqueue_split_pop()
1813 vq->used_elems[idx].ndescs = elem->ndescs; in virtqueue_split_pop()
1816 vq->inuse++; in virtqueue_split_pop()
1818 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); in virtqueue_split_pop()
1836 VirtIODevice *vdev = vq->vdev; in virtqueue_packed_pop()
1855 max = vq->vring.num; in virtqueue_packed_pop()
1857 if (vq->inuse >= vq->vring.num) { in virtqueue_packed_pop()
1862 i = vq->last_avail_idx; in virtqueue_packed_pop()
1870 if (caches->desc.len < max * sizeof(VRingDesc)) { in virtqueue_packed_pop()
1875 desc_cache = &caches->desc; in virtqueue_packed_pop()
1886 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, in virtqueue_packed_pop()
1906 VIRTQUEUE_MAX_SIZE - out_num, true, in virtqueue_packed_pop()
1940 elem->out_addr[i] = addr[i]; in virtqueue_packed_pop()
1941 elem->out_sg[i] = iov[i]; in virtqueue_packed_pop()
1944 elem->in_addr[i] = addr[out_num + i]; in virtqueue_packed_pop()
1945 elem->in_sg[i] = iov[out_num + i]; in virtqueue_packed_pop()
1948 elem->index = id; in virtqueue_packed_pop()
1949 elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries; in virtqueue_packed_pop()
1952 vq->used_elems[vq->last_avail_idx].index = elem->index; in virtqueue_packed_pop()
1953 vq->used_elems[vq->last_avail_idx].len = elem->len; in virtqueue_packed_pop()
1954 vq->used_elems[vq->last_avail_idx].ndescs = elem->ndescs; in virtqueue_packed_pop()
1957 vq->last_avail_idx += elem->ndescs; in virtqueue_packed_pop()
1958 vq->inuse += elem->ndescs; in virtqueue_packed_pop()
1960 if (vq->last_avail_idx >= vq->vring.num) { in virtqueue_packed_pop()
1961 vq->last_avail_idx -= vq->vring.num; in virtqueue_packed_pop()
1962 vq->last_avail_wrap_counter ^= 1; in virtqueue_packed_pop()
1965 vq->shadow_avail_idx = vq->last_avail_idx; in virtqueue_packed_pop()
1966 vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter; in virtqueue_packed_pop()
1968 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); in virtqueue_packed_pop()
1981 if (virtio_device_disabled(vq->vdev)) { in virtqueue_pop()
1985 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_pop()
1998 VirtIODevice *vdev = vq->vdev; in virtqueue_packed_drop_all()
2008 desc_cache = &caches->desc; in virtqueue_packed_drop_all()
2012 while (vq->inuse < vq->vring.num) { in virtqueue_packed_drop_all()
2013 unsigned int idx = vq->last_avail_idx; in virtqueue_packed_drop_all()
2019 vq->last_avail_idx , true); in virtqueue_packed_drop_all()
2020 if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) { in virtqueue_packed_drop_all()
2026 vq->vring.num, &idx, false)) { in virtqueue_packed_drop_all()
2035 vq->last_avail_idx += elem.ndescs; in virtqueue_packed_drop_all()
2036 if (vq->last_avail_idx >= vq->vring.num) { in virtqueue_packed_drop_all()
2037 vq->last_avail_idx -= vq->vring.num; in virtqueue_packed_drop_all()
2038 vq->last_avail_wrap_counter ^= 1; in virtqueue_packed_drop_all()
2049 VirtIODevice *vdev = vq->vdev; in virtqueue_split_drop_all()
2052 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { in virtqueue_split_drop_all()
2056 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { in virtqueue_split_drop_all()
2059 vq->inuse++; in virtqueue_split_drop_all()
2060 vq->last_avail_idx++; in virtqueue_split_drop_all()
2062 vring_set_avail_event(vq, vq->last_avail_idx); in virtqueue_split_drop_all()
2081 struct VirtIODevice *vdev = vq->vdev; in virtqueue_drop_all()
2083 if (virtio_device_disabled(vq->vdev)) { in virtqueue_drop_all()
2097 * In the meanwhile, since the in-memory layout of VirtQueueElement
2128 elem->index = data.index; in qemu_get_virtqueue_element()
2130 for (i = 0; i < elem->in_num; i++) { in qemu_get_virtqueue_element()
2131 elem->in_addr[i] = data.in_addr[i]; in qemu_get_virtqueue_element()
2134 for (i = 0; i < elem->out_num; i++) { in qemu_get_virtqueue_element()
2135 elem->out_addr[i] = data.out_addr[i]; in qemu_get_virtqueue_element()
2138 for (i = 0; i < elem->in_num; i++) { in qemu_get_virtqueue_element()
2140 elem->in_sg[i].iov_base = 0; in qemu_get_virtqueue_element()
2141 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; in qemu_get_virtqueue_element()
2144 for (i = 0; i < elem->out_num; i++) { in qemu_get_virtqueue_element()
2146 elem->out_sg[i].iov_base = 0; in qemu_get_virtqueue_element()
2147 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; in qemu_get_virtqueue_element()
2151 qemu_get_be32s(f, &elem->ndescs); in qemu_get_virtqueue_element()
2165 data.index = elem->index; in qemu_put_virtqueue_element()
2166 data.in_num = elem->in_num; in qemu_put_virtqueue_element()
2167 data.out_num = elem->out_num; in qemu_put_virtqueue_element()
2169 for (i = 0; i < elem->in_num; i++) { in qemu_put_virtqueue_element()
2170 data.in_addr[i] = elem->in_addr[i]; in qemu_put_virtqueue_element()
2173 for (i = 0; i < elem->out_num; i++) { in qemu_put_virtqueue_element()
2174 data.out_addr[i] = elem->out_addr[i]; in qemu_put_virtqueue_element()
2177 for (i = 0; i < elem->in_num; i++) { in qemu_put_virtqueue_element()
2180 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; in qemu_put_virtqueue_element()
2183 for (i = 0; i < elem->out_num; i++) { in qemu_put_virtqueue_element()
2185 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; in qemu_put_virtqueue_element()
2189 qemu_put_be32s(f, &elem->ndescs); in qemu_put_virtqueue_element()
2205 if (k->notify) { in virtio_notify_vector()
2206 k->notify(qbus->parent, vector); in virtio_notify_vector()
2221 return -EFAULT; in virtio_validate_features()
2224 if (k->validate_features) { in virtio_validate_features()
2225 return k->validate_features(vdev); in virtio_validate_features()
2238 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && in virtio_set_status()
2247 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) != in virtio_set_status()
2252 if (k->set_status) { in virtio_set_status()
2253 ret = k->set_status(vdev, val); in virtio_set_status()
2256 vdev->name, val, vdev->status); in virtio_set_status()
2259 vdev->status = val; in virtio_set_status()
2284 vdev->vq[i].vring.desc = 0; in __virtio_queue_reset()
2285 vdev->vq[i].vring.avail = 0; in __virtio_queue_reset()
2286 vdev->vq[i].vring.used = 0; in __virtio_queue_reset()
2287 vdev->vq[i].last_avail_idx = 0; in __virtio_queue_reset()
2288 vdev->vq[i].shadow_avail_idx = 0; in __virtio_queue_reset()
2289 vdev->vq[i].used_idx = 0; in __virtio_queue_reset()
2290 vdev->vq[i].last_avail_wrap_counter = true; in __virtio_queue_reset()
2291 vdev->vq[i].shadow_avail_wrap_counter = true; in __virtio_queue_reset()
2292 vdev->vq[i].used_wrap_counter = true; in __virtio_queue_reset()
2294 vdev->vq[i].signalled_used = 0; in __virtio_queue_reset()
2295 vdev->vq[i].signalled_used_valid = false; in __virtio_queue_reset()
2296 vdev->vq[i].notification = true; in __virtio_queue_reset()
2297 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; in __virtio_queue_reset()
2298 vdev->vq[i].inuse = 0; in __virtio_queue_reset()
2299 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); in __virtio_queue_reset()
2306 if (k->queue_reset) { in virtio_queue_reset()
2307 k->queue_reset(vdev, queue_index); in virtio_queue_reset()
2320 * be re-enabled for new machine types only, and also after in virtio_queue_enable()
2329 if (k->queue_enable) { in virtio_queue_enable()
2330 k->queue_enable(vdev, queue_index); in virtio_queue_enable()
2336 if (!vdev->vq[n].vring.num) { in virtio_queue_set_addr()
2339 vdev->vq[n].vring.desc = addr; in virtio_queue_set_addr()
2345 return vdev->vq[n].vring.desc; in virtio_queue_get_addr()
2351 if (!vdev->vq[n].vring.num) { in virtio_queue_set_rings()
2354 vdev->vq[n].vring.desc = desc; in virtio_queue_set_rings()
2355 vdev->vq[n].vring.avail = avail; in virtio_queue_set_rings()
2356 vdev->vq[n].vring.used = used; in virtio_queue_set_rings()
2365 if (!!num != !!vdev->vq[n].vring.num || in virtio_queue_set_num()
2370 vdev->vq[n].vring.num = num; in virtio_queue_set_num()
2375 return QLIST_FIRST(&vdev->vector_queues[vector]); in virtio_vector_first_queue()
2385 return vdev->vq[n].vring.num; in virtio_queue_get_num()
2390 return vdev->vq[n].vring.num_default; in virtio_queue_get_max_num()
2411 /* virtio-1 compliant devices cannot change the alignment */ in virtio_queue_set_align()
2413 error_report("tried to modify queue alignment for virtio-1 device"); in virtio_queue_set_align()
2420 assert(k->has_variable_vring_alignment); in virtio_queue_set_align()
2423 vdev->vq[n].vring.align = align; in virtio_queue_set_align()
2430 if (!vq->vring.desc) { in virtio_queue_set_shadow_avail_idx()
2435 * 16-bit data for packed VQs include 1-bit wrap counter and in virtio_queue_set_shadow_avail_idx()
2436 * 15-bit shadow_avail_idx. in virtio_queue_set_shadow_avail_idx()
2438 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_set_shadow_avail_idx()
2439 vq->shadow_avail_wrap_counter = (shadow_avail_idx >> 15) & 0x1; in virtio_queue_set_shadow_avail_idx()
2440 vq->shadow_avail_idx = shadow_avail_idx & 0x7FFF; in virtio_queue_set_shadow_avail_idx()
2442 vq->shadow_avail_idx = shadow_avail_idx; in virtio_queue_set_shadow_avail_idx()
2448 if (vq->vring.desc && vq->handle_output) { in virtio_queue_notify_vq()
2449 VirtIODevice *vdev = vq->vdev; in virtio_queue_notify_vq()
2451 if (unlikely(vdev->broken)) { in virtio_queue_notify_vq()
2455 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); in virtio_queue_notify_vq()
2456 vq->handle_output(vdev, vq); in virtio_queue_notify_vq()
2458 if (unlikely(vdev->start_on_kick)) { in virtio_queue_notify_vq()
2466 VirtQueue *vq = &vdev->vq[n]; in virtio_queue_notify()
2468 if (unlikely(!vq->vring.desc || vdev->broken)) { in virtio_queue_notify()
2472 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); in virtio_queue_notify()
2473 if (vq->host_notifier_enabled) { in virtio_queue_notify()
2474 event_notifier_set(&vq->host_notifier); in virtio_queue_notify()
2475 } else if (vq->handle_output) { in virtio_queue_notify()
2476 vq->handle_output(vdev, vq); in virtio_queue_notify()
2478 if (unlikely(vdev->start_on_kick)) { in virtio_queue_notify()
2486 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : in virtio_queue_vector()
2492 VirtQueue *vq = &vdev->vq[n]; in virtio_queue_set_vector()
2495 if (vdev->vector_queues && in virtio_queue_set_vector()
2496 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { in virtio_queue_set_vector()
2499 vdev->vq[n].vector = vector; in virtio_queue_set_vector()
2500 if (vdev->vector_queues && in virtio_queue_set_vector()
2502 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); in virtio_queue_set_vector()
2513 if (vdev->vq[i].vring.num == 0) in virtio_add_queue()
2520 vdev->vq[i].vring.num = queue_size; in virtio_add_queue()
2521 vdev->vq[i].vring.num_default = queue_size; in virtio_add_queue()
2522 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; in virtio_add_queue()
2523 vdev->vq[i].handle_output = handle_output; in virtio_add_queue()
2524 vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size); in virtio_add_queue()
2526 return &vdev->vq[i]; in virtio_add_queue()
2531 vq->vring.num = 0; in virtio_delete_queue()
2532 vq->vring.num_default = 0; in virtio_delete_queue()
2533 vq->handle_output = NULL; in virtio_delete_queue()
2534 g_free(vq->used_elems); in virtio_delete_queue()
2535 vq->used_elems = NULL; in virtio_delete_queue()
2545 virtio_delete_queue(&vdev->vq[n]); in virtio_del_queue()
2550 uint8_t old = qatomic_read(&vdev->isr); in virtio_set_isr()
2556 qatomic_or(&vdev->isr, value); in virtio_set_isr()
2569 !vq->inuse && virtio_queue_empty(vq)) { in virtio_split_should_notify()
2577 v = vq->signalled_used_valid; in virtio_split_should_notify()
2578 vq->signalled_used_valid = true; in virtio_split_should_notify()
2579 old = vq->signalled_used; in virtio_split_should_notify()
2580 new = vq->signalled_used = vq->used_idx; in virtio_split_should_notify()
2591 off -= vq->vring.num; in vring_packed_need_event()
2610 vring_packed_event_read(vdev, &caches->avail, &e); in virtio_packed_should_notify()
2612 old = vq->signalled_used; in virtio_packed_should_notify()
2613 new = vq->signalled_used = vq->used_idx; in virtio_packed_should_notify()
2614 v = vq->signalled_used_valid; in virtio_packed_should_notify()
2615 vq->signalled_used_valid = true; in virtio_packed_should_notify()
2623 return !v || vring_packed_need_event(vq, vq->used_wrap_counter, in virtio_packed_should_notify()
2643 trace_virtio_notify_irqfd_deferred_fn(vq->vdev, vq); in virtio_notify_irqfd_deferred_fn()
2659 * windows drivers included in virtio-win 1.8.0 (circa 2015) are in virtio_notify_irqfd()
2672 virtio_set_isr(vq->vdev, 0x1); in virtio_notify_irqfd()
2673 defer_call(virtio_notify_irqfd_deferred_fn, &vq->guest_notifier); in virtio_notify_irqfd()
2678 virtio_set_isr(vq->vdev, 0x1); in virtio_irq()
2679 virtio_notify_vector(vq->vdev, vq->vector); in virtio_irq()
2696 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) in virtio_notify_config()
2700 vdev->generation++; in virtio_notify_config()
2701 virtio_notify_vector(vdev, vdev->config_vector); in virtio_notify_config()
2708 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); in virtio_device_endian_needed()
2710 return vdev->device_endian != virtio_default_endian(); in virtio_device_endian_needed()
2713 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; in virtio_device_endian_needed()
2720 return (vdev->host_features >> 32) != 0; in virtio_64bit_features_needed()
2743 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { in virtio_ringsize_needed()
2756 return k->has_extra_state && in virtio_extra_state_needed()
2757 k->has_extra_state(qbus->parent); in virtio_extra_state_needed()
2764 return vdev->broken; in virtio_broken_needed()
2771 return vdev->started; in virtio_started_needed()
2778 return vdev->disabled; in virtio_disabled_needed()
2859 if (!k->load_extra_state) { in get_extra_state()
2860 return -1; in get_extra_state()
2862 return k->load_extra_state(qbus->parent, f); in get_extra_state()
2873 k->save_extra_state(qbus->parent, f); in put_extra_state()
2983 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); in virtio_save()
2986 if (k->save_config) { in virtio_save()
2987 k->save_config(qbus->parent, f); in virtio_save()
2990 qemu_put_8s(f, &vdev->status); in virtio_save()
2991 qemu_put_8s(f, &vdev->isr); in virtio_save()
2992 qemu_put_be16s(f, &vdev->queue_sel); in virtio_save()
2994 qemu_put_be32(f, vdev->config_len); in virtio_save()
2995 qemu_put_buffer(f, vdev->config, vdev->config_len); in virtio_save()
2998 if (vdev->vq[i].vring.num == 0) in virtio_save()
3005 if (vdev->vq[i].vring.num == 0) in virtio_save()
3008 qemu_put_be32(f, vdev->vq[i].vring.num); in virtio_save()
3009 if (k->has_variable_vring_alignment) { in virtio_save()
3010 qemu_put_be32(f, vdev->vq[i].vring.align); in virtio_save()
3014 * subsections for VIRTIO-1 devices. in virtio_save()
3016 qemu_put_be64(f, vdev->vq[i].vring.desc); in virtio_save()
3017 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); in virtio_save()
3018 if (k->save_queue) { in virtio_save()
3019 k->save_queue(qbus->parent, i, f); in virtio_save()
3023 if (vdc->save != NULL) { in virtio_save()
3024 vdc->save(vdev, f); in virtio_save()
3027 if (vdc->vmsd) { in virtio_save()
3028 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL); in virtio_save()
3053 return virtio_load(vdev, f, dc->vmsd->version_id); in virtio_device_get()
3065 bool bad = (val & ~(vdev->host_features)) != 0; in virtio_set_features_nocheck()
3067 val &= vdev->host_features; in virtio_set_features_nocheck()
3068 if (k->set_features) { in virtio_set_features_nocheck()
3069 k->set_features(vdev, val); in virtio_set_features_nocheck()
3071 vdev->guest_features = val; in virtio_set_features_nocheck()
3072 return bad ? -1 : 0; in virtio_set_features_nocheck()
3086 data->ret = virtio_set_features_nocheck(data->vdev, data->val); in virtio_set_features_nocheck_bh()
3087 aio_co_wake(data->co); in virtio_set_features_nocheck_bh()
3115 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { in virtio_set_features()
3116 return -EINVAL; in virtio_set_features()
3122 __func__, vdev->name); in virtio_set_features()
3130 if (vdev->vq[i].vring.num != 0) { in virtio_set_features()
3136 if (!virtio_device_started(vdev, vdev->status) && in virtio_set_features()
3138 vdev->start_on_kick = true; in virtio_set_features()
3153 vdev->device_endian = virtio_current_cpu_endian(); in virtio_reset()
3156 vdev->device_endian = virtio_default_endian(); in virtio_reset()
3159 if (k->get_vhost) { in virtio_reset()
3160 struct vhost_dev *hdev = k->get_vhost(vdev); in virtio_reset()
3161 /* Only reset when vhost back-end is connected */ in virtio_reset()
3162 if (hdev && hdev->vhost_ops) { in virtio_reset()
3167 if (k->reset) { in virtio_reset()
3168 k->reset(vdev); in virtio_reset()
3171 vdev->start_on_kick = false; in virtio_reset()
3172 vdev->started = false; in virtio_reset()
3173 vdev->broken = false; in virtio_reset()
3175 vdev->queue_sel = 0; in virtio_reset()
3176 vdev->status = 0; in virtio_reset()
3177 vdev->disabled = false; in virtio_reset()
3178 qatomic_set(&vdev->isr, 0); in virtio_reset()
3179 vdev->config_vector = VIRTIO_NO_VECTOR; in virtio_reset()
3180 virtio_notify_vector(vdev, vdev->config_vector); in virtio_reset()
3192 DeviceState *proxy = DEVICE(BUS(bus)->parent); in virtio_device_check_notification_compatibility()
3195 k->ioeventfd_enabled(proxy)) { in virtio_device_check_notification_compatibility()
3204 size_t config_size = params->min_size; in virtio_get_config_size()
3205 const VirtIOFeature *feature_sizes = params->feature_sizes; in virtio_get_config_size()
3214 assert(config_size <= params->max_size); in virtio_get_config_size()
3233 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; in virtio_load()
3235 if (k->load_config) { in virtio_load()
3236 ret = k->load_config(qbus->parent, f); in virtio_load()
3241 qemu_get_8s(f, &vdev->status); in virtio_load()
3242 qemu_get_8s(f, &vdev->isr); in virtio_load()
3243 qemu_get_be16s(f, &vdev->queue_sel); in virtio_load()
3244 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { in virtio_load()
3245 return -1; in virtio_load()
3250 * Temporarily set guest_features low bits - needed by in virtio_load()
3254 * Note: devices should always test host features in future - don't create in virtio_load()
3257 vdev->guest_features = features; in virtio_load()
3266 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); in virtio_load()
3268 while (config_len > vdev->config_len) { in virtio_load()
3270 config_len--; in virtio_load()
3273 if (vdc->pre_load_queues) { in virtio_load()
3274 ret = vdc->pre_load_queues(vdev); in virtio_load()
3284 return -1; in virtio_load()
3288 vdev->vq[i].vring.num = qemu_get_be32(f); in virtio_load()
3289 if (k->has_variable_vring_alignment) { in virtio_load()
3290 vdev->vq[i].vring.align = qemu_get_be32(f); in virtio_load()
3292 vdev->vq[i].vring.desc = qemu_get_be64(f); in virtio_load()
3293 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); in virtio_load()
3294 vdev->vq[i].signalled_used_valid = false; in virtio_load()
3295 vdev->vq[i].notification = true; in virtio_load()
3297 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { in virtio_load()
3300 i, vdev->vq[i].last_avail_idx); in virtio_load()
3301 return -1; in virtio_load()
3303 if (k->load_queue) { in virtio_load()
3304 ret = k->load_queue(qbus->parent, i, f); in virtio_load()
3312 if (vdc->load != NULL) { in virtio_load()
3313 ret = vdc->load(vdev, f, version_id); in virtio_load()
3319 if (vdc->vmsd) { in virtio_load()
3320 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id); in virtio_load()
3332 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { in virtio_load()
3333 vdev->device_endian = virtio_default_endian(); in virtio_load()
3338 * Subsection load filled vdev->guest_features. Run them in virtio_load()
3339 * through virtio_set_features to sanity-check them against in virtio_load()
3342 uint64_t features64 = vdev->guest_features; in virtio_load()
3346 features64, vdev->host_features); in virtio_load()
3347 return -1; in virtio_load()
3353 features, vdev->host_features); in virtio_load()
3354 return -1; in virtio_load()
3358 if (!virtio_device_started(vdev, vdev->status) && in virtio_load()
3360 vdev->start_on_kick = true; in virtio_load()
3365 if (vdev->vq[i].vring.desc) { in virtio_load()
3369 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so in virtio_load()
3381 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx; in virtio_load()
3382 vdev->vq[i].shadow_avail_wrap_counter = in virtio_load()
3383 vdev->vq[i].last_avail_wrap_counter; in virtio_load()
3387 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; in virtio_load()
3389 if (nheads > vdev->vq[i].vring.num) { in virtio_load()
3392 i, vdev->vq[i].vring.num, in virtio_load()
3393 vring_avail_idx(&vdev->vq[i]), in virtio_load()
3394 vdev->vq[i].last_avail_idx, nheads); in virtio_load()
3395 vdev->vq[i].used_idx = 0; in virtio_load()
3396 vdev->vq[i].shadow_avail_idx = 0; in virtio_load()
3397 vdev->vq[i].inuse = 0; in virtio_load()
3400 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); in virtio_load()
3401 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); in virtio_load()
3409 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - in virtio_load()
3410 vdev->vq[i].used_idx); in virtio_load()
3411 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { in virtio_load()
3412 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " in virtio_load()
3414 i, vdev->vq[i].vring.num, in virtio_load()
3415 vdev->vq[i].last_avail_idx, in virtio_load()
3416 vdev->vq[i].used_idx); in virtio_load()
3417 return -1; in virtio_load()
3422 if (vdc->post_load) { in virtio_load()
3423 ret = vdc->post_load(vdev); in virtio_load()
3434 qemu_del_vm_change_state_handler(vdev->vmstate); in virtio_cleanup()
3442 bool backend_run = running && virtio_device_started(vdev, vdev->status); in virtio_vmstate_change()
3443 vdev->vm_running = running; in virtio_vmstate_change()
3446 virtio_set_status(vdev, vdev->status); in virtio_vmstate_change()
3449 if (k->vmstate_change) { in virtio_vmstate_change()
3450 k->vmstate_change(qbus->parent, backend_run); in virtio_vmstate_change()
3454 int ret = virtio_set_status(vdev, vdev->status); in virtio_vmstate_change()
3467 object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev, in virtio_instance_init_common()
3478 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; in virtio_init()
3481 vdev->vector_queues = in virtio_init()
3482 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); in virtio_init()
3485 vdev->start_on_kick = false; in virtio_init()
3486 vdev->started = false; in virtio_init()
3487 vdev->vhost_started = false; in virtio_init()
3488 vdev->device_id = device_id; in virtio_init()
3489 vdev->status = 0; in virtio_init()
3490 qatomic_set(&vdev->isr, 0); in virtio_init()
3491 vdev->queue_sel = 0; in virtio_init()
3492 vdev->config_vector = VIRTIO_NO_VECTOR; in virtio_init()
3493 vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX); in virtio_init()
3494 vdev->vm_running = runstate_is_running(); in virtio_init()
3495 vdev->broken = false; in virtio_init()
3497 vdev->vq[i].vector = VIRTIO_NO_VECTOR; in virtio_init()
3498 vdev->vq[i].vdev = vdev; in virtio_init()
3499 vdev->vq[i].queue_index = i; in virtio_init()
3500 vdev->vq[i].host_notifier_enabled = false; in virtio_init()
3503 vdev->name = virtio_id_to_name(device_id); in virtio_init()
3504 vdev->config_len = config_size; in virtio_init()
3505 if (vdev->config_len) { in virtio_init()
3506 vdev->config = g_malloc0(config_size); in virtio_init()
3508 vdev->config = NULL; in virtio_init()
3510 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), in virtio_init()
3512 vdev->device_endian = virtio_default_endian(); in virtio_init()
3513 vdev->use_guest_notifier_mask = true; in virtio_init()
3523 switch (vdev->device_id) { in virtio_legacy_allowed()
3542 return vdev->disable_legacy_check; in virtio_legacy_check_disabled()
3547 return vdev->vq[n].vring.desc; in virtio_queue_get_desc_addr()
3560 if (k->queue_enabled) { in virtio_queue_enabled()
3561 return k->queue_enabled(qbus->parent, n); in virtio_queue_enabled()
3568 return vdev->vq[n].vring.avail; in virtio_queue_get_avail_addr()
3573 return vdev->vq[n].vring.used; in virtio_queue_get_used_addr()
3578 return sizeof(VRingDesc) * vdev->vq[n].vring.num; in virtio_queue_get_desc_size()
3591 sizeof(uint16_t) * vdev->vq[n].vring.num + s; in virtio_queue_get_avail_size()
3604 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s; in virtio_queue_get_used_size()
3612 avail = vdev->vq[n].last_avail_idx; in virtio_queue_packed_get_last_avail_idx()
3613 avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15; in virtio_queue_packed_get_last_avail_idx()
3615 used = vdev->vq[n].used_idx; in virtio_queue_packed_get_last_avail_idx()
3616 used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15; in virtio_queue_packed_get_last_avail_idx()
3624 return vdev->vq[n].last_avail_idx; in virtio_queue_split_get_last_avail_idx()
3639 struct VirtQueue *vq = &vdev->vq[n]; in virtio_queue_packed_set_last_avail_idx()
3641 vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff; in virtio_queue_packed_set_last_avail_idx()
3642 vq->last_avail_wrap_counter = in virtio_queue_packed_set_last_avail_idx()
3643 vq->shadow_avail_wrap_counter = !!(idx & 0x8000); in virtio_queue_packed_set_last_avail_idx()
3645 vq->used_idx = idx & 0x7fff; in virtio_queue_packed_set_last_avail_idx()
3646 vq->used_wrap_counter = !!(idx & 0x8000); in virtio_queue_packed_set_last_avail_idx()
3652 vdev->vq[n].last_avail_idx = idx; in virtio_queue_split_set_last_avail_idx()
3653 vdev->vq[n].shadow_avail_idx = idx; in virtio_queue_split_set_last_avail_idx()
3676 if (vdev->vq[n].vring.desc) { in virtio_queue_split_restore_last_avail_idx()
3677 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]); in virtio_queue_split_restore_last_avail_idx()
3678 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx; in virtio_queue_split_restore_last_avail_idx()
3699 if (vdev->vq[n].vring.desc) { in virtio_queue_split_update_used_idx()
3700 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); in virtio_queue_split_update_used_idx()
3715 vdev->vq[n].signalled_used_valid = false; in virtio_queue_invalidate_signalled_used()
3720 return vdev->vq + n; in virtio_get_queue()
3725 return vq->queue_index; in virtio_get_queue_index()
3747 event_notifier_set_handler(&vq->guest_notifier, in virtio_queue_set_guest_notifier_fd_handler()
3750 event_notifier_set_handler(&vq->guest_notifier, NULL); in virtio_queue_set_guest_notifier_fd_handler()
3755 virtio_queue_guest_notifier_read(&vq->guest_notifier); in virtio_queue_set_guest_notifier_fd_handler()
3763 n = &vdev->config_notifier; in virtio_config_set_guest_notifier_fd_handler()
3778 return &vq->guest_notifier; in virtio_queue_get_guest_notifier()
3793 return vq->vring.desc && !virtio_queue_empty(vq); in virtio_queue_host_notifier_aio_poll()
3815 * Re-enable them. (And if detach has not been used before, notifications in virtio_queue_aio_attach_host_notifier()
3824 aio_set_event_notifier(ctx, &vq->host_notifier, in virtio_queue_aio_attach_host_notifier()
3828 aio_set_event_notifier_poll(ctx, &vq->host_notifier, in virtio_queue_aio_attach_host_notifier()
3837 event_notifier_set(&vq->host_notifier); in virtio_queue_aio_attach_host_notifier()
3843 * function does not pop all elements. When the virtqueue is left non-empty
3853 aio_set_event_notifier(ctx, &vq->host_notifier, in virtio_queue_aio_attach_host_notifier_no_poll()
3863 event_notifier_set(&vq->host_notifier); in virtio_queue_aio_attach_host_notifier_no_poll()
3868 aio_set_event_notifier(ctx, &vq->host_notifier, NULL, NULL, NULL); in virtio_queue_aio_detach_host_notifier()
3877 * we potentially re-attach it. The attach_host_notifier functions will in virtio_queue_aio_detach_host_notifier()
3892 return &vq->host_notifier; in virtio_queue_get_host_notifier()
3897 return &vdev->config_notifier; in virtio_config_get_guest_notifier()
3902 vq->host_notifier_enabled = enabled; in virtio_queue_set_host_notifier_enabled()
3911 if (k->set_host_notifier_mr) { in virtio_queue_set_host_notifier_mr()
3912 return k->set_host_notifier_mr(qbus->parent, n, mr, assign); in virtio_queue_set_host_notifier_mr()
3915 return -1; in virtio_queue_set_host_notifier_mr()
3920 g_free(vdev->bus_name); in virtio_device_set_child_bus_name()
3921 vdev->bus_name = g_strdup(bus_name); in virtio_device_set_child_bus_name()
3933 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET; in virtio_error()
3937 vdev->broken = true; in virtio_error()
3946 if (vdev->vq[i].vring.num == 0) { in virtio_memory_listener_commit()
3960 assert(!vdc->vmsd || !vdc->load); in virtio_device_realize()
3962 if (vdc->realize != NULL) { in virtio_device_realize()
3963 vdc->realize(dev, &err); in virtio_device_realize()
3974 vdc->unrealize(dev); in virtio_device_realize()
3981 vdc->unrealize(dev); in virtio_device_realize()
3985 vdev->listener.commit = virtio_memory_listener_commit; in virtio_device_realize()
3986 vdev->listener.name = "virtio"; in virtio_device_realize()
3987 memory_listener_register(&vdev->listener, vdev->dma_as); in virtio_device_realize()
3995 memory_listener_unregister(&vdev->listener); in virtio_device_unrealize()
3998 if (vdc->unrealize != NULL) { in virtio_device_unrealize()
3999 vdc->unrealize(dev); in virtio_device_unrealize()
4002 g_free(vdev->bus_name); in virtio_device_unrealize()
4003 vdev->bus_name = NULL; in virtio_device_unrealize()
4009 if (!vdev->vq) { in virtio_device_free_virtqueues()
4014 if (vdev->vq[i].vring.num == 0) { in virtio_device_free_virtqueues()
4017 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); in virtio_device_free_virtqueues()
4019 g_free(vdev->vq); in virtio_device_free_virtqueues()
4028 g_free(vdev->config); in virtio_device_instance_finalize()
4029 g_free(vdev->vector_queues); in virtio_device_instance_finalize()
4034 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
4035 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true),
4036 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice,
4051 VirtQueue *vq = &vdev->vq[n]; in virtio_device_start_ioeventfd_impl()
4060 event_notifier_set_handler(&vq->host_notifier, in virtio_device_start_ioeventfd_impl()
4066 VirtQueue *vq = &vdev->vq[n]; in virtio_device_start_ioeventfd_impl()
4067 if (!vq->vring.num) { in virtio_device_start_ioeventfd_impl()
4070 event_notifier_set(&vq->host_notifier); in virtio_device_start_ioeventfd_impl()
4077 while (--n >= 0) { in virtio_device_start_ioeventfd_impl()
4078 VirtQueue *vq = &vdev->vq[n]; in virtio_device_start_ioeventfd_impl()
4083 event_notifier_set_handler(&vq->host_notifier, NULL); in virtio_device_start_ioeventfd_impl()
4093 while (--i >= 0) { in virtio_device_start_ioeventfd_impl()
4121 VirtQueue *vq = &vdev->vq[n]; in virtio_device_stop_ioeventfd_impl()
4126 event_notifier_set_handler(&vq->host_notifier, NULL); in virtio_device_stop_ioeventfd_impl()
4166 dc->realize = virtio_device_realize; in virtio_device_class_init()
4167 dc->unrealize = virtio_device_unrealize; in virtio_device_class_init()
4168 dc->bus_type = TYPE_VIRTIO_BUS; in virtio_device_class_init()
4170 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; in virtio_device_class_init()
4171 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; in virtio_device_class_init()
4173 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; in virtio_device_class_init()
4203 status->name = g_strdup(vdev->name); in qmp_x_query_virtio_queue_status()
4204 status->queue_index = vdev->vq[queue].queue_index; in qmp_x_query_virtio_queue_status()
4205 status->inuse = vdev->vq[queue].inuse; in qmp_x_query_virtio_queue_status()
4206 status->vring_num = vdev->vq[queue].vring.num; in qmp_x_query_virtio_queue_status()
4207 status->vring_num_default = vdev->vq[queue].vring.num_default; in qmp_x_query_virtio_queue_status()
4208 status->vring_align = vdev->vq[queue].vring.align; in qmp_x_query_virtio_queue_status()
4209 status->vring_desc = vdev->vq[queue].vring.desc; in qmp_x_query_virtio_queue_status()
4210 status->vring_avail = vdev->vq[queue].vring.avail; in qmp_x_query_virtio_queue_status()
4211 status->vring_used = vdev->vq[queue].vring.used; in qmp_x_query_virtio_queue_status()
4212 status->used_idx = vdev->vq[queue].used_idx; in qmp_x_query_virtio_queue_status()
4213 status->signalled_used = vdev->vq[queue].signalled_used; in qmp_x_query_virtio_queue_status()
4214 status->signalled_used_valid = vdev->vq[queue].signalled_used_valid; in qmp_x_query_virtio_queue_status()
4216 if (vdev->vhost_started) { in qmp_x_query_virtio_queue_status()
4218 struct vhost_dev *hdev = vdc->get_vhost(vdev); in qmp_x_query_virtio_queue_status()
4220 /* check if vq index exists for vhost as well */ in qmp_x_query_virtio_queue_status()
4221 if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) { in qmp_x_query_virtio_queue_status()
4222 status->has_last_avail_idx = true; in qmp_x_query_virtio_queue_status()
4225 hdev->vhost_ops->vhost_get_vq_index(hdev, queue); in qmp_x_query_virtio_queue_status()
4230 status->last_avail_idx = in qmp_x_query_virtio_queue_status()
4231 hdev->vhost_ops->vhost_get_vring_base(hdev, &state); in qmp_x_query_virtio_queue_status()
4234 status->has_shadow_avail_idx = true; in qmp_x_query_virtio_queue_status()
4235 status->has_last_avail_idx = true; in qmp_x_query_virtio_queue_status()
4236 status->last_avail_idx = vdev->vq[queue].last_avail_idx; in qmp_x_query_virtio_queue_status()
4237 status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx; in qmp_x_query_virtio_queue_status()
4266 node->value = g_strdup(map[i].value); in qmp_decode_vring_desc_flags()
4267 node->next = list; in qmp_decode_vring_desc_flags()
4294 vq = &vdev->vq[queue]; in qmp_x_query_virtio_queue_element()
4313 max = vq->vring.num; in qmp_x_query_virtio_queue_element()
4316 head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num); in qmp_x_query_virtio_queue_element()
4318 head = vring_avail_ring(vq, index % vq->vring.num); in qmp_x_query_virtio_queue_element()
4327 if (caches->desc.len < max * sizeof(VRingDesc)) { in qmp_x_query_virtio_queue_element()
4332 desc_cache = &caches->desc; in qmp_x_query_virtio_queue_element()
4336 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, in qmp_x_query_virtio_queue_element()
4350 element->avail = g_new0(VirtioRingAvail, 1); in qmp_x_query_virtio_queue_element()
4351 element->used = g_new0(VirtioRingUsed, 1); in qmp_x_query_virtio_queue_element()
4352 element->name = g_strdup(vdev->name); in qmp_x_query_virtio_queue_element()
4353 element->index = head; in qmp_x_query_virtio_queue_element()
4354 element->avail->flags = vring_avail_flags(vq); in qmp_x_query_virtio_queue_element()
4355 element->avail->idx = vring_avail_idx(vq); in qmp_x_query_virtio_queue_element()
4356 element->avail->ring = head; in qmp_x_query_virtio_queue_element()
4357 element->used->flags = vring_used_flags(vq); in qmp_x_query_virtio_queue_element()
4358 element->used->idx = vring_used_idx(vq); in qmp_x_query_virtio_queue_element()
4367 node->value = g_new0(VirtioRingDesc, 1); in qmp_x_query_virtio_queue_element()
4368 node->value->addr = desc.addr; in qmp_x_query_virtio_queue_element()
4369 node->value->len = desc.len; in qmp_x_query_virtio_queue_element()
4370 node->value->flags = qmp_decode_vring_desc_flags(desc.flags); in qmp_x_query_virtio_queue_element()
4371 node->next = list; in qmp_x_query_virtio_queue_element()
4377 element->descs = list; in qmp_x_query_virtio_queue_element()
4406 DeviceState *transport = qdev_get_parent_bus(dev)->parent; in type_init()
4409 &transport->mem_reentrancy_guard); in type_init()