221         struct vhost_virtqueue *vq = dev->vqs + i;  in vhost_sync_dirty_bitmap()  local
223         if (!vq->used_phys && !vq->used_size) {  in vhost_sync_dirty_bitmap()
229             hwaddr used_phys = vq->used_phys, used_size = vq->used_size;  in vhost_sync_dirty_bitmap()
269                                   end_addr, vq->used_phys,  in vhost_sync_dirty_bitmap()
270                                   range_get_last(vq->used_phys, vq->used_size));  in vhost_sync_dirty_bitmap()
524         struct vhost_virtqueue *vq = dev->vqs + i;  in vhost_verify_ring_mappings()  local
526         if (vq->desc_phys == 0) {  in vhost_verify_ring_mappings()
532                 vq->desc, vq->desc_phys, vq->desc_size,  in vhost_verify_ring_mappings()
540                 vq->avail, vq->avail_phys, vq->avail_size,  in vhost_verify_ring_mappings()
548                 vq->used, vq->used_phys, vq->used_size,  in vhost_verify_ring_mappings()
957                                     struct vhost_virtqueue *vq,  in vhost_virtqueue_set_addr()  argument
965         r = dev->vhost_ops->vhost_vq_get_addr(dev, &addr, vq);  in vhost_virtqueue_set_addr()
971         addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc;  in vhost_virtqueue_set_addr()
972         addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail;  in vhost_virtqueue_set_addr()
973         addr.used_user_addr = (uint64_t)(unsigned long)vq->used;  in vhost_virtqueue_set_addr()
976     addr.log_guest_addr = vq->used_phys;  in vhost_virtqueue_set_addr()
1252                           struct vhost_virtqueue *vq,  in vhost_virtqueue_start()  argument
1275     vq->num = state.num = virtio_queue_get_num(vdev, idx);  in vhost_virtqueue_start()
1298     vq->desc_size = s = l = virtio_queue_get_desc_size(vdev, idx);  in vhost_virtqueue_start()
1299     vq->desc_phys = a;  in vhost_virtqueue_start()
1300     vq->desc = vhost_memory_map(dev, a, &l, false);  in vhost_virtqueue_start()
1301     if (!vq->desc || l != s) {  in vhost_virtqueue_start()
1305     vq->avail_size = s = l = virtio_queue_get_avail_size(vdev, idx);  in vhost_virtqueue_start()
1306     vq->avail_phys = a = virtio_queue_get_avail_addr(vdev, idx);  in vhost_virtqueue_start()
1307     vq->avail = vhost_memory_map(dev, a, &l, false);  in vhost_virtqueue_start()
1308     if (!vq->avail || l != s) {  in vhost_virtqueue_start()
1312     vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);  in vhost_virtqueue_start()
1313     vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);  in vhost_virtqueue_start()
1314     vq->used = vhost_memory_map(dev, a, &l, true);  in vhost_virtqueue_start()
1315     if (!vq->used || l != s) {  in vhost_virtqueue_start()
1320     r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);  in vhost_virtqueue_start()
1333     event_notifier_test_and_clear(&vq->masked_notifier);  in vhost_virtqueue_start()
1358     vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),  in vhost_virtqueue_start()
1361     vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),  in vhost_virtqueue_start()
1364     vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),  in vhost_virtqueue_start()
1372                          struct vhost_virtqueue *vq,  in vhost_virtqueue_stop()  argument
1408     vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),  in vhost_virtqueue_stop()
1410     vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),  in vhost_virtqueue_stop()
1412     vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),  in vhost_virtqueue_stop()
1442     struct vhost_virtqueue *vq = container_of(n, struct vhost_virtqueue,  in vhost_virtqueue_error_notifier()  local
1444     struct vhost_dev *dev = vq->dev;  in vhost_virtqueue_error_notifier()
1445     int index = vq - dev->vqs;  in vhost_virtqueue_error_notifier()
1454                                 struct vhost_virtqueue *vq, int n)  in vhost_virtqueue_init()  argument
1460     int r = event_notifier_init(&vq->masked_notifier, 0);  in vhost_virtqueue_init()
1465     file.fd = event_notifier_get_wfd(&vq->masked_notifier);  in vhost_virtqueue_init()
1472     vq->dev = dev;  in vhost_virtqueue_init()
1475         r = event_notifier_init(&vq->error_notifier, 0);  in vhost_virtqueue_init()
1480         file.fd = event_notifier_get_fd(&vq->error_notifier);  in vhost_virtqueue_init()
1487         event_notifier_set_handler(&vq->error_notifier,  in vhost_virtqueue_init()
1494     event_notifier_cleanup(&vq->error_notifier);  in vhost_virtqueue_init()
1496     event_notifier_cleanup(&vq->masked_notifier);  in vhost_virtqueue_init()
1500 static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)  in vhost_virtqueue_cleanup()  argument
1502     event_notifier_cleanup(&vq->masked_notifier);  in vhost_virtqueue_cleanup()
1503     if (vq->dev->vhost_ops->vhost_set_vring_err) {  in vhost_virtqueue_cleanup()
1504         event_notifier_set_handler(&vq->error_notifier, NULL);  in vhost_virtqueue_cleanup()
1505         event_notifier_cleanup(&vq->error_notifier);  in vhost_virtqueue_cleanup()
1773     struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;  in vhost_virtqueue_pending()  local
1775     return event_notifier_test_and_clear(&vq->masked_notifier);  in vhost_virtqueue_pending()
2097             struct vhost_virtqueue *vq = hdev->vqs + i;  in vhost_dev_start()  local
2098             r = vhost_device_iotlb_miss(hdev, vq->used_phys, true);  in vhost_dev_start()