69 uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)  in vhost_svq_available_slots()  argument
71     return svq->num_free;  in vhost_svq_available_slots()
83 static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,  in vhost_svq_translate_addr()  argument
104             map = vhost_iova_tree_find_gpa(svq->iova_tree, &needle);  in vhost_svq_translate_addr()
111             map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);  in vhost_svq_translate_addr()
154 static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,  in vhost_svq_vring_write_descs()  argument
159     uint16_t i = svq->free_head, last = svq->free_head;  in vhost_svq_vring_write_descs()
162     vring_desc_t *descs = svq->vring.desc;  in vhost_svq_vring_write_descs()
169     ok = vhost_svq_translate_addr(svq, sg, iovec, num, addr);  in vhost_svq_vring_write_descs()
177             descs[i].next = cpu_to_le16(svq->desc_next[i]);  in vhost_svq_vring_write_descs()
185         i = svq->desc_next[i];  in vhost_svq_vring_write_descs()
188     svq->free_head = svq->desc_next[last];  in vhost_svq_vring_write_descs()
192 static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,  in vhost_svq_add_split()  argument
199     vring_avail_t *avail = svq->vring.avail;  in vhost_svq_add_split()
203     *head = svq->free_head;  in vhost_svq_add_split()
212     ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, out_addr,  in vhost_svq_add_split()
218     ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, in_addr, false,  in vhost_svq_add_split()
228     avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);  in vhost_svq_add_split()
230     svq->shadow_avail_idx++;  in vhost_svq_add_split()
234     avail->idx = cpu_to_le16(svq->shadow_avail_idx);  in vhost_svq_add_split()
239 static void vhost_svq_kick(VhostShadowVirtqueue *svq)  in vhost_svq_kick()  argument
249     if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {  in vhost_svq_kick()
251                 *(uint16_t *)(&svq->vring.used->ring[svq->vring.num]));  in vhost_svq_kick()
252 …      needs_kick = vring_need_event(avail_event, svq->shadow_avail_idx, svq->shadow_avail_idx - 1);  in vhost_svq_kick()
255                 !(svq->vring.used->flags & cpu_to_le16(VRING_USED_F_NO_NOTIFY));  in vhost_svq_kick()
262     event_notifier_set(&svq->hdev_kick);  in vhost_svq_kick()
270 int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg,  in vhost_svq_add()  argument
279     if (unlikely(ndescs > vhost_svq_available_slots(svq))) {  in vhost_svq_add()
283     ok = vhost_svq_add_split(svq, out_sg, out_num, out_addr, in_sg, in_num,  in vhost_svq_add()
289     svq->num_free -= ndescs;  in vhost_svq_add()
290     svq->desc_state[qemu_head].elem = elem;  in vhost_svq_add()
291     svq->desc_state[qemu_head].ndescs = ndescs;  in vhost_svq_add()
292     vhost_svq_kick(svq);  in vhost_svq_add()
297 static int vhost_svq_add_element(VhostShadowVirtqueue *svq,  in vhost_svq_add_element()  argument
300     return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->out_addr,  in vhost_svq_add_element()
317 static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)  in vhost_handle_guest_kick()  argument
320     event_notifier_test_and_clear(&svq->svq_kick);  in vhost_handle_guest_kick()
324         virtio_queue_set_notification(svq->vq, false);  in vhost_handle_guest_kick()
330             if (svq->next_guest_avail_elem) {  in vhost_handle_guest_kick()
331                 elem = g_steal_pointer(&svq->next_guest_avail_elem);  in vhost_handle_guest_kick()
333                 elem = virtqueue_pop(svq->vq, sizeof(*elem));  in vhost_handle_guest_kick()
340             if (svq->ops) {  in vhost_handle_guest_kick()
341                 r = svq->ops->avail_handler(svq, elem, svq->ops_opaque);  in vhost_handle_guest_kick()
343                 r = vhost_svq_add_element(svq, elem);  in vhost_handle_guest_kick()
358                     svq->next_guest_avail_elem = g_steal_pointer(&elem);  in vhost_handle_guest_kick()
368         virtio_queue_set_notification(svq->vq, true);  in vhost_handle_guest_kick()
369     } while (!virtio_queue_empty(svq->vq));  in vhost_handle_guest_kick()
379     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);  in vhost_handle_guest_kick_notifier()  local
381     vhost_handle_guest_kick(svq);  in vhost_handle_guest_kick_notifier()
384 static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)  in vhost_svq_more_used()  argument
386     uint16_t *used_idx = &svq->vring.used->idx;  in vhost_svq_more_used()
387     if (svq->last_used_idx != svq->shadow_used_idx) {  in vhost_svq_more_used()
391     svq->shadow_used_idx = le16_to_cpu(*(volatile uint16_t *)used_idx);  in vhost_svq_more_used()
393     return svq->last_used_idx != svq->shadow_used_idx;  in vhost_svq_more_used()
405 static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)  in vhost_svq_enable_notification()  argument
407     if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {  in vhost_svq_enable_notification()
408         uint16_t *used_event = (uint16_t *)&svq->vring.avail->ring[svq->vring.num];  in vhost_svq_enable_notification()
409         *used_event = cpu_to_le16(svq->shadow_used_idx);  in vhost_svq_enable_notification()
411         svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);  in vhost_svq_enable_notification()
416     return !vhost_svq_more_used(svq);  in vhost_svq_enable_notification()
419 static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)  in vhost_svq_disable_notification()  argument
425     if (!virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {  in vhost_svq_disable_notification()
426         svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);  in vhost_svq_disable_notification()
430 static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,  in vhost_svq_last_desc_of_chain()  argument
434         i = svq->desc_next[i];  in vhost_svq_last_desc_of_chain()
441 static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,  in vhost_svq_get_buf()  argument
444     const vring_used_t *used = svq->vring.used;  in vhost_svq_get_buf()
448     if (!vhost_svq_more_used(svq)) {  in vhost_svq_get_buf()
454     last_used = svq->last_used_idx & (svq->vring.num - 1);  in vhost_svq_get_buf()
458     svq->last_used_idx++;  in vhost_svq_get_buf()
459     if (unlikely(used_elem.id >= svq->vring.num)) {  in vhost_svq_get_buf()
461                       svq->vdev->name, used_elem.id);  in vhost_svq_get_buf()
465     if (unlikely(!svq->desc_state[used_elem.id].ndescs)) {  in vhost_svq_get_buf()
468             svq->vdev->name, used_elem.id);  in vhost_svq_get_buf()
472     num = svq->desc_state[used_elem.id].ndescs;  in vhost_svq_get_buf()
473     svq->desc_state[used_elem.id].ndescs = 0;  in vhost_svq_get_buf()
474     last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id);  in vhost_svq_get_buf()
475     svq->desc_next[last_used_chain] = svq->free_head;  in vhost_svq_get_buf()
476     svq->free_head = used_elem.id;  in vhost_svq_get_buf()
477     svq->num_free += num;  in vhost_svq_get_buf()
480     return g_steal_pointer(&svq->desc_state[used_elem.id].elem);  in vhost_svq_get_buf()
486 void vhost_svq_push_elem(VhostShadowVirtqueue *svq,  in vhost_svq_push_elem()  argument
489     virtqueue_push(svq->vq, elem, len);  in vhost_svq_push_elem()
490     if (svq->next_guest_avail_elem) {  in vhost_svq_push_elem()
495         vhost_handle_guest_kick(svq);  in vhost_svq_push_elem()
499 static void vhost_svq_flush(VhostShadowVirtqueue *svq,  in vhost_svq_flush()  argument
502     VirtQueue *vq = svq->vq;  in vhost_svq_flush()
508         vhost_svq_disable_notification(svq);  in vhost_svq_flush()
511             g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);  in vhost_svq_flush()
516             if (unlikely(i >= svq->vring.num)) {  in vhost_svq_flush()
519                          i, svq->vring.num);  in vhost_svq_flush()
528         event_notifier_set(&svq->svq_call);  in vhost_svq_flush()
530         if (check_for_avail_queue && svq->next_guest_avail_elem) {  in vhost_svq_flush()
535             vhost_handle_guest_kick(svq);  in vhost_svq_flush()
537     } while (!vhost_svq_enable_notification(svq));  in vhost_svq_flush()
550 size_t vhost_svq_poll(VhostShadowVirtqueue *svq, size_t num)  in vhost_svq_poll()  argument
560             if (vhost_svq_more_used(svq)) {  in vhost_svq_poll()
569         elem = vhost_svq_get_buf(svq, &r);  in vhost_svq_poll()
586     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,  in vhost_svq_handle_call()  local
589     vhost_svq_flush(svq, true);  in vhost_svq_handle_call()
600 void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)  in vhost_svq_set_svq_call_fd()  argument
609         memset(&svq->svq_call, 0, sizeof(svq->svq_call));  in vhost_svq_set_svq_call_fd()
611         event_notifier_init_fd(&svq->svq_call, call_fd);  in vhost_svq_set_svq_call_fd()
620 void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,  in vhost_svq_get_vring_addr()  argument
623     addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;  in vhost_svq_get_vring_addr()
624     addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;  in vhost_svq_get_vring_addr()
625     addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;  in vhost_svq_get_vring_addr()
628 size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)  in vhost_svq_driver_area_size()  argument
630     size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;  in vhost_svq_driver_area_size()
631     size_t avail_size = offsetof(vring_avail_t, ring[svq->vring.num]) +  in vhost_svq_driver_area_size()
637 size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)  in vhost_svq_device_area_size()  argument
639     size_t used_size = offsetof(vring_used_t, ring[svq->vring.num]) +  in vhost_svq_device_area_size()
652 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)  in vhost_svq_set_svq_kick_fd()  argument
654     EventNotifier *svq_kick = &svq->svq_kick;  in vhost_svq_set_svq_kick_fd()
682 void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,  in vhost_svq_start()  argument
687     event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);  in vhost_svq_start()
688     svq->next_guest_avail_elem = NULL;  in vhost_svq_start()
689     svq->shadow_avail_idx = 0;  in vhost_svq_start()
690     svq->shadow_used_idx = 0;  in vhost_svq_start()
691     svq->last_used_idx = 0;  in vhost_svq_start()
692     svq->vdev = vdev;  in vhost_svq_start()
693     svq->vq = vq;  in vhost_svq_start()
694     svq->iova_tree = iova_tree;  in vhost_svq_start()
696     svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));  in vhost_svq_start()
697     svq->num_free = svq->vring.num;  in vhost_svq_start()
698     svq->vring.desc = mmap(NULL, vhost_svq_driver_area_size(svq),  in vhost_svq_start()
701     desc_size = sizeof(vring_desc_t) * svq->vring.num;  in vhost_svq_start()
702     svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);  in vhost_svq_start()
703     svq->vring.used = mmap(NULL, vhost_svq_device_area_size(svq),  in vhost_svq_start()
706     svq->desc_state = g_new0(SVQDescState, svq->vring.num);  in vhost_svq_start()
707     svq->desc_next = g_new0(uint16_t, svq->vring.num);  in vhost_svq_start()
708     for (unsigned i = 0; i < svq->vring.num - 1; i++) {  in vhost_svq_start()
709         svq->desc_next[i] = i + 1;  in vhost_svq_start()
717 void vhost_svq_stop(VhostShadowVirtqueue *svq)  in vhost_svq_stop()  argument
719     vhost_svq_set_svq_kick_fd(svq, VHOST_FILE_UNBIND);  in vhost_svq_stop()
722     if (!svq->vq) {  in vhost_svq_stop()
727     vhost_svq_flush(svq, false);  in vhost_svq_stop()
729     for (unsigned i = 0; i < svq->vring.num; ++i) {  in vhost_svq_stop()
731         elem = g_steal_pointer(&svq->desc_state[i].elem);  in vhost_svq_stop()
737             virtqueue_unpop(svq->vq, elem, 0);  in vhost_svq_stop()
741     next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);  in vhost_svq_stop()
743         virtqueue_unpop(svq->vq, next_avail_elem, 0);  in vhost_svq_stop()
745     svq->vq = NULL;  in vhost_svq_stop()
746     g_free(svq->desc_next);  in vhost_svq_stop()
747     g_free(svq->desc_state);  in vhost_svq_stop()
748     munmap(svq->vring.desc, vhost_svq_driver_area_size(svq));  in vhost_svq_stop()
749     munmap(svq->vring.used, vhost_svq_device_area_size(svq));  in vhost_svq_stop()
750     event_notifier_set_handler(&svq->hdev_call, NULL);  in vhost_svq_stop()
763     VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);  in vhost_svq_new()  local
765     event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);  in vhost_svq_new()
766     svq->ops = ops;  in vhost_svq_new()
767     svq->ops_opaque = ops_opaque;  in vhost_svq_new()
768     return svq;  in vhost_svq_new()