Lines Matching refs:svq

69 uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)  in vhost_svq_available_slots()  argument
71 return svq->num_free; in vhost_svq_available_slots()
83 static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq, in vhost_svq_translate_addr() argument
104 map = vhost_iova_tree_find_gpa(svq->iova_tree, &needle); in vhost_svq_translate_addr()
111 map = vhost_iova_tree_find_iova(svq->iova_tree, &needle); in vhost_svq_translate_addr()
154 static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg, in vhost_svq_vring_write_descs() argument
159 uint16_t i = svq->free_head, last = svq->free_head; in vhost_svq_vring_write_descs()
162 vring_desc_t *descs = svq->vring.desc; in vhost_svq_vring_write_descs()
169 ok = vhost_svq_translate_addr(svq, sg, iovec, num, addr); in vhost_svq_vring_write_descs()
177 descs[i].next = cpu_to_le16(svq->desc_next[i]); in vhost_svq_vring_write_descs()
185 i = svq->desc_next[i]; in vhost_svq_vring_write_descs()
188 svq->free_head = svq->desc_next[last]; in vhost_svq_vring_write_descs()
192 static bool vhost_svq_add_split(VhostShadowVirtqueue *svq, in vhost_svq_add_split() argument
199 vring_avail_t *avail = svq->vring.avail; in vhost_svq_add_split()
203 *head = svq->free_head; in vhost_svq_add_split()
212 ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, out_addr, in vhost_svq_add_split()
218 ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, in_addr, false, in vhost_svq_add_split()
228 avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1); in vhost_svq_add_split()
230 svq->shadow_avail_idx++; in vhost_svq_add_split()
234 avail->idx = cpu_to_le16(svq->shadow_avail_idx); in vhost_svq_add_split()
239 static void vhost_svq_kick(VhostShadowVirtqueue *svq) in vhost_svq_kick() argument
249 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in vhost_svq_kick()
251 *(uint16_t *)(&svq->vring.used->ring[svq->vring.num])); in vhost_svq_kick()
252 … needs_kick = vring_need_event(avail_event, svq->shadow_avail_idx, svq->shadow_avail_idx - 1); in vhost_svq_kick()
255 !(svq->vring.used->flags & cpu_to_le16(VRING_USED_F_NO_NOTIFY)); in vhost_svq_kick()
262 event_notifier_set(&svq->hdev_kick); in vhost_svq_kick()
270 int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg, in vhost_svq_add() argument
279 if (unlikely(ndescs > vhost_svq_available_slots(svq))) { in vhost_svq_add()
283 ok = vhost_svq_add_split(svq, out_sg, out_num, out_addr, in_sg, in_num, in vhost_svq_add()
289 svq->num_free -= ndescs; in vhost_svq_add()
290 svq->desc_state[qemu_head].elem = elem; in vhost_svq_add()
291 svq->desc_state[qemu_head].ndescs = ndescs; in vhost_svq_add()
292 vhost_svq_kick(svq); in vhost_svq_add()
297 static int vhost_svq_add_element(VhostShadowVirtqueue *svq, in vhost_svq_add_element() argument
300 return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->out_addr, in vhost_svq_add_element()
317 static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq) in vhost_handle_guest_kick() argument
320 event_notifier_test_and_clear(&svq->svq_kick); in vhost_handle_guest_kick()
324 virtio_queue_set_notification(svq->vq, false); in vhost_handle_guest_kick()
330 if (svq->next_guest_avail_elem) { in vhost_handle_guest_kick()
331 elem = g_steal_pointer(&svq->next_guest_avail_elem); in vhost_handle_guest_kick()
333 elem = virtqueue_pop(svq->vq, sizeof(*elem)); in vhost_handle_guest_kick()
340 if (svq->ops) { in vhost_handle_guest_kick()
341 r = svq->ops->avail_handler(svq, elem, svq->ops_opaque); in vhost_handle_guest_kick()
343 r = vhost_svq_add_element(svq, elem); in vhost_handle_guest_kick()
358 svq->next_guest_avail_elem = g_steal_pointer(&elem); in vhost_handle_guest_kick()
368 virtio_queue_set_notification(svq->vq, true); in vhost_handle_guest_kick()
369 } while (!virtio_queue_empty(svq->vq)); in vhost_handle_guest_kick()
379 VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick); in vhost_handle_guest_kick_notifier() local
381 vhost_handle_guest_kick(svq); in vhost_handle_guest_kick_notifier()
384 static bool vhost_svq_more_used(VhostShadowVirtqueue *svq) in vhost_svq_more_used() argument
386 uint16_t *used_idx = &svq->vring.used->idx; in vhost_svq_more_used()
387 if (svq->last_used_idx != svq->shadow_used_idx) { in vhost_svq_more_used()
391 svq->shadow_used_idx = le16_to_cpu(*(volatile uint16_t *)used_idx); in vhost_svq_more_used()
393 return svq->last_used_idx != svq->shadow_used_idx; in vhost_svq_more_used()
405 static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq) in vhost_svq_enable_notification() argument
407 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in vhost_svq_enable_notification()
408 uint16_t *used_event = (uint16_t *)&svq->vring.avail->ring[svq->vring.num]; in vhost_svq_enable_notification()
409 *used_event = cpu_to_le16(svq->shadow_used_idx); in vhost_svq_enable_notification()
411 svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT); in vhost_svq_enable_notification()
416 return !vhost_svq_more_used(svq); in vhost_svq_enable_notification()
419 static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq) in vhost_svq_disable_notification() argument
425 if (!virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in vhost_svq_disable_notification()
426 svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT); in vhost_svq_disable_notification()
430 static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq, in vhost_svq_last_desc_of_chain() argument
434 i = svq->desc_next[i]; in vhost_svq_last_desc_of_chain()
441 static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq, in vhost_svq_get_buf() argument
444 const vring_used_t *used = svq->vring.used; in vhost_svq_get_buf()
448 if (!vhost_svq_more_used(svq)) { in vhost_svq_get_buf()
454 last_used = svq->last_used_idx & (svq->vring.num - 1); in vhost_svq_get_buf()
458 svq->last_used_idx++; in vhost_svq_get_buf()
459 if (unlikely(used_elem.id >= svq->vring.num)) { in vhost_svq_get_buf()
461 svq->vdev->name, used_elem.id); in vhost_svq_get_buf()
465 if (unlikely(!svq->desc_state[used_elem.id].ndescs)) { in vhost_svq_get_buf()
468 svq->vdev->name, used_elem.id); in vhost_svq_get_buf()
472 num = svq->desc_state[used_elem.id].ndescs; in vhost_svq_get_buf()
473 svq->desc_state[used_elem.id].ndescs = 0; in vhost_svq_get_buf()
474 last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id); in vhost_svq_get_buf()
475 svq->desc_next[last_used_chain] = svq->free_head; in vhost_svq_get_buf()
476 svq->free_head = used_elem.id; in vhost_svq_get_buf()
477 svq->num_free += num; in vhost_svq_get_buf()
480 return g_steal_pointer(&svq->desc_state[used_elem.id].elem); in vhost_svq_get_buf()
486 void vhost_svq_push_elem(VhostShadowVirtqueue *svq, in vhost_svq_push_elem() argument
489 virtqueue_push(svq->vq, elem, len); in vhost_svq_push_elem()
490 if (svq->next_guest_avail_elem) { in vhost_svq_push_elem()
495 vhost_handle_guest_kick(svq); in vhost_svq_push_elem()
499 static void vhost_svq_flush(VhostShadowVirtqueue *svq, in vhost_svq_flush() argument
502 VirtQueue *vq = svq->vq; in vhost_svq_flush()
508 vhost_svq_disable_notification(svq); in vhost_svq_flush()
511 g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len); in vhost_svq_flush()
516 if (unlikely(i >= svq->vring.num)) { in vhost_svq_flush()
519 i, svq->vring.num); in vhost_svq_flush()
528 event_notifier_set(&svq->svq_call); in vhost_svq_flush()
530 if (check_for_avail_queue && svq->next_guest_avail_elem) { in vhost_svq_flush()
535 vhost_handle_guest_kick(svq); in vhost_svq_flush()
537 } while (!vhost_svq_enable_notification(svq)); in vhost_svq_flush()
550 size_t vhost_svq_poll(VhostShadowVirtqueue *svq, size_t num) in vhost_svq_poll() argument
560 if (vhost_svq_more_used(svq)) { in vhost_svq_poll()
569 elem = vhost_svq_get_buf(svq, &r); in vhost_svq_poll()
586 VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, in vhost_svq_handle_call() local
589 vhost_svq_flush(svq, true); in vhost_svq_handle_call()
600 void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd) in vhost_svq_set_svq_call_fd() argument
609 memset(&svq->svq_call, 0, sizeof(svq->svq_call)); in vhost_svq_set_svq_call_fd()
611 event_notifier_init_fd(&svq->svq_call, call_fd); in vhost_svq_set_svq_call_fd()
620 void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq, in vhost_svq_get_vring_addr() argument
623 addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc; in vhost_svq_get_vring_addr()
624 addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail; in vhost_svq_get_vring_addr()
625 addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used; in vhost_svq_get_vring_addr()
628 size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq) in vhost_svq_driver_area_size() argument
630 size_t desc_size = sizeof(vring_desc_t) * svq->vring.num; in vhost_svq_driver_area_size()
631 size_t avail_size = offsetof(vring_avail_t, ring[svq->vring.num]) + in vhost_svq_driver_area_size()
637 size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq) in vhost_svq_device_area_size() argument
639 size_t used_size = offsetof(vring_used_t, ring[svq->vring.num]) + in vhost_svq_device_area_size()
652 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd) in vhost_svq_set_svq_kick_fd() argument
654 EventNotifier *svq_kick = &svq->svq_kick; in vhost_svq_set_svq_kick_fd()
682 void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev, in vhost_svq_start() argument
687 event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call); in vhost_svq_start()
688 svq->next_guest_avail_elem = NULL; in vhost_svq_start()
689 svq->shadow_avail_idx = 0; in vhost_svq_start()
690 svq->shadow_used_idx = 0; in vhost_svq_start()
691 svq->last_used_idx = 0; in vhost_svq_start()
692 svq->vdev = vdev; in vhost_svq_start()
693 svq->vq = vq; in vhost_svq_start()
694 svq->iova_tree = iova_tree; in vhost_svq_start()
696 svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq)); in vhost_svq_start()
697 svq->num_free = svq->vring.num; in vhost_svq_start()
698 svq->vring.desc = mmap(NULL, vhost_svq_driver_area_size(svq), in vhost_svq_start()
701 desc_size = sizeof(vring_desc_t) * svq->vring.num; in vhost_svq_start()
702 svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size); in vhost_svq_start()
703 svq->vring.used = mmap(NULL, vhost_svq_device_area_size(svq), in vhost_svq_start()
706 svq->desc_state = g_new0(SVQDescState, svq->vring.num); in vhost_svq_start()
707 svq->desc_next = g_new0(uint16_t, svq->vring.num); in vhost_svq_start()
708 for (unsigned i = 0; i < svq->vring.num - 1; i++) { in vhost_svq_start()
709 svq->desc_next[i] = i + 1; in vhost_svq_start()
717 void vhost_svq_stop(VhostShadowVirtqueue *svq) in vhost_svq_stop() argument
719 vhost_svq_set_svq_kick_fd(svq, VHOST_FILE_UNBIND); in vhost_svq_stop()
722 if (!svq->vq) { in vhost_svq_stop()
727 vhost_svq_flush(svq, false); in vhost_svq_stop()
729 for (unsigned i = 0; i < svq->vring.num; ++i) { in vhost_svq_stop()
731 elem = g_steal_pointer(&svq->desc_state[i].elem); in vhost_svq_stop()
737 virtqueue_unpop(svq->vq, elem, 0); in vhost_svq_stop()
741 next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem); in vhost_svq_stop()
743 virtqueue_unpop(svq->vq, next_avail_elem, 0); in vhost_svq_stop()
745 svq->vq = NULL; in vhost_svq_stop()
746 g_free(svq->desc_next); in vhost_svq_stop()
747 g_free(svq->desc_state); in vhost_svq_stop()
748 munmap(svq->vring.desc, vhost_svq_driver_area_size(svq)); in vhost_svq_stop()
749 munmap(svq->vring.used, vhost_svq_device_area_size(svq)); in vhost_svq_stop()
750 event_notifier_set_handler(&svq->hdev_call, NULL); in vhost_svq_stop()
763 VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1); in vhost_svq_new() local
765 event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND); in vhost_svq_new()
766 svq->ops = ops; in vhost_svq_new()
767 svq->ops_opaque = ops_opaque; in vhost_svq_new()
768 return svq; in vhost_svq_new()