Lines Matching +full:no +full:- +full:map
4 * SPDX-FileCopyrightText: Red Hat, Inc. 2021
5 * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
7 * SPDX-License-Identifier: GPL-2.0-or-later
11 #include "hw/virtio/vhost-shadow-virtqueue.h"
13 #include "qemu/error-report.h"
15 #include "qemu/main-loop.h"
18 #include "linux-headers/linux/vhost.h"
71 return svq->num_free; in vhost_svq_available_slots()
94 const DMAMap *map; in vhost_svq_translate_addr() local
99 /* Search the GPA->IOVA tree */ in vhost_svq_translate_addr()
104 map = vhost_iova_tree_find_gpa(svq->iova_tree, &needle); in vhost_svq_translate_addr()
106 /* Search the IOVA->HVA tree */ in vhost_svq_translate_addr()
111 map = vhost_iova_tree_find_iova(svq->iova_tree, &needle); in vhost_svq_translate_addr()
115 * Map cannot be NULL since iova map contains all guest space and in vhost_svq_translate_addr()
118 if (unlikely(!map)) { in vhost_svq_translate_addr()
125 off = needle.translated_addr - map->translated_addr; in vhost_svq_translate_addr()
126 addrs[i] = map->iova + off; in vhost_svq_translate_addr()
129 int128_makes64(iovec[i].iov_len - 1)); in vhost_svq_translate_addr()
130 map_last = int128_make64(map->translated_addr + map->size); in vhost_svq_translate_addr()
159 uint16_t i = svq->free_head, last = svq->free_head; in vhost_svq_vring_write_descs()
162 vring_desc_t *descs = svq->vring.desc; in vhost_svq_vring_write_descs()
177 descs[i].next = cpu_to_le16(svq->desc_next[i]); in vhost_svq_vring_write_descs()
185 i = svq->desc_next[i]; in vhost_svq_vring_write_descs()
188 svq->free_head = svq->desc_next[last]; in vhost_svq_vring_write_descs()
199 vring_avail_t *avail = svq->vring.avail; in vhost_svq_add_split()
203 *head = svq->free_head; in vhost_svq_add_split()
208 "Guest provided element with no descriptors"); in vhost_svq_add_split()
225 * Put the entry in the available array (but don't update avail->idx until in vhost_svq_add_split()
228 avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1); in vhost_svq_add_split()
229 avail->ring[avail_idx] = cpu_to_le16(*head); in vhost_svq_add_split()
230 svq->shadow_avail_idx++; in vhost_svq_add_split()
234 avail->idx = cpu_to_le16(svq->shadow_avail_idx); in vhost_svq_add_split()
249 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in vhost_svq_kick()
251 *(uint16_t *)(&svq->vring.used->ring[svq->vring.num])); in vhost_svq_kick()
252 … needs_kick = vring_need_event(avail_event, svq->shadow_avail_idx, svq->shadow_avail_idx - 1); in vhost_svq_kick()
255 !(svq->vring.used->flags & cpu_to_le16(VRING_USED_F_NO_NOTIFY)); in vhost_svq_kick()
262 event_notifier_set(&svq->hdev_kick); in vhost_svq_kick()
268 * Return -EINVAL if element is invalid, -ENOSPC if dev queue is full
280 return -ENOSPC; in vhost_svq_add()
286 return -EINVAL; in vhost_svq_add()
289 svq->num_free -= ndescs; in vhost_svq_add()
290 svq->desc_state[qemu_head].elem = elem; in vhost_svq_add()
291 svq->desc_state[qemu_head].ndescs = ndescs; in vhost_svq_add()
300 return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->out_addr, in vhost_svq_add_element()
301 elem->in_sg, elem->in_num, elem->in_addr, elem); in vhost_svq_add_element()
320 event_notifier_test_and_clear(&svq->svq_kick); in vhost_handle_guest_kick()
324 virtio_queue_set_notification(svq->vq, false); in vhost_handle_guest_kick()
330 if (svq->next_guest_avail_elem) { in vhost_handle_guest_kick()
331 elem = g_steal_pointer(&svq->next_guest_avail_elem); in vhost_handle_guest_kick()
333 elem = virtqueue_pop(svq->vq, sizeof(*elem)); in vhost_handle_guest_kick()
340 if (svq->ops) { in vhost_handle_guest_kick()
341 r = svq->ops->avail_handler(svq, elem, svq->ops_opaque); in vhost_handle_guest_kick()
346 if (r == -ENOSPC) { in vhost_handle_guest_kick()
350 * scatter-gather segments. If that happens, the buffer in vhost_handle_guest_kick()
358 svq->next_guest_avail_elem = g_steal_pointer(&elem); in vhost_handle_guest_kick()
368 virtio_queue_set_notification(svq->vq, true); in vhost_handle_guest_kick()
369 } while (!virtio_queue_empty(svq->vq)); in vhost_handle_guest_kick()
386 uint16_t *used_idx = &svq->vring.used->idx; in vhost_svq_more_used()
387 if (svq->last_used_idx != svq->shadow_used_idx) { in vhost_svq_more_used()
391 svq->shadow_used_idx = le16_to_cpu(*(volatile uint16_t *)used_idx); in vhost_svq_more_used()
393 return svq->last_used_idx != svq->shadow_used_idx; in vhost_svq_more_used()
403 * callbacks. True if SVQ used vring has no more pending buffers.
407 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in vhost_svq_enable_notification()
408 uint16_t *used_event = (uint16_t *)&svq->vring.avail->ring[svq->vring.num]; in vhost_svq_enable_notification()
409 *used_event = cpu_to_le16(svq->shadow_used_idx); in vhost_svq_enable_notification()
411 svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT); in vhost_svq_enable_notification()
422 * No need to disable notification in the event idx case, since used event in vhost_svq_disable_notification()
425 if (!virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in vhost_svq_disable_notification()
426 svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT); in vhost_svq_disable_notification()
433 for (uint16_t j = 0; j < (num - 1); ++j) { in vhost_svq_last_desc_of_chain()
434 i = svq->desc_next[i]; in vhost_svq_last_desc_of_chain()
444 const vring_used_t *used = svq->vring.used; in vhost_svq_get_buf()
454 last_used = svq->last_used_idx & (svq->vring.num - 1); in vhost_svq_get_buf()
455 used_elem.id = le32_to_cpu(used->ring[last_used].id); in vhost_svq_get_buf()
456 used_elem.len = le32_to_cpu(used->ring[last_used].len); in vhost_svq_get_buf()
458 svq->last_used_idx++; in vhost_svq_get_buf()
459 if (unlikely(used_elem.id >= svq->vring.num)) { in vhost_svq_get_buf()
461 svq->vdev->name, used_elem.id); in vhost_svq_get_buf()
465 if (unlikely(!svq->desc_state[used_elem.id].ndescs)) { in vhost_svq_get_buf()
468 svq->vdev->name, used_elem.id); in vhost_svq_get_buf()
472 num = svq->desc_state[used_elem.id].ndescs; in vhost_svq_get_buf()
473 svq->desc_state[used_elem.id].ndescs = 0; in vhost_svq_get_buf()
475 svq->desc_next[last_used_chain] = svq->free_head; in vhost_svq_get_buf()
476 svq->free_head = used_elem.id; in vhost_svq_get_buf()
477 svq->num_free += num; in vhost_svq_get_buf()
480 return g_steal_pointer(&svq->desc_state[used_elem.id].elem); in vhost_svq_get_buf()
489 virtqueue_push(svq->vq, elem, len); in vhost_svq_push_elem()
490 if (svq->next_guest_avail_elem) { in vhost_svq_push_elem()
502 VirtQueue *vq = svq->vq; in vhost_svq_flush()
516 if (unlikely(i >= svq->vring.num)) { in vhost_svq_flush()
519 i, svq->vring.num); in vhost_svq_flush()
528 event_notifier_set(&svq->svq_call); in vhost_svq_flush()
530 if (check_for_avail_queue && svq->next_guest_avail_elem) { in vhost_svq_flush()
554 while (num--) { in vhost_svq_poll()
564 if (unlikely(g_get_monotonic_time() - start_us > 10e6)) { in vhost_svq_poll()
581 * Note that we are not making any buffers available in the loop, there is no
609 memset(&svq->svq_call, 0, sizeof(svq->svq_call)); in vhost_svq_set_svq_call_fd()
611 event_notifier_init_fd(&svq->svq_call, call_fd); in vhost_svq_set_svq_call_fd()
623 addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc; in vhost_svq_get_vring_addr()
624 addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail; in vhost_svq_get_vring_addr()
625 addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used; in vhost_svq_get_vring_addr()
630 size_t desc_size = sizeof(vring_desc_t) * svq->vring.num; in vhost_svq_driver_area_size()
631 size_t avail_size = offsetof(vring_avail_t, ring[svq->vring.num]) + in vhost_svq_driver_area_size()
639 size_t used_size = offsetof(vring_used_t, ring[svq->vring.num]) + in vhost_svq_device_area_size()
654 EventNotifier *svq_kick = &svq->svq_kick; in vhost_svq_set_svq_kick_fd()
665 * they arrive at the new file descriptor in the switch, so there is no in vhost_svq_set_svq_kick_fd()
687 event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call); in vhost_svq_start()
688 svq->next_guest_avail_elem = NULL; in vhost_svq_start()
689 svq->shadow_avail_idx = 0; in vhost_svq_start()
690 svq->shadow_used_idx = 0; in vhost_svq_start()
691 svq->last_used_idx = 0; in vhost_svq_start()
692 svq->vdev = vdev; in vhost_svq_start()
693 svq->vq = vq; in vhost_svq_start()
694 svq->iova_tree = iova_tree; in vhost_svq_start()
696 svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq)); in vhost_svq_start()
697 svq->num_free = svq->vring.num; in vhost_svq_start()
698 svq->vring.desc = mmap(NULL, vhost_svq_driver_area_size(svq), in vhost_svq_start()
700 -1, 0); in vhost_svq_start()
701 desc_size = sizeof(vring_desc_t) * svq->vring.num; in vhost_svq_start()
702 svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size); in vhost_svq_start()
703 svq->vring.used = mmap(NULL, vhost_svq_device_area_size(svq), in vhost_svq_start()
705 -1, 0); in vhost_svq_start()
706 svq->desc_state = g_new0(SVQDescState, svq->vring.num); in vhost_svq_start()
707 svq->desc_next = g_new0(uint16_t, svq->vring.num); in vhost_svq_start()
708 for (unsigned i = 0; i < svq->vring.num - 1; i++) { in vhost_svq_start()
709 svq->desc_next[i] = i + 1; in vhost_svq_start()
722 if (!svq->vq) { in vhost_svq_stop()
729 for (unsigned i = 0; i < svq->vring.num; ++i) { in vhost_svq_stop()
731 elem = g_steal_pointer(&svq->desc_state[i].elem); in vhost_svq_stop()
737 virtqueue_unpop(svq->vq, elem, 0); in vhost_svq_stop()
741 next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem); in vhost_svq_stop()
743 virtqueue_unpop(svq->vq, next_avail_elem, 0); in vhost_svq_stop()
745 svq->vq = NULL; in vhost_svq_stop()
746 g_free(svq->desc_next); in vhost_svq_stop()
747 g_free(svq->desc_state); in vhost_svq_stop()
748 munmap(svq->vring.desc, vhost_svq_driver_area_size(svq)); in vhost_svq_stop()
749 munmap(svq->vring.used, vhost_svq_device_area_size(svq)); in vhost_svq_stop()
750 event_notifier_set_handler(&svq->hdev_call, NULL); in vhost_svq_stop()
765 event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND); in vhost_svq_new()
766 svq->ops = ops; in vhost_svq_new()
767 svq->ops_opaque = ops_opaque; in vhost_svq_new()