108 	struct vring vring;  member
142 	} vring;  member
353 static bool vring_need_unmap_buffer(const struct vring_virtqueue *vring,  in vring_need_unmap_buffer()  argument
356 	return vring->use_map_api && (extra->addr != DMA_MAPPING_ERROR);  in vring_need_unmap_buffer()
639 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);  in virtqueue_add_split()
651 		desc = vq->split.vring.desc;  in virtqueue_add_split()
721 		virtqueue_add_desc_split(vq, vq->split.vring.desc,  in virtqueue_add_split()
734 		if (vq->free_head >= vq->split.vring.num)  in virtqueue_add_split()
735 			vq->free_head -= vq->split.vring.num;  in virtqueue_add_split()
751 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);  in virtqueue_add_split()
752 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(vq->vq.vdev, head);  in virtqueue_add_split()
758 	vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,  in virtqueue_add_split()
813 					vring_avail_event(&vq->split.vring)),  in virtqueue_kick_prepare_split()
816 		needs_kick = !(vq->split.vring.used->flags &  in virtqueue_kick_prepare_split()
870 	while (vq->split.vring.desc[i].flags & nextflag) {  in detach_buf_split_in_order()
901 			vq->split.vring.used->idx);  in virtqueue_poll_split()
941 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));  in virtqueue_get_buf_ctx_split()
943 			vq->split.vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx_split()
945 			vq->split.vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx_split()
947 	if (unlikely(i >= vq->split.vring.num)) {  in virtqueue_get_buf_ctx_split()
965 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split()
979 	unsigned int num = vq->split.vring.num;  in virtqueue_get_buf_ctx_split_in_order()
1007 				    vq->split.vring.used->ring[last_used_idx].id);  in virtqueue_get_buf_ctx_split_in_order()
1009 				     vq->split.vring.used->ring[last_used_idx].len);  in virtqueue_get_buf_ctx_split_in_order()
1035 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split_in_order()
1058 			vring_used_event(&vq->split.vring) = 0x0;  in virtqueue_disable_cb_split()
1060 			vq->split.vring.avail->flags =  in virtqueue_disable_cb_split()
1080 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_prepare_split()
1084 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(vq->vq.vdev,  in virtqueue_enable_cb_prepare_split()
1104 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_delayed_split()
1112 			&vring_used_event(&vq->split.vring),  in virtqueue_enable_cb_delayed_split()
1115 	if (unlikely((u16)(virtio16_to_cpu(vq->vq.vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
1132 	for (i = 0; i < vq->split.vring.num; i++) {  in virtqueue_detach_unused_buf_split()
1142 		vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,  in virtqueue_detach_unused_buf_split()
1148 	BUG_ON(vq->vq.num_free != vq->split.vring.num);  in virtqueue_detach_unused_buf_split()
1168 			vring_split->vring.avail->flags = cpu_to_virtio16(vdev,  in virtqueue_vring_init_split()
1177 	num = vq->split.vring.num;  in virtqueue_reset_split()
1179 	vq->split.vring.avail->flags = 0;  in virtqueue_reset_split()
1180 	vq->split.vring.avail->idx = 0;  in virtqueue_reset_split()
1183 	vq->split.vring.avail->ring[num] = 0;  in virtqueue_reset_split()
1185 	vq->split.vring.used->flags = 0;  in virtqueue_reset_split()
1186 	vq->split.vring.used->idx = 0;  in virtqueue_reset_split()
1189 	*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;  in virtqueue_reset_split()
1210 	u32 num = vring_split->vring.num;  in vring_alloc_state_extra_split()
1237 			 vring_split->vring.desc,  in vring_free_split()
1285 	vring_init(&vring_split->vring, num, queue, vring_align);  in vring_alloc_queue_split()
1348 	virtqueue_init(vq, vring_split->vring.num);  in __vring_new_virtqueue_split()
1412 	virtqueue_init(vq, vring_split.vring.num);  in virtqueue_resize_split()
1551 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);  in virtqueue_add_indirect_packed()
1552 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *  in virtqueue_add_indirect_packed()
1554 	vq->packed.vring.desc[head].id = cpu_to_le16(id);  in virtqueue_add_indirect_packed()
1570 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1578 	if (n >= vq->packed.vring.num) {  in virtqueue_add_indirect_packed()
1649 		BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1664 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed()
1666 	desc = vq->packed.vring.desc;  in virtqueue_add_packed()
1678 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1713 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed()
1744 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed()
1765 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed()
1823 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed_in_order()
1825 	desc = vq->packed.vring.desc;  in virtqueue_add_packed_in_order()
1871 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed_in_order()
1902 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed_in_order()
1921 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed_in_order()
1953 	snapshot.u32 = *(u32 *)vq->packed.vring.device;  in virtqueue_kick_prepare_packed()
1969 		event_idx -= vq->packed.vring.num;  in virtqueue_kick_prepare_packed()
2042 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);  in is_used_desc_packed()
2071 	if (unlikely(last_used >= vq->packed.vring.num)) {  in update_last_used_idx_packed()
2072 		last_used -= vq->packed.vring.num;  in update_last_used_idx_packed()
2086 				&vq->packed.vring.driver->off_wrap,  in update_last_used_idx_packed()
2102 	unsigned int num = vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed_in_order()
2127 			le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed_in_order()
2129 			le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed_in_order()
2165 	unsigned int num = vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed()
2189 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed()
2190 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed()
2225 		vq->packed.vring.driver->flags =  in virtqueue_disable_cb_packed()
2240 		vq->packed.vring.driver->off_wrap =  in virtqueue_enable_cb_prepare_packed()
2253 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_prepare_packed()
2275 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;  in virtqueue_enable_cb_delayed_packed()
2280 		if (used_idx >= vq->packed.vring.num) {  in virtqueue_enable_cb_delayed_packed()
2281 			used_idx -= vq->packed.vring.num;  in virtqueue_enable_cb_delayed_packed()
2285 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |  in virtqueue_enable_cb_delayed_packed()
2299 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_delayed_packed()
2328 	for (i = 0; i < vq->packed.vring.num; i++) {  in virtqueue_detach_unused_buf_packed()
2341 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);  in virtqueue_detach_unused_buf_packed()
2370 	if (vring_packed->vring.desc)  in vring_free_packed()
2372 				 vring_packed->vring.desc,  in vring_free_packed()
2376 	if (vring_packed->vring.driver)  in vring_free_packed()
2378 				 vring_packed->vring.driver,  in vring_free_packed()
2382 	if (vring_packed->vring.device)  in vring_free_packed()
2384 				 vring_packed->vring.device,  in vring_free_packed()
2410 	vring_packed->vring.desc         = ring;  in vring_alloc_queue_packed()
2423 	vring_packed->vring.driver          = driver;  in vring_alloc_queue_packed()
2434 	vring_packed->vring.device          = device;  in vring_alloc_queue_packed()
2437 	vring_packed->vring.num = num;  in vring_alloc_queue_packed()
2450 	u32 num = vring_packed->vring.num;  in vring_alloc_state_extra_packed()
2484 		vring_packed->vring.driver->flags =  in virtqueue_vring_init_packed()
2507 	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);  in virtqueue_reset_packed()
2508 	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);  in virtqueue_reset_packed()
2511 	memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);  in virtqueue_reset_packed()
2512 	virtqueue_init(vq, vq->packed.vring.num);  in virtqueue_reset_packed()
2568 	virtqueue_init(vq, vring_packed->vring.num);  in __vring_new_virtqueue_packed()
2625 	virtqueue_init(vq, vring_packed.vring.num);  in virtqueue_resize_packed()
3418 		vring_packed.vring.num = num;  in vring_new_virtqueue()
3419 		vring_packed.vring.desc = pages;  in vring_new_virtqueue()
3426 	vring_init(&vring_split.vring, num, pages, vring_align);  in vring_new_virtqueue()
3441 					 vq->packed.vring.desc,  in vring_free()
3447 					 vq->packed.vring.driver,  in vring_free()
3453 					 vq->packed.vring.device,  in vring_free()
3462 					 vq->split.vring.desc,  in vring_free()
3547 	return virtqueue_is_packed(vq) ? vq->packed.vring.num :  in virtqueue_get_vring_size()
3548 				      vq->split.vring.num;  in virtqueue_get_vring_size()
3648 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);  in virtqueue_get_avail_addr()
3662 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);  in virtqueue_get_used_addr()
3667 const struct vring *virtqueue_get_vring(const struct virtqueue *vq)  in virtqueue_get_vring()
3669 	return &to_vvq(vq)->split.vring;  in virtqueue_get_vring()