Lines Matching full:packed
90 /* Is this a packed ring? */
139 /* Available for packed ring */
177 } packed; member
911 * Packed ring specific functions - *_packed().
993 head = vq->packed.next_avail_idx; in virtqueue_add_indirect_packed()
1005 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_indirect_packed()
1029 vq->packed.vring.desc[head].addr = cpu_to_le64(addr); in virtqueue_add_indirect_packed()
1030 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg * in virtqueue_add_indirect_packed()
1032 vq->packed.vring.desc[head].id = cpu_to_le16(id); in virtqueue_add_indirect_packed()
1035 vq->packed.desc_extra[id].addr = addr; in virtqueue_add_indirect_packed()
1036 vq->packed.desc_extra[id].len = total_sg * in virtqueue_add_indirect_packed()
1038 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1039 vq->packed.avail_used_flags; in virtqueue_add_indirect_packed()
1048 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1049 vq->packed.avail_used_flags); in virtqueue_add_indirect_packed()
1056 if (n >= vq->packed.vring.num) { in virtqueue_add_indirect_packed()
1058 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_indirect_packed()
1059 vq->packed.avail_used_flags ^= in virtqueue_add_indirect_packed()
1063 vq->packed.next_avail_idx = n; in virtqueue_add_indirect_packed()
1064 vq->free_head = vq->packed.desc_state[id].next; in virtqueue_add_indirect_packed()
1067 vq->packed.desc_state[id].num = 1; in virtqueue_add_indirect_packed()
1068 vq->packed.desc_state[id].data = data; in virtqueue_add_indirect_packed()
1069 vq->packed.desc_state[id].indir_desc = desc; in virtqueue_add_indirect_packed()
1070 vq->packed.desc_state[id].last = id; in virtqueue_add_indirect_packed()
1125 head = vq->packed.next_avail_idx; in virtqueue_add_packed()
1126 avail_used_flags = vq->packed.avail_used_flags; in virtqueue_add_packed()
1128 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect); in virtqueue_add_packed()
1130 desc = vq->packed.vring.desc; in virtqueue_add_packed()
1142 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_packed()
1153 flags = cpu_to_le16(vq->packed.avail_used_flags | in virtqueue_add_packed()
1166 vq->packed.desc_extra[curr].addr = addr; in virtqueue_add_packed()
1167 vq->packed.desc_extra[curr].len = sg->length; in virtqueue_add_packed()
1168 vq->packed.desc_extra[curr].flags = in virtqueue_add_packed()
1172 curr = vq->packed.desc_state[curr].next; in virtqueue_add_packed()
1174 if ((unlikely(++i >= vq->packed.vring.num))) { in virtqueue_add_packed()
1176 vq->packed.avail_used_flags ^= in virtqueue_add_packed()
1184 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_packed()
1190 vq->packed.next_avail_idx = i; in virtqueue_add_packed()
1194 vq->packed.desc_state[id].num = descs_used; in virtqueue_add_packed()
1195 vq->packed.desc_state[id].data = data; in virtqueue_add_packed()
1196 vq->packed.desc_state[id].indir_desc = ctx; in virtqueue_add_packed()
1197 vq->packed.desc_state[id].last = prev; in virtqueue_add_packed()
1205 vq->packed.vring.desc[head].flags = head_flags; in virtqueue_add_packed()
1217 vq->packed.avail_used_flags = avail_used_flags; in virtqueue_add_packed()
1224 if (i >= vq->packed.vring.num) in virtqueue_add_packed()
1253 old = vq->packed.next_avail_idx - vq->num_added; in virtqueue_kick_prepare_packed()
1254 new = vq->packed.next_avail_idx; in virtqueue_kick_prepare_packed()
1257 snapshot.u32 = *(u32 *)vq->packed.vring.device; in virtqueue_kick_prepare_packed()
1272 if (wrap_counter != vq->packed.avail_wrap_counter) in virtqueue_kick_prepare_packed()
1273 event_idx -= vq->packed.vring.num; in virtqueue_kick_prepare_packed()
1288 state = &vq->packed.desc_state[id]; in detach_buf_packed()
1293 vq->packed.desc_state[state->last].next = vq->free_head; in detach_buf_packed()
1301 &vq->packed.desc_extra[curr]); in detach_buf_packed()
1302 curr = vq->packed.desc_state[curr].next; in detach_buf_packed()
1315 len = vq->packed.desc_extra[id].len; in detach_buf_packed()
1333 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags); in is_used_desc_packed()
1343 vq->packed.used_wrap_counter); in more_used_packed()
1371 id = le16_to_cpu(vq->packed.vring.desc[last_used].id); in virtqueue_get_buf_ctx_packed()
1372 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len); in virtqueue_get_buf_ctx_packed()
1374 if (unlikely(id >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1378 if (unlikely(!vq->packed.desc_state[id].data)) { in virtqueue_get_buf_ctx_packed()
1384 ret = vq->packed.desc_state[id].data; in virtqueue_get_buf_ctx_packed()
1387 vq->last_used_idx += vq->packed.desc_state[id].num; in virtqueue_get_buf_ctx_packed()
1388 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1389 vq->last_used_idx -= vq->packed.vring.num; in virtqueue_get_buf_ctx_packed()
1390 vq->packed.used_wrap_counter ^= 1; in virtqueue_get_buf_ctx_packed()
1398 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC) in virtqueue_get_buf_ctx_packed()
1400 &vq->packed.vring.driver->off_wrap, in virtqueue_get_buf_ctx_packed()
1402 (vq->packed.used_wrap_counter << in virtqueue_get_buf_ctx_packed()
1415 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_disable_cb_packed()
1416 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; in virtqueue_disable_cb_packed()
1417 vq->packed.vring.driver->flags = in virtqueue_disable_cb_packed()
1418 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_disable_cb_packed()
1434 vq->packed.vring.driver->off_wrap = in virtqueue_enable_cb_prepare_packed()
1436 (vq->packed.used_wrap_counter << in virtqueue_enable_cb_prepare_packed()
1445 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_prepare_packed()
1446 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_prepare_packed()
1449 vq->packed.vring.driver->flags = in virtqueue_enable_cb_prepare_packed()
1450 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_prepare_packed()
1454 return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter << in virtqueue_enable_cb_prepare_packed()
1485 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4; in virtqueue_enable_cb_delayed_packed()
1486 wrap_counter = vq->packed.used_wrap_counter; in virtqueue_enable_cb_delayed_packed()
1489 if (used_idx >= vq->packed.vring.num) { in virtqueue_enable_cb_delayed_packed()
1490 used_idx -= vq->packed.vring.num; in virtqueue_enable_cb_delayed_packed()
1494 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx | in virtqueue_enable_cb_delayed_packed()
1504 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_delayed_packed()
1505 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_delayed_packed()
1508 vq->packed.vring.driver->flags = in virtqueue_enable_cb_delayed_packed()
1509 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_delayed_packed()
1520 vq->packed.used_wrap_counter)) { in virtqueue_enable_cb_delayed_packed()
1537 for (i = 0; i < vq->packed.vring.num; i++) { in virtqueue_detach_unused_buf_packed()
1538 if (!vq->packed.desc_state[i].data) in virtqueue_detach_unused_buf_packed()
1541 buf = vq->packed.desc_state[i].data; in virtqueue_detach_unused_buf_packed()
1547 BUG_ON(vq->vq.num_free != vq->packed.vring.num); in virtqueue_detach_unused_buf_packed()
1624 vq->packed.ring_dma_addr = ring_dma_addr; in vring_create_virtqueue_packed()
1625 vq->packed.driver_event_dma_addr = driver_event_dma_addr; in vring_create_virtqueue_packed()
1626 vq->packed.device_event_dma_addr = device_event_dma_addr; in vring_create_virtqueue_packed()
1628 vq->packed.ring_size_in_bytes = ring_size_in_bytes; in vring_create_virtqueue_packed()
1629 vq->packed.event_size_in_bytes = event_size_in_bytes; in vring_create_virtqueue_packed()
1631 vq->packed.vring.num = num; in vring_create_virtqueue_packed()
1632 vq->packed.vring.desc = ring; in vring_create_virtqueue_packed()
1633 vq->packed.vring.driver = driver; in vring_create_virtqueue_packed()
1634 vq->packed.vring.device = device; in vring_create_virtqueue_packed()
1636 vq->packed.next_avail_idx = 0; in vring_create_virtqueue_packed()
1637 vq->packed.avail_wrap_counter = 1; in vring_create_virtqueue_packed()
1638 vq->packed.used_wrap_counter = 1; in vring_create_virtqueue_packed()
1639 vq->packed.event_flags_shadow = 0; in vring_create_virtqueue_packed()
1640 vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; in vring_create_virtqueue_packed()
1642 vq->packed.desc_state = kmalloc_array(num, in vring_create_virtqueue_packed()
1645 if (!vq->packed.desc_state) in vring_create_virtqueue_packed()
1648 memset(vq->packed.desc_state, 0, in vring_create_virtqueue_packed()
1654 vq->packed.desc_state[i].next = i + 1; in vring_create_virtqueue_packed()
1656 vq->packed.desc_extra = kmalloc_array(num, in vring_create_virtqueue_packed()
1659 if (!vq->packed.desc_extra) in vring_create_virtqueue_packed()
1662 memset(vq->packed.desc_extra, 0, in vring_create_virtqueue_packed()
1667 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; in vring_create_virtqueue_packed()
1668 vq->packed.vring.driver->flags = in vring_create_virtqueue_packed()
1669 cpu_to_le16(vq->packed.event_flags_shadow); in vring_create_virtqueue_packed()
1675 kfree(vq->packed.desc_state); in vring_create_virtqueue_packed()
2188 vq->packed.ring_size_in_bytes, in vring_del_virtqueue()
2189 vq->packed.vring.desc, in vring_del_virtqueue()
2190 vq->packed.ring_dma_addr); in vring_del_virtqueue()
2193 vq->packed.event_size_in_bytes, in vring_del_virtqueue()
2194 vq->packed.vring.driver, in vring_del_virtqueue()
2195 vq->packed.driver_event_dma_addr); in vring_del_virtqueue()
2198 vq->packed.event_size_in_bytes, in vring_del_virtqueue()
2199 vq->packed.vring.device, in vring_del_virtqueue()
2200 vq->packed.device_event_dma_addr); in vring_del_virtqueue()
2202 kfree(vq->packed.desc_state); in vring_del_virtqueue()
2203 kfree(vq->packed.desc_extra); in vring_del_virtqueue()
2257 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2291 return vq->packed.ring_dma_addr; in virtqueue_get_desc_addr()
2304 return vq->packed.driver_event_dma_addr; in virtqueue_get_avail_addr()
2318 return vq->packed.device_event_dma_addr; in virtqueue_get_used_addr()