1 /* 2 * DMA helper functions 3 * 4 * Copyright (c) 2009,2020 Red Hat 5 * 6 * This work is licensed under the terms of the GNU General Public License 7 * (GNU GPL), version 2 or later. 8 */ 9 10 #include "qemu/osdep.h" 11 #include "sysemu/block-backend.h" 12 #include "sysemu/dma.h" 13 #include "trace/trace-root.h" 14 #include "qemu/thread.h" 15 #include "qemu/main-loop.h" 16 #include "sysemu/cpu-timers.h" 17 #include "qemu/range.h" 18 19 /* #define DEBUG_IOMMU */ 20 21 MemTxResult dma_memory_set(AddressSpace *as, dma_addr_t addr, 22 uint8_t c, dma_addr_t len, MemTxAttrs attrs) 23 { 24 dma_barrier(as, DMA_DIRECTION_FROM_DEVICE); 25 26 #define FILLBUF_SIZE 512 27 uint8_t fillbuf[FILLBUF_SIZE]; 28 int l; 29 MemTxResult error = MEMTX_OK; 30 31 memset(fillbuf, c, FILLBUF_SIZE); 32 while (len > 0) { 33 l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE; 34 error |= address_space_write(as, addr, attrs, fillbuf, l); 35 len -= l; 36 addr += l; 37 } 38 39 return error; 40 } 41 42 void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint, 43 AddressSpace *as) 44 { 45 qsg->sg = g_malloc(alloc_hint * sizeof(ScatterGatherEntry)); 46 qsg->nsg = 0; 47 qsg->nalloc = alloc_hint; 48 qsg->size = 0; 49 qsg->as = as; 50 qsg->dev = dev; 51 object_ref(OBJECT(dev)); 52 } 53 54 void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len) 55 { 56 if (qsg->nsg == qsg->nalloc) { 57 qsg->nalloc = 2 * qsg->nalloc + 1; 58 qsg->sg = g_realloc(qsg->sg, qsg->nalloc * sizeof(ScatterGatherEntry)); 59 } 60 qsg->sg[qsg->nsg].base = base; 61 qsg->sg[qsg->nsg].len = len; 62 qsg->size += len; 63 ++qsg->nsg; 64 } 65 66 void qemu_sglist_destroy(QEMUSGList *qsg) 67 { 68 object_unref(OBJECT(qsg->dev)); 69 g_free(qsg->sg); 70 memset(qsg, 0, sizeof(*qsg)); 71 } 72 73 typedef struct { 74 BlockAIOCB common; 75 AioContext *ctx; 76 BlockAIOCB *acb; 77 QEMUSGList *sg; 78 uint32_t align; 79 uint64_t offset; 80 DMADirection dir; 81 int sg_cur_index; 82 dma_addr_t sg_cur_byte; 83 QEMUIOVector iov; 84 QEMUBH *bh; 85 DMAIOFunc *io_func; 86 void *io_func_opaque; 87 } DMAAIOCB; 88 89 static void dma_blk_cb(void *opaque, int ret); 90 91 static void reschedule_dma(void *opaque) 92 { 93 DMAAIOCB *dbs = (DMAAIOCB *)opaque; 94 95 assert(!dbs->acb && dbs->bh); 96 qemu_bh_delete(dbs->bh); 97 dbs->bh = NULL; 98 dma_blk_cb(dbs, 0); 99 } 100 101 static void dma_blk_unmap(DMAAIOCB *dbs) 102 { 103 int i; 104 105 for (i = 0; i < dbs->iov.niov; ++i) { 106 dma_memory_unmap(dbs->sg->as, dbs->iov.iov[i].iov_base, 107 dbs->iov.iov[i].iov_len, dbs->dir, 108 dbs->iov.iov[i].iov_len); 109 } 110 qemu_iovec_reset(&dbs->iov); 111 } 112 113 static void dma_complete(DMAAIOCB *dbs, int ret) 114 { 115 trace_dma_complete(dbs, ret, dbs->common.cb); 116 117 assert(!dbs->acb && !dbs->bh); 118 dma_blk_unmap(dbs); 119 if (dbs->common.cb) { 120 dbs->common.cb(dbs->common.opaque, ret); 121 } 122 qemu_iovec_destroy(&dbs->iov); 123 qemu_aio_unref(dbs); 124 } 125 126 static void dma_blk_cb(void *opaque, int ret) 127 { 128 DMAAIOCB *dbs = (DMAAIOCB *)opaque; 129 dma_addr_t cur_addr, cur_len; 130 void *mem; 131 132 trace_dma_blk_cb(dbs, ret); 133 134 dbs->acb = NULL; 135 dbs->offset += dbs->iov.size; 136 137 if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) { 138 dma_complete(dbs, ret); 139 return; 140 } 141 dma_blk_unmap(dbs); 142 143 while (dbs->sg_cur_index < dbs->sg->nsg) { 144 cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte; 145 cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte; 146 mem = dma_memory_map(dbs->sg->as, cur_addr, &cur_len, dbs->dir); 147 /* 148 * Make reads deterministic in icount mode. Windows sometimes issues 149 * disk read requests with overlapping SGs. It leads 150 * to non-determinism, because resulting buffer contents may be mixed 151 * from several sectors. This code splits all SGs into several 152 * groups. SGs in every group do not overlap. 153 */ 154 if (mem && icount_enabled() && dbs->dir == DMA_DIRECTION_FROM_DEVICE) { 155 int i; 156 for (i = 0 ; i < dbs->iov.niov ; ++i) { 157 if (ranges_overlap((intptr_t)dbs->iov.iov[i].iov_base, 158 dbs->iov.iov[i].iov_len, (intptr_t)mem, 159 cur_len)) { 160 dma_memory_unmap(dbs->sg->as, mem, cur_len, 161 dbs->dir, cur_len); 162 mem = NULL; 163 break; 164 } 165 } 166 } 167 if (!mem) 168 break; 169 qemu_iovec_add(&dbs->iov, mem, cur_len); 170 dbs->sg_cur_byte += cur_len; 171 if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) { 172 dbs->sg_cur_byte = 0; 173 ++dbs->sg_cur_index; 174 } 175 } 176 177 if (dbs->iov.size == 0) { 178 trace_dma_map_wait(dbs); 179 dbs->bh = aio_bh_new(dbs->ctx, reschedule_dma, dbs); 180 cpu_register_map_client(dbs->bh); 181 return; 182 } 183 184 if (!QEMU_IS_ALIGNED(dbs->iov.size, dbs->align)) { 185 qemu_iovec_discard_back(&dbs->iov, 186 QEMU_ALIGN_DOWN(dbs->iov.size, dbs->align)); 187 } 188 189 aio_context_acquire(dbs->ctx); 190 dbs->acb = dbs->io_func(dbs->offset, &dbs->iov, 191 dma_blk_cb, dbs, dbs->io_func_opaque); 192 aio_context_release(dbs->ctx); 193 assert(dbs->acb); 194 } 195 196 static void dma_aio_cancel(BlockAIOCB *acb) 197 { 198 DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); 199 200 trace_dma_aio_cancel(dbs); 201 202 assert(!(dbs->acb && dbs->bh)); 203 if (dbs->acb) { 204 /* This will invoke dma_blk_cb. */ 205 blk_aio_cancel_async(dbs->acb); 206 return; 207 } 208 209 if (dbs->bh) { 210 cpu_unregister_map_client(dbs->bh); 211 qemu_bh_delete(dbs->bh); 212 dbs->bh = NULL; 213 } 214 if (dbs->common.cb) { 215 dbs->common.cb(dbs->common.opaque, -ECANCELED); 216 } 217 } 218 219 static AioContext *dma_get_aio_context(BlockAIOCB *acb) 220 { 221 DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); 222 223 return dbs->ctx; 224 } 225 226 static const AIOCBInfo dma_aiocb_info = { 227 .aiocb_size = sizeof(DMAAIOCB), 228 .cancel_async = dma_aio_cancel, 229 .get_aio_context = dma_get_aio_context, 230 }; 231 232 BlockAIOCB *dma_blk_io(AioContext *ctx, 233 QEMUSGList *sg, uint64_t offset, uint32_t align, 234 DMAIOFunc *io_func, void *io_func_opaque, 235 BlockCompletionFunc *cb, 236 void *opaque, DMADirection dir) 237 { 238 DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, NULL, cb, opaque); 239 240 trace_dma_blk_io(dbs, io_func_opaque, offset, (dir == DMA_DIRECTION_TO_DEVICE)); 241 242 dbs->acb = NULL; 243 dbs->sg = sg; 244 dbs->ctx = ctx; 245 dbs->offset = offset; 246 dbs->align = align; 247 dbs->sg_cur_index = 0; 248 dbs->sg_cur_byte = 0; 249 dbs->dir = dir; 250 dbs->io_func = io_func; 251 dbs->io_func_opaque = io_func_opaque; 252 dbs->bh = NULL; 253 qemu_iovec_init(&dbs->iov, sg->nsg); 254 dma_blk_cb(dbs, 0); 255 return &dbs->common; 256 } 257 258 259 static 260 BlockAIOCB *dma_blk_read_io_func(int64_t offset, QEMUIOVector *iov, 261 BlockCompletionFunc *cb, void *cb_opaque, 262 void *opaque) 263 { 264 BlockBackend *blk = opaque; 265 return blk_aio_preadv(blk, offset, iov, 0, cb, cb_opaque); 266 } 267 268 BlockAIOCB *dma_blk_read(BlockBackend *blk, 269 QEMUSGList *sg, uint64_t offset, uint32_t align, 270 void (*cb)(void *opaque, int ret), void *opaque) 271 { 272 return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, 273 dma_blk_read_io_func, blk, cb, opaque, 274 DMA_DIRECTION_FROM_DEVICE); 275 } 276 277 static 278 BlockAIOCB *dma_blk_write_io_func(int64_t offset, QEMUIOVector *iov, 279 BlockCompletionFunc *cb, void *cb_opaque, 280 void *opaque) 281 { 282 BlockBackend *blk = opaque; 283 return blk_aio_pwritev(blk, offset, iov, 0, cb, cb_opaque); 284 } 285 286 BlockAIOCB *dma_blk_write(BlockBackend *blk, 287 QEMUSGList *sg, uint64_t offset, uint32_t align, 288 void (*cb)(void *opaque, int ret), void *opaque) 289 { 290 return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, 291 dma_blk_write_io_func, blk, cb, opaque, 292 DMA_DIRECTION_TO_DEVICE); 293 } 294 295 296 static uint64_t dma_buf_rw(uint8_t *ptr, int32_t len, QEMUSGList *sg, 297 DMADirection dir) 298 { 299 uint64_t resid; 300 int sg_cur_index; 301 302 resid = sg->size; 303 sg_cur_index = 0; 304 len = MIN(len, resid); 305 while (len > 0) { 306 ScatterGatherEntry entry = sg->sg[sg_cur_index++]; 307 int32_t xfer = MIN(len, entry.len); 308 dma_memory_rw(sg->as, entry.base, ptr, xfer, dir, 309 MEMTXATTRS_UNSPECIFIED); 310 ptr += xfer; 311 len -= xfer; 312 resid -= xfer; 313 } 314 315 return resid; 316 } 317 318 uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg) 319 { 320 return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_FROM_DEVICE); 321 } 322 323 uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg) 324 { 325 return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_TO_DEVICE); 326 } 327 328 void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie, 329 QEMUSGList *sg, enum BlockAcctType type) 330 { 331 block_acct_start(blk_get_stats(blk), cookie, sg->size, type); 332 } 333 334 uint64_t dma_aligned_pow2_mask(uint64_t start, uint64_t end, int max_addr_bits) 335 { 336 uint64_t max_mask = UINT64_MAX, addr_mask = end - start; 337 uint64_t alignment_mask, size_mask; 338 339 if (max_addr_bits != 64) { 340 max_mask = (1ULL << max_addr_bits) - 1; 341 } 342 343 alignment_mask = start ? (start & -start) - 1 : max_mask; 344 alignment_mask = MIN(alignment_mask, max_mask); 345 size_mask = MIN(addr_mask, max_mask); 346 347 if (alignment_mask <= size_mask) { 348 /* Increase the alignment of start */ 349 return alignment_mask; 350 } else { 351 /* Find the largest page mask from size */ 352 if (addr_mask == UINT64_MAX) { 353 return UINT64_MAX; 354 } 355 return (1ULL << (63 - clz64(addr_mask + 1))) - 1; 356 } 357 } 358 359