Lines Matching full:dir
156 size_t offset, size_t size, enum dma_data_direction dir, in dma_map_page_attrs() argument
162 BUG_ON(!valid_dma_direction(dir)); in dma_map_page_attrs()
169 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
171 addr = iommu_dma_map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
173 addr = ops->map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
174 kmsan_handle_dma(page, offset, size, dir); in dma_map_page_attrs()
175 trace_dma_map_page(dev, page_to_phys(page) + offset, addr, size, dir, in dma_map_page_attrs()
177 debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); in dma_map_page_attrs()
184 enum dma_data_direction dir, unsigned long attrs) in dma_unmap_page_attrs() argument
188 BUG_ON(!valid_dma_direction(dir)); in dma_unmap_page_attrs()
191 dma_direct_unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
193 iommu_dma_unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
195 ops->unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
196 trace_dma_unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
197 debug_dma_unmap_page(dev, addr, size, dir); in dma_unmap_page_attrs()
202 int nents, enum dma_data_direction dir, unsigned long attrs) in __dma_map_sg_attrs() argument
207 BUG_ON(!valid_dma_direction(dir)); in __dma_map_sg_attrs()
214 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); in __dma_map_sg_attrs()
216 ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); in __dma_map_sg_attrs()
218 ents = ops->map_sg(dev, sg, nents, dir, attrs); in __dma_map_sg_attrs()
221 kmsan_handle_dma_sg(sg, nents, dir); in __dma_map_sg_attrs()
222 trace_dma_map_sg(dev, sg, nents, ents, dir, attrs); in __dma_map_sg_attrs()
223 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); in __dma_map_sg_attrs()
226 trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs); in __dma_map_sg_attrs()
238 * @dir: DMA direction
242 * nents segments for the @dir DMA operation by the @dev device.
251 int nents, enum dma_data_direction dir, unsigned long attrs) in dma_map_sg_attrs() argument
255 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); in dma_map_sg_attrs()
266 * @dir: DMA direction
270 * object for the @dir DMA operation by the @dev device. After success, the
290 enum dma_data_direction dir, unsigned long attrs) in dma_map_sgtable() argument
294 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); in dma_map_sgtable()
303 int nents, enum dma_data_direction dir, in dma_unmap_sg_attrs() argument
308 BUG_ON(!valid_dma_direction(dir)); in dma_unmap_sg_attrs()
309 trace_dma_unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
310 debug_dma_unmap_sg(dev, sg, nents, dir); in dma_unmap_sg_attrs()
313 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
315 iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
317 ops->unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
322 size_t size, enum dma_data_direction dir, unsigned long attrs) in dma_map_resource() argument
327 BUG_ON(!valid_dma_direction(dir)); in dma_map_resource()
333 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); in dma_map_resource()
335 addr = iommu_dma_map_resource(dev, phys_addr, size, dir, attrs); in dma_map_resource()
337 addr = ops->map_resource(dev, phys_addr, size, dir, attrs); in dma_map_resource()
339 trace_dma_map_resource(dev, phys_addr, addr, size, dir, attrs); in dma_map_resource()
340 debug_dma_map_resource(dev, phys_addr, size, dir, addr, attrs); in dma_map_resource()
346 enum dma_data_direction dir, unsigned long attrs) in dma_unmap_resource() argument
350 BUG_ON(!valid_dma_direction(dir)); in dma_unmap_resource()
354 iommu_dma_unmap_resource(dev, addr, size, dir, attrs); in dma_unmap_resource()
356 ops->unmap_resource(dev, addr, size, dir, attrs); in dma_unmap_resource()
357 trace_dma_unmap_resource(dev, addr, size, dir, attrs); in dma_unmap_resource()
358 debug_dma_unmap_resource(dev, addr, size, dir); in dma_unmap_resource()
364 enum dma_data_direction dir) in __dma_sync_single_for_cpu() argument
368 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_single_for_cpu()
370 dma_direct_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
372 iommu_dma_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
374 ops->sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
375 trace_dma_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
376 debug_dma_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
381 size_t size, enum dma_data_direction dir) in __dma_sync_single_for_device() argument
385 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_single_for_device()
387 dma_direct_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
389 iommu_dma_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
391 ops->sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
392 trace_dma_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
393 debug_dma_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
398 int nelems, enum dma_data_direction dir) in __dma_sync_sg_for_cpu() argument
402 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_sg_for_cpu()
404 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
406 iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
408 ops->sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
409 trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
410 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
415 int nelems, enum dma_data_direction dir) in __dma_sync_sg_for_device() argument
419 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_sg_for_device()
421 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
423 iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
425 ops->sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
426 trace_dma_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
427 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
668 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) in __dma_alloc_pages() argument
681 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); in __dma_alloc_pages()
683 return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp); in __dma_alloc_pages()
686 return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); in __dma_alloc_pages()
690 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) in dma_alloc_pages() argument
692 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); in dma_alloc_pages()
696 size, dir, gfp, 0); in dma_alloc_pages()
697 debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0); in dma_alloc_pages()
699 trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0); in dma_alloc_pages()
706 dma_addr_t dma_handle, enum dma_data_direction dir) in __dma_free_pages() argument
712 dma_direct_free_pages(dev, size, page, dma_handle, dir); in __dma_free_pages()
714 dma_common_free_pages(dev, size, page, dma_handle, dir); in __dma_free_pages()
716 ops->free_pages(dev, size, page, dma_handle, dir); in __dma_free_pages()
720 dma_addr_t dma_handle, enum dma_data_direction dir) in dma_free_pages() argument
722 trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0); in dma_free_pages()
723 debug_dma_unmap_page(dev, dma_handle, size, dir); in dma_free_pages()
724 __dma_free_pages(dev, size, page, dma_handle, dir); in dma_free_pages()
742 enum dma_data_direction dir, gfp_t gfp) in alloc_single_sgt() argument
752 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp); in alloc_single_sgt()
766 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) in dma_alloc_noncontiguous() argument
776 sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs); in dma_alloc_noncontiguous()
778 sgt = alloc_single_sgt(dev, size, dir, gfp); in dma_alloc_noncontiguous()
782 trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs); in dma_alloc_noncontiguous()
783 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); in dma_alloc_noncontiguous()
785 trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs); in dma_alloc_noncontiguous()
792 struct sg_table *sgt, enum dma_data_direction dir) in free_single_sgt() argument
795 dir); in free_single_sgt()
801 struct sg_table *sgt, enum dma_data_direction dir) in dma_free_noncontiguous() argument
803 trace_dma_free_sgt(dev, sgt, size, dir); in dma_free_noncontiguous()
804 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); in dma_free_noncontiguous()
807 iommu_dma_free_noncontiguous(dev, size, sgt, dir); in dma_free_noncontiguous()
809 free_single_sgt(dev, size, sgt, dir); in dma_free_noncontiguous()