| /linux/drivers/iommu/ |
| H A D | iova.c | 22 static bool iova_rcache_insert(struct iova_domain *iovad,
25 static unsigned long iova_rcache_get(struct iova_domain *iovad,
28 static void free_iova_rcaches(struct iova_domain *iovad);
29 static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
30 static void free_global_cached_iovas(struct iova_domain *iovad);
38 init_iova_domain(struct iova_domain *iovad, unsigned long granule, in init_iova_domain() argument
48 spin_lock_init(&iovad->iova_rbtree_lock); in init_iova_domain()
49 iovad->rbroot = RB_ROOT; in init_iova_domain()
50 iovad->cached_node = &iovad in init_iova_domain()
63 __get_cached_rbnode(struct iova_domain * iovad,unsigned long limit_pfn) __get_cached_rbnode() argument
72 __cached_rbnode_insert_update(struct iova_domain * iovad,struct iova * new) __cached_rbnode_insert_update() argument
81 __cached_rbnode_delete_update(struct iova_domain * iovad,struct iova * free) __cached_rbnode_delete_update() argument
99 iova_find_limit(struct iova_domain * iovad,unsigned long limit_pfn) iova_find_limit() argument
164 __alloc_and_insert_iova_range(struct iova_domain * iovad,unsigned long size,unsigned long limit_pfn,struct iova * new,bool size_aligned) __alloc_and_insert_iova_range() argument
252 alloc_iova(struct iova_domain * iovad,unsigned long size,unsigned long limit_pfn,bool size_aligned) alloc_iova() argument
276 private_find_iova(struct iova_domain * iovad,unsigned long pfn) private_find_iova() argument
296 remove_iova(struct iova_domain * iovad,struct iova * iova) remove_iova() argument
310 find_iova(struct iova_domain * iovad,unsigned long pfn) find_iova() argument
330 __free_iova(struct iova_domain * iovad,struct iova * iova) __free_iova() argument
349 free_iova(struct iova_domain * iovad,unsigned long pfn) free_iova() argument
377 alloc_iova_fast(struct iova_domain * iovad,unsigned long size,unsigned long limit_pfn,bool flush_rcache) alloc_iova_fast() argument
425 free_iova_fast(struct iova_domain * iovad,unsigned long pfn,unsigned long size) free_iova_fast() argument
434 iova_domain_free_rcaches(struct iova_domain * iovad) iova_domain_free_rcaches() argument
446 put_iova_domain(struct iova_domain * iovad) put_iova_domain() argument
484 __insert_new_range(struct iova_domain * iovad,unsigned long pfn_lo,unsigned long pfn_hi) __insert_new_range() argument
515 reserve_iova(struct iova_domain * iovad,unsigned long pfn_lo,unsigned long pfn_hi) reserve_iova() argument
590 struct iova_domain *iovad; global() member
618 iova_magazine_free_pfns(struct iova_magazine * mag,struct iova_domain * iovad) iova_magazine_free_pfns() argument
713 iova_domain_init_rcaches(struct iova_domain * iovad) iova_domain_init_rcaches() argument
769 __iova_rcache_insert(struct iova_domain * iovad,struct iova_rcache * rcache,unsigned long iova_pfn) __iova_rcache_insert() argument
807 iova_rcache_insert(struct iova_domain * iovad,unsigned long pfn,unsigned long size) iova_rcache_insert() argument
862 iova_rcache_get(struct iova_domain * iovad,unsigned long size,unsigned long limit_pfn) iova_rcache_get() argument
877 free_iova_rcaches(struct iova_domain * iovad) free_iova_rcaches() argument
905 free_cpu_cached_iovas(unsigned int cpu,struct iova_domain * iovad) free_cpu_cached_iovas() argument
925 free_global_cached_iovas(struct iova_domain * iovad) free_global_cached_iovas() argument
945 struct iova_domain *iovad; iova_cpuhp_dead() local
[all...] |
| H A D | dma-iommu.c | 58 struct iova_domain iovad; member
154 free_iova_fast(&cookie->iovad, in fq_ring_free_locked()
428 if (cookie->iovad.granule) { in iommu_put_dma_cookie()
430 put_iova_domain(&cookie->iovad); in iommu_put_dma_cookie()
475 struct iova_domain *iovad = &cookie->iovad; in cookie_init_hw_msi_region() local
479 start -= iova_offset(iovad, start); in cookie_init_hw_msi_region()
480 num_pages = iova_align(iovad, end - start) >> iova_shift(iovad); in cookie_init_hw_msi_region()
491 start += iovad in cookie_init_hw_msi_region()
507 iova_reserve_pci_windows(struct pci_dev * dev,struct iova_domain * iovad) iova_reserve_pci_windows() argument
556 struct iova_domain *iovad = &cookie->iovad; iova_reserve_iommu_regions() local
665 struct iova_domain *iovad; iommu_dma_init_domain() local
748 struct iova_domain *iovad = &cookie->iovad; iommu_dma_alloc_iova() local
793 struct iova_domain *iovad = &domain->iova_cookie->iovad; iommu_dma_free_iova() local
812 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_unmap() local
835 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_map() local
929 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_alloc_noncontiguous() local
1149 struct iova_domain *iovad = &domain->iova_cookie->iovad; iommu_dma_map_swiotlb() local
1187 iova_unaligned(struct iova_domain * iovad,phys_addr_t phys,size_t size) iova_unaligned() argument
1202 struct iova_domain *iovad = &cookie->iovad; iommu_dma_map_page() local
1383 struct iova_domain *iovad = &cookie->iovad; iommu_dma_map_sg() local
1771 struct iova_domain *iovad; dma_iova_try_alloc() local
1827 struct iova_domain *iovad = &cookie->iovad; dma_iova_free() local
1855 struct iova_domain *iovad = &domain->iova_cookie->iovad; iommu_dma_iova_bounce_and_link() local
1878 struct iova_domain *iovad = &cookie->iovad; iommu_dma_iova_link_swiotlb() local
1945 struct iova_domain *iovad = &cookie->iovad; dma_iova_link() local
1978 struct iova_domain *iovad = &cookie->iovad; dma_iova_sync() local
1993 struct iova_domain *iovad = &cookie->iovad; iommu_dma_iova_unlink_range_slow() local
2027 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_iova_unlink() local
[all...] |
| /linux/include/linux/ |
| H A D | iova.h | 48 static inline unsigned long iova_shift(struct iova_domain *iovad) in iova_shift() argument
50 return __ffs(iovad->granule); in iova_shift()
53 static inline unsigned long iova_mask(struct iova_domain *iovad) in iova_mask() argument
55 return iovad->granule - 1; in iova_mask()
58 static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova) in iova_offset() argument
60 return iova & iova_mask(iovad); in iova_offset()
63 static inline size_t iova_align(struct iova_domain *iovad, size_t size) in iova_align() argument
65 return ALIGN(size, iovad->granule); in iova_align()
68 static inline size_t iova_align_down(struct iova_domain *iovad, size_t size) in iova_align_down() argument
70 return ALIGN_DOWN(size, iovad in iova_align_down()
73 iova_dma_addr(struct iova_domain * iovad,struct iova * iova) iova_dma_addr() argument
78 iova_pfn(struct iova_domain * iovad,dma_addr_t iova) iova_pfn() argument
115 free_iova(struct iova_domain * iovad,unsigned long pfn) free_iova() argument
119 __free_iova(struct iova_domain * iovad,struct iova * iova) __free_iova() argument
123 alloc_iova(struct iova_domain * iovad,unsigned long size,unsigned long limit_pfn,bool size_aligned) alloc_iova() argument
131 free_iova_fast(struct iova_domain * iovad,unsigned long pfn,unsigned long size) free_iova_fast() argument
137 alloc_iova_fast(struct iova_domain * iovad,unsigned long size,unsigned long limit_pfn,bool flush_rcache) alloc_iova_fast() argument
145 reserve_iova(struct iova_domain * iovad,unsigned long pfn_lo,unsigned long pfn_hi) reserve_iova() argument
152 init_iova_domain(struct iova_domain * iovad,unsigned long granule,unsigned long start_pfn) init_iova_domain() argument
158 find_iova(struct iova_domain * iovad,unsigned long pfn) find_iova() argument
164 put_iova_domain(struct iova_domain * iovad) put_iova_domain() argument
[all...] |
| /linux/drivers/vdpa/vdpa_user/ |
| H A D | iova_domain.c | 358 vduse_domain_alloc_iova(struct iova_domain *iovad, in vduse_domain_alloc_iova() argument
361 unsigned long shift = iova_shift(iovad); in vduse_domain_alloc_iova()
362 unsigned long iova_len = iova_align(iovad, size) >> shift; in vduse_domain_alloc_iova()
365 iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true); in vduse_domain_alloc_iova()
370 static void vduse_domain_free_iova(struct iova_domain *iovad, in vduse_domain_free_iova() argument
373 unsigned long shift = iova_shift(iovad); in vduse_domain_free_iova()
374 unsigned long iova_len = iova_align(iovad, size) >> shift; in vduse_domain_free_iova()
376 free_iova_fast(iovad, iova >> shift, iova_len); in vduse_domain_free_iova()
404 struct iova_domain *iovad = &domain->stream_iovad; in vduse_domain_map_page() local
407 dma_addr_t iova = vduse_domain_alloc_iova(iovad, siz in vduse_domain_map_page()
437 struct iova_domain *iovad = &domain->stream_iovad; vduse_domain_unmap_page() local
452 struct iova_domain *iovad = &domain->consistent_iovad; vduse_domain_alloc_coherent() local
486 struct iova_domain *iovad = &domain->consistent_iovad; vduse_domain_free_coherent() local
[all...] |
| /linux/drivers/staging/media/ipu7/ |
| H A D | ipu7-dma.c | 174 iova = alloc_iova(&mmu->dmap->iovad, count, in ipu7_dma_alloc()
236 __free_iova(&mmu->dmap->iovad, iova); in ipu7_dma_alloc()
249 struct iova *iova = find_iova(&mmu->dmap->iovad, PHYS_PFN(dma_handle)); in ipu7_dma_free()
291 __free_iova(&mmu->dmap->iovad, iova); in ipu7_dma_free()
336 struct iova *iova = find_iova(&mmu->dmap->iovad, in ipu7_dma_unmap_sg()
370 __free_iova(&mmu->dmap->iovad, iova); in ipu7_dma_unmap_sg()
407 lo = iova_pfn(&mmu->dmap->iovad, IPU_FW_CODE_REGION_START); in ipu7_dma_map_sg()
408 hi = iova_pfn(&mmu->dmap->iovad, IPU_FW_CODE_REGION_END) - 1U; in ipu7_dma_map_sg()
409 iova = reserve_iova(&mmu->dmap->iovad, lo, hi); in ipu7_dma_map_sg()
416 iova = alloc_iova(&mmu->dmap->iovad, npage in ipu7_dma_map_sg()
[all...] |
| H A D | ipu7-mmu.c | 407 iova = alloc_iova(&mmu->dmap->iovad, n_pages, in allocate_trash_buffer()
452 __free_iova(&mmu->dmap->iovad, iova); in allocate_trash_buffer()
670 init_iova_domain(&dmap->iovad, SZ_4K, base_pfn); in alloc_dma_mapping()
758 iova = find_iova(&dmap->iovad, PHYS_PFN(mmu->iova_trash_page)); in ipu7_mmu_destroy()
763 __free_iova(&dmap->iovad, iova); in ipu7_mmu_destroy()
851 put_iova_domain(&dmap->iovad); in ipu7_mmu_cleanup()
|
| H A D | ipu7-dma.h | 20 struct iova_domain iovad; member
|
| /linux/drivers/media/pci/intel/ipu6/ |
| H A D | ipu6-dma.c | 174 iova = alloc_iova(&mmu->dmap->iovad, count, in ipu6_dma_alloc()
236 __free_iova(&mmu->dmap->iovad, iova); in ipu6_dma_alloc()
249 struct iova *iova = find_iova(&mmu->dmap->iovad, PHYS_PFN(dma_handle)); in ipu6_dma_free()
291 __free_iova(&mmu->dmap->iovad, iova); in ipu6_dma_free()
336 struct iova *iova = find_iova(&mmu->dmap->iovad, in ipu6_dma_unmap_sg()
370 __free_iova(&mmu->dmap->iovad, iova); in ipu6_dma_unmap_sg()
400 iova = alloc_iova(&mmu->dmap->iovad, npages, in ipu6_dma_map_sg()
|
| H A D | ipu6-mmu.c | 424 iova = alloc_iova(&mmu->dmap->iovad, n_pages, in allocate_trash_buffer()
469 __free_iova(&mmu->dmap->iovad, iova); in allocate_trash_buffer()
626 init_iova_domain(&dmap->iovad, SZ_4K, 1); in alloc_dma_mapping()
713 iova = find_iova(&dmap->iovad, PHYS_PFN(mmu->iova_trash_page)); in ipu6_mmu_destroy()
718 __free_iova(&dmap->iovad, iova); in ipu6_mmu_destroy()
804 put_iova_domain(&dmap->iovad); in ipu6_mmu_cleanup()
|
| H A D | ipu6-dma.h | 17 struct iova_domain iovad; member
|