Lines Matching +full:iommu +full:- +full:map
30 * 1) If an IOMMU unit has been probed, the IOMMU API is used to make memory
32 * 2) If no IOMMU unit is probed, the DMA API is used to allocate physically
35 * In both cases CPU read and writes are performed by creating a write-combined
74 * Used for objects flattened using the IOMMU API
86 /* array of base.mem->size pages (+ dma_addr_ts) */
103 /* Only used if IOMMU if present */
130 return (u64)gk20a_instobj(memory)->mn->offset << 12; in gk20a_instobj_addr()
136 return (u64)gk20a_instobj(memory)->mn->length << 12; in gk20a_instobj_size()
145 struct gk20a_instmem *imem = obj->base.imem; in gk20a_instobj_iommu_recycle_vaddr()
147 WARN_ON(obj->use_cpt); in gk20a_instobj_iommu_recycle_vaddr()
148 list_del(&obj->vaddr_node); in gk20a_instobj_iommu_recycle_vaddr()
149 vunmap(obj->base.vaddr); in gk20a_instobj_iommu_recycle_vaddr()
150 obj->base.vaddr = NULL; in gk20a_instobj_iommu_recycle_vaddr()
151 imem->vaddr_use -= nvkm_memory_size(&obj->base.base.memory); in gk20a_instobj_iommu_recycle_vaddr()
152 nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n", imem->vaddr_use, in gk20a_instobj_iommu_recycle_vaddr()
153 imem->vaddr_max); in gk20a_instobj_iommu_recycle_vaddr()
162 while (imem->vaddr_use + size > imem->vaddr_max) { in gk20a_instmem_vaddr_gc()
164 if (list_empty(&imem->vaddr_lru)) in gk20a_instmem_vaddr_gc()
168 list_first_entry(&imem->vaddr_lru, in gk20a_instmem_vaddr_gc()
177 struct gk20a_instmem *imem = node->imem; in gk20a_instobj_acquire_dma()
178 struct nvkm_ltc *ltc = imem->base.subdev.device->ltc; in gk20a_instobj_acquire_dma()
182 return node->vaddr; in gk20a_instobj_acquire_dma()
189 struct gk20a_instmem *imem = node->base.imem; in gk20a_instobj_acquire_iommu()
190 struct nvkm_ltc *ltc = imem->base.subdev.device->ltc; in gk20a_instobj_acquire_iommu()
195 mutex_lock(&imem->lock); in gk20a_instobj_acquire_iommu()
197 if (node->base.vaddr) { in gk20a_instobj_acquire_iommu()
198 if (!node->use_cpt) { in gk20a_instobj_acquire_iommu()
200 list_del(&node->vaddr_node); in gk20a_instobj_acquire_iommu()
208 /* map the pages */ in gk20a_instobj_acquire_iommu()
209 node->base.vaddr = vmap(node->pages, size >> PAGE_SHIFT, VM_MAP, in gk20a_instobj_acquire_iommu()
211 if (!node->base.vaddr) { in gk20a_instobj_acquire_iommu()
212 nvkm_error(&imem->base.subdev, "cannot map instobj - " in gk20a_instobj_acquire_iommu()
217 imem->vaddr_use += size; in gk20a_instobj_acquire_iommu()
218 nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n", in gk20a_instobj_acquire_iommu()
219 imem->vaddr_use, imem->vaddr_max); in gk20a_instobj_acquire_iommu()
222 node->use_cpt++; in gk20a_instobj_acquire_iommu()
223 mutex_unlock(&imem->lock); in gk20a_instobj_acquire_iommu()
225 return node->base.vaddr; in gk20a_instobj_acquire_iommu()
232 struct gk20a_instmem *imem = node->imem; in gk20a_instobj_release_dma()
233 struct nvkm_ltc *ltc = imem->base.subdev.device->ltc; in gk20a_instobj_release_dma()
235 /* in case we got a write-combined mapping */ in gk20a_instobj_release_dma()
244 struct gk20a_instmem *imem = node->base.imem; in gk20a_instobj_release_iommu()
245 struct nvkm_ltc *ltc = imem->base.subdev.device->ltc; in gk20a_instobj_release_iommu()
247 mutex_lock(&imem->lock); in gk20a_instobj_release_iommu()
250 if (WARN_ON(node->use_cpt == 0)) in gk20a_instobj_release_iommu()
254 if (--node->use_cpt == 0) in gk20a_instobj_release_iommu()
255 list_add_tail(&node->vaddr_node, &imem->vaddr_lru); in gk20a_instobj_release_iommu()
258 mutex_unlock(&imem->lock); in gk20a_instobj_release_iommu()
269 return node->vaddr[offset / 4]; in gk20a_instobj_rd32()
277 node->vaddr[offset / 4] = data; in gk20a_instobj_wr32()
285 struct nvkm_vmm_map map = { in gk20a_instobj_map() local
286 .memory = &node->base.memory, in gk20a_instobj_map()
288 .mem = node->mn, in gk20a_instobj_map()
291 return nvkm_vmm_map(vmm, vma, argv, argc, &map); in gk20a_instobj_map()
298 struct gk20a_instmem *imem = node->base.imem; in gk20a_instobj_dtor_dma()
299 struct device *dev = imem->base.subdev.device->dev; in gk20a_instobj_dtor_dma()
301 if (unlikely(!node->base.vaddr)) in gk20a_instobj_dtor_dma()
304 dma_free_attrs(dev, (u64)node->base.mn->length << PAGE_SHIFT, in gk20a_instobj_dtor_dma()
305 node->base.vaddr, node->handle, imem->attrs); in gk20a_instobj_dtor_dma()
315 struct gk20a_instmem *imem = node->base.imem; in gk20a_instobj_dtor_iommu()
316 struct device *dev = imem->base.subdev.device->dev; in gk20a_instobj_dtor_iommu()
317 struct nvkm_mm_node *r = node->base.mn; in gk20a_instobj_dtor_iommu()
323 mutex_lock(&imem->lock); in gk20a_instobj_dtor_iommu()
326 if (node->base.vaddr) in gk20a_instobj_dtor_iommu()
329 mutex_unlock(&imem->lock); in gk20a_instobj_dtor_iommu()
331 /* clear IOMMU bit to unmap pages */ in gk20a_instobj_dtor_iommu()
332 r->offset &= ~BIT(imem->iommu_bit - imem->iommu_pgshift); in gk20a_instobj_dtor_iommu()
335 for (i = 0; i < node->base.mn->length; i++) { in gk20a_instobj_dtor_iommu()
336 iommu_unmap(imem->domain, in gk20a_instobj_dtor_iommu()
337 (r->offset + i) << imem->iommu_pgshift, PAGE_SIZE); in gk20a_instobj_dtor_iommu()
338 dma_unmap_page(dev, node->dma_addrs[i], PAGE_SIZE, in gk20a_instobj_dtor_iommu()
340 __free_page(node->pages[i]); in gk20a_instobj_dtor_iommu()
344 mutex_lock(imem->mm_mutex); in gk20a_instobj_dtor_iommu()
345 nvkm_mm_free(imem->mm, &r); in gk20a_instobj_dtor_iommu()
346 mutex_unlock(imem->mm_mutex); in gk20a_instobj_dtor_iommu()
361 .map = gk20a_instobj_map,
373 .map = gk20a_instobj_map,
387 struct nvkm_subdev *subdev = &imem->base.subdev; in gk20a_instobj_ctor_dma()
388 struct device *dev = subdev->device->dev; in gk20a_instobj_ctor_dma()
391 return -ENOMEM; in gk20a_instobj_ctor_dma()
392 *_node = &node->base; in gk20a_instobj_ctor_dma()
394 nvkm_memory_ctor(&gk20a_instobj_func_dma, &node->base.base.memory); in gk20a_instobj_ctor_dma()
395 node->base.base.memory.ptrs = &gk20a_instobj_ptrs; in gk20a_instobj_ctor_dma()
397 node->base.vaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT, in gk20a_instobj_ctor_dma()
398 &node->handle, GFP_KERNEL, in gk20a_instobj_ctor_dma()
399 imem->attrs); in gk20a_instobj_ctor_dma()
400 if (!node->base.vaddr) { in gk20a_instobj_ctor_dma()
402 return -ENOMEM; in gk20a_instobj_ctor_dma()
406 if (unlikely(node->handle & (align - 1))) in gk20a_instobj_ctor_dma()
409 &node->handle, align); in gk20a_instobj_ctor_dma()
412 node->r.type = 12; in gk20a_instobj_ctor_dma()
413 node->r.offset = node->handle >> 12; in gk20a_instobj_ctor_dma()
414 node->r.length = (npages << PAGE_SHIFT) >> 12; in gk20a_instobj_ctor_dma()
416 node->base.mn = &node->r; in gk20a_instobj_ctor_dma()
425 struct nvkm_subdev *subdev = &imem->base.subdev; in gk20a_instobj_ctor_iommu()
426 struct device *dev = subdev->device->dev; in gk20a_instobj_ctor_iommu()
435 if (!(node = kzalloc(sizeof(*node) + ((sizeof(node->pages[0]) + in gk20a_instobj_ctor_iommu()
436 sizeof(*node->dma_addrs)) * npages), GFP_KERNEL))) in gk20a_instobj_ctor_iommu()
437 return -ENOMEM; in gk20a_instobj_ctor_iommu()
438 *_node = &node->base; in gk20a_instobj_ctor_iommu()
439 node->dma_addrs = (void *)(node->pages + npages); in gk20a_instobj_ctor_iommu()
441 nvkm_memory_ctor(&gk20a_instobj_func_iommu, &node->base.base.memory); in gk20a_instobj_ctor_iommu()
442 node->base.base.memory.ptrs = &gk20a_instobj_ptrs; in gk20a_instobj_ctor_iommu()
450 ret = -ENOMEM; in gk20a_instobj_ctor_iommu()
453 node->pages[i] = p; in gk20a_instobj_ctor_iommu()
457 ret = -ENOMEM; in gk20a_instobj_ctor_iommu()
460 node->dma_addrs[i] = dma_adr; in gk20a_instobj_ctor_iommu()
463 mutex_lock(imem->mm_mutex); in gk20a_instobj_ctor_iommu()
465 ret = nvkm_mm_head(imem->mm, 0, 1, npages, npages, in gk20a_instobj_ctor_iommu()
466 align >> imem->iommu_pgshift, &r); in gk20a_instobj_ctor_iommu()
467 mutex_unlock(imem->mm_mutex); in gk20a_instobj_ctor_iommu()
469 nvkm_error(subdev, "IOMMU space is full!\n"); in gk20a_instobj_ctor_iommu()
473 /* Map into GPU address space */ in gk20a_instobj_ctor_iommu()
475 u32 offset = (r->offset + i) << imem->iommu_pgshift; in gk20a_instobj_ctor_iommu()
477 ret = iommu_map(imem->domain, offset, node->dma_addrs[i], in gk20a_instobj_ctor_iommu()
481 nvkm_error(subdev, "IOMMU mapping failure: %d\n", ret); in gk20a_instobj_ctor_iommu()
483 while (i-- > 0) { in gk20a_instobj_ctor_iommu()
484 offset -= PAGE_SIZE; in gk20a_instobj_ctor_iommu()
485 iommu_unmap(imem->domain, offset, PAGE_SIZE); in gk20a_instobj_ctor_iommu()
491 /* IOMMU bit tells that an address is to be resolved through the IOMMU */ in gk20a_instobj_ctor_iommu()
492 r->offset |= BIT(imem->iommu_bit - imem->iommu_pgshift); in gk20a_instobj_ctor_iommu()
494 node->base.mn = r; in gk20a_instobj_ctor_iommu()
498 mutex_lock(imem->mm_mutex); in gk20a_instobj_ctor_iommu()
499 nvkm_mm_free(imem->mm, &r); in gk20a_instobj_ctor_iommu()
500 mutex_unlock(imem->mm_mutex); in gk20a_instobj_ctor_iommu()
503 for (i = 0; i < npages && node->pages[i] != NULL; i++) { in gk20a_instobj_ctor_iommu()
504 dma_addr_t dma_addr = node->dma_addrs[i]; in gk20a_instobj_ctor_iommu()
508 __free_page(node->pages[i]); in gk20a_instobj_ctor_iommu()
519 struct nvkm_subdev *subdev = &imem->base.subdev; in gk20a_instobj_new()
524 imem->domain ? "IOMMU" : "DMA", size, align); in gk20a_instobj_new()
530 if (imem->domain) in gk20a_instobj_new()
536 *pmemory = node ? &node->base.memory : NULL; in gk20a_instobj_new()
540 node->imem = imem; in gk20a_instobj_new()
543 size, align, (u64)node->mn->offset << 12); in gk20a_instobj_new()
554 if (!list_empty(&imem->vaddr_lru)) in gk20a_instmem_dtor()
555 nvkm_warn(&base->subdev, "instobj LRU not empty!\n"); in gk20a_instmem_dtor()
557 if (imem->vaddr_use != 0) in gk20a_instmem_dtor()
558 nvkm_warn(&base->subdev, "instobj vmap area not empty! " in gk20a_instmem_dtor()
559 "0x%x bytes still mapped\n", imem->vaddr_use); in gk20a_instmem_dtor()
577 struct nvkm_device_tegra *tdev = device->func->tegra(device); in gk20a_instmem_new()
581 return -ENOMEM; in gk20a_instmem_new()
582 nvkm_instmem_ctor(&gk20a_instmem, device, type, inst, &imem->base); in gk20a_instmem_new()
583 mutex_init(&imem->lock); in gk20a_instmem_new()
584 *pimem = &imem->base; in gk20a_instmem_new()
586 /* do not allow more than 1MB of CPU-mapped instmem */ in gk20a_instmem_new()
587 imem->vaddr_use = 0; in gk20a_instmem_new()
588 imem->vaddr_max = 0x100000; in gk20a_instmem_new()
589 INIT_LIST_HEAD(&imem->vaddr_lru); in gk20a_instmem_new()
591 if (tdev->iommu.domain) { in gk20a_instmem_new()
592 imem->mm_mutex = &tdev->iommu.mutex; in gk20a_instmem_new()
593 imem->mm = &tdev->iommu.mm; in gk20a_instmem_new()
594 imem->domain = tdev->iommu.domain; in gk20a_instmem_new()
595 imem->iommu_pgshift = tdev->iommu.pgshift; in gk20a_instmem_new()
596 imem->iommu_bit = tdev->func->iommu_bit; in gk20a_instmem_new()
598 nvkm_info(&imem->base.subdev, "using IOMMU\n"); in gk20a_instmem_new()
600 imem->attrs = DMA_ATTR_WEAK_ORDERING | in gk20a_instmem_new()
603 nvkm_info(&imem->base.subdev, "using DMA API\n"); in gk20a_instmem_new()