Lines Matching +full:gfx +full:- +full:mem
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
33 #include <linux/dma-mapping.h>
41 #include <linux/dma-buf.h>
78 return ttm_range_man_init(&adev->mman.bdev, type, in amdgpu_ttm_init_on_chip()
83 * amdgpu_evict_flags - Compute placement flags
93 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_evict_flags()
103 if (bo->type == ttm_bo_type_sg) { in amdgpu_evict_flags()
104 placement->num_placement = 0; in amdgpu_evict_flags()
105 placement->num_busy_placement = 0; in amdgpu_evict_flags()
111 placement->placement = &placements; in amdgpu_evict_flags()
112 placement->busy_placement = &placements; in amdgpu_evict_flags()
113 placement->num_placement = 1; in amdgpu_evict_flags()
114 placement->num_busy_placement = 1; in amdgpu_evict_flags()
119 if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) { in amdgpu_evict_flags()
120 placement->num_placement = 0; in amdgpu_evict_flags()
121 placement->num_busy_placement = 0; in amdgpu_evict_flags()
125 switch (bo->resource->mem_type) { in amdgpu_evict_flags()
130 placement->num_placement = 0; in amdgpu_evict_flags()
131 placement->num_busy_placement = 0; in amdgpu_evict_flags()
135 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_evict_flags()
138 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && in amdgpu_evict_flags()
139 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && in amdgpu_evict_flags()
150 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; in amdgpu_evict_flags()
151 abo->placements[0].lpfn = 0; in amdgpu_evict_flags()
152 abo->placement.busy_placement = &abo->placements[1]; in amdgpu_evict_flags()
153 abo->placement.num_busy_placement = 1; in amdgpu_evict_flags()
166 *placement = abo->placement; in amdgpu_evict_flags()
170 * amdgpu_ttm_map_buffer - Map memory into the GART windows
172 * @mem: memory object to map
184 struct ttm_resource *mem, in amdgpu_ttm_map_buffer() argument
189 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_map_buffer()
198 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < in amdgpu_ttm_map_buffer()
201 if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT)) in amdgpu_ttm_map_buffer()
202 return -EINVAL; in amdgpu_ttm_map_buffer()
205 if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) { in amdgpu_ttm_map_buffer()
206 *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) + in amdgpu_ttm_map_buffer()
207 mm_cur->start; in amdgpu_ttm_map_buffer()
216 offset = mm_cur->start & ~PAGE_MASK; in amdgpu_ttm_map_buffer()
221 *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset); in amdgpu_ttm_map_buffer()
223 *addr = adev->gmc.gart_start; in amdgpu_ttm_map_buffer()
228 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_map_buffer()
231 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_map_buffer()
239 src_addr += job->ibs[0].gpu_addr; in amdgpu_ttm_map_buffer()
241 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); in amdgpu_ttm_map_buffer()
243 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, in amdgpu_ttm_map_buffer()
246 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_map_buffer()
247 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_map_buffer()
249 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem); in amdgpu_ttm_map_buffer()
253 cpu_addr = &job->ibs[0].ptr[num_dw]; in amdgpu_ttm_map_buffer()
255 if (mem->mem_type == TTM_PL_TT) { in amdgpu_ttm_map_buffer()
258 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT]; in amdgpu_ttm_map_buffer()
263 dma_address = mm_cur->start; in amdgpu_ttm_map_buffer()
264 dma_address += adev->vm_manager.vram_base_offset; in amdgpu_ttm_map_buffer()
278 * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
287 * The function copies @size bytes from {src->mem + src->offset} to
288 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
299 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_copy_mem_to_mem()
304 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_ttm_copy_mem_to_mem()
306 return -EINVAL; in amdgpu_ttm_copy_mem_to_mem()
309 amdgpu_res_first(src->mem, src->offset, size, &src_mm); in amdgpu_ttm_copy_mem_to_mem()
310 amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm); in amdgpu_ttm_copy_mem_to_mem()
312 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
321 r = amdgpu_ttm_map_buffer(src->bo, src->mem, &src_mm, in amdgpu_ttm_copy_mem_to_mem()
326 r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, &dst_mm, in amdgpu_ttm_copy_mem_to_mem()
343 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
351 * amdgpu_move_blit - Copy an entire buffer to another buffer
361 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_move_blit()
369 src.mem = old_mem; in amdgpu_move_blit()
370 dst.mem = new_mem; in amdgpu_move_blit()
375 new_mem->size, in amdgpu_move_blit()
377 bo->base.resv, &fence); in amdgpu_move_blit()
382 if (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_move_blit()
383 (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) { in amdgpu_move_blit()
397 if (bo->type == ttm_bo_type_kernel) in amdgpu_move_blit()
412 * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
417 struct ttm_resource *mem) in amdgpu_mem_visible() argument
419 u64 mem_size = (u64)mem->size; in amdgpu_mem_visible()
423 if (mem->mem_type == TTM_PL_SYSTEM || in amdgpu_mem_visible()
424 mem->mem_type == TTM_PL_TT) in amdgpu_mem_visible()
426 if (mem->mem_type != TTM_PL_VRAM) in amdgpu_mem_visible()
429 amdgpu_res_first(mem, 0, mem_size, &cursor); in amdgpu_mem_visible()
444 return end <= adev->gmc.visible_vram_size; in amdgpu_mem_visible()
448 * amdgpu_bo_move - Move a buffer object to a new memory location
459 struct ttm_resource *old_mem = bo->resource; in amdgpu_bo_move()
462 if (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
463 new_mem->mem_type == AMDGPU_PL_PREEMPT) { in amdgpu_bo_move()
464 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); in amdgpu_bo_move()
470 adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_bo_move()
472 if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
473 bo->ttm == NULL)) { in amdgpu_bo_move()
477 if (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
478 (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
479 new_mem->mem_type == AMDGPU_PL_PREEMPT)) { in amdgpu_bo_move()
483 if ((old_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
484 old_mem->mem_type == AMDGPU_PL_PREEMPT) && in amdgpu_bo_move()
485 new_mem->mem_type == TTM_PL_SYSTEM) { in amdgpu_bo_move()
490 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); in amdgpu_bo_move()
491 ttm_resource_free(bo, &bo->resource); in amdgpu_bo_move()
496 if (old_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
497 old_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
498 old_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
499 old_mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_bo_move()
500 new_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
501 new_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
502 new_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
503 new_mem->mem_type == AMDGPU_PL_DOORBELL) { in amdgpu_bo_move()
509 if (bo->type == ttm_bo_type_device && in amdgpu_bo_move()
510 new_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
511 old_mem->mem_type != TTM_PL_VRAM) { in amdgpu_bo_move()
512 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU in amdgpu_bo_move()
515 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; in amdgpu_bo_move()
518 if (adev->mman.buffer_funcs_enabled) { in amdgpu_bo_move()
519 if (((old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
520 new_mem->mem_type == TTM_PL_VRAM) || in amdgpu_bo_move()
521 (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
522 new_mem->mem_type == TTM_PL_SYSTEM))) { in amdgpu_bo_move()
523 hop->fpfn = 0; in amdgpu_bo_move()
524 hop->lpfn = 0; in amdgpu_bo_move()
525 hop->mem_type = TTM_PL_TT; in amdgpu_bo_move()
526 hop->flags = TTM_PL_FLAG_TEMPORARY; in amdgpu_bo_move()
527 return -EMULTIHOP; in amdgpu_bo_move()
532 r = -ENODEV; in amdgpu_bo_move()
548 trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type); in amdgpu_bo_move()
551 atomic64_add(bo->base.size, &adev->num_bytes_moved); in amdgpu_bo_move()
557 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
562 struct ttm_resource *mem) in amdgpu_ttm_io_mem_reserve() argument
565 size_t bus_size = (size_t)mem->size; in amdgpu_ttm_io_mem_reserve()
567 switch (mem->mem_type) { in amdgpu_ttm_io_mem_reserve()
575 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
577 if ((mem->bus.offset + bus_size) > adev->gmc.visible_vram_size) in amdgpu_ttm_io_mem_reserve()
578 return -EINVAL; in amdgpu_ttm_io_mem_reserve()
580 if (adev->mman.aper_base_kaddr && in amdgpu_ttm_io_mem_reserve()
581 mem->placement & TTM_PL_FLAG_CONTIGUOUS) in amdgpu_ttm_io_mem_reserve()
582 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + in amdgpu_ttm_io_mem_reserve()
583 mem->bus.offset; in amdgpu_ttm_io_mem_reserve()
585 mem->bus.offset += adev->gmc.aper_base; in amdgpu_ttm_io_mem_reserve()
586 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
589 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
590 mem->bus.offset += adev->doorbell.base; in amdgpu_ttm_io_mem_reserve()
591 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
592 mem->bus.caching = ttm_uncached; in amdgpu_ttm_io_mem_reserve()
595 return -EINVAL; in amdgpu_ttm_io_mem_reserve()
603 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_io_mem_pfn()
606 amdgpu_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, in amdgpu_ttm_io_mem_pfn()
609 if (bo->resource->mem_type == AMDGPU_PL_DOORBELL) in amdgpu_ttm_io_mem_pfn()
610 return ((uint64_t)(adev->doorbell.base + cursor.start)) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
612 return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
616 * amdgpu_ttm_domain_start - Returns GPU start address
628 return adev->gmc.gart_start; in amdgpu_ttm_domain_start()
630 return adev->gmc.vram_start; in amdgpu_ttm_domain_start()
654 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
663 struct ttm_tt *ttm = bo->tbo.ttm; in amdgpu_ttm_tt_get_user_pages()
665 unsigned long start = gtt->userptr; in amdgpu_ttm_tt_get_user_pages()
674 mm = bo->notifier.mm; in amdgpu_ttm_tt_get_user_pages()
677 return -EFAULT; in amdgpu_ttm_tt_get_user_pages()
681 return -ESRCH; in amdgpu_ttm_tt_get_user_pages()
686 r = -EFAULT; in amdgpu_ttm_tt_get_user_pages()
689 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && in amdgpu_ttm_tt_get_user_pages()
690 vma->vm_file)) { in amdgpu_ttm_tt_get_user_pages()
691 r = -EPERM; in amdgpu_ttm_tt_get_user_pages()
696 r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages, in amdgpu_ttm_tt_get_user_pages()
708 /* amdgpu_ttm_tt_discard_user_pages - Discard range and pfn array allocations
715 if (gtt && gtt->userptr && range) in amdgpu_ttm_tt_discard_user_pages()
720 * amdgpu_ttm_tt_get_user_pages_done - stop HMM track the CPU page table change
730 if (!gtt || !gtt->userptr || !range) in amdgpu_ttm_tt_get_user_pages_done()
734 gtt->userptr, ttm->num_pages); in amdgpu_ttm_tt_get_user_pages_done()
736 WARN_ONCE(!range->hmm_pfns, "No user pages to check\n"); in amdgpu_ttm_tt_get_user_pages_done()
743 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
753 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_set_user_pages()
754 ttm->pages[i] = pages ? pages[i] : NULL; in amdgpu_ttm_tt_set_user_pages()
758 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
767 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_pin_userptr()
773 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, in amdgpu_ttm_tt_pin_userptr()
774 (u64)ttm->num_pages << PAGE_SHIFT, in amdgpu_ttm_tt_pin_userptr()
780 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_pin_userptr()
785 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_tt_pin_userptr()
786 ttm->num_pages); in amdgpu_ttm_tt_pin_userptr()
791 kfree(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
792 ttm->sg = NULL; in amdgpu_ttm_tt_pin_userptr()
797 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
804 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_unpin_userptr()
809 if (!ttm->sg || !ttm->sg->sgl) in amdgpu_ttm_tt_unpin_userptr()
813 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_unpin_userptr()
814 sg_free_table(ttm->sg); in amdgpu_ttm_tt_unpin_userptr()
828 uint64_t total_pages = ttm->num_pages; in amdgpu_ttm_gart_bind_gfx9_mqd()
829 int num_xcc = max(1U, adev->gfx.num_xcc_per_xcp); in amdgpu_ttm_gart_bind_gfx9_mqd()
841 gtt->offset + (page_idx << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
842 1, >t->ttm.dma_address[page_idx], flags); in amdgpu_ttm_gart_bind_gfx9_mqd()
844 * Ctrl pages - modify the memory type to NC (ctrl_flags) from in amdgpu_ttm_gart_bind_gfx9_mqd()
848 gtt->offset + ((page_idx + 1) << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
849 pages_per_xcc - 1, in amdgpu_ttm_gart_bind_gfx9_mqd()
850 >t->ttm.dma_address[page_idx + 1], in amdgpu_ttm_gart_bind_gfx9_mqd()
860 struct ttm_tt *ttm = tbo->ttm; in amdgpu_ttm_gart_bind()
866 if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) { in amdgpu_ttm_gart_bind()
869 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_gart_bind()
870 gtt->ttm.dma_address, flags); in amdgpu_ttm_gart_bind()
875 * amdgpu_ttm_backend_bind - Bind GTT memory
890 return -EINVAL; in amdgpu_ttm_backend_bind()
892 if (gtt->bound) in amdgpu_ttm_backend_bind()
895 if (gtt->userptr) { in amdgpu_ttm_backend_bind()
901 } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) { in amdgpu_ttm_backend_bind()
902 if (!ttm->sg) { in amdgpu_ttm_backend_bind()
906 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_bind()
911 ttm->sg = sgt; in amdgpu_ttm_backend_bind()
914 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_backend_bind()
915 ttm->num_pages); in amdgpu_ttm_backend_bind()
918 if (!ttm->num_pages) { in amdgpu_ttm_backend_bind()
920 ttm->num_pages, bo_mem, ttm); in amdgpu_ttm_backend_bind()
923 if (bo_mem->mem_type != TTM_PL_TT || in amdgpu_ttm_backend_bind()
925 gtt->offset = AMDGPU_BO_INVALID_OFFSET; in amdgpu_ttm_backend_bind()
933 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; in amdgpu_ttm_backend_bind()
934 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_backend_bind()
935 gtt->ttm.dma_address, flags); in amdgpu_ttm_backend_bind()
936 gtt->bound = true; in amdgpu_ttm_backend_bind()
941 * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
950 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_alloc_gart()
952 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_alloc_gart()
959 if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_alloc_gart()
972 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
974 placements.flags = bo->resource->placement; in amdgpu_ttm_alloc_gart()
981 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp); in amdgpu_ttm_alloc_gart()
984 gtt->offset = (u64)tmp->start << PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
987 ttm_resource_free(bo, &bo->resource); in amdgpu_ttm_alloc_gart()
994 * amdgpu_ttm_recover_gart - Rebind GTT pages
1001 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); in amdgpu_ttm_recover_gart()
1004 if (!tbo->ttm) in amdgpu_ttm_recover_gart()
1007 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource); in amdgpu_ttm_recover_gart()
1012 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1024 if (gtt->userptr) { in amdgpu_ttm_backend_unbind()
1026 } else if (ttm->sg && gtt->gobj->import_attach) { in amdgpu_ttm_backend_unbind()
1029 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_unbind()
1030 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL); in amdgpu_ttm_backend_unbind()
1031 ttm->sg = NULL; in amdgpu_ttm_backend_unbind()
1034 if (!gtt->bound) in amdgpu_ttm_backend_unbind()
1037 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_backend_unbind()
1041 amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); in amdgpu_ttm_backend_unbind()
1042 gtt->bound = false; in amdgpu_ttm_backend_unbind()
1050 if (gtt->usertask) in amdgpu_ttm_backend_destroy()
1051 put_task_struct(gtt->usertask); in amdgpu_ttm_backend_destroy()
1053 ttm_tt_fini(>t->ttm); in amdgpu_ttm_backend_destroy()
1058 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1068 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_tt_create()
1077 gtt->gobj = &bo->base; in amdgpu_ttm_tt_create()
1078 if (adev->gmc.mem_partitions && abo->xcp_id >= 0) in amdgpu_ttm_tt_create()
1079 gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); in amdgpu_ttm_tt_create()
1081 gtt->pool_id = abo->xcp_id; in amdgpu_ttm_tt_create()
1083 if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) in amdgpu_ttm_tt_create()
1089 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { in amdgpu_ttm_tt_create()
1093 return >t->ttm; in amdgpu_ttm_tt_create()
1097 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1113 if (gtt->userptr) { in amdgpu_ttm_tt_populate()
1114 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); in amdgpu_ttm_tt_populate()
1115 if (!ttm->sg) in amdgpu_ttm_tt_populate()
1116 return -ENOMEM; in amdgpu_ttm_tt_populate()
1120 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_populate()
1123 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_populate()
1124 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_populate()
1126 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_populate()
1131 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_populate()
1132 ttm->pages[i]->mapping = bdev->dev_mapping; in amdgpu_ttm_tt_populate()
1138 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1153 if (gtt->userptr) { in amdgpu_ttm_tt_unpopulate()
1155 kfree(ttm->sg); in amdgpu_ttm_tt_unpopulate()
1156 ttm->sg = NULL; in amdgpu_ttm_tt_unpopulate()
1160 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_unpopulate()
1163 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_unpopulate()
1164 ttm->pages[i]->mapping = NULL; in amdgpu_ttm_tt_unpopulate()
1168 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_unpopulate()
1169 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_unpopulate()
1171 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_unpopulate()
1177 * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
1188 if (!tbo->ttm) in amdgpu_ttm_tt_get_userptr()
1189 return -EINVAL; in amdgpu_ttm_tt_get_userptr()
1191 gtt = (void *)tbo->ttm; in amdgpu_ttm_tt_get_userptr()
1192 *user_addr = gtt->userptr; in amdgpu_ttm_tt_get_userptr()
1197 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1213 if (!bo->ttm) { in amdgpu_ttm_tt_set_userptr()
1215 bo->ttm = amdgpu_ttm_tt_create(bo, 0); in amdgpu_ttm_tt_set_userptr()
1216 if (bo->ttm == NULL) in amdgpu_ttm_tt_set_userptr()
1217 return -ENOMEM; in amdgpu_ttm_tt_set_userptr()
1221 bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; in amdgpu_ttm_tt_set_userptr()
1223 gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_tt_set_userptr()
1224 gtt->userptr = addr; in amdgpu_ttm_tt_set_userptr()
1225 gtt->userflags = flags; in amdgpu_ttm_tt_set_userptr()
1227 if (gtt->usertask) in amdgpu_ttm_tt_set_userptr()
1228 put_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1229 gtt->usertask = current->group_leader; in amdgpu_ttm_tt_set_userptr()
1230 get_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1236 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1245 if (gtt->usertask == NULL) in amdgpu_ttm_tt_get_usermm()
1248 return gtt->usertask->mm; in amdgpu_ttm_tt_get_usermm()
1252 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1262 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_affect_userptr()
1268 size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE; in amdgpu_ttm_tt_affect_userptr()
1269 if (gtt->userptr > end || gtt->userptr + size <= start) in amdgpu_ttm_tt_affect_userptr()
1273 *userptr = gtt->userptr; in amdgpu_ttm_tt_affect_userptr()
1278 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1284 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_is_userptr()
1291 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1300 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_is_readonly()
1304 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1307 * @mem: The memory registry backing this ttm_tt object
1311 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem) in amdgpu_ttm_tt_pde_flags() argument
1315 if (mem && mem->mem_type != TTM_PL_SYSTEM) in amdgpu_ttm_tt_pde_flags()
1318 if (mem && (mem->mem_type == TTM_PL_TT || in amdgpu_ttm_tt_pde_flags()
1319 mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_ttm_tt_pde_flags()
1320 mem->mem_type == AMDGPU_PL_PREEMPT)) { in amdgpu_ttm_tt_pde_flags()
1323 if (ttm->caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1327 if (mem && mem->mem_type == TTM_PL_VRAM && in amdgpu_ttm_tt_pde_flags()
1328 mem->bus.caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1335 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1339 * @mem: The memory registry backing this ttm_tt object
1344 struct ttm_resource *mem) in amdgpu_ttm_tt_pte_flags() argument
1346 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem); in amdgpu_ttm_tt_pte_flags()
1348 flags |= adev->gart.gart_pte_flags; in amdgpu_ttm_tt_pte_flags()
1358 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1376 if (bo->resource->mem_type == TTM_PL_SYSTEM) in amdgpu_ttm_bo_eviction_valuable()
1379 if (bo->type == ttm_bo_type_kernel && in amdgpu_ttm_bo_eviction_valuable()
1387 dma_resv_for_each_fence(&resv_cursor, bo->base.resv, in amdgpu_ttm_bo_eviction_valuable()
1389 if (amdkfd_fence_check_mm(f, current->mm)) in amdgpu_ttm_bo_eviction_valuable()
1400 if (bo->resource->mem_type == AMDGPU_PL_PREEMPT) in amdgpu_ttm_bo_eviction_valuable()
1403 if (bo->resource->mem_type == TTM_PL_TT && in amdgpu_ttm_bo_eviction_valuable()
1415 uint64_t bytes = 4 - (pos & 0x3); in amdgpu_ttm_vram_mm_access()
1421 mask &= 0xffffffff >> (bytes - size) * 8; in amdgpu_ttm_vram_mm_access()
1441 size -= bytes; in amdgpu_ttm_vram_mm_access()
1450 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory_sdma()
1459 return -EINVAL; in amdgpu_ttm_access_memory_sdma()
1461 if (!adev->mman.sdma_access_ptr) in amdgpu_ttm_access_memory_sdma()
1462 return -EACCES; in amdgpu_ttm_access_memory_sdma()
1465 return -ENODEV; in amdgpu_ttm_access_memory_sdma()
1468 memcpy(adev->mman.sdma_access_ptr, buf, len); in amdgpu_ttm_access_memory_sdma()
1470 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_access_memory_sdma()
1471 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_access_memory_sdma()
1478 amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm); in amdgpu_ttm_access_memory_sdma()
1479 src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) + in amdgpu_ttm_access_memory_sdma()
1481 dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo); in amdgpu_ttm_access_memory_sdma()
1485 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr, in amdgpu_ttm_access_memory_sdma()
1488 amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]); in amdgpu_ttm_access_memory_sdma()
1489 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_access_memory_sdma()
1493 if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout)) in amdgpu_ttm_access_memory_sdma()
1494 r = -ETIMEDOUT; in amdgpu_ttm_access_memory_sdma()
1498 memcpy(buf, adev->mman.sdma_access_ptr, len); in amdgpu_ttm_access_memory_sdma()
1505 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1521 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory()
1525 if (bo->resource->mem_type != TTM_PL_VRAM) in amdgpu_ttm_access_memory()
1526 return -EIO; in amdgpu_ttm_access_memory()
1532 amdgpu_res_first(bo->resource, offset, len, &cursor); in amdgpu_ttm_access_memory()
1538 size -= count; in amdgpu_ttm_access_memory()
1540 /* using MM to access rest vram and handle un-aligned address */ in amdgpu_ttm_access_memory()
1579 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1587 amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_fini()
1588 NULL, &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_fini()
1595 * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
1603 amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_fini()
1605 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_fini()
1609 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1617 uint64_t vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_fw_reserve_vram_init()
1619 adev->mman.fw_vram_usage_va = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1620 adev->mman.fw_vram_usage_reserved_bo = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1622 if (adev->mman.fw_vram_usage_size == 0 || in amdgpu_ttm_fw_reserve_vram_init()
1623 adev->mman.fw_vram_usage_size > vram_size) in amdgpu_ttm_fw_reserve_vram_init()
1627 adev->mman.fw_vram_usage_start_offset, in amdgpu_ttm_fw_reserve_vram_init()
1628 adev->mman.fw_vram_usage_size, in amdgpu_ttm_fw_reserve_vram_init()
1629 &adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_init()
1630 &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_init()
1634 * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
1642 u64 vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_drv_reserve_vram_init()
1644 adev->mman.drv_vram_usage_va = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1645 adev->mman.drv_vram_usage_reserved_bo = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1647 if (adev->mman.drv_vram_usage_size == 0 || in amdgpu_ttm_drv_reserve_vram_init()
1648 adev->mman.drv_vram_usage_size > vram_size) in amdgpu_ttm_drv_reserve_vram_init()
1652 adev->mman.drv_vram_usage_start_offset, in amdgpu_ttm_drv_reserve_vram_init()
1653 adev->mman.drv_vram_usage_size, in amdgpu_ttm_drv_reserve_vram_init()
1654 &adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_init()
1655 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_init()
1663 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1671 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_reserve_vram_fini()
1673 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT; in amdgpu_ttm_training_reserve_vram_fini()
1674 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL); in amdgpu_ttm_training_reserve_vram_fini()
1675 ctx->c2p_bo = NULL; in amdgpu_ttm_training_reserve_vram_fini()
1683 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_data_block_init()
1687 ctx->c2p_train_data_offset = in amdgpu_ttm_training_data_block_init()
1688 ALIGN((adev->gmc.mc_vram_size - reserve_size - SZ_1M), SZ_1M); in amdgpu_ttm_training_data_block_init()
1689 ctx->p2c_train_data_offset = in amdgpu_ttm_training_data_block_init()
1690 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET); in amdgpu_ttm_training_data_block_init()
1691 ctx->train_data_size = in amdgpu_ttm_training_data_block_init()
1695 ctx->train_data_size, in amdgpu_ttm_training_data_block_init()
1696 ctx->p2c_train_data_offset, in amdgpu_ttm_training_data_block_init()
1697 ctx->c2p_train_data_offset); in amdgpu_ttm_training_data_block_init()
1706 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_reserve_tmr()
1711 if (adev->bios && !amdgpu_sriov_vf(adev)) { in amdgpu_ttm_reserve_tmr()
1725 if (adev->bios) in amdgpu_ttm_reserve_tmr()
1729 if (!adev->bios && in amdgpu_ttm_reserve_tmr()
1736 /* reserve vram for mem train according to TMR location */ in amdgpu_ttm_reserve_tmr()
1739 ctx->c2p_train_data_offset, in amdgpu_ttm_reserve_tmr()
1740 ctx->train_data_size, in amdgpu_ttm_reserve_tmr()
1741 &ctx->c2p_bo, in amdgpu_ttm_reserve_tmr()
1748 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS; in amdgpu_ttm_reserve_tmr()
1751 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_reserve_tmr()
1753 adev, adev->gmc.real_vram_size - reserve_size, in amdgpu_ttm_reserve_tmr()
1754 reserve_size, &adev->mman.fw_reserved_memory, NULL); in amdgpu_ttm_reserve_tmr()
1757 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, in amdgpu_ttm_reserve_tmr()
1772 if (!adev->gmc.is_app_apu || !adev->gmc.num_mem_partitions) in amdgpu_ttm_pools_init()
1775 adev->mman.ttm_pools = kcalloc(adev->gmc.num_mem_partitions, in amdgpu_ttm_pools_init()
1776 sizeof(*adev->mman.ttm_pools), in amdgpu_ttm_pools_init()
1778 if (!adev->mman.ttm_pools) in amdgpu_ttm_pools_init()
1779 return -ENOMEM; in amdgpu_ttm_pools_init()
1781 for (i = 0; i < adev->gmc.num_mem_partitions; i++) { in amdgpu_ttm_pools_init()
1782 ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev, in amdgpu_ttm_pools_init()
1783 adev->gmc.mem_partitions[i].numa.node, in amdgpu_ttm_pools_init()
1793 if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools) in amdgpu_ttm_pools_fini()
1796 for (i = 0; i < adev->gmc.num_mem_partitions; i++) in amdgpu_ttm_pools_fini()
1797 ttm_pool_fini(&adev->mman.ttm_pools[i]); in amdgpu_ttm_pools_fini()
1799 kfree(adev->mman.ttm_pools); in amdgpu_ttm_pools_fini()
1800 adev->mman.ttm_pools = NULL; in amdgpu_ttm_pools_fini()
1804 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1809 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1817 mutex_init(&adev->mman.gtt_window_lock); in amdgpu_ttm_init()
1820 r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev, in amdgpu_ttm_init()
1821 adev_to_drm(adev)->anon_inode->i_mapping, in amdgpu_ttm_init()
1822 adev_to_drm(adev)->vma_offset_manager, in amdgpu_ttm_init()
1823 adev->need_swiotlb, in amdgpu_ttm_init()
1824 dma_addressing_limited(adev->dev)); in amdgpu_ttm_init()
1835 adev->mman.initialized = true; in amdgpu_ttm_init()
1848 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_ttm_init()
1849 adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, in amdgpu_ttm_init()
1850 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1852 else if (adev->gmc.is_app_apu) in amdgpu_ttm_init()
1857 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, in amdgpu_ttm_init()
1858 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1882 if (adev->mman.discovery_bin) { in amdgpu_ttm_init()
1889 * This is used for VGA emulation and pre-OS scanout buffers to in amdgpu_ttm_init()
1890 * avoid display artifacts while transitioning between pre-OS in amdgpu_ttm_init()
1893 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_init()
1895 adev->mman.stolen_vga_size, in amdgpu_ttm_init()
1896 &adev->mman.stolen_vga_memory, in amdgpu_ttm_init()
1901 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, in amdgpu_ttm_init()
1902 adev->mman.stolen_extended_size, in amdgpu_ttm_init()
1903 &adev->mman.stolen_extended_memory, in amdgpu_ttm_init()
1910 adev->mman.stolen_reserved_offset, in amdgpu_ttm_init()
1911 adev->mman.stolen_reserved_size, in amdgpu_ttm_init()
1912 &adev->mman.stolen_reserved_memory, in amdgpu_ttm_init()
1921 (unsigned int)(adev->gmc.real_vram_size / (1024 * 1024))); in amdgpu_ttm_init()
1926 if (amdgpu_gtt_size == -1) in amdgpu_ttm_init()
1941 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_DOORBELL, adev->doorbell.size / PAGE_SIZE); in amdgpu_ttm_init()
1961 /* Initialize various on-chip memory pools */ in amdgpu_ttm_init()
1962 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size); in amdgpu_ttm_init()
1968 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size); in amdgpu_ttm_init()
1974 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size); in amdgpu_ttm_init()
1981 &adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_init()
1982 &adev->mman.sdma_access_ptr)) in amdgpu_ttm_init()
1989 * amdgpu_ttm_fini - De-initialize the TTM memory pools
1995 if (!adev->mman.initialized) in amdgpu_ttm_fini()
2002 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_fini()
2003 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); in amdgpu_ttm_fini()
2004 amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); in amdgpu_ttm_fini()
2006 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, NULL, in amdgpu_ttm_fini()
2008 if (adev->mman.stolen_reserved_size) in amdgpu_ttm_fini()
2009 amdgpu_bo_free_kernel(&adev->mman.stolen_reserved_memory, in amdgpu_ttm_fini()
2012 amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_fini()
2013 &adev->mman.sdma_access_ptr); in amdgpu_ttm_fini()
2019 if (adev->mman.aper_base_kaddr) in amdgpu_ttm_fini()
2020 iounmap(adev->mman.aper_base_kaddr); in amdgpu_ttm_fini()
2021 adev->mman.aper_base_kaddr = NULL; in amdgpu_ttm_fini()
2029 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); in amdgpu_ttm_fini()
2030 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS); in amdgpu_ttm_fini()
2031 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA); in amdgpu_ttm_fini()
2032 ttm_device_fini(&adev->mman.bdev); in amdgpu_ttm_fini()
2033 adev->mman.initialized = false; in amdgpu_ttm_fini()
2038 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
2048 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); in amdgpu_ttm_set_buffer_funcs_status()
2052 if (!adev->mman.initialized || amdgpu_in_reset(adev) || in amdgpu_ttm_set_buffer_funcs_status()
2053 adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu) in amdgpu_ttm_set_buffer_funcs_status()
2060 ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_set_buffer_funcs_status()
2061 sched = &ring->sched; in amdgpu_ttm_set_buffer_funcs_status()
2062 r = drm_sched_entity_init(&adev->mman.high_pr, in amdgpu_ttm_set_buffer_funcs_status()
2071 r = drm_sched_entity_init(&adev->mman.low_pr, in amdgpu_ttm_set_buffer_funcs_status()
2080 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2081 drm_sched_entity_destroy(&adev->mman.low_pr); in amdgpu_ttm_set_buffer_funcs_status()
2082 dma_fence_put(man->move); in amdgpu_ttm_set_buffer_funcs_status()
2083 man->move = NULL; in amdgpu_ttm_set_buffer_funcs_status()
2088 size = adev->gmc.real_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2090 size = adev->gmc.visible_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2091 man->size = size; in amdgpu_ttm_set_buffer_funcs_status()
2092 adev->mman.buffer_funcs_enabled = enable; in amdgpu_ttm_set_buffer_funcs_status()
2097 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2112 struct drm_sched_entity *entity = delayed ? &adev->mman.low_pr : in amdgpu_ttm_prepare_job()
2113 &adev->mman.high_pr; in amdgpu_ttm_prepare_job()
2121 (*job)->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ? in amdgpu_ttm_prepare_job()
2122 adev->gmc.pdb0_bo : in amdgpu_ttm_prepare_job()
2123 adev->gart.bo); in amdgpu_ttm_prepare_job()
2124 (*job)->vm_needs_flush = true; in amdgpu_ttm_prepare_job()
2129 return drm_sched_job_add_resv_dependencies(&(*job)->base, resv, in amdgpu_ttm_prepare_job()
2139 struct amdgpu_device *adev = ring->adev; in amdgpu_copy_buffer()
2146 if (!direct_submit && !ring->sched.ready) { in amdgpu_copy_buffer()
2148 return -EINVAL; in amdgpu_copy_buffer()
2151 max_bytes = adev->mman.buffer_funcs->copy_max_bytes; in amdgpu_copy_buffer()
2153 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_copy_buffer()
2162 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, in amdgpu_copy_buffer()
2167 byte_count -= cur_size_in_bytes; in amdgpu_copy_buffer()
2170 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_copy_buffer()
2171 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_copy_buffer()
2193 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_fill_mem()
2200 max_bytes = adev->mman.buffer_funcs->fill_max_bytes; in amdgpu_ttm_fill_mem()
2202 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8); in amdgpu_ttm_fill_mem()
2211 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr, in amdgpu_ttm_fill_mem()
2215 byte_count -= cur_size; in amdgpu_ttm_fill_mem()
2218 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_fill_mem()
2219 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_fill_mem()
2230 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_fill_buffer()
2231 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_fill_buffer()
2236 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_fill_buffer()
2238 return -EINVAL; in amdgpu_fill_buffer()
2241 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst); in amdgpu_fill_buffer()
2243 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2251 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &dst, in amdgpu_fill_buffer()
2267 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2275 * amdgpu_ttm_evict_resources - evict memory buffers
2294 man = ttm_manager_type(&adev->mman.bdev, mem_type); in amdgpu_ttm_evict_resources()
2298 return -EINVAL; in amdgpu_ttm_evict_resources()
2301 return ttm_resource_manager_evict_all(&adev->mman.bdev, man); in amdgpu_ttm_evict_resources()
2308 struct amdgpu_device *adev = m->private; in amdgpu_ttm_page_pool_show()
2310 return ttm_pool_debugfs(&adev->mman.bdev.pool, m); in amdgpu_ttm_page_pool_show()
2316 * amdgpu_ttm_vram_read - Linear read access to VRAM
2323 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_read()
2327 return -EINVAL; in amdgpu_ttm_vram_read()
2329 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_read()
2330 return -ENXIO; in amdgpu_ttm_vram_read()
2332 size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos)); in amdgpu_ttm_vram_read()
2339 return -EFAULT; in amdgpu_ttm_vram_read()
2344 size -= bytes; in amdgpu_ttm_vram_read()
2351 * amdgpu_ttm_vram_write - Linear write access to VRAM
2358 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_write()
2363 return -EINVAL; in amdgpu_ttm_vram_write()
2365 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2366 return -ENXIO; in amdgpu_ttm_vram_write()
2371 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2383 size -= 4; in amdgpu_ttm_vram_write()
2397 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2406 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_read()
2412 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_read()
2417 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_read()
2432 return -EPERM; in amdgpu_iomem_read()
2435 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_read()
2436 return -EPERM; in amdgpu_iomem_read()
2442 return -EFAULT; in amdgpu_iomem_read()
2444 size -= bytes; in amdgpu_iomem_read()
2453 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2462 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_write()
2467 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_write()
2472 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_write()
2483 return -EPERM; in amdgpu_iomem_write()
2486 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_write()
2487 return -EPERM; in amdgpu_iomem_write()
2493 return -EFAULT; in amdgpu_iomem_write()
2495 size -= bytes; in amdgpu_iomem_write()
2515 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_ttm_debugfs_init()
2516 struct dentry *root = minor->debugfs_root; in amdgpu_ttm_debugfs_init()
2519 &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size); in amdgpu_ttm_debugfs_init()
2524 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2527 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2530 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2533 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2536 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()