Lines Matching +full:long +full:- +full:ram +full:- +full:code
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
3 * Copyright 2020-2021 Advanced Micro Devices, Inc.
25 #include <linux/dma-direction.h>
26 #include <linux/dma-mapping.h>
52 struct amdgpu_device *adev = ring->adev; in svm_migrate_gart_map()
62 *gart_addr = adev->gmc.gart_start; in svm_migrate_gart_map()
64 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in svm_migrate_gart_map()
67 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in svm_migrate_gart_map()
76 src_addr += job->ibs[0].gpu_addr; in svm_migrate_gart_map()
78 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); in svm_migrate_gart_map()
79 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, in svm_migrate_gart_map()
82 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in svm_migrate_gart_map()
83 WARN_ON(job->ibs[0].length_dw > num_dw); in svm_migrate_gart_map()
89 pte_flags |= adev->gart.gart_pte_flags; in svm_migrate_gart_map()
91 cpu_addr = &job->ibs[0].ptr[num_dw]; in svm_migrate_gart_map()
101 * svm_migrate_copy_memory_gart - sdma copy data between ram and vram
110 * ram address uses GART table continuous entries mapping to ram pages,
120 * 0 - OK, otherwise error code
130 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in svm_migrate_copy_memory_gart()
136 mutex_lock(&adev->mman.gtt_window_lock); in svm_migrate_copy_memory_gart()
151 dev_err(adev->dev, "fail %d create gart mapping\n", r); in svm_migrate_copy_memory_gart()
158 dev_err(adev->dev, "fail %d to copy memory\n", r); in svm_migrate_copy_memory_gart()
164 npages -= size; in svm_migrate_copy_memory_gart()
172 mutex_unlock(&adev->mman.gtt_window_lock); in svm_migrate_copy_memory_gart()
178 * svm_migrate_copy_done - wait for memory copy sdma is done
189 * 0 - success
190 * otherwise - error code from dma fence signal
206 unsigned long
207 svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr) in svm_migrate_addr_to_pfn()
209 return (addr + adev->kfd.pgmap.range.start) >> PAGE_SHIFT; in svm_migrate_addr_to_pfn()
213 svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn) in svm_migrate_get_vram_page()
218 svm_range_bo_ref(prange->svm_bo); in svm_migrate_get_vram_page()
219 page->zone_device_data = prange->svm_bo; in svm_migrate_get_vram_page()
224 svm_migrate_put_vram_page(struct amdgpu_device *adev, unsigned long addr) in svm_migrate_put_vram_page()
233 static unsigned long
236 unsigned long addr; in svm_migrate_addr()
239 return (addr - adev->kfd.pgmap.range.start); in svm_migrate_addr()
243 svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr) in svm_migrate_get_sys_page()
254 static void svm_migrate_put_sys_page(unsigned long addr) in svm_migrate_put_sys_page()
263 static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate) in svm_migrate_unsuccessful_pages()
265 unsigned long upages = 0; in svm_migrate_unsuccessful_pages()
266 unsigned long i; in svm_migrate_unsuccessful_pages()
268 for (i = 0; i < migrate->npages; i++) { in svm_migrate_unsuccessful_pages()
269 if (migrate->src[i] & MIGRATE_PFN_VALID && in svm_migrate_unsuccessful_pages()
270 !(migrate->src[i] & MIGRATE_PFN_MIGRATE)) in svm_migrate_unsuccessful_pages()
281 uint64_t npages = migrate->cpages; in svm_migrate_copy_to_vram()
282 struct amdgpu_device *adev = node->adev; in svm_migrate_copy_to_vram()
283 struct device *dev = adev->dev; in svm_migrate_copy_to_vram()
290 pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start, in svm_migrate_copy_to_vram()
291 prange->last, ttm_res_offset); in svm_migrate_copy_to_vram()
296 amdgpu_res_first(prange->ttm_res, ttm_res_offset, in svm_migrate_copy_to_vram()
302 migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); in svm_migrate_copy_to_vram()
303 svm_migrate_get_vram_page(prange, migrate->dst[i]); in svm_migrate_copy_to_vram()
304 migrate->dst[i] = migrate_pfn(migrate->dst[i]); in svm_migrate_copy_to_vram()
306 spage = migrate_pfn_to_page(migrate->src[i]); in svm_migrate_copy_to_vram()
319 adev, src + i - j, in svm_migrate_copy_to_vram()
320 dst + i - j, j, in svm_migrate_copy_to_vram()
336 if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) { in svm_migrate_copy_to_vram()
337 r = svm_migrate_copy_memory_gart(adev, src + i - j, in svm_migrate_copy_to_vram()
338 dst + i - j, j + 1, in svm_migrate_copy_to_vram()
350 r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, in svm_migrate_copy_to_vram()
356 while (i--) { in svm_migrate_copy_to_vram()
358 migrate->dst[i] = 0; in svm_migrate_copy_to_vram()
367 migrate->dst[i] = 0; in svm_migrate_copy_to_vram()
369 migrate->dst[i + 1] = 0; in svm_migrate_copy_to_vram()
371 migrate->dst[i + 2] = 0; in svm_migrate_copy_to_vram()
373 migrate->dst[i + 3] = 0; in svm_migrate_copy_to_vram()
380 static long
385 struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); in svm_migrate_vma_to_vram()
386 uint64_t npages = (end - start) >> PAGE_SHIFT; in svm_migrate_vma_to_vram()
387 struct amdgpu_device *adev = node->adev; in svm_migrate_vma_to_vram()
391 unsigned long cpages = 0; in svm_migrate_vma_to_vram()
392 unsigned long mpages = 0; in svm_migrate_vma_to_vram()
395 int r = -ENOMEM; in svm_migrate_vma_to_vram()
414 kfd_smi_event_migration_start(node, p->lead_thread->pid, in svm_migrate_vma_to_vram()
416 0, node->id, prange->prefetch_loc, in svm_migrate_vma_to_vram()
417 prange->preferred_loc, trigger); in svm_migrate_vma_to_vram()
421 dev_err(adev->dev, "%s: vma setup fail %d range [0x%lx 0x%lx]\n", in svm_migrate_vma_to_vram()
422 __func__, r, prange->start, prange->last); in svm_migrate_vma_to_vram()
429 prange->start, prange->last); in svm_migrate_vma_to_vram()
444 mpages = cpages - svm_migrate_unsuccessful_pages(&migrate); in svm_migrate_vma_to_vram()
448 kfd_smi_event_migration_end(node, p->lead_thread->pid, in svm_migrate_vma_to_vram()
450 0, node->id, trigger); in svm_migrate_vma_to_vram()
452 svm_range_dma_unmap_dev(adev->dev, scratch, 0, npages); in svm_migrate_vma_to_vram()
460 WRITE_ONCE(pdd->page_in, pdd->page_in + mpages); in svm_migrate_vma_to_vram()
468 * svm_migrate_ram_to_vram - migrate svm range from system to device
479 * 0 - OK, otherwise error code
483 unsigned long start_mgr, unsigned long last_mgr, in svm_migrate_ram_to_vram()
486 unsigned long addr, start, end; in svm_migrate_ram_to_vram()
490 unsigned long mpages = 0; in svm_migrate_ram_to_vram()
491 long r = 0; in svm_migrate_ram_to_vram()
493 if (start_mgr < prange->start || last_mgr > prange->last) { in svm_migrate_ram_to_vram()
495 start_mgr, last_mgr, prange->start, prange->last); in svm_migrate_ram_to_vram()
496 return -EFAULT; in svm_migrate_ram_to_vram()
502 return -ENODEV; in svm_migrate_ram_to_vram()
506 prange->svms, start_mgr, last_mgr, prange->start, prange->last, in svm_migrate_ram_to_vram()
514 dev_dbg(node->adev->dev, "fail %ld to alloc vram\n", r); in svm_migrate_ram_to_vram()
517 ttm_res_offset = (start_mgr - prange->start + prange->offset) << PAGE_SHIFT; in svm_migrate_ram_to_vram()
520 unsigned long next; in svm_migrate_ram_to_vram()
526 next = min(vma->vm_end, end); in svm_migrate_ram_to_vram()
534 ttm_res_offset += next - addr; in svm_migrate_ram_to_vram()
539 prange->actual_loc = best_loc; in svm_migrate_ram_to_vram()
540 prange->vram_pages += mpages; in svm_migrate_ram_to_vram()
541 } else if (!prange->actual_loc) { in svm_migrate_ram_to_vram()
543 * sys ram drop svm_bo got from svm_range_vram_node_new in svm_migrate_ram_to_vram()
553 struct svm_range_bo *svm_bo = page->zone_device_data; in svm_migrate_page_free()
556 pr_debug_ratelimited("ref: %d\n", kref_read(&svm_bo->kref)); in svm_migrate_page_free()
566 struct device *dev = adev->dev; in svm_migrate_copy_to_ram()
574 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start, in svm_migrate_copy_to_ram()
575 prange->last); in svm_migrate_copy_to_ram()
577 addr = migrate->start; in svm_migrate_copy_to_ram()
585 spage = migrate_pfn_to_page(migrate->src[i]); in svm_migrate_copy_to_ram()
588 prange->svms, prange->start, prange->last); in svm_migrate_copy_to_ram()
590 r = svm_migrate_copy_memory_gart(adev, dst + i - j, in svm_migrate_copy_to_ram()
591 src + i - j, j, in svm_migrate_copy_to_ram()
601 if (j > 0 && src[i] != src[i - 1] + PAGE_SIZE) { in svm_migrate_copy_to_ram()
602 r = svm_migrate_copy_memory_gart(adev, dst + i - j, in svm_migrate_copy_to_ram()
603 src + i - j, j, in svm_migrate_copy_to_ram()
611 dpage = svm_migrate_get_sys_page(migrate->vma, addr); in svm_migrate_copy_to_ram()
614 prange->svms, prange->start, prange->last); in svm_migrate_copy_to_ram()
615 r = -ENOMEM; in svm_migrate_copy_to_ram()
622 dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r); in svm_migrate_copy_to_ram()
629 migrate->dst[i] = migrate_pfn(page_to_pfn(dpage)); in svm_migrate_copy_to_ram()
633 r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j, in svm_migrate_copy_to_ram()
638 pr_debug("failed %d copy to ram\n", r); in svm_migrate_copy_to_ram()
639 while (i--) { in svm_migrate_copy_to_ram()
641 migrate->dst[i] = 0; in svm_migrate_copy_to_ram()
649 * svm_migrate_vma_to_ram - migrate range inside one vma from device to system
657 * @fault_page: is from vmf->page, svm_migrate_to_ram(), this is CPU page fault callback
659 * Context: Process context, caller hold mmap read lock, prange->migrate_mutex
662 * negative values - indicate error
663 * positive values or zero - number of pages got migrated
665 static long
670 struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); in svm_migrate_vma_to_ram()
671 uint64_t npages = (end - start) >> PAGE_SHIFT; in svm_migrate_vma_to_ram()
672 unsigned long upages = npages; in svm_migrate_vma_to_ram()
673 unsigned long cpages = 0; in svm_migrate_vma_to_ram()
674 unsigned long mpages = 0; in svm_migrate_vma_to_ram()
675 struct amdgpu_device *adev = node->adev; in svm_migrate_vma_to_ram()
681 int r = -ENOMEM; in svm_migrate_vma_to_ram()
688 if (adev->gmc.xgmi.connected_to_cpu) in svm_migrate_vma_to_ram()
704 kfd_smi_event_migration_start(node, p->lead_thread->pid, in svm_migrate_vma_to_ram()
706 node->id, 0, prange->prefetch_loc, in svm_migrate_vma_to_ram()
707 prange->preferred_loc, trigger); in svm_migrate_vma_to_ram()
711 dev_err(adev->dev, "%s: vma setup fail %d range [0x%lx 0x%lx]\n", in svm_migrate_vma_to_ram()
712 __func__, r, prange->start, prange->last); in svm_migrate_vma_to_ram()
719 prange->start, prange->last); in svm_migrate_vma_to_ram()
740 kfd_smi_event_migration_end(node, p->lead_thread->pid, in svm_migrate_vma_to_ram()
742 node->id, 0, trigger); in svm_migrate_vma_to_ram()
744 svm_range_dma_unmap_dev(adev->dev, scratch, 0, npages); in svm_migrate_vma_to_ram()
750 mpages = cpages - upages; in svm_migrate_vma_to_ram()
753 WRITE_ONCE(pdd->page_out, pdd->page_out + mpages); in svm_migrate_vma_to_ram()
760 * svm_migrate_vram_to_ram - migrate svm range from device to system
762 * @mm: process mm, use current->mm if NULL
763 * @start_mgr: start page need be migrated to sys ram
764 * @last_mgr: last page need be migrated to sys ram
766 * @fault_page: is from vmf->page, svm_migrate_to_ram(), this is CPU page fault callback
768 * Context: Process context, caller hold mmap read lock, prange->migrate_mutex
771 * 0 - OK, otherwise error code
774 unsigned long start_mgr, unsigned long last_mgr, in svm_migrate_vram_to_ram()
779 unsigned long addr; in svm_migrate_vram_to_ram()
780 unsigned long start; in svm_migrate_vram_to_ram()
781 unsigned long end; in svm_migrate_vram_to_ram()
782 unsigned long mpages = 0; in svm_migrate_vram_to_ram()
783 long r = 0; in svm_migrate_vram_to_ram()
785 /* this pragne has no any vram page to migrate to sys ram */ in svm_migrate_vram_to_ram()
786 if (!prange->actual_loc) { in svm_migrate_vram_to_ram()
787 pr_debug("[0x%lx 0x%lx] already migrated to ram\n", in svm_migrate_vram_to_ram()
788 prange->start, prange->last); in svm_migrate_vram_to_ram()
792 if (start_mgr < prange->start || last_mgr > prange->last) { in svm_migrate_vram_to_ram()
794 start_mgr, last_mgr, prange->start, prange->last); in svm_migrate_vram_to_ram()
795 return -EFAULT; in svm_migrate_vram_to_ram()
798 node = svm_range_get_node_by_id(prange, prange->actual_loc); in svm_migrate_vram_to_ram()
800 pr_debug("failed to get kfd node by id 0x%x\n", prange->actual_loc); in svm_migrate_vram_to_ram()
801 return -ENODEV; in svm_migrate_vram_to_ram()
803 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] from gpu 0x%x to ram\n", in svm_migrate_vram_to_ram()
804 prange->svms, prange, start_mgr, last_mgr, in svm_migrate_vram_to_ram()
805 prange->actual_loc); in svm_migrate_vram_to_ram()
811 unsigned long next; in svm_migrate_vram_to_ram()
816 r = -EFAULT; in svm_migrate_vram_to_ram()
820 next = min(vma->vm_end, end); in svm_migrate_vram_to_ram()
833 prange->vram_pages -= mpages; in svm_migrate_vram_to_ram()
838 if (prange->vram_pages == 0 && prange->ttm_res) { in svm_migrate_vram_to_ram()
839 prange->actual_loc = 0; in svm_migrate_vram_to_ram()
848 * svm_migrate_vram_to_vram - migrate svm range from device to device
851 * @start: start page need be migrated to sys ram
852 * @last: last page need be migrated to sys ram
853 * @mm: process mm, use current->mm if NULL
858 * migrate all vram pages in prange to sys ram, then migrate
859 * [start, last] pages from sys ram to gpu node best_loc.
862 * 0 - OK, otherwise error code
866 unsigned long start, unsigned long last, in svm_migrate_vram_to_vram()
876 pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc); in svm_migrate_vram_to_vram()
879 r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last, in svm_migrate_vram_to_vram()
883 } while (prange->actual_loc && --retries); in svm_migrate_vram_to_vram()
885 if (prange->actual_loc) in svm_migrate_vram_to_vram()
886 return -EDEADLK; in svm_migrate_vram_to_vram()
893 unsigned long start, unsigned long last, in svm_migrate_to_vram()
896 if (!prange->actual_loc || prange->actual_loc == best_loc) in svm_migrate_to_vram()
907 * svm_migrate_to_ram - CPU page fault handler
913 * 0 - OK
914 * VM_FAULT_SIGBUS - notice application to have SIGBUS page fault
918 unsigned long start, last, size; in svm_migrate_to_ram()
919 unsigned long addr = vmf->address; in svm_migrate_to_ram()
926 svm_bo = vmf->page->zone_device_data; in svm_migrate_to_ram()
931 if (!mmget_not_zero(svm_bo->eviction_fence->mm)) { in svm_migrate_to_ram()
936 mm = svm_bo->eviction_fence->mm; in svm_migrate_to_ram()
937 if (mm != vmf->vma->vm_mm) in svm_migrate_to_ram()
946 if (READ_ONCE(p->svms.faulting_task) == current) { in svm_migrate_to_ram()
947 pr_debug("skipping ram migration\n"); in svm_migrate_to_ram()
952 pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr); in svm_migrate_to_ram()
955 mutex_lock(&p->svms.lock); in svm_migrate_to_ram()
957 prange = svm_range_from_addr(&p->svms, addr, NULL); in svm_migrate_to_ram()
959 pr_debug("failed get range svms 0x%p addr 0x%lx\n", &p->svms, addr); in svm_migrate_to_ram()
960 r = -EFAULT; in svm_migrate_to_ram()
964 mutex_lock(&prange->migrate_mutex); in svm_migrate_to_ram()
966 if (!prange->actual_loc) in svm_migrate_to_ram()
970 size = 1UL << prange->granularity; in svm_migrate_to_ram()
971 start = max(ALIGN_DOWN(addr, size), prange->start); in svm_migrate_to_ram()
972 last = min(ALIGN(addr + 1, size) - 1, prange->last); in svm_migrate_to_ram()
974 r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm, start, last, in svm_migrate_to_ram()
975 KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU, vmf->page); in svm_migrate_to_ram()
978 r, prange->svms, prange, start, last); in svm_migrate_to_ram()
981 mutex_unlock(&prange->migrate_mutex); in svm_migrate_to_ram()
983 mutex_unlock(&p->svms.lock); in svm_migrate_to_ram()
985 pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr); in svm_migrate_to_ram()
1002 struct amdgpu_kfd_dev *kfddev = &adev->kfd; in kgd2kfd_init_zone_device()
1005 unsigned long size; in kgd2kfd_init_zone_device()
1010 return -EINVAL; in kgd2kfd_init_zone_device()
1012 if (adev->gmc.is_app_apu) in kgd2kfd_init_zone_device()
1015 pgmap = &kfddev->pgmap; in kgd2kfd_init_zone_device()
1021 size = ALIGN(adev->gmc.real_vram_size, 2ULL << 20); in kgd2kfd_init_zone_device()
1022 if (adev->gmc.xgmi.connected_to_cpu) { in kgd2kfd_init_zone_device()
1023 pgmap->range.start = adev->gmc.aper_base; in kgd2kfd_init_zone_device()
1024 pgmap->range.end = adev->gmc.aper_base + adev->gmc.aper_size - 1; in kgd2kfd_init_zone_device()
1025 pgmap->type = MEMORY_DEVICE_COHERENT; in kgd2kfd_init_zone_device()
1027 res = devm_request_free_mem_region(adev->dev, &iomem_resource, size); in kgd2kfd_init_zone_device()
1030 pgmap->range.start = res->start; in kgd2kfd_init_zone_device()
1031 pgmap->range.end = res->end; in kgd2kfd_init_zone_device()
1032 pgmap->type = MEMORY_DEVICE_PRIVATE; in kgd2kfd_init_zone_device()
1035 pgmap->nr_range = 1; in kgd2kfd_init_zone_device()
1036 pgmap->ops = &svm_migrate_pgmap_ops; in kgd2kfd_init_zone_device()
1037 pgmap->owner = SVM_ADEV_PGMAP_OWNER(adev); in kgd2kfd_init_zone_device()
1038 pgmap->flags = 0; in kgd2kfd_init_zone_device()
1039 /* Device manager releases device-specific resources, memory region and in kgd2kfd_init_zone_device()
1042 r = devm_memremap_pages(adev->dev, pgmap); in kgd2kfd_init_zone_device()
1045 if (pgmap->type == MEMORY_DEVICE_PRIVATE) in kgd2kfd_init_zone_device()
1046 devm_release_mem_region(adev->dev, res->start, resource_size(res)); in kgd2kfd_init_zone_device()
1048 pgmap->type = 0; in kgd2kfd_init_zone_device()