27 #include <linux/migrate.h>
181  * @mfence: migrate fence
263 static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate)  in svm_migrate_unsuccessful_pages()  argument
268 	for (i = 0; i < migrate->npages; i++) {  in svm_migrate_unsuccessful_pages()
269 		if (migrate->src[i] & MIGRATE_PFN_VALID &&  in svm_migrate_unsuccessful_pages()
270 		    !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in svm_migrate_unsuccessful_pages()
278 			 struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_vram()  argument
281 	uint64_t npages = migrate->cpages;  in svm_migrate_copy_to_vram()
302 		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);  in svm_migrate_copy_to_vram()
303 		svm_migrate_get_vram_page(prange, migrate->dst[i]);  in svm_migrate_copy_to_vram()
304 		migrate->dst[i] = migrate_pfn(migrate->dst[i]);  in svm_migrate_copy_to_vram()
306 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_vram()
358 			migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
367 		migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
369 		migrate->dst[i + 1] = 0;  in svm_migrate_copy_to_vram()
371 		migrate->dst[i + 2] = 0;  in svm_migrate_copy_to_vram()
373 		migrate->dst[i + 3] = 0;  in svm_migrate_copy_to_vram()
390 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_vram()  local
397 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_vram()
398 	migrate.vma = vma;  in svm_migrate_vma_to_vram()
399 	migrate.start = start;  in svm_migrate_vma_to_vram()
400 	migrate.end = end;  in svm_migrate_vma_to_vram()
401 	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;  in svm_migrate_vma_to_vram()
402 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_vram()
405 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_vram()
410 	migrate.src = buf;  in svm_migrate_vma_to_vram()
411 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_vram()
412 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_vram()
419 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_vram()
426 	cpages = migrate.cpages;  in svm_migrate_vma_to_vram()
428 		pr_debug("failed collect migrate sys pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_vram()
438 	r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);  in svm_migrate_vma_to_vram()
439 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_vram()
442 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_vram()
444 	mpages = cpages - svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_vram()
446 			 mpages, cpages, migrate.npages);  in svm_migrate_vma_to_vram()
468  * svm_migrate_ram_to_vram - migrate svm range from system to device
470  * @best_loc: the device to migrate to
471  * @start_mgr: start page to migrate
472  * @last_mgr: last page to migrate
529 			pr_debug("failed %ld to migrate\n", r);  in svm_migrate_ram_to_vram()
563 			struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_ram()  argument
577 	addr = migrate->start;  in svm_migrate_copy_to_ram()
585 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_ram()
611 		dpage = svm_migrate_get_sys_page(migrate->vma, addr);  in svm_migrate_copy_to_ram()
629 		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));  in svm_migrate_copy_to_ram()
641 			migrate->dst[i] = 0;  in svm_migrate_copy_to_ram()
649  * svm_migrate_vma_to_ram - migrate range inside one vma from device to system
655  * @node: kfd node device to migrate from
678 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_ram()  local
683 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_ram()
684 	migrate.vma = vma;  in svm_migrate_vma_to_ram()
685 	migrate.start = start;  in svm_migrate_vma_to_ram()
686 	migrate.end = end;  in svm_migrate_vma_to_ram()
687 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_ram()
689 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_COHERENT;  in svm_migrate_vma_to_ram()
691 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;  in svm_migrate_vma_to_ram()
694 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_ram()
699 	migrate.src = buf;  in svm_migrate_vma_to_ram()
700 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_ram()
701 	migrate.fault_page = fault_page;  in svm_migrate_vma_to_ram()
702 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_ram()
709 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_ram()
716 	cpages = migrate.cpages;  in svm_migrate_vma_to_ram()
718 		pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_ram()
720 		upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
729 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_ram()
731 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_ram()
733 	upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
735 		 upages, cpages, migrate.npages);  in svm_migrate_vma_to_ram()
738 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_ram()
760  * svm_migrate_vram_to_ram - migrate svm range from device to system
785 	/* this pragne has no any vram page to migrate to sys ram */  in svm_migrate_vram_to_ram()
824 			pr_debug("failed %ld to migrate prange %p\n", r, prange);  in svm_migrate_vram_to_ram()
848  * svm_migrate_vram_to_vram - migrate svm range from device to device
850  * @best_loc: the device to migrate to
858  * migrate all vram pages in prange to sys ram, then migrate
977 		pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",  in svm_migrate_to_ram()