9 #include <linux/migrate.h>
24 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_skip()  local
28 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_skip()
29 		migrate->src[migrate->npages++] = 0;  in migrate_vma_collect_skip()
40 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_hole()  local
48 		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;  in migrate_vma_collect_hole()
49 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_hole()
50 		migrate->npages++;  in migrate_vma_collect_hole()
51 		migrate->cpages++;  in migrate_vma_collect_hole()
62 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_pmd()  local
63 	struct folio *fault_folio = migrate->fault_page ?  in migrate_vma_collect_pmd()
64 		page_folio(migrate->fault_page) : NULL;  in migrate_vma_collect_pmd()
128 				migrate->cpages++;  in migrate_vma_collect_pmd()
145 			if (!(migrate->flags &  in migrate_vma_collect_pmd()
147 			    pgmap->owner != migrate->pgmap_owner)  in migrate_vma_collect_pmd()
157 			    (migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) {  in migrate_vma_collect_pmd()
159 				migrate->cpages++;  in migrate_vma_collect_pmd()
162 			page = vm_normal_page(migrate->vma, addr, pte);  in migrate_vma_collect_pmd()
164 			    !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) {  in migrate_vma_collect_pmd()
169 				if (!(migrate->flags &  in migrate_vma_collect_pmd()
171 					pgmap->owner != migrate->pgmap_owner)  in migrate_vma_collect_pmd()
230 			migrate->cpages++;  in migrate_vma_collect_pmd()
282 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_pmd()
283 		migrate->src[migrate->npages++] = mpfn;  in migrate_vma_collect_pmd()
304  * @migrate: migrate struct containing all migration information
310 static void migrate_vma_collect(struct migrate_vma *migrate)  in migrate_vma_collect()  argument
320 		migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
321 		migrate->pgmap_owner);  in migrate_vma_collect()
324 	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
325 			&migrate_vma_walk_ops, migrate);  in migrate_vma_collect()
328 	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);  in migrate_vma_collect()
457  * @migrate: migrate struct containing all migration information
461  * restored because we cannot migrate them.
466 static void migrate_vma_unmap(struct migrate_vma *migrate)  in migrate_vma_unmap()  argument
468 	migrate->cpages = migrate_device_unmap(migrate->src, migrate->npages,  in migrate_vma_unmap()
469 					migrate->fault_page);  in migrate_vma_unmap()
473  * migrate_vma_setup() - prepare to migrate a range of memory
479  * Prepare to migrate a range of memory virtual address range by collecting all
494  * Note that the caller does not have to migrate all the pages that are marked
496  * device memory to system memory.  If the caller cannot migrate a device page
513  * then migrate_vma_pages() to migrate struct page information from the source
514  * struct page to the destination struct page.  If it fails to migrate the
586 static void migrate_vma_insert_page(struct migrate_vma *migrate,  in migrate_vma_insert_page()  argument
592 	struct vm_area_struct *vma = migrate->vma;  in migrate_vma_insert_page()
704 				struct migrate_vma *migrate)  in __migrate_device_pages()  argument
733 			VM_BUG_ON(!migrate);  in __migrate_device_pages()
734 			addr = migrate->start + i*PAGE_SIZE;  in __migrate_device_pages()
740 					migrate->vma->vm_mm, addr, migrate->end,  in __migrate_device_pages()
741 					migrate->pgmap_owner);  in __migrate_device_pages()
744 			migrate_vma_insert_page(migrate, addr, newpage,  in __migrate_device_pages()
778 		if (migrate && migrate->fault_page == page)  in __migrate_device_pages()
792  * migrate_device_pages() - migrate meta-data from src page to dst page
794  * @dst_pfns: array of pfns allocated by the driver to migrate memory to
797  * Equivalent to migrate_vma_pages(). This is called to migrate struct page
808  * migrate_vma_pages() - migrate meta-data from src page to dst page
809  * @migrate: migrate struct containing all migration information
815 void migrate_vma_pages(struct migrate_vma *migrate)  in migrate_vma_pages()  argument
817 	__migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate);  in migrate_vma_pages()
876  * @dst_pfns: array of pfns allocated by the driver to migrate memory to
892  * @migrate: migrate struct containing all migration information
901 void migrate_vma_finalize(struct migrate_vma *migrate)  in migrate_vma_finalize()  argument
903 	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,  in migrate_vma_finalize()
904 				  migrate->fault_page);  in migrate_vma_finalize()
925  * migrate_device_range() - migrate device private pfns to normal memory.
927  * @start: starting pfn in the range to migrate.
928  * @npages: number of pages to migrate.
958  * migrate_device_pfns() - migrate device private pfns to normal memory.
959  * @src_pfns: pre-popluated array of source device private pfns to migrate.
960  * @npages: number of pages to migrate.
963  * array of device pages to migrate.
979  * Migrate a device coherent folio back to normal memory. The caller should have