34 	struct hmm_range	*range;  member
51 			 struct hmm_range *range, unsigned long cpu_flags)  in hmm_pfns_fill()  argument
53 	unsigned long i = (addr - range->start) >> PAGE_SHIFT;  in hmm_pfns_fill()
56 		range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS;  in hmm_pfns_fill()
57 		range->hmm_pfns[i] |= cpu_flags;  in hmm_pfns_fill()
63  * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s)
64  * @addr: range virtual start address (inclusive)
65  * @end: range virtual end address (exclusive)
71  * or whenever there is no page directory covering the virtual address range.
100 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_pte_need_fault()  local
104 	 * consider the default flags requested for the range. The API can  in hmm_pte_need_fault()
108 	 * fault a range with specific flags. For the latter one it is a  in hmm_pte_need_fault()
112 	pfn_req_flags &= range->pfn_flags_mask;  in hmm_pte_need_fault()
113 	pfn_req_flags |= range->default_flags;  in hmm_pte_need_fault()
135 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_range_need_fault()  local
144 	if (!((range->default_flags | range->pfn_flags_mask) &  in hmm_range_need_fault()
161 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_hole()  local
166 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_hole()
168 	hmm_pfns = &range->hmm_pfns[i];  in hmm_vma_walk_hole()
174 		return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR);  in hmm_vma_walk_hole()
178 	return hmm_pfns_fill(addr, end, range, 0);  in hmm_vma_walk_hole()
187 static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,  in pmd_to_hmm_pfn_flags()  argument
202 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_handle_pmd()  local
208 	cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);  in hmm_vma_handle_pmd()
227 static inline unsigned long pte_to_hmm_pfn_flags(struct hmm_range *range,  in pte_to_hmm_pfn_flags()  argument
240 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_handle_pte()  local
269 		    range->dev_private_owner) {  in hmm_vma_handle_pte()
303 	cpu_flags = pte_to_hmm_pfn_flags(range, pte);  in hmm_vma_handle_pte()
340 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_handle_absent_pmd()  local
348 	    range->dev_private_owner) {  in hmm_vma_handle_absent_pmd()
379 	return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);  in hmm_vma_handle_absent_pmd()
387 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_handle_absent_pmd()  local
392 	return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);  in hmm_vma_handle_absent_pmd()
402 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_pmd()  local
404 		&range->hmm_pfns[(start - range->start) >> PAGE_SHIFT];  in hmm_vma_walk_pmd()
421 		return hmm_pfns_fill(start, end, range, 0);  in hmm_vma_walk_pmd()
454 		return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);  in hmm_vma_walk_pmd()
474 static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range *range,  in pud_to_hmm_pfn_flags()  argument
488 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_pud()  local
511 		i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pud()
513 		hmm_pfns = &range->hmm_pfns[i];  in hmm_vma_walk_pud()
515 		cpu_flags = pud_to_hmm_pfn_flags(range, pud);  in hmm_vma_walk_pud()
549 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_hugetlb_entry()  local
560 	i = (start - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_hugetlb_entry()
561 	pfn_req_flags = range->hmm_pfns[i];  in hmm_vma_walk_hugetlb_entry()
562 	cpu_flags = pte_to_hmm_pfn_flags(range, entry) |  in hmm_vma_walk_hugetlb_entry()
586 		range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS;  in hmm_vma_walk_hugetlb_entry()
587 		range->hmm_pfns[i] |= pfn | cpu_flags;  in hmm_vma_walk_hugetlb_entry()
601 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_test()  local
616 	 * If a fault is requested for an unsupported range then it is a hard  in hmm_vma_walk_test()
620 				 range->hmm_pfns +  in hmm_vma_walk_test()
621 					 ((start - range->start) >> PAGE_SHIFT),  in hmm_vma_walk_test()
625 	hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);  in hmm_vma_walk_test()
641  * hmm_range_fault - try to fault some address in a virtual address range
642  * @range:	argument structure
649  * -EPERM:	Invalid permission (e.g., asking for write and range is read
651  * -EBUSY:	The range has been invalidated and the caller needs to wait for
659 int hmm_range_fault(struct hmm_range *range)  in hmm_range_fault()  argument
662 		.range = range,  in hmm_range_fault()
663 		.last = range->start,  in hmm_range_fault()
665 	struct mm_struct *mm = range->notifier->mm;  in hmm_range_fault()
671 		/* If range is no longer valid force retry. */  in hmm_range_fault()
672 		if (mmu_interval_check_retry(range->notifier,  in hmm_range_fault()
673 					     range->notifier_seq))  in hmm_range_fault()
675 		ret = walk_page_range(mm, hmm_vma_walk.last, range->end,  in hmm_range_fault()