linux/mm/memory.c

796 		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
799 	vm_flags_t vm_flags = dst_vma->vm_flags;
861 		folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma);
891 		pte_marker marker = copy_pte_marker(entry, dst_vma);
898 	if (!userfaultfd_wp(dst_vma))
917 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
938 	folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE);
939 	folio_add_lru_vma(new_folio, dst_vma);
943 	pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot);
944 	pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
945 	if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
948 	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
952 static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
969 	if (!userfaultfd_wp(dst_vma))
972 	set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
983 copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1013 								  nr, dst_vma, src_vma))) {
1020 			folio_dup_file_rmap_ptes(folio, page, nr, dst_vma);
1023 		__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte,
1036 		if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) {
1039 			err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
1046 		folio_dup_file_rmap_pte(folio, page, dst_vma);
1051 	__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1);
1078 copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1082 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1152 						  dst_vma, src_vma,
1174 		ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte,
1233 copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1237 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1252 					    addr, dst_vma, src_vma);
1261 		if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1269 copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1273 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1298 		if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1306 copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1310 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1322 		if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1335 vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1338 	 * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
1343 	if (userfaultfd_wp(dst_vma))
1362 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1367 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1374 	if (!vma_needs_copy(dst_vma, src_vma))
1378 		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
1410 		if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,