27 							unsigned long addr,  in hugetlb_get_unmapped_area_bottomup()  argument
45 	addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_bottomup()
47 	if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {  in hugetlb_get_unmapped_area_bottomup()
48 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_bottomup()
51 		addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_bottomup()
54 	return addr;  in hugetlb_get_unmapped_area_bottomup()
65 	unsigned long addr = addr0;  in hugetlb_get_unmapped_area_topdown()  local
77 	addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_topdown()
85 	if (addr & ~PAGE_MASK) {  in hugetlb_get_unmapped_area_topdown()
86 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_topdown()
90 		addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_topdown()
93 	return addr;  in hugetlb_get_unmapped_area_topdown()
97 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,  in hugetlb_get_unmapped_area()  argument
114 		if (prepare_hugepage_range(file, addr, len))  in hugetlb_get_unmapped_area()
116 		return addr;  in hugetlb_get_unmapped_area()
119 	if (addr) {  in hugetlb_get_unmapped_area()
120 		addr = ALIGN(addr, huge_page_size(h));  in hugetlb_get_unmapped_area()
121 		vma = find_vma(mm, addr);  in hugetlb_get_unmapped_area()
122 		if (task_size - len >= addr &&  in hugetlb_get_unmapped_area()
123 		    (!vma || addr + len <= vm_start_gap(vma)))  in hugetlb_get_unmapped_area()
124 			return addr;  in hugetlb_get_unmapped_area()
127 		return hugetlb_get_unmapped_area_bottomup(file, addr, len,  in hugetlb_get_unmapped_area()
130 		return hugetlb_get_unmapped_area_topdown(file, addr, len,  in hugetlb_get_unmapped_area()
282 			unsigned long addr, unsigned long sz)  in huge_pte_alloc()  argument
289 	pgd = pgd_offset(mm, addr);  in huge_pte_alloc()
290 	p4d = p4d_offset(pgd, addr);  in huge_pte_alloc()
291 	pud = pud_alloc(mm, p4d, addr);  in huge_pte_alloc()
296 	pmd = pmd_alloc(mm, pud, addr);  in huge_pte_alloc()
301 	return pte_alloc_huge(mm, pmd, addr);  in huge_pte_alloc()
305 		       unsigned long addr, unsigned long sz)  in huge_pte_offset()  argument
312 	pgd = pgd_offset(mm, addr);  in huge_pte_offset()
315 	p4d = p4d_offset(pgd, addr);  in huge_pte_offset()
318 	pud = pud_offset(p4d, addr);  in huge_pte_offset()
323 	pmd = pmd_offset(pud, addr);  in huge_pte_offset()
328 	return pte_offset_huge(pmd, addr);  in huge_pte_offset()
331 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,  in __set_huge_pte_at()  argument
353 	addr &= ~(size - 1);  in __set_huge_pte_at()
360 	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);  in __set_huge_pte_at()
363 		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,  in __set_huge_pte_at()
367 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,  in set_huge_pte_at()  argument
370 	__set_huge_pte_at(mm, addr, ptep, entry);  in set_huge_pte_at()
373 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,  in huge_ptep_get_and_clear()  argument
397 	addr &= ~(size - 1);  in huge_ptep_get_and_clear()
401 	maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);  in huge_ptep_get_and_clear()
404 		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,  in huge_ptep_get_and_clear()
423 			   unsigned long addr)  in hugetlb_free_pte_range()  argument
428 	pte_free_tlb(tlb, token, addr);  in hugetlb_free_pte_range()
433 				   unsigned long addr, unsigned long end,  in hugetlb_free_pmd_range()  argument
440 	start = addr;  in hugetlb_free_pmd_range()
441 	pmd = pmd_offset(pud, addr);  in hugetlb_free_pmd_range()
443 		next = pmd_addr_end(addr, end);  in hugetlb_free_pmd_range()
449 			hugetlb_free_pte_range(tlb, pmd, addr);  in hugetlb_free_pmd_range()
450 	} while (pmd++, addr = next, addr != end);  in hugetlb_free_pmd_range()
470 				   unsigned long addr, unsigned long end,  in hugetlb_free_pud_range()  argument
477 	start = addr;  in hugetlb_free_pud_range()
478 	pud = pud_offset(p4d, addr);  in hugetlb_free_pud_range()
480 		next = pud_addr_end(addr, end);  in hugetlb_free_pud_range()
486 			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,  in hugetlb_free_pud_range()
488 	} while (pud++, addr = next, addr != end);  in hugetlb_free_pud_range()
508 			    unsigned long addr, unsigned long end,  in hugetlb_free_pgd_range()  argument
515 	addr &= PMD_MASK;  in hugetlb_free_pgd_range()
516 	if (addr < floor) {  in hugetlb_free_pgd_range()
517 		addr += PMD_SIZE;  in hugetlb_free_pgd_range()
518 		if (!addr)  in hugetlb_free_pgd_range()
528 	if (addr > end - 1)  in hugetlb_free_pgd_range()
531 	pgd = pgd_offset(tlb->mm, addr);  in hugetlb_free_pgd_range()
532 	p4d = p4d_offset(pgd, addr);  in hugetlb_free_pgd_range()
534 		next = p4d_addr_end(addr, end);  in hugetlb_free_pgd_range()
537 		hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);  in hugetlb_free_pgd_range()
538 	} while (p4d++, addr = next, addr != end);  in hugetlb_free_pgd_range()