1 // SPDX-License-Identifier: GPL-2.0
3  * SPARC64 Huge TLB page support.
17 #include <asm/tlb.h>
22 /* Slightly simplified from the non-hugepage variant because by
48 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_bottomup()
64 	struct mm_struct *mm = current->mm;  in hugetlb_get_unmapped_area_topdown()
68 	/* This should only ever run for 32-bit processes.  */  in hugetlb_get_unmapped_area_topdown()
74 	info.high_limit = mm->mmap_base;  in hugetlb_get_unmapped_area_topdown()
81 	 * so fall back to the bottom-up function here. This scenario  in hugetlb_get_unmapped_area_topdown()
86 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_topdown()
101 	struct mm_struct *mm = current->mm;  in hugetlb_get_unmapped_area()
109 		return -EINVAL;  in hugetlb_get_unmapped_area()
111 		return -ENOMEM;  in hugetlb_get_unmapped_area()
115 			return -EINVAL;  in hugetlb_get_unmapped_area()
122 		if (task_size - len >= addr &&  in hugetlb_get_unmapped_area()
126 	if (mm->get_unmapped_area == arch_get_unmapped_area)  in hugetlb_get_unmapped_area()
191 	if (vma->vm_flags & VM_SPARC_ADI)  in arch_make_huge_pte()
268 	unsigned long size = 1UL << huge_tte_to_shift(pte);  in huge_tte_to_size()  local
270 	if (size == REAL_HPAGE_SIZE)  in huge_tte_to_size()
271 		size = HPAGE_SIZE;  in huge_tte_to_size()
272 	return size;  in huge_tte_to_size()
329 	unsigned long i, size;  in set_huge_pte_at()  local
332 	size = huge_tte_to_size(entry);  in set_huge_pte_at()
335 	if (size >= PUD_SIZE)  in set_huge_pte_at()
337 	else if (size >= PMD_SIZE)  in set_huge_pte_at()
342 	nptes = size >> shift;  in set_huge_pte_at()
345 		mm->context.hugetlb_pte_count += nptes;  in set_huge_pte_at()
347 	addr &= ~(size - 1);  in set_huge_pte_at()
351 	for (i = 0; i < nptes; i++)  in set_huge_pte_at()
352 		ptep[i] = __pte(pte_val(entry) + (i << shift));  in set_huge_pte_at()
356 	if (size == HPAGE_SIZE)  in set_huge_pte_at()
364 	unsigned int i, nptes, orig_shift, shift;  in huge_ptep_get_and_clear()  local
365 	unsigned long size;  in huge_ptep_get_and_clear()  local
369 	size = huge_tte_to_size(entry);  in huge_ptep_get_and_clear()
372 	if (size >= PUD_SIZE)  in huge_ptep_get_and_clear()
374 	else if (size >= PMD_SIZE)  in huge_ptep_get_and_clear()
379 	nptes = size >> shift;  in huge_ptep_get_and_clear()
383 		mm->context.hugetlb_pte_count -= nptes;  in huge_ptep_get_and_clear()
385 	addr &= ~(size - 1);  in huge_ptep_get_and_clear()
386 	for (i = 0; i < nptes; i++)  in huge_ptep_get_and_clear()
387 		ptep[i] = __pte(0UL);  in huge_ptep_get_and_clear()
391 	if (size == HPAGE_SIZE)  in huge_ptep_get_and_clear()
410 static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,  in hugetlb_free_pte_range()  argument
416 	pte_free_tlb(tlb, token, addr);  in hugetlb_free_pte_range()
417 	mm_dec_nr_ptes(tlb->mm);  in hugetlb_free_pte_range()
420 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,  in hugetlb_free_pmd_range()  argument
437 			hugetlb_free_pte_range(tlb, pmd, addr);  in hugetlb_free_pmd_range()
448 	if (end - 1 > ceiling - 1)  in hugetlb_free_pmd_range()
453 	pmd_free_tlb(tlb, pmd, start);  in hugetlb_free_pmd_range()
454 	mm_dec_nr_pmds(tlb->mm);  in hugetlb_free_pmd_range()
457 static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,  in hugetlb_free_pud_range()  argument
474 			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,  in hugetlb_free_pud_range()
486 	if (end - 1 > ceiling - 1)  in hugetlb_free_pud_range()
491 	pud_free_tlb(tlb, pud, start);  in hugetlb_free_pud_range()
492 	mm_dec_nr_puds(tlb->mm);  in hugetlb_free_pud_range()
495 void hugetlb_free_pgd_range(struct mmu_gather *tlb,  in hugetlb_free_pgd_range()  argument
514 	if (end - 1 > ceiling - 1)  in hugetlb_free_pgd_range()
515 		end -= PMD_SIZE;  in hugetlb_free_pgd_range()
516 	if (addr > end - 1)  in hugetlb_free_pgd_range()
519 	pgd = pgd_offset(tlb->mm, addr);  in hugetlb_free_pgd_range()
525 		hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);  in hugetlb_free_pgd_range()