arm64/mm/mmu.c

177 static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
185 		 * are deferred to the end of alloc_init_cont_pte().
197 	} while (ptep++, addr += PAGE_SIZE, addr != end);
201 				unsigned long end, phys_addr_t phys,
231 		next = pte_cont_addr_end(addr, end);
242 	} while (addr = next, addr != end);
252 static void init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
261 		next = pmd_addr_end(addr, end);
282 	} while (pmdp++, addr = next, addr != end);
286 				unsigned long end, phys_addr_t phys,
319 		next = pmd_cont_addr_end(addr, end);
330 	} while (addr = next, addr != end);
335 static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
364 		next = pud_addr_end(addr, end);
388 	} while (pudp++, addr = next, addr != end);
393 static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
422 		next = p4d_addr_end(addr, end);
431 	} while (p4dp++, addr = next, addr != end);
442 	unsigned long addr, end, next;
454 	end = PAGE_ALIGN(virt + size);
457 		next = pgd_addr_end(addr, end);
461 	} while (pgdp++, addr = next, addr != end);
573 				  phys_addr_t end, pgprot_t prot, int flags)
575 	__create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
648 	phys_addr_t start, end;
680 	for_each_mem_range(i, &start, &end) {
681 		if (start >= end)
688 		__map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL),
797 void __pi_map_range(u64 *pgd, u64 start, u64 end, u64 pa, pgprot_t prot,
806 	u64 end   = __pa_symbol(__idmap_text_end);
809 	__pi_map_range(&ptep, start, end, start, PAGE_KERNEL_ROX,
855 static bool pgtable_range_aligned(unsigned long start, unsigned long end,
869 	if (end - 1 > ceiling - 1)
875 				    unsigned long end, bool free_mapped,
892 	} while (addr += PAGE_SIZE, addr < end);
896 				    unsigned long end, bool free_mapped,
903 		next = pmd_addr_end(addr, end);
925 	} while (addr = next, addr < end);
929 				    unsigned long end, bool free_mapped,
936 		next = pud_addr_end(addr, end);
958 	} while (addr = next, addr < end);
962 				    unsigned long end, bool free_mapped,
969 		next = p4d_addr_end(addr, end);
977 	} while (addr = next, addr < end);
980 static void unmap_hotplug_range(unsigned long addr, unsigned long end,
995 		next = pgd_addr_end(addr, end);
1003 	} while (addr = next, addr < end);
1007 				 unsigned long end, unsigned long floor,
1022 	} while (addr += PAGE_SIZE, addr < end);
1024 	if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK))
1044 				 unsigned long end, unsigned long floor,
1051 		next = pmd_addr_end(addr, end);
1059 	} while (addr = next, addr < end);
1064 	if (!pgtable_range_aligned(start, end, floor, ceiling, PUD_MASK))
1084 				 unsigned long end, unsigned long floor,
1091 		next = pud_addr_end(addr, end);
1099 	} while (addr = next, addr < end);
1104 	if (!pgtable_range_aligned(start, end, floor, ceiling, P4D_MASK))
1124 				 unsigned long end, unsigned long floor,
1131 		next = p4d_addr_end(addr, end);
1139 	} while (addr = next, addr < end);
1144 	if (!pgtable_range_aligned(start, end, floor, ceiling, PGDIR_MASK))
1163 static void free_empty_tables(unsigned long addr, unsigned long end,
1170 		next = pgd_addr_end(addr, end);
1178 	} while (addr = next, addr < end);
1196 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
1199 	WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
1200 	/* [start, end] should be within one section */
1201 	WARN_ON_ONCE(end - start > PAGES_PER_SECTION * sizeof(struct page));
1204 	    (end - start < PAGES_PER_SECTION * sizeof(struct page)))
1205 		return vmemmap_populate_basepages(start, end, node, altmap);
1207 		return vmemmap_populate_hugepages(start, end, node, altmap);
1211 void vmemmap_free(unsigned long start, unsigned long end,
1214 	WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
1216 	unmap_hotplug_range(start, end, true, altmap);
1217 	free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END);
1295 	unsigned long next, end;
1307 	end = addr + PUD_SIZE;
1311 	} while (pmdp++, next += PMD_SIZE, next != end);
1322 	unsigned long end = start + size;
1325 	WARN_ON((start < PAGE_OFFSET) || (end > PAGE_END));
1327 	unmap_hotplug_range(start, end, false, NULL);
1328 	free_empty_tables(start, end, PAGE_OFFSET, PAGE_END);
1341 		 * the end physical address. In this case set start to zero
1355 	 * also be derived from its end points.
1358 	mhp_range.end =  end_linear_pa;
1424 		unsigned long end = start + (1UL << PA_SECTION_SHIFT);
1436 			pr_warn("Boot memory [%lx %lx] offlining attempted\n", start, end);
1446 			pr_err("Boot memory [%lx %lx] offlined\n", start, end);
1474 	phys_addr_t start, end, addr;
1486 	for_each_mem_range(i, &start, &end) {
1487 		for (addr = start; addr < end; addr += (1UL << PA_SECTION_SHIFT)) {