Lines Matching +full:addr +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0
5 #include <asm/page-states.h>
37 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
50 static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode) in kasan_populate() argument
54 pgtable_populate(start, end, mode); in kasan_populate()
81 * +- 0 -------------+ +- shadow start -+ in kasan_populate_shadow()
84 * +-end of ident map+ / +----------------+ in kasan_populate_shadow()
87 * +- vmalloc area -+ / | mapping | in kasan_populate_shadow()
89 * +- modules vaddr -+ / +----------------+ in kasan_populate_shadow()
91 * +- shadow start -+ +----------------+ in kasan_populate_shadow()
92 * | 1/8 addr space | | zero pg mapping| (untracked) in kasan_populate_shadow()
93 * +- shadow end ----+---------+- shadow end ---+ in kasan_populate_shadow()
96 * +- 0 -------------+ +- shadow start -+ in kasan_populate_shadow()
99 * +-end of ident map+ / +----------------+ in kasan_populate_shadow()
102 * +- vmalloc area -+ / +----------------+ in kasan_populate_shadow()
104 * +- modules vaddr -+ / +----------------+ in kasan_populate_shadow()
106 * +- shadow start -+ +----------------+ in kasan_populate_shadow()
107 * | 1/8 addr space | | zero pg mapping| (untracked) in kasan_populate_shadow()
108 * +- shadow end ----+---------+- shadow end ---+ in kasan_populate_shadow()
129 static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, in kasan_pgd_populate_zero_shadow() argument
130 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
132 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pgd_populate_zero_shadow()
133 IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { in kasan_pgd_populate_zero_shadow()
140 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
141 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
143 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_p4d_populate_zero_shadow()
144 IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { in kasan_p4d_populate_zero_shadow()
151 static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, in kasan_pud_populate_zero_shadow() argument
152 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
154 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pud_populate_zero_shadow()
155 IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { in kasan_pud_populate_zero_shadow()
162 static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, in kasan_pmd_populate_zero_shadow() argument
163 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
165 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pmd_populate_zero_shadow()
166 IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { in kasan_pmd_populate_zero_shadow()
173 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
175 if (mode == POPULATE_KASAN_ZERO_SHADOW) { in kasan_pte_populate_zero_shadow()
185 static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, in kasan_pgd_populate_zero_shadow() argument
186 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
191 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
192 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
197 static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, in kasan_pud_populate_zero_shadow() argument
198 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
203 static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, in kasan_pmd_populate_zero_shadow() argument
204 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
209 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
257 static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode) in _pa() argument
259 switch (mode) { in _pa()
261 return -1; in _pa()
263 return addr; in _pa()
265 return __abs_lowcore_pa(addr); in _pa()
268 addr = physmem_alloc_top_down(RR_VMEM, size, size); in _pa()
269 memset((void *)addr, 0, size); in _pa()
270 return addr; in _pa()
273 return -1; in _pa()
277 static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end) in can_large_pud() argument
280 IS_ALIGNED(addr, PUD_SIZE) && (end - addr) >= PUD_SIZE; in can_large_pud()
283 static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end) in can_large_pmd() argument
286 IS_ALIGNED(addr, PMD_SIZE) && (end - addr) >= PMD_SIZE; in can_large_pmd()
289 static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end, in pgtable_pte_populate() argument
290 enum populate_mode mode) in pgtable_pte_populate() argument
295 pte = pte_offset_kernel(pmd, addr); in pgtable_pte_populate()
296 for (; addr < end; addr += PAGE_SIZE, pte++) { in pgtable_pte_populate()
298 if (kasan_pte_populate_zero_shadow(pte, mode)) in pgtable_pte_populate()
300 entry = __pte(_pa(addr, PAGE_SIZE, mode)); in pgtable_pte_populate()
308 if (mode == POPULATE_DIRECT) in pgtable_pte_populate()
312 static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end, in pgtable_pmd_populate() argument
313 enum populate_mode mode) in pgtable_pmd_populate() argument
319 pmd = pmd_offset(pud, addr); in pgtable_pmd_populate()
320 for (; addr < end; addr = next, pmd++) { in pgtable_pmd_populate()
321 next = pmd_addr_end(addr, end); in pgtable_pmd_populate()
323 if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode)) in pgtable_pmd_populate()
325 if (can_large_pmd(pmd, addr, next)) { in pgtable_pmd_populate()
326 entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode)); in pgtable_pmd_populate()
339 pgtable_pte_populate(pmd, addr, next, mode); in pgtable_pmd_populate()
341 if (mode == POPULATE_DIRECT) in pgtable_pmd_populate()
345 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, in pgtable_pud_populate() argument
346 enum populate_mode mode) in pgtable_pud_populate() argument
352 pud = pud_offset(p4d, addr); in pgtable_pud_populate()
353 for (; addr < end; addr = next, pud++) { in pgtable_pud_populate()
354 next = pud_addr_end(addr, end); in pgtable_pud_populate()
356 if (kasan_pud_populate_zero_shadow(pud, addr, next, mode)) in pgtable_pud_populate()
358 if (can_large_pud(pud, addr, next)) { in pgtable_pud_populate()
359 entry = __pud(_pa(addr, _REGION3_SIZE, mode)); in pgtable_pud_populate()
372 pgtable_pmd_populate(pud, addr, next, mode); in pgtable_pud_populate()
374 if (mode == POPULATE_DIRECT) in pgtable_pud_populate()
378 static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end, in pgtable_p4d_populate() argument
379 enum populate_mode mode) in pgtable_p4d_populate() argument
385 p4d = p4d_offset(pgd, addr); in pgtable_p4d_populate()
386 for (; addr < end; addr = next, p4d++) { in pgtable_p4d_populate()
387 next = p4d_addr_end(addr, end); in pgtable_p4d_populate()
389 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) in pgtable_p4d_populate()
394 pgtable_pud_populate(p4d, addr, next, mode); in pgtable_p4d_populate()
398 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode) in pgtable_populate() argument
404 pgd = pgd_offset(&init_mm, addr); in pgtable_populate()
405 for (; addr < end; addr = next, pgd++) { in pgtable_populate()
406 next = pgd_addr_end(addr, end); in pgtable_populate()
408 if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode)) in pgtable_populate()
414 if (mode == POPULATE_KASAN_SHALLOW) in pgtable_populate()
417 pgtable_p4d_populate(pgd, addr, next, mode); in pgtable_populate()
429 * Mark whole memory as no-dat. This must be done before any in setup_vmem()
434 __arch_set_page_nodat((void *)start, (end - start) >> PAGE_SHIFT); in setup_vmem()