130b1bc86SAndrew Jones #ifndef _ASMARM64_PGTABLE_H_
230b1bc86SAndrew Jones #define _ASMARM64_PGTABLE_H_
330b1bc86SAndrew Jones /*
430b1bc86SAndrew Jones * Adapted from arch/arm64/include/asm/pgtable.h
530b1bc86SAndrew Jones * include/asm-generic/pgtable-nopmd.h
630b1bc86SAndrew Jones * include/linux/mm.h
730b1bc86SAndrew Jones *
830b1bc86SAndrew Jones * Note: some Linux function APIs have been modified. Nothing crazy,
930b1bc86SAndrew Jones * but if a function took, for example, an mm_struct, then
1030b1bc86SAndrew Jones * that was either removed or replaced.
1149f758b8SAndrew Jones *
1249f758b8SAndrew Jones * Copyright (C) 2017, Red Hat Inc, Andrew Jones <drjones@redhat.com>
1349f758b8SAndrew Jones *
1449f758b8SAndrew Jones * This work is licensed under the terms of the GNU GPL, version 2.
1530b1bc86SAndrew Jones */
1630b1bc86SAndrew Jones #include <alloc.h>
177510bc97SAlexandru Elisei #include <alloc_page.h>
1830b1bc86SAndrew Jones #include <asm/setup.h>
1930b1bc86SAndrew Jones #include <asm/page.h>
2030b1bc86SAndrew Jones #include <asm/pgtable-hwdef.h>
2130b1bc86SAndrew Jones
2270cea146SAlexandru Elisei #include <linux/compiler.h>
2370cea146SAlexandru Elisei
24f02b6363SAndrew Jones /*
25f02b6363SAndrew Jones * We can convert va <=> pa page table addresses with simple casts
26f02b6363SAndrew Jones * because we always allocate their pages with alloc_page(), and
27f02b6363SAndrew Jones * alloc_page() always returns identity mapped pages.
28f02b6363SAndrew Jones */
29f02b6363SAndrew Jones #define pgtable_va(x) ((void *)(unsigned long)(x))
30f02b6363SAndrew Jones #define pgtable_pa(x) ((unsigned long)(x))
31f02b6363SAndrew Jones
3230b1bc86SAndrew Jones #define pgd_none(pgd) (!pgd_val(pgd))
33*a2d06852SNikos Nikoleris #define pud_none(pud) (!pud_val(pud))
3430b1bc86SAndrew Jones #define pmd_none(pmd) (!pmd_val(pmd))
3530b1bc86SAndrew Jones #define pte_none(pte) (!pte_val(pte))
3630b1bc86SAndrew Jones
3702f1cdc8SAlexandru Elisei #define pgd_valid(pgd) (pgd_val(pgd) & PGD_VALID)
38*a2d06852SNikos Nikoleris #define pud_valid(pud) (pud_val(pud) & PUD_VALID)
3902f1cdc8SAlexandru Elisei #define pmd_valid(pmd) (pmd_val(pmd) & PMD_SECT_VALID)
4002f1cdc8SAlexandru Elisei #define pte_valid(pte) (pte_val(pte) & PTE_VALID)
4102f1cdc8SAlexandru Elisei
4202f1cdc8SAlexandru Elisei #define pmd_huge(pmd) \
4302f1cdc8SAlexandru Elisei ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT)
4402f1cdc8SAlexandru Elisei
4530b1bc86SAndrew Jones #define pgd_index(addr) \
4630b1bc86SAndrew Jones (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
4730b1bc86SAndrew Jones #define pgd_offset(pgtable, addr) ((pgtable) + pgd_index(addr))
4830b1bc86SAndrew Jones
4930b1bc86SAndrew Jones #define pgd_free(pgd) free(pgd)
pgd_alloc(void)5030b1bc86SAndrew Jones static inline pgd_t *pgd_alloc(void)
5130b1bc86SAndrew Jones {
5230b1bc86SAndrew Jones pgd_t *pgd = memalign(PAGE_SIZE, PTRS_PER_PGD * sizeof(pgd_t));
5330b1bc86SAndrew Jones memset(pgd, 0, PTRS_PER_PGD * sizeof(pgd_t));
5430b1bc86SAndrew Jones return pgd;
5530b1bc86SAndrew Jones }
5630b1bc86SAndrew Jones
pgd_page_vaddr(pgd_t pgd)57*a2d06852SNikos Nikoleris static inline pud_t *pgd_page_vaddr(pgd_t pgd)
58*a2d06852SNikos Nikoleris {
59*a2d06852SNikos Nikoleris return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
60*a2d06852SNikos Nikoleris }
61*a2d06852SNikos Nikoleris
pud_page_vaddr(pud_t pud)62*a2d06852SNikos Nikoleris static inline pmd_t *pud_page_vaddr(pud_t pud)
63*a2d06852SNikos Nikoleris {
64*a2d06852SNikos Nikoleris return pgtable_va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
65*a2d06852SNikos Nikoleris }
6630b1bc86SAndrew Jones
pmd_page_vaddr(pmd_t pmd)6730b1bc86SAndrew Jones static inline pte_t *pmd_page_vaddr(pmd_t pmd)
6830b1bc86SAndrew Jones {
69f02b6363SAndrew Jones return pgtable_va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK);
7030b1bc86SAndrew Jones }
7130b1bc86SAndrew Jones
72*a2d06852SNikos Nikoleris #if PGTABLE_LEVELS > 2
73*a2d06852SNikos Nikoleris #define pmd_index(addr) \
74*a2d06852SNikos Nikoleris (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
75*a2d06852SNikos Nikoleris #define pmd_offset(pud, addr) \
76*a2d06852SNikos Nikoleris (pud_page_vaddr(*(pud)) + pmd_index(addr))
77*a2d06852SNikos Nikoleris #define pmd_free(pmd) free_page(pmd)
pmd_alloc_one(void)78*a2d06852SNikos Nikoleris static inline pmd_t *pmd_alloc_one(void)
79*a2d06852SNikos Nikoleris {
80*a2d06852SNikos Nikoleris assert(PTRS_PER_PMD * sizeof(pmd_t) == PAGE_SIZE);
81*a2d06852SNikos Nikoleris pmd_t *pmd = alloc_page();
82*a2d06852SNikos Nikoleris return pmd;
83*a2d06852SNikos Nikoleris }
pmd_alloc(pud_t * pud,unsigned long addr)84*a2d06852SNikos Nikoleris static inline pmd_t *pmd_alloc(pud_t *pud, unsigned long addr)
85*a2d06852SNikos Nikoleris {
86*a2d06852SNikos Nikoleris if (pud_none(*pud)) {
87*a2d06852SNikos Nikoleris pud_t entry;
88*a2d06852SNikos Nikoleris pud_val(entry) = pgtable_pa(pmd_alloc_one()) | PMD_TYPE_TABLE;
89*a2d06852SNikos Nikoleris WRITE_ONCE(*pud, entry);
90*a2d06852SNikos Nikoleris }
91*a2d06852SNikos Nikoleris return pmd_offset(pud, addr);
92*a2d06852SNikos Nikoleris }
93*a2d06852SNikos Nikoleris #else
94*a2d06852SNikos Nikoleris #define pmd_offset(pud, addr) ((pmd_t *)pud)
95*a2d06852SNikos Nikoleris #define pmd_free(pmd)
96*a2d06852SNikos Nikoleris #define pmd_alloc(pud, addr) pmd_offset(pud, addr)
97*a2d06852SNikos Nikoleris #endif
98*a2d06852SNikos Nikoleris
99*a2d06852SNikos Nikoleris #if PGTABLE_LEVELS > 3
100*a2d06852SNikos Nikoleris #define pud_index(addr) \
101*a2d06852SNikos Nikoleris (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
102*a2d06852SNikos Nikoleris #define pud_offset(pgd, addr) \
103*a2d06852SNikos Nikoleris (pgd_page_vaddr(*(pgd)) + pud_index(addr))
104*a2d06852SNikos Nikoleris #define pud_free(pud) free_page(pud)
pud_alloc_one(void)105*a2d06852SNikos Nikoleris static inline pud_t *pud_alloc_one(void)
106*a2d06852SNikos Nikoleris {
107*a2d06852SNikos Nikoleris assert(PTRS_PER_PUD * sizeof(pud_t) == PAGE_SIZE);
108*a2d06852SNikos Nikoleris pud_t *pud = alloc_page();
109*a2d06852SNikos Nikoleris return pud;
110*a2d06852SNikos Nikoleris }
pud_alloc(pgd_t * pgd,unsigned long addr)111*a2d06852SNikos Nikoleris static inline pud_t *pud_alloc(pgd_t *pgd, unsigned long addr)
112*a2d06852SNikos Nikoleris {
113*a2d06852SNikos Nikoleris if (pgd_none(*pgd)) {
114*a2d06852SNikos Nikoleris pgd_t entry;
115*a2d06852SNikos Nikoleris pgd_val(entry) = pgtable_pa(pud_alloc_one()) | PMD_TYPE_TABLE;
116*a2d06852SNikos Nikoleris WRITE_ONCE(*pgd, entry);
117*a2d06852SNikos Nikoleris }
118*a2d06852SNikos Nikoleris return pud_offset(pgd, addr);
119*a2d06852SNikos Nikoleris }
120*a2d06852SNikos Nikoleris #else
121*a2d06852SNikos Nikoleris #define pud_offset(pgd, addr) ((pud_t *)pgd)
122*a2d06852SNikos Nikoleris #define pud_free(pud)
123*a2d06852SNikos Nikoleris #define pud_alloc(pgd, addr) pud_offset(pgd, addr)
124*a2d06852SNikos Nikoleris #endif
125*a2d06852SNikos Nikoleris
12630b1bc86SAndrew Jones #define pte_index(addr) \
12730b1bc86SAndrew Jones (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
12830b1bc86SAndrew Jones #define pte_offset(pmd, addr) \
12930b1bc86SAndrew Jones (pmd_page_vaddr(*(pmd)) + pte_index(addr))
13030b1bc86SAndrew Jones
131031755dbSPaolo Bonzini #define pte_free(pte) free_page(pte)
pte_alloc_one(void)13230b1bc86SAndrew Jones static inline pte_t *pte_alloc_one(void)
13330b1bc86SAndrew Jones {
134031755dbSPaolo Bonzini assert(PTRS_PER_PTE * sizeof(pte_t) == PAGE_SIZE);
135031755dbSPaolo Bonzini pte_t *pte = alloc_page();
13630b1bc86SAndrew Jones return pte;
13730b1bc86SAndrew Jones }
pte_alloc(pmd_t * pmd,unsigned long addr)13830b1bc86SAndrew Jones static inline pte_t *pte_alloc(pmd_t *pmd, unsigned long addr)
13930b1bc86SAndrew Jones {
14030b1bc86SAndrew Jones if (pmd_none(*pmd)) {
14170cea146SAlexandru Elisei pmd_t entry;
14270cea146SAlexandru Elisei pmd_val(entry) = pgtable_pa(pte_alloc_one()) | PMD_TYPE_TABLE;
14370cea146SAlexandru Elisei WRITE_ONCE(*pmd, entry);
14430b1bc86SAndrew Jones }
14530b1bc86SAndrew Jones return pte_offset(pmd, addr);
14630b1bc86SAndrew Jones }
14730b1bc86SAndrew Jones
14830b1bc86SAndrew Jones #endif /* _ASMARM64_PGTABLE_H_ */
149