1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Radix MMU support
4 *
5 * Copyright (C) 2024, IBM Inc, Nicholas Piggin <npiggin@gmail.com>
6 *
7 * Derived from Linux kernel MMU code.
8 */
9 #include <asm/mmu.h>
10 #include <asm/setup.h>
11 #include <asm/smp.h>
12 #include <asm/page.h>
13 #include <asm/io.h>
14 #include <asm/processor.h>
15 #include <asm/hcall.h>
16
17 #include "alloc_page.h"
18 #include "vmalloc.h"
19 #include <asm/pgtable-hwdef.h>
20 #include <asm/pgtable.h>
21
22 #include <linux/compiler.h>
23
24 static pgd_t *identity_pgd;
25
vm_available(void)26 bool vm_available(void) /* weak override */
27 {
28 return cpu_has_radix;
29 }
30
mmu_enabled(void)31 bool mmu_enabled(void)
32 {
33 return current_cpu()->pgtable != NULL;
34 }
35
mmu_enable(pgd_t * pgtable)36 void mmu_enable(pgd_t *pgtable)
37 {
38 struct cpu *cpu = current_cpu();
39
40 if (!pgtable)
41 pgtable = identity_pgd;
42
43 cpu->pgtable = pgtable;
44
45 assert(!in_usermode());
46 mtmsr(mfmsr() | (MSR_IR|MSR_DR));
47 }
48
mmu_disable(void)49 void mmu_disable(void)
50 {
51 struct cpu *cpu = current_cpu();
52
53 cpu->pgtable = NULL;
54
55 assert(!in_usermode());
56 mtmsr(mfmsr() & ~(MSR_IR|MSR_DR));
57 }
58
get_pte(pgd_t * pgtable,uintptr_t vaddr)59 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
60 {
61 pgd_t *pgd = pgd_offset(pgtable, vaddr);
62 pud_t *pud = pud_alloc(pgd, vaddr);
63 pmd_t *pmd = pmd_alloc(pud, vaddr);
64 pte_t *pte = pte_alloc(pmd, vaddr);
65
66 return &pte_val(*pte);
67 }
68
install_pte(pgd_t * pgtable,uintptr_t vaddr,pteval_t pte)69 static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte)
70 {
71 pteval_t *p_pte = get_pte(pgtable, vaddr);
72
73 if (READ_ONCE(*p_pte) & cpu_to_be64(_PAGE_VALID)) {
74 WRITE_ONCE(*p_pte, 0);
75 flush_tlb_page(vaddr);
76 }
77
78 WRITE_ONCE(*p_pte, cpu_to_be64(pte));
79
80 return p_pte;
81 }
82
install_page_prot(pgd_t * pgtable,phys_addr_t phys,uintptr_t vaddr,pgprot_t prot)83 static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys,
84 uintptr_t vaddr, pgprot_t prot)
85 {
86 pteval_t pte = phys;
87 pte |= _PAGE_VALID | _PAGE_PTE;
88 pte |= pgprot_val(prot);
89 return install_pte(pgtable, vaddr, pte);
90 }
91
install_page(pgd_t * pgtable,phys_addr_t phys,void * virt)92 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt)
93 {
94 if (!pgtable)
95 pgtable = identity_pgd;
96
97 return install_page_prot(pgtable, phys, (uintptr_t)virt,
98 __pgprot(_PAGE_VALID | _PAGE_PTE |
99 _PAGE_READ | _PAGE_WRITE |
100 _PAGE_EXEC | _PAGE_ACCESSED |
101 _PAGE_DIRTY));
102 }
103
follow_pte(pgd_t * pgtable,uintptr_t vaddr)104 static pteval_t *follow_pte(pgd_t *pgtable, uintptr_t vaddr)
105 {
106 pgd_t *pgd;
107 pud_t *pud;
108 pmd_t *pmd;
109 pte_t *pte;
110
111 pgd = pgd_offset(pgtable, vaddr);
112 if (!pgd_valid(*pgd))
113 return NULL;
114
115 pud = pud_offset(pgd, vaddr);
116 if (!pud_valid(*pud))
117 return NULL;
118
119 pmd = pmd_offset(pud, vaddr);
120 if (!pmd_valid(*pmd))
121 return NULL;
122 if (pmd_huge(*pmd))
123 return &pmd_val(*pmd);
124
125 pte = pte_offset(pmd, vaddr);
126 if (!pte_valid(*pte))
127 return NULL;
128
129 return &pte_val(*pte);
130 }
131
virt_to_pte_phys(pgd_t * pgtable,void * virt)132 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *virt)
133 {
134 phys_addr_t mask;
135 pteval_t *pteval;
136
137 if (!pgtable)
138 pgtable = identity_pgd;
139
140 pteval = follow_pte(pgtable, (uintptr_t)virt);
141 if (!pteval) {
142 install_page(pgtable, (phys_addr_t)(unsigned long)virt, virt);
143 return (phys_addr_t)(unsigned long)virt;
144 }
145
146 if (pmd_huge(__pmd(*pteval)))
147 mask = PMD_MASK;
148 else
149 mask = PAGE_MASK;
150
151 return (be64_to_cpu(*pteval) & PHYS_MASK & mask) |
152 ((phys_addr_t)(unsigned long)virt & ~mask);
153 }
154
155 struct partition_table_entry {
156 uint64_t dw0;
157 uint64_t dw1;
158 };
159
160 static struct partition_table_entry *partition_table;
161
162 struct process_table_entry {
163 uint64_t dw0;
164 uint64_t dw1;
165 };
166
167 static struct process_table_entry *process_table;
168
setup_mmu(phys_addr_t phys_end,void * unused)169 void *setup_mmu(phys_addr_t phys_end, void *unused)
170 {
171 phys_addr_t addr;
172 uint64_t dw0, dw1;
173
174 if (identity_pgd)
175 goto enable;
176
177 assert_msg(cpu_has_radix, "MMU support requires radix MMU.");
178
179 /* 32G address is reserved for vmalloc, cap phys_end at 31G */
180 if (phys_end > (31ul << 30)) {
181 /* print warning */
182 phys_end = 31ul << 30;
183 }
184
185 init_alloc_vpage((void *)(32ul << 30));
186
187 process_table = memalign_pages(SZ_4K, SZ_4K);
188 memset(process_table, 0, SZ_4K);
189
190 identity_pgd = pgd_alloc_one();
191
192 dw0 = (unsigned long)identity_pgd;
193 dw0 |= 16UL - 3; /* 64K pgd size */
194 dw0 |= (0x2UL << 61) | (0x5UL << 5); /* 52-bit virt */
195 process_table[1].dw0 = cpu_to_be64(dw0);
196
197 if (machine_is_pseries()) {
198 int ret;
199
200 ret = hcall(H_REGISTER_PROCESS_TABLE, PTBL_NEW | PTBL_RADIX | PTBL_GTSE, process_table, 0, 0 /* 4K size */);
201 assert_msg(!ret, "H_REGISTER_PROCESS_TABLE failed! err=%d\n", ret);
202 } else if (machine_is_powernv()) {
203 partition_table = memalign_pages(SZ_4K, SZ_4K);
204 memset(partition_table, 0, SZ_4K);
205
206 /* Reuse dw0 for partition table */
207 dw0 |= 1ULL << 63; /* Host radix */
208 dw1 = (unsigned long)process_table; /* 4K size */
209 partition_table[0].dw0 = cpu_to_be64(dw0);
210 partition_table[0].dw1 = cpu_to_be64(dw1);
211
212 } else {
213 /* Only pseries and powernv support radix so far */
214 assert(0);
215 }
216
217 /*
218 * Avoid mapping page 0 so NULL dereferences fault. Test images
219 * run relocated well above 0, so nothing useful here except
220 * real-mode interrupt entry code.
221 */
222 for (addr = PAGE_SIZE; addr < phys_end; addr += PAGE_SIZE)
223 install_page(identity_pgd, addr, __va(addr));
224
225 enable:
226 if (machine_is_powernv()) {
227 mtspr(SPR_PTCR, (unsigned long)partition_table); /* 4KB size */
228
229 mtspr(SPR_LPIDR, 0);
230 /* Set LPCR[UPRT] and LPCR[HR] for radix */
231 mtspr(SPR_LPCR, mfspr(SPR_LPCR) | (1ULL << 22) | (1ULL << 20));
232 }
233
234 /* PID=1 is used because PID=0 is also mapped in quadrant 3 */
235 mtspr(SPR_PIDR, 1);
236
237 mmu_enable(identity_pgd);
238
239 return identity_pgd;
240 }
241
__virt_to_phys(unsigned long addr)242 phys_addr_t __virt_to_phys(unsigned long addr)
243 {
244 if (mmu_enabled()) {
245 pgd_t *pgtable = current_cpu()->pgtable;
246 return virt_to_pte_phys(pgtable, (void *)addr);
247 }
248 return addr;
249 }
250
__phys_to_virt(phys_addr_t addr)251 unsigned long __phys_to_virt(phys_addr_t addr)
252 {
253 /*
254 * We don't guarantee that phys_to_virt(virt_to_phys(vaddr)) == vaddr, but
255 * the default page tables do identity map all physical addresses, which
256 * means phys_to_virt(virt_to_phys((void *)paddr)) == paddr.
257 */
258 assert(!mmu_enabled() || __virt_to_phys(addr) == addr);
259 return addr;
260 }
261