1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * s390x MMU
4 *
5 * Copyright (c) 2017 Red Hat Inc
6 *
7 * Authors:
8 * David Hildenbrand <david@redhat.com>
9 */
10
11 #include <libcflat.h>
12 #include <asm/pgtable.h>
13 #include <asm/arch_def.h>
14 #include <asm/barrier.h>
15 #include <asm/interrupt.h>
16 #include <vmalloc.h>
17 #include "mmu.h"
18
19 /*
20 * The naming convention used here is the same as used in the Linux kernel;
21 * this is the correspondence between the s390x architectural names and the
22 * Linux ones:
23 *
24 * pgd - region 1 table entry
25 * p4d - region 2 table entry
26 * pud - region 3 table entry
27 * pmd - segment table entry
28 * pte - page table entry
29 */
30
31 static pgd_t *table_root;
32
mmu_enable(pgd_t * pgtable)33 static void mmu_enable(pgd_t *pgtable)
34 {
35 const uint64_t asce = __pa(pgtable) | ASCE_DT_REGION1 |
36 REGION_TABLE_LENGTH;
37
38 /* set primary asce */
39 lctlg(1, asce);
40 assert(stctg(1) == asce);
41
42 /* enable dat (primary == 0 set as default) */
43 enable_dat();
44
45 /* we can now also use DAT in all interrupt handlers */
46 irq_set_dat_mode(true, AS_PRIM);
47 }
48
49 /*
50 * Get the pud (region 3) DAT table entry for the given address and root,
51 * allocating it if necessary
52 */
get_pud(pgd_t * pgtable,uintptr_t vaddr)53 static inline pud_t *get_pud(pgd_t *pgtable, uintptr_t vaddr)
54 {
55 pgd_t *pgd = pgd_offset(pgtable, vaddr);
56 p4d_t *p4d = p4d_alloc(pgd, vaddr);
57 pud_t *pud = pud_alloc(p4d, vaddr);
58
59 return pud;
60 }
61
62 /*
63 * Get the pmd (segment) DAT table entry for the given address and pud,
64 * allocating it if necessary.
65 * The pud must not be huge.
66 */
get_pmd(pud_t * pud,uintptr_t vaddr)67 static inline pmd_t *get_pmd(pud_t *pud, uintptr_t vaddr)
68 {
69 pmd_t *pmd;
70
71 assert(!pud_huge(*pud));
72 pmd = pmd_alloc(pud, vaddr);
73 return pmd;
74 }
75
76 /*
77 * Get the pte (page) DAT table entry for the given address and pmd,
78 * allocating it if necessary.
79 * The pmd must not be large.
80 */
get_pte(pmd_t * pmd,uintptr_t vaddr)81 static inline pte_t *get_pte(pmd_t *pmd, uintptr_t vaddr)
82 {
83 pte_t *pte;
84
85 assert(!pmd_large(*pmd));
86 pte = pte_alloc(pmd, vaddr);
87 return pte;
88 }
89
90 /*
91 * Splits a large pmd (segment) DAT table entry into equivalent 4kB small
92 * pages.
93 * @pmd The pmd to split, it must be large.
94 * @va the virtual address corresponding to this pmd.
95 */
split_pmd(pmd_t * pmd,uintptr_t va)96 static void split_pmd(pmd_t *pmd, uintptr_t va)
97 {
98 phys_addr_t pa = pmd_val(*pmd) & SEGMENT_ENTRY_SFAA;
99 unsigned long i, prot;
100 pte_t *pte;
101
102 assert(pmd_large(*pmd));
103 pte = alloc_pages(PAGE_TABLE_ORDER);
104 prot = pmd_val(*pmd) & (SEGMENT_ENTRY_IEP | SEGMENT_ENTRY_P);
105 for (i = 0; i < PAGE_TABLE_ENTRIES; i++)
106 pte_val(pte[i]) = pa | PAGE_SIZE * i | prot;
107 idte_pmdp(va, &pmd_val(*pmd));
108 pmd_val(*pmd) = __pa(pte) | SEGMENT_ENTRY_TT_SEGMENT;
109
110 }
111
112 /*
113 * Splits a huge pud (region 3) DAT table entry into equivalent 1MB large
114 * pages.
115 * @pud The pud to split, it must be huge.
116 * @va the virtual address corresponding to this pud.
117 */
split_pud(pud_t * pud,uintptr_t va)118 static void split_pud(pud_t *pud, uintptr_t va)
119 {
120 phys_addr_t pa = pud_val(*pud) & REGION3_ENTRY_RFAA;
121 unsigned long i, prot;
122 pmd_t *pmd;
123
124 assert(pud_huge(*pud));
125 pmd = alloc_pages(SEGMENT_TABLE_ORDER);
126 prot = pud_val(*pud) & (REGION3_ENTRY_IEP | REGION_ENTRY_P);
127 for (i = 0; i < SEGMENT_TABLE_ENTRIES; i++)
128 pmd_val(pmd[i]) = pa | SZ_1M * i | prot | SEGMENT_ENTRY_FC | SEGMENT_ENTRY_TT_SEGMENT;
129 idte_pudp(va, &pud_val(*pud));
130 pud_val(*pud) = __pa(pmd) | REGION_ENTRY_TT_REGION3 | REGION_TABLE_LENGTH;
131 }
132
get_dat_entry(pgd_t * pgtable,void * vaddr,enum pgt_level level)133 void *get_dat_entry(pgd_t *pgtable, void *vaddr, enum pgt_level level)
134 {
135 uintptr_t va = (uintptr_t)vaddr;
136 pgd_t *pgd;
137 p4d_t *p4d;
138 pud_t *pud;
139 pmd_t *pmd;
140
141 assert(level && (level <= 5));
142 pgd = pgd_offset(pgtable, va);
143 if (level == pgtable_level_pgd)
144 return pgd;
145 p4d = p4d_alloc(pgd, va);
146 if (level == pgtable_level_p4d)
147 return p4d;
148 pud = pud_alloc(p4d, va);
149
150 if (level == pgtable_level_pud)
151 return pud;
152 if (!pud_none(*pud) && pud_huge(*pud))
153 split_pud(pud, va);
154 pmd = get_pmd(pud, va);
155 if (level == pgtable_level_pmd)
156 return pmd;
157 if (!pmd_none(*pmd) && pmd_large(*pmd))
158 split_pmd(pmd, va);
159 return get_pte(pmd, va);
160 }
161
split_page(pgd_t * pgtable,void * vaddr,enum pgt_level level)162 void *split_page(pgd_t *pgtable, void *vaddr, enum pgt_level level)
163 {
164 assert((level >= 3) && (level <= 5));
165 return get_dat_entry(pgtable ? pgtable : table_root, vaddr, level);
166 }
167
virt_to_pte_phys(pgd_t * pgtable,void * vaddr)168 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *vaddr)
169 {
170 uintptr_t va = (uintptr_t)vaddr;
171 pud_t *pud;
172 pmd_t *pmd;
173 pte_t *pte;
174
175 pud = get_pud(pgtable, va);
176 if (pud_huge(*pud))
177 return (pud_val(*pud) & REGION3_ENTRY_RFAA) | (va & ~REGION3_ENTRY_RFAA);
178 pmd = get_pmd(pud, va);
179 if (pmd_large(*pmd))
180 return (pmd_val(*pmd) & SEGMENT_ENTRY_SFAA) | (va & ~SEGMENT_ENTRY_SFAA);
181 pte = get_pte(pmd, va);
182 return (pte_val(*pte) & PAGE_MASK) | (va & ~PAGE_MASK);
183 }
184
185 /*
186 * Get the DAT table entry of the given level for the given address,
187 * splitting if necessary. If the entry was not invalid, invalidate it, and
188 * return the pointer to the entry and, if requested, its old value.
189 * @pgtable root of the page tables
190 * @vaddr virtual address
191 * @level 3 (for 2GB pud), 4 (for 1MB pmd) or 5 (for 4kB pages)
192 * @old if not NULL, will be written with the old value of the DAT table
193 * entry before invalidation
194 */
dat_get_and_invalidate(pgd_t * pgtable,void * vaddr,enum pgt_level level,unsigned long * old)195 static void *dat_get_and_invalidate(pgd_t *pgtable, void *vaddr, enum pgt_level level, unsigned long *old)
196 {
197 unsigned long va = (unsigned long)vaddr;
198 void *ptr;
199
200 ptr = get_dat_entry(pgtable, vaddr, level);
201 if (old)
202 *old = *(unsigned long *)ptr;
203 if ((level == pgtable_level_pgd) && !pgd_none(*(pgd_t *)ptr))
204 idte_pgdp(va, ptr);
205 else if ((level == pgtable_level_p4d) && !p4d_none(*(p4d_t *)ptr))
206 idte_p4dp(va, ptr);
207 else if ((level == pgtable_level_pud) && !pud_none(*(pud_t *)ptr))
208 idte_pudp(va, ptr);
209 else if ((level == pgtable_level_pmd) && !pmd_none(*(pmd_t *)ptr))
210 idte_pmdp(va, ptr);
211 else if (!pte_none(*(pte_t *)ptr))
212 ipte(va, ptr);
213 return ptr;
214 }
215
cleanup_pmd(pmd_t * pmd)216 static void cleanup_pmd(pmd_t *pmd)
217 {
218 /* was invalid or large, nothing to do */
219 if (pmd_none(*pmd) || pmd_large(*pmd))
220 return;
221 /* was not large, free the corresponding page table */
222 free_pages((void *)(pmd_val(*pmd) & PAGE_MASK));
223 }
224
cleanup_pud(pud_t * pud)225 static void cleanup_pud(pud_t *pud)
226 {
227 unsigned long i;
228 pmd_t *pmd;
229
230 /* was invalid or large, nothing to do */
231 if (pud_none(*pud) || pud_huge(*pud))
232 return;
233 /* recursively clean up all pmds if needed */
234 pmd = (pmd_t *)(pud_val(*pud) & PAGE_MASK);
235 for (i = 0; i < SEGMENT_TABLE_ENTRIES; i++)
236 cleanup_pmd(pmd + i);
237 /* free the corresponding segment table */
238 free_pages(pmd);
239 }
240
241 /*
242 * Set the DAT entry for the given level of the given virtual address. If a
243 * mapping already existed, it is overwritten. If an existing mapping with
244 * smaller pages existed, all the lower tables are freed.
245 * Returns the pointer to the DAT table entry.
246 * @pgtable root of the page tables
247 * @val the new value for the DAT table entry
248 * @vaddr the virtual address
249 * @level 3 for pud (region 3), 4 for pmd (segment) and 5 for pte (pages)
250 */
set_dat_entry(pgd_t * pgtable,unsigned long val,void * vaddr,enum pgt_level level)251 static void *set_dat_entry(pgd_t *pgtable, unsigned long val, void *vaddr, enum pgt_level level)
252 {
253 unsigned long old, *res;
254
255 res = dat_get_and_invalidate(pgtable, vaddr, level, &old);
256 if (level == pgtable_level_pmd)
257 cleanup_pmd((pmd_t *)&old);
258 if (level == pgtable_level_pud)
259 cleanup_pud((pud_t *)&old);
260 *res = val;
261 return res;
262 }
263
install_page(pgd_t * pgtable,phys_addr_t phys,void * vaddr)264 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *vaddr)
265 {
266 assert(IS_ALIGNED(phys, PAGE_SIZE));
267 assert(IS_ALIGNED((uintptr_t)vaddr, PAGE_SIZE));
268 return set_dat_entry(pgtable, phys, vaddr, pgtable_level_pte);
269 }
270
install_large_page(pgd_t * pgtable,phys_addr_t phys,void * vaddr)271 pmdval_t *install_large_page(pgd_t *pgtable, phys_addr_t phys, void *vaddr)
272 {
273 assert(IS_ALIGNED(phys, SZ_1M));
274 assert(IS_ALIGNED((uintptr_t)vaddr, SZ_1M));
275 return set_dat_entry(pgtable, phys | SEGMENT_ENTRY_FC, vaddr, pgtable_level_pmd);
276 }
277
install_huge_page(pgd_t * pgtable,phys_addr_t phys,void * vaddr)278 pudval_t *install_huge_page(pgd_t *pgtable, phys_addr_t phys, void *vaddr)
279 {
280 assert(IS_ALIGNED(phys, SZ_2G));
281 assert(IS_ALIGNED((uintptr_t)vaddr, SZ_2G));
282 return set_dat_entry(pgtable, phys | REGION3_ENTRY_FC | REGION_ENTRY_TT_REGION3, vaddr, pgtable_level_pud);
283 }
284
protect_dat_entry(void * vaddr,unsigned long prot,enum pgt_level level)285 void protect_dat_entry(void *vaddr, unsigned long prot, enum pgt_level level)
286 {
287 unsigned long old, *ptr;
288
289 ptr = dat_get_and_invalidate(table_root, vaddr, level, &old);
290 *ptr = old | prot;
291 }
292
unprotect_dat_entry(void * vaddr,unsigned long prot,enum pgt_level level)293 void unprotect_dat_entry(void *vaddr, unsigned long prot, enum pgt_level level)
294 {
295 unsigned long old, *ptr;
296
297 ptr = dat_get_and_invalidate(table_root, vaddr, level, &old);
298 *ptr = old & ~prot;
299 }
300
protect_range(void * start,unsigned long len,unsigned long prot)301 void protect_range(void *start, unsigned long len, unsigned long prot)
302 {
303 uintptr_t curr = (uintptr_t)start & PAGE_MASK;
304
305 len &= PAGE_MASK;
306 for (; len; len -= PAGE_SIZE, curr += PAGE_SIZE)
307 protect_dat_entry((void *)curr, prot, 5);
308 }
309
unprotect_range(void * start,unsigned long len,unsigned long prot)310 void unprotect_range(void *start, unsigned long len, unsigned long prot)
311 {
312 uintptr_t curr = (uintptr_t)start & PAGE_MASK;
313
314 len &= PAGE_MASK;
315 for (; len; len -= PAGE_SIZE, curr += PAGE_SIZE)
316 unprotect_dat_entry((void *)curr, prot, 5);
317 }
318
setup_identity(pgd_t * pgtable,phys_addr_t start_addr,phys_addr_t end_addr)319 static void setup_identity(pgd_t *pgtable, phys_addr_t start_addr,
320 phys_addr_t end_addr)
321 {
322 phys_addr_t cur;
323
324 start_addr &= PAGE_MASK;
325 for (cur = start_addr; true; cur += PAGE_SIZE) {
326 if (start_addr < end_addr && cur >= end_addr)
327 break;
328 if (start_addr > end_addr && cur <= end_addr)
329 break;
330 install_page(pgtable, cur, __va(cur));
331 }
332 }
333
setup_mmu(phys_addr_t phys_end,void * unused)334 void *setup_mmu(phys_addr_t phys_end, void *unused)
335 {
336 pgd_t *page_root;
337
338 /* allocate a region-1 table */
339 page_root = pgd_alloc_one();
340
341 /* map all physical memory 1:1 */
342 setup_identity(page_root, 0, phys_end);
343
344 /* generate 128MB of invalid adresses at the end (for testing PGM) */
345 init_alloc_vpage((void *) -(1UL << 27));
346 setup_identity(page_root, -(1UL << 27), 0);
347
348 /* finally enable DAT with the new table */
349 mmu_enable(page_root);
350 table_root = page_root;
351 return page_root;
352 }
353