1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * s390x MMU 4 * 5 * Copyright (c) 2017 Red Hat Inc 6 * 7 * Authors: 8 * David Hildenbrand <david@redhat.com> 9 */ 10 11 #include <libcflat.h> 12 #include <asm/pgtable.h> 13 #include <asm/arch_def.h> 14 #include <asm/barrier.h> 15 #include <vmalloc.h> 16 #include "mmu.h" 17 18 /* 19 * The naming convention used here is the same as used in the Linux kernel; 20 * this is the correspondence between the s390x architectural names and the 21 * Linux ones: 22 * 23 * pgd - region 1 table entry 24 * p4d - region 2 table entry 25 * pud - region 3 table entry 26 * pmd - segment table entry 27 * pte - page table entry 28 */ 29 30 static pgd_t *table_root; 31 32 void configure_dat(int enable) 33 { 34 uint64_t mask; 35 36 if (enable) 37 mask = extract_psw_mask() | PSW_MASK_DAT; 38 else 39 mask = extract_psw_mask() & ~PSW_MASK_DAT; 40 41 load_psw_mask(mask); 42 } 43 44 static void mmu_enable(pgd_t *pgtable) 45 { 46 struct lowcore *lc = NULL; 47 const uint64_t asce = __pa(pgtable) | ASCE_DT_REGION1 | 48 REGION_TABLE_LENGTH; 49 50 /* set primary asce */ 51 lctlg(1, asce); 52 assert(stctg(1) == asce); 53 54 /* enable dat (primary == 0 set as default) */ 55 configure_dat(1); 56 57 /* we can now also use DAT unconditionally in our PGM handler */ 58 lc->pgm_new_psw.mask |= PSW_MASK_DAT; 59 } 60 61 /* 62 * Get the pud (region 3) DAT table entry for the given address and root, 63 * allocating it if necessary 64 */ 65 static inline pud_t *get_pud(pgd_t *pgtable, uintptr_t vaddr) 66 { 67 pgd_t *pgd = pgd_offset(pgtable, vaddr); 68 p4d_t *p4d = p4d_alloc(pgd, vaddr); 69 pud_t *pud = pud_alloc(p4d, vaddr); 70 71 return pud; 72 } 73 74 /* 75 * Get the pmd (segment) DAT table entry for the given address and pud, 76 * allocating it if necessary. 77 * The pud must not be huge. 78 */ 79 static inline pmd_t *get_pmd(pud_t *pud, uintptr_t vaddr) 80 { 81 pmd_t *pmd; 82 83 assert(!pud_huge(*pud)); 84 pmd = pmd_alloc(pud, vaddr); 85 return pmd; 86 } 87 88 /* 89 * Get the pte (page) DAT table entry for the given address and pmd, 90 * allocating it if necessary. 91 * The pmd must not be large. 92 */ 93 static inline pte_t *get_pte(pmd_t *pmd, uintptr_t vaddr) 94 { 95 pte_t *pte; 96 97 assert(!pmd_large(*pmd)); 98 pte = pte_alloc(pmd, vaddr); 99 return pte; 100 } 101 102 /* 103 * Splits a large pmd (segment) DAT table entry into equivalent 4kB small 104 * pages. 105 * @pmd The pmd to split, it must be large. 106 * @va the virtual address corresponding to this pmd. 107 */ 108 static void split_pmd(pmd_t *pmd, uintptr_t va) 109 { 110 phys_addr_t pa = pmd_val(*pmd) & SEGMENT_ENTRY_SFAA; 111 unsigned long i, prot; 112 pte_t *pte; 113 114 assert(pmd_large(*pmd)); 115 pte = alloc_pages(PAGE_TABLE_ORDER); 116 prot = pmd_val(*pmd) & (SEGMENT_ENTRY_IEP | SEGMENT_ENTRY_P); 117 for (i = 0; i < PAGE_TABLE_ENTRIES; i++) 118 pte_val(pte[i]) = pa | PAGE_SIZE * i | prot; 119 idte_pmdp(va, &pmd_val(*pmd)); 120 pmd_val(*pmd) = __pa(pte) | SEGMENT_ENTRY_TT_SEGMENT; 121 122 } 123 124 /* 125 * Splits a huge pud (region 3) DAT table entry into equivalent 1MB large 126 * pages. 127 * @pud The pud to split, it must be huge. 128 * @va the virtual address corresponding to this pud. 129 */ 130 static void split_pud(pud_t *pud, uintptr_t va) 131 { 132 phys_addr_t pa = pud_val(*pud) & REGION3_ENTRY_RFAA; 133 unsigned long i, prot; 134 pmd_t *pmd; 135 136 assert(pud_huge(*pud)); 137 pmd = alloc_pages(SEGMENT_TABLE_ORDER); 138 prot = pud_val(*pud) & (REGION3_ENTRY_IEP | REGION_ENTRY_P); 139 for (i = 0; i < SEGMENT_TABLE_ENTRIES; i++) 140 pmd_val(pmd[i]) = pa | SZ_1M * i | prot | SEGMENT_ENTRY_FC | SEGMENT_ENTRY_TT_SEGMENT; 141 idte_pudp(va, &pud_val(*pud)); 142 pud_val(*pud) = __pa(pmd) | REGION_ENTRY_TT_REGION3 | REGION_TABLE_LENGTH; 143 } 144 145 void *get_dat_entry(pgd_t *pgtable, void *vaddr, enum pgt_level level) 146 { 147 uintptr_t va = (uintptr_t)vaddr; 148 pgd_t *pgd; 149 p4d_t *p4d; 150 pud_t *pud; 151 pmd_t *pmd; 152 153 assert(level && (level <= 5)); 154 pgd = pgd_offset(pgtable, va); 155 if (level == pgtable_level_pgd) 156 return pgd; 157 p4d = p4d_alloc(pgd, va); 158 if (level == pgtable_level_p4d) 159 return p4d; 160 pud = pud_alloc(p4d, va); 161 162 if (level == pgtable_level_pud) 163 return pud; 164 if (!pud_none(*pud) && pud_huge(*pud)) 165 split_pud(pud, va); 166 pmd = get_pmd(pud, va); 167 if (level == pgtable_level_pmd) 168 return pmd; 169 if (!pmd_none(*pmd) && pmd_large(*pmd)) 170 split_pmd(pmd, va); 171 return get_pte(pmd, va); 172 } 173 174 void *split_page(pgd_t *pgtable, void *vaddr, enum pgt_level level) 175 { 176 assert((level >= 3) && (level <= 5)); 177 return get_dat_entry(pgtable ? pgtable : table_root, vaddr, level); 178 } 179 180 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *vaddr) 181 { 182 uintptr_t va = (uintptr_t)vaddr; 183 pud_t *pud; 184 pmd_t *pmd; 185 pte_t *pte; 186 187 pud = get_pud(pgtable, va); 188 if (pud_huge(*pud)) 189 return (pud_val(*pud) & REGION3_ENTRY_RFAA) | (va & ~REGION3_ENTRY_RFAA); 190 pmd = get_pmd(pud, va); 191 if (pmd_large(*pmd)) 192 return (pmd_val(*pmd) & SEGMENT_ENTRY_SFAA) | (va & ~SEGMENT_ENTRY_SFAA); 193 pte = get_pte(pmd, va); 194 return (pte_val(*pte) & PAGE_MASK) | (va & ~PAGE_MASK); 195 } 196 197 /* 198 * Get the DAT table entry of the given level for the given address, 199 * splitting if necessary. If the entry was not invalid, invalidate it, and 200 * return the pointer to the entry and, if requested, its old value. 201 * @pgtable root of the page tables 202 * @vaddr virtual address 203 * @level 3 (for 2GB pud), 4 (for 1MB pmd) or 5 (for 4kB pages) 204 * @old if not NULL, will be written with the old value of the DAT table 205 * entry before invalidation 206 */ 207 static void *dat_get_and_invalidate(pgd_t *pgtable, void *vaddr, enum pgt_level level, unsigned long *old) 208 { 209 unsigned long va = (unsigned long)vaddr; 210 void *ptr; 211 212 ptr = get_dat_entry(pgtable, vaddr, level); 213 if (old) 214 *old = *(unsigned long *)ptr; 215 if ((level == pgtable_level_pgd) && !pgd_none(*(pgd_t *)ptr)) 216 idte_pgdp(va, ptr); 217 else if ((level == pgtable_level_p4d) && !p4d_none(*(p4d_t *)ptr)) 218 idte_p4dp(va, ptr); 219 else if ((level == pgtable_level_pud) && !pud_none(*(pud_t *)ptr)) 220 idte_pudp(va, ptr); 221 else if ((level == pgtable_level_pmd) && !pmd_none(*(pmd_t *)ptr)) 222 idte_pmdp(va, ptr); 223 else if (!pte_none(*(pte_t *)ptr)) 224 ipte(va, ptr); 225 return ptr; 226 } 227 228 static void cleanup_pmd(pmd_t *pmd) 229 { 230 /* was invalid or large, nothing to do */ 231 if (pmd_none(*pmd) || pmd_large(*pmd)) 232 return; 233 /* was not large, free the corresponding page table */ 234 free_pages((void *)(pmd_val(*pmd) & PAGE_MASK)); 235 } 236 237 static void cleanup_pud(pud_t *pud) 238 { 239 unsigned long i; 240 pmd_t *pmd; 241 242 /* was invalid or large, nothing to do */ 243 if (pud_none(*pud) || pud_huge(*pud)) 244 return; 245 /* recursively clean up all pmds if needed */ 246 pmd = (pmd_t *)(pud_val(*pud) & PAGE_MASK); 247 for (i = 0; i < SEGMENT_TABLE_ENTRIES; i++) 248 cleanup_pmd(pmd + i); 249 /* free the corresponding segment table */ 250 free_pages(pmd); 251 } 252 253 /* 254 * Set the DAT entry for the given level of the given virtual address. If a 255 * mapping already existed, it is overwritten. If an existing mapping with 256 * smaller pages existed, all the lower tables are freed. 257 * Returns the pointer to the DAT table entry. 258 * @pgtable root of the page tables 259 * @val the new value for the DAT table entry 260 * @vaddr the virtual address 261 * @level 3 for pud (region 3), 4 for pmd (segment) and 5 for pte (pages) 262 */ 263 static void *set_dat_entry(pgd_t *pgtable, unsigned long val, void *vaddr, enum pgt_level level) 264 { 265 unsigned long old, *res; 266 267 res = dat_get_and_invalidate(pgtable, vaddr, level, &old); 268 if (level == pgtable_level_pmd) 269 cleanup_pmd((pmd_t *)&old); 270 if (level == pgtable_level_pud) 271 cleanup_pud((pud_t *)&old); 272 *res = val; 273 return res; 274 } 275 276 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *vaddr) 277 { 278 assert(IS_ALIGNED(phys, PAGE_SIZE)); 279 assert(IS_ALIGNED((uintptr_t)vaddr, PAGE_SIZE)); 280 return set_dat_entry(pgtable, phys, vaddr, pgtable_level_pte); 281 } 282 283 pmdval_t *install_large_page(pgd_t *pgtable, phys_addr_t phys, void *vaddr) 284 { 285 assert(IS_ALIGNED(phys, SZ_1M)); 286 assert(IS_ALIGNED((uintptr_t)vaddr, SZ_1M)); 287 return set_dat_entry(pgtable, phys | SEGMENT_ENTRY_FC, vaddr, pgtable_level_pmd); 288 } 289 290 pudval_t *install_huge_page(pgd_t *pgtable, phys_addr_t phys, void *vaddr) 291 { 292 assert(IS_ALIGNED(phys, SZ_2G)); 293 assert(IS_ALIGNED((uintptr_t)vaddr, SZ_2G)); 294 return set_dat_entry(pgtable, phys | REGION3_ENTRY_FC | REGION_ENTRY_TT_REGION3, vaddr, pgtable_level_pud); 295 } 296 297 void protect_dat_entry(void *vaddr, unsigned long prot, enum pgt_level level) 298 { 299 unsigned long old, *ptr; 300 301 ptr = dat_get_and_invalidate(table_root, vaddr, level, &old); 302 *ptr = old | prot; 303 } 304 305 void unprotect_dat_entry(void *vaddr, unsigned long prot, enum pgt_level level) 306 { 307 unsigned long old, *ptr; 308 309 ptr = dat_get_and_invalidate(table_root, vaddr, level, &old); 310 *ptr = old & ~prot; 311 } 312 313 void protect_range(void *start, unsigned long len, unsigned long prot) 314 { 315 uintptr_t curr = (uintptr_t)start & PAGE_MASK; 316 317 len &= PAGE_MASK; 318 for (; len; len -= PAGE_SIZE, curr += PAGE_SIZE) 319 protect_dat_entry((void *)curr, prot, 5); 320 } 321 322 void unprotect_range(void *start, unsigned long len, unsigned long prot) 323 { 324 uintptr_t curr = (uintptr_t)start & PAGE_MASK; 325 326 len &= PAGE_MASK; 327 for (; len; len -= PAGE_SIZE, curr += PAGE_SIZE) 328 unprotect_dat_entry((void *)curr, prot, 5); 329 } 330 331 static void setup_identity(pgd_t *pgtable, phys_addr_t start_addr, 332 phys_addr_t end_addr) 333 { 334 phys_addr_t cur; 335 336 start_addr &= PAGE_MASK; 337 for (cur = start_addr; true; cur += PAGE_SIZE) { 338 if (start_addr < end_addr && cur >= end_addr) 339 break; 340 if (start_addr > end_addr && cur <= end_addr) 341 break; 342 install_page(pgtable, cur, __va(cur)); 343 } 344 } 345 346 void *setup_mmu(phys_addr_t phys_end, void *unused) 347 { 348 pgd_t *page_root; 349 350 /* allocate a region-1 table */ 351 page_root = pgd_alloc_one(); 352 353 /* map all physical memory 1:1 */ 354 setup_identity(page_root, 0, phys_end); 355 356 /* generate 128MB of invalid adresses at the end (for testing PGM) */ 357 init_alloc_vpage((void *) -(1UL << 27)); 358 setup_identity(page_root, -(1UL << 27), 0); 359 360 /* finally enable DAT with the new table */ 361 mmu_enable(page_root); 362 table_root = page_root; 363 return page_root; 364 } 365