1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Radix MMU support 4 * 5 * Copyright (C) 2024, IBM Inc, Nicholas Piggin <npiggin@gmail.com> 6 * 7 * Derived from Linux kernel MMU code. 8 */ 9 #include <asm/mmu.h> 10 #include <asm/setup.h> 11 #include <asm/smp.h> 12 #include <asm/page.h> 13 #include <asm/io.h> 14 #include <asm/processor.h> 15 #include <asm/hcall.h> 16 17 #include "alloc_page.h" 18 #include "vmalloc.h" 19 #include <asm/pgtable-hwdef.h> 20 #include <asm/pgtable.h> 21 22 #include <linux/compiler.h> 23 24 static pgd_t *identity_pgd; 25 26 bool vm_available(void) /* weak override */ 27 { 28 return cpu_has_radix; 29 } 30 31 bool mmu_enabled(void) 32 { 33 return current_cpu()->pgtable != NULL; 34 } 35 36 void mmu_enable(pgd_t *pgtable) 37 { 38 struct cpu *cpu = current_cpu(); 39 40 if (!pgtable) 41 pgtable = identity_pgd; 42 43 cpu->pgtable = pgtable; 44 45 assert(!in_usermode()); 46 mtmsr(mfmsr() | (MSR_IR|MSR_DR)); 47 } 48 49 void mmu_disable(void) 50 { 51 struct cpu *cpu = current_cpu(); 52 53 cpu->pgtable = NULL; 54 55 assert(!in_usermode()); 56 mtmsr(mfmsr() & ~(MSR_IR|MSR_DR)); 57 } 58 59 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr) 60 { 61 pgd_t *pgd = pgd_offset(pgtable, vaddr); 62 pud_t *pud = pud_alloc(pgd, vaddr); 63 pmd_t *pmd = pmd_alloc(pud, vaddr); 64 pte_t *pte = pte_alloc(pmd, vaddr); 65 66 return &pte_val(*pte); 67 } 68 69 static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte) 70 { 71 pteval_t *p_pte = get_pte(pgtable, vaddr); 72 73 if (READ_ONCE(*p_pte) & cpu_to_be64(_PAGE_VALID)) { 74 WRITE_ONCE(*p_pte, 0); 75 flush_tlb_page(vaddr); 76 } 77 78 WRITE_ONCE(*p_pte, cpu_to_be64(pte)); 79 80 return p_pte; 81 } 82 83 static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys, 84 uintptr_t vaddr, pgprot_t prot) 85 { 86 pteval_t pte = phys; 87 pte |= _PAGE_VALID | _PAGE_PTE; 88 pte |= pgprot_val(prot); 89 return install_pte(pgtable, vaddr, pte); 90 } 91 92 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt) 93 { 94 if (!pgtable) 95 pgtable = identity_pgd; 96 97 return install_page_prot(pgtable, phys, (uintptr_t)virt, 98 __pgprot(_PAGE_VALID | _PAGE_PTE | 99 _PAGE_READ | _PAGE_WRITE | 100 _PAGE_EXEC | _PAGE_ACCESSED | 101 _PAGE_DIRTY)); 102 } 103 104 static pteval_t *follow_pte(pgd_t *pgtable, uintptr_t vaddr) 105 { 106 pgd_t *pgd; 107 pud_t *pud; 108 pmd_t *pmd; 109 pte_t *pte; 110 111 pgd = pgd_offset(pgtable, vaddr); 112 if (!pgd_valid(*pgd)) 113 return NULL; 114 115 pud = pud_offset(pgd, vaddr); 116 if (!pud_valid(*pud)) 117 return NULL; 118 119 pmd = pmd_offset(pud, vaddr); 120 if (!pmd_valid(*pmd)) 121 return NULL; 122 if (pmd_huge(*pmd)) 123 return &pmd_val(*pmd); 124 125 pte = pte_offset(pmd, vaddr); 126 if (!pte_valid(*pte)) 127 return NULL; 128 129 return &pte_val(*pte); 130 } 131 132 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *virt) 133 { 134 phys_addr_t mask; 135 pteval_t *pteval; 136 137 if (!pgtable) 138 pgtable = identity_pgd; 139 140 pteval = follow_pte(pgtable, (uintptr_t)virt); 141 if (!pteval) { 142 install_page(pgtable, (phys_addr_t)(unsigned long)virt, virt); 143 return (phys_addr_t)(unsigned long)virt; 144 } 145 146 if (pmd_huge(__pmd(*pteval))) 147 mask = PMD_MASK; 148 else 149 mask = PAGE_MASK; 150 151 return (be64_to_cpu(*pteval) & PHYS_MASK & mask) | 152 ((phys_addr_t)(unsigned long)virt & ~mask); 153 } 154 155 struct partition_table_entry { 156 uint64_t dw0; 157 uint64_t dw1; 158 }; 159 160 static struct partition_table_entry *partition_table; 161 162 struct process_table_entry { 163 uint64_t dw0; 164 uint64_t dw1; 165 }; 166 167 static struct process_table_entry *process_table; 168 169 void *setup_mmu(phys_addr_t phys_end, void *unused) 170 { 171 phys_addr_t addr; 172 uint64_t dw0, dw1; 173 174 if (identity_pgd) 175 goto enable; 176 177 assert_msg(cpu_has_radix, "MMU support requires radix MMU."); 178 179 /* 32G address is reserved for vmalloc, cap phys_end at 31G */ 180 if (phys_end > (31ul << 30)) { 181 /* print warning */ 182 phys_end = 31ul << 30; 183 } 184 185 init_alloc_vpage((void *)(32ul << 30)); 186 187 process_table = memalign_pages(SZ_4K, SZ_4K); 188 memset(process_table, 0, SZ_4K); 189 190 identity_pgd = pgd_alloc_one(); 191 192 dw0 = (unsigned long)identity_pgd; 193 dw0 |= 16UL - 3; /* 64K pgd size */ 194 dw0 |= (0x2UL << 61) | (0x5UL << 5); /* 52-bit virt */ 195 process_table[1].dw0 = cpu_to_be64(dw0); 196 197 if (machine_is_pseries()) { 198 int ret; 199 200 ret = hcall(H_REGISTER_PROCESS_TABLE, PTBL_NEW | PTBL_RADIX | PTBL_GTSE, process_table, 0, 0 /* 4K size */); 201 assert_msg(!ret, "H_REGISTER_PROCESS_TABLE failed! err=%d\n", ret); 202 } else if (machine_is_powernv()) { 203 partition_table = memalign_pages(SZ_4K, SZ_4K); 204 memset(partition_table, 0, SZ_4K); 205 206 /* Reuse dw0 for partition table */ 207 dw0 |= 1ULL << 63; /* Host radix */ 208 dw1 = (unsigned long)process_table; /* 4K size */ 209 partition_table[0].dw0 = cpu_to_be64(dw0); 210 partition_table[0].dw1 = cpu_to_be64(dw1); 211 212 } else { 213 /* Only pseries and powernv support radix so far */ 214 assert(0); 215 } 216 217 /* 218 * Avoid mapping page 0 so NULL dereferences fault. Test images 219 * run relocated well above 0, so nothing useful here except 220 * real-mode interrupt entry code. 221 */ 222 for (addr = PAGE_SIZE; addr < phys_end; addr += PAGE_SIZE) 223 install_page(identity_pgd, addr, __va(addr)); 224 225 enable: 226 if (machine_is_powernv()) { 227 mtspr(SPR_PTCR, (unsigned long)partition_table); /* 4KB size */ 228 229 mtspr(SPR_LPIDR, 0); 230 /* Set LPCR[UPRT] and LPCR[HR] for radix */ 231 mtspr(SPR_LPCR, mfspr(SPR_LPCR) | (1ULL << 22) | (1ULL << 20)); 232 } 233 234 /* PID=1 is used because PID=0 is also mapped in quadrant 3 */ 235 mtspr(SPR_PIDR, 1); 236 237 mmu_enable(identity_pgd); 238 239 return identity_pgd; 240 } 241 242 phys_addr_t __virt_to_phys(unsigned long addr) 243 { 244 if (mmu_enabled()) { 245 pgd_t *pgtable = current_cpu()->pgtable; 246 return virt_to_pte_phys(pgtable, (void *)addr); 247 } 248 return addr; 249 } 250 251 unsigned long __phys_to_virt(phys_addr_t addr) 252 { 253 /* 254 * We don't guarantee that phys_to_virt(virt_to_phys(vaddr)) == vaddr, but 255 * the default page tables do identity map all physical addresses, which 256 * means phys_to_virt(virt_to_phys((void *)paddr)) == paddr. 257 */ 258 assert(!mmu_enabled() || __virt_to_phys(addr) == addr); 259 return addr; 260 } 261