1 /* 2 * MMU enable and page table manipulation functions 3 * 4 * Copyright (C) 2014, Red Hat Inc, Andrew Jones <drjones@redhat.com> 5 * 6 * This work is licensed under the terms of the GNU LGPL, version 2. 7 */ 8 #include <cpumask.h> 9 #include <memregions.h> 10 #include <asm/setup.h> 11 #include <asm/thread_info.h> 12 #include <asm/mmu.h> 13 #include <asm/setup.h> 14 #include <asm/page.h> 15 #include <asm/io.h> 16 17 #include "alloc_page.h" 18 #include "vmalloc.h" 19 #include <asm/pgtable-hwdef.h> 20 #include <asm/pgtable.h> 21 22 #include <linux/compiler.h> 23 24 pgd_t *mmu_idmap; 25 26 /* CPU 0 starts with disabled MMU */ 27 static cpumask_t mmu_enabled_cpumask; 28 29 bool mmu_enabled(void) 30 { 31 /* 32 * mmu_enabled is called from places that are guarding the 33 * use of exclusive ops (which require the mmu to be enabled). 34 * That means we CANNOT call anything from here that may use a 35 * spinlock, atomic bitop, etc., otherwise we'll recurse. 36 * [cpumask_]test_bit is safe though. 37 */ 38 if (is_user()) { 39 int cpu = current_thread_info()->cpu; 40 return cpumask_test_cpu(cpu, &mmu_enabled_cpumask); 41 } 42 43 return __mmu_enabled(); 44 } 45 46 void mmu_mark_enabled(int cpu) 47 { 48 cpumask_set_cpu(cpu, &mmu_enabled_cpumask); 49 } 50 51 void mmu_mark_disabled(int cpu) 52 { 53 cpumask_clear_cpu(cpu, &mmu_enabled_cpumask); 54 } 55 56 extern void asm_mmu_enable(phys_addr_t pgtable); 57 void mmu_enable(pgd_t *pgtable) 58 { 59 struct thread_info *info = current_thread_info(); 60 61 asm_mmu_enable(__pa(pgtable)); 62 63 info->pgtable = pgtable; 64 mmu_mark_enabled(info->cpu); 65 } 66 67 extern void asm_mmu_disable(void); 68 void mmu_disable(void) 69 { 70 unsigned long sp = current_stack_pointer; 71 int cpu = current_thread_info()->cpu; 72 73 assert_msg(__virt_to_phys(sp) == sp, 74 "Attempting to disable MMU with non-identity mapped stack"); 75 76 mmu_mark_disabled(cpu); 77 78 asm_mmu_disable(); 79 } 80 81 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr) 82 { 83 pgd_t *pgd = pgd_offset(pgtable, vaddr); 84 pud_t *pud = pud_alloc(pgd, vaddr); 85 pmd_t *pmd = pmd_alloc(pud, vaddr); 86 pte_t *pte = pte_alloc(pmd, vaddr); 87 88 return &pte_val(*pte); 89 } 90 91 static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte) 92 { 93 pteval_t *p_pte = get_pte(pgtable, vaddr); 94 95 WRITE_ONCE(*p_pte, pte); 96 flush_tlb_page(vaddr); 97 return p_pte; 98 } 99 100 static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys, 101 uintptr_t vaddr, pgprot_t prot) 102 { 103 pteval_t pte = phys; 104 pte |= PTE_TYPE_PAGE | PTE_AF | PTE_SHARED; 105 pte |= pgprot_val(prot); 106 return install_pte(pgtable, vaddr, pte); 107 } 108 109 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt) 110 { 111 return install_page_prot(pgtable, phys, (uintptr_t)virt, 112 __pgprot(PTE_WBWA | PTE_USER)); 113 } 114 115 /* 116 * NOTE: The Arm architecture might require the use of a 117 * break-before-make sequence before making changes to a PTE and 118 * certain conditions are met (see Arm ARM D5-2669 for AArch64 and 119 * B3-1378 for AArch32 for more details). 120 */ 121 pteval_t *follow_pte(pgd_t *pgtable, uintptr_t vaddr) 122 { 123 pgd_t *pgd; 124 pud_t *pud; 125 pmd_t *pmd; 126 pte_t *pte; 127 128 pgd = pgd_offset(pgtable, vaddr); 129 if (!pgd_valid(*pgd)) 130 return NULL; 131 132 pud = pud_offset(pgd, vaddr); 133 if (!pud_valid(*pud)) 134 return NULL; 135 136 pmd = pmd_offset(pud, vaddr); 137 if (!pmd_valid(*pmd)) 138 return NULL; 139 if (pmd_huge(*pmd)) 140 return &pmd_val(*pmd); 141 142 pte = pte_offset(pmd, vaddr); 143 if (!pte_valid(*pte)) 144 return NULL; 145 146 return &pte_val(*pte); 147 } 148 149 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *virt) 150 { 151 phys_addr_t mask; 152 pteval_t *pteval; 153 154 pteval = follow_pte(pgtable, (uintptr_t)virt); 155 if (!pteval) { 156 install_page(pgtable, (phys_addr_t)(unsigned long)virt, virt); 157 return (phys_addr_t)(unsigned long)virt; 158 } 159 160 if (pmd_huge(__pmd(*pteval))) 161 mask = PMD_MASK; 162 else 163 mask = PAGE_MASK; 164 165 return (*pteval & PHYS_MASK & mask) | 166 ((phys_addr_t)(unsigned long)virt & ~mask); 167 } 168 169 void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset, 170 phys_addr_t phys_start, phys_addr_t phys_end, 171 pgprot_t prot) 172 { 173 phys_addr_t paddr = phys_start & PAGE_MASK; 174 uintptr_t vaddr = virt_offset & PAGE_MASK; 175 uintptr_t virt_end = phys_end - paddr + vaddr; 176 177 for (; vaddr < virt_end; vaddr += PAGE_SIZE, paddr += PAGE_SIZE) 178 install_page_prot(pgtable, paddr, vaddr, prot); 179 } 180 181 void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, 182 phys_addr_t phys_start, phys_addr_t phys_end, 183 pgprot_t prot) 184 { 185 phys_addr_t paddr = phys_start & PMD_MASK; 186 uintptr_t vaddr = virt_offset & PMD_MASK; 187 uintptr_t virt_end = phys_end - paddr + vaddr; 188 pgd_t *pgd; 189 pud_t *pud; 190 pmd_t *pmd; 191 pmd_t entry; 192 193 for (; vaddr < virt_end; vaddr += PMD_SIZE, paddr += PMD_SIZE) { 194 pmd_val(entry) = paddr; 195 pmd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S; 196 pmd_val(entry) |= pgprot_val(prot); 197 pgd = pgd_offset(pgtable, vaddr); 198 pud = pud_alloc(pgd, vaddr); 199 pmd = pmd_alloc(pud, vaddr); 200 WRITE_ONCE(*pmd, entry); 201 flush_tlb_page(vaddr); 202 } 203 } 204 205 void *setup_mmu(phys_addr_t phys_end, void *unused) 206 { 207 struct mem_region *r; 208 209 /* 3G-4G region is reserved for vmalloc, cap phys_end at 3G */ 210 if (phys_end > (3ul << 30)) 211 phys_end = 3ul << 30; 212 213 #ifdef __aarch64__ 214 init_alloc_vpage((void*)(4ul << 30)); 215 216 assert_msg(system_supports_granule(PAGE_SIZE), 217 "Unsupported translation granule %ld\n", PAGE_SIZE); 218 #endif 219 220 if (!mmu_idmap) 221 mmu_idmap = alloc_page(); 222 223 for (r = mem_regions; r->end; ++r) { 224 if (r->flags & (MR_F_IO | MR_F_RESERVED)) { 225 continue; 226 } else if (r->flags & MR_F_CODE) { 227 /* armv8 requires code shared between EL1 and EL0 to be read-only */ 228 mmu_set_range_ptes(mmu_idmap, r->start, r->start, r->end, 229 __pgprot(PTE_WBWA | PTE_USER | PTE_RDONLY)); 230 } else { 231 mmu_set_range_ptes(mmu_idmap, r->start, r->start, r->end, 232 __pgprot(PTE_WBWA | PTE_USER)); 233 } 234 } 235 236 mmu_enable(mmu_idmap); 237 return mmu_idmap; 238 } 239 240 void __iomem *__ioremap(phys_addr_t phys_addr, size_t size) 241 { 242 phys_addr_t paddr_aligned = phys_addr & PAGE_MASK; 243 phys_addr_t paddr_end = PAGE_ALIGN(phys_addr + size); 244 pgprot_t prot = __pgprot(PTE_UNCACHED | PTE_USER | PTE_UXN | PTE_PXN); 245 pgd_t *pgtable; 246 247 assert(sizeof(long) == 8 || !(phys_addr >> 32)); 248 249 if (mmu_enabled()) { 250 pgtable = current_thread_info()->pgtable; 251 } else { 252 if (!mmu_idmap) 253 mmu_idmap = alloc_page(); 254 pgtable = mmu_idmap; 255 } 256 257 mmu_set_range_ptes(pgtable, paddr_aligned, paddr_aligned, 258 paddr_end, prot); 259 260 return (void __iomem *)(unsigned long)phys_addr; 261 } 262 263 phys_addr_t __virt_to_phys(unsigned long addr) 264 { 265 if (mmu_enabled()) { 266 pgd_t *pgtable = current_thread_info()->pgtable; 267 return virt_to_pte_phys(pgtable, (void *)addr); 268 } 269 return addr; 270 } 271 272 unsigned long __phys_to_virt(phys_addr_t addr) 273 { 274 /* 275 * We don't guarantee that phys_to_virt(virt_to_phys(vaddr)) == vaddr, but 276 * the default page tables do identity map all physical addresses, which 277 * means phys_to_virt(virt_to_phys((void *)paddr)) == paddr. 278 */ 279 assert(!mmu_enabled() || __virt_to_phys(addr) == addr); 280 return addr; 281 } 282 283 void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr) 284 { 285 pteval_t *p_pte = follow_pte(pgtable, vaddr); 286 if (p_pte) { 287 pteval_t entry = *p_pte & ~PTE_USER; 288 WRITE_ONCE(*p_pte, entry); 289 flush_tlb_page(vaddr); 290 } 291 } 292