1 /* 2 * MMU enable and page table manipulation functions 3 * 4 * Copyright (C) 2014, Red Hat Inc, Andrew Jones <drjones@redhat.com> 5 * 6 * This work is licensed under the terms of the GNU LGPL, version 2. 7 */ 8 #include <asm/setup.h> 9 #include <asm/thread_info.h> 10 #include <asm/cpumask.h> 11 #include <asm/mmu.h> 12 #include <asm/setup.h> 13 #include <asm/page.h> 14 #include <asm/io.h> 15 16 #include "alloc_page.h" 17 #include "vmalloc.h" 18 #include <asm/pgtable-hwdef.h> 19 #include <asm/pgtable.h> 20 21 #include <linux/compiler.h> 22 23 pgd_t *mmu_idmap; 24 25 /* CPU 0 starts with disabled MMU */ 26 static cpumask_t mmu_enabled_cpumask; 27 28 bool mmu_enabled(void) 29 { 30 /* 31 * mmu_enabled is called from places that are guarding the 32 * use of exclusive ops (which require the mmu to be enabled). 33 * That means we CANNOT call anything from here that may use a 34 * spinlock, atomic bitop, etc., otherwise we'll recurse. 35 * [cpumask_]test_bit is safe though. 36 */ 37 if (is_user()) { 38 int cpu = current_thread_info()->cpu; 39 return cpumask_test_cpu(cpu, &mmu_enabled_cpumask); 40 } 41 42 return __mmu_enabled(); 43 } 44 45 void mmu_mark_enabled(int cpu) 46 { 47 cpumask_set_cpu(cpu, &mmu_enabled_cpumask); 48 } 49 50 void mmu_mark_disabled(int cpu) 51 { 52 cpumask_clear_cpu(cpu, &mmu_enabled_cpumask); 53 } 54 55 extern void asm_mmu_enable(phys_addr_t pgtable); 56 void mmu_enable(pgd_t *pgtable) 57 { 58 struct thread_info *info = current_thread_info(); 59 60 asm_mmu_enable(__pa(pgtable)); 61 62 info->pgtable = pgtable; 63 mmu_mark_enabled(info->cpu); 64 } 65 66 extern void asm_mmu_disable(void); 67 void mmu_disable(void) 68 { 69 unsigned long sp = current_stack_pointer; 70 int cpu = current_thread_info()->cpu; 71 72 assert_msg(__virt_to_phys(sp) == sp, 73 "Attempting to disable MMU with non-identity mapped stack"); 74 75 mmu_mark_disabled(cpu); 76 77 asm_mmu_disable(); 78 } 79 80 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr) 81 { 82 pgd_t *pgd = pgd_offset(pgtable, vaddr); 83 pud_t *pud = pud_alloc(pgd, vaddr); 84 pmd_t *pmd = pmd_alloc(pud, vaddr); 85 pte_t *pte = pte_alloc(pmd, vaddr); 86 87 return &pte_val(*pte); 88 } 89 90 static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte) 91 { 92 pteval_t *p_pte = get_pte(pgtable, vaddr); 93 94 WRITE_ONCE(*p_pte, pte); 95 flush_tlb_page(vaddr); 96 return p_pte; 97 } 98 99 static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys, 100 uintptr_t vaddr, pgprot_t prot) 101 { 102 pteval_t pte = phys; 103 pte |= PTE_TYPE_PAGE | PTE_AF | PTE_SHARED; 104 pte |= pgprot_val(prot); 105 return install_pte(pgtable, vaddr, pte); 106 } 107 108 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt) 109 { 110 return install_page_prot(pgtable, phys, (uintptr_t)virt, 111 __pgprot(PTE_WBWA | PTE_USER)); 112 } 113 114 /* 115 * NOTE: The Arm architecture might require the use of a 116 * break-before-make sequence before making changes to a PTE and 117 * certain conditions are met (see Arm ARM D5-2669 for AArch64 and 118 * B3-1378 for AArch32 for more details). 119 */ 120 pteval_t *follow_pte(pgd_t *pgtable, uintptr_t vaddr) 121 { 122 pgd_t *pgd; 123 pud_t *pud; 124 pmd_t *pmd; 125 pte_t *pte; 126 127 pgd = pgd_offset(pgtable, vaddr); 128 if (!pgd_valid(*pgd)) 129 return NULL; 130 131 pud = pud_offset(pgd, vaddr); 132 if (!pud_valid(*pud)) 133 return NULL; 134 135 pmd = pmd_offset(pud, vaddr); 136 if (!pmd_valid(*pmd)) 137 return NULL; 138 if (pmd_huge(*pmd)) 139 return &pmd_val(*pmd); 140 141 pte = pte_offset(pmd, vaddr); 142 if (!pte_valid(*pte)) 143 return NULL; 144 145 return &pte_val(*pte); 146 } 147 148 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *virt) 149 { 150 phys_addr_t mask; 151 pteval_t *pteval; 152 153 pteval = follow_pte(pgtable, (uintptr_t)virt); 154 if (!pteval) { 155 install_page(pgtable, (phys_addr_t)(unsigned long)virt, virt); 156 return (phys_addr_t)(unsigned long)virt; 157 } 158 159 if (pmd_huge(__pmd(*pteval))) 160 mask = PMD_MASK; 161 else 162 mask = PAGE_MASK; 163 164 return (*pteval & PHYS_MASK & mask) | 165 ((phys_addr_t)(unsigned long)virt & ~mask); 166 } 167 168 void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset, 169 phys_addr_t phys_start, phys_addr_t phys_end, 170 pgprot_t prot) 171 { 172 phys_addr_t paddr = phys_start & PAGE_MASK; 173 uintptr_t vaddr = virt_offset & PAGE_MASK; 174 uintptr_t virt_end = phys_end - paddr + vaddr; 175 176 for (; vaddr < virt_end; vaddr += PAGE_SIZE, paddr += PAGE_SIZE) 177 install_page_prot(pgtable, paddr, vaddr, prot); 178 } 179 180 void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, 181 phys_addr_t phys_start, phys_addr_t phys_end, 182 pgprot_t prot) 183 { 184 phys_addr_t paddr = phys_start & PMD_MASK; 185 uintptr_t vaddr = virt_offset & PMD_MASK; 186 uintptr_t virt_end = phys_end - paddr + vaddr; 187 pgd_t *pgd; 188 pud_t *pud; 189 pmd_t *pmd; 190 pmd_t entry; 191 192 for (; vaddr < virt_end; vaddr += PMD_SIZE, paddr += PMD_SIZE) { 193 pmd_val(entry) = paddr; 194 pmd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S; 195 pmd_val(entry) |= pgprot_val(prot); 196 pgd = pgd_offset(pgtable, vaddr); 197 pud = pud_alloc(pgd, vaddr); 198 pmd = pmd_alloc(pud, vaddr); 199 WRITE_ONCE(*pmd, entry); 200 flush_tlb_page(vaddr); 201 } 202 } 203 204 void *setup_mmu(phys_addr_t phys_end, void *unused) 205 { 206 struct mem_region *r; 207 208 /* 3G-4G region is reserved for vmalloc, cap phys_end at 3G */ 209 if (phys_end > (3ul << 30)) 210 phys_end = 3ul << 30; 211 212 #ifdef __aarch64__ 213 init_alloc_vpage((void*)(4ul << 30)); 214 215 assert_msg(system_supports_granule(PAGE_SIZE), 216 "Unsupported translation granule %ld\n", PAGE_SIZE); 217 #endif 218 219 if (!mmu_idmap) 220 mmu_idmap = alloc_page(); 221 222 for (r = mem_regions; r->end; ++r) { 223 if (r->flags & MR_F_IO) { 224 continue; 225 } else if (r->flags & MR_F_CODE) { 226 /* armv8 requires code shared between EL1 and EL0 to be read-only */ 227 mmu_set_range_ptes(mmu_idmap, r->start, r->start, r->end, 228 __pgprot(PTE_WBWA | PTE_USER | PTE_RDONLY)); 229 } else { 230 mmu_set_range_ptes(mmu_idmap, r->start, r->start, r->end, 231 __pgprot(PTE_WBWA | PTE_USER)); 232 } 233 } 234 235 mmu_enable(mmu_idmap); 236 return mmu_idmap; 237 } 238 239 void __iomem *__ioremap(phys_addr_t phys_addr, size_t size) 240 { 241 phys_addr_t paddr_aligned = phys_addr & PAGE_MASK; 242 phys_addr_t paddr_end = PAGE_ALIGN(phys_addr + size); 243 pgprot_t prot = __pgprot(PTE_UNCACHED | PTE_USER | PTE_UXN | PTE_PXN); 244 pgd_t *pgtable; 245 246 assert(sizeof(long) == 8 || !(phys_addr >> 32)); 247 248 if (mmu_enabled()) { 249 pgtable = current_thread_info()->pgtable; 250 } else { 251 if (!mmu_idmap) 252 mmu_idmap = alloc_page(); 253 pgtable = mmu_idmap; 254 } 255 256 mmu_set_range_ptes(pgtable, paddr_aligned, paddr_aligned, 257 paddr_end, prot); 258 259 return (void __iomem *)(unsigned long)phys_addr; 260 } 261 262 phys_addr_t __virt_to_phys(unsigned long addr) 263 { 264 if (mmu_enabled()) { 265 pgd_t *pgtable = current_thread_info()->pgtable; 266 return virt_to_pte_phys(pgtable, (void *)addr); 267 } 268 return addr; 269 } 270 271 unsigned long __phys_to_virt(phys_addr_t addr) 272 { 273 /* 274 * We don't guarantee that phys_to_virt(virt_to_phys(vaddr)) == vaddr, but 275 * the default page tables do identity map all physical addresses, which 276 * means phys_to_virt(virt_to_phys((void *)paddr)) == paddr. 277 */ 278 assert(!mmu_enabled() || __virt_to_phys(addr) == addr); 279 return addr; 280 } 281 282 void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr) 283 { 284 pteval_t *p_pte = follow_pte(pgtable, vaddr); 285 if (p_pte) { 286 pteval_t entry = *p_pte & ~PTE_USER; 287 WRITE_ONCE(*p_pte, entry); 288 flush_tlb_page(vaddr); 289 } 290 } 291