1 /* 2 * MMU enable and page table manipulation functions 3 * 4 * Copyright (C) 2014, Red Hat Inc, Andrew Jones <drjones@redhat.com> 5 * 6 * This work is licensed under the terms of the GNU LGPL, version 2. 7 */ 8 #include <asm/setup.h> 9 #include <asm/thread_info.h> 10 #include <asm/cpumask.h> 11 #include <asm/mmu.h> 12 #include <asm/setup.h> 13 #include <asm/page.h> 14 15 #include "alloc_page.h" 16 #include "vmalloc.h" 17 #include <asm/pgtable-hwdef.h> 18 #include <asm/pgtable.h> 19 20 #include <linux/compiler.h> 21 22 extern unsigned long etext; 23 24 pgd_t *mmu_idmap; 25 26 /* CPU 0 starts with disabled MMU */ 27 static cpumask_t mmu_disabled_cpumask = { {1} }; 28 unsigned int mmu_disabled_cpu_count = 1; 29 30 bool __mmu_enabled(void) 31 { 32 int cpu = current_thread_info()->cpu; 33 34 /* 35 * mmu_enabled is called from places that are guarding the 36 * use of exclusive ops (which require the mmu to be enabled). 37 * That means we CANNOT call anything from here that may use a 38 * spinlock, atomic bitop, etc., otherwise we'll recurse. 39 * [cpumask_]test_bit is safe though. 40 */ 41 return !cpumask_test_cpu(cpu, &mmu_disabled_cpumask); 42 } 43 44 void mmu_mark_enabled(int cpu) 45 { 46 if (cpumask_test_and_clear_cpu(cpu, &mmu_disabled_cpumask)) 47 --mmu_disabled_cpu_count; 48 } 49 50 void mmu_mark_disabled(int cpu) 51 { 52 if (!cpumask_test_and_set_cpu(cpu, &mmu_disabled_cpumask)) 53 ++mmu_disabled_cpu_count; 54 } 55 56 extern void asm_mmu_enable(phys_addr_t pgtable); 57 void mmu_enable(pgd_t *pgtable) 58 { 59 struct thread_info *info = current_thread_info(); 60 61 asm_mmu_enable(__pa(pgtable)); 62 63 info->pgtable = pgtable; 64 mmu_mark_enabled(info->cpu); 65 } 66 67 extern void asm_mmu_disable(void); 68 void mmu_disable(void) 69 { 70 unsigned long sp = current_stack_pointer; 71 int cpu = current_thread_info()->cpu; 72 73 assert_msg(__virt_to_phys(sp) == sp, 74 "Attempting to disable MMU with non-identity mapped stack"); 75 76 mmu_mark_disabled(cpu); 77 78 asm_mmu_disable(); 79 } 80 81 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr) 82 { 83 pgd_t *pgd = pgd_offset(pgtable, vaddr); 84 pmd_t *pmd = pmd_alloc(pgd, vaddr); 85 pte_t *pte = pte_alloc(pmd, vaddr); 86 87 return &pte_val(*pte); 88 } 89 90 static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte) 91 { 92 pteval_t *p_pte = get_pte(pgtable, vaddr); 93 94 WRITE_ONCE(*p_pte, pte); 95 flush_tlb_page(vaddr); 96 return p_pte; 97 } 98 99 static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys, 100 uintptr_t vaddr, pgprot_t prot) 101 { 102 pteval_t pte = phys; 103 pte |= PTE_TYPE_PAGE | PTE_AF | PTE_SHARED; 104 pte |= pgprot_val(prot); 105 return install_pte(pgtable, vaddr, pte); 106 } 107 108 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt) 109 { 110 return install_page_prot(pgtable, phys, (uintptr_t)virt, 111 __pgprot(PTE_WBWA | PTE_USER)); 112 } 113 114 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *mem) 115 { 116 return (*get_pte(pgtable, (uintptr_t)mem) & PHYS_MASK & -PAGE_SIZE) 117 + ((ulong)mem & (PAGE_SIZE - 1)); 118 } 119 120 void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset, 121 phys_addr_t phys_start, phys_addr_t phys_end, 122 pgprot_t prot) 123 { 124 phys_addr_t paddr = phys_start & PAGE_MASK; 125 uintptr_t vaddr = virt_offset & PAGE_MASK; 126 uintptr_t virt_end = phys_end - paddr + vaddr; 127 128 for (; vaddr < virt_end; vaddr += PAGE_SIZE, paddr += PAGE_SIZE) 129 install_page_prot(pgtable, paddr, vaddr, prot); 130 } 131 132 void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, 133 phys_addr_t phys_start, phys_addr_t phys_end, 134 pgprot_t prot) 135 { 136 phys_addr_t paddr = phys_start & PGDIR_MASK; 137 uintptr_t vaddr = virt_offset & PGDIR_MASK; 138 uintptr_t virt_end = phys_end - paddr + vaddr; 139 pgd_t *pgd; 140 pgd_t entry; 141 142 for (; vaddr < virt_end; vaddr += PGDIR_SIZE, paddr += PGDIR_SIZE) { 143 pgd_val(entry) = paddr; 144 pgd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S; 145 pgd_val(entry) |= pgprot_val(prot); 146 pgd = pgd_offset(pgtable, vaddr); 147 WRITE_ONCE(*pgd, entry); 148 flush_tlb_page(vaddr); 149 } 150 } 151 152 void *setup_mmu(phys_addr_t phys_end) 153 { 154 uintptr_t code_end = (uintptr_t)&etext; 155 struct mem_region *r; 156 157 /* 0G-1G = I/O, 1G-3G = identity, 3G-4G = vmalloc */ 158 if (phys_end > (3ul << 30)) 159 phys_end = 3ul << 30; 160 161 #ifdef __aarch64__ 162 init_alloc_vpage((void*)(4ul << 30)); 163 164 assert_msg(system_supports_granule(PAGE_SIZE), 165 "Unsupported translation granule %ld\n", PAGE_SIZE); 166 #endif 167 168 mmu_idmap = alloc_page(); 169 170 for (r = mem_regions; r->end; ++r) { 171 if (!(r->flags & MR_F_IO)) 172 continue; 173 mmu_set_range_sect(mmu_idmap, r->start, r->start, r->end, 174 __pgprot(PMD_SECT_UNCACHED | PMD_SECT_USER)); 175 } 176 177 /* armv8 requires code shared between EL1 and EL0 to be read-only */ 178 mmu_set_range_ptes(mmu_idmap, PHYS_OFFSET, 179 PHYS_OFFSET, code_end, 180 __pgprot(PTE_WBWA | PTE_RDONLY | PTE_USER)); 181 182 mmu_set_range_ptes(mmu_idmap, code_end, 183 code_end, phys_end, 184 __pgprot(PTE_WBWA | PTE_USER)); 185 186 mmu_enable(mmu_idmap); 187 return mmu_idmap; 188 } 189 190 phys_addr_t __virt_to_phys(unsigned long addr) 191 { 192 if (mmu_enabled()) { 193 pgd_t *pgtable = current_thread_info()->pgtable; 194 return virt_to_pte_phys(pgtable, (void *)addr); 195 } 196 return addr; 197 } 198 199 unsigned long __phys_to_virt(phys_addr_t addr) 200 { 201 /* 202 * We don't guarantee that phys_to_virt(virt_to_phys(vaddr)) == vaddr, but 203 * the default page tables do identity map all physical addresses, which 204 * means phys_to_virt(virt_to_phys((void *)paddr)) == paddr. 205 */ 206 assert(!mmu_enabled() || __virt_to_phys(addr) == addr); 207 return addr; 208 } 209 210 void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr) 211 { 212 pgd_t *pgd; 213 pmd_t *pmd; 214 pte_t *pte; 215 216 if (!mmu_enabled()) 217 return; 218 219 pgd = pgd_offset(pgtable, vaddr); 220 assert(pgd_valid(*pgd)); 221 pmd = pmd_offset(pgd, vaddr); 222 assert(pmd_valid(*pmd)); 223 224 if (pmd_huge(*pmd)) { 225 pmd_t entry = __pmd(pmd_val(*pmd) & ~PMD_SECT_USER); 226 WRITE_ONCE(*pmd, entry); 227 goto out_flush_tlb; 228 } 229 230 pte = pte_offset(pmd, vaddr); 231 assert(pte_valid(*pte)); 232 pte_t entry = __pte(pte_val(*pte) & ~PTE_USER); 233 WRITE_ONCE(*pte, entry); 234 235 out_flush_tlb: 236 flush_tlb_page(vaddr); 237 } 238