1 /* 2 * MMU enable and page table manipulation functions 3 * 4 * Copyright (C) 2014, Red Hat Inc, Andrew Jones <drjones@redhat.com> 5 * 6 * This work is licensed under the terms of the GNU LGPL, version 2. 7 */ 8 #include <asm/setup.h> 9 #include <asm/thread_info.h> 10 #include <asm/cpumask.h> 11 #include <asm/mmu.h> 12 #include <asm/setup.h> 13 #include <asm/page.h> 14 15 #include "alloc_page.h" 16 #include "vmalloc.h" 17 #include <asm/pgtable-hwdef.h> 18 #include <asm/pgtable.h> 19 20 #include <linux/compiler.h> 21 22 extern unsigned long etext; 23 24 pgd_t *mmu_idmap; 25 26 /* CPU 0 starts with disabled MMU */ 27 static cpumask_t mmu_disabled_cpumask = { {1} }; 28 unsigned int mmu_disabled_cpu_count = 1; 29 30 bool __mmu_enabled(void) 31 { 32 int cpu = current_thread_info()->cpu; 33 34 /* 35 * mmu_enabled is called from places that are guarding the 36 * use of exclusive ops (which require the mmu to be enabled). 37 * That means we CANNOT call anything from here that may use a 38 * spinlock, atomic bitop, etc., otherwise we'll recurse. 39 * [cpumask_]test_bit is safe though. 40 */ 41 return !cpumask_test_cpu(cpu, &mmu_disabled_cpumask); 42 } 43 44 void mmu_mark_enabled(int cpu) 45 { 46 if (cpumask_test_and_clear_cpu(cpu, &mmu_disabled_cpumask)) 47 --mmu_disabled_cpu_count; 48 } 49 50 void mmu_mark_disabled(int cpu) 51 { 52 if (!cpumask_test_and_set_cpu(cpu, &mmu_disabled_cpumask)) 53 ++mmu_disabled_cpu_count; 54 } 55 56 extern void asm_mmu_enable(phys_addr_t pgtable); 57 void mmu_enable(pgd_t *pgtable) 58 { 59 struct thread_info *info = current_thread_info(); 60 61 asm_mmu_enable(__pa(pgtable)); 62 flush_tlb_all(); 63 64 info->pgtable = pgtable; 65 mmu_mark_enabled(info->cpu); 66 } 67 68 extern void asm_mmu_disable(void); 69 void mmu_disable(void) 70 { 71 int cpu = current_thread_info()->cpu; 72 73 mmu_mark_disabled(cpu); 74 75 asm_mmu_disable(); 76 } 77 78 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr) 79 { 80 pgd_t *pgd = pgd_offset(pgtable, vaddr); 81 pmd_t *pmd = pmd_alloc(pgd, vaddr); 82 pte_t *pte = pte_alloc(pmd, vaddr); 83 84 return &pte_val(*pte); 85 } 86 87 static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte) 88 { 89 pteval_t *p_pte = get_pte(pgtable, vaddr); 90 91 WRITE_ONCE(*p_pte, pte); 92 flush_tlb_page(vaddr); 93 return p_pte; 94 } 95 96 static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys, 97 uintptr_t vaddr, pgprot_t prot) 98 { 99 pteval_t pte = phys; 100 pte |= PTE_TYPE_PAGE | PTE_AF | PTE_SHARED; 101 pte |= pgprot_val(prot); 102 return install_pte(pgtable, vaddr, pte); 103 } 104 105 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt) 106 { 107 return install_page_prot(pgtable, phys, (uintptr_t)virt, 108 __pgprot(PTE_WBWA | PTE_USER)); 109 } 110 111 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *mem) 112 { 113 return (*get_pte(pgtable, (uintptr_t)mem) & PHYS_MASK & -PAGE_SIZE) 114 + ((ulong)mem & (PAGE_SIZE - 1)); 115 } 116 117 void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset, 118 phys_addr_t phys_start, phys_addr_t phys_end, 119 pgprot_t prot) 120 { 121 phys_addr_t paddr = phys_start & PAGE_MASK; 122 uintptr_t vaddr = virt_offset & PAGE_MASK; 123 uintptr_t virt_end = phys_end - paddr + vaddr; 124 125 for (; vaddr < virt_end; vaddr += PAGE_SIZE, paddr += PAGE_SIZE) 126 install_page_prot(pgtable, paddr, vaddr, prot); 127 } 128 129 void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, 130 phys_addr_t phys_start, phys_addr_t phys_end, 131 pgprot_t prot) 132 { 133 phys_addr_t paddr = phys_start & PGDIR_MASK; 134 uintptr_t vaddr = virt_offset & PGDIR_MASK; 135 uintptr_t virt_end = phys_end - paddr + vaddr; 136 pgd_t *pgd; 137 pgd_t entry; 138 139 for (; vaddr < virt_end; vaddr += PGDIR_SIZE, paddr += PGDIR_SIZE) { 140 pgd_val(entry) = paddr; 141 pgd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S; 142 pgd_val(entry) |= pgprot_val(prot); 143 pgd = pgd_offset(pgtable, vaddr); 144 WRITE_ONCE(*pgd, entry); 145 flush_tlb_page(vaddr); 146 } 147 } 148 149 void *setup_mmu(phys_addr_t phys_end) 150 { 151 uintptr_t code_end = (uintptr_t)&etext; 152 153 /* 0G-1G = I/O, 1G-3G = identity, 3G-4G = vmalloc */ 154 if (phys_end > (3ul << 30)) 155 phys_end = 3ul << 30; 156 157 #ifdef __aarch64__ 158 init_alloc_vpage((void*)(4ul << 30)); 159 #endif 160 161 mmu_idmap = alloc_page(); 162 163 /* 164 * mach-virt I/O regions: 165 * - The first 1G (arm/arm64) 166 * - 512M at 256G (arm64, arm uses highmem=off) 167 * - 512G at 512G (arm64, arm uses highmem=off) 168 */ 169 mmu_set_range_sect(mmu_idmap, 170 0, 0, (1ul << 30), 171 __pgprot(PMD_SECT_UNCACHED | PMD_SECT_USER)); 172 #ifdef __aarch64__ 173 mmu_set_range_sect(mmu_idmap, 174 (1ul << 38), (1ul << 38), (1ul << 38) | (1ul << 29), 175 __pgprot(PMD_SECT_UNCACHED | PMD_SECT_USER)); 176 mmu_set_range_sect(mmu_idmap, 177 (1ul << 39), (1ul << 39), (1ul << 40), 178 __pgprot(PMD_SECT_UNCACHED | PMD_SECT_USER)); 179 #endif 180 181 /* armv8 requires code shared between EL1 and EL0 to be read-only */ 182 mmu_set_range_ptes(mmu_idmap, PHYS_OFFSET, 183 PHYS_OFFSET, code_end, 184 __pgprot(PTE_WBWA | PTE_RDONLY | PTE_USER)); 185 186 mmu_set_range_ptes(mmu_idmap, code_end, 187 code_end, phys_end, 188 __pgprot(PTE_WBWA | PTE_USER)); 189 190 mmu_enable(mmu_idmap); 191 return mmu_idmap; 192 } 193 194 phys_addr_t __virt_to_phys(unsigned long addr) 195 { 196 if (mmu_enabled()) { 197 pgd_t *pgtable = current_thread_info()->pgtable; 198 return virt_to_pte_phys(pgtable, (void *)addr); 199 } 200 return addr; 201 } 202 203 unsigned long __phys_to_virt(phys_addr_t addr) 204 { 205 /* 206 * We don't guarantee that phys_to_virt(virt_to_phys(vaddr)) == vaddr, but 207 * the default page tables do identity map all physical addresses, which 208 * means phys_to_virt(virt_to_phys((void *)paddr)) == paddr. 209 */ 210 assert(!mmu_enabled() || __virt_to_phys(addr) == addr); 211 return addr; 212 } 213 214 void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr) 215 { 216 pgd_t *pgd; 217 pmd_t *pmd; 218 pte_t *pte; 219 220 if (!mmu_enabled()) 221 return; 222 223 pgd = pgd_offset(pgtable, vaddr); 224 assert(pgd_valid(*pgd)); 225 pmd = pmd_offset(pgd, vaddr); 226 assert(pmd_valid(*pmd)); 227 228 if (pmd_huge(*pmd)) { 229 pmd_t entry = __pmd(pmd_val(*pmd) & ~PMD_SECT_USER); 230 WRITE_ONCE(*pmd, entry); 231 goto out_flush_tlb; 232 } 233 234 pte = pte_offset(pmd, vaddr); 235 assert(pte_valid(*pte)); 236 pte_t entry = __pte(pte_val(*pte) & ~PTE_USER); 237 WRITE_ONCE(*pte, entry); 238 239 out_flush_tlb: 240 flush_tlb_page(vaddr); 241 } 242