1 /*
2 * MMU enable and page table manipulation functions
3 *
4 * Copyright (C) 2014, Red Hat Inc, Andrew Jones <drjones@redhat.com>
5 *
6 * This work is licensed under the terms of the GNU LGPL, version 2.
7 */
8 #include <cpumask.h>
9 #include <memregions.h>
10 #include <asm/setup.h>
11 #include <asm/thread_info.h>
12 #include <asm/mmu.h>
13 #include <asm/setup.h>
14 #include <asm/page.h>
15 #include <asm/io.h>
16
17 #include "alloc_page.h"
18 #include "vmalloc.h"
19 #include <asm/pgtable-hwdef.h>
20 #include <asm/pgtable.h>
21
22 #include <linux/compiler.h>
23
24 pgd_t *mmu_idmap;
25
26 /* CPU 0 starts with disabled MMU */
27 static cpumask_t mmu_enabled_cpumask;
28
mmu_enabled(void)29 bool mmu_enabled(void)
30 {
31 /*
32 * mmu_enabled is called from places that are guarding the
33 * use of exclusive ops (which require the mmu to be enabled).
34 * That means we CANNOT call anything from here that may use a
35 * spinlock, atomic bitop, etc., otherwise we'll recurse.
36 * [cpumask_]test_bit is safe though.
37 */
38 if (is_user()) {
39 int cpu = current_thread_info()->cpu;
40 return cpumask_test_cpu(cpu, &mmu_enabled_cpumask);
41 }
42
43 return __mmu_enabled();
44 }
45
mmu_mark_enabled(int cpu)46 void mmu_mark_enabled(int cpu)
47 {
48 cpumask_set_cpu(cpu, &mmu_enabled_cpumask);
49 }
50
mmu_mark_disabled(int cpu)51 void mmu_mark_disabled(int cpu)
52 {
53 cpumask_clear_cpu(cpu, &mmu_enabled_cpumask);
54 }
55
56 extern void asm_mmu_enable(phys_addr_t pgtable);
mmu_enable(pgd_t * pgtable)57 void mmu_enable(pgd_t *pgtable)
58 {
59 struct thread_info *info = current_thread_info();
60
61 asm_mmu_enable(__pa(pgtable));
62
63 info->pgtable = pgtable;
64 mmu_mark_enabled(info->cpu);
65 }
66
67 extern void asm_mmu_disable(void);
mmu_disable(void)68 void mmu_disable(void)
69 {
70 unsigned long sp = current_stack_pointer;
71 int cpu = current_thread_info()->cpu;
72
73 assert_msg(__virt_to_phys(sp) == sp,
74 "Attempting to disable MMU with non-identity mapped stack");
75
76 mmu_mark_disabled(cpu);
77
78 asm_mmu_disable();
79 }
80
get_pte(pgd_t * pgtable,uintptr_t vaddr)81 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
82 {
83 pgd_t *pgd = pgd_offset(pgtable, vaddr);
84 pud_t *pud = pud_alloc(pgd, vaddr);
85 pmd_t *pmd = pmd_alloc(pud, vaddr);
86 pte_t *pte = pte_alloc(pmd, vaddr);
87
88 return &pte_val(*pte);
89 }
90
install_pte(pgd_t * pgtable,uintptr_t vaddr,pteval_t pte)91 static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte)
92 {
93 pteval_t *p_pte = get_pte(pgtable, vaddr);
94
95 WRITE_ONCE(*p_pte, pte);
96 flush_tlb_page(vaddr);
97 return p_pte;
98 }
99
install_page_prot(pgd_t * pgtable,phys_addr_t phys,uintptr_t vaddr,pgprot_t prot)100 static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys,
101 uintptr_t vaddr, pgprot_t prot)
102 {
103 pteval_t pte = phys;
104 pte |= PTE_TYPE_PAGE | PTE_AF | PTE_SHARED;
105 pte |= pgprot_val(prot);
106 return install_pte(pgtable, vaddr, pte);
107 }
108
install_page(pgd_t * pgtable,phys_addr_t phys,void * virt)109 pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt)
110 {
111 return install_page_prot(pgtable, phys, (uintptr_t)virt,
112 __pgprot(PTE_WBWA | PTE_USER));
113 }
114
115 /*
116 * NOTE: The Arm architecture might require the use of a
117 * break-before-make sequence before making changes to a PTE and
118 * certain conditions are met (see Arm ARM D5-2669 for AArch64 and
119 * B3-1378 for AArch32 for more details).
120 */
follow_pte(pgd_t * pgtable,uintptr_t vaddr)121 pteval_t *follow_pte(pgd_t *pgtable, uintptr_t vaddr)
122 {
123 pgd_t *pgd;
124 pud_t *pud;
125 pmd_t *pmd;
126 pte_t *pte;
127
128 pgd = pgd_offset(pgtable, vaddr);
129 if (!pgd_valid(*pgd))
130 return NULL;
131
132 pud = pud_offset(pgd, vaddr);
133 if (!pud_valid(*pud))
134 return NULL;
135
136 pmd = pmd_offset(pud, vaddr);
137 if (!pmd_valid(*pmd))
138 return NULL;
139 if (pmd_huge(*pmd))
140 return &pmd_val(*pmd);
141
142 pte = pte_offset(pmd, vaddr);
143 if (!pte_valid(*pte))
144 return NULL;
145
146 return &pte_val(*pte);
147 }
148
virt_to_pte_phys(pgd_t * pgtable,void * virt)149 phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *virt)
150 {
151 phys_addr_t mask;
152 pteval_t *pteval;
153
154 pteval = follow_pte(pgtable, (uintptr_t)virt);
155 if (!pteval) {
156 install_page(pgtable, (phys_addr_t)(unsigned long)virt, virt);
157 return (phys_addr_t)(unsigned long)virt;
158 }
159
160 if (pmd_huge(__pmd(*pteval)))
161 mask = PMD_MASK;
162 else
163 mask = PAGE_MASK;
164
165 return (*pteval & PHYS_MASK & mask) |
166 ((phys_addr_t)(unsigned long)virt & ~mask);
167 }
168
mmu_set_range_ptes(pgd_t * pgtable,uintptr_t virt_offset,phys_addr_t phys_start,phys_addr_t phys_end,pgprot_t prot)169 void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset,
170 phys_addr_t phys_start, phys_addr_t phys_end,
171 pgprot_t prot)
172 {
173 phys_addr_t paddr = phys_start & PAGE_MASK;
174 uintptr_t vaddr = virt_offset & PAGE_MASK;
175 uintptr_t virt_end = phys_end - paddr + vaddr;
176
177 for (; vaddr < virt_end; vaddr += PAGE_SIZE, paddr += PAGE_SIZE)
178 install_page_prot(pgtable, paddr, vaddr, prot);
179 }
180
mmu_set_range_sect(pgd_t * pgtable,uintptr_t virt_offset,phys_addr_t phys_start,phys_addr_t phys_end,pgprot_t prot)181 void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset,
182 phys_addr_t phys_start, phys_addr_t phys_end,
183 pgprot_t prot)
184 {
185 phys_addr_t paddr = phys_start & PMD_MASK;
186 uintptr_t vaddr = virt_offset & PMD_MASK;
187 uintptr_t virt_end = phys_end - paddr + vaddr;
188 pgd_t *pgd;
189 pud_t *pud;
190 pmd_t *pmd;
191 pmd_t entry;
192
193 for (; vaddr < virt_end; vaddr += PMD_SIZE, paddr += PMD_SIZE) {
194 pmd_val(entry) = paddr;
195 pmd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
196 pmd_val(entry) |= pgprot_val(prot);
197 pgd = pgd_offset(pgtable, vaddr);
198 pud = pud_alloc(pgd, vaddr);
199 pmd = pmd_alloc(pud, vaddr);
200 WRITE_ONCE(*pmd, entry);
201 flush_tlb_page(vaddr);
202 }
203 }
204
setup_mmu(phys_addr_t phys_end,void * unused)205 void *setup_mmu(phys_addr_t phys_end, void *unused)
206 {
207 struct mem_region *r;
208
209 /* 3G-4G region is reserved for vmalloc, cap phys_end at 3G */
210 if (phys_end > (3ul << 30))
211 phys_end = 3ul << 30;
212
213 #ifdef __aarch64__
214 init_alloc_vpage((void*)(4ul << 30));
215
216 assert_msg(system_supports_granule(PAGE_SIZE),
217 "Unsupported translation granule %ld\n", PAGE_SIZE);
218 #endif
219
220 if (!mmu_idmap)
221 mmu_idmap = alloc_page();
222
223 for (r = mem_regions; r->end; ++r) {
224 if (r->flags & (MR_F_IO | MR_F_RESERVED)) {
225 continue;
226 } else if (r->flags & MR_F_CODE) {
227 /* armv8 requires code shared between EL1 and EL0 to be read-only */
228 mmu_set_range_ptes(mmu_idmap, r->start, r->start, r->end,
229 __pgprot(PTE_WBWA | PTE_USER | PTE_RDONLY));
230 } else {
231 mmu_set_range_ptes(mmu_idmap, r->start, r->start, r->end,
232 __pgprot(PTE_WBWA | PTE_USER));
233 }
234 }
235
236 mmu_enable(mmu_idmap);
237 return mmu_idmap;
238 }
239
__ioremap(phys_addr_t phys_addr,size_t size)240 void __iomem *__ioremap(phys_addr_t phys_addr, size_t size)
241 {
242 phys_addr_t paddr_aligned = phys_addr & PAGE_MASK;
243 phys_addr_t paddr_end = PAGE_ALIGN(phys_addr + size);
244 pgprot_t prot = __pgprot(PTE_UNCACHED | PTE_USER | PTE_UXN | PTE_PXN);
245 pgd_t *pgtable;
246
247 assert(sizeof(long) == 8 || !(phys_addr >> 32));
248
249 if (mmu_enabled()) {
250 pgtable = current_thread_info()->pgtable;
251 } else {
252 if (!mmu_idmap)
253 mmu_idmap = alloc_page();
254 pgtable = mmu_idmap;
255 }
256
257 mmu_set_range_ptes(pgtable, paddr_aligned, paddr_aligned,
258 paddr_end, prot);
259
260 return (void __iomem *)(unsigned long)phys_addr;
261 }
262
__virt_to_phys(unsigned long addr)263 phys_addr_t __virt_to_phys(unsigned long addr)
264 {
265 if (mmu_enabled()) {
266 pgd_t *pgtable = current_thread_info()->pgtable;
267 return virt_to_pte_phys(pgtable, (void *)addr);
268 }
269 return addr;
270 }
271
__phys_to_virt(phys_addr_t addr)272 unsigned long __phys_to_virt(phys_addr_t addr)
273 {
274 /*
275 * We don't guarantee that phys_to_virt(virt_to_phys(vaddr)) == vaddr, but
276 * the default page tables do identity map all physical addresses, which
277 * means phys_to_virt(virt_to_phys((void *)paddr)) == paddr.
278 */
279 assert(!mmu_enabled() || __virt_to_phys(addr) == addr);
280 return addr;
281 }
282
mmu_clear_user(pgd_t * pgtable,unsigned long vaddr)283 void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
284 {
285 pteval_t *p_pte = follow_pte(pgtable, vaddr);
286 if (p_pte) {
287 pteval_t entry = *p_pte & ~PTE_USER;
288 WRITE_ONCE(*p_pte, entry);
289 flush_tlb_page(vaddr);
290 }
291 }
292