1 /*
2 * arch/sh/mm/tlb-flush_64.c
3 *
4 * Copyright (C) 2000, 2001 Paolo Alberelli
5 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
6 * Copyright (C) 2003 - 2009 Paul Mundt
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 */
12 #include <linux/signal.h>
13 #include <linux/rwsem.h>
14 #include <linux/sched.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19 #include <linux/ptrace.h>
20 #include <linux/mman.h>
21 #include <linux/mm.h>
22 #include <linux/smp.h>
23 #include <linux/perf_event.h>
24 #include <linux/interrupt.h>
25 #include <asm/system.h>
26 #include <asm/io.h>
27 #include <asm/tlb.h>
28 #include <asm/uaccess.h>
29 #include <asm/pgalloc.h>
30 #include <asm/mmu_context.h>
31
32 extern void die(const char *,struct pt_regs *,long);
33
34 #define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
35 #define PPROT(flag) PFLAG(pgprot_val(prot),flag)
36
print_prots(pgprot_t prot)37 static inline void print_prots(pgprot_t prot)
38 {
39 printk("prot is 0x%016llx\n",pgprot_val(prot));
40
41 printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
42 PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
43 }
44
print_vma(struct vm_area_struct * vma)45 static inline void print_vma(struct vm_area_struct *vma)
46 {
47 printk("vma start 0x%08lx\n", vma->vm_start);
48 printk("vma end 0x%08lx\n", vma->vm_end);
49
50 print_prots(vma->vm_page_prot);
51 printk("vm_flags 0x%08lx\n", vma->vm_flags);
52 }
53
print_task(struct task_struct * tsk)54 static inline void print_task(struct task_struct *tsk)
55 {
56 printk("Task pid %d\n", task_pid_nr(tsk));
57 }
58
lookup_pte(struct mm_struct * mm,unsigned long address)59 static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
60 {
61 pgd_t *dir;
62 pud_t *pud;
63 pmd_t *pmd;
64 pte_t *pte;
65 pte_t entry;
66
67 dir = pgd_offset(mm, address);
68 if (pgd_none(*dir))
69 return NULL;
70
71 pud = pud_offset(dir, address);
72 if (pud_none(*pud))
73 return NULL;
74
75 pmd = pmd_offset(pud, address);
76 if (pmd_none(*pmd))
77 return NULL;
78
79 pte = pte_offset_kernel(pmd, address);
80 entry = *pte;
81 if (pte_none(entry) || !pte_present(entry))
82 return NULL;
83
84 return pte;
85 }
86
87 /*
88 * This routine handles page faults. It determines the address,
89 * and the problem, and then passes it off to one of the appropriate
90 * routines.
91 */
do_page_fault(struct pt_regs * regs,unsigned long writeaccess,unsigned long textaccess,unsigned long address)92 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
93 unsigned long textaccess, unsigned long address)
94 {
95 struct task_struct *tsk;
96 struct mm_struct *mm;
97 struct vm_area_struct * vma;
98 const struct exception_table_entry *fixup;
99 pte_t *pte;
100 int fault;
101
102 /* SIM
103 * Note this is now called with interrupts still disabled
104 * This is to cope with being called for a missing IO port
105 * address with interrupts disabled. This should be fixed as
106 * soon as we have a better 'fast path' miss handler.
107 *
108 * Plus take care how you try and debug this stuff.
109 * For example, writing debug data to a port which you
110 * have just faulted on is not going to work.
111 */
112
113 tsk = current;
114 mm = tsk->mm;
115
116 /* Not an IO address, so reenable interrupts */
117 local_irq_enable();
118
119 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
120
121 /*
122 * If we're in an interrupt or have no user
123 * context, we must not take the fault..
124 */
125 if (in_atomic() || !mm)
126 goto no_context;
127
128 /* TLB misses upon some cache flushes get done under cli() */
129 down_read(&mm->mmap_sem);
130
131 vma = find_vma(mm, address);
132
133 if (!vma) {
134 #ifdef DEBUG_FAULT
135 print_task(tsk);
136 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
137 __func__, __LINE__,
138 address,regs->pc,textaccess,writeaccess);
139 show_regs(regs);
140 #endif
141 goto bad_area;
142 }
143 if (vma->vm_start <= address) {
144 goto good_area;
145 }
146
147 if (!(vma->vm_flags & VM_GROWSDOWN)) {
148 #ifdef DEBUG_FAULT
149 print_task(tsk);
150 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
151 __func__, __LINE__,
152 address,regs->pc,textaccess,writeaccess);
153 show_regs(regs);
154
155 print_vma(vma);
156 #endif
157 goto bad_area;
158 }
159 if (expand_stack(vma, address)) {
160 #ifdef DEBUG_FAULT
161 print_task(tsk);
162 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
163 __func__, __LINE__,
164 address,regs->pc,textaccess,writeaccess);
165 show_regs(regs);
166 #endif
167 goto bad_area;
168 }
169 /*
170 * Ok, we have a good vm_area for this memory access, so
171 * we can handle it..
172 */
173 good_area:
174 if (textaccess) {
175 if (!(vma->vm_flags & VM_EXEC))
176 goto bad_area;
177 } else {
178 if (writeaccess) {
179 if (!(vma->vm_flags & VM_WRITE))
180 goto bad_area;
181 } else {
182 if (!(vma->vm_flags & VM_READ))
183 goto bad_area;
184 }
185 }
186
187 /*
188 * If for any reason at all we couldn't handle the fault,
189 * make sure we exit gracefully rather than endlessly redo
190 * the fault.
191 */
192 fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
193 if (unlikely(fault & VM_FAULT_ERROR)) {
194 if (fault & VM_FAULT_OOM)
195 goto out_of_memory;
196 else if (fault & VM_FAULT_SIGBUS)
197 goto do_sigbus;
198 BUG();
199 }
200
201 if (fault & VM_FAULT_MAJOR) {
202 tsk->maj_flt++;
203 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
204 regs, address);
205 } else {
206 tsk->min_flt++;
207 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
208 regs, address);
209 }
210
211 /* If we get here, the page fault has been handled. Do the TLB refill
212 now from the newly-setup PTE, to avoid having to fault again right
213 away on the same instruction. */
214 pte = lookup_pte (mm, address);
215 if (!pte) {
216 /* From empirical evidence, we can get here, due to
217 !pte_present(pte). (e.g. if a swap-in occurs, and the page
218 is swapped back out again before the process that wanted it
219 gets rescheduled?) */
220 goto no_pte;
221 }
222
223 __do_tlb_refill(address, textaccess, pte);
224
225 no_pte:
226
227 up_read(&mm->mmap_sem);
228 return;
229
230 /*
231 * Something tried to access memory that isn't in our memory map..
232 * Fix it, but check if it's kernel or user first..
233 */
234 bad_area:
235 #ifdef DEBUG_FAULT
236 printk("fault:bad area\n");
237 #endif
238 up_read(&mm->mmap_sem);
239
240 if (user_mode(regs)) {
241 static int count=0;
242 siginfo_t info;
243 if (count < 4) {
244 /* This is really to help debug faults when starting
245 * usermode, so only need a few */
246 count++;
247 printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
248 address, task_pid_nr(current), current->comm,
249 (unsigned long) regs->pc);
250 #if 0
251 show_regs(regs);
252 #endif
253 }
254 if (is_global_init(tsk)) {
255 panic("INIT had user mode bad_area\n");
256 }
257 tsk->thread.address = address;
258 tsk->thread.error_code = writeaccess;
259 info.si_signo = SIGSEGV;
260 info.si_errno = 0;
261 info.si_addr = (void *) address;
262 force_sig_info(SIGSEGV, &info, tsk);
263 return;
264 }
265
266 no_context:
267 #ifdef DEBUG_FAULT
268 printk("fault:No context\n");
269 #endif
270 /* Are we prepared to handle this kernel fault? */
271 fixup = search_exception_tables(regs->pc);
272 if (fixup) {
273 regs->pc = fixup->fixup;
274 return;
275 }
276
277 /*
278 * Oops. The kernel tried to access some bad page. We'll have to
279 * terminate things with extreme prejudice.
280 *
281 */
282 if (address < PAGE_SIZE)
283 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
284 else
285 printk(KERN_ALERT "Unable to handle kernel paging request");
286 printk(" at virtual address %08lx\n", address);
287 printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
288 die("Oops", regs, writeaccess);
289 do_exit(SIGKILL);
290
291 /*
292 * We ran out of memory, or some other thing happened to us that made
293 * us unable to handle the page fault gracefully.
294 */
295 out_of_memory:
296 up_read(&mm->mmap_sem);
297 if (!user_mode(regs))
298 goto no_context;
299 pagefault_out_of_memory();
300 return;
301
302 do_sigbus:
303 printk("fault:Do sigbus\n");
304 up_read(&mm->mmap_sem);
305
306 /*
307 * Send a sigbus, regardless of whether we were in kernel
308 * or user mode.
309 */
310 tsk->thread.address = address;
311 tsk->thread.error_code = writeaccess;
312 tsk->thread.trap_no = 14;
313 force_sig(SIGBUS, tsk);
314
315 /* Kernel mode? Handle exceptions or die */
316 if (!user_mode(regs))
317 goto no_context;
318 }
319
local_flush_tlb_one(unsigned long asid,unsigned long page)320 void local_flush_tlb_one(unsigned long asid, unsigned long page)
321 {
322 unsigned long long match, pteh=0, lpage;
323 unsigned long tlb;
324
325 /*
326 * Sign-extend based on neff.
327 */
328 lpage = neff_sign_extend(page);
329 match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
330 match |= lpage;
331
332 for_each_itlb_entry(tlb) {
333 asm volatile ("getcfg %1, 0, %0"
334 : "=r" (pteh)
335 : "r" (tlb) );
336
337 if (pteh == match) {
338 __flush_tlb_slot(tlb);
339 break;
340 }
341 }
342
343 for_each_dtlb_entry(tlb) {
344 asm volatile ("getcfg %1, 0, %0"
345 : "=r" (pteh)
346 : "r" (tlb) );
347
348 if (pteh == match) {
349 __flush_tlb_slot(tlb);
350 break;
351 }
352
353 }
354 }
355
local_flush_tlb_page(struct vm_area_struct * vma,unsigned long page)356 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
357 {
358 unsigned long flags;
359
360 if (vma->vm_mm) {
361 page &= PAGE_MASK;
362 local_irq_save(flags);
363 local_flush_tlb_one(get_asid(), page);
364 local_irq_restore(flags);
365 }
366 }
367
local_flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)368 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
369 unsigned long end)
370 {
371 unsigned long flags;
372 unsigned long long match, pteh=0, pteh_epn, pteh_low;
373 unsigned long tlb;
374 unsigned int cpu = smp_processor_id();
375 struct mm_struct *mm;
376
377 mm = vma->vm_mm;
378 if (cpu_context(cpu, mm) == NO_CONTEXT)
379 return;
380
381 local_irq_save(flags);
382
383 start &= PAGE_MASK;
384 end &= PAGE_MASK;
385
386 match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
387
388 /* Flush ITLB */
389 for_each_itlb_entry(tlb) {
390 asm volatile ("getcfg %1, 0, %0"
391 : "=r" (pteh)
392 : "r" (tlb) );
393
394 pteh_epn = pteh & PAGE_MASK;
395 pteh_low = pteh & ~PAGE_MASK;
396
397 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
398 __flush_tlb_slot(tlb);
399 }
400
401 /* Flush DTLB */
402 for_each_dtlb_entry(tlb) {
403 asm volatile ("getcfg %1, 0, %0"
404 : "=r" (pteh)
405 : "r" (tlb) );
406
407 pteh_epn = pteh & PAGE_MASK;
408 pteh_low = pteh & ~PAGE_MASK;
409
410 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
411 __flush_tlb_slot(tlb);
412 }
413
414 local_irq_restore(flags);
415 }
416
local_flush_tlb_mm(struct mm_struct * mm)417 void local_flush_tlb_mm(struct mm_struct *mm)
418 {
419 unsigned long flags;
420 unsigned int cpu = smp_processor_id();
421
422 if (cpu_context(cpu, mm) == NO_CONTEXT)
423 return;
424
425 local_irq_save(flags);
426
427 cpu_context(cpu, mm) = NO_CONTEXT;
428 if (mm == current->mm)
429 activate_context(mm, cpu);
430
431 local_irq_restore(flags);
432 }
433
local_flush_tlb_all(void)434 void local_flush_tlb_all(void)
435 {
436 /* Invalidate all, including shared pages, excluding fixed TLBs */
437 unsigned long flags, tlb;
438
439 local_irq_save(flags);
440
441 /* Flush each ITLB entry */
442 for_each_itlb_entry(tlb)
443 __flush_tlb_slot(tlb);
444
445 /* Flush each DTLB entry */
446 for_each_dtlb_entry(tlb)
447 __flush_tlb_slot(tlb);
448
449 local_irq_restore(flags);
450 }
451
local_flush_tlb_kernel_range(unsigned long start,unsigned long end)452 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
453 {
454 /* FIXME: Optimize this later.. */
455 flush_tlb_all();
456 }
457
__flush_tlb_global(void)458 void __flush_tlb_global(void)
459 {
460 flush_tlb_all();
461 }
462
__update_tlb(struct vm_area_struct * vma,unsigned long address,pte_t pte)463 void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
464 {
465 }
466