1 /*
2 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 *
6 * Based on the IA-32 version:
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
8 */
9
10 #include <linux/mm.h>
11 #include <linux/hugetlb.h>
12 #include <asm/pgtable.h>
13 #include <asm/pgalloc.h>
14 #include <asm/cacheflush.h>
15 #include <asm/machdep.h>
16
__hash_page_huge(unsigned long ea,unsigned long access,unsigned long vsid,pte_t * ptep,unsigned long trap,int local,int ssize,unsigned int shift,unsigned int mmu_psize)17 int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
18 pte_t *ptep, unsigned long trap, int local, int ssize,
19 unsigned int shift, unsigned int mmu_psize)
20 {
21 unsigned long old_pte, new_pte;
22 unsigned long va, rflags, pa, sz;
23 long slot;
24
25 BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
26
27 /* Search the Linux page table for a match with va */
28 va = hpt_va(ea, vsid, ssize);
29
30 /* At this point, we have a pte (old_pte) which can be used to build
31 * or update an HPTE. There are 2 cases:
32 *
33 * 1. There is a valid (present) pte with no associated HPTE (this is
34 * the most common case)
35 * 2. There is a valid (present) pte with an associated HPTE. The
36 * current values of the pp bits in the HPTE prevent access
37 * because we are doing software DIRTY bit management and the
38 * page is currently not DIRTY.
39 */
40
41
42 do {
43 old_pte = pte_val(*ptep);
44 /* If PTE busy, retry the access */
45 if (unlikely(old_pte & _PAGE_BUSY))
46 return 0;
47 /* If PTE permissions don't match, take page fault */
48 if (unlikely(access & ~old_pte))
49 return 1;
50 /* Try to lock the PTE, add ACCESSED and DIRTY if it was
51 * a write access */
52 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
53 if (access & _PAGE_RW)
54 new_pte |= _PAGE_DIRTY;
55 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
56 old_pte, new_pte));
57
58 rflags = 0x2 | (!(new_pte & _PAGE_RW));
59 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
60 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
61 sz = ((1UL) << shift);
62 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
63 /* No CPU has hugepages but lacks no execute, so we
64 * don't need to worry about that case */
65 rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
66
67 /* Check if pte already has an hpte (case 2) */
68 if (unlikely(old_pte & _PAGE_HASHPTE)) {
69 /* There MIGHT be an HPTE for this pte */
70 unsigned long hash, slot;
71
72 hash = hpt_hash(va, shift, ssize);
73 if (old_pte & _PAGE_F_SECOND)
74 hash = ~hash;
75 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
76 slot += (old_pte & _PAGE_F_GIX) >> 12;
77
78 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize,
79 ssize, local) == -1)
80 old_pte &= ~_PAGE_HPTEFLAGS;
81 }
82
83 if (likely(!(old_pte & _PAGE_HASHPTE))) {
84 unsigned long hash = hpt_hash(va, shift, ssize);
85 unsigned long hpte_group;
86
87 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
88
89 repeat:
90 hpte_group = ((hash & htab_hash_mask) *
91 HPTES_PER_GROUP) & ~0x7UL;
92
93 /* clear HPTE slot informations in new PTE */
94 #ifdef CONFIG_PPC_64K_PAGES
95 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
96 #else
97 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
98 #endif
99 /* Add in WIMG bits */
100 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
101 _PAGE_COHERENT | _PAGE_GUARDED));
102
103 /* Insert into the hash table, primary slot */
104 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
105 mmu_psize, ssize);
106
107 /* Primary is full, try the secondary */
108 if (unlikely(slot == -1)) {
109 hpte_group = ((~hash & htab_hash_mask) *
110 HPTES_PER_GROUP) & ~0x7UL;
111 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
112 HPTE_V_SECONDARY,
113 mmu_psize, ssize);
114 if (slot == -1) {
115 if (mftb() & 0x1)
116 hpte_group = ((hash & htab_hash_mask) *
117 HPTES_PER_GROUP)&~0x7UL;
118
119 ppc_md.hpte_remove(hpte_group);
120 goto repeat;
121 }
122 }
123
124 /*
125 * Hypervisor failure. Restore old pte and return -1
126 * similar to __hash_page_*
127 */
128 if (unlikely(slot == -2)) {
129 *ptep = __pte(old_pte);
130 hash_failure_debug(ea, access, vsid, trap, ssize,
131 mmu_psize, old_pte);
132 return -1;
133 }
134
135 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
136 }
137
138 /*
139 * No need to use ldarx/stdcx here
140 */
141 *ptep = __pte(new_pte & ~_PAGE_BUSY);
142 return 0;
143 }
144