1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright IBM Corp. 2007, 2011
4 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
5 */
6
7 #include <linux/cpufeature.h>
8 #include <linux/export.h>
9 #include <linux/sched.h>
10 #include <linux/kernel.h>
11 #include <linux/errno.h>
12 #include <linux/gfp.h>
13 #include <linux/mm.h>
14 #include <linux/swap.h>
15 #include <linux/smp.h>
16 #include <linux/spinlock.h>
17 #include <linux/rcupdate.h>
18 #include <linux/slab.h>
19 #include <linux/leafops.h>
20 #include <linux/sysctl.h>
21 #include <linux/ksm.h>
22 #include <linux/mman.h>
23
24 #include <asm/tlbflush.h>
25 #include <asm/mmu_context.h>
26 #include <asm/page-states.h>
27 #include <asm/machine.h>
28
pgprot_writecombine(pgprot_t prot)29 pgprot_t pgprot_writecombine(pgprot_t prot)
30 {
31 /*
32 * mio_wb_bit_mask may be set on a different CPU, but it is only set
33 * once at init and only read afterwards.
34 */
35 return __pgprot(pgprot_val(prot) | mio_wb_bit_mask);
36 }
37 EXPORT_SYMBOL_GPL(pgprot_writecombine);
38
ptep_ipte_local(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)39 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
40 pte_t *ptep, int nodat)
41 {
42 unsigned long opt, asce;
43
44 if (machine_has_tlb_guest()) {
45 opt = 0;
46 asce = READ_ONCE(mm->context.gmap_asce);
47 if (asce == 0UL || nodat)
48 opt |= IPTE_NODAT;
49 if (asce != -1UL) {
50 asce = asce ? : mm->context.asce;
51 opt |= IPTE_GUEST_ASCE;
52 }
53 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
54 } else {
55 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
56 }
57 }
58
ptep_ipte_global(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)59 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
60 pte_t *ptep, int nodat)
61 {
62 unsigned long opt, asce;
63
64 if (machine_has_tlb_guest()) {
65 opt = 0;
66 asce = READ_ONCE(mm->context.gmap_asce);
67 if (asce == 0UL || nodat)
68 opt |= IPTE_NODAT;
69 if (asce != -1UL) {
70 asce = asce ? : mm->context.asce;
71 opt |= IPTE_GUEST_ASCE;
72 }
73 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
74 } else {
75 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
76 }
77 }
78
ptep_flush_direct(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)79 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
80 unsigned long addr, pte_t *ptep,
81 int nodat)
82 {
83 pte_t old;
84
85 old = *ptep;
86 if (unlikely(pte_val(old) & _PAGE_INVALID))
87 return old;
88 atomic_inc(&mm->context.flush_count);
89 if (cpu_has_tlb_lc() &&
90 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
91 ptep_ipte_local(mm, addr, ptep, nodat);
92 else
93 ptep_ipte_global(mm, addr, ptep, nodat);
94 atomic_dec(&mm->context.flush_count);
95 return old;
96 }
97
ptep_flush_lazy(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)98 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
99 unsigned long addr, pte_t *ptep,
100 int nodat)
101 {
102 pte_t old;
103
104 old = *ptep;
105 if (unlikely(pte_val(old) & _PAGE_INVALID))
106 return old;
107 atomic_inc(&mm->context.flush_count);
108 if (cpumask_equal(&mm->context.cpu_attach_mask,
109 cpumask_of(smp_processor_id()))) {
110 set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_INVALID)));
111 mm->context.flush_mm = 1;
112 } else
113 ptep_ipte_global(mm, addr, ptep, nodat);
114 atomic_dec(&mm->context.flush_count);
115 return old;
116 }
117
ptep_xchg_direct(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t new)118 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
119 pte_t *ptep, pte_t new)
120 {
121 pte_t old;
122
123 preempt_disable();
124 old = ptep_flush_direct(mm, addr, ptep, 1);
125 set_pte(ptep, new);
126 preempt_enable();
127 return old;
128 }
129 EXPORT_SYMBOL(ptep_xchg_direct);
130
131 /*
132 * Caller must check that new PTE only differs in _PAGE_PROTECT HW bit, so that
133 * RDP can be used instead of IPTE. See also comments at pte_allow_rdp().
134 */
ptep_reset_dat_prot(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t new)135 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
136 pte_t new)
137 {
138 preempt_disable();
139 atomic_inc(&mm->context.flush_count);
140 if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
141 __ptep_rdp(addr, ptep, 1);
142 else
143 __ptep_rdp(addr, ptep, 0);
144 /*
145 * PTE is not invalidated by RDP, only _PAGE_PROTECT is cleared. That
146 * means it is still valid and active, and must not be changed according
147 * to the architecture. But writing a new value that only differs in SW
148 * bits is allowed.
149 */
150 set_pte(ptep, new);
151 atomic_dec(&mm->context.flush_count);
152 preempt_enable();
153 }
154 EXPORT_SYMBOL(ptep_reset_dat_prot);
155
ptep_xchg_lazy(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t new)156 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
157 pte_t *ptep, pte_t new)
158 {
159 pte_t old;
160
161 preempt_disable();
162 old = ptep_flush_lazy(mm, addr, ptep, 1);
163 set_pte(ptep, new);
164 preempt_enable();
165 return old;
166 }
167 EXPORT_SYMBOL(ptep_xchg_lazy);
168
ptep_modify_prot_start(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)169 pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,
170 pte_t *ptep)
171 {
172 return ptep_flush_lazy(vma->vm_mm, addr, ptep, 1);
173 }
174
ptep_modify_prot_commit(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t old_pte,pte_t pte)175 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
176 pte_t *ptep, pte_t old_pte, pte_t pte)
177 {
178 set_pte(ptep, pte);
179 }
180
pmdp_idte_local(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)181 static inline void pmdp_idte_local(struct mm_struct *mm,
182 unsigned long addr, pmd_t *pmdp)
183 {
184 if (machine_has_tlb_guest())
185 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_LOCAL);
186 else
187 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
188 }
189
pmdp_idte_global(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)190 static inline void pmdp_idte_global(struct mm_struct *mm,
191 unsigned long addr, pmd_t *pmdp)
192 {
193 if (machine_has_tlb_guest()) {
194 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
195 mm->context.asce, IDTE_GLOBAL);
196 } else {
197 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
198 }
199 }
200
pmdp_flush_direct(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)201 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
202 unsigned long addr, pmd_t *pmdp)
203 {
204 pmd_t old;
205
206 old = *pmdp;
207 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
208 return old;
209 atomic_inc(&mm->context.flush_count);
210 if (cpu_has_tlb_lc() &&
211 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
212 pmdp_idte_local(mm, addr, pmdp);
213 else
214 pmdp_idte_global(mm, addr, pmdp);
215 atomic_dec(&mm->context.flush_count);
216 return old;
217 }
218
pmdp_flush_lazy(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)219 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
220 unsigned long addr, pmd_t *pmdp)
221 {
222 pmd_t old;
223
224 old = *pmdp;
225 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
226 return old;
227 atomic_inc(&mm->context.flush_count);
228 if (cpumask_equal(&mm->context.cpu_attach_mask,
229 cpumask_of(smp_processor_id()))) {
230 set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID)));
231 mm->context.flush_mm = 1;
232 } else {
233 pmdp_idte_global(mm, addr, pmdp);
234 }
235 atomic_dec(&mm->context.flush_count);
236 return old;
237 }
238
pmdp_xchg_direct(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t new)239 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
240 pmd_t *pmdp, pmd_t new)
241 {
242 pmd_t old;
243
244 preempt_disable();
245 old = pmdp_flush_direct(mm, addr, pmdp);
246 set_pmd(pmdp, new);
247 preempt_enable();
248 return old;
249 }
250 EXPORT_SYMBOL(pmdp_xchg_direct);
251
pmdp_xchg_lazy(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t new)252 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
253 pmd_t *pmdp, pmd_t new)
254 {
255 pmd_t old;
256
257 preempt_disable();
258 old = pmdp_flush_lazy(mm, addr, pmdp);
259 set_pmd(pmdp, new);
260 preempt_enable();
261 return old;
262 }
263 EXPORT_SYMBOL(pmdp_xchg_lazy);
264
pudp_idte_local(struct mm_struct * mm,unsigned long addr,pud_t * pudp)265 static inline void pudp_idte_local(struct mm_struct *mm,
266 unsigned long addr, pud_t *pudp)
267 {
268 if (machine_has_tlb_guest())
269 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
270 mm->context.asce, IDTE_LOCAL);
271 else
272 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
273 }
274
pudp_idte_global(struct mm_struct * mm,unsigned long addr,pud_t * pudp)275 static inline void pudp_idte_global(struct mm_struct *mm,
276 unsigned long addr, pud_t *pudp)
277 {
278 if (machine_has_tlb_guest())
279 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
280 mm->context.asce, IDTE_GLOBAL);
281 else
282 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
283 }
284
pudp_flush_direct(struct mm_struct * mm,unsigned long addr,pud_t * pudp)285 static inline pud_t pudp_flush_direct(struct mm_struct *mm,
286 unsigned long addr, pud_t *pudp)
287 {
288 pud_t old;
289
290 old = *pudp;
291 if (pud_val(old) & _REGION_ENTRY_INVALID)
292 return old;
293 atomic_inc(&mm->context.flush_count);
294 if (cpu_has_tlb_lc() &&
295 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
296 pudp_idte_local(mm, addr, pudp);
297 else
298 pudp_idte_global(mm, addr, pudp);
299 atomic_dec(&mm->context.flush_count);
300 return old;
301 }
302
pudp_xchg_direct(struct mm_struct * mm,unsigned long addr,pud_t * pudp,pud_t new)303 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr,
304 pud_t *pudp, pud_t new)
305 {
306 pud_t old;
307
308 preempt_disable();
309 old = pudp_flush_direct(mm, addr, pudp);
310 set_pud(pudp, new);
311 preempt_enable();
312 return old;
313 }
314 EXPORT_SYMBOL(pudp_xchg_direct);
315
316 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
pgtable_trans_huge_deposit(struct mm_struct * mm,pmd_t * pmdp,pgtable_t pgtable)317 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
318 pgtable_t pgtable)
319 {
320 struct list_head *lh = (struct list_head *) pgtable;
321
322 assert_spin_locked(pmd_lockptr(mm, pmdp));
323
324 /* FIFO */
325 if (!pmd_huge_pte(mm, pmdp))
326 INIT_LIST_HEAD(lh);
327 else
328 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
329 pmd_huge_pte(mm, pmdp) = pgtable;
330 }
331
pgtable_trans_huge_withdraw(struct mm_struct * mm,pmd_t * pmdp)332 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
333 {
334 struct list_head *lh;
335 pgtable_t pgtable;
336 pte_t *ptep;
337
338 assert_spin_locked(pmd_lockptr(mm, pmdp));
339
340 /* FIFO */
341 pgtable = pmd_huge_pte(mm, pmdp);
342 lh = (struct list_head *) pgtable;
343 if (list_empty(lh))
344 pmd_huge_pte(mm, pmdp) = NULL;
345 else {
346 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
347 list_del(lh);
348 }
349 ptep = (pte_t *) pgtable;
350 set_pte(ptep, __pte(_PAGE_INVALID));
351 ptep++;
352 set_pte(ptep, __pte(_PAGE_INVALID));
353 return pgtable;
354 }
355 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
356