1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright IBM Corp. 2007, 2011
4 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
5 */
6
7 #include <linux/cpufeature.h>
8 #include <linux/sched.h>
9 #include <linux/kernel.h>
10 #include <linux/errno.h>
11 #include <linux/gfp.h>
12 #include <linux/mm.h>
13 #include <linux/swap.h>
14 #include <linux/smp.h>
15 #include <linux/spinlock.h>
16 #include <linux/rcupdate.h>
17 #include <linux/slab.h>
18 #include <linux/swapops.h>
19 #include <linux/sysctl.h>
20 #include <linux/ksm.h>
21 #include <linux/mman.h>
22
23 #include <asm/tlb.h>
24 #include <asm/tlbflush.h>
25 #include <asm/mmu_context.h>
26 #include <asm/page-states.h>
27 #include <asm/machine.h>
28
pgprot_writecombine(pgprot_t prot)29 pgprot_t pgprot_writecombine(pgprot_t prot)
30 {
31 /*
32 * mio_wb_bit_mask may be set on a different CPU, but it is only set
33 * once at init and only read afterwards.
34 */
35 return __pgprot(pgprot_val(prot) | mio_wb_bit_mask);
36 }
37 EXPORT_SYMBOL_GPL(pgprot_writecombine);
38
ptep_ipte_local(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)39 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
40 pte_t *ptep, int nodat)
41 {
42 unsigned long opt, asce;
43
44 if (machine_has_tlb_guest()) {
45 opt = 0;
46 asce = READ_ONCE(mm->context.gmap_asce);
47 if (asce == 0UL || nodat)
48 opt |= IPTE_NODAT;
49 if (asce != -1UL) {
50 asce = asce ? : mm->context.asce;
51 opt |= IPTE_GUEST_ASCE;
52 }
53 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
54 } else {
55 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
56 }
57 }
58
ptep_ipte_global(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)59 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
60 pte_t *ptep, int nodat)
61 {
62 unsigned long opt, asce;
63
64 if (machine_has_tlb_guest()) {
65 opt = 0;
66 asce = READ_ONCE(mm->context.gmap_asce);
67 if (asce == 0UL || nodat)
68 opt |= IPTE_NODAT;
69 if (asce != -1UL) {
70 asce = asce ? : mm->context.asce;
71 opt |= IPTE_GUEST_ASCE;
72 }
73 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
74 } else {
75 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
76 }
77 }
78
ptep_flush_direct(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)79 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
80 unsigned long addr, pte_t *ptep,
81 int nodat)
82 {
83 pte_t old;
84
85 old = *ptep;
86 if (unlikely(pte_val(old) & _PAGE_INVALID))
87 return old;
88 atomic_inc(&mm->context.flush_count);
89 if (cpu_has_tlb_lc() &&
90 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
91 ptep_ipte_local(mm, addr, ptep, nodat);
92 else
93 ptep_ipte_global(mm, addr, ptep, nodat);
94 atomic_dec(&mm->context.flush_count);
95 return old;
96 }
97
ptep_flush_lazy(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int nodat)98 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
99 unsigned long addr, pte_t *ptep,
100 int nodat)
101 {
102 pte_t old;
103
104 old = *ptep;
105 if (unlikely(pte_val(old) & _PAGE_INVALID))
106 return old;
107 atomic_inc(&mm->context.flush_count);
108 if (cpumask_equal(&mm->context.cpu_attach_mask,
109 cpumask_of(smp_processor_id()))) {
110 set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_INVALID)));
111 mm->context.flush_mm = 1;
112 } else
113 ptep_ipte_global(mm, addr, ptep, nodat);
114 atomic_dec(&mm->context.flush_count);
115 return old;
116 }
117
pgste_get_lock(pte_t * ptep)118 static inline pgste_t pgste_get_lock(pte_t *ptep)
119 {
120 unsigned long value = 0;
121 #ifdef CONFIG_PGSTE
122 unsigned long *ptr = (unsigned long *)(ptep + PTRS_PER_PTE);
123
124 do {
125 value = __atomic64_or_barrier(PGSTE_PCL_BIT, ptr);
126 } while (value & PGSTE_PCL_BIT);
127 value |= PGSTE_PCL_BIT;
128 #endif
129 return __pgste(value);
130 }
131
pgste_set_unlock(pte_t * ptep,pgste_t pgste)132 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
133 {
134 #ifdef CONFIG_PGSTE
135 barrier();
136 WRITE_ONCE(*(unsigned long *)(ptep + PTRS_PER_PTE), pgste_val(pgste) & ~PGSTE_PCL_BIT);
137 #endif
138 }
139
pgste_get(pte_t * ptep)140 static inline pgste_t pgste_get(pte_t *ptep)
141 {
142 unsigned long pgste = 0;
143 #ifdef CONFIG_PGSTE
144 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
145 #endif
146 return __pgste(pgste);
147 }
148
pgste_set(pte_t * ptep,pgste_t pgste)149 static inline void pgste_set(pte_t *ptep, pgste_t pgste)
150 {
151 #ifdef CONFIG_PGSTE
152 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
153 #endif
154 }
155
pgste_update_all(pte_t pte,pgste_t pgste,struct mm_struct * mm)156 static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
157 struct mm_struct *mm)
158 {
159 #ifdef CONFIG_PGSTE
160 unsigned long address, bits, skey;
161
162 if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID)
163 return pgste;
164 address = pte_val(pte) & PAGE_MASK;
165 skey = (unsigned long) page_get_storage_key(address);
166 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
167 /* Transfer page changed & referenced bit to guest bits in pgste */
168 pgste = set_pgste_bit(pgste, bits << 48); /* GR bit & GC bit */
169 /* Copy page access key and fetch protection bit to pgste */
170 pgste = clear_pgste_bit(pgste, PGSTE_ACC_BITS | PGSTE_FP_BIT);
171 pgste = set_pgste_bit(pgste, (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56);
172 #endif
173 return pgste;
174
175 }
176
pgste_set_key(pte_t * ptep,pgste_t pgste,pte_t entry,struct mm_struct * mm)177 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
178 struct mm_struct *mm)
179 {
180 #ifdef CONFIG_PGSTE
181 unsigned long address;
182 unsigned long nkey;
183
184 if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID)
185 return;
186 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
187 address = pte_val(entry) & PAGE_MASK;
188 /*
189 * Set page access key and fetch protection bit from pgste.
190 * The guest C/R information is still in the PGSTE, set real
191 * key C/R to 0.
192 */
193 nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
194 nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
195 page_set_storage_key(address, nkey, 0);
196 #endif
197 }
198
pgste_set_pte(pte_t * ptep,pgste_t pgste,pte_t entry)199 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
200 {
201 #ifdef CONFIG_PGSTE
202 if ((pte_val(entry) & _PAGE_PRESENT) &&
203 (pte_val(entry) & _PAGE_WRITE) &&
204 !(pte_val(entry) & _PAGE_INVALID)) {
205 if (!machine_has_esop()) {
206 /*
207 * Without enhanced suppression-on-protection force
208 * the dirty bit on for all writable ptes.
209 */
210 entry = set_pte_bit(entry, __pgprot(_PAGE_DIRTY));
211 entry = clear_pte_bit(entry, __pgprot(_PAGE_PROTECT));
212 }
213 if (!(pte_val(entry) & _PAGE_PROTECT))
214 /* This pte allows write access, set user-dirty */
215 pgste = set_pgste_bit(pgste, PGSTE_UC_BIT);
216 }
217 #endif
218 set_pte(ptep, entry);
219 return pgste;
220 }
221
pgste_pte_notify(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pgste_t pgste)222 static inline pgste_t pgste_pte_notify(struct mm_struct *mm,
223 unsigned long addr,
224 pte_t *ptep, pgste_t pgste)
225 {
226 #ifdef CONFIG_PGSTE
227 unsigned long bits;
228
229 bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT);
230 if (bits) {
231 pgste = __pgste(pgste_val(pgste) ^ bits);
232 ptep_notify(mm, addr, ptep, bits);
233 }
234 #endif
235 return pgste;
236 }
237
ptep_xchg_start(struct mm_struct * mm,unsigned long addr,pte_t * ptep)238 static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
239 unsigned long addr, pte_t *ptep)
240 {
241 pgste_t pgste = __pgste(0);
242
243 if (mm_has_pgste(mm)) {
244 pgste = pgste_get_lock(ptep);
245 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
246 }
247 return pgste;
248 }
249
ptep_xchg_commit(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pgste_t pgste,pte_t old,pte_t new)250 static inline pte_t ptep_xchg_commit(struct mm_struct *mm,
251 unsigned long addr, pte_t *ptep,
252 pgste_t pgste, pte_t old, pte_t new)
253 {
254 if (mm_has_pgste(mm)) {
255 if (pte_val(old) & _PAGE_INVALID)
256 pgste_set_key(ptep, pgste, new, mm);
257 if (pte_val(new) & _PAGE_INVALID) {
258 pgste = pgste_update_all(old, pgste, mm);
259 if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
260 _PGSTE_GPS_USAGE_UNUSED)
261 old = set_pte_bit(old, __pgprot(_PAGE_UNUSED));
262 }
263 pgste = pgste_set_pte(ptep, pgste, new);
264 pgste_set_unlock(ptep, pgste);
265 } else {
266 set_pte(ptep, new);
267 }
268 return old;
269 }
270
ptep_xchg_direct(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t new)271 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
272 pte_t *ptep, pte_t new)
273 {
274 pgste_t pgste;
275 pte_t old;
276 int nodat;
277
278 preempt_disable();
279 pgste = ptep_xchg_start(mm, addr, ptep);
280 nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
281 old = ptep_flush_direct(mm, addr, ptep, nodat);
282 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
283 preempt_enable();
284 return old;
285 }
286 EXPORT_SYMBOL(ptep_xchg_direct);
287
288 /*
289 * Caller must check that new PTE only differs in _PAGE_PROTECT HW bit, so that
290 * RDP can be used instead of IPTE. See also comments at pte_allow_rdp().
291 */
ptep_reset_dat_prot(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t new)292 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
293 pte_t new)
294 {
295 preempt_disable();
296 atomic_inc(&mm->context.flush_count);
297 if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
298 __ptep_rdp(addr, ptep, 0, 0, 1);
299 else
300 __ptep_rdp(addr, ptep, 0, 0, 0);
301 /*
302 * PTE is not invalidated by RDP, only _PAGE_PROTECT is cleared. That
303 * means it is still valid and active, and must not be changed according
304 * to the architecture. But writing a new value that only differs in SW
305 * bits is allowed.
306 */
307 set_pte(ptep, new);
308 atomic_dec(&mm->context.flush_count);
309 preempt_enable();
310 }
311 EXPORT_SYMBOL(ptep_reset_dat_prot);
312
ptep_xchg_lazy(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t new)313 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
314 pte_t *ptep, pte_t new)
315 {
316 pgste_t pgste;
317 pte_t old;
318 int nodat;
319
320 preempt_disable();
321 pgste = ptep_xchg_start(mm, addr, ptep);
322 nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
323 old = ptep_flush_lazy(mm, addr, ptep, nodat);
324 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
325 preempt_enable();
326 return old;
327 }
328 EXPORT_SYMBOL(ptep_xchg_lazy);
329
ptep_modify_prot_start(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)330 pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,
331 pte_t *ptep)
332 {
333 pgste_t pgste;
334 pte_t old;
335 int nodat;
336 struct mm_struct *mm = vma->vm_mm;
337
338 preempt_disable();
339 pgste = ptep_xchg_start(mm, addr, ptep);
340 nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
341 old = ptep_flush_lazy(mm, addr, ptep, nodat);
342 if (mm_has_pgste(mm)) {
343 pgste = pgste_update_all(old, pgste, mm);
344 pgste_set(ptep, pgste);
345 }
346 return old;
347 }
348
ptep_modify_prot_commit(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t old_pte,pte_t pte)349 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
350 pte_t *ptep, pte_t old_pte, pte_t pte)
351 {
352 pgste_t pgste;
353 struct mm_struct *mm = vma->vm_mm;
354
355 if (mm_has_pgste(mm)) {
356 pgste = pgste_get(ptep);
357 pgste_set_key(ptep, pgste, pte, mm);
358 pgste = pgste_set_pte(ptep, pgste, pte);
359 pgste_set_unlock(ptep, pgste);
360 } else {
361 set_pte(ptep, pte);
362 }
363 preempt_enable();
364 }
365
pmdp_idte_local(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)366 static inline void pmdp_idte_local(struct mm_struct *mm,
367 unsigned long addr, pmd_t *pmdp)
368 {
369 if (machine_has_tlb_guest())
370 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
371 mm->context.asce, IDTE_LOCAL);
372 else
373 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
374 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
375 gmap_pmdp_idte_local(mm, addr);
376 }
377
pmdp_idte_global(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)378 static inline void pmdp_idte_global(struct mm_struct *mm,
379 unsigned long addr, pmd_t *pmdp)
380 {
381 if (machine_has_tlb_guest()) {
382 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
383 mm->context.asce, IDTE_GLOBAL);
384 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
385 gmap_pmdp_idte_global(mm, addr);
386 } else if (cpu_has_idte()) {
387 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
388 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
389 gmap_pmdp_idte_global(mm, addr);
390 } else {
391 __pmdp_csp(pmdp);
392 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
393 gmap_pmdp_csp(mm, addr);
394 }
395 }
396
pmdp_flush_direct(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)397 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
398 unsigned long addr, pmd_t *pmdp)
399 {
400 pmd_t old;
401
402 old = *pmdp;
403 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
404 return old;
405 atomic_inc(&mm->context.flush_count);
406 if (cpu_has_tlb_lc() &&
407 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
408 pmdp_idte_local(mm, addr, pmdp);
409 else
410 pmdp_idte_global(mm, addr, pmdp);
411 atomic_dec(&mm->context.flush_count);
412 return old;
413 }
414
pmdp_flush_lazy(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)415 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
416 unsigned long addr, pmd_t *pmdp)
417 {
418 pmd_t old;
419
420 old = *pmdp;
421 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
422 return old;
423 atomic_inc(&mm->context.flush_count);
424 if (cpumask_equal(&mm->context.cpu_attach_mask,
425 cpumask_of(smp_processor_id()))) {
426 set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID)));
427 mm->context.flush_mm = 1;
428 if (mm_has_pgste(mm))
429 gmap_pmdp_invalidate(mm, addr);
430 } else {
431 pmdp_idte_global(mm, addr, pmdp);
432 }
433 atomic_dec(&mm->context.flush_count);
434 return old;
435 }
436
437 #ifdef CONFIG_PGSTE
pmd_lookup(struct mm_struct * mm,unsigned long addr,pmd_t ** pmdp)438 static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp)
439 {
440 struct vm_area_struct *vma;
441 pgd_t *pgd;
442 p4d_t *p4d;
443 pud_t *pud;
444
445 /* We need a valid VMA, otherwise this is clearly a fault. */
446 vma = vma_lookup(mm, addr);
447 if (!vma)
448 return -EFAULT;
449
450 pgd = pgd_offset(mm, addr);
451 if (!pgd_present(*pgd))
452 return -ENOENT;
453
454 p4d = p4d_offset(pgd, addr);
455 if (!p4d_present(*p4d))
456 return -ENOENT;
457
458 pud = pud_offset(p4d, addr);
459 if (!pud_present(*pud))
460 return -ENOENT;
461
462 /* Large PUDs are not supported yet. */
463 if (pud_leaf(*pud))
464 return -EFAULT;
465
466 *pmdp = pmd_offset(pud, addr);
467 return 0;
468 }
469 #endif
470
pmdp_xchg_direct(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t new)471 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
472 pmd_t *pmdp, pmd_t new)
473 {
474 pmd_t old;
475
476 preempt_disable();
477 old = pmdp_flush_direct(mm, addr, pmdp);
478 set_pmd(pmdp, new);
479 preempt_enable();
480 return old;
481 }
482 EXPORT_SYMBOL(pmdp_xchg_direct);
483
pmdp_xchg_lazy(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t new)484 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
485 pmd_t *pmdp, pmd_t new)
486 {
487 pmd_t old;
488
489 preempt_disable();
490 old = pmdp_flush_lazy(mm, addr, pmdp);
491 set_pmd(pmdp, new);
492 preempt_enable();
493 return old;
494 }
495 EXPORT_SYMBOL(pmdp_xchg_lazy);
496
pudp_idte_local(struct mm_struct * mm,unsigned long addr,pud_t * pudp)497 static inline void pudp_idte_local(struct mm_struct *mm,
498 unsigned long addr, pud_t *pudp)
499 {
500 if (machine_has_tlb_guest())
501 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
502 mm->context.asce, IDTE_LOCAL);
503 else
504 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
505 }
506
pudp_idte_global(struct mm_struct * mm,unsigned long addr,pud_t * pudp)507 static inline void pudp_idte_global(struct mm_struct *mm,
508 unsigned long addr, pud_t *pudp)
509 {
510 if (machine_has_tlb_guest())
511 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
512 mm->context.asce, IDTE_GLOBAL);
513 else if (cpu_has_idte())
514 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
515 else
516 /*
517 * Invalid bit position is the same for pmd and pud, so we can
518 * reuse _pmd_csp() here
519 */
520 __pmdp_csp((pmd_t *) pudp);
521 }
522
pudp_flush_direct(struct mm_struct * mm,unsigned long addr,pud_t * pudp)523 static inline pud_t pudp_flush_direct(struct mm_struct *mm,
524 unsigned long addr, pud_t *pudp)
525 {
526 pud_t old;
527
528 old = *pudp;
529 if (pud_val(old) & _REGION_ENTRY_INVALID)
530 return old;
531 atomic_inc(&mm->context.flush_count);
532 if (cpu_has_tlb_lc() &&
533 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
534 pudp_idte_local(mm, addr, pudp);
535 else
536 pudp_idte_global(mm, addr, pudp);
537 atomic_dec(&mm->context.flush_count);
538 return old;
539 }
540
pudp_xchg_direct(struct mm_struct * mm,unsigned long addr,pud_t * pudp,pud_t new)541 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr,
542 pud_t *pudp, pud_t new)
543 {
544 pud_t old;
545
546 preempt_disable();
547 old = pudp_flush_direct(mm, addr, pudp);
548 set_pud(pudp, new);
549 preempt_enable();
550 return old;
551 }
552 EXPORT_SYMBOL(pudp_xchg_direct);
553
554 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
pgtable_trans_huge_deposit(struct mm_struct * mm,pmd_t * pmdp,pgtable_t pgtable)555 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
556 pgtable_t pgtable)
557 {
558 struct list_head *lh = (struct list_head *) pgtable;
559
560 assert_spin_locked(pmd_lockptr(mm, pmdp));
561
562 /* FIFO */
563 if (!pmd_huge_pte(mm, pmdp))
564 INIT_LIST_HEAD(lh);
565 else
566 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
567 pmd_huge_pte(mm, pmdp) = pgtable;
568 }
569
pgtable_trans_huge_withdraw(struct mm_struct * mm,pmd_t * pmdp)570 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
571 {
572 struct list_head *lh;
573 pgtable_t pgtable;
574 pte_t *ptep;
575
576 assert_spin_locked(pmd_lockptr(mm, pmdp));
577
578 /* FIFO */
579 pgtable = pmd_huge_pte(mm, pmdp);
580 lh = (struct list_head *) pgtable;
581 if (list_empty(lh))
582 pmd_huge_pte(mm, pmdp) = NULL;
583 else {
584 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
585 list_del(lh);
586 }
587 ptep = (pte_t *) pgtable;
588 set_pte(ptep, __pte(_PAGE_INVALID));
589 ptep++;
590 set_pte(ptep, __pte(_PAGE_INVALID));
591 return pgtable;
592 }
593 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
594
595 #ifdef CONFIG_PGSTE
ptep_set_pte_at(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t entry)596 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
597 pte_t *ptep, pte_t entry)
598 {
599 pgste_t pgste;
600
601 /* the mm_has_pgste() check is done in set_pte_at() */
602 preempt_disable();
603 pgste = pgste_get_lock(ptep);
604 pgste = clear_pgste_bit(pgste, _PGSTE_GPS_ZERO);
605 pgste_set_key(ptep, pgste, entry, mm);
606 pgste = pgste_set_pte(ptep, pgste, entry);
607 pgste_set_unlock(ptep, pgste);
608 preempt_enable();
609 }
610
ptep_set_notify(struct mm_struct * mm,unsigned long addr,pte_t * ptep)611 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
612 {
613 pgste_t pgste;
614
615 preempt_disable();
616 pgste = pgste_get_lock(ptep);
617 pgste = set_pgste_bit(pgste, PGSTE_IN_BIT);
618 pgste_set_unlock(ptep, pgste);
619 preempt_enable();
620 }
621
622 /**
623 * ptep_force_prot - change access rights of a locked pte
624 * @mm: pointer to the process mm_struct
625 * @addr: virtual address in the guest address space
626 * @ptep: pointer to the page table entry
627 * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE
628 * @bit: pgste bit to set (e.g. for notification)
629 *
630 * Returns 0 if the access rights were changed and -EAGAIN if the current
631 * and requested access rights are incompatible.
632 */
ptep_force_prot(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int prot,unsigned long bit)633 int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
634 pte_t *ptep, int prot, unsigned long bit)
635 {
636 pte_t entry;
637 pgste_t pgste;
638 int pte_i, pte_p, nodat;
639
640 pgste = pgste_get_lock(ptep);
641 entry = *ptep;
642 /* Check pte entry after all locks have been acquired */
643 pte_i = pte_val(entry) & _PAGE_INVALID;
644 pte_p = pte_val(entry) & _PAGE_PROTECT;
645 if ((pte_i && (prot != PROT_NONE)) ||
646 (pte_p && (prot & PROT_WRITE))) {
647 pgste_set_unlock(ptep, pgste);
648 return -EAGAIN;
649 }
650 /* Change access rights and set pgste bit */
651 nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
652 if (prot == PROT_NONE && !pte_i) {
653 ptep_flush_direct(mm, addr, ptep, nodat);
654 pgste = pgste_update_all(entry, pgste, mm);
655 entry = set_pte_bit(entry, __pgprot(_PAGE_INVALID));
656 }
657 if (prot == PROT_READ && !pte_p) {
658 ptep_flush_direct(mm, addr, ptep, nodat);
659 entry = clear_pte_bit(entry, __pgprot(_PAGE_INVALID));
660 entry = set_pte_bit(entry, __pgprot(_PAGE_PROTECT));
661 }
662 pgste = set_pgste_bit(pgste, bit);
663 pgste = pgste_set_pte(ptep, pgste, entry);
664 pgste_set_unlock(ptep, pgste);
665 return 0;
666 }
667
ptep_shadow_pte(struct mm_struct * mm,unsigned long saddr,pte_t * sptep,pte_t * tptep,pte_t pte)668 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
669 pte_t *sptep, pte_t *tptep, pte_t pte)
670 {
671 pgste_t spgste, tpgste;
672 pte_t spte, tpte;
673 int rc = -EAGAIN;
674
675 if (!(pte_val(*tptep) & _PAGE_INVALID))
676 return 0; /* already shadowed */
677 spgste = pgste_get_lock(sptep);
678 spte = *sptep;
679 if (!(pte_val(spte) & _PAGE_INVALID) &&
680 !((pte_val(spte) & _PAGE_PROTECT) &&
681 !(pte_val(pte) & _PAGE_PROTECT))) {
682 spgste = set_pgste_bit(spgste, PGSTE_VSIE_BIT);
683 tpgste = pgste_get_lock(tptep);
684 tpte = __pte((pte_val(spte) & PAGE_MASK) |
685 (pte_val(pte) & _PAGE_PROTECT));
686 /* don't touch the storage key - it belongs to parent pgste */
687 tpgste = pgste_set_pte(tptep, tpgste, tpte);
688 pgste_set_unlock(tptep, tpgste);
689 rc = 1;
690 }
691 pgste_set_unlock(sptep, spgste);
692 return rc;
693 }
694
ptep_unshadow_pte(struct mm_struct * mm,unsigned long saddr,pte_t * ptep)695 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
696 {
697 pgste_t pgste;
698 int nodat;
699
700 pgste = pgste_get_lock(ptep);
701 /* notifier is called by the caller */
702 nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
703 ptep_flush_direct(mm, saddr, ptep, nodat);
704 /* don't touch the storage key - it belongs to parent pgste */
705 pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
706 pgste_set_unlock(ptep, pgste);
707 }
708
ptep_zap_swap_entry(struct mm_struct * mm,swp_entry_t entry)709 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
710 {
711 if (!non_swap_entry(entry))
712 dec_mm_counter(mm, MM_SWAPENTS);
713 else if (is_migration_entry(entry)) {
714 struct folio *folio = pfn_swap_entry_folio(entry);
715
716 dec_mm_counter(mm, mm_counter(folio));
717 }
718 free_swap_and_cache(entry);
719 }
720
ptep_zap_unused(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int reset)721 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
722 pte_t *ptep, int reset)
723 {
724 unsigned long pgstev;
725 pgste_t pgste;
726 pte_t pte;
727
728 /* Zap unused and logically-zero pages */
729 preempt_disable();
730 pgste = pgste_get_lock(ptep);
731 pgstev = pgste_val(pgste);
732 pte = *ptep;
733 if (!reset && pte_swap(pte) &&
734 ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
735 (pgstev & _PGSTE_GPS_ZERO))) {
736 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
737 pte_clear(mm, addr, ptep);
738 }
739 if (reset)
740 pgste = clear_pgste_bit(pgste, _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
741 pgste_set_unlock(ptep, pgste);
742 preempt_enable();
743 }
744
ptep_zap_key(struct mm_struct * mm,unsigned long addr,pte_t * ptep)745 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
746 {
747 unsigned long ptev;
748 pgste_t pgste;
749
750 /* Clear storage key ACC and F, but set R/C */
751 preempt_disable();
752 pgste = pgste_get_lock(ptep);
753 pgste = clear_pgste_bit(pgste, PGSTE_ACC_BITS | PGSTE_FP_BIT);
754 pgste = set_pgste_bit(pgste, PGSTE_GR_BIT | PGSTE_GC_BIT);
755 ptev = pte_val(*ptep);
756 if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
757 page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0);
758 pgste_set_unlock(ptep, pgste);
759 preempt_enable();
760 }
761
762 /*
763 * Test and reset if a guest page is dirty
764 */
ptep_test_and_clear_uc(struct mm_struct * mm,unsigned long addr,pte_t * ptep)765 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr,
766 pte_t *ptep)
767 {
768 pgste_t pgste;
769 pte_t pte;
770 bool dirty;
771 int nodat;
772
773 pgste = pgste_get_lock(ptep);
774 dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
775 pgste = clear_pgste_bit(pgste, PGSTE_UC_BIT);
776 pte = *ptep;
777 if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
778 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
779 nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
780 ptep_ipte_global(mm, addr, ptep, nodat);
781 if (machine_has_esop() || !(pte_val(pte) & _PAGE_WRITE))
782 pte = set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
783 else
784 pte = set_pte_bit(pte, __pgprot(_PAGE_INVALID));
785 set_pte(ptep, pte);
786 }
787 pgste_set_unlock(ptep, pgste);
788 return dirty;
789 }
790 EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc);
791
set_guest_storage_key(struct mm_struct * mm,unsigned long addr,unsigned char key,bool nq)792 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
793 unsigned char key, bool nq)
794 {
795 unsigned long keyul, paddr;
796 spinlock_t *ptl;
797 pgste_t old, new;
798 pmd_t *pmdp;
799 pte_t *ptep;
800
801 /*
802 * If we don't have a PTE table and if there is no huge page mapped,
803 * we can ignore attempts to set the key to 0, because it already is 0.
804 */
805 switch (pmd_lookup(mm, addr, &pmdp)) {
806 case -ENOENT:
807 return key ? -EFAULT : 0;
808 case 0:
809 break;
810 default:
811 return -EFAULT;
812 }
813 again:
814 ptl = pmd_lock(mm, pmdp);
815 if (!pmd_present(*pmdp)) {
816 spin_unlock(ptl);
817 return key ? -EFAULT : 0;
818 }
819
820 if (pmd_leaf(*pmdp)) {
821 paddr = pmd_val(*pmdp) & HPAGE_MASK;
822 paddr |= addr & ~HPAGE_MASK;
823 /*
824 * Huge pmds need quiescing operations, they are
825 * always mapped.
826 */
827 page_set_storage_key(paddr, key, 1);
828 spin_unlock(ptl);
829 return 0;
830 }
831 spin_unlock(ptl);
832
833 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
834 if (!ptep)
835 goto again;
836 new = old = pgste_get_lock(ptep);
837 new = clear_pgste_bit(new, PGSTE_GR_BIT | PGSTE_GC_BIT |
838 PGSTE_ACC_BITS | PGSTE_FP_BIT);
839 keyul = (unsigned long) key;
840 new = set_pgste_bit(new, (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48);
841 new = set_pgste_bit(new, (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56);
842 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
843 unsigned long bits, skey;
844
845 paddr = pte_val(*ptep) & PAGE_MASK;
846 skey = (unsigned long) page_get_storage_key(paddr);
847 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
848 skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
849 /* Set storage key ACC and FP */
850 page_set_storage_key(paddr, skey, !nq);
851 /* Merge host changed & referenced into pgste */
852 new = set_pgste_bit(new, bits << 52);
853 }
854 /* changing the guest storage key is considered a change of the page */
855 if ((pgste_val(new) ^ pgste_val(old)) &
856 (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
857 new = set_pgste_bit(new, PGSTE_UC_BIT);
858
859 pgste_set_unlock(ptep, new);
860 pte_unmap_unlock(ptep, ptl);
861 return 0;
862 }
863 EXPORT_SYMBOL(set_guest_storage_key);
864
865 /*
866 * Conditionally set a guest storage key (handling csske).
867 * oldkey will be updated when either mr or mc is set and a pointer is given.
868 *
869 * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
870 * storage key was updated and -EFAULT on access errors.
871 */
cond_set_guest_storage_key(struct mm_struct * mm,unsigned long addr,unsigned char key,unsigned char * oldkey,bool nq,bool mr,bool mc)872 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
873 unsigned char key, unsigned char *oldkey,
874 bool nq, bool mr, bool mc)
875 {
876 unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT;
877 int rc;
878
879 /* we can drop the pgste lock between getting and setting the key */
880 if (mr | mc) {
881 rc = get_guest_storage_key(current->mm, addr, &tmp);
882 if (rc)
883 return rc;
884 if (oldkey)
885 *oldkey = tmp;
886 if (!mr)
887 mask |= _PAGE_REFERENCED;
888 if (!mc)
889 mask |= _PAGE_CHANGED;
890 if (!((tmp ^ key) & mask))
891 return 0;
892 }
893 rc = set_guest_storage_key(current->mm, addr, key, nq);
894 return rc < 0 ? rc : 1;
895 }
896 EXPORT_SYMBOL(cond_set_guest_storage_key);
897
898 /*
899 * Reset a guest reference bit (rrbe), returning the reference and changed bit.
900 *
901 * Returns < 0 in case of error, otherwise the cc to be reported to the guest.
902 */
reset_guest_reference_bit(struct mm_struct * mm,unsigned long addr)903 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
904 {
905 spinlock_t *ptl;
906 unsigned long paddr;
907 pgste_t old, new;
908 pmd_t *pmdp;
909 pte_t *ptep;
910 int cc = 0;
911
912 /*
913 * If we don't have a PTE table and if there is no huge page mapped,
914 * the storage key is 0 and there is nothing for us to do.
915 */
916 switch (pmd_lookup(mm, addr, &pmdp)) {
917 case -ENOENT:
918 return 0;
919 case 0:
920 break;
921 default:
922 return -EFAULT;
923 }
924 again:
925 ptl = pmd_lock(mm, pmdp);
926 if (!pmd_present(*pmdp)) {
927 spin_unlock(ptl);
928 return 0;
929 }
930
931 if (pmd_leaf(*pmdp)) {
932 paddr = pmd_val(*pmdp) & HPAGE_MASK;
933 paddr |= addr & ~HPAGE_MASK;
934 cc = page_reset_referenced(paddr);
935 spin_unlock(ptl);
936 return cc;
937 }
938 spin_unlock(ptl);
939
940 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
941 if (!ptep)
942 goto again;
943 new = old = pgste_get_lock(ptep);
944 /* Reset guest reference bit only */
945 new = clear_pgste_bit(new, PGSTE_GR_BIT);
946
947 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
948 paddr = pte_val(*ptep) & PAGE_MASK;
949 cc = page_reset_referenced(paddr);
950 /* Merge real referenced bit into host-set */
951 new = set_pgste_bit(new, ((unsigned long)cc << 53) & PGSTE_HR_BIT);
952 }
953 /* Reflect guest's logical view, not physical */
954 cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49;
955 /* Changing the guest storage key is considered a change of the page */
956 if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT)
957 new = set_pgste_bit(new, PGSTE_UC_BIT);
958
959 pgste_set_unlock(ptep, new);
960 pte_unmap_unlock(ptep, ptl);
961 return cc;
962 }
963 EXPORT_SYMBOL(reset_guest_reference_bit);
964
get_guest_storage_key(struct mm_struct * mm,unsigned long addr,unsigned char * key)965 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
966 unsigned char *key)
967 {
968 unsigned long paddr;
969 spinlock_t *ptl;
970 pgste_t pgste;
971 pmd_t *pmdp;
972 pte_t *ptep;
973
974 /*
975 * If we don't have a PTE table and if there is no huge page mapped,
976 * the storage key is 0.
977 */
978 *key = 0;
979
980 switch (pmd_lookup(mm, addr, &pmdp)) {
981 case -ENOENT:
982 return 0;
983 case 0:
984 break;
985 default:
986 return -EFAULT;
987 }
988 again:
989 ptl = pmd_lock(mm, pmdp);
990 if (!pmd_present(*pmdp)) {
991 spin_unlock(ptl);
992 return 0;
993 }
994
995 if (pmd_leaf(*pmdp)) {
996 paddr = pmd_val(*pmdp) & HPAGE_MASK;
997 paddr |= addr & ~HPAGE_MASK;
998 *key = page_get_storage_key(paddr);
999 spin_unlock(ptl);
1000 return 0;
1001 }
1002 spin_unlock(ptl);
1003
1004 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
1005 if (!ptep)
1006 goto again;
1007 pgste = pgste_get_lock(ptep);
1008 *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
1009 paddr = pte_val(*ptep) & PAGE_MASK;
1010 if (!(pte_val(*ptep) & _PAGE_INVALID))
1011 *key = page_get_storage_key(paddr);
1012 /* Reflect guest's logical view, not physical */
1013 *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
1014 pgste_set_unlock(ptep, pgste);
1015 pte_unmap_unlock(ptep, ptl);
1016 return 0;
1017 }
1018 EXPORT_SYMBOL(get_guest_storage_key);
1019
1020 /**
1021 * pgste_perform_essa - perform ESSA actions on the PGSTE.
1022 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1023 * @hva: the host virtual address of the page whose PGSTE is to be processed
1024 * @orc: the specific action to perform, see the ESSA_SET_* macros.
1025 * @oldpte: the PTE will be saved there if the pointer is not NULL.
1026 * @oldpgste: the old PGSTE will be saved there if the pointer is not NULL.
1027 *
1028 * Return: 1 if the page is to be added to the CBRL, otherwise 0,
1029 * or < 0 in case of error. -EINVAL is returned for invalid values
1030 * of orc, -EFAULT for invalid addresses.
1031 */
pgste_perform_essa(struct mm_struct * mm,unsigned long hva,int orc,unsigned long * oldpte,unsigned long * oldpgste)1032 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1033 unsigned long *oldpte, unsigned long *oldpgste)
1034 {
1035 struct vm_area_struct *vma;
1036 unsigned long pgstev;
1037 spinlock_t *ptl;
1038 pgste_t pgste;
1039 pte_t *ptep;
1040 int res = 0;
1041
1042 WARN_ON_ONCE(orc > ESSA_MAX);
1043 if (unlikely(orc > ESSA_MAX))
1044 return -EINVAL;
1045
1046 vma = vma_lookup(mm, hva);
1047 if (!vma || is_vm_hugetlb_page(vma))
1048 return -EFAULT;
1049 ptep = get_locked_pte(mm, hva, &ptl);
1050 if (unlikely(!ptep))
1051 return -EFAULT;
1052 pgste = pgste_get_lock(ptep);
1053 pgstev = pgste_val(pgste);
1054 if (oldpte)
1055 *oldpte = pte_val(*ptep);
1056 if (oldpgste)
1057 *oldpgste = pgstev;
1058
1059 switch (orc) {
1060 case ESSA_GET_STATE:
1061 break;
1062 case ESSA_SET_STABLE:
1063 pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
1064 pgstev |= _PGSTE_GPS_USAGE_STABLE;
1065 break;
1066 case ESSA_SET_UNUSED:
1067 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1068 pgstev |= _PGSTE_GPS_USAGE_UNUSED;
1069 if (pte_val(*ptep) & _PAGE_INVALID)
1070 res = 1;
1071 break;
1072 case ESSA_SET_VOLATILE:
1073 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1074 pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
1075 if (pte_val(*ptep) & _PAGE_INVALID)
1076 res = 1;
1077 break;
1078 case ESSA_SET_POT_VOLATILE:
1079 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1080 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1081 pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE;
1082 break;
1083 }
1084 if (pgstev & _PGSTE_GPS_ZERO) {
1085 pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
1086 break;
1087 }
1088 if (!(pgstev & PGSTE_GC_BIT)) {
1089 pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
1090 res = 1;
1091 break;
1092 }
1093 break;
1094 case ESSA_SET_STABLE_RESIDENT:
1095 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1096 pgstev |= _PGSTE_GPS_USAGE_STABLE;
1097 /*
1098 * Since the resident state can go away any time after this
1099 * call, we will not make this page resident. We can revisit
1100 * this decision if a guest will ever start using this.
1101 */
1102 break;
1103 case ESSA_SET_STABLE_IF_RESIDENT:
1104 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1105 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1106 pgstev |= _PGSTE_GPS_USAGE_STABLE;
1107 }
1108 break;
1109 case ESSA_SET_STABLE_NODAT:
1110 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1111 pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT;
1112 break;
1113 default:
1114 /* we should never get here! */
1115 break;
1116 }
1117 /* If we are discarding a page, set it to logical zero */
1118 if (res)
1119 pgstev |= _PGSTE_GPS_ZERO;
1120
1121 pgste = __pgste(pgstev);
1122 pgste_set_unlock(ptep, pgste);
1123 pte_unmap_unlock(ptep, ptl);
1124 return res;
1125 }
1126 EXPORT_SYMBOL(pgste_perform_essa);
1127
1128 /**
1129 * set_pgste_bits - set specific PGSTE bits.
1130 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1131 * @hva: the host virtual address of the page whose PGSTE is to be processed
1132 * @bits: a bitmask representing the bits that will be touched
1133 * @value: the values of the bits to be written. Only the bits in the mask
1134 * will be written.
1135 *
1136 * Return: 0 on success, < 0 in case of error.
1137 */
set_pgste_bits(struct mm_struct * mm,unsigned long hva,unsigned long bits,unsigned long value)1138 int set_pgste_bits(struct mm_struct *mm, unsigned long hva,
1139 unsigned long bits, unsigned long value)
1140 {
1141 struct vm_area_struct *vma;
1142 spinlock_t *ptl;
1143 pgste_t new;
1144 pte_t *ptep;
1145
1146 vma = vma_lookup(mm, hva);
1147 if (!vma || is_vm_hugetlb_page(vma))
1148 return -EFAULT;
1149 ptep = get_locked_pte(mm, hva, &ptl);
1150 if (unlikely(!ptep))
1151 return -EFAULT;
1152 new = pgste_get_lock(ptep);
1153
1154 new = clear_pgste_bit(new, bits);
1155 new = set_pgste_bit(new, value & bits);
1156
1157 pgste_set_unlock(ptep, new);
1158 pte_unmap_unlock(ptep, ptl);
1159 return 0;
1160 }
1161 EXPORT_SYMBOL(set_pgste_bits);
1162
1163 /**
1164 * get_pgste - get the current PGSTE for the given address.
1165 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1166 * @hva: the host virtual address of the page whose PGSTE is to be processed
1167 * @pgstep: will be written with the current PGSTE for the given address.
1168 *
1169 * Return: 0 on success, < 0 in case of error.
1170 */
get_pgste(struct mm_struct * mm,unsigned long hva,unsigned long * pgstep)1171 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep)
1172 {
1173 struct vm_area_struct *vma;
1174 spinlock_t *ptl;
1175 pte_t *ptep;
1176
1177 vma = vma_lookup(mm, hva);
1178 if (!vma || is_vm_hugetlb_page(vma))
1179 return -EFAULT;
1180 ptep = get_locked_pte(mm, hva, &ptl);
1181 if (unlikely(!ptep))
1182 return -EFAULT;
1183 *pgstep = pgste_val(pgste_get(ptep));
1184 pte_unmap_unlock(ptep, ptl);
1185 return 0;
1186 }
1187 EXPORT_SYMBOL(get_pgste);
1188 #endif
1189