Lines Matching +full:mm +full:- +full:0
1 // SPDX-License-Identifier: GPL-2.0
12 #include <linux/mm.h>
26 #include <asm/page-states.h>
39 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, in ptep_ipte_local() argument
45 opt = 0; in ptep_ipte_local()
46 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_local()
47 if (asce == 0UL || nodat) in ptep_ipte_local()
49 if (asce != -1UL) { in ptep_ipte_local()
50 asce = asce ? : mm->context.asce; in ptep_ipte_local()
55 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local()
59 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, in ptep_ipte_global() argument
65 opt = 0; in ptep_ipte_global()
66 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_global()
67 if (asce == 0UL || nodat) in ptep_ipte_global()
69 if (asce != -1UL) { in ptep_ipte_global()
70 asce = asce ? : mm->context.asce; in ptep_ipte_global()
75 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global()
79 static inline pte_t ptep_flush_direct(struct mm_struct *mm, in ptep_flush_direct() argument
88 atomic_inc(&mm->context.flush_count); in ptep_flush_direct()
90 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in ptep_flush_direct()
91 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
93 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
94 atomic_dec(&mm->context.flush_count); in ptep_flush_direct()
98 static inline pte_t ptep_flush_lazy(struct mm_struct *mm, in ptep_flush_lazy() argument
107 atomic_inc(&mm->context.flush_count); in ptep_flush_lazy()
108 if (cpumask_equal(&mm->context.cpu_attach_mask, in ptep_flush_lazy()
111 mm->context.flush_mm = 1; in ptep_flush_lazy()
113 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_lazy()
114 atomic_dec(&mm->context.flush_count); in ptep_flush_lazy()
120 unsigned long value = 0; in pgste_get_lock()
142 unsigned long pgste = 0; in pgste_get()
157 struct mm_struct *mm) in pgste_update_all() argument
162 if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID) in pgste_update_all()
178 struct mm_struct *mm) in pgste_set_key() argument
184 if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID) in pgste_set_key()
191 * key C/R to 0. in pgste_set_key()
195 page_set_storage_key(address, nkey, 0); in pgste_set_key()
207 * Without enhanced suppression-on-protection force in pgste_set_pte()
214 /* This pte allows write access, set user-dirty */ in pgste_set_pte()
222 static inline pgste_t pgste_pte_notify(struct mm_struct *mm, in pgste_pte_notify() argument
232 ptep_notify(mm, addr, ptep, bits); in pgste_pte_notify()
238 static inline pgste_t ptep_xchg_start(struct mm_struct *mm, in ptep_xchg_start() argument
241 pgste_t pgste = __pgste(0); in ptep_xchg_start()
243 if (mm_has_pgste(mm)) { in ptep_xchg_start()
245 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_xchg_start()
250 static inline pte_t ptep_xchg_commit(struct mm_struct *mm, in ptep_xchg_commit() argument
254 if (mm_has_pgste(mm)) { in ptep_xchg_commit()
256 pgste_set_key(ptep, pgste, new, mm); in ptep_xchg_commit()
258 pgste = pgste_update_all(old, pgste, mm); in ptep_xchg_commit()
271 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, in ptep_xchg_direct() argument
279 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_direct()
281 old = ptep_flush_direct(mm, addr, ptep, nodat); in ptep_xchg_direct()
282 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_direct()
292 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep, in ptep_reset_dat_prot() argument
296 atomic_inc(&mm->context.flush_count); in ptep_reset_dat_prot()
297 if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in ptep_reset_dat_prot()
298 __ptep_rdp(addr, ptep, 0, 0, 1); in ptep_reset_dat_prot()
300 __ptep_rdp(addr, ptep, 0, 0, 0); in ptep_reset_dat_prot()
308 atomic_dec(&mm->context.flush_count); in ptep_reset_dat_prot()
313 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, in ptep_xchg_lazy() argument
321 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_lazy()
323 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_xchg_lazy()
324 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_lazy()
336 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_start() local
339 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_modify_prot_start()
341 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_modify_prot_start()
342 if (mm_has_pgste(mm)) { in ptep_modify_prot_start()
343 pgste = pgste_update_all(old, pgste, mm); in ptep_modify_prot_start()
353 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_commit() local
355 if (mm_has_pgste(mm)) { in ptep_modify_prot_commit()
357 pgste_set_key(ptep, pgste, pte, mm); in ptep_modify_prot_commit()
366 static inline void pmdp_idte_local(struct mm_struct *mm, in pmdp_idte_local() argument
371 mm->context.asce, IDTE_LOCAL); in pmdp_idte_local()
373 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); in pmdp_idte_local()
374 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_local()
375 gmap_pmdp_idte_local(mm, addr); in pmdp_idte_local()
378 static inline void pmdp_idte_global(struct mm_struct *mm, in pmdp_idte_global() argument
383 mm->context.asce, IDTE_GLOBAL); in pmdp_idte_global()
384 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
385 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
387 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); in pmdp_idte_global()
388 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
389 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
392 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
393 gmap_pmdp_csp(mm, addr); in pmdp_idte_global()
397 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, in pmdp_flush_direct() argument
405 atomic_inc(&mm->context.flush_count); in pmdp_flush_direct()
407 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pmdp_flush_direct()
408 pmdp_idte_local(mm, addr, pmdp); in pmdp_flush_direct()
410 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_direct()
411 atomic_dec(&mm->context.flush_count); in pmdp_flush_direct()
415 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, in pmdp_flush_lazy() argument
423 atomic_inc(&mm->context.flush_count); in pmdp_flush_lazy()
424 if (cpumask_equal(&mm->context.cpu_attach_mask, in pmdp_flush_lazy()
427 mm->context.flush_mm = 1; in pmdp_flush_lazy()
428 if (mm_has_pgste(mm)) in pmdp_flush_lazy()
429 gmap_pmdp_invalidate(mm, addr); in pmdp_flush_lazy()
431 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_lazy()
433 atomic_dec(&mm->context.flush_count); in pmdp_flush_lazy()
438 static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp) in pmd_lookup() argument
446 vma = vma_lookup(mm, addr); in pmd_lookup()
448 return -EFAULT; in pmd_lookup()
450 pgd = pgd_offset(mm, addr); in pmd_lookup()
452 return -ENOENT; in pmd_lookup()
456 return -ENOENT; in pmd_lookup()
460 return -ENOENT; in pmd_lookup()
464 return -EFAULT; in pmd_lookup()
467 return 0; in pmd_lookup()
471 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_direct() argument
477 old = pmdp_flush_direct(mm, addr, pmdp); in pmdp_xchg_direct()
484 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_lazy() argument
490 old = pmdp_flush_lazy(mm, addr, pmdp); in pmdp_xchg_lazy()
497 static inline void pudp_idte_local(struct mm_struct *mm, in pudp_idte_local() argument
502 mm->context.asce, IDTE_LOCAL); in pudp_idte_local()
504 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); in pudp_idte_local()
507 static inline void pudp_idte_global(struct mm_struct *mm, in pudp_idte_global() argument
512 mm->context.asce, IDTE_GLOBAL); in pudp_idte_global()
514 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); in pudp_idte_global()
523 static inline pud_t pudp_flush_direct(struct mm_struct *mm, in pudp_flush_direct() argument
531 atomic_inc(&mm->context.flush_count); in pudp_flush_direct()
533 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pudp_flush_direct()
534 pudp_idte_local(mm, addr, pudp); in pudp_flush_direct()
536 pudp_idte_global(mm, addr, pudp); in pudp_flush_direct()
537 atomic_dec(&mm->context.flush_count); in pudp_flush_direct()
541 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pudp_xchg_direct() argument
547 old = pudp_flush_direct(mm, addr, pudp); in pudp_xchg_direct()
555 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
560 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_deposit()
563 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
566 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); in pgtable_trans_huge_deposit()
567 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
570 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) in pgtable_trans_huge_withdraw() argument
576 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_withdraw()
579 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
582 pmd_huge_pte(mm, pmdp) = NULL; in pgtable_trans_huge_withdraw()
584 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; in pgtable_trans_huge_withdraw()
596 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, in ptep_set_pte_at() argument
605 pgste_set_key(ptep, pgste, entry, mm); in ptep_set_pte_at()
611 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_set_notify() argument
623 * ptep_force_prot - change access rights of a locked pte
624 * @mm: pointer to the process mm_struct
630 * Returns 0 if the access rights were changed and -EAGAIN if the current
633 int ptep_force_prot(struct mm_struct *mm, unsigned long addr, in ptep_force_prot() argument
648 return -EAGAIN; in ptep_force_prot()
653 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
654 pgste = pgste_update_all(entry, pgste, mm); in ptep_force_prot()
658 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
665 return 0; in ptep_force_prot()
668 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, in ptep_shadow_pte() argument
673 int rc = -EAGAIN; in ptep_shadow_pte()
676 return 0; /* already shadowed */ in ptep_shadow_pte()
686 /* don't touch the storage key - it belongs to parent pgste */ in ptep_shadow_pte()
695 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) in ptep_unshadow_pte() argument
703 ptep_flush_direct(mm, saddr, ptep, nodat); in ptep_unshadow_pte()
704 /* don't touch the storage key - it belongs to parent pgste */ in ptep_unshadow_pte()
709 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) in ptep_zap_swap_entry() argument
712 dec_mm_counter(mm, MM_SWAPENTS); in ptep_zap_swap_entry()
716 dec_mm_counter(mm, mm_counter(folio)); in ptep_zap_swap_entry()
721 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, in ptep_zap_unused() argument
728 /* Zap unused and logically-zero pages */ in ptep_zap_unused()
736 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); in ptep_zap_unused()
737 pte_clear(mm, addr, ptep); in ptep_zap_unused()
745 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_zap_key() argument
757 page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0); in ptep_zap_key()
765 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr, in ptep_test_and_clear_uc() argument
778 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_test_and_clear_uc()
780 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_test_and_clear_uc()
792 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in set_guest_storage_key() argument
803 * we can ignore attempts to set the key to 0, because it already is 0. in set_guest_storage_key()
805 switch (pmd_lookup(mm, addr, &pmdp)) { in set_guest_storage_key()
806 case -ENOENT: in set_guest_storage_key()
807 return key ? -EFAULT : 0; in set_guest_storage_key()
808 case 0: in set_guest_storage_key()
811 return -EFAULT; in set_guest_storage_key()
814 ptl = pmd_lock(mm, pmdp); in set_guest_storage_key()
817 return key ? -EFAULT : 0; in set_guest_storage_key()
829 return 0; in set_guest_storage_key()
833 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in set_guest_storage_key()
861 return 0; in set_guest_storage_key()
869 * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
870 * storage key was updated and -EFAULT on access errors.
872 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in cond_set_guest_storage_key() argument
881 rc = get_guest_storage_key(current->mm, addr, &tmp); in cond_set_guest_storage_key()
891 return 0; in cond_set_guest_storage_key()
893 rc = set_guest_storage_key(current->mm, addr, key, nq); in cond_set_guest_storage_key()
894 return rc < 0 ? rc : 1; in cond_set_guest_storage_key()
901 * Returns < 0 in case of error, otherwise the cc to be reported to the guest.
903 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) in reset_guest_reference_bit() argument
910 int cc = 0; in reset_guest_reference_bit()
914 * the storage key is 0 and there is nothing for us to do. in reset_guest_reference_bit()
916 switch (pmd_lookup(mm, addr, &pmdp)) { in reset_guest_reference_bit()
917 case -ENOENT: in reset_guest_reference_bit()
918 return 0; in reset_guest_reference_bit()
919 case 0: in reset_guest_reference_bit()
922 return -EFAULT; in reset_guest_reference_bit()
925 ptl = pmd_lock(mm, pmdp); in reset_guest_reference_bit()
928 return 0; in reset_guest_reference_bit()
940 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in reset_guest_reference_bit()
950 /* Merge real referenced bit into host-set */ in reset_guest_reference_bit()
965 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, in get_guest_storage_key() argument
976 * the storage key is 0. in get_guest_storage_key()
978 *key = 0; in get_guest_storage_key()
980 switch (pmd_lookup(mm, addr, &pmdp)) { in get_guest_storage_key()
981 case -ENOENT: in get_guest_storage_key()
982 return 0; in get_guest_storage_key()
983 case 0: in get_guest_storage_key()
986 return -EFAULT; in get_guest_storage_key()
989 ptl = pmd_lock(mm, pmdp); in get_guest_storage_key()
992 return 0; in get_guest_storage_key()
1000 return 0; in get_guest_storage_key()
1004 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in get_guest_storage_key()
1016 return 0; in get_guest_storage_key()
1021 * pgste_perform_essa - perform ESSA actions on the PGSTE.
1022 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1028 * Return: 1 if the page is to be added to the CBRL, otherwise 0,
1029 * or < 0 in case of error. -EINVAL is returned for invalid values
1030 * of orc, -EFAULT for invalid addresses.
1032 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, in pgste_perform_essa() argument
1040 int res = 0; in pgste_perform_essa()
1044 return -EINVAL; in pgste_perform_essa()
1046 vma = vma_lookup(mm, hva); in pgste_perform_essa()
1048 return -EFAULT; in pgste_perform_essa()
1049 ptep = get_locked_pte(mm, hva, &ptl); in pgste_perform_essa()
1051 return -EFAULT; in pgste_perform_essa()
1129 * set_pgste_bits - set specific PGSTE bits.
1130 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1136 * Return: 0 on success, < 0 in case of error.
1138 int set_pgste_bits(struct mm_struct *mm, unsigned long hva, in set_pgste_bits() argument
1146 vma = vma_lookup(mm, hva); in set_pgste_bits()
1148 return -EFAULT; in set_pgste_bits()
1149 ptep = get_locked_pte(mm, hva, &ptl); in set_pgste_bits()
1151 return -EFAULT; in set_pgste_bits()
1159 return 0; in set_pgste_bits()
1164 * get_pgste - get the current PGSTE for the given address.
1165 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1169 * Return: 0 on success, < 0 in case of error.
1171 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) in get_pgste() argument
1177 vma = vma_lookup(mm, hva); in get_pgste()
1179 return -EFAULT; in get_pgste()
1180 ptep = get_locked_pte(mm, hva, &ptl); in get_pgste()
1182 return -EFAULT; in get_pgste()
1185 return 0; in get_pgste()