1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Helper functions for KVM guest address space mapping code
4 *
5 * Copyright IBM Corp. 2007, 2025
6 */
7
8 #include <linux/export.h>
9 #include <linux/mm_types.h>
10 #include <linux/mmap_lock.h>
11 #include <linux/mm.h>
12 #include <linux/hugetlb.h>
13 #include <linux/swap.h>
14 #include <linux/swapops.h>
15 #include <linux/pagewalk.h>
16 #include <linux/ksm.h>
17 #include <asm/gmap_helpers.h>
18 #include <asm/pgtable.h>
19
20 /**
21 * ptep_zap_swap_entry() - discard a swap entry.
22 * @mm: the mm
23 * @entry: the swap entry that needs to be zapped
24 *
25 * Discards the given swap entry. If the swap entry was an actual swap
26 * entry (and not a migration entry, for example), the actual swapped
27 * page is also discarded from swap.
28 */
ptep_zap_swap_entry(struct mm_struct * mm,swp_entry_t entry)29 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
30 {
31 if (!non_swap_entry(entry))
32 dec_mm_counter(mm, MM_SWAPENTS);
33 else if (is_migration_entry(entry))
34 dec_mm_counter(mm, mm_counter(pfn_swap_entry_folio(entry)));
35 free_swap_and_cache(entry);
36 }
37
38 /**
39 * gmap_helper_zap_one_page() - discard a page if it was swapped.
40 * @mm: the mm
41 * @vmaddr: the userspace virtual address that needs to be discarded
42 *
43 * If the given address maps to a swap entry, discard it.
44 *
45 * Context: needs to be called while holding the mmap lock.
46 */
gmap_helper_zap_one_page(struct mm_struct * mm,unsigned long vmaddr)47 void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr)
48 {
49 struct vm_area_struct *vma;
50 spinlock_t *ptl;
51 pgste_t pgste;
52 pte_t *ptep;
53
54 mmap_assert_locked(mm);
55
56 /* Find the vm address for the guest address */
57 vma = vma_lookup(mm, vmaddr);
58 if (!vma || is_vm_hugetlb_page(vma))
59 return;
60
61 /* Get pointer to the page table entry */
62 ptep = get_locked_pte(mm, vmaddr, &ptl);
63 if (unlikely(!ptep))
64 return;
65 if (pte_swap(*ptep)) {
66 preempt_disable();
67 pgste = pgste_get_lock(ptep);
68
69 ptep_zap_swap_entry(mm, pte_to_swp_entry(*ptep));
70 pte_clear(mm, vmaddr, ptep);
71
72 pgste_set_unlock(ptep, pgste);
73 preempt_enable();
74 }
75 pte_unmap_unlock(ptep, ptl);
76 }
77 EXPORT_SYMBOL_GPL(gmap_helper_zap_one_page);
78
79 /**
80 * gmap_helper_discard() - discard user pages in the given range
81 * @mm: the mm
82 * @vmaddr: starting userspace address
83 * @end: end address (first address outside the range)
84 *
85 * All userpace pages in the range [@vamddr, @end) are discarded and unmapped.
86 *
87 * Context: needs to be called while holding the mmap lock.
88 */
gmap_helper_discard(struct mm_struct * mm,unsigned long vmaddr,unsigned long end)89 void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end)
90 {
91 struct vm_area_struct *vma;
92
93 mmap_assert_locked(mm);
94
95 while (vmaddr < end) {
96 vma = find_vma_intersection(mm, vmaddr, end);
97 if (!vma)
98 return;
99 if (!is_vm_hugetlb_page(vma))
100 zap_page_range_single(vma, vmaddr, min(end, vma->vm_end) - vmaddr, NULL);
101 vmaddr = vma->vm_end;
102 }
103 }
104 EXPORT_SYMBOL_GPL(gmap_helper_discard);
105
find_zeropage_pte_entry(pte_t * pte,unsigned long addr,unsigned long end,struct mm_walk * walk)106 static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr,
107 unsigned long end, struct mm_walk *walk)
108 {
109 unsigned long *found_addr = walk->private;
110
111 /* Return 1 of the page is a zeropage. */
112 if (is_zero_pfn(pte_pfn(*pte))) {
113 /*
114 * Shared zeropage in e.g., a FS DAX mapping? We cannot do the
115 * right thing and likely don't care: FAULT_FLAG_UNSHARE
116 * currently only works in COW mappings, which is also where
117 * mm_forbids_zeropage() is checked.
118 */
119 if (!is_cow_mapping(walk->vma->vm_flags))
120 return -EFAULT;
121
122 *found_addr = addr;
123 return 1;
124 }
125 return 0;
126 }
127
128 static const struct mm_walk_ops find_zeropage_ops = {
129 .pte_entry = find_zeropage_pte_entry,
130 .walk_lock = PGWALK_WRLOCK,
131 };
132
133 /** __gmap_helper_unshare_zeropages() - unshare all shared zeropages
134 * @mm: the mm whose zero pages are to be unshared
135 *
136 * Unshare all shared zeropages, replacing them by anonymous pages. Note that
137 * we cannot simply zap all shared zeropages, because this could later
138 * trigger unexpected userfaultfd missing events.
139 *
140 * This must be called after mm->context.allow_cow_sharing was
141 * set to 0, to avoid future mappings of shared zeropages.
142 *
143 * mm contracts with s390, that even if mm were to remove a page table,
144 * and racing with walk_page_range_vma() calling pte_offset_map_lock()
145 * would fail, it will never insert a page table containing empty zero
146 * pages once mm_forbids_zeropage(mm) i.e.
147 * mm->context.allow_cow_sharing is set to 0.
148 */
__gmap_helper_unshare_zeropages(struct mm_struct * mm)149 static int __gmap_helper_unshare_zeropages(struct mm_struct *mm)
150 {
151 struct vm_area_struct *vma;
152 VMA_ITERATOR(vmi, mm, 0);
153 unsigned long addr;
154 vm_fault_t fault;
155 int rc;
156
157 for_each_vma(vmi, vma) {
158 /*
159 * We could only look at COW mappings, but it's more future
160 * proof to catch unexpected zeropages in other mappings and
161 * fail.
162 */
163 if ((vma->vm_flags & VM_PFNMAP) || is_vm_hugetlb_page(vma))
164 continue;
165 addr = vma->vm_start;
166
167 retry:
168 rc = walk_page_range_vma(vma, addr, vma->vm_end,
169 &find_zeropage_ops, &addr);
170 if (rc < 0)
171 return rc;
172 else if (!rc)
173 continue;
174
175 /* addr was updated by find_zeropage_pte_entry() */
176 fault = handle_mm_fault(vma, addr,
177 FAULT_FLAG_UNSHARE | FAULT_FLAG_REMOTE,
178 NULL);
179 if (fault & VM_FAULT_OOM)
180 return -ENOMEM;
181 /*
182 * See break_ksm(): even after handle_mm_fault() returned 0, we
183 * must start the lookup from the current address, because
184 * handle_mm_fault() may back out if there's any difficulty.
185 *
186 * VM_FAULT_SIGBUS and VM_FAULT_SIGSEGV are unexpected but
187 * maybe they could trigger in the future on concurrent
188 * truncation. In that case, the shared zeropage would be gone
189 * and we can simply retry and make progress.
190 */
191 cond_resched();
192 goto retry;
193 }
194
195 return 0;
196 }
197
198 /**
199 * gmap_helper_disable_cow_sharing() - disable all COW sharing
200 *
201 * Disable most COW-sharing of memory pages for the whole process:
202 * (1) Disable KSM and unmerge/unshare any KSM pages.
203 * (2) Disallow shared zeropages and unshare any zerpages that are mapped.
204 *
205 * Not that we currently don't bother with COW-shared pages that are shared
206 * with parent/child processes due to fork().
207 */
gmap_helper_disable_cow_sharing(void)208 int gmap_helper_disable_cow_sharing(void)
209 {
210 struct mm_struct *mm = current->mm;
211 int rc;
212
213 mmap_assert_write_locked(mm);
214
215 if (!mm->context.allow_cow_sharing)
216 return 0;
217
218 mm->context.allow_cow_sharing = 0;
219
220 /* Replace all shared zeropages by anonymous pages. */
221 rc = __gmap_helper_unshare_zeropages(mm);
222 /*
223 * Make sure to disable KSM (if enabled for the whole process or
224 * individual VMAs). Note that nothing currently hinders user space
225 * from re-enabling it.
226 */
227 if (!rc)
228 rc = ksm_disable(mm);
229 if (rc)
230 mm->context.allow_cow_sharing = 1;
231 return rc;
232 }
233 EXPORT_SYMBOL_GPL(gmap_helper_disable_cow_sharing);
234