1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #define _GNU_SOURCE
4 #include <stdbool.h>
5 #include <stdint.h>
6 #include <fcntl.h>
7 #include <assert.h>
8 #include <linux/mman.h>
9 #include <sys/mman.h>
10 #include "../kselftest.h"
11 #include "thp_settings.h"
12 #include "uffd-common.h"
13
14 static int pagemap_fd;
15 static size_t pagesize;
16 static int nr_pagesizes = 1;
17 static int nr_thpsizes;
18 static size_t thpsizes[20];
19 static int nr_hugetlbsizes;
20 static size_t hugetlbsizes[10];
21
sz2ord(size_t size)22 static int sz2ord(size_t size)
23 {
24 return __builtin_ctzll(size / pagesize);
25 }
26
detect_thp_sizes(size_t sizes[],int max)27 static int detect_thp_sizes(size_t sizes[], int max)
28 {
29 int count = 0;
30 unsigned long orders;
31 size_t kb;
32 int i;
33
34 /* thp not supported at all. */
35 if (!read_pmd_pagesize())
36 return 0;
37
38 orders = thp_supported_orders();
39
40 for (i = 0; orders && count < max; i++) {
41 if (!(orders & (1UL << i)))
42 continue;
43 orders &= ~(1UL << i);
44 kb = (pagesize >> 10) << i;
45 sizes[count++] = kb * 1024;
46 ksft_print_msg("[INFO] detected THP size: %zu KiB\n", kb);
47 }
48
49 return count;
50 }
51
mmap_aligned(size_t size,int prot,int flags)52 static void *mmap_aligned(size_t size, int prot, int flags)
53 {
54 size_t mmap_size = size * 2;
55 char *mmap_mem, *mem;
56
57 mmap_mem = mmap(NULL, mmap_size, prot, flags, -1, 0);
58 if (mmap_mem == MAP_FAILED)
59 return mmap_mem;
60
61 mem = (char *)(((uintptr_t)mmap_mem + size - 1) & ~(size - 1));
62 munmap(mmap_mem, mem - mmap_mem);
63 munmap(mem + size, mmap_mem + mmap_size - mem - size);
64
65 return mem;
66 }
67
alloc_one_folio(size_t size,bool private,bool hugetlb)68 static void *alloc_one_folio(size_t size, bool private, bool hugetlb)
69 {
70 bool thp = !hugetlb && size > pagesize;
71 int flags = MAP_ANONYMOUS;
72 int prot = PROT_READ | PROT_WRITE;
73 char *mem, *addr;
74
75 assert((size & (size - 1)) == 0);
76
77 if (private)
78 flags |= MAP_PRIVATE;
79 else
80 flags |= MAP_SHARED;
81
82 /*
83 * For THP, we must explicitly enable the THP size, allocate twice the
84 * required space then manually align.
85 */
86 if (thp) {
87 struct thp_settings settings = *thp_current_settings();
88
89 if (private)
90 settings.hugepages[sz2ord(size)].enabled = THP_ALWAYS;
91 else
92 settings.shmem_hugepages[sz2ord(size)].enabled = SHMEM_ALWAYS;
93
94 thp_push_settings(&settings);
95
96 mem = mmap_aligned(size, prot, flags);
97 } else {
98 if (hugetlb) {
99 flags |= MAP_HUGETLB;
100 flags |= __builtin_ctzll(size) << MAP_HUGE_SHIFT;
101 }
102
103 mem = mmap(NULL, size, prot, flags, -1, 0);
104 }
105
106 if (mem == MAP_FAILED) {
107 mem = NULL;
108 goto out;
109 }
110
111 assert(((uintptr_t)mem & (size - 1)) == 0);
112
113 /*
114 * Populate the folio by writing the first byte and check that all pages
115 * are populated. Finally set the whole thing to non-zero data to avoid
116 * kernel from mapping it back to the zero page.
117 */
118 mem[0] = 1;
119 for (addr = mem; addr < mem + size; addr += pagesize) {
120 if (!pagemap_is_populated(pagemap_fd, addr)) {
121 munmap(mem, size);
122 mem = NULL;
123 goto out;
124 }
125 }
126 memset(mem, 1, size);
127 out:
128 if (thp)
129 thp_pop_settings();
130
131 return mem;
132 }
133
check_uffd_wp_state(void * mem,size_t size,bool expect)134 static bool check_uffd_wp_state(void *mem, size_t size, bool expect)
135 {
136 uint64_t pte;
137 void *addr;
138
139 for (addr = mem; addr < mem + size; addr += pagesize) {
140 pte = pagemap_get_entry(pagemap_fd, addr);
141 if (!!(pte & PM_UFFD_WP) != expect) {
142 ksft_test_result_fail("uffd-wp not %s for pte %lu!\n",
143 expect ? "set" : "clear",
144 (addr - mem) / pagesize);
145 return false;
146 }
147 }
148
149 return true;
150 }
151
range_is_swapped(void * addr,size_t size)152 static bool range_is_swapped(void *addr, size_t size)
153 {
154 for (; size; addr += pagesize, size -= pagesize)
155 if (!pagemap_is_swapped(pagemap_fd, addr))
156 return false;
157 return true;
158 }
159
test_one_folio(size_t size,bool private,bool swapout,bool hugetlb)160 static void test_one_folio(size_t size, bool private, bool swapout, bool hugetlb)
161 {
162 struct uffdio_writeprotect wp_prms;
163 uint64_t features = 0;
164 void *addr = NULL;
165 void *mem = NULL;
166
167 assert(!(hugetlb && swapout));
168
169 ksft_print_msg("[RUN] %s(size=%zu, private=%s, swapout=%s, hugetlb=%s)\n",
170 __func__,
171 size,
172 private ? "true" : "false",
173 swapout ? "true" : "false",
174 hugetlb ? "true" : "false");
175
176 /* Allocate a folio of required size and type. */
177 mem = alloc_one_folio(size, private, hugetlb);
178 if (!mem) {
179 ksft_test_result_fail("alloc_one_folio() failed\n");
180 goto out;
181 }
182
183 /* Register range for uffd-wp. */
184 if (userfaultfd_open(&features)) {
185 if (errno == ENOENT)
186 ksft_test_result_skip("userfaultfd not available\n");
187 else
188 ksft_test_result_fail("userfaultfd_open() failed\n");
189 goto out;
190 }
191 if (uffd_register(uffd, mem, size, false, true, false)) {
192 ksft_test_result_fail("uffd_register() failed\n");
193 goto out;
194 }
195 wp_prms.mode = UFFDIO_WRITEPROTECT_MODE_WP;
196 wp_prms.range.start = (uintptr_t)mem;
197 wp_prms.range.len = size;
198 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp_prms)) {
199 ksft_test_result_fail("ioctl(UFFDIO_WRITEPROTECT) failed\n");
200 goto out;
201 }
202
203 if (swapout) {
204 madvise(mem, size, MADV_PAGEOUT);
205 if (!range_is_swapped(mem, size)) {
206 ksft_test_result_skip("MADV_PAGEOUT did not work, is swap enabled?\n");
207 goto out;
208 }
209 }
210
211 /* Check that uffd-wp is set for all PTEs in range. */
212 if (!check_uffd_wp_state(mem, size, true))
213 goto out;
214
215 /*
216 * Move the mapping to a new, aligned location. Since
217 * UFFD_FEATURE_EVENT_REMAP is not set, we expect the uffd-wp bit for
218 * each PTE to be cleared in the new mapping.
219 */
220 addr = mmap_aligned(size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS);
221 if (addr == MAP_FAILED) {
222 ksft_test_result_fail("mmap_aligned() failed\n");
223 goto out;
224 }
225 if (mremap(mem, size, size, MREMAP_FIXED | MREMAP_MAYMOVE, addr) == MAP_FAILED) {
226 ksft_test_result_fail("mremap() failed\n");
227 munmap(addr, size);
228 goto out;
229 }
230 mem = addr;
231
232 /* Check that uffd-wp is cleared for all PTEs in range. */
233 if (!check_uffd_wp_state(mem, size, false))
234 goto out;
235
236 ksft_test_result_pass("%s(size=%zu, private=%s, swapout=%s, hugetlb=%s)\n",
237 __func__,
238 size,
239 private ? "true" : "false",
240 swapout ? "true" : "false",
241 hugetlb ? "true" : "false");
242 out:
243 if (mem)
244 munmap(mem, size);
245 if (uffd >= 0) {
246 close(uffd);
247 uffd = -1;
248 }
249 }
250
251 struct testcase {
252 size_t *sizes;
253 int *nr_sizes;
254 bool private;
255 bool swapout;
256 bool hugetlb;
257 };
258
259 static const struct testcase testcases[] = {
260 /* base pages. */
261 {
262 .sizes = &pagesize,
263 .nr_sizes = &nr_pagesizes,
264 .private = false,
265 .swapout = false,
266 .hugetlb = false,
267 },
268 {
269 .sizes = &pagesize,
270 .nr_sizes = &nr_pagesizes,
271 .private = true,
272 .swapout = false,
273 .hugetlb = false,
274 },
275 {
276 .sizes = &pagesize,
277 .nr_sizes = &nr_pagesizes,
278 .private = false,
279 .swapout = true,
280 .hugetlb = false,
281 },
282 {
283 .sizes = &pagesize,
284 .nr_sizes = &nr_pagesizes,
285 .private = true,
286 .swapout = true,
287 .hugetlb = false,
288 },
289
290 /* thp. */
291 {
292 .sizes = thpsizes,
293 .nr_sizes = &nr_thpsizes,
294 .private = false,
295 .swapout = false,
296 .hugetlb = false,
297 },
298 {
299 .sizes = thpsizes,
300 .nr_sizes = &nr_thpsizes,
301 .private = true,
302 .swapout = false,
303 .hugetlb = false,
304 },
305 {
306 .sizes = thpsizes,
307 .nr_sizes = &nr_thpsizes,
308 .private = false,
309 .swapout = true,
310 .hugetlb = false,
311 },
312 {
313 .sizes = thpsizes,
314 .nr_sizes = &nr_thpsizes,
315 .private = true,
316 .swapout = true,
317 .hugetlb = false,
318 },
319
320 /* hugetlb. */
321 {
322 .sizes = hugetlbsizes,
323 .nr_sizes = &nr_hugetlbsizes,
324 .private = false,
325 .swapout = false,
326 .hugetlb = true,
327 },
328 {
329 .sizes = hugetlbsizes,
330 .nr_sizes = &nr_hugetlbsizes,
331 .private = true,
332 .swapout = false,
333 .hugetlb = true,
334 },
335 };
336
main(int argc,char ** argv)337 int main(int argc, char **argv)
338 {
339 struct thp_settings settings;
340 int i, j, plan = 0;
341
342 pagesize = getpagesize();
343 nr_thpsizes = detect_thp_sizes(thpsizes, ARRAY_SIZE(thpsizes));
344 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes,
345 ARRAY_SIZE(hugetlbsizes));
346
347 /* If THP is supported, save THP settings and initially disable THP. */
348 if (nr_thpsizes) {
349 thp_save_settings();
350 thp_read_settings(&settings);
351 for (i = 0; i < NR_ORDERS; i++) {
352 settings.hugepages[i].enabled = THP_NEVER;
353 settings.shmem_hugepages[i].enabled = SHMEM_NEVER;
354 }
355 thp_push_settings(&settings);
356 }
357
358 for (i = 0; i < ARRAY_SIZE(testcases); i++)
359 plan += *testcases[i].nr_sizes;
360 ksft_set_plan(plan);
361
362 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
363 if (pagemap_fd < 0)
364 ksft_exit_fail_msg("opening pagemap failed\n");
365
366 for (i = 0; i < ARRAY_SIZE(testcases); i++) {
367 const struct testcase *tc = &testcases[i];
368
369 for (j = 0; j < *tc->nr_sizes; j++)
370 test_one_folio(tc->sizes[j], tc->private, tc->swapout,
371 tc->hugetlb);
372 }
373
374 /* If THP is supported, restore original THP settings. */
375 if (nr_thpsizes)
376 thp_restore_settings();
377
378 i = ksft_get_fail_cnt();
379 if (i)
380 ksft_exit_fail_msg("%d out of %d tests failed\n",
381 i, ksft_test_num());
382 ksft_exit_pass();
383 }
384