1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Userfaultfd unit tests.
4 *
5 * Copyright (C) 2015-2023 Red Hat, Inc.
6 */
7
8 #include "uffd-common.h"
9
10 #include "../../../../mm/gup_test.h"
11
12 #ifdef __NR_userfaultfd
13
14 /* The unit test doesn't need a large or random size, make it 32MB for now */
15 #define UFFD_TEST_MEM_SIZE (32UL << 20)
16
17 #define MEM_ANON BIT_ULL(0)
18 #define MEM_SHMEM BIT_ULL(1)
19 #define MEM_SHMEM_PRIVATE BIT_ULL(2)
20 #define MEM_HUGETLB BIT_ULL(3)
21 #define MEM_HUGETLB_PRIVATE BIT_ULL(4)
22
23 #define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
24 MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
25
26 #define ALIGN_UP(x, align_to) \
27 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
28
29 struct mem_type {
30 const char *name;
31 unsigned int mem_flag;
32 uffd_test_ops_t *mem_ops;
33 bool shared;
34 };
35 typedef struct mem_type mem_type_t;
36
37 mem_type_t mem_types[] = {
38 {
39 .name = "anon",
40 .mem_flag = MEM_ANON,
41 .mem_ops = &anon_uffd_test_ops,
42 .shared = false,
43 },
44 {
45 .name = "shmem",
46 .mem_flag = MEM_SHMEM,
47 .mem_ops = &shmem_uffd_test_ops,
48 .shared = true,
49 },
50 {
51 .name = "shmem-private",
52 .mem_flag = MEM_SHMEM_PRIVATE,
53 .mem_ops = &shmem_uffd_test_ops,
54 .shared = false,
55 },
56 {
57 .name = "hugetlb",
58 .mem_flag = MEM_HUGETLB,
59 .mem_ops = &hugetlb_uffd_test_ops,
60 .shared = true,
61 },
62 {
63 .name = "hugetlb-private",
64 .mem_flag = MEM_HUGETLB_PRIVATE,
65 .mem_ops = &hugetlb_uffd_test_ops,
66 .shared = false,
67 },
68 };
69
70 /* Arguments to be passed over to each uffd unit test */
71 struct uffd_test_args {
72 mem_type_t *mem_type;
73 };
74 typedef struct uffd_test_args uffd_test_args_t;
75
76 /* Returns: UFFD_TEST_* */
77 typedef void (*uffd_test_fn)(uffd_test_args_t *);
78
79 typedef struct {
80 const char *name;
81 uffd_test_fn uffd_fn;
82 unsigned int mem_targets;
83 uint64_t uffd_feature_required;
84 uffd_test_case_ops_t *test_case_ops;
85 } uffd_test_case_t;
86
uffd_test_report(void)87 static void uffd_test_report(void)
88 {
89 printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n",
90 ksft_get_pass_cnt(),
91 ksft_get_xskip_cnt(),
92 ksft_get_fail_cnt(),
93 ksft_test_num());
94 }
95
uffd_test_pass(void)96 static void uffd_test_pass(void)
97 {
98 printf("done\n");
99 ksft_inc_pass_cnt();
100 }
101
102 #define uffd_test_start(...) do { \
103 printf("Testing "); \
104 printf(__VA_ARGS__); \
105 printf("... "); \
106 fflush(stdout); \
107 } while (0)
108
109 #define uffd_test_fail(...) do { \
110 printf("failed [reason: "); \
111 printf(__VA_ARGS__); \
112 printf("]\n"); \
113 ksft_inc_fail_cnt(); \
114 } while (0)
115
uffd_test_skip(const char * message)116 static void uffd_test_skip(const char *message)
117 {
118 printf("skipped [reason: %s]\n", message);
119 ksft_inc_xskip_cnt();
120 }
121
122 /*
123 * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll
124 * return 1 even if some test failed as long as uffd supported, because in
125 * that case we still want to proceed with the rest uffd unit tests.
126 */
test_uffd_api(bool use_dev)127 static int test_uffd_api(bool use_dev)
128 {
129 struct uffdio_api uffdio_api;
130 int uffd;
131
132 uffd_test_start("UFFDIO_API (with %s)",
133 use_dev ? "/dev/userfaultfd" : "syscall");
134
135 if (use_dev)
136 uffd = uffd_open_dev(UFFD_FLAGS);
137 else
138 uffd = uffd_open_sys(UFFD_FLAGS);
139 if (uffd < 0) {
140 uffd_test_skip("cannot open userfaultfd handle");
141 return 0;
142 }
143
144 /* Test wrong UFFD_API */
145 uffdio_api.api = 0xab;
146 uffdio_api.features = 0;
147 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
148 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
149 goto out;
150 }
151
152 /* Test wrong feature bit */
153 uffdio_api.api = UFFD_API;
154 uffdio_api.features = BIT_ULL(63);
155 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
156 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
157 goto out;
158 }
159
160 /* Test normal UFFDIO_API */
161 uffdio_api.api = UFFD_API;
162 uffdio_api.features = 0;
163 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
164 uffd_test_fail("UFFDIO_API should succeed but failed");
165 goto out;
166 }
167
168 /* Test double requests of UFFDIO_API with a random feature set */
169 uffdio_api.features = BIT_ULL(0);
170 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
171 uffd_test_fail("UFFDIO_API should reject initialized uffd");
172 goto out;
173 }
174
175 uffd_test_pass();
176 out:
177 close(uffd);
178 /* We have a valid uffd handle */
179 return 1;
180 }
181
182 /*
183 * This function initializes the global variables. TODO: remove global
184 * vars and then remove this.
185 */
186 static int
uffd_setup_environment(uffd_test_args_t * args,uffd_test_case_t * test,mem_type_t * mem_type,const char ** errmsg)187 uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test,
188 mem_type_t *mem_type, const char **errmsg)
189 {
190 map_shared = mem_type->shared;
191 uffd_test_ops = mem_type->mem_ops;
192 uffd_test_case_ops = test->test_case_ops;
193
194 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))
195 page_size = default_huge_page_size();
196 else
197 page_size = psize();
198
199 nr_pages = UFFD_TEST_MEM_SIZE / page_size;
200 /* TODO: remove this global var.. it's so ugly */
201 nr_cpus = 1;
202
203 /* Initialize test arguments */
204 args->mem_type = mem_type;
205
206 return uffd_test_ctx_init(test->uffd_feature_required, errmsg);
207 }
208
uffd_feature_supported(uffd_test_case_t * test)209 static bool uffd_feature_supported(uffd_test_case_t *test)
210 {
211 uint64_t features;
212
213 if (uffd_get_features(&features))
214 return false;
215
216 return (features & test->uffd_feature_required) ==
217 test->uffd_feature_required;
218 }
219
pagemap_open(void)220 static int pagemap_open(void)
221 {
222 int fd = open("/proc/self/pagemap", O_RDONLY);
223
224 if (fd < 0)
225 err("open pagemap");
226
227 return fd;
228 }
229
230 /* This macro let __LINE__ works in err() */
231 #define pagemap_check_wp(value, wp) do { \
232 if (!!(value & PM_UFFD_WP) != wp) \
233 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
234 } while (0)
235
236 typedef struct {
237 int parent_uffd, child_uffd;
238 } fork_event_args;
239
fork_event_consumer(void * data)240 static void *fork_event_consumer(void *data)
241 {
242 fork_event_args *args = data;
243 struct uffd_msg msg = { 0 };
244
245 /* Read until a full msg received */
246 while (uffd_read_msg(args->parent_uffd, &msg));
247
248 if (msg.event != UFFD_EVENT_FORK)
249 err("wrong message: %u\n", msg.event);
250
251 /* Just to be properly freed later */
252 args->child_uffd = msg.arg.fork.ufd;
253 return NULL;
254 }
255
256 typedef struct {
257 int gup_fd;
258 bool pinned;
259 } pin_args;
260
261 /*
262 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired
263 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy
264 * all needs of the test cases (e.g., trigger unshare, trigger fork() early
265 * CoW, etc.).
266 */
pin_pages(pin_args * args,void * buffer,size_t size)267 static int pin_pages(pin_args *args, void *buffer, size_t size)
268 {
269 struct pin_longterm_test test = {
270 .addr = (uintptr_t)buffer,
271 .size = size,
272 /* Read-only pins */
273 .flags = 0,
274 };
275
276 if (args->pinned)
277 err("already pinned");
278
279 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
280 if (args->gup_fd < 0)
281 return -errno;
282
283 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
284 /* Even if gup_test existed, can be an old gup_test / kernel */
285 close(args->gup_fd);
286 return -errno;
287 }
288 args->pinned = true;
289 return 0;
290 }
291
unpin_pages(pin_args * args)292 static void unpin_pages(pin_args *args)
293 {
294 if (!args->pinned)
295 err("unpin without pin first");
296 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
297 err("PIN_LONGTERM_TEST_STOP");
298 close(args->gup_fd);
299 args->pinned = false;
300 }
301
pagemap_test_fork(int uffd,bool with_event,bool test_pin)302 static int pagemap_test_fork(int uffd, bool with_event, bool test_pin)
303 {
304 fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };
305 pthread_t thread;
306 pid_t child;
307 uint64_t value;
308 int fd, result;
309
310 /* Prepare a thread to resolve EVENT_FORK */
311 if (with_event) {
312 if (pthread_create(&thread, NULL, fork_event_consumer, &args))
313 err("pthread_create()");
314 }
315
316 child = fork();
317 if (!child) {
318 /* Open the pagemap fd of the child itself */
319 pin_args args = {};
320
321 fd = pagemap_open();
322
323 if (test_pin && pin_pages(&args, area_dst, page_size))
324 /*
325 * Normally when reach here we have pinned in
326 * previous tests, so shouldn't fail anymore
327 */
328 err("pin page failed in child");
329
330 value = pagemap_get_entry(fd, area_dst);
331 /*
332 * After fork(), we should handle uffd-wp bit differently:
333 *
334 * (1) when with EVENT_FORK, it should persist
335 * (2) when without EVENT_FORK, it should be dropped
336 */
337 pagemap_check_wp(value, with_event);
338 if (test_pin)
339 unpin_pages(&args);
340 /* Succeed */
341 exit(0);
342 }
343 waitpid(child, &result, 0);
344
345 if (with_event) {
346 if (pthread_join(thread, NULL))
347 err("pthread_join()");
348 if (args.child_uffd < 0)
349 err("Didn't receive child uffd");
350 close(args.child_uffd);
351 }
352
353 return result;
354 }
355
uffd_wp_unpopulated_test(uffd_test_args_t * args)356 static void uffd_wp_unpopulated_test(uffd_test_args_t *args)
357 {
358 uint64_t value;
359 int pagemap_fd;
360
361 if (uffd_register(uffd, area_dst, nr_pages * page_size,
362 false, true, false))
363 err("register failed");
364
365 pagemap_fd = pagemap_open();
366
367 /* Test applying pte marker to anon unpopulated */
368 wp_range(uffd, (uint64_t)area_dst, page_size, true);
369 value = pagemap_get_entry(pagemap_fd, area_dst);
370 pagemap_check_wp(value, true);
371
372 /* Test unprotect on anon pte marker */
373 wp_range(uffd, (uint64_t)area_dst, page_size, false);
374 value = pagemap_get_entry(pagemap_fd, area_dst);
375 pagemap_check_wp(value, false);
376
377 /* Test zap on anon marker */
378 wp_range(uffd, (uint64_t)area_dst, page_size, true);
379 if (madvise(area_dst, page_size, MADV_DONTNEED))
380 err("madvise(MADV_DONTNEED) failed");
381 value = pagemap_get_entry(pagemap_fd, area_dst);
382 pagemap_check_wp(value, false);
383
384 /* Test fault in after marker removed */
385 *area_dst = 1;
386 value = pagemap_get_entry(pagemap_fd, area_dst);
387 pagemap_check_wp(value, false);
388 /* Drop it to make pte none again */
389 if (madvise(area_dst, page_size, MADV_DONTNEED))
390 err("madvise(MADV_DONTNEED) failed");
391
392 /* Test read-zero-page upon pte marker */
393 wp_range(uffd, (uint64_t)area_dst, page_size, true);
394 *(volatile char *)area_dst;
395 /* Drop it to make pte none again */
396 if (madvise(area_dst, page_size, MADV_DONTNEED))
397 err("madvise(MADV_DONTNEED) failed");
398
399 uffd_test_pass();
400 }
401
uffd_wp_fork_test_common(uffd_test_args_t * args,bool with_event)402 static void uffd_wp_fork_test_common(uffd_test_args_t *args,
403 bool with_event)
404 {
405 int pagemap_fd;
406 uint64_t value;
407
408 if (uffd_register(uffd, area_dst, nr_pages * page_size,
409 false, true, false))
410 err("register failed");
411
412 pagemap_fd = pagemap_open();
413
414 /* Touch the page */
415 *area_dst = 1;
416 wp_range(uffd, (uint64_t)area_dst, page_size, true);
417 value = pagemap_get_entry(pagemap_fd, area_dst);
418 pagemap_check_wp(value, true);
419 if (pagemap_test_fork(uffd, with_event, false)) {
420 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
421 with_event ? "missing" : "stall");
422 goto out;
423 }
424
425 /*
426 * This is an attempt for zapping the pgtable so as to test the
427 * markers.
428 *
429 * For private mappings, PAGEOUT will only work on exclusive ptes
430 * (PM_MMAP_EXCLUSIVE) which we should satisfy.
431 *
432 * For shared, PAGEOUT may not work. Use DONTNEED instead which
433 * plays a similar role of zapping (rather than freeing the page)
434 * to expose pte markers.
435 */
436 if (args->mem_type->shared) {
437 if (madvise(area_dst, page_size, MADV_DONTNEED))
438 err("MADV_DONTNEED");
439 } else {
440 /*
441 * NOTE: ignore retval because private-hugetlb doesn't yet
442 * support swapping, so it could fail.
443 */
444 madvise(area_dst, page_size, MADV_PAGEOUT);
445 }
446
447 /* Uffd-wp should persist even swapped out */
448 value = pagemap_get_entry(pagemap_fd, area_dst);
449 pagemap_check_wp(value, true);
450 if (pagemap_test_fork(uffd, with_event, false)) {
451 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
452 with_event ? "missing" : "stall");
453 goto out;
454 }
455
456 /* Unprotect; this tests swap pte modifications */
457 wp_range(uffd, (uint64_t)area_dst, page_size, false);
458 value = pagemap_get_entry(pagemap_fd, area_dst);
459 pagemap_check_wp(value, false);
460
461 /* Fault in the page from disk */
462 *area_dst = 2;
463 value = pagemap_get_entry(pagemap_fd, area_dst);
464 pagemap_check_wp(value, false);
465 uffd_test_pass();
466 out:
467 if (uffd_unregister(uffd, area_dst, nr_pages * page_size))
468 err("unregister failed");
469 close(pagemap_fd);
470 }
471
uffd_wp_fork_test(uffd_test_args_t * args)472 static void uffd_wp_fork_test(uffd_test_args_t *args)
473 {
474 uffd_wp_fork_test_common(args, false);
475 }
476
uffd_wp_fork_with_event_test(uffd_test_args_t * args)477 static void uffd_wp_fork_with_event_test(uffd_test_args_t *args)
478 {
479 uffd_wp_fork_test_common(args, true);
480 }
481
uffd_wp_fork_pin_test_common(uffd_test_args_t * args,bool with_event)482 static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args,
483 bool with_event)
484 {
485 int pagemap_fd;
486 pin_args pin_args = {};
487
488 if (uffd_register(uffd, area_dst, page_size, false, true, false))
489 err("register failed");
490
491 pagemap_fd = pagemap_open();
492
493 /* Touch the page */
494 *area_dst = 1;
495 wp_range(uffd, (uint64_t)area_dst, page_size, true);
496
497 /*
498 * 1. First pin, then fork(). This tests fork() special path when
499 * doing early CoW if the page is private.
500 */
501 if (pin_pages(&pin_args, area_dst, page_size)) {
502 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
503 "or unprivileged");
504 close(pagemap_fd);
505 uffd_unregister(uffd, area_dst, page_size);
506 return;
507 }
508
509 if (pagemap_test_fork(uffd, with_event, false)) {
510 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
511 with_event ? "missing" : "stall");
512 unpin_pages(&pin_args);
513 goto out;
514 }
515
516 unpin_pages(&pin_args);
517
518 /*
519 * 2. First fork(), then pin (in the child, where test_pin==true).
520 * This tests COR, aka, page unsharing on private memories.
521 */
522 if (pagemap_test_fork(uffd, with_event, true)) {
523 uffd_test_fail("Detected %s uffd-wp bit when RO pin",
524 with_event ? "missing" : "stall");
525 goto out;
526 }
527 uffd_test_pass();
528 out:
529 if (uffd_unregister(uffd, area_dst, page_size))
530 err("register failed");
531 close(pagemap_fd);
532 }
533
uffd_wp_fork_pin_test(uffd_test_args_t * args)534 static void uffd_wp_fork_pin_test(uffd_test_args_t *args)
535 {
536 uffd_wp_fork_pin_test_common(args, false);
537 }
538
uffd_wp_fork_pin_with_event_test(uffd_test_args_t * args)539 static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args)
540 {
541 uffd_wp_fork_pin_test_common(args, true);
542 }
543
check_memory_contents(char * p)544 static void check_memory_contents(char *p)
545 {
546 unsigned long i, j;
547 uint8_t expected_byte;
548
549 for (i = 0; i < nr_pages; ++i) {
550 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
551 for (j = 0; j < page_size; j++) {
552 uint8_t v = *(uint8_t *)(p + (i * page_size) + j);
553 if (v != expected_byte)
554 err("unexpected page contents");
555 }
556 }
557 }
558
uffd_minor_test_common(bool test_collapse,bool test_wp)559 static void uffd_minor_test_common(bool test_collapse, bool test_wp)
560 {
561 unsigned long p;
562 pthread_t uffd_mon;
563 char c;
564 struct uffd_args args = { 0 };
565
566 /*
567 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
568 * both do not make much sense.
569 */
570 assert(!(test_collapse && test_wp));
571
572 if (uffd_register(uffd, area_dst_alias, nr_pages * page_size,
573 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
574 false, test_wp, true))
575 err("register failure");
576
577 /*
578 * After registering with UFFD, populate the non-UFFD-registered side of
579 * the shared mapping. This should *not* trigger any UFFD minor faults.
580 */
581 for (p = 0; p < nr_pages; ++p)
582 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
583 page_size);
584
585 args.apply_wp = test_wp;
586 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
587 err("uffd_poll_thread create");
588
589 /*
590 * Read each of the pages back using the UFFD-registered mapping. We
591 * expect that the first time we touch a page, it will result in a minor
592 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
593 * page's contents, and then issuing a CONTINUE ioctl.
594 */
595 check_memory_contents(area_dst_alias);
596
597 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
598 err("pipe write");
599 if (pthread_join(uffd_mon, NULL))
600 err("join() failed");
601
602 if (test_collapse) {
603 if (madvise(area_dst_alias, nr_pages * page_size,
604 MADV_COLLAPSE)) {
605 /* It's fine to fail for this one... */
606 uffd_test_skip("MADV_COLLAPSE failed");
607 return;
608 }
609
610 uffd_test_ops->check_pmd_mapping(area_dst,
611 nr_pages * page_size /
612 read_pmd_pagesize());
613 /*
614 * This won't cause uffd-fault - it purely just makes sure there
615 * was no corruption.
616 */
617 check_memory_contents(area_dst_alias);
618 }
619
620 if (args.missing_faults != 0 || args.minor_faults != nr_pages)
621 uffd_test_fail("stats check error");
622 else
623 uffd_test_pass();
624 }
625
uffd_minor_test(uffd_test_args_t * args)626 void uffd_minor_test(uffd_test_args_t *args)
627 {
628 uffd_minor_test_common(false, false);
629 }
630
uffd_minor_wp_test(uffd_test_args_t * args)631 void uffd_minor_wp_test(uffd_test_args_t *args)
632 {
633 uffd_minor_test_common(false, true);
634 }
635
uffd_minor_collapse_test(uffd_test_args_t * args)636 void uffd_minor_collapse_test(uffd_test_args_t *args)
637 {
638 uffd_minor_test_common(true, false);
639 }
640
641 static sigjmp_buf jbuf, *sigbuf;
642
sighndl(int sig,siginfo_t * siginfo,void * ptr)643 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
644 {
645 if (sig == SIGBUS) {
646 if (sigbuf)
647 siglongjmp(*sigbuf, 1);
648 abort();
649 }
650 }
651
652 /*
653 * For non-cooperative userfaultfd test we fork() a process that will
654 * generate pagefaults, will mremap the area monitored by the
655 * userfaultfd and at last this process will release the monitored
656 * area.
657 * For the anonymous and shared memory the area is divided into two
658 * parts, the first part is accessed before mremap, and the second
659 * part is accessed after mremap. Since hugetlbfs does not support
660 * mremap, the entire monitored area is accessed in a single pass for
661 * HUGETLB_TEST.
662 * The release of the pages currently generates event for shmem and
663 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
664 * for hugetlb.
665 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
666 * monitored area, generate pagefaults and test that signal is delivered.
667 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
668 * test robustness use case - we release monitored area, fork a process
669 * that will generate pagefaults and verify signal is generated.
670 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
671 * feature. Using monitor thread, verify no userfault events are generated.
672 */
faulting_process(int signal_test,bool wp)673 static int faulting_process(int signal_test, bool wp)
674 {
675 unsigned long nr, i;
676 unsigned long long count;
677 unsigned long split_nr_pages;
678 unsigned long lastnr;
679 struct sigaction act;
680 volatile unsigned long signalled = 0;
681
682 split_nr_pages = (nr_pages + 1) / 2;
683
684 if (signal_test) {
685 sigbuf = &jbuf;
686 memset(&act, 0, sizeof(act));
687 act.sa_sigaction = sighndl;
688 act.sa_flags = SA_SIGINFO;
689 if (sigaction(SIGBUS, &act, 0))
690 err("sigaction");
691 lastnr = (unsigned long)-1;
692 }
693
694 for (nr = 0; nr < split_nr_pages; nr++) {
695 volatile int steps = 1;
696 unsigned long offset = nr * page_size;
697
698 if (signal_test) {
699 if (sigsetjmp(*sigbuf, 1) != 0) {
700 if (steps == 1 && nr == lastnr)
701 err("Signal repeated");
702
703 lastnr = nr;
704 if (signal_test == 1) {
705 if (steps == 1) {
706 /* This is a MISSING request */
707 steps++;
708 if (copy_page(uffd, offset, wp))
709 signalled++;
710 } else {
711 /* This is a WP request */
712 assert(steps == 2);
713 wp_range(uffd,
714 (__u64)area_dst +
715 offset,
716 page_size, false);
717 }
718 } else {
719 signalled++;
720 continue;
721 }
722 }
723 }
724
725 count = *area_count(area_dst, nr);
726 if (count != count_verify[nr])
727 err("nr %lu memory corruption %llu %llu\n",
728 nr, count, count_verify[nr]);
729 /*
730 * Trigger write protection if there is by writing
731 * the same value back.
732 */
733 *area_count(area_dst, nr) = count;
734 }
735
736 if (signal_test)
737 return signalled != split_nr_pages;
738
739 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
740 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
741 if (area_dst == MAP_FAILED)
742 err("mremap");
743 /* Reset area_src since we just clobbered it */
744 area_src = NULL;
745
746 for (; nr < nr_pages; nr++) {
747 count = *area_count(area_dst, nr);
748 if (count != count_verify[nr]) {
749 err("nr %lu memory corruption %llu %llu\n",
750 nr, count, count_verify[nr]);
751 }
752 /*
753 * Trigger write protection if there is by writing
754 * the same value back.
755 */
756 *area_count(area_dst, nr) = count;
757 }
758
759 uffd_test_ops->release_pages(area_dst);
760
761 for (nr = 0; nr < nr_pages; nr++)
762 for (i = 0; i < page_size; i++)
763 if (*(area_dst + nr * page_size + i) != 0)
764 err("page %lu offset %lu is not zero", nr, i);
765
766 return 0;
767 }
768
uffd_sigbus_test_common(bool wp)769 static void uffd_sigbus_test_common(bool wp)
770 {
771 unsigned long userfaults;
772 pthread_t uffd_mon;
773 pid_t pid;
774 int err;
775 char c;
776 struct uffd_args args = { 0 };
777
778 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
779
780 if (uffd_register(uffd, area_dst, nr_pages * page_size,
781 true, wp, false))
782 err("register failure");
783
784 if (faulting_process(1, wp))
785 err("faulting process failed");
786
787 uffd_test_ops->release_pages(area_dst);
788
789 args.apply_wp = wp;
790 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
791 err("uffd_poll_thread create");
792
793 pid = fork();
794 if (pid < 0)
795 err("fork");
796
797 if (!pid)
798 exit(faulting_process(2, wp));
799
800 waitpid(pid, &err, 0);
801 if (err)
802 err("faulting process failed");
803 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
804 err("pipe write");
805 if (pthread_join(uffd_mon, (void **)&userfaults))
806 err("pthread_join()");
807
808 if (userfaults)
809 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
810 else
811 uffd_test_pass();
812 }
813
uffd_sigbus_test(uffd_test_args_t * args)814 static void uffd_sigbus_test(uffd_test_args_t *args)
815 {
816 uffd_sigbus_test_common(false);
817 }
818
uffd_sigbus_wp_test(uffd_test_args_t * args)819 static void uffd_sigbus_wp_test(uffd_test_args_t *args)
820 {
821 uffd_sigbus_test_common(true);
822 }
823
uffd_events_test_common(bool wp)824 static void uffd_events_test_common(bool wp)
825 {
826 pthread_t uffd_mon;
827 pid_t pid;
828 int err;
829 char c;
830 struct uffd_args args = { 0 };
831
832 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
833 if (uffd_register(uffd, area_dst, nr_pages * page_size,
834 true, wp, false))
835 err("register failure");
836
837 args.apply_wp = wp;
838 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
839 err("uffd_poll_thread create");
840
841 pid = fork();
842 if (pid < 0)
843 err("fork");
844
845 if (!pid)
846 exit(faulting_process(0, wp));
847
848 waitpid(pid, &err, 0);
849 if (err)
850 err("faulting process failed");
851 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
852 err("pipe write");
853 if (pthread_join(uffd_mon, NULL))
854 err("pthread_join()");
855
856 if (args.missing_faults != nr_pages)
857 uffd_test_fail("Fault counts wrong");
858 else
859 uffd_test_pass();
860 }
861
uffd_events_test(uffd_test_args_t * args)862 static void uffd_events_test(uffd_test_args_t *args)
863 {
864 uffd_events_test_common(false);
865 }
866
uffd_events_wp_test(uffd_test_args_t * args)867 static void uffd_events_wp_test(uffd_test_args_t *args)
868 {
869 uffd_events_test_common(true);
870 }
871
retry_uffdio_zeropage(int ufd,struct uffdio_zeropage * uffdio_zeropage)872 static void retry_uffdio_zeropage(int ufd,
873 struct uffdio_zeropage *uffdio_zeropage)
874 {
875 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
876 uffdio_zeropage->range.len,
877 0);
878 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
879 if (uffdio_zeropage->zeropage != -EEXIST)
880 err("UFFDIO_ZEROPAGE error: %"PRId64,
881 (int64_t)uffdio_zeropage->zeropage);
882 } else {
883 err("UFFDIO_ZEROPAGE error: %"PRId64,
884 (int64_t)uffdio_zeropage->zeropage);
885 }
886 }
887
do_uffdio_zeropage(int ufd,bool has_zeropage)888 static bool do_uffdio_zeropage(int ufd, bool has_zeropage)
889 {
890 struct uffdio_zeropage uffdio_zeropage = { 0 };
891 int ret;
892 __s64 res;
893
894 uffdio_zeropage.range.start = (unsigned long) area_dst;
895 uffdio_zeropage.range.len = page_size;
896 uffdio_zeropage.mode = 0;
897 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
898 res = uffdio_zeropage.zeropage;
899 if (ret) {
900 /* real retval in ufdio_zeropage.zeropage */
901 if (has_zeropage)
902 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
903 else if (res != -EINVAL)
904 err("UFFDIO_ZEROPAGE not -EINVAL");
905 } else if (has_zeropage) {
906 if (res != page_size)
907 err("UFFDIO_ZEROPAGE unexpected size");
908 else
909 retry_uffdio_zeropage(ufd, &uffdio_zeropage);
910 return true;
911 } else
912 err("UFFDIO_ZEROPAGE succeeded");
913
914 return false;
915 }
916
917 /*
918 * Registers a range with MISSING mode only for zeropage test. Return true
919 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
920 * because we want to detect .ioctls along the way.
921 */
922 static bool
uffd_register_detect_zeropage(int uffd,void * addr,uint64_t len)923 uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
924 {
925 uint64_t ioctls = 0;
926
927 if (uffd_register_with_ioctls(uffd, addr, len, true,
928 false, false, &ioctls))
929 err("zeropage register fail");
930
931 return ioctls & (1 << _UFFDIO_ZEROPAGE);
932 }
933
934 /* exercise UFFDIO_ZEROPAGE */
uffd_zeropage_test(uffd_test_args_t * args)935 static void uffd_zeropage_test(uffd_test_args_t *args)
936 {
937 bool has_zeropage;
938 int i;
939
940 has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size);
941 if (area_dst_alias)
942 /* Ignore the retval; we already have it */
943 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size);
944
945 if (do_uffdio_zeropage(uffd, has_zeropage))
946 for (i = 0; i < page_size; i++)
947 if (area_dst[i] != 0)
948 err("data non-zero at offset %d\n", i);
949
950 if (uffd_unregister(uffd, area_dst, page_size))
951 err("unregister");
952
953 if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size))
954 err("unregister");
955
956 uffd_test_pass();
957 }
958
uffd_register_poison(int uffd,void * addr,uint64_t len)959 static void uffd_register_poison(int uffd, void *addr, uint64_t len)
960 {
961 uint64_t ioctls = 0;
962 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
963
964 if (uffd_register_with_ioctls(uffd, addr, len, true,
965 false, false, &ioctls))
966 err("poison register fail");
967
968 if ((ioctls & expected) != expected)
969 err("registered area doesn't support COPY and POISON ioctls");
970 }
971
do_uffdio_poison(int uffd,unsigned long offset)972 static void do_uffdio_poison(int uffd, unsigned long offset)
973 {
974 struct uffdio_poison uffdio_poison = { 0 };
975 int ret;
976 __s64 res;
977
978 uffdio_poison.range.start = (unsigned long) area_dst + offset;
979 uffdio_poison.range.len = page_size;
980 uffdio_poison.mode = 0;
981 ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison);
982 res = uffdio_poison.updated;
983
984 if (ret)
985 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
986 else if (res != page_size)
987 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
988 }
989
uffd_poison_handle_fault(struct uffd_msg * msg,struct uffd_args * args)990 static void uffd_poison_handle_fault(
991 struct uffd_msg *msg, struct uffd_args *args)
992 {
993 unsigned long offset;
994
995 if (msg->event != UFFD_EVENT_PAGEFAULT)
996 err("unexpected msg event %u", msg->event);
997
998 if (msg->arg.pagefault.flags &
999 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
1000 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1001
1002 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1003 offset &= ~(page_size-1);
1004
1005 /* Odd pages -> copy zeroed page; even pages -> poison. */
1006 if (offset & page_size)
1007 copy_page(uffd, offset, false);
1008 else
1009 do_uffdio_poison(uffd, offset);
1010 }
1011
uffd_poison_test(uffd_test_args_t * targs)1012 static void uffd_poison_test(uffd_test_args_t *targs)
1013 {
1014 pthread_t uffd_mon;
1015 char c;
1016 struct uffd_args args = { 0 };
1017 struct sigaction act = { 0 };
1018 unsigned long nr_sigbus = 0;
1019 unsigned long nr;
1020
1021 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1022
1023 uffd_register_poison(uffd, area_dst, nr_pages * page_size);
1024 memset(area_src, 0, nr_pages * page_size);
1025
1026 args.handle_fault = uffd_poison_handle_fault;
1027 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1028 err("uffd_poll_thread create");
1029
1030 sigbuf = &jbuf;
1031 act.sa_sigaction = sighndl;
1032 act.sa_flags = SA_SIGINFO;
1033 if (sigaction(SIGBUS, &act, 0))
1034 err("sigaction");
1035
1036 for (nr = 0; nr < nr_pages; ++nr) {
1037 unsigned long offset = nr * page_size;
1038 const char *bytes = (const char *) area_dst + offset;
1039 const char *i;
1040
1041 if (sigsetjmp(*sigbuf, 1)) {
1042 /*
1043 * Access below triggered a SIGBUS, which was caught by
1044 * sighndl, which then jumped here. Count this SIGBUS,
1045 * and move on to next page.
1046 */
1047 ++nr_sigbus;
1048 continue;
1049 }
1050
1051 for (i = bytes; i < bytes + page_size; ++i) {
1052 if (*i)
1053 err("nonzero byte in area_dst (%p) at %p: %u",
1054 area_dst, i, *i);
1055 }
1056 }
1057
1058 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1059 err("pipe write");
1060 if (pthread_join(uffd_mon, NULL))
1061 err("pthread_join()");
1062
1063 if (nr_sigbus != nr_pages / 2)
1064 err("expected to receive %lu SIGBUS, actually received %lu",
1065 nr_pages / 2, nr_sigbus);
1066
1067 uffd_test_pass();
1068 }
1069
1070 static void
uffd_move_handle_fault_common(struct uffd_msg * msg,struct uffd_args * args,unsigned long len)1071 uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args,
1072 unsigned long len)
1073 {
1074 unsigned long offset;
1075
1076 if (msg->event != UFFD_EVENT_PAGEFAULT)
1077 err("unexpected msg event %u", msg->event);
1078
1079 if (msg->arg.pagefault.flags &
1080 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
1081 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1082
1083 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1084 offset &= ~(len-1);
1085
1086 if (move_page(uffd, offset, len))
1087 args->missing_faults++;
1088 }
1089
uffd_move_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1090 static void uffd_move_handle_fault(struct uffd_msg *msg,
1091 struct uffd_args *args)
1092 {
1093 uffd_move_handle_fault_common(msg, args, page_size);
1094 }
1095
uffd_move_pmd_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1096 static void uffd_move_pmd_handle_fault(struct uffd_msg *msg,
1097 struct uffd_args *args)
1098 {
1099 uffd_move_handle_fault_common(msg, args, read_pmd_pagesize());
1100 }
1101
1102 static void
uffd_move_test_common(uffd_test_args_t * targs,unsigned long chunk_size,void (* handle_fault)(struct uffd_msg * msg,struct uffd_args * args))1103 uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size,
1104 void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args))
1105 {
1106 unsigned long nr;
1107 pthread_t uffd_mon;
1108 char c;
1109 unsigned long long count;
1110 struct uffd_args args = { 0 };
1111 char *orig_area_src, *orig_area_dst;
1112 unsigned long step_size, step_count;
1113 unsigned long src_offs = 0;
1114 unsigned long dst_offs = 0;
1115
1116 /* Prevent source pages from being mapped more than once */
1117 if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK))
1118 err("madvise(MADV_DONTFORK) failure");
1119
1120 if (uffd_register(uffd, area_dst, nr_pages * page_size,
1121 true, false, false))
1122 err("register failure");
1123
1124 args.handle_fault = handle_fault;
1125 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1126 err("uffd_poll_thread create");
1127
1128 step_size = chunk_size / page_size;
1129 step_count = nr_pages / step_size;
1130
1131 if (chunk_size > page_size) {
1132 char *aligned_src = ALIGN_UP(area_src, chunk_size);
1133 char *aligned_dst = ALIGN_UP(area_dst, chunk_size);
1134
1135 if (aligned_src != area_src || aligned_dst != area_dst) {
1136 src_offs = (aligned_src - area_src) / page_size;
1137 dst_offs = (aligned_dst - area_dst) / page_size;
1138 step_count--;
1139 }
1140 orig_area_src = area_src;
1141 orig_area_dst = area_dst;
1142 area_src = aligned_src;
1143 area_dst = aligned_dst;
1144 }
1145
1146 /*
1147 * Read each of the pages back using the UFFD-registered mapping. We
1148 * expect that the first time we touch a page, it will result in a missing
1149 * fault. uffd_poll_thread will resolve the fault by moving source
1150 * page to destination.
1151 */
1152 for (nr = 0; nr < step_count * step_size; nr += step_size) {
1153 unsigned long i;
1154
1155 /* Check area_src content */
1156 for (i = 0; i < step_size; i++) {
1157 count = *area_count(area_src, nr + i);
1158 if (count != count_verify[src_offs + nr + i])
1159 err("nr %lu source memory invalid %llu %llu\n",
1160 nr + i, count, count_verify[src_offs + nr + i]);
1161 }
1162
1163 /* Faulting into area_dst should move the page or the huge page */
1164 for (i = 0; i < step_size; i++) {
1165 count = *area_count(area_dst, nr + i);
1166 if (count != count_verify[dst_offs + nr + i])
1167 err("nr %lu memory corruption %llu %llu\n",
1168 nr, count, count_verify[dst_offs + nr + i]);
1169 }
1170
1171 /* Re-check area_src content which should be empty */
1172 for (i = 0; i < step_size; i++) {
1173 count = *area_count(area_src, nr + i);
1174 if (count != 0)
1175 err("nr %lu move failed %llu %llu\n",
1176 nr, count, count_verify[src_offs + nr + i]);
1177 }
1178 }
1179 if (step_size > page_size) {
1180 area_src = orig_area_src;
1181 area_dst = orig_area_dst;
1182 }
1183
1184 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1185 err("pipe write");
1186 if (pthread_join(uffd_mon, NULL))
1187 err("join() failed");
1188
1189 if (args.missing_faults != step_count || args.minor_faults != 0)
1190 uffd_test_fail("stats check error");
1191 else
1192 uffd_test_pass();
1193 }
1194
uffd_move_test(uffd_test_args_t * targs)1195 static void uffd_move_test(uffd_test_args_t *targs)
1196 {
1197 uffd_move_test_common(targs, page_size, uffd_move_handle_fault);
1198 }
1199
uffd_move_pmd_test(uffd_test_args_t * targs)1200 static void uffd_move_pmd_test(uffd_test_args_t *targs)
1201 {
1202 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1203 err("madvise(MADV_HUGEPAGE) failure");
1204 uffd_move_test_common(targs, read_pmd_pagesize(),
1205 uffd_move_pmd_handle_fault);
1206 }
1207
uffd_move_pmd_split_test(uffd_test_args_t * targs)1208 static void uffd_move_pmd_split_test(uffd_test_args_t *targs)
1209 {
1210 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1211 err("madvise(MADV_NOHUGEPAGE) failure");
1212 uffd_move_test_common(targs, read_pmd_pagesize(),
1213 uffd_move_pmd_handle_fault);
1214 }
1215
prevent_hugepages(const char ** errmsg)1216 static int prevent_hugepages(const char **errmsg)
1217 {
1218 /* This should be done before source area is populated */
1219 if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) {
1220 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
1221 if (errno != EINVAL) {
1222 if (errmsg)
1223 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";
1224 return -errno;
1225 }
1226 }
1227 return 0;
1228 }
1229
request_hugepages(const char ** errmsg)1230 static int request_hugepages(const char **errmsg)
1231 {
1232 /* This should be done before source area is populated */
1233 if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) {
1234 if (errmsg) {
1235 *errmsg = (errno == EINVAL) ?
1236 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :
1237 "madvise(MADV_HUGEPAGE) failed";
1238 }
1239 return -errno;
1240 }
1241 return 0;
1242 }
1243
1244 struct uffd_test_case_ops uffd_move_test_case_ops = {
1245 .post_alloc = prevent_hugepages,
1246 };
1247
1248 struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
1249 .post_alloc = request_hugepages,
1250 };
1251
1252 /*
1253 * Test the returned uffdio_register.ioctls with different register modes.
1254 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
1255 */
1256 static void
do_register_ioctls_test(uffd_test_args_t * args,bool miss,bool wp,bool minor)1257 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)
1258 {
1259 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
1260 mem_type_t *mem_type = args->mem_type;
1261 int ret;
1262
1263 ret = uffd_register_with_ioctls(uffd, area_dst, page_size,
1264 miss, wp, minor, &ioctls);
1265
1266 /*
1267 * Handle special cases of UFFDIO_REGISTER here where it should
1268 * just fail with -EINVAL first..
1269 *
1270 * Case 1: register MINOR on anon
1271 * Case 2: register with no mode selected
1272 */
1273 if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
1274 (!miss && !wp && !minor)) {
1275 if (ret != -EINVAL)
1276 err("register (miss=%d, wp=%d, minor=%d) failed "
1277 "with wrong errno=%d", miss, wp, minor, ret);
1278 return;
1279 }
1280
1281 /* UFFDIO_REGISTER should succeed, then check ioctls returned */
1282 if (miss)
1283 expected |= BIT_ULL(_UFFDIO_COPY);
1284 if (wp)
1285 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
1286 if (minor)
1287 expected |= BIT_ULL(_UFFDIO_CONTINUE);
1288
1289 if ((ioctls & expected) != expected)
1290 err("unexpected uffdio_register.ioctls "
1291 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
1292 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
1293
1294 if (uffd_unregister(uffd, area_dst, page_size))
1295 err("unregister");
1296 }
1297
uffd_register_ioctls_test(uffd_test_args_t * args)1298 static void uffd_register_ioctls_test(uffd_test_args_t *args)
1299 {
1300 int miss, wp, minor;
1301
1302 for (miss = 0; miss <= 1; miss++)
1303 for (wp = 0; wp <= 1; wp++)
1304 for (minor = 0; minor <= 1; minor++)
1305 do_register_ioctls_test(args, miss, wp, minor);
1306
1307 uffd_test_pass();
1308 }
1309
1310 uffd_test_case_t uffd_tests[] = {
1311 {
1312 /* Test returned uffdio_register.ioctls. */
1313 .name = "register-ioctls",
1314 .uffd_fn = uffd_register_ioctls_test,
1315 .mem_targets = MEM_ALL,
1316 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
1317 UFFD_FEATURE_MISSING_SHMEM |
1318 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1319 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1320 UFFD_FEATURE_MINOR_HUGETLBFS |
1321 UFFD_FEATURE_MINOR_SHMEM,
1322 },
1323 {
1324 .name = "zeropage",
1325 .uffd_fn = uffd_zeropage_test,
1326 .mem_targets = MEM_ALL,
1327 .uffd_feature_required = 0,
1328 },
1329 {
1330 .name = "move",
1331 .uffd_fn = uffd_move_test,
1332 .mem_targets = MEM_ANON,
1333 .uffd_feature_required = UFFD_FEATURE_MOVE,
1334 .test_case_ops = &uffd_move_test_case_ops,
1335 },
1336 {
1337 .name = "move-pmd",
1338 .uffd_fn = uffd_move_pmd_test,
1339 .mem_targets = MEM_ANON,
1340 .uffd_feature_required = UFFD_FEATURE_MOVE,
1341 .test_case_ops = &uffd_move_test_pmd_case_ops,
1342 },
1343 {
1344 .name = "move-pmd-split",
1345 .uffd_fn = uffd_move_pmd_split_test,
1346 .mem_targets = MEM_ANON,
1347 .uffd_feature_required = UFFD_FEATURE_MOVE,
1348 .test_case_ops = &uffd_move_test_pmd_case_ops,
1349 },
1350 {
1351 .name = "wp-fork",
1352 .uffd_fn = uffd_wp_fork_test,
1353 .mem_targets = MEM_ALL,
1354 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1355 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1356 },
1357 {
1358 .name = "wp-fork-with-event",
1359 .uffd_fn = uffd_wp_fork_with_event_test,
1360 .mem_targets = MEM_ALL,
1361 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1362 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1363 /* when set, child process should inherit uffd-wp bits */
1364 UFFD_FEATURE_EVENT_FORK,
1365 },
1366 {
1367 .name = "wp-fork-pin",
1368 .uffd_fn = uffd_wp_fork_pin_test,
1369 .mem_targets = MEM_ALL,
1370 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1371 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1372 },
1373 {
1374 .name = "wp-fork-pin-with-event",
1375 .uffd_fn = uffd_wp_fork_pin_with_event_test,
1376 .mem_targets = MEM_ALL,
1377 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1378 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1379 /* when set, child process should inherit uffd-wp bits */
1380 UFFD_FEATURE_EVENT_FORK,
1381 },
1382 {
1383 .name = "wp-unpopulated",
1384 .uffd_fn = uffd_wp_unpopulated_test,
1385 .mem_targets = MEM_ANON,
1386 .uffd_feature_required =
1387 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
1388 },
1389 {
1390 .name = "minor",
1391 .uffd_fn = uffd_minor_test,
1392 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1393 .uffd_feature_required =
1394 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
1395 },
1396 {
1397 .name = "minor-wp",
1398 .uffd_fn = uffd_minor_wp_test,
1399 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1400 .uffd_feature_required =
1401 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
1402 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1403 /*
1404 * HACK: here we leveraged WP_UNPOPULATED to detect whether
1405 * minor mode supports wr-protect. There's no feature flag
1406 * for it so this is the best we can test against.
1407 */
1408 UFFD_FEATURE_WP_UNPOPULATED,
1409 },
1410 {
1411 .name = "minor-collapse",
1412 .uffd_fn = uffd_minor_collapse_test,
1413 /* MADV_COLLAPSE only works with shmem */
1414 .mem_targets = MEM_SHMEM,
1415 /* We can't test MADV_COLLAPSE, so try our luck */
1416 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
1417 },
1418 {
1419 .name = "sigbus",
1420 .uffd_fn = uffd_sigbus_test,
1421 .mem_targets = MEM_ALL,
1422 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1423 UFFD_FEATURE_EVENT_FORK,
1424 },
1425 {
1426 .name = "sigbus-wp",
1427 .uffd_fn = uffd_sigbus_wp_test,
1428 .mem_targets = MEM_ALL,
1429 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1430 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP,
1431 },
1432 {
1433 .name = "events",
1434 .uffd_fn = uffd_events_test,
1435 .mem_targets = MEM_ALL,
1436 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1437 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
1438 },
1439 {
1440 .name = "events-wp",
1441 .uffd_fn = uffd_events_wp_test,
1442 .mem_targets = MEM_ALL,
1443 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1444 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
1445 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1446 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1447 },
1448 {
1449 .name = "poison",
1450 .uffd_fn = uffd_poison_test,
1451 .mem_targets = MEM_ALL,
1452 .uffd_feature_required = UFFD_FEATURE_POISON,
1453 },
1454 };
1455
usage(const char * prog)1456 static void usage(const char *prog)
1457 {
1458 printf("usage: %s [-f TESTNAME]\n", prog);
1459 puts("");
1460 puts(" -f: test name to filter (e.g., event)");
1461 puts(" -h: show the help msg");
1462 puts(" -l: list tests only");
1463 puts("");
1464 exit(KSFT_FAIL);
1465 }
1466
main(int argc,char * argv[])1467 int main(int argc, char *argv[])
1468 {
1469 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
1470 int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
1471 const char *test_filter = NULL;
1472 bool list_only = false;
1473 uffd_test_case_t *test;
1474 mem_type_t *mem_type;
1475 uffd_test_args_t args;
1476 const char *errmsg;
1477 int has_uffd, opt;
1478 int i, j;
1479
1480 while ((opt = getopt(argc, argv, "f:hl")) != -1) {
1481 switch (opt) {
1482 case 'f':
1483 test_filter = optarg;
1484 break;
1485 case 'l':
1486 list_only = true;
1487 break;
1488 case 'h':
1489 default:
1490 /* Unknown */
1491 usage(argv[0]);
1492 break;
1493 }
1494 }
1495
1496 if (!test_filter && !list_only) {
1497 has_uffd = test_uffd_api(false);
1498 has_uffd |= test_uffd_api(true);
1499
1500 if (!has_uffd) {
1501 printf("Userfaultfd not supported or unprivileged, skip all tests\n");
1502 exit(KSFT_SKIP);
1503 }
1504 }
1505
1506 for (i = 0; i < n_tests; i++) {
1507 test = &uffd_tests[i];
1508 if (test_filter && !strstr(test->name, test_filter))
1509 continue;
1510 if (list_only) {
1511 printf("%s\n", test->name);
1512 continue;
1513 }
1514 for (j = 0; j < n_mems; j++) {
1515 mem_type = &mem_types[j];
1516 if (!(test->mem_targets & mem_type->mem_flag))
1517 continue;
1518
1519 uffd_test_start("%s on %s", test->name, mem_type->name);
1520 if ((mem_type->mem_flag == MEM_HUGETLB ||
1521 mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&
1522 (default_huge_page_size() == 0)) {
1523 uffd_test_skip("huge page size is 0, feature missing?");
1524 continue;
1525 }
1526 if (!uffd_feature_supported(test)) {
1527 uffd_test_skip("feature missing");
1528 continue;
1529 }
1530 if (uffd_setup_environment(&args, test, mem_type,
1531 &errmsg)) {
1532 uffd_test_skip(errmsg);
1533 continue;
1534 }
1535 test->uffd_fn(&args);
1536 uffd_test_ctx_clear();
1537 }
1538 }
1539
1540 if (!list_only)
1541 uffd_test_report();
1542
1543 return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;
1544 }
1545
1546 #else /* __NR_userfaultfd */
1547
1548 #warning "missing __NR_userfaultfd definition"
1549
main(void)1550 int main(void)
1551 {
1552 printf("Skipping %s (missing __NR_userfaultfd)\n", __file__);
1553 return KSFT_SKIP;
1554 }
1555
1556 #endif /* __NR_userfaultfd */
1557