1 // SPDX-License-Identifier: GPL-2.0-only
5  *  Copyright (C) 2015-2023  Red Hat, Inc.
8 #include "uffd-common.h"
27 	((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
53 		.name = "shmem-private",
65 		.name = "hugetlb-private",
192 	map_shared = mem_type->shared;  in uffd_setup_environment()
193 	uffd_test_ops = mem_type->mem_ops;  in uffd_setup_environment()
194 	uffd_test_case_ops = test->test_case_ops;  in uffd_setup_environment()
196 	if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))  in uffd_setup_environment()
207 	args->mem_type = mem_type;  in uffd_setup_environment()
209 	return uffd_test_ctx_init(test->uffd_feature_required, errmsg);  in uffd_setup_environment()
219 	return (features & test->uffd_feature_required) ==  in uffd_feature_supported()
220 	    test->uffd_feature_required;  in uffd_feature_supported()
234 #define  pagemap_check_wp(value, wp) do {				\  argument
235 		if (!!(value & PM_UFFD_WP) != wp)			\
236 			err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
251 	while (uffd_read_msg(args->parent_uffd, &msg));  in fork_event_consumer()
257 	args->child_uffd = msg.arg.fork.ufd;  in fork_event_consumer()
277 		/* Read-only pins */  in pin_pages()
281 	if (args->pinned)  in pin_pages()
284 	args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);  in pin_pages()
285 	if (args->gup_fd < 0)  in pin_pages()
286 		return -errno;  in pin_pages()
288 	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {  in pin_pages()
290 		close(args->gup_fd);  in pin_pages()
291 		return -errno;  in pin_pages()
293 	args->pinned = true;  in pin_pages()
299 	if (!args->pinned)  in unpin_pages()
301 	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))  in unpin_pages()
303 	close(args->gup_fd);  in unpin_pages()
304 	args->pinned = false;  in unpin_pages()
309 	fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };  in pagemap_test_fork()
340 		 * After fork(), we should handle uffd-wp bit differently:  in pagemap_test_fork()
400 	/* Test read-zero-page upon pte marker */  in uffd_wp_unpopulated_test()
428 		uffd_test_fail("Detected %s uffd-wp bit in child in present pte",  in uffd_wp_fork_test_common()
444 	if (args->mem_type->shared) {  in uffd_wp_fork_test_common()
449 		 * NOTE: ignore retval because private-hugetlb doesn't yet  in uffd_wp_fork_test_common()
455 	/* Uffd-wp should persist even swapped out */  in uffd_wp_fork_test_common()
459 		uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",  in uffd_wp_fork_test_common()
518 		uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",  in uffd_wp_fork_pin_test_common()
531 		uffd_test_fail("Detected %s uffd-wp bit when RO pin",  in uffd_wp_fork_pin_test_common()
558 		expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));  in check_memory_contents()
575 	 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing  in uffd_minor_test_common()
581 			  /* NOTE! MADV_COLLAPSE may not work with uffd-wp */  in uffd_minor_test_common()
586 	 * After registering with UFFD, populate the non-UFFD-registered side of  in uffd_minor_test_common()
590 		memset(area_dst + (p * page_size), p % ((uint8_t)-1),  in uffd_minor_test_common()
598 	 * Read each of the pages back using the UFFD-registered mapping. We  in uffd_minor_test_common()
600 	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the  in uffd_minor_test_common()
618 		uffd_test_ops->check_pmd_mapping(area_dst,  in uffd_minor_test_common()
622 		 * This won't cause uffd-fault - it purely just makes sure there  in uffd_minor_test_common()
661  * For non-cooperative userfaultfd test we fork() a process that will
676  * test robustness use case - we release monitored area, fork a process
681 static int faulting_process(int signal_test, bool wp)  in faulting_process()  argument
699 		lastnr = (unsigned long)-1;  in faulting_process()
716 						if (copy_page(uffd, offset, wp))  in faulting_process()
719 						/* This is a WP request */  in faulting_process()
767 	uffd_test_ops->release_pages(area_dst);  in faulting_process()
777 static void uffd_sigbus_test_common(bool wp)  in uffd_sigbus_test_common()  argument
791 			  true, wp, false))  in uffd_sigbus_test_common()
794 	if (faulting_process(1, wp))  in uffd_sigbus_test_common()
797 	uffd_test_ops->release_pages(area_dst);  in uffd_sigbus_test_common()
799 	args.apply_wp = wp;  in uffd_sigbus_test_common()
811 		exit(faulting_process(2, wp));  in uffd_sigbus_test_common()
837 static void uffd_events_test_common(bool wp)  in uffd_events_test_common()  argument
849 			  true, wp, false))  in uffd_events_test_common()
852 	args.apply_wp = wp;  in uffd_events_test_common()
864 		exit(faulting_process(0, wp));  in uffd_events_test_common()
893 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
894 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
897 		if (uffdio_zeropage->zeropage != -EEXIST)  in retry_uffdio_zeropage()
899 			    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
902 		    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
921 		else if (res != -EINVAL)  in do_uffdio_zeropage()
922 			err("UFFDIO_ZEROPAGE not -EINVAL");  in do_uffdio_zeropage()
966 				err("data non-zero at offset %d\n", i);  in uffd_zeropage_test()
1013 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_poison_handle_fault()
1014 		err("unexpected msg event %u", msg->event);  in uffd_poison_handle_fault()
1016 	if (msg->arg.pagefault.flags &  in uffd_poison_handle_fault()
1018 		err("unexpected fault type %llu", msg->arg.pagefault.flags);  in uffd_poison_handle_fault()
1020 	offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_poison_handle_fault()
1021 	offset &= ~(page_size-1);  in uffd_poison_handle_fault()
1023 	/* Odd pages -> copy zeroed page; even pages -> poison. */  in uffd_poison_handle_fault()
1094 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_move_handle_fault_common()
1095 		err("unexpected msg event %u", msg->event);  in uffd_move_handle_fault_common()
1097 	if (msg->arg.pagefault.flags &  in uffd_move_handle_fault_common()
1099 		err("unexpected fault type %llu", msg->arg.pagefault.flags);  in uffd_move_handle_fault_common()
1101 	offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_move_handle_fault_common()
1102 	offset &= ~(len-1);  in uffd_move_handle_fault_common()
1105 		args->missing_faults++;  in uffd_move_handle_fault_common()
1154 			src_offs = (aligned_src - area_src) / page_size;  in uffd_move_test_common()
1155 			dst_offs = (aligned_dst - area_dst) / page_size;  in uffd_move_test_common()
1156 			step_count--;  in uffd_move_test_common()
1165 	 * Read each of the pages back using the UFFD-registered mapping. We  in uffd_move_test_common()
1173 		/* Check area_src content */  in uffd_move_test_common()
1189 		/* Re-check area_src content which should be empty */  in uffd_move_test_common()
1242 			return -errno;  in prevent_hugepages()
1257 		return -errno;  in request_hugepages()
1275 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)  in do_register_ioctls_test()  argument
1278 	mem_type_t *mem_type = args->mem_type;  in do_register_ioctls_test()
1282 					miss, wp, minor, &ioctls);  in do_register_ioctls_test()
1286 	 * just fail with -EINVAL first..  in do_register_ioctls_test()
1291 	if ((minor && (mem_type->mem_flag == MEM_ANON)) ||  in do_register_ioctls_test()
1292 	    (!miss && !wp && !minor)) {  in do_register_ioctls_test()
1293 		if (ret != -EINVAL)  in do_register_ioctls_test()
1294 			err("register (miss=%d, wp=%d, minor=%d) failed "  in do_register_ioctls_test()
1295 			    "with wrong errno=%d", miss, wp, minor, ret);  in do_register_ioctls_test()
1302 	if (wp)  in do_register_ioctls_test()
1309 		    "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "  in do_register_ioctls_test()
1310 		    "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);  in do_register_ioctls_test()
1318 	int miss, wp, minor;  in uffd_register_ioctls_test()  local
1321 		for (wp = 0; wp <= 1; wp++)  in uffd_register_ioctls_test()
1323 				do_register_ioctls_test(args, miss, wp, minor);  in uffd_register_ioctls_test()
1331 		.name = "register-ioctls",
1355 		.name = "move-pmd",
1362 		.name = "move-pmd-split",
1369 		.name = "wp-fork",
1376 		.name = "wp-fork-with-event",
1381 		/* when set, child process should inherit uffd-wp bits */
1385 		.name = "wp-fork-pin",
1392 		.name = "wp-fork-pin-with-event",
1397 		/* when set, child process should inherit uffd-wp bits */
1401 		.name = "wp-unpopulated",
1415 		.name = "minor-wp",
1423 		 * minor mode supports wr-protect.  There's no feature flag
1429 		.name = "minor-collapse",
1444 		.name = "sigbus-wp",
1459 		.name = "events-wp",
1477 	printf("usage: %s [-f TESTNAME]\n", prog);  in usage()
1479 	puts(" -f: test name to filter (e.g., event)");  in usage()
1480 	puts(" -h: show the help msg");  in usage()
1481 	puts(" -l: list tests only");  in usage()
1499 	while ((opt = getopt(argc, argv, "f:hl")) != -1) {  in main()
1527 		if (test_filter && !strstr(test->name, test_filter))  in main()
1530 			printf("%s\n", test->name);  in main()
1535 			if (!(test->mem_targets & mem_type->mem_flag))  in main()
1538 			uffd_test_start("%s on %s", test->name, mem_type->name);  in main()
1539 			if ((mem_type->mem_flag == MEM_HUGETLB ||  in main()
1540 			    mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&  in main()
1554 			test->uffd_fn(&args);  in main()