1 // SPDX-License-Identifier: GPL-2.0-only
11  * There are three threads running per CPU:
13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15  *    area_src), and increments a per-page counter in the same page,
18  * 2) another per-CPU thread handles the userfaults generated by
22  * 3) one last per-CPU thread transfers the memory in the background
24  *    2). Each cpu thread takes cares of transferring a portion of the
32  * per-CPU threads 1 by triggering userfaults inside
80 /* Whether to test uffd write-protection */
94 	int cpu;  member
99 /* pthread_mutex_t starts at page offset 0 */
104  * to avoid non alignment faults on non-x86 archs.
110 				  sizeof(unsigned long long) - 1) &	\
112 						  -  1)))
123     "# 10MiB-~6GiB 999 bounces anonymous test, "
144 		uffd_stats[i].cpu = i;  in uffd_stats_reset()
189 static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)  in noop_alias_mapping()  argument
255 static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)  in hugetlb_alias_mapping()  argument
261 	 * MADV_DONTEED won't work. So exercise -EEXIST on a alias  in hugetlb_alias_mapping()
263 	 * this way we'll exercise the -EEXEC at the fs level.  in hugetlb_alias_mapping()
265 	*start = (unsigned long) area_dst_alias + offset;  in hugetlb_alias_mapping()
284 			   MAP_ANONYMOUS | MAP_SHARED, -1, 0);  in shmem_allocate_area()
295 	void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
346 	/* Undo write-protect, do wakeup after that */  in wp_range()
357 	unsigned long cpu = (unsigned long) arg;  in locking_thread()  local
367 		seed = (unsigned int) time(NULL) - bounces;  in locking_thread()
369 			seed += cpu;  in locking_thread()
377 		page_nr = -bounces;  in locking_thread()
379 			page_nr += cpu * nr_pages_per_cpu;  in locking_thread()
445 					page_nr, cpu, area_dst + page_nr * page_size,  in locking_thread()
465 		if (time(NULL) - start > 1)  in locking_thread()
469 				time(NULL) - start);  in locking_thread()
476 			    unsigned long offset)  in retry_copy_page()  argument
478 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
479 				     uffdio_copy->len,  in retry_copy_page()
480 				     offset);  in retry_copy_page()
483 		if (uffdio_copy->copy != -EEXIST) {  in retry_copy_page()
485 				uffdio_copy->copy);  in retry_copy_page()
490 			uffdio_copy->copy); exit(1);  in retry_copy_page()
494 static int __copy_page(int ufd, unsigned long offset, bool retry)  in __copy_page()  argument
498 	if (offset >= nr_pages * page_size) {  in __copy_page()
499 		fprintf(stderr, "unexpected offset %lu\n", offset);  in __copy_page()
502 	uffdio_copy.dst = (unsigned long) area_dst + offset;  in __copy_page()
503 	uffdio_copy.src = (unsigned long) area_src + offset;  in __copy_page()
512 		if (uffdio_copy.copy != -EEXIST) {  in __copy_page()
523 			retry_copy_page(ufd, &uffdio_copy, offset);  in __copy_page()
530 static int copy_page_retry(int ufd, unsigned long offset)  in copy_page_retry()  argument
532 	return __copy_page(ufd, offset, true);  in copy_page_retry()
535 static int copy_page(int ufd, unsigned long offset)  in copy_page()  argument
537 	return __copy_page(ufd, offset, false);  in copy_page()
561 	unsigned long offset;  in uffd_handle_page_fault()  local
563 	if (msg->event != UFFD_EVENT_PAGEFAULT) {  in uffd_handle_page_fault()
564 		fprintf(stderr, "unexpected msg event %u\n", msg->event);  in uffd_handle_page_fault()
568 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
569 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
570 		stats->wp_faults++;  in uffd_handle_page_fault()
574 		    msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) {  in uffd_handle_page_fault()
579 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
580 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
582 		if (copy_page(uffd, offset))  in uffd_handle_page_fault()
583 			stats->missing_faults++;  in uffd_handle_page_fault()
590 	unsigned long cpu = stats->cpu;  in uffd_poll_thread()  local
599 	pollfd[1].fd = pipefd[cpu*2];  in uffd_poll_thread()
603 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
641 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
678 	unsigned long cpu = (unsigned long) arg;  in background_thread()  local
681 	start_nr = cpu * nr_pages_per_cpu;  in background_thread()
682 	end_nr = (cpu+1) * nr_pages_per_cpu;  in background_thread()
690 	 * If we need to test uffd-wp, set it up now.  Then we'll have  in background_thread()
692 	 * can be write-protected for testing  in background_thread()
710 	unsigned long cpu;  in stress()  local
716 	for (cpu = 0; cpu < nr_cpus; cpu++) {  in stress()
717 		if (pthread_create(&locking_threads[cpu], &attr,  in stress()
718 				   locking_thread, (void *)cpu))  in stress()
721 			if (pthread_create(&uffd_threads[cpu], &attr,  in stress()
723 					   (void *)&uffd_stats[cpu]))  in stress()
726 			if (pthread_create(&uffd_threads[cpu], &attr,  in stress()
728 					   (void *)&uffd_stats[cpu]))  in stress()
732 		if (pthread_create(&background_threads[cpu], &attr,  in stress()
733 				   background_thread, (void *)cpu))  in stress()
736 	for (cpu = 0; cpu < nr_cpus; cpu++)  in stress()
737 		if (pthread_join(background_threads[cpu], NULL))  in stress()
745 	 * area_src (but they're guaranteed to get -EEXIST from  in stress()
749 	if (uffd_test_ops->release_pages(area_src))  in stress()
754 	for (cpu = 0; cpu < nr_cpus; cpu++)  in stress()
755 		if (pthread_join(locking_threads[cpu], NULL))  in stress()
758 	for (cpu = 0; cpu < nr_cpus; cpu++) {  in stress()
761 			if (write(pipefd[cpu*2+1], &c, 1) != 1) {  in stress()
765 			if (pthread_join(uffd_threads[cpu],  in stress()
766 					 (void *)&uffd_stats[cpu]))  in stress()
769 			if (pthread_cancel(uffd_threads[cpu]))  in stress()
771 			if (pthread_join(uffd_threads[cpu], NULL))  in stress()
817  * For non-cooperative userfaultfd test we fork() a process that will
832  * test robustness use case - we release monitored area, fork a process
860 		lastnr = (unsigned long)-1;  in faulting_process()
865 		unsigned long offset = nr * page_size;  in faulting_process()  local
879 						if (copy_page(uffd, offset))  in faulting_process()
886 							 offset,  in faulting_process()
938 	if (uffd_test_ops->release_pages(area_dst))  in faulting_process()
953 				  unsigned long offset)  in retry_uffdio_zeropage()  argument
955 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
956 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
957 				     offset);  in retry_uffdio_zeropage()
959 		if (uffdio_zeropage->zeropage != -EEXIST) {  in retry_uffdio_zeropage()
961 				uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
966 			uffdio_zeropage->zeropage); exit(1);  in retry_uffdio_zeropage()
970 static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry)  in __uffdio_zeropage()  argument
976 	has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);  in __uffdio_zeropage()
978 	if (offset >= nr_pages * page_size) {  in __uffdio_zeropage()
979 		fprintf(stderr, "unexpected offset %lu\n", offset);  in __uffdio_zeropage()
982 	uffdio_zeropage.range.start = (unsigned long) area_dst + offset;  in __uffdio_zeropage()
989 			if (uffdio_zeropage.zeropage == -EEXIST) {  in __uffdio_zeropage()
990 				fprintf(stderr, "UFFDIO_ZEROPAGE -EEXIST\n");  in __uffdio_zeropage()
998 			if (uffdio_zeropage.zeropage != -EINVAL) {  in __uffdio_zeropage()
1000 					"UFFDIO_ZEROPAGE not -EINVAL %Ld\n",  in __uffdio_zeropage()
1013 						      offset);  in __uffdio_zeropage()
1026 static int uffdio_zeropage(int ufd, unsigned long offset)  in uffdio_zeropage()  argument
1028 	return __uffdio_zeropage(ufd, offset, false);  in uffdio_zeropage()
1040 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_zeropage_test()
1055 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_zeropage_test()
1088 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_events_test()
1107 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_events_test()
1161 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_sig_test()
1179 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_sig_test()
1190 	if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_sig_test()
1234 	unsigned long cpu;  in userfaultfd_stress()  local
1238 	uffd_test_ops->allocate_area((void **)&area_src);  in userfaultfd_stress()
1241 	uffd_test_ops->allocate_area((void **)&area_dst);  in userfaultfd_stress()
1261 		 * zero, so leave a placeholder below always non-zero  in userfaultfd_stress()
1273 	for (cpu = 0; cpu < nr_cpus; cpu++) {  in userfaultfd_stress()
1274 		if (pipe2(&pipefd[cpu*2], O_CLOEXEC | O_NONBLOCK)) {  in userfaultfd_stress()
1293 	while (bounces--) {  in userfaultfd_stress()
1323 		expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_stress()
1348 		 * return -EEXIST). The problem comes at the next  in userfaultfd_stress()
1353 		 * area_src would lead to -EEXIST failure during the  in userfaultfd_stress()
1364 		if (uffd_test_ops->release_pages(area_dst))  in userfaultfd_stress()
1428  * Copied from mlock2-tests.c
1456 		/* Only enable write-protect test for anonymous test */  in set_test_type()