134 static int child_memcmp_fn(char *mem, size_t size,  in child_memcmp_fn()  argument
141 	memcpy(old, mem, size);  in child_memcmp_fn()
149 	return memcmp(old, mem, size);  in child_memcmp_fn()
152 static int child_vmsplice_memcmp_fn(char *mem, size_t size,  in child_vmsplice_memcmp_fn()  argument
156 		.iov_base = mem,  in child_vmsplice_memcmp_fn()
168 	memcpy(old, mem, size);  in child_vmsplice_memcmp_fn()
181 	if (munmap(mem, size) < 0)  in child_vmsplice_memcmp_fn()
199 typedef int (*child_fn)(char *mem, size_t size, struct comm_pipes *comm_pipes);
201 static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect,  in do_test_cow_in_parent()  argument
219 		exit(fn(mem, size, &comm_pipes));  in do_test_cow_in_parent()
230 		ret = mprotect(mem, size, PROT_READ);  in do_test_cow_in_parent()
231 		ret |= mprotect(mem, size, PROT_READ|PROT_WRITE);  in do_test_cow_in_parent()
241 	memset(mem, 0xff, size);  in do_test_cow_in_parent()
255 static void test_cow_in_parent(char *mem, size_t size)  in test_cow_in_parent()  argument
257 	do_test_cow_in_parent(mem, size, false, child_memcmp_fn);  in test_cow_in_parent()
260 static void test_cow_in_parent_mprotect(char *mem, size_t size)  in test_cow_in_parent_mprotect()  argument
262 	do_test_cow_in_parent(mem, size, true, child_memcmp_fn);  in test_cow_in_parent_mprotect()
265 static void test_vmsplice_in_child(char *mem, size_t size)  in test_vmsplice_in_child()  argument
267 	do_test_cow_in_parent(mem, size, false, child_vmsplice_memcmp_fn);  in test_vmsplice_in_child()
270 static void test_vmsplice_in_child_mprotect(char *mem, size_t size)  in test_vmsplice_in_child_mprotect()  argument
272 	do_test_cow_in_parent(mem, size, true, child_vmsplice_memcmp_fn);  in test_vmsplice_in_child_mprotect()
275 static void do_test_vmsplice_in_parent(char *mem, size_t size,  in do_test_vmsplice_in_parent()  argument
279 		.iov_base = mem,  in do_test_vmsplice_in_parent()
291 	memcpy(old, mem, size);  in do_test_vmsplice_in_parent()
321 		memset(mem, 0xff, size);  in do_test_vmsplice_in_parent()
336 	if (munmap(mem, size) < 0) {  in do_test_vmsplice_in_parent()
370 static void test_vmsplice_before_fork(char *mem, size_t size)  in test_vmsplice_before_fork()  argument
372 	do_test_vmsplice_in_parent(mem, size, true);  in test_vmsplice_before_fork()
375 static void test_vmsplice_after_fork(char *mem, size_t size)  in test_vmsplice_after_fork()  argument
377 	do_test_vmsplice_in_parent(mem, size, false);  in test_vmsplice_after_fork()
381 static void do_test_iouring(char *mem, size_t size, bool use_fork)  in do_test_iouring()  argument
427 	iov.iov_base = mem;  in do_test_iouring()
461 		ret = mprotect(mem, size, PROT_READ);  in do_test_iouring()
463 		ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_iouring()
474 	memset(mem, 0xff, size);  in do_test_iouring()
480 	io_uring_prep_write_fixed(sqe, fd, mem, size, 0, 0);  in do_test_iouring()
512 	ksft_test_result(!memcmp(mem, tmp, size),  in do_test_iouring()
532 static void test_iouring_ro(char *mem, size_t size)  in test_iouring_ro()  argument
534 	do_test_iouring(mem, size, false);  in test_iouring_ro()
537 static void test_iouring_fork(char *mem, size_t size)  in test_iouring_fork()  argument
539 	do_test_iouring(mem, size, true);  in test_iouring_fork()
551 static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test,  in do_test_ro_pin()  argument
621 		ret = mprotect(mem, size, PROT_READ);  in do_test_ro_pin()
623 		ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_ro_pin()
634 	args.addr = (__u64)(uintptr_t)mem;  in do_test_ro_pin()
647 	memset(mem, 0xff, size);  in do_test_ro_pin()
658 		ksft_test_result(!memcmp(mem, tmp, size),  in do_test_ro_pin()
681 static void test_ro_pin_on_shared(char *mem, size_t size)  in test_ro_pin_on_shared()  argument
683 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, false);  in test_ro_pin_on_shared()
686 static void test_ro_fast_pin_on_shared(char *mem, size_t size)  in test_ro_fast_pin_on_shared()  argument
688 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, true);  in test_ro_fast_pin_on_shared()
691 static void test_ro_pin_on_ro_previously_shared(char *mem, size_t size)  in test_ro_pin_on_ro_previously_shared()  argument
693 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, false);  in test_ro_pin_on_ro_previously_shared()
696 static void test_ro_fast_pin_on_ro_previously_shared(char *mem, size_t size)  in test_ro_fast_pin_on_ro_previously_shared()  argument
698 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, true);  in test_ro_fast_pin_on_ro_previously_shared()
701 static void test_ro_pin_on_ro_exclusive(char *mem, size_t size)  in test_ro_pin_on_ro_exclusive()  argument
703 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, false);  in test_ro_pin_on_ro_exclusive()
706 static void test_ro_fast_pin_on_ro_exclusive(char *mem, size_t size)  in test_ro_fast_pin_on_ro_exclusive()  argument
708 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, true);  in test_ro_fast_pin_on_ro_exclusive()
711 typedef void (*test_fn)(char *mem, size_t size);
715 	char *mem;  in do_run_with_base_page()  local
718 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE,  in do_run_with_base_page()
720 	if (mem == MAP_FAILED) {  in do_run_with_base_page()
725 	ret = madvise(mem, pagesize, MADV_NOHUGEPAGE);  in do_run_with_base_page()
733 	memset(mem, 0, pagesize);  in do_run_with_base_page()
736 		madvise(mem, pagesize, MADV_PAGEOUT);  in do_run_with_base_page()
737 		if (!pagemap_is_swapped(pagemap_fd, mem)) {  in do_run_with_base_page()
743 	fn(mem, pagesize);  in do_run_with_base_page()
745 	munmap(mem, pagesize);  in do_run_with_base_page()
773 	char *mem, *mmap_mem, *tmp, *mremap_mem = MAP_FAILED;  in do_run_with_thp()  local
787 	mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));  in do_run_with_thp()
789 	ret = madvise(mem, thpsize, MADV_HUGEPAGE);  in do_run_with_thp()
799 	mem[0] = 0;  in do_run_with_thp()
800 	if (!pagemap_is_populated(pagemap_fd, mem + thpsize - pagesize)) {  in do_run_with_thp()
804 	memset(mem, 0, thpsize);  in do_run_with_thp()
819 		ret = mprotect(mem + pagesize, pagesize, PROT_READ);  in do_run_with_thp()
824 		ret = mprotect(mem + pagesize, pagesize, PROT_READ | PROT_WRITE);  in do_run_with_thp()
836 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTNEED);  in do_run_with_thp()
851 		if (mem == MAP_FAILED) {  in do_run_with_thp()
855 		tmp = mremap(mem + mremap_size, mremap_size, mremap_size,  in do_run_with_thp()
869 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTFORK);  in do_run_with_thp()
883 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DOFORK);  in do_run_with_thp()
897 		madvise(mem, size, MADV_PAGEOUT);  in do_run_with_thp()
898 		if (!range_is_swapped(mem, size)) {  in do_run_with_thp()
907 	fn(mem, size);  in do_run_with_thp()
973 	char *mem, *dummy;  in run_with_hugetlb()  local
980 	mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, flags, -1, 0);  in run_with_hugetlb()
981 	if (mem == MAP_FAILED) {  in run_with_hugetlb()
987 	memset(mem, 0, hugetlbsize);  in run_with_hugetlb()
1000 	fn(mem, hugetlbsize);  in run_with_hugetlb()
1002 	munmap(mem, hugetlbsize);  in run_with_hugetlb()
1192 static void do_test_anon_thp_collapse(char *mem, size_t size,  in do_test_anon_thp_collapse()  argument
1209 	ret = mprotect(mem + pagesize, pagesize, PROT_READ);  in do_test_anon_thp_collapse()
1214 	ret = mprotect(mem + pagesize, pagesize, PROT_READ | PROT_WRITE);  in do_test_anon_thp_collapse()
1223 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1235 		ret = madvise(mem + size / 2, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1243 		ret = madvise(mem, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1261 			exit(child_memcmp_fn(mem, size, &comm_pipes));  in do_test_anon_thp_collapse()
1264 			exit(child_memcmp_fn(mem, size / 2, &comm_pipes));  in do_test_anon_thp_collapse()
1267 			exit(child_memcmp_fn(mem + size / 2, size / 2,  in do_test_anon_thp_collapse()
1287 		ret = madvise(mem, size, MADV_DOFORK);  in do_test_anon_thp_collapse()
1297 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1311 	memset(mem, 0xff, size);  in do_test_anon_thp_collapse()
1325 static void test_anon_thp_collapse_unshared(char *mem, size_t size)  in test_anon_thp_collapse_unshared()  argument
1327 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UNSHARED);  in test_anon_thp_collapse_unshared()
1330 static void test_anon_thp_collapse_fully_shared(char *mem, size_t size)  in test_anon_thp_collapse_fully_shared()  argument
1332 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_FULLY_SHARED);  in test_anon_thp_collapse_fully_shared()
1335 static void test_anon_thp_collapse_lower_shared(char *mem, size_t size)  in test_anon_thp_collapse_lower_shared()  argument
1337 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_LOWER_SHARED);  in test_anon_thp_collapse_lower_shared()
1340 static void test_anon_thp_collapse_upper_shared(char *mem, size_t size)  in test_anon_thp_collapse_upper_shared()  argument
1342 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UPPER_SHARED);  in test_anon_thp_collapse_upper_shared()
1407 typedef void (*non_anon_test_fn)(char *mem, const char *smem, size_t size);
1409 static void test_cow(char *mem, const char *smem, size_t size)  in test_cow()  argument
1417 	memset(mem, 0xff, size);  in test_cow()
1425 static void test_ro_pin(char *mem, const char *smem, size_t size)  in test_ro_pin()  argument
1427 	do_test_ro_pin(mem, size, RO_PIN_TEST, false);  in test_ro_pin()
1430 static void test_ro_fast_pin(char *mem, const char *smem, size_t size)  in test_ro_fast_pin()  argument
1432 	do_test_ro_pin(mem, size, RO_PIN_TEST, true);  in test_ro_fast_pin()
1437 	char *mem, *smem, tmp;  in run_with_zeropage()  local
1441 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE,  in run_with_zeropage()
1443 	if (mem == MAP_FAILED) {  in run_with_zeropage()
1449 	if (mem == MAP_FAILED) {  in run_with_zeropage()
1455 	tmp = *mem + *smem;  in run_with_zeropage()
1458 	fn(mem, smem, pagesize);  in run_with_zeropage()
1460 	munmap(mem, pagesize);  in run_with_zeropage()
1467 	char *mem, *smem, *mmap_mem, *mmap_smem, tmp;  in run_with_huge_zeropage()  local
1494 	mem = (char *)(((uintptr_t)mmap_mem + pmdsize) & ~(pmdsize - 1));  in run_with_huge_zeropage()
1497 	ret = madvise(mem, pmdsize, MADV_HUGEPAGE);  in run_with_huge_zeropage()
1509 	tmp = *mem + *smem;  in run_with_huge_zeropage()
1511 	if (!pagemap_is_populated(pagemap_fd, mem + pagesize) ||  in run_with_huge_zeropage()
1517 	fn(mem, smem, pmdsize);  in run_with_huge_zeropage()
1526 	char *mem, *smem, tmp;  in run_with_memfd()  local
1544 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);  in run_with_memfd()
1545 	if (mem == MAP_FAILED) {  in run_with_memfd()
1550 	if (mem == MAP_FAILED) {  in run_with_memfd()
1556 	tmp = *mem + *smem;  in run_with_memfd()
1559 	fn(mem, smem, pagesize);  in run_with_memfd()
1561 	munmap(mem, pagesize);  in run_with_memfd()
1570 	char *mem, *smem, tmp;  in run_with_tmpfile()  local
1595 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);  in run_with_tmpfile()
1596 	if (mem == MAP_FAILED) {  in run_with_tmpfile()
1601 	if (mem == MAP_FAILED) {  in run_with_tmpfile()
1607 	tmp = *mem + *smem;  in run_with_tmpfile()
1610 	fn(mem, smem, pagesize);  in run_with_tmpfile()
1612 	munmap(mem, pagesize);  in run_with_tmpfile()
1623 	char *mem, *smem, tmp;  in run_with_memfd_hugetlb()  local
1644 	mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd,  in run_with_memfd_hugetlb()
1646 	if (mem == MAP_FAILED) {  in run_with_memfd_hugetlb()
1651 	if (mem == MAP_FAILED) {  in run_with_memfd_hugetlb()
1657 	tmp = *mem + *smem;  in run_with_memfd_hugetlb()
1660 	fn(mem, smem, hugetlbsize);  in run_with_memfd_hugetlb()
1662 	munmap(mem, hugetlbsize);  in run_with_memfd_hugetlb()
1663 	if (mem != MAP_FAILED)  in run_with_memfd_hugetlb()