1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #define _GNU_SOURCE 4 #include <stdbool.h> 5 #include <stdint.h> 6 #include <fcntl.h> 7 #include <assert.h> 8 #include <linux/mman.h> 9 #include <sys/mman.h> 10 #include "../kselftest.h" 11 #include "thp_settings.h" 12 #include "uffd-common.h" 13 14 static int pagemap_fd; 15 static size_t pagesize; 16 static int nr_pagesizes = 1; 17 static int nr_thpsizes; 18 static size_t thpsizes[20]; 19 static int nr_hugetlbsizes; 20 static size_t hugetlbsizes[10]; 21 22 static int sz2ord(size_t size) 23 { 24 return __builtin_ctzll(size / pagesize); 25 } 26 27 static int detect_thp_sizes(size_t sizes[], int max) 28 { 29 int count = 0; 30 unsigned long orders; 31 size_t kb; 32 int i; 33 34 /* thp not supported at all. */ 35 if (!read_pmd_pagesize()) 36 return 0; 37 38 orders = thp_supported_orders(); 39 40 for (i = 0; orders && count < max; i++) { 41 if (!(orders & (1UL << i))) 42 continue; 43 orders &= ~(1UL << i); 44 kb = (pagesize >> 10) << i; 45 sizes[count++] = kb * 1024; 46 ksft_print_msg("[INFO] detected THP size: %zu KiB\n", kb); 47 } 48 49 return count; 50 } 51 52 static void *mmap_aligned(size_t size, int prot, int flags) 53 { 54 size_t mmap_size = size * 2; 55 char *mmap_mem, *mem; 56 57 mmap_mem = mmap(NULL, mmap_size, prot, flags, -1, 0); 58 if (mmap_mem == MAP_FAILED) 59 return mmap_mem; 60 61 mem = (char *)(((uintptr_t)mmap_mem + size - 1) & ~(size - 1)); 62 munmap(mmap_mem, mem - mmap_mem); 63 munmap(mem + size, mmap_mem + mmap_size - mem - size); 64 65 return mem; 66 } 67 68 static void *alloc_one_folio(size_t size, bool private, bool hugetlb) 69 { 70 bool thp = !hugetlb && size > pagesize; 71 int flags = MAP_ANONYMOUS; 72 int prot = PROT_READ | PROT_WRITE; 73 char *mem, *addr; 74 75 assert((size & (size - 1)) == 0); 76 77 if (private) 78 flags |= MAP_PRIVATE; 79 else 80 flags |= MAP_SHARED; 81 82 /* 83 * For THP, we must explicitly enable the THP size, allocate twice the 84 * required space then manually align. 85 */ 86 if (thp) { 87 struct thp_settings settings = *thp_current_settings(); 88 89 if (private) 90 settings.hugepages[sz2ord(size)].enabled = THP_ALWAYS; 91 else 92 settings.shmem_hugepages[sz2ord(size)].enabled = SHMEM_ALWAYS; 93 94 thp_push_settings(&settings); 95 96 mem = mmap_aligned(size, prot, flags); 97 } else { 98 if (hugetlb) { 99 flags |= MAP_HUGETLB; 100 flags |= __builtin_ctzll(size) << MAP_HUGE_SHIFT; 101 } 102 103 mem = mmap(NULL, size, prot, flags, -1, 0); 104 } 105 106 if (mem == MAP_FAILED) { 107 mem = NULL; 108 goto out; 109 } 110 111 assert(((uintptr_t)mem & (size - 1)) == 0); 112 113 /* 114 * Populate the folio by writing the first byte and check that all pages 115 * are populated. Finally set the whole thing to non-zero data to avoid 116 * kernel from mapping it back to the zero page. 117 */ 118 mem[0] = 1; 119 for (addr = mem; addr < mem + size; addr += pagesize) { 120 if (!pagemap_is_populated(pagemap_fd, addr)) { 121 munmap(mem, size); 122 mem = NULL; 123 goto out; 124 } 125 } 126 memset(mem, 1, size); 127 out: 128 if (thp) 129 thp_pop_settings(); 130 131 return mem; 132 } 133 134 static bool check_uffd_wp_state(void *mem, size_t size, bool expect) 135 { 136 uint64_t pte; 137 void *addr; 138 139 for (addr = mem; addr < mem + size; addr += pagesize) { 140 pte = pagemap_get_entry(pagemap_fd, addr); 141 if (!!(pte & PM_UFFD_WP) != expect) { 142 ksft_test_result_fail("uffd-wp not %s for pte %lu!\n", 143 expect ? "set" : "clear", 144 (addr - mem) / pagesize); 145 return false; 146 } 147 } 148 149 return true; 150 } 151 152 static bool range_is_swapped(void *addr, size_t size) 153 { 154 for (; size; addr += pagesize, size -= pagesize) 155 if (!pagemap_is_swapped(pagemap_fd, addr)) 156 return false; 157 return true; 158 } 159 160 static void test_one_folio(size_t size, bool private, bool swapout, bool hugetlb) 161 { 162 struct uffdio_writeprotect wp_prms; 163 uint64_t features = 0; 164 void *addr = NULL; 165 void *mem = NULL; 166 167 assert(!(hugetlb && swapout)); 168 169 ksft_print_msg("[RUN] %s(size=%zu, private=%s, swapout=%s, hugetlb=%s)\n", 170 __func__, 171 size, 172 private ? "true" : "false", 173 swapout ? "true" : "false", 174 hugetlb ? "true" : "false"); 175 176 /* Allocate a folio of required size and type. */ 177 mem = alloc_one_folio(size, private, hugetlb); 178 if (!mem) { 179 ksft_test_result_fail("alloc_one_folio() failed\n"); 180 goto out; 181 } 182 183 /* Register range for uffd-wp. */ 184 if (userfaultfd_open(&features)) { 185 if (errno == ENOENT) 186 ksft_test_result_skip("userfaultfd not available\n"); 187 else 188 ksft_test_result_fail("userfaultfd_open() failed\n"); 189 goto out; 190 } 191 if (uffd_register(uffd, mem, size, false, true, false)) { 192 ksft_test_result_fail("uffd_register() failed\n"); 193 goto out; 194 } 195 wp_prms.mode = UFFDIO_WRITEPROTECT_MODE_WP; 196 wp_prms.range.start = (uintptr_t)mem; 197 wp_prms.range.len = size; 198 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp_prms)) { 199 ksft_test_result_fail("ioctl(UFFDIO_WRITEPROTECT) failed\n"); 200 goto out; 201 } 202 203 if (swapout) { 204 madvise(mem, size, MADV_PAGEOUT); 205 if (!range_is_swapped(mem, size)) { 206 ksft_test_result_skip("MADV_PAGEOUT did not work, is swap enabled?\n"); 207 goto out; 208 } 209 } 210 211 /* Check that uffd-wp is set for all PTEs in range. */ 212 if (!check_uffd_wp_state(mem, size, true)) 213 goto out; 214 215 /* 216 * Move the mapping to a new, aligned location. Since 217 * UFFD_FEATURE_EVENT_REMAP is not set, we expect the uffd-wp bit for 218 * each PTE to be cleared in the new mapping. 219 */ 220 addr = mmap_aligned(size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS); 221 if (addr == MAP_FAILED) { 222 ksft_test_result_fail("mmap_aligned() failed\n"); 223 goto out; 224 } 225 if (mremap(mem, size, size, MREMAP_FIXED | MREMAP_MAYMOVE, addr) == MAP_FAILED) { 226 ksft_test_result_fail("mremap() failed\n"); 227 munmap(addr, size); 228 goto out; 229 } 230 mem = addr; 231 232 /* Check that uffd-wp is cleared for all PTEs in range. */ 233 if (!check_uffd_wp_state(mem, size, false)) 234 goto out; 235 236 ksft_test_result_pass("%s(size=%zu, private=%s, swapout=%s, hugetlb=%s)\n", 237 __func__, 238 size, 239 private ? "true" : "false", 240 swapout ? "true" : "false", 241 hugetlb ? "true" : "false"); 242 out: 243 if (mem) 244 munmap(mem, size); 245 if (uffd >= 0) { 246 close(uffd); 247 uffd = -1; 248 } 249 } 250 251 struct testcase { 252 size_t *sizes; 253 int *nr_sizes; 254 bool private; 255 bool swapout; 256 bool hugetlb; 257 }; 258 259 static const struct testcase testcases[] = { 260 /* base pages. */ 261 { 262 .sizes = &pagesize, 263 .nr_sizes = &nr_pagesizes, 264 .private = false, 265 .swapout = false, 266 .hugetlb = false, 267 }, 268 { 269 .sizes = &pagesize, 270 .nr_sizes = &nr_pagesizes, 271 .private = true, 272 .swapout = false, 273 .hugetlb = false, 274 }, 275 { 276 .sizes = &pagesize, 277 .nr_sizes = &nr_pagesizes, 278 .private = false, 279 .swapout = true, 280 .hugetlb = false, 281 }, 282 { 283 .sizes = &pagesize, 284 .nr_sizes = &nr_pagesizes, 285 .private = true, 286 .swapout = true, 287 .hugetlb = false, 288 }, 289 290 /* thp. */ 291 { 292 .sizes = thpsizes, 293 .nr_sizes = &nr_thpsizes, 294 .private = false, 295 .swapout = false, 296 .hugetlb = false, 297 }, 298 { 299 .sizes = thpsizes, 300 .nr_sizes = &nr_thpsizes, 301 .private = true, 302 .swapout = false, 303 .hugetlb = false, 304 }, 305 { 306 .sizes = thpsizes, 307 .nr_sizes = &nr_thpsizes, 308 .private = false, 309 .swapout = true, 310 .hugetlb = false, 311 }, 312 { 313 .sizes = thpsizes, 314 .nr_sizes = &nr_thpsizes, 315 .private = true, 316 .swapout = true, 317 .hugetlb = false, 318 }, 319 320 /* hugetlb. */ 321 { 322 .sizes = hugetlbsizes, 323 .nr_sizes = &nr_hugetlbsizes, 324 .private = false, 325 .swapout = false, 326 .hugetlb = true, 327 }, 328 { 329 .sizes = hugetlbsizes, 330 .nr_sizes = &nr_hugetlbsizes, 331 .private = true, 332 .swapout = false, 333 .hugetlb = true, 334 }, 335 }; 336 337 int main(int argc, char **argv) 338 { 339 struct thp_settings settings; 340 int i, j, plan = 0; 341 342 pagesize = getpagesize(); 343 nr_thpsizes = detect_thp_sizes(thpsizes, ARRAY_SIZE(thpsizes)); 344 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, 345 ARRAY_SIZE(hugetlbsizes)); 346 347 /* If THP is supported, save THP settings and initially disable THP. */ 348 if (nr_thpsizes) { 349 thp_save_settings(); 350 thp_read_settings(&settings); 351 for (i = 0; i < NR_ORDERS; i++) { 352 settings.hugepages[i].enabled = THP_NEVER; 353 settings.shmem_hugepages[i].enabled = SHMEM_NEVER; 354 } 355 thp_push_settings(&settings); 356 } 357 358 for (i = 0; i < ARRAY_SIZE(testcases); i++) 359 plan += *testcases[i].nr_sizes; 360 ksft_set_plan(plan); 361 362 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 363 if (pagemap_fd < 0) 364 ksft_exit_fail_msg("opening pagemap failed\n"); 365 366 for (i = 0; i < ARRAY_SIZE(testcases); i++) { 367 const struct testcase *tc = &testcases[i]; 368 369 for (j = 0; j < *tc->nr_sizes; j++) 370 test_one_folio(tc->sizes[j], tc->private, tc->swapout, 371 tc->hugetlb); 372 } 373 374 /* If THP is supported, restore original THP settings. */ 375 if (nr_thpsizes) 376 thp_restore_settings(); 377 378 i = ksft_get_fail_cnt(); 379 if (i) 380 ksft_exit_fail_msg("%d out of %d tests failed\n", 381 i, ksft_test_num()); 382 ksft_exit_pass(); 383 } 384