1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * GUP long-term page pinning tests.
4 *
5 * Copyright 2023, Red Hat, Inc.
6 *
7 * Author(s): David Hildenbrand <david@redhat.com>
8 */
9 #define _GNU_SOURCE
10 #include <stdlib.h>
11 #include <string.h>
12 #include <stdbool.h>
13 #include <stdint.h>
14 #include <unistd.h>
15 #include <errno.h>
16 #include <fcntl.h>
17 #include <assert.h>
18 #include <sys/mman.h>
19 #include <sys/ioctl.h>
20 #include <sys/vfs.h>
21 #include <linux/magic.h>
22 #include <linux/memfd.h>
23
24 #include "local_config.h"
25 #ifdef LOCAL_CONFIG_HAVE_LIBURING
26 #include <liburing.h>
27 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
28
29 #include "../../../../mm/gup_test.h"
30 #include "../kselftest.h"
31 #include "vm_util.h"
32
33 static size_t pagesize;
34 static int nr_hugetlbsizes;
35 static size_t hugetlbsizes[10];
36 static int gup_fd;
37
get_fs_type(int fd)38 static __fsword_t get_fs_type(int fd)
39 {
40 struct statfs fs;
41 int ret;
42
43 do {
44 ret = fstatfs(fd, &fs);
45 } while (ret && errno == EINTR);
46
47 return ret ? 0 : fs.f_type;
48 }
49
fs_is_unknown(__fsword_t fs_type)50 static bool fs_is_unknown(__fsword_t fs_type)
51 {
52 /*
53 * We only support some filesystems in our tests when dealing with
54 * R/W long-term pinning. For these filesystems, we can be fairly sure
55 * whether they support it or not.
56 */
57 switch (fs_type) {
58 case TMPFS_MAGIC:
59 case HUGETLBFS_MAGIC:
60 case BTRFS_SUPER_MAGIC:
61 case EXT4_SUPER_MAGIC:
62 case XFS_SUPER_MAGIC:
63 return false;
64 default:
65 return true;
66 }
67 }
68
fs_supports_writable_longterm_pinning(__fsword_t fs_type)69 static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type)
70 {
71 assert(!fs_is_unknown(fs_type));
72 switch (fs_type) {
73 case TMPFS_MAGIC:
74 case HUGETLBFS_MAGIC:
75 return true;
76 default:
77 return false;
78 }
79 }
80
81 enum test_type {
82 TEST_TYPE_RO,
83 TEST_TYPE_RO_FAST,
84 TEST_TYPE_RW,
85 TEST_TYPE_RW_FAST,
86 #ifdef LOCAL_CONFIG_HAVE_LIBURING
87 TEST_TYPE_IOURING,
88 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
89 };
90
do_test(int fd,size_t size,enum test_type type,bool shared)91 static void do_test(int fd, size_t size, enum test_type type, bool shared)
92 {
93 __fsword_t fs_type = get_fs_type(fd);
94 bool should_work;
95 char *mem;
96 int ret;
97
98 if (ftruncate(fd, size)) {
99 if (errno == ENOENT) {
100 skip_test_dodgy_fs("ftruncate()");
101 } else {
102 ksft_test_result_fail("ftruncate() failed (%s)\n", strerror(errno));
103 }
104 return;
105 }
106
107 if (fallocate(fd, 0, 0, size)) {
108 if (size == pagesize)
109 ksft_test_result_fail("fallocate() failed (%s)\n", strerror(errno));
110 else
111 ksft_test_result_skip("need more free huge pages\n");
112 return;
113 }
114
115 mem = mmap(NULL, size, PROT_READ | PROT_WRITE,
116 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0);
117 if (mem == MAP_FAILED) {
118 if (size == pagesize || shared)
119 ksft_test_result_fail("mmap() failed (%s)\n", strerror(errno));
120 else
121 ksft_test_result_skip("need more free huge pages\n");
122 return;
123 }
124
125 /* Fault in the page such that GUP-fast can pin it directly. */
126 memset(mem, 0, size);
127
128 switch (type) {
129 case TEST_TYPE_RO:
130 case TEST_TYPE_RO_FAST:
131 /*
132 * Cover more cases regarding unsharing decisions when
133 * long-term R/O pinning by mapping the page R/O.
134 */
135 ret = mprotect(mem, size, PROT_READ);
136 if (ret) {
137 ksft_test_result_fail("mprotect() failed (%s)\n", strerror(errno));
138 goto munmap;
139 }
140 /* FALLTHROUGH */
141 case TEST_TYPE_RW:
142 case TEST_TYPE_RW_FAST: {
143 struct pin_longterm_test args;
144 const bool fast = type == TEST_TYPE_RO_FAST ||
145 type == TEST_TYPE_RW_FAST;
146 const bool rw = type == TEST_TYPE_RW ||
147 type == TEST_TYPE_RW_FAST;
148
149 if (gup_fd < 0) {
150 ksft_test_result_skip("gup_test not available\n");
151 break;
152 }
153
154 if (rw && shared && fs_is_unknown(fs_type)) {
155 ksft_test_result_skip("Unknown filesystem\n");
156 return;
157 }
158 /*
159 * R/O pinning or pinning in a private mapping is always
160 * expected to work. Otherwise, we expect long-term R/W pinning
161 * to only succeed for special fielesystems.
162 */
163 should_work = !shared || !rw ||
164 fs_supports_writable_longterm_pinning(fs_type);
165
166 args.addr = (__u64)(uintptr_t)mem;
167 args.size = size;
168 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0;
169 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0;
170 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args);
171 if (ret && errno == EINVAL) {
172 ksft_test_result_skip("PIN_LONGTERM_TEST_START failed (EINVAL)n");
173 break;
174 } else if (ret && errno == EFAULT) {
175 ksft_test_result(!should_work, "Should have failed\n");
176 break;
177 } else if (ret) {
178 ksft_test_result_fail("PIN_LONGTERM_TEST_START failed (%s)\n",
179 strerror(errno));
180 break;
181 }
182
183 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP))
184 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed (%s)\n",
185 strerror(errno));
186
187 /*
188 * TODO: if the kernel ever supports long-term R/W pinning on
189 * some previously unsupported filesystems, we might want to
190 * perform some additional tests for possible data corruptions.
191 */
192 ksft_test_result(should_work, "Should have worked\n");
193 break;
194 }
195 #ifdef LOCAL_CONFIG_HAVE_LIBURING
196 case TEST_TYPE_IOURING: {
197 struct io_uring ring;
198 struct iovec iov;
199
200 /* io_uring always pins pages writable. */
201 if (shared && fs_is_unknown(fs_type)) {
202 ksft_test_result_skip("Unknown filesystem\n");
203 return;
204 }
205 should_work = !shared ||
206 fs_supports_writable_longterm_pinning(fs_type);
207
208 /* Skip on errors, as we might just lack kernel support. */
209 ret = io_uring_queue_init(1, &ring, 0);
210 if (ret < 0) {
211 ksft_test_result_skip("io_uring_queue_init() failed (%s)\n",
212 strerror(-ret));
213 break;
214 }
215 /*
216 * Register the range as a fixed buffer. This will FOLL_WRITE |
217 * FOLL_PIN | FOLL_LONGTERM the range.
218 */
219 iov.iov_base = mem;
220 iov.iov_len = size;
221 ret = io_uring_register_buffers(&ring, &iov, 1);
222 /* Only new kernels return EFAULT. */
223 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP ||
224 errno == EFAULT)) {
225 ksft_test_result(!should_work, "Should have failed (%s)\n",
226 strerror(errno));
227 } else if (ret) {
228 /*
229 * We might just lack support or have insufficient
230 * MEMLOCK limits.
231 */
232 ksft_test_result_skip("io_uring_register_buffers() failed (%s)\n",
233 strerror(-ret));
234 } else {
235 ksft_test_result(should_work, "Should have worked\n");
236 io_uring_unregister_buffers(&ring);
237 }
238
239 io_uring_queue_exit(&ring);
240 break;
241 }
242 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
243 default:
244 assert(false);
245 }
246
247 munmap:
248 munmap(mem, size);
249 }
250
251 typedef void (*test_fn)(int fd, size_t size);
252
run_with_memfd(test_fn fn,const char * desc)253 static void run_with_memfd(test_fn fn, const char *desc)
254 {
255 int fd;
256
257 ksft_print_msg("[RUN] %s ... with memfd\n", desc);
258
259 fd = memfd_create("test", 0);
260 if (fd < 0) {
261 ksft_test_result_fail("memfd_create() failed (%s)\n", strerror(errno));
262 return;
263 }
264
265 fn(fd, pagesize);
266 close(fd);
267 }
268
run_with_tmpfile(test_fn fn,const char * desc)269 static void run_with_tmpfile(test_fn fn, const char *desc)
270 {
271 FILE *file;
272 int fd;
273
274 ksft_print_msg("[RUN] %s ... with tmpfile\n", desc);
275
276 file = tmpfile();
277 if (!file) {
278 ksft_test_result_fail("tmpfile() failed (%s)\n", strerror(errno));
279 return;
280 }
281
282 fd = fileno(file);
283 if (fd < 0) {
284 ksft_test_result_fail("fileno() failed (%s)\n", strerror(errno));
285 goto close;
286 }
287
288 fn(fd, pagesize);
289 close:
290 fclose(file);
291 }
292
run_with_local_tmpfile(test_fn fn,const char * desc)293 static void run_with_local_tmpfile(test_fn fn, const char *desc)
294 {
295 char filename[] = __FILE__"_tmpfile_XXXXXX";
296 int fd;
297
298 ksft_print_msg("[RUN] %s ... with local tmpfile\n", desc);
299
300 fd = mkstemp(filename);
301 if (fd < 0) {
302 ksft_test_result_fail("mkstemp() failed (%s)\n", strerror(errno));
303 return;
304 }
305
306 if (unlink(filename)) {
307 ksft_test_result_fail("unlink() failed (%s)\n", strerror(errno));
308 goto close;
309 }
310
311 fn(fd, pagesize);
312 close:
313 close(fd);
314 }
315
run_with_memfd_hugetlb(test_fn fn,const char * desc,size_t hugetlbsize)316 static void run_with_memfd_hugetlb(test_fn fn, const char *desc,
317 size_t hugetlbsize)
318 {
319 int flags = MFD_HUGETLB;
320 int fd;
321
322 ksft_print_msg("[RUN] %s ... with memfd hugetlb (%zu kB)\n", desc,
323 hugetlbsize / 1024);
324
325 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT;
326
327 fd = memfd_create("test", flags);
328 if (fd < 0) {
329 ksft_test_result_skip("memfd_create() failed (%s)\n", strerror(errno));
330 return;
331 }
332
333 fn(fd, hugetlbsize);
334 close(fd);
335 }
336
337 struct test_case {
338 const char *desc;
339 test_fn fn;
340 };
341
test_shared_rw_pin(int fd,size_t size)342 static void test_shared_rw_pin(int fd, size_t size)
343 {
344 do_test(fd, size, TEST_TYPE_RW, true);
345 }
346
test_shared_rw_fast_pin(int fd,size_t size)347 static void test_shared_rw_fast_pin(int fd, size_t size)
348 {
349 do_test(fd, size, TEST_TYPE_RW_FAST, true);
350 }
351
test_shared_ro_pin(int fd,size_t size)352 static void test_shared_ro_pin(int fd, size_t size)
353 {
354 do_test(fd, size, TEST_TYPE_RO, true);
355 }
356
test_shared_ro_fast_pin(int fd,size_t size)357 static void test_shared_ro_fast_pin(int fd, size_t size)
358 {
359 do_test(fd, size, TEST_TYPE_RO_FAST, true);
360 }
361
test_private_rw_pin(int fd,size_t size)362 static void test_private_rw_pin(int fd, size_t size)
363 {
364 do_test(fd, size, TEST_TYPE_RW, false);
365 }
366
test_private_rw_fast_pin(int fd,size_t size)367 static void test_private_rw_fast_pin(int fd, size_t size)
368 {
369 do_test(fd, size, TEST_TYPE_RW_FAST, false);
370 }
371
test_private_ro_pin(int fd,size_t size)372 static void test_private_ro_pin(int fd, size_t size)
373 {
374 do_test(fd, size, TEST_TYPE_RO, false);
375 }
376
test_private_ro_fast_pin(int fd,size_t size)377 static void test_private_ro_fast_pin(int fd, size_t size)
378 {
379 do_test(fd, size, TEST_TYPE_RO_FAST, false);
380 }
381
382 #ifdef LOCAL_CONFIG_HAVE_LIBURING
test_shared_iouring(int fd,size_t size)383 static void test_shared_iouring(int fd, size_t size)
384 {
385 do_test(fd, size, TEST_TYPE_IOURING, true);
386 }
387
test_private_iouring(int fd,size_t size)388 static void test_private_iouring(int fd, size_t size)
389 {
390 do_test(fd, size, TEST_TYPE_IOURING, false);
391 }
392 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
393
394 static const struct test_case test_cases[] = {
395 {
396 "R/W longterm GUP pin in MAP_SHARED file mapping",
397 test_shared_rw_pin,
398 },
399 {
400 "R/W longterm GUP-fast pin in MAP_SHARED file mapping",
401 test_shared_rw_fast_pin,
402 },
403 {
404 "R/O longterm GUP pin in MAP_SHARED file mapping",
405 test_shared_ro_pin,
406 },
407 {
408 "R/O longterm GUP-fast pin in MAP_SHARED file mapping",
409 test_shared_ro_fast_pin,
410 },
411 {
412 "R/W longterm GUP pin in MAP_PRIVATE file mapping",
413 test_private_rw_pin,
414 },
415 {
416 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping",
417 test_private_rw_fast_pin,
418 },
419 {
420 "R/O longterm GUP pin in MAP_PRIVATE file mapping",
421 test_private_ro_pin,
422 },
423 {
424 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping",
425 test_private_ro_fast_pin,
426 },
427 #ifdef LOCAL_CONFIG_HAVE_LIBURING
428 {
429 "io_uring fixed buffer with MAP_SHARED file mapping",
430 test_shared_iouring,
431 },
432 {
433 "io_uring fixed buffer with MAP_PRIVATE file mapping",
434 test_private_iouring,
435 },
436 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
437 };
438
run_test_case(struct test_case const * test_case)439 static void run_test_case(struct test_case const *test_case)
440 {
441 int i;
442
443 run_with_memfd(test_case->fn, test_case->desc);
444 run_with_tmpfile(test_case->fn, test_case->desc);
445 run_with_local_tmpfile(test_case->fn, test_case->desc);
446 for (i = 0; i < nr_hugetlbsizes; i++)
447 run_with_memfd_hugetlb(test_case->fn, test_case->desc,
448 hugetlbsizes[i]);
449 }
450
tests_per_test_case(void)451 static int tests_per_test_case(void)
452 {
453 return 3 + nr_hugetlbsizes;
454 }
455
main(int argc,char ** argv)456 int main(int argc, char **argv)
457 {
458 int i, err;
459
460 pagesize = getpagesize();
461 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes,
462 ARRAY_SIZE(hugetlbsizes));
463
464 ksft_print_header();
465 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case());
466
467 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
468
469 for (i = 0; i < ARRAY_SIZE(test_cases); i++)
470 run_test_case(&test_cases[i]);
471
472 err = ksft_get_fail_cnt();
473 if (err)
474 ksft_exit_fail_msg("%d out of %d tests failed\n",
475 err, ksft_test_num());
476 ksft_exit_pass();
477 }
478