1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2020 Facebook */
3 #include <test_progs.h>
4 #include <unistd.h>
5 #include <sys/syscall.h>
6 #include <task_local_storage_helpers.h>
7 #include "bpf_iter_ipv6_route.skel.h"
8 #include "bpf_iter_netlink.skel.h"
9 #include "bpf_iter_bpf_map.skel.h"
10 #include "bpf_iter_tasks.skel.h"
11 #include "bpf_iter_task_stack.skel.h"
12 #include "bpf_iter_task_file.skel.h"
13 #include "bpf_iter_task_vmas.skel.h"
14 #include "bpf_iter_task_btf.skel.h"
15 #include "bpf_iter_tcp4.skel.h"
16 #include "bpf_iter_tcp6.skel.h"
17 #include "bpf_iter_udp4.skel.h"
18 #include "bpf_iter_udp6.skel.h"
19 #include "bpf_iter_unix.skel.h"
20 #include "bpf_iter_vma_offset.skel.h"
21 #include "bpf_iter_test_kern1.skel.h"
22 #include "bpf_iter_test_kern2.skel.h"
23 #include "bpf_iter_test_kern3.skel.h"
24 #include "bpf_iter_test_kern4.skel.h"
25 #include "bpf_iter_bpf_hash_map.skel.h"
26 #include "bpf_iter_bpf_percpu_hash_map.skel.h"
27 #include "bpf_iter_bpf_array_map.skel.h"
28 #include "bpf_iter_bpf_percpu_array_map.skel.h"
29 #include "bpf_iter_bpf_sk_storage_helpers.skel.h"
30 #include "bpf_iter_bpf_sk_storage_map.skel.h"
31 #include "bpf_iter_test_kern5.skel.h"
32 #include "bpf_iter_test_kern6.skel.h"
33 #include "bpf_iter_bpf_link.skel.h"
34 #include "bpf_iter_ksym.skel.h"
35 #include "bpf_iter_sockmap.skel.h"
36
test_btf_id_or_null(void)37 static void test_btf_id_or_null(void)
38 {
39 struct bpf_iter_test_kern3 *skel;
40
41 skel = bpf_iter_test_kern3__open_and_load();
42 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
43 bpf_iter_test_kern3__destroy(skel);
44 return;
45 }
46 }
47
do_dummy_read_opts(struct bpf_program * prog,struct bpf_iter_attach_opts * opts)48 static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
49 {
50 struct bpf_link *link;
51 char buf[16] = {};
52 int iter_fd, len;
53
54 link = bpf_program__attach_iter(prog, opts);
55 if (!ASSERT_OK_PTR(link, "attach_iter"))
56 return;
57
58 iter_fd = bpf_iter_create(bpf_link__fd(link));
59 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
60 goto free_link;
61
62 /* not check contents, but ensure read() ends without error */
63 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
64 ;
65 ASSERT_GE(len, 0, "read");
66
67 close(iter_fd);
68
69 free_link:
70 bpf_link__destroy(link);
71 }
72
do_dummy_read(struct bpf_program * prog)73 static void do_dummy_read(struct bpf_program *prog)
74 {
75 do_dummy_read_opts(prog, NULL);
76 }
77
do_read_map_iter_fd(struct bpf_object_skeleton ** skel,struct bpf_program * prog,struct bpf_map * map)78 static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
79 struct bpf_map *map)
80 {
81 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
82 union bpf_iter_link_info linfo;
83 struct bpf_link *link;
84 char buf[16] = {};
85 int iter_fd, len;
86
87 memset(&linfo, 0, sizeof(linfo));
88 linfo.map.map_fd = bpf_map__fd(map);
89 opts.link_info = &linfo;
90 opts.link_info_len = sizeof(linfo);
91 link = bpf_program__attach_iter(prog, &opts);
92 if (!ASSERT_OK_PTR(link, "attach_map_iter"))
93 return;
94
95 iter_fd = bpf_iter_create(bpf_link__fd(link));
96 if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
97 bpf_link__destroy(link);
98 return;
99 }
100
101 /* Close link and map fd prematurely */
102 bpf_link__destroy(link);
103 bpf_object__destroy_skeleton(*skel);
104 *skel = NULL;
105
106 /* Try to let map free work to run first if map is freed */
107 usleep(100);
108 /* Memory used by both sock map and sock local storage map are
109 * freed after two synchronize_rcu() calls, so wait for it
110 */
111 kern_sync_rcu();
112 kern_sync_rcu();
113
114 /* Read after both map fd and link fd are closed */
115 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
116 ;
117 ASSERT_GE(len, 0, "read_iterator");
118
119 close(iter_fd);
120 }
121
read_fd_into_buffer(int fd,char * buf,int size)122 static int read_fd_into_buffer(int fd, char *buf, int size)
123 {
124 int bufleft = size;
125 int len;
126
127 do {
128 len = read(fd, buf, bufleft);
129 if (len > 0) {
130 buf += len;
131 bufleft -= len;
132 }
133 } while (len > 0);
134
135 return len < 0 ? len : size - bufleft;
136 }
137
test_ipv6_route(void)138 static void test_ipv6_route(void)
139 {
140 struct bpf_iter_ipv6_route *skel;
141
142 skel = bpf_iter_ipv6_route__open_and_load();
143 if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
144 return;
145
146 do_dummy_read(skel->progs.dump_ipv6_route);
147
148 bpf_iter_ipv6_route__destroy(skel);
149 }
150
test_netlink(void)151 static void test_netlink(void)
152 {
153 struct bpf_iter_netlink *skel;
154
155 skel = bpf_iter_netlink__open_and_load();
156 if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
157 return;
158
159 do_dummy_read(skel->progs.dump_netlink);
160
161 bpf_iter_netlink__destroy(skel);
162 }
163
test_bpf_map(void)164 static void test_bpf_map(void)
165 {
166 struct bpf_iter_bpf_map *skel;
167
168 skel = bpf_iter_bpf_map__open_and_load();
169 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
170 return;
171
172 do_dummy_read(skel->progs.dump_bpf_map);
173
174 bpf_iter_bpf_map__destroy(skel);
175 }
176
check_bpf_link_info(const struct bpf_program * prog)177 static void check_bpf_link_info(const struct bpf_program *prog)
178 {
179 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
180 union bpf_iter_link_info linfo;
181 struct bpf_link_info info = {};
182 struct bpf_link *link;
183 __u32 info_len;
184 int err;
185
186 memset(&linfo, 0, sizeof(linfo));
187 linfo.task.tid = getpid();
188 opts.link_info = &linfo;
189 opts.link_info_len = sizeof(linfo);
190
191 link = bpf_program__attach_iter(prog, &opts);
192 if (!ASSERT_OK_PTR(link, "attach_iter"))
193 return;
194
195 info_len = sizeof(info);
196 err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
197 ASSERT_OK(err, "bpf_link_get_info_by_fd");
198 ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
199
200 bpf_link__destroy(link);
201 }
202
203 static pthread_mutex_t do_nothing_mutex;
204
do_nothing_wait(void * arg)205 static void *do_nothing_wait(void *arg)
206 {
207 pthread_mutex_lock(&do_nothing_mutex);
208 pthread_mutex_unlock(&do_nothing_mutex);
209
210 pthread_exit(arg);
211 }
212
test_task_common_nocheck(struct bpf_iter_attach_opts * opts,int * num_unknown,int * num_known)213 static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
214 int *num_unknown, int *num_known)
215 {
216 struct bpf_iter_tasks *skel;
217 pthread_t thread_id;
218 void *ret;
219
220 skel = bpf_iter_tasks__open_and_load();
221 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
222 return;
223
224 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
225
226 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
227 "pthread_create");
228
229 skel->bss->tid = sys_gettid();
230
231 do_dummy_read_opts(skel->progs.dump_task, opts);
232
233 *num_unknown = skel->bss->num_unknown_tid;
234 *num_known = skel->bss->num_known_tid;
235
236 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
237 ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
238 "pthread_join");
239
240 bpf_iter_tasks__destroy(skel);
241 }
242
test_task_common(struct bpf_iter_attach_opts * opts,int num_unknown,int num_known)243 static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
244 {
245 int num_unknown_tid, num_known_tid;
246
247 test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
248 ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
249 ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
250 }
251
run_test_task_tid(void * arg)252 static void *run_test_task_tid(void *arg)
253 {
254 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
255 union bpf_iter_link_info linfo;
256 int num_unknown_tid, num_known_tid;
257
258 ASSERT_NEQ(getpid(), sys_gettid(), "check_new_thread_id");
259
260 memset(&linfo, 0, sizeof(linfo));
261 linfo.task.tid = sys_gettid();
262 opts.link_info = &linfo;
263 opts.link_info_len = sizeof(linfo);
264 test_task_common(&opts, 0, 1);
265
266 linfo.task.tid = 0;
267 linfo.task.pid = getpid();
268 /* This includes the parent thread, this thread, watchdog timer thread
269 * and the do_nothing_wait thread
270 */
271 test_task_common(&opts, 3, 1);
272
273 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
274 ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");
275 ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
276
277 return NULL;
278 }
279
test_task_tid(void)280 static void test_task_tid(void)
281 {
282 pthread_t thread_id;
283
284 /* Create a new thread so pid and tid aren't the same */
285 ASSERT_OK(pthread_create(&thread_id, NULL, &run_test_task_tid, NULL),
286 "pthread_create");
287 ASSERT_FALSE(pthread_join(thread_id, NULL), "pthread_join");
288 }
289
test_task_pid(void)290 static void test_task_pid(void)
291 {
292 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
293 union bpf_iter_link_info linfo;
294
295 memset(&linfo, 0, sizeof(linfo));
296 linfo.task.pid = getpid();
297 opts.link_info = &linfo;
298 opts.link_info_len = sizeof(linfo);
299
300 test_task_common(&opts, 2, 1);
301 }
302
test_task_pidfd(void)303 static void test_task_pidfd(void)
304 {
305 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
306 union bpf_iter_link_info linfo;
307 int pidfd;
308
309 pidfd = sys_pidfd_open(getpid(), 0);
310 if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
311 return;
312
313 memset(&linfo, 0, sizeof(linfo));
314 linfo.task.pid_fd = pidfd;
315 opts.link_info = &linfo;
316 opts.link_info_len = sizeof(linfo);
317
318 test_task_common(&opts, 2, 1);
319
320 close(pidfd);
321 }
322
test_task_sleepable(void)323 static void test_task_sleepable(void)
324 {
325 struct bpf_iter_tasks *skel;
326 int pid, status, err, data_pipe[2], finish_pipe[2], c;
327 char *test_data = NULL;
328 char *test_data_long = NULL;
329 char *data[2];
330
331 if (!ASSERT_OK(pipe(data_pipe), "data_pipe") ||
332 !ASSERT_OK(pipe(finish_pipe), "finish_pipe"))
333 return;
334
335 skel = bpf_iter_tasks__open_and_load();
336 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
337 return;
338
339 pid = fork();
340 if (!ASSERT_GE(pid, 0, "fork"))
341 return;
342
343 if (pid == 0) {
344 /* child */
345 close(data_pipe[0]);
346 close(finish_pipe[1]);
347
348 test_data = malloc(sizeof(char) * 10);
349 strncpy(test_data, "test_data", 10);
350 test_data[9] = '\0';
351
352 test_data_long = malloc(sizeof(char) * 5000);
353 for (int i = 0; i < 5000; ++i) {
354 if (i % 2 == 0)
355 test_data_long[i] = 'b';
356 else
357 test_data_long[i] = 'a';
358 }
359 test_data_long[4999] = '\0';
360
361 data[0] = test_data;
362 data[1] = test_data_long;
363
364 write(data_pipe[1], &data, sizeof(data));
365
366 /* keep child alive until after the test */
367 err = read(finish_pipe[0], &c, 1);
368 if (err != 1)
369 exit(-1);
370
371 close(data_pipe[1]);
372 close(finish_pipe[0]);
373 _exit(0);
374 }
375
376 /* parent */
377 close(data_pipe[1]);
378 close(finish_pipe[0]);
379
380 err = read(data_pipe[0], &data, sizeof(data));
381 ASSERT_EQ(err, sizeof(data), "read_check");
382
383 skel->bss->user_ptr = data[0];
384 skel->bss->user_ptr_long = data[1];
385 skel->bss->pid = pid;
386
387 do_dummy_read(skel->progs.dump_task_sleepable);
388
389 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
390 "num_expected_failure_copy_from_user_task");
391 ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
392 "num_success_copy_from_user_task");
393 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task_str, 0,
394 "num_expected_failure_copy_from_user_task_str");
395 ASSERT_GT(skel->bss->num_success_copy_from_user_task_str, 0,
396 "num_success_copy_from_user_task_str");
397
398 bpf_iter_tasks__destroy(skel);
399
400 write(finish_pipe[1], &c, 1);
401 err = waitpid(pid, &status, 0);
402 ASSERT_EQ(err, pid, "waitpid");
403 ASSERT_EQ(status, 0, "zero_child_exit");
404
405 close(data_pipe[0]);
406 close(finish_pipe[1]);
407 }
408
test_task_stack(void)409 static void test_task_stack(void)
410 {
411 struct bpf_iter_task_stack *skel;
412
413 skel = bpf_iter_task_stack__open_and_load();
414 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
415 return;
416
417 do_dummy_read(skel->progs.dump_task_stack);
418 do_dummy_read(skel->progs.get_task_user_stacks);
419
420 ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");
421
422 bpf_iter_task_stack__destroy(skel);
423 }
424
test_task_file(void)425 static void test_task_file(void)
426 {
427 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
428 struct bpf_iter_task_file *skel;
429 union bpf_iter_link_info linfo;
430 pthread_t thread_id;
431 void *ret;
432
433 skel = bpf_iter_task_file__open_and_load();
434 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
435 return;
436
437 skel->bss->tgid = getpid();
438
439 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
440
441 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
442 "pthread_create");
443
444 memset(&linfo, 0, sizeof(linfo));
445 linfo.task.tid = getpid();
446 opts.link_info = &linfo;
447 opts.link_info_len = sizeof(linfo);
448
449 do_dummy_read_opts(skel->progs.dump_task_file, &opts);
450
451 ASSERT_EQ(skel->bss->count, 0, "check_count");
452 ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
453
454 skel->bss->last_tgid = 0;
455 skel->bss->count = 0;
456 skel->bss->unique_tgid_count = 0;
457
458 do_dummy_read(skel->progs.dump_task_file);
459
460 ASSERT_EQ(skel->bss->count, 0, "check_count");
461 ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
462
463 check_bpf_link_info(skel->progs.dump_task_file);
464
465 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
466 ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
467 ASSERT_NULL(ret, "pthread_join");
468
469 bpf_iter_task_file__destroy(skel);
470 }
471
472 #define TASKBUFSZ 32768
473
474 static char taskbuf[TASKBUFSZ];
475
do_btf_read(struct bpf_iter_task_btf * skel)476 static int do_btf_read(struct bpf_iter_task_btf *skel)
477 {
478 struct bpf_program *prog = skel->progs.dump_task_struct;
479 struct bpf_iter_task_btf__bss *bss = skel->bss;
480 int iter_fd = -1, err;
481 struct bpf_link *link;
482 char *buf = taskbuf;
483 int ret = 0;
484
485 link = bpf_program__attach_iter(prog, NULL);
486 if (!ASSERT_OK_PTR(link, "attach_iter"))
487 return ret;
488
489 iter_fd = bpf_iter_create(bpf_link__fd(link));
490 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
491 goto free_link;
492
493 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
494 if (bss->skip) {
495 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
496 ret = 1;
497 test__skip();
498 goto free_link;
499 }
500
501 if (!ASSERT_GE(err, 0, "read"))
502 goto free_link;
503
504 ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
505 "check for btf representation of task_struct in iter data");
506 free_link:
507 if (iter_fd > 0)
508 close(iter_fd);
509 bpf_link__destroy(link);
510 return ret;
511 }
512
test_task_btf(void)513 static void test_task_btf(void)
514 {
515 struct bpf_iter_task_btf__bss *bss;
516 struct bpf_iter_task_btf *skel;
517 int ret;
518
519 skel = bpf_iter_task_btf__open_and_load();
520 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
521 return;
522
523 bss = skel->bss;
524
525 ret = do_btf_read(skel);
526 if (ret)
527 goto cleanup;
528
529 if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
530 goto cleanup;
531
532 ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");
533
534 cleanup:
535 bpf_iter_task_btf__destroy(skel);
536 }
537
test_tcp4(void)538 static void test_tcp4(void)
539 {
540 struct bpf_iter_tcp4 *skel;
541
542 skel = bpf_iter_tcp4__open_and_load();
543 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
544 return;
545
546 do_dummy_read(skel->progs.dump_tcp4);
547
548 bpf_iter_tcp4__destroy(skel);
549 }
550
test_tcp6(void)551 static void test_tcp6(void)
552 {
553 struct bpf_iter_tcp6 *skel;
554
555 skel = bpf_iter_tcp6__open_and_load();
556 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
557 return;
558
559 do_dummy_read(skel->progs.dump_tcp6);
560
561 bpf_iter_tcp6__destroy(skel);
562 }
563
test_udp4(void)564 static void test_udp4(void)
565 {
566 struct bpf_iter_udp4 *skel;
567
568 skel = bpf_iter_udp4__open_and_load();
569 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
570 return;
571
572 do_dummy_read(skel->progs.dump_udp4);
573
574 bpf_iter_udp4__destroy(skel);
575 }
576
test_udp6(void)577 static void test_udp6(void)
578 {
579 struct bpf_iter_udp6 *skel;
580
581 skel = bpf_iter_udp6__open_and_load();
582 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
583 return;
584
585 do_dummy_read(skel->progs.dump_udp6);
586
587 bpf_iter_udp6__destroy(skel);
588 }
589
test_unix(void)590 static void test_unix(void)
591 {
592 struct bpf_iter_unix *skel;
593
594 skel = bpf_iter_unix__open_and_load();
595 if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
596 return;
597
598 do_dummy_read(skel->progs.dump_unix);
599
600 bpf_iter_unix__destroy(skel);
601 }
602
603 /* The expected string is less than 16 bytes */
do_read_with_fd(int iter_fd,const char * expected,bool read_one_char)604 static int do_read_with_fd(int iter_fd, const char *expected,
605 bool read_one_char)
606 {
607 int len, read_buf_len, start;
608 char buf[16] = {};
609
610 read_buf_len = read_one_char ? 1 : 16;
611 start = 0;
612 while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
613 start += len;
614 if (!ASSERT_LT(start, 16, "read"))
615 return -1;
616 read_buf_len = read_one_char ? 1 : 16 - start;
617 }
618 if (!ASSERT_GE(len, 0, "read"))
619 return -1;
620
621 if (!ASSERT_STREQ(buf, expected, "read"))
622 return -1;
623
624 return 0;
625 }
626
test_anon_iter(bool read_one_char)627 static void test_anon_iter(bool read_one_char)
628 {
629 struct bpf_iter_test_kern1 *skel;
630 struct bpf_link *link;
631 int iter_fd, err;
632
633 skel = bpf_iter_test_kern1__open_and_load();
634 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
635 return;
636
637 err = bpf_iter_test_kern1__attach(skel);
638 if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
639 goto out;
640 }
641
642 link = skel->links.dump_task;
643 iter_fd = bpf_iter_create(bpf_link__fd(link));
644 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
645 goto out;
646
647 do_read_with_fd(iter_fd, "abcd", read_one_char);
648 close(iter_fd);
649
650 out:
651 bpf_iter_test_kern1__destroy(skel);
652 }
653
do_read(const char * path,const char * expected)654 static int do_read(const char *path, const char *expected)
655 {
656 int err, iter_fd;
657
658 iter_fd = open(path, O_RDONLY);
659 if (!ASSERT_GE(iter_fd, 0, "open"))
660 return -1;
661
662 err = do_read_with_fd(iter_fd, expected, false);
663 close(iter_fd);
664 return err;
665 }
666
test_file_iter(void)667 static void test_file_iter(void)
668 {
669 const char *path = "/sys/fs/bpf/bpf_iter_test1";
670 struct bpf_iter_test_kern1 *skel1;
671 struct bpf_iter_test_kern2 *skel2;
672 struct bpf_link *link;
673 int err;
674
675 skel1 = bpf_iter_test_kern1__open_and_load();
676 if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
677 return;
678
679 link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
680 if (!ASSERT_OK_PTR(link, "attach_iter"))
681 goto out;
682
683 /* unlink this path if it exists. */
684 unlink(path);
685
686 err = bpf_link__pin(link, path);
687 if (!ASSERT_OK(err, "pin_iter"))
688 goto free_link;
689
690 err = do_read(path, "abcd");
691 if (err)
692 goto unlink_path;
693
694 /* file based iterator seems working fine. Let us a link update
695 * of the underlying link and `cat` the iterator again, its content
696 * should change.
697 */
698 skel2 = bpf_iter_test_kern2__open_and_load();
699 if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
700 goto unlink_path;
701
702 err = bpf_link__update_program(link, skel2->progs.dump_task);
703 if (!ASSERT_OK(err, "update_prog"))
704 goto destroy_skel2;
705
706 do_read(path, "ABCD");
707
708 destroy_skel2:
709 bpf_iter_test_kern2__destroy(skel2);
710 unlink_path:
711 unlink(path);
712 free_link:
713 bpf_link__destroy(link);
714 out:
715 bpf_iter_test_kern1__destroy(skel1);
716 }
717
test_overflow(bool test_e2big_overflow,bool ret1)718 static void test_overflow(bool test_e2big_overflow, bool ret1)
719 {
720 __u32 map_info_len, total_read_len, expected_read_len;
721 int err, iter_fd, map1_fd, map2_fd, len;
722 struct bpf_map_info map_info = {};
723 struct bpf_iter_test_kern4 *skel;
724 struct bpf_link *link;
725 __u32 iter_size;
726 char *buf;
727
728 skel = bpf_iter_test_kern4__open();
729 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
730 return;
731
732 /* create two maps: bpf program will only do bpf_seq_write
733 * for these two maps. The goal is one map output almost
734 * fills seq_file buffer and then the other will trigger
735 * overflow and needs restart.
736 */
737 map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
738 if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
739 goto out;
740 map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
741 if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
742 goto free_map1;
743
744 /* bpf_seq_printf kernel buffer is 8 pages, so one map
745 * bpf_seq_write will mostly fill it, and the other map
746 * will partially fill and then trigger overflow and need
747 * bpf_seq_read restart.
748 */
749 iter_size = sysconf(_SC_PAGE_SIZE) << 3;
750
751 if (test_e2big_overflow) {
752 skel->rodata->print_len = (iter_size + 8) / 8;
753 expected_read_len = 2 * (iter_size + 8);
754 } else if (!ret1) {
755 skel->rodata->print_len = (iter_size - 8) / 8;
756 expected_read_len = 2 * (iter_size - 8);
757 } else {
758 skel->rodata->print_len = 1;
759 expected_read_len = 2 * 8;
760 }
761 skel->rodata->ret1 = ret1;
762
763 if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
764 "bpf_iter_test_kern4__load"))
765 goto free_map2;
766
767 /* setup filtering map_id in bpf program */
768 map_info_len = sizeof(map_info);
769 err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
770 if (!ASSERT_OK(err, "get_map_info"))
771 goto free_map2;
772 skel->bss->map1_id = map_info.id;
773
774 err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
775 if (!ASSERT_OK(err, "get_map_info"))
776 goto free_map2;
777 skel->bss->map2_id = map_info.id;
778
779 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
780 if (!ASSERT_OK_PTR(link, "attach_iter"))
781 goto free_map2;
782
783 iter_fd = bpf_iter_create(bpf_link__fd(link));
784 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
785 goto free_link;
786
787 buf = malloc(expected_read_len);
788 if (!ASSERT_OK_PTR(buf, "malloc"))
789 goto close_iter;
790
791 /* do read */
792 total_read_len = 0;
793 if (test_e2big_overflow) {
794 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
795 total_read_len += len;
796
797 ASSERT_EQ(len, -1, "read");
798 ASSERT_EQ(errno, E2BIG, "read");
799 goto free_buf;
800 } else if (!ret1) {
801 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
802 total_read_len += len;
803
804 if (!ASSERT_GE(len, 0, "read"))
805 goto free_buf;
806 } else {
807 do {
808 len = read(iter_fd, buf, expected_read_len);
809 if (len > 0)
810 total_read_len += len;
811 } while (len > 0 || len == -EAGAIN);
812
813 if (!ASSERT_GE(len, 0, "read"))
814 goto free_buf;
815 }
816
817 if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
818 goto free_buf;
819
820 if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
821 goto free_buf;
822
823 if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
824 goto free_buf;
825
826 ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");
827
828 free_buf:
829 free(buf);
830 close_iter:
831 close(iter_fd);
832 free_link:
833 bpf_link__destroy(link);
834 free_map2:
835 close(map2_fd);
836 free_map1:
837 close(map1_fd);
838 out:
839 bpf_iter_test_kern4__destroy(skel);
840 }
841
test_bpf_hash_map(void)842 static void test_bpf_hash_map(void)
843 {
844 __u32 expected_key_a = 0, expected_key_b = 0;
845 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
846 struct bpf_iter_bpf_hash_map *skel;
847 int err, i, len, map_fd, iter_fd;
848 union bpf_iter_link_info linfo;
849 __u64 val, expected_val = 0;
850 struct bpf_link *link;
851 struct key_t {
852 int a;
853 int b;
854 int c;
855 } key;
856 char buf[64];
857
858 skel = bpf_iter_bpf_hash_map__open();
859 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
860 return;
861
862 skel->bss->in_test_mode = true;
863
864 err = bpf_iter_bpf_hash_map__load(skel);
865 if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
866 goto out;
867
868 /* iterator with hashmap2 and hashmap3 should fail */
869 memset(&linfo, 0, sizeof(linfo));
870 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
871 opts.link_info = &linfo;
872 opts.link_info_len = sizeof(linfo);
873 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
874 if (!ASSERT_ERR_PTR(link, "attach_iter"))
875 goto out;
876
877 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
878 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
879 if (!ASSERT_ERR_PTR(link, "attach_iter"))
880 goto out;
881
882 /* hashmap1 should be good, update map values here */
883 map_fd = bpf_map__fd(skel->maps.hashmap1);
884 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
885 key.a = i + 1;
886 key.b = i + 2;
887 key.c = i + 3;
888 val = i + 4;
889 expected_key_a += key.a;
890 expected_key_b += key.b;
891 expected_val += val;
892
893 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
894 if (!ASSERT_OK(err, "map_update"))
895 goto out;
896 }
897
898 /* Sleepable program is prohibited for hash map iterator */
899 linfo.map.map_fd = map_fd;
900 link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
901 if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
902 goto out;
903
904 linfo.map.map_fd = map_fd;
905 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
906 if (!ASSERT_OK_PTR(link, "attach_iter"))
907 goto out;
908
909 iter_fd = bpf_iter_create(bpf_link__fd(link));
910 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
911 goto free_link;
912
913 /* do some tests */
914 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
915 ;
916 if (!ASSERT_GE(len, 0, "read"))
917 goto close_iter;
918
919 /* test results */
920 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
921 goto close_iter;
922 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
923 goto close_iter;
924 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
925 goto close_iter;
926
927 close_iter:
928 close(iter_fd);
929 free_link:
930 bpf_link__destroy(link);
931 out:
932 bpf_iter_bpf_hash_map__destroy(skel);
933 }
934
test_bpf_percpu_hash_map(void)935 static void test_bpf_percpu_hash_map(void)
936 {
937 __u32 expected_key_a = 0, expected_key_b = 0;
938 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
939 struct bpf_iter_bpf_percpu_hash_map *skel;
940 int err, i, j, len, map_fd, iter_fd;
941 union bpf_iter_link_info linfo;
942 __u32 expected_val = 0;
943 struct bpf_link *link;
944 struct key_t {
945 int a;
946 int b;
947 int c;
948 } key;
949 char buf[64];
950 void *val;
951
952 skel = bpf_iter_bpf_percpu_hash_map__open();
953 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
954 return;
955
956 skel->rodata->num_cpus = bpf_num_possible_cpus();
957 val = malloc(8 * bpf_num_possible_cpus());
958 if (!ASSERT_OK_PTR(val, "malloc"))
959 goto out;
960
961 err = bpf_iter_bpf_percpu_hash_map__load(skel);
962 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
963 goto out;
964
965 /* update map values here */
966 map_fd = bpf_map__fd(skel->maps.hashmap1);
967 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
968 key.a = i + 1;
969 key.b = i + 2;
970 key.c = i + 3;
971 expected_key_a += key.a;
972 expected_key_b += key.b;
973
974 for (j = 0; j < bpf_num_possible_cpus(); j++) {
975 *(__u32 *)(val + j * 8) = i + j;
976 expected_val += i + j;
977 }
978
979 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
980 if (!ASSERT_OK(err, "map_update"))
981 goto out;
982 }
983
984 memset(&linfo, 0, sizeof(linfo));
985 linfo.map.map_fd = map_fd;
986 opts.link_info = &linfo;
987 opts.link_info_len = sizeof(linfo);
988 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
989 if (!ASSERT_OK_PTR(link, "attach_iter"))
990 goto out;
991
992 iter_fd = bpf_iter_create(bpf_link__fd(link));
993 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
994 goto free_link;
995
996 /* do some tests */
997 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
998 ;
999 if (!ASSERT_GE(len, 0, "read"))
1000 goto close_iter;
1001
1002 /* test results */
1003 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
1004 goto close_iter;
1005 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
1006 goto close_iter;
1007 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1008 goto close_iter;
1009
1010 close_iter:
1011 close(iter_fd);
1012 free_link:
1013 bpf_link__destroy(link);
1014 out:
1015 bpf_iter_bpf_percpu_hash_map__destroy(skel);
1016 free(val);
1017 }
1018
test_bpf_array_map(void)1019 static void test_bpf_array_map(void)
1020 {
1021 __u64 val, expected_val = 0, res_first_val, first_val = 0;
1022 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1023 __u32 key, expected_key = 0, res_first_key;
1024 int err, i, map_fd, hash_fd, iter_fd;
1025 struct bpf_iter_bpf_array_map *skel;
1026 union bpf_iter_link_info linfo;
1027 struct bpf_link *link;
1028 char buf[64] = {};
1029 int len, start;
1030
1031 skel = bpf_iter_bpf_array_map__open_and_load();
1032 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1033 return;
1034
1035 map_fd = bpf_map__fd(skel->maps.arraymap1);
1036 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1037 val = i + 4;
1038 expected_key += i;
1039 expected_val += val;
1040
1041 if (i == 0)
1042 first_val = val;
1043
1044 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
1045 if (!ASSERT_OK(err, "map_update"))
1046 goto out;
1047 }
1048
1049 memset(&linfo, 0, sizeof(linfo));
1050 linfo.map.map_fd = map_fd;
1051 opts.link_info = &linfo;
1052 opts.link_info_len = sizeof(linfo);
1053 link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
1054 if (!ASSERT_OK_PTR(link, "attach_iter"))
1055 goto out;
1056
1057 iter_fd = bpf_iter_create(bpf_link__fd(link));
1058 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1059 goto free_link;
1060
1061 /* do some tests */
1062 start = 0;
1063 while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
1064 start += len;
1065 if (!ASSERT_GE(len, 0, "read"))
1066 goto close_iter;
1067
1068 /* test results */
1069 res_first_key = *(__u32 *)buf;
1070 res_first_val = *(__u64 *)(buf + sizeof(__u32));
1071 if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
1072 !ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
1073 goto close_iter;
1074
1075 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1076 goto close_iter;
1077 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1078 goto close_iter;
1079
1080 hash_fd = bpf_map__fd(skel->maps.hashmap1);
1081 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1082 err = bpf_map_lookup_elem(map_fd, &i, &val);
1083 if (!ASSERT_OK(err, "map_lookup arraymap1"))
1084 goto close_iter;
1085 if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
1086 goto close_iter;
1087
1088 val = i + 4;
1089 err = bpf_map_lookup_elem(hash_fd, &val, &key);
1090 if (!ASSERT_OK(err, "map_lookup hashmap1"))
1091 goto close_iter;
1092 if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
1093 goto close_iter;
1094 }
1095
1096 close_iter:
1097 close(iter_fd);
1098 free_link:
1099 bpf_link__destroy(link);
1100 out:
1101 bpf_iter_bpf_array_map__destroy(skel);
1102 }
1103
test_bpf_array_map_iter_fd(void)1104 static void test_bpf_array_map_iter_fd(void)
1105 {
1106 struct bpf_iter_bpf_array_map *skel;
1107
1108 skel = bpf_iter_bpf_array_map__open_and_load();
1109 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1110 return;
1111
1112 do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,
1113 skel->maps.arraymap1);
1114
1115 bpf_iter_bpf_array_map__destroy(skel);
1116 }
1117
test_bpf_percpu_array_map(void)1118 static void test_bpf_percpu_array_map(void)
1119 {
1120 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1121 struct bpf_iter_bpf_percpu_array_map *skel;
1122 __u32 expected_key = 0, expected_val = 0;
1123 union bpf_iter_link_info linfo;
1124 int err, i, j, map_fd, iter_fd;
1125 struct bpf_link *link;
1126 char buf[64];
1127 void *val;
1128 int len;
1129
1130 skel = bpf_iter_bpf_percpu_array_map__open();
1131 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
1132 return;
1133
1134 skel->rodata->num_cpus = bpf_num_possible_cpus();
1135 val = malloc(8 * bpf_num_possible_cpus());
1136 if (!ASSERT_OK_PTR(val, "malloc"))
1137 goto out;
1138
1139 err = bpf_iter_bpf_percpu_array_map__load(skel);
1140 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
1141 goto out;
1142
1143 /* update map values here */
1144 map_fd = bpf_map__fd(skel->maps.arraymap1);
1145 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1146 expected_key += i;
1147
1148 for (j = 0; j < bpf_num_possible_cpus(); j++) {
1149 *(__u32 *)(val + j * 8) = i + j;
1150 expected_val += i + j;
1151 }
1152
1153 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
1154 if (!ASSERT_OK(err, "map_update"))
1155 goto out;
1156 }
1157
1158 memset(&linfo, 0, sizeof(linfo));
1159 linfo.map.map_fd = map_fd;
1160 opts.link_info = &linfo;
1161 opts.link_info_len = sizeof(linfo);
1162 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
1163 if (!ASSERT_OK_PTR(link, "attach_iter"))
1164 goto out;
1165
1166 iter_fd = bpf_iter_create(bpf_link__fd(link));
1167 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1168 goto free_link;
1169
1170 /* do some tests */
1171 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1172 ;
1173 if (!ASSERT_GE(len, 0, "read"))
1174 goto close_iter;
1175
1176 /* test results */
1177 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1178 goto close_iter;
1179 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1180 goto close_iter;
1181
1182 close_iter:
1183 close(iter_fd);
1184 free_link:
1185 bpf_link__destroy(link);
1186 out:
1187 bpf_iter_bpf_percpu_array_map__destroy(skel);
1188 free(val);
1189 }
1190
1191 /* An iterator program deletes all local storage in a map. */
test_bpf_sk_storage_delete(void)1192 static void test_bpf_sk_storage_delete(void)
1193 {
1194 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1195 struct bpf_iter_bpf_sk_storage_helpers *skel;
1196 union bpf_iter_link_info linfo;
1197 int err, len, map_fd, iter_fd;
1198 struct bpf_link *link;
1199 int sock_fd = -1;
1200 __u32 val = 42;
1201 char buf[64];
1202
1203 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1204 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1205 return;
1206
1207 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1208
1209 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1210 if (!ASSERT_GE(sock_fd, 0, "socket"))
1211 goto out;
1212
1213 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1214 if (!ASSERT_OK(err, "map_update"))
1215 goto out;
1216
1217 memset(&linfo, 0, sizeof(linfo));
1218 linfo.map.map_fd = map_fd;
1219 opts.link_info = &linfo;
1220 opts.link_info_len = sizeof(linfo);
1221 link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
1222 &opts);
1223 if (!ASSERT_OK_PTR(link, "attach_iter"))
1224 goto out;
1225
1226 iter_fd = bpf_iter_create(bpf_link__fd(link));
1227 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1228 goto free_link;
1229
1230 /* do some tests */
1231 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1232 ;
1233 if (!ASSERT_GE(len, 0, "read"))
1234 goto close_iter;
1235
1236 /* test results */
1237 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1238
1239 /* Note: The following assertions serve to ensure
1240 * the value was deleted. It does so by asserting
1241 * that bpf_map_lookup_elem has failed. This might
1242 * seem counterintuitive at first.
1243 */
1244 ASSERT_ERR(err, "bpf_map_lookup_elem");
1245 ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");
1246
1247 close_iter:
1248 close(iter_fd);
1249 free_link:
1250 bpf_link__destroy(link);
1251 out:
1252 if (sock_fd >= 0)
1253 close(sock_fd);
1254 bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1255 }
1256
1257 /* This creates a socket and its local storage. It then runs a task_iter BPF
1258 * program that replaces the existing socket local storage with the tgid of the
1259 * only task owning a file descriptor to this socket, this process, prog_tests.
1260 * It then runs a tcp socket iterator that negates the value in the existing
1261 * socket local storage, the test verifies that the resulting value is -pid.
1262 */
test_bpf_sk_storage_get(void)1263 static void test_bpf_sk_storage_get(void)
1264 {
1265 struct bpf_iter_bpf_sk_storage_helpers *skel;
1266 int err, map_fd, val = -1;
1267 int sock_fd = -1;
1268
1269 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1270 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1271 return;
1272
1273 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1274 if (!ASSERT_GE(sock_fd, 0, "socket"))
1275 goto out;
1276
1277 err = listen(sock_fd, 1);
1278 if (!ASSERT_OK(err, "listen"))
1279 goto close_socket;
1280
1281 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1282
1283 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1284 if (!ASSERT_OK(err, "bpf_map_update_elem"))
1285 goto close_socket;
1286
1287 do_dummy_read(skel->progs.fill_socket_owner);
1288
1289 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1290 if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
1291 !ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
1292 goto close_socket;
1293
1294 do_dummy_read(skel->progs.negate_socket_local_storage);
1295
1296 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1297 ASSERT_OK(err, "bpf_map_lookup_elem");
1298 ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");
1299
1300 close_socket:
1301 close(sock_fd);
1302 out:
1303 bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1304 }
1305
test_bpf_sk_storage_map_iter_fd(void)1306 static void test_bpf_sk_storage_map_iter_fd(void)
1307 {
1308 struct bpf_iter_bpf_sk_storage_map *skel;
1309
1310 skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1311 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1312 return;
1313
1314 do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,
1315 skel->maps.sk_stg_map);
1316
1317 bpf_iter_bpf_sk_storage_map__destroy(skel);
1318 }
1319
test_bpf_sk_storage_map(void)1320 static void test_bpf_sk_storage_map(void)
1321 {
1322 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1323 int err, i, len, map_fd, iter_fd, num_sockets;
1324 struct bpf_iter_bpf_sk_storage_map *skel;
1325 union bpf_iter_link_info linfo;
1326 int sock_fd[3] = {-1, -1, -1};
1327 __u32 val, expected_val = 0;
1328 struct bpf_link *link;
1329 char buf[64];
1330
1331 skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1332 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1333 return;
1334
1335 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1336 num_sockets = ARRAY_SIZE(sock_fd);
1337 for (i = 0; i < num_sockets; i++) {
1338 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
1339 if (!ASSERT_GE(sock_fd[i], 0, "socket"))
1340 goto out;
1341
1342 val = i + 1;
1343 expected_val += val;
1344
1345 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
1346 BPF_NOEXIST);
1347 if (!ASSERT_OK(err, "map_update"))
1348 goto out;
1349 }
1350
1351 memset(&linfo, 0, sizeof(linfo));
1352 linfo.map.map_fd = map_fd;
1353 opts.link_info = &linfo;
1354 opts.link_info_len = sizeof(linfo);
1355 link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
1356 err = libbpf_get_error(link);
1357 if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
1358 if (!err)
1359 bpf_link__destroy(link);
1360 goto out;
1361 }
1362
1363 link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
1364 if (!ASSERT_OK_PTR(link, "attach_iter"))
1365 goto out;
1366
1367 iter_fd = bpf_iter_create(bpf_link__fd(link));
1368 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1369 goto free_link;
1370
1371 skel->bss->to_add_val = time(NULL);
1372 /* do some tests */
1373 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1374 ;
1375 if (!ASSERT_GE(len, 0, "read"))
1376 goto close_iter;
1377
1378 /* test results */
1379 if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
1380 goto close_iter;
1381
1382 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1383 goto close_iter;
1384
1385 for (i = 0; i < num_sockets; i++) {
1386 err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
1387 if (!ASSERT_OK(err, "map_lookup") ||
1388 !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
1389 break;
1390 }
1391
1392 close_iter:
1393 close(iter_fd);
1394 free_link:
1395 bpf_link__destroy(link);
1396 out:
1397 for (i = 0; i < num_sockets; i++) {
1398 if (sock_fd[i] >= 0)
1399 close(sock_fd[i]);
1400 }
1401 bpf_iter_bpf_sk_storage_map__destroy(skel);
1402 }
1403
test_rdonly_buf_out_of_bound(void)1404 static void test_rdonly_buf_out_of_bound(void)
1405 {
1406 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1407 struct bpf_iter_test_kern5 *skel;
1408 union bpf_iter_link_info linfo;
1409 struct bpf_link *link;
1410
1411 skel = bpf_iter_test_kern5__open_and_load();
1412 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
1413 return;
1414
1415 memset(&linfo, 0, sizeof(linfo));
1416 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
1417 opts.link_info = &linfo;
1418 opts.link_info_len = sizeof(linfo);
1419 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
1420 if (!ASSERT_ERR_PTR(link, "attach_iter"))
1421 bpf_link__destroy(link);
1422
1423 bpf_iter_test_kern5__destroy(skel);
1424 }
1425
test_buf_neg_offset(void)1426 static void test_buf_neg_offset(void)
1427 {
1428 struct bpf_iter_test_kern6 *skel;
1429
1430 skel = bpf_iter_test_kern6__open_and_load();
1431 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
1432 bpf_iter_test_kern6__destroy(skel);
1433 }
1434
test_link_iter(void)1435 static void test_link_iter(void)
1436 {
1437 struct bpf_iter_bpf_link *skel;
1438
1439 skel = bpf_iter_bpf_link__open_and_load();
1440 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
1441 return;
1442
1443 do_dummy_read(skel->progs.dump_bpf_link);
1444
1445 bpf_iter_bpf_link__destroy(skel);
1446 }
1447
test_ksym_iter(void)1448 static void test_ksym_iter(void)
1449 {
1450 struct bpf_iter_ksym *skel;
1451
1452 skel = bpf_iter_ksym__open_and_load();
1453 if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
1454 return;
1455
1456 do_dummy_read(skel->progs.dump_ksym);
1457
1458 bpf_iter_ksym__destroy(skel);
1459 }
1460
1461 #define CMP_BUFFER_SIZE 1024
1462 static char task_vma_output[CMP_BUFFER_SIZE];
1463 static char proc_maps_output[CMP_BUFFER_SIZE];
1464
1465 /* remove \0 and \t from str, and only keep the first line */
str_strip_first_line(char * str)1466 static void str_strip_first_line(char *str)
1467 {
1468 char *dst = str, *src = str;
1469
1470 do {
1471 if (*src == ' ' || *src == '\t')
1472 src++;
1473 else
1474 *(dst++) = *(src++);
1475
1476 } while (*src != '\0' && *src != '\n');
1477
1478 *dst = '\0';
1479 }
1480
test_task_vma_common(struct bpf_iter_attach_opts * opts)1481 static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
1482 {
1483 int err, iter_fd = -1, proc_maps_fd = -1;
1484 struct bpf_iter_task_vmas *skel;
1485 int len, read_size = 4;
1486 char maps_path[64];
1487
1488 skel = bpf_iter_task_vmas__open();
1489 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1490 return;
1491
1492 skel->bss->pid = getpid();
1493 skel->bss->one_task = opts ? 1 : 0;
1494
1495 err = bpf_iter_task_vmas__load(skel);
1496 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1497 goto out;
1498
1499 skel->links.proc_maps = bpf_program__attach_iter(
1500 skel->progs.proc_maps, opts);
1501
1502 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1503 skel->links.proc_maps = NULL;
1504 goto out;
1505 }
1506
1507 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1508 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1509 goto out;
1510
1511 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1512 * to trigger seq_file corner cases.
1513 */
1514 len = 0;
1515 while (len < CMP_BUFFER_SIZE) {
1516 err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1517 MIN(read_size, CMP_BUFFER_SIZE - len));
1518 if (!err)
1519 break;
1520 if (!ASSERT_GE(err, 0, "read_iter_fd"))
1521 goto out;
1522 len += err;
1523 }
1524 if (opts)
1525 ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
1526
1527 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1528 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1529 proc_maps_fd = open(maps_path, O_RDONLY);
1530 if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
1531 goto out;
1532 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1533 if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
1534 goto out;
1535
1536 /* strip and compare the first line of the two files */
1537 str_strip_first_line(task_vma_output);
1538 str_strip_first_line(proc_maps_output);
1539
1540 ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
1541
1542 check_bpf_link_info(skel->progs.proc_maps);
1543
1544 out:
1545 close(proc_maps_fd);
1546 close(iter_fd);
1547 bpf_iter_task_vmas__destroy(skel);
1548 }
1549
test_task_vma_dead_task(void)1550 static void test_task_vma_dead_task(void)
1551 {
1552 struct bpf_iter_task_vmas *skel;
1553 int wstatus, child_pid = -1;
1554 time_t start_tm, cur_tm;
1555 int err, iter_fd = -1;
1556 int wait_sec = 3;
1557
1558 skel = bpf_iter_task_vmas__open();
1559 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1560 return;
1561
1562 skel->bss->pid = getpid();
1563
1564 err = bpf_iter_task_vmas__load(skel);
1565 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1566 goto out;
1567
1568 skel->links.proc_maps = bpf_program__attach_iter(
1569 skel->progs.proc_maps, NULL);
1570
1571 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1572 skel->links.proc_maps = NULL;
1573 goto out;
1574 }
1575
1576 start_tm = time(NULL);
1577 cur_tm = start_tm;
1578
1579 child_pid = fork();
1580 if (child_pid == 0) {
1581 /* Fork short-lived processes in the background. */
1582 while (cur_tm < start_tm + wait_sec) {
1583 system("echo > /dev/null");
1584 cur_tm = time(NULL);
1585 }
1586 exit(0);
1587 }
1588
1589 if (!ASSERT_GE(child_pid, 0, "fork_child"))
1590 goto out;
1591
1592 while (cur_tm < start_tm + wait_sec) {
1593 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1594 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1595 goto out;
1596
1597 /* Drain all data from iter_fd. */
1598 while (cur_tm < start_tm + wait_sec) {
1599 err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);
1600 if (!ASSERT_GE(err, 0, "read_iter_fd"))
1601 goto out;
1602
1603 cur_tm = time(NULL);
1604
1605 if (err == 0)
1606 break;
1607 }
1608
1609 close(iter_fd);
1610 iter_fd = -1;
1611 }
1612
1613 check_bpf_link_info(skel->progs.proc_maps);
1614
1615 out:
1616 waitpid(child_pid, &wstatus, 0);
1617 close(iter_fd);
1618 bpf_iter_task_vmas__destroy(skel);
1619 }
1620
test_bpf_sockmap_map_iter_fd(void)1621 void test_bpf_sockmap_map_iter_fd(void)
1622 {
1623 struct bpf_iter_sockmap *skel;
1624
1625 skel = bpf_iter_sockmap__open_and_load();
1626 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
1627 return;
1628
1629 do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);
1630
1631 bpf_iter_sockmap__destroy(skel);
1632 }
1633
test_task_vma(void)1634 static void test_task_vma(void)
1635 {
1636 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1637 union bpf_iter_link_info linfo;
1638
1639 memset(&linfo, 0, sizeof(linfo));
1640 linfo.task.tid = getpid();
1641 opts.link_info = &linfo;
1642 opts.link_info_len = sizeof(linfo);
1643
1644 test_task_vma_common(&opts);
1645 test_task_vma_common(NULL);
1646 }
1647
1648 /* uprobe attach point */
trigger_func(int arg)1649 static noinline int trigger_func(int arg)
1650 {
1651 asm volatile ("");
1652 return arg + 1;
1653 }
1654
test_task_vma_offset_common(struct bpf_iter_attach_opts * opts,bool one_proc)1655 static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
1656 {
1657 struct bpf_iter_vma_offset *skel;
1658 char buf[16] = {};
1659 int iter_fd, len;
1660 int pgsz, shift;
1661
1662 skel = bpf_iter_vma_offset__open_and_load();
1663 if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
1664 return;
1665
1666 skel->bss->pid = getpid();
1667 skel->bss->address = (uintptr_t)trigger_func;
1668 for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
1669 ;
1670 skel->bss->page_shift = shift;
1671
1672 skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
1673 if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
1674 goto exit;
1675
1676 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
1677 if (!ASSERT_GT(iter_fd, 0, "create_iter"))
1678 goto exit;
1679
1680 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1681 ;
1682 buf[15] = 0;
1683 ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
1684
1685 ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
1686 if (one_proc)
1687 ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1688 else
1689 ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1690
1691 close(iter_fd);
1692
1693 exit:
1694 bpf_iter_vma_offset__destroy(skel);
1695 }
1696
test_task_vma_offset(void)1697 static void test_task_vma_offset(void)
1698 {
1699 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1700 union bpf_iter_link_info linfo;
1701
1702 memset(&linfo, 0, sizeof(linfo));
1703 linfo.task.pid = getpid();
1704 opts.link_info = &linfo;
1705 opts.link_info_len = sizeof(linfo);
1706
1707 test_task_vma_offset_common(&opts, true);
1708
1709 linfo.task.pid = 0;
1710 linfo.task.tid = getpid();
1711 test_task_vma_offset_common(&opts, true);
1712
1713 test_task_vma_offset_common(NULL, false);
1714 }
1715
test_bpf_iter(void)1716 void test_bpf_iter(void)
1717 {
1718 ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
1719
1720 if (test__start_subtest("btf_id_or_null"))
1721 test_btf_id_or_null();
1722 if (test__start_subtest("ipv6_route"))
1723 test_ipv6_route();
1724 if (test__start_subtest("netlink"))
1725 test_netlink();
1726 if (test__start_subtest("bpf_map"))
1727 test_bpf_map();
1728 if (test__start_subtest("task_tid"))
1729 test_task_tid();
1730 if (test__start_subtest("task_pid"))
1731 test_task_pid();
1732 if (test__start_subtest("task_pidfd"))
1733 test_task_pidfd();
1734 if (test__start_subtest("task_sleepable"))
1735 test_task_sleepable();
1736 if (test__start_subtest("task_stack"))
1737 test_task_stack();
1738 if (test__start_subtest("task_file"))
1739 test_task_file();
1740 if (test__start_subtest("task_vma"))
1741 test_task_vma();
1742 if (test__start_subtest("task_vma_dead_task"))
1743 test_task_vma_dead_task();
1744 if (test__start_subtest("task_btf"))
1745 test_task_btf();
1746 if (test__start_subtest("tcp4"))
1747 test_tcp4();
1748 if (test__start_subtest("tcp6"))
1749 test_tcp6();
1750 if (test__start_subtest("udp4"))
1751 test_udp4();
1752 if (test__start_subtest("udp6"))
1753 test_udp6();
1754 if (test__start_subtest("unix"))
1755 test_unix();
1756 if (test__start_subtest("anon"))
1757 test_anon_iter(false);
1758 if (test__start_subtest("anon-read-one-char"))
1759 test_anon_iter(true);
1760 if (test__start_subtest("file"))
1761 test_file_iter();
1762 if (test__start_subtest("overflow"))
1763 test_overflow(false, false);
1764 if (test__start_subtest("overflow-e2big"))
1765 test_overflow(true, false);
1766 if (test__start_subtest("prog-ret-1"))
1767 test_overflow(false, true);
1768 if (test__start_subtest("bpf_hash_map"))
1769 test_bpf_hash_map();
1770 if (test__start_subtest("bpf_percpu_hash_map"))
1771 test_bpf_percpu_hash_map();
1772 if (test__start_subtest("bpf_array_map"))
1773 test_bpf_array_map();
1774 if (test__start_subtest("bpf_array_map_iter_fd"))
1775 test_bpf_array_map_iter_fd();
1776 if (test__start_subtest("bpf_percpu_array_map"))
1777 test_bpf_percpu_array_map();
1778 if (test__start_subtest("bpf_sk_storage_map"))
1779 test_bpf_sk_storage_map();
1780 if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
1781 test_bpf_sk_storage_map_iter_fd();
1782 if (test__start_subtest("bpf_sk_storage_delete"))
1783 test_bpf_sk_storage_delete();
1784 if (test__start_subtest("bpf_sk_storage_get"))
1785 test_bpf_sk_storage_get();
1786 if (test__start_subtest("rdonly-buf-out-of-bound"))
1787 test_rdonly_buf_out_of_bound();
1788 if (test__start_subtest("buf-neg-offset"))
1789 test_buf_neg_offset();
1790 if (test__start_subtest("link-iter"))
1791 test_link_iter();
1792 if (test__start_subtest("ksym"))
1793 test_ksym_iter();
1794 if (test__start_subtest("bpf_sockmap_map_iter_fd"))
1795 test_bpf_sockmap_map_iter_fd();
1796 if (test__start_subtest("vma_offset"))
1797 test_task_vma_offset();
1798 }
1799