1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2022 Facebook */
3
4 #include <vmlinux.h>
5 #include <string.h>
6 #include <stdbool.h>
7 #include <bpf/bpf_helpers.h>
8 #include <bpf/bpf_tracing.h>
9 #include "bpf_misc.h"
10 #include "errno.h"
11
12 #define PAGE_SIZE_64K 65536
13
14 char _license[] SEC("license") = "GPL";
15
16 int pid, err, val;
17
18 struct ringbuf_sample {
19 int pid;
20 int seq;
21 long value;
22 char comm[16];
23 };
24
25 struct {
26 __uint(type, BPF_MAP_TYPE_RINGBUF);
27 __uint(max_entries, 4096);
28 } ringbuf SEC(".maps");
29
30 struct {
31 __uint(type, BPF_MAP_TYPE_ARRAY);
32 __uint(max_entries, 1);
33 __type(key, __u32);
34 __type(value, __u32);
35 } array_map SEC(".maps");
36
37 SEC("?tp/syscalls/sys_enter_nanosleep")
test_read_write(void * ctx)38 int test_read_write(void *ctx)
39 {
40 char write_data[64] = "hello there, world!!";
41 char read_data[64] = {};
42 struct bpf_dynptr ptr;
43 int i;
44
45 if (bpf_get_current_pid_tgid() >> 32 != pid)
46 return 0;
47
48 bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(write_data), 0, &ptr);
49
50 /* Write data into the dynptr */
51 err = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
52
53 /* Read the data that was written into the dynptr */
54 err = err ?: bpf_dynptr_read(read_data, sizeof(read_data), &ptr, 0, 0);
55
56 /* Ensure the data we read matches the data we wrote */
57 for (i = 0; i < sizeof(read_data); i++) {
58 if (read_data[i] != write_data[i]) {
59 err = 1;
60 break;
61 }
62 }
63
64 bpf_ringbuf_discard_dynptr(&ptr, 0);
65 return 0;
66 }
67
68 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_data(void * ctx)69 int test_dynptr_data(void *ctx)
70 {
71 __u32 key = 0, val = 235, *map_val;
72 struct bpf_dynptr ptr;
73 __u32 map_val_size;
74 void *data;
75
76 map_val_size = sizeof(*map_val);
77
78 if (bpf_get_current_pid_tgid() >> 32 != pid)
79 return 0;
80
81 bpf_map_update_elem(&array_map, &key, &val, 0);
82
83 map_val = bpf_map_lookup_elem(&array_map, &key);
84 if (!map_val) {
85 err = 1;
86 return 0;
87 }
88
89 bpf_dynptr_from_mem(map_val, map_val_size, 0, &ptr);
90
91 /* Try getting a data slice that is out of range */
92 data = bpf_dynptr_data(&ptr, map_val_size + 1, 1);
93 if (data) {
94 err = 2;
95 return 0;
96 }
97
98 /* Try getting more bytes than available */
99 data = bpf_dynptr_data(&ptr, 0, map_val_size + 1);
100 if (data) {
101 err = 3;
102 return 0;
103 }
104
105 data = bpf_dynptr_data(&ptr, 0, sizeof(__u32));
106 if (!data) {
107 err = 4;
108 return 0;
109 }
110
111 *(__u32 *)data = 999;
112
113 err = bpf_probe_read_kernel(&val, sizeof(val), data);
114 if (err)
115 return 0;
116
117 if (val != *(int *)data)
118 err = 5;
119
120 return 0;
121 }
122
ringbuf_callback(__u32 index,void * data)123 static int ringbuf_callback(__u32 index, void *data)
124 {
125 struct ringbuf_sample *sample;
126
127 struct bpf_dynptr *ptr = (struct bpf_dynptr *)data;
128
129 sample = bpf_dynptr_data(ptr, 0, sizeof(*sample));
130 if (!sample)
131 err = 2;
132 else
133 sample->pid += index;
134
135 return 0;
136 }
137
138 SEC("?tp/syscalls/sys_enter_nanosleep")
test_ringbuf(void * ctx)139 int test_ringbuf(void *ctx)
140 {
141 struct bpf_dynptr ptr;
142 struct ringbuf_sample *sample;
143
144 if (bpf_get_current_pid_tgid() >> 32 != pid)
145 return 0;
146
147 val = 100;
148
149 /* check that you can reserve a dynamic size reservation */
150 err = bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr);
151
152 sample = err ? NULL : bpf_dynptr_data(&ptr, 0, sizeof(*sample));
153 if (!sample) {
154 err = 1;
155 goto done;
156 }
157
158 sample->pid = 10;
159
160 /* Can pass dynptr to callback functions */
161 bpf_loop(10, ringbuf_callback, &ptr, 0);
162
163 if (sample->pid != 55)
164 err = 2;
165
166 done:
167 bpf_ringbuf_discard_dynptr(&ptr, 0);
168 return 0;
169 }
170
171 SEC("?cgroup_skb/egress")
test_skb_readonly(struct __sk_buff * skb)172 int test_skb_readonly(struct __sk_buff *skb)
173 {
174 __u8 write_data[2] = {1, 2};
175 struct bpf_dynptr ptr;
176 int ret;
177
178 if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
179 err = 1;
180 return 1;
181 }
182
183 /* since cgroup skbs are read only, writes should fail */
184 ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
185 if (ret != -EINVAL) {
186 err = 2;
187 return 1;
188 }
189
190 return 1;
191 }
192
193 SEC("?cgroup_skb/egress")
test_dynptr_skb_data(struct __sk_buff * skb)194 int test_dynptr_skb_data(struct __sk_buff *skb)
195 {
196 struct bpf_dynptr ptr;
197 __u64 *data;
198
199 if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
200 err = 1;
201 return 1;
202 }
203
204 /* This should return NULL. Must use bpf_dynptr_slice API */
205 data = bpf_dynptr_data(&ptr, 0, 1);
206 if (data) {
207 err = 2;
208 return 1;
209 }
210
211 return 1;
212 }
213
214 SEC("?tc")
test_dynptr_skb_meta_data(struct __sk_buff * skb)215 int test_dynptr_skb_meta_data(struct __sk_buff *skb)
216 {
217 struct bpf_dynptr meta;
218 __u8 *md;
219 int ret;
220
221 err = 1;
222 ret = bpf_dynptr_from_skb_meta(skb, 0, &meta);
223 if (ret)
224 return 1;
225
226 /* This should return NULL. Must use bpf_dynptr_slice API */
227 err = 2;
228 md = bpf_dynptr_data(&meta, 0, sizeof(*md));
229 if (md)
230 return 1;
231
232 err = 0;
233 return 1;
234 }
235
236 /* Check that skb metadata dynptr ops don't accept any flags. */
237 SEC("?tc")
test_dynptr_skb_meta_flags(struct __sk_buff * skb)238 int test_dynptr_skb_meta_flags(struct __sk_buff *skb)
239 {
240 const __u64 INVALID_FLAGS = ~0ULL;
241 struct bpf_dynptr meta;
242 __u8 buf;
243 int ret;
244
245 err = 1;
246 ret = bpf_dynptr_from_skb_meta(skb, INVALID_FLAGS, &meta);
247 if (ret != -EINVAL)
248 return 1;
249
250 err = 2;
251 ret = bpf_dynptr_from_skb_meta(skb, 0, &meta);
252 if (ret)
253 return 1;
254
255 err = 3;
256 ret = bpf_dynptr_read(&buf, 0, &meta, 0, INVALID_FLAGS);
257 if (ret != -EINVAL)
258 return 1;
259
260 err = 4;
261 ret = bpf_dynptr_write(&meta, 0, &buf, 0, INVALID_FLAGS);
262 if (ret != -EINVAL)
263 return 1;
264
265 err = 0;
266 return 1;
267 }
268
269 SEC("tp/syscalls/sys_enter_nanosleep")
test_adjust(void * ctx)270 int test_adjust(void *ctx)
271 {
272 struct bpf_dynptr ptr;
273 __u32 bytes = 64;
274 __u32 off = 10;
275 __u32 trim = 15;
276
277 if (bpf_get_current_pid_tgid() >> 32 != pid)
278 return 0;
279
280 err = bpf_ringbuf_reserve_dynptr(&ringbuf, bytes, 0, &ptr);
281 if (err) {
282 err = 1;
283 goto done;
284 }
285
286 if (bpf_dynptr_size(&ptr) != bytes) {
287 err = 2;
288 goto done;
289 }
290
291 /* Advance the dynptr by off */
292 err = bpf_dynptr_adjust(&ptr, off, bpf_dynptr_size(&ptr));
293 if (err) {
294 err = 3;
295 goto done;
296 }
297
298 if (bpf_dynptr_size(&ptr) != bytes - off) {
299 err = 4;
300 goto done;
301 }
302
303 /* Trim the dynptr */
304 err = bpf_dynptr_adjust(&ptr, off, 15);
305 if (err) {
306 err = 5;
307 goto done;
308 }
309
310 /* Check that the size was adjusted correctly */
311 if (bpf_dynptr_size(&ptr) != trim - off) {
312 err = 6;
313 goto done;
314 }
315
316 done:
317 bpf_ringbuf_discard_dynptr(&ptr, 0);
318 return 0;
319 }
320
321 SEC("tp/syscalls/sys_enter_nanosleep")
test_adjust_err(void * ctx)322 int test_adjust_err(void *ctx)
323 {
324 char write_data[45] = "hello there, world!!";
325 struct bpf_dynptr ptr;
326 __u32 size = 64;
327 __u32 off = 20;
328
329 if (bpf_get_current_pid_tgid() >> 32 != pid)
330 return 0;
331
332 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) {
333 err = 1;
334 goto done;
335 }
336
337 /* Check that start can't be greater than end */
338 if (bpf_dynptr_adjust(&ptr, 5, 1) != -EINVAL) {
339 err = 2;
340 goto done;
341 }
342
343 /* Check that start can't be greater than size */
344 if (bpf_dynptr_adjust(&ptr, size + 1, size + 1) != -ERANGE) {
345 err = 3;
346 goto done;
347 }
348
349 /* Check that end can't be greater than size */
350 if (bpf_dynptr_adjust(&ptr, 0, size + 1) != -ERANGE) {
351 err = 4;
352 goto done;
353 }
354
355 if (bpf_dynptr_adjust(&ptr, off, size)) {
356 err = 5;
357 goto done;
358 }
359
360 /* Check that you can't write more bytes than available into the dynptr
361 * after you've adjusted it
362 */
363 if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) {
364 err = 6;
365 goto done;
366 }
367
368 /* Check that even after adjusting, submitting/discarding
369 * a ringbuf dynptr works
370 */
371 bpf_ringbuf_submit_dynptr(&ptr, 0);
372 return 0;
373
374 done:
375 bpf_ringbuf_discard_dynptr(&ptr, 0);
376 return 0;
377 }
378
379 SEC("tp/syscalls/sys_enter_nanosleep")
test_zero_size_dynptr(void * ctx)380 int test_zero_size_dynptr(void *ctx)
381 {
382 char write_data = 'x', read_data;
383 struct bpf_dynptr ptr;
384 __u32 size = 64;
385
386 if (bpf_get_current_pid_tgid() >> 32 != pid)
387 return 0;
388
389 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) {
390 err = 1;
391 goto done;
392 }
393
394 /* After this, the dynptr has a size of 0 */
395 if (bpf_dynptr_adjust(&ptr, size, size)) {
396 err = 2;
397 goto done;
398 }
399
400 /* Test that reading + writing non-zero bytes is not ok */
401 if (bpf_dynptr_read(&read_data, sizeof(read_data), &ptr, 0, 0) != -E2BIG) {
402 err = 3;
403 goto done;
404 }
405
406 if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) {
407 err = 4;
408 goto done;
409 }
410
411 /* Test that reading + writing 0 bytes from a 0-size dynptr is ok */
412 if (bpf_dynptr_read(&read_data, 0, &ptr, 0, 0)) {
413 err = 5;
414 goto done;
415 }
416
417 if (bpf_dynptr_write(&ptr, 0, &write_data, 0, 0)) {
418 err = 6;
419 goto done;
420 }
421
422 err = 0;
423
424 done:
425 bpf_ringbuf_discard_dynptr(&ptr, 0);
426 return 0;
427 }
428
429 SEC("tp/syscalls/sys_enter_nanosleep")
test_dynptr_is_null(void * ctx)430 int test_dynptr_is_null(void *ctx)
431 {
432 struct bpf_dynptr ptr1;
433 struct bpf_dynptr ptr2;
434 __u64 size = 4;
435
436 if (bpf_get_current_pid_tgid() >> 32 != pid)
437 return 0;
438
439 /* Pass in invalid flags, get back an invalid dynptr */
440 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 123, &ptr1) != -EINVAL) {
441 err = 1;
442 goto exit_early;
443 }
444
445 /* Test that the invalid dynptr is null */
446 if (!bpf_dynptr_is_null(&ptr1)) {
447 err = 2;
448 goto exit_early;
449 }
450
451 /* Get a valid dynptr */
452 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr2)) {
453 err = 3;
454 goto exit;
455 }
456
457 /* Test that the valid dynptr is not null */
458 if (bpf_dynptr_is_null(&ptr2)) {
459 err = 4;
460 goto exit;
461 }
462
463 exit:
464 bpf_ringbuf_discard_dynptr(&ptr2, 0);
465 exit_early:
466 bpf_ringbuf_discard_dynptr(&ptr1, 0);
467 return 0;
468 }
469
470 SEC("cgroup_skb/egress")
test_dynptr_is_rdonly(struct __sk_buff * skb)471 int test_dynptr_is_rdonly(struct __sk_buff *skb)
472 {
473 struct bpf_dynptr ptr1;
474 struct bpf_dynptr ptr2;
475 struct bpf_dynptr ptr3;
476
477 /* Pass in invalid flags, get back an invalid dynptr */
478 if (bpf_dynptr_from_skb(skb, 123, &ptr1) != -EINVAL) {
479 err = 1;
480 return 0;
481 }
482
483 /* Test that an invalid dynptr is_rdonly returns false */
484 if (bpf_dynptr_is_rdonly(&ptr1)) {
485 err = 2;
486 return 0;
487 }
488
489 /* Get a read-only dynptr */
490 if (bpf_dynptr_from_skb(skb, 0, &ptr2)) {
491 err = 3;
492 return 0;
493 }
494
495 /* Test that the dynptr is read-only */
496 if (!bpf_dynptr_is_rdonly(&ptr2)) {
497 err = 4;
498 return 0;
499 }
500
501 /* Get a read-writeable dynptr */
502 if (bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr3)) {
503 err = 5;
504 goto done;
505 }
506
507 /* Test that the dynptr is read-only */
508 if (bpf_dynptr_is_rdonly(&ptr3)) {
509 err = 6;
510 goto done;
511 }
512
513 done:
514 bpf_ringbuf_discard_dynptr(&ptr3, 0);
515 return 0;
516 }
517
518 SEC("cgroup_skb/egress")
test_dynptr_clone(struct __sk_buff * skb)519 int test_dynptr_clone(struct __sk_buff *skb)
520 {
521 struct bpf_dynptr ptr1;
522 struct bpf_dynptr ptr2;
523 __u32 off = 2, size;
524
525 /* Get a dynptr */
526 if (bpf_dynptr_from_skb(skb, 0, &ptr1)) {
527 err = 1;
528 return 0;
529 }
530
531 if (bpf_dynptr_adjust(&ptr1, off, bpf_dynptr_size(&ptr1))) {
532 err = 2;
533 return 0;
534 }
535
536 /* Clone the dynptr */
537 if (bpf_dynptr_clone(&ptr1, &ptr2)) {
538 err = 3;
539 return 0;
540 }
541
542 size = bpf_dynptr_size(&ptr1);
543
544 /* Check that the clone has the same size and rd-only */
545 if (bpf_dynptr_size(&ptr2) != size) {
546 err = 4;
547 return 0;
548 }
549
550 if (bpf_dynptr_is_rdonly(&ptr2) != bpf_dynptr_is_rdonly(&ptr1)) {
551 err = 5;
552 return 0;
553 }
554
555 /* Advance and trim the original dynptr */
556 bpf_dynptr_adjust(&ptr1, 5, 5);
557
558 /* Check that only original dynptr was affected, and the clone wasn't */
559 if (bpf_dynptr_size(&ptr2) != size) {
560 err = 6;
561 return 0;
562 }
563
564 return 0;
565 }
566
567 SEC("?cgroup_skb/egress")
test_dynptr_skb_no_buff(struct __sk_buff * skb)568 int test_dynptr_skb_no_buff(struct __sk_buff *skb)
569 {
570 struct bpf_dynptr ptr;
571 __u64 *data;
572
573 if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
574 err = 1;
575 return 1;
576 }
577
578 /* This may return NULL. SKB may require a buffer */
579 data = bpf_dynptr_slice(&ptr, 0, NULL, 1);
580
581 return !!data;
582 }
583
584 SEC("?cgroup_skb/egress")
test_dynptr_skb_strcmp(struct __sk_buff * skb)585 int test_dynptr_skb_strcmp(struct __sk_buff *skb)
586 {
587 struct bpf_dynptr ptr;
588 char *data;
589
590 if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
591 err = 1;
592 return 1;
593 }
594
595 /* This may return NULL. SKB may require a buffer */
596 data = bpf_dynptr_slice(&ptr, 0, NULL, 10);
597 if (data) {
598 bpf_strncmp(data, 10, "foo");
599 return 1;
600 }
601
602 return 1;
603 }
604
605 SEC("tp_btf/kfree_skb")
BPF_PROG(test_dynptr_skb_tp_btf,void * skb,void * location)606 int BPF_PROG(test_dynptr_skb_tp_btf, void *skb, void *location)
607 {
608 __u8 write_data[2] = {1, 2};
609 struct bpf_dynptr ptr;
610 int ret;
611
612 if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
613 err = 1;
614 return 1;
615 }
616
617 /* since tp_btf skbs are read only, writes should fail */
618 ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
619 if (ret != -EINVAL) {
620 err = 2;
621 return 1;
622 }
623
624 return 1;
625 }
626
bpf_memcmp(const char * a,const char * b,u32 size)627 static inline int bpf_memcmp(const char *a, const char *b, u32 size)
628 {
629 int i;
630
631 bpf_for(i, 0, size) {
632 if (a[i] != b[i])
633 return a[i] < b[i] ? -1 : 1;
634 }
635 return 0;
636 }
637
638 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_copy(void * ctx)639 int test_dynptr_copy(void *ctx)
640 {
641 char data[] = "hello there, world!!";
642 char buf[32] = {'\0'};
643 __u32 sz = sizeof(data);
644 struct bpf_dynptr src, dst;
645
646 bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src);
647 bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst);
648
649 /* Test basic case of copying contiguous memory backed dynptrs */
650 err = bpf_dynptr_write(&src, 0, data, sz, 0);
651 err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz);
652 err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0);
653 err = err ?: bpf_memcmp(data, buf, sz);
654
655 /* Test that offsets are handled correctly */
656 err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5);
657 err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0);
658 err = err ?: bpf_memcmp(data + 5, buf, sz - 5);
659
660 bpf_ringbuf_discard_dynptr(&src, 0);
661 bpf_ringbuf_discard_dynptr(&dst, 0);
662 return 0;
663 }
664
665 SEC("xdp")
test_dynptr_copy_xdp(struct xdp_md * xdp)666 int test_dynptr_copy_xdp(struct xdp_md *xdp)
667 {
668 struct bpf_dynptr ptr_buf, ptr_xdp;
669 char data[] = "qwertyuiopasdfghjkl";
670 char buf[32] = {'\0'};
671 __u32 len = sizeof(data), xdp_data_size;
672 int i, chunks = 200;
673
674 /* ptr_xdp is backed by non-contiguous memory */
675 bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
676 xdp_data_size = bpf_dynptr_size(&ptr_xdp);
677 bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf);
678
679 /* Destination dynptr is backed by non-contiguous memory */
680 bpf_for(i, 0, chunks) {
681 err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0);
682 if (err)
683 goto out;
684 }
685
686 err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks);
687 if (err)
688 goto out;
689
690 bpf_for(i, 0, chunks) {
691 __builtin_memset(buf, 0, sizeof(buf));
692 err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0);
693 if (err)
694 goto out;
695 if (bpf_memcmp(data, buf, len) != 0)
696 goto out;
697 }
698
699 /* Source dynptr is backed by non-contiguous memory */
700 __builtin_memset(buf, 0, sizeof(buf));
701 bpf_for(i, 0, chunks) {
702 err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0);
703 if (err)
704 goto out;
705 }
706
707 err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks);
708 if (err)
709 goto out;
710
711 bpf_for(i, 0, chunks) {
712 __builtin_memset(buf, 0, sizeof(buf));
713 err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0);
714 if (err)
715 goto out;
716 if (bpf_memcmp(data, buf, len) != 0)
717 goto out;
718 }
719
720 /* Both source and destination dynptrs are backed by non-contiguous memory */
721 err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1));
722 if (err)
723 goto out;
724
725 bpf_for(i, 0, chunks - 1) {
726 __builtin_memset(buf, 0, sizeof(buf));
727 err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0);
728 if (err)
729 goto out;
730 if (bpf_memcmp(data, buf, len) != 0)
731 goto out;
732 }
733
734 if (bpf_dynptr_copy(&ptr_xdp, xdp_data_size - 3000, &ptr_xdp, 0, len * chunks) != -E2BIG)
735 err = 1;
736
737 out:
738 bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
739 return XDP_DROP;
740 }
741
742 char memset_zero_data[] = "data to be zeroed";
743
744 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_memset_zero(void * ctx)745 int test_dynptr_memset_zero(void *ctx)
746 {
747 __u32 data_sz = sizeof(memset_zero_data);
748 char zeroes[32] = {'\0'};
749 struct bpf_dynptr ptr;
750
751 err = bpf_dynptr_from_mem(memset_zero_data, data_sz, 0, &ptr);
752 err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, 0);
753 err = err ?: bpf_memcmp(zeroes, memset_zero_data, data_sz);
754
755 return 0;
756 }
757
758 #define DYNPTR_MEMSET_VAL 42
759
760 char memset_notzero_data[] = "data to be overwritten";
761
762 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_memset_notzero(void * ctx)763 int test_dynptr_memset_notzero(void *ctx)
764 {
765 u32 data_sz = sizeof(memset_notzero_data);
766 struct bpf_dynptr ptr;
767 char expected[32];
768
769 __builtin_memset(expected, DYNPTR_MEMSET_VAL, data_sz);
770
771 err = bpf_dynptr_from_mem(memset_notzero_data, data_sz, 0, &ptr);
772 err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, DYNPTR_MEMSET_VAL);
773 err = err ?: bpf_memcmp(expected, memset_notzero_data, data_sz);
774
775 return 0;
776 }
777
778 char memset_zero_offset_data[] = "data to be zeroed partially";
779
780 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_memset_zero_offset(void * ctx)781 int test_dynptr_memset_zero_offset(void *ctx)
782 {
783 char expected[] = "data to \0\0\0\0eroed partially";
784 __u32 data_sz = sizeof(memset_zero_offset_data);
785 struct bpf_dynptr ptr;
786
787 err = bpf_dynptr_from_mem(memset_zero_offset_data, data_sz, 0, &ptr);
788 err = err ?: bpf_dynptr_memset(&ptr, 8, 4, 0);
789 err = err ?: bpf_memcmp(expected, memset_zero_offset_data, data_sz);
790
791 return 0;
792 }
793
794 char memset_zero_adjusted_data[] = "data to be zeroed partially";
795
796 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_memset_zero_adjusted(void * ctx)797 int test_dynptr_memset_zero_adjusted(void *ctx)
798 {
799 char expected[] = "data\0\0\0\0be zeroed partially";
800 __u32 data_sz = sizeof(memset_zero_adjusted_data);
801 struct bpf_dynptr ptr;
802
803 err = bpf_dynptr_from_mem(memset_zero_adjusted_data, data_sz, 0, &ptr);
804 err = err ?: bpf_dynptr_adjust(&ptr, 4, 8);
805 err = err ?: bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0);
806 err = err ?: bpf_memcmp(expected, memset_zero_adjusted_data, data_sz);
807
808 return 0;
809 }
810
811 char memset_overflow_data[] = "memset overflow data";
812
813 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_memset_overflow(void * ctx)814 int test_dynptr_memset_overflow(void *ctx)
815 {
816 __u32 data_sz = sizeof(memset_overflow_data);
817 struct bpf_dynptr ptr;
818 int ret;
819
820 err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr);
821 ret = bpf_dynptr_memset(&ptr, 0, data_sz + 1, 0);
822 if (ret != -E2BIG)
823 err = 1;
824
825 return 0;
826 }
827
828 SEC("?tp/syscalls/sys_enter_nanosleep")
test_dynptr_memset_overflow_offset(void * ctx)829 int test_dynptr_memset_overflow_offset(void *ctx)
830 {
831 __u32 data_sz = sizeof(memset_overflow_data);
832 struct bpf_dynptr ptr;
833 int ret;
834
835 err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr);
836 ret = bpf_dynptr_memset(&ptr, 1, data_sz, 0);
837 if (ret != -E2BIG)
838 err = 1;
839
840 return 0;
841 }
842
843 SEC("?cgroup_skb/egress")
test_dynptr_memset_readonly(struct __sk_buff * skb)844 int test_dynptr_memset_readonly(struct __sk_buff *skb)
845 {
846 struct bpf_dynptr ptr;
847 int ret;
848
849 err = bpf_dynptr_from_skb(skb, 0, &ptr);
850
851 /* cgroup skbs are read only, memset should fail */
852 ret = bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0);
853 if (ret != -EINVAL)
854 err = 1;
855
856 return 0;
857 }
858
859 #define min_t(type, x, y) ({ \
860 type __x = (x); \
861 type __y = (y); \
862 __x < __y ? __x : __y; })
863
864 SEC("xdp")
test_dynptr_memset_xdp_chunks(struct xdp_md * xdp)865 int test_dynptr_memset_xdp_chunks(struct xdp_md *xdp)
866 {
867 u32 data_sz, chunk_sz, offset = 0;
868 const int max_chunks = 200;
869 struct bpf_dynptr ptr_xdp;
870 char expected_buf[32];
871 char buf[32];
872 int i;
873
874 __builtin_memset(expected_buf, DYNPTR_MEMSET_VAL, sizeof(expected_buf));
875
876 /* ptr_xdp is backed by non-contiguous memory */
877 bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
878 data_sz = bpf_dynptr_size(&ptr_xdp);
879
880 err = bpf_dynptr_memset(&ptr_xdp, 0, data_sz, DYNPTR_MEMSET_VAL);
881 if (err) {
882 /* bpf_dynptr_memset() eventually called bpf_xdp_pointer()
883 * where if data_sz is greater than 0xffff, -EFAULT will be
884 * returned. For 64K page size, data_sz is greater than
885 * 64K, so error is expected and let us zero out error and
886 * return success.
887 */
888 if (data_sz >= PAGE_SIZE_64K)
889 err = 0;
890 goto out;
891 }
892
893 bpf_for(i, 0, max_chunks) {
894 offset = i * sizeof(buf);
895 if (offset >= data_sz)
896 goto out;
897 chunk_sz = min_t(u32, sizeof(buf), data_sz - offset);
898 err = bpf_dynptr_read(&buf, chunk_sz, &ptr_xdp, offset, 0);
899 if (err)
900 goto out;
901 err = bpf_memcmp(buf, expected_buf, sizeof(buf));
902 if (err)
903 goto out;
904 }
905 out:
906 return XDP_DROP;
907 }
908
909 void *user_ptr;
910 /* Contains the copy of the data pointed by user_ptr.
911 * Size 384 to make it not fit into a single kernel chunk when copying
912 * but less than the maximum bpf stack size (512).
913 */
914 char expected_str[384];
915 __u32 test_len[7] = {0/* placeholder */, 0, 1, 2, 255, 256, 257};
916
917 typedef int (*bpf_read_dynptr_fn_t)(struct bpf_dynptr *dptr, u64 off,
918 u64 size, const void *unsafe_ptr);
919
920 /* Returns the offset just before the end of the maximum sized xdp fragment.
921 * Any write larger than 32 bytes will be split between 2 fragments.
922 */
xdp_near_frag_end_offset(void)923 __u32 xdp_near_frag_end_offset(void)
924 {
925 const __u32 headroom = 256;
926 const __u32 max_frag_size = __PAGE_SIZE - headroom - sizeof(struct skb_shared_info);
927
928 /* 32 bytes before the approximate end of the fragment */
929 return max_frag_size - 32;
930 }
931
932 /* Use __always_inline on test_dynptr_probe[_str][_xdp]() and callbacks
933 * of type bpf_read_dynptr_fn_t to prevent compiler from generating
934 * indirect calls that make program fail to load with "unknown opcode" error.
935 */
test_dynptr_probe(void * ptr,bpf_read_dynptr_fn_t bpf_read_dynptr_fn)936 static __always_inline void test_dynptr_probe(void *ptr, bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
937 {
938 char buf[sizeof(expected_str)];
939 struct bpf_dynptr ptr_buf;
940 int i;
941
942 if (bpf_get_current_pid_tgid() >> 32 != pid)
943 return;
944
945 err = bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf);
946
947 bpf_for(i, 0, ARRAY_SIZE(test_len)) {
948 __u32 len = test_len[i];
949
950 err = err ?: bpf_read_dynptr_fn(&ptr_buf, 0, test_len[i], ptr);
951 if (len > sizeof(buf))
952 break;
953 err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0);
954
955 if (err || bpf_memcmp(expected_str, buf, len))
956 err = 1;
957
958 /* Reset buffer and dynptr */
959 __builtin_memset(buf, 0, sizeof(buf));
960 err = err ?: bpf_dynptr_write(&ptr_buf, 0, buf, len, 0);
961 }
962 bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
963 }
964
test_dynptr_probe_str(void * ptr,bpf_read_dynptr_fn_t bpf_read_dynptr_fn)965 static __always_inline void test_dynptr_probe_str(void *ptr,
966 bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
967 {
968 char buf[sizeof(expected_str)];
969 struct bpf_dynptr ptr_buf;
970 __u32 cnt, i;
971
972 if (bpf_get_current_pid_tgid() >> 32 != pid)
973 return;
974
975 bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf);
976
977 bpf_for(i, 0, ARRAY_SIZE(test_len)) {
978 __u32 len = test_len[i];
979
980 cnt = bpf_read_dynptr_fn(&ptr_buf, 0, len, ptr);
981 if (cnt != len)
982 err = 1;
983
984 if (len > sizeof(buf))
985 continue;
986 err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0);
987 if (!len)
988 continue;
989 if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0')
990 err = 1;
991 }
992 bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
993 }
994
test_dynptr_probe_xdp(struct xdp_md * xdp,void * ptr,bpf_read_dynptr_fn_t bpf_read_dynptr_fn)995 static __always_inline void test_dynptr_probe_xdp(struct xdp_md *xdp, void *ptr,
996 bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
997 {
998 struct bpf_dynptr ptr_xdp;
999 char buf[sizeof(expected_str)];
1000 __u32 off, i;
1001
1002 if (bpf_get_current_pid_tgid() >> 32 != pid)
1003 return;
1004
1005 off = xdp_near_frag_end_offset();
1006 err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
1007
1008 bpf_for(i, 0, ARRAY_SIZE(test_len)) {
1009 __u32 len = test_len[i];
1010
1011 err = err ?: bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr);
1012 if (len > sizeof(buf))
1013 continue;
1014 err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0);
1015 if (err || bpf_memcmp(expected_str, buf, len))
1016 err = 1;
1017 /* Reset buffer and dynptr */
1018 __builtin_memset(buf, 0, sizeof(buf));
1019 err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0);
1020 }
1021 }
1022
test_dynptr_probe_str_xdp(struct xdp_md * xdp,void * ptr,bpf_read_dynptr_fn_t bpf_read_dynptr_fn)1023 static __always_inline void test_dynptr_probe_str_xdp(struct xdp_md *xdp, void *ptr,
1024 bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
1025 {
1026 struct bpf_dynptr ptr_xdp;
1027 char buf[sizeof(expected_str)];
1028 __u32 cnt, off, i;
1029
1030 if (bpf_get_current_pid_tgid() >> 32 != pid)
1031 return;
1032
1033 off = xdp_near_frag_end_offset();
1034 err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
1035 if (err)
1036 return;
1037
1038 bpf_for(i, 0, ARRAY_SIZE(test_len)) {
1039 __u32 len = test_len[i];
1040
1041 cnt = bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr);
1042 if (cnt != len)
1043 err = 1;
1044
1045 if (len > sizeof(buf))
1046 continue;
1047 err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0);
1048
1049 if (!len)
1050 continue;
1051 if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0')
1052 err = 1;
1053
1054 __builtin_memset(buf, 0, sizeof(buf));
1055 err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0);
1056 }
1057 }
1058
1059 SEC("xdp")
test_probe_read_user_dynptr(struct xdp_md * xdp)1060 int test_probe_read_user_dynptr(struct xdp_md *xdp)
1061 {
1062 test_dynptr_probe(user_ptr, bpf_probe_read_user_dynptr);
1063 if (!err)
1064 test_dynptr_probe_xdp(xdp, user_ptr, bpf_probe_read_user_dynptr);
1065 return XDP_PASS;
1066 }
1067
1068 SEC("xdp")
test_probe_read_kernel_dynptr(struct xdp_md * xdp)1069 int test_probe_read_kernel_dynptr(struct xdp_md *xdp)
1070 {
1071 test_dynptr_probe(expected_str, bpf_probe_read_kernel_dynptr);
1072 if (!err)
1073 test_dynptr_probe_xdp(xdp, expected_str, bpf_probe_read_kernel_dynptr);
1074 return XDP_PASS;
1075 }
1076
1077 SEC("xdp")
test_probe_read_user_str_dynptr(struct xdp_md * xdp)1078 int test_probe_read_user_str_dynptr(struct xdp_md *xdp)
1079 {
1080 test_dynptr_probe_str(user_ptr, bpf_probe_read_user_str_dynptr);
1081 if (!err)
1082 test_dynptr_probe_str_xdp(xdp, user_ptr, bpf_probe_read_user_str_dynptr);
1083 return XDP_PASS;
1084 }
1085
1086 SEC("xdp")
test_probe_read_kernel_str_dynptr(struct xdp_md * xdp)1087 int test_probe_read_kernel_str_dynptr(struct xdp_md *xdp)
1088 {
1089 test_dynptr_probe_str(expected_str, bpf_probe_read_kernel_str_dynptr);
1090 if (!err)
1091 test_dynptr_probe_str_xdp(xdp, expected_str, bpf_probe_read_kernel_str_dynptr);
1092 return XDP_PASS;
1093 }
1094
1095 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
test_copy_from_user_dynptr(void * ctx)1096 int test_copy_from_user_dynptr(void *ctx)
1097 {
1098 test_dynptr_probe(user_ptr, bpf_copy_from_user_dynptr);
1099 return 0;
1100 }
1101
1102 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
test_copy_from_user_str_dynptr(void * ctx)1103 int test_copy_from_user_str_dynptr(void *ctx)
1104 {
1105 test_dynptr_probe_str(user_ptr, bpf_copy_from_user_str_dynptr);
1106 return 0;
1107 }
1108
bpf_copy_data_from_user_task(struct bpf_dynptr * dptr,u64 off,u64 size,const void * unsafe_ptr)1109 static int bpf_copy_data_from_user_task(struct bpf_dynptr *dptr, u64 off,
1110 u64 size, const void *unsafe_ptr)
1111 {
1112 struct task_struct *task = bpf_get_current_task_btf();
1113
1114 return bpf_copy_from_user_task_dynptr(dptr, off, size, unsafe_ptr, task);
1115 }
1116
bpf_copy_data_from_user_task_str(struct bpf_dynptr * dptr,u64 off,u64 size,const void * unsafe_ptr)1117 static int bpf_copy_data_from_user_task_str(struct bpf_dynptr *dptr, u64 off,
1118 u64 size, const void *unsafe_ptr)
1119 {
1120 struct task_struct *task = bpf_get_current_task_btf();
1121
1122 return bpf_copy_from_user_task_str_dynptr(dptr, off, size, unsafe_ptr, task);
1123 }
1124
1125 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
test_copy_from_user_task_dynptr(void * ctx)1126 int test_copy_from_user_task_dynptr(void *ctx)
1127 {
1128 test_dynptr_probe(user_ptr, bpf_copy_data_from_user_task);
1129 return 0;
1130 }
1131
1132 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
test_copy_from_user_task_str_dynptr(void * ctx)1133 int test_copy_from_user_task_str_dynptr(void *ctx)
1134 {
1135 test_dynptr_probe_str(user_ptr, bpf_copy_data_from_user_task_str);
1136 return 0;
1137 }
1138