1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/bpf.h>
4 #include <bpf/bpf_helpers.h>
5 #include <bpf/bpf_core_read.h>
6 #include "../../../include/linux/filter.h"
7 #include "bpf_misc.h"
8 #include <stdbool.h>
9 #include "bpf_kfuncs.h"
10
11 SEC("raw_tp")
12 __arch_x86_64
13 __log_level(4) __msg("stack depth 8")
14 __xlated("4: r5 = 5")
15 __xlated("5: r0 = ")
16 __xlated("6: r0 = &(void __percpu *)(r0)")
17 __xlated("7: r0 = *(u32 *)(r0 +0)")
18 __xlated("8: exit")
19 __success
simple(void)20 __naked void simple(void)
21 {
22 asm volatile (
23 "r1 = 1;"
24 "r2 = 2;"
25 "r3 = 3;"
26 "r4 = 4;"
27 "r5 = 5;"
28 "*(u64 *)(r10 - 16) = r1;"
29 "*(u64 *)(r10 - 24) = r2;"
30 "*(u64 *)(r10 - 32) = r3;"
31 "*(u64 *)(r10 - 40) = r4;"
32 "*(u64 *)(r10 - 48) = r5;"
33 "call %[bpf_get_smp_processor_id];"
34 "r5 = *(u64 *)(r10 - 48);"
35 "r4 = *(u64 *)(r10 - 40);"
36 "r3 = *(u64 *)(r10 - 32);"
37 "r2 = *(u64 *)(r10 - 24);"
38 "r1 = *(u64 *)(r10 - 16);"
39 "exit;"
40 :
41 : __imm(bpf_get_smp_processor_id)
42 : __clobber_all);
43 }
44
45 /* The logic for detecting and verifying bpf_fastcall pattern is the same for
46 * any arch, however x86 differs from arm64 or riscv64 in a way
47 * bpf_get_smp_processor_id is rewritten:
48 * - on x86 it is done by verifier
49 * - on arm64 and riscv64 it is done by jit
50 *
51 * Which leads to different xlated patterns for different archs:
52 * - on x86 the call is expanded as 3 instructions
53 * - on arm64 and riscv64 the call remains as is
54 * (but spills/fills are still removed)
55 *
56 * It is really desirable to check instruction indexes in the xlated
57 * patterns, so add this canary test to check that function rewrite by
58 * jit is correctly processed by bpf_fastcall logic, keep the rest of the
59 * tests as x86.
60 */
61 SEC("raw_tp")
62 __arch_arm64
63 __arch_riscv64
64 __xlated("0: r1 = 1")
65 __xlated("1: call bpf_get_smp_processor_id")
66 __xlated("2: exit")
67 __success
canary_arm64_riscv64(void)68 __naked void canary_arm64_riscv64(void)
69 {
70 asm volatile (
71 "r1 = 1;"
72 "*(u64 *)(r10 - 16) = r1;"
73 "call %[bpf_get_smp_processor_id];"
74 "r1 = *(u64 *)(r10 - 16);"
75 "exit;"
76 :
77 : __imm(bpf_get_smp_processor_id)
78 : __clobber_all);
79 }
80
81 SEC("raw_tp")
82 __arch_x86_64
83 __xlated("1: r0 = &(void __percpu *)(r0)")
84 __xlated("...")
85 __xlated("3: exit")
86 __success
canary_zero_spills(void)87 __naked void canary_zero_spills(void)
88 {
89 asm volatile (
90 "call %[bpf_get_smp_processor_id];"
91 "exit;"
92 :
93 : __imm(bpf_get_smp_processor_id)
94 : __clobber_all);
95 }
96
97 SEC("raw_tp")
98 __arch_x86_64
99 __log_level(4) __msg("stack depth 16")
100 __xlated("1: *(u64 *)(r10 -16) = r1")
101 __xlated("...")
102 __xlated("3: r0 = &(void __percpu *)(r0)")
103 __xlated("...")
104 __xlated("5: r2 = *(u64 *)(r10 -16)")
105 __success
wrong_reg_in_pattern1(void)106 __naked void wrong_reg_in_pattern1(void)
107 {
108 asm volatile (
109 "r1 = 1;"
110 "*(u64 *)(r10 - 16) = r1;"
111 "call %[bpf_get_smp_processor_id];"
112 "r2 = *(u64 *)(r10 - 16);"
113 "exit;"
114 :
115 : __imm(bpf_get_smp_processor_id)
116 : __clobber_all);
117 }
118
119 SEC("raw_tp")
120 __arch_x86_64
121 __xlated("1: *(u64 *)(r10 -16) = r6")
122 __xlated("...")
123 __xlated("3: r0 = &(void __percpu *)(r0)")
124 __xlated("...")
125 __xlated("5: r6 = *(u64 *)(r10 -16)")
126 __success
wrong_reg_in_pattern2(void)127 __naked void wrong_reg_in_pattern2(void)
128 {
129 asm volatile (
130 "r6 = 1;"
131 "*(u64 *)(r10 - 16) = r6;"
132 "call %[bpf_get_smp_processor_id];"
133 "r6 = *(u64 *)(r10 - 16);"
134 "exit;"
135 :
136 : __imm(bpf_get_smp_processor_id)
137 : __clobber_all);
138 }
139
140 SEC("raw_tp")
141 __arch_x86_64
142 __xlated("1: *(u64 *)(r10 -16) = r0")
143 __xlated("...")
144 __xlated("3: r0 = &(void __percpu *)(r0)")
145 __xlated("...")
146 __xlated("5: r0 = *(u64 *)(r10 -16)")
147 __success
wrong_reg_in_pattern3(void)148 __naked void wrong_reg_in_pattern3(void)
149 {
150 asm volatile (
151 "r0 = 1;"
152 "*(u64 *)(r10 - 16) = r0;"
153 "call %[bpf_get_smp_processor_id];"
154 "r0 = *(u64 *)(r10 - 16);"
155 "exit;"
156 :
157 : __imm(bpf_get_smp_processor_id)
158 : __clobber_all);
159 }
160
161 SEC("raw_tp")
162 __arch_x86_64
163 __xlated("2: *(u64 *)(r2 -16) = r1")
164 __xlated("...")
165 __xlated("4: r0 = &(void __percpu *)(r0)")
166 __xlated("...")
167 __xlated("6: r1 = *(u64 *)(r10 -16)")
168 __success
wrong_base_in_pattern(void)169 __naked void wrong_base_in_pattern(void)
170 {
171 asm volatile (
172 "r1 = 1;"
173 "r2 = r10;"
174 "*(u64 *)(r2 - 16) = r1;"
175 "call %[bpf_get_smp_processor_id];"
176 "r1 = *(u64 *)(r10 - 16);"
177 "exit;"
178 :
179 : __imm(bpf_get_smp_processor_id)
180 : __clobber_all);
181 }
182
183 SEC("raw_tp")
184 __arch_x86_64
185 __xlated("1: *(u64 *)(r10 -16) = r1")
186 __xlated("...")
187 __xlated("3: r0 = &(void __percpu *)(r0)")
188 __xlated("...")
189 __xlated("5: r2 = 1")
190 __success
wrong_insn_in_pattern(void)191 __naked void wrong_insn_in_pattern(void)
192 {
193 asm volatile (
194 "r1 = 1;"
195 "*(u64 *)(r10 - 16) = r1;"
196 "call %[bpf_get_smp_processor_id];"
197 "r2 = 1;"
198 "r1 = *(u64 *)(r10 - 16);"
199 "exit;"
200 :
201 : __imm(bpf_get_smp_processor_id)
202 : __clobber_all);
203 }
204
205 SEC("raw_tp")
206 __arch_x86_64
207 __xlated("2: *(u64 *)(r10 -16) = r1")
208 __xlated("...")
209 __xlated("4: r0 = &(void __percpu *)(r0)")
210 __xlated("...")
211 __xlated("6: r1 = *(u64 *)(r10 -8)")
212 __success
wrong_off_in_pattern1(void)213 __naked void wrong_off_in_pattern1(void)
214 {
215 asm volatile (
216 "r1 = 1;"
217 "*(u64 *)(r10 - 8) = r1;"
218 "*(u64 *)(r10 - 16) = r1;"
219 "call %[bpf_get_smp_processor_id];"
220 "r1 = *(u64 *)(r10 - 8);"
221 "exit;"
222 :
223 : __imm(bpf_get_smp_processor_id)
224 : __clobber_all);
225 }
226
227 SEC("raw_tp")
228 __arch_x86_64
229 __xlated("1: *(u32 *)(r10 -4) = r1")
230 __xlated("...")
231 __xlated("3: r0 = &(void __percpu *)(r0)")
232 __xlated("...")
233 __xlated("5: r1 = *(u32 *)(r10 -4)")
234 __success
wrong_off_in_pattern2(void)235 __naked void wrong_off_in_pattern2(void)
236 {
237 asm volatile (
238 "r1 = 1;"
239 "*(u32 *)(r10 - 4) = r1;"
240 "call %[bpf_get_smp_processor_id];"
241 "r1 = *(u32 *)(r10 - 4);"
242 "exit;"
243 :
244 : __imm(bpf_get_smp_processor_id)
245 : __clobber_all);
246 }
247
248 SEC("raw_tp")
249 __arch_x86_64
250 __xlated("1: *(u32 *)(r10 -16) = r1")
251 __xlated("...")
252 __xlated("3: r0 = &(void __percpu *)(r0)")
253 __xlated("...")
254 __xlated("5: r1 = *(u32 *)(r10 -16)")
255 __success
wrong_size_in_pattern(void)256 __naked void wrong_size_in_pattern(void)
257 {
258 asm volatile (
259 "r1 = 1;"
260 "*(u32 *)(r10 - 16) = r1;"
261 "call %[bpf_get_smp_processor_id];"
262 "r1 = *(u32 *)(r10 - 16);"
263 "exit;"
264 :
265 : __imm(bpf_get_smp_processor_id)
266 : __clobber_all);
267 }
268
269 SEC("raw_tp")
270 __arch_x86_64
271 __xlated("2: *(u32 *)(r10 -8) = r1")
272 __xlated("...")
273 __xlated("4: r0 = &(void __percpu *)(r0)")
274 __xlated("...")
275 __xlated("6: r1 = *(u32 *)(r10 -8)")
276 __success
partial_pattern(void)277 __naked void partial_pattern(void)
278 {
279 asm volatile (
280 "r1 = 1;"
281 "r2 = 2;"
282 "*(u32 *)(r10 - 8) = r1;"
283 "*(u64 *)(r10 - 16) = r2;"
284 "call %[bpf_get_smp_processor_id];"
285 "r2 = *(u64 *)(r10 - 16);"
286 "r1 = *(u32 *)(r10 - 8);"
287 "exit;"
288 :
289 : __imm(bpf_get_smp_processor_id)
290 : __clobber_all);
291 }
292
293 SEC("raw_tp")
294 __arch_x86_64
295 __xlated("0: r1 = 1")
296 __xlated("1: r2 = 2")
297 /* not patched, spills for -8, -16 not removed */
298 __xlated("2: *(u64 *)(r10 -8) = r1")
299 __xlated("3: *(u64 *)(r10 -16) = r2")
300 __xlated("...")
301 __xlated("5: r0 = &(void __percpu *)(r0)")
302 __xlated("...")
303 __xlated("7: r2 = *(u64 *)(r10 -16)")
304 __xlated("8: r1 = *(u64 *)(r10 -8)")
305 /* patched, spills for -24, -32 removed */
306 __xlated("...")
307 __xlated("10: r0 = &(void __percpu *)(r0)")
308 __xlated("...")
309 __xlated("12: exit")
310 __success
min_stack_offset(void)311 __naked void min_stack_offset(void)
312 {
313 asm volatile (
314 "r1 = 1;"
315 "r2 = 2;"
316 /* this call won't be patched */
317 "*(u64 *)(r10 - 8) = r1;"
318 "*(u64 *)(r10 - 16) = r2;"
319 "call %[bpf_get_smp_processor_id];"
320 "r2 = *(u64 *)(r10 - 16);"
321 "r1 = *(u64 *)(r10 - 8);"
322 /* this call would be patched */
323 "*(u64 *)(r10 - 24) = r1;"
324 "*(u64 *)(r10 - 32) = r2;"
325 "call %[bpf_get_smp_processor_id];"
326 "r2 = *(u64 *)(r10 - 32);"
327 "r1 = *(u64 *)(r10 - 24);"
328 "exit;"
329 :
330 : __imm(bpf_get_smp_processor_id)
331 : __clobber_all);
332 }
333
334 SEC("raw_tp")
335 __arch_x86_64
336 __xlated("1: *(u64 *)(r10 -8) = r1")
337 __xlated("...")
338 __xlated("3: r0 = &(void __percpu *)(r0)")
339 __xlated("...")
340 __xlated("5: r1 = *(u64 *)(r10 -8)")
341 __success
bad_fixed_read(void)342 __naked void bad_fixed_read(void)
343 {
344 asm volatile (
345 "r1 = 1;"
346 "*(u64 *)(r10 - 8) = r1;"
347 "call %[bpf_get_smp_processor_id];"
348 "r1 = *(u64 *)(r10 - 8);"
349 "r1 = r10;"
350 "r1 += -8;"
351 "r1 = *(u64 *)(r1 - 0);"
352 "exit;"
353 :
354 : __imm(bpf_get_smp_processor_id)
355 : __clobber_all);
356 }
357
358 SEC("raw_tp")
359 __arch_x86_64
360 __xlated("1: *(u64 *)(r10 -8) = r1")
361 __xlated("...")
362 __xlated("3: r0 = &(void __percpu *)(r0)")
363 __xlated("...")
364 __xlated("5: r1 = *(u64 *)(r10 -8)")
365 __success
bad_fixed_write(void)366 __naked void bad_fixed_write(void)
367 {
368 asm volatile (
369 "r1 = 1;"
370 "*(u64 *)(r10 - 8) = r1;"
371 "call %[bpf_get_smp_processor_id];"
372 "r1 = *(u64 *)(r10 - 8);"
373 "r1 = r10;"
374 "r1 += -8;"
375 "*(u64 *)(r1 - 0) = r1;"
376 "exit;"
377 :
378 : __imm(bpf_get_smp_processor_id)
379 : __clobber_all);
380 }
381
382 SEC("raw_tp")
383 __arch_x86_64
384 __xlated("6: *(u64 *)(r10 -16) = r1")
385 __xlated("...")
386 __xlated("8: r0 = &(void __percpu *)(r0)")
387 __xlated("...")
388 __xlated("10: r1 = *(u64 *)(r10 -16)")
389 __success
bad_varying_read(void)390 __naked void bad_varying_read(void)
391 {
392 asm volatile (
393 "r6 = *(u64 *)(r1 + 0);" /* random scalar value */
394 "r6 &= 0x7;" /* r6 range [0..7] */
395 "r6 += 0x2;" /* r6 range [2..9] */
396 "r7 = 0;"
397 "r7 -= r6;" /* r7 range [-9..-2] */
398 "r1 = 1;"
399 "*(u64 *)(r10 - 16) = r1;"
400 "call %[bpf_get_smp_processor_id];"
401 "r1 = *(u64 *)(r10 - 16);"
402 "r1 = r10;"
403 "r1 += r7;"
404 "r1 = *(u8 *)(r1 - 0);" /* touches slot [-16..-9] where spills are stored */
405 "exit;"
406 :
407 : __imm(bpf_get_smp_processor_id)
408 : __clobber_all);
409 }
410
411 SEC("raw_tp")
412 __arch_x86_64
413 __xlated("6: *(u64 *)(r10 -16) = r1")
414 __xlated("...")
415 __xlated("8: r0 = &(void __percpu *)(r0)")
416 __xlated("...")
417 __xlated("10: r1 = *(u64 *)(r10 -16)")
418 __success
bad_varying_write(void)419 __naked void bad_varying_write(void)
420 {
421 asm volatile (
422 "r6 = *(u64 *)(r1 + 0);" /* random scalar value */
423 "r6 &= 0x7;" /* r6 range [0..7] */
424 "r6 += 0x2;" /* r6 range [2..9] */
425 "r7 = 0;"
426 "r7 -= r6;" /* r7 range [-9..-2] */
427 "r1 = 1;"
428 "*(u64 *)(r10 - 16) = r1;"
429 "call %[bpf_get_smp_processor_id];"
430 "r1 = *(u64 *)(r10 - 16);"
431 "r1 = r10;"
432 "r1 += r7;"
433 "*(u8 *)(r1 - 0) = r7;" /* touches slot [-16..-9] where spills are stored */
434 "exit;"
435 :
436 : __imm(bpf_get_smp_processor_id)
437 : __clobber_all);
438 }
439
440 SEC("raw_tp")
441 __arch_x86_64
442 __xlated("1: *(u64 *)(r10 -8) = r1")
443 __xlated("...")
444 __xlated("3: r0 = &(void __percpu *)(r0)")
445 __xlated("...")
446 __xlated("5: r1 = *(u64 *)(r10 -8)")
447 __success
bad_write_in_subprog(void)448 __naked void bad_write_in_subprog(void)
449 {
450 asm volatile (
451 "r1 = 1;"
452 "*(u64 *)(r10 - 8) = r1;"
453 "call %[bpf_get_smp_processor_id];"
454 "r1 = *(u64 *)(r10 - 8);"
455 "r1 = r10;"
456 "r1 += -8;"
457 "call bad_write_in_subprog_aux;"
458 "exit;"
459 :
460 : __imm(bpf_get_smp_processor_id)
461 : __clobber_all);
462 }
463
464 __used
bad_write_in_subprog_aux(void)465 __naked static void bad_write_in_subprog_aux(void)
466 {
467 asm volatile (
468 "r0 = 1;"
469 "*(u64 *)(r1 - 0) = r0;" /* invalidates bpf_fastcall contract for caller: */
470 "exit;" /* caller stack at -8 used outside of the pattern */
471 ::: __clobber_all);
472 }
473
474 SEC("raw_tp")
475 __arch_x86_64
476 __xlated("1: *(u64 *)(r10 -8) = r1")
477 __xlated("...")
478 __xlated("3: r0 = &(void __percpu *)(r0)")
479 __xlated("...")
480 __xlated("5: r1 = *(u64 *)(r10 -8)")
481 __success
bad_helper_write(void)482 __naked void bad_helper_write(void)
483 {
484 asm volatile (
485 "r1 = 1;"
486 /* bpf_fastcall pattern with stack offset -8 */
487 "*(u64 *)(r10 - 8) = r1;"
488 "call %[bpf_get_smp_processor_id];"
489 "r1 = *(u64 *)(r10 - 8);"
490 "r1 = r10;"
491 "r1 += -8;"
492 "r2 = 1;"
493 "r3 = 42;"
494 /* read dst is fp[-8], thus bpf_fastcall rewrite not applied */
495 "call %[bpf_probe_read_kernel];"
496 "exit;"
497 :
498 : __imm(bpf_get_smp_processor_id),
499 __imm(bpf_probe_read_kernel)
500 : __clobber_all);
501 }
502
503 SEC("raw_tp")
504 __arch_x86_64
505 /* main, not patched */
506 __xlated("1: *(u64 *)(r10 -8) = r1")
507 __xlated("...")
508 __xlated("3: r0 = &(void __percpu *)(r0)")
509 __xlated("...")
510 __xlated("5: r1 = *(u64 *)(r10 -8)")
511 __xlated("...")
512 __xlated("9: call pc+1")
513 __xlated("...")
514 __xlated("10: exit")
515 /* subprogram, patched */
516 __xlated("11: r1 = 1")
517 __xlated("...")
518 __xlated("13: r0 = &(void __percpu *)(r0)")
519 __xlated("...")
520 __xlated("15: exit")
521 __success
invalidate_one_subprog(void)522 __naked void invalidate_one_subprog(void)
523 {
524 asm volatile (
525 "r1 = 1;"
526 "*(u64 *)(r10 - 8) = r1;"
527 "call %[bpf_get_smp_processor_id];"
528 "r1 = *(u64 *)(r10 - 8);"
529 "r1 = r10;"
530 "r1 += -8;"
531 "r1 = *(u64 *)(r1 - 0);"
532 "call invalidate_one_subprog_aux;"
533 "exit;"
534 :
535 : __imm(bpf_get_smp_processor_id)
536 : __clobber_all);
537 }
538
539 __used
invalidate_one_subprog_aux(void)540 __naked static void invalidate_one_subprog_aux(void)
541 {
542 asm volatile (
543 "r1 = 1;"
544 "*(u64 *)(r10 - 8) = r1;"
545 "call %[bpf_get_smp_processor_id];"
546 "r1 = *(u64 *)(r10 - 8);"
547 "exit;"
548 :
549 : __imm(bpf_get_smp_processor_id)
550 : __clobber_all);
551 }
552
553 SEC("raw_tp")
554 __arch_x86_64
555 /* main */
556 __xlated("0: r1 = 1")
557 __xlated("...")
558 __xlated("2: r0 = &(void __percpu *)(r0)")
559 __xlated("...")
560 __xlated("4: call pc+1")
561 __xlated("5: exit")
562 /* subprogram */
563 __xlated("6: r1 = 1")
564 __xlated("...")
565 __xlated("8: r0 = &(void __percpu *)(r0)")
566 __xlated("...")
567 __xlated("10: *(u64 *)(r10 -16) = r1")
568 __xlated("11: exit")
569 __success
subprogs_use_independent_offsets(void)570 __naked void subprogs_use_independent_offsets(void)
571 {
572 asm volatile (
573 "r1 = 1;"
574 "*(u64 *)(r10 - 16) = r1;"
575 "call %[bpf_get_smp_processor_id];"
576 "r1 = *(u64 *)(r10 - 16);"
577 "call subprogs_use_independent_offsets_aux;"
578 "exit;"
579 :
580 : __imm(bpf_get_smp_processor_id)
581 : __clobber_all);
582 }
583
584 __used
subprogs_use_independent_offsets_aux(void)585 __naked static void subprogs_use_independent_offsets_aux(void)
586 {
587 asm volatile (
588 "r1 = 1;"
589 "*(u64 *)(r10 - 24) = r1;"
590 "call %[bpf_get_smp_processor_id];"
591 "r1 = *(u64 *)(r10 - 24);"
592 "*(u64 *)(r10 - 16) = r1;"
593 "exit;"
594 :
595 : __imm(bpf_get_smp_processor_id)
596 : __clobber_all);
597 }
598
599 SEC("raw_tp")
600 __arch_x86_64
601 __log_level(4) __msg("stack depth 8")
602 __xlated("2: r0 = &(void __percpu *)(r0)")
603 __success
helper_call_does_not_prevent_bpf_fastcall(void)604 __naked void helper_call_does_not_prevent_bpf_fastcall(void)
605 {
606 asm volatile (
607 "r1 = 1;"
608 "*(u64 *)(r10 - 8) = r1;"
609 "call %[bpf_get_smp_processor_id];"
610 "r1 = *(u64 *)(r10 - 8);"
611 "*(u64 *)(r10 - 8) = r1;"
612 "call %[bpf_get_prandom_u32];"
613 "r1 = *(u64 *)(r10 - 8);"
614 "exit;"
615 :
616 : __imm(bpf_get_smp_processor_id),
617 __imm(bpf_get_prandom_u32)
618 : __clobber_all);
619 }
620
621 SEC("raw_tp")
622 __arch_x86_64
623 __log_level(4) __msg("stack depth 24")
624 /* may_goto counter at -24 */
625 __xlated("0: *(u64 *)(r10 -24) =")
626 /* may_goto timestamp at -16 */
627 __xlated("1: *(u64 *)(r10 -16) =")
628 __xlated("2: r1 = 1")
629 __xlated("...")
630 __xlated("4: r0 = &(void __percpu *)(r0)")
631 __xlated("...")
632 /* may_goto expansion starts */
633 __xlated("6: r11 = *(u64 *)(r10 -24)")
634 __xlated("7: if r11 == 0x0 goto pc+6")
635 __xlated("8: r11 -= 1")
636 __xlated("9: if r11 != 0x0 goto pc+2")
637 __xlated("10: r11 = -24")
638 __xlated("11: call unknown")
639 __xlated("12: *(u64 *)(r10 -24) = r11")
640 /* may_goto expansion ends */
641 __xlated("13: *(u64 *)(r10 -8) = r1")
642 __xlated("14: exit")
643 __success
may_goto_interaction_x86_64(void)644 __naked void may_goto_interaction_x86_64(void)
645 {
646 asm volatile (
647 "r1 = 1;"
648 "*(u64 *)(r10 - 16) = r1;"
649 "call %[bpf_get_smp_processor_id];"
650 "r1 = *(u64 *)(r10 - 16);"
651 ".8byte %[may_goto];"
652 /* just touch some stack at -8 */
653 "*(u64 *)(r10 - 8) = r1;"
654 "exit;"
655 :
656 : __imm(bpf_get_smp_processor_id),
657 __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, +1 /* offset */, 0))
658 : __clobber_all);
659 }
660
661 SEC("raw_tp")
662 __arch_arm64
663 __log_level(4) __msg("stack depth 16")
664 /* may_goto counter at -16 */
665 __xlated("0: *(u64 *)(r10 -16) =")
666 __xlated("1: r1 = 1")
667 __xlated("2: call bpf_get_smp_processor_id")
668 /* may_goto expansion starts */
669 __xlated("3: r11 = *(u64 *)(r10 -16)")
670 __xlated("4: if r11 == 0x0 goto pc+3")
671 __xlated("5: r11 -= 1")
672 __xlated("6: *(u64 *)(r10 -16) = r11")
673 /* may_goto expansion ends */
674 __xlated("7: *(u64 *)(r10 -8) = r1")
675 __xlated("8: exit")
676 __success
may_goto_interaction_arm64(void)677 __naked void may_goto_interaction_arm64(void)
678 {
679 asm volatile (
680 "r1 = 1;"
681 "*(u64 *)(r10 - 16) = r1;"
682 "call %[bpf_get_smp_processor_id];"
683 "r1 = *(u64 *)(r10 - 16);"
684 ".8byte %[may_goto];"
685 /* just touch some stack at -8 */
686 "*(u64 *)(r10 - 8) = r1;"
687 "exit;"
688 :
689 : __imm(bpf_get_smp_processor_id),
690 __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, +1 /* offset */, 0))
691 : __clobber_all);
692 }
693
694 __used
dummy_loop_callback(void)695 __naked static void dummy_loop_callback(void)
696 {
697 asm volatile (
698 "r0 = 0;"
699 "exit;"
700 ::: __clobber_all);
701 }
702
703 SEC("raw_tp")
704 __arch_x86_64
705 __log_level(4) __msg("stack depth 32+0")
706 __xlated("2: r1 = 1")
707 __xlated("3: r0 =")
708 __xlated("4: r0 = &(void __percpu *)(r0)")
709 __xlated("5: r0 = *(u32 *)(r0 +0)")
710 /* bpf_loop params setup */
711 __xlated("6: r2 =")
712 __xlated("7: r3 = 0")
713 __xlated("8: r4 = 0")
714 __xlated("...")
715 /* ... part of the inlined bpf_loop */
716 __xlated("12: *(u64 *)(r10 -32) = r6")
717 __xlated("13: *(u64 *)(r10 -24) = r7")
718 __xlated("14: *(u64 *)(r10 -16) = r8")
719 __xlated("...")
720 __xlated("21: call pc+8") /* dummy_loop_callback */
721 /* ... last insns of the bpf_loop_interaction1 */
722 __xlated("...")
723 __xlated("28: r0 = 0")
724 __xlated("29: exit")
725 /* dummy_loop_callback */
726 __xlated("30: r0 = 0")
727 __xlated("31: exit")
728 __success
bpf_loop_interaction1(void)729 __naked int bpf_loop_interaction1(void)
730 {
731 asm volatile (
732 "r1 = 1;"
733 /* bpf_fastcall stack region at -16, but could be removed */
734 "*(u64 *)(r10 - 16) = r1;"
735 "call %[bpf_get_smp_processor_id];"
736 "r1 = *(u64 *)(r10 - 16);"
737 "r2 = %[dummy_loop_callback];"
738 "r3 = 0;"
739 "r4 = 0;"
740 "call %[bpf_loop];"
741 "r0 = 0;"
742 "exit;"
743 :
744 : __imm_ptr(dummy_loop_callback),
745 __imm(bpf_get_smp_processor_id),
746 __imm(bpf_loop)
747 : __clobber_common
748 );
749 }
750
751 SEC("raw_tp")
752 __arch_x86_64
753 __log_level(4) __msg("stack depth 40+0")
754 /* call bpf_get_smp_processor_id */
755 __xlated("2: r1 = 42")
756 __xlated("3: r0 =")
757 __xlated("4: r0 = &(void __percpu *)(r0)")
758 __xlated("5: r0 = *(u32 *)(r0 +0)")
759 /* call bpf_get_prandom_u32 */
760 __xlated("6: *(u64 *)(r10 -16) = r1")
761 __xlated("7: call")
762 __xlated("8: r1 = *(u64 *)(r10 -16)")
763 __xlated("...")
764 /* ... part of the inlined bpf_loop */
765 __xlated("15: *(u64 *)(r10 -40) = r6")
766 __xlated("16: *(u64 *)(r10 -32) = r7")
767 __xlated("17: *(u64 *)(r10 -24) = r8")
768 __success
bpf_loop_interaction2(void)769 __naked int bpf_loop_interaction2(void)
770 {
771 asm volatile (
772 "r1 = 42;"
773 /* bpf_fastcall stack region at -16, cannot be removed */
774 "*(u64 *)(r10 - 16) = r1;"
775 "call %[bpf_get_smp_processor_id];"
776 "r1 = *(u64 *)(r10 - 16);"
777 "*(u64 *)(r10 - 16) = r1;"
778 "call %[bpf_get_prandom_u32];"
779 "r1 = *(u64 *)(r10 - 16);"
780 "r2 = %[dummy_loop_callback];"
781 "r3 = 0;"
782 "r4 = 0;"
783 "call %[bpf_loop];"
784 "r0 = 0;"
785 "exit;"
786 :
787 : __imm_ptr(dummy_loop_callback),
788 __imm(bpf_get_smp_processor_id),
789 __imm(bpf_get_prandom_u32),
790 __imm(bpf_loop)
791 : __clobber_common
792 );
793 }
794
795 SEC("raw_tp")
796 __arch_x86_64
797 __log_level(4)
798 __msg("stack depth 512+0")
799 /* just to print xlated version when debugging */
800 __xlated("r0 = &(void __percpu *)(r0)")
801 __success
802 /* cumulative_stack_depth() stack usage is MAX_BPF_STACK,
803 * called subprogram uses an additional slot for bpf_fastcall spill/fill,
804 * since bpf_fastcall spill/fill could be removed the program still fits
805 * in MAX_BPF_STACK and should be accepted.
806 */
cumulative_stack_depth(void)807 __naked int cumulative_stack_depth(void)
808 {
809 asm volatile(
810 "r1 = 42;"
811 "*(u64 *)(r10 - %[max_bpf_stack]) = r1;"
812 "call cumulative_stack_depth_subprog;"
813 "exit;"
814 :
815 : __imm_const(max_bpf_stack, MAX_BPF_STACK)
816 : __clobber_all
817 );
818 }
819
820 __used
cumulative_stack_depth_subprog(void)821 __naked static void cumulative_stack_depth_subprog(void)
822 {
823 asm volatile (
824 "*(u64 *)(r10 - 8) = r1;"
825 "call %[bpf_get_smp_processor_id];"
826 "r1 = *(u64 *)(r10 - 8);"
827 "exit;"
828 :: __imm(bpf_get_smp_processor_id) : __clobber_all);
829 }
830
831 SEC("cgroup/getsockname_unix")
832 __xlated("0: r2 = 1")
833 /* bpf_cast_to_kern_ctx is replaced by a single assignment */
834 __xlated("1: r0 = r1")
835 __xlated("2: r0 = r2")
836 __xlated("3: exit")
837 __success
kfunc_bpf_cast_to_kern_ctx(void)838 __naked void kfunc_bpf_cast_to_kern_ctx(void)
839 {
840 asm volatile (
841 "r2 = 1;"
842 "*(u64 *)(r10 - 32) = r2;"
843 "call %[bpf_cast_to_kern_ctx];"
844 "r2 = *(u64 *)(r10 - 32);"
845 "r0 = r2;"
846 "exit;"
847 :
848 : __imm(bpf_cast_to_kern_ctx)
849 : __clobber_all);
850 }
851
852 SEC("raw_tp")
853 __xlated("3: r3 = 1")
854 /* bpf_rdonly_cast is replaced by a single assignment */
855 __xlated("4: r0 = r1")
856 __xlated("5: r0 = r3")
kfunc_bpf_rdonly_cast(void)857 void kfunc_bpf_rdonly_cast(void)
858 {
859 asm volatile (
860 "r2 = %[btf_id];"
861 "r3 = 1;"
862 "*(u64 *)(r10 - 32) = r3;"
863 "call %[bpf_rdonly_cast];"
864 "r3 = *(u64 *)(r10 - 32);"
865 "r0 = r3;"
866 :
867 : __imm(bpf_rdonly_cast),
868 [btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr))
869 : __clobber_common);
870 }
871
872 /* BTF FUNC records are not generated for kfuncs referenced
873 * from inline assembly. These records are necessary for
874 * libbpf to link the program. The function below is a hack
875 * to ensure that BTF FUNC records are generated.
876 */
kfunc_root(void)877 void kfunc_root(void)
878 {
879 bpf_cast_to_kern_ctx(0);
880 bpf_rdonly_cast(0, 0);
881 }
882
883 char _license[] SEC("license") = "GPL";
884