1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2025 Google LLC. */
3
4 #include <linux/bpf.h>
5 #include <bpf/bpf_helpers.h>
6 #include "../../../include/linux/filter.h"
7 #include "bpf_misc.h"
8
9 #ifdef CAN_USE_LOAD_ACQ_STORE_REL
10
11 SEC("socket")
12 __description("load-acquire, 8-bit")
13 __success __success_unpriv __retval(0)
load_acquire_8(void)14 __naked void load_acquire_8(void)
15 {
16 asm volatile (
17 "r0 = 0;"
18 "w1 = 0xfe;"
19 "*(u8 *)(r10 - 1) = w1;"
20 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1));
21 "if r2 == r1 goto 1f;"
22 "r0 = 1;"
23 "1:"
24 "exit;"
25 :
26 : __imm_insn(load_acquire_insn,
27 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1))
28 : __clobber_all);
29 }
30
31 SEC("socket")
32 __description("load-acquire, 16-bit")
33 __success __success_unpriv __retval(0)
load_acquire_16(void)34 __naked void load_acquire_16(void)
35 {
36 asm volatile (
37 "r0 = 0;"
38 "w1 = 0xfedc;"
39 "*(u16 *)(r10 - 2) = w1;"
40 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2));
41 "if r2 == r1 goto 1f;"
42 "r0 = 1;"
43 "1:"
44 "exit;"
45 :
46 : __imm_insn(load_acquire_insn,
47 BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2))
48 : __clobber_all);
49 }
50
51 SEC("socket")
52 __description("load-acquire, 32-bit")
53 __success __success_unpriv __retval(0)
load_acquire_32(void)54 __naked void load_acquire_32(void)
55 {
56 asm volatile (
57 "r0 = 0;"
58 "w1 = 0xfedcba09;"
59 "*(u32 *)(r10 - 4) = w1;"
60 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4));
61 "if r2 == r1 goto 1f;"
62 "r0 = 1;"
63 "1:"
64 "exit;"
65 :
66 : __imm_insn(load_acquire_insn,
67 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4))
68 : __clobber_all);
69 }
70
71 SEC("socket")
72 __description("load-acquire, 64-bit")
73 __success __success_unpriv __retval(0)
load_acquire_64(void)74 __naked void load_acquire_64(void)
75 {
76 asm volatile (
77 "r0 = 0;"
78 "r1 = 0xfedcba0987654321 ll;"
79 "*(u64 *)(r10 - 8) = r1;"
80 ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8));
81 "if r2 == r1 goto 1f;"
82 "r0 = 1;"
83 "1:"
84 "exit;"
85 :
86 : __imm_insn(load_acquire_insn,
87 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
88 : __clobber_all);
89 }
90
91 SEC("socket")
92 __description("load-acquire with uninitialized src_reg")
93 __failure __failure_unpriv __msg("R2 !read_ok")
load_acquire_with_uninitialized_src_reg(void)94 __naked void load_acquire_with_uninitialized_src_reg(void)
95 {
96 asm volatile (
97 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0));
98 "exit;"
99 :
100 : __imm_insn(load_acquire_insn,
101 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
102 : __clobber_all);
103 }
104
105 SEC("socket")
106 __description("load-acquire with non-pointer src_reg")
107 __failure __failure_unpriv __msg("R1 invalid mem access 'scalar'")
load_acquire_with_non_pointer_src_reg(void)108 __naked void load_acquire_with_non_pointer_src_reg(void)
109 {
110 asm volatile (
111 "r1 = 0;"
112 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0));
113 "exit;"
114 :
115 : __imm_insn(load_acquire_insn,
116 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
117 : __clobber_all);
118 }
119
120 SEC("socket")
121 __description("misaligned load-acquire")
122 __failure __failure_unpriv __msg("misaligned stack access off")
__flag(BPF_F_ANY_ALIGNMENT)123 __flag(BPF_F_ANY_ALIGNMENT)
124 __naked void load_acquire_misaligned(void)
125 {
126 asm volatile (
127 "r1 = 0;"
128 "*(u64 *)(r10 - 8) = r1;"
129 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5));
130 "exit;"
131 :
132 : __imm_insn(load_acquire_insn,
133 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5))
134 : __clobber_all);
135 }
136
137 SEC("socket")
138 __description("load-acquire from ctx pointer")
139 __failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed")
load_acquire_from_ctx_pointer(void)140 __naked void load_acquire_from_ctx_pointer(void)
141 {
142 asm volatile (
143 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0));
144 "exit;"
145 :
146 : __imm_insn(load_acquire_insn,
147 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
148 : __clobber_all);
149 }
150
151 SEC("xdp")
152 __description("load-acquire from pkt pointer")
153 __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed")
load_acquire_from_pkt_pointer(void)154 __naked void load_acquire_from_pkt_pointer(void)
155 {
156 asm volatile (
157 "r2 = *(u32 *)(r1 + %[xdp_md_data]);"
158 "r3 = *(u32 *)(r1 + %[xdp_md_data_end]);"
159 "r1 = r2;"
160 "r1 += 8;"
161 "if r1 >= r3 goto l0_%=;"
162 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
163 "l0_%=: r0 = 0;"
164 "exit;"
165 :
166 : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)),
167 __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)),
168 __imm_insn(load_acquire_insn,
169 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
170 : __clobber_all);
171 }
172
173 SEC("flow_dissector")
174 __description("load-acquire from flow_keys pointer")
175 __failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed")
load_acquire_from_flow_keys_pointer(void)176 __naked void load_acquire_from_flow_keys_pointer(void)
177 {
178 asm volatile (
179 "r2 = *(u64 *)(r1 + %[__sk_buff_flow_keys]);"
180 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
181 "exit;"
182 :
183 : __imm_const(__sk_buff_flow_keys,
184 offsetof(struct __sk_buff, flow_keys)),
185 __imm_insn(load_acquire_insn,
186 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
187 : __clobber_all);
188 }
189
190 SEC("sk_reuseport")
191 __description("load-acquire from sock pointer")
192 __failure __msg("BPF_ATOMIC loads from R2 sock is not allowed")
load_acquire_from_sock_pointer(void)193 __naked void load_acquire_from_sock_pointer(void)
194 {
195 asm volatile (
196 "r2 = *(u64 *)(r1 + %[sk_reuseport_md_sk]);"
197 // w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family)));
198 ".8byte %[load_acquire_insn];"
199 "exit;"
200 :
201 : __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)),
202 __imm_insn(load_acquire_insn,
203 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2,
204 offsetof(struct bpf_sock, family)))
205 : __clobber_all);
206 }
207
208 SEC("socket")
209 __description("load-acquire with invalid register R15")
210 __failure __failure_unpriv __msg("R15 is invalid")
load_acquire_with_invalid_reg(void)211 __naked void load_acquire_with_invalid_reg(void)
212 {
213 asm volatile (
214 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0));
215 "exit;"
216 :
217 : __imm_insn(load_acquire_insn,
218 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0))
219 : __clobber_all);
220 }
221
222 #else /* CAN_USE_LOAD_ACQ_STORE_REL */
223
224 SEC("socket")
225 __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test")
226 __success
dummy_test(void)227 int dummy_test(void)
228 {
229 return 0;
230 }
231
232 #endif /* CAN_USE_LOAD_ACQ_STORE_REL */
233
234 char _license[] SEC("license") = "GPL";
235