1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #ifndef __USDT_BPF_H__
4 #define __USDT_BPF_H__
5
6 #include <linux/errno.h>
7 #include "bpf_helpers.h"
8 #include "bpf_tracing.h"
9
10 /* Below types and maps are internal implementation details of libbpf's USDT
11 * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
12 * be considered an unstable API as well and might be adjusted based on user
13 * feedback from using libbpf's USDT support in production.
14 */
15
16 /* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
17 * map that keeps track of USDT argument specifications. This might be
18 * necessary if there are a lot of USDT attachments.
19 */
20 #ifndef BPF_USDT_MAX_SPEC_CNT
21 #define BPF_USDT_MAX_SPEC_CNT 256
22 #endif
23 /* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
24 * map that keeps track of IP (memory address) mapping to USDT argument
25 * specification.
26 * Note, if kernel supports BPF cookies, this map is not used and could be
27 * resized all the way to 1 to save a bit of memory.
28 */
29 #ifndef BPF_USDT_MAX_IP_CNT
30 #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
31 #endif
32
33 enum __bpf_usdt_arg_type {
34 BPF_USDT_ARG_CONST,
35 BPF_USDT_ARG_REG,
36 BPF_USDT_ARG_REG_DEREF,
37 };
38
39 struct __bpf_usdt_arg_spec {
40 /* u64 scalar interpreted depending on arg_type, see below */
41 __u64 val_off;
42 /* arg location case, see bpf_usdt_arg() for details */
43 enum __bpf_usdt_arg_type arg_type;
44 /* offset of referenced register within struct pt_regs */
45 short reg_off;
46 /* whether arg should be interpreted as signed value */
47 bool arg_signed;
48 /* number of bits that need to be cleared and, optionally,
49 * sign-extended to cast arguments that are 1, 2, or 4 bytes
50 * long into final 8-byte u64/s64 value returned to user
51 */
52 char arg_bitshift;
53 };
54
55 /* should match USDT_MAX_ARG_CNT in usdt.c exactly */
56 #define BPF_USDT_MAX_ARG_CNT 12
57 struct __bpf_usdt_spec {
58 struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
59 __u64 usdt_cookie;
60 short arg_cnt;
61 };
62
63 struct {
64 __uint(type, BPF_MAP_TYPE_ARRAY);
65 __uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
66 __type(key, int);
67 __type(value, struct __bpf_usdt_spec);
68 } __bpf_usdt_specs SEC(".maps") __weak;
69
70 struct {
71 __uint(type, BPF_MAP_TYPE_HASH);
72 __uint(max_entries, BPF_USDT_MAX_IP_CNT);
73 __type(key, long);
74 __type(value, __u32);
75 } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
76
77 extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;
78
79 static __always_inline
__bpf_usdt_spec_id(struct pt_regs * ctx)80 int __bpf_usdt_spec_id(struct pt_regs *ctx)
81 {
82 if (!LINUX_HAS_BPF_COOKIE) {
83 long ip = PT_REGS_IP(ctx);
84 int *spec_id_ptr;
85
86 spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
87 return spec_id_ptr ? *spec_id_ptr : -ESRCH;
88 }
89
90 return bpf_get_attach_cookie(ctx);
91 }
92
93 /* Return number of USDT arguments defined for currently traced USDT. */
94 __weak __hidden
bpf_usdt_arg_cnt(struct pt_regs * ctx)95 int bpf_usdt_arg_cnt(struct pt_regs *ctx)
96 {
97 struct __bpf_usdt_spec *spec;
98 int spec_id;
99
100 spec_id = __bpf_usdt_spec_id(ctx);
101 if (spec_id < 0)
102 return -ESRCH;
103
104 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
105 if (!spec)
106 return -ESRCH;
107
108 return spec->arg_cnt;
109 }
110
111 /* Returns the size in bytes of the #*arg_num* (zero-indexed) USDT argument.
112 * Returns negative error if argument is not found or arg_num is invalid.
113 */
114 static __always_inline
bpf_usdt_arg_size(struct pt_regs * ctx,__u64 arg_num)115 int bpf_usdt_arg_size(struct pt_regs *ctx, __u64 arg_num)
116 {
117 struct __bpf_usdt_arg_spec *arg_spec;
118 struct __bpf_usdt_spec *spec;
119 int spec_id;
120
121 spec_id = __bpf_usdt_spec_id(ctx);
122 if (spec_id < 0)
123 return -ESRCH;
124
125 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
126 if (!spec)
127 return -ESRCH;
128
129 if (arg_num >= BPF_USDT_MAX_ARG_CNT)
130 return -ENOENT;
131 barrier_var(arg_num);
132 if (arg_num >= spec->arg_cnt)
133 return -ENOENT;
134
135 arg_spec = &spec->args[arg_num];
136
137 /* arg_spec->arg_bitshift = 64 - arg_sz * 8
138 * so: arg_sz = (64 - arg_spec->arg_bitshift) / 8
139 */
140 return (unsigned int)(64 - arg_spec->arg_bitshift) / 8;
141 }
142
143 /* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
144 * Returns 0 on success; negative error, otherwise.
145 * On error *res is guaranteed to be set to zero.
146 */
147 __weak __hidden
bpf_usdt_arg(struct pt_regs * ctx,__u64 arg_num,long * res)148 int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)
149 {
150 struct __bpf_usdt_spec *spec;
151 struct __bpf_usdt_arg_spec *arg_spec;
152 unsigned long val;
153 int err, spec_id;
154
155 *res = 0;
156
157 spec_id = __bpf_usdt_spec_id(ctx);
158 if (spec_id < 0)
159 return -ESRCH;
160
161 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
162 if (!spec)
163 return -ESRCH;
164
165 if (arg_num >= BPF_USDT_MAX_ARG_CNT)
166 return -ENOENT;
167 barrier_var(arg_num);
168 if (arg_num >= spec->arg_cnt)
169 return -ENOENT;
170
171 arg_spec = &spec->args[arg_num];
172 switch (arg_spec->arg_type) {
173 case BPF_USDT_ARG_CONST:
174 /* Arg is just a constant ("-4@$-9" in USDT arg spec).
175 * value is recorded in arg_spec->val_off directly.
176 */
177 val = arg_spec->val_off;
178 break;
179 case BPF_USDT_ARG_REG:
180 /* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
181 * so we read the contents of that register directly from
182 * struct pt_regs. To keep things simple user-space parts
183 * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
184 */
185 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
186 if (err)
187 return err;
188 break;
189 case BPF_USDT_ARG_REG_DEREF:
190 /* Arg is in memory addressed by register, plus some offset
191 * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
192 * identified like with BPF_USDT_ARG_REG case, and the offset
193 * is in arg_spec->val_off. We first fetch register contents
194 * from pt_regs, then do another user-space probe read to
195 * fetch argument value itself.
196 */
197 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
198 if (err)
199 return err;
200 err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
201 if (err)
202 return err;
203 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
204 val >>= arg_spec->arg_bitshift;
205 #endif
206 break;
207 default:
208 return -EINVAL;
209 }
210
211 /* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
212 * necessary upper arg_bitshift bits, with sign extension if argument
213 * is signed
214 */
215 val <<= arg_spec->arg_bitshift;
216 if (arg_spec->arg_signed)
217 val = ((long)val) >> arg_spec->arg_bitshift;
218 else
219 val = val >> arg_spec->arg_bitshift;
220 *res = val;
221 return 0;
222 }
223
224 /* Retrieve user-specified cookie value provided during attach as
225 * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
226 * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
227 * utilizing BPF cookies internally, so user can't use BPF cookie directly
228 * for USDT programs and has to use bpf_usdt_cookie() API instead.
229 */
230 __weak __hidden
bpf_usdt_cookie(struct pt_regs * ctx)231 long bpf_usdt_cookie(struct pt_regs *ctx)
232 {
233 struct __bpf_usdt_spec *spec;
234 int spec_id;
235
236 spec_id = __bpf_usdt_spec_id(ctx);
237 if (spec_id < 0)
238 return 0;
239
240 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
241 if (!spec)
242 return 0;
243
244 return spec->usdt_cookie;
245 }
246
247 /* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
248 #define ___bpf_usdt_args0() ctx
249 #define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); _x; })
250 #define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); _x; })
251 #define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); _x; })
252 #define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); _x; })
253 #define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); _x; })
254 #define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); _x; })
255 #define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); _x; })
256 #define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); _x; })
257 #define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); _x; })
258 #define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); _x; })
259 #define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); _x; })
260 #define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); _x; })
261 #define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)
262
263 /*
264 * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
265 * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
266 * Original struct pt_regs * context is preserved as 'ctx' argument.
267 */
268 #define BPF_USDT(name, args...) \
269 name(struct pt_regs *ctx); \
270 static __always_inline typeof(name(0)) \
271 ____##name(struct pt_regs *ctx, ##args); \
272 typeof(name(0)) name(struct pt_regs *ctx) \
273 { \
274 _Pragma("GCC diagnostic push") \
275 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
276 return ____##name(___bpf_usdt_args(args)); \
277 _Pragma("GCC diagnostic pop") \
278 } \
279 static __always_inline typeof(name(0)) \
280 ____##name(struct pt_regs *ctx, ##args)
281
282 #endif /* __USDT_BPF_H__ */
283