1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2 /* Copyright (c) 2021 Facebook */
3 #ifndef __SKEL_INTERNAL_H
4 #define __SKEL_INTERNAL_H
5
6 #ifdef __KERNEL__
7 #include <linux/fdtable.h>
8 #include <linux/mm.h>
9 #include <linux/mman.h>
10 #include <linux/slab.h>
11 #include <linux/bpf.h>
12 #else
13 #include <unistd.h>
14 #include <sys/syscall.h>
15 #include <sys/mman.h>
16 #include <linux/keyctl.h>
17 #include <stdlib.h>
18 #include "bpf.h"
19 #endif
20
21 #ifndef SHA256_DIGEST_LENGTH
22 #define SHA256_DIGEST_LENGTH 32
23 #endif
24
25 #ifndef __NR_bpf
26 # if defined(__mips__) && defined(_ABIO32)
27 # define __NR_bpf 4355
28 # elif defined(__mips__) && defined(_ABIN32)
29 # define __NR_bpf 6319
30 # elif defined(__mips__) && defined(_ABI64)
31 # define __NR_bpf 5315
32 # endif
33 #endif
34
35 /* This file is a base header for auto-generated *.lskel.h files.
36 * Its contents will change and may become part of auto-generation in the future.
37 *
38 * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent
39 * and will change from one version of libbpf to another and features
40 * requested during loader program generation.
41 */
42 struct bpf_map_desc {
43 /* output of the loader prog */
44 int map_fd;
45 /* input for the loader prog */
46 __u32 max_entries;
47 __aligned_u64 initial_value;
48 };
49 struct bpf_prog_desc {
50 int prog_fd;
51 };
52
53 enum {
54 BPF_SKEL_KERNEL = (1ULL << 0),
55 };
56
57 struct bpf_loader_ctx {
58 __u32 sz;
59 __u32 flags;
60 __u32 log_level;
61 __u32 log_size;
62 __u64 log_buf;
63 };
64
65 struct bpf_load_and_run_opts {
66 struct bpf_loader_ctx *ctx;
67 const void *data;
68 const void *insns;
69 __u32 data_sz;
70 __u32 insns_sz;
71 const char *errstr;
72 void *signature;
73 __u32 signature_sz;
74 __s32 keyring_id;
75 void *excl_prog_hash;
76 __u32 excl_prog_hash_sz;
77 };
78
79 long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size);
80
skel_sys_bpf(enum bpf_cmd cmd,union bpf_attr * attr,unsigned int size)81 static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
82 unsigned int size)
83 {
84 #ifdef __KERNEL__
85 return kern_sys_bpf(cmd, attr, size);
86 #else
87 return syscall(__NR_bpf, cmd, attr, size);
88 #endif
89 }
90
91 #ifdef __KERNEL__
close(int fd)92 static inline int close(int fd)
93 {
94 return close_fd(fd);
95 }
96
skel_alloc(size_t size)97 static inline void *skel_alloc(size_t size)
98 {
99 struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL);
100
101 if (!ctx)
102 return NULL;
103 ctx->flags |= BPF_SKEL_KERNEL;
104 return ctx;
105 }
106
skel_free(const void * p)107 static inline void skel_free(const void *p)
108 {
109 kfree(p);
110 }
111
112 /* skel->bss/rodata maps are populated the following way:
113 *
114 * For kernel use:
115 * skel_prep_map_data() allocates kernel memory that kernel module can directly access.
116 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value.
117 * The loader program will perform probe_read_kernel() from maps.rodata.initial_value.
118 * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and
119 * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree
120 * is not necessary.
121 *
122 * For user space:
123 * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly.
124 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value.
125 * The loader program will perform copy_from_user() from maps.rodata.initial_value.
126 * skel_finalize_map_data() remaps bpf array map value from the kernel memory into
127 * skel->rodata address.
128 *
129 * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for
130 * both kernel and user space. The generated loader program does
131 * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value
132 * depending on bpf_loader_ctx->flags.
133 */
skel_free_map_data(void * p,__u64 addr,size_t sz)134 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz)
135 {
136 if (addr != ~0ULL)
137 kvfree(p);
138 /* When addr == ~0ULL the 'p' points to
139 * ((struct bpf_array *)map)->value. See skel_finalize_map_data.
140 */
141 }
142
skel_prep_map_data(const void * val,size_t mmap_sz,size_t val_sz)143 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz)
144 {
145 void *addr;
146
147 addr = kvmalloc(val_sz, GFP_KERNEL);
148 if (!addr)
149 return NULL;
150 memcpy(addr, val, val_sz);
151 return addr;
152 }
153
skel_finalize_map_data(__u64 * init_val,size_t mmap_sz,int flags,int fd)154 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd)
155 {
156 struct bpf_map *map;
157 void *addr = NULL;
158
159 kvfree((void *) (long) *init_val);
160 *init_val = ~0ULL;
161
162 /* At this point bpf_load_and_run() finished without error and
163 * 'fd' is a valid bpf map FD. All sanity checks below should succeed.
164 */
165 map = bpf_map_get(fd);
166 if (IS_ERR(map))
167 return NULL;
168 if (map->map_type != BPF_MAP_TYPE_ARRAY)
169 goto out;
170 addr = ((struct bpf_array *)map)->value;
171 /* the addr stays valid, since FD is not closed */
172 out:
173 bpf_map_put(map);
174 return addr;
175 }
176
177 #else
178
skel_alloc(size_t size)179 static inline void *skel_alloc(size_t size)
180 {
181 return calloc(1, size);
182 }
183
skel_free(void * p)184 static inline void skel_free(void *p)
185 {
186 free(p);
187 }
188
skel_free_map_data(void * p,__u64 addr,size_t sz)189 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz)
190 {
191 munmap(p, sz);
192 }
193
skel_prep_map_data(const void * val,size_t mmap_sz,size_t val_sz)194 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz)
195 {
196 void *addr;
197
198 addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
199 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
200 if (addr == (void *) -1)
201 return NULL;
202 memcpy(addr, val, val_sz);
203 return addr;
204 }
205
skel_finalize_map_data(__u64 * init_val,size_t mmap_sz,int flags,int fd)206 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd)
207 {
208 void *addr;
209
210 addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0);
211 if (addr == (void *) -1)
212 return NULL;
213 return addr;
214 }
215 #endif
216
skel_closenz(int fd)217 static inline int skel_closenz(int fd)
218 {
219 if (fd > 0)
220 return close(fd);
221 return -EINVAL;
222 }
223
224 #ifndef offsetofend
225 #define offsetofend(TYPE, MEMBER) \
226 (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER)))
227 #endif
228
skel_map_create(enum bpf_map_type map_type,const char * map_name,__u32 key_size,__u32 value_size,__u32 max_entries,const void * excl_prog_hash,__u32 excl_prog_hash_sz)229 static inline int skel_map_create(enum bpf_map_type map_type,
230 const char *map_name,
231 __u32 key_size,
232 __u32 value_size,
233 __u32 max_entries,
234 const void *excl_prog_hash,
235 __u32 excl_prog_hash_sz)
236 {
237 const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size);
238 union bpf_attr attr;
239
240 memset(&attr, 0, attr_sz);
241
242 attr.map_type = map_type;
243 attr.excl_prog_hash = (unsigned long) excl_prog_hash;
244 attr.excl_prog_hash_size = excl_prog_hash_sz;
245
246 strncpy(attr.map_name, map_name, sizeof(attr.map_name));
247 attr.key_size = key_size;
248 attr.value_size = value_size;
249 attr.max_entries = max_entries;
250
251 return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz);
252 }
253
skel_map_update_elem(int fd,const void * key,const void * value,__u64 flags)254 static inline int skel_map_update_elem(int fd, const void *key,
255 const void *value, __u64 flags)
256 {
257 const size_t attr_sz = offsetofend(union bpf_attr, flags);
258 union bpf_attr attr;
259
260 memset(&attr, 0, attr_sz);
261 attr.map_fd = fd;
262 attr.key = (long) key;
263 attr.value = (long) value;
264 attr.flags = flags;
265
266 return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);
267 }
268
skel_map_delete_elem(int fd,const void * key)269 static inline int skel_map_delete_elem(int fd, const void *key)
270 {
271 const size_t attr_sz = offsetofend(union bpf_attr, flags);
272 union bpf_attr attr;
273
274 memset(&attr, 0, attr_sz);
275 attr.map_fd = fd;
276 attr.key = (long)key;
277
278 return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
279 }
280
skel_map_get_fd_by_id(__u32 id)281 static inline int skel_map_get_fd_by_id(__u32 id)
282 {
283 const size_t attr_sz = offsetofend(union bpf_attr, flags);
284 union bpf_attr attr;
285
286 memset(&attr, 0, attr_sz);
287 attr.map_id = id;
288
289 return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz);
290 }
291
skel_raw_tracepoint_open(const char * name,int prog_fd)292 static inline int skel_raw_tracepoint_open(const char *name, int prog_fd)
293 {
294 const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd);
295 union bpf_attr attr;
296
297 memset(&attr, 0, attr_sz);
298 attr.raw_tracepoint.name = (long) name;
299 attr.raw_tracepoint.prog_fd = prog_fd;
300
301 return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz);
302 }
303
skel_link_create(int prog_fd,int target_fd,enum bpf_attach_type attach_type)304 static inline int skel_link_create(int prog_fd, int target_fd,
305 enum bpf_attach_type attach_type)
306 {
307 const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len);
308 union bpf_attr attr;
309
310 memset(&attr, 0, attr_sz);
311 attr.link_create.prog_fd = prog_fd;
312 attr.link_create.target_fd = target_fd;
313 attr.link_create.attach_type = attach_type;
314
315 return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz);
316 }
317
skel_obj_get_info_by_fd(int fd)318 static inline int skel_obj_get_info_by_fd(int fd)
319 {
320 const size_t attr_sz = offsetofend(union bpf_attr, info);
321 __u8 sha[SHA256_DIGEST_LENGTH];
322 struct bpf_map_info info;
323 __u32 info_len = sizeof(info);
324 union bpf_attr attr;
325
326 memset(&info, 0, sizeof(info));
327 info.hash = (long) &sha;
328 info.hash_size = SHA256_DIGEST_LENGTH;
329
330 memset(&attr, 0, attr_sz);
331 attr.info.bpf_fd = fd;
332 attr.info.info = (long) &info;
333 attr.info.info_len = info_len;
334 return skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz);
335 }
336
skel_map_freeze(int fd)337 static inline int skel_map_freeze(int fd)
338 {
339 const size_t attr_sz = offsetofend(union bpf_attr, map_fd);
340 union bpf_attr attr;
341
342 memset(&attr, 0, attr_sz);
343 attr.map_fd = fd;
344
345 return skel_sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz);
346 }
347 #ifdef __KERNEL__
348 #define set_err
349 #else
350 #define set_err err = -errno
351 #endif
352
bpf_load_and_run(struct bpf_load_and_run_opts * opts)353 static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
354 {
355 const size_t prog_load_attr_sz = offsetofend(union bpf_attr, keyring_id);
356 const size_t test_run_attr_sz = offsetofend(union bpf_attr, test);
357 int map_fd = -1, prog_fd = -1, key = 0, err;
358 union bpf_attr attr;
359
360 err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1,
361 opts->excl_prog_hash, opts->excl_prog_hash_sz);
362 if (map_fd < 0) {
363 opts->errstr = "failed to create loader map";
364 set_err;
365 goto out;
366 }
367
368 err = skel_map_update_elem(map_fd, &key, opts->data, 0);
369 if (err < 0) {
370 opts->errstr = "failed to update loader map";
371 set_err;
372 goto out;
373 }
374
375 #ifndef __KERNEL__
376 err = skel_map_freeze(map_fd);
377 if (err < 0) {
378 opts->errstr = "failed to freeze map";
379 set_err;
380 goto out;
381 }
382 err = skel_obj_get_info_by_fd(map_fd);
383 if (err < 0) {
384 opts->errstr = "failed to fetch obj info";
385 set_err;
386 goto out;
387 }
388 #endif
389
390 memset(&attr, 0, prog_load_attr_sz);
391 attr.prog_type = BPF_PROG_TYPE_SYSCALL;
392 attr.insns = (long) opts->insns;
393 attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn);
394 attr.license = (long) "Dual BSD/GPL";
395 #ifndef __KERNEL__
396 attr.signature = (long) opts->signature;
397 attr.signature_size = opts->signature_sz;
398 #else
399 if (opts->signature || opts->signature_sz)
400 pr_warn("signatures are not supported from bpf_preload\n");
401 #endif
402 attr.keyring_id = opts->keyring_id;
403 memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog"));
404 attr.fd_array = (long) &map_fd;
405 attr.log_level = opts->ctx->log_level;
406 attr.log_size = opts->ctx->log_size;
407 attr.log_buf = opts->ctx->log_buf;
408 attr.prog_flags = BPF_F_SLEEPABLE;
409 err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz);
410 if (prog_fd < 0) {
411 opts->errstr = "failed to load loader prog";
412 set_err;
413 goto out;
414 }
415
416 memset(&attr, 0, test_run_attr_sz);
417 attr.test.prog_fd = prog_fd;
418 attr.test.ctx_in = (long) opts->ctx;
419 attr.test.ctx_size_in = opts->ctx->sz;
420 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz);
421 if (err < 0 || (int)attr.test.retval < 0) {
422 if (err < 0) {
423 opts->errstr = "failed to execute loader prog";
424 set_err;
425 } else {
426 opts->errstr = "error returned by loader prog";
427 err = (int)attr.test.retval;
428 #ifndef __KERNEL__
429 errno = -err;
430 #endif
431 }
432 goto out;
433 }
434 err = 0;
435 out:
436 if (map_fd >= 0)
437 close(map_fd);
438 if (prog_fd >= 0)
439 close(prog_fd);
440 return err;
441 }
442
443 #endif
444