1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __TASK_LOCAL_DATA_BPF_H
3 #define __TASK_LOCAL_DATA_BPF_H
4
5 /*
6 * Task local data is a library that facilitates sharing per-task data
7 * between user space and bpf programs.
8 *
9 *
10 * USAGE
11 *
12 * A TLD, an entry of data in task local data, first needs to be created by the
13 * user space. This is done by calling user space API, TLD_DEFINE_KEY() or
14 * tld_create_key(), with the name of the TLD and the size.
15 *
16 * TLD_DEFINE_KEY(prio, "priority", sizeof(int));
17 *
18 * or
19 *
20 * void func_call(...) {
21 * tld_key_t prio, in_cs;
22 *
23 * prio = tld_create_key("priority", sizeof(int));
24 * in_cs = tld_create_key("in_critical_section", sizeof(bool));
25 * ...
26 *
27 * A key associated with the TLD, which has an opaque type tld_key_t, will be
28 * initialized or returned. It can be used to get a pointer to the TLD in the
29 * user space by calling tld_get_data().
30 *
31 * In a bpf program, tld_object_init() first needs to be called to initialized a
32 * tld_object on the stack. Then, TLDs can be accessed by calling tld_get_data().
33 * The API will try to fetch the key by the name and use it to locate the data.
34 * A pointer to the TLD will be returned. It also caches the key in a task local
35 * storage map, tld_key_map, whose value type, struct tld_keys, must be defined
36 * by the developer.
37 *
38 * struct tld_keys {
39 * tld_key_t prio;
40 * tld_key_t in_cs;
41 * };
42 *
43 * SEC("struct_ops")
44 * void prog(struct task_struct task, ...)
45 * {
46 * struct tld_object tld_obj;
47 * int err, *p;
48 *
49 * err = tld_object_init(task, &tld_obj);
50 * if (err)
51 * return;
52 *
53 * p = tld_get_data(&tld_obj, prio, "priority", sizeof(int));
54 * if (p)
55 * // do something depending on *p
56 */
57 #include <errno.h>
58 #include <bpf/bpf_helpers.h>
59
60 #define TLD_ROUND_MASK(x, y) ((__typeof__(x))((y) - 1))
61 #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1)
62
63 #define TLD_MAX_DATA_CNT (__PAGE_SIZE / sizeof(struct tld_metadata) - 1)
64
65 #ifndef TLD_NAME_LEN
66 #define TLD_NAME_LEN 62
67 #endif
68
69 #ifndef TLD_KEY_MAP_CREATE_RETRY
70 #define TLD_KEY_MAP_CREATE_RETRY 10
71 #endif
72
73 typedef struct {
74 __s16 off;
75 } tld_key_t;
76
77 struct tld_metadata {
78 char name[TLD_NAME_LEN];
79 __u16 size;
80 };
81
82 struct tld_meta_u {
83 __u8 cnt;
84 __u16 size;
85 struct tld_metadata metadata[TLD_MAX_DATA_CNT];
86 };
87
88 struct tld_data_u {
89 __u64 start; /* offset of tld_data_u->data in a page */
90 char data[__PAGE_SIZE - sizeof(__u64)];
91 };
92
93 struct tld_map_value {
94 struct tld_data_u __uptr *data;
95 struct tld_meta_u __uptr *meta;
96 };
97
98 typedef struct tld_uptr_dummy {
99 struct tld_data_u data[0];
100 struct tld_meta_u meta[0];
101 } *tld_uptr_dummy_t;
102
103 struct tld_object {
104 struct tld_map_value *data_map;
105 struct tld_keys *key_map;
106 /*
107 * Force the compiler to generate the actual definition of tld_meta_u
108 * and tld_data_u in BTF. Without it, tld_meta_u and u_tld_data will
109 * be BTF_KIND_FWD.
110 */
111 tld_uptr_dummy_t dummy[0];
112 };
113
114 /*
115 * Map value of tld_key_map for caching keys. Must be defined by the developer.
116 * Members should be tld_key_t and passed to the 3rd argument of tld_fetch_key().
117 */
118 struct tld_keys;
119
120 struct {
121 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
122 __uint(map_flags, BPF_F_NO_PREALLOC);
123 __type(key, int);
124 __type(value, struct tld_map_value);
125 } tld_data_map SEC(".maps");
126
127 struct {
128 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
129 __uint(map_flags, BPF_F_NO_PREALLOC);
130 __type(key, int);
131 __type(value, struct tld_keys);
132 } tld_key_map SEC(".maps");
133
134 /**
135 * tld_object_init() - Initialize a tld_object.
136 *
137 * @task: The task_struct of the target task
138 * @tld_obj: A pointer to a tld_object to be initialized
139 *
140 * Return 0 on success; -ENODATA if the user space did not initialize task local data
141 * for the current task through tld_get_data(); -ENOMEM if the creation of tld_key_map
142 * fails
143 */
144 __attribute__((unused))
tld_object_init(struct task_struct * task,struct tld_object * tld_obj)145 static int tld_object_init(struct task_struct *task, struct tld_object *tld_obj)
146 {
147 int i;
148
149 tld_obj->data_map = bpf_task_storage_get(&tld_data_map, task, 0, 0);
150 if (!tld_obj->data_map)
151 return -ENODATA;
152
153 bpf_for(i, 0, TLD_KEY_MAP_CREATE_RETRY) {
154 tld_obj->key_map = bpf_task_storage_get(&tld_key_map, task, 0,
155 BPF_LOCAL_STORAGE_GET_F_CREATE);
156 if (likely(tld_obj->key_map))
157 break;
158 }
159 if (!tld_obj->key_map)
160 return -ENOMEM;
161
162 return 0;
163 }
164
165 /*
166 * Return the offset of TLD if @name is found. Otherwise, return the current TLD count
167 * using the nonpositive range so that the next tld_get_data() can skip fetching key if
168 * no new TLD is added or start comparing name from the first newly added TLD.
169 */
170 __attribute__((unused))
__tld_fetch_key(struct tld_object * tld_obj,const char * name,int i_start)171 static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_start)
172 {
173 struct tld_metadata *metadata;
174 int i, cnt, start, off = 0;
175
176 if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta)
177 return 0;
178
179 start = tld_obj->data_map->data->start;
180 cnt = tld_obj->data_map->meta->cnt;
181 metadata = tld_obj->data_map->meta->metadata;
182
183 bpf_for(i, 0, cnt) {
184 if (i >= TLD_MAX_DATA_CNT)
185 break;
186
187 if (i >= i_start && !bpf_strncmp(metadata[i].name, TLD_NAME_LEN, name))
188 return start + off;
189
190 off += TLD_ROUND_UP(metadata[i].size, 8);
191 }
192
193 return -cnt;
194 }
195
196 /**
197 * tld_get_data() - Retrieve a pointer to the TLD associated with the name.
198 *
199 * @tld_obj: A pointer to a valid tld_object initialized by tld_object_init()
200 * @key: The cached key of the TLD in tld_key_map
201 * @name: The name of the key associated with a TLD
202 * @size: The size of the TLD. Must be a known constant value
203 *
204 * Return a pointer to the TLD associated with @name; NULL if not found or @size is too
205 * big. @key is used to cache the key if the TLD is found to speed up subsequent calls.
206 * It should be defined as an member of tld_keys of tld_key_t type by the developer.
207 */
208 #define tld_get_data(tld_obj, key, name, size) \
209 ({ \
210 void *data = NULL, *_data = (tld_obj)->data_map->data; \
211 long off = (tld_obj)->key_map->key.off; \
212 int cnt; \
213 \
214 if (likely(_data)) { \
215 if (likely(off > 0)) { \
216 barrier_var(off); \
217 if (likely(off < __PAGE_SIZE - size)) \
218 data = _data + off; \
219 } else { \
220 cnt = -(off); \
221 if (likely((tld_obj)->data_map->meta) && \
222 cnt < (tld_obj)->data_map->meta->cnt) { \
223 off = __tld_fetch_key(tld_obj, name, cnt); \
224 (tld_obj)->key_map->key.off = off; \
225 \
226 if (likely(off < __PAGE_SIZE - size)) { \
227 barrier_var(off); \
228 if (off > 0) \
229 data = _data + off; \
230 } \
231 } \
232 } \
233 } \
234 data; \
235 })
236
237 #endif
238