1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/pid.h> 8 #include <linux/sched.h> 9 #include <linux/rculist.h> 10 #include <linux/list.h> 11 #include <linux/hash.h> 12 #include <linux/types.h> 13 #include <linux/spinlock.h> 14 #include <linux/bpf.h> 15 #include <linux/bpf_local_storage.h> 16 #include <linux/filter.h> 17 #include <uapi/linux/btf.h> 18 #include <linux/btf_ids.h> 19 #include <linux/rcupdate_trace.h> 20 21 DEFINE_BPF_STORAGE_CACHE(task_cache); 22 23 static DEFINE_PER_CPU(int, bpf_task_storage_busy); 24 25 static void bpf_task_storage_lock(void) 26 { 27 cant_migrate(); 28 this_cpu_inc(bpf_task_storage_busy); 29 } 30 31 static void bpf_task_storage_unlock(void) 32 { 33 this_cpu_dec(bpf_task_storage_busy); 34 } 35 36 static bool bpf_task_storage_trylock(void) 37 { 38 cant_migrate(); 39 if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) { 40 this_cpu_dec(bpf_task_storage_busy); 41 return false; 42 } 43 return true; 44 } 45 46 static struct bpf_local_storage __rcu **task_storage_ptr(void *owner) 47 { 48 struct task_struct *task = owner; 49 50 return &task->bpf_storage; 51 } 52 53 static struct bpf_local_storage_data * 54 task_storage_lookup(struct task_struct *task, struct bpf_map *map, 55 bool cacheit_lockit) 56 { 57 struct bpf_local_storage *task_storage; 58 struct bpf_local_storage_map *smap; 59 60 task_storage = 61 rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held()); 62 if (!task_storage) 63 return NULL; 64 65 smap = (struct bpf_local_storage_map *)map; 66 return bpf_local_storage_lookup(task_storage, smap, cacheit_lockit); 67 } 68 69 void bpf_task_storage_free(struct task_struct *task) 70 { 71 struct bpf_local_storage *local_storage; 72 73 migrate_disable(); 74 rcu_read_lock(); 75 76 local_storage = rcu_dereference(task->bpf_storage); 77 if (!local_storage) 78 goto out; 79 80 bpf_task_storage_lock(); 81 bpf_local_storage_destroy(local_storage); 82 bpf_task_storage_unlock(); 83 out: 84 rcu_read_unlock(); 85 migrate_enable(); 86 } 87 88 static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key) 89 { 90 struct bpf_local_storage_data *sdata; 91 struct task_struct *task; 92 unsigned int f_flags; 93 struct pid *pid; 94 int fd, err; 95 96 fd = *(int *)key; 97 pid = pidfd_get_pid(fd, &f_flags); 98 if (IS_ERR(pid)) 99 return ERR_CAST(pid); 100 101 /* We should be in an RCU read side critical section, it should be safe 102 * to call pid_task. 103 */ 104 WARN_ON_ONCE(!rcu_read_lock_held()); 105 task = pid_task(pid, PIDTYPE_PID); 106 if (!task) { 107 err = -ENOENT; 108 goto out; 109 } 110 111 bpf_task_storage_lock(); 112 sdata = task_storage_lookup(task, map, true); 113 bpf_task_storage_unlock(); 114 put_pid(pid); 115 return sdata ? sdata->data : NULL; 116 out: 117 put_pid(pid); 118 return ERR_PTR(err); 119 } 120 121 static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, 122 void *value, u64 map_flags) 123 { 124 struct bpf_local_storage_data *sdata; 125 struct task_struct *task; 126 unsigned int f_flags; 127 struct pid *pid; 128 int fd, err; 129 130 if ((map_flags & BPF_F_LOCK) && btf_record_has_field(map->record, BPF_UPTR)) 131 return -EOPNOTSUPP; 132 133 fd = *(int *)key; 134 pid = pidfd_get_pid(fd, &f_flags); 135 if (IS_ERR(pid)) 136 return PTR_ERR(pid); 137 138 /* We should be in an RCU read side critical section, it should be safe 139 * to call pid_task. 140 */ 141 WARN_ON_ONCE(!rcu_read_lock_held()); 142 task = pid_task(pid, PIDTYPE_PID); 143 if (!task) { 144 err = -ENOENT; 145 goto out; 146 } 147 148 bpf_task_storage_lock(); 149 sdata = bpf_local_storage_update( 150 task, (struct bpf_local_storage_map *)map, value, map_flags, 151 true, GFP_ATOMIC); 152 bpf_task_storage_unlock(); 153 154 err = PTR_ERR_OR_ZERO(sdata); 155 out: 156 put_pid(pid); 157 return err; 158 } 159 160 static int task_storage_delete(struct task_struct *task, struct bpf_map *map, 161 bool nobusy) 162 { 163 struct bpf_local_storage_data *sdata; 164 165 sdata = task_storage_lookup(task, map, false); 166 if (!sdata) 167 return -ENOENT; 168 169 if (!nobusy) 170 return -EBUSY; 171 172 bpf_selem_unlink(SELEM(sdata), false); 173 174 return 0; 175 } 176 177 static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key) 178 { 179 struct task_struct *task; 180 unsigned int f_flags; 181 struct pid *pid; 182 int fd, err; 183 184 fd = *(int *)key; 185 pid = pidfd_get_pid(fd, &f_flags); 186 if (IS_ERR(pid)) 187 return PTR_ERR(pid); 188 189 /* We should be in an RCU read side critical section, it should be safe 190 * to call pid_task. 191 */ 192 WARN_ON_ONCE(!rcu_read_lock_held()); 193 task = pid_task(pid, PIDTYPE_PID); 194 if (!task) { 195 err = -ENOENT; 196 goto out; 197 } 198 199 bpf_task_storage_lock(); 200 err = task_storage_delete(task, map, true); 201 bpf_task_storage_unlock(); 202 out: 203 put_pid(pid); 204 return err; 205 } 206 207 /* Called by bpf_task_storage_get*() helpers */ 208 static void *__bpf_task_storage_get(struct bpf_map *map, 209 struct task_struct *task, void *value, 210 u64 flags, gfp_t gfp_flags, bool nobusy) 211 { 212 struct bpf_local_storage_data *sdata; 213 214 sdata = task_storage_lookup(task, map, nobusy); 215 if (sdata) 216 return sdata->data; 217 218 /* only allocate new storage, when the task is refcounted */ 219 if (refcount_read(&task->usage) && 220 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) { 221 sdata = bpf_local_storage_update( 222 task, (struct bpf_local_storage_map *)map, value, 223 BPF_NOEXIST, false, gfp_flags); 224 return IS_ERR(sdata) ? NULL : sdata->data; 225 } 226 227 return NULL; 228 } 229 230 /* *gfp_flags* is a hidden argument provided by the verifier */ 231 BPF_CALL_5(bpf_task_storage_get_recur, struct bpf_map *, map, struct task_struct *, 232 task, void *, value, u64, flags, gfp_t, gfp_flags) 233 { 234 bool nobusy; 235 void *data; 236 237 WARN_ON_ONCE(!bpf_rcu_lock_held()); 238 if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task) 239 return (unsigned long)NULL; 240 241 nobusy = bpf_task_storage_trylock(); 242 data = __bpf_task_storage_get(map, task, value, flags, 243 gfp_flags, nobusy); 244 if (nobusy) 245 bpf_task_storage_unlock(); 246 return (unsigned long)data; 247 } 248 249 /* *gfp_flags* is a hidden argument provided by the verifier */ 250 BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, 251 task, void *, value, u64, flags, gfp_t, gfp_flags) 252 { 253 void *data; 254 255 WARN_ON_ONCE(!bpf_rcu_lock_held()); 256 if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task) 257 return (unsigned long)NULL; 258 259 bpf_task_storage_lock(); 260 data = __bpf_task_storage_get(map, task, value, flags, 261 gfp_flags, true); 262 bpf_task_storage_unlock(); 263 return (unsigned long)data; 264 } 265 266 BPF_CALL_2(bpf_task_storage_delete_recur, struct bpf_map *, map, struct task_struct *, 267 task) 268 { 269 bool nobusy; 270 int ret; 271 272 WARN_ON_ONCE(!bpf_rcu_lock_held()); 273 if (!task) 274 return -EINVAL; 275 276 nobusy = bpf_task_storage_trylock(); 277 /* This helper must only be called from places where the lifetime of the task 278 * is guaranteed. Either by being refcounted or by being protected 279 * by an RCU read-side critical section. 280 */ 281 ret = task_storage_delete(task, map, nobusy); 282 if (nobusy) 283 bpf_task_storage_unlock(); 284 return ret; 285 } 286 287 BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *, 288 task) 289 { 290 int ret; 291 292 WARN_ON_ONCE(!bpf_rcu_lock_held()); 293 if (!task) 294 return -EINVAL; 295 296 bpf_task_storage_lock(); 297 /* This helper must only be called from places where the lifetime of the task 298 * is guaranteed. Either by being refcounted or by being protected 299 * by an RCU read-side critical section. 300 */ 301 ret = task_storage_delete(task, map, true); 302 bpf_task_storage_unlock(); 303 return ret; 304 } 305 306 static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key) 307 { 308 return -ENOTSUPP; 309 } 310 311 static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr) 312 { 313 return bpf_local_storage_map_alloc(attr, &task_cache, true); 314 } 315 316 static void task_storage_map_free(struct bpf_map *map) 317 { 318 bpf_local_storage_map_free(map, &task_cache, &bpf_task_storage_busy); 319 } 320 321 BTF_ID_LIST_GLOBAL_SINGLE(bpf_local_storage_map_btf_id, struct, bpf_local_storage_map) 322 const struct bpf_map_ops task_storage_map_ops = { 323 .map_meta_equal = bpf_map_meta_equal, 324 .map_alloc_check = bpf_local_storage_map_alloc_check, 325 .map_alloc = task_storage_map_alloc, 326 .map_free = task_storage_map_free, 327 .map_get_next_key = notsupp_get_next_key, 328 .map_lookup_elem = bpf_pid_task_storage_lookup_elem, 329 .map_update_elem = bpf_pid_task_storage_update_elem, 330 .map_delete_elem = bpf_pid_task_storage_delete_elem, 331 .map_check_btf = bpf_local_storage_map_check_btf, 332 .map_mem_usage = bpf_local_storage_map_mem_usage, 333 .map_btf_id = &bpf_local_storage_map_btf_id[0], 334 .map_owner_storage_ptr = task_storage_ptr, 335 }; 336 337 const struct bpf_func_proto bpf_task_storage_get_recur_proto = { 338 .func = bpf_task_storage_get_recur, 339 .gpl_only = false, 340 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 341 .arg1_type = ARG_CONST_MAP_PTR, 342 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 343 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 344 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 345 .arg4_type = ARG_ANYTHING, 346 }; 347 348 const struct bpf_func_proto bpf_task_storage_get_proto = { 349 .func = bpf_task_storage_get, 350 .gpl_only = false, 351 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 352 .arg1_type = ARG_CONST_MAP_PTR, 353 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 354 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 355 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 356 .arg4_type = ARG_ANYTHING, 357 }; 358 359 const struct bpf_func_proto bpf_task_storage_delete_recur_proto = { 360 .func = bpf_task_storage_delete_recur, 361 .gpl_only = false, 362 .ret_type = RET_INTEGER, 363 .arg1_type = ARG_CONST_MAP_PTR, 364 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 365 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 366 }; 367 368 const struct bpf_func_proto bpf_task_storage_delete_proto = { 369 .func = bpf_task_storage_delete, 370 .gpl_only = false, 371 .ret_type = RET_INTEGER, 372 .arg1_type = ARG_CONST_MAP_PTR, 373 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 374 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 375 }; 376