Lines Matching +full:cpu +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
26 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_free_percpu()
27 free_percpu(array->pptrs[i]); in bpf_array_free_percpu()
37 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_alloc_percpu()
38 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, in bpf_array_alloc_percpu()
42 return -ENOMEM; in bpf_array_alloc_percpu()
44 array->pptrs[i] = ptr; in bpf_array_alloc_percpu()
54 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_alloc_check()
58 if (attr->max_entries == 0 || attr->key_size != 4 || in array_map_alloc_check()
59 attr->value_size == 0 || in array_map_alloc_check()
60 attr->map_flags & ~ARRAY_CREATE_FLAG_MASK || in array_map_alloc_check()
61 !bpf_map_flags_access_ok(attr->map_flags) || in array_map_alloc_check()
63 return -EINVAL; in array_map_alloc_check()
65 if (attr->map_type != BPF_MAP_TYPE_ARRAY && in array_map_alloc_check()
66 attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP)) in array_map_alloc_check()
67 return -EINVAL; in array_map_alloc_check()
69 if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY && in array_map_alloc_check()
70 attr->map_flags & BPF_F_PRESERVE_ELEMS) in array_map_alloc_check()
71 return -EINVAL; in array_map_alloc_check()
73 /* avoid overflow on round_up(map->value_size) */ in array_map_alloc_check()
74 if (attr->value_size > INT_MAX) in array_map_alloc_check()
75 return -E2BIG; in array_map_alloc_check()
82 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_alloc()
89 elem_size = round_up(attr->value_size, 8); in array_map_alloc()
91 max_entries = attr->max_entries; in array_map_alloc()
97 mask64 = fls_long(max_entries - 1); in array_map_alloc()
99 mask64 -= 1; in array_map_alloc()
104 * since cpu will speculate within index_mask limits in array_map_alloc()
108 if (max_entries < attr->max_entries) in array_map_alloc()
109 return ERR_PTR(-E2BIG); in array_map_alloc()
116 /* rely on vmalloc() to return page-aligned memory and in array_map_alloc()
117 * ensure array->value is exactly page-aligned in array_map_alloc()
119 if (attr->map_flags & BPF_F_MMAPABLE) { in array_map_alloc()
127 /* allocate all map elements and zero-initialize them */ in array_map_alloc()
128 if (attr->map_flags & BPF_F_MMAPABLE) { in array_map_alloc()
134 return ERR_PTR(-ENOMEM); in array_map_alloc()
136 - offsetof(struct bpf_array, value); in array_map_alloc()
141 return ERR_PTR(-ENOMEM); in array_map_alloc()
142 array->index_mask = index_mask; in array_map_alloc()
143 array->map.bypass_spec_v1 = bypass_spec_v1; in array_map_alloc()
145 /* copy mandatory map attributes */ in array_map_alloc()
146 bpf_map_init_from_attr(&array->map, attr); in array_map_alloc()
147 array->elem_size = elem_size; in array_map_alloc()
151 return ERR_PTR(-ENOMEM); in array_map_alloc()
154 return &array->map; in array_map_alloc()
159 return array->value + (u64)array->elem_size * index; in array_map_elem_ptr()
163 static void *array_map_lookup_elem(struct bpf_map *map, void *key) in array_map_lookup_elem() argument
165 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_lookup_elem()
168 if (unlikely(index >= array->map.max_entries)) in array_map_lookup_elem()
171 return array->value + (u64)array->elem_size * (index & array->index_mask); in array_map_lookup_elem()
174 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, in array_map_direct_value_addr() argument
177 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_addr()
179 if (map->max_entries != 1) in array_map_direct_value_addr()
180 return -ENOTSUPP; in array_map_direct_value_addr()
181 if (off >= map->value_size) in array_map_direct_value_addr()
182 return -EINVAL; in array_map_direct_value_addr()
184 *imm = (unsigned long)array->value; in array_map_direct_value_addr()
188 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, in array_map_direct_value_meta() argument
191 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_meta()
192 u64 base = (unsigned long)array->value; in array_map_direct_value_meta()
193 u64 range = array->elem_size; in array_map_direct_value_meta()
195 if (map->max_entries != 1) in array_map_direct_value_meta()
196 return -ENOTSUPP; in array_map_direct_value_meta()
198 return -ENOENT; in array_map_direct_value_meta()
200 *off = imm - base; in array_map_direct_value_meta()
205 static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in array_map_gen_lookup() argument
207 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_gen_lookup()
209 u32 elem_size = array->elem_size; in array_map_gen_lookup()
214 if (map->map_flags & BPF_F_INNER_MAP) in array_map_gen_lookup()
215 return -EOPNOTSUPP; in array_map_gen_lookup()
219 if (!map->bypass_spec_v1) { in array_map_gen_lookup()
220 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); in array_map_gen_lookup()
221 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_map_gen_lookup()
223 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3); in array_map_gen_lookup()
234 return insn - insn_buf; in array_map_gen_lookup()
238 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) in percpu_array_map_lookup_elem() argument
240 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_elem()
243 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_elem()
246 return this_cpu_ptr(array->pptrs[index & array->index_mask]); in percpu_array_map_lookup_elem()
249 static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in percpu_array_map_lookup_percpu_elem() argument
251 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_percpu_elem()
254 if (cpu >= nr_cpu_ids) in percpu_array_map_lookup_percpu_elem()
257 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_percpu_elem()
260 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu); in percpu_array_map_lookup_percpu_elem()
263 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_array_copy() argument
265 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_copy()
268 int cpu, off = 0; in bpf_percpu_array_copy() local
271 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_copy()
272 return -ENOENT; in bpf_percpu_array_copy()
274 /* per_cpu areas are zero-filled and bpf programs can only in bpf_percpu_array_copy()
278 size = array->elem_size; in bpf_percpu_array_copy()
280 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_copy()
281 for_each_possible_cpu(cpu) { in bpf_percpu_array_copy()
282 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_copy()
283 check_and_init_map_value(map, value + off); in bpf_percpu_array_copy()
291 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in array_map_get_next_key() argument
293 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_get_next_key()
297 if (index >= array->map.max_entries) { in array_map_get_next_key()
302 if (index == array->map.max_entries - 1) in array_map_get_next_key()
303 return -ENOENT; in array_map_get_next_key()
310 static long array_map_update_elem(struct bpf_map *map, void *key, void *value, in array_map_update_elem() argument
313 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_update_elem()
319 return -EINVAL; in array_map_update_elem()
321 if (unlikely(index >= array->map.max_entries)) in array_map_update_elem()
322 /* all elements were pre-allocated, cannot insert a new one */ in array_map_update_elem()
323 return -E2BIG; in array_map_update_elem()
327 return -EEXIST; in array_map_update_elem()
330 !btf_record_has_field(map->record, BPF_SPIN_LOCK))) in array_map_update_elem()
331 return -EINVAL; in array_map_update_elem()
333 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_update_elem()
334 val = this_cpu_ptr(array->pptrs[index & array->index_mask]); in array_map_update_elem()
335 copy_map_value(map, val, value); in array_map_update_elem()
336 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
338 val = array->value + in array_map_update_elem()
339 (u64)array->elem_size * (index & array->index_mask); in array_map_update_elem()
341 copy_map_value_locked(map, val, value, false); in array_map_update_elem()
343 copy_map_value(map, val, value); in array_map_update_elem()
344 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
349 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_array_update() argument
352 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_update()
355 int cpu, off = 0; in bpf_percpu_array_update() local
360 return -EINVAL; in bpf_percpu_array_update()
362 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_update()
363 /* all elements were pre-allocated, cannot insert a new one */ in bpf_percpu_array_update()
364 return -E2BIG; in bpf_percpu_array_update()
368 return -EEXIST; in bpf_percpu_array_update()
371 * will be copied into per-cpu area. bpf programs can only access in bpf_percpu_array_update()
373 * returned or zeros which were zero-filled by percpu_alloc, in bpf_percpu_array_update()
376 size = array->elem_size; in bpf_percpu_array_update()
378 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_update()
379 for_each_possible_cpu(cpu) { in bpf_percpu_array_update()
380 copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off); in bpf_percpu_array_update()
381 bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_update()
389 static long array_map_delete_elem(struct bpf_map *map, void *key) in array_map_delete_elem() argument
391 return -EINVAL; in array_map_delete_elem()
399 static void array_map_free_timers(struct bpf_map *map) in array_map_free_timers() argument
401 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free_timers()
405 if (!btf_record_has_field(map->record, BPF_TIMER)) in array_map_free_timers()
408 for (i = 0; i < array->map.max_entries; i++) in array_map_free_timers()
409 bpf_obj_free_timer(map->record, array_map_elem_ptr(array, i)); in array_map_free_timers()
412 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
413 static void array_map_free(struct bpf_map *map) in array_map_free() argument
415 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free()
418 if (!IS_ERR_OR_NULL(map->record)) { in array_map_free()
419 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_free()
420 for (i = 0; i < array->map.max_entries; i++) { in array_map_free()
421 void __percpu *pptr = array->pptrs[i & array->index_mask]; in array_map_free()
422 int cpu; in array_map_free() local
424 for_each_possible_cpu(cpu) { in array_map_free()
425 bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu)); in array_map_free()
430 for (i = 0; i < array->map.max_entries; i++) in array_map_free()
431 bpf_obj_free_fields(map->record, array_map_elem_ptr(array, i)); in array_map_free()
435 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) in array_map_free()
438 if (array->map.map_flags & BPF_F_MMAPABLE) in array_map_free()
444 static void array_map_seq_show_elem(struct bpf_map *map, void *key, in array_map_seq_show_elem() argument
451 value = array_map_lookup_elem(map, key); in array_map_seq_show_elem()
457 if (map->btf_key_type_id) in array_map_seq_show_elem()
459 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in array_map_seq_show_elem()
465 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key, in percpu_array_map_seq_show_elem() argument
468 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_seq_show_elem()
471 int cpu; in percpu_array_map_seq_show_elem() local
476 pptr = array->pptrs[index & array->index_mask]; in percpu_array_map_seq_show_elem()
477 for_each_possible_cpu(cpu) { in percpu_array_map_seq_show_elem()
478 seq_printf(m, "\tcpu%d: ", cpu); in percpu_array_map_seq_show_elem()
479 btf_type_seq_show(map->btf, map->btf_value_type_id, in percpu_array_map_seq_show_elem()
480 per_cpu_ptr(pptr, cpu), m); in percpu_array_map_seq_show_elem()
488 static int array_map_check_btf(const struct bpf_map *map, in array_map_check_btf() argument
495 /* One exception for keyless BTF: .bss/.data/.rodata map */ in array_map_check_btf()
497 if (map->map_type != BPF_MAP_TYPE_ARRAY || in array_map_check_btf()
498 map->max_entries != 1) in array_map_check_btf()
499 return -EINVAL; in array_map_check_btf()
501 if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC) in array_map_check_btf()
502 return -EINVAL; in array_map_check_btf()
507 if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT) in array_map_check_btf()
508 return -EINVAL; in array_map_check_btf()
515 return -EINVAL; in array_map_check_btf()
520 static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) in array_map_mmap() argument
522 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mmap()
525 if (!(map->map_flags & BPF_F_MMAPABLE)) in array_map_mmap()
526 return -EINVAL; in array_map_mmap()
528 if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > in array_map_mmap()
529 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) in array_map_mmap()
530 return -EINVAL; in array_map_mmap()
533 vma->vm_pgoff + pgoff); in array_map_mmap()
541 return meta0->map_flags & BPF_F_INNER_MAP ? true : in array_map_meta_equal()
542 meta0->max_entries == meta1->max_entries; in array_map_meta_equal()
546 struct bpf_map *map; member
553 struct bpf_iter_seq_array_map_info *info = seq->private; in bpf_array_map_seq_start()
554 struct bpf_map *map = info->map; in bpf_array_map_seq_start() local
558 if (info->index >= map->max_entries) in bpf_array_map_seq_start()
563 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_start()
564 index = info->index & array->index_mask; in bpf_array_map_seq_start()
565 if (info->percpu_value_buf) in bpf_array_map_seq_start()
566 return array->pptrs[index]; in bpf_array_map_seq_start()
572 struct bpf_iter_seq_array_map_info *info = seq->private; in bpf_array_map_seq_next()
573 struct bpf_map *map = info->map; in bpf_array_map_seq_next() local
578 ++info->index; in bpf_array_map_seq_next()
579 if (info->index >= map->max_entries) in bpf_array_map_seq_next()
582 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_next()
583 index = info->index & array->index_mask; in bpf_array_map_seq_next()
584 if (info->percpu_value_buf) in bpf_array_map_seq_next()
585 return array->pptrs[index]; in bpf_array_map_seq_next()
591 struct bpf_iter_seq_array_map_info *info = seq->private; in __bpf_array_map_seq_show()
593 struct bpf_map *map = info->map; in __bpf_array_map_seq_show() local
594 struct bpf_array *array = container_of(map, struct bpf_array, map); in __bpf_array_map_seq_show()
597 int off = 0, cpu = 0; in __bpf_array_map_seq_show() local
607 ctx.map = info->map; in __bpf_array_map_seq_show()
609 ctx.key = &info->index; in __bpf_array_map_seq_show()
611 if (!info->percpu_value_buf) { in __bpf_array_map_seq_show()
615 size = array->elem_size; in __bpf_array_map_seq_show()
616 for_each_possible_cpu(cpu) { in __bpf_array_map_seq_show()
617 copy_map_value_long(map, info->percpu_value_buf + off, in __bpf_array_map_seq_show()
618 per_cpu_ptr(pptr, cpu)); in __bpf_array_map_seq_show()
619 check_and_init_map_value(map, info->percpu_value_buf + off); in __bpf_array_map_seq_show()
622 ctx.value = info->percpu_value_buf; in __bpf_array_map_seq_show()
644 struct bpf_map *map = aux->map; in bpf_iter_init_array_map() local
645 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_iter_init_array_map()
649 if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in bpf_iter_init_array_map()
650 buf_size = array->elem_size * num_possible_cpus(); in bpf_iter_init_array_map()
653 return -ENOMEM; in bpf_iter_init_array_map()
655 seq_info->percpu_value_buf = value_buf; in bpf_iter_init_array_map()
658 /* bpf_iter_attach_map() acquires a map uref, and the uref may be in bpf_iter_init_array_map()
659 * released before or in the middle of iterating map elements, so in bpf_iter_init_array_map()
660 * acquire an extra map uref for iterator. in bpf_iter_init_array_map()
662 bpf_map_inc_with_uref(map); in bpf_iter_init_array_map()
663 seq_info->map = map; in bpf_iter_init_array_map()
671 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_array_map()
672 kfree(seq_info->percpu_value_buf); in bpf_iter_fini_array_map()
689 static long bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn, in bpf_for_each_array_elem() argument
699 return -EINVAL; in bpf_for_each_array_elem()
701 is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in bpf_for_each_array_elem()
702 array = container_of(map, struct bpf_array, map); in bpf_for_each_array_elem()
705 for (i = 0; i < map->max_entries; i++) { in bpf_for_each_array_elem()
707 val = this_cpu_ptr(array->pptrs[i]); in bpf_for_each_array_elem()
712 ret = callback_fn((u64)(long)map, (u64)(long)&key, in bpf_for_each_array_elem()
714 /* return value: 0 - continue, 1 - stop and return */ in bpf_for_each_array_elem()
724 static u64 array_map_mem_usage(const struct bpf_map *map) in array_map_mem_usage() argument
726 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mem_usage()
727 bool percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_mem_usage()
728 u32 elem_size = array->elem_size; in array_map_mem_usage()
729 u64 entries = map->max_entries; in array_map_mem_usage()
736 if (map->map_flags & BPF_F_MMAPABLE) { in array_map_mem_usage()
795 /* only file descriptors can be stored in this type of map */ in fd_array_map_alloc_check()
796 if (attr->value_size != sizeof(u32)) in fd_array_map_alloc_check()
797 return -EINVAL; in fd_array_map_alloc_check()
798 /* Program read-only/write-only not supported for special maps yet. */ in fd_array_map_alloc_check()
799 if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) in fd_array_map_alloc_check()
800 return -EINVAL; in fd_array_map_alloc_check()
804 static void fd_array_map_free(struct bpf_map *map) in fd_array_map_free() argument
806 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_free()
810 for (i = 0; i < array->map.max_entries; i++) in fd_array_map_free()
811 BUG_ON(array->ptrs[i] != NULL); in fd_array_map_free()
816 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) in fd_array_map_lookup_elem() argument
818 return ERR_PTR(-EOPNOTSUPP); in fd_array_map_lookup_elem()
822 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_array_map_lookup_elem() argument
827 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_array_map_lookup_elem()
828 return -ENOTSUPP; in bpf_fd_array_map_lookup_elem()
831 elem = array_map_lookup_elem(map, key); in bpf_fd_array_map_lookup_elem()
833 *value = map->ops->map_fd_sys_lookup_elem(ptr); in bpf_fd_array_map_lookup_elem()
835 ret = -ENOENT; in bpf_fd_array_map_lookup_elem()
842 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_array_map_update_elem() argument
845 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_update_elem()
850 return -EINVAL; in bpf_fd_array_map_update_elem()
852 if (index >= array->map.max_entries) in bpf_fd_array_map_update_elem()
853 return -E2BIG; in bpf_fd_array_map_update_elem()
856 new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_array_map_update_elem()
860 if (map->ops->map_poke_run) { in bpf_fd_array_map_update_elem()
861 mutex_lock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
862 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
863 map->ops->map_poke_run(map, index, old_ptr, new_ptr); in bpf_fd_array_map_update_elem()
864 mutex_unlock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
866 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
870 map->ops->map_fd_put_ptr(map, old_ptr, true); in bpf_fd_array_map_update_elem()
874 static long __fd_array_map_delete_elem(struct bpf_map *map, void *key, bool need_defer) in __fd_array_map_delete_elem() argument
876 struct bpf_array *array = container_of(map, struct bpf_array, map); in __fd_array_map_delete_elem()
880 if (index >= array->map.max_entries) in __fd_array_map_delete_elem()
881 return -E2BIG; in __fd_array_map_delete_elem()
883 if (map->ops->map_poke_run) { in __fd_array_map_delete_elem()
884 mutex_lock(&array->aux->poke_mutex); in __fd_array_map_delete_elem()
885 old_ptr = xchg(array->ptrs + index, NULL); in __fd_array_map_delete_elem()
886 map->ops->map_poke_run(map, index, old_ptr, NULL); in __fd_array_map_delete_elem()
887 mutex_unlock(&array->aux->poke_mutex); in __fd_array_map_delete_elem()
889 old_ptr = xchg(array->ptrs + index, NULL); in __fd_array_map_delete_elem()
893 map->ops->map_fd_put_ptr(map, old_ptr, need_defer); in __fd_array_map_delete_elem()
896 return -ENOENT; in __fd_array_map_delete_elem()
900 static long fd_array_map_delete_elem(struct bpf_map *map, void *key) in fd_array_map_delete_elem() argument
902 return __fd_array_map_delete_elem(map, key, true); in fd_array_map_delete_elem()
905 static void *prog_fd_array_get_ptr(struct bpf_map *map, in prog_fd_array_get_ptr() argument
913 if (!bpf_prog_map_compatible(map, prog)) { in prog_fd_array_get_ptr()
915 return ERR_PTR(-EINVAL); in prog_fd_array_get_ptr()
921 static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) in prog_fd_array_put_ptr() argument
929 return ((struct bpf_prog *)ptr)->aux->id; in prog_fd_array_sys_lookup_elem()
932 /* decrement refcnt of all bpf_progs that are stored in this map */
933 static void bpf_fd_array_map_clear(struct bpf_map *map, bool need_defer) in bpf_fd_array_map_clear() argument
935 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_clear()
938 for (i = 0; i < array->map.max_entries; i++) in bpf_fd_array_map_clear()
939 __fd_array_map_delete_elem(map, &i, need_defer); in bpf_fd_array_map_clear()
942 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, in prog_array_map_seq_show_elem() argument
950 elem = array_map_lookup_elem(map, key); in prog_array_map_seq_show_elem()
956 btf_type_seq_show(map->btf, map->btf_value_type_id, in prog_array_map_seq_show_elem()
970 static int prog_array_map_poke_track(struct bpf_map *map, in prog_array_map_poke_track() argument
977 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_track()
978 mutex_lock(&aux->poke_mutex); in prog_array_map_poke_track()
979 list_for_each_entry(elem, &aux->poke_progs, list) { in prog_array_map_poke_track()
980 if (elem->aux == prog_aux) in prog_array_map_poke_track()
986 ret = -ENOMEM; in prog_array_map_poke_track()
990 INIT_LIST_HEAD(&elem->list); in prog_array_map_poke_track()
995 elem->aux = prog_aux; in prog_array_map_poke_track()
997 list_add_tail(&elem->list, &aux->poke_progs); in prog_array_map_poke_track()
999 mutex_unlock(&aux->poke_mutex); in prog_array_map_poke_track()
1003 static void prog_array_map_poke_untrack(struct bpf_map *map, in prog_array_map_poke_untrack() argument
1009 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_untrack()
1010 mutex_lock(&aux->poke_mutex); in prog_array_map_poke_untrack()
1011 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) { in prog_array_map_poke_untrack()
1012 if (elem->aux == prog_aux) { in prog_array_map_poke_untrack()
1013 list_del_init(&elem->list); in prog_array_map_poke_untrack()
1018 mutex_unlock(&aux->poke_mutex); in prog_array_map_poke_untrack()
1027 static void prog_array_map_poke_run(struct bpf_map *map, u32 key, in prog_array_map_poke_run() argument
1034 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_run()
1035 WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex)); in prog_array_map_poke_run()
1037 list_for_each_entry(elem, &aux->poke_progs, list) { in prog_array_map_poke_run()
1041 for (i = 0; i < elem->aux->size_poke_tab; i++) { in prog_array_map_poke_run()
1042 poke = &elem->aux->poke_tab[i]; in prog_array_map_poke_run()
1047 * not aux->prog since it might not be stable yet and in prog_array_map_poke_run()
1051 * entry. We skip these as poke->tailcall_target_stable in prog_array_map_poke_run()
1054 * poke->tailcall_target_stable are successively in prog_array_map_poke_run()
1057 * non-activated poke entries. in prog_array_map_poke_run()
1063 if (!READ_ONCE(poke->tailcall_target_stable)) in prog_array_map_poke_run()
1065 if (poke->reason != BPF_POKE_REASON_TAIL_CALL) in prog_array_map_poke_run()
1067 if (poke->tail_call.map != map || in prog_array_map_poke_run()
1068 poke->tail_call.key != key) in prog_array_map_poke_run()
1078 struct bpf_map *map = container_of(work, struct bpf_array_aux, in prog_array_map_clear_deferred() local
1079 work)->map; in prog_array_map_clear_deferred()
1080 bpf_fd_array_map_clear(map, true); in prog_array_map_clear_deferred()
1081 bpf_map_put(map); in prog_array_map_clear_deferred()
1084 static void prog_array_map_clear(struct bpf_map *map) in prog_array_map_clear() argument
1086 struct bpf_array_aux *aux = container_of(map, struct bpf_array, in prog_array_map_clear()
1087 map)->aux; in prog_array_map_clear()
1088 bpf_map_inc(map); in prog_array_map_clear()
1089 schedule_work(&aux->work); in prog_array_map_clear()
1095 struct bpf_map *map; in prog_array_map_alloc() local
1099 return ERR_PTR(-ENOMEM); in prog_array_map_alloc()
1101 INIT_WORK(&aux->work, prog_array_map_clear_deferred); in prog_array_map_alloc()
1102 INIT_LIST_HEAD(&aux->poke_progs); in prog_array_map_alloc()
1103 mutex_init(&aux->poke_mutex); in prog_array_map_alloc()
1105 map = array_map_alloc(attr); in prog_array_map_alloc()
1106 if (IS_ERR(map)) { in prog_array_map_alloc()
1108 return map; in prog_array_map_alloc()
1111 container_of(map, struct bpf_array, map)->aux = aux; in prog_array_map_alloc()
1112 aux->map = map; in prog_array_map_alloc()
1114 return map; in prog_array_map_alloc()
1117 static void prog_array_map_free(struct bpf_map *map) in prog_array_map_free() argument
1122 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_free()
1123 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) { in prog_array_map_free()
1124 list_del_init(&elem->list); in prog_array_map_free()
1128 fd_array_map_free(map); in prog_array_map_free()
1131 /* prog_array->aux->{type,jited} is a runtime binding.
1162 ee->event = perf_file->private_data; in bpf_event_entry_gen()
1163 ee->perf_file = perf_file; in bpf_event_entry_gen()
1164 ee->map_file = map_file; in bpf_event_entry_gen()
1175 fput(ee->perf_file); in __bpf_event_entry_free()
1181 call_rcu(&ee->rcu, __bpf_event_entry_free); in bpf_event_entry_free_rcu()
1184 static void *perf_event_fd_array_get_ptr(struct bpf_map *map, in perf_event_fd_array_get_ptr() argument
1196 ee = ERR_PTR(-EOPNOTSUPP); in perf_event_fd_array_get_ptr()
1197 event = perf_file->private_data; in perf_event_fd_array_get_ptr()
1198 if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP) in perf_event_fd_array_get_ptr()
1204 ee = ERR_PTR(-ENOMEM); in perf_event_fd_array_get_ptr()
1210 static void perf_event_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) in perf_event_fd_array_put_ptr() argument
1216 static void perf_event_fd_array_release(struct bpf_map *map, in perf_event_fd_array_release() argument
1219 struct bpf_array *array = container_of(map, struct bpf_array, map); in perf_event_fd_array_release()
1223 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_release()
1227 for (i = 0; i < array->map.max_entries; i++) { in perf_event_fd_array_release()
1228 ee = READ_ONCE(array->ptrs[i]); in perf_event_fd_array_release()
1229 if (ee && ee->map_file == map_file) in perf_event_fd_array_release()
1230 __fd_array_map_delete_elem(map, &i, true); in perf_event_fd_array_release()
1235 static void perf_event_fd_array_map_free(struct bpf_map *map) in perf_event_fd_array_map_free() argument
1237 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_map_free()
1238 bpf_fd_array_map_clear(map, false); in perf_event_fd_array_map_free()
1239 fd_array_map_free(map); in perf_event_fd_array_map_free()
1259 static void *cgroup_fd_array_get_ptr(struct bpf_map *map, in cgroup_fd_array_get_ptr() argument
1266 static void cgroup_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) in cgroup_fd_array_put_ptr() argument
1272 static void cgroup_fd_array_free(struct bpf_map *map) in cgroup_fd_array_free() argument
1274 bpf_fd_array_map_clear(map, false); in cgroup_fd_array_free()
1275 fd_array_map_free(map); in cgroup_fd_array_free()
1296 struct bpf_map *map, *inner_map_meta; in array_of_map_alloc() local
1298 inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); in array_of_map_alloc()
1302 map = array_map_alloc(attr); in array_of_map_alloc()
1303 if (IS_ERR(map)) { in array_of_map_alloc()
1305 return map; in array_of_map_alloc()
1308 map->inner_map_meta = inner_map_meta; in array_of_map_alloc()
1310 return map; in array_of_map_alloc()
1313 static void array_of_map_free(struct bpf_map *map) in array_of_map_free() argument
1315 /* map->inner_map_meta is only accessed by syscall which in array_of_map_free()
1318 bpf_map_meta_free(map->inner_map_meta); in array_of_map_free()
1319 bpf_fd_array_map_clear(map, false); in array_of_map_free()
1320 fd_array_map_free(map); in array_of_map_free()
1323 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key) in array_of_map_lookup_elem() argument
1325 struct bpf_map **inner_map = array_map_lookup_elem(map, key); in array_of_map_lookup_elem()
1333 static int array_of_map_gen_lookup(struct bpf_map *map, in array_of_map_gen_lookup() argument
1336 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_of_map_gen_lookup()
1337 u32 elem_size = array->elem_size; in array_of_map_gen_lookup()
1345 if (!map->bypass_spec_v1) { in array_of_map_gen_lookup()
1346 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6); in array_of_map_gen_lookup()
1347 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_of_map_gen_lookup()
1349 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); in array_of_map_gen_lookup()
1361 return insn - insn_buf; in array_of_map_gen_lookup()