Lines Matching full:array
23 static void bpf_array_free_percpu(struct bpf_array *array) in bpf_array_free_percpu() argument
27 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_free_percpu()
28 free_percpu(array->pptrs[i]); in bpf_array_free_percpu()
33 static int bpf_array_alloc_percpu(struct bpf_array *array) in bpf_array_alloc_percpu() argument
38 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_alloc_percpu()
39 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, in bpf_array_alloc_percpu()
42 bpf_array_free_percpu(array); in bpf_array_alloc_percpu()
45 array->pptrs[i] = ptr; in bpf_array_alloc_percpu()
91 struct bpf_array *array; in array_map_alloc() local
107 /* round up array size to nearest power of 2, in array_map_alloc()
116 array_size = sizeof(*array); in array_map_alloc()
121 * ensure array->value is exactly page-aligned in array_map_alloc()
139 array = data + PAGE_ALIGN(sizeof(struct bpf_array)) in array_map_alloc()
142 array = bpf_map_area_alloc(array_size, numa_node); in array_map_alloc()
144 if (!array) in array_map_alloc()
146 array->index_mask = index_mask; in array_map_alloc()
147 array->map.bypass_spec_v1 = bypass_spec_v1; in array_map_alloc()
150 bpf_map_init_from_attr(&array->map, attr); in array_map_alloc()
151 array->elem_size = elem_size; in array_map_alloc()
153 if (percpu && bpf_array_alloc_percpu(array)) { in array_map_alloc()
154 bpf_map_area_free(array); in array_map_alloc()
158 return &array->map; in array_map_alloc()
161 static void *array_map_elem_ptr(struct bpf_array* array, u32 index) in array_map_elem_ptr() argument
163 return array->value + (u64)array->elem_size * index; in array_map_elem_ptr()
169 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_lookup_elem() local
172 if (unlikely(index >= array->map.max_entries)) in array_map_lookup_elem()
175 return array->value + (u64)array->elem_size * (index & array->index_mask); in array_map_lookup_elem()
181 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_get_hash() local
183 sha256(array->value, (u64)array->elem_size * array->map.max_entries, in array_map_get_hash()
185 memcpy(array->map.sha, hash_buf, sizeof(array->map.sha)); in array_map_get_hash()
192 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_addr() local
199 *imm = (unsigned long)array->value; in array_map_direct_value_addr()
206 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_meta() local
207 u64 base = (unsigned long)array->value; in array_map_direct_value_meta()
208 u64 range = array->elem_size; in array_map_direct_value_meta()
222 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_gen_lookup() local
224 u32 elem_size = array->elem_size; in array_map_gen_lookup()
236 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_map_gen_lookup()
255 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_elem() local
258 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_elem()
261 return this_cpu_ptr(array->pptrs[index & array->index_mask]); in percpu_array_map_lookup_elem()
267 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_gen_lookup() local
282 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_0, array->index_mask); in percpu_array_map_gen_lookup()
298 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_percpu_elem() local
304 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_percpu_elem()
307 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu); in percpu_array_map_lookup_percpu_elem()
312 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_copy() local
318 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_copy()
325 size = array->elem_size; in bpf_percpu_array_copy()
327 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_copy()
366 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_update_elem() local
374 if (unlikely(index >= array->map.max_entries)) in array_map_update_elem()
386 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_update_elem()
387 val = this_cpu_ptr(array->pptrs[index & array->index_mask]); in array_map_update_elem()
389 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
391 val = array->value + in array_map_update_elem()
392 (u64)array->elem_size * (index & array->index_mask); in array_map_update_elem()
397 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
405 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_update() local
416 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_update()
430 size = array->elem_size; in bpf_percpu_array_update()
432 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_update()
437 bpf_obj_free_fields(array->map.record, ptr); in bpf_percpu_array_update()
444 bpf_obj_free_fields(array->map.record, ptr); in bpf_percpu_array_update()
457 static void *array_map_vmalloc_addr(struct bpf_array *array) in array_map_vmalloc_addr() argument
459 return (void *)round_down((unsigned long)array, PAGE_SIZE); in array_map_vmalloc_addr()
464 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free_internal_structs() local
471 for (i = 0; i < array->map.max_entries; i++) in array_map_free_internal_structs()
472 bpf_map_free_internal_structs(map, array_map_elem_ptr(array, i)); in array_map_free_internal_structs()
478 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free() local
482 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_free()
483 for (i = 0; i < array->map.max_entries; i++) { in array_map_free()
484 void __percpu *pptr = array->pptrs[i & array->index_mask]; in array_map_free()
493 for (i = 0; i < array->map.max_entries; i++) in array_map_free()
494 bpf_obj_free_fields(map->record, array_map_elem_ptr(array, i)); in array_map_free()
498 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) in array_map_free()
499 bpf_array_free_percpu(array); in array_map_free()
501 if (array->map.map_flags & BPF_F_MMAPABLE) in array_map_free()
502 bpf_map_area_free(array_map_vmalloc_addr(array)); in array_map_free()
504 bpf_map_area_free(array); in array_map_free()
531 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_seq_show_elem() local
539 pptr = array->pptrs[index & array->index_mask]; in percpu_array_map_seq_show_elem()
569 * Bpf array can only take a u32 key. This check makes sure in array_map_check_btf()
580 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mmap() local
581 pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT; in array_map_mmap()
587 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) in array_map_mmap()
590 return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), in array_map_mmap()
613 struct bpf_array *array; in bpf_array_map_seq_start() local
621 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_start()
622 index = info->index & array->index_mask; in bpf_array_map_seq_start()
624 return (void *)(uintptr_t)array->pptrs[index]; in bpf_array_map_seq_start()
625 return array_map_elem_ptr(array, index); in bpf_array_map_seq_start()
632 struct bpf_array *array; in bpf_array_map_seq_next() local
640 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_next()
641 index = info->index & array->index_mask; in bpf_array_map_seq_next()
643 return (void *)(uintptr_t)array->pptrs[index]; in bpf_array_map_seq_next()
644 return array_map_elem_ptr(array, index); in bpf_array_map_seq_next()
652 struct bpf_array *array = container_of(map, struct bpf_array, map); in __bpf_array_map_seq_show() local
673 size = array->elem_size; in __bpf_array_map_seq_show()
703 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_iter_init_array_map() local
708 buf_size = array->elem_size * num_possible_cpus(); in bpf_iter_init_array_map()
751 struct bpf_array *array; in bpf_for_each_array_elem() local
762 array = container_of(map, struct bpf_array, map); in bpf_for_each_array_elem()
765 val = this_cpu_ptr(array->pptrs[i]); in bpf_for_each_array_elem()
767 val = array_map_elem_ptr(array, i); in bpf_for_each_array_elem()
782 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mem_usage() local
784 u32 elem_size = array->elem_size; in array_map_mem_usage()
786 u64 usage = sizeof(*array); in array_map_mem_usage()
864 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_free() local
868 for (i = 0; i < array->map.max_entries; i++) in fd_array_map_free()
869 BUG_ON(array->ptrs[i] != NULL); in fd_array_map_free()
871 bpf_map_area_free(array); in fd_array_map_free()
903 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_update_elem() local
910 if (index >= array->map.max_entries) in bpf_fd_array_map_update_elem()
919 mutex_lock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
920 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
922 mutex_unlock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
924 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
934 struct bpf_array *array = container_of(map, struct bpf_array, map); in __fd_array_map_delete_elem() local
938 if (index >= array->map.max_entries) in __fd_array_map_delete_elem()
942 mutex_lock(&array->aux->poke_mutex); in __fd_array_map_delete_elem()
943 old_ptr = xchg(array->ptrs + index, NULL); in __fd_array_map_delete_elem()
945 mutex_unlock(&array->aux->poke_mutex); in __fd_array_map_delete_elem()
947 old_ptr = xchg(array->ptrs + index, NULL); in __fd_array_map_delete_elem()
1015 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_clear() local
1018 for (i = 0; i < array->map.max_entries; i++) in bpf_fd_array_map_clear()
1299 struct bpf_array *array = container_of(map, struct bpf_array, map); in perf_event_fd_array_release() local
1307 for (i = 0; i < array->map.max_entries; i++) { in perf_event_fd_array_release()
1308 ee = READ_ONCE(array->ptrs[i]); in perf_event_fd_array_release()
1416 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_of_map_gen_lookup() local
1417 u32 elem_size = array->elem_size; in array_of_map_gen_lookup()
1427 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_of_map_gen_lookup()