25 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_free_percpu()
36 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_alloc_percpu()
138 	/* allocate all map elements and zero-initialize them */  in array_map_alloc()
158 	array->map.bypass_spec_v1 = bypass_spec_v1;  in array_map_alloc()
160 	/* copy mandatory map attributes */  in array_map_alloc()
161 	bpf_map_init_from_attr(&array->map, attr);  in array_map_alloc()
162 	bpf_map_charge_move(&array->map.memory, &mem);  in array_map_alloc()
166 		bpf_map_charge_finish(&array->map.memory);  in array_map_alloc()
171 	return &array->map;  in array_map_alloc()
175 static void *array_map_lookup_elem(struct bpf_map *map, void *key)  in array_map_lookup_elem()  argument
177 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_lookup_elem()
180 	if (unlikely(index >= array->map.max_entries))  in array_map_lookup_elem()
186 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,  in array_map_direct_value_addr()  argument
189 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_addr()
191 	if (map->max_entries != 1)  in array_map_direct_value_addr()
193 	if (off >= map->value_size)  in array_map_direct_value_addr()
200 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,  in array_map_direct_value_meta()  argument
203 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_meta()
207 	if (map->max_entries != 1)  in array_map_direct_value_meta()
217 static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)  in array_map_gen_lookup()  argument
219 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_gen_lookup()
221 	u32 elem_size = round_up(map->value_size, 8);  in array_map_gen_lookup()
226 	if (map->map_flags & BPF_F_INNER_MAP)  in array_map_gen_lookup()
231 	if (!map->bypass_spec_v1) {  in array_map_gen_lookup()
232 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);  in array_map_gen_lookup()
235 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);  in array_map_gen_lookup()
250 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)  in percpu_array_map_lookup_elem()  argument
252 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_elem()
255 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_elem()
261 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)  in bpf_percpu_array_copy()  argument
263 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_copy()
269 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_copy()
276 	size = round_up(map->value_size, 8);  in bpf_percpu_array_copy()
288 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)  in array_map_get_next_key()  argument
290 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_get_next_key()
294 	if (index >= array->map.max_entries) {  in array_map_get_next_key()
299 	if (index == array->map.max_entries - 1)  in array_map_get_next_key()
307 static int array_map_update_elem(struct bpf_map *map, void *key, void *value,  in array_map_update_elem()  argument
310 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_update_elem()
318 	if (unlikely(index >= array->map.max_entries))  in array_map_update_elem()
327 		     !map_value_has_spin_lock(map)))  in array_map_update_elem()
330 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in array_map_update_elem()
332 		       value, map->value_size);  in array_map_update_elem()
337 			copy_map_value_locked(map, val, value, false);  in array_map_update_elem()
339 			copy_map_value(map, val, value);  in array_map_update_elem()
344 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,  in bpf_percpu_array_update()  argument
347 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_update()
357 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_update()
371 	size = round_up(map->value_size, 8);  in bpf_percpu_array_update()
383 static int array_map_delete_elem(struct bpf_map *map, void *key)  in array_map_delete_elem()  argument
393 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
394 static void array_map_free(struct bpf_map *map)  in array_map_free()  argument
396 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free()
398 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_free()
401 	if (array->map.map_flags & BPF_F_MMAPABLE)  in array_map_free()
407 static void array_map_seq_show_elem(struct bpf_map *map, void *key,  in array_map_seq_show_elem()  argument
414 	value = array_map_lookup_elem(map, key);  in array_map_seq_show_elem()
420 	if (map->btf_key_type_id)  in array_map_seq_show_elem()
422 	btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);  in array_map_seq_show_elem()
428 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,  in percpu_array_map_seq_show_elem()  argument
431 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_seq_show_elem()
442 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in percpu_array_map_seq_show_elem()
451 static int array_map_check_btf(const struct bpf_map *map,  in array_map_check_btf()  argument
458 	/* One exception for keyless BTF: .bss/.data/.rodata map */  in array_map_check_btf()
460 		if (map->map_type != BPF_MAP_TYPE_ARRAY ||  in array_map_check_btf()
461 		    map->max_entries != 1)  in array_map_check_btf()
483 static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)  in array_map_mmap()  argument
485 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_mmap()
488 	if (!(map->map_flags & BPF_F_MMAPABLE))  in array_map_mmap()
492 	    PAGE_ALIGN((u64)array->map.max_entries * array->elem_size))  in array_map_mmap()
509 	struct bpf_map *map;  member
517 	struct bpf_map *map = info->map;  in bpf_array_map_seq_start()  local
521 	if (info->index >= map->max_entries)  in bpf_array_map_seq_start()
526 	array = container_of(map, struct bpf_array, map);  in bpf_array_map_seq_start()
536 	struct bpf_map *map = info->map;  in bpf_array_map_seq_next()  local
542 	if (info->index >= map->max_entries)  in bpf_array_map_seq_next()
545 	array = container_of(map, struct bpf_array, map);  in bpf_array_map_seq_next()
556 	struct bpf_map *map = info->map;  in __bpf_array_map_seq_show()  local
569 	ctx.map = info->map;  in __bpf_array_map_seq_show()
577 			size = round_up(map->value_size, 8);  in __bpf_array_map_seq_show()
606 	struct bpf_map *map = aux->map;  in bpf_iter_init_array_map()  local
610 	if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in bpf_iter_init_array_map()
611 		buf_size = round_up(map->value_size, 8) * num_possible_cpus();  in bpf_iter_init_array_map()
619 	seq_info->map = map;  in bpf_iter_init_array_map()
686 	/* only file descriptors can be stored in this type of map */  in fd_array_map_alloc_check()
695 static void fd_array_map_free(struct bpf_map *map)  in fd_array_map_free()  argument
697 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_free()
701 	for (i = 0; i < array->map.max_entries; i++)  in fd_array_map_free()
707 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)  in fd_array_map_lookup_elem()  argument
713 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)  in bpf_fd_array_map_lookup_elem()  argument
718 	if (!map->ops->map_fd_sys_lookup_elem)  in bpf_fd_array_map_lookup_elem()
722 	elem = array_map_lookup_elem(map, key);  in bpf_fd_array_map_lookup_elem()
724 		*value = map->ops->map_fd_sys_lookup_elem(ptr);  in bpf_fd_array_map_lookup_elem()
733 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,  in bpf_fd_array_map_update_elem()  argument
736 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_update_elem()
743 	if (index >= array->map.max_entries)  in bpf_fd_array_map_update_elem()
747 	new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);  in bpf_fd_array_map_update_elem()
751 	if (map->ops->map_poke_run) {  in bpf_fd_array_map_update_elem()
754 		map->ops->map_poke_run(map, index, old_ptr, new_ptr);  in bpf_fd_array_map_update_elem()
761 		map->ops->map_fd_put_ptr(old_ptr);  in bpf_fd_array_map_update_elem()
765 static int fd_array_map_delete_elem(struct bpf_map *map, void *key)  in fd_array_map_delete_elem()  argument
767 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_delete_elem()
771 	if (index >= array->map.max_entries)  in fd_array_map_delete_elem()
774 	if (map->ops->map_poke_run) {  in fd_array_map_delete_elem()
777 		map->ops->map_poke_run(map, index, old_ptr, NULL);  in fd_array_map_delete_elem()
784 		map->ops->map_fd_put_ptr(old_ptr);  in fd_array_map_delete_elem()
791 static void *prog_fd_array_get_ptr(struct bpf_map *map,  in prog_fd_array_get_ptr()  argument
794 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in prog_fd_array_get_ptr()
818 /* decrement refcnt of all bpf_progs that are stored in this map */
819 static void bpf_fd_array_map_clear(struct bpf_map *map)  in bpf_fd_array_map_clear()  argument
821 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_clear()
824 	for (i = 0; i < array->map.max_entries; i++)  in bpf_fd_array_map_clear()
825 		fd_array_map_delete_elem(map, &i);  in bpf_fd_array_map_clear()
828 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,  in prog_array_map_seq_show_elem()  argument
836 	elem = array_map_lookup_elem(map, key);  in prog_array_map_seq_show_elem()
842 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in prog_array_map_seq_show_elem()
856 static int prog_array_map_poke_track(struct bpf_map *map,  in prog_array_map_poke_track()  argument
863 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_track()
889 static void prog_array_map_poke_untrack(struct bpf_map *map,  in prog_array_map_poke_untrack()  argument
895 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_untrack()
907 static void prog_array_map_poke_run(struct bpf_map *map, u32 key,  in prog_array_map_poke_run()  argument
915 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_run()
959 			if (poke->tail_call.map != map ||  in prog_array_map_poke_run()
1002 	struct bpf_map *map = container_of(work, struct bpf_array_aux,  in prog_array_map_clear_deferred()  local
1003 					   work)->map;  in prog_array_map_clear_deferred()
1004 	bpf_fd_array_map_clear(map);  in prog_array_map_clear_deferred()
1005 	bpf_map_put(map);  in prog_array_map_clear_deferred()
1008 static void prog_array_map_clear(struct bpf_map *map)  in prog_array_map_clear()  argument
1010 	struct bpf_array_aux *aux = container_of(map, struct bpf_array,  in prog_array_map_clear()
1011 						 map)->aux;  in prog_array_map_clear()
1012 	bpf_map_inc(map);  in prog_array_map_clear()
1019 	struct bpf_map *map;  in prog_array_map_alloc()  local
1029 	map = array_map_alloc(attr);  in prog_array_map_alloc()
1030 	if (IS_ERR(map)) {  in prog_array_map_alloc()
1032 		return map;  in prog_array_map_alloc()
1035 	container_of(map, struct bpf_array, map)->aux = aux;  in prog_array_map_alloc()
1036 	aux->map = map;  in prog_array_map_alloc()
1038 	return map;  in prog_array_map_alloc()
1041 static void prog_array_map_free(struct bpf_map *map)  in prog_array_map_free()  argument
1046 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_free()
1052 	fd_array_map_free(map);  in prog_array_map_free()
1109 static void *perf_event_fd_array_get_ptr(struct bpf_map *map,  in perf_event_fd_array_get_ptr()  argument
1140 static void perf_event_fd_array_release(struct bpf_map *map,  in perf_event_fd_array_release()  argument
1143 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in perf_event_fd_array_release()
1147 	if (map->map_flags & BPF_F_PRESERVE_ELEMS)  in perf_event_fd_array_release()
1151 	for (i = 0; i < array->map.max_entries; i++) {  in perf_event_fd_array_release()
1154 			fd_array_map_delete_elem(map, &i);  in perf_event_fd_array_release()
1159 static void perf_event_fd_array_map_free(struct bpf_map *map)  in perf_event_fd_array_map_free()  argument
1161 	if (map->map_flags & BPF_F_PRESERVE_ELEMS)  in perf_event_fd_array_map_free()
1162 		bpf_fd_array_map_clear(map);  in perf_event_fd_array_map_free()
1163 	fd_array_map_free(map);  in perf_event_fd_array_map_free()
1184 static void *cgroup_fd_array_get_ptr(struct bpf_map *map,  in cgroup_fd_array_get_ptr()  argument
1197 static void cgroup_fd_array_free(struct bpf_map *map)  in cgroup_fd_array_free()  argument
1199 	bpf_fd_array_map_clear(map);  in cgroup_fd_array_free()
1200 	fd_array_map_free(map);  in cgroup_fd_array_free()
1222 	struct bpf_map *map, *inner_map_meta;  in array_of_map_alloc()  local
1228 	map = array_map_alloc(attr);  in array_of_map_alloc()
1229 	if (IS_ERR(map)) {  in array_of_map_alloc()
1231 		return map;  in array_of_map_alloc()
1234 	map->inner_map_meta = inner_map_meta;  in array_of_map_alloc()
1236 	return map;  in array_of_map_alloc()
1239 static void array_of_map_free(struct bpf_map *map)  in array_of_map_free()  argument
1241 	/* map->inner_map_meta is only accessed by syscall which  in array_of_map_free()
1244 	bpf_map_meta_free(map->inner_map_meta);  in array_of_map_free()
1245 	bpf_fd_array_map_clear(map);  in array_of_map_free()
1246 	fd_array_map_free(map);  in array_of_map_free()
1249 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)  in array_of_map_lookup_elem()  argument
1251 	struct bpf_map **inner_map = array_map_lookup_elem(map, key);  in array_of_map_lookup_elem()
1259 static int array_of_map_gen_lookup(struct bpf_map *map,  in array_of_map_gen_lookup()  argument
1262 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_of_map_gen_lookup()
1263 	u32 elem_size = round_up(map->value_size, 8);  in array_of_map_gen_lookup()
1271 	if (!map->bypass_spec_v1) {  in array_of_map_gen_lookup()
1272 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);  in array_of_map_gen_lookup()
1275 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);  in array_of_map_gen_lookup()