Lines Matching refs:htab
133 static inline bool htab_is_prealloc(const struct bpf_htab *htab) in htab_is_prealloc() argument
135 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
138 static void htab_init_buckets(struct bpf_htab *htab) in htab_init_buckets() argument
142 for (i = 0; i < htab->n_buckets; i++) { in htab_init_buckets()
143 INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); in htab_init_buckets()
144 raw_res_spin_lock_init(&htab->buckets[i].raw_lock); in htab_init_buckets()
168 static bool htab_is_lru(const struct bpf_htab *htab) in htab_is_lru() argument
170 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
171 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
174 static bool htab_is_percpu(const struct bpf_htab *htab) in htab_is_percpu() argument
176 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
177 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
180 static inline bool is_fd_htab(const struct bpf_htab *htab) in is_fd_htab() argument
182 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; in is_fd_htab()
206 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) in get_htab_elem() argument
208 return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); in get_htab_elem()
215 static bool htab_has_extra_elems(struct bpf_htab *htab) in htab_has_extra_elems() argument
217 return !htab_is_percpu(htab) && !htab_is_lru(htab) && !is_fd_htab(htab); in htab_has_extra_elems()
220 static void htab_free_prealloced_internal_structs(struct bpf_htab *htab) in htab_free_prealloced_internal_structs() argument
222 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_internal_structs()
225 if (htab_has_extra_elems(htab)) in htab_free_prealloced_internal_structs()
231 elem = get_htab_elem(htab, i); in htab_free_prealloced_internal_structs()
232 bpf_map_free_internal_structs(&htab->map, in htab_free_prealloced_internal_structs()
233 htab_elem_value(elem, htab->map.key_size)); in htab_free_prealloced_internal_structs()
238 static void htab_free_prealloced_fields(struct bpf_htab *htab) in htab_free_prealloced_fields() argument
240 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_fields()
243 if (IS_ERR_OR_NULL(htab->map.record)) in htab_free_prealloced_fields()
245 if (htab_has_extra_elems(htab)) in htab_free_prealloced_fields()
250 elem = get_htab_elem(htab, i); in htab_free_prealloced_fields()
251 if (htab_is_percpu(htab)) { in htab_free_prealloced_fields()
252 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in htab_free_prealloced_fields()
256 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in htab_free_prealloced_fields()
260 bpf_obj_free_fields(htab->map.record, in htab_free_prealloced_fields()
261 htab_elem_value(elem, htab->map.key_size)); in htab_free_prealloced_fields()
268 static void htab_free_elems(struct bpf_htab *htab) in htab_free_elems() argument
272 if (!htab_is_percpu(htab)) in htab_free_elems()
275 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
278 pptr = htab_elem_get_ptr(get_htab_elem(htab, i), in htab_free_elems()
279 htab->map.key_size); in htab_free_elems()
284 bpf_map_area_free(htab->elems); in htab_free_elems()
298 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, in prealloc_lru_pop() argument
301 struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); in prealloc_lru_pop()
305 bpf_map_inc_elem_count(&htab->map); in prealloc_lru_pop()
307 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
314 static int prealloc_init(struct bpf_htab *htab) in prealloc_init() argument
316 u32 num_entries = htab->map.max_entries; in prealloc_init()
319 if (htab_has_extra_elems(htab)) in prealloc_init()
322 htab->elems = bpf_map_area_alloc((u64)htab->elem_size * num_entries, in prealloc_init()
323 htab->map.numa_node); in prealloc_init()
324 if (!htab->elems) in prealloc_init()
327 if (!htab_is_percpu(htab)) in prealloc_init()
331 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
334 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in prealloc_init()
338 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
344 if (htab_is_lru(htab)) in prealloc_init()
345 err = bpf_lru_init(&htab->lru, in prealloc_init()
346 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
350 htab); in prealloc_init()
352 err = pcpu_freelist_init(&htab->freelist); in prealloc_init()
357 if (htab_is_lru(htab)) in prealloc_init()
358 bpf_lru_populate(&htab->lru, htab->elems, in prealloc_init()
360 htab->elem_size, num_entries); in prealloc_init()
362 pcpu_freelist_populate(&htab->freelist, in prealloc_init()
363 htab->elems + offsetof(struct htab_elem, fnode), in prealloc_init()
364 htab->elem_size, num_entries); in prealloc_init()
369 htab_free_elems(htab); in prealloc_init()
373 static void prealloc_destroy(struct bpf_htab *htab) in prealloc_destroy() argument
375 htab_free_elems(htab); in prealloc_destroy()
377 if (htab_is_lru(htab)) in prealloc_destroy()
378 bpf_lru_destroy(&htab->lru); in prealloc_destroy()
380 pcpu_freelist_destroy(&htab->freelist); in prealloc_destroy()
383 static int alloc_extra_elems(struct bpf_htab *htab) in alloc_extra_elems() argument
389 pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8, in alloc_extra_elems()
395 l = pcpu_freelist_pop(&htab->freelist); in alloc_extra_elems()
402 htab->extra_elems = pptr; in alloc_extra_elems()
499 static int htab_set_dtor(struct bpf_htab *htab, void (*dtor)(void *, void *)) in htab_set_dtor() argument
501 u32 key_size = htab->map.key_size; in htab_set_dtor()
507 if (IS_ERR_OR_NULL(htab->map.record)) in htab_set_dtor()
514 hrec->record = btf_record_dup(htab->map.record); in htab_set_dtor()
520 ma = htab_is_percpu(htab) ? &htab->pcpu_ma : &htab->ma; in htab_set_dtor()
528 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_check_btf() local
530 if (htab_is_prealloc(htab)) in htab_map_check_btf()
536 if (htab_is_percpu(htab)) in htab_map_check_btf()
537 return htab_set_dtor(htab, htab_pcpu_mem_dtor); in htab_map_check_btf()
539 return htab_set_dtor(htab, htab_mem_dtor); in htab_map_check_btf()
553 struct bpf_htab *htab; in htab_map_alloc() local
556 htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE); in htab_map_alloc()
557 if (!htab) in htab_map_alloc()
560 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
567 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
569 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
570 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
578 if (htab->map.max_entries > 1UL << 31) in htab_map_alloc()
581 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
583 htab->elem_size = sizeof(struct htab_elem) + in htab_map_alloc()
584 round_up(htab->map.key_size, 8); in htab_map_alloc()
586 htab->elem_size += sizeof(void *); in htab_map_alloc()
588 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
591 if (htab->n_buckets > U32_MAX / sizeof(struct bucket)) in htab_map_alloc()
594 err = bpf_map_init_elem_count(&htab->map); in htab_map_alloc()
599 htab->buckets = bpf_map_area_alloc(htab->n_buckets * in htab_map_alloc()
601 htab->map.numa_node); in htab_map_alloc()
602 if (!htab->buckets) in htab_map_alloc()
605 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
606 htab->hashrnd = 0; in htab_map_alloc()
608 htab->hashrnd = get_random_u32(); in htab_map_alloc()
610 htab_init_buckets(htab); in htab_map_alloc()
627 htab->use_percpu_counter = true; in htab_map_alloc()
629 if (htab->use_percpu_counter) { in htab_map_alloc()
630 err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL); in htab_map_alloc()
636 err = prealloc_init(htab); in htab_map_alloc()
640 if (htab_has_extra_elems(htab)) { in htab_map_alloc()
641 err = alloc_extra_elems(htab); in htab_map_alloc()
646 err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false); in htab_map_alloc()
650 err = bpf_mem_alloc_init(&htab->pcpu_ma, in htab_map_alloc()
651 round_up(htab->map.value_size, 8), true); in htab_map_alloc()
657 return &htab->map; in htab_map_alloc()
660 prealloc_destroy(htab); in htab_map_alloc()
662 if (htab->use_percpu_counter) in htab_map_alloc()
663 percpu_counter_destroy(&htab->pcount); in htab_map_alloc()
664 bpf_map_area_free(htab->buckets); in htab_map_alloc()
665 bpf_mem_alloc_destroy(&htab->pcpu_ma); in htab_map_alloc()
666 bpf_mem_alloc_destroy(&htab->ma); in htab_map_alloc()
668 bpf_map_free_elem_count(&htab->map); in htab_map_alloc()
670 bpf_map_area_free(htab); in htab_map_alloc()
681 static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) in __select_bucket() argument
683 return &htab->buckets[hash & (htab->n_buckets - 1)]; in __select_bucket()
686 static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash) in select_bucket() argument
688 return &__select_bucket(htab, hash)->head; in select_bucket()
734 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem() local
743 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_elem()
745 head = select_bucket(htab, hash); in __htab_map_lookup_elem()
747 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in __htab_map_lookup_elem()
837 static void check_and_free_fields(struct bpf_htab *htab, in check_and_free_fields() argument
840 if (IS_ERR_OR_NULL(htab->map.record)) in check_and_free_fields()
843 if (htab_is_percpu(htab)) { in check_and_free_fields()
844 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in check_and_free_fields()
848 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in check_and_free_fields()
850 void *map_value = htab_elem_value(elem, htab->map.key_size); in check_and_free_fields()
852 bpf_obj_free_fields(htab->map.record, map_value); in check_and_free_fields()
861 struct bpf_htab *htab = arg; in htab_lru_map_delete_node() local
870 b = __select_bucket(htab, tgt_l->hash); in htab_lru_map_delete_node()
880 bpf_map_dec_elem_count(&htab->map); in htab_lru_map_delete_node()
887 check_and_free_fields(htab, l); in htab_lru_map_delete_node()
894 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key() local
907 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_get_next_key()
909 head = select_bucket(htab, hash); in htab_map_get_next_key()
912 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in htab_map_get_next_key()
928 i = hash & (htab->n_buckets - 1); in htab_map_get_next_key()
933 for (; i < htab->n_buckets; i++) { in htab_map_get_next_key()
934 head = select_bucket(htab, i); in htab_map_get_next_key()
950 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) in htab_elem_free() argument
952 check_and_free_fields(htab, l); in htab_elem_free()
954 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
955 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); in htab_elem_free()
956 bpf_mem_cache_free(&htab->ma, l); in htab_elem_free()
959 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) in htab_put_fd_value() argument
961 struct bpf_map *map = &htab->map; in htab_put_fd_value()
970 static bool is_map_full(struct bpf_htab *htab) in is_map_full() argument
972 if (htab->use_percpu_counter) in is_map_full()
973 return __percpu_counter_compare(&htab->pcount, htab->map.max_entries, in is_map_full()
975 return atomic_read(&htab->count) >= htab->map.max_entries; in is_map_full()
978 static void inc_elem_count(struct bpf_htab *htab) in inc_elem_count() argument
980 bpf_map_inc_elem_count(&htab->map); in inc_elem_count()
982 if (htab->use_percpu_counter) in inc_elem_count()
983 percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH); in inc_elem_count()
985 atomic_inc(&htab->count); in inc_elem_count()
988 static void dec_elem_count(struct bpf_htab *htab) in dec_elem_count() argument
990 bpf_map_dec_elem_count(&htab->map); in dec_elem_count()
992 if (htab->use_percpu_counter) in dec_elem_count()
993 percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH); in dec_elem_count()
995 atomic_dec(&htab->count); in dec_elem_count()
999 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) in free_htab_elem() argument
1001 htab_put_fd_value(htab, l); in free_htab_elem()
1003 if (htab_is_prealloc(htab)) { in free_htab_elem()
1004 bpf_map_dec_elem_count(&htab->map); in free_htab_elem()
1005 check_and_free_fields(htab, l); in free_htab_elem()
1006 pcpu_freelist_push(&htab->freelist, &l->fnode); in free_htab_elem()
1008 dec_elem_count(htab); in free_htab_elem()
1009 htab_elem_free(htab, l); in free_htab_elem()
1013 static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, in pcpu_copy_value() argument
1021 copy_map_value(&htab->map, ptr, value); in pcpu_copy_value()
1022 bpf_obj_free_fields(htab->map.record, ptr); in pcpu_copy_value()
1024 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
1031 copy_map_value(&htab->map, ptr, value); in pcpu_copy_value()
1032 bpf_obj_free_fields(htab->map.record, ptr); in pcpu_copy_value()
1039 copy_map_value(&htab->map, ptr, val); in pcpu_copy_value()
1040 bpf_obj_free_fields(htab->map.record, ptr); in pcpu_copy_value()
1045 static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, in pcpu_init_value() argument
1059 copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value); in pcpu_init_value()
1061 zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); in pcpu_init_value()
1064 pcpu_copy_value(htab, pptr, value, onallcpus, map_flags); in pcpu_init_value()
1068 static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab) in fd_htab_map_needs_adjust() argument
1070 return is_fd_htab(htab) && BITS_PER_LONG == 64; in fd_htab_map_needs_adjust()
1073 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, in alloc_htab_elem() argument
1078 u32 size = htab->map.value_size; in alloc_htab_elem()
1079 bool prealloc = htab_is_prealloc(htab); in alloc_htab_elem()
1088 pl_new = this_cpu_ptr(htab->extra_elems); in alloc_htab_elem()
1094 l = __pcpu_freelist_pop(&htab->freelist); in alloc_htab_elem()
1098 bpf_map_inc_elem_count(&htab->map); in alloc_htab_elem()
1101 if (is_map_full(htab)) in alloc_htab_elem()
1109 inc_elem_count(htab); in alloc_htab_elem()
1110 l_new = bpf_mem_cache_alloc(&htab->ma); in alloc_htab_elem()
1123 void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); in alloc_htab_elem()
1126 bpf_mem_cache_free(&htab->ma, l_new); in alloc_htab_elem()
1134 pcpu_init_value(htab, pptr, value, onallcpus, map_flags); in alloc_htab_elem()
1138 } else if (fd_htab_map_needs_adjust(htab)) { in alloc_htab_elem()
1142 copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); in alloc_htab_elem()
1148 dec_elem_count(htab); in alloc_htab_elem()
1152 static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, in check_flags() argument
1170 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem() local
1186 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_update_elem()
1188 b = __select_bucket(htab, hash); in htab_map_update_elem()
1196 htab->n_buckets); in htab_map_update_elem()
1197 ret = check_flags(htab, l_old, map_flags); in htab_map_update_elem()
1219 ret = check_flags(htab, l_old, map_flags); in htab_map_update_elem()
1237 l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false, in htab_map_update_elem()
1255 if (htab_is_prealloc(htab)) in htab_map_update_elem()
1256 check_and_free_fields(htab, l_old); in htab_map_update_elem()
1259 if (l_old && !htab_is_prealloc(htab)) in htab_map_update_elem()
1260 free_htab_elem(htab, l_old); in htab_map_update_elem()
1267 static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) in htab_lru_push_free() argument
1269 check_and_free_fields(htab, elem); in htab_lru_push_free()
1270 bpf_map_dec_elem_count(&htab->map); in htab_lru_push_free()
1271 bpf_lru_push_free(&htab->lru, &elem->lru_node); in htab_lru_push_free()
1277 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem() local
1293 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_update_elem()
1295 b = __select_bucket(htab, hash); in htab_lru_map_update_elem()
1303 l_new = prealloc_lru_pop(htab, key, hash); in htab_lru_map_update_elem()
1306 copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); in htab_lru_map_update_elem()
1314 ret = check_flags(htab, l_old, map_flags); in htab_lru_map_update_elem()
1333 htab_lru_push_free(htab, l_new); in htab_lru_map_update_elem()
1335 htab_lru_push_free(htab, l_old); in htab_lru_map_update_elem()
1353 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem_in_place() local
1370 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_update_elem_in_place()
1372 b = __select_bucket(htab, hash); in htab_map_update_elem_in_place()
1381 ret = check_flags(htab, l_old, map_flags); in htab_map_update_elem_in_place()
1388 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), in htab_map_update_elem_in_place()
1397 l_new = alloc_htab_elem(htab, key, value, key_size, in htab_map_update_elem_in_place()
1416 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem() local
1432 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_lru_percpu_map_update_elem()
1434 b = __select_bucket(htab, hash); in __htab_lru_percpu_map_update_elem()
1443 l_new = prealloc_lru_pop(htab, key, hash); in __htab_lru_percpu_map_update_elem()
1454 ret = check_flags(htab, l_old, map_flags); in __htab_lru_percpu_map_update_elem()
1462 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), in __htab_lru_percpu_map_update_elem()
1465 pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), in __htab_lru_percpu_map_update_elem()
1475 bpf_map_dec_elem_count(&htab->map); in __htab_lru_percpu_map_update_elem()
1476 bpf_lru_push_free(&htab->lru, &l_new->lru_node); in __htab_lru_percpu_map_update_elem()
1497 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem() local
1509 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_delete_elem()
1510 b = __select_bucket(htab, hash); in htab_map_delete_elem()
1526 free_htab_elem(htab, l); in htab_map_delete_elem()
1532 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem() local
1544 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_delete_elem()
1545 b = __select_bucket(htab, hash); in htab_lru_map_delete_elem()
1561 htab_lru_push_free(htab, l); in htab_lru_map_delete_elem()
1565 static void delete_all_elements(struct bpf_htab *htab) in delete_all_elements() argument
1572 for (i = 0; i < htab->n_buckets; i++) { in delete_all_elements()
1573 struct hlist_nulls_head *head = select_bucket(htab, i); in delete_all_elements()
1579 htab_elem_free(htab, l); in delete_all_elements()
1585 static void htab_free_malloced_internal_structs(struct bpf_htab *htab) in htab_free_malloced_internal_structs() argument
1590 for (i = 0; i < htab->n_buckets; i++) { in htab_free_malloced_internal_structs()
1591 struct hlist_nulls_head *head = select_bucket(htab, i); in htab_free_malloced_internal_structs()
1597 bpf_map_free_internal_structs(&htab->map, in htab_free_malloced_internal_structs()
1598 htab_elem_value(l, htab->map.key_size)); in htab_free_malloced_internal_structs()
1607 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free_internal_structs() local
1613 if (htab_is_prealloc(htab)) in htab_map_free_internal_structs()
1614 htab_free_prealloced_internal_structs(htab); in htab_map_free_internal_structs()
1616 htab_free_malloced_internal_structs(htab); in htab_map_free_internal_structs()
1622 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free() local
1633 if (!htab_is_prealloc(htab)) { in htab_map_free()
1634 delete_all_elements(htab); in htab_map_free()
1636 htab_free_prealloced_fields(htab); in htab_map_free()
1637 prealloc_destroy(htab); in htab_map_free()
1641 free_percpu(htab->extra_elems); in htab_map_free()
1642 bpf_map_area_free(htab->buckets); in htab_map_free()
1643 bpf_mem_alloc_destroy(&htab->pcpu_ma); in htab_map_free()
1644 bpf_mem_alloc_destroy(&htab->ma); in htab_map_free()
1645 if (htab->use_percpu_counter) in htab_map_free()
1646 percpu_counter_destroy(&htab->pcount); in htab_map_free()
1647 bpf_map_area_free(htab); in htab_map_free()
1675 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_elem() local
1685 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_and_delete_elem()
1686 b = __select_bucket(htab, hash); in __htab_map_lookup_and_delete_elem()
1706 copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_elem()
1707 check_and_init_map_value(&htab->map, value + off); in __htab_map_lookup_and_delete_elem()
1727 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_elem()
1729 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_elem()
1772 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_batch() local
1812 if (batch >= htab->n_buckets) in __htab_map_lookup_and_delete_batch()
1815 key_size = htab->map.key_size; in __htab_map_lookup_and_delete_batch()
1816 value_size = htab->map.value_size; in __htab_map_lookup_and_delete_batch()
1843 b = &htab->buckets[batch]; in __htab_map_lookup_and_delete_batch()
1903 copy_map_value(&htab->map, dst_val, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_batch()
1904 check_and_init_map_value(&htab->map, dst_val); in __htab_map_lookup_and_delete_batch()
1907 copy_map_value_long(&htab->map, dst_val + off, in __htab_map_lookup_and_delete_batch()
1909 check_and_init_map_value(&htab->map, dst_val + off); in __htab_map_lookup_and_delete_batch()
1915 if (is_fd_htab(htab)) { in __htab_map_lookup_and_delete_batch()
1958 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_batch()
1960 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_batch()
1967 if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { in __htab_map_lookup_and_delete_batch()
1984 if (batch >= htab->n_buckets) { in __htab_map_lookup_and_delete_batch()
2077 struct bpf_htab *htab; member
2087 const struct bpf_htab *htab = info->htab; in bpf_hash_map_seq_find_next() local
2096 if (bucket_id >= htab->n_buckets) in bpf_hash_map_seq_find_next()
2110 b = &htab->buckets[bucket_id++]; in bpf_hash_map_seq_find_next()
2115 for (i = bucket_id; i < htab->n_buckets; i++) { in bpf_hash_map_seq_find_next()
2116 b = &htab->buckets[i]; in bpf_hash_map_seq_find_next()
2233 seq_info->htab = container_of(map, struct bpf_htab, map); in bpf_iter_init_hash_map()
2262 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_for_each_hash_elem() local
2278 is_percpu = htab_is_percpu(htab); in bpf_for_each_hash_elem()
2284 for (i = 0; i < htab->n_buckets; i++) { in bpf_for_each_hash_elem()
2285 b = &htab->buckets[i]; in bpf_for_each_hash_elem()
2314 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_mem_usage() local
2315 u32 value_size = round_up(htab->map.value_size, 8); in htab_map_mem_usage()
2316 bool prealloc = htab_is_prealloc(htab); in htab_map_mem_usage()
2317 bool percpu = htab_is_percpu(htab); in htab_map_mem_usage()
2318 bool lru = htab_is_lru(htab); in htab_map_mem_usage()
2322 sizeof(struct bucket) * htab->n_buckets; in htab_map_mem_usage()
2326 if (htab_has_extra_elems(htab)) in htab_map_mem_usage()
2329 usage += htab->elem_size * num_entries; in htab_map_mem_usage()
2338 num_entries = htab->use_percpu_counter ? in htab_map_mem_usage()
2339 percpu_counter_sum(&htab->pcount) : in htab_map_mem_usage()
2340 atomic_read(&htab->count); in htab_map_mem_usage()
2341 usage += (htab->elem_size + LLIST_NODE_SZ) * num_entries; in htab_map_mem_usage()
2368 BATCH_OPS(htab),
2510 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update() local
2514 if (htab_is_lru(htab)) in bpf_percpu_hash_update()
2606 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free() local
2612 for (i = 0; i < htab->n_buckets; i++) { in fd_htab_map_free()
2613 head = select_bucket(htab, i); in fd_htab_map_free()
2735 BATCH_OPS(htab),