54 static bool selem_linked_to_storage_lockless(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage_lockless()  argument
56 	return !hlist_unhashed_lockless(&selem->snode);  in selem_linked_to_storage_lockless()
59 static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage()  argument
61 	return !hlist_unhashed(&selem->snode);  in selem_linked_to_storage()
64 static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)  in selem_linked_to_map()  argument
66 	return !hlist_unhashed(&selem->map_node);  in selem_linked_to_map()
73 	struct bpf_local_storage_elem *selem;  in bpf_selem_alloc()  local
79 		selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size,  in bpf_selem_alloc()
82 		selem = bpf_map_kzalloc(&smap->map, smap->elem_size,  in bpf_selem_alloc()
86 	if (selem) {  in bpf_selem_alloc()
87 		RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_alloc()
88 		atomic_set(&selem->state, 0);  in bpf_selem_alloc()
89 		selem->use_kmalloc_nolock = smap->use_kmalloc_nolock;  in bpf_selem_alloc()
93 			copy_map_value(&smap->map, SDATA(selem)->data, value);  in bpf_selem_alloc()
95 				bpf_obj_swap_uptrs(smap->map.record, SDATA(selem)->data, value);  in bpf_selem_alloc()
97 		return selem;  in bpf_selem_alloc()
169 	struct bpf_local_storage_elem *selem;  in __bpf_selem_free_rcu()  local
172 	selem = container_of(rcu, struct bpf_local_storage_elem, rcu);  in __bpf_selem_free_rcu()
174 	smap = rcu_dereference_check(SDATA(selem)->smap, 1);  in __bpf_selem_free_rcu()
177 		bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in __bpf_selem_free_rcu()
178 	kfree(selem);  in __bpf_selem_free_rcu()
192 static void __bpf_selem_free(struct bpf_local_storage_elem *selem,  in __bpf_selem_free()  argument
196 		call_rcu(&selem->rcu, __bpf_selem_free_rcu);  in __bpf_selem_free()
198 		call_rcu_tasks_trace(&selem->rcu, __bpf_selem_free_trace_rcu);  in __bpf_selem_free()
203 	struct bpf_local_storage_elem *selem;  in bpf_selem_free_rcu()  local
206 	selem = container_of(rcu, struct bpf_local_storage_elem, rcu);  in bpf_selem_free_rcu()
208 	smap = rcu_dereference_check(SDATA(selem)->smap, 1);  in bpf_selem_free_rcu()
211 		bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_free_rcu()
212 	kfree_nolock(selem);  in bpf_selem_free_rcu()
224 void bpf_selem_free(struct bpf_local_storage_elem *selem,  in bpf_selem_free()  argument
227 	if (!selem->use_kmalloc_nolock) {  in bpf_selem_free()
233 		__bpf_selem_free(selem, reuse_now);  in bpf_selem_free()
242 		call_rcu(&selem->rcu, bpf_selem_free_rcu);  in bpf_selem_free()
246 	call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_trace_rcu);  in bpf_selem_free()
251 	struct bpf_local_storage_elem *selem;  in bpf_selem_free_list()  local
259 	hlist_for_each_entry_safe(selem, n, list, free_node)  in bpf_selem_free_list()
260 		bpf_selem_free(selem, reuse_now);  in bpf_selem_free_list()
263 static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage_nolock_misc()  argument
272 	    SDATA(selem))  in bpf_selem_unlink_storage_nolock_misc()
298 					    struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage_nolock()  argument
304 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_storage_nolock()
306 	free_local_storage = hlist_is_singular_node(&selem->snode,  in bpf_selem_unlink_storage_nolock()
309 	bpf_selem_unlink_storage_nolock_misc(selem, smap, local_storage,  in bpf_selem_unlink_storage_nolock()
312 	hlist_del_init_rcu(&selem->snode);  in bpf_selem_unlink_storage_nolock()
314 	hlist_add_head(&selem->free_node, free_selem_list);  in bpf_selem_unlink_storage_nolock()
320 				   struct bpf_local_storage_elem *selem)  in bpf_selem_link_storage_nolock()  argument
324 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_link_storage_nolock()
327 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
328 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
331 static int bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink_map()  argument
339 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink_map()
341 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_map()
347 	hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_map()
353 static void bpf_selem_unlink_map_nolock(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink_map_nolock()  argument
355 	hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_map_nolock()
360 		       struct bpf_local_storage_elem *selem)  in bpf_selem_link_map()  argument
372 	hlist_add_head_rcu(&selem->map_node, &b->list);  in bpf_selem_link_map()
379 				      struct bpf_local_storage_elem *selem)  in bpf_selem_link_map_nolock()  argument
381 	hlist_add_head_rcu(&selem->map_node, &b->list);  in bpf_selem_link_map_nolock()
388 int bpf_selem_unlink(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink()  argument
396 	if (unlikely(!selem_linked_to_storage_lockless(selem)))  in bpf_selem_unlink()
400 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink()
407 	if (likely(selem_linked_to_storage(selem))) {  in bpf_selem_unlink()
412 		err = bpf_selem_unlink_map(selem);  in bpf_selem_unlink()
417 			local_storage, selem, &selem_free_list);  in bpf_selem_unlink()
435 static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_nofail()  argument
444 	local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held());  in bpf_selem_unlink_nofail()
445 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_nofail()
456 			if (likely(selem_linked_to_map(selem))) {  in bpf_selem_unlink_nofail()
457 				hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_nofail()
458 				bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_unlink_nofail()
474 			RCU_INIT_POINTER(SDATA(selem)->smap, NULL);  in bpf_selem_unlink_nofail()
480 			if (likely(selem_linked_to_storage(selem))) {  in bpf_selem_unlink_nofail()
481 				free_storage = hlist_is_singular_node(&selem->snode,  in bpf_selem_unlink_nofail()
490 						selem, smap, local_storage,  in bpf_selem_unlink_nofail()
492 				hlist_del_init_rcu(&selem->snode);  in bpf_selem_unlink_nofail()
498 			RCU_INIT_POINTER(selem->local_storage, NULL);  in bpf_selem_unlink_nofail()
502 		atomic_or(in_map_free ? SELEM_MAP_UNLINKED : SELEM_STORAGE_UNLINKED, &selem->state);  in bpf_selem_unlink_nofail()
511 	    atomic_cmpxchg(&selem->state, SELEM_UNLINKED, SELEM_TOFREE) == SELEM_UNLINKED)  in bpf_selem_unlink_nofail()
512 		bpf_selem_free(selem, true);  in bpf_selem_unlink_nofail()
520 				      struct bpf_local_storage_elem *selem)  in __bpf_local_storage_insert_cache()  argument
534 	if (selem_linked_to_storage(selem))  in __bpf_local_storage_insert_cache()
535 		rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));  in __bpf_local_storage_insert_cache()
634 	struct bpf_local_storage_elem *alloc_selem, *selem = NULL;  in bpf_local_storage_update()  local
659 		selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);  in bpf_local_storage_update()
660 		if (!selem)  in bpf_local_storage_update()
663 		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);  in bpf_local_storage_update()
665 			bpf_selem_free(selem, true);  in bpf_local_storage_update()
670 		return SDATA(selem);  in bpf_local_storage_update()
693 	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);  in bpf_local_storage_update()
720 		selem = SELEM(old_sdata);  in bpf_local_storage_update()
732 	bpf_selem_link_map_nolock(b, selem);  in bpf_local_storage_update()
735 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
753 	return err ? ERR_PTR(err) : SDATA(selem);  in bpf_local_storage_update()
821 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_destroy()  local
832 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)  in bpf_local_storage_destroy()
833 		bpf_selem_unlink_nofail(selem, NULL);  in bpf_local_storage_destroy()
912 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_map_free()  local
940 		hlist_for_each_entry_rcu(selem, &b->list, map_node) {  in bpf_local_storage_map_free()
941 			bpf_selem_unlink_nofail(selem, b);  in bpf_local_storage_map_free()