22 	      struct bpf_local_storage *local_storage)  in select_bucket()  argument
24 	return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)];  in select_bucket()
108 	struct bpf_local_storage *local_storage;  in __bpf_local_storage_free_trace_rcu()  local
114 	local_storage = container_of(rcu, struct bpf_local_storage, rcu);  in __bpf_local_storage_free_trace_rcu()
115 	kfree(local_storage);  in __bpf_local_storage_free_trace_rcu()
119 static void __bpf_local_storage_free(struct bpf_local_storage *local_storage,  in __bpf_local_storage_free()  argument
123 		kfree_rcu(local_storage, rcu);  in __bpf_local_storage_free()
125 		call_rcu_tasks_trace(&local_storage->rcu,  in __bpf_local_storage_free()
131 	struct bpf_local_storage *local_storage;  in bpf_local_storage_free_rcu()  local
133 	local_storage = container_of(rcu, struct bpf_local_storage, rcu);  in bpf_local_storage_free_rcu()
134 	kfree_nolock(local_storage);  in bpf_local_storage_free_rcu()
146 static void bpf_local_storage_free(struct bpf_local_storage *local_storage,  in bpf_local_storage_free()  argument
149 	if (!local_storage)  in bpf_local_storage_free()
152 	if (!local_storage->use_kmalloc_nolock) {  in bpf_local_storage_free()
153 		__bpf_local_storage_free(local_storage, reuse_now);  in bpf_local_storage_free()
158 		call_rcu(&local_storage->rcu, bpf_local_storage_free_rcu);  in bpf_local_storage_free()
162 	call_rcu_tasks_trace(&local_storage->rcu,  in bpf_local_storage_free()
265 						 struct bpf_local_storage *local_storage,  in bpf_selem_unlink_storage_nolock_misc()  argument
268 	void *owner = local_storage->owner;  in bpf_selem_unlink_storage_nolock_misc()
271 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock_misc()
273 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock_misc()
275 	if (pin_owner && !refcount_inc_not_zero(&local_storage->owner_refcnt))  in bpf_selem_unlink_storage_nolock_misc()
278 	uncharge += free_local_storage ? sizeof(*local_storage) : 0;  in bpf_selem_unlink_storage_nolock_misc()
279 	mem_uncharge(smap, local_storage->owner, uncharge);  in bpf_selem_unlink_storage_nolock_misc()
280 	local_storage->mem_charge -= uncharge;  in bpf_selem_unlink_storage_nolock_misc()
283 		local_storage->owner = NULL;  in bpf_selem_unlink_storage_nolock_misc()
290 		refcount_dec(&local_storage->owner_refcnt);  in bpf_selem_unlink_storage_nolock_misc()
297 static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_unlink_storage_nolock()  argument
307 						    &local_storage->list);  in bpf_selem_unlink_storage_nolock()
309 	bpf_selem_unlink_storage_nolock_misc(selem, smap, local_storage,  in bpf_selem_unlink_storage_nolock()
319 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_link_storage_nolock()  argument
325 	local_storage->mem_charge += smap->elem_size;  in bpf_selem_link_storage_nolock()
327 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
328 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
333 	struct bpf_local_storage *local_storage;  in bpf_selem_unlink_map()  local
339 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink_map()
342 	b = select_bucket(smap, local_storage);  in bpf_selem_unlink_map()
359 		       struct bpf_local_storage *local_storage,  in bpf_selem_link_map()  argument
366 	b = select_bucket(smap, local_storage);  in bpf_selem_link_map()
390 	struct bpf_local_storage *local_storage;  in bpf_selem_unlink()  local
400 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink()
403 	err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_selem_unlink()
417 			local_storage, selem, &selem_free_list);  in bpf_selem_unlink()
420 	raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_selem_unlink()
425 		bpf_local_storage_free(local_storage, false);  in bpf_selem_unlink()
439 	struct bpf_local_storage *local_storage;  in bpf_selem_unlink_nofail()  local
444 	local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held());  in bpf_selem_unlink_nofail()
448 		b = b ? : select_bucket(smap, local_storage);  in bpf_selem_unlink_nofail()
477 	if (local_storage) {  in bpf_selem_unlink_nofail()
478 		err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_selem_unlink_nofail()
482 								      &local_storage->list);  in bpf_selem_unlink_nofail()
490 						selem, smap, local_storage,  in bpf_selem_unlink_nofail()
495 			raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_selem_unlink_nofail()
498 			RCU_INIT_POINTER(selem->local_storage, NULL);  in bpf_selem_unlink_nofail()
515 		bpf_local_storage_free(local_storage, true);  in bpf_selem_unlink_nofail()
518 void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,  in __bpf_local_storage_insert_cache()  argument
530 	err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);  in __bpf_local_storage_insert_cache()
535 		rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));  in __bpf_local_storage_insert_cache()
536 	raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);  in __bpf_local_storage_insert_cache()
635 	struct bpf_local_storage *local_storage;  in bpf_local_storage_update()  local
651 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),  in bpf_local_storage_update()
653 	if (!local_storage || hlist_empty(&local_storage->list)) {  in bpf_local_storage_update()
679 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
697 	err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_update()
702 	if (unlikely(hlist_empty(&local_storage->list))) {  in bpf_local_storage_update()
712 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
724 	b = select_bucket(smap, local_storage);  in bpf_local_storage_update()
735 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
740 		bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),  in bpf_local_storage_update()
746 	raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_update()
819 u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage)  in bpf_local_storage_destroy()  argument
832 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)  in bpf_local_storage_destroy()
835 	if (!refcount_dec_and_test(&local_storage->owner_refcnt)) {  in bpf_local_storage_destroy()
836 		while (refcount_read(&local_storage->owner_refcnt))  in bpf_local_storage_destroy()
845 	return local_storage->mem_charge;  in bpf_local_storage_destroy()