22 	struct bpf_map map;  member
29 static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map)  in map_to_storage()  argument
31 	return container_of(map, struct bpf_cgroup_storage_map, map);  in map_to_storage()
34 static bool attach_type_isolated(const struct bpf_map *map)  in attach_type_isolated()  argument
36 	return map->key_size == sizeof(struct bpf_cgroup_storage_key);  in attach_type_isolated()
39 static int bpf_cgroup_storage_key_cmp(const struct bpf_cgroup_storage_map *map,  in bpf_cgroup_storage_key_cmp()  argument
42 	if (attach_type_isolated(&map->map)) {  in bpf_cgroup_storage_key_cmp()
67 cgroup_storage_lookup(struct bpf_cgroup_storage_map *map,  in cgroup_storage_lookup()  argument
70 	struct rb_root *root = &map->root;  in cgroup_storage_lookup()
74 		spin_lock_bh(&map->lock);  in cgroup_storage_lookup()
82 		switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {  in cgroup_storage_lookup()
91 				spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
97 		spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
102 static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map,  in cgroup_storage_insert()  argument
105 	struct rb_root *root = &map->root;  in cgroup_storage_insert()
114 		switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {  in cgroup_storage_insert()
134 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_lookup_elem()  local
137 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
144 static int cgroup_storage_update_elem(struct bpf_map *map, void *key,  in cgroup_storage_update_elem()  argument
154 		     !map_value_has_spin_lock(map)))  in cgroup_storage_update_elem()
157 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
163 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
168 			   map->value_size,  in cgroup_storage_update_elem()
170 			   map->numa_node);  in cgroup_storage_update_elem()
174 	memcpy(&new->data[0], value, map->value_size);  in cgroup_storage_update_elem()
175 	check_and_init_map_lock(map, new->data);  in cgroup_storage_update_elem()
186 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_copy()  local
192 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
215 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_update()  local
224 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
249 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_get_next_key()  local
252 	spin_lock_bh(&map->lock);  in cgroup_storage_get_next_key()
254 	if (list_empty(&map->list))  in cgroup_storage_get_next_key()
258 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
266 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
270 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
272 	if (attach_type_isolated(&map->map)) {  in cgroup_storage_get_next_key()
282 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
289 	struct bpf_cgroup_storage_map *map;  in cgroup_storage_map_alloc()  local
315 	map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),  in cgroup_storage_map_alloc()
317 	if (!map) {  in cgroup_storage_map_alloc()
322 	bpf_map_charge_move(&map->map.memory, &mem);  in cgroup_storage_map_alloc()
324 	/* copy mandatory map attributes */  in cgroup_storage_map_alloc()
325 	bpf_map_init_from_attr(&map->map, attr);  in cgroup_storage_map_alloc()
327 	spin_lock_init(&map->lock);  in cgroup_storage_map_alloc()
328 	map->root = RB_ROOT;  in cgroup_storage_map_alloc()
329 	INIT_LIST_HEAD(&map->list);  in cgroup_storage_map_alloc()
331 	return &map->map;  in cgroup_storage_map_alloc()
336 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_map_free()  local
337 	struct list_head *storages = &map->list;  in cgroup_storage_map_free()
349 	WARN_ON(!RB_EMPTY_ROOT(&map->root));  in cgroup_storage_map_free()
350 	WARN_ON(!list_empty(&map->list));  in cgroup_storage_map_free()
352 	kfree(map);  in cgroup_storage_map_free()
355 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)  in cgroup_storage_delete_elem()  argument
360 static int cgroup_storage_check_btf(const struct bpf_map *map,  in cgroup_storage_check_btf()  argument
365 	if (attach_type_isolated(map)) {  in cgroup_storage_check_btf()
418 static void cgroup_storage_seq_show_elem(struct bpf_map *map, void *key,  in cgroup_storage_seq_show_elem()  argument
421 	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);  in cgroup_storage_seq_show_elem()
426 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
432 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in cgroup_storage_seq_show_elem()
433 	stype = cgroup_storage_type(map);  in cgroup_storage_seq_show_elem()
436 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
443 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
479 static size_t bpf_cgroup_storage_calculate_size(struct bpf_map *map, u32 *pages)  in bpf_cgroup_storage_calculate_size()  argument
483 	if (cgroup_storage_type(map) == BPF_CGROUP_STORAGE_SHARED) {  in bpf_cgroup_storage_calculate_size()
484 		size = sizeof(struct bpf_storage_buffer) + map->value_size;  in bpf_cgroup_storage_calculate_size()
488 		size = map->value_size;  in bpf_cgroup_storage_calculate_size()
500 	struct bpf_map *map;  in bpf_cgroup_storage_alloc()  local
505 	map = prog->aux->cgroup_storage[stype];  in bpf_cgroup_storage_alloc()
506 	if (!map)  in bpf_cgroup_storage_alloc()
509 	size = bpf_cgroup_storage_calculate_size(map, &pages);  in bpf_cgroup_storage_alloc()
511 	if (bpf_map_charge_memlock(map, pages))  in bpf_cgroup_storage_alloc()
515 			       __GFP_ZERO | GFP_USER, map->numa_node);  in bpf_cgroup_storage_alloc()
522 		storage->buf = kmalloc_node(size, flags, map->numa_node);  in bpf_cgroup_storage_alloc()
525 		check_and_init_map_lock(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
532 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
537 	bpf_map_uncharge_memlock(map, pages);  in bpf_cgroup_storage_alloc()
563 	struct bpf_map *map;  in bpf_cgroup_storage_free()  local
569 	map = &storage->map->map;  in bpf_cgroup_storage_free()
571 	bpf_cgroup_storage_calculate_size(map, &pages);  in bpf_cgroup_storage_free()
572 	bpf_map_uncharge_memlock(map, pages);  in bpf_cgroup_storage_free()
574 	stype = cgroup_storage_type(map);  in bpf_cgroup_storage_free()
585 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_link()  local
593 	map = storage->map;  in bpf_cgroup_storage_link()
595 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_link()
596 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
597 	list_add(&storage->list_map, &map->list);  in bpf_cgroup_storage_link()
599 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_link()
604 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_unlink()  local
610 	map = storage->map;  in bpf_cgroup_storage_unlink()
612 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_unlink()
613 	root = &map->root;  in bpf_cgroup_storage_unlink()
618 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_unlink()