484 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_sys_lookup_elem()  local
492 	kvalue = &st_map->kvalue;  in bpf_struct_ops_map_sys_lookup_elem()
504 	memcpy(uvalue, st_map->uvalue, map->value_size);  in bpf_struct_ops_map_sys_lookup_elem()
523 static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_put_progs()  argument
527 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_put_progs()
528 		if (!st_map->links[i])  in bpf_struct_ops_map_put_progs()
530 		bpf_link_put(st_map->links[i]);  in bpf_struct_ops_map_put_progs()
531 		st_map->links[i] = NULL;  in bpf_struct_ops_map_put_progs()
535 static void bpf_struct_ops_map_dissoc_progs(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_dissoc_progs()  argument
539 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_dissoc_progs()
540 		if (!st_map->links[i])  in bpf_struct_ops_map_dissoc_progs()
542 		bpf_prog_disassoc_struct_ops(st_map->links[i]->prog);  in bpf_struct_ops_map_dissoc_progs()
546 static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_free_image()  argument
550 	for (i = 0; i < st_map->image_pages_cnt; i++)  in bpf_struct_ops_map_free_image()
551 		bpf_struct_ops_image_free(st_map->image_pages[i]);  in bpf_struct_ops_map_free_image()
552 	st_map->image_pages_cnt = 0;  in bpf_struct_ops_map_free_image()
652 static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_add_ksyms()  argument
656 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_add_ksyms()
657 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_add_ksyms()
659 		bpf_image_ksym_add(st_map->ksyms[i]);  in bpf_struct_ops_map_add_ksyms()
663 static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_del_ksyms()  argument
667 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_del_ksyms()
668 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_del_ksyms()
670 		bpf_image_ksym_del(st_map->ksyms[i]);  in bpf_struct_ops_map_del_ksyms()
674 static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_free_ksyms()  argument
678 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_free_ksyms()
679 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_free_ksyms()
681 		kfree(st_map->ksyms[i]);  in bpf_struct_ops_map_free_ksyms()
682 		st_map->ksyms[i] = NULL;  in bpf_struct_ops_map_free_ksyms()
689 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_update_elem()  local
690 	const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;  in bpf_struct_ops_map_update_elem()
711 	err = check_zero_holes(st_map->btf, st_ops_desc->value_type, value);  in bpf_struct_ops_map_update_elem()
716 	err = check_zero_holes(st_map->btf, t, uvalue->data);  in bpf_struct_ops_map_update_elem()
727 	uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;  in bpf_struct_ops_map_update_elem()
728 	kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;  in bpf_struct_ops_map_update_elem()
730 	mutex_lock(&st_map->lock);  in bpf_struct_ops_map_update_elem()
742 	plink = st_map->links;  in bpf_struct_ops_map_update_elem()
743 	pksym = st_map->ksyms;  in bpf_struct_ops_map_update_elem()
744 	tname = btf_name_by_offset(st_map->btf, t->name_off);  in bpf_struct_ops_map_update_elem()
754 		mname = btf_name_by_offset(st_map->btf, member->name_off);  in bpf_struct_ops_map_update_elem()
755 		ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);  in bpf_struct_ops_map_update_elem()
780 			mtype = btf_type_by_id(st_map->btf, member->type);  in bpf_struct_ops_map_update_elem()
781 			mtype = btf_resolve_size(st_map->btf, mtype, &msize);  in bpf_struct_ops_map_update_elem()
815 		bpf_prog_assoc_struct_ops(prog, &st_map->map);  in bpf_struct_ops_map_update_elem()
839 						st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);  in bpf_struct_ops_map_update_elem()
844 			st_map->image_pages[st_map->image_pages_cnt++] = image;  in bpf_struct_ops_map_update_elem()
866 	for (i = 0; i < st_map->image_pages_cnt; i++) {  in bpf_struct_ops_map_update_elem()
867 		err = arch_protect_bpf_trampoline(st_map->image_pages[i],  in bpf_struct_ops_map_update_elem()
873 	if (st_map->map.map_flags & BPF_F_LINK) {  in bpf_struct_ops_map_update_elem()
907 	bpf_struct_ops_map_free_ksyms(st_map);  in bpf_struct_ops_map_update_elem()
908 	bpf_struct_ops_map_free_image(st_map);  in bpf_struct_ops_map_update_elem()
909 	bpf_struct_ops_map_put_progs(st_map);  in bpf_struct_ops_map_update_elem()
914 	mutex_unlock(&st_map->lock);  in bpf_struct_ops_map_update_elem()
916 		bpf_struct_ops_map_add_ksyms(st_map);  in bpf_struct_ops_map_update_elem()
923 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_delete_elem()  local
925 	st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_delete_elem()
926 	if (st_map->map.map_flags & BPF_F_LINK)  in bpf_struct_ops_map_delete_elem()
929 	prev_state = cmpxchg(&st_map->kvalue.common.state,  in bpf_struct_ops_map_delete_elem()
934 		st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, NULL);  in bpf_struct_ops_map_delete_elem()
951 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_seq_show_elem()  local
961 		btf_type_seq_show(st_map->btf,  in bpf_struct_ops_map_seq_show_elem()
972 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in __bpf_struct_ops_map_free()  local
974 	if (st_map->links)  in __bpf_struct_ops_map_free()
975 		bpf_struct_ops_map_put_progs(st_map);  in __bpf_struct_ops_map_free()
976 	if (st_map->ksyms)  in __bpf_struct_ops_map_free()
977 		bpf_struct_ops_map_free_ksyms(st_map);  in __bpf_struct_ops_map_free()
978 	bpf_map_area_free(st_map->links);  in __bpf_struct_ops_map_free()
979 	bpf_map_area_free(st_map->ksyms);  in __bpf_struct_ops_map_free()
980 	bpf_struct_ops_map_free_image(st_map);  in __bpf_struct_ops_map_free()
981 	bpf_map_area_free(st_map->uvalue);  in __bpf_struct_ops_map_free()
982 	bpf_map_area_free(st_map);  in __bpf_struct_ops_map_free()
987 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_free()  local
993 	if (btf_is_module(st_map->btf))  in bpf_struct_ops_map_free()
994 		module_put(st_map->st_ops_desc->st_ops->owner);  in bpf_struct_ops_map_free()
996 	bpf_struct_ops_map_dissoc_progs(st_map);  in bpf_struct_ops_map_free()
998 	bpf_struct_ops_map_del_ksyms(st_map);  in bpf_struct_ops_map_free()
1046 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_alloc()  local
1092 	st_map_size = sizeof(*st_map) +  in bpf_struct_ops_map_alloc()
1098 	st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);  in bpf_struct_ops_map_alloc()
1099 	if (!st_map) {  in bpf_struct_ops_map_alloc()
1104 	st_map->st_ops_desc = st_ops_desc;  in bpf_struct_ops_map_alloc()
1105 	map = &st_map->map;  in bpf_struct_ops_map_alloc()
1107 	st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);  in bpf_struct_ops_map_alloc()
1108 	st_map->funcs_cnt = count_func_ptrs(btf, t);  in bpf_struct_ops_map_alloc()
1109 	st_map->links =  in bpf_struct_ops_map_alloc()
1110 		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),  in bpf_struct_ops_map_alloc()
1113 	st_map->ksyms =  in bpf_struct_ops_map_alloc()
1114 		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),  in bpf_struct_ops_map_alloc()
1116 	if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {  in bpf_struct_ops_map_alloc()
1120 	st_map->btf = btf;  in bpf_struct_ops_map_alloc()
1122 	mutex_init(&st_map->lock);  in bpf_struct_ops_map_alloc()
1137 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_mem_usage()  local
1138 	const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;  in bpf_struct_ops_map_mem_usage()
1142 	usage = sizeof(*st_map) +  in bpf_struct_ops_map_mem_usage()
1145 	usage += st_map->funcs_cnt * sizeof(struct bpf_link *);  in bpf_struct_ops_map_mem_usage()
1146 	usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);  in bpf_struct_ops_map_mem_usage()
1171 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_get()  local
1175 	st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);  in bpf_struct_ops_get()
1177 	map = __bpf_map_inc_not_zero(&st_map->map, false);  in bpf_struct_ops_get()
1185 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_put()  local
1188 	st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);  in bpf_struct_ops_put()
1190 	bpf_map_put(&st_map->map);  in bpf_struct_ops_put()
1197 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_id()  local
1200 	st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);  in bpf_struct_ops_id()
1202 	return st_map->map.id;  in bpf_struct_ops_id()
1208 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_valid_to_reg()  local
1213 		smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY;  in bpf_struct_ops_valid_to_reg()
1219 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_link_dealloc()  local
1222 	st_map = (struct bpf_struct_ops_map *)  in bpf_struct_ops_map_link_dealloc()
1224 	if (st_map) {  in bpf_struct_ops_map_link_dealloc()
1225 		st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);  in bpf_struct_ops_map_link_dealloc()
1226 		bpf_map_put(&st_map->map);  in bpf_struct_ops_map_link_dealloc()
1263 	struct bpf_struct_ops_map *st_map, *old_st_map;  in bpf_struct_ops_map_link_update()  local
1269 	st_map = container_of(new_map, struct bpf_struct_ops_map, map);  in bpf_struct_ops_map_link_update()
1274 	if (!st_map->st_ops_desc->st_ops->update)  in bpf_struct_ops_map_link_update()
1291 	if (st_map->st_ops_desc != old_st_map->st_ops_desc) {  in bpf_struct_ops_map_link_update()
1296 	err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link);  in bpf_struct_ops_map_link_update()
1313 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_link_detach()  local
1323 	st_map = container_of(map, struct bpf_struct_ops_map, map);  in bpf_struct_ops_map_link_detach()
1325 	st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);  in bpf_struct_ops_map_link_detach()
1331 	bpf_map_put(&st_map->map);  in bpf_struct_ops_map_link_detach()
1363 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_link_create()  local
1371 	st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_link_create()
1396 	err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data, &link->link);  in bpf_struct_ops_link_create()
1473 	struct bpf_struct_ops_map *st_map;  in bpf_prog_get_assoc_struct_ops()  local
1480 	st_map = (struct bpf_struct_ops_map *)st_ops_assoc;  in bpf_prog_get_assoc_struct_ops()
1482 	return &st_map->kvalue.data;  in bpf_prog_get_assoc_struct_ops()
1488 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_map_struct_ops_info_fill()  local
1490 	info->btf_vmlinux_id = btf_obj_id(st_map->btf);  in bpf_map_struct_ops_info_fill()