Lines Matching +full:ext +full:- +full:gen
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
70 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
223 [BPF_PROG_TYPE_EXT] = "ext",
298 if (err != -EPERM || geteuid() != 0) in pr_perm_msg()
315 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", in pr_perm_msg()
331 fd = -1; \
342 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ in libbpf_set_strict_mode()
388 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
407 /* BPF program support non-linear XDP buffer */
445 * program. For the entry-point (main) BPF program, this is always
446 * zero. For a sub-program, this gets reset before each of main BPF
448 * whether sub-program was already appended to the main program, and
460 * entry-point BPF programs this includes the size of main program
461 * itself plus all the used sub-programs, appended at the end
510 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
619 /* BTF fd index to be patched in for insn->off, this is
620 * 0 for vmlinux BTF, index in obj->fd_array for module
713 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
717 /* vmlinux BTF override for CO-RE relocations */
762 zclose(prog->fd); in bpf_program__unload()
764 zfree(&prog->func_info); in bpf_program__unload()
765 zfree(&prog->line_info); in bpf_program__unload()
774 zfree(&prog->name); in bpf_program__exit()
775 zfree(&prog->sec_name); in bpf_program__exit()
776 zfree(&prog->insns); in bpf_program__exit()
777 zfree(&prog->reloc_desc); in bpf_program__exit()
779 prog->nr_reloc = 0; in bpf_program__exit()
780 prog->insns_cnt = 0; in bpf_program__exit()
781 prog->sec_idx = -1; in bpf_program__exit()
786 return BPF_CLASS(insn->code) == BPF_JMP && in insn_is_subprog_call()
787 BPF_OP(insn->code) == BPF_CALL && in insn_is_subprog_call()
788 BPF_SRC(insn->code) == BPF_K && in insn_is_subprog_call()
789 insn->src_reg == BPF_PSEUDO_CALL && in insn_is_subprog_call()
790 insn->dst_reg == 0 && in insn_is_subprog_call()
791 insn->off == 0; in insn_is_subprog_call()
796 return insn->code == (BPF_JMP | BPF_CALL); in is_call_insn()
801 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; in insn_is_pseudo_func()
812 return -EINVAL; in bpf_object__init_prog()
816 prog->obj = obj; in bpf_object__init_prog()
818 prog->sec_idx = sec_idx; in bpf_object__init_prog()
819 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
820 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
822 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
824 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
825 prog->fd = -1; in bpf_object__init_prog()
826 prog->exception_cb_idx = -1; in bpf_object__init_prog()
833 prog->autoload = false; in bpf_object__init_prog()
837 prog->autoload = true; in bpf_object__init_prog()
840 prog->autoattach = true; in bpf_object__init_prog()
843 prog->log_level = obj->log_level; in bpf_object__init_prog()
845 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
846 if (!prog->sec_name) in bpf_object__init_prog()
849 prog->name = strdup(name); in bpf_object__init_prog()
850 if (!prog->name) in bpf_object__init_prog()
853 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
854 if (!prog->insns) in bpf_object__init_prog()
856 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
862 return -ENOMEM; in bpf_object__init_prog()
869 Elf_Data *symbols = obj->efile.symbols; in bpf_object__add_programs()
871 void *data = sec_data->d_buf; in bpf_object__add_programs()
872 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; in bpf_object__add_programs()
877 progs = obj->programs; in bpf_object__add_programs()
878 nr_progs = obj->nr_programs; in bpf_object__add_programs()
879 nr_syms = symbols->d_size / sizeof(Elf64_Sym); in bpf_object__add_programs()
884 if (sym->st_shndx != sec_idx) in bpf_object__add_programs()
886 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) in bpf_object__add_programs()
889 prog_sz = sym->st_size; in bpf_object__add_programs()
890 sec_off = sym->st_value; in bpf_object__add_programs()
892 name = elf_sym_str(obj, sym->st_name); in bpf_object__add_programs()
896 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
902 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
905 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { in bpf_object__add_programs()
907 return -ENOTSUP; in bpf_object__add_programs()
916 * In this case the original obj->programs in bpf_object__add_programs()
922 return -ENOMEM; in bpf_object__add_programs()
924 obj->programs = progs; in bpf_object__add_programs()
933 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) in bpf_object__add_programs()
934 prog->sym_global = true; in bpf_object__add_programs()
941 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
942 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) in bpf_object__add_programs()
943 prog->mark_btf_static = true; in bpf_object__add_programs()
946 obj->nr_programs = nr_progs; in bpf_object__add_programs()
954 struct bpf_program *prog = obj->programs; in bpf_object_bswap_progs()
958 for (p = 0; p < obj->nr_programs; p++, prog++) { in bpf_object_bswap_progs()
959 insn = prog->insns; in bpf_object_bswap_progs()
960 for (i = 0; i < prog->insns_cnt; i++, insn++) in bpf_object_bswap_progs()
963 pr_debug("converted %zu BPF programs to native byte order\n", obj->nr_programs); in bpf_object_bswap_progs()
988 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) in find_member_by_name()
1051 if (kern_data_member->type == kern_type_id) in find_struct_ops_kern_types()
1057 return -EINVAL; in find_struct_ops_kern_types()
1071 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; in bpf_map__is_struct_ops()
1079 for (i = 0; i < obj->nr_programs; i++) { in is_valid_st_ops_program()
1080 if (&obj->programs[i] == prog) in is_valid_st_ops_program()
1081 return prog->type == BPF_PROG_TYPE_STRUCT_OPS; in is_valid_st_ops_program()
1098 for (i = 0; i < obj->nr_programs; ++i) { in bpf_object_adjust_struct_ops_autoload()
1102 prog = &obj->programs[i]; in bpf_object_adjust_struct_ops_autoload()
1103 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) in bpf_object_adjust_struct_ops_autoload()
1106 for (j = 0; j < obj->nr_maps; ++j) { in bpf_object_adjust_struct_ops_autoload()
1109 map = &obj->maps[j]; in bpf_object_adjust_struct_ops_autoload()
1113 type = btf__type_by_id(obj->btf, map->st_ops->type_id); in bpf_object_adjust_struct_ops_autoload()
1116 slot_prog = map->st_ops->progs[k]; in bpf_object_adjust_struct_ops_autoload()
1121 if (map->autocreate) in bpf_object_adjust_struct_ops_autoload()
1126 prog->autoload = should_load; in bpf_object_adjust_struct_ops_autoload()
1138 struct bpf_object *obj = map->obj; in bpf_map__init_kern_struct_ops()
1139 const struct btf *btf = obj->btf; in bpf_map__init_kern_struct_ops()
1147 st_ops = map->st_ops; in bpf_map__init_kern_struct_ops()
1148 type = btf__type_by_id(btf, st_ops->type_id); in bpf_map__init_kern_struct_ops()
1149 tname = btf__name_by_offset(btf, type->name_off); in bpf_map__init_kern_struct_ops()
1157 kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux; in bpf_map__init_kern_struct_ops()
1160 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); in bpf_map__init_kern_struct_ops()
1162 map->mod_btf_fd = mod_btf ? mod_btf->fd : -1; in bpf_map__init_kern_struct_ops()
1163 map->def.value_size = kern_vtype->size; in bpf_map__init_kern_struct_ops()
1164 map->btf_vmlinux_value_type_id = kern_vtype_id; in bpf_map__init_kern_struct_ops()
1166 st_ops->kern_vdata = calloc(1, kern_vtype->size); in bpf_map__init_kern_struct_ops()
1167 if (!st_ops->kern_vdata) in bpf_map__init_kern_struct_ops()
1168 return -ENOMEM; in bpf_map__init_kern_struct_ops()
1170 data = st_ops->data; in bpf_map__init_kern_struct_ops()
1171 kern_data_off = kern_data_member->offset / 8; in bpf_map__init_kern_struct_ops()
1172 kern_data = st_ops->kern_vdata + kern_data_off; in bpf_map__init_kern_struct_ops()
1185 mname = btf__name_by_offset(btf, member->name_off); in bpf_map__init_kern_struct_ops()
1186 moff = member->offset / 8; in bpf_map__init_kern_struct_ops()
1188 msize = btf__resolve_size(btf, member->type); in bpf_map__init_kern_struct_ops()
1191 map->name, mname); in bpf_map__init_kern_struct_ops()
1199 map->name, mname); in bpf_map__init_kern_struct_ops()
1200 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1203 if (st_ops->progs[i]) { in bpf_map__init_kern_struct_ops()
1211 st_ops->progs[i]->autoload = false; in bpf_map__init_kern_struct_ops()
1212 st_ops->progs[i] = NULL; in bpf_map__init_kern_struct_ops()
1215 /* Skip all-zero/NULL fields if they are not present in the kernel BTF */ in bpf_map__init_kern_struct_ops()
1217 map->name, mname); in bpf_map__init_kern_struct_ops()
1221 kern_member_idx = kern_member - btf_members(kern_type); in bpf_map__init_kern_struct_ops()
1225 map->name, mname); in bpf_map__init_kern_struct_ops()
1226 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1229 kern_moff = kern_member->offset / 8; in bpf_map__init_kern_struct_ops()
1232 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); in bpf_map__init_kern_struct_ops()
1233 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, in bpf_map__init_kern_struct_ops()
1235 if (BTF_INFO_KIND(mtype->info) != in bpf_map__init_kern_struct_ops()
1236 BTF_INFO_KIND(kern_mtype->info)) { in bpf_map__init_kern_struct_ops()
1238 map->name, mname, BTF_INFO_KIND(mtype->info), in bpf_map__init_kern_struct_ops()
1239 BTF_INFO_KIND(kern_mtype->info)); in bpf_map__init_kern_struct_ops()
1240 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1249 if (st_ops->progs[i] && st_ops->progs[i] != prog) in bpf_map__init_kern_struct_ops()
1250 st_ops->progs[i]->autoload = false; in bpf_map__init_kern_struct_ops()
1253 st_ops->progs[i] = prog; in bpf_map__init_kern_struct_ops()
1259 map->name, mname); in bpf_map__init_kern_struct_ops()
1260 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1264 kern_mtype->type, in bpf_map__init_kern_struct_ops()
1267 /* mtype->type must be a func_proto which was in bpf_map__init_kern_struct_ops()
1273 map->name, mname); in bpf_map__init_kern_struct_ops()
1274 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1278 prog->attach_btf_obj_fd = mod_btf->fd; in bpf_map__init_kern_struct_ops()
1283 if (!prog->attach_btf_id) { in bpf_map__init_kern_struct_ops()
1284 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1285 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1288 /* struct_ops BPF prog can be re-used between multiple in bpf_map__init_kern_struct_ops()
1293 if (prog->attach_btf_id != kern_type_id) { in bpf_map__init_kern_struct_ops()
1295 map->name, mname, prog->name, prog->sec_name, prog->type, in bpf_map__init_kern_struct_ops()
1296 prog->attach_btf_id, kern_type_id); in bpf_map__init_kern_struct_ops()
1297 return -EINVAL; in bpf_map__init_kern_struct_ops()
1299 if (prog->expected_attach_type != kern_member_idx) { in bpf_map__init_kern_struct_ops()
1301 map->name, mname, prog->name, prog->sec_name, prog->type, in bpf_map__init_kern_struct_ops()
1302 prog->expected_attach_type, kern_member_idx); in bpf_map__init_kern_struct_ops()
1303 return -EINVAL; in bpf_map__init_kern_struct_ops()
1306 st_ops->kern_func_off[i] = kern_data_off + kern_moff; in bpf_map__init_kern_struct_ops()
1309 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
1318 map->name, mname, (ssize_t)msize, in bpf_map__init_kern_struct_ops()
1320 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1324 map->name, mname, (unsigned int)msize, in bpf_map__init_kern_struct_ops()
1338 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_kern_struct_ops_maps()
1339 map = &obj->maps[i]; in bpf_object__init_kern_struct_ops_maps()
1344 if (!map->autocreate) in bpf_object__init_kern_struct_ops_maps()
1367 if (shndx == -1) in init_struct_ops_maps()
1370 btf = obj->btf; in init_struct_ops_maps()
1376 return -EINVAL; in init_struct_ops_maps()
1382 type = btf__type_by_id(obj->btf, vsi->type); in init_struct_ops_maps()
1383 var_name = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1385 type_id = btf__resolve_type(obj->btf, vsi->type); in init_struct_ops_maps()
1388 vsi->type, sec_name); in init_struct_ops_maps()
1389 return -EINVAL; in init_struct_ops_maps()
1392 type = btf__type_by_id(obj->btf, type_id); in init_struct_ops_maps()
1393 tname = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1396 return -ENOTSUP; in init_struct_ops_maps()
1400 return -EINVAL; in init_struct_ops_maps()
1407 map->sec_idx = shndx; in init_struct_ops_maps()
1408 map->sec_offset = vsi->offset; in init_struct_ops_maps()
1409 map->name = strdup(var_name); in init_struct_ops_maps()
1410 if (!map->name) in init_struct_ops_maps()
1411 return -ENOMEM; in init_struct_ops_maps()
1412 map->btf_value_type_id = type_id; in init_struct_ops_maps()
1418 map->autocreate = false; in init_struct_ops_maps()
1423 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; in init_struct_ops_maps()
1424 map->def.key_size = sizeof(int); in init_struct_ops_maps()
1425 map->def.value_size = type->size; in init_struct_ops_maps()
1426 map->def.max_entries = 1; in init_struct_ops_maps()
1427 map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0; in init_struct_ops_maps()
1428 map->autoattach = true; in init_struct_ops_maps()
1430 map->st_ops = calloc(1, sizeof(*map->st_ops)); in init_struct_ops_maps()
1431 if (!map->st_ops) in init_struct_ops_maps()
1432 return -ENOMEM; in init_struct_ops_maps()
1433 st_ops = map->st_ops; in init_struct_ops_maps()
1434 st_ops->data = malloc(type->size); in init_struct_ops_maps()
1435 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); in init_struct_ops_maps()
1436 st_ops->kern_func_off = malloc(btf_vlen(type) * in init_struct_ops_maps()
1437 sizeof(*st_ops->kern_func_off)); in init_struct_ops_maps()
1438 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) in init_struct_ops_maps()
1439 return -ENOMEM; in init_struct_ops_maps()
1441 if (vsi->offset + type->size > data->d_size) { in init_struct_ops_maps()
1444 return -EINVAL; in init_struct_ops_maps()
1447 memcpy(st_ops->data, in init_struct_ops_maps()
1448 data->d_buf + vsi->offset, in init_struct_ops_maps()
1449 type->size); in init_struct_ops_maps()
1450 st_ops->type_id = type_id; in init_struct_ops_maps()
1453 tname, type_id, var_name, vsi->offset); in init_struct_ops_maps()
1464 for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) { in bpf_object_init_struct_ops()
1465 struct elf_sec_desc *desc = &obj->efile.secs[sec_idx]; in bpf_object_init_struct_ops()
1467 if (desc->sec_type != SEC_ST_OPS) in bpf_object_init_struct_ops()
1472 return -LIBBPF_ERRNO__FORMAT; in bpf_object_init_struct_ops()
1474 err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data); in bpf_object_init_struct_ops()
1493 return ERR_PTR(-ENOMEM); in bpf_object__new()
1496 strcpy(obj->path, path); in bpf_object__new()
1498 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); in bpf_object__new()
1501 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); in bpf_object__new()
1502 end = strchr(obj->name, '.'); in bpf_object__new()
1507 obj->efile.fd = -1; in bpf_object__new()
1514 obj->efile.obj_buf = obj_buf; in bpf_object__new()
1515 obj->efile.obj_buf_sz = obj_buf_sz; in bpf_object__new()
1516 obj->efile.btf_maps_shndx = -1; in bpf_object__new()
1517 obj->kconfig_map_idx = -1; in bpf_object__new()
1519 obj->kern_version = get_kernel_version(); in bpf_object__new()
1520 obj->state = OBJ_OPEN; in bpf_object__new()
1527 if (!obj->efile.elf) in bpf_object__elf_finish()
1530 elf_end(obj->efile.elf); in bpf_object__elf_finish()
1531 obj->efile.elf = NULL; in bpf_object__elf_finish()
1532 obj->efile.ehdr = NULL; in bpf_object__elf_finish()
1533 obj->efile.symbols = NULL; in bpf_object__elf_finish()
1534 obj->efile.arena_data = NULL; in bpf_object__elf_finish()
1536 zfree(&obj->efile.secs); in bpf_object__elf_finish()
1537 obj->efile.sec_cnt = 0; in bpf_object__elf_finish()
1538 zclose(obj->efile.fd); in bpf_object__elf_finish()
1539 obj->efile.obj_buf = NULL; in bpf_object__elf_finish()
1540 obj->efile.obj_buf_sz = 0; in bpf_object__elf_finish()
1549 if (obj->efile.elf) { in bpf_object__elf_init()
1551 return -LIBBPF_ERRNO__LIBELF; in bpf_object__elf_init()
1554 if (obj->efile.obj_buf_sz > 0) { in bpf_object__elf_init()
1556 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); in bpf_object__elf_init()
1558 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); in bpf_object__elf_init()
1559 if (obj->efile.fd < 0) { in bpf_object__elf_init()
1560 err = -errno; in bpf_object__elf_init()
1561 pr_warn("elf: failed to open %s: %s\n", obj->path, errstr(err)); in bpf_object__elf_init()
1565 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); in bpf_object__elf_init()
1569 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1570 err = -LIBBPF_ERRNO__LIBELF; in bpf_object__elf_init()
1574 obj->efile.elf = elf; in bpf_object__elf_init()
1577 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1578 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); in bpf_object__elf_init()
1583 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1584 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); in bpf_object__elf_init()
1588 obj->efile.ehdr = ehdr = elf64_getehdr(elf); in bpf_object__elf_init()
1589 if (!obj->efile.ehdr) { in bpf_object__elf_init()
1590 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1591 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1596 if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB && in bpf_object__elf_init()
1597 ehdr->e_ident[EI_DATA] != ELFDATA2MSB) { in bpf_object__elf_init()
1598 err = -LIBBPF_ERRNO__ENDIAN; in bpf_object__elf_init()
1599 pr_warn("elf: '%s' has unknown byte order\n", obj->path); in bpf_object__elf_init()
1603 obj->byteorder = ehdr->e_ident[EI_DATA]; in bpf_object__elf_init()
1605 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { in bpf_object__elf_init()
1607 obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1608 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1613 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { in bpf_object__elf_init()
1615 obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1616 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1621 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) { in bpf_object__elf_init()
1622 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); in bpf_object__elf_init()
1623 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1636 return obj->byteorder == ELFDATA2LSB; in is_native_endianness()
1638 return obj->byteorder == ELFDATA2MSB; in is_native_endianness()
1648 pr_warn("invalid license section in %s\n", obj->path); in bpf_object__init_license()
1649 return -LIBBPF_ERRNO__FORMAT; in bpf_object__init_license()
1651 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't in bpf_object__init_license()
1654 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); in bpf_object__init_license()
1655 pr_debug("license of %s is %s\n", obj->path, obj->license); in bpf_object__init_license()
1665 pr_warn("invalid kver section in %s\n", obj->path); in bpf_object__init_kversion()
1666 return -LIBBPF_ERRNO__FORMAT; in bpf_object__init_kversion()
1669 obj->kern_version = kver; in bpf_object__init_kversion()
1670 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); in bpf_object__init_kversion()
1688 return -EINVAL; in find_elf_sec_sz()
1693 *size = data->d_size; in find_elf_sec_sz()
1697 return -ENOENT; in find_elf_sec_sz()
1702 Elf_Data *symbols = obj->efile.symbols; in find_elf_var_sym()
1706 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { in find_elf_var_sym()
1709 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) in find_elf_var_sym()
1712 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && in find_elf_var_sym()
1713 ELF64_ST_BIND(sym->st_info) != STB_WEAK) in find_elf_var_sym()
1716 sname = elf_sym_str(obj, sym->st_name); in find_elf_var_sym()
1719 return ERR_PTR(-EIO); in find_elf_var_sym()
1725 return ERR_PTR(-ENOENT); in find_elf_var_sym()
1747 const char *name = "libbpf-placeholder-fd"; in create_placeholder_fd()
1754 return -errno; in create_placeholder_fd()
1759 return -errno; in create_placeholder_fd()
1768 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, in bpf_object__add_map()
1769 sizeof(*obj->maps), obj->nr_maps + 1); in bpf_object__add_map()
1773 map = &obj->maps[obj->nr_maps++]; in bpf_object__add_map()
1774 map->obj = obj; in bpf_object__add_map()
1787 map->fd = create_placeholder_fd(); in bpf_object__add_map()
1788 if (map->fd < 0) in bpf_object__add_map()
1789 return ERR_PTR(map->fd); in bpf_object__add_map()
1790 map->inner_map_fd = -1; in bpf_object__add_map()
1791 map->autocreate = true; in bpf_object__add_map()
1810 switch (map->def.type) { in bpf_map_mmap_sz()
1812 return array_map_mmap_sz(map->def.value_size, map->def.max_entries); in bpf_map_mmap_sz()
1814 return page_sz * map->def.max_entries; in bpf_map_mmap_sz()
1824 if (!map->mmaped) in bpf_map_mmap_resize()
1825 return -EINVAL; in bpf_map_mmap_resize()
1830 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); in bpf_map_mmap_resize()
1832 return -errno; in bpf_map_mmap_resize()
1834 memcpy(mmaped, map->mmaped, min(old_sz, new_sz)); in bpf_map_mmap_resize()
1835 munmap(map->mmaped, old_sz); in bpf_map_mmap_resize()
1836 map->mmaped = mmaped; in bpf_map_mmap_resize()
1870 * '.rodata.abracad' kernel and user-visible name. in internal_map_name()
1879 sfx_len = BPF_OBJ_NAME_LEN - 1; in internal_map_name()
1885 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); in internal_map_name()
1887 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, in internal_map_name()
1912 if (!map->btf_value_type_id) in map_is_mmapable()
1915 t = btf__type_by_id(obj->btf, map->btf_value_type_id); in map_is_mmapable()
1921 vt = btf__type_by_id(obj->btf, vsi->type); in map_is_mmapable()
1925 if (btf_var(vt)->linkage != BTF_VAR_STATIC) in map_is_mmapable()
1945 map->libbpf_type = type; in bpf_object__init_internal_map()
1946 map->sec_idx = sec_idx; in bpf_object__init_internal_map()
1947 map->sec_offset = 0; in bpf_object__init_internal_map()
1948 map->real_name = strdup(real_name); in bpf_object__init_internal_map()
1949 map->name = internal_map_name(obj, real_name); in bpf_object__init_internal_map()
1950 if (!map->real_name || !map->name) { in bpf_object__init_internal_map()
1951 zfree(&map->real_name); in bpf_object__init_internal_map()
1952 zfree(&map->name); in bpf_object__init_internal_map()
1953 return -ENOMEM; in bpf_object__init_internal_map()
1956 def = &map->def; in bpf_object__init_internal_map()
1957 def->type = BPF_MAP_TYPE_ARRAY; in bpf_object__init_internal_map()
1958 def->key_size = sizeof(int); in bpf_object__init_internal_map()
1959 def->value_size = data_sz; in bpf_object__init_internal_map()
1960 def->max_entries = 1; in bpf_object__init_internal_map()
1961 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG in bpf_object__init_internal_map()
1968 def->map_flags |= BPF_F_MMAPABLE; in bpf_object__init_internal_map()
1971 map->name, map->sec_idx, map->sec_offset, def->map_flags); in bpf_object__init_internal_map()
1974 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, in bpf_object__init_internal_map()
1975 MAP_SHARED | MAP_ANONYMOUS, -1, 0); in bpf_object__init_internal_map()
1976 if (map->mmaped == MAP_FAILED) { in bpf_object__init_internal_map()
1977 err = -errno; in bpf_object__init_internal_map()
1978 map->mmaped = NULL; in bpf_object__init_internal_map()
1979 pr_warn("failed to alloc map '%s' content buffer: %s\n", map->name, errstr(err)); in bpf_object__init_internal_map()
1980 zfree(&map->real_name); in bpf_object__init_internal_map()
1981 zfree(&map->name); in bpf_object__init_internal_map()
1986 memcpy(map->mmaped, data, data_sz); in bpf_object__init_internal_map()
1988 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); in bpf_object__init_internal_map()
1999 * Populate obj->maps with libbpf internal maps. in bpf_object__init_global_data_maps()
2001 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { in bpf_object__init_global_data_maps()
2002 sec_desc = &obj->efile.secs[sec_idx]; in bpf_object__init_global_data_maps()
2005 if (!sec_desc->data || sec_desc->data->d_size == 0) in bpf_object__init_global_data_maps()
2008 switch (sec_desc->sec_type) { in bpf_object__init_global_data_maps()
2013 sec_desc->data->d_buf, in bpf_object__init_global_data_maps()
2014 sec_desc->data->d_size); in bpf_object__init_global_data_maps()
2017 obj->has_rodata = true; in bpf_object__init_global_data_maps()
2021 sec_desc->data->d_buf, in bpf_object__init_global_data_maps()
2022 sec_desc->data->d_size); in bpf_object__init_global_data_maps()
2029 sec_desc->data->d_size); in bpf_object__init_global_data_maps()
2047 for (i = 0; i < obj->nr_extern; i++) { in find_extern_by_name()
2048 if (strcmp(obj->externs[i].name, name) == 0) in find_extern_by_name()
2049 return &obj->externs[i]; in find_extern_by_name()
2060 for (i = 0; i < obj->nr_extern; i++) { in find_extern_by_name_with_len()
2061 ext_name = obj->externs[i].name; in find_extern_by_name_with_len()
2063 return &obj->externs[i]; in find_extern_by_name_with_len()
2068 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, in set_kcfg_value_tri() argument
2071 switch (ext->kcfg.type) { in set_kcfg_value_tri()
2075 ext->name, value); in set_kcfg_value_tri()
2076 return -EINVAL; in set_kcfg_value_tri()
2096 ext->name, value); in set_kcfg_value_tri()
2097 return -EINVAL; in set_kcfg_value_tri()
2099 ext->is_set = true; in set_kcfg_value_tri()
2103 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, in set_kcfg_value_str() argument
2108 if (ext->kcfg.type != KCFG_CHAR_ARR) { in set_kcfg_value_str()
2110 ext->name, value); in set_kcfg_value_str()
2111 return -EINVAL; in set_kcfg_value_str()
2115 if (len < 2 || value[len - 1] != '"') { in set_kcfg_value_str()
2117 ext->name, value); in set_kcfg_value_str()
2118 return -EINVAL; in set_kcfg_value_str()
2122 len -= 2; in set_kcfg_value_str()
2123 if (len >= ext->kcfg.sz) { in set_kcfg_value_str()
2125 ext->name, value, len, ext->kcfg.sz - 1); in set_kcfg_value_str()
2126 len = ext->kcfg.sz - 1; in set_kcfg_value_str()
2130 ext->is_set = true; in set_kcfg_value_str()
2142 err = -errno; in parse_u64()
2148 return -EINVAL; in parse_u64()
2153 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) in is_kcfg_value_in_range() argument
2155 int bit_sz = ext->kcfg.sz * 8; in is_kcfg_value_in_range()
2157 if (ext->kcfg.sz == 8) in is_kcfg_value_in_range()
2160 /* Validate that value stored in u64 fits in integer of `ext->sz` in is_kcfg_value_in_range()
2165 * -2^(Y-1) <= X <= 2^(Y-1) - 1 in is_kcfg_value_in_range()
2166 * 0 <= X + 2^(Y-1) <= 2^Y - 1 in is_kcfg_value_in_range()
2167 * 0 <= X + 2^(Y-1) < 2^Y in is_kcfg_value_in_range()
2169 * For unsigned target integer, check that all the (64 - Y) bits are in is_kcfg_value_in_range()
2172 if (ext->kcfg.is_signed) in is_kcfg_value_in_range()
2173 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); in is_kcfg_value_in_range()
2178 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, in set_kcfg_value_num() argument
2181 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && in set_kcfg_value_num()
2182 ext->kcfg.type != KCFG_BOOL) { in set_kcfg_value_num()
2184 ext->name, (unsigned long long)value); in set_kcfg_value_num()
2185 return -EINVAL; in set_kcfg_value_num()
2187 if (ext->kcfg.type == KCFG_BOOL && value > 1) { in set_kcfg_value_num()
2189 ext->name, (unsigned long long)value); in set_kcfg_value_num()
2190 return -EINVAL; in set_kcfg_value_num()
2193 if (!is_kcfg_value_in_range(ext, value)) { in set_kcfg_value_num()
2195 ext->name, (unsigned long long)value, ext->kcfg.sz); in set_kcfg_value_num()
2196 return -ERANGE; in set_kcfg_value_num()
2198 switch (ext->kcfg.sz) { in set_kcfg_value_num()
2212 return -EINVAL; in set_kcfg_value_num()
2214 ext->is_set = true; in set_kcfg_value_num()
2221 struct extern_desc *ext; in bpf_object__process_kconfig_line() local
2233 return -EINVAL; in bpf_object__process_kconfig_line()
2238 if (buf[len - 1] == '\n') in bpf_object__process_kconfig_line()
2239 buf[len - 1] = '\0'; in bpf_object__process_kconfig_line()
2245 return -EINVAL; in bpf_object__process_kconfig_line()
2248 ext = find_extern_by_name(obj, buf); in bpf_object__process_kconfig_line()
2249 if (!ext || ext->is_set) in bpf_object__process_kconfig_line()
2252 ext_val = data + ext->kcfg.data_off; in bpf_object__process_kconfig_line()
2257 err = set_kcfg_value_tri(ext, ext_val, *value); in bpf_object__process_kconfig_line()
2260 err = set_kcfg_value_str(ext, ext_val, value); in bpf_object__process_kconfig_line()
2266 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); in bpf_object__process_kconfig_line()
2269 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { in bpf_object__process_kconfig_line()
2270 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); in bpf_object__process_kconfig_line()
2271 return -EINVAL; in bpf_object__process_kconfig_line()
2273 err = set_kcfg_value_num(ext, ext_val, num); in bpf_object__process_kconfig_line()
2278 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); in bpf_object__process_kconfig_line()
2290 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); in bpf_object__read_kconfig_file()
2292 return -EINVAL; in bpf_object__read_kconfig_file()
2294 return -ENAMETOOLONG; in bpf_object__read_kconfig_file()
2303 return -ENOENT; in bpf_object__read_kconfig_file()
2329 err = -errno; in bpf_object__read_kconfig_mem()
2330 pr_warn("failed to open in-memory Kconfig: %s\n", errstr(err)); in bpf_object__read_kconfig_mem()
2337 pr_warn("error parsing in-memory Kconfig line '%s': %s\n", in bpf_object__read_kconfig_mem()
2349 struct extern_desc *last_ext = NULL, *ext; in bpf_object__init_kconfig_map() local
2353 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__init_kconfig_map()
2354 ext = &obj->externs[i]; in bpf_object__init_kconfig_map()
2355 if (ext->type == EXT_KCFG) in bpf_object__init_kconfig_map()
2356 last_ext = ext; in bpf_object__init_kconfig_map()
2362 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; in bpf_object__init_kconfig_map()
2364 ".kconfig", obj->efile.symbols_shndx, in bpf_object__init_kconfig_map()
2369 obj->kconfig_map_idx = obj->nr_maps - 1; in bpf_object__init_kconfig_map()
2384 *res_id = t->type; in skip_mods_and_typedefs()
2385 t = btf__type_by_id(btf, t->type); in skip_mods_and_typedefs()
2400 t = skip_mods_and_typedefs(btf, t->type, res_id); in resolve_func_ptr()
2447 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); in get_map_field_int()
2448 const char *name = btf__name_by_offset(btf, m->name_off); in get_map_field_int()
2458 arr_t = btf__type_by_id(btf, t->type); in get_map_field_int()
2461 map_name, name, t->type); in get_map_field_int()
2470 *res = arr_info->nelems; in get_map_field_int()
2477 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); in get_map_field_long()
2478 const char *name = btf__name_by_offset(btf, m->name_off); in get_map_field_long()
2505 *res = e->val; in get_map_field_long()
2520 return -EINVAL; in pathname_concat()
2522 return -ENAMETOOLONG; in pathname_concat()
2561 const char *name = btf__name_by_offset(btf, m->name_off); in parse_btf_map_def()
2565 return -EINVAL; in parse_btf_map_def()
2568 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) in parse_btf_map_def()
2569 return -EINVAL; in parse_btf_map_def()
2570 map_def->parts |= MAP_DEF_MAP_TYPE; in parse_btf_map_def()
2572 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) in parse_btf_map_def()
2573 return -EINVAL; in parse_btf_map_def()
2574 map_def->parts |= MAP_DEF_MAX_ENTRIES; in parse_btf_map_def()
2576 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) in parse_btf_map_def()
2577 return -EINVAL; in parse_btf_map_def()
2578 map_def->parts |= MAP_DEF_MAP_FLAGS; in parse_btf_map_def()
2580 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) in parse_btf_map_def()
2581 return -EINVAL; in parse_btf_map_def()
2582 map_def->parts |= MAP_DEF_NUMA_NODE; in parse_btf_map_def()
2587 return -EINVAL; in parse_btf_map_def()
2588 if (map_def->key_size && map_def->key_size != sz) { in parse_btf_map_def()
2590 map_name, map_def->key_size, sz); in parse_btf_map_def()
2591 return -EINVAL; in parse_btf_map_def()
2593 map_def->key_size = sz; in parse_btf_map_def()
2594 map_def->parts |= MAP_DEF_KEY_SIZE; in parse_btf_map_def()
2598 t = btf__type_by_id(btf, m->type); in parse_btf_map_def()
2601 map_name, m->type); in parse_btf_map_def()
2602 return -EINVAL; in parse_btf_map_def()
2607 return -EINVAL; in parse_btf_map_def()
2609 sz = btf__resolve_size(btf, t->type); in parse_btf_map_def()
2612 map_name, t->type, (ssize_t)sz); in parse_btf_map_def()
2615 if (map_def->key_size && map_def->key_size != sz) { in parse_btf_map_def()
2617 map_name, map_def->key_size, (ssize_t)sz); in parse_btf_map_def()
2618 return -EINVAL; in parse_btf_map_def()
2620 map_def->key_size = sz; in parse_btf_map_def()
2621 map_def->key_type_id = t->type; in parse_btf_map_def()
2622 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; in parse_btf_map_def()
2627 return -EINVAL; in parse_btf_map_def()
2628 if (map_def->value_size && map_def->value_size != sz) { in parse_btf_map_def()
2630 map_name, map_def->value_size, sz); in parse_btf_map_def()
2631 return -EINVAL; in parse_btf_map_def()
2633 map_def->value_size = sz; in parse_btf_map_def()
2634 map_def->parts |= MAP_DEF_VALUE_SIZE; in parse_btf_map_def()
2638 t = btf__type_by_id(btf, m->type); in parse_btf_map_def()
2641 map_name, m->type); in parse_btf_map_def()
2642 return -EINVAL; in parse_btf_map_def()
2647 return -EINVAL; in parse_btf_map_def()
2649 sz = btf__resolve_size(btf, t->type); in parse_btf_map_def()
2652 map_name, t->type, (ssize_t)sz); in parse_btf_map_def()
2655 if (map_def->value_size && map_def->value_size != sz) { in parse_btf_map_def()
2657 map_name, map_def->value_size, (ssize_t)sz); in parse_btf_map_def()
2658 return -EINVAL; in parse_btf_map_def()
2660 map_def->value_size = sz; in parse_btf_map_def()
2661 map_def->value_type_id = t->type; in parse_btf_map_def()
2662 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; in parse_btf_map_def()
2665 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type); in parse_btf_map_def()
2666 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY; in parse_btf_map_def()
2667 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value"; in parse_btf_map_def()
2672 pr_warn("map '%s': multi-level inner maps not supported.\n", in parse_btf_map_def()
2674 return -ENOTSUP; in parse_btf_map_def()
2676 if (i != vlen - 1) { in parse_btf_map_def()
2679 return -EINVAL; in parse_btf_map_def()
2682 pr_warn("map '%s': should be map-in-map or prog-array.\n", in parse_btf_map_def()
2684 return -ENOTSUP; in parse_btf_map_def()
2686 if (map_def->value_size && map_def->value_size != 4) { in parse_btf_map_def()
2688 map_name, map_def->value_size); in parse_btf_map_def()
2689 return -EINVAL; in parse_btf_map_def()
2691 map_def->value_size = 4; in parse_btf_map_def()
2692 t = btf__type_by_id(btf, m->type); in parse_btf_map_def()
2695 map_name, desc, m->type); in parse_btf_map_def()
2696 return -EINVAL; in parse_btf_map_def()
2698 if (!btf_is_array(t) || btf_array(t)->nelems) { in parse_btf_map_def()
2699 pr_warn("map '%s': %s spec is not a zero-sized array.\n", in parse_btf_map_def()
2701 return -EINVAL; in parse_btf_map_def()
2703 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); in parse_btf_map_def()
2707 return -EINVAL; in parse_btf_map_def()
2709 t = skip_mods_and_typedefs(btf, t->type, NULL); in parse_btf_map_def()
2712 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n", in parse_btf_map_def()
2714 return -EINVAL; in parse_btf_map_def()
2719 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", in parse_btf_map_def()
2721 return -EINVAL; in parse_btf_map_def()
2729 map_def->parts |= MAP_DEF_INNER_MAP; in parse_btf_map_def()
2735 return -EINVAL; in parse_btf_map_def()
2738 return -EINVAL; in parse_btf_map_def()
2742 return -EINVAL; in parse_btf_map_def()
2744 map_def->pinning = val; in parse_btf_map_def()
2745 map_def->parts |= MAP_DEF_PINNING; in parse_btf_map_def()
2750 return -EINVAL; in parse_btf_map_def()
2751 map_def->map_extra = map_extra; in parse_btf_map_def()
2752 map_def->parts |= MAP_DEF_MAP_EXTRA; in parse_btf_map_def()
2756 return -ENOTSUP; in parse_btf_map_def()
2762 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { in parse_btf_map_def()
2764 return -EINVAL; in parse_btf_map_def()
2779 * a power-of-2 multiple of kernel's page size. If user diligently in adjust_ringbuf_sz()
2786 * user-set size to satisfy both user size request and kernel in adjust_ringbuf_sz()
2795 * very close to UINT_MAX but is not a power-of-2 multiple of in adjust_ringbuf_sz()
2803 return map->def.type == BPF_MAP_TYPE_RINGBUF || in map_is_ringbuf()
2804 map->def.type == BPF_MAP_TYPE_USER_RINGBUF; in map_is_ringbuf()
2809 map->def.type = def->map_type; in fill_map_from_def()
2810 map->def.key_size = def->key_size; in fill_map_from_def()
2811 map->def.value_size = def->value_size; in fill_map_from_def()
2812 map->def.max_entries = def->max_entries; in fill_map_from_def()
2813 map->def.map_flags = def->map_flags; in fill_map_from_def()
2814 map->map_extra = def->map_extra; in fill_map_from_def()
2816 map->numa_node = def->numa_node; in fill_map_from_def()
2817 map->btf_key_type_id = def->key_type_id; in fill_map_from_def()
2818 map->btf_value_type_id = def->value_type_id; in fill_map_from_def()
2820 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ in fill_map_from_def()
2822 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); in fill_map_from_def()
2824 if (def->parts & MAP_DEF_MAP_TYPE) in fill_map_from_def()
2825 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); in fill_map_from_def()
2827 if (def->parts & MAP_DEF_KEY_TYPE) in fill_map_from_def()
2829 map->name, def->key_type_id, def->key_size); in fill_map_from_def()
2830 else if (def->parts & MAP_DEF_KEY_SIZE) in fill_map_from_def()
2831 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); in fill_map_from_def()
2833 if (def->parts & MAP_DEF_VALUE_TYPE) in fill_map_from_def()
2835 map->name, def->value_type_id, def->value_size); in fill_map_from_def()
2836 else if (def->parts & MAP_DEF_VALUE_SIZE) in fill_map_from_def()
2837 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); in fill_map_from_def()
2839 if (def->parts & MAP_DEF_MAX_ENTRIES) in fill_map_from_def()
2840 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); in fill_map_from_def()
2841 if (def->parts & MAP_DEF_MAP_FLAGS) in fill_map_from_def()
2842 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags); in fill_map_from_def()
2843 if (def->parts & MAP_DEF_MAP_EXTRA) in fill_map_from_def()
2844 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name, in fill_map_from_def()
2845 (unsigned long long)def->map_extra); in fill_map_from_def()
2846 if (def->parts & MAP_DEF_PINNING) in fill_map_from_def()
2847 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); in fill_map_from_def()
2848 if (def->parts & MAP_DEF_NUMA_NODE) in fill_map_from_def()
2849 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); in fill_map_from_def()
2851 if (def->parts & MAP_DEF_INNER_MAP) in fill_map_from_def()
2852 pr_debug("map '%s': found inner map definition.\n", map->name); in fill_map_from_def()
2880 var = btf__type_by_id(obj->btf, vi->type); in bpf_object__init_user_btf_map()
2882 map_name = btf__name_by_offset(obj->btf, var->name_off); in bpf_object__init_user_btf_map()
2886 return -EINVAL; in bpf_object__init_user_btf_map()
2888 if ((__u64)vi->offset + vi->size > data->d_size) { in bpf_object__init_user_btf_map()
2890 return -EINVAL; in bpf_object__init_user_btf_map()
2895 return -EINVAL; in bpf_object__init_user_btf_map()
2897 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { in bpf_object__init_user_btf_map()
2899 map_name, btf_var_linkage_str(var_extra->linkage)); in bpf_object__init_user_btf_map()
2900 return -EOPNOTSUPP; in bpf_object__init_user_btf_map()
2903 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); in bpf_object__init_user_btf_map()
2907 return -EINVAL; in bpf_object__init_user_btf_map()
2909 if (def->size > vi->size) { in bpf_object__init_user_btf_map()
2911 return -EINVAL; in bpf_object__init_user_btf_map()
2917 map->name = strdup(map_name); in bpf_object__init_user_btf_map()
2918 if (!map->name) { in bpf_object__init_user_btf_map()
2920 return -ENOMEM; in bpf_object__init_user_btf_map()
2922 map->libbpf_type = LIBBPF_MAP_UNSPEC; in bpf_object__init_user_btf_map()
2923 map->def.type = BPF_MAP_TYPE_UNSPEC; in bpf_object__init_user_btf_map()
2924 map->sec_idx = sec_idx; in bpf_object__init_user_btf_map()
2925 map->sec_offset = vi->offset; in bpf_object__init_user_btf_map()
2926 map->btf_var_idx = var_idx; in bpf_object__init_user_btf_map()
2928 map_name, map->sec_idx, map->sec_offset); in bpf_object__init_user_btf_map()
2930 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); in bpf_object__init_user_btf_map()
2939 pr_warn("map '%s': couldn't build pin path.\n", map->name); in bpf_object__init_user_btf_map()
2945 map->inner_map = calloc(1, sizeof(*map->inner_map)); in bpf_object__init_user_btf_map()
2946 if (!map->inner_map) in bpf_object__init_user_btf_map()
2947 return -ENOMEM; in bpf_object__init_user_btf_map()
2948 map->inner_map->fd = create_placeholder_fd(); in bpf_object__init_user_btf_map()
2949 if (map->inner_map->fd < 0) in bpf_object__init_user_btf_map()
2950 return map->inner_map->fd; in bpf_object__init_user_btf_map()
2951 map->inner_map->sec_idx = sec_idx; in bpf_object__init_user_btf_map()
2952 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); in bpf_object__init_user_btf_map()
2953 if (!map->inner_map->name) in bpf_object__init_user_btf_map()
2954 return -ENOMEM; in bpf_object__init_user_btf_map()
2955 sprintf(map->inner_map->name, "%s.inner", map_name); in bpf_object__init_user_btf_map()
2957 fill_map_from_def(map->inner_map, &inner_def); in bpf_object__init_user_btf_map()
2974 mmap_sz = bpf_map_mmap_sz(obj->arena_map); in init_arena_map_data()
2978 return -E2BIG; in init_arena_map_data()
2981 obj->arena_data = malloc(data_sz); in init_arena_map_data()
2982 if (!obj->arena_data) in init_arena_map_data()
2983 return -ENOMEM; in init_arena_map_data()
2984 memcpy(obj->arena_data, data, data_sz); in init_arena_map_data()
2985 obj->arena_data_sz = data_sz; in init_arena_map_data()
2988 map->mmaped = obj->arena_data; in init_arena_map_data()
3003 if (obj->efile.btf_maps_shndx < 0) in bpf_object__init_user_btf_maps()
3006 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); in bpf_object__init_user_btf_maps()
3010 MAPS_ELF_SEC, obj->path); in bpf_object__init_user_btf_maps()
3011 return -EINVAL; in bpf_object__init_user_btf_maps()
3014 nr_types = btf__type_cnt(obj->btf); in bpf_object__init_user_btf_maps()
3016 t = btf__type_by_id(obj->btf, i); in bpf_object__init_user_btf_maps()
3019 name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__init_user_btf_maps()
3022 obj->efile.btf_maps_sec_btf_id = i; in bpf_object__init_user_btf_maps()
3029 return -ENOENT; in bpf_object__init_user_btf_maps()
3035 obj->efile.btf_maps_shndx, in bpf_object__init_user_btf_maps()
3042 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_user_btf_maps()
3043 struct bpf_map *map = &obj->maps[i]; in bpf_object__init_user_btf_maps()
3045 if (map->def.type != BPF_MAP_TYPE_ARENA) in bpf_object__init_user_btf_maps()
3048 if (obj->arena_map) { in bpf_object__init_user_btf_maps()
3050 map->name, obj->arena_map->name); in bpf_object__init_user_btf_maps()
3051 return -EINVAL; in bpf_object__init_user_btf_maps()
3053 obj->arena_map = map; in bpf_object__init_user_btf_maps()
3055 if (obj->efile.arena_data) { in bpf_object__init_user_btf_maps()
3056 err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx, in bpf_object__init_user_btf_maps()
3057 obj->efile.arena_data->d_buf, in bpf_object__init_user_btf_maps()
3058 obj->efile.arena_data->d_size); in bpf_object__init_user_btf_maps()
3063 if (obj->efile.arena_data && !obj->arena_map) { in bpf_object__init_user_btf_maps()
3066 return -ENOENT; in bpf_object__init_user_btf_maps()
3098 return sh->sh_flags & SHF_EXECINSTR; in section_have_execinstr()
3140 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); in bpf_object__sanitize_btf()
3146 t->size = 1; in bpf_object__sanitize_btf()
3155 name = (char *)btf__name_by_offset(btf, t->name_off); in bpf_object__sanitize_btf()
3163 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); in bpf_object__sanitize_btf()
3166 m->offset = v->offset * 8; in bpf_object__sanitize_btf()
3167 m->type = v->type; in bpf_object__sanitize_btf()
3169 vt = (void *)btf__type_by_id(btf, v->type); in bpf_object__sanitize_btf()
3170 m->name_off = vt->name_off; in bpf_object__sanitize_btf()
3173 starts_with_qmark(btf__name_by_offset(btf, t->name_off))) { in bpf_object__sanitize_btf()
3177 name = (char *)btf__name_by_offset(btf, t->name_off); in bpf_object__sanitize_btf()
3183 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); in bpf_object__sanitize_btf()
3184 t->size = sizeof(__u32); /* kernel enforced */ in bpf_object__sanitize_btf()
3187 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); in bpf_object__sanitize_btf()
3190 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); in bpf_object__sanitize_btf()
3192 /* replace FLOAT with an equally-sized empty STRUCT; in bpf_object__sanitize_btf()
3196 t->name_off = 0; in bpf_object__sanitize_btf()
3197 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); in bpf_object__sanitize_btf()
3200 t->name_off = 0; in bpf_object__sanitize_btf()
3201 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); in bpf_object__sanitize_btf()
3204 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); in bpf_object__sanitize_btf()
3219 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); in bpf_object__sanitize_btf()
3221 m->type = enum64_placeholder_id; in bpf_object__sanitize_btf()
3222 m->offset = 0; in bpf_object__sanitize_btf()
3232 return obj->efile.btf_maps_shndx >= 0 || in libbpf_needs_btf()
3233 obj->efile.has_st_ops || in libbpf_needs_btf()
3234 obj->nr_extern > 0; in libbpf_needs_btf()
3239 return obj->efile.has_st_ops; in kernel_needs_btf()
3246 int err = -ENOENT; in bpf_object__init_btf()
3249 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); in bpf_object__init_btf()
3250 err = libbpf_get_error(obj->btf); in bpf_object__init_btf()
3252 obj->btf = NULL; in bpf_object__init_btf()
3256 /* enforce 8-byte pointers for BPF-targeted BTFs */ in bpf_object__init_btf()
3257 btf__set_pointer_size(obj->btf, 8); in bpf_object__init_btf()
3263 if (!obj->btf) { in bpf_object__init_btf()
3268 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); in bpf_object__init_btf()
3269 err = libbpf_get_error(obj->btf_ext); in bpf_object__init_btf()
3273 obj->btf_ext = NULL; in bpf_object__init_btf()
3277 /* setup .BTF.ext to ELF section mapping */ in bpf_object__init_btf()
3278 ext_segs[0] = &obj->btf_ext->func_info; in bpf_object__init_btf()
3279 ext_segs[1] = &obj->btf_ext->line_info; in bpf_object__init_btf()
3280 ext_segs[2] = &obj->btf_ext->core_relo_info; in bpf_object__init_btf()
3287 if (seg->sec_cnt == 0) in bpf_object__init_btf()
3290 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); in bpf_object__init_btf()
3291 if (!seg->sec_idxs) { in bpf_object__init_btf()
3292 err = -ENOMEM; in bpf_object__init_btf()
3303 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); in bpf_object__init_btf()
3310 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); in bpf_object__init_btf()
3327 return a->offset - b->offset; in compare_vsi_off()
3334 const char *sec_name = btf__name_by_offset(btf, t->name_off); in btf_fixup_datasec()
3341 return -ENOENT; in btf_fixup_datasec()
3344 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and in btf_fixup_datasec()
3356 * to be optional. But the STV_HIDDEN handling is non-optional for any in btf_fixup_datasec()
3357 * non-extern DATASEC, so the variable fixup loop below handles both in btf_fixup_datasec()
3358 * functions at the same time, paying the cost of BTF VAR <-> ELF in btf_fixup_datasec()
3361 if (t->size == 0) { in btf_fixup_datasec()
3366 return -ENOENT; in btf_fixup_datasec()
3369 t->size = size; in btf_fixup_datasec()
3379 t_var = btf__type_by_id(btf, vsi->type); in btf_fixup_datasec()
3381 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name); in btf_fixup_datasec()
3382 return -EINVAL; in btf_fixup_datasec()
3386 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN) in btf_fixup_datasec()
3389 var_name = btf__name_by_offset(btf, t_var->name_off); in btf_fixup_datasec()
3393 return -ENOENT; in btf_fixup_datasec()
3400 return -ENOENT; in btf_fixup_datasec()
3404 vsi->offset = sym->st_value; in btf_fixup_datasec()
3413 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in btf_fixup_datasec()
3414 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL) in btf_fixup_datasec()
3415 var->linkage = BTF_VAR_STATIC; in btf_fixup_datasec()
3427 if (!obj->btf) in bpf_object_fixup_btf()
3430 n = btf__type_cnt(obj->btf); in bpf_object_fixup_btf()
3432 struct btf_type *t = btf_type_by_id(obj->btf, i); in bpf_object_fixup_btf()
3440 err = btf_fixup_datasec(obj, obj->btf, t); in bpf_object_fixup_btf()
3451 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || in prog_needs_vmlinux_btf()
3452 prog->type == BPF_PROG_TYPE_LSM) in prog_needs_vmlinux_btf()
3458 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) in prog_needs_vmlinux_btf()
3475 /* CO-RE relocations need kernel BTF, only when btf_custom_path in obj_needs_vmlinux_btf()
3478 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) in obj_needs_vmlinux_btf()
3482 for (i = 0; i < obj->nr_extern; i++) { in obj_needs_vmlinux_btf()
3483 const struct extern_desc *ext; in obj_needs_vmlinux_btf() local
3485 ext = &obj->externs[i]; in obj_needs_vmlinux_btf()
3486 if (ext->type == EXT_KSYM && ext->ksym.type_id) in obj_needs_vmlinux_btf()
3491 if (!prog->autoload) in obj_needs_vmlinux_btf()
3510 if (obj->btf_vmlinux || obj->gen_loader) in bpf_object__load_vmlinux_btf()
3516 obj->btf_vmlinux = btf__load_vmlinux_btf(); in bpf_object__load_vmlinux_btf()
3517 err = libbpf_get_error(obj->btf_vmlinux); in bpf_object__load_vmlinux_btf()
3520 obj->btf_vmlinux = NULL; in bpf_object__load_vmlinux_btf()
3528 struct btf *kern_btf = obj->btf; in bpf_object__sanitize_and_load_btf()
3532 if (!obj->btf) in bpf_object__sanitize_and_load_btf()
3537 err = -EOPNOTSUPP; in bpf_object__sanitize_and_load_btf()
3552 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__sanitize_and_load_btf()
3553 struct bpf_program *prog = &obj->programs[i]; in bpf_object__sanitize_and_load_btf()
3558 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) in bpf_object__sanitize_and_load_btf()
3561 n = btf__type_cnt(obj->btf); in bpf_object__sanitize_and_load_btf()
3563 t = btf_type_by_id(obj->btf, j); in bpf_object__sanitize_and_load_btf()
3567 name = btf__str_by_offset(obj->btf, t->name_off); in bpf_object__sanitize_and_load_btf()
3568 if (strcmp(name, prog->name) != 0) in bpf_object__sanitize_and_load_btf()
3571 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); in bpf_object__sanitize_and_load_btf()
3582 raw_data = btf__raw_data(obj->btf, &sz); in bpf_object__sanitize_and_load_btf()
3588 /* enforce 8-byte pointers for BPF-targeted BTFs */ in bpf_object__sanitize_and_load_btf()
3589 btf__set_pointer_size(obj->btf, 8); in bpf_object__sanitize_and_load_btf()
3595 if (obj->gen_loader) { in bpf_object__sanitize_and_load_btf()
3600 return -ENOMEM; in bpf_object__sanitize_and_load_btf()
3601 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); in bpf_object__sanitize_and_load_btf()
3603 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. in bpf_object__sanitize_and_load_btf()
3608 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, in bpf_object__sanitize_and_load_btf()
3609 obj->log_level ? 1 : 0, obj->token_fd); in bpf_object__sanitize_and_load_btf()
3614 btf__set_fd(obj->btf, btf__fd(kern_btf)); in bpf_object__sanitize_and_load_btf()
3615 btf__set_fd(kern_btf, -1); in bpf_object__sanitize_and_load_btf()
3638 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); in elf_sym_str()
3641 off, obj->path, elf_errmsg(-1)); in elf_sym_str()
3652 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); in elf_sec_str()
3655 off, obj->path, elf_errmsg(-1)); in elf_sec_str()
3666 scn = elf_getscn(obj->efile.elf, idx); in elf_sec_by_idx()
3669 idx, obj->path, elf_errmsg(-1)); in elf_sec_by_idx()
3678 Elf *elf = obj->efile.elf; in elf_sec_by_name()
3704 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); in elf_sec_hdr()
3723 name = elf_sec_str(obj, sh->sh_name); in elf_sec_name()
3726 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); in elf_sec_name()
3744 obj->path, elf_errmsg(-1)); in elf_sec_data()
3753 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) in elf_sym_by_idx()
3756 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; in elf_sym_by_idx()
3761 if (idx >= data->d_size / sizeof(Elf64_Rel)) in elf_rel_by_idx()
3764 return (Elf64_Rel *)data->d_buf + idx; in elf_rel_by_idx()
3776 if (hdr->sh_type == SHT_STRTAB) in ignore_elf_section()
3780 if (hdr->sh_type == SHT_LLVM_ADDRSIG) in ignore_elf_section()
3784 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && in ignore_elf_section()
3793 name += sizeof(".rel") - 1; in ignore_elf_section()
3798 /* .BTF and .BTF.ext don't need relocations */ in ignore_elf_section()
3812 if (a->sec_idx != b->sec_idx) in cmp_progs()
3813 return a->sec_idx < b->sec_idx ? -1 : 1; in cmp_progs()
3816 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; in cmp_progs()
3822 Elf *elf = obj->efile.elf; in bpf_object__elf_collect()
3831 /* ELF section indices are 0-based, but sec #0 is special "invalid" in bpf_object__elf_collect()
3836 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { in bpf_object__elf_collect()
3838 obj->path, elf_errmsg(-1)); in bpf_object__elf_collect()
3839 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3841 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); in bpf_object__elf_collect()
3842 if (!obj->efile.secs) in bpf_object__elf_collect()
3843 return -ENOMEM; in bpf_object__elf_collect()
3852 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3854 if (sh->sh_type == SHT_SYMTAB) { in bpf_object__elf_collect()
3855 if (obj->efile.symbols) { in bpf_object__elf_collect()
3856 pr_warn("elf: multiple symbol tables in %s\n", obj->path); in bpf_object__elf_collect()
3857 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3862 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3866 obj->efile.symbols = data; in bpf_object__elf_collect()
3867 obj->efile.symbols_shndx = idx; in bpf_object__elf_collect()
3868 obj->efile.strtabidx = sh->sh_link; in bpf_object__elf_collect()
3872 if (!obj->efile.symbols) { in bpf_object__elf_collect()
3874 obj->path); in bpf_object__elf_collect()
3875 return -ENOENT; in bpf_object__elf_collect()
3881 sec_desc = &obj->efile.secs[idx]; in bpf_object__elf_collect()
3885 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3887 name = elf_sec_str(obj, sh->sh_name); in bpf_object__elf_collect()
3889 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3896 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3899 idx, name, (unsigned long)data->d_size, in bpf_object__elf_collect()
3900 (int)sh->sh_link, (unsigned long)sh->sh_flags, in bpf_object__elf_collect()
3901 (int)sh->sh_type); in bpf_object__elf_collect()
3904 err = bpf_object__init_license(obj, data->d_buf, data->d_size); in bpf_object__elf_collect()
3908 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); in bpf_object__elf_collect()
3913 return -ENOTSUP; in bpf_object__elf_collect()
3915 obj->efile.btf_maps_shndx = idx; in bpf_object__elf_collect()
3917 if (sh->sh_type != SHT_PROGBITS) in bpf_object__elf_collect()
3918 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3921 if (sh->sh_type != SHT_PROGBITS) in bpf_object__elf_collect()
3922 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3924 } else if (sh->sh_type == SHT_SYMTAB) { in bpf_object__elf_collect()
3926 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) { in bpf_object__elf_collect()
3927 if (sh->sh_flags & SHF_EXECINSTR) { in bpf_object__elf_collect()
3929 obj->efile.text_shndx = idx; in bpf_object__elf_collect()
3935 sec_desc->sec_type = SEC_DATA; in bpf_object__elf_collect()
3936 sec_desc->shdr = sh; in bpf_object__elf_collect()
3937 sec_desc->data = data; in bpf_object__elf_collect()
3940 sec_desc->sec_type = SEC_RODATA; in bpf_object__elf_collect()
3941 sec_desc->shdr = sh; in bpf_object__elf_collect()
3942 sec_desc->data = data; in bpf_object__elf_collect()
3947 sec_desc->sec_type = SEC_ST_OPS; in bpf_object__elf_collect()
3948 sec_desc->shdr = sh; in bpf_object__elf_collect()
3949 sec_desc->data = data; in bpf_object__elf_collect()
3950 obj->efile.has_st_ops = true; in bpf_object__elf_collect()
3952 obj->efile.arena_data = data; in bpf_object__elf_collect()
3953 obj->efile.arena_data_shndx = idx; in bpf_object__elf_collect()
3958 } else if (sh->sh_type == SHT_REL) { in bpf_object__elf_collect()
3959 int targ_sec_idx = sh->sh_info; /* points to other section */ in bpf_object__elf_collect()
3961 if (sh->sh_entsize != sizeof(Elf64_Rel) || in bpf_object__elf_collect()
3962 targ_sec_idx >= obj->efile.sec_cnt) in bpf_object__elf_collect()
3963 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3978 sec_desc->sec_type = SEC_RELO; in bpf_object__elf_collect()
3979 sec_desc->shdr = sh; in bpf_object__elf_collect()
3980 sec_desc->data = data; in bpf_object__elf_collect()
3981 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 || in bpf_object__elf_collect()
3983 sec_desc->sec_type = SEC_BSS; in bpf_object__elf_collect()
3984 sec_desc->shdr = sh; in bpf_object__elf_collect()
3985 sec_desc->data = data; in bpf_object__elf_collect()
3988 (size_t)sh->sh_size); in bpf_object__elf_collect()
3992 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { in bpf_object__elf_collect()
3993 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); in bpf_object__elf_collect()
3994 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
4001 /* sort BPF programs by section name and in-section instruction offset in bpf_object__elf_collect()
4004 if (obj->nr_programs) in bpf_object__elf_collect()
4005 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); in bpf_object__elf_collect()
4012 int bind = ELF64_ST_BIND(sym->st_info); in sym_is_extern()
4014 return sym->st_shndx == SHN_UNDEF && in sym_is_extern()
4016 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE; in sym_is_extern()
4021 int bind = ELF64_ST_BIND(sym->st_info); in sym_is_subprog()
4022 int type = ELF64_ST_TYPE(sym->st_info); in sym_is_subprog()
4025 if (sym->st_shndx != text_shndx) in sym_is_subprog()
4043 return -ESRCH; in find_extern_btf_id()
4052 tname = btf__name_by_offset(btf, t->name_off); in find_extern_btf_id()
4057 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) in find_extern_btf_id()
4058 return -EINVAL; in find_extern_btf_id()
4061 return -EINVAL; in find_extern_btf_id()
4066 return -ENOENT; in find_extern_btf_id()
4075 return -ESRCH; in find_extern_sec_btf_id()
4086 if (vs->type == ext_btf_id) in find_extern_sec_btf_id()
4091 return -ENOENT; in find_extern_sec_btf_id()
4101 name = btf__name_by_offset(btf, t->name_off); in find_kcfg_type()
4110 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; in find_kcfg_type()
4113 if (t->size == 1) in find_kcfg_type()
4115 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) in find_kcfg_type()
4120 if (t->size != 4) in find_kcfg_type()
4130 if (btf_array(t)->nelems == 0) in find_kcfg_type()
4132 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) in find_kcfg_type()
4145 if (a->type != b->type) in cmp_externs()
4146 return a->type < b->type ? -1 : 1; in cmp_externs()
4148 if (a->type == EXT_KCFG) { in cmp_externs()
4150 if (a->kcfg.align != b->kcfg.align) in cmp_externs()
4151 return a->kcfg.align > b->kcfg.align ? -1 : 1; in cmp_externs()
4153 if (a->kcfg.sz != b->kcfg.sz) in cmp_externs()
4154 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; in cmp_externs()
4158 return strcmp(a->name, b->name); in cmp_externs()
4196 vt = btf__type_by_id(btf, vs->type); in add_dummy_ksym_var()
4220 struct extern_desc *ext; in bpf_object__collect_externs() local
4227 if (!obj->efile.symbols) in bpf_object__collect_externs()
4230 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); in bpf_object__collect_externs()
4232 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) in bpf_object__collect_externs()
4233 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_externs()
4235 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); in bpf_object__collect_externs()
4239 n = sh->sh_size / sh->sh_entsize; in bpf_object__collect_externs()
4246 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_externs()
4249 ext_name = elf_sym_str(obj, sym->st_name); in bpf_object__collect_externs()
4253 ext = obj->externs; in bpf_object__collect_externs()
4254 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); in bpf_object__collect_externs()
4255 if (!ext) in bpf_object__collect_externs()
4256 return -ENOMEM; in bpf_object__collect_externs()
4257 obj->externs = ext; in bpf_object__collect_externs()
4258 ext = &ext[obj->nr_extern]; in bpf_object__collect_externs()
4259 memset(ext, 0, sizeof(*ext)); in bpf_object__collect_externs()
4260 obj->nr_extern++; in bpf_object__collect_externs()
4262 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); in bpf_object__collect_externs()
4263 if (ext->btf_id <= 0) { in bpf_object__collect_externs()
4265 ext_name, ext->btf_id); in bpf_object__collect_externs()
4266 return ext->btf_id; in bpf_object__collect_externs()
4268 t = btf__type_by_id(obj->btf, ext->btf_id); in bpf_object__collect_externs()
4269 ext->name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__collect_externs()
4270 ext->sym_idx = i; in bpf_object__collect_externs()
4271 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; in bpf_object__collect_externs()
4273 ext_essent_len = bpf_core_essential_name_len(ext->name); in bpf_object__collect_externs()
4274 ext->essent_name = NULL; in bpf_object__collect_externs()
4275 if (ext_essent_len != strlen(ext->name)) { in bpf_object__collect_externs()
4276 ext->essent_name = strndup(ext->name, ext_essent_len); in bpf_object__collect_externs()
4277 if (!ext->essent_name) in bpf_object__collect_externs()
4278 return -ENOMEM; in bpf_object__collect_externs()
4281 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); in bpf_object__collect_externs()
4282 if (ext->sec_btf_id <= 0) { in bpf_object__collect_externs()
4284 ext_name, ext->btf_id, ext->sec_btf_id); in bpf_object__collect_externs()
4285 return ext->sec_btf_id; in bpf_object__collect_externs()
4287 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); in bpf_object__collect_externs()
4288 sec_name = btf__name_by_offset(obj->btf, sec->name_off); in bpf_object__collect_externs()
4293 ext->name, KCONFIG_SEC); in bpf_object__collect_externs()
4294 return -ENOTSUP; in bpf_object__collect_externs()
4297 ext->type = EXT_KCFG; in bpf_object__collect_externs()
4298 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); in bpf_object__collect_externs()
4299 if (ext->kcfg.sz <= 0) { in bpf_object__collect_externs()
4301 ext_name, ext->kcfg.sz); in bpf_object__collect_externs()
4302 return ext->kcfg.sz; in bpf_object__collect_externs()
4304 ext->kcfg.align = btf__align_of(obj->btf, t->type); in bpf_object__collect_externs()
4305 if (ext->kcfg.align <= 0) { in bpf_object__collect_externs()
4307 ext_name, ext->kcfg.align); in bpf_object__collect_externs()
4308 return -EINVAL; in bpf_object__collect_externs()
4310 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, in bpf_object__collect_externs()
4311 &ext->kcfg.is_signed); in bpf_object__collect_externs()
4312 if (ext->kcfg.type == KCFG_UNKNOWN) { in bpf_object__collect_externs()
4314 return -ENOTSUP; in bpf_object__collect_externs()
4318 ext->type = EXT_KSYM; in bpf_object__collect_externs()
4319 skip_mods_and_typedefs(obj->btf, t->type, in bpf_object__collect_externs()
4320 &ext->ksym.type_id); in bpf_object__collect_externs()
4323 return -ENOTSUP; in bpf_object__collect_externs()
4326 pr_debug("collected %d externs total\n", obj->nr_extern); in bpf_object__collect_externs()
4328 if (!obj->nr_extern) in bpf_object__collect_externs()
4332 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); in bpf_object__collect_externs()
4336 * pretending that each extern is a 8-byte variable in bpf_object__collect_externs()
4339 /* find existing 4-byte integer type in BTF to use for fake in bpf_object__collect_externs()
4342 int int_btf_id = find_int_btf_id(obj->btf); in bpf_object__collect_externs()
4344 * will be used to replace the vs->type and in bpf_object__collect_externs()
4350 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); in bpf_object__collect_externs()
4351 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__collect_externs()
4352 ext = &obj->externs[i]; in bpf_object__collect_externs()
4353 if (ext->type != EXT_KSYM) in bpf_object__collect_externs()
4356 i, ext->sym_idx, ext->name); in bpf_object__collect_externs()
4365 vt = (void *)btf__type_by_id(obj->btf, vs->type); in bpf_object__collect_externs()
4366 ext_name = btf__name_by_offset(obj->btf, vt->name_off); in bpf_object__collect_externs()
4367 ext = find_extern_by_name(obj, ext_name); in bpf_object__collect_externs()
4368 if (!ext) { in bpf_object__collect_externs()
4371 return -ESRCH; in bpf_object__collect_externs()
4378 func_proto = btf__type_by_id(obj->btf, in bpf_object__collect_externs()
4379 vt->type); in bpf_object__collect_externs()
4387 dummy_var->name_off; in bpf_object__collect_externs()
4388 vs->type = dummy_var_btf_id; in bpf_object__collect_externs()
4389 vt->info &= ~0xffff; in bpf_object__collect_externs()
4390 vt->info |= BTF_FUNC_GLOBAL; in bpf_object__collect_externs()
4392 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; in bpf_object__collect_externs()
4393 vt->type = int_btf_id; in bpf_object__collect_externs()
4395 vs->offset = off; in bpf_object__collect_externs()
4396 vs->size = sizeof(int); in bpf_object__collect_externs()
4398 sec->size = off; in bpf_object__collect_externs()
4405 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__collect_externs()
4406 ext = &obj->externs[i]; in bpf_object__collect_externs()
4407 if (ext->type != EXT_KCFG) in bpf_object__collect_externs()
4410 ext->kcfg.data_off = roundup(off, ext->kcfg.align); in bpf_object__collect_externs()
4411 off = ext->kcfg.data_off + ext->kcfg.sz; in bpf_object__collect_externs()
4413 i, ext->sym_idx, ext->kcfg.data_off, ext->name); in bpf_object__collect_externs()
4415 sec->size = off; in bpf_object__collect_externs()
4420 t = btf__type_by_id(obj->btf, vs->type); in bpf_object__collect_externs()
4421 ext_name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__collect_externs()
4422 ext = find_extern_by_name(obj, ext_name); in bpf_object__collect_externs()
4423 if (!ext) { in bpf_object__collect_externs()
4426 return -ESRCH; in bpf_object__collect_externs()
4428 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; in bpf_object__collect_externs()
4429 vs->offset = ext->kcfg.data_off; in bpf_object__collect_externs()
4437 return prog->sec_idx == obj->efile.text_shndx; in prog_is_subprog()
4449 if (!strcmp(prog->name, name)) in bpf_object__find_program_by_name()
4458 switch (obj->efile.secs[shndx].sec_type) { in bpf_object__shndx_is_data()
4471 return shndx == obj->efile.btf_maps_shndx; in bpf_object__shndx_is_maps()
4477 if (shndx == obj->efile.symbols_shndx) in bpf_object__section_to_libbpf_map_type()
4480 switch (obj->efile.secs[shndx].sec_type) { in bpf_object__section_to_libbpf_map_type()
4497 struct bpf_insn *insn = &prog->insns[insn_idx]; in bpf_program__record_reloc()
4498 size_t map_idx, nr_maps = prog->obj->nr_maps; in bpf_program__record_reloc()
4499 struct bpf_object *obj = prog->obj; in bpf_program__record_reloc()
4500 __u32 shdr_idx = sym->st_shndx; in bpf_program__record_reloc()
4507 prog->name, sym_name, insn_idx, insn->code); in bpf_program__record_reloc()
4508 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4512 int sym_idx = ELF64_R_SYM(rel->r_info); in bpf_program__record_reloc()
4513 int i, n = obj->nr_extern; in bpf_program__record_reloc()
4514 struct extern_desc *ext; in bpf_program__record_reloc() local
4517 ext = &obj->externs[i]; in bpf_program__record_reloc()
4518 if (ext->sym_idx == sym_idx) in bpf_program__record_reloc()
4523 prog->name, sym_name, sym_idx); in bpf_program__record_reloc()
4524 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4527 prog->name, i, ext->name, ext->sym_idx, insn_idx); in bpf_program__record_reloc()
4528 if (insn->code == (BPF_JMP | BPF_CALL)) in bpf_program__record_reloc()
4529 reloc_desc->type = RELO_EXTERN_CALL; in bpf_program__record_reloc()
4531 reloc_desc->type = RELO_EXTERN_LD64; in bpf_program__record_reloc()
4532 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4533 reloc_desc->ext_idx = i; in bpf_program__record_reloc()
4537 /* sub-program call relocation */ in bpf_program__record_reloc()
4539 if (insn->src_reg != BPF_PSEUDO_CALL) { in bpf_program__record_reloc()
4540 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); in bpf_program__record_reloc()
4541 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4544 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { in bpf_program__record_reloc()
4547 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4548 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4550 if (sym->st_value % BPF_INSN_SZ) { in bpf_program__record_reloc()
4552 prog->name, sym_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4553 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4555 reloc_desc->type = RELO_CALL; in bpf_program__record_reloc()
4556 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4557 reloc_desc->sym_off = sym->st_value; in bpf_program__record_reloc()
4563 prog->name, sym_name, shdr_idx); in bpf_program__record_reloc()
4564 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4568 if (sym_is_subprog(sym, obj->efile.text_shndx)) { in bpf_program__record_reloc()
4569 /* global_func: sym->st_value = offset in the section, insn->imm = 0. in bpf_program__record_reloc()
4570 * local_func: sym->st_value = 0, insn->imm = offset in the section. in bpf_program__record_reloc()
4572 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { in bpf_program__record_reloc()
4574 prog->name, sym_name, (size_t)sym->st_value, insn->imm); in bpf_program__record_reloc()
4575 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4578 reloc_desc->type = RELO_SUBPROG_ADDR; in bpf_program__record_reloc()
4579 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4580 reloc_desc->sym_off = sym->st_value; in bpf_program__record_reloc()
4588 if (shdr_idx == obj->efile.arena_data_shndx) { in bpf_program__record_reloc()
4589 reloc_desc->type = RELO_DATA; in bpf_program__record_reloc()
4590 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4591 reloc_desc->map_idx = obj->arena_map - obj->maps; in bpf_program__record_reloc()
4592 reloc_desc->sym_off = sym->st_value; in bpf_program__record_reloc()
4600 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4601 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4604 map = &obj->maps[map_idx]; in bpf_program__record_reloc()
4605 if (map->libbpf_type != type || in bpf_program__record_reloc()
4606 map->sec_idx != sym->st_shndx || in bpf_program__record_reloc()
4607 map->sec_offset != sym->st_value) in bpf_program__record_reloc()
4610 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4611 map->sec_offset, insn_idx); in bpf_program__record_reloc()
4616 prog->name, sym_sec_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4617 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4619 reloc_desc->type = RELO_LD64; in bpf_program__record_reloc()
4620 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4621 reloc_desc->map_idx = map_idx; in bpf_program__record_reloc()
4622 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ in bpf_program__record_reloc()
4629 prog->name, sym_sec_name); in bpf_program__record_reloc()
4630 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4633 map = &obj->maps[map_idx]; in bpf_program__record_reloc()
4634 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) in bpf_program__record_reloc()
4637 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4638 map->sec_offset, insn_idx); in bpf_program__record_reloc()
4643 prog->name, sym_sec_name); in bpf_program__record_reloc()
4644 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4647 reloc_desc->type = RELO_DATA; in bpf_program__record_reloc()
4648 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4649 reloc_desc->map_idx = map_idx; in bpf_program__record_reloc()
4650 reloc_desc->sym_off = sym->st_value; in bpf_program__record_reloc()
4656 return insn_idx >= prog->sec_insn_off && in prog_contains_insn()
4657 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; in prog_contains_insn()
4663 int l = 0, r = obj->nr_programs - 1, m; in find_prog_by_sec_insn()
4666 if (!obj->nr_programs) in find_prog_by_sec_insn()
4670 m = l + (r - l + 1) / 2; in find_prog_by_sec_insn()
4671 prog = &obj->programs[m]; in find_prog_by_sec_insn()
4673 if (prog->sec_idx < sec_idx || in find_prog_by_sec_insn()
4674 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) in find_prog_by_sec_insn()
4677 r = m - 1; in find_prog_by_sec_insn()
4682 prog = &obj->programs[l]; in find_prog_by_sec_insn()
4683 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) in find_prog_by_sec_insn()
4692 size_t sec_idx = shdr->sh_info, sym_idx; in bpf_object__collect_prog_relos()
4703 if (sec_idx >= obj->efile.sec_cnt) in bpf_object__collect_prog_relos()
4704 return -EINVAL; in bpf_object__collect_prog_relos()
4709 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4711 relo_sec_name = elf_sec_str(obj, shdr->sh_name); in bpf_object__collect_prog_relos()
4714 return -EINVAL; in bpf_object__collect_prog_relos()
4718 nrels = shdr->sh_size / shdr->sh_entsize; in bpf_object__collect_prog_relos()
4724 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4727 sym_idx = ELF64_R_SYM(rel->r_info); in bpf_object__collect_prog_relos()
4732 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4735 if (sym->st_shndx >= obj->efile.sec_cnt) { in bpf_object__collect_prog_relos()
4737 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); in bpf_object__collect_prog_relos()
4738 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4741 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { in bpf_object__collect_prog_relos()
4743 relo_sec_name, (size_t)rel->r_offset, i); in bpf_object__collect_prog_relos()
4744 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4747 insn_idx = rel->r_offset / BPF_INSN_SZ; in bpf_object__collect_prog_relos()
4754 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0) in bpf_object__collect_prog_relos()
4755 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); in bpf_object__collect_prog_relos()
4757 sym_name = elf_sym_str(obj, sym->st_name); in bpf_object__collect_prog_relos()
4770 relos = libbpf_reallocarray(prog->reloc_desc, in bpf_object__collect_prog_relos()
4771 prog->nr_reloc + 1, sizeof(*relos)); in bpf_object__collect_prog_relos()
4773 return -ENOMEM; in bpf_object__collect_prog_relos()
4774 prog->reloc_desc = relos; in bpf_object__collect_prog_relos()
4777 insn_idx -= prog->sec_insn_off; in bpf_object__collect_prog_relos()
4778 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], in bpf_object__collect_prog_relos()
4783 prog->nr_reloc++; in bpf_object__collect_prog_relos()
4792 if (!obj->btf) in map_fill_btf_type_info()
4793 return -ENOENT; in map_fill_btf_type_info()
4795 /* if it's BTF-defined map, we don't need to search for type IDs. in map_fill_btf_type_info()
4799 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) in map_fill_btf_type_info()
4807 return -ENOENT; in map_fill_btf_type_info()
4809 id = btf__find_by_name(obj->btf, map->real_name); in map_fill_btf_type_info()
4813 map->btf_key_type_id = 0; in map_fill_btf_type_info()
4814 map->btf_value_type_id = id; in map_fill_btf_type_info()
4830 err = -errno; in bpf_get_map_info_from_fdinfo()
4838 info->type = val; in bpf_get_map_info_from_fdinfo()
4840 info->key_size = val; in bpf_get_map_info_from_fdinfo()
4842 info->value_size = val; in bpf_get_map_info_from_fdinfo()
4844 info->max_entries = val; in bpf_get_map_info_from_fdinfo()
4846 info->map_flags = val; in bpf_get_map_info_from_fdinfo()
4856 return map->obj->state >= OBJ_PREPARED || map->reused; in map_is_created()
4861 return map->autocreate; in bpf_map__autocreate()
4867 return libbpf_err(-EBUSY); in bpf_map__set_autocreate()
4869 map->autocreate = autocreate; in bpf_map__set_autocreate()
4876 return libbpf_err(-EINVAL); in bpf_map__set_autoattach()
4878 map->autoattach = autoattach; in bpf_map__set_autoattach()
4884 return map->autoattach; in bpf_map__autoattach()
4902 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) in bpf_map__reuse_fd()
4903 new_name = strdup(map->name); in bpf_map__reuse_fd()
4908 return libbpf_err(-errno); in bpf_map__reuse_fd()
4917 err = -errno; in bpf_map__reuse_fd()
4921 err = reuse_fd(map->fd, new_fd); in bpf_map__reuse_fd()
4925 free(map->name); in bpf_map__reuse_fd()
4927 map->name = new_name; in bpf_map__reuse_fd()
4928 map->def.type = info.type; in bpf_map__reuse_fd()
4929 map->def.key_size = info.key_size; in bpf_map__reuse_fd()
4930 map->def.value_size = info.value_size; in bpf_map__reuse_fd()
4931 map->def.max_entries = info.max_entries; in bpf_map__reuse_fd()
4932 map->def.map_flags = info.map_flags; in bpf_map__reuse_fd()
4933 map->btf_key_type_id = info.btf_key_type_id; in bpf_map__reuse_fd()
4934 map->btf_value_type_id = info.btf_value_type_id; in bpf_map__reuse_fd()
4935 map->reused = true; in bpf_map__reuse_fd()
4936 map->map_extra = info.map_extra; in bpf_map__reuse_fd()
4947 return map->def.max_entries; in bpf_map__max_entries()
4952 if (!bpf_map_type__is_map_in_map(map->def.type)) in bpf_map__inner_map()
4955 return map->inner_map; in bpf_map__inner_map()
4961 return libbpf_err(-EBUSY); in bpf_map__set_max_entries()
4963 map->def.max_entries = max_entries; in bpf_map__set_max_entries()
4965 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ in bpf_map__set_max_entries()
4967 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); in bpf_map__set_max_entries()
4975 int bpffs_fd = -1, token_fd, err; in bpf_object_prepare_token()
4980 if (obj->token_path && obj->token_path[0] == '\0') { in bpf_object_prepare_token()
4981 pr_debug("object '%s': token is prevented, skipping...\n", obj->name); in bpf_object_prepare_token()
4985 mandatory = obj->token_path != NULL; in bpf_object_prepare_token()
4988 bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH; in bpf_object_prepare_token()
4991 err = -errno; in bpf_object_prepare_token()
4993 obj->name, errstr(err), bpffs_path, in bpf_object_prepare_token()
5001 if (!mandatory && token_fd == -ENOENT) { in bpf_object_prepare_token()
5003 obj->name, bpffs_path); in bpf_object_prepare_token()
5007 obj->name, token_fd, bpffs_path, in bpf_object_prepare_token()
5012 obj->feat_cache = calloc(1, sizeof(*obj->feat_cache)); in bpf_object_prepare_token()
5013 if (!obj->feat_cache) { in bpf_object_prepare_token()
5015 return -ENOMEM; in bpf_object_prepare_token()
5018 obj->token_fd = token_fd; in bpf_object_prepare_token()
5019 obj->feat_cache->token_fd = token_fd; in bpf_object_prepare_token()
5033 .token_fd = obj->token_fd, in bpf_object__probe_loading()
5034 .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, in bpf_object__probe_loading()
5037 if (obj->gen_loader) in bpf_object__probe_loading()
5053 return -ret; in bpf_object__probe_loading()
5062 if (obj->gen_loader) in kernel_supports()
5068 if (obj->token_fd) in kernel_supports()
5069 return feat_supported(obj->feat_cache, feat_id); in kernel_supports()
5090 return (map_info.type == map->def.type && in map_is_reuse_compat()
5091 map_info.key_size == map->def.key_size && in map_is_reuse_compat()
5092 map_info.value_size == map->def.value_size && in map_is_reuse_compat()
5093 map_info.max_entries == map->def.max_entries && in map_is_reuse_compat()
5094 map_info.map_flags == map->def.map_flags && in map_is_reuse_compat()
5095 map_info.map_extra == map->map_extra); in map_is_reuse_compat()
5103 pin_fd = bpf_obj_get(map->pin_path); in bpf_object__reuse_map()
5105 err = -errno; in bpf_object__reuse_map()
5106 if (err == -ENOENT) { in bpf_object__reuse_map()
5108 map->pin_path); in bpf_object__reuse_map()
5113 map->pin_path, errstr(err)); in bpf_object__reuse_map()
5119 map->pin_path); in bpf_object__reuse_map()
5121 return -EINVAL; in bpf_object__reuse_map()
5129 map->pinned = true; in bpf_object__reuse_map()
5130 pr_debug("reused pinned map at '%s'\n", map->pin_path); in bpf_object__reuse_map()
5138 enum libbpf_map_type map_type = map->libbpf_type; in bpf_object__populate_internal_map()
5142 if (obj->gen_loader) { in bpf_object__populate_internal_map()
5143 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, in bpf_object__populate_internal_map()
5144 map->mmaped, map->def.value_size); in bpf_object__populate_internal_map()
5146 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); in bpf_object__populate_internal_map()
5150 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); in bpf_object__populate_internal_map()
5152 err = -errno; in bpf_object__populate_internal_map()
5158 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ in bpf_object__populate_internal_map()
5160 err = bpf_map_freeze(map->fd); in bpf_object__populate_internal_map()
5162 err = -errno; in bpf_object__populate_internal_map()
5163 pr_warn("map '%s': failed to freeze as read-only: %s\n", in bpf_object__populate_internal_map()
5169 /* Remap anonymous mmap()-ed "map initialization image" as in bpf_object__populate_internal_map()
5170 * a BPF map-backed mmap()-ed memory, but preserving the same in bpf_object__populate_internal_map()
5179 if (map->def.map_flags & BPF_F_MMAPABLE) { in bpf_object__populate_internal_map()
5183 if (map->def.map_flags & BPF_F_RDONLY_PROG) in bpf_object__populate_internal_map()
5187 mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map->fd, 0); in bpf_object__populate_internal_map()
5189 err = -errno; in bpf_object__populate_internal_map()
5190 pr_warn("map '%s': failed to re-mmap() contents: %s\n", in bpf_object__populate_internal_map()
5194 map->mmaped = mmaped; in bpf_object__populate_internal_map()
5195 } else if (map->mmaped) { in bpf_object__populate_internal_map()
5196 munmap(map->mmaped, mmap_sz); in bpf_object__populate_internal_map()
5197 map->mmaped = NULL; in bpf_object__populate_internal_map()
5208 struct bpf_map_def *def = &map->def; in bpf_object__create_map()
5213 map_name = map->name; in bpf_object__create_map()
5214 create_attr.map_ifindex = map->map_ifindex; in bpf_object__create_map()
5215 create_attr.map_flags = def->map_flags; in bpf_object__create_map()
5216 create_attr.numa_node = map->numa_node; in bpf_object__create_map()
5217 create_attr.map_extra = map->map_extra; in bpf_object__create_map()
5218 create_attr.token_fd = obj->token_fd; in bpf_object__create_map()
5219 if (obj->token_fd) in bpf_object__create_map()
5223 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; in bpf_object__create_map()
5224 if (map->mod_btf_fd >= 0) { in bpf_object__create_map()
5225 create_attr.value_type_btf_obj_fd = map->mod_btf_fd; in bpf_object__create_map()
5230 if (obj->btf && btf__fd(obj->btf) >= 0) { in bpf_object__create_map()
5231 create_attr.btf_fd = btf__fd(obj->btf); in bpf_object__create_map()
5232 create_attr.btf_key_type_id = map->btf_key_type_id; in bpf_object__create_map()
5233 create_attr.btf_value_type_id = map->btf_value_type_id; in bpf_object__create_map()
5236 if (bpf_map_type__is_map_in_map(def->type)) { in bpf_object__create_map()
5237 if (map->inner_map) { in bpf_object__create_map()
5238 err = map_set_def_max_entries(map->inner_map); in bpf_object__create_map()
5241 err = bpf_object__create_map(obj, map->inner_map, true); in bpf_object__create_map()
5244 map->name, errstr(err)); in bpf_object__create_map()
5247 map->inner_map_fd = map->inner_map->fd; in bpf_object__create_map()
5249 if (map->inner_map_fd >= 0) in bpf_object__create_map()
5250 create_attr.inner_map_fd = map->inner_map_fd; in bpf_object__create_map()
5253 switch (def->type) { in bpf_object__create_map()
5271 map->btf_key_type_id = 0; in bpf_object__create_map()
5272 map->btf_value_type_id = 0; in bpf_object__create_map()
5281 if (obj->gen_loader) { in bpf_object__create_map()
5282 bpf_gen__map_create(obj->gen_loader, def->type, map_name, in bpf_object__create_map()
5283 def->key_size, def->value_size, def->max_entries, in bpf_object__create_map()
5284 &create_attr, is_inner ? -1 : map - obj->maps); in bpf_object__create_map()
5289 * will be reset to -1 eventually. in bpf_object__create_map()
5291 map_fd = map->fd; in bpf_object__create_map()
5293 map_fd = bpf_map_create(def->type, map_name, in bpf_object__create_map()
5294 def->key_size, def->value_size, in bpf_object__create_map()
5295 def->max_entries, &create_attr); in bpf_object__create_map()
5298 err = -errno; in bpf_object__create_map()
5300 map->name, errstr(err)); in bpf_object__create_map()
5304 map->btf_key_type_id = 0; in bpf_object__create_map()
5305 map->btf_value_type_id = 0; in bpf_object__create_map()
5306 map_fd = bpf_map_create(def->type, map_name, in bpf_object__create_map()
5307 def->key_size, def->value_size, in bpf_object__create_map()
5308 def->max_entries, &create_attr); in bpf_object__create_map()
5311 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { in bpf_object__create_map()
5312 if (obj->gen_loader) in bpf_object__create_map()
5313 map->inner_map->fd = -1; in bpf_object__create_map()
5314 bpf_map__destroy(map->inner_map); in bpf_object__create_map()
5315 zfree(&map->inner_map); in bpf_object__create_map()
5321 /* obj->gen_loader case, prevent reuse_fd() from closing map_fd */ in bpf_object__create_map()
5322 if (map->fd == map_fd) in bpf_object__create_map()
5328 * map->fd stays valid but now point to what map_fd points to. in bpf_object__create_map()
5330 return reuse_fd(map->fd, map_fd); in bpf_object__create_map()
5339 for (i = 0; i < map->init_slots_sz; i++) { in init_map_in_map_slots()
5340 if (!map->init_slots[i]) in init_map_in_map_slots()
5343 targ_map = map->init_slots[i]; in init_map_in_map_slots()
5344 fd = targ_map->fd; in init_map_in_map_slots()
5346 if (obj->gen_loader) { in init_map_in_map_slots()
5347 bpf_gen__populate_outer_map(obj->gen_loader, in init_map_in_map_slots()
5348 map - obj->maps, i, in init_map_in_map_slots()
5349 targ_map - obj->maps); in init_map_in_map_slots()
5351 err = bpf_map_update_elem(map->fd, &i, &fd, 0); in init_map_in_map_slots()
5354 err = -errno; in init_map_in_map_slots()
5356 map->name, i, targ_map->name, fd, errstr(err)); in init_map_in_map_slots()
5360 map->name, i, targ_map->name, fd); in init_map_in_map_slots()
5363 zfree(&map->init_slots); in init_map_in_map_slots()
5364 map->init_slots_sz = 0; in init_map_in_map_slots()
5375 if (obj->gen_loader) in init_prog_array_slots()
5376 return -ENOTSUP; in init_prog_array_slots()
5378 for (i = 0; i < map->init_slots_sz; i++) { in init_prog_array_slots()
5379 if (!map->init_slots[i]) in init_prog_array_slots()
5382 targ_prog = map->init_slots[i]; in init_prog_array_slots()
5385 err = bpf_map_update_elem(map->fd, &i, &fd, 0); in init_prog_array_slots()
5387 err = -errno; in init_prog_array_slots()
5389 map->name, i, targ_prog->name, fd, errstr(err)); in init_prog_array_slots()
5393 map->name, i, targ_prog->name, fd); in init_prog_array_slots()
5396 zfree(&map->init_slots); in init_prog_array_slots()
5397 map->init_slots_sz = 0; in init_prog_array_slots()
5407 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_init_prog_arrays()
5408 map = &obj->maps[i]; in bpf_object_init_prog_arrays()
5410 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY) in bpf_object_init_prog_arrays()
5422 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) { in map_set_def_max_entries()
5428 map->name, nr_cpus); in map_set_def_max_entries()
5431 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); in map_set_def_max_entries()
5432 map->def.max_entries = nr_cpus; in map_set_def_max_entries()
5446 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__create_maps()
5447 map = &obj->maps[i]; in bpf_object__create_maps()
5451 * loading, if we detect that at least one of the to-be-loaded in bpf_object__create_maps()
5456 * but also it allows to have CO-RE applications that use in bpf_object__create_maps()
5458 * If those global variable-using programs are not loaded at in bpf_object__create_maps()
5464 map->autocreate = false; in bpf_object__create_maps()
5466 if (!map->autocreate) { in bpf_object__create_maps()
5467 pr_debug("map '%s': skipped auto-creating...\n", map->name); in bpf_object__create_maps()
5477 if (map->pin_path) { in bpf_object__create_maps()
5481 map->name); in bpf_object__create_maps()
5484 if (retried && map->fd < 0) { in bpf_object__create_maps()
5486 map->name); in bpf_object__create_maps()
5487 err = -ENOENT; in bpf_object__create_maps()
5492 if (map->reused) { in bpf_object__create_maps()
5494 map->name, map->fd); in bpf_object__create_maps()
5501 map->name, map->fd); in bpf_object__create_maps()
5507 } else if (map->def.type == BPF_MAP_TYPE_ARENA) { in bpf_object__create_maps()
5508 map->mmaped = mmap((void *)(long)map->map_extra, in bpf_object__create_maps()
5510 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, in bpf_object__create_maps()
5511 map->fd, 0); in bpf_object__create_maps()
5512 if (map->mmaped == MAP_FAILED) { in bpf_object__create_maps()
5513 err = -errno; in bpf_object__create_maps()
5514 map->mmaped = NULL; in bpf_object__create_maps()
5516 map->name, errstr(err)); in bpf_object__create_maps()
5519 if (obj->arena_data) { in bpf_object__create_maps()
5520 memcpy(map->mmaped, obj->arena_data, obj->arena_data_sz); in bpf_object__create_maps()
5521 zfree(&obj->arena_data); in bpf_object__create_maps()
5524 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { in bpf_object__create_maps()
5531 if (map->pin_path && !map->pinned) { in bpf_object__create_maps()
5534 if (!retried && err == -EEXIST) { in bpf_object__create_maps()
5538 pr_warn("map '%s': failed to auto-pin at '%s': %s\n", in bpf_object__create_maps()
5539 map->name, map->pin_path, errstr(err)); in bpf_object__create_maps()
5548 pr_warn("map '%s': failed to create: %s\n", map->name, errstr(err)); in bpf_object__create_maps()
5551 zclose(obj->maps[j].fd); in bpf_object__create_maps()
5565 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5572 for (i = n - 5; i >= 0; i--) { in bpf_core_essential_name_len()
5584 free(cands->cands); in bpf_core_free_cands()
5601 local_t = btf__type_by_id(local_cand->btf, local_cand->id); in bpf_core_add_cands()
5602 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); in bpf_core_add_cands()
5610 targ_name = btf__name_by_offset(targ_btf, t->name_off); in bpf_core_add_cands()
5621 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", in bpf_core_add_cands()
5622 local_cand->id, btf_kind_str(local_t), in bpf_core_add_cands()
5625 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, in bpf_core_add_cands()
5626 sizeof(*cands->cands)); in bpf_core_add_cands()
5628 return -ENOMEM; in bpf_core_add_cands()
5630 cand = &new_cands[cands->len]; in bpf_core_add_cands()
5631 cand->btf = targ_btf; in bpf_core_add_cands()
5632 cand->id = i; in bpf_core_add_cands()
5634 cands->cands = new_cands; in bpf_core_add_cands()
5635 cands->len++; in bpf_core_add_cands()
5649 if (obj->btf_modules_loaded) in load_module_btfs()
5652 if (obj->gen_loader) in load_module_btfs()
5656 obj->btf_modules_loaded = true; in load_module_btfs()
5671 err = -errno; in load_module_btfs()
5680 err = -errno; in load_module_btfs()
5692 err = -errno; in load_module_btfs()
5697 /* ignore non-module BTFs */ in load_module_btfs()
5703 btf = btf_get_from_fd(fd, obj->btf_vmlinux); in load_module_btfs()
5711 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, in load_module_btfs()
5712 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); in load_module_btfs()
5716 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; in load_module_btfs()
5718 mod_btf->btf = btf; in load_module_btfs()
5719 mod_btf->id = id; in load_module_btfs()
5720 mod_btf->fd = fd; in load_module_btfs()
5721 mod_btf->name = strdup(name); in load_module_btfs()
5722 if (!mod_btf->name) { in load_module_btfs()
5723 err = -ENOMEM; in load_module_btfs()
5751 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
5753 local_name = btf__name_by_offset(local_btf, local_t->name_off); in bpf_core_find_cands()
5755 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
5760 return ERR_PTR(-ENOMEM); in bpf_core_find_cands()
5763 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; in bpf_core_find_cands()
5769 if (cands->len) in bpf_core_find_cands()
5773 if (obj->btf_vmlinux_override) in bpf_core_find_cands()
5781 for (i = 0; i < obj->btf_module_cnt; i++) { in bpf_core_find_cands()
5783 obj->btf_modules[i].btf, in bpf_core_find_cands()
5784 obj->btf_modules[i].name, in bpf_core_find_cands()
5785 btf__type_cnt(obj->btf_vmlinux), in bpf_core_find_cands()
5798 * type-based CO-RE relocations and follow slightly different rules than
5799 * field-based relocations. This function assumes that root types were already
5800 * checked for name match. Beyond that initial root-level name check, names
5802 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5805 * - for ENUMs, the size is ignored;
5806 * - for INT, size and signedness are ignored;
5807 * - for ARRAY, dimensionality is ignored, element types are checked for
5809 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
5810 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5811 * - FUNC_PROTOs are compatible if they have compatible signature: same
5814 * more experience with using BPF CO-RE relocations.
5843 relos = libbpf_reallocarray(prog->reloc_desc, in record_relo_core()
5844 prog->nr_reloc + 1, sizeof(*relos)); in record_relo_core()
5846 return -ENOMEM; in record_relo_core()
5847 relo = &relos[prog->nr_reloc]; in record_relo_core()
5848 relo->type = RELO_CORE; in record_relo_core()
5849 relo->insn_idx = insn_idx; in record_relo_core()
5850 relo->core_relo = core_relo; in record_relo_core()
5851 prog->reloc_desc = relos; in record_relo_core()
5852 prog->nr_reloc++; in record_relo_core()
5861 for (i = 0; i < prog->nr_reloc; i++) { in find_relo_core()
5862 relo = &prog->reloc_desc[i]; in find_relo_core()
5863 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) in find_relo_core()
5866 return relo->core_relo; in find_relo_core()
5881 const char *prog_name = prog->name; in bpf_core_resolve_relo()
5884 __u32 local_id = relo->type_id; in bpf_core_resolve_relo()
5889 return -EINVAL; in bpf_core_resolve_relo()
5891 local_name = btf__name_by_offset(local_btf, local_type->name_off); in bpf_core_resolve_relo()
5893 return -EINVAL; in bpf_core_resolve_relo()
5895 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && in bpf_core_resolve_relo()
5897 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); in bpf_core_resolve_relo()
5929 if (obj->btf_ext->core_relo_info.len == 0) in bpf_object__relocate_core()
5933 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); in bpf_object__relocate_core()
5934 err = libbpf_get_error(obj->btf_vmlinux_override); in bpf_object__relocate_core()
5947 seg = &obj->btf_ext->core_relo_info; in bpf_object__relocate_core()
5950 sec_idx = seg->sec_idxs[sec_num]; in bpf_object__relocate_core()
5953 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); in bpf_object__relocate_core()
5955 err = -EINVAL; in bpf_object__relocate_core()
5959 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); in bpf_object__relocate_core()
5962 if (rec->insn_off % BPF_INSN_SZ) in bpf_object__relocate_core()
5963 return -EINVAL; in bpf_object__relocate_core()
5964 insn_idx = rec->insn_off / BPF_INSN_SZ; in bpf_object__relocate_core()
5969 * appends all the .BTF.ext info that used to belong to that in bpf_object__relocate_core()
5971 * This is similar to what x86-64 linker does for relocations. in bpf_object__relocate_core()
5975 …pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subpro… in bpf_object__relocate_core()
5979 /* no need to apply CO-RE relocation if the program is in bpf_object__relocate_core()
5982 if (!prog->autoload) in bpf_object__relocate_core()
5986 * program's frame of reference; (sub-)program code is not yet in bpf_object__relocate_core()
5987 * relocated, so it's enough to just subtract in-section offset in bpf_object__relocate_core()
5989 insn_idx = insn_idx - prog->sec_insn_off; in bpf_object__relocate_core()
5990 if (insn_idx >= prog->insns_cnt) in bpf_object__relocate_core()
5991 return -EINVAL; in bpf_object__relocate_core()
5992 insn = &prog->insns[insn_idx]; in bpf_object__relocate_core()
5997 prog->name, i, errstr(err)); in bpf_object__relocate_core()
6001 if (prog->obj->gen_loader) in bpf_object__relocate_core()
6004 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); in bpf_object__relocate_core()
6007 prog->name, i, errstr(err)); in bpf_object__relocate_core()
6011 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); in bpf_object__relocate_core()
6014 prog->name, i, insn_idx, errstr(err)); in bpf_object__relocate_core()
6021 /* obj->btf_vmlinux and module BTFs are freed after object load */ in bpf_object__relocate_core()
6022 btf__free(obj->btf_vmlinux_override); in bpf_object__relocate_core()
6023 obj->btf_vmlinux_override = NULL; in bpf_object__relocate_core()
6027 bpf_core_free_cands(entry->pvalue); in bpf_object__relocate_core()
6045 prog->name, relo_idx, insn_idx, map_idx, map->name); in poison_map_ldimm64()
6049 insn->code = BPF_JMP | BPF_CALL; in poison_map_ldimm64()
6050 insn->dst_reg = 0; in poison_map_ldimm64()
6051 insn->src_reg = 0; in poison_map_ldimm64()
6052 insn->off = 0; in poison_map_ldimm64()
6056 * where lower 123 is map index into obj->maps[] array in poison_map_ldimm64()
6058 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx; in poison_map_ldimm64()
6070 int ext_idx, const struct extern_desc *ext) in poison_kfunc_call() argument
6073 prog->name, relo_idx, insn_idx, ext->name); in poison_kfunc_call()
6076 insn->code = BPF_JMP | BPF_CALL; in poison_kfunc_call()
6077 insn->dst_reg = 0; in poison_kfunc_call()
6078 insn->src_reg = 0; in poison_kfunc_call()
6079 insn->off = 0; in poison_kfunc_call()
6083 * where lower 123 is extern index into obj->externs[] array in poison_kfunc_call()
6085 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx; in poison_kfunc_call()
6089 * - map references;
6090 * - global variable references;
6091 * - extern references.
6098 for (i = 0; i < prog->nr_reloc; i++) { in bpf_object__relocate_data()
6099 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_object__relocate_data()
6100 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate_data()
6102 struct extern_desc *ext; in bpf_object__relocate_data() local
6104 switch (relo->type) { in bpf_object__relocate_data()
6106 map = &obj->maps[relo->map_idx]; in bpf_object__relocate_data()
6107 if (obj->gen_loader) { in bpf_object__relocate_data()
6109 insn[0].imm = relo->map_idx; in bpf_object__relocate_data()
6110 } else if (map->autocreate) { in bpf_object__relocate_data()
6112 insn[0].imm = map->fd; in bpf_object__relocate_data()
6114 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6115 relo->map_idx, map); in bpf_object__relocate_data()
6119 map = &obj->maps[relo->map_idx]; in bpf_object__relocate_data()
6120 insn[1].imm = insn[0].imm + relo->sym_off; in bpf_object__relocate_data()
6121 if (obj->gen_loader) { in bpf_object__relocate_data()
6123 insn[0].imm = relo->map_idx; in bpf_object__relocate_data()
6124 } else if (map->autocreate) { in bpf_object__relocate_data()
6126 insn[0].imm = map->fd; in bpf_object__relocate_data()
6128 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6129 relo->map_idx, map); in bpf_object__relocate_data()
6133 ext = &obj->externs[relo->ext_idx]; in bpf_object__relocate_data()
6134 if (ext->type == EXT_KCFG) { in bpf_object__relocate_data()
6135 if (obj->gen_loader) { in bpf_object__relocate_data()
6137 insn[0].imm = obj->kconfig_map_idx; in bpf_object__relocate_data()
6140 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; in bpf_object__relocate_data()
6142 insn[1].imm = ext->kcfg.data_off; in bpf_object__relocate_data()
6144 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ in bpf_object__relocate_data()
6146 insn[0].imm = ext->ksym.kernel_btf_id; in bpf_object__relocate_data()
6147 insn[1].imm = ext->ksym.kernel_btf_obj_fd; in bpf_object__relocate_data()
6149 insn[0].imm = (__u32)ext->ksym.addr; in bpf_object__relocate_data()
6150 insn[1].imm = ext->ksym.addr >> 32; in bpf_object__relocate_data()
6155 ext = &obj->externs[relo->ext_idx]; in bpf_object__relocate_data()
6157 if (ext->is_set) { in bpf_object__relocate_data()
6158 insn[0].imm = ext->ksym.kernel_btf_id; in bpf_object__relocate_data()
6159 insn[0].off = ext->ksym.btf_fd_idx; in bpf_object__relocate_data()
6161 poison_kfunc_call(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6162 relo->ext_idx, ext); in bpf_object__relocate_data()
6168 prog->name, i); in bpf_object__relocate_data()
6169 return -EINVAL; in bpf_object__relocate_data()
6181 prog->name, i, relo->type); in bpf_object__relocate_data()
6182 return -EINVAL; in bpf_object__relocate_data()
6203 sec_idx = ext_info->sec_idxs[sec_num]; in adjust_prog_btf_ext_info()
6205 if (prog->sec_idx != sec_idx) in adjust_prog_btf_ext_info()
6211 if (insn_off < prog->sec_insn_off) in adjust_prog_btf_ext_info()
6213 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) in adjust_prog_btf_ext_info()
6218 copy_end = rec + ext_info->rec_size; in adjust_prog_btf_ext_info()
6222 return -ENOENT; in adjust_prog_btf_ext_info()
6224 /* append func/line info of a given (sub-)program to the main in adjust_prog_btf_ext_info()
6227 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; in adjust_prog_btf_ext_info()
6228 new_sz = old_sz + (copy_end - copy_start); in adjust_prog_btf_ext_info()
6231 return -ENOMEM; in adjust_prog_btf_ext_info()
6233 *prog_rec_cnt = new_sz / ext_info->rec_size; in adjust_prog_btf_ext_info()
6234 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); in adjust_prog_btf_ext_info()
6236 /* Kernel instruction offsets are in units of 8-byte in adjust_prog_btf_ext_info()
6237 * instructions, while .BTF.ext instruction offsets generated in adjust_prog_btf_ext_info()
6242 off_adj = prog->sub_insn_off - prog->sec_insn_off; in adjust_prog_btf_ext_info()
6245 for (; rec < rec_end; rec += ext_info->rec_size) { in adjust_prog_btf_ext_info()
6250 *prog_rec_sz = ext_info->rec_size; in adjust_prog_btf_ext_info()
6254 return -ENOENT; in adjust_prog_btf_ext_info()
6264 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't in reloc_prog_func_and_line_info()
6267 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) in reloc_prog_func_and_line_info()
6273 if (main_prog != prog && !main_prog->func_info) in reloc_prog_func_and_line_info()
6276 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, in reloc_prog_func_and_line_info()
6277 &main_prog->func_info, in reloc_prog_func_and_line_info()
6278 &main_prog->func_info_cnt, in reloc_prog_func_and_line_info()
6279 &main_prog->func_info_rec_size); in reloc_prog_func_and_line_info()
6281 if (err != -ENOENT) { in reloc_prog_func_and_line_info()
6282 pr_warn("prog '%s': error relocating .BTF.ext function info: %s\n", in reloc_prog_func_and_line_info()
6283 prog->name, errstr(err)); in reloc_prog_func_and_line_info()
6286 if (main_prog->func_info) { in reloc_prog_func_and_line_info()
6291 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); in reloc_prog_func_and_line_info()
6295 …pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext … in reloc_prog_func_and_line_info()
6296 prog->name); in reloc_prog_func_and_line_info()
6301 if (main_prog != prog && !main_prog->line_info) in reloc_prog_func_and_line_info()
6304 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, in reloc_prog_func_and_line_info()
6305 &main_prog->line_info, in reloc_prog_func_and_line_info()
6306 &main_prog->line_info_cnt, in reloc_prog_func_and_line_info()
6307 &main_prog->line_info_rec_size); in reloc_prog_func_and_line_info()
6309 if (err != -ENOENT) { in reloc_prog_func_and_line_info()
6310 pr_warn("prog '%s': error relocating .BTF.ext line info: %s\n", in reloc_prog_func_and_line_info()
6311 prog->name, errstr(err)); in reloc_prog_func_and_line_info()
6314 if (main_prog->line_info) { in reloc_prog_func_and_line_info()
6319 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); in reloc_prog_func_and_line_info()
6323 …pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line… in reloc_prog_func_and_line_info()
6324 prog->name); in reloc_prog_func_and_line_info()
6334 if (insn_idx == relo->insn_idx) in cmp_relo_by_insn_idx()
6336 return insn_idx < relo->insn_idx ? -1 : 1; in cmp_relo_by_insn_idx()
6341 if (!prog->nr_reloc) in find_prog_insn_relo()
6343 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, in find_prog_insn_relo()
6344 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); in find_prog_insn_relo()
6349 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; in append_subprog_relos()
6355 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); in append_subprog_relos()
6361 return -ENOMEM; in append_subprog_relos()
6362 if (subprog->nr_reloc) in append_subprog_relos()
6363 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, in append_subprog_relos()
6364 sizeof(*relos) * subprog->nr_reloc); in append_subprog_relos()
6366 for (i = main_prog->nr_reloc; i < new_cnt; i++) in append_subprog_relos()
6367 relos[i].insn_idx += subprog->sub_insn_off; in append_subprog_relos()
6371 main_prog->reloc_desc = relos; in append_subprog_relos()
6372 main_prog->nr_reloc = new_cnt; in append_subprog_relos()
6384 subprog->sub_insn_off = main_prog->insns_cnt; in bpf_object__append_subprog_code()
6386 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; in bpf_object__append_subprog_code()
6387 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); in bpf_object__append_subprog_code()
6389 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); in bpf_object__append_subprog_code()
6390 return -ENOMEM; in bpf_object__append_subprog_code()
6392 main_prog->insns = insns; in bpf_object__append_subprog_code()
6393 main_prog->insns_cnt = new_cnt; in bpf_object__append_subprog_code()
6395 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, in bpf_object__append_subprog_code()
6396 subprog->insns_cnt * sizeof(*insns)); in bpf_object__append_subprog_code()
6398 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", in bpf_object__append_subprog_code()
6399 main_prog->name, subprog->insns_cnt, subprog->name); in bpf_object__append_subprog_code()
6422 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { in bpf_object__reloc_code()
6423 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6428 if (relo && relo->type == RELO_EXTERN_CALL) in bpf_object__reloc_code()
6433 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { in bpf_object__reloc_code()
6435 prog->name, insn_idx, relo->type); in bpf_object__reloc_code()
6436 return -LIBBPF_ERRNO__RELOC; in bpf_object__reloc_code()
6439 /* sub-program instruction index is a combination of in bpf_object__reloc_code()
6442 * call always has imm = -1, but for static functions in bpf_object__reloc_code()
6443 * relocation is against STT_SECTION and insn->imm in bpf_object__reloc_code()
6446 * for subprog addr relocation, the relo->sym_off + insn->imm is in bpf_object__reloc_code()
6449 if (relo->type == RELO_CALL) in bpf_object__reloc_code()
6450 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; in bpf_object__reloc_code()
6452 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; in bpf_object__reloc_code()
6459 prog->name, insn_idx); in bpf_object__reloc_code()
6460 return -LIBBPF_ERRNO__RELOC; in bpf_object__reloc_code()
6465 * offset necessary, insns->imm is relative to in bpf_object__reloc_code()
6468 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; in bpf_object__reloc_code()
6471 /* we enforce that sub-programs should be in .text section */ in bpf_object__reloc_code()
6472 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); in bpf_object__reloc_code()
6474 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", in bpf_object__reloc_code()
6475 prog->name); in bpf_object__reloc_code()
6476 return -LIBBPF_ERRNO__RELOC; in bpf_object__reloc_code()
6482 * - append it at the end of main program's instructions blog; in bpf_object__reloc_code()
6483 * - process is recursively, while current program is put on hold; in bpf_object__reloc_code()
6484 * - if that subprogram calls some other not yet processes in bpf_object__reloc_code()
6489 if (subprog->sub_insn_off == 0) { in bpf_object__reloc_code()
6498 /* main_prog->insns memory could have been re-allocated, so in bpf_object__reloc_code()
6501 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6508 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; in bpf_object__reloc_code()
6511 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); in bpf_object__reloc_code()
6518 * Relocate sub-program calls.
6520 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6521 * main prog) is processed separately. For each subprog (non-entry functions,
6530 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6546 * subprog->sub_insn_off as zero at all times and won't be appended to current
6555 * +--------+ +-------+
6557 * +--+---+ +--+-+-+ +---+--+
6559 * +--+---+ +------+ +---+--+
6562 * +---+-------+ +------+----+
6564 * +-----------+ +-----------+
6569 * +-----------+------+
6571 * +-----------+------+
6576 * +-----------+------+------+
6578 * +-----------+------+------+
6587 * +-----------+------+
6589 * +-----------+------+
6592 * +-----------+------+------+
6594 * +-----------+------+------+
6607 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate_calls()
6608 subprog = &obj->programs[i]; in bpf_object__relocate_calls()
6612 subprog->sub_insn_off = 0; in bpf_object__relocate_calls()
6629 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__free_relocs()
6630 prog = &obj->programs[i]; in bpf_object__free_relocs()
6631 zfree(&prog->reloc_desc); in bpf_object__free_relocs()
6632 prog->nr_reloc = 0; in bpf_object__free_relocs()
6641 if (a->insn_idx != b->insn_idx) in cmp_relocs()
6642 return a->insn_idx < b->insn_idx ? -1 : 1; in cmp_relocs()
6645 if (a->type != b->type) in cmp_relocs()
6646 return a->type < b->type ? -1 : 1; in cmp_relocs()
6655 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__sort_relos()
6656 struct bpf_program *p = &obj->programs[i]; in bpf_object__sort_relos()
6658 if (!p->nr_reloc) in bpf_object__sort_relos()
6661 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); in bpf_object__sort_relos()
6671 if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG)) in bpf_prog_assign_exc_cb()
6674 n = btf__type_cnt(obj->btf); in bpf_prog_assign_exc_cb()
6679 t = btf_type_by_id(obj->btf, i); in bpf_prog_assign_exc_cb()
6680 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1) in bpf_prog_assign_exc_cb()
6683 name = btf__str_by_offset(obj->btf, t->name_off); in bpf_prog_assign_exc_cb()
6687 t = btf_type_by_id(obj->btf, t->type); in bpf_prog_assign_exc_cb()
6690 prog->name); in bpf_prog_assign_exc_cb()
6691 return -EINVAL; in bpf_prog_assign_exc_cb()
6693 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0) in bpf_prog_assign_exc_cb()
6699 if (prog->exception_cb_idx >= 0) { in bpf_prog_assign_exc_cb()
6700 prog->exception_cb_idx = -1; in bpf_prog_assign_exc_cb()
6707 prog->name); in bpf_prog_assign_exc_cb()
6708 return -EINVAL; in bpf_prog_assign_exc_cb()
6711 for (j = 0; j < obj->nr_programs; j++) { in bpf_prog_assign_exc_cb()
6712 struct bpf_program *subprog = &obj->programs[j]; in bpf_prog_assign_exc_cb()
6716 if (strcmp(name, subprog->name) != 0) in bpf_prog_assign_exc_cb()
6718 /* Enforce non-hidden, as from verifier point of in bpf_prog_assign_exc_cb()
6722 if (!subprog->sym_global || subprog->mark_btf_static) { in bpf_prog_assign_exc_cb()
6723 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n", in bpf_prog_assign_exc_cb()
6724 prog->name, subprog->name); in bpf_prog_assign_exc_cb()
6725 return -EINVAL; in bpf_prog_assign_exc_cb()
6728 if (prog->exception_cb_idx >= 0) { in bpf_prog_assign_exc_cb()
6730 prog->name, subprog->name); in bpf_prog_assign_exc_cb()
6731 return -EINVAL; in bpf_prog_assign_exc_cb()
6733 prog->exception_cb_idx = j; in bpf_prog_assign_exc_cb()
6737 if (prog->exception_cb_idx >= 0) in bpf_prog_assign_exc_cb()
6740 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); in bpf_prog_assign_exc_cb()
6741 return -ENOENT; in bpf_prog_assign_exc_cb()
6779 /* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef,
6781 * with this approach we don't need any extra arch-specific #ifdef guards
6804 t = btf__type_by_id(btf, t->type); in need_func_arg_type_fixup()
6806 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) { in need_func_arg_type_fixup()
6807 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>"; in need_func_arg_type_fixup()
6813 t = skip_mods_and_typedefs(btf, t->type, NULL); in need_func_arg_type_fixup()
6820 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>"; in need_func_arg_type_fixup()
6825 switch (prog->type) { in need_func_arg_type_fixup()
6845 if (btf_is_int(t) && t->size == 8) in need_func_arg_type_fixup()
6854 prog->name, subprog_name, arg_idx, ctx_name); in need_func_arg_type_fixup()
6865 /* caller already validated FUNC -> FUNC_PROTO validity */ in clone_func_btf_info()
6867 fn_proto_t = btf_type_by_id(btf, fn_t->type); in clone_func_btf_info()
6878 fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */ in clone_func_btf_info()
6880 orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */ in clone_func_btf_info()
6881 ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */ in clone_func_btf_info()
6887 return -EINVAL; in clone_func_btf_info()
6895 name_off = p->name_off; in clone_func_btf_info()
6897 err = btf__add_func_param(btf, "", p->type); in clone_func_btf_info()
6903 p->name_off = name_off; /* use remembered str offset */ in clone_func_btf_info()
6906 /* clone FUNC now, btf__add_func() enforces non-empty name, so use in clone_func_btf_info()
6910 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id); in clone_func_btf_info()
6912 return -EINVAL; in clone_func_btf_info()
6915 fn_t->name_off = fn_name_off; /* reuse original string */ in clone_func_btf_info()
6932 struct btf *btf = obj->btf; in bpf_program_fixup_func_info()
6939 /* no .BTF.ext, no problem */ in bpf_program_fixup_func_info()
6940 if (!obj->btf_ext || !prog->func_info) in bpf_program_fixup_func_info()
6951 if (global_ctx_map[i].prog_type != prog->type) in bpf_program_fixup_func_info()
6960 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids)); in bpf_program_fixup_func_info()
6962 return -ENOMEM; in bpf_program_fixup_func_info()
6963 for (i = 0; i < prog->func_info_cnt; i++) { in bpf_program_fixup_func_info()
6964 func_rec = prog->func_info + prog->func_info_rec_size * i; in bpf_program_fixup_func_info()
6965 orig_ids[i] = func_rec->type_id; in bpf_program_fixup_func_info()
6970 * clone and adjust FUNC -> FUNC_PROTO combo in bpf_program_fixup_func_info()
6977 if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0) in bpf_program_fixup_func_info()
6981 orig_fn_id = t->type; in bpf_program_fixup_func_info()
6986 /* sanity check FUNC -> FUNC_PROTO chain, just in case */ in bpf_program_fixup_func_info()
6987 fn_proto_t = btf_type_by_id(btf, fn_t->type); in bpf_program_fixup_func_info()
6993 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) { in bpf_program_fixup_func_info()
6994 if (orig_ids[rec_idx] == t->type) { in bpf_program_fixup_func_info()
6995 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx; in bpf_program_fixup_func_info()
7005 arg_idx = btf_decl_tag(t)->component_idx; in bpf_program_fixup_func_info()
7011 fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>"; in bpf_program_fixup_func_info()
7012 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name)) in bpf_program_fixup_func_info()
7016 if (func_rec->type_id == orig_fn_id) { in bpf_program_fixup_func_info()
7026 func_rec->type_id = fn_id; in bpf_program_fixup_func_info()
7029 /* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument; in bpf_program_fixup_func_info()
7031 * funcs share the same program type, so need only PTR -> in bpf_program_fixup_func_info()
7038 err = -EINVAL; in bpf_program_fixup_func_info()
7044 tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx); in bpf_program_fixup_func_info()
7046 err = -EINVAL; in bpf_program_fixup_func_info()
7051 fn_t = btf_type_by_id(btf, func_rec->type_id); in bpf_program_fixup_func_info()
7052 fn_proto_t = btf_type_by_id(btf, fn_t->type); in bpf_program_fixup_func_info()
7056 p->type = ptr_id; in bpf_program_fixup_func_info()
7072 if (obj->btf_ext) { in bpf_object__relocate()
7075 pr_warn("failed to perform CO-RE relocations: %s\n", in bpf_object__relocate()
7082 /* Before relocating calls pre-process relocations and mark in bpf_object__relocate()
7089 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
7090 prog = &obj->programs[i]; in bpf_object__relocate()
7091 for (j = 0; j < prog->nr_reloc; j++) { in bpf_object__relocate()
7092 struct reloc_desc *relo = &prog->reloc_desc[j]; in bpf_object__relocate()
7093 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate()
7096 if (relo->type == RELO_SUBPROG_ADDR) in bpf_object__relocate()
7108 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
7109 prog = &obj->programs[i]; in bpf_object__relocate()
7110 /* sub-program's sub-calls are relocated within the context of in bpf_object__relocate()
7115 if (!prog->autoload) in bpf_object__relocate()
7121 prog->name, errstr(err)); in bpf_object__relocate()
7129 if (prog->exception_cb_idx >= 0) { in bpf_object__relocate()
7130 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; in bpf_object__relocate()
7137 if (subprog->sub_insn_off == 0) { in bpf_object__relocate()
7147 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
7148 prog = &obj->programs[i]; in bpf_object__relocate()
7151 if (!prog->autoload) in bpf_object__relocate()
7158 prog->name, errstr(err)); in bpf_object__relocate()
7162 /* Fix up .BTF.ext information, if necessary */ in bpf_object__relocate()
7165 pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %s\n", in bpf_object__relocate()
7166 prog->name, errstr(err)); in bpf_object__relocate()
7194 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) in bpf_object__collect_map_relos()
7195 return -EINVAL; in bpf_object__collect_map_relos()
7196 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); in bpf_object__collect_map_relos()
7198 return -EINVAL; in bpf_object__collect_map_relos()
7200 nrels = shdr->sh_size / shdr->sh_entsize; in bpf_object__collect_map_relos()
7205 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_map_relos()
7208 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); in bpf_object__collect_map_relos()
7211 i, (size_t)ELF64_R_SYM(rel->r_info)); in bpf_object__collect_map_relos()
7212 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_map_relos()
7214 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; in bpf_object__collect_map_relos()
7216 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", in bpf_object__collect_map_relos()
7217 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, in bpf_object__collect_map_relos()
7218 (size_t)rel->r_offset, sym->st_name, name); in bpf_object__collect_map_relos()
7220 for (j = 0; j < obj->nr_maps; j++) { in bpf_object__collect_map_relos()
7221 map = &obj->maps[j]; in bpf_object__collect_map_relos()
7222 if (map->sec_idx != obj->efile.btf_maps_shndx) in bpf_object__collect_map_relos()
7225 vi = btf_var_secinfos(sec) + map->btf_var_idx; in bpf_object__collect_map_relos()
7226 if (vi->offset <= rel->r_offset && in bpf_object__collect_map_relos()
7227 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size) in bpf_object__collect_map_relos()
7230 if (j == obj->nr_maps) { in bpf_object__collect_map_relos()
7231 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n", in bpf_object__collect_map_relos()
7232 i, name, (size_t)rel->r_offset); in bpf_object__collect_map_relos()
7233 return -EINVAL; in bpf_object__collect_map_relos()
7236 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type); in bpf_object__collect_map_relos()
7237 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY; in bpf_object__collect_map_relos()
7240 if (sym->st_shndx != obj->efile.btf_maps_shndx) { in bpf_object__collect_map_relos()
7241 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", in bpf_object__collect_map_relos()
7243 return -LIBBPF_ERRNO__RELOC; in bpf_object__collect_map_relos()
7245 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && in bpf_object__collect_map_relos()
7246 map->def.key_size != sizeof(int)) { in bpf_object__collect_map_relos()
7247 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", in bpf_object__collect_map_relos()
7248 i, map->name, sizeof(int)); in bpf_object__collect_map_relos()
7249 return -EINVAL; in bpf_object__collect_map_relos()
7255 return -ESRCH; in bpf_object__collect_map_relos()
7262 return -ESRCH; in bpf_object__collect_map_relos()
7264 if (targ_prog->sec_idx != sym->st_shndx || in bpf_object__collect_map_relos()
7265 targ_prog->sec_insn_off * 8 != sym->st_value || in bpf_object__collect_map_relos()
7267 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n", in bpf_object__collect_map_relos()
7269 return -LIBBPF_ERRNO__RELOC; in bpf_object__collect_map_relos()
7272 return -EINVAL; in bpf_object__collect_map_relos()
7275 var = btf__type_by_id(obj->btf, vi->type); in bpf_object__collect_map_relos()
7276 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); in bpf_object__collect_map_relos()
7278 return -EINVAL; in bpf_object__collect_map_relos()
7279 member = btf_members(def) + btf_vlen(def) - 1; in bpf_object__collect_map_relos()
7280 mname = btf__name_by_offset(obj->btf, member->name_off); in bpf_object__collect_map_relos()
7282 return -EINVAL; in bpf_object__collect_map_relos()
7284 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; in bpf_object__collect_map_relos()
7285 if (rel->r_offset - vi->offset < moff) in bpf_object__collect_map_relos()
7286 return -EINVAL; in bpf_object__collect_map_relos()
7288 moff = rel->r_offset - vi->offset - moff; in bpf_object__collect_map_relos()
7293 return -EINVAL; in bpf_object__collect_map_relos()
7295 if (moff >= map->init_slots_sz) { in bpf_object__collect_map_relos()
7297 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); in bpf_object__collect_map_relos()
7299 return -ENOMEM; in bpf_object__collect_map_relos()
7300 map->init_slots = tmp; in bpf_object__collect_map_relos()
7301 memset(map->init_slots + map->init_slots_sz, 0, in bpf_object__collect_map_relos()
7302 (new_sz - map->init_slots_sz) * host_ptr_sz); in bpf_object__collect_map_relos()
7303 map->init_slots_sz = new_sz; in bpf_object__collect_map_relos()
7305 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; in bpf_object__collect_map_relos()
7308 i, map->name, moff, type, name); in bpf_object__collect_map_relos()
7318 for (i = 0; i < obj->efile.sec_cnt; i++) { in bpf_object__collect_relos()
7319 struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; in bpf_object__collect_relos()
7324 if (sec_desc->sec_type != SEC_RELO) in bpf_object__collect_relos()
7327 shdr = sec_desc->shdr; in bpf_object__collect_relos()
7328 data = sec_desc->data; in bpf_object__collect_relos()
7329 idx = shdr->sh_info; in bpf_object__collect_relos()
7331 if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) { in bpf_object__collect_relos()
7333 return -LIBBPF_ERRNO__INTERNAL; in bpf_object__collect_relos()
7336 if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) in bpf_object__collect_relos()
7338 else if (idx == obj->efile.btf_maps_shndx) in bpf_object__collect_relos()
7352 if (BPF_CLASS(insn->code) == BPF_JMP && in insn_is_helper_call()
7353 BPF_OP(insn->code) == BPF_CALL && in insn_is_helper_call()
7354 BPF_SRC(insn->code) == BPF_K && in insn_is_helper_call()
7355 insn->src_reg == 0 && in insn_is_helper_call()
7356 insn->dst_reg == 0) { in insn_is_helper_call()
7357 *func_id = insn->imm; in insn_is_helper_call()
7365 struct bpf_insn *insn = prog->insns; in bpf_object__sanitize_prog()
7369 if (obj->gen_loader) in bpf_object__sanitize_prog()
7372 for (i = 0; i < prog->insns_cnt; i++, insn++) { in bpf_object__sanitize_prog()
7384 insn->imm = BPF_FUNC_probe_read; in bpf_object__sanitize_prog()
7389 insn->imm = BPF_FUNC_probe_read_str; in bpf_object__sanitize_prog()
7401 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7408 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) in libbpf_prepare_prog_load()
7409 opts->expected_attach_type = 0; in libbpf_prepare_prog_load()
7412 opts->prog_flags |= BPF_F_SLEEPABLE; in libbpf_prepare_prog_load()
7414 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) in libbpf_prepare_prog_load()
7415 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; in libbpf_prepare_prog_load()
7418 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) { in libbpf_prepare_prog_load()
7423 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; in libbpf_prepare_prog_load()
7424 opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI; in libbpf_prepare_prog_load()
7427 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { in libbpf_prepare_prog_load()
7431 attach_name = strchr(prog->sec_name, '/'); in libbpf_prepare_prog_load()
7442 …pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\… in libbpf_prepare_prog_load()
7443 prog->name); in libbpf_prepare_prog_load()
7444 return -EINVAL; in libbpf_prepare_prog_load()
7453 prog->attach_btf_obj_fd = btf_obj_fd; in libbpf_prepare_prog_load()
7454 prog->attach_btf_id = btf_type_id; in libbpf_prepare_prog_load()
7457 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because in libbpf_prepare_prog_load()
7461 opts->attach_btf_obj_fd = btf_obj_fd; in libbpf_prepare_prog_load()
7462 opts->attach_btf_id = btf_type_id; in libbpf_prepare_prog_load()
7478 __u32 log_level = prog->log_level; in bpf_object_load_prog()
7484 switch (prog->type) { in bpf_object_load_prog()
7491 prog->name, prog->sec_name); in bpf_object_load_prog()
7492 return -EINVAL; in bpf_object_load_prog()
7494 if (prog->attach_btf_id == 0) { in bpf_object_load_prog()
7496 prog->name); in bpf_object_load_prog()
7497 return -EINVAL; in bpf_object_load_prog()
7505 return -EINVAL; in bpf_object_load_prog()
7508 prog_name = prog->name; in bpf_object_load_prog()
7509 load_attr.attach_prog_fd = prog->attach_prog_fd; in bpf_object_load_prog()
7510 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; in bpf_object_load_prog()
7511 load_attr.attach_btf_id = prog->attach_btf_id; in bpf_object_load_prog()
7513 load_attr.prog_ifindex = prog->prog_ifindex; in bpf_object_load_prog()
7514 load_attr.expected_attach_type = prog->expected_attach_type; in bpf_object_load_prog()
7517 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) { in bpf_object_load_prog()
7518 load_attr.prog_btf_fd = btf__fd(obj->btf); in bpf_object_load_prog()
7519 load_attr.func_info = prog->func_info; in bpf_object_load_prog()
7520 load_attr.func_info_rec_size = prog->func_info_rec_size; in bpf_object_load_prog()
7521 load_attr.func_info_cnt = prog->func_info_cnt; in bpf_object_load_prog()
7522 load_attr.line_info = prog->line_info; in bpf_object_load_prog()
7523 load_attr.line_info_rec_size = prog->line_info_rec_size; in bpf_object_load_prog()
7524 load_attr.line_info_cnt = prog->line_info_cnt; in bpf_object_load_prog()
7527 load_attr.prog_flags = prog->prog_flags; in bpf_object_load_prog()
7528 load_attr.fd_array = obj->fd_array; in bpf_object_load_prog()
7530 load_attr.token_fd = obj->token_fd; in bpf_object_load_prog()
7531 if (obj->token_fd) in bpf_object_load_prog()
7535 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { in bpf_object_load_prog()
7536 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); in bpf_object_load_prog()
7539 prog->name, errstr(err)); in bpf_object_load_prog()
7542 insns = prog->insns; in bpf_object_load_prog()
7543 insns_cnt = prog->insns_cnt; in bpf_object_load_prog()
7546 if (obj->gen_loader) { in bpf_object_load_prog()
7547 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, in bpf_object_load_prog()
7549 prog - obj->programs); in bpf_object_load_prog()
7550 *prog_fd = -1; in bpf_object_load_prog()
7561 if (prog->log_buf) { in bpf_object_load_prog()
7562 log_buf = prog->log_buf; in bpf_object_load_prog()
7563 log_buf_size = prog->log_size; in bpf_object_load_prog()
7565 } else if (obj->log_buf) { in bpf_object_load_prog()
7566 log_buf = obj->log_buf; in bpf_object_load_prog()
7567 log_buf_size = obj->log_size; in bpf_object_load_prog()
7573 ret = -ENOMEM; in bpf_object_load_prog()
7586 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); in bpf_object_load_prog()
7589 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", in bpf_object_load_prog()
7590 prog->name, log_buf); in bpf_object_load_prog()
7593 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { in bpf_object_load_prog()
7597 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_load_prog()
7598 map = &prog->obj->maps[i]; in bpf_object_load_prog()
7599 if (map->libbpf_type != LIBBPF_MAP_RODATA) in bpf_object_load_prog()
7602 if (bpf_prog_bind_map(ret, map->fd, NULL)) { in bpf_object_load_prog()
7604 prog->name, map->real_name, errstr(errno)); in bpf_object_load_prog()
7624 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). in bpf_object_load_prog()
7629 ret = -errno; in bpf_object_load_prog()
7631 /* post-process verifier log to improve error descriptions */ in bpf_object_load_prog()
7634 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno)); in bpf_object_load_prog()
7638 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", in bpf_object_load_prog()
7639 prog->name, log_buf); in bpf_object_load_prog()
7655 p = cur - 1; in find_prev_line()
7656 while (p - 1 >= buf && *(p - 1) != '\n') in find_prev_line()
7657 p--; in find_prev_line()
7665 /* size of the remaining log content to the right from the to-be-replaced part */ in patch_log()
7666 size_t rem_sz = (buf + log_sz) - (orig + orig_sz); in patch_log()
7671 * shift log contents by (patch_sz - orig_sz) bytes to the right in patch_log()
7672 * starting from after to-be-replaced part of the log. in patch_log()
7675 * shift log contents by (orig_sz - patch_sz) bytes to the left in patch_log()
7676 * starting from after to-be-replaced part of the log in patch_log()
7685 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; in patch_log()
7687 } else if (patch_sz - orig_sz > buf_sz - log_sz) { in patch_log()
7689 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); in patch_log()
7703 /* Expected log for failed and not properly guarded CO-RE relocation: in fixup_log_failed_core_relo()
7704 * line1 -> 123: (85) call unknown#195896080 in fixup_log_failed_core_relo()
7705 * line2 -> invalid func unknown#195896080 in fixup_log_failed_core_relo()
7706 * line3 -> <anything else or end of buffer> in fixup_log_failed_core_relo()
7709 * instruction index to find corresponding CO-RE relocation and in fixup_log_failed_core_relo()
7711 * failed CO-RE relocation. in fixup_log_failed_core_relo()
7725 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); in fixup_log_failed_core_relo()
7731 "%d: <invalid CO-RE relocation>\n" in fixup_log_failed_core_relo()
7732 "failed to resolve CO-RE relocation %s%s\n", in fixup_log_failed_core_relo()
7735 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); in fixup_log_failed_core_relo()
7743 * line1 -> 123: (85) call unknown#2001000345 in fixup_log_missing_map_load()
7744 * line2 -> invalid func unknown#2001000345 in fixup_log_missing_map_load()
7745 * line3 -> <anything else or end of buffer> in fixup_log_missing_map_load()
7748 * "345" in "2001000345" is a map index in obj->maps to fetch map name. in fixup_log_missing_map_load()
7750 struct bpf_object *obj = prog->obj; in fixup_log_missing_map_load()
7758 map_idx -= POISON_LDIMM64_MAP_BASE; in fixup_log_missing_map_load()
7759 if (map_idx < 0 || map_idx >= obj->nr_maps) in fixup_log_missing_map_load()
7761 map = &obj->maps[map_idx]; in fixup_log_missing_map_load()
7766 insn_idx, map->name); in fixup_log_missing_map_load()
7768 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); in fixup_log_missing_map_load()
7776 * line1 -> 123: (85) call unknown#2002000345 in fixup_log_missing_kfunc_call()
7777 * line2 -> invalid func unknown#2002000345 in fixup_log_missing_kfunc_call()
7778 * line3 -> <anything else or end of buffer> in fixup_log_missing_kfunc_call()
7781 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name. in fixup_log_missing_kfunc_call()
7783 struct bpf_object *obj = prog->obj; in fixup_log_missing_kfunc_call()
7784 const struct extern_desc *ext; in fixup_log_missing_kfunc_call() local
7791 ext_idx -= POISON_CALL_KFUNC_BASE; in fixup_log_missing_kfunc_call()
7792 if (ext_idx < 0 || ext_idx >= obj->nr_extern) in fixup_log_missing_kfunc_call()
7794 ext = &obj->externs[ext_idx]; in fixup_log_missing_kfunc_call()
7799 insn_idx, ext->name); in fixup_log_missing_kfunc_call()
7801 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); in fixup_log_missing_kfunc_call()
7816 next_line = buf + log_sz - 1; in fixup_verifier_log()
7828 /* failed CO-RE relocation case */ in fixup_verifier_log()
7856 struct bpf_object *obj = prog->obj; in bpf_program_record_relos()
7859 for (i = 0; i < prog->nr_reloc; i++) { in bpf_program_record_relos()
7860 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_program_record_relos()
7861 struct extern_desc *ext = &obj->externs[relo->ext_idx]; in bpf_program_record_relos() local
7864 switch (relo->type) { in bpf_program_record_relos()
7866 if (ext->type != EXT_KSYM) in bpf_program_record_relos()
7868 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? in bpf_program_record_relos()
7870 bpf_gen__record_extern(obj->gen_loader, ext->name, in bpf_program_record_relos()
7871 ext->is_weak, !ext->ksym.type_id, in bpf_program_record_relos()
7872 true, kind, relo->insn_idx); in bpf_program_record_relos()
7875 bpf_gen__record_extern(obj->gen_loader, ext->name, in bpf_program_record_relos()
7876 ext->is_weak, false, false, BTF_KIND_FUNC, in bpf_program_record_relos()
7877 relo->insn_idx); in bpf_program_record_relos()
7881 .insn_off = relo->insn_idx * 8, in bpf_program_record_relos()
7882 .type_id = relo->core_relo->type_id, in bpf_program_record_relos()
7883 .access_str_off = relo->core_relo->access_str_off, in bpf_program_record_relos()
7884 .kind = relo->core_relo->kind, in bpf_program_record_relos()
7887 bpf_gen__record_relo_core(obj->gen_loader, &cr); in bpf_program_record_relos()
7904 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__load_progs()
7905 prog = &obj->programs[i]; in bpf_object__load_progs()
7908 if (!prog->autoload) { in bpf_object__load_progs()
7909 pr_debug("prog '%s': skipped loading\n", prog->name); in bpf_object__load_progs()
7912 prog->log_level |= log_level; in bpf_object__load_progs()
7914 if (obj->gen_loader) in bpf_object__load_progs()
7917 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, in bpf_object__load_progs()
7918 obj->license, obj->kern_version, &prog->fd); in bpf_object__load_progs()
7920 pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err)); in bpf_object__load_progs()
7935 for (i = 0; i < obj->nr_programs; i++) { in bpf_object_prepare_progs()
7936 prog = &obj->programs[i]; in bpf_object_prepare_progs()
7952 prog->sec_def = find_sec_def(prog->sec_name); in bpf_object_init_progs()
7953 if (!prog->sec_def) { in bpf_object_init_progs()
7956 prog->name, prog->sec_name); in bpf_object_init_progs()
7960 prog->type = prog->sec_def->prog_type; in bpf_object_init_progs()
7961 prog->expected_attach_type = prog->sec_def->expected_attach_type; in bpf_object_init_progs()
7966 if (prog->sec_def->prog_setup_fn) { in bpf_object_init_progs()
7967 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); in bpf_object_init_progs()
7970 prog->name, errstr(err)); in bpf_object_init_progs()
7991 return ERR_PTR(-EINVAL); in bpf_object_open()
7996 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); in bpf_object_open()
8000 return ERR_PTR(-EINVAL); in bpf_object_open()
8014 return ERR_PTR(-EINVAL); in bpf_object_open()
8016 return ERR_PTR(-EINVAL); in bpf_object_open()
8026 return ERR_PTR(-ENAMETOOLONG); in bpf_object_open()
8032 obj->log_buf = log_buf; in bpf_object_open()
8033 obj->log_size = log_size; in bpf_object_open()
8034 obj->log_level = log_level; in bpf_object_open()
8037 obj->token_path = strdup(token_path); in bpf_object_open()
8038 if (!obj->token_path) { in bpf_object_open()
8039 err = -ENOMEM; in bpf_object_open()
8047 err = -ENAMETOOLONG; in bpf_object_open()
8050 obj->btf_custom_path = strdup(btf_tmp_path); in bpf_object_open()
8051 if (!obj->btf_custom_path) { in bpf_object_open()
8052 err = -ENOMEM; in bpf_object_open()
8059 obj->kconfig = strdup(kconfig); in bpf_object_open()
8060 if (!obj->kconfig) { in bpf_object_open()
8061 err = -ENOMEM; in bpf_object_open()
8088 return libbpf_err_ptr(-EINVAL); in bpf_object__open_file()
8105 return libbpf_err_ptr(-EINVAL); in bpf_object__open_mem()
8108 snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz); in bpf_object__open_mem()
8118 return libbpf_err(-EINVAL); in bpf_object_unload()
8120 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_unload()
8121 zclose(obj->maps[i].fd); in bpf_object_unload()
8122 if (obj->maps[i].st_ops) in bpf_object_unload()
8123 zfree(&obj->maps[i].st_ops->kern_vdata); in bpf_object_unload()
8126 for (i = 0; i < obj->nr_programs; i++) in bpf_object_unload()
8127 bpf_program__unload(&obj->programs[i]); in bpf_object_unload()
8140 m->def.map_flags &= ~BPF_F_MMAPABLE; in bpf_object__sanitize_maps()
8158 err = -errno; in libbpf_kallsyms_parse()
8170 err = -EINVAL; in libbpf_kallsyms_parse()
8188 struct extern_desc *ext; in kallsyms_cb() local
8193 ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name); in kallsyms_cb()
8195 ext = find_extern_by_name(obj, sym_name); in kallsyms_cb()
8196 if (!ext || ext->type != EXT_KSYM) in kallsyms_cb()
8199 t = btf__type_by_id(obj->btf, ext->btf_id); in kallsyms_cb()
8203 if (ext->is_set && ext->ksym.addr != sym_addr) { in kallsyms_cb()
8205 sym_name, ext->ksym.addr, sym_addr); in kallsyms_cb()
8206 return -EINVAL; in kallsyms_cb()
8208 if (!ext->is_set) { in kallsyms_cb()
8209 ext->is_set = true; in kallsyms_cb()
8210 ext->ksym.addr = sym_addr; in kallsyms_cb()
8229 btf = obj->btf_vmlinux; in find_ksym_btf_id()
8233 if (id == -ENOENT) { in find_ksym_btf_id()
8238 for (i = 0; i < obj->btf_module_cnt; i++) { in find_ksym_btf_id()
8240 mod_btf = &obj->btf_modules[i]; in find_ksym_btf_id()
8241 btf = mod_btf->btf; in find_ksym_btf_id()
8243 if (id != -ENOENT) in find_ksym_btf_id()
8248 return -ESRCH; in find_ksym_btf_id()
8256 struct extern_desc *ext) in bpf_object__resolve_ksym_var_btf_id() argument
8265 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); in bpf_object__resolve_ksym_var_btf_id()
8267 if (id == -ESRCH && ext->is_weak) in bpf_object__resolve_ksym_var_btf_id()
8270 ext->name); in bpf_object__resolve_ksym_var_btf_id()
8275 local_type_id = ext->ksym.type_id; in bpf_object__resolve_ksym_var_btf_id()
8279 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); in bpf_object__resolve_ksym_var_btf_id()
8280 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); in bpf_object__resolve_ksym_var_btf_id()
8282 err = bpf_core_types_are_compat(obj->btf, local_type_id, in bpf_object__resolve_ksym_var_btf_id()
8288 local_type = btf__type_by_id(obj->btf, local_type_id); in bpf_object__resolve_ksym_var_btf_id()
8289 local_name = btf__name_by_offset(obj->btf, local_type->name_off); in bpf_object__resolve_ksym_var_btf_id()
8290 targ_name = btf__name_by_offset(btf, targ_type->name_off); in bpf_object__resolve_ksym_var_btf_id()
8293 ext->name, local_type_id, in bpf_object__resolve_ksym_var_btf_id()
8296 return -EINVAL; in bpf_object__resolve_ksym_var_btf_id()
8299 ext->is_set = true; in bpf_object__resolve_ksym_var_btf_id()
8300 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; in bpf_object__resolve_ksym_var_btf_id()
8301 ext->ksym.kernel_btf_id = id; in bpf_object__resolve_ksym_var_btf_id()
8303 ext->name, id, btf_kind_str(targ_var), targ_var_name); in bpf_object__resolve_ksym_var_btf_id()
8309 struct extern_desc *ext) in bpf_object__resolve_ksym_func_btf_id() argument
8317 local_func_proto_id = ext->ksym.type_id; in bpf_object__resolve_ksym_func_btf_id()
8319 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, in bpf_object__resolve_ksym_func_btf_id()
8322 if (kfunc_id == -ESRCH && ext->is_weak) in bpf_object__resolve_ksym_func_btf_id()
8325 ext->name); in bpf_object__resolve_ksym_func_btf_id()
8330 kfunc_proto_id = kern_func->type; in bpf_object__resolve_ksym_func_btf_id()
8332 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, in bpf_object__resolve_ksym_func_btf_id()
8335 if (ext->is_weak) in bpf_object__resolve_ksym_func_btf_id()
8339 ext->name, local_func_proto_id, in bpf_object__resolve_ksym_func_btf_id()
8340 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id); in bpf_object__resolve_ksym_func_btf_id()
8341 return -EINVAL; in bpf_object__resolve_ksym_func_btf_id()
8345 if (mod_btf && !mod_btf->fd_array_idx) { in bpf_object__resolve_ksym_func_btf_id()
8346 /* insn->off is s16 */ in bpf_object__resolve_ksym_func_btf_id()
8347 if (obj->fd_array_cnt == INT16_MAX) { in bpf_object__resolve_ksym_func_btf_id()
8349 ext->name, mod_btf->fd_array_idx); in bpf_object__resolve_ksym_func_btf_id()
8350 return -E2BIG; in bpf_object__resolve_ksym_func_btf_id()
8353 if (!obj->fd_array_cnt) in bpf_object__resolve_ksym_func_btf_id()
8354 obj->fd_array_cnt = 1; in bpf_object__resolve_ksym_func_btf_id()
8356 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), in bpf_object__resolve_ksym_func_btf_id()
8357 obj->fd_array_cnt + 1); in bpf_object__resolve_ksym_func_btf_id()
8360 mod_btf->fd_array_idx = obj->fd_array_cnt; in bpf_object__resolve_ksym_func_btf_id()
8362 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; in bpf_object__resolve_ksym_func_btf_id()
8365 ext->is_set = true; in bpf_object__resolve_ksym_func_btf_id()
8366 ext->ksym.kernel_btf_id = kfunc_id; in bpf_object__resolve_ksym_func_btf_id()
8367 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0; in bpf_object__resolve_ksym_func_btf_id()
8370 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call. in bpf_object__resolve_ksym_func_btf_id()
8371 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64. in bpf_object__resolve_ksym_func_btf_id()
8373 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; in bpf_object__resolve_ksym_func_btf_id()
8375 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id); in bpf_object__resolve_ksym_func_btf_id()
8383 struct extern_desc *ext; in bpf_object__resolve_ksyms_btf_id() local
8386 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_ksyms_btf_id()
8387 ext = &obj->externs[i]; in bpf_object__resolve_ksyms_btf_id()
8388 if (ext->type != EXT_KSYM || !ext->ksym.type_id) in bpf_object__resolve_ksyms_btf_id()
8391 if (obj->gen_loader) { in bpf_object__resolve_ksyms_btf_id()
8392 ext->is_set = true; in bpf_object__resolve_ksyms_btf_id()
8393 ext->ksym.kernel_btf_obj_fd = 0; in bpf_object__resolve_ksyms_btf_id()
8394 ext->ksym.kernel_btf_id = 0; in bpf_object__resolve_ksyms_btf_id()
8397 t = btf__type_by_id(obj->btf, ext->btf_id); in bpf_object__resolve_ksyms_btf_id()
8399 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); in bpf_object__resolve_ksyms_btf_id()
8401 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); in bpf_object__resolve_ksyms_btf_id()
8413 struct extern_desc *ext; in bpf_object__resolve_externs() local
8417 if (obj->nr_extern == 0) in bpf_object__resolve_externs()
8420 if (obj->kconfig_map_idx >= 0) in bpf_object__resolve_externs()
8421 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; in bpf_object__resolve_externs()
8423 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
8424 ext = &obj->externs[i]; in bpf_object__resolve_externs()
8426 if (ext->type == EXT_KSYM) { in bpf_object__resolve_externs()
8427 if (ext->ksym.type_id) in bpf_object__resolve_externs()
8432 } else if (ext->type == EXT_KCFG) { in bpf_object__resolve_externs()
8433 void *ext_ptr = kcfg_data + ext->kcfg.data_off; in bpf_object__resolve_externs()
8437 if (str_has_pfx(ext->name, "CONFIG_")) { in bpf_object__resolve_externs()
8443 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { in bpf_object__resolve_externs()
8446 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); in bpf_object__resolve_externs()
8447 return -EINVAL; in bpf_object__resolve_externs()
8449 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { in bpf_object__resolve_externs()
8451 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { in bpf_object__resolve_externs()
8453 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { in bpf_object__resolve_externs()
8461 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); in bpf_object__resolve_externs()
8462 return -EINVAL; in bpf_object__resolve_externs()
8465 err = set_kcfg_value_num(ext, ext_ptr, value); in bpf_object__resolve_externs()
8469 ext->name, (long long)value); in bpf_object__resolve_externs()
8471 pr_warn("extern '%s': unrecognized extern kind\n", ext->name); in bpf_object__resolve_externs()
8472 return -EINVAL; in bpf_object__resolve_externs()
8478 return -EINVAL; in bpf_object__resolve_externs()
8480 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
8481 ext = &obj->externs[i]; in bpf_object__resolve_externs()
8482 if (ext->type == EXT_KCFG && !ext->is_set) { in bpf_object__resolve_externs()
8491 return -EINVAL; in bpf_object__resolve_externs()
8496 return -EINVAL; in bpf_object__resolve_externs()
8501 return -EINVAL; in bpf_object__resolve_externs()
8503 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
8504 ext = &obj->externs[i]; in bpf_object__resolve_externs()
8506 if (!ext->is_set && !ext->is_weak) { in bpf_object__resolve_externs()
8507 pr_warn("extern '%s' (strong): not resolved\n", ext->name); in bpf_object__resolve_externs()
8508 return -ESRCH; in bpf_object__resolve_externs()
8509 } else if (!ext->is_set) { in bpf_object__resolve_externs()
8511 ext->name); in bpf_object__resolve_externs()
8524 st_ops = map->st_ops; in bpf_map_prepare_vdata()
8525 type = btf__type_by_id(map->obj->btf, st_ops->type_id); in bpf_map_prepare_vdata()
8527 struct bpf_program *prog = st_ops->progs[i]; in bpf_map_prepare_vdata()
8535 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; in bpf_map_prepare_vdata()
8545 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_prepare_struct_ops()
8546 map = &obj->maps[i]; in bpf_object_prepare_struct_ops()
8551 if (!map->autocreate) in bpf_object_prepare_struct_ops()
8564 /* unpin any maps that were auto-pinned during load */ in bpf_object_unpin()
8565 for (i = 0; i < obj->nr_maps; i++) in bpf_object_unpin()
8566 if (obj->maps[i].pinned && !obj->maps[i].reused) in bpf_object_unpin()
8567 bpf_map__unpin(&obj->maps[i], NULL); in bpf_object_unpin()
8575 zfree(&obj->fd_array); in bpf_object_post_load_cleanup()
8578 for (i = 0; i < obj->btf_module_cnt; i++) { in bpf_object_post_load_cleanup()
8579 close(obj->btf_modules[i].fd); in bpf_object_post_load_cleanup()
8580 btf__free(obj->btf_modules[i].btf); in bpf_object_post_load_cleanup()
8581 free(obj->btf_modules[i].name); in bpf_object_post_load_cleanup()
8583 obj->btf_module_cnt = 0; in bpf_object_post_load_cleanup()
8584 zfree(&obj->btf_modules); in bpf_object_post_load_cleanup()
8587 btf__free(obj->btf_vmlinux); in bpf_object_post_load_cleanup()
8588 obj->btf_vmlinux = NULL; in bpf_object_post_load_cleanup()
8595 if (obj->state >= OBJ_PREPARED) { in bpf_object_prepare()
8596 pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name); in bpf_object_prepare()
8597 return -EINVAL; in bpf_object_prepare()
8603 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); in bpf_object_prepare()
8607 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); in bpf_object_prepare()
8615 obj->state = OBJ_LOADED; in bpf_object_prepare()
8619 obj->state = OBJ_PREPARED; in bpf_object_prepare()
8628 return libbpf_err(-EINVAL); in bpf_object_load()
8630 if (obj->state >= OBJ_LOADED) { in bpf_object_load()
8631 pr_warn("object '%s': load can't be attempted twice\n", obj->name); in bpf_object_load()
8632 return libbpf_err(-EINVAL); in bpf_object_load()
8635 /* Disallow kernel loading programs of non-native endianness but in bpf_object_load()
8636 * permit cross-endian creation of "light skeleton". in bpf_object_load()
8638 if (obj->gen_loader) { in bpf_object_load()
8639 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); in bpf_object_load()
8641 pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name); in bpf_object_load()
8642 return libbpf_err(-LIBBPF_ERRNO__ENDIAN); in bpf_object_load()
8645 if (obj->state < OBJ_PREPARED) { in bpf_object_load()
8654 if (obj->gen_loader) { in bpf_object_load()
8656 if (obj->btf) in bpf_object_load()
8657 btf__set_fd(obj->btf, -1); in bpf_object_load()
8659 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); in bpf_object_load()
8663 obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */ in bpf_object_load()
8668 pr_warn("failed to load object '%s'\n", obj->path); in bpf_object_load()
8692 return -ENOMEM; in make_parent_dir()
8696 err = -errno; in make_parent_dir()
8712 return -EINVAL; in check_path()
8716 return -ENOMEM; in check_path()
8721 err = -errno; in check_path()
8727 err = -EINVAL; in check_path()
8737 if (prog->fd < 0) { in bpf_program__pin()
8738 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); in bpf_program__pin()
8739 return libbpf_err(-EINVAL); in bpf_program__pin()
8750 if (bpf_obj_pin(prog->fd, path)) { in bpf_program__pin()
8751 err = -errno; in bpf_program__pin()
8752 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err)); in bpf_program__pin()
8756 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); in bpf_program__pin()
8764 if (prog->fd < 0) { in bpf_program__unpin()
8765 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); in bpf_program__unpin()
8766 return libbpf_err(-EINVAL); in bpf_program__unpin()
8775 return libbpf_err(-errno); in bpf_program__unpin()
8777 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); in bpf_program__unpin()
8787 return libbpf_err(-EINVAL); in bpf_map__pin()
8790 if (map->fd < 0) { in bpf_map__pin()
8791 pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name); in bpf_map__pin()
8792 return libbpf_err(-EINVAL); in bpf_map__pin()
8795 if (map->pin_path) { in bpf_map__pin()
8796 if (path && strcmp(path, map->pin_path)) { in bpf_map__pin()
8798 bpf_map__name(map), map->pin_path, path); in bpf_map__pin()
8799 return libbpf_err(-EINVAL); in bpf_map__pin()
8800 } else if (map->pinned) { in bpf_map__pin()
8801 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", in bpf_map__pin()
8802 bpf_map__name(map), map->pin_path); in bpf_map__pin()
8809 return libbpf_err(-EINVAL); in bpf_map__pin()
8810 } else if (map->pinned) { in bpf_map__pin()
8812 return libbpf_err(-EEXIST); in bpf_map__pin()
8815 map->pin_path = strdup(path); in bpf_map__pin()
8816 if (!map->pin_path) { in bpf_map__pin()
8817 err = -errno; in bpf_map__pin()
8822 err = make_parent_dir(map->pin_path); in bpf_map__pin()
8826 err = check_path(map->pin_path); in bpf_map__pin()
8830 if (bpf_obj_pin(map->fd, map->pin_path)) { in bpf_map__pin()
8831 err = -errno; in bpf_map__pin()
8835 map->pinned = true; in bpf_map__pin()
8836 pr_debug("pinned map '%s'\n", map->pin_path); in bpf_map__pin()
8851 return libbpf_err(-EINVAL); in bpf_map__unpin()
8854 if (map->pin_path) { in bpf_map__unpin()
8855 if (path && strcmp(path, map->pin_path)) { in bpf_map__unpin()
8857 bpf_map__name(map), map->pin_path, path); in bpf_map__unpin()
8858 return libbpf_err(-EINVAL); in bpf_map__unpin()
8860 path = map->pin_path; in bpf_map__unpin()
8864 return libbpf_err(-EINVAL); in bpf_map__unpin()
8873 return libbpf_err(-errno); in bpf_map__unpin()
8875 map->pinned = false; in bpf_map__unpin()
8888 return libbpf_err(-errno); in bpf_map__set_pin_path()
8891 free(map->pin_path); in bpf_map__set_pin_path()
8892 map->pin_path = new; in bpf_map__set_pin_path()
8901 return map->pin_path; in bpf_map__pin_path()
8906 return map->pinned; in bpf_map__is_pinned()
8925 return libbpf_err(-ENOENT); in bpf_object__pin_maps()
8927 if (obj->state < OBJ_PREPARED) { in bpf_object__pin_maps()
8929 return libbpf_err(-ENOENT); in bpf_object__pin_maps()
8936 if (!map->autocreate) in bpf_object__pin_maps()
8945 } else if (!map->pin_path) { in bpf_object__pin_maps()
8958 if (!map->pin_path) in bpf_object__pin_maps()
8973 return libbpf_err(-ENOENT); in bpf_object__unpin_maps()
8985 } else if (!map->pin_path) { in bpf_object__unpin_maps()
9004 return libbpf_err(-ENOENT); in bpf_object__pin_programs()
9006 if (obj->state < OBJ_LOADED) { in bpf_object__pin_programs()
9008 return libbpf_err(-ENOENT); in bpf_object__pin_programs()
9012 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__pin_programs()
9025 if (pathname_concat(buf, sizeof(buf), path, prog->name)) in bpf_object__pin_programs()
9040 return libbpf_err(-ENOENT); in bpf_object__unpin_programs()
9045 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__unpin_programs()
9091 if (map->inner_map) { in bpf_map__destroy()
9092 bpf_map__destroy(map->inner_map); in bpf_map__destroy()
9093 zfree(&map->inner_map); in bpf_map__destroy()
9096 zfree(&map->init_slots); in bpf_map__destroy()
9097 map->init_slots_sz = 0; in bpf_map__destroy()
9099 if (map->mmaped && map->mmaped != map->obj->arena_data) in bpf_map__destroy()
9100 munmap(map->mmaped, bpf_map_mmap_sz(map)); in bpf_map__destroy()
9101 map->mmaped = NULL; in bpf_map__destroy()
9103 if (map->st_ops) { in bpf_map__destroy()
9104 zfree(&map->st_ops->data); in bpf_map__destroy()
9105 zfree(&map->st_ops->progs); in bpf_map__destroy()
9106 zfree(&map->st_ops->kern_func_off); in bpf_map__destroy()
9107 zfree(&map->st_ops); in bpf_map__destroy()
9110 zfree(&map->name); in bpf_map__destroy()
9111 zfree(&map->real_name); in bpf_map__destroy()
9112 zfree(&map->pin_path); in bpf_map__destroy()
9114 if (map->fd >= 0) in bpf_map__destroy()
9115 zclose(map->fd); in bpf_map__destroy()
9132 usdt_manager_free(obj->usdt_man); in bpf_object__close()
9133 obj->usdt_man = NULL; in bpf_object__close()
9135 bpf_gen__free(obj->gen_loader); in bpf_object__close()
9138 btf__free(obj->btf); in bpf_object__close()
9139 btf__free(obj->btf_vmlinux); in bpf_object__close()
9140 btf_ext__free(obj->btf_ext); in bpf_object__close()
9142 for (i = 0; i < obj->nr_maps; i++) in bpf_object__close()
9143 bpf_map__destroy(&obj->maps[i]); in bpf_object__close()
9145 zfree(&obj->btf_custom_path); in bpf_object__close()
9146 zfree(&obj->kconfig); in bpf_object__close()
9148 for (i = 0; i < obj->nr_extern; i++) in bpf_object__close()
9149 zfree(&obj->externs[i].essent_name); in bpf_object__close()
9151 zfree(&obj->externs); in bpf_object__close()
9152 obj->nr_extern = 0; in bpf_object__close()
9154 zfree(&obj->maps); in bpf_object__close()
9155 obj->nr_maps = 0; in bpf_object__close()
9157 if (obj->programs && obj->nr_programs) { in bpf_object__close()
9158 for (i = 0; i < obj->nr_programs; i++) in bpf_object__close()
9159 bpf_program__exit(&obj->programs[i]); in bpf_object__close()
9161 zfree(&obj->programs); in bpf_object__close()
9163 zfree(&obj->feat_cache); in bpf_object__close()
9164 zfree(&obj->token_path); in bpf_object__close()
9165 if (obj->token_fd > 0) in bpf_object__close()
9166 close(obj->token_fd); in bpf_object__close()
9168 zfree(&obj->arena_data); in bpf_object__close()
9175 return obj ? obj->name : libbpf_err_ptr(-EINVAL); in bpf_object__name()
9180 return obj ? obj->kern_version : 0; in bpf_object__kversion()
9185 return obj->token_fd ?: -1; in bpf_object__token_fd()
9190 return obj ? obj->btf : NULL; in bpf_object__btf()
9195 return obj->btf ? btf__fd(obj->btf) : -1; in bpf_object__btf_fd()
9200 if (obj->state >= OBJ_LOADED) in bpf_object__set_kversion()
9201 return libbpf_err(-EINVAL); in bpf_object__set_kversion()
9203 obj->kern_version = kern_version; in bpf_object__set_kversion()
9210 struct bpf_gen *gen; in bpf_object__gen_loader() local
9213 return libbpf_err(-EFAULT); in bpf_object__gen_loader()
9215 return libbpf_err(-EINVAL); in bpf_object__gen_loader()
9216 gen = calloc(sizeof(*gen), 1); in bpf_object__gen_loader()
9217 if (!gen) in bpf_object__gen_loader()
9218 return libbpf_err(-ENOMEM); in bpf_object__gen_loader()
9219 gen->opts = opts; in bpf_object__gen_loader()
9220 gen->swapped_endian = !is_native_endianness(obj); in bpf_object__gen_loader()
9221 obj->gen_loader = gen; in bpf_object__gen_loader()
9229 size_t nr_programs = obj->nr_programs; in __bpf_program__iter()
9237 return forward ? &obj->programs[0] : in __bpf_program__iter()
9238 &obj->programs[nr_programs - 1]; in __bpf_program__iter()
9240 if (p->obj != obj) { in __bpf_program__iter()
9245 idx = (p - obj->programs) + (forward ? 1 : -1); in __bpf_program__iter()
9246 if (idx >= obj->nr_programs || idx < 0) in __bpf_program__iter()
9248 return &obj->programs[idx]; in __bpf_program__iter()
9277 prog->prog_ifindex = ifindex; in bpf_program__set_ifindex()
9282 return prog->name; in bpf_program__name()
9287 return prog->sec_name; in bpf_program__section_name()
9292 return prog->autoload; in bpf_program__autoload()
9297 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_autoload()
9298 return libbpf_err(-EINVAL); in bpf_program__set_autoload()
9300 prog->autoload = autoload; in bpf_program__set_autoload()
9306 return prog->autoattach; in bpf_program__autoattach()
9311 prog->autoattach = autoattach; in bpf_program__set_autoattach()
9316 return prog->insns; in bpf_program__insns()
9321 return prog->insns_cnt; in bpf_program__insn_cnt()
9329 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_insns()
9330 return libbpf_err(-EBUSY); in bpf_program__set_insns()
9332 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); in bpf_program__set_insns()
9335 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); in bpf_program__set_insns()
9336 return libbpf_err(-ENOMEM); in bpf_program__set_insns()
9340 prog->insns = insns; in bpf_program__set_insns()
9341 prog->insns_cnt = new_insn_cnt; in bpf_program__set_insns()
9348 return libbpf_err(-EINVAL); in bpf_program__fd()
9350 if (prog->fd < 0) in bpf_program__fd()
9351 return libbpf_err(-ENOENT); in bpf_program__fd()
9353 return prog->fd; in bpf_program__fd()
9361 return prog->type; in bpf_program__type()
9372 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_type()
9373 return libbpf_err(-EBUSY); in bpf_program__set_type()
9376 if (prog->type == type) in bpf_program__set_type()
9379 prog->type = type; in bpf_program__set_type()
9383 * fallback handler, which by definition is program type-agnostic and in bpf_program__set_type()
9384 * is a catch-all custom handler, optionally set by the application, in bpf_program__set_type()
9387 if (prog->sec_def != &custom_fallback_def) in bpf_program__set_type()
9388 prog->sec_def = NULL; in bpf_program__set_type()
9397 return prog->expected_attach_type; in bpf_program__expected_attach_type()
9403 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_expected_attach_type()
9404 return libbpf_err(-EBUSY); in bpf_program__set_expected_attach_type()
9406 prog->expected_attach_type = type; in bpf_program__set_expected_attach_type()
9412 return prog->prog_flags; in bpf_program__flags()
9417 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_flags()
9418 return libbpf_err(-EBUSY); in bpf_program__set_flags()
9420 prog->prog_flags = flags; in bpf_program__set_flags()
9426 return prog->log_level; in bpf_program__log_level()
9431 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_log_level()
9432 return libbpf_err(-EBUSY); in bpf_program__set_log_level()
9434 prog->log_level = log_level; in bpf_program__set_log_level()
9440 *log_size = prog->log_size; in bpf_program__log_buf()
9441 return prog->log_buf; in bpf_program__log_buf()
9447 return libbpf_err(-EINVAL); in bpf_program__set_log_buf()
9448 if (prog->log_size > UINT_MAX) in bpf_program__set_log_buf()
9449 return libbpf_err(-EINVAL); in bpf_program__set_log_buf()
9450 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_log_buf()
9451 return libbpf_err(-EBUSY); in bpf_program__set_log_buf()
9453 prog->log_buf = log_buf; in bpf_program__set_log_buf()
9454 prog->log_size = log_size; in bpf_program__set_log_buf()
9525 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace),
9594 return libbpf_err(-EINVAL); in libbpf_register_prog_handler()
9597 return libbpf_err(-E2BIG); in libbpf_register_prog_handler()
9603 return libbpf_err(-ENOMEM); in libbpf_register_prog_handler()
9609 return libbpf_err(-EBUSY); in libbpf_register_prog_handler()
9614 sec_def->sec = sec ? strdup(sec) : NULL; in libbpf_register_prog_handler()
9615 if (sec && !sec_def->sec) in libbpf_register_prog_handler()
9616 return libbpf_err(-ENOMEM); in libbpf_register_prog_handler()
9618 sec_def->prog_type = prog_type; in libbpf_register_prog_handler()
9619 sec_def->expected_attach_type = exp_attach_type; in libbpf_register_prog_handler()
9620 sec_def->cookie = OPTS_GET(opts, cookie, 0); in libbpf_register_prog_handler()
9622 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL); in libbpf_register_prog_handler()
9623 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL); in libbpf_register_prog_handler()
9624 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL); in libbpf_register_prog_handler()
9626 sec_def->handler_id = ++last_custom_sec_def_handler_id; in libbpf_register_prog_handler()
9633 return sec_def->handler_id; in libbpf_register_prog_handler()
9642 return libbpf_err(-EINVAL); in libbpf_unregister_prog_handler()
9656 return libbpf_err(-ENOENT); in libbpf_unregister_prog_handler()
9660 custom_sec_defs[i - 1] = custom_sec_defs[i]; in libbpf_unregister_prog_handler()
9661 custom_sec_def_cnt--; in libbpf_unregister_prog_handler()
9677 size_t len = strlen(sec_def->sec); in sec_def_matches()
9680 if (sec_def->sec[len - 1] == '/') { in sec_def_matches()
9681 if (str_has_pfx(sec_name, sec_def->sec)) in sec_def_matches()
9687 * well-formed SEC("type/extras") with proper '/' separator in sec_def_matches()
9689 if (sec_def->sec[len - 1] == '+') { in sec_def_matches()
9690 len--; in sec_def_matches()
9692 if (strncmp(sec_name, sec_def->sec, len) != 0) in sec_def_matches()
9700 return strcmp(sec_name, sec_def->sec) == 0; in sec_def_matches()
9745 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) in libbpf_get_type_names()
9748 if (!(sec_def->cookie & SEC_ATTACHABLE)) in libbpf_get_type_names()
9770 return libbpf_err(-EINVAL); in libbpf_prog_type_by_name()
9774 *prog_type = sec_def->prog_type; in libbpf_prog_type_by_name()
9775 *expected_attach_type = sec_def->expected_attach_type; in libbpf_prog_type_by_name()
9786 return libbpf_err(-ESRCH); in libbpf_prog_type_by_name()
9828 for (i = 0; i < obj->nr_maps; i++) { in find_struct_ops_map_by_offset()
9829 map = &obj->maps[i]; in find_struct_ops_map_by_offset()
9832 if (map->sec_idx == sec_idx && in find_struct_ops_map_by_offset()
9833 map->sec_offset <= offset && in find_struct_ops_map_by_offset()
9834 offset - map->sec_offset < map->def.value_size) in find_struct_ops_map_by_offset()
9841 /* Collect the reloc from ELF, populate the st_ops->progs[], and update
9842 * st_ops->data for shadow type.
9861 btf = obj->btf; in bpf_object__collect_st_ops_relos()
9862 nrels = shdr->sh_size / shdr->sh_entsize; in bpf_object__collect_st_ops_relos()
9867 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_st_ops_relos()
9870 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); in bpf_object__collect_st_ops_relos()
9873 (size_t)ELF64_R_SYM(rel->r_info)); in bpf_object__collect_st_ops_relos()
9874 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_st_ops_relos()
9877 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; in bpf_object__collect_st_ops_relos()
9878 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); in bpf_object__collect_st_ops_relos()
9880 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n", in bpf_object__collect_st_ops_relos()
9881 (size_t)rel->r_offset); in bpf_object__collect_st_ops_relos()
9882 return -EINVAL; in bpf_object__collect_st_ops_relos()
9885 moff = rel->r_offset - map->sec_offset; in bpf_object__collect_st_ops_relos()
9886 shdr_idx = sym->st_shndx; in bpf_object__collect_st_ops_relos()
9887 st_ops = map->st_ops; in bpf_object__collect_st_ops_relos()
9888 …pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %… in bpf_object__collect_st_ops_relos()
9889 map->name, in bpf_object__collect_st_ops_relos()
9890 (long long)(rel->r_info >> 32), in bpf_object__collect_st_ops_relos()
9891 (long long)sym->st_value, in bpf_object__collect_st_ops_relos()
9892 shdr_idx, (size_t)rel->r_offset, in bpf_object__collect_st_ops_relos()
9893 map->sec_offset, sym->st_name, name); in bpf_object__collect_st_ops_relos()
9896 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n", in bpf_object__collect_st_ops_relos()
9897 map->name, (size_t)rel->r_offset, shdr_idx); in bpf_object__collect_st_ops_relos()
9898 return -LIBBPF_ERRNO__RELOC; in bpf_object__collect_st_ops_relos()
9900 if (sym->st_value % BPF_INSN_SZ) { in bpf_object__collect_st_ops_relos()
9902 map->name, (unsigned long long)sym->st_value); in bpf_object__collect_st_ops_relos()
9903 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_st_ops_relos()
9905 insn_idx = sym->st_value / BPF_INSN_SZ; in bpf_object__collect_st_ops_relos()
9907 type = btf__type_by_id(btf, st_ops->type_id); in bpf_object__collect_st_ops_relos()
9911 map->name, moff); in bpf_object__collect_st_ops_relos()
9912 return -EINVAL; in bpf_object__collect_st_ops_relos()
9914 member_idx = member - btf_members(type); in bpf_object__collect_st_ops_relos()
9915 name = btf__name_by_offset(btf, member->name_off); in bpf_object__collect_st_ops_relos()
9917 if (!resolve_func_ptr(btf, member->type, NULL)) { in bpf_object__collect_st_ops_relos()
9919 map->name, name); in bpf_object__collect_st_ops_relos()
9920 return -EINVAL; in bpf_object__collect_st_ops_relos()
9926 map->name, shdr_idx, name); in bpf_object__collect_st_ops_relos()
9927 return -EINVAL; in bpf_object__collect_st_ops_relos()
9931 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { in bpf_object__collect_st_ops_relos()
9933 map->name, prog->name); in bpf_object__collect_st_ops_relos()
9934 return -EINVAL; in bpf_object__collect_st_ops_relos()
9937 st_ops->progs[member_idx] = prog; in bpf_object__collect_st_ops_relos()
9939 /* st_ops->data will be exposed to users, being returned by in bpf_object__collect_st_ops_relos()
9945 *((struct bpf_program **)(st_ops->data + moff)) = prog; in bpf_object__collect_st_ops_relos()
9992 return -ENAMETOOLONG; in find_btf_by_prefix_kind()
10042 err = -EINVAL; in libbpf_find_prog_btf_id()
10073 mod_len = fn_name - mod_name; in find_kernel_btf_id()
10078 ret = find_attach_btf_id(obj->btf_vmlinux, in find_kernel_btf_id()
10086 if (ret != -ENOENT) in find_kernel_btf_id()
10094 for (i = 0; i < obj->btf_module_cnt; i++) { in find_kernel_btf_id()
10095 const struct module_btf *mod = &obj->btf_modules[i]; in find_kernel_btf_id()
10097 if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0) in find_kernel_btf_id()
10100 ret = find_attach_btf_id(mod->btf, in find_kernel_btf_id()
10104 *btf_obj_fd = mod->fd; in find_kernel_btf_id()
10108 if (ret == -ENOENT) in find_kernel_btf_id()
10114 return -ESRCH; in find_kernel_btf_id()
10120 enum bpf_attach_type attach_type = prog->expected_attach_type; in libbpf_find_attach_btf_id()
10121 __u32 attach_prog_fd = prog->attach_prog_fd; in libbpf_find_attach_btf_id()
10125 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { in libbpf_find_attach_btf_id()
10127 pr_warn("prog '%s': attach program FD is not set\n", prog->name); in libbpf_find_attach_btf_id()
10128 return -EINVAL; in libbpf_find_attach_btf_id()
10130 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd); in libbpf_find_attach_btf_id()
10133 prog->name, attach_prog_fd, attach_name, errstr(err)); in libbpf_find_attach_btf_id()
10142 if (prog->obj->gen_loader) { in libbpf_find_attach_btf_id()
10143 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); in libbpf_find_attach_btf_id()
10147 err = find_kernel_btf_id(prog->obj, attach_name, in libbpf_find_attach_btf_id()
10153 prog->name, attach_name, errstr(err)); in libbpf_find_attach_btf_id()
10166 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
10177 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
10180 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) in libbpf_attach_type_by_name()
10181 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
10182 if (!(sec_def->cookie & SEC_ATTACHABLE)) in libbpf_attach_type_by_name()
10183 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
10185 *attach_type = sec_def->expected_attach_type; in libbpf_attach_type_by_name()
10192 return libbpf_err(-EINVAL); in bpf_map__fd()
10194 return -1; in bpf_map__fd()
10195 return map->fd; in bpf_map__fd()
10201 * their user-visible name differs from kernel-visible name. Users see in map_uses_real_name()
10206 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) in map_uses_real_name()
10208 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) in map_uses_real_name()
10219 return map->real_name; in bpf_map__name()
10221 return map->name; in bpf_map__name()
10226 return map->def.type; in bpf_map__type()
10232 return libbpf_err(-EBUSY); in bpf_map__set_type()
10233 map->def.type = type; in bpf_map__set_type()
10239 return map->def.map_flags; in bpf_map__map_flags()
10245 return libbpf_err(-EBUSY); in bpf_map__set_map_flags()
10246 map->def.map_flags = flags; in bpf_map__set_map_flags()
10252 return map->map_extra; in bpf_map__map_extra()
10258 return libbpf_err(-EBUSY); in bpf_map__set_map_extra()
10259 map->map_extra = map_extra; in bpf_map__set_map_extra()
10265 return map->numa_node; in bpf_map__numa_node()
10271 return libbpf_err(-EBUSY); in bpf_map__set_numa_node()
10272 map->numa_node = numa_node; in bpf_map__set_numa_node()
10278 return map->def.key_size; in bpf_map__key_size()
10284 return libbpf_err(-EBUSY); in bpf_map__set_key_size()
10285 map->def.key_size = size; in bpf_map__set_key_size()
10291 return map->def.value_size; in bpf_map__value_size()
10305 btf = bpf_object__btf(map->obj); in map_btf_datasec_resize()
10307 return -ENOENT; in map_btf_datasec_resize()
10314 return -EINVAL; in map_btf_datasec_resize()
10322 return -EINVAL; in map_btf_datasec_resize()
10326 var = &btf_var_secinfos(datasec_type)[vlen - 1]; in map_btf_datasec_resize()
10327 var_type = btf_type_by_id(btf, var->type); in map_btf_datasec_resize()
10328 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL); in map_btf_datasec_resize()
10332 return -EINVAL; in map_btf_datasec_resize()
10337 element_sz = btf__resolve_size(btf, array->type); in map_btf_datasec_resize()
10338 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) { in map_btf_datasec_resize()
10341 return -EINVAL; in map_btf_datasec_resize()
10345 nr_elements = (size - var->offset) / element_sz; in map_btf_datasec_resize()
10346 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements); in map_btf_datasec_resize()
10353 datasec_type = btf_type_by_id(btf, map->btf_value_type_id); in map_btf_datasec_resize()
10354 var = &btf_var_secinfos(datasec_type)[vlen - 1]; in map_btf_datasec_resize()
10355 var_type = btf_type_by_id(btf, var->type); in map_btf_datasec_resize()
10358 datasec_type->size = size; in map_btf_datasec_resize()
10359 var->size = size - var->offset; in map_btf_datasec_resize()
10360 var_type->type = new_array_id; in map_btf_datasec_resize()
10368 return libbpf_err(-EBUSY); in bpf_map__set_value_size()
10370 if (map->mmaped) { in bpf_map__set_value_size()
10374 if (map->def.type != BPF_MAP_TYPE_ARRAY) in bpf_map__set_value_size()
10375 return libbpf_err(-EOPNOTSUPP); in bpf_map__set_value_size()
10378 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries); in bpf_map__set_value_size()
10381 pr_warn("map '%s': failed to resize memory-mapped region: %s\n", in bpf_map__set_value_size()
10386 if (err && err != -ENOENT) { in bpf_map__set_value_size()
10389 map->btf_value_type_id = 0; in bpf_map__set_value_size()
10390 map->btf_key_type_id = 0; in bpf_map__set_value_size()
10394 map->def.value_size = size; in bpf_map__set_value_size()
10400 return map ? map->btf_key_type_id : 0; in bpf_map__btf_key_type_id()
10405 return map ? map->btf_value_type_id : 0; in bpf_map__btf_value_type_id()
10414 return libbpf_err(-EBUSY); in bpf_map__set_initial_value()
10416 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG) in bpf_map__set_initial_value()
10417 return libbpf_err(-EINVAL); in bpf_map__set_initial_value()
10419 if (map->def.type == BPF_MAP_TYPE_ARENA) in bpf_map__set_initial_value()
10420 actual_sz = map->obj->arena_data_sz; in bpf_map__set_initial_value()
10422 actual_sz = map->def.value_size; in bpf_map__set_initial_value()
10424 return libbpf_err(-EINVAL); in bpf_map__set_initial_value()
10426 memcpy(map->mmaped, data, size); in bpf_map__set_initial_value()
10434 *psize = map->def.value_size; in bpf_map__initial_value()
10435 return map->st_ops->data; in bpf_map__initial_value()
10438 if (!map->mmaped) in bpf_map__initial_value()
10441 if (map->def.type == BPF_MAP_TYPE_ARENA) in bpf_map__initial_value()
10442 *psize = map->obj->arena_data_sz; in bpf_map__initial_value()
10444 *psize = map->def.value_size; in bpf_map__initial_value()
10446 return map->mmaped; in bpf_map__initial_value()
10451 return map->libbpf_type != LIBBPF_MAP_UNSPEC; in bpf_map__is_internal()
10456 return map->map_ifindex; in bpf_map__ifindex()
10462 return libbpf_err(-EBUSY); in bpf_map__set_ifindex()
10463 map->map_ifindex = ifindex; in bpf_map__set_ifindex()
10469 if (!bpf_map_type__is_map_in_map(map->def.type)) { in bpf_map__set_inner_map_fd()
10471 return libbpf_err(-EINVAL); in bpf_map__set_inner_map_fd()
10473 if (map->inner_map_fd != -1) { in bpf_map__set_inner_map_fd()
10475 return libbpf_err(-EINVAL); in bpf_map__set_inner_map_fd()
10477 if (map->inner_map) { in bpf_map__set_inner_map_fd()
10478 bpf_map__destroy(map->inner_map); in bpf_map__set_inner_map_fd()
10479 zfree(&map->inner_map); in bpf_map__set_inner_map_fd()
10481 map->inner_map_fd = fd; in bpf_map__set_inner_map_fd()
10491 if (!obj || !obj->maps) in __bpf_map__iter()
10494 s = obj->maps; in __bpf_map__iter()
10495 e = obj->maps + obj->nr_maps; in __bpf_map__iter()
10503 idx = (m - obj->maps) + i; in __bpf_map__iter()
10504 if (idx >= obj->nr_maps || idx < 0) in __bpf_map__iter()
10506 return &obj->maps[idx]; in __bpf_map__iter()
10513 return obj->maps; in bpf_object__next_map()
10522 if (!obj->nr_maps) in bpf_object__prev_map()
10524 return obj->maps + obj->nr_maps - 1; in bpf_object__prev_map()
10527 return __bpf_map__iter(next, obj, -1); in bpf_object__prev_map()
10541 if (pos->real_name && strcmp(pos->real_name, name) == 0) in bpf_object__find_map_by_name()
10547 if (strcmp(pos->real_name, name) == 0) in bpf_object__find_map_by_name()
10551 if (strcmp(pos->name, name) == 0) in bpf_object__find_map_by_name()
10567 return -ENOENT; in validate_map_op()
10569 if (map->def.key_size != key_sz) { in validate_map_op()
10571 map->name, key_sz, map->def.key_size); in validate_map_op()
10572 return -EINVAL; in validate_map_op()
10575 if (map->fd < 0) { in validate_map_op()
10576 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name); in validate_map_op()
10577 return -EINVAL; in validate_map_op()
10583 switch (map->def.type) { in validate_map_op()
10589 size_t elem_sz = roundup(map->def.value_size, 8); in validate_map_op()
10592 … pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", in validate_map_op()
10593 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); in validate_map_op()
10594 return -EINVAL; in validate_map_op()
10599 if (map->def.value_size != value_sz) { in validate_map_op()
10601 map->name, value_sz, map->def.value_size); in validate_map_op()
10602 return -EINVAL; in validate_map_op()
10619 return bpf_map_lookup_elem_flags(map->fd, key, value, flags); in bpf_map__lookup_elem()
10632 return bpf_map_update_elem(map->fd, key, value, flags); in bpf_map__update_elem()
10644 return bpf_map_delete_elem_flags(map->fd, key, flags); in bpf_map__delete_elem()
10657 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); in bpf_map__lookup_and_delete_elem()
10669 return bpf_map_get_next_key(map->fd, cur_key, next_key); in bpf_map__get_next_key()
10678 errno = -PTR_ERR(ptr); in libbpf_get_error()
10685 return -errno; in libbpf_get_error()
10696 prog->name); in bpf_link__update_program()
10697 return libbpf_err(-EINVAL); in bpf_link__update_program()
10716 link->disconnected = true; in bpf_link__disconnect()
10726 if (!link->disconnected && link->detach) in bpf_link__destroy()
10727 err = link->detach(link); in bpf_link__destroy()
10728 if (link->pin_path) in bpf_link__destroy()
10729 free(link->pin_path); in bpf_link__destroy()
10730 if (link->dealloc) in bpf_link__destroy()
10731 link->dealloc(link); in bpf_link__destroy()
10740 return link->fd; in bpf_link__fd()
10745 return link->pin_path; in bpf_link__pin_path()
10750 return libbpf_err_errno(close(link->fd)); in bpf_link__detach_fd()
10760 fd = -errno; in bpf_link__open()
10768 return libbpf_err_ptr(-ENOMEM); in bpf_link__open()
10770 link->detach = &bpf_link__detach_fd; in bpf_link__open()
10771 link->fd = fd; in bpf_link__open()
10773 link->pin_path = strdup(path); in bpf_link__open()
10774 if (!link->pin_path) { in bpf_link__open()
10776 return libbpf_err_ptr(-ENOMEM); in bpf_link__open()
10784 return bpf_link_detach(link->fd) ? -errno : 0; in bpf_link__detach()
10791 if (link->pin_path) in bpf_link__pin()
10792 return libbpf_err(-EBUSY); in bpf_link__pin()
10800 link->pin_path = strdup(path); in bpf_link__pin()
10801 if (!link->pin_path) in bpf_link__pin()
10802 return libbpf_err(-ENOMEM); in bpf_link__pin()
10804 if (bpf_obj_pin(link->fd, link->pin_path)) { in bpf_link__pin()
10805 err = -errno; in bpf_link__pin()
10806 zfree(&link->pin_path); in bpf_link__pin()
10810 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); in bpf_link__pin()
10818 if (!link->pin_path) in bpf_link__unpin()
10819 return libbpf_err(-EINVAL); in bpf_link__unpin()
10821 err = unlink(link->pin_path); in bpf_link__unpin()
10823 return -errno; in bpf_link__unpin()
10825 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); in bpf_link__unpin()
10826 zfree(&link->pin_path); in bpf_link__unpin()
10847 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) in bpf_link_perf_detach()
10848 err = -errno; in bpf_link_perf_detach()
10850 if (perf_link->perf_event_fd != link->fd) in bpf_link_perf_detach()
10851 close(perf_link->perf_event_fd); in bpf_link_perf_detach()
10852 close(link->fd); in bpf_link_perf_detach()
10855 if (perf_link->legacy_probe_name) { in bpf_link_perf_detach()
10856 if (perf_link->legacy_is_kprobe) { in bpf_link_perf_detach()
10857 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, in bpf_link_perf_detach()
10858 perf_link->legacy_is_retprobe); in bpf_link_perf_detach()
10860 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name, in bpf_link_perf_detach()
10861 perf_link->legacy_is_retprobe); in bpf_link_perf_detach()
10872 free(perf_link->legacy_probe_name); in bpf_link_perf_dealloc()
10880 int prog_fd, link_fd = -1, err; in bpf_program__attach_perf_event_opts()
10884 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_perf_event_opts()
10888 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10889 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_perf_event_opts()
10894 prog->name); in bpf_program__attach_perf_event_opts()
10895 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_perf_event_opts()
10900 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_perf_event_opts()
10901 link->link.detach = &bpf_link_perf_detach; in bpf_program__attach_perf_event_opts()
10902 link->link.dealloc = &bpf_link_perf_dealloc; in bpf_program__attach_perf_event_opts()
10903 link->perf_event_fd = pfd; in bpf_program__attach_perf_event_opts()
10906 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { in bpf_program__attach_perf_event_opts()
10912 err = -errno; in bpf_program__attach_perf_event_opts()
10914 prog->name, pfd, errstr(err)); in bpf_program__attach_perf_event_opts()
10917 link->link.fd = link_fd; in bpf_program__attach_perf_event_opts()
10920 pr_warn("prog '%s': user context value is not supported\n", prog->name); in bpf_program__attach_perf_event_opts()
10921 err = -EOPNOTSUPP; in bpf_program__attach_perf_event_opts()
10926 err = -errno; in bpf_program__attach_perf_event_opts()
10928 prog->name, pfd, errstr(err)); in bpf_program__attach_perf_event_opts()
10929 if (err == -EPROTO) in bpf_program__attach_perf_event_opts()
10931 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10934 link->link.fd = pfd; in bpf_program__attach_perf_event_opts()
10937 err = -errno; in bpf_program__attach_perf_event_opts()
10939 prog->name, pfd, errstr(err)); in bpf_program__attach_perf_event_opts()
10943 return &link->link; in bpf_program__attach_perf_event_opts()
10957 * this function is expected to parse integer in the range of [0, 2^31-1] from
10968 err = -errno; in parse_uint_from_file()
10974 err = err == EOF ? -EIO : -errno; in parse_uint_from_file()
11022 return -EINVAL; in perf_event_open_probe()
11054 pid < 0 ? -1 : pid /* pid */, in perf_event_open_probe()
11055 pid == -1 ? 0 : -1 /* cpu */, in perf_event_open_probe()
11056 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_open_probe()
11057 return pfd >= 0 ? pfd : -errno; in perf_event_open_probe()
11071 return -EINVAL; in append_to_file()
11075 return -errno; in append_to_file()
11078 err = -errno; in append_to_file()
11089 static int has_debugfs = -1; in use_debugfs()
11151 return append_to_file(tracefs_kprobe_events(), "-:%s/%s", in remove_kprobe_event_legacy()
11194 pid < 0 ? -1 : pid, /* pid */ in perf_event_kprobe_open_legacy()
11195 pid == -1 ? 0 : -1, /* cpu */ in perf_event_kprobe_open_legacy()
11196 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_kprobe_open_legacy()
11198 err = -errno; in perf_event_kprobe_open_legacy()
11283 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_opts()
11298 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_kprobe_opts()
11302 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_kprobe_opts()
11303 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_kprobe_opts()
11308 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_opts()
11314 -1 /* pid */, 0 /* ref_ctr_off */); in bpf_program__attach_kprobe_opts()
11323 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_kprobe_opts()
11326 offset, -1 /* pid */); in bpf_program__attach_kprobe_opts()
11329 err = -errno; in bpf_program__attach_kprobe_opts()
11331 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
11341 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
11349 perf_link->legacy_probe_name = legacy_probe; in bpf_program__attach_kprobe_opts()
11350 perf_link->legacy_is_kprobe = true; in bpf_program__attach_kprobe_opts()
11351 perf_link->legacy_is_retprobe = retprobe; in bpf_program__attach_kprobe_opts()
11383 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_ksyscall()
11385 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { in bpf_program__attach_ksyscall()
11452 struct kprobe_multi_resolve *res = data->res; in avail_kallsyms_cb()
11455 if (!glob_match(sym_name, res->pattern)) in avail_kallsyms_cb()
11458 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) { in avail_kallsyms_cb()
11463 * make multi-kprobe usability a bit better: if no match is in avail_kallsyms_cb()
11477 snprintf(sym_trim, sizeof(sym_trim), "%.*s", (int)(sym_sfx - sym_name), sym_name); in avail_kallsyms_cb()
11478 if (!bsearch(&psym_trim, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) in avail_kallsyms_cb()
11482 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); in avail_kallsyms_cb()
11486 res->addrs[res->cnt++] = (unsigned long)sym_addr; in avail_kallsyms_cb()
11502 err = -errno; in libbpf_available_kallsyms_parse()
11516 err = -EINVAL; in libbpf_available_kallsyms_parse()
11520 if (!glob_match(sym_name, res->pattern)) in libbpf_available_kallsyms_parse()
11529 err = -errno; in libbpf_available_kallsyms_parse()
11538 err = -ENOENT; in libbpf_available_kallsyms_parse()
11550 if (res->cnt == 0) in libbpf_available_kallsyms_parse()
11551 err = -ENOENT; in libbpf_available_kallsyms_parse()
11564 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; in has_available_filter_functions_addrs()
11577 err = -errno; in libbpf_available_kprobes_parse()
11590 err = -EINVAL; in libbpf_available_kprobes_parse()
11594 if (!glob_match(sym_name, res->pattern)) in libbpf_available_kprobes_parse()
11597 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, in libbpf_available_kprobes_parse()
11598 sizeof(*res->addrs), res->cnt + 1); in libbpf_available_kprobes_parse()
11602 res->addrs[res->cnt++] = (unsigned long)sym_addr; in libbpf_available_kprobes_parse()
11605 if (res->cnt == 0) in libbpf_available_kprobes_parse()
11606 err = -ENOENT; in libbpf_available_kprobes_parse()
11632 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11637 prog->name); in bpf_program__attach_kprobe_multi_opts()
11638 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11648 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11650 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11652 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11654 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11656 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11668 prog->name, pattern, res.cnt); in bpf_program__attach_kprobe_multi_opts()
11669 err = -EINVAL; in bpf_program__attach_kprobe_multi_opts()
11681 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
11693 err = -ENOMEM; in bpf_program__attach_kprobe_multi_opts()
11696 link->detach = &bpf_link__detach_fd; in bpf_program__attach_kprobe_multi_opts()
11700 err = -errno; in bpf_program__attach_kprobe_multi_opts()
11702 prog->name, errstr(err)); in bpf_program__attach_kprobe_multi_opts()
11705 link->fd = link_fd; in bpf_program__attach_kprobe_multi_opts()
11725 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ in attach_kprobe()
11726 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) in attach_kprobe()
11729 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); in attach_kprobe()
11731 func_name = prog->sec_name + sizeof("kretprobe/") - 1; in attach_kprobe()
11733 func_name = prog->sec_name + sizeof("kprobe/") - 1; in attach_kprobe()
11735 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); in attach_kprobe()
11738 return -EINVAL; in attach_kprobe()
11743 return -EINVAL; in attach_kprobe()
11759 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ in attach_ksyscall()
11760 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) in attach_ksyscall()
11763 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); in attach_ksyscall()
11765 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; in attach_ksyscall()
11767 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; in attach_ksyscall()
11770 return *link ? 0 : -errno; in attach_ksyscall()
11782 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ in attach_kprobe_multi()
11783 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || in attach_kprobe_multi()
11784 strcmp(prog->sec_name, "kretprobe.multi") == 0) in attach_kprobe_multi()
11787 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); in attach_kprobe_multi()
11789 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; in attach_kprobe_multi()
11791 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; in attach_kprobe_multi()
11793 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); in attach_kprobe_multi()
11796 return -EINVAL; in attach_kprobe_multi()
11814 /* no auto-attach for SEC("kprobe.session") */ in attach_kprobe_session()
11815 if (strcmp(prog->sec_name, "kprobe.session") == 0) in attach_kprobe_session()
11818 spec = prog->sec_name + sizeof("kprobe.session/") - 1; in attach_kprobe_session()
11819 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); in attach_kprobe_session()
11822 return -EINVAL; in attach_kprobe_session()
11827 return *link ? 0 : -errno; in attach_kprobe_session()
11834 int n, ret = -EINVAL; in attach_uprobe_multi()
11838 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe_multi()
11842 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ in attach_uprobe_multi()
11849 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); in attach_uprobe_multi()
11853 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe_multi()
11854 prog->sec_name); in attach_uprobe_multi()
11888 return append_to_file(tracefs_uprobe_events(), "-:%s/%s", in remove_uprobe_event_legacy()
11929 pid < 0 ? -1 : pid, /* pid */ in perf_event_uprobe_open_legacy()
11930 pid == -1 ? 0 : -1, /* cpu */ in perf_event_uprobe_open_legacy()
11931 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_uprobe_open_legacy()
11933 err = -errno; in perf_event_uprobe_open_legacy()
11981 ret = -LIBBPF_ERRNO__FORMAT; in elf_find_func_offset_from_archive()
11988 elf_errmsg(-1)); in elf_find_func_offset_from_archive()
11989 ret = -LIBBPF_ERRNO__LIBELF; in elf_find_func_offset_from_archive()
12016 return "/lib/x86_64-linux-gnu"; in arch_specific_lib_paths()
12018 return "/lib/i386-linux-gnu"; in arch_specific_lib_paths()
12020 return "/lib/s390x-linux-gnu"; in arch_specific_lib_paths()
12022 return "/lib/s390-linux-gnu"; in arch_specific_lib_paths()
12024 return "/lib/arm-linux-gnueabi"; in arch_specific_lib_paths()
12026 return "/lib/arm-linux-gnueabihf"; in arch_specific_lib_paths()
12028 return "/lib/aarch64-linux-gnu"; in arch_specific_lib_paths()
12030 return "/lib/mips64el-linux-gnuabi64"; in arch_specific_lib_paths()
12032 return "/lib/mipsel-linux-gnu"; in arch_specific_lib_paths()
12034 return "/lib/powerpc64le-linux-gnu"; in arch_specific_lib_paths()
12036 return "/lib/sparc64-linux-gnu"; in arch_specific_lib_paths()
12038 return "/lib/riscv64-linux-gnu"; in arch_specific_lib_paths()
12073 seg_len = next_path ? next_path - s : strlen(s); in resolve_full_path()
12084 return -ENOENT; in resolve_full_path()
12107 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
12112 prog->name); in bpf_program__attach_uprobe_multi()
12113 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
12132 * - syms and offsets are mutually exclusive in bpf_program__attach_uprobe_multi()
12133 * - ref_ctr_offsets and cookies are optional in bpf_program__attach_uprobe_multi()
12139 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
12141 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
12145 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
12148 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
12152 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
12159 prog->name, path, errstr(err)); in bpf_program__attach_uprobe_multi()
12193 err = -ENOMEM; in bpf_program__attach_uprobe_multi()
12196 link->detach = &bpf_link__detach_fd; in bpf_program__attach_uprobe_multi()
12200 err = -errno; in bpf_program__attach_uprobe_multi()
12201 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n", in bpf_program__attach_uprobe_multi()
12202 prog->name, errstr(err)); in bpf_program__attach_uprobe_multi()
12205 link->fd = link_fd; in bpf_program__attach_uprobe_multi()
12232 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
12240 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
12247 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1))); in bpf_program__attach_uprobe_opts()
12254 prog->name, binary_path, errstr(err)); in bpf_program__attach_uprobe_opts()
12283 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_uprobe_opts()
12287 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_uprobe_opts()
12288 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_uprobe_opts()
12293 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
12303 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
12310 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_uprobe_opts()
12316 err = -errno; in bpf_program__attach_uprobe_opts()
12318 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
12329 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
12337 perf_link->legacy_probe_name = legacy_probe; in bpf_program__attach_uprobe_opts()
12338 perf_link->legacy_is_kprobe = false; in bpf_program__attach_uprobe_opts()
12339 perf_link->legacy_is_retprobe = retprobe; in bpf_program__attach_uprobe_opts()
12351 /* Format of u[ret]probe section definition supporting auto-attach:
12358 * specified (and auto-attach is not possible) or the above format is specified for
12359 * auto-attach.
12365 int n, c, ret = -EINVAL; in attach_uprobe()
12370 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe()
12374 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ in attach_uprobe()
12379 prog->name, prog->sec_name); in attach_uprobe()
12397 prog->name); in attach_uprobe()
12401 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); in attach_uprobe()
12405 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe()
12406 prog->sec_name); in attach_uprobe()
12432 struct bpf_object *obj = prog->obj; in bpf_program__attach_usdt()
12438 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_usdt()
12442 prog->name); in bpf_program__attach_usdt()
12443 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_usdt()
12447 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_usdt()
12453 prog->name, binary_path, errstr(err)); in bpf_program__attach_usdt()
12462 if (IS_ERR(obj->usdt_man)) in bpf_program__attach_usdt()
12463 return libbpf_ptr(obj->usdt_man); in bpf_program__attach_usdt()
12464 if (!obj->usdt_man) { in bpf_program__attach_usdt()
12465 obj->usdt_man = usdt_manager_new(obj); in bpf_program__attach_usdt()
12466 if (IS_ERR(obj->usdt_man)) in bpf_program__attach_usdt()
12467 return libbpf_ptr(obj->usdt_man); in bpf_program__attach_usdt()
12471 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, in bpf_program__attach_usdt()
12487 /* no auto-attach for just SEC("usdt") */ in attach_usdt()
12496 err = -EINVAL; in attach_usdt()
12498 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, in attach_usdt()
12517 return -errno; in determine_tracepoint_id()
12521 return -E2BIG; in determine_tracepoint_id()
12546 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, in perf_event_open_tracepoint()
12547 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_open_tracepoint()
12549 err = -errno; in perf_event_open_tracepoint()
12568 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tracepoint_opts()
12575 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
12584 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
12604 /* no auto-attach for SEC("tp") or SEC("tracepoint") */ in attach_tp()
12605 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) in attach_tp()
12608 sec_name = strdup(prog->sec_name); in attach_tp()
12610 return -ENOMEM; in attach_tp()
12613 if (str_has_pfx(prog->sec_name, "tp/")) in attach_tp()
12614 tp_cat = sec_name + sizeof("tp/") - 1; in attach_tp()
12616 tp_cat = sec_name + sizeof("tracepoint/") - 1; in attach_tp()
12620 return -EINVAL; in attach_tp()
12640 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_raw_tracepoint_opts()
12644 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_raw_tracepoint_opts()
12645 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_raw_tracepoint_opts()
12650 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_raw_tracepoint_opts()
12651 link->detach = &bpf_link__detach_fd; in bpf_program__attach_raw_tracepoint_opts()
12657 pfd = -errno; in bpf_program__attach_raw_tracepoint_opts()
12660 prog->name, tp_name, errstr(pfd)); in bpf_program__attach_raw_tracepoint_opts()
12663 link->fd = pfd; in bpf_program__attach_raw_tracepoint_opts()
12689 if (!str_has_pfx(prog->sec_name, prefixes[i])) in attach_raw_tp()
12693 /* no auto-attach case of, e.g., SEC("raw_tp") */ in attach_raw_tp()
12694 if (prog->sec_name[pfx_len] == '\0') in attach_raw_tp()
12697 if (prog->sec_name[pfx_len] != '/') in attach_raw_tp()
12700 tp_name = prog->sec_name + pfx_len + 1; in attach_raw_tp()
12706 prog->name, prog->sec_name); in attach_raw_tp()
12707 return -EINVAL; in attach_raw_tp()
12723 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_btf_id()
12727 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_btf_id()
12728 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_btf_id()
12733 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_btf_id()
12734 link->detach = &bpf_link__detach_fd; in bpf_program__attach_btf_id()
12740 pfd = -errno; in bpf_program__attach_btf_id()
12743 prog->name, errstr(pfd)); in bpf_program__attach_btf_id()
12746 link->fd = pfd; in bpf_program__attach_btf_id()
12789 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program_attach_fd()
12790 return libbpf_err_ptr(-EINVAL); in bpf_program_attach_fd()
12795 return libbpf_err_ptr(-ENOMEM); in bpf_program_attach_fd()
12796 link->detach = &bpf_link__detach_fd; in bpf_program_attach_fd()
12801 link_fd = -errno; in bpf_program_attach_fd()
12804 prog->name, target_name, in bpf_program_attach_fd()
12808 link->fd = link_fd; in bpf_program_attach_fd()
12845 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tcx()
12850 /* validate we don't have unexpected combinations of non-zero fields */ in bpf_program__attach_tcx()
12853 prog->name); in bpf_program__attach_tcx()
12854 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tcx()
12858 prog->name); in bpf_program__attach_tcx()
12859 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tcx()
12880 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_netkit()
12885 /* validate we don't have unexpected combinations of non-zero fields */ in bpf_program__attach_netkit()
12888 prog->name); in bpf_program__attach_netkit()
12889 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_netkit()
12893 prog->name); in bpf_program__attach_netkit()
12894 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_netkit()
12913 prog->name); in bpf_program__attach_freplace()
12914 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_freplace()
12917 if (prog->type != BPF_PROG_TYPE_EXT) { in bpf_program__attach_freplace()
12919 prog->name); in bpf_program__attach_freplace()
12920 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_freplace()
12926 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd); in bpf_program__attach_freplace()
12952 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_iter()
12959 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_iter()
12960 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_iter()
12965 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_iter()
12966 link->detach = &bpf_link__detach_fd; in bpf_program__attach_iter()
12971 link_fd = -errno; in bpf_program__attach_iter()
12974 prog->name, errstr(link_fd)); in bpf_program__attach_iter()
12977 link->fd = link_fd; in bpf_program__attach_iter()
12995 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_netfilter()
12999 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_netfilter()
13000 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_netfilter()
13005 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_netfilter()
13007 link->detach = &bpf_link__detach_fd; in bpf_program__attach_netfilter()
13016 link_fd = -errno; in bpf_program__attach_netfilter()
13019 prog->name, errstr(link_fd)); in bpf_program__attach_netfilter()
13022 link->fd = link_fd; in bpf_program__attach_netfilter()
13032 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_program__attach()
13033 return libbpf_err_ptr(-EOPNOTSUPP); in bpf_program__attach()
13037 prog->name); in bpf_program__attach()
13038 return libbpf_err_ptr(-EINVAL); in bpf_program__attach()
13041 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); in bpf_program__attach()
13045 /* When calling bpf_program__attach() explicitly, auto-attach support in bpf_program__attach()
13051 return libbpf_err_ptr(-EOPNOTSUPP); in bpf_program__attach()
13068 if (st_link->map_fd < 0) in bpf_link__detach_struct_ops()
13070 return bpf_map_delete_elem(link->fd, &zero); in bpf_link__detach_struct_ops()
13072 return close(link->fd); in bpf_link__detach_struct_ops()
13082 pr_warn("map '%s': can't attach non-struct_ops map\n", map->name); in bpf_map__attach_struct_ops()
13083 return libbpf_err_ptr(-EINVAL); in bpf_map__attach_struct_ops()
13086 if (map->fd < 0) { in bpf_map__attach_struct_ops()
13087 pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name); in bpf_map__attach_struct_ops()
13088 return libbpf_err_ptr(-EINVAL); in bpf_map__attach_struct_ops()
13093 return libbpf_err_ptr(-EINVAL); in bpf_map__attach_struct_ops()
13096 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); in bpf_map__attach_struct_ops()
13102 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) { in bpf_map__attach_struct_ops()
13107 link->link.detach = bpf_link__detach_struct_ops; in bpf_map__attach_struct_ops()
13109 if (!(map->def.map_flags & BPF_F_LINK)) { in bpf_map__attach_struct_ops()
13111 link->link.fd = map->fd; in bpf_map__attach_struct_ops()
13112 link->map_fd = -1; in bpf_map__attach_struct_ops()
13113 return &link->link; in bpf_map__attach_struct_ops()
13116 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL); in bpf_map__attach_struct_ops()
13122 link->link.fd = fd; in bpf_map__attach_struct_ops()
13123 link->map_fd = map->fd; in bpf_map__attach_struct_ops()
13125 return &link->link; in bpf_map__attach_struct_ops()
13138 return libbpf_err(-EINVAL); in bpf_link__update_map()
13140 if (map->fd < 0) { in bpf_link__update_map()
13141 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name); in bpf_link__update_map()
13142 return libbpf_err(-EINVAL); in bpf_link__update_map()
13147 if (st_ops_link->map_fd < 0) in bpf_link__update_map()
13148 return libbpf_err(-EINVAL); in bpf_link__update_map()
13150 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); in bpf_link__update_map()
13156 if (err && err != -EBUSY) in bpf_link__update_map()
13159 err = bpf_link_update(link->fd, map->fd, NULL); in bpf_link__update_map()
13163 st_ops_link->map_fd = map->fd; in bpf_link__update_map()
13178 __u64 data_tail = header->data_tail; in perf_event_read_simple()
13185 ehdr = base + (data_tail & (mmap_size - 1)); in perf_event_read_simple()
13186 ehdr_size = ehdr->size; in perf_event_read_simple()
13190 size_t len_first = base + mmap_size - copy_start; in perf_event_read_simple()
13191 size_t len_secnd = ehdr_size - len_first; in perf_event_read_simple()
13225 /* sample_cb and lost_cb are higher-level common-case callbacks */
13264 if (cpu_buf->base && in perf_buffer__free_cpu_buf()
13265 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) in perf_buffer__free_cpu_buf()
13266 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); in perf_buffer__free_cpu_buf()
13267 if (cpu_buf->fd >= 0) { in perf_buffer__free_cpu_buf()
13268 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); in perf_buffer__free_cpu_buf()
13269 close(cpu_buf->fd); in perf_buffer__free_cpu_buf()
13271 free(cpu_buf->buf); in perf_buffer__free_cpu_buf()
13281 if (pb->cpu_bufs) { in perf_buffer__free()
13282 for (i = 0; i < pb->cpu_cnt; i++) { in perf_buffer__free()
13283 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; in perf_buffer__free()
13288 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); in perf_buffer__free()
13291 free(pb->cpu_bufs); in perf_buffer__free()
13293 if (pb->epoll_fd >= 0) in perf_buffer__free()
13294 close(pb->epoll_fd); in perf_buffer__free()
13295 free(pb->events); in perf_buffer__free()
13308 return ERR_PTR(-ENOMEM); in perf_buffer__open_cpu_buf()
13310 cpu_buf->pb = pb; in perf_buffer__open_cpu_buf()
13311 cpu_buf->cpu = cpu; in perf_buffer__open_cpu_buf()
13312 cpu_buf->map_key = map_key; in perf_buffer__open_cpu_buf()
13314 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, in perf_buffer__open_cpu_buf()
13315 -1, PERF_FLAG_FD_CLOEXEC); in perf_buffer__open_cpu_buf()
13316 if (cpu_buf->fd < 0) { in perf_buffer__open_cpu_buf()
13317 err = -errno; in perf_buffer__open_cpu_buf()
13323 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, in perf_buffer__open_cpu_buf()
13325 cpu_buf->fd, 0); in perf_buffer__open_cpu_buf()
13326 if (cpu_buf->base == MAP_FAILED) { in perf_buffer__open_cpu_buf()
13327 cpu_buf->base = NULL; in perf_buffer__open_cpu_buf()
13328 err = -errno; in perf_buffer__open_cpu_buf()
13334 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { in perf_buffer__open_cpu_buf()
13335 err = -errno; in perf_buffer__open_cpu_buf()
13363 return libbpf_err_ptr(-EINVAL); in perf_buffer__new()
13393 return libbpf_err_ptr(-EINVAL); in perf_buffer__new_raw()
13396 return libbpf_err_ptr(-EINVAL); in perf_buffer__new_raw()
13418 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) { in __perf_buffer__new()
13421 return ERR_PTR(-EINVAL); in __perf_buffer__new()
13424 /* best-effort sanity checks */ in __perf_buffer__new()
13429 err = -errno; in __perf_buffer__new()
13431 * -EBADFD, -EFAULT, or -E2BIG on real error in __perf_buffer__new()
13433 if (err != -EINVAL) { in __perf_buffer__new()
13444 return ERR_PTR(-EINVAL); in __perf_buffer__new()
13450 return ERR_PTR(-ENOMEM); in __perf_buffer__new()
13452 pb->event_cb = p->event_cb; in __perf_buffer__new()
13453 pb->sample_cb = p->sample_cb; in __perf_buffer__new()
13454 pb->lost_cb = p->lost_cb; in __perf_buffer__new()
13455 pb->ctx = p->ctx; in __perf_buffer__new()
13457 pb->page_size = getpagesize(); in __perf_buffer__new()
13458 pb->mmap_size = pb->page_size * page_cnt; in __perf_buffer__new()
13459 pb->map_fd = map_fd; in __perf_buffer__new()
13461 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); in __perf_buffer__new()
13462 if (pb->epoll_fd < 0) { in __perf_buffer__new()
13463 err = -errno; in __perf_buffer__new()
13469 if (p->cpu_cnt > 0) { in __perf_buffer__new()
13470 pb->cpu_cnt = p->cpu_cnt; in __perf_buffer__new()
13472 pb->cpu_cnt = libbpf_num_possible_cpus(); in __perf_buffer__new()
13473 if (pb->cpu_cnt < 0) { in __perf_buffer__new()
13474 err = pb->cpu_cnt; in __perf_buffer__new()
13477 if (map.max_entries && map.max_entries < pb->cpu_cnt) in __perf_buffer__new()
13478 pb->cpu_cnt = map.max_entries; in __perf_buffer__new()
13481 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); in __perf_buffer__new()
13482 if (!pb->events) { in __perf_buffer__new()
13483 err = -ENOMEM; in __perf_buffer__new()
13487 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); in __perf_buffer__new()
13488 if (!pb->cpu_bufs) { in __perf_buffer__new()
13489 err = -ENOMEM; in __perf_buffer__new()
13500 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { in __perf_buffer__new()
13504 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; in __perf_buffer__new()
13505 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; in __perf_buffer__new()
13510 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) in __perf_buffer__new()
13513 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); in __perf_buffer__new()
13519 pb->cpu_bufs[j] = cpu_buf; in __perf_buffer__new()
13521 err = bpf_map_update_elem(pb->map_fd, &map_key, in __perf_buffer__new()
13522 &cpu_buf->fd, 0); in __perf_buffer__new()
13524 err = -errno; in __perf_buffer__new()
13525 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", in __perf_buffer__new()
13526 cpu, map_key, cpu_buf->fd, in __perf_buffer__new()
13531 pb->events[j].events = EPOLLIN; in __perf_buffer__new()
13532 pb->events[j].data.ptr = cpu_buf; in __perf_buffer__new()
13533 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, in __perf_buffer__new()
13534 &pb->events[j]) < 0) { in __perf_buffer__new()
13535 err = -errno; in __perf_buffer__new()
13537 cpu, cpu_buf->fd, in __perf_buffer__new()
13543 pb->cpu_cnt = j; in __perf_buffer__new()
13572 struct perf_buffer *pb = cpu_buf->pb; in perf_buffer__process_record()
13576 if (pb->event_cb) in perf_buffer__process_record()
13577 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); in perf_buffer__process_record()
13579 switch (e->type) { in perf_buffer__process_record()
13583 if (pb->sample_cb) in perf_buffer__process_record()
13584 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); in perf_buffer__process_record()
13590 if (pb->lost_cb) in perf_buffer__process_record()
13591 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); in perf_buffer__process_record()
13595 pr_warn("unknown perf sample type %d\n", e->type); in perf_buffer__process_record()
13606 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, in perf_buffer__process_records()
13607 pb->page_size, &cpu_buf->buf, in perf_buffer__process_records()
13608 &cpu_buf->buf_size, in perf_buffer__process_records()
13617 return pb->epoll_fd; in perf_buffer__epoll_fd()
13624 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); in perf_buffer__poll()
13626 return -errno; in perf_buffer__poll()
13629 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; in perf_buffer__poll()
13645 return pb->cpu_cnt; in perf_buffer__buffer_cnt()
13657 if (buf_idx >= pb->cpu_cnt) in perf_buffer__buffer_fd()
13658 return libbpf_err(-EINVAL); in perf_buffer__buffer_fd()
13660 cpu_buf = pb->cpu_bufs[buf_idx]; in perf_buffer__buffer_fd()
13662 return libbpf_err(-ENOENT); in perf_buffer__buffer_fd()
13664 return cpu_buf->fd; in perf_buffer__buffer_fd()
13671 if (buf_idx >= pb->cpu_cnt) in perf_buffer__buffer()
13672 return libbpf_err(-EINVAL); in perf_buffer__buffer()
13674 cpu_buf = pb->cpu_bufs[buf_idx]; in perf_buffer__buffer()
13676 return libbpf_err(-ENOENT); in perf_buffer__buffer()
13678 *buf = cpu_buf->base; in perf_buffer__buffer()
13679 *buf_size = pb->mmap_size; in perf_buffer__buffer()
13688 * - 0 on success;
13689 * - <0 on failure.
13695 if (buf_idx >= pb->cpu_cnt) in perf_buffer__consume_buffer()
13696 return libbpf_err(-EINVAL); in perf_buffer__consume_buffer()
13698 cpu_buf = pb->cpu_bufs[buf_idx]; in perf_buffer__consume_buffer()
13700 return libbpf_err(-ENOENT); in perf_buffer__consume_buffer()
13709 for (i = 0; i < pb->cpu_cnt; i++) { in perf_buffer__consume()
13710 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; in perf_buffer__consume()
13732 return libbpf_err(-EINVAL); in bpf_program__set_attach_target()
13734 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_attach_target()
13735 return libbpf_err(-EINVAL); in bpf_program__set_attach_target()
13741 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
13747 attach_prog_fd, prog->obj->token_fd); in bpf_program__set_attach_target()
13752 return libbpf_err(-EINVAL); in bpf_program__set_attach_target()
13755 err = bpf_object__load_vmlinux_btf(prog->obj, true); in bpf_program__set_attach_target()
13758 err = find_kernel_btf_id(prog->obj, attach_func_name, in bpf_program__set_attach_target()
13759 prog->expected_attach_type, in bpf_program__set_attach_target()
13765 prog->attach_btf_id = btf_id; in bpf_program__set_attach_target()
13766 prog->attach_btf_obj_fd = btf_obj_fd; in bpf_program__set_attach_target()
13767 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
13773 int err = 0, n, len, start, end = -1; in parse_cpu_mask_str()
13779 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ in parse_cpu_mask_str()
13785 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); in parse_cpu_mask_str()
13788 err = -EINVAL; in parse_cpu_mask_str()
13796 err = -EINVAL; in parse_cpu_mask_str()
13801 err = -ENOMEM; in parse_cpu_mask_str()
13805 memset(tmp + *mask_sz, 0, start - *mask_sz); in parse_cpu_mask_str()
13806 memset(tmp + start, 1, end - start + 1); in parse_cpu_mask_str()
13812 return -EINVAL; in parse_cpu_mask_str()
13828 err = -errno; in parse_cpu_mask_file()
13835 err = len ? -errno : -EINVAL; in parse_cpu_mask_file()
13841 return -E2BIG; in parse_cpu_mask_file()
13882 struct bpf_map **map = map_skel->map; in populate_skeleton_maps()
13883 const char *name = map_skel->name; in populate_skeleton_maps()
13884 void **mmaped = map_skel->mmaped; in populate_skeleton_maps()
13889 return -ESRCH; in populate_skeleton_maps()
13892 /* externs shouldn't be pre-setup from user code */ in populate_skeleton_maps()
13893 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) in populate_skeleton_maps()
13894 *mmaped = (*map)->mmaped; in populate_skeleton_maps()
13907 struct bpf_program **prog = prog_skel->prog; in populate_skeleton_progs()
13908 const char *name = prog_skel->name; in populate_skeleton_progs()
13913 return -ESRCH; in populate_skeleton_progs()
13925 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts); in bpf_object__open_skeleton()
13929 s->name, errstr(err)); in bpf_object__open_skeleton()
13933 *s->obj = obj; in bpf_object__open_skeleton()
13934 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz); in bpf_object__open_skeleton()
13936 pr_warn("failed to populate skeleton maps for '%s': %s\n", s->name, errstr(err)); in bpf_object__open_skeleton()
13940 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz); in bpf_object__open_skeleton()
13942 pr_warn("failed to populate skeleton progs for '%s': %s\n", s->name, errstr(err)); in bpf_object__open_skeleton()
13960 if (!s->obj) in bpf_object__open_subskeleton()
13961 return libbpf_err(-EINVAL); in bpf_object__open_subskeleton()
13963 btf = bpf_object__btf(s->obj); in bpf_object__open_subskeleton()
13966 bpf_object__name(s->obj)); in bpf_object__open_subskeleton()
13967 return libbpf_err(-errno); in bpf_object__open_subskeleton()
13970 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz); in bpf_object__open_subskeleton()
13976 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz); in bpf_object__open_subskeleton()
13982 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) { in bpf_object__open_subskeleton()
13983 var_skel = (void *)s->vars + var_idx * s->var_skel_sz; in bpf_object__open_subskeleton()
13984 map = *var_skel->map; in bpf_object__open_subskeleton()
13992 return libbpf_err(-EINVAL); in bpf_object__open_subskeleton()
13998 var_type = btf__type_by_id(btf, var->type); in bpf_object__open_subskeleton()
13999 var_name = btf__name_by_offset(btf, var_type->name_off); in bpf_object__open_subskeleton()
14000 if (strcmp(var_name, var_skel->name) == 0) { in bpf_object__open_subskeleton()
14001 *var_skel->addr = map->mmaped + var->offset; in bpf_object__open_subskeleton()
14013 free(s->maps); in bpf_object__destroy_subskeleton()
14014 free(s->progs); in bpf_object__destroy_subskeleton()
14015 free(s->vars); in bpf_object__destroy_subskeleton()
14023 err = bpf_object__load(*s->obj); in bpf_object__load_skeleton()
14025 pr_warn("failed to load BPF skeleton '%s': %s\n", s->name, errstr(err)); in bpf_object__load_skeleton()
14029 for (i = 0; i < s->map_cnt; i++) { in bpf_object__load_skeleton()
14030 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz; in bpf_object__load_skeleton()
14031 struct bpf_map *map = *map_skel->map; in bpf_object__load_skeleton()
14033 if (!map_skel->mmaped) in bpf_object__load_skeleton()
14036 *map_skel->mmaped = map->mmaped; in bpf_object__load_skeleton()
14046 for (i = 0; i < s->prog_cnt; i++) { in bpf_object__attach_skeleton()
14047 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz; in bpf_object__attach_skeleton()
14048 struct bpf_program *prog = *prog_skel->prog; in bpf_object__attach_skeleton()
14049 struct bpf_link **link = prog_skel->link; in bpf_object__attach_skeleton()
14051 if (!prog->autoload || !prog->autoattach) in bpf_object__attach_skeleton()
14054 /* auto-attaching not supported for this program */ in bpf_object__attach_skeleton()
14055 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_object__attach_skeleton()
14058 /* if user already set the link manually, don't attempt auto-attach */ in bpf_object__attach_skeleton()
14062 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); in bpf_object__attach_skeleton()
14064 pr_warn("prog '%s': failed to auto-attach: %s\n", in bpf_object__attach_skeleton()
14069 /* It's possible that for some SEC() definitions auto-attach in bpf_object__attach_skeleton()
14074 * auto-attached. But if not, it shouldn't trigger skeleton's in bpf_object__attach_skeleton()
14082 for (i = 0; i < s->map_cnt; i++) { in bpf_object__attach_skeleton()
14083 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz; in bpf_object__attach_skeleton()
14084 struct bpf_map *map = *map_skel->map; in bpf_object__attach_skeleton()
14087 if (!map->autocreate || !map->autoattach) in bpf_object__attach_skeleton()
14095 if (s->map_skel_sz < offsetofend(struct bpf_map_skeleton, link)) { in bpf_object__attach_skeleton()
14096 pr_warn("map '%s': BPF skeleton version is old, skipping map auto-attachment...\n", in bpf_object__attach_skeleton()
14101 link = map_skel->link; in bpf_object__attach_skeleton()
14107 err = -errno; in bpf_object__attach_skeleton()
14108 pr_warn("map '%s': failed to auto-attach: %s\n", in bpf_object__attach_skeleton()
14121 for (i = 0; i < s->prog_cnt; i++) { in bpf_object__detach_skeleton()
14122 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz; in bpf_object__detach_skeleton()
14123 struct bpf_link **link = prog_skel->link; in bpf_object__detach_skeleton()
14129 if (s->map_skel_sz < sizeof(struct bpf_map_skeleton)) in bpf_object__detach_skeleton()
14132 for (i = 0; i < s->map_cnt; i++) { in bpf_object__detach_skeleton()
14133 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz; in bpf_object__detach_skeleton()
14134 struct bpf_link **link = map_skel->link; in bpf_object__detach_skeleton()
14149 if (s->obj) in bpf_object__destroy_skeleton()
14150 bpf_object__close(*s->obj); in bpf_object__destroy_skeleton()
14151 free(s->maps); in bpf_object__destroy_skeleton()
14152 free(s->progs); in bpf_object__destroy_skeleton()