Lines Matching +full:function +full:- +full:enumerator
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
33 /* raw BTF data in non-native endianness */
45 * +--------------------------------+
47 * +--------------------------------+
51 * types_data-+ |
52 * strs_data------------+
63 * +----------+ +---------+ +-----------+
65 * +----------+ +---------+ +-----------+
69 * types_data----+ |
70 * strs_data------------------+
72 * +----------+---------+-----------+
74 * raw_data----->+----------+---------+-----------+
79 size_t types_data_cap; /* used size stored in hdr->type_len */
87 size_t strs_data_cap; /* used size stored in hdr->str_len */
142 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); in btf_add_mem()
159 p = btf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); in btf_ensure_mem()
161 return -ENOMEM; in btf_ensure_mem()
170 p = btf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), in btf_add_type_idx_entry()
171 btf->nr_types + 1, BTF_MAX_NR_TYPES, 1); in btf_add_type_idx_entry()
173 return -ENOMEM; in btf_add_type_idx_entry()
181 h->magic = bswap_16(h->magic); in btf_bswap_hdr()
182 h->hdr_len = bswap_32(h->hdr_len); in btf_bswap_hdr()
183 h->type_off = bswap_32(h->type_off); in btf_bswap_hdr()
184 h->type_len = bswap_32(h->type_len); in btf_bswap_hdr()
185 h->str_off = bswap_32(h->str_off); in btf_bswap_hdr()
186 h->str_len = bswap_32(h->str_len); in btf_bswap_hdr()
191 struct btf_header *hdr = btf->hdr; in btf_parse_hdr()
194 if (btf->raw_size < sizeof(struct btf_header)) { in btf_parse_hdr()
196 return -EINVAL; in btf_parse_hdr()
199 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_parse_hdr()
200 btf->swapped_endian = true; in btf_parse_hdr()
201 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { in btf_parse_hdr()
202 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", in btf_parse_hdr()
203 bswap_32(hdr->hdr_len)); in btf_parse_hdr()
204 return -ENOTSUP; in btf_parse_hdr()
207 } else if (hdr->magic != BTF_MAGIC) { in btf_parse_hdr()
208 pr_debug("Invalid BTF magic:%x\n", hdr->magic); in btf_parse_hdr()
209 return -EINVAL; in btf_parse_hdr()
212 meta_left = btf->raw_size - sizeof(*hdr); in btf_parse_hdr()
215 return -EINVAL; in btf_parse_hdr()
218 if (meta_left < hdr->type_off) { in btf_parse_hdr()
219 pr_debug("Invalid BTF type section offset:%u\n", hdr->type_off); in btf_parse_hdr()
220 return -EINVAL; in btf_parse_hdr()
223 if (meta_left < hdr->str_off) { in btf_parse_hdr()
224 pr_debug("Invalid BTF string section offset:%u\n", hdr->str_off); in btf_parse_hdr()
225 return -EINVAL; in btf_parse_hdr()
228 if (hdr->type_off >= hdr->str_off) { in btf_parse_hdr()
230 return -EINVAL; in btf_parse_hdr()
233 if (hdr->type_off & 0x02) { in btf_parse_hdr()
235 return -EINVAL; in btf_parse_hdr()
243 const struct btf_header *hdr = btf->hdr; in btf_parse_str_sec()
244 const char *start = btf->strs_data; in btf_parse_str_sec()
245 const char *end = start + btf->hdr->str_len; in btf_parse_str_sec()
247 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || in btf_parse_str_sec()
248 start[0] || end[-1]) { in btf_parse_str_sec()
250 return -EINVAL; in btf_parse_str_sec()
287 return -EINVAL; in btf_type_size()
293 t->name_off = bswap_32(t->name_off); in btf_bswap_type_base()
294 t->info = bswap_32(t->info); in btf_bswap_type_base()
295 t->type = bswap_32(t->type); in btf_bswap_type_base()
322 e->name_off = bswap_32(e->name_off); in btf_bswap_type_rest()
323 e->val = bswap_32(e->val); in btf_bswap_type_rest()
328 a->type = bswap_32(a->type); in btf_bswap_type_rest()
329 a->index_type = bswap_32(a->index_type); in btf_bswap_type_rest()
330 a->nelems = bswap_32(a->nelems); in btf_bswap_type_rest()
335 m->name_off = bswap_32(m->name_off); in btf_bswap_type_rest()
336 m->type = bswap_32(m->type); in btf_bswap_type_rest()
337 m->offset = bswap_32(m->offset); in btf_bswap_type_rest()
342 p->name_off = bswap_32(p->name_off); in btf_bswap_type_rest()
343 p->type = bswap_32(p->type); in btf_bswap_type_rest()
347 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); in btf_bswap_type_rest()
351 v->type = bswap_32(v->type); in btf_bswap_type_rest()
352 v->offset = bswap_32(v->offset); in btf_bswap_type_rest()
353 v->size = bswap_32(v->size); in btf_bswap_type_rest()
358 return -EINVAL; in btf_bswap_type_rest()
364 struct btf_header *hdr = btf->hdr; in btf_parse_type_sec()
365 void *next_type = btf->types_data; in btf_parse_type_sec()
366 void *end_type = next_type + hdr->type_len; in btf_parse_type_sec()
380 if (btf->swapped_endian) in btf_parse_type_sec()
388 return -EINVAL; in btf_parse_type_sec()
391 if (btf->swapped_endian && btf_bswap_type_rest(next_type)) in btf_parse_type_sec()
392 return -EINVAL; in btf_parse_type_sec()
394 err = btf_add_type_idx_entry(btf, next_type - btf->types_data); in btf_parse_type_sec()
399 btf->nr_types++; in btf_parse_type_sec()
404 return -EINVAL; in btf_parse_type_sec()
412 return btf->nr_types; in btf__get_nr_types()
415 /* internal helper returning non-const pointer to a type */
421 return btf->types_data + btf->type_offs[type_id]; in btf_type_by_id()
426 if (type_id > btf->nr_types) in btf__type_by_id()
437 for (i = 1; i <= btf->nr_types; i++) { in determine_ptr_size()
442 name = btf__name_by_offset(btf, t->name_off); in determine_ptr_size()
448 if (t->size != 4 && t->size != 8) in determine_ptr_size()
450 return t->size; in determine_ptr_size()
454 return -1; in determine_ptr_size()
459 if (!btf->ptr_sz) in btf_ptr_sz()
460 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf_ptr_sz()
461 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; in btf_ptr_sz()
467 * type, this function returns 0. In the latter case, native architecture's
474 if (!btf->ptr_sz) in btf__pointer_size()
475 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf__pointer_size()
477 if (btf->ptr_sz < 0) in btf__pointer_size()
481 return btf->ptr_sz; in btf__pointer_size()
490 return -EINVAL; in btf__set_pointer_size()
491 btf->ptr_sz = ptr_sz; in btf__set_pointer_size()
509 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; in btf__endianness()
511 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; in btf__endianness()
517 return -EINVAL; in btf__set_endianness()
519 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); in btf__set_endianness()
520 if (!btf->swapped_endian) { in btf__set_endianness()
521 free(btf->raw_data_swapped); in btf__set_endianness()
522 btf->raw_data_swapped = NULL; in btf__set_endianness()
544 __s64 size = -1; in btf__resolve_size()
556 size = t->size; in btf__resolve_size()
566 type_id = t->type; in btf__resolve_size()
570 if (nelems && array->nelems > UINT32_MAX / nelems) in btf__resolve_size()
571 return -E2BIG; in btf__resolve_size()
572 nelems *= array->nelems; in btf__resolve_size()
573 type_id = array->type; in btf__resolve_size()
576 return -EINVAL; in btf__resolve_size()
584 return -EINVAL; in btf__resolve_size()
586 return -E2BIG; in btf__resolve_size()
599 return min(btf_ptr_sz(btf), (size_t)t->size); in btf__align_of()
606 return btf__align_of(btf, t->type); in btf__align_of()
608 return btf__align_of(btf, btf_array(t)->type); in btf__align_of()
616 align = btf__align_of(btf, m->type); in btf__align_of()
639 type_id = t->type; in btf__resolve_type()
645 return -EINVAL; in btf__resolve_type()
657 for (i = 1; i <= btf->nr_types; i++) { in btf__find_by_name()
659 const char *name = btf__name_by_offset(btf, t->name_off); in btf__find_by_name()
665 return -ENOENT; in btf__find_by_name()
676 for (i = 1; i <= btf->nr_types; i++) { in btf__find_by_name_kind()
682 name = btf__name_by_offset(btf, t->name_off); in btf__find_by_name_kind()
687 return -ENOENT; in btf__find_by_name_kind()
692 return (void *)btf->hdr != btf->raw_data; in btf_is_modifiable()
700 if (btf->fd >= 0) in btf__free()
701 close(btf->fd); in btf__free()
705 * in-memory representation for header, types, and strings in btf__free()
710 free(btf->hdr); in btf__free()
711 free(btf->types_data); in btf__free()
712 free(btf->strs_data); in btf__free()
714 free(btf->raw_data); in btf__free()
715 free(btf->raw_data_swapped); in btf__free()
716 free(btf->type_offs); in btf__free()
726 return ERR_PTR(-ENOMEM); in btf__new_empty()
728 btf->fd = -1; in btf__new_empty()
729 btf->ptr_sz = sizeof(void *); in btf__new_empty()
730 btf->swapped_endian = false; in btf__new_empty()
733 btf->raw_size = sizeof(struct btf_header) + 1; in btf__new_empty()
734 btf->raw_data = calloc(1, btf->raw_size); in btf__new_empty()
735 if (!btf->raw_data) { in btf__new_empty()
737 return ERR_PTR(-ENOMEM); in btf__new_empty()
740 btf->hdr = btf->raw_data; in btf__new_empty()
741 btf->hdr->hdr_len = sizeof(struct btf_header); in btf__new_empty()
742 btf->hdr->magic = BTF_MAGIC; in btf__new_empty()
743 btf->hdr->version = BTF_VERSION; in btf__new_empty()
745 btf->types_data = btf->raw_data + btf->hdr->hdr_len; in btf__new_empty()
746 btf->strs_data = btf->raw_data + btf->hdr->hdr_len; in btf__new_empty()
747 btf->hdr->str_len = 1; /* empty string at offset 0 */ in btf__new_empty()
759 return ERR_PTR(-ENOMEM); in btf__new()
761 btf->raw_data = malloc(size); in btf__new()
762 if (!btf->raw_data) { in btf__new()
763 err = -ENOMEM; in btf__new()
766 memcpy(btf->raw_data, data, size); in btf__new()
767 btf->raw_size = size; in btf__new()
769 btf->hdr = btf->raw_data; in btf__new()
774 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; in btf__new()
775 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; in btf__new()
782 btf->fd = -1; in btf__new()
796 int err = 0, fd = -1, idx = 0; in btf__parse_elf()
804 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); in btf__parse_elf()
809 err = -errno; in btf__parse_elf()
814 err = -LIBBPF_ERRNO__FORMAT; in btf__parse_elf()
868 err = -ENOENT; in btf__parse_elf()
871 btf = btf__new(btf_data->d_buf, btf_data->d_size); in btf__parse_elf()
888 *btf_ext = btf_ext__new(btf_ext_data->d_buf, in btf__parse_elf()
889 btf_ext_data->d_size); in btf__parse_elf()
927 err = -errno; in btf__parse_raw()
933 err = -EIO; in btf__parse_raw()
938 err = -EPROTO; in btf__parse_raw()
944 err = -errno; in btf__parse_raw()
949 err = -errno; in btf__parse_raw()
954 err = -errno; in btf__parse_raw()
958 /* pre-alloc memory and read all of BTF data */ in btf__parse_raw()
961 err = -ENOMEM; in btf__parse_raw()
965 err = -EIO; in btf__parse_raw()
987 if (!IS_ERR(btf) || PTR_ERR(btf) != -EPROTO) in btf__parse()
998 return a->offset - b->offset; in compare_vsi_off()
1005 const char *name = btf__name_by_offset(btf, t->name_off); in btf_fixup_datasec()
1013 return -ENOENT; in btf_fixup_datasec()
1019 if (t->size) in btf_fixup_datasec()
1023 if (ret || !size || (t->size && t->size != size)) { in btf_fixup_datasec()
1025 return -ENOENT; in btf_fixup_datasec()
1028 t->size = size; in btf_fixup_datasec()
1031 t_var = btf__type_by_id(btf, vsi->type); in btf_fixup_datasec()
1035 pr_debug("Non-VAR type seen in section %s\n", name); in btf_fixup_datasec()
1036 return -EINVAL; in btf_fixup_datasec()
1039 if (var->linkage == BTF_VAR_STATIC) in btf_fixup_datasec()
1042 name = btf__name_by_offset(btf, t_var->name_off); in btf_fixup_datasec()
1045 return -ENOENT; in btf_fixup_datasec()
1052 return -ENOENT; in btf_fixup_datasec()
1055 vsi->offset = off; in btf_fixup_datasec()
1068 for (i = 1; i <= btf->nr_types; i++) { in btf__finalize_data()
1095 if (btf->fd >= 0) in btf__load()
1096 return -EEXIST; in btf__load()
1102 return -ENOMEM; in btf__load()
1109 err = -ENOMEM; in btf__load()
1113 btf->raw_size = raw_size; in btf__load()
1114 btf->raw_data = raw_data; in btf__load()
1116 btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false); in btf__load()
1117 if (btf->fd < 0) { in btf__load()
1125 err = -errno; in btf__load()
1139 return btf->fd; in btf__fd()
1144 btf->fd = fd; in btf__set_fd()
1149 struct btf_header *hdr = btf->hdr; in btf_get_raw_data()
1155 data = swap_endian ? btf->raw_data_swapped : btf->raw_data; in btf_get_raw_data()
1157 *size = btf->raw_size; in btf_get_raw_data()
1161 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; in btf_get_raw_data()
1167 memcpy(p, hdr, hdr->hdr_len); in btf_get_raw_data()
1170 p += hdr->hdr_len; in btf_get_raw_data()
1172 memcpy(p, btf->types_data, hdr->type_len); in btf_get_raw_data()
1174 for (i = 1; i <= btf->nr_types; i++) { in btf_get_raw_data()
1175 t = p + btf->type_offs[i]; in btf_get_raw_data()
1176 /* btf_bswap_type_rest() relies on native t->info, so in btf_get_raw_data()
1185 p += hdr->type_len; in btf_get_raw_data()
1187 memcpy(p, btf->strs_data, hdr->str_len); in btf_get_raw_data()
1188 p += hdr->str_len; in btf_get_raw_data()
1203 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); in btf__get_raw_data()
1207 btf->raw_size = data_sz; in btf__get_raw_data()
1208 if (btf->swapped_endian) in btf__get_raw_data()
1209 btf->raw_data_swapped = data; in btf__get_raw_data()
1211 btf->raw_data = data; in btf__get_raw_data()
1218 if (offset < btf->hdr->str_len) in btf__str_by_offset()
1219 return btf->strs_data + offset; in btf__str_by_offset()
1245 * let's start with a sane default - 4KiB here - and resize it only if in btf__get_from_id()
1252 err = -ENOMEM; in btf__get_from_id()
1266 err = -ENOMEM; in btf__get_from_id()
1308 return -EINVAL; in btf__get_map_kv_tids()
1322 return -EINVAL; in btf__get_map_kv_tids()
1328 return -EINVAL; in btf__get_map_kv_tids()
1334 key_size = btf__resolve_size(btf, key->type); in btf__get_map_kv_tids()
1343 return -EINVAL; in btf__get_map_kv_tids()
1346 value_size = btf__resolve_size(btf, value->type); in btf__get_map_kv_tids()
1355 return -EINVAL; in btf__get_map_kv_tids()
1358 *key_type_id = key->type; in btf__get_map_kv_tids()
1359 *value_type_id = value->type; in btf__get_map_kv_tids()
1367 const char *str = btf->strs_data + (long)key; in strs_hash_fn()
1375 const char *str1 = btf->strs_data + (long)key1; in strs_hash_equal_fn()
1376 const char *str2 = btf->strs_data + (long)key2; in strs_hash_equal_fn()
1383 if (btf->raw_data) { in btf_invalidate_raw_data()
1384 free(btf->raw_data); in btf_invalidate_raw_data()
1385 btf->raw_data = NULL; in btf_invalidate_raw_data()
1387 if (btf->raw_data_swapped) { in btf_invalidate_raw_data()
1388 free(btf->raw_data_swapped); in btf_invalidate_raw_data()
1389 btf->raw_data_swapped = NULL; in btf_invalidate_raw_data()
1411 hdr = malloc(btf->hdr->hdr_len); in btf_ensure_modifiable()
1412 types = malloc(btf->hdr->type_len); in btf_ensure_modifiable()
1413 strs = malloc(btf->hdr->str_len); in btf_ensure_modifiable()
1417 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); in btf_ensure_modifiable()
1418 memcpy(types, btf->types_data, btf->hdr->type_len); in btf_ensure_modifiable()
1419 memcpy(strs, btf->strs_data, btf->hdr->str_len); in btf_ensure_modifiable()
1429 strs_end = strs + btf->hdr->str_len; in btf_ensure_modifiable()
1435 if (err == -EEXIST) in btf_ensure_modifiable()
1442 btf->hdr = hdr; in btf_ensure_modifiable()
1443 btf->types_data = types; in btf_ensure_modifiable()
1444 btf->types_data_cap = btf->hdr->type_len; in btf_ensure_modifiable()
1445 btf->strs_data = strs; in btf_ensure_modifiable()
1446 btf->strs_data_cap = btf->hdr->str_len; in btf_ensure_modifiable()
1447 btf->strs_hash = hash; in btf_ensure_modifiable()
1451 btf->strs_deduped = btf->hdr->str_len <= 1; in btf_ensure_modifiable()
1463 return -ENOMEM; in btf_ensure_modifiable()
1468 return btf_add_mem(&btf->strs_data, &btf->strs_data_cap, 1, in btf_add_str_mem()
1469 btf->hdr->str_len, BTF_MAX_STR_OFFSET, add_sz); in btf_add_str_mem()
1474 * - >0 offset into string section, if string is found;
1475 * - -ENOENT, if string is not in the string section;
1476 * - <0, on any other error.
1485 return -ENOMEM; in btf__find_str()
1491 return -ENOMEM; in btf__find_str()
1493 new_off = btf->hdr->str_len; in btf__find_str()
1496 if (hashmap__find(btf->strs_hash, (void *)new_off, (void **)&old_off)) in btf__find_str()
1499 return -ENOENT; in btf__find_str()
1504 * - > 0 offset into string section, on success;
1505 * - < 0, on error.
1514 return -ENOMEM; in btf__add_str()
1516 /* Hashmap keys are always offsets within btf->strs_data, so to even in btf__add_str()
1519 * until btf->hdr->str_len is incremented, that string is just a piece in btf__add_str()
1522 * ready to be used, only a simple btf->hdr->str_len increment away. in btf__add_str()
1527 return -ENOMEM; in btf__add_str()
1529 new_off = btf->hdr->str_len; in btf__add_str()
1536 err = hashmap__insert(btf->strs_hash, (void *)new_off, (void *)new_off, in btf__add_str()
1538 if (err == -EEXIST) in btf__add_str()
1543 btf->hdr->str_len += len; /* new unique string, adjust data length */ in btf__add_str()
1549 return btf_add_mem(&btf->types_data, &btf->types_data_cap, 1, in btf_add_type_mem()
1550 btf->hdr->type_len, UINT_MAX, add_sz); in btf_add_type_mem()
1560 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); in btf_type_inc_vlen()
1565 * - *name* - non-empty, non-NULL type name;
1566 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
1567 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
1569 * - >0, type ID of newly added BTF type;
1570 * - <0, on error.
1577 /* non-empty name */ in btf__add_int()
1579 return -EINVAL; in btf__add_int()
1581 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) in btf__add_int()
1582 return -EINVAL; in btf__add_int()
1584 return -EINVAL; in btf__add_int()
1588 return -ENOMEM; in btf__add_int()
1593 return -ENOMEM; in btf__add_int()
1603 t->name_off = name_off; in btf__add_int()
1604 t->info = btf_type_info(BTF_KIND_INT, 0, 0); in btf__add_int()
1605 t->size = byte_sz; in btf__add_int()
1609 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_int()
1613 btf->hdr->type_len += sz; in btf__add_int()
1614 btf->hdr->str_off += sz; in btf__add_int()
1615 btf->nr_types++; in btf__add_int()
1616 return btf->nr_types; in btf__add_int()
1626 return -EINVAL; in validate_type_id()
1630 /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
1637 return -EINVAL; in btf_add_ref_kind()
1640 return -ENOMEM; in btf_add_ref_kind()
1645 return -ENOMEM; in btf_add_ref_kind()
1653 t->name_off = name_off; in btf_add_ref_kind()
1654 t->info = btf_type_info(kind, 0, 0); in btf_add_ref_kind()
1655 t->type = ref_type_id; in btf_add_ref_kind()
1657 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf_add_ref_kind()
1661 btf->hdr->type_len += sz; in btf_add_ref_kind()
1662 btf->hdr->str_off += sz; in btf_add_ref_kind()
1663 btf->nr_types++; in btf_add_ref_kind()
1664 return btf->nr_types; in btf_add_ref_kind()
1669 * - *ref_type_id* - referenced type ID, it might not exist yet;
1671 * - >0, type ID of newly added BTF type;
1672 * - <0, on error.
1681 * - *index_type_id* - type ID of the type describing array index;
1682 * - *elem_type_id* - type ID of the type describing array element;
1683 * - *nr_elems* - the size of the array;
1685 * - >0, type ID of newly added BTF type;
1686 * - <0, on error.
1695 return -EINVAL; in btf__add_array()
1698 return -ENOMEM; in btf__add_array()
1703 return -ENOMEM; in btf__add_array()
1705 t->name_off = 0; in btf__add_array()
1706 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); in btf__add_array()
1707 t->size = 0; in btf__add_array()
1710 a->type = elem_type_id; in btf__add_array()
1711 a->index_type = index_type_id; in btf__add_array()
1712 a->nelems = nr_elems; in btf__add_array()
1714 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_array()
1718 btf->hdr->type_len += sz; in btf__add_array()
1719 btf->hdr->str_off += sz; in btf__add_array()
1720 btf->nr_types++; in btf__add_array()
1721 return btf->nr_types; in btf__add_array()
1724 /* generic STRUCT/UNION append function */
1731 return -ENOMEM; in btf_add_composite()
1736 return -ENOMEM; in btf_add_composite()
1747 t->name_off = name_off; in btf_add_composite()
1748 t->info = btf_type_info(kind, 0, 0); in btf_add_composite()
1749 t->size = bytes_sz; in btf_add_composite()
1751 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf_add_composite()
1755 btf->hdr->type_len += sz; in btf_add_composite()
1756 btf->hdr->str_off += sz; in btf_add_composite()
1757 btf->nr_types++; in btf_add_composite()
1758 return btf->nr_types; in btf_add_composite()
1763 * - *name* - name of the struct, can be NULL or empty for anonymous structs;
1764 * - *byte_sz* - size of the struct, in bytes;
1770 * - >0, type ID of newly added BTF type;
1771 * - <0, on error.
1780 * - *name* - name of the union, can be NULL or empty for anonymous union;
1781 * - *byte_sz* - size of the union, in bytes;
1788 * - >0, type ID of newly added BTF type;
1789 * - <0, on error.
1798 * - *name* - name of the field, can be NULL or empty for anonymous field;
1799 * - *type_id* - type ID for the type describing field type;
1800 * - *bit_offset* - bit offset of the start of the field within struct/union;
1801 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
1803 * - 0, on success;
1804 * - <0, on error.
1815 if (btf->nr_types == 0) in btf__add_field()
1816 return -EINVAL; in btf__add_field()
1817 t = btf_type_by_id(btf, btf->nr_types); in btf__add_field()
1819 return -EINVAL; in btf__add_field()
1822 return -EINVAL; in btf__add_field()
1823 /* best-effort bit field offset/size enforcement */ in btf__add_field()
1826 return -EINVAL; in btf__add_field()
1830 return -EINVAL; in btf__add_field()
1834 return -ENOMEM; in btf__add_field()
1839 return -ENOMEM; in btf__add_field()
1847 m->name_off = name_off; in btf__add_field()
1848 m->type = type_id; in btf__add_field()
1849 m->offset = bit_offset | (bit_size << 24); in btf__add_field()
1852 t = btf_type_by_id(btf, btf->nr_types); in btf__add_field()
1854 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); in btf__add_field()
1856 btf->hdr->type_len += sz; in btf__add_field()
1857 btf->hdr->str_off += sz; in btf__add_field()
1863 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
1864 * - *byte_sz* - size of the enum, in bytes.
1867 * declaration). Enumerator values can be added by btf__add_enum_value()
1871 * - >0, type ID of newly added BTF type;
1872 * - <0, on error.
1880 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) in btf__add_enum()
1881 return -EINVAL; in btf__add_enum()
1884 return -ENOMEM; in btf__add_enum()
1889 return -ENOMEM; in btf__add_enum()
1898 t->name_off = name_off; in btf__add_enum()
1899 t->info = btf_type_info(BTF_KIND_ENUM, 0, 0); in btf__add_enum()
1900 t->size = byte_sz; in btf__add_enum()
1902 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_enum()
1906 btf->hdr->type_len += sz; in btf__add_enum()
1907 btf->hdr->str_off += sz; in btf__add_enum()
1908 btf->nr_types++; in btf__add_enum()
1909 return btf->nr_types; in btf__add_enum()
1914 * - *name* - name of the enumerator value, can't be NULL or empty;
1915 * - *value* - integer value corresponding to enum value *name*;
1917 * - 0, on success;
1918 * - <0, on error.
1927 if (btf->nr_types == 0) in btf__add_enum_value()
1928 return -EINVAL; in btf__add_enum_value()
1929 t = btf_type_by_id(btf, btf->nr_types); in btf__add_enum_value()
1931 return -EINVAL; in btf__add_enum_value()
1933 /* non-empty name */ in btf__add_enum_value()
1935 return -EINVAL; in btf__add_enum_value()
1937 return -E2BIG; in btf__add_enum_value()
1941 return -ENOMEM; in btf__add_enum_value()
1946 return -ENOMEM; in btf__add_enum_value()
1952 v->name_off = name_off; in btf__add_enum_value()
1953 v->val = value; in btf__add_enum_value()
1956 t = btf_type_by_id(btf, btf->nr_types); in btf__add_enum_value()
1959 btf->hdr->type_len += sz; in btf__add_enum_value()
1960 btf->hdr->str_off += sz; in btf__add_enum_value()
1966 * - *name*, non-empty/non-NULL name;
1967 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
1970 * - >0, type ID of newly added BTF type;
1971 * - <0, on error.
1976 return -EINVAL; in btf__add_fwd()
1988 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); in btf__add_fwd()
1993 * values; we also assume a standard 4-byte size for it in btf__add_fwd()
1997 return -EINVAL; in btf__add_fwd()
2003 * - *name*, non-empty/non-NULL name;
2004 * - *ref_type_id* - referenced type ID, it might not exist yet;
2006 * - >0, type ID of newly added BTF type;
2007 * - <0, on error.
2012 return -EINVAL; in btf__add_typedef()
2019 * - *ref_type_id* - referenced type ID, it might not exist yet;
2021 * - >0, type ID of newly added BTF type;
2022 * - <0, on error.
2031 * - *ref_type_id* - referenced type ID, it might not exist yet;
2033 * - >0, type ID of newly added BTF type;
2034 * - <0, on error.
2043 * - *ref_type_id* - referenced type ID, it might not exist yet;
2045 * - >0, type ID of newly added BTF type;
2046 * - <0, on error.
2055 * - *name*, non-empty/non-NULL name;
2056 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2058 * - >0, type ID of newly added BTF type;
2059 * - <0, on error.
2067 return -EINVAL; in btf__add_func()
2070 return -EINVAL; in btf__add_func()
2076 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); in btf__add_func()
2083 * - *ret_type_id* - type ID for return result of a function.
2085 * Function prototype initially has no arguments, but they can be added by
2090 * - >0, type ID of newly added BTF type;
2091 * - <0, on error.
2099 return -EINVAL; in btf__add_func_proto()
2102 return -ENOMEM; in btf__add_func_proto()
2107 return -ENOMEM; in btf__add_func_proto()
2112 t->name_off = 0; in btf__add_func_proto()
2113 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); in btf__add_func_proto()
2114 t->type = ret_type_id; in btf__add_func_proto()
2116 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_func_proto()
2120 btf->hdr->type_len += sz; in btf__add_func_proto()
2121 btf->hdr->str_off += sz; in btf__add_func_proto()
2122 btf->nr_types++; in btf__add_func_proto()
2123 return btf->nr_types; in btf__add_func_proto()
2127 * Append new function parameter for current FUNC_PROTO type with:
2128 * - *name* - parameter name, can be NULL or empty;
2129 * - *type_id* - type ID describing the type of the parameter.
2131 * - 0, on success;
2132 * - <0, on error.
2141 return -EINVAL; in btf__add_func_param()
2144 if (btf->nr_types == 0) in btf__add_func_param()
2145 return -EINVAL; in btf__add_func_param()
2146 t = btf_type_by_id(btf, btf->nr_types); in btf__add_func_param()
2148 return -EINVAL; in btf__add_func_param()
2152 return -ENOMEM; in btf__add_func_param()
2157 return -ENOMEM; in btf__add_func_param()
2165 p->name_off = name_off; in btf__add_func_param()
2166 p->type = type_id; in btf__add_func_param()
2169 t = btf_type_by_id(btf, btf->nr_types); in btf__add_func_param()
2172 btf->hdr->type_len += sz; in btf__add_func_param()
2173 btf->hdr->str_off += sz; in btf__add_func_param()
2179 * - *name* - non-empty/non-NULL name;
2180 * - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2182 * - *type_id* - type ID of the type describing the type of the variable.
2184 * - >0, type ID of newly added BTF type;
2185 * - <0, on error.
2193 /* non-empty name */ in btf__add_var()
2195 return -EINVAL; in btf__add_var()
2198 return -EINVAL; in btf__add_var()
2200 return -EINVAL; in btf__add_var()
2204 return -ENOMEM; in btf__add_var()
2209 return -ENOMEM; in btf__add_var()
2215 t->name_off = name_off; in btf__add_var()
2216 t->info = btf_type_info(BTF_KIND_VAR, 0, 0); in btf__add_var()
2217 t->type = type_id; in btf__add_var()
2220 v->linkage = linkage; in btf__add_var()
2222 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_var()
2226 btf->hdr->type_len += sz; in btf__add_var()
2227 btf->hdr->str_off += sz; in btf__add_var()
2228 btf->nr_types++; in btf__add_var()
2229 return btf->nr_types; in btf__add_var()
2234 * - *name* - non-empty/non-NULL name;
2235 * - *byte_sz* - data section size, in bytes.
2241 * - >0, type ID of newly added BTF type;
2242 * - <0, on error.
2249 /* non-empty name */ in btf__add_datasec()
2251 return -EINVAL; in btf__add_datasec()
2254 return -ENOMEM; in btf__add_datasec()
2259 return -ENOMEM; in btf__add_datasec()
2266 t->name_off = name_off; in btf__add_datasec()
2267 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); in btf__add_datasec()
2268 t->size = byte_sz; in btf__add_datasec()
2270 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_datasec()
2274 btf->hdr->type_len += sz; in btf__add_datasec()
2275 btf->hdr->str_off += sz; in btf__add_datasec()
2276 btf->nr_types++; in btf__add_datasec()
2277 return btf->nr_types; in btf__add_datasec()
2282 * - *var_type_id* - type ID, describing type of the variable;
2283 * - *offset* - variable offset within data section, in bytes;
2284 * - *byte_sz* - variable size, in bytes.
2287 * - 0, on success;
2288 * - <0, on error.
2297 if (btf->nr_types == 0) in btf__add_datasec_var_info()
2298 return -EINVAL; in btf__add_datasec_var_info()
2299 t = btf_type_by_id(btf, btf->nr_types); in btf__add_datasec_var_info()
2301 return -EINVAL; in btf__add_datasec_var_info()
2304 return -EINVAL; in btf__add_datasec_var_info()
2308 return -ENOMEM; in btf__add_datasec_var_info()
2313 return -ENOMEM; in btf__add_datasec_var_info()
2315 v->type = var_type_id; in btf__add_datasec_var_info()
2316 v->offset = offset; in btf__add_datasec_var_info()
2317 v->size = byte_sz; in btf__add_datasec_var_info()
2320 t = btf_type_by_id(btf, btf->nr_types); in btf__add_datasec_var_info()
2323 btf->hdr->type_len += sz; in btf__add_datasec_var_info()
2324 btf->hdr->str_off += sz; in btf__add_datasec_var_info()
2345 if (ext_sec->len == 0) in btf_ext_setup_info()
2348 if (ext_sec->off & 0x03) { in btf_ext_setup_info()
2350 ext_sec->desc); in btf_ext_setup_info()
2351 return -EINVAL; in btf_ext_setup_info()
2354 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; in btf_ext_setup_info()
2355 info_left = ext_sec->len; in btf_ext_setup_info()
2357 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { in btf_ext_setup_info()
2359 ext_sec->desc, ext_sec->off, ext_sec->len); in btf_ext_setup_info()
2360 return -EINVAL; in btf_ext_setup_info()
2365 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); in btf_ext_setup_info()
2366 return -EINVAL; in btf_ext_setup_info()
2371 if (record_size < ext_sec->min_rec_size || in btf_ext_setup_info()
2374 ext_sec->desc, record_size); in btf_ext_setup_info()
2375 return -EINVAL; in btf_ext_setup_info()
2379 info_left -= sizeof(__u32); in btf_ext_setup_info()
2383 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc); in btf_ext_setup_info()
2384 return -EINVAL; in btf_ext_setup_info()
2394 ext_sec->desc); in btf_ext_setup_info()
2395 return -EINVAL; in btf_ext_setup_info()
2398 num_records = sinfo->num_info; in btf_ext_setup_info()
2401 ext_sec->desc); in btf_ext_setup_info()
2402 return -EINVAL; in btf_ext_setup_info()
2409 ext_sec->desc); in btf_ext_setup_info()
2410 return -EINVAL; in btf_ext_setup_info()
2413 info_left -= total_record_size; in btf_ext_setup_info()
2417 ext_info = ext_sec->ext_info; in btf_ext_setup_info()
2418 ext_info->len = ext_sec->len - sizeof(__u32); in btf_ext_setup_info()
2419 ext_info->rec_size = record_size; in btf_ext_setup_info()
2420 ext_info->info = info + sizeof(__u32); in btf_ext_setup_info()
2428 .off = btf_ext->hdr->func_info_off, in btf_ext_setup_func_info()
2429 .len = btf_ext->hdr->func_info_len, in btf_ext_setup_func_info()
2431 .ext_info = &btf_ext->func_info, in btf_ext_setup_func_info()
2441 .off = btf_ext->hdr->line_info_off, in btf_ext_setup_line_info()
2442 .len = btf_ext->hdr->line_info_len, in btf_ext_setup_line_info()
2444 .ext_info = &btf_ext->line_info, in btf_ext_setup_line_info()
2454 .off = btf_ext->hdr->core_relo_off, in btf_ext_setup_core_relos()
2455 .len = btf_ext->hdr->core_relo_len, in btf_ext_setup_core_relos()
2457 .ext_info = &btf_ext->core_relo_info, in btf_ext_setup_core_relos()
2469 data_size < hdr->hdr_len) { in btf_ext_parse_hdr()
2471 return -EINVAL; in btf_ext_parse_hdr()
2474 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_ext_parse_hdr()
2475 pr_warn("BTF.ext in non-native endianness is not supported\n"); in btf_ext_parse_hdr()
2476 return -ENOTSUP; in btf_ext_parse_hdr()
2477 } else if (hdr->magic != BTF_MAGIC) { in btf_ext_parse_hdr()
2478 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); in btf_ext_parse_hdr()
2479 return -EINVAL; in btf_ext_parse_hdr()
2482 if (hdr->version != BTF_VERSION) { in btf_ext_parse_hdr()
2483 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); in btf_ext_parse_hdr()
2484 return -ENOTSUP; in btf_ext_parse_hdr()
2487 if (hdr->flags) { in btf_ext_parse_hdr()
2488 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); in btf_ext_parse_hdr()
2489 return -ENOTSUP; in btf_ext_parse_hdr()
2492 if (data_size == hdr->hdr_len) { in btf_ext_parse_hdr()
2494 return -EINVAL; in btf_ext_parse_hdr()
2504 free(btf_ext->data); in btf_ext__free()
2519 return ERR_PTR(-ENOMEM); in btf_ext__new()
2521 btf_ext->data_size = size; in btf_ext__new()
2522 btf_ext->data = malloc(size); in btf_ext__new()
2523 if (!btf_ext->data) { in btf_ext__new()
2524 err = -ENOMEM; in btf_ext__new()
2527 memcpy(btf_ext->data, data, size); in btf_ext__new()
2529 if (btf_ext->hdr->hdr_len < in btf_ext__new()
2540 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) in btf_ext__new()
2557 *size = btf_ext->data_size; in btf_ext__get_raw_data()
2558 return btf_ext->data; in btf_ext__get_raw_data()
2573 record_size = ext_info->rec_size; in btf_ext_reloc_info()
2574 sinfo = ext_info->info; in btf_ext_reloc_info()
2575 remain_len = ext_info->len; in btf_ext_reloc_info()
2577 records_len = sinfo->num_info * record_size; in btf_ext_reloc_info()
2578 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); in btf_ext_reloc_info()
2580 remain_len -= sec_hdrlen + records_len; in btf_ext_reloc_info()
2588 return -ENOMEM; in btf_ext_reloc_info()
2590 memcpy(data + existing_len, sinfo->data, records_len); in btf_ext_reloc_info()
2594 for (i = 0; i < sinfo->num_info; i++) { in btf_ext_reloc_info()
2602 *cnt += sinfo->num_info; in btf_ext_reloc_info()
2606 return -ENOENT; in btf_ext_reloc_info()
2614 return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name, in btf_ext__reloc_func_info()
2623 return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name, in btf_ext__reloc_line_info()
2629 return btf_ext->func_info.rec_size; in btf_ext__func_info_rec_size()
2634 return btf_ext->line_info.rec_size; in btf_ext__line_info_rec_size()
2654 * memory in-place with deduplicated types and strings without any loss of
2659 * If function returns with error, type/string data might be garbled and should
2664 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
2771 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
2775 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
2778 * hashing function, we can limit number of canonical types for each unique type
2784 * `btf_dedup_is_equiv` function.
2794 return -EINVAL; in btf__dedup()
2798 return -ENOMEM; in btf__dedup()
2836 #define BTF_UNPROCESSED_ID ((__u32)-1)
2837 #define BTF_IN_PROGRESS_ID ((__u32)-2)
2840 /* .BTF section to be deduped in-place */
2885 hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash)
2889 return hashmap__append(d->dedup_table, in btf_dedup_table_add()
2896 if (d->hypot_cnt == d->hypot_cap) { in btf_dedup_hypot_map_add()
2899 d->hypot_cap += max((size_t)16, d->hypot_cap / 2); in btf_dedup_hypot_map_add()
2900 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); in btf_dedup_hypot_map_add()
2902 return -ENOMEM; in btf_dedup_hypot_map_add()
2903 d->hypot_list = new_list; in btf_dedup_hypot_map_add()
2905 d->hypot_list[d->hypot_cnt++] = from_id; in btf_dedup_hypot_map_add()
2906 d->hypot_map[from_id] = to_id; in btf_dedup_hypot_map_add()
2914 for (i = 0; i < d->hypot_cnt; i++) in btf_dedup_clear_hypot_map()
2915 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; in btf_dedup_clear_hypot_map()
2916 d->hypot_cnt = 0; in btf_dedup_clear_hypot_map()
2921 hashmap__free(d->dedup_table); in btf_dedup_free()
2922 d->dedup_table = NULL; in btf_dedup_free()
2924 free(d->map); in btf_dedup_free()
2925 d->map = NULL; in btf_dedup_free()
2927 free(d->hypot_map); in btf_dedup_free()
2928 d->hypot_map = NULL; in btf_dedup_free()
2930 free(d->hypot_list); in btf_dedup_free()
2931 d->hypot_list = NULL; in btf_dedup_free()
2959 return ERR_PTR(-ENOMEM); in btf_dedup_new()
2961 d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds; in btf_dedup_new()
2963 if (opts && opts->dedup_table_size == 1) in btf_dedup_new()
2966 d->btf = btf; in btf_dedup_new()
2967 d->btf_ext = btf_ext; in btf_dedup_new()
2969 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); in btf_dedup_new()
2970 if (IS_ERR(d->dedup_table)) { in btf_dedup_new()
2971 err = PTR_ERR(d->dedup_table); in btf_dedup_new()
2972 d->dedup_table = NULL; in btf_dedup_new()
2976 d->map = malloc(sizeof(__u32) * (1 + btf->nr_types)); in btf_dedup_new()
2977 if (!d->map) { in btf_dedup_new()
2978 err = -ENOMEM; in btf_dedup_new()
2982 d->map[0] = 0; in btf_dedup_new()
2983 for (i = 1; i <= btf->nr_types; i++) { in btf_dedup_new()
2984 struct btf_type *t = btf_type_by_id(d->btf, i); in btf_dedup_new()
2986 /* VAR and DATASEC are never deduped and are self-canonical */ in btf_dedup_new()
2988 d->map[i] = i; in btf_dedup_new()
2990 d->map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
2993 d->hypot_map = malloc(sizeof(__u32) * (1 + btf->nr_types)); in btf_dedup_new()
2994 if (!d->hypot_map) { in btf_dedup_new()
2995 err = -ENOMEM; in btf_dedup_new()
2998 for (i = 0; i <= btf->nr_types; i++) in btf_dedup_new()
2999 d->hypot_map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
3022 for (i = 1; i <= d->btf->nr_types; i++) { in btf_for_each_str_off()
3023 t = btf_type_by_id(d->btf, i); in btf_for_each_str_off()
3024 r = fn(&t->name_off, ctx); in btf_for_each_str_off()
3035 r = fn(&m->name_off, ctx); in btf_for_each_str_off()
3047 r = fn(&m->name_off, ctx); in btf_for_each_str_off()
3059 r = fn(&m->name_off, ctx); in btf_for_each_str_off()
3071 if (!d->btf_ext) in btf_for_each_str_off()
3074 line_data_cur = d->btf_ext->line_info.info; in btf_for_each_str_off()
3075 line_data_end = d->btf_ext->line_info.info + d->btf_ext->line_info.len; in btf_for_each_str_off()
3076 rec_size = d->btf_ext->line_info.rec_size; in btf_for_each_str_off()
3081 __u32 num_info = sec->num_info; in btf_for_each_str_off()
3083 r = fn(&sec->sec_name_off, ctx); in btf_for_each_str_off()
3090 r = fn(&line_info->file_name_off, ctx); in btf_for_each_str_off()
3093 r = fn(&line_info->line_off, ctx); in btf_for_each_str_off()
3108 return strcmp(p1->str, p2->str); in str_sort_by_content()
3116 if (p1->str != p2->str) in str_sort_by_offset()
3117 return p1->str < p2->str ? -1 : 1; in str_sort_by_offset()
3125 if (str_ptr != p->str) in btf_dedup_str_ptr_cmp()
3126 return (const char *)str_ptr < p->str ? -1 : 1; in btf_dedup_str_ptr_cmp()
3139 s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt, in btf_str_mark_as_used()
3142 return -EINVAL; in btf_str_mark_as_used()
3143 s->used = true; in btf_str_mark_as_used()
3156 s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt, in btf_str_remap_offset()
3159 return -EINVAL; in btf_str_remap_offset()
3160 *str_off_ptr = s->new_off; in btf_str_remap_offset()
3177 char *start = d->btf->strs_data; in btf_dedup_strings()
3178 char *end = start + d->btf->hdr->str_len; in btf_dedup_strings()
3189 if (d->btf->strs_deduped) in btf_dedup_strings()
3200 err = -ENOMEM; in btf_dedup_strings()
3214 tmp_strs = malloc(d->btf->hdr->str_len); in btf_dedup_strings()
3216 err = -ENOMEM; in btf_dedup_strings()
3255 int new_off = p - tmp_strs; in btf_dedup_strings()
3271 d->btf->hdr->str_len = p - tmp_strs; in btf_dedup_strings()
3272 memmove(start, tmp_strs, d->btf->hdr->str_len); in btf_dedup_strings()
3273 end = start + d->btf->hdr->str_len; in btf_dedup_strings()
3283 d->btf->hdr->str_len = end - start; in btf_dedup_strings()
3284 d->btf->strs_deduped = true; in btf_dedup_strings()
3296 h = hash_combine(0, t->name_off); in btf_hash_common()
3297 h = hash_combine(h, t->info); in btf_hash_common()
3298 h = hash_combine(h, t->size); in btf_hash_common()
3304 return t1->name_off == t2->name_off && in btf_equal_common()
3305 t1->info == t2->info && in btf_equal_common()
3306 t1->size == t2->size; in btf_equal_common()
3338 h = hash_combine(0, t->name_off); in btf_hash_enum()
3339 h = hash_combine(h, t->info & ~0xffff); in btf_hash_enum()
3340 h = hash_combine(h, t->size); in btf_hash_enum()
3358 if (m1->name_off != m2->name_off || m1->val != m2->val) in btf_equal_enum()
3376 return t1->name_off == t2->name_off && in btf_compat_enum()
3377 (t1->info & ~0xffff) == (t2->info & ~0xffff) && in btf_compat_enum()
3378 t1->size == t2->size; in btf_compat_enum()
3394 h = hash_combine(h, member->name_off); in btf_hash_struct()
3395 h = hash_combine(h, member->offset); in btf_hash_struct()
3420 if (m1->name_off != m2->name_off || m1->offset != m2->offset) in btf_shallow_equal_struct()
3438 h = hash_combine(h, info->type); in btf_hash_array()
3439 h = hash_combine(h, info->index_type); in btf_hash_array()
3440 h = hash_combine(h, info->nelems); in btf_hash_array()
3448 * This function is called during reference types deduplication to compare
3460 return info1->type == info2->type && in btf_equal_array()
3461 info1->index_type == info2->index_type && in btf_equal_array()
3462 info1->nelems == info2->nelems; in btf_equal_array()
3475 return btf_array(t1)->nelems == btf_array(t2)->nelems; in btf_compat_array()
3491 h = hash_combine(h, member->name_off); in btf_hash_fnproto()
3492 h = hash_combine(h, member->type); in btf_hash_fnproto()
3502 * This function is called during reference types deduplication to compare
3518 if (m1->name_off != m2->name_off || m1->type != m2->type) in btf_equal_fnproto()
3538 if (t1->name_off != t2->name_off || t1->info != t2->info) in btf_compat_fnproto()
3545 if (m1->name_off != m2->name_off) in btf_compat_fnproto()
3557 * canonical and is added into `btf_dedup->dedup_table` as another candidate.
3561 struct btf_type *t = btf_type_by_id(d->btf, type_id); in btf_dedup_prim_type()
3587 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3588 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3599 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3600 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3605 if (d->opts.dont_resolve_fwds) in btf_dedup_prim_type()
3614 d->map[cand_id] = type_id; in btf_dedup_prim_type()
3622 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3623 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3632 return -EINVAL; in btf_dedup_prim_type()
3635 d->map[type_id] = new_id; in btf_dedup_prim_type()
3637 return -ENOMEM; in btf_dedup_prim_type()
3646 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_prim_types()
3659 return d->map[type_id] <= BTF_MAX_NR_TYPES; in is_type_mapped()
3669 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_type_id()
3670 type_id = d->map[type_id]; in resolve_type_id()
3682 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3685 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_fwd_id()
3686 type_id = d->map[type_id]; in resolve_fwd_id()
3688 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3708 * - >0, if type graphs are equivalent;
3709 * - 0, if not equivalent;
3710 * - <0, on error.
3712 * Algorithm performs side-by-side DFS traversal of both type graphs and checks
3717 * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`).
3724 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will
3727 * - `canon_id` is mapped to different type. This is contradiction in our
3741 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
3766 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
3770 * - Both types in canonical and candidate graphs are FWDs. If they are
3774 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
3776 * - Type in canonical graph is FWD, while type in candidate is concrete
3785 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
3790 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
3809 hypot_type_id = d->hypot_map[canon_id]; in btf_dedup_is_equiv()
3814 return -ENOMEM; in btf_dedup_is_equiv()
3816 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_is_equiv()
3817 canon_type = btf_type_by_id(d->btf, canon_id); in btf_dedup_is_equiv()
3821 if (cand_type->name_off != canon_type->name_off) in btf_dedup_is_equiv()
3824 /* FWD <--> STRUCT/UNION equivalence check, if enabled */ in btf_dedup_is_equiv()
3825 if (!d->opts.dont_resolve_fwds in btf_dedup_is_equiv()
3849 if (d->opts.dont_resolve_fwds) in btf_dedup_is_equiv()
3863 if (cand_type->info != canon_type->info) in btf_dedup_is_equiv()
3865 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
3875 cand_arr->index_type, canon_arr->index_type); in btf_dedup_is_equiv()
3878 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); in btf_dedup_is_equiv()
3892 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); in btf_dedup_is_equiv()
3908 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
3915 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); in btf_dedup_is_equiv()
3925 return -EINVAL; in btf_dedup_is_equiv()
3935 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
3963 for (i = 0; i < d->hypot_cnt; i++) { in btf_dedup_merge_hypot_map()
3964 cand_type_id = d->hypot_list[i]; in btf_dedup_merge_hypot_map()
3965 targ_type_id = d->hypot_map[cand_type_id]; in btf_dedup_merge_hypot_map()
3968 t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); in btf_dedup_merge_hypot_map()
3969 c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); in btf_dedup_merge_hypot_map()
3974 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because in btf_dedup_merge_hypot_map()
3983 d->map[c_id] = t_id; in btf_dedup_merge_hypot_map()
3985 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
3997 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
4014 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4015 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4021 * as canonical for itself and is added into btf_dedup->dedup_table hash map
4034 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_struct_type()
4037 t = btf_type_by_id(d->btf, type_id); in btf_dedup_struct_type()
4045 __u32 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_struct_type()
4055 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because in btf_dedup_struct_type()
4058 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_struct_type()
4073 d->map[type_id] = new_id; in btf_dedup_struct_type()
4075 return -ENOMEM; in btf_dedup_struct_type()
4084 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_struct_types()
4114 * btf_dedup->dedup_table as another possible canonical representative.
4125 if (d->map[type_id] == BTF_IN_PROGRESS_ID) in btf_dedup_ref_type()
4126 return -ELOOP; in btf_dedup_ref_type()
4127 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_ref_type()
4130 t = btf_type_by_id(d->btf, type_id); in btf_dedup_ref_type()
4131 d->map[type_id] = BTF_IN_PROGRESS_ID; in btf_dedup_ref_type()
4140 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4143 t->type = ref_type_id; in btf_dedup_ref_type()
4147 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4148 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4159 ref_type_id = btf_dedup_ref_type(d, info->type); in btf_dedup_ref_type()
4162 info->type = ref_type_id; in btf_dedup_ref_type()
4164 ref_type_id = btf_dedup_ref_type(d, info->index_type); in btf_dedup_ref_type()
4167 info->index_type = ref_type_id; in btf_dedup_ref_type()
4171 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4172 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4186 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4189 t->type = ref_type_id; in btf_dedup_ref_type()
4194 ref_type_id = btf_dedup_ref_type(d, param->type); in btf_dedup_ref_type()
4197 param->type = ref_type_id; in btf_dedup_ref_type()
4203 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4204 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4214 return -EINVAL; in btf_dedup_ref_type()
4217 d->map[type_id] = new_id; in btf_dedup_ref_type()
4219 return -ENOMEM; in btf_dedup_ref_type()
4228 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_ref_types()
4233 /* we won't need d->dedup_table anymore */ in btf_dedup_ref_types()
4234 hashmap__free(d->dedup_table); in btf_dedup_ref_types()
4235 d->dedup_table = NULL; in btf_dedup_ref_types()
4245 * duplicates. During compaction btf_dedup->hypot_map array is reused to store
4258 d->hypot_map[0] = 0; in btf_dedup_compact_types()
4259 for (i = 1; i <= d->btf->nr_types; i++) in btf_dedup_compact_types()
4260 d->hypot_map[i] = BTF_UNPROCESSED_ID; in btf_dedup_compact_types()
4262 p = d->btf->types_data; in btf_dedup_compact_types()
4264 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_compact_types()
4265 if (d->map[i] != i) in btf_dedup_compact_types()
4268 len = btf_type_size(btf__type_by_id(d->btf, i)); in btf_dedup_compact_types()
4272 memmove(p, btf__type_by_id(d->btf, i), len); in btf_dedup_compact_types()
4273 d->hypot_map[i] = next_type_id; in btf_dedup_compact_types()
4274 d->btf->type_offs[next_type_id] = p - d->btf->types_data; in btf_dedup_compact_types()
4280 d->btf->nr_types = next_type_id - 1; in btf_dedup_compact_types()
4281 d->btf->type_offs_cap = d->btf->nr_types + 1; in btf_dedup_compact_types()
4282 d->btf->hdr->type_len = p - d->btf->types_data; in btf_dedup_compact_types()
4283 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, in btf_dedup_compact_types()
4286 return -ENOMEM; in btf_dedup_compact_types()
4287 d->btf->type_offs = new_offs; in btf_dedup_compact_types()
4288 d->btf->hdr->str_off = d->btf->hdr->type_len; in btf_dedup_compact_types()
4289 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; in btf_dedup_compact_types()
4296 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
4304 new_type_id = d->hypot_map[resolved_type_id]; in btf_dedup_remap_type_id()
4306 return -EINVAL; in btf_dedup_remap_type_id()
4315 * deduped type ID is stored in btf_dedup->hypot_map and is populated during
4322 struct btf_type *t = btf_type_by_id(d->btf, type_id); in btf_dedup_remap_type()
4338 r = btf_dedup_remap_type_id(d, t->type); in btf_dedup_remap_type()
4341 t->type = r; in btf_dedup_remap_type()
4347 r = btf_dedup_remap_type_id(d, arr_info->type); in btf_dedup_remap_type()
4350 arr_info->type = r; in btf_dedup_remap_type()
4351 r = btf_dedup_remap_type_id(d, arr_info->index_type); in btf_dedup_remap_type()
4354 arr_info->index_type = r; in btf_dedup_remap_type()
4364 r = btf_dedup_remap_type_id(d, member->type); in btf_dedup_remap_type()
4367 member->type = r; in btf_dedup_remap_type()
4377 r = btf_dedup_remap_type_id(d, t->type); in btf_dedup_remap_type()
4380 t->type = r; in btf_dedup_remap_type()
4383 r = btf_dedup_remap_type_id(d, param->type); in btf_dedup_remap_type()
4386 param->type = r; in btf_dedup_remap_type()
4397 r = btf_dedup_remap_type_id(d, var->type); in btf_dedup_remap_type()
4400 var->type = r; in btf_dedup_remap_type()
4407 return -EINVAL; in btf_dedup_remap_type()
4417 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_remap_types()
4426 * Probe few well-known locations for vmlinux kernel image and try to load BTF
4438 { "/boot/vmlinux-%1$s" }, in libbpf_find_kernel_btf()
4439 { "/lib/modules/%1$s/vmlinux-%1$s" }, in libbpf_find_kernel_btf()
4442 { "/usr/lib/debug/boot/vmlinux-%1$s" }, in libbpf_find_kernel_btf()
4443 { "/usr/lib/debug/boot/vmlinux-%1$s.debug" }, in libbpf_find_kernel_btf()
4473 return ERR_PTR(-ESRCH); in libbpf_find_kernel_btf()