Lines Matching +full:enum +full:- +full:cnt +full:- +full:name
1 // SPDX-License-Identifier: GPL-2.0
74 * NOTE that we cannot assume any reference-order.
80 * object describing "void *". This type-reference is done
87 * - Each line started with "[?]" is a btf_type object
88 * - [?] is the type_id of the btf_type object.
89 * - CONST/PTR is the BTF_KIND_XXX
90 * - "(anon)" is the name of the type. It just
91 * happens that CONST and PTR has no name.
92 * - type_id=XXX is the 'u32 type' in btf_type
106 * have a name.
116 * an array: "btf->types".
126 * check this type-reference in the first pass.
129 * checking the name is a valid offset to the string section).
137 * 1) does exist in the BTF (i.e. in btf->types[])
170 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
172 * +-----------------------------------------+
177 #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
207 enum btf_kfunc_hook {
225 enum {
242 u32 cnt; member
247 u32 cnt; member
275 char name[MODULE_NAME_LEN]; member
277 __u32 *base_id_map; /* map from distilled base BTF -> vmlinux BTF ids */
280 enum verifier_phase {
291 enum visit_state {
297 enum resolve_mode {
319 enum verifier_phase phase;
320 enum resolve_mode resolve_mode;
330 [BTF_KIND_ENUM] = "ENUM",
348 return btf_kind_str[BTF_INFO_KIND(t->info)]; in btf_type_str()
356 * 128-bit int); if we are at the end of our safe buffer and have
363 /* Type name size */
377 * One challenge with showing nested data is we want to skip 0-valued
382 * pass is signalled by show->state.depth_check being set, and if we
383 * encounter a non-zero value we set show->state.depth_to_show to
401 * as we traverse the object's data. skbuff-like semantics are
404 * - obj.head points to the start of the toplevel object for display
405 * - obj.size is the size of the toplevel object
406 * - obj.data points to the current point in the original data at
429 int status; /* non-zero for error */
432 char name[BTF_SHOW_NAME_SIZE]; /* space for member name/type */ member
478 * type through t->type AND its size cannot in btf_type_is_modifier()
479 * be determined without following the t->type. in btf_type_is_modifier()
484 switch (BTF_INFO_KIND(t->info)) { in btf_type_is_modifier()
503 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; in btf_type_is_datasec()
508 return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; in btf_type_is_decl_tag()
531 return btf->kernel_btf && !btf->base_btf; in btf_is_vmlinux()
539 total += btf->nr_types; in btf_nr_types()
540 btf = btf->base_btf; in btf_nr_types()
546 s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind) in btf_find_by_name_kind() argument
555 if (BTF_INFO_KIND(t->info) != kind) in btf_find_by_name_kind()
558 tname = btf_name_by_offset(btf, t->name_off); in btf_find_by_name_kind()
559 if (!strcmp(tname, name)) in btf_find_by_name_kind()
563 return -ENOENT; in btf_find_by_name_kind()
566 s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p) in bpf_find_btf_id() argument
576 return -EINVAL; in bpf_find_btf_id()
578 ret = btf_find_by_name_kind(btf, name, kind); in bpf_find_btf_id()
588 /* If name is not found in vmlinux's BTF then search in module's BTFs */ in bpf_find_btf_id()
598 ret = btf_find_by_name_kind(btf, name, kind); in bpf_find_btf_id()
617 id = t->type; in btf_type_skip_modifiers()
618 t = btf_type_by_id(btf, t->type); in btf_type_skip_modifiers()
636 return btf_type_skip_modifiers(btf, t->type, res_id); in btf_type_resolve_ptr()
666 * another type (through member->type).
672 * btf_type_is_array() because its element (array->type)
675 * member-type repeated by array->nelems of times.
689 /* t->size can be used */
692 switch (BTF_INFO_KIND(t->info)) { in btf_type_has_size()
752 return kind_ops[BTF_INFO_KIND(t->info)]; in btf_type_ops()
760 while (offset < btf->start_str_off) in btf_name_offset_valid()
761 btf = btf->base_btf; in btf_name_offset_valid()
763 offset -= btf->start_str_off; in btf_name_offset_valid()
764 return offset < btf->hdr.str_len; in btf_name_offset_valid()
779 while (offset < btf->start_str_off) in btf_str_by_offset()
780 btf = btf->base_btf; in btf_str_by_offset()
782 offset -= btf->start_str_off; in btf_str_by_offset()
783 if (offset < btf->hdr.str_len) in btf_str_by_offset()
784 return &btf->strings[offset]; in btf_str_by_offset()
833 const char *name; in __btf_name_by_offset() local
838 name = btf_str_by_offset(btf, offset); in __btf_name_by_offset()
839 return name ?: "(invalid-name-offset)"; in __btf_name_by_offset()
849 while (type_id < btf->start_id) in btf_type_by_id()
850 btf = btf->base_btf; in btf_type_by_id()
852 type_id -= btf->start_id; in btf_type_by_id()
853 if (type_id >= btf->nr_types) in btf_type_by_id()
855 return btf->types[type_id]; in btf_type_by_id()
894 id = m->type; in btf_member_is_reg_int()
902 u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset); in btf_member_is_reg_int()
903 u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset); in btf_member_is_reg_int()
914 BITS_PER_BYTE_MASKED(m->offset) || in btf_member_is_reg_int()
915 BITS_ROUNDUP_BYTES(m->offset) != expected_offset || in btf_member_is_reg_int()
930 BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) { in btf_type_skip_qualifiers()
931 t = btf_type_by_id(btf, t->type); in btf_type_skip_qualifiers()
942 * Populate show->state.name with type name information.
943 * Format of type name is
955 const char *name = NULL, *prefix = "", *parens = ""; in btf_show_name() local
956 const struct btf_member *m = show->state.member; in btf_show_name()
959 u32 id = show->state.type_id; in btf_show_name()
965 show->state.name[0] = '\0'; in btf_show_name()
968 * Don't show type name if we're showing an array member; in btf_show_name()
972 if (show->state.array_member) in btf_show_name()
975 /* Retrieve member name, if any. */ in btf_show_name()
977 member = btf_name_by_offset(show->btf, m->name_off); in btf_show_name()
979 id = m->type; in btf_show_name()
987 * in our show->state points at the resolved type of the typedef. in btf_show_name()
989 t = btf_type_by_id(show->btf, id); in btf_show_name()
995 * array suffixes while ensuring the type name for a typedef in btf_show_name()
1011 * We also want to get typedef name while proceeding to resolve in btf_show_name()
1017 switch (BTF_INFO_KIND(t->info)) { in btf_show_name()
1019 if (!name) in btf_show_name()
1020 name = btf_name_by_offset(show->btf, in btf_show_name()
1021 t->name_off); in btf_show_name()
1023 id = t->type; in btf_show_name()
1032 array_suffix -= 2; in btf_show_name()
1033 id = array->type; in btf_show_name()
1038 ptr_suffix -= 1; in btf_show_name()
1039 id = t->type; in btf_show_name()
1047 t = btf_type_skip_qualifiers(show->btf, id); in btf_show_name()
1053 if (!name) in btf_show_name()
1054 name = btf_name_by_offset(show->btf, t->name_off); in btf_show_name()
1056 switch (BTF_INFO_KIND(t->info)) { in btf_show_name()
1059 prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ? in btf_show_name()
1067 prefix = "enum"; in btf_show_name()
1076 /* typedef does not require struct/union/enum prefix */ in btf_show_name()
1080 if (!name) in btf_show_name()
1081 name = ""; in btf_show_name()
1083 /* Even if we don't want type name info, we want parentheses etc */ in btf_show_name()
1084 if (show->flags & BTF_SHOW_NONAME) in btf_show_name()
1085 snprintf(show->state.name, sizeof(show->state.name), "%s", in btf_show_name()
1088 snprintf(show->state.name, sizeof(show->state.name), in btf_show_name()
1094 /* ...next is our prefix (struct, enum, etc) */ in btf_show_name()
1096 strlen(prefix) > 0 && strlen(name) > 0 ? " " : "", in btf_show_name()
1097 /* ...this is the type name itself */ in btf_show_name()
1098 name, in btf_show_name()
1103 return show->state.name; in btf_show_name()
1111 if ((indent - show->state.depth) >= indents) in __btf_show_indent()
1112 return indent - show->state.depth; in __btf_show_indent()
1118 return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show); in btf_show_indent()
1123 return show->flags & BTF_SHOW_COMPACT ? "" : "\n"; in btf_show_newline()
1128 if (show->state.depth == 0) in btf_show_delim()
1131 if ((show->flags & BTF_SHOW_COMPACT) && show->state.type && in btf_show_delim()
1132 BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION) in btf_show_delim()
1142 if (!show->state.depth_check) { in btf_show()
1144 show->showfn(show, fmt, args); in btf_show()
1157 (show->flags & BTF_SHOW_ZERO) || \
1158 show->state.depth == 0) { \
1164 if (show->state.depth > show->state.depth_to_show) \
1165 show->state.depth_to_show = show->state.depth; \
1175 if (show->state.depth > show->state.depth_to_show) \
1176 show->state.depth_to_show = show->state.depth; \
1182 return show->obj.head + show->obj.size - data; in btf_show_obj_size_left()
1188 return data >= show->obj.data && in btf_show_obj_is_safe()
1189 (data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE); in btf_show_obj_is_safe()
1201 return show->obj.safe + (data - show->obj.data); in __btf_show_obj_safe()
1206 * Return a safe-to-access version of data pointed to by @data.
1210 * If BTF_SHOW_UNSAFE is specified, just return data as-is; no
1229 * We use stack data as opposed to per-CPU buffers because the
1241 if (show->flags & BTF_SHOW_UNSAFE) in btf_show_obj_safe()
1244 rt = btf_resolve_size(show->btf, t, &size); in btf_show_obj_safe()
1246 show->state.status = PTR_ERR(rt); in btf_show_obj_safe()
1255 if (show->state.depth == 0) { in btf_show_obj_safe()
1256 show->obj.size = size; in btf_show_obj_safe()
1257 show->obj.head = data; in btf_show_obj_safe()
1276 * - the current type size is within the safe buffer; or in btf_show_obj_safe()
1277 * - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in in btf_show_obj_safe()
1294 show->state.status = copy_from_kernel_nofault(show->obj.safe, in btf_show_obj_safe()
1296 if (!show->state.status) { in btf_show_obj_safe()
1297 show->obj.data = data; in btf_show_obj_safe()
1298 safe = show->obj.safe; in btf_show_obj_safe()
1313 show->state.type = t; in btf_show_start_type()
1314 show->state.type_id = type_id; in btf_show_start_type()
1315 show->state.name[0] = '\0'; in btf_show_start_type()
1322 show->state.type = NULL; in btf_show_end_type()
1323 show->state.type_id = 0; in btf_show_end_type()
1324 show->state.name[0] = '\0'; in btf_show_end_type()
1339 show->state.depth++; in btf_show_start_aggr_type()
1346 show->state.depth--; in btf_show_end_aggr_type()
1355 show->state.member = m; in btf_show_start_member()
1360 show->state.array_member = 1; in btf_show_start_array_member()
1366 show->state.member = NULL; in btf_show_end_member()
1371 show->state.array_member = 0; in btf_show_end_array_member()
1381 show->state.array_encoding = array_encoding; in btf_show_start_array_type()
1382 show->state.array_terminated = 0; in btf_show_start_array_type()
1388 show->state.array_encoding = 0; in btf_show_end_array_type()
1389 show->state.array_terminated = 0; in btf_show_end_array_type()
1419 struct bpf_verifier_log *log = &env->log; in btf_verifier_log()
1435 struct bpf_verifier_log *log = &env->log; in __btf_verifier_log_type()
1436 struct btf *btf = env->btf; in __btf_verifier_log_type()
1442 if (log->level == BPF_LOG_KERNEL) { in __btf_verifier_log_type()
1445 * Skip those prints for in-kernel BTF verification. in __btf_verifier_log_type()
1451 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in __btf_verifier_log_type()
1456 env->log_type_id, in __btf_verifier_log_type()
1458 __btf_name_by_offset(btf, t->name_off), in __btf_verifier_log_type()
1462 btf_type_ops(t)->log_details(env, t); in __btf_verifier_log_type()
1485 struct bpf_verifier_log *log = &env->log; in btf_verifier_log_member()
1486 struct btf *btf = env->btf; in btf_verifier_log_member()
1492 if (log->level == BPF_LOG_KERNEL) { in btf_verifier_log_member()
1497 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in btf_verifier_log_member()
1507 if (env->phase != CHECK_META) in btf_verifier_log_member()
1513 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1514 member->type, in btf_verifier_log_member()
1515 BTF_MEMBER_BITFIELD_SIZE(member->offset), in btf_verifier_log_member()
1516 BTF_MEMBER_BIT_OFFSET(member->offset)); in btf_verifier_log_member()
1519 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1520 member->type, member->offset); in btf_verifier_log_member()
1538 struct bpf_verifier_log *log = &env->log; in btf_verifier_log_vsi()
1543 if (log->level == BPF_LOG_KERNEL && !fmt) in btf_verifier_log_vsi()
1545 if (env->phase != CHECK_META) in btf_verifier_log_vsi()
1549 vsi->type, vsi->offset, vsi->size); in btf_verifier_log_vsi()
1563 struct bpf_verifier_log *log = &env->log; in btf_verifier_log_hdr()
1564 const struct btf *btf = env->btf; in btf_verifier_log_hdr()
1570 if (log->level == BPF_LOG_KERNEL) in btf_verifier_log_hdr()
1572 hdr = &btf->hdr; in btf_verifier_log_hdr()
1573 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic); in btf_verifier_log_hdr()
1574 __btf_verifier_log(log, "version: %u\n", hdr->version); in btf_verifier_log_hdr()
1575 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags); in btf_verifier_log_hdr()
1576 __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len); in btf_verifier_log_hdr()
1577 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off); in btf_verifier_log_hdr()
1578 __btf_verifier_log(log, "type_len: %u\n", hdr->type_len); in btf_verifier_log_hdr()
1579 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off); in btf_verifier_log_hdr()
1580 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len); in btf_verifier_log_hdr()
1586 struct btf *btf = env->btf; in btf_add_type()
1588 if (btf->types_size == btf->nr_types) { in btf_add_type()
1594 if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { in btf_add_type()
1596 return -E2BIG; in btf_add_type()
1599 expand_by = max_t(u32, btf->types_size >> 2, 16); in btf_add_type()
1601 btf->types_size + expand_by); in btf_add_type()
1606 return -ENOMEM; in btf_add_type()
1608 if (btf->nr_types == 0) { in btf_add_type()
1609 if (!btf->base_btf) { in btf_add_type()
1612 btf->nr_types++; in btf_add_type()
1615 memcpy(new_types, btf->types, in btf_add_type()
1616 sizeof(*btf->types) * btf->nr_types); in btf_add_type()
1619 kvfree(btf->types); in btf_add_type()
1620 btf->types = new_types; in btf_add_type()
1621 btf->types_size = new_size; in btf_add_type()
1624 btf->types[btf->nr_types++] = t; in btf_add_type()
1637 btf->id = id; in btf_alloc_id()
1642 return -ENOSPC; in btf_alloc_id()
1652 * In map-in-map, calling map_delete_elem() on outer in btf_free_id()
1661 idr_remove(&btf_idr, btf->id); in btf_free_id()
1667 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; in btf_free_kfunc_set_tab()
1672 for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) in btf_free_kfunc_set_tab()
1673 kfree(tab->sets[hook]); in btf_free_kfunc_set_tab()
1675 btf->kfunc_set_tab = NULL; in btf_free_kfunc_set_tab()
1680 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_free_dtor_kfunc_tab()
1685 btf->dtor_kfunc_tab = NULL; in btf_free_dtor_kfunc_tab()
1694 for (i = 0; i < tab->cnt; i++) in btf_struct_metas_free()
1695 btf_record_free(tab->types[i].record); in btf_struct_metas_free()
1701 struct btf_struct_metas *tab = btf->struct_meta_tab; in btf_free_struct_meta_tab()
1704 btf->struct_meta_tab = NULL; in btf_free_struct_meta_tab()
1709 struct btf_struct_ops_tab *tab = btf->struct_ops_tab; in btf_free_struct_ops_tab()
1715 for (i = 0; i < tab->cnt; i++) in btf_free_struct_ops_tab()
1716 bpf_struct_ops_desc_release(&tab->ops[i]); in btf_free_struct_ops_tab()
1719 btf->struct_ops_tab = NULL; in btf_free_struct_ops_tab()
1728 kvfree(btf->types); in btf_free()
1729 kvfree(btf->resolved_sizes); in btf_free()
1730 kvfree(btf->resolved_ids); in btf_free()
1731 /* vmlinux does not allocate btf->data, it simply points it at in btf_free()
1735 kvfree(btf->data); in btf_free()
1736 kvfree(btf->base_id_map); in btf_free()
1749 return btf->name; in btf_get_name()
1754 refcount_inc(&btf->refcnt); in btf_get()
1759 if (btf && refcount_dec_and_test(&btf->refcnt)) { in btf_put()
1761 call_rcu(&btf->rcu, btf_free_rcu); in btf_put()
1767 return btf->base_btf; in btf_base_btf()
1772 return &btf->hdr; in btf_header()
1777 btf->base_btf = (struct btf *)base_btf; in btf_set_base_btf()
1778 btf->start_id = btf_nr_types(base_btf); in btf_set_base_btf()
1779 btf->start_str_off = base_btf->hdr.str_len; in btf_set_base_btf()
1784 struct btf *btf = env->btf; in env_resolve_init()
1785 u32 nr_types = btf->nr_types; in env_resolve_init()
1805 btf->resolved_sizes = resolved_sizes; in env_resolve_init()
1806 btf->resolved_ids = resolved_ids; in env_resolve_init()
1807 env->visit_states = visit_states; in env_resolve_init()
1815 return -ENOMEM; in env_resolve_init()
1820 kvfree(env->visit_states); in btf_verifier_env_free()
1827 switch (env->resolve_mode) { in env_type_is_resolve_sink()
1829 /* int, enum or void is a sink */ in env_type_is_resolve_sink()
1832 /* int, enum, void, struct, array, func or func_proto is a sink in env_type_is_resolve_sink()
1838 /* int, enum, void, ptr, func or func_proto is a sink in env_type_is_resolve_sink()
1853 if (type_id < env->btf->start_id) in env_type_is_resolved()
1856 return env->visit_states[type_id - env->btf->start_id] == RESOLVED; in env_type_is_resolved()
1862 const struct btf *btf = env->btf; in env_stack_push()
1865 if (env->top_stack == MAX_RESOLVE_DEPTH) in env_stack_push()
1866 return -E2BIG; in env_stack_push()
1868 if (type_id < btf->start_id in env_stack_push()
1869 || env->visit_states[type_id - btf->start_id] != NOT_VISITED) in env_stack_push()
1870 return -EEXIST; in env_stack_push()
1872 env->visit_states[type_id - btf->start_id] = VISITED; in env_stack_push()
1874 v = &env->stack[env->top_stack++]; in env_stack_push()
1875 v->t = t; in env_stack_push()
1876 v->type_id = type_id; in env_stack_push()
1877 v->next_member = 0; in env_stack_push()
1879 if (env->resolve_mode == RESOLVE_TBD) { in env_stack_push()
1881 env->resolve_mode = RESOLVE_PTR; in env_stack_push()
1883 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY; in env_stack_push()
1892 env->stack[env->top_stack - 1].next_member = next_member; in env_stack_set_next_member()
1899 u32 type_id = env->stack[--(env->top_stack)].type_id; in env_stack_pop_resolved()
1900 struct btf *btf = env->btf; in env_stack_pop_resolved()
1902 type_id -= btf->start_id; /* adjust to local type id */ in env_stack_pop_resolved()
1903 btf->resolved_sizes[type_id] = resolved_size; in env_stack_pop_resolved()
1904 btf->resolved_ids[type_id] = resolved_type_id; in env_stack_pop_resolved()
1905 env->visit_states[type_id] = RESOLVED; in env_stack_pop_resolved()
1910 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL; in env_stack_peak()
1913 /* Resolve the size of a passed-in "type"
1943 switch (BTF_INFO_KIND(type->info)) { in __btf_resolve_size()
1944 /* type->size can be used */ in __btf_resolve_size()
1951 size = type->size; in __btf_resolve_size()
1964 id = type->type; in __btf_resolve_size()
1965 type = btf_type_by_id(btf, type->type); in __btf_resolve_size()
1972 if (nelems && array->nelems > U32_MAX / nelems) in __btf_resolve_size()
1973 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1974 nelems *= array->nelems; in __btf_resolve_size()
1975 type = btf_type_by_id(btf, array->type); in __btf_resolve_size()
1980 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1984 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1988 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1996 *elem_id = array ? array->type : 0; in __btf_resolve_size()
2012 while (type_id < btf->start_id) in btf_resolved_type_id()
2013 btf = btf->base_btf; in btf_resolved_type_id()
2015 return btf->resolved_ids[type_id - btf->start_id]; in btf_resolved_type_id()
2028 while (type_id < btf->start_id) in btf_resolved_type_size()
2029 btf = btf->base_btf; in btf_resolved_type_size()
2031 return btf->resolved_sizes[type_id - btf->start_id]; in btf_resolved_type_size()
2046 size = size_type->size; in btf_type_id_size()
2061 size = size_type->size; in btf_type_id_size()
2084 return -EINVAL; in btf_df_check_member()
2094 return -EINVAL; in btf_df_check_kflag_member()
2098 * int, enum and modifier types have their specific callback functions.
2105 if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) { in btf_generic_check_kflag_member()
2108 return -EINVAL; in btf_generic_check_kflag_member()
2111 /* bitfield size is 0, so member->offset represents bit offset only. in btf_generic_check_kflag_member()
2114 return btf_type_ops(member_type)->check_member(env, struct_type, in btf_generic_check_kflag_member()
2122 btf_verifier_log_basic(env, v->t, "Unsupported resolve"); in btf_df_resolve()
2123 return -EINVAL; in btf_df_resolve()
2130 btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info)); in btf_df_show()
2139 u32 struct_bits_off = member->offset; in btf_int_check_member()
2140 u32 struct_size = struct_type->size; in btf_int_check_member()
2144 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) { in btf_int_check_member()
2147 return -EINVAL; in btf_int_check_member()
2158 return -EINVAL; in btf_int_check_member()
2162 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { in btf_int_check_member()
2165 return -EINVAL; in btf_int_check_member()
2178 u32 struct_size = struct_type->size; in btf_int_check_kflag_member()
2185 return -EINVAL; in btf_int_check_kflag_member()
2189 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); in btf_int_check_kflag_member()
2190 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); in btf_int_check_kflag_member()
2199 return -EINVAL; in btf_int_check_kflag_member()
2206 return -EINVAL; in btf_int_check_kflag_member()
2214 return -EINVAL; in btf_int_check_kflag_member()
2218 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { in btf_int_check_kflag_member()
2221 return -EINVAL; in btf_int_check_kflag_member()
2238 return -EINVAL; in btf_int_check_meta()
2243 return -EINVAL; in btf_int_check_meta()
2248 return -EINVAL; in btf_int_check_meta()
2255 return -EINVAL; in btf_int_check_meta()
2263 return -EINVAL; in btf_int_check_meta()
2266 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) { in btf_int_check_meta()
2268 return -EINVAL; in btf_int_check_meta()
2283 return -ENOTSUPP; in btf_int_check_meta()
2298 t->size, BTF_INT_OFFSET(int_data), in btf_int_log()
2341 /* shake out un-needed bits by shift/or operations */ in btf_int128_shift()
2343 upper_num = lower_num << (left_shift_bits - 64); in btf_int128_shift()
2347 (lower_num >> (64 - left_shift_bits)); in btf_int128_shift()
2352 lower_num = upper_num >> (right_shift_bits - 64); in btf_int128_shift()
2356 (upper_num << (64 - right_shift_bits)); in btf_int128_shift()
2385 left_shift_bits = BITS_PER_U128 - nr_copy_bits; in btf_bitfield_show()
2387 right_shift_bits = BITS_PER_U128 - nr_bits; in btf_bitfield_show()
2456 if (show->state.array_encoding == BTF_INT_CHAR) { in btf_int_show()
2458 if (show->state.array_terminated) in btf_int_show()
2461 show->state.array_terminated = 1; in btf_int_show()
2498 u32 resolved_type_id = member->type; in btf_modifier_check_member()
2500 struct btf *btf = env->btf; in btf_modifier_check_member()
2506 return -EINVAL; in btf_modifier_check_member()
2512 return btf_type_ops(resolved_type)->check_member(env, struct_type, in btf_modifier_check_member()
2523 u32 resolved_type_id = member->type; in btf_modifier_check_kflag_member()
2525 struct btf *btf = env->btf; in btf_modifier_check_kflag_member()
2531 return -EINVAL; in btf_modifier_check_kflag_member()
2537 return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type, in btf_modifier_check_kflag_member()
2549 struct_size = struct_type->size; in btf_ptr_check_member()
2550 struct_bits_off = member->offset; in btf_ptr_check_member()
2556 return -EINVAL; in btf_ptr_check_member()
2559 if (struct_size - bytes_offset < sizeof(void *)) { in btf_ptr_check_member()
2562 return -EINVAL; in btf_ptr_check_member()
2576 return -EINVAL; in btf_ref_type_check_meta()
2581 return -EINVAL; in btf_ref_type_check_meta()
2584 if (!BTF_TYPE_ID_VALID(t->type)) { in btf_ref_type_check_meta()
2586 return -EINVAL; in btf_ref_type_check_meta()
2589 /* typedef/type_tag type must have a valid name, and other ref types, in btf_ref_type_check_meta()
2590 * volatile, const, restrict, should have a null name. in btf_ref_type_check_meta()
2592 if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) { in btf_ref_type_check_meta()
2593 if (!t->name_off || in btf_ref_type_check_meta()
2594 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_ref_type_check_meta()
2595 btf_verifier_log_type(env, t, "Invalid name"); in btf_ref_type_check_meta()
2596 return -EINVAL; in btf_ref_type_check_meta()
2598 } else if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPE_TAG) { in btf_ref_type_check_meta()
2599 value = btf_name_by_offset(env->btf, t->name_off); in btf_ref_type_check_meta()
2601 btf_verifier_log_type(env, t, "Invalid name"); in btf_ref_type_check_meta()
2602 return -EINVAL; in btf_ref_type_check_meta()
2605 if (t->name_off) { in btf_ref_type_check_meta()
2606 btf_verifier_log_type(env, t, "Invalid name"); in btf_ref_type_check_meta()
2607 return -EINVAL; in btf_ref_type_check_meta()
2619 const struct btf_type *t = v->t; in btf_modifier_resolve()
2621 u32 next_type_id = t->type; in btf_modifier_resolve()
2622 struct btf *btf = env->btf; in btf_modifier_resolve()
2626 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_modifier_resolve()
2627 return -EINVAL; in btf_modifier_resolve()
2637 * save us a few type-following when we use it later (e.g. in in btf_modifier_resolve()
2648 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_modifier_resolve()
2649 return -EINVAL; in btf_modifier_resolve()
2662 const struct btf_type *t = v->t; in btf_var_resolve()
2663 u32 next_type_id = t->type; in btf_var_resolve()
2664 struct btf *btf = env->btf; in btf_var_resolve()
2668 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_var_resolve()
2669 return -EINVAL; in btf_var_resolve()
2695 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_var_resolve()
2696 return -EINVAL; in btf_var_resolve()
2708 const struct btf_type *t = v->t; in btf_ptr_resolve()
2709 u32 next_type_id = t->type; in btf_ptr_resolve()
2710 struct btf *btf = env->btf; in btf_ptr_resolve()
2714 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_ptr_resolve()
2715 return -EINVAL; in btf_ptr_resolve()
2724 * to a ptr (last-resolved-ptr). in btf_ptr_resolve()
2726 * We now need to continue from the last-resolved-ptr to in btf_ptr_resolve()
2727 * ensure the last-resolved-ptr will not referring back to in btf_ptr_resolve()
2751 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_ptr_resolve()
2752 return -EINVAL; in btf_ptr_resolve()
2766 if (btf->resolved_ids) in btf_modifier_show()
2771 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_modifier_show()
2780 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_var_show()
2794 if (show->flags & BTF_SHOW_PTR_RAW) in btf_ptr_show()
2804 btf_verifier_log(env, "type_id=%u", t->type); in btf_ref_type_log()
2831 return -EINVAL; in btf_fwd_check_meta()
2834 if (t->type) { in btf_fwd_check_meta()
2836 return -EINVAL; in btf_fwd_check_meta()
2839 /* fwd type must have a valid name */ in btf_fwd_check_meta()
2840 if (!t->name_off || in btf_fwd_check_meta()
2841 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_fwd_check_meta()
2842 btf_verifier_log_type(env, t, "Invalid name"); in btf_fwd_check_meta()
2843 return -EINVAL; in btf_fwd_check_meta()
2871 u32 struct_bits_off = member->offset; in btf_array_check_member()
2874 struct btf *btf = env->btf; in btf_array_check_member()
2879 return -EINVAL; in btf_array_check_member()
2882 array_type_id = member->type; in btf_array_check_member()
2884 struct_size = struct_type->size; in btf_array_check_member()
2886 if (struct_size - bytes_offset < array_size) { in btf_array_check_member()
2889 return -EINVAL; in btf_array_check_member()
2906 return -EINVAL; in btf_array_check_meta()
2909 /* array type should not have a name */ in btf_array_check_meta()
2910 if (t->name_off) { in btf_array_check_meta()
2911 btf_verifier_log_type(env, t, "Invalid name"); in btf_array_check_meta()
2912 return -EINVAL; in btf_array_check_meta()
2917 return -EINVAL; in btf_array_check_meta()
2922 return -EINVAL; in btf_array_check_meta()
2925 if (t->size) { in btf_array_check_meta()
2927 return -EINVAL; in btf_array_check_meta()
2931 * so !array->type and !array->index_type are not allowed. in btf_array_check_meta()
2933 if (!array->type || !BTF_TYPE_ID_VALID(array->type)) { in btf_array_check_meta()
2935 return -EINVAL; in btf_array_check_meta()
2938 if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) { in btf_array_check_meta()
2940 return -EINVAL; in btf_array_check_meta()
2951 const struct btf_array *array = btf_type_array(v->t); in btf_array_resolve()
2954 struct btf *btf = env->btf; in btf_array_resolve()
2957 /* Check array->index_type */ in btf_array_resolve()
2958 index_type_id = array->index_type; in btf_array_resolve()
2962 btf_verifier_log_type(env, v->t, "Invalid index"); in btf_array_resolve()
2963 return -EINVAL; in btf_array_resolve()
2973 btf_verifier_log_type(env, v->t, "Invalid index"); in btf_array_resolve()
2974 return -EINVAL; in btf_array_resolve()
2977 /* Check array->type */ in btf_array_resolve()
2978 elem_type_id = array->type; in btf_array_resolve()
2982 btf_verifier_log_type(env, v->t, in btf_array_resolve()
2984 return -EINVAL; in btf_array_resolve()
2993 btf_verifier_log_type(env, v->t, "Invalid elem"); in btf_array_resolve()
2994 return -EINVAL; in btf_array_resolve()
2998 btf_verifier_log_type(env, v->t, "Invalid array of int"); in btf_array_resolve()
2999 return -EINVAL; in btf_array_resolve()
3002 if (array->nelems && elem_size > U32_MAX / array->nelems) { in btf_array_resolve()
3003 btf_verifier_log_type(env, v->t, in btf_array_resolve()
3005 return -EINVAL; in btf_array_resolve()
3008 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems); in btf_array_resolve()
3019 array->type, array->index_type, array->nelems); in btf_array_log()
3032 elem_type_id = array->type; in __btf_array_show()
3035 elem_size = elem_type->size; in __btf_array_show()
3058 for (i = 0; i < array->nelems; i++) { in __btf_array_show()
3062 elem_ops->show(btf, elem_type, elem_type_id, data, in __btf_array_show()
3068 if (show->state.array_terminated) in __btf_array_show()
3079 const struct btf_member *m = show->state.member; in btf_array_show()
3082 * First check if any members would be shown (are non-zero). in btf_array_show()
3084 * details on how this works at a high-level. in btf_array_show()
3086 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { in btf_array_show()
3087 if (!show->state.depth_check) { in btf_array_show()
3088 show->state.depth_check = show->state.depth + 1; in btf_array_show()
3089 show->state.depth_to_show = 0; in btf_array_show()
3092 show->state.member = m; in btf_array_show()
3094 if (show->state.depth_check != show->state.depth + 1) in btf_array_show()
3096 show->state.depth_check = 0; in btf_array_show()
3098 if (show->state.depth_to_show <= show->state.depth) in btf_array_show()
3102 * non-zero array member(s). in btf_array_show()
3122 u32 struct_bits_off = member->offset; in btf_struct_check_member()
3128 return -EINVAL; in btf_struct_check_member()
3131 struct_size = struct_type->size; in btf_struct_check_member()
3133 if (struct_size - bytes_offset < member_type->size) { in btf_struct_check_member()
3136 return -EINVAL; in btf_struct_check_member()
3146 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION; in btf_struct_check_meta()
3149 struct btf *btf = env->btf; in btf_struct_check_meta()
3150 u32 struct_size = t->size; in btf_struct_check_meta()
3159 return -EINVAL; in btf_struct_check_meta()
3162 /* struct type either no name or a valid one */ in btf_struct_check_meta()
3163 if (t->name_off && in btf_struct_check_meta()
3164 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_struct_check_meta()
3165 btf_verifier_log_type(env, t, "Invalid name"); in btf_struct_check_meta()
3166 return -EINVAL; in btf_struct_check_meta()
3173 if (!btf_name_offset_valid(btf, member->name_off)) { in btf_struct_check_meta()
3176 member->name_off); in btf_struct_check_meta()
3177 return -EINVAL; in btf_struct_check_meta()
3180 /* struct member either no name or a valid one */ in btf_struct_check_meta()
3181 if (member->name_off && in btf_struct_check_meta()
3182 !btf_name_valid_identifier(btf, member->name_off)) { in btf_struct_check_meta()
3183 btf_verifier_log_member(env, t, member, "Invalid name"); in btf_struct_check_meta()
3184 return -EINVAL; in btf_struct_check_meta()
3187 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) { in btf_struct_check_meta()
3190 return -EINVAL; in btf_struct_check_meta()
3197 return -EINVAL; in btf_struct_check_meta()
3207 return -EINVAL; in btf_struct_check_meta()
3213 return -EINVAL; in btf_struct_check_meta()
3234 if (v->next_member) { in btf_struct_resolve()
3239 last_member = btf_type_member(v->t) + v->next_member - 1; in btf_struct_resolve()
3240 last_member_type_id = last_member->type; in btf_struct_resolve()
3243 return -EINVAL; in btf_struct_resolve()
3245 last_member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3247 if (btf_type_kflag(v->t)) in btf_struct_resolve()
3248 err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t, in btf_struct_resolve()
3252 err = btf_type_ops(last_member_type)->check_member(env, v->t, in btf_struct_resolve()
3259 for_each_member_from(i, v->next_member, v->t, member) { in btf_struct_resolve()
3260 u32 member_type_id = member->type; in btf_struct_resolve()
3261 const struct btf_type *member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3266 btf_verifier_log_member(env, v->t, member, in btf_struct_resolve()
3268 return -EINVAL; in btf_struct_resolve()
3277 if (btf_type_kflag(v->t)) in btf_struct_resolve()
3278 err = btf_type_ops(member_type)->check_kflag_member(env, v->t, in btf_struct_resolve()
3282 err = btf_type_ops(member_type)->check_member(env, v->t, in btf_struct_resolve()
3297 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); in btf_struct_log()
3300 enum {
3306 enum btf_field_type type;
3320 u32 off, int sz, enum btf_field_type field_type, in btf_find_struct()
3325 if (t->size != sz) in btf_find_struct()
3327 info->type = field_type; in btf_find_struct()
3328 info->off = off; in btf_find_struct()
3335 enum btf_field_type type; in btf_find_kptr()
3342 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3346 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3351 if (btf_type_is_type_tag(btf_type_by_id(btf, t->type))) in btf_find_kptr()
3352 return -EINVAL; in btf_find_kptr()
3353 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_find_kptr()
3363 return -EINVAL; in btf_find_kptr()
3369 t = btf_type_skip_modifiers(btf, t->type, &res_id); in btf_find_kptr()
3372 return -EINVAL; in btf_find_kptr()
3374 info->type = type; in btf_find_kptr()
3375 info->off = off; in btf_find_kptr()
3376 info->kptr.type_id = res_id; in btf_find_kptr()
3391 if (pt != btf_type_by_id(btf, t->type)) in btf_find_next_decl_tag()
3393 if (btf_type_decl_tag(t)->component_idx != comp_idx) in btf_find_next_decl_tag()
3395 if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) in btf_find_next_decl_tag()
3399 return -ENOENT; in btf_find_next_decl_tag()
3415 value = __btf_name_by_offset(btf, t->name_off) + len; in btf_find_decl_tag_value()
3420 return ERR_PTR(-EEXIST); in btf_find_decl_tag_value()
3429 enum btf_field_type head_type) in btf_find_graph_root()
3437 if (t->size != sz) in btf_find_graph_root()
3441 return -EINVAL; in btf_find_graph_root()
3444 return -EINVAL; in btf_find_graph_root()
3445 value_type = kstrndup(value_type, node_field_name - value_type, GFP_KERNEL | __GFP_NOWARN); in btf_find_graph_root()
3447 return -ENOMEM; in btf_find_graph_root()
3454 return -EINVAL; in btf_find_graph_root()
3455 info->type = head_type; in btf_find_graph_root()
3456 info->off = off; in btf_find_graph_root()
3457 info->graph_root.value_btf_id = id; in btf_find_graph_root()
3458 info->graph_root.node_name = node_field_name; in btf_find_graph_root()
3463 if (field_mask & field_type && !strcmp(name, field_type_str)) { \
3473 const char *name = __btf_name_by_offset(btf, var_type->name_off); in btf_get_field_type() local
3476 if (!strcmp(name, "bpf_spin_lock")) { in btf_get_field_type()
3478 return -E2BIG; in btf_get_field_type()
3485 if (!strcmp(name, "bpf_res_spin_lock")) { in btf_get_field_type()
3487 return -E2BIG; in btf_get_field_type()
3494 if (!strcmp(name, "bpf_timer")) { in btf_get_field_type()
3496 return -E2BIG; in btf_get_field_type()
3503 if (!strcmp(name, "bpf_wq")) { in btf_get_field_type()
3505 return -E2BIG; in btf_get_field_type()
3556 return -EINVAL; in btf_repeat_fields()
3564 return -E2BIG; in btf_repeat_fields()
3596 return -E2BIG; in btf_find_nested_struct()
3610 err = btf_repeat_fields(info, info_cnt, ret, nelems - 1, t->size); in btf_find_nested_struct()
3639 nelems *= array->nelems; in btf_find_field_one()
3640 var_type = btf_type_by_id(btf, array->type); in btf_find_field_one()
3643 return -E2BIG; in btf_find_field_one()
3651 sz = var_type->size; in btf_find_field_one()
3701 return -EFAULT; in btf_find_field_one()
3707 return -E2BIG; in btf_find_field_one()
3709 ret = btf_repeat_fields(info, info_cnt, 1, nelems - 1, sz); in btf_find_field_one()
3727 member->type); in btf_find_struct_field()
3732 return -EINVAL; in btf_find_struct_field()
3738 &info[idx], info_cnt - idx, level); in btf_find_struct_field()
3755 const struct btf_type *var = btf_type_by_id(btf, vsi->type); in btf_find_datasec_var()
3756 const struct btf_type *var_type = btf_type_by_id(btf, var->type); in btf_find_datasec_var()
3758 off = vsi->offset; in btf_find_datasec_var()
3759 ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, in btf_find_datasec_var()
3761 &info[idx], info_cnt - idx, in btf_find_datasec_var()
3778 return -EINVAL; in btf_find_field()
3795 * in vmlinux or module BTFs, by name and kind. in btf_parse_kptr()
3797 t = btf_type_by_id(btf, info->kptr.type_id); in btf_parse_kptr()
3798 id = bpf_find_btf_id(__btf_name_by_offset(btf, t->name_off), BTF_INFO_KIND(t->info), in btf_parse_kptr()
3800 if (id == -ENOENT) { in btf_parse_kptr()
3807 field->kptr.dtor = NULL; in btf_parse_kptr()
3808 id = info->kptr.type_id; in btf_parse_kptr()
3818 if (info->type == BPF_KPTR_REF) { in btf_parse_kptr()
3836 ret = -ENOENT; in btf_parse_kptr()
3843 ret = -ENXIO; in btf_parse_kptr()
3851 dtor_func_name = __btf_name_by_offset(kptr_btf, dtor_func->name_off); in btf_parse_kptr()
3854 ret = -EINVAL; in btf_parse_kptr()
3857 field->kptr.dtor = (void *)addr; in btf_parse_kptr()
3861 field->kptr.btf_id = id; in btf_parse_kptr()
3862 field->kptr.btf = kptr_btf; in btf_parse_kptr()
3863 field->kptr.module = mod; in btf_parse_kptr()
3883 t = btf_type_by_id(btf, info->graph_root.value_btf_id); in btf_parse_graph_root()
3889 if (strcmp(info->graph_root.node_name, in btf_parse_graph_root()
3890 __btf_name_by_offset(btf, member->name_off))) in btf_parse_graph_root()
3892 /* Invalid BTF, two members with same name */ in btf_parse_graph_root()
3894 return -EINVAL; in btf_parse_graph_root()
3895 n = btf_type_by_id(btf, member->type); in btf_parse_graph_root()
3897 return -EINVAL; in btf_parse_graph_root()
3898 if (strcmp(node_type_name, __btf_name_by_offset(btf, n->name_off))) in btf_parse_graph_root()
3899 return -EINVAL; in btf_parse_graph_root()
3902 return -EINVAL; in btf_parse_graph_root()
3905 return -EINVAL; in btf_parse_graph_root()
3907 field->graph_root.btf = (struct btf *)btf; in btf_parse_graph_root()
3908 field->graph_root.value_btf_id = info->graph_root.value_btf_id; in btf_parse_graph_root()
3909 field->graph_root.node_offset = offset; in btf_parse_graph_root()
3912 return -ENOENT; in btf_parse_graph_root()
3935 if (a->offset < b->offset) in btf_field_cmp()
3936 return -1; in btf_field_cmp()
3937 else if (a->offset > b->offset) in btf_field_cmp()
3948 int ret, i, cnt; in btf_parse_fields() local
3956 cnt = ret; in btf_parse_fields()
3960 rec = kzalloc(offsetof(struct btf_record, fields[cnt]), GFP_KERNEL | __GFP_NOWARN); in btf_parse_fields()
3962 return ERR_PTR(-ENOMEM); in btf_parse_fields()
3964 rec->spin_lock_off = -EINVAL; in btf_parse_fields()
3965 rec->res_spin_lock_off = -EINVAL; in btf_parse_fields()
3966 rec->timer_off = -EINVAL; in btf_parse_fields()
3967 rec->wq_off = -EINVAL; in btf_parse_fields()
3968 rec->refcount_off = -EINVAL; in btf_parse_fields()
3969 for (i = 0; i < cnt; i++) { in btf_parse_fields()
3973 ret = -EFAULT; in btf_parse_fields()
3977 ret = -EEXIST; in btf_parse_fields()
3982 rec->field_mask |= info_arr[i].type; in btf_parse_fields()
3983 rec->fields[i].offset = info_arr[i].off; in btf_parse_fields()
3984 rec->fields[i].type = info_arr[i].type; in btf_parse_fields()
3985 rec->fields[i].size = field_type_size; in btf_parse_fields()
3989 WARN_ON_ONCE(rec->spin_lock_off >= 0); in btf_parse_fields()
3991 rec->spin_lock_off = rec->fields[i].offset; in btf_parse_fields()
3994 WARN_ON_ONCE(rec->spin_lock_off >= 0); in btf_parse_fields()
3996 rec->res_spin_lock_off = rec->fields[i].offset; in btf_parse_fields()
3999 WARN_ON_ONCE(rec->timer_off >= 0); in btf_parse_fields()
4001 rec->timer_off = rec->fields[i].offset; in btf_parse_fields()
4004 WARN_ON_ONCE(rec->wq_off >= 0); in btf_parse_fields()
4006 rec->wq_off = rec->fields[i].offset; in btf_parse_fields()
4009 WARN_ON_ONCE(rec->refcount_off >= 0); in btf_parse_fields()
4011 rec->refcount_off = rec->fields[i].offset; in btf_parse_fields()
4017 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4022 ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4027 ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4035 ret = -EFAULT; in btf_parse_fields()
4038 rec->cnt++; in btf_parse_fields()
4041 if (rec->spin_lock_off >= 0 && rec->res_spin_lock_off >= 0) { in btf_parse_fields()
4042 ret = -EINVAL; in btf_parse_fields()
4049 (rec->spin_lock_off < 0 && rec->res_spin_lock_off < 0)) { in btf_parse_fields()
4050 ret = -EINVAL; in btf_parse_fields()
4054 if (rec->refcount_off < 0 && in btf_parse_fields()
4057 ret = -EINVAL; in btf_parse_fields()
4061 sort_r(rec->fields, rec->cnt, sizeof(struct btf_field), btf_field_cmp, in btf_parse_fields()
4082 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR))) in btf_check_and_fixup_fields()
4084 for (i = 0; i < rec->cnt; i++) { in btf_check_and_fixup_fields()
4089 if (rec->fields[i].type == BPF_UPTR) { in btf_check_and_fixup_fields()
4093 if (btf_is_kernel(rec->fields[i].kptr.btf)) in btf_check_and_fixup_fields()
4094 return -EINVAL; in btf_check_and_fixup_fields()
4095 t = btf_type_by_id(rec->fields[i].kptr.btf, in btf_check_and_fixup_fields()
4096 rec->fields[i].kptr.btf_id); in btf_check_and_fixup_fields()
4097 if (!t->size) in btf_check_and_fixup_fields()
4098 return -EINVAL; in btf_check_and_fixup_fields()
4099 if (t->size > PAGE_SIZE) in btf_check_and_fixup_fields()
4100 return -E2BIG; in btf_check_and_fixup_fields()
4104 if (!(rec->fields[i].type & BPF_GRAPH_ROOT)) in btf_check_and_fixup_fields()
4106 btf_id = rec->fields[i].graph_root.value_btf_id; in btf_check_and_fixup_fields()
4109 return -EFAULT; in btf_check_and_fixup_fields()
4110 rec->fields[i].graph_root.value_rec = meta->record; in btf_check_and_fixup_fields()
4116 if (!(rec->field_mask & BPF_GRAPH_NODE)) in btf_check_and_fixup_fields()
4125 * - A type can only be owned by another type in user BTF if it in btf_check_and_fixup_fields()
4127 * - A type can only _own_ another type in user BTF if it has a in btf_check_and_fixup_fields()
4137 * A -> B -> C in btf_check_and_fixup_fields()
4139 * - A is an root, e.g. has bpf_rb_root. in btf_check_and_fixup_fields()
4140 * - B is both a root and node, e.g. has bpf_rb_node and in btf_check_and_fixup_fields()
4142 * - C is only an root, e.g. has bpf_list_node in btf_check_and_fixup_fields()
4147 * A -> B in btf_check_and_fixup_fields()
4149 * - A is both an root and node. in btf_check_and_fixup_fields()
4150 * - B is only an node. in btf_check_and_fixup_fields()
4152 if (meta->record->field_mask & BPF_GRAPH_ROOT) in btf_check_and_fixup_fields()
4153 return -ELOOP; in btf_check_and_fixup_fields()
4172 member->type); in __btf_struct_show()
4186 member->type, in __btf_struct_show()
4195 ops->show(btf, member_type, member->type, in __btf_struct_show()
4209 const struct btf_member *m = show->state.member; in btf_struct_show()
4212 * First check if any members would be shown (are non-zero). in btf_struct_show()
4214 * details on how this works at a high-level. in btf_struct_show()
4216 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { in btf_struct_show()
4217 if (!show->state.depth_check) { in btf_struct_show()
4218 show->state.depth_check = show->state.depth + 1; in btf_struct_show()
4219 show->state.depth_to_show = 0; in btf_struct_show()
4223 show->state.member = m; in btf_struct_show()
4224 if (show->state.depth_check != show->state.depth + 1) in btf_struct_show()
4226 show->state.depth_check = 0; in btf_struct_show()
4228 if (show->state.depth_to_show <= show->state.depth) in btf_struct_show()
4232 * non-zero child values. in btf_struct_show()
4253 u32 struct_bits_off = member->offset; in btf_enum_check_member()
4259 return -EINVAL; in btf_enum_check_member()
4262 struct_size = struct_type->size; in btf_enum_check_member()
4264 if (struct_size - bytes_offset < member_type->size) { in btf_enum_check_member()
4267 return -EINVAL; in btf_enum_check_member()
4281 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); in btf_enum_check_kflag_member()
4282 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); in btf_enum_check_kflag_member()
4287 return -EINVAL; in btf_enum_check_kflag_member()
4294 return -EINVAL; in btf_enum_check_kflag_member()
4297 struct_size = struct_type->size; in btf_enum_check_kflag_member()
4302 return -EINVAL; in btf_enum_check_kflag_member()
4313 struct btf *btf = env->btf; in btf_enum_check_meta()
4325 return -EINVAL; in btf_enum_check_meta()
4328 if (t->size > 8 || !is_power_of_2(t->size)) { in btf_enum_check_meta()
4330 return -EINVAL; in btf_enum_check_meta()
4333 /* enum type either no name or a valid one */ in btf_enum_check_meta()
4334 if (t->name_off && in btf_enum_check_meta()
4335 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum_check_meta()
4336 btf_verifier_log_type(env, t, "Invalid name"); in btf_enum_check_meta()
4337 return -EINVAL; in btf_enum_check_meta()
4346 return -EINVAL; in btf_enum_check_meta()
4349 /* enum member must have a valid name */ in btf_enum_check_meta()
4352 btf_verifier_log_type(env, t, "Invalid name"); in btf_enum_check_meta()
4353 return -EINVAL; in btf_enum_check_meta()
4356 if (env->log.level == BPF_LOG_KERNEL) in btf_enum_check_meta()
4370 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); in btf_enum_log()
4421 struct btf *btf = env->btf; in btf_enum64_check_meta()
4433 return -EINVAL; in btf_enum64_check_meta()
4436 if (t->size > 8 || !is_power_of_2(t->size)) { in btf_enum64_check_meta()
4438 return -EINVAL; in btf_enum64_check_meta()
4441 /* enum type either no name or a valid one */ in btf_enum64_check_meta()
4442 if (t->name_off && in btf_enum64_check_meta()
4443 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum64_check_meta()
4444 btf_verifier_log_type(env, t, "Invalid name"); in btf_enum64_check_meta()
4445 return -EINVAL; in btf_enum64_check_meta()
4454 return -EINVAL; in btf_enum64_check_meta()
4457 /* enum member must have a valid name */ in btf_enum64_check_meta()
4460 btf_verifier_log_type(env, t, "Invalid name"); in btf_enum64_check_meta()
4461 return -EINVAL; in btf_enum64_check_meta()
4464 if (env->log.level == BPF_LOG_KERNEL) in btf_enum64_check_meta()
4529 return -EINVAL; in btf_func_proto_check_meta()
4532 if (t->name_off) { in btf_func_proto_check_meta()
4533 btf_verifier_log_type(env, t, "Invalid name"); in btf_func_proto_check_meta()
4534 return -EINVAL; in btf_func_proto_check_meta()
4539 return -EINVAL; in btf_func_proto_check_meta()
4553 btf_verifier_log(env, "return=%u args=(", t->type); in btf_func_proto_log()
4566 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4568 for (i = 1; i < nr_args - 1; i++) in btf_func_proto_log()
4570 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4574 const struct btf_param *last_arg = &args[nr_args - 1]; in btf_func_proto_log()
4576 if (last_arg->type) in btf_func_proto_log()
4577 btf_verifier_log(env, ", %u %s", last_arg->type, in btf_func_proto_log()
4578 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4579 last_arg->name_off)); in btf_func_proto_log()
4596 * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO)
4610 if (!t->name_off || in btf_func_check_meta()
4611 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_func_check_meta()
4612 btf_verifier_log_type(env, t, "Invalid name"); in btf_func_check_meta()
4613 return -EINVAL; in btf_func_check_meta()
4618 return -EINVAL; in btf_func_check_meta()
4623 return -EINVAL; in btf_func_check_meta()
4634 const struct btf_type *t = v->t; in btf_func_resolve()
4635 u32 next_type_id = t->type; in btf_func_resolve()
4666 return -EINVAL; in btf_var_check_meta()
4671 return -EINVAL; in btf_var_check_meta()
4676 return -EINVAL; in btf_var_check_meta()
4679 if (!t->name_off || in btf_var_check_meta()
4680 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_var_check_meta()
4681 btf_verifier_log_type(env, t, "Invalid name"); in btf_var_check_meta()
4682 return -EINVAL; in btf_var_check_meta()
4686 if (!t->type || !BTF_TYPE_ID_VALID(t->type)) { in btf_var_check_meta()
4688 return -EINVAL; in btf_var_check_meta()
4692 if (var->linkage != BTF_VAR_STATIC && in btf_var_check_meta()
4693 var->linkage != BTF_VAR_GLOBAL_ALLOCATED) { in btf_var_check_meta()
4695 return -EINVAL; in btf_var_check_meta()
4707 btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage); in btf_var_log()
4732 return -EINVAL; in btf_datasec_check_meta()
4735 if (!t->size) { in btf_datasec_check_meta()
4737 return -EINVAL; in btf_datasec_check_meta()
4742 return -EINVAL; in btf_datasec_check_meta()
4745 if (!t->name_off || in btf_datasec_check_meta()
4746 !btf_name_valid_section(env->btf, t->name_off)) { in btf_datasec_check_meta()
4747 btf_verifier_log_type(env, t, "Invalid name"); in btf_datasec_check_meta()
4748 return -EINVAL; in btf_datasec_check_meta()
4755 if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) { in btf_datasec_check_meta()
4758 return -EINVAL; in btf_datasec_check_meta()
4761 if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) { in btf_datasec_check_meta()
4764 return -EINVAL; in btf_datasec_check_meta()
4767 if (!vsi->size || vsi->size > t->size) { in btf_datasec_check_meta()
4770 return -EINVAL; in btf_datasec_check_meta()
4773 last_vsi_end_off = vsi->offset + vsi->size; in btf_datasec_check_meta()
4774 if (last_vsi_end_off > t->size) { in btf_datasec_check_meta()
4777 return -EINVAL; in btf_datasec_check_meta()
4781 sum += vsi->size; in btf_datasec_check_meta()
4784 if (t->size < sum) { in btf_datasec_check_meta()
4786 return -EINVAL; in btf_datasec_check_meta()
4796 struct btf *btf = env->btf; in btf_datasec_resolve()
4799 env->resolve_mode = RESOLVE_TBD; in btf_datasec_resolve()
4800 for_each_vsi_from(i, v->next_member, v->t, vsi) { in btf_datasec_resolve()
4801 u32 var_type_id = vsi->type, type_id, type_size = 0; in btf_datasec_resolve()
4802 const struct btf_type *var_type = btf_type_by_id(env->btf, in btf_datasec_resolve()
4805 btf_verifier_log_vsi(env, v->t, vsi, in btf_datasec_resolve()
4807 return -EINVAL; in btf_datasec_resolve()
4816 type_id = var_type->type; in btf_datasec_resolve()
4818 btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); in btf_datasec_resolve()
4819 return -EINVAL; in btf_datasec_resolve()
4822 if (vsi->size < type_size) { in btf_datasec_resolve()
4823 btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); in btf_datasec_resolve()
4824 return -EINVAL; in btf_datasec_resolve()
4835 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); in btf_datasec_log()
4851 __btf_name_by_offset(btf, t->name_off)); in btf_datasec_show()
4853 var = btf_type_by_id(btf, vsi->type); in btf_datasec_show()
4856 btf_type_ops(var)->show(btf, var, vsi->type, in btf_datasec_show()
4857 data + vsi->offset, bits_offset, show); in btf_datasec_show()
4877 return -EINVAL; in btf_float_check_meta()
4882 return -EINVAL; in btf_float_check_meta()
4885 if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 && in btf_float_check_meta()
4886 t->size != 16) { in btf_float_check_meta()
4888 return -EINVAL; in btf_float_check_meta()
4909 * that types after CO-RE can pass the kernel BTF verifier. in btf_float_check_member()
4911 align_bytes = min_t(u64, sizeof(void *), member_type->size); in btf_float_check_member()
4913 div64_u64_rem(member->offset, align_bits, &misalign_bits); in btf_float_check_member()
4917 return -EINVAL; in btf_float_check_member()
4920 start_offset_bytes = member->offset / BITS_PER_BYTE; in btf_float_check_member()
4921 end_offset_bytes = start_offset_bytes + member_type->size; in btf_float_check_member()
4922 if (end_offset_bytes > struct_type->size) { in btf_float_check_member()
4925 return -EINVAL; in btf_float_check_member()
4934 btf_verifier_log(env, "size=%u", t->size); in btf_float_log()
4959 return -EINVAL; in btf_decl_tag_check_meta()
4962 value = btf_name_by_offset(env->btf, t->name_off); in btf_decl_tag_check_meta()
4965 return -EINVAL; in btf_decl_tag_check_meta()
4970 return -EINVAL; in btf_decl_tag_check_meta()
4973 component_idx = btf_type_decl_tag(t)->component_idx; in btf_decl_tag_check_meta()
4974 if (component_idx < -1) { in btf_decl_tag_check_meta()
4976 return -EINVAL; in btf_decl_tag_check_meta()
4988 const struct btf_type *t = v->t; in btf_decl_tag_resolve()
4989 u32 next_type_id = t->type; in btf_decl_tag_resolve()
4990 struct btf *btf = env->btf; in btf_decl_tag_resolve()
4996 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_decl_tag_resolve()
4997 return -EINVAL; in btf_decl_tag_resolve()
5004 component_idx = btf_type_decl_tag(t)->component_idx; in btf_decl_tag_resolve()
5005 if (component_idx != -1) { in btf_decl_tag_resolve()
5007 btf_verifier_log_type(env, v->t, "Invalid component_idx"); in btf_decl_tag_resolve()
5008 return -EINVAL; in btf_decl_tag_resolve()
5015 next_type = btf_type_by_id(btf, next_type->type); in btf_decl_tag_resolve()
5020 btf_verifier_log_type(env, v->t, "Invalid component_idx"); in btf_decl_tag_resolve()
5021 return -EINVAL; in btf_decl_tag_resolve()
5032 btf_verifier_log(env, "type=%u component_idx=%d", t->type, in btf_decl_tag_log()
5033 btf_type_decl_tag(t)->component_idx); in btf_decl_tag_log()
5054 btf = env->btf; in btf_func_proto_check()
5058 /* Check func return type which could be "void" (t->type == 0) */ in btf_func_proto_check()
5059 if (t->type) { in btf_func_proto_check()
5060 u32 ret_type_id = t->type; in btf_func_proto_check()
5065 return -EINVAL; in btf_func_proto_check()
5070 return -EINVAL; in btf_func_proto_check()
5083 return -EINVAL; in btf_func_proto_check()
5091 if (!args[nr_args - 1].type) { in btf_func_proto_check()
5092 if (args[nr_args - 1].name_off) { in btf_func_proto_check()
5095 return -EINVAL; in btf_func_proto_check()
5097 nr_args--; in btf_func_proto_check()
5108 return -EINVAL; in btf_func_proto_check()
5113 return -EINVAL; in btf_func_proto_check()
5121 return -EINVAL; in btf_func_proto_check()
5133 return -EINVAL; in btf_func_proto_check()
5148 btf = env->btf; in btf_func_check()
5149 proto_type = btf_type_by_id(btf, t->type); in btf_func_check()
5153 return -EINVAL; in btf_func_check()
5161 return -EINVAL; in btf_func_check()
5199 env->log_type_id, meta_left, sizeof(*t)); in btf_check_meta()
5200 return -EINVAL; in btf_check_meta()
5202 meta_left -= sizeof(*t); in btf_check_meta()
5204 if (t->info & ~BTF_INFO_MASK) { in btf_check_meta()
5206 env->log_type_id, t->info); in btf_check_meta()
5207 return -EINVAL; in btf_check_meta()
5210 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX || in btf_check_meta()
5211 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) { in btf_check_meta()
5213 env->log_type_id, BTF_INFO_KIND(t->info)); in btf_check_meta()
5214 return -EINVAL; in btf_check_meta()
5217 if (!btf_name_offset_valid(env->btf, t->name_off)) { in btf_check_meta()
5219 env->log_type_id, t->name_off); in btf_check_meta()
5220 return -EINVAL; in btf_check_meta()
5223 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left); in btf_check_meta()
5227 meta_left -= var_meta_size; in btf_check_meta()
5229 return saved_meta_left - meta_left; in btf_check_meta()
5234 struct btf *btf = env->btf; in btf_check_all_metas()
5238 hdr = &btf->hdr; in btf_check_all_metas()
5239 cur = btf->nohdr_data + hdr->type_off; in btf_check_all_metas()
5240 end = cur + hdr->type_len; in btf_check_all_metas()
5242 env->log_type_id = btf->base_btf ? btf->start_id : 1; in btf_check_all_metas()
5247 meta_size = btf_check_meta(env, t, end - cur); in btf_check_all_metas()
5253 env->log_type_id++; in btf_check_all_metas()
5263 struct btf *btf = env->btf; in btf_resolve_valid()
5288 u32 elem_type_id = array->type; in btf_resolve_valid()
5293 (array->nelems * elem_size == in btf_resolve_valid()
5303 u32 save_log_type_id = env->log_type_id; in btf_resolve()
5307 env->resolve_mode = RESOLVE_TBD; in btf_resolve()
5310 env->log_type_id = v->type_id; in btf_resolve()
5311 err = btf_type_ops(v->t)->resolve(env, v); in btf_resolve()
5314 env->log_type_id = type_id; in btf_resolve()
5315 if (err == -E2BIG) { in btf_resolve()
5319 } else if (err == -EEXIST) { in btf_resolve()
5326 err = -EINVAL; in btf_resolve()
5329 env->log_type_id = save_log_type_id; in btf_resolve()
5335 struct btf *btf = env->btf; in btf_check_all_types()
5344 env->phase++; in btf_check_all_types()
5345 for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) { in btf_check_all_types()
5346 type_id = btf->start_id + i; in btf_check_all_types()
5349 env->log_type_id = type_id; in btf_check_all_types()
5369 const struct btf_header *hdr = &env->btf->hdr; in btf_parse_type_sec()
5373 if (hdr->type_off & (sizeof(u32) - 1)) { in btf_parse_type_sec()
5375 return -EINVAL; in btf_parse_type_sec()
5378 if (!env->btf->base_btf && !hdr->type_len) { in btf_parse_type_sec()
5380 return -EINVAL; in btf_parse_type_sec()
5393 struct btf *btf = env->btf; in btf_parse_str_sec()
5396 hdr = &btf->hdr; in btf_parse_str_sec()
5397 start = btf->nohdr_data + hdr->str_off; in btf_parse_str_sec()
5398 end = start + hdr->str_len; in btf_parse_str_sec()
5400 if (end != btf->data + btf->data_size) { in btf_parse_str_sec()
5402 return -EINVAL; in btf_parse_str_sec()
5405 btf->strings = start; in btf_parse_str_sec()
5407 if (btf->base_btf && !hdr->str_len) in btf_parse_str_sec()
5409 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || end[-1]) { in btf_parse_str_sec()
5411 return -EINVAL; in btf_parse_str_sec()
5413 if (!btf->base_btf && start[0]) { in btf_parse_str_sec()
5415 return -EINVAL; in btf_parse_str_sec()
5431 return (int)(x->off - y->off) ? : (int)(x->len - y->len); in btf_sec_info_cmp()
5442 btf = env->btf; in btf_check_sec_info()
5443 hdr = &btf->hdr; in btf_check_sec_info()
5455 expected_total = btf_data_size - hdr->hdr_len; in btf_check_sec_info()
5459 return -EINVAL; in btf_check_sec_info()
5464 return -EINVAL; in btf_check_sec_info()
5468 return -EINVAL; in btf_check_sec_info()
5470 if (expected_total - total < secs[i].len) { in btf_check_sec_info()
5473 return -EINVAL; in btf_check_sec_info()
5481 return -EINVAL; in btf_check_sec_info()
5493 btf = env->btf; in btf_parse_hdr()
5494 btf_data_size = btf->data_size; in btf_parse_hdr()
5498 return -EINVAL; in btf_parse_hdr()
5501 hdr = btf->data; in btf_parse_hdr()
5502 hdr_len = hdr->hdr_len; in btf_parse_hdr()
5505 return -EINVAL; in btf_parse_hdr()
5509 if (hdr_len > sizeof(btf->hdr)) { in btf_parse_hdr()
5510 u8 *expected_zero = btf->data + sizeof(btf->hdr); in btf_parse_hdr()
5511 u8 *end = btf->data + hdr_len; in btf_parse_hdr()
5516 return -E2BIG; in btf_parse_hdr()
5521 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); in btf_parse_hdr()
5522 memcpy(&btf->hdr, btf->data, hdr_copy); in btf_parse_hdr()
5524 hdr = &btf->hdr; in btf_parse_hdr()
5528 if (hdr->magic != BTF_MAGIC) { in btf_parse_hdr()
5530 return -EINVAL; in btf_parse_hdr()
5533 if (hdr->version != BTF_VERSION) { in btf_parse_hdr()
5535 return -ENOTSUPP; in btf_parse_hdr()
5538 if (hdr->flags) { in btf_parse_hdr()
5540 return -ENOTSUPP; in btf_parse_hdr()
5543 if (!btf->base_btf && btf_data_size == hdr->hdr_len) { in btf_parse_hdr()
5545 return -EINVAL; in btf_parse_hdr()
5567 BUILD_BUG_ON(offsetof(struct btf_id_set, cnt) != 0); in btf_parse_struct_metas()
5572 return ERR_PTR(-ENOMEM); in btf_parse_struct_metas()
5573 aof->cnt = 0; in btf_parse_struct_metas()
5586 new_aof = krealloc(aof, offsetof(struct btf_id_set, ids[aof->cnt + 1]), in btf_parse_struct_metas()
5589 ret = -ENOMEM; in btf_parse_struct_metas()
5593 aof->ids[aof->cnt++] = id; in btf_parse_struct_metas()
5605 ret = -EINVAL; in btf_parse_struct_metas()
5613 new_aof = krealloc(aof, offsetof(struct btf_id_set, ids[aof->cnt + 1]), in btf_parse_struct_metas()
5616 ret = -ENOMEM; in btf_parse_struct_metas()
5620 aof->ids[aof->cnt++] = i; in btf_parse_struct_metas()
5623 if (!aof->cnt) { in btf_parse_struct_metas()
5627 sort(&aof->ids, aof->cnt, sizeof(aof->ids[0]), btf_id_cmp_func, NULL); in btf_parse_struct_metas()
5644 if (btf_id_set_contains(aof, member->type)) in btf_parse_struct_metas()
5649 tab_cnt = tab ? tab->cnt : 0; in btf_parse_struct_metas()
5653 ret = -ENOMEM; in btf_parse_struct_metas()
5657 new_tab->cnt = 0; in btf_parse_struct_metas()
5660 type = &tab->types[tab->cnt]; in btf_parse_struct_metas()
5661 type->btf_id = i; in btf_parse_struct_metas()
5664 BPF_KPTR, t->size); in btf_parse_struct_metas()
5667 ret = PTR_ERR_OR_ZERO(record) ?: -EFAULT; in btf_parse_struct_metas()
5670 type->record = record; in btf_parse_struct_metas()
5671 tab->cnt++; in btf_parse_struct_metas()
5687 tab = btf->struct_meta_tab; in btf_find_struct_meta()
5690 return bsearch(&btf_id, tab->types, tab->cnt, sizeof(tab->types[0]), btf_id_cmp_func); in btf_find_struct_meta()
5696 int i, n, good_id = start_id - 1; in btf_check_type_tags()
5707 return -EINVAL; in btf_check_type_tags()
5715 if (!chain_limit--) { in btf_check_type_tags()
5717 return -ELOOP; in btf_check_type_tags()
5722 return -EINVAL; in btf_check_type_tags()
5730 cur_id = t->type; in btf_check_type_tags()
5733 return -EINVAL; in btf_check_type_tags()
5750 err = -EFAULT; in finalize_log()
5757 bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); in btf_parse()
5758 char __user *log_ubuf = u64_to_user_ptr(attr->btf_log_buf); in btf_parse()
5765 if (attr->btf_size > BTF_MAX_SIZE) in btf_parse()
5766 return ERR_PTR(-E2BIG); in btf_parse()
5770 return ERR_PTR(-ENOMEM); in btf_parse()
5775 err = bpf_vlog_init(&env->log, attr->btf_log_level, in btf_parse()
5776 log_ubuf, attr->btf_log_size); in btf_parse()
5782 err = -ENOMEM; in btf_parse()
5785 env->btf = btf; in btf_parse()
5787 data = kvmalloc(attr->btf_size, GFP_KERNEL | __GFP_NOWARN); in btf_parse()
5789 err = -ENOMEM; in btf_parse()
5793 btf->data = data; in btf_parse()
5794 btf->data_size = attr->btf_size; in btf_parse()
5796 if (copy_from_bpfptr(data, btf_data, attr->btf_size)) { in btf_parse()
5797 err = -EFAULT; in btf_parse()
5805 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse()
5819 struct_meta_tab = btf_parse_struct_metas(&env->log, btf); in btf_parse()
5824 btf->struct_meta_tab = struct_meta_tab; in btf_parse()
5829 for (i = 0; i < struct_meta_tab->cnt; i++) { in btf_parse()
5830 err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record); in btf_parse()
5836 err = finalize_log(&env->log, uattr, uattr_size); in btf_parse()
5841 refcount_set(&btf->refcnt, 1); in btf_parse()
5847 /* overwrite err with -ENOSPC or -EFAULT */ in btf_parse()
5848 ret = finalize_log(&env->log, uattr, uattr_size); in btf_parse()
5875 enum {
5880 __ctx_convert_unused, /* to avoid empty enum in extreme .config */
5892 static const struct btf_type *find_canonical_prog_ctx_type(enum bpf_prog_type prog_type) in find_canonical_prog_ctx_type()
5905 return btf_type_by_id(btf_vmlinux, ctx_type->type); in find_canonical_prog_ctx_type()
5908 static int find_kern_ctx_type_id(enum bpf_prog_type prog_type) in find_kern_ctx_type_id()
5915 return -EFAULT; in find_kern_ctx_type_id()
5921 return ctx_type->type; in find_kern_ctx_type_id()
5934 const struct btf_type *t, enum bpf_prog_type prog_type, in btf_is_prog_ctx_type()
5940 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5947 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5950 tname = btf_name_by_offset(btf, t->name_off); in btf_is_prog_ctx_type()
5957 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5966 tname = btf_name_by_offset(btf, t->name_off); in btf_is_prog_ctx_type()
5968 bpf_log(log, "arg#%d struct doesn't have a name\n", arg); in btf_is_prog_ctx_type()
5979 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off); in btf_is_prog_ctx_type()
5988 /* only compare that prog's ctx type name is the same as in btf_is_prog_ctx_type()
5999 * underlying struct and check name again in btf_is_prog_ctx_type()
6004 ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type); in btf_is_prog_ctx_type()
6010 /* forward declarations for arch-specific underlying types of
6011 * bpf_user_pt_regs_t; this avoids the need for arch-specific #ifdef
6021 enum bpf_prog_type prog_type, in btf_validate_prog_ctx_type()
6022 enum bpf_attach_type attach_type) in btf_validate_prog_ctx_type()
6029 return -EINVAL; in btf_validate_prog_ctx_type()
6031 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6036 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6039 tname = btf_name_by_offset(btf, t->name_off); in btf_validate_prog_ctx_type()
6047 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6053 tname = btf_name_by_offset(btf, t->name_off); in btf_validate_prog_ctx_type()
6055 bpf_log(log, "arg#%d type doesn't have a name\n", arg); in btf_validate_prog_ctx_type()
6056 return -EINVAL; in btf_validate_prog_ctx_type()
6079 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6090 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6096 strncmp(tname, "bpf_iter__", sizeof("bpf_iter__") - 1) == 0) in btf_validate_prog_ctx_type()
6103 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6113 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6128 return -EINVAL; in btf_validate_prog_ctx_type()
6133 ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type); in btf_validate_prog_ctx_type()
6141 return -EINVAL; in btf_validate_prog_ctx_type()
6144 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off); in btf_validate_prog_ctx_type()
6147 return -EINVAL; in btf_validate_prog_ctx_type()
6156 enum bpf_prog_type prog_type, in btf_translate_to_vmlinux()
6160 return -ENOENT; in btf_translate_to_vmlinux()
6164 int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_type) in get_kern_ctx_btf_id()
6174 kctx_type_id = kctx_member->type; in get_kern_ctx_btf_id()
6178 return -EINVAL; in get_kern_ctx_btf_id()
6187 static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name, in BTF_ID()
6194 return ERR_PTR(-ENOENT); in BTF_ID()
6198 err = -ENOMEM; in BTF_ID()
6201 env->btf = btf; in BTF_ID()
6203 btf->data = data; in BTF_ID()
6204 btf->data_size = data_size; in BTF_ID()
6205 btf->kernel_btf = true; in BTF_ID()
6206 snprintf(btf->name, sizeof(btf->name), "%s", name); in BTF_ID()
6212 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in BTF_ID()
6226 refcount_set(&btf->refcnt, 1); in BTF_ID()
6232 kvfree(btf->types); in BTF_ID()
6247 return ERR_PTR(-ENOMEM); in btf_parse_vmlinux()
6249 log = &env->log; in btf_parse_vmlinux()
6250 log->level = BPF_LOG_KERNEL; in btf_parse_vmlinux()
6251 btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF); in btf_parse_vmlinux()
6273 if (!btf->base_btf || !btf->base_id_map) in btf_relocate_id()
6275 return btf->base_id_map[id]; in btf_relocate_id()
6293 return ERR_PTR(-EINVAL); in btf_parse_module()
6297 return ERR_PTR(-ENOMEM); in btf_parse_module()
6299 log = &env->log; in btf_parse_module()
6300 log->level = BPF_LOG_KERNEL; in btf_parse_module()
6314 err = -ENOMEM; in btf_parse_module()
6317 env->btf = btf; in btf_parse_module()
6319 btf->base_btf = base_btf; in btf_parse_module()
6320 btf->start_id = base_btf->nr_types; in btf_parse_module()
6321 btf->start_str_off = base_btf->hdr.str_len; in btf_parse_module()
6322 btf->kernel_btf = true; in btf_parse_module()
6323 snprintf(btf->name, sizeof(btf->name), "%s", module_name); in btf_parse_module()
6325 btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); in btf_parse_module()
6326 if (!btf->data) { in btf_parse_module()
6327 err = -ENOMEM; in btf_parse_module()
6330 btf->data_size = data_size; in btf_parse_module()
6336 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse_module()
6351 err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map); in btf_parse_module()
6359 refcount_set(&btf->refcnt, 1); in btf_parse_module()
6367 kvfree(btf->data); in btf_parse_module()
6368 kvfree(btf->types); in btf_parse_module()
6378 struct bpf_prog *tgt_prog = prog->aux->dst_prog; in bpf_prog_get_target_btf()
6381 return tgt_prog->aux->btf; in bpf_prog_get_target_btf()
6383 return prog->aux->attach_btf; in bpf_prog_get_target_btf()
6389 t = btf_type_skip_modifiers(btf, t->type, NULL); in is_int_ptr()
6409 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); in get_ctx_arg_idx()
6414 t = btf_type_skip_modifiers(btf, func_proto->type, NULL); in get_ctx_arg_idx()
6415 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); in get_ctx_arg_idx()
6424 enum bpf_attach_type atype = prog->expected_attach_type; in prog_args_trusted()
6426 switch (prog->type) { in prog_args_trusted()
6654 bool btf_ctx_access(int off, int size, enum bpf_access_type type, in btf_ctx_access()
6658 const struct btf_type *t = prog->aux->attach_func_proto; in btf_ctx_access()
6659 struct bpf_prog *tgt_prog = prog->aux->dst_prog; in btf_ctx_access()
6661 const char *tname = prog->aux->attach_func_name; in btf_ctx_access()
6662 struct bpf_verifier_log *log = info->log; in btf_ctx_access()
6680 if (prog->aux->attach_btf_trace) { in btf_ctx_access()
6681 /* skip first 'void *__data' argument in btf_trace_##name typedef */ in btf_ctx_access()
6683 nr_args--; in btf_ctx_access()
6687 bpf_log(log, "func '%s' doesn't have %d-th argument\n", in btf_ctx_access()
6693 switch (prog->expected_attach_type) { in btf_ctx_access()
6696 info->is_retval = true; in btf_ctx_access()
6704 * While the LSM programs are BPF_MODIFY_RETURN-like in btf_ctx_access()
6708 * return -EINVAL; in btf_ctx_access()
6715 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6724 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_ctx_access()
6733 bpf_log(log, "func '%s' doesn't have %d-th argument\n", in btf_ctx_access()
6746 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6754 __btf_name_by_offset(btf, t->name_off), in btf_ctx_access()
6766 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { in btf_ctx_access()
6767 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; in btf_ctx_access()
6770 type = base_type(ctx_arg_info->reg_type); in btf_ctx_access()
6771 flag = type_flag(ctx_arg_info->reg_type); in btf_ctx_access()
6772 if (ctx_arg_info->offset == off && type == PTR_TO_BUF && in btf_ctx_access()
6774 info->reg_type = ctx_arg_info->reg_type; in btf_ctx_access()
6779 if (t->type == 0) in btf_ctx_access()
6790 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { in btf_ctx_access()
6791 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; in btf_ctx_access()
6793 if (ctx_arg_info->offset == off) { in btf_ctx_access()
6794 if (!ctx_arg_info->btf_id) { in btf_ctx_access()
6799 info->reg_type = ctx_arg_info->reg_type; in btf_ctx_access()
6800 info->btf = ctx_arg_info->btf ? : btf_vmlinux; in btf_ctx_access()
6801 info->btf_id = ctx_arg_info->btf_id; in btf_ctx_access()
6802 info->ref_obj_id = ctx_arg_info->ref_obj_id; in btf_ctx_access()
6807 info->reg_type = PTR_TO_BTF_ID; in btf_ctx_access()
6809 info->reg_type |= PTR_TRUSTED; in btf_ctx_access()
6812 info->reg_type |= PTR_MAYBE_NULL; in btf_ctx_access()
6814 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { in btf_ctx_access()
6815 struct btf *btf = prog->aux->attach_btf; in btf_ctx_access()
6820 t = btf_type_by_id(btf, prog->aux->attach_btf_id); in btf_ctx_access()
6823 tname = btf_name_by_offset(btf, t->name_off); in btf_ctx_access()
6827 tname += sizeof("btf_trace_") - 1; in btf_ctx_access()
6833 info->reg_type |= PTR_MAYBE_NULL; in btf_ctx_access()
6839 /* If we don't know NULL-ness specification and the tracepoint in btf_ctx_access()
6844 info->reg_type |= PTR_MAYBE_NULL; in btf_ctx_access()
6848 enum bpf_prog_type tgt_type; in btf_ctx_access()
6850 if (tgt_prog->type == BPF_PROG_TYPE_EXT) in btf_ctx_access()
6851 tgt_type = tgt_prog->aux->saved_dst_prog_type; in btf_ctx_access()
6853 tgt_type = tgt_prog->type; in btf_ctx_access()
6857 info->btf = btf_vmlinux; in btf_ctx_access()
6858 info->btf_id = ret; in btf_ctx_access()
6865 info->btf = btf; in btf_ctx_access()
6866 info->btf_id = t->type; in btf_ctx_access()
6867 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6870 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_ctx_access()
6872 info->reg_type |= MEM_USER; in btf_ctx_access()
6874 info->reg_type |= MEM_PERCPU; in btf_ctx_access()
6879 info->btf_id = t->type; in btf_ctx_access()
6880 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6889 tname, arg, info->btf_id, btf_type_str(t), in btf_ctx_access()
6890 __btf_name_by_offset(btf, t->name_off)); in btf_ctx_access()
6898 info->reg_type = SCALAR_VALUE; in btf_ctx_access()
6904 enum bpf_struct_walk_result {
6913 u32 *next_btf_id, enum bpf_type_flag *flag, in btf_struct_walk()
6924 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_struct_walk()
6925 tname = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
6928 return -EINVAL; in btf_struct_walk()
6932 if (BTF_INFO_KIND(t->info) == BTF_KIND_UNION && vlen != 1 && !(*flag & PTR_UNTRUSTED)) in btf_struct_walk()
6940 if (off + size > t->size) { in btf_struct_walk()
6949 member = btf_type_member(t) + vlen - 1; in btf_struct_walk()
6950 mtype = btf_type_skip_modifiers(btf, member->type, in btf_struct_walk()
6956 if (array_elem->nelems != 0) in btf_struct_walk()
6964 t = btf_type_skip_modifiers(btf, array_elem->type, in btf_struct_walk()
6973 off = (off - moff) % t->size; in btf_struct_walk()
6979 return -EACCES; in btf_struct_walk()
7025 mid = member->type; in btf_struct_walk()
7026 mtype = btf_type_by_id(btf, member->type); in btf_struct_walk()
7027 mname = __btf_name_by_offset(btf, member->name_off); in btf_struct_walk()
7034 return -EFAULT; in btf_struct_walk()
7046 * linearize a multi-dimensional array. in btf_struct_walk()
7064 * When accessing outer->array[1][0], it moves in btf_struct_walk()
7090 elem_idx = (off - moff) / msize; in btf_struct_walk()
7110 off -= moff; in btf_struct_walk()
7116 enum bpf_type_flag tmp_flag = 0; in btf_struct_walk()
7123 return -EACCES; in btf_struct_walk()
7127 t = btf_type_by_id(btf, mtype->type); in btf_struct_walk()
7129 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
7141 stype = btf_type_skip_modifiers(btf, mtype->type, &id); in btf_struct_walk()
7155 * space. e.g. skb->cb[]. in btf_struct_walk()
7161 return -EACCES; in btf_struct_walk()
7167 return -EINVAL; in btf_struct_walk()
7172 int off, int size, enum bpf_access_type atype __maybe_unused, in btf_struct_access()
7173 u32 *next_btf_id, enum bpf_type_flag *flag, in btf_struct_access()
7176 const struct btf *btf = reg->btf; in btf_struct_access()
7177 enum bpf_type_flag tmp_flag = 0; in btf_struct_access()
7179 u32 id = reg->btf_id; in btf_struct_access()
7182 while (type_is_alloc(reg->type)) { in btf_struct_access()
7190 rec = meta->record; in btf_struct_access()
7191 for (i = 0; i < rec->cnt; i++) { in btf_struct_access()
7192 struct btf_field *field = &rec->fields[i]; in btf_struct_access()
7193 u32 offset = field->offset; in btf_struct_access()
7194 if (off < offset + field->size && offset < off + size) { in btf_struct_access()
7197 btf_field_type_name(field->type)); in btf_struct_access()
7198 return -EACCES; in btf_struct_access()
7213 if (type_is_alloc(reg->type)) in btf_struct_access()
7236 return -EINVAL; in btf_struct_access()
7241 return -EINVAL; in btf_struct_access()
7265 enum bpf_type_flag flag = 0; in btf_struct_ids_match()
7309 t = btf_type_by_id(btf, t->type); in __get_type_size()
7311 return -EINVAL; in __get_type_size()
7317 return t->size; in __get_type_size()
7318 return -EINVAL; in __get_type_size()
7349 m->arg_size[i] = 8; in btf_distill_func_proto()
7350 m->arg_flags[i] = 0; in btf_distill_func_proto()
7352 m->ret_size = 8; in btf_distill_func_proto()
7353 m->ret_flags = 0; in btf_distill_func_proto()
7354 m->nr_args = MAX_BPF_FUNC_REG_ARGS; in btf_distill_func_proto()
7363 return -EINVAL; in btf_distill_func_proto()
7365 ret = __get_type_size(btf, func->type, &t); in btf_distill_func_proto()
7370 return -EINVAL; in btf_distill_func_proto()
7372 m->ret_size = ret; in btf_distill_func_proto()
7373 m->ret_flags = __get_type_fmodel_flags(t); in btf_distill_func_proto()
7376 if (i == nargs - 1 && args[i].type == 0) { in btf_distill_func_proto()
7380 return -EINVAL; in btf_distill_func_proto()
7389 return -EINVAL; in btf_distill_func_proto()
7395 return -EINVAL; in btf_distill_func_proto()
7397 m->arg_size[i] = ret; in btf_distill_func_proto()
7398 m->arg_flags[i] = __get_type_fmodel_flags(t); in btf_distill_func_proto()
7400 m->nr_args = nargs; in btf_distill_func_proto()
7408 * EINVAL - function prototype mismatch
7409 * EFAULT - verifier bug
7410 * 0 - 99% match. The last 1% is validated by the verifier.
7420 fn1 = btf_name_by_offset(btf1, t1->name_off); in btf_check_func_type_match()
7421 fn2 = btf_name_by_offset(btf2, t2->name_off); in btf_check_func_type_match()
7425 return -EINVAL; in btf_check_func_type_match()
7429 return -EINVAL; in btf_check_func_type_match()
7432 t1 = btf_type_by_id(btf1, t1->type); in btf_check_func_type_match()
7434 return -EFAULT; in btf_check_func_type_match()
7435 t2 = btf_type_by_id(btf2, t2->type); in btf_check_func_type_match()
7437 return -EFAULT; in btf_check_func_type_match()
7447 return -EINVAL; in btf_check_func_type_match()
7450 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); in btf_check_func_type_match()
7451 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); in btf_check_func_type_match()
7452 if (t1->info != t2->info) { in btf_check_func_type_match()
7457 return -EINVAL; in btf_check_func_type_match()
7464 if (t1->info != t2->info) { in btf_check_func_type_match()
7468 return -EINVAL; in btf_check_func_type_match()
7470 if (btf_type_has_size(t1) && t1->size != t2->size) { in btf_check_func_type_match()
7473 i, fn1, t1->size, in btf_check_func_type_match()
7474 fn2, t2->size); in btf_check_func_type_match()
7475 return -EINVAL; in btf_check_func_type_match()
7488 return -EINVAL; in btf_check_func_type_match()
7490 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); in btf_check_func_type_match()
7491 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); in btf_check_func_type_match()
7496 return -EINVAL; in btf_check_func_type_match()
7502 return -EINVAL; in btf_check_func_type_match()
7510 s1 = btf_name_by_offset(btf1, t1->name_off); in btf_check_func_type_match()
7511 s2 = btf_name_by_offset(btf2, t2->name_off); in btf_check_func_type_match()
7516 return -EINVAL; in btf_check_func_type_match()
7526 struct btf *btf1 = prog->aux->btf; in btf_check_type_match()
7530 if (!prog->aux->func_info) { in btf_check_type_match()
7532 return -EINVAL; in btf_check_type_match()
7535 btf_id = prog->aux->func_info[0].type_id; in btf_check_type_match()
7537 return -EFAULT; in btf_check_type_match()
7541 return -EFAULT; in btf_check_type_match()
7548 const char *name; in btf_is_dynptr_ptr() local
7550 t = btf_type_by_id(btf, t->type); /* skip PTR */ in btf_is_dynptr_ptr()
7553 t = btf_type_by_id(btf, t->type); in btf_is_dynptr_ptr()
7558 name = btf_str_by_offset(btf, t->name_off); in btf_is_dynptr_ptr()
7559 return name && strcmp(name, "bpf_dynptr") == 0; in btf_is_dynptr_ptr()
7566 const char *name; member
7569 u16 cnt; member
7593 type_id = t->type; in btf_get_ptr_to_btf_id()
7594 t = btf_type_by_id(btf, t->type); in btf_get_ptr_to_btf_id()
7596 type_id = t->type; in btf_get_ptr_to_btf_id()
7597 t = btf_type_by_id(btf, t->type); in btf_get_ptr_to_btf_id()
7605 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), in btf_get_ptr_to_btf_id()
7609 if (cc->cnt != 1) { in btf_get_ptr_to_btf_id()
7611 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), in btf_get_ptr_to_btf_id()
7612 cc->cnt == 0 ? "has no matches" : "is ambiguous"); in btf_get_ptr_to_btf_id()
7613 err = cc->cnt == 0 ? -ENOENT : -ESRCH; in btf_get_ptr_to_btf_id()
7616 if (btf_is_module(cc->cands[0].btf)) { in btf_get_ptr_to_btf_id()
7618 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off)); in btf_get_ptr_to_btf_id()
7619 err = -EOPNOTSUPP; in btf_get_ptr_to_btf_id()
7622 kern_type_id = cc->cands[0].id; in btf_get_ptr_to_btf_id()
7632 enum btf_arg_tag {
7640 /* Process BTF of a function to produce high-level expectation of function
7644 * EFAULT - there is a verifier bug. Abort verification.
7645 * EINVAL - cannot convert BTF.
7646 * 0 - Successfully processed BTF and constructed argument expectations.
7650 bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL; in btf_prepare_func_args()
7652 struct bpf_verifier_log *log = &env->log; in btf_prepare_func_args()
7653 struct bpf_prog *prog = env->prog; in btf_prepare_func_args()
7654 enum bpf_prog_type prog_type = prog->type; in btf_prepare_func_args()
7655 struct btf *btf = prog->aux->btf; in btf_prepare_func_args()
7661 if (sub->args_cached) in btf_prepare_func_args()
7664 if (!prog->aux->func_info) { in btf_prepare_func_args()
7666 return -EFAULT; in btf_prepare_func_args()
7669 btf_id = prog->aux->func_info[subprog].type_id; in btf_prepare_func_args()
7672 return -EINVAL; in btf_prepare_func_args()
7674 return -EFAULT; in btf_prepare_func_args()
7684 return -EFAULT; in btf_prepare_func_args()
7686 tname = btf_name_by_offset(btf, fn_t->name_off); in btf_prepare_func_args()
7688 if (prog->aux->func_info_aux[subprog].unreliable) { in btf_prepare_func_args()
7690 return -EFAULT; in btf_prepare_func_args()
7693 prog_type = prog->aux->dst_prog->type; in btf_prepare_func_args()
7695 t = btf_type_by_id(btf, fn_t->type); in btf_prepare_func_args()
7698 return -EFAULT; in btf_prepare_func_args()
7704 return -EINVAL; in btf_prepare_func_args()
7707 return -EINVAL; in btf_prepare_func_args()
7710 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7712 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7715 return -EINVAL; in btf_prepare_func_args()
7719 return -EINVAL; in btf_prepare_func_args()
7733 const char *tag = __btf_name_by_offset(btf, tag_t->name_off) + 4; in btf_prepare_func_args()
7738 return -EOPNOTSUPP; in btf_prepare_func_args()
7753 return -EOPNOTSUPP; in btf_prepare_func_args()
7756 if (id != -ENOENT) { in btf_prepare_func_args()
7763 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7770 return -EINVAL; in btf_prepare_func_args()
7774 prog->expected_attach_type)) in btf_prepare_func_args()
7775 return -EINVAL; in btf_prepare_func_args()
7776 sub->args[i].arg_type = ARG_PTR_TO_CTX; in btf_prepare_func_args()
7782 return -EINVAL; in btf_prepare_func_args()
7784 sub->args[i].arg_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY; in btf_prepare_func_args()
7792 return -EINVAL; in btf_prepare_func_args()
7799 sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; in btf_prepare_func_args()
7801 sub->args[i].arg_type |= PTR_MAYBE_NULL; in btf_prepare_func_args()
7802 sub->args[i].btf_id = kern_type_id; in btf_prepare_func_args()
7808 return -EINVAL; in btf_prepare_func_args()
7810 sub->args[i].arg_type = ARG_PTR_TO_ARENA; in btf_prepare_func_args()
7818 return -EINVAL; in btf_prepare_func_args()
7821 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_prepare_func_args()
7825 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), in btf_prepare_func_args()
7827 return -EINVAL; in btf_prepare_func_args()
7830 sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; in btf_prepare_func_args()
7832 sub->args[i].arg_type &= ~PTR_MAYBE_NULL; in btf_prepare_func_args()
7833 sub->args[i].mem_size = mem_size; in btf_prepare_func_args()
7840 return -EINVAL; in btf_prepare_func_args()
7843 sub->args[i].arg_type = ARG_ANYTHING; in btf_prepare_func_args()
7847 return -EINVAL; in btf_prepare_func_args()
7850 return -EINVAL; in btf_prepare_func_args()
7853 sub->arg_cnt = nargs; in btf_prepare_func_args()
7854 sub->args_cached = true; in btf_prepare_func_args()
7864 show->btf = btf; in btf_type_show()
7865 memset(&show->state, 0, sizeof(show->state)); in btf_type_show()
7866 memset(&show->obj, 0, sizeof(show->obj)); in btf_type_show()
7868 btf_type_ops(t)->show(btf, t, type_id, obj, 0, show); in btf_type_show()
7874 seq_vprintf((struct seq_file *)show->target, fmt, args); in btf_seq_show()
7911 len = vsnprintf(show->target, ssnprintf->len_left, fmt, args); in btf_snprintf_show()
7914 ssnprintf->len_left = 0; in btf_snprintf_show()
7915 ssnprintf->len = len; in btf_snprintf_show()
7916 } else if (len >= ssnprintf->len_left) { in btf_snprintf_show()
7918 ssnprintf->len_left = 0; in btf_snprintf_show()
7919 ssnprintf->len += len; in btf_snprintf_show()
7921 ssnprintf->len_left -= len; in btf_snprintf_show()
7922 ssnprintf->len += len; in btf_snprintf_show()
7923 show->target += len; in btf_snprintf_show()
7951 const struct btf *btf = filp->private_data; in bpf_btf_show_fdinfo()
7953 seq_printf(m, "btf_id:\t%u\n", btf->id); in bpf_btf_show_fdinfo()
7959 btf_put(filp->private_data); in btf_release()
8010 refcount_inc(&btf->refcnt); in btf_get_by_fd()
8027 uinfo = u64_to_user_ptr(attr->info.info); in btf_get_info_by_fd()
8028 uinfo_len = attr->info.info_len; in btf_get_info_by_fd()
8033 return -EFAULT; in btf_get_info_by_fd()
8035 info.id = btf->id; in btf_get_info_by_fd()
8037 btf_copy = min_t(u32, btf->data_size, info.btf_size); in btf_get_info_by_fd()
8038 if (copy_to_user(ubtf, btf->data, btf_copy)) in btf_get_info_by_fd()
8039 return -EFAULT; in btf_get_info_by_fd()
8040 info.btf_size = btf->data_size; in btf_get_info_by_fd()
8042 info.kernel_btf = btf->kernel_btf; in btf_get_info_by_fd()
8044 uname = u64_to_user_ptr(info.name); in btf_get_info_by_fd()
8047 return -EINVAL; in btf_get_info_by_fd()
8049 name_len = strlen(btf->name); in btf_get_info_by_fd()
8054 if (copy_to_user(uname, btf->name, name_len + 1)) in btf_get_info_by_fd()
8055 return -EFAULT; in btf_get_info_by_fd()
8059 if (copy_to_user(uname, btf->name, uname_len - 1)) in btf_get_info_by_fd()
8060 return -EFAULT; in btf_get_info_by_fd()
8061 if (put_user(zero, uname + uname_len - 1)) in btf_get_info_by_fd()
8062 return -EFAULT; in btf_get_info_by_fd()
8063 /* let user-space know about too short buffer */ in btf_get_info_by_fd()
8064 ret = -ENOSPC; in btf_get_info_by_fd()
8069 put_user(info_copy, &uattr->info.info_len)) in btf_get_info_by_fd()
8070 return -EFAULT; in btf_get_info_by_fd()
8082 if (!btf || !refcount_inc_not_zero(&btf->refcnt)) in btf_get_fd_by_id()
8083 btf = ERR_PTR(-ENOENT); in btf_get_fd_by_id()
8098 return btf->id; in btf_obj_id()
8103 return btf->kernel_btf; in btf_is_kernel()
8108 return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0; in btf_is_module()
8111 enum {
8137 if (mod->btf_data_size == 0 || in btf_module_notify()
8146 err = -ENOMEM; in btf_module_notify()
8149 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, in btf_module_notify()
8150 mod->btf_base_data, mod->btf_base_data_size); in btf_module_notify()
8155 mod->name, PTR_ERR(btf)); in btf_module_notify()
8171 btf_mod->module = module; in btf_module_notify()
8172 btf_mod->btf = btf; in btf_module_notify()
8173 list_add(&btf_mod->list, &btf_modules); in btf_module_notify()
8184 attr->attr.name = btf->name; in btf_module_notify()
8185 attr->attr.mode = 0444; in btf_module_notify()
8186 attr->size = btf->data_size; in btf_module_notify()
8187 attr->private = btf->data; in btf_module_notify()
8188 attr->read_new = sysfs_bin_attr_simple_read; in btf_module_notify()
8193 mod->name, err); in btf_module_notify()
8199 btf_mod->sysfs_attr = attr; in btf_module_notify()
8206 if (btf_mod->module != module) in btf_module_notify()
8209 btf_mod->flags |= BTF_MODULE_F_LIVE; in btf_module_notify()
8217 if (btf_mod->module != module) in btf_module_notify()
8220 list_del(&btf_mod->list); in btf_module_notify()
8221 if (btf_mod->sysfs_attr) in btf_module_notify()
8222 sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); in btf_module_notify()
8223 purge_cand_cache(btf_mod->btf); in btf_module_notify()
8224 btf_put(btf_mod->btf); in btf_module_notify()
8225 kfree(btf_mod->sysfs_attr); in btf_module_notify()
8257 if (btf_mod->btf != btf) in btf_try_get_module()
8265 if ((btf_mod->flags & BTF_MODULE_F_LIVE) && try_module_get(btf_mod->module)) in btf_try_get_module()
8266 res = btf_mod->module; in btf_try_get_module()
8296 if (btf_mod->module != module) in btf_get_module_btf()
8299 btf_get(btf_mod->btf); in btf_get_module_btf()
8300 btf = btf_mod->btf; in btf_get_module_btf()
8313 return -ENOENT; in check_btf_kconfigs()
8320 BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags) in BPF_CALL_4() argument
8327 return -EINVAL; in BPF_CALL_4()
8329 if (name_sz <= 1 || name[name_sz - 1]) in BPF_CALL_4()
8330 return -EINVAL; in BPF_CALL_4()
8332 ret = bpf_find_btf_id(name, kind, &btf); in BPF_CALL_4()
8357 #define BTF_TRACING_TYPE(name, type) BTF_ID(struct, type) in BTF_ID_LIST_GLOBAL() argument
8361 /* Validate well-formedness of iter argument type. in BTF_ID_LIST_GLOBAL()
8369 const char *name; in BTF_ID_LIST_GLOBAL() local
8373 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8376 t = btf_type_skip_modifiers(btf, arg->type, NULL); in BTF_ID_LIST_GLOBAL()
8378 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8379 t = btf_type_skip_modifiers(btf, t->type, &btf_id); in BTF_ID_LIST_GLOBAL()
8381 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8383 name = btf_name_by_offset(btf, t->name_off); in BTF_ID_LIST_GLOBAL()
8384 if (!name || strncmp(name, ITER_PREFIX, sizeof(ITER_PREFIX) - 1)) in BTF_ID_LIST_GLOBAL()
8385 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8401 if (!flags || (flags & (flags - 1))) in btf_check_iter_kfuncs()
8402 return -EINVAL; in btf_check_iter_kfuncs()
8407 return -EINVAL; in btf_check_iter_kfuncs()
8417 if (t->size == 0 || (t->size % 8)) in btf_check_iter_kfuncs()
8418 return -EINVAL; in btf_check_iter_kfuncs()
8423 iter_name = btf_name_by_offset(btf, t->name_off) + sizeof(ITER_PREFIX) - 1; in btf_check_iter_kfuncs()
8433 return -EINVAL; in btf_check_iter_kfuncs()
8437 return -EINVAL; in btf_check_iter_kfuncs()
8441 t = btf_type_skip_modifiers(btf, func->type, NULL); in btf_check_iter_kfuncs()
8443 return -EINVAL; in btf_check_iter_kfuncs()
8448 t = btf_type_by_id(btf, func->type); in btf_check_iter_kfuncs()
8450 return -EINVAL; in btf_check_iter_kfuncs()
8462 /* any kfunc should be FUNC -> FUNC_PROTO */ in btf_check_kfunc_protos()
8465 return -EINVAL; in btf_check_kfunc_protos()
8467 /* sanity check kfunc name */ in btf_check_kfunc_protos()
8468 func_name = btf_name_by_offset(btf, func->name_off); in btf_check_kfunc_protos()
8470 return -EINVAL; in btf_check_kfunc_protos()
8472 func = btf_type_by_id(btf, func->type); in btf_check_kfunc_protos()
8474 return -EINVAL; in btf_check_kfunc_protos()
8487 static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, in btf_populate_kfunc_set()
8491 struct btf_id_set8 *add_set = kset->set; in btf_populate_kfunc_set()
8493 bool add_filter = !!kset->filter; in btf_populate_kfunc_set()
8500 ret = -EINVAL; in btf_populate_kfunc_set()
8504 if (!add_set->cnt) in btf_populate_kfunc_set()
8507 tab = btf->kfunc_set_tab; in btf_populate_kfunc_set()
8512 hook_filter = &tab->hook_filters[hook]; in btf_populate_kfunc_set()
8513 for (i = 0; i < hook_filter->nr_filters; i++) { in btf_populate_kfunc_set()
8514 if (hook_filter->filters[i] == kset->filter) { in btf_populate_kfunc_set()
8520 if (add_filter && hook_filter->nr_filters == BTF_KFUNC_FILTER_MAX_CNT) { in btf_populate_kfunc_set()
8521 ret = -E2BIG; in btf_populate_kfunc_set()
8529 return -ENOMEM; in btf_populate_kfunc_set()
8530 btf->kfunc_set_tab = tab; in btf_populate_kfunc_set()
8533 set = tab->sets[hook]; in btf_populate_kfunc_set()
8538 ret = -EINVAL; in btf_populate_kfunc_set()
8546 * hence re-sorting the final set again is required to make binary in btf_populate_kfunc_set()
8552 set_cnt = set ? set->cnt : 0; in btf_populate_kfunc_set()
8554 if (set_cnt > U32_MAX - add_set->cnt) { in btf_populate_kfunc_set()
8555 ret = -EOVERFLOW; in btf_populate_kfunc_set()
8559 if (set_cnt + add_set->cnt > BTF_KFUNC_SET_MAX_CNT) { in btf_populate_kfunc_set()
8560 ret = -E2BIG; in btf_populate_kfunc_set()
8565 set = krealloc(tab->sets[hook], in btf_populate_kfunc_set()
8566 offsetof(struct btf_id_set8, pairs[set_cnt + add_set->cnt]), in btf_populate_kfunc_set()
8569 ret = -ENOMEM; in btf_populate_kfunc_set()
8573 /* For newly allocated set, initialize set->cnt to 0 */ in btf_populate_kfunc_set()
8574 if (!tab->sets[hook]) in btf_populate_kfunc_set()
8575 set->cnt = 0; in btf_populate_kfunc_set()
8576 tab->sets[hook] = set; in btf_populate_kfunc_set()
8579 memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0])); in btf_populate_kfunc_set()
8581 for (i = set->cnt; i < set->cnt + add_set->cnt; i++) in btf_populate_kfunc_set()
8582 set->pairs[i].id = btf_relocate_id(btf, set->pairs[i].id); in btf_populate_kfunc_set()
8584 set->cnt += add_set->cnt; in btf_populate_kfunc_set()
8586 sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL); in btf_populate_kfunc_set()
8589 hook_filter = &tab->hook_filters[hook]; in btf_populate_kfunc_set()
8590 hook_filter->filters[hook_filter->nr_filters++] = kset->filter; in btf_populate_kfunc_set()
8599 enum btf_kfunc_hook hook, in __btf_kfunc_id_set_contains()
8609 if (!btf->kfunc_set_tab) in __btf_kfunc_id_set_contains()
8611 hook_filter = &btf->kfunc_set_tab->hook_filters[hook]; in __btf_kfunc_id_set_contains()
8612 for (i = 0; i < hook_filter->nr_filters; i++) { in __btf_kfunc_id_set_contains()
8613 if (hook_filter->filters[i](prog, kfunc_btf_id)) in __btf_kfunc_id_set_contains()
8616 set = btf->kfunc_set_tab->sets[hook]; in __btf_kfunc_id_set_contains()
8626 static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) in bpf_prog_type_to_kfunc_hook()
8677 * protection for looking up a well-formed btf->kfunc_set_tab.
8683 enum bpf_prog_type prog_type = resolve_prog_type(prog); in btf_kfunc_id_set_contains()
8684 enum btf_kfunc_hook hook; in btf_kfunc_id_set_contains()
8701 static int __register_btf_kfunc_id_set(enum btf_kfunc_hook hook, in __register_btf_kfunc_id_set()
8707 btf = btf_get_module_btf(kset->owner); in __register_btf_kfunc_id_set()
8709 return check_btf_kconfigs(kset->owner, "kfunc"); in __register_btf_kfunc_id_set()
8713 for (i = 0; i < kset->set->cnt; i++) { in __register_btf_kfunc_id_set()
8714 ret = btf_check_kfunc_protos(btf, btf_relocate_id(btf, kset->set->pairs[i].id), in __register_btf_kfunc_id_set()
8715 kset->set->pairs[i].flags); in __register_btf_kfunc_id_set()
8728 int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, in register_btf_kfunc_id_set()
8731 enum btf_kfunc_hook hook; in register_btf_kfunc_id_set()
8736 if (!(kset->set->flags & BTF_SET8_KFUNCS)) { in register_btf_kfunc_id_set()
8737 WARN_ON(!kset->owner); in register_btf_kfunc_id_set()
8738 return -EINVAL; in register_btf_kfunc_id_set()
8755 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_find_dtor_kfunc()
8759 return -ENOENT; in btf_find_dtor_kfunc()
8760 /* Even though the size of tab->dtors[0] is > sizeof(u32), we only need in btf_find_dtor_kfunc()
8764 dtor = bsearch(&btf_id, tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func); in btf_find_dtor_kfunc()
8766 return -ENOENT; in btf_find_dtor_kfunc()
8767 return dtor->kfunc_btf_id; in btf_find_dtor_kfunc()
8770 static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc *dtors, u32 cnt) in btf_check_dtor_kfuncs() argument
8777 for (i = 0; i < cnt; i++) { in btf_check_dtor_kfuncs()
8782 return -EINVAL; in btf_check_dtor_kfuncs()
8784 dtor_func_proto = btf_type_by_id(btf, dtor_func->type); in btf_check_dtor_kfuncs()
8786 return -EINVAL; in btf_check_dtor_kfuncs()
8789 t = btf_type_by_id(btf, dtor_func_proto->type); in btf_check_dtor_kfuncs()
8791 return -EINVAL; in btf_check_dtor_kfuncs()
8795 return -EINVAL; in btf_check_dtor_kfuncs()
8802 return -EINVAL; in btf_check_dtor_kfuncs()
8824 ret = -E2BIG; in register_btf_id_dtor_kfuncs()
8833 tab = btf->dtor_kfunc_tab; in register_btf_id_dtor_kfuncs()
8836 ret = -EINVAL; in register_btf_id_dtor_kfuncs()
8840 tab_cnt = tab ? tab->cnt : 0; in register_btf_id_dtor_kfuncs()
8841 if (tab_cnt > U32_MAX - add_cnt) { in register_btf_id_dtor_kfuncs()
8842 ret = -EOVERFLOW; in register_btf_id_dtor_kfuncs()
8847 ret = -E2BIG; in register_btf_id_dtor_kfuncs()
8851 tab = krealloc(btf->dtor_kfunc_tab, in register_btf_id_dtor_kfuncs()
8855 ret = -ENOMEM; in register_btf_id_dtor_kfuncs()
8859 if (!btf->dtor_kfunc_tab) in register_btf_id_dtor_kfuncs()
8860 tab->cnt = 0; in register_btf_id_dtor_kfuncs()
8861 btf->dtor_kfunc_tab = tab; in register_btf_id_dtor_kfuncs()
8863 memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0])); in register_btf_id_dtor_kfuncs()
8867 tab->dtors[i].btf_id = btf_relocate_id(btf, tab->dtors[i].btf_id); in register_btf_id_dtor_kfuncs()
8868 tab->dtors[i].kfunc_btf_id = btf_relocate_id(btf, tab->dtors[i].kfunc_btf_id); in register_btf_id_dtor_kfuncs()
8871 tab->cnt += add_cnt; in register_btf_id_dtor_kfuncs()
8873 sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL); in register_btf_id_dtor_kfuncs()
8886 * type-based CO-RE relocations and follow slightly different rules than
8887 * field-based relocations. This function assumes that root types were already
8888 * checked for name match. Beyond that initial root-level name check, names
8890 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs/ENUM64s are considered compatible, but
8893 * - for ENUMs/ENUM64s, the size is ignored;
8894 * - for INT, size and signedness are ignored;
8895 * - for ARRAY, dimensionality is ignored, element types are checked for
8897 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
8898 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
8899 * - FUNC_PROTOs are compatible if they have compatible signature: same
8902 * more experience with using BPF CO-RE relocations.
8922 /* check X___Y name pattern, where X and Y are not underscores */ in bpf_core_is_flavor_sep()
8928 size_t bpf_core_essential_name_len(const char *name) in bpf_core_essential_name_len() argument
8930 size_t n = strlen(name); in bpf_core_essential_name_len()
8933 for (i = n - 5; i >= 0; i--) { in bpf_core_essential_name_len()
8934 if (bpf_core_is_flavor_sep(name + i)) in bpf_core_essential_name_len()
8942 if (!cands->cnt) in bpf_free_cands()
8950 kfree(cands->name); in bpf_free_cands_from_cache()
8971 bpf_log(log, "[%d]%s(", i, cc->name); in __print_cand_cache()
8972 for (j = 0; j < cc->cnt; j++) { in __print_cand_cache()
8973 bpf_log(log, "%d", cc->cands[j].id); in __print_cand_cache()
8974 if (j < cc->cnt - 1) in __print_cand_cache()
8994 return jhash(cands->name, cands->name_len, 0); in hash_cands()
9003 if (cc && cc->name_len == cands->name_len && in check_cand_cache()
9004 !strncmp(cc->name, cands->name, cands->name_len)) in check_cand_cache()
9009 static size_t sizeof_cands(int cnt) in sizeof_cands() argument
9011 return offsetof(struct bpf_cand_cache, cands[cnt]); in sizeof_cands()
9024 new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL); in populate_cand_cache()
9027 return ERR_PTR(-ENOMEM); in populate_cand_cache()
9029 /* strdup the name, since it will stay in cache. in populate_cand_cache()
9030 * the cands->name points to strings in prog's BTF and the prog can be unloaded. in populate_cand_cache()
9032 new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL); in populate_cand_cache()
9034 if (!new_cands->name) { in populate_cand_cache()
9036 return ERR_PTR(-ENOMEM); in populate_cand_cache()
9055 * since new module might have candidates with the name in __purge_cand_cache()
9065 for (j = 0; j < cc->cnt; j++) in __purge_cand_cache()
9066 if (cc->cands[j].btf == btf) { in __purge_cand_cache()
9096 if (btf_kind(t) != cands->kind) in bpf_core_add_cands()
9099 targ_name = btf_name_by_offset(targ_btf, t->name_off); in bpf_core_add_cands()
9104 * for non-existing name will have a chance to schedule(). in bpf_core_add_cands()
9108 if (strncmp(cands->name, targ_name, cands->name_len) != 0) in bpf_core_add_cands()
9112 if (targ_essent_len != cands->name_len) in bpf_core_add_cands()
9115 /* most of the time there is only one candidate for a given kind+name pair */ in bpf_core_add_cands()
9116 new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL); in bpf_core_add_cands()
9119 return ERR_PTR(-ENOMEM); in bpf_core_add_cands()
9122 memcpy(new_cands, cands, sizeof_cands(cands->cnt)); in bpf_core_add_cands()
9125 cands->cands[cands->cnt].btf = targ_btf; in bpf_core_add_cands()
9126 cands->cands[cands->cnt].id = i; in bpf_core_add_cands()
9127 cands->cnt++; in bpf_core_add_cands()
9136 const struct btf *local_btf = ctx->btf; in bpf_core_find_cands()
9141 const char *name; in bpf_core_find_cands() local
9148 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
9152 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
9154 name = btf_name_by_offset(local_btf, local_type->name_off); in bpf_core_find_cands()
9155 if (str_is_empty(name)) in bpf_core_find_cands()
9156 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
9157 local_essent_len = bpf_core_essential_name_len(name); in bpf_core_find_cands()
9160 cands->name = name; in bpf_core_find_cands()
9161 cands->kind = btf_kind(local_type); in bpf_core_find_cands()
9162 cands->name_len = local_essent_len; in bpf_core_find_cands()
9167 if (cc->cnt) in bpf_core_find_cands()
9177 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 */ in bpf_core_find_cands()
9179 /* populate cache even when cands->cnt == 0 */ in bpf_core_find_cands()
9185 if (cc->cnt) in bpf_core_find_cands()
9189 /* cands is a pointer to stack here and cands->cnt == 0 */ in bpf_core_find_cands()
9192 /* if cache has it return it even if cc->cnt == 0 */ in bpf_core_find_cands()
9212 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 in bpf_core_find_cands()
9213 * or pointer to stack if cands->cnd == 0. in bpf_core_find_cands()
9214 * Copy it into the cache even when cands->cnt == 0 and in bpf_core_find_cands()
9223 bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL; in bpf_core_apply()
9235 return -ENOMEM; in bpf_core_apply()
9237 type = btf_type_by_id(ctx->btf, relo->type_id); in bpf_core_apply()
9239 bpf_log(ctx->log, "relo #%u: bad type id %u\n", in bpf_core_apply()
9240 relo_idx, relo->type_id); in bpf_core_apply()
9242 return -EINVAL; in bpf_core_apply()
9250 cc = bpf_core_find_cands(ctx, relo->type_id); in bpf_core_apply()
9252 bpf_log(ctx->log, "target candidate search failed for %d\n", in bpf_core_apply()
9253 relo->type_id); in bpf_core_apply()
9257 if (cc->cnt) { in bpf_core_apply()
9258 cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL); in bpf_core_apply()
9260 err = -ENOMEM; in bpf_core_apply()
9264 for (i = 0; i < cc->cnt; i++) { in bpf_core_apply()
9265 bpf_log(ctx->log, in bpf_core_apply()
9266 "CO-RE relocating %s %s: found target candidate [%d]\n", in bpf_core_apply()
9267 btf_kind_str[cc->kind], cc->name, cc->cands[i].id); in bpf_core_apply()
9268 cands.cands[i].btf = cc->cands[i].btf; in bpf_core_apply()
9269 cands.cands[i].id = cc->cands[i].id; in bpf_core_apply()
9271 cands.len = cc->cnt; in bpf_core_apply()
9279 err = bpf_core_calc_relo_insn((void *)ctx->log, relo, relo_idx, ctx->btf, &cands, specs, in bpf_core_apply()
9284 err = bpf_core_patch_insn((void *)ctx->log, insn, relo->insn_off / 8, relo, relo_idx, in bpf_core_apply()
9292 if (ctx->log->level & BPF_LOG_LEVEL2) in bpf_core_apply()
9293 print_cand_cache(ctx->log); in bpf_core_apply()
9302 struct btf *btf = reg->btf; in btf_nested_type_is_trusted()
9310 walk_type = btf_type_by_id(btf, reg->btf_id); in btf_nested_type_is_trusted()
9314 tname = btf_name_by_offset(btf, walk_type->name_off); in btf_nested_type_is_trusted()
9320 safe_id = btf_find_by_name_kind(btf, safe_tname, BTF_INFO_KIND(walk_type->info)); in btf_nested_type_is_trusted()
9329 const char *m_name = __btf_name_by_offset(btf, member->name_off); in btf_nested_type_is_trusted()
9330 const struct btf_type *mtype = btf_type_by_id(btf, member->type); in btf_nested_type_is_trusted()
9336 btf_type_skip_modifiers(btf, mtype->type, &id); in btf_nested_type_is_trusted()
9337 /* If we match on both type and name, the field is considered trusted. */ in btf_nested_type_is_trusted()
9352 size_t pattern_len = sizeof(NOCAST_ALIAS_SUFFIX) - sizeof(char); in btf_type_ids_nocast_alias()
9362 reg_name = btf_name_by_offset(reg_btf, reg_type->name_off); in btf_type_ids_nocast_alias()
9363 arg_name = btf_name_by_offset(arg_btf, arg_type->name_off); in btf_type_ids_nocast_alias()
9369 * if the strings are the same size, they can't possibly be no-cast in btf_type_ids_nocast_alias()
9372 * because they are _not_ no-cast aliases, they are the same type. in btf_type_ids_nocast_alias()
9377 /* Either of the two names must be the other name, suffixed with ___init. */ in btf_type_ids_nocast_alias()
9393 /* ___init suffix must come at the end of the name */ in btf_type_ids_nocast_alias()
9408 tab = btf->struct_ops_tab; in btf_add_struct_ops()
9413 return -ENOMEM; in btf_add_struct_ops()
9414 tab->capacity = 4; in btf_add_struct_ops()
9415 btf->struct_ops_tab = tab; in btf_add_struct_ops()
9418 for (i = 0; i < tab->cnt; i++) in btf_add_struct_ops()
9419 if (tab->ops[i].st_ops == st_ops) in btf_add_struct_ops()
9420 return -EEXIST; in btf_add_struct_ops()
9422 if (tab->cnt == tab->capacity) { in btf_add_struct_ops()
9425 ops[tab->capacity * 2]), in btf_add_struct_ops()
9428 return -ENOMEM; in btf_add_struct_ops()
9430 tab->capacity *= 2; in btf_add_struct_ops()
9431 btf->struct_ops_tab = tab; in btf_add_struct_ops()
9434 tab->ops[btf->struct_ops_tab->cnt].st_ops = st_ops; in btf_add_struct_ops()
9436 err = bpf_struct_ops_desc_init(&tab->ops[btf->struct_ops_tab->cnt], btf, log); in btf_add_struct_ops()
9440 btf->struct_ops_tab->cnt++; in btf_add_struct_ops()
9450 u32 cnt; in bpf_struct_ops_find_value() local
9454 if (!btf->struct_ops_tab) in bpf_struct_ops_find_value()
9457 cnt = btf->struct_ops_tab->cnt; in bpf_struct_ops_find_value()
9458 st_ops_list = btf->struct_ops_tab->ops; in bpf_struct_ops_find_value()
9459 for (i = 0; i < cnt; i++) { in bpf_struct_ops_find_value()
9472 u32 cnt; in bpf_struct_ops_find() local
9476 if (!btf->struct_ops_tab) in bpf_struct_ops_find()
9479 cnt = btf->struct_ops_tab->cnt; in bpf_struct_ops_find()
9480 st_ops_list = btf->struct_ops_tab->ops; in bpf_struct_ops_find()
9481 for (i = 0; i < cnt; i++) { in bpf_struct_ops_find()
9495 btf = btf_get_module_btf(st_ops->owner); in __register_bpf_struct_ops()
9497 return check_btf_kconfigs(st_ops->owner, "struct_ops"); in __register_bpf_struct_ops()
9503 err = -ENOMEM; in __register_bpf_struct_ops()
9507 log->level = BPF_LOG_KERNEL; in __register_bpf_struct_ops()
9528 param_name = btf_name_by_offset(btf, arg->name_off); in btf_param_match_suffix()
9534 param_name += len - suffix_len; in btf_param_match_suffix()