Merge branch 'bpf-btf' into iproute2-next

Daniel Borkmann  says:

====================

Main part of this set is to: i) avoid strict af_alg kernel dependency,
ii) add loader support for bpf to bpf calls and iii) add btf loader
support with an option to annotate maps. For details please see the
individual patches. Thanks!

====================

Signed-off-by: David Ahern <dsahern@gmail.com>

include/bpf_elf.h

@@ -41,4 +41,13 @@ struct bpf_elf_map {
 	__u32 inner_idx;
 };
 
+#define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val)		\
+	struct ____btf_map_##name {				\
+		type_key key;					\
+		type_val value;					\
+	};							\
+	struct ____btf_map_##name				\
+	    __attribute__ ((section(".maps." #name), used))	\
+	    ____btf_map_##name = { }
+
 #endif /* __BPF_ELF__ */

include/bpf_util.h

@@ -14,6 +14,7 @@
 #define __BPF_UTIL__
 
 #include <linux/bpf.h>
+#include <linux/btf.h>
 #include <linux/filter.h>
 #include <linux/magic.h>
 #include <linux/elf-em.h>

lib/bpf.c

@@ -388,6 +388,8 @@ struct bpf_prog_data {
 
 struct bpf_map_ext {
 	struct bpf_prog_data owner;
+	unsigned int btf_id_key;
+	unsigned int btf_id_val;
 };
 
 static int bpf_derive_elf_map_from_fdinfo(int fd, struct bpf_elf_map *map,
@@ -1104,7 +1106,8 @@ int bpf_prog_load(enum bpf_prog_type type, const struct bpf_insn *insns,
 #ifdef HAVE_ELF
 struct bpf_elf_prog {
 	enum bpf_prog_type	type;
-	const struct bpf_insn	*insns;
+	struct bpf_insn		*insns;
+	unsigned int		insns_num;
 	size_t			size;
 	const char		*license;
 };
@@ -1119,25 +1122,41 @@ struct bpf_config {
 	unsigned int		jit_enabled;
 };
 
+struct bpf_btf {
+	const struct btf_header	*hdr;
+	const void		*raw;
+	const char		*strings;
+	const struct btf_type	**types;
+	int			types_num;
+};
+
 struct bpf_elf_ctx {
 	struct bpf_config	cfg;
 	Elf			*elf_fd;
 	GElf_Ehdr		elf_hdr;
 	Elf_Data		*sym_tab;
 	Elf_Data		*str_tab;
+	Elf_Data		*btf_data;
+	char			obj_uid[64];
 	int			obj_fd;
+	int			btf_fd;
 	int			map_fds[ELF_MAX_MAPS];
 	struct bpf_elf_map	maps[ELF_MAX_MAPS];
 	struct bpf_map_ext	maps_ext[ELF_MAX_MAPS];
+	struct bpf_elf_prog	prog_text;
+	struct bpf_btf		btf;
 	int			sym_num;
 	int			map_num;
 	int			map_len;
 	bool			*sec_done;
 	int			sec_maps;
+	int			sec_text;
+	int			sec_btf;
 	char			license[ELF_MAX_LICENSE_LEN];
 	enum bpf_prog_type	type;
 	__u32			ifindex;
 	bool			verbose;
+	bool			noafalg;
 	struct bpf_elf_st	stat;
 	struct bpf_hash_entry	*ht[256];
 	char			*log;
@@ -1157,6 +1176,11 @@ struct bpf_map_data {
 	struct bpf_elf_map	*ent;
 };
 
+static bool bpf_log_has_data(struct bpf_elf_ctx *ctx)
+{
+	return ctx->log && ctx->log[0];
+}
+
 static __check_format_string(2, 3) void
 bpf_dump_error(struct bpf_elf_ctx *ctx, const char *format, ...)
 {
@@ -1166,7 +1190,7 @@ bpf_dump_error(struct bpf_elf_ctx *ctx, const char *format, ...)
 	vfprintf(stderr, format, vl);
 	va_end(vl);
 
-	if (ctx->log && ctx->log[0]) {
+	if (bpf_log_has_data(ctx)) {
 		if (ctx->verbose) {
 			fprintf(stderr, "%s\n", ctx->log);
 		} else {
@@ -1213,7 +1237,9 @@ static int bpf_log_realloc(struct bpf_elf_ctx *ctx)
 
 static int bpf_map_create(enum bpf_map_type type, uint32_t size_key,
 			  uint32_t size_value, uint32_t max_elem,
-			  uint32_t flags, int inner_fd, uint32_t ifindex)
+			  uint32_t flags, int inner_fd, int btf_fd,
+			  uint32_t ifindex, uint32_t btf_id_key,
+			  uint32_t btf_id_val)
 {
 	union bpf_attr attr = {};
 
@@ -1224,10 +1250,30 @@ static int bpf_map_create(enum bpf_map_type type, uint32_t size_key,
 	attr.map_flags = flags;
 	attr.inner_map_fd = inner_fd;
 	attr.map_ifindex = ifindex;
+	attr.btf_fd = btf_fd;
+	attr.btf_key_type_id = btf_id_key;
+	attr.btf_value_type_id = btf_id_val;
 
 	return bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
 }
 
+static int bpf_btf_load(void *btf, size_t size_btf,
+			char *log, size_t size_log)
+{
+	union bpf_attr attr = {};
+
+	attr.btf = bpf_ptr_to_u64(btf);
+	attr.btf_size = size_btf;
+
+	if (size_log > 0) {
+		attr.btf_log_buf = bpf_ptr_to_u64(log);
+		attr.btf_log_size = size_log;
+		attr.btf_log_level = 1;
+	}
+
+	return bpf(BPF_BTF_LOAD, &attr, sizeof(attr));
+}
+
 static int bpf_obj_pin(int fd, const char *pathname)
 {
 	union bpf_attr attr = {};
@@ -1253,22 +1299,15 @@ static int bpf_obj_hash(const char *object, uint8_t *out, size_t len)
 		return -EINVAL;
 
 	cfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
-	if (cfd < 0) {
-		fprintf(stderr, "Cannot get AF_ALG socket: %s\n",
-			strerror(errno));
+	if (cfd < 0)
 		return cfd;
-	}
 
 	ret = bind(cfd, (struct sockaddr *)&alg, sizeof(alg));
-	if (ret < 0) {
-		fprintf(stderr, "Error binding socket: %s\n", strerror(errno));
+	if (ret < 0)
 		goto out_cfd;
-	}
 
 	ofd = accept(cfd, NULL, 0);
 	if (ofd < 0) {
-		fprintf(stderr, "Error accepting socket: %s\n",
-			strerror(errno));
 		ret = ofd;
 		goto out_cfd;
 	}
@@ -1313,29 +1352,7 @@ out_cfd:
 	return ret;
 }
 
-static const char *bpf_get_obj_uid(const char *pathname)
-{
-	static bool bpf_uid_cached;
-	static char bpf_uid[64];
-	uint8_t tmp[20];
-	int ret;
-
-	if (bpf_uid_cached)
-		goto done;
-
-	ret = bpf_obj_hash(pathname, tmp, sizeof(tmp));
-	if (ret) {
-		fprintf(stderr, "Object hashing failed!\n");
-		return NULL;
-	}
-
-	hexstring_n2a(tmp, sizeof(tmp), bpf_uid, sizeof(bpf_uid));
-	bpf_uid_cached = true;
-done:
-	return bpf_uid;
-}
-
-static int bpf_init_env(const char *pathname)
+static void bpf_init_env(void)
 {
 	struct rlimit limit = {
 		.rlim_cur = RLIM_INFINITY,
@@ -1345,15 +1362,8 @@ static int bpf_init_env(const char *pathname)
 	/* Don't bother in case we fail! */
 	setrlimit(RLIMIT_MEMLOCK, &limit);
 
-	if (!bpf_get_work_dir(BPF_PROG_TYPE_UNSPEC)) {
+	if (!bpf_get_work_dir(BPF_PROG_TYPE_UNSPEC))
 		fprintf(stderr, "Continuing without mounted eBPF fs. Too old kernel?\n");
-		return 0;
-	}
-
-	if (!bpf_get_obj_uid(pathname))
-		return -1;
-
-	return 0;
 }
 
 static const char *bpf_custom_pinning(const struct bpf_elf_ctx *ctx,
@@ -1389,7 +1399,7 @@ static void bpf_make_pathname(char *pathname, size_t len, const char *name,
 	case PIN_OBJECT_NS:
 		snprintf(pathname, len, "%s/%s/%s",
 			 bpf_get_work_dir(ctx->type),
-			 bpf_get_obj_uid(NULL), name);
+			 ctx->obj_uid, name);
 		break;
 	case PIN_GLOBAL_NS:
 		snprintf(pathname, len, "%s/%s/%s",
@@ -1422,7 +1432,7 @@ static int bpf_make_obj_path(const struct bpf_elf_ctx *ctx)
 	int ret;
 
 	snprintf(tmp, sizeof(tmp), "%s/%s", bpf_get_work_dir(ctx->type),
-		 bpf_get_obj_uid(NULL));
+		 ctx->obj_uid);
 
 	ret = mkdir(tmp, S_IRWXU);
 	if (ret && errno != EEXIST) {
@@ -1639,7 +1649,8 @@ static int bpf_map_attach(const char *name, struct bpf_elf_ctx *ctx,
 	ifindex = bpf_map_offload_neutral(map->type) ? 0 : ctx->ifindex;
 	errno = 0;
 	fd = bpf_map_create(map->type, map->size_key, map->size_value,
-			    map->max_elem, map->flags, map_inner_fd, ifindex);
+			    map->max_elem, map->flags, map_inner_fd, ctx->btf_fd,
+			    ifindex, ext->btf_id_key, ext->btf_id_val);
 
 	if (fd < 0 || ctx->verbose) {
 		bpf_map_report(fd, name, map, ctx, map_inner_fd);
@@ -1664,8 +1675,80 @@ static const char *bpf_str_tab_name(const struct bpf_elf_ctx *ctx,
 	return ctx->str_tab->d_buf + sym->st_name;
 }
 
+static int bpf_btf_find(struct bpf_elf_ctx *ctx, const char *name)
+{
+	const struct btf_type *type;
+	const char *res;
+	int id;
+
+	for (id = 1; id < ctx->btf.types_num; id++) {
+		type = ctx->btf.types[id];
+		if (type->name_off >= ctx->btf.hdr->str_len)
+			continue;
+		res = &ctx->btf.strings[type->name_off];
+		if (!strcmp(res, name))
+			return id;
+	}
+
+	return -ENOENT;
+}
+
+static int bpf_btf_find_kv(struct bpf_elf_ctx *ctx, const struct bpf_elf_map *map,
+			   const char *name, uint32_t *id_key, uint32_t *id_val)
+{
+	const struct btf_member *key, *val;
+	const struct btf_type *type;
+	char btf_name[512];
+	const char *res;
+	int id;
+
+	snprintf(btf_name, sizeof(btf_name), "____btf_map_%s", name);
+	id = bpf_btf_find(ctx, btf_name);
+	if (id < 0)
+		return id;
+
+	type = ctx->btf.types[id];
+	if (BTF_INFO_KIND(type->info) != BTF_KIND_STRUCT)
+		return -EINVAL;
+	if (BTF_INFO_VLEN(type->info) != 2)
+		return -EINVAL;
+
+	key = ((void *) type) + sizeof(*type);
+	val = key + 1;
+	if (!key->type || key->type >= ctx->btf.types_num ||
+	    !val->type || val->type >= ctx->btf.types_num)
+		return -EINVAL;
+
+	if (key->name_off >= ctx->btf.hdr->str_len ||
+	    val->name_off >= ctx->btf.hdr->str_len)
+		return -EINVAL;
+
+	res = &ctx->btf.strings[key->name_off];
+	if (strcmp(res, "key"))
+		return -EINVAL;
+
+	res = &ctx->btf.strings[val->name_off];
+	if (strcmp(res, "value"))
+		return -EINVAL;
+
+	*id_key = key->type;
+	*id_val = val->type;
+	return 0;
+}
+
+static void bpf_btf_annotate(struct bpf_elf_ctx *ctx, int which, const char *name)
+{
+	uint32_t id_key = 0, id_val = 0;
+
+	if (!bpf_btf_find_kv(ctx, &ctx->maps[which], name, &id_key, &id_val)) {
+		ctx->maps_ext[which].btf_id_key = id_key;
+		ctx->maps_ext[which].btf_id_val = id_val;
+	}
+}
+
 static const char *bpf_map_fetch_name(struct bpf_elf_ctx *ctx, int which)
 {
+	const char *name;
 	GElf_Sym sym;
 	int i;
 
@@ -1679,7 +1762,9 @@ static const char *bpf_map_fetch_name(struct bpf_elf_ctx *ctx, int which)
 		    sym.st_value / ctx->map_len != which)
 			continue;
 
-		return bpf_str_tab_name(ctx, &sym);
+		name = bpf_str_tab_name(ctx, &sym);
+		bpf_btf_annotate(ctx, which, name);
+		return name;
 	}
 
 	return NULL;
@@ -1691,6 +1776,12 @@ static int bpf_maps_attach_all(struct bpf_elf_ctx *ctx)
 	const char *map_name;
 
 	for (i = 0; i < ctx->map_num; i++) {
+		if (ctx->maps[i].pinning == PIN_OBJECT_NS &&
+		    ctx->noafalg) {
+			fprintf(stderr, "Missing kernel AF_ALG support for PIN_OBJECT_NS!\n");
+			return -ENOTSUP;
+		}
+
 		map_name = bpf_map_fetch_name(ctx, i);
 		if (!map_name)
 			return -EIO;
@@ -1893,9 +1984,9 @@ static int bpf_fetch_maps_end(struct bpf_elf_ctx *ctx)
 	}
 
 	memcpy(ctx->maps, fixup, sizeof(fixup));
-
-	printf("Note: %zu bytes struct bpf_elf_map fixup performed due to size mismatch!\n",
-	       sizeof(struct bpf_elf_map) - ctx->map_len);
+	if (ctx->verbose)
+		printf("%zu bytes struct bpf_elf_map fixup performed due to size mismatch!\n",
+		       sizeof(struct bpf_elf_map) - ctx->map_len);
 	return 0;
 }
 
@@ -1927,12 +2018,224 @@ static int bpf_fetch_strtab(struct bpf_elf_ctx *ctx, int section,
 	return 0;
 }
 
+static int bpf_fetch_text(struct bpf_elf_ctx *ctx, int section,
+			  struct bpf_elf_sec_data *data)
+{
+	ctx->sec_text = section;
+	ctx->sec_done[section] = true;
+	return 0;
+}
+
+static void bpf_btf_report(int fd, struct bpf_elf_ctx *ctx)
+{
+	fprintf(stderr, "\nBTF debug data section \'.BTF\' %s%s (%d)!\n",
+		fd < 0 ? "rejected: " : "loaded",
+		fd < 0 ? strerror(errno) : "",
+		fd < 0 ? errno : fd);
+
+	fprintf(stderr, " - Length:       %zu\n", ctx->btf_data->d_size);
+
+	bpf_dump_error(ctx, "Verifier analysis:\n\n");
+}
+
+static int bpf_btf_attach(struct bpf_elf_ctx *ctx)
+{
+	int tries = 0, fd;
+retry:
+	errno = 0;
+	fd = bpf_btf_load(ctx->btf_data->d_buf, ctx->btf_data->d_size,
+			  ctx->log, ctx->log_size);
+	if (fd < 0 || ctx->verbose) {
+		if (fd < 0 && (errno == ENOSPC || !ctx->log_size)) {
+			if (tries++ < 10 && !bpf_log_realloc(ctx))
+				goto retry;
+
+			fprintf(stderr, "Log buffer too small to dump verifier log %zu bytes (%d tries)!\n",
+				ctx->log_size, tries);
+			return fd;
+		}
+
+		if (bpf_log_has_data(ctx))
+			bpf_btf_report(fd, ctx);
+	}
+
+	return fd;
+}
+
+static int bpf_fetch_btf_begin(struct bpf_elf_ctx *ctx, int section,
+			       struct bpf_elf_sec_data *data)
+{
+	ctx->btf_data = data->sec_data;
+	ctx->sec_btf = section;
+	ctx->sec_done[section] = true;
+	return 0;
+}
+
+static int bpf_btf_check_header(struct bpf_elf_ctx *ctx)
+{
+	const struct btf_header *hdr = ctx->btf_data->d_buf;
+	const char *str_start, *str_end;
+	unsigned int data_len;
+
+	if (hdr->magic != BTF_MAGIC) {
+		fprintf(stderr, "Object has wrong BTF magic: %x, expected: %x!\n",
+			hdr->magic, BTF_MAGIC);
+		return -EINVAL;
+	}
+
+	if (hdr->version != BTF_VERSION) {
+		fprintf(stderr, "Object has wrong BTF version: %u, expected: %u!\n",
+			hdr->version, BTF_VERSION);
+		return -EINVAL;
+	}
+
+	if (hdr->flags) {
+		fprintf(stderr, "Object has unsupported BTF flags %x!\n",
+			hdr->flags);
+		return -EINVAL;
+	}
+
+	data_len = ctx->btf_data->d_size - sizeof(*hdr);
+	if (data_len < hdr->type_off ||
+	    data_len < hdr->str_off ||
+	    data_len < hdr->type_len + hdr->str_len ||
+	    hdr->type_off >= hdr->str_off ||
+	    hdr->type_off + hdr->type_len != hdr->str_off ||
+	    hdr->str_off + hdr->str_len != data_len ||
+	    (hdr->type_off & (sizeof(uint32_t) - 1))) {
+		fprintf(stderr, "Object has malformed BTF data!\n");
+		return -EINVAL;
+	}
+
+	ctx->btf.hdr = hdr;
+	ctx->btf.raw = hdr + 1;
+
+	str_start = ctx->btf.raw + hdr->str_off;
+	str_end = str_start + hdr->str_len;
+	if (!hdr->str_len ||
+	    hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
+	    str_start[0] || str_end[-1]) {
+		fprintf(stderr, "Object has malformed BTF string data!\n");
+		return -EINVAL;
+	}
+
+	ctx->btf.strings = str_start;
+	return 0;
+}
+
+static int bpf_btf_register_type(struct bpf_elf_ctx *ctx,
+				 const struct btf_type *type)
+{
+	int cur = ctx->btf.types_num, num = cur + 1;
+	const struct btf_type **types;
+
+	types = realloc(ctx->btf.types, num * sizeof(type));
+	if (!types) {
+		free(ctx->btf.types);
+		ctx->btf.types = NULL;
+		ctx->btf.types_num = 0;
+		return -ENOMEM;
+	}
+
+	ctx->btf.types = types;
+	ctx->btf.types[cur] = type;
+	ctx->btf.types_num = num;
+	return 0;
+}
+
+static struct btf_type btf_type_void;
+
+static int bpf_btf_prep_type_data(struct bpf_elf_ctx *ctx)
+{
+	const void *type_cur = ctx->btf.raw + ctx->btf.hdr->type_off;
+	const void *type_end = ctx->btf.raw + ctx->btf.hdr->str_off;
+	const struct btf_type *type;
+	uint16_t var_len;
+	int ret, kind;
+
+	ret = bpf_btf_register_type(ctx, &btf_type_void);
+	if (ret < 0)
+		return ret;
+
+	while (type_cur < type_end) {
+		type = type_cur;
+		type_cur += sizeof(*type);
+
+		var_len = BTF_INFO_VLEN(type->info);
+		kind = BTF_INFO_KIND(type->info);
+
+		switch (kind) {
+		case BTF_KIND_INT:
+			type_cur += sizeof(int);
+			break;
+		case BTF_KIND_ARRAY:
+			type_cur += sizeof(struct btf_array);
+			break;
+		case BTF_KIND_STRUCT:
+		case BTF_KIND_UNION:
+			type_cur += var_len * sizeof(struct btf_member);
+			break;
+		case BTF_KIND_ENUM:
+			type_cur += var_len * sizeof(struct btf_enum);
+			break;
+		case BTF_KIND_TYPEDEF:
+		case BTF_KIND_PTR:
+		case BTF_KIND_FWD:
+		case BTF_KIND_VOLATILE:
+		case BTF_KIND_CONST:
+		case BTF_KIND_RESTRICT:
+			break;
+		default:
+			fprintf(stderr, "Object has unknown BTF type: %u!\n", kind);
+			return -EINVAL;
+		}
+
+		ret = bpf_btf_register_type(ctx, type);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int bpf_btf_prep_data(struct bpf_elf_ctx *ctx)
+{
+	int ret = bpf_btf_check_header(ctx);
+
+	if (!ret)
+		return bpf_btf_prep_type_data(ctx);
+	return ret;
+}
+
+static void bpf_fetch_btf_end(struct bpf_elf_ctx *ctx)
+{
+	int fd = bpf_btf_attach(ctx);
+
+	if (fd < 0)
+		return;
+	ctx->btf_fd = fd;
+	if (bpf_btf_prep_data(ctx) < 0) {
+		close(ctx->btf_fd);
+		ctx->btf_fd = 0;
+	}
+}
+
 static bool bpf_has_map_data(const struct bpf_elf_ctx *ctx)
 {
 	return ctx->sym_tab && ctx->str_tab && ctx->sec_maps;
 }
 
-static int bpf_fetch_ancillary(struct bpf_elf_ctx *ctx)
+static bool bpf_has_btf_data(const struct bpf_elf_ctx *ctx)
+{
+	return ctx->sec_btf;
+}
+
+static bool bpf_has_call_data(const struct bpf_elf_ctx *ctx)
+{
+	return ctx->sec_text;
+}
+
+static int bpf_fetch_ancillary(struct bpf_elf_ctx *ctx, bool check_text_sec)
 {
 	struct bpf_elf_sec_data data;
 	int i, ret = -1;
@@ -1948,12 +2251,20 @@ static int bpf_fetch_ancillary(struct bpf_elf_ctx *ctx)
 		else if (data.sec_hdr.sh_type == SHT_PROGBITS &&
 			 !strcmp(data.sec_name, ELF_SECTION_LICENSE))
 			ret = bpf_fetch_license(ctx, i, &data);
+		else if (data.sec_hdr.sh_type == SHT_PROGBITS &&
+			 (data.sec_hdr.sh_flags & SHF_EXECINSTR) &&
+			 !strcmp(data.sec_name, ".text") &&
+			 check_text_sec)
+			ret = bpf_fetch_text(ctx, i, &data);
 		else if (data.sec_hdr.sh_type == SHT_SYMTAB &&
 			 !strcmp(data.sec_name, ".symtab"))
 			ret = bpf_fetch_symtab(ctx, i, &data);
 		else if (data.sec_hdr.sh_type == SHT_STRTAB &&
 			 !strcmp(data.sec_name, ".strtab"))
 			ret = bpf_fetch_strtab(ctx, i, &data);
+		else if (data.sec_hdr.sh_type == SHT_PROGBITS &&
+			 !strcmp(data.sec_name, ".BTF"))
+			ret = bpf_fetch_btf_begin(ctx, i, &data);
 		if (ret < 0) {
 			fprintf(stderr, "Error parsing section %d! Perhaps check with readelf -a?\n",
 				i);
@@ -1961,6 +2272,8 @@ static int bpf_fetch_ancillary(struct bpf_elf_ctx *ctx)
 		}
 	}
 
+	if (bpf_has_btf_data(ctx))
+		bpf_fetch_btf_end(ctx);
 	if (bpf_has_map_data(ctx)) {
 		ret = bpf_fetch_maps_end(ctx);
 		if (ret < 0) {
@@ -1992,17 +2305,18 @@ static int bpf_fetch_prog(struct bpf_elf_ctx *ctx, const char *section,
 		ret = bpf_fill_section_data(ctx, i, &data);
 		if (ret < 0 ||
 		    !(data.sec_hdr.sh_type == SHT_PROGBITS &&
-		      data.sec_hdr.sh_flags & SHF_EXECINSTR &&
+		      (data.sec_hdr.sh_flags & SHF_EXECINSTR) &&
 		      !strcmp(data.sec_name, section)))
 			continue;
 
 		*sseen = true;
 
 		memset(&prog, 0, sizeof(prog));
-		prog.type    = ctx->type;
-		prog.insns   = data.sec_data->d_buf;
-		prog.size    = data.sec_data->d_size;
-		prog.license = ctx->license;
+		prog.type      = ctx->type;
+		prog.license   = ctx->license;
+		prog.size      = data.sec_data->d_size;
+		prog.insns_num = prog.size / sizeof(struct bpf_insn);
+		prog.insns     = data.sec_data->d_buf;
 
 		fd = bpf_prog_attach(section, &prog, ctx);
 		if (fd < 0)
@@ -2015,84 +2329,120 @@ static int bpf_fetch_prog(struct bpf_elf_ctx *ctx, const char *section,
 	return fd;
 }
 
-struct bpf_tail_call_props {
-	unsigned int total;
-	unsigned int jited;
+struct bpf_relo_props {
+	struct bpf_tail_call {
+		unsigned int total;
+		unsigned int jited;
+	} tc;
+	int main_num;
 };
 
+static int bpf_apply_relo_map(struct bpf_elf_ctx *ctx, struct bpf_elf_prog *prog,
+			      GElf_Rel *relo, GElf_Sym *sym,
+			      struct bpf_relo_props *props)
+{
+	unsigned int insn_off = relo->r_offset / sizeof(struct bpf_insn);
+	unsigned int map_idx = sym->st_value / ctx->map_len;
+
+	if (insn_off >= prog->insns_num)
+		return -EINVAL;
+	if (prog->insns[insn_off].code != (BPF_LD | BPF_IMM | BPF_DW)) {
+		fprintf(stderr, "ELF contains relo data for non ld64 instruction at offset %u! Compiler bug?!\n",
+			insn_off);
+		return -EINVAL;
+	}
+
+	if (map_idx >= ARRAY_SIZE(ctx->map_fds))
+		return -EINVAL;
+	if (!ctx->map_fds[map_idx])
+		return -EINVAL;
+	if (ctx->maps[map_idx].type == BPF_MAP_TYPE_PROG_ARRAY) {
+		props->tc.total++;
+		if (ctx->maps_ext[map_idx].owner.jited ||
+		    (ctx->maps_ext[map_idx].owner.type == 0 &&
+		     ctx->cfg.jit_enabled))
+			props->tc.jited++;
+	}
+
+	prog->insns[insn_off].src_reg = BPF_PSEUDO_MAP_FD;
+	prog->insns[insn_off].imm = ctx->map_fds[map_idx];
+	return 0;
+}
+
+static int bpf_apply_relo_call(struct bpf_elf_ctx *ctx, struct bpf_elf_prog *prog,
+			       GElf_Rel *relo, GElf_Sym *sym,
+			       struct bpf_relo_props *props)
+{
+	unsigned int insn_off = relo->r_offset / sizeof(struct bpf_insn);
+	struct bpf_elf_prog *prog_text = &ctx->prog_text;
+
+	if (insn_off >= prog->insns_num)
+		return -EINVAL;
+	if (prog->insns[insn_off].code != (BPF_JMP | BPF_CALL) &&
+	    prog->insns[insn_off].src_reg != BPF_PSEUDO_CALL) {
+		fprintf(stderr, "ELF contains relo data for non call instruction at offset %u! Compiler bug?!\n",
+			insn_off);
+		return -EINVAL;
+	}
+
+	if (!props->main_num) {
+		struct bpf_insn *insns = realloc(prog->insns,
+						 prog->size + prog_text->size);
+		if (!insns)
+			return -ENOMEM;
+
+		memcpy(insns + prog->insns_num, prog_text->insns,
+		       prog_text->size);
+		props->main_num = prog->insns_num;
+		prog->insns = insns;
+		prog->insns_num += prog_text->insns_num;
+		prog->size += prog_text->size;
+	}
+
+	prog->insns[insn_off].imm += props->main_num - insn_off;
+	return 0;
+}
+
 static int bpf_apply_relo_data(struct bpf_elf_ctx *ctx,
 			       struct bpf_elf_sec_data *data_relo,
-			       struct bpf_elf_sec_data *data_insn,
-			       struct bpf_tail_call_props *props)
+			       struct bpf_elf_prog *prog,
+			       struct bpf_relo_props *props)
 {
-	Elf_Data *idata = data_insn->sec_data;
 	GElf_Shdr *rhdr = &data_relo->sec_hdr;
 	int relo_ent, relo_num = rhdr->sh_size / rhdr->sh_entsize;
-	struct bpf_insn *insns = idata->d_buf;
-	unsigned int num_insns = idata->d_size / sizeof(*insns);
 
 	for (relo_ent = 0; relo_ent < relo_num; relo_ent++) {
-		unsigned int ioff, rmap;
 		GElf_Rel relo;
 		GElf_Sym sym;
+		int ret = -EIO;
 
 		if (gelf_getrel(data_relo->sec_data, relo_ent, &relo) != &relo)
 			return -EIO;
-
-		ioff = relo.r_offset / sizeof(struct bpf_insn);
-		if (ioff >= num_insns ||
-		    insns[ioff].code != (BPF_LD | BPF_IMM | BPF_DW)) {
-			fprintf(stderr, "ELF contains relo data for non ld64 instruction at offset %u! Compiler bug?!\n",
-				ioff);
-			fprintf(stderr, " - Current section: %s\n", data_relo->sec_name);
-			if (ioff < num_insns &&
-			    insns[ioff].code == (BPF_JMP | BPF_CALL))
-				fprintf(stderr, " - Try to annotate functions with always_inline attribute!\n");
-			return -EINVAL;
-		}
-
 		if (gelf_getsym(ctx->sym_tab, GELF_R_SYM(relo.r_info), &sym) != &sym)
 			return -EIO;
-		if (sym.st_shndx != ctx->sec_maps) {
-			fprintf(stderr, "ELF contains non-map related relo data in entry %u pointing to section %u! Compiler bug?!\n",
+
+		if (sym.st_shndx == ctx->sec_maps)
+			ret = bpf_apply_relo_map(ctx, prog, &relo, &sym, props);
+		else if (sym.st_shndx == ctx->sec_text)
+			ret = bpf_apply_relo_call(ctx, prog, &relo, &sym, props);
+		else
+			fprintf(stderr, "ELF contains non-{map,call} related relo data in entry %u pointing to section %u! Compiler bug?!\n",
 				relo_ent, sym.st_shndx);
-			return -EIO;
-		}
-
-		rmap = sym.st_value / ctx->map_len;
-		if (rmap >= ARRAY_SIZE(ctx->map_fds))
-			return -EINVAL;
-		if (!ctx->map_fds[rmap])
-			return -EINVAL;
-		if (ctx->maps[rmap].type == BPF_MAP_TYPE_PROG_ARRAY) {
-			props->total++;
-			if (ctx->maps_ext[rmap].owner.jited ||
-			    (ctx->maps_ext[rmap].owner.type == 0 &&
-			     ctx->cfg.jit_enabled))
-				props->jited++;
-		}
-
-		if (ctx->verbose)
-			fprintf(stderr, "Map \'%s\' (%d) injected into prog section \'%s\' at offset %u!\n",
-				bpf_str_tab_name(ctx, &sym), ctx->map_fds[rmap],
-				data_insn->sec_name, ioff);
-
-		insns[ioff].src_reg = BPF_PSEUDO_MAP_FD;
-		insns[ioff].imm     = ctx->map_fds[rmap];
+		if (ret < 0)
+			return ret;
 	}
 
 	return 0;
 }
 
 static int bpf_fetch_prog_relo(struct bpf_elf_ctx *ctx, const char *section,
-			       bool *lderr, bool *sseen)
+			       bool *lderr, bool *sseen, struct bpf_elf_prog *prog)
 {
 	struct bpf_elf_sec_data data_relo, data_insn;
-	struct bpf_elf_prog prog;
 	int ret, idx, i, fd = -1;
 
 	for (i = 1; i < ctx->elf_hdr.e_shnum; i++) {
-		struct bpf_tail_call_props props = {};
+		struct bpf_relo_props props = {};
 
 		ret = bpf_fill_section_data(ctx, i, &data_relo);
 		if (ret < 0 || data_relo.sec_hdr.sh_type != SHT_REL)
@@ -2103,40 +2453,54 @@ static int bpf_fetch_prog_relo(struct bpf_elf_ctx *ctx, const char *section,
 		ret = bpf_fill_section_data(ctx, idx, &data_insn);
 		if (ret < 0 ||
 		    !(data_insn.sec_hdr.sh_type == SHT_PROGBITS &&
-		      data_insn.sec_hdr.sh_flags & SHF_EXECINSTR &&
+		      (data_insn.sec_hdr.sh_flags & SHF_EXECINSTR) &&
 		      !strcmp(data_insn.sec_name, section)))
 			continue;
+		if (sseen)
+			*sseen = true;
 
-		*sseen = true;
-
-		ret = bpf_apply_relo_data(ctx, &data_relo, &data_insn, &props);
-		if (ret < 0) {
+		memset(prog, 0, sizeof(*prog));
+		prog->type = ctx->type;
+		prog->license = ctx->license;
+		prog->size = data_insn.sec_data->d_size;
+		prog->insns_num = prog->size / sizeof(struct bpf_insn);
+		prog->insns = malloc(prog->size);
+		if (!prog->insns) {
 			*lderr = true;
-			return ret;
+			return -ENOMEM;
 		}
 
-		memset(&prog, 0, sizeof(prog));
-		prog.type    = ctx->type;
-		prog.insns   = data_insn.sec_data->d_buf;
-		prog.size    = data_insn.sec_data->d_size;
-		prog.license = ctx->license;
+		memcpy(prog->insns, data_insn.sec_data->d_buf, prog->size);
 
-		fd = bpf_prog_attach(section, &prog, ctx);
+		ret = bpf_apply_relo_data(ctx, &data_relo, prog, &props);
+		if (ret < 0) {
+			*lderr = true;
+			if (ctx->sec_text != idx)
+				free(prog->insns);
+			return ret;
+		}
+		if (ctx->sec_text == idx) {
+			fd = 0;
+			goto out;
+		}
+
+		fd = bpf_prog_attach(section, prog, ctx);
+		free(prog->insns);
 		if (fd < 0) {
 			*lderr = true;
-			if (props.total) {
+			if (props.tc.total) {
 				if (ctx->cfg.jit_enabled &&
-				    props.total != props.jited)
+				    props.tc.total != props.tc.jited)
 					fprintf(stderr, "JIT enabled, but only %u/%u tail call maps in the program have JITed owner!\n",
-						props.jited, props.total);
+						props.tc.jited, props.tc.total);
 				if (!ctx->cfg.jit_enabled &&
-				    props.jited)
+				    props.tc.jited)
 					fprintf(stderr, "JIT disabled, but %u/%u tail call maps in the program have JITed owner!\n",
-						props.jited, props.total);
+						props.tc.jited, props.tc.total);
 			}
 			return fd;
 		}
-
+out:
 		ctx->sec_done[i]   = true;
 		ctx->sec_done[idx] = true;
 		break;
@@ -2148,10 +2512,18 @@ static int bpf_fetch_prog_relo(struct bpf_elf_ctx *ctx, const char *section,
 static int bpf_fetch_prog_sec(struct bpf_elf_ctx *ctx, const char *section)
 {
 	bool lderr = false, sseen = false;
+	struct bpf_elf_prog prog;
 	int ret = -1;
 
-	if (bpf_has_map_data(ctx))
-		ret = bpf_fetch_prog_relo(ctx, section, &lderr, &sseen);
+	if (bpf_has_call_data(ctx)) {
+		ret = bpf_fetch_prog_relo(ctx, ".text", &lderr, NULL,
+					  &ctx->prog_text);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (bpf_has_map_data(ctx) || bpf_has_call_data(ctx))
+		ret = bpf_fetch_prog_relo(ctx, section, &lderr, &sseen, &prog);
 	if (ret < 0 && !lderr)
 		ret = bpf_fetch_prog(ctx, section, &sseen);
 	if (ret < 0 && !sseen)
@@ -2446,14 +2818,24 @@ static int bpf_elf_ctx_init(struct bpf_elf_ctx *ctx, const char *pathname,
 			    enum bpf_prog_type type, __u32 ifindex,
 			    bool verbose)
 {
-	int ret = -EINVAL;
+	uint8_t tmp[20];
+	int ret;
 
-	if (elf_version(EV_CURRENT) == EV_NONE ||
-	    bpf_init_env(pathname))
-		return ret;
+	if (elf_version(EV_CURRENT) == EV_NONE)
+		return -EINVAL;
+
+	bpf_init_env();
 
 	memset(ctx, 0, sizeof(*ctx));
 	bpf_get_cfg(ctx);
+
+	ret = bpf_obj_hash(pathname, tmp, sizeof(tmp));
+	if (ret)
+		ctx->noafalg = true;
+	else
+		hexstring_n2a(tmp, sizeof(tmp), ctx->obj_uid,
+			      sizeof(ctx->obj_uid));
+
 	ctx->verbose = verbose;
 	ctx->type    = type;
 	ctx->ifindex = ifindex;
@@ -2529,6 +2911,10 @@ static void bpf_maps_teardown(struct bpf_elf_ctx *ctx)
 		if (ctx->map_fds[i])
 			close(ctx->map_fds[i]);
 	}
+
+	if (ctx->btf_fd)
+		close(ctx->btf_fd);
+	free(ctx->btf.types);
 }
 
 static void bpf_elf_ctx_destroy(struct bpf_elf_ctx *ctx, bool failure)
@@ -2538,6 +2924,7 @@ static void bpf_elf_ctx_destroy(struct bpf_elf_ctx *ctx, bool failure)
 
 	bpf_hash_destroy(ctx);
 
+	free(ctx->prog_text.insns);
 	free(ctx->sec_done);
 	free(ctx->log);
 
@@ -2559,7 +2946,7 @@ static int bpf_obj_open(const char *pathname, enum bpf_prog_type type,
 		return ret;
 	}
 
-	ret = bpf_fetch_ancillary(ctx);
+	ret = bpf_fetch_ancillary(ctx, strcmp(section, ".text"));
 	if (ret < 0) {
 		fprintf(stderr, "Error fetching ELF ancillary data!\n");
 		goto out;