check.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
 */

#define _GNU_SOURCE /* memmem() */
#include <fnmatch.h>
#include <string.h>
#include <stdlib.h>
#include <inttypes.h>
#include <sys/mman.h>

#include <objtool/builtin.h>
#include <objtool/cfi.h>
#include <objtool/arch.h>
#include <objtool/disas.h>
#include <objtool/check.h>
#include <objtool/klp.h>
#include <objtool/special.h>
#include <objtool/trace.h>
#include <objtool/warn.h>
#include <objtool/util.h>

#include <linux/objtool_types.h>
#include <linux/hashtable.h>
#include <linux/kernel.h>
#include <linux/static_call_types.h>
#include <linux/string.h>

static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;

static struct cfi_init_state initial_func_cfi;
static struct cfi_state init_cfi;
static struct cfi_state func_cfi;
static struct cfi_state force_undefined_cfi;

struct disas_context *objtool_disas_ctx;

size_t sym_name_max_len;

struct instruction *find_insn(struct objtool_file *file,
			      struct section *sec, unsigned long offset)
{
	struct instruction *insn;

	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
		if (insn->sec == sec && insn->offset == offset)
			return insn;
	}

	return NULL;
}

struct instruction *next_insn_same_sec(struct objtool_file *file,
				       struct instruction *insn)
{
	if (insn->idx == INSN_CHUNK_MAX)
		return find_insn(file, insn->sec, insn->offset + insn->len);

	insn++;
	if (!insn->len)
		return NULL;

	return insn;
}

struct instruction *next_insn_same_func(struct objtool_file *file,
				       struct instruction *insn)
{
	struct instruction *next = next_insn_same_sec(file, insn);
	struct symbol *func = insn_func(insn);

	if (!func)
		return NULL;

	if (next && insn_func(next) == func)
		return next;

	/* Check if we're already in the subfunction: */
	if (func == func->cfunc)
		return NULL;

	/* Move to the subfunction: */
	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
}

static struct instruction *prev_insn_same_sec(struct objtool_file *file,
					      struct instruction *insn)
{
	if (insn->idx == 0) {
		if (insn->prev_len)
			return find_insn(file, insn->sec, insn->offset - insn->prev_len);
		return NULL;
	}

	return insn - 1;
}

static struct instruction *prev_insn_same_sym(struct objtool_file *file,
					      struct instruction *insn)
{
	struct instruction *prev = prev_insn_same_sec(file, insn);

	if (prev && insn_func(prev) == insn_func(insn))
		return prev;

	return NULL;
}

#define for_each_insn(file, insn)					\
	for (struct section *__sec, *__fake = (struct section *)1;	\
	     __fake; __fake = NULL)					\
		for_each_sec(file->elf, __sec)				\
			sec_for_each_insn(file, __sec, insn)


#define sym_for_each_insn(file, sym, insn)				\
	for (insn = find_insn(file, sym->sec, sym->offset);		\
	     insn && insn->offset < sym->offset + sym->len;		\
	     insn = next_insn_same_sec(file, insn))

#define sym_for_each_insn_continue_reverse(file, sym, insn)		\
	for (insn = prev_insn_same_sec(file, insn);			\
	     insn && insn->offset >= sym->offset;			\
	     insn = prev_insn_same_sec(file, insn))

#define sec_for_each_insn_from(file, insn)				\
	for (; insn; insn = next_insn_same_sec(file, insn))

#define sec_for_each_insn_continue(file, insn)				\
	for (insn = next_insn_same_sec(file, insn); insn;		\
	     insn = next_insn_same_sec(file, insn))

static inline struct reloc *insn_jump_table(struct instruction *insn)
{
	if (insn->type == INSN_JUMP_DYNAMIC ||
	    insn->type == INSN_CALL_DYNAMIC)
		return insn->_jump_table;

	return NULL;
}

static inline unsigned long insn_jump_table_size(struct instruction *insn)
{
	if (insn->type == INSN_JUMP_DYNAMIC ||
	    insn->type == INSN_CALL_DYNAMIC)
		return insn->_jump_table_size;

	return 0;
}

static bool is_jump_table_jump(struct instruction *insn)
{
	struct alt_group *alt_group = insn->alt_group;

	if (insn_jump_table(insn))
		return true;

	/* Retpoline alternative for a jump table? */
	return alt_group && alt_group->orig_group &&
	       insn_jump_table(alt_group->orig_group->first_insn);
}

static bool is_sibling_call(struct instruction *insn)
{
	/*
	 * Assume only STT_FUNC calls have jump-tables.
	 */
	if (insn_func(insn)) {
		/* An indirect jump is either a sibling call or a jump to a table. */
		if (insn->type == INSN_JUMP_DYNAMIC)
			return !is_jump_table_jump(insn);
	}

	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
	return (is_static_jump(insn) && insn_call_dest(insn));
}

/*
 * Checks if a function is a Rust "noreturn" one.
 */
static bool is_rust_noreturn(const struct symbol *func)
{
	/*
	 * If it does not start with "_R", then it is not a Rust symbol.
	 */
	if (strncmp(func->name, "_R", 2))
		return false;

	/*
	 * These are just heuristics -- we do not control the precise symbol
	 * name, due to the crate disambiguators (which depend on the compiler)
	 * as well as changes to the source code itself between versions (since
	 * these come from the Rust standard library).
	 */
	return str_ends_with(func->name, "_4core3num20from_str_radix_panic")				||
	       str_ends_with(func->name, "_4core3num22from_ascii_radix_panic")				||
	       str_ends_with(func->name, "_4core3num28from_ascii_bytes_radix_panic")			||
	       str_ends_with(func->name, "_4core3str16slice_error_fail")				||
	       str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail")		||
	       str_ends_with(func->name, "_4core6option13expect_failed")				||
	       str_ends_with(func->name, "_4core6option13unwrap_failed")				||
	       str_ends_with(func->name, "_4core6result13unwrap_failed")				||
	       str_ends_with(func->name, "_4core9panicking5panic")					||
	       str_ends_with(func->name, "_4core9panicking9panic_fmt")					||
	       str_ends_with(func->name, "_4core9panicking14panic_explicit")				||
	       str_ends_with(func->name, "_4core9panicking14panic_nounwind")				||
	       str_ends_with(func->name, "_4core9panicking18panic_bounds_check")			||
	       str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt")			||
	       str_ends_with(func->name, "_4core9panicking19assert_failed_inner")			||
	       str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference")		||
	       str_ends_with(func->name, "_4core9panicking32panic_null_reference_constructed")		||
	       str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference")	||
	       str_ends_with(func->name, "_7___rustc17rust_begin_unwind")				||
	       strstr(func->name, "_4core9panicking13assert_failed")					||
	       strstr(func->name, "_4core9panicking11panic_const24panic_const_")			||
	       (strstr(func->name, "_4core5slice5index") &&
		strstr(func->name, "slice_") &&
		str_ends_with(func->name, "_fail"));
}

/*
 * This checks to see if the given function is a "noreturn" function.
 *
 * For global functions which are outside the scope of this object file, we
 * have to keep a manual list of them.
 *
 * For local functions, we have to detect them manually by simply looking for
 * the lack of a return instruction.
 */
static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
				int recursion)
{
	int i;
	struct instruction *insn;
	bool empty = true;

#define NORETURN(func) __stringify(func),
	static const char * const global_noreturns[] = {
#include "noreturns.h"
	};
#undef NORETURN

	if (!func)
		return false;

	if (!is_local_sym(func)) {
		if (is_rust_noreturn(func))
			return true;

		for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
			if (!strcmp(func->name, global_noreturns[i]))
				return true;
	}

	if (is_weak_sym(func))
		return false;

	if (!func->len)
		return false;

	insn = find_insn(file, func->sec, func->offset);
	if (!insn || !insn_func(insn))
		return false;

	func_for_each_insn(file, func, insn) {
		empty = false;

		if (insn->type == INSN_RETURN)
			return false;
	}

	if (empty)
		return false;

	/*
	 * A function can have a sibling call instead of a return.  In that
	 * case, the function's dead-end status depends on whether the target
	 * of the sibling call returns.
	 */
	func_for_each_insn(file, func, insn) {
		if (is_sibling_call(insn)) {
			struct instruction *dest = insn->jump_dest;

			if (!dest)
				/* sibling call to another file */
				return false;

			/* local sibling call */
			if (recursion == 5) {
				/*
				 * Infinite recursion: two functions have
				 * sibling calls to each other.  This is a very
				 * rare case.  It means they aren't dead ends.
				 */
				return false;
			}

			return __dead_end_function(file, insn_func(dest), recursion+1);
		}
	}

	return true;
}

static bool dead_end_function(struct objtool_file *file, struct symbol *func)
{
	return __dead_end_function(file, func, 0);
}

static void init_cfi_state(struct cfi_state *cfi)
{
	int i;

	for (i = 0; i < CFI_NUM_REGS; i++) {
		cfi->regs[i].base = CFI_UNDEFINED;
		cfi->vals[i].base = CFI_UNDEFINED;
	}
	cfi->cfa.base = CFI_UNDEFINED;
	cfi->drap_reg = CFI_UNDEFINED;
	cfi->drap_offset = -1;
}

static void init_insn_state(struct objtool_file *file, struct insn_state *state,
			    struct section *sec)
{
	memset(state, 0, sizeof(*state));
	init_cfi_state(&state->cfi);

	if (opts.noinstr && sec)
		state->noinstr = sec->noinstr;
}

static struct cfi_state *cfi_alloc(void)
{
	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
	if (!cfi) {
		ERROR_GLIBC("calloc");
		exit(1);
	}
	nr_cfi++;
	return cfi;
}

static int cfi_bits;
static struct hlist_head *cfi_hash;

static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
{
	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
		      (void *)cfi2 + sizeof(cfi2->hash),
		      sizeof(struct cfi_state) - sizeof(struct hlist_node));
}

static inline u32 cfi_key(struct cfi_state *cfi)
{
	return jhash((void *)cfi + sizeof(cfi->hash),
		     sizeof(*cfi) - sizeof(cfi->hash), 0);
}

static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
{
	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
	struct cfi_state *obj;

	hlist_for_each_entry(obj, head, hash) {
		if (!cficmp(cfi, obj)) {
			nr_cfi_cache++;
			return obj;
		}
	}

	obj = cfi_alloc();
	*obj = *cfi;
	hlist_add_head(&obj->hash, head);

	return obj;
}

static void cfi_hash_add(struct cfi_state *cfi)
{
	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];

	hlist_add_head(&cfi->hash, head);
}

static void *cfi_hash_alloc(unsigned long size)
{
	cfi_bits = max(10, ilog2(size));
	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
			PROT_READ|PROT_WRITE,
			MAP_PRIVATE|MAP_ANON, -1, 0);
	if (cfi_hash == (void *)-1L) {
		ERROR_GLIBC("mmap fail cfi_hash");
		cfi_hash = NULL;
	}  else if (opts.stats) {
		printf("cfi_bits: %d\n", cfi_bits);
	}

	return cfi_hash;
}

static unsigned long nr_insns;
static unsigned long nr_insns_visited;

/*
 * Call the arch-specific instruction decoder for all the instructions and add
 * them to the global instruction list.
 */
static int decode_instructions(struct objtool_file *file)
{
	struct section *sec;
	struct symbol *func;
	unsigned long offset;
	struct instruction *insn;

	for_each_sec(file->elf, sec) {
		struct instruction *insns = NULL;
		u8 prev_len = 0;
		u8 idx = 0;

		if (!is_text_sec(sec))
			continue;

		if (strcmp(sec->name, ".altinstr_replacement") &&
		    strcmp(sec->name, ".altinstr_aux") &&
		    strncmp(sec->name, ".discard.", 9))
			sec->text = true;

		if (!strcmp(sec->name, ".noinstr.text") ||
		    !strcmp(sec->name, ".entry.text") ||
		    !strcmp(sec->name, ".cpuidle.text") ||
		    !strncmp(sec->name, ".text..__x86.", 13))
			sec->noinstr = true;

		/*
		 * .init.text code is ran before userspace and thus doesn't
		 * strictly need retpolines, except for modules which are
		 * loaded late, they very much do need retpoline in their
		 * .init.text
		 */
		if (!strcmp(sec->name, ".init.text") && !opts.module)
			sec->init = true;

		for (offset = 0; offset < sec_size(sec); offset += insn->len) {
			if (!insns || idx == INSN_CHUNK_MAX) {
				insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn));
				if (!insns) {
					ERROR_GLIBC("calloc");
					return -1;
				}
				idx = 0;
			} else {
				idx++;
			}
			insn = &insns[idx];
			insn->idx = idx;

			INIT_LIST_HEAD(&insn->call_node);
			insn->sec = sec;
			insn->offset = offset;
			insn->prev_len = prev_len;

			if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn))
				return -1;

			prev_len = insn->len;

			/*
			 * By default, "ud2" is a dead end unless otherwise
			 * annotated, because GCC 7 inserts it for certain
			 * divide-by-zero cases.
			 */
			if (insn->type == INSN_BUG)
				insn->dead_end = true;

			hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
			nr_insns++;
		}

		sec_for_each_sym(sec, func) {
			if (!is_notype_sym(func) && !is_func_sym(func))
				continue;

			if (func->offset == sec_size(sec)) {
				/* Heuristic: likely an "end" symbol */
				if (is_notype_sym(func))
					continue;
				ERROR("%s(): STT_FUNC at end of section", func->name);
				return -1;
			}

			if (func->embedded_insn || is_alias_sym(func))
				continue;

			if (!find_insn(file, sec, func->offset)) {
				ERROR("%s(): can't find starting instruction", func->name);
				return -1;
			}

			sym_for_each_insn(file, func, insn) {
				insn->_sym = func;
				if (is_func_sym(func) &&
				    insn->type == INSN_ENDBR &&
				    list_empty(&insn->call_node)) {
					if (insn->offset == func->offset) {
						list_add_tail(&insn->call_node, &file->endbr_list);
						file->nr_endbr++;
					} else {
						file->nr_endbr_int++;
					}
				}
			}
		}
	}

	if (opts.stats)
		printf("nr_insns: %lu\n", nr_insns);

	return 0;
}

/*
 * Known pv_ops*[] arrays.
 */
static struct {
	const char *name;
	int idx_off;
} pv_ops_tables[] = {
	{ .name = "pv_ops", },
	{ .name = "pv_ops_lock", },
	{ .name = NULL, .idx_off = -1 }
};

/*
 * Get index offset for a pv_ops* array.
 */
int pv_ops_idx_off(const char *symname)
{
	int idx;

	for (idx = 0; pv_ops_tables[idx].name; idx++) {
		if (!strcmp(symname, pv_ops_tables[idx].name))
			break;
	}

	return pv_ops_tables[idx].idx_off;
}

/*
 * Read a pv_ops*[] .data table to find the static initialized values.
 */
static int add_pv_ops(struct objtool_file *file, int pv_ops_idx)
{
	struct symbol *sym, *func;
	unsigned long off, end;
	struct reloc *reloc;
	int idx, idx_off;
	const char *symname;

	symname = pv_ops_tables[pv_ops_idx].name;
	sym = find_symbol_by_name(file->elf, symname);
	if (!sym) {
		ERROR("Unknown pv_ops array %s", symname);
		return -1;
	}

	off = sym->offset;
	end = off + sym->len;
	idx_off = pv_ops_tables[pv_ops_idx].idx_off;
	if (idx_off < 0) {
		ERROR("pv_ops array %s has unknown index offset", symname);
		return -1;
	}

	for (;;) {
		reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
		if (!reloc)
			break;

		idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);

		func = reloc->sym;
		if (is_sec_sym(func))
			func = find_symbol_by_offset(reloc->sym->sec,
						     reloc_addend(reloc));
		if (!func) {
			ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc),
				   "can't find func at %s[%d]", symname, idx);
			return -1;
		}

		if (objtool_pv_add(file, idx + idx_off, func))
			return -1;

		off = reloc_offset(reloc) + 1;
		if (off > end)
			break;
	}

	return 0;
}

/*
 * Allocate and initialize file->pv_ops[].
 */
static int init_pv_ops(struct objtool_file *file)
{
	struct symbol *sym;
	int idx, nr;

	if (!opts.noinstr)
		return 0;

	file->pv_ops = NULL;

	nr = 0;
	for (idx = 0; pv_ops_tables[idx].name; idx++) {
		sym = find_symbol_by_name(file->elf, pv_ops_tables[idx].name);
		if (!sym) {
			pv_ops_tables[idx].idx_off = -1;
			continue;
		}
		pv_ops_tables[idx].idx_off = nr;
		nr += sym->len / sizeof(unsigned long);
	}

	if (nr == 0)
		return 0;

	file->pv_ops = calloc(nr, sizeof(struct pv_state));
	if (!file->pv_ops) {
		ERROR_GLIBC("calloc");
		return -1;
	}

	for (idx = 0; idx < nr; idx++)
		INIT_LIST_HEAD(&file->pv_ops[idx].targets);

	for (idx = 0; pv_ops_tables[idx].name; idx++) {
		if (pv_ops_tables[idx].idx_off < 0)
			continue;
		if (add_pv_ops(file, idx))
			return -1;
	}

	return 0;
}

static bool is_livepatch_module(struct objtool_file *file)
{
	struct section *sec;

	if (!opts.module)
		return false;

	sec = find_section_by_name(file->elf, ".modinfo");
	if (!sec)
		return false;

	return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12);
}

static int create_static_call_sections(struct objtool_file *file)
{
	struct static_call_site *site;
	struct section *sec;
	struct instruction *insn;
	struct symbol *key_sym;
	char *key_name, *tmp;
	int idx;

	sec = find_section_by_name(file->elf, ".static_call_sites");
	if (sec) {
		/*
		 * Livepatch modules may have already extracted the static call
		 * site entries to take advantage of vmlinux static call
		 * privileges.
		 */
		if (!file->klp)
			WARN("file already has .static_call_sites section, skipping");

		return 0;
	}

	if (list_empty(&file->static_call_list))
		return 0;

	idx = 0;
	list_for_each_entry(insn, &file->static_call_list, call_node)
		idx++;

	sec = elf_create_section_pair(file->elf, ".static_call_sites",
				      sizeof(*site), idx, idx * 2);
	if (!sec)
		return -1;

	/* Allow modules to modify the low bits of static_call_site::key */
	sec->sh.sh_flags |= SHF_WRITE;

	idx = 0;
	list_for_each_entry(insn, &file->static_call_list, call_node) {

		/* populate reloc for 'addr' */
		if (!elf_init_reloc_text_sym(file->elf, sec,
					     idx * sizeof(*site), idx * 2,
					     insn->sec, insn->offset))
			return -1;

		/* find key symbol */
		key_name = strdup(insn_call_dest(insn)->name);
		if (!key_name) {
			ERROR_GLIBC("strdup");
			return -1;
		}
		if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
			    STATIC_CALL_TRAMP_PREFIX_LEN)) {
			ERROR("static_call: trampoline name malformed: %s", key_name);
			return -1;
		}
		tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
		memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);

		key_sym = find_symbol_by_name(file->elf, tmp);
		if (!key_sym) {
			if (!opts.module) {
				ERROR("static_call: can't find static_call_key symbol: %s", tmp);
				return -1;
			}

			/*
			 * For modules(), the key might not be exported, which
			 * means the module can make static calls but isn't
			 * allowed to change them.
			 *
			 * In that case we temporarily set the key to be the
			 * trampoline address.  This is fixed up in
			 * static_call_add_module().
			 */
			key_sym = insn_call_dest(insn);
		}

		/* populate reloc for 'key' */
		if (!elf_init_reloc_data_sym(file->elf, sec,
					     idx * sizeof(*site) + 4,
					     (idx * 2) + 1, key_sym,
					     is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
			return -1;

		idx++;
	}

	return 0;
}

static int create_retpoline_sites_sections(struct objtool_file *file)
{
	struct instruction *insn;
	struct section *sec;
	int idx;

	sec = find_section_by_name(file->elf, ".retpoline_sites");
	if (sec) {
		WARN("file already has .retpoline_sites, skipping");
		return 0;
	}

	idx = 0;
	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
		idx++;

	if (!idx)
		return 0;

	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
				      sizeof(int), idx, idx);
	if (!sec)
		return -1;

	idx = 0;
	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {

		if (!elf_init_reloc_text_sym(file->elf, sec,
					     idx * sizeof(int), idx,
					     insn->sec, insn->offset))
			return -1;

		idx++;
	}

	return 0;
}

static int create_return_sites_sections(struct objtool_file *file)
{
	struct instruction *insn;
	struct section *sec;
	int idx;

	sec = find_section_by_name(file->elf, ".return_sites");
	if (sec) {
		WARN("file already has .return_sites, skipping");
		return 0;
	}

	idx = 0;
	list_for_each_entry(insn, &file->return_thunk_list, call_node)
		idx++;

	if (!idx)
		return 0;

	sec = elf_create_section_pair(file->elf, ".return_sites",
				      sizeof(int), idx, idx);
	if (!sec)
		return -1;

	idx = 0;
	list_for_each_entry(insn, &file->return_thunk_list, call_node) {

		if (!elf_init_reloc_text_sym(file->elf, sec,
					     idx * sizeof(int), idx,
					     insn->sec, insn->offset))
			return -1;

		idx++;
	}

	return 0;
}

static int create_ibt_endbr_seal_sections(struct objtool_file *file)
{
	struct instruction *insn;
	struct section *sec;
	int idx;

	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
	if (sec) {
		WARN("file already has .ibt_endbr_seal, skipping");
		return 0;
	}

	idx = 0;
	list_for_each_entry(insn, &file->endbr_list, call_node)
		idx++;

	if (opts.stats) {
		printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
		printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
		printf("ibt: superfluous ENDBR:       %d\n", idx);
	}

	if (!idx)
		return 0;

	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
				      sizeof(int), idx, idx);
	if (!sec)
		return -1;

	idx = 0;
	list_for_each_entry(insn, &file->endbr_list, call_node) {

		int *site = (int *)sec->data->d_buf + idx;
		struct symbol *func = insn_func(insn);
		*site = 0;

		if (opts.module && func && insn->offset == func->offset &&
		    (!strcmp(func->name, "init_module") ||
		     !strcmp(func->name, "cleanup_module"))) {
			ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead",
			      func->name);
			return -1;
		}

		if (!elf_init_reloc_text_sym(file->elf, sec,
					     idx * sizeof(int), idx,
					     insn->sec, insn->offset))
			return -1;

		idx++;
	}

	return 0;
}

/*
* Grow __cfi_ symbols to fill the NOP gap between the 'mov <hash>, %rax' and
* the start of the function.
*/
static int grow_cfi_symbols(struct objtool_file *file)
{
	struct symbol *sym;

	for_each_sym(file->elf, sym) {
		if (!is_func_sym(sym) || !strstarts(sym->name, "__cfi_") ||
		    sym->len != 5)
			continue;

		if (!find_func_by_offset(sym->sec, sym->offset + sym->len + opts.prefix))
			continue;

		sym->len += opts.prefix;
		sym->sym.st_size = sym->len;
		if (elf_write_symbol(file->elf, sym))
			return -1;
	}

	return 0;
}

static int create_cfi_sections(struct objtool_file *file)
{
	struct section *sec;
	struct symbol *sym;
	int idx;

	sec = find_section_by_name(file->elf, ".cfi_sites");
	if (sec) {
		WARN("file already has .cfi_sites section, skipping");
		return 0;
	}

	idx = 0;
	for_each_sym(file->elf, sym) {
		if (!is_func_sym(sym))
			continue;

		if (strncmp(sym->name, "__cfi_", 6))
			continue;

		idx++;
	}

	sec = elf_create_section_pair(file->elf, ".cfi_sites",
				      sizeof(unsigned int), idx, idx);
	if (!sec)
		return -1;

	idx = 0;
	for_each_sym(file->elf, sym) {
		if (!is_func_sym(sym))
			continue;

		if (strncmp(sym->name, "__cfi_", 6))
			continue;

		if (!elf_init_reloc_text_sym(file->elf, sec,
					     idx * sizeof(unsigned int), idx,
					     sym->sec, sym->offset))
			return -1;

		idx++;
	}

	return 0;
}

static int create_mcount_loc_sections(struct objtool_file *file)
{
	size_t addr_size = elf_addr_size(file->elf);
	struct instruction *insn;
	struct section *sec;
	int idx;

	sec = find_section_by_name(file->elf, "__mcount_loc");
	if (sec) {
		/*
		 * Livepatch modules have already extracted their __mcount_loc
		 * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case.
		 */
		if (!file->klp)
			WARN("file already has __mcount_loc section, skipping");

		return 0;
	}

	if (list_empty(&file->mcount_loc_list))
		return 0;

	idx = 0;
	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
		idx++;

	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
				      idx, idx);
	if (!sec)
		return -1;

	sec->sh.sh_addralign = addr_size;

	idx = 0;
	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {

		struct reloc *reloc;

		reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
					       insn->sec, insn->offset);
		if (!reloc)