linux - tools/objtool/check.c

4971 lines · 113.6 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2015-2017 Josh Poimboeuf <[email protected]>
4	*/
5
6	#define _GNU_SOURCE /* memmem() */
7	#include <fnmatch.h>
8	#include <string.h>
9	#include <stdlib.h>
10	#include <inttypes.h>
11	#include <sys/mman.h>
12
13	#include <objtool/builtin.h>
14	#include <objtool/cfi.h>
15	#include <objtool/arch.h>
16	#include <objtool/disas.h>
17	#include <objtool/check.h>
18	#include <objtool/special.h>
19	#include <objtool/trace.h>
20	#include <objtool/warn.h>
21	#include <objtool/util.h>
22
23	#include <linux/objtool_types.h>
24	#include <linux/hashtable.h>
25	#include <linux/kernel.h>
26	#include <linux/static_call_types.h>
27	#include <linux/string.h>
28
29	static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
30
31	static struct cfi_init_state initial_func_cfi;
32	static struct cfi_state init_cfi;
33	static struct cfi_state func_cfi;
34	static struct cfi_state force_undefined_cfi;
35
36	struct disas_context *objtool_disas_ctx;
37
38	size_t sym_name_max_len;
39
40	struct instruction find_insn(struct objtool_file file,
41	struct section *sec, unsigned long offset)
42	{
43	struct instruction *insn;
44
45	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
46	if (insn->sec == sec && insn->offset == offset)
47	return insn;
48	}
49
50	return NULL;
51	}
52
53	struct instruction next_insn_same_sec(struct objtool_file file,
54	struct instruction *insn)
55	{
56	if (insn->idx == INSN_CHUNK_MAX)
57	return find_insn(file, insn->sec, insn->offset + insn->len);
58
59	insn++;
60	if (!insn->len)
61	return NULL;
62
63	return insn;
64	}
65
66	struct instruction next_insn_same_func(struct objtool_file file,
67	struct instruction *insn)
68	{
69	struct instruction *next = next_insn_same_sec(file, insn);
70	struct symbol *func = insn_func(insn);
71
72	if (!func)
73	return NULL;
74
75	if (next && insn_func(next) == func)
76	return next;
77
78	/* Check if we're already in the subfunction: */
79	if (func == func->cfunc)
80	return NULL;
81
82	/* Move to the subfunction: */
83	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
84	}
85
86	static struct instruction prev_insn_same_sec(struct objtool_file file,
87	struct instruction *insn)
88	{
89	if (insn->idx == 0) {
90	if (insn->prev_len)
91	return find_insn(file, insn->sec, insn->offset - insn->prev_len);
92	return NULL;
93	}
94
95	return insn - 1;
96	}
97
98	static struct instruction prev_insn_same_sym(struct objtool_file file,
99	struct instruction *insn)
100	{
101	struct instruction *prev = prev_insn_same_sec(file, insn);
102
103	if (prev && insn_func(prev) == insn_func(insn))
104	return prev;
105
106	return NULL;
107	}
108
109	#define for_each_insn(file, insn) \
110	for (struct section __sec, __fake = (struct section *)1; \
111	__fake; __fake = NULL) \
112	for_each_sec(file->elf, __sec) \
113	sec_for_each_insn(file, __sec, insn)
114
115
116	#define sym_for_each_insn(file, sym, insn) \
117	for (insn = find_insn(file, sym->sec, sym->offset); \
118	insn && insn->offset < sym->offset + sym->len; \
119	insn = next_insn_same_sec(file, insn))
120
121	#define sym_for_each_insn_continue_reverse(file, sym, insn) \
122	for (insn = prev_insn_same_sec(file, insn); \
123	insn && insn->offset >= sym->offset; \
124	insn = prev_insn_same_sec(file, insn))
125
126	#define sec_for_each_insn_from(file, insn) \
127	for (; insn; insn = next_insn_same_sec(file, insn))
128
129	#define sec_for_each_insn_continue(file, insn) \
130	for (insn = next_insn_same_sec(file, insn); insn; \
131	insn = next_insn_same_sec(file, insn))
132
133	static inline struct reloc insn_jump_table(struct instruction insn)
134	{
135	if (insn->type == INSN_JUMP_DYNAMIC \|\|
136	insn->type == INSN_CALL_DYNAMIC)
137	return insn->_jump_table;
138
139	return NULL;
140	}
141
142	static inline unsigned long insn_jump_table_size(struct instruction *insn)
143	{
144	if (insn->type == INSN_JUMP_DYNAMIC \|\|
145	insn->type == INSN_CALL_DYNAMIC)
146	return insn->_jump_table_size;
147
148	return 0;
149	}
150
151	static bool is_jump_table_jump(struct instruction *insn)
152	{
153	struct alt_group *alt_group = insn->alt_group;
154
155	if (insn_jump_table(insn))
156	return true;
157
158	/* Retpoline alternative for a jump table? */
159	return alt_group && alt_group->orig_group &&
160	insn_jump_table(alt_group->orig_group->first_insn);
161	}
162
163	static bool is_sibling_call(struct instruction *insn)
164	{
165	/*
166	* Assume only STT_FUNC calls have jump-tables.
167	*/
168	if (insn_func(insn)) {
169	/* An indirect jump is either a sibling call or a jump to a table. */
170	if (insn->type == INSN_JUMP_DYNAMIC)
171	return !is_jump_table_jump(insn);
172	}
173
174	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
175	return (is_static_jump(insn) && insn_call_dest(insn));
176	}
177
178	/*
179	* Checks if a function is a Rust "noreturn" one.
180	*/
181	static bool is_rust_noreturn(const struct symbol *func)
182	{
183	/*
184	* If it does not start with "_R", then it is not a Rust symbol.
185	*/
186	if (strncmp(func->name, "_R", 2))
187	return false;
188
189	/*
190	* These are just heuristics -- we do not control the precise symbol
191	* name, due to the crate disambiguators (which depend on the compiler)
192	* as well as changes to the source code itself between versions (since
193	* these come from the Rust standard library).
194	*/
195	return str_ends_with(func->name, "_4core3num20from_str_radix_panic") \|\|
196	str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") \|\|
197	str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") \|\|
198	str_ends_with(func->name, "_4core6option13expect_failed") \|\|
199	str_ends_with(func->name, "_4core6option13unwrap_failed") \|\|
200	str_ends_with(func->name, "_4core6result13unwrap_failed") \|\|
201	str_ends_with(func->name, "_4core9panicking5panic") \|\|
202	str_ends_with(func->name, "_4core9panicking9panic_fmt") \|\|
203	str_ends_with(func->name, "_4core9panicking14panic_explicit") \|\|
204	str_ends_with(func->name, "_4core9panicking14panic_nounwind") \|\|
205	str_ends_with(func->name, "_4core9panicking18panic_bounds_check") \|\|
206	str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") \|\|
207	str_ends_with(func->name, "_4core9panicking19assert_failed_inner") \|\|
208	str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") \|\|
209	str_ends_with(func->name, "_4core9panicking32panic_null_reference_constructed") \|\|
210	str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") \|\|
211	str_ends_with(func->name, "_7___rustc17rust_begin_unwind") \|\|
212	strstr(func->name, "_4core9panicking13assert_failed") \|\|
213	strstr(func->name, "_4core9panicking11panic_const24panic_const_") \|\|
214	(strstr(func->name, "_4core5slice5index") &&
215	strstr(func->name, "slice_") &&
216	str_ends_with(func->name, "_fail"));
217	}
218
219	/*
220	* This checks to see if the given function is a "noreturn" function.
221	*
222	* For global functions which are outside the scope of this object file, we
223	* have to keep a manual list of them.
224	*
225	* For local functions, we have to detect them manually by simply looking for
226	* the lack of a return instruction.
227	*/
228	static bool __dead_end_function(struct objtool_file file, struct symbol func,
229	int recursion)
230	{
231	int i;
232	struct instruction *insn;
233	bool empty = true;
234
235	#define NORETURN(func) __stringify(func),
236	static const char * const global_noreturns[] = {
237	#include "noreturns.h"
238	};
239	#undef NORETURN
240
241	if (!func)
242	return false;
243
244	if (!is_local_sym(func)) {
245	if (is_rust_noreturn(func))
246	return true;
247
248	for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
249	if (!strcmp(func->name, global_noreturns[i]))
250	return true;
251	}
252
253	if (is_weak_sym(func))
254	return false;
255
256	if (!func->len)
257	return false;
258
259	insn = find_insn(file, func->sec, func->offset);
260	if (!insn \|\| !insn_func(insn))
261	return false;
262
263	func_for_each_insn(file, func, insn) {
264	empty = false;
265
266	if (insn->type == INSN_RETURN)
267	return false;
268	}
269
270	if (empty)
271	return false;
272
273	/*
274	* A function can have a sibling call instead of a return. In that
275	* case, the function's dead-end status depends on whether the target
276	* of the sibling call returns.
277	*/
278	func_for_each_insn(file, func, insn) {
279	if (is_sibling_call(insn)) {
280	struct instruction *dest = insn->jump_dest;
281
282	if (!dest)
283	/* sibling call to another file */
284	return false;
285
286	/* local sibling call */
287	if (recursion == 5) {
288	/*
289	* Infinite recursion: two functions have
290	* sibling calls to each other. This is a very
291	* rare case. It means they aren't dead ends.
292	*/
293	return false;
294	}
295
296	return __dead_end_function(file, insn_func(dest), recursion+1);
297	}
298	}
299
300	return true;
301	}
302
303	static bool dead_end_function(struct objtool_file file, struct symbol func)
304	{
305	return __dead_end_function(file, func, 0);
306	}
307
308	static void init_cfi_state(struct cfi_state *cfi)
309	{
310	int i;
311
312	for (i = 0; i < CFI_NUM_REGS; i++) {
313	cfi->regs[i].base = CFI_UNDEFINED;
314	cfi->vals[i].base = CFI_UNDEFINED;
315	}
316	cfi->cfa.base = CFI_UNDEFINED;
317	cfi->drap_reg = CFI_UNDEFINED;
318	cfi->drap_offset = -1;
319	}
320
321	static void init_insn_state(struct objtool_file file, struct insn_state state,
322	struct section *sec)
323	{
324	memset(state, 0, sizeof(*state));
325	init_cfi_state(&state->cfi);
326
327	if (opts.noinstr && sec)
328	state->noinstr = sec->noinstr;
329	}
330
331	static struct cfi_state *cfi_alloc(void)
332	{
333	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
334	if (!cfi) {
335	ERROR_GLIBC("calloc");
336	exit(1);
337	}
338	nr_cfi++;
339	return cfi;
340	}
341
342	static int cfi_bits;
343	static struct hlist_head *cfi_hash;
344
345	static inline bool cficmp(struct cfi_state cfi1, struct cfi_state cfi2)
346	{
347	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
348	(void *)cfi2 + sizeof(cfi2->hash),
349	sizeof(struct cfi_state) - sizeof(struct hlist_node));
350	}
351
352	static inline u32 cfi_key(struct cfi_state *cfi)
353	{
354	return jhash((void *)cfi + sizeof(cfi->hash),
355	sizeof(*cfi) - sizeof(cfi->hash), 0);
356	}
357
358	static struct cfi_state cfi_hash_find_or_add(struct cfi_state cfi)
359	{
360	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
361	struct cfi_state *obj;
362
363	hlist_for_each_entry(obj, head, hash) {
364	if (!cficmp(cfi, obj)) {
365	nr_cfi_cache++;
366	return obj;
367	}
368	}
369
370	obj = cfi_alloc();
371	obj = cfi;
372	hlist_add_head(&obj->hash, head);
373
374	return obj;
375	}
376
377	static void cfi_hash_add(struct cfi_state *cfi)
378	{
379	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
380
381	hlist_add_head(&cfi->hash, head);
382	}
383
384	static void *cfi_hash_alloc(unsigned long size)
385	{
386	cfi_bits = max(10, ilog2(size));
387	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
388	PROT_READ\|PROT_WRITE,
389	MAP_PRIVATE\|MAP_ANON, -1, 0);
390	if (cfi_hash == (void *)-1L) {
391	ERROR_GLIBC("mmap fail cfi_hash");
392	cfi_hash = NULL;
393	} else if (opts.stats) {
394	printf("cfi_bits: %d\n", cfi_bits);
395	}
396
397	return cfi_hash;
398	}
399
400	static unsigned long nr_insns;
401	static unsigned long nr_insns_visited;
402
403	/*
404	* Call the arch-specific instruction decoder for all the instructions and add
405	* them to the global instruction list.
406	*/
407	static int decode_instructions(struct objtool_file *file)
408	{
409	struct section *sec;
410	struct symbol *func;
411	unsigned long offset;
412	struct instruction *insn;
413
414	for_each_sec(file->elf, sec) {
415	struct instruction *insns = NULL;
416	u8 prev_len = 0;
417	u8 idx = 0;
418
419	if (!is_text_sec(sec))
420	continue;
421
422	if (strcmp(sec->name, ".altinstr_replacement") &&
423	strcmp(sec->name, ".altinstr_aux") &&
424	strncmp(sec->name, ".discard.", 9))
425	sec->text = true;
426
427	if (!strcmp(sec->name, ".noinstr.text") \|\|
428	!strcmp(sec->name, ".entry.text") \|\|
429	!strcmp(sec->name, ".cpuidle.text") \|\|
430	!strncmp(sec->name, ".text..__x86.", 13))
431	sec->noinstr = true;
432
433	/*
434	* .init.text code is ran before userspace and thus doesn't
435	* strictly need retpolines, except for modules which are
436	* loaded late, they very much do need retpoline in their
437	* .init.text
438	*/
439	if (!strcmp(sec->name, ".init.text") && !opts.module)
440	sec->init = true;
441
442	for (offset = 0; offset < sec_size(sec); offset += insn->len) {
443	if (!insns \|\| idx == INSN_CHUNK_MAX) {
444	insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn));
445	if (!insns) {
446	ERROR_GLIBC("calloc");
447	return -1;
448	}
449	idx = 0;
450	} else {
451	idx++;
452	}
453	insn = &insns[idx];
454	insn->idx = idx;
455
456	INIT_LIST_HEAD(&insn->call_node);
457	insn->sec = sec;
458	insn->offset = offset;
459	insn->prev_len = prev_len;
460
461	if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn))
462	return -1;
463
464	prev_len = insn->len;
465
466	/*
467	* By default, "ud2" is a dead end unless otherwise
468	* annotated, because GCC 7 inserts it for certain
469	* divide-by-zero cases.
470	*/
471	if (insn->type == INSN_BUG)
472	insn->dead_end = true;
473
474	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
475	nr_insns++;
476	}
477
478	sec_for_each_sym(sec, func) {
479	if (!is_notype_sym(func) && !is_func_sym(func))
480	continue;
481
482	if (func->offset == sec_size(sec)) {
483	/* Heuristic: likely an "end" symbol */
484	if (is_notype_sym(func))
485	continue;
486	ERROR("%s(): STT_FUNC at end of section", func->name);
487	return -1;
488	}
489
490	if (func->embedded_insn \|\| is_alias_sym(func))
491	continue;
492
493	if (!find_insn(file, sec, func->offset)) {
494	ERROR("%s(): can't find starting instruction", func->name);
495	return -1;
496	}
497
498	sym_for_each_insn(file, func, insn) {
499	insn->_sym = func;
500	if (is_func_sym(func) &&
501	insn->type == INSN_ENDBR &&
502	list_empty(&insn->call_node)) {
503	if (insn->offset == func->offset) {
504	list_add_tail(&insn->call_node, &file->endbr_list);
505	file->nr_endbr++;
506	} else {
507	file->nr_endbr_int++;
508	}
509	}
510	}
511	}
512	}
513
514	if (opts.stats)
515	printf("nr_insns: %lu\n", nr_insns);
516
517	return 0;
518	}
519
520	/*
521	* Known pv_ops*[] arrays.
522	*/
523	static struct {
524	const char *name;
525	int idx_off;
526	} pv_ops_tables[] = {
527	{ .name = "pv_ops", },
528	{ .name = "pv_ops_lock", },
529	{ .name = NULL, .idx_off = -1 }
530	};
531
532	/*
533	* Get index offset for a pv_ops* array.
534	*/
535	int pv_ops_idx_off(const char *symname)
536	{
537	int idx;
538
539	for (idx = 0; pv_ops_tables[idx].name; idx++) {
540	if (!strcmp(symname, pv_ops_tables[idx].name))
541	break;
542	}
543
544	return pv_ops_tables[idx].idx_off;
545	}
546
547	/*
548	* Read a pv_ops*[] .data table to find the static initialized values.
549	*/
550	static int add_pv_ops(struct objtool_file *file, int pv_ops_idx)
551	{
552	struct symbol sym, func;
553	unsigned long off, end;
554	struct reloc *reloc;
555	int idx, idx_off;
556	const char *symname;
557
558	symname = pv_ops_tables[pv_ops_idx].name;
559	sym = find_symbol_by_name(file->elf, symname);
560	if (!sym) {
561	ERROR("Unknown pv_ops array %s", symname);
562	return -1;
563	}
564
565	off = sym->offset;
566	end = off + sym->len;
567	idx_off = pv_ops_tables[pv_ops_idx].idx_off;
568	if (idx_off < 0) {
569	ERROR("pv_ops array %s has unknown index offset", symname);
570	return -1;
571	}
572
573	for (;;) {
574	reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
575	if (!reloc)
576	break;
577
578	idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
579
580	func = reloc->sym;
581	if (is_sec_sym(func))
582	func = find_symbol_by_offset(reloc->sym->sec,
583	reloc_addend(reloc));
584	if (!func) {
585	ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc),
586	"can't find func at %s[%d]", symname, idx);
587	return -1;
588	}
589
590	if (objtool_pv_add(file, idx + idx_off, func))
591	return -1;
592
593	off = reloc_offset(reloc) + 1;
594	if (off > end)
595	break;
596	}
597
598	return 0;
599	}
600
601	/*
602	* Allocate and initialize file->pv_ops[].
603	*/
604	static int init_pv_ops(struct objtool_file *file)
605	{
606	struct symbol *sym;
607	int idx, nr;
608
609	if (!opts.noinstr)
610	return 0;
611
612	file->pv_ops = NULL;
613
614	nr = 0;
615	for (idx = 0; pv_ops_tables[idx].name; idx++) {
616	sym = find_symbol_by_name(file->elf, pv_ops_tables[idx].name);
617	if (!sym) {
618	pv_ops_tables[idx].idx_off = -1;
619	continue;
620	}
621	pv_ops_tables[idx].idx_off = nr;
622	nr += sym->len / sizeof(unsigned long);
623	}
624
625	if (nr == 0)
626	return 0;
627
628	file->pv_ops = calloc(nr, sizeof(struct pv_state));
629	if (!file->pv_ops) {
630	ERROR_GLIBC("calloc");
631	return -1;
632	}
633
634	for (idx = 0; idx < nr; idx++)
635	INIT_LIST_HEAD(&file->pv_ops[idx].targets);
636
637	for (idx = 0; pv_ops_tables[idx].name; idx++) {
638	if (pv_ops_tables[idx].idx_off < 0)
639	continue;
640	if (add_pv_ops(file, idx))
641	return -1;
642	}
643
644	return 0;
645	}
646
647	static bool is_livepatch_module(struct objtool_file *file)
648	{
649	struct section *sec;
650
651	if (!opts.module)
652	return false;
653
654	sec = find_section_by_name(file->elf, ".modinfo");
655	if (!sec)
656	return false;
657
658	return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12);
659	}
660
661	static int create_static_call_sections(struct objtool_file *file)
662	{
663	struct static_call_site *site;
664	struct section *sec;
665	struct instruction *insn;
666	struct symbol *key_sym;
667	char key_name, tmp;
668	int idx;
669
670	sec = find_section_by_name(file->elf, ".static_call_sites");
671	if (sec) {
672	/*
673	* Livepatch modules may have already extracted the static call
674	* site entries to take advantage of vmlinux static call
675	* privileges.
676	*/
677	if (!file->klp)
678	WARN("file already has .static_call_sites section, skipping");
679
680	return 0;
681	}
682
683	if (list_empty(&file->static_call_list))
684	return 0;
685
686	idx = 0;
687	list_for_each_entry(insn, &file->static_call_list, call_node)
688	idx++;
689
690	sec = elf_create_section_pair(file->elf, ".static_call_sites",
691	sizeof(site), idx, idx 2);
692	if (!sec)
693	return -1;
694
695	/* Allow modules to modify the low bits of static_call_site::key */
696	sec->sh.sh_flags \|= SHF_WRITE;
697
698	idx = 0;
699	list_for_each_entry(insn, &file->static_call_list, call_node) {
700
701	/* populate reloc for 'addr' */
702	if (!elf_init_reloc_text_sym(file->elf, sec,
703	idx * sizeof(site), idx 2,
704	insn->sec, insn->offset))
705	return -1;
706
707	/* find key symbol */
708	key_name = strdup(insn_call_dest(insn)->name);
709	if (!key_name) {
710	ERROR_GLIBC("strdup");
711	return -1;
712	}
713	if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
714	STATIC_CALL_TRAMP_PREFIX_LEN)) {
715	ERROR("static_call: trampoline name malformed: %s", key_name);
716	return -1;
717	}
718	tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
719	memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
720
721	key_sym = find_symbol_by_name(file->elf, tmp);
722	if (!key_sym) {
723	if (!opts.module) {
724	ERROR("static_call: can't find static_call_key symbol: %s", tmp);
725	return -1;
726	}
727
728	/*
729	* For modules(), the key might not be exported, which
730	* means the module can make static calls but isn't
731	* allowed to change them.
732	*
733	* In that case we temporarily set the key to be the
734	* trampoline address. This is fixed up in
735	* static_call_add_module().
736	*/
737	key_sym = insn_call_dest(insn);
738	}
739
740	/* populate reloc for 'key' */
741	if (!elf_init_reloc_data_sym(file->elf, sec,
742	idx * sizeof(*site) + 4,
743	(idx * 2) + 1, key_sym,
744	is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
745	return -1;
746
747	idx++;
748	}
749
750	return 0;
751	}
752
753	static int create_retpoline_sites_sections(struct objtool_file *file)
754	{
755	struct instruction *insn;
756	struct section *sec;
757	int idx;
758
759	sec = find_section_by_name(file->elf, ".retpoline_sites");
760	if (sec) {
761	WARN("file already has .retpoline_sites, skipping");
762	return 0;
763	}
764
765	idx = 0;
766	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
767	idx++;
768
769	if (!idx)
770	return 0;
771
772	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
773	sizeof(int), idx, idx);
774	if (!sec)
775	return -1;
776
777	idx = 0;
778	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
779
780	if (!elf_init_reloc_text_sym(file->elf, sec,
781	idx * sizeof(int), idx,
782	insn->sec, insn->offset))
783	return -1;
784
785	idx++;
786	}
787
788	return 0;
789	}
790
791	static int create_return_sites_sections(struct objtool_file *file)
792	{
793	struct instruction *insn;
794	struct section *sec;
795	int idx;
796
797	sec = find_section_by_name(file->elf, ".return_sites");
798	if (sec) {
799	WARN("file already has .return_sites, skipping");
800	return 0;
801	}
802
803	idx = 0;
804	list_for_each_entry(insn, &file->return_thunk_list, call_node)
805	idx++;
806
807	if (!idx)
808	return 0;
809
810	sec = elf_create_section_pair(file->elf, ".return_sites",
811	sizeof(int), idx, idx);
812	if (!sec)
813	return -1;
814
815	idx = 0;
816	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
817
818	if (!elf_init_reloc_text_sym(file->elf, sec,
819	idx * sizeof(int), idx,
820	insn->sec, insn->offset))
821	return -1;
822
823	idx++;
824	}
825
826	return 0;
827	}
828
829	static int create_ibt_endbr_seal_sections(struct objtool_file *file)
830	{
831	struct instruction *insn;
832	struct section *sec;
833	int idx;
834
835	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
836	if (sec) {
837	WARN("file already has .ibt_endbr_seal, skipping");
838	return 0;
839	}
840
841	idx = 0;
842	list_for_each_entry(insn, &file->endbr_list, call_node)
843	idx++;
844
845	if (opts.stats) {
846	printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
847	printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
848	printf("ibt: superfluous ENDBR: %d\n", idx);
849	}
850
851	if (!idx)
852	return 0;
853
854	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
855	sizeof(int), idx, idx);
856	if (!sec)
857	return -1;
858
859	idx = 0;
860	list_for_each_entry(insn, &file->endbr_list, call_node) {
861
862	int site = (int )sec->data->d_buf + idx;
863	struct symbol *func = insn_func(insn);
864	*site = 0;
865
866	if (opts.module && func && insn->offset == func->offset &&
867	(!strcmp(func->name, "init_module") \|\|
868	!strcmp(func->name, "cleanup_module"))) {
869	ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead",
870	func->name);
871	return -1;
872	}
873
874	if (!elf_init_reloc_text_sym(file->elf, sec,
875	idx * sizeof(int), idx,
876	insn->sec, insn->offset))
877	return -1;
878
879	idx++;
880	}
881
882	return 0;
883	}
884
885	/*
886	* Grow __cfi_ symbols to fill the NOP gap between the 'mov <hash>, %rax' and
887	* the start of the function.
888	*/
889	static int grow_cfi_symbols(struct objtool_file *file)
890	{
891	struct symbol *sym;
892
893	for_each_sym(file->elf, sym) {
894	if (!is_func_sym(sym) \|\| !strstarts(sym->name, "__cfi_") \|\|
895	sym->len != 5)
896	continue;
897
898	if (!find_func_by_offset(sym->sec, sym->offset + sym->len + opts.prefix))
899	continue;
900
901	sym->len += opts.prefix;
902	sym->sym.st_size = sym->len;
903	if (elf_write_symbol(file->elf, sym))
904	return -1;
905	}
906
907	return 0;
908	}
909
910	static int create_cfi_sections(struct objtool_file *file)
911	{
912	struct section *sec;
913	struct symbol *sym;
914	int idx;
915
916	sec = find_section_by_name(file->elf, ".cfi_sites");
917	if (sec) {
918	WARN("file already has .cfi_sites section, skipping");
919	return 0;
920	}
921
922	idx = 0;
923	for_each_sym(file->elf, sym) {
924	if (!is_func_sym(sym))
925	continue;
926
927	if (strncmp(sym->name, "__cfi_", 6))
928	continue;
929
930	idx++;
931	}
932
933	sec = elf_create_section_pair(file->elf, ".cfi_sites",
934	sizeof(unsigned int), idx, idx);
935	if (!sec)
936	return -1;
937
938	idx = 0;
939	for_each_sym(file->elf, sym) {
940	if (!is_func_sym(sym))
941	continue;
942
943	if (strncmp(sym->name, "__cfi_", 6))
944	continue;
945
946	if (!elf_init_reloc_text_sym(file->elf, sec,
947	idx * sizeof(unsigned int), idx,
948	sym->sec, sym->offset))
949	return -1;
950
951	idx++;
952	}
953
954	return 0;
955	}
956
957	static int create_mcount_loc_sections(struct objtool_file *file)
958	{
959	size_t addr_size = elf_addr_size(file->elf);
960	struct instruction *insn;
961	struct section *sec;
962	int idx;
963
964	sec = find_section_by_name(file->elf, "__mcount_loc");
965	if (sec) {
966	/*
967	* Livepatch modules have already extracted their __mcount_loc
968	* entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case.
969	*/
970	if (!file->klp)
971	WARN("file already has __mcount_loc section, skipping");
972
973	return 0;
974	}
975
976	if (list_empty(&file->mcount_loc_list))
977	return 0;
978
979	idx = 0;
980	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
981	idx++;
982
983	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
984	idx, idx);
985	if (!sec)
986	return -1;
987
988	sec->sh.sh_addralign = addr_size;
989
990	idx = 0;
991	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
992
993	struct reloc *reloc;
994
995	reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
996	insn->sec, insn->offset);
997	if (!reloc)
998	return -1;
999
1000	set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
1001
1002	idx++;
1003	}
1004
1005	return 0;
1006	}
1007
1008	static int create_direct_call_sections(struct objtool_file *file)
1009	{
1010	struct instruction *insn;
1011	struct section *sec;
1012	int idx;
1013
1014	sec = find_section_by_name(file->elf, ".call_sites");
1015	if (sec) {
1016	WARN("file already has .call_sites section, skipping");
1017	return 0;
1018	}
1019
1020	if (list_empty(&file->call_list))
1021	return 0;
1022
1023	idx = 0;
1024	list_for_each_entry(insn, &file->call_list, call_node)
1025	idx++;
1026
1027	sec = elf_create_section_pair(file->elf, ".call_sites",
1028	sizeof(unsigned int), idx, idx);
1029	if (!sec)
1030	return -1;
1031
1032	idx = 0;
1033	list_for_each_entry(insn, &file->call_list, call_node) {
1034
1035	if (!elf_init_reloc_text_sym(file->elf, sec,
1036	idx * sizeof(unsigned int), idx,
1037	insn->sec, insn->offset))
1038	return -1;
1039
1040	idx++;
1041	}
1042
1043	return 0;
1044	}
1045
1046	/*
1047	* Warnings shouldn't be reported for ignored functions.
1048	*/
1049	static int add_ignores(struct objtool_file *file)
1050	{
1051	struct section *rsec;
1052	struct symbol *func;
1053	struct reloc *reloc;
1054
1055	rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1056	if (!rsec)
1057	return 0;
1058
1059	for_each_reloc(rsec, reloc) {
1060	switch (reloc->sym->type) {
1061	case STT_FUNC:
1062	func = reloc->sym;
1063	break;
1064
1065	case STT_SECTION:
1066	func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1067	if (!func)
1068	continue;
1069	break;
1070
1071	default:
1072	ERROR("unexpected relocation symbol type in %s: %d",
1073	rsec->name, reloc->sym->type);
1074	return -1;
1075	}
1076
1077	func->ignore = true;
1078	if (func->cfunc)
1079	func->cfunc->ignore = true;
1080	}
1081
1082	return 0;
1083	}
1084
1085	/*
1086	* This is a whitelist of functions that is allowed to be called with AC set.
1087	* The list is meant to be minimal and only contains compiler instrumentation
1088	* ABI and a few functions used to implement *_{to,from}_user() functions.
1089	*
1090	* These functions must not directly change AC, but may PUSHF/POPF.
1091	*/
1092	static const char *uaccess_safe_builtin[] = {
1093	/* KASAN */
1094	"kasan_report",
1095	"kasan_check_range",
1096	/* KASAN out-of-line */
1097	"__asan_loadN_noabort",
1098	"__asan_load1_noabort",
1099	"__asan_load2_noabort",
1100	"__asan_load4_noabort",
1101	"__asan_load8_noabort",
1102	"__asan_load16_noabort",
1103	"__asan_storeN_noabort",
1104	"__asan_store1_noabort",
1105	"__asan_store2_noabort",
1106	"__asan_store4_noabort",
1107	"__asan_store8_noabort",
1108	"__asan_store16_noabort",
1109	"__kasan_check_read",
1110	"__kasan_check_write",
1111	/* KASAN in-line */
1112	"__asan_report_load_n_noabort",
1113	"__asan_report_load1_noabort",
1114	"__asan_report_load2_noabort",
1115	"__asan_report_load4_noabort",
1116	"__asan_report_load8_noabort",
1117	"__asan_report_load16_noabort",
1118	"__asan_report_store_n_noabort",
1119	"__asan_report_store1_noabort",
1120	"__asan_report_store2_noabort",
1121	"__asan_report_store4_noabort",
1122	"__asan_report_store8_noabort",
1123	"__asan_report_store16_noabort",
1124	/* KCSAN */
1125	"__kcsan_check_access",
1126	"__kcsan_mb",
1127	"__kcsan_wmb",
1128	"__kcsan_rmb",
1129	"__kcsan_release",
1130	"kcsan_found_watchpoint",
1131	"kcsan_setup_watchpoint",
1132	"kcsan_check_scoped_accesses",
1133	"kcsan_disable_current",
1134	"kcsan_enable_current_nowarn",
1135	/* KCSAN/TSAN */
1136	"__tsan_func_entry",
1137	"__tsan_func_exit",
1138	"__tsan_read_range",
1139	"__tsan_write_range",
1140	"__tsan_read1",
1141	"__tsan_read2",
1142	"__tsan_read4",
1143	"__tsan_read8",
1144	"__tsan_read16",
1145	"__tsan_write1",
1146	"__tsan_write2",
1147	"__tsan_write4",
1148	"__tsan_write8",
1149	"__tsan_write16",
1150	"__tsan_read_write1",
1151	"__tsan_read_write2",
1152	"__tsan_read_write4",
1153	"__tsan_read_write8",
1154	"__tsan_read_write16",
1155	"__tsan_volatile_read1",
1156	"__tsan_volatile_read2",
1157	"__tsan_volatile_read4",
1158	"__tsan_volatile_read8",
1159	"__tsan_volatile_read16",
1160	"__tsan_volatile_write1",
1161	"__tsan_volatile_write2",
1162	"__tsan_volatile_write4",
1163	"__tsan_volatile_write8",
1164	"__tsan_volatile_write16",
1165	"__tsan_atomic8_load",
1166	"__tsan_atomic16_load",
1167	"__tsan_atomic32_load",
1168	"__tsan_atomic64_load",
1169	"__tsan_atomic8_store",
1170	"__tsan_atomic16_store",
1171	"__tsan_atomic32_store",
1172	"__tsan_atomic64_store",
1173	"__tsan_atomic8_exchange",
1174	"__tsan_atomic16_exchange",
1175	"__tsan_atomic32_exchange",
1176	"__tsan_atomic64_exchange",
1177	"__tsan_atomic8_fetch_add",
1178	"__tsan_atomic16_fetch_add",
1179	"__tsan_atomic32_fetch_add",
1180	"__tsan_atomic64_fetch_add",
1181	"__tsan_atomic8_fetch_sub",
1182	"__tsan_atomic16_fetch_sub",
1183	"__tsan_atomic32_fetch_sub",
1184	"__tsan_atomic64_fetch_sub",
1185	"__tsan_atomic8_fetch_and",
1186	"__tsan_atomic16_fetch_and",
1187	"__tsan_atomic32_fetch_and",
1188	"__tsan_atomic64_fetch_and",
1189	"__tsan_atomic8_fetch_or",
1190	"__tsan_atomic16_fetch_or",
1191	"__tsan_atomic32_fetch_or",
1192	"__tsan_atomic64_fetch_or",
1193	"__tsan_atomic8_fetch_xor",
1194	"__tsan_atomic16_fetch_xor",
1195	"__tsan_atomic32_fetch_xor",
1196	"__tsan_atomic64_fetch_xor",
1197	"__tsan_atomic8_fetch_nand",
1198	"__tsan_atomic16_fetch_nand",
1199	"__tsan_atomic32_fetch_nand",
1200	"__tsan_atomic64_fetch_nand",
1201	"__tsan_atomic8_compare_exchange_strong",
1202	"__tsan_atomic16_compare_exchange_strong",
1203	"__tsan_atomic32_compare_exchange_strong",
1204	"__tsan_atomic64_compare_exchange_strong",
1205	"__tsan_atomic8_compare_exchange_weak",
1206	"__tsan_atomic16_compare_exchange_weak",
1207	"__tsan_atomic32_compare_exchange_weak",
1208	"__tsan_atomic64_compare_exchange_weak",
1209	"__tsan_atomic8_compare_exchange_val",
1210	"__tsan_atomic16_compare_exchange_val",
1211	"__tsan_atomic32_compare_exchange_val",
1212	"__tsan_atomic64_compare_exchange_val",
1213	"__tsan_atomic_thread_fence",
1214	"__tsan_atomic_signal_fence",
1215	"__tsan_unaligned_read16",
1216	"__tsan_unaligned_write16",
1217	/* KCOV */
1218	"write_comp_data",
1219	"check_kcov_mode",
1220	"__sanitizer_cov_trace_pc",
1221	"__sanitizer_cov_trace_const_cmp1",
1222	"__sanitizer_cov_trace_const_cmp2",
1223	"__sanitizer_cov_trace_const_cmp4",
1224	"__sanitizer_cov_trace_const_cmp8",
1225	"__sanitizer_cov_trace_cmp1",
1226	"__sanitizer_cov_trace_cmp2",
1227	"__sanitizer_cov_trace_cmp4",
1228	"__sanitizer_cov_trace_cmp8",
1229	"__sanitizer_cov_trace_switch",
1230	/* KMSAN */
1231	"kmsan_copy_to_user",
1232	"kmsan_disable_current",
1233	"kmsan_enable_current",
1234	"kmsan_report",
1235	"kmsan_unpoison_entry_regs",
1236	"kmsan_unpoison_memory",
1237	"__msan_chain_origin",
1238	"__msan_get_context_state",
1239	"__msan_instrument_asm_store",
1240	"__msan_metadata_ptr_for_load_1",
1241	"__msan_metadata_ptr_for_load_2",
1242	"__msan_metadata_ptr_for_load_4",
1243	"__msan_metadata_ptr_for_load_8",
1244	"__msan_metadata_ptr_for_load_n",
1245	"__msan_metadata_ptr_for_store_1",
1246	"__msan_metadata_ptr_for_store_2",
1247	"__msan_metadata_ptr_for_store_4",
1248	"__msan_metadata_ptr_for_store_8",
1249	"__msan_metadata_ptr_for_store_n",
1250	"__msan_poison_alloca",
1251	"__msan_warning",
1252	/* UBSAN */
1253	"ubsan_type_mismatch_common",
1254	"__ubsan_handle_type_mismatch",
1255	"__ubsan_handle_type_mismatch_v1",
1256	"__ubsan_handle_shift_out_of_bounds",
1257	"__ubsan_handle_load_invalid_value",
1258	/* KSTACK_ERASE */
1259	"__sanitizer_cov_stack_depth",
1260	/* TRACE_BRANCH_PROFILING */
1261	"ftrace_likely_update",
1262	/* STACKPROTECTOR */
1263	"__stack_chk_fail",
1264	/* misc */
1265	"csum_partial_copy_generic",
1266	"copy_mc_fragile",
1267	"copy_mc_fragile_handle_tail",
1268	"copy_mc_enhanced_fast_string",
1269	"rep_stos_alternative",
1270	"rep_movs_alternative",
1271	"copy_to_nontemporal",
1272	NULL
1273	};
1274
1275	static void add_uaccess_safe(struct objtool_file *file)
1276	{
1277	struct symbol *func;
1278	const char **name;
1279
1280	if (!opts.uaccess)
1281	return;
1282
1283	for (name = uaccess_safe_builtin; *name; name++) {
1284	func = find_symbol_by_name(file->elf, *name);
1285	if (!func)
1286	continue;
1287
1288	func->uaccess_safe = true;
1289	}
1290	}
1291
1292	/*
1293	* Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1294	* will be added to the .retpoline_sites section.
1295	*/
1296	__weak bool arch_is_retpoline(struct symbol *sym)
1297	{
1298	return false;
1299	}
1300
1301	/*
1302	* Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1303	* will be added to the .return_sites section.
1304	*/
1305	__weak bool arch_is_rethunk(struct symbol *sym)
1306	{
1307	return false;
1308	}
1309
1310	/*
1311	* Symbols that are embedded inside other instructions, because sometimes crazy
1312	* code exists. These are mostly ignored for validation purposes.
1313	*/
1314	__weak bool arch_is_embedded_insn(struct symbol *sym)
1315	{
1316	return false;
1317	}
1318
1319	struct reloc insn_reloc(struct objtool_file file, struct instruction *insn)
1320	{
1321	struct reloc *reloc;
1322
1323	if (!file \|\| insn->no_reloc \|\| insn->fake)
1324	return NULL;
1325
1326	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1327	insn->offset, insn->len);
1328	if (!reloc) {
1329	insn->no_reloc = 1;
1330	return NULL;
1331	}
1332
1333	return reloc;
1334	}
1335
1336	static void remove_insn_ops(struct instruction *insn)
1337	{
1338	struct stack_op op, next;
1339
1340	for (op = insn->stack_ops; op; op = next) {
1341	next = op->next;
1342	free(op);
1343	}
1344	insn->stack_ops = NULL;
1345	}
1346
1347	static int annotate_call_site(struct objtool_file *file,
1348	struct instruction *insn, bool sibling)
1349	{
1350	struct reloc *reloc = insn_reloc(file, insn);
1351	struct symbol *sym = insn_call_dest(insn);
1352
1353	if (!sym)
1354	sym = reloc->sym;
1355
1356	if (sym->static_call_tramp) {
1357	list_add_tail(&insn->call_node, &file->static_call_list);
1358	return 0;
1359	}
1360
1361	if (sym->retpoline_thunk) {
1362	list_add_tail(&insn->call_node, &file->retpoline_call_list);
1363	return 0;
1364	}
1365
1366	/*
1367	* Many compilers cannot disable KCOV or sanitizer calls with a function
1368	* attribute so they need a little help, NOP out any such calls from
1369	* noinstr text.
1370	*/
1371	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1372	if (reloc)
1373	set_reloc_type(file->elf, reloc, R_NONE);
1374
1375	if (elf_write_insn(file->elf, insn->sec,
1376	insn->offset, insn->len,
1377	sibling ? arch_ret_insn(insn->len)
1378	: arch_nop_insn(insn->len))) {
1379	return -1;
1380	}
1381
1382	insn->type = sibling ? INSN_RETURN : INSN_NOP;
1383
1384	if (sibling) {
1385	/*
1386	* We've replaced the tail-call JMP insn by two new
1387	* insn: RET; INT3, except we only have a single struct
1388	* insn here. Mark it retpoline_safe to avoid the SLS
1389	* warning, instead of adding another insn.
1390	*/
1391	insn->retpoline_safe = true;
1392	}
1393
1394	return 0;
1395	}
1396
1397	if (opts.mcount && sym->fentry) {
1398	if (sibling)
1399	WARN_INSN(insn, "tail call to __fentry__ !?!?");
1400	if (opts.mnop) {
1401	if (reloc)
1402	set_reloc_type(file->elf, reloc, R_NONE);
1403
1404	if (elf_write_insn(file->elf, insn->sec,
1405	insn->offset, insn->len,
1406	arch_nop_insn(insn->len))) {
1407	return -1;
1408	}
1409
1410	insn->type = INSN_NOP;
1411	}
1412
1413	list_add_tail(&insn->call_node, &file->mcount_loc_list);
1414	return 0;
1415	}
1416
1417	if (insn->type == INSN_CALL && !insn->sec->init &&
1418	!insn->_call_dest->embedded_insn)
1419	list_add_tail(&insn->call_node, &file->call_list);
1420
1421	if (!sibling && dead_end_function(file, sym))
1422	insn->dead_end = true;
1423
1424	return 0;
1425	}
1426
1427	static int add_call_dest(struct objtool_file file, struct instruction insn,
1428	struct symbol *dest, bool sibling)
1429	{
1430	insn->_call_dest = dest;
1431	if (!dest)
1432	return 0;
1433
1434	/*
1435	* Whatever stack impact regular CALLs have, should be undone
1436	* by the RETURN of the called function.
1437	*
1438	* Annotated intra-function calls retain the stack_ops but
1439	* are converted to JUMP, see read_intra_function_calls().
1440	*/
1441	remove_insn_ops(insn);
1442
1443	return annotate_call_site(file, insn, sibling);
1444	}
1445
1446	static int add_retpoline_call(struct objtool_file file, struct instruction insn)
1447	{
1448	/*
1449	* Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1450	* so convert them accordingly.
1451	*/
1452	switch (insn->type) {
1453	case INSN_CALL:
1454	insn->type = INSN_CALL_DYNAMIC;
1455	break;
1456	case INSN_JUMP_UNCONDITIONAL:
1457	insn->type = INSN_JUMP_DYNAMIC;
1458	break;
1459	case INSN_JUMP_CONDITIONAL:
1460	insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1461	break;
1462	default:
1463	return 0;
1464	}
1465
1466	insn->retpoline_safe = true;
1467
1468	/*
1469	* Whatever stack impact regular CALLs have, should be undone
1470	* by the RETURN of the called function.
1471	*
1472	* Annotated intra-function calls retain the stack_ops but
1473	* are converted to JUMP, see read_intra_function_calls().
1474	*/
1475	remove_insn_ops(insn);
1476
1477	return annotate_call_site(file, insn, false);
1478	}
1479
1480	static void add_return_call(struct objtool_file file, struct instruction insn, bool add)
1481	{
1482	/*
1483	* Return thunk tail calls are really just returns in disguise,
1484	* so convert them accordingly.
1485	*/
1486	insn->type = INSN_RETURN;
1487	insn->retpoline_safe = true;
1488
1489	if (add)
1490	list_add_tail(&insn->call_node, &file->return_thunk_list);
1491	}
1492
1493	static bool is_first_func_insn(struct objtool_file *file,
1494	struct instruction *insn)
1495	{
1496	struct symbol *func = insn_func(insn);
1497
1498	if (!func)
1499	return false;
1500
1501	if (insn->offset == func->offset)
1502	return true;
1503
1504	/* Allow direct CALL/JMP past ENDBR */
1505	if (opts.ibt) {
1506	struct instruction *prev = prev_insn_same_sym(file, insn);
1507
1508	if (prev && prev->type == INSN_ENDBR &&
1509	insn->offset == func->offset + prev->len)
1510	return true;
1511	}
1512
1513	return false;
1514	}
1515
1516	/*
1517	* Find the destination instructions for all jumps.
1518	*/
1519	static int add_jump_destinations(struct objtool_file *file)
1520	{
1521	struct instruction *insn;
1522	struct reloc *reloc;
1523
1524	for_each_insn(file, insn) {
1525	struct symbol *func = insn_func(insn);
1526	struct instruction *dest_insn;
1527	struct section *dest_sec;
1528	struct symbol *dest_sym;
1529	unsigned long dest_off;
1530
1531	if (!is_static_jump(insn))
1532	continue;
1533
1534	if (insn->jump_dest) {
1535	/*
1536	* handle_group_alt() may have previously set
1537	* 'jump_dest' for some alternatives.
1538	*/
1539	continue;
1540	}
1541
1542	reloc = insn_reloc(file, insn);
1543	if (!reloc) {
1544	dest_sec = insn->sec;
1545	dest_off = arch_jump_destination(insn);
1546	dest_sym = dest_sec->sym;
1547	} else {
1548	dest_sym = reloc->sym;
1549	if (is_undef_sym(dest_sym)) {
1550	if (dest_sym->retpoline_thunk) {
1551	if (add_retpoline_call(file, insn))
1552	return -1;
1553	continue;
1554	}
1555
1556	if (dest_sym->return_thunk) {
1557	add_return_call(file, insn, true);
1558	continue;
1559	}
1560
1561	/* External symbol */
1562	if (func) {
1563	/* External sibling call */
1564	if (add_call_dest(file, insn, dest_sym, true))
1565	return -1;
1566	continue;
1567	}
1568
1569	/* Non-func asm code jumping to external symbol */
1570	continue;
1571	}
1572
1573	dest_sec = dest_sym->sec;
1574	dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc);
1575	}
1576
1577	dest_insn = find_insn(file, dest_sec, dest_off);
1578	if (!dest_insn) {
1579	struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1580
1581	/*
1582	* retbleed_untrain_ret() jumps to
1583	* __x86_return_thunk(), but objtool can't find
1584	* the thunk's starting RET instruction,
1585	* because the RET is also in the middle of
1586	* another instruction. Objtool only knows
1587	* about the outer instruction.
1588	*/
1589	if (sym && sym->embedded_insn) {
1590	add_return_call(file, insn, false);
1591	continue;
1592	}
1593
1594	/*
1595	* GCOV/KCOV dead code can jump to the end of
1596	* the function/section.
1597	*/
1598	if (file->ignore_unreachables && func &&
1599	dest_sec == insn->sec &&
1600	dest_off == func->offset + func->len)
1601	continue;
1602
1603	ERROR_INSN(insn, "can't find jump dest instruction at %s",
1604	offstr(dest_sec, dest_off));
1605	return -1;
1606	}
1607
1608	if (!dest_sym \|\| is_sec_sym(dest_sym)) {
1609	dest_sym = insn_sym(dest_insn);
1610	if (!dest_sym)
1611	goto set_jump_dest;
1612	}
1613
1614	if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) {
1615	if (add_retpoline_call(file, insn))
1616	return -1;
1617	continue;
1618	}
1619
1620	if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) {
1621	add_return_call(file, insn, true);
1622	continue;
1623	}
1624
1625	if (!insn_sym(insn) \|\| insn_sym(insn)->pfunc == dest_sym->pfunc)
1626	goto set_jump_dest;
1627
1628	/*
1629	* Internal cross-function jump.
1630	*/
1631
1632	if (is_first_func_insn(file, dest_insn)) {
1633	/* Internal sibling call */
1634	if (add_call_dest(file, insn, dest_sym, true))
1635	return -1;
1636	continue;
1637	}
1638
1639	set_jump_dest:
1640	insn->jump_dest = dest_insn;
1641	}
1642
1643	return 0;
1644	}
1645
1646	static struct symbol find_call_destination(struct section sec, unsigned long offset)
1647	{
1648	struct symbol *call_dest;
1649
1650	call_dest = find_func_by_offset(sec, offset);
1651	if (!call_dest)
1652	call_dest = find_symbol_by_offset(sec, offset);
1653
1654	return call_dest;
1655	}
1656
1657	/*
1658	* Find the destination instructions for all calls.
1659	*/
1660	static int add_call_destinations(struct objtool_file *file)
1661	{
1662	struct instruction *insn;
1663	unsigned long dest_off;
1664	struct symbol *dest;
1665	struct reloc *reloc;
1666
1667	for_each_insn(file, insn) {
1668	struct symbol *func = insn_func(insn);
1669	if (insn->type != INSN_CALL)
1670	continue;
1671
1672	reloc = insn_reloc(file, insn);
1673	if (!reloc) {
1674	dest_off = arch_jump_destination(insn);
1675	dest = find_call_destination(insn->sec, dest_off);
1676
1677	if (add_call_dest(file, insn, dest, false))
1678	return -1;
1679
1680	if (func && func->ignore)
1681	continue;
1682
1683	if (!insn_call_dest(insn)) {
1684	ERROR_INSN(insn, "unannotated intra-function call");
1685	return -1;
1686	}
1687
1688	if (func && !is_func_sym(insn_call_dest(insn))) {
1689	ERROR_INSN(insn, "unsupported call to non-function");
1690	return -1;
1691	}
1692
1693	} else if (is_sec_sym(reloc->sym)) {
1694	dest_off = arch_insn_adjusted_addend(insn, reloc);
1695	dest = find_call_destination(reloc->sym->sec, dest_off);
1696	if (!dest) {
1697	ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1698	reloc->sym->sec->name, dest_off);
1699	return -1;
1700	}
1701
1702	if (add_call_dest(file, insn, dest, false))
1703	return -1;
1704
1705	} else if (reloc->sym->retpoline_thunk) {
1706	if (add_retpoline_call(file, insn))
1707	return -1;
1708
1709	} else {
1710	if (add_call_dest(file, insn, reloc->sym, false))
1711	return -1;
1712	}
1713	}
1714
1715	return 0;
1716	}
1717
1718	/*
1719	* The .alternatives section requires some extra special care over and above
1720	* other special sections because alternatives are patched in place.
1721	*/
1722	static int handle_group_alt(struct objtool_file *file,
1723	struct special_alt *special_alt,
1724	struct instruction *orig_insn,
1725	struct instruction **new_insn)
1726	{
1727	struct instruction last_new_insn = NULL, insn, *nop = NULL;
1728	struct alt_group orig_alt_group, new_alt_group;
1729	unsigned long dest_off;
1730
1731	orig_alt_group = orig_insn->alt_group;
1732	if (!orig_alt_group) {
1733	struct instruction *last_orig_insn = NULL;
1734
1735	orig_alt_group = calloc(1, sizeof(*orig_alt_group));
1736	if (!orig_alt_group) {
1737	ERROR_GLIBC("calloc");
1738	return -1;
1739	}
1740	orig_alt_group->cfi = calloc(special_alt->orig_len,
1741	sizeof(struct cfi_state *));
1742	if (!orig_alt_group->cfi) {
1743	ERROR_GLIBC("calloc");
1744	return -1;
1745	}
1746
1747	insn = orig_insn;
1748	sec_for_each_insn_from(file, insn) {
1749	if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1750	break;
1751
1752	insn->alt_group = orig_alt_group;
1753	last_orig_insn = insn;
1754	}
1755	orig_alt_group->orig_group = NULL;
1756	orig_alt_group->first_insn = orig_insn;
1757	orig_alt_group->last_insn = last_orig_insn;
1758	orig_alt_group->nop = NULL;
1759	orig_alt_group->ignore = orig_insn->ignore_alts;
1760	orig_alt_group->feature = 0;
1761	} else {
1762	if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1763	orig_alt_group->first_insn->offset != special_alt->orig_len) {
1764	ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1765	orig_alt_group->last_insn->offset +
1766	orig_alt_group->last_insn->len -
1767	orig_alt_group->first_insn->offset,
1768	special_alt->orig_len);
1769	return -1;
1770	}
1771	}
1772
1773	new_alt_group = calloc(1, sizeof(*new_alt_group));
1774	if (!new_alt_group) {
1775	ERROR_GLIBC("calloc");
1776	return -1;
1777	}
1778
1779	if (special_alt->new_len < special_alt->orig_len) {
1780	/*
1781	* Insert a fake nop at the end to make the replacement
1782	* alt_group the same size as the original. This is needed to
1783	* allow propagate_alt_cfi() to do its magic. When the last
1784	* instruction affects the stack, the instruction after it (the
1785	* nop) will propagate the new state to the shared CFI array.
1786	*/
1787	nop = calloc(1, sizeof(*nop));
1788	if (!nop) {
1789	ERROR_GLIBC("calloc");
1790	return -1;
1791	}
1792	memset(nop, 0, sizeof(*nop));
1793
1794	nop->sec = special_alt->new_sec;
1795	nop->offset = special_alt->new_off + special_alt->new_len;
1796	nop->len = special_alt->orig_len - special_alt->new_len;
1797	nop->type = INSN_NOP;
1798	nop->alt_group = new_alt_group;
1799	nop->fake = 1;
1800	}
1801
1802	if (!special_alt->new_len) {
1803	*new_insn = nop;
1804	goto end;
1805	}
1806
1807	insn = *new_insn;
1808	sec_for_each_insn_from(file, insn) {
1809	struct reloc *alt_reloc;
1810
1811	if (insn->offset >= special_alt->new_off + special_alt->new_len)
1812	break;
1813
1814	last_new_insn = insn;
1815
1816	insn->alt_group = new_alt_group;
1817
1818	/*
1819	* Since alternative replacement code is copy/pasted by the
1820	* kernel after applying relocations, generally such code can't
1821	* have relative-address relocation references to outside the
1822	* .altinstr_replacement section, unless the arch's
1823	* alternatives code can adjust the relative offsets
1824	* accordingly.
1825	*/
1826	alt_reloc = insn_reloc(file, insn);
1827	if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1828	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1829
1830	ERROR_INSN(insn, "unsupported relocation in alternatives section");
1831	return -1;
1832	}
1833
1834	if (!is_static_jump(insn))
1835	continue;
1836
1837	if (!insn->immediate)
1838	continue;
1839
1840	dest_off = arch_jump_destination(insn);
1841	if (dest_off == special_alt->new_off + special_alt->new_len) {
1842	insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1843	if (!insn->jump_dest) {
1844	ERROR_INSN(insn, "can't find alternative jump destination");
1845	return -1;
1846	}
1847	}
1848	}
1849
1850	if (!last_new_insn) {
1851	ERROR_FUNC(special_alt->new_sec, special_alt->new_off,
1852	"can't find last new alternative instruction");
1853	return -1;
1854	}
1855
1856	end:
1857	new_alt_group->orig_group = orig_alt_group;
1858	new_alt_group->first_insn = *new_insn;
1859	new_alt_group->last_insn = last_new_insn;
1860	new_alt_group->nop = nop;
1861	new_alt_group->ignore = (*new_insn)->ignore_alts;
1862	new_alt_group->cfi = orig_alt_group->cfi;
1863	new_alt_group->feature = special_alt->feature;
1864	return 0;
1865	}
1866
1867	/*
1868	* A jump table entry can either convert a nop to a jump or a jump to a nop.
1869	* If the original instruction is a jump, make the alt entry an effective nop
1870	* by just skipping the original instruction.
1871	*/
1872	static int handle_jump_alt(struct objtool_file *file,
1873	struct special_alt *special_alt,
1874	struct instruction *orig_insn,
1875	struct instruction **new_insn)
1876	{
1877	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1878	orig_insn->type != INSN_NOP) {
1879
1880	ERROR_INSN(orig_insn, "unsupported instruction at jump label");
1881	return -1;
1882	}
1883
1884	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1885	struct reloc *reloc = insn_reloc(file, orig_insn);
1886
1887	if (reloc)
1888	set_reloc_type(file->elf, reloc, R_NONE);
1889
1890	if (elf_write_insn(file->elf, orig_insn->sec,
1891	orig_insn->offset, orig_insn->len,
1892	arch_nop_insn(orig_insn->len))) {
1893	return -1;
1894	}
1895
1896	orig_insn->type = INSN_NOP;
1897	}
1898
1899	if (orig_insn->type == INSN_NOP) {
1900	if (orig_insn->len == 2)
1901	file->jl_nop_short++;
1902	else
1903	file->jl_nop_long++;
1904
1905	return 0;
1906	}
1907
1908	if (orig_insn->len == 2)
1909	file->jl_short++;
1910	else
1911	file->jl_long++;
1912
1913	*new_insn = next_insn_same_sec(file, orig_insn);
1914	return 0;
1915	}
1916
1917	/*
1918	* Read all the special sections which have alternate instructions which can be
1919	* patched in or redirected to at runtime. Each instruction having alternate
1920	* instruction(s) has them added to its insn->alts list, which will be
1921	* traversed in validate_branch().
1922	*/
1923	static int add_special_section_alts(struct objtool_file *file)
1924	{
1925	struct list_head special_alts;
1926	struct instruction orig_insn, new_insn;
1927	struct special_alt special_alt, tmp;
1928	enum alternative_type alt_type;
1929	struct alternative *alt;
1930	struct alternative *a;
1931
1932	if (special_get_alts(file->elf, &special_alts))
1933	return -1;
1934
1935	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1936
1937	orig_insn = find_insn(file, special_alt->orig_sec,
1938	special_alt->orig_off);
1939	if (!orig_insn) {
1940	ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off,
1941	"special: can't find orig instruction");
1942	return -1;
1943	}
1944
1945	new_insn = NULL;
1946	if (!special_alt->group \|\| special_alt->new_len) {
1947	new_insn = find_insn(file, special_alt->new_sec,
1948	special_alt->new_off);
1949	if (!new_insn) {
1950	ERROR_FUNC(special_alt->new_sec, special_alt->new_off,
1951	"special: can't find new instruction");
1952	return -1;
1953	}
1954	}
1955
1956	if (special_alt->group) {
1957	if (!special_alt->orig_len) {
1958	ERROR_INSN(orig_insn, "empty alternative entry");
1959	continue;
1960	}
1961
1962	if (handle_group_alt(file, special_alt, orig_insn, &new_insn))
1963	return -1;
1964
1965	alt_type = ALT_TYPE_INSTRUCTIONS;
1966
1967	} else if (special_alt->jump_or_nop) {
1968	if (handle_jump_alt(file, special_alt, orig_insn, &new_insn))
1969	return -1;
1970
1971	alt_type = ALT_TYPE_JUMP_TABLE;
1972	} else {
1973	alt_type = ALT_TYPE_EX_TABLE;
1974	}
1975
1976	alt = calloc(1, sizeof(*alt));
1977	if (!alt) {
1978	ERROR_GLIBC("calloc");
1979	return -1;
1980	}
1981
1982	alt->insn = new_insn;
1983	alt->type = alt_type;
1984	alt->next = NULL;
1985
1986	/*
1987	* Store alternatives in the same order they have been
1988	* defined.
1989	*/
1990	if (!orig_insn->alts) {
1991	orig_insn->alts = alt;
1992	} else {
1993	for (a = orig_insn->alts; a->next; a = a->next)
1994	;
1995	a->next = alt;
1996	}
1997
1998	list_del(&special_alt->list);
1999	free(special_alt);
2000	}
2001
2002	if (opts.stats) {
2003	printf("jl\\\tNOP\tJMP\n");
2004	printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2005	printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2006	}
2007
2008	return 0;
2009	}
2010
2011	__weak unsigned long arch_jump_table_sym_offset(struct reloc reloc, struct reloc table)
2012	{
2013	return reloc->sym->offset + reloc_addend(reloc);
2014	}
2015
2016	static int add_jump_table(struct objtool_file file, struct instruction insn)
2017	{
2018	unsigned long table_size = insn_jump_table_size(insn);
2019	struct symbol *pfunc = insn_func(insn)->pfunc;
2020	struct reloc *table = insn_jump_table(insn);
2021	struct instruction *dest_insn;
2022	unsigned int prev_offset = 0;
2023	struct reloc *reloc = table;
2024	struct alternative *alt;
2025	unsigned long sym_offset;
2026
2027	/*
2028	* Each @reloc is a switch table relocation which points to the target
2029	* instruction.
2030	*/
2031	for_each_reloc_from(table->sec, reloc) {
2032
2033	/* Check for the end of the table: */
2034	if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size)
2035	break;
2036	if (reloc != table && is_jump_table(reloc))
2037	break;
2038
2039	/* Make sure the table entries are consecutive: */
2040	if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc))
2041	break;
2042
2043	sym_offset = arch_jump_table_sym_offset(reloc, table);
2044
2045	/* Detect function pointers from contiguous objects: */
2046	if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset)
2047	break;
2048
2049	/*
2050	* Clang sometimes leaves dangling unused jump table entries
2051	* which point to the end of the function. Ignore them.
2052	*/
2053	if (reloc->sym->sec == pfunc->sec &&
2054	sym_offset == pfunc->offset + pfunc->len)
2055	goto next;
2056
2057	dest_insn = find_insn(file, reloc->sym->sec, sym_offset);
2058	if (!dest_insn)
2059	break;
2060
2061	/* Make sure the destination is in the same function: */
2062	if (!insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != pfunc)
2063	break;
2064
2065	alt = calloc(1, sizeof(*alt));
2066	if (!alt) {
2067	ERROR_GLIBC("calloc");
2068	return -1;
2069	}
2070
2071	alt->insn = dest_insn;
2072	alt->next = insn->alts;
2073	insn->alts = alt;
2074	next:
2075	prev_offset = reloc_offset(reloc);
2076	}
2077
2078	if (!prev_offset) {
2079	ERROR_INSN(insn, "can't find switch jump table");
2080	return -1;
2081	}
2082
2083	return 0;
2084	}
2085
2086	/*
2087	* find_jump_table() - Given a dynamic jump, find the switch jump table
2088	* associated with it.
2089	*/
2090	static void find_jump_table(struct objtool_file file, struct symbol func,
2091	struct instruction *insn)
2092	{
2093	struct reloc *table_reloc;
2094	struct instruction dest_insn, orig_insn = insn;
2095	unsigned long table_size;
2096	unsigned long sym_offset;
2097
2098	/*
2099	* Backward search using the @first_jump_src links, these help avoid
2100	* much of the 'in between' code. Which avoids us getting confused by
2101	* it.
2102	*/
2103	for (;
2104	insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2105	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2106
2107	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2108	break;
2109
2110	/* allow small jumps within the range */
2111	if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2112	insn->jump_dest &&
2113	(insn->jump_dest->offset <= insn->offset \|\|
2114	insn->jump_dest->offset > orig_insn->offset))
2115	break;
2116
2117	table_reloc = arch_find_switch_table(file, insn, &table_size);
2118	if (!table_reloc)
2119	continue;
2120
2121	sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc);
2122
2123	dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset);
2124	if (!dest_insn \|\| !insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != func)
2125	continue;
2126
2127	set_jump_table(table_reloc);
2128	orig_insn->_jump_table = table_reloc;
2129	orig_insn->_jump_table_size = table_size;
2130
2131	break;
2132	}
2133	}
2134
2135	/*
2136	* First pass: Mark the head of each jump table so that in the next pass,
2137	* we know when a given jump table ends and the next one starts.
2138	*/
2139	static void mark_func_jump_tables(struct objtool_file *file,
2140	struct symbol *func)
2141	{
2142	struct instruction insn, last = NULL;
2143
2144	func_for_each_insn(file, func, insn) {
2145	if (!last)
2146	last = insn;
2147
2148	/*
2149	* Store back-pointers for forward jumps such
2150	* that find_jump_table() can back-track using those and
2151	* avoid some potentially confusing code.
2152	*/
2153	if (insn->jump_dest &&
2154	insn->jump_dest->offset > insn->offset &&
2155	!insn->jump_dest->first_jump_src) {
2156
2157	insn->jump_dest->first_jump_src = insn;
2158	last = insn->jump_dest;
2159	}
2160
2161	if (insn->type != INSN_JUMP_DYNAMIC)
2162	continue;
2163
2164	find_jump_table(file, func, insn);
2165	}
2166	}
2167
2168	static int add_func_jump_tables(struct objtool_file *file,
2169	struct symbol *func)
2170	{
2171	struct instruction *insn;
2172
2173	func_for_each_insn(file, func, insn) {
2174	if (!insn_jump_table(insn))
2175	continue;
2176
2177	if (add_jump_table(file, insn))
2178	return -1;
2179	}
2180
2181	return 0;
2182	}
2183
2184	/*
2185	* For some switch statements, gcc generates a jump table in the .rodata
2186	* section which contains a list of addresses within the function to jump to.
2187	* This finds these jump tables and adds them to the insn->alts lists.
2188	*/
2189	static int add_jump_table_alts(struct objtool_file *file)
2190	{
2191	struct symbol *func;
2192
2193	if (!file->rodata)
2194	return 0;
2195
2196	for_each_sym(file->elf, func) {
2197	if (!is_func_sym(func) \|\| is_alias_sym(func))
2198	continue;
2199
2200	mark_func_jump_tables(file, func);
2201	if (add_func_jump_tables(file, func))
2202	return -1;
2203	}
2204
2205	return 0;
2206	}
2207
2208	static void set_func_state(struct cfi_state *state)
2209	{
2210	state->cfa = initial_func_cfi.cfa;
2211	memcpy(&state->regs, &initial_func_cfi.regs,
2212	CFI_NUM_REGS * sizeof(struct cfi_reg));
2213	state->stack_size = initial_func_cfi.cfa.offset;
2214	state->type = UNWIND_HINT_TYPE_CALL;
2215	}
2216
2217	static int read_unwind_hints(struct objtool_file *file)
2218	{
2219	struct cfi_state cfi = init_cfi;
2220	struct section *sec;
2221	struct unwind_hint *hint;
2222	struct instruction *insn;
2223	struct reloc *reloc;
2224	unsigned long offset;
2225	int i;
2226
2227	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2228	if (!sec)
2229	return 0;
2230
2231	if (!sec->rsec) {
2232	ERROR("missing .rela.discard.unwind_hints section");
2233	return -1;
2234	}
2235
2236	if (sec_size(sec) % sizeof(struct unwind_hint)) {
2237	ERROR("struct unwind_hint size mismatch");
2238	return -1;
2239	}
2240
2241	file->hints = true;
2242
2243	for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) {
2244	hint = (struct unwind_hint *)sec->data->d_buf + i;
2245
2246	reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2247	if (!reloc) {
2248	ERROR("can't find reloc for unwind_hints[%d]", i);
2249	return -1;
2250	}
2251
2252	offset = reloc->sym->offset + reloc_addend(reloc);
2253
2254	insn = find_insn(file, reloc->sym->sec, offset);
2255	if (!insn) {
2256	ERROR("can't find insn for unwind_hints[%d]", i);
2257	return -1;
2258	}
2259
2260	insn->hint = true;
2261
2262	if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2263	insn->cfi = &force_undefined_cfi;
2264	continue;
2265	}
2266
2267	if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2268	insn->hint = false;
2269	insn->save = true;
2270	continue;
2271	}
2272
2273	if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2274	insn->restore = true;
2275	continue;
2276	}
2277
2278	if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2279	struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2280
2281	if (sym && is_global_sym(sym)) {
2282	if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2283	ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2284	return -1;
2285	}
2286	}
2287	}
2288
2289	if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2290	insn->cfi = &func_cfi;
2291	continue;
2292	}
2293
2294	if (insn->cfi)
2295	cfi = *(insn->cfi);
2296
2297	if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2298	ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2299	return -1;
2300	}
2301
2302	cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2303	cfi.type = hint->type;
2304	cfi.signal = hint->signal;
2305
2306	insn->cfi = cfi_hash_find_or_add(&cfi);
2307	}
2308
2309	return 0;
2310	}
2311
2312	static int read_annotate(struct objtool_file *file,
2313	int (func)(struct objtool_file file, int type, struct instruction *insn))
2314	{
2315	struct section *sec;
2316	struct instruction *insn;
2317	struct reloc *reloc;
2318	uint64_t offset;
2319	int type;
2320
2321	sec = find_section_by_name(file->elf, ".discard.annotate_insn");
2322	if (!sec)
2323	return 0;
2324
2325	if (!sec->rsec)
2326	return 0;
2327
2328	if (sec->sh.sh_entsize != 8) {
2329	static bool warned = false;
2330	if (!warned && opts.verbose) {
2331	WARN("%s: dodgy linker, sh_entsize != 8", sec->name);
2332	warned = true;
2333	}
2334	sec->sh.sh_entsize = 8;
2335	}
2336
2337	if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) {
2338	ERROR("bad .discard.annotate_insn section: missing relocs");
2339	return -1;
2340	}
2341
2342	for_each_reloc(sec->rsec, reloc) {
2343	type = annotype(file->elf, sec, reloc);
2344	offset = reloc->sym->offset + reloc_addend(reloc);
2345	insn = find_insn(file, reloc->sym->sec, offset);
2346
2347	if (!insn) {
2348	ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type);
2349	return -1;
2350	}
2351
2352	if (func(file, type, insn))
2353	return -1;
2354	}
2355
2356	return 0;
2357	}
2358
2359	static int __annotate_early(struct objtool_file file, int type, struct instruction insn)
2360	{
2361	switch (type) {
2362
2363	/* Must be before add_special_section_alts() */
2364	case ANNOTYPE_IGNORE_ALTS:
2365	insn->ignore_alts = true;
2366	break;
2367
2368	/*
2369	* Must be before read_unwind_hints() since that needs insn->noendbr.
2370	*/
2371	case ANNOTYPE_NOENDBR:
2372	insn->noendbr = 1;
2373	break;
2374
2375	default:
2376	break;
2377	}
2378
2379	return 0;
2380	}
2381
2382	static int __annotate_ifc(struct objtool_file file, int type, struct instruction insn)
2383	{
2384	unsigned long dest_off;
2385
2386	if (type != ANNOTYPE_INTRA_FUNCTION_CALL)
2387	return 0;
2388
2389	if (insn->type != INSN_CALL) {
2390	ERROR_INSN(insn, "intra_function_call not a direct call");
2391	return -1;
2392	}
2393
2394	/*
2395	* Treat intra-function CALLs as JMPs, but with a stack_op.
2396	* See add_call_destinations(), which strips stack_ops from
2397	* normal CALLs.
2398	*/
2399	insn->type = INSN_JUMP_UNCONDITIONAL;
2400
2401	dest_off = arch_jump_destination(insn);
2402	insn->jump_dest = find_insn(file, insn->sec, dest_off);
2403	if (!insn->jump_dest) {
2404	ERROR_INSN(insn, "can't find call dest at %s+0x%lx",
2405	insn->sec->name, dest_off);
2406	return -1;
2407	}
2408
2409	return 0;
2410	}
2411
2412	static int __annotate_late(struct objtool_file file, int type, struct instruction insn)
2413	{
2414	struct symbol *sym;
2415
2416	switch (type) {
2417	case ANNOTYPE_NOENDBR:
2418	/* early */
2419	break;
2420
2421	case ANNOTYPE_RETPOLINE_SAFE:
2422	if (insn->type != INSN_JUMP_DYNAMIC &&
2423	insn->type != INSN_CALL_DYNAMIC &&
2424	insn->type != INSN_RETURN &&
2425	insn->type != INSN_NOP) {
2426	ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2427	return -1;
2428	}
2429
2430	insn->retpoline_safe = true;
2431	break;
2432
2433	case ANNOTYPE_INSTR_BEGIN:
2434	insn->instr++;
2435	break;
2436
2437	case ANNOTYPE_INSTR_END:
2438	insn->instr--;
2439	break;
2440
2441	case ANNOTYPE_UNRET_BEGIN:
2442	insn->unret = 1;
2443	break;
2444
2445	case ANNOTYPE_IGNORE_ALTS:
2446	/* early */
2447	break;
2448
2449	case ANNOTYPE_INTRA_FUNCTION_CALL:
2450	/* ifc */
2451	break;
2452
2453	case ANNOTYPE_REACHABLE:
2454	insn->dead_end = false;
2455	break;
2456
2457	case ANNOTYPE_NOCFI:
2458	sym = insn_sym(insn);
2459	if (!sym) {
2460	ERROR_INSN(insn, "dodgy NOCFI annotation");
2461	return -1;
2462	}
2463	sym->nocfi = 1;
2464	break;
2465
2466	default:
2467	ERROR_INSN(insn, "Unknown annotation type: %d", type);
2468	return -1;
2469	}
2470
2471	return 0;
2472	}
2473
2474	/*
2475	* Return true if name matches an instrumentation function, where calls to that
2476	* function from noinstr code can safely be removed, but compilers won't do so.
2477	*/
2478	static bool is_profiling_func(const char *name)
2479	{
2480	/*
2481	* Many compilers cannot disable KCOV with a function attribute.
2482	*/
2483	if (!strncmp(name, "__sanitizer_cov_", 16))
2484	return true;
2485
2486	return false;
2487	}
2488
2489	static int classify_symbols(struct objtool_file *file)
2490	{
2491	struct symbol *func;
2492	size_t len;
2493
2494	for_each_sym(file->elf, func) {
2495	if (is_notype_sym(func) && strstarts(func->name, ".L"))
2496	func->local_label = true;
2497
2498	if (!is_global_sym(func))
2499	continue;
2500
2501	if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2502	strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2503	func->static_call_tramp = true;
2504
2505	if (arch_is_retpoline(func))
2506	func->retpoline_thunk = true;
2507
2508	if (arch_is_rethunk(func))
2509	func->return_thunk = true;
2510
2511	if (arch_is_embedded_insn(func))
2512	func->embedded_insn = true;
2513
2514	if (arch_ftrace_match(func->name))
2515	func->fentry = true;
2516
2517	if (is_profiling_func(func->name))
2518	func->profiling_func = true;
2519
2520	len = strlen(func->name);
2521	if (len > sym_name_max_len)
2522	sym_name_max_len = len;
2523	}
2524
2525	return 0;
2526	}
2527
2528	static void mark_rodata(struct objtool_file *file)
2529	{
2530	struct section *sec;
2531
2532	/*
2533	* Search for the following rodata sections, each of which can
2534	* potentially contain jump tables:
2535	*
2536	* - .rodata: can contain GCC switch tables
2537	* - .rodata.<func>: same, if -fdata-sections is being used
2538	* - .data.rel.ro.c_jump_table: contains C annotated jump tables
2539	*
2540	* .rodata.str1.* sections are ignored; they don't contain jump tables.
2541	*/
2542	for_each_sec(file->elf, sec) {
2543	if (is_rodata_sec(sec)) {
2544	file->rodata = true;
2545	return;
2546	}
2547	}
2548	}
2549
2550	static void mark_holes(struct objtool_file *file)
2551	{
2552	struct instruction *insn;
2553	bool in_hole = false;
2554
2555	if (!opts.link)
2556	return;
2557
2558	/*
2559	* Whole archive runs might encounter dead code from weak symbols.
2560	* This is where the linker will have dropped the weak symbol in
2561	* favour of a regular symbol, but leaves the code in place.
2562	*/
2563	for_each_insn(file, insn) {
2564	if (insn_sym(insn) \|\| !find_symbol_hole_containing(insn->sec, insn->offset)) {
2565	in_hole = false;
2566	continue;
2567	}
2568
2569	/* Skip function padding and pfx code */
2570	if (!in_hole && insn->type == INSN_NOP)
2571	continue;
2572
2573	in_hole = true;
2574	insn->hole = 1;
2575
2576	/*
2577	* If this hole jumps to a .cold function, mark it ignore.
2578	*/
2579	if (insn->jump_dest) {
2580	struct symbol *dest_func = insn_func(insn->jump_dest);
2581
2582	if (dest_func && is_cold_func(dest_func))
2583	dest_func->ignore = true;
2584	}
2585	}
2586	}
2587
2588	static bool validate_branch_enabled(void)
2589	{
2590	return opts.stackval \|\|
2591	opts.orc \|\|
2592	opts.uaccess;
2593	}
2594
2595	static bool alts_needed(void)
2596	{
2597	return validate_branch_enabled() \|\|
2598	opts.noinstr \|\|
2599	opts.hack_jump_label \|\|
2600	opts.disas \|\|
2601	opts.checksum;
2602	}
2603
2604	int decode_file(struct objtool_file *file)
2605	{
2606	arch_initial_func_cfi_state(&initial_func_cfi);
2607	init_cfi_state(&init_cfi);
2608	init_cfi_state(&func_cfi);
2609	set_func_state(&func_cfi);
2610	init_cfi_state(&force_undefined_cfi);
2611	force_undefined_cfi.force_undefined = true;
2612
2613	if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
2614	return -1;
2615
2616	cfi_hash_add(&init_cfi);
2617	cfi_hash_add(&func_cfi);
2618
2619	file->klp = is_livepatch_module(file);
2620
2621	mark_rodata(file);
2622
2623	if (init_pv_ops(file))
2624	return -1;
2625
2626	/*
2627	* Must be before add_{jump_call}_destination.
2628	*/
2629	if (classify_symbols(file))
2630	return -1;
2631
2632	if (decode_instructions(file))
2633	return -1;
2634
2635	if (add_ignores(file))
2636	return -1;
2637
2638	add_uaccess_safe(file);
2639
2640	if (read_annotate(file, __annotate_early))
2641	return -1;
2642
2643	/*
2644	* Must be before add_jump_destinations(), which depends on 'func'
2645	* being set for alternatives, to enable proper sibling call detection.
2646	*/
2647	if (alts_needed()) {
2648	if (add_special_section_alts(file))
2649	return -1;
2650	}
2651
2652	if (add_jump_destinations(file))
2653	return -1;
2654
2655	/*
2656	* Must be before add_call_destination(); it changes INSN_CALL to
2657	* INSN_JUMP.
2658	*/
2659	if (read_annotate(file, __annotate_ifc))
2660	return -1;
2661
2662	if (add_call_destinations(file))
2663	return -1;
2664
2665	if (add_jump_table_alts(file))
2666	return -1;
2667
2668	if (read_unwind_hints(file))
2669	return -1;
2670
2671	/* Must be after add_jump_destinations() */
2672	mark_holes(file);
2673
2674	/*
2675	* Must be after add_call_destinations() such that it can override
2676	* dead_end_function() marks.
2677	*/
2678	if (read_annotate(file, __annotate_late))
2679	return -1;
2680
2681	return 0;
2682	}
2683
2684	static bool is_special_call(struct instruction *insn)
2685	{
2686	if (insn->type == INSN_CALL) {
2687	struct symbol *dest = insn_call_dest(insn);
2688
2689	if (!dest)
2690	return false;
2691
2692	if (dest->fentry \|\| dest->embedded_insn)
2693	return true;
2694	}
2695
2696	return false;
2697	}
2698
2699	static bool has_modified_stack_frame(struct instruction insn, struct insn_state state)
2700	{
2701	struct cfi_state *cfi = &state->cfi;
2702	int i;
2703
2704	if (cfi->cfa.base != initial_func_cfi.cfa.base \|\| cfi->drap)
2705	return true;
2706
2707	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2708	return true;
2709
2710	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2711	return true;
2712
2713	for (i = 0; i < CFI_NUM_REGS; i++) {
2714	if (cfi->regs[i].base != initial_func_cfi.regs[i].base \|\|
2715	cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2716	return true;
2717	}
2718
2719	return false;
2720	}
2721
2722	static bool check_reg_frame_pos(const struct cfi_reg *reg,
2723	int expected_offset)
2724	{
2725	return reg->base == CFI_CFA &&
2726	reg->offset == expected_offset;
2727	}
2728
2729	static bool has_valid_stack_frame(struct insn_state *state)
2730	{
2731	struct cfi_state *cfi = &state->cfi;
2732
2733	if (cfi->cfa.base == CFI_BP &&
2734	check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2735	check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2736	return true;
2737
2738	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2739	return true;
2740
2741	return false;
2742	}
2743
2744	static int update_cfi_state_regs(struct instruction *insn,
2745	struct cfi_state *cfi,
2746	struct stack_op *op)
2747	{
2748	struct cfi_reg *cfa = &cfi->cfa;
2749
2750	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2751	return 0;
2752
2753	/* push */
2754	if (op->dest.type == OP_DEST_PUSH \|\| op->dest.type == OP_DEST_PUSHF)
2755	cfa->offset += 8;
2756
2757	/* pop */
2758	if (op->src.type == OP_SRC_POP \|\| op->src.type == OP_SRC_POPF)
2759	cfa->offset -= 8;
2760
2761	/* add immediate to sp */
2762	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2763	op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2764	cfa->offset -= op->src.offset;
2765
2766	return 0;
2767	}
2768
2769	static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2770	{
2771	if (arch_callee_saved_reg(reg) &&
2772	cfi->regs[reg].base == CFI_UNDEFINED) {
2773	cfi->regs[reg].base = base;
2774	cfi->regs[reg].offset = offset;
2775	}
2776	}
2777
2778	static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2779	{
2780	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2781	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2782	}
2783
2784	/*
2785	* A note about DRAP stack alignment:
2786	*
2787	* GCC has the concept of a DRAP register, which is used to help keep track of
2788	* the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2789	* register. The typical DRAP pattern is:
2790	*
2791	* 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2792	* 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2793	* 41 ff 72 f8 pushq -0x8(%r10)
2794	* 55 push %rbp
2795	* 48 89 e5 mov %rsp,%rbp
2796	* (more pushes)
2797	* 41 52 push %r10
2798	* ...
2799	* 41 5a pop %r10
2800	* (more pops)
2801	* 5d pop %rbp
2802	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2803	* c3 retq
2804	*
2805	* There are some variations in the epilogues, like:
2806	*
2807	* 5b pop %rbx
2808	* 41 5a pop %r10
2809	* 41 5c pop %r12
2810	* 41 5d pop %r13
2811	* 41 5e pop %r14
2812	* c9 leaveq
2813	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2814	* c3 retq
2815	*
2816	* and:
2817	*
2818	* 4c 8b 55 e8 mov -0x18(%rbp),%r10
2819	* 48 8b 5d e0 mov -0x20(%rbp),%rbx
2820	* 4c 8b 65 f0 mov -0x10(%rbp),%r12
2821	* 4c 8b 6d f8 mov -0x8(%rbp),%r13
2822	* c9 leaveq
2823	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2824	* c3 retq
2825	*
2826	* Sometimes r13 is used as the DRAP register, in which case it's saved and
2827	* restored beforehand:
2828	*
2829	* 41 55 push %r13
2830	* 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2831	* 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2832	* ...
2833	* 49 8d 65 f0 lea -0x10(%r13),%rsp
2834	* 41 5d pop %r13
2835	* c3 retq
2836	*/
2837	static int update_cfi_state(struct instruction *insn,
2838	struct instruction *next_insn,
2839	struct cfi_state cfi, struct stack_op op)
2840	{
2841	struct cfi_reg *cfa = &cfi->cfa;
2842	struct cfi_reg *regs = cfi->regs;
2843
2844	/* ignore UNWIND_HINT_UNDEFINED regions */
2845	if (cfi->force_undefined)
2846	return 0;
2847
2848	/* stack operations don't make sense with an undefined CFA */
2849	if (cfa->base == CFI_UNDEFINED) {
2850	if (insn_func(insn)) {
2851	WARN_INSN(insn, "undefined stack state");
2852	return 1;
2853	}
2854	return 0;
2855	}
2856
2857	if (cfi->type == UNWIND_HINT_TYPE_REGS \|\|
2858	cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2859	return update_cfi_state_regs(insn, cfi, op);
2860
2861	switch (op->dest.type) {
2862
2863	case OP_DEST_REG:
2864	switch (op->src.type) {
2865
2866	case OP_SRC_REG:
2867	if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2868	cfa->base == CFI_SP &&
2869	check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2870
2871	/* mov %rsp, %rbp */
2872	cfa->base = op->dest.reg;
2873	cfi->bp_scratch = false;
2874	}
2875
2876	else if (op->src.reg == CFI_SP &&
2877	op->dest.reg == CFI_BP && cfi->drap) {
2878
2879	/* drap: mov %rsp, %rbp */
2880	regs[CFI_BP].base = CFI_BP;
2881	regs[CFI_BP].offset = -cfi->stack_size;
2882	cfi->bp_scratch = false;
2883	}
2884
2885	else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2886
2887	/*
2888	* mov %rsp, %reg
2889	*
2890	* This is needed for the rare case where GCC
2891	* does:
2892	*
2893	* mov %rsp, %rax
2894	* ...
2895	* mov %rax, %rsp
2896	*/
2897	cfi->vals[op->dest.reg].base = CFI_CFA;
2898	cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2899	}
2900
2901	else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2902	(cfa->base == CFI_BP \|\| cfa->base == cfi->drap_reg)) {
2903
2904	/*
2905	* mov %rbp, %rsp
2906	*
2907	* Restore the original stack pointer (Clang).
2908	*/
2909	cfi->stack_size = -cfi->regs[CFI_BP].offset;
2910	}
2911
2912	else if (op->dest.reg == cfa->base) {
2913
2914	/* mov %reg, %rsp */
2915	if (cfa->base == CFI_SP &&
2916	cfi->vals[op->src.reg].base == CFI_CFA) {
2917
2918	/*
2919	* This is needed for the rare case
2920	* where GCC does something dumb like:
2921	*
2922	* lea 0x8(%rsp), %rcx
2923	* ...
2924	* mov %rcx, %rsp
2925	*/
2926	cfa->offset = -cfi->vals[op->src.reg].offset;
2927	cfi->stack_size = cfa->offset;
2928
2929	} else if (cfa->base == CFI_SP &&
2930	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2931	cfi->vals[op->src.reg].offset == cfa->offset) {
2932
2933	/*
2934	* Stack swizzle:
2935	*
2936	* 1: mov %rsp, (%[tos])
2937	* 2: mov %[tos], %rsp
2938	* ...
2939	* 3: pop %rsp
2940	*
2941	* Where:
2942	*
2943	* 1 - places a pointer to the previous
2944	* stack at the Top-of-Stack of the
2945	* new stack.
2946	*
2947	* 2 - switches to the new stack.
2948	*
2949	* 3 - pops the Top-of-Stack to restore
2950	* the original stack.
2951	*
2952	* Note: we set base to SP_INDIRECT
2953	* here and preserve offset. Therefore
2954	* when the unwinder reaches ToS it
2955	* will dereference SP and then add the
2956	* offset to find the next frame, IOW:
2957	* (%rsp) + offset.
2958	*/
2959	cfa->base = CFI_SP_INDIRECT;
2960
2961	} else {
2962	cfa->base = CFI_UNDEFINED;
2963	cfa->offset = 0;
2964	}
2965	}
2966
2967	else if (op->dest.reg == CFI_SP &&
2968	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2969	cfi->vals[op->src.reg].offset == cfa->offset) {
2970
2971	/*
2972	* The same stack swizzle case 2) as above. But
2973	* because we can't change cfa->base, case 3)
2974	* will become a regular POP. Pretend we're a
2975	* PUSH so things don't go unbalanced.
2976	*/
2977	cfi->stack_size += 8;
2978	}
2979
2980	else if (cfi->vals[op->src.reg].base == CFI_CFA) {
2981	/*
2982	* Clang RSP musical chairs:
2983	*
2984	* mov %rsp, %rdx [handled above]
2985	* ...
2986	* mov %rdx, %rbx [handled here]
2987	* ...
2988	* mov %rbx, %rsp [handled above]
2989	*/
2990	cfi->vals[op->dest.reg].base = CFI_CFA;
2991	cfi->vals[op->dest.reg].offset = cfi->vals[op->src.reg].offset;
2992	}
2993
2994
2995	break;
2996
2997	case OP_SRC_ADD:
2998	if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2999
3000	/* add imm, %rsp */
3001	cfi->stack_size -= op->src.offset;
3002	if (cfa->base == CFI_SP)
3003	cfa->offset -= op->src.offset;
3004	break;
3005	}
3006
3007	if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
3008	insn_sym(insn)->frame_pointer) {
3009	/* addi.d fp,sp,imm on LoongArch */
3010	if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
3011	cfa->base = CFI_BP;
3012	cfa->offset = 0;
3013	}
3014	break;
3015	}
3016
3017	if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3018	/* addi.d sp,fp,imm on LoongArch */
3019	if (cfa->base == CFI_BP && cfa->offset == 0) {
3020	if (insn_sym(insn)->frame_pointer) {
3021	cfa->base = CFI_SP;
3022	cfa->offset = -op->src.offset;
3023	}
3024	} else {
3025	/* lea disp(%rbp), %rsp */
3026	cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3027	}
3028	break;
3029	}
3030
3031	if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3032
3033	/* drap: lea disp(%rsp), %drap */
3034	cfi->drap_reg = op->dest.reg;
3035
3036	/*
3037	* lea disp(%rsp), %reg
3038	*
3039	* This is needed for the rare case where GCC
3040	* does something dumb like:
3041	*
3042	* lea 0x8(%rsp), %rcx
3043	* ...
3044	* mov %rcx, %rsp
3045	*/
3046	cfi->vals[op->dest.reg].base = CFI_CFA;
3047	cfi->vals[op->dest.reg].offset = \
3048	-cfi->stack_size + op->src.offset;
3049
3050	break;
3051	}
3052
3053	if (cfi->drap && op->dest.reg == CFI_SP &&
3054	op->src.reg == cfi->drap_reg) {
3055
3056	/* drap: lea disp(%drap), %rsp */
3057	cfa->base = CFI_SP;
3058	cfa->offset = cfi->stack_size = -op->src.offset;
3059	cfi->drap_reg = CFI_UNDEFINED;
3060	cfi->drap = false;
3061	break;
3062	}
3063
3064	if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3065	WARN_INSN(insn, "unsupported stack register modification");
3066	return -1;
3067	}
3068
3069	break;
3070
3071	case OP_SRC_AND:
3072	if (op->dest.reg != CFI_SP \|\|
3073	(cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) \|\|
3074	(cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3075	WARN_INSN(insn, "unsupported stack pointer realignment");
3076	return -1;
3077	}
3078
3079	if (cfi->drap_reg != CFI_UNDEFINED) {
3080	/* drap: and imm, %rsp */
3081	cfa->base = cfi->drap_reg;
3082	cfa->offset = cfi->stack_size = 0;
3083	cfi->drap = true;
3084	}
3085
3086	/*
3087	* Older versions of GCC (4.8ish) realign the stack
3088	* without DRAP, with a frame pointer.
3089	*/
3090
3091	break;
3092
3093	case OP_SRC_POP:
3094	case OP_SRC_POPF:
3095	if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3096
3097	/* pop %rsp; # restore from a stack swizzle */
3098	cfa->base = CFI_SP;
3099	break;
3100	}
3101
3102	if (!cfi->drap && op->dest.reg == cfa->base) {
3103
3104	/* pop %rbp */
3105	cfa->base = CFI_SP;
3106	}
3107
3108	if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3109	op->dest.reg == cfi->drap_reg &&
3110	cfi->drap_offset == -cfi->stack_size) {
3111
3112	/* drap: pop %drap */
3113	cfa->base = cfi->drap_reg;
3114	cfa->offset = 0;
3115	cfi->drap_offset = -1;
3116
3117	} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3118
3119	/* pop %reg */
3120	restore_reg(cfi, op->dest.reg);
3121	}
3122
3123	cfi->stack_size -= 8;
3124	if (cfa->base == CFI_SP)
3125	cfa->offset -= 8;
3126
3127	break;
3128
3129	case OP_SRC_REG_INDIRECT:
3130	if (!cfi->drap && op->dest.reg == cfa->base &&
3131	op->dest.reg == CFI_BP) {
3132
3133	/* mov disp(%rsp), %rbp */
3134	cfa->base = CFI_SP;
3135	cfa->offset = cfi->stack_size;
3136	}
3137
3138	if (cfi->drap && op->src.reg == CFI_BP &&
3139	op->src.offset == cfi->drap_offset) {
3140
3141	/* drap: mov disp(%rbp), %drap */
3142	cfa->base = cfi->drap_reg;
3143	cfa->offset = 0;
3144	cfi->drap_offset = -1;
3145	}
3146
3147	if (cfi->drap && op->src.reg == CFI_BP &&
3148	op->src.offset == regs[op->dest.reg].offset) {
3149
3150	/* drap: mov disp(%rbp), %reg */
3151	restore_reg(cfi, op->dest.reg);
3152
3153	} else if (op->src.reg == cfa->base &&
3154	op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3155
3156	/* mov disp(%rbp), %reg */
3157	/* mov disp(%rsp), %reg */
3158	restore_reg(cfi, op->dest.reg);
3159
3160	} else if (op->src.reg == CFI_SP &&
3161	op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3162
3163	/* mov disp(%rsp), %reg */
3164	restore_reg(cfi, op->dest.reg);
3165	}
3166
3167	break;
3168
3169	default:
3170	WARN_INSN(insn, "unknown stack-related instruction");
3171	return -1;
3172	}
3173
3174	break;
3175
3176	case OP_DEST_PUSH:
3177	case OP_DEST_PUSHF:
3178	cfi->stack_size += 8;
3179	if (cfa->base == CFI_SP)
3180	cfa->offset += 8;
3181
3182	if (op->src.type != OP_SRC_REG)
3183	break;
3184
3185	if (cfi->drap) {
3186	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3187
3188	/* drap: push %drap */
3189	cfa->base = CFI_BP_INDIRECT;
3190	cfa->offset = -cfi->stack_size;
3191
3192	/* save drap so we know when to restore it */
3193	cfi->drap_offset = -cfi->stack_size;
3194
3195	} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3196
3197	/* drap: push %rbp */
3198	cfi->stack_size = 0;
3199
3200	} else {
3201
3202	/* drap: push %reg */
3203	save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3204	}
3205
3206	} else {
3207
3208	/* push %reg */
3209	save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3210	}
3211
3212	/* detect when asm code uses rbp as a scratch register */
3213	if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3214	cfa->base != CFI_BP)
3215	cfi->bp_scratch = true;
3216	break;
3217
3218	case OP_DEST_REG_INDIRECT:
3219
3220	if (cfi->drap) {
3221	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3222
3223	/* drap: mov %drap, disp(%rbp) */
3224	cfa->base = CFI_BP_INDIRECT;
3225	cfa->offset = op->dest.offset;
3226
3227	/* save drap offset so we know when to restore it */
3228	cfi->drap_offset = op->dest.offset;
3229	} else {
3230
3231	/* drap: mov reg, disp(%rbp) */
3232	save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3233	}
3234
3235	} else if (op->dest.reg == cfa->base) {
3236
3237	/* mov reg, disp(%rbp) */
3238	/* mov reg, disp(%rsp) */
3239	save_reg(cfi, op->src.reg, CFI_CFA,
3240	op->dest.offset - cfi->cfa.offset);
3241
3242	} else if (op->dest.reg == CFI_SP) {
3243
3244	/* mov reg, disp(%rsp) */
3245	save_reg(cfi, op->src.reg, CFI_CFA,
3246	op->dest.offset - cfi->stack_size);
3247
3248	} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3249
3250	/* mov %rsp, (%reg); # setup a stack swizzle. */
3251	cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3252	cfi->vals[op->dest.reg].offset = cfa->offset;
3253	}
3254
3255	break;
3256
3257	case OP_DEST_MEM:
3258	if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3259	WARN_INSN(insn, "unknown stack-related memory operation");
3260	return -1;
3261	}
3262
3263	/* pop mem */
3264	cfi->stack_size -= 8;
3265	if (cfa->base == CFI_SP)
3266	cfa->offset -= 8;
3267
3268	break;
3269
3270	default:
3271	WARN_INSN(insn, "unknown stack-related instruction");
3272	return -1;
3273	}
3274
3275	return 0;
3276	}
3277
3278	/*
3279	* The stack layouts of alternatives instructions can sometimes diverge when
3280	* they have stack modifications. That's fine as long as the potential stack
3281	* layouts don't conflict at any given potential instruction boundary.
3282	*
3283	* Flatten the CFIs of the different alternative code streams (both original
3284	* and replacement) into a single shared CFI array which can be used to detect
3285	* conflicts and nicely feed a linear array of ORC entries to the unwinder.
3286	*/
3287	static int propagate_alt_cfi(struct objtool_file file, struct instruction insn)
3288	{
3289	struct cfi_state **alt_cfi;
3290	int group_off;
3291
3292	if (!insn->alt_group)
3293	return 0;
3294
3295	if (!insn->cfi) {
3296	WARN("CFI missing");
3297	return -1;
3298	}
3299
3300	alt_cfi = insn->alt_group->cfi;
3301	group_off = insn->offset - insn->alt_group->first_insn->offset;
3302
3303	if (!alt_cfi[group_off]) {
3304	alt_cfi[group_off] = insn->cfi;
3305	} else {
3306	if (cficmp(alt_cfi[group_off], insn->cfi)) {
3307	struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3308	struct instruction *orig = orig_group->first_insn;
3309	WARN_INSN(orig, "stack layout conflict in alternatives: %s",
3310	offstr(insn->sec, insn->offset));
3311	return -1;
3312	}
3313	}
3314
3315	return 0;
3316	}
3317
3318	static int noinline handle_insn_ops(struct instruction *insn,
3319	struct instruction *next_insn,
3320	struct insn_state *state)
3321	{
3322	struct insn_state prev_state __maybe_unused = *state;
3323	struct stack_op *op;
3324	int ret = 0;
3325
3326	for (op = insn->stack_ops; op; op = op->next) {
3327
3328	ret = update_cfi_state(insn, next_insn, &state->cfi, op);
3329	if (ret)
3330	goto done;
3331
3332	if (!opts.uaccess \|\| !insn->alt_group)
3333	continue;
3334
3335	if (op->dest.type == OP_DEST_PUSHF) {
3336	if (!state->uaccess_stack) {
3337	state->uaccess_stack = 1;
3338	} else if (state->uaccess_stack >> 31) {
3339	WARN_INSN(insn, "PUSHF stack exhausted");
3340	ret = 1;
3341	goto done;
3342	}
3343	state->uaccess_stack <<= 1;
3344	state->uaccess_stack \|= state->uaccess;
3345	}
3346
3347	if (op->src.type == OP_SRC_POPF) {
3348	if (state->uaccess_stack) {
3349	state->uaccess = state->uaccess_stack & 1;
3350	state->uaccess_stack >>= 1;
3351	if (state->uaccess_stack == 1)
3352	state->uaccess_stack = 0;
3353	}
3354	}
3355	}
3356
3357	done:
3358	TRACE_INSN_STATE(insn, &prev_state, state);
3359
3360	return ret;
3361	}
3362
3363	static bool insn_cfi_match(struct instruction insn, struct cfi_state cfi2)
3364	{
3365	struct cfi_state *cfi1 = insn->cfi;
3366	int i;
3367
3368	if (!cfi1) {
3369	WARN("CFI missing");
3370	return false;
3371	}
3372
3373	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3374
3375	WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3376	cfi1->cfa.base, cfi1->cfa.offset,
3377	cfi2->cfa.base, cfi2->cfa.offset);
3378	return false;
3379
3380	}
3381
3382	if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3383	for (i = 0; i < CFI_NUM_REGS; i++) {
3384
3385	if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg)))
3386	continue;
3387
3388	WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3389	i, cfi1->regs[i].base, cfi1->regs[i].offset,
3390	i, cfi2->regs[i].base, cfi2->regs[i].offset);
3391	}
3392	return false;
3393	}
3394
3395	if (cfi1->type != cfi2->type) {
3396
3397	WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3398	cfi1->type, cfi2->type);
3399	return false;
3400	}
3401
3402	if (cfi1->drap != cfi2->drap \|\|
3403	(cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) \|\|
3404	(cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3405
3406	WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3407	cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3408	cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3409	return false;
3410	}
3411
3412	return true;
3413	}
3414
3415	static inline bool func_uaccess_safe(struct symbol *func)
3416	{
3417	if (func)
3418	return func->uaccess_safe;
3419
3420	return false;
3421	}
3422
3423	static inline const char call_dest_name(struct instruction insn)
3424	{
3425	static char pvname[19];
3426	struct reloc *reloc;
3427	int idx;
3428
3429	if (insn_call_dest(insn))
3430	return insn_call_dest(insn)->name;
3431
3432	reloc = insn_reloc(NULL, insn);
3433	if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3434	idx = (reloc_addend(reloc) / sizeof(void *));
3435	snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3436	return pvname;
3437	}
3438
3439	return "{dynamic}";
3440	}
3441
3442	static bool pv_call_dest(struct objtool_file file, struct instruction insn)
3443	{
3444	struct symbol *target;
3445	struct reloc *reloc;
3446	int idx;
3447
3448	reloc = insn_reloc(file, insn);
3449	if (!reloc \|\| strcmp(reloc->sym->name, "pv_ops"))
3450	return false;
3451
3452	idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *);
3453
3454	if (file->pv_ops[idx].clean)
3455	return true;
3456
3457	file->pv_ops[idx].clean = true;
3458
3459	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3460	if (!target->sec->noinstr) {
3461	WARN("pv_ops[%d]: %s", idx, target->name);
3462	file->pv_ops[idx].clean = false;
3463	}
3464	}
3465
3466	return file->pv_ops[idx].clean;
3467	}
3468
3469	static inline bool noinstr_call_dest(struct objtool_file *file,
3470	struct instruction *insn,
3471	struct symbol *func)
3472	{
3473	/*
3474	* We can't deal with indirect function calls at present;
3475	* assume they're instrumented.
3476	*/
3477	if (!func) {
3478	if (file->pv_ops)
3479	return pv_call_dest(file, insn);
3480
3481	return false;
3482	}
3483
3484	/*
3485	* If the symbol is from a noinstr section; we good.
3486	*/
3487	if (func->sec->noinstr)
3488	return true;
3489
3490	/*
3491	* If the symbol is a static_call trampoline, we can't tell.
3492	*/
3493	if (func->static_call_tramp)
3494	return true;
3495
3496	/*
3497	* The __ubsan_handle_*() calls are like WARN(), they only happen when
3498	* something 'BAD' happened. At the risk of taking the machine down,
3499	* let them proceed to get the message out.
3500	*/
3501	if (!strncmp(func->name, "__ubsan_handle_", 15))
3502	return true;
3503
3504	return false;
3505	}
3506
3507	static int validate_call(struct objtool_file *file,
3508	struct instruction *insn,
3509	struct insn_state *state)
3510	{
3511	if (state->noinstr && state->instr <= 0 &&
3512	!noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3513	WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3514	return 1;
3515	}
3516
3517	if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3518	WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3519	return 1;
3520	}
3521
3522	if (state->df) {
3523	WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3524	return 1;
3525	}
3526
3527	return 0;
3528	}
3529
3530	static int validate_sibling_call(struct objtool_file *file,
3531	struct instruction *insn,
3532	struct insn_state *state)
3533	{
3534	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3535	WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3536	return 1;
3537	}
3538
3539	return validate_call(file, insn, state);
3540	}
3541
3542	static int validate_return(struct symbol func, struct instruction insn, struct insn_state *state)
3543	{
3544	if (state->noinstr && state->instr > 0) {
3545	WARN_INSN(insn, "return with instrumentation enabled");
3546	return 1;
3547	}
3548
3549	if (state->uaccess && !func_uaccess_safe(func)) {
3550	WARN_INSN(insn, "return with UACCESS enabled");
3551	return 1;
3552	}
3553
3554	if (!state->uaccess && func_uaccess_safe(func)) {
3555	WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3556	return 1;
3557	}
3558
3559	if (state->df) {
3560	WARN_INSN(insn, "return with DF set");
3561	return 1;
3562	}
3563
3564	if (func && has_modified_stack_frame(insn, state)) {
3565	WARN_INSN(insn, "return with modified stack frame");
3566	return 1;
3567	}
3568
3569	if (state->cfi.bp_scratch) {
3570	WARN_INSN(insn, "BP used as a scratch register");
3571	return 1;
3572	}
3573
3574	return 0;
3575	}
3576
3577	static struct instruction next_insn_to_validate(struct objtool_file file,
3578	struct instruction *insn)
3579	{
3580	struct alt_group *alt_group = insn->alt_group;
3581
3582	/*
3583	* Simulate the fact that alternatives are patched in-place. When the
3584	* end of a replacement alt_group is reached, redirect objtool flow to
3585	* the end of the original alt_group.
3586	*
3587	* insn->alts->insn -> alt_group->first_insn
3588	* ...
3589	* alt_group->last_insn
3590	* [alt_group->nop] -> next(orig_group->last_insn)
3591	*/
3592	if (alt_group) {
3593	if (alt_group->nop) {
3594	/* ->nop implies ->orig_group */
3595	if (insn == alt_group->last_insn)
3596	return alt_group->nop;
3597	if (insn == alt_group->nop)
3598	goto next_orig;
3599	}
3600	if (insn == alt_group->last_insn && alt_group->orig_group)
3601	goto next_orig;
3602	}
3603
3604	return next_insn_same_sec(file, insn);
3605
3606	next_orig:
3607	return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3608	}
3609
3610	static bool skip_alt_group(struct instruction *insn)
3611	{
3612	struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL;
3613
3614	if (!insn->alt_group)
3615	return false;
3616
3617	/* ANNOTATE_IGNORE_ALTERNATIVE */
3618	if (insn->alt_group->ignore) {
3619	TRACE_ALT(insn, "alt group ignored");
3620	return true;
3621	}
3622
3623	/*
3624	* For NOP patched with CLAC/STAC, only follow the latter to avoid
3625	* impossible code paths combining patched CLAC with unpatched STAC
3626	* or vice versa.
3627	*
3628	* ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus
3629	* requested not to do that to avoid hurting .s file readability
3630	* around CLAC/STAC alternative sites.
3631	*/
3632
3633	if (!alt_insn)
3634	return false;
3635
3636	/* Don't override ASM_{CLAC,STAC}_UNSAFE */
3637	if (alt_insn->alt_group && alt_insn->alt_group->ignore)
3638	return false;
3639
3640	return alt_insn->type == INSN_CLAC \|\| alt_insn->type == INSN_STAC;
3641	}
3642
3643	static int validate_branch(struct objtool_file file, struct symbol func,
3644	struct instruction *insn, struct insn_state state);
3645	static int do_validate_branch(struct objtool_file file, struct symbol func,
3646	struct instruction insn, struct insn_state state);
3647
3648	static int validate_insn(struct objtool_file file, struct symbol func,
3649	struct instruction insn, struct insn_state statep,
3650	struct instruction prev_insn, struct instruction next_insn,
3651	bool *dead_end)
3652	{
3653	char *alt_name __maybe_unused = NULL;
3654	struct alternative *alt;
3655	u8 visited;
3656	int ret;
3657
3658	/*
3659	* Any returns before the end of this function are effectively dead
3660	* ends, i.e. validate_branch() has reached the end of the branch.
3661	*/
3662	*dead_end = true;
3663
3664	visited = VISITED_BRANCH << statep->uaccess;
3665	if (insn->visited & VISITED_BRANCH_MASK) {
3666	if (!insn->hint && !insn_cfi_match(insn, &statep->cfi))
3667	return 1;
3668
3669	if (insn->visited & visited) {
3670	TRACE_INSN(insn, "already visited");
3671	return 0;
3672	}
3673	} else {
3674	nr_insns_visited++;
3675	}
3676
3677	if (statep->noinstr)
3678	statep->instr += insn->instr;
3679
3680	if (insn->hint) {
3681	if (insn->restore) {
3682	struct instruction save_insn, i;
3683
3684	i = insn;
3685	save_insn = NULL;
3686
3687	sym_for_each_insn_continue_reverse(file, func, i) {
3688	if (i->save) {
3689	save_insn = i;
3690	break;
3691	}
3692	}
3693
3694	if (!save_insn) {
3695	WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3696	return 1;
3697	}
3698
3699	if (!save_insn->visited) {
3700	/*
3701	* If the restore hint insn is at the
3702	* beginning of a basic block and was
3703	* branched to from elsewhere, and the
3704	* save insn hasn't been visited yet,
3705	* defer following this branch for now.
3706	* It will be seen later via the
3707	* straight-line path.
3708	*/
3709	if (!prev_insn) {
3710	TRACE_INSN(insn, "defer restore");
3711	return 0;
3712	}
3713
3714	WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3715	return 1;
3716	}
3717
3718	insn->cfi = save_insn->cfi;
3719	nr_cfi_reused++;
3720	}
3721
3722	statep->cfi = *insn->cfi;
3723	} else {
3724	/* XXX track if we actually changed statep->cfi */
3725
3726	if (prev_insn && !cficmp(prev_insn->cfi, &statep->cfi)) {
3727	insn->cfi = prev_insn->cfi;
3728	nr_cfi_reused++;
3729	} else {
3730	insn->cfi = cfi_hash_find_or_add(&statep->cfi);
3731	}
3732	}
3733
3734	insn->visited \|= visited;
3735
3736	if (propagate_alt_cfi(file, insn))
3737	return 1;
3738
3739	if (insn->alts) {
3740	for (alt = insn->alts; alt; alt = alt->next) {
3741	TRACE_ALT_BEGIN(insn, alt, alt_name);
3742	ret = validate_branch(file, func, alt->insn, *statep);
3743	TRACE_ALT_END(insn, alt, alt_name);
3744	if (ret) {
3745	BT_INSN(insn, "(alt)");
3746	return ret;
3747	}
3748	}
3749	TRACE_ALT_INFO_NOADDR(insn, "/ ", "DEFAULT");
3750	}
3751
3752	if (skip_alt_group(insn))
3753	return 0;
3754
3755	if (handle_insn_ops(insn, next_insn, statep))
3756	return 1;
3757
3758	switch (insn->type) {
3759
3760	case INSN_RETURN:
3761	TRACE_INSN(insn, "return");
3762	return validate_return(func, insn, statep);
3763
3764	case INSN_CALL:
3765	case INSN_CALL_DYNAMIC:
3766	if (insn->type == INSN_CALL)
3767	TRACE_INSN(insn, "call");
3768	else
3769	TRACE_INSN(insn, "indirect call");
3770
3771	ret = validate_call(file, insn, statep);
3772	if (ret)
3773	return ret;
3774
3775	if (opts.stackval && func && !is_special_call(insn) &&
3776	!has_valid_stack_frame(statep)) {
3777	WARN_INSN(insn, "call without frame pointer save/setup");
3778	return 1;
3779	}
3780
3781	break;
3782
3783	case INSN_JUMP_CONDITIONAL:
3784	case INSN_JUMP_UNCONDITIONAL:
3785	if (is_sibling_call(insn)) {
3786	TRACE_INSN(insn, "sibling call");
3787	ret = validate_sibling_call(file, insn, statep);
3788	if (ret)
3789	return ret;
3790
3791	} else if (insn->jump_dest) {
3792	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3793	TRACE_INSN(insn, "unconditional jump");
3794	else
3795	TRACE_INSN(insn, "jump taken");
3796
3797	ret = validate_branch(file, func, insn->jump_dest, *statep);
3798	if (ret) {
3799	BT_INSN(insn, "(branch)");
3800	return ret;
3801	}
3802	}
3803
3804	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3805	return 0;
3806
3807	TRACE_INSN(insn, "jump not taken");
3808	break;
3809
3810	case INSN_JUMP_DYNAMIC:
3811	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3812	TRACE_INSN(insn, "indirect jump");
3813	if (is_sibling_call(insn)) {
3814	ret = validate_sibling_call(file, insn, statep);
3815	if (ret)
3816	return ret;
3817	}
3818
3819	if (insn->type == INSN_JUMP_DYNAMIC)
3820	return 0;
3821
3822	break;
3823
3824	case INSN_SYSCALL:
3825	TRACE_INSN(insn, "syscall");
3826	if (func && (!next_insn \|\| !next_insn->hint)) {
3827	WARN_INSN(insn, "unsupported instruction in callable function");
3828	return 1;
3829	}
3830
3831	break;
3832
3833	case INSN_SYSRET:
3834	TRACE_INSN(insn, "sysret");
3835	if (func && (!next_insn \|\| !next_insn->hint)) {
3836	WARN_INSN(insn, "unsupported instruction in callable function");
3837	return 1;
3838	}
3839
3840	return 0;
3841
3842	case INSN_STAC:
3843	TRACE_INSN(insn, "stac");
3844	if (!opts.uaccess)
3845	break;
3846
3847	if (statep->uaccess) {
3848	WARN_INSN(insn, "recursive UACCESS enable");
3849	return 1;
3850	}
3851
3852	statep->uaccess = true;
3853	break;
3854
3855	case INSN_CLAC:
3856	TRACE_INSN(insn, "clac");
3857	if (!opts.uaccess)
3858	break;
3859
3860	if (!statep->uaccess && func) {
3861	WARN_INSN(insn, "redundant UACCESS disable");
3862	return 1;
3863	}
3864
3865	if (func_uaccess_safe(func) && !statep->uaccess_stack) {
3866	WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3867	return 1;
3868	}
3869
3870	statep->uaccess = false;
3871	break;
3872
3873	case INSN_STD:
3874	TRACE_INSN(insn, "std");
3875	if (statep->df) {
3876	WARN_INSN(insn, "recursive STD");
3877	return 1;
3878	}
3879
3880	statep->df = true;
3881	break;
3882
3883	case INSN_CLD:
3884	TRACE_INSN(insn, "cld");
3885	if (!statep->df && func) {
3886	WARN_INSN(insn, "redundant CLD");
3887	return 1;
3888	}
3889
3890	statep->df = false;
3891	break;
3892
3893	default:
3894	break;
3895	}
3896
3897	if (insn->dead_end)
3898	TRACE_INSN(insn, "dead end");
3899
3900	*dead_end = insn->dead_end;
3901	return 0;
3902	}
3903
3904	/*
3905	* Follow the branch starting at the given instruction, and recursively follow
3906	* any other branches (jumps). Meanwhile, track the frame pointer state at
3907	* each instruction and validate all the rules described in
3908	* tools/objtool/Documentation/objtool.txt.
3909	*/
3910	static int do_validate_branch(struct objtool_file file, struct symbol func,
3911	struct instruction insn, struct insn_state state)
3912	{
3913	struct instruction next_insn, prev_insn = NULL;
3914	bool dead_end;
3915	int ret;
3916
3917	if (func && func->ignore)
3918	return 0;
3919
3920	do {
3921	insn->trace = 0;
3922	next_insn = next_insn_to_validate(file, insn);
3923
3924	if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3925	/* Ignore KCFI type preambles, which always fall through */
3926	if (is_prefix_func(func))
3927	return 0;
3928
3929	if (file->ignore_unreachables)
3930	return 0;
3931
3932	WARN("%s() falls through to next function %s()",
3933	func->name, insn_func(insn)->name);
3934	func->warned = 1;
3935
3936	return 1;
3937	}
3938
3939	ret = validate_insn(file, func, insn, state, prev_insn, next_insn,
3940	&dead_end);
3941
3942	if (!insn->trace) {
3943	if (ret)
3944	TRACE_INSN(insn, "warning (%d)", ret);
3945	else
3946	TRACE_INSN(insn, NULL);
3947	}
3948
3949	if (!dead_end && !next_insn) {
3950	if (state->cfi.cfa.base == CFI_UNDEFINED)
3951	return 0;
3952	if (file->ignore_unreachables)
3953	return 0;
3954
3955	WARN("%s%sunexpected end of section %s",
3956	func ? func->name : "", func ? "(): " : "",
3957	insn->sec->name);
3958	return 1;
3959	}
3960
3961	prev_insn = insn;
3962	insn = next_insn;
3963
3964	} while (!dead_end);
3965
3966	return ret;
3967	}
3968
3969	static int validate_branch(struct objtool_file file, struct symbol func,
3970	struct instruction *insn, struct insn_state state)
3971	{
3972	int ret;
3973
3974	trace_depth_inc();
3975	ret = do_validate_branch(file, func, insn, &state);
3976	trace_depth_dec();
3977
3978	return ret;
3979	}
3980
3981	static int validate_unwind_hint(struct objtool_file *file,
3982	struct instruction *insn,
3983	struct insn_state *state)
3984	{
3985	if (insn->hint && !insn->visited) {
3986	struct symbol *func = insn_func(insn);
3987	int ret;
3988
3989	ret = validate_branch(file, func, insn, *state);
3990	if (ret)
3991	BT_INSN(insn, "<=== (hint)");
3992	return ret;
3993	}
3994
3995	return 0;
3996	}
3997
3998	static int validate_unwind_hints(struct objtool_file file, struct section sec)
3999	{
4000	struct instruction *insn;
4001	struct insn_state state;
4002	int warnings = 0;
4003
4004	if (!file->hints)
4005	return 0;
4006
4007	init_insn_state(file, &state, sec);
4008
4009	if (sec) {
4010	sec_for_each_insn(file, sec, insn)
4011	warnings += validate_unwind_hint(file, insn, &state);
4012	} else {
4013	for_each_insn(file, insn)
4014	warnings += validate_unwind_hint(file, insn, &state);
4015	}
4016
4017	return warnings;
4018	}
4019
4020	/*
4021	* Validate rethunk entry constraint: must untrain RET before the first RET.
4022	*
4023	* Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
4024	* before an actual RET instruction.
4025	*/
4026	static int validate_unret(struct objtool_file file, struct instruction insn)
4027	{
4028	struct instruction next, dest;
4029	int ret;
4030
4031	for (;;) {
4032	next = next_insn_to_validate(file, insn);
4033
4034	if (insn->visited & VISITED_UNRET)
4035	return 0;
4036
4037	insn->visited \|= VISITED_UNRET;
4038
4039	if (insn->alts) {
4040	struct alternative *alt;
4041	for (alt = insn->alts; alt; alt = alt->next) {
4042	ret = validate_unret(file, alt->insn);
4043	if (ret) {
4044	BT_INSN(insn, "(alt)");
4045	return ret;
4046	}
4047	}
4048	}
4049
4050	switch (insn->type) {
4051
4052	case INSN_CALL_DYNAMIC:
4053	case INSN_JUMP_DYNAMIC:
4054	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4055	WARN_INSN(insn, "early indirect call");
4056	return 1;
4057
4058	case INSN_JUMP_UNCONDITIONAL:
4059	case INSN_JUMP_CONDITIONAL:
4060	if (!is_sibling_call(insn)) {
4061	if (!insn->jump_dest) {
4062	WARN_INSN(insn, "unresolved jump target after linking?!?");
4063	return 1;
4064	}
4065	ret = validate_unret(file, insn->jump_dest);
4066	if (ret) {
4067	BT_INSN(insn, "(branch%s)",
4068	insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
4069	return ret;
4070	}
4071
4072	if (insn->type == INSN_JUMP_UNCONDITIONAL)
4073	return 0;
4074
4075	break;
4076	}
4077
4078	/* fallthrough */
4079	case INSN_CALL:
4080	dest = find_insn(file, insn_call_dest(insn)->sec,
4081	insn_call_dest(insn)->offset);
4082	if (!dest) {
4083	WARN("Unresolved function after linking!?: %s",
4084	insn_call_dest(insn)->name);
4085	return 1;
4086	}
4087
4088	ret = validate_unret(file, dest);
4089	if (ret) {
4090	BT_INSN(insn, "(call)");
4091	return ret;
4092	}
4093	/*
4094	* If a call returns without error, it must have seen UNTRAIN_RET.
4095	* Therefore any non-error return is a success.
4096	*/
4097	return 0;
4098
4099	case INSN_RETURN:
4100	WARN_INSN(insn, "RET before UNTRAIN");
4101	return 1;
4102
4103	case INSN_SYSCALL:
4104	break;
4105
4106	case INSN_SYSRET:
4107	return 0;
4108
4109	case INSN_NOP:
4110	if (insn->retpoline_safe)
4111	return 0;
4112	break;
4113
4114	default:
4115	break;
4116	}
4117
4118	if (insn->dead_end)
4119	return 0;
4120
4121	if (!next) {
4122	WARN_INSN(insn, "teh end!");
4123	return 1;
4124	}
4125	insn = next;
4126	}
4127
4128	return 0;
4129	}
4130
4131	/*
4132	* Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
4133	* VALIDATE_UNRET_END before RET.
4134	*/
4135	static int validate_unrets(struct objtool_file *file)
4136	{
4137	struct instruction *insn;
4138	int warnings = 0;
4139
4140	for_each_insn(file, insn) {
4141	if (!insn->unret)
4142	continue;
4143
4144	warnings += validate_unret(file, insn);
4145	}
4146
4147	return warnings;
4148	}
4149
4150	static int validate_retpoline(struct objtool_file *file)
4151	{
4152	struct instruction *insn;
4153	int warnings = 0;
4154
4155	for_each_insn(file, insn) {
4156	if (insn->type != INSN_JUMP_DYNAMIC &&
4157	insn->type != INSN_CALL_DYNAMIC &&
4158	insn->type != INSN_RETURN)
4159	continue;
4160
4161	if (insn->retpoline_safe)
4162	continue;
4163
4164	if (insn->sec->init)
4165	continue;
4166
4167	if (insn->type == INSN_RETURN) {
4168	if (opts.rethunk) {
4169	WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4170	warnings++;
4171	}
4172	continue;
4173	}
4174
4175	WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4176	insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4177	warnings++;
4178	}
4179
4180	if (!opts.cfi)
4181	return warnings;
4182
4183	/*
4184	* kCFI call sites look like:
4185	*
4186	* movl $(-0x12345678), %r10d
4187	* addl -4(%r11), %r10d
4188	* jz 1f
4189	* ud2
4190	* 1: cs call __x86_indirect_thunk_r11
4191	*
4192	* Verify all indirect calls are kCFI adorned by checking for the
4193	* UD2. Notably, doing __nocfi calls to regular (cfi) functions is
4194	* broken.
4195	*/
4196	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
4197	struct symbol *sym = insn_sym(insn);
4198
4199	if (sym && (is_notype_sym(sym) \|\|
4200	is_func_sym(sym)) && !sym->nocfi) {
4201	struct instruction *prev =
4202	prev_insn_same_sym(file, insn);
4203
4204	if (!prev \|\| prev->type != INSN_BUG) {
4205	WARN_INSN(insn, "no-cfi indirect call!");
4206	warnings++;
4207	}
4208	}
4209	}
4210
4211	return warnings;
4212	}
4213
4214	static bool is_kasan_insn(struct instruction *insn)
4215	{
4216	return (insn->type == INSN_CALL &&
4217	!strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4218	}
4219
4220	static bool is_ubsan_insn(struct instruction *insn)
4221	{
4222	return (insn->type == INSN_CALL &&
4223	!strcmp(insn_call_dest(insn)->name,
4224	"__ubsan_handle_builtin_unreachable"));
4225	}
4226
4227	static bool ignore_unreachable_insn(struct objtool_file file, struct instruction insn)
4228	{
4229	struct symbol *func = insn_func(insn);
4230	struct instruction *prev_insn;
4231	int i;
4232
4233	if (insn->type == INSN_NOP \|\| insn->type == INSN_TRAP \|\|
4234	insn->hole \|\| (func && func->ignore))
4235	return true;
4236
4237	/*
4238	* Ignore alternative replacement instructions. This can happen
4239	* when a whitelisted function uses one of the ALTERNATIVE macros.
4240	*/
4241	if (!strcmp(insn->sec->name, ".altinstr_replacement") \|\|
4242	!strcmp(insn->sec->name, ".altinstr_aux"))
4243	return true;
4244
4245	if (!func)
4246	return false;
4247
4248	if (func->static_call_tramp)
4249	return true;
4250
4251	/*
4252	* CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4253	* __builtin_unreachable(). The BUG() macro has an unreachable() after
4254	* the UD2, which causes GCC's undefined trap logic to emit another UD2
4255	* (or occasionally a JMP to UD2).
4256	*
4257	* It may also insert a UD2 after calling a __noreturn function.
4258	*/
4259	prev_insn = prev_insn_same_sec(file, insn);
4260	if (prev_insn && prev_insn->dead_end &&
4261	(insn->type == INSN_BUG \|\|
4262	(insn->type == INSN_JUMP_UNCONDITIONAL &&
4263	insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4264	return true;
4265
4266	/*
4267	* Check if this (or a subsequent) instruction is related to
4268	* CONFIG_UBSAN or CONFIG_KASAN.
4269	*
4270	* End the search at 5 instructions to avoid going into the weeds.
4271	*/
4272	for (i = 0; i < 5; i++) {
4273
4274	if (is_kasan_insn(insn) \|\| is_ubsan_insn(insn))
4275	return true;
4276
4277	if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4278	if (insn->jump_dest &&
4279	insn_func(insn->jump_dest) == func) {
4280	insn = insn->jump_dest;
4281	continue;
4282	}
4283
4284	break;
4285	}
4286
4287	if (insn->offset + insn->len >= func->offset + func->len)
4288	break;
4289
4290	insn = next_insn_same_sec(file, insn);
4291	}
4292
4293	return false;
4294	}
4295
4296	/*
4297	* For FineIBT or kCFI, a certain number of bytes preceding the function may be
4298	* NOPs. Those NOPs may be rewritten at runtime and executed, so give them a
4299	* proper function name: __pfx_<func>.
4300	*/
4301	static int create_prefix_symbol(struct objtool_file file, struct symbol func)
4302	{
4303	struct instruction insn, prev;
4304	char name[SYM_NAME_LEN];
4305	struct cfi_state *cfi;
4306
4307	if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) {
4308	WARN("%s: symbol name too long, can't create __pfx_ symbol",
4309	func->name);
4310	return 0;
4311	}
4312
4313	if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name))
4314	return -1;
4315
4316	if (!elf_create_symbol(file->elf, name, func->sec,
4317	GELF_ST_BIND(func->sym.st_info),
4318	GELF_ST_TYPE(func->sym.st_info),
4319	func->offset - opts.prefix, opts.prefix))
4320	return -1;
4321
4322	/* Propagate insn->cfi to the prefix code */
4323	insn = find_insn(file, func->sec, func->offset);
4324	if (!insn \|\| !insn->cfi)
4325	return 0;
4326
4327	cfi = cfi_hash_find_or_add(insn->cfi);
4328	for (prev = find_insn(file, func->sec, func->offset - opts.prefix);
4329	prev && prev != insn;
4330	prev = next_insn_same_sec(file, prev))
4331	prev->cfi = cfi;
4332
4333	return 0;
4334	}
4335
4336	static int create_prefix_symbols(struct objtool_file *file)
4337	{
4338	struct section *pfe_sec;
4339	struct symbol *func;
4340	struct reloc *reloc;
4341
4342	for_each_sec(file->elf, pfe_sec) {
4343	if (strcmp(pfe_sec->name, "__patchable_function_entries"))
4344	continue;
4345	if (!pfe_sec->rsec)
4346	continue;
4347
4348	for_each_reloc(pfe_sec->rsec, reloc) {
4349	func = find_func_by_offset(reloc->sym->sec,
4350	reloc->sym->offset + reloc_addend(reloc) + opts.prefix);
4351	if (func && create_prefix_symbol(file, func))
4352	return -1;
4353	}
4354	}
4355
4356	return 0;
4357	}
4358
4359	static int validate_symbol(struct objtool_file file, struct section sec,
4360	struct symbol sym, struct insn_state state)
4361	{
4362	struct instruction *insn;
4363	struct symbol *func;
4364	int ret;
4365
4366	if (!sym->len) {
4367	WARN("%s() is missing an ELF size annotation", sym->name);
4368	return 1;
4369	}
4370
4371	if (sym->pfunc != sym \|\| is_alias_sym(sym))
4372	return 0;
4373
4374	insn = find_insn(file, sec, sym->offset);
4375	if (!insn \|\| insn->visited)
4376	return 0;
4377
4378	if (opts.uaccess)
4379	state->uaccess = sym->uaccess_safe;
4380
4381	func = insn_func(insn);
4382
4383	if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) {
4384	trace_enable();
4385	TRACE("%s: validation begin\n", sym->name);
4386	}
4387
4388	ret = validate_branch(file, func, insn, *state);
4389	if (ret)
4390	BT_INSN(insn, "<=== (sym)");
4391
4392	TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end");
4393	trace_disable();
4394
4395	return ret;
4396	}
4397
4398	static int validate_section(struct objtool_file file, struct section sec)
4399	{
4400	struct insn_state state;
4401	struct symbol *func;
4402	int warnings = 0;
4403
4404	sec_for_each_sym(sec, func) {
4405	if (!is_func_sym(func))
4406	continue;
4407
4408	init_insn_state(file, &state, sec);
4409	set_func_state(&state.cfi);
4410
4411	warnings += validate_symbol(file, sec, func, &state);
4412	}
4413
4414	return warnings;
4415	}
4416
4417	static int validate_noinstr_sections(struct objtool_file *file)
4418	{
4419	struct section *sec;
4420	int warnings = 0;
4421
4422	sec = find_section_by_name(file->elf, ".noinstr.text");
4423	if (sec) {
4424	warnings += validate_section(file, sec);
4425	warnings += validate_unwind_hints(file, sec);
4426	}
4427
4428	sec = find_section_by_name(file->elf, ".entry.text");
4429	if (sec) {
4430	warnings += validate_section(file, sec);
4431	warnings += validate_unwind_hints(file, sec);
4432	}
4433
4434	sec = find_section_by_name(file->elf, ".cpuidle.text");
4435	if (sec) {
4436	warnings += validate_section(file, sec);
4437	warnings += validate_unwind_hints(file, sec);
4438	}
4439
4440	return warnings;
4441	}
4442
4443	static int validate_functions(struct objtool_file *file)
4444	{
4445	struct section *sec;
4446	int warnings = 0;
4447
4448	for_each_sec(file->elf, sec) {
4449	if (!is_text_sec(sec))
4450	continue;
4451
4452	warnings += validate_section(file, sec);
4453	}
4454
4455	return warnings;
4456	}
4457
4458	static void mark_endbr_used(struct instruction *insn)
4459	{
4460	if (!list_empty(&insn->call_node))
4461	list_del_init(&insn->call_node);
4462	}
4463
4464	static bool noendbr_range(struct objtool_file file, struct instruction insn)
4465	{
4466	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4467	struct instruction *first;
4468
4469	if (!sym)
4470	return false;
4471
4472	first = find_insn(file, sym->sec, sym->offset);
4473	if (!first)
4474	return false;
4475
4476	if (first->type != INSN_ENDBR && !first->noendbr)
4477	return false;
4478
4479	return insn->offset == sym->offset + sym->len;
4480	}
4481
4482	static int __validate_ibt_insn(struct objtool_file file, struct instruction insn,
4483	struct instruction *dest)
4484	{
4485	if (dest->type == INSN_ENDBR) {
4486	mark_endbr_used(dest);
4487	return 0;
4488	}
4489
4490	if (insn_func(dest) && insn_func(insn) &&
4491	insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4492	/*
4493	* Anything from->to self is either _THIS_IP_ or
4494	* IRET-to-self.
4495	*
4496	* There is no sane way to annotate _THIS_IP_ since the
4497	* compiler treats the relocation as a constant and is
4498	* happy to fold in offsets, skewing any annotation we
4499	* do, leading to vast amounts of false-positives.
4500	*
4501	* There's also compiler generated _THIS_IP_ through
4502	* KCOV and such which we have no hope of annotating.
4503	*
4504	* As such, blanket accept self-references without
4505	* issue.
4506	*/
4507	return 0;
4508	}
4509
4510	/*
4511	* Accept anything ANNOTATE_NOENDBR.
4512	*/
4513	if (dest->noendbr)
4514	return 0;
4515
4516	/*
4517	* Accept if this is the instruction after a symbol
4518	* that is (no)endbr -- typical code-range usage.
4519	*/
4520	if (noendbr_range(file, dest))
4521	return 0;
4522
4523	WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4524	return 1;
4525	}
4526
4527	static int validate_ibt_insn(struct objtool_file file, struct instruction insn)
4528	{
4529	struct instruction *dest;
4530	struct reloc *reloc;
4531	unsigned long off;
4532	int warnings = 0;
4533
4534	/*
4535	* Looking for function pointer load relocations. Ignore
4536	* direct/indirect branches:
4537	*/
4538	switch (insn->type) {
4539
4540	case INSN_CALL:
4541	case INSN_CALL_DYNAMIC:
4542	case INSN_JUMP_CONDITIONAL:
4543	case INSN_JUMP_UNCONDITIONAL:
4544	case INSN_JUMP_DYNAMIC:
4545	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4546	case INSN_RETURN:
4547	case INSN_NOP:
4548	return 0;
4549
4550	case INSN_LEA_RIP:
4551	if (!insn_reloc(file, insn)) {
4552	/* local function pointer reference without reloc */
4553
4554	off = arch_jump_destination(insn);
4555
4556	dest = find_insn(file, insn->sec, off);
4557	if (!dest) {
4558	WARN_INSN(insn, "corrupt function pointer reference");
4559	return 1;
4560	}
4561
4562	return __validate_ibt_insn(file, insn, dest);
4563	}
4564	break;
4565
4566	default:
4567	break;
4568	}
4569
4570	for (reloc = insn_reloc(file, insn);
4571	reloc;
4572	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4573	reloc_offset(reloc) + 1,
4574	(insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4575
4576	off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc);
4577
4578	dest = find_insn(file, reloc->sym->sec, off);
4579	if (!dest)
4580	continue;
4581
4582	warnings += __validate_ibt_insn(file, insn, dest);
4583	}
4584
4585	return warnings;
4586	}
4587
4588	static int validate_ibt_data_reloc(struct objtool_file *file,
4589	struct reloc *reloc)
4590	{
4591	struct instruction *dest;
4592
4593	dest = find_insn(file, reloc->sym->sec,
4594	reloc->sym->offset + reloc_addend(reloc));
4595	if (!dest)
4596	return 0;
4597
4598	if (dest->type == INSN_ENDBR) {
4599	mark_endbr_used(dest);
4600	return 0;
4601	}
4602
4603	if (dest->noendbr)
4604	return 0;
4605
4606	WARN_FUNC(reloc->sec->base, reloc_offset(reloc),
4607	"data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4608
4609	return 1;
4610	}
4611
4612	/*
4613	* Validate IBT rules and remove used ENDBR instructions from the seal list.
4614	* Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4615	* NOPs) later, in create_ibt_endbr_seal_sections().
4616	*/
4617	static int validate_ibt(struct objtool_file *file)
4618	{
4619	struct section *sec;
4620	struct reloc *reloc;
4621	struct instruction *insn;
4622	int warnings = 0;
4623
4624	for_each_insn(file, insn)
4625	warnings += validate_ibt_insn(file, insn);
4626
4627	for_each_sec(file->elf, sec) {
4628
4629	/* Already done by validate_ibt_insn() */
4630	if (is_text_sec(sec))
4631	continue;
4632
4633	if (!sec->rsec)
4634	continue;
4635
4636	/*
4637	* These sections can reference text addresses, but not with
4638	* the intent to indirect branch to them.
4639	*/
4640	if ((!strncmp(sec->name, ".discard", 8) &&
4641	strcmp(sec->name, ".discard.ibt_endbr_noseal")) \|\|
4642	!strncmp(sec->name, ".debug", 6) \|\|
4643	!strcmp(sec->name, ".altinstructions") \|\|
4644	!strcmp(sec->name, ".ibt_endbr_seal") \|\|
4645	!strcmp(sec->name, ".kcfi_traps") \|\|
4646	!strcmp(sec->name, ".orc_unwind_ip") \|\|
4647	!strcmp(sec->name, ".retpoline_sites") \|\|
4648	!strcmp(sec->name, ".smp_locks") \|\|
4649	!strcmp(sec->name, ".static_call_sites") \|\|
4650	!strcmp(sec->name, "_error_injection_whitelist") \|\|
4651	!strcmp(sec->name, "_kprobe_blacklist") \|\|
4652	!strcmp(sec->name, "__bug_table") \|\|
4653	!strcmp(sec->name, "__ex_table") \|\|
4654	!strcmp(sec->name, "__jump_table") \|\|
4655	!strcmp(sec->name, ".init.klp_funcs") \|\|
4656	!strcmp(sec->name, "__mcount_loc") \|\|
4657	!strcmp(sec->name, ".llvm.call-graph-profile") \|\|
4658	!strcmp(sec->name, ".llvm_bb_addr_map") \|\|
4659	!strcmp(sec->name, "__tracepoints") \|\|
4660	!strcmp(sec->name, ".return_sites") \|\|
4661	!strcmp(sec->name, ".call_sites") \|\|
4662	!strcmp(sec->name, "__patchable_function_entries"))
4663	continue;
4664
4665	for_each_reloc(sec->rsec, reloc)
4666	warnings += validate_ibt_data_reloc(file, reloc);
4667	}
4668
4669	return warnings;
4670	}
4671
4672	static int validate_sls(struct objtool_file *file)
4673	{
4674	struct instruction insn, next_insn;
4675	int warnings = 0;
4676
4677	for_each_insn(file, insn) {
4678	next_insn = next_insn_same_sec(file, insn);
4679
4680	if (insn->retpoline_safe)
4681	continue;
4682
4683	switch (insn->type) {
4684	case INSN_RETURN:
4685	if (!next_insn \|\| next_insn->type != INSN_TRAP) {
4686	WARN_INSN(insn, "missing int3 after ret");
4687	warnings++;
4688	}
4689
4690	break;
4691	case INSN_JUMP_DYNAMIC:
4692	if (!next_insn \|\| next_insn->type != INSN_TRAP) {
4693	WARN_INSN(insn, "missing int3 after indirect jump");
4694	warnings++;
4695	}
4696	break;
4697	default:
4698	break;
4699	}
4700	}
4701
4702	return warnings;
4703	}
4704
4705	static int validate_reachable_instructions(struct objtool_file *file)
4706	{
4707	struct instruction insn, prev_insn;
4708	struct symbol *call_dest;
4709	int warnings = 0;
4710
4711	if (file->ignore_unreachables)
4712	return 0;
4713
4714	for_each_insn(file, insn) {
4715	if (insn->visited \|\| ignore_unreachable_insn(file, insn))
4716	continue;
4717
4718	prev_insn = prev_insn_same_sec(file, insn);
4719	if (prev_insn && prev_insn->dead_end) {
4720	call_dest = insn_call_dest(prev_insn);
4721	if (call_dest) {
4722	WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h",
4723	call_dest->name);
4724	warnings++;
4725	continue;
4726	}
4727	}
4728
4729	WARN_INSN(insn, "unreachable instruction");
4730	warnings++;
4731	}
4732
4733	return warnings;
4734	}
4735
4736	__weak bool arch_absolute_reloc(struct elf elf, struct reloc reloc)
4737	{
4738	unsigned int type = reloc_type(reloc);
4739	size_t sz = elf_addr_size(elf);
4740
4741	return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32);
4742	}
4743
4744	static int check_abs_references(struct objtool_file *file)
4745	{
4746	struct section *sec;
4747	struct reloc *reloc;
4748	int ret = 0;
4749
4750	for_each_sec(file->elf, sec) {
4751	/* absolute references in non-loadable sections are fine */
4752	if (!(sec->sh.sh_flags & SHF_ALLOC))
4753	continue;
4754
4755	/* section must have an associated .rela section */
4756	if (!sec->rsec)
4757	continue;
4758
4759	/*
4760	* Special case for compiler generated metadata that is not
4761	* consumed until after boot.
4762	*/
4763	if (!strcmp(sec->name, "__patchable_function_entries"))
4764	continue;
4765
4766	for_each_reloc(sec->rsec, reloc) {
4767	if (arch_absolute_reloc(file->elf, reloc)) {
4768	WARN("section %s has absolute relocation at offset 0x%llx",
4769	sec->name, (unsigned long long)reloc_offset(reloc));
4770	ret++;
4771	}
4772	}
4773	}
4774	return ret;
4775	}
4776
4777	struct insn_chunk {
4778	void *addr;
4779	struct insn_chunk *next;
4780	};
4781
4782	/*
4783	* Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4784	* which can trigger more allocations for .debug_* sections whose data hasn't
4785	* been read yet.
4786	*/
4787	void free_insns(struct objtool_file *file)
4788	{
4789	struct instruction *insn;
4790	struct insn_chunk chunks = NULL, chunk;
4791
4792	for_each_insn(file, insn) {
4793	if (!insn->idx) {
4794	chunk = malloc(sizeof(*chunk));
4795	chunk->addr = insn;
4796	chunk->next = chunks;
4797	chunks = chunk;
4798	}
4799	}
4800
4801	for (chunk = chunks; chunk; chunk = chunk->next)
4802	free(chunk->addr);
4803	}
4804
4805	const char objtool_disas_insn(struct instruction insn)
4806	{
4807	struct disas_context *dctx = objtool_disas_ctx;
4808
4809	if (!dctx)
4810	return "";
4811
4812	disas_insn(dctx, insn);
4813	return disas_result(dctx);
4814	}
4815
4816	int check(struct objtool_file *file)
4817	{
4818	struct disas_context *disas_ctx = NULL;
4819	int ret = 0, warnings = 0;
4820
4821	/*
4822	* Create a disassembly context if we might disassemble any
4823	* instruction or function.
4824	*/
4825	if (opts.verbose \|\| opts.backtrace \|\| opts.trace \|\| opts.disas) {
4826	disas_ctx = disas_context_create(file);
4827	if (!disas_ctx) {
4828	opts.disas = false;
4829	opts.trace = false;
4830	}
4831	objtool_disas_ctx = disas_ctx;
4832	}
4833
4834	ret = decode_file(file);
4835	if (ret)
4836	goto out;
4837
4838	if (!nr_insns)
4839	goto out;
4840
4841	if (opts.retpoline)
4842	warnings += validate_retpoline(file);
4843
4844	if (validate_branch_enabled()) {
4845	int w = 0;
4846
4847	w += validate_functions(file);
4848	w += validate_unwind_hints(file, NULL);
4849	if (!w)
4850	w += validate_reachable_instructions(file);
4851
4852	warnings += w;
4853
4854	} else if (opts.noinstr) {
4855	warnings += validate_noinstr_sections(file);
4856	}
4857
4858	if (opts.unret) {
4859	/*
4860	* Must be after validate_branch() and friends, it plays
4861	* further games with insn->visited.
4862	*/
4863	warnings += validate_unrets(file);
4864	}
4865
4866	if (opts.ibt)
4867	warnings += validate_ibt(file);
4868
4869	if (opts.sls)
4870	warnings += validate_sls(file);
4871
4872	if (opts.static_call) {
4873	ret = create_static_call_sections(file);
4874	if (ret)
4875	goto out;
4876	}
4877
4878	if (opts.retpoline) {
4879	ret = create_retpoline_sites_sections(file);
4880	if (ret)
4881	goto out;
4882	}
4883
4884	if (opts.rethunk) {
4885	ret = create_return_sites_sections(file);
4886	if (ret)
4887	goto out;
4888
4889	if (opts.hack_skylake) {
4890	ret = create_direct_call_sections(file);
4891	if (ret)
4892	goto out;
4893	}
4894	}
4895
4896	if (opts.mcount) {
4897	ret = create_mcount_loc_sections(file);
4898	if (ret)
4899	goto out;
4900	}
4901
4902	if (opts.prefix) {
4903	if (!opts.cfi) {
4904	ret = create_prefix_symbols(file);
4905	if (ret)
4906	goto out;
4907	} else {
4908	ret = grow_cfi_symbols(file);
4909	if (ret)
4910	goto out;
4911
4912	if (opts.fineibt) {
4913	ret = create_cfi_sections(file);
4914	if (ret)
4915	goto out;
4916	}
4917	}
4918	}
4919
4920	if (opts.ibt) {
4921	ret = create_ibt_endbr_seal_sections(file);
4922	if (ret)
4923	goto out;
4924	}
4925
4926	if (opts.noabs)
4927	warnings += check_abs_references(file);
4928
4929	if (opts.orc && nr_insns) {
4930	ret = orc_create(file);
4931	if (ret)
4932	goto out;
4933	}
4934
4935	if (opts.stats) {
4936	printf("nr_insns_visited: %ld\n", nr_insns_visited);
4937	printf("nr_cfi: %ld\n", nr_cfi);
4938	printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4939	printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4940	}
4941
4942	out:
4943	if (ret \|\| warnings) {
4944	if (opts.werror && warnings)
4945	ret = 1;
4946
4947	if (opts.verbose) {
4948	if (opts.werror && warnings)
4949	WARN("%d warning(s) upgraded to errors", warnings);
4950	disas_warned_funcs(disas_ctx);
4951	}
4952	}
4953
4954	if (opts.disas)
4955	disas_funcs(disas_ctx);
4956
4957	if (disas_ctx) {
4958	disas_context_destroy(disas_ctx);
4959	objtool_disas_ctx = NULL;
4960	}
4961
4962	free_insns(file);
4963
4964	if (!ret && !warnings)
4965	return 0;
4966
4967	if (opts.backup && make_backup())
4968	return 1;
4969
4970	return ret;
4971	}