linux - tools/objtool/check.c

4979 lines · 113.9 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2015-2017 Josh Poimboeuf <[email protected]>
4	*/
5
6	#define _GNU_SOURCE /* memmem() */
7	#include <fnmatch.h>
8	#include <string.h>
9	#include <stdlib.h>
10	#include <inttypes.h>
11	#include <sys/mman.h>
12
13	#include <objtool/builtin.h>
14	#include <objtool/cfi.h>
15	#include <objtool/arch.h>
16	#include <objtool/disas.h>
17	#include <objtool/check.h>
18	#include <objtool/klp.h>
19	#include <objtool/special.h>
20	#include <objtool/trace.h>
21	#include <objtool/warn.h>
22	#include <objtool/util.h>
23
24	#include <linux/objtool_types.h>
25	#include <linux/hashtable.h>
26	#include <linux/kernel.h>
27	#include <linux/static_call_types.h>
28	#include <linux/string.h>
29
30	static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
31
32	static struct cfi_init_state initial_func_cfi;
33	static struct cfi_state init_cfi;
34	static struct cfi_state func_cfi;
35	static struct cfi_state force_undefined_cfi;
36
37	struct disas_context *objtool_disas_ctx;
38
39	size_t sym_name_max_len;
40
41	struct instruction find_insn(struct objtool_file file,
42	struct section *sec, unsigned long offset)
43	{
44	struct instruction *insn;
45
46	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
47	if (insn->sec == sec && insn->offset == offset)
48	return insn;
49	}
50
51	return NULL;
52	}
53
54	struct instruction next_insn_same_sec(struct objtool_file file,
55	struct instruction *insn)
56	{
57	if (insn->idx == INSN_CHUNK_MAX)
58	return find_insn(file, insn->sec, insn->offset + insn->len);
59
60	insn++;
61	if (!insn->len)
62	return NULL;
63
64	return insn;
65	}
66
67	struct instruction next_insn_same_func(struct objtool_file file,
68	struct instruction *insn)
69	{
70	struct instruction *next = next_insn_same_sec(file, insn);
71	struct symbol *func = insn_func(insn);
72
73	if (!func)
74	return NULL;
75
76	if (next && insn_func(next) == func)
77	return next;
78
79	/* Check if we're already in the subfunction: */
80	if (func == func->cfunc)
81	return NULL;
82
83	/* Move to the subfunction: */
84	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
85	}
86
87	static struct instruction prev_insn_same_sec(struct objtool_file file,
88	struct instruction *insn)
89	{
90	if (insn->idx == 0) {
91	if (insn->prev_len)
92	return find_insn(file, insn->sec, insn->offset - insn->prev_len);
93	return NULL;
94	}
95
96	return insn - 1;
97	}
98
99	static struct instruction prev_insn_same_sym(struct objtool_file file,
100	struct instruction *insn)
101	{
102	struct instruction *prev = prev_insn_same_sec(file, insn);
103
104	if (prev && insn_func(prev) == insn_func(insn))
105	return prev;
106
107	return NULL;
108	}
109
110	#define for_each_insn(file, insn) \
111	for (struct section __sec, __fake = (struct section *)1; \
112	__fake; __fake = NULL) \
113	for_each_sec(file->elf, __sec) \
114	sec_for_each_insn(file, __sec, insn)
115
116
117	#define sym_for_each_insn(file, sym, insn) \
118	for (insn = find_insn(file, sym->sec, sym->offset); \
119	insn && insn->offset < sym->offset + sym->len; \
120	insn = next_insn_same_sec(file, insn))
121
122	#define sym_for_each_insn_continue_reverse(file, sym, insn) \
123	for (insn = prev_insn_same_sec(file, insn); \
124	insn && insn->offset >= sym->offset; \
125	insn = prev_insn_same_sec(file, insn))
126
127	#define sec_for_each_insn_from(file, insn) \
128	for (; insn; insn = next_insn_same_sec(file, insn))
129
130	#define sec_for_each_insn_continue(file, insn) \
131	for (insn = next_insn_same_sec(file, insn); insn; \
132	insn = next_insn_same_sec(file, insn))
133
134	static inline struct reloc insn_jump_table(struct instruction insn)
135	{
136	if (insn->type == INSN_JUMP_DYNAMIC \|\|
137	insn->type == INSN_CALL_DYNAMIC)
138	return insn->_jump_table;
139
140	return NULL;
141	}
142
143	static inline unsigned long insn_jump_table_size(struct instruction *insn)
144	{
145	if (insn->type == INSN_JUMP_DYNAMIC \|\|
146	insn->type == INSN_CALL_DYNAMIC)
147	return insn->_jump_table_size;
148
149	return 0;
150	}
151
152	static bool is_jump_table_jump(struct instruction *insn)
153	{
154	struct alt_group *alt_group = insn->alt_group;
155
156	if (insn_jump_table(insn))
157	return true;
158
159	/* Retpoline alternative for a jump table? */
160	return alt_group && alt_group->orig_group &&
161	insn_jump_table(alt_group->orig_group->first_insn);
162	}
163
164	static bool is_sibling_call(struct instruction *insn)
165	{
166	/*
167	* Assume only STT_FUNC calls have jump-tables.
168	*/
169	if (insn_func(insn)) {
170	/* An indirect jump is either a sibling call or a jump to a table. */
171	if (insn->type == INSN_JUMP_DYNAMIC)
172	return !is_jump_table_jump(insn);
173	}
174
175	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
176	return (is_static_jump(insn) && insn_call_dest(insn));
177	}
178
179	/*
180	* Checks if a function is a Rust "noreturn" one.
181	*/
182	static bool is_rust_noreturn(const struct symbol *func)
183	{
184	/*
185	* If it does not start with "_R", then it is not a Rust symbol.
186	*/
187	if (strncmp(func->name, "_R", 2))
188	return false;
189
190	/*
191	* These are just heuristics -- we do not control the precise symbol
192	* name, due to the crate disambiguators (which depend on the compiler)
193	* as well as changes to the source code itself between versions (since
194	* these come from the Rust standard library).
195	*/
196	return str_ends_with(func->name, "_4core3num20from_str_radix_panic") \|\|
197	str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") \|\|
198	str_ends_with(func->name, "_4core3num28from_ascii_bytes_radix_panic") \|\|
199	str_ends_with(func->name, "_4core3str16slice_error_fail") \|\|
200	str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") \|\|
201	str_ends_with(func->name, "_4core6option13expect_failed") \|\|
202	str_ends_with(func->name, "_4core6option13unwrap_failed") \|\|
203	str_ends_with(func->name, "_4core6result13unwrap_failed") \|\|
204	str_ends_with(func->name, "_4core9panicking5panic") \|\|
205	str_ends_with(func->name, "_4core9panicking9panic_fmt") \|\|
206	str_ends_with(func->name, "_4core9panicking14panic_explicit") \|\|
207	str_ends_with(func->name, "_4core9panicking14panic_nounwind") \|\|
208	str_ends_with(func->name, "_4core9panicking18panic_bounds_check") \|\|
209	str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") \|\|
210	str_ends_with(func->name, "_4core9panicking19assert_failed_inner") \|\|
211	str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") \|\|
212	str_ends_with(func->name, "_4core9panicking32panic_null_reference_constructed") \|\|
213	str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") \|\|
214	str_ends_with(func->name, "_7___rustc17rust_begin_unwind") \|\|
215	strstr(func->name, "_4core9panicking13assert_failed") \|\|
216	strstr(func->name, "_4core9panicking11panic_const24panic_const_") \|\|
217	(strstr(func->name, "_4core5slice5index") &&
218	strstr(func->name, "slice_") &&
219	str_ends_with(func->name, "_fail"));
220	}
221
222	/*
223	* This checks to see if the given function is a "noreturn" function.
224	*
225	* For global functions which are outside the scope of this object file, we
226	* have to keep a manual list of them.
227	*
228	* For local functions, we have to detect them manually by simply looking for
229	* the lack of a return instruction.
230	*/
231	static bool __dead_end_function(struct objtool_file file, struct symbol func,
232	int recursion)
233	{
234	int i;
235	struct instruction *insn;
236	bool empty = true;
237
238	#define NORETURN(func) __stringify(func),
239	static const char * const global_noreturns[] = {
240	#include "noreturns.h"
241	};
242	#undef NORETURN
243
244	if (!func)
245	return false;
246
247	if (!is_local_sym(func)) {
248	if (is_rust_noreturn(func))
249	return true;
250
251	for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
252	if (!strcmp(func->name, global_noreturns[i]))
253	return true;
254	}
255
256	if (is_weak_sym(func))
257	return false;
258
259	if (!func->len)
260	return false;
261
262	insn = find_insn(file, func->sec, func->offset);
263	if (!insn \|\| !insn_func(insn))
264	return false;
265
266	func_for_each_insn(file, func, insn) {
267	empty = false;
268
269	if (insn->type == INSN_RETURN)
270	return false;
271	}
272
273	if (empty)
274	return false;
275
276	/*
277	* A function can have a sibling call instead of a return. In that
278	* case, the function's dead-end status depends on whether the target
279	* of the sibling call returns.
280	*/
281	func_for_each_insn(file, func, insn) {
282	if (is_sibling_call(insn)) {
283	struct instruction *dest = insn->jump_dest;
284
285	if (!dest)
286	/* sibling call to another file */
287	return false;
288
289	/* local sibling call */
290	if (recursion == 5) {
291	/*
292	* Infinite recursion: two functions have
293	* sibling calls to each other. This is a very
294	* rare case. It means they aren't dead ends.
295	*/
296	return false;
297	}
298
299	return __dead_end_function(file, insn_func(dest), recursion+1);
300	}
301	}
302
303	return true;
304	}
305
306	static bool dead_end_function(struct objtool_file file, struct symbol func)
307	{
308	return __dead_end_function(file, func, 0);
309	}
310
311	static void init_cfi_state(struct cfi_state *cfi)
312	{
313	int i;
314
315	for (i = 0; i < CFI_NUM_REGS; i++) {
316	cfi->regs[i].base = CFI_UNDEFINED;
317	cfi->vals[i].base = CFI_UNDEFINED;
318	}
319	cfi->cfa.base = CFI_UNDEFINED;
320	cfi->drap_reg = CFI_UNDEFINED;
321	cfi->drap_offset = -1;
322	}
323
324	static void init_insn_state(struct objtool_file file, struct insn_state state,
325	struct section *sec)
326	{
327	memset(state, 0, sizeof(*state));
328	init_cfi_state(&state->cfi);
329
330	if (opts.noinstr && sec)
331	state->noinstr = sec->noinstr;
332	}
333
334	static struct cfi_state *cfi_alloc(void)
335	{
336	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
337	if (!cfi) {
338	ERROR_GLIBC("calloc");
339	exit(1);
340	}
341	nr_cfi++;
342	return cfi;
343	}
344
345	static int cfi_bits;
346	static struct hlist_head *cfi_hash;
347
348	static inline bool cficmp(struct cfi_state cfi1, struct cfi_state cfi2)
349	{
350	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
351	(void *)cfi2 + sizeof(cfi2->hash),
352	sizeof(struct cfi_state) - sizeof(struct hlist_node));
353	}
354
355	static inline u32 cfi_key(struct cfi_state *cfi)
356	{
357	return jhash((void *)cfi + sizeof(cfi->hash),
358	sizeof(*cfi) - sizeof(cfi->hash), 0);
359	}
360
361	static struct cfi_state cfi_hash_find_or_add(struct cfi_state cfi)
362	{
363	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
364	struct cfi_state *obj;
365
366	hlist_for_each_entry(obj, head, hash) {
367	if (!cficmp(cfi, obj)) {
368	nr_cfi_cache++;
369	return obj;
370	}
371	}
372
373	obj = cfi_alloc();
374	obj = cfi;
375	hlist_add_head(&obj->hash, head);
376
377	return obj;
378	}
379
380	static void cfi_hash_add(struct cfi_state *cfi)
381	{
382	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
383
384	hlist_add_head(&cfi->hash, head);
385	}
386
387	static void *cfi_hash_alloc(unsigned long size)
388	{
389	cfi_bits = max(10, ilog2(size));
390	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
391	PROT_READ\|PROT_WRITE,
392	MAP_PRIVATE\|MAP_ANON, -1, 0);
393	if (cfi_hash == (void *)-1L) {
394	ERROR_GLIBC("mmap fail cfi_hash");
395	cfi_hash = NULL;
396	} else if (opts.stats) {
397	printf("cfi_bits: %d\n", cfi_bits);
398	}
399
400	return cfi_hash;
401	}
402
403	static unsigned long nr_insns;
404	static unsigned long nr_insns_visited;
405
406	/*
407	* Call the arch-specific instruction decoder for all the instructions and add
408	* them to the global instruction list.
409	*/
410	static int decode_instructions(struct objtool_file *file)
411	{
412	struct section *sec;
413	struct symbol *func;
414	unsigned long offset;
415	struct instruction *insn;
416
417	for_each_sec(file->elf, sec) {
418	struct instruction *insns = NULL;
419	u8 prev_len = 0;
420	u8 idx = 0;
421
422	if (!is_text_sec(sec))
423	continue;
424
425	if (strcmp(sec->name, ".altinstr_replacement") &&
426	strcmp(sec->name, ".altinstr_aux") &&
427	strncmp(sec->name, ".discard.", 9))
428	sec->text = true;
429
430	if (!strcmp(sec->name, ".noinstr.text") \|\|
431	!strcmp(sec->name, ".entry.text") \|\|
432	!strcmp(sec->name, ".cpuidle.text") \|\|
433	!strncmp(sec->name, ".text..__x86.", 13))
434	sec->noinstr = true;
435
436	/*
437	* .init.text code is ran before userspace and thus doesn't
438	* strictly need retpolines, except for modules which are
439	* loaded late, they very much do need retpoline in their
440	* .init.text
441	*/
442	if (!strcmp(sec->name, ".init.text") && !opts.module)
443	sec->init = true;
444
445	for (offset = 0; offset < sec_size(sec); offset += insn->len) {
446	if (!insns \|\| idx == INSN_CHUNK_MAX) {
447	insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn));
448	if (!insns) {
449	ERROR_GLIBC("calloc");
450	return -1;
451	}
452	idx = 0;
453	} else {
454	idx++;
455	}
456	insn = &insns[idx];
457	insn->idx = idx;
458
459	INIT_LIST_HEAD(&insn->call_node);
460	insn->sec = sec;
461	insn->offset = offset;
462	insn->prev_len = prev_len;
463
464	if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn))
465	return -1;
466
467	prev_len = insn->len;
468
469	/*
470	* By default, "ud2" is a dead end unless otherwise
471	* annotated, because GCC 7 inserts it for certain
472	* divide-by-zero cases.
473	*/
474	if (insn->type == INSN_BUG)
475	insn->dead_end = true;
476
477	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
478	nr_insns++;
479	}
480
481	sec_for_each_sym(sec, func) {
482	if (!is_notype_sym(func) && !is_func_sym(func))
483	continue;
484
485	if (func->offset == sec_size(sec)) {
486	/* Heuristic: likely an "end" symbol */
487	if (is_notype_sym(func))
488	continue;
489	ERROR("%s(): STT_FUNC at end of section", func->name);
490	return -1;
491	}
492
493	if (func->embedded_insn \|\| is_alias_sym(func))
494	continue;
495
496	if (!find_insn(file, sec, func->offset)) {
497	ERROR("%s(): can't find starting instruction", func->name);
498	return -1;
499	}
500
501	sym_for_each_insn(file, func, insn) {
502	insn->_sym = func;
503	if (is_func_sym(func) &&
504	insn->type == INSN_ENDBR &&
505	list_empty(&insn->call_node)) {
506	if (insn->offset == func->offset) {
507	list_add_tail(&insn->call_node, &file->endbr_list);
508	file->nr_endbr++;
509	} else {
510	file->nr_endbr_int++;
511	}
512	}
513	}
514	}
515	}
516
517	if (opts.stats)
518	printf("nr_insns: %lu\n", nr_insns);
519
520	return 0;
521	}
522
523	/*
524	* Known pv_ops*[] arrays.
525	*/
526	static struct {
527	const char *name;
528	int idx_off;
529	} pv_ops_tables[] = {
530	{ .name = "pv_ops", },
531	{ .name = "pv_ops_lock", },
532	{ .name = NULL, .idx_off = -1 }
533	};
534
535	/*
536	* Get index offset for a pv_ops* array.
537	*/
538	int pv_ops_idx_off(const char *symname)
539	{
540	int idx;
541
542	for (idx = 0; pv_ops_tables[idx].name; idx++) {
543	if (!strcmp(symname, pv_ops_tables[idx].name))
544	break;
545	}
546
547	return pv_ops_tables[idx].idx_off;
548	}
549
550	/*
551	* Read a pv_ops*[] .data table to find the static initialized values.
552	*/
553	static int add_pv_ops(struct objtool_file *file, int pv_ops_idx)
554	{
555	struct symbol sym, func;
556	unsigned long off, end;
557	struct reloc *reloc;
558	int idx, idx_off;
559	const char *symname;
560
561	symname = pv_ops_tables[pv_ops_idx].name;
562	sym = find_symbol_by_name(file->elf, symname);
563	if (!sym) {
564	ERROR("Unknown pv_ops array %s", symname);
565	return -1;
566	}
567
568	off = sym->offset;
569	end = off + sym->len;
570	idx_off = pv_ops_tables[pv_ops_idx].idx_off;
571	if (idx_off < 0) {
572	ERROR("pv_ops array %s has unknown index offset", symname);
573	return -1;
574	}
575
576	for (;;) {
577	reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
578	if (!reloc)
579	break;
580
581	idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
582
583	func = reloc->sym;
584	if (is_sec_sym(func))
585	func = find_symbol_by_offset(reloc->sym->sec,
586	reloc_addend(reloc));
587	if (!func) {
588	ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc),
589	"can't find func at %s[%d]", symname, idx);
590	return -1;
591	}
592
593	if (objtool_pv_add(file, idx + idx_off, func))
594	return -1;
595
596	off = reloc_offset(reloc) + 1;
597	if (off > end)
598	break;
599	}
600
601	return 0;
602	}
603
604	/*
605	* Allocate and initialize file->pv_ops[].
606	*/
607	static int init_pv_ops(struct objtool_file *file)
608	{
609	struct symbol *sym;
610	int idx, nr;
611
612	if (!opts.noinstr)
613	return 0;
614
615	file->pv_ops = NULL;
616
617	nr = 0;
618	for (idx = 0; pv_ops_tables[idx].name; idx++) {
619	sym = find_symbol_by_name(file->elf, pv_ops_tables[idx].name);
620	if (!sym) {
621	pv_ops_tables[idx].idx_off = -1;
622	continue;
623	}
624	pv_ops_tables[idx].idx_off = nr;
625	nr += sym->len / sizeof(unsigned long);
626	}
627
628	if (nr == 0)
629	return 0;
630
631	file->pv_ops = calloc(nr, sizeof(struct pv_state));
632	if (!file->pv_ops) {
633	ERROR_GLIBC("calloc");
634	return -1;
635	}
636
637	for (idx = 0; idx < nr; idx++)
638	INIT_LIST_HEAD(&file->pv_ops[idx].targets);
639
640	for (idx = 0; pv_ops_tables[idx].name; idx++) {
641	if (pv_ops_tables[idx].idx_off < 0)
642	continue;
643	if (add_pv_ops(file, idx))
644	return -1;
645	}
646
647	return 0;
648	}
649
650	static bool is_livepatch_module(struct objtool_file *file)
651	{
652	struct section *sec;
653
654	if (!opts.module)
655	return false;
656
657	sec = find_section_by_name(file->elf, ".modinfo");
658	if (!sec)
659	return false;
660
661	return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12);
662	}
663
664	static int create_static_call_sections(struct objtool_file *file)
665	{
666	struct static_call_site *site;
667	struct section *sec;
668	struct instruction *insn;
669	struct symbol *key_sym;
670	char key_name, tmp;
671	int idx;
672
673	sec = find_section_by_name(file->elf, ".static_call_sites");
674	if (sec) {
675	/*
676	* Livepatch modules may have already extracted the static call
677	* site entries to take advantage of vmlinux static call
678	* privileges.
679	*/
680	if (!file->klp)
681	WARN("file already has .static_call_sites section, skipping");
682
683	return 0;
684	}
685
686	if (list_empty(&file->static_call_list))
687	return 0;
688
689	idx = 0;
690	list_for_each_entry(insn, &file->static_call_list, call_node)
691	idx++;
692
693	sec = elf_create_section_pair(file->elf, ".static_call_sites",
694	sizeof(site), idx, idx 2);
695	if (!sec)
696	return -1;
697
698	/* Allow modules to modify the low bits of static_call_site::key */
699	sec->sh.sh_flags \|= SHF_WRITE;
700
701	idx = 0;
702	list_for_each_entry(insn, &file->static_call_list, call_node) {
703
704	/* populate reloc for 'addr' */
705	if (!elf_init_reloc_text_sym(file->elf, sec,
706	idx * sizeof(site), idx 2,
707	insn->sec, insn->offset))
708	return -1;
709
710	/* find key symbol */
711	key_name = strdup(insn_call_dest(insn)->name);
712	if (!key_name) {
713	ERROR_GLIBC("strdup");
714	return -1;
715	}
716	if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
717	STATIC_CALL_TRAMP_PREFIX_LEN)) {
718	ERROR("static_call: trampoline name malformed: %s", key_name);
719	return -1;
720	}
721	tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
722	memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
723
724	key_sym = find_symbol_by_name(file->elf, tmp);
725	if (!key_sym) {
726	if (!opts.module) {
727	ERROR("static_call: can't find static_call_key symbol: %s", tmp);
728	return -1;
729	}
730
731	/*
732	* For modules(), the key might not be exported, which
733	* means the module can make static calls but isn't
734	* allowed to change them.
735	*
736	* In that case we temporarily set the key to be the
737	* trampoline address. This is fixed up in
738	* static_call_add_module().
739	*/
740	key_sym = insn_call_dest(insn);
741	}
742
743	/* populate reloc for 'key' */
744	if (!elf_init_reloc_data_sym(file->elf, sec,
745	idx * sizeof(*site) + 4,
746	(idx * 2) + 1, key_sym,
747	is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
748	return -1;
749
750	idx++;
751	}
752
753	return 0;
754	}
755
756	static int create_retpoline_sites_sections(struct objtool_file *file)
757	{
758	struct instruction *insn;
759	struct section *sec;
760	int idx;
761
762	sec = find_section_by_name(file->elf, ".retpoline_sites");
763	if (sec) {
764	WARN("file already has .retpoline_sites, skipping");
765	return 0;
766	}
767
768	idx = 0;
769	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
770	idx++;
771
772	if (!idx)
773	return 0;
774
775	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
776	sizeof(int), idx, idx);
777	if (!sec)
778	return -1;
779
780	idx = 0;
781	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
782
783	if (!elf_init_reloc_text_sym(file->elf, sec,
784	idx * sizeof(int), idx,
785	insn->sec, insn->offset))
786	return -1;
787
788	idx++;
789	}
790
791	return 0;
792	}
793
794	static int create_return_sites_sections(struct objtool_file *file)
795	{
796	struct instruction *insn;
797	struct section *sec;
798	int idx;
799
800	sec = find_section_by_name(file->elf, ".return_sites");
801	if (sec) {
802	WARN("file already has .return_sites, skipping");
803	return 0;
804	}
805
806	idx = 0;
807	list_for_each_entry(insn, &file->return_thunk_list, call_node)
808	idx++;
809
810	if (!idx)
811	return 0;
812
813	sec = elf_create_section_pair(file->elf, ".return_sites",
814	sizeof(int), idx, idx);
815	if (!sec)
816	return -1;
817
818	idx = 0;
819	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
820
821	if (!elf_init_reloc_text_sym(file->elf, sec,
822	idx * sizeof(int), idx,
823	insn->sec, insn->offset))
824	return -1;
825
826	idx++;
827	}
828
829	return 0;
830	}
831
832	static int create_ibt_endbr_seal_sections(struct objtool_file *file)
833	{
834	struct instruction *insn;
835	struct section *sec;
836	int idx;
837
838	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
839	if (sec) {
840	WARN("file already has .ibt_endbr_seal, skipping");
841	return 0;
842	}
843
844	idx = 0;
845	list_for_each_entry(insn, &file->endbr_list, call_node)
846	idx++;
847
848	if (opts.stats) {
849	printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
850	printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
851	printf("ibt: superfluous ENDBR: %d\n", idx);
852	}
853
854	if (!idx)
855	return 0;
856
857	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
858	sizeof(int), idx, idx);
859	if (!sec)
860	return -1;
861
862	idx = 0;
863	list_for_each_entry(insn, &file->endbr_list, call_node) {
864
865	int site = (int )sec->data->d_buf + idx;
866	struct symbol *func = insn_func(insn);
867	*site = 0;
868
869	if (opts.module && func && insn->offset == func->offset &&
870	(!strcmp(func->name, "init_module") \|\|
871	!strcmp(func->name, "cleanup_module"))) {
872	ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead",
873	func->name);
874	return -1;
875	}
876
877	if (!elf_init_reloc_text_sym(file->elf, sec,
878	idx * sizeof(int), idx,
879	insn->sec, insn->offset))
880	return -1;
881
882	idx++;
883	}
884
885	return 0;
886	}
887
888	/*
889	* Grow __cfi_ symbols to fill the NOP gap between the 'mov <hash>, %rax' and
890	* the start of the function.
891	*/
892	static int grow_cfi_symbols(struct objtool_file *file)
893	{
894	struct symbol *sym;
895
896	for_each_sym(file->elf, sym) {
897	if (!is_func_sym(sym) \|\| !strstarts(sym->name, "__cfi_") \|\|
898	sym->len != 5)
899	continue;
900
901	if (!find_func_by_offset(sym->sec, sym->offset + sym->len + opts.prefix))
902	continue;
903
904	sym->len += opts.prefix;
905	sym->sym.st_size = sym->len;
906	if (elf_write_symbol(file->elf, sym))
907	return -1;
908	}
909
910	return 0;
911	}
912
913	static int create_cfi_sections(struct objtool_file *file)
914	{
915	struct section *sec;
916	struct symbol *sym;
917	int idx;
918
919	sec = find_section_by_name(file->elf, ".cfi_sites");
920	if (sec) {
921	WARN("file already has .cfi_sites section, skipping");
922	return 0;
923	}
924
925	idx = 0;
926	for_each_sym(file->elf, sym) {
927	if (!is_func_sym(sym))
928	continue;
929
930	if (strncmp(sym->name, "__cfi_", 6))
931	continue;
932
933	idx++;
934	}
935
936	sec = elf_create_section_pair(file->elf, ".cfi_sites",
937	sizeof(unsigned int), idx, idx);
938	if (!sec)
939	return -1;
940
941	idx = 0;
942	for_each_sym(file->elf, sym) {
943	if (!is_func_sym(sym))
944	continue;
945
946	if (strncmp(sym->name, "__cfi_", 6))
947	continue;
948
949	if (!elf_init_reloc_text_sym(file->elf, sec,
950	idx * sizeof(unsigned int), idx,
951	sym->sec, sym->offset))
952	return -1;
953
954	idx++;
955	}
956
957	return 0;
958	}
959
960	static int create_mcount_loc_sections(struct objtool_file *file)
961	{
962	size_t addr_size = elf_addr_size(file->elf);
963	struct instruction *insn;
964	struct section *sec;
965	int idx;
966
967	sec = find_section_by_name(file->elf, "__mcount_loc");
968	if (sec) {
969	/*
970	* Livepatch modules have already extracted their __mcount_loc
971	* entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case.
972	*/
973	if (!file->klp)
974	WARN("file already has __mcount_loc section, skipping");
975
976	return 0;
977	}
978
979	if (list_empty(&file->mcount_loc_list))
980	return 0;
981
982	idx = 0;
983	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
984	idx++;
985
986	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
987	idx, idx);
988	if (!sec)
989	return -1;
990
991	sec->sh.sh_addralign = addr_size;
992
993	idx = 0;
994	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
995
996	struct reloc *reloc;
997
998	reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
999	insn->sec, insn->offset);
1000	if (!reloc)
1001	return -1;
1002
1003	set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
1004
1005	idx++;
1006	}
1007
1008	return 0;
1009	}
1010
1011	static int create_direct_call_sections(struct objtool_file *file)
1012	{
1013	struct instruction *insn;
1014	struct section *sec;
1015	int idx;
1016
1017	sec = find_section_by_name(file->elf, ".call_sites");
1018	if (sec) {
1019	WARN("file already has .call_sites section, skipping");
1020	return 0;
1021	}
1022
1023	if (list_empty(&file->call_list))
1024	return 0;
1025
1026	idx = 0;
1027	list_for_each_entry(insn, &file->call_list, call_node)
1028	idx++;
1029
1030	sec = elf_create_section_pair(file->elf, ".call_sites",
1031	sizeof(unsigned int), idx, idx);
1032	if (!sec)
1033	return -1;
1034
1035	idx = 0;
1036	list_for_each_entry(insn, &file->call_list, call_node) {
1037
1038	if (!elf_init_reloc_text_sym(file->elf, sec,
1039	idx * sizeof(unsigned int), idx,
1040	insn->sec, insn->offset))
1041	return -1;
1042
1043	idx++;
1044	}
1045
1046	return 0;
1047	}
1048
1049	/*
1050	* Warnings shouldn't be reported for ignored functions.
1051	*/
1052	static int add_ignores(struct objtool_file *file)
1053	{
1054	struct section *rsec;
1055	struct symbol *func;
1056	struct reloc *reloc;
1057
1058	rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1059	if (!rsec)
1060	return 0;
1061
1062	for_each_reloc(rsec, reloc) {
1063	switch (reloc->sym->type) {
1064	case STT_FUNC:
1065	func = reloc->sym;
1066	break;
1067
1068	case STT_SECTION:
1069	func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1070	if (!func)
1071	continue;
1072	break;
1073
1074	default:
1075	ERROR("unexpected relocation symbol type in %s: %d",
1076	rsec->name, reloc->sym->type);
1077	return -1;
1078	}
1079
1080	func->ignore = true;
1081	if (func->cfunc)
1082	func->cfunc->ignore = true;
1083	}
1084
1085	return 0;
1086	}
1087
1088	/*
1089	* This is a whitelist of functions that is allowed to be called with AC set.
1090	* The list is meant to be minimal and only contains compiler instrumentation
1091	* ABI and a few functions used to implement *_{to,from}_user() functions.
1092	*
1093	* These functions must not directly change AC, but may PUSHF/POPF.
1094	*/
1095	static const char *uaccess_safe_builtin[] = {
1096	/* KASAN */
1097	"kasan_report",
1098	"kasan_check_range",
1099	/* KASAN out-of-line */
1100	"__asan_loadN_noabort",
1101	"__asan_load1_noabort",
1102	"__asan_load2_noabort",
1103	"__asan_load4_noabort",
1104	"__asan_load8_noabort",
1105	"__asan_load16_noabort",
1106	"__asan_storeN_noabort",
1107	"__asan_store1_noabort",
1108	"__asan_store2_noabort",
1109	"__asan_store4_noabort",
1110	"__asan_store8_noabort",
1111	"__asan_store16_noabort",
1112	"__kasan_check_read",
1113	"__kasan_check_write",
1114	/* KASAN in-line */
1115	"__asan_report_load_n_noabort",
1116	"__asan_report_load1_noabort",
1117	"__asan_report_load2_noabort",
1118	"__asan_report_load4_noabort",
1119	"__asan_report_load8_noabort",
1120	"__asan_report_load16_noabort",
1121	"__asan_report_store_n_noabort",
1122	"__asan_report_store1_noabort",
1123	"__asan_report_store2_noabort",
1124	"__asan_report_store4_noabort",
1125	"__asan_report_store8_noabort",
1126	"__asan_report_store16_noabort",
1127	/* KCSAN */
1128	"__kcsan_check_access",
1129	"__kcsan_mb",
1130	"__kcsan_wmb",
1131	"__kcsan_rmb",
1132	"__kcsan_release",
1133	"kcsan_found_watchpoint",
1134	"kcsan_setup_watchpoint",
1135	"kcsan_check_scoped_accesses",
1136	"kcsan_disable_current",
1137	"kcsan_enable_current_nowarn",
1138	/* KCSAN/TSAN */
1139	"__tsan_func_entry",
1140	"__tsan_func_exit",
1141	"__tsan_read_range",
1142	"__tsan_write_range",
1143	"__tsan_read1",
1144	"__tsan_read2",
1145	"__tsan_read4",
1146	"__tsan_read8",
1147	"__tsan_read16",
1148	"__tsan_write1",
1149	"__tsan_write2",
1150	"__tsan_write4",
1151	"__tsan_write8",
1152	"__tsan_write16",
1153	"__tsan_read_write1",
1154	"__tsan_read_write2",
1155	"__tsan_read_write4",
1156	"__tsan_read_write8",
1157	"__tsan_read_write16",
1158	"__tsan_volatile_read1",
1159	"__tsan_volatile_read2",
1160	"__tsan_volatile_read4",
1161	"__tsan_volatile_read8",
1162	"__tsan_volatile_read16",
1163	"__tsan_volatile_write1",
1164	"__tsan_volatile_write2",
1165	"__tsan_volatile_write4",
1166	"__tsan_volatile_write8",
1167	"__tsan_volatile_write16",
1168	"__tsan_atomic8_load",
1169	"__tsan_atomic16_load",
1170	"__tsan_atomic32_load",
1171	"__tsan_atomic64_load",
1172	"__tsan_atomic8_store",
1173	"__tsan_atomic16_store",
1174	"__tsan_atomic32_store",
1175	"__tsan_atomic64_store",
1176	"__tsan_atomic8_exchange",
1177	"__tsan_atomic16_exchange",
1178	"__tsan_atomic32_exchange",
1179	"__tsan_atomic64_exchange",
1180	"__tsan_atomic8_fetch_add",
1181	"__tsan_atomic16_fetch_add",
1182	"__tsan_atomic32_fetch_add",
1183	"__tsan_atomic64_fetch_add",
1184	"__tsan_atomic8_fetch_sub",
1185	"__tsan_atomic16_fetch_sub",
1186	"__tsan_atomic32_fetch_sub",
1187	"__tsan_atomic64_fetch_sub",
1188	"__tsan_atomic8_fetch_and",
1189	"__tsan_atomic16_fetch_and",
1190	"__tsan_atomic32_fetch_and",
1191	"__tsan_atomic64_fetch_and",
1192	"__tsan_atomic8_fetch_or",
1193	"__tsan_atomic16_fetch_or",
1194	"__tsan_atomic32_fetch_or",
1195	"__tsan_atomic64_fetch_or",
1196	"__tsan_atomic8_fetch_xor",
1197	"__tsan_atomic16_fetch_xor",
1198	"__tsan_atomic32_fetch_xor",
1199	"__tsan_atomic64_fetch_xor",
1200	"__tsan_atomic8_fetch_nand",
1201	"__tsan_atomic16_fetch_nand",
1202	"__tsan_atomic32_fetch_nand",
1203	"__tsan_atomic64_fetch_nand",
1204	"__tsan_atomic8_compare_exchange_strong",
1205	"__tsan_atomic16_compare_exchange_strong",
1206	"__tsan_atomic32_compare_exchange_strong",
1207	"__tsan_atomic64_compare_exchange_strong",
1208	"__tsan_atomic8_compare_exchange_weak",
1209	"__tsan_atomic16_compare_exchange_weak",
1210	"__tsan_atomic32_compare_exchange_weak",
1211	"__tsan_atomic64_compare_exchange_weak",
1212	"__tsan_atomic8_compare_exchange_val",
1213	"__tsan_atomic16_compare_exchange_val",
1214	"__tsan_atomic32_compare_exchange_val",
1215	"__tsan_atomic64_compare_exchange_val",
1216	"__tsan_atomic_thread_fence",
1217	"__tsan_atomic_signal_fence",
1218	"__tsan_unaligned_read16",
1219	"__tsan_unaligned_write16",
1220	/* KCOV */
1221	"write_comp_data",
1222	"check_kcov_mode",
1223	"__sanitizer_cov_trace_pc",
1224	"__sanitizer_cov_trace_const_cmp1",
1225	"__sanitizer_cov_trace_const_cmp2",
1226	"__sanitizer_cov_trace_const_cmp4",
1227	"__sanitizer_cov_trace_const_cmp8",
1228	"__sanitizer_cov_trace_cmp1",
1229	"__sanitizer_cov_trace_cmp2",
1230	"__sanitizer_cov_trace_cmp4",
1231	"__sanitizer_cov_trace_cmp8",
1232	"__sanitizer_cov_trace_switch",
1233	/* KMSAN */
1234	"kmsan_copy_to_user",
1235	"kmsan_disable_current",
1236	"kmsan_enable_current",
1237	"kmsan_report",
1238	"kmsan_unpoison_entry_regs",
1239	"kmsan_unpoison_memory",
1240	"__msan_chain_origin",
1241	"__msan_get_context_state",
1242	"__msan_instrument_asm_store",
1243	"__msan_metadata_ptr_for_load_1",
1244	"__msan_metadata_ptr_for_load_2",
1245	"__msan_metadata_ptr_for_load_4",
1246	"__msan_metadata_ptr_for_load_8",
1247	"__msan_metadata_ptr_for_load_n",
1248	"__msan_metadata_ptr_for_store_1",
1249	"__msan_metadata_ptr_for_store_2",
1250	"__msan_metadata_ptr_for_store_4",
1251	"__msan_metadata_ptr_for_store_8",
1252	"__msan_metadata_ptr_for_store_n",
1253	"__msan_poison_alloca",
1254	"__msan_warning",
1255	/* UBSAN */
1256	"ubsan_type_mismatch_common",
1257	"__ubsan_handle_type_mismatch",
1258	"__ubsan_handle_type_mismatch_v1",
1259	"__ubsan_handle_shift_out_of_bounds",
1260	"__ubsan_handle_load_invalid_value",
1261	/* KSTACK_ERASE */
1262	"__sanitizer_cov_stack_depth",
1263	/* TRACE_BRANCH_PROFILING */
1264	"ftrace_likely_update",
1265	/* STACKPROTECTOR */
1266	"__stack_chk_fail",
1267	/* misc */
1268	"csum_partial_copy_generic",
1269	"copy_mc_fragile",
1270	"copy_mc_fragile_handle_tail",
1271	"copy_mc_enhanced_fast_string",
1272	"rep_stos_alternative",
1273	"rep_movs_alternative",
1274	"copy_to_nontemporal",
1275	NULL
1276	};
1277
1278	static void add_uaccess_safe(struct objtool_file *file)
1279	{
1280	struct symbol *func;
1281	const char **name;
1282
1283	if (!opts.uaccess)
1284	return;
1285
1286	for (name = uaccess_safe_builtin; *name; name++) {
1287	func = find_symbol_by_name(file->elf, *name);
1288	if (!func)
1289	continue;
1290
1291	func->uaccess_safe = true;
1292	}
1293	}
1294
1295	/*
1296	* Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1297	* will be added to the .retpoline_sites section.
1298	*/
1299	__weak bool arch_is_retpoline(struct symbol *sym)
1300	{
1301	return false;
1302	}
1303
1304	/*
1305	* Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1306	* will be added to the .return_sites section.
1307	*/
1308	__weak bool arch_is_rethunk(struct symbol *sym)
1309	{
1310	return false;
1311	}
1312
1313	/*
1314	* Symbols that are embedded inside other instructions, because sometimes crazy
1315	* code exists. These are mostly ignored for validation purposes.
1316	*/
1317	__weak bool arch_is_embedded_insn(struct symbol *sym)
1318	{
1319	return false;
1320	}
1321
1322	struct reloc insn_reloc(struct objtool_file file, struct instruction *insn)
1323	{
1324	struct reloc *reloc;
1325
1326	if (!file \|\| insn->no_reloc \|\| insn->fake)
1327	return NULL;
1328
1329	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1330	insn->offset, insn->len);
1331	if (!reloc) {
1332	insn->no_reloc = 1;
1333	return NULL;
1334	}
1335
1336	return reloc;
1337	}
1338
1339	static void remove_insn_ops(struct instruction *insn)
1340	{
1341	struct stack_op op, next;
1342
1343	for (op = insn->stack_ops; op; op = next) {
1344	next = op->next;
1345	free(op);
1346	}
1347	insn->stack_ops = NULL;
1348	}
1349
1350	static int annotate_call_site(struct objtool_file *file,
1351	struct instruction *insn, bool sibling)
1352	{
1353	struct reloc *reloc = insn_reloc(file, insn);
1354	struct symbol *sym = insn_call_dest(insn);
1355
1356	if (!sym)
1357	sym = reloc->sym;
1358
1359	if (sym->static_call_tramp) {
1360	list_add_tail(&insn->call_node, &file->static_call_list);
1361	return 0;
1362	}
1363
1364	if (sym->retpoline_thunk) {
1365	list_add_tail(&insn->call_node, &file->retpoline_call_list);
1366	return 0;
1367	}
1368
1369	/*
1370	* Many compilers cannot disable KCOV or sanitizer calls with a function
1371	* attribute so they need a little help, NOP out any such calls from
1372	* noinstr text.
1373	*/
1374	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1375	if (reloc)
1376	set_reloc_type(file->elf, reloc, R_NONE);
1377
1378	if (elf_write_insn(file->elf, insn->sec,
1379	insn->offset, insn->len,
1380	sibling ? arch_ret_insn(insn->len)
1381	: arch_nop_insn(insn->len))) {
1382	return -1;
1383	}
1384
1385	insn->type = sibling ? INSN_RETURN : INSN_NOP;
1386
1387	if (sibling) {
1388	/*
1389	* We've replaced the tail-call JMP insn by two new
1390	* insn: RET; INT3, except we only have a single struct
1391	* insn here. Mark it retpoline_safe to avoid the SLS
1392	* warning, instead of adding another insn.
1393	*/
1394	insn->retpoline_safe = true;
1395	}
1396
1397	return 0;
1398	}
1399
1400	if (opts.mcount && sym->fentry) {
1401	if (sibling)
1402	WARN_INSN(insn, "tail call to __fentry__ !?!?");
1403	if (opts.mnop) {
1404	if (reloc)
1405	set_reloc_type(file->elf, reloc, R_NONE);
1406
1407	if (elf_write_insn(file->elf, insn->sec,
1408	insn->offset, insn->len,
1409	arch_nop_insn(insn->len))) {
1410	return -1;
1411	}
1412
1413	insn->type = INSN_NOP;
1414	}
1415
1416	list_add_tail(&insn->call_node, &file->mcount_loc_list);
1417	return 0;
1418	}
1419
1420	if (insn->type == INSN_CALL && !insn->sec->init &&
1421	!insn->_call_dest->embedded_insn)
1422	list_add_tail(&insn->call_node, &file->call_list);
1423
1424	if (!sibling && dead_end_function(file, sym))
1425	insn->dead_end = true;
1426
1427	return 0;
1428	}
1429
1430	static int add_call_dest(struct objtool_file file, struct instruction insn,
1431	struct symbol *dest, bool sibling)
1432	{
1433	insn->_call_dest = dest;
1434	if (!dest)
1435	return 0;
1436
1437	/*
1438	* Whatever stack impact regular CALLs have, should be undone
1439	* by the RETURN of the called function.
1440	*
1441	* Annotated intra-function calls retain the stack_ops but
1442	* are converted to JUMP, see read_intra_function_calls().
1443	*/
1444	remove_insn_ops(insn);
1445
1446	return annotate_call_site(file, insn, sibling);
1447	}
1448
1449	static int add_retpoline_call(struct objtool_file file, struct instruction insn)
1450	{
1451	/*
1452	* Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1453	* so convert them accordingly.
1454	*/
1455	switch (insn->type) {
1456	case INSN_CALL:
1457	insn->type = INSN_CALL_DYNAMIC;
1458	break;
1459	case INSN_JUMP_UNCONDITIONAL:
1460	insn->type = INSN_JUMP_DYNAMIC;
1461	break;
1462	case INSN_JUMP_CONDITIONAL:
1463	insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1464	break;
1465	default:
1466	return 0;
1467	}
1468
1469	insn->retpoline_safe = true;
1470
1471	/*
1472	* Whatever stack impact regular CALLs have, should be undone
1473	* by the RETURN of the called function.
1474	*
1475	* Annotated intra-function calls retain the stack_ops but
1476	* are converted to JUMP, see read_intra_function_calls().
1477	*/
1478	remove_insn_ops(insn);
1479
1480	return annotate_call_site(file, insn, false);
1481	}
1482
1483	static void add_return_call(struct objtool_file file, struct instruction insn, bool add)
1484	{
1485	/*
1486	* Return thunk tail calls are really just returns in disguise,
1487	* so convert them accordingly.
1488	*/
1489	insn->type = INSN_RETURN;
1490	insn->retpoline_safe = true;
1491
1492	if (add)
1493	list_add_tail(&insn->call_node, &file->return_thunk_list);
1494	}
1495
1496	static bool is_first_func_insn(struct objtool_file *file,
1497	struct instruction *insn)
1498	{
1499	struct symbol *func = insn_func(insn);
1500
1501	if (!func)
1502	return false;
1503
1504	if (insn->offset == func->offset)
1505	return true;
1506
1507	/* Allow direct CALL/JMP past ENDBR */
1508	if (opts.ibt) {
1509	struct instruction *prev = prev_insn_same_sym(file, insn);
1510
1511	if (prev && prev->type == INSN_ENDBR &&
1512	insn->offset == func->offset + prev->len)
1513	return true;
1514	}
1515
1516	return false;
1517	}
1518
1519	/*
1520	* Find the destination instructions for all jumps.
1521	*/
1522	static int add_jump_destinations(struct objtool_file *file)
1523	{
1524	struct instruction *insn;
1525	struct reloc *reloc;
1526
1527	for_each_insn(file, insn) {
1528	struct symbol *func = insn_func(insn);
1529	struct instruction *dest_insn;
1530	struct section *dest_sec;
1531	struct symbol *dest_sym;
1532	unsigned long dest_off;
1533
1534	if (!is_static_jump(insn))
1535	continue;
1536
1537	if (insn->jump_dest) {
1538	/*
1539	* handle_group_alt() may have previously set
1540	* 'jump_dest' for some alternatives.
1541	*/
1542	continue;
1543	}
1544
1545	reloc = insn_reloc(file, insn);
1546	if (!reloc) {
1547	dest_sec = insn->sec;
1548	dest_off = arch_jump_destination(insn);
1549	dest_sym = dest_sec->sym;
1550	} else {
1551	dest_sym = reloc->sym;
1552	if (is_undef_sym(dest_sym)) {
1553	if (dest_sym->retpoline_thunk) {
1554	if (add_retpoline_call(file, insn))
1555	return -1;
1556	continue;
1557	}
1558
1559	if (dest_sym->return_thunk) {
1560	add_return_call(file, insn, true);
1561	continue;
1562	}
1563
1564	/* External symbol */
1565	if (func) {
1566	/* External sibling call */
1567	if (add_call_dest(file, insn, dest_sym, true))
1568	return -1;
1569	continue;
1570	}
1571
1572	/* Non-func asm code jumping to external symbol */
1573	continue;
1574	}
1575
1576	dest_sec = dest_sym->sec;
1577	dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc);
1578	}
1579
1580	dest_insn = find_insn(file, dest_sec, dest_off);
1581	if (!dest_insn) {
1582	struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1583
1584	/*
1585	* retbleed_untrain_ret() jumps to
1586	* __x86_return_thunk(), but objtool can't find
1587	* the thunk's starting RET instruction,
1588	* because the RET is also in the middle of
1589	* another instruction. Objtool only knows
1590	* about the outer instruction.
1591	*/
1592	if (sym && sym->embedded_insn) {
1593	add_return_call(file, insn, false);
1594	continue;
1595	}
1596
1597	/*
1598	* GCOV/KCOV dead code can jump to the end of
1599	* the function/section.
1600	*/
1601	if (file->ignore_unreachables && func &&
1602	dest_sec == insn->sec &&
1603	dest_off == func->offset + func->len)
1604	continue;
1605
1606	ERROR_INSN(insn, "can't find jump dest instruction at %s",
1607	offstr(dest_sec, dest_off));
1608	return -1;
1609	}
1610
1611	if (!dest_sym \|\| is_sec_sym(dest_sym)) {
1612	dest_sym = insn_sym(dest_insn);
1613	if (!dest_sym)
1614	goto set_jump_dest;
1615	}
1616
1617	if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) {
1618	if (add_retpoline_call(file, insn))
1619	return -1;
1620	continue;
1621	}
1622
1623	if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) {
1624	add_return_call(file, insn, true);
1625	continue;
1626	}
1627
1628	if (!insn_sym(insn) \|\| insn_sym(insn)->pfunc == dest_sym->pfunc)
1629	goto set_jump_dest;
1630
1631	/*
1632	* Internal cross-function jump.
1633	*/
1634
1635	if (is_first_func_insn(file, dest_insn)) {
1636	/* Internal sibling call */
1637	if (add_call_dest(file, insn, dest_sym, true))
1638	return -1;
1639	continue;
1640	}
1641
1642	set_jump_dest:
1643	insn->jump_dest = dest_insn;
1644	}
1645
1646	return 0;
1647	}
1648
1649	static struct symbol find_call_destination(struct section sec, unsigned long offset)
1650	{
1651	struct symbol *call_dest;
1652
1653	call_dest = find_func_by_offset(sec, offset);
1654	if (!call_dest)
1655	call_dest = find_symbol_by_offset(sec, offset);
1656
1657	return call_dest;
1658	}
1659
1660	/*
1661	* Find the destination instructions for all calls.
1662	*/
1663	static int add_call_destinations(struct objtool_file *file)
1664	{
1665	struct instruction *insn;
1666	unsigned long dest_off;
1667	struct symbol *dest;
1668	struct reloc *reloc;
1669
1670	for_each_insn(file, insn) {
1671	struct symbol *func = insn_func(insn);
1672	if (insn->type != INSN_CALL)
1673	continue;
1674
1675	reloc = insn_reloc(file, insn);
1676	if (!reloc) {
1677	dest_off = arch_jump_destination(insn);
1678	dest = find_call_destination(insn->sec, dest_off);
1679
1680	if (add_call_dest(file, insn, dest, false))
1681	return -1;
1682
1683	if (func && func->ignore)
1684	continue;
1685
1686	if (!insn_call_dest(insn)) {
1687	ERROR_INSN(insn, "unannotated intra-function call");
1688	return -1;
1689	}
1690
1691	if (func && !is_func_sym(insn_call_dest(insn))) {
1692	ERROR_INSN(insn, "unsupported call to non-function");
1693	return -1;
1694	}
1695
1696	} else if (is_sec_sym(reloc->sym)) {
1697	dest_off = arch_insn_adjusted_addend(insn, reloc);
1698	dest = find_call_destination(reloc->sym->sec, dest_off);
1699	if (!dest) {
1700	ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1701	reloc->sym->sec->name, dest_off);
1702	return -1;
1703	}
1704
1705	if (add_call_dest(file, insn, dest, false))
1706	return -1;
1707
1708	} else if (reloc->sym->retpoline_thunk) {
1709	if (add_retpoline_call(file, insn))
1710	return -1;
1711
1712	} else {
1713	if (add_call_dest(file, insn, reloc->sym, false))
1714	return -1;
1715	}
1716	}
1717
1718	return 0;
1719	}
1720
1721	/*
1722	* The .alternatives section requires some extra special care over and above
1723	* other special sections because alternatives are patched in place.
1724	*/
1725	static int handle_group_alt(struct objtool_file *file,
1726	struct special_alt *special_alt,
1727	struct instruction *orig_insn,
1728	struct instruction **new_insn)
1729	{
1730	struct instruction last_new_insn = NULL, insn, *nop = NULL;
1731	struct alt_group orig_alt_group, new_alt_group;
1732	unsigned long dest_off;
1733
1734	orig_alt_group = orig_insn->alt_group;
1735	if (!orig_alt_group) {
1736	struct instruction *last_orig_insn = NULL;
1737
1738	orig_alt_group = calloc(1, sizeof(*orig_alt_group));
1739	if (!orig_alt_group) {
1740	ERROR_GLIBC("calloc");
1741	return -1;
1742	}
1743	orig_alt_group->cfi = calloc(special_alt->orig_len,
1744	sizeof(struct cfi_state *));
1745	if (!orig_alt_group->cfi) {
1746	ERROR_GLIBC("calloc");
1747	return -1;
1748	}
1749
1750	insn = orig_insn;
1751	sec_for_each_insn_from(file, insn) {
1752	if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1753	break;
1754
1755	insn->alt_group = orig_alt_group;
1756	last_orig_insn = insn;
1757	}
1758	orig_alt_group->orig_group = NULL;
1759	orig_alt_group->first_insn = orig_insn;
1760	orig_alt_group->last_insn = last_orig_insn;
1761	orig_alt_group->nop = NULL;
1762	orig_alt_group->ignore = orig_insn->ignore_alts;
1763	orig_alt_group->feature = 0;
1764	} else {
1765	if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1766	orig_alt_group->first_insn->offset != special_alt->orig_len) {
1767	ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1768	orig_alt_group->last_insn->offset +
1769	orig_alt_group->last_insn->len -
1770	orig_alt_group->first_insn->offset,
1771	special_alt->orig_len);
1772	return -1;
1773	}
1774	}
1775
1776	new_alt_group = calloc(1, sizeof(*new_alt_group));
1777	if (!new_alt_group) {
1778	ERROR_GLIBC("calloc");
1779	return -1;
1780	}
1781
1782	if (special_alt->new_len < special_alt->orig_len) {
1783	/*
1784	* Insert a fake nop at the end to make the replacement
1785	* alt_group the same size as the original. This is needed to
1786	* allow propagate_alt_cfi() to do its magic. When the last
1787	* instruction affects the stack, the instruction after it (the
1788	* nop) will propagate the new state to the shared CFI array.
1789	*/
1790	nop = calloc(1, sizeof(*nop));
1791	if (!nop) {
1792	ERROR_GLIBC("calloc");
1793	return -1;
1794	}
1795	memset(nop, 0, sizeof(*nop));
1796
1797	nop->sec = special_alt->new_sec;
1798	nop->offset = special_alt->new_off + special_alt->new_len;
1799	nop->len = special_alt->orig_len - special_alt->new_len;
1800	nop->type = INSN_NOP;
1801	nop->alt_group = new_alt_group;
1802	nop->fake = 1;
1803	}
1804
1805	if (!special_alt->new_len) {
1806	*new_insn = nop;
1807	goto end;
1808	}
1809
1810	insn = *new_insn;
1811	sec_for_each_insn_from(file, insn) {
1812	struct reloc *alt_reloc;
1813
1814	if (insn->offset >= special_alt->new_off + special_alt->new_len)
1815	break;
1816
1817	last_new_insn = insn;
1818
1819	insn->alt_group = new_alt_group;
1820
1821	/*
1822	* Since alternative replacement code is copy/pasted by the
1823	* kernel after applying relocations, generally such code can't
1824	* have relative-address relocation references to outside the
1825	* .altinstr_replacement section, unless the arch's
1826	* alternatives code can adjust the relative offsets
1827	* accordingly.
1828	*/
1829	alt_reloc = insn_reloc(file, insn);
1830	if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1831	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1832
1833	ERROR_INSN(insn, "unsupported relocation in alternatives section");
1834	return -1;
1835	}
1836
1837	if (!is_static_jump(insn))
1838	continue;
1839
1840	if (!insn->immediate)
1841	continue;
1842
1843	dest_off = arch_jump_destination(insn);
1844	if (dest_off == special_alt->new_off + special_alt->new_len) {
1845	insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1846	if (!insn->jump_dest) {
1847	ERROR_INSN(insn, "can't find alternative jump destination");
1848	return -1;
1849	}
1850	}
1851	}
1852
1853	if (!last_new_insn) {
1854	ERROR_FUNC(special_alt->new_sec, special_alt->new_off,
1855	"can't find last new alternative instruction");
1856	return -1;
1857	}
1858
1859	end:
1860	new_alt_group->orig_group = orig_alt_group;
1861	new_alt_group->first_insn = *new_insn;
1862	new_alt_group->last_insn = last_new_insn;
1863	new_alt_group->nop = nop;
1864	new_alt_group->ignore = (*new_insn)->ignore_alts;
1865	new_alt_group->cfi = orig_alt_group->cfi;
1866	new_alt_group->feature = special_alt->feature;
1867	return 0;
1868	}
1869
1870	/*
1871	* A jump table entry can either convert a nop to a jump or a jump to a nop.
1872	* If the original instruction is a jump, make the alt entry an effective nop
1873	* by just skipping the original instruction.
1874	*/
1875	static int handle_jump_alt(struct objtool_file *file,
1876	struct special_alt *special_alt,
1877	struct instruction *orig_insn,
1878	struct instruction **new_insn)
1879	{
1880	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1881	orig_insn->type != INSN_NOP) {
1882
1883	ERROR_INSN(orig_insn, "unsupported instruction at jump label");
1884	return -1;
1885	}
1886
1887	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1888	struct reloc *reloc = insn_reloc(file, orig_insn);
1889
1890	if (reloc)
1891	set_reloc_type(file->elf, reloc, R_NONE);
1892
1893	if (elf_write_insn(file->elf, orig_insn->sec,
1894	orig_insn->offset, orig_insn->len,
1895	arch_nop_insn(orig_insn->len))) {
1896	return -1;
1897	}
1898
1899	orig_insn->type = INSN_NOP;
1900	}
1901
1902	if (orig_insn->type == INSN_NOP) {
1903	if (orig_insn->len == 2)
1904	file->jl_nop_short++;
1905	else
1906	file->jl_nop_long++;
1907
1908	return 0;
1909	}
1910
1911	if (orig_insn->len == 2)
1912	file->jl_short++;
1913	else
1914	file->jl_long++;
1915
1916	*new_insn = next_insn_same_sec(file, orig_insn);
1917	return 0;
1918	}
1919
1920	/*
1921	* Read all the special sections which have alternate instructions which can be
1922	* patched in or redirected to at runtime. Each instruction having alternate
1923	* instruction(s) has them added to its insn->alts list, which will be
1924	* traversed in validate_branch().
1925	*/
1926	static int add_special_section_alts(struct objtool_file *file)
1927	{
1928	struct list_head special_alts;
1929	struct instruction orig_insn, new_insn;
1930	struct special_alt special_alt, tmp;
1931	enum alternative_type alt_type;
1932	struct alternative *alt;
1933	struct alternative *a;
1934
1935	if (special_get_alts(file->elf, &special_alts))
1936	return -1;
1937
1938	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1939
1940	orig_insn = find_insn(file, special_alt->orig_sec,
1941	special_alt->orig_off);
1942	if (!orig_insn) {
1943	ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off,
1944	"special: can't find orig instruction");
1945	return -1;
1946	}
1947
1948	new_insn = NULL;
1949	if (!special_alt->group \|\| special_alt->new_len) {
1950	new_insn = find_insn(file, special_alt->new_sec,
1951	special_alt->new_off);
1952	if (!new_insn) {
1953	ERROR_FUNC(special_alt->new_sec, special_alt->new_off,
1954	"special: can't find new instruction");
1955	return -1;
1956	}
1957	}
1958
1959	if (special_alt->group) {
1960	if (!special_alt->orig_len) {
1961	ERROR_INSN(orig_insn, "empty alternative entry");
1962	continue;
1963	}
1964
1965	if (handle_group_alt(file, special_alt, orig_insn, &new_insn))
1966	return -1;
1967
1968	alt_type = ALT_TYPE_INSTRUCTIONS;
1969
1970	} else if (special_alt->jump_or_nop) {
1971	if (handle_jump_alt(file, special_alt, orig_insn, &new_insn))
1972	return -1;
1973
1974	alt_type = ALT_TYPE_JUMP_TABLE;
1975	} else {
1976	alt_type = ALT_TYPE_EX_TABLE;
1977	}
1978
1979	alt = calloc(1, sizeof(*alt));
1980	if (!alt) {
1981	ERROR_GLIBC("calloc");
1982	return -1;
1983	}
1984
1985	alt->insn = new_insn;
1986	alt->type = alt_type;
1987	alt->next = NULL;
1988
1989	/*
1990	* Store alternatives in the same order they have been
1991	* defined.
1992	*/
1993	if (!orig_insn->alts) {
1994	orig_insn->alts = alt;
1995	} else {
1996	for (a = orig_insn->alts; a->next; a = a->next)
1997	;
1998	a->next = alt;
1999	}
2000
2001	list_del(&special_alt->list);
2002	free(special_alt);
2003	}
2004
2005	if (opts.stats) {
2006	printf("jl\\\tNOP\tJMP\n");
2007	printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2008	printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2009	}
2010
2011	return 0;
2012	}
2013
2014	__weak unsigned long arch_jump_table_sym_offset(struct reloc reloc, struct reloc table)
2015	{
2016	return reloc->sym->offset + reloc_addend(reloc);
2017	}
2018
2019	static int add_jump_table(struct objtool_file file, struct instruction insn)
2020	{
2021	unsigned long table_size = insn_jump_table_size(insn);
2022	struct symbol *pfunc = insn_func(insn)->pfunc;
2023	struct reloc *table = insn_jump_table(insn);
2024	struct instruction *dest_insn;
2025	unsigned int prev_offset = 0;
2026	struct reloc *reloc = table;
2027	struct alternative *alt;
2028	unsigned long sym_offset;
2029
2030	/*
2031	* Each @reloc is a switch table relocation which points to the target
2032	* instruction.
2033	*/
2034	for_each_reloc_from(table->sec, reloc) {
2035
2036	/* Check for the end of the table: */
2037	if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size)
2038	break;
2039	if (reloc != table && is_jump_table(reloc))
2040	break;
2041
2042	/* Make sure the table entries are consecutive: */
2043	if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc))
2044	break;
2045
2046	sym_offset = arch_jump_table_sym_offset(reloc, table);
2047
2048	/* Detect function pointers from contiguous objects: */
2049	if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset)
2050	break;
2051
2052	/*
2053	* Clang sometimes leaves dangling unused jump table entries
2054	* which point to the end of the function. Ignore them.
2055	*/
2056	if (reloc->sym->sec == pfunc->sec &&
2057	sym_offset == pfunc->offset + pfunc->len)
2058	goto next;
2059
2060	dest_insn = find_insn(file, reloc->sym->sec, sym_offset);
2061	if (!dest_insn)
2062	break;
2063
2064	/* Make sure the destination is in the same function: */
2065	if (!insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != pfunc)
2066	break;
2067
2068	alt = calloc(1, sizeof(*alt));
2069	if (!alt) {
2070	ERROR_GLIBC("calloc");
2071	return -1;
2072	}
2073
2074	alt->insn = dest_insn;
2075	alt->next = insn->alts;
2076	insn->alts = alt;
2077	next:
2078	prev_offset = reloc_offset(reloc);
2079	}
2080
2081	if (!prev_offset) {
2082	ERROR_INSN(insn, "can't find switch jump table");
2083	return -1;
2084	}
2085
2086	return 0;
2087	}
2088
2089	/*
2090	* find_jump_table() - Given a dynamic jump, find the switch jump table
2091	* associated with it.
2092	*/
2093	static void find_jump_table(struct objtool_file file, struct symbol func,
2094	struct instruction *insn)
2095	{
2096	struct reloc *table_reloc;
2097	struct instruction dest_insn, orig_insn = insn;
2098	unsigned long table_size;
2099	unsigned long sym_offset;
2100
2101	/*
2102	* Backward search using the @first_jump_src links, these help avoid
2103	* much of the 'in between' code. Which avoids us getting confused by
2104	* it.
2105	*/
2106	for (;
2107	insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2108	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2109
2110	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2111	break;
2112
2113	/* allow small jumps within the range */
2114	if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2115	insn->jump_dest &&
2116	(insn->jump_dest->offset <= insn->offset \|\|
2117	insn->jump_dest->offset > orig_insn->offset))
2118	break;
2119
2120	table_reloc = arch_find_switch_table(file, insn, &table_size);
2121	if (!table_reloc)
2122	continue;
2123
2124	sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc);
2125
2126	dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset);
2127	if (!dest_insn \|\| !insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != func)
2128	continue;
2129
2130	set_jump_table(table_reloc);
2131	orig_insn->_jump_table = table_reloc;
2132	orig_insn->_jump_table_size = table_size;
2133
2134	break;
2135	}
2136	}
2137
2138	/*
2139	* First pass: Mark the head of each jump table so that in the next pass,
2140	* we know when a given jump table ends and the next one starts.
2141	*/
2142	static void mark_func_jump_tables(struct objtool_file *file,
2143	struct symbol *func)
2144	{
2145	struct instruction insn, last = NULL;
2146
2147	func_for_each_insn(file, func, insn) {
2148	if (!last)
2149	last = insn;
2150
2151	/*
2152	* Store back-pointers for forward jumps such
2153	* that find_jump_table() can back-track using those and
2154	* avoid some potentially confusing code.
2155	*/
2156	if (insn->jump_dest &&
2157	insn->jump_dest->offset > insn->offset &&
2158	!insn->jump_dest->first_jump_src) {
2159
2160	insn->jump_dest->first_jump_src = insn;
2161	last = insn->jump_dest;
2162	}
2163
2164	if (insn->type != INSN_JUMP_DYNAMIC)
2165	continue;
2166
2167	find_jump_table(file, func, insn);
2168	}
2169	}
2170
2171	static int add_func_jump_tables(struct objtool_file *file,
2172	struct symbol *func)
2173	{
2174	struct instruction *insn;
2175
2176	func_for_each_insn(file, func, insn) {
2177	if (!insn_jump_table(insn))
2178	continue;
2179
2180	if (add_jump_table(file, insn))
2181	return -1;
2182	}
2183
2184	return 0;
2185	}
2186
2187	/*
2188	* For some switch statements, gcc generates a jump table in the .rodata
2189	* section which contains a list of addresses within the function to jump to.
2190	* This finds these jump tables and adds them to the insn->alts lists.
2191	*/
2192	static int add_jump_table_alts(struct objtool_file *file)
2193	{
2194	struct symbol *func;
2195
2196	if (!file->rodata)
2197	return 0;
2198
2199	for_each_sym(file->elf, func) {
2200	if (!is_func_sym(func) \|\| is_alias_sym(func))
2201	continue;
2202
2203	mark_func_jump_tables(file, func);
2204	if (add_func_jump_tables(file, func))
2205	return -1;
2206	}
2207
2208	return 0;
2209	}
2210
2211	static void set_func_state(struct cfi_state *state)
2212	{
2213	state->cfa = initial_func_cfi.cfa;
2214	memcpy(&state->regs, &initial_func_cfi.regs,
2215	CFI_NUM_REGS * sizeof(struct cfi_reg));
2216	state->stack_size = initial_func_cfi.cfa.offset;
2217	state->type = UNWIND_HINT_TYPE_CALL;
2218	}
2219
2220	static int read_unwind_hints(struct objtool_file *file)
2221	{
2222	struct cfi_state cfi = init_cfi;
2223	struct section *sec;
2224	struct unwind_hint *hint;
2225	struct instruction *insn;
2226	struct reloc *reloc;
2227	unsigned long offset;
2228	int i;
2229
2230	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2231	if (!sec)
2232	return 0;
2233
2234	if (!sec->rsec) {
2235	ERROR("missing .rela.discard.unwind_hints section");
2236	return -1;
2237	}
2238
2239	if (sec_size(sec) % sizeof(struct unwind_hint)) {
2240	ERROR("struct unwind_hint size mismatch");
2241	return -1;
2242	}
2243
2244	file->hints = true;
2245
2246	for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) {
2247	hint = (struct unwind_hint *)sec->data->d_buf + i;
2248
2249	reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2250	if (!reloc) {
2251	ERROR("can't find reloc for unwind_hints[%d]", i);
2252	return -1;
2253	}
2254
2255	offset = reloc->sym->offset + reloc_addend(reloc);
2256
2257	insn = find_insn(file, reloc->sym->sec, offset);
2258	if (!insn) {
2259	ERROR("can't find insn for unwind_hints[%d]", i);
2260	return -1;
2261	}
2262
2263	insn->hint = true;
2264
2265	if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2266	insn->cfi = &force_undefined_cfi;
2267	continue;
2268	}
2269
2270	if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2271	insn->hint = false;
2272	insn->save = true;
2273	continue;
2274	}
2275
2276	if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2277	insn->restore = true;
2278	continue;
2279	}
2280
2281	if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2282	struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2283
2284	if (sym && is_global_sym(sym)) {
2285	if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2286	ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2287	return -1;
2288	}
2289	}
2290	}
2291
2292	if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2293	insn->cfi = &func_cfi;
2294	continue;
2295	}
2296
2297	if (insn->cfi)
2298	cfi = *(insn->cfi);
2299
2300	if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2301	ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2302	return -1;
2303	}
2304
2305	cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2306	cfi.type = hint->type;
2307	cfi.signal = hint->signal;
2308
2309	insn->cfi = cfi_hash_find_or_add(&cfi);
2310	}
2311
2312	return 0;
2313	}
2314
2315	static int read_annotate(struct objtool_file *file,
2316	int (func)(struct objtool_file file, int type, struct instruction *insn))
2317	{
2318	struct section *sec;
2319	struct instruction *insn;
2320	struct reloc *reloc;
2321	uint64_t offset;
2322	int type;
2323
2324	sec = find_section_by_name(file->elf, ".discard.annotate_insn");
2325	if (!sec)
2326	return 0;
2327
2328	if (!sec->rsec)
2329	return 0;
2330
2331	if (sec->sh.sh_entsize != 8) {
2332	static bool warned = false;
2333	if (!warned && opts.verbose) {
2334	WARN("%s: dodgy linker, sh_entsize != 8", sec->name);
2335	warned = true;
2336	}
2337	sec->sh.sh_entsize = 8;
2338	}
2339
2340	if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) {
2341	ERROR("bad .discard.annotate_insn section: missing relocs");
2342	return -1;
2343	}
2344
2345	for_each_reloc(sec->rsec, reloc) {
2346	type = annotype(file->elf, sec, reloc);
2347	offset = reloc->sym->offset + reloc_addend(reloc);
2348	insn = find_insn(file, reloc->sym->sec, offset);
2349
2350	if (!insn) {
2351	ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type);
2352	return -1;
2353	}
2354
2355	if (func(file, type, insn))
2356	return -1;
2357	}
2358
2359	return 0;
2360	}
2361
2362	static int __annotate_early(struct objtool_file file, int type, struct instruction insn)
2363	{
2364	switch (type) {
2365
2366	/* Must be before add_special_section_alts() */
2367	case ANNOTYPE_IGNORE_ALTS:
2368	insn->ignore_alts = true;
2369	break;
2370
2371	/*
2372	* Must be before read_unwind_hints() since that needs insn->noendbr.
2373	*/
2374	case ANNOTYPE_NOENDBR:
2375	insn->noendbr = 1;
2376	break;
2377
2378	default:
2379	break;
2380	}
2381
2382	return 0;
2383	}
2384
2385	static int __annotate_ifc(struct objtool_file file, int type, struct instruction insn)
2386	{
2387	unsigned long dest_off;
2388
2389	if (type != ANNOTYPE_INTRA_FUNCTION_CALL)
2390	return 0;
2391
2392	if (insn->type != INSN_CALL) {
2393	ERROR_INSN(insn, "intra_function_call not a direct call");
2394	return -1;
2395	}
2396
2397	/*
2398	* Treat intra-function CALLs as JMPs, but with a stack_op.
2399	* See add_call_destinations(), which strips stack_ops from
2400	* normal CALLs.
2401	*/
2402	insn->type = INSN_JUMP_UNCONDITIONAL;
2403
2404	dest_off = arch_jump_destination(insn);
2405	insn->jump_dest = find_insn(file, insn->sec, dest_off);
2406	if (!insn->jump_dest) {
2407	ERROR_INSN(insn, "can't find call dest at %s+0x%lx",
2408	insn->sec->name, dest_off);
2409	return -1;
2410	}
2411
2412	return 0;
2413	}
2414
2415	static int __annotate_late(struct objtool_file file, int type, struct instruction insn)
2416	{
2417	struct symbol *sym;
2418
2419	switch (type) {
2420	case ANNOTYPE_NOENDBR:
2421	/* early */
2422	break;
2423
2424	case ANNOTYPE_RETPOLINE_SAFE:
2425	if (insn->type != INSN_JUMP_DYNAMIC &&
2426	insn->type != INSN_CALL_DYNAMIC &&
2427	insn->type != INSN_RETURN &&
2428	insn->type != INSN_NOP) {
2429	ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2430	return -1;
2431	}
2432
2433	insn->retpoline_safe = true;
2434	break;
2435
2436	case ANNOTYPE_INSTR_BEGIN:
2437	insn->instr++;
2438	break;
2439
2440	case ANNOTYPE_INSTR_END:
2441	insn->instr--;
2442	break;
2443
2444	case ANNOTYPE_UNRET_BEGIN:
2445	insn->unret = 1;
2446	break;
2447
2448	case ANNOTYPE_IGNORE_ALTS:
2449	/* early */
2450	break;
2451
2452	case ANNOTYPE_INTRA_FUNCTION_CALL:
2453	/* ifc */
2454	break;
2455
2456	case ANNOTYPE_REACHABLE:
2457	insn->dead_end = false;
2458	break;
2459
2460	case ANNOTYPE_NOCFI:
2461	sym = insn_sym(insn);
2462	if (!sym) {
2463	ERROR_INSN(insn, "dodgy NOCFI annotation");
2464	return -1;
2465	}
2466	sym->nocfi = 1;
2467	break;
2468
2469	default:
2470	ERROR_INSN(insn, "Unknown annotation type: %d", type);
2471	return -1;
2472	}
2473
2474	return 0;
2475	}
2476
2477	/*
2478	* Return true if name matches an instrumentation function, where calls to that
2479	* function from noinstr code can safely be removed, but compilers won't do so.
2480	*/
2481	static bool is_profiling_func(const char *name)
2482	{
2483	/*
2484	* Many compilers cannot disable KCOV with a function attribute.
2485	*/
2486	if (!strncmp(name, "__sanitizer_cov_", 16))
2487	return true;
2488
2489	return false;
2490	}
2491
2492	static int classify_symbols(struct objtool_file *file)
2493	{
2494	struct symbol *func;
2495	size_t len;
2496
2497	for_each_sym(file->elf, func) {
2498	if (is_notype_sym(func) && strstarts(func->name, ".L"))
2499	func->local_label = true;
2500
2501	if (!is_global_sym(func))
2502	continue;
2503
2504	if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2505	strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2506	func->static_call_tramp = true;
2507
2508	if (arch_is_retpoline(func))
2509	func->retpoline_thunk = true;
2510
2511	if (arch_is_rethunk(func))
2512	func->return_thunk = true;
2513
2514	if (arch_is_embedded_insn(func))
2515	func->embedded_insn = true;
2516
2517	if (arch_ftrace_match(func->name))
2518	func->fentry = true;
2519
2520	if (is_profiling_func(func->name))
2521	func->profiling_func = true;
2522
2523	len = strlen(func->name);
2524	if (len > sym_name_max_len)
2525	sym_name_max_len = len;
2526	}
2527
2528	return 0;
2529	}
2530
2531	static void mark_rodata(struct objtool_file *file)
2532	{
2533	struct section *sec;
2534
2535	/*
2536	* Search for the following rodata sections, each of which can
2537	* potentially contain jump tables:
2538	*
2539	* - .rodata: can contain GCC switch tables
2540	* - .rodata.<func>: same, if -fdata-sections is being used
2541	* - .data.rel.ro.c_jump_table: contains C annotated jump tables
2542	*
2543	* .rodata.str1.* sections are ignored; they don't contain jump tables.
2544	*/
2545	for_each_sec(file->elf, sec) {
2546	if (is_rodata_sec(sec)) {
2547	file->rodata = true;
2548	return;
2549	}
2550	}
2551	}
2552
2553	static void mark_holes(struct objtool_file *file)
2554	{
2555	struct instruction *insn;
2556	bool in_hole = false;
2557
2558	if (!opts.link)
2559	return;
2560
2561	/*
2562	* Whole archive runs might encounter dead code from weak symbols.
2563	* This is where the linker will have dropped the weak symbol in
2564	* favour of a regular symbol, but leaves the code in place.
2565	*/
2566	for_each_insn(file, insn) {
2567	if (insn_sym(insn) \|\| !find_symbol_hole_containing(insn->sec, insn->offset)) {
2568	in_hole = false;
2569	continue;
2570	}
2571
2572	/* Skip function padding and pfx code */
2573	if (!in_hole && insn->type == INSN_NOP)
2574	continue;
2575
2576	in_hole = true;
2577	insn->hole = 1;
2578
2579	/*
2580	* If this hole jumps to a .cold function, mark it ignore.
2581	*/
2582	if (insn->jump_dest) {
2583	struct symbol *dest_func = insn_func(insn->jump_dest);
2584
2585	if (dest_func && is_cold_func(dest_func))
2586	dest_func->ignore = true;
2587	}
2588	}
2589	}
2590
2591	static bool validate_branch_enabled(void)
2592	{
2593	return opts.stackval \|\|
2594	opts.orc \|\|
2595	opts.uaccess;
2596	}
2597
2598	static bool alts_needed(void)
2599	{
2600	return validate_branch_enabled() \|\|
2601	opts.noinstr \|\|
2602	opts.hack_jump_label \|\|
2603	opts.disas \|\|
2604	opts.checksum;
2605	}
2606
2607	int decode_file(struct objtool_file *file)
2608	{
2609	arch_initial_func_cfi_state(&initial_func_cfi);
2610	init_cfi_state(&init_cfi);
2611	init_cfi_state(&func_cfi);
2612	set_func_state(&func_cfi);
2613	init_cfi_state(&force_undefined_cfi);
2614	force_undefined_cfi.force_undefined = true;
2615
2616	if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
2617	return -1;
2618
2619	cfi_hash_add(&init_cfi);
2620	cfi_hash_add(&func_cfi);
2621
2622	file->klp = is_livepatch_module(file);
2623
2624	mark_rodata(file);
2625
2626	if (init_pv_ops(file))
2627	return -1;
2628
2629	/*
2630	* Must be before add_{jump_call}_destination.
2631	*/
2632	if (classify_symbols(file))
2633	return -1;
2634
2635	if (decode_instructions(file))
2636	return -1;
2637
2638	if (add_ignores(file))
2639	return -1;
2640
2641	add_uaccess_safe(file);
2642
2643	if (read_annotate(file, __annotate_early))
2644	return -1;
2645
2646	/*
2647	* Must be before add_jump_destinations(), which depends on 'func'
2648	* being set for alternatives, to enable proper sibling call detection.
2649	*/
2650	if (alts_needed()) {
2651	if (add_special_section_alts(file))
2652	return -1;
2653	}
2654
2655	if (add_jump_destinations(file))
2656	return -1;
2657
2658	/*
2659	* Must be before add_call_destination(); it changes INSN_CALL to
2660	* INSN_JUMP.
2661	*/
2662	if (read_annotate(file, __annotate_ifc))
2663	return -1;
2664
2665	if (add_call_destinations(file))
2666	return -1;
2667
2668	if (add_jump_table_alts(file))
2669	return -1;
2670
2671	if (read_unwind_hints(file))
2672	return -1;
2673
2674	/* Must be after add_jump_destinations() */
2675	mark_holes(file);
2676
2677	/*
2678	* Must be after add_call_destinations() such that it can override
2679	* dead_end_function() marks.
2680	*/
2681	if (read_annotate(file, __annotate_late))
2682	return -1;
2683
2684	return 0;
2685	}
2686
2687	static bool is_special_call(struct instruction *insn)
2688	{
2689	if (insn->type == INSN_CALL) {
2690	struct symbol *dest = insn_call_dest(insn);
2691
2692	if (!dest)
2693	return false;
2694
2695	if (dest->fentry \|\| dest->embedded_insn)
2696	return true;
2697	}
2698
2699	return false;
2700	}
2701
2702	static bool has_modified_stack_frame(struct instruction insn, struct insn_state state)
2703	{
2704	struct cfi_state *cfi = &state->cfi;
2705	int i;
2706
2707	if (cfi->cfa.base != initial_func_cfi.cfa.base \|\| cfi->drap)
2708	return true;
2709
2710	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2711	return true;
2712
2713	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2714	return true;
2715
2716	for (i = 0; i < CFI_NUM_REGS; i++) {
2717	if (cfi->regs[i].base != initial_func_cfi.regs[i].base \|\|
2718	cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2719	return true;
2720	}
2721
2722	return false;
2723	}
2724
2725	static bool check_reg_frame_pos(const struct cfi_reg *reg,
2726	int expected_offset)
2727	{
2728	return reg->base == CFI_CFA &&
2729	reg->offset == expected_offset;
2730	}
2731
2732	static bool has_valid_stack_frame(struct insn_state *state)
2733	{
2734	struct cfi_state *cfi = &state->cfi;
2735
2736	if (cfi->cfa.base == CFI_BP &&
2737	check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2738	check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2739	return true;
2740
2741	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2742	return true;
2743
2744	return false;
2745	}
2746
2747	static int update_cfi_state_regs(struct instruction *insn,
2748	struct cfi_state *cfi,
2749	struct stack_op *op)
2750	{
2751	struct cfi_reg *cfa = &cfi->cfa;
2752
2753	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2754	return 0;
2755
2756	/* push */
2757	if (op->dest.type == OP_DEST_PUSH \|\| op->dest.type == OP_DEST_PUSHF)
2758	cfa->offset += 8;
2759
2760	/* pop */
2761	if (op->src.type == OP_SRC_POP \|\| op->src.type == OP_SRC_POPF)
2762	cfa->offset -= 8;
2763
2764	/* add immediate to sp */
2765	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2766	op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2767	cfa->offset -= op->src.offset;
2768
2769	return 0;
2770	}
2771
2772	static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2773	{
2774	if (arch_callee_saved_reg(reg) &&
2775	cfi->regs[reg].base == CFI_UNDEFINED) {
2776	cfi->regs[reg].base = base;
2777	cfi->regs[reg].offset = offset;
2778	}
2779	}
2780
2781	static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2782	{
2783	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2784	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2785	}
2786
2787	/*
2788	* A note about DRAP stack alignment:
2789	*
2790	* GCC has the concept of a DRAP register, which is used to help keep track of
2791	* the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2792	* register. The typical DRAP pattern is:
2793	*
2794	* 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2795	* 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2796	* 41 ff 72 f8 pushq -0x8(%r10)
2797	* 55 push %rbp
2798	* 48 89 e5 mov %rsp,%rbp
2799	* (more pushes)
2800	* 41 52 push %r10
2801	* ...
2802	* 41 5a pop %r10
2803	* (more pops)
2804	* 5d pop %rbp
2805	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2806	* c3 retq
2807	*
2808	* There are some variations in the epilogues, like:
2809	*
2810	* 5b pop %rbx
2811	* 41 5a pop %r10
2812	* 41 5c pop %r12
2813	* 41 5d pop %r13
2814	* 41 5e pop %r14
2815	* c9 leaveq
2816	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2817	* c3 retq
2818	*
2819	* and:
2820	*
2821	* 4c 8b 55 e8 mov -0x18(%rbp),%r10
2822	* 48 8b 5d e0 mov -0x20(%rbp),%rbx
2823	* 4c 8b 65 f0 mov -0x10(%rbp),%r12
2824	* 4c 8b 6d f8 mov -0x8(%rbp),%r13
2825	* c9 leaveq
2826	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2827	* c3 retq
2828	*
2829	* Sometimes r13 is used as the DRAP register, in which case it's saved and
2830	* restored beforehand:
2831	*
2832	* 41 55 push %r13
2833	* 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2834	* 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2835	* ...
2836	* 49 8d 65 f0 lea -0x10(%r13),%rsp
2837	* 41 5d pop %r13
2838	* c3 retq
2839	*/
2840	static int update_cfi_state(struct instruction *insn,
2841	struct instruction *next_insn,
2842	struct cfi_state cfi, struct stack_op op)
2843	{
2844	struct cfi_reg *cfa = &cfi->cfa;
2845	struct cfi_reg *regs = cfi->regs;
2846
2847	/* ignore UNWIND_HINT_UNDEFINED regions */
2848	if (cfi->force_undefined)
2849	return 0;
2850
2851	/* stack operations don't make sense with an undefined CFA */
2852	if (cfa->base == CFI_UNDEFINED) {
2853	if (insn_func(insn)) {
2854	WARN_INSN(insn, "undefined stack state");
2855	return 1;
2856	}
2857	return 0;
2858	}
2859
2860	if (cfi->type == UNWIND_HINT_TYPE_REGS \|\|
2861	cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2862	return update_cfi_state_regs(insn, cfi, op);
2863
2864	switch (op->dest.type) {
2865
2866	case OP_DEST_REG:
2867	switch (op->src.type) {
2868
2869	case OP_SRC_REG:
2870	if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2871	cfa->base == CFI_SP &&
2872	check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2873
2874	/* mov %rsp, %rbp */
2875	cfa->base = op->dest.reg;
2876	cfi->bp_scratch = false;
2877	}
2878
2879	else if (op->src.reg == CFI_SP &&
2880	op->dest.reg == CFI_BP && cfi->drap) {
2881
2882	/* drap: mov %rsp, %rbp */
2883	regs[CFI_BP].base = CFI_BP;
2884	regs[CFI_BP].offset = -cfi->stack_size;
2885	cfi->bp_scratch = false;
2886	}
2887
2888	else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2889
2890	/*
2891	* mov %rsp, %reg
2892	*
2893	* This is needed for the rare case where GCC
2894	* does:
2895	*
2896	* mov %rsp, %rax
2897	* ...
2898	* mov %rax, %rsp
2899	*/
2900	cfi->vals[op->dest.reg].base = CFI_CFA;
2901	cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2902	}
2903
2904	else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2905	(cfa->base == CFI_BP \|\| cfa->base == cfi->drap_reg)) {
2906
2907	/*
2908	* mov %rbp, %rsp
2909	*
2910	* Restore the original stack pointer (Clang).
2911	*/
2912	cfi->stack_size = -cfi->regs[CFI_BP].offset;
2913	}
2914
2915	else if (op->dest.reg == cfa->base) {
2916
2917	/* mov %reg, %rsp */
2918	if (cfa->base == CFI_SP &&
2919	cfi->vals[op->src.reg].base == CFI_CFA) {
2920
2921	/*
2922	* This is needed for the rare case
2923	* where GCC does something dumb like:
2924	*
2925	* lea 0x8(%rsp), %rcx
2926	* ...
2927	* mov %rcx, %rsp
2928	*/
2929	cfa->offset = -cfi->vals[op->src.reg].offset;
2930	cfi->stack_size = cfa->offset;
2931
2932	} else if (cfa->base == CFI_SP &&
2933	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2934	cfi->vals[op->src.reg].offset == cfa->offset) {
2935
2936	/*
2937	* Stack swizzle:
2938	*
2939	* 1: mov %rsp, (%[tos])
2940	* 2: mov %[tos], %rsp
2941	* ...
2942	* 3: pop %rsp
2943	*
2944	* Where:
2945	*
2946	* 1 - places a pointer to the previous
2947	* stack at the Top-of-Stack of the
2948	* new stack.
2949	*
2950	* 2 - switches to the new stack.
2951	*
2952	* 3 - pops the Top-of-Stack to restore
2953	* the original stack.
2954	*
2955	* Note: we set base to SP_INDIRECT
2956	* here and preserve offset. Therefore
2957	* when the unwinder reaches ToS it
2958	* will dereference SP and then add the
2959	* offset to find the next frame, IOW:
2960	* (%rsp) + offset.
2961	*/
2962	cfa->base = CFI_SP_INDIRECT;
2963
2964	} else {
2965	cfa->base = CFI_UNDEFINED;
2966	cfa->offset = 0;
2967	}
2968	}
2969
2970	else if (op->dest.reg == CFI_SP &&
2971	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2972	cfi->vals[op->src.reg].offset == cfa->offset) {
2973
2974	/*
2975	* The same stack swizzle case 2) as above. But
2976	* because we can't change cfa->base, case 3)
2977	* will become a regular POP. Pretend we're a
2978	* PUSH so things don't go unbalanced.
2979	*/
2980	cfi->stack_size += 8;
2981	}
2982
2983	else if (cfi->vals[op->src.reg].base == CFI_CFA) {
2984	/*
2985	* Clang RSP musical chairs:
2986	*
2987	* mov %rsp, %rdx [handled above]
2988	* ...
2989	* mov %rdx, %rbx [handled here]
2990	* ...
2991	* mov %rbx, %rsp [handled above]
2992	*/
2993	cfi->vals[op->dest.reg].base = CFI_CFA;
2994	cfi->vals[op->dest.reg].offset = cfi->vals[op->src.reg].offset;
2995	}
2996
2997
2998	break;
2999
3000	case OP_SRC_ADD:
3001	if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
3002
3003	/* add imm, %rsp */
3004	cfi->stack_size -= op->src.offset;
3005	if (cfa->base == CFI_SP)
3006	cfa->offset -= op->src.offset;
3007	break;
3008	}
3009
3010	if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
3011	insn_sym(insn)->frame_pointer) {
3012	/* addi.d fp,sp,imm on LoongArch */
3013	if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
3014	cfa->base = CFI_BP;
3015	cfa->offset = 0;
3016	}
3017	break;
3018	}
3019
3020	if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3021	/* addi.d sp,fp,imm on LoongArch */
3022	if (cfa->base == CFI_BP && cfa->offset == 0) {
3023	if (insn_sym(insn)->frame_pointer) {
3024	cfa->base = CFI_SP;
3025	cfa->offset = -op->src.offset;
3026	}
3027	} else {
3028	/* lea disp(%rbp), %rsp */
3029	cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3030	}
3031	break;
3032	}
3033
3034	if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3035
3036	/* drap: lea disp(%rsp), %drap */
3037	cfi->drap_reg = op->dest.reg;
3038
3039	/*
3040	* lea disp(%rsp), %reg
3041	*
3042	* This is needed for the rare case where GCC
3043	* does something dumb like:
3044	*
3045	* lea 0x8(%rsp), %rcx
3046	* ...
3047	* mov %rcx, %rsp
3048	*/
3049	cfi->vals[op->dest.reg].base = CFI_CFA;
3050	cfi->vals[op->dest.reg].offset = \
3051	-cfi->stack_size + op->src.offset;
3052
3053	break;
3054	}
3055
3056	if (cfi->drap && op->dest.reg == CFI_SP &&
3057	op->src.reg == cfi->drap_reg) {
3058
3059	/* drap: lea disp(%drap), %rsp */
3060	cfa->base = CFI_SP;
3061	cfa->offset = cfi->stack_size = -op->src.offset;
3062	cfi->drap_reg = CFI_UNDEFINED;
3063	cfi->drap = false;
3064	break;
3065	}
3066
3067	if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3068	WARN_INSN(insn, "unsupported stack register modification");
3069	return -1;
3070	}
3071
3072	break;
3073
3074	case OP_SRC_AND:
3075	if (op->dest.reg != CFI_SP \|\|
3076	(cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) \|\|
3077	(cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3078	WARN_INSN(insn, "unsupported stack pointer realignment");
3079	return -1;
3080	}
3081
3082	if (cfi->drap_reg != CFI_UNDEFINED) {
3083	/* drap: and imm, %rsp */
3084	cfa->base = cfi->drap_reg;
3085	cfa->offset = cfi->stack_size = 0;
3086	cfi->drap = true;
3087	}
3088
3089	/*
3090	* Older versions of GCC (4.8ish) realign the stack
3091	* without DRAP, with a frame pointer.
3092	*/
3093
3094	break;
3095
3096	case OP_SRC_POP:
3097	case OP_SRC_POPF:
3098	if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3099
3100	/* pop %rsp; # restore from a stack swizzle */
3101	cfa->base = CFI_SP;
3102	break;
3103	}
3104
3105	if (!cfi->drap && op->dest.reg == cfa->base) {
3106
3107	/* pop %rbp */
3108	cfa->base = CFI_SP;
3109	}
3110
3111	if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3112	op->dest.reg == cfi->drap_reg &&
3113	cfi->drap_offset == -cfi->stack_size) {
3114
3115	/* drap: pop %drap */
3116	cfa->base = cfi->drap_reg;
3117	cfa->offset = 0;
3118	cfi->drap_offset = -1;
3119
3120	} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3121
3122	/* pop %reg */
3123	restore_reg(cfi, op->dest.reg);
3124	}
3125
3126	cfi->stack_size -= 8;
3127	if (cfa->base == CFI_SP)
3128	cfa->offset -= 8;
3129
3130	break;
3131
3132	case OP_SRC_REG_INDIRECT:
3133	if (!cfi->drap && op->dest.reg == cfa->base &&
3134	op->dest.reg == CFI_BP) {
3135
3136	/* mov disp(%rsp), %rbp */
3137	cfa->base = CFI_SP;
3138	cfa->offset = cfi->stack_size;
3139	}
3140
3141	if (cfi->drap && op->src.reg == CFI_BP &&
3142	op->src.offset == cfi->drap_offset) {
3143
3144	/* drap: mov disp(%rbp), %drap */
3145	cfa->base = cfi->drap_reg;
3146	cfa->offset = 0;
3147	cfi->drap_offset = -1;
3148	}
3149
3150	if (cfi->drap && op->src.reg == CFI_BP &&
3151	op->src.offset == regs[op->dest.reg].offset) {
3152
3153	/* drap: mov disp(%rbp), %reg */
3154	restore_reg(cfi, op->dest.reg);
3155
3156	} else if (op->src.reg == cfa->base &&
3157	op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3158
3159	/* mov disp(%rbp), %reg */
3160	/* mov disp(%rsp), %reg */
3161	restore_reg(cfi, op->dest.reg);
3162
3163	} else if (op->src.reg == CFI_SP &&
3164	op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3165
3166	/* mov disp(%rsp), %reg */
3167	restore_reg(cfi, op->dest.reg);
3168	}
3169
3170	break;
3171
3172	default:
3173	WARN_INSN(insn, "unknown stack-related instruction");
3174	return -1;
3175	}
3176
3177	break;
3178
3179	case OP_DEST_PUSH:
3180	case OP_DEST_PUSHF:
3181	cfi->stack_size += 8;
3182	if (cfa->base == CFI_SP)
3183	cfa->offset += 8;
3184
3185	if (op->src.type != OP_SRC_REG)
3186	break;
3187
3188	if (cfi->drap) {
3189	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3190
3191	/* drap: push %drap */
3192	cfa->base = CFI_BP_INDIRECT;
3193	cfa->offset = -cfi->stack_size;
3194
3195	/* save drap so we know when to restore it */
3196	cfi->drap_offset = -cfi->stack_size;
3197
3198	} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3199
3200	/* drap: push %rbp */
3201	cfi->stack_size = 0;
3202
3203	} else {
3204
3205	/* drap: push %reg */
3206	save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3207	}
3208
3209	} else {
3210
3211	/* push %reg */
3212	save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3213	}
3214
3215	/* detect when asm code uses rbp as a scratch register */
3216	if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3217	cfa->base != CFI_BP)
3218	cfi->bp_scratch = true;
3219	break;
3220
3221	case OP_DEST_REG_INDIRECT:
3222
3223	if (cfi->drap) {
3224	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3225
3226	/* drap: mov %drap, disp(%rbp) */
3227	cfa->base = CFI_BP_INDIRECT;
3228	cfa->offset = op->dest.offset;
3229
3230	/* save drap offset so we know when to restore it */
3231	cfi->drap_offset = op->dest.offset;
3232	} else {
3233
3234	/* drap: mov reg, disp(%rbp) */
3235	save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3236	}
3237
3238	} else if (op->dest.reg == cfa->base) {
3239
3240	/* mov reg, disp(%rbp) */
3241	/* mov reg, disp(%rsp) */
3242	save_reg(cfi, op->src.reg, CFI_CFA,
3243	op->dest.offset - cfi->cfa.offset);
3244
3245	} else if (op->dest.reg == CFI_SP) {
3246
3247	/* mov reg, disp(%rsp) */
3248	save_reg(cfi, op->src.reg, CFI_CFA,
3249	op->dest.offset - cfi->stack_size);
3250
3251	} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3252
3253	/* mov %rsp, (%reg); # setup a stack swizzle. */
3254	cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3255	cfi->vals[op->dest.reg].offset = cfa->offset;
3256	}
3257
3258	break;
3259
3260	case OP_DEST_MEM:
3261	if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3262	WARN_INSN(insn, "unknown stack-related memory operation");
3263	return -1;
3264	}
3265
3266	/* pop mem */
3267	cfi->stack_size -= 8;
3268	if (cfa->base == CFI_SP)
3269	cfa->offset -= 8;
3270
3271	break;
3272
3273	default:
3274	WARN_INSN(insn, "unknown stack-related instruction");
3275	return -1;
3276	}
3277
3278	return 0;
3279	}
3280
3281	/*
3282	* The stack layouts of alternatives instructions can sometimes diverge when
3283	* they have stack modifications. That's fine as long as the potential stack
3284	* layouts don't conflict at any given potential instruction boundary.
3285	*
3286	* Flatten the CFIs of the different alternative code streams (both original
3287	* and replacement) into a single shared CFI array which can be used to detect
3288	* conflicts and nicely feed a linear array of ORC entries to the unwinder.
3289	*/
3290	static int propagate_alt_cfi(struct objtool_file file, struct instruction insn)
3291	{
3292	struct cfi_state **alt_cfi;
3293	int group_off;
3294
3295	if (!insn->alt_group)
3296	return 0;
3297
3298	if (!insn->cfi) {
3299	WARN("CFI missing");
3300	return -1;
3301	}
3302
3303	alt_cfi = insn->alt_group->cfi;
3304	group_off = insn->offset - insn->alt_group->first_insn->offset;
3305
3306	if (!alt_cfi[group_off]) {
3307	alt_cfi[group_off] = insn->cfi;
3308	} else {
3309	if (cficmp(alt_cfi[group_off], insn->cfi)) {
3310	struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3311	struct instruction *orig = orig_group->first_insn;
3312	WARN_INSN(orig, "stack layout conflict in alternatives: %s",
3313	offstr(insn->sec, insn->offset));
3314	return -1;
3315	}
3316	}
3317
3318	return 0;
3319	}
3320
3321	static int noinline handle_insn_ops(struct instruction *insn,
3322	struct instruction *next_insn,
3323	struct insn_state *state)
3324	{
3325	struct insn_state prev_state __maybe_unused = *state;
3326	struct stack_op *op;
3327	int ret = 0;
3328
3329	for (op = insn->stack_ops; op; op = op->next) {
3330
3331	ret = update_cfi_state(insn, next_insn, &state->cfi, op);
3332	if (ret)
3333	goto done;
3334
3335	if (!opts.uaccess \|\| !insn->alt_group)
3336	continue;
3337
3338	if (op->dest.type == OP_DEST_PUSHF) {
3339	if (!state->uaccess_stack) {
3340	state->uaccess_stack = 1;
3341	} else if (state->uaccess_stack >> 31) {
3342	WARN_INSN(insn, "PUSHF stack exhausted");
3343	ret = 1;
3344	goto done;
3345	}
3346	state->uaccess_stack <<= 1;
3347	state->uaccess_stack \|= state->uaccess;
3348	}
3349
3350	if (op->src.type == OP_SRC_POPF) {
3351	if (state->uaccess_stack) {
3352	state->uaccess = state->uaccess_stack & 1;
3353	state->uaccess_stack >>= 1;
3354	if (state->uaccess_stack == 1)
3355	state->uaccess_stack = 0;
3356	}
3357	}
3358	}
3359
3360	done:
3361	TRACE_INSN_STATE(insn, &prev_state, state);
3362
3363	return ret;
3364	}
3365
3366	static bool insn_cfi_match(struct instruction insn, struct cfi_state cfi2)
3367	{
3368	struct cfi_state *cfi1 = insn->cfi;
3369	int i;
3370
3371	if (!cfi1) {
3372	WARN("CFI missing");
3373	return false;
3374	}
3375
3376	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3377
3378	WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3379	cfi1->cfa.base, cfi1->cfa.offset,
3380	cfi2->cfa.base, cfi2->cfa.offset);
3381	return false;
3382
3383	}
3384
3385	if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3386	for (i = 0; i < CFI_NUM_REGS; i++) {
3387
3388	if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg)))
3389	continue;
3390
3391	WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3392	i, cfi1->regs[i].base, cfi1->regs[i].offset,
3393	i, cfi2->regs[i].base, cfi2->regs[i].offset);
3394	}
3395	return false;
3396	}
3397
3398	if (cfi1->type != cfi2->type) {
3399
3400	WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3401	cfi1->type, cfi2->type);
3402	return false;
3403	}
3404
3405	if (cfi1->drap != cfi2->drap \|\|
3406	(cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) \|\|
3407	(cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3408
3409	WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3410	cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3411	cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3412	return false;
3413	}
3414
3415	return true;
3416	}
3417
3418	static inline bool func_uaccess_safe(struct symbol *func)
3419	{
3420	if (func)
3421	return func->uaccess_safe;
3422
3423	return false;
3424	}
3425
3426	static inline const char call_dest_name(struct instruction insn)
3427	{
3428	static char pvname[19];
3429	struct reloc *reloc;
3430	int idx;
3431
3432	if (insn_call_dest(insn))
3433	return insn_call_dest(insn)->name;
3434
3435	reloc = insn_reloc(NULL, insn);
3436	if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3437	idx = (reloc_addend(reloc) / sizeof(void *));
3438	snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3439	return pvname;
3440	}
3441
3442	return "{dynamic}";
3443	}
3444
3445	static bool pv_call_dest(struct objtool_file file, struct instruction insn)
3446	{
3447	struct symbol *target;
3448	struct reloc *reloc;
3449	int idx;
3450
3451	reloc = insn_reloc(file, insn);
3452	if (!reloc \|\| strcmp(reloc->sym->name, "pv_ops"))
3453	return false;
3454
3455	idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *);
3456
3457	if (file->pv_ops[idx].clean)
3458	return true;
3459
3460	file->pv_ops[idx].clean = true;
3461
3462	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3463	if (!target->sec->noinstr) {
3464	WARN("pv_ops[%d]: %s", idx, target->name);
3465	file->pv_ops[idx].clean = false;
3466	}
3467	}
3468
3469	return file->pv_ops[idx].clean;
3470	}
3471
3472	static inline bool noinstr_call_dest(struct objtool_file *file,
3473	struct instruction *insn,
3474	struct symbol *func)
3475	{
3476	/*
3477	* We can't deal with indirect function calls at present;
3478	* assume they're instrumented.
3479	*/
3480	if (!func) {
3481	if (file->pv_ops)
3482	return pv_call_dest(file, insn);
3483
3484	return false;
3485	}
3486
3487	/*
3488	* If the symbol is from a noinstr section; we good.
3489	*/
3490	if (func->sec->noinstr)
3491	return true;
3492
3493	/*
3494	* If the symbol is a static_call trampoline, we can't tell.
3495	*/
3496	if (func->static_call_tramp)
3497	return true;
3498
3499	/*
3500	* The __ubsan_handle_*() calls are like WARN(), they only happen when
3501	* something 'BAD' happened. At the risk of taking the machine down,
3502	* let them proceed to get the message out.
3503	*/
3504	if (!strncmp(func->name, "__ubsan_handle_", 15))
3505	return true;
3506
3507	return false;
3508	}
3509
3510	static int validate_call(struct objtool_file *file,
3511	struct instruction *insn,
3512	struct insn_state *state)
3513	{
3514	if (state->noinstr && state->instr <= 0 &&
3515	!noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3516	WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3517	return 1;
3518	}
3519
3520	if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3521	WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3522	return 1;
3523	}
3524
3525	if (state->df) {
3526	WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3527	return 1;
3528	}
3529
3530	return 0;
3531	}
3532
3533	static int validate_sibling_call(struct objtool_file *file,
3534	struct instruction *insn,
3535	struct insn_state *state)
3536	{
3537	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3538	WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3539	return 1;
3540	}
3541
3542	return validate_call(file, insn, state);
3543	}
3544
3545	static int validate_return(struct symbol func, struct instruction insn, struct insn_state *state)
3546	{
3547	if (state->noinstr && state->instr > 0) {
3548	WARN_INSN(insn, "return with instrumentation enabled");
3549	return 1;
3550	}
3551
3552	if (state->uaccess && !func_uaccess_safe(func)) {
3553	WARN_INSN(insn, "return with UACCESS enabled");
3554	return 1;
3555	}
3556
3557	if (!state->uaccess && func_uaccess_safe(func)) {
3558	WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3559	return 1;
3560	}
3561
3562	if (state->df) {
3563	WARN_INSN(insn, "return with DF set");
3564	return 1;
3565	}
3566
3567	if (func && has_modified_stack_frame(insn, state)) {
3568	WARN_INSN(insn, "return with modified stack frame");
3569	return 1;
3570	}
3571
3572	if (state->cfi.bp_scratch) {
3573	WARN_INSN(insn, "BP used as a scratch register");
3574	return 1;
3575	}
3576
3577	return 0;
3578	}
3579
3580	static struct instruction next_insn_to_validate(struct objtool_file file,
3581	struct instruction *insn)
3582	{
3583	struct alt_group *alt_group = insn->alt_group;
3584
3585	/*
3586	* Simulate the fact that alternatives are patched in-place. When the
3587	* end of a replacement alt_group is reached, redirect objtool flow to
3588	* the end of the original alt_group.
3589	*
3590	* insn->alts->insn -> alt_group->first_insn
3591	* ...
3592	* alt_group->last_insn
3593	* [alt_group->nop] -> next(orig_group->last_insn)
3594	*/
3595	if (alt_group) {
3596	if (alt_group->nop) {
3597	/* ->nop implies ->orig_group */
3598	if (insn == alt_group->last_insn)
3599	return alt_group->nop;
3600	if (insn == alt_group->nop)
3601	goto next_orig;
3602	}
3603	if (insn == alt_group->last_insn && alt_group->orig_group)
3604	goto next_orig;
3605	}
3606
3607	return next_insn_same_sec(file, insn);
3608
3609	next_orig:
3610	return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3611	}
3612
3613	static bool skip_alt_group(struct instruction *insn)
3614	{
3615	struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL;
3616
3617	if (!insn->alt_group)
3618	return false;
3619
3620	/* ANNOTATE_IGNORE_ALTERNATIVE */
3621	if (insn->alt_group->ignore) {
3622	TRACE_ALT(insn, "alt group ignored");
3623	return true;
3624	}
3625
3626	/*
3627	* For NOP patched with CLAC/STAC, only follow the latter to avoid
3628	* impossible code paths combining patched CLAC with unpatched STAC
3629	* or vice versa.
3630	*
3631	* ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus
3632	* requested not to do that to avoid hurting .s file readability
3633	* around CLAC/STAC alternative sites.
3634	*/
3635
3636	if (!alt_insn)
3637	return false;
3638
3639	/* Don't override ASM_{CLAC,STAC}_UNSAFE */
3640	if (alt_insn->alt_group && alt_insn->alt_group->ignore)
3641	return false;
3642
3643	return alt_insn->type == INSN_CLAC \|\| alt_insn->type == INSN_STAC;
3644	}
3645
3646	static int validate_branch(struct objtool_file file, struct symbol func,
3647	struct instruction *insn, struct insn_state state);
3648	static int do_validate_branch(struct objtool_file file, struct symbol func,
3649	struct instruction insn, struct insn_state state);
3650
3651	static int validate_insn(struct objtool_file file, struct symbol func,
3652	struct instruction insn, struct insn_state statep,
3653	struct instruction prev_insn, struct instruction next_insn,
3654	bool *dead_end)
3655	{
3656	char *alt_name __maybe_unused = NULL;
3657	struct alternative *alt;
3658	u8 visited;
3659	int ret;
3660
3661	/*
3662	* Any returns before the end of this function are effectively dead
3663	* ends, i.e. validate_branch() has reached the end of the branch.
3664	*/
3665	*dead_end = true;
3666
3667	visited = VISITED_BRANCH << statep->uaccess;
3668	if (insn->visited & VISITED_BRANCH_MASK) {
3669	if (!insn->hint && !insn_cfi_match(insn, &statep->cfi))
3670	return 1;
3671
3672	if (insn->visited & visited) {
3673	TRACE_INSN(insn, "already visited");
3674	return 0;
3675	}
3676	} else {
3677	nr_insns_visited++;
3678	}
3679
3680	if (statep->noinstr)
3681	statep->instr += insn->instr;
3682
3683	if (insn->hint) {
3684	if (insn->restore) {
3685	struct instruction save_insn, i;
3686
3687	i = insn;
3688	save_insn = NULL;
3689
3690	sym_for_each_insn_continue_reverse(file, func, i) {
3691	if (i->save) {
3692	save_insn = i;
3693	break;
3694	}
3695	}
3696
3697	if (!save_insn) {
3698	WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3699	return 1;
3700	}
3701
3702	if (!save_insn->visited) {
3703	/*
3704	* If the restore hint insn is at the
3705	* beginning of a basic block and was
3706	* branched to from elsewhere, and the
3707	* save insn hasn't been visited yet,
3708	* defer following this branch for now.
3709	* It will be seen later via the
3710	* straight-line path.
3711	*/
3712	if (!prev_insn) {
3713	TRACE_INSN(insn, "defer restore");
3714	return 0;
3715	}
3716
3717	WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3718	return 1;
3719	}
3720
3721	insn->cfi = save_insn->cfi;
3722	nr_cfi_reused++;
3723	}
3724
3725	statep->cfi = *insn->cfi;
3726	} else {
3727	/* XXX track if we actually changed statep->cfi */
3728
3729	if (prev_insn && !cficmp(prev_insn->cfi, &statep->cfi)) {
3730	insn->cfi = prev_insn->cfi;
3731	nr_cfi_reused++;
3732	} else {
3733	insn->cfi = cfi_hash_find_or_add(&statep->cfi);
3734	}
3735	}
3736
3737	insn->visited \|= visited;
3738
3739	if (propagate_alt_cfi(file, insn))
3740	return 1;
3741
3742	if (insn->alts) {
3743	for (alt = insn->alts; alt; alt = alt->next) {
3744	TRACE_ALT_BEGIN(insn, alt, alt_name);
3745	ret = validate_branch(file, func, alt->insn, *statep);
3746	TRACE_ALT_END(insn, alt, alt_name);
3747	if (ret) {
3748	BT_INSN(insn, "(alt)");
3749	return ret;
3750	}
3751	}
3752	TRACE_ALT_INFO_NOADDR(insn, "/ ", "DEFAULT");
3753	}
3754
3755	if (skip_alt_group(insn))
3756	return 0;
3757
3758	if (handle_insn_ops(insn, next_insn, statep))
3759	return 1;
3760
3761	switch (insn->type) {
3762
3763	case INSN_RETURN:
3764	TRACE_INSN(insn, "return");
3765	return validate_return(func, insn, statep);
3766
3767	case INSN_CALL:
3768	case INSN_CALL_DYNAMIC:
3769	if (insn->type == INSN_CALL)
3770	TRACE_INSN(insn, "call");
3771	else
3772	TRACE_INSN(insn, "indirect call");
3773
3774	ret = validate_call(file, insn, statep);
3775	if (ret)
3776	return ret;
3777
3778	if (opts.stackval && func && !is_special_call(insn) &&
3779	!has_valid_stack_frame(statep)) {
3780	WARN_INSN(insn, "call without frame pointer save/setup");
3781	return 1;
3782	}
3783
3784	break;
3785
3786	case INSN_JUMP_CONDITIONAL:
3787	case INSN_JUMP_UNCONDITIONAL:
3788	if (is_sibling_call(insn)) {
3789	TRACE_INSN(insn, "sibling call");
3790	ret = validate_sibling_call(file, insn, statep);
3791	if (ret)
3792	return ret;
3793
3794	} else if (insn->jump_dest) {
3795	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3796	TRACE_INSN(insn, "unconditional jump");
3797	else
3798	TRACE_INSN(insn, "jump taken");
3799
3800	ret = validate_branch(file, func, insn->jump_dest, *statep);
3801	if (ret) {
3802	BT_INSN(insn, "(branch)");
3803	return ret;
3804	}
3805	}
3806
3807	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3808	return 0;
3809
3810	TRACE_INSN(insn, "jump not taken");
3811	break;
3812
3813	case INSN_JUMP_DYNAMIC:
3814	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3815	TRACE_INSN(insn, "indirect jump");
3816	if (is_sibling_call(insn)) {
3817	ret = validate_sibling_call(file, insn, statep);
3818	if (ret)
3819	return ret;
3820	}
3821
3822	if (insn->type == INSN_JUMP_DYNAMIC)
3823	return 0;
3824
3825	break;
3826
3827	case INSN_SYSCALL:
3828	TRACE_INSN(insn, "syscall");
3829	if (func && (!next_insn \|\| !next_insn->hint)) {
3830	WARN_INSN(insn, "unsupported instruction in callable function");
3831	return 1;
3832	}
3833
3834	break;
3835
3836	case INSN_SYSRET:
3837	TRACE_INSN(insn, "sysret");
3838	if (func && (!next_insn \|\| !next_insn->hint)) {
3839	WARN_INSN(insn, "unsupported instruction in callable function");
3840	return 1;
3841	}
3842
3843	return 0;
3844
3845	case INSN_STAC:
3846	TRACE_INSN(insn, "stac");
3847	if (!opts.uaccess)
3848	break;
3849
3850	if (statep->uaccess) {
3851	WARN_INSN(insn, "recursive UACCESS enable");
3852	return 1;
3853	}
3854
3855	statep->uaccess = true;
3856	break;
3857
3858	case INSN_CLAC:
3859	TRACE_INSN(insn, "clac");
3860	if (!opts.uaccess)
3861	break;
3862
3863	if (!statep->uaccess && func) {
3864	WARN_INSN(insn, "redundant UACCESS disable");
3865	return 1;
3866	}
3867
3868	if (func_uaccess_safe(func) && !statep->uaccess_stack) {
3869	WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3870	return 1;
3871	}
3872
3873	statep->uaccess = false;
3874	break;
3875
3876	case INSN_STD:
3877	TRACE_INSN(insn, "std");
3878	if (statep->df) {
3879	WARN_INSN(insn, "recursive STD");
3880	return 1;
3881	}
3882
3883	statep->df = true;
3884	break;
3885
3886	case INSN_CLD:
3887	TRACE_INSN(insn, "cld");
3888	if (!statep->df && func) {
3889	WARN_INSN(insn, "redundant CLD");
3890	return 1;
3891	}
3892
3893	statep->df = false;
3894	break;
3895
3896	default:
3897	break;
3898	}
3899
3900	if (insn->dead_end)
3901	TRACE_INSN(insn, "dead end");
3902
3903	*dead_end = insn->dead_end;
3904	return 0;
3905	}
3906
3907	/*
3908	* Follow the branch starting at the given instruction, and recursively follow
3909	* any other branches (jumps). Meanwhile, track the frame pointer state at
3910	* each instruction and validate all the rules described in
3911	* tools/objtool/Documentation/objtool.txt.
3912	*/
3913	static int do_validate_branch(struct objtool_file file, struct symbol func,
3914	struct instruction insn, struct insn_state state)
3915	{
3916	struct instruction next_insn, prev_insn = NULL;
3917	bool dead_end;
3918	int ret;
3919
3920	if (func && func->ignore)
3921	return 0;
3922
3923	do {
3924	insn->trace = 0;
3925	next_insn = next_insn_to_validate(file, insn);
3926
3927	if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3928	/* Ignore KCFI type preambles, which always fall through */
3929	if (is_prefix_func(func))
3930	return 0;
3931
3932	if (file->ignore_unreachables)
3933	return 0;
3934
3935	WARN("%s() falls through to next function %s()",
3936	func->name, insn_func(insn)->name);
3937	func->warned = 1;
3938
3939	return 1;
3940	}
3941
3942	ret = validate_insn(file, func, insn, state, prev_insn, next_insn,
3943	&dead_end);
3944
3945	if (!insn->trace) {
3946	if (ret)
3947	TRACE_INSN(insn, "warning (%d)", ret);
3948	else
3949	TRACE_INSN(insn, NULL);
3950	}
3951
3952	if (!dead_end && !next_insn) {
3953	if (state->cfi.cfa.base == CFI_UNDEFINED)
3954	return 0;
3955	if (file->ignore_unreachables)
3956	return 0;
3957
3958	WARN("%s%sunexpected end of section %s",
3959	func ? func->name : "", func ? "(): " : "",
3960	insn->sec->name);
3961	return 1;
3962	}
3963
3964	prev_insn = insn;
3965	insn = next_insn;
3966
3967	} while (!dead_end);
3968
3969	return ret;
3970	}
3971
3972	static int validate_branch(struct objtool_file file, struct symbol func,
3973	struct instruction *insn, struct insn_state state)
3974	{
3975	int ret;
3976
3977	trace_depth_inc();
3978	ret = do_validate_branch(file, func, insn, &state);
3979	trace_depth_dec();
3980
3981	return ret;
3982	}
3983
3984	static int validate_unwind_hint(struct objtool_file *file,
3985	struct instruction *insn,
3986	struct insn_state *state)
3987	{
3988	if (insn->hint && !insn->visited) {
3989	struct symbol *func = insn_func(insn);
3990	int ret;
3991
3992	ret = validate_branch(file, func, insn, *state);
3993	if (ret)
3994	BT_INSN(insn, "<=== (hint)");
3995	return ret;
3996	}
3997
3998	return 0;
3999	}
4000
4001	static int validate_unwind_hints(struct objtool_file file, struct section sec)
4002	{
4003	struct instruction *insn;
4004	struct insn_state state;
4005	int warnings = 0;
4006
4007	if (!file->hints)
4008	return 0;
4009
4010	init_insn_state(file, &state, sec);
4011
4012	if (sec) {
4013	sec_for_each_insn(file, sec, insn)
4014	warnings += validate_unwind_hint(file, insn, &state);
4015	} else {
4016	for_each_insn(file, insn)
4017	warnings += validate_unwind_hint(file, insn, &state);
4018	}
4019
4020	return warnings;
4021	}
4022
4023	/*
4024	* Validate rethunk entry constraint: must untrain RET before the first RET.
4025	*
4026	* Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
4027	* before an actual RET instruction.
4028	*/
4029	static int validate_unret(struct objtool_file file, struct instruction insn)
4030	{
4031	struct instruction next, dest;
4032	int ret;
4033
4034	for (;;) {
4035	next = next_insn_to_validate(file, insn);
4036
4037	if (insn->visited & VISITED_UNRET)
4038	return 0;
4039
4040	insn->visited \|= VISITED_UNRET;
4041
4042	if (insn->alts) {
4043	struct alternative *alt;
4044	for (alt = insn->alts; alt; alt = alt->next) {
4045	ret = validate_unret(file, alt->insn);
4046	if (ret) {
4047	BT_INSN(insn, "(alt)");
4048	return ret;
4049	}
4050	}
4051	}
4052
4053	switch (insn->type) {
4054
4055	case INSN_CALL_DYNAMIC:
4056	case INSN_JUMP_DYNAMIC:
4057	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4058	WARN_INSN(insn, "early indirect call");
4059	return 1;
4060
4061	case INSN_JUMP_UNCONDITIONAL:
4062	case INSN_JUMP_CONDITIONAL:
4063	if (!is_sibling_call(insn)) {
4064	if (!insn->jump_dest) {
4065	WARN_INSN(insn, "unresolved jump target after linking?!?");
4066	return 1;
4067	}
4068	ret = validate_unret(file, insn->jump_dest);
4069	if (ret) {
4070	BT_INSN(insn, "(branch%s)",
4071	insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
4072	return ret;
4073	}
4074
4075	if (insn->type == INSN_JUMP_UNCONDITIONAL)
4076	return 0;
4077
4078	break;
4079	}
4080
4081	/* fallthrough */
4082	case INSN_CALL:
4083	dest = find_insn(file, insn_call_dest(insn)->sec,
4084	insn_call_dest(insn)->offset);
4085	if (!dest) {
4086	WARN("Unresolved function after linking!?: %s",
4087	insn_call_dest(insn)->name);
4088	return 1;
4089	}
4090
4091	ret = validate_unret(file, dest);
4092	if (ret) {
4093	BT_INSN(insn, "(call)");
4094	return ret;
4095	}
4096	/*
4097	* If a call returns without error, it must have seen UNTRAIN_RET.
4098	* Therefore any non-error return is a success.
4099	*/
4100	return 0;
4101
4102	case INSN_RETURN:
4103	WARN_INSN(insn, "RET before UNTRAIN");
4104	return 1;
4105
4106	case INSN_SYSCALL:
4107	break;
4108
4109	case INSN_SYSRET:
4110	return 0;
4111
4112	case INSN_NOP:
4113	if (insn->retpoline_safe)
4114	return 0;
4115	break;
4116
4117	default:
4118	break;
4119	}
4120
4121	if (insn->dead_end)
4122	return 0;
4123
4124	if (!next) {
4125	WARN_INSN(insn, "teh end!");
4126	return 1;
4127	}
4128	insn = next;
4129	}
4130
4131	return 0;
4132	}
4133
4134	/*
4135	* Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
4136	* VALIDATE_UNRET_END before RET.
4137	*/
4138	static int validate_unrets(struct objtool_file *file)
4139	{
4140	struct instruction *insn;
4141	int warnings = 0;
4142
4143	for_each_insn(file, insn) {
4144	if (!insn->unret)
4145	continue;
4146
4147	warnings += validate_unret(file, insn);
4148	}
4149
4150	return warnings;
4151	}
4152
4153	static int validate_retpoline(struct objtool_file *file)
4154	{
4155	struct instruction *insn;
4156	int warnings = 0;
4157
4158	for_each_insn(file, insn) {
4159	if (insn->type != INSN_JUMP_DYNAMIC &&
4160	insn->type != INSN_CALL_DYNAMIC &&
4161	insn->type != INSN_RETURN)
4162	continue;
4163
4164	if (insn->retpoline_safe)
4165	continue;
4166
4167	if (insn->sec->init)
4168	continue;
4169
4170	if (insn->type == INSN_RETURN) {
4171	if (opts.rethunk) {
4172	WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4173	warnings++;
4174	}
4175	continue;
4176	}
4177
4178	WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4179	insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4180	warnings++;
4181	}
4182
4183	if (!opts.cfi)
4184	return warnings;
4185
4186	/*
4187	* kCFI call sites look like:
4188	*
4189	* movl $(-0x12345678), %r10d
4190	* addl -4(%r11), %r10d
4191	* jz 1f
4192	* ud2
4193	* 1: cs call __x86_indirect_thunk_r11
4194	*
4195	* Verify all indirect calls are kCFI adorned by checking for the
4196	* UD2. Notably, doing __nocfi calls to regular (cfi) functions is
4197	* broken.
4198	*/
4199	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
4200	struct symbol *sym = insn_sym(insn);
4201
4202	if (sym && (is_notype_sym(sym) \|\|
4203	is_func_sym(sym)) && !sym->nocfi) {
4204	struct instruction *prev =
4205	prev_insn_same_sym(file, insn);
4206
4207	if (!prev \|\| prev->type != INSN_BUG) {
4208	WARN_INSN(insn, "no-cfi indirect call!");
4209	warnings++;
4210	}
4211	}
4212	}
4213
4214	return warnings;
4215	}
4216
4217	static bool is_kasan_insn(struct instruction *insn)
4218	{
4219	return (insn->type == INSN_CALL &&
4220	!strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4221	}
4222
4223	static bool is_ubsan_insn(struct instruction *insn)
4224	{
4225	return (insn->type == INSN_CALL &&
4226	!strcmp(insn_call_dest(insn)->name,
4227	"__ubsan_handle_builtin_unreachable"));
4228	}
4229
4230	static bool ignore_unreachable_insn(struct objtool_file file, struct instruction insn)
4231	{
4232	struct symbol *func = insn_func(insn);
4233	struct instruction *prev_insn;
4234	int i;
4235
4236	if (insn->type == INSN_NOP \|\| insn->type == INSN_TRAP \|\|
4237	insn->hole \|\| (func && func->ignore))
4238	return true;
4239
4240	/*
4241	* Ignore alternative replacement instructions. This can happen
4242	* when a whitelisted function uses one of the ALTERNATIVE macros.
4243	*/
4244	if (!strcmp(insn->sec->name, ".altinstr_replacement") \|\|
4245	!strcmp(insn->sec->name, ".altinstr_aux"))
4246	return true;
4247
4248	if (!func)
4249	return false;
4250
4251	if (func->static_call_tramp)
4252	return true;
4253
4254	/*
4255	* CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4256	* __builtin_unreachable(). The BUG() macro has an unreachable() after
4257	* the UD2, which causes GCC's undefined trap logic to emit another UD2
4258	* (or occasionally a JMP to UD2).
4259	*
4260	* It may also insert a UD2 after calling a __noreturn function.
4261	*/
4262	prev_insn = prev_insn_same_sec(file, insn);
4263	if (prev_insn && prev_insn->dead_end &&
4264	(insn->type == INSN_BUG \|\|
4265	(insn->type == INSN_JUMP_UNCONDITIONAL &&
4266	insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4267	return true;
4268
4269	/*
4270	* Check if this (or a subsequent) instruction is related to
4271	* CONFIG_UBSAN or CONFIG_KASAN.
4272	*
4273	* End the search at 5 instructions to avoid going into the weeds.
4274	*/
4275	for (i = 0; i < 5; i++) {
4276
4277	if (is_kasan_insn(insn) \|\| is_ubsan_insn(insn))
4278	return true;
4279
4280	if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4281	if (insn->jump_dest &&
4282	insn_func(insn->jump_dest) == func) {
4283	insn = insn->jump_dest;
4284	continue;
4285	}
4286
4287	break;
4288	}
4289
4290	if (insn->offset + insn->len >= func->offset + func->len)
4291	break;
4292
4293	insn = next_insn_same_sec(file, insn);
4294	}
4295
4296	return false;
4297	}
4298
4299	/*
4300	* For FineIBT or kCFI, a certain number of bytes preceding the function may be
4301	* NOPs. Those NOPs may be rewritten at runtime and executed, so give them a
4302	* proper function name: __pfx_<func>.
4303	*/
4304	static int create_prefix_symbol(struct objtool_file file, struct symbol func)
4305	{
4306	struct instruction insn, prev;
4307	char name[SYM_NAME_LEN];
4308	struct cfi_state *cfi;
4309
4310	if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) {
4311	WARN("%s: symbol name too long, can't create __pfx_ symbol",
4312	func->name);
4313	return 0;
4314	}
4315
4316	if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name))
4317	return -1;
4318
4319	if (!elf_create_symbol(file->elf, name, func->sec,
4320	GELF_ST_BIND(func->sym.st_info),
4321	GELF_ST_TYPE(func->sym.st_info),
4322	func->offset - opts.prefix, opts.prefix))
4323	return -1;
4324
4325	/* Propagate insn->cfi to the prefix code */
4326	insn = find_insn(file, func->sec, func->offset);
4327	if (!insn \|\| !insn->cfi)
4328	return 0;
4329
4330	cfi = cfi_hash_find_or_add(insn->cfi);
4331	for (prev = find_insn(file, func->sec, func->offset - opts.prefix);
4332	prev && prev != insn;
4333	prev = next_insn_same_sec(file, prev))
4334	prev->cfi = cfi;
4335
4336	return 0;
4337	}
4338
4339	static int create_prefix_symbols(struct objtool_file *file)
4340	{
4341	struct section *pfe_sec;
4342	struct symbol *func;
4343	struct reloc *reloc;
4344
4345	for_each_sec(file->elf, pfe_sec) {
4346	if (strcmp(pfe_sec->name, "__patchable_function_entries"))
4347	continue;
4348	if (!pfe_sec->rsec)
4349	continue;
4350
4351	for_each_reloc(pfe_sec->rsec, reloc) {
4352	func = find_func_by_offset(reloc->sym->sec,
4353	reloc->sym->offset + reloc_addend(reloc) + opts.prefix);
4354	if (func && create_prefix_symbol(file, func))
4355	return -1;
4356	}
4357	}
4358
4359	return 0;
4360	}
4361
4362	static int validate_symbol(struct objtool_file file, struct section sec,
4363	struct symbol sym, struct insn_state state)
4364	{
4365	struct instruction *insn;
4366	struct symbol *func;
4367	int ret;
4368
4369	if (!sym->len) {
4370	WARN("%s() is missing an ELF size annotation", sym->name);
4371	return 1;
4372	}
4373
4374	if (sym->pfunc != sym \|\| is_alias_sym(sym))
4375	return 0;
4376
4377	insn = find_insn(file, sec, sym->offset);
4378	if (!insn \|\| insn->visited)
4379	return 0;
4380
4381	if (opts.uaccess)
4382	state->uaccess = sym->uaccess_safe;
4383
4384	func = insn_func(insn);
4385
4386	if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) {
4387	trace_enable();
4388	TRACE("%s: validation begin\n", sym->name);
4389	}
4390
4391	ret = validate_branch(file, func, insn, *state);
4392	if (ret)
4393	BT_INSN(insn, "<=== (sym)");
4394
4395	TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end");
4396	trace_disable();
4397
4398	return ret;
4399	}
4400
4401	static int validate_section(struct objtool_file file, struct section sec)
4402	{
4403	struct insn_state state;
4404	struct symbol *func;
4405	int warnings = 0;
4406
4407	sec_for_each_sym(sec, func) {
4408	if (!is_func_sym(func))
4409	continue;
4410
4411	init_insn_state(file, &state, sec);
4412	set_func_state(&state.cfi);
4413
4414	warnings += validate_symbol(file, sec, func, &state);
4415	}
4416
4417	return warnings;
4418	}
4419
4420	static int validate_noinstr_sections(struct objtool_file *file)
4421	{
4422	struct section *sec;
4423	int warnings = 0;
4424
4425	sec = find_section_by_name(file->elf, ".noinstr.text");
4426	if (sec) {
4427	warnings += validate_section(file, sec);
4428	warnings += validate_unwind_hints(file, sec);
4429	}
4430
4431	sec = find_section_by_name(file->elf, ".entry.text");
4432	if (sec) {
4433	warnings += validate_section(file, sec);
4434	warnings += validate_unwind_hints(file, sec);
4435	}
4436
4437	sec = find_section_by_name(file->elf, ".cpuidle.text");
4438	if (sec) {
4439	warnings += validate_section(file, sec);
4440	warnings += validate_unwind_hints(file, sec);
4441	}
4442
4443	return warnings;
4444	}
4445
4446	static int validate_functions(struct objtool_file *file)
4447	{
4448	struct section *sec;
4449	int warnings = 0;
4450
4451	for_each_sec(file->elf, sec) {
4452	if (!is_text_sec(sec))
4453	continue;
4454
4455	warnings += validate_section(file, sec);
4456	}
4457
4458	return warnings;
4459	}
4460
4461	static void mark_endbr_used(struct instruction *insn)
4462	{
4463	if (!list_empty(&insn->call_node))
4464	list_del_init(&insn->call_node);
4465	}
4466
4467	static bool noendbr_range(struct objtool_file file, struct instruction insn)
4468	{
4469	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4470	struct instruction *first;
4471
4472	if (!sym)
4473	return false;
4474
4475	first = find_insn(file, sym->sec, sym->offset);
4476	if (!first)
4477	return false;
4478
4479	if (first->type != INSN_ENDBR && !first->noendbr)
4480	return false;
4481
4482	return insn->offset == sym->offset + sym->len;
4483	}
4484
4485	static int __validate_ibt_insn(struct objtool_file file, struct instruction insn,
4486	struct instruction *dest)
4487	{
4488	if (dest->type == INSN_ENDBR) {
4489	mark_endbr_used(dest);
4490	return 0;
4491	}
4492
4493	if (insn_func(dest) && insn_func(insn) &&
4494	insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4495	/*
4496	* Anything from->to self is either _THIS_IP_ or
4497	* IRET-to-self.
4498	*
4499	* There is no sane way to annotate _THIS_IP_ since the
4500	* compiler treats the relocation as a constant and is
4501	* happy to fold in offsets, skewing any annotation we
4502	* do, leading to vast amounts of false-positives.
4503	*
4504	* There's also compiler generated _THIS_IP_ through
4505	* KCOV and such which we have no hope of annotating.
4506	*
4507	* As such, blanket accept self-references without
4508	* issue.
4509	*/
4510	return 0;
4511	}
4512
4513	/*
4514	* Accept anything ANNOTATE_NOENDBR.
4515	*/
4516	if (dest->noendbr)
4517	return 0;
4518
4519	/*
4520	* Accept if this is the instruction after a symbol
4521	* that is (no)endbr -- typical code-range usage.
4522	*/
4523	if (noendbr_range(file, dest))
4524	return 0;
4525
4526	WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4527	return 1;
4528	}
4529
4530	static int validate_ibt_insn(struct objtool_file file, struct instruction insn)
4531	{
4532	struct instruction *dest;
4533	struct reloc *reloc;
4534	unsigned long off;
4535	int warnings = 0;
4536
4537	/*
4538	* Looking for function pointer load relocations. Ignore
4539	* direct/indirect branches:
4540	*/
4541	switch (insn->type) {
4542
4543	case INSN_CALL:
4544	case INSN_CALL_DYNAMIC:
4545	case INSN_JUMP_CONDITIONAL:
4546	case INSN_JUMP_UNCONDITIONAL:
4547	case INSN_JUMP_DYNAMIC:
4548	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4549	case INSN_RETURN:
4550	case INSN_NOP:
4551	return 0;
4552
4553	case INSN_LEA_RIP:
4554	if (!insn_reloc(file, insn)) {
4555	/* local function pointer reference without reloc */
4556
4557	off = arch_jump_destination(insn);
4558
4559	dest = find_insn(file, insn->sec, off);
4560	if (!dest) {
4561	WARN_INSN(insn, "corrupt function pointer reference");
4562	return 1;
4563	}
4564
4565	return __validate_ibt_insn(file, insn, dest);
4566	}
4567	break;
4568
4569	default:
4570	break;
4571	}
4572
4573	for (reloc = insn_reloc(file, insn);
4574	reloc;
4575	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4576	reloc_offset(reloc) + 1,
4577	(insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4578
4579	off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc);
4580
4581	dest = find_insn(file, reloc->sym->sec, off);
4582	if (!dest)
4583	continue;
4584
4585	warnings += __validate_ibt_insn(file, insn, dest);
4586	}
4587
4588	return warnings;
4589	}
4590
4591	static int validate_ibt_data_reloc(struct objtool_file *file,
4592	struct reloc *reloc)
4593	{
4594	struct instruction *dest;
4595
4596	dest = find_insn(file, reloc->sym->sec,
4597	reloc->sym->offset + reloc_addend(reloc));
4598	if (!dest)
4599	return 0;
4600
4601	if (dest->type == INSN_ENDBR) {
4602	mark_endbr_used(dest);
4603	return 0;
4604	}
4605
4606	if (dest->noendbr)
4607	return 0;
4608
4609	WARN_FUNC(reloc->sec->base, reloc_offset(reloc),
4610	"data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4611
4612	return 1;
4613	}
4614
4615	/*
4616	* Validate IBT rules and remove used ENDBR instructions from the seal list.
4617	* Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4618	* NOPs) later, in create_ibt_endbr_seal_sections().
4619	*/
4620	static int validate_ibt(struct objtool_file *file)
4621	{
4622	struct section *sec;
4623	struct reloc *reloc;
4624	struct instruction *insn;
4625	int warnings = 0;
4626
4627	for_each_insn(file, insn)
4628	warnings += validate_ibt_insn(file, insn);
4629
4630	for_each_sec(file->elf, sec) {
4631
4632	/* Already done by validate_ibt_insn() */
4633	if (is_text_sec(sec))
4634	continue;
4635
4636	if (!sec->rsec)
4637	continue;
4638
4639	/*
4640	* These sections can reference text addresses, but not with
4641	* the intent to indirect branch to them.
4642	*/
4643	if ((!strncmp(sec->name, ".discard", 8) &&
4644	strcmp(sec->name, ".discard.ibt_endbr_noseal")) \|\|
4645	!strncmp(sec->name, ".debug", 6) \|\|
4646	!strcmp(sec->name, ".altinstructions") \|\|
4647	!strcmp(sec->name, ".ibt_endbr_seal") \|\|
4648	!strcmp(sec->name, ".kcfi_traps") \|\|
4649	!strcmp(sec->name, ".orc_unwind_ip") \|\|
4650	!strcmp(sec->name, ".retpoline_sites") \|\|
4651	!strcmp(sec->name, ".static_call_sites") \|\|
4652	!strcmp(sec->name, "_error_injection_whitelist") \|\|
4653	!strcmp(sec->name, "_kprobe_blacklist") \|\|
4654	!strcmp(sec->name, "__bug_table") \|\|
4655	!strcmp(sec->name, "__ex_table") \|\|
4656	!strcmp(sec->name, "__jump_table") \|\|
4657	!strcmp(sec->name, ".init.klp_funcs") \|\|
4658	!strcmp(sec->name, "__mcount_loc") \|\|
4659	!strcmp(sec->name, ".llvm.call-graph-profile") \|\|
4660	!strcmp(sec->name, ".llvm_bb_addr_map") \|\|
4661	!strcmp(sec->name, "__tracepoints") \|\|
4662	!strcmp(sec->name, ".return_sites") \|\|
4663	!strcmp(sec->name, ".call_sites") \|\|
4664	!strcmp(sec->name, "__patchable_function_entries"))
4665	continue;
4666
4667	for_each_reloc(sec->rsec, reloc)
4668	warnings += validate_ibt_data_reloc(file, reloc);
4669	}
4670
4671	return warnings;
4672	}
4673
4674	static int validate_sls(struct objtool_file *file)
4675	{
4676	struct instruction insn, next_insn;
4677	int warnings = 0;
4678
4679	for_each_insn(file, insn) {
4680	next_insn = next_insn_same_sec(file, insn);
4681
4682	if (insn->retpoline_safe)
4683	continue;
4684
4685	switch (insn->type) {
4686	case INSN_RETURN:
4687	if (!next_insn \|\| next_insn->type != INSN_TRAP) {
4688	WARN_INSN(insn, "missing int3 after ret");
4689	warnings++;
4690	}
4691
4692	break;
4693	case INSN_JUMP_DYNAMIC:
4694	if (!next_insn \|\| next_insn->type != INSN_TRAP) {
4695	WARN_INSN(insn, "missing int3 after indirect jump");
4696	warnings++;
4697	}
4698	break;
4699	default:
4700	break;
4701	}
4702	}
4703
4704	return warnings;
4705	}
4706
4707	static int validate_reachable_instructions(struct objtool_file *file)
4708	{
4709	struct instruction insn, prev_insn;
4710	struct symbol *call_dest;
4711	int warnings = 0;
4712
4713	if (file->ignore_unreachables)
4714	return 0;
4715
4716	for_each_insn(file, insn) {
4717	if (insn->visited \|\| ignore_unreachable_insn(file, insn))
4718	continue;
4719
4720	prev_insn = prev_insn_same_sec(file, insn);
4721	if (prev_insn && prev_insn->dead_end) {
4722	call_dest = insn_call_dest(prev_insn);
4723	if (call_dest) {
4724	WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h",
4725	call_dest->name);
4726	warnings++;
4727	continue;
4728	}
4729	}
4730
4731	WARN_INSN(insn, "unreachable instruction");
4732	warnings++;
4733	}
4734
4735	return warnings;
4736	}
4737
4738	__weak bool arch_absolute_reloc(struct elf elf, struct reloc reloc)
4739	{
4740	unsigned int type = reloc_type(reloc);
4741	size_t sz = elf_addr_size(elf);
4742
4743	return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32);
4744	}
4745
4746	static int check_abs_references(struct objtool_file *file)
4747	{
4748	struct section *sec;
4749	struct reloc *reloc;
4750	int ret = 0;
4751
4752	for_each_sec(file->elf, sec) {
4753	/* absolute references in non-loadable sections are fine */
4754	if (!(sec->sh.sh_flags & SHF_ALLOC))
4755	continue;
4756
4757	/* section must have an associated .rela section */
4758	if (!sec->rsec)
4759	continue;
4760
4761	/*
4762	* Special case for compiler generated metadata that is not
4763	* consumed until after boot.
4764	*/
4765	if (!strcmp(sec->name, "__patchable_function_entries"))
4766	continue;
4767
4768	for_each_reloc(sec->rsec, reloc) {
4769	if (arch_absolute_reloc(file->elf, reloc)) {
4770	WARN("section %s has absolute relocation at offset 0x%llx",
4771	sec->name, (unsigned long long)reloc_offset(reloc));
4772	ret++;
4773	}
4774	}
4775	}
4776	return ret;
4777	}
4778
4779	struct insn_chunk {
4780	void *addr;
4781	struct insn_chunk *next;
4782	};
4783
4784	/*
4785	* Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4786	* which can trigger more allocations for .debug_* sections whose data hasn't
4787	* been read yet.
4788	*/
4789	void free_insns(struct objtool_file *file)
4790	{
4791	struct instruction *insn;
4792	struct insn_chunk chunks = NULL, chunk;
4793
4794	for_each_insn(file, insn) {
4795	if (!insn->idx) {
4796	chunk = malloc(sizeof(*chunk));
4797	chunk->addr = insn;
4798	chunk->next = chunks;
4799	chunks = chunk;
4800	}
4801	}
4802
4803	for (chunk = chunks; chunk; chunk = chunk->next)
4804	free(chunk->addr);
4805	}
4806
4807	const char objtool_disas_insn(struct instruction insn)
4808	{
4809	struct disas_context *dctx = objtool_disas_ctx;
4810
4811	if (!dctx)
4812	return "";
4813
4814	disas_insn(dctx, insn);
4815	return disas_result(dctx);
4816	}
4817
4818	int check(struct objtool_file *file)
4819	{
4820	struct disas_context *disas_ctx = NULL;
4821	int ret = 0, warnings = 0;
4822
4823	/*
4824	* Create a disassembly context if we might disassemble any
4825	* instruction or function.
4826	*/
4827	if (opts.verbose \|\| opts.backtrace \|\| opts.trace \|\| opts.disas) {
4828	disas_ctx = disas_context_create(file);
4829	if (!disas_ctx) {
4830	opts.disas = false;
4831	opts.trace = false;
4832	}
4833	objtool_disas_ctx = disas_ctx;
4834	}
4835
4836	ret = decode_file(file);
4837	if (ret)
4838	goto out;
4839
4840	if (!nr_insns)
4841	goto out;
4842
4843	if (opts.retpoline)
4844	warnings += validate_retpoline(file);
4845
4846	if (validate_branch_enabled()) {
4847	int w = 0;
4848
4849	w += validate_functions(file);
4850	w += validate_unwind_hints(file, NULL);
4851	if (!w)
4852	w += validate_reachable_instructions(file);
4853
4854	warnings += w;
4855
4856	} else if (opts.noinstr) {
4857	warnings += validate_noinstr_sections(file);
4858	}
4859
4860	if (opts.unret) {
4861	/*
4862	* Must be after validate_branch() and friends, it plays
4863	* further games with insn->visited.
4864	*/
4865	warnings += validate_unrets(file);
4866	}
4867
4868	if (opts.ibt)
4869	warnings += validate_ibt(file);
4870
4871	if (opts.sls)
4872	warnings += validate_sls(file);
4873
4874	if (opts.static_call) {
4875	ret = create_static_call_sections(file);
4876	if (ret)
4877	goto out;
4878	}
4879
4880	if (opts.retpoline) {
4881	ret = create_retpoline_sites_sections(file);
4882	if (ret)
4883	goto out;
4884	}
4885
4886	if (opts.rethunk) {
4887	ret = create_return_sites_sections(file);
4888	if (ret)
4889	goto out;
4890
4891	if (opts.hack_skylake) {
4892	ret = create_direct_call_sections(file);
4893	if (ret)
4894	goto out;
4895	}
4896	}
4897
4898	if (opts.mcount) {
4899	ret = create_mcount_loc_sections(file);
4900	if (ret)
4901	goto out;
4902	}
4903
4904	if (opts.prefix) {
4905	if (!opts.cfi) {
4906	ret = create_prefix_symbols(file);
4907	if (ret)
4908	goto out;
4909	} else {
4910	ret = grow_cfi_symbols(file);
4911	if (ret)
4912	goto out;
4913
4914	if (opts.fineibt) {
4915	ret = create_cfi_sections(file);
4916	if (ret)
4917	goto out;
4918	}
4919	}
4920	}
4921
4922	if (opts.ibt) {
4923	ret = create_ibt_endbr_seal_sections(file);
4924	if (ret)
4925	goto out;
4926	}
4927
4928	if (opts.klp_symids) {
4929	ret = klp_create_symid_sections(file);
4930	if (ret)
4931	goto out;
4932	}
4933
4934	if (opts.noabs)
4935	warnings += check_abs_references(file);
4936
4937	if (opts.orc && nr_insns) {
4938	ret = orc_create(file);
4939	if (ret)
4940	goto out;
4941	}
4942
4943	if (opts.stats) {
4944	printf("nr_insns_visited: %ld\n", nr_insns_visited);
4945	printf("nr_cfi: %ld\n", nr_cfi);
4946	printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4947	printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4948	}
4949
4950	out:
4951	if (ret \|\| warnings) {
4952	if (opts.werror && warnings)
4953	ret = 1;
4954
4955	if (opts.verbose) {
4956	if (opts.werror && warnings)
4957	WARN("%d warning(s) upgraded to errors", warnings);
4958	disas_warned_funcs(disas_ctx);
4959	}
4960	}
4961
4962	if (opts.disas)
4963	disas_funcs(disas_ctx);
4964
4965	if (disas_ctx) {
4966	disas_context_destroy(disas_ctx);
4967	objtool_disas_ctx = NULL;
4968	}
4969
4970	free_insns(file);
4971
4972	if (!ret && !warnings)
4973	return 0;
4974
4975	if (opts.backup && make_backup())
4976	return 1;
4977
4978	return ret;
4979	}