linux - tools/objtool/check.c

4953 lines · 114.0 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2015-2017 Josh Poimboeuf <[email protected]>
4	*/
5
6	#include <string.h>
7	#include <stdlib.h>
8	#include <inttypes.h>
9	#include <sys/mman.h>
10
11	#include <objtool/builtin.h>
12	#include <objtool/cfi.h>
13	#include <objtool/arch.h>
14	#include <objtool/check.h>
15	#include <objtool/special.h>
16	#include <objtool/warn.h>
17	#include <objtool/endianness.h>
18
19	#include <linux/objtool_types.h>
20	#include <linux/hashtable.h>
21	#include <linux/kernel.h>
22	#include <linux/static_call_types.h>
23	#include <linux/string.h>
24
25	struct alternative {
26	struct alternative *next;
27	struct instruction *insn;
28	bool skip_orig;
29	};
30
31	static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33	static struct cfi_init_state initial_func_cfi;
34	static struct cfi_state init_cfi;
35	static struct cfi_state func_cfi;
36	static struct cfi_state force_undefined_cfi;
37
38	struct instruction find_insn(struct objtool_file file,
39	struct section *sec, unsigned long offset)
40	{
41	struct instruction *insn;
42
43	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
44	if (insn->sec == sec && insn->offset == offset)
45	return insn;
46	}
47
48	return NULL;
49	}
50
51	struct instruction next_insn_same_sec(struct objtool_file file,
52	struct instruction *insn)
53	{
54	if (insn->idx == INSN_CHUNK_MAX)
55	return find_insn(file, insn->sec, insn->offset + insn->len);
56
57	insn++;
58	if (!insn->len)
59	return NULL;
60
61	return insn;
62	}
63
64	static struct instruction next_insn_same_func(struct objtool_file file,
65	struct instruction *insn)
66	{
67	struct instruction *next = next_insn_same_sec(file, insn);
68	struct symbol *func = insn_func(insn);
69
70	if (!func)
71	return NULL;
72
73	if (next && insn_func(next) == func)
74	return next;
75
76	/* Check if we're already in the subfunction: */
77	if (func == func->cfunc)
78	return NULL;
79
80	/* Move to the subfunction: */
81	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
82	}
83
84	static struct instruction prev_insn_same_sec(struct objtool_file file,
85	struct instruction *insn)
86	{
87	if (insn->idx == 0) {
88	if (insn->prev_len)
89	return find_insn(file, insn->sec, insn->offset - insn->prev_len);
90	return NULL;
91	}
92
93	return insn - 1;
94	}
95
96	static struct instruction prev_insn_same_sym(struct objtool_file file,
97	struct instruction *insn)
98	{
99	struct instruction *prev = prev_insn_same_sec(file, insn);
100
101	if (prev && insn_func(prev) == insn_func(insn))
102	return prev;
103
104	return NULL;
105	}
106
107	#define for_each_insn(file, insn) \
108	for (struct section __sec, __fake = (struct section *)1; \
109	__fake; __fake = NULL) \
110	for_each_sec(file, __sec) \
111	sec_for_each_insn(file, __sec, insn)
112
113	#define func_for_each_insn(file, func, insn) \
114	for (insn = find_insn(file, func->sec, func->offset); \
115	insn; \
116	insn = next_insn_same_func(file, insn))
117
118	#define sym_for_each_insn(file, sym, insn) \
119	for (insn = find_insn(file, sym->sec, sym->offset); \
120	insn && insn->offset < sym->offset + sym->len; \
121	insn = next_insn_same_sec(file, insn))
122
123	#define sym_for_each_insn_continue_reverse(file, sym, insn) \
124	for (insn = prev_insn_same_sec(file, insn); \
125	insn && insn->offset >= sym->offset; \
126	insn = prev_insn_same_sec(file, insn))
127
128	#define sec_for_each_insn_from(file, insn) \
129	for (; insn; insn = next_insn_same_sec(file, insn))
130
131	#define sec_for_each_insn_continue(file, insn) \
132	for (insn = next_insn_same_sec(file, insn); insn; \
133	insn = next_insn_same_sec(file, insn))
134
135	static inline struct symbol insn_call_dest(struct instruction insn)
136	{
137	if (insn->type == INSN_JUMP_DYNAMIC \|\|
138	insn->type == INSN_CALL_DYNAMIC)
139	return NULL;
140
141	return insn->_call_dest;
142	}
143
144	static inline struct reloc insn_jump_table(struct instruction insn)
145	{
146	if (insn->type == INSN_JUMP_DYNAMIC \|\|
147	insn->type == INSN_CALL_DYNAMIC)
148	return insn->_jump_table;
149
150	return NULL;
151	}
152
153	static bool is_jump_table_jump(struct instruction *insn)
154	{
155	struct alt_group *alt_group = insn->alt_group;
156
157	if (insn_jump_table(insn))
158	return true;
159
160	/* Retpoline alternative for a jump table? */
161	return alt_group && alt_group->orig_group &&
162	insn_jump_table(alt_group->orig_group->first_insn);
163	}
164
165	static bool is_sibling_call(struct instruction *insn)
166	{
167	/*
168	* Assume only STT_FUNC calls have jump-tables.
169	*/
170	if (insn_func(insn)) {
171	/* An indirect jump is either a sibling call or a jump to a table. */
172	if (insn->type == INSN_JUMP_DYNAMIC)
173	return !is_jump_table_jump(insn);
174	}
175
176	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
177	return (is_static_jump(insn) && insn_call_dest(insn));
178	}
179
180	/*
181	* Checks if a string ends with another.
182	*/
183	static bool str_ends_with(const char s, const char sub)
184	{
185	const int slen = strlen(s);
186	const int sublen = strlen(sub);
187
188	if (sublen > slen)
189	return 0;
190
191	return !memcmp(s + slen - sublen, sub, sublen);
192	}
193
194	/*
195	* Checks if a function is a Rust "noreturn" one.
196	*/
197	static bool is_rust_noreturn(const struct symbol *func)
198	{
199	/*
200	* If it does not start with "_R", then it is not a Rust symbol.
201	*/
202	if (strncmp(func->name, "_R", 2))
203	return false;
204
205	/*
206	* These are just heuristics -- we do not control the precise symbol
207	* name, due to the crate disambiguators (which depend on the compiler)
208	* as well as changes to the source code itself between versions (since
209	* these come from the Rust standard library).
210	*/
211	return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") \|\|
212	str_ends_with(func->name, "_4core6option13unwrap_failed") \|\|
213	str_ends_with(func->name, "_4core6result13unwrap_failed") \|\|
214	str_ends_with(func->name, "_4core9panicking5panic") \|\|
215	str_ends_with(func->name, "_4core9panicking9panic_fmt") \|\|
216	str_ends_with(func->name, "_4core9panicking14panic_explicit") \|\|
217	str_ends_with(func->name, "_4core9panicking14panic_nounwind") \|\|
218	str_ends_with(func->name, "_4core9panicking18panic_bounds_check") \|\|
219	str_ends_with(func->name, "_4core9panicking19assert_failed_inner") \|\|
220	str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") \|\|
221	strstr(func->name, "_4core9panicking11panic_const24panic_const_") \|\|
222	(strstr(func->name, "_4core5slice5index24slice_") &&
223	str_ends_with(func->name, "_fail"));
224	}
225
226	/*
227	* This checks to see if the given function is a "noreturn" function.
228	*
229	* For global functions which are outside the scope of this object file, we
230	* have to keep a manual list of them.
231	*
232	* For local functions, we have to detect them manually by simply looking for
233	* the lack of a return instruction.
234	*/
235	static bool __dead_end_function(struct objtool_file file, struct symbol func,
236	int recursion)
237	{
238	int i;
239	struct instruction *insn;
240	bool empty = true;
241
242	#define NORETURN(func) __stringify(func),
243	static const char * const global_noreturns[] = {
244	#include "noreturns.h"
245	};
246	#undef NORETURN
247
248	if (!func)
249	return false;
250
251	if (func->bind == STB_GLOBAL \|\| func->bind == STB_WEAK) {
252	if (is_rust_noreturn(func))
253	return true;
254
255	for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
256	if (!strcmp(func->name, global_noreturns[i]))
257	return true;
258	}
259
260	if (func->bind == STB_WEAK)
261	return false;
262
263	if (!func->len)
264	return false;
265
266	insn = find_insn(file, func->sec, func->offset);
267	if (!insn \|\| !insn_func(insn))
268	return false;
269
270	func_for_each_insn(file, func, insn) {
271	empty = false;
272
273	if (insn->type == INSN_RETURN)
274	return false;
275	}
276
277	if (empty)
278	return false;
279
280	/*
281	* A function can have a sibling call instead of a return. In that
282	* case, the function's dead-end status depends on whether the target
283	* of the sibling call returns.
284	*/
285	func_for_each_insn(file, func, insn) {
286	if (is_sibling_call(insn)) {
287	struct instruction *dest = insn->jump_dest;
288
289	if (!dest)
290	/* sibling call to another file */
291	return false;
292
293	/* local sibling call */
294	if (recursion == 5) {
295	/*
296	* Infinite recursion: two functions have
297	* sibling calls to each other. This is a very
298	* rare case. It means they aren't dead ends.
299	*/
300	return false;
301	}
302
303	return __dead_end_function(file, insn_func(dest), recursion+1);
304	}
305	}
306
307	return true;
308	}
309
310	static bool dead_end_function(struct objtool_file file, struct symbol func)
311	{
312	return __dead_end_function(file, func, 0);
313	}
314
315	static void init_cfi_state(struct cfi_state *cfi)
316	{
317	int i;
318
319	for (i = 0; i < CFI_NUM_REGS; i++) {
320	cfi->regs[i].base = CFI_UNDEFINED;
321	cfi->vals[i].base = CFI_UNDEFINED;
322	}
323	cfi->cfa.base = CFI_UNDEFINED;
324	cfi->drap_reg = CFI_UNDEFINED;
325	cfi->drap_offset = -1;
326	}
327
328	static void init_insn_state(struct objtool_file file, struct insn_state state,
329	struct section *sec)
330	{
331	memset(state, 0, sizeof(*state));
332	init_cfi_state(&state->cfi);
333
334	/*
335	* We need the full vmlinux for noinstr validation, otherwise we can
336	* not correctly determine insn_call_dest(insn)->sec (external symbols
337	* do not have a section).
338	*/
339	if (opts.link && opts.noinstr && sec)
340	state->noinstr = sec->noinstr;
341	}
342
343	static struct cfi_state *cfi_alloc(void)
344	{
345	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
346	if (!cfi) {
347	WARN("calloc failed");
348	exit(1);
349	}
350	nr_cfi++;
351	return cfi;
352	}
353
354	static int cfi_bits;
355	static struct hlist_head *cfi_hash;
356
357	static inline bool cficmp(struct cfi_state cfi1, struct cfi_state cfi2)
358	{
359	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
360	(void *)cfi2 + sizeof(cfi2->hash),
361	sizeof(struct cfi_state) - sizeof(struct hlist_node));
362	}
363
364	static inline u32 cfi_key(struct cfi_state *cfi)
365	{
366	return jhash((void *)cfi + sizeof(cfi->hash),
367	sizeof(*cfi) - sizeof(cfi->hash), 0);
368	}
369
370	static struct cfi_state cfi_hash_find_or_add(struct cfi_state cfi)
371	{
372	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
373	struct cfi_state *obj;
374
375	hlist_for_each_entry(obj, head, hash) {
376	if (!cficmp(cfi, obj)) {
377	nr_cfi_cache++;
378	return obj;
379	}
380	}
381
382	obj = cfi_alloc();
383	obj = cfi;
384	hlist_add_head(&obj->hash, head);
385
386	return obj;
387	}
388
389	static void cfi_hash_add(struct cfi_state *cfi)
390	{
391	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
392
393	hlist_add_head(&cfi->hash, head);
394	}
395
396	static void *cfi_hash_alloc(unsigned long size)
397	{
398	cfi_bits = max(10, ilog2(size));
399	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
400	PROT_READ\|PROT_WRITE,
401	MAP_PRIVATE\|MAP_ANON, -1, 0);
402	if (cfi_hash == (void *)-1L) {
403	WARN("mmap fail cfi_hash");
404	cfi_hash = NULL;
405	} else if (opts.stats) {
406	printf("cfi_bits: %d\n", cfi_bits);
407	}
408
409	return cfi_hash;
410	}
411
412	static unsigned long nr_insns;
413	static unsigned long nr_insns_visited;
414
415	/*
416	* Call the arch-specific instruction decoder for all the instructions and add
417	* them to the global instruction list.
418	*/
419	static int decode_instructions(struct objtool_file *file)
420	{
421	struct section *sec;
422	struct symbol *func;
423	unsigned long offset;
424	struct instruction *insn;
425	int ret;
426
427	for_each_sec(file, sec) {
428	struct instruction *insns = NULL;
429	u8 prev_len = 0;
430	u8 idx = 0;
431
432	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
433	continue;
434
435	if (strcmp(sec->name, ".altinstr_replacement") &&
436	strcmp(sec->name, ".altinstr_aux") &&
437	strncmp(sec->name, ".discard.", 9))
438	sec->text = true;
439
440	if (!strcmp(sec->name, ".noinstr.text") \|\|
441	!strcmp(sec->name, ".entry.text") \|\|
442	!strcmp(sec->name, ".cpuidle.text") \|\|
443	!strncmp(sec->name, ".text..__x86.", 13))
444	sec->noinstr = true;
445
446	/*
447	* .init.text code is ran before userspace and thus doesn't
448	* strictly need retpolines, except for modules which are
449	* loaded late, they very much do need retpoline in their
450	* .init.text
451	*/
452	if (!strcmp(sec->name, ".init.text") && !opts.module)
453	sec->init = true;
454
455	for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
456	if (!insns \|\| idx == INSN_CHUNK_MAX) {
457	insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
458	if (!insns) {
459	WARN("malloc failed");
460	return -1;
461	}
462	idx = 0;
463	} else {
464	idx++;
465	}
466	insn = &insns[idx];
467	insn->idx = idx;
468
469	INIT_LIST_HEAD(&insn->call_node);
470	insn->sec = sec;
471	insn->offset = offset;
472	insn->prev_len = prev_len;
473
474	ret = arch_decode_instruction(file, sec, offset,
475	sec->sh.sh_size - offset,
476	insn);
477	if (ret)
478	return ret;
479
480	prev_len = insn->len;
481
482	/*
483	* By default, "ud2" is a dead end unless otherwise
484	* annotated, because GCC 7 inserts it for certain
485	* divide-by-zero cases.
486	*/
487	if (insn->type == INSN_BUG)
488	insn->dead_end = true;
489
490	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
491	nr_insns++;
492	}
493
494	// printf("%s: last chunk used: %d\n", sec->name, (int)idx);
495
496	sec_for_each_sym(sec, func) {
497	if (func->type != STT_NOTYPE && func->type != STT_FUNC)
498	continue;
499
500	if (func->offset == sec->sh.sh_size) {
501	/* Heuristic: likely an "end" symbol */
502	if (func->type == STT_NOTYPE)
503	continue;
504	WARN("%s(): STT_FUNC at end of section",
505	func->name);
506	return -1;
507	}
508
509	if (func->embedded_insn \|\| func->alias != func)
510	continue;
511
512	if (!find_insn(file, sec, func->offset)) {
513	WARN("%s(): can't find starting instruction",
514	func->name);
515	return -1;
516	}
517
518	sym_for_each_insn(file, func, insn) {
519	insn->sym = func;
520	if (func->type == STT_FUNC &&
521	insn->type == INSN_ENDBR &&
522	list_empty(&insn->call_node)) {
523	if (insn->offset == func->offset) {
524	list_add_tail(&insn->call_node, &file->endbr_list);
525	file->nr_endbr++;
526	} else {
527	file->nr_endbr_int++;
528	}
529	}
530	}
531	}
532	}
533
534	if (opts.stats)
535	printf("nr_insns: %lu\n", nr_insns);
536
537	return 0;
538	}
539
540	/*
541	* Read the pv_ops[] .data table to find the static initialized values.
542	*/
543	static int add_pv_ops(struct objtool_file file, const char symname)
544	{
545	struct symbol sym, func;
546	unsigned long off, end;
547	struct reloc *reloc;
548	int idx;
549
550	sym = find_symbol_by_name(file->elf, symname);
551	if (!sym)
552	return 0;
553
554	off = sym->offset;
555	end = off + sym->len;
556	for (;;) {
557	reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
558	if (!reloc)
559	break;
560
561	func = reloc->sym;
562	if (func->type == STT_SECTION)
563	func = find_symbol_by_offset(reloc->sym->sec,
564	reloc_addend(reloc));
565
566	idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
567
568	objtool_pv_add(file, idx, func);
569
570	off = reloc_offset(reloc) + 1;
571	if (off > end)
572	break;
573	}
574
575	return 0;
576	}
577
578	/*
579	* Allocate and initialize file->pv_ops[].
580	*/
581	static int init_pv_ops(struct objtool_file *file)
582	{
583	static const char *pv_ops_tables[] = {
584	"pv_ops",
585	"xen_cpu_ops",
586	"xen_irq_ops",
587	"xen_mmu_ops",
588	NULL,
589	};
590	const char *pv_ops;
591	struct symbol *sym;
592	int idx, nr;
593
594	if (!opts.noinstr)
595	return 0;
596
597	file->pv_ops = NULL;
598
599	sym = find_symbol_by_name(file->elf, "pv_ops");
600	if (!sym)
601	return 0;
602
603	nr = sym->len / sizeof(unsigned long);
604	file->pv_ops = calloc(sizeof(struct pv_state), nr);
605	if (!file->pv_ops)
606	return -1;
607
608	for (idx = 0; idx < nr; idx++)
609	INIT_LIST_HEAD(&file->pv_ops[idx].targets);
610
611	for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
612	add_pv_ops(file, pv_ops);
613
614	return 0;
615	}
616
617	static struct instruction find_last_insn(struct objtool_file file,
618	struct section *sec)
619	{
620	struct instruction *insn = NULL;
621	unsigned int offset;
622	unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
623
624	for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
625	insn = find_insn(file, sec, offset);
626
627	return insn;
628	}
629
630	/*
631	* Mark "ud2" instructions and manually annotated dead ends.
632	*/
633	static int add_dead_ends(struct objtool_file *file)
634	{
635	struct section *rsec;
636	struct reloc *reloc;
637	struct instruction *insn;
638	uint64_t offset;
639
640	/*
641	* Check for manually annotated dead ends.
642	*/
643	rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
644	if (!rsec)
645	goto reachable;
646
647	for_each_reloc(rsec, reloc) {
648	if (reloc->sym->type == STT_SECTION) {
649	offset = reloc_addend(reloc);
650	} else if (reloc->sym->local_label) {
651	offset = reloc->sym->offset;
652	} else {
653	WARN("unexpected relocation symbol type in %s", rsec->name);
654	return -1;
655	}
656
657	insn = find_insn(file, reloc->sym->sec, offset);
658	if (insn)
659	insn = prev_insn_same_sec(file, insn);
660	else if (offset == reloc->sym->sec->sh.sh_size) {
661	insn = find_last_insn(file, reloc->sym->sec);
662	if (!insn) {
663	WARN("can't find unreachable insn at %s+0x%" PRIx64,
664	reloc->sym->sec->name, offset);
665	return -1;
666	}
667	} else {
668	WARN("can't find unreachable insn at %s+0x%" PRIx64,
669	reloc->sym->sec->name, offset);
670	return -1;
671	}
672
673	insn->dead_end = true;
674	}
675
676	reachable:
677	/*
678	* These manually annotated reachable checks are needed for GCC 4.4,
679	* where the Linux unreachable() macro isn't supported. In that case
680	* GCC doesn't know the "ud2" is fatal, so it generates code as if it's
681	* not a dead end.
682	*/
683	rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
684	if (!rsec)
685	return 0;
686
687	for_each_reloc(rsec, reloc) {
688	if (reloc->sym->type == STT_SECTION) {
689	offset = reloc_addend(reloc);
690	} else if (reloc->sym->local_label) {
691	offset = reloc->sym->offset;
692	} else {
693	WARN("unexpected relocation symbol type in %s", rsec->name);
694	return -1;
695	}
696
697	insn = find_insn(file, reloc->sym->sec, offset);
698	if (insn)
699	insn = prev_insn_same_sec(file, insn);
700	else if (offset == reloc->sym->sec->sh.sh_size) {
701	insn = find_last_insn(file, reloc->sym->sec);
702	if (!insn) {
703	WARN("can't find reachable insn at %s+0x%" PRIx64,
704	reloc->sym->sec->name, offset);
705	return -1;
706	}
707	} else {
708	WARN("can't find reachable insn at %s+0x%" PRIx64,
709	reloc->sym->sec->name, offset);
710	return -1;
711	}
712
713	insn->dead_end = false;
714	}
715
716	return 0;
717	}
718
719	static int create_static_call_sections(struct objtool_file *file)
720	{
721	struct static_call_site *site;
722	struct section *sec;
723	struct instruction *insn;
724	struct symbol *key_sym;
725	char key_name, tmp;
726	int idx;
727
728	sec = find_section_by_name(file->elf, ".static_call_sites");
729	if (sec) {
730	INIT_LIST_HEAD(&file->static_call_list);
731	WARN("file already has .static_call_sites section, skipping");
732	return 0;
733	}
734
735	if (list_empty(&file->static_call_list))
736	return 0;
737
738	idx = 0;
739	list_for_each_entry(insn, &file->static_call_list, call_node)
740	idx++;
741
742	sec = elf_create_section_pair(file->elf, ".static_call_sites",
743	sizeof(site), idx, idx 2);
744	if (!sec)
745	return -1;
746
747	/* Allow modules to modify the low bits of static_call_site::key */
748	sec->sh.sh_flags \|= SHF_WRITE;
749
750	idx = 0;
751	list_for_each_entry(insn, &file->static_call_list, call_node) {
752
753	/* populate reloc for 'addr' */
754	if (!elf_init_reloc_text_sym(file->elf, sec,
755	idx * sizeof(site), idx 2,
756	insn->sec, insn->offset))
757	return -1;
758
759	/* find key symbol */
760	key_name = strdup(insn_call_dest(insn)->name);
761	if (!key_name) {
762	perror("strdup");
763	return -1;
764	}
765	if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
766	STATIC_CALL_TRAMP_PREFIX_LEN)) {
767	WARN("static_call: trampoline name malformed: %s", key_name);
768	free(key_name);
769	return -1;
770	}
771	tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
772	memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
773
774	key_sym = find_symbol_by_name(file->elf, tmp);
775	if (!key_sym) {
776	if (!opts.module) {
777	WARN("static_call: can't find static_call_key symbol: %s", tmp);
778	free(key_name);
779	return -1;
780	}
781
782	/*
783	* For modules(), the key might not be exported, which
784	* means the module can make static calls but isn't
785	* allowed to change them.
786	*
787	* In that case we temporarily set the key to be the
788	* trampoline address. This is fixed up in
789	* static_call_add_module().
790	*/
791	key_sym = insn_call_dest(insn);
792	}
793	free(key_name);
794
795	/* populate reloc for 'key' */
796	if (!elf_init_reloc_data_sym(file->elf, sec,
797	idx * sizeof(*site) + 4,
798	(idx * 2) + 1, key_sym,
799	is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
800	return -1;
801
802	idx++;
803	}
804
805	return 0;
806	}
807
808	static int create_retpoline_sites_sections(struct objtool_file *file)
809	{
810	struct instruction *insn;
811	struct section *sec;
812	int idx;
813
814	sec = find_section_by_name(file->elf, ".retpoline_sites");
815	if (sec) {
816	WARN("file already has .retpoline_sites, skipping");
817	return 0;
818	}
819
820	idx = 0;
821	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
822	idx++;
823
824	if (!idx)
825	return 0;
826
827	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
828	sizeof(int), idx, idx);
829	if (!sec)
830	return -1;
831
832	idx = 0;
833	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
834
835	if (!elf_init_reloc_text_sym(file->elf, sec,
836	idx * sizeof(int), idx,
837	insn->sec, insn->offset))
838	return -1;
839
840	idx++;
841	}
842
843	return 0;
844	}
845
846	static int create_return_sites_sections(struct objtool_file *file)
847	{
848	struct instruction *insn;
849	struct section *sec;
850	int idx;
851
852	sec = find_section_by_name(file->elf, ".return_sites");
853	if (sec) {
854	WARN("file already has .return_sites, skipping");
855	return 0;
856	}
857
858	idx = 0;
859	list_for_each_entry(insn, &file->return_thunk_list, call_node)
860	idx++;
861
862	if (!idx)
863	return 0;
864
865	sec = elf_create_section_pair(file->elf, ".return_sites",
866	sizeof(int), idx, idx);
867	if (!sec)
868	return -1;
869
870	idx = 0;
871	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
872
873	if (!elf_init_reloc_text_sym(file->elf, sec,
874	idx * sizeof(int), idx,
875	insn->sec, insn->offset))
876	return -1;
877
878	idx++;
879	}
880
881	return 0;
882	}
883
884	static int create_ibt_endbr_seal_sections(struct objtool_file *file)
885	{
886	struct instruction *insn;
887	struct section *sec;
888	int idx;
889
890	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
891	if (sec) {
892	WARN("file already has .ibt_endbr_seal, skipping");
893	return 0;
894	}
895
896	idx = 0;
897	list_for_each_entry(insn, &file->endbr_list, call_node)
898	idx++;
899
900	if (opts.stats) {
901	printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
902	printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
903	printf("ibt: superfluous ENDBR: %d\n", idx);
904	}
905
906	if (!idx)
907	return 0;
908
909	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
910	sizeof(int), idx, idx);
911	if (!sec)
912	return -1;
913
914	idx = 0;
915	list_for_each_entry(insn, &file->endbr_list, call_node) {
916
917	int site = (int )sec->data->d_buf + idx;
918	struct symbol *sym = insn->sym;
919	*site = 0;
920
921	if (opts.module && sym && sym->type == STT_FUNC &&
922	insn->offset == sym->offset &&
923	(!strcmp(sym->name, "init_module") \|\|
924	!strcmp(sym->name, "cleanup_module")))
925	WARN("%s(): not an indirect call target", sym->name);
926
927	if (!elf_init_reloc_text_sym(file->elf, sec,
928	idx * sizeof(int), idx,
929	insn->sec, insn->offset))
930	return -1;
931
932	idx++;
933	}
934
935	return 0;
936	}
937
938	static int create_cfi_sections(struct objtool_file *file)
939	{
940	struct section *sec;
941	struct symbol *sym;
942	int idx;
943
944	sec = find_section_by_name(file->elf, ".cfi_sites");
945	if (sec) {
946	INIT_LIST_HEAD(&file->call_list);
947	WARN("file already has .cfi_sites section, skipping");
948	return 0;
949	}
950
951	idx = 0;
952	for_each_sym(file, sym) {
953	if (sym->type != STT_FUNC)
954	continue;
955
956	if (strncmp(sym->name, "__cfi_", 6))
957	continue;
958
959	idx++;
960	}
961
962	sec = elf_create_section_pair(file->elf, ".cfi_sites",
963	sizeof(unsigned int), idx, idx);
964	if (!sec)
965	return -1;
966
967	idx = 0;
968	for_each_sym(file, sym) {
969	if (sym->type != STT_FUNC)
970	continue;
971
972	if (strncmp(sym->name, "__cfi_", 6))
973	continue;
974
975	if (!elf_init_reloc_text_sym(file->elf, sec,
976	idx * sizeof(unsigned int), idx,
977	sym->sec, sym->offset))
978	return -1;
979
980	idx++;
981	}
982
983	return 0;
984	}
985
986	static int create_mcount_loc_sections(struct objtool_file *file)
987	{
988	size_t addr_size = elf_addr_size(file->elf);
989	struct instruction *insn;
990	struct section *sec;
991	int idx;
992
993	sec = find_section_by_name(file->elf, "__mcount_loc");
994	if (sec) {
995	INIT_LIST_HEAD(&file->mcount_loc_list);
996	WARN("file already has __mcount_loc section, skipping");
997	return 0;
998	}
999
1000	if (list_empty(&file->mcount_loc_list))
1001	return 0;
1002
1003	idx = 0;
1004	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
1005	idx++;
1006
1007	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
1008	idx, idx);
1009	if (!sec)
1010	return -1;
1011
1012	sec->sh.sh_addralign = addr_size;
1013
1014	idx = 0;
1015	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
1016
1017	struct reloc *reloc;
1018
1019	reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
1020	insn->sec, insn->offset);
1021	if (!reloc)
1022	return -1;
1023
1024	set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
1025
1026	idx++;
1027	}
1028
1029	return 0;
1030	}
1031
1032	static int create_direct_call_sections(struct objtool_file *file)
1033	{
1034	struct instruction *insn;
1035	struct section *sec;
1036	int idx;
1037
1038	sec = find_section_by_name(file->elf, ".call_sites");
1039	if (sec) {
1040	INIT_LIST_HEAD(&file->call_list);
1041	WARN("file already has .call_sites section, skipping");
1042	return 0;
1043	}
1044
1045	if (list_empty(&file->call_list))
1046	return 0;
1047
1048	idx = 0;
1049	list_for_each_entry(insn, &file->call_list, call_node)
1050	idx++;
1051
1052	sec = elf_create_section_pair(file->elf, ".call_sites",
1053	sizeof(unsigned int), idx, idx);
1054	if (!sec)
1055	return -1;
1056
1057	idx = 0;
1058	list_for_each_entry(insn, &file->call_list, call_node) {
1059
1060	if (!elf_init_reloc_text_sym(file->elf, sec,
1061	idx * sizeof(unsigned int), idx,
1062	insn->sec, insn->offset))
1063	return -1;
1064
1065	idx++;
1066	}
1067
1068	return 0;
1069	}
1070
1071	/*
1072	* Warnings shouldn't be reported for ignored functions.
1073	*/
1074	static void add_ignores(struct objtool_file *file)
1075	{
1076	struct instruction *insn;
1077	struct section *rsec;
1078	struct symbol *func;
1079	struct reloc *reloc;
1080
1081	rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1082	if (!rsec)
1083	return;
1084
1085	for_each_reloc(rsec, reloc) {
1086	switch (reloc->sym->type) {
1087	case STT_FUNC:
1088	func = reloc->sym;
1089	break;
1090
1091	case STT_SECTION:
1092	func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1093	if (!func)
1094	continue;
1095	break;
1096
1097	default:
1098	WARN("unexpected relocation symbol type in %s: %d",
1099	rsec->name, reloc->sym->type);
1100	continue;
1101	}
1102
1103	func_for_each_insn(file, func, insn)
1104	insn->ignore = true;
1105	}
1106	}
1107
1108	/*
1109	* This is a whitelist of functions that is allowed to be called with AC set.
1110	* The list is meant to be minimal and only contains compiler instrumentation
1111	* ABI and a few functions used to implement *_{to,from}_user() functions.
1112	*
1113	* These functions must not directly change AC, but may PUSHF/POPF.
1114	*/
1115	static const char *uaccess_safe_builtin[] = {
1116	/* KASAN */
1117	"kasan_report",
1118	"kasan_check_range",
1119	/* KASAN out-of-line */
1120	"__asan_loadN_noabort",
1121	"__asan_load1_noabort",
1122	"__asan_load2_noabort",
1123	"__asan_load4_noabort",
1124	"__asan_load8_noabort",
1125	"__asan_load16_noabort",
1126	"__asan_storeN_noabort",
1127	"__asan_store1_noabort",
1128	"__asan_store2_noabort",
1129	"__asan_store4_noabort",
1130	"__asan_store8_noabort",
1131	"__asan_store16_noabort",
1132	"__kasan_check_read",
1133	"__kasan_check_write",
1134	/* KASAN in-line */
1135	"__asan_report_load_n_noabort",
1136	"__asan_report_load1_noabort",
1137	"__asan_report_load2_noabort",
1138	"__asan_report_load4_noabort",
1139	"__asan_report_load8_noabort",
1140	"__asan_report_load16_noabort",
1141	"__asan_report_store_n_noabort",
1142	"__asan_report_store1_noabort",
1143	"__asan_report_store2_noabort",
1144	"__asan_report_store4_noabort",
1145	"__asan_report_store8_noabort",
1146	"__asan_report_store16_noabort",
1147	/* KCSAN */
1148	"__kcsan_check_access",
1149	"__kcsan_mb",
1150	"__kcsan_wmb",
1151	"__kcsan_rmb",
1152	"__kcsan_release",
1153	"kcsan_found_watchpoint",
1154	"kcsan_setup_watchpoint",
1155	"kcsan_check_scoped_accesses",
1156	"kcsan_disable_current",
1157	"kcsan_enable_current_nowarn",
1158	/* KCSAN/TSAN */
1159	"__tsan_func_entry",
1160	"__tsan_func_exit",
1161	"__tsan_read_range",
1162	"__tsan_write_range",
1163	"__tsan_read1",
1164	"__tsan_read2",
1165	"__tsan_read4",
1166	"__tsan_read8",
1167	"__tsan_read16",
1168	"__tsan_write1",
1169	"__tsan_write2",
1170	"__tsan_write4",
1171	"__tsan_write8",
1172	"__tsan_write16",
1173	"__tsan_read_write1",
1174	"__tsan_read_write2",
1175	"__tsan_read_write4",
1176	"__tsan_read_write8",
1177	"__tsan_read_write16",
1178	"__tsan_volatile_read1",
1179	"__tsan_volatile_read2",
1180	"__tsan_volatile_read4",
1181	"__tsan_volatile_read8",
1182	"__tsan_volatile_read16",
1183	"__tsan_volatile_write1",
1184	"__tsan_volatile_write2",
1185	"__tsan_volatile_write4",
1186	"__tsan_volatile_write8",
1187	"__tsan_volatile_write16",
1188	"__tsan_atomic8_load",
1189	"__tsan_atomic16_load",
1190	"__tsan_atomic32_load",
1191	"__tsan_atomic64_load",
1192	"__tsan_atomic8_store",
1193	"__tsan_atomic16_store",
1194	"__tsan_atomic32_store",
1195	"__tsan_atomic64_store",
1196	"__tsan_atomic8_exchange",
1197	"__tsan_atomic16_exchange",
1198	"__tsan_atomic32_exchange",
1199	"__tsan_atomic64_exchange",
1200	"__tsan_atomic8_fetch_add",
1201	"__tsan_atomic16_fetch_add",
1202	"__tsan_atomic32_fetch_add",
1203	"__tsan_atomic64_fetch_add",
1204	"__tsan_atomic8_fetch_sub",
1205	"__tsan_atomic16_fetch_sub",
1206	"__tsan_atomic32_fetch_sub",
1207	"__tsan_atomic64_fetch_sub",
1208	"__tsan_atomic8_fetch_and",
1209	"__tsan_atomic16_fetch_and",
1210	"__tsan_atomic32_fetch_and",
1211	"__tsan_atomic64_fetch_and",
1212	"__tsan_atomic8_fetch_or",
1213	"__tsan_atomic16_fetch_or",
1214	"__tsan_atomic32_fetch_or",
1215	"__tsan_atomic64_fetch_or",
1216	"__tsan_atomic8_fetch_xor",
1217	"__tsan_atomic16_fetch_xor",
1218	"__tsan_atomic32_fetch_xor",
1219	"__tsan_atomic64_fetch_xor",
1220	"__tsan_atomic8_fetch_nand",
1221	"__tsan_atomic16_fetch_nand",
1222	"__tsan_atomic32_fetch_nand",
1223	"__tsan_atomic64_fetch_nand",
1224	"__tsan_atomic8_compare_exchange_strong",
1225	"__tsan_atomic16_compare_exchange_strong",
1226	"__tsan_atomic32_compare_exchange_strong",
1227	"__tsan_atomic64_compare_exchange_strong",
1228	"__tsan_atomic8_compare_exchange_weak",
1229	"__tsan_atomic16_compare_exchange_weak",
1230	"__tsan_atomic32_compare_exchange_weak",
1231	"__tsan_atomic64_compare_exchange_weak",
1232	"__tsan_atomic8_compare_exchange_val",
1233	"__tsan_atomic16_compare_exchange_val",
1234	"__tsan_atomic32_compare_exchange_val",
1235	"__tsan_atomic64_compare_exchange_val",
1236	"__tsan_atomic_thread_fence",
1237	"__tsan_atomic_signal_fence",
1238	"__tsan_unaligned_read16",
1239	"__tsan_unaligned_write16",
1240	/* KCOV */
1241	"write_comp_data",
1242	"check_kcov_mode",
1243	"__sanitizer_cov_trace_pc",
1244	"__sanitizer_cov_trace_const_cmp1",
1245	"__sanitizer_cov_trace_const_cmp2",
1246	"__sanitizer_cov_trace_const_cmp4",
1247	"__sanitizer_cov_trace_const_cmp8",
1248	"__sanitizer_cov_trace_cmp1",
1249	"__sanitizer_cov_trace_cmp2",
1250	"__sanitizer_cov_trace_cmp4",
1251	"__sanitizer_cov_trace_cmp8",
1252	"__sanitizer_cov_trace_switch",
1253	/* KMSAN */
1254	"kmsan_copy_to_user",
1255	"kmsan_disable_current",
1256	"kmsan_enable_current",
1257	"kmsan_report",
1258	"kmsan_unpoison_entry_regs",
1259	"kmsan_unpoison_memory",
1260	"__msan_chain_origin",
1261	"__msan_get_context_state",
1262	"__msan_instrument_asm_store",
1263	"__msan_metadata_ptr_for_load_1",
1264	"__msan_metadata_ptr_for_load_2",
1265	"__msan_metadata_ptr_for_load_4",
1266	"__msan_metadata_ptr_for_load_8",
1267	"__msan_metadata_ptr_for_load_n",
1268	"__msan_metadata_ptr_for_store_1",
1269	"__msan_metadata_ptr_for_store_2",
1270	"__msan_metadata_ptr_for_store_4",
1271	"__msan_metadata_ptr_for_store_8",
1272	"__msan_metadata_ptr_for_store_n",
1273	"__msan_poison_alloca",
1274	"__msan_warning",
1275	/* UBSAN */
1276	"ubsan_type_mismatch_common",
1277	"__ubsan_handle_type_mismatch",
1278	"__ubsan_handle_type_mismatch_v1",
1279	"__ubsan_handle_shift_out_of_bounds",
1280	"__ubsan_handle_load_invalid_value",
1281	/* STACKLEAK */
1282	"stackleak_track_stack",
1283	/* misc */
1284	"csum_partial_copy_generic",
1285	"copy_mc_fragile",
1286	"copy_mc_fragile_handle_tail",
1287	"copy_mc_enhanced_fast_string",
1288	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1289	"rep_stos_alternative",
1290	"rep_movs_alternative",
1291	"__copy_user_nocache",
1292	NULL
1293	};
1294
1295	static void add_uaccess_safe(struct objtool_file *file)
1296	{
1297	struct symbol *func;
1298	const char **name;
1299
1300	if (!opts.uaccess)
1301	return;
1302
1303	for (name = uaccess_safe_builtin; *name; name++) {
1304	func = find_symbol_by_name(file->elf, *name);
1305	if (!func)
1306	continue;
1307
1308	func->uaccess_safe = true;
1309	}
1310	}
1311
1312	/*
1313	* FIXME: For now, just ignore any alternatives which add retpolines. This is
1314	* a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1315	* But it at least allows objtool to understand the control flow around the
1316	* retpoline.
1317	*/
1318	static int add_ignore_alternatives(struct objtool_file *file)
1319	{
1320	struct section *rsec;
1321	struct reloc *reloc;
1322	struct instruction *insn;
1323
1324	rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1325	if (!rsec)
1326	return 0;
1327
1328	for_each_reloc(rsec, reloc) {
1329	if (reloc->sym->type != STT_SECTION) {
1330	WARN("unexpected relocation symbol type in %s", rsec->name);
1331	return -1;
1332	}
1333
1334	insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1335	if (!insn) {
1336	WARN("bad .discard.ignore_alts entry");
1337	return -1;
1338	}
1339
1340	insn->ignore_alts = true;
1341	}
1342
1343	return 0;
1344	}
1345
1346	/*
1347	* Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1348	* will be added to the .retpoline_sites section.
1349	*/
1350	__weak bool arch_is_retpoline(struct symbol *sym)
1351	{
1352	return false;
1353	}
1354
1355	/*
1356	* Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1357	* will be added to the .return_sites section.
1358	*/
1359	__weak bool arch_is_rethunk(struct symbol *sym)
1360	{
1361	return false;
1362	}
1363
1364	/*
1365	* Symbols that are embedded inside other instructions, because sometimes crazy
1366	* code exists. These are mostly ignored for validation purposes.
1367	*/
1368	__weak bool arch_is_embedded_insn(struct symbol *sym)
1369	{
1370	return false;
1371	}
1372
1373	static struct reloc insn_reloc(struct objtool_file file, struct instruction *insn)
1374	{
1375	struct reloc *reloc;
1376
1377	if (insn->no_reloc)
1378	return NULL;
1379
1380	if (!file)
1381	return NULL;
1382
1383	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1384	insn->offset, insn->len);
1385	if (!reloc) {
1386	insn->no_reloc = 1;
1387	return NULL;
1388	}
1389
1390	return reloc;
1391	}
1392
1393	static void remove_insn_ops(struct instruction *insn)
1394	{
1395	struct stack_op op, next;
1396
1397	for (op = insn->stack_ops; op; op = next) {
1398	next = op->next;
1399	free(op);
1400	}
1401	insn->stack_ops = NULL;
1402	}
1403
1404	static void annotate_call_site(struct objtool_file *file,
1405	struct instruction *insn, bool sibling)
1406	{
1407	struct reloc *reloc = insn_reloc(file, insn);
1408	struct symbol *sym = insn_call_dest(insn);
1409
1410	if (!sym)
1411	sym = reloc->sym;
1412
1413	/*
1414	* Alternative replacement code is just template code which is
1415	* sometimes copied to the original instruction. For now, don't
1416	* annotate it. (In the future we might consider annotating the
1417	* original instruction if/when it ever makes sense to do so.)
1418	*/
1419	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1420	return;
1421
1422	if (sym->static_call_tramp) {
1423	list_add_tail(&insn->call_node, &file->static_call_list);
1424	return;
1425	}
1426
1427	if (sym->retpoline_thunk) {
1428	list_add_tail(&insn->call_node, &file->retpoline_call_list);
1429	return;
1430	}
1431
1432	/*
1433	* Many compilers cannot disable KCOV or sanitizer calls with a function
1434	* attribute so they need a little help, NOP out any such calls from
1435	* noinstr text.
1436	*/
1437	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1438	if (reloc)
1439	set_reloc_type(file->elf, reloc, R_NONE);
1440
1441	elf_write_insn(file->elf, insn->sec,
1442	insn->offset, insn->len,
1443	sibling ? arch_ret_insn(insn->len)
1444	: arch_nop_insn(insn->len));
1445
1446	insn->type = sibling ? INSN_RETURN : INSN_NOP;
1447
1448	if (sibling) {
1449	/*
1450	* We've replaced the tail-call JMP insn by two new
1451	* insn: RET; INT3, except we only have a single struct
1452	* insn here. Mark it retpoline_safe to avoid the SLS
1453	* warning, instead of adding another insn.
1454	*/
1455	insn->retpoline_safe = true;
1456	}
1457
1458	return;
1459	}
1460
1461	if (opts.mcount && sym->fentry) {
1462	if (sibling)
1463	WARN_INSN(insn, "tail call to __fentry__ !?!?");
1464	if (opts.mnop) {
1465	if (reloc)
1466	set_reloc_type(file->elf, reloc, R_NONE);
1467
1468	elf_write_insn(file->elf, insn->sec,
1469	insn->offset, insn->len,
1470	arch_nop_insn(insn->len));
1471
1472	insn->type = INSN_NOP;
1473	}
1474
1475	list_add_tail(&insn->call_node, &file->mcount_loc_list);
1476	return;
1477	}
1478
1479	if (insn->type == INSN_CALL && !insn->sec->init)
1480	list_add_tail(&insn->call_node, &file->call_list);
1481
1482	if (!sibling && dead_end_function(file, sym))
1483	insn->dead_end = true;
1484	}
1485
1486	static void add_call_dest(struct objtool_file file, struct instruction insn,
1487	struct symbol *dest, bool sibling)
1488	{
1489	insn->_call_dest = dest;
1490	if (!dest)
1491	return;
1492
1493	/*
1494	* Whatever stack impact regular CALLs have, should be undone
1495	* by the RETURN of the called function.
1496	*
1497	* Annotated intra-function calls retain the stack_ops but
1498	* are converted to JUMP, see read_intra_function_calls().
1499	*/
1500	remove_insn_ops(insn);
1501
1502	annotate_call_site(file, insn, sibling);
1503	}
1504
1505	static void add_retpoline_call(struct objtool_file file, struct instruction insn)
1506	{
1507	/*
1508	* Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1509	* so convert them accordingly.
1510	*/
1511	switch (insn->type) {
1512	case INSN_CALL:
1513	insn->type = INSN_CALL_DYNAMIC;
1514	break;
1515	case INSN_JUMP_UNCONDITIONAL:
1516	insn->type = INSN_JUMP_DYNAMIC;
1517	break;
1518	case INSN_JUMP_CONDITIONAL:
1519	insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1520	break;
1521	default:
1522	return;
1523	}
1524
1525	insn->retpoline_safe = true;
1526
1527	/*
1528	* Whatever stack impact regular CALLs have, should be undone
1529	* by the RETURN of the called function.
1530	*
1531	* Annotated intra-function calls retain the stack_ops but
1532	* are converted to JUMP, see read_intra_function_calls().
1533	*/
1534	remove_insn_ops(insn);
1535
1536	annotate_call_site(file, insn, false);
1537	}
1538
1539	static void add_return_call(struct objtool_file file, struct instruction insn, bool add)
1540	{
1541	/*
1542	* Return thunk tail calls are really just returns in disguise,
1543	* so convert them accordingly.
1544	*/
1545	insn->type = INSN_RETURN;
1546	insn->retpoline_safe = true;
1547
1548	if (add)
1549	list_add_tail(&insn->call_node, &file->return_thunk_list);
1550	}
1551
1552	static bool is_first_func_insn(struct objtool_file *file,
1553	struct instruction insn, struct symbol sym)
1554	{
1555	if (insn->offset == sym->offset)
1556	return true;
1557
1558	/* Allow direct CALL/JMP past ENDBR */
1559	if (opts.ibt) {
1560	struct instruction *prev = prev_insn_same_sym(file, insn);
1561
1562	if (prev && prev->type == INSN_ENDBR &&
1563	insn->offset == sym->offset + prev->len)
1564	return true;
1565	}
1566
1567	return false;
1568	}
1569
1570	/*
1571	* A sibling call is a tail-call to another symbol -- to differentiate from a
1572	* recursive tail-call which is to the same symbol.
1573	*/
1574	static bool jump_is_sibling_call(struct objtool_file *file,
1575	struct instruction from, struct instruction to)
1576	{
1577	struct symbol *fs = from->sym;
1578	struct symbol *ts = to->sym;
1579
1580	/* Not a sibling call if from/to a symbol hole */
1581	if (!fs \|\| !ts)
1582	return false;
1583
1584	/* Not a sibling call if not targeting the start of a symbol. */
1585	if (!is_first_func_insn(file, to, ts))
1586	return false;
1587
1588	/* Disallow sibling calls into STT_NOTYPE */
1589	if (ts->type == STT_NOTYPE)
1590	return false;
1591
1592	/* Must not be self to be a sibling */
1593	return fs->pfunc != ts->pfunc;
1594	}
1595
1596	/*
1597	* Find the destination instructions for all jumps.
1598	*/
1599	static int add_jump_destinations(struct objtool_file *file)
1600	{
1601	struct instruction insn, jump_dest;
1602	struct reloc *reloc;
1603	struct section *dest_sec;
1604	unsigned long dest_off;
1605
1606	for_each_insn(file, insn) {
1607	if (insn->jump_dest) {
1608	/*
1609	* handle_group_alt() may have previously set
1610	* 'jump_dest' for some alternatives.
1611	*/
1612	continue;
1613	}
1614	if (!is_static_jump(insn))
1615	continue;
1616
1617	reloc = insn_reloc(file, insn);
1618	if (!reloc) {
1619	dest_sec = insn->sec;
1620	dest_off = arch_jump_destination(insn);
1621	} else if (reloc->sym->type == STT_SECTION) {
1622	dest_sec = reloc->sym->sec;
1623	dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1624	} else if (reloc->sym->retpoline_thunk) {
1625	add_retpoline_call(file, insn);
1626	continue;
1627	} else if (reloc->sym->return_thunk) {
1628	add_return_call(file, insn, true);
1629	continue;
1630	} else if (insn_func(insn)) {
1631	/*
1632	* External sibling call or internal sibling call with
1633	* STT_FUNC reloc.
1634	*/
1635	add_call_dest(file, insn, reloc->sym, true);
1636	continue;
1637	} else if (reloc->sym->sec->idx) {
1638	dest_sec = reloc->sym->sec;
1639	dest_off = reloc->sym->sym.st_value +
1640	arch_dest_reloc_offset(reloc_addend(reloc));
1641	} else {
1642	/* non-func asm code jumping to another file */
1643	continue;
1644	}
1645
1646	jump_dest = find_insn(file, dest_sec, dest_off);
1647	if (!jump_dest) {
1648	struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1649
1650	/*
1651	* This is a special case for retbleed_untrain_ret().
1652	* It jumps to __x86_return_thunk(), but objtool
1653	* can't find the thunk's starting RET
1654	* instruction, because the RET is also in the
1655	* middle of another instruction. Objtool only
1656	* knows about the outer instruction.
1657	*/
1658	if (sym && sym->embedded_insn) {
1659	add_return_call(file, insn, false);
1660	continue;
1661	}
1662
1663	WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1664	dest_sec->name, dest_off);
1665	return -1;
1666	}
1667
1668	/*
1669	* An intra-TU jump in retpoline.o might not have a relocation
1670	* for its jump dest, in which case the above
1671	* add_{retpoline,return}_call() didn't happen.
1672	*/
1673	if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1674	if (jump_dest->sym->retpoline_thunk) {
1675	add_retpoline_call(file, insn);
1676	continue;
1677	}
1678	if (jump_dest->sym->return_thunk) {
1679	add_return_call(file, insn, true);
1680	continue;
1681	}
1682	}
1683
1684	/*
1685	* Cross-function jump.
1686	*/
1687	if (insn_func(insn) && insn_func(jump_dest) &&
1688	insn_func(insn) != insn_func(jump_dest)) {
1689
1690	/*
1691	* For GCC 8+, create parent/child links for any cold
1692	* subfunctions. This is _mostly_ redundant with a
1693	* similar initialization in read_symbols().
1694	*
1695	* If a function has aliases, we want the first such
1696	* function in the symbol table to be the subfunction's
1697	* parent. In that case we overwrite the
1698	* initialization done in read_symbols().
1699	*
1700	* However this code can't completely replace the
1701	* read_symbols() code because this doesn't detect the
1702	* case where the parent function's only reference to a
1703	* subfunction is through a jump table.
1704	*/
1705	if (!strstr(insn_func(insn)->name, ".cold") &&
1706	strstr(insn_func(jump_dest)->name, ".cold")) {
1707	insn_func(insn)->cfunc = insn_func(jump_dest);
1708	insn_func(jump_dest)->pfunc = insn_func(insn);
1709	}
1710	}
1711
1712	if (jump_is_sibling_call(file, insn, jump_dest)) {
1713	/*
1714	* Internal sibling call without reloc or with
1715	* STT_SECTION reloc.
1716	*/
1717	add_call_dest(file, insn, insn_func(jump_dest), true);
1718	continue;
1719	}
1720
1721	insn->jump_dest = jump_dest;
1722	}
1723
1724	return 0;
1725	}
1726
1727	static struct symbol find_call_destination(struct section sec, unsigned long offset)
1728	{
1729	struct symbol *call_dest;
1730
1731	call_dest = find_func_by_offset(sec, offset);
1732	if (!call_dest)
1733	call_dest = find_symbol_by_offset(sec, offset);
1734
1735	return call_dest;
1736	}
1737
1738	/*
1739	* Find the destination instructions for all calls.
1740	*/
1741	static int add_call_destinations(struct objtool_file *file)
1742	{
1743	struct instruction *insn;
1744	unsigned long dest_off;
1745	struct symbol *dest;
1746	struct reloc *reloc;
1747
1748	for_each_insn(file, insn) {
1749	if (insn->type != INSN_CALL)
1750	continue;
1751
1752	reloc = insn_reloc(file, insn);
1753	if (!reloc) {
1754	dest_off = arch_jump_destination(insn);
1755	dest = find_call_destination(insn->sec, dest_off);
1756
1757	add_call_dest(file, insn, dest, false);
1758
1759	if (insn->ignore)
1760	continue;
1761
1762	if (!insn_call_dest(insn)) {
1763	WARN_INSN(insn, "unannotated intra-function call");
1764	return -1;
1765	}
1766
1767	if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1768	WARN_INSN(insn, "unsupported call to non-function");
1769	return -1;
1770	}
1771
1772	} else if (reloc->sym->type == STT_SECTION) {
1773	dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1774	dest = find_call_destination(reloc->sym->sec, dest_off);
1775	if (!dest) {
1776	WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1777	reloc->sym->sec->name, dest_off);
1778	return -1;
1779	}
1780
1781	add_call_dest(file, insn, dest, false);
1782
1783	} else if (reloc->sym->retpoline_thunk) {
1784	add_retpoline_call(file, insn);
1785
1786	} else
1787	add_call_dest(file, insn, reloc->sym, false);
1788	}
1789
1790	return 0;
1791	}
1792
1793	/*
1794	* The .alternatives section requires some extra special care over and above
1795	* other special sections because alternatives are patched in place.
1796	*/
1797	static int handle_group_alt(struct objtool_file *file,
1798	struct special_alt *special_alt,
1799	struct instruction *orig_insn,
1800	struct instruction **new_insn)
1801	{
1802	struct instruction last_new_insn = NULL, insn, *nop = NULL;
1803	struct alt_group orig_alt_group, new_alt_group;
1804	unsigned long dest_off;
1805
1806	orig_alt_group = orig_insn->alt_group;
1807	if (!orig_alt_group) {
1808	struct instruction *last_orig_insn = NULL;
1809
1810	orig_alt_group = malloc(sizeof(*orig_alt_group));
1811	if (!orig_alt_group) {
1812	WARN("malloc failed");
1813	return -1;
1814	}
1815	orig_alt_group->cfi = calloc(special_alt->orig_len,
1816	sizeof(struct cfi_state *));
1817	if (!orig_alt_group->cfi) {
1818	WARN("calloc failed");
1819	return -1;
1820	}
1821
1822	insn = orig_insn;
1823	sec_for_each_insn_from(file, insn) {
1824	if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1825	break;
1826
1827	insn->alt_group = orig_alt_group;
1828	last_orig_insn = insn;
1829	}
1830	orig_alt_group->orig_group = NULL;
1831	orig_alt_group->first_insn = orig_insn;
1832	orig_alt_group->last_insn = last_orig_insn;
1833	orig_alt_group->nop = NULL;
1834	} else {
1835	if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1836	orig_alt_group->first_insn->offset != special_alt->orig_len) {
1837	WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1838	orig_alt_group->last_insn->offset +
1839	orig_alt_group->last_insn->len -
1840	orig_alt_group->first_insn->offset,
1841	special_alt->orig_len);
1842	return -1;
1843	}
1844	}
1845
1846	new_alt_group = malloc(sizeof(*new_alt_group));
1847	if (!new_alt_group) {
1848	WARN("malloc failed");
1849	return -1;
1850	}
1851
1852	if (special_alt->new_len < special_alt->orig_len) {
1853	/*
1854	* Insert a fake nop at the end to make the replacement
1855	* alt_group the same size as the original. This is needed to
1856	* allow propagate_alt_cfi() to do its magic. When the last
1857	* instruction affects the stack, the instruction after it (the
1858	* nop) will propagate the new state to the shared CFI array.
1859	*/
1860	nop = malloc(sizeof(*nop));
1861	if (!nop) {
1862	WARN("malloc failed");
1863	return -1;
1864	}
1865	memset(nop, 0, sizeof(*nop));
1866
1867	nop->sec = special_alt->new_sec;
1868	nop->offset = special_alt->new_off + special_alt->new_len;
1869	nop->len = special_alt->orig_len - special_alt->new_len;
1870	nop->type = INSN_NOP;
1871	nop->sym = orig_insn->sym;
1872	nop->alt_group = new_alt_group;
1873	nop->ignore = orig_insn->ignore_alts;
1874	}
1875
1876	if (!special_alt->new_len) {
1877	*new_insn = nop;
1878	goto end;
1879	}
1880
1881	insn = *new_insn;
1882	sec_for_each_insn_from(file, insn) {
1883	struct reloc *alt_reloc;
1884
1885	if (insn->offset >= special_alt->new_off + special_alt->new_len)
1886	break;
1887
1888	last_new_insn = insn;
1889
1890	insn->ignore = orig_insn->ignore_alts;
1891	insn->sym = orig_insn->sym;
1892	insn->alt_group = new_alt_group;
1893
1894	/*
1895	* Since alternative replacement code is copy/pasted by the
1896	* kernel after applying relocations, generally such code can't
1897	* have relative-address relocation references to outside the
1898	* .altinstr_replacement section, unless the arch's
1899	* alternatives code can adjust the relative offsets
1900	* accordingly.
1901	*/
1902	alt_reloc = insn_reloc(file, insn);
1903	if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1904	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1905
1906	WARN_INSN(insn, "unsupported relocation in alternatives section");
1907	return -1;
1908	}
1909
1910	if (!is_static_jump(insn))
1911	continue;
1912
1913	if (!insn->immediate)
1914	continue;
1915
1916	dest_off = arch_jump_destination(insn);
1917	if (dest_off == special_alt->new_off + special_alt->new_len) {
1918	insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1919	if (!insn->jump_dest) {
1920	WARN_INSN(insn, "can't find alternative jump destination");
1921	return -1;
1922	}
1923	}
1924	}
1925
1926	if (!last_new_insn) {
1927	WARN_FUNC("can't find last new alternative instruction",
1928	special_alt->new_sec, special_alt->new_off);
1929	return -1;
1930	}
1931
1932	end:
1933	new_alt_group->orig_group = orig_alt_group;
1934	new_alt_group->first_insn = *new_insn;
1935	new_alt_group->last_insn = last_new_insn;
1936	new_alt_group->nop = nop;
1937	new_alt_group->cfi = orig_alt_group->cfi;
1938	return 0;
1939	}
1940
1941	/*
1942	* A jump table entry can either convert a nop to a jump or a jump to a nop.
1943	* If the original instruction is a jump, make the alt entry an effective nop
1944	* by just skipping the original instruction.
1945	*/
1946	static int handle_jump_alt(struct objtool_file *file,
1947	struct special_alt *special_alt,
1948	struct instruction *orig_insn,
1949	struct instruction **new_insn)
1950	{
1951	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1952	orig_insn->type != INSN_NOP) {
1953
1954	WARN_INSN(orig_insn, "unsupported instruction at jump label");
1955	return -1;
1956	}
1957
1958	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1959	struct reloc *reloc = insn_reloc(file, orig_insn);
1960
1961	if (reloc)
1962	set_reloc_type(file->elf, reloc, R_NONE);
1963	elf_write_insn(file->elf, orig_insn->sec,
1964	orig_insn->offset, orig_insn->len,
1965	arch_nop_insn(orig_insn->len));
1966	orig_insn->type = INSN_NOP;
1967	}
1968
1969	if (orig_insn->type == INSN_NOP) {
1970	if (orig_insn->len == 2)
1971	file->jl_nop_short++;
1972	else
1973	file->jl_nop_long++;
1974
1975	return 0;
1976	}
1977
1978	if (orig_insn->len == 2)
1979	file->jl_short++;
1980	else
1981	file->jl_long++;
1982
1983	*new_insn = next_insn_same_sec(file, orig_insn);
1984	return 0;
1985	}
1986
1987	/*
1988	* Read all the special sections which have alternate instructions which can be
1989	* patched in or redirected to at runtime. Each instruction having alternate
1990	* instruction(s) has them added to its insn->alts list, which will be
1991	* traversed in validate_branch().
1992	*/
1993	static int add_special_section_alts(struct objtool_file *file)
1994	{
1995	struct list_head special_alts;
1996	struct instruction orig_insn, new_insn;
1997	struct special_alt special_alt, tmp;
1998	struct alternative *alt;
1999	int ret;
2000
2001	ret = special_get_alts(file->elf, &special_alts);
2002	if (ret)
2003	return ret;
2004
2005	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
2006
2007	orig_insn = find_insn(file, special_alt->orig_sec,
2008	special_alt->orig_off);
2009	if (!orig_insn) {
2010	WARN_FUNC("special: can't find orig instruction",
2011	special_alt->orig_sec, special_alt->orig_off);
2012	ret = -1;
2013	goto out;
2014	}
2015
2016	new_insn = NULL;
2017	if (!special_alt->group \|\| special_alt->new_len) {
2018	new_insn = find_insn(file, special_alt->new_sec,
2019	special_alt->new_off);
2020	if (!new_insn) {
2021	WARN_FUNC("special: can't find new instruction",
2022	special_alt->new_sec,
2023	special_alt->new_off);
2024	ret = -1;
2025	goto out;
2026	}
2027	}
2028
2029	if (special_alt->group) {
2030	if (!special_alt->orig_len) {
2031	WARN_INSN(orig_insn, "empty alternative entry");
2032	continue;
2033	}
2034
2035	ret = handle_group_alt(file, special_alt, orig_insn,
2036	&new_insn);
2037	if (ret)
2038	goto out;
2039	} else if (special_alt->jump_or_nop) {
2040	ret = handle_jump_alt(file, special_alt, orig_insn,
2041	&new_insn);
2042	if (ret)
2043	goto out;
2044	}
2045
2046	alt = malloc(sizeof(*alt));
2047	if (!alt) {
2048	WARN("malloc failed");
2049	ret = -1;
2050	goto out;
2051	}
2052
2053	alt->insn = new_insn;
2054	alt->skip_orig = special_alt->skip_orig;
2055	orig_insn->ignore_alts \|= special_alt->skip_alt;
2056	alt->next = orig_insn->alts;
2057	orig_insn->alts = alt;
2058
2059	list_del(&special_alt->list);
2060	free(special_alt);
2061	}
2062
2063	if (opts.stats) {
2064	printf("jl\\\tNOP\tJMP\n");
2065	printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2066	printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2067	}
2068
2069	out:
2070	return ret;
2071	}
2072
2073	static int add_jump_table(struct objtool_file file, struct instruction insn,
2074	struct reloc *next_table)
2075	{
2076	struct symbol *pfunc = insn_func(insn)->pfunc;
2077	struct reloc *table = insn_jump_table(insn);
2078	struct instruction *dest_insn;
2079	unsigned int prev_offset = 0;
2080	struct reloc *reloc = table;
2081	struct alternative *alt;
2082
2083	/*
2084	* Each @reloc is a switch table relocation which points to the target
2085	* instruction.
2086	*/
2087	for_each_reloc_from(table->sec, reloc) {
2088
2089	/* Check for the end of the table: */
2090	if (reloc != table && reloc == next_table)
2091	break;
2092
2093	/* Make sure the table entries are consecutive: */
2094	if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2095	break;
2096
2097	/* Detect function pointers from contiguous objects: */
2098	if (reloc->sym->sec == pfunc->sec &&
2099	reloc_addend(reloc) == pfunc->offset)
2100	break;
2101
2102	dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2103	if (!dest_insn)
2104	break;
2105
2106	/* Make sure the destination is in the same function: */
2107	if (!insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != pfunc)
2108	break;
2109
2110	alt = malloc(sizeof(*alt));
2111	if (!alt) {
2112	WARN("malloc failed");
2113	return -1;
2114	}
2115
2116	alt->insn = dest_insn;
2117	alt->next = insn->alts;
2118	insn->alts = alt;
2119	prev_offset = reloc_offset(reloc);
2120	}
2121
2122	if (!prev_offset) {
2123	WARN_INSN(insn, "can't find switch jump table");
2124	return -1;
2125	}
2126
2127	return 0;
2128	}
2129
2130	/*
2131	* find_jump_table() - Given a dynamic jump, find the switch jump table
2132	* associated with it.
2133	*/
2134	static struct reloc find_jump_table(struct objtool_file file,
2135	struct symbol *func,
2136	struct instruction *insn)
2137	{
2138	struct reloc *table_reloc;
2139	struct instruction dest_insn, orig_insn = insn;
2140
2141	/*
2142	* Backward search using the @first_jump_src links, these help avoid
2143	* much of the 'in between' code. Which avoids us getting confused by
2144	* it.
2145	*/
2146	for (;
2147	insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2148	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2149
2150	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2151	break;
2152
2153	/* allow small jumps within the range */
2154	if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2155	insn->jump_dest &&
2156	(insn->jump_dest->offset <= insn->offset \|\|
2157	insn->jump_dest->offset > orig_insn->offset))
2158	break;
2159
2160	table_reloc = arch_find_switch_table(file, insn);
2161	if (!table_reloc)
2162	continue;
2163	dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2164	if (!dest_insn \|\| !insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != func)
2165	continue;
2166
2167	return table_reloc;
2168	}
2169
2170	return NULL;
2171	}
2172
2173	/*
2174	* First pass: Mark the head of each jump table so that in the next pass,
2175	* we know when a given jump table ends and the next one starts.
2176	*/
2177	static void mark_func_jump_tables(struct objtool_file *file,
2178	struct symbol *func)
2179	{
2180	struct instruction insn, last = NULL;
2181	struct reloc *reloc;
2182
2183	func_for_each_insn(file, func, insn) {
2184	if (!last)
2185	last = insn;
2186
2187	/*
2188	* Store back-pointers for unconditional forward jumps such
2189	* that find_jump_table() can back-track using those and
2190	* avoid some potentially confusing code.
2191	*/
2192	if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2193	insn->offset > last->offset &&
2194	insn->jump_dest->offset > insn->offset &&
2195	!insn->jump_dest->first_jump_src) {
2196
2197	insn->jump_dest->first_jump_src = insn;
2198	last = insn->jump_dest;
2199	}
2200
2201	if (insn->type != INSN_JUMP_DYNAMIC)
2202	continue;
2203
2204	reloc = find_jump_table(file, func, insn);
2205	if (reloc)
2206	insn->_jump_table = reloc;
2207	}
2208	}
2209
2210	static int add_func_jump_tables(struct objtool_file *file,
2211	struct symbol *func)
2212	{
2213	struct instruction insn, insn_t1 = NULL, *insn_t2;
2214	int ret = 0;
2215
2216	func_for_each_insn(file, func, insn) {
2217	if (!insn_jump_table(insn))
2218	continue;
2219
2220	if (!insn_t1) {
2221	insn_t1 = insn;
2222	continue;
2223	}
2224
2225	insn_t2 = insn;
2226
2227	ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2228	if (ret)
2229	return ret;
2230
2231	insn_t1 = insn_t2;
2232	}
2233
2234	if (insn_t1)
2235	ret = add_jump_table(file, insn_t1, NULL);
2236
2237	return ret;
2238	}
2239
2240	/*
2241	* For some switch statements, gcc generates a jump table in the .rodata
2242	* section which contains a list of addresses within the function to jump to.
2243	* This finds these jump tables and adds them to the insn->alts lists.
2244	*/
2245	static int add_jump_table_alts(struct objtool_file *file)
2246	{
2247	struct symbol *func;
2248	int ret;
2249
2250	if (!file->rodata)
2251	return 0;
2252
2253	for_each_sym(file, func) {
2254	if (func->type != STT_FUNC)
2255	continue;
2256
2257	mark_func_jump_tables(file, func);
2258	ret = add_func_jump_tables(file, func);
2259	if (ret)
2260	return ret;
2261	}
2262
2263	return 0;
2264	}
2265
2266	static void set_func_state(struct cfi_state *state)
2267	{
2268	state->cfa = initial_func_cfi.cfa;
2269	memcpy(&state->regs, &initial_func_cfi.regs,
2270	CFI_NUM_REGS * sizeof(struct cfi_reg));
2271	state->stack_size = initial_func_cfi.cfa.offset;
2272	state->type = UNWIND_HINT_TYPE_CALL;
2273	}
2274
2275	static int read_unwind_hints(struct objtool_file *file)
2276	{
2277	struct cfi_state cfi = init_cfi;
2278	struct section *sec;
2279	struct unwind_hint *hint;
2280	struct instruction *insn;
2281	struct reloc *reloc;
2282	unsigned long offset;
2283	int i;
2284
2285	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2286	if (!sec)
2287	return 0;
2288
2289	if (!sec->rsec) {
2290	WARN("missing .rela.discard.unwind_hints section");
2291	return -1;
2292	}
2293
2294	if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2295	WARN("struct unwind_hint size mismatch");
2296	return -1;
2297	}
2298
2299	file->hints = true;
2300
2301	for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2302	hint = (struct unwind_hint *)sec->data->d_buf + i;
2303
2304	reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2305	if (!reloc) {
2306	WARN("can't find reloc for unwind_hints[%d]", i);
2307	return -1;
2308	}
2309
2310	if (reloc->sym->type == STT_SECTION) {
2311	offset = reloc_addend(reloc);
2312	} else if (reloc->sym->local_label) {
2313	offset = reloc->sym->offset;
2314	} else {
2315	WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2316	return -1;
2317	}
2318
2319	insn = find_insn(file, reloc->sym->sec, offset);
2320	if (!insn) {
2321	WARN("can't find insn for unwind_hints[%d]", i);
2322	return -1;
2323	}
2324
2325	insn->hint = true;
2326
2327	if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2328	insn->cfi = &force_undefined_cfi;
2329	continue;
2330	}
2331
2332	if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2333	insn->hint = false;
2334	insn->save = true;
2335	continue;
2336	}
2337
2338	if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2339	insn->restore = true;
2340	continue;
2341	}
2342
2343	if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2344	struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2345
2346	if (sym && sym->bind == STB_GLOBAL) {
2347	if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2348	WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2349	}
2350	}
2351	}
2352
2353	if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2354	insn->cfi = &func_cfi;
2355	continue;
2356	}
2357
2358	if (insn->cfi)
2359	cfi = *(insn->cfi);
2360
2361	if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2362	WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2363	return -1;
2364	}
2365
2366	cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2367	cfi.type = hint->type;
2368	cfi.signal = hint->signal;
2369
2370	insn->cfi = cfi_hash_find_or_add(&cfi);
2371	}
2372
2373	return 0;
2374	}
2375
2376	static int read_noendbr_hints(struct objtool_file *file)
2377	{
2378	struct instruction *insn;
2379	struct section *rsec;
2380	struct reloc *reloc;
2381
2382	rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2383	if (!rsec)
2384	return 0;
2385
2386	for_each_reloc(rsec, reloc) {
2387	insn = find_insn(file, reloc->sym->sec,
2388	reloc->sym->offset + reloc_addend(reloc));
2389	if (!insn) {
2390	WARN("bad .discard.noendbr entry");
2391	return -1;
2392	}
2393
2394	insn->noendbr = 1;
2395	}
2396
2397	return 0;
2398	}
2399
2400	static int read_retpoline_hints(struct objtool_file *file)
2401	{
2402	struct section *rsec;
2403	struct instruction *insn;
2404	struct reloc *reloc;
2405
2406	rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2407	if (!rsec)
2408	return 0;
2409
2410	for_each_reloc(rsec, reloc) {
2411	if (reloc->sym->type != STT_SECTION) {
2412	WARN("unexpected relocation symbol type in %s", rsec->name);
2413	return -1;
2414	}
2415
2416	insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2417	if (!insn) {
2418	WARN("bad .discard.retpoline_safe entry");
2419	return -1;
2420	}
2421
2422	if (insn->type != INSN_JUMP_DYNAMIC &&
2423	insn->type != INSN_CALL_DYNAMIC &&
2424	insn->type != INSN_RETURN &&
2425	insn->type != INSN_NOP) {
2426	WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2427	return -1;
2428	}
2429
2430	insn->retpoline_safe = true;
2431	}
2432
2433	return 0;
2434	}
2435
2436	static int read_instr_hints(struct objtool_file *file)
2437	{
2438	struct section *rsec;
2439	struct instruction *insn;
2440	struct reloc *reloc;
2441
2442	rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2443	if (!rsec)
2444	return 0;
2445
2446	for_each_reloc(rsec, reloc) {
2447	if (reloc->sym->type != STT_SECTION) {
2448	WARN("unexpected relocation symbol type in %s", rsec->name);
2449	return -1;
2450	}
2451
2452	insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2453	if (!insn) {
2454	WARN("bad .discard.instr_end entry");
2455	return -1;
2456	}
2457
2458	insn->instr--;
2459	}
2460
2461	rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2462	if (!rsec)
2463	return 0;
2464
2465	for_each_reloc(rsec, reloc) {
2466	if (reloc->sym->type != STT_SECTION) {
2467	WARN("unexpected relocation symbol type in %s", rsec->name);
2468	return -1;
2469	}
2470
2471	insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2472	if (!insn) {
2473	WARN("bad .discard.instr_begin entry");
2474	return -1;
2475	}
2476
2477	insn->instr++;
2478	}
2479
2480	return 0;
2481	}
2482
2483	static int read_validate_unret_hints(struct objtool_file *file)
2484	{
2485	struct section *rsec;
2486	struct instruction *insn;
2487	struct reloc *reloc;
2488
2489	rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2490	if (!rsec)
2491	return 0;
2492
2493	for_each_reloc(rsec, reloc) {
2494	if (reloc->sym->type != STT_SECTION) {
2495	WARN("unexpected relocation symbol type in %s", rsec->name);
2496	return -1;
2497	}
2498
2499	insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2500	if (!insn) {
2501	WARN("bad .discard.instr_end entry");
2502	return -1;
2503	}
2504	insn->unret = 1;
2505	}
2506
2507	return 0;
2508	}
2509
2510
2511	static int read_intra_function_calls(struct objtool_file *file)
2512	{
2513	struct instruction *insn;
2514	struct section *rsec;
2515	struct reloc *reloc;
2516
2517	rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2518	if (!rsec)
2519	return 0;
2520
2521	for_each_reloc(rsec, reloc) {
2522	unsigned long dest_off;
2523
2524	if (reloc->sym->type != STT_SECTION) {
2525	WARN("unexpected relocation symbol type in %s",
2526	rsec->name);
2527	return -1;
2528	}
2529
2530	insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2531	if (!insn) {
2532	WARN("bad .discard.intra_function_call entry");
2533	return -1;
2534	}
2535
2536	if (insn->type != INSN_CALL) {
2537	WARN_INSN(insn, "intra_function_call not a direct call");
2538	return -1;
2539	}
2540
2541	/*
2542	* Treat intra-function CALLs as JMPs, but with a stack_op.
2543	* See add_call_destinations(), which strips stack_ops from
2544	* normal CALLs.
2545	*/
2546	insn->type = INSN_JUMP_UNCONDITIONAL;
2547
2548	dest_off = arch_jump_destination(insn);
2549	insn->jump_dest = find_insn(file, insn->sec, dest_off);
2550	if (!insn->jump_dest) {
2551	WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2552	insn->sec->name, dest_off);
2553	return -1;
2554	}
2555	}
2556
2557	return 0;
2558	}
2559
2560	/*
2561	* Return true if name matches an instrumentation function, where calls to that
2562	* function from noinstr code can safely be removed, but compilers won't do so.
2563	*/
2564	static bool is_profiling_func(const char *name)
2565	{
2566	/*
2567	* Many compilers cannot disable KCOV with a function attribute.
2568	*/
2569	if (!strncmp(name, "__sanitizer_cov_", 16))
2570	return true;
2571
2572	/*
2573	* Some compilers currently do not remove __tsan_func_entry/exit nor
2574	* __tsan_atomic_signal_fence (used for barrier instrumentation) with
2575	* the __no_sanitize_thread attribute, remove them. Once the kernel's
2576	* minimum Clang version is 14.0, this can be removed.
2577	*/
2578	if (!strncmp(name, "__tsan_func_", 12) \|\|
2579	!strcmp(name, "__tsan_atomic_signal_fence"))
2580	return true;
2581
2582	return false;
2583	}
2584
2585	static int classify_symbols(struct objtool_file *file)
2586	{
2587	struct symbol *func;
2588
2589	for_each_sym(file, func) {
2590	if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2591	func->local_label = true;
2592
2593	if (func->bind != STB_GLOBAL)
2594	continue;
2595
2596	if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2597	strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2598	func->static_call_tramp = true;
2599
2600	if (arch_is_retpoline(func))
2601	func->retpoline_thunk = true;
2602
2603	if (arch_is_rethunk(func))
2604	func->return_thunk = true;
2605
2606	if (arch_is_embedded_insn(func))
2607	func->embedded_insn = true;
2608
2609	if (arch_ftrace_match(func->name))
2610	func->fentry = true;
2611
2612	if (is_profiling_func(func->name))
2613	func->profiling_func = true;
2614	}
2615
2616	return 0;
2617	}
2618
2619	static void mark_rodata(struct objtool_file *file)
2620	{
2621	struct section *sec;
2622	bool found = false;
2623
2624	/*
2625	* Search for the following rodata sections, each of which can
2626	* potentially contain jump tables:
2627	*
2628	* - .rodata: can contain GCC switch tables
2629	* - .rodata.<func>: same, if -fdata-sections is being used
2630	* - .rodata..c_jump_table: contains C annotated jump tables
2631	*
2632	* .rodata.str1.* sections are ignored; they don't contain jump tables.
2633	*/
2634	for_each_sec(file, sec) {
2635	if (!strncmp(sec->name, ".rodata", 7) &&
2636	!strstr(sec->name, ".str1.")) {
2637	sec->rodata = true;
2638	found = true;
2639	}
2640	}
2641
2642	file->rodata = found;
2643	}
2644
2645	static int decode_sections(struct objtool_file *file)
2646	{
2647	int ret;
2648
2649	mark_rodata(file);
2650
2651	ret = init_pv_ops(file);
2652	if (ret)
2653	return ret;
2654
2655	/*
2656	* Must be before add_{jump_call}_destination.
2657	*/
2658	ret = classify_symbols(file);
2659	if (ret)
2660	return ret;
2661
2662	ret = decode_instructions(file);
2663	if (ret)
2664	return ret;
2665
2666	add_ignores(file);
2667	add_uaccess_safe(file);
2668
2669	ret = add_ignore_alternatives(file);
2670	if (ret)
2671	return ret;
2672
2673	/*
2674	* Must be before read_unwind_hints() since that needs insn->noendbr.
2675	*/
2676	ret = read_noendbr_hints(file);
2677	if (ret)
2678	return ret;
2679
2680	/*
2681	* Must be before add_jump_destinations(), which depends on 'func'
2682	* being set for alternatives, to enable proper sibling call detection.
2683	*/
2684	if (opts.stackval \|\| opts.orc \|\| opts.uaccess \|\| opts.noinstr) {
2685	ret = add_special_section_alts(file);
2686	if (ret)
2687	return ret;
2688	}
2689
2690	ret = add_jump_destinations(file);
2691	if (ret)
2692	return ret;
2693
2694	/*
2695	* Must be before add_call_destination(); it changes INSN_CALL to
2696	* INSN_JUMP.
2697	*/
2698	ret = read_intra_function_calls(file);
2699	if (ret)
2700	return ret;
2701
2702	ret = add_call_destinations(file);
2703	if (ret)
2704	return ret;
2705
2706	/*
2707	* Must be after add_call_destinations() such that it can override
2708	* dead_end_function() marks.
2709	*/
2710	ret = add_dead_ends(file);
2711	if (ret)
2712	return ret;
2713
2714	ret = add_jump_table_alts(file);
2715	if (ret)
2716	return ret;
2717
2718	ret = read_unwind_hints(file);
2719	if (ret)
2720	return ret;
2721
2722	ret = read_retpoline_hints(file);
2723	if (ret)
2724	return ret;
2725
2726	ret = read_instr_hints(file);
2727	if (ret)
2728	return ret;
2729
2730	ret = read_validate_unret_hints(file);
2731	if (ret)
2732	return ret;
2733
2734	return 0;
2735	}
2736
2737	static bool is_special_call(struct instruction *insn)
2738	{
2739	if (insn->type == INSN_CALL) {
2740	struct symbol *dest = insn_call_dest(insn);
2741
2742	if (!dest)
2743	return false;
2744
2745	if (dest->fentry \|\| dest->embedded_insn)
2746	return true;
2747	}
2748
2749	return false;
2750	}
2751
2752	static bool has_modified_stack_frame(struct instruction insn, struct insn_state state)
2753	{
2754	struct cfi_state *cfi = &state->cfi;
2755	int i;
2756
2757	if (cfi->cfa.base != initial_func_cfi.cfa.base \|\| cfi->drap)
2758	return true;
2759
2760	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2761	return true;
2762
2763	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2764	return true;
2765
2766	for (i = 0; i < CFI_NUM_REGS; i++) {
2767	if (cfi->regs[i].base != initial_func_cfi.regs[i].base \|\|
2768	cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2769	return true;
2770	}
2771
2772	return false;
2773	}
2774
2775	static bool check_reg_frame_pos(const struct cfi_reg *reg,
2776	int expected_offset)
2777	{
2778	return reg->base == CFI_CFA &&
2779	reg->offset == expected_offset;
2780	}
2781
2782	static bool has_valid_stack_frame(struct insn_state *state)
2783	{
2784	struct cfi_state *cfi = &state->cfi;
2785
2786	if (cfi->cfa.base == CFI_BP &&
2787	check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2788	check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2789	return true;
2790
2791	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2792	return true;
2793
2794	return false;
2795	}
2796
2797	static int update_cfi_state_regs(struct instruction *insn,
2798	struct cfi_state *cfi,
2799	struct stack_op *op)
2800	{
2801	struct cfi_reg *cfa = &cfi->cfa;
2802
2803	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2804	return 0;
2805
2806	/* push */
2807	if (op->dest.type == OP_DEST_PUSH \|\| op->dest.type == OP_DEST_PUSHF)
2808	cfa->offset += 8;
2809
2810	/* pop */
2811	if (op->src.type == OP_SRC_POP \|\| op->src.type == OP_SRC_POPF)
2812	cfa->offset -= 8;
2813
2814	/* add immediate to sp */
2815	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2816	op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2817	cfa->offset -= op->src.offset;
2818
2819	return 0;
2820	}
2821
2822	static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2823	{
2824	if (arch_callee_saved_reg(reg) &&
2825	cfi->regs[reg].base == CFI_UNDEFINED) {
2826	cfi->regs[reg].base = base;
2827	cfi->regs[reg].offset = offset;
2828	}
2829	}
2830
2831	static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2832	{
2833	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2834	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2835	}
2836
2837	/*
2838	* A note about DRAP stack alignment:
2839	*
2840	* GCC has the concept of a DRAP register, which is used to help keep track of
2841	* the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2842	* register. The typical DRAP pattern is:
2843	*
2844	* 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2845	* 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2846	* 41 ff 72 f8 pushq -0x8(%r10)
2847	* 55 push %rbp
2848	* 48 89 e5 mov %rsp,%rbp
2849	* (more pushes)
2850	* 41 52 push %r10
2851	* ...
2852	* 41 5a pop %r10
2853	* (more pops)
2854	* 5d pop %rbp
2855	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2856	* c3 retq
2857	*
2858	* There are some variations in the epilogues, like:
2859	*
2860	* 5b pop %rbx
2861	* 41 5a pop %r10
2862	* 41 5c pop %r12
2863	* 41 5d pop %r13
2864	* 41 5e pop %r14
2865	* c9 leaveq
2866	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2867	* c3 retq
2868	*
2869	* and:
2870	*
2871	* 4c 8b 55 e8 mov -0x18(%rbp),%r10
2872	* 48 8b 5d e0 mov -0x20(%rbp),%rbx
2873	* 4c 8b 65 f0 mov -0x10(%rbp),%r12
2874	* 4c 8b 6d f8 mov -0x8(%rbp),%r13
2875	* c9 leaveq
2876	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2877	* c3 retq
2878	*
2879	* Sometimes r13 is used as the DRAP register, in which case it's saved and
2880	* restored beforehand:
2881	*
2882	* 41 55 push %r13
2883	* 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2884	* 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2885	* ...
2886	* 49 8d 65 f0 lea -0x10(%r13),%rsp
2887	* 41 5d pop %r13
2888	* c3 retq
2889	*/
2890	static int update_cfi_state(struct instruction *insn,
2891	struct instruction *next_insn,
2892	struct cfi_state cfi, struct stack_op op)
2893	{
2894	struct cfi_reg *cfa = &cfi->cfa;
2895	struct cfi_reg *regs = cfi->regs;
2896
2897	/* ignore UNWIND_HINT_UNDEFINED regions */
2898	if (cfi->force_undefined)
2899	return 0;
2900
2901	/* stack operations don't make sense with an undefined CFA */
2902	if (cfa->base == CFI_UNDEFINED) {
2903	if (insn_func(insn)) {
2904	WARN_INSN(insn, "undefined stack state");
2905	return -1;
2906	}
2907	return 0;
2908	}
2909
2910	if (cfi->type == UNWIND_HINT_TYPE_REGS \|\|
2911	cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2912	return update_cfi_state_regs(insn, cfi, op);
2913
2914	switch (op->dest.type) {
2915
2916	case OP_DEST_REG:
2917	switch (op->src.type) {
2918
2919	case OP_SRC_REG:
2920	if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2921	cfa->base == CFI_SP &&
2922	check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2923
2924	/* mov %rsp, %rbp */
2925	cfa->base = op->dest.reg;
2926	cfi->bp_scratch = false;
2927	}
2928
2929	else if (op->src.reg == CFI_SP &&
2930	op->dest.reg == CFI_BP && cfi->drap) {
2931
2932	/* drap: mov %rsp, %rbp */
2933	regs[CFI_BP].base = CFI_BP;
2934	regs[CFI_BP].offset = -cfi->stack_size;
2935	cfi->bp_scratch = false;
2936	}
2937
2938	else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2939
2940	/*
2941	* mov %rsp, %reg
2942	*
2943	* This is needed for the rare case where GCC
2944	* does:
2945	*
2946	* mov %rsp, %rax
2947	* ...
2948	* mov %rax, %rsp
2949	*/
2950	cfi->vals[op->dest.reg].base = CFI_CFA;
2951	cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2952	}
2953
2954	else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2955	(cfa->base == CFI_BP \|\| cfa->base == cfi->drap_reg)) {
2956
2957	/*
2958	* mov %rbp, %rsp
2959	*
2960	* Restore the original stack pointer (Clang).
2961	*/
2962	cfi->stack_size = -cfi->regs[CFI_BP].offset;
2963	}
2964
2965	else if (op->dest.reg == cfa->base) {
2966
2967	/* mov %reg, %rsp */
2968	if (cfa->base == CFI_SP &&
2969	cfi->vals[op->src.reg].base == CFI_CFA) {
2970
2971	/*
2972	* This is needed for the rare case
2973	* where GCC does something dumb like:
2974	*
2975	* lea 0x8(%rsp), %rcx
2976	* ...
2977	* mov %rcx, %rsp
2978	*/
2979	cfa->offset = -cfi->vals[op->src.reg].offset;
2980	cfi->stack_size = cfa->offset;
2981
2982	} else if (cfa->base == CFI_SP &&
2983	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2984	cfi->vals[op->src.reg].offset == cfa->offset) {
2985
2986	/*
2987	* Stack swizzle:
2988	*
2989	* 1: mov %rsp, (%[tos])
2990	* 2: mov %[tos], %rsp
2991	* ...
2992	* 3: pop %rsp
2993	*
2994	* Where:
2995	*
2996	* 1 - places a pointer to the previous
2997	* stack at the Top-of-Stack of the
2998	* new stack.
2999	*
3000	* 2 - switches to the new stack.
3001	*
3002	* 3 - pops the Top-of-Stack to restore
3003	* the original stack.
3004	*
3005	* Note: we set base to SP_INDIRECT
3006	* here and preserve offset. Therefore
3007	* when the unwinder reaches ToS it
3008	* will dereference SP and then add the
3009	* offset to find the next frame, IOW:
3010	* (%rsp) + offset.
3011	*/
3012	cfa->base = CFI_SP_INDIRECT;
3013
3014	} else {
3015	cfa->base = CFI_UNDEFINED;
3016	cfa->offset = 0;
3017	}
3018	}
3019
3020	else if (op->dest.reg == CFI_SP &&
3021	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
3022	cfi->vals[op->src.reg].offset == cfa->offset) {
3023
3024	/*
3025	* The same stack swizzle case 2) as above. But
3026	* because we can't change cfa->base, case 3)
3027	* will become a regular POP. Pretend we're a
3028	* PUSH so things don't go unbalanced.
3029	*/
3030	cfi->stack_size += 8;
3031	}
3032
3033
3034	break;
3035
3036	case OP_SRC_ADD:
3037	if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
3038
3039	/* add imm, %rsp */
3040	cfi->stack_size -= op->src.offset;
3041	if (cfa->base == CFI_SP)
3042	cfa->offset -= op->src.offset;
3043	break;
3044	}
3045
3046	if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
3047	insn->sym->frame_pointer) {
3048	/* addi.d fp,sp,imm on LoongArch */
3049	if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
3050	cfa->base = CFI_BP;
3051	cfa->offset = 0;
3052	}
3053	break;
3054	}
3055
3056	if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3057	/* addi.d sp,fp,imm on LoongArch */
3058	if (cfa->base == CFI_BP && cfa->offset == 0) {
3059	if (insn->sym->frame_pointer) {
3060	cfa->base = CFI_SP;
3061	cfa->offset = -op->src.offset;
3062	}
3063	} else {
3064	/* lea disp(%rbp), %rsp */
3065	cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3066	}
3067	break;
3068	}
3069
3070	if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3071
3072	/* drap: lea disp(%rsp), %drap */
3073	cfi->drap_reg = op->dest.reg;
3074
3075	/*
3076	* lea disp(%rsp), %reg
3077	*
3078	* This is needed for the rare case where GCC
3079	* does something dumb like:
3080	*
3081	* lea 0x8(%rsp), %rcx
3082	* ...
3083	* mov %rcx, %rsp
3084	*/
3085	cfi->vals[op->dest.reg].base = CFI_CFA;
3086	cfi->vals[op->dest.reg].offset = \
3087	-cfi->stack_size + op->src.offset;
3088
3089	break;
3090	}
3091
3092	if (cfi->drap && op->dest.reg == CFI_SP &&
3093	op->src.reg == cfi->drap_reg) {
3094
3095	/* drap: lea disp(%drap), %rsp */
3096	cfa->base = CFI_SP;
3097	cfa->offset = cfi->stack_size = -op->src.offset;
3098	cfi->drap_reg = CFI_UNDEFINED;
3099	cfi->drap = false;
3100	break;
3101	}
3102
3103	if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3104	WARN_INSN(insn, "unsupported stack register modification");
3105	return -1;
3106	}
3107
3108	break;
3109
3110	case OP_SRC_AND:
3111	if (op->dest.reg != CFI_SP \|\|
3112	(cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) \|\|
3113	(cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3114	WARN_INSN(insn, "unsupported stack pointer realignment");
3115	return -1;
3116	}
3117
3118	if (cfi->drap_reg != CFI_UNDEFINED) {
3119	/* drap: and imm, %rsp */
3120	cfa->base = cfi->drap_reg;
3121	cfa->offset = cfi->stack_size = 0;
3122	cfi->drap = true;
3123	}
3124
3125	/*
3126	* Older versions of GCC (4.8ish) realign the stack
3127	* without DRAP, with a frame pointer.
3128	*/
3129
3130	break;
3131
3132	case OP_SRC_POP:
3133	case OP_SRC_POPF:
3134	if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3135
3136	/* pop %rsp; # restore from a stack swizzle */
3137	cfa->base = CFI_SP;
3138	break;
3139	}
3140
3141	if (!cfi->drap && op->dest.reg == cfa->base) {
3142
3143	/* pop %rbp */
3144	cfa->base = CFI_SP;
3145	}
3146
3147	if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3148	op->dest.reg == cfi->drap_reg &&
3149	cfi->drap_offset == -cfi->stack_size) {
3150
3151	/* drap: pop %drap */
3152	cfa->base = cfi->drap_reg;
3153	cfa->offset = 0;
3154	cfi->drap_offset = -1;
3155
3156	} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3157
3158	/* pop %reg */
3159	restore_reg(cfi, op->dest.reg);
3160	}
3161
3162	cfi->stack_size -= 8;
3163	if (cfa->base == CFI_SP)
3164	cfa->offset -= 8;
3165
3166	break;
3167
3168	case OP_SRC_REG_INDIRECT:
3169	if (!cfi->drap && op->dest.reg == cfa->base &&
3170	op->dest.reg == CFI_BP) {
3171
3172	/* mov disp(%rsp), %rbp */
3173	cfa->base = CFI_SP;
3174	cfa->offset = cfi->stack_size;
3175	}
3176
3177	if (cfi->drap && op->src.reg == CFI_BP &&
3178	op->src.offset == cfi->drap_offset) {
3179
3180	/* drap: mov disp(%rbp), %drap */
3181	cfa->base = cfi->drap_reg;
3182	cfa->offset = 0;
3183	cfi->drap_offset = -1;
3184	}
3185
3186	if (cfi->drap && op->src.reg == CFI_BP &&
3187	op->src.offset == regs[op->dest.reg].offset) {
3188
3189	/* drap: mov disp(%rbp), %reg */
3190	restore_reg(cfi, op->dest.reg);
3191
3192	} else if (op->src.reg == cfa->base &&
3193	op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3194
3195	/* mov disp(%rbp), %reg */
3196	/* mov disp(%rsp), %reg */
3197	restore_reg(cfi, op->dest.reg);
3198
3199	} else if (op->src.reg == CFI_SP &&
3200	op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3201
3202	/* mov disp(%rsp), %reg */
3203	restore_reg(cfi, op->dest.reg);
3204	}
3205
3206	break;
3207
3208	default:
3209	WARN_INSN(insn, "unknown stack-related instruction");
3210	return -1;
3211	}
3212
3213	break;
3214
3215	case OP_DEST_PUSH:
3216	case OP_DEST_PUSHF:
3217	cfi->stack_size += 8;
3218	if (cfa->base == CFI_SP)
3219	cfa->offset += 8;
3220
3221	if (op->src.type != OP_SRC_REG)
3222	break;
3223
3224	if (cfi->drap) {
3225	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3226
3227	/* drap: push %drap */
3228	cfa->base = CFI_BP_INDIRECT;
3229	cfa->offset = -cfi->stack_size;
3230
3231	/* save drap so we know when to restore it */
3232	cfi->drap_offset = -cfi->stack_size;
3233
3234	} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3235
3236	/* drap: push %rbp */
3237	cfi->stack_size = 0;
3238
3239	} else {
3240
3241	/* drap: push %reg */
3242	save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3243	}
3244
3245	} else {
3246
3247	/* push %reg */
3248	save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3249	}
3250
3251	/* detect when asm code uses rbp as a scratch register */
3252	if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3253	cfa->base != CFI_BP)
3254	cfi->bp_scratch = true;
3255	break;
3256
3257	case OP_DEST_REG_INDIRECT:
3258
3259	if (cfi->drap) {
3260	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3261
3262	/* drap: mov %drap, disp(%rbp) */
3263	cfa->base = CFI_BP_INDIRECT;
3264	cfa->offset = op->dest.offset;
3265
3266	/* save drap offset so we know when to restore it */
3267	cfi->drap_offset = op->dest.offset;
3268	} else {
3269
3270	/* drap: mov reg, disp(%rbp) */
3271	save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3272	}
3273
3274	} else if (op->dest.reg == cfa->base) {
3275
3276	/* mov reg, disp(%rbp) */
3277	/* mov reg, disp(%rsp) */
3278	save_reg(cfi, op->src.reg, CFI_CFA,
3279	op->dest.offset - cfi->cfa.offset);
3280
3281	} else if (op->dest.reg == CFI_SP) {
3282
3283	/* mov reg, disp(%rsp) */
3284	save_reg(cfi, op->src.reg, CFI_CFA,
3285	op->dest.offset - cfi->stack_size);
3286
3287	} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3288
3289	/* mov %rsp, (%reg); # setup a stack swizzle. */
3290	cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3291	cfi->vals[op->dest.reg].offset = cfa->offset;
3292	}
3293
3294	break;
3295
3296	case OP_DEST_MEM:
3297	if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3298	WARN_INSN(insn, "unknown stack-related memory operation");
3299	return -1;
3300	}
3301
3302	/* pop mem */
3303	cfi->stack_size -= 8;
3304	if (cfa->base == CFI_SP)
3305	cfa->offset -= 8;
3306
3307	break;
3308
3309	default:
3310	WARN_INSN(insn, "unknown stack-related instruction");
3311	return -1;
3312	}
3313
3314	return 0;
3315	}
3316
3317	/*
3318	* The stack layouts of alternatives instructions can sometimes diverge when
3319	* they have stack modifications. That's fine as long as the potential stack
3320	* layouts don't conflict at any given potential instruction boundary.
3321	*
3322	* Flatten the CFIs of the different alternative code streams (both original
3323	* and replacement) into a single shared CFI array which can be used to detect
3324	* conflicts and nicely feed a linear array of ORC entries to the unwinder.
3325	*/
3326	static int propagate_alt_cfi(struct objtool_file file, struct instruction insn)
3327	{
3328	struct cfi_state **alt_cfi;
3329	int group_off;
3330
3331	if (!insn->alt_group)
3332	return 0;
3333
3334	if (!insn->cfi) {
3335	WARN("CFI missing");
3336	return -1;
3337	}
3338
3339	alt_cfi = insn->alt_group->cfi;
3340	group_off = insn->offset - insn->alt_group->first_insn->offset;
3341
3342	if (!alt_cfi[group_off]) {
3343	alt_cfi[group_off] = insn->cfi;
3344	} else {
3345	if (cficmp(alt_cfi[group_off], insn->cfi)) {
3346	struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3347	struct instruction *orig = orig_group->first_insn;
3348	char *where = offstr(insn->sec, insn->offset);
3349	WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3350	free(where);
3351	return -1;
3352	}
3353	}
3354
3355	return 0;
3356	}
3357
3358	static int handle_insn_ops(struct instruction *insn,
3359	struct instruction *next_insn,
3360	struct insn_state *state)
3361	{
3362	struct stack_op *op;
3363
3364	for (op = insn->stack_ops; op; op = op->next) {
3365
3366	if (update_cfi_state(insn, next_insn, &state->cfi, op))
3367	return 1;
3368
3369	if (!insn->alt_group)
3370	continue;
3371
3372	if (op->dest.type == OP_DEST_PUSHF) {
3373	if (!state->uaccess_stack) {
3374	state->uaccess_stack = 1;
3375	} else if (state->uaccess_stack >> 31) {
3376	WARN_INSN(insn, "PUSHF stack exhausted");
3377	return 1;
3378	}
3379	state->uaccess_stack <<= 1;
3380	state->uaccess_stack \|= state->uaccess;
3381	}
3382
3383	if (op->src.type == OP_SRC_POPF) {
3384	if (state->uaccess_stack) {
3385	state->uaccess = state->uaccess_stack & 1;
3386	state->uaccess_stack >>= 1;
3387	if (state->uaccess_stack == 1)
3388	state->uaccess_stack = 0;
3389	}
3390	}
3391	}
3392
3393	return 0;
3394	}
3395
3396	static bool insn_cfi_match(struct instruction insn, struct cfi_state cfi2)
3397	{
3398	struct cfi_state *cfi1 = insn->cfi;
3399	int i;
3400
3401	if (!cfi1) {
3402	WARN("CFI missing");
3403	return false;
3404	}
3405
3406	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3407
3408	WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3409	cfi1->cfa.base, cfi1->cfa.offset,
3410	cfi2->cfa.base, cfi2->cfa.offset);
3411
3412	} else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3413	for (i = 0; i < CFI_NUM_REGS; i++) {
3414	if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3415	sizeof(struct cfi_reg)))
3416	continue;
3417
3418	WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3419	i, cfi1->regs[i].base, cfi1->regs[i].offset,
3420	i, cfi2->regs[i].base, cfi2->regs[i].offset);
3421	break;
3422	}
3423
3424	} else if (cfi1->type != cfi2->type) {
3425
3426	WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3427	cfi1->type, cfi2->type);
3428
3429	} else if (cfi1->drap != cfi2->drap \|\|
3430	(cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) \|\|
3431	(cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3432
3433	WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3434	cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3435	cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3436
3437	} else
3438	return true;
3439
3440	return false;
3441	}
3442
3443	static inline bool func_uaccess_safe(struct symbol *func)
3444	{
3445	if (func)
3446	return func->uaccess_safe;
3447
3448	return false;
3449	}
3450
3451	static inline const char call_dest_name(struct instruction insn)
3452	{
3453	static char pvname[19];
3454	struct reloc *reloc;
3455	int idx;
3456
3457	if (insn_call_dest(insn))
3458	return insn_call_dest(insn)->name;
3459
3460	reloc = insn_reloc(NULL, insn);
3461	if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3462	idx = (reloc_addend(reloc) / sizeof(void *));
3463	snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3464	return pvname;
3465	}
3466
3467	return "{dynamic}";
3468	}
3469
3470	static bool pv_call_dest(struct objtool_file file, struct instruction insn)
3471	{
3472	struct symbol *target;
3473	struct reloc *reloc;
3474	int idx;
3475
3476	reloc = insn_reloc(file, insn);
3477	if (!reloc \|\| strcmp(reloc->sym->name, "pv_ops"))
3478	return false;
3479
3480	idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3481
3482	if (file->pv_ops[idx].clean)
3483	return true;
3484
3485	file->pv_ops[idx].clean = true;
3486
3487	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3488	if (!target->sec->noinstr) {
3489	WARN("pv_ops[%d]: %s", idx, target->name);
3490	file->pv_ops[idx].clean = false;
3491	}
3492	}
3493
3494	return file->pv_ops[idx].clean;
3495	}
3496
3497	static inline bool noinstr_call_dest(struct objtool_file *file,
3498	struct instruction *insn,
3499	struct symbol *func)
3500	{
3501	/*
3502	* We can't deal with indirect function calls at present;
3503	* assume they're instrumented.
3504	*/
3505	if (!func) {
3506	if (file->pv_ops)
3507	return pv_call_dest(file, insn);
3508
3509	return false;
3510	}
3511
3512	/*
3513	* If the symbol is from a noinstr section; we good.
3514	*/
3515	if (func->sec->noinstr)
3516	return true;
3517
3518	/*
3519	* If the symbol is a static_call trampoline, we can't tell.
3520	*/
3521	if (func->static_call_tramp)
3522	return true;
3523
3524	/*
3525	* The __ubsan_handle_*() calls are like WARN(), they only happen when
3526	* something 'BAD' happened. At the risk of taking the machine down,
3527	* let them proceed to get the message out.
3528	*/
3529	if (!strncmp(func->name, "__ubsan_handle_", 15))
3530	return true;
3531
3532	return false;
3533	}
3534
3535	static int validate_call(struct objtool_file *file,
3536	struct instruction *insn,
3537	struct insn_state *state)
3538	{
3539	if (state->noinstr && state->instr <= 0 &&
3540	!noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3541	WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3542	return 1;
3543	}
3544
3545	if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3546	WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3547	return 1;
3548	}
3549
3550	if (state->df) {
3551	WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3552	return 1;
3553	}
3554
3555	return 0;
3556	}
3557
3558	static int validate_sibling_call(struct objtool_file *file,
3559	struct instruction *insn,
3560	struct insn_state *state)
3561	{
3562	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3563	WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3564	return 1;
3565	}
3566
3567	return validate_call(file, insn, state);
3568	}
3569
3570	static int validate_return(struct symbol func, struct instruction insn, struct insn_state *state)
3571	{
3572	if (state->noinstr && state->instr > 0) {
3573	WARN_INSN(insn, "return with instrumentation enabled");
3574	return 1;
3575	}
3576
3577	if (state->uaccess && !func_uaccess_safe(func)) {
3578	WARN_INSN(insn, "return with UACCESS enabled");
3579	return 1;
3580	}
3581
3582	if (!state->uaccess && func_uaccess_safe(func)) {
3583	WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3584	return 1;
3585	}
3586
3587	if (state->df) {
3588	WARN_INSN(insn, "return with DF set");
3589	return 1;
3590	}
3591
3592	if (func && has_modified_stack_frame(insn, state)) {
3593	WARN_INSN(insn, "return with modified stack frame");
3594	return 1;
3595	}
3596
3597	if (state->cfi.bp_scratch) {
3598	WARN_INSN(insn, "BP used as a scratch register");
3599	return 1;
3600	}
3601
3602	return 0;
3603	}
3604
3605	static struct instruction next_insn_to_validate(struct objtool_file file,
3606	struct instruction *insn)
3607	{
3608	struct alt_group *alt_group = insn->alt_group;
3609
3610	/*
3611	* Simulate the fact that alternatives are patched in-place. When the
3612	* end of a replacement alt_group is reached, redirect objtool flow to
3613	* the end of the original alt_group.
3614	*
3615	* insn->alts->insn -> alt_group->first_insn
3616	* ...
3617	* alt_group->last_insn
3618	* [alt_group->nop] -> next(orig_group->last_insn)
3619	*/
3620	if (alt_group) {
3621	if (alt_group->nop) {
3622	/* ->nop implies ->orig_group */
3623	if (insn == alt_group->last_insn)
3624	return alt_group->nop;
3625	if (insn == alt_group->nop)
3626	goto next_orig;
3627	}
3628	if (insn == alt_group->last_insn && alt_group->orig_group)
3629	goto next_orig;
3630	}
3631
3632	return next_insn_same_sec(file, insn);
3633
3634	next_orig:
3635	return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3636	}
3637
3638	/*
3639	* Follow the branch starting at the given instruction, and recursively follow
3640	* any other branches (jumps). Meanwhile, track the frame pointer state at
3641	* each instruction and validate all the rules described in
3642	* tools/objtool/Documentation/objtool.txt.
3643	*/
3644	static int validate_branch(struct objtool_file file, struct symbol func,
3645	struct instruction *insn, struct insn_state state)
3646	{
3647	struct alternative *alt;
3648	struct instruction next_insn, prev_insn = NULL;
3649	struct section *sec;
3650	u8 visited;
3651	int ret;
3652
3653	sec = insn->sec;
3654
3655	while (1) {
3656	next_insn = next_insn_to_validate(file, insn);
3657
3658	if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3659	/* Ignore KCFI type preambles, which always fall through */
3660	if (!strncmp(func->name, "__cfi_", 6) \|\|
3661	!strncmp(func->name, "__pfx_", 6))
3662	return 0;
3663
3664	WARN("%s() falls through to next function %s()",
3665	func->name, insn_func(insn)->name);
3666	return 1;
3667	}
3668
3669	if (func && insn->ignore) {
3670	WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3671	return 1;
3672	}
3673
3674	visited = VISITED_BRANCH << state.uaccess;
3675	if (insn->visited & VISITED_BRANCH_MASK) {
3676	if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3677	return 1;
3678
3679	if (insn->visited & visited)
3680	return 0;
3681	} else {
3682	nr_insns_visited++;
3683	}
3684
3685	if (state.noinstr)
3686	state.instr += insn->instr;
3687
3688	if (insn->hint) {
3689	if (insn->restore) {
3690	struct instruction save_insn, i;
3691
3692	i = insn;
3693	save_insn = NULL;
3694
3695	sym_for_each_insn_continue_reverse(file, func, i) {
3696	if (i->save) {
3697	save_insn = i;
3698	break;
3699	}
3700	}
3701
3702	if (!save_insn) {
3703	WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3704	return 1;
3705	}
3706
3707	if (!save_insn->visited) {
3708	/*
3709	* If the restore hint insn is at the
3710	* beginning of a basic block and was
3711	* branched to from elsewhere, and the
3712	* save insn hasn't been visited yet,
3713	* defer following this branch for now.
3714	* It will be seen later via the
3715	* straight-line path.
3716	*/
3717	if (!prev_insn)
3718	return 0;
3719
3720	WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3721	return 1;
3722	}
3723
3724	insn->cfi = save_insn->cfi;
3725	nr_cfi_reused++;
3726	}
3727
3728	state.cfi = *insn->cfi;
3729	} else {
3730	/* XXX track if we actually changed state.cfi */
3731
3732	if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3733	insn->cfi = prev_insn->cfi;
3734	nr_cfi_reused++;
3735	} else {
3736	insn->cfi = cfi_hash_find_or_add(&state.cfi);
3737	}
3738	}
3739
3740	insn->visited \|= visited;
3741
3742	if (propagate_alt_cfi(file, insn))
3743	return 1;
3744
3745	if (!insn->ignore_alts && insn->alts) {
3746	bool skip_orig = false;
3747
3748	for (alt = insn->alts; alt; alt = alt->next) {
3749	if (alt->skip_orig)
3750	skip_orig = true;
3751
3752	ret = validate_branch(file, func, alt->insn, state);
3753	if (ret) {
3754	BT_INSN(insn, "(alt)");
3755	return ret;
3756	}
3757	}
3758
3759	if (skip_orig)
3760	return 0;
3761	}
3762
3763	if (handle_insn_ops(insn, next_insn, &state))
3764	return 1;
3765
3766	switch (insn->type) {
3767
3768	case INSN_RETURN:
3769	return validate_return(func, insn, &state);
3770
3771	case INSN_CALL:
3772	case INSN_CALL_DYNAMIC:
3773	ret = validate_call(file, insn, &state);
3774	if (ret)
3775	return ret;
3776
3777	if (opts.stackval && func && !is_special_call(insn) &&
3778	!has_valid_stack_frame(&state)) {
3779	WARN_INSN(insn, "call without frame pointer save/setup");
3780	return 1;
3781	}
3782
3783	if (insn->dead_end)
3784	return 0;
3785
3786	break;
3787
3788	case INSN_JUMP_CONDITIONAL:
3789	case INSN_JUMP_UNCONDITIONAL:
3790	if (is_sibling_call(insn)) {
3791	ret = validate_sibling_call(file, insn, &state);
3792	if (ret)
3793	return ret;
3794
3795	} else if (insn->jump_dest) {
3796	ret = validate_branch(file, func,
3797	insn->jump_dest, state);
3798	if (ret) {
3799	BT_INSN(insn, "(branch)");
3800	return ret;
3801	}
3802	}
3803
3804	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3805	return 0;
3806
3807	break;
3808
3809	case INSN_JUMP_DYNAMIC:
3810	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3811	if (is_sibling_call(insn)) {
3812	ret = validate_sibling_call(file, insn, &state);
3813	if (ret)
3814	return ret;
3815	}
3816
3817	if (insn->type == INSN_JUMP_DYNAMIC)
3818	return 0;
3819
3820	break;
3821
3822	case INSN_CONTEXT_SWITCH:
3823	if (func) {
3824	if (!next_insn \|\| !next_insn->hint) {
3825	WARN_INSN(insn, "unsupported instruction in callable function");
3826	return 1;
3827	}
3828	break;
3829	}
3830	return 0;
3831
3832	case INSN_STAC:
3833	if (state.uaccess) {
3834	WARN_INSN(insn, "recursive UACCESS enable");
3835	return 1;
3836	}
3837
3838	state.uaccess = true;
3839	break;
3840
3841	case INSN_CLAC:
3842	if (!state.uaccess && func) {
3843	WARN_INSN(insn, "redundant UACCESS disable");
3844	return 1;
3845	}
3846
3847	if (func_uaccess_safe(func) && !state.uaccess_stack) {
3848	WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3849	return 1;
3850	}
3851
3852	state.uaccess = false;
3853	break;
3854
3855	case INSN_STD:
3856	if (state.df) {
3857	WARN_INSN(insn, "recursive STD");
3858	return 1;
3859	}
3860
3861	state.df = true;
3862	break;
3863
3864	case INSN_CLD:
3865	if (!state.df && func) {
3866	WARN_INSN(insn, "redundant CLD");
3867	return 1;
3868	}
3869
3870	state.df = false;
3871	break;
3872
3873	default:
3874	break;
3875	}
3876
3877	if (insn->dead_end)
3878	return 0;
3879
3880	if (!next_insn) {
3881	if (state.cfi.cfa.base == CFI_UNDEFINED)
3882	return 0;
3883	WARN("%s: unexpected end of section", sec->name);
3884	return 1;
3885	}
3886
3887	prev_insn = insn;
3888	insn = next_insn;
3889	}
3890
3891	return 0;
3892	}
3893
3894	static int validate_unwind_hint(struct objtool_file *file,
3895	struct instruction *insn,
3896	struct insn_state *state)
3897	{
3898	if (insn->hint && !insn->visited && !insn->ignore) {
3899	int ret = validate_branch(file, insn_func(insn), insn, *state);
3900	if (ret)
3901	BT_INSN(insn, "<=== (hint)");
3902	return ret;
3903	}
3904
3905	return 0;
3906	}
3907
3908	static int validate_unwind_hints(struct objtool_file file, struct section sec)
3909	{
3910	struct instruction *insn;
3911	struct insn_state state;
3912	int warnings = 0;
3913
3914	if (!file->hints)
3915	return 0;
3916
3917	init_insn_state(file, &state, sec);
3918
3919	if (sec) {
3920	sec_for_each_insn(file, sec, insn)
3921	warnings += validate_unwind_hint(file, insn, &state);
3922	} else {
3923	for_each_insn(file, insn)
3924	warnings += validate_unwind_hint(file, insn, &state);
3925	}
3926
3927	return warnings;
3928	}
3929
3930	/*
3931	* Validate rethunk entry constraint: must untrain RET before the first RET.
3932	*
3933	* Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3934	* before an actual RET instruction.
3935	*/
3936	static int validate_unret(struct objtool_file file, struct instruction insn)
3937	{
3938	struct instruction next, dest;
3939	int ret;
3940
3941	for (;;) {
3942	next = next_insn_to_validate(file, insn);
3943
3944	if (insn->visited & VISITED_UNRET)
3945	return 0;
3946
3947	insn->visited \|= VISITED_UNRET;
3948
3949	if (!insn->ignore_alts && insn->alts) {
3950	struct alternative *alt;
3951	bool skip_orig = false;
3952
3953	for (alt = insn->alts; alt; alt = alt->next) {
3954	if (alt->skip_orig)
3955	skip_orig = true;
3956
3957	ret = validate_unret(file, alt->insn);
3958	if (ret) {
3959	BT_INSN(insn, "(alt)");
3960	return ret;
3961	}
3962	}
3963
3964	if (skip_orig)
3965	return 0;
3966	}
3967
3968	switch (insn->type) {
3969
3970	case INSN_CALL_DYNAMIC:
3971	case INSN_JUMP_DYNAMIC:
3972	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3973	WARN_INSN(insn, "early indirect call");
3974	return 1;
3975
3976	case INSN_JUMP_UNCONDITIONAL:
3977	case INSN_JUMP_CONDITIONAL:
3978	if (!is_sibling_call(insn)) {
3979	if (!insn->jump_dest) {
3980	WARN_INSN(insn, "unresolved jump target after linking?!?");
3981	return -1;
3982	}
3983	ret = validate_unret(file, insn->jump_dest);
3984	if (ret) {
3985	BT_INSN(insn, "(branch%s)",
3986	insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3987	return ret;
3988	}
3989
3990	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3991	return 0;
3992
3993	break;
3994	}
3995
3996	/* fallthrough */
3997	case INSN_CALL:
3998	dest = find_insn(file, insn_call_dest(insn)->sec,
3999	insn_call_dest(insn)->offset);
4000	if (!dest) {
4001	WARN("Unresolved function after linking!?: %s",
4002	insn_call_dest(insn)->name);
4003	return -1;
4004	}
4005
4006	ret = validate_unret(file, dest);
4007	if (ret) {
4008	BT_INSN(insn, "(call)");
4009	return ret;
4010	}
4011	/*
4012	* If a call returns without error, it must have seen UNTRAIN_RET.
4013	* Therefore any non-error return is a success.
4014	*/
4015	return 0;
4016
4017	case INSN_RETURN:
4018	WARN_INSN(insn, "RET before UNTRAIN");
4019	return 1;
4020
4021	case INSN_NOP:
4022	if (insn->retpoline_safe)
4023	return 0;
4024	break;
4025
4026	default:
4027	break;
4028	}
4029
4030	if (!next) {
4031	WARN_INSN(insn, "teh end!");
4032	return -1;
4033	}
4034	insn = next;
4035	}
4036
4037	return 0;
4038	}
4039
4040	/*
4041	* Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
4042	* VALIDATE_UNRET_END before RET.
4043	*/
4044	static int validate_unrets(struct objtool_file *file)
4045	{
4046	struct instruction *insn;
4047	int ret, warnings = 0;
4048
4049	for_each_insn(file, insn) {
4050	if (!insn->unret)
4051	continue;
4052
4053	ret = validate_unret(file, insn);
4054	if (ret < 0) {
4055	WARN_INSN(insn, "Failed UNRET validation");
4056	return ret;
4057	}
4058	warnings += ret;
4059	}
4060
4061	return warnings;
4062	}
4063
4064	static int validate_retpoline(struct objtool_file *file)
4065	{
4066	struct instruction *insn;
4067	int warnings = 0;
4068
4069	for_each_insn(file, insn) {
4070	if (insn->type != INSN_JUMP_DYNAMIC &&
4071	insn->type != INSN_CALL_DYNAMIC &&
4072	insn->type != INSN_RETURN)
4073	continue;
4074
4075	if (insn->retpoline_safe)
4076	continue;
4077
4078	if (insn->sec->init)
4079	continue;
4080
4081	if (insn->type == INSN_RETURN) {
4082	if (opts.rethunk) {
4083	WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4084	} else
4085	continue;
4086	} else {
4087	WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4088	insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4089	}
4090
4091	warnings++;
4092	}
4093
4094	return warnings;
4095	}
4096
4097	static bool is_kasan_insn(struct instruction *insn)
4098	{
4099	return (insn->type == INSN_CALL &&
4100	!strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4101	}
4102
4103	static bool is_ubsan_insn(struct instruction *insn)
4104	{
4105	return (insn->type == INSN_CALL &&
4106	!strcmp(insn_call_dest(insn)->name,
4107	"__ubsan_handle_builtin_unreachable"));
4108	}
4109
4110	static bool ignore_unreachable_insn(struct objtool_file file, struct instruction insn)
4111	{
4112	int i;
4113	struct instruction *prev_insn;
4114
4115	if (insn->ignore \|\| insn->type == INSN_NOP \|\| insn->type == INSN_TRAP)
4116	return true;
4117
4118	/*
4119	* Ignore alternative replacement instructions. This can happen
4120	* when a whitelisted function uses one of the ALTERNATIVE macros.
4121	*/
4122	if (!strcmp(insn->sec->name, ".altinstr_replacement") \|\|
4123	!strcmp(insn->sec->name, ".altinstr_aux"))
4124	return true;
4125
4126	/*
4127	* Whole archive runs might encounter dead code from weak symbols.
4128	* This is where the linker will have dropped the weak symbol in
4129	* favour of a regular symbol, but leaves the code in place.
4130	*
4131	* In this case we'll find a piece of code (whole function) that is not
4132	* covered by a !section symbol. Ignore them.
4133	*/
4134	if (opts.link && !insn_func(insn)) {
4135	int size = find_symbol_hole_containing(insn->sec, insn->offset);
4136	unsigned long end = insn->offset + size;
4137
4138	if (!size) /* not a hole */
4139	return false;
4140
4141	if (size < 0) /* hole until the end */
4142	return true;
4143
4144	sec_for_each_insn_continue(file, insn) {
4145	/*
4146	* If we reach a visited instruction at or before the
4147	* end of the hole, ignore the unreachable.
4148	*/
4149	if (insn->visited)
4150	return true;
4151
4152	if (insn->offset >= end)
4153	break;
4154
4155	/*
4156	* If this hole jumps to a .cold function, mark it ignore too.
4157	*/
4158	if (insn->jump_dest && insn_func(insn->jump_dest) &&
4159	strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4160	struct instruction *dest = insn->jump_dest;
4161	func_for_each_insn(file, insn_func(dest), dest)
4162	dest->ignore = true;
4163	}
4164	}
4165
4166	return false;
4167	}
4168
4169	if (!insn_func(insn))
4170	return false;
4171
4172	if (insn_func(insn)->static_call_tramp)
4173	return true;
4174
4175	/*
4176	* CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4177	* __builtin_unreachable(). The BUG() macro has an unreachable() after
4178	* the UD2, which causes GCC's undefined trap logic to emit another UD2
4179	* (or occasionally a JMP to UD2).
4180	*
4181	* It may also insert a UD2 after calling a __noreturn function.
4182	*/
4183	prev_insn = prev_insn_same_sec(file, insn);
4184	if (prev_insn->dead_end &&
4185	(insn->type == INSN_BUG \|\|
4186	(insn->type == INSN_JUMP_UNCONDITIONAL &&
4187	insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4188	return true;
4189
4190	/*
4191	* Check if this (or a subsequent) instruction is related to
4192	* CONFIG_UBSAN or CONFIG_KASAN.
4193	*
4194	* End the search at 5 instructions to avoid going into the weeds.
4195	*/
4196	for (i = 0; i < 5; i++) {
4197
4198	if (is_kasan_insn(insn) \|\| is_ubsan_insn(insn))
4199	return true;
4200
4201	if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4202	if (insn->jump_dest &&
4203	insn_func(insn->jump_dest) == insn_func(insn)) {
4204	insn = insn->jump_dest;
4205	continue;
4206	}
4207
4208	break;
4209	}
4210
4211	if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4212	break;
4213
4214	insn = next_insn_same_sec(file, insn);
4215	}
4216
4217	return false;
4218	}
4219
4220	static int add_prefix_symbol(struct objtool_file file, struct symbol func)
4221	{
4222	struct instruction insn, prev;
4223	struct cfi_state *cfi;
4224
4225	insn = find_insn(file, func->sec, func->offset);
4226	if (!insn)
4227	return -1;
4228
4229	for (prev = prev_insn_same_sec(file, insn);
4230	prev;
4231	prev = prev_insn_same_sec(file, prev)) {
4232	u64 offset;
4233
4234	if (prev->type != INSN_NOP)
4235	return -1;
4236
4237	offset = func->offset - prev->offset;
4238
4239	if (offset > opts.prefix)
4240	return -1;
4241
4242	if (offset < opts.prefix)
4243	continue;
4244
4245	elf_create_prefix_symbol(file->elf, func, opts.prefix);
4246	break;
4247	}
4248
4249	if (!prev)
4250	return -1;
4251
4252	if (!insn->cfi) {
4253	/*
4254	* This can happen if stack validation isn't enabled or the
4255	* function is annotated with STACK_FRAME_NON_STANDARD.
4256	*/
4257	return 0;
4258	}
4259
4260	/* Propagate insn->cfi to the prefix code */
4261	cfi = cfi_hash_find_or_add(insn->cfi);
4262	for (; prev != insn; prev = next_insn_same_sec(file, prev))
4263	prev->cfi = cfi;
4264
4265	return 0;
4266	}
4267
4268	static int add_prefix_symbols(struct objtool_file *file)
4269	{
4270	struct section *sec;
4271	struct symbol *func;
4272
4273	for_each_sec(file, sec) {
4274	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4275	continue;
4276
4277	sec_for_each_sym(sec, func) {
4278	if (func->type != STT_FUNC)
4279	continue;
4280
4281	add_prefix_symbol(file, func);
4282	}
4283	}
4284
4285	return 0;
4286	}
4287
4288	static int validate_symbol(struct objtool_file file, struct section sec,
4289	struct symbol sym, struct insn_state state)
4290	{
4291	struct instruction *insn;
4292	int ret;
4293
4294	if (!sym->len) {
4295	WARN("%s() is missing an ELF size annotation", sym->name);
4296	return 1;
4297	}
4298
4299	if (sym->pfunc != sym \|\| sym->alias != sym)
4300	return 0;
4301
4302	insn = find_insn(file, sec, sym->offset);
4303	if (!insn \|\| insn->ignore \|\| insn->visited)
4304	return 0;
4305
4306	state->uaccess = sym->uaccess_safe;
4307
4308	ret = validate_branch(file, insn_func(insn), insn, *state);
4309	if (ret)
4310	BT_INSN(insn, "<=== (sym)");
4311	return ret;
4312	}
4313
4314	static int validate_section(struct objtool_file file, struct section sec)
4315	{
4316	struct insn_state state;
4317	struct symbol *func;
4318	int warnings = 0;
4319
4320	sec_for_each_sym(sec, func) {
4321	if (func->type != STT_FUNC)
4322	continue;
4323
4324	init_insn_state(file, &state, sec);
4325	set_func_state(&state.cfi);
4326
4327	warnings += validate_symbol(file, sec, func, &state);
4328	}
4329
4330	return warnings;
4331	}
4332
4333	static int validate_noinstr_sections(struct objtool_file *file)
4334	{
4335	struct section *sec;
4336	int warnings = 0;
4337
4338	sec = find_section_by_name(file->elf, ".noinstr.text");
4339	if (sec) {
4340	warnings += validate_section(file, sec);
4341	warnings += validate_unwind_hints(file, sec);
4342	}
4343
4344	sec = find_section_by_name(file->elf, ".entry.text");
4345	if (sec) {
4346	warnings += validate_section(file, sec);
4347	warnings += validate_unwind_hints(file, sec);
4348	}
4349
4350	sec = find_section_by_name(file->elf, ".cpuidle.text");
4351	if (sec) {
4352	warnings += validate_section(file, sec);
4353	warnings += validate_unwind_hints(file, sec);
4354	}
4355
4356	return warnings;
4357	}
4358
4359	static int validate_functions(struct objtool_file *file)
4360	{
4361	struct section *sec;
4362	int warnings = 0;
4363
4364	for_each_sec(file, sec) {
4365	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4366	continue;
4367
4368	warnings += validate_section(file, sec);
4369	}
4370
4371	return warnings;
4372	}
4373
4374	static void mark_endbr_used(struct instruction *insn)
4375	{
4376	if (!list_empty(&insn->call_node))
4377	list_del_init(&insn->call_node);
4378	}
4379
4380	static bool noendbr_range(struct objtool_file file, struct instruction insn)
4381	{
4382	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4383	struct instruction *first;
4384
4385	if (!sym)
4386	return false;
4387
4388	first = find_insn(file, sym->sec, sym->offset);
4389	if (!first)
4390	return false;
4391
4392	if (first->type != INSN_ENDBR && !first->noendbr)
4393	return false;
4394
4395	return insn->offset == sym->offset + sym->len;
4396	}
4397
4398	static int __validate_ibt_insn(struct objtool_file file, struct instruction insn,
4399	struct instruction *dest)
4400	{
4401	if (dest->type == INSN_ENDBR) {
4402	mark_endbr_used(dest);
4403	return 0;
4404	}
4405
4406	if (insn_func(dest) && insn_func(insn) &&
4407	insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4408	/*
4409	* Anything from->to self is either _THIS_IP_ or
4410	* IRET-to-self.
4411	*
4412	* There is no sane way to annotate _THIS_IP_ since the
4413	* compiler treats the relocation as a constant and is
4414	* happy to fold in offsets, skewing any annotation we
4415	* do, leading to vast amounts of false-positives.
4416	*
4417	* There's also compiler generated _THIS_IP_ through
4418	* KCOV and such which we have no hope of annotating.
4419	*
4420	* As such, blanket accept self-references without
4421	* issue.
4422	*/
4423	return 0;
4424	}
4425
4426	/*
4427	* Accept anything ANNOTATE_NOENDBR.
4428	*/
4429	if (dest->noendbr)
4430	return 0;
4431
4432	/*
4433	* Accept if this is the instruction after a symbol
4434	* that is (no)endbr -- typical code-range usage.
4435	*/
4436	if (noendbr_range(file, dest))
4437	return 0;
4438
4439	WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4440	return 1;
4441	}
4442
4443	static int validate_ibt_insn(struct objtool_file file, struct instruction insn)
4444	{
4445	struct instruction *dest;
4446	struct reloc *reloc;
4447	unsigned long off;
4448	int warnings = 0;
4449
4450	/*
4451	* Looking for function pointer load relocations. Ignore
4452	* direct/indirect branches:
4453	*/
4454	switch (insn->type) {
4455
4456	case INSN_CALL:
4457	case INSN_CALL_DYNAMIC:
4458	case INSN_JUMP_CONDITIONAL:
4459	case INSN_JUMP_UNCONDITIONAL:
4460	case INSN_JUMP_DYNAMIC:
4461	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4462	case INSN_RETURN:
4463	case INSN_NOP:
4464	return 0;
4465
4466	case INSN_LEA_RIP:
4467	if (!insn_reloc(file, insn)) {
4468	/* local function pointer reference without reloc */
4469
4470	off = arch_jump_destination(insn);
4471
4472	dest = find_insn(file, insn->sec, off);
4473	if (!dest) {
4474	WARN_INSN(insn, "corrupt function pointer reference");
4475	return 1;
4476	}
4477
4478	return __validate_ibt_insn(file, insn, dest);
4479	}
4480	break;
4481
4482	default:
4483	break;
4484	}
4485
4486	for (reloc = insn_reloc(file, insn);
4487	reloc;
4488	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4489	reloc_offset(reloc) + 1,
4490	(insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4491
4492	off = reloc->sym->offset;
4493	if (reloc_type(reloc) == R_X86_64_PC32 \|\|
4494	reloc_type(reloc) == R_X86_64_PLT32)
4495	off += arch_dest_reloc_offset(reloc_addend(reloc));
4496	else
4497	off += reloc_addend(reloc);
4498
4499	dest = find_insn(file, reloc->sym->sec, off);
4500	if (!dest)
4501	continue;
4502
4503	warnings += __validate_ibt_insn(file, insn, dest);
4504	}
4505
4506	return warnings;
4507	}
4508
4509	static int validate_ibt_data_reloc(struct objtool_file *file,
4510	struct reloc *reloc)
4511	{
4512	struct instruction *dest;
4513
4514	dest = find_insn(file, reloc->sym->sec,
4515	reloc->sym->offset + reloc_addend(reloc));
4516	if (!dest)
4517	return 0;
4518
4519	if (dest->type == INSN_ENDBR) {
4520	mark_endbr_used(dest);
4521	return 0;
4522	}
4523
4524	if (dest->noendbr)
4525	return 0;
4526
4527	WARN_FUNC("data relocation to !ENDBR: %s",
4528	reloc->sec->base, reloc_offset(reloc),
4529	offstr(dest->sec, dest->offset));
4530
4531	return 1;
4532	}
4533
4534	/*
4535	* Validate IBT rules and remove used ENDBR instructions from the seal list.
4536	* Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4537	* NOPs) later, in create_ibt_endbr_seal_sections().
4538	*/
4539	static int validate_ibt(struct objtool_file *file)
4540	{
4541	struct section *sec;
4542	struct reloc *reloc;
4543	struct instruction *insn;
4544	int warnings = 0;
4545
4546	for_each_insn(file, insn)
4547	warnings += validate_ibt_insn(file, insn);
4548
4549	for_each_sec(file, sec) {
4550
4551	/* Already done by validate_ibt_insn() */
4552	if (sec->sh.sh_flags & SHF_EXECINSTR)
4553	continue;
4554
4555	if (!sec->rsec)
4556	continue;
4557
4558	/*
4559	* These sections can reference text addresses, but not with
4560	* the intent to indirect branch to them.
4561	*/
4562	if ((!strncmp(sec->name, ".discard", 8) &&
4563	strcmp(sec->name, ".discard.ibt_endbr_noseal")) \|\|
4564	!strncmp(sec->name, ".debug", 6) \|\|
4565	!strcmp(sec->name, ".altinstructions") \|\|
4566	!strcmp(sec->name, ".ibt_endbr_seal") \|\|
4567	!strcmp(sec->name, ".orc_unwind_ip") \|\|
4568	!strcmp(sec->name, ".parainstructions") \|\|
4569	!strcmp(sec->name, ".retpoline_sites") \|\|
4570	!strcmp(sec->name, ".smp_locks") \|\|
4571	!strcmp(sec->name, ".static_call_sites") \|\|
4572	!strcmp(sec->name, "_error_injection_whitelist") \|\|
4573	!strcmp(sec->name, "_kprobe_blacklist") \|\|
4574	!strcmp(sec->name, "__bug_table") \|\|
4575	!strcmp(sec->name, "__ex_table") \|\|
4576	!strcmp(sec->name, "__jump_table") \|\|
4577	!strcmp(sec->name, "__mcount_loc") \|\|
4578	!strcmp(sec->name, ".kcfi_traps") \|\|
4579	!strcmp(sec->name, ".llvm.call-graph-profile") \|\|
4580	!strcmp(sec->name, ".llvm_bb_addr_map") \|\|
4581	!strcmp(sec->name, "__tracepoints") \|\|
4582	strstr(sec->name, "__patchable_function_entries"))
4583	continue;
4584
4585	for_each_reloc(sec->rsec, reloc)
4586	warnings += validate_ibt_data_reloc(file, reloc);
4587	}
4588
4589	return warnings;
4590	}
4591
4592	static int validate_sls(struct objtool_file *file)
4593	{
4594	struct instruction insn, next_insn;
4595	int warnings = 0;
4596
4597	for_each_insn(file, insn) {
4598	next_insn = next_insn_same_sec(file, insn);
4599
4600	if (insn->retpoline_safe)
4601	continue;
4602
4603	switch (insn->type) {
4604	case INSN_RETURN:
4605	if (!next_insn \|\| next_insn->type != INSN_TRAP) {
4606	WARN_INSN(insn, "missing int3 after ret");
4607	warnings++;
4608	}
4609
4610	break;
4611	case INSN_JUMP_DYNAMIC:
4612	if (!next_insn \|\| next_insn->type != INSN_TRAP) {
4613	WARN_INSN(insn, "missing int3 after indirect jump");
4614	warnings++;
4615	}
4616	break;
4617	default:
4618	break;
4619	}
4620	}
4621
4622	return warnings;
4623	}
4624
4625	static bool ignore_noreturn_call(struct instruction *insn)
4626	{
4627	struct symbol *call_dest = insn_call_dest(insn);
4628
4629	/*
4630	* FIXME: hack, we need a real noreturn solution
4631	*
4632	* Problem is, exc_double_fault() may or may not return, depending on
4633	* whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4634	* to the kernel config.
4635	*
4636	* Other potential ways to fix it:
4637	*
4638	* - have compiler communicate __noreturn functions somehow
4639	* - remove CONFIG_X86_ESPFIX64
4640	* - read the .config file
4641	* - add a cmdline option
4642	* - create a generic objtool annotation format (vs a bunch of custom
4643	* formats) and annotate it
4644	*/
4645	if (!strcmp(call_dest->name, "exc_double_fault")) {
4646	/* prevent further unreachable warnings for the caller */
4647	insn->sym->warned = 1;
4648	return true;
4649	}
4650
4651	return false;
4652	}
4653
4654	static int validate_reachable_instructions(struct objtool_file *file)
4655	{
4656	struct instruction insn, prev_insn;
4657	struct symbol *call_dest;
4658	int warnings = 0;
4659
4660	if (file->ignore_unreachables)
4661	return 0;
4662
4663	for_each_insn(file, insn) {
4664	if (insn->visited \|\| ignore_unreachable_insn(file, insn))
4665	continue;
4666
4667	prev_insn = prev_insn_same_sec(file, insn);
4668	if (prev_insn && prev_insn->dead_end) {
4669	call_dest = insn_call_dest(prev_insn);
4670	if (call_dest && !ignore_noreturn_call(prev_insn)) {
4671	WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4672	call_dest->name);
4673	warnings++;
4674	continue;
4675	}
4676	}
4677
4678	WARN_INSN(insn, "unreachable instruction");
4679	warnings++;
4680	}
4681
4682	return warnings;
4683	}
4684
4685	/* 'funcs' is a space-separated list of function names */
4686	static int disas_funcs(const char *funcs)
4687	{
4688	const char objdump_str, cross_compile;
4689	int size, ret;
4690	char *cmd;
4691
4692	cross_compile = getenv("CROSS_COMPILE");
4693
4694	objdump_str = "%sobjdump -wdr %s \| gawk -M -v _funcs='%s' '"
4695	"BEGIN { split(_funcs, funcs); }"
4696	"/^$/ { func_match = 0; }"
4697	"/<.*>:/ { "
4698	"f = gensub(/.<(.)>:/, \"\\\\1\", 1);"
4699	"for (i in funcs) {"
4700	"if (funcs[i] == f) {"
4701	"func_match = 1;"
4702	"base = strtonum(\"0x\" $1);"
4703	"break;"
4704	"}"
4705	"}"
4706	"}"
4707	"{"
4708	"if (func_match) {"
4709	"addr = strtonum(\"0x\" $1);"
4710	"printf(\"%%04x \", addr - base);"
4711	"print;"
4712	"}"
4713	"}' 1>&2";
4714
4715	/* fake snprintf() to calculate the size */
4716	size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4717	if (size <= 0) {
4718	WARN("objdump string size calculation failed");
4719	return -1;
4720	}
4721
4722	cmd = malloc(size);
4723
4724	/* real snprintf() */
4725	snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4726	ret = system(cmd);
4727	if (ret) {
4728	WARN("disassembly failed: %d", ret);
4729	return -1;
4730	}
4731
4732	return 0;
4733	}
4734
4735	static int disas_warned_funcs(struct objtool_file *file)
4736	{
4737	struct symbol *sym;
4738	char funcs = NULL, tmp;
4739
4740	for_each_sym(file, sym) {
4741	if (sym->warned) {
4742	if (!funcs) {
4743	funcs = malloc(strlen(sym->name) + 1);
4744	strcpy(funcs, sym->name);
4745	} else {
4746	tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4747	sprintf(tmp, "%s %s", funcs, sym->name);
4748	free(funcs);
4749	funcs = tmp;
4750	}
4751	}
4752	}
4753
4754	if (funcs)
4755	disas_funcs(funcs);
4756
4757	return 0;
4758	}
4759
4760	struct insn_chunk {
4761	void *addr;
4762	struct insn_chunk *next;
4763	};
4764
4765	/*
4766	* Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4767	* which can trigger more allocations for .debug_* sections whose data hasn't
4768	* been read yet.
4769	*/
4770	static void free_insns(struct objtool_file *file)
4771	{
4772	struct instruction *insn;
4773	struct insn_chunk chunks = NULL, chunk;
4774
4775	for_each_insn(file, insn) {
4776	if (!insn->idx) {
4777	chunk = malloc(sizeof(*chunk));
4778	chunk->addr = insn;
4779	chunk->next = chunks;
4780	chunks = chunk;
4781	}
4782	}
4783
4784	for (chunk = chunks; chunk; chunk = chunk->next)
4785	free(chunk->addr);
4786	}
4787
4788	int check(struct objtool_file *file)
4789	{
4790	int ret, warnings = 0;
4791
4792	arch_initial_func_cfi_state(&initial_func_cfi);
4793	init_cfi_state(&init_cfi);
4794	init_cfi_state(&func_cfi);
4795	set_func_state(&func_cfi);
4796	init_cfi_state(&force_undefined_cfi);
4797	force_undefined_cfi.force_undefined = true;
4798
4799	if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4800	goto out;
4801
4802	cfi_hash_add(&init_cfi);
4803	cfi_hash_add(&func_cfi);
4804
4805	ret = decode_sections(file);
4806	if (ret < 0)
4807	goto out;
4808
4809	warnings += ret;
4810
4811	if (!nr_insns)
4812	goto out;
4813
4814	if (opts.retpoline) {
4815	ret = validate_retpoline(file);
4816	if (ret < 0)
4817	return ret;
4818	warnings += ret;
4819	}
4820
4821	if (opts.stackval \|\| opts.orc \|\| opts.uaccess) {
4822	ret = validate_functions(file);
4823	if (ret < 0)
4824	goto out;
4825	warnings += ret;
4826
4827	ret = validate_unwind_hints(file, NULL);
4828	if (ret < 0)
4829	goto out;
4830	warnings += ret;
4831
4832	if (!warnings) {
4833	ret = validate_reachable_instructions(file);
4834	if (ret < 0)
4835	goto out;
4836	warnings += ret;
4837	}
4838
4839	} else if (opts.noinstr) {
4840	ret = validate_noinstr_sections(file);
4841	if (ret < 0)
4842	goto out;
4843	warnings += ret;
4844	}
4845
4846	if (opts.unret) {
4847	/*
4848	* Must be after validate_branch() and friends, it plays
4849	* further games with insn->visited.
4850	*/
4851	ret = validate_unrets(file);
4852	if (ret < 0)
4853	return ret;
4854	warnings += ret;
4855	}
4856
4857	if (opts.ibt) {
4858	ret = validate_ibt(file);
4859	if (ret < 0)
4860	goto out;
4861	warnings += ret;
4862	}
4863
4864	if (opts.sls) {
4865	ret = validate_sls(file);
4866	if (ret < 0)
4867	goto out;
4868	warnings += ret;
4869	}
4870
4871	if (opts.static_call) {
4872	ret = create_static_call_sections(file);
4873	if (ret < 0)
4874	goto out;
4875	warnings += ret;
4876	}
4877
4878	if (opts.retpoline) {
4879	ret = create_retpoline_sites_sections(file);
4880	if (ret < 0)
4881	goto out;
4882	warnings += ret;
4883	}
4884
4885	if (opts.cfi) {
4886	ret = create_cfi_sections(file);
4887	if (ret < 0)
4888	goto out;
4889	warnings += ret;
4890	}
4891
4892	if (opts.rethunk) {
4893	ret = create_return_sites_sections(file);
4894	if (ret < 0)
4895	goto out;
4896	warnings += ret;
4897
4898	if (opts.hack_skylake) {
4899	ret = create_direct_call_sections(file);
4900	if (ret < 0)
4901	goto out;
4902	warnings += ret;
4903	}
4904	}
4905
4906	if (opts.mcount) {
4907	ret = create_mcount_loc_sections(file);
4908	if (ret < 0)
4909	goto out;
4910	warnings += ret;
4911	}
4912
4913	if (opts.prefix) {
4914	ret = add_prefix_symbols(file);
4915	if (ret < 0)
4916	return ret;
4917	warnings += ret;
4918	}
4919
4920	if (opts.ibt) {
4921	ret = create_ibt_endbr_seal_sections(file);
4922	if (ret < 0)
4923	goto out;
4924	warnings += ret;
4925	}
4926
4927	if (opts.orc && nr_insns) {
4928	ret = orc_create(file);
4929	if (ret < 0)
4930	goto out;
4931	warnings += ret;
4932	}
4933
4934	free_insns(file);
4935
4936	if (opts.verbose)
4937	disas_warned_funcs(file);
4938
4939	if (opts.stats) {
4940	printf("nr_insns_visited: %ld\n", nr_insns_visited);
4941	printf("nr_cfi: %ld\n", nr_cfi);
4942	printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4943	printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4944	}
4945
4946	out:
4947	/*
4948	* For now, don't fail the kernel build on fatal warnings. These
4949	* errors are still fairly common due to the growing matrix of
4950	* supported toolchains and their recent pace of change.
4951	*/
4952	return 0;
4953	}