projects/linux·blob v7.2 · refs

2213 lines · 55.5 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2#define _GNU_SOURCE /* memmem() */
3#include <subcmd/parse-options.h>
4#include <stdlib.h>
5#include <string.h>
6#include <libgen.h>
7#include <stdio.h>
8#include <ctype.h>
9
10#include <objtool/objtool.h>
11#include <objtool/warn.h>
12#include <objtool/arch.h>
13#include <objtool/klp.h>
14#include <objtool/util.h>
15#include <arch/special.h>
16
17#include <linux/align.h>
18#include <linux/objtool_types.h>
19#include <linux/livepatch_external.h>
20#include <linux/stringify.h>
21#include <linux/string.h>
22#include <linux/jhash.h>
23
24#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
25
26struct elfs {
27 struct elf *orig, *patched, *out;
28 const char *modname;
29};
30
31struct export {
32 struct hlist_node hash;
33 char *mod, *sym;
34};
35
36bool debug, debug_correlate, debug_clone;
37int indent;
38
39static const char * const klp_diff_usage[] = {
40 "objtool klp diff [<options>] <in1.o> <in2.o> <out.o>",
41 NULL,
42};
43
44static const struct option klp_diff_options[] = {
45 OPT_GROUP("Options:"),
46 OPT_BOOLEAN('d', "debug", &debug, "enable all debug output"),
47 OPT_BOOLEAN(0, "debug-correlate", &debug_correlate, "enable correlation debug output"),
48 OPT_BOOLEAN(0, "debug-clone", &debug_clone, "enable cloning debug output"),
49 OPT_END(),
50};
51
52static DEFINE_HASHTABLE(exports, 15);
53
54static char *escape_str(const char *orig)
55{
56 size_t len = 0;
57 const char *a;
58 char *b, *new;
59
60 for (a = orig; *a; a++) {
61 switch (*a) {
62 case '\001': len += 5; break;
63 case '\n':
64 case '\t': len += 2; break;
65 default: len++;
66 }
67 }
68
69 new = malloc(len + 1);
70 if (!new)
71 return NULL;
72
73 for (a = orig, b = new; *a; a++) {
74 switch (*a) {
75 case '\001': memcpy(b, "<SOH>", 5); b += 5; break;
76 case '\n': *b++ = '\\'; *b++ = 'n'; break;
77 case '\t': *b++ = '\\'; *b++ = 't'; break;
78 default: *b++ = *a;
79 }
80 }
81
82 *b = '\0';
83 return new;
84}
85
86static int read_exports(void)
87{
88 const char *symvers = "Module.symvers";
89 char line[1024], *path = NULL;
90 unsigned int line_num = 1;
91 FILE *file;
92
93 file = fopen(symvers, "r");
94 if (!file) {
95 path = top_level_dir(symvers);
96 if (!path) {
97 ERROR("can't open '%s', \"objtool diff\" should be run from the kernel tree", symvers);
98 return -1;
99 }
100
101 file = fopen(path, "r");
102 if (!file) {
103 ERROR_GLIBC("fopen");
104 return -1;
105 }
106 }
107
108 while (fgets(line, 1024, file)) {
109 char *sym, *mod, *type;
110 struct export *export;
111
112 sym = strchr(line, '\t');
113 if (!sym) {
114 ERROR("malformed Module.symvers (sym) at line %d", line_num);
115 return -1;
116 }
117
118 *sym++ = '\0';
119
120 mod = strchr(sym, '\t');
121 if (!mod) {
122 ERROR("malformed Module.symvers (mod) at line %d", line_num);
123 return -1;
124 }
125
126 *mod++ = '\0';
127
128 type = strchr(mod, '\t');
129 if (!type) {
130 ERROR("malformed Module.symvers (type) at line %d", line_num);
131 return -1;
132 }
133
134 *type++ = '\0';
135
136 if (*sym == '\0' || *mod == '\0') {
137 ERROR("malformed Module.symvers at line %d", line_num);
138 return -1;
139 }
140
141 export = calloc(1, sizeof(*export));
142 if (!export) {
143 ERROR_GLIBC("calloc");
144 return -1;
145 }
146
147 export->mod = strdup(mod);
148 if (!export->mod) {
149 ERROR_GLIBC("strdup");
150 return -1;
151 }
152
153 export->sym = strdup(sym);
154 if (!export->sym) {
155 ERROR_GLIBC("strdup");
156 return -1;
157 }
158
159 hash_add(exports, &export->hash, str_hash(sym));
160 }
161
162 free(path);
163 fclose(file);
164
165 return 0;
166}
167
168static int read_sym_checksums(struct elf *elf)
169{
170 struct section *sec;
171
172 sec = find_section_by_name(elf, ".discard.sym_checksum");
173 if (!sec) {
174 ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool klp checksum'?",
175 elf->name);
176 return -1;
177 }
178
179 if (!sec->rsec) {
180 ERROR("missing reloc section for .discard.sym_checksum");
181 return -1;
182 }
183
184 if (sec_size(sec) % sizeof(struct sym_checksum)) {
185 ERROR("struct sym_checksum size mismatch");
186 return -1;
187 }
188
189 for (int i = 0; i < sec_size(sec) / sizeof(struct sym_checksum); i++) {
190 struct sym_checksum *sym_checksum;
191 struct reloc *reloc;
192 struct symbol *sym;
193
194 sym_checksum = (struct sym_checksum *)sec->data->d_buf + i;
195
196 reloc = find_reloc_by_dest(elf, sec, i * sizeof(*sym_checksum));
197 if (!reloc) {
198 ERROR("can't find reloc for sym_checksum[%d]", i);
199 return -1;
200 }
201
202 sym = reloc->sym;
203
204 if (is_sec_sym(sym)) {
205 ERROR("not sure how to handle section %s", sym->name);
206 return -1;
207 }
208
209 if (is_func_sym(sym) || is_object_sym(sym))
210 sym->csum.checksum = sym_checksum->checksum;
211 }
212
213 return 0;
214}
215
216static struct symbol *first_file_symbol(struct elf *elf)
217{
218 struct symbol *sym;
219
220 for_each_sym(elf, sym) {
221 if (is_file_sym(sym))
222 return sym;
223 }
224
225 return NULL;
226}
227
228static struct symbol *next_file_symbol(struct elf *elf, struct symbol *sym)
229{
230 for_each_sym_continue(elf, sym) {
231 if (is_file_sym(sym))
232 return sym;
233 }
234
235 return NULL;
236}
237
238/*
239 * Certain static local variables should never be correlated. They will be
240 * used in place rather than referencing the originals.
241 */
242static bool is_uncorrelated_static_local(struct symbol *sym)
243{
244 static const char * const vars[] = {
245 "__already_done",
246 "__func__",
247 "__key",
248 "__warned",
249 "_entry",
250 "_entry_ptr",
251 "_rs",
252 "descriptor",
253 "CSWTCH",
254 };
255 const char *dot;
256
257 if (!is_object_sym(sym) || !is_local_sym(sym))
258 return false;
259
260 /* WARN_ONCE, etc */
261 if (!strcmp(sym->sec->name, ".data..once"))
262 return true;
263
264 dot = strchr(sym->name, '.');
265 if (!dot)
266 return false;
267
268 for (int i = 0; i < ARRAY_SIZE(vars); i++) {
269 size_t len = strlen(vars[i]);
270
271 /* GCC: <var>.<id> */
272 if (strstarts(sym->name, vars[i]) && (sym->name[len] == '.'))
273 return true;
274
275 /* Clang: <func>.<var>[.<id>] */
276 if (strstarts(dot + 1, vars[i]) &&
277 (dot[1 + len] == '.' || dot[1 + len] == '\0'))
278 return true;
279 }
280
281 return false;
282}
283
284/*
285 * .L symbols are assembler-local labels not present in kallsyms. They must
286 * never become KLP relocations; instead their data is cloned into the patch
287 * module. This covers .Ltmp* (Clang temp labels), .L__const.* (Clang local
288 * constants), and any other assembler-local pattern.
289 */
290static bool is_local_label(struct symbol *sym)
291{
292 return strstarts(sym->name, ".L");
293}
294
295static bool is_special_section(struct section *sec)
296{
297 static const char * const specials[] = {
298 ".altinstructions",
299 ".kcfi_traps",
300 ".smp_locks",
301 "__bug_table",
302 "__ex_table",
303 "__jump_table",
304 "__mcount_loc",
305
306 /*
307 * Extract .static_call_sites here to inherit non-module
308 * preferential treatment. The later static call processing
309 * during klp module build will be skipped when it sees this
310 * section already exists.
311 */
312 ".static_call_sites",
313 };
314
315 static const char * const non_special_discards[] = {
316 ".discard.addressable",
317 ".discard.sym_checksum",
318 };
319
320 if (is_text_sec(sec))
321 return false;
322
323 for (int i = 0; i < ARRAY_SIZE(specials); i++) {
324 if (!strcmp(sec->name, specials[i]))
325 return true;
326 }
327
328 /* Most .discard data sections are special */
329 for (int i = 0; i < ARRAY_SIZE(non_special_discards); i++) {
330 if (!strcmp(sec->name, non_special_discards[i]))
331 return false;
332 }
333
334 return strstarts(sec->name, ".discard.");
335}
336
337/*
338 * These sections are referenced by special sections but aren't considered
339 * special sections themselves.
340 */
341static bool is_special_section_aux(struct section *sec)
342{
343 static const char * const specials_aux[] = {
344 ".altinstr_replacement",
345 ".altinstr_aux",
346 };
347
348 for (int i = 0; i < ARRAY_SIZE(specials_aux); i++) {
349 if (!strcmp(sec->name, specials_aux[i]))
350 return true;
351 }
352
353 return false;
354}
355
356/*
357 * Symbols created by ___ADDRESSABLE() are only used to convince the toolchain
358 * not to optimize out the referenced symbol.
359 */
360static bool is_addressable_sym(struct symbol *sym)
361{
362 return !strcmp(sym->sec->name, ".discard.addressable");
363}
364
365/*
366 * ABS symbols are typically assembly .set/.equ constants which are never
367 * referenced by relocations. (Exclude FILE symbols which are also SHN_ABS.)
368 */
369static bool is_abs_sym(struct symbol *sym)
370{
371 return sym->sym.st_shndx == SHN_ABS && !is_file_sym(sym);
372}
373
374static bool is_initcall_sym(struct symbol *sym)
375{
376 return strstarts(sym->name, "__initcall__") ||
377 strstarts(sym->name, "__initstub__");
378}
379
380/*
381 * Some .rodata is anonymous and can't be correlated due to there being no
382 * symbol names.
383 *
384 * The .rodata.cst* sections aren't technically anonymous, they're SHF_MERGE
385 * constant pool sections containing small fixed-size data (lookup tables,
386 * bitmasks) which are only read by value, so pointer equivalence isn't needed.
387 * They are typically referenced by UBSAN data sections.
388 */
389static bool is_anonymous_rodata(struct symbol *sym)
390{
391 return is_rodata_sec(sym->sec) &&
392 (!is_object_sym(sym) || strstarts(sym->sec->name, ".rodata.cst"));
393}
394
395/*
396 * These symbols should never be correlated, so their local patched versions
397 * are used instead of linking to the originals.
398 */
399static bool dont_correlate(struct symbol *sym)
400{
401 return is_file_sym(sym) ||
402 is_null_sym(sym) ||
403 is_sec_sym(sym) ||
404 is_abs_sym(sym) ||
405 is_prefix_func(sym) ||
406 is_uncorrelated_static_local(sym) ||
407 is_local_label(sym) ||
408 is_string_sec(sym->sec) ||
409 is_anonymous_rodata(sym) ||
410 is_initcall_sym(sym) ||
411 is_addressable_sym(sym) ||
412 is_special_section(sym->sec) ||
413 is_special_section_aux(sym->sec);
414}
415
416static const char *llvm_suffix(const char *name)
417{
418 return strstr(name, ".llvm.");
419}
420
421static bool is_llvm_sym(struct symbol *sym)
422{
423 return llvm_suffix(sym->name);
424}
425
426/*
427 * Determine if two symbols have compatible source file origins:
428 *
429 * - If both symbols are local, only return true if they belong to the same
430 * ELF file symbol.
431 *
432 * - If both symbols are global, always return true, as globals don't have
433 * file associations.
434 *
435 * - If they have different scopes, also return true, as the patch might have
436 * changed the symbol's scope.
437 *
438 * Works for both same-ELF (direct pointer compare) and cross-ELF
439 * (compare via file->twin) cases.
440 */
441static bool maybe_same_file(struct symbol *sym1, struct symbol *sym2)
442{
443 if (!sym1->file || !sym2->file)
444 return true;
445 if (sym1->file == sym2->file)
446 return true;
447 return sym1->file->twin == sym2->file;
448}
449
450/*
451 * Similar to maybe_same_file(), but strict: no scope changes allowed.
452 *
453 * Works for both same-ELF (direct pointer compare) and cross-ELF
454 * (compare via file->twin) cases.
455 */
456static bool same_file(struct symbol *sym1, struct symbol *sym2)
457{
458 if (llvm_suffix(sym1->name) && llvm_suffix(sym2->name))
459 return true;
460 if (!sym1->file && !sym2->file)
461 return true;
462 if (!sym1->file || !sym2->file)
463 return false;
464 if (sym1->file == sym2->file)
465 return true;
466 return sym1->file->twin == sym2->file;
467}
468
469/*
470 * Is it a local symbol, or at least was it local in the translation unit
471 * before LLVM promoted it?
472 */
473static bool is_tu_local_sym(struct symbol *sym)
474{
475 return is_local_sym(sym) || is_llvm_sym(sym);
476}
477
478/*
479 * Try to find sym1's twin in patched using deterministic matching.
480 *
481 * Multiple symbols can share a demangled name (e.g., static functions in
482 * different TUs). This function counts same-named candidates through a
483 * funnel of progressively tighter filters. Each level is a strict subset
484 * of the previous one.
485 *
486 * The widest level that yields a 1:1 match wins. Narrower levels are only
487 * needed when the wider level is ambiguous (count > 1).
488 *
489 * Candidates are pre-filtered by maybe_same_file(), which narrows most
490 * local symbols to their own TU. For example, 19 different static
491 * type_show() functions across vmlinux.o each see only one candidate after
492 * pre-filtering, so they match immediately at Level 1.
493 *
494 * Level 1 (name): Works when the demangled name is unique after
495 * pre-filtering. Handles most symbols: unique globals like copy_signal(),
496 * or per-TU locals like pcspkr_probe().
497 *
498 * Level 2 (scope): Filters by local-vs-global (TU-local-vs-not). Example:
499 * parse_header() exists as both a static and a global function. Level 1
500 * sees both (same demangled name), but Level 2 separates them by scope.
501 *
502 * Level 3 (file): Strict file matching via same_file(), which rejects scope
503 * changes. Example: LLVM-promoted foo.llvm.12345 (global, no FILE symbol)
504 * vs genuine local foo (has FILE symbol). Both are TU-local so Level 2
505 * can't distinguish them, but same_file() rejects the pair because one has
506 * a file association and the other doesn't.
507 *
508 * Level 4 (checksum): Distinguishes by function checksum. Example:
509 * usb_devnode.llvm.AAA and usb_devnode.llvm.BBB are two LLVM-promoted
510 * functions from different TUs with the same demangled name. After a TU
511 * change, the .llvm. hashes change but the functions themselves may be
512 * unchanged. Level 4 matches each to the patched candidate with the
513 * same checksum.
514 */
515static struct symbol *find_twin(struct elfs *e, struct symbol *sym1)
516{
517 struct symbol *name_last = NULL, *scope_last = NULL,
518 *file_last = NULL, *csum_last = NULL;
519 unsigned int name_orig = 0, name_patched = 0;
520 unsigned int scope_orig = 0, scope_patched = 0;
521 unsigned int file_orig = 0, file_patched = 0;
522 unsigned int csum_orig = 0, csum_patched = 0;
523 struct symbol *sym2, *match = NULL;
524
525 /* Count orig candidates */
526 for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) {
527 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
528 (!maybe_same_file(sym1, sym2)))
529 continue;
530
531 /* Level 1: name match (widest filter) */
532 name_orig++;
533
534 /* Level 2: scope (scope changes allowed) */
535 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2))
536 continue;
537 scope_orig++;
538
539 /* Level 3: file (scope changes disallowed) */
540 if (!same_file(sym1, sym2))
541 continue;
542 file_orig++;
543
544 /* Level 4: checksum (unchanged symbols) */
545 if (sym1->len != sym2->len || !sym1->csum.checksum ||
546 sym1->csum.checksum != sym2->csum.checksum)
547 continue;
548 csum_orig++;
549 }
550
551 /* Count patched candidates */
552 for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) {
553 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
554 !maybe_same_file(sym1, sym2))
555 continue;
556
557 /* Level 1 */
558 name_patched++;
559 name_last = sym2;
560
561 /* Level 2 */
562 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2))
563 continue;
564 scope_patched++;
565 scope_last = sym2;
566
567 /* Level 3 */
568 if (!same_file(sym1, sym2))
569 continue;
570 file_patched++;
571 file_last = sym2;
572
573 /* Level 4 */
574 if (sym1->len != sym2->len || !sym1->csum.checksum ||
575 sym1->csum.checksum != sym2->csum.checksum)
576 continue;
577 csum_patched++;
578 csum_last = sym2;
579 }
580
581 /* Return the widest level that yields a unique (1:1) match */
582 if (name_orig == 1 && name_patched == 1)
583 match = name_last;
584 else if (scope_orig == 1 && scope_patched == 1)
585 match = scope_last;
586 else if (file_orig == 1 && file_patched == 1)
587 match = file_last;
588 else if (csum_orig == 1 && csum_patched == 1)
589 match = csum_last;
590
591 if (!match)
592 return NULL;
593
594 if (name_orig != 1 || name_patched != 1)
595 dbg_correlate("find_twin(): %s%s -> %s%s",
596 sym1->name, is_func_sym(sym1) ? "()" : "",
597 match->name, is_func_sym(match) ? "()" : "");
598
599 return match;
600}
601
602struct llvm_suffix_pair {
603 struct hlist_node hash;
604 const char *orig;
605 const char *patched;
606};
607
608static DECLARE_HASHTABLE(suffix_map, 7);
609
610/*
611 * Build a mapping of known orig-to-patched LLVM suffixes based on
612 * already-correlated symbol pairs. All promoted symbols from the same TU
613 * share the same .llvm.<hash> suffix, so one correlated pair seeds the map
614 * for the entire TU.
615 */
616static int update_suffix_map(struct elf *elf)
617{
618 struct llvm_suffix_pair *entry;
619 struct symbol *sym;
620
621 for_each_sym(elf, sym) {
622 const char *s1, *s2;
623 bool found;
624
625 if (!sym->twin)
626 continue;
627
628 s1 = llvm_suffix(sym->name);
629 s2 = llvm_suffix(sym->twin->name);
630
631 if (!s1 || !s2)
632 continue;
633
634 found = false;
635 hash_for_each_possible(suffix_map, entry, hash, str_hash(s1)) {
636 if (!strcmp(entry->orig, s1)) {
637 found = true;
638 break;
639 }
640 }
641 if (found)
642 continue;
643
644 entry = calloc(1, sizeof(*entry));
645 if (!entry) {
646 ERROR_GLIBC("calloc");
647 return -1;
648 }
649
650 entry->orig = s1;
651 entry->patched = s2;
652 hash_add(suffix_map, &entry->hash, str_hash(s1));
653 }
654
655 return 0;
656}
657
658/*
659 * Match by translating the symbol's .llvm.<hash> suffix through the suffix
660 * map to find the corresponding hash suffix for the patched object.
661 *
662 * Example: In the original kernel, TU drivers/base/core.c contains
663 * foo.llvm.12345 and bar.llvm.12345 (same TU, same hash). After patching,
664 * they become foo.llvm.67890 and bar.llvm.67890. If foo was already
665 * correlated by find_twin() (e.g., unique by name), the suffix map records
666 * .llvm.12345 -> .llvm.67890. When processing bar.llvm.12345, this
667 * function looks up .llvm.12345, gets .llvm.67890, constructs the name
668 * bar.llvm.67890, and finds the match.
669 */
670static struct symbol *find_twin_suffixed(struct elf *elf, struct symbol *sym1)
671{
672 const char *suffix, *patched_suffix = NULL;
673 struct symbol *sym2, *match = NULL;
674 char name[SYM_NAME_LEN];
675 struct llvm_suffix_pair *entry;
676 int count = 0;
677
678 suffix = llvm_suffix(sym1->name);
679 if (!suffix)
680 return NULL;
681
682 hash_for_each_possible(suffix_map, entry, hash, str_hash(suffix)) {
683 if (!strcmp(entry->orig, suffix)) {
684 patched_suffix = entry->patched;
685 break;
686 }
687 }
688 if (!patched_suffix)
689 return NULL;
690
691 if (snprintf_check(name, SYM_NAME_LEN, "%s%s",
692 sym1->demangled_name, patched_suffix))
693 return NULL;
694
695 for_each_sym_by_name(elf, name, sym2) {
696 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate)
697 continue;
698 count++;
699 match = sym2;
700 }
701
702 if (count != 1)
703 return NULL;
704
705 dbg_correlate("find_suffixed_twin(): %s%s -> %s%s",
706 sym1->name, is_func_sym(sym1) ? "()" : "",
707 match->name, is_func_sym(match) ? "()" : "");
708
709 return match;
710}
711
712/*
713 * Last-resort positional matching.
714 *
715 * Finds a symbol with the same position in the symbol table among
716 * same-demangled-name candidates, similar to livepatch sympos. Note that
717 * LLVM-promoted symbols are globals, which come after locals in the symbol
718 * table, so we have to be careful not to compare different scopes.
719 *
720 * Example: arch/x86/events/intel/core.c defines many __quirk variables via
721 * X86_MATCH_*() macros. In the symbol table they appear as __quirk.90,
722 * __quirk.97, __quirk.101, etc., all with demangled name __quirk, same
723 * scope, and same FILE symbol. No deterministic filter can distinguish
724 * them, so they're matched by position: the 1st __quirk in orig matches the
725 * 1st in patched, the 2nd matches the 2nd, etc.
726 *
727 * This is less deterministic than the other strategies, so it's done last.
728 */
729static struct symbol *find_twin_positional(struct elfs *e, struct symbol *sym1)
730{
731 unsigned int idx_orig = 0, idx_patched = 0;
732 unsigned int sym1_pos = 0;
733 struct symbol *sym2, *match = NULL;
734
735 for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) {
736 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
737 !maybe_same_file(sym1, sym2))
738 continue;
739 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) ||
740 is_llvm_sym(sym1) != is_llvm_sym(sym2))
741 continue;
742 if (sym1 == sym2)
743 sym1_pos = idx_orig;
744 idx_orig++;
745 }
746
747 for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) {
748 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
749 !maybe_same_file(sym1, sym2))
750 continue;
751 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) ||
752 is_llvm_sym(sym1) != is_llvm_sym(sym2))
753 continue;
754 if (idx_patched == sym1_pos)
755 match = sym2;
756 idx_patched++;
757 }
758
759 if (idx_orig != idx_patched)
760 return NULL;
761
762 dbg_correlate("find_twin_positional(): %s%s -> %s%s",
763 sym1->name, is_func_sym(sym1) ? "()" : "",
764 match->name, is_func_sym(match) ? "()" : "");
765
766 return match;
767}
768
769/*
770 * Correlate symbols between the orig and patched objects. This is a
771 * prerequisite for detecting changed functions, as well as for properly
772 * translating relocations so they point to the correct symbol.
773 */
774static int correlate_symbols(struct elfs *e)
775{
776 struct symbol *file1_sym, *file2_sym;
777 struct symbol *sym1, *sym2;
778 bool progress;
779
780 for_each_sym(e->orig, sym1)
781 sym1->dont_correlate = dont_correlate(sym1);
782 for_each_sym(e->patched, sym2)
783 sym2->dont_correlate = dont_correlate(sym2);
784
785 /* Correlate FILE symbols */
786 file1_sym = first_file_symbol(e->orig);
787 file2_sym = first_file_symbol(e->patched);
788
789 for (; ; file1_sym = next_file_symbol(e->orig, file1_sym),
790 file2_sym = next_file_symbol(e->patched, file2_sym)) {
791
792 if (!file1_sym && file2_sym) {
793 ERROR("FILE symbol mismatch: NULL != %s", file2_sym->name);
794 return -1;
795 }
796
797 if (file1_sym && !file2_sym) {
798 ERROR("FILE symbol mismatch: %s != NULL", file1_sym->name);
799 return -1;
800 }
801
802 if (!file1_sym)
803 break;
804
805 if (strcmp(file1_sym->name, file2_sym->name)) {
806 ERROR("FILE symbol mismatch: %s != %s", file1_sym->name, file2_sym->name);
807 return -1;
808 }
809
810 file1_sym->twin = file2_sym;
811 file2_sym->twin = file1_sym;
812 }
813
814
815 /*
816 * Correlate in two phases: loop deterministic levels until no more
817 * progress, then use positional fallback for the rest. This prevents
818 * the nondeterministic positional matching from stealing symbols that
819 * have deterministic matches.
820 */
821 hash_init(suffix_map);
822 do {
823 progress = false;
824 for_each_sym(e->orig, sym1) {
825 if (sym1->twin || sym1->dont_correlate)
826 continue;
827 sym2 = find_twin(e, sym1);
828 if (!sym2)
829 continue;
830 sym1->twin = sym2;
831 sym2->twin = sym1;
832 progress = true;
833 }
834
835 if (update_suffix_map(e->orig))
836 return -1;
837
838 for_each_sym(e->orig, sym1) {
839 if (sym1->twin || sym1->dont_correlate)
840 continue;
841 sym2 = find_twin_suffixed(e->patched, sym1);
842 if (!sym2)
843 continue;
844 sym1->twin = sym2;
845 sym2->twin = sym1;
846 progress = true;
847 }
848 } while (progress);
849
850 for_each_sym(e->orig, sym1) {
851 if (sym1->twin || sym1->dont_correlate)
852 continue;
853 sym2 = find_twin_positional(e, sym1);
854 if (!sym2)
855 continue;
856 sym1->twin = sym2;
857 sym2->twin = sym1;
858 }
859
860 for_each_sym(e->orig, sym1) {
861 if (sym1->twin || sym1->dont_correlate)
862 continue;
863 WARN("no correlation: %s", sym1->name);
864 }
865
866 return 0;
867}
868
869/* "sympos" is used by livepatch to disambiguate duplicate symbol names */
870static unsigned long find_sympos(struct elf *elf, struct symbol *sym)
871{
872 bool vmlinux = str_ends_with(objname, "vmlinux.o");
873 unsigned long sympos = 0, nr_matches = 0;
874 bool has_dup = false;
875 struct symbol *s;
876
877 if (sym->bind != STB_LOCAL)
878 return 0;
879
880 if (vmlinux && is_func_sym(sym)) {
881 /*
882 * HACK: Unfortunately, symbol ordering can differ between
883 * vmlinux.o and vmlinux due to the linker script emitting
884 * .text.unlikely* before .text*. Count .text.unlikely* first.
885 *
886 * TODO: Disambiguate symbols more reliably (checksums?)
887 */
888 for_each_sym(elf, s) {
889 if (strstarts(s->sec->name, ".text.unlikely") &&
890 !strcmp(s->name, sym->name)) {
891 nr_matches++;
892 if (s == sym)
893 sympos = nr_matches;
894 else
895 has_dup = true;
896 }
897 }
898 for_each_sym(elf, s) {
899 if (!strstarts(s->sec->name, ".text.unlikely") &&
900 !strcmp(s->name, sym->name)) {
901 nr_matches++;
902 if (s == sym)
903 sympos = nr_matches;
904 else
905 has_dup = true;
906 }
907 }
908 } else {
909 for_each_sym(elf, s) {
910 if (!strcmp(s->name, sym->name)) {
911 nr_matches++;
912 if (s == sym)
913 sympos = nr_matches;
914 else
915 has_dup = true;
916 }
917 }
918 }
919
920 if (!sympos) {
921 ERROR("can't find sympos for %s", sym->name);
922 return ULONG_MAX;
923 }
924
925 return has_dup ? sympos : 0;
926}
927
928static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym);
929
930static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym,
931 bool data_too)
932{
933 struct section *out_sec = NULL;
934 unsigned long offset = 0;
935 struct symbol *out_sym;
936
937 if (data_too && !is_undef_sym(patched_sym)) {
938 struct section *patched_sec = patched_sym->sec;
939
940 out_sec = find_section_by_name(elf, patched_sec->name);
941 if (!out_sec) {
942 out_sec = elf_create_section(elf, patched_sec->name, 0,
943 patched_sec->sh.sh_entsize,
944 patched_sec->sh.sh_type,
945 patched_sec->sh.sh_addralign,
946 patched_sec->sh.sh_flags);
947 if (!out_sec)
948 return NULL;
949 }
950
951 if (is_string_sec(patched_sym->sec)) {
952 out_sym = elf_create_section_symbol(elf, out_sec);
953 if (!out_sym)
954 return NULL;
955
956 goto sym_created;
957 }
958
959 if (!is_sec_sym(patched_sym))
960 offset = ALIGN(sec_size(out_sec), out_sec->sh.sh_addralign);
961
962 if (patched_sym->len || is_sec_sym(patched_sym)) {
963 void *data = NULL;
964 size_t size;
965
966 /* bss doesn't have data */
967 if (patched_sym->sec->data && patched_sym->sec->data->d_buf)
968 data = patched_sym->sec->data->d_buf + patched_sym->offset;
969
970 if (is_sec_sym(patched_sym))
971 size = sec_size(patched_sym->sec);
972 else
973 size = patched_sym->len;
974
975 if (!elf_add_data(elf, out_sec, data, size))
976 return NULL;
977 }
978 }
979
980 out_sym = elf_create_symbol(elf, patched_sym->name, out_sec,
981 patched_sym->bind, patched_sym->type,
982 offset, patched_sym->len);
983 if (!out_sym)
984 return NULL;
985
986sym_created:
987 patched_sym->clone = out_sym;
988 out_sym->clone = patched_sym;
989
990 return out_sym;
991}
992
993static const char *sym_type(struct symbol *sym)
994{
995 switch (sym->type) {
996 case STT_NOTYPE: return "NOTYPE";
997 case STT_OBJECT: return "OBJECT";
998 case STT_FUNC: return "FUNC";
999 case STT_SECTION: return "SECTION";
1000 case STT_FILE: return "FILE";
1001 default: return "UNKNOWN";
1002 }
1003}
1004
1005static const char *sym_bind(struct symbol *sym)
1006{
1007 switch (sym->bind) {
1008 case STB_LOCAL: return "LOCAL";
1009 case STB_GLOBAL: return "GLOBAL";
1010 case STB_WEAK: return "WEAK";
1011 default: return "UNKNOWN";
1012 }
1013}
1014
1015/*
1016 * Copy a symbol to the output object, optionally including its data and
1017 * relocations.
1018 */
1019static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym,
1020 bool data_too)
1021{
1022 struct symbol *pfx;
1023
1024 if (patched_sym->clone)
1025 return patched_sym->clone;
1026
1027 dbg_clone("%s%s", patched_sym->name, data_too ? " [+DATA]" : "");
1028
1029 /* Make sure the prefix gets cloned first */
1030 if (is_func_sym(patched_sym) && data_too) {
1031 pfx = get_func_prefix(patched_sym);
1032 if (pfx)
1033 clone_symbol(e, pfx, true);
1034 }
1035
1036 if (!__clone_symbol(e->out, patched_sym, data_too))
1037 return NULL;
1038
1039 if (data_too && clone_sym_relocs(e, patched_sym))
1040 return NULL;
1041
1042 return patched_sym->clone;
1043}
1044
1045static void mark_included_function(struct symbol *func)
1046{
1047 struct symbol *pfx;
1048
1049 func->included = 1;
1050
1051 /* Include prefix function */
1052 pfx = get_func_prefix(func);
1053 if (pfx)
1054 pfx->included = 1;
1055
1056 /* Make sure .cold parent+child always stay together */
1057 if (func->cfunc && func->cfunc != func)
1058 func->cfunc->included = 1;
1059 if (func->pfunc && func->pfunc != func)
1060 func->pfunc->included = 1;
1061}
1062
1063/*
1064 * Copy all changed functions (and their dependencies) from the patched object
1065 * to the output object.
1066 */
1067static int mark_changed_functions(struct elfs *e)
1068{
1069 struct symbol *orig_sym, *patched_sym;
1070 bool changed = false;
1071
1072 /* Find changed functions */
1073 for_each_sym(e->orig, orig_sym) {
1074 if (orig_sym->dont_correlate)
1075 continue;
1076
1077 patched_sym = orig_sym->twin;
1078 if (!patched_sym)
1079 continue;
1080
1081 if (orig_sym->csum.checksum != patched_sym->csum.checksum) {
1082 if (!is_func_sym(orig_sym)) {
1083 ERROR("changed data: %s", orig_sym->name);
1084 return -1;
1085 }
1086
1087 patched_sym->changed = 1;
1088 mark_included_function(patched_sym);
1089 changed = true;
1090 }
1091 }
1092
1093 /* Find added functions and print them */
1094 for_each_sym(e->patched, patched_sym) {
1095 if (!is_func_sym(patched_sym) || patched_sym->dont_correlate)
1096 continue;
1097
1098 if (!patched_sym->twin) {
1099 printf("%s: new function: %s\n", objname, patched_sym->name);
1100 mark_included_function(patched_sym);
1101 changed = true;
1102 }
1103 }
1104
1105 /* Print changed functions */
1106 for_each_sym(e->patched, patched_sym) {
1107 if (patched_sym->changed)
1108 printf("%s: changed function: %s\n", objname, patched_sym->name);
1109 }
1110
1111 return !changed ? 1 : 0;
1112}
1113
1114static int clone_included_functions(struct elfs *e)
1115{
1116 struct symbol *patched_sym;
1117
1118 for_each_sym(e->patched, patched_sym) {
1119 if (patched_sym->included) {
1120 if (!clone_symbol(e, patched_sym, true))
1121 return -1;
1122 }
1123 }
1124
1125 return 0;
1126}
1127
1128static struct export *find_export(struct symbol *sym)
1129{
1130 struct export *export;
1131
1132 hash_for_each_possible(exports, export, hash, str_hash(sym->name)) {
1133 if (!strcmp(export->sym, sym->name))
1134 return export;
1135 }
1136
1137 return NULL;
1138}
1139
1140static const char *__find_modname(struct elfs *e)
1141{
1142 struct section *sec;
1143 char *name;
1144
1145 sec = find_section_by_name(e->orig, ".modinfo");
1146 if (!sec) {
1147 ERROR("missing .modinfo section");
1148 return NULL;
1149 }
1150
1151 name = memmem(sec->data->d_buf, sec_size(sec), "\0name=", 6);
1152 if (name)
1153 return name + 6;
1154
1155 name = strdup(e->orig->name);
1156 if (!name) {
1157 ERROR_GLIBC("strdup");
1158 return NULL;
1159 }
1160
1161 for (char *c = name; *c; c++) {
1162 if (*c == '/')
1163 name = c + 1;
1164 else if (*c == '-')
1165 *c = '_';
1166 else if (*c == '.') {
1167 *c = '\0';
1168 break;
1169 }
1170 }
1171
1172 return name;
1173}
1174
1175/* Get the object's module name as defined by the kernel (and klp_object) */
1176static const char *find_modname(struct elfs *e)
1177{
1178 const char *modname;
1179
1180 if (e->modname)
1181 return e->modname;
1182
1183 modname = __find_modname(e);
1184 e->modname = modname;
1185 return modname;
1186}
1187
1188/*
1189 * Copying a function from its native compiled environment to a kernel module
1190 * removes its natural access to local functions/variables and unexported
1191 * globals. References to such symbols need to be converted to KLP relocs so
1192 * the kernel arch relocation code knows to apply them and where to find the
1193 * symbols. Particularly, duplicate static symbols need to be disambiguated.
1194 */
1195static bool klp_reloc_needed(struct reloc *patched_reloc)
1196{
1197 struct symbol *patched_sym = patched_reloc->sym;
1198 struct export *export;
1199
1200 /* no external symbol to reference */
1201 if (patched_sym->dont_correlate)
1202 return false;
1203
1204 /* For included functions, a regular reloc will do. */
1205 if (patched_sym->included)
1206 return false;
1207
1208 /*
1209 * If exported by a module, it has to be a klp reloc. Thanks to the
1210 * clusterfunk that is late module patching, the patch module is
1211 * allowed to be loaded before any modules it depends on.
1212 *
1213 * If exported by vmlinux, a normal reloc will do.
1214 */
1215 export = find_export(patched_sym);
1216 if (export)
1217 return strcmp(export->mod, "vmlinux");
1218
1219 if (!patched_sym->twin) {
1220 /*
1221 * Presumably the symbol and its reference were added by the
1222 * patch. The symbol could be defined in this .o or in another
1223 * .o in the patch module.
1224 *
1225 * This check needs to be *after* the export check due to the
1226 * possibility of the patch adding a new UNDEF reference to an
1227 * exported symbol.
1228 */
1229 return false;
1230 }
1231
1232 /* Unexported symbol which lives in the original vmlinux or module. */
1233 return true;
1234}
1235
1236/* Return -1 error, 0 success, 1 skip */
1237static int convert_reloc_sym_to_secsym(struct elf *elf, struct reloc *reloc)
1238{
1239 struct symbol *sym = reloc->sym;
1240 struct section *sec = sym->sec;
1241
1242 if (is_sec_sym(sym))
1243 return 0;
1244
1245 if (!sec->sym && !elf_create_section_symbol(elf, sec))
1246 return -1;
1247
1248 reloc->sym = sec->sym;
1249 set_reloc_sym(elf, reloc, sec->sym->idx);
1250 set_reloc_addend(elf, reloc, sym->offset + reloc_addend(reloc));
1251 return 0;
1252}
1253
1254/* Return -1 error, 0 success, 1 skip */
1255static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc)
1256{
1257 struct symbol *sym = reloc->sym;
1258 struct section *sec = sym->sec;
1259
1260 if (!is_sec_sym(sym))
1261 return 0;
1262
1263 /* If the symbol has a dedicated section, it's easy to find */
1264 sym = find_symbol_by_offset(sec, 0);
1265 if (sym && sym->len == sec_size(sec))
1266 goto found_sym;
1267
1268 /* No dedicated section; find the symbol manually */
1269 sym = find_symbol_containing_inclusive(sec, arch_adjusted_addend(reloc));
1270 if (!sym) {
1271 /*
1272 * This is presumably an .altinstr_replacement section which is
1273 * empty due to it only having zero-length replacement(s).
1274 */
1275 if (!sec_size(sec))
1276 return 1;
1277
1278 /*
1279 * .rodata is a mixed bag of named objects and anonymous data.
1280 *
1281 * Convert section symbol references to named object symbols
1282 * when possible, to preserve pointer identity for const
1283 * structs like file_operations. Otherwise a section symbol is
1284 * fine.
1285 */
1286 if (is_rodata_sec(sec))
1287 return 0;
1288
1289 /*
1290 * This can happen for special section references to weak code
1291 * whose symbol has been stripped by the linker.
1292 */
1293 return -1;
1294 }
1295
1296found_sym:
1297 reloc->sym = sym;
1298 set_reloc_sym(elf, reloc, sym->idx);
1299 set_reloc_addend(elf, reloc, reloc_addend(reloc) - sym->offset);
1300 return 0;
1301}
1302
1303/*
1304 * Sections with anonymous or uncorrelated data (strings, UBSAN data, Clang
1305 * anonymous constants) need section symbol references.
1306 */
1307static bool is_uncorrelated_section(struct section *sec)
1308{
1309 return is_string_sec(sec) ||
1310 strstarts(sec->name, ".data..Lubsan") || /* GCC */
1311 strstarts(sec->name, ".data..L__unnamed_") || /* Clang */
1312 strstarts(sec->name, ".data..Lanon."); /* Clang */
1313}
1314
1315/*
1316 * Convert a relocation symbol reference to the needed format: either a section
1317 * symbol or the underlying symbol itself. Return -1 error, 0 success, 1 skip.
1318 */
1319static int convert_reloc_sym(struct elf *elf, struct reloc *reloc)
1320{
1321 struct section *sec = reloc->sym->sec;
1322
1323 if (reloc_type(reloc) == R_NONE)
1324 return 1;
1325
1326 if (is_uncorrelated_section(sec))
1327 return convert_reloc_sym_to_secsym(elf, reloc);
1328
1329 /* Everything else: references should use named symbols. */
1330 return convert_reloc_secsym_to_sym(elf, reloc);
1331}
1332
1333/*
1334 * Convert a regular relocation to a klp relocation (sort of).
1335 */
1336static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc,
1337 struct section *sec, unsigned long offset,
1338 struct export *export)
1339{
1340 struct symbol *patched_sym = patched_reloc->sym;
1341 s64 addend = reloc_addend(patched_reloc);
1342 const char *sym_modname, *sym_orig_name;
1343 static struct section *klp_relocs;
1344 struct symbol *sym, *klp_sym;
1345 unsigned long klp_reloc_off;
1346 char sym_name[SYM_NAME_LEN];
1347 struct klp_reloc klp_reloc;
1348 unsigned long sympos;
1349
1350 if (!patched_sym->twin) {
1351 ERROR("unexpected klp reloc for new symbol %s", patched_sym->name);
1352 return -1;
1353 }
1354
1355 /*
1356 * Keep the original reloc intact for now to avoid breaking objtool run
1357 * which relies on proper relocations for many of its features. This
1358 * will be disabled later by "objtool klp post-link".
1359 *
1360 * Convert it to UNDEF (and WEAK to avoid modpost warnings).
1361 */
1362
1363 sym = patched_sym->clone;
1364 if (!sym) {
1365 /* STB_WEAK: avoid modpost undefined symbol warnings */
1366 sym = elf_create_symbol(e->out, patched_sym->name, NULL,
1367 STB_WEAK, patched_sym->type, 0, 0);
1368 if (!sym)
1369 return -1;
1370
1371 patched_sym->clone = sym;
1372 sym->clone = patched_sym;
1373 }
1374
1375 if (!elf_create_reloc(e->out, sec, offset, sym, addend, reloc_type(patched_reloc)))
1376 return -1;
1377
1378 /*
1379 * Create the KLP symbol.
1380 */
1381
1382 if (export) {
1383 sym_modname = export->mod;
1384 sym_orig_name = export->sym;
1385 sympos = 0;
1386 } else {
1387 sym_modname = find_modname(e);
1388 if (!sym_modname)
1389 return -1;
1390
1391 sym_orig_name = patched_sym->twin->name;
1392 sympos = find_sympos(e->orig, patched_sym->twin);
1393 if (sympos == ULONG_MAX)
1394 return -1;
1395 }
1396
1397 /* symbol format: .klp.sym.modname.sym_name,sympos */
1398 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_SYM_PREFIX "%s.%s,%ld",
1399 sym_modname, sym_orig_name, sympos))
1400 return -1;
1401
1402 klp_sym = find_symbol_by_name(e->out, sym_name);
1403 if (!klp_sym) {
1404 __dbg_clone("%s", sym_name);
1405
1406 /* STB_WEAK: avoid modpost undefined symbol warnings */
1407 klp_sym = elf_create_symbol(e->out, sym_name, NULL,
1408 STB_WEAK, patched_sym->type, 0, 0);
1409 if (!klp_sym)
1410 return -1;
1411 }
1412
1413 /*
1414 * Create the __klp_relocs entry. This will be converted to an actual
1415 * KLP rela by "objtool klp post-link".
1416 *
1417 * This intermediate step is necessary to prevent corruption by the
1418 * linker, which doesn't know how to properly handle two rela sections
1419 * applying to the same base section.
1420 */
1421
1422 if (!klp_relocs) {
1423 klp_relocs = elf_create_section(e->out, KLP_RELOCS_SEC, 0,
1424 0, SHT_PROGBITS, 8, SHF_ALLOC);
1425 if (!klp_relocs)
1426 return -1;
1427 }
1428
1429 klp_reloc_off = sec_size(klp_relocs);
1430 memset(&klp_reloc, 0, sizeof(klp_reloc));
1431
1432 klp_reloc.type = reloc_type(patched_reloc);
1433 if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc)))
1434 return -1;
1435
1436 /* klp_reloc.offset */
1437 if (!sec->sym && !elf_create_section_symbol(e->out, sec))
1438 return -1;
1439
1440 if (!elf_create_reloc(e->out, klp_relocs,
1441 klp_reloc_off + offsetof(struct klp_reloc, offset),
1442 sec->sym, offset, R_ABS64))
1443 return -1;
1444
1445 /* klp_reloc.sym */
1446 if (!elf_create_reloc(e->out, klp_relocs,
1447 klp_reloc_off + offsetof(struct klp_reloc, sym),
1448 klp_sym, addend, R_ABS64))
1449 return -1;
1450
1451 return 0;
1452}
1453
1454#define dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp) \
1455 dbg_clone("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \
1456 sec->name, offset, patched_sym->name, \
1457 addend >= 0 ? "+" : "-", labs(addend), \
1458 sym_type(patched_sym), \
1459 is_sec_sym(patched_sym) ? "" : " ", \
1460 is_sec_sym(patched_sym) ? "" : sym_bind(patched_sym), \
1461 is_undef_sym(patched_sym) ? " UNDEF" : "", \
1462 export ? " EXPORTED" : "", \
1463 klp ? " KLP" : "")
1464
1465/* Copy a reloc and its symbol to the output object */
1466static int clone_reloc(struct elfs *e, struct reloc *patched_reloc,
1467 struct section *sec, unsigned long offset)
1468{
1469 struct symbol *patched_sym = patched_reloc->sym;
1470 struct export *export = find_export(patched_sym);
1471 long addend = reloc_addend(patched_reloc);
1472 struct symbol *out_sym;
1473 bool klp;
1474
1475 klp = klp_reloc_needed(patched_reloc);
1476
1477 dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp);
1478
1479 if (klp) {
1480 if (clone_reloc_klp(e, patched_reloc, sec, offset, export))
1481 return -1;
1482
1483 return 0;
1484 }
1485
1486 /*
1487 * Why !export sets 'data_too':
1488 *
1489 * Unexported non-klp symbols need to live in the patch module,
1490 * otherwise there will be unresolved symbols. Notably, this includes:
1491 *
1492 * - New functions/data
1493 * - String sections
1494 * - Special section entries
1495 * - Uncorrelated static local variables
1496 * - UBSAN sections
1497 */
1498 out_sym = clone_symbol(e, patched_sym, patched_sym->included || !export);
1499 if (!out_sym)
1500 return -1;
1501
1502 /*
1503 * For strings, all references use section symbols, thanks to
1504 * convert_reloc_sym(). clone_symbol() has cloned an empty
1505 * version of the string section. Now copy the string itself.
1506 */
1507 if (is_string_sec(patched_sym->sec)) {
1508 const char *str = patched_sym->sec->data->d_buf + addend;
1509
1510 __dbg_clone("\"%s\"", escape_str(str));
1511
1512 addend = elf_add_string(e->out, out_sym->sec, str);
1513 if (addend == -1)
1514 return -1;
1515 }
1516
1517 if (!elf_create_reloc(e->out, sec, offset, out_sym, addend,
1518 reloc_type(patched_reloc)))
1519 return -1;
1520
1521 return 0;
1522}
1523
1524/* Copy all relocs needed for a symbol's contents */
1525static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym)
1526{
1527 struct section *patched_rsec = patched_sym->sec->rsec;
1528 struct reloc *patched_reloc;
1529 unsigned long start, end;
1530 struct symbol *out_sym;
1531
1532 out_sym = patched_sym->clone;
1533 if (!out_sym) {
1534 ERROR("no clone for %s", patched_sym->name);
1535 return -1;
1536 }
1537
1538 if (!patched_rsec)
1539 return 0;
1540
1541 if (!is_sec_sym(patched_sym) && !patched_sym->len)
1542 return 0;
1543
1544 if (is_string_sec(patched_sym->sec))
1545 return 0;
1546
1547 if (is_sec_sym(patched_sym)) {
1548 start = 0;
1549 end = sec_size(patched_sym->sec);
1550 } else {
1551 start = patched_sym->offset;
1552 end = start + patched_sym->len;
1553 }
1554
1555 for_each_reloc(patched_rsec, patched_reloc) {
1556 unsigned long offset;
1557 int ret;
1558
1559 if (reloc_offset(patched_reloc) < start ||
1560 reloc_offset(patched_reloc) >= end)
1561 continue;
1562
1563 /*
1564 * Skip any reloc referencing .altinstr_aux. Its code is
1565 * always patched by alternatives. See ALTERNATIVE_TERNARY().
1566 */
1567 if (patched_reloc->sym->sec &&
1568 !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux"))
1569 continue;
1570
1571 ret = convert_reloc_sym(e->patched, patched_reloc);
1572 if (ret < 0) {
1573 ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc),
1574 "failed to convert reloc sym '%s' to its proper format",
1575 patched_reloc->sym->name);
1576 return -1;
1577 }
1578 if (ret > 0)
1579 continue;
1580
1581 offset = out_sym->offset + (reloc_offset(patched_reloc) - patched_sym->offset);
1582
1583 if (clone_reloc(e, patched_reloc, out_sym->sec, offset))
1584 return -1;
1585 }
1586 return 0;
1587
1588}
1589
1590static int create_fake_symbol(struct elf *elf, struct section *sec,
1591 unsigned long offset, size_t size)
1592{
1593 char name[SYM_NAME_LEN];
1594 unsigned int type;
1595 static int ctr;
1596 char *c;
1597
1598 if (snprintf_check(name, SYM_NAME_LEN, "%s_%d", sec->name, ctr++))
1599 return -1;
1600
1601 for (c = name; *c; c++)
1602 if (*c == '.')
1603 *c = '_';
1604
1605 /*
1606 * STT_NOTYPE: Prevent objtool from validating .altinstr_replacement
1607 * while still allowing objdump to disassemble it.
1608 */
1609 type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT;
1610 return elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size) ? 0 : -1;
1611}
1612
1613/*
1614 * Special sections (alternatives, etc) are basically arrays of structs.
1615 * For all the special sections, create a symbol for each struct entry. This
1616 * is a bit cumbersome, but it makes the extracting of the individual entries
1617 * much more straightforward.
1618 *
1619 * There are three ways to identify the entry sizes for a special section:
1620 *
1621 * 1) ELF section header sh_entsize: Ideally this would be used almost
1622 * everywhere. But unfortunately the toolchains make it difficult. The
1623 * assembler .[push]section directive syntax only takes entsize when
1624 * combined with SHF_MERGE. But Clang disallows combining SHF_MERGE with
1625 * SHF_WRITE. And some special sections do need to be writable.
1626 *
1627 * Another place this wouldn't work is .altinstr_replacement, whose entries
1628 * don't have a fixed size.
1629 *
1630 * 2) ANNOTATE_DATA_SPECIAL: This is a lightweight objtool annotation which
1631 * points to the beginning of each entry. The size of the entry is then
1632 * inferred by the location of the subsequent annotation (or end of
1633 * section).
1634 *
1635 * 3) Simple array of pointers: If the special section is just a basic array of
1636 * pointers, the entry size can be inferred by the number of relocations.
1637 * No annotations needed.
1638 *
1639 * Note I also tried to create per-entry symbols at the time of creation, in
1640 * the original [inline] asm. Unfortunately, creating uniquely named symbols
1641 * is trickier than one might think, especially with Clang inline asm. I
1642 * eventually just gave up trying to make that work, in favor of using
1643 * ANNOTATE_DATA_SPECIAL and creating the symbols here after the fact.
1644 */
1645static int create_fake_symbols(struct elf *elf)
1646{
1647 struct section *sec;
1648 struct reloc *reloc;
1649
1650 /*
1651 * 1) Make symbols for all the ANNOTATE_DATA_SPECIAL entries:
1652 */
1653
1654 sec = find_section_by_name(elf, ".discard.annotate_data");
1655 if (!sec || !sec->rsec)
1656 goto entsize;
1657
1658 for_each_reloc(sec->rsec, reloc) {
1659 unsigned long offset, size;
1660 struct reloc *next_reloc;
1661
1662 if (annotype(elf, sec, reloc) != ANNOTYPE_DATA_SPECIAL)
1663 continue;
1664
1665 offset = reloc_addend(reloc);
1666
1667 size = 0;
1668 next_reloc = reloc;
1669 for_each_reloc_continue(sec->rsec, next_reloc) {
1670 if (annotype(elf, sec, next_reloc) != ANNOTYPE_DATA_SPECIAL ||
1671 next_reloc->sym->sec != reloc->sym->sec)
1672 continue;
1673
1674 size = reloc_addend(next_reloc) - offset;
1675 break;
1676 }
1677
1678 if (!size)
1679 size = sec_size(reloc->sym->sec) - offset;
1680
1681 if (create_fake_symbol(elf, reloc->sym->sec, offset, size))
1682 return -1;
1683 }
1684
1685 /*
1686 * 2) Make symbols for sh_entsize, and simple arrays of pointers:
1687 */
1688entsize:
1689 for_each_sec(elf, sec) {
1690 unsigned int entry_size;
1691 unsigned long offset;
1692
1693 if (!is_special_section(sec) || find_symbol_by_offset(sec, 0))
1694 continue;
1695
1696 if (!sec->rsec) {
1697 ERROR("%s: missing special section relocations", sec->name);
1698 return -1;
1699 }
1700
1701 entry_size = sec->sh.sh_entsize;
1702 if (!entry_size) {
1703 entry_size = arch_reloc_size(sec->rsec->relocs);
1704 if (sec_size(sec) != entry_size * sec_num_entries(sec->rsec)) {
1705 ERROR("%s: missing special section entsize or annotations", sec->name);
1706 return -1;
1707 }
1708 }
1709
1710 for (offset = 0; offset < sec_size(sec); offset += entry_size) {
1711 if (create_fake_symbol(elf, sec, offset, entry_size))
1712 return -1;
1713 }
1714 }
1715
1716 return 0;
1717}
1718
1719/* Keep a special section entry if it references an included function */
1720static bool should_keep_special_sym(struct elf *elf, struct symbol *sym)
1721{
1722 bool annotate_insn = !strcmp(sym->sec->name, ".discard.annotate_insn");
1723 struct reloc *reloc;
1724
1725 if (is_sec_sym(sym) || !sym->sec->rsec)
1726 return false;
1727
1728 sym_for_each_reloc(elf, sym, reloc) {
1729 if (convert_reloc_sym(elf, reloc))
1730 continue;
1731
1732 if (!reloc->sym->clone || is_undef_sym(reloc->sym->clone))
1733 continue;
1734
1735 /*
1736 * Keep special section references to cloned functions.
1737 * In some cases annotate_insn can also reference cloned alt
1738 * replacement fake symbols; keep those references as well.
1739 */
1740 if (is_func_sym(reloc->sym) ||
1741 (annotate_insn && is_notype_sym(reloc->sym)))
1742 return true;
1743 }
1744
1745 return false;
1746}
1747
1748/*
1749 * Klp relocations aren't allowed for __jump_table and .static_call_sites if
1750 * the referenced symbol lives in a kernel module, because such klp relocs may
1751 * be applied after static branch/call init, resulting in code corruption.
1752 *
1753 * Validate a special section entry to avoid that. Note that an inert
1754 * tracepoint or pr_debug() is harmless enough, in that case just skip the
1755 * entry and print a warning. Otherwise, return an error.
1756 *
1757 * TODO: This is only a temporary limitation which will be fixed when livepatch
1758 * adds support for submodules: fully self-contained modules which are embedded
1759 * in the top-level livepatch module's data and which can be loaded on demand
1760 * when their corresponding to-be-patched module gets loaded. Then klp relocs
1761 * can be retired.
1762 *
1763 * Return:
1764 * -1: error: validation failed
1765 * 1: warning: disabled tracepoint or pr_debug()
1766 * 0: success
1767 */
1768static int validate_special_section_klp_reloc(struct elfs *e, struct symbol *sym)
1769{
1770 bool static_branch = !strcmp(sym->sec->name, "__jump_table");
1771 bool static_call = !strcmp(sym->sec->name, ".static_call_sites");
1772 const char *code_sym = NULL;
1773 unsigned long code_offset = 0;
1774 struct reloc *reloc;
1775 int ret = 0;
1776
1777 if (!static_branch && !static_call)
1778 return 0;
1779
1780 sym_for_each_reloc(e->patched, sym, reloc) {
1781 const char *sym_modname;
1782 struct export *export;
1783
1784 if (convert_reloc_sym(e->patched, reloc))
1785 continue;
1786
1787 /* Static branch/call keys are always STT_OBJECT */
1788 if (reloc->sym->type != STT_OBJECT) {
1789
1790 /* Save code location which can be printed below */
1791 if (reloc->sym->type == STT_FUNC && !code_sym) {
1792 code_sym = reloc->sym->name;
1793 code_offset = reloc_addend(reloc);
1794 }
1795
1796 continue;
1797 }
1798
1799 if (!klp_reloc_needed(reloc))
1800 continue;
1801
1802 export = find_export(reloc->sym);
1803 if (export) {
1804 sym_modname = export->mod;
1805 } else {
1806 sym_modname = find_modname(e);
1807 if (!sym_modname)
1808 return -1;
1809 }
1810
1811 /* vmlinux keys are ok */
1812 if (!strcmp(sym_modname, "vmlinux"))
1813 continue;
1814
1815 if (!code_sym)
1816 code_sym = "<unknown>";
1817
1818 if (static_branch) {
1819 if (strstarts(reloc->sym->name, "__tracepoint_")) {
1820 WARN("%s: disabling unsupported tracepoint %s",
1821 code_sym, reloc->sym->name + 13);
1822 ret = 1;
1823 continue;
1824 }
1825
1826 if (strstr(reloc->sym->name, "__UNIQUE_ID_ddebug_")) {
1827 WARN("%s: disabling unsupported pr_debug()",
1828 code_sym);
1829 ret = 1;
1830 continue;
1831 }
1832
1833 ERROR("%s+0x%lx: unsupported static branch key %s. Use static_key_enabled() instead",
1834 code_sym, code_offset, reloc->sym->name);
1835 return -1;
1836 }
1837
1838 /* static call */
1839 if (strstarts(reloc->sym->name, "__SCK__tp_func_")) {
1840 ret = 1;
1841 continue;
1842 }
1843
1844 ERROR("%s()+0x%lx: unsupported static call key %s. Use KLP_STATIC_CALL() instead",
1845 code_sym, code_offset, reloc->sym->name);
1846 return -1;
1847 }
1848
1849 return ret;
1850}
1851
1852static int clone_special_section(struct elfs *e, struct section *patched_sec)
1853{
1854 struct symbol *patched_sym;
1855
1856 /*
1857 * Extract all special section symbols (and their dependencies) which
1858 * reference included functions.
1859 */
1860 sec_for_each_sym(patched_sec, patched_sym) {
1861 int ret;
1862
1863 if (!is_object_sym(patched_sym))
1864 continue;
1865
1866 if (!should_keep_special_sym(e->patched, patched_sym))
1867 continue;
1868
1869 ret = validate_special_section_klp_reloc(e, patched_sym);
1870 if (ret < 0)
1871 return -1;
1872 if (ret > 0)
1873 continue;
1874
1875 if (!clone_symbol(e, patched_sym, true))
1876 return -1;
1877 }
1878
1879 return 0;
1880}
1881
1882/* Extract only the needed bits from special sections */
1883static int clone_special_sections(struct elfs *e)
1884{
1885 struct section *sec, *annotate_insn = NULL;
1886
1887 for_each_sec(e->patched, sec) {
1888 if (is_special_section(sec)) {
1889 if (!strcmp(sec->name, ".discard.annotate_insn")) {
1890 annotate_insn = sec;
1891 continue;
1892 }
1893 if (clone_special_section(e, sec))
1894 return -1;
1895 }
1896 }
1897
1898 /*
1899 * Do .discard.annotate_insn last, it can reference other special
1900 * sections (alt replacements) so they need to be cloned first.
1901 */
1902 if (annotate_insn) {
1903 if (clone_special_section(e, annotate_insn))
1904 return -1;
1905 }
1906
1907 return 0;
1908}
1909
1910/*
1911 * Create .init.klp_objects and .init.klp_funcs sections which are intermediate
1912 * sections provided as input to the patch module's init code for building the
1913 * klp_patch, klp_object and klp_func structs for the livepatch API.
1914 */
1915static int create_klp_sections(struct elfs *e)
1916{
1917 size_t obj_size = sizeof(struct klp_object_ext);
1918 size_t func_size = sizeof(struct klp_func_ext);
1919 struct section *obj_sec, *funcs_sec, *str_sec;
1920 struct symbol *funcs_sym, *str_sym, *sym;
1921 char sym_name[SYM_NAME_LEN];
1922 unsigned int nr_funcs = 0;
1923 const char *modname;
1924 void *obj_data;
1925 s64 addend;
1926
1927 obj_sec = elf_create_section_pair(e->out, KLP_OBJECTS_SEC, obj_size, 0, 0);
1928 if (!obj_sec)
1929 return -1;
1930
1931 funcs_sec = elf_create_section_pair(e->out, KLP_FUNCS_SEC, func_size, 0, 0);
1932 if (!funcs_sec)
1933 return -1;
1934
1935 funcs_sym = elf_create_section_symbol(e->out, funcs_sec);
1936 if (!funcs_sym)
1937 return -1;
1938
1939 str_sec = elf_create_section(e->out, KLP_STRINGS_SEC, 0, 0,
1940 SHT_PROGBITS, 1,
1941 SHF_ALLOC | SHF_STRINGS | SHF_MERGE);
1942 if (!str_sec)
1943 return -1;
1944
1945 if (elf_add_string(e->out, str_sec, "") == -1)
1946 return -1;
1947
1948 str_sym = elf_create_section_symbol(e->out, str_sec);
1949 if (!str_sym)
1950 return -1;
1951
1952 /* allocate klp_object_ext */
1953 obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size);
1954 if (!obj_data)
1955 return -1;
1956
1957 modname = find_modname(e);
1958 if (!modname)
1959 return -1;
1960
1961 /* klp_object_ext.name */
1962 if (strcmp(modname, "vmlinux")) {
1963 addend = elf_add_string(e->out, str_sec, modname);
1964 if (addend == -1)
1965 return -1;
1966
1967 if (!elf_create_reloc(e->out, obj_sec,
1968 offsetof(struct klp_object_ext, name),
1969 str_sym, addend, R_ABS64))
1970 return -1;
1971 }
1972
1973 /* klp_object_ext.funcs */
1974 if (!elf_create_reloc(e->out, obj_sec, offsetof(struct klp_object_ext, funcs),
1975 funcs_sym, 0, R_ABS64))
1976 return -1;
1977
1978 for_each_sym(e->out, sym) {
1979 unsigned long offset = nr_funcs * func_size;
1980 unsigned long sympos;
1981 void *func_data;
1982
1983 if (!is_func_sym(sym) || is_cold_func(sym) ||
1984 !sym->clone || !sym->clone->changed)
1985 continue;
1986
1987 /* allocate klp_func_ext */
1988 func_data = elf_add_data(e->out, funcs_sec, NULL, func_size);
1989 if (!func_data)
1990 return -1;
1991
1992 /* klp_func_ext.old_name */
1993 addend = elf_add_string(e->out, str_sec, sym->clone->twin->name);
1994 if (addend == -1)
1995 return -1;
1996
1997 if (!elf_create_reloc(e->out, funcs_sec,
1998 offset + offsetof(struct klp_func_ext, old_name),
1999 str_sym, addend, R_ABS64))
2000 return -1;
2001
2002 /* klp_func_ext.new_func */
2003 if (!elf_create_reloc(e->out, funcs_sec,
2004 offset + offsetof(struct klp_func_ext, new_func),
2005 sym, 0, R_ABS64))
2006 return -1;
2007
2008 /* klp_func_ext.sympos */
2009 BUILD_BUG_ON(sizeof(sympos) != sizeof_field(struct klp_func_ext, sympos));
2010 sympos = find_sympos(e->orig, sym->clone->twin);
2011 if (sympos == ULONG_MAX)
2012 return -1;
2013 memcpy(func_data + offsetof(struct klp_func_ext, sympos), &sympos,
2014 sizeof_field(struct klp_func_ext, sympos));
2015
2016 nr_funcs++;
2017 }
2018
2019 /* klp_object_ext.nr_funcs */
2020 BUILD_BUG_ON(sizeof(nr_funcs) != sizeof_field(struct klp_object_ext, nr_funcs));
2021 memcpy(obj_data + offsetof(struct klp_object_ext, nr_funcs), &nr_funcs,
2022 sizeof_field(struct klp_object_ext, nr_funcs));
2023
2024 /*
2025 * Find callback pointers created by KLP_PRE_PATCH_CALLBACK() and
2026 * friends, and add them to the klp object.
2027 */
2028
2029 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_PATCH_PREFIX "%s", modname))
2030 return -1;
2031
2032 sym = find_symbol_by_name(e->out, sym_name);
2033 if (sym) {
2034 struct reloc *reloc;
2035
2036 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2037
2038 if (!elf_create_reloc(e->out, obj_sec,
2039 offsetof(struct klp_object_ext, callbacks) +
2040 offsetof(struct klp_callbacks, pre_patch),
2041 reloc->sym, reloc_addend(reloc), R_ABS64))
2042 return -1;
2043 }
2044
2045 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_PATCH_PREFIX "%s", modname))
2046 return -1;
2047
2048 sym = find_symbol_by_name(e->out, sym_name);
2049 if (sym) {
2050 struct reloc *reloc;
2051
2052 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2053
2054 if (!elf_create_reloc(e->out, obj_sec,
2055 offsetof(struct klp_object_ext, callbacks) +
2056 offsetof(struct klp_callbacks, post_patch),
2057 reloc->sym, reloc_addend(reloc), R_ABS64))
2058 return -1;
2059 }
2060
2061 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_UNPATCH_PREFIX "%s", modname))
2062 return -1;
2063
2064 sym = find_symbol_by_name(e->out, sym_name);
2065 if (sym) {
2066 struct reloc *reloc;
2067
2068 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2069
2070 if (!elf_create_reloc(e->out, obj_sec,
2071 offsetof(struct klp_object_ext, callbacks) +
2072 offsetof(struct klp_callbacks, pre_unpatch),
2073 reloc->sym, reloc_addend(reloc), R_ABS64))
2074 return -1;
2075 }
2076
2077 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_UNPATCH_PREFIX "%s", modname))
2078 return -1;
2079
2080 sym = find_symbol_by_name(e->out, sym_name);
2081 if (sym) {
2082 struct reloc *reloc;
2083
2084 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2085
2086 if (!elf_create_reloc(e->out, obj_sec,
2087 offsetof(struct klp_object_ext, callbacks) +
2088 offsetof(struct klp_callbacks, post_unpatch),
2089 reloc->sym, reloc_addend(reloc), R_ABS64))
2090 return -1;
2091 }
2092
2093 return 0;
2094}
2095
2096/*
2097 * Copy all .modinfo import_ns= tags to ensure all namespaced exported symbols
2098 * can be accessed via normal relocs.
2099 */
2100static int copy_import_ns(struct elfs *e)
2101{
2102 struct section *patched_sec, *out_sec = NULL;
2103 char *import_ns, *data_end;
2104
2105 patched_sec = find_section_by_name(e->patched, ".modinfo");
2106 if (!patched_sec)
2107 return 0;
2108
2109 import_ns = patched_sec->data->d_buf;
2110 if (!import_ns)
2111 return 0;
2112
2113 for (data_end = import_ns + sec_size(patched_sec);
2114 import_ns < data_end;
2115 import_ns += strlen(import_ns) + 1) {
2116
2117 import_ns = memmem(import_ns, data_end - import_ns, "import_ns=", 10);
2118 if (!import_ns)
2119 return 0;
2120
2121 if (!out_sec) {
2122 out_sec = find_section_by_name(e->out, ".modinfo");
2123 if (!out_sec) {
2124 out_sec = elf_create_section(e->out, ".modinfo", 0,
2125 patched_sec->sh.sh_entsize,
2126 patched_sec->sh.sh_type,
2127 patched_sec->sh.sh_addralign,
2128 patched_sec->sh.sh_flags);
2129 if (!out_sec)
2130 return -1;
2131 }
2132 }
2133
2134 if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1))
2135 return -1;
2136 }
2137
2138 return 0;
2139}
2140
2141int cmd_klp_diff(int argc, const char **argv)
2142{
2143 struct elfs e = {0};
2144 int ret;
2145
2146 argc = parse_options(argc, argv, klp_diff_options, klp_diff_usage, 0);
2147 if (argc != 3)
2148 usage_with_options(klp_diff_usage, klp_diff_options);
2149
2150 if (debug) {
2151 debug_correlate = true;
2152 debug_clone = true;
2153 }
2154
2155 objname = argv[0];
2156
2157 e.orig = elf_open_read(argv[0], O_RDONLY);
2158 e.patched = elf_open_read(argv[1], O_RDONLY);
2159 e.out = NULL;
2160
2161 if (!e.orig || !e.patched)
2162 return -1;
2163
2164 if (read_exports())
2165 return -1;
2166
2167 if (read_sym_checksums(e.orig))
2168 return -1;
2169
2170 if (read_sym_checksums(e.patched))
2171 return -1;
2172
2173 if (correlate_symbols(&e))
2174 return -1;
2175
2176 ret = mark_changed_functions(&e);
2177 if (ret < 0)
2178 return -1;
2179 if (ret > 0)
2180 return 0;
2181
2182 e.out = elf_create_file(&e.orig->ehdr, argv[2]);
2183 if (!e.out)
2184 return -1;
2185
2186 /*
2187 * Special section fake symbols are needed so that individual special
2188 * section entries can be extracted by clone_special_sections().
2189 *
2190 * Note the fake symbols are also needed by clone_included_functions()
2191 * because __WARN_printf() call sites add references to bug table
2192 * entries in the calling functions.
2193 */
2194 if (create_fake_symbols(e.patched))
2195 return -1;
2196
2197 if (clone_included_functions(&e))
2198 return -1;
2199
2200 if (clone_special_sections(&e))
2201 return -1;
2202
2203 if (create_klp_sections(&e))
2204 return -1;
2205
2206 if (copy_import_ns(&e))
2207 return -1;
2208
2209 if (elf_write(e.out))
2210 return -1;
2211
2212 return elf_close(e.out);
2213}