projects/linux·blob master · refs

2294 lines · 57.5 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2#define _GNU_SOURCE /* memmem() */
3#include <subcmd/parse-options.h>
4#include <stdlib.h>
5#include <string.h>
6#include <libgen.h>
7#include <stdio.h>
8#include <ctype.h>
9
10#include <objtool/objtool.h>
11#include <objtool/warn.h>
12#include <objtool/arch.h>
13#include <objtool/klp.h>
14#include <objtool/util.h>
15#include <objtool/special.h>
16
17#include <linux/align.h>
18#include <linux/objtool_types.h>
19#include <linux/livepatch_external.h>
20#include <linux/stringify.h>
21#include <linux/string.h>
22#include <linux/jhash.h>
23
24#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
25
26struct elfs {
27 struct elf *orig, *patched, *out;
28 const char *modname;
29};
30
31struct export {
32 struct hlist_node hash;
33 char *mod;
34 char *sym;
35 bool mod_ns;
36};
37
38bool debug, debug_correlate, debug_clone;
39int indent;
40
41static const char * const klp_diff_usage[] = {
42 "objtool klp diff [<options>] <in1.o> <in2.o> <out.o>",
43 NULL,
44};
45
46static const struct option klp_diff_options[] = {
47 OPT_GROUP("Options:"),
48 OPT_BOOLEAN('d', "debug", &debug, "enable all debug output"),
49 OPT_BOOLEAN(0, "debug-correlate", &debug_correlate, "enable correlation debug output"),
50 OPT_BOOLEAN(0, "debug-clone", &debug_clone, "enable cloning debug output"),
51 OPT_END(),
52};
53
54static DEFINE_HASHTABLE(exports, 15);
55
56static char *escape_str(const char *orig)
57{
58 size_t len = 0;
59 const char *a;
60 char *b, *new;
61
62 for (a = orig; *a; a++) {
63 switch (*a) {
64 case '\001': len += 5; break;
65 case '\n':
66 case '\t': len += 2; break;
67 default: len++;
68 }
69 }
70
71 new = malloc(len + 1);
72 if (!new)
73 return NULL;
74
75 for (a = orig, b = new; *a; a++) {
76 switch (*a) {
77 case '\001': memcpy(b, "<SOH>", 5); b += 5; break;
78 case '\n': *b++ = '\\'; *b++ = 'n'; break;
79 case '\t': *b++ = '\\'; *b++ = 't'; break;
80 default: *b++ = *a;
81 }
82 }
83
84 *b = '\0';
85 return new;
86}
87
88/*
89 * Convert a build-tree object path to a runtime module name: strip
90 * directory components, replace '-' with '_', and remove file
91 * extensions. Examples:
92 *
93 * "arch/x86/kvm/kvm" -> "kvm"
94 * "arch/x86/kvm/kvm-intel" -> "kvm_intel".
95 *
96 * Used by read_exports() to normalize Module.symvers entries and by
97 * __find_modname() as a fallback when .modinfo lacks a "name=" tag.
98 */
99static char *normalize_modname(char *name)
100{
101 char *slash = strrchr(name, '/');
102
103 if (slash)
104 name = slash + 1;
105
106 for (char *c = name; *c; c++) {
107 if (*c == '-')
108 *c = '_';
109 else if (*c == '.') {
110 *c = '\0';
111 break;
112 }
113 }
114 return name;
115}
116
117static int read_exports(void)
118{
119 const char *symvers = "Module.symvers";
120 char line[1024], *path = NULL;
121 unsigned int line_num = 0;
122 FILE *file;
123
124 file = fopen(symvers, "r");
125 if (!file) {
126 path = top_level_dir(symvers);
127 if (!path) {
128 ERROR("can't open '%s', \"objtool diff\" should be run from the kernel tree", symvers);
129 return -1;
130 }
131
132 file = fopen(path, "r");
133 if (!file) {
134 ERROR_GLIBC("fopen");
135 return -1;
136 }
137 }
138
139 while (fgets(line, 1024, file)) {
140 char *sym, *mod, *type, *namespace;
141 struct export *export;
142
143 line_num++;
144
145 sym = strchr(line, '\t');
146 if (!sym) {
147 ERROR("malformed Module.symvers (sym) at line %d", line_num);
148 return -1;
149 }
150
151 *sym++ = '\0';
152
153 mod = strchr(sym, '\t');
154 if (!mod) {
155 ERROR("malformed Module.symvers (mod) at line %d", line_num);
156 return -1;
157 }
158
159 *mod++ = '\0';
160
161 type = strchr(mod, '\t');
162 if (!type) {
163 ERROR("malformed Module.symvers (type) at line %d", line_num);
164 return -1;
165 }
166
167 *type++ = '\0';
168
169 namespace = strchr(type, '\t');
170 if (!namespace) {
171 ERROR("malformed Module.symvers (namespace) at line %d", line_num);
172 return -1;
173 }
174
175 *namespace++ = '\0';
176
177 if (*sym == '\0' || *mod == '\0') {
178 ERROR("malformed Module.symvers at line %d", line_num);
179 return -1;
180 }
181
182 export = calloc(1, sizeof(*export));
183 if (!export) {
184 ERROR_GLIBC("calloc");
185 return -1;
186 }
187
188 export->mod = strdup(mod);
189 if (!export->mod) {
190 ERROR_GLIBC("strdup");
191 return -1;
192 }
193
194 if (strcmp(export->mod, "vmlinux"))
195 export->mod = normalize_modname(export->mod);
196
197 export->sym = strdup(sym);
198 if (!export->sym) {
199 ERROR_GLIBC("strdup");
200 return -1;
201 }
202
203 /* EXPORT_SYMBOL_FOR_MODULES() */
204 export->mod_ns = strstarts(namespace, "module:");
205
206 hash_add(exports, &export->hash, str_hash(sym));
207 }
208
209 free(path);
210 fclose(file);
211
212 return 0;
213}
214
215static int read_sym_checksums(struct elf *elf)
216{
217 struct section *sec;
218
219 sec = find_section_by_name(elf, ".discard.sym_checksum");
220 if (!sec) {
221 ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool klp checksum'?",
222 elf->name);
223 return -1;
224 }
225
226 if (!sec->rsec) {
227 ERROR("missing reloc section for .discard.sym_checksum");
228 return -1;
229 }
230
231 if (sec_size(sec) % sizeof(struct sym_checksum)) {
232 ERROR("struct sym_checksum size mismatch");
233 return -1;
234 }
235
236 for (int i = 0; i < sec_size(sec) / sizeof(struct sym_checksum); i++) {
237 struct sym_checksum *sym_checksum;
238 struct reloc *reloc;
239 struct symbol *sym;
240
241 sym_checksum = (struct sym_checksum *)sec->data->d_buf + i;
242
243 reloc = find_reloc_by_dest(elf, sec, i * sizeof(*sym_checksum));
244 if (!reloc) {
245 ERROR("can't find reloc for sym_checksum[%d]", i);
246 return -1;
247 }
248
249 sym = reloc->sym;
250
251 if (is_sec_sym(sym)) {
252 ERROR("not sure how to handle section %s", sym->name);
253 return -1;
254 }
255
256 if (is_func_sym(sym) || is_object_sym(sym))
257 sym->csum.checksum = sym_checksum->checksum;
258 }
259
260 return 0;
261}
262
263static struct symbol *first_file_symbol(struct elf *elf)
264{
265 struct symbol *sym;
266
267 for_each_sym(elf, sym) {
268 if (is_file_sym(sym))
269 return sym;
270 }
271
272 return NULL;
273}
274
275static struct symbol *next_file_symbol(struct elf *elf, struct symbol *sym)
276{
277 for_each_sym_continue(elf, sym) {
278 if (is_file_sym(sym))
279 return sym;
280 }
281
282 return NULL;
283}
284
285/*
286 * Certain static local variables should never be correlated. They will be
287 * used in place rather than referencing the originals.
288 */
289static bool is_uncorrelated_static_local(struct symbol *sym)
290{
291 static const char * const vars[] = {
292 "__already_done",
293 "__func__",
294 "__key",
295 "__warned",
296 "_entry",
297 "_entry_ptr",
298 "_rs",
299 "descriptor",
300 "CSWTCH",
301 };
302 const char *dot;
303
304 if (!is_object_sym(sym) || !is_local_sym(sym))
305 return false;
306
307 /* WARN_ONCE, etc */
308 if (!strcmp(sym->sec->name, ".data..once"))
309 return true;
310
311 dot = strchr(sym->name, '.');
312 if (!dot)
313 return false;
314
315 for (int i = 0; i < ARRAY_SIZE(vars); i++) {
316 size_t len = strlen(vars[i]);
317
318 /* GCC: <var>.<id> */
319 if (strstarts(sym->name, vars[i]) && (sym->name[len] == '.'))
320 return true;
321
322 /* Clang: <func>.<var>[.<id>] */
323 if (strstarts(dot + 1, vars[i]) &&
324 (dot[1 + len] == '.' || dot[1 + len] == '\0'))
325 return true;
326 }
327
328 return false;
329}
330
331/*
332 * .L symbols are assembler-local labels not present in kallsyms. They must
333 * never become KLP relocations; instead their data is cloned into the patch
334 * module. This covers .Ltmp* (Clang temp labels), .L__const.* (Clang local
335 * constants), and any other assembler-local pattern.
336 */
337static bool is_local_label(struct symbol *sym)
338{
339 return strstarts(sym->name, ".L");
340}
341
342static bool is_special_section(struct section *sec)
343{
344 static const char * const specials[] = {
345 ".altinstructions",
346 ".kcfi_traps",
347 "__bug_table",
348 "__ex_table",
349 "__jump_table",
350 "__mcount_loc",
351
352 /*
353 * Extract .static_call_sites here to inherit non-module
354 * preferential treatment. The later static call processing
355 * during klp module build will be skipped when it sees this
356 * section already exists.
357 */
358 ".static_call_sites",
359 };
360
361 static const char * const non_special_discards[] = {
362 ".discard.addressable",
363 ".discard.sym_checksum",
364 };
365
366 if (is_text_sec(sec))
367 return false;
368
369 for (int i = 0; i < ARRAY_SIZE(specials); i++) {
370 if (!strcmp(sec->name, specials[i]))
371 return true;
372 }
373
374 /* Most .discard data sections are special */
375 for (int i = 0; i < ARRAY_SIZE(non_special_discards); i++) {
376 if (!strcmp(sec->name, non_special_discards[i]))
377 return false;
378 }
379
380 return strstarts(sec->name, ".discard.");
381}
382
383/*
384 * These sections are referenced by special sections but aren't considered
385 * special sections themselves.
386 */
387static bool is_special_section_aux(struct section *sec)
388{
389 static const char * const specials_aux[] = {
390 ".altinstr_replacement",
391 ".altinstr_aux",
392 };
393
394 for (int i = 0; i < ARRAY_SIZE(specials_aux); i++) {
395 if (!strcmp(sec->name, specials_aux[i]))
396 return true;
397 }
398
399 return false;
400}
401
402/*
403 * Symbols created by ___ADDRESSABLE() are only used to convince the toolchain
404 * not to optimize out the referenced symbol.
405 */
406static bool is_addressable_sym(struct symbol *sym)
407{
408 return !strcmp(sym->sec->name, ".discard.addressable");
409}
410
411/*
412 * ABS symbols are typically assembly .set/.equ constants which are never
413 * referenced by relocations. (Exclude FILE symbols which are also SHN_ABS.)
414 */
415static bool is_abs_sym(struct symbol *sym)
416{
417 return sym->sym.st_shndx == SHN_ABS && !is_file_sym(sym);
418}
419
420static bool is_initcall_sym(struct symbol *sym)
421{
422 return strstarts(sym->name, "__initcall__") ||
423 strstarts(sym->name, "__initstub__");
424}
425
426/*
427 * Some .rodata is anonymous and can't be correlated due to there being no
428 * symbol names.
429 *
430 * The .rodata.cst* sections aren't technically anonymous, they're SHF_MERGE
431 * constant pool sections containing small fixed-size data (lookup tables,
432 * bitmasks) which are only read by value, so pointer equivalence isn't needed.
433 * They are typically referenced by UBSAN data sections.
434 */
435static bool is_anonymous_rodata(struct symbol *sym)
436{
437 return is_rodata_sec(sym->sec) &&
438 (!is_object_sym(sym) || strstarts(sym->sec->name, ".rodata.cst"));
439}
440
441/*
442 * These symbols should never be correlated, so their local patched versions
443 * are used instead of linking to the originals.
444 */
445static bool dont_correlate(struct symbol *sym)
446{
447 return is_file_sym(sym) ||
448 is_null_sym(sym) ||
449 is_sec_sym(sym) ||
450 is_abs_sym(sym) ||
451 is_prefix_func(sym) ||
452 is_uncorrelated_static_local(sym) ||
453 is_local_label(sym) ||
454 is_string_sec(sym->sec) ||
455 is_anonymous_rodata(sym) ||
456 is_initcall_sym(sym) ||
457 is_addressable_sym(sym) ||
458 is_special_section(sym->sec) ||
459 is_special_section_aux(sym->sec);
460}
461
462static const char *llvm_suffix(const char *name)
463{
464 return strstr(name, ".llvm.");
465}
466
467static bool is_llvm_sym(struct symbol *sym)
468{
469 return llvm_suffix(sym->name);
470}
471
472/*
473 * Determine if two symbols have compatible source file origins:
474 *
475 * - If both symbols are local, only return true if they belong to the same
476 * ELF file symbol.
477 *
478 * - If both symbols are global, always return true, as globals don't have
479 * file associations.
480 *
481 * - If they have different scopes, also return true, as the patch might have
482 * changed the symbol's scope.
483 *
484 * Works for both same-ELF (direct pointer compare) and cross-ELF
485 * (compare via file->twin) cases.
486 */
487static bool maybe_same_file(struct symbol *sym1, struct symbol *sym2)
488{
489 if (!sym1->file || !sym2->file)
490 return true;
491 if (sym1->file == sym2->file)
492 return true;
493 return sym1->file->twin == sym2->file;
494}
495
496/*
497 * Similar to maybe_same_file(), but strict: no scope changes allowed.
498 *
499 * Works for both same-ELF (direct pointer compare) and cross-ELF
500 * (compare via file->twin) cases.
501 */
502static bool same_file(struct symbol *sym1, struct symbol *sym2)
503{
504 if (llvm_suffix(sym1->name) && llvm_suffix(sym2->name))
505 return true;
506 if (!sym1->file && !sym2->file)
507 return true;
508 if (!sym1->file || !sym2->file)
509 return false;
510 if (sym1->file == sym2->file)
511 return true;
512 return sym1->file->twin == sym2->file;
513}
514
515/*
516 * Is it a local symbol, or at least was it local in the translation unit
517 * before LLVM promoted it?
518 */
519static bool is_tu_local_sym(struct symbol *sym)
520{
521 return is_local_sym(sym) || is_llvm_sym(sym);
522}
523
524/*
525 * Try to find sym1's twin in patched using deterministic matching.
526 *
527 * Multiple symbols can share a demangled name (e.g., static functions in
528 * different TUs). This function counts same-named candidates through a
529 * funnel of progressively tighter filters. Each level is a strict subset
530 * of the previous one.
531 *
532 * The widest level that yields a 1:1 match wins. Narrower levels are only
533 * needed when the wider level is ambiguous (count > 1).
534 *
535 * Candidates are pre-filtered by maybe_same_file(), which narrows most
536 * local symbols to their own TU. For example, 19 different static
537 * type_show() functions across vmlinux.o each see only one candidate after
538 * pre-filtering, so they match immediately at Level 1.
539 *
540 * Level 1 (name): Works when the demangled name is unique after
541 * pre-filtering. Handles most symbols: unique globals like copy_signal(),
542 * or per-TU locals like pcspkr_probe().
543 *
544 * Level 2 (scope): Filters by local-vs-global (TU-local-vs-not). Example:
545 * parse_header() exists as both a static and a global function. Level 1
546 * sees both (same demangled name), but Level 2 separates them by scope.
547 *
548 * Level 3 (file): Strict file matching via same_file(), which rejects scope
549 * changes. Example: LLVM-promoted foo.llvm.12345 (global, no FILE symbol)
550 * vs genuine local foo (has FILE symbol). Both are TU-local so Level 2
551 * can't distinguish them, but same_file() rejects the pair because one has
552 * a file association and the other doesn't.
553 *
554 * Level 4 (checksum): Distinguishes by function checksum. Example:
555 * usb_devnode.llvm.AAA and usb_devnode.llvm.BBB are two LLVM-promoted
556 * functions from different TUs with the same demangled name. After a TU
557 * change, the .llvm. hashes change but the functions themselves may be
558 * unchanged. Level 4 matches each to the patched candidate with the
559 * same checksum.
560 */
561static struct symbol *find_twin(struct elfs *e, struct symbol *sym1)
562{
563 struct symbol *name_last = NULL, *scope_last = NULL,
564 *file_last = NULL, *csum_last = NULL;
565 unsigned int name_orig = 0, name_patched = 0;
566 unsigned int scope_orig = 0, scope_patched = 0;
567 unsigned int file_orig = 0, file_patched = 0;
568 unsigned int csum_orig = 0, csum_patched = 0;
569 struct symbol *sym2, *match = NULL;
570
571 /* Count orig candidates */
572 for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) {
573 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
574 (!maybe_same_file(sym1, sym2)))
575 continue;
576
577 /* Level 1: name match (widest filter) */
578 name_orig++;
579
580 /* Level 2: scope (scope changes allowed) */
581 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2))
582 continue;
583 scope_orig++;
584
585 /* Level 3: file (scope changes disallowed) */
586 if (!same_file(sym1, sym2))
587 continue;
588 file_orig++;
589
590 /* Level 4: checksum (unchanged symbols) */
591 if (sym1->len != sym2->len || !sym1->csum.checksum ||
592 sym1->csum.checksum != sym2->csum.checksum)
593 continue;
594 csum_orig++;
595 }
596
597 /* Count patched candidates */
598 for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) {
599 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
600 !maybe_same_file(sym1, sym2))
601 continue;
602
603 /* Level 1 */
604 name_patched++;
605 name_last = sym2;
606
607 /* Level 2 */
608 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2))
609 continue;
610 scope_patched++;
611 scope_last = sym2;
612
613 /* Level 3 */
614 if (!same_file(sym1, sym2))
615 continue;
616 file_patched++;
617 file_last = sym2;
618
619 /* Level 4 */
620 if (sym1->len != sym2->len || !sym1->csum.checksum ||
621 sym1->csum.checksum != sym2->csum.checksum)
622 continue;
623 csum_patched++;
624 csum_last = sym2;
625 }
626
627 /* Return the widest level that yields a unique (1:1) match */
628 if (name_orig == 1 && name_patched == 1)
629 match = name_last;
630 else if (scope_orig == 1 && scope_patched == 1)
631 match = scope_last;
632 else if (file_orig == 1 && file_patched == 1)
633 match = file_last;
634 else if (csum_orig == 1 && csum_patched == 1)
635 match = csum_last;
636
637 if (!match)
638 return NULL;
639
640 if (name_orig != 1 || name_patched != 1)
641 dbg_correlate("find_twin(): %s%s -> %s%s",
642 sym1->name, is_func_sym(sym1) ? "()" : "",
643 match->name, is_func_sym(match) ? "()" : "");
644
645 return match;
646}
647
648struct llvm_suffix_pair {
649 struct hlist_node hash;
650 const char *orig;
651 const char *patched;
652};
653
654static DECLARE_HASHTABLE(suffix_map, 7);
655
656/*
657 * Build a mapping of known orig-to-patched LLVM suffixes based on
658 * already-correlated symbol pairs. All promoted symbols from the same TU
659 * share the same .llvm.<hash> suffix, so one correlated pair seeds the map
660 * for the entire TU.
661 */
662static int update_suffix_map(struct elf *elf)
663{
664 struct llvm_suffix_pair *entry;
665 struct symbol *sym;
666
667 for_each_sym(elf, sym) {
668 const char *s1, *s2;
669 bool found;
670
671 if (!sym->twin)
672 continue;
673
674 s1 = llvm_suffix(sym->name);
675 s2 = llvm_suffix(sym->twin->name);
676
677 if (!s1 || !s2)
678 continue;
679
680 found = false;
681 hash_for_each_possible(suffix_map, entry, hash, str_hash(s1)) {
682 if (!strcmp(entry->orig, s1)) {
683 found = true;
684 break;
685 }
686 }
687 if (found)
688 continue;
689
690 entry = calloc(1, sizeof(*entry));
691 if (!entry) {
692 ERROR_GLIBC("calloc");
693 return -1;
694 }
695
696 entry->orig = s1;
697 entry->patched = s2;
698 hash_add(suffix_map, &entry->hash, str_hash(s1));
699 }
700
701 return 0;
702}
703
704/*
705 * Match by translating the symbol's .llvm.<hash> suffix through the suffix
706 * map to find the corresponding hash suffix for the patched object.
707 *
708 * Example: In the original kernel, TU drivers/base/core.c contains
709 * foo.llvm.12345 and bar.llvm.12345 (same TU, same hash). After patching,
710 * they become foo.llvm.67890 and bar.llvm.67890. If foo was already
711 * correlated by find_twin() (e.g., unique by name), the suffix map records
712 * .llvm.12345 -> .llvm.67890. When processing bar.llvm.12345, this
713 * function looks up .llvm.12345, gets .llvm.67890, constructs the name
714 * bar.llvm.67890, and finds the match.
715 */
716static struct symbol *find_twin_suffixed(struct elf *elf, struct symbol *sym1)
717{
718 const char *suffix, *patched_suffix = NULL;
719 struct symbol *sym2, *match = NULL;
720 char name[SYM_NAME_LEN];
721 struct llvm_suffix_pair *entry;
722 int count = 0;
723
724 suffix = llvm_suffix(sym1->name);
725 if (!suffix)
726 return NULL;
727
728 hash_for_each_possible(suffix_map, entry, hash, str_hash(suffix)) {
729 if (!strcmp(entry->orig, suffix)) {
730 patched_suffix = entry->patched;
731 break;
732 }
733 }
734 if (!patched_suffix)
735 return NULL;
736
737 if (snprintf_check(name, SYM_NAME_LEN, "%s%s",
738 sym1->demangled_name, patched_suffix))
739 return NULL;
740
741 for_each_sym_by_name(elf, name, sym2) {
742 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate)
743 continue;
744 count++;
745 match = sym2;
746 }
747
748 if (count != 1)
749 return NULL;
750
751 dbg_correlate("find_suffixed_twin(): %s%s -> %s%s",
752 sym1->name, is_func_sym(sym1) ? "()" : "",
753 match->name, is_func_sym(match) ? "()" : "");
754
755 return match;
756}
757
758/*
759 * Last-resort positional matching.
760 *
761 * Finds a symbol with the same position in the symbol table among
762 * same-demangled-name candidates, similar to livepatch sympos. Note that
763 * LLVM-promoted symbols are globals, which come after locals in the symbol
764 * table, so we have to be careful not to compare different scopes.
765 *
766 * Example: arch/x86/events/intel/core.c defines many __quirk variables via
767 * X86_MATCH_*() macros. In the symbol table they appear as __quirk.90,
768 * __quirk.97, __quirk.101, etc., all with demangled name __quirk, same
769 * scope, and same FILE symbol. No deterministic filter can distinguish
770 * them, so they're matched by position: the 1st __quirk in orig matches the
771 * 1st in patched, the 2nd matches the 2nd, etc.
772 *
773 * This is less deterministic than the other strategies, so it's done last.
774 */
775static struct symbol *find_twin_positional(struct elfs *e, struct symbol *sym1)
776{
777 unsigned int idx_orig = 0, idx_patched = 0;
778 unsigned int sym1_pos = 0;
779 struct symbol *sym2, *match = NULL;
780
781 for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) {
782 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
783 !maybe_same_file(sym1, sym2))
784 continue;
785 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) ||
786 is_llvm_sym(sym1) != is_llvm_sym(sym2))
787 continue;
788 if (sym1 == sym2)
789 sym1_pos = idx_orig;
790 idx_orig++;
791 }
792
793 for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) {
794 if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
795 !maybe_same_file(sym1, sym2))
796 continue;
797 if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) ||
798 is_llvm_sym(sym1) != is_llvm_sym(sym2))
799 continue;
800 if (idx_patched == sym1_pos)
801 match = sym2;
802 idx_patched++;
803 }
804
805 if (idx_orig != idx_patched)
806 return NULL;
807
808 dbg_correlate("find_twin_positional(): %s%s -> %s%s",
809 sym1->name, is_func_sym(sym1) ? "()" : "",
810 match->name, is_func_sym(match) ? "()" : "");
811
812 return match;
813}
814
815/*
816 * Correlate symbols between the orig and patched objects. This is a
817 * prerequisite for detecting changed functions, as well as for properly
818 * translating relocations so they point to the correct symbol.
819 */
820static int correlate_symbols(struct elfs *e)
821{
822 struct symbol *file1_sym, *file2_sym;
823 struct symbol *sym1, *sym2;
824 bool progress;
825
826 for_each_sym(e->orig, sym1)
827 sym1->dont_correlate = dont_correlate(sym1);
828 for_each_sym(e->patched, sym2)
829 sym2->dont_correlate = dont_correlate(sym2);
830
831 /* Correlate FILE symbols */
832 file1_sym = first_file_symbol(e->orig);
833 file2_sym = first_file_symbol(e->patched);
834
835 for (; ; file1_sym = next_file_symbol(e->orig, file1_sym),
836 file2_sym = next_file_symbol(e->patched, file2_sym)) {
837
838 if (!file1_sym && file2_sym) {
839 ERROR("FILE symbol mismatch: NULL != %s", file2_sym->name);
840 return -1;
841 }
842
843 if (file1_sym && !file2_sym) {
844 ERROR("FILE symbol mismatch: %s != NULL", file1_sym->name);
845 return -1;
846 }
847
848 if (!file1_sym)
849 break;
850
851 if (strcmp(file1_sym->name, file2_sym->name)) {
852 ERROR("FILE symbol mismatch: %s != %s", file1_sym->name, file2_sym->name);
853 return -1;
854 }
855
856 file1_sym->twin = file2_sym;
857 file2_sym->twin = file1_sym;
858 }
859
860
861 /*
862 * Correlate in two phases: loop deterministic levels until no more
863 * progress, then use positional fallback for the rest. This prevents
864 * the nondeterministic positional matching from stealing symbols that
865 * have deterministic matches.
866 */
867 hash_init(suffix_map);
868 do {
869 progress = false;
870 for_each_sym(e->orig, sym1) {
871 if (sym1->twin || sym1->dont_correlate)
872 continue;
873 sym2 = find_twin(e, sym1);
874 if (!sym2)
875 continue;
876 sym1->twin = sym2;
877 sym2->twin = sym1;
878 progress = true;
879 }
880
881 if (update_suffix_map(e->orig))
882 return -1;
883
884 for_each_sym(e->orig, sym1) {
885 if (sym1->twin || sym1->dont_correlate)
886 continue;
887 sym2 = find_twin_suffixed(e->patched, sym1);
888 if (!sym2)
889 continue;
890 sym1->twin = sym2;
891 sym2->twin = sym1;
892 progress = true;
893 }
894 } while (progress);
895
896 for_each_sym(e->orig, sym1) {
897 if (sym1->twin || sym1->dont_correlate)
898 continue;
899 sym2 = find_twin_positional(e, sym1);
900 if (!sym2)
901 continue;
902 sym1->twin = sym2;
903 sym2->twin = sym1;
904 }
905
906 for_each_sym(e->orig, sym1) {
907 if (sym1->twin || sym1->dont_correlate)
908 continue;
909 WARN("no correlation: %s", sym1->name);
910 }
911
912 return 0;
913}
914
915static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym);
916
917static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym,
918 bool data_too)
919{
920 struct section *out_sec = NULL;
921 unsigned long offset = 0;
922 struct symbol *out_sym;
923
924 if (data_too && !is_undef_sym(patched_sym)) {
925 struct section *patched_sec = patched_sym->sec;
926
927 out_sec = find_section_by_name(elf, patched_sec->name);
928 if (!out_sec) {
929 out_sec = elf_create_section(elf, patched_sec->name, 0,
930 patched_sec->sh.sh_entsize,
931 patched_sec->sh.sh_type,
932 patched_sec->sh.sh_addralign,
933 patched_sec->sh.sh_flags);
934 if (!out_sec)
935 return NULL;
936 }
937
938 if (is_string_sec(patched_sym->sec)) {
939 out_sym = elf_create_section_symbol(elf, out_sec);
940 if (!out_sym)
941 return NULL;
942
943 goto sym_created;
944 }
945
946 if (!is_sec_sym(patched_sym))
947 offset = ALIGN(sec_size(out_sec), out_sec->sh.sh_addralign);
948
949 if (patched_sym->len || is_sec_sym(patched_sym)) {
950 void *data = NULL;
951 size_t size;
952
953 /* bss doesn't have data */
954 if (patched_sym->sec->data && patched_sym->sec->data->d_buf)
955 data = patched_sym->sec->data->d_buf + patched_sym->offset;
956
957 if (is_sec_sym(patched_sym))
958 size = sec_size(patched_sym->sec);
959 else
960 size = patched_sym->len;
961
962 if (!elf_add_data(elf, out_sec, data, size))
963 return NULL;
964 }
965 }
966
967 out_sym = elf_create_symbol(elf, patched_sym->name, out_sec,
968 patched_sym->bind, patched_sym->type,
969 offset, patched_sym->len);
970 if (!out_sym)
971 return NULL;
972
973sym_created:
974 patched_sym->clone = out_sym;
975 out_sym->clone = patched_sym;
976
977 return out_sym;
978}
979
980static const char *sym_type(struct symbol *sym)
981{
982 switch (sym->type) {
983 case STT_NOTYPE: return "NOTYPE";
984 case STT_OBJECT: return "OBJECT";
985 case STT_FUNC: return "FUNC";
986 case STT_SECTION: return "SECTION";
987 case STT_FILE: return "FILE";
988 default: return "UNKNOWN";
989 }
990}
991
992static const char *sym_bind(struct symbol *sym)
993{
994 switch (sym->bind) {
995 case STB_LOCAL: return "LOCAL";
996 case STB_GLOBAL: return "GLOBAL";
997 case STB_WEAK: return "WEAK";
998 default: return "UNKNOWN";
999 }
1000}
1001
1002/*
1003 * Copy a symbol to the output object, optionally including its data and
1004 * relocations.
1005 */
1006static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym,
1007 bool data_too)
1008{
1009 struct symbol *pfx;
1010
1011 if (patched_sym->clone)
1012 return patched_sym->clone;
1013
1014 dbg_clone("%s%s", patched_sym->name, data_too ? " [+DATA]" : "");
1015
1016 /* Make sure the prefix gets cloned first */
1017 if (is_func_sym(patched_sym) && data_too) {
1018 pfx = get_func_prefix(patched_sym);
1019 if (pfx)
1020 clone_symbol(e, pfx, true);
1021 }
1022
1023 if (!__clone_symbol(e->out, patched_sym, data_too))
1024 return NULL;
1025
1026 if (data_too && clone_sym_relocs(e, patched_sym))
1027 return NULL;
1028
1029 return patched_sym->clone;
1030}
1031
1032static void mark_included_function(struct symbol *func)
1033{
1034 struct symbol *pfx;
1035
1036 func->included = 1;
1037
1038 /* Include prefix function */
1039 pfx = get_func_prefix(func);
1040 if (pfx)
1041 pfx->included = 1;
1042
1043 /* Make sure .cold parent+child always stay together */
1044 if (func->cfunc && func->cfunc != func)
1045 func->cfunc->included = 1;
1046 if (func->pfunc && func->pfunc != func)
1047 func->pfunc->included = 1;
1048}
1049
1050/*
1051 * Copy all changed functions (and their dependencies) from the patched object
1052 * to the output object.
1053 */
1054static int mark_changed_functions(struct elfs *e)
1055{
1056 struct symbol *orig_sym, *patched_sym;
1057 bool changed = false;
1058
1059 /* Find changed functions */
1060 for_each_sym(e->orig, orig_sym) {
1061 if (orig_sym->dont_correlate)
1062 continue;
1063
1064 patched_sym = orig_sym->twin;
1065 if (!patched_sym)
1066 continue;
1067
1068 if (orig_sym->csum.checksum != patched_sym->csum.checksum) {
1069 if (!is_func_sym(orig_sym)) {
1070 ERROR("changed data: %s", orig_sym->name);
1071 return -1;
1072 }
1073
1074 patched_sym->changed = 1;
1075 mark_included_function(patched_sym);
1076 changed = true;
1077 }
1078 }
1079
1080 /* Find added functions and print them */
1081 for_each_sym(e->patched, patched_sym) {
1082 if (!is_func_sym(patched_sym) || patched_sym->dont_correlate)
1083 continue;
1084
1085 if (!patched_sym->twin) {
1086 printf("%s: new function: %s\n", objname, patched_sym->name);
1087 mark_included_function(patched_sym);
1088 changed = true;
1089 }
1090 }
1091
1092 /* Print changed functions */
1093 for_each_sym(e->patched, patched_sym) {
1094 if (patched_sym->changed)
1095 printf("%s: changed function: %s\n", objname, patched_sym->name);
1096 }
1097
1098 return !changed ? 1 : 0;
1099}
1100
1101static int clone_included_functions(struct elfs *e)
1102{
1103 struct symbol *patched_sym;
1104
1105 for_each_sym(e->patched, patched_sym) {
1106 if (patched_sym->included) {
1107 if (!clone_symbol(e, patched_sym, true))
1108 return -1;
1109 }
1110 }
1111
1112 return 0;
1113}
1114
1115static struct export *find_export(struct symbol *sym)
1116{
1117 struct export *export;
1118
1119 if (is_local_sym(sym))
1120 return NULL;
1121
1122 hash_for_each_possible(exports, export, hash, str_hash(sym->name)) {
1123 if (!strcmp(export->sym, sym->name))
1124 return export;
1125 }
1126
1127 return NULL;
1128}
1129
1130static const char *__find_modname(struct elfs *e)
1131{
1132 struct section *sec;
1133 char *name;
1134
1135 sec = find_section_by_name(e->orig, ".modinfo");
1136 if (!sec) {
1137 ERROR("missing .modinfo section");
1138 return NULL;
1139 }
1140
1141 name = memmem(sec->data->d_buf, sec_size(sec), "\0name=", 6);
1142 if (name)
1143 return name + 6;
1144
1145 name = strdup(e->orig->name);
1146 if (!name) {
1147 ERROR_GLIBC("strdup");
1148 return NULL;
1149 }
1150
1151 return normalize_modname(name);
1152}
1153
1154/* Get the object's module name as defined by the kernel (and klp_object) */
1155static const char *find_modname(struct elfs *e)
1156{
1157 const char *modname;
1158
1159 if (e->modname)
1160 return e->modname;
1161
1162 modname = __find_modname(e);
1163 e->modname = modname;
1164 return modname;
1165}
1166
1167/*
1168 * Copying a function from its native compiled environment to a kernel module
1169 * removes its natural access to local functions/variables and unexported
1170 * globals. References to such symbols need to be converted to KLP relocs so
1171 * the kernel arch relocation code knows to apply them and where to find the
1172 * symbols. Particularly, duplicate static symbols need to be disambiguated.
1173 */
1174static bool klp_reloc_needed(struct reloc *patched_reloc)
1175{
1176 struct symbol *patched_sym = patched_reloc->sym;
1177 struct export *export;
1178
1179 /* no external symbol to reference */
1180 if (patched_sym->dont_correlate)
1181 return false;
1182
1183 /* For included functions, a regular reloc will do. */
1184 if (patched_sym->included)
1185 return false;
1186
1187 /*
1188 * If exported by a module, it has to be a klp reloc. Thanks to the
1189 * clusterfunk that is late module patching, the patch module is
1190 * allowed to be loaded before any modules it depends on.
1191 *
1192 * If exported by vmlinux to all modules, a normal reloc will do.
1193 */
1194 export = find_export(patched_sym);
1195 if (export) {
1196 if (strcmp(export->mod, "vmlinux"))
1197 return true;
1198
1199 /* EXPORT_SYMBOL_FOR_MODULES() gets a klp reloc */
1200 return export->mod_ns;
1201 }
1202
1203 if (!patched_sym->twin) {
1204 /*
1205 * Presumably the symbol and its reference were added by the
1206 * patch. The symbol could be defined in this .o or in another
1207 * .o in the patch module.
1208 *
1209 * This check needs to be *after* the export check due to the
1210 * possibility of the patch adding a new UNDEF reference to an
1211 * exported symbol.
1212 */
1213 return false;
1214 }
1215
1216 /* Unexported symbol which lives in the original vmlinux or module. */
1217 return true;
1218}
1219
1220/* Return -1 error, 0 success, 1 skip */
1221static int convert_reloc_sym_to_secsym(struct elf *elf, struct reloc *reloc)
1222{
1223 struct symbol *sym = reloc->sym;
1224 struct section *sec = sym->sec;
1225
1226 if (is_sec_sym(sym))
1227 return 0;
1228
1229 if (!sec->sym && !elf_create_section_symbol(elf, sec))
1230 return -1;
1231
1232 reloc->sym = sec->sym;
1233 set_reloc_sym(elf, reloc, sec->sym->idx);
1234 set_reloc_addend(elf, reloc, sym->offset + reloc_addend(reloc));
1235 return 0;
1236}
1237
1238/* Return -1 error, 0 success, 1 skip */
1239static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc)
1240{
1241 struct symbol *sym = reloc->sym;
1242 struct section *sec = sym->sec;
1243
1244 if (!is_sec_sym(sym))
1245 return 0;
1246
1247 /* If the symbol has a dedicated section, it's easy to find */
1248 sym = find_symbol_by_offset(sec, 0);
1249 if (sym && sym->len == sec_size(sec))
1250 goto found_sym;
1251
1252 /* No dedicated section; find the symbol manually */
1253 sym = find_symbol_containing_inclusive(sec, arch_adjusted_addend(reloc));
1254 if (!sym) {
1255 /*
1256 * This is presumably an .altinstr_replacement section which is
1257 * empty due to it only having zero-length replacement(s).
1258 */
1259 if (!sec_size(sec))
1260 return 1;
1261
1262 /*
1263 * .rodata is a mixed bag of named objects and anonymous data.
1264 *
1265 * Convert section symbol references to named object symbols
1266 * when possible, to preserve pointer identity for const
1267 * structs like file_operations. Otherwise a section symbol is
1268 * fine.
1269 */
1270 if (is_rodata_sec(sec))
1271 return 0;
1272
1273 /*
1274 * This can happen for special section references to weak code
1275 * whose symbol has been stripped by the linker.
1276 */
1277 return -1;
1278 }
1279
1280found_sym:
1281 reloc->sym = sym;
1282 set_reloc_sym(elf, reloc, sym->idx);
1283 set_reloc_addend(elf, reloc, reloc_addend(reloc) - sym->offset);
1284 return 0;
1285}
1286
1287/*
1288 * Sections with anonymous or uncorrelated data (strings, UBSAN data, Clang
1289 * anonymous constants) need section symbol references.
1290 */
1291static bool is_uncorrelated_section(struct section *sec)
1292{
1293 return is_string_sec(sec) ||
1294 strstarts(sec->name, ".data..Lubsan") || /* GCC */
1295 strstarts(sec->name, ".data..L__unnamed_") || /* Clang */
1296 strstarts(sec->name, ".data..Lanon."); /* Clang */
1297}
1298
1299/*
1300 * Convert a relocation symbol reference to the needed format: either a section
1301 * symbol or the underlying symbol itself. Return -1 error, 0 success, 1 skip.
1302 */
1303static int convert_reloc_sym(struct elf *elf, struct reloc *reloc)
1304{
1305 struct section *sec = reloc->sym->sec;
1306
1307 if (reloc_type(reloc) == R_NONE)
1308 return 1;
1309
1310 if (is_uncorrelated_section(sec))
1311 return convert_reloc_sym_to_secsym(elf, reloc);
1312
1313 /* Everything else: references should use named symbols. */
1314 return convert_reloc_secsym_to_sym(elf, reloc);
1315}
1316
1317/*
1318 * Check if the original module already has a dependency on dep_mod, i.e. it
1319 * already references at least one export from that module.
1320 */
1321static bool has_module_dep(struct elfs *e, const char *dep_mod)
1322{
1323 struct symbol *sym;
1324
1325 for_each_sym(e->orig, sym) {
1326 struct export *exp;
1327
1328 if (!is_undef_sym(sym) || is_weak_sym(sym))
1329 continue;
1330
1331 exp = find_export(sym);
1332 if (exp && !strcmp(exp->mod, dep_mod))
1333 return true;
1334 }
1335
1336 return false;
1337}
1338
1339/*
1340 * Convert a regular relocation to a klp relocation (sort of).
1341 */
1342static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc,
1343 struct section *sec, unsigned long offset,
1344 struct export *export)
1345{
1346 const char *sym_modname, *sym_orig_name, *sec_objname;
1347 struct symbol *patched_sym = patched_reloc->sym;
1348 s64 addend = reloc_addend(patched_reloc);
1349 char tombstone_name[SYM_NAME_LEN];
1350 struct symbol *sym, *klp_sym;
1351 unsigned long klp_reloc_off;
1352 struct section *klp_relocs;
1353 char sec_name[SEC_NAME_LEN];
1354 char sym_name[SYM_NAME_LEN];
1355 struct klp_reloc klp_reloc;
1356 unsigned long sympos;
1357
1358 if (!patched_sym->twin) {
1359 if (!export) {
1360 ERROR("unexpected klp reloc for new symbol %s", patched_sym->name);
1361 return -1;
1362 }
1363
1364 if (strcmp(export->mod, "vmlinux") &&
1365 !has_module_dep(e, export->mod)) {
1366 ERROR("%s: new reference to %s (exported by %s) would create an undeclared module dependency",
1367 patched_sym->name, export->sym, export->mod);
1368 return -1;
1369 }
1370 }
1371
1372 /*
1373 * Keep the original reloc intact for now to avoid breaking objtool run
1374 * which relies on proper relocations for many of its features. This
1375 * reloc now targets a functionally dead tombstone symbol and will be
1376 * disabled later by "objtool klp post-link".
1377 *
1378 * Convert the symbol to UNDEF/WEAK and rename to
1379 * .klp.tombstone.sym_name to prevent modpost from printing warnings or
1380 * creating false module dependencies. The prefix is hidden from the
1381 * objtool run itself by read_symbols().
1382 */
1383
1384 sym = patched_sym->clone;
1385 if (!sym) {
1386 if (snprintf_check(tombstone_name, SYM_NAME_LEN,
1387 KLP_TOMBSTONE_PREFIX "%s", patched_sym->name))
1388 return -1;
1389
1390 sym = elf_create_symbol(e->out, tombstone_name, NULL,
1391 STB_WEAK, patched_sym->type, 0, 0);
1392 if (!sym)
1393 return -1;
1394
1395 patched_sym->clone = sym;
1396 sym->clone = patched_sym;
1397 }
1398
1399 if (!elf_create_reloc(e->out, sec, offset, sym, addend, reloc_type(patched_reloc)))
1400 return -1;
1401
1402 /*
1403 * Create the KLP symbol.
1404 */
1405
1406 if (export) {
1407 sym_modname = export->mod;
1408 sym_orig_name = export->sym;
1409 sympos = 0;
1410 } else {
1411 sym_modname = find_modname(e);
1412 if (!sym_modname)
1413 return -1;
1414
1415 sym_orig_name = patched_sym->twin->name;
1416 sympos = klp_find_sympos(e->orig, patched_sym->twin);
1417 if (sympos == ULONG_MAX)
1418 return -1;
1419 }
1420
1421 /* symbol format: .klp.sym.modname.sym_name,sympos */
1422 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_SYM_PREFIX "%s.%s,%ld",
1423 sym_modname, sym_orig_name, sympos))
1424 return -1;
1425
1426 klp_sym = find_symbol_by_name(e->out, sym_name);
1427 if (!klp_sym) {
1428 __dbg_clone("%s", sym_name);
1429
1430 /* STB_WEAK: avoid modpost undefined symbol warnings */
1431 klp_sym = elf_create_symbol(e->out, sym_name, NULL,
1432 STB_WEAK, patched_sym->type, 0, 0);
1433 if (!klp_sym)
1434 return -1;
1435 }
1436
1437 /*
1438 * Create the __klp_relocs.<objname> entry. This will be converted to
1439 * an actual KLP rela by "objtool klp post-link".
1440 *
1441 * This intermediate step is necessary to prevent corruption by the
1442 * linker, which doesn't know how to properly handle two rela sections
1443 * applying to the same base section.
1444 *
1445 * The objname decides when the reloc gets applied. A reference to a
1446 * vmlinux symbol goes in the vmlinux section so it gets applied when
1447 * the patch module loads. Everything else goes in the patched
1448 * object's section, applied when the patched module is loaded.
1449 */
1450
1451 if (!strcmp(sym_modname, "vmlinux")) {
1452 sec_objname = "vmlinux";
1453 } else {
1454 sec_objname = find_modname(e);
1455 if (!sec_objname)
1456 return -1;
1457 }
1458
1459 /* section format: __klp_relocs.objname */
1460 if (snprintf_check(sec_name, SEC_NAME_LEN,
1461 KLP_RELOCS_SEC ".%s", sec_objname))
1462 return -1;
1463
1464 klp_relocs = find_section_by_name(e->out, sec_name);
1465 if (!klp_relocs) {
1466 klp_relocs = elf_create_section(e->out, sec_name, 0,
1467 0, SHT_PROGBITS, 8, SHF_ALLOC);
1468 if (!klp_relocs)
1469 return -1;
1470 }
1471
1472 klp_reloc_off = sec_size(klp_relocs);
1473 memset(&klp_reloc, 0, sizeof(klp_reloc));
1474
1475 klp_reloc.type = reloc_type(patched_reloc);
1476 if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc)))
1477 return -1;
1478
1479 /* klp_reloc.offset */
1480 if (!sec->sym && !elf_create_section_symbol(e->out, sec))
1481 return -1;
1482
1483 if (!elf_create_reloc(e->out, klp_relocs,
1484 klp_reloc_off + offsetof(struct klp_reloc, offset),
1485 sec->sym, offset, R_ABS64))
1486 return -1;
1487
1488 /* klp_reloc.sym */
1489 if (!elf_create_reloc(e->out, klp_relocs,
1490 klp_reloc_off + offsetof(struct klp_reloc, sym),
1491 klp_sym, addend, R_ABS64))
1492 return -1;
1493
1494 return 0;
1495}
1496
1497#define dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp) \
1498 dbg_clone("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \
1499 sec->name, offset, patched_sym->name, \
1500 addend >= 0 ? "+" : "-", labs(addend), \
1501 sym_type(patched_sym), \
1502 is_sec_sym(patched_sym) ? "" : " ", \
1503 is_sec_sym(patched_sym) ? "" : sym_bind(patched_sym), \
1504 is_undef_sym(patched_sym) ? " UNDEF" : "", \
1505 export ? " EXPORTED" : "", \
1506 klp ? " KLP" : "")
1507
1508/* Copy a reloc and its symbol to the output object */
1509static int clone_reloc(struct elfs *e, struct reloc *patched_reloc,
1510 struct section *sec, unsigned long offset)
1511{
1512 struct symbol *patched_sym = patched_reloc->sym;
1513 struct export *export = find_export(patched_sym);
1514 long addend = reloc_addend(patched_reloc);
1515 struct symbol *out_sym;
1516 bool klp;
1517
1518 klp = klp_reloc_needed(patched_reloc);
1519
1520 dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp);
1521
1522 if (klp) {
1523 if (clone_reloc_klp(e, patched_reloc, sec, offset, export))
1524 return -1;
1525
1526 return 0;
1527 }
1528
1529 /*
1530 * Why !export sets 'data_too':
1531 *
1532 * Unexported non-klp symbols need to live in the patch module,
1533 * otherwise there will be unresolved symbols. Notably, this includes:
1534 *
1535 * - New functions/data
1536 * - String sections
1537 * - Special section entries
1538 * - Uncorrelated static local variables
1539 * - UBSAN sections
1540 */
1541 out_sym = clone_symbol(e, patched_sym, patched_sym->included || !export);
1542 if (!out_sym)
1543 return -1;
1544
1545 /*
1546 * For strings, all references use section symbols, thanks to
1547 * convert_reloc_sym(). clone_symbol() has cloned an empty
1548 * version of the string section. Now copy the string itself.
1549 */
1550 if (is_string_sec(patched_sym->sec)) {
1551 const char *str = patched_sym->sec->data->d_buf + addend;
1552
1553 __dbg_clone("\"%s\"", escape_str(str));
1554
1555 addend = elf_add_string(e->out, out_sym->sec, str);
1556 if (addend == -1)
1557 return -1;
1558 }
1559
1560 if (!elf_create_reloc(e->out, sec, offset, out_sym, addend,
1561 reloc_type(patched_reloc)))
1562 return -1;
1563
1564 return 0;
1565}
1566
1567/* Copy all relocs needed for a symbol's contents */
1568static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym)
1569{
1570 struct section *patched_rsec = patched_sym->sec->rsec;
1571 struct reloc *patched_reloc;
1572 unsigned long start, end;
1573 struct symbol *out_sym;
1574
1575 out_sym = patched_sym->clone;
1576 if (!out_sym) {
1577 ERROR("no clone for %s", patched_sym->name);
1578 return -1;
1579 }
1580
1581 if (!patched_rsec)
1582 return 0;
1583
1584 if (!is_sec_sym(patched_sym) && !patched_sym->len)
1585 return 0;
1586
1587 if (is_string_sec(patched_sym->sec))
1588 return 0;
1589
1590 if (is_sec_sym(patched_sym)) {
1591 start = 0;
1592 end = sec_size(patched_sym->sec);
1593 } else {
1594 start = patched_sym->offset;
1595 end = start + patched_sym->len;
1596 }
1597
1598 for_each_reloc(patched_rsec, patched_reloc) {
1599 unsigned long offset;
1600 int ret;
1601
1602 if (reloc_offset(patched_reloc) < start ||
1603 reloc_offset(patched_reloc) >= end)
1604 continue;
1605
1606 /*
1607 * Skip any reloc referencing .altinstr_aux. Its code is
1608 * always patched by alternatives. See ALTERNATIVE_TERNARY().
1609 */
1610 if (patched_reloc->sym->sec &&
1611 !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux"))
1612 continue;
1613
1614 if (arch_alt_ignore_new_reloc(patched_sym->sec,
1615 reloc_offset(patched_reloc)))
1616 continue;
1617
1618 ret = convert_reloc_sym(e->patched, patched_reloc);
1619 if (ret < 0) {
1620 ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc),
1621 "failed to convert reloc sym '%s' to its proper format",
1622 patched_reloc->sym->name);
1623 return -1;
1624 }
1625 if (ret > 0)
1626 continue;
1627
1628 offset = out_sym->offset + (reloc_offset(patched_reloc) - patched_sym->offset);
1629
1630 if (clone_reloc(e, patched_reloc, out_sym->sec, offset))
1631 return -1;
1632 }
1633 return 0;
1634
1635}
1636
1637static int create_fake_symbol(struct elf *elf, struct section *sec,
1638 unsigned long offset, size_t size)
1639{
1640 char name[SYM_NAME_LEN];
1641 struct symbol *sym;
1642 unsigned int type;
1643 static int ctr;
1644 char *c;
1645
1646 if (snprintf_check(name, SYM_NAME_LEN, "%s_%d", sec->name, ctr++))
1647 return -1;
1648
1649 for (c = name; *c; c++)
1650 if (*c == '.')
1651 *c = '_';
1652
1653 /*
1654 * STT_NOTYPE: Prevent objtool from validating .altinstr_replacement
1655 * while still allowing objdump to disassemble it.
1656 */
1657 type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT;
1658
1659 sym = elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size);
1660 if (!sym)
1661 return -1;
1662
1663 sym->fake = 1;
1664 return 0;
1665}
1666
1667static bool has_fake_symbols(struct section *sec)
1668{
1669 struct symbol *sym;
1670
1671 sec_for_each_sym(sec, sym)
1672 if (sym->fake)
1673 return true;
1674
1675 return false;
1676}
1677
1678/*
1679 * Special sections (alternatives, etc) are basically arrays of structs.
1680 * For all the special sections, create a symbol for each struct entry. This
1681 * is a bit cumbersome, but it makes the extracting of the individual entries
1682 * much more straightforward.
1683 *
1684 * There are three ways to identify the entry sizes for a special section:
1685 *
1686 * 1) ELF section header sh_entsize: Ideally this would be used almost
1687 * everywhere. But unfortunately the toolchains make it difficult. The
1688 * assembler .[push]section directive syntax only takes entsize when
1689 * combined with SHF_MERGE. But Clang disallows combining SHF_MERGE with
1690 * SHF_WRITE. And some special sections do need to be writable.
1691 *
1692 * Another place this wouldn't work is .altinstr_replacement, whose entries
1693 * don't have a fixed size.
1694 *
1695 * 2) ANNOTATE_DATA_SPECIAL: This is a lightweight objtool annotation which
1696 * points to the beginning of each entry. The size of the entry is then
1697 * inferred by the location of the subsequent annotation (or end of
1698 * section).
1699 *
1700 * 3) Simple array of pointers: If the special section is just a basic array of
1701 * pointers, the entry size can be inferred by the number of relocations.
1702 * No annotations needed.
1703 *
1704 * Note I also tried to create per-entry symbols at the time of creation, in
1705 * the original [inline] asm. Unfortunately, creating uniquely named symbols
1706 * is trickier than one might think, especially with Clang inline asm. I
1707 * eventually just gave up trying to make that work, in favor of using
1708 * ANNOTATE_DATA_SPECIAL and creating the symbols here after the fact.
1709 */
1710static int create_fake_symbols(struct elf *elf)
1711{
1712 struct section *sec;
1713 struct reloc *reloc;
1714
1715 /*
1716 * 1) Make symbols for all the ANNOTATE_DATA_SPECIAL entries:
1717 */
1718
1719 sec = find_section_by_name(elf, ".discard.annotate_data");
1720 if (!sec || !sec->rsec)
1721 goto entsize;
1722
1723 for_each_reloc(sec->rsec, reloc) {
1724 unsigned long offset, size;
1725 struct reloc *next_reloc;
1726 bool last = true;
1727
1728 if (annotype(elf, sec, reloc) != ANNOTYPE_DATA_SPECIAL)
1729 continue;
1730
1731 offset = reloc_addend(reloc);
1732
1733 /*
1734 * Find the start of the next entry so the fake symbol size can
1735 * be calculated.
1736 */
1737 next_reloc = reloc;
1738 for_each_reloc_continue(sec->rsec, next_reloc) {
1739 if (annotype(elf, sec, next_reloc) != ANNOTYPE_DATA_SPECIAL ||
1740 next_reloc->sym->sec != reloc->sym->sec)
1741 continue;
1742
1743 size = reloc_addend(next_reloc) - offset;
1744 last = false;
1745 break;
1746 }
1747
1748 /*
1749 * If no next entry found, this is the last entry, so its size
1750 * is from the current offset to the end of the section.
1751 */
1752 if (last)
1753 size = sec_size(reloc->sym->sec) - offset;
1754
1755 if (create_fake_symbol(elf, reloc->sym->sec, offset, size))
1756 return -1;
1757 }
1758
1759 /*
1760 * 2) Make symbols for sh_entsize, and simple arrays of pointers:
1761 */
1762entsize:
1763 for_each_sec(elf, sec) {
1764 unsigned int entry_size;
1765 unsigned long offset;
1766
1767 if (!is_special_section(sec))
1768 continue;
1769
1770 /* Skip sections already handled by step 1 above */
1771 if (has_fake_symbols(sec))
1772 continue;
1773
1774 if (!sec->rsec) {
1775 ERROR("%s: missing special section relocations", sec->name);
1776 return -1;
1777 }
1778
1779 entry_size = sec->sh.sh_entsize;
1780 if (!entry_size) {
1781 entry_size = arch_reloc_size(sec->rsec->relocs);
1782 if (sec_size(sec) != entry_size * sec_num_entries(sec->rsec)) {
1783 ERROR("%s: missing special section entsize or annotations", sec->name);
1784 return -1;
1785 }
1786 }
1787
1788 for (offset = 0; offset < sec_size(sec); offset += entry_size) {
1789 if (create_fake_symbol(elf, sec, offset, entry_size))
1790 return -1;
1791 }
1792 }
1793
1794 return 0;
1795}
1796
1797/* Keep a special section entry if it references an included function */
1798static bool should_keep_special_sym(struct elf *elf, struct symbol *sym)
1799{
1800 bool annotate_insn = !strcmp(sym->sec->name, ".discard.annotate_insn");
1801 struct reloc *reloc;
1802
1803 if (is_sec_sym(sym) || !sym->sec->rsec)
1804 return false;
1805
1806 sym_for_each_reloc(elf, sym, reloc) {
1807 if (convert_reloc_sym(elf, reloc))
1808 continue;
1809
1810 if (!reloc->sym->clone || is_undef_sym(reloc->sym->clone))
1811 continue;
1812
1813 /*
1814 * Keep special section references to cloned functions.
1815 * In some cases annotate_insn can also reference cloned alt
1816 * replacement fake symbols; keep those references as well.
1817 */
1818 if (is_func_sym(reloc->sym) ||
1819 (annotate_insn && is_notype_sym(reloc->sym)))
1820 return true;
1821 }
1822
1823 return false;
1824}
1825
1826/*
1827 * Klp relocations aren't allowed for __jump_table and .static_call_sites if
1828 * the referenced symbol lives in a kernel module, because such klp relocs may
1829 * be applied after static branch/call init, resulting in code corruption.
1830 *
1831 * Validate a special section entry to avoid that. Note that an inert
1832 * tracepoint or pr_debug() is harmless enough, in that case just skip the
1833 * entry and print a warning. Otherwise, return an error.
1834 *
1835 * TODO: This is only a temporary limitation which will be fixed when livepatch
1836 * adds support for submodules: fully self-contained modules which are embedded
1837 * in the top-level livepatch module's data and which can be loaded on demand
1838 * when their corresponding to-be-patched module gets loaded. Then klp relocs
1839 * can be retired.
1840 *
1841 * Return:
1842 * -1: error: validation failed
1843 * 1: warning: disabled tracepoint or pr_debug()
1844 * 0: success
1845 */
1846static int validate_special_section_klp_reloc(struct elfs *e, struct symbol *sym)
1847{
1848 bool static_branch = !strcmp(sym->sec->name, "__jump_table");
1849 bool static_call = !strcmp(sym->sec->name, ".static_call_sites");
1850 const char *code_sym = NULL;
1851 unsigned long code_offset = 0;
1852 struct reloc *reloc;
1853 int ret = 0;
1854
1855 if (!static_branch && !static_call)
1856 return 0;
1857
1858 sym_for_each_reloc(e->patched, sym, reloc) {
1859 const char *sym_modname;
1860 struct export *export;
1861
1862 if (convert_reloc_sym(e->patched, reloc))
1863 continue;
1864
1865 /* Static branch/call keys are always STT_OBJECT */
1866 if (reloc->sym->type != STT_OBJECT) {
1867
1868 /* Save code location which can be printed below */
1869 if (reloc->sym->type == STT_FUNC && !code_sym) {
1870 code_sym = reloc->sym->name;
1871 code_offset = reloc_addend(reloc);
1872 }
1873
1874 continue;
1875 }
1876
1877 if (!klp_reloc_needed(reloc))
1878 continue;
1879
1880 export = find_export(reloc->sym);
1881 if (export) {
1882 sym_modname = export->mod;
1883 } else {
1884 sym_modname = find_modname(e);
1885 if (!sym_modname)
1886 return -1;
1887 }
1888
1889 /* vmlinux keys are ok */
1890 if (!strcmp(sym_modname, "vmlinux"))
1891 continue;
1892
1893 if (!code_sym)
1894 code_sym = "<unknown>";
1895
1896 if (static_branch) {
1897 if (strstarts(reloc->sym->name, "__tracepoint_")) {
1898 WARN("%s: disabling unsupported tracepoint %s",
1899 code_sym, reloc->sym->name + 13);
1900 ret = 1;
1901 continue;
1902 }
1903
1904 if (strstr(reloc->sym->name, "__UNIQUE_ID_ddebug_")) {
1905 WARN("%s: disabling unsupported pr_debug()",
1906 code_sym);
1907 ret = 1;
1908 continue;
1909 }
1910
1911 ERROR("%s+0x%lx: unsupported static branch key %s. Use static_key_enabled() instead",
1912 code_sym, code_offset, reloc->sym->name);
1913 return -1;
1914 }
1915
1916 /* static call */
1917 if (strstarts(reloc->sym->name, "__SCK__tp_func_")) {
1918 ret = 1;
1919 continue;
1920 }
1921
1922 ERROR("%s()+0x%lx: unsupported static call key %s. Use KLP_STATIC_CALL() instead",
1923 code_sym, code_offset, reloc->sym->name);
1924 return -1;
1925 }
1926
1927 return ret;
1928}
1929
1930static int clone_special_section(struct elfs *e, struct section *patched_sec)
1931{
1932 struct symbol *patched_sym;
1933
1934 /*
1935 * Extract all special section symbols (and their dependencies) which
1936 * reference included functions.
1937 */
1938 sec_for_each_sym(patched_sec, patched_sym) {
1939 int ret;
1940
1941 if (!is_object_sym(patched_sym))
1942 continue;
1943
1944 if (!should_keep_special_sym(e->patched, patched_sym))
1945 continue;
1946
1947 ret = validate_special_section_klp_reloc(e, patched_sym);
1948 if (ret < 0)
1949 return -1;
1950 if (ret > 0)
1951 continue;
1952
1953 if (!clone_symbol(e, patched_sym, true))
1954 return -1;
1955 }
1956
1957 return 0;
1958}
1959
1960/* Extract only the needed bits from special sections */
1961static int clone_special_sections(struct elfs *e)
1962{
1963 struct section *sec, *annotate_insn = NULL;
1964
1965 for_each_sec(e->patched, sec) {
1966 if (is_special_section(sec)) {
1967 if (!strcmp(sec->name, ".discard.annotate_insn")) {
1968 annotate_insn = sec;
1969 continue;
1970 }
1971 if (clone_special_section(e, sec))
1972 return -1;
1973 }
1974 }
1975
1976 /*
1977 * Do .discard.annotate_insn last, it can reference other special
1978 * sections (alt replacements) so they need to be cloned first.
1979 */
1980 if (annotate_insn) {
1981 if (clone_special_section(e, annotate_insn))
1982 return -1;
1983 }
1984
1985 return 0;
1986}
1987
1988/*
1989 * Create .init.klp_objects and .init.klp_funcs sections which are intermediate
1990 * sections provided as input to the patch module's init code for building the
1991 * klp_patch, klp_object and klp_func structs for the livepatch API.
1992 */
1993static int create_klp_sections(struct elfs *e)
1994{
1995 size_t obj_size = sizeof(struct klp_object_ext);
1996 size_t func_size = sizeof(struct klp_func_ext);
1997 struct section *obj_sec, *funcs_sec, *str_sec;
1998 struct symbol *funcs_sym, *str_sym, *sym;
1999 char sym_name[SYM_NAME_LEN];
2000 unsigned int nr_funcs = 0;
2001 const char *modname;
2002 void *obj_data;
2003 s64 addend;
2004
2005 obj_sec = elf_create_section_pair(e->out, KLP_OBJECTS_SEC, obj_size, 0, 0);
2006 if (!obj_sec)
2007 return -1;
2008
2009 funcs_sec = elf_create_section_pair(e->out, KLP_FUNCS_SEC, func_size, 0, 0);
2010 if (!funcs_sec)
2011 return -1;
2012
2013 funcs_sym = elf_create_section_symbol(e->out, funcs_sec);
2014 if (!funcs_sym)
2015 return -1;
2016
2017 str_sec = elf_create_section(e->out, KLP_STRINGS_SEC, 0, 0,
2018 SHT_PROGBITS, 1,
2019 SHF_ALLOC | SHF_STRINGS | SHF_MERGE);
2020 if (!str_sec)
2021 return -1;
2022
2023 if (elf_add_string(e->out, str_sec, "") == -1)
2024 return -1;
2025
2026 str_sym = elf_create_section_symbol(e->out, str_sec);
2027 if (!str_sym)
2028 return -1;
2029
2030 /* allocate klp_object_ext */
2031 obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size);
2032 if (!obj_data)
2033 return -1;
2034
2035 modname = find_modname(e);
2036 if (!modname)
2037 return -1;
2038
2039 /* klp_object_ext.name */
2040 if (strcmp(modname, "vmlinux")) {
2041 addend = elf_add_string(e->out, str_sec, modname);
2042 if (addend == -1)
2043 return -1;
2044
2045 if (!elf_create_reloc(e->out, obj_sec,
2046 offsetof(struct klp_object_ext, name),
2047 str_sym, addend, R_ABS64))
2048 return -1;
2049 }
2050
2051 /* klp_object_ext.funcs */
2052 if (!elf_create_reloc(e->out, obj_sec, offsetof(struct klp_object_ext, funcs),
2053 funcs_sym, 0, R_ABS64))
2054 return -1;
2055
2056 for_each_sym(e->out, sym) {
2057 unsigned long offset = nr_funcs * func_size;
2058 unsigned long sympos;
2059 void *func_data;
2060
2061 if (!is_func_sym(sym) || is_cold_func(sym) ||
2062 !sym->clone || !sym->clone->changed)
2063 continue;
2064
2065 /* allocate klp_func_ext */
2066 func_data = elf_add_data(e->out, funcs_sec, NULL, func_size);
2067 if (!func_data)
2068 return -1;
2069
2070 /* klp_func_ext.old_name */
2071 addend = elf_add_string(e->out, str_sec, sym->clone->twin->name);
2072 if (addend == -1)
2073 return -1;
2074
2075 if (!elf_create_reloc(e->out, funcs_sec,
2076 offset + offsetof(struct klp_func_ext, old_name),
2077 str_sym, addend, R_ABS64))
2078 return -1;
2079
2080 /* klp_func_ext.new_func */
2081 if (!elf_create_reloc(e->out, funcs_sec,
2082 offset + offsetof(struct klp_func_ext, new_func),
2083 sym, 0, R_ABS64))
2084 return -1;
2085
2086 /* klp_func_ext.sympos */
2087 BUILD_BUG_ON(sizeof(sympos) != sizeof_field(struct klp_func_ext, sympos));
2088 sympos = klp_find_sympos(e->orig, sym->clone->twin);
2089 if (sympos == ULONG_MAX)
2090 return -1;
2091 memcpy(func_data + offsetof(struct klp_func_ext, sympos), &sympos,
2092 sizeof_field(struct klp_func_ext, sympos));
2093
2094 nr_funcs++;
2095 }
2096
2097 /* klp_object_ext.nr_funcs */
2098 BUILD_BUG_ON(sizeof(nr_funcs) != sizeof_field(struct klp_object_ext, nr_funcs));
2099 memcpy(obj_data + offsetof(struct klp_object_ext, nr_funcs), &nr_funcs,
2100 sizeof_field(struct klp_object_ext, nr_funcs));
2101
2102 /*
2103 * Find callback pointers created by KLP_PRE_PATCH_CALLBACK() and
2104 * friends, and add them to the klp object.
2105 */
2106
2107 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_PATCH_PREFIX "%s", modname))
2108 return -1;
2109
2110 sym = find_symbol_by_name(e->out, sym_name);
2111 if (sym) {
2112 struct reloc *reloc;
2113
2114 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2115
2116 if (!elf_create_reloc(e->out, obj_sec,
2117 offsetof(struct klp_object_ext, callbacks) +
2118 offsetof(struct klp_callbacks, pre_patch),
2119 reloc->sym, reloc_addend(reloc), R_ABS64))
2120 return -1;
2121 }
2122
2123 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_PATCH_PREFIX "%s", modname))
2124 return -1;
2125
2126 sym = find_symbol_by_name(e->out, sym_name);
2127 if (sym) {
2128 struct reloc *reloc;
2129
2130 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2131
2132 if (!elf_create_reloc(e->out, obj_sec,
2133 offsetof(struct klp_object_ext, callbacks) +
2134 offsetof(struct klp_callbacks, post_patch),
2135 reloc->sym, reloc_addend(reloc), R_ABS64))
2136 return -1;
2137 }
2138
2139 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_UNPATCH_PREFIX "%s", modname))
2140 return -1;
2141
2142 sym = find_symbol_by_name(e->out, sym_name);
2143 if (sym) {
2144 struct reloc *reloc;
2145
2146 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2147
2148 if (!elf_create_reloc(e->out, obj_sec,
2149 offsetof(struct klp_object_ext, callbacks) +
2150 offsetof(struct klp_callbacks, pre_unpatch),
2151 reloc->sym, reloc_addend(reloc), R_ABS64))
2152 return -1;
2153 }
2154
2155 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_UNPATCH_PREFIX "%s", modname))
2156 return -1;
2157
2158 sym = find_symbol_by_name(e->out, sym_name);
2159 if (sym) {
2160 struct reloc *reloc;
2161
2162 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset);
2163
2164 if (!elf_create_reloc(e->out, obj_sec,
2165 offsetof(struct klp_object_ext, callbacks) +
2166 offsetof(struct klp_callbacks, post_unpatch),
2167 reloc->sym, reloc_addend(reloc), R_ABS64))
2168 return -1;
2169 }
2170
2171 return 0;
2172}
2173
2174/*
2175 * Copy all .modinfo import_ns= tags to ensure all namespaced exported symbols
2176 * can be accessed via normal relocs.
2177 */
2178static int copy_import_ns(struct elfs *e)
2179{
2180 struct section *patched_sec, *out_sec = NULL;
2181 char *import_ns, *data_end;
2182
2183 patched_sec = find_section_by_name(e->patched, ".modinfo");
2184 if (!patched_sec)
2185 return 0;
2186
2187 import_ns = patched_sec->data->d_buf;
2188 if (!import_ns)
2189 return 0;
2190
2191 for (data_end = import_ns + sec_size(patched_sec);
2192 import_ns < data_end;
2193 import_ns += strlen(import_ns) + 1) {
2194
2195 import_ns = memmem(import_ns, data_end - import_ns, "import_ns=", 10);
2196 if (!import_ns)
2197 return 0;
2198
2199 if (!out_sec) {
2200 out_sec = find_section_by_name(e->out, ".modinfo");
2201 if (!out_sec) {
2202 out_sec = elf_create_section(e->out, ".modinfo", 0,
2203 patched_sec->sh.sh_entsize,
2204 patched_sec->sh.sh_type,
2205 patched_sec->sh.sh_addralign,
2206 patched_sec->sh.sh_flags);
2207 if (!out_sec)
2208 return -1;
2209 }
2210 }
2211
2212 if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1))
2213 return -1;
2214 }
2215
2216 return 0;
2217}
2218
2219int cmd_klp_diff(int argc, const char **argv)
2220{
2221 struct elfs e = {0};
2222 int ret;
2223
2224 argc = parse_options(argc, argv, klp_diff_options, klp_diff_usage, 0);
2225 if (argc != 3)
2226 usage_with_options(klp_diff_usage, klp_diff_options);
2227
2228 if (debug) {
2229 debug_correlate = true;
2230 debug_clone = true;
2231 }
2232
2233 objname = argv[0];
2234
2235 e.orig = elf_open_read(argv[0], O_RDONLY);
2236 e.patched = elf_open_read(argv[1], O_RDONLY);
2237 e.out = NULL;
2238
2239 if (!e.orig || !e.patched)
2240 return -1;
2241
2242 if (klp_sympos_init(e.orig))
2243 return -1;
2244
2245 if (read_exports())
2246 return -1;
2247
2248 if (read_sym_checksums(e.orig))
2249 return -1;
2250
2251 if (read_sym_checksums(e.patched))
2252 return -1;
2253
2254 if (correlate_symbols(&e))
2255 return -1;
2256
2257 ret = mark_changed_functions(&e);
2258 if (ret < 0)
2259 return -1;
2260 if (ret > 0)
2261 return 0;
2262
2263 e.out = elf_create_file(&e.orig->ehdr, argv[2]);
2264 if (!e.out)
2265 return -1;
2266
2267 /*
2268 * Special section fake symbols are needed so that individual special
2269 * section entries can be extracted by clone_special_sections().
2270 *
2271 * Note the fake symbols are also needed by clone_included_functions()
2272 * because __WARN_printf() call sites add references to bug table
2273 * entries in the calling functions.
2274 */
2275 if (create_fake_symbols(e.patched))
2276 return -1;
2277
2278 if (clone_included_functions(&e))
2279 return -1;
2280
2281 if (clone_special_sections(&e))
2282 return -1;
2283
2284 if (create_klp_sections(&e))
2285 return -1;
2286
2287 if (copy_import_ns(&e))
2288 return -1;
2289
2290 if (elf_write(e.out))
2291 return -1;
2292
2293 return elf_close(e.out);
2294}