projects/linux·blob uaccess-cleanups · refs

1010 lines · 23.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * sorttable.c: Sort the kernel's table
4 *
5 * Added ORC unwind tables sort support and other updates:
6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
7 * Shile Zhang <[email protected]>
8 *
9 * Copyright 2011 - 2012 Cavium, Inc.
10 *
11 * Based on code taken from recortmcount.c which is:
12 *
13 * Copyright 2009 John F. Reiser <[email protected]>. All rights reserved.
14 *
15 * Restructured to fit Linux format, as well as other updates:
16 * Copyright 2010 Steven Rostedt <[email protected]>, Red Hat Inc.
17 */
18
19/*
20 * Strategy: alter the vmlinux file in-place.
21 */
22
23#include <sys/types.h>
24#include <sys/stat.h>
25#include <getopt.h>
26#include <fcntl.h>
27#include <stdio.h>
28#include <stdlib.h>
29#include <stdbool.h>
30#include <string.h>
31#include <unistd.h>
32#include <errno.h>
33#include <pthread.h>
34
35#include "elf-parse.h"
36
37#ifndef EM_ARCOMPACT
38#define EM_ARCOMPACT 93
39#endif
40
41#ifndef EM_XTENSA
42#define EM_XTENSA 94
43#endif
44
45#ifndef EM_AARCH64
46#define EM_AARCH64 183
47#endif
48
49#ifndef EM_MICROBLAZE
50#define EM_MICROBLAZE 189
51#endif
52
53#ifndef EM_ARCV2
54#define EM_ARCV2 195
55#endif
56
57#ifndef EM_RISCV
58#define EM_RISCV 243
59#endif
60
61#ifndef EM_LOONGARCH
62#define EM_LOONGARCH 258
63#endif
64
65typedef void (*table_sort_t)(char *, int);
66
67/*
68 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of
69 * the way to -256..-1, to avoid conflicting with real section
70 * indices.
71 */
72#define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1))
73
74static inline int is_shndx_special(unsigned int i)
75{
76 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE;
77}
78
79/* Accessor for sym->st_shndx, hides ugliness of "64k sections" */
80static inline unsigned int get_secindex(unsigned int shndx,
81 unsigned int sym_offs,
82 const Elf32_Word *symtab_shndx_start)
83{
84 if (is_shndx_special(shndx))
85 return SPECIAL(shndx);
86 if (shndx != SHN_XINDEX)
87 return shndx;
88 return elf_parser.r(&symtab_shndx_start[sym_offs]);
89}
90
91static int compare_extable_32(const void *a, const void *b)
92{
93 Elf32_Addr av = elf_parser.r(a);
94 Elf32_Addr bv = elf_parser.r(b);
95
96 if (av < bv)
97 return -1;
98 return av > bv;
99}
100
101static int compare_extable_64(const void *a, const void *b)
102{
103 Elf64_Addr av = elf_parser.r8(a);
104 Elf64_Addr bv = elf_parser.r8(b);
105
106 if (av < bv)
107 return -1;
108 return av > bv;
109}
110
111static int (*compare_extable)(const void *a, const void *b);
112
113static inline void *get_index(void *start, int entsize, int index)
114{
115 return start + (entsize * index);
116}
117
118static int extable_ent_size;
119static int long_size __maybe_unused;
120
121#define ERRSTR_MAXSZ 256
122
123#ifdef UNWINDER_ORC_ENABLED
124/* ORC unwinder only support X86_64 */
125#include <asm/orc_types.h>
126
127static char g_err[ERRSTR_MAXSZ];
128static int *g_orc_ip_table;
129static struct orc_entry *g_orc_table;
130
131static pthread_t orc_sort_thread;
132
133static inline unsigned long orc_ip(const int *ip)
134{
135 return (unsigned long)ip + *ip;
136}
137
138static int orc_sort_cmp(const void *_a, const void *_b)
139{
140 struct orc_entry *orc_a, *orc_b;
141 const int *a = g_orc_ip_table + *(int *)_a;
142 const int *b = g_orc_ip_table + *(int *)_b;
143 unsigned long a_val = orc_ip(a);
144 unsigned long b_val = orc_ip(b);
145
146 if (a_val > b_val)
147 return 1;
148 if (a_val < b_val)
149 return -1;
150
151 /*
152 * The "weak" section terminator entries need to always be on the left
153 * to ensure the lookup code skips them in favor of real entries.
154 * These terminator entries exist to handle any gaps created by
155 * whitelisted .o files which didn't get objtool generation.
156 */
157 orc_a = g_orc_table + (a - g_orc_ip_table);
158 orc_b = g_orc_table + (b - g_orc_ip_table);
159 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED)
160 return 0;
161 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
162}
163
164static void *sort_orctable(void *arg)
165{
166 int i;
167 int *idxs = NULL;
168 int *tmp_orc_ip_table = NULL;
169 struct orc_entry *tmp_orc_table = NULL;
170 unsigned int *orc_ip_size = (unsigned int *)arg;
171 unsigned int num_entries = *orc_ip_size / sizeof(int);
172 unsigned int orc_size = num_entries * sizeof(struct orc_entry);
173
174 idxs = (int *)malloc(*orc_ip_size);
175 if (!idxs) {
176 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
177 strerror(errno));
178 pthread_exit(g_err);
179 }
180
181 tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
182 if (!tmp_orc_ip_table) {
183 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
184 strerror(errno));
185 pthread_exit(g_err);
186 }
187
188 tmp_orc_table = (struct orc_entry *)malloc(orc_size);
189 if (!tmp_orc_table) {
190 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
191 strerror(errno));
192 pthread_exit(g_err);
193 }
194
195 /* initialize indices array, convert ip_table to absolute address */
196 for (i = 0; i < num_entries; i++) {
197 idxs[i] = i;
198 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
199 }
200 memcpy(tmp_orc_table, g_orc_table, orc_size);
201
202 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
203
204 for (i = 0; i < num_entries; i++) {
205 if (idxs[i] == i)
206 continue;
207
208 /* convert back to relative address */
209 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
210 g_orc_table[i] = tmp_orc_table[idxs[i]];
211 }
212
213 free(idxs);
214 free(tmp_orc_ip_table);
215 free(tmp_orc_table);
216 pthread_exit(NULL);
217}
218#endif
219
220#ifdef MCOUNT_SORT_ENABLED
221
222static int compare_values_64(const void *a, const void *b)
223{
224 uint64_t av = *(uint64_t *)a;
225 uint64_t bv = *(uint64_t *)b;
226
227 if (av < bv)
228 return -1;
229 return av > bv;
230}
231
232static int compare_values_32(const void *a, const void *b)
233{
234 uint32_t av = *(uint32_t *)a;
235 uint32_t bv = *(uint32_t *)b;
236
237 if (av < bv)
238 return -1;
239 return av > bv;
240}
241
242static int (*compare_values)(const void *a, const void *b);
243
244/* Only used for sorting mcount table */
245static void rela_write_addend(Elf_Rela *rela, uint64_t val)
246{
247 elf_parser.rela_write_addend(rela, val);
248}
249
250struct func_info {
251 uint64_t addr;
252 uint64_t size;
253};
254
255/* List of functions created by: nm -S vmlinux */
256static struct func_info *function_list;
257static int function_list_size;
258
259/* Allocate functions in 1k blocks */
260#define FUNC_BLK_SIZE 1024
261#define FUNC_BLK_MASK (FUNC_BLK_SIZE - 1)
262
263static int add_field(uint64_t addr, uint64_t size)
264{
265 struct func_info *fi;
266 int fsize = function_list_size;
267
268 if (!(fsize & FUNC_BLK_MASK)) {
269 fsize += FUNC_BLK_SIZE;
270 fi = realloc(function_list, fsize * sizeof(struct func_info));
271 if (!fi)
272 return -1;
273 function_list = fi;
274 }
275 fi = &function_list[function_list_size++];
276 fi->addr = addr;
277 fi->size = size;
278 return 0;
279}
280
281/* Used for when mcount/fentry is before the function entry */
282static int before_func;
283
284/* Only return match if the address lies inside the function size */
285static int cmp_func_addr(const void *K, const void *A)
286{
287 uint64_t key = *(const uint64_t *)K;
288 const struct func_info *a = A;
289
290 if (key + before_func < a->addr)
291 return -1;
292 return key >= a->addr + a->size;
293}
294
295/* Find the function in function list that is bounded by the function size */
296static int find_func(uint64_t key)
297{
298 return bsearch(&key, function_list, function_list_size,
299 sizeof(struct func_info), cmp_func_addr) != NULL;
300}
301
302static int cmp_funcs(const void *A, const void *B)
303{
304 const struct func_info *a = A;
305 const struct func_info *b = B;
306
307 if (a->addr < b->addr)
308 return -1;
309 return a->addr > b->addr;
310}
311
312static int parse_symbols(const char *fname)
313{
314 FILE *fp;
315 char addr_str[20]; /* Only need 17, but round up to next int size */
316 char size_str[20];
317 char type;
318
319 fp = fopen(fname, "r");
320 if (!fp) {
321 perror(fname);
322 return -1;
323 }
324
325 while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) {
326 uint64_t addr;
327 uint64_t size;
328
329 /* Only care about functions */
330 if (type != 't' && type != 'T' && type != 'W')
331 continue;
332
333 addr = strtoull(addr_str, NULL, 16);
334 size = strtoull(size_str, NULL, 16);
335 if (add_field(addr, size) < 0)
336 return -1;
337 }
338 fclose(fp);
339
340 qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs);
341
342 return 0;
343}
344
345static pthread_t mcount_sort_thread;
346static bool sort_reloc;
347
348static long rela_type;
349
350static char m_err[ERRSTR_MAXSZ];
351
352struct elf_mcount_loc {
353 Elf_Ehdr *ehdr;
354 Elf_Shdr *init_data_sec;
355 uint64_t start_mcount_loc;
356 uint64_t stop_mcount_loc;
357};
358
359/* Fill the array with the content of the relocs */
360static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc)
361{
362 Elf_Shdr *shdr_start;
363 Elf_Rela *rel;
364 unsigned int shnum;
365 unsigned int count = 0;
366 int shentsize;
367 void *array_end = ptr + size;
368
369 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
370 shentsize = ehdr_shentsize(ehdr);
371
372 shnum = ehdr_shnum(ehdr);
373 if (shnum == SHN_UNDEF)
374 shnum = shdr_size(shdr_start);
375
376 for (int i = 0; i < shnum; i++) {
377 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
378 void *end;
379
380 if (shdr_type(shdr) != SHT_RELA)
381 continue;
382
383 rel = (void *)ehdr + shdr_offset(shdr);
384 end = (void *)rel + shdr_size(shdr);
385
386 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
387 uint64_t offset = rela_offset(rel);
388
389 if (offset >= start_loc && offset < start_loc + size) {
390 if (ptr + long_size > array_end) {
391 snprintf(m_err, ERRSTR_MAXSZ,
392 "Too many relocations");
393 return -1;
394 }
395
396 /* Make sure this has the correct type */
397 if (rela_info(rel) != rela_type) {
398 snprintf(m_err, ERRSTR_MAXSZ,
399 "rela has type %lx but expected %lx\n",
400 (long)rela_info(rel), rela_type);
401 return -1;
402 }
403
404 if (long_size == 4)
405 *(uint32_t *)ptr = rela_addend(rel);
406 else
407 *(uint64_t *)ptr = rela_addend(rel);
408 ptr += long_size;
409 count++;
410 }
411 }
412 }
413 return count;
414}
415
416/* Put the sorted vals back into the relocation elements */
417static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc)
418{
419 Elf_Shdr *shdr_start;
420 Elf_Rela *rel;
421 unsigned int shnum;
422 int shentsize;
423
424 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
425 shentsize = ehdr_shentsize(ehdr);
426
427 shnum = ehdr_shnum(ehdr);
428 if (shnum == SHN_UNDEF)
429 shnum = shdr_size(shdr_start);
430
431 for (int i = 0; i < shnum; i++) {
432 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
433 void *end;
434
435 if (shdr_type(shdr) != SHT_RELA)
436 continue;
437
438 rel = (void *)ehdr + shdr_offset(shdr);
439 end = (void *)rel + shdr_size(shdr);
440
441 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
442 uint64_t offset = rela_offset(rel);
443
444 if (offset >= start_loc && offset < start_loc + size) {
445 if (long_size == 4)
446 rela_write_addend(rel, *(uint32_t *)ptr);
447 else
448 rela_write_addend(rel, *(uint64_t *)ptr);
449 ptr += long_size;
450 }
451 }
452 }
453}
454
455static int fill_addrs(void *ptr, uint64_t size, void *addrs)
456{
457 void *end = ptr + size;
458 int count = 0;
459
460 for (; ptr < end; ptr += long_size, addrs += long_size, count++) {
461 if (long_size == 4)
462 *(uint32_t *)ptr = elf_parser.r(addrs);
463 else
464 *(uint64_t *)ptr = elf_parser.r8(addrs);
465 }
466 return count;
467}
468
469static void replace_addrs(void *ptr, uint64_t size, void *addrs)
470{
471 void *end = ptr + size;
472
473 for (; ptr < end; ptr += long_size, addrs += long_size) {
474 if (long_size == 4)
475 elf_parser.w(*(uint32_t *)ptr, addrs);
476 else
477 elf_parser.w8(*(uint64_t *)ptr, addrs);
478 }
479}
480
481/* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
482static void *sort_mcount_loc(void *arg)
483{
484 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg;
485 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec)
486 + shdr_offset(emloc->init_data_sec);
487 uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc;
488 unsigned char *start_loc = (void *)emloc->ehdr + offset;
489 Elf_Ehdr *ehdr = emloc->ehdr;
490 void *e_msg = NULL;
491 void *vals;
492 int count;
493
494 vals = malloc(long_size * size);
495 if (!vals) {
496 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array");
497 pthread_exit(m_err);
498 }
499
500 if (sort_reloc) {
501 count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc);
502 /* gcc may use relocs to save the addresses, but clang does not. */
503 if (!count) {
504 count = fill_addrs(vals, size, start_loc);
505 sort_reloc = 0;
506 }
507 } else
508 count = fill_addrs(vals, size, start_loc);
509
510 if (count < 0) {
511 e_msg = m_err;
512 goto out;
513 }
514
515 if (count != size / long_size) {
516 snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n",
517 (int)(size / long_size), count);
518 e_msg = m_err;
519 goto out;
520 }
521
522 /* zero out any locations not found by function list */
523 if (function_list_size) {
524 for (void *ptr = vals; ptr < vals + size; ptr += long_size) {
525 uint64_t key;
526
527 key = long_size == 4 ? *(uint32_t *)ptr : *(uint64_t *)ptr;
528 if (!find_func(key)) {
529 if (long_size == 4)
530 *(uint32_t *)ptr = 0;
531 else
532 *(uint64_t *)ptr = 0;
533 }
534 }
535 }
536
537 compare_values = long_size == 4 ? compare_values_32 : compare_values_64;
538
539 qsort(vals, count, long_size, compare_values);
540
541 if (sort_reloc)
542 replace_relocs(vals, size, ehdr, emloc->start_mcount_loc);
543 else
544 replace_addrs(vals, size, start_loc);
545
546out:
547 free(vals);
548
549 pthread_exit(e_msg);
550}
551
552/* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */
553static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec,
554 const char *strtab)
555{
556 Elf_Sym *sym, *end_sym;
557 int symentsize = shdr_entsize(symtab_sec);
558 int found = 0;
559
560 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec);
561 end_sym = (void *)sym + shdr_size(symtab_sec);
562
563 while (sym < end_sym) {
564 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) {
565 emloc->start_mcount_loc = sym_value(sym);
566 if (++found == 2)
567 break;
568 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) {
569 emloc->stop_mcount_loc = sym_value(sym);
570 if (++found == 2)
571 break;
572 }
573 sym = (void *)sym + symentsize;
574 }
575
576 if (!emloc->start_mcount_loc) {
577 fprintf(stderr, "get start_mcount_loc error!");
578 return;
579 }
580
581 if (!emloc->stop_mcount_loc) {
582 fprintf(stderr, "get stop_mcount_loc error!");
583 return;
584 }
585}
586#else /* MCOUNT_SORT_ENABLED */
587static inline int parse_symbols(const char *fname) { return 0; }
588#endif
589
590static int do_sort(Elf_Ehdr *ehdr,
591 char const *const fname,
592 table_sort_t custom_sort)
593{
594 int rc = -1;
595 Elf_Shdr *shdr_start;
596 Elf_Shdr *strtab_sec = NULL;
597 Elf_Shdr *symtab_sec = NULL;
598 Elf_Shdr *extab_sec = NULL;
599 Elf_Shdr *string_sec;
600 Elf_Sym *sym;
601 const Elf_Sym *symtab;
602 Elf32_Word *symtab_shndx = NULL;
603 Elf_Sym *sort_needed_sym = NULL;
604 Elf_Shdr *sort_needed_sec;
605 uint32_t *sort_needed_loc;
606 void *sym_start;
607 void *sym_end;
608 const char *secstrings;
609 const char *strtab;
610 char *extab_image;
611 int sort_need_index;
612 int symentsize;
613 int shentsize;
614 int idx;
615 int i;
616 unsigned int shnum;
617 unsigned int shstrndx;
618#ifdef MCOUNT_SORT_ENABLED
619 struct elf_mcount_loc mstruct = {0};
620#endif
621#ifdef UNWINDER_ORC_ENABLED
622 unsigned int orc_ip_size = 0;
623 unsigned int orc_size = 0;
624 unsigned int orc_num_entries = 0;
625#endif
626
627 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
628 shentsize = ehdr_shentsize(ehdr);
629
630 shstrndx = ehdr_shstrndx(ehdr);
631 if (shstrndx == SHN_XINDEX)
632 shstrndx = shdr_link(shdr_start);
633 string_sec = get_index(shdr_start, shentsize, shstrndx);
634 secstrings = (const char *)ehdr + shdr_offset(string_sec);
635
636 shnum = ehdr_shnum(ehdr);
637 if (shnum == SHN_UNDEF)
638 shnum = shdr_size(shdr_start);
639
640 for (i = 0; i < shnum; i++) {
641 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
642
643 idx = shdr_name(shdr);
644 if (!strcmp(secstrings + idx, "__ex_table"))
645 extab_sec = shdr;
646 if (!strcmp(secstrings + idx, ".symtab"))
647 symtab_sec = shdr;
648 if (!strcmp(secstrings + idx, ".strtab"))
649 strtab_sec = shdr;
650
651 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX)
652 symtab_shndx = (Elf32_Word *)((const char *)ehdr +
653 shdr_offset(shdr));
654
655#ifdef MCOUNT_SORT_ENABLED
656 /* locate the .init.data section in vmlinux */
657 if (!strcmp(secstrings + idx, ".init.data"))
658 mstruct.init_data_sec = shdr;
659#endif
660
661#ifdef UNWINDER_ORC_ENABLED
662 /* locate the ORC unwind tables */
663 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
664 orc_ip_size = shdr_size(shdr);
665 g_orc_ip_table = (int *)((void *)ehdr +
666 shdr_offset(shdr));
667 }
668 if (!strcmp(secstrings + idx, ".orc_unwind")) {
669 orc_size = shdr_size(shdr);
670 g_orc_table = (struct orc_entry *)((void *)ehdr +
671 shdr_offset(shdr));
672 }
673#endif
674 } /* for loop */
675
676#ifdef UNWINDER_ORC_ENABLED
677 if (!g_orc_ip_table || !g_orc_table) {
678 fprintf(stderr,
679 "incomplete ORC unwind tables in file: %s\n", fname);
680 goto out;
681 }
682
683 orc_num_entries = orc_ip_size / sizeof(int);
684 if (orc_ip_size % sizeof(int) != 0 ||
685 orc_size % sizeof(struct orc_entry) != 0 ||
686 orc_num_entries != orc_size / sizeof(struct orc_entry)) {
687 fprintf(stderr,
688 "inconsistent ORC unwind table entries in file: %s\n",
689 fname);
690 goto out;
691 }
692
693 /* create thread to sort ORC unwind tables concurrently */
694 if (pthread_create(&orc_sort_thread, NULL,
695 sort_orctable, &orc_ip_size)) {
696 fprintf(stderr,
697 "pthread_create orc_sort_thread failed '%s': %s\n",
698 strerror(errno), fname);
699 goto out;
700 }
701#endif
702 if (!extab_sec) {
703 fprintf(stderr, "no __ex_table in file: %s\n", fname);
704 goto out;
705 }
706
707 if (!symtab_sec) {
708 fprintf(stderr, "no .symtab in file: %s\n", fname);
709 goto out;
710 }
711
712 if (!strtab_sec) {
713 fprintf(stderr, "no .strtab in file: %s\n", fname);
714 goto out;
715 }
716
717 extab_image = (void *)ehdr + shdr_offset(extab_sec);
718 strtab = (const char *)ehdr + shdr_offset(strtab_sec);
719 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec));
720
721#ifdef MCOUNT_SORT_ENABLED
722 mstruct.ehdr = ehdr;
723 get_mcount_loc(&mstruct, symtab_sec, strtab);
724
725 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) {
726 fprintf(stderr,
727 "incomplete mcount's sort in file: %s\n",
728 fname);
729 goto out;
730 }
731
732 /* create thread to sort mcount_loc concurrently */
733 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) {
734 fprintf(stderr,
735 "pthread_create mcount_sort_thread failed '%s': %s\n",
736 strerror(errno), fname);
737 goto out;
738 }
739#endif
740
741 if (custom_sort) {
742 custom_sort(extab_image, shdr_size(extab_sec));
743 } else {
744 int num_entries = shdr_size(extab_sec) / extable_ent_size;
745 qsort(extab_image, num_entries,
746 extable_ent_size, compare_extable);
747 }
748
749 /* find the flag main_extable_sort_needed */
750 sym_start = (void *)ehdr + shdr_offset(symtab_sec);
751 sym_end = sym_start + shdr_size(symtab_sec);
752 symentsize = shdr_entsize(symtab_sec);
753
754 for (sym = sym_start; (void *)sym + symentsize < sym_end;
755 sym = (void *)sym + symentsize) {
756 if (sym_type(sym) != STT_OBJECT)
757 continue;
758 if (!strcmp(strtab + sym_name(sym),
759 "main_extable_sort_needed")) {
760 sort_needed_sym = sym;
761 break;
762 }
763 }
764
765 if (!sort_needed_sym) {
766 fprintf(stderr,
767 "no main_extable_sort_needed symbol in file: %s\n",
768 fname);
769 goto out;
770 }
771
772 sort_need_index = get_secindex(sym_shndx(sym),
773 ((void *)sort_needed_sym - (void *)symtab) / symentsize,
774 symtab_shndx);
775 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index);
776 sort_needed_loc = (void *)ehdr +
777 shdr_offset(sort_needed_sec) +
778 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec);
779
780 /* extable has been sorted, clear the flag */
781 elf_parser.w(0, sort_needed_loc);
782 rc = 0;
783
784out:
785#ifdef UNWINDER_ORC_ENABLED
786 if (orc_sort_thread) {
787 void *retval = NULL;
788 /* wait for ORC tables sort done */
789 rc = pthread_join(orc_sort_thread, &retval);
790 if (rc) {
791 fprintf(stderr,
792 "pthread_join failed '%s': %s\n",
793 strerror(errno), fname);
794 } else if (retval) {
795 rc = -1;
796 fprintf(stderr,
797 "failed to sort ORC tables '%s': %s\n",
798 (char *)retval, fname);
799 }
800 }
801#endif
802
803#ifdef MCOUNT_SORT_ENABLED
804 if (mcount_sort_thread) {
805 void *retval = NULL;
806 /* wait for mcount sort done */
807 rc = pthread_join(mcount_sort_thread, &retval);
808 if (rc) {
809 fprintf(stderr,
810 "pthread_join failed '%s': %s\n",
811 strerror(errno), fname);
812 } else if (retval) {
813 rc = -1;
814 fprintf(stderr,
815 "failed to sort mcount '%s': %s\n",
816 (char *)retval, fname);
817 }
818 }
819#endif
820 return rc;
821}
822
823static int compare_relative_table(const void *a, const void *b)
824{
825 int32_t av = (int32_t)elf_parser.r(a);
826 int32_t bv = (int32_t)elf_parser.r(b);
827
828 if (av < bv)
829 return -1;
830 if (av > bv)
831 return 1;
832 return 0;
833}
834
835static void sort_relative_table(char *extab_image, int image_size)
836{
837 int i = 0;
838
839 /*
840 * Do the same thing the runtime sort does, first normalize to
841 * being relative to the start of the section.
842 */
843 while (i < image_size) {
844 uint32_t *loc = (uint32_t *)(extab_image + i);
845 elf_parser.w(elf_parser.r(loc) + i, loc);
846 i += 4;
847 }
848
849 qsort(extab_image, image_size / 8, 8, compare_relative_table);
850
851 /* Now denormalize. */
852 i = 0;
853 while (i < image_size) {
854 uint32_t *loc = (uint32_t *)(extab_image + i);
855 elf_parser.w(elf_parser.r(loc) - i, loc);
856 i += 4;
857 }
858}
859
860static void sort_relative_table_with_data(char *extab_image, int image_size)
861{
862 int i = 0;
863
864 while (i < image_size) {
865 uint32_t *loc = (uint32_t *)(extab_image + i);
866
867 elf_parser.w(elf_parser.r(loc) + i, loc);
868 elf_parser.w(elf_parser.r(loc + 1) + i + 4, loc + 1);
869 /* Don't touch the fixup type or data */
870
871 i += sizeof(uint32_t) * 3;
872 }
873
874 qsort(extab_image, image_size / 12, 12, compare_relative_table);
875
876 i = 0;
877 while (i < image_size) {
878 uint32_t *loc = (uint32_t *)(extab_image + i);
879
880 elf_parser.w(elf_parser.r(loc) - i, loc);
881 elf_parser.w(elf_parser.r(loc + 1) - (i + 4), loc + 1);
882 /* Don't touch the fixup type or data */
883
884 i += sizeof(uint32_t) * 3;
885 }
886}
887
888static int do_file(char const *const fname, void *addr)
889{
890 Elf_Ehdr *ehdr = addr;
891 table_sort_t custom_sort = NULL;
892
893 switch (elf_map_machine(ehdr)) {
894#ifdef MCOUNT_SORT_ENABLED
895 case EM_AARCH64:
896 /* arm64 also needs RELA-based weak-function fixups. */
897 sort_reloc = true;
898 rela_type = 0x403;
899 /* fallthrough */
900 case EM_RISCV:
901 /* arm64 and RISC-V place patchable entries before the function. */
902 before_func = 8;
903#else
904 case EM_AARCH64:
905 case EM_RISCV:
906#endif
907 /* fallthrough */
908 case EM_386:
909 case EM_LOONGARCH:
910 case EM_S390:
911 case EM_X86_64:
912 custom_sort = sort_relative_table_with_data;
913 break;
914 case EM_PARISC:
915 case EM_PPC:
916 case EM_PPC64:
917 custom_sort = sort_relative_table;
918 break;
919 case EM_ARCOMPACT:
920 case EM_ARCV2:
921 case EM_ARM:
922 case EM_MICROBLAZE:
923 case EM_MIPS:
924 case EM_XTENSA:
925 break;
926 default:
927 fprintf(stderr, "unrecognized e_machine %d %s\n",
928 elf_parser.r2(&ehdr->e32.e_machine), fname);
929 return -1;
930 }
931
932 switch (elf_map_long_size(addr)) {
933 case 4:
934 compare_extable = compare_extable_32,
935 long_size = 4;
936 extable_ent_size = 8;
937
938 if (elf_parser.r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) ||
939 elf_parser.r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) {
940 fprintf(stderr,
941 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname);
942 return -1;
943 }
944
945 break;
946 case 8:
947 compare_extable = compare_extable_64,
948 long_size = 8;
949 extable_ent_size = 16;
950
951 if (elf_parser.r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) ||
952 elf_parser.r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) {
953 fprintf(stderr,
954 "unrecognized ET_EXEC/ET_DYN file: %s\n",
955 fname);
956 return -1;
957 }
958
959 break;
960 default:
961 fprintf(stderr, "unrecognized ELF class %d %s\n",
962 ehdr->e32.e_ident[EI_CLASS], fname);
963 return -1;
964 }
965
966 return do_sort(ehdr, fname, custom_sort);
967}
968
969int main(int argc, char *argv[])
970{
971 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */
972 size_t size = 0;
973 void *addr = NULL;
974 int c;
975
976 while ((c = getopt(argc, argv, "s:")) >= 0) {
977 switch (c) {
978 case 's':
979 if (parse_symbols(optarg) < 0) {
980 fprintf(stderr, "Could not parse %s\n", optarg);
981 return -1;
982 }
983 break;
984 default:
985 fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n");
986 return 0;
987 }
988 }
989
990 if ((argc - optind) < 1) {
991 fprintf(stderr, "usage: sorttable vmlinux...\n");
992 return 0;
993 }
994
995 /* Process each file in turn, allowing deep failure. */
996 for (i = optind; i < argc; i++) {
997 addr = elf_map(argv[i], &size, (1 << ET_EXEC) | (1 << ET_DYN));
998 if (!addr) {
999 ++n_error;
1000 continue;
1001 }
1002
1003 if (do_file(argv[i], addr))
1004 ++n_error;
1005
1006 elf_unmap(addr, size);
1007 }
1008
1009 return !!n_error;
1010}