projects/linux·blob master · refs

519 lines · 13.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AArch64 loadable module support.
4 *
5 * Copyright (C) 2012 ARM Limited
6 *
7 * Author: Will Deacon <[email protected]>
8 */
9
10#define pr_fmt(fmt) "Modules: " fmt
11
12#include <linux/bitops.h>
13#include <linux/elf.h>
14#include <linux/ftrace.h>
15#include <linux/kasan.h>
16#include <linux/kernel.h>
17#include <linux/mm.h>
18#include <linux/moduleloader.h>
19#include <linux/random.h>
20#include <linux/scs.h>
21
22#include <asm/alternative.h>
23#include <asm/insn.h>
24#include <asm/scs.h>
25#include <asm/sections.h>
26#include <asm/text-patching.h>
27
28enum aarch64_reloc_op {
29 RELOC_OP_NONE,
30 RELOC_OP_ABS,
31 RELOC_OP_PREL,
32 RELOC_OP_PAGE,
33};
34
35static u64 do_reloc(enum aarch64_reloc_op reloc_op, __le32 *place, u64 val)
36{
37 switch (reloc_op) {
38 case RELOC_OP_ABS:
39 return val;
40 case RELOC_OP_PREL:
41 return val - (u64)place;
42 case RELOC_OP_PAGE:
43 return (val & ~0xfff) - ((u64)place & ~0xfff);
44 case RELOC_OP_NONE:
45 return 0;
46 }
47
48 pr_err("do_reloc: unknown relocation operation %d\n", reloc_op);
49 return 0;
50}
51
52#define WRITE_PLACE(place, val, mod) do { \
53 __typeof__(val) __val = (val); \
54 \
55 if (mod->state == MODULE_STATE_UNFORMED) \
56 *(place) = __val; \
57 else \
58 aarch64_insn_copy(place, &(__val), sizeof(*place)); \
59} while (0)
60
61static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len,
62 struct module *me)
63{
64 s64 sval = do_reloc(op, place, val);
65
66 /*
67 * The ELF psABI for AArch64 documents the 16-bit and 32-bit place
68 * relative and absolute relocations as having a range of [-2^15, 2^16)
69 * or [-2^31, 2^32), respectively. However, in order to be able to
70 * detect overflows reliably, we have to choose whether we interpret
71 * such quantities as signed or as unsigned, and stick with it.
72 * The way we organize our address space requires a signed
73 * interpretation of 32-bit relative references, so let's use that
74 * for all R_AARCH64_PRELxx relocations. This means our upper
75 * bound for overflow detection should be Sxx_MAX rather than Uxx_MAX.
76 */
77
78 switch (len) {
79 case 16:
80 WRITE_PLACE((s16 *)place, sval, me);
81 switch (op) {
82 case RELOC_OP_ABS:
83 if (sval < 0 || sval > U16_MAX)
84 return -ERANGE;
85 break;
86 case RELOC_OP_PREL:
87 if (sval < S16_MIN || sval > S16_MAX)
88 return -ERANGE;
89 break;
90 default:
91 pr_err("Invalid 16-bit data relocation (%d)\n", op);
92 return 0;
93 }
94 break;
95 case 32:
96 WRITE_PLACE((s32 *)place, sval, me);
97 switch (op) {
98 case RELOC_OP_ABS:
99 if (sval < 0 || sval > U32_MAX)
100 return -ERANGE;
101 break;
102 case RELOC_OP_PREL:
103 if (sval < S32_MIN || sval > S32_MAX)
104 return -ERANGE;
105 break;
106 default:
107 pr_err("Invalid 32-bit data relocation (%d)\n", op);
108 return 0;
109 }
110 break;
111 case 64:
112 WRITE_PLACE((s64 *)place, sval, me);
113 break;
114 default:
115 pr_err("Invalid length (%d) for data relocation\n", len);
116 return 0;
117 }
118 return 0;
119}
120
121enum aarch64_insn_movw_imm_type {
122 AARCH64_INSN_IMM_MOVNZ,
123 AARCH64_INSN_IMM_MOVKZ,
124};
125
126static int reloc_insn_movw(enum aarch64_reloc_op op, __le32 *place, u64 val,
127 int lsb, enum aarch64_insn_movw_imm_type imm_type,
128 struct module *me)
129{
130 u64 imm;
131 s64 sval;
132 u32 insn = le32_to_cpu(*place);
133
134 sval = do_reloc(op, place, val);
135 imm = sval >> lsb;
136
137 if (imm_type == AARCH64_INSN_IMM_MOVNZ) {
138 /*
139 * For signed MOVW relocations, we have to manipulate the
140 * instruction encoding depending on whether or not the
141 * immediate is less than zero.
142 */
143 insn &= ~(3 << 29);
144 if (sval >= 0) {
145 /* >=0: Set the instruction to MOVZ (opcode 10b). */
146 insn |= 2 << 29;
147 } else {
148 /*
149 * <0: Set the instruction to MOVN (opcode 00b).
150 * Since we've masked the opcode already, we
151 * don't need to do anything other than
152 * inverting the new immediate field.
153 */
154 imm = ~imm;
155 }
156 }
157
158 /* Update the instruction with the new encoding. */
159 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
160 WRITE_PLACE(place, cpu_to_le32(insn), me);
161
162 if (imm > U16_MAX)
163 return -ERANGE;
164
165 return 0;
166}
167
168static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val,
169 int lsb, int len, enum aarch64_insn_imm_type imm_type,
170 struct module *me)
171{
172 u64 imm, imm_mask;
173 s64 sval;
174 u32 insn = le32_to_cpu(*place);
175
176 /* Calculate the relocation value. */
177 sval = do_reloc(op, place, val);
178 sval >>= lsb;
179
180 /* Extract the value bits and shift them to bit 0. */
181 imm_mask = (BIT(lsb + len) - 1) >> lsb;
182 imm = sval & imm_mask;
183
184 /* Update the instruction's immediate field. */
185 insn = aarch64_insn_encode_immediate(imm_type, insn, imm);
186 WRITE_PLACE(place, cpu_to_le32(insn), me);
187
188 /*
189 * Extract the upper value bits (including the sign bit) and
190 * shift them to bit 0.
191 */
192 sval = (s64)(sval & ~(imm_mask >> 1)) >> (len - 1);
193
194 /*
195 * Overflow has occurred if the upper bits are not all equal to
196 * the sign bit of the value.
197 */
198 if ((u64)(sval + 1) >= 2)
199 return -ERANGE;
200
201 return 0;
202}
203
204static int reloc_insn_adrp(struct module *mod, Elf64_Shdr *sechdrs,
205 __le32 *place, u64 val, struct module *me)
206{
207 u32 insn;
208
209 if (!is_forbidden_offset_for_adrp(place))
210 return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21,
211 AARCH64_INSN_IMM_ADR, me);
212
213 /* patch ADRP to ADR if it is in range */
214 if (!reloc_insn_imm(RELOC_OP_PREL, place, val & ~0xfff, 0, 21,
215 AARCH64_INSN_IMM_ADR, me)) {
216 insn = le32_to_cpu(*place);
217 insn &= ~BIT(31);
218 } else {
219 /* out of range for ADR -> emit a veneer */
220 val = module_emit_veneer_for_adrp(mod, sechdrs, place, val & ~0xfff);
221 if (!val)
222 return -ENOEXEC;
223 insn = aarch64_insn_gen_branch_imm((u64)place, val,
224 AARCH64_INSN_BRANCH_NOLINK);
225 }
226
227 WRITE_PLACE(place, cpu_to_le32(insn), me);
228 return 0;
229}
230
231int apply_relocate_add(Elf64_Shdr *sechdrs,
232 const char *strtab,
233 unsigned int symindex,
234 unsigned int relsec,
235 struct module *me)
236{
237 unsigned int i;
238 int ovf;
239 bool overflow_check;
240 Elf64_Sym *sym;
241 void *loc;
242 u64 val;
243 Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr;
244
245 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
246 /* loc corresponds to P in the AArch64 ELF document. */
247 loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
248 + rel[i].r_offset;
249
250 /* sym is the ELF symbol we're referring to. */
251 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
252 + ELF64_R_SYM(rel[i].r_info);
253
254 /* val corresponds to (S + A) in the AArch64 ELF document. */
255 val = sym->st_value + rel[i].r_addend;
256
257 /* Check for overflow by default. */
258 overflow_check = true;
259
260 /* Perform the static relocation. */
261 switch (ELF64_R_TYPE(rel[i].r_info)) {
262 /* Null relocations. */
263 case R_ARM_NONE:
264 case R_AARCH64_NONE:
265 ovf = 0;
266 break;
267
268 /* Data relocations. */
269 case R_AARCH64_ABS64:
270 overflow_check = false;
271 ovf = reloc_data(RELOC_OP_ABS, loc, val, 64, me);
272 break;
273 case R_AARCH64_ABS32:
274 ovf = reloc_data(RELOC_OP_ABS, loc, val, 32, me);
275 break;
276 case R_AARCH64_ABS16:
277 ovf = reloc_data(RELOC_OP_ABS, loc, val, 16, me);
278 break;
279 case R_AARCH64_PREL64:
280 overflow_check = false;
281 ovf = reloc_data(RELOC_OP_PREL, loc, val, 64, me);
282 break;
283 case R_AARCH64_PREL32:
284 ovf = reloc_data(RELOC_OP_PREL, loc, val, 32, me);
285 break;
286 case R_AARCH64_PREL16:
287 ovf = reloc_data(RELOC_OP_PREL, loc, val, 16, me);
288 break;
289
290 /* MOVW instruction relocations. */
291 case R_AARCH64_MOVW_UABS_G0_NC:
292 overflow_check = false;
293 fallthrough;
294 case R_AARCH64_MOVW_UABS_G0:
295 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
296 AARCH64_INSN_IMM_MOVKZ, me);
297 break;
298 case R_AARCH64_MOVW_UABS_G1_NC:
299 overflow_check = false;
300 fallthrough;
301 case R_AARCH64_MOVW_UABS_G1:
302 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
303 AARCH64_INSN_IMM_MOVKZ, me);
304 break;
305 case R_AARCH64_MOVW_UABS_G2_NC:
306 overflow_check = false;
307 fallthrough;
308 case R_AARCH64_MOVW_UABS_G2:
309 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
310 AARCH64_INSN_IMM_MOVKZ, me);
311 break;
312 case R_AARCH64_MOVW_UABS_G3:
313 /* We're using the top bits so we can't overflow. */
314 overflow_check = false;
315 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48,
316 AARCH64_INSN_IMM_MOVKZ, me);
317 break;
318 case R_AARCH64_MOVW_SABS_G0:
319 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
320 AARCH64_INSN_IMM_MOVNZ, me);
321 break;
322 case R_AARCH64_MOVW_SABS_G1:
323 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
324 AARCH64_INSN_IMM_MOVNZ, me);
325 break;
326 case R_AARCH64_MOVW_SABS_G2:
327 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
328 AARCH64_INSN_IMM_MOVNZ, me);
329 break;
330 case R_AARCH64_MOVW_PREL_G0_NC:
331 overflow_check = false;
332 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
333 AARCH64_INSN_IMM_MOVKZ, me);
334 break;
335 case R_AARCH64_MOVW_PREL_G0:
336 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
337 AARCH64_INSN_IMM_MOVNZ, me);
338 break;
339 case R_AARCH64_MOVW_PREL_G1_NC:
340 overflow_check = false;
341 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
342 AARCH64_INSN_IMM_MOVKZ, me);
343 break;
344 case R_AARCH64_MOVW_PREL_G1:
345 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
346 AARCH64_INSN_IMM_MOVNZ, me);
347 break;
348 case R_AARCH64_MOVW_PREL_G2_NC:
349 overflow_check = false;
350 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
351 AARCH64_INSN_IMM_MOVKZ, me);
352 break;
353 case R_AARCH64_MOVW_PREL_G2:
354 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
355 AARCH64_INSN_IMM_MOVNZ, me);
356 break;
357 case R_AARCH64_MOVW_PREL_G3:
358 /* We're using the top bits so we can't overflow. */
359 overflow_check = false;
360 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48,
361 AARCH64_INSN_IMM_MOVNZ, me);
362 break;
363
364 /* Immediate instruction relocations. */
365 case R_AARCH64_LD_PREL_LO19:
366 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
367 AARCH64_INSN_IMM_19, me);
368 break;
369 case R_AARCH64_ADR_PREL_LO21:
370 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21,
371 AARCH64_INSN_IMM_ADR, me);
372 break;
373 case R_AARCH64_ADR_PREL_PG_HI21_NC:
374 overflow_check = false;
375 fallthrough;
376 case R_AARCH64_ADR_PREL_PG_HI21:
377 ovf = reloc_insn_adrp(me, sechdrs, loc, val, me);
378 if (ovf && ovf != -ERANGE)
379 return ovf;
380 break;
381 case R_AARCH64_ADD_ABS_LO12_NC:
382 case R_AARCH64_LDST8_ABS_LO12_NC:
383 overflow_check = false;
384 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12,
385 AARCH64_INSN_IMM_12, me);
386 break;
387 case R_AARCH64_LDST16_ABS_LO12_NC:
388 overflow_check = false;
389 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11,
390 AARCH64_INSN_IMM_12, me);
391 break;
392 case R_AARCH64_LDST32_ABS_LO12_NC:
393 overflow_check = false;
394 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10,
395 AARCH64_INSN_IMM_12, me);
396 break;
397 case R_AARCH64_LDST64_ABS_LO12_NC:
398 overflow_check = false;
399 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9,
400 AARCH64_INSN_IMM_12, me);
401 break;
402 case R_AARCH64_LDST128_ABS_LO12_NC:
403 overflow_check = false;
404 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8,
405 AARCH64_INSN_IMM_12, me);
406 break;
407 case R_AARCH64_TSTBR14:
408 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14,
409 AARCH64_INSN_IMM_14, me);
410 break;
411 case R_AARCH64_CONDBR19:
412 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
413 AARCH64_INSN_IMM_19, me);
414 break;
415 case R_AARCH64_JUMP26:
416 case R_AARCH64_CALL26:
417 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26,
418 AARCH64_INSN_IMM_26, me);
419 if (ovf == -ERANGE) {
420 val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym);
421 if (!val)
422 return -ENOEXEC;
423 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2,
424 26, AARCH64_INSN_IMM_26, me);
425 }
426 break;
427
428 default:
429 pr_err("module %s: unsupported RELA relocation: %llu\n",
430 me->name, ELF64_R_TYPE(rel[i].r_info));
431 return -ENOEXEC;
432 }
433
434 if (overflow_check && ovf == -ERANGE)
435 goto overflow;
436
437 }
438
439 return 0;
440
441overflow:
442 pr_err("module %s: overflow in relocation type %d val %Lx\n",
443 me->name, (int)ELF64_R_TYPE(rel[i].r_info), val);
444 return -ENOEXEC;
445}
446
447static inline void __init_plt(struct plt_entry *plt, unsigned long addr)
448{
449 *plt = get_plt_entry(addr, plt);
450}
451
452static int module_init_ftrace_plt(const Elf_Ehdr *hdr,
453 const Elf_Shdr *sechdrs,
454 struct module *mod)
455{
456#if defined(CONFIG_DYNAMIC_FTRACE)
457 const Elf_Shdr *s;
458 struct plt_entry *plts;
459
460 s = find_section(hdr, sechdrs, ".text.ftrace_trampoline");
461 if (!s)
462 return -ENOEXEC;
463
464 plts = (void *)s->sh_addr;
465
466 __init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR);
467
468 mod->arch.ftrace_trampolines = plts;
469
470 s = find_section(hdr, sechdrs, ".init.text.ftrace_trampoline");
471 if (!s)
472 return -ENOEXEC;
473
474 plts = (void *)s->sh_addr;
475
476 __init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR);
477
478 mod->arch.init_ftrace_trampolines = plts;
479
480#endif
481 return 0;
482}
483
484int module_finalize(const Elf_Ehdr *hdr,
485 const Elf_Shdr *sechdrs,
486 struct module *me)
487{
488 const Elf_Shdr *s;
489 int ret;
490
491 s = find_section(hdr, sechdrs, ".altinstructions");
492 if (s) {
493 ret = apply_alternatives_module((void *)s->sh_addr, s->sh_size);
494 if (ret < 0) {
495 pr_err("module %s: error occurred when applying alternatives\n", me->name);
496 return ret;
497 }
498 }
499
500 if (scs_is_dynamic()) {
501 s = find_section(hdr, sechdrs, ".init.eh_frame");
502 if (s) {
503 /*
504 * Because we can reject modules that are malformed
505 * so SCS patching fails, skip dry run and try to patch
506 * it in place. If patching fails, the module would not
507 * be loaded anyway.
508 */
509 ret = __pi_scs_patch((void *)s->sh_addr, s->sh_size, true);
510 if (ret) {
511 pr_err("module %s: error occurred during dynamic SCS patching (%d)\n",
512 me->name, ret);
513 return -ENOEXEC;
514 }
515 }
516 }
517
518 return module_init_ftrace_plt(hdr, sechdrs, me);
519}