| 1 | |
| 2 | #define pr_fmt(fmt) "SMP alternatives: " fmt
|
| 3 |
|
| 4 | #include <linux/mmu_context.h>
|
| 5 | #include <linux/perf_event.h>
|
| 6 | #include <linux/vmalloc.h>
|
| 7 | #include <linux/memory.h>
|
| 8 | #include <linux/execmem.h>
|
| 9 | #include <linux/cleanup.h>
|
| 10 | #include <linux/kgdb.h>
|
| 11 | #include <linux/mmap_lock.h>
|
| 12 |
|
| 13 | #include <asm/text-patching.h>
|
| 14 | #include <asm/insn.h>
|
| 15 | #include <asm/insn-eval.h>
|
| 16 | #include <asm/ibt.h>
|
| 17 | #include <asm/set_memory.h>
|
| 18 | #include <asm/nmi.h>
|
| 19 |
|
| 20 | int __read_mostly alternatives_patched;
|
| 21 |
|
| 22 | EXPORT_SYMBOL_GPL(alternatives_patched);
|
| 23 |
|
| 24 | #define MAX_PATCH_LEN (255-1)
|
| 25 |
|
| 26 | #define DA_ALL (~0)
|
| 27 | #define DA_ALT 0x01
|
| 28 | #define DA_RET 0x02
|
| 29 | #define DA_RETPOLINE 0x04
|
| 30 | #define DA_ENDBR 0x08
|
| 31 | #define DA_SMP 0x10
|
| 32 |
|
| 33 | static unsigned int debug_alternative;
|
| 34 |
|
| 35 | static int __init debug_alt(char *str)
|
| 36 | {
|
| 37 | if (str && *str == '=')
|
| 38 | str++;
|
| 39 |
|
| 40 | if (!str || kstrtouint(str, 0, &debug_alternative))
|
| 41 | debug_alternative = DA_ALL;
|
| 42 |
|
| 43 | return 1;
|
| 44 | }
|
| 45 | __setup("debug-alternative", debug_alt);
|
| 46 |
|
| 47 | #define DPRINTK(type, fmt, args...) \
|
| 48 | do { \
|
| 49 | if (debug_alternative & DA_##type) \
|
| 50 | printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \
|
| 51 | } while (0)
|
| 52 |
|
| 53 | #define DUMP_BYTES(type, buf, len, fmt, args...) \
|
| 54 | do { \
|
| 55 | if (unlikely(debug_alternative & DA_##type)) { \
|
| 56 | int j; \
|
| 57 | \
|
| 58 | if (!(len)) \
|
| 59 | break; \
|
| 60 | \
|
| 61 | printk(KERN_DEBUG pr_fmt(fmt), ##args); \
|
| 62 | for (j = 0; j < (len) - 1; j++) \
|
| 63 | printk(KERN_CONT "%02hhx ", buf[j]); \
|
| 64 | printk(KERN_CONT "%02hhx\n", buf[j]); \
|
| 65 | } \
|
| 66 | } while (0)
|
| 67 |
|
| 68 | static const unsigned char x86nops[] =
|
| 69 | {
|
| 70 | BYTES_NOP1,
|
| 71 | BYTES_NOP2,
|
| 72 | BYTES_NOP3,
|
| 73 | BYTES_NOP4,
|
| 74 | BYTES_NOP5,
|
| 75 | BYTES_NOP6,
|
| 76 | BYTES_NOP7,
|
| 77 | BYTES_NOP8,
|
| 78 | #ifdef CONFIG_64BIT
|
| 79 | BYTES_NOP9,
|
| 80 | BYTES_NOP10,
|
| 81 | BYTES_NOP11,
|
| 82 | #endif
|
| 83 | };
|
| 84 |
|
| 85 | const unsigned char * const x86_nops[ASM_NOP_MAX+1] =
|
| 86 | {
|
| 87 | NULL,
|
| 88 | x86nops,
|
| 89 | x86nops + 1,
|
| 90 | x86nops + 1 + 2,
|
| 91 | x86nops + 1 + 2 + 3,
|
| 92 | x86nops + 1 + 2 + 3 + 4,
|
| 93 | x86nops + 1 + 2 + 3 + 4 + 5,
|
| 94 | x86nops + 1 + 2 + 3 + 4 + 5 + 6,
|
| 95 | x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
|
| 96 | #ifdef CONFIG_64BIT
|
| 97 | x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
|
| 98 | x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9,
|
| 99 | x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10,
|
| 100 | #endif
|
| 101 | };
|
| 102 |
|
| 103 | #ifdef CONFIG_FINEIBT
|
| 104 | static bool cfi_paranoid __ro_after_init;
|
| 105 | #endif
|
| 106 |
|
| 107 | #ifdef CONFIG_MITIGATION_ITS
|
| 108 |
|
| 109 | #ifdef CONFIG_MODULES
|
| 110 | static struct module *its_mod;
|
| 111 | #endif
|
| 112 | static void *its_page;
|
| 113 | static unsigned int its_offset;
|
| 114 | struct its_array its_pages;
|
| 115 |
|
| 116 | static void *__its_alloc(struct its_array *pages)
|
| 117 | {
|
| 118 | void *page __free(execmem) = execmem_alloc_rw(EXECMEM_MODULE_TEXT, PAGE_SIZE);
|
| 119 | if (!page)
|
| 120 | return NULL;
|
| 121 |
|
| 122 | void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *),
|
| 123 | GFP_KERNEL);
|
| 124 | if (!tmp)
|
| 125 | return NULL;
|
| 126 |
|
| 127 | pages->pages = tmp;
|
| 128 | pages->pages[pages->num++] = page;
|
| 129 |
|
| 130 | return no_free_ptr(page);
|
| 131 | }
|
| 132 |
|
| 133 |
|
| 134 | static void *its_init_thunk(void *thunk, int reg)
|
| 135 | {
|
| 136 | u8 *bytes = thunk;
|
| 137 | int offset = 0;
|
| 138 | int i = 0;
|
| 139 |
|
| 140 | #ifdef CONFIG_FINEIBT
|
| 141 | if (cfi_paranoid) {
|
| 142 | |
| 143 | * When ITS uses indirect branch thunk the fineibt_paranoid
|
| 144 | * caller sequence doesn't fit in the caller site. So put the
|
| 145 | * remaining part of the sequence (UDB + JNE) into the ITS
|
| 146 | * thunk.
|
| 147 |
|
| 148 | bytes[i++] = 0xd6;
|
| 149 | bytes[i++] = 0x75;
|
| 150 | bytes[i++] = 0xfd;
|
| 151 |
|
| 152 | offset = 1;
|
| 153 | }
|
| 154 | #endif
|
| 155 |
|
| 156 | if (reg >= 8) {
|
| 157 | bytes[i++] = 0x41;
|
| 158 | reg -= 8;
|
| 159 | }
|
| 160 | bytes[i++] = 0xff;
|
| 161 | bytes[i++] = 0xe0 + reg;
|
| 162 | bytes[i++] = 0xcc;
|
| 163 |
|
| 164 | return thunk + offset;
|
| 165 | }
|
| 166 |
|
| 167 | static void its_pages_protect(struct its_array *pages)
|
| 168 | {
|
| 169 | for (int i = 0; i < pages->num; i++) {
|
| 170 | void *page = pages->pages[i];
|
| 171 | execmem_restore_rox(page, PAGE_SIZE);
|
| 172 | }
|
| 173 | }
|
| 174 |
|
| 175 | static void its_fini_core(void)
|
| 176 | {
|
| 177 | if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
|
| 178 | its_pages_protect(&its_pages);
|
| 179 | kfree(its_pages.pages);
|
| 180 | }
|
| 181 |
|
| 182 | #ifdef CONFIG_MODULES
|
| 183 | void its_init_mod(struct module *mod)
|
| 184 | {
|
| 185 | if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
|
| 186 | return;
|
| 187 |
|
| 188 | mutex_lock(&text_mutex);
|
| 189 | its_mod = mod;
|
| 190 | its_page = NULL;
|
| 191 | }
|
| 192 |
|
| 193 | void its_fini_mod(struct module *mod)
|
| 194 | {
|
| 195 | if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
|
| 196 | return;
|
| 197 |
|
| 198 | WARN_ON_ONCE(its_mod != mod);
|
| 199 |
|
| 200 | its_mod = NULL;
|
| 201 | its_page = NULL;
|
| 202 | mutex_unlock(&text_mutex);
|
| 203 |
|
| 204 | if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
|
| 205 | its_pages_protect(&mod->arch.its_pages);
|
| 206 | }
|
| 207 |
|
| 208 | void its_free_mod(struct module *mod)
|
| 209 | {
|
| 210 | if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
|
| 211 | return;
|
| 212 |
|
| 213 | for (int i = 0; i < mod->arch.its_pages.num; i++) {
|
| 214 | void *page = mod->arch.its_pages.pages[i];
|
| 215 | execmem_free(page);
|
| 216 | }
|
| 217 | kfree(mod->arch.its_pages.pages);
|
| 218 | }
|
| 219 | #endif
|
| 220 |
|
| 221 | static void *its_alloc(void)
|
| 222 | {
|
| 223 | struct its_array *pages = &its_pages;
|
| 224 | void *page;
|
| 225 |
|
| 226 | #ifdef CONFIG_MODULES
|
| 227 | if (its_mod)
|
| 228 | pages = &its_mod->arch.its_pages;
|
| 229 | #endif
|
| 230 |
|
| 231 | page = __its_alloc(pages);
|
| 232 | if (!page)
|
| 233 | return NULL;
|
| 234 |
|
| 235 | if (pages == &its_pages)
|
| 236 | set_memory_x((unsigned long)page, 1);
|
| 237 |
|
| 238 | return page;
|
| 239 | }
|
| 240 |
|
| 241 | static void *its_allocate_thunk(int reg)
|
| 242 | {
|
| 243 | int size = 3 + (reg / 8);
|
| 244 | void *thunk;
|
| 245 |
|
| 246 | #ifdef CONFIG_FINEIBT
|
| 247 | |
| 248 | * The ITS thunk contains an indirect jump and an int3 instruction so
|
| 249 | * its size is 3 or 4 bytes depending on the register used. If CFI
|
| 250 | * paranoid is used then 3 extra bytes are added in the ITS thunk to
|
| 251 | * complete the fineibt_paranoid caller sequence.
|
| 252 |
|
| 253 | if (cfi_paranoid)
|
| 254 | size += 3;
|
| 255 | #endif
|
| 256 |
|
| 257 | if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {
|
| 258 | its_page = its_alloc();
|
| 259 | if (!its_page) {
|
| 260 | pr_err("ITS page allocation failed\n");
|
| 261 | return NULL;
|
| 262 | }
|
| 263 | memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE);
|
| 264 | its_offset = 32;
|
| 265 | }
|
| 266 |
|
| 267 | |
| 268 | * If the indirect branch instruction will be in the lower half
|
| 269 | * of a cacheline, then update the offset to reach the upper half.
|
| 270 |
|
| 271 | if ((its_offset + size - 1) % 64 < 32)
|
| 272 | its_offset = ((its_offset - 1) | 0x3F) + 33;
|
| 273 |
|
| 274 | thunk = its_page + its_offset;
|
| 275 | its_offset += size;
|
| 276 |
|
| 277 | return its_init_thunk(thunk, reg);
|
| 278 | }
|
| 279 |
|
| 280 | u8 *its_static_thunk(int reg)
|
| 281 | {
|
| 282 | u8 *thunk = __x86_indirect_its_thunk_array[reg];
|
| 283 |
|
| 284 | #ifdef CONFIG_FINEIBT
|
| 285 |
|
| 286 | if (cfi_paranoid)
|
| 287 | return thunk - 2;
|
| 288 | #endif
|
| 289 | return thunk;
|
| 290 | }
|
| 291 |
|
| 292 | #else
|
| 293 | static inline void its_fini_core(void) {}
|
| 294 | #endif
|
| 295 |
|
| 296 | |
| 297 | * Nomenclature for variable names to simplify and clarify this code and ease
|
| 298 | * any potential staring at it:
|
| 299 | *
|
| 300 | * @instr: source address of the original instructions in the kernel text as
|
| 301 | * generated by the compiler.
|
| 302 | *
|
| 303 | * @buf: temporary buffer on which the patching operates. This buffer is
|
| 304 | * eventually text-poked into the kernel image.
|
| 305 | *
|
| 306 | * @replacement/@repl: pointer to the opcodes which are replacing @instr, located
|
| 307 | * in the .altinstr_replacement section.
|
| 308 |
|
| 309 |
|
| 310 | |
| 311 | * Fill the buffer with a single effective instruction of size @len.
|
| 312 | *
|
| 313 | * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info)
|
| 314 | * for every single-byte NOP, try to generate the maximally available NOP of
|
| 315 | * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for
|
| 316 | * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and
|
| 317 | * *jump* over instead of executing long and daft NOPs.
|
| 318 |
|
| 319 | static void add_nop(u8 *buf, unsigned int len)
|
| 320 | {
|
| 321 | u8 *target = buf + len;
|
| 322 |
|
| 323 | if (!len)
|
| 324 | return;
|
| 325 |
|
| 326 | if (len <= ASM_NOP_MAX) {
|
| 327 | memcpy(buf, x86_nops[len], len);
|
| 328 | return;
|
| 329 | }
|
| 330 |
|
| 331 | if (len < 128) {
|
| 332 | __text_gen_insn(buf, JMP8_INSN_OPCODE, buf, target, JMP8_INSN_SIZE);
|
| 333 | buf += JMP8_INSN_SIZE;
|
| 334 | } else {
|
| 335 | __text_gen_insn(buf, JMP32_INSN_OPCODE, buf, target, JMP32_INSN_SIZE);
|
| 336 | buf += JMP32_INSN_SIZE;
|
| 337 | }
|
| 338 |
|
| 339 | for (;buf < target; buf++)
|
| 340 | *buf = INT3_INSN_OPCODE;
|
| 341 | }
|
| 342 |
|
| 343 | |
| 344 | * Find the offset of the first non-NOP instruction starting at @offset
|
| 345 | * but no further than @len.
|
| 346 |
|
| 347 | static int skip_nops(u8 *buf, int offset, int len)
|
| 348 | {
|
| 349 | struct insn insn;
|
| 350 |
|
| 351 | for (; offset < len; offset += insn.length) {
|
| 352 | if (insn_decode_kernel(&insn, &buf[offset]))
|
| 353 | break;
|
| 354 |
|
| 355 | if (!insn_is_nop(&insn))
|
| 356 | break;
|
| 357 | }
|
| 358 |
|
| 359 | return offset;
|
| 360 | }
|
| 361 |
|
| 362 | |
| 363 | * "noinline" to cause control flow change and thus invalidate I$ and
|
| 364 | * cause refetch after modification.
|
| 365 |
|
| 366 | static void noinline optimize_nops(const u8 * const instr, u8 *buf, size_t len)
|
| 367 | {
|
| 368 | for (int next, i = 0; i < len; i = next) {
|
| 369 | struct insn insn;
|
| 370 |
|
| 371 | if (insn_decode_kernel(&insn, &buf[i]))
|
| 372 | return;
|
| 373 |
|
| 374 | next = i + insn.length;
|
| 375 |
|
| 376 | if (insn_is_nop(&insn)) {
|
| 377 | int nop = i;
|
| 378 |
|
| 379 |
|
| 380 | if (i + insn.length == len)
|
| 381 | return;
|
| 382 |
|
| 383 | next = skip_nops(buf, next, len);
|
| 384 |
|
| 385 | add_nop(buf + nop, next - nop);
|
| 386 | DUMP_BYTES(ALT, buf, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, next);
|
| 387 | }
|
| 388 | }
|
| 389 | }
|
| 390 |
|
| 391 | |
| 392 | * In this context, "source" is where the instructions are placed in the
|
| 393 | * section .altinstr_replacement, for example during kernel build by the
|
| 394 | * toolchain.
|
| 395 | * "Destination" is where the instructions are being patched in by this
|
| 396 | * machinery.
|
| 397 | *
|
| 398 | * The source offset is:
|
| 399 | *
|
| 400 | * src_imm = target - src_next_ip (1)
|
| 401 | *
|
| 402 | * and the target offset is:
|
| 403 | *
|
| 404 | * dst_imm = target - dst_next_ip (2)
|
| 405 | *
|
| 406 | * so rework (1) as an expression for target like:
|
| 407 | *
|
| 408 | * target = src_imm + src_next_ip (1a)
|
| 409 | *
|
| 410 | * and substitute in (2) to get:
|
| 411 | *
|
| 412 | * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3)
|
| 413 | *
|
| 414 | * Now, since the instruction stream is 'identical' at src and dst (it
|
| 415 | * is being copied after all) it can be stated that:
|
| 416 | *
|
| 417 | * src_next_ip = src + ip_offset
|
| 418 | * dst_next_ip = dst + ip_offset (4)
|
| 419 | *
|
| 420 | * Substitute (4) in (3) and observe ip_offset being cancelled out to
|
| 421 | * obtain:
|
| 422 | *
|
| 423 | * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset)
|
| 424 | * = src_imm + src - dst + ip_offset - ip_offset
|
| 425 | * = src_imm + src - dst (5)
|
| 426 | *
|
| 427 | * IOW, only the relative displacement of the code block matters.
|
| 428 |
|
| 429 |
|
| 430 | #define apply_reloc_n(n_, p_, d_) \
|
| 431 | do { \
|
| 432 | s32 v = *(s##n_ *)(p_); \
|
| 433 | v += (d_); \
|
| 434 | BUG_ON((v >> 31) != (v >> (n_-1))); \
|
| 435 | *(s##n_ *)(p_) = (s##n_)v; \
|
| 436 | } while (0)
|
| 437 |
|
| 438 |
|
| 439 | static __always_inline
|
| 440 | void apply_reloc(int n, void *ptr, uintptr_t diff)
|
| 441 | {
|
| 442 | switch (n) {
|
| 443 | case 1: apply_reloc_n(8, ptr, diff); break;
|
| 444 | case 2: apply_reloc_n(16, ptr, diff); break;
|
| 445 | case 4: apply_reloc_n(32, ptr, diff); break;
|
| 446 | default: BUG();
|
| 447 | }
|
| 448 | }
|
| 449 |
|
| 450 | static __always_inline
|
| 451 | bool need_reloc(unsigned long offset, u8 *src, size_t src_len)
|
| 452 | {
|
| 453 | u8 *target = src + offset;
|
| 454 | |
| 455 | * If the target is inside the patched block, it's relative to the
|
| 456 | * block itself and does not need relocation.
|
| 457 |
|
| 458 | return (target < src || target > src + src_len);
|
| 459 | }
|
| 460 |
|
| 461 | static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len)
|
| 462 | {
|
| 463 | for (int next, i = 0; i < instrlen; i = next) {
|
| 464 | struct insn insn;
|
| 465 |
|
| 466 | if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i])))
|
| 467 | return;
|
| 468 |
|
| 469 | next = i + insn.length;
|
| 470 |
|
| 471 | switch (insn.opcode.bytes[0]) {
|
| 472 | case 0x0f:
|
| 473 | if (insn.opcode.bytes[1] < 0x80 ||
|
| 474 | insn.opcode.bytes[1] > 0x8f)
|
| 475 | break;
|
| 476 |
|
| 477 | fallthrough;
|
| 478 | case 0x70 ... 0x7f:
|
| 479 | case JMP8_INSN_OPCODE:
|
| 480 | case JMP32_INSN_OPCODE:
|
| 481 | case CALL_INSN_OPCODE:
|
| 482 | if (need_reloc(next + insn.immediate.value, repl, repl_len)) {
|
| 483 | apply_reloc(insn.immediate.nbytes,
|
| 484 | buf + i + insn_offset_immediate(&insn),
|
| 485 | repl - instr);
|
| 486 | }
|
| 487 |
|
| 488 | |
| 489 | * Where possible, convert JMP.d32 into JMP.d8.
|
| 490 |
|
| 491 | if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) {
|
| 492 | s32 imm = insn.immediate.value;
|
| 493 | imm += repl - instr;
|
| 494 | imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE;
|
| 495 | if ((imm >> 31) == (imm >> 7)) {
|
| 496 | buf[i+0] = JMP8_INSN_OPCODE;
|
| 497 | buf[i+1] = (s8)imm;
|
| 498 |
|
| 499 | memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2);
|
| 500 | }
|
| 501 | }
|
| 502 | break;
|
| 503 | }
|
| 504 |
|
| 505 | if (insn_rip_relative(&insn)) {
|
| 506 | if (need_reloc(next + insn.displacement.value, repl, repl_len)) {
|
| 507 | apply_reloc(insn.displacement.nbytes,
|
| 508 | buf + i + insn_offset_displacement(&insn),
|
| 509 | repl - instr);
|
| 510 | }
|
| 511 | }
|
| 512 | }
|
| 513 | }
|
| 514 |
|
| 515 | void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len)
|
| 516 | {
|
| 517 | __apply_relocation(buf, instr, instrlen, repl, repl_len);
|
| 518 | optimize_nops(instr, buf, instrlen);
|
| 519 | }
|
| 520 |
|
| 521 |
|
| 522 | DEFINE_ASM_FUNC(nop_func, "", .entry.text);
|
| 523 | EXPORT_SYMBOL_GPL(nop_func);
|
| 524 |
|
| 525 | noinstr void BUG_func(void)
|
| 526 | {
|
| 527 | BUG();
|
| 528 | }
|
| 529 | EXPORT_SYMBOL(BUG_func);
|
| 530 |
|
| 531 | #define CALL_RIP_REL_OPCODE 0xff
|
| 532 | #define CALL_RIP_REL_MODRM 0x15
|
| 533 |
|
| 534 | |
| 535 | * Rewrite the "call BUG_func" replacement to point to the target of the
|
| 536 | * indirect pv_ops call "call *disp(%ip)".
|
| 537 |
|
| 538 | static unsigned int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a)
|
| 539 | {
|
| 540 | void *target, *bug = &BUG_func;
|
| 541 | s32 disp;
|
| 542 |
|
| 543 | if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) {
|
| 544 | pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n");
|
| 545 | BUG();
|
| 546 | }
|
| 547 |
|
| 548 | if (a->instrlen != 6 ||
|
| 549 | instr[0] != CALL_RIP_REL_OPCODE ||
|
| 550 | instr[1] != CALL_RIP_REL_MODRM) {
|
| 551 | pr_err("ALT_FLAG_DIRECT_CALL set for unrecognized indirect call\n");
|
| 552 | BUG();
|
| 553 | }
|
| 554 |
|
| 555 |
|
| 556 | disp = *(s32 *)(instr + 2);
|
| 557 | #ifdef CONFIG_X86_64
|
| 558 |
|
| 559 |
|
| 560 | target = *(void **)(instr + a->instrlen + disp);
|
| 561 | #else
|
| 562 |
|
| 563 |
|
| 564 | target = *(void **)disp;
|
| 565 | #endif
|
| 566 | if (!target)
|
| 567 | target = bug;
|
| 568 |
|
| 569 |
|
| 570 | *(s32 *)(insn_buff + 1) += target - bug;
|
| 571 |
|
| 572 | if (target == &nop_func)
|
| 573 | return 0;
|
| 574 |
|
| 575 | return 5;
|
| 576 | }
|
| 577 |
|
| 578 | static inline u8 * instr_va(struct alt_instr *i)
|
| 579 | {
|
| 580 | return (u8 *)&i->instr_offset + i->instr_offset;
|
| 581 | }
|
| 582 |
|
| 583 | struct patch_site {
|
| 584 | u8 *instr;
|
| 585 | struct alt_instr *alt;
|
| 586 | u8 buff[MAX_PATCH_LEN];
|
| 587 | u8 len;
|
| 588 | };
|
| 589 |
|
| 590 | static struct alt_instr * __init_or_module analyze_patch_site(struct patch_site *ps,
|
| 591 | struct alt_instr *start,
|
| 592 | struct alt_instr *end)
|
| 593 | {
|
| 594 | struct alt_instr *alt = start;
|
| 595 |
|
| 596 | ps->instr = instr_va(start);
|
| 597 |
|
| 598 | |
| 599 | * In case of nested ALTERNATIVE()s the outer alternative might add
|
| 600 | * more padding. To ensure consistent patching find the max padding for
|
| 601 | * all alt_instr entries for this site (nested alternatives result in
|
| 602 | * consecutive entries).
|
| 603 | * Find the last alt_instr eligible for patching at the site.
|
| 604 |
|
| 605 | for (; alt < end && instr_va(alt) == ps->instr; alt++) {
|
| 606 | ps->len = max(ps->len, alt->instrlen);
|
| 607 |
|
| 608 | BUG_ON(alt->cpuid >= (NCAPINTS + NBUGINTS) * 32);
|
| 609 | |
| 610 | * Patch if either:
|
| 611 | * - feature is present
|
| 612 | * - feature not present but ALT_FLAG_NOT is set to mean,
|
| 613 | * patch if feature is *NOT* present.
|
| 614 |
|
| 615 | if (!boot_cpu_has(alt->cpuid) != !(alt->flags & ALT_FLAG_NOT))
|
| 616 | ps->alt = alt;
|
| 617 | }
|
| 618 |
|
| 619 | BUG_ON(ps->len > sizeof(ps->buff));
|
| 620 |
|
| 621 | return alt;
|
| 622 | }
|
| 623 |
|
| 624 | static void __init_or_module prep_patch_site(struct patch_site *ps)
|
| 625 | {
|
| 626 | struct alt_instr *alt = ps->alt;
|
| 627 | u8 buff_sz;
|
| 628 | u8 *repl;
|
| 629 |
|
| 630 | if (!alt) {
|
| 631 |
|
| 632 | memcpy(ps->buff, ps->instr, ps->len);
|
| 633 | return;
|
| 634 | }
|
| 635 |
|
| 636 | repl = (u8 *)&alt->repl_offset + alt->repl_offset;
|
| 637 | DPRINTK(ALT, "feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d) flags: 0x%x",
|
| 638 | alt->cpuid >> 5, alt->cpuid & 0x1f,
|
| 639 | ps->instr, ps->instr, ps->len,
|
| 640 | repl, alt->replacementlen, alt->flags);
|
| 641 |
|
| 642 | memcpy(ps->buff, repl, alt->replacementlen);
|
| 643 | buff_sz = alt->replacementlen;
|
| 644 |
|
| 645 | if (alt->flags & ALT_FLAG_DIRECT_CALL)
|
| 646 | buff_sz = alt_replace_call(ps->instr, ps->buff, alt);
|
| 647 |
|
| 648 | for (; buff_sz < ps->len; buff_sz++)
|
| 649 | ps->buff[buff_sz] = 0x90;
|
| 650 |
|
| 651 | __apply_relocation(ps->buff, ps->instr, ps->len, repl, alt->replacementlen);
|
| 652 |
|
| 653 | DUMP_BYTES(ALT, ps->instr, ps->len, "%px: old_insn: ", ps->instr);
|
| 654 | DUMP_BYTES(ALT, repl, alt->replacementlen, "%px: rpl_insn: ", repl);
|
| 655 | DUMP_BYTES(ALT, ps->buff, ps->len, "%px: final_insn: ", ps->instr);
|
| 656 | }
|
| 657 |
|
| 658 | static void __init_or_module patch_site(struct patch_site *ps)
|
| 659 | {
|
| 660 | optimize_nops(ps->instr, ps->buff, ps->len);
|
| 661 | text_poke_early(ps->instr, ps->buff, ps->len);
|
| 662 | }
|
| 663 |
|
| 664 | |
| 665 | * Replace instructions with better alternatives for this CPU type. This runs
|
| 666 | * before SMP is initialized to avoid SMP problems with self modifying code.
|
| 667 | * This implies that asymmetric systems where APs have less capabilities than
|
| 668 | * the boot processor are not handled. Tough. Make sure you disable such
|
| 669 | * features by hand.
|
| 670 | *
|
| 671 | * Marked "noinline" to cause control flow change and thus insn cache
|
| 672 | * to refetch changed I$ lines.
|
| 673 |
|
| 674 | void __init_or_module noinline apply_alternatives(struct alt_instr *start,
|
| 675 | struct alt_instr *end)
|
| 676 | {
|
| 677 | struct alt_instr *a;
|
| 678 |
|
| 679 | DPRINTK(ALT, "alt table %px, -> %px", start, end);
|
| 680 |
|
| 681 | |
| 682 | * KASAN_SHADOW_START is defined using
|
| 683 | * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here.
|
| 684 | * During the process, KASAN becomes confused seeing partial LA57
|
| 685 | * conversion and triggers a false-positive out-of-bound report.
|
| 686 | *
|
| 687 | * Disable KASAN until the patching is complete.
|
| 688 |
|
| 689 | kasan_disable_current();
|
| 690 |
|
| 691 | |
| 692 | * The scan order should be from start to end. A later scanned
|
| 693 | * alternative code can overwrite previously scanned alternative code.
|
| 694 | * Some kernel functions (e.g. memcpy, memset, etc) use this order to
|
| 695 | * patch code.
|
| 696 | *
|
| 697 | * So be careful if you want to change the scan order to any other
|
| 698 | * order.
|
| 699 |
|
| 700 | a = start;
|
| 701 | while (a < end) {
|
| 702 | struct patch_site ps = {
|
| 703 | .alt = NULL,
|
| 704 | .len = 0
|
| 705 | };
|
| 706 |
|
| 707 | a = analyze_patch_site(&ps, a, end);
|
| 708 | prep_patch_site(&ps);
|
| 709 | patch_site(&ps);
|
| 710 | }
|
| 711 |
|
| 712 | kasan_enable_current();
|
| 713 | }
|
| 714 |
|
| 715 | static inline bool is_jcc32(struct insn *insn)
|
| 716 | {
|
| 717 |
|
| 718 | return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;
|
| 719 | }
|
| 720 |
|
| 721 | #if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL)
|
| 722 |
|
| 723 | |
| 724 | * [CS]{,3} CALL/JMP *%\reg [INT3]*
|
| 725 |
|
| 726 | static int emit_indirect(int op, int reg, u8 *bytes, int len)
|
| 727 | {
|
| 728 | int cs = 0, bp = 0;
|
| 729 | int i = 0;
|
| 730 | u8 modrm;
|
| 731 |
|
| 732 | |
| 733 | * Set @len to the excess bytes after writing the instruction.
|
| 734 |
|
| 735 | len -= 2 + (reg >= 8);
|
| 736 | WARN_ON_ONCE(len < 0);
|
| 737 |
|
| 738 | switch (op) {
|
| 739 | case CALL_INSN_OPCODE:
|
| 740 | modrm = 0x10;
|
| 741 | |
| 742 | * Additional NOP is better than prefix decode penalty.
|
| 743 |
|
| 744 | if (len <= 3)
|
| 745 | cs = len;
|
| 746 | break;
|
| 747 |
|
| 748 | case JMP32_INSN_OPCODE:
|
| 749 | modrm = 0x20;
|
| 750 | bp = len;
|
| 751 | break;
|
| 752 |
|
| 753 | default:
|
| 754 | WARN_ON_ONCE(1);
|
| 755 | return -1;
|
| 756 | }
|
| 757 |
|
| 758 | while (cs--)
|
| 759 | bytes[i++] = 0x2e;
|
| 760 |
|
| 761 | if (reg >= 8) {
|
| 762 | bytes[i++] = 0x41;
|
| 763 | reg -= 8;
|
| 764 | }
|
| 765 |
|
| 766 | modrm |= 0xc0;
|
| 767 | modrm += reg;
|
| 768 |
|
| 769 | bytes[i++] = 0xff;
|
| 770 | bytes[i++] = modrm;
|
| 771 |
|
| 772 | while (bp--)
|
| 773 | bytes[i++] = 0xcc;
|
| 774 |
|
| 775 | return i;
|
| 776 | }
|
| 777 |
|
| 778 | static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,
|
| 779 | void *call_dest, void *jmp_dest)
|
| 780 | {
|
| 781 | u8 op = insn->opcode.bytes[0];
|
| 782 | int i = 0;
|
| 783 |
|
| 784 | |
| 785 | * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional
|
| 786 | * tail-calls. Deal with them.
|
| 787 |
|
| 788 | if (is_jcc32(insn)) {
|
| 789 | bytes[i++] = op;
|
| 790 | op = insn->opcode.bytes[1];
|
| 791 | goto clang_jcc;
|
| 792 | }
|
| 793 |
|
| 794 | if (insn->length == 6)
|
| 795 | bytes[i++] = 0x2e;
|
| 796 |
|
| 797 | switch (op) {
|
| 798 | case CALL_INSN_OPCODE:
|
| 799 | __text_gen_insn(bytes+i, op, addr+i,
|
| 800 | call_dest,
|
| 801 | CALL_INSN_SIZE);
|
| 802 | i += CALL_INSN_SIZE;
|
| 803 | break;
|
| 804 |
|
| 805 | case JMP32_INSN_OPCODE:
|
| 806 | clang_jcc:
|
| 807 | __text_gen_insn(bytes+i, op, addr+i,
|
| 808 | jmp_dest,
|
| 809 | JMP32_INSN_SIZE);
|
| 810 | i += JMP32_INSN_SIZE;
|
| 811 | break;
|
| 812 |
|
| 813 | default:
|
| 814 | WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr);
|
| 815 | return -1;
|
| 816 | }
|
| 817 |
|
| 818 | WARN_ON_ONCE(i != insn->length);
|
| 819 |
|
| 820 | return i;
|
| 821 | }
|
| 822 |
|
| 823 | static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
|
| 824 | {
|
| 825 | return __emit_trampoline(addr, insn, bytes,
|
| 826 | __x86_indirect_call_thunk_array[reg],
|
| 827 | __x86_indirect_jump_thunk_array[reg]);
|
| 828 | }
|
| 829 |
|
| 830 | #ifdef CONFIG_MITIGATION_ITS
|
| 831 | static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
|
| 832 | {
|
| 833 | u8 *thunk = __x86_indirect_its_thunk_array[reg];
|
| 834 | u8 *tmp = its_allocate_thunk(reg);
|
| 835 |
|
| 836 | if (tmp)
|
| 837 | thunk = tmp;
|
| 838 |
|
| 839 | return __emit_trampoline(addr, insn, bytes, thunk, thunk);
|
| 840 | }
|
| 841 |
|
| 842 |
|
| 843 | static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
|
| 844 | {
|
| 845 | if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
|
| 846 | return false;
|
| 847 |
|
| 848 |
|
| 849 | addr += 1 + reg / 8;
|
| 850 |
|
| 851 |
|
| 852 | return !(addr & 0x20);
|
| 853 | }
|
| 854 | #else
|
| 855 |
|
| 856 | #ifdef CONFIG_FINEIBT
|
| 857 | static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
|
| 858 | {
|
| 859 | return false;
|
| 860 | }
|
| 861 | #endif
|
| 862 |
|
| 863 | #endif
|
| 864 |
|
| 865 | |
| 866 | * Rewrite the compiler generated retpoline thunk calls.
|
| 867 | *
|
| 868 | * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate
|
| 869 | * indirect instructions, avoiding the extra indirection.
|
| 870 | *
|
| 871 | * For example, convert:
|
| 872 | *
|
| 873 | * CALL __x86_indirect_thunk_\reg
|
| 874 | *
|
| 875 | * into:
|
| 876 | *
|
| 877 | * CALL *%\reg
|
| 878 | *
|
| 879 | * It also tries to inline spectre_v2=retpoline,lfence when size permits.
|
| 880 |
|
| 881 | static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
|
| 882 | {
|
| 883 | retpoline_thunk_t *target;
|
| 884 | int reg, ret, i = 0;
|
| 885 | u8 op, cc;
|
| 886 |
|
| 887 | target = addr + insn->length + insn->immediate.value;
|
| 888 | reg = target - __x86_indirect_thunk_array;
|
| 889 |
|
| 890 | if (WARN_ON_ONCE(reg & ~0xf))
|
| 891 | return -1;
|
| 892 |
|
| 893 |
|
| 894 | BUG_ON(reg == 4);
|
| 895 |
|
| 896 | if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&
|
| 897 | !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
|
| 898 | if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
|
| 899 | return emit_call_track_retpoline(addr, insn, reg, bytes);
|
| 900 |
|
| 901 | return -1;
|
| 902 | }
|
| 903 |
|
| 904 | op = insn->opcode.bytes[0];
|
| 905 |
|
| 906 | |
| 907 | * Convert:
|
| 908 | *
|
| 909 | * Jcc.d32 __x86_indirect_thunk_\reg
|
| 910 | *
|
| 911 | * into:
|
| 912 | *
|
| 913 | * Jncc.d8 1f
|
| 914 | * [ LFENCE ]
|
| 915 | * JMP *%\reg
|
| 916 | * [ NOP ]
|
| 917 | * 1:
|
| 918 |
|
| 919 | if (is_jcc32(insn)) {
|
| 920 | cc = insn->opcode.bytes[1] & 0xf;
|
| 921 | cc ^= 1;
|
| 922 |
|
| 923 | bytes[i++] = 0x70 + cc;
|
| 924 | bytes[i++] = insn->length - 2;
|
| 925 |
|
| 926 |
|
| 927 | op = JMP32_INSN_OPCODE;
|
| 928 | }
|
| 929 |
|
| 930 | |
| 931 | * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE.
|
| 932 |
|
| 933 | if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
|
| 934 | bytes[i++] = 0x0f;
|
| 935 | bytes[i++] = 0xae;
|
| 936 | bytes[i++] = 0xe8;
|
| 937 | }
|
| 938 |
|
| 939 | #ifdef CONFIG_MITIGATION_ITS
|
| 940 | |
| 941 | * Check if the address of last byte of emitted-indirect is in
|
| 942 | * lower-half of the cacheline. Such branches need ITS mitigation.
|
| 943 |
|
| 944 | if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg))
|
| 945 | return emit_its_trampoline(addr, insn, reg, bytes);
|
| 946 | #endif
|
| 947 |
|
| 948 | ret = emit_indirect(op, reg, bytes + i, insn->length - i);
|
| 949 | if (ret < 0)
|
| 950 | return ret;
|
| 951 | i += ret;
|
| 952 |
|
| 953 | for (; i < insn->length;)
|
| 954 | bytes[i++] = BYTES_NOP1;
|
| 955 |
|
| 956 | return i;
|
| 957 | }
|
| 958 |
|
| 959 | |
| 960 | * Generated by 'objtool --retpoline'.
|
| 961 |
|
| 962 | void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
|
| 963 | {
|
| 964 | s32 *s;
|
| 965 |
|
| 966 | for (s = start; s < end; s++) {
|
| 967 | void *addr = (void *)s + *s;
|
| 968 | struct insn insn;
|
| 969 | int len, ret;
|
| 970 | u8 bytes[16];
|
| 971 | u8 op1, op2;
|
| 972 | u8 *dest;
|
| 973 |
|
| 974 | ret = insn_decode_kernel(&insn, addr);
|
| 975 | if (WARN_ON_ONCE(ret < 0))
|
| 976 | continue;
|
| 977 |
|
| 978 | op1 = insn.opcode.bytes[0];
|
| 979 | op2 = insn.opcode.bytes[1];
|
| 980 |
|
| 981 | switch (op1) {
|
| 982 | case 0x70 ... 0x7f:
|
| 983 |
|
| 984 | WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
|
| 985 | continue;
|
| 986 |
|
| 987 | case CALL_INSN_OPCODE:
|
| 988 | case JMP32_INSN_OPCODE:
|
| 989 |
|
| 990 | dest = addr + insn.length + insn.immediate.value;
|
| 991 | if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) {
|
| 992 | WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
|
| 993 | continue;
|
| 994 | }
|
| 995 | break;
|
| 996 |
|
| 997 | case 0x0f:
|
| 998 | if (op2 >= 0x80 && op2 <= 0x8f)
|
| 999 | break;
|
| 1000 | fallthrough;
|
| 1001 | default:
|
| 1002 | WARN_ON_ONCE(1);
|
| 1003 | continue;
|
| 1004 | }
|
| 1005 |
|
| 1006 | DPRINTK(RETPOLINE, "retpoline at: %pS (%px) len: %d to: %pS",
|
| 1007 | addr, addr, insn.length,
|
| 1008 | addr + insn.length + insn.immediate.value);
|
| 1009 |
|
| 1010 | len = patch_retpoline(addr, &insn, bytes);
|
| 1011 | if (len == insn.length) {
|
| 1012 | optimize_nops(addr, bytes, len);
|
| 1013 | DUMP_BYTES(RETPOLINE, ((u8*)addr), len, "%px: orig: ", addr);
|
| 1014 | DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr);
|
| 1015 | text_poke_early(addr, bytes, len);
|
| 1016 | }
|
| 1017 | }
|
| 1018 | }
|
| 1019 |
|
| 1020 | #ifdef CONFIG_MITIGATION_RETHUNK
|
| 1021 |
|
| 1022 | bool cpu_wants_rethunk(void)
|
| 1023 | {
|
| 1024 | return cpu_feature_enabled(X86_FEATURE_RETHUNK);
|
| 1025 | }
|
| 1026 |
|
| 1027 | bool cpu_wants_rethunk_at(void *addr)
|
| 1028 | {
|
| 1029 | if (!cpu_feature_enabled(X86_FEATURE_RETHUNK))
|
| 1030 | return false;
|
| 1031 | if (x86_return_thunk != its_return_thunk)
|
| 1032 | return true;
|
| 1033 |
|
| 1034 | return !((unsigned long)addr & 0x20);
|
| 1035 | }
|
| 1036 |
|
| 1037 | |
| 1038 | * Rewrite the compiler generated return thunk tail-calls.
|
| 1039 | *
|
| 1040 | * For example, convert:
|
| 1041 | *
|
| 1042 | * JMP __x86_return_thunk
|
| 1043 | *
|
| 1044 | * into:
|
| 1045 | *
|
| 1046 | * RET
|
| 1047 |
|
| 1048 | static int patch_return(void *addr, struct insn *insn, u8 *bytes)
|
| 1049 | {
|
| 1050 | int i = 0;
|
| 1051 |
|
| 1052 |
|
| 1053 | if (cpu_wants_rethunk_at(addr)) {
|
| 1054 | i = JMP32_INSN_SIZE;
|
| 1055 | __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i);
|
| 1056 | } else {
|
| 1057 |
|
| 1058 | bytes[i++] = RET_INSN_OPCODE;
|
| 1059 | }
|
| 1060 |
|
| 1061 | for (; i < insn->length;)
|
| 1062 | bytes[i++] = INT3_INSN_OPCODE;
|
| 1063 | return i;
|
| 1064 | }
|
| 1065 |
|
| 1066 | void __init_or_module noinline apply_returns(s32 *start, s32 *end)
|
| 1067 | {
|
| 1068 | s32 *s;
|
| 1069 |
|
| 1070 | if (cpu_wants_rethunk())
|
| 1071 | static_call_force_reinit();
|
| 1072 |
|
| 1073 | for (s = start; s < end; s++) {
|
| 1074 | void *dest = NULL, *addr = (void *)s + *s;
|
| 1075 | struct insn insn;
|
| 1076 | int len, ret;
|
| 1077 | u8 bytes[16];
|
| 1078 | u8 op;
|
| 1079 |
|
| 1080 | ret = insn_decode_kernel(&insn, addr);
|
| 1081 | if (WARN_ON_ONCE(ret < 0))
|
| 1082 | continue;
|
| 1083 |
|
| 1084 | op = insn.opcode.bytes[0];
|
| 1085 | if (op == JMP32_INSN_OPCODE)
|
| 1086 | dest = addr + insn.length + insn.immediate.value;
|
| 1087 |
|
| 1088 | if (__static_call_fixup(addr, op, dest) ||
|
| 1089 | WARN_ONCE(dest != &__x86_return_thunk,
|
| 1090 | "missing return thunk: %pS-%pS: %*ph",
|
| 1091 | addr, dest, 5, addr))
|
| 1092 | continue;
|
| 1093 |
|
| 1094 | DPRINTK(RET, "return thunk at: %pS (%px) len: %d to: %pS",
|
| 1095 | addr, addr, insn.length,
|
| 1096 | addr + insn.length + insn.immediate.value);
|
| 1097 |
|
| 1098 | len = patch_return(addr, &insn, bytes);
|
| 1099 | if (len == insn.length) {
|
| 1100 | DUMP_BYTES(RET, ((u8*)addr), len, "%px: orig: ", addr);
|
| 1101 | DUMP_BYTES(RET, ((u8*)bytes), len, "%px: repl: ", addr);
|
| 1102 | text_poke_early(addr, bytes, len);
|
| 1103 | }
|
| 1104 | }
|
| 1105 | }
|
| 1106 | #else
|
| 1107 | void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
|
| 1108 | #endif
|
| 1109 |
|
| 1110 | #else
|
| 1111 |
|
| 1112 | void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }
|
| 1113 | void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
|
| 1114 |
|
| 1115 | #endif
|
| 1116 |
|
| 1117 | #ifdef CONFIG_X86_KERNEL_IBT
|
| 1118 |
|
| 1119 | __noendbr bool is_endbr(u32 *val)
|
| 1120 | {
|
| 1121 | u32 endbr;
|
| 1122 |
|
| 1123 | __get_kernel_nofault(&endbr, val, u32, Efault);
|
| 1124 | return __is_endbr(endbr);
|
| 1125 |
|
| 1126 | Efault:
|
| 1127 | return false;
|
| 1128 | }
|
| 1129 |
|
| 1130 | #ifdef CONFIG_FINEIBT
|
| 1131 |
|
| 1132 | static __noendbr bool exact_endbr(u32 *val)
|
| 1133 | {
|
| 1134 | u32 endbr;
|
| 1135 |
|
| 1136 | __get_kernel_nofault(&endbr, val, u32, Efault);
|
| 1137 | return endbr == gen_endbr();
|
| 1138 |
|
| 1139 | Efault:
|
| 1140 | return false;
|
| 1141 | }
|
| 1142 |
|
| 1143 | #endif
|
| 1144 |
|
| 1145 | static void poison_cfi(void *addr);
|
| 1146 |
|
| 1147 | static void __init_or_module poison_endbr(void *addr)
|
| 1148 | {
|
| 1149 | u32 poison = gen_endbr_poison();
|
| 1150 |
|
| 1151 | if (WARN_ON_ONCE(!is_endbr(addr)))
|
| 1152 | return;
|
| 1153 |
|
| 1154 | DPRINTK(ENDBR, "ENDBR at: %pS (%px)", addr, addr);
|
| 1155 |
|
| 1156 | |
| 1157 | * When we have IBT, the lack of ENDBR will trigger #CP
|
| 1158 |
|
| 1159 | DUMP_BYTES(ENDBR, ((u8*)addr), 4, "%px: orig: ", addr);
|
| 1160 | DUMP_BYTES(ENDBR, ((u8*)&poison), 4, "%px: repl: ", addr);
|
| 1161 | text_poke_early(addr, &poison, 4);
|
| 1162 | }
|
| 1163 |
|
| 1164 | |
| 1165 | * Generated by: objtool --ibt
|
| 1166 | *
|
| 1167 | * Seal the functions for indirect calls by clobbering the ENDBR instructions
|
| 1168 | * and the kCFI hash value.
|
| 1169 |
|
| 1170 | void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end)
|
| 1171 | {
|
| 1172 | s32 *s;
|
| 1173 |
|
| 1174 | for (s = start; s < end; s++) {
|
| 1175 | void *addr = (void *)s + *s;
|
| 1176 |
|
| 1177 | poison_endbr(addr);
|
| 1178 | if (IS_ENABLED(CONFIG_FINEIBT))
|
| 1179 | poison_cfi(addr - CFI_OFFSET);
|
| 1180 | }
|
| 1181 | }
|
| 1182 |
|
| 1183 | #else
|
| 1184 |
|
| 1185 | void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }
|
| 1186 |
|
| 1187 | #endif
|
| 1188 |
|
| 1189 | #ifdef CONFIG_CFI_AUTO_DEFAULT
|
| 1190 | # define __CFI_DEFAULT CFI_AUTO
|
| 1191 | #elif defined(CONFIG_CFI)
|
| 1192 | # define __CFI_DEFAULT CFI_KCFI
|
| 1193 | #else
|
| 1194 | # define __CFI_DEFAULT CFI_OFF
|
| 1195 | #endif
|
| 1196 |
|
| 1197 | enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;
|
| 1198 | static bool cfi_debug __ro_after_init;
|
| 1199 |
|
| 1200 | #ifdef CONFIG_FINEIBT_BHI
|
| 1201 | bool cfi_bhi __ro_after_init = false;
|
| 1202 | #endif
|
| 1203 |
|
| 1204 | #ifdef CONFIG_FINEIBT
|
| 1205 | |
| 1206 | * <fineibt_preamble_start>:
|
| 1207 | * 0: f3 0f 1e fa endbr64
|
| 1208 | * 4: 2d 78 56 34 12 sub $0x12345678, %eax
|
| 1209 | * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
|
| 1210 | * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
|
| 1211 | *
|
| 1212 | * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
|
| 1213 | * UDB on x86_64 and raises #UD.
|
| 1214 |
|
| 1215 | asm( ".pushsection .rodata \n"
|
| 1216 | "fineibt_preamble_start: \n"
|
| 1217 | " endbr64 \n"
|
| 1218 | " subl $0x12345678, %eax \n"
|
| 1219 | "fineibt_preamble_bhi: \n"
|
| 1220 | " cs jne.d32 fineibt_preamble_start+0x13 \n"
|
| 1221 | "#fineibt_func: \n"
|
| 1222 | " nopl -42(%rax) \n"
|
| 1223 | "fineibt_preamble_end: \n"
|
| 1224 | ".popsection\n"
|
| 1225 | );
|
| 1226 |
|
| 1227 | extern u8 fineibt_preamble_start[];
|
| 1228 | extern u8 fineibt_preamble_bhi[];
|
| 1229 | extern u8 fineibt_preamble_end[];
|
| 1230 |
|
| 1231 | #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
|
| 1232 | #define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
|
| 1233 | #define fineibt_preamble_ud 0x13
|
| 1234 | #define fineibt_preamble_hash 5
|
| 1235 |
|
| 1236 | #define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
|
| 1237 | #endif
|
| 1238 |
|
| 1239 | #ifdef CONFIG_CFI
|
| 1240 | u32 cfi_get_func_hash(void *func)
|
| 1241 | {
|
| 1242 | u32 hash;
|
| 1243 |
|
| 1244 | func -= cfi_get_offset();
|
| 1245 | switch (cfi_mode) {
|
| 1246 | #ifdef CONFIG_FINEIBT
|
| 1247 | case CFI_FINEIBT:
|
| 1248 | func += fineibt_preamble_hash;
|
| 1249 | break;
|
| 1250 | #endif
|
| 1251 | case CFI_KCFI:
|
| 1252 | func += 1;
|
| 1253 | break;
|
| 1254 | default:
|
| 1255 | return 0;
|
| 1256 | }
|
| 1257 |
|
| 1258 | if (get_kernel_nofault(hash, func))
|
| 1259 | return 0;
|
| 1260 |
|
| 1261 | return hash;
|
| 1262 | }
|
| 1263 |
|
| 1264 | int cfi_get_func_arity(void *func)
|
| 1265 | {
|
| 1266 | bhi_thunk *target;
|
| 1267 | s32 disp;
|
| 1268 |
|
| 1269 | if (cfi_mode != CFI_FINEIBT && !cfi_bhi)
|
| 1270 | return 0;
|
| 1271 |
|
| 1272 | if (get_kernel_nofault(disp, func - 4))
|
| 1273 | return 0;
|
| 1274 |
|
| 1275 | target = func + disp;
|
| 1276 | return target - __bhi_args;
|
| 1277 | }
|
| 1278 | #endif
|
| 1279 |
|
| 1280 | #ifdef CONFIG_FINEIBT
|
| 1281 |
|
| 1282 | static bool cfi_rand __ro_after_init = true;
|
| 1283 | static u32 cfi_seed __ro_after_init;
|
| 1284 |
|
| 1285 | |
| 1286 | * Re-hash the CFI hash with a boot-time seed while making sure the result is
|
| 1287 | * not a valid ENDBR instruction.
|
| 1288 |
|
| 1289 | static u32 cfi_rehash(u32 hash)
|
| 1290 | {
|
| 1291 | hash ^= cfi_seed;
|
| 1292 | while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) {
|
| 1293 | bool lsb = hash & 1;
|
| 1294 | hash >>= 1;
|
| 1295 | if (lsb)
|
| 1296 | hash ^= 0x80200003;
|
| 1297 | }
|
| 1298 | return hash;
|
| 1299 | }
|
| 1300 |
|
| 1301 | static __init int cfi_parse_cmdline(char *str)
|
| 1302 | {
|
| 1303 | if (!str)
|
| 1304 | return -EINVAL;
|
| 1305 |
|
| 1306 | while (str) {
|
| 1307 | char *next = strchr(str, ',');
|
| 1308 | if (next) {
|
| 1309 | *next = 0;
|
| 1310 | next++;
|
| 1311 | }
|
| 1312 |
|
| 1313 | if (!strcmp(str, "auto")) {
|
| 1314 | cfi_mode = CFI_AUTO;
|
| 1315 | } else if (!strcmp(str, "off")) {
|
| 1316 | cfi_mode = CFI_OFF;
|
| 1317 | cfi_rand = false;
|
| 1318 | } else if (!strcmp(str, "debug")) {
|
| 1319 | cfi_debug = true;
|
| 1320 | } else if (!strcmp(str, "kcfi")) {
|
| 1321 | cfi_mode = CFI_KCFI;
|
| 1322 | } else if (!strcmp(str, "fineibt")) {
|
| 1323 | cfi_mode = CFI_FINEIBT;
|
| 1324 | } else if (!strcmp(str, "norand")) {
|
| 1325 | cfi_rand = false;
|
| 1326 | } else if (!strcmp(str, "warn")) {
|
| 1327 | pr_alert("CFI: mismatch non-fatal!\n");
|
| 1328 | cfi_warn = true;
|
| 1329 | } else if (!strcmp(str, "paranoid")) {
|
| 1330 | if (cfi_mode == CFI_FINEIBT) {
|
| 1331 | cfi_paranoid = true;
|
| 1332 | } else {
|
| 1333 | pr_err("CFI: ignoring paranoid; depends on fineibt.\n");
|
| 1334 | }
|
| 1335 | } else if (!strcmp(str, "bhi")) {
|
| 1336 | #ifdef CONFIG_FINEIBT_BHI
|
| 1337 | if (cfi_mode == CFI_FINEIBT) {
|
| 1338 | cfi_bhi = true;
|
| 1339 | } else {
|
| 1340 | pr_err("CFI: ignoring bhi; depends on fineibt.\n");
|
| 1341 | }
|
| 1342 | #else
|
| 1343 | pr_err("CFI: ignoring bhi; depends on FINEIBT_BHI=y.\n");
|
| 1344 | #endif
|
| 1345 | } else {
|
| 1346 | pr_err("CFI: Ignoring unknown option (%s).", str);
|
| 1347 | }
|
| 1348 |
|
| 1349 | str = next;
|
| 1350 | }
|
| 1351 |
|
| 1352 | return 0;
|
| 1353 | }
|
| 1354 | early_param("cfi", cfi_parse_cmdline);
|
| 1355 |
|
| 1356 | |
| 1357 | * kCFI FineIBT
|
| 1358 | *
|
| 1359 | * __cfi_\func: __cfi_\func:
|
| 1360 | * movl $0x12345678,%eax // 5 endbr64 // 4
|
| 1361 | * nop subl $0x12345678,%eax // 5
|
| 1362 | * nop jne.d32,pn \func+3 // 7
|
| 1363 | * nop
|
| 1364 | * nop
|
| 1365 | * nop
|
| 1366 | * nop
|
| 1367 | * nop
|
| 1368 | * nop
|
| 1369 | * nop
|
| 1370 | * nop
|
| 1371 | * nop
|
| 1372 | * \func: \func:
|
| 1373 | * endbr64 nopl -42(%rax)
|
| 1374 | *
|
| 1375 | *
|
| 1376 | * caller: caller:
|
| 1377 | * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%eax // 5
|
| 1378 | * addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4
|
| 1379 | * je 1f // 2 nop5 // 5
|
| 1380 | * ud2 // 2
|
| 1381 | * 1: cs call __x86_indirect_thunk_r11 // 6 call *%r11; nop3; // 6
|
| 1382 | *
|
| 1383 | *
|
| 1384 | * Notably, the FineIBT sequences are crafted such that branches are presumed
|
| 1385 | * non-taken. This is based on Agner Fog's optimization manual, which states:
|
| 1386 | *
|
| 1387 | * "Make conditional jumps most often not taken: The efficiency and throughput
|
| 1388 | * for not-taken branches is better than for taken branches on most
|
| 1389 | * processors. Therefore, it is good to place the most frequent branch first"
|
| 1390 | *
|
| 1391 | * NOTE: Update the kCFI caller sequence to make use of this observation:
|
| 1392 | *
|
| 1393 | * kCFI kCFI-OPT
|
| 1394 | *
|
| 1395 | * caller: caller:
|
| 1396 | * movl $(-0x12345678),%r10d // 6 movl $(-0x12345678),%r10d // 6
|
| 1397 | * addl $-15(%r11),%r10d // 4 addl $-15(%r11),%r10d // 4
|
| 1398 | * je 1f // 2 jne . + 3 // 2
|
| 1399 | * ud2 // 2 test $0xd6, %al // 2
|
| 1400 | * 1: cs call __x86_indirect_thunk_r11 // 6 1: cs call __x86_indirect_thunk_r11 // 6
|
| 1401 | *
|
| 1402 | * This new test clobbers eflags, but those are clobbered by the hash test
|
| 1403 | * anyway.
|
| 1404 |
|
| 1405 |
|
| 1406 | |
| 1407 | * <fineibt_caller_start>:
|
| 1408 | * 0: b8 78 56 34 12 mov $0x12345678, %eax
|
| 1409 | * 5: 4d 8d 5b f0 lea -0x10(%r11), %r11
|
| 1410 | * 9: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
|
| 1411 |
|
| 1412 | asm( ".pushsection .rodata \n"
|
| 1413 | "fineibt_caller_start: \n"
|
| 1414 | " movl $0x12345678, %eax \n"
|
| 1415 | " lea -0x10(%r11), %r11 \n"
|
| 1416 | ASM_NOP5
|
| 1417 | "fineibt_caller_end: \n"
|
| 1418 | ".popsection \n"
|
| 1419 | );
|
| 1420 |
|
| 1421 | extern u8 fineibt_caller_start[];
|
| 1422 | extern u8 fineibt_caller_end[];
|
| 1423 |
|
| 1424 | #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
|
| 1425 | #define fineibt_caller_hash 1
|
| 1426 |
|
| 1427 | #define fineibt_caller_jmp (fineibt_caller_size - 2)
|
| 1428 |
|
| 1429 | |
| 1430 | * Since FineIBT does hash validation on the callee side it is prone to
|
| 1431 | * circumvention attacks where a 'naked' ENDBR instruction exists that
|
| 1432 | * is not part of the fineibt_preamble sequence.
|
| 1433 | *
|
| 1434 | * Notably the x86 entry points must be ENDBR and equally cannot be
|
| 1435 | * fineibt_preamble.
|
| 1436 | *
|
| 1437 | * The fineibt_paranoid caller sequence adds additional caller side
|
| 1438 | * hash validation. This stops such circumvention attacks dead, but at the cost
|
| 1439 | * of adding a load.
|
| 1440 | *
|
| 1441 | * <fineibt_paranoid_start>:
|
| 1442 | * 0: b8 78 56 34 12 mov $0x12345678, %eax
|
| 1443 | * 5: 41 3b 43 f5 cmp -0x11(%r11), %eax
|
| 1444 | * 9: 2e 4d 8d 5b <f0> cs lea -0x10(%r11), %r11
|
| 1445 | * e: 75 fd jne d <fineibt_paranoid_start+0xd>
|
| 1446 | * 10: 41 ff d3 call *%r11
|
| 1447 | * 13: 90 nop
|
| 1448 | *
|
| 1449 | * Notably LEA does not modify flags and can be reordered with the CMP,
|
| 1450 | * avoiding a dependency. Again, using a non-taken (backwards) branch
|
| 1451 | * for the failure case, abusing LEA's immediate 0xf0 as LOCK prefix for the
|
| 1452 | * Jcc.d8, causing #UD.
|
| 1453 |
|
| 1454 | asm( ".pushsection .rodata \n"
|
| 1455 | "fineibt_paranoid_start: \n"
|
| 1456 | " mov $0x12345678, %eax \n"
|
| 1457 | " cmpl -11(%r11), %eax \n"
|
| 1458 | " .byte 0x2e \n"
|
| 1459 | " lea -0x10(%r11), %r11 \n"
|
| 1460 | "#fineibt_caller_size: \n"
|
| 1461 | " jne fineibt_paranoid_start+0xd \n"
|
| 1462 | "fineibt_paranoid_ind: \n"
|
| 1463 | " cs call *%r11 \n"
|
| 1464 | "fineibt_paranoid_end: \n"
|
| 1465 | ".popsection \n"
|
| 1466 | );
|
| 1467 |
|
| 1468 | extern u8 fineibt_paranoid_start[];
|
| 1469 | extern u8 fineibt_paranoid_ind[];
|
| 1470 | extern u8 fineibt_paranoid_end[];
|
| 1471 |
|
| 1472 | #define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start)
|
| 1473 | #define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_start)
|
| 1474 | #define fineibt_paranoid_ud 0xd
|
| 1475 |
|
| 1476 | static u32 decode_preamble_hash(void *addr, int *reg)
|
| 1477 | {
|
| 1478 | u8 *p = addr;
|
| 1479 |
|
| 1480 |
|
| 1481 | if (p[0] >= 0xb8 && p[0] < 0xc0) {
|
| 1482 | if (reg)
|
| 1483 | *reg = p[0] - 0xb8;
|
| 1484 | return *(u32 *)(addr + 1);
|
| 1485 | }
|
| 1486 |
|
| 1487 | return 0;
|
| 1488 | }
|
| 1489 |
|
| 1490 | static u32 decode_caller_hash(void *addr)
|
| 1491 | {
|
| 1492 | u8 *p = addr;
|
| 1493 |
|
| 1494 |
|
| 1495 | if (p[0] == 0x41 && p[1] == 0xba)
|
| 1496 | return -*(u32 *)(addr + 2);
|
| 1497 |
|
| 1498 |
|
| 1499 | if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp)
|
| 1500 | return -*(u32 *)(addr + 2);
|
| 1501 |
|
| 1502 | return 0;
|
| 1503 | }
|
| 1504 |
|
| 1505 |
|
| 1506 | static int cfi_disable_callers(s32 *start, s32 *end)
|
| 1507 | {
|
| 1508 | |
| 1509 | * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate
|
| 1510 | * in tact for later usage. Also see decode_caller_hash() and
|
| 1511 | * cfi_rewrite_callers().
|
| 1512 |
|
| 1513 | const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp };
|
| 1514 | s32 *s;
|
| 1515 |
|
| 1516 | for (s = start; s < end; s++) {
|
| 1517 | void *addr = (void *)s + *s;
|
| 1518 | u32 hash;
|
| 1519 |
|
| 1520 | addr -= fineibt_caller_size;
|
| 1521 | hash = decode_caller_hash(addr);
|
| 1522 | if (!hash)
|
| 1523 | continue;
|
| 1524 |
|
| 1525 | text_poke_early(addr, jmp, 2);
|
| 1526 | }
|
| 1527 |
|
| 1528 | return 0;
|
| 1529 | }
|
| 1530 |
|
| 1531 | static int cfi_enable_callers(s32 *start, s32 *end)
|
| 1532 | {
|
| 1533 | |
| 1534 | * Re-enable (and update) kCFI, undo what cfi_disable_callers() did.
|
| 1535 |
|
| 1536 | const u8 udne[] = { 0x75, 0x01, 0xa8, 0xd6 };
|
| 1537 | const u8 mov[] = { 0x41, 0xba };
|
| 1538 | s32 *s;
|
| 1539 |
|
| 1540 | for (s = start; s < end; s++) {
|
| 1541 | void *addr = (void *)s + *s;
|
| 1542 | u32 hash;
|
| 1543 |
|
| 1544 | addr -= fineibt_caller_size;
|
| 1545 | hash = decode_caller_hash(addr);
|
| 1546 | if (!hash)
|
| 1547 | continue;
|
| 1548 |
|
| 1549 | |
| 1550 | * See the kCFI/FineIBT comment above -- update note.
|
| 1551 |
|
| 1552 | text_poke_early(addr + 10, udne, 4);
|
| 1553 | text_poke_early(addr, mov, 2);
|
| 1554 | }
|
| 1555 |
|
| 1556 | return 0;
|
| 1557 | }
|
| 1558 |
|
| 1559 |
|
| 1560 | static int cfi_rand_preamble(s32 *start, s32 *end)
|
| 1561 | {
|
| 1562 | s32 *s;
|
| 1563 |
|
| 1564 | for (s = start; s < end; s++) {
|
| 1565 | void *addr = (void *)s + *s;
|
| 1566 | u32 hash;
|
| 1567 |
|
| 1568 | hash = decode_preamble_hash(addr, NULL);
|
| 1569 | if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
|
| 1570 | addr, addr, 5, addr))
|
| 1571 | return -EINVAL;
|
| 1572 |
|
| 1573 | hash = cfi_rehash(hash);
|
| 1574 | text_poke_early(addr + 1, &hash, 4);
|
| 1575 | }
|
| 1576 |
|
| 1577 | return 0;
|
| 1578 | }
|
| 1579 |
|
| 1580 | |
| 1581 | * Inline the bhi-arity 1 case:
|
| 1582 | *
|
| 1583 | * __cfi_foo:
|
| 1584 | * 0: f3 0f 1e fa endbr64
|
| 1585 | * 4: 2d 78 56 34 12 sub $0x12345678, %eax
|
| 1586 | * 9: 49 0f 45 fa cmovne %rax, %rdi
|
| 1587 | * d: 2e 75 03 jne,pn foo+0x3
|
| 1588 | *
|
| 1589 | * foo:
|
| 1590 | * 10: 0f 1f 40 <d6> nopl -42(%rax)
|
| 1591 | *
|
| 1592 | * Notably, this scheme is incompatible with permissive CFI
|
| 1593 | * because the CMOVcc is unconditional and RDI will have been
|
| 1594 | * clobbered.
|
| 1595 |
|
| 1596 | asm( ".pushsection .rodata \n"
|
| 1597 | "fineibt_bhi1_start: \n"
|
| 1598 | " cmovne %rax, %rdi \n"
|
| 1599 | " cs jne fineibt_bhi1_func + 0x3 \n"
|
| 1600 | "fineibt_bhi1_func: \n"
|
| 1601 | " nopl -42(%rax) \n"
|
| 1602 | "fineibt_bhi1_end: \n"
|
| 1603 | ".popsection \n"
|
| 1604 | );
|
| 1605 |
|
| 1606 | extern u8 fineibt_bhi1_start[];
|
| 1607 | extern u8 fineibt_bhi1_end[];
|
| 1608 |
|
| 1609 | #define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start)
|
| 1610 |
|
| 1611 | static void cfi_fineibt_bhi_preamble(void *addr, int arity)
|
| 1612 | {
|
| 1613 | u8 bytes[MAX_INSN_SIZE];
|
| 1614 |
|
| 1615 | if (!arity)
|
| 1616 | return;
|
| 1617 |
|
| 1618 | if (!cfi_warn && arity == 1) {
|
| 1619 | text_poke_early(addr + fineibt_preamble_bhi,
|
| 1620 | fineibt_bhi1_start, fineibt_bhi1_size);
|
| 1621 | return;
|
| 1622 | }
|
| 1623 |
|
| 1624 | |
| 1625 | * Replace the bytes at fineibt_preamble_bhi with a CALL instruction
|
| 1626 | * that lines up exactly with the end of the preamble, such that the
|
| 1627 | * return address will be foo+0.
|
| 1628 | *
|
| 1629 | * __cfi_foo:
|
| 1630 | * 0: f3 0f 1e fa endbr64
|
| 1631 | * 4: 2d 78 56 34 12 sub $0x12345678, %eax
|
| 1632 | * 9: 2e 2e e8 DD DD DD DD cs cs call __bhi_args[arity]
|
| 1633 |
|
| 1634 | bytes[0] = 0x2e;
|
| 1635 | bytes[1] = 0x2e;
|
| 1636 | __text_gen_insn(bytes + 2, CALL_INSN_OPCODE,
|
| 1637 | addr + fineibt_preamble_bhi + 2,
|
| 1638 | __bhi_args[arity], CALL_INSN_SIZE);
|
| 1639 |
|
| 1640 | text_poke_early(addr + fineibt_preamble_bhi, bytes, 7);
|
| 1641 | }
|
| 1642 |
|
| 1643 | static int cfi_rewrite_preamble(s32 *start, s32 *end)
|
| 1644 | {
|
| 1645 | s32 *s;
|
| 1646 |
|
| 1647 | for (s = start; s < end; s++) {
|
| 1648 | void *addr = (void *)s + *s;
|
| 1649 | int arity;
|
| 1650 | u32 hash;
|
| 1651 |
|
| 1652 | |
| 1653 | * When the function doesn't start with ENDBR the compiler will
|
| 1654 | * have determined there are no indirect calls to it and we
|
| 1655 | * don't need no CFI either.
|
| 1656 |
|
| 1657 | if (!is_endbr(addr + CFI_OFFSET))
|
| 1658 | continue;
|
| 1659 |
|
| 1660 | hash = decode_preamble_hash(addr, &arity);
|
| 1661 | if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
|
| 1662 | addr, addr, 5, addr))
|
| 1663 | return -EINVAL;
|
| 1664 |
|
| 1665 | |
| 1666 | * FineIBT relies on being at func-16, so if the preamble is
|
| 1667 | * actually larger than that, place it the tail end.
|
| 1668 | *
|
| 1669 | * NOTE: this is possible with things like DEBUG_CALL_THUNKS
|
| 1670 | * and DEBUG_FORCE_FUNCTION_ALIGN_64B.
|
| 1671 |
|
| 1672 | addr += CFI_OFFSET - fineibt_prefix_size;
|
| 1673 |
|
| 1674 | text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);
|
| 1675 | WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678);
|
| 1676 | text_poke_early(addr + fineibt_preamble_hash, &hash, 4);
|
| 1677 |
|
| 1678 | WARN_ONCE(!IS_ENABLED(CONFIG_FINEIBT_BHI) && arity,
|
| 1679 | "kCFI preamble has wrong register at: %pS %*ph\n",
|
| 1680 | addr, 5, addr);
|
| 1681 |
|
| 1682 | if (cfi_bhi)
|
| 1683 | cfi_fineibt_bhi_preamble(addr, arity);
|
| 1684 | }
|
| 1685 |
|
| 1686 | return 0;
|
| 1687 | }
|
| 1688 |
|
| 1689 | static void cfi_rewrite_endbr(s32 *start, s32 *end)
|
| 1690 | {
|
| 1691 | s32 *s;
|
| 1692 |
|
| 1693 | for (s = start; s < end; s++) {
|
| 1694 | void *addr = (void *)s + *s;
|
| 1695 |
|
| 1696 | if (!exact_endbr(addr + CFI_OFFSET))
|
| 1697 | continue;
|
| 1698 |
|
| 1699 | poison_endbr(addr + CFI_OFFSET);
|
| 1700 | }
|
| 1701 | }
|
| 1702 |
|
| 1703 |
|
| 1704 | static int cfi_rand_callers(s32 *start, s32 *end)
|
| 1705 | {
|
| 1706 | s32 *s;
|
| 1707 |
|
| 1708 | for (s = start; s < end; s++) {
|
| 1709 | void *addr = (void *)s + *s;
|
| 1710 | u32 hash;
|
| 1711 |
|
| 1712 | addr -= fineibt_caller_size;
|
| 1713 | hash = decode_caller_hash(addr);
|
| 1714 | if (hash) {
|
| 1715 | hash = -cfi_rehash(hash);
|
| 1716 | text_poke_early(addr + 2, &hash, 4);
|
| 1717 | }
|
| 1718 | }
|
| 1719 |
|
| 1720 | return 0;
|
| 1721 | }
|
| 1722 |
|
| 1723 | static int emit_paranoid_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
|
| 1724 | {
|
| 1725 | u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2;
|
| 1726 |
|
| 1727 | #ifdef CONFIG_MITIGATION_ITS
|
| 1728 | u8 *tmp = its_allocate_thunk(reg);
|
| 1729 | if (tmp)
|
| 1730 | thunk = tmp;
|
| 1731 | #endif
|
| 1732 |
|
| 1733 | return __emit_trampoline(addr, insn, bytes, thunk, thunk);
|
| 1734 | }
|
| 1735 |
|
| 1736 | static int cfi_rewrite_callers(s32 *start, s32 *end)
|
| 1737 | {
|
| 1738 | s32 *s;
|
| 1739 |
|
| 1740 | for (s = start; s < end; s++) {
|
| 1741 | void *addr = (void *)s + *s;
|
| 1742 | struct insn insn;
|
| 1743 | u8 bytes[20];
|
| 1744 | u32 hash;
|
| 1745 | int ret;
|
| 1746 | u8 op;
|
| 1747 |
|
| 1748 | addr -= fineibt_caller_size;
|
| 1749 | hash = decode_caller_hash(addr);
|
| 1750 | if (!hash)
|
| 1751 | continue;
|
| 1752 |
|
| 1753 | if (!cfi_paranoid) {
|
| 1754 | text_poke_early(addr, fineibt_caller_start, fineibt_caller_size);
|
| 1755 | WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678);
|
| 1756 | text_poke_early(addr + fineibt_caller_hash, &hash, 4);
|
| 1757 |
|
| 1758 | continue;
|
| 1759 | }
|
| 1760 |
|
| 1761 |
|
| 1762 | ret = insn_decode_kernel(&insn, addr + fineibt_caller_size);
|
| 1763 | if (WARN_ON_ONCE(ret < 0))
|
| 1764 | continue;
|
| 1765 |
|
| 1766 | op = insn.opcode.bytes[0];
|
| 1767 | if (op != CALL_INSN_OPCODE && op != JMP32_INSN_OPCODE) {
|
| 1768 | WARN_ON_ONCE(1);
|
| 1769 | continue;
|
| 1770 | }
|
| 1771 |
|
| 1772 | memcpy(bytes, fineibt_paranoid_start, fineibt_paranoid_size);
|
| 1773 | memcpy(bytes + fineibt_caller_hash, &hash, 4);
|
| 1774 |
|
| 1775 | if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + fineibt_paranoid_ind, 11)) {
|
| 1776 | emit_paranoid_trampoline(addr + fineibt_caller_size,
|
| 1777 | &insn, 11, bytes + fineibt_caller_size);
|
| 1778 | } else {
|
| 1779 | int len = fineibt_paranoid_size - fineibt_paranoid_ind;
|
| 1780 | ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind, len);
|
| 1781 | if (WARN_ON_ONCE(ret != len))
|
| 1782 | continue;
|
| 1783 | }
|
| 1784 |
|
| 1785 | text_poke_early(addr, bytes, fineibt_paranoid_size);
|
| 1786 | }
|
| 1787 |
|
| 1788 | return 0;
|
| 1789 | }
|
| 1790 |
|
| 1791 | #define pr_cfi_debug(X...) if (cfi_debug) pr_info(X)
|
| 1792 |
|
| 1793 | #define FINEIBT_WARN(_f, _v) \
|
| 1794 | WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v)
|
| 1795 |
|
| 1796 | static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
|
| 1797 | s32 *start_cfi, s32 *end_cfi, bool builtin)
|
| 1798 | {
|
| 1799 | int ret;
|
| 1800 |
|
| 1801 | if (FINEIBT_WARN(fineibt_preamble_size, 20) ||
|
| 1802 | FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) ||
|
| 1803 | FINEIBT_WARN(fineibt_caller_size, 14) ||
|
| 1804 | FINEIBT_WARN(fineibt_paranoid_size, 20) ||
|
| 1805 | WARN_ON_ONCE(CFI_OFFSET < fineibt_prefix_size))
|
| 1806 | return;
|
| 1807 |
|
| 1808 | if (cfi_mode == CFI_AUTO) {
|
| 1809 | cfi_mode = CFI_KCFI;
|
| 1810 | if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) {
|
| 1811 | |
| 1812 | * FRED has much saner context on exception entry and
|
| 1813 | * is less easy to take advantage of.
|
| 1814 |
|
| 1815 | if (!cpu_feature_enabled(X86_FEATURE_FRED))
|
| 1816 | cfi_paranoid = true;
|
| 1817 | cfi_mode = CFI_FINEIBT;
|
| 1818 | }
|
| 1819 | }
|
| 1820 |
|
| 1821 | |
| 1822 | * Rewrite the callers to not use the __cfi_ stubs, such that we might
|
| 1823 | * rewrite them. This disables all CFI. If this succeeds but any of the
|
| 1824 | * later stages fails, we're without CFI.
|
| 1825 |
|
| 1826 | pr_cfi_debug("CFI: disabling all indirect call checking\n");
|
| 1827 | ret = cfi_disable_callers(start_retpoline, end_retpoline);
|
| 1828 | if (ret)
|
| 1829 | goto err;
|
| 1830 |
|
| 1831 | if (cfi_rand) {
|
| 1832 | if (builtin) {
|
| 1833 | cfi_seed = get_random_u32();
|
| 1834 | cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
|
| 1835 | cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
|
| 1836 | }
|
| 1837 | pr_cfi_debug("CFI: cfi_seed: 0x%08x\n", cfi_seed);
|
| 1838 |
|
| 1839 | pr_cfi_debug("CFI: rehashing all preambles\n");
|
| 1840 | ret = cfi_rand_preamble(start_cfi, end_cfi);
|
| 1841 | if (ret)
|
| 1842 | goto err;
|
| 1843 |
|
| 1844 | pr_cfi_debug("CFI: rehashing all indirect calls\n");
|
| 1845 | ret = cfi_rand_callers(start_retpoline, end_retpoline);
|
| 1846 | if (ret)
|
| 1847 | goto err;
|
| 1848 | } else {
|
| 1849 | pr_cfi_debug("CFI: rehashing disabled\n");
|
| 1850 | }
|
| 1851 |
|
| 1852 | switch (cfi_mode) {
|
| 1853 | case CFI_OFF:
|
| 1854 | if (builtin)
|
| 1855 | pr_info("CFI: disabled\n");
|
| 1856 | return;
|
| 1857 |
|
| 1858 | case CFI_KCFI:
|
| 1859 | pr_cfi_debug("CFI: re-enabling all indirect call checking\n");
|
| 1860 | ret = cfi_enable_callers(start_retpoline, end_retpoline);
|
| 1861 | if (ret)
|
| 1862 | goto err;
|
| 1863 |
|
| 1864 | if (builtin)
|
| 1865 | pr_info("CFI: Using %sretpoline kCFI\n",
|
| 1866 | cfi_rand ? "rehashed " : "");
|
| 1867 | return;
|
| 1868 |
|
| 1869 | case CFI_FINEIBT:
|
| 1870 | pr_cfi_debug("CFI: adding FineIBT to all preambles\n");
|
| 1871 |
|
| 1872 | ret = cfi_rewrite_preamble(start_cfi, end_cfi);
|
| 1873 | if (ret)
|
| 1874 | goto err;
|
| 1875 |
|
| 1876 |
|
| 1877 | pr_cfi_debug("CFI: rewriting indirect call sites to use FineIBT\n");
|
| 1878 | ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
|
| 1879 | if (ret)
|
| 1880 | goto err;
|
| 1881 |
|
| 1882 |
|
| 1883 | pr_cfi_debug("CFI: removing old endbr insns\n");
|
| 1884 | cfi_rewrite_endbr(start_cfi, end_cfi);
|
| 1885 |
|
| 1886 | if (builtin) {
|
| 1887 | pr_info("Using %sFineIBT%s CFI\n",
|
| 1888 | cfi_paranoid ? "paranoid " : "",
|
| 1889 | cfi_bhi ? "+BHI" : "");
|
| 1890 | }
|
| 1891 | return;
|
| 1892 |
|
| 1893 | default:
|
| 1894 | break;
|
| 1895 | }
|
| 1896 |
|
| 1897 | err:
|
| 1898 | pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n");
|
| 1899 | }
|
| 1900 |
|
| 1901 | static inline void poison_hash(void *addr)
|
| 1902 | {
|
| 1903 | *(u32 *)addr = 0;
|
| 1904 | }
|
| 1905 |
|
| 1906 | static void poison_cfi(void *addr)
|
| 1907 | {
|
| 1908 | |
| 1909 | * Compilers manage to be inconsistent with ENDBR vs __cfi prefixes,
|
| 1910 | * some (static) functions for which they can determine the address
|
| 1911 | * is never taken do not get a __cfi prefix, but *DO* get an ENDBR.
|
| 1912 | *
|
| 1913 | * As such, these functions will get sealed, but we need to be careful
|
| 1914 | * to not unconditionally scribble the previous function.
|
| 1915 |
|
| 1916 | switch (cfi_mode) {
|
| 1917 | case CFI_FINEIBT:
|
| 1918 | |
| 1919 | * FineIBT preamble is at func-16.
|
| 1920 |
|
| 1921 | addr += CFI_OFFSET - fineibt_prefix_size;
|
| 1922 |
|
| 1923 | |
| 1924 | * FineIBT prefix should start with an ENDBR.
|
| 1925 |
|
| 1926 | if (!is_endbr(addr))
|
| 1927 | break;
|
| 1928 |
|
| 1929 | |
| 1930 | * __cfi_\func:
|
| 1931 | * nopl -42(%rax)
|
| 1932 | * sub $0, %eax
|
| 1933 | * jne \func+3
|
| 1934 | * \func:
|
| 1935 | * nopl -42(%rax)
|
| 1936 |
|
| 1937 | poison_endbr(addr);
|
| 1938 | poison_hash(addr + fineibt_preamble_hash);
|
| 1939 | break;
|
| 1940 |
|
| 1941 | case CFI_KCFI:
|
| 1942 | |
| 1943 | * kCFI prefix should start with a valid hash.
|
| 1944 |
|
| 1945 | if (!decode_preamble_hash(addr, NULL))
|
| 1946 | break;
|
| 1947 |
|
| 1948 | |
| 1949 | * __cfi_\func:
|
| 1950 | * movl $0, %eax
|
| 1951 | * .skip 11, 0x90
|
| 1952 |
|
| 1953 | poison_hash(addr + 1);
|
| 1954 | break;
|
| 1955 |
|
| 1956 | default:
|
| 1957 | break;
|
| 1958 | }
|
| 1959 | }
|
| 1960 |
|
| 1961 | |
| 1962 | * When regs->ip points to a 0xD6 byte in the FineIBT preamble,
|
| 1963 | * return true and fill out target and type.
|
| 1964 | *
|
| 1965 | * We check the preamble by checking for the ENDBR instruction relative to the
|
| 1966 | * UDB instruction.
|
| 1967 |
|
| 1968 | static bool decode_fineibt_preamble(struct pt_regs *regs, unsigned long *target, u32 *type)
|
| 1969 | {
|
| 1970 | unsigned long addr = regs->ip - fineibt_preamble_ud;
|
| 1971 | u32 hash;
|
| 1972 |
|
| 1973 | if (!exact_endbr((void *)addr))
|
| 1974 | return false;
|
| 1975 |
|
| 1976 | *target = addr + fineibt_prefix_size;
|
| 1977 |
|
| 1978 | __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
|
| 1979 | *type = (u32)regs->ax + hash;
|
| 1980 |
|
| 1981 | |
| 1982 | * Since regs->ip points to the middle of an instruction; it cannot
|
| 1983 | * continue with the normal fixup.
|
| 1984 |
|
| 1985 | regs->ip = *target;
|
| 1986 |
|
| 1987 | return true;
|
| 1988 |
|
| 1989 | Efault:
|
| 1990 | return false;
|
| 1991 | }
|
| 1992 |
|
| 1993 | |
| 1994 | * regs->ip points to one of the UD2 in __bhi_args[].
|
| 1995 |
|
| 1996 | static bool decode_fineibt_bhi(struct pt_regs *regs, unsigned long *target, u32 *type)
|
| 1997 | {
|
| 1998 | unsigned long addr;
|
| 1999 | u32 hash;
|
| 2000 |
|
| 2001 | if (!cfi_bhi)
|
| 2002 | return false;
|
| 2003 |
|
| 2004 | if (regs->ip < (unsigned long)__bhi_args ||
|
| 2005 | regs->ip >= (unsigned long)__bhi_args_end)
|
| 2006 | return false;
|
| 2007 |
|
| 2008 | |
| 2009 | * Fetch the return address from the stack, this points to the
|
| 2010 | * FineIBT preamble. Since the CALL instruction is in the 5 last
|
| 2011 | * bytes of the preamble, the return address is in fact the target
|
| 2012 | * address.
|
| 2013 |
|
| 2014 | __get_kernel_nofault(&addr, regs->sp, unsigned long, Efault);
|
| 2015 | *target = addr;
|
| 2016 |
|
| 2017 | addr -= fineibt_prefix_size;
|
| 2018 | if (!exact_endbr((void *)addr))
|
| 2019 | return false;
|
| 2020 |
|
| 2021 | __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
|
| 2022 | *type = (u32)regs->ax + hash;
|
| 2023 |
|
| 2024 | |
| 2025 | * The UD2 sites are constructed with a RET immediately following,
|
| 2026 | * as such the non-fatal case can use the regular fixup.
|
| 2027 |
|
| 2028 | return true;
|
| 2029 |
|
| 2030 | Efault:
|
| 2031 | return false;
|
| 2032 | }
|
| 2033 |
|
| 2034 | static bool is_paranoid_thunk(unsigned long addr)
|
| 2035 | {
|
| 2036 | u32 thunk;
|
| 2037 |
|
| 2038 | __get_kernel_nofault(&thunk, (u32 *)addr, u32, Efault);
|
| 2039 | return (thunk & 0x00FFFFFF) == 0xfd75d6;
|
| 2040 |
|
| 2041 | Efault:
|
| 2042 | return false;
|
| 2043 | }
|
| 2044 |
|
| 2045 | |
| 2046 | * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
|
| 2047 | * sequence, or to UDB + Jcc.d8 for cfi_paranoid + ITS thunk.
|
| 2048 |
|
| 2049 | static bool decode_fineibt_paranoid(struct pt_regs *regs, unsigned long *target, u32 *type)
|
| 2050 | {
|
| 2051 | unsigned long addr = regs->ip - fineibt_paranoid_ud;
|
| 2052 |
|
| 2053 | if (!cfi_paranoid)
|
| 2054 | return false;
|
| 2055 |
|
| 2056 | if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) {
|
| 2057 | *target = regs->r11 + fineibt_prefix_size;
|
| 2058 | *type = regs->ax;
|
| 2059 |
|
| 2060 | |
| 2061 | * Since the trapping instruction is the exact, but LOCK prefixed,
|
| 2062 | * Jcc.d8 that got us here, the normal fixup will work.
|
| 2063 |
|
| 2064 | return true;
|
| 2065 | }
|
| 2066 |
|
| 2067 | |
| 2068 | * The cfi_paranoid + ITS thunk combination results in:
|
| 2069 | *
|
| 2070 | * 0: b8 78 56 34 12 mov $0x12345678, %eax
|
| 2071 | * 5: 41 3b 43 f7 cmp -11(%r11), %eax
|
| 2072 | * a: 2e 3d 8d 5b f0 cs lea -0x10(%r11), %r11
|
| 2073 | * e: 2e e8 XX XX XX XX cs call __x86_indirect_paranoid_thunk_r11
|
| 2074 | *
|
| 2075 | * Where the paranoid_thunk looks like:
|
| 2076 | *
|
| 2077 | * 1d: <d6> udb
|
| 2078 | * __x86_indirect_paranoid_thunk_r11:
|
| 2079 | * 1e: 75 fd jne 1d
|
| 2080 | * __x86_indirect_its_thunk_r11:
|
| 2081 | * 20: 41 ff eb jmp *%r11
|
| 2082 | * 23: cc int3
|
| 2083 | *
|
| 2084 |
|
| 2085 | if (is_paranoid_thunk(regs->ip)) {
|
| 2086 | *target = regs->r11 + fineibt_prefix_size;
|
| 2087 | *type = regs->ax;
|
| 2088 |
|
| 2089 | regs->ip = *target;
|
| 2090 | return true;
|
| 2091 | }
|
| 2092 |
|
| 2093 | return false;
|
| 2094 | }
|
| 2095 |
|
| 2096 | bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type)
|
| 2097 | {
|
| 2098 | if (decode_fineibt_paranoid(regs, target, type))
|
| 2099 | return true;
|
| 2100 |
|
| 2101 | if (decode_fineibt_bhi(regs, target, type))
|
| 2102 | return true;
|
| 2103 |
|
| 2104 | return decode_fineibt_preamble(regs, target, type);
|
| 2105 | }
|
| 2106 |
|
| 2107 | #else
|
| 2108 |
|
| 2109 | static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
|
| 2110 | s32 *start_cfi, s32 *end_cfi, bool builtin)
|
| 2111 | {
|
| 2112 | if (IS_ENABLED(CONFIG_CFI) && builtin)
|
| 2113 | pr_info("CFI: Using standard kCFI\n");
|
| 2114 | }
|
| 2115 |
|
| 2116 | #ifdef CONFIG_X86_KERNEL_IBT
|
| 2117 | static void poison_cfi(void *addr) { }
|
| 2118 | #endif
|
| 2119 |
|
| 2120 | #endif
|
| 2121 |
|
| 2122 | void __init_or_module apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
|
| 2123 | s32 *start_cfi, s32 *end_cfi)
|
| 2124 | {
|
| 2125 | return __apply_fineibt(start_retpoline, end_retpoline,
|
| 2126 | start_cfi, end_cfi,
|
| 2127 | false);
|
| 2128 | }
|
| 2129 |
|
| 2130 | |
| 2131 | * Self-test for the INT3 based CALL emulation code.
|
| 2132 | *
|
| 2133 | * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
|
| 2134 | * properly and that there is a stack gap between the INT3 frame and the
|
| 2135 | * previous context. Without this gap doing a virtual PUSH on the interrupted
|
| 2136 | * stack would corrupt the INT3 IRET frame.
|
| 2137 | *
|
| 2138 | * See entry_{32,64}.S for more details.
|
| 2139 |
|
| 2140 |
|
| 2141 | extern void int3_selftest_asm(unsigned int *ptr);
|
| 2142 |
|
| 2143 | asm (
|
| 2144 | " .pushsection .init.text, \"ax\", @progbits\n"
|
| 2145 | " .type int3_selftest_asm, @function\n"
|
| 2146 | "int3_selftest_asm:\n"
|
| 2147 | ANNOTATE_NOENDBR "\n"
|
| 2148 | |
| 2149 | * INT3 padded with NOP to CALL_INSN_SIZE. The INT3 triggers an
|
| 2150 | * exception, then the int3_exception_nb notifier emulates a call to
|
| 2151 | * int3_selftest_callee().
|
| 2152 |
|
| 2153 | " int3; nop; nop; nop; nop\n"
|
| 2154 | ASM_RET
|
| 2155 | " .size int3_selftest_asm, . - int3_selftest_asm\n"
|
| 2156 | " .popsection\n"
|
| 2157 | );
|
| 2158 |
|
| 2159 | extern void int3_selftest_callee(unsigned int *ptr);
|
| 2160 |
|
| 2161 | asm (
|
| 2162 | " .pushsection .init.text, \"ax\", @progbits\n"
|
| 2163 | " .type int3_selftest_callee, @function\n"
|
| 2164 | "int3_selftest_callee:\n"
|
| 2165 | ANNOTATE_NOENDBR "\n"
|
| 2166 | " movl $0x1234, (%" _ASM_ARG1 ")\n"
|
| 2167 | ASM_RET
|
| 2168 | " .size int3_selftest_callee, . - int3_selftest_callee\n"
|
| 2169 | " .popsection\n"
|
| 2170 | );
|
| 2171 |
|
| 2172 | extern void int3_selftest_ip(void);
|
| 2173 |
|
| 2174 | static int __init
|
| 2175 | int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
|
| 2176 | {
|
| 2177 | unsigned long selftest = (unsigned long)&int3_selftest_asm;
|
| 2178 | struct die_args *args = data;
|
| 2179 | struct pt_regs *regs = args->regs;
|
| 2180 | unsigned long ip;
|
| 2181 |
|
| 2182 | OPTIMIZER_HIDE_VAR(selftest);
|
| 2183 |
|
| 2184 | if (!regs || user_mode(regs))
|
| 2185 | return NOTIFY_DONE;
|
| 2186 |
|
| 2187 | if (val != DIE_INT3)
|
| 2188 | return NOTIFY_DONE;
|
| 2189 |
|
| 2190 | if (regs->ip - INT3_INSN_SIZE != selftest)
|
| 2191 | return NOTIFY_DONE;
|
| 2192 |
|
| 2193 | ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
|
| 2194 | int3_emulate_call(regs, ip, (unsigned long)&int3_selftest_callee);
|
| 2195 | return NOTIFY_STOP;
|
| 2196 | }
|
| 2197 |
|
| 2198 |
|
| 2199 | static noinline void __init int3_selftest(void)
|
| 2200 | {
|
| 2201 | static __initdata struct notifier_block int3_exception_nb = {
|
| 2202 | .notifier_call = int3_exception_notify,
|
| 2203 | .priority = INT_MAX-1,
|
| 2204 | };
|
| 2205 | unsigned int val = 0;
|
| 2206 |
|
| 2207 | BUG_ON(register_die_notifier(&int3_exception_nb));
|
| 2208 |
|
| 2209 | |
| 2210 | * Basically: int3_selftest_callee(&val); but really complicated :-)
|
| 2211 |
|
| 2212 | int3_selftest_asm(&val);
|
| 2213 |
|
| 2214 | BUG_ON(val != 0x1234);
|
| 2215 |
|
| 2216 | unregister_die_notifier(&int3_exception_nb);
|
| 2217 | }
|
| 2218 |
|
| 2219 | static __initdata int __alt_reloc_selftest_addr;
|
| 2220 |
|
| 2221 | extern void __init __alt_reloc_selftest(void *arg);
|
| 2222 | __visible noinline void __init __alt_reloc_selftest(void *arg)
|
| 2223 | {
|
| 2224 | WARN_ON(arg != &__alt_reloc_selftest_addr);
|
| 2225 | }
|
| 2226 |
|
| 2227 | static noinline void __init alt_reloc_selftest(void)
|
| 2228 | {
|
| 2229 | |
| 2230 | * Tests text_poke_apply_relocation().
|
| 2231 | *
|
| 2232 | * This has a relative immediate (CALL) in a place other than the first
|
| 2233 | * instruction and additionally on x86_64 we get a RIP-relative LEA:
|
| 2234 | *
|
| 2235 | * lea 0x0(%rip),%rdi # 5d0: R_X86_64_PC32 .init.data+0x5566c
|
| 2236 | * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4
|
| 2237 | *
|
| 2238 | * Getting this wrong will either crash and burn or tickle the WARN
|
| 2239 | * above.
|
| 2240 |
|
| 2241 | asm_inline volatile (
|
| 2242 | ALTERNATIVE("", "lea %[mem], %%" _ASM_ARG1 "; call __alt_reloc_selftest;", X86_FEATURE_ALWAYS)
|
| 2243 | : ASM_CALL_CONSTRAINT
|
| 2244 | : [mem] "m" (__alt_reloc_selftest_addr)
|
| 2245 | : _ASM_ARG1
|
| 2246 | );
|
| 2247 | }
|
| 2248 |
|
| 2249 | void __init alternative_instructions(void)
|
| 2250 | {
|
| 2251 | u64 ibt;
|
| 2252 |
|
| 2253 | int3_selftest();
|
| 2254 |
|
| 2255 | |
| 2256 | * The patching is not fully atomic, so try to avoid local
|
| 2257 | * interruptions that might execute the to be patched code.
|
| 2258 | * Other CPUs are not running.
|
| 2259 |
|
| 2260 | stop_nmi();
|
| 2261 |
|
| 2262 | |
| 2263 | * Don't stop machine check exceptions while patching.
|
| 2264 | * MCEs only happen when something got corrupted and in this
|
| 2265 | * case we must do something about the corruption.
|
| 2266 | * Ignoring it is worse than an unlikely patching race.
|
| 2267 | * Also machine checks tend to be broadcast and if one CPU
|
| 2268 | * goes into machine check the others follow quickly, so we don't
|
| 2269 | * expect a machine check to cause undue problems during to code
|
| 2270 | * patching.
|
| 2271 |
|
| 2272 |
|
| 2273 | |
| 2274 | * Make sure to set (artificial) features depending on used paravirt
|
| 2275 | * functions which can later influence alternative patching.
|
| 2276 |
|
| 2277 | paravirt_set_cap();
|
| 2278 |
|
| 2279 |
|
| 2280 | ibt = ibt_save( true);
|
| 2281 |
|
| 2282 | __apply_fineibt(__retpoline_sites, __retpoline_sites_end,
|
| 2283 | __cfi_sites, __cfi_sites_end, true);
|
| 2284 | cfi_debug = false;
|
| 2285 |
|
| 2286 | |
| 2287 | * Rewrite the retpolines, must be done before alternatives since
|
| 2288 | * those can rewrite the retpoline thunks.
|
| 2289 |
|
| 2290 | apply_retpolines(__retpoline_sites, __retpoline_sites_end);
|
| 2291 | apply_returns(__return_sites, __return_sites_end);
|
| 2292 |
|
| 2293 | its_fini_core();
|
| 2294 |
|
| 2295 | |
| 2296 | * Adjust all CALL instructions to point to func()-10, including
|
| 2297 | * those in .altinstr_replacement.
|
| 2298 |
|
| 2299 | callthunks_patch_builtin_calls();
|
| 2300 |
|
| 2301 | apply_alternatives(__alt_instructions, __alt_instructions_end);
|
| 2302 |
|
| 2303 | |
| 2304 | * Seal all functions that do not have their address taken.
|
| 2305 |
|
| 2306 | apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
|
| 2307 |
|
| 2308 | ibt_restore(ibt);
|
| 2309 |
|
| 2310 | restart_nmi();
|
| 2311 | alternatives_patched = 1;
|
| 2312 |
|
| 2313 | alt_reloc_selftest();
|
| 2314 | }
|
| 2315 |
|
| 2316 | |
| 2317 | * text_poke_early - Update instructions on a live kernel at boot time
|
| 2318 | * @addr: address to modify
|
| 2319 | * @opcode: source of the copy
|
| 2320 | * @len: length to copy
|
| 2321 | *
|
| 2322 | * When you use this code to patch more than one byte of an instruction
|
| 2323 | * you need to make sure that other CPUs cannot execute this code in parallel.
|
| 2324 | * Also no thread must be currently preempted in the middle of these
|
| 2325 | * instructions. And on the local CPU you need to be protected against NMI or
|
| 2326 | * MCE handlers seeing an inconsistent instruction while you patch.
|
| 2327 |
|
| 2328 | void __init_or_module text_poke_early(void *addr, const void *opcode,
|
| 2329 | size_t len)
|
| 2330 | {
|
| 2331 | unsigned long flags;
|
| 2332 |
|
| 2333 | if (boot_cpu_has(X86_FEATURE_NX) &&
|
| 2334 | is_module_text_address((unsigned long)addr)) {
|
| 2335 | |
| 2336 | * Modules text is marked initially as non-executable, so the
|
| 2337 | * code cannot be running and speculative code-fetches are
|
| 2338 | * prevented. Just change the code.
|
| 2339 |
|
| 2340 | memcpy(addr, opcode, len);
|
| 2341 | } else {
|
| 2342 | local_irq_save(flags);
|
| 2343 | memcpy(addr, opcode, len);
|
| 2344 | sync_core();
|
| 2345 | local_irq_restore(flags);
|
| 2346 |
|
| 2347 | |
| 2348 | * Could also do a CLFLUSH here to speed up CPU recovery; but
|
| 2349 | * that causes hangs on some VIA CPUs.
|
| 2350 |
|
| 2351 | }
|
| 2352 | }
|
| 2353 |
|
| 2354 | __ro_after_init struct mm_struct *text_poke_mm;
|
| 2355 | __ro_after_init unsigned long text_poke_mm_addr;
|
| 2356 |
|
| 2357 | |
| 2358 | * Text poking creates and uses a mapping in the lower half of the
|
| 2359 | * address space. Relax LASS enforcement when accessing the poking
|
| 2360 | * address.
|
| 2361 | *
|
| 2362 | * objtool enforces a strict policy of "no function calls within AC=1
|
| 2363 | * regions". Adhere to the policy by using inline versions of
|
| 2364 | * memcpy()/memset() that will never result in a function call.
|
| 2365 |
|
| 2366 |
|
| 2367 | static void text_poke_memcpy(void *dst, const void *src, size_t len)
|
| 2368 | {
|
| 2369 | lass_stac();
|
| 2370 | __inline_memcpy(dst, src, len);
|
| 2371 | lass_clac();
|
| 2372 | }
|
| 2373 |
|
| 2374 | static void text_poke_memset(void *dst, const void *src, size_t len)
|
| 2375 | {
|
| 2376 | int c = *(const int *)src;
|
| 2377 |
|
| 2378 | lass_stac();
|
| 2379 | __inline_memset(dst, c, len);
|
| 2380 | lass_clac();
|
| 2381 | }
|
| 2382 |
|
| 2383 | typedef void text_poke_f(void *dst, const void *src, size_t len);
|
| 2384 |
|
| 2385 | static void __poke_vmalloc_pages(struct page **pages, void *addr,
|
| 2386 | bool cross_page_boundary)
|
| 2387 | {
|
| 2388 | pages[0] = vmalloc_to_page(addr);
|
| 2389 | if (cross_page_boundary)
|
| 2390 | pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
|
| 2391 | }
|
| 2392 |
|
| 2393 | static void poke_vmalloc_pages(struct page **pages, void *addr,
|
| 2394 | bool cross_page_boundary)
|
| 2395 | {
|
| 2396 | if (in_dbg_master()) {
|
| 2397 | |
| 2398 | * If called from kgdb cannot sleep, but all other CPUs stopped
|
| 2399 | * anyway so safe to proceed without locks
|
| 2400 |
|
| 2401 | __poke_vmalloc_pages(pages, addr, cross_page_boundary);
|
| 2402 | } else {
|
| 2403 | |
| 2404 | * execmem ROX ranges are shared between modules and can be
|
| 2405 | * collapsed to huge PMD entries, and this collapse can happen
|
| 2406 | * concurrently with a racing set_memory_rox().
|
| 2407 | *
|
| 2408 | * Prevent vmalloc_to_page() from racing by acquiring an
|
| 2409 | * init_mm read lock which pairs with the init_mm write lock in
|
| 2410 | * cpa_collapse_large_pages().
|
| 2411 |
|
| 2412 | guard(mmap_read_lock)(&init_mm);
|
| 2413 | __poke_vmalloc_pages(pages, addr, cross_page_boundary);
|
| 2414 | }
|
| 2415 | }
|
| 2416 |
|
| 2417 | static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len)
|
| 2418 | {
|
| 2419 | bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
|
| 2420 | struct page *pages[2] = {NULL};
|
| 2421 | struct mm_struct *prev_mm;
|
| 2422 | unsigned long flags;
|
| 2423 | pte_t pte, *ptep;
|
| 2424 | spinlock_t *ptl;
|
| 2425 | pgprot_t pgprot;
|
| 2426 |
|
| 2427 | |
| 2428 | * While boot memory allocator is running we cannot use struct pages as
|
| 2429 | * they are not yet initialized. There is no way to recover.
|
| 2430 |
|
| 2431 | BUG_ON(!after_bootmem);
|
| 2432 |
|
| 2433 | if (!core_kernel_text((unsigned long)addr)) {
|
| 2434 | poke_vmalloc_pages(pages, addr, cross_page_boundary);
|
| 2435 | } else {
|
| 2436 | pages[0] = virt_to_page(addr);
|
| 2437 | WARN_ON(!PageReserved(pages[0]));
|
| 2438 | if (cross_page_boundary)
|
| 2439 | pages[1] = virt_to_page(addr + PAGE_SIZE);
|
| 2440 | }
|
| 2441 | |
| 2442 | * If something went wrong, crash and burn since recovery paths are not
|
| 2443 | * implemented.
|
| 2444 |
|
| 2445 | BUG_ON(!pages[0] || (cross_page_boundary && !pages[1]));
|
| 2446 |
|
| 2447 | |
| 2448 | * Map the page without the global bit, as TLB flushing is done with
|
| 2449 | * flush_tlb_mm_range(), which is intended for non-global PTEs.
|
| 2450 |
|
| 2451 | pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL);
|
| 2452 |
|
| 2453 | |
| 2454 | * The lock is not really needed, but this allows to avoid open-coding.
|
| 2455 |
|
| 2456 | ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl);
|
| 2457 |
|
| 2458 | |
| 2459 | * This must not fail; preallocated in poking_init().
|
| 2460 |
|
| 2461 | VM_BUG_ON(!ptep);
|
| 2462 |
|
| 2463 | local_irq_save(flags);
|
| 2464 |
|
| 2465 | pte = mk_pte(pages[0], pgprot);
|
| 2466 | set_pte_at(text_poke_mm, text_poke_mm_addr, ptep, pte);
|
| 2467 |
|
| 2468 | if (cross_page_boundary) {
|
| 2469 | pte = mk_pte(pages[1], pgprot);
|
| 2470 | set_pte_at(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1, pte);
|
| 2471 | }
|
| 2472 |
|
| 2473 | |
| 2474 | * Loading the temporary mm behaves as a compiler barrier, which
|
| 2475 | * guarantees that the PTE will be set at the time memcpy() is done.
|
| 2476 |
|
| 2477 | prev_mm = use_temporary_mm(text_poke_mm);
|
| 2478 |
|
| 2479 | kasan_disable_current();
|
| 2480 | func((u8 *)text_poke_mm_addr + offset_in_page(addr), src, len);
|
| 2481 | kasan_enable_current();
|
| 2482 |
|
| 2483 | |
| 2484 | * Ensure that the PTE is only cleared after the instructions of memcpy
|
| 2485 | * were issued by using a compiler barrier.
|
| 2486 |
|
| 2487 | barrier();
|
| 2488 |
|
| 2489 | pte_clear(text_poke_mm, text_poke_mm_addr, ptep);
|
| 2490 | if (cross_page_boundary)
|
| 2491 | pte_clear(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1);
|
| 2492 |
|
| 2493 | |
| 2494 | * Loading the previous page-table hierarchy requires a serializing
|
| 2495 | * instruction that already allows the core to see the updated version.
|
| 2496 | * Xen-PV is assumed to serialize execution in a similar manner.
|
| 2497 |
|
| 2498 | unuse_temporary_mm(prev_mm);
|
| 2499 |
|
| 2500 | |
| 2501 | * Flushing the TLB might involve IPIs, which would require enabled
|
| 2502 | * IRQs, but not if the mm is not used, as it is in this point.
|
| 2503 |
|
| 2504 | flush_tlb_mm_range(text_poke_mm, text_poke_mm_addr, text_poke_mm_addr +
|
| 2505 | (cross_page_boundary ? 2 : 1) * PAGE_SIZE,
|
| 2506 | PAGE_SHIFT, false);
|
| 2507 |
|
| 2508 | if (func == text_poke_memcpy) {
|
| 2509 | |
| 2510 | * If the text does not match what we just wrote then something is
|
| 2511 | * fundamentally screwy; there's nothing we can really do about that.
|
| 2512 |
|
| 2513 | BUG_ON(memcmp(addr, src, len));
|
| 2514 | }
|
| 2515 |
|
| 2516 | local_irq_restore(flags);
|
| 2517 | pte_unmap_unlock(ptep, ptl);
|
| 2518 | return addr;
|
| 2519 | }
|
| 2520 |
|
| 2521 | |
| 2522 | * text_poke - Update instructions on a live kernel
|
| 2523 | * @addr: address to modify
|
| 2524 | * @opcode: source of the copy
|
| 2525 | * @len: length to copy
|
| 2526 | *
|
| 2527 | * Only atomic text poke/set should be allowed when not doing early patching.
|
| 2528 | * It means the size must be writable atomically and the address must be aligned
|
| 2529 | * in a way that permits an atomic write. It also makes sure we fit on a single
|
| 2530 | * page.
|
| 2531 | *
|
| 2532 | * Note that the caller must ensure that if the modified code is part of a
|
| 2533 | * module, the module would not be removed during poking. This can be achieved
|
| 2534 | * by registering a module notifier, and ordering module removal and patching
|
| 2535 | * through a mutex.
|
| 2536 |
|
| 2537 | void *text_poke(void *addr, const void *opcode, size_t len)
|
| 2538 | {
|
| 2539 | lockdep_assert_held(&text_mutex);
|
| 2540 |
|
| 2541 | return __text_poke(text_poke_memcpy, addr, opcode, len);
|
| 2542 | }
|
| 2543 |
|
| 2544 | |
| 2545 | * text_poke_kgdb - Update instructions on a live kernel by kgdb
|
| 2546 | * @addr: address to modify
|
| 2547 | * @opcode: source of the copy
|
| 2548 | * @len: length to copy
|
| 2549 | *
|
| 2550 | * Only atomic text poke/set should be allowed when not doing early patching.
|
| 2551 | * It means the size must be writable atomically and the address must be aligned
|
| 2552 | * in a way that permits an atomic write. It also makes sure we fit on a single
|
| 2553 | * page.
|
| 2554 | *
|
| 2555 | * Context: should only be used by kgdb, which ensures no other core is running,
|
| 2556 | * despite the fact it does not hold the text_mutex.
|
| 2557 |
|
| 2558 | void *text_poke_kgdb(void *addr, const void *opcode, size_t len)
|
| 2559 | {
|
| 2560 | return __text_poke(text_poke_memcpy, addr, opcode, len);
|
| 2561 | }
|
| 2562 |
|
| 2563 | void *text_poke_copy_locked(void *addr, const void *opcode, size_t len,
|
| 2564 | bool core_ok)
|
| 2565 | {
|
| 2566 | unsigned long start = (unsigned long)addr;
|
| 2567 | size_t patched = 0;
|
| 2568 |
|
| 2569 | if (WARN_ON_ONCE(!core_ok && core_kernel_text(start)))
|
| 2570 | return NULL;
|
| 2571 |
|
| 2572 | while (patched < len) {
|
| 2573 | unsigned long ptr = start + patched;
|
| 2574 | size_t s;
|
| 2575 |
|
| 2576 | s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);
|
| 2577 |
|
| 2578 | __text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s);
|
| 2579 | patched += s;
|
| 2580 | }
|
| 2581 | return addr;
|
| 2582 | }
|
| 2583 |
|
| 2584 | |
| 2585 | * text_poke_copy - Copy instructions into (an unused part of) RX memory
|
| 2586 | * @addr: address to modify
|
| 2587 | * @opcode: source of the copy
|
| 2588 | * @len: length to copy, could be more than 2x PAGE_SIZE
|
| 2589 | *
|
| 2590 | * Not safe against concurrent execution; useful for JITs to dump
|
| 2591 | * new code blocks into unused regions of RX memory. Can be used in
|
| 2592 | * conjunction with synchronize_rcu_tasks() to wait for existing
|
| 2593 | * execution to quiesce after having made sure no existing functions
|
| 2594 | * pointers are live.
|
| 2595 |
|
| 2596 | void *text_poke_copy(void *addr, const void *opcode, size_t len)
|
| 2597 | {
|
| 2598 | mutex_lock(&text_mutex);
|
| 2599 | addr = text_poke_copy_locked(addr, opcode, len, false);
|
| 2600 | mutex_unlock(&text_mutex);
|
| 2601 | return addr;
|
| 2602 | }
|
| 2603 |
|
| 2604 | |
| 2605 | * text_poke_set - memset into (an unused part of) RX memory
|
| 2606 | * @addr: address to modify
|
| 2607 | * @c: the byte to fill the area with
|
| 2608 | * @len: length to copy, could be more than 2x PAGE_SIZE
|
| 2609 | *
|
| 2610 | * This is useful to overwrite unused regions of RX memory with illegal
|
| 2611 | * instructions.
|
| 2612 |
|
| 2613 | void *text_poke_set(void *addr, int c, size_t len)
|
| 2614 | {
|
| 2615 | unsigned long start = (unsigned long)addr;
|
| 2616 | size_t patched = 0;
|
| 2617 |
|
| 2618 | if (WARN_ON_ONCE(core_kernel_text(start)))
|
| 2619 | return NULL;
|
| 2620 |
|
| 2621 | mutex_lock(&text_mutex);
|
| 2622 | while (patched < len) {
|
| 2623 | unsigned long ptr = start + patched;
|
| 2624 | size_t s;
|
| 2625 |
|
| 2626 | s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);
|
| 2627 |
|
| 2628 | __text_poke(text_poke_memset, (void *)ptr, (void *)&c, s);
|
| 2629 | patched += s;
|
| 2630 | }
|
| 2631 | mutex_unlock(&text_mutex);
|
| 2632 | return addr;
|
| 2633 | }
|
| 2634 |
|
| 2635 | static void do_sync_core(void *info)
|
| 2636 | {
|
| 2637 | sync_core();
|
| 2638 | }
|
| 2639 |
|
| 2640 | void smp_text_poke_sync_each_cpu(void)
|
| 2641 | {
|
| 2642 | on_each_cpu(do_sync_core, NULL, 1);
|
| 2643 | }
|
| 2644 |
|
| 2645 | |
| 2646 | * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of
|
| 2647 | * this thing. When len == 6 everything is prefixed with 0x0f and we map
|
| 2648 | * opcode to Jcc.d8, using len to distinguish.
|
| 2649 |
|
| 2650 | struct smp_text_poke_loc {
|
| 2651 |
|
| 2652 | s32 rel_addr;
|
| 2653 | s32 disp;
|
| 2654 | u8 len;
|
| 2655 | u8 opcode;
|
| 2656 | const u8 text[TEXT_POKE_MAX_OPCODE_SIZE];
|
| 2657 |
|
| 2658 | u8 old;
|
| 2659 | };
|
| 2660 |
|
| 2661 | #define TEXT_POKE_ARRAY_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc))
|
| 2662 |
|
| 2663 | static struct smp_text_poke_array {
|
| 2664 | struct smp_text_poke_loc vec[TEXT_POKE_ARRAY_MAX];
|
| 2665 | int nr_entries;
|
| 2666 | } text_poke_array;
|
| 2667 |
|
| 2668 | static DEFINE_PER_CPU(atomic_t, text_poke_array_refs);
|
| 2669 |
|
| 2670 | |
| 2671 | * These four __always_inline annotations imply noinstr, necessary
|
| 2672 | * due to smp_text_poke_int3_handler() being noinstr:
|
| 2673 |
|
| 2674 |
|
| 2675 | static __always_inline bool try_get_text_poke_array(void)
|
| 2676 | {
|
| 2677 | atomic_t *refs = this_cpu_ptr(&text_poke_array_refs);
|
| 2678 |
|
| 2679 | if (!raw_atomic_inc_not_zero(refs))
|
| 2680 | return false;
|
| 2681 |
|
| 2682 | return true;
|
| 2683 | }
|
| 2684 |
|
| 2685 | static __always_inline void put_text_poke_array(void)
|
| 2686 | {
|
| 2687 | atomic_t *refs = this_cpu_ptr(&text_poke_array_refs);
|
| 2688 |
|
| 2689 | smp_mb__before_atomic();
|
| 2690 | raw_atomic_dec(refs);
|
| 2691 | }
|
| 2692 |
|
| 2693 | static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tpl)
|
| 2694 | {
|
| 2695 | return _stext + tpl->rel_addr;
|
| 2696 | }
|
| 2697 |
|
| 2698 | static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b)
|
| 2699 | {
|
| 2700 | if (tpl_a < text_poke_addr(tpl_b))
|
| 2701 | return -1;
|
| 2702 | if (tpl_a > text_poke_addr(tpl_b))
|
| 2703 | return 1;
|
| 2704 | return 0;
|
| 2705 | }
|
| 2706 |
|
| 2707 | noinstr int smp_text_poke_int3_handler(struct pt_regs *regs)
|
| 2708 | {
|
| 2709 | struct smp_text_poke_loc *tpl;
|
| 2710 | int ret = 0;
|
| 2711 | void *ip;
|
| 2712 |
|
| 2713 | if (user_mode(regs))
|
| 2714 | return 0;
|
| 2715 |
|
| 2716 | |
| 2717 | * Having observed our INT3 instruction, we now must observe
|
| 2718 | * text_poke_array with non-zero refcount:
|
| 2719 | *
|
| 2720 | * text_poke_array_refs = 1 INT3
|
| 2721 | * WMB RMB
|
| 2722 | * write INT3 if (text_poke_array_refs != 0)
|
| 2723 |
|
| 2724 | smp_rmb();
|
| 2725 |
|
| 2726 | if (!try_get_text_poke_array())
|
| 2727 | return 0;
|
| 2728 |
|
| 2729 | |
| 2730 | * Discount the INT3. See smp_text_poke_batch_finish().
|
| 2731 |
|
| 2732 | ip = (void *) regs->ip - INT3_INSN_SIZE;
|
| 2733 |
|
| 2734 | |
| 2735 | * Skip the binary search if there is a single member in the vector.
|
| 2736 |
|
| 2737 | if (unlikely(text_poke_array.nr_entries > 1)) {
|
| 2738 | tpl = __inline_bsearch(ip, text_poke_array.vec, text_poke_array.nr_entries,
|
| 2739 | sizeof(struct smp_text_poke_loc),
|
| 2740 | patch_cmp);
|
| 2741 | if (!tpl)
|
| 2742 | goto out_put;
|
| 2743 | } else {
|
| 2744 | tpl = text_poke_array.vec;
|
| 2745 | if (text_poke_addr(tpl) != ip)
|
| 2746 | goto out_put;
|
| 2747 | }
|
| 2748 |
|
| 2749 | ip += tpl->len;
|
| 2750 |
|
| 2751 | switch (tpl->opcode) {
|
| 2752 | case INT3_INSN_OPCODE:
|
| 2753 | |
| 2754 | * Someone poked an explicit INT3, they'll want to handle it,
|
| 2755 | * do not consume.
|
| 2756 |
|
| 2757 | goto out_put;
|
| 2758 |
|
| 2759 | case RET_INSN_OPCODE:
|
| 2760 | int3_emulate_ret(regs);
|
| 2761 | break;
|
| 2762 |
|
| 2763 | case CALL_INSN_OPCODE:
|
| 2764 | int3_emulate_call(regs, (long)ip, (long)ip + tpl->disp);
|
| 2765 | break;
|
| 2766 |
|
| 2767 | case JMP32_INSN_OPCODE:
|
| 2768 | case JMP8_INSN_OPCODE:
|
| 2769 | int3_emulate_jmp(regs, (long)ip + tpl->disp);
|
| 2770 | break;
|
| 2771 |
|
| 2772 | case 0x70 ... 0x7f:
|
| 2773 | int3_emulate_jcc(regs, tpl->opcode & 0xf, (long)ip, tpl->disp);
|
| 2774 | break;
|
| 2775 |
|
| 2776 | default:
|
| 2777 | BUG();
|
| 2778 | }
|
| 2779 |
|
| 2780 | ret = 1;
|
| 2781 |
|
| 2782 | out_put:
|
| 2783 | put_text_poke_array();
|
| 2784 | return ret;
|
| 2785 | }
|
| 2786 |
|
| 2787 | |
| 2788 | * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP
|
| 2789 | *
|
| 2790 | * Input state:
|
| 2791 | * text_poke_array.vec: vector of instructions to patch
|
| 2792 | * text_poke_array.nr_entries: number of entries in the vector
|
| 2793 | *
|
| 2794 | * Modify multi-byte instructions by using INT3 breakpoints on SMP.
|
| 2795 | * We completely avoid using stop_machine() here, and achieve the
|
| 2796 | * synchronization using INT3 breakpoints and SMP cross-calls.
|
| 2797 | *
|
| 2798 | * The way it is done:
|
| 2799 | * - For each entry in the vector:
|
| 2800 | * - add an INT3 trap to the address that will be patched
|
| 2801 | * - SMP sync all CPUs
|
| 2802 | * - For each entry in the vector:
|
| 2803 | * - update all but the first byte of the patched range
|
| 2804 | * - SMP sync all CPUs
|
| 2805 | * - For each entry in the vector:
|
| 2806 | * - replace the first byte (INT3) by the first byte of the
|
| 2807 | * replacing opcode
|
| 2808 | * - SMP sync all CPUs
|
| 2809 |
|
| 2810 | void smp_text_poke_batch_finish(void)
|
| 2811 | {
|
| 2812 | unsigned char int3 = INT3_INSN_OPCODE;
|
| 2813 | unsigned int i;
|
| 2814 | int do_sync;
|
| 2815 |
|
| 2816 | if (!text_poke_array.nr_entries)
|
| 2817 | return;
|
| 2818 |
|
| 2819 | lockdep_assert_held(&text_mutex);
|
| 2820 |
|
| 2821 | |
| 2822 | * Corresponds to the implicit memory barrier in try_get_text_poke_array() to
|
| 2823 | * ensure reading a non-zero refcount provides up to date text_poke_array data.
|
| 2824 |
|
| 2825 | for_each_possible_cpu(i)
|
| 2826 | atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1);
|
| 2827 |
|
| 2828 | |
| 2829 | * Function tracing can enable thousands of places that need to be
|
| 2830 | * updated. This can take quite some time, and with full kernel debugging
|
| 2831 | * enabled, this could cause the softlockup watchdog to trigger.
|
| 2832 | * This function gets called every 256 entries added to be patched.
|
| 2833 | * Call cond_resched() here to make sure that other tasks can get scheduled
|
| 2834 | * while processing all the functions being patched.
|
| 2835 |
|
| 2836 | cond_resched();
|
| 2837 |
|
| 2838 | |
| 2839 | * Corresponding read barrier in INT3 notifier for making sure the
|
| 2840 | * text_poke_array.nr_entries and handler are correctly ordered wrt. patching.
|
| 2841 |
|
| 2842 | smp_wmb();
|
| 2843 |
|
| 2844 | |
| 2845 | * First step: add a INT3 trap to the address that will be patched.
|
| 2846 |
|
| 2847 | for (i = 0; i < text_poke_array.nr_entries; i++) {
|
| 2848 | text_poke_array.vec[i].old = *(u8 *)text_poke_addr(&text_poke_array.vec[i]);
|
| 2849 | text_poke(text_poke_addr(&text_poke_array.vec[i]), &int3, INT3_INSN_SIZE);
|
| 2850 | }
|
| 2851 |
|
| 2852 | smp_text_poke_sync_each_cpu();
|
| 2853 |
|
| 2854 | |
| 2855 | * Second step: update all but the first byte of the patched range.
|
| 2856 |
|
| 2857 | for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) {
|
| 2858 | u8 old[TEXT_POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, };
|
| 2859 | u8 _new[TEXT_POKE_MAX_OPCODE_SIZE+1];
|
| 2860 | const u8 *new = text_poke_array.vec[i].text;
|
| 2861 | int len = text_poke_array.vec[i].len;
|
| 2862 |
|
| 2863 | if (len - INT3_INSN_SIZE > 0) {
|
| 2864 | memcpy(old + INT3_INSN_SIZE,
|
| 2865 | text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE,
|
| 2866 | len - INT3_INSN_SIZE);
|
| 2867 |
|
| 2868 | if (len == 6) {
|
| 2869 | _new[0] = 0x0f;
|
| 2870 | memcpy(_new + 1, new, 5);
|
| 2871 | new = _new;
|
| 2872 | }
|
| 2873 |
|
| 2874 | text_poke(text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE,
|
| 2875 | new + INT3_INSN_SIZE,
|
| 2876 | len - INT3_INSN_SIZE);
|
| 2877 |
|
| 2878 | do_sync++;
|
| 2879 | }
|
| 2880 |
|
| 2881 | |
| 2882 | * Emit a perf event to record the text poke, primarily to
|
| 2883 | * support Intel PT decoding which must walk the executable code
|
| 2884 | * to reconstruct the trace. The flow up to here is:
|
| 2885 | * - write INT3 byte
|
| 2886 | * - IPI-SYNC
|
| 2887 | * - write instruction tail
|
| 2888 | * At this point the actual control flow will be through the
|
| 2889 | * INT3 and handler and not hit the old or new instruction.
|
| 2890 | * Intel PT outputs FUP/TIP packets for the INT3, so the flow
|
| 2891 | * can still be decoded. Subsequently:
|
| 2892 | * - emit RECORD_TEXT_POKE with the new instruction
|
| 2893 | * - IPI-SYNC
|
| 2894 | * - write first byte
|
| 2895 | * - IPI-SYNC
|
| 2896 | * So before the text poke event timestamp, the decoder will see
|
| 2897 | * either the old instruction flow or FUP/TIP of INT3. After the
|
| 2898 | * text poke event timestamp, the decoder will see either the
|
| 2899 | * new instruction flow or FUP/TIP of INT3. Thus decoders can
|
| 2900 | * use the timestamp as the point at which to modify the
|
| 2901 | * executable code.
|
| 2902 | * The old instruction is recorded so that the event can be
|
| 2903 | * processed forwards or backwards.
|
| 2904 |
|
| 2905 | perf_event_text_poke(text_poke_addr(&text_poke_array.vec[i]), old, len, new, len);
|
| 2906 | }
|
| 2907 |
|
| 2908 | if (do_sync) {
|
| 2909 | |
| 2910 | * According to Intel, this core syncing is very likely
|
| 2911 | * not necessary and we'd be safe even without it. But
|
| 2912 | * better safe than sorry (plus there's not only Intel).
|
| 2913 |
|
| 2914 | smp_text_poke_sync_each_cpu();
|
| 2915 | }
|
| 2916 |
|
| 2917 | |
| 2918 | * Third step: replace the first byte (INT3) by the first byte of the
|
| 2919 | * replacing opcode.
|
| 2920 |
|
| 2921 | for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) {
|
| 2922 | u8 byte = text_poke_array.vec[i].text[0];
|
| 2923 |
|
| 2924 | if (text_poke_array.vec[i].len == 6)
|
| 2925 | byte = 0x0f;
|
| 2926 |
|
| 2927 | if (byte == INT3_INSN_OPCODE)
|
| 2928 | continue;
|
| 2929 |
|
| 2930 | text_poke(text_poke_addr(&text_poke_array.vec[i]), &byte, INT3_INSN_SIZE);
|
| 2931 | do_sync++;
|
| 2932 | }
|
| 2933 |
|
| 2934 | if (do_sync)
|
| 2935 | smp_text_poke_sync_each_cpu();
|
| 2936 |
|
| 2937 | |
| 2938 | * Remove and wait for refs to be zero.
|
| 2939 | *
|
| 2940 | * Notably, if after step-3 above the INT3 got removed, then the
|
| 2941 | * smp_text_poke_sync_each_cpu() will have serialized against any running INT3
|
| 2942 | * handlers and the below spin-wait will not happen.
|
| 2943 | *
|
| 2944 | * IOW. unless the replacement instruction is INT3, this case goes
|
| 2945 | * unused.
|
| 2946 |
|
| 2947 | for_each_possible_cpu(i) {
|
| 2948 | atomic_t *refs = per_cpu_ptr(&text_poke_array_refs, i);
|
| 2949 |
|
| 2950 | if (unlikely(!atomic_dec_and_test(refs)))
|
| 2951 | atomic_cond_read_acquire(refs, !VAL);
|
| 2952 | }
|
| 2953 |
|
| 2954 |
|
| 2955 | text_poke_array.nr_entries = 0;
|
| 2956 | }
|
| 2957 |
|
| 2958 | static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate)
|
| 2959 | {
|
| 2960 | struct smp_text_poke_loc *tpl;
|
| 2961 | struct insn insn;
|
| 2962 | int ret, i = 0;
|
| 2963 |
|
| 2964 | tpl = &text_poke_array.vec[text_poke_array.nr_entries++];
|
| 2965 |
|
| 2966 | if (len == 6)
|
| 2967 | i = 1;
|
| 2968 | memcpy((void *)tpl->text, opcode+i, len-i);
|
| 2969 | if (!emulate)
|
| 2970 | emulate = opcode;
|
| 2971 |
|
| 2972 | ret = insn_decode_kernel(&insn, emulate);
|
| 2973 | BUG_ON(ret < 0);
|
| 2974 |
|
| 2975 | tpl->rel_addr = addr - (void *)_stext;
|
| 2976 | tpl->len = len;
|
| 2977 | tpl->opcode = insn.opcode.bytes[0];
|
| 2978 |
|
| 2979 | if (is_jcc32(&insn)) {
|
| 2980 | |
| 2981 | * Map Jcc.d32 onto Jcc.d8 and use len to distinguish.
|
| 2982 |
|
| 2983 | tpl->opcode = insn.opcode.bytes[1] - 0x10;
|
| 2984 | }
|
| 2985 |
|
| 2986 | switch (tpl->opcode) {
|
| 2987 | case RET_INSN_OPCODE:
|
| 2988 | case JMP32_INSN_OPCODE:
|
| 2989 | case JMP8_INSN_OPCODE:
|
| 2990 | |
| 2991 | * Control flow instructions without implied execution of the
|
| 2992 | * next instruction can be padded with INT3.
|
| 2993 |
|
| 2994 | for (i = insn.length; i < len; i++)
|
| 2995 | BUG_ON(tpl->text[i] != INT3_INSN_OPCODE);
|
| 2996 | break;
|
| 2997 |
|
| 2998 | default:
|
| 2999 | BUG_ON(len != insn.length);
|
| 3000 | }
|
| 3001 |
|
| 3002 | switch (tpl->opcode) {
|
| 3003 | case INT3_INSN_OPCODE:
|
| 3004 | case RET_INSN_OPCODE:
|
| 3005 | break;
|
| 3006 |
|
| 3007 | case CALL_INSN_OPCODE:
|
| 3008 | case JMP32_INSN_OPCODE:
|
| 3009 | case JMP8_INSN_OPCODE:
|
| 3010 | case 0x70 ... 0x7f:
|
| 3011 | tpl->disp = insn.immediate.value;
|
| 3012 | break;
|
| 3013 |
|
| 3014 | default:
|
| 3015 | switch (len) {
|
| 3016 | case 2:
|
| 3017 | BUG_ON(memcmp(emulate, x86_nops[len], len));
|
| 3018 | tpl->opcode = JMP8_INSN_OPCODE;
|
| 3019 | tpl->disp = 0;
|
| 3020 | break;
|
| 3021 |
|
| 3022 | case 5:
|
| 3023 | BUG_ON(memcmp(emulate, x86_nops[len], len));
|
| 3024 | tpl->opcode = JMP32_INSN_OPCODE;
|
| 3025 | tpl->disp = 0;
|
| 3026 | break;
|
| 3027 |
|
| 3028 | default:
|
| 3029 | BUG();
|
| 3030 | }
|
| 3031 | break;
|
| 3032 | }
|
| 3033 | }
|
| 3034 |
|
| 3035 | |
| 3036 | * We hard rely on the text_poke_array.vec being ordered; ensure this is so by flushing
|
| 3037 | * early if needed.
|
| 3038 |
|
| 3039 | static bool text_poke_addr_ordered(void *addr)
|
| 3040 | {
|
| 3041 | WARN_ON_ONCE(!addr);
|
| 3042 |
|
| 3043 | if (!text_poke_array.nr_entries)
|
| 3044 | return true;
|
| 3045 |
|
| 3046 | |
| 3047 | * If the last current entry's address is higher than the
|
| 3048 | * new entry's address we'd like to add, then ordering
|
| 3049 | * is violated and we must first flush all pending patching
|
| 3050 | * requests:
|
| 3051 |
|
| 3052 | if (text_poke_addr(text_poke_array.vec + text_poke_array.nr_entries-1) > addr)
|
| 3053 | return false;
|
| 3054 |
|
| 3055 | return true;
|
| 3056 | }
|
| 3057 |
|
| 3058 | |
| 3059 | * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched
|
| 3060 | * @addr: address to patch
|
| 3061 | * @opcode: opcode of new instruction
|
| 3062 | * @len: length to copy
|
| 3063 | * @emulate: instruction to be emulated
|
| 3064 | *
|
| 3065 | * Add a new instruction to the current queue of to-be-patched instructions
|
| 3066 | * the kernel maintains. The patching request will not be executed immediately,
|
| 3067 | * but becomes part of an array of patching requests, optimized for batched
|
| 3068 | * execution. All pending patching requests will be executed on the next
|
| 3069 | * smp_text_poke_batch_finish() call.
|
| 3070 |
|
| 3071 | void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate)
|
| 3072 | {
|
| 3073 | if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr))
|
| 3074 | smp_text_poke_batch_finish();
|
| 3075 | __smp_text_poke_batch_add(addr, opcode, len, emulate);
|
| 3076 | }
|
| 3077 |
|
| 3078 | |
| 3079 | * smp_text_poke_single() -- update instruction on live kernel on SMP immediately
|
| 3080 | * @addr: address to patch
|
| 3081 | * @opcode: opcode of new instruction
|
| 3082 | * @len: length to copy
|
| 3083 | * @emulate: instruction to be emulated
|
| 3084 | *
|
| 3085 | * Update a single instruction with the vector in the stack, avoiding
|
| 3086 | * dynamically allocated memory. This function should be used when it is
|
| 3087 | * not possible to allocate memory for a vector. The single instruction
|
| 3088 | * is patched in immediately.
|
| 3089 |
|
| 3090 | void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate)
|
| 3091 | {
|
| 3092 | smp_text_poke_batch_add(addr, opcode, len, emulate);
|
| 3093 | smp_text_poke_batch_finish();
|
| 3094 | }
|