| 1 | /* SPDX-License-Identifier: GPL-2.0 */
|
| 2 | /*
|
| 3 | * Common place for both 32- and 64-bit entry routines.
|
| 4 | */
|
| 5 |
|
| 6 | |
| 7 | |
| 8 | |
| 9 | |
| 10 | |
| 11 | |
| 12 | |
| 13 | |
| 14 | |
| 15 | |
| 16 |
|
| 17 | |
| 18 |
|
| 19 | .pushsection .noinstr.text, "ax"
|
| 20 |
|
| 21 | /* Clobbers AX, CX, DX */
|
| 22 | SYM_FUNC_START(write_ibpb)
|
| 23 | ANNOTATE_NOENDBR
|
| 24 | movl $MSR_IA32_PRED_CMD, %ecx
|
| 25 | movl _ASM_RIP(x86_pred_cmd), %eax
|
| 26 | xorl %edx, %edx
|
| 27 | wrmsr
|
| 28 |
|
| 29 | /* Make sure IBPB clears return stack preductions too. */
|
| 30 | FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_BUG_IBPB_NO_RET
|
| 31 | RET
|
| 32 | SYM_FUNC_END(write_ibpb)
|
| 33 | EXPORT_SYMBOL_FOR_KVM(write_ibpb);
|
| 34 |
|
| 35 | SYM_FUNC_START(__WARN_trap)
|
| 36 | ANNOTATE_NOENDBR
|
| 37 | ANNOTATE_REACHABLE
|
| 38 | ud1 (%edx), %_ASM_ARG1
|
| 39 | RET
|
| 40 | SYM_FUNC_END(__WARN_trap)
|
| 41 | EXPORT_SYMBOL(__WARN_trap)
|
| 42 |
|
| 43 | .popsection
|
| 44 |
|
| 45 | /*
|
| 46 | * Define the VERW operand that is disguised as entry code so that
|
| 47 | * it can be referenced with KPTI enabled. This ensures VERW can be
|
| 48 | * used late in exit-to-user path after page tables are switched.
|
| 49 | */
|
| 50 | .pushsection .entry.text, "ax"
|
| 51 |
|
| 52 | .align L1_CACHE_BYTES, 0xcc
|
| 53 | SYM_CODE_START_NOALIGN(x86_verw_sel)
|
| 54 | UNWIND_HINT_UNDEFINED
|
| 55 | ANNOTATE_NOENDBR
|
| 56 | .word __KERNEL_DS
|
| 57 | .align L1_CACHE_BYTES, 0xcc
|
| 58 | SYM_CODE_END(x86_verw_sel);
|
| 59 | EXPORT_SYMBOL_FOR_KVM(x86_verw_sel);
|
| 60 |
|
| 61 | .popsection
|
| 62 |
|
| 63 | THUNK warn_thunk_thunk, __warn_thunk
|
| 64 |
|
| 65 | /*
|
| 66 | * Clang's implementation of TLS stack cookies requires the variable in
|
| 67 | * question to be a TLS variable. If the variable happens to be defined as an
|
| 68 | * ordinary variable with external linkage in the same compilation unit (which
|
| 69 | * amounts to the whole of vmlinux with LTO enabled), Clang will drop the
|
| 70 | * segment register prefix from the references, resulting in broken code. Work
|
| 71 | * around this by avoiding the symbol used in -mstack-protector-guard-symbol=
|
| 72 | * entirely in the C code, and use an alias emitted by the linker script
|
| 73 | * instead.
|
| 74 | */
|
| 75 | #if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)
|
| 76 | EXPORT_SYMBOL(__ref_stack_chk_guard);
|
| 77 | #endif
|
| 78 |
|
| 79 | #if IS_ENABLED(CONFIG_KVM_INTEL)
|
| 80 | .macro IDT_DO_EVENT_IRQOFF call_insn call_target
|
| 81 | /*
|
| 82 | * Unconditionally create a stack frame, getting the correct RSP on the
|
| 83 | * stack (for x86-64) would take two instructions anyways, and RBP can
|
| 84 | * be used to restore RSP to make objtool happy (see below).
|
| 85 | */
|
| 86 | push %_ASM_BP
|
| 87 | mov %_ASM_SP, %_ASM_BP
|
| 88 |
|
| 89 | #ifdef CONFIG_X86_64
|
| 90 | /*
|
| 91 | * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
|
| 92 | * creating the synthetic interrupt stack frame for the IRQ/NMI.
|
| 93 | */
|
| 94 | and $-16, %rsp
|
| 95 | push $__KERNEL_DS
|
| 96 | push %rbp
|
| 97 | #endif
|
| 98 | pushf
|
| 99 | push $__KERNEL_CS
|
| 100 | \call_insn \call_target
|
| 101 |
|
| 102 | /*
|
| 103 | * "Restore" RSP from RBP, even though IRET has already unwound RSP to
|
| 104 | * the correct value. objtool doesn't know the callee will IRET and,
|
| 105 | * without the explicit restore, thinks the stack is getting walloped.
|
| 106 | * Using an unwind hint is problematic due to x86-64's dynamic alignment.
|
| 107 | */
|
| 108 | leave
|
| 109 | RET
|
| 110 | .endm
|
| 111 |
|
| 112 | #ifndef CONFIG_X86_64
|
| 113 | .pushsection .text, "ax"
|
| 114 | SYM_FUNC_START(idt_do_interrupt_irqoff)
|
| 115 | IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
|
| 116 | SYM_FUNC_END(idt_do_interrupt_irqoff)
|
| 117 | .popsection
|
| 118 | #endif
|
| 119 |
|
| 120 | .pushsection .noinstr.text, "ax"
|
| 121 | SYM_FUNC_START(idt_do_nmi_irqoff)
|
| 122 | IDT_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
|
| 123 | SYM_FUNC_END(idt_do_nmi_irqoff)
|
| 124 | .popsection
|
| 125 | #endif
|