projects/linux·blob master · refs

240 lines · 6.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * alternative runtime patching
4 * inspired by the ARM64 and x86 version
5 *
6 * Copyright (C) 2021 Sifive.
7 */
8
9#include <linux/init.h>
10#include <linux/module.h>
11#include <linux/cpu.h>
12#include <linux/uaccess.h>
13#include <asm/alternative.h>
14#include <asm/module.h>
15#include <asm/sections.h>
16#include <asm/vdso.h>
17#include <asm/vendorid_list.h>
18#include <asm/sbi.h>
19#include <asm/csr.h>
20#include <asm/insn.h>
21#include <asm/text-patching.h>
22
23struct cpu_manufacturer_info_t {
24 unsigned long vendor_id;
25 unsigned long arch_id;
26 unsigned long imp_id;
27 void (*patch_func)(struct alt_entry *begin, struct alt_entry *end,
28 unsigned long archid, unsigned long impid,
29 unsigned int stage);
30};
31
32static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info)
33{
34#ifdef CONFIG_RISCV_M_MODE
35 cpu_mfr_info->vendor_id = csr_read(CSR_MVENDORID);
36 cpu_mfr_info->arch_id = csr_read(CSR_MARCHID);
37 cpu_mfr_info->imp_id = csr_read(CSR_MIMPID);
38#else
39 cpu_mfr_info->vendor_id = sbi_get_mvendorid();
40 cpu_mfr_info->arch_id = sbi_get_marchid();
41 cpu_mfr_info->imp_id = sbi_get_mimpid();
42#endif
43
44 switch (cpu_mfr_info->vendor_id) {
45#ifdef CONFIG_ERRATA_ANDES
46 case ANDES_VENDOR_ID:
47 cpu_mfr_info->patch_func = andes_errata_patch_func;
48 break;
49#endif
50#ifdef CONFIG_ERRATA_MIPS
51 case MIPS_VENDOR_ID:
52 cpu_mfr_info->patch_func = mips_errata_patch_func;
53 break;
54#endif
55#ifdef CONFIG_ERRATA_SIFIVE
56 case SIFIVE_VENDOR_ID:
57 cpu_mfr_info->patch_func = sifive_errata_patch_func;
58 break;
59#endif
60#ifdef CONFIG_ERRATA_THEAD
61 case THEAD_VENDOR_ID:
62 cpu_mfr_info->patch_func = thead_errata_patch_func;
63 break;
64#endif
65 default:
66 cpu_mfr_info->patch_func = NULL;
67 }
68}
69
70static u32 riscv_instruction_at(void *p)
71{
72 u16 *parcel = p;
73
74 return (u32)parcel[0] | (u32)parcel[1] << 16;
75}
76
77static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,
78 u32 jalr_insn, int patch_offset)
79{
80 u32 call[2] = { auipc_insn, jalr_insn };
81 s32 imm;
82
83 /* get and adjust new target address */
84 imm = riscv_insn_extract_utype_itype_imm(auipc_insn, jalr_insn);
85 imm -= patch_offset;
86
87 /* update instructions */
88 riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm);
89
90 /* patch the call place again */
91 patch_text_nosync(ptr, call, sizeof(u32) * 2);
92}
93
94static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset)
95{
96 s32 imm;
97
98 /* get and adjust new target address */
99 imm = riscv_insn_extract_jtype_imm(jal_insn);
100 imm -= patch_offset;
101
102 /* update instruction */
103 riscv_insn_insert_jtype_imm(&jal_insn, imm);
104
105 /* patch the call place again */
106 patch_text_nosync(ptr, &jal_insn, sizeof(u32));
107}
108
109void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
110 int patch_offset)
111{
112 int num_insn = len / sizeof(u32);
113 int i;
114
115 for (i = 0; i < num_insn; i++) {
116 u32 insn = riscv_instruction_at(alt_ptr + i * sizeof(u32));
117
118 /*
119 * May be the start of an auipc + jalr pair
120 * Needs to check that at least one more instruction
121 * is in the list.
122 */
123 if (riscv_insn_is_auipc(insn) && i < num_insn - 1) {
124 u32 insn2 = riscv_instruction_at(alt_ptr + (i + 1) * sizeof(u32));
125
126 if (!riscv_insn_is_jalr(insn2))
127 continue;
128
129 /* if instruction pair is a call, it will use the ra register */
130 if (RV_EXTRACT_RD_REG(insn) != 1)
131 continue;
132
133 riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32),
134 insn, insn2, patch_offset);
135 i++;
136 }
137
138 if (riscv_insn_is_jal(insn)) {
139 s32 imm = riscv_insn_extract_jtype_imm(insn);
140
141 /* Don't modify jumps inside the alternative block */
142 if ((alt_ptr + i * sizeof(u32) + imm) >= alt_ptr &&
143 (alt_ptr + i * sizeof(u32) + imm) < (alt_ptr + len))
144 continue;
145
146 riscv_alternative_fix_jal(alt_ptr + i * sizeof(u32),
147 insn, patch_offset);
148 }
149 }
150}
151
152/*
153 * This is called very early in the boot process (directly after we run
154 * a feature detect on the boot CPU). No need to worry about other CPUs
155 * here.
156 */
157static void __init_or_module _apply_alternatives(struct alt_entry *begin,
158 struct alt_entry *end,
159 unsigned int stage)
160{
161 struct cpu_manufacturer_info_t cpu_mfr_info;
162
163 riscv_fill_cpu_mfr_info(&cpu_mfr_info);
164
165 riscv_cpufeature_patch_func(begin, end, stage);
166
167 if (!cpu_mfr_info.patch_func)
168 return;
169
170 cpu_mfr_info.patch_func(begin, end,
171 cpu_mfr_info.arch_id,
172 cpu_mfr_info.imp_id,
173 stage);
174}
175
176static __always_inline
177void __init apply_vdso_alternatives(void *base, size_t alternatives_begin, size_t alternatives_end)
178{
179 if (alternatives_begin == alternatives_end)
180 return;
181
182 _apply_alternatives(base + alternatives_begin,
183 base + alternatives_end,
184 RISCV_ALTERNATIVES_BOOT);
185}
186
187void __init apply_boot_alternatives(void)
188{
189 /* If called on non-boot cpu things could go wrong */
190 WARN_ON(smp_processor_id() != 0);
191
192 _apply_alternatives((struct alt_entry *)__alt_start,
193 (struct alt_entry *)__alt_end,
194 RISCV_ALTERNATIVES_BOOT);
195
196 if (IS_ENABLED(CONFIG_MMU))
197 apply_vdso_alternatives(vdso_start,
198 __vdso_alternatives_start_offset,
199 __vdso_alternatives_end_offset);
200
201 if (IS_ENABLED(CONFIG_RISCV_USER_CFI))
202 apply_vdso_alternatives(vdso_cfi_start,
203 __vdso_alternatives_start_cfi_offset,
204 __vdso_alternatives_end_cfi_offset);
205
206 if (IS_ENABLED(CONFIG_COMPAT))
207 apply_vdso_alternatives(compat_vdso_start,
208 compat__vdso_alternatives_start_offset,
209 compat__vdso_alternatives_end_offset);
210}
211
212/*
213 * apply_early_boot_alternatives() is called from setup_vm() with MMU-off.
214 *
215 * Following requirements should be honoured for it to work correctly:
216 * 1) It should use PC-relative addressing for accessing kernel symbols.
217 * To achieve this we always use GCC cmodel=medany.
218 * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
219 * so disable compiler instrumentation when FTRACE is enabled.
220 *
221 * Currently, the above requirements are honoured by using custom CFLAGS
222 * for alternative.o in kernel/Makefile.
223 */
224void __init apply_early_boot_alternatives(void)
225{
226#ifdef CONFIG_RISCV_ALTERNATIVE_EARLY
227 _apply_alternatives((struct alt_entry *)__alt_start,
228 (struct alt_entry *)__alt_end,
229 RISCV_ALTERNATIVES_EARLY_BOOT);
230#endif
231}
232
233#ifdef CONFIG_MODULES
234void apply_module_alternatives(void *start, size_t length)
235{
236 _apply_alternatives((struct alt_entry *)start,
237 (struct alt_entry *)(start + length),
238 RISCV_ALTERNATIVES_MODULE);
239}
240#endif