projects/linux·blob master · refs

382 lines · 8.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2
3#define pr_fmt(fmt) "callthunks: " fmt
4
5#include <linux/debugfs.h>
6#include <linux/kallsyms.h>
7#include <linux/memory.h>
8#include <linux/moduleloader.h>
9#include <linux/static_call.h>
10
11#include <asm/alternative.h>
12#include <asm/asm-offsets.h>
13#include <asm/cpu.h>
14#include <asm/ftrace.h>
15#include <asm/insn.h>
16#include <asm/kexec.h>
17#include <asm/nospec-branch.h>
18#include <asm/sections.h>
19#include <asm/switch_to.h>
20#include <asm/sync_core.h>
21#include <asm/text-patching.h>
22#include <asm/xen/hypercall.h>
23
24static int __initdata_or_module debug_callthunks;
25
26#define MAX_PATCH_LEN (255-1)
27
28#define prdbg(fmt, args...) \
29do { \
30 if (debug_callthunks) \
31 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
32} while(0)
33
34static int __init debug_thunks(char *str)
35{
36 debug_callthunks = 1;
37 return 1;
38}
39__setup("debug-callthunks", debug_thunks);
40
41#ifdef CONFIG_CALL_THUNKS_DEBUG
42DEFINE_PER_CPU(u64, __x86_call_count);
43DEFINE_PER_CPU(u64, __x86_ret_count);
44DEFINE_PER_CPU(u64, __x86_stuffs_count);
45DEFINE_PER_CPU(u64, __x86_ctxsw_count);
46EXPORT_PER_CPU_SYMBOL_GPL(__x86_ctxsw_count);
47EXPORT_PER_CPU_SYMBOL_GPL(__x86_call_count);
48#endif
49
50extern s32 __call_sites[], __call_sites_end[];
51
52struct core_text {
53 unsigned long base;
54 unsigned long end;
55 const char *name;
56};
57
58static bool thunks_initialized __ro_after_init;
59
60static const struct core_text builtin_coretext = {
61 .base = (unsigned long)_text,
62 .end = (unsigned long)_etext,
63 .name = "builtin",
64};
65
66asm (
67 ".pushsection .rodata \n"
68 ".global skl_call_thunk_template \n"
69 "skl_call_thunk_template: \n"
70 __stringify(INCREMENT_CALL_DEPTH)" \n"
71 ".global skl_call_thunk_tail \n"
72 "skl_call_thunk_tail: \n"
73 ".popsection \n"
74);
75
76extern u8 skl_call_thunk_template[];
77extern u8 skl_call_thunk_tail[];
78
79#define SKL_TMPL_SIZE \
80 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
81
82extern void error_entry(void);
83extern void xen_error_entry(void);
84extern void paranoid_entry(void);
85
86static inline bool within_coretext(const struct core_text *ct, void *addr)
87{
88 unsigned long p = (unsigned long)addr;
89
90 return ct->base <= p && p < ct->end;
91}
92
93static inline bool within_module_coretext(void *addr)
94{
95 bool ret = false;
96
97#ifdef CONFIG_MODULES
98 struct module *mod;
99
100 guard(rcu)();
101 mod = __module_address((unsigned long)addr);
102 if (mod && within_module_core((unsigned long)addr, mod))
103 ret = true;
104#endif
105 return ret;
106}
107
108static bool is_coretext(const struct core_text *ct, void *addr)
109{
110 if (ct && within_coretext(ct, addr))
111 return true;
112 if (within_coretext(&builtin_coretext, addr))
113 return true;
114 return within_module_coretext(addr);
115}
116
117static bool skip_addr(void *dest)
118{
119 if (dest == error_entry)
120 return true;
121 if (dest == paranoid_entry)
122 return true;
123 if (dest == xen_error_entry)
124 return true;
125 /* Does FILL_RSB... */
126 if (dest == __switch_to_asm)
127 return true;
128 /* Accounts directly */
129 if (dest == ret_from_fork)
130 return true;
131#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
132 if (dest == soft_restart_cpu)
133 return true;
134#endif
135#ifdef CONFIG_FUNCTION_TRACER
136 if (dest == __fentry__)
137 return true;
138#endif
139#ifdef CONFIG_KEXEC_CORE
140# ifdef CONFIG_X86_64
141 if (dest >= (void *)__relocate_kernel_start &&
142 dest < (void *)__relocate_kernel_end)
143 return true;
144# else
145 if (dest >= (void *)relocate_kernel &&
146 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
147 return true;
148# endif
149#endif
150 return false;
151}
152
153static __init_or_module void *call_get_dest(void *addr)
154{
155 struct insn insn;
156 void *dest;
157 int ret;
158
159 ret = insn_decode_kernel(&insn, addr);
160 if (ret)
161 return ERR_PTR(ret);
162
163 /* Patched out call? */
164 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
165 return NULL;
166
167 dest = addr + insn.length + insn.immediate.value;
168 if (skip_addr(dest))
169 return NULL;
170 return dest;
171}
172
173static const u8 nops[] = {
174 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
177 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
178};
179
180static void *patch_dest(void *dest, bool direct)
181{
182 unsigned int tsize = SKL_TMPL_SIZE;
183 u8 insn_buff[MAX_PATCH_LEN];
184 u8 *pad = dest - tsize;
185
186 memcpy(insn_buff, skl_call_thunk_template, tsize);
187 text_poke_apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize);
188
189 /* Already patched? */
190 if (!bcmp(pad, insn_buff, tsize))
191 return pad;
192
193 /* Ensure there are nops */
194 if (bcmp(pad, nops, tsize)) {
195 pr_warn_once("Invalid padding area for %pS\n", dest);
196 return NULL;
197 }
198
199 if (direct)
200 memcpy(pad, insn_buff, tsize);
201 else
202 text_poke_copy_locked(pad, insn_buff, tsize, true);
203 return pad;
204}
205
206static __init_or_module void patch_call(void *addr, const struct core_text *ct)
207{
208 void *pad, *dest;
209 u8 bytes[8];
210
211 if (!within_coretext(ct, addr))
212 return;
213
214 dest = call_get_dest(addr);
215 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
216 return;
217
218 if (!is_coretext(ct, dest))
219 return;
220
221 pad = patch_dest(dest, within_coretext(ct, dest));
222 if (!pad)
223 return;
224
225 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
226 dest, dest, pad);
227 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
228 text_poke_early(addr, bytes, CALL_INSN_SIZE);
229}
230
231static __init_or_module void
232patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
233{
234 s32 *s;
235
236 for (s = start; s < end; s++)
237 patch_call((void *)s + *s, ct);
238}
239
240static __init_or_module void
241callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
242{
243 prdbg("Patching call sites %s\n", ct->name);
244 patch_call_sites(cs->call_start, cs->call_end, ct);
245 prdbg("Patching call sites done%s\n", ct->name);
246}
247
248void __init callthunks_patch_builtin_calls(void)
249{
250 struct callthunk_sites cs = {
251 .call_start = __call_sites,
252 .call_end = __call_sites_end,
253 };
254
255 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
256 return;
257
258 pr_info("Setting up call depth tracking\n");
259 mutex_lock(&text_mutex);
260 callthunks_setup(&cs, &builtin_coretext);
261 thunks_initialized = true;
262 mutex_unlock(&text_mutex);
263}
264
265void *callthunks_translate_call_dest(void *dest)
266{
267 void *target;
268
269 lockdep_assert_held(&text_mutex);
270
271 if (!thunks_initialized || skip_addr(dest))
272 return dest;
273
274 if (!is_coretext(NULL, dest))
275 return dest;
276
277 target = patch_dest(dest, false);
278 return target ? : dest;
279}
280
281#ifdef CONFIG_BPF_JIT
282static bool is_callthunk(void *addr)
283{
284 unsigned int tmpl_size = SKL_TMPL_SIZE;
285 u8 insn_buff[MAX_PATCH_LEN];
286 unsigned long dest;
287 u8 *pad;
288
289 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
290 if (!thunks_initialized || skip_addr((void *)dest))
291 return false;
292
293 pad = (void *)(dest - tmpl_size);
294
295 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
296 text_poke_apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size);
297
298 return !bcmp(pad, insn_buff, tmpl_size);
299}
300
301int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip)
302{
303 unsigned int tmpl_size = SKL_TMPL_SIZE;
304 u8 insn_buff[MAX_PATCH_LEN];
305
306 if (!thunks_initialized)
307 return 0;
308
309 /* Is function call target a thunk? */
310 if (func && is_callthunk(func))
311 return 0;
312
313 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
314 text_poke_apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size);
315
316 memcpy(*pprog, insn_buff, tmpl_size);
317 *pprog += tmpl_size;
318 return tmpl_size;
319}
320#endif
321
322#ifdef CONFIG_MODULES
323void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
324 struct module *mod)
325{
326 struct core_text ct = {
327 .base = (unsigned long)mod->mem[MOD_TEXT].base,
328 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
329 .name = mod->name,
330 };
331
332 if (!thunks_initialized)
333 return;
334
335 mutex_lock(&text_mutex);
336 callthunks_setup(cs, &ct);
337 mutex_unlock(&text_mutex);
338}
339#endif /* CONFIG_MODULES */
340
341#if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
342static int callthunks_debug_show(struct seq_file *m, void *p)
343{
344 unsigned long cpu = (unsigned long)m->private;
345
346 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
347 per_cpu(__x86_call_count, cpu),
348 per_cpu(__x86_ret_count, cpu),
349 per_cpu(__x86_stuffs_count, cpu),
350 per_cpu(__x86_ctxsw_count, cpu));
351 return 0;
352}
353
354static int callthunks_debug_open(struct inode *inode, struct file *file)
355{
356 return single_open(file, callthunks_debug_show, inode->i_private);
357}
358
359static const struct file_operations dfs_ops = {
360 .open = callthunks_debug_open,
361 .read = seq_read,
362 .llseek = seq_lseek,
363 .release = single_release,
364};
365
366static int __init callthunks_debugfs_init(void)
367{
368 struct dentry *dir;
369 unsigned long cpu;
370
371 dir = debugfs_create_dir("callthunks", NULL);
372 for_each_possible_cpu(cpu) {
373 void *arg = (void *)cpu;
374 char name [10];
375
376 sprintf(name, "cpu%lu", cpu);
377 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
378 }
379 return 0;
380}
381__initcall(callthunks_debugfs_init);
382#endif