projects/linux·blob master · refs

444 lines · 11.6 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2009 Sunplus Core Technology Co., Ltd.
4 * Chen Liqin <[email protected]>
5 * Lennox Wu <[email protected]>
6 * Copyright (C) 2012 Regents of the University of California
7 * Copyright (C) 2017 SiFive
8 */
9
10#include <linux/bitfield.h>
11#include <linux/cpu.h>
12#include <linux/kernel.h>
13#include <linux/sched.h>
14#include <linux/sched/debug.h>
15#include <linux/sched/task_stack.h>
16#include <linux/tick.h>
17#include <linux/ptrace.h>
18#include <linux/uaccess.h>
19#include <linux/personality.h>
20#include <linux/entry-common.h>
21
22#include <asm/asm-prototypes.h>
23#include <asm/unistd.h>
24#include <asm/processor.h>
25#include <asm/csr.h>
26#include <asm/stacktrace.h>
27#include <asm/string.h>
28#include <asm/switch_to.h>
29#include <asm/thread_info.h>
30#include <asm/cpuidle.h>
31#include <asm/vector.h>
32#include <asm/cpufeature.h>
33#include <asm/exec.h>
34#include <asm/usercfi.h>
35
36#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
37#include <linux/stackprotector.h>
38unsigned long __stack_chk_guard __read_mostly;
39EXPORT_SYMBOL(__stack_chk_guard);
40#endif
41
42extern asmlinkage void ret_from_fork_kernel_asm(void);
43extern asmlinkage void ret_from_fork_user_asm(void);
44
45void noinstr arch_cpu_idle(void)
46{
47 cpu_do_idle();
48}
49
50int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
51{
52 if (!unaligned_ctl_available())
53 return -EINVAL;
54
55 tsk->thread.align_ctl = val;
56 return 0;
57}
58
59int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
60{
61 if (!unaligned_ctl_available())
62 return -EINVAL;
63
64 return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
65}
66
67void __show_regs(struct pt_regs *regs)
68{
69 show_regs_print_info(KERN_DEFAULT);
70
71 if (!user_mode(regs)) {
72 pr_cont("epc : %pS\n", (void *)regs->epc);
73 pr_cont(" ra : %pS\n", (void *)regs->ra);
74 }
75
76 pr_cont("epc : " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n",
77 regs->epc, regs->ra, regs->sp);
78 pr_cont(" gp : " REG_FMT " tp : " REG_FMT " t0 : " REG_FMT "\n",
79 regs->gp, regs->tp, regs->t0);
80 pr_cont(" t1 : " REG_FMT " t2 : " REG_FMT " s0 : " REG_FMT "\n",
81 regs->t1, regs->t2, regs->s0);
82 pr_cont(" s1 : " REG_FMT " a0 : " REG_FMT " a1 : " REG_FMT "\n",
83 regs->s1, regs->a0, regs->a1);
84 pr_cont(" a2 : " REG_FMT " a3 : " REG_FMT " a4 : " REG_FMT "\n",
85 regs->a2, regs->a3, regs->a4);
86 pr_cont(" a5 : " REG_FMT " a6 : " REG_FMT " a7 : " REG_FMT "\n",
87 regs->a5, regs->a6, regs->a7);
88 pr_cont(" s2 : " REG_FMT " s3 : " REG_FMT " s4 : " REG_FMT "\n",
89 regs->s2, regs->s3, regs->s4);
90 pr_cont(" s5 : " REG_FMT " s6 : " REG_FMT " s7 : " REG_FMT "\n",
91 regs->s5, regs->s6, regs->s7);
92 pr_cont(" s8 : " REG_FMT " s9 : " REG_FMT " s10: " REG_FMT "\n",
93 regs->s8, regs->s9, regs->s10);
94 pr_cont(" s11: " REG_FMT " t3 : " REG_FMT " t4 : " REG_FMT "\n",
95 regs->s11, regs->t3, regs->t4);
96 pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT " ssp : " REG_FMT "\n",
97 regs->t5, regs->t6, get_active_shstk(current));
98
99 pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n",
100 regs->status, regs->badaddr, regs->cause);
101}
102void show_regs(struct pt_regs *regs)
103{
104 __show_regs(regs);
105 if (!user_mode(regs))
106 dump_backtrace(regs, NULL, KERN_DEFAULT);
107}
108
109unsigned long arch_align_stack(unsigned long sp)
110{
111 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
112 sp -= get_random_u32_below(PAGE_SIZE);
113 return sp & ~0xf;
114}
115
116#ifdef CONFIG_COMPAT
117static bool compat_mode_supported __read_mostly;
118
119bool compat_elf_check_arch(Elf32_Ehdr *hdr)
120{
121 return compat_mode_supported &&
122 hdr->e_machine == EM_RISCV &&
123 hdr->e_ident[EI_CLASS] == ELFCLASS32;
124}
125
126static int __init compat_mode_detect(void)
127{
128 unsigned long tmp = csr_read(CSR_STATUS);
129
130 csr_write(CSR_STATUS, (tmp & ~SR_UXL) | SR_UXL_32);
131 compat_mode_supported =
132 (csr_read(CSR_STATUS) & SR_UXL) == SR_UXL_32;
133
134 csr_write(CSR_STATUS, tmp);
135
136 pr_info("riscv: ELF compat mode %s",
137 compat_mode_supported ? "supported" : "unsupported");
138
139 return 0;
140}
141early_initcall(compat_mode_detect);
142#endif
143
144void start_thread(struct pt_regs *regs, unsigned long pc,
145 unsigned long sp)
146{
147 regs->status = SR_PIE;
148 if (has_fpu()) {
149 regs->status |= SR_FS_INITIAL;
150 /*
151 * Restore the initial value to the FP register
152 * before starting the user program.
153 */
154 fstate_restore(current, regs);
155 }
156 regs->epc = pc;
157 regs->sp = sp;
158
159 /*
160 * clear shadow stack state on exec.
161 * libc will set it later via prctl.
162 */
163 set_shstk_lock(current, false);
164 set_shstk_status(current, false);
165 set_shstk_base(current, 0, 0);
166 set_active_shstk(current, 0);
167 /*
168 * disable indirect branch tracking on exec.
169 * libc will enable it later via prctl.
170 */
171 set_indir_lp_lock(current, false);
172 set_indir_lp_status(current, false);
173
174#ifdef CONFIG_64BIT
175 regs->status &= ~SR_UXL;
176
177 if (is_compat_task())
178 regs->status |= SR_UXL_32;
179 else
180 regs->status |= SR_UXL_64;
181#endif
182}
183
184void flush_thread(void)
185{
186#ifdef CONFIG_FPU
187 /*
188 * Reset FPU state and context
189 * frm: round to nearest, ties to even (IEEE default)
190 * fflags: accrued exceptions cleared
191 */
192 fstate_off(current, task_pt_regs(current));
193 memset(&current->thread.fstate, 0, sizeof(current->thread.fstate));
194#endif
195#ifdef CONFIG_RISCV_ISA_V
196 /* Reset vector state */
197 riscv_v_vstate_ctrl_init(current);
198 riscv_v_vstate_off(task_pt_regs(current));
199 kfree(current->thread.vstate.datap);
200 memset(&current->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
201 clear_tsk_thread_flag(current, TIF_RISCV_V_DEFER_RESTORE);
202#endif
203#ifdef CONFIG_RISCV_ISA_SUPM
204 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
205 envcfg_update_bits(current, ENVCFG_PMM, ENVCFG_PMM_PMLEN_0);
206#endif
207}
208
209void arch_release_task_struct(struct task_struct *tsk)
210{
211 /* Free the vector context of datap. */
212 if (has_vector() || has_xtheadvector())
213 riscv_v_thread_free(tsk);
214}
215
216int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
217{
218 fstate_save(src, task_pt_regs(src));
219 *dst = *src;
220 /* clear entire V context, including datap for a new task */
221 memset(&dst->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
222 memset(&dst->thread.kernel_vstate, 0, sizeof(struct __riscv_v_ext_state));
223 clear_tsk_thread_flag(dst, TIF_RISCV_V_DEFER_RESTORE);
224
225 return 0;
226}
227
228asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs)
229{
230 fn(fn_arg);
231
232 syscall_exit_to_user_mode(regs);
233}
234
235asmlinkage void ret_from_fork_user(struct pt_regs *regs)
236{
237 syscall_exit_to_user_mode(regs);
238}
239
240int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
241{
242 u64 clone_flags = args->flags;
243 unsigned long usp = args->stack;
244 unsigned long tls = args->tls;
245 unsigned long ssp = 0;
246 struct pt_regs *childregs = task_pt_regs(p);
247
248 /* Ensure all threads in this mm have the same pointer masking mode. */
249 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM) && p->mm && (clone_flags & CLONE_VM))
250 set_bit(MM_CONTEXT_LOCK_PMLEN, &p->mm->context.flags);
251
252 memset(&p->thread.s, 0, sizeof(p->thread.s));
253
254 /* p->thread holds context to be restored by __switch_to() */
255 if (unlikely(args->fn)) {
256 /* Kernel thread */
257 memset(childregs, 0, sizeof(struct pt_regs));
258 /* Supervisor/Machine, irqs on: */
259 childregs->status = SR_PP | SR_PIE;
260
261 p->thread.s[0] = (unsigned long)args->fn;
262 p->thread.s[1] = (unsigned long)args->fn_arg;
263 p->thread.ra = (unsigned long)ret_from_fork_kernel_asm;
264 } else {
265 /* allocate new shadow stack if needed. In case of CLONE_VM we have to */
266 ssp = shstk_alloc_thread_stack(p, args);
267 if (IS_ERR_VALUE(ssp))
268 return PTR_ERR((void *)ssp);
269
270 *childregs = *(current_pt_regs());
271 /* Turn off status.VS */
272 riscv_v_vstate_off(childregs);
273 if (usp) /* User fork */
274 childregs->sp = usp;
275 /* if needed, set new ssp */
276 if (ssp)
277 set_active_shstk(p, ssp);
278 if (clone_flags & CLONE_SETTLS)
279 childregs->tp = tls;
280 childregs->a0 = 0; /* Return value of fork() */
281 p->thread.ra = (unsigned long)ret_from_fork_user_asm;
282 }
283 p->thread.riscv_v_flags = 0;
284 if (has_vector() || has_xtheadvector())
285 riscv_v_thread_alloc(p);
286 p->thread.sp = (unsigned long)childregs; /* kernel sp */
287 return 0;
288}
289
290void __init arch_task_cache_init(void)
291{
292 riscv_v_setup_ctx_cache();
293}
294
295#ifdef CONFIG_RISCV_ISA_SUPM
296enum {
297 PMLEN_0 = 0,
298 PMLEN_7 = 7,
299 PMLEN_16 = 16,
300};
301
302static bool have_user_pmlen_7;
303static bool have_user_pmlen_16;
304
305/*
306 * Control the relaxed ABI allowing tagged user addresses into the kernel.
307 */
308static unsigned int tagged_addr_disabled;
309
310long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
311{
312 unsigned long valid_mask = PR_PMLEN_MASK | PR_TAGGED_ADDR_ENABLE;
313 struct thread_info *ti = task_thread_info(task);
314 struct mm_struct *mm = task->mm;
315 unsigned long pmm;
316 u8 pmlen;
317
318 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
319 return -EINVAL;
320
321 if (is_compat_thread(ti))
322 return -EINVAL;
323
324 if (arg & ~valid_mask)
325 return -EINVAL;
326
327 /*
328 * Prefer the smallest PMLEN that satisfies the user's request,
329 * in case choosing a larger PMLEN has a performance impact.
330 */
331 pmlen = FIELD_GET(PR_PMLEN_MASK, arg);
332 if (pmlen == PMLEN_0) {
333 pmm = ENVCFG_PMM_PMLEN_0;
334 } else if (pmlen <= PMLEN_7 && have_user_pmlen_7) {
335 pmlen = PMLEN_7;
336 pmm = ENVCFG_PMM_PMLEN_7;
337 } else if (pmlen <= PMLEN_16 && have_user_pmlen_16) {
338 pmlen = PMLEN_16;
339 pmm = ENVCFG_PMM_PMLEN_16;
340 } else {
341 return -EINVAL;
342 }
343
344 /*
345 * Do not allow the enabling of the tagged address ABI if globally
346 * disabled via sysctl abi.tagged_addr_disabled, if pointer masking
347 * is disabled for userspace.
348 */
349 if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen))
350 return -EINVAL;
351
352 if (!(arg & PR_TAGGED_ADDR_ENABLE))
353 pmlen = PMLEN_0;
354
355 if (mmap_write_lock_killable(mm))
356 return -EINTR;
357
358 if (test_bit(MM_CONTEXT_LOCK_PMLEN, &mm->context.flags) && mm->context.pmlen != pmlen) {
359 mmap_write_unlock(mm);
360 return -EBUSY;
361 }
362
363 envcfg_update_bits(task, ENVCFG_PMM, pmm);
364 mm->context.pmlen = pmlen;
365
366 mmap_write_unlock(mm);
367
368 return 0;
369}
370
371long get_tagged_addr_ctrl(struct task_struct *task)
372{
373 struct thread_info *ti = task_thread_info(task);
374 long ret = 0;
375
376 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
377 return -EINVAL;
378
379 if (is_compat_thread(ti))
380 return -EINVAL;
381
382 /*
383 * The mm context's pmlen is set only when the tagged address ABI is
384 * enabled, so the effective PMLEN must be extracted from envcfg.PMM.
385 */
386 switch (task->thread.envcfg & ENVCFG_PMM) {
387 case ENVCFG_PMM_PMLEN_7:
388 ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_7);
389 break;
390 case ENVCFG_PMM_PMLEN_16:
391 ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_16);
392 break;
393 }
394
395 if (task->mm->context.pmlen)
396 ret |= PR_TAGGED_ADDR_ENABLE;
397
398 return ret;
399}
400
401static bool try_to_set_pmm(unsigned long value)
402{
403 csr_set(CSR_ENVCFG, value);
404 return (csr_read_clear(CSR_ENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value;
405}
406
407/*
408 * Global sysctl to disable the tagged user addresses support. This control
409 * only prevents the tagged address ABI enabling via prctl() and does not
410 * disable it for tasks that already opted in to the relaxed ABI.
411 */
412
413static const struct ctl_table tagged_addr_sysctl_table[] = {
414 {
415 .procname = "tagged_addr_disabled",
416 .mode = 0644,
417 .data = &tagged_addr_disabled,
418 .maxlen = sizeof(int),
419 .proc_handler = proc_dointvec_minmax,
420 .extra1 = SYSCTL_ZERO,
421 .extra2 = SYSCTL_ONE,
422 },
423};
424
425static int __init tagged_addr_init(void)
426{
427 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
428 return 0;
429
430 /*
431 * envcfg.PMM is a WARL field. Detect which values are supported.
432 * Assume the supported PMLEN values are the same on all harts.
433 */
434 csr_clear(CSR_ENVCFG, ENVCFG_PMM);
435 have_user_pmlen_7 = try_to_set_pmm(ENVCFG_PMM_PMLEN_7);
436 have_user_pmlen_16 = try_to_set_pmm(ENVCFG_PMM_PMLEN_16);
437
438 if (!register_sysctl("abi", tagged_addr_sysctl_table))
439 return -EINVAL;
440
441 return 0;
442}
443core_initcall(tagged_addr_init);
444#endif /* CONFIG_RISCV_ISA_SUPM */