projects/linux·blob master · refs

1134 lines · 27.8 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm/kernel/entry-armv.S
4 *
5 * Copyright (C) 1996,1997,1998 Russell King.
6 * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
7 * nommu support by Hyok S. Choi (hyok.choi@samsung.com)
8 *
9 * Low-level vector interface routines
10 *
11 * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction
12 * that causes it to save wrong values... Be aware!
13 */
14
15#include <linux/init.h>
16
17#include <asm/assembler.h>
18#include <asm/page.h>
19#include <asm/glue-df.h>
20#include <asm/glue-pf.h>
21#include <asm/vfpmacros.h>
22#include <asm/thread_notify.h>
23#include <asm/unwind.h>
24#include <asm/unistd.h>
25#include <asm/tls.h>
26#include <asm/system_info.h>
27#include <asm/uaccess-asm.h>
28#include <asm/kasan_def.h>
29
30#include "entry-header.S"
31#include <asm/probes.h>
32
33#ifdef CONFIG_HAVE_LD_DEAD_CODE_DATA_ELIMINATION
34#define RELOC_TEXT_NONE .reloc .text, R_ARM_NONE, .
35#else
36#define RELOC_TEXT_NONE
37#endif
38
39/*
40 * Interrupt handling.
41 */
42 .macro irq_handler, from_user:req
43 mov r1, sp
44 ldr_this_cpu r2, irq_stack_ptr, r2, r3
45 .if \from_user == 0
46 @
47 @ If we took the interrupt while running in the kernel, we may already
48 @ be using the IRQ stack, so revert to the original value in that case.
49 @
50 subs r3, r2, r1 @ SP above bottom of IRQ stack?
51 rsbscs r3, r3, #THREAD_SIZE @ ... and below the top?
52#ifdef CONFIG_VMAP_STACK
53 ldr_va r3, high_memory, cc @ End of the linear region
54 cmpcc r3, r1 @ Stack pointer was below it?
55#endif
56 bcc 0f @ If not, switch to the IRQ stack
57 mov r0, r1
58 bl generic_handle_arch_irq
59 b 1f
600:
61 .endif
62
63 mov_l r0, generic_handle_arch_irq
64 bl call_with_stack
651:
66 .endm
67
68 .macro pabt_helper
69 @ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5
70#ifdef MULTI_PABORT
71 ldr_va ip, processor, offset=PROCESSOR_PABT_FUNC
72 bl_r ip
73#else
74 bl CPU_PABORT_HANDLER
75#endif
76 .endm
77
78 .macro dabt_helper
79
80 @
81 @ Call the processor-specific abort handler:
82 @
83 @ r2 - pt_regs
84 @ r4 - aborted context pc
85 @ r5 - aborted context psr
86 @
87 @ The abort handler must return the aborted address in r0, and
88 @ the fault status register in r1. r9 must be preserved.
89 @
90#ifdef MULTI_DABORT
91 ldr_va ip, processor, offset=PROCESSOR_DABT_FUNC
92 bl_r ip
93#else
94 bl CPU_DABORT_HANDLER
95#endif
96 .endm
97
98 .section .entry.text,"ax",%progbits
99
100/*
101 * Invalid mode handlers
102 */
103 .macro inv_entry, reason
104 sub sp, sp, #PT_REGS_SIZE
105 ARM( stmib sp, {r1 - lr} )
106 THUMB( stmia sp, {r0 - r12} )
107 THUMB( str sp, [sp, #S_SP] )
108 THUMB( str lr, [sp, #S_LR] )
109 mov r1, #\reason
110 .endm
111
112__pabt_invalid:
113 inv_entry BAD_PREFETCH
114 b common_invalid
115ENDPROC(__pabt_invalid)
116
117__dabt_invalid:
118 inv_entry BAD_DATA
119 b common_invalid
120ENDPROC(__dabt_invalid)
121
122__irq_invalid:
123 inv_entry BAD_IRQ
124 b common_invalid
125ENDPROC(__irq_invalid)
126
127__und_invalid:
128 inv_entry BAD_UNDEFINSTR
129
130 @
131 @ XXX fall through to common_invalid
132 @
133
134@
135@ common_invalid - generic code for failed exception (re-entrant version of handlers)
136@
137common_invalid:
138 zero_fp
139
140 ldmia r0, {r4 - r6}
141 add r0, sp, #S_PC @ here for interlock avoidance
142 mov r7, #-1 @ "" "" "" ""
143 str r4, [sp] @ save preserved r0
144 stmia r0, {r5 - r7} @ lr_<exception>,
145 @ cpsr_<exception>, "old_r0"
146
147 mov r0, sp
148 b bad_mode
149ENDPROC(__und_invalid)
150
151/*
152 * SVC mode handlers
153 */
154
155#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
156#define SPFIX(code...) code
157#else
158#define SPFIX(code...)
159#endif
160
161 .macro svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1
162 UNWIND(.fnstart )
163 sub sp, sp, #(SVC_REGS_SIZE + \stack_hole)
164 THUMB( add sp, r1 ) @ get SP in a GPR without
165 THUMB( sub r1, sp, r1 ) @ using a temp register
166
167 .if \overflow_check
168 UNWIND(.save {r0 - pc} )
169 do_overflow_check (SVC_REGS_SIZE + \stack_hole)
170 .endif
171
172#ifdef CONFIG_THUMB2_KERNEL
173 tst r1, #4 @ test stack pointer alignment
174 sub r1, sp, r1 @ restore original R1
175 sub sp, r1 @ restore original SP
176#else
177 SPFIX( tst sp, #4 )
178#endif
179 SPFIX( subne sp, sp, #4 )
180
181 ARM( stmib sp, {r1 - r12} )
182 THUMB( stmia sp, {r0 - r12} ) @ No STMIB in Thumb-2
183
184 ldmia r0, {r3 - r5}
185 add r7, sp, #S_SP @ here for interlock avoidance
186 mov r6, #-1 @ "" "" "" ""
187 add r2, sp, #(SVC_REGS_SIZE + \stack_hole)
188 SPFIX( addne r2, r2, #4 )
189 str r3, [sp] @ save the "real" r0 copied
190 @ from the exception stack
191
192 mov r3, lr
193
194 @
195 @ We are now ready to fill in the remaining blanks on the stack:
196 @
197 @ r2 - sp_svc
198 @ r3 - lr_svc
199 @ r4 - lr_<exception>, already fixed up for correct return/restart
200 @ r5 - spsr_<exception>
201 @ r6 - orig_r0 (see pt_regs definition in ptrace.h)
202 @
203 stmia r7, {r2 - r6}
204
205 get_thread_info tsk
206 uaccess_entry tsk, r0, r1, r2, \uaccess
207
208 .if \trace
209#ifdef CONFIG_TRACE_IRQFLAGS
210 bl trace_hardirqs_off
211#endif
212 .endif
213 .endm
214
215 .align 5
216__dabt_svc:
217 svc_entry uaccess=0
218 mov r2, sp
219 dabt_helper
220 THUMB( ldr r5, [sp, #S_PSR] ) @ potentially updated CPSR
221 svc_exit r5 @ return from exception
222 UNWIND(.fnend )
223ENDPROC(__dabt_svc)
224
225 .align 5
226__irq_svc:
227 svc_entry
228 irq_handler from_user=0
229
230#ifdef CONFIG_PREEMPTION
231 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
232 ldr r0, [tsk, #TI_FLAGS] @ get flags
233 teq r8, #0 @ if preempt count != 0
234 movne r0, #0 @ force flags to 0
235 tst r0, #_TIF_NEED_RESCHED
236 blne svc_preempt
237#endif
238
239 svc_exit r5, irq = 1 @ return from exception
240 UNWIND(.fnend )
241ENDPROC(__irq_svc)
242
243 .ltorg
244
245#ifdef CONFIG_PREEMPTION
246svc_preempt:
247 mov r8, lr
2481: bl preempt_schedule_irq @ irq en/disable is done inside
249 ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS
250 tst r0, #_TIF_NEED_RESCHED
251 reteq r8 @ go again
252 b 1b
253#endif
254
255__und_fault:
256 @ Correct the PC such that it is pointing at the instruction
257 @ which caused the fault. If the faulting instruction was ARM
258 @ the PC will be pointing at the next instruction, and have to
259 @ subtract 4. Otherwise, it is Thumb, and the PC will be
260 @ pointing at the second half of the Thumb instruction. We
261 @ have to subtract 2.
262 ldr r2, [r0, #S_PC]
263 sub r2, r2, r1
264 str r2, [r0, #S_PC]
265 b do_undefinstr
266ENDPROC(__und_fault)
267
268 .align 5
269__und_svc:
270#ifdef CONFIG_KPROBES
271 @ If a kprobe is about to simulate a "stmdb sp..." instruction,
272 @ it obviously needs free stack space which then will belong to
273 @ the saved context.
274 svc_entry MAX_STACK_SIZE
275#else
276 svc_entry
277#endif
278
279 mov r1, #4 @ PC correction to apply
280 THUMB( tst r5, #PSR_T_BIT ) @ exception taken in Thumb mode?
281 THUMB( movne r1, #2 ) @ if so, fix up PC correction
282 mov r0, sp @ struct pt_regs *regs
283 bl __und_fault
284
285__und_svc_finish:
286 get_thread_info tsk
287 ldr r5, [sp, #S_PSR] @ Get SVC cpsr
288 svc_exit r5 @ return from exception
289 UNWIND(.fnend )
290ENDPROC(__und_svc)
291
292 .align 5
293__pabt_svc:
294 svc_entry
295 mov r2, sp @ regs
296 pabt_helper
297 svc_exit r5 @ return from exception
298 UNWIND(.fnend )
299ENDPROC(__pabt_svc)
300
301 .align 5
302__fiq_svc:
303 svc_entry trace=0
304 mov r0, sp @ struct pt_regs *regs
305 bl handle_fiq_as_nmi
306 svc_exit_via_fiq
307 UNWIND(.fnend )
308ENDPROC(__fiq_svc)
309
310/*
311 * Abort mode handlers
312 */
313
314@
315@ Taking a FIQ in abort mode is similar to taking a FIQ in SVC mode
316@ and reuses the same macros. However in abort mode we must also
317@ save/restore lr_abt and spsr_abt to make nested aborts safe.
318@
319 .align 5
320__fiq_abt:
321 svc_entry trace=0
322
323 ARM( msr cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
324 THUMB( mov r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
325 THUMB( msr cpsr_c, r0 )
326 mov r1, lr @ Save lr_abt
327 mrs r2, spsr @ Save spsr_abt, abort is now safe
328 ARM( msr cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
329 THUMB( mov r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
330 THUMB( msr cpsr_c, r0 )
331 stmfd sp!, {r1 - r2}
332
333 add r0, sp, #8 @ struct pt_regs *regs
334 bl handle_fiq_as_nmi
335
336 ldmfd sp!, {r1 - r2}
337 ARM( msr cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
338 THUMB( mov r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
339 THUMB( msr cpsr_c, r0 )
340 mov lr, r1 @ Restore lr_abt, abort is unsafe
341 msr spsr_cxsf, r2 @ Restore spsr_abt
342 ARM( msr cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
343 THUMB( mov r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
344 THUMB( msr cpsr_c, r0 )
345
346 svc_exit_via_fiq
347 UNWIND(.fnend )
348ENDPROC(__fiq_abt)
349
350/*
351 * User mode handlers
352 *
353 * EABI note: sp_svc is always 64-bit aligned here, so should PT_REGS_SIZE
354 */
355
356#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (PT_REGS_SIZE & 7)
357#error "sizeof(struct pt_regs) must be a multiple of 8"
358#endif
359
360 .macro usr_entry, trace=1, uaccess=1
361 UNWIND(.fnstart )
362 UNWIND(.cantunwind ) @ don't unwind the user space
363 sub sp, sp, #PT_REGS_SIZE
364 ARM( stmib sp, {r1 - r12} )
365 THUMB( stmia sp, {r0 - r12} )
366
367 ATRAP( mrc p15, 0, r7, c1, c0, 0)
368 ATRAP( ldr_va r8, cr_alignment)
369
370 ldmia r0, {r3 - r5}
371 add r0, sp, #S_PC @ here for interlock avoidance
372 mov r6, #-1 @ "" "" "" ""
373
374 str r3, [sp] @ save the "real" r0 copied
375 @ from the exception stack
376
377 @
378 @ We are now ready to fill in the remaining blanks on the stack:
379 @
380 @ r4 - lr_<exception>, already fixed up for correct return/restart
381 @ r5 - spsr_<exception>
382 @ r6 - orig_r0 (see pt_regs definition in ptrace.h)
383 @
384 @ Also, separately save sp_usr and lr_usr
385 @
386 stmia r0, {r4 - r6}
387 ARM( stmdb r0, {sp, lr}^ )
388 THUMB( store_user_sp_lr r0, r1, S_SP - S_PC )
389
390 .if \uaccess
391 uaccess_disable ip
392 .endif
393
394 @ Enable the alignment trap while in kernel mode
395 ATRAP( teq r8, r7)
396 ATRAP( mcrne p15, 0, r8, c1, c0, 0)
397
398 reload_current r7, r8
399
400 @
401 @ Clear FP to mark the first stack frame
402 @
403 zero_fp
404
405 .if \trace
406#ifdef CONFIG_TRACE_IRQFLAGS
407 bl trace_hardirqs_off
408#endif
409 ct_user_exit save = 0
410 .endif
411 .endm
412
413 .macro kuser_cmpxchg_check
414#if !defined(CONFIG_CPU_32v6K) && defined(CONFIG_KUSER_HELPERS)
415#ifndef CONFIG_MMU
416#warning "NPTL on non MMU needs fixing"
417#else
418 @ Make sure our user space atomic helper is restarted
419 @ if it was interrupted in a critical region. Here we
420 @ perform a quick test inline since it should be false
421 @ 99.9999% of the time. The rest is done out of line.
422 ldr r0, =TASK_SIZE
423 cmp r4, r0
424 blhs kuser_cmpxchg64_fixup
425#endif
426#endif
427 .endm
428
429 .align 5
430__dabt_usr:
431 usr_entry uaccess=0
432 kuser_cmpxchg_check
433 mov r2, sp
434 dabt_helper
435 b ret_from_exception
436 UNWIND(.fnend )
437ENDPROC(__dabt_usr)
438
439 .align 5
440__irq_usr:
441 usr_entry
442 kuser_cmpxchg_check
443 irq_handler from_user=1
444 get_thread_info tsk
445 mov why, #0
446 b ret_to_user_from_irq
447 UNWIND(.fnend )
448ENDPROC(__irq_usr)
449
450 .ltorg
451
452 .align 5
453__und_usr:
454 usr_entry uaccess=0
455
456 @ IRQs must be enabled before attempting to read the instruction from
457 @ user space since that could cause a page/translation fault if the
458 @ page table was modified by another CPU.
459 enable_irq
460
461 tst r5, #PSR_T_BIT @ Thumb mode?
462 mov r1, #2 @ set insn size to 2 for Thumb
463 bne 0f @ handle as Thumb undef exception
464#ifdef CONFIG_FPE_NWFPE
465 adr r9, ret_from_exception
466 bl call_fpe @ returns via R9 on success
467#endif
468 mov r1, #4 @ set insn size to 4 for ARM
4690: mov r0, sp
470 uaccess_disable ip
471 bl __und_fault
472 b ret_from_exception
473 UNWIND(.fnend)
474ENDPROC(__und_usr)
475
476 .align 5
477__pabt_usr:
478 usr_entry
479 mov r2, sp @ regs
480 pabt_helper
481 UNWIND(.fnend )
482 /* fall through */
483/*
484 * This is the return code to user mode for abort handlers
485 */
486ENTRY(ret_from_exception)
487 UNWIND(.fnstart )
488 UNWIND(.cantunwind )
489 get_thread_info tsk
490 mov why, #0
491 b ret_to_user
492 UNWIND(.fnend )
493ENDPROC(__pabt_usr)
494ENDPROC(ret_from_exception)
495
496 .align 5
497__fiq_usr:
498 usr_entry trace=0
499 kuser_cmpxchg_check
500 mov r0, sp @ struct pt_regs *regs
501 bl handle_fiq_as_nmi
502 get_thread_info tsk
503 restore_user_regs fast = 0, offset = 0
504 UNWIND(.fnend )
505ENDPROC(__fiq_usr)
506
507/*
508 * Register switch for ARMv3 and ARMv4 processors
509 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info
510 * previous and next are guaranteed not to be the same.
511 */
512ENTRY(__switch_to)
513 UNWIND(.fnstart )
514 UNWIND(.cantunwind )
515 add ip, r1, #TI_CPU_SAVE
516 ARM( stmia ip!, {r4 - sl, fp, sp, lr} ) @ Store most regs on stack
517 THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack
518 THUMB( str sp, [ip], #4 )
519 THUMB( str lr, [ip], #4 )
520 ldr r4, [r2, #TI_TP_VALUE]
521 ldr r5, [r2, #TI_TP_VALUE + 4]
522#ifdef CONFIG_CPU_USE_DOMAINS
523 mrc p15, 0, r6, c3, c0, 0 @ Get domain register
524 str r6, [r1, #TI_CPU_DOMAIN] @ Save old domain register
525 ldr r6, [r2, #TI_CPU_DOMAIN]
526#endif
527 switch_tls r1, r4, r5, r3, r7
528#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \
529 !defined(CONFIG_STACKPROTECTOR_PER_TASK)
530 ldr r8, =__stack_chk_guard
531 .if (TSK_STACK_CANARY > IMM12_MASK)
532 add r9, r2, #TSK_STACK_CANARY & ~IMM12_MASK
533 ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK]
534 .else
535 ldr r9, [r2, #TSK_STACK_CANARY & IMM12_MASK]
536 .endif
537#endif
538 mov r7, r2 @ Preserve 'next'
539#ifdef CONFIG_CPU_USE_DOMAINS
540 mcr p15, 0, r6, c3, c0, 0 @ Set domain register
541#endif
542 mov r5, r0
543 add r4, r2, #TI_CPU_SAVE
544 ldr r0, =thread_notify_head
545 mov r1, #THREAD_NOTIFY_SWITCH
546 bl atomic_notifier_call_chain
547#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \
548 !defined(CONFIG_STACKPROTECTOR_PER_TASK)
549 str r9, [r8]
550#endif
551 mov r0, r5
552#if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK)
553 set_current r7, r8
554 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously
555#else
556 mov r1, r7
557 ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously
558#ifdef CONFIG_VMAP_STACK
559 @
560 @ For a non-lazy mm switch, check_vmalloc_seq() has ensured that
561 @ that the active mm's page tables have mappings for the prev
562 @ task's stack and the next task's stack.
563 @
564 @ For a lazy mm switch the active mm's page tables have mappings
565 @ for the prev task's stack but might not have mappings for the
566 @ new task's stack. Do a dummy read from the new stack while
567 @ running from the old stack so that we can rely on
568 @ do_translation_fault() to populate missing PMD entries covering the
569 @ new task's stack in the old task's page tables.
570 @
571 ldr r2, [ip]
572#ifdef CONFIG_KASAN_VMALLOC
573 @ Also dummy read from the KASAN shadow memory for the new stack if we
574 @ are using KASAN
575 mov_l r2, KASAN_SHADOW_OFFSET
576 add r2, r2, ip, lsr #KASAN_SHADOW_SCALE_SHIFT
577 ldrb r2, [r2]
578#endif
579#endif
580
581 @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what
582 @ effectuates the task switch, as that is what causes the observable
583 @ values of current and current_thread_info to change. When
584 @ CONFIG_THREAD_INFO_IN_TASK=y, setting current (and therefore
585 @ current_thread_info) is done explicitly, and the update of SP just
586 @ switches us to another stack, with few other side effects. In order
587 @ to prevent this distinction from causing any inconsistencies, let's
588 @ keep the 'set_current' call as close as we can to the update of SP.
589 set_current r1, r2
590 mov sp, ip
591 ret lr
592#endif
593 UNWIND(.fnend )
594ENDPROC(__switch_to)
595
596#ifdef CONFIG_VMAP_STACK
597 .text
598 .align 2
599__bad_stack:
600 @
601 @ We've just detected an overflow. We need to load the address of this
602 @ CPU's overflow stack into the stack pointer register. We have only one
603 @ scratch register so let's use a sequence of ADDs including one
604 @ involving the PC, and decorate them with PC-relative group
605 @ relocations. As these are ARM only, switch to ARM mode first.
606 @
607 @ We enter here with IP clobbered and its value stashed on the mode
608 @ stack.
609 @
610THUMB( bx pc )
611THUMB( nop )
612THUMB( .arm )
613 ldr_this_cpu_armv6 ip, overflow_stack_ptr
614
615 str sp, [ip, #-4]! @ Preserve original SP value
616 mov sp, ip @ Switch to overflow stack
617 pop {ip} @ Original SP in IP
618
619#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
620 mov ip, ip @ mov expected by unwinder
621 push {fp, ip, lr, pc} @ GCC flavor frame record
622#else
623 str ip, [sp, #-8]! @ store original SP
624 push {fpreg, lr} @ Clang flavor frame record
625#endif
626UNWIND( ldr ip, [r0, #4] ) @ load exception LR
627UNWIND( str ip, [sp, #12] ) @ store in the frame record
628 ldr ip, [r0, #12] @ reload IP
629
630 @ Store the original GPRs to the new stack.
631 svc_entry uaccess=0, overflow_check=0
632
633UNWIND( .save {sp, pc} )
634UNWIND( .save {fpreg, lr} )
635UNWIND( .setfp fpreg, sp )
636
637 ldr fpreg, [sp, #S_SP] @ Add our frame record
638 @ to the linked list
639#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
640 ldr r1, [fp, #4] @ reload SP at entry
641 add fp, fp, #12
642#else
643 ldr r1, [fpreg, #8]
644#endif
645 str r1, [sp, #S_SP] @ store in pt_regs
646
647 @ Stash the regs for handle_bad_stack
648 mov r0, sp
649
650 @ Time to die
651 bl handle_bad_stack
652 nop
653UNWIND( .fnend )
654ENDPROC(__bad_stack)
655#endif
656
657 __INIT
658
659/*
660 * User helpers.
661 *
662 * Each segment is 32-byte aligned and will be moved to the top of the high
663 * vector page. New segments (if ever needed) must be added in front of
664 * existing ones. This mechanism should be used only for things that are
665 * really small and justified, and not be abused freely.
666 *
667 * See Documentation/arch/arm/kernel_user_helpers.rst for formal definitions.
668 */
669 THUMB( .arm )
670
671 .macro usr_ret, reg
672#ifdef CONFIG_ARM_THUMB
673 bx \reg
674#else
675 ret \reg
676#endif
677 .endm
678
679 .macro kuser_pad, sym, size
680 .if (. - \sym) & 3
681 .rept 4 - (. - \sym) & 3
682 .byte 0
683 .endr
684 .endif
685 .rept (\size - (. - \sym)) / 4
686 .word 0xe7fddef1
687 .endr
688 .endm
689
690#ifdef CONFIG_KUSER_HELPERS
691 .align 5
692 .globl __kuser_helper_start
693__kuser_helper_start:
694
695/*
696 * Due to the length of some sequences, __kuser_cmpxchg64 spans 2 regular
697 * kuser "slots", therefore 0xffff0f80 is not used as a valid entry point.
698 */
699
700__kuser_cmpxchg64: @ 0xffff0f60
701
702#if defined(CONFIG_CPU_32v6K)
703
704 stmfd sp!, {r4, r5, r6, r7}
705 ldrd r4, r5, [r0] @ load old val
706 ldrd r6, r7, [r1] @ load new val
707 smp_dmb arm
7081: ldrexd r0, r1, [r2] @ load current val
709 eors r3, r0, r4 @ compare with oldval (1)
710 eorseq r3, r1, r5 @ compare with oldval (2)
711 strexdeq r3, r6, r7, [r2] @ store newval if eq
712 teqeq r3, #1 @ success?
713 beq 1b @ if no then retry
714 smp_dmb arm
715 rsbs r0, r3, #0 @ set returned val and C flag
716 ldmfd sp!, {r4, r5, r6, r7}
717 usr_ret lr
718
719#elif !defined(CONFIG_SMP)
720
721#ifdef CONFIG_MMU
722
723 /*
724 * The only thing that can break atomicity in this cmpxchg64
725 * implementation is either an IRQ or a data abort exception
726 * causing another process/thread to be scheduled in the middle of
727 * the critical sequence. The same strategy as for cmpxchg is used.
728 */
729 stmfd sp!, {r4, r5, r6, lr}
730 ldmia r0, {r4, r5} @ load old val
731 ldmia r1, {r6, lr} @ load new val
7321: ldmia r2, {r0, r1} @ load current val
733 eors r3, r0, r4 @ compare with oldval (1)
734 eorseq r3, r1, r5 @ compare with oldval (2)
7352: stmiaeq r2, {r6, lr} @ store newval if eq
736 rsbs r0, r3, #0 @ set return val and C flag
737 ldmfd sp!, {r4, r5, r6, pc}
738
739 .text
740kuser_cmpxchg64_fixup:
741 @ Called from kuser_cmpxchg_fixup.
742 @ r4 = address of interrupted insn (must be preserved).
743 @ sp = saved regs. r7 and r8 are clobbered.
744 @ 1b = first critical insn, 2b = last critical insn.
745 @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
746 mov r7, #0xffff0fff
747 sub r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64)))
748 subs r8, r4, r7
749 rsbscs r8, r8, #(2b - 1b)
750 strcs r7, [sp, #S_PC]
751#if __LINUX_ARM_ARCH__ < 6
752 bcc kuser_cmpxchg32_fixup
753#endif
754 ret lr
755 .previous
756
757#else
758#warning "NPTL on non MMU needs fixing"
759 mov r0, #-1
760 adds r0, r0, #0
761 usr_ret lr
762#endif
763
764#else
765#error "incoherent kernel configuration"
766#endif
767
768 kuser_pad __kuser_cmpxchg64, 64
769
770__kuser_memory_barrier: @ 0xffff0fa0
771 smp_dmb arm
772 usr_ret lr
773
774 kuser_pad __kuser_memory_barrier, 32
775
776__kuser_cmpxchg: @ 0xffff0fc0
777
778#if __LINUX_ARM_ARCH__ < 6
779
780#ifdef CONFIG_MMU
781
782 /*
783 * The only thing that can break atomicity in this cmpxchg
784 * implementation is either an IRQ or a data abort exception
785 * causing another process/thread to be scheduled in the middle
786 * of the critical sequence. To prevent this, code is added to
787 * the IRQ and data abort exception handlers to set the pc back
788 * to the beginning of the critical section if it is found to be
789 * within that critical section (see kuser_cmpxchg_fixup).
790 */
7911: ldr r3, [r2] @ load current val
792 subs r3, r3, r0 @ compare with oldval
7932: streq r1, [r2] @ store newval if eq
794 rsbs r0, r3, #0 @ set return val and C flag
795 usr_ret lr
796
797 .text
798kuser_cmpxchg32_fixup:
799 @ Called from kuser_cmpxchg_check macro.
800 @ r4 = address of interrupted insn (must be preserved).
801 @ sp = saved regs. r7 and r8 are clobbered.
802 @ 1b = first critical insn, 2b = last critical insn.
803 @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
804 mov r7, #0xffff0fff
805 sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg)))
806 subs r8, r4, r7
807 rsbscs r8, r8, #(2b - 1b)
808 strcs r7, [sp, #S_PC]
809 ret lr
810 .previous
811
812#else
813#warning "NPTL on non MMU needs fixing"
814 mov r0, #-1
815 adds r0, r0, #0
816 usr_ret lr
817#endif
818
819#else
820
821 smp_dmb arm
8221: ldrex r3, [r2]
823 subs r3, r3, r0
824 strexeq r3, r1, [r2]
825 teqeq r3, #1
826 beq 1b
827 rsbs r0, r3, #0
828 /* beware -- each __kuser slot must be 8 instructions max */
829 ALT_SMP(b __kuser_memory_barrier)
830 ALT_UP(usr_ret lr)
831
832#endif
833
834 kuser_pad __kuser_cmpxchg, 32
835
836__kuser_get_tls: @ 0xffff0fe0
837 ldr r0, [pc, #(16 - 8)] @ read TLS, set in kuser_get_tls_init
838 usr_ret lr
839 mrc p15, 0, r0, c13, c0, 3 @ 0xffff0fe8 hardware TLS code
840 kuser_pad __kuser_get_tls, 16
841 .rep 3
842 .word 0 @ 0xffff0ff0 software TLS value, then
843 .endr @ pad up to __kuser_helper_version
844
845__kuser_helper_version: @ 0xffff0ffc
846 .word ((__kuser_helper_end - __kuser_helper_start) >> 5)
847
848 .globl __kuser_helper_end
849__kuser_helper_end:
850
851#endif
852
853 THUMB( .thumb )
854
855/*
856 * Vector stubs.
857 *
858 * This code is copied to 0xffff1000 so we can use branches in the
859 * vectors, rather than ldr's. Note that this code must not exceed
860 * a page size.
861 *
862 * Common stub entry macro:
863 * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
864 *
865 * SP points to a minimal amount of processor-private memory, the address
866 * of which is copied into r0 for the mode specific abort handler.
867 */
868 .macro vector_stub, name, mode, correction=0
869 .align 5
870#ifdef CONFIG_HARDEN_BRANCH_HISTORY
871vector_bhb_bpiall_\name:
872 mcr p15, 0, r0, c7, c5, 6 @ BPIALL
873 @ isb not needed due to "movs pc, lr" in the vector stub
874 @ which gives a "context synchronisation".
875#endif
876
877vector_\name:
878 .if \correction
879 sub lr, lr, #\correction
880 .endif
881
882 @ Save r0, lr_<exception> (parent PC)
883 stmia sp, {r0, lr} @ save r0, lr
884
885 @ Save spsr_<exception> (parent CPSR)
886.Lvec_\name:
887 mrs lr, spsr
888 str lr, [sp, #8] @ save spsr
889
890 @
891 @ Prepare for SVC32 mode. IRQs remain disabled.
892 @
893 mrs r0, cpsr
894 eor r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE)
895 msr spsr_cxsf, r0
896
897 @
898 @ the branch table must immediately follow this code
899 @
900 and lr, lr, #0x0f
901 THUMB( adr r0, 1f )
902 THUMB( ldr lr, [r0, lr, lsl #2] )
903 mov r0, sp
904 ARM( ldr lr, [pc, lr, lsl #2] )
905 movs pc, lr @ branch to handler in SVC mode
906ENDPROC(vector_\name)
907
908#ifdef CONFIG_HARDEN_BRANCH_HISTORY
909 .subsection 1
910 .align 5
911vector_bhb_loop8_\name:
912 .if \correction
913 sub lr, lr, #\correction
914 .endif
915
916 @ Save r0, lr_<exception> (parent PC)
917 stmia sp, {r0, lr}
918
919 @ bhb workaround
920 mov r0, #8
9213: W(b) . + 4
922 subs r0, r0, #1
923 bne 3b
924 dsb nsh
925 @ isb not needed due to "movs pc, lr" in the vector stub
926 @ which gives a "context synchronisation".
927 b .Lvec_\name
928ENDPROC(vector_bhb_loop8_\name)
929 .previous
930#endif
931
932 .align 2
933 @ handler addresses follow this label
9341:
935 .endm
936
937 .section .stubs, "ax", %progbits
938 @ These need to remain at the start of the section so that
939 @ they are in range of the 'SWI' entries in the vector tables
940 @ located 4k down.
941.L__vector_swi:
942 .word vector_swi
943#ifdef CONFIG_HARDEN_BRANCH_HISTORY
944.L__vector_bhb_loop8_swi:
945 .word vector_bhb_loop8_swi
946.L__vector_bhb_bpiall_swi:
947 .word vector_bhb_bpiall_swi
948#endif
949
950vector_rst:
951 ARM( swi SYS_ERROR0 )
952 THUMB( svc #0 )
953 THUMB( nop )
954 b vector_und
955
956/*
957 * Interrupt dispatcher
958 */
959 vector_stub irq, IRQ_MODE, 4
960
961 .long __irq_usr @ 0 (USR_26 / USR_32)
962 .long __irq_invalid @ 1 (FIQ_26 / FIQ_32)
963 .long __irq_invalid @ 2 (IRQ_26 / IRQ_32)
964 .long __irq_svc @ 3 (SVC_26 / SVC_32)
965 .long __irq_invalid @ 4
966 .long __irq_invalid @ 5
967 .long __irq_invalid @ 6
968 .long __irq_invalid @ 7
969 .long __irq_invalid @ 8
970 .long __irq_invalid @ 9
971 .long __irq_invalid @ a
972 .long __irq_invalid @ b
973 .long __irq_invalid @ c
974 .long __irq_invalid @ d
975 .long __irq_invalid @ e
976 .long __irq_invalid @ f
977
978/*
979 * Data abort dispatcher
980 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
981 */
982 vector_stub dabt, ABT_MODE, 8
983
984 .long __dabt_usr @ 0 (USR_26 / USR_32)
985 .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32)
986 .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32)
987 .long __dabt_svc @ 3 (SVC_26 / SVC_32)
988 .long __dabt_invalid @ 4
989 .long __dabt_invalid @ 5
990 .long __dabt_invalid @ 6
991 .long __dabt_invalid @ 7
992 .long __dabt_invalid @ 8
993 .long __dabt_invalid @ 9
994 .long __dabt_invalid @ a
995 .long __dabt_invalid @ b
996 .long __dabt_invalid @ c
997 .long __dabt_invalid @ d
998 .long __dabt_invalid @ e
999 .long __dabt_invalid @ f
1000
1001/*
1002 * Prefetch abort dispatcher
1003 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
1004 */
1005 vector_stub pabt, ABT_MODE, 4
1006
1007 .long __pabt_usr @ 0 (USR_26 / USR_32)
1008 .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32)
1009 .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32)
1010 .long __pabt_svc @ 3 (SVC_26 / SVC_32)
1011 .long __pabt_invalid @ 4
1012 .long __pabt_invalid @ 5
1013 .long __pabt_invalid @ 6
1014 .long __pabt_invalid @ 7
1015 .long __pabt_invalid @ 8
1016 .long __pabt_invalid @ 9
1017 .long __pabt_invalid @ a
1018 .long __pabt_invalid @ b
1019 .long __pabt_invalid @ c
1020 .long __pabt_invalid @ d
1021 .long __pabt_invalid @ e
1022 .long __pabt_invalid @ f
1023
1024/*
1025 * Undef instr entry dispatcher
1026 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
1027 */
1028 vector_stub und, UND_MODE
1029
1030 .long __und_usr @ 0 (USR_26 / USR_32)
1031 .long __und_invalid @ 1 (FIQ_26 / FIQ_32)
1032 .long __und_invalid @ 2 (IRQ_26 / IRQ_32)
1033 .long __und_svc @ 3 (SVC_26 / SVC_32)
1034 .long __und_invalid @ 4
1035 .long __und_invalid @ 5
1036 .long __und_invalid @ 6
1037 .long __und_invalid @ 7
1038 .long __und_invalid @ 8
1039 .long __und_invalid @ 9
1040 .long __und_invalid @ a
1041 .long __und_invalid @ b
1042 .long __und_invalid @ c
1043 .long __und_invalid @ d
1044 .long __und_invalid @ e
1045 .long __und_invalid @ f
1046
1047 .align 5
1048
1049/*=============================================================================
1050 * Address exception handler
1051 *-----------------------------------------------------------------------------
1052 * These aren't too critical.
1053 * (they're not supposed to happen, and won't happen in 32-bit data mode).
1054 */
1055
1056vector_addrexcptn:
1057 b vector_addrexcptn
1058
1059/*=============================================================================
1060 * FIQ "NMI" handler
1061 *-----------------------------------------------------------------------------
1062 * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86
1063 * systems. This must be the last vector stub, so lets place it in its own
1064 * subsection.
1065 */
1066 .subsection 2
1067 vector_stub fiq, FIQ_MODE, 4
1068
1069 .long __fiq_usr @ 0 (USR_26 / USR_32)
1070 .long __fiq_svc @ 1 (FIQ_26 / FIQ_32)
1071 .long __fiq_svc @ 2 (IRQ_26 / IRQ_32)
1072 .long __fiq_svc @ 3 (SVC_26 / SVC_32)
1073 .long __fiq_svc @ 4
1074 .long __fiq_svc @ 5
1075 .long __fiq_svc @ 6
1076 .long __fiq_abt @ 7
1077 .long __fiq_svc @ 8
1078 .long __fiq_svc @ 9
1079 .long __fiq_svc @ a
1080 .long __fiq_svc @ b
1081 .long __fiq_svc @ c
1082 .long __fiq_svc @ d
1083 .long __fiq_svc @ e
1084 .long __fiq_svc @ f
1085
1086 .globl vector_fiq
1087
1088 .section .vectors, "ax", %progbits
1089 RELOC_TEXT_NONE
1090 W(b) vector_rst
1091 W(b) vector_und
1092ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_swi )
1093THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_swi )
1094 W(ldr) pc, .
1095 W(b) vector_pabt
1096 W(b) vector_dabt
1097 W(b) vector_addrexcptn
1098 W(b) vector_irq
1099 W(b) vector_fiq
1100
1101#ifdef CONFIG_HARDEN_BRANCH_HISTORY
1102 .section .vectors.bhb.loop8, "ax", %progbits
1103 RELOC_TEXT_NONE
1104 W(b) vector_rst
1105 W(b) vector_bhb_loop8_und
1106ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_loop8_swi )
1107THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_loop8_swi )
1108 W(ldr) pc, .
1109 W(b) vector_bhb_loop8_pabt
1110 W(b) vector_bhb_loop8_dabt
1111 W(b) vector_addrexcptn
1112 W(b) vector_bhb_loop8_irq
1113 W(b) vector_bhb_loop8_fiq
1114
1115 .section .vectors.bhb.bpiall, "ax", %progbits
1116 RELOC_TEXT_NONE
1117 W(b) vector_rst
1118 W(b) vector_bhb_bpiall_und
1119ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_bpiall_swi )
1120THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_bpiall_swi )
1121 W(ldr) pc, .
1122 W(b) vector_bhb_bpiall_pabt
1123 W(b) vector_bhb_bpiall_dabt
1124 W(b) vector_addrexcptn
1125 W(b) vector_bhb_bpiall_irq
1126 W(b) vector_bhb_bpiall_fiq
1127#endif
1128
1129 .data
1130 .align 2
1131
1132 .globl cr_alignment
1133cr_alignment:
1134 .space 4