projects/linux·blob master · refs

654 lines · 18.7 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * S390 low-level entry points.
4 *
5 * Copyright IBM Corp. 1999, 2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 */
10
11#include <linux/export.h>
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/asm-extable.h>
15#include <asm/alternative.h>
16#include <asm/processor.h>
17#include <asm/cache.h>
18#include <asm/dwarf.h>
19#include <asm/errno.h>
20#include <asm/ptrace.h>
21#include <asm/thread_info.h>
22#include <asm/asm-offsets.h>
23#include <asm/unistd.h>
24#include <asm/page.h>
25#include <asm/sigp.h>
26#include <asm/bug.h>
27#include <asm/irq.h>
28#include <asm/fpu-insn.h>
29#include <asm/setup.h>
30#include <asm/nmi.h>
31#include <asm/nospec-insn.h>
32#include <asm/lowcore.h>
33#include <asm/machine.h>
34#include "entry.h"
35
36_LPP_OFFSET = __LC_LPP
37
38 .macro STBEAR address
39 ALTERNATIVE "nop", ".insn s,0xb2010000,\address", ALT_FACILITY(193)
40 .endm
41
42 .macro LBEAR address
43 ALTERNATIVE "nop", ".insn s,0xb2000000,\address", ALT_FACILITY(193)
44 .endm
45
46 .macro LPSWEY address, lpswe
47 ALTERNATIVE_2 "b \lpswe;nopr", \
48 ".insn siy,0xeb0000000071,\address,0", ALT_FACILITY(193), \
49 __stringify(.insn siy,0xeb0000000071,LOWCORE_ALT_ADDRESS+\address,0), \
50 ALT_FEATURE(MFEATURE_LOWCORE)
51 .endm
52
53 .macro MBEAR reg, lowcore
54 ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK(\lowcore)),\
55 ALT_FACILITY(193)
56 .endm
57
58 .macro CHECK_VMAP_STACK savearea, lowcore, oklabel
59 lgr %r14,%r15
60 nill %r14,0x10000 - THREAD_SIZE
61 oill %r14,STACK_INIT_OFFSET
62 clg %r14,__LC_KERNEL_STACK(\lowcore)
63 je \oklabel
64 clg %r14,__LC_ASYNC_STACK(\lowcore)
65 je \oklabel
66 clg %r14,__LC_MCCK_STACK(\lowcore)
67 je \oklabel
68 clg %r14,__LC_NODAT_STACK(\lowcore)
69 je \oklabel
70 clg %r14,__LC_RESTART_STACK(\lowcore)
71 je \oklabel
72 la %r14,\savearea(\lowcore)
73 j stack_invalid
74 .endm
75
76 /*
77 * The TSTMSK macro generates a test-under-mask instruction by
78 * calculating the memory offset for the specified mask value.
79 * Mask value can be any constant. The macro shifts the mask
80 * value to calculate the memory offset for the test-under-mask
81 * instruction.
82 */
83 .macro TSTMSK addr, mask, size=8, bytepos=0
84 .if (\bytepos < \size) && (\mask >> 8)
85 .if (\mask & 0xff)
86 .error "Mask exceeds byte boundary"
87 .endif
88 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
89 .exitm
90 .endif
91 .ifeq \mask
92 .error "Mask must not be zero"
93 .endif
94 off = \size - \bytepos - 1
95 tm off+\addr, \mask
96 .endm
97
98 .macro BPOFF
99 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", ALT_SPEC(82)
100 .endm
101
102 .macro BPON
103 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82)
104 .endm
105
106 .macro BPENTER tif_ptr,tif_mask
107 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
108 "j .+12; nop; nop", ALT_SPEC(82)
109 .endm
110
111 .macro BPEXIT tif_ptr,tif_mask
112 TSTMSK \tif_ptr,\tif_mask
113 ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \
114 "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82)
115 .endm
116
117#if IS_ENABLED(CONFIG_KVM)
118 .macro SIEEXIT sie_control,lowcore
119 lg %r9,\sie_control # get control block pointer
120 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
121 lctlg %c1,%c1,__LC_USER_ASCE(\lowcore) # load primary asce
122 lg %r9,__LC_CURRENT(\lowcore)
123 mvi __TI_sie(%r9),0
124 larl %r9,sie_exit # skip forward to sie_exit
125 .endm
126#endif
127
128 .macro STACKLEAK_ERASE
129#ifdef CONFIG_KSTACK_ERASE
130 brasl %r14,stackleak_erase_on_task_stack
131#endif
132 .endm
133
134 GEN_BR_THUNK %r14
135
136 .section .kprobes.text, "ax"
137.Ldummy:
138 /*
139 * The following nop exists only in order to avoid that the next
140 * symbol starts at the beginning of the kprobes text section.
141 * In that case there would be several symbols at the same address.
142 * E.g. objdump would take an arbitrary symbol when disassembling
143 * the code.
144 * With the added nop in between this cannot happen.
145 */
146 nop 0
147
148/*
149 * Scheduler resume function, called by __switch_to
150 * gpr2 = (task_struct *)prev
151 * gpr3 = (task_struct *)next
152 * Returns:
153 * gpr2 = prev
154 */
155SYM_FUNC_START(__switch_to_asm)
156 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
157 lghi %r4,__TASK_stack
158 lghi %r1,__TASK_thread
159 llill %r5,STACK_INIT_OFFSET
160 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev
161 lg %r15,0(%r4,%r3) # start of kernel stack of next
162 agr %r15,%r5 # end of kernel stack of next
163 GET_LC %r13
164 stg %r3,__LC_CURRENT(%r13) # store task struct of next
165 stg %r15,__LC_KERNEL_STACK(%r13) # store end of kernel stack
166 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next
167 lay %r4,__TASK_pid(%r3)
168 mvc __LC_CURRENT_PID(4,%r13),0(%r4) # store pid of next
169 ALTERNATIVE "nop", "lpp _LPP_OFFSET(%r13)", ALT_FACILITY(40)
170#ifdef CONFIG_STACKPROTECTOR
171 lg %r3,__TASK_stack_canary(%r3)
172 stg %r3,__LC_STACK_CANARY(%r13)
173#endif
174 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
175 BR_EX %r14
176SYM_FUNC_END(__switch_to_asm)
177
178#if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS)
179
180SYM_FUNC_START(__WARN_trap)
181 mc MONCODE_BUG_ARG(%r0),0
182 BR_EX %r14
183SYM_FUNC_END(__WARN_trap)
184EXPORT_SYMBOL(__WARN_trap)
185
186#endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */
187
188#if IS_ENABLED(CONFIG_KVM)
189/*
190 * __sie64a calling convention:
191 * %r2 pointer to sie control block phys
192 * %r3 pointer to sie control block virt
193 * %r4 guest register save area
194 * %r5 guest asce
195 */
196SYM_FUNC_START(__sie64a)
197 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
198 GET_LC %r13
199 lg %r14,__LC_CURRENT(%r13)
200 stg %r2,__SF_SIE_CONTROL_PHYS(%r15) # save sie block physical..
201 stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses
202 stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area
203 stg %r5,__SF_SIE_GUEST_ASCE(%r15) # save guest asce
204 xc __SF_SIE_RETURN(8,%r15),__SF_SIE_RETURN(%r15) # return code = 0
205 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r14) # copy thread flags
206 lmg %r0,%r13,0(%r4) # load guest gprs 0-13
207 mvi __TI_sie(%r14),1
208 stosm __SF_SIE_IRQ(%r15),0x03 # enable interrupts
209 lctlg %c1,%c1,__SF_SIE_GUEST_ASCE(%r15) # load primary asce
210 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
211 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
212 tm __SIE_PROG20+3(%r14),3 # last exit...
213 jnz .Lsie_skip
214 lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr
215 BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
216.Lsie_entry:
217 sie 0(%r14)
218# Let the next instruction be NOP to avoid triggering a machine check
219# and handling it in a guest as result of the instruction execution.
220 nopr 7
221.Lsie_leave:
222 BPOFF
223 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
224.Lsie_skip:
225 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
226 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
227 GET_LC %r14
228 lctlg %c1,%c1,__LC_USER_ASCE(%r14) # load primary asce
229 lg %r14,__LC_CURRENT(%r14)
230 mvi __TI_sie(%r14),0
231SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL)
232 stnsm __SF_SIE_IRQ(%r15),0xfc # disable interrupts
233 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area
234 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
235 xgr %r0,%r0 # clear guest registers to
236 xgr %r1,%r1 # prevent speculative use
237 xgr %r3,%r3
238 xgr %r4,%r4
239 xgr %r5,%r5
240 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
241 lg %r2,__SF_SIE_RETURN(%r15) # return sie return code
242 BR_EX %r14
243SYM_FUNC_END(__sie64a)
244EXPORT_SYMBOL(__sie64a)
245EXPORT_SYMBOL(sie_exit)
246#endif
247
248/*
249 * SVC interrupt handler routine. System calls are synchronous events and
250 * are entered with interrupts disabled.
251 */
252
253SYM_CODE_START(system_call)
254 STMG_LC %r8,%r15,__LC_SAVE_AREA
255 GET_LC %r13
256 stpt __LC_SYS_ENTER_TIMER(%r13)
257 BPOFF
258 lghi %r14,0
259.Lsysc_per:
260 STBEAR __LC_LAST_BREAK(%r13)
261 lg %r15,__LC_KERNEL_STACK(%r13)
262 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
263 stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
264 # clear user controlled register to prevent speculative use
265 xgr %r0,%r0
266 xgr %r1,%r1
267 xgr %r4,%r4
268 xgr %r5,%r5
269 xgr %r6,%r6
270 xgr %r7,%r7
271 xgr %r8,%r8
272 xgr %r9,%r9
273 xgr %r10,%r10
274 xgr %r11,%r11
275 xgr %r12,%r12
276 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
277 mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13)
278 MBEAR %r2,%r13
279 lgr %r3,%r14
280 brasl %r14,__do_syscall
281 STACKLEAK_ERASE
282 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
283 BPON
284 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
285 stpt __LC_EXIT_TIMER(%r13)
286 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
287 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
288SYM_CODE_END(system_call)
289
290#
291# a new process exits the kernel with ret_from_fork
292#
293SYM_CODE_START(ret_from_fork)
294 lgr %r3,%r11
295 brasl %r14,__ret_from_fork
296 STACKLEAK_ERASE
297 GET_LC %r13
298 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
299 BPON
300 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
301 stpt __LC_EXIT_TIMER(%r13)
302 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
303 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
304SYM_CODE_END(ret_from_fork)
305
306/*
307 * Program check handler routine
308 */
309
310SYM_CODE_START(pgm_check_handler)
311 STMG_LC %r8,%r15,__LC_SAVE_AREA
312 GET_LC %r13
313 stpt __LC_SYS_ENTER_TIMER(%r13)
314 BPOFF
315 lmg %r8,%r9,__LC_PGM_OLD_PSW(%r13)
316 xgr %r10,%r10
317 tmhh %r8,0x0001 # coming from user space?
318 jo 3f # -> fault in user space
319#if IS_ENABLED(CONFIG_KVM)
320 lg %r11,__LC_CURRENT(%r13)
321 tm __TI_sie(%r11),0xff
322 jz 1f
323 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
324 SIEEXIT __SF_SIE_CONTROL(%r15),%r13
325 lghi %r10,PGM_FLAG_GUEST_FAULT
326#endif
3271: tmhh %r8,0x4000 # PER bit set in old PSW ?
328 jnz 2f # -> enabled, can't be a double fault
329 tm __LC_PGM_ILC+3(%r13),0x80 # check for per exception
330 jnz .Lpgm_svcper # -> single stepped svc
3312: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
332 # CHECK_VMAP_STACK branches to stack_invalid or 4f
333 CHECK_VMAP_STACK __LC_SAVE_AREA,%r13,4f
3343: lg %r15,__LC_KERNEL_STACK(%r13)
3354: la %r11,STACK_FRAME_OVERHEAD(%r15)
336 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
337 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
338 stmg %r0,%r7,__PT_R0(%r11)
339 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13)
340 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r13)
341 stmg %r8,%r9,__PT_PSW(%r11)
342 # clear user controlled registers to prevent speculative use
343 xgr %r0,%r0
344 xgr %r1,%r1
345 xgr %r4,%r4
346 xgr %r5,%r5
347 xgr %r6,%r6
348 xgr %r7,%r7
349 xgr %r12,%r12
350 lgr %r2,%r11
351 lgr %r3,%r10
352 brasl %r14,__do_pgm_check
353 tmhh %r8,0x0001 # returning to user space?
354 jno .Lpgm_exit_kernel
355 STACKLEAK_ERASE
356 BPON
357 stpt __LC_EXIT_TIMER(%r13)
358.Lpgm_exit_kernel:
359 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
360 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
361 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
362 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
363
364#
365# single stepped system call
366#
367.Lpgm_svcper:
368 mvc __LC_RETURN_PSW(8,%r13),__LC_SVC_NEW_PSW(%r13)
369 larl %r14,.Lsysc_per
370 stg %r14,__LC_RETURN_PSW+8(%r13)
371 lghi %r14,SYSCALL_FLAG_PER_TRAP
372 LBEAR __LC_PGM_LAST_BREAK(%r13)
373 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
374SYM_CODE_END(pgm_check_handler)
375
376/*
377 * Interrupt handler macro used for external and IO interrupts.
378 */
379.macro INT_HANDLER name,lc_old_psw,handler
380SYM_CODE_START(\name)
381 STMG_LC %r8,%r15,__LC_SAVE_AREA
382 GET_LC %r13
383#ifdef CONFIG_NO_HZ_COMMON
384 larl %r12,kernel_cpustat
385 ag %r12,__LC_PERCPU_OFFSET(%r13)
386 asi __KCPUSTAT_SEQUENCE(%r12),1
387#endif
388 stcke __LC_INT_CLOCK(%r13)
389 stpt __LC_SYS_ENTER_TIMER(%r13)
390 larl %r10,s390_idle
391 ag %r10,__LC_PERCPU_OFFSET(%r13)
392 mvc __IDLE_CLOCK_EXIT(16,%r10),__LC_INT_CLOCK(%r13)
393#ifdef CONFIG_NO_HZ_COMMON
394 asi __KCPUSTAT_SEQUENCE(%r12),1
395#endif
396 STBEAR __LC_LAST_BREAK(%r13)
397 BPOFF
398 lmg %r8,%r9,\lc_old_psw(%r13)
399 tmhh %r8,0x0001 # interrupting from user ?
400 jnz 1f
401#if IS_ENABLED(CONFIG_KVM)
402 lg %r10,__LC_CURRENT(%r13)
403 tm __TI_sie(%r10),0xff
404 jz 0f
405 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
406 SIEEXIT __SF_SIE_CONTROL(%r15),%r13
407#endif
4080: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
409 j 2f
4101: lg %r15,__LC_KERNEL_STACK(%r13)
4112: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
412 la %r11,STACK_FRAME_OVERHEAD(%r15)
413 stmg %r0,%r7,__PT_R0(%r11)
414 # clear user controlled registers to prevent speculative use
415 xgr %r0,%r0
416 xgr %r1,%r1
417 xgr %r3,%r3
418 xgr %r4,%r4
419 xgr %r5,%r5
420 xgr %r6,%r6
421 xgr %r7,%r7
422 xgr %r12,%r12
423 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
424 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13)
425 MBEAR %r11,%r13
426 stmg %r8,%r9,__PT_PSW(%r11)
427 lgr %r2,%r11 # pass pointer to pt_regs
428 brasl %r14,\handler
429 mvc __LC_RETURN_PSW(16,%r13),__PT_PSW(%r11)
430 tmhh %r8,0x0001 # returning to user ?
431 jno 2f
432 STACKLEAK_ERASE
433 BPON
434 stpt __LC_EXIT_TIMER(%r13)
4352: LBEAR __PT_LAST_BREAK(%r11)
436 lmg %r0,%r15,__PT_R0(%r11)
437 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
438SYM_CODE_END(\name)
439.endm
440
441 .section .irqentry.text, "ax"
442
443INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
444INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
445
446 .section .kprobes.text, "ax"
447
448/*
449 * Machine check handler routines
450 */
451SYM_CODE_START(mcck_int_handler)
452 BPOFF
453 GET_LC %r13
454 lmg %r8,%r9,__LC_MCK_OLD_PSW(%r13)
455 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_SYSTEM_DAMAGE
456 jo .Lmcck_panic # yes -> rest of mcck code invalid
457 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_CR_VALID
458 jno .Lmcck_panic # control registers invalid -> panic
459 ptlb
460 lay %r14,__LC_CPU_TIMER_SAVE_AREA(%r13)
461 mvc __LC_MCCK_ENTER_TIMER(8,%r13),0(%r14)
462 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_CPU_TIMER_VALID
463 jo 3f
464 la %r14,__LC_SYS_ENTER_TIMER(%r13)
465 clc 0(8,%r14),__LC_EXIT_TIMER(%r13)
466 jl 1f
467 la %r14,__LC_EXIT_TIMER(%r13)
4681: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER(%r13)
469 jl 2f
470 la %r14,__LC_LAST_UPDATE_TIMER(%r13)
4712: spt 0(%r14)
472 mvc __LC_MCCK_ENTER_TIMER(8,%r13),0(%r14)
4733: TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_PSW_MWP_VALID
474 jno .Lmcck_panic
475 tmhh %r8,0x0001 # interrupting from user ?
476 jnz .Lmcck_user
477 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_PSW_IA_VALID
478 jno .Lmcck_panic
479#if IS_ENABLED(CONFIG_KVM)
480 lg %r10,__LC_CURRENT(%r13)
481 tm __TI_sie(%r10),0xff
482 jz .Lmcck_user
483 # Need to compare the address instead of __TI_SIE flag.
484 # Otherwise there would be a race between setting the flag
485 # and entering SIE (or leaving and clearing the flag). This
486 # would cause machine checks targeted at the guest to be
487 # handled by the host.
488 larl %r14,.Lsie_entry
489 clgrjl %r9,%r14, 4f
490 larl %r14,.Lsie_leave
491 clgrjhe %r9,%r14, 4f
492 lg %r10,__LC_PCPU(%r13)
493 oi __PCPU_FLAGS+7(%r10), _CIF_MCCK_GUEST
4944: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
495 SIEEXIT __SF_SIE_CONTROL(%r15),%r13
496#endif
497.Lmcck_user:
498 lg %r15,__LC_MCCK_STACK(%r13)
499 la %r11,STACK_FRAME_OVERHEAD(%r15)
500 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
501 lay %r14,__LC_GPREGS_SAVE_AREA(%r13)
502 mvc __PT_R0(128,%r11),0(%r14)
503 # clear user controlled registers to prevent speculative use
504 xgr %r0,%r0
505 xgr %r1,%r1
506 xgr %r3,%r3
507 xgr %r4,%r4
508 xgr %r5,%r5
509 xgr %r6,%r6
510 xgr %r7,%r7
511 xgr %r10,%r10
512 xgr %r12,%r12
513 stmg %r8,%r9,__PT_PSW(%r11)
514 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
515 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
516 lgr %r2,%r11 # pass pointer to pt_regs
517 brasl %r14,s390_do_machine_check
518 lmg %r0,%r10,__PT_R0(%r11)
519 mvc __LC_RETURN_MCCK_PSW(16,%r13),__PT_PSW(%r11) # move return PSW
520 tm __LC_RETURN_MCCK_PSW+1(%r13),0x01 # returning to user ?
521 jno 0f
522 BPON
523 stpt __LC_EXIT_TIMER(%r13)
5240: ALTERNATIVE "brcl 0,0", __stringify(lay %r12,__LC_LAST_BREAK_SAVE_AREA(%r13)),\
525 ALT_FACILITY(193)
526 LBEAR 0(%r12)
527 lmg %r11,%r15,__PT_R11(%r11)
528 LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
529
530.Lmcck_panic:
531 /*
532 * Iterate over all possible CPU addresses in the range 0..0xffff
533 * and stop each CPU using signal processor. Use compare and swap
534 * to allow just one CPU-stopper and prevent concurrent CPUs from
535 * stopping each other while leaving the others running.
536 */
537 lhi %r5,0
538 lhi %r6,1
539 larl %r7,stop_lock
540 cs %r5,%r6,0(%r7) # single CPU-stopper only
541 jnz 4f
542 larl %r7,this_cpu
543 stap 0(%r7) # this CPU address
544 lh %r4,0(%r7)
545 nilh %r4,0
546 lhi %r0,1
547 sll %r0,16 # CPU counter
548 lhi %r3,0 # next CPU address
5490: cr %r3,%r4
550 je 2f
5511: sigp %r1,%r3,SIGP_STOP # stop next CPU
552 brc SIGP_CC_BUSY,1b
5532: ahi %r3,1
554 brct %r0,0b
5553: sigp %r1,%r4,SIGP_STOP # stop this CPU
556 brc SIGP_CC_BUSY,3b
5574: j 4b
558SYM_CODE_END(mcck_int_handler)
559
560SYM_CODE_START(restart_int_handler)
561 ALTERNATIVE "nop", "lpp _LPP_OFFSET", ALT_FACILITY(40)
562 stg %r15,__LC_SAVE_AREA_RESTART
563 TSTMSK __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
564 jz 0f
565 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA
5660: larl %r15,daton_psw
567 lpswe 0(%r15) # turn dat on, keep irqs off
568.Ldaton:
569 GET_LC %r15
570 lg %r15,__LC_RESTART_STACK(%r15)
571 xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
572 stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
573 GET_LC %r13
574 mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART(%r13)
575 mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW(%r13)
576 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
577 lg %r1,__LC_RESTART_FN(%r13) # load fn, parm & source cpu
578 lg %r2,__LC_RESTART_DATA(%r13)
579 lgf %r3,__LC_RESTART_SOURCE(%r13)
580 ltgr %r3,%r3 # test source cpu address
581 jm 1f # negative -> skip source stop
5820: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
583 brc 10,0b # wait for status stored
5841: basr %r14,%r1 # call function
585 stap __SF_EMPTY(%r15) # store cpu address
586 llgh %r3,__SF_EMPTY(%r15)
5872: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
588 brc 2,2b
5893: j 3b
590SYM_CODE_END(restart_int_handler)
591
592 __INIT
593SYM_CODE_START(early_pgm_check_handler)
594 STMG_LC %r8,%r15,__LC_SAVE_AREA
595 GET_LC %r13
596 aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE)
597 la %r11,STACK_FRAME_OVERHEAD(%r15)
598 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
599 stmg %r0,%r7,__PT_R0(%r11)
600 mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW(%r13)
601 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13)
602 lgr %r2,%r11
603 brasl %r14,__do_early_pgm_check
604 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
605 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
606 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
607SYM_CODE_END(early_pgm_check_handler)
608 __FINIT
609
610 .section .kprobes.text, "ax"
611
612/*
613 * The synchronous or the asynchronous stack pointer is invalid. We are dead.
614 * No need to properly save the registers, we are going to panic anyway.
615 * Setup a pt_regs so that show_trace can provide a good call trace.
616 * Copy lowcore of the current CPU to stack_invalid_lowcore before it
617 * is modified by printing debug messages and crash handling.
618 */
619SYM_CODE_START(stack_invalid)
620 GET_LC %r15
621 lg %r15,__LC_NODAT_STACK(%r15) # change to panic stack
622 la %r11,STACK_FRAME_OVERHEAD(%r15)
623 stmg %r0,%r7,__PT_R0(%r11)
624 stmg %r8,%r9,__PT_PSW(%r11)
625 mvc __PT_R8(64,%r11),0(%r14)
626 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
627 GET_LC %r2
628 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r2)
629 /* copy lowcore; skip if used by another CPU */
630 larl %r4,stack_invalid_lowcore_used
631 lhi %r1,0
632 lhi %r3,1
633 cs %r1,%r3,0(%r4)
634 jne .Lskip
635 larl %r0,stack_invalid_lowcore
636 lghi %r1,LC_PAGES * PAGE_SIZE
637 lgr %r4,%r2
638 lgr %r5,%r1
639 mvcl %r0,%r4
640.Lskip:
641 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
642 lgr %r2,%r11 # pass pointer to pt_regs
643 jg kernel_stack_invalid
644SYM_CODE_END(stack_invalid)
645
646 .section .data, "aw"
647 .balign 4
648SYM_DATA_LOCAL(stop_lock, .long 0)
649SYM_DATA_LOCAL(this_cpu, .short 0)
650 .balign 8
651SYM_DATA_START_LOCAL(daton_psw)
652 .quad PSW_KERNEL_BITS
653 .quad .Ldaton
654SYM_DATA_END(daton_psw)