projects/linux·blob master · refs

579 lines · 13.9 KiB

1/*
2 * arch/xtensa/kernel/align.S
3 *
4 * Handle unalignment and load/store exceptions.
5 *
6 * This file is subject to the terms and conditions of the GNU General
7 * Public License. See the file "COPYING" in the main directory of
8 * this archive for more details.
9 *
10 * Copyright (C) 2001 - 2005 Tensilica, Inc.
11 * Copyright (C) 2014 Cadence Design Systems Inc.
12 *
13 * Rewritten by Chris Zankel <chris@zankel.net>
14 *
15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca>
17 */
18
19#include <linux/linkage.h>
20#include <asm/current.h>
21#include <asm/asm-offsets.h>
22#include <asm/asmmacro.h>
23#include <asm/processor.h>
24#ifdef CONFIG_MMU
25#include <asm/asm-uaccess.h>
26#endif
27
28#if XCHAL_UNALIGNED_LOAD_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE
29#define LOAD_EXCEPTION_HANDLER
30#endif
31
32#if XCHAL_UNALIGNED_STORE_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE
33#define STORE_EXCEPTION_HANDLER
34#endif
35
36#if defined LOAD_EXCEPTION_HANDLER || defined STORE_EXCEPTION_HANDLER
37#define ANY_EXCEPTION_HANDLER
38#endif
39
40#if XCHAL_HAVE_WINDOWED && defined CONFIG_MMU
41#define UNALIGNED_USER_EXCEPTION
42#endif
43
44/* Big and little endian 16-bit values are located in
45 * different halves of a register. HWORD_START helps to
46 * abstract the notion of extracting a 16-bit value from a
47 * register.
48 * We also have to define new shifting instructions because
49 * lsb and msb are on 'opposite' ends in a register for
50 * different endian machines.
51 *
52 * Assume a memory region in ascending address:
53 * 0 1 2 3|4 5 6 7
54 *
55 * When loading one word into a register, the content of that register is:
56 * LE 3 2 1 0, 7 6 5 4
57 * BE 0 1 2 3, 4 5 6 7
58 *
59 * Masking the bits of the higher/lower address means:
60 * LE X X 0 0, 0 0 X X
61 * BE 0 0 X X, X X 0 0
62 *
63 * Shifting to higher/lower addresses, means:
64 * LE shift left / shift right
65 * BE shift right / shift left
66 *
67 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means:
68 * LE mask 0 0 X X / shift left
69 * BE shift left / mask 0 0 X X
70 */
71
72#if XCHAL_HAVE_BE
73
74#define HWORD_START 16
75#define INSN_OP0 28
76#define INSN_T 24
77#define INSN_OP1 16
78
79.macro __ssa8r r; ssa8l \r; .endm
80.macro __sh r, s; srl \r, \s; .endm
81.macro __sl r, s; sll \r, \s; .endm
82.macro __exth r, s; extui \r, \s, 0, 16; .endm
83.macro __extl r, s; slli \r, \s, 16; .endm
84
85#else
86
87#define HWORD_START 0
88#define INSN_OP0 0
89#define INSN_T 4
90#define INSN_OP1 12
91
92.macro __ssa8r r; ssa8b \r; .endm
93.macro __sh r, s; sll \r, \s; .endm
94.macro __sl r, s; srl \r, \s; .endm
95.macro __exth r, s; slli \r, \s, 16; .endm
96.macro __extl r, s; extui \r, \s, 0, 16; .endm
97
98#endif
99
100/*
101 * xxxx xxxx = imm8 field
102 * yyyy = imm4 field
103 * ssss = s field
104 * tttt = t field
105 *
106 * 16 0
107 * -------------------
108 * L32I.N yyyy ssss tttt 1000
109 * S32I.N yyyy ssss tttt 1001
110 *
111 * 23 0
112 * -----------------------------
113 * L8UI xxxx xxxx 0000 ssss tttt 0010
114 * L16UI xxxx xxxx 0001 ssss tttt 0010
115 * L32I xxxx xxxx 0010 ssss tttt 0010
116 * XXX 0011 ssss tttt 0010
117 * XXX 0100 ssss tttt 0010
118 * S16I xxxx xxxx 0101 ssss tttt 0010
119 * S32I xxxx xxxx 0110 ssss tttt 0010
120 * XXX 0111 ssss tttt 0010
121 * XXX 1000 ssss tttt 0010
122 * L16SI xxxx xxxx 1001 ssss tttt 0010
123 * XXX 1010 0010
124 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported
125 * XXX 1100 0010
126 * XXX 1101 0010
127 * XXX 1110 0010
128 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported
129 * -----------------------------
130 * ^ ^ ^
131 * sub-opcode (NIBBLE_R) -+ | |
132 * t field (NIBBLE_T) -----------+ |
133 * major opcode (NIBBLE_OP0) --------------+
134 */
135
136#define OP0_L32I_N 0x8 /* load immediate narrow */
137#define OP0_S32I_N 0x9 /* store immediate narrow */
138#define OP0_LSAI 0x2 /* load/store */
139#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */
140#define OP1_SI_BIT 2 /* OP1 bit number for stores */
141
142#define OP1_L8UI 0x0
143#define OP1_L32I 0x2
144#define OP1_L16UI 0x1
145#define OP1_L16SI 0x9
146#define OP1_L32AI 0xb
147
148#define OP1_S32I 0x6
149#define OP1_S16I 0x5
150#define OP1_S32RI 0xf
151
152/*
153 * Entry condition:
154 *
155 * a0: trashed, original value saved on stack (PT_AREG0)
156 * a1: a1
157 * a2: new stack pointer, original in DEPC
158 * a3: a3
159 * depc: a2, original value saved on stack (PT_DEPC)
160 * excsave_1: dispatch table
161 *
162 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
163 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
164 */
165
166 .literal_position
167#ifdef CONFIG_XTENSA_LOAD_STORE
168ENTRY(fast_load_store)
169
170 call0 .Lsave_and_load_instruction
171
172 /* Analyze the instruction (load or store?). */
173
174 extui a0, a4, INSN_OP0, 4 # get insn.op0 nibble
175
176#if XCHAL_HAVE_DENSITY
177 _beqi a0, OP0_L32I_N, 1f # L32I.N, jump
178#endif
179 bnei a0, OP0_LSAI, .Linvalid_instruction
180 /* 'store indicator bit' set, jump */
181 bbsi.l a4, OP1_SI_BIT + INSN_OP1, .Linvalid_instruction
182
1831:
184 movi a3, ~3
185 and a3, a3, a8 # align memory address
186
187 __ssa8 a8
188
189#ifdef CONFIG_MMU
190 /* l32e can't be used here even when it's available. */
191 rsr a6, ps
192 bbci.l a6, PS_UM_BIT, 1f # kernel mode
193 movi a5, 8
194 access_ok a3, a5, a6, a2, .Linvalid_instruction
1951:
196#endif
197 l32i a5, a3, 0
198 l32i a6, a3, 4
199 __src_b a3, a5, a6 # a3 has the data word
200
201#if XCHAL_HAVE_DENSITY
202 addi a7, a7, 2 # increment PC (assume 16-bit insn)
203 _beqi a0, OP0_L32I_N, .Lload_w# l32i.n: jump
204 addi a7, a7, 1
205#else
206 addi a7, a7, 3
207#endif
208
209 extui a5, a4, INSN_OP1, 4
210 _beqi a5, OP1_L32I, .Lload_w
211 bnei a5, OP1_L8UI, .Lload16
212 extui a3, a3, 0, 8
213 j .Lload_w
214
215ENDPROC(fast_load_store)
216#endif
217
218/*
219 * Entry condition:
220 *
221 * a0: trashed, original value saved on stack (PT_AREG0)
222 * a1: a1
223 * a2: new stack pointer, original in DEPC
224 * a3: a3
225 * depc: a2, original value saved on stack (PT_DEPC)
226 * excsave_1: dispatch table
227 *
228 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
229 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
230 */
231
232#ifdef ANY_EXCEPTION_HANDLER
233ENTRY(fast_unaligned)
234
235 call0 .Lsave_and_load_instruction
236
237 /* Analyze the instruction (load or store?). */
238
239 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble
240
241#if XCHAL_HAVE_DENSITY
242 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump
243 addi a6, a5, -OP0_S32I_N
244 _beqz a6, .Lstore # S32I.N, do a store
245#endif
246 /* 'store indicator bit' not set, jump */
247 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload
248
249#ifdef STORE_EXCEPTION_HANDLER
250
251 /* Store: Jump to table entry to get the value in the source register.*/
252
253.Lstore:movi a5, .Lstore_table # table
254 extui a6, a4, INSN_T, 4 # get source register
255 addx8 a5, a6, a5
256 jx a5 # jump into table
257#endif
258#ifdef LOAD_EXCEPTION_HANDLER
259
260 /* Load: Load memory address. */
261
262.Lload: movi a3, ~3
263 and a3, a3, a8 # align memory address
264
265 __ssa8 a8
266#ifdef UNALIGNED_USER_EXCEPTION
267 addi a3, a3, 8
268 l32e a5, a3, -8
269 l32e a6, a3, -4
270#else
271 l32i a5, a3, 0
272 l32i a6, a3, 4
273#endif
274 __src_b a3, a5, a6 # a3 has the data word
275
276#if XCHAL_HAVE_DENSITY
277 addi a7, a7, 2 # increment PC (assume 16-bit insn)
278
279 extui a5, a4, INSN_OP0, 4
280 _beqi a5, OP0_L32I_N, .Lload_w# l32i.n: jump
281
282 addi a7, a7, 1
283#else
284 addi a7, a7, 3
285#endif
286
287 extui a5, a4, INSN_OP1, 4
288 _beqi a5, OP1_L32I, .Lload_w # l32i: jump
289#endif
290#ifdef LOAD_EXCEPTION_HANDLER
291.Lload16:
292 extui a3, a3, 0, 16 # extract lower 16 bits
293 _beqi a5, OP1_L16UI, .Lload_w
294 addi a5, a5, -OP1_L16SI
295 _bnez a5, .Linvalid_instruction
296
297 /* sign extend value */
298#if XCHAL_HAVE_SEXT
299 sext a3, a3, 15
300#else
301 slli a3, a3, 16
302 srai a3, a3, 16
303#endif
304
305 /* Set target register. */
306
307.Lload_w:
308 extui a4, a4, INSN_T, 4 # extract target register
309 movi a5, .Lload_table
310 addx8 a4, a4, a5
311 jx a4 # jump to entry for target register
312
313 .align 8
314.Lload_table:
315 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8
316 mov a1, a3; _j .Lexit; .align 8 # fishy??
317 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8
318 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8
319 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8
320 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8
321 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8
322 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8
323 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8
324 mov a9, a3 ; _j .Lexit; .align 8
325 mov a10, a3 ; _j .Lexit; .align 8
326 mov a11, a3 ; _j .Lexit; .align 8
327 mov a12, a3 ; _j .Lexit; .align 8
328 mov a13, a3 ; _j .Lexit; .align 8
329 mov a14, a3 ; _j .Lexit; .align 8
330 mov a15, a3 ; _j .Lexit; .align 8
331#endif
332#ifdef STORE_EXCEPTION_HANDLER
333.Lstore_table:
334 l32i a3, a2, PT_AREG0; _j .Lstore_w; .align 8
335 mov a3, a1; _j .Lstore_w; .align 8 # fishy??
336 l32i a3, a2, PT_AREG2; _j .Lstore_w; .align 8
337 l32i a3, a2, PT_AREG3; _j .Lstore_w; .align 8
338 l32i a3, a2, PT_AREG4; _j .Lstore_w; .align 8
339 l32i a3, a2, PT_AREG5; _j .Lstore_w; .align 8
340 l32i a3, a2, PT_AREG6; _j .Lstore_w; .align 8
341 l32i a3, a2, PT_AREG7; _j .Lstore_w; .align 8
342 l32i a3, a2, PT_AREG8; _j .Lstore_w; .align 8
343 mov a3, a9 ; _j .Lstore_w; .align 8
344 mov a3, a10 ; _j .Lstore_w; .align 8
345 mov a3, a11 ; _j .Lstore_w; .align 8
346 mov a3, a12 ; _j .Lstore_w; .align 8
347 mov a3, a13 ; _j .Lstore_w; .align 8
348 mov a3, a14 ; _j .Lstore_w; .align 8
349 mov a3, a15 ; _j .Lstore_w; .align 8
350#endif
351
352 /* We cannot handle this exception. */
353
354 .extern _kernel_exception
355.Linvalid_instruction:
356
357 movi a4, 0
358 rsr a3, excsave1
359 s32i a4, a3, EXC_TABLE_FIXUP
360
361 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */
362
363 l32i a0, a2, PT_SAR
364 l32i a8, a2, PT_AREG8
365 l32i a7, a2, PT_AREG7
366 l32i a6, a2, PT_AREG6
367 l32i a5, a2, PT_AREG5
368 l32i a4, a2, PT_AREG4
369 wsr a0, sar
370 mov a1, a2
371
372 rsr a0, ps
373 bbsi.l a0, PS_UM_BIT, 2f # jump if user mode
374
375 movi a0, _kernel_exception
376 jx a0
377
3782: movi a0, _user_exception
379 jx a0
380
381#ifdef STORE_EXCEPTION_HANDLER
382
383 # a7: instruction pointer, a4: instruction, a3: value
384.Lstore_w:
385 movi a6, 0 # mask: ffffffff:00000000
386
387#if XCHAL_HAVE_DENSITY
388 addi a7, a7, 2 # incr. PC,assume 16-bit instruction
389
390 extui a5, a4, INSN_OP0, 4 # extract OP0
391 addi a5, a5, -OP0_S32I_N
392 _beqz a5, 1f # s32i.n: jump
393
394 addi a7, a7, 1 # increment PC, 32-bit instruction
395#else
396 addi a7, a7, 3 # increment PC, 32-bit instruction
397#endif
398
399 extui a5, a4, INSN_OP1, 4 # extract OP1
400 _beqi a5, OP1_S32I, 1f # jump if 32 bit store
401 _bnei a5, OP1_S16I, .Linvalid_instruction
402
403 movi a5, -1
404 __extl a3, a3 # get 16-bit value
405 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000
406
407 /* Get memory address */
408
4091:
410 movi a4, ~3
411 and a4, a4, a8 # align memory address
412
413 /* Insert value into memory */
414
415 movi a5, -1 # mask: ffffffff:XXXX0000
416#ifdef UNALIGNED_USER_EXCEPTION
417 addi a4, a4, 8
418#endif
419
420 __ssa8r a8
421 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE)
422 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE)
423#ifdef UNALIGNED_USER_EXCEPTION
424 l32e a5, a4, -8
425#else
426 l32i a5, a4, 0 # load lower address word
427#endif
428 and a5, a5, a8 # mask
429 __sh a8, a3 # shift value
430 or a5, a5, a8 # or with original value
431#ifdef UNALIGNED_USER_EXCEPTION
432 s32e a5, a4, -8
433 l32e a8, a4, -4
434#else
435 s32i a5, a4, 0 # store
436 l32i a8, a4, 4 # same for upper address word
437#endif
438 __sl a5, a3
439 and a6, a8, a6
440 or a6, a6, a5
441#ifdef UNALIGNED_USER_EXCEPTION
442 s32e a6, a4, -4
443#else
444 s32i a6, a4, 4
445#endif
446#endif
447
448.Lexit:
449#if XCHAL_HAVE_LOOPS
450 rsr a4, lend # check if we reached LEND
451 bne a7, a4, 1f
452 rsr a4, lcount # and LCOUNT != 0
453 beqz a4, 1f
454 addi a4, a4, -1 # decrement LCOUNT and set
455 rsr a7, lbeg # set PC to LBEGIN
456 wsr a4, lcount
457#endif
458
4591: wsr a7, epc1 # skip emulated instruction
460
461 /* Update icount if we're single-stepping in userspace. */
462 rsr a4, icountlevel
463 beqz a4, 1f
464 bgeui a4, LOCKLEVEL + 1, 1f
465 rsr a4, icount
466 addi a4, a4, 1
467 wsr a4, icount
4681:
469 movi a4, 0
470 rsr a3, excsave1
471 s32i a4, a3, EXC_TABLE_FIXUP
472
473 /* Restore working register */
474
475 l32i a0, a2, PT_SAR
476 l32i a8, a2, PT_AREG8
477 l32i a7, a2, PT_AREG7
478 l32i a6, a2, PT_AREG6
479 l32i a5, a2, PT_AREG5
480 l32i a4, a2, PT_AREG4
481 l32i a3, a2, PT_AREG3
482
483 /* restore SAR and return */
484
485 wsr a0, sar
486 l32i a0, a2, PT_AREG0
487 l32i a2, a2, PT_AREG2
488 rfe
489
490 .align 4
491.Lsave_and_load_instruction:
492
493 /* Save some working register */
494
495 s32i a3, a2, PT_AREG3
496 s32i a4, a2, PT_AREG4
497 s32i a5, a2, PT_AREG5
498 s32i a6, a2, PT_AREG6
499 s32i a7, a2, PT_AREG7
500 s32i a8, a2, PT_AREG8
501
502 rsr a4, depc
503 s32i a4, a2, PT_AREG2
504
505 rsr a5, sar
506 s32i a5, a2, PT_SAR
507
508 rsr a3, excsave1
509 movi a4, fast_unaligned_fixup
510 s32i a4, a3, EXC_TABLE_FIXUP
511
512 rsr a8, excvaddr # load unaligned memory address
513
514 /* Now, identify one of the following load/store instructions.
515 *
516 * The only possible danger of a double exception on the
517 * following l32i instructions is kernel code in vmalloc
518 * memory. The processor was just executing at the EPC_1
519 * address, and indeed, already fetched the instruction. That
520 * guarantees a TLB mapping, which hasn't been replaced by
521 * this unaligned exception handler that uses only static TLB
522 * mappings. However, high-level interrupt handlers might
523 * modify TLB entries, so for the generic case, we register a
524 * TABLE_FIXUP handler here, too.
525 */
526
527 /* a3...a6 saved on stack, a2 = SP */
528
529 /* Extract the instruction that caused the unaligned access. */
530
531 rsr a7, epc1 # load exception address
532 movi a3, ~3
533 and a3, a3, a7 # mask lower bits
534
535 l32i a4, a3, 0 # load 2 words
536 l32i a5, a3, 4
537
538 __ssa8 a7
539 __src_b a4, a4, a5 # a4 has the instruction
540
541 ret
542
543ENDPROC(fast_unaligned)
544
545ENTRY(fast_unaligned_fixup)
546
547 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
548 wsr a3, excsave1
549
550 l32i a8, a2, PT_AREG8
551 l32i a7, a2, PT_AREG7
552 l32i a6, a2, PT_AREG6
553 l32i a5, a2, PT_AREG5
554 l32i a4, a2, PT_SAR
555 l32i a0, a2, PT_AREG2
556 wsr a4, sar
557 wsr a0, depc # restore depc and a0
558 l32i a4, a2, PT_AREG4
559
560 rsr a0, exccause
561 s32i a0, a2, PT_DEPC # mark as a regular exception
562
563 rsr a0, ps
564 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode
565
566 rsr a0, exccause
567 addx4 a0, a0, a3 # find entry in table
568 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler
569 l32i a3, a2, PT_AREG3
570 jx a0
5711:
572 rsr a0, exccause
573 addx4 a0, a0, a3 # find entry in table
574 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
575 l32i a3, a2, PT_AREG3
576 jx a0
577
578ENDPROC(fast_unaligned_fixup)
579#endif