projects/linux·blob master · refs

1295 lines · 31.4 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * PARISC TLB and cache flushing support
4 * Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
5 * Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
6 * Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
7 */
8
9/*
10 * NOTE: fdc,fic, and pdc instructions that use base register modification
11 * should only use index and base registers that are not shadowed,
12 * so that the fast path emulation in the non access miss handler
13 * can be used.
14 */
15
16#ifdef CONFIG_64BIT
17 .level 2.0w
18#else
19 .level 2.0
20#endif
21
22#include <asm/psw.h>
23#include <asm/assembly.h>
24#include <asm/cache.h>
25#include <asm/ldcw.h>
26#include <asm/alternative.h>
27#include <linux/linkage.h>
28#include <linux/init.h>
29#include <linux/pgtable.h>
30
31 .section .text.hot
32 .align 16
33
34ENTRY_CFI(flush_tlb_all_local)
35 /*
36 * The pitlbe and pdtlbe instructions should only be used to
37 * flush the entire tlb. Also, there needs to be no intervening
38 * tlb operations, e.g. tlb misses, so the operation needs
39 * to happen in real mode with all interruptions disabled.
40 */
41
42 /* pcxt_ssm_bug - relied upon translation! PA 2.0 Arch. F-4 and F-5 */
43 rsm PSW_SM_I, %r19 /* save I-bit state */
44 load32 PA(1f), %r1
45 nop
46 nop
47 nop
48 nop
49 nop
50
51 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
52 mtctl %r0, %cr17 /* Clear IIASQ tail */
53 mtctl %r0, %cr17 /* Clear IIASQ head */
54 mtctl %r1, %cr18 /* IIAOQ head */
55 ldo 4(%r1), %r1
56 mtctl %r1, %cr18 /* IIAOQ tail */
57 load32 REAL_MODE_PSW, %r1
58 mtctl %r1, %ipsw
59 rfi
60 nop
61
621: load32 PA(cache_info), %r1
63
64 /* Flush Instruction Tlb */
65
6688: LDREG ITLB_SID_BASE(%r1), %r20
67 LDREG ITLB_SID_STRIDE(%r1), %r21
68 LDREG ITLB_SID_COUNT(%r1), %r22
69 LDREG ITLB_OFF_BASE(%r1), %arg0
70 LDREG ITLB_OFF_STRIDE(%r1), %arg1
71 LDREG ITLB_OFF_COUNT(%r1), %arg2
72 LDREG ITLB_LOOP(%r1), %arg3
73
74 addib,COND(=) -1, %arg3, fitoneloop /* Preadjust and test */
75 movb,<,n %arg3, %r31, fitdone /* If loop < 0, skip */
76 copy %arg0, %r28 /* Init base addr */
77
78fitmanyloop: /* Loop if LOOP >= 2 */
79 mtsp %r20, %sr1
80 add %r21, %r20, %r20 /* increment space */
81 copy %arg2, %r29 /* Init middle loop count */
82
83fitmanymiddle: /* Loop if LOOP >= 2 */
84 addib,COND(>) -1, %r31, fitmanymiddle /* Adjusted inner loop decr */
85 pitlbe %r0(%sr1, %r28)
86 pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */
87 addib,COND(>) -1, %r29, fitmanymiddle /* Middle loop decr */
88 copy %arg3, %r31 /* Re-init inner loop count */
89
90 movb,tr %arg0, %r28, fitmanyloop /* Re-init base addr */
91 addib,COND(<=),n -1, %r22, fitdone /* Outer loop count decr */
92
93fitoneloop: /* Loop if LOOP = 1 */
94 mtsp %r20, %sr1
95 copy %arg0, %r28 /* init base addr */
96 copy %arg2, %r29 /* init middle loop count */
97
98fitonemiddle: /* Loop if LOOP = 1 */
99 addib,COND(>) -1, %r29, fitonemiddle /* Middle loop count decr */
100 pitlbe,m %arg1(%sr1, %r28) /* pitlbe for one loop */
101
102 addib,COND(>) -1, %r22, fitoneloop /* Outer loop count decr */
103 add %r21, %r20, %r20 /* increment space */
104
105fitdone:
106 ALTERNATIVE(88b, fitdone, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
107
108 /* Flush Data Tlb */
109
110 LDREG DTLB_SID_BASE(%r1), %r20
111 LDREG DTLB_SID_STRIDE(%r1), %r21
112 LDREG DTLB_SID_COUNT(%r1), %r22
113 LDREG DTLB_OFF_BASE(%r1), %arg0
114 LDREG DTLB_OFF_STRIDE(%r1), %arg1
115 LDREG DTLB_OFF_COUNT(%r1), %arg2
116 LDREG DTLB_LOOP(%r1), %arg3
117
118 addib,COND(=) -1, %arg3, fdtoneloop /* Preadjust and test */
119 movb,<,n %arg3, %r31, fdtdone /* If loop < 0, skip */
120 copy %arg0, %r28 /* Init base addr */
121
122fdtmanyloop: /* Loop if LOOP >= 2 */
123 mtsp %r20, %sr1
124 add %r21, %r20, %r20 /* increment space */
125 copy %arg2, %r29 /* Init middle loop count */
126
127fdtmanymiddle: /* Loop if LOOP >= 2 */
128 addib,COND(>) -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */
129 pdtlbe %r0(%sr1, %r28)
130 pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */
131 addib,COND(>) -1, %r29, fdtmanymiddle /* Middle loop decr */
132 copy %arg3, %r31 /* Re-init inner loop count */
133
134 movb,tr %arg0, %r28, fdtmanyloop /* Re-init base addr */
135 addib,COND(<=),n -1, %r22,fdtdone /* Outer loop count decr */
136
137fdtoneloop: /* Loop if LOOP = 1 */
138 mtsp %r20, %sr1
139 copy %arg0, %r28 /* init base addr */
140 copy %arg2, %r29 /* init middle loop count */
141
142fdtonemiddle: /* Loop if LOOP = 1 */
143 addib,COND(>) -1, %r29, fdtonemiddle /* Middle loop count decr */
144 pdtlbe,m %arg1(%sr1, %r28) /* pdtlbe for one loop */
145
146 addib,COND(>) -1, %r22, fdtoneloop /* Outer loop count decr */
147 add %r21, %r20, %r20 /* increment space */
148
149
150fdtdone:
151 /*
152 * Switch back to virtual mode
153 */
154 /* pcxt_ssm_bug */
155 rsm PSW_SM_I, %r0
156 load32 2f, %r1
157 nop
158 nop
159 nop
160 nop
161 nop
162
163 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
164 mtctl %r0, %cr17 /* Clear IIASQ tail */
165 mtctl %r0, %cr17 /* Clear IIASQ head */
166 mtctl %r1, %cr18 /* IIAOQ head */
167 ldo 4(%r1), %r1
168 mtctl %r1, %cr18 /* IIAOQ tail */
169 load32 KERNEL_PSW, %r1
170 or %r1, %r19, %r1 /* I-bit to state on entry */
171 mtctl %r1, %ipsw /* restore I-bit (entire PSW) */
172 rfi
173 nop
174
1752: bv %r0(%r2)
176 nop
177
178 /*
179 * When running in qemu, drop whole flush_tlb_all_local function and
180 * replace by one pdtlbe instruction, for which QEMU will drop all
181 * local TLB entries.
182 */
1833: pdtlbe %r0(%sr1,%r0)
184 bv,n %r0(%r2)
185 ALTERNATIVE_CODE(flush_tlb_all_local, 2, ALT_COND_RUN_ON_QEMU, 3b)
186ENDPROC_CFI(flush_tlb_all_local)
187
188 .import cache_info,data
189
190ENTRY_CFI(flush_instruction_cache_local)
19188: load32 cache_info, %r1
192
193 /* Flush Instruction Cache */
194
195 LDREG ICACHE_BASE(%r1), %arg0
196 LDREG ICACHE_STRIDE(%r1), %arg1
197 LDREG ICACHE_COUNT(%r1), %arg2
198 LDREG ICACHE_LOOP(%r1), %arg3
199 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
200 mtsp %r0, %sr1
201 addib,COND(=) -1, %arg3, fioneloop /* Preadjust and test */
202 movb,<,n %arg3, %r31, fisync /* If loop < 0, do sync */
203
204fimanyloop: /* Loop if LOOP >= 2 */
205 addib,COND(>) -1, %r31, fimanyloop /* Adjusted inner loop decr */
206 fice %r0(%sr1, %arg0)
207 fice,m %arg1(%sr1, %arg0) /* Last fice and addr adjust */
208 movb,tr %arg3, %r31, fimanyloop /* Re-init inner loop count */
209 addib,COND(<=),n -1, %arg2, fisync /* Outer loop decr */
210
211fioneloop: /* Loop if LOOP = 1 */
212 /* Some implementations may flush with a single fice instruction */
213 cmpib,COND(>>=),n 15, %arg2, fioneloop2
214
215fioneloop1:
216 fice,m %arg1(%sr1, %arg0)
217 fice,m %arg1(%sr1, %arg0)
218 fice,m %arg1(%sr1, %arg0)
219 fice,m %arg1(%sr1, %arg0)
220 fice,m %arg1(%sr1, %arg0)
221 fice,m %arg1(%sr1, %arg0)
222 fice,m %arg1(%sr1, %arg0)
223 fice,m %arg1(%sr1, %arg0)
224 fice,m %arg1(%sr1, %arg0)
225 fice,m %arg1(%sr1, %arg0)
226 fice,m %arg1(%sr1, %arg0)
227 fice,m %arg1(%sr1, %arg0)
228 fice,m %arg1(%sr1, %arg0)
229 fice,m %arg1(%sr1, %arg0)
230 fice,m %arg1(%sr1, %arg0)
231 addib,COND(>) -16, %arg2, fioneloop1
232 fice,m %arg1(%sr1, %arg0)
233
234 /* Check if done */
235 cmpb,COND(=),n %arg2, %r0, fisync /* Predict branch taken */
236
237fioneloop2:
238 addib,COND(>) -1, %arg2, fioneloop2 /* Outer loop count decr */
239 fice,m %arg1(%sr1, %arg0) /* Fice for one loop */
240
241fisync:
242 sync
243 mtsm %r22 /* restore I-bit */
24489: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
245 bv %r0(%r2)
246 nop
247ENDPROC_CFI(flush_instruction_cache_local)
248
249
250 .import cache_info, data
251ENTRY_CFI(flush_data_cache_local)
25288: load32 cache_info, %r1
253
254 /* Flush Data Cache */
255
256 LDREG DCACHE_BASE(%r1), %arg0
257 LDREG DCACHE_STRIDE(%r1), %arg1
258 LDREG DCACHE_COUNT(%r1), %arg2
259 LDREG DCACHE_LOOP(%r1), %arg3
260 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
261 mtsp %r0, %sr1
262 addib,COND(=) -1, %arg3, fdoneloop /* Preadjust and test */
263 movb,<,n %arg3, %r31, fdsync /* If loop < 0, do sync */
264
265fdmanyloop: /* Loop if LOOP >= 2 */
266 addib,COND(>) -1, %r31, fdmanyloop /* Adjusted inner loop decr */
267 fdce %r0(%sr1, %arg0)
268 fdce,m %arg1(%sr1, %arg0) /* Last fdce and addr adjust */
269 movb,tr %arg3, %r31, fdmanyloop /* Re-init inner loop count */
270 addib,COND(<=),n -1, %arg2, fdsync /* Outer loop decr */
271
272fdoneloop: /* Loop if LOOP = 1 */
273 /* Some implementations may flush with a single fdce instruction */
274 cmpib,COND(>>=),n 15, %arg2, fdoneloop2
275
276fdoneloop1:
277 fdce,m %arg1(%sr1, %arg0)
278 fdce,m %arg1(%sr1, %arg0)
279 fdce,m %arg1(%sr1, %arg0)
280 fdce,m %arg1(%sr1, %arg0)
281 fdce,m %arg1(%sr1, %arg0)
282 fdce,m %arg1(%sr1, %arg0)
283 fdce,m %arg1(%sr1, %arg0)
284 fdce,m %arg1(%sr1, %arg0)
285 fdce,m %arg1(%sr1, %arg0)
286 fdce,m %arg1(%sr1, %arg0)
287 fdce,m %arg1(%sr1, %arg0)
288 fdce,m %arg1(%sr1, %arg0)
289 fdce,m %arg1(%sr1, %arg0)
290 fdce,m %arg1(%sr1, %arg0)
291 fdce,m %arg1(%sr1, %arg0)
292 addib,COND(>) -16, %arg2, fdoneloop1
293 fdce,m %arg1(%sr1, %arg0)
294
295 /* Check if done */
296 cmpb,COND(=),n %arg2, %r0, fdsync /* Predict branch taken */
297
298fdoneloop2:
299 addib,COND(>) -1, %arg2, fdoneloop2 /* Outer loop count decr */
300 fdce,m %arg1(%sr1, %arg0) /* Fdce for one loop */
301
302fdsync:
303 sync
304 mtsm %r22 /* restore I-bit */
30589: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
306 bv %r0(%r2)
307 nop
308ENDPROC_CFI(flush_data_cache_local)
309
310/* Clear page using kernel mapping. */
311
312ENTRY_CFI(clear_page_asm)
313#ifdef CONFIG_64BIT
314
315 /* Unroll the loop. */
316 ldi (PAGE_SIZE / 128), %r1
317
3181:
319 std %r0, 0(%r26)
320 std %r0, 8(%r26)
321 std %r0, 16(%r26)
322 std %r0, 24(%r26)
323 std %r0, 32(%r26)
324 std %r0, 40(%r26)
325 std %r0, 48(%r26)
326 std %r0, 56(%r26)
327 std %r0, 64(%r26)
328 std %r0, 72(%r26)
329 std %r0, 80(%r26)
330 std %r0, 88(%r26)
331 std %r0, 96(%r26)
332 std %r0, 104(%r26)
333 std %r0, 112(%r26)
334 std %r0, 120(%r26)
335
336 /* Note reverse branch hint for addib is taken. */
337 addib,COND(>),n -1, %r1, 1b
338 ldo 128(%r26), %r26
339
340#else
341
342 /*
343 * Note that until (if) we start saving the full 64-bit register
344 * values on interrupt, we can't use std on a 32 bit kernel.
345 */
346 ldi (PAGE_SIZE / 64), %r1
347
3481:
349 stw %r0, 0(%r26)
350 stw %r0, 4(%r26)
351 stw %r0, 8(%r26)
352 stw %r0, 12(%r26)
353 stw %r0, 16(%r26)
354 stw %r0, 20(%r26)
355 stw %r0, 24(%r26)
356 stw %r0, 28(%r26)
357 stw %r0, 32(%r26)
358 stw %r0, 36(%r26)
359 stw %r0, 40(%r26)
360 stw %r0, 44(%r26)
361 stw %r0, 48(%r26)
362 stw %r0, 52(%r26)
363 stw %r0, 56(%r26)
364 stw %r0, 60(%r26)
365
366 addib,COND(>),n -1, %r1, 1b
367 ldo 64(%r26), %r26
368#endif
369 bv %r0(%r2)
370 nop
371ENDPROC_CFI(clear_page_asm)
372
373/* Copy page using kernel mapping. */
374
375ENTRY_CFI(copy_page_asm)
376#ifdef CONFIG_64BIT
377 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
378 * Unroll the loop by hand and arrange insn appropriately.
379 * Prefetch doesn't improve performance on rp3440.
380 * GCC probably can do this just as well...
381 */
382
383 ldi (PAGE_SIZE / 128), %r1
384
3851: ldd 0(%r25), %r19
386 ldd 8(%r25), %r20
387
388 ldd 16(%r25), %r21
389 ldd 24(%r25), %r22
390 std %r19, 0(%r26)
391 std %r20, 8(%r26)
392
393 ldd 32(%r25), %r19
394 ldd 40(%r25), %r20
395 std %r21, 16(%r26)
396 std %r22, 24(%r26)
397
398 ldd 48(%r25), %r21
399 ldd 56(%r25), %r22
400 std %r19, 32(%r26)
401 std %r20, 40(%r26)
402
403 ldd 64(%r25), %r19
404 ldd 72(%r25), %r20
405 std %r21, 48(%r26)
406 std %r22, 56(%r26)
407
408 ldd 80(%r25), %r21
409 ldd 88(%r25), %r22
410 std %r19, 64(%r26)
411 std %r20, 72(%r26)
412
413 ldd 96(%r25), %r19
414 ldd 104(%r25), %r20
415 std %r21, 80(%r26)
416 std %r22, 88(%r26)
417
418 ldd 112(%r25), %r21
419 ldd 120(%r25), %r22
420 ldo 128(%r25), %r25
421 std %r19, 96(%r26)
422 std %r20, 104(%r26)
423
424 std %r21, 112(%r26)
425 std %r22, 120(%r26)
426
427 /* Note reverse branch hint for addib is taken. */
428 addib,COND(>),n -1, %r1, 1b
429 ldo 128(%r26), %r26
430
431#else
432
433 /*
434 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
435 * bundles (very restricted rules for bundling).
436 * Note that until (if) we start saving
437 * the full 64 bit register values on interrupt, we can't
438 * use ldd/std on a 32 bit kernel.
439 */
440 ldw 0(%r25), %r19
441 ldi (PAGE_SIZE / 64), %r1
442
4431:
444 ldw 4(%r25), %r20
445 ldw 8(%r25), %r21
446 ldw 12(%r25), %r22
447 stw %r19, 0(%r26)
448 stw %r20, 4(%r26)
449 stw %r21, 8(%r26)
450 stw %r22, 12(%r26)
451 ldw 16(%r25), %r19
452 ldw 20(%r25), %r20
453 ldw 24(%r25), %r21
454 ldw 28(%r25), %r22
455 stw %r19, 16(%r26)
456 stw %r20, 20(%r26)
457 stw %r21, 24(%r26)
458 stw %r22, 28(%r26)
459 ldw 32(%r25), %r19
460 ldw 36(%r25), %r20
461 ldw 40(%r25), %r21
462 ldw 44(%r25), %r22
463 stw %r19, 32(%r26)
464 stw %r20, 36(%r26)
465 stw %r21, 40(%r26)
466 stw %r22, 44(%r26)
467 ldw 48(%r25), %r19
468 ldw 52(%r25), %r20
469 ldw 56(%r25), %r21
470 ldw 60(%r25), %r22
471 stw %r19, 48(%r26)
472 stw %r20, 52(%r26)
473 ldo 64(%r25), %r25
474 stw %r21, 56(%r26)
475 stw %r22, 60(%r26)
476 ldo 64(%r26), %r26
477 addib,COND(>),n -1, %r1, 1b
478 ldw 0(%r25), %r19
479#endif
480 bv %r0(%r2)
481 nop
482ENDPROC_CFI(copy_page_asm)
483
484/*
485 * NOTE: Code in clear_user_page has a hard coded dependency on the
486 * maximum alias boundary being 4 Mb. We've been assured by the
487 * parisc chip designers that there will not ever be a parisc
488 * chip with a larger alias boundary (Never say never :-) ).
489 *
490 * Yah, what about the PA8800 and PA8900 processors?
491 *
492 * Subtle: the dtlb miss handlers support the temp alias region by
493 * "knowing" that if a dtlb miss happens within the temp alias
494 * region it must have occurred while in clear_user_page. Since
495 * this routine makes use of processor local translations, we
496 * don't want to insert them into the kernel page table. Instead,
497 * we load up some general registers (they need to be registers
498 * which aren't shadowed) with the physical page numbers (preshifted
499 * for tlb insertion) needed to insert the translations. When we
500 * miss on the translation, the dtlb miss handler inserts the
501 * translation into the tlb using these values:
502 *
503 * %r26 physical address of "to" translation
504 * %r23 physical address of "from" translation
505 */
506
507 /*
508 * copy_user_page_asm() performs a page copy using mappings
509 * equivalent to the user page mappings. It can be used to
510 * implement copy_user_page() but unfortunately both the `from'
511 * and `to' pages need to be flushed through mappings equivalent
512 * to the user mappings after the copy because the kernel accesses
513 * the `from' page through the kmap kernel mapping and the `to'
514 * page needs to be flushed since code can be copied. As a
515 * result, this implementation is less efficient than the simpler
516 * copy using the kernel mapping. It only needs the `from' page
517 * to flushed via the user mapping. The kunmap routines handle
518 * the flushes needed for the kernel mapping.
519 *
520 * I'm still keeping this around because it may be possible to
521 * use it if more information is passed into copy_user_page().
522 * Have to do some measurements to see if it is worthwhile to
523 * lobby for such a change.
524 *
525 */
526
527ENTRY_CFI(copy_user_page_asm)
528 /* Convert virtual `to' and `from' addresses to physical addresses.
529 Move `from' physical address to non shadowed register. */
530 ldil L%(__PAGE_OFFSET), %r1
531 sub %r26, %r1, %r26
532 sub %r25, %r1, %r23
533
534 ldil L%(TMPALIAS_MAP_START), %r28
535 dep_safe %r24, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
536 depi_safe 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
537 copy %r28, %r29
538 depi_safe 1, 31-TMPALIAS_SIZE_BITS,1, %r29 /* Form aliased virtual address 'from' */
539
540 /* Purge any old translations */
541
542#ifdef CONFIG_PA20
543 pdtlb,l %r0(%r28)
544 pdtlb,l %r0(%r29)
545#else
5460: pdtlb %r0(%r28)
5471: pdtlb %r0(%r29)
548 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
549 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
550#endif
551
552#ifdef CONFIG_64BIT
553 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
554 * Unroll the loop by hand and arrange insn appropriately.
555 * GCC probably can do this just as well.
556 */
557
558 ldd 0(%r29), %r19
559 ldi (PAGE_SIZE / 128), %r1
560
5611: ldd 8(%r29), %r20
562
563 ldd 16(%r29), %r21
564 ldd 24(%r29), %r22
565 std %r19, 0(%r28)
566 std %r20, 8(%r28)
567
568 ldd 32(%r29), %r19
569 ldd 40(%r29), %r20
570 std %r21, 16(%r28)
571 std %r22, 24(%r28)
572
573 ldd 48(%r29), %r21
574 ldd 56(%r29), %r22
575 std %r19, 32(%r28)
576 std %r20, 40(%r28)
577
578 ldd 64(%r29), %r19
579 ldd 72(%r29), %r20
580 std %r21, 48(%r28)
581 std %r22, 56(%r28)
582
583 ldd 80(%r29), %r21
584 ldd 88(%r29), %r22
585 std %r19, 64(%r28)
586 std %r20, 72(%r28)
587
588 ldd 96(%r29), %r19
589 ldd 104(%r29), %r20
590 std %r21, 80(%r28)
591 std %r22, 88(%r28)
592
593 ldd 112(%r29), %r21
594 ldd 120(%r29), %r22
595 std %r19, 96(%r28)
596 std %r20, 104(%r28)
597
598 ldo 128(%r29), %r29
599 std %r21, 112(%r28)
600 std %r22, 120(%r28)
601 ldo 128(%r28), %r28
602
603 /* conditional branches nullify on forward taken branch, and on
604 * non-taken backward branch. Note that .+4 is a backwards branch.
605 * The ldd should only get executed if the branch is taken.
606 */
607 addib,COND(>),n -1, %r1, 1b /* bundle 10 */
608 ldd 0(%r29), %r19 /* start next loads */
609
610#else
611 ldi (PAGE_SIZE / 64), %r1
612
613 /*
614 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
615 * bundles (very restricted rules for bundling). It probably
616 * does OK on PCXU and better, but we could do better with
617 * ldd/std instructions. Note that until (if) we start saving
618 * the full 64 bit register values on interrupt, we can't
619 * use ldd/std on a 32 bit kernel.
620 */
621
6221: ldw 0(%r29), %r19
623 ldw 4(%r29), %r20
624 ldw 8(%r29), %r21
625 ldw 12(%r29), %r22
626 stw %r19, 0(%r28)
627 stw %r20, 4(%r28)
628 stw %r21, 8(%r28)
629 stw %r22, 12(%r28)
630 ldw 16(%r29), %r19
631 ldw 20(%r29), %r20
632 ldw 24(%r29), %r21
633 ldw 28(%r29), %r22
634 stw %r19, 16(%r28)
635 stw %r20, 20(%r28)
636 stw %r21, 24(%r28)
637 stw %r22, 28(%r28)
638 ldw 32(%r29), %r19
639 ldw 36(%r29), %r20
640 ldw 40(%r29), %r21
641 ldw 44(%r29), %r22
642 stw %r19, 32(%r28)
643 stw %r20, 36(%r28)
644 stw %r21, 40(%r28)
645 stw %r22, 44(%r28)
646 ldw 48(%r29), %r19
647 ldw 52(%r29), %r20
648 ldw 56(%r29), %r21
649 ldw 60(%r29), %r22
650 stw %r19, 48(%r28)
651 stw %r20, 52(%r28)
652 stw %r21, 56(%r28)
653 stw %r22, 60(%r28)
654 ldo 64(%r28), %r28
655
656 addib,COND(>) -1, %r1,1b
657 ldo 64(%r29), %r29
658#endif
659
660 bv %r0(%r2)
661 nop
662ENDPROC_CFI(copy_user_page_asm)
663
664ENTRY_CFI(clear_user_page_asm)
665 tophys_r1 %r26
666
667 ldil L%(TMPALIAS_MAP_START), %r28
668 dep_safe %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
669 depi_safe 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
670
671 /* Purge any old translation */
672
673#ifdef CONFIG_PA20
674 pdtlb,l %r0(%r28)
675#else
6760: pdtlb %r0(%r28)
677 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
678#endif
679
680#ifdef CONFIG_64BIT
681 ldi (PAGE_SIZE / 128), %r1
682
683 /* PREFETCH (Write) has not (yet) been proven to help here */
684 /* #define PREFETCHW_OP ldd 256(%0), %r0 */
685
6861: std %r0, 0(%r28)
687 std %r0, 8(%r28)
688 std %r0, 16(%r28)
689 std %r0, 24(%r28)
690 std %r0, 32(%r28)
691 std %r0, 40(%r28)
692 std %r0, 48(%r28)
693 std %r0, 56(%r28)
694 std %r0, 64(%r28)
695 std %r0, 72(%r28)
696 std %r0, 80(%r28)
697 std %r0, 88(%r28)
698 std %r0, 96(%r28)
699 std %r0, 104(%r28)
700 std %r0, 112(%r28)
701 std %r0, 120(%r28)
702 addib,COND(>) -1, %r1, 1b
703 ldo 128(%r28), %r28
704
705#else /* ! CONFIG_64BIT */
706 ldi (PAGE_SIZE / 64), %r1
707
7081: stw %r0, 0(%r28)
709 stw %r0, 4(%r28)
710 stw %r0, 8(%r28)
711 stw %r0, 12(%r28)
712 stw %r0, 16(%r28)
713 stw %r0, 20(%r28)
714 stw %r0, 24(%r28)
715 stw %r0, 28(%r28)
716 stw %r0, 32(%r28)
717 stw %r0, 36(%r28)
718 stw %r0, 40(%r28)
719 stw %r0, 44(%r28)
720 stw %r0, 48(%r28)
721 stw %r0, 52(%r28)
722 stw %r0, 56(%r28)
723 stw %r0, 60(%r28)
724 addib,COND(>) -1, %r1, 1b
725 ldo 64(%r28), %r28
726#endif /* CONFIG_64BIT */
727
728 bv %r0(%r2)
729 nop
730ENDPROC_CFI(clear_user_page_asm)
731
732ENTRY_CFI(flush_dcache_page_asm)
733 ldil L%(TMPALIAS_MAP_START), %r28
734 dep_safe %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
735 depi_safe 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
736
737 /* Purge any old translation */
738
739#ifdef CONFIG_PA20
740 pdtlb,l %r0(%r28)
741#else
7420: pdtlb %r0(%r28)
743 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
744#endif
745
74688: ldil L%dcache_stride, %r1
747 ldw R%dcache_stride(%r1), r31
748
749#ifdef CONFIG_64BIT
750 depdi,z 1, 63-PAGE_SHIFT,1, %r25
751#else
752 depwi,z 1, 31-PAGE_SHIFT,1, %r25
753#endif
754 add %r28, %r25, %r25
755 sub %r25, r31, %r25
756
7571: fdc,m r31(%r28)
758 fdc,m r31(%r28)
759 fdc,m r31(%r28)
760 fdc,m r31(%r28)
761 fdc,m r31(%r28)
762 fdc,m r31(%r28)
763 fdc,m r31(%r28)
764 fdc,m r31(%r28)
765 fdc,m r31(%r28)
766 fdc,m r31(%r28)
767 fdc,m r31(%r28)
768 fdc,m r31(%r28)
769 fdc,m r31(%r28)
770 fdc,m r31(%r28)
771 fdc,m r31(%r28)
772 cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
773 fdc,m r31(%r28)
774
77589: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
776 sync
777 bv %r0(%r2)
778 nop
779ENDPROC_CFI(flush_dcache_page_asm)
780
781ENTRY_CFI(purge_dcache_page_asm)
782 ldil L%(TMPALIAS_MAP_START), %r28
783 dep_safe %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
784 depi_safe 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
785
786 /* Purge any old translation */
787
788#ifdef CONFIG_PA20
789 pdtlb,l %r0(%r28)
790#else
7910: pdtlb %r0(%r28)
792 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
793#endif
794
79588: ldil L%dcache_stride, %r1
796 ldw R%dcache_stride(%r1), r31
797
798#ifdef CONFIG_64BIT
799 depdi,z 1, 63-PAGE_SHIFT,1, %r25
800#else
801 depwi,z 1, 31-PAGE_SHIFT,1, %r25
802#endif
803 add %r28, %r25, %r25
804 sub %r25, r31, %r25
805
8061: pdc,m r31(%r28)
807 pdc,m r31(%r28)
808 pdc,m r31(%r28)
809 pdc,m r31(%r28)
810 pdc,m r31(%r28)
811 pdc,m r31(%r28)
812 pdc,m r31(%r28)
813 pdc,m r31(%r28)
814 pdc,m r31(%r28)
815 pdc,m r31(%r28)
816 pdc,m r31(%r28)
817 pdc,m r31(%r28)
818 pdc,m r31(%r28)
819 pdc,m r31(%r28)
820 pdc,m r31(%r28)
821 cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
822 pdc,m r31(%r28)
823
82489: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
825 sync
826 bv %r0(%r2)
827 nop
828ENDPROC_CFI(purge_dcache_page_asm)
829
830ENTRY_CFI(flush_icache_page_asm)
831 ldil L%(TMPALIAS_MAP_START), %r28
832 dep_safe %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
833 depi_safe 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
834
835 /* Purge any old translation. Note that the FIC instruction
836 * may use either the instruction or data TLB. Given that we
837 * have a flat address space, it's not clear which TLB will be
838 * used. So, we purge both entries. */
839
840#ifdef CONFIG_PA20
841 pdtlb,l %r0(%r28)
8421: pitlb,l %r0(%sr4,%r28)
843 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
844#else
8450: pdtlb %r0(%r28)
8461: pitlb %r0(%sr4,%r28)
847 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
848 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
849 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
850#endif
851
85288: ldil L%icache_stride, %r1
853 ldw R%icache_stride(%r1), %r31
854
855#ifdef CONFIG_64BIT
856 depdi,z 1, 63-PAGE_SHIFT,1, %r25
857#else
858 depwi,z 1, 31-PAGE_SHIFT,1, %r25
859#endif
860 add %r28, %r25, %r25
861 sub %r25, %r31, %r25
862
863 /* fic only has the type 26 form on PA1.1, requiring an
864 * explicit space specification, so use %sr4 */
8651: fic,m %r31(%sr4,%r28)
866 fic,m %r31(%sr4,%r28)
867 fic,m %r31(%sr4,%r28)
868 fic,m %r31(%sr4,%r28)
869 fic,m %r31(%sr4,%r28)
870 fic,m %r31(%sr4,%r28)
871 fic,m %r31(%sr4,%r28)
872 fic,m %r31(%sr4,%r28)
873 fic,m %r31(%sr4,%r28)
874 fic,m %r31(%sr4,%r28)
875 fic,m %r31(%sr4,%r28)
876 fic,m %r31(%sr4,%r28)
877 fic,m %r31(%sr4,%r28)
878 fic,m %r31(%sr4,%r28)
879 fic,m %r31(%sr4,%r28)
880 cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
881 fic,m %r31(%sr4,%r28)
882
88389: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
884 sync
885 bv %r0(%r2)
886 nop
887ENDPROC_CFI(flush_icache_page_asm)
888
889ENTRY_CFI(flush_kernel_dcache_page_asm)
89088: ldil L%dcache_stride, %r1
891 ldw R%dcache_stride(%r1), %r23
892 depi_safe 0, 31,PAGE_SHIFT, %r26 /* Clear any offset bits */
893
894#ifdef CONFIG_64BIT
895 depdi,z 1, 63-PAGE_SHIFT,1, %r25
896#else
897 depwi,z 1, 31-PAGE_SHIFT,1, %r25
898#endif
899 add %r26, %r25, %r25
900 sub %r25, %r23, %r25
901
9021: fdc,m %r23(%r26)
903 fdc,m %r23(%r26)
904 fdc,m %r23(%r26)
905 fdc,m %r23(%r26)
906 fdc,m %r23(%r26)
907 fdc,m %r23(%r26)
908 fdc,m %r23(%r26)
909 fdc,m %r23(%r26)
910 fdc,m %r23(%r26)
911 fdc,m %r23(%r26)
912 fdc,m %r23(%r26)
913 fdc,m %r23(%r26)
914 fdc,m %r23(%r26)
915 fdc,m %r23(%r26)
916 fdc,m %r23(%r26)
917 cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
918 fdc,m %r23(%r26)
919
92089: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
921 sync
922 bv %r0(%r2)
923 nop
924ENDPROC_CFI(flush_kernel_dcache_page_asm)
925
926ENTRY_CFI(purge_kernel_dcache_page_asm)
92788: ldil L%dcache_stride, %r1
928 ldw R%dcache_stride(%r1), %r23
929 depi_safe 0, 31,PAGE_SHIFT, %r26 /* Clear any offset bits */
930
931#ifdef CONFIG_64BIT
932 depdi,z 1, 63-PAGE_SHIFT,1, %r25
933#else
934 depwi,z 1, 31-PAGE_SHIFT,1, %r25
935#endif
936 add %r26, %r25, %r25
937 sub %r25, %r23, %r25
938
9391: pdc,m %r23(%r26)
940 pdc,m %r23(%r26)
941 pdc,m %r23(%r26)
942 pdc,m %r23(%r26)
943 pdc,m %r23(%r26)
944 pdc,m %r23(%r26)
945 pdc,m %r23(%r26)
946 pdc,m %r23(%r26)
947 pdc,m %r23(%r26)
948 pdc,m %r23(%r26)
949 pdc,m %r23(%r26)
950 pdc,m %r23(%r26)
951 pdc,m %r23(%r26)
952 pdc,m %r23(%r26)
953 pdc,m %r23(%r26)
954 cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
955 pdc,m %r23(%r26)
956
95789: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
958 sync
959 bv %r0(%r2)
960 nop
961ENDPROC_CFI(purge_kernel_dcache_page_asm)
962
963ENTRY_CFI(flush_user_dcache_range_asm)
96488: ldil L%dcache_stride, %r1
965 ldw R%dcache_stride(%r1), %r23
966 ldo -1(%r23), %r21
967 ANDCM %r26, %r21, %r26
968
969#ifdef CONFIG_64BIT
970 depd,z %r23, 59, 60, %r21
971#else
972 depw,z %r23, 27, 28, %r21
973#endif
974 add %r26, %r21, %r22
975 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
9761: add %r22, %r21, %r22
977 fdc,m %r23(%sr3, %r26)
978 fdc,m %r23(%sr3, %r26)
979 fdc,m %r23(%sr3, %r26)
980 fdc,m %r23(%sr3, %r26)
981 fdc,m %r23(%sr3, %r26)
982 fdc,m %r23(%sr3, %r26)
983 fdc,m %r23(%sr3, %r26)
984 fdc,m %r23(%sr3, %r26)
985 fdc,m %r23(%sr3, %r26)
986 fdc,m %r23(%sr3, %r26)
987 fdc,m %r23(%sr3, %r26)
988 fdc,m %r23(%sr3, %r26)
989 fdc,m %r23(%sr3, %r26)
990 fdc,m %r23(%sr3, %r26)
991 fdc,m %r23(%sr3, %r26)
992 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
993 fdc,m %r23(%sr3, %r26)
994
9952: cmpb,COND(>>),n %r25, %r26, 2b
996 fdc,m %r23(%sr3, %r26)
997
99889: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
999 sync
1000 bv %r0(%r2)
1001 nop
1002ENDPROC_CFI(flush_user_dcache_range_asm)
1003
1004ENTRY_CFI(flush_kernel_dcache_range_asm)
100588: ldil L%dcache_stride, %r1
1006 ldw R%dcache_stride(%r1), %r23
1007 ldo -1(%r23), %r21
1008 ANDCM %r26, %r21, %r26
1009
1010#ifdef CONFIG_64BIT
1011 depd,z %r23, 59, 60, %r21
1012#else
1013 depw,z %r23, 27, 28, %r21
1014#endif
1015 add %r26, %r21, %r22
1016 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
10171: add %r22, %r21, %r22
1018 fdc,m %r23(%r26)
1019 fdc,m %r23(%r26)
1020 fdc,m %r23(%r26)
1021 fdc,m %r23(%r26)
1022 fdc,m %r23(%r26)
1023 fdc,m %r23(%r26)
1024 fdc,m %r23(%r26)
1025 fdc,m %r23(%r26)
1026 fdc,m %r23(%r26)
1027 fdc,m %r23(%r26)
1028 fdc,m %r23(%r26)
1029 fdc,m %r23(%r26)
1030 fdc,m %r23(%r26)
1031 fdc,m %r23(%r26)
1032 fdc,m %r23(%r26)
1033 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1034 fdc,m %r23(%r26)
1035
10362: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
1037 fdc,m %r23(%r26)
1038
1039 sync
104089: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1041 bv %r0(%r2)
1042 nop
1043ENDPROC_CFI(flush_kernel_dcache_range_asm)
1044
1045ENTRY_CFI(purge_kernel_dcache_range_asm)
104688: ldil L%dcache_stride, %r1
1047 ldw R%dcache_stride(%r1), %r23
1048 ldo -1(%r23), %r21
1049 ANDCM %r26, %r21, %r26
1050
1051#ifdef CONFIG_64BIT
1052 depd,z %r23, 59, 60, %r21
1053#else
1054 depw,z %r23, 27, 28, %r21
1055#endif
1056 add %r26, %r21, %r22
1057 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
10581: add %r22, %r21, %r22
1059 pdc,m %r23(%r26)
1060 pdc,m %r23(%r26)
1061 pdc,m %r23(%r26)
1062 pdc,m %r23(%r26)
1063 pdc,m %r23(%r26)
1064 pdc,m %r23(%r26)
1065 pdc,m %r23(%r26)
1066 pdc,m %r23(%r26)
1067 pdc,m %r23(%r26)
1068 pdc,m %r23(%r26)
1069 pdc,m %r23(%r26)
1070 pdc,m %r23(%r26)
1071 pdc,m %r23(%r26)
1072 pdc,m %r23(%r26)
1073 pdc,m %r23(%r26)
1074 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1075 pdc,m %r23(%r26)
1076
10772: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
1078 pdc,m %r23(%r26)
1079
1080 sync
108189: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1082 bv %r0(%r2)
1083 nop
1084ENDPROC_CFI(purge_kernel_dcache_range_asm)
1085
1086ENTRY_CFI(flush_user_icache_range_asm)
108788: ldil L%icache_stride, %r1
1088 ldw R%icache_stride(%r1), %r23
1089 ldo -1(%r23), %r21
1090 ANDCM %r26, %r21, %r26
1091
1092#ifdef CONFIG_64BIT
1093 depd,z %r23, 59, 60, %r21
1094#else
1095 depw,z %r23, 27, 28, %r21
1096#endif
1097 add %r26, %r21, %r22
1098 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
10991: add %r22, %r21, %r22
1100 fic,m %r23(%sr3, %r26)
1101 fic,m %r23(%sr3, %r26)
1102 fic,m %r23(%sr3, %r26)
1103 fic,m %r23(%sr3, %r26)
1104 fic,m %r23(%sr3, %r26)
1105 fic,m %r23(%sr3, %r26)
1106 fic,m %r23(%sr3, %r26)
1107 fic,m %r23(%sr3, %r26)
1108 fic,m %r23(%sr3, %r26)
1109 fic,m %r23(%sr3, %r26)
1110 fic,m %r23(%sr3, %r26)
1111 fic,m %r23(%sr3, %r26)
1112 fic,m %r23(%sr3, %r26)
1113 fic,m %r23(%sr3, %r26)
1114 fic,m %r23(%sr3, %r26)
1115 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1116 fic,m %r23(%sr3, %r26)
1117
11182: cmpb,COND(>>),n %r25, %r26, 2b
1119 fic,m %r23(%sr3, %r26)
1120
112189: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1122 sync
1123 bv %r0(%r2)
1124 nop
1125ENDPROC_CFI(flush_user_icache_range_asm)
1126
1127ENTRY_CFI(flush_kernel_icache_page)
112888: ldil L%icache_stride, %r1
1129 ldw R%icache_stride(%r1), %r23
1130
1131#ifdef CONFIG_64BIT
1132 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1133#else
1134 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1135#endif
1136 add %r26, %r25, %r25
1137 sub %r25, %r23, %r25
1138
1139
11401: fic,m %r23(%sr4, %r26)
1141 fic,m %r23(%sr4, %r26)
1142 fic,m %r23(%sr4, %r26)
1143 fic,m %r23(%sr4, %r26)
1144 fic,m %r23(%sr4, %r26)
1145 fic,m %r23(%sr4, %r26)
1146 fic,m %r23(%sr4, %r26)
1147 fic,m %r23(%sr4, %r26)
1148 fic,m %r23(%sr4, %r26)
1149 fic,m %r23(%sr4, %r26)
1150 fic,m %r23(%sr4, %r26)
1151 fic,m %r23(%sr4, %r26)
1152 fic,m %r23(%sr4, %r26)
1153 fic,m %r23(%sr4, %r26)
1154 fic,m %r23(%sr4, %r26)
1155 cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
1156 fic,m %r23(%sr4, %r26)
1157
115889: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1159 sync
1160 bv %r0(%r2)
1161 nop
1162ENDPROC_CFI(flush_kernel_icache_page)
1163
1164ENTRY_CFI(flush_kernel_icache_range_asm)
116588: ldil L%icache_stride, %r1
1166 ldw R%icache_stride(%r1), %r23
1167 ldo -1(%r23), %r21
1168 ANDCM %r26, %r21, %r26
1169
1170#ifdef CONFIG_64BIT
1171 depd,z %r23, 59, 60, %r21
1172#else
1173 depw,z %r23, 27, 28, %r21
1174#endif
1175 add %r26, %r21, %r22
1176 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
11771: add %r22, %r21, %r22
1178 fic,m %r23(%sr4, %r26)
1179 fic,m %r23(%sr4, %r26)
1180 fic,m %r23(%sr4, %r26)
1181 fic,m %r23(%sr4, %r26)
1182 fic,m %r23(%sr4, %r26)
1183 fic,m %r23(%sr4, %r26)
1184 fic,m %r23(%sr4, %r26)
1185 fic,m %r23(%sr4, %r26)
1186 fic,m %r23(%sr4, %r26)
1187 fic,m %r23(%sr4, %r26)
1188 fic,m %r23(%sr4, %r26)
1189 fic,m %r23(%sr4, %r26)
1190 fic,m %r23(%sr4, %r26)
1191 fic,m %r23(%sr4, %r26)
1192 fic,m %r23(%sr4, %r26)
1193 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1194 fic,m %r23(%sr4, %r26)
1195
11962: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
1197 fic,m %r23(%sr4, %r26)
1198
119989: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1200 sync
1201 bv %r0(%r2)
1202 nop
1203ENDPROC_CFI(flush_kernel_icache_range_asm)
1204
1205 .text
1206
1207 /* align should cover use of rfi in disable_sr_hashing_asm and
1208 * srdis_done.
1209 */
1210 .align 256
1211ENTRY_CFI(disable_sr_hashing_asm)
1212 /*
1213 * Switch to real mode
1214 */
1215 /* pcxt_ssm_bug */
1216 rsm PSW_SM_I, %r0
1217 load32 PA(1f), %r1
1218 nop
1219 nop
1220 nop
1221 nop
1222 nop
1223
1224 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1225 mtctl %r0, %cr17 /* Clear IIASQ tail */
1226 mtctl %r0, %cr17 /* Clear IIASQ head */
1227 mtctl %r1, %cr18 /* IIAOQ head */
1228 ldo 4(%r1), %r1
1229 mtctl %r1, %cr18 /* IIAOQ tail */
1230 load32 REAL_MODE_PSW, %r1
1231 mtctl %r1, %ipsw
1232 rfi
1233 nop
1234
12351: cmpib,=,n SRHASH_PCXST, %r26,srdis_pcxs
1236 cmpib,=,n SRHASH_PCXL, %r26,srdis_pcxl
1237 cmpib,=,n SRHASH_PA20, %r26,srdis_pa20
1238 b,n srdis_done
1239
1240srdis_pcxs:
1241
1242 /* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1243
1244 .word 0x141c1a00 /* mfdiag %dr0, %r28 */
1245 .word 0x141c1a00 /* must issue twice */
1246 depwi 0,18,1, %r28 /* Clear DHE (dcache hash enable) */
1247 depwi 0,20,1, %r28 /* Clear IHE (icache hash enable) */
1248 .word 0x141c1600 /* mtdiag %r28, %dr0 */
1249 .word 0x141c1600 /* must issue twice */
1250 b,n srdis_done
1251
1252srdis_pcxl:
1253
1254 /* Disable Space Register Hashing for PCXL */
1255
1256 .word 0x141c0600 /* mfdiag %dr0, %r28 */
1257 depwi 0,28,2, %r28 /* Clear DHASH_EN & IHASH_EN */
1258 .word 0x141c0240 /* mtdiag %r28, %dr0 */
1259 b,n srdis_done
1260
1261srdis_pa20:
1262
1263 /* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1264
1265 .word 0x144008bc /* mfdiag %dr2, %r28 */
1266 depdi 0, 54,1, %r28 /* clear DIAG_SPHASH_ENAB (bit 54) */
1267 .word 0x145c1840 /* mtdiag %r28, %dr2 */
1268
1269
1270srdis_done:
1271 /* Switch back to virtual mode */
1272 rsm PSW_SM_I, %r0 /* prep to load iia queue */
1273 load32 2f, %r1
1274 nop
1275 nop
1276 nop
1277 nop
1278 nop
1279
1280 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1281 mtctl %r0, %cr17 /* Clear IIASQ tail */
1282 mtctl %r0, %cr17 /* Clear IIASQ head */
1283 mtctl %r1, %cr18 /* IIAOQ head */
1284 ldo 4(%r1), %r1
1285 mtctl %r1, %cr18 /* IIAOQ tail */
1286 load32 KERNEL_PSW, %r1
1287 mtctl %r1, %ipsw
1288 rfi
1289 nop
1290
12912: bv %r0(%r2)
1292 nop
1293ENDPROC_CFI(disable_sr_hashing_asm)
1294
1295 .end