| 1 | /* SPDX-License-Identifier: GPL-2.0-only */
|
| 2 | /*
|
| 3 | * linux/arch/arm/lib/copy_template.s
|
| 4 | *
|
| 5 | * Code template for optimized memory copy functions
|
| 6 | *
|
| 7 | * Author: Nicolas Pitre
|
| 8 | * Created: Sep 28, 2005
|
| 9 | * Copyright: MontaVista Software, Inc.
|
| 10 | */
|
| 11 |
|
| 12 | /*
|
| 13 | * Theory of operation
|
| 14 | * -------------------
|
| 15 | *
|
| 16 | * This file provides the core code for a forward memory copy used in
|
| 17 | * the implementation of memcopy(), copy_to_user() and copy_from_user().
|
| 18 | *
|
| 19 | * The including file must define the following accessor macros
|
| 20 | * according to the need of the given function:
|
| 21 | *
|
| 22 | * ldr1w ptr reg abort
|
| 23 | *
|
| 24 | * This loads one word from 'ptr', stores it in 'reg' and increments
|
| 25 | * 'ptr' to the next word. The 'abort' argument is used for fixup tables.
|
| 26 | *
|
| 27 | * ldr4w ptr reg1 reg2 reg3 reg4 abort
|
| 28 | * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
|
| 29 | *
|
| 30 | * This loads four or eight words starting from 'ptr', stores them
|
| 31 | * in provided registers and increments 'ptr' past those words.
|
| 32 | * The'abort' argument is used for fixup tables.
|
| 33 | *
|
| 34 | * ldr1b ptr reg cond abort
|
| 35 | *
|
| 36 | * Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
|
| 37 | * It also must apply the condition code if provided, otherwise the
|
| 38 | * "al" condition is assumed by default.
|
| 39 | *
|
| 40 | * str1w ptr reg abort
|
| 41 | * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
|
| 42 | * str1b ptr reg cond abort
|
| 43 | *
|
| 44 | * Same as their ldr* counterparts, but data is stored to 'ptr' location
|
| 45 | * rather than being loaded.
|
| 46 | *
|
| 47 | * enter reg1 reg2
|
| 48 | *
|
| 49 | * Preserve the provided registers on the stack plus any additional
|
| 50 | * data as needed by the implementation including this code. Called
|
| 51 | * upon code entry.
|
| 52 | *
|
| 53 | * usave reg1 reg2
|
| 54 | *
|
| 55 | * Unwind annotation macro is corresponding for 'enter' macro.
|
| 56 | * It tell unwinder that preserved some provided registers on the stack
|
| 57 | * and additional data by a prior 'enter' macro.
|
| 58 | *
|
| 59 | * exit reg1 reg2
|
| 60 | *
|
| 61 | * Restore registers with the values previously saved with the
|
| 62 | * 'preserv' macro. Called upon code termination.
|
| 63 | *
|
| 64 | * LDR1W_SHIFT
|
| 65 | * STR1W_SHIFT
|
| 66 | *
|
| 67 | * Correction to be applied to the "ip" register when branching into
|
| 68 | * the ldr1w or str1w instructions (some of these macros may expand to
|
| 69 | * than one 32bit instruction in Thumb-2)
|
| 70 | */
|
| 71 |
|
| 72 | UNWIND( .fnstart )
|
| 73 | enter r4, UNWIND(fpreg,) lr
|
| 74 | UNWIND( .setfp fpreg, sp )
|
| 75 | UNWIND( mov fpreg, sp )
|
| 76 |
|
| 77 | subs r2, r2, |
| 78 | blt 8f
|
| 79 | ands ip, r0, |
| 80 | PLD( pld [r1, |
| 81 | bne 9f
|
| 82 | ands ip, r1, |
| 83 | bne 10f
|
| 84 |
|
| 85 | 1: subs r2, r2, |
| 86 | stmfd sp!, {r5, r6, r8, r9}
|
| 87 | blt 5f
|
| 88 |
|
| 89 | CALGN( ands ip, r0, |
| 90 | CALGN( rsb r3, ip, |
| 91 | CALGN( sbcsne r4, r3, r2 ) @ C is always set here
|
| 92 | CALGN( bcs 2f )
|
| 93 | CALGN( adr r4, 6f )
|
| 94 | CALGN( subs r2, r2, r3 ) @ C gets set
|
| 95 | CALGN( add pc, r4, ip )
|
| 96 |
|
| 97 | PLD( pld [r1, |
| 98 | 2: PLD( subs r2, r2, |
| 99 | PLD( pld [r1, |
| 100 | PLD( blt 4f )
|
| 101 | PLD( pld [r1, |
| 102 | PLD( pld [r1, |
| 103 |
|
| 104 | 3: PLD( pld [r1, |
| 105 | 4: ldr8w r1, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
|
| 106 | subs r2, r2, |
| 107 | str8w r0, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
|
| 108 | bge 3b
|
| 109 | PLD( cmn r2, |
| 110 | PLD( bge 4b )
|
| 111 |
|
| 112 | 5: ands ip, r2, |
| 113 | rsb ip, ip, |
| 114 | |
| 115 | lsl ip, ip, |
| 116 | |
| 117 | addne pc, pc, ip @ C is always clear here
|
| 118 | b 7f
|
| 119 | 6:
|
| 120 | .rept (1 << LDR1W_SHIFT)
|
| 121 | W(nop)
|
| 122 | .endr
|
| 123 | ldr1w r1, r3, abort=20f
|
| 124 | ldr1w r1, r4, abort=20f
|
| 125 | ldr1w r1, r5, abort=20f
|
| 126 | ldr1w r1, r6, abort=20f
|
| 127 | ldr1w r1, r8, abort=20f
|
| 128 | ldr1w r1, r9, abort=20f
|
| 129 | ldr1w r1, lr, abort=20f
|
| 130 |
|
| 131 | |
| 132 | lsl ip, ip, |
| 133 | |
| 134 | lsr ip, ip, |
| 135 | |
| 136 | add pc, pc, ip
|
| 137 | nop
|
| 138 | .rept (1 << STR1W_SHIFT)
|
| 139 | W(nop)
|
| 140 | .endr
|
| 141 | str1w r0, r3, abort=20f
|
| 142 | str1w r0, r4, abort=20f
|
| 143 | str1w r0, r5, abort=20f
|
| 144 | str1w r0, r6, abort=20f
|
| 145 | str1w r0, r8, abort=20f
|
| 146 | str1w r0, r9, abort=20f
|
| 147 | str1w r0, lr, abort=20f
|
| 148 |
|
| 149 | CALGN( bcs 2b )
|
| 150 |
|
| 151 | 7: ldmfd sp!, {r5, r6, r8, r9}
|
| 152 |
|
| 153 | 8: movs r2, r2, lsl |
| 154 | ldr1b r1, r3, ne, abort=21f
|
| 155 | ldr1b r1, r4, cs, abort=21f
|
| 156 | ldr1b r1, ip, cs, abort=21f
|
| 157 | str1b r0, r3, ne, abort=21f
|
| 158 | str1b r0, r4, cs, abort=21f
|
| 159 | str1b r0, ip, cs, abort=21f
|
| 160 |
|
| 161 | exit r4, UNWIND(fpreg,) pc
|
| 162 |
|
| 163 | 9: rsb ip, ip, |
| 164 | cmp ip, |
| 165 | ldr1b r1, r3, gt, abort=21f
|
| 166 | ldr1b r1, r4, ge, abort=21f
|
| 167 | ldr1b r1, lr, abort=21f
|
| 168 | str1b r0, r3, gt, abort=21f
|
| 169 | str1b r0, r4, ge, abort=21f
|
| 170 | subs r2, r2, ip
|
| 171 | str1b r0, lr, abort=21f
|
| 172 | blt 8b
|
| 173 | ands ip, r1, |
| 174 | beq 1b
|
| 175 |
|
| 176 | 10: bic r1, r1, |
| 177 | cmp ip, |
| 178 | ldr1w r1, lr, abort=21f
|
| 179 | beq 17f
|
| 180 | bgt 18f
|
| 181 |
|
| 182 |
|
| 183 | .macro forward_copy_shift pull push
|
| 184 |
|
| 185 | subs r2, r2, |
| 186 | blt 14f
|
| 187 |
|
| 188 | CALGN( ands ip, r0, |
| 189 | CALGN( rsb ip, ip, |
| 190 | CALGN( sbcsne r4, ip, r2 ) @ C is always set here
|
| 191 | CALGN( subcc r2, r2, ip )
|
| 192 | CALGN( bcc 15f )
|
| 193 |
|
| 194 | 11: stmfd sp!, {r5, r6, r8 - r10}
|
| 195 |
|
| 196 | PLD( pld [r1, |
| 197 | PLD( subs r2, r2, |
| 198 | PLD( pld [r1, |
| 199 | PLD( blt 13f )
|
| 200 | PLD( pld [r1, |
| 201 | PLD( pld [r1, |
| 202 |
|
| 203 | 12: PLD( pld [r1, |
| 204 | 13: ldr4w r1, r4, r5, r6, r8, abort=19f
|
| 205 | mov r3, lr, lspull |
| 206 | subs r2, r2, |
| 207 | ldr4w r1, r9, r10, ip, lr, abort=19f
|
| 208 | orr r3, r3, r4, lspush |
| 209 | mov r4, r4, lspull |
| 210 | orr r4, r4, r5, lspush |
| 211 | mov r5, r5, lspull |
| 212 | orr r5, r5, r6, lspush |
| 213 | mov r6, r6, lspull |
| 214 | orr r6, r6, r8, lspush |
| 215 | mov r8, r8, lspull |
| 216 | orr r8, r8, r9, lspush |
| 217 | mov r9, r9, lspull |
| 218 | orr r9, r9, r10, lspush |
| 219 | mov r10, r10, lspull |
| 220 | orr r10, r10, ip, lspush |
| 221 | mov ip, ip, lspull |
| 222 | orr ip, ip, lr, lspush |
| 223 | str8w r0, r3, r4, r5, r6, r8, r9, r10, ip, abort=19f
|
| 224 | bge 12b
|
| 225 | PLD( cmn r2, |
| 226 | PLD( bge 13b )
|
| 227 |
|
| 228 | ldmfd sp!, {r5, r6, r8 - r10}
|
| 229 |
|
| 230 | 14: ands ip, r2, |
| 231 | beq 16f
|
| 232 |
|
| 233 | 15: mov r3, lr, lspull |
| 234 | ldr1w r1, lr, abort=21f
|
| 235 | subs ip, ip, |
| 236 | orr r3, r3, lr, lspush |
| 237 | str1w r0, r3, abort=21f
|
| 238 | bgt 15b
|
| 239 | CALGN( cmp r2, |
| 240 | CALGN( bge 11b )
|
| 241 |
|
| 242 | 16: sub r1, r1, |
| 243 | b 8b
|
| 244 |
|
| 245 | .endm
|
| 246 |
|
| 247 |
|
| 248 | forward_copy_shift pull=8 push=24
|
| 249 |
|
| 250 | 17: forward_copy_shift pull=16 push=16
|
| 251 |
|
| 252 | 18: forward_copy_shift pull=24 push=8
|
| 253 |
|
| 254 | UNWIND( .fnend )
|
| 255 |
|
| 256 | /*
|
| 257 | * Abort preamble and completion macros.
|
| 258 | * If a fixup handler is required then those macros must surround it.
|
| 259 | * It is assumed that the fixup code will handle the private part of
|
| 260 | * the exit macro.
|
| 261 | */
|
| 262 |
|
| 263 | .macro copy_abort_preamble
|
| 264 | 19: ldmfd sp!, {r5, r6, r8 - r10}
|
| 265 | b 21f
|
| 266 | 20: ldmfd sp!, {r5, r6, r8, r9}
|
| 267 | 21:
|
| 268 | .endm
|
| 269 |
|
| 270 | .macro copy_abort_end
|
| 271 | ldmfd sp!, {r4, UNWIND(fpreg,) pc}
|
| 272 | .endm
|
| 273 |
|