| 1 | /* SPDX-License-Identifier: GPL-2.0 */
|
| 2 | /*
|
| 3 | * arch/alpha/lib/ev6-memset.S
|
| 4 | *
|
| 5 | * This is an efficient (and relatively small) implementation of the C library
|
| 6 | * "memset()" function for the 21264 implementation of Alpha.
|
| 7 | *
|
| 8 | * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
|
| 9 | *
|
| 10 | * Much of the information about 21264 scheduling/coding comes from:
|
| 11 | * Compiler Writer's Guide for the Alpha 21264
|
| 12 | * abbreviated as 'CWG' in other comments here
|
| 13 | * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
|
| 14 | * Scheduling notation:
|
| 15 | * E - either cluster
|
| 16 | * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
|
| 17 | * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
|
| 18 | * The algorithm for the leading and trailing quadwords remains the same,
|
| 19 | * however the loop has been unrolled to enable better memory throughput,
|
| 20 | * and the code has been replicated for each of the entry points: __memset
|
| 21 | * and __memset16 to permit better scheduling to eliminate the stalling
|
| 22 | * encountered during the mask replication.
|
| 23 | * A future enhancement might be to put in a byte store loop for really
|
| 24 | * small (say < 32 bytes) memset()s. Whether or not that change would be
|
| 25 | * a win in the kernel would depend upon the contextual usage.
|
| 26 | * WARNING: Maintaining this is going to be more work than the above version,
|
| 27 | * as fixes will need to be made in multiple places. The performance gain
|
| 28 | * is worth it.
|
| 29 | */
|
| 30 | #include <linux/export.h>
|
| 31 | .set noat
|
| 32 | .set noreorder
|
| 33 | .text
|
| 34 | .globl memset
|
| 35 | .globl __memset
|
| 36 | .globl ___memset
|
| 37 | .globl __memset16
|
| 38 | .globl __constant_c_memset
|
| 39 |
|
| 40 | .ent ___memset
|
| 41 | .align 5
|
| 42 | ___memset:
|
| 43 | .frame $30,0,$26,0
|
| 44 | .prologue 0
|
| 45 |
|
| 46 | /*
|
| 47 | * Serious stalling happens. The only way to mitigate this is to
|
| 48 | * undertake a major re-write to interleave the constant materialization
|
| 49 | * with other parts of the fall-through code. This is important, even
|
| 50 | * though it makes maintenance tougher.
|
| 51 | * Do this later.
|
| 52 | */
|
| 53 | and $17,255,$1 # E : 00000000000000ch
|
| 54 | insbl $17,1,$2 # U : 000000000000ch00
|
| 55 | bis $16,$16,$0 # E : return value
|
| 56 | ble $18,end_b # U : zero length requested?
|
| 57 |
|
| 58 | addq $18,$16,$6 # E : max address to write to
|
| 59 | bis $1,$2,$17 # E : 000000000000chch
|
| 60 | insbl $1,2,$3 # U : 0000000000ch0000
|
| 61 | insbl $1,3,$4 # U : 00000000ch000000
|
| 62 |
|
| 63 | or $3,$4,$3 # E : 00000000chch0000
|
| 64 | inswl $17,4,$5 # U : 0000chch00000000
|
| 65 | xor $16,$6,$1 # E : will complete write be within one quadword?
|
| 66 | inswl $17,6,$2 # U : chch000000000000
|
| 67 |
|
| 68 | or $17,$3,$17 # E : 00000000chchchch
|
| 69 | or $2,$5,$2 # E : chchchch00000000
|
| 70 | bic $1,7,$1 # E : fit within a single quadword?
|
| 71 | and $16,7,$3 # E : Target addr misalignment
|
| 72 |
|
| 73 | or $17,$2,$17 # E : chchchchchchchch
|
| 74 | beq $1,within_quad_b # U :
|
| 75 | nop # E :
|
| 76 | beq $3,aligned_b # U : target is 0mod8
|
| 77 |
|
| 78 | /*
|
| 79 | * Target address is misaligned, and won't fit within a quadword
|
| 80 | */
|
| 81 | ldq_u $4,0($16) |
| 82 | bis $16,$16,$5 |
| 83 | insql $17,$16,$2 |
| 84 | subq $3,8,$3 |
| 85 |
|
| 86 | addq $18,$3,$18 |
| 87 | mskql $4,$16,$4 |
| 88 | subq $16,$3,$16 |
| 89 | bis $2,$4,$1 |
| 90 |
|
| 91 | nop
|
| 92 | stq_u $1,0($5) |
| 93 | nop
|
| 94 | nop
|
| 95 |
|
| 96 | .align 4
|
| 97 | aligned_b:
|
| 98 | /*
|
| 99 | * We are now guaranteed to be quad aligned, with at least
|
| 100 | * one partial quad to write.
|
| 101 | */
|
| 102 |
|
| 103 | sra $18,3,$3 |
| 104 | and $18,7,$18 |
| 105 | bis $16,$16,$5 |
| 106 | beq $3,no_quad_b |
| 107 |
|
| 108 | /*
|
| 109 | * it's worth the effort to unroll this and use wh64 if possible
|
| 110 | * Lifted a bunch of code from clear_user.S
|
| 111 | * At this point, entry values are:
|
| 112 | * $16 Current destination address
|
| 113 | * $5 A copy of $16
|
| 114 | * $6 The max quadword address to write to
|
| 115 | * $18 Number trailer bytes
|
| 116 | * $3 Number quads to write
|
| 117 | */
|
| 118 |
|
| 119 | and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop)
|
| 120 | subq $3, 16, $4 # E : Only try to unroll if > 128 bytes
|
| 121 | subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64)
|
| 122 | blt $4, loop_b # U :
|
| 123 |
|
| 124 | /*
|
| 125 | * We know we've got at least 16 quads, minimum of one trip
|
| 126 | * through unrolled loop. Do a quad at a time to get us 0mod64
|
| 127 | * aligned.
|
| 128 | */
|
| 129 |
|
| 130 | nop |
| 131 | nop |
| 132 | nop |
| 133 | beq $1, $bigalign_b |
| 134 |
|
| 135 | $alignmod64_b:
|
| 136 | stq $17, 0($5) |
| 137 | subq $3, 1, $3 |
| 138 | addq $1, 8, $1 |
| 139 | addq $5, 8, $4 |
| 140 |
|
| 141 | nop
|
| 142 | nop
|
| 143 | addq $5, 8, $5 |
| 144 | blt $1, $alignmod64_b |
| 145 |
|
| 146 | $bigalign_b:
|
| 147 | /*
|
| 148 | * $3 - number quads left to go
|
| 149 | * $5 - target address (aligned 0mod64)
|
| 150 | * $17 - mask of stuff to store
|
| 151 | * Scratch registers available: $7, $2, $4, $1
|
| 152 | * we know that we'll be taking a minimum of one trip through
|
| 153 | * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
|
| 154 | * Assumes the wh64 needs to be for 2 trips through the loop in the future
|
| 155 | * The wh64 is issued on for the starting destination address for trip +2
|
| 156 | * through the loop, and if there are less than two trips left, the target
|
| 157 | * address will be for the current trip.
|
| 158 | */
|
| 159 |
|
| 160 | $do_wh64_b:
|
| 161 | wh64 ($4) # L1 : memory subsystem write hint
|
| 162 | subq $3, 24, $2 # E : For determining future wh64 addresses
|
| 163 | stq $17, 0($5) # L :
|
| 164 | nop # E :
|
| 165 |
|
| 166 | addq $5, 128, $4 # E : speculative target of next wh64
|
| 167 | stq $17, 8($5) # L :
|
| 168 | stq $17, 16($5) # L :
|
| 169 | addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr)
|
| 170 |
|
| 171 | stq $17, 24($5) # L :
|
| 172 | stq $17, 32($5) # L :
|
| 173 | cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle
|
| 174 | nop
|
| 175 |
|
| 176 | stq $17, 40($5) # L :
|
| 177 | stq $17, 48($5) # L :
|
| 178 | subq $3, 16, $2 # E : Repeat the loop at least once more?
|
| 179 | nop
|
| 180 |
|
| 181 | stq $17, 56($5) # L :
|
| 182 | addq $5, 64, $5 # E :
|
| 183 | subq $3, 8, $3 # E :
|
| 184 | bge $2, $do_wh64_b # U :
|
| 185 |
|
| 186 | nop
|
| 187 | nop
|
| 188 | nop
|
| 189 | beq $3, no_quad_b # U : Might have finished already
|
| 190 |
|
| 191 | .align 4
|
| 192 | /*
|
| 193 | * Simple loop for trailing quadwords, or for small amounts
|
| 194 | * of data (where we can't use an unrolled loop and wh64)
|
| 195 | */
|
| 196 | loop_b:
|
| 197 | stq $17,0($5) |
| 198 | subq $3,1,$3 |
| 199 | addq $5,8,$5 |
| 200 | bne $3,loop_b |
| 201 |
|
| 202 | no_quad_b:
|
| 203 | /*
|
| 204 | * Write 0..7 trailing bytes.
|
| 205 | */
|
| 206 | nop |
| 207 | beq $18,end_b |
| 208 | ldq $7,0($5) |
| 209 | mskqh $7,$6,$2 |
| 210 |
|
| 211 | insqh $17,$6,$4 |
| 212 | bis $2,$4,$1 |
| 213 | stq $1,0($5) |
| 214 | ret $31,($26),1 |
| 215 |
|
| 216 | within_quad_b:
|
| 217 | ldq_u $1,0($16) |
| 218 | insql $17,$16,$2 |
| 219 | mskql $1,$16,$4 |
| 220 | bis $2,$4,$2 |
| 221 |
|
| 222 | mskql $2,$6,$4 |
| 223 | mskqh $1,$6,$2 |
| 224 | bis $2,$4,$1 |
| 225 | stq_u $1,0($16) |
| 226 |
|
| 227 | end_b:
|
| 228 | nop
|
| 229 | nop
|
| 230 | nop
|
| 231 | ret $31,($26),1 |
| 232 | .end ___memset
|
| 233 | EXPORT_SYMBOL(___memset)
|
| 234 |
|
| 235 | /*
|
| 236 | * This is the original body of code, prior to replication and
|
| 237 | * rescheduling. Leave it here, as there may be calls to this
|
| 238 | * entry point.
|
| 239 | */
|
| 240 | .align 4
|
| 241 | .ent __constant_c_memset
|
| 242 | __constant_c_memset:
|
| 243 | .frame $30,0,$26,0
|
| 244 | .prologue 0
|
| 245 |
|
| 246 | addq $18,$16,$6 |
| 247 | bis $16,$16,$0 |
| 248 | xor $16,$6,$1 |
| 249 | ble $18,end |
| 250 |
|
| 251 | bic $1,7,$1 |
| 252 | beq $1,within_one_quad |
| 253 | and $16,7,$3 |
| 254 | beq $3,aligned |
| 255 |
|
| 256 | /*
|
| 257 | * Target address is misaligned, and won't fit within a quadword
|
| 258 | */
|
| 259 | ldq_u $4,0($16) # L : Fetch first partial
|
| 260 | bis $16,$16,$5 # E : Save the address
|
| 261 | insql $17,$16,$2 # U : Insert new bytes
|
| 262 | subq $3,8,$3 # E : Invert (for addressing uses)
|
| 263 |
|
| 264 | addq $18,$3,$18 # E : $18 is new count ($3 is negative)
|
| 265 | mskql $4,$16,$4 # U : clear relevant parts of the quad
|
| 266 | subq $16,$3,$16 # E : $16 is new aligned destination
|
| 267 | bis $2,$4,$1 # E : Final bytes
|
| 268 |
|
| 269 | nop
|
| 270 | stq_u $1,0($5) # L : Store result
|
| 271 | nop
|
| 272 | nop
|
| 273 |
|
| 274 | .align 4
|
| 275 | aligned:
|
| 276 | /*
|
| 277 | * We are now guaranteed to be quad aligned, with at least
|
| 278 | * one partial quad to write.
|
| 279 | */
|
| 280 |
|
| 281 | sra $18,3,$3 # U : Number of remaining quads to write
|
| 282 | and $18,7,$18 # E : Number of trailing bytes to write
|
| 283 | bis $16,$16,$5 # E : Save dest address
|
| 284 | beq $3,no_quad # U : tail stuff only
|
| 285 |
|
| 286 | /*
|
| 287 | * it's worth the effort to unroll this and use wh64 if possible
|
| 288 | * Lifted a bunch of code from clear_user.S
|
| 289 | * At this point, entry values are:
|
| 290 | * $16 Current destination address
|
| 291 | * $5 A copy of $16
|
| 292 | * $6 The max quadword address to write to
|
| 293 | * $18 Number trailer bytes
|
| 294 | * $3 Number quads to write
|
| 295 | */
|
| 296 |
|
| 297 | and $16, 0x3f, $2 |
| 298 | subq $3, 16, $4 |
| 299 | subq $2, 0x40, $1 |
| 300 | blt $4, loop |
| 301 |
|
| 302 | /*
|
| 303 | * We know we've got at least 16 quads, minimum of one trip
|
| 304 | * through unrolled loop. Do a quad at a time to get us 0mod64
|
| 305 | * aligned.
|
| 306 | */
|
| 307 |
|
| 308 | nop # E :
|
| 309 | nop # E :
|
| 310 | nop # E :
|
| 311 | beq $1, $bigalign # U :
|
| 312 |
|
| 313 | $alignmod64:
|
| 314 | stq $17, 0($5) # L :
|
| 315 | subq $3, 1, $3 # E : For consistency later
|
| 316 | addq $1, 8, $1 # E : Increment towards zero for alignment
|
| 317 | addq $5, 8, $4 # E : Initial wh64 address (filler instruction)
|
| 318 |
|
| 319 | nop
|
| 320 | nop
|
| 321 | addq $5, 8, $5 # E : Inc address
|
| 322 | blt $1, $alignmod64 # U :
|
| 323 |
|
| 324 | $bigalign:
|
| 325 | /*
|
| 326 | * $3 - number quads left to go
|
| 327 | * $5 - target address (aligned 0mod64)
|
| 328 | * $17 - mask of stuff to store
|
| 329 | * Scratch registers available: $7, $2, $4, $1
|
| 330 | * we know that we'll be taking a minimum of one trip through
|
| 331 | * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
|
| 332 | * Assumes the wh64 needs to be for 2 trips through the loop in the future
|
| 333 | * The wh64 is issued on for the starting destination address for trip +2
|
| 334 | * through the loop, and if there are less than two trips left, the target
|
| 335 | * address will be for the current trip.
|
| 336 | */
|
| 337 |
|
| 338 | $do_wh64:
|
| 339 | wh64 ($4) |
| 340 | subq $3, 24, $2 |
| 341 | stq $17, 0($5) |
| 342 | nop |
| 343 |
|
| 344 | addq $5, 128, $4 |
| 345 | stq $17, 8($5) |
| 346 | stq $17, 16($5) |
| 347 | addq $5, 64, $7 |
| 348 |
|
| 349 | stq $17, 24($5) |
| 350 | stq $17, 32($5) |
| 351 | cmovlt $2, $7, $4 |
| 352 | nop
|
| 353 |
|
| 354 | stq $17, 40($5) |
| 355 | stq $17, 48($5) |
| 356 | subq $3, 16, $2 |
| 357 | nop
|
| 358 |
|
| 359 | stq $17, 56($5) |
| 360 | addq $5, 64, $5 |
| 361 | subq $3, 8, $3 |
| 362 | bge $2, $do_wh64 |
| 363 |
|
| 364 | nop
|
| 365 | nop
|
| 366 | nop
|
| 367 | beq $3, no_quad |
| 368 |
|
| 369 | .align 4
|
| 370 | /*
|
| 371 | * Simple loop for trailing quadwords, or for small amounts
|
| 372 | * of data (where we can't use an unrolled loop and wh64)
|
| 373 | */
|
| 374 | loop:
|
| 375 | stq $17,0($5) # L :
|
| 376 | subq $3,1,$3 # E : Decrement number quads left
|
| 377 | addq $5,8,$5 # E : Inc address
|
| 378 | bne $3,loop # U : more?
|
| 379 |
|
| 380 | no_quad:
|
| 381 | /*
|
| 382 | * Write 0..7 trailing bytes.
|
| 383 | */
|
| 384 | nop # E :
|
| 385 | beq $18,end # U : All done?
|
| 386 | ldq $7,0($5) # L :
|
| 387 | mskqh $7,$6,$2 # U : Mask final quad
|
| 388 |
|
| 389 | insqh $17,$6,$4 # U : New bits
|
| 390 | bis $2,$4,$1 # E : Put it all together
|
| 391 | stq $1,0($5) # L : And back to memory
|
| 392 | ret $31,($26),1 # L0 :
|
| 393 |
|
| 394 | within_one_quad:
|
| 395 | ldq_u $1,0($16) # L :
|
| 396 | insql $17,$16,$2 # U : New bits
|
| 397 | mskql $1,$16,$4 # U : Clear old
|
| 398 | bis $2,$4,$2 # E : New result
|
| 399 |
|
| 400 | mskql $2,$6,$4 # U :
|
| 401 | mskqh $1,$6,$2 # U :
|
| 402 | bis $2,$4,$1 # E :
|
| 403 | stq_u $1,0($16) # L :
|
| 404 |
|
| 405 | end:
|
| 406 | nop
|
| 407 | nop
|
| 408 | nop
|
| 409 | ret $31,($26),1 # L0 :
|
| 410 | .end __constant_c_memset
|
| 411 | EXPORT_SYMBOL(__constant_c_memset)
|
| 412 |
|
| 413 | /*
|
| 414 | * This is a replicant of the __constant_c_memset code, rescheduled
|
| 415 | * to mask stalls. Note that entry point names also had to change
|
| 416 | */
|
| 417 | .align 5
|
| 418 | .ent __memset16
|
| 419 |
|
| 420 | __memset16:
|
| 421 | .frame $30,0,$26,0
|
| 422 | .prologue 0
|
| 423 |
|
| 424 | inswl $17,0,$5 # U : 000000000000c1c2
|
| 425 | inswl $17,2,$2 # U : 00000000c1c20000
|
| 426 | bis $16,$16,$0 # E : return value
|
| 427 | addq $18,$16,$6 # E : max address to write to
|
| 428 |
|
| 429 | ble $18, end_w # U : zero length requested?
|
| 430 | inswl $17,4,$3 # U : 0000c1c200000000
|
| 431 | inswl $17,6,$4 # U : c1c2000000000000
|
| 432 | xor $16,$6,$1 # E : will complete write be within one quadword?
|
| 433 |
|
| 434 | or $2,$5,$2 # E : 00000000c1c2c1c2
|
| 435 | or $3,$4,$17 # E : c1c2c1c200000000
|
| 436 | bic $1,7,$1 # E : fit within a single quadword
|
| 437 | and $16,7,$3 # E : Target addr misalignment
|
| 438 |
|
| 439 | or $17,$2,$17 # E : c1c2c1c2c1c2c1c2
|
| 440 | beq $1,within_quad_w # U :
|
| 441 | nop
|
| 442 | beq $3,aligned_w # U : target is 0mod8
|
| 443 |
|
| 444 | /*
|
| 445 | * Target address is misaligned, and won't fit within a quadword
|
| 446 | */
|
| 447 | ldq_u $4,0($16) |
| 448 | bis $16,$16,$5 |
| 449 | insql $17,$16,$2 |
| 450 | subq $3,8,$3 |
| 451 |
|
| 452 | addq $18,$3,$18 |
| 453 | mskql $4,$16,$4 |
| 454 | subq $16,$3,$16 |
| 455 | bis $2,$4,$1 |
| 456 |
|
| 457 | nop
|
| 458 | stq_u $1,0($5) |
| 459 | nop
|
| 460 | nop
|
| 461 |
|
| 462 | .align 4
|
| 463 | aligned_w:
|
| 464 | /*
|
| 465 | * We are now guaranteed to be quad aligned, with at least
|
| 466 | * one partial quad to write.
|
| 467 | */
|
| 468 |
|
| 469 | sra $18,3,$3 |
| 470 | and $18,7,$18 |
| 471 | bis $16,$16,$5 |
| 472 | beq $3,no_quad_w |
| 473 |
|
| 474 | /*
|
| 475 | * it's worth the effort to unroll this and use wh64 if possible
|
| 476 | * Lifted a bunch of code from clear_user.S
|
| 477 | * At this point, entry values are:
|
| 478 | * $16 Current destination address
|
| 479 | * $5 A copy of $16
|
| 480 | * $6 The max quadword address to write to
|
| 481 | * $18 Number trailer bytes
|
| 482 | * $3 Number quads to write
|
| 483 | */
|
| 484 |
|
| 485 | and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop)
|
| 486 | subq $3, 16, $4 # E : Only try to unroll if > 128 bytes
|
| 487 | subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64)
|
| 488 | blt $4, loop_w # U :
|
| 489 |
|
| 490 | /*
|
| 491 | * We know we've got at least 16 quads, minimum of one trip
|
| 492 | * through unrolled loop. Do a quad at a time to get us 0mod64
|
| 493 | * aligned.
|
| 494 | */
|
| 495 |
|
| 496 | nop |
| 497 | nop |
| 498 | nop |
| 499 | beq $1, $bigalign_w |
| 500 |
|
| 501 | $alignmod64_w:
|
| 502 | stq $17, 0($5) |
| 503 | subq $3, 1, $3 |
| 504 | addq $1, 8, $1 |
| 505 | addq $5, 8, $4 |
| 506 |
|
| 507 | nop
|
| 508 | nop
|
| 509 | addq $5, 8, $5 |
| 510 | blt $1, $alignmod64_w |
| 511 |
|
| 512 | $bigalign_w:
|
| 513 | /*
|
| 514 | * $3 - number quads left to go
|
| 515 | * $5 - target address (aligned 0mod64)
|
| 516 | * $17 - mask of stuff to store
|
| 517 | * Scratch registers available: $7, $2, $4, $1
|
| 518 | * we know that we'll be taking a minimum of one trip through
|
| 519 | * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
|
| 520 | * Assumes the wh64 needs to be for 2 trips through the loop in the future
|
| 521 | * The wh64 is issued on for the starting destination address for trip +2
|
| 522 | * through the loop, and if there are less than two trips left, the target
|
| 523 | * address will be for the current trip.
|
| 524 | */
|
| 525 |
|
| 526 | $do_wh64_w:
|
| 527 | wh64 ($4) # L1 : memory subsystem write hint
|
| 528 | subq $3, 24, $2 # E : For determining future wh64 addresses
|
| 529 | stq $17, 0($5) # L :
|
| 530 | nop # E :
|
| 531 |
|
| 532 | addq $5, 128, $4 # E : speculative target of next wh64
|
| 533 | stq $17, 8($5) # L :
|
| 534 | stq $17, 16($5) # L :
|
| 535 | addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr)
|
| 536 |
|
| 537 | stq $17, 24($5) # L :
|
| 538 | stq $17, 32($5) # L :
|
| 539 | cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle
|
| 540 | nop
|
| 541 |
|
| 542 | stq $17, 40($5) # L :
|
| 543 | stq $17, 48($5) # L :
|
| 544 | subq $3, 16, $2 # E : Repeat the loop at least once more?
|
| 545 | nop
|
| 546 |
|
| 547 | stq $17, 56($5) # L :
|
| 548 | addq $5, 64, $5 # E :
|
| 549 | subq $3, 8, $3 # E :
|
| 550 | bge $2, $do_wh64_w # U :
|
| 551 |
|
| 552 | nop
|
| 553 | nop
|
| 554 | nop
|
| 555 | beq $3, no_quad_w # U : Might have finished already
|
| 556 |
|
| 557 | .align 4
|
| 558 | /*
|
| 559 | * Simple loop for trailing quadwords, or for small amounts
|
| 560 | * of data (where we can't use an unrolled loop and wh64)
|
| 561 | */
|
| 562 | loop_w:
|
| 563 | stq $17,0($5) |
| 564 | subq $3,1,$3 |
| 565 | addq $5,8,$5 |
| 566 | bne $3,loop_w |
| 567 |
|
| 568 | no_quad_w:
|
| 569 | /*
|
| 570 | * Write 0..7 trailing bytes.
|
| 571 | */
|
| 572 | nop |
| 573 | beq $18,end_w |
| 574 | ldq $7,0($5) |
| 575 | mskqh $7,$6,$2 |
| 576 |
|
| 577 | insqh $17,$6,$4 |
| 578 | bis $2,$4,$1 |
| 579 | stq $1,0($5) |
| 580 | ret $31,($26),1 |
| 581 |
|
| 582 | within_quad_w:
|
| 583 | ldq_u $1,0($16) |
| 584 | insql $17,$16,$2 |
| 585 | mskql $1,$16,$4 |
| 586 | bis $2,$4,$2 |
| 587 |
|
| 588 | mskql $2,$6,$4 |
| 589 | mskqh $1,$6,$2 |
| 590 | bis $2,$4,$1 |
| 591 | stq_u $1,0($16) |
| 592 |
|
| 593 | end_w:
|
| 594 | nop
|
| 595 | nop
|
| 596 | nop
|
| 597 | ret $31,($26),1 |
| 598 |
|
| 599 | .end __memset16
|
| 600 | EXPORT_SYMBOL(__memset16)
|
| 601 |
|
| 602 | memset = ___memset
|
| 603 | __memset = ___memset
|
| 604 | EXPORT_SYMBOL(memset)
|
| 605 | EXPORT_SYMBOL(__memset)
|