projects/linux·blob master · refs

158 lines · 3.7 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (c) 2012-2022, Arm Limited.
4 *
5 * Adapted from the original at:
6 * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strcmp.S
7 */
8
9#include <linux/linkage.h>
10#include <asm/assembler.h>
11
12/* Assumptions:
13 *
14 * ARMv8-a, AArch64.
15 * MTE compatible.
16 */
17
18#define L(label) .L ## label
19
20#define REP8_01 0x0101010101010101
21#define REP8_7f 0x7f7f7f7f7f7f7f7f
22
23#define src1 x0
24#define src2 x1
25#define result x0
26
27#define data1 x2
28#define data1w w2
29#define data2 x3
30#define data2w w3
31#define has_nul x4
32#define diff x5
33#define off1 x5
34#define syndrome x6
35#define tmp x6
36#define data3 x7
37#define zeroones x8
38#define shift x9
39#define off2 x10
40
41/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
42 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
43 can be done in parallel across the entire word.
44 Since carry propagation makes 0x1 bytes before a NUL byte appear
45 NUL too in big-endian, byte-reverse the data before the NUL check. */
46
47
48SYM_FUNC_START(__pi_strcmp)
49 sub off2, src2, src1
50 mov zeroones, REP8_01
51 and tmp, src1, 7
52 tst off2, 7
53 b.ne L(misaligned8)
54 cbnz tmp, L(mutual_align)
55
56 .p2align 4
57
58L(loop_aligned):
59 ldr data2, [src1, off2]
60 ldr data1, [src1], 8
61L(start_realigned):
62 sub has_nul, data1, zeroones
63 orr tmp, data1, REP8_7f
64 bics has_nul, has_nul, tmp /* Non-zero if NUL terminator. */
65 ccmp data1, data2, 0, eq
66 b.eq L(loop_aligned)
67 eor diff, data1, data2
68 orr syndrome, diff, has_nul
69L(end):
70 rev syndrome, syndrome
71 rev data1, data1
72 rev data2, data2
73 clz shift, syndrome
74 /* The most-significant-non-zero bit of the syndrome marks either the
75 first bit that is different, or the top bit of the first zero byte.
76 Shifting left now will bring the critical information into the
77 top bits. */
78 lsl data1, data1, shift
79 lsl data2, data2, shift
80 /* But we need to zero-extend (char is unsigned) the value and then
81 perform a signed 32-bit subtraction. */
82 lsr data1, data1, 56
83 sub result, data1, data2, lsr 56
84 ret
85
86 .p2align 4
87
88L(mutual_align):
89 /* Sources are mutually aligned, but are not currently at an
90 alignment boundary. Round down the addresses and then mask off
91 the bytes that precede the start point. */
92 bic src1, src1, 7
93 ldr data2, [src1, off2]
94 ldr data1, [src1], 8
95 neg shift, src2, lsl 3 /* Bits to alignment -64. */
96 mov tmp, -1
97 lsr tmp, tmp, shift
98 orr data1, data1, tmp
99 orr data2, data2, tmp
100 b L(start_realigned)
101
102L(misaligned8):
103 /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always
104 checking to make sure that we don't access beyond the end of SRC2. */
105 cbz tmp, L(src1_aligned)
106L(do_misaligned):
107 ldrb data1w, [src1], 1
108 ldrb data2w, [src2], 1
109 cmp data1w, 0
110 ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */
111 b.ne L(done)
112 tst src1, 7
113 b.ne L(do_misaligned)
114
115L(src1_aligned):
116 neg shift, src2, lsl 3
117 bic src2, src2, 7
118 ldr data3, [src2], 8
119 lsr tmp, zeroones, shift
120 orr data3, data3, tmp
121 sub has_nul, data3, zeroones
122 orr tmp, data3, REP8_7f
123 bics has_nul, has_nul, tmp
124 b.ne L(tail)
125
126 sub off1, src2, src1
127
128 .p2align 4
129
130L(loop_unaligned):
131 ldr data3, [src1, off1]
132 ldr data2, [src1, off2]
133 sub has_nul, data3, zeroones
134 orr tmp, data3, REP8_7f
135 ldr data1, [src1], 8
136 bics has_nul, has_nul, tmp
137 ccmp data1, data2, 0, eq
138 b.eq L(loop_unaligned)
139
140 lsl tmp, has_nul, shift
141 eor diff, data1, data2
142 orr syndrome, diff, tmp
143 cbnz syndrome, L(end)
144L(tail):
145 ldr data1, [src1]
146 neg shift, shift
147 lsr data2, data3, shift
148 lsr has_nul, has_nul, shift
149 eor diff, data1, data2
150 orr syndrome, diff, has_nul
151 b L(end)
152
153L(done):
154 sub result, data1, data2
155 ret
156SYM_FUNC_END(__pi_strcmp)
157SYM_FUNC_ALIAS_WEAK(strcmp, __pi_strcmp)
158EXPORT_SYMBOL_NOKASAN(strcmp)