projects/linux·blob master · refs

485 lines · 9.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Optimized string functions
4 *
5 * S390 version
6 * Copyright IBM Corp. 2004
7 * Author(s): Martin Schwidefsky ([email protected])
8 */
9
10#define IN_ARCH_STRING_C 1
11#ifndef __NO_FORTIFY
12# define __NO_FORTIFY
13#endif
14
15#include <linux/types.h>
16#include <linux/string.h>
17#include <linux/export.h>
18#include <asm/facility.h>
19#include <asm/asm.h>
20
21#define SYMBOL_FUNCTION_ALIAS(alias, name) \
22asm(".globl " __stringify(alias) "\n\t" \
23 ".set " __stringify(alias) "," __stringify(name))
24
25#ifdef __HAVE_ARCH_MEMMOVE
26noinstr void *__memmove(void *dest, const void *src, size_t n)
27{
28 const char *s = src;
29 char *d = dest;
30
31 if (!n)
32 return dest;
33 if ((d <= s || d >= s + n)) {
34 /* Forward copy */
35 while (n >= 256) {
36 asm volatile(
37 " mvc 0(256,%[d]),0(%[s])\n"
38 :
39 : [d] "a" (d), [s] "a" (s)
40 : "memory");
41 d += 256;
42 s += 256;
43 n -= 256;
44 }
45 if (n) {
46 asm volatile(
47 " exrl %[n],0f\n"
48 " j 1f\n"
49 "0: mvc 0(1,%[d]),0(%[s])\n"
50 "1:"
51 :
52 : [d] "a" (d), [s] "a" (s), [n] "a" (n - 1)
53 : "memory");
54 }
55 return dest;
56 }
57 /* Backward copy */
58 if (test_facility(61)) {
59 /* Use mvcrl instruction if available */
60 while (n >= 256) {
61 asm volatile(
62 " lghi %%r0,255\n"
63 " .insn sse,0xe50a00000000,%[d],%[s]\n"
64 : [d] "=Q" (*(d + n - 256))
65 : [s] "Q" (*(s + n - 256))
66 : "0", "memory");
67 n -= 256;
68 }
69 if (n) {
70 asm volatile(
71 " lgr %%r0,%[n]\n"
72 " .insn sse,0xe50a00000000,%[d],%[s]\n"
73 : [d] "=Q" (*d)
74 : [s] "Q" (*s), [n] "d" (n - 1)
75 : "0", "memory");
76 }
77 } else {
78 while (n--)
79 d[n] = s[n];
80 }
81 return dest;
82}
83SYMBOL_FUNCTION_ALIAS(memmove, __memmove);
84EXPORT_SYMBOL(__memmove);
85EXPORT_SYMBOL(memmove);
86#endif
87
88#ifdef __HAVE_ARCH_MEMSET
89noinstr void *__memset(void *s, int c, size_t n)
90{
91 char *xs = s;
92
93 if (!n)
94 return s;
95 if (!c) {
96 /* Clear memory */
97 while (n >= 256) {
98 asm volatile(
99 " xc 0(256,%[xs]),0(%[xs])"
100 :
101 : [xs] "a" (xs)
102 : "cc", "memory");
103 xs += 256;
104 n -= 256;
105 }
106 if (!n)
107 return s;
108 asm volatile(
109 " exrl %[n],0f\n"
110 " j 1f\n"
111 "0: xc 0(1,%[xs]),0(%[xs])\n"
112 "1:"
113 :
114 : [xs] "a" (xs), [n] "a" (n - 1)
115 : "cc", "memory");
116 } else {
117 /* Fill memory */
118 while (n >= 256) {
119 *xs = c;
120 asm volatile(
121 " mvc 1(255,%[xs]),0(%[xs])"
122 :
123 : [xs] "a" (xs)
124 : "memory");
125 xs += 256;
126 n -= 256;
127 }
128 if (!n)
129 return s;
130 *xs = c;
131 if (n == 1)
132 return s;
133 asm volatile(
134 " exrl %[n],0f\n"
135 " j 1f\n"
136 "0: mvc 1(1,%[xs]),0(%[xs])\n"
137 "1:"
138 :
139 : [xs] "a" (xs), [n] "a" (n - 2)
140 : "memory");
141 }
142 return s;
143}
144SYMBOL_FUNCTION_ALIAS(memset, __memset);
145EXPORT_SYMBOL(__memset);
146EXPORT_SYMBOL(memset);
147#endif
148
149#ifdef __HAVE_ARCH_MEMCPY
150noinstr void *__memcpy(void *dest, const void *src, size_t n)
151{
152 void *d = dest;
153
154 if (!n)
155 return d;
156 while (n >= 256) {
157 asm volatile(
158 " mvc 0(256,%[dest]),0(%[src])"
159 :
160 : [dest] "a" (dest), [src] "a" (src)
161 : "memory");
162 dest += 256;
163 src += 256;
164 n -= 256;
165 }
166 if (!n)
167 return d;
168 asm volatile(
169 " exrl %[n],1f\n"
170 " j 2f\n"
171 "1: mvc 0(1,%[dest]),0(%[src])\n"
172 "2:"
173 :
174 : [dest] "a" (dest), [src] "a" (src), [n] "a" (n - 1)
175 : "memory");
176 return d;
177}
178SYMBOL_FUNCTION_ALIAS(memcpy, __memcpy);
179EXPORT_SYMBOL(__memcpy);
180EXPORT_SYMBOL(memcpy);
181#endif
182
183#define DEFINE_MEMSET(_bits, _bytes, _type) \
184void *__memset##_bits(_type *s, _type v, size_t n) \
185{ \
186 _type *xs = s; \
187 \
188 if (!n) \
189 return s; \
190 while (n >= 256) { \
191 *xs = v; \
192 asm volatile( \
193 " mvc %[_b](256-%[_b],%[xs]),0(%[xs])\n" \
194 : \
195 : [xs] "a" (xs), [_b] "i" (_bytes) \
196 : "memory"); \
197 xs = (_type *)((char *)xs + 256); \
198 n -= 256; \
199 } \
200 if (!n) \
201 return s; \
202 *xs = v; \
203 if (n == _bytes) \
204 return s; \
205 n -= _bytes + 1; \
206 asm volatile( \
207 " exrl %[n],1f\n" \
208 " j 2f\n" \
209 "1: mvc %[_b](1,%[xs]),0(%[xs])\n" \
210 "2:" \
211 : \
212 : [n] "a" (n), [xs] "a" (xs), [_b] "i" (_bytes) \
213 : "memory"); \
214 return s; \
215} \
216EXPORT_SYMBOL(__memset##_bits)
217
218#ifdef __HAVE_ARCH_MEMSET16
219DEFINE_MEMSET(16, 2, uint16_t);
220#endif
221
222#ifdef __HAVE_ARCH_MEMSET32
223DEFINE_MEMSET(32, 4, uint32_t);
224#endif
225
226#ifdef __HAVE_ARCH_MEMSET64
227DEFINE_MEMSET(64, 8, uint64_t);
228#endif
229
230/*
231 * Helper functions to find the end of a string
232 */
233static inline char *__strend(const char *s)
234{
235 unsigned long e = 0;
236
237 asm volatile(
238 " lghi 0,0\n"
239 "0: srst %[e],%[s]\n"
240 " jo 0b"
241 : [e] "+&a" (e), [s] "+&a" (s)
242 :
243 : "cc", "memory", "0");
244 return (char *)e;
245}
246
247static inline char *__strnend(const char *s, size_t n)
248{
249 const char *p = s + n;
250
251 asm volatile(
252 " lghi 0,0\n"
253 "0: srst %[p],%[s]\n"
254 " jo 0b"
255 : [p] "+&d" (p), [s] "+&a" (s)
256 :
257 : "cc", "memory", "0");
258 return (char *)p;
259}
260
261/**
262 * strlen - Find the length of a string
263 * @s: The string to be sized
264 *
265 * returns the length of @s
266 */
267#ifdef __HAVE_ARCH_STRLEN
268size_t strlen(const char *s)
269{
270 return __strend(s) - s;
271}
272EXPORT_SYMBOL(strlen);
273#endif
274
275/**
276 * strnlen - Find the length of a length-limited string
277 * @s: The string to be sized
278 * @n: The maximum number of bytes to search
279 *
280 * returns the minimum of the length of @s and @n
281 */
282#ifdef __HAVE_ARCH_STRNLEN
283size_t strnlen(const char *s, size_t n)
284{
285 return __strnend(s, n) - s;
286}
287EXPORT_SYMBOL(strnlen);
288#endif
289
290/**
291 * strcat - Append one %NUL-terminated string to another
292 * @dest: The string to be appended to
293 * @src: The string to append to it
294 *
295 * returns a pointer to @dest
296 */
297#ifdef __HAVE_ARCH_STRCAT
298char *strcat(char *dest, const char *src)
299{
300 unsigned long dummy = 0;
301 char *ret = dest;
302
303 asm volatile(
304 " lghi 0,0\n"
305 "0: srst %[dummy],%[dest]\n"
306 " jo 0b\n"
307 "1: mvst %[dummy],%[src]\n"
308 " jo 1b"
309 : [dummy] "+&a" (dummy), [dest] "+&a" (dest), [src] "+&a" (src)
310 :
311 : "cc", "memory", "0");
312 return ret;
313}
314EXPORT_SYMBOL(strcat);
315#endif
316
317/**
318 * strncat - Append a length-limited, %NUL-terminated string to another
319 * @dest: The string to be appended to
320 * @src: The string to append to it
321 * @n: The maximum numbers of bytes to copy
322 *
323 * returns a pointer to @dest
324 */
325#ifdef __HAVE_ARCH_STRNCAT
326char *strncat(char *dest, const char *src, size_t n)
327{
328 size_t len = __strnend(src, n) - src;
329 char *p = __strend(dest);
330
331 p[len] = '\0';
332 memcpy(p, src, len);
333 return dest;
334}
335EXPORT_SYMBOL(strncat);
336#endif
337
338/**
339 * strcmp - Compare two strings
340 * @s1: One string
341 * @s2: Another string
342 *
343 * returns 0 if @s1 and @s2 are equal,
344 * < 0 if @s1 is less than @s2
345 * > 0 if @s1 is greater than @s2
346 */
347#ifdef __HAVE_ARCH_STRCMP
348int strcmp(const char *s1, const char *s2)
349{
350 int ret = 0;
351
352 asm volatile(
353 " lghi 0,0\n"
354 "0: clst %[s1],%[s2]\n"
355 " jo 0b\n"
356 " je 1f\n"
357 " ic %[ret],0(%[s1])\n"
358 " ic 0,0(%[s2])\n"
359 " sr %[ret],0\n"
360 "1:"
361 : [ret] "+&d" (ret), [s1] "+&a" (s1), [s2] "+&a" (s2)
362 :
363 : "cc", "memory", "0");
364 return ret;
365}
366EXPORT_SYMBOL(strcmp);
367#endif
368
369static inline int clcle(const char *s1, unsigned long l1,
370 const char *s2, unsigned long l2)
371{
372 union register_pair r1 = { .even = (unsigned long)s1, .odd = l1, };
373 union register_pair r3 = { .even = (unsigned long)s2, .odd = l2, };
374 int cc;
375
376 asm volatile(
377 "0: clcle %[r1],%[r3],0\n"
378 " jo 0b\n"
379 CC_IPM(cc)
380 : CC_OUT(cc, cc), [r1] "+d" (r1.pair), [r3] "+d" (r3.pair)
381 :
382 : CC_CLOBBER_LIST("memory"));
383 return CC_TRANSFORM(cc);
384}
385
386/**
387 * strstr - Find the first substring in a %NUL terminated string
388 * @s1: The string to be searched
389 * @s2: The string to search for
390 */
391#ifdef __HAVE_ARCH_STRSTR
392char *strstr(const char *s1, const char *s2)
393{
394 int l1, l2;
395
396 l2 = __strend(s2) - s2;
397 if (!l2)
398 return (char *) s1;
399 l1 = __strend(s1) - s1;
400 while (l1-- >= l2) {
401 int cc;
402
403 cc = clcle(s1, l2, s2, l2);
404 if (!cc)
405 return (char *) s1;
406 s1++;
407 }
408 return NULL;
409}
410EXPORT_SYMBOL(strstr);
411#endif
412
413/**
414 * memchr - Find a character in an area of memory.
415 * @s: The memory area
416 * @c: The byte to search for
417 * @n: The size of the area.
418 *
419 * returns the address of the first occurrence of @c, or %NULL
420 * if @c is not found
421 */
422#ifdef __HAVE_ARCH_MEMCHR
423void *memchr(const void *s, int c, size_t n)
424{
425 const void *ret = s + n;
426
427 asm volatile(
428 " lgr 0,%[c]\n"
429 "0: srst %[ret],%[s]\n"
430 " jo 0b\n"
431 " jl 1f\n"
432 " la %[ret],0\n"
433 "1:"
434 : [ret] "+&a" (ret), [s] "+&a" (s)
435 : [c] "d" (c)
436 : "cc", "memory", "0");
437 return (void *) ret;
438}
439EXPORT_SYMBOL(memchr);
440#endif
441
442/**
443 * memcmp - Compare two areas of memory
444 * @s1: One area of memory
445 * @s2: Another area of memory
446 * @n: The size of the area.
447 */
448#ifdef __HAVE_ARCH_MEMCMP
449int memcmp(const void *s1, const void *s2, size_t n)
450{
451 int ret;
452
453 ret = clcle(s1, n, s2, n);
454 if (ret)
455 ret = ret == 1 ? -1 : 1;
456 return ret;
457}
458EXPORT_SYMBOL(memcmp);
459#endif
460
461/**
462 * memscan - Find a character in an area of memory.
463 * @s: The memory area
464 * @c: The byte to search for
465 * @n: The size of the area.
466 *
467 * returns the address of the first occurrence of @c, or 1 byte past
468 * the area if @c is not found
469 */
470#ifdef __HAVE_ARCH_MEMSCAN
471void *memscan(void *s, int c, size_t n)
472{
473 const void *ret = s + n;
474
475 asm volatile(
476 " lgr 0,%[c]\n"
477 "0: srst %[ret],%[s]\n"
478 " jo 0b"
479 : [ret] "+&a" (ret), [s] "+&a" (s)
480 : [c] "d" (c)
481 : "cc", "memory", "0");
482 return (void *)ret;
483}
484EXPORT_SYMBOL(memscan);
485#endif