projects/linux·blob master · refs

505 lines · 13.2 KiB

1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-arm-bits.h
4 *
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <[email protected]>
6 */
7
8#include "rseq-bits-template.h"
9
10#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
11 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
12
13static inline __attribute__((always_inline))
14int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
15{
16 RSEQ_INJECT_C(9)
17
18 __asm__ __volatile__ goto (
19 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
20 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
21#ifdef RSEQ_COMPARE_TWICE
22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
24#endif
25 /* Start rseq by storing table entry pointer into rseq_cs. */
26 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
28 RSEQ_INJECT_ASM(3)
29 "ldr r0, %[v]\n\t"
30 "cmp %[expect], r0\n\t"
31 "bne %l[cmpfail]\n\t"
32 RSEQ_INJECT_ASM(4)
33#ifdef RSEQ_COMPARE_TWICE
34 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
35 "ldr r0, %[v]\n\t"
36 "cmp %[expect], r0\n\t"
37 "bne %l[error2]\n\t"
38#endif
39 /* final store */
40 "str %[newv], %[v]\n\t"
41 "2:\n\t"
42 RSEQ_INJECT_ASM(5)
43 "b 5f\n\t"
44 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
45 "5:\n\t"
46 : /* gcc asm goto does not allow outputs */
47 : [cpu_id] "r" (cpu),
48 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
49 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
50 [v] "m" (*v),
51 [expect] "r" (expect),
52 [newv] "r" (newv)
53 RSEQ_INJECT_INPUT
54 : "r0", "memory", "cc"
55 RSEQ_INJECT_CLOBBER
56 : abort, cmpfail
57#ifdef RSEQ_COMPARE_TWICE
58 , error1, error2
59#endif
60 );
61 rseq_after_asm_goto();
62 return 0;
63abort:
64 rseq_after_asm_goto();
65 RSEQ_INJECT_FAILED
66 return -1;
67cmpfail:
68 rseq_after_asm_goto();
69 return 1;
70#ifdef RSEQ_COMPARE_TWICE
71error1:
72 rseq_after_asm_goto();
73 rseq_bug("cpu_id comparison failed");
74error2:
75 rseq_after_asm_goto();
76 rseq_bug("expected value comparison failed");
77#endif
78}
79
80static inline __attribute__((always_inline))
81int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
82 long voffp, intptr_t *load, int cpu)
83{
84 RSEQ_INJECT_C(9)
85
86 __asm__ __volatile__ goto (
87 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
89#ifdef RSEQ_COMPARE_TWICE
90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
92#endif
93 /* Start rseq by storing table entry pointer into rseq_cs. */
94 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
95 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
96 RSEQ_INJECT_ASM(3)
97 "ldr r0, %[v]\n\t"
98 "cmp %[expectnot], r0\n\t"
99 "beq %l[cmpfail]\n\t"
100 RSEQ_INJECT_ASM(4)
101#ifdef RSEQ_COMPARE_TWICE
102 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
103 "ldr r0, %[v]\n\t"
104 "cmp %[expectnot], r0\n\t"
105 "beq %l[error2]\n\t"
106#endif
107 "str r0, %[load]\n\t"
108 "add r0, %[voffp]\n\t"
109 "ldr r0, [r0]\n\t"
110 /* final store */
111 "str r0, %[v]\n\t"
112 "2:\n\t"
113 RSEQ_INJECT_ASM(5)
114 "b 5f\n\t"
115 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
116 "5:\n\t"
117 : /* gcc asm goto does not allow outputs */
118 : [cpu_id] "r" (cpu),
119 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
120 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
121 /* final store input */
122 [v] "m" (*v),
123 [expectnot] "r" (expectnot),
124 [voffp] "Ir" (voffp),
125 [load] "m" (*load)
126 RSEQ_INJECT_INPUT
127 : "r0", "memory", "cc"
128 RSEQ_INJECT_CLOBBER
129 : abort, cmpfail
130#ifdef RSEQ_COMPARE_TWICE
131 , error1, error2
132#endif
133 );
134 rseq_after_asm_goto();
135 return 0;
136abort:
137 rseq_after_asm_goto();
138 RSEQ_INJECT_FAILED
139 return -1;
140cmpfail:
141 rseq_after_asm_goto();
142 return 1;
143#ifdef RSEQ_COMPARE_TWICE
144error1:
145 rseq_after_asm_goto();
146 rseq_bug("cpu_id comparison failed");
147error2:
148 rseq_after_asm_goto();
149 rseq_bug("expected value comparison failed");
150#endif
151}
152
153static inline __attribute__((always_inline))
154int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
155{
156 RSEQ_INJECT_C(9)
157
158 __asm__ __volatile__ goto (
159 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
160#ifdef RSEQ_COMPARE_TWICE
161 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
162#endif
163 /* Start rseq by storing table entry pointer into rseq_cs. */
164 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
165 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
166 RSEQ_INJECT_ASM(3)
167#ifdef RSEQ_COMPARE_TWICE
168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
169#endif
170 "ldr r0, %[v]\n\t"
171 "add r0, %[count]\n\t"
172 /* final store */
173 "str r0, %[v]\n\t"
174 "2:\n\t"
175 RSEQ_INJECT_ASM(4)
176 "b 5f\n\t"
177 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
178 "5:\n\t"
179 : /* gcc asm goto does not allow outputs */
180 : [cpu_id] "r" (cpu),
181 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
182 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
183 [v] "m" (*v),
184 [count] "Ir" (count)
185 RSEQ_INJECT_INPUT
186 : "r0", "memory", "cc"
187 RSEQ_INJECT_CLOBBER
188 : abort
189#ifdef RSEQ_COMPARE_TWICE
190 , error1
191#endif
192 );
193 rseq_after_asm_goto();
194 return 0;
195abort:
196 rseq_after_asm_goto();
197 RSEQ_INJECT_FAILED
198 return -1;
199#ifdef RSEQ_COMPARE_TWICE
200error1:
201 rseq_after_asm_goto();
202 rseq_bug("cpu_id comparison failed");
203#endif
204}
205
206static inline __attribute__((always_inline))
207int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
208 intptr_t *v2, intptr_t expect2,
209 intptr_t newv, int cpu)
210{
211 RSEQ_INJECT_C(9)
212
213 __asm__ __volatile__ goto (
214 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
216#ifdef RSEQ_COMPARE_TWICE
217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
220#endif
221 /* Start rseq by storing table entry pointer into rseq_cs. */
222 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
224 RSEQ_INJECT_ASM(3)
225 "ldr r0, %[v]\n\t"
226 "cmp %[expect], r0\n\t"
227 "bne %l[cmpfail]\n\t"
228 RSEQ_INJECT_ASM(4)
229 "ldr r0, %[v2]\n\t"
230 "cmp %[expect2], r0\n\t"
231 "bne %l[cmpfail]\n\t"
232 RSEQ_INJECT_ASM(5)
233#ifdef RSEQ_COMPARE_TWICE
234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
235 "ldr r0, %[v]\n\t"
236 "cmp %[expect], r0\n\t"
237 "bne %l[error2]\n\t"
238 "ldr r0, %[v2]\n\t"
239 "cmp %[expect2], r0\n\t"
240 "bne %l[error3]\n\t"
241#endif
242 /* final store */
243 "str %[newv], %[v]\n\t"
244 "2:\n\t"
245 RSEQ_INJECT_ASM(6)
246 "b 5f\n\t"
247 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
248 "5:\n\t"
249 : /* gcc asm goto does not allow outputs */
250 : [cpu_id] "r" (cpu),
251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
253 /* cmp2 input */
254 [v2] "m" (*v2),
255 [expect2] "r" (expect2),
256 /* final store input */
257 [v] "m" (*v),
258 [expect] "r" (expect),
259 [newv] "r" (newv)
260 RSEQ_INJECT_INPUT
261 : "r0", "memory", "cc"
262 RSEQ_INJECT_CLOBBER
263 : abort, cmpfail
264#ifdef RSEQ_COMPARE_TWICE
265 , error1, error2, error3
266#endif
267 );
268 rseq_after_asm_goto();
269 return 0;
270abort:
271 rseq_after_asm_goto();
272 RSEQ_INJECT_FAILED
273 return -1;
274cmpfail:
275 rseq_after_asm_goto();
276 return 1;
277#ifdef RSEQ_COMPARE_TWICE
278error1:
279 rseq_after_asm_goto();
280 rseq_bug("cpu_id comparison failed");
281error2:
282 rseq_after_asm_goto();
283 rseq_bug("1st expected value comparison failed");
284error3:
285 rseq_after_asm_goto();
286 rseq_bug("2nd expected value comparison failed");
287#endif
288}
289
290#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
291 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
292
293#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
294 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
295
296static inline __attribute__((always_inline))
297int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
298 intptr_t *v2, intptr_t newv2,
299 intptr_t newv, int cpu)
300{
301 RSEQ_INJECT_C(9)
302
303 __asm__ __volatile__ goto (
304 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
305 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
306#ifdef RSEQ_COMPARE_TWICE
307 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
308 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
309#endif
310 /* Start rseq by storing table entry pointer into rseq_cs. */
311 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
312 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
313 RSEQ_INJECT_ASM(3)
314 "ldr r0, %[v]\n\t"
315 "cmp %[expect], r0\n\t"
316 "bne %l[cmpfail]\n\t"
317 RSEQ_INJECT_ASM(4)
318#ifdef RSEQ_COMPARE_TWICE
319 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
320 "ldr r0, %[v]\n\t"
321 "cmp %[expect], r0\n\t"
322 "bne %l[error2]\n\t"
323#endif
324 /* try store */
325 "str %[newv2], %[v2]\n\t"
326 RSEQ_INJECT_ASM(5)
327#ifdef RSEQ_TEMPLATE_MO_RELEASE
328 "dmb\n\t" /* full mb provides store-release */
329#endif
330 /* final store */
331 "str %[newv], %[v]\n\t"
332 "2:\n\t"
333 RSEQ_INJECT_ASM(6)
334 "b 5f\n\t"
335 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
336 "5:\n\t"
337 : /* gcc asm goto does not allow outputs */
338 : [cpu_id] "r" (cpu),
339 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
340 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
341 /* try store input */
342 [v2] "m" (*v2),
343 [newv2] "r" (newv2),
344 /* final store input */
345 [v] "m" (*v),
346 [expect] "r" (expect),
347 [newv] "r" (newv)
348 RSEQ_INJECT_INPUT
349 : "r0", "memory", "cc"
350 RSEQ_INJECT_CLOBBER
351 : abort, cmpfail
352#ifdef RSEQ_COMPARE_TWICE
353 , error1, error2
354#endif
355 );
356 rseq_after_asm_goto();
357 return 0;
358abort:
359 rseq_after_asm_goto();
360 RSEQ_INJECT_FAILED
361 return -1;
362cmpfail:
363 rseq_after_asm_goto();
364 return 1;
365#ifdef RSEQ_COMPARE_TWICE
366error1:
367 rseq_after_asm_goto();
368 rseq_bug("cpu_id comparison failed");
369error2:
370 rseq_after_asm_goto();
371 rseq_bug("expected value comparison failed");
372#endif
373}
374
375
376static inline __attribute__((always_inline))
377int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
378 void *dst, void *src, size_t len,
379 intptr_t newv, int cpu)
380{
381 uint32_t rseq_scratch[3];
382
383 RSEQ_INJECT_C(9)
384
385 __asm__ __volatile__ goto (
386 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
387 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
388#ifdef RSEQ_COMPARE_TWICE
389 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
390 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
391#endif
392 "str %[src], %[rseq_scratch0]\n\t"
393 "str %[dst], %[rseq_scratch1]\n\t"
394 "str %[len], %[rseq_scratch2]\n\t"
395 /* Start rseq by storing table entry pointer into rseq_cs. */
396 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
397 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
398 RSEQ_INJECT_ASM(3)
399 "ldr r0, %[v]\n\t"
400 "cmp %[expect], r0\n\t"
401 "bne 5f\n\t"
402 RSEQ_INJECT_ASM(4)
403#ifdef RSEQ_COMPARE_TWICE
404 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
405 "ldr r0, %[v]\n\t"
406 "cmp %[expect], r0\n\t"
407 "bne 7f\n\t"
408#endif
409 /* try memcpy */
410 "cmp %[len], #0\n\t" \
411 "beq 333f\n\t" \
412 "222:\n\t" \
413 "ldrb %%r0, [%[src]]\n\t" \
414 "strb %%r0, [%[dst]]\n\t" \
415 "adds %[src], #1\n\t" \
416 "adds %[dst], #1\n\t" \
417 "subs %[len], #1\n\t" \
418 "bne 222b\n\t" \
419 "333:\n\t" \
420 RSEQ_INJECT_ASM(5)
421#ifdef RSEQ_TEMPLATE_MO_RELEASE
422 "dmb\n\t" /* full mb provides store-release */
423#endif
424 /* final store */
425 "str %[newv], %[v]\n\t"
426 "2:\n\t"
427 RSEQ_INJECT_ASM(6)
428 /* teardown */
429 "ldr %[len], %[rseq_scratch2]\n\t"
430 "ldr %[dst], %[rseq_scratch1]\n\t"
431 "ldr %[src], %[rseq_scratch0]\n\t"
432 "b 8f\n\t"
433 RSEQ_ASM_DEFINE_ABORT(3, 4,
434 /* teardown */
435 "ldr %[len], %[rseq_scratch2]\n\t"
436 "ldr %[dst], %[rseq_scratch1]\n\t"
437 "ldr %[src], %[rseq_scratch0]\n\t",
438 abort, 1b, 2b, 4f)
439 RSEQ_ASM_DEFINE_CMPFAIL(5,
440 /* teardown */
441 "ldr %[len], %[rseq_scratch2]\n\t"
442 "ldr %[dst], %[rseq_scratch1]\n\t"
443 "ldr %[src], %[rseq_scratch0]\n\t",
444 cmpfail)
445#ifdef RSEQ_COMPARE_TWICE
446 RSEQ_ASM_DEFINE_CMPFAIL(6,
447 /* teardown */
448 "ldr %[len], %[rseq_scratch2]\n\t"
449 "ldr %[dst], %[rseq_scratch1]\n\t"
450 "ldr %[src], %[rseq_scratch0]\n\t",
451 error1)
452 RSEQ_ASM_DEFINE_CMPFAIL(7,
453 /* teardown */
454 "ldr %[len], %[rseq_scratch2]\n\t"
455 "ldr %[dst], %[rseq_scratch1]\n\t"
456 "ldr %[src], %[rseq_scratch0]\n\t",
457 error2)
458#endif
459 "8:\n\t"
460 : /* gcc asm goto does not allow outputs */
461 : [cpu_id] "r" (cpu),
462 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
463 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
464 /* final store input */
465 [v] "m" (*v),
466 [expect] "r" (expect),
467 [newv] "r" (newv),
468 /* try memcpy input */
469 [dst] "r" (dst),
470 [src] "r" (src),
471 [len] "r" (len),
472 [rseq_scratch0] "m" (rseq_scratch[0]),
473 [rseq_scratch1] "m" (rseq_scratch[1]),
474 [rseq_scratch2] "m" (rseq_scratch[2])
475 RSEQ_INJECT_INPUT
476 : "r0", "memory", "cc"
477 RSEQ_INJECT_CLOBBER
478 : abort, cmpfail
479#ifdef RSEQ_COMPARE_TWICE
480 , error1, error2
481#endif
482 );
483 rseq_after_asm_goto();
484 return 0;
485abort:
486 rseq_after_asm_goto();
487 RSEQ_INJECT_FAILED
488 return -1;
489cmpfail:
490 rseq_after_asm_goto();
491 return 1;
492#ifdef RSEQ_COMPARE_TWICE
493error1:
494 rseq_after_asm_goto();
495 rseq_bug("cpu_id comparison failed");
496error2:
497 rseq_after_asm_goto();
498 rseq_bug("expected value comparison failed");
499#endif
500}
501
502#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
503 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
504
505#include "rseq-bits-reset.h"