projects/linux·blob master · refs

462 lines · 12.6 KiB

1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * Author: Paul Burton <[email protected]>
4 * (C) Copyright 2018 MIPS Tech LLC
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <[email protected]>
6 */
7
8#include "rseq-bits-template.h"
9
10#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
11 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
12
13static inline __attribute__((always_inline))
14int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
15{
16 RSEQ_INJECT_C(9)
17
18 __asm__ __volatile__ goto (
19 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
20 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
21#ifdef RSEQ_COMPARE_TWICE
22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
24#endif
25 /* Start rseq by storing table entry pointer into rseq_cs. */
26 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
28 RSEQ_INJECT_ASM(3)
29 LONG_L " $4, %[v]\n\t"
30 "bne $4, %[expect], %l[cmpfail]\n\t"
31 RSEQ_INJECT_ASM(4)
32#ifdef RSEQ_COMPARE_TWICE
33 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
34 LONG_L " $4, %[v]\n\t"
35 "bne $4, %[expect], %l[error2]\n\t"
36#endif
37 /* final store */
38 LONG_S " %[newv], %[v]\n\t"
39 "2:\n\t"
40 RSEQ_INJECT_ASM(5)
41 "b 5f\n\t"
42 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
43 "5:\n\t"
44 : /* gcc asm goto does not allow outputs */
45 : [cpu_id] "r" (cpu),
46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
48 [v] "m" (*v),
49 [expect] "r" (expect),
50 [newv] "r" (newv)
51 RSEQ_INJECT_INPUT
52 : "$4", "memory"
53 RSEQ_INJECT_CLOBBER
54 : abort, cmpfail
55#ifdef RSEQ_COMPARE_TWICE
56 , error1, error2
57#endif
58 );
59 return 0;
60abort:
61 RSEQ_INJECT_FAILED
62 return -1;
63cmpfail:
64 return 1;
65#ifdef RSEQ_COMPARE_TWICE
66error1:
67 rseq_bug("cpu_id comparison failed");
68error2:
69 rseq_bug("expected value comparison failed");
70#endif
71}
72
73static inline __attribute__((always_inline))
74int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
75 long voffp, intptr_t *load, int cpu)
76{
77 RSEQ_INJECT_C(9)
78
79 __asm__ __volatile__ goto (
80 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
81 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
82#ifdef RSEQ_COMPARE_TWICE
83 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
84 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
85#endif
86 /* Start rseq by storing table entry pointer into rseq_cs. */
87 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
89 RSEQ_INJECT_ASM(3)
90 LONG_L " $4, %[v]\n\t"
91 "beq $4, %[expectnot], %l[cmpfail]\n\t"
92 RSEQ_INJECT_ASM(4)
93#ifdef RSEQ_COMPARE_TWICE
94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
95 LONG_L " $4, %[v]\n\t"
96 "beq $4, %[expectnot], %l[error2]\n\t"
97#endif
98 LONG_S " $4, %[load]\n\t"
99 LONG_ADDI " $4, %[voffp]\n\t"
100 LONG_L " $4, 0($4)\n\t"
101 /* final store */
102 LONG_S " $4, %[v]\n\t"
103 "2:\n\t"
104 RSEQ_INJECT_ASM(5)
105 "b 5f\n\t"
106 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
107 "5:\n\t"
108 : /* gcc asm goto does not allow outputs */
109 : [cpu_id] "r" (cpu),
110 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
111 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
112 /* final store input */
113 [v] "m" (*v),
114 [expectnot] "r" (expectnot),
115 [voffp] "Ir" (voffp),
116 [load] "m" (*load)
117 RSEQ_INJECT_INPUT
118 : "$4", "memory"
119 RSEQ_INJECT_CLOBBER
120 : abort, cmpfail
121#ifdef RSEQ_COMPARE_TWICE
122 , error1, error2
123#endif
124 );
125 return 0;
126abort:
127 RSEQ_INJECT_FAILED
128 return -1;
129cmpfail:
130 return 1;
131#ifdef RSEQ_COMPARE_TWICE
132error1:
133 rseq_bug("cpu_id comparison failed");
134error2:
135 rseq_bug("expected value comparison failed");
136#endif
137}
138
139static inline __attribute__((always_inline))
140int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
141{
142 RSEQ_INJECT_C(9)
143
144 __asm__ __volatile__ goto (
145 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
146#ifdef RSEQ_COMPARE_TWICE
147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
148#endif
149 /* Start rseq by storing table entry pointer into rseq_cs. */
150 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
152 RSEQ_INJECT_ASM(3)
153#ifdef RSEQ_COMPARE_TWICE
154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
155#endif
156 LONG_L " $4, %[v]\n\t"
157 LONG_ADDI " $4, %[count]\n\t"
158 /* final store */
159 LONG_S " $4, %[v]\n\t"
160 "2:\n\t"
161 RSEQ_INJECT_ASM(4)
162 "b 5f\n\t"
163 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
164 "5:\n\t"
165 : /* gcc asm goto does not allow outputs */
166 : [cpu_id] "r" (cpu),
167 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
168 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
169 [v] "m" (*v),
170 [count] "Ir" (count)
171 RSEQ_INJECT_INPUT
172 : "$4", "memory"
173 RSEQ_INJECT_CLOBBER
174 : abort
175#ifdef RSEQ_COMPARE_TWICE
176 , error1
177#endif
178 );
179 return 0;
180abort:
181 RSEQ_INJECT_FAILED
182 return -1;
183#ifdef RSEQ_COMPARE_TWICE
184error1:
185 rseq_bug("cpu_id comparison failed");
186#endif
187}
188
189static inline __attribute__((always_inline))
190int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
191 intptr_t *v2, intptr_t expect2,
192 intptr_t newv, int cpu)
193{
194 RSEQ_INJECT_C(9)
195
196 __asm__ __volatile__ goto (
197 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
198 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
199#ifdef RSEQ_COMPARE_TWICE
200 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
201 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
202 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
203#endif
204 /* Start rseq by storing table entry pointer into rseq_cs. */
205 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
206 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
207 RSEQ_INJECT_ASM(3)
208 LONG_L " $4, %[v]\n\t"
209 "bne $4, %[expect], %l[cmpfail]\n\t"
210 RSEQ_INJECT_ASM(4)
211 LONG_L " $4, %[v2]\n\t"
212 "bne $4, %[expect2], %l[cmpfail]\n\t"
213 RSEQ_INJECT_ASM(5)
214#ifdef RSEQ_COMPARE_TWICE
215 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
216 LONG_L " $4, %[v]\n\t"
217 "bne $4, %[expect], %l[error2]\n\t"
218 LONG_L " $4, %[v2]\n\t"
219 "bne $4, %[expect2], %l[error3]\n\t"
220#endif
221 /* final store */
222 LONG_S " %[newv], %[v]\n\t"
223 "2:\n\t"
224 RSEQ_INJECT_ASM(6)
225 "b 5f\n\t"
226 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
227 "5:\n\t"
228 : /* gcc asm goto does not allow outputs */
229 : [cpu_id] "r" (cpu),
230 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
231 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
232 /* cmp2 input */
233 [v2] "m" (*v2),
234 [expect2] "r" (expect2),
235 /* final store input */
236 [v] "m" (*v),
237 [expect] "r" (expect),
238 [newv] "r" (newv)
239 RSEQ_INJECT_INPUT
240 : "$4", "memory"
241 RSEQ_INJECT_CLOBBER
242 : abort, cmpfail
243#ifdef RSEQ_COMPARE_TWICE
244 , error1, error2, error3
245#endif
246 );
247 return 0;
248abort:
249 RSEQ_INJECT_FAILED
250 return -1;
251cmpfail:
252 return 1;
253#ifdef RSEQ_COMPARE_TWICE
254error1:
255 rseq_bug("cpu_id comparison failed");
256error2:
257 rseq_bug("1st expected value comparison failed");
258error3:
259 rseq_bug("2nd expected value comparison failed");
260#endif
261}
262
263#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
264 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
265
266#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
267 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
268
269static inline __attribute__((always_inline))
270int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
271 intptr_t *v2, intptr_t newv2,
272 intptr_t newv, int cpu)
273{
274 RSEQ_INJECT_C(9)
275
276 __asm__ __volatile__ goto (
277 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
278 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
279#ifdef RSEQ_COMPARE_TWICE
280 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
281 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
282#endif
283 /* Start rseq by storing table entry pointer into rseq_cs. */
284 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
286 RSEQ_INJECT_ASM(3)
287 LONG_L " $4, %[v]\n\t"
288 "bne $4, %[expect], %l[cmpfail]\n\t"
289 RSEQ_INJECT_ASM(4)
290#ifdef RSEQ_COMPARE_TWICE
291 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
292 LONG_L " $4, %[v]\n\t"
293 "bne $4, %[expect], %l[error2]\n\t"
294#endif
295 /* try store */
296 LONG_S " %[newv2], %[v2]\n\t"
297 RSEQ_INJECT_ASM(5)
298#ifdef RSEQ_TEMPLATE_MO_RELEASE
299 "sync\n\t" /* full sync provides store-release */
300#endif
301 /* final store */
302 LONG_S " %[newv], %[v]\n\t"
303 "2:\n\t"
304 RSEQ_INJECT_ASM(6)
305 "b 5f\n\t"
306 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
307 "5:\n\t"
308 : /* gcc asm goto does not allow outputs */
309 : [cpu_id] "r" (cpu),
310 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
311 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
312 /* try store input */
313 [v2] "m" (*v2),
314 [newv2] "r" (newv2),
315 /* final store input */
316 [v] "m" (*v),
317 [expect] "r" (expect),
318 [newv] "r" (newv)
319 RSEQ_INJECT_INPUT
320 : "$4", "memory"
321 RSEQ_INJECT_CLOBBER
322 : abort, cmpfail
323#ifdef RSEQ_COMPARE_TWICE
324 , error1, error2
325#endif
326 );
327 return 0;
328abort:
329 RSEQ_INJECT_FAILED
330 return -1;
331cmpfail:
332 return 1;
333#ifdef RSEQ_COMPARE_TWICE
334error1:
335 rseq_bug("cpu_id comparison failed");
336error2:
337 rseq_bug("expected value comparison failed");
338#endif
339}
340
341static inline __attribute__((always_inline))
342int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
343 void *dst, void *src, size_t len,
344 intptr_t newv, int cpu)
345{
346 uintptr_t rseq_scratch[3];
347
348 RSEQ_INJECT_C(9)
349
350 __asm__ __volatile__ goto (
351 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
352 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
353#ifdef RSEQ_COMPARE_TWICE
354 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
356#endif
357 LONG_S " %[src], %[rseq_scratch0]\n\t"
358 LONG_S " %[dst], %[rseq_scratch1]\n\t"
359 LONG_S " %[len], %[rseq_scratch2]\n\t"
360 /* Start rseq by storing table entry pointer into rseq_cs. */
361 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
363 RSEQ_INJECT_ASM(3)
364 LONG_L " $4, %[v]\n\t"
365 "bne $4, %[expect], 5f\n\t"
366 RSEQ_INJECT_ASM(4)
367#ifdef RSEQ_COMPARE_TWICE
368 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
369 LONG_L " $4, %[v]\n\t"
370 "bne $4, %[expect], 7f\n\t"
371#endif
372 /* try memcpy */
373 "beqz %[len], 333f\n\t" \
374 "222:\n\t" \
375 "lb $4, 0(%[src])\n\t" \
376 "sb $4, 0(%[dst])\n\t" \
377 LONG_ADDI " %[src], 1\n\t" \
378 LONG_ADDI " %[dst], 1\n\t" \
379 LONG_ADDI " %[len], -1\n\t" \
380 "bnez %[len], 222b\n\t" \
381 "333:\n\t" \
382 RSEQ_INJECT_ASM(5)
383#ifdef RSEQ_TEMPLATE_MO_RELEASE
384 "sync\n\t" /* full sync provides store-release */
385#endif
386 /* final store */
387 LONG_S " %[newv], %[v]\n\t"
388 "2:\n\t"
389 RSEQ_INJECT_ASM(6)
390 /* teardown */
391 LONG_L " %[len], %[rseq_scratch2]\n\t"
392 LONG_L " %[dst], %[rseq_scratch1]\n\t"
393 LONG_L " %[src], %[rseq_scratch0]\n\t"
394 "b 8f\n\t"
395 RSEQ_ASM_DEFINE_ABORT(3, 4,
396 /* teardown */
397 LONG_L " %[len], %[rseq_scratch2]\n\t"
398 LONG_L " %[dst], %[rseq_scratch1]\n\t"
399 LONG_L " %[src], %[rseq_scratch0]\n\t",
400 abort, 1b, 2b, 4f)
401 RSEQ_ASM_DEFINE_CMPFAIL(5,
402 /* teardown */
403 LONG_L " %[len], %[rseq_scratch2]\n\t"
404 LONG_L " %[dst], %[rseq_scratch1]\n\t"
405 LONG_L " %[src], %[rseq_scratch0]\n\t",
406 cmpfail)
407#ifdef RSEQ_COMPARE_TWICE
408 RSEQ_ASM_DEFINE_CMPFAIL(6,
409 /* teardown */
410 LONG_L " %[len], %[rseq_scratch2]\n\t"
411 LONG_L " %[dst], %[rseq_scratch1]\n\t"
412 LONG_L " %[src], %[rseq_scratch0]\n\t",
413 error1)
414 RSEQ_ASM_DEFINE_CMPFAIL(7,
415 /* teardown */
416 LONG_L " %[len], %[rseq_scratch2]\n\t"
417 LONG_L " %[dst], %[rseq_scratch1]\n\t"
418 LONG_L " %[src], %[rseq_scratch0]\n\t",
419 error2)
420#endif
421 "8:\n\t"
422 : /* gcc asm goto does not allow outputs */
423 : [cpu_id] "r" (cpu),
424 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
425 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
426 /* final store input */
427 [v] "m" (*v),
428 [expect] "r" (expect),
429 [newv] "r" (newv),
430 /* try memcpy input */
431 [dst] "r" (dst),
432 [src] "r" (src),
433 [len] "r" (len),
434 [rseq_scratch0] "m" (rseq_scratch[0]),
435 [rseq_scratch1] "m" (rseq_scratch[1]),
436 [rseq_scratch2] "m" (rseq_scratch[2])
437 RSEQ_INJECT_INPUT
438 : "$4", "memory"
439 RSEQ_INJECT_CLOBBER
440 : abort, cmpfail
441#ifdef RSEQ_COMPARE_TWICE
442 , error1, error2
443#endif
444 );
445 return 0;
446abort:
447 RSEQ_INJECT_FAILED
448 return -1;
449cmpfail:
450 return 1;
451#ifdef RSEQ_COMPARE_TWICE
452error1:
453 rseq_bug("cpu_id comparison failed");
454error2:
455 rseq_bug("expected value comparison failed");
456#endif
457}
458
459#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
460 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
461
462#include "rseq-bits-reset.h"