projects/linux·blob master · refs

454 lines · 12.3 KiB

1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-ppc-bits.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <[email protected]>
6 * (C) Copyright 2016-2018 - Boqun Feng <[email protected]>
7 */
8
9#include "rseq-bits-template.h"
10
11#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
13
14static inline __attribute__((always_inline))
15int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
16{
17 RSEQ_INJECT_C(9)
18
19 __asm__ __volatile__ goto (
20 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
21 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
22#ifdef RSEQ_COMPARE_TWICE
23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
25#endif
26 /* Start rseq by storing table entry pointer into rseq_cs. */
27 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
28 /* cmp cpuid */
29 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
30 RSEQ_INJECT_ASM(3)
31 /* cmp @v equal to @expect */
32 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
33 RSEQ_INJECT_ASM(4)
34#ifdef RSEQ_COMPARE_TWICE
35 /* cmp cpuid */
36 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
37 /* cmp @v equal to @expect */
38 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
39#endif
40 /* final store */
41 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
42 RSEQ_INJECT_ASM(5)
43 RSEQ_ASM_DEFINE_ABORT(4, abort)
44 : /* gcc asm goto does not allow outputs */
45 : [cpu_id] "r" (cpu),
46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
48 [v] "m" (*v),
49 [expect] "r" (expect),
50 [newv] "r" (newv)
51 RSEQ_INJECT_INPUT
52 : "memory", "cc", "r17"
53 RSEQ_INJECT_CLOBBER
54 : abort, cmpfail
55#ifdef RSEQ_COMPARE_TWICE
56 , error1, error2
57#endif
58 );
59 rseq_after_asm_goto();
60 return 0;
61abort:
62 rseq_after_asm_goto();
63 RSEQ_INJECT_FAILED
64 return -1;
65cmpfail:
66 rseq_after_asm_goto();
67 return 1;
68#ifdef RSEQ_COMPARE_TWICE
69error1:
70 rseq_after_asm_goto();
71 rseq_bug("cpu_id comparison failed");
72error2:
73 rseq_after_asm_goto();
74 rseq_bug("expected value comparison failed");
75#endif
76}
77
78static inline __attribute__((always_inline))
79int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
80 long voffp, intptr_t *load, int cpu)
81{
82 RSEQ_INJECT_C(9)
83
84 __asm__ __volatile__ goto (
85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
87#ifdef RSEQ_COMPARE_TWICE
88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
89 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
90#endif
91 /* Start rseq by storing table entry pointer into rseq_cs. */
92 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
93 /* cmp cpuid */
94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
95 RSEQ_INJECT_ASM(3)
96 /* cmp @v not equal to @expectnot */
97 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
98 RSEQ_INJECT_ASM(4)
99#ifdef RSEQ_COMPARE_TWICE
100 /* cmp cpuid */
101 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
102 /* cmp @v not equal to @expectnot */
103 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
104#endif
105 /* load the value of @v */
106 RSEQ_ASM_OP_R_LOAD(v)
107 /* store it in @load */
108 RSEQ_ASM_OP_R_STORE(load)
109 /* dereference voffp(v) */
110 RSEQ_ASM_OP_R_LOADX(voffp)
111 /* final store the value at voffp(v) */
112 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
113 RSEQ_INJECT_ASM(5)
114 RSEQ_ASM_DEFINE_ABORT(4, abort)
115 : /* gcc asm goto does not allow outputs */
116 : [cpu_id] "r" (cpu),
117 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
118 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
119 /* final store input */
120 [v] "m" (*v),
121 [expectnot] "r" (expectnot),
122 [voffp] "b" (voffp),
123 [load] "m" (*load)
124 RSEQ_INJECT_INPUT
125 : "memory", "cc", "r17"
126 RSEQ_INJECT_CLOBBER
127 : abort, cmpfail
128#ifdef RSEQ_COMPARE_TWICE
129 , error1, error2
130#endif
131 );
132 rseq_after_asm_goto();
133 return 0;
134abort:
135 rseq_after_asm_goto();
136 RSEQ_INJECT_FAILED
137 return -1;
138cmpfail:
139 rseq_after_asm_goto();
140 return 1;
141#ifdef RSEQ_COMPARE_TWICE
142error1:
143 rseq_after_asm_goto();
144 rseq_bug("cpu_id comparison failed");
145error2:
146 rseq_after_asm_goto();
147 rseq_bug("expected value comparison failed");
148#endif
149}
150
151static inline __attribute__((always_inline))
152int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
153{
154 RSEQ_INJECT_C(9)
155
156 __asm__ __volatile__ goto (
157 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
158#ifdef RSEQ_COMPARE_TWICE
159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
160#endif
161 /* Start rseq by storing table entry pointer into rseq_cs. */
162 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
163 /* cmp cpuid */
164 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
165 RSEQ_INJECT_ASM(3)
166#ifdef RSEQ_COMPARE_TWICE
167 /* cmp cpuid */
168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
169#endif
170 /* load the value of @v */
171 RSEQ_ASM_OP_R_LOAD(v)
172 /* add @count to it */
173 RSEQ_ASM_OP_R_ADD(count)
174 /* final store */
175 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
176 RSEQ_INJECT_ASM(4)
177 RSEQ_ASM_DEFINE_ABORT(4, abort)
178 : /* gcc asm goto does not allow outputs */
179 : [cpu_id] "r" (cpu),
180 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
181 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
182 /* final store input */
183 [v] "m" (*v),
184 [count] "r" (count)
185 RSEQ_INJECT_INPUT
186 : "memory", "cc", "r17"
187 RSEQ_INJECT_CLOBBER
188 : abort
189#ifdef RSEQ_COMPARE_TWICE
190 , error1
191#endif
192 );
193 rseq_after_asm_goto();
194 return 0;
195abort:
196 rseq_after_asm_goto();
197 RSEQ_INJECT_FAILED
198 return -1;
199#ifdef RSEQ_COMPARE_TWICE
200error1:
201 rseq_after_asm_goto();
202 rseq_bug("cpu_id comparison failed");
203#endif
204}
205
206static inline __attribute__((always_inline))
207int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
208 intptr_t *v2, intptr_t expect2,
209 intptr_t newv, int cpu)
210{
211 RSEQ_INJECT_C(9)
212
213 __asm__ __volatile__ goto (
214 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
216#ifdef RSEQ_COMPARE_TWICE
217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
220#endif
221 /* Start rseq by storing table entry pointer into rseq_cs. */
222 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
223 /* cmp cpuid */
224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
225 RSEQ_INJECT_ASM(3)
226 /* cmp @v equal to @expect */
227 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
228 RSEQ_INJECT_ASM(4)
229 /* cmp @v2 equal to @expct2 */
230 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
231 RSEQ_INJECT_ASM(5)
232#ifdef RSEQ_COMPARE_TWICE
233 /* cmp cpuid */
234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
235 /* cmp @v equal to @expect */
236 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
237 /* cmp @v2 equal to @expct2 */
238 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
239#endif
240 /* final store */
241 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
242 RSEQ_INJECT_ASM(6)
243 RSEQ_ASM_DEFINE_ABORT(4, abort)
244 : /* gcc asm goto does not allow outputs */
245 : [cpu_id] "r" (cpu),
246 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
247 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
248 /* cmp2 input */
249 [v2] "m" (*v2),
250 [expect2] "r" (expect2),
251 /* final store input */
252 [v] "m" (*v),
253 [expect] "r" (expect),
254 [newv] "r" (newv)
255 RSEQ_INJECT_INPUT
256 : "memory", "cc", "r17"
257 RSEQ_INJECT_CLOBBER
258 : abort, cmpfail
259#ifdef RSEQ_COMPARE_TWICE
260 , error1, error2, error3
261#endif
262 );
263 rseq_after_asm_goto();
264 return 0;
265abort:
266 rseq_after_asm_goto();
267 RSEQ_INJECT_FAILED
268 return -1;
269cmpfail:
270 rseq_after_asm_goto();
271 return 1;
272#ifdef RSEQ_COMPARE_TWICE
273error1:
274 rseq_after_asm_goto();
275 rseq_bug("cpu_id comparison failed");
276error2:
277 rseq_after_asm_goto();
278 rseq_bug("1st expected value comparison failed");
279error3:
280 rseq_after_asm_goto();
281 rseq_bug("2nd expected value comparison failed");
282#endif
283}
284
285#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
286 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
287
288#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
289 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
290
291static inline __attribute__((always_inline))
292int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
293 intptr_t *v2, intptr_t newv2,
294 intptr_t newv, int cpu)
295{
296 RSEQ_INJECT_C(9)
297
298 __asm__ __volatile__ goto (
299 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
300 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
301#ifdef RSEQ_COMPARE_TWICE
302 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
303 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
304#endif
305 /* Start rseq by storing table entry pointer into rseq_cs. */
306 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
307 /* cmp cpuid */
308 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
309 RSEQ_INJECT_ASM(3)
310 /* cmp @v equal to @expect */
311 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
312 RSEQ_INJECT_ASM(4)
313#ifdef RSEQ_COMPARE_TWICE
314 /* cmp cpuid */
315 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
316 /* cmp @v equal to @expect */
317 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
318#endif
319 /* try store */
320 RSEQ_ASM_OP_STORE(newv2, v2)
321 RSEQ_INJECT_ASM(5)
322#ifdef RSEQ_TEMPLATE_MO_RELEASE
323 /* for 'release' */
324 "lwsync\n\t"
325#endif
326 /* final store */
327 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
328 RSEQ_INJECT_ASM(6)
329 RSEQ_ASM_DEFINE_ABORT(4, abort)
330 : /* gcc asm goto does not allow outputs */
331 : [cpu_id] "r" (cpu),
332 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
333 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
334 /* try store input */
335 [v2] "m" (*v2),
336 [newv2] "r" (newv2),
337 /* final store input */
338 [v] "m" (*v),
339 [expect] "r" (expect),
340 [newv] "r" (newv)
341 RSEQ_INJECT_INPUT
342 : "memory", "cc", "r17"
343 RSEQ_INJECT_CLOBBER
344 : abort, cmpfail
345#ifdef RSEQ_COMPARE_TWICE
346 , error1, error2
347#endif
348 );
349 rseq_after_asm_goto();
350 return 0;
351abort:
352 rseq_after_asm_goto();
353 RSEQ_INJECT_FAILED
354 return -1;
355cmpfail:
356 rseq_after_asm_goto();
357 return 1;
358#ifdef RSEQ_COMPARE_TWICE
359error1:
360 rseq_after_asm_goto();
361 rseq_bug("cpu_id comparison failed");
362error2:
363 rseq_after_asm_goto();
364 rseq_bug("expected value comparison failed");
365#endif
366}
367
368static inline __attribute__((always_inline))
369int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
370 void *dst, void *src, size_t len,
371 intptr_t newv, int cpu)
372{
373 RSEQ_INJECT_C(9)
374
375 __asm__ __volatile__ goto (
376 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
377 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
378#ifdef RSEQ_COMPARE_TWICE
379 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
380 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
381#endif
382 /* setup for mempcy */
383 "mr %%r19, %[len]\n\t"
384 "mr %%r20, %[src]\n\t"
385 "mr %%r21, %[dst]\n\t"
386 /* Start rseq by storing table entry pointer into rseq_cs. */
387 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
388 /* cmp cpuid */
389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
390 RSEQ_INJECT_ASM(3)
391 /* cmp @v equal to @expect */
392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
393 RSEQ_INJECT_ASM(4)
394#ifdef RSEQ_COMPARE_TWICE
395 /* cmp cpuid */
396 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
397 /* cmp @v equal to @expect */
398 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
399#endif
400 /* try memcpy */
401 RSEQ_ASM_OP_R_MEMCPY()
402 RSEQ_INJECT_ASM(5)
403#ifdef RSEQ_TEMPLATE_MO_RELEASE
404 /* for 'release' */
405 "lwsync\n\t"
406#endif
407 /* final store */
408 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
409 RSEQ_INJECT_ASM(6)
410 /* teardown */
411 RSEQ_ASM_DEFINE_ABORT(4, abort)
412 : /* gcc asm goto does not allow outputs */
413 : [cpu_id] "r" (cpu),
414 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
415 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
416 /* final store input */
417 [v] "m" (*v),
418 [expect] "r" (expect),
419 [newv] "r" (newv),
420 /* try memcpy input */
421 [dst] "r" (dst),
422 [src] "r" (src),
423 [len] "r" (len)
424 RSEQ_INJECT_INPUT
425 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
426 RSEQ_INJECT_CLOBBER
427 : abort, cmpfail
428#ifdef RSEQ_COMPARE_TWICE
429 , error1, error2
430#endif
431 );
432 rseq_after_asm_goto();
433 return 0;
434abort:
435 rseq_after_asm_goto();
436 RSEQ_INJECT_FAILED
437 return -1;
438cmpfail:
439 rseq_after_asm_goto();
440 return 1;
441#ifdef RSEQ_COMPARE_TWICE
442error1:
443 rseq_after_asm_goto();
444 rseq_bug("cpu_id comparison failed");
445error2:
446 rseq_after_asm_goto();
447 rseq_bug("expected value comparison failed");
448#endif
449}
450
451#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
452 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
453
454#include "rseq-bits-reset.h"