projects/linux·blob master · refs

353 lines · 10.2 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
4 * Copyright (C) 2012 Regents of the University of California
5 * Copyright (C) 2017 SiFive
6 */
7
8#ifndef _ASM_RISCV_ATOMIC_H
9#define _ASM_RISCV_ATOMIC_H
10
11#ifdef CONFIG_GENERIC_ATOMIC64
12# include <asm-generic/atomic64.h>
13#else
14# if (__riscv_xlen < 64)
15# error "64-bit atomics require XLEN to be at least 64"
16# endif
17#endif
18
19#include <asm/cmpxchg.h>
20
21#define __atomic_acquire_fence() \
22 __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
23
24#define __atomic_release_fence() \
25 __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
26
27static __always_inline int arch_atomic_read(const atomic_t *v)
28{
29 return READ_ONCE(v->counter);
30}
31static __always_inline void arch_atomic_set(atomic_t *v, int i)
32{
33 WRITE_ONCE(v->counter, i);
34}
35
36#ifndef CONFIG_GENERIC_ATOMIC64
37#define ATOMIC64_INIT(i) { (i) }
38static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
39{
40 return READ_ONCE(v->counter);
41}
42static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
43{
44 WRITE_ONCE(v->counter, i);
45}
46#endif
47
48/*
49 * First, the atomic ops that have no ordering constraints and therefore don't
50 * have the AQ or RL bits set. These don't return anything, so there's only
51 * one version to worry about.
52 */
53#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
54static __always_inline \
55void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
56{ \
57 __asm__ __volatile__ ( \
58 " amo" #asm_op "." #asm_type " zero, %1, %0" \
59 : "+A" (v->counter) \
60 : "r" (I) \
61 : "memory"); \
62} \
63
64#ifdef CONFIG_GENERIC_ATOMIC64
65#define ATOMIC_OPS(op, asm_op, I) \
66 ATOMIC_OP (op, asm_op, I, w, int, )
67#else
68#define ATOMIC_OPS(op, asm_op, I) \
69 ATOMIC_OP (op, asm_op, I, w, int, ) \
70 ATOMIC_OP (op, asm_op, I, d, s64, 64)
71#endif
72
73ATOMIC_OPS(add, add, i)
74ATOMIC_OPS(sub, add, -i)
75ATOMIC_OPS(and, and, i)
76ATOMIC_OPS( or, or, i)
77ATOMIC_OPS(xor, xor, i)
78
79#undef ATOMIC_OP
80#undef ATOMIC_OPS
81
82/*
83 * Atomic ops that have ordered, relaxed, acquire, and release variants.
84 * There's two flavors of these: the arithmetic ops have both fetch and return
85 * versions, while the logical ops only have fetch versions.
86 */
87#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
88static __always_inline \
89c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \
90 atomic##prefix##_t *v) \
91{ \
92 register c_type ret; \
93 __asm__ __volatile__ ( \
94 " amo" #asm_op "." #asm_type " %1, %2, %0" \
95 : "+A" (v->counter), "=r" (ret) \
96 : "r" (I) \
97 : "memory"); \
98 return ret; \
99} \
100static __always_inline \
101c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
102{ \
103 register c_type ret; \
104 __asm__ __volatile__ ( \
105 " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \
106 : "+A" (v->counter), "=r" (ret) \
107 : "r" (I) \
108 : "memory"); \
109 return ret; \
110}
111
112#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
113static __always_inline \
114c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \
115 atomic##prefix##_t *v) \
116{ \
117 return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \
118} \
119static __always_inline \
120c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \
121{ \
122 return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \
123}
124
125#ifdef CONFIG_GENERIC_ATOMIC64
126#define ATOMIC_OPS(op, asm_op, c_op, I) \
127 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
128 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, )
129#else
130#define ATOMIC_OPS(op, asm_op, c_op, I) \
131 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
132 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \
133 ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \
134 ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64)
135#endif
136
137ATOMIC_OPS(add, add, +, i)
138ATOMIC_OPS(sub, add, +, -i)
139
140#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
141#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
142#define arch_atomic_add_return arch_atomic_add_return
143#define arch_atomic_sub_return arch_atomic_sub_return
144
145#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
146#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
147#define arch_atomic_fetch_add arch_atomic_fetch_add
148#define arch_atomic_fetch_sub arch_atomic_fetch_sub
149
150#ifndef CONFIG_GENERIC_ATOMIC64
151#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
152#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
153#define arch_atomic64_add_return arch_atomic64_add_return
154#define arch_atomic64_sub_return arch_atomic64_sub_return
155
156#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
157#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
158#define arch_atomic64_fetch_add arch_atomic64_fetch_add
159#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
160#endif
161
162#undef ATOMIC_OPS
163
164#ifdef CONFIG_GENERIC_ATOMIC64
165#define ATOMIC_OPS(op, asm_op, I) \
166 ATOMIC_FETCH_OP(op, asm_op, I, w, int, )
167#else
168#define ATOMIC_OPS(op, asm_op, I) \
169 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \
170 ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64)
171#endif
172
173ATOMIC_OPS(and, and, i)
174ATOMIC_OPS( or, or, i)
175ATOMIC_OPS(xor, xor, i)
176
177#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
178#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
179#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
180#define arch_atomic_fetch_and arch_atomic_fetch_and
181#define arch_atomic_fetch_or arch_atomic_fetch_or
182#define arch_atomic_fetch_xor arch_atomic_fetch_xor
183
184#ifndef CONFIG_GENERIC_ATOMIC64
185#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
186#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
187#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
188#define arch_atomic64_fetch_and arch_atomic64_fetch_and
189#define arch_atomic64_fetch_or arch_atomic64_fetch_or
190#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
191#endif
192
193#undef ATOMIC_OPS
194
195#undef ATOMIC_FETCH_OP
196#undef ATOMIC_OP_RETURN
197
198#define _arch_atomic_fetch_add_unless(_prev, _rc, counter, _a, _u, sfx) \
199({ \
200 __asm__ __volatile__ ( \
201 "0: lr." sfx " %[p], %[c]\n" \
202 " beq %[p], %[u], 1f\n" \
203 " add %[rc], %[p], %[a]\n" \
204 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
205 " bnez %[rc], 0b\n" \
206 RISCV_FULL_BARRIER \
207 "1:\n" \
208 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
209 : [a]"r" (_a), [u]"r" (_u) \
210 : "memory"); \
211})
212
213/* This is required to provide a full barrier on success. */
214static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
215{
216 int prev, rc;
217
218 _arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "w");
219
220 return prev;
221}
222#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
223
224#ifndef CONFIG_GENERIC_ATOMIC64
225static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
226{
227 s64 prev;
228 long rc;
229
230 _arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "d");
231
232 return prev;
233}
234#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
235#endif
236
237#define _arch_atomic_inc_unless_negative(_prev, _rc, counter, sfx) \
238({ \
239 __asm__ __volatile__ ( \
240 "0: lr." sfx " %[p], %[c]\n" \
241 " bltz %[p], 1f\n" \
242 " addi %[rc], %[p], 1\n" \
243 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
244 " bnez %[rc], 0b\n" \
245 RISCV_FULL_BARRIER \
246 "1:\n" \
247 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
248 : \
249 : "memory"); \
250})
251
252static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
253{
254 int prev, rc;
255
256 _arch_atomic_inc_unless_negative(prev, rc, v->counter, "w");
257
258 return !(prev < 0);
259}
260
261#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
262
263#define _arch_atomic_dec_unless_positive(_prev, _rc, counter, sfx) \
264({ \
265 __asm__ __volatile__ ( \
266 "0: lr." sfx " %[p], %[c]\n" \
267 " bgtz %[p], 1f\n" \
268 " addi %[rc], %[p], -1\n" \
269 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
270 " bnez %[rc], 0b\n" \
271 RISCV_FULL_BARRIER \
272 "1:\n" \
273 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
274 : \
275 : "memory"); \
276})
277
278static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v)
279{
280 int prev, rc;
281
282 _arch_atomic_dec_unless_positive(prev, rc, v->counter, "w");
283
284 return !(prev > 0);
285}
286
287#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
288
289#define _arch_atomic_dec_if_positive(_prev, _rc, counter, sfx) \
290({ \
291 __asm__ __volatile__ ( \
292 "0: lr." sfx " %[p], %[c]\n" \
293 " addi %[rc], %[p], -1\n" \
294 " bltz %[rc], 1f\n" \
295 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
296 " bnez %[rc], 0b\n" \
297 RISCV_FULL_BARRIER \
298 "1:\n" \
299 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
300 : \
301 : "memory"); \
302})
303
304static __always_inline int arch_atomic_dec_if_positive(atomic_t *v)
305{
306 int prev, rc;
307
308 _arch_atomic_dec_if_positive(prev, rc, v->counter, "w");
309
310 return prev - 1;
311}
312
313#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
314
315#ifndef CONFIG_GENERIC_ATOMIC64
316static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v)
317{
318 s64 prev;
319 long rc;
320
321 _arch_atomic_inc_unless_negative(prev, rc, v->counter, "d");
322
323 return !(prev < 0);
324}
325
326#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
327
328static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v)
329{
330 s64 prev;
331 long rc;
332
333 _arch_atomic_dec_unless_positive(prev, rc, v->counter, "d");
334
335 return !(prev > 0);
336}
337
338#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
339
340static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
341{
342 s64 prev;
343 long rc;
344
345 _arch_atomic_dec_if_positive(prev, rc, v->counter, "d");
346
347 return prev - 1;
348}
349
350#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
351#endif
352
353#endif /* _ASM_RISCV_ATOMIC_H */