projects/linux·blob master · refs

283 lines · 6.3 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ALPHA_CMPXCHG_H
3#define _ALPHA_CMPXCHG_H
4
5/*
6 * Atomic exchange.
7 * Since it can be used to implement critical sections
8 * it must clobber "memory" (also for interrupts in UP).
9 */
10
11static inline unsigned long
12____xchg_u8(volatile char *m, unsigned long val)
13{
14 unsigned long ret, tmp, addr64;
15
16 __asm__ __volatile__(
17 " andnot %4,7,%3\n"
18 " insbl %1,%4,%1\n"
19 "1: ldq_l %2,0(%3)\n"
20 " extbl %2,%4,%0\n"
21 " mskbl %2,%4,%2\n"
22 " or %1,%2,%2\n"
23 " stq_c %2,0(%3)\n"
24 " beq %2,2f\n"
25 ".subsection 2\n"
26 "2: br 1b\n"
27 ".previous"
28 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
29 : "r" ((long)m), "1" (val) : "memory");
30
31 return ret;
32}
33
34static inline unsigned long
35____xchg_u16(volatile short *m, unsigned long val)
36{
37 unsigned long ret, tmp, addr64;
38
39 __asm__ __volatile__(
40 " andnot %4,7,%3\n"
41 " inswl %1,%4,%1\n"
42 "1: ldq_l %2,0(%3)\n"
43 " extwl %2,%4,%0\n"
44 " mskwl %2,%4,%2\n"
45 " or %1,%2,%2\n"
46 " stq_c %2,0(%3)\n"
47 " beq %2,2f\n"
48 ".subsection 2\n"
49 "2: br 1b\n"
50 ".previous"
51 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
52 : "r" ((long)m), "1" (val) : "memory");
53
54 return ret;
55}
56
57static inline unsigned long
58____xchg_u32(volatile int *m, unsigned long val)
59{
60 unsigned long dummy;
61
62 __asm__ __volatile__(
63 "1: ldl_l %0,%4\n"
64 " bis $31,%3,%1\n"
65 " stl_c %1,%2\n"
66 " beq %1,2f\n"
67 ".subsection 2\n"
68 "2: br 1b\n"
69 ".previous"
70 : "=&r" (val), "=&r" (dummy), "=m" (*m)
71 : "rI" (val), "m" (*m) : "memory");
72
73 return val;
74}
75
76static inline unsigned long
77____xchg_u64(volatile long *m, unsigned long val)
78{
79 unsigned long dummy;
80
81 __asm__ __volatile__(
82 "1: ldq_l %0,%4\n"
83 " bis $31,%3,%1\n"
84 " stq_c %1,%2\n"
85 " beq %1,2f\n"
86 ".subsection 2\n"
87 "2: br 1b\n"
88 ".previous"
89 : "=&r" (val), "=&r" (dummy), "=m" (*m)
90 : "rI" (val), "m" (*m) : "memory");
91
92 return val;
93}
94
95/* This function doesn't exist, so you'll get a linker error
96 if something tries to do an invalid xchg(). */
97extern void __xchg_called_with_bad_pointer(void);
98
99static __always_inline unsigned long
100____xchg(volatile void *ptr, unsigned long x, int size)
101{
102 return
103 size == 1 ? ____xchg_u8(ptr, x) :
104 size == 2 ? ____xchg_u16(ptr, x) :
105 size == 4 ? ____xchg_u32(ptr, x) :
106 size == 8 ? ____xchg_u64(ptr, x) :
107 (__xchg_called_with_bad_pointer(), x);
108}
109
110/*
111 * Atomic compare and exchange. Compare OLD with MEM, if identical,
112 * store NEW in MEM. Return the initial value in MEM. Success is
113 * indicated by comparing RETURN with OLD.
114 */
115
116static inline unsigned long
117____cmpxchg_u8(volatile char *m, unsigned char old, unsigned char new)
118{
119 unsigned long prev, tmp, cmp, addr64;
120
121 __asm__ __volatile__(
122 " andnot %5,7,%4\n"
123 " insbl %1,%5,%1\n"
124 "1: ldq_l %2,0(%4)\n"
125 " extbl %2,%5,%0\n"
126 " cmpeq %0,%6,%3\n"
127 " beq %3,2f\n"
128 " mskbl %2,%5,%2\n"
129 " or %1,%2,%2\n"
130 " stq_c %2,0(%4)\n"
131 " beq %2,3f\n"
132 "2:\n"
133 ".subsection 2\n"
134 "3: br 1b\n"
135 ".previous"
136 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
137 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
138
139 return prev;
140}
141
142static inline unsigned long
143____cmpxchg_u16(volatile short *m, unsigned short old, unsigned short new)
144{
145 unsigned long prev, tmp, cmp, addr64;
146
147 __asm__ __volatile__(
148 " andnot %5,7,%4\n"
149 " inswl %1,%5,%1\n"
150 "1: ldq_l %2,0(%4)\n"
151 " extwl %2,%5,%0\n"
152 " cmpeq %0,%6,%3\n"
153 " beq %3,2f\n"
154 " mskwl %2,%5,%2\n"
155 " or %1,%2,%2\n"
156 " stq_c %2,0(%4)\n"
157 " beq %2,3f\n"
158 "2:\n"
159 ".subsection 2\n"
160 "3: br 1b\n"
161 ".previous"
162 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
163 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
164
165 return prev;
166}
167
168static inline unsigned long
169____cmpxchg_u32(volatile int *m, int old, int new)
170{
171 unsigned long prev, cmp;
172
173 __asm__ __volatile__(
174 "1: ldl_l %0,%5\n"
175 " cmpeq %0,%3,%1\n"
176 " beq %1,2f\n"
177 " mov %4,%1\n"
178 " stl_c %1,%2\n"
179 " beq %1,3f\n"
180 "2:\n"
181 ".subsection 2\n"
182 "3: br 1b\n"
183 ".previous"
184 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
185 : "r"((long) old), "r"(new), "m"(*m) : "memory");
186
187 return prev;
188}
189
190static inline unsigned long
191____cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
192{
193 unsigned long prev, cmp;
194
195 __asm__ __volatile__(
196 "1: ldq_l %0,%5\n"
197 " cmpeq %0,%3,%1\n"
198 " beq %1,2f\n"
199 " mov %4,%1\n"
200 " stq_c %1,%2\n"
201 " beq %1,3f\n"
202 "2:\n"
203 ".subsection 2\n"
204 "3: br 1b\n"
205 ".previous"
206 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
207 : "r"((long) old), "r"(new), "m"(*m) : "memory");
208
209 return prev;
210}
211
212/* This function doesn't exist, so you'll get a linker error
213 if something tries to do an invalid cmpxchg(). */
214extern void __cmpxchg_called_with_bad_pointer(void);
215
216static __always_inline unsigned long
217____cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
218 int size)
219{
220 return
221 size == 1 ? ____cmpxchg_u8(ptr, old, new) :
222 size == 2 ? ____cmpxchg_u16(ptr, old, new) :
223 size == 4 ? ____cmpxchg_u32(ptr, old, new) :
224 size == 8 ? ____cmpxchg_u64(ptr, old, new) :
225 (__cmpxchg_called_with_bad_pointer(), old);
226}
227
228#define xchg_local(ptr, x) \
229({ \
230 __typeof__(*(ptr)) _x_ = (x); \
231 (__typeof__(*(ptr))) ____xchg((ptr), (unsigned long)_x_, \
232 sizeof(*(ptr))); \
233})
234
235#define arch_cmpxchg_local(ptr, o, n) \
236({ \
237 __typeof__(*(ptr)) _o_ = (o); \
238 __typeof__(*(ptr)) _n_ = (n); \
239 (__typeof__(*(ptr))) ____cmpxchg((ptr), (unsigned long)_o_, \
240 (unsigned long)_n_, \
241 sizeof(*(ptr))); \
242})
243
244#define arch_cmpxchg64_local(ptr, o, n) \
245({ \
246 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
247 cmpxchg_local((ptr), (o), (n)); \
248})
249
250/*
251 * The leading and the trailing memory barriers guarantee that these
252 * operations are fully ordered.
253 */
254#define arch_xchg(ptr, x) \
255({ \
256 __typeof__(*(ptr)) __ret; \
257 __typeof__(*(ptr)) _x_ = (x); \
258 smp_mb(); \
259 __ret = (__typeof__(*(ptr))) \
260 ____xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
261 smp_mb(); \
262 __ret; \
263})
264
265#define arch_cmpxchg(ptr, o, n) \
266({ \
267 __typeof__(*(ptr)) __ret; \
268 __typeof__(*(ptr)) _o_ = (o); \
269 __typeof__(*(ptr)) _n_ = (n); \
270 smp_mb(); \
271 __ret = (__typeof__(*(ptr))) ____cmpxchg((ptr), \
272 (unsigned long)_o_, (unsigned long)_n_, sizeof(*(ptr)));\
273 smp_mb(); \
274 __ret; \
275})
276
277#define arch_cmpxchg64(ptr, o, n) \
278({ \
279 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
280 arch_cmpxchg((ptr), (o), (n)); \
281})
282
283#endif /* _ALPHA_CMPXCHG_H */