projects/linux·blob master · refs

542 lines · 16.0 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2013 ARM Ltd.
4 */
5#ifndef __ASM_PERCPU_H
6#define __ASM_PERCPU_H
7
8#include <linux/bits.h>
9#include <linux/preempt.h>
10#include <linux/stringify.h>
11
12#include <asm/alternative.h>
13#include <asm/cmpxchg.h>
14#include <asm/gpr-num.h>
15#include <asm/stack_pointer.h>
16#include <asm/sysreg.h>
17
18static inline void set_my_cpu_offset(unsigned long off)
19{
20 asm volatile(ALTERNATIVE("msr tpidr_el1, %0",
21 "msr tpidr_el2, %0",
22 ARM64_HAS_VIRT_HOST_EXTN)
23 :: "r" (off) : "memory");
24}
25
26static inline unsigned long __hyp_my_cpu_offset(void)
27{
28 /*
29 * Non-VHE hyp code runs with preemption disabled. No need to hazard
30 * the register access against barrier() as in __kern_my_cpu_offset.
31 */
32 return read_sysreg(tpidr_el2);
33}
34
35#define __KERN_ASM_CPU_OFFSET(dst) \
36 ALTERNATIVE("mrs " dst ", tpidr_el1", \
37 "mrs " dst ", tpidr_el2", \
38 ARM64_HAS_VIRT_HOST_EXTN)
39
40static inline unsigned long __kern_my_cpu_offset(void)
41{
42 unsigned long off;
43
44 /*
45 * We want to allow caching the value, so avoid using volatile and
46 * instead use a fake stack read to hazard against barrier().
47 */
48 asm(
49 __KERN_ASM_CPU_OFFSET("%0")
50 : "=r" (off)
51 : "Q" (*(const unsigned long *)current_stack_pointer)
52 );
53
54 return off;
55}
56
57#define PCPU_GPR_PCP_SHIFT 0
58#define PCPU_GPR_PCP GENMASK(4, 0)
59#define PCPU_GPR_OFF_SHIFT 5
60#define PCPU_GPR_OFF GENMASK(9, 5)
61#define PCPU_GPR_ADDR_SHIFT 10
62#define PCPU_GPR_ADDR GENMASK(14, 10)
63
64#define __VAL_PCPU_GPRS(pcp, off, addr) \
65 "(" \
66 "(" __GPR_NUM(pcp) " << " __stringify(PCPU_GPR_PCP_SHIFT) ") | " \
67 "(" __GPR_NUM(off) " << " __stringify(PCPU_GPR_OFF_SHIFT) ") | " \
68 "(" __GPR_NUM(addr) " << " __stringify(PCPU_GPR_ADDR_SHIFT) ")" \
69 ")"
70
71#define __ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr) \
72 ".if (" \
73 "(" __GPR_NUM(pcp) " == " __GPR_NUM(off) ") || " \
74 "(" __GPR_NUM(pcp) " == " __GPR_NUM(addr) ") || " \
75 "(" __GPR_NUM(off) " == " __GPR_NUM(addr) ")" \
76 " )\n" \
77 ".error \"PCPU GPRS overlap: {" pcp "," off "," addr "}\"\n" \
78 ".endif\n"
79
80#define ____PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \
81 "// ____PCPU_GPRS_BEGIN(" gprs ", " pcp ", " off ", " addr")\n" \
82 __DEFINE_ASM_GPR_NUMS \
83 __DEFINE_ASM_GPR_ALIASES \
84 __ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr) \
85 " mov w" off ", #" __VAL_PCPU_GPRS(pcp, off, addr) "\n" \
86 " strh w" off ", " gprs "\n" \
87 __KERN_ASM_CPU_OFFSET(off) "\n"
88
89/*
90 * Begin a PCPU GPR critical section which requires <addr> (and <off>).
91 *
92 * At the start of the critical section, and upon any (preemptible) exception
93 * until __PCPU_GPRS_END():
94 * - <off> will be set to the current CPU's percpu offset.
95 * - <addr> will be set to <pcp> + <off>.
96 *
97 * The <pcp>, <off>, and <addr> registers must be distinct GPRs.
98 *
99 * <gprs> must be '&current_thread_info()->pcpu_gprs', as a memory operand
100 * which can be written both at the start and end of the critical section
101 * (e.g. using "=Qo" constraints).
102 */
103#define __PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \
104 ____PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \
105 " add " addr ", " pcp ", " off "\n"
106
107/*
108 * Begin a PCPU GPR critical section which only requires <off> and does not
109 * require <addr>.
110 *
111 * This is only for operations that can use register-offset addressing,
112 * e.g. STR <Xt>, [<Xn>, <Xm>].
113 *
114 * All other details are the same as __PCPU_GPRS_BEGIN().
115 */
116#define __PCPU_GPRS_BEGIN_OFFSET(gprs, pcp, off) \
117 ____PCPU_GPRS_BEGIN(gprs, pcp, off, "xzr")
118
119/*
120 * End a PCPU GPR critical section.
121 */
122#define __PCPU_GPRS_END(gprs) \
123 " strh wzr, " gprs "\n"
124
125#ifdef __KVM_NVHE_HYPERVISOR__
126#define __my_cpu_offset __hyp_my_cpu_offset()
127#else
128#define __my_cpu_offset __kern_my_cpu_offset()
129#endif
130
131#define PERCPU_RW_OPS(w, sfx, sz) \
132static inline unsigned long __percpu_read_##sz(void __percpu *pcp) \
133{ \
134 u16 *gprs = &current_thread_info()->pcpu_gprs; \
135 unsigned long off; \
136 unsigned long val; \
137 \
138 asm volatile( \
139 __PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]") \
140 " ldr" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n" \
141 __PCPU_GPRS_END("%[gprs]") \
142 : [gprs] "=Qo" (*gprs), \
143 [off] "=&r" (off), \
144 [val] "=&r" (val) \
145 : [pcp] "r" (pcp) \
146 : "memory" \
147 ); \
148 \
149 return val; \
150} \
151 \
152static inline void __percpu_write_##sz(void __percpu *pcp, unsigned long val) \
153{ \
154 u16 *gprs = &current_thread_info()->pcpu_gprs; \
155 unsigned long off; \
156 \
157 asm volatile( \
158 __PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]") \
159 " str" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n" \
160 __PCPU_GPRS_END("%[gprs]") \
161 : [gprs] "=Qo" (*gprs), \
162 [off] "=&r" (off) \
163 : [pcp] "r" (pcp), \
164 [val] "r" ((u##sz)val) \
165 : "memory" \
166 ); \
167}
168
169#define __PERCPU_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
170static inline void \
171__percpu_##name##_case_##sz(void __percpu *pcp, unsigned long val) \
172{ \
173 u16 *gprs = &current_thread_info()->pcpu_gprs; \
174 unsigned long addr; \
175 unsigned long off; \
176 unsigned int loop; \
177 u##sz tmp; \
178 \
179 asm volatile ( \
180 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
181 ARM64_LSE_ATOMIC_INSN( \
182 /* LL/SC */ \
183 "1: ldxr" #sfx "\t%" #w "[tmp], [%[addr]]\n" \
184 #op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
185 " stxr" #sfx "\t%w[loop], %" #w "[tmp], [%[addr]]\n" \
186 " cbnz %w[loop], 1b", \
187 /* LSE atomics */ \
188 #op_lse #sfx "\t%" #w "[val], %" #w "[tmp], [%[addr]]\n"\
189 __nops(3)) \
190 __PCPU_GPRS_END("%[gprs]") \
191 : [gprs] "=Qo" (*gprs), \
192 [addr] "=&r" (addr), \
193 [off] "=&r" (off), \
194 [loop] "=&r" (loop), \
195 [tmp] "=&r" (tmp) \
196 : [pcp] "r" (pcp), \
197 [val] "r" ((u##sz)(val)) \
198 : "memory" \
199 ); \
200}
201
202#define __PERCPU_RET_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
203static inline u##sz \
204__percpu_##name##_return_case_##sz(void __percpu *pcp, unsigned long val) \
205{ \
206 u16 *gprs = &current_thread_info()->pcpu_gprs; \
207 unsigned long addr; \
208 unsigned long off; \
209 unsigned int loop; \
210 u##sz ret; \
211 \
212 asm volatile ( \
213 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
214 ARM64_LSE_ATOMIC_INSN( \
215 /* LL/SC */ \
216 "1: ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n" \
217 #op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
218 " stxr" #sfx "\t%w[loop], %" #w "[ret], [%[addr]]\n" \
219 " cbnz %w[loop], 1b", \
220 /* LSE atomics */ \
221 #op_lse #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n"\
222 #op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
223 __nops(2)) \
224 __PCPU_GPRS_END("%[gprs]") \
225 : [gprs] "=Qo" (*gprs), \
226 [addr] "=&r" (addr), \
227 [off] "=&r" (off), \
228 [loop] "=&r" (loop), \
229 [ret] "=&r" (ret) \
230 : [pcp] "r" (pcp), \
231 [val] "r" ((u##sz)(val)) \
232 : "memory" \
233 ); \
234 \
235 return ret; \
236}
237
238#define PERCPU_OP(name, op_llsc, op_lse) \
239 __PERCPU_OP_CASE(w, b, name, 8, op_llsc, op_lse) \
240 __PERCPU_OP_CASE(w, h, name, 16, op_llsc, op_lse) \
241 __PERCPU_OP_CASE(w, , name, 32, op_llsc, op_lse) \
242 __PERCPU_OP_CASE( , , name, 64, op_llsc, op_lse)
243
244#define PERCPU_RET_OP(name, op_llsc, op_lse) \
245 __PERCPU_RET_OP_CASE(w, b, name, 8, op_llsc, op_lse) \
246 __PERCPU_RET_OP_CASE(w, h, name, 16, op_llsc, op_lse) \
247 __PERCPU_RET_OP_CASE(w, , name, 32, op_llsc, op_lse) \
248 __PERCPU_RET_OP_CASE( , , name, 64, op_llsc, op_lse)
249
250PERCPU_RW_OPS(w, b, 8)
251PERCPU_RW_OPS(w, h, 16)
252PERCPU_RW_OPS(w, , 32)
253PERCPU_RW_OPS( , , 64)
254
255/*
256 * Use value-returning atomics for CPU-local ops as they are more likely
257 * to execute "near" to the CPU (e.g. in L1$).
258 *
259 * https://lore.kernel.org/r/e7d539ed-ced0-4b96-8ecd-048a5b803b85@paulmck-laptop
260 */
261PERCPU_OP(add, add, ldadd)
262PERCPU_OP(andnot, bic, ldclr)
263PERCPU_OP(or, orr, ldset)
264PERCPU_RET_OP(add, add, ldadd)
265
266#undef PERCPU_RW_OPS
267#undef __PERCPU_OP_CASE
268#undef __PERCPU_RET_OP_CASE
269#undef PERCPU_OP
270#undef PERCPU_RET_OP
271
272#define PERCPU_XCHG_OP(w, sfx, sz) \
273static inline unsigned long \
274__percpu_xchg_case_##sz(void __percpu *pcp, u##sz val) \
275{ \
276 u16 *gprs = &current_thread_info()->pcpu_gprs; \
277 unsigned long addr; \
278 unsigned long off; \
279 unsigned int loop; \
280 unsigned long ret; \
281 \
282 asm volatile ( \
283 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
284 ARM64_LSE_ATOMIC_INSN( \
285 /* LL/SC */ \
286 " prfm pstl1strm, [%[addr]]\n" \
287 "1: ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n" \
288 " stxr" #sfx "\t%w[loop], %" #w "[val], [%[addr]]\n" \
289 " cbnz %w[loop], 1b\n" \
290 , \
291 /* LSE atomics */ \
292 " swp" #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n" \
293 __nops(3) \
294 ) \
295 __PCPU_GPRS_END("%[gprs]") \
296 : [gprs] "=Qo" (*gprs), \
297 [addr] "=&r" (addr), \
298 [off] "=&r" (off), \
299 [loop] "=&r" (loop), \
300 [ret] "=&r" (ret) \
301 : [pcp] "r" (pcp), \
302 [val] "r" (val) \
303 : "memory" \
304 ); \
305 \
306 return ret; \
307}
308
309#define PERCPU_CMPXCHG_OP(w, sfx, sz) \
310static inline unsigned long \
311__percpu_cmpxchg_case_##sz(void __percpu *pcp, \
312 u##sz old, \
313 u##sz new) \
314{ \
315 /* \
316 * Sub-word sizes require zero extension so that EOR+CBNZ don't \
317 * consume non-zero upper bits of the register containing "old".\
318 */ \
319 xwreg_t(w) cmpval = xwreg_zero_extend(old, w, sz); \
320 u16 *gprs = &current_thread_info()->pcpu_gprs; \
321 unsigned long addr; \
322 unsigned long off; \
323 unsigned long tmp; \
324 unsigned long oldval; \
325 \
326 asm volatile ( \
327 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
328 ARM64_LSE_ATOMIC_INSN( \
329 /* LL/SC */ \
330 " prfm pstl1strm, [%[addr]]\n" \
331 "1: ldxr" #sfx "\t%" #w "[oldval], [%[addr]]\n" \
332 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
333 " cbnz %" #w "[tmp], 2f\n" \
334 " stxr" #sfx "\t%w[tmp], %" #w "[new], [%[addr]]\n" \
335 " cbnz %w[tmp], 1b\n" \
336 "2:\n" \
337 , \
338 /* LSE atomics */ \
339 " mov %" #w "[oldval], %" #w "[old]\n" \
340 " cas" #sfx "\t%" #w "[oldval], %" #w "[new], [%[addr]]\n"\
341 __nops(4) \
342 ) \
343 __PCPU_GPRS_END("%[gprs]") \
344 : [gprs] "=Qo" (*gprs), \
345 [addr] "=&r" (addr), \
346 [off] "=&r" (off), \
347 [tmp] "=&r" (tmp), \
348 [oldval] "=&r" (oldval) \
349 : [pcp] "r" (pcp), \
350 [old] "r" (cmpval), \
351 [new] "r" (new) \
352 : "memory" \
353 ); \
354 \
355 return oldval; \
356}
357
358PERCPU_XCHG_OP(w, b, 8)
359PERCPU_XCHG_OP(w, h, 16)
360PERCPU_XCHG_OP(w, , 32)
361PERCPU_XCHG_OP(x, , 64)
362
363PERCPU_CMPXCHG_OP(w, b, 8)
364PERCPU_CMPXCHG_OP(w, h, 16)
365PERCPU_CMPXCHG_OP(w, , 32)
366PERCPU_CMPXCHG_OP(x, , 64)
367
368#undef PERCPU_XCHG_OP
369#undef PERCPU_CMPXCHG_OP
370
371#define _pcp_wrap(op, pcp, ...) \
372({ \
373 op(&(pcp), __VA_ARGS__); \
374})
375
376#define _pcp_wrap_return(op, pcp, args...) \
377({ \
378 (typeof(pcp))op(&(pcp), ##args); \
379})
380
381#define _pcp_wrap_xchg(op, pcp, val) \
382({ \
383 (typeof(pcp))op(&(pcp), (unsigned long)(val)); \
384})
385
386#define _pcp_wrap_cmpxchg(op, pcp, old, new) \
387({ \
388 (typeof(pcp))op(&(pcp), (unsigned long)(old), \
389 (unsigned long)(new)); \
390})
391
392#define this_cpu_read_1(pcp) \
393 _pcp_wrap_return(__percpu_read_8, pcp)
394#define this_cpu_read_2(pcp) \
395 _pcp_wrap_return(__percpu_read_16, pcp)
396#define this_cpu_read_4(pcp) \
397 _pcp_wrap_return(__percpu_read_32, pcp)
398#define this_cpu_read_8(pcp) \
399 _pcp_wrap_return(__percpu_read_64, pcp)
400
401#define this_cpu_write_1(pcp, val) \
402 _pcp_wrap(__percpu_write_8, pcp, (unsigned long)(val))
403#define this_cpu_write_2(pcp, val) \
404 _pcp_wrap(__percpu_write_16, pcp, (unsigned long)(val))
405#define this_cpu_write_4(pcp, val) \
406 _pcp_wrap(__percpu_write_32, pcp, (unsigned long)(val))
407#define this_cpu_write_8(pcp, val) \
408 _pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val))
409
410#define this_cpu_add_1(pcp, val) \
411 _pcp_wrap(__percpu_add_case_8, pcp, val)
412#define this_cpu_add_2(pcp, val) \
413 _pcp_wrap(__percpu_add_case_16, pcp, val)
414#define this_cpu_add_4(pcp, val) \
415 _pcp_wrap(__percpu_add_case_32, pcp, val)
416#define this_cpu_add_8(pcp, val) \
417 _pcp_wrap(__percpu_add_case_64, pcp, val)
418
419#define this_cpu_add_return_1(pcp, val) \
420 _pcp_wrap_return(__percpu_add_return_case_8, pcp, val)
421#define this_cpu_add_return_2(pcp, val) \
422 _pcp_wrap_return(__percpu_add_return_case_16, pcp, val)
423#define this_cpu_add_return_4(pcp, val) \
424 _pcp_wrap_return(__percpu_add_return_case_32, pcp, val)
425#define this_cpu_add_return_8(pcp, val) \
426 _pcp_wrap_return(__percpu_add_return_case_64, pcp, val)
427
428#define this_cpu_and_1(pcp, val) \
429 _pcp_wrap(__percpu_andnot_case_8, pcp, ~(u8)(val))
430#define this_cpu_and_2(pcp, val) \
431 _pcp_wrap(__percpu_andnot_case_16, pcp, ~(u16)(val))
432#define this_cpu_and_4(pcp, val) \
433 _pcp_wrap(__percpu_andnot_case_32, pcp, ~(u32)(val))
434#define this_cpu_and_8(pcp, val) \
435 _pcp_wrap(__percpu_andnot_case_64, pcp, ~(u64)(val))
436
437#define this_cpu_or_1(pcp, val) \
438 _pcp_wrap(__percpu_or_case_8, pcp, val)
439#define this_cpu_or_2(pcp, val) \
440 _pcp_wrap(__percpu_or_case_16, pcp, val)
441#define this_cpu_or_4(pcp, val) \
442 _pcp_wrap(__percpu_or_case_32, pcp, val)
443#define this_cpu_or_8(pcp, val) \
444 _pcp_wrap(__percpu_or_case_64, pcp, val)
445
446#define this_cpu_xchg_1(pcp, val) \
447 _pcp_wrap_xchg(__percpu_xchg_case_8, pcp, val)
448#define this_cpu_xchg_2(pcp, val) \
449 _pcp_wrap_xchg(__percpu_xchg_case_16, pcp, val)
450#define this_cpu_xchg_4(pcp, val) \
451 _pcp_wrap_xchg(__percpu_xchg_case_32, pcp, val)
452#define this_cpu_xchg_8(pcp, val) \
453 _pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val)
454
455#define this_cpu_cmpxchg_1(pcp, o, n) \
456 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_8, pcp, o, n)
457#define this_cpu_cmpxchg_2(pcp, o, n) \
458 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_16, pcp, o, n)
459#define this_cpu_cmpxchg_4(pcp, o, n) \
460 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_32, pcp, o, n)
461#define this_cpu_cmpxchg_8(pcp, o, n) \
462 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_64, pcp, o, n)
463
464#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
465
466static inline u128
467__percpu_cmpxchg_128(void __percpu *pcp, u128 old, u128 new)
468{
469 u16 *gprs = &current_thread_info()->pcpu_gprs;
470 unsigned long addr;
471 unsigned long off;
472 union __u128_halves r, o = { .full = (old) },
473 n = { .full = (new) };
474 register unsigned long ol asm ("x0") = o.low;
475 register unsigned long oh asm ("x1") = o.high;
476 register unsigned long nl asm ("x2") = n.low;
477 register unsigned long nh asm ("x3") = n.high;
478 unsigned long rl, rh;
479 unsigned long tmp;
480
481 asm volatile (
482 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")
483 ARM64_LSE_ATOMIC_INSN(
484 /* LL/SC */
485 " prfm pstl1strm, [%[addr]]\n"
486 "1: ldxp %[rl], %[rh], [%[addr]]\n"
487 " cmp %[rl], %[ol]\n"
488 " ccmp %[rh], %[oh], 0, eq\n"
489 " b.ne 2f\n"
490 " stxp %w[tmp], %[nl], %[nh], [%[addr]]\n"
491 " cbnz %w[tmp], 1b\n"
492 "2:\n"
493 ,
494 /* LSE atomics */
495 " casp %[ol], %[oh], %[nl], %[nh], [%[addr]]\n"
496 " mov %[rl], %[ol]\n"
497 " mov %[rh], %[oh]\n"
498 __nops(4)
499 )
500 __PCPU_GPRS_END("%[gprs]")
501 : [gprs] "=Qo" (*gprs),
502 [addr] "=&r" (addr),
503 [off] "=&r" (off),
504 [tmp] "=&r" (tmp),
505 [ol] "+&r" (ol),
506 [oh] "+&r" (oh),
507 [rl] "=&r" (rl),
508 [rh] "=&r" (rh)
509 : [pcp] "r" (pcp),
510 [nl] "r" (nl),
511 [nh] "r" (nh)
512 : "memory", "cc"
513 );
514
515 r.low = rl;
516 r.high = rh;
517
518 return r.full;
519}
520
521#define this_cpu_cmpxchg128(pcp, o, n) \
522 _pcp_wrap_return(__percpu_cmpxchg_128, pcp, o, n)
523
524#ifdef __KVM_NVHE_HYPERVISOR__
525extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
526#define __per_cpu_offset
527#define per_cpu_offset(cpu) __hyp_per_cpu_offset((cpu))
528#endif
529
530#include <asm-generic/percpu.h>
531
532/* Redefine macros for nVHE hyp under DEBUG_PREEMPT to avoid its dependencies. */
533#if defined(__KVM_NVHE_HYPERVISOR__) && defined(CONFIG_DEBUG_PREEMPT)
534#undef this_cpu_ptr
535#define this_cpu_ptr raw_cpu_ptr
536#undef __this_cpu_read
537#define __this_cpu_read raw_cpu_read
538#undef __this_cpu_write
539#define __this_cpu_write raw_cpu_write
540#endif
541
542#endif /* __ASM_PERCPU_H */