| 1 |
|
| 2 | #ifndef __ARCH_S390_PERCPU__
|
| 3 | #define __ARCH_S390_PERCPU__
|
| 4 |
|
| 5 | #include <linux/preempt.h>
|
| 6 | #include <asm/cmpxchg.h>
|
| 7 | #include <asm/march.h>
|
| 8 | #include <asm/asm.h>
|
| 9 |
|
| 10 | |
| 11 | * s390 uses its own implementation for per cpu data, the offset of
|
| 12 | * the cpu local data area is cached in the cpu's lowcore memory.
|
| 13 |
|
| 14 | #define __my_cpu_offset get_lowcore()->percpu_offset
|
| 15 |
|
| 16 | #define arch_raw_cpu_ptr(_ptr) \
|
| 17 | ({ \
|
| 18 | unsigned long tcp_ptr__; \
|
| 19 | \
|
| 20 | tcp_ptr__ = (__force unsigned long)(_ptr); \
|
| 21 | asm_inline volatile( \
|
| 22 | ALTERNATIVE("ag %[__ptr__],%[offzero](%%r0)\n", \
|
| 23 | "ag %[__ptr__],%[offalt](%%r0)\n", \
|
| 24 | ALT_FEATURE(MFEATURE_LOWCORE)) \
|
| 25 | : [__ptr__] "+d" (tcp_ptr__) \
|
| 26 | : [offzero] "i" (LC_PERCPU_OFFSET), \
|
| 27 | [offalt] "i" (LC_PERCPU_OFFSET + LOWCORE_ALT_ADDRESS), \
|
| 28 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 29 | : "cc"); \
|
| 30 | (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \
|
| 31 | })
|
| 32 |
|
| 33 | |
| 34 | * We use a compare-and-swap loop since that uses less cpu cycles than
|
| 35 | * disabling and enabling interrupts like the generic variant would do.
|
| 36 |
|
| 37 | #define arch_this_cpu_to_op_simple(pcp, val, op) \
|
| 38 | ({ \
|
| 39 | typedef typeof(pcp) pcp_op_T__; \
|
| 40 | pcp_op_T__ old__, new__, prev__; \
|
| 41 | pcp_op_T__ *ptr__; \
|
| 42 | preempt_disable_notrace(); \
|
| 43 | ptr__ = raw_cpu_ptr(&(pcp)); \
|
| 44 | prev__ = READ_ONCE(*ptr__); \
|
| 45 | do { \
|
| 46 | old__ = prev__; \
|
| 47 | new__ = old__ op (val); \
|
| 48 | prev__ = cmpxchg(ptr__, old__, new__); \
|
| 49 | } while (prev__ != old__); \
|
| 50 | preempt_enable_notrace(); \
|
| 51 | new__; \
|
| 52 | })
|
| 53 |
|
| 54 | #define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 55 | #define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 56 | #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 57 | #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 58 | #define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
| 59 | #define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
| 60 | #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
| 61 | #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
| 62 |
|
| 63 | |
| 64 | * Macros to be used for percpu code section based on atomic instructions.
|
| 65 | *
|
| 66 | * Avoid the need to use preempt_disable() / preempt_enable() pairs and the
|
| 67 | * conditional preempt_schedule_notrace() function calls which come with
|
| 68 | * this. The idea is that this_cpu operations based on atomic instructions are
|
| 69 | * guarded with mviy instructions:
|
| 70 | *
|
| 71 | * - The first mviy instruction writes the register number of the percpu address
|
| 72 | * variable and the even register number of a register pair (which encodes two
|
| 73 | * registers: the even register for the percpu offset and the odd register for
|
| 74 | * the percpu pointer) to lowcore. This also indicates that a percpu code
|
| 75 | * section is executed.
|
| 76 | *
|
| 77 | * - The mviy instruction is followed by the lg instruction which loads the
|
| 78 | * percpu offset into the even register of the pair and the la instruction
|
| 79 | * which adds the percpu offset and the percpu address into the odd register
|
| 80 | * of the pair (the percpu pointer register).
|
| 81 | *
|
| 82 | * - Afterwards the atomic percpu operation follows.
|
| 83 | *
|
| 84 | * - Then a second mviy instruction writes a zero to lowcore, which indicates
|
| 85 | * the end of the percpu code section.
|
| 86 | *
|
| 87 | * - In case of an interrupt/exception/nmi the encoded register numbers which
|
| 88 | * were written to lowcore are copied to the exception frame (pt_regs), and a
|
| 89 | * zero is written to lowcore.
|
| 90 | *
|
| 91 | * - On return to the previous context it is checked if a percpu code section
|
| 92 | * was executed (saved register value not zero), and if the process was
|
| 93 | * migrated to a different cpu. The content of the percpu offset register
|
| 94 | * (even register of the pair) is reloaded with the current cpu's percpu
|
| 95 | * offset and the percpu pointer register (odd register of the pair) is
|
| 96 | * recalculated so it points to the percpu variable of the new cpu.
|
| 97 | *
|
| 98 | * Inline assemblies making use of this typically have a code sequence like:
|
| 99 | *
|
| 100 | * __PCPU_BEGIN(...) <- start of percpu code section
|
| 101 | * atomic_op <- atomic op
|
| 102 | * __PCPU_END(...) <- end of percpu code section
|
| 103 |
|
| 104 |
|
| 105 | #define LC_ALT_ADDR __stringify(LOWCORE_ALT_ADDRESS)
|
| 106 |
|
| 107 | #define DEFINE_GR_NUM \
|
| 108 | ".macro _GR_NUM opd, gr\n" \
|
| 109 | ".set \\opd,255\n" \
|
| 110 | ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \
|
| 111 | " .ifc \\gr,%%r\\rs\n" \
|
| 112 | " \\opd = \\rs\n" \
|
| 113 | " .endif\n" \
|
| 114 | ".endr\n" \
|
| 115 | ".if \\opd == 255\n" \
|
| 116 | " .error \"Illegal register number\"\n" \
|
| 117 | ".endif\n" \
|
| 118 | ".endm\n"
|
| 119 |
|
| 120 | #define UNDEF_GR_NUM \
|
| 121 | ".purgem _GR_NUM\n"
|
| 122 |
|
| 123 | #define PCPU_REG_PCP_SHIFT 0
|
| 124 | #define PCPU_REG_PCP GENMASK(3, 0)
|
| 125 | #define PCPU_REG_OFF_SHIFT 4
|
| 126 | #define PCPU_REG_OFF GENMASK(7, 4)
|
| 127 |
|
| 128 | #define __PCPU_MVIY(lcreg, imm) \
|
| 129 | ALTERNATIVE(" mviy " lcreg "(%%r0)," imm "\n", \
|
| 130 | " mviy " lcreg "+" LC_ALT_ADDR "(%%r0)," imm "\n", \
|
| 131 | ALT_FEATURE(MFEATURE_LOWCORE))
|
| 132 |
|
| 133 | #define __PCPU_LG(regoff, lcoff) \
|
| 134 | ALTERNATIVE(" lg " regoff ", " lcoff "(%%r0)\n", \
|
| 135 | " lg " regoff ", " lcoff "+" LC_ALT_ADDR "(%%r0)\n", \
|
| 136 | ALT_FEATURE(MFEATURE_LOWCORE))
|
| 137 |
|
| 138 | #define __PCPU_LA(regptr, regoff, regpcp) \
|
| 139 | " la " regptr ",0(" regoff "," regpcp ")\n"
|
| 140 |
|
| 141 | #define __PCPU_CALC_REGVAL(regpcp, regoff) \
|
| 142 | "(" regpcp " << " __stringify(PCPU_REG_PCP_SHIFT) ") |" \
|
| 143 | "(" regoff " << " __stringify(PCPU_REG_OFF_SHIFT) ")"
|
| 144 |
|
| 145 | #define __PCPU_CHECK_REGS(regpcp, regoff, regptr) \
|
| 146 | ".if " regoff " & 1\n" \
|
| 147 | " .error \"Percpu offset register must be even\"\n" \
|
| 148 | ".endif\n" \
|
| 149 | ".if (" regpcp " == 0) || (" regoff " == 0)\n" \
|
| 150 | " .error \"Percpu address and offset register must be non-zero\"\n"\
|
| 151 | ".endif\n" \
|
| 152 | ".if " regptr " != " regoff " + 1\n" \
|
| 153 | " .error \"Percpu pointer and offset register must be a pair\"\n" \
|
| 154 | ".endif\n" \
|
| 155 | ".if (" regpcp " == " regoff ") || (" regpcp " == " regptr ")\n" \
|
| 156 | " .error \"Percpu registers must be distinct\"\n" \
|
| 157 | ".endif\n"
|
| 158 |
|
| 159 | #define __PCPU_BEGIN(lcreg, lcoff, regpcp, regoff, regptr) \
|
| 160 | DEFINE_GR_NUM \
|
| 161 | "_GR_NUM .Lregpcp, " regpcp "\n" \
|
| 162 | "_GR_NUM .Lregoff, " regoff "\n" \
|
| 163 | "_GR_NUM .Lregptr, " regptr "\n" \
|
| 164 | UNDEF_GR_NUM \
|
| 165 | __PCPU_CHECK_REGS(".Lregpcp", ".Lregoff", ".Lregptr") \
|
| 166 | ".set .Lregval, " __PCPU_CALC_REGVAL(".Lregpcp", ".Lregoff") "\n" \
|
| 167 | __PCPU_MVIY(lcreg, ".Lregval") \
|
| 168 | __PCPU_LG(regoff, lcoff) \
|
| 169 | __PCPU_LA(regptr, regoff, regpcp)
|
| 170 |
|
| 171 | #define __PCPU_END(lcreg) \
|
| 172 | __PCPU_MVIY(lcreg, "0")
|
| 173 |
|
| 174 | #ifndef MARCH_HAS_Z196_FEATURES
|
| 175 |
|
| 176 | #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 177 | #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 178 | #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 179 | #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
| 180 | #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
| 181 | #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
| 182 | #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
| 183 | #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
| 184 |
|
| 185 | #else
|
| 186 |
|
| 187 | #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \
|
| 188 | do { \
|
| 189 | typedef typeof(pcp) pcp_op_T__; \
|
| 190 | union register_pair rp__; \
|
| 191 | pcp_op_T__ val__ = (val); \
|
| 192 | pcp_op_T__ *ptr__; \
|
| 193 | \
|
| 194 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 195 | if (__builtin_constant_p(val__) && \
|
| 196 | ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \
|
| 197 | asm_inline volatile( \
|
| 198 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 199 | "%[pair__]","%N[pair__]") \
|
| 200 | op2 " 0(%N[pair__]),%[val__]\n" \
|
| 201 | __PCPU_END("%[lcreg]") \
|
| 202 | : [pair__] "=&a" (rp__.pair), "+m" (*ptr__), \
|
| 203 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 204 | : [val__] "i" ((szcast)val__), [ptr__] "a" (ptr__), \
|
| 205 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 206 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 207 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 208 | : "cc"); \
|
| 209 | } else { \
|
| 210 | asm_inline volatile( \
|
| 211 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 212 | "%[pair__]","%N[pair__]") \
|
| 213 | op1 " %[val__],%[val__],0(%N[pair__])\n" \
|
| 214 | __PCPU_END("%[lcreg]") \
|
| 215 | : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \
|
| 216 | "+m" (*ptr__), \
|
| 217 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 218 | : [ptr__] "a" (ptr__), \
|
| 219 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 220 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 221 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 222 | : "cc"); \
|
| 223 | } \
|
| 224 | } while (0)
|
| 225 |
|
| 226 | #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
|
| 227 | #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
|
| 228 |
|
| 229 | #define arch_this_cpu_add_return(pcp, val, op) \
|
| 230 | ({ \
|
| 231 | typedef typeof(pcp) pcp_op_T__; \
|
| 232 | union register_pair rp__; \
|
| 233 | pcp_op_T__ val__ = (val); \
|
| 234 | pcp_op_T__ old__, *ptr__; \
|
| 235 | \
|
| 236 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 237 | asm_inline volatile( \
|
| 238 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 239 | "%[pair__]","%N[pair__]") \
|
| 240 | op " %[old__],%[val__],0(%N[pair__])\n" \
|
| 241 | __PCPU_END("%[lcreg]") \
|
| 242 | : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \
|
| 243 | "+m" (*ptr__), \
|
| 244 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 245 | : [val__] "d" (val__), [ptr__] "a" (ptr__), \
|
| 246 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 247 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 248 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 249 | : "cc"); \
|
| 250 | old__ + val__; \
|
| 251 | })
|
| 252 |
|
| 253 | #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
|
| 254 | #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
|
| 255 |
|
| 256 | #define arch_this_cpu_to_op(pcp, val, op) \
|
| 257 | do { \
|
| 258 | typedef typeof(pcp) pcp_op_T__; \
|
| 259 | union register_pair rp__; \
|
| 260 | pcp_op_T__ val__ = (val); \
|
| 261 | pcp_op_T__ *ptr__; \
|
| 262 | \
|
| 263 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 264 | asm_inline volatile( \
|
| 265 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 266 | "%[pair__]","%N[pair__]") \
|
| 267 | op " %[val__],%[val__],0(%N[pair__])\n" \
|
| 268 | __PCPU_END("%[lcreg]") \
|
| 269 | : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \
|
| 270 | "+m" (*ptr__), \
|
| 271 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 272 | : [ptr__] "a" (ptr__), \
|
| 273 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 274 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 275 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 276 | : "cc"); \
|
| 277 | } while (0)
|
| 278 |
|
| 279 | #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan")
|
| 280 | #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang")
|
| 281 | #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao")
|
| 282 | #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog")
|
| 283 |
|
| 284 | #endif
|
| 285 |
|
| 286 | #define arch_this_cpu_read(pcp, op) \
|
| 287 | ({ \
|
| 288 | typedef typeof(pcp) pcp_op_T__; \
|
| 289 | union register_pair rp__; \
|
| 290 | unsigned long res__; \
|
| 291 | pcp_op_T__ *ptr__; \
|
| 292 | \
|
| 293 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 294 | asm_inline volatile( \
|
| 295 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 296 | "%[pair__]","%N[pair__]") \
|
| 297 | op " %[res__],0(%N[pair__])\n" \
|
| 298 | __PCPU_END("%[lcreg]") \
|
| 299 | : [res__] "=&d" (res__), [pair__] "=&a" (rp__.pair), \
|
| 300 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 301 | : [ptr__] "a" (ptr__), \
|
| 302 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 303 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 304 | "m" (*ptr__), \
|
| 305 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 306 | : "cc"); \
|
| 307 | (pcp_op_T__)res__; \
|
| 308 | })
|
| 309 |
|
| 310 | #define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc")
|
| 311 | #define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh")
|
| 312 | #define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf")
|
| 313 | #define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg")
|
| 314 |
|
| 315 | #define arch_this_cpu_write(pcp, val, op) \
|
| 316 | do { \
|
| 317 | typedef typeof(pcp) pcp_op_T__; \
|
| 318 | union register_pair rp__; \
|
| 319 | pcp_op_T__ *ptr__, val__ = (val); \
|
| 320 | \
|
| 321 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 322 | asm_inline volatile( \
|
| 323 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 324 | "%[pair__]","%N[pair__]") \
|
| 325 | op " %[val__],0(%N[pair__])\n" \
|
| 326 | __PCPU_END("%[lcreg]") \
|
| 327 | : [pair__] "=&a" (rp__.pair), "=m" (*ptr__), \
|
| 328 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 329 | : [val__] "d" (val__), [ptr__] "a" (ptr__), \
|
| 330 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 331 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 332 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 333 | : "cc"); \
|
| 334 | } while (0)
|
| 335 |
|
| 336 | #define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc")
|
| 337 | #define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth")
|
| 338 | #define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st")
|
| 339 | #define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg")
|
| 340 |
|
| 341 | #define arch_this_cpu_cmpxchg_simple(pcp, oval, nval) \
|
| 342 | ({ \
|
| 343 | typedef typeof(pcp) pcp_op_T__; \
|
| 344 | pcp_op_T__ ret__; \
|
| 345 | pcp_op_T__ *ptr__; \
|
| 346 | preempt_disable_notrace(); \
|
| 347 | ptr__ = raw_cpu_ptr(&(pcp)); \
|
| 348 | ret__ = cmpxchg(ptr__, oval, nval); \
|
| 349 | preempt_enable_notrace(); \
|
| 350 | ret__; \
|
| 351 | })
|
| 352 |
|
| 353 | #define arch_this_cpu_cmpxchg(pcp, oval, nval, op) \
|
| 354 | ({ \
|
| 355 | typedef typeof(pcp) pcp_op_T__; \
|
| 356 | pcp_op_T__ old__ = (oval), new__ = (nval); \
|
| 357 | union register_pair rp__; \
|
| 358 | pcp_op_T__ *ptr__; \
|
| 359 | \
|
| 360 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 361 | asm_inline volatile( \
|
| 362 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 363 | "%[pair__]","%N[pair__]") \
|
| 364 | op " %[old__],%[new__],0(%N[pair__])\n" \
|
| 365 | __PCPU_END("%[lcreg]") \
|
| 366 | : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \
|
| 367 | "+m" (*ptr__), \
|
| 368 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 369 | : [new__] "d" (new__), [ptr__] "a" (ptr__), \
|
| 370 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 371 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 372 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 373 | : "memory", "cc"); \
|
| 374 | old__; \
|
| 375 | })
|
| 376 |
|
| 377 | #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval)
|
| 378 | #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval)
|
| 379 | #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "cs")
|
| 380 | #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "csg")
|
| 381 |
|
| 382 | #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
|
| 383 |
|
| 384 | #define this_cpu_cmpxchg128(pcp, oval, nval) \
|
| 385 | ({ \
|
| 386 | typedef typeof(pcp) pcp_op_T__; \
|
| 387 | u128 old__ = (oval), new__ = (nval); \
|
| 388 | union register_pair rp__; \
|
| 389 | pcp_op_T__ *ptr__; \
|
| 390 | \
|
| 391 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 392 | asm_inline volatile( \
|
| 393 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 394 | "%[pair__]","%N[pair__]") \
|
| 395 | " cdsg %[old__],%[new__],0(%N[pair__])\n" \
|
| 396 | __PCPU_END("%[lcreg]") \
|
| 397 | : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \
|
| 398 | "+m" (*ptr__), \
|
| 399 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 400 | : [new__] "d" (new__), [ptr__] "a" (ptr__), \
|
| 401 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 402 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 403 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 404 | : "memory", "cc"); \
|
| 405 | old__; \
|
| 406 | })
|
| 407 |
|
| 408 | #ifdef __HAVE_ASM_FLAG_OUTPUTS__
|
| 409 |
|
| 410 | #define arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, op) \
|
| 411 | ({ \
|
| 412 | typedef typeof(pcp) pcp_op_T__; \
|
| 413 | typeof(ovalp) ovalp__ = (ovalp); \
|
| 414 | pcp_op_T__ old__ = *ovalp__; \
|
| 415 | pcp_op_T__ new__ = (nval); \
|
| 416 | union register_pair rp__; \
|
| 417 | pcp_op_T__ *ptr__; \
|
| 418 | int cc__; \
|
| 419 | \
|
| 420 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 421 | asm_inline volatile( \
|
| 422 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 423 | "%[pair__]","%N[pair__]") \
|
| 424 | op " %[old__],%[new__],0(%N[pair__])\n" \
|
| 425 | __PCPU_END("%[lcreg]") \
|
| 426 | : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \
|
| 427 | "+m" (*ptr__), "=@cc" (cc__), \
|
| 428 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 429 | : [new__] "d" (new__), [ptr__] "a" (ptr__), \
|
| 430 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 431 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 432 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 433 | : "memory"); \
|
| 434 | if (unlikely(cc__)) \
|
| 435 | *ovalp__ = old__; \
|
| 436 | likely(cc__ == 0); \
|
| 437 | })
|
| 438 |
|
| 439 | #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "cs")
|
| 440 | #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "csg")
|
| 441 |
|
| 442 | #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) this_cpu_try_cmpxchg_8(pcp, ovalp, nval)
|
| 443 |
|
| 444 | #endif
|
| 445 |
|
| 446 | #define arch_this_cpu_xchg_simple(pcp, nval) \
|
| 447 | ({ \
|
| 448 | typeof(pcp) *ptr__; \
|
| 449 | typeof(pcp) ret__; \
|
| 450 | preempt_disable_notrace(); \
|
| 451 | ptr__ = raw_cpu_ptr(&(pcp)); \
|
| 452 | ret__ = xchg(ptr__, nval); \
|
| 453 | preempt_enable_notrace(); \
|
| 454 | ret__; \
|
| 455 | })
|
| 456 |
|
| 457 | #define arch_this_cpu_xchg(pcp, nval, ldop, csop) \
|
| 458 | ({ \
|
| 459 | typedef typeof(pcp) pcp_op_T__; \
|
| 460 | pcp_op_T__ old__, new__ = (nval); \
|
| 461 | union register_pair rp__; \
|
| 462 | pcp_op_T__ *ptr__; \
|
| 463 | \
|
| 464 | ptr__ = PERCPU_PTR(&(pcp)); \
|
| 465 | asm_inline volatile( \
|
| 466 | __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \
|
| 467 | "%[pair__]","%N[pair__]") \
|
| 468 | " " ldop " %[old__],0(%N[pair__])\n" \
|
| 469 | "0: " csop " %[old__],%[new__],0(%N[pair__])\n"\
|
| 470 | " jnz 0b\n" \
|
| 471 | __PCPU_END("%[lcreg]") \
|
| 472 | : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \
|
| 473 | "+m" (*ptr__), \
|
| 474 | "=m" (((struct lowcore *)0)->percpu_register) \
|
| 475 | : [new__] "d" (new__), [ptr__] "a" (ptr__), \
|
| 476 | [lcreg] "i" (LC_PERCPU_REGISTER), \
|
| 477 | [lcoff] "i" (LC_PERCPU_OFFSET), \
|
| 478 | "m" (((struct lowcore *)0)->percpu_offset) \
|
| 479 | : "memory", "cc"); \
|
| 480 | old__; \
|
| 481 | })
|
| 482 |
|
| 483 | #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval)
|
| 484 | #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval)
|
| 485 | #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval, "l", "cs")
|
| 486 | #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval, "lg", "csg")
|
| 487 |
|
| 488 | #include <asm-generic/percpu.h>
|
| 489 |
|
| 490 | #endif
|