| 1 | #ifndef __LINUX_PERCPU_H
|
| 2 | #define __LINUX_PERCPU_H
|
| 3 |
|
| 4 | #include <linux/mmdebug.h>
|
| 5 | #include <linux/preempt.h>
|
| 6 | #include <linux/smp.h>
|
| 7 | #include <linux/cpumask.h>
|
| 8 | #include <linux/pfn.h>
|
| 9 | #include <linux/init.h>
|
| 10 |
|
| 11 | #include <asm/percpu.h>
|
| 12 |
|
| 13 |
|
| 14 | #ifdef CONFIG_MODULES
|
| 15 | #define PERCPU_MODULE_RESERVE (8 << 10)
|
| 16 | #else
|
| 17 | #define PERCPU_MODULE_RESERVE 0
|
| 18 | #endif
|
| 19 |
|
| 20 | #ifndef PERCPU_ENOUGH_ROOM
|
| 21 | #define PERCPU_ENOUGH_ROOM \
|
| 22 | (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
|
| 23 | PERCPU_MODULE_RESERVE)
|
| 24 | #endif
|
| 25 |
|
| 26 | |
| 27 | * Must be an lvalue. Since @var must be a simple identifier,
|
| 28 | * we force a syntax error here if it isn't.
|
| 29 |
|
| 30 | #define get_cpu_var(var) (*({ \
|
| 31 | preempt_disable(); \
|
| 32 | &__get_cpu_var(var); }))
|
| 33 |
|
| 34 | |
| 35 | * The weird & is necessary because sparse considers (void)(var) to be
|
| 36 | * a direct dereference of percpu variable (var).
|
| 37 |
|
| 38 | #define put_cpu_var(var) do { \
|
| 39 | (void)&(var); \
|
| 40 | preempt_enable(); \
|
| 41 | } while (0)
|
| 42 |
|
| 43 | #define get_cpu_ptr(var) ({ \
|
| 44 | preempt_disable(); \
|
| 45 | this_cpu_ptr(var); })
|
| 46 |
|
| 47 | #define put_cpu_ptr(var) do { \
|
| 48 | (void)(var); \
|
| 49 | preempt_enable(); \
|
| 50 | } while (0)
|
| 51 |
|
| 52 |
|
| 53 | #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
|
| 54 |
|
| 55 | |
| 56 | * Percpu allocator can serve percpu allocations before slab is
|
| 57 | * initialized which allows slab to depend on the percpu allocator.
|
| 58 | * The following two parameters decide how much resource to
|
| 59 | * preallocate for this. Keep PERCPU_DYNAMIC_RESERVE equal to or
|
| 60 | * larger than PERCPU_DYNAMIC_EARLY_SIZE.
|
| 61 |
|
| 62 | #define PERCPU_DYNAMIC_EARLY_SLOTS 128
|
| 63 | #define PERCPU_DYNAMIC_EARLY_SIZE (12 << 10)
|
| 64 |
|
| 65 | |
| 66 | * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
|
| 67 | * back on the first chunk for dynamic percpu allocation if arch is
|
| 68 | * manually allocating and mapping it for faster access (as a part of
|
| 69 | * large page mapping for example).
|
| 70 | *
|
| 71 | * The following values give between one and two pages of free space
|
| 72 | * after typical minimal boot (2-way SMP, single disk and NIC) with
|
| 73 | * both defconfig and a distro config on x86_64 and 32. More
|
| 74 | * intelligent way to determine this would be nice.
|
| 75 |
|
| 76 | #if BITS_PER_LONG > 32
|
| 77 | #define PERCPU_DYNAMIC_RESERVE (20 << 10)
|
| 78 | #else
|
| 79 | #define PERCPU_DYNAMIC_RESERVE (12 << 10)
|
| 80 | #endif
|
| 81 |
|
| 82 | extern void *pcpu_base_addr;
|
| 83 | extern const unsigned long *pcpu_unit_offsets;
|
| 84 |
|
| 85 | struct pcpu_group_info {
|
| 86 | int nr_units;
|
| 87 | unsigned long base_offset;
|
| 88 | unsigned int *cpu_map; |
| 89 | * entries contain NR_CPUS
|
| 90 | };
|
| 91 |
|
| 92 | struct pcpu_alloc_info {
|
| 93 | size_t static_size;
|
| 94 | size_t reserved_size;
|
| 95 | size_t dyn_size;
|
| 96 | size_t unit_size;
|
| 97 | size_t atom_size;
|
| 98 | size_t alloc_size;
|
| 99 | size_t __ai_size;
|
| 100 | int nr_groups;
|
| 101 | struct pcpu_group_info groups[];
|
| 102 | };
|
| 103 |
|
| 104 | enum pcpu_fc {
|
| 105 | PCPU_FC_AUTO,
|
| 106 | PCPU_FC_EMBED,
|
| 107 | PCPU_FC_PAGE,
|
| 108 |
|
| 109 | PCPU_FC_NR,
|
| 110 | };
|
| 111 | extern const char * const pcpu_fc_names[PCPU_FC_NR];
|
| 112 |
|
| 113 | extern enum pcpu_fc pcpu_chosen_fc;
|
| 114 |
|
| 115 | typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size,
|
| 116 | size_t align);
|
| 117 | typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);
|
| 118 | typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
|
| 119 | typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to);
|
| 120 |
|
| 121 | extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
|
| 122 | int nr_units);
|
| 123 | extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);
|
| 124 |
|
| 125 | extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
|
| 126 | void *base_addr);
|
| 127 |
|
| 128 | #ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
|
| 129 | extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
|
| 130 | size_t atom_size,
|
| 131 | pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
|
| 132 | pcpu_fc_alloc_fn_t alloc_fn,
|
| 133 | pcpu_fc_free_fn_t free_fn);
|
| 134 | #endif
|
| 135 |
|
| 136 | #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
|
| 137 | extern int __init pcpu_page_first_chunk(size_t reserved_size,
|
| 138 | pcpu_fc_alloc_fn_t alloc_fn,
|
| 139 | pcpu_fc_free_fn_t free_fn,
|
| 140 | pcpu_fc_populate_pte_fn_t populate_pte_fn);
|
| 141 | #endif
|
| 142 |
|
| 143 | |
| 144 | * Use this to get to a cpu's version of the per-cpu object
|
| 145 | * dynamically allocated. Non-atomic access to the current CPU's
|
| 146 | * version should probably be combined with get_cpu()/put_cpu().
|
| 147 |
|
| 148 | #ifdef CONFIG_SMP
|
| 149 | #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
|
| 150 | #else
|
| 151 | #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
|
| 152 | #endif
|
| 153 |
|
| 154 | extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
|
| 155 | extern bool is_kernel_percpu_address(unsigned long addr);
|
| 156 |
|
| 157 | #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
|
| 158 | extern void __init setup_per_cpu_areas(void);
|
| 159 | #endif
|
| 160 | extern void __init percpu_init_late(void);
|
| 161 |
|
| 162 | extern void __percpu *__alloc_percpu(size_t size, size_t align);
|
| 163 | extern void free_percpu(void __percpu *__pdata);
|
| 164 | extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
|
| 165 |
|
| 166 | #define alloc_percpu(type) \
|
| 167 | (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
|
| 168 |
|
| 169 | |
| 170 | * Branching function to split up a function into a set of functions that
|
| 171 | * are called for different scalar sizes of the objects handled.
|
| 172 |
|
| 173 |
|
| 174 | extern void __bad_size_call_parameter(void);
|
| 175 |
|
| 176 | #ifdef CONFIG_DEBUG_PREEMPT
|
| 177 | extern void __this_cpu_preempt_check(const char *op);
|
| 178 | #else
|
| 179 | static inline void __this_cpu_preempt_check(const char *op) { }
|
| 180 | #endif
|
| 181 |
|
| 182 | #define __pcpu_size_call_return(stem, variable) \
|
| 183 | ({ typeof(variable) pscr_ret__; \
|
| 184 | __verify_pcpu_ptr(&(variable)); \
|
| 185 | switch(sizeof(variable)) { \
|
| 186 | case 1: pscr_ret__ = stem##1(variable);break; \
|
| 187 | case 2: pscr_ret__ = stem##2(variable);break; \
|
| 188 | case 4: pscr_ret__ = stem##4(variable);break; \
|
| 189 | case 8: pscr_ret__ = stem##8(variable);break; \
|
| 190 | default: \
|
| 191 | __bad_size_call_parameter();break; \
|
| 192 | } \
|
| 193 | pscr_ret__; \
|
| 194 | })
|
| 195 |
|
| 196 | #define __pcpu_size_call_return2(stem, variable, ...) \
|
| 197 | ({ \
|
| 198 | typeof(variable) pscr2_ret__; \
|
| 199 | __verify_pcpu_ptr(&(variable)); \
|
| 200 | switch(sizeof(variable)) { \
|
| 201 | case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
|
| 202 | case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
|
| 203 | case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
|
| 204 | case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
|
| 205 | default: \
|
| 206 | __bad_size_call_parameter(); break; \
|
| 207 | } \
|
| 208 | pscr2_ret__; \
|
| 209 | })
|
| 210 |
|
| 211 | |
| 212 | * Special handling for cmpxchg_double. cmpxchg_double is passed two
|
| 213 | * percpu variables. The first has to be aligned to a double word
|
| 214 | * boundary and the second has to follow directly thereafter.
|
| 215 | * We enforce this on all architectures even if they don't support
|
| 216 | * a double cmpxchg instruction, since it's a cheap requirement, and it
|
| 217 | * avoids breaking the requirement for architectures with the instruction.
|
| 218 |
|
| 219 | #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
|
| 220 | ({ \
|
| 221 | bool pdcrb_ret__; \
|
| 222 | __verify_pcpu_ptr(&pcp1); \
|
| 223 | BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
|
| 224 | VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
|
| 225 | VM_BUG_ON((unsigned long)(&pcp2) != \
|
| 226 | (unsigned long)(&pcp1) + sizeof(pcp1)); \
|
| 227 | switch(sizeof(pcp1)) { \
|
| 228 | case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
|
| 229 | case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
|
| 230 | case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
|
| 231 | case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
|
| 232 | default: \
|
| 233 | __bad_size_call_parameter(); break; \
|
| 234 | } \
|
| 235 | pdcrb_ret__; \
|
| 236 | })
|
| 237 |
|
| 238 | #define __pcpu_size_call(stem, variable, ...) \
|
| 239 | do { \
|
| 240 | __verify_pcpu_ptr(&(variable)); \
|
| 241 | switch(sizeof(variable)) { \
|
| 242 | case 1: stem##1(variable, __VA_ARGS__);break; \
|
| 243 | case 2: stem##2(variable, __VA_ARGS__);break; \
|
| 244 | case 4: stem##4(variable, __VA_ARGS__);break; \
|
| 245 | case 8: stem##8(variable, __VA_ARGS__);break; \
|
| 246 | default: \
|
| 247 | __bad_size_call_parameter();break; \
|
| 248 | } \
|
| 249 | } while (0)
|
| 250 |
|
| 251 | |
| 252 | * this_cpu operations (C) 2008-2013 Christoph Lameter <[email protected]>
|
| 253 | *
|
| 254 | * Optimized manipulation for memory allocated through the per cpu
|
| 255 | * allocator or for addresses of per cpu variables.
|
| 256 | *
|
| 257 | * These operation guarantee exclusivity of access for other operations
|
| 258 | * on the *same* processor. The assumption is that per cpu data is only
|
| 259 | * accessed by a single processor instance (the current one).
|
| 260 | *
|
| 261 | * The first group is used for accesses that must be done in a
|
| 262 | * preemption safe way since we know that the context is not preempt
|
| 263 | * safe. Interrupts may occur. If the interrupt modifies the variable
|
| 264 | * too then RMW actions will not be reliable.
|
| 265 | *
|
| 266 | * The arch code can provide optimized functions in two ways:
|
| 267 | *
|
| 268 | * 1. Override the function completely. F.e. define this_cpu_add().
|
| 269 | * The arch must then ensure that the various scalar format passed
|
| 270 | * are handled correctly.
|
| 271 | *
|
| 272 | * 2. Provide functions for certain scalar sizes. F.e. provide
|
| 273 | * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
|
| 274 | * sized RMW actions. If arch code does not provide operations for
|
| 275 | * a scalar size then the fallback in the generic code will be
|
| 276 | * used.
|
| 277 |
|
| 278 |
|
| 279 | #define _this_cpu_generic_read(pcp) \
|
| 280 | ({ typeof(pcp) ret__; \
|
| 281 | preempt_disable(); \
|
| 282 | ret__ = *this_cpu_ptr(&(pcp)); \
|
| 283 | preempt_enable(); \
|
| 284 | ret__; \
|
| 285 | })
|
| 286 |
|
| 287 | #ifndef this_cpu_read
|
| 288 | # ifndef this_cpu_read_1
|
| 289 | # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
|
| 290 | # endif
|
| 291 | # ifndef this_cpu_read_2
|
| 292 | # define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
|
| 293 | # endif
|
| 294 | # ifndef this_cpu_read_4
|
| 295 | # define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
|
| 296 | # endif
|
| 297 | # ifndef this_cpu_read_8
|
| 298 | # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
|
| 299 | # endif
|
| 300 | # define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
|
| 301 | #endif
|
| 302 |
|
| 303 | #define _this_cpu_generic_to_op(pcp, val, op) \
|
| 304 | do { \
|
| 305 | unsigned long flags; \
|
| 306 | raw_local_irq_save(flags); \
|
| 307 | *raw_cpu_ptr(&(pcp)) op val; \
|
| 308 | raw_local_irq_restore(flags); \
|
| 309 | } while (0)
|
| 310 |
|
| 311 | #ifndef this_cpu_write
|
| 312 | # ifndef this_cpu_write_1
|
| 313 | # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 314 | # endif
|
| 315 | # ifndef this_cpu_write_2
|
| 316 | # define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 317 | # endif
|
| 318 | # ifndef this_cpu_write_4
|
| 319 | # define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 320 | # endif
|
| 321 | # ifndef this_cpu_write_8
|
| 322 | # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 323 | # endif
|
| 324 | # define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
|
| 325 | #endif
|
| 326 |
|
| 327 | #ifndef this_cpu_add
|
| 328 | # ifndef this_cpu_add_1
|
| 329 | # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 330 | # endif
|
| 331 | # ifndef this_cpu_add_2
|
| 332 | # define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 333 | # endif
|
| 334 | # ifndef this_cpu_add_4
|
| 335 | # define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 336 | # endif
|
| 337 | # ifndef this_cpu_add_8
|
| 338 | # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 339 | # endif
|
| 340 | # define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
|
| 341 | #endif
|
| 342 |
|
| 343 | #ifndef this_cpu_sub
|
| 344 | # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val))
|
| 345 | #endif
|
| 346 |
|
| 347 | #ifndef this_cpu_inc
|
| 348 | # define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
|
| 349 | #endif
|
| 350 |
|
| 351 | #ifndef this_cpu_dec
|
| 352 | # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
|
| 353 | #endif
|
| 354 |
|
| 355 | #ifndef this_cpu_and
|
| 356 | # ifndef this_cpu_and_1
|
| 357 | # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 358 | # endif
|
| 359 | # ifndef this_cpu_and_2
|
| 360 | # define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 361 | # endif
|
| 362 | # ifndef this_cpu_and_4
|
| 363 | # define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 364 | # endif
|
| 365 | # ifndef this_cpu_and_8
|
| 366 | # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 367 | # endif
|
| 368 | # define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
|
| 369 | #endif
|
| 370 |
|
| 371 | #ifndef this_cpu_or
|
| 372 | # ifndef this_cpu_or_1
|
| 373 | # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 374 | # endif
|
| 375 | # ifndef this_cpu_or_2
|
| 376 | # define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 377 | # endif
|
| 378 | # ifndef this_cpu_or_4
|
| 379 | # define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 380 | # endif
|
| 381 | # ifndef this_cpu_or_8
|
| 382 | # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 383 | # endif
|
| 384 | # define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
|
| 385 | #endif
|
| 386 |
|
| 387 | #define _this_cpu_generic_add_return(pcp, val) \
|
| 388 | ({ \
|
| 389 | typeof(pcp) ret__; \
|
| 390 | unsigned long flags; \
|
| 391 | raw_local_irq_save(flags); \
|
| 392 | raw_cpu_add(pcp, val); \
|
| 393 | ret__ = raw_cpu_read(pcp); \
|
| 394 | raw_local_irq_restore(flags); \
|
| 395 | ret__; \
|
| 396 | })
|
| 397 |
|
| 398 | #ifndef this_cpu_add_return
|
| 399 | # ifndef this_cpu_add_return_1
|
| 400 | # define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 401 | # endif
|
| 402 | # ifndef this_cpu_add_return_2
|
| 403 | # define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 404 | # endif
|
| 405 | # ifndef this_cpu_add_return_4
|
| 406 | # define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 407 | # endif
|
| 408 | # ifndef this_cpu_add_return_8
|
| 409 | # define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 410 | # endif
|
| 411 | # define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
|
| 412 | #endif
|
| 413 |
|
| 414 | #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
|
| 415 | #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
|
| 416 | #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
|
| 417 |
|
| 418 | #define _this_cpu_generic_xchg(pcp, nval) \
|
| 419 | ({ typeof(pcp) ret__; \
|
| 420 | unsigned long flags; \
|
| 421 | raw_local_irq_save(flags); \
|
| 422 | ret__ = raw_cpu_read(pcp); \
|
| 423 | raw_cpu_write(pcp, nval); \
|
| 424 | raw_local_irq_restore(flags); \
|
| 425 | ret__; \
|
| 426 | })
|
| 427 |
|
| 428 | #ifndef this_cpu_xchg
|
| 429 | # ifndef this_cpu_xchg_1
|
| 430 | # define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 431 | # endif
|
| 432 | # ifndef this_cpu_xchg_2
|
| 433 | # define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 434 | # endif
|
| 435 | # ifndef this_cpu_xchg_4
|
| 436 | # define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 437 | # endif
|
| 438 | # ifndef this_cpu_xchg_8
|
| 439 | # define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 440 | # endif
|
| 441 | # define this_cpu_xchg(pcp, nval) \
|
| 442 | __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval)
|
| 443 | #endif
|
| 444 |
|
| 445 | #define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
|
| 446 | ({ \
|
| 447 | typeof(pcp) ret__; \
|
| 448 | unsigned long flags; \
|
| 449 | raw_local_irq_save(flags); \
|
| 450 | ret__ = raw_cpu_read(pcp); \
|
| 451 | if (ret__ == (oval)) \
|
| 452 | raw_cpu_write(pcp, nval); \
|
| 453 | raw_local_irq_restore(flags); \
|
| 454 | ret__; \
|
| 455 | })
|
| 456 |
|
| 457 | #ifndef this_cpu_cmpxchg
|
| 458 | # ifndef this_cpu_cmpxchg_1
|
| 459 | # define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 460 | # endif
|
| 461 | # ifndef this_cpu_cmpxchg_2
|
| 462 | # define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 463 | # endif
|
| 464 | # ifndef this_cpu_cmpxchg_4
|
| 465 | # define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 466 | # endif
|
| 467 | # ifndef this_cpu_cmpxchg_8
|
| 468 | # define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 469 | # endif
|
| 470 | # define this_cpu_cmpxchg(pcp, oval, nval) \
|
| 471 | __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
|
| 472 | #endif
|
| 473 |
|
| 474 | |
| 475 | * cmpxchg_double replaces two adjacent scalars at once. The first
|
| 476 | * two parameters are per cpu variables which have to be of the same
|
| 477 | * size. A truth value is returned to indicate success or failure
|
| 478 | * (since a double register result is difficult to handle). There is
|
| 479 | * very limited hardware support for these operations, so only certain
|
| 480 | * sizes may work.
|
| 481 |
|
| 482 | #define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 483 | ({ \
|
| 484 | int ret__; \
|
| 485 | unsigned long flags; \
|
| 486 | raw_local_irq_save(flags); \
|
| 487 | ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
|
| 488 | oval1, oval2, nval1, nval2); \
|
| 489 | raw_local_irq_restore(flags); \
|
| 490 | ret__; \
|
| 491 | })
|
| 492 |
|
| 493 | #ifndef this_cpu_cmpxchg_double
|
| 494 | # ifndef this_cpu_cmpxchg_double_1
|
| 495 | # define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 496 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 497 | # endif
|
| 498 | # ifndef this_cpu_cmpxchg_double_2
|
| 499 | # define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 500 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 501 | # endif
|
| 502 | # ifndef this_cpu_cmpxchg_double_4
|
| 503 | # define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 504 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 505 | # endif
|
| 506 | # ifndef this_cpu_cmpxchg_double_8
|
| 507 | # define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 508 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 509 | # endif
|
| 510 | # define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 511 | __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
|
| 512 | #endif
|
| 513 |
|
| 514 | |
| 515 | * Generic percpu operations for contexts where we do not want to do
|
| 516 | * any checks for preemptiosn.
|
| 517 | *
|
| 518 | * If there is no other protection through preempt disable and/or
|
| 519 | * disabling interupts then one of these RMW operations can show unexpected
|
| 520 | * behavior because the execution thread was rescheduled on another processor
|
| 521 | * or an interrupt occurred and the same percpu variable was modified from
|
| 522 | * the interrupt context.
|
| 523 |
|
| 524 | #ifndef raw_cpu_read
|
| 525 | # ifndef raw_cpu_read_1
|
| 526 | # define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
|
| 527 | # endif
|
| 528 | # ifndef raw_cpu_read_2
|
| 529 | # define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
|
| 530 | # endif
|
| 531 | # ifndef raw_cpu_read_4
|
| 532 | # define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
|
| 533 | # endif
|
| 534 | # ifndef raw_cpu_read_8
|
| 535 | # define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
|
| 536 | # endif
|
| 537 | # define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))
|
| 538 | #endif
|
| 539 |
|
| 540 | #define raw_cpu_generic_to_op(pcp, val, op) \
|
| 541 | do { \
|
| 542 | *raw_cpu_ptr(&(pcp)) op val; \
|
| 543 | } while (0)
|
| 544 |
|
| 545 |
|
| 546 | #ifndef raw_cpu_write
|
| 547 | # ifndef raw_cpu_write_1
|
| 548 | # define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
| 549 | # endif
|
| 550 | # ifndef raw_cpu_write_2
|
| 551 | # define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
| 552 | # endif
|
| 553 | # ifndef raw_cpu_write_4
|
| 554 | # define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
| 555 | # endif
|
| 556 | # ifndef raw_cpu_write_8
|
| 557 | # define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
| 558 | # endif
|
| 559 | # define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))
|
| 560 | #endif
|
| 561 |
|
| 562 | #ifndef raw_cpu_add
|
| 563 | # ifndef raw_cpu_add_1
|
| 564 | # define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
| 565 | # endif
|
| 566 | # ifndef raw_cpu_add_2
|
| 567 | # define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
| 568 | # endif
|
| 569 | # ifndef raw_cpu_add_4
|
| 570 | # define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
| 571 | # endif
|
| 572 | # ifndef raw_cpu_add_8
|
| 573 | # define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
| 574 | # endif
|
| 575 | # define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))
|
| 576 | #endif
|
| 577 |
|
| 578 | #ifndef raw_cpu_sub
|
| 579 | # define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val))
|
| 580 | #endif
|
| 581 |
|
| 582 | #ifndef raw_cpu_inc
|
| 583 | # define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1)
|
| 584 | #endif
|
| 585 |
|
| 586 | #ifndef raw_cpu_dec
|
| 587 | # define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1)
|
| 588 | #endif
|
| 589 |
|
| 590 | #ifndef raw_cpu_and
|
| 591 | # ifndef raw_cpu_and_1
|
| 592 | # define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
| 593 | # endif
|
| 594 | # ifndef raw_cpu_and_2
|
| 595 | # define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
| 596 | # endif
|
| 597 | # ifndef raw_cpu_and_4
|
| 598 | # define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
| 599 | # endif
|
| 600 | # ifndef raw_cpu_and_8
|
| 601 | # define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
| 602 | # endif
|
| 603 | # define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))
|
| 604 | #endif
|
| 605 |
|
| 606 | #ifndef raw_cpu_or
|
| 607 | # ifndef raw_cpu_or_1
|
| 608 | # define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
| 609 | # endif
|
| 610 | # ifndef raw_cpu_or_2
|
| 611 | # define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
| 612 | # endif
|
| 613 | # ifndef raw_cpu_or_4
|
| 614 | # define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
| 615 | # endif
|
| 616 | # ifndef raw_cpu_or_8
|
| 617 | # define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
| 618 | # endif
|
| 619 | # define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))
|
| 620 | #endif
|
| 621 |
|
| 622 | #define raw_cpu_generic_add_return(pcp, val) \
|
| 623 | ({ \
|
| 624 | raw_cpu_add(pcp, val); \
|
| 625 | raw_cpu_read(pcp); \
|
| 626 | })
|
| 627 |
|
| 628 | #ifndef raw_cpu_add_return
|
| 629 | # ifndef raw_cpu_add_return_1
|
| 630 | # define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
| 631 | # endif
|
| 632 | # ifndef raw_cpu_add_return_2
|
| 633 | # define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
| 634 | # endif
|
| 635 | # ifndef raw_cpu_add_return_4
|
| 636 | # define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
| 637 | # endif
|
| 638 | # ifndef raw_cpu_add_return_8
|
| 639 | # define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
| 640 | # endif
|
| 641 | # define raw_cpu_add_return(pcp, val) \
|
| 642 | __pcpu_size_call_return2(raw_add_return_, pcp, val)
|
| 643 | #endif
|
| 644 |
|
| 645 | #define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
|
| 646 | #define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
|
| 647 | #define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
|
| 648 |
|
| 649 | #define raw_cpu_generic_xchg(pcp, nval) \
|
| 650 | ({ typeof(pcp) ret__; \
|
| 651 | ret__ = raw_cpu_read(pcp); \
|
| 652 | raw_cpu_write(pcp, nval); \
|
| 653 | ret__; \
|
| 654 | })
|
| 655 |
|
| 656 | #ifndef raw_cpu_xchg
|
| 657 | # ifndef raw_cpu_xchg_1
|
| 658 | # define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
| 659 | # endif
|
| 660 | # ifndef raw_cpu_xchg_2
|
| 661 | # define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
| 662 | # endif
|
| 663 | # ifndef raw_cpu_xchg_4
|
| 664 | # define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
| 665 | # endif
|
| 666 | # ifndef raw_cpu_xchg_8
|
| 667 | # define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
| 668 | # endif
|
| 669 | # define raw_cpu_xchg(pcp, nval) \
|
| 670 | __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)
|
| 671 | #endif
|
| 672 |
|
| 673 | #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
|
| 674 | ({ \
|
| 675 | typeof(pcp) ret__; \
|
| 676 | ret__ = raw_cpu_read(pcp); \
|
| 677 | if (ret__ == (oval)) \
|
| 678 | raw_cpu_write(pcp, nval); \
|
| 679 | ret__; \
|
| 680 | })
|
| 681 |
|
| 682 | #ifndef raw_cpu_cmpxchg
|
| 683 | # ifndef raw_cpu_cmpxchg_1
|
| 684 | # define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 685 | # endif
|
| 686 | # ifndef raw_cpu_cmpxchg_2
|
| 687 | # define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 688 | # endif
|
| 689 | # ifndef raw_cpu_cmpxchg_4
|
| 690 | # define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 691 | # endif
|
| 692 | # ifndef raw_cpu_cmpxchg_8
|
| 693 | # define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 694 | # endif
|
| 695 | # define raw_cpu_cmpxchg(pcp, oval, nval) \
|
| 696 | __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
|
| 697 | #endif
|
| 698 |
|
| 699 | #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 700 | ({ \
|
| 701 | int __ret = 0; \
|
| 702 | if (raw_cpu_read(pcp1) == (oval1) && \
|
| 703 | raw_cpu_read(pcp2) == (oval2)) { \
|
| 704 | raw_cpu_write(pcp1, (nval1)); \
|
| 705 | raw_cpu_write(pcp2, (nval2)); \
|
| 706 | __ret = 1; \
|
| 707 | } \
|
| 708 | (__ret); \
|
| 709 | })
|
| 710 |
|
| 711 | #ifndef raw_cpu_cmpxchg_double
|
| 712 | # ifndef raw_cpu_cmpxchg_double_1
|
| 713 | # define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 714 | raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 715 | # endif
|
| 716 | # ifndef raw_cpu_cmpxchg_double_2
|
| 717 | # define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 718 | raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 719 | # endif
|
| 720 | # ifndef raw_cpu_cmpxchg_double_4
|
| 721 | # define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 722 | raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 723 | # endif
|
| 724 | # ifndef raw_cpu_cmpxchg_double_8
|
| 725 | # define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 726 | raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 727 | # endif
|
| 728 | # define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 729 | __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
|
| 730 | #endif
|
| 731 |
|
| 732 | |
| 733 | * Generic percpu operations for context that are safe from preemption/interrupts.
|
| 734 |
|
| 735 | #ifndef __this_cpu_read
|
| 736 | # define __this_cpu_read(pcp) \
|
| 737 | (__this_cpu_preempt_check("read"),__pcpu_size_call_return(raw_cpu_read_, (pcp)))
|
| 738 | #endif
|
| 739 |
|
| 740 | #ifndef __this_cpu_write
|
| 741 | # define __this_cpu_write(pcp, val) \
|
| 742 | do { __this_cpu_preempt_check("write"); \
|
| 743 | __pcpu_size_call(raw_cpu_write_, (pcp), (val)); \
|
| 744 | } while (0)
|
| 745 | #endif
|
| 746 |
|
| 747 | #ifndef __this_cpu_add
|
| 748 | # define __this_cpu_add(pcp, val) \
|
| 749 | do { __this_cpu_preempt_check("add"); \
|
| 750 | __pcpu_size_call(raw_cpu_add_, (pcp), (val)); \
|
| 751 | } while (0)
|
| 752 | #endif
|
| 753 |
|
| 754 | #ifndef __this_cpu_sub
|
| 755 | # define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val))
|
| 756 | #endif
|
| 757 |
|
| 758 | #ifndef __this_cpu_inc
|
| 759 | # define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
|
| 760 | #endif
|
| 761 |
|
| 762 | #ifndef __this_cpu_dec
|
| 763 | # define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
|
| 764 | #endif
|
| 765 |
|
| 766 | #ifndef __this_cpu_and
|
| 767 | # define __this_cpu_and(pcp, val) \
|
| 768 | do { __this_cpu_preempt_check("and"); \
|
| 769 | __pcpu_size_call(raw_cpu_and_, (pcp), (val)); \
|
| 770 | } while (0)
|
| 771 |
|
| 772 | #endif
|
| 773 |
|
| 774 | #ifndef __this_cpu_or
|
| 775 | # define __this_cpu_or(pcp, val) \
|
| 776 | do { __this_cpu_preempt_check("or"); \
|
| 777 | __pcpu_size_call(raw_cpu_or_, (pcp), (val)); \
|
| 778 | } while (0)
|
| 779 | #endif
|
| 780 |
|
| 781 | #ifndef __this_cpu_add_return
|
| 782 | # define __this_cpu_add_return(pcp, val) \
|
| 783 | (__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val))
|
| 784 | #endif
|
| 785 |
|
| 786 | #define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
|
| 787 | #define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
|
| 788 | #define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
|
| 789 |
|
| 790 | #ifndef __this_cpu_xchg
|
| 791 | # define __this_cpu_xchg(pcp, nval) \
|
| 792 | (__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval))
|
| 793 | #endif
|
| 794 |
|
| 795 | #ifndef __this_cpu_cmpxchg
|
| 796 | # define __this_cpu_cmpxchg(pcp, oval, nval) \
|
| 797 | (__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval))
|
| 798 | #endif
|
| 799 |
|
| 800 | #ifndef __this_cpu_cmpxchg_double
|
| 801 | # define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 802 | (__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)))
|
| 803 | #endif
|
| 804 |
|
| 805 | #endif
|