| 1 | #ifndef __LINUX_PERCPU_H
|
| 2 | #define __LINUX_PERCPU_H
|
| 3 |
|
| 4 | #include <linux/preempt.h>
|
| 5 | #include <linux/smp.h>
|
| 6 | #include <linux/cpumask.h>
|
| 7 | #include <linux/pfn.h>
|
| 8 | #include <linux/init.h>
|
| 9 |
|
| 10 | #include <asm/percpu.h>
|
| 11 |
|
| 12 |
|
| 13 | #ifdef CONFIG_MODULES
|
| 14 | #define PERCPU_MODULE_RESERVE (8 << 10)
|
| 15 | #else
|
| 16 | #define PERCPU_MODULE_RESERVE 0
|
| 17 | #endif
|
| 18 |
|
| 19 | #ifndef PERCPU_ENOUGH_ROOM
|
| 20 | #define PERCPU_ENOUGH_ROOM \
|
| 21 | (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
|
| 22 | PERCPU_MODULE_RESERVE)
|
| 23 | #endif
|
| 24 |
|
| 25 | |
| 26 | * Must be an lvalue. Since @var must be a simple identifier,
|
| 27 | * we force a syntax error here if it isn't.
|
| 28 |
|
| 29 | #define get_cpu_var(var) (*({ \
|
| 30 | preempt_disable(); \
|
| 31 | &__get_cpu_var(var); }))
|
| 32 |
|
| 33 | |
| 34 | * The weird & is necessary because sparse considers (void)(var) to be
|
| 35 | * a direct dereference of percpu variable (var).
|
| 36 |
|
| 37 | #define put_cpu_var(var) do { \
|
| 38 | (void)&(var); \
|
| 39 | preempt_enable(); \
|
| 40 | } while (0)
|
| 41 |
|
| 42 | #define get_cpu_ptr(var) ({ \
|
| 43 | preempt_disable(); \
|
| 44 | this_cpu_ptr(var); })
|
| 45 |
|
| 46 | #define put_cpu_ptr(var) do { \
|
| 47 | (void)(var); \
|
| 48 | preempt_enable(); \
|
| 49 | } while (0)
|
| 50 |
|
| 51 |
|
| 52 | #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
|
| 53 |
|
| 54 | |
| 55 | * Percpu allocator can serve percpu allocations before slab is
|
| 56 | * initialized which allows slab to depend on the percpu allocator.
|
| 57 | * The following two parameters decide how much resource to
|
| 58 | * preallocate for this. Keep PERCPU_DYNAMIC_RESERVE equal to or
|
| 59 | * larger than PERCPU_DYNAMIC_EARLY_SIZE.
|
| 60 |
|
| 61 | #define PERCPU_DYNAMIC_EARLY_SLOTS 128
|
| 62 | #define PERCPU_DYNAMIC_EARLY_SIZE (12 << 10)
|
| 63 |
|
| 64 | |
| 65 | * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
|
| 66 | * back on the first chunk for dynamic percpu allocation if arch is
|
| 67 | * manually allocating and mapping it for faster access (as a part of
|
| 68 | * large page mapping for example).
|
| 69 | *
|
| 70 | * The following values give between one and two pages of free space
|
| 71 | * after typical minimal boot (2-way SMP, single disk and NIC) with
|
| 72 | * both defconfig and a distro config on x86_64 and 32. More
|
| 73 | * intelligent way to determine this would be nice.
|
| 74 |
|
| 75 | #if BITS_PER_LONG > 32
|
| 76 | #define PERCPU_DYNAMIC_RESERVE (20 << 10)
|
| 77 | #else
|
| 78 | #define PERCPU_DYNAMIC_RESERVE (12 << 10)
|
| 79 | #endif
|
| 80 |
|
| 81 | extern void *pcpu_base_addr;
|
| 82 | extern const unsigned long *pcpu_unit_offsets;
|
| 83 |
|
| 84 | struct pcpu_group_info {
|
| 85 | int nr_units;
|
| 86 | unsigned long base_offset;
|
| 87 | unsigned int *cpu_map; |
| 88 | * entries contain NR_CPUS
|
| 89 | };
|
| 90 |
|
| 91 | struct pcpu_alloc_info {
|
| 92 | size_t static_size;
|
| 93 | size_t reserved_size;
|
| 94 | size_t dyn_size;
|
| 95 | size_t unit_size;
|
| 96 | size_t atom_size;
|
| 97 | size_t alloc_size;
|
| 98 | size_t __ai_size;
|
| 99 | int nr_groups;
|
| 100 | struct pcpu_group_info groups[];
|
| 101 | };
|
| 102 |
|
| 103 | enum pcpu_fc {
|
| 104 | PCPU_FC_AUTO,
|
| 105 | PCPU_FC_EMBED,
|
| 106 | PCPU_FC_PAGE,
|
| 107 |
|
| 108 | PCPU_FC_NR,
|
| 109 | };
|
| 110 | extern const char * const pcpu_fc_names[PCPU_FC_NR];
|
| 111 |
|
| 112 | extern enum pcpu_fc pcpu_chosen_fc;
|
| 113 |
|
| 114 | typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size,
|
| 115 | size_t align);
|
| 116 | typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);
|
| 117 | typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
|
| 118 | typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to);
|
| 119 |
|
| 120 | extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
|
| 121 | int nr_units);
|
| 122 | extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);
|
| 123 |
|
| 124 | extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
|
| 125 | void *base_addr);
|
| 126 |
|
| 127 | #ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
|
| 128 | extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
|
| 129 | size_t atom_size,
|
| 130 | pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
|
| 131 | pcpu_fc_alloc_fn_t alloc_fn,
|
| 132 | pcpu_fc_free_fn_t free_fn);
|
| 133 | #endif
|
| 134 |
|
| 135 | #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
|
| 136 | extern int __init pcpu_page_first_chunk(size_t reserved_size,
|
| 137 | pcpu_fc_alloc_fn_t alloc_fn,
|
| 138 | pcpu_fc_free_fn_t free_fn,
|
| 139 | pcpu_fc_populate_pte_fn_t populate_pte_fn);
|
| 140 | #endif
|
| 141 |
|
| 142 | |
| 143 | * Use this to get to a cpu's version of the per-cpu object
|
| 144 | * dynamically allocated. Non-atomic access to the current CPU's
|
| 145 | * version should probably be combined with get_cpu()/put_cpu().
|
| 146 |
|
| 147 | #ifdef CONFIG_SMP
|
| 148 | #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
|
| 149 | #else
|
| 150 | #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
|
| 151 | #endif
|
| 152 |
|
| 153 | extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
|
| 154 | extern bool is_kernel_percpu_address(unsigned long addr);
|
| 155 |
|
| 156 | #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
|
| 157 | extern void __init setup_per_cpu_areas(void);
|
| 158 | #endif
|
| 159 | extern void __init percpu_init_late(void);
|
| 160 |
|
| 161 | extern void __percpu *__alloc_percpu(size_t size, size_t align);
|
| 162 | extern void free_percpu(void __percpu *__pdata);
|
| 163 | extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
|
| 164 |
|
| 165 | #define alloc_percpu(type) \
|
| 166 | (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
|
| 167 |
|
| 168 | |
| 169 | * Branching function to split up a function into a set of functions that
|
| 170 | * are called for different scalar sizes of the objects handled.
|
| 171 |
|
| 172 |
|
| 173 | extern void __bad_size_call_parameter(void);
|
| 174 |
|
| 175 | #define __pcpu_size_call_return(stem, variable) \
|
| 176 | ({ typeof(variable) pscr_ret__; \
|
| 177 | __verify_pcpu_ptr(&(variable)); \
|
| 178 | switch(sizeof(variable)) { \
|
| 179 | case 1: pscr_ret__ = stem##1(variable);break; \
|
| 180 | case 2: pscr_ret__ = stem##2(variable);break; \
|
| 181 | case 4: pscr_ret__ = stem##4(variable);break; \
|
| 182 | case 8: pscr_ret__ = stem##8(variable);break; \
|
| 183 | default: \
|
| 184 | __bad_size_call_parameter();break; \
|
| 185 | } \
|
| 186 | pscr_ret__; \
|
| 187 | })
|
| 188 |
|
| 189 | #define __pcpu_size_call_return2(stem, variable, ...) \
|
| 190 | ({ \
|
| 191 | typeof(variable) pscr2_ret__; \
|
| 192 | __verify_pcpu_ptr(&(variable)); \
|
| 193 | switch(sizeof(variable)) { \
|
| 194 | case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
|
| 195 | case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
|
| 196 | case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
|
| 197 | case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
|
| 198 | default: \
|
| 199 | __bad_size_call_parameter(); break; \
|
| 200 | } \
|
| 201 | pscr2_ret__; \
|
| 202 | })
|
| 203 |
|
| 204 | |
| 205 | * Special handling for cmpxchg_double. cmpxchg_double is passed two
|
| 206 | * percpu variables. The first has to be aligned to a double word
|
| 207 | * boundary and the second has to follow directly thereafter.
|
| 208 | * We enforce this on all architectures even if they don't support
|
| 209 | * a double cmpxchg instruction, since it's a cheap requirement, and it
|
| 210 | * avoids breaking the requirement for architectures with the instruction.
|
| 211 |
|
| 212 | #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
|
| 213 | ({ \
|
| 214 | bool pdcrb_ret__; \
|
| 215 | __verify_pcpu_ptr(&pcp1); \
|
| 216 | BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
|
| 217 | VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
|
| 218 | VM_BUG_ON((unsigned long)(&pcp2) != \
|
| 219 | (unsigned long)(&pcp1) + sizeof(pcp1)); \
|
| 220 | switch(sizeof(pcp1)) { \
|
| 221 | case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
|
| 222 | case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
|
| 223 | case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
|
| 224 | case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
|
| 225 | default: \
|
| 226 | __bad_size_call_parameter(); break; \
|
| 227 | } \
|
| 228 | pdcrb_ret__; \
|
| 229 | })
|
| 230 |
|
| 231 | #define __pcpu_size_call(stem, variable, ...) \
|
| 232 | do { \
|
| 233 | __verify_pcpu_ptr(&(variable)); \
|
| 234 | switch(sizeof(variable)) { \
|
| 235 | case 1: stem##1(variable, __VA_ARGS__);break; \
|
| 236 | case 2: stem##2(variable, __VA_ARGS__);break; \
|
| 237 | case 4: stem##4(variable, __VA_ARGS__);break; \
|
| 238 | case 8: stem##8(variable, __VA_ARGS__);break; \
|
| 239 | default: \
|
| 240 | __bad_size_call_parameter();break; \
|
| 241 | } \
|
| 242 | } while (0)
|
| 243 |
|
| 244 | |
| 245 | * Optimized manipulation for memory allocated through the per cpu
|
| 246 | * allocator or for addresses of per cpu variables.
|
| 247 | *
|
| 248 | * These operation guarantee exclusivity of access for other operations
|
| 249 | * on the *same* processor. The assumption is that per cpu data is only
|
| 250 | * accessed by a single processor instance (the current one).
|
| 251 | *
|
| 252 | * The first group is used for accesses that must be done in a
|
| 253 | * preemption safe way since we know that the context is not preempt
|
| 254 | * safe. Interrupts may occur. If the interrupt modifies the variable
|
| 255 | * too then RMW actions will not be reliable.
|
| 256 | *
|
| 257 | * The arch code can provide optimized functions in two ways:
|
| 258 | *
|
| 259 | * 1. Override the function completely. F.e. define this_cpu_add().
|
| 260 | * The arch must then ensure that the various scalar format passed
|
| 261 | * are handled correctly.
|
| 262 | *
|
| 263 | * 2. Provide functions for certain scalar sizes. F.e. provide
|
| 264 | * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
|
| 265 | * sized RMW actions. If arch code does not provide operations for
|
| 266 | * a scalar size then the fallback in the generic code will be
|
| 267 | * used.
|
| 268 |
|
| 269 |
|
| 270 | #define _this_cpu_generic_read(pcp) \
|
| 271 | ({ typeof(pcp) ret__; \
|
| 272 | preempt_disable(); \
|
| 273 | ret__ = *this_cpu_ptr(&(pcp)); \
|
| 274 | preempt_enable(); \
|
| 275 | ret__; \
|
| 276 | })
|
| 277 |
|
| 278 | #ifndef this_cpu_read
|
| 279 | # ifndef this_cpu_read_1
|
| 280 | # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
|
| 281 | # endif
|
| 282 | # ifndef this_cpu_read_2
|
| 283 | # define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
|
| 284 | # endif
|
| 285 | # ifndef this_cpu_read_4
|
| 286 | # define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
|
| 287 | # endif
|
| 288 | # ifndef this_cpu_read_8
|
| 289 | # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
|
| 290 | # endif
|
| 291 | # define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
|
| 292 | #endif
|
| 293 |
|
| 294 | #define _this_cpu_generic_to_op(pcp, val, op) \
|
| 295 | do { \
|
| 296 | unsigned long flags; \
|
| 297 | raw_local_irq_save(flags); \
|
| 298 | *__this_cpu_ptr(&(pcp)) op val; \
|
| 299 | raw_local_irq_restore(flags); \
|
| 300 | } while (0)
|
| 301 |
|
| 302 | #ifndef this_cpu_write
|
| 303 | # ifndef this_cpu_write_1
|
| 304 | # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 305 | # endif
|
| 306 | # ifndef this_cpu_write_2
|
| 307 | # define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 308 | # endif
|
| 309 | # ifndef this_cpu_write_4
|
| 310 | # define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 311 | # endif
|
| 312 | # ifndef this_cpu_write_8
|
| 313 | # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
| 314 | # endif
|
| 315 | # define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
|
| 316 | #endif
|
| 317 |
|
| 318 | #ifndef this_cpu_add
|
| 319 | # ifndef this_cpu_add_1
|
| 320 | # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 321 | # endif
|
| 322 | # ifndef this_cpu_add_2
|
| 323 | # define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 324 | # endif
|
| 325 | # ifndef this_cpu_add_4
|
| 326 | # define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 327 | # endif
|
| 328 | # ifndef this_cpu_add_8
|
| 329 | # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
| 330 | # endif
|
| 331 | # define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
|
| 332 | #endif
|
| 333 |
|
| 334 | #ifndef this_cpu_sub
|
| 335 | # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(val))
|
| 336 | #endif
|
| 337 |
|
| 338 | #ifndef this_cpu_inc
|
| 339 | # define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
|
| 340 | #endif
|
| 341 |
|
| 342 | #ifndef this_cpu_dec
|
| 343 | # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
|
| 344 | #endif
|
| 345 |
|
| 346 | #ifndef this_cpu_and
|
| 347 | # ifndef this_cpu_and_1
|
| 348 | # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 349 | # endif
|
| 350 | # ifndef this_cpu_and_2
|
| 351 | # define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 352 | # endif
|
| 353 | # ifndef this_cpu_and_4
|
| 354 | # define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 355 | # endif
|
| 356 | # ifndef this_cpu_and_8
|
| 357 | # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
| 358 | # endif
|
| 359 | # define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
|
| 360 | #endif
|
| 361 |
|
| 362 | #ifndef this_cpu_or
|
| 363 | # ifndef this_cpu_or_1
|
| 364 | # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 365 | # endif
|
| 366 | # ifndef this_cpu_or_2
|
| 367 | # define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 368 | # endif
|
| 369 | # ifndef this_cpu_or_4
|
| 370 | # define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 371 | # endif
|
| 372 | # ifndef this_cpu_or_8
|
| 373 | # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
| 374 | # endif
|
| 375 | # define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
|
| 376 | #endif
|
| 377 |
|
| 378 | #ifndef this_cpu_xor
|
| 379 | # ifndef this_cpu_xor_1
|
| 380 | # define this_cpu_xor_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
|
| 381 | # endif
|
| 382 | # ifndef this_cpu_xor_2
|
| 383 | # define this_cpu_xor_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
|
| 384 | # endif
|
| 385 | # ifndef this_cpu_xor_4
|
| 386 | # define this_cpu_xor_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
|
| 387 | # endif
|
| 388 | # ifndef this_cpu_xor_8
|
| 389 | # define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
|
| 390 | # endif
|
| 391 | # define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
|
| 392 | #endif
|
| 393 |
|
| 394 | #define _this_cpu_generic_add_return(pcp, val) \
|
| 395 | ({ \
|
| 396 | typeof(pcp) ret__; \
|
| 397 | unsigned long flags; \
|
| 398 | raw_local_irq_save(flags); \
|
| 399 | __this_cpu_add(pcp, val); \
|
| 400 | ret__ = __this_cpu_read(pcp); \
|
| 401 | raw_local_irq_restore(flags); \
|
| 402 | ret__; \
|
| 403 | })
|
| 404 |
|
| 405 | #ifndef this_cpu_add_return
|
| 406 | # ifndef this_cpu_add_return_1
|
| 407 | # define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 408 | # endif
|
| 409 | # ifndef this_cpu_add_return_2
|
| 410 | # define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 411 | # endif
|
| 412 | # ifndef this_cpu_add_return_4
|
| 413 | # define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 414 | # endif
|
| 415 | # ifndef this_cpu_add_return_8
|
| 416 | # define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
| 417 | # endif
|
| 418 | # define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
|
| 419 | #endif
|
| 420 |
|
| 421 | #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(val))
|
| 422 | #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
|
| 423 | #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
|
| 424 |
|
| 425 | #define _this_cpu_generic_xchg(pcp, nval) \
|
| 426 | ({ typeof(pcp) ret__; \
|
| 427 | unsigned long flags; \
|
| 428 | raw_local_irq_save(flags); \
|
| 429 | ret__ = __this_cpu_read(pcp); \
|
| 430 | __this_cpu_write(pcp, nval); \
|
| 431 | raw_local_irq_restore(flags); \
|
| 432 | ret__; \
|
| 433 | })
|
| 434 |
|
| 435 | #ifndef this_cpu_xchg
|
| 436 | # ifndef this_cpu_xchg_1
|
| 437 | # define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 438 | # endif
|
| 439 | # ifndef this_cpu_xchg_2
|
| 440 | # define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 441 | # endif
|
| 442 | # ifndef this_cpu_xchg_4
|
| 443 | # define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 444 | # endif
|
| 445 | # ifndef this_cpu_xchg_8
|
| 446 | # define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
| 447 | # endif
|
| 448 | # define this_cpu_xchg(pcp, nval) \
|
| 449 | __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval)
|
| 450 | #endif
|
| 451 |
|
| 452 | #define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
|
| 453 | ({ \
|
| 454 | typeof(pcp) ret__; \
|
| 455 | unsigned long flags; \
|
| 456 | raw_local_irq_save(flags); \
|
| 457 | ret__ = __this_cpu_read(pcp); \
|
| 458 | if (ret__ == (oval)) \
|
| 459 | __this_cpu_write(pcp, nval); \
|
| 460 | raw_local_irq_restore(flags); \
|
| 461 | ret__; \
|
| 462 | })
|
| 463 |
|
| 464 | #ifndef this_cpu_cmpxchg
|
| 465 | # ifndef this_cpu_cmpxchg_1
|
| 466 | # define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 467 | # endif
|
| 468 | # ifndef this_cpu_cmpxchg_2
|
| 469 | # define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 470 | # endif
|
| 471 | # ifndef this_cpu_cmpxchg_4
|
| 472 | # define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 473 | # endif
|
| 474 | # ifndef this_cpu_cmpxchg_8
|
| 475 | # define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 476 | # endif
|
| 477 | # define this_cpu_cmpxchg(pcp, oval, nval) \
|
| 478 | __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
|
| 479 | #endif
|
| 480 |
|
| 481 | |
| 482 | * cmpxchg_double replaces two adjacent scalars at once. The first
|
| 483 | * two parameters are per cpu variables which have to be of the same
|
| 484 | * size. A truth value is returned to indicate success or failure
|
| 485 | * (since a double register result is difficult to handle). There is
|
| 486 | * very limited hardware support for these operations, so only certain
|
| 487 | * sizes may work.
|
| 488 |
|
| 489 | #define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 490 | ({ \
|
| 491 | int ret__; \
|
| 492 | unsigned long flags; \
|
| 493 | raw_local_irq_save(flags); \
|
| 494 | ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \
|
| 495 | oval1, oval2, nval1, nval2); \
|
| 496 | raw_local_irq_restore(flags); \
|
| 497 | ret__; \
|
| 498 | })
|
| 499 |
|
| 500 | #ifndef this_cpu_cmpxchg_double
|
| 501 | # ifndef this_cpu_cmpxchg_double_1
|
| 502 | # define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 503 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 504 | # endif
|
| 505 | # ifndef this_cpu_cmpxchg_double_2
|
| 506 | # define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 507 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 508 | # endif
|
| 509 | # ifndef this_cpu_cmpxchg_double_4
|
| 510 | # define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 511 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 512 | # endif
|
| 513 | # ifndef this_cpu_cmpxchg_double_8
|
| 514 | # define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 515 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 516 | # endif
|
| 517 | # define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 518 | __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
|
| 519 | #endif
|
| 520 |
|
| 521 | |
| 522 | * Generic percpu operations for context that are safe from preemption/interrupts.
|
| 523 | * Either we do not care about races or the caller has the
|
| 524 | * responsibility of handling preemption/interrupt issues. Arch code can still
|
| 525 | * override these instructions since the arch per cpu code may be more
|
| 526 | * efficient and may actually get race freeness for free (that is the
|
| 527 | * case for x86 for example).
|
| 528 | *
|
| 529 | * If there is no other protection through preempt disable and/or
|
| 530 | * disabling interupts then one of these RMW operations can show unexpected
|
| 531 | * behavior because the execution thread was rescheduled on another processor
|
| 532 | * or an interrupt occurred and the same percpu variable was modified from
|
| 533 | * the interrupt context.
|
| 534 |
|
| 535 | #ifndef __this_cpu_read
|
| 536 | # ifndef __this_cpu_read_1
|
| 537 | # define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp)))
|
| 538 | # endif
|
| 539 | # ifndef __this_cpu_read_2
|
| 540 | # define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp)))
|
| 541 | # endif
|
| 542 | # ifndef __this_cpu_read_4
|
| 543 | # define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp)))
|
| 544 | # endif
|
| 545 | # ifndef __this_cpu_read_8
|
| 546 | # define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp)))
|
| 547 | # endif
|
| 548 | # define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp))
|
| 549 | #endif
|
| 550 |
|
| 551 | #define __this_cpu_generic_to_op(pcp, val, op) \
|
| 552 | do { \
|
| 553 | *__this_cpu_ptr(&(pcp)) op val; \
|
| 554 | } while (0)
|
| 555 |
|
| 556 | #ifndef __this_cpu_write
|
| 557 | # ifndef __this_cpu_write_1
|
| 558 | # define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
|
| 559 | # endif
|
| 560 | # ifndef __this_cpu_write_2
|
| 561 | # define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
|
| 562 | # endif
|
| 563 | # ifndef __this_cpu_write_4
|
| 564 | # define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
|
| 565 | # endif
|
| 566 | # ifndef __this_cpu_write_8
|
| 567 | # define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
|
| 568 | # endif
|
| 569 | # define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val))
|
| 570 | #endif
|
| 571 |
|
| 572 | #ifndef __this_cpu_add
|
| 573 | # ifndef __this_cpu_add_1
|
| 574 | # define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
|
| 575 | # endif
|
| 576 | # ifndef __this_cpu_add_2
|
| 577 | # define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
|
| 578 | # endif
|
| 579 | # ifndef __this_cpu_add_4
|
| 580 | # define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
|
| 581 | # endif
|
| 582 | # ifndef __this_cpu_add_8
|
| 583 | # define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
|
| 584 | # endif
|
| 585 | # define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val))
|
| 586 | #endif
|
| 587 |
|
| 588 | #ifndef __this_cpu_sub
|
| 589 | # define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(val))
|
| 590 | #endif
|
| 591 |
|
| 592 | #ifndef __this_cpu_inc
|
| 593 | # define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
|
| 594 | #endif
|
| 595 |
|
| 596 | #ifndef __this_cpu_dec
|
| 597 | # define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
|
| 598 | #endif
|
| 599 |
|
| 600 | #ifndef __this_cpu_and
|
| 601 | # ifndef __this_cpu_and_1
|
| 602 | # define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
|
| 603 | # endif
|
| 604 | # ifndef __this_cpu_and_2
|
| 605 | # define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
|
| 606 | # endif
|
| 607 | # ifndef __this_cpu_and_4
|
| 608 | # define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
|
| 609 | # endif
|
| 610 | # ifndef __this_cpu_and_8
|
| 611 | # define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
|
| 612 | # endif
|
| 613 | # define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val))
|
| 614 | #endif
|
| 615 |
|
| 616 | #ifndef __this_cpu_or
|
| 617 | # ifndef __this_cpu_or_1
|
| 618 | # define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
|
| 619 | # endif
|
| 620 | # ifndef __this_cpu_or_2
|
| 621 | # define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
|
| 622 | # endif
|
| 623 | # ifndef __this_cpu_or_4
|
| 624 | # define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
|
| 625 | # endif
|
| 626 | # ifndef __this_cpu_or_8
|
| 627 | # define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
|
| 628 | # endif
|
| 629 | # define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val))
|
| 630 | #endif
|
| 631 |
|
| 632 | #ifndef __this_cpu_xor
|
| 633 | # ifndef __this_cpu_xor_1
|
| 634 | # define __this_cpu_xor_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
|
| 635 | # endif
|
| 636 | # ifndef __this_cpu_xor_2
|
| 637 | # define __this_cpu_xor_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
|
| 638 | # endif
|
| 639 | # ifndef __this_cpu_xor_4
|
| 640 | # define __this_cpu_xor_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
|
| 641 | # endif
|
| 642 | # ifndef __this_cpu_xor_8
|
| 643 | # define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
|
| 644 | # endif
|
| 645 | # define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))
|
| 646 | #endif
|
| 647 |
|
| 648 | #define __this_cpu_generic_add_return(pcp, val) \
|
| 649 | ({ \
|
| 650 | __this_cpu_add(pcp, val); \
|
| 651 | __this_cpu_read(pcp); \
|
| 652 | })
|
| 653 |
|
| 654 | #ifndef __this_cpu_add_return
|
| 655 | # ifndef __this_cpu_add_return_1
|
| 656 | # define __this_cpu_add_return_1(pcp, val) __this_cpu_generic_add_return(pcp, val)
|
| 657 | # endif
|
| 658 | # ifndef __this_cpu_add_return_2
|
| 659 | # define __this_cpu_add_return_2(pcp, val) __this_cpu_generic_add_return(pcp, val)
|
| 660 | # endif
|
| 661 | # ifndef __this_cpu_add_return_4
|
| 662 | # define __this_cpu_add_return_4(pcp, val) __this_cpu_generic_add_return(pcp, val)
|
| 663 | # endif
|
| 664 | # ifndef __this_cpu_add_return_8
|
| 665 | # define __this_cpu_add_return_8(pcp, val) __this_cpu_generic_add_return(pcp, val)
|
| 666 | # endif
|
| 667 | # define __this_cpu_add_return(pcp, val) \
|
| 668 | __pcpu_size_call_return2(__this_cpu_add_return_, pcp, val)
|
| 669 | #endif
|
| 670 |
|
| 671 | #define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(val))
|
| 672 | #define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
|
| 673 | #define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
|
| 674 |
|
| 675 | #define __this_cpu_generic_xchg(pcp, nval) \
|
| 676 | ({ typeof(pcp) ret__; \
|
| 677 | ret__ = __this_cpu_read(pcp); \
|
| 678 | __this_cpu_write(pcp, nval); \
|
| 679 | ret__; \
|
| 680 | })
|
| 681 |
|
| 682 | #ifndef __this_cpu_xchg
|
| 683 | # ifndef __this_cpu_xchg_1
|
| 684 | # define __this_cpu_xchg_1(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
|
| 685 | # endif
|
| 686 | # ifndef __this_cpu_xchg_2
|
| 687 | # define __this_cpu_xchg_2(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
|
| 688 | # endif
|
| 689 | # ifndef __this_cpu_xchg_4
|
| 690 | # define __this_cpu_xchg_4(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
|
| 691 | # endif
|
| 692 | # ifndef __this_cpu_xchg_8
|
| 693 | # define __this_cpu_xchg_8(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
|
| 694 | # endif
|
| 695 | # define __this_cpu_xchg(pcp, nval) \
|
| 696 | __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval)
|
| 697 | #endif
|
| 698 |
|
| 699 | #define __this_cpu_generic_cmpxchg(pcp, oval, nval) \
|
| 700 | ({ \
|
| 701 | typeof(pcp) ret__; \
|
| 702 | ret__ = __this_cpu_read(pcp); \
|
| 703 | if (ret__ == (oval)) \
|
| 704 | __this_cpu_write(pcp, nval); \
|
| 705 | ret__; \
|
| 706 | })
|
| 707 |
|
| 708 | #ifndef __this_cpu_cmpxchg
|
| 709 | # ifndef __this_cpu_cmpxchg_1
|
| 710 | # define __this_cpu_cmpxchg_1(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 711 | # endif
|
| 712 | # ifndef __this_cpu_cmpxchg_2
|
| 713 | # define __this_cpu_cmpxchg_2(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 714 | # endif
|
| 715 | # ifndef __this_cpu_cmpxchg_4
|
| 716 | # define __this_cpu_cmpxchg_4(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 717 | # endif
|
| 718 | # ifndef __this_cpu_cmpxchg_8
|
| 719 | # define __this_cpu_cmpxchg_8(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
|
| 720 | # endif
|
| 721 | # define __this_cpu_cmpxchg(pcp, oval, nval) \
|
| 722 | __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval)
|
| 723 | #endif
|
| 724 |
|
| 725 | #define __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 726 | ({ \
|
| 727 | int __ret = 0; \
|
| 728 | if (__this_cpu_read(pcp1) == (oval1) && \
|
| 729 | __this_cpu_read(pcp2) == (oval2)) { \
|
| 730 | __this_cpu_write(pcp1, (nval1)); \
|
| 731 | __this_cpu_write(pcp2, (nval2)); \
|
| 732 | __ret = 1; \
|
| 733 | } \
|
| 734 | (__ret); \
|
| 735 | })
|
| 736 |
|
| 737 | #ifndef __this_cpu_cmpxchg_double
|
| 738 | # ifndef __this_cpu_cmpxchg_double_1
|
| 739 | # define __this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 740 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 741 | # endif
|
| 742 | # ifndef __this_cpu_cmpxchg_double_2
|
| 743 | # define __this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 744 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 745 | # endif
|
| 746 | # ifndef __this_cpu_cmpxchg_double_4
|
| 747 | # define __this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 748 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 749 | # endif
|
| 750 | # ifndef __this_cpu_cmpxchg_double_8
|
| 751 | # define __this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 752 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
| 753 | # endif
|
| 754 | # define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
| 755 | __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
|
| 756 | #endif
|
| 757 |
|
| 758 | #endif
|