Green shading in the line number column means the source is part of the translation unit, red means it is conditionally excluded. Highlighted line numbers link to the translation unit page. Highlighted macros link to the macro page.
1: #ifndef __LINUX_PERCPU_H 2: #define __LINUX_PERCPU_H 3: 4: #include <linux/preempt.h> 5: #include <linux/smp.h> 6: #include <linux/cpumask.h> 7: #include <linux/pfn.h> 8: #include <linux/init.h> 9: 10: #include <asm/percpu.h> 11: 12: /* enough to cover all DEFINE_PER_CPUs in modules */ 13: #ifdef CONFIG_MODULES 14: #define PERCPU_MODULE_RESERVE (8 << 10) 15: #else 16: #define PERCPU_MODULE_RESERVE 0 17: #endif 18: 19: #ifndef PERCPU_ENOUGH_ROOM 20: #define PERCPU_ENOUGH_ROOM \ 21: (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \ 22: PERCPU_MODULE_RESERVE) 23: #endif 24: 25: /* 26: * Must be an lvalue. Since @var must be a simple identifier, 27: * we force a syntax error here if it isn't. 28: */ 29: #define get_cpu_var(var) (*({ \ 30: preempt_disable(); \ 31: &__get_cpu_var(var); })) 32: 33: /* 34: * The weird & is necessary because sparse considers (void)(var) to be 35: * a direct dereference of percpu variable (var). 36: */ 37: #define put_cpu_var(var) do { \ 38: (void)&(var); \ 39: preempt_enable(); \ 40: } while (0) 41: 42: #define get_cpu_ptr(var) ({ \ 43: preempt_disable(); \ 44: this_cpu_ptr(var); }) 45: 46: #define put_cpu_ptr(var) do { \ 47: (void)(var); \ 48: preempt_enable(); \ 49: } while (0) 50: 51: /* minimum unit size, also is the maximum supported allocation size */ 52: #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) 53: 54: /* 55: * Percpu allocator can serve percpu allocations before slab is 56: * initialized which allows slab to depend on the percpu allocator. 57: * The following two parameters decide how much resource to 58: * preallocate for this. Keep PERCPU_DYNAMIC_RESERVE equal to or 59: * larger than PERCPU_DYNAMIC_EARLY_SIZE. 60: */ 61: #define PERCPU_DYNAMIC_EARLY_SLOTS 128 62: #define PERCPU_DYNAMIC_EARLY_SIZE (12 << 10) 63: 64: /* 65: * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy 66: * back on the first chunk for dynamic percpu allocation if arch is 67: * manually allocating and mapping it for faster access (as a part of 68: * large page mapping for example). 69: * 70: * The following values give between one and two pages of free space 71: * after typical minimal boot (2-way SMP, single disk and NIC) with 72: * both defconfig and a distro config on x86_64 and 32. More 73: * intelligent way to determine this would be nice. 74: */ 75: #if BITS_PER_LONG > 32 76: #define PERCPU_DYNAMIC_RESERVE (20 << 10) 77: #else 78: #define PERCPU_DYNAMIC_RESERVE (12 << 10) 79: #endif 80: 81: extern void *pcpu_base_addr; 82: extern const unsigned long *pcpu_unit_offsets; 83: 84: struct pcpu_group_info { 85: int nr_units; /* aligned # of units */ 86: unsigned long base_offset; /* base address offset */ 87: unsigned int *cpu_map; /* unit->cpu map, empty 88: * entries contain NR_CPUS */ 89: }; 90: 91: struct pcpu_alloc_info { 92: size_t static_size; 93: size_t reserved_size; 94: size_t dyn_size; 95: size_t unit_size; 96: size_t atom_size; 97: size_t alloc_size; 98: size_t __ai_size; /* internal, don't use */ 99: int nr_groups; /* 0 if grouping unnecessary */ 100: struct pcpu_group_info groups[]; 101: }; 102: 103: enum pcpu_fc { 104: PCPU_FC_AUTO, 105: PCPU_FC_EMBED, 106: PCPU_FC_PAGE, 107: 108: PCPU_FC_NR, 109: }; 110: extern const char * const pcpu_fc_names[PCPU_FC_NR]; 111: 112: extern enum pcpu_fc pcpu_chosen_fc; 113: 114: typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, 115: size_t align); 116: typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size); 117: typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); 118: typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); 119: 120: extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, 121: int nr_units); 122: extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai); 123: 124: extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, 125: void *base_addr); 126: 127: #ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK 128: extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 129: size_t atom_size, 130: pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 131: pcpu_fc_alloc_fn_t alloc_fn, 132: pcpu_fc_free_fn_t free_fn); 133: #endif 134: 135: #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 136: extern int __init pcpu_page_first_chunk(size_t reserved_size, 137: pcpu_fc_alloc_fn_t alloc_fn, 138: pcpu_fc_free_fn_t free_fn, 139: pcpu_fc_populate_pte_fn_t populate_pte_fn); 140: #endif 141: 142: /* 143: * Use this to get to a cpu's version of the per-cpu object 144: * dynamically allocated. Non-atomic access to the current CPU's 145: * version should probably be combined with get_cpu()/put_cpu(). 146: */ 147: #ifdef CONFIG_SMP 148: #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) 149: #else 150: #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) 151: #endif 152: 153: extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); 154: extern bool is_kernel_percpu_address(unsigned long addr); 155: 156: #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) 157: extern void __init setup_per_cpu_areas(void); 158: #endif 159: extern void __init percpu_init_late(void); 160: 161: extern void __percpu *__alloc_percpu(size_t size, size_t align); 162: extern void free_percpu(void __percpu *__pdata); 163: extern phys_addr_t per_cpu_ptr_to_phys(void *addr); 164: 165: #define alloc_percpu(type) \ 166: (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type)) 167: 168: /* 169: * Branching function to split up a function into a set of functions that 170: * are called for different scalar sizes of the objects handled. 171: */ 172: 173: extern void __bad_size_call_parameter(void); 174: 175: #define __pcpu_size_call_return(stem, variable) \ 176: ({ typeof(variable) pscr_ret__; \ 177: __verify_pcpu_ptr(&(variable)); \ 178: switch(sizeof(variable)) { \ 179: case 1: pscr_ret__ = stem##1(variable);break; \ 180: case 2: pscr_ret__ = stem##2(variable);break; \ 181: case 4: pscr_ret__ = stem##4(variable);break; \ 182: case 8: pscr_ret__ = stem##8(variable);break; \ 183: default: \ 184: __bad_size_call_parameter();break; \ 185: } \ 186: pscr_ret__; \ 187: }) 188: 189: #define __pcpu_size_call_return2(stem, variable, ...) \ 190: ({ \ 191: typeof(variable) pscr2_ret__; \ 192: __verify_pcpu_ptr(&(variable)); \ 193: switch(sizeof(variable)) { \ 194: case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ 195: case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ 196: case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ 197: case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ 198: default: \ 199: __bad_size_call_parameter(); break; \ 200: } \ 201: pscr2_ret__; \ 202: }) 203: 204: /* 205: * Special handling for cmpxchg_double. cmpxchg_double is passed two 206: * percpu variables. The first has to be aligned to a double word 207: * boundary and the second has to follow directly thereafter. 208: * We enforce this on all architectures even if they don't support 209: * a double cmpxchg instruction, since it's a cheap requirement, and it 210: * avoids breaking the requirement for architectures with the instruction. 211: */ 212: #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \ 213: ({ \ 214: bool pdcrb_ret__; \ 215: __verify_pcpu_ptr(&pcp1); \ 216: BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ 217: VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \ 218: VM_BUG_ON((unsigned long)(&pcp2) != \ 219: (unsigned long)(&pcp1) + sizeof(pcp1)); \ 220: switch(sizeof(pcp1)) { \ 221: case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \ 222: case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \ 223: case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \ 224: case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \ 225: default: \ 226: __bad_size_call_parameter(); break; \ 227: } \ 228: pdcrb_ret__; \ 229: }) 230: 231: #define __pcpu_size_call(stem, variable, ...) \ 232: do { \ 233: __verify_pcpu_ptr(&(variable)); \ 234: switch(sizeof(variable)) { \ 235: case 1: stem##1(variable, __VA_ARGS__);break; \ 236: case 2: stem##2(variable, __VA_ARGS__);break; \ 237: case 4: stem##4(variable, __VA_ARGS__);break; \ 238: case 8: stem##8(variable, __VA_ARGS__);break; \ 239: default: \ 240: __bad_size_call_parameter();break; \ 241: } \ 242: } while (0) 243: 244: /* 245: * Optimized manipulation for memory allocated through the per cpu 246: * allocator or for addresses of per cpu variables. 247: * 248: * These operation guarantee exclusivity of access for other operations 249: * on the *same* processor. The assumption is that per cpu data is only 250: * accessed by a single processor instance (the current one). 251: * 252: * The first group is used for accesses that must be done in a 253: * preemption safe way since we know that the context is not preempt 254: * safe. Interrupts may occur. If the interrupt modifies the variable 255: * too then RMW actions will not be reliable. 256: * 257: * The arch code can provide optimized functions in two ways: 258: * 259: * 1. Override the function completely. F.e. define this_cpu_add(). 260: * The arch must then ensure that the various scalar format passed 261: * are handled correctly. 262: * 263: * 2. Provide functions for certain scalar sizes. F.e. provide 264: * this_cpu_add_2() to provide per cpu atomic operations for 2 byte 265: * sized RMW actions. If arch code does not provide operations for 266: * a scalar size then the fallback in the generic code will be 267: * used. 268: */ 269: 270: #define _this_cpu_generic_read(pcp) \ 271: ({ typeof(pcp) ret__; \ 272: preempt_disable(); \ 273: ret__ = *this_cpu_ptr(&(pcp)); \ 274: preempt_enable(); \ 275: ret__; \ 276: }) 277: 278: #ifndef this_cpu_read 279: # ifndef this_cpu_read_1 280: # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp) 281: # endif 282: # ifndef this_cpu_read_2 283: # define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp) 284: # endif 285: # ifndef this_cpu_read_4 286: # define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp) 287: # endif 288: # ifndef this_cpu_read_8 289: # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) 290: # endif 291: # define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp)) 292: #endif 293: 294: #define _this_cpu_generic_to_op(pcp, val, op) \ 295: do { \ 296: unsigned long flags; \ 297: raw_local_irq_save(flags); \ 298: *__this_cpu_ptr(&(pcp)) op val; \ 299: raw_local_irq_restore(flags); \ 300: } while (0) 301: 302: #ifndef this_cpu_write 303: # ifndef this_cpu_write_1 304: # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 305: # endif 306: # ifndef this_cpu_write_2 307: # define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 308: # endif 309: # ifndef this_cpu_write_4 310: # define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 311: # endif 312: # ifndef this_cpu_write_8 313: # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 314: # endif 315: # define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val)) 316: #endif 317: 318: #ifndef this_cpu_add 319: # ifndef this_cpu_add_1 320: # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 321: # endif 322: # ifndef this_cpu_add_2 323: # define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 324: # endif 325: # ifndef this_cpu_add_4 326: # define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 327: # endif 328: # ifndef this_cpu_add_8 329: # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 330: # endif 331: # define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val)) 332: #endif 333: 334: #ifndef this_cpu_sub 335: # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val)) 336: #endif 337: 338: #ifndef this_cpu_inc 339: # define this_cpu_inc(pcp) this_cpu_add((pcp), 1) 340: #endif 341: 342: #ifndef this_cpu_dec 343: # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1) 344: #endif 345: 346: #ifndef this_cpu_and 347: # ifndef this_cpu_and_1 348: # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 349: # endif 350: # ifndef this_cpu_and_2 351: # define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 352: # endif 353: # ifndef this_cpu_and_4 354: # define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 355: # endif 356: # ifndef this_cpu_and_8 357: # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 358: # endif 359: # define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val)) 360: #endif 361: 362: #ifndef this_cpu_or 363: # ifndef this_cpu_or_1 364: # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 365: # endif 366: # ifndef this_cpu_or_2 367: # define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 368: # endif 369: # ifndef this_cpu_or_4 370: # define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 371: # endif 372: # ifndef this_cpu_or_8 373: # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 374: # endif 375: # define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) 376: #endif 377: 378: #define _this_cpu_generic_add_return(pcp, val) \ 379: ({ \ 380: typeof(pcp) ret__; \ 381: unsigned long flags; \ 382: raw_local_irq_save(flags); \ 383: __this_cpu_add(pcp, val); \ 384: ret__ = __this_cpu_read(pcp); \ 385: raw_local_irq_restore(flags); \ 386: ret__; \ 387: }) 388: 389: #ifndef this_cpu_add_return 390: # ifndef this_cpu_add_return_1 391: # define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val) 392: # endif 393: # ifndef this_cpu_add_return_2 394: # define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val) 395: # endif 396: # ifndef this_cpu_add_return_4 397: # define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val) 398: # endif 399: # ifndef this_cpu_add_return_8 400: # define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val) 401: # endif 402: # define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) 403: #endif 404: 405: #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val)) 406: #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) 407: #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) 408: 409: #define _this_cpu_generic_xchg(pcp, nval) \ 410: ({ typeof(pcp) ret__; \ 411: unsigned long flags; \ 412: raw_local_irq_save(flags); \ 413: ret__ = __this_cpu_read(pcp); \ 414: __this_cpu_write(pcp, nval); \ 415: raw_local_irq_restore(flags); \ 416: ret__; \ 417: }) 418: 419: #ifndef this_cpu_xchg 420: # ifndef this_cpu_xchg_1 421: # define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval) 422: # endif 423: # ifndef this_cpu_xchg_2 424: # define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval) 425: # endif 426: # ifndef this_cpu_xchg_4 427: # define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval) 428: # endif 429: # ifndef this_cpu_xchg_8 430: # define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval) 431: # endif 432: # define this_cpu_xchg(pcp, nval) \ 433: __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval) 434: #endif 435: 436: #define _this_cpu_generic_cmpxchg(pcp, oval, nval) \ 437: ({ \ 438: typeof(pcp) ret__; \ 439: unsigned long flags; \ 440: raw_local_irq_save(flags); \ 441: ret__ = __this_cpu_read(pcp); \ 442: if (ret__ == (oval)) \ 443: __this_cpu_write(pcp, nval); \ 444: raw_local_irq_restore(flags); \ 445: ret__; \ 446: }) 447: 448: #ifndef this_cpu_cmpxchg 449: # ifndef this_cpu_cmpxchg_1 450: # define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) 451: # endif 452: # ifndef this_cpu_cmpxchg_2 453: # define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) 454: # endif 455: # ifndef this_cpu_cmpxchg_4 456: # define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) 457: # endif 458: # ifndef this_cpu_cmpxchg_8 459: # define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) 460: # endif 461: # define this_cpu_cmpxchg(pcp, oval, nval) \ 462: __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) 463: #endif 464: 465: /* 466: * cmpxchg_double replaces two adjacent scalars at once. The first 467: * two parameters are per cpu variables which have to be of the same 468: * size. A truth value is returned to indicate success or failure 469: * (since a double register result is difficult to handle). There is 470: * very limited hardware support for these operations, so only certain 471: * sizes may work. 472: */ 473: #define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 474: ({ \ 475: int ret__; \ 476: unsigned long flags; \ 477: raw_local_irq_save(flags); \ 478: ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \ 479: oval1, oval2, nval1, nval2); \ 480: raw_local_irq_restore(flags); \ 481: ret__; \ 482: }) 483: 484: #ifndef this_cpu_cmpxchg_double 485: # ifndef this_cpu_cmpxchg_double_1 486: # define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 487: _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 488: # endif 489: # ifndef this_cpu_cmpxchg_double_2 490: # define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 491: _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 492: # endif 493: # ifndef this_cpu_cmpxchg_double_4 494: # define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 495: _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 496: # endif 497: # ifndef this_cpu_cmpxchg_double_8 498: # define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 499: _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 500: # endif 501: # define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 502: __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) 503: #endif 504: 505: /* 506: * Generic percpu operations for context that are safe from preemption/interrupts. 507: * Either we do not care about races or the caller has the 508: * responsibility of handling preemption/interrupt issues. Arch code can still 509: * override these instructions since the arch per cpu code may be more 510: * efficient and may actually get race freeness for free (that is the 511: * case for x86 for example). 512: * 513: * If there is no other protection through preempt disable and/or 514: * disabling interupts then one of these RMW operations can show unexpected 515: * behavior because the execution thread was rescheduled on another processor 516: * or an interrupt occurred and the same percpu variable was modified from 517: * the interrupt context. 518: */ 519: #ifndef __this_cpu_read 520: # ifndef __this_cpu_read_1 521: # define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp))) 522: # endif 523: # ifndef __this_cpu_read_2 524: # define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp))) 525: # endif 526: # ifndef __this_cpu_read_4 527: # define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp))) 528: # endif 529: # ifndef __this_cpu_read_8 530: # define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp))) 531: # endif 532: # define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp)) 533: #endif 534: 535: #define __this_cpu_generic_to_op(pcp, val, op) \ 536: do { \ 537: *__this_cpu_ptr(&(pcp)) op val; \ 538: } while (0) 539: 540: #ifndef __this_cpu_write 541: # ifndef __this_cpu_write_1 542: # define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 543: # endif 544: # ifndef __this_cpu_write_2 545: # define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 546: # endif 547: # ifndef __this_cpu_write_4 548: # define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 549: # endif 550: # ifndef __this_cpu_write_8 551: # define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 552: # endif 553: # define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val)) 554: #endif 555: 556: #ifndef __this_cpu_add 557: # ifndef __this_cpu_add_1 558: # define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 559: # endif 560: # ifndef __this_cpu_add_2 561: # define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 562: # endif 563: # ifndef __this_cpu_add_4 564: # define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 565: # endif 566: # ifndef __this_cpu_add_8 567: # define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 568: # endif 569: # define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val)) 570: #endif 571: 572: #ifndef __this_cpu_sub 573: # define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val)) 574: #endif 575: 576: #ifndef __this_cpu_inc 577: # define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1) 578: #endif 579: 580: #ifndef __this_cpu_dec 581: # define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1) 582: #endif 583: 584: #ifndef __this_cpu_and 585: # ifndef __this_cpu_and_1 586: # define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 587: # endif 588: # ifndef __this_cpu_and_2 589: # define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 590: # endif 591: # ifndef __this_cpu_and_4 592: # define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 593: # endif 594: # ifndef __this_cpu_and_8 595: # define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 596: # endif 597: # define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val)) 598: #endif 599: 600: #ifndef __this_cpu_or 601: # ifndef __this_cpu_or_1 602: # define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 603: # endif 604: # ifndef __this_cpu_or_2 605: # define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 606: # endif 607: # ifndef __this_cpu_or_4 608: # define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 609: # endif 610: # ifndef __this_cpu_or_8 611: # define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 612: # endif 613: # define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val)) 614: #endif 615: 616: #define __this_cpu_generic_add_return(pcp, val) \ 617: ({ \ 618: __this_cpu_add(pcp, val); \ 619: __this_cpu_read(pcp); \ 620: }) 621: 622: #ifndef __this_cpu_add_return 623: # ifndef __this_cpu_add_return_1 624: # define __this_cpu_add_return_1(pcp, val) __this_cpu_generic_add_return(pcp, val) 625: # endif 626: # ifndef __this_cpu_add_return_2 627: # define __this_cpu_add_return_2(pcp, val) __this_cpu_generic_add_return(pcp, val) 628: # endif 629: # ifndef __this_cpu_add_return_4 630: # define __this_cpu_add_return_4(pcp, val) __this_cpu_generic_add_return(pcp, val) 631: # endif 632: # ifndef __this_cpu_add_return_8 633: # define __this_cpu_add_return_8(pcp, val) __this_cpu_generic_add_return(pcp, val) 634: # endif 635: # define __this_cpu_add_return(pcp, val) \ 636: __pcpu_size_call_return2(__this_cpu_add_return_, pcp, val) 637: #endif 638: 639: #define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val)) 640: #define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1) 641: #define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1) 642: 643: #define __this_cpu_generic_xchg(pcp, nval) \ 644: ({ typeof(pcp) ret__; \ 645: ret__ = __this_cpu_read(pcp); \ 646: __this_cpu_write(pcp, nval); \ 647: ret__; \ 648: }) 649: 650: #ifndef __this_cpu_xchg 651: # ifndef __this_cpu_xchg_1 652: # define __this_cpu_xchg_1(pcp, nval) __this_cpu_generic_xchg(pcp, nval) 653: # endif 654: # ifndef __this_cpu_xchg_2 655: # define __this_cpu_xchg_2(pcp, nval) __this_cpu_generic_xchg(pcp, nval) 656: # endif 657: # ifndef __this_cpu_xchg_4 658: # define __this_cpu_xchg_4(pcp, nval) __this_cpu_generic_xchg(pcp, nval) 659: # endif 660: # ifndef __this_cpu_xchg_8 661: # define __this_cpu_xchg_8(pcp, nval) __this_cpu_generic_xchg(pcp, nval) 662: # endif 663: # define __this_cpu_xchg(pcp, nval) \ 664: __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval) 665: #endif 666: 667: #define __this_cpu_generic_cmpxchg(pcp, oval, nval) \ 668: ({ \ 669: typeof(pcp) ret__; \ 670: ret__ = __this_cpu_read(pcp); \ 671: if (ret__ == (oval)) \ 672: __this_cpu_write(pcp, nval); \ 673: ret__; \ 674: }) 675: 676: #ifndef __this_cpu_cmpxchg 677: # ifndef __this_cpu_cmpxchg_1 678: # define __this_cpu_cmpxchg_1(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) 679: # endif 680: # ifndef __this_cpu_cmpxchg_2 681: # define __this_cpu_cmpxchg_2(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) 682: # endif 683: # ifndef __this_cpu_cmpxchg_4 684: # define __this_cpu_cmpxchg_4(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) 685: # endif 686: # ifndef __this_cpu_cmpxchg_8 687: # define __this_cpu_cmpxchg_8(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) 688: # endif 689: # define __this_cpu_cmpxchg(pcp, oval, nval) \ 690: __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval) 691: #endif 692: 693: #define __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 694: ({ \ 695: int __ret = 0; \ 696: if (__this_cpu_read(pcp1) == (oval1) && \ 697: __this_cpu_read(pcp2) == (oval2)) { \ 698: __this_cpu_write(pcp1, (nval1)); \ 699: __this_cpu_write(pcp2, (nval2)); \ 700: __ret = 1; \ 701: } \ 702: (__ret); \ 703: }) 704: 705: #ifndef __this_cpu_cmpxchg_double 706: # ifndef __this_cpu_cmpxchg_double_1 707: # define __this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 708: __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 709: # endif 710: # ifndef __this_cpu_cmpxchg_double_2 711: # define __this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 712: __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 713: # endif 714: # ifndef __this_cpu_cmpxchg_double_4 715: # define __this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 716: __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 717: # endif 718: # ifndef __this_cpu_cmpxchg_double_8 719: # define __this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 720: __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 721: # endif 722: # define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 723: __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) 724: #endif 725: 726: #endif /* __LINUX_PERCPU_H */ 727: