Green shading in the line number column means the source is part of the translation unit, red means it is conditionally excluded. Highlighted line numbers link to the translation unit page. Highlighted macros link to the macro page.
1: /* 2: * Sleepable Read-Copy Update mechanism for mutual exclusion 3: * 4: * This program is free software; you can redistribute it and/or modify 5: * it under the terms of the GNU General Public License as published by 6: * the Free Software Foundation; either version 2 of the License, or 7: * (at your option) any later version. 8: * 9: * This program is distributed in the hope that it will be useful, 10: * but WITHOUT ANY WARRANTY; without even the implied warranty of 11: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12: * GNU General Public License for more details. 13: * 14: * You should have received a copy of the GNU General Public License 15: * along with this program; if not, write to the Free Software 16: * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17: * 18: * Copyright (C) IBM Corporation, 2006 19: * Copyright (C) Fujitsu, 2012 20: * 21: * Author: Paul McKenney <paulmck@us.ibm.com> 22: * Lai Jiangshan <laijs@cn.fujitsu.com> 23: * 24: * For detailed explanation of Read-Copy Update mechanism see - 25: * Documentation/RCU/ *.txt 26: * 27: */ 28: 29: #ifndef _LINUX_SRCU_H 30: #define _LINUX_SRCU_H 31: 32: #include <linux/mutex.h> 33: #include <linux/rcupdate.h> 34: #include <linux/workqueue.h> 35: 36: struct srcu_struct_array { 37: unsigned long c[2]; 38: unsigned long seq[2]; 39: }; 40: 41: struct rcu_batch { 42: struct rcu_head *head, **tail; 43: }; 44: 45: #define RCU_BATCH_INIT(name) { NULL, &(name.head) } 46: 47: struct srcu_struct { 48: unsigned completed; 49: struct srcu_struct_array __percpu *per_cpu_ref; 50: spinlock_t queue_lock; /* protect ->batch_queue, ->running */ 51: bool running; 52: /* callbacks just queued */ 53: struct rcu_batch batch_queue; 54: /* callbacks try to do the first check_zero */ 55: struct rcu_batch batch_check0; 56: /* callbacks done with the first check_zero and the flip */ 57: struct rcu_batch batch_check1; 58: struct rcu_batch batch_done; 59: struct delayed_work work; 60: #ifdef CONFIG_DEBUG_LOCK_ALLOC 61: struct lockdep_map dep_map; 62: #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 63: }; 64: 65: #ifdef CONFIG_DEBUG_LOCK_ALLOC 66: 67: int __init_srcu_struct(struct srcu_struct *sp, const char *name, 68: struct lock_class_key *key); 69: 70: #define init_srcu_struct(sp) \ 71: ({ \ 72: static struct lock_class_key __srcu_key; \ 73: \ 74: __init_srcu_struct((sp), #sp, &__srcu_key); \ 75: }) 76: 77: #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, 78: #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 79: 80: int init_srcu_struct(struct srcu_struct *sp); 81: 82: #define __SRCU_DEP_MAP_INIT(srcu_name) 83: #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 84: 85: void process_srcu(struct work_struct *work); 86: 87: #define __SRCU_STRUCT_INIT(name) \ 88: { \ 89: .completed = -300, \ 90: .per_cpu_ref = &name##_srcu_array, \ 91: .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \ 92: .running = false, \ 93: .batch_queue = RCU_BATCH_INIT(name.batch_queue), \ 94: .batch_check0 = RCU_BATCH_INIT(name.batch_check0), \ 95: .batch_check1 = RCU_BATCH_INIT(name.batch_check1), \ 96: .batch_done = RCU_BATCH_INIT(name.batch_done), \ 97: .work = __DELAYED_WORK_INITIALIZER(name.work, process_srcu, 0),\ __SRCU_DEP_MAP_INIT(name) \ 99: } 100: 101: /* 102: * define and init a srcu struct at build time. 103: * dont't call init_srcu_struct() nor cleanup_srcu_struct() on it. 104: */ 105: #define DEFINE_SRCU(name) \ 106: static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ struct srcu_struct name = __SRCU_STRUCT_INIT(name); 108: 109: #define DEFINE_STATIC_SRCU(name) \ 110: static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ static struct srcu_struct name = __SRCU_STRUCT_INIT(name); 112: 113: /** 114: * call_srcu() - Queue a callback for invocation after an SRCU grace period 115: * @sp: srcu_struct in queue the callback 116: * @head: structure to be used for queueing the SRCU callback. 117: * @func: function to be invoked after the SRCU grace period 118: * 119: * The callback function will be invoked some time after a full SRCU 120: * grace period elapses, in other words after all pre-existing SRCU 121: * read-side critical sections have completed. However, the callback 122: * function might well execute concurrently with other SRCU read-side 123: * critical sections that started after call_srcu() was invoked. SRCU 124: * read-side critical sections are delimited by srcu_read_lock() and 125: * srcu_read_unlock(), and may be nested. 126: * 127: * The callback will be invoked from process context, but must nevertheless 128: * be fast and must not block. 129: */ 130: void call_srcu(struct srcu_struct *sp, struct rcu_head *head, 131: void (*func)(struct rcu_head *head)); 132: 133: void cleanup_srcu_struct(struct srcu_struct *sp); 134: int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp); 135: void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); 136: void synchronize_srcu(struct srcu_struct *sp); 137: void synchronize_srcu_expedited(struct srcu_struct *sp); 138: long srcu_batches_completed(struct srcu_struct *sp); 139: void srcu_barrier(struct srcu_struct *sp); 140: 141: #ifdef CONFIG_DEBUG_LOCK_ALLOC 142: 143: /** 144: * srcu_read_lock_held - might we be in SRCU read-side critical section? 145: * 146: * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU 147: * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, 148: * this assumes we are in an SRCU read-side critical section unless it can 149: * prove otherwise. 150: * 151: * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot 152: * and while lockdep is disabled. 153: * 154: * Note that SRCU is based on its own statemachine and it doesn't 155: * relies on normal RCU, it can be called from the CPU which 156: * is in the idle loop from an RCU point of view or offline. 157: */ 158: static inline int srcu_read_lock_held(struct srcu_struct *sp) 159: { 160: if (!debug_lockdep_rcu_enabled()) 161: return 1; 162: return lock_is_held(&sp->dep_map); 163: } 164: 165: #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 166: 167: static inline int srcu_read_lock_held(struct srcu_struct *sp) 168: { 169: return 1; 170: } 171: 172: #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 173: 174: /** 175: * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing 176: * @p: the pointer to fetch and protect for later dereferencing 177: * @sp: pointer to the srcu_struct, which is used to check that we 178: * really are in an SRCU read-side critical section. 179: * @c: condition to check for update-side use 180: * 181: * If PROVE_RCU is enabled, invoking this outside of an RCU read-side 182: * critical section will result in an RCU-lockdep splat, unless @c evaluates 183: * to 1. The @c argument will normally be a logical expression containing 184: * lockdep_is_held() calls. 185: */ 186: #define srcu_dereference_check(p, sp, c) \ 187: __rcu_dereference_check((p), srcu_read_lock_held(sp) || (c), __rcu) 188: 189: /** 190: * srcu_dereference - fetch SRCU-protected pointer for later dereferencing 191: * @p: the pointer to fetch and protect for later dereferencing 192: * @sp: pointer to the srcu_struct, which is used to check that we 193: * really are in an SRCU read-side critical section. 194: * 195: * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU 196: * is enabled, invoking this outside of an RCU read-side critical 197: * section will result in an RCU-lockdep splat. 198: */ 199: #define srcu_dereference(p, sp) srcu_dereference_check((p), (sp), 0) 200: 201: /** 202: * srcu_read_lock - register a new reader for an SRCU-protected structure. 203: * @sp: srcu_struct in which to register the new reader. 204: * 205: * Enter an SRCU read-side critical section. Note that SRCU read-side 206: * critical sections may be nested. However, it is illegal to 207: * call anything that waits on an SRCU grace period for the same 208: * srcu_struct, whether directly or indirectly. Please note that 209: * one way to indirectly wait on an SRCU grace period is to acquire 210: * a mutex that is held elsewhere while calling synchronize_srcu() or 211: * synchronize_srcu_expedited(). 212: * 213: * Note that srcu_read_lock() and the matching srcu_read_unlock() must 214: * occur in the same context, for example, it is illegal to invoke 215: * srcu_read_unlock() in an irq handler if the matching srcu_read_lock() 216: * was invoked in process context. 217: */ 218: static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) 219: { 220: int retval = __srcu_read_lock(sp); 221: 222: rcu_lock_acquire(&(sp)->dep_map); 223: return retval; 224: } 225: 226: /** 227: * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. 228: * @sp: srcu_struct in which to unregister the old reader. 229: * @idx: return value from corresponding srcu_read_lock(). 230: * 231: * Exit an SRCU read-side critical section. 232: */ 233: static inline void srcu_read_unlock(struct srcu_struct *sp, int idx) 234: __releases(sp) 235: { 236: rcu_lock_release(&(sp)->dep_map); 237: __srcu_read_unlock(sp, idx); 238: } 239: 240: /** 241: * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock 242: * 243: * Converts the preceding srcu_read_unlock into a two-way memory barrier. 244: * 245: * Call this after srcu_read_unlock, to guarantee that all memory operations 246: * that occur after smp_mb__after_srcu_read_unlock will appear to happen after 247: * the preceding srcu_read_unlock. 248: */ 249: static inline void smp_mb__after_srcu_read_unlock(void) 250: { 251: /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */ 252: } 253: 254: #endif 255: