Green shading in the line number column means the source is part of the translation unit, red means it is conditionally excluded. Highlighted line numbers link to the translation unit page. Highlighted macros link to the macro page.
1: /* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+ 2: * 3: * Written by David Howells (dhowells@redhat.com). 4: * 5: * Derived from asm-x86/semaphore.h 6: * 7: * 8: * The MSW of the count is the negated number of active writers and waiting 9: * lockers, and the LSW is the total number of active locks 10: * 11: * The lock count is initialized to 0 (no active and no waiting lockers). 12: * 13: * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an 14: * uncontended lock. This can be determined because XADD returns the old value. 15: * Readers increment by 1 and see a positive value when uncontended, negative 16: * if there are writers (and maybe) readers waiting (in which case it goes to 17: * sleep). 18: * 19: * The value of WAITING_BIAS supports up to 32766 waiting processes. This can 20: * be extended to 65534 by manually checking the whole MSW rather than relying 21: * on the S flag. 22: * 23: * The value of ACTIVE_BIAS supports up to 65535 active processes. 24: * 25: * This should be totally fair - if anything is waiting, a process that wants a 26: * lock will go to the back of the queue. When the currently active lock is 27: * released, if there's a writer at the front of the queue, then that and only 28: * that will be woken up; if there's a bunch of consequtive readers at the 29: * front, then they'll all be woken up, but no other readers will be. 30: */ 31: 32: #ifndef _ASM_X86_RWSEM_H 33: #define _ASM_X86_RWSEM_H 34: 35: #ifndef _LINUX_RWSEM_H 36: #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" 37: #endif 38: 39: #ifdef __KERNEL__ 40: #include <asm/asm.h> 41: 42: /* 43: * The bias values and the counter type limits the number of 44: * potential readers/writers to 32767 for 32 bits and 2147483647 45: * for 64 bits. 46: */ 47: 48: #ifdef CONFIG_X86_64 49: # define RWSEM_ACTIVE_MASK 0xffffffffL 50: #else 51: # define RWSEM_ACTIVE_MASK 0x0000ffffL 52: #endif 53: 54: #define RWSEM_UNLOCKED_VALUE 0x00000000L 55: #define RWSEM_ACTIVE_BIAS 0x00000001L 56: #define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) 57: #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS 58: #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) 59: 60: /* 61: * lock for reading 62: */ 63: static inline void __down_read(struct rw_semaphore *sem) 64: { 65: asm volatile("# beginning down_read\n\t" 66: LOCK_PREFIX _ASM_INC "(%1)\n\t" 67: /* adds 0x00000001 */ 68: " jns 1f\n" 69: " call call_rwsem_down_read_failed\n" 70: "1:\n\t" 71: "# ending down_read\n\t" 72: : "+m" (sem->count) 73: : "a" (sem) 74: : "memory", "cc"); 75: } 76: 77: /* 78: * trylock for reading -- returns 1 if successful, 0 if contention 79: */ 80: static inline int __down_read_trylock(struct rw_semaphore *sem) 81: { 82: long result, tmp; 83: asm volatile("# beginning __down_read_trylock\n\t" 84: " mov %0,%1\n\t" 85: "1:\n\t" 86: " mov %1,%2\n\t" 87: " add %3,%2\n\t" 88: " jle 2f\n\t" 89: LOCK_PREFIX " cmpxchg %2,%0\n\t" 90: " jnz 1b\n\t" 91: "2:\n\t" 92: "# ending __down_read_trylock\n\t" 93: : "+m" (sem->count), "=&a" (result), "=&r" (tmp) 94: : "i" (RWSEM_ACTIVE_READ_BIAS) 95: : "memory", "cc"); 96: return result >= 0 ? 1 : 0; 97: } 98: 99: /* 100: * lock for writing 101: */ 102: static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) 103: { 104: long tmp; 105: asm volatile("# beginning down_write\n\t" 106: LOCK_PREFIX " xadd %1,(%2)\n\t" 107: /* adds 0xffff0001, returns the old value */ 108: " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" 109: /* was the active mask 0 before? */ 110: " jz 1f\n" 111: " call call_rwsem_down_write_failed\n" 112: "1:\n" 113: "# ending down_write" 114: : "+m" (sem->count), "=d" (tmp) 115: : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS) 116: : "memory", "cc"); 117: } 118: 119: static inline void __down_write(struct rw_semaphore *sem) 120: { 121: __down_write_nested(sem, 0); 122: } 123: 124: /* 125: * trylock for writing -- returns 1 if successful, 0 if contention 126: */ 127: static inline int __down_write_trylock(struct rw_semaphore *sem) 128: { 129: long result, tmp; 130: asm volatile("# beginning __down_write_trylock\n\t" 131: " mov %0,%1\n\t" 132: "1:\n\t" 133: " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" 134: /* was the active mask 0 before? */ 135: " jnz 2f\n\t" 136: " mov %1,%2\n\t" 137: " add %3,%2\n\t" 138: LOCK_PREFIX " cmpxchg %2,%0\n\t" 139: " jnz 1b\n\t" 140: "2:\n\t" 141: " sete %b1\n\t" 142: " movzbl %b1, %k1\n\t" 143: "# ending __down_write_trylock\n\t" 144: : "+m" (sem->count), "=&a" (result), "=&r" (tmp) 145: : "er" (RWSEM_ACTIVE_WRITE_BIAS) 146: : "memory", "cc"); 147: return result; 148: } 149: 150: /* 151: * unlock after reading 152: */ 153: static inline void __up_read(struct rw_semaphore *sem) 154: { 155: long tmp; 156: asm volatile("# beginning __up_read\n\t" 157: LOCK_PREFIX " xadd %1,(%2)\n\t" 158: /* subtracts 1, returns the old value */ 159: " jns 1f\n\t" 160: " call call_rwsem_wake\n" /* expects old value in %edx */ 161: "1:\n" 162: "# ending __up_read\n" 163: : "+m" (sem->count), "=d" (tmp) 164: : "a" (sem), "1" (-RWSEM_ACTIVE_READ_BIAS) 165: : "memory", "cc"); 166: } 167: 168: /* 169: * unlock after writing 170: */ 171: static inline void __up_write(struct rw_semaphore *sem) 172: { 173: long tmp; 174: asm volatile("# beginning __up_write\n\t" 175: LOCK_PREFIX " xadd %1,(%2)\n\t" 176: /* subtracts 0xffff0001, returns the old value */ 177: " jns 1f\n\t" 178: " call call_rwsem_wake\n" /* expects old value in %edx */ 179: "1:\n\t" 180: "# ending __up_write\n" 181: : "+m" (sem->count), "=d" (tmp) 182: : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS) 183: : "memory", "cc"); 184: } 185: 186: /* 187: * downgrade write lock to read lock 188: */ 189: static inline void __downgrade_write(struct rw_semaphore *sem) 190: { 191: asm volatile("# beginning __downgrade_write\n\t" 192: LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t" 193: /* 194: * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386) 195: * 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64) 196: */ 197: " jns 1f\n\t" 198: " call call_rwsem_downgrade_wake\n" 199: "1:\n\t" 200: "# ending __downgrade_write\n" 201: : "+m" (sem->count) 202: : "a" (sem), "er" (-RWSEM_WAITING_BIAS) 203: : "memory", "cc"); 204: } 205: 206: /* 207: * implement atomic add functionality 208: */ 209: static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) 210: { 211: asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0" 212: : "+m" (sem->count) 213: : "er" (delta)); 214: } 215: 216: /* 217: * implement exchange and add functionality 218: */ 219: static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) 220: { 221: return delta + xadd(&sem->count, delta); 222: } 223: 224: #endif /* __KERNEL__ */ 225: #endif /* _ASM_X86_RWSEM_H */ 226: