Green shading in the line number column means the source is part of the translation unit, red means it is conditionally excluded. Highlighted line numbers link to the translation unit page. Highlighted macros link to the macro page.
1: #ifndef _ASM_X86_BARRIER_H 2: #define _ASM_X86_BARRIER_H 3: 4: #include <asm/alternative.h> 5: #include <asm/nops.h> 6: 7: /* 8: * Force strict CPU ordering. 9: * And yes, this is required on UP too when we're talking 10: * to devices. 11: */ 12: 13: #ifdef CONFIG_X86_32 14: /* 15: * Some non-Intel clones support out of order store. wmb() ceases to be a 16: * nop for these. 17: */ 18: #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2) 19: #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2) 20: #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM) 21: #else 22: #define mb() asm volatile("mfence":::"memory") 23: #define rmb() asm volatile("lfence":::"memory") 24: #define wmb() asm volatile("sfence" ::: "memory") 25: #endif 26: 27: /** 28: * read_barrier_depends - Flush all pending reads that subsequents reads 29: * depend on. 30: * 31: * No data-dependent reads from memory-like regions are ever reordered 32: * over this barrier. All reads preceding this primitive are guaranteed 33: * to access memory (but not necessarily other CPUs' caches) before any 34: * reads following this primitive that depend on the data return by 35: * any of the preceding reads. This primitive is much lighter weight than 36: * rmb() on most CPUs, and is never heavier weight than is 37: * rmb(). 38: * 39: * These ordering constraints are respected by both the local CPU 40: * and the compiler. 41: * 42: * Ordering is not guaranteed by anything other than these primitives, 43: * not even by data dependencies. See the documentation for 44: * memory_barrier() for examples and URLs to more information. 45: * 46: * For example, the following code would force ordering (the initial 47: * value of "a" is zero, "b" is one, and "p" is "&a"): 48: * 49: * <programlisting> 50: * CPU 0 CPU 1 51: * 52: * b = 2; 53: * memory_barrier(); 54: * p = &b; q = p; 55: * read_barrier_depends(); 56: * d = *q; 57: * </programlisting> 58: * 59: * because the read of "*q" depends on the read of "p" and these 60: * two reads are separated by a read_barrier_depends(). However, 61: * the following code, with the same initial values for "a" and "b": 62: * 63: * <programlisting> 64: * CPU 0 CPU 1 65: * 66: * a = 2; 67: * memory_barrier(); 68: * b = 3; y = b; 69: * read_barrier_depends(); 70: * x = a; 71: * </programlisting> 72: * 73: * does not enforce ordering, since there is no data dependency between 74: * the read of "a" and the read of "b". Therefore, on some CPUs, such 75: * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb() 76: * in cases like this where there are no data dependencies. 77: **/ 78: 79: #define read_barrier_depends() do { } while (0) 80: 81: #ifdef CONFIG_SMP 82: #define smp_mb() mb() 83: #ifdef CONFIG_X86_PPRO_FENCE 84: # define smp_rmb() rmb() 85: #else 86: # define smp_rmb() barrier() 87: #endif 88: #ifdef CONFIG_X86_OOSTORE 89: # define smp_wmb() wmb() 90: #else 91: # define smp_wmb() barrier() 92: #endif 93: #define smp_read_barrier_depends() read_barrier_depends() 94: #define set_mb(var, value) do { (void)xchg(&var, value); } while (0) 95: #else 96: #define smp_mb() barrier() 97: #define smp_rmb() barrier() 98: #define smp_wmb() barrier() 99: #define smp_read_barrier_depends() do { } while (0) 100: #define set_mb(var, value) do { var = value; barrier(); } while (0) 101: #endif 102: 103: /* 104: * Stop RDTSC speculation. This is needed when you need to use RDTSC 105: * (or get_cycles or vread that possibly accesses the TSC) in a defined 106: * code region. 107: * 108: * (Could use an alternative three way for this if there was one.) 109: */ 110: static __always_inline void rdtsc_barrier(void) 111: { 112: alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC); 113: alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC); 114: } 115: 116: #endif /* _ASM_X86_BARRIER_H */ 117: