Green shading in the line number column means the source is part of the translation unit, red means it is conditionally excluded. Highlighted line numbers link to the translation unit page. Highlighted macros link to the macro page.
1: #ifndef _LINUX_MM_TYPES_H 2: #define _LINUX_MM_TYPES_H 3: 4: #include <linux/auxvec.h> 5: #include <linux/types.h> 6: #include <linux/threads.h> 7: #include <linux/list.h> 8: #include <linux/spinlock.h> 9: #include <linux/rbtree.h> 10: #include <linux/rwsem.h> 11: #include <linux/completion.h> 12: #include <linux/cpumask.h> 13: #include <linux/page-debug-flags.h> 14: #include <linux/uprobes.h> 15: #include <linux/page-flags-layout.h> 16: #include <asm/page.h> 17: #include <asm/mmu.h> 18: 19: #ifndef AT_VECTOR_SIZE_ARCH 20: #define AT_VECTOR_SIZE_ARCH 0 21: #endif 22: #define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1)) 23: 24: struct address_space; 25: 26: #define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) 27: #define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \ 28: IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) 29: #define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) 30: 31: /* 32: * Each physical page in the system has a struct page associated with 33: * it to keep track of whatever it is we are using the page for at the 34: * moment. Note that we have no way to track which tasks are using 35: * a page, though if it is a pagecache page, rmap structures can tell us 36: * who is mapping it. 37: * 38: * The objects in struct page are organized in double word blocks in 39: * order to allows us to use atomic double word operations on portions 40: * of struct page. That is currently only used by slub but the arrangement 41: * allows the use of atomic double word operations on the flags/mapping 42: * and lru list pointers also. 43: */ 44: struct page { 45: /* First double word block */ 46: unsigned long flags; /* Atomic flags, some possibly 47: * updated asynchronously */ 48: union { 49: struct address_space *mapping; /* If low bit clear, points to 50: * inode address_space, or NULL. 51: * If page mapped as anonymous 52: * memory, low bit is set, and 53: * it points to anon_vma object: 54: * see PAGE_MAPPING_ANON below. 55: */ 56: void *s_mem; /* slab first object */ 57: }; 58: 59: /* Second double word */ 60: struct { 61: union { 62: pgoff_t index; /* Our offset within mapping. */ 63: void *freelist; /* sl[aou]b first free object */ 64: bool pfmemalloc; /* If set by the page allocator, 65: * ALLOC_NO_WATERMARKS was set 66: * and the low watermark was not 67: * met implying that the system 68: * is under some pressure. The 69: * caller should try ensure 70: * this page is only used to 71: * free other pages. 72: */ 73: }; 74: 75: union { 76: #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ 77: defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 78: /* Used for cmpxchg_double in slub */ 79: unsigned long counters; 80: #else 81: /* 82: * Keep _count separate from slub cmpxchg_double data. 83: * As the rest of the double word is protected by 84: * slab_lock but _count is not. 85: */ 86: unsigned counters; 87: #endif 88: 89: struct { 90: 91: union { 92: /* 93: * Count of ptes mapped in 94: * mms, to show when page is 95: * mapped & limit reverse map 96: * searches. 97: * 98: * Used also for tail pages 99: * refcounting instead of 100: * _count. Tail pages cannot 101: * be mapped and keeping the 102: * tail page _count zero at 103: * all times guarantees 104: * get_page_unless_zero() will 105: * never succeed on tail 106: * pages. 107: */ 108: atomic_t _mapcount; 109: 110: struct { /* SLUB */ 111: unsigned inuse:16; 112: unsigned objects:15; 113: unsigned frozen:1; 114: }; 115: int units; /* SLOB */ 116: }; 117: atomic_t _count; /* Usage count, see below. */ 118: }; 119: unsigned int active; /* SLAB */ 120: }; 121: }; 122: 123: /* Third double word block */ 124: union { 125: struct list_head lru; /* Pageout list, eg. active_list 126: * protected by zone->lru_lock ! 127: */ 128: struct { /* slub per cpu partial pages */ 129: struct page *next; /* Next partial slab */ 130: #ifdef CONFIG_64BIT 131: int pages; /* Nr of partial slabs left */ 132: int pobjects; /* Approximate # of objects */ 133: #else 134: short int pages; 135: short int pobjects; 136: #endif 137: }; 138: 139: struct list_head list; /* slobs list of pages */ 140: struct slab *slab_page; /* slab fields */ 141: struct rcu_head rcu_head; /* Used by SLAB 142: * when destroying via RCU 143: */ 144: #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS 145: pgtable_t pmd_huge_pte; /* protected by page->ptl */ 146: #endif 147: }; 148: 149: /* Remainder is not double word aligned */ 150: union { 151: unsigned long private; /* Mapping-private opaque data: 152: * usually used for buffer_heads 153: * if PagePrivate set; used for 154: * swp_entry_t if PageSwapCache; 155: * indicates order in the buddy 156: * system if PG_buddy is set. 157: */ 158: #if USE_SPLIT_PTE_PTLOCKS 159: #if ALLOC_SPLIT_PTLOCKS 160: spinlock_t *ptl; 161: #else 162: spinlock_t ptl; 163: #endif 164: #endif 165: struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */ 166: struct page *first_page; /* Compound tail pages */ 167: }; 168: 169: /* 170: * On machines where all RAM is mapped into kernel address space, 171: * we can simply calculate the virtual address. On machines with 172: * highmem some memory is mapped into kernel virtual memory 173: * dynamically, so we need a place to store that address. 174: * Note that this field could be 16 bits on x86 ... ;) 175: * 176: * Architectures with slow multiplication can define 177: * WANT_PAGE_VIRTUAL in asm/page.h 178: */ 179: #if defined(WANT_PAGE_VIRTUAL) 180: void *virtual; /* Kernel virtual address (NULL if 181: not kmapped, ie. highmem) */ 182: #endif /* WANT_PAGE_VIRTUAL */ 183: #ifdef CONFIG_WANT_PAGE_DEBUG_FLAGS 184: unsigned long debug_flags; /* Use atomic bitops on this */ 185: #endif 186: 187: #ifdef CONFIG_KMEMCHECK 188: /* 189: * kmemcheck wants to track the status of each byte in a page; this 190: * is a pointer to such a status block. NULL if not tracked. 191: */ 192: void *shadow; 193: #endif 194: 195: #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 196: int _last_cpupid; 197: #endif 198: } 199: /* 200: * The struct page can be forced to be double word aligned so that atomic ops 201: * on double words work. The SLUB allocator can make use of such a feature. 202: */ 203: #ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE 204: __aligned(2 * sizeof(unsigned long)) 205: #endif 206: ; 207: 208: struct page_frag { 209: struct page *page; 210: #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) 211: __u32 offset; 212: __u32 size; 213: #else 214: __u16 offset; 215: __u16 size; 216: #endif 217: }; 218: 219: typedef unsigned long __nocast vm_flags_t; 220: 221: /* 222: * A region containing a mapping of a non-memory backed file under NOMMU 223: * conditions. These are held in a global tree and are pinned by the VMAs that 224: * map parts of them. 225: */ 226: struct vm_region { 227: struct rb_node vm_rb; /* link in global region tree */ 228: vm_flags_t vm_flags; /* VMA vm_flags */ 229: unsigned long vm_start; /* start address of region */ 230: unsigned long vm_end; /* region initialised to here */ 231: unsigned long vm_top; /* region allocated to here */ 232: unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ 233: struct file *vm_file; /* the backing file or NULL */ 234: 235: int vm_usage; /* region usage count (access under nommu_region_sem) */ 236: bool vm_icache_flushed : 1; /* true if the icache has been flushed for 237: * this region */ 238: }; 239: 240: /* 241: * This struct defines a memory VMM memory area. There is one of these 242: * per VM-area/task. A VM area is any part of the process virtual memory 243: * space that has a special rule for the page-fault handlers (ie a shared 244: * library, the executable area etc). 245: */ 246: struct vm_area_struct { 247: /* The first cache line has the info for VMA tree walking. */ 248: 249: unsigned long vm_start; /* Our start address within vm_mm. */ 250: unsigned long vm_end; /* The first byte after our end address 251: within vm_mm. */ 252: 253: /* linked list of VM areas per task, sorted by address */ 254: struct vm_area_struct *vm_next, *vm_prev; 255: 256: struct rb_node vm_rb; 257: 258: /* 259: * Largest free memory gap in bytes to the left of this VMA. 260: * Either between this VMA and vma->vm_prev, or between one of the 261: * VMAs below us in the VMA rbtree and its ->vm_prev. This helps 262: * get_unmapped_area find a free area of the right size. 263: */ 264: unsigned long rb_subtree_gap; 265: 266: /* Second cache line starts here. */ 267: 268: struct mm_struct *vm_mm; /* The address space we belong to. */ 269: pgprot_t vm_page_prot; /* Access permissions of this VMA. */ 270: unsigned long vm_flags; /* Flags, see mm.h. */ 271: 272: /* 273: * For areas with an address space and backing store, 274: * linkage into the address_space->i_mmap interval tree, or 275: * linkage of vma in the address_space->i_mmap_nonlinear list. 276: */ 277: union { 278: struct { 279: struct rb_node rb; 280: unsigned long rb_subtree_last; 281: } linear; 282: struct list_head nonlinear; 283: } shared; 284: 285: /* 286: * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma 287: * list, after a COW of one of the file pages. A MAP_SHARED vma 288: * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack 289: * or brk vma (with NULL file) can only be in an anon_vma list. 290: */ 291: struct list_head anon_vma_chain; /* Serialized by mmap_sem & 292: * page_table_lock */ 293: struct anon_vma *anon_vma; /* Serialized by page_table_lock */ 294: 295: /* Function pointers to deal with this struct. */ 296: const struct vm_operations_struct *vm_ops; 297: 298: /* Information about our backing store: */ 299: unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE 300: units, *not* PAGE_CACHE_SIZE */ 301: struct file * vm_file; /* File we map to (can be NULL). */ 302: void * vm_private_data; /* was vm_pte (shared mem) */ 303: 304: #ifndef CONFIG_MMU 305: struct vm_region *vm_region; /* NOMMU mapping region */ 306: #endif 307: #ifdef CONFIG_NUMA 308: struct mempolicy *vm_policy; /* NUMA policy for the VMA */ 309: #endif 310: }; 311: 312: struct core_thread { 313: struct task_struct *task; 314: struct core_thread *next; 315: }; 316: 317: struct core_state { 318: atomic_t nr_threads; 319: struct core_thread dumper; 320: struct completion startup; 321: }; 322: 323: enum { 324: MM_FILEPAGES, 325: MM_ANONPAGES, 326: MM_SWAPENTS, 327: NR_MM_COUNTERS 328: }; 329: 330: #if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU) 331: #define SPLIT_RSS_COUNTING 332: /* per-thread cached information, */ 333: struct task_rss_stat { 334: int events; /* for synchronization threshold */ 335: int count[NR_MM_COUNTERS]; 336: }; 337: #endif /* USE_SPLIT_PTE_PTLOCKS */ 338: 339: struct mm_rss_stat { 340: atomic_long_t count[NR_MM_COUNTERS]; 341: }; 342: 343: struct kioctx_table; 344: struct mm_struct { 345: struct vm_area_struct * mmap; /* list of VMAs */ 346: struct rb_root mm_rb; 347: struct vm_area_struct * mmap_cache; /* last find_vma result */ 348: #ifdef CONFIG_MMU 349: unsigned long (*get_unmapped_area) (struct file *filp, 350: unsigned long addr, unsigned long len, 351: unsigned long pgoff, unsigned long flags); 352: #endif 353: unsigned long mmap_base; /* base of mmap area */ 354: unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */ 355: unsigned long task_size; /* size of task vm space */ 356: unsigned long highest_vm_end; /* highest vma end address */ 357: pgd_t * pgd; 358: atomic_t mm_users; /* How many users with user space? */ 359: atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */ 360: atomic_long_t nr_ptes; /* Page table pages */ 361: int map_count; /* number of VMAs */ 362: 363: spinlock_t page_table_lock; /* Protects page tables and some counters */ 364: struct rw_semaphore mmap_sem; 365: 366: struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung 367: * together off init_mm.mmlist, and are protected 368: * by mmlist_lock 369: */ 370: 371: 372: unsigned long hiwater_rss; /* High-watermark of RSS usage */ 373: unsigned long hiwater_vm; /* High-water virtual memory usage */ 374: 375: unsigned long total_vm; /* Total pages mapped */ 376: unsigned long locked_vm; /* Pages that have PG_mlocked set */ 377: unsigned long pinned_vm; /* Refcount permanently increased */ 378: unsigned long shared_vm; /* Shared pages (files) */ 379: unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */ 380: unsigned long stack_vm; /* VM_GROWSUP/DOWN */ 381: unsigned long def_flags; 382: unsigned long start_code, end_code, start_data, end_data; 383: unsigned long start_brk, brk, start_stack; 384: unsigned long arg_start, arg_end, env_start, env_end; 385: 386: unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */ 387: 388: /* 389: * Special counters, in some configurations protected by the 390: * page_table_lock, in other configurations by being atomic. 391: */ 392: struct mm_rss_stat rss_stat; 393: 394: struct linux_binfmt *binfmt; 395: 396: cpumask_var_t cpu_vm_mask_var; 397: 398: /* Architecture-specific MM context */ 399: mm_context_t context; 400: 401: unsigned long flags; /* Must use atomic bitops to access the bits */ 402: 403: struct core_state *core_state; /* coredumping support */ 404: #ifdef CONFIG_AIO 405: spinlock_t ioctx_lock; 406: struct kioctx_table __rcu *ioctx_table; 407: #endif 408: #ifdef CONFIG_MM_OWNER 409: /* 410: * "owner" points to a task that is regarded as the canonical 411: * user/owner of this mm. All of the following must be true in 412: * order for it to be changed: 413: * 414: * current == mm->owner 415: * current->mm != mm 416: * new_owner->mm == mm 417: * new_owner->alloc_lock is held 418: */ 419: struct task_struct __rcu *owner; 420: #endif 421: 422: /* store ref to file /proc/<pid>/exe symlink points to */ 423: struct file *exe_file; 424: #ifdef CONFIG_MMU_NOTIFIER 425: struct mmu_notifier_mm *mmu_notifier_mm; 426: #endif 427: #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 428: pgtable_t pmd_huge_pte; /* protected by page_table_lock */ 429: #endif 430: #ifdef CONFIG_CPUMASK_OFFSTACK 431: struct cpumask cpumask_allocation; 432: #endif 433: #ifdef CONFIG_NUMA_BALANCING 434: /* 435: * numa_next_scan is the next time that the PTEs will be marked 436: * pte_numa. NUMA hinting faults will gather statistics and migrate 437: * pages to new nodes if necessary. 438: */ 439: unsigned long numa_next_scan; 440: 441: /* Restart point for scanning and setting pte_numa */ 442: unsigned long numa_scan_offset; 443: 444: /* numa_scan_seq prevents two threads setting pte_numa */ 445: int numa_scan_seq; 446: #endif 447: #if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) 448: /* 449: * An operation with batched TLB flushing is going on. Anything that 450: * can move process memory needs to flush the TLB when moving a 451: * PROT_NONE or PROT_NUMA mapped page. 452: */ 453: bool tlb_flush_pending; 454: #endif 455: struct uprobes_state uprobes_state; 456: }; 457: 458: static inline void mm_init_cpumask(struct mm_struct *mm) 459: { 460: #ifdef CONFIG_CPUMASK_OFFSTACK 461: mm->cpu_vm_mask_var = &mm->cpumask_allocation; 462: #endif 463: } 464: 465: /* Future-safe accessor for struct mm_struct's cpu_vm_mask. */ 466: static inline cpumask_t *mm_cpumask(struct mm_struct *mm) 467: { 468: return mm->cpu_vm_mask_var; 469: } 470: 471: #if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) 472: /* 473: * Memory barriers to keep this state in sync are graciously provided by 474: * the page table locks, outside of which no page table modifications happen. 475: * The barriers below prevent the compiler from re-ordering the instructions 476: * around the memory barriers that are already present in the code. 477: */ 478: static inline bool mm_tlb_flush_pending(struct mm_struct *mm) 479: { 480: barrier(); 481: return mm->tlb_flush_pending; 482: } 483: static inline void set_tlb_flush_pending(struct mm_struct *mm) 484: { 485: mm->tlb_flush_pending = true; 486: 487: /* 488: * Guarantee that the tlb_flush_pending store does not leak into the 489: * critical section updating the page tables 490: */ 491: smp_mb__before_spinlock(); 492: } 493: /* Clearing is done after a TLB flush, which also provides a barrier. */ 494: static inline void clear_tlb_flush_pending(struct mm_struct *mm) 495: { 496: barrier(); 497: mm->tlb_flush_pending = false; 498: } 499: #else 500: static inline bool mm_tlb_flush_pending(struct mm_struct *mm) 501: { 502: return false; 503: } 504: static inline void set_tlb_flush_pending(struct mm_struct *mm) 505: { 506: } 507: static inline void clear_tlb_flush_pending(struct mm_struct *mm) 508: { 509: } 510: #endif 511: 512: #endif /* _LINUX_MM_TYPES_H */ 513: