Linux系统启动之后，物理内存的布局是怎么样的？

struct page *mem_map;

#if defined(CONFIG_FLATMEM) #define ARCH_PFN_OFFSET (0UL) #define __pfn_to_page(pfn) (mem_map + ((pfn) - ARCH_PFN_OFFSET)) #define __page_to_pfn(page) ((unsigned long)((page) - mem_map) + \ ARCH_PFN_OFFSET) #endif

//部分代码 typedef struct pglist_data { struct page *node_mem_map; } pg_data_t;

#if defined(CONFIG_SPARSEMEM_VMEMMAP) /* memmap is virtually contiguous. */ #define __pfn_to_page(pfn) (vmemmap + (pfn)) #define __page_to_pfn(page) (unsigned long)((page) - vmemmap) #endif

struct mem_section { unsigned long section_mem_map; struct mem_section_usage *usage; #ifdef CONFIG_PAGE_EXTENSION /* * 如果使用了 SPARSEMEM，pgdat 将没有 page_ext 指针。 * 我们会使用 section。 */ struct page_ext *page_ext; unsigned long pad; #endif // 警告：mem_section 的大小必须是2的幂，以便计算和使用 SECTION_ROOT_MASK。 };

#ifdef CONFIG_SPARSEMEM_EXTREME extern struct mem_section **mem_section; #else extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]; #endif

#define SECTION_SIZE_BITS 27 // 128MB #define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT) // 15 #define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT) #define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1)) #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section)) // 4096 / 16 = 256 # define MAX_PHYSMEM_BITS (pgtable_l5_enabled() ? 52 : 46) #define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS) // 46 - 27 = 19 #define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)//1 << 19 #define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) #define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT) // 2048 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) #define SECTIONS_WIDTH SECTIONS_SHIFT // 19 #else #define SECTIONS_WIDTH 0 #endif /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */ #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) // 64 - 19 = 45 #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))

#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) #define __page_to_pfn(pg) \ ({ const struct page *__pg = (pg); \ int __sec = page_to_section(__pg); \ (unsigned long)(__pg - __section_mem_map_addr(__nr_to_section(__sec))); \ }) #define __pfn_to_page(pfn) \ ({ unsigned long __pfn = (pfn); \ struct mem_section *__sec = __pfn_to_section(__pfn); \ __section_mem_map_addr(__sec) + __pfn; \ }) #endif

static inline struct page *__section_mem_map_addr(struct mem_section *section) { unsigned long map = section->section_mem_map; map &= SECTION_MAP_MASK; return (struct page *)map; }

static void __meminit sparse_init_one_section(struct mem_section *ms, unsigned long pnum, struct page *mem_map, struct mem_section_usage *usage, unsigned long flags) { ms->section_mem_map &= ~SECTION_MAP_MASK; ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | SECTION_HAS_MEM_MAP | flags; ms->usage = usage; } static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) { unsigned long coded_mem_map = (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); BUG_ON(coded_mem_map & ~SECTION_MAP_MASK); return coded_mem_map; }

enum { SECTION_MARKED_PRESENT_BIT, SECTION_HAS_MEM_MAP_BIT, SECTION_IS_ONLINE_BIT, //热插拔 SECTION_IS_EARLY_BIT, #ifdef CONFIG_ZONE_DEVICE SECTION_TAINT_ZONE_DEVICE_BIT, #endif SECTION_MAP_LAST_BIT, };

$ numactl -H available: 2 nodes (0-1) node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 node 0 size: 65441 MB node 0 free: 311 MB node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 node 1 size: 65536 MB node 1 free: 3704 MB node distances: node 0 1 0: 10 21 1: 21 10

$ numastat node0 node1 numa_hit 85567733 180252357 numa_miss 974366 107879428 numa_foreign 107879428 974366 interleave_hit 3475 6018 local_node 85554510 180592099 other_node 987585 107539786

#include<stdio.h> #include<numa.h> int main() { if (numa_available() < 0) printf("本系统不支持 NUMA\n"); else printf("最大 NUMA 节点 id: %d\n", numa_max_node()); return 0; }

gcc -o test test.c -lnuma

#include <numaif.h> long set_mempolicy(int mode, const unsigned long *nodemask, unsigned long maxnode);

$ numactl -s policy: default //默认策略 preferred node: current physcpubind: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 cpubind: 0 1 nodebind: 0 1 membind: 0 1

$ numactl --membind=0 --cpunodebind=0 ./a.out $ numactl --membind=1 --cpunodebind=0 ./a.out

$ numactl --physcpubind=0 ./a.out #绑定到 0 号 CPU $ numactl --physcpubind=0-5 ./a.out #绑定到 0～5 号 CPU

[root@929533e6b321 test]# cat /proc/cpuinfo | grep processor processor : 0 processor : 1 processor : 2 processor : 3 processor : 4 processor : 5

typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t; /* Array of node states. */ nodemask_t node_states[NR_NODE_STATES] __read_mostly = { [N_POSSIBLE] = NODE_MASK_ALL, [N_ONLINE] = { { [0] = 1UL } }, #ifndef CONFIG_NUMA [N_NORMAL_MEMORY] = { { [0] = 1UL } }, #ifdef CONFIG_HIGHMEM [N_HIGH_MEMORY] = { { [0] = 1UL } }, #endif [N_MEMORY] = { { [0] = 1UL } }, [N_CPU] = { { [0] = 1UL } }, #endif /* NUMA */ }; /* Bitmasks that are kept for all the nodes. */ enum node_states { N_POSSIBLE, /* The node could become online at some point */ N_ONLINE, /* The node is online */ N_NORMAL_MEMORY, /* The node has regular memory */ #ifdef CONFIG_HIGHMEM N_HIGH_MEMORY, /* The node has regular or high memory */ #else N_HIGH_MEMORY = N_NORMAL_MEMORY, #endif N_MEMORY, /* The node has memory(regular, high, movable) */ N_CPU, /* The node has one or more cpus */ N_GENERIC_INITIATOR,/* The node has one or more Generic Initiators */ NR_NODE_STATES };

#define MAX_NUMNODES (1 << NODES_SHIFT) struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;

config NODES_SHIFT int "Maximum NUMA Nodes (as a power of 2)" if !MAXSMP range 1 10 default "10" if MAXSMP default "6" if X86_64 default "3" depends on NUMA help Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables.

typedef struct pglist_data { /* * node_zones 只包含此节点的内存区域（zone）。并不是所有区域都会被填充，但它是完整的区域列表。 * 它被本节点的 node_zonelists 引用，以及其他节点的 node_zonelists 引用。 */ struct zone node_zones[MAX_NR_ZONES]; /* * node_zonelists 包含对所有节点中所有区域的引用。 * 通常，前面的区域引用是指向此节点的 node_zones。 */ struct zonelist node_zonelists[MAX_ZONELISTS]; int nr_zones; /* 当前节点中已填充的内存区域数量 */ #ifdef CONFIG_FLATMEM /* means !SPARSEMEM */ struct page *node_mem_map; #ifdef CONFIG_PAGE_EXTENSION struct page_ext *node_page_ext; #endif #endif #if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) /* * 在你期望 node_start_pfn、node_present_pages、node_spanned_pages 或 nr_zones 保持不变时，必须持有此锁。 * 同时还在延迟的页面初始化期间同步 pgdat->first_deferred_pfn。 * 通过 pgdat_resize_lock() 和 pgdat_resize_unlock() 函数来操作 node_size_lock，而不需要检查 CONFIG_MEMORY_HOTPLUG 或 CONFIG_DEFERRED_STRUCT_PAGE_INIT。 * 嵌套在 zone->lock 和 zone->span_seqlock 之上。 */ spinlock_t node_size_lock; #endif unsigned long node_start_pfn; unsigned long node_present_pages; /* 物理页的总数 */ unsigned long node_spanned_pages; /* 包含空洞的物理页总数 */ int node_id; /* 节点 id */ wait_queue_head_t kswapd_wait; wait_queue_head_t pfmemalloc_wait; /* 用于不同原因的回收限制的工作队列。 */ wait_queue_head_t reclaim_wait[NR_VMSCAN_THROTTLE]; atomic_t nr_writeback_throttled; /* 受限制写回的任务数 */ unsigned long nr_reclaim_start; /* 在限制开始时写入的页面数 */ struct task_struct *kswapd; /* 在 mem_hotplug_begin/done() 期间受保护 */ int kswapd_order; enum zone_type kswapd_highest_zoneidx; int kswapd_failures; /* 'reclaimed == 0' 的次数 */ #ifdef CONFIG_COMPACTION int kcompactd_max_order; enum zone_type kcompactd_highest_zoneidx; wait_queue_head_t kcompactd_wait; struct task_struct *kcompactd; bool proactive_compact_trigger; #endif /* 这是每个节点的保留页数量，不可用于用户空间分配。*/ unsigned long totalreserve_pages; #ifdef CONFIG_NUMA /* 如果存在更多的未映射页面，则节点回收变得活跃。*/ unsigned long min_unmapped_pages; unsigned long min_slab_pages; #endif /* CONFIG_NUMA */ /* 用于页面回收的写密集型字段（Write-intensive fields）*/ ZONE_PADDING(_pad1_) #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* 如果在大型机器上的内存初始化被延迟执行，则这是需要初始化的第一个 PFN。*/ unsigned long first_deferred_pfn; #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ #ifdef CONFIG_TRANSPARENT_HUGEPAGE struct deferred_split deferred_split_queue; #endif /* 页面回收扫描器常用的字段 */ /* * 注意：如果启用了 MEMCG，则此字段未使用。 * 使用 mem_cgroup_lruvec() 来查找 lruvecs。 */ struct lruvec __lruvec; unsigned long flags; ZONE_PADDING(_pad2_) /* Per-node vmstats，每个节点的虚拟内存统计 */ struct per_cpu_nodestat __percpu *per_cpu_nodestats; atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; } pg_data_t;

enum zone_type { #ifdef CONFIG_ZONE_DMA ZONE_DMA, #endif #ifdef CONFIG_ZONE_DMA32 ZONE_DMA32, #endif ZONE_NORMAL, #ifdef CONFIG_HIGHMEM ZONE_HIGHMEM, #endif ZONE_MOVABLE, #ifdef CONFIG_ZONE_DEVICE ZONE_DEVICE, #endif __MAX_NR_ZONES // DEFINE(MAX_NR_ZONES, __MAX_NR_ZONES); };

$ cat /proc/zoneinfo | grep Node Node 0, zone DMA Node 0, zone DMA32 Node 0, zone Normal Node 1, zone Normal Node 2, zone Normal Node 2, zone Movable Node 3, zone Normal Node 3, zone Device

[root@929533e6b321 test]# cat /proc/zoneinfo Node 0, zone DMA per-node stats nr_inactive_anon 111525 nr_active_anon 17352 nr_inactive_file 211346 nr_active_file 124539 nr_unevictable 0 nr_slab_reclaimable 13793 nr_slab_unreclaimable 7843 nr_isolated_anon 0 nr_isolated_file 0 nr_anon_pages 42969 nr_mapped 56654 nr_file_pages 420009 nr_dirty 1 nr_writeback 0 nr_writeback_temp 0 nr_shmem 84124 nr_shmem_hugepages 0 nr_shmem_pmdmapped 0 nr_file_hugepages 0 nr_file_pmdmapped 0 nr_anon_transparent_hugepages 4 nr_vmscan_write 0 nr_vmscan_immediate_reclaim 0 nr_dirtied 245843 nr_written 245771 nr_kernel_misc_reclaimable 0 nr_kernel_stack 8864 pages free 3977 min 44 low 55 high 66 spanned 4095 present 3998 managed 3977 protection: (0, 2992, 5928, 5928, 5928) nr_free_pages 3977 nr_zone_inactive_anon 0 nr_zone_active_anon 0 nr_zone_inactive_file 0 nr_zone_active_file 0 nr_zone_unevictable 0 nr_zone_write_pending 0 nr_mlock 0 nr_page_table_pages 0 nr_bounce 0 nr_free_cma 0 pagesets cpu: 0 count: 0 high: 0 batch: 1 vm stats threshold: 6 cpu: 1 count: 0 high: 0 batch: 1 ··· ··· vm stats threshold: 6 cpu: 5 count: 0 high: 0 batch: 1 vm stats threshold: 6 node_unreclaimable: 0 start_pfn: 1 Node 0, zone DMA32 pages free 765587 min 8505 low 10631 high 12757 spanned 1044480 present 782336 managed 765952 protection: (0, 0, 2936, 2936, 2936) nr_free_pages 765587 nr_zone_inactive_anon 0 nr_zone_active_anon 0 nr_zone_inactive_file 0 nr_zone_active_file 0 nr_zone_unevictable 0 nr_zone_write_pending 0 nr_mlock 0 nr_page_table_pages 0 nr_bounce 0 nr_free_cma 0 pagesets cpu: 0 count: 365 high: 378 batch: 63 vm stats threshold: 36 cpu: 1 count: 0 high: 378 batch: 63 ··· ··· vm stats threshold: 36 cpu: 5 count: 0 high: 378 batch: 63 vm stats threshold: 36 node_unreclaimable: 0 start_pfn: 4096 Node 0, zone Normal pages free 250616 min 8346 low 10432 high 12518 spanned 786432 present 786432 managed 751697 protection: (0, 0, 0, 0, 0) nr_free_pages 250616 nr_zone_inactive_anon 111525 nr_zone_active_anon 17352 nr_zone_inactive_file 211346 nr_zone_active_file 124539 nr_zone_unevictable 0 nr_zone_write_pending 1 nr_mlock 0 nr_page_table_pages 1166 nr_bounce 0 nr_free_cma 0 pagesets cpu: 0 count: 325 high: 378 batch: 63 vm stats threshold: 36 cpu: 1 count: 259 high: 378 batch: 63 ··· ··· vm stats threshold: 36 cpu: 5 count: 131 high: 378 batch: 63 vm stats threshold: 36 node_unreclaimable: 0 start_pfn: 1048576 Node 0, zone Movable pages free 0 min 0 low 0 high 0 spanned 0 present 0 managed 0 protection: (0, 0, 0, 0, 0) Node 0, zone Device pages free 0 min 0 low 0 high 0 spanned 0 present 0 managed 0 protection: (0, 0, 0, 0, 0)

struct zone { /* 几乎都是读操作的字段 */ /* 内存区域水位标记，通过 *_wmark_pages(zone) 宏访问 */ unsigned long _watermark[NR_WMARK]; unsigned long watermark_boost; unsigned long nr_reserved_highatomic; /* * 我们不知道要分配的内存是否可释放，或者最终是否会被释放，所以为了避免浪费几个GB的大量内存，我们必须保留一些较低内存区域的内存。 *（否则我们就会冒着在较低区域运行OOM的风险，尽管较高区域有大量可用的内存） * 此数组在运行时根据 sysctl_lowmem_reserve_ratio 系统控制参数的变化进行重新计算。 */ long lowmem_reserve[MAX_NR_ZONES]; #ifdef CONFIG_NUMA int node; #endif // 指向所在的 NUMA 节点 pglist_data struct pglist_data *zone_pgdat; struct per_cpu_pages __percpu *per_cpu_pageset; struct per_cpu_zonestat __percpu *per_cpu_zonestats; /* * the high and batch values are copied to individual pagesets for * faster access */ int pageset_high; int pageset_batch; #ifndef CONFIG_SPARSEMEM /* * 用于 pageblock_nr_pages 块的标志位。参见 pageblock-flags.h。 * 在 SPARSEMEM 中，此映射存储在 struct mem_section 中。 */ unsigned long *pageblock_flags; #endif /* CONFIG_SPARSEMEM */ /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ unsigned long zone_start_pfn; atomic_long_t managed_pages; unsigned long spanned_pages; unsigned long present_pages; #if defined(CONFIG_MEMORY_HOTPLUG) unsigned long present_early_pages; #endif #ifdef CONFIG_CMA unsigned long cma_pages; #endif const char *name; #ifdef CONFIG_MEMORY_ISOLATION unsigned long nr_isolate_pageblock; #endif #ifdef CONFIG_MEMORY_HOTPLUG /* see spanned/present_pages for more description */ seqlock_t span_seqlock; #endif int initialized; /* 用于页面分配器的写密集型字段 */ ZONE_PADDING(_pad1_) /* 不同大小的空闲区域 */ struct free_area free_area[MAX_ORDER]; /* 区域标志，见下面的定义 */ unsigned long flags; /* 主要用于保护 free_area */ spinlock_t lock; /* 用于 compaction（压缩）和 vm 统计的写密集型字段 */ ZONE_PADDING(_pad2_) /* * 当空闲页低于此点时，在读取空闲页数时会采取额外步骤， * 以避免 per-cpu 计数器漂移，从而导致水位标记被突破 */ unsigned long percpu_drift_mark; #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* compaction（压缩）空闲扫描器应该从哪个 pfn 开始 */ unsigned long compact_cached_free_pfn; /* compaction（压缩）迁移扫描器应该从哪个 pfn 开始 */ unsigned long compact_cached_migrate_pfn[ASYNC_AND_SYNC]; unsigned long compact_init_migrate_pfn; unsigned long compact_init_free_pfn; #endif #ifdef CONFIG_COMPACTION /* * 在压缩失败时，会跳过 1<<compact_defer_shift 次压缩， * 然后再次尝试。自上次失败以来尝试的次数由 compact_considered 跟踪。 * compact_order_failed 是最小的压缩失败阶数。 */ unsigned int compact_considered; unsigned int compact_defer_shift; int compact_order_failed; #endif #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* 当 PG_migrate_skip 位应该被清除时设置为 true */ bool compact_blockskip_flush; #endif bool contiguous; ZONE_PADDING(_pad3_) /* 区域统计数据 */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; } ____cacheline_internodealigned_in_smp;

/* * Add a wild amount of padding here to ensure data fall into separate * cachelines. There are very few zone structures in the machine, so space * consumption is not a concern here. */ #if defined(CONFIG_SMP) struct zone_padding { char x[0]; } ____cacheline_internodealigned_in_smp; #define ZONE_PADDING(name) struct zone_padding name; #else #define ZONE_PADDING(name) #endif

[root@929533e6b321 /]# cat /proc/sys/vm/lowmem_reserve_ratio 256 256 32 0 0 [root@929533e6b321 /]# sysctl -a|grep lowmem_reserve_ratio vm.lowmem_reserve_ratio = 256 256 32 0 0

[root@929533e6b321 /]# cat /proc/zoneinfo | grep Node Node 0, zone DMA Node 0, zone DMA32 Node 0, zone Normal Node 0, zone Movable Node 0, zone Device

[root@929533e6b321 /]# cat /proc/zoneinfo|grep managed managed 3977 //DMA managed 765952 //DMA32 managed 751697 //Normal managed 0 //Movable managed 0 //Device

[root@929533e6b321 /]# cat /proc/zoneinfo|grep protection protection: (0, 2992, 5928, 5928, 5928) //DMA protection: (0, 0, 2936, 2936, 2936) //DMA32 protection: (0, 0, 0, 0, 0) //Normal protection: (0, 0, 0, 0, 0) //Movable protection: (0, 0, 0, 0, 0) //Device

$ sysctl -w vm.lowmem_reserve_ratio=256 256 32 0 0

$ cat /proc/sys/vm/lowmem_reserve_ratio 256 256 32 $ sysctl -a|grep lowmem_reserve_ratio vm.lowmem_reserve_ratio = 256 256 32 $ cat /proc/zoneinfo | grep Node Node 0, zone DMA Node 0, zone DMA32 Node 0, zone Normal Node 1, zone Normal $ cat /proc/zoneinfo|grep managed managed 3957 //DMA managed 433090 //DMA32 managed 15987300 //Normal 0 managed 16511520 //Normal 1 $ cat /proc/zoneinfo|grep pro protection: (0, 1691, 64142, 64142) //DMA protection: (0, 0, 62450, 62450) //DMA32 protection: (0, 0, 0, 0) //Normal 0 protection: (0, 0, 0, 0) //Normal 1

enum zone_watermarks { WMARK_MIN, WMARK_LOW, WMARK_HIGH, WMARK_PROMO, NR_WMARK };

#define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost) #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost) #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost) #define wmark_pages(z, i) (z->_watermark[i] + z->watermark_boost)

$ cat /proc/zoneinfo ··· Node 0, zone Normal pages free 329518 //空闲内存页数 min 8346 //_watermark[WMARK_MIN] low 10432 //_watermark[WMARK_LOW] high 12518 //_watermark[WMARK_HIGH] spanned 786432 present 786432 managed 751697 protection: (0, 0, 0, 0, 0) ···

$ cat /proc/sys/vm/min_free_kbytes 67584

struct zone { ··· struct per_cpu_pages __percpu *per_cpu_pageset; ··· } ____cacheline_internodealigned_in_smp;

/* Fields and list protected by pagesets local_lock in page_alloc.c */ struct per_cpu_pages { spinlock_t lock; /* Protects lists field */ int count; /* number of pages in the list */ int high; /* high watermark, emptying needed */ int batch; /* chunk size for buddy add/remove */ short free_factor; /* batch scaling factor during free */ #ifdef CONFIG_NUMA short expire; /* When 0, remote pagesets are drained */ #endif /* Lists of pages, one per migrate type stored on the pcp-lists */ struct list_head lists[NR_PCP_LISTS]; } ____cacheline_aligned_in_smp;

struct page { /* 原子标志，一些可能会异步更新 */ unsigned long flags; /* * 这个联合中有五个字（20/40 字节）可用。 * 警告：第一个字的第 0 位用于 PageTail()。这意味着此联合的其他用户 * 必须避免使用该位，以避免冲突和错误的 PageTail()。 */ union { struct {/* 页面缓存（Page cache）和匿名页面（anonymous pages） */ /** * @lru: 页面退出列表，例如由 lruvec->lru_lock 保护的 active_list。 * 有时被页面所有者用作通用列表。 */ union { struct list_head lru; struct { /* 或者，对于不可驱逐的“LRU 列表”槽位 */ /* 始终为偶数，以取消 PageTail */ void *__filler; /* 计算 page/folio 的 mlock 数 */ unsigned int mlock_count; }; /* 或者，空闲页面 */ struct list_head buddy_list; struct list_head pcp_list; }; /* 有关 PAGE_MAPPING_FLAGS，请参见 page-flags.h */ struct address_space *mapping; pgoff_t index; /* 在映射中的偏移量。*/ /** * @private: 映射私有的不透明数据。 * 如果 PagePrivate，则通常用于 buffer_heads。 * 如果 PageSwapCache，则用于 swp_entry_t。 * 如果 PageBuddy，则表示在伙伴系统中的阶数。 */ unsigned long private; }; struct { /* 由 netstack 使用的 page_pool */ /*@pp_magic: 用于避免回收非 page_pool 分配的页面。*/ unsigned long pp_magic; struct page_pool *pp; unsigned long _pp_mapping_pad; unsigned long dma_addr; union { /* dma_addr_upper：在 32 位体系结构上可能需要 64 位值。*/ unsigned long dma_addr_upper; /* 用于 frag 页面支持，在带有 64 位 DMA 的 32 位体系结构上不支持。*/ atomic_long_t pp_frag_count; }; }; struct { /* 复合页面（compound page）的首页面 */ unsigned long compound_head; /* 第 0 位被置位 */ /* 仅对第一个尾部页面 */ unsigned char compound_dtor; unsigned char compound_order; atomic_t compound_mapcount; atomic_t compound_pincount; #ifdef CONFIG_64BIT unsigned int compound_nr; /* 1 << compound_order */ #endif }; struct { /* 复合页面（compound page）的尾页面 */ unsigned long _compound_pad_1; /* compound_head */ unsigned long _compound_pad_2; /* 用于 global 和 memcg */ struct list_head deferred_list; }; struct { /* 页表页 */ unsigned long _pt_pad_1; /* compound_head */ pgtable_t pmd_huge_pte; /* protected by page->ptl */ unsigned long _pt_pad_2; /* mapping */ union { struct mm_struct *pt_mm; /* x86 pgds only */ atomic_t pt_frag_refcount; /* powerpc */ }; #if ALLOC_SPLIT_PTLOCKS spinlock_t *ptl; #else spinlock_t ptl; #endif }; struct { /* ZONE_DEVICE pages */ /* pgmap: 指向托管设备页面映射。 */ struct dev_pagemap *pgmap; void *zone_device_data; /* * ZONE_DEVICE 私有页面被视为已映射的页面，因此下面的 3 个字 * 保存源匿名或页面缓存页面的映射、索引和私有字段， * 在页面迁移到设备私有内存时使用。 * ZONE_DEVICE MEMORY_DEVICE_FS_DAX 页面在映射 pmem 后备的 DAX 文件时也使用映射、索引和私有字段。 */ }; /* rcu_head: 可以使用此项通过 RCU 来释放页面。 */ struct rcu_head rcu_head; }; union { /* 此联合的大小为 4 字节。*/ /* 如果页面可以映射到用户空间，则编码了此页被页表引用的次数。*/ atomic_t _mapcount; /* * 如果页既不是 PageSlab 也不能映射到用户空间， * 则存储在此处的值可能有助于确定此页面的用途。 * 有关当前存储在此处的页面类型列表，请参见 page-flags.h。 */ unsigned int page_type; }; /* 使用计数。*请勿直接使用*。请参见 page_ref.h */ atomic_t _refcount; #ifdef CONFIG_MEMCG unsigned long memcg_data; #endif /* * 在将所有 RAM 映射到内核地址空间的机器上， * 我们可以简单地计算虚拟地址。在具有高内存的机器上， * 一些内存被动态地映射到内核虚拟内存中， * 因此我们需要一个地方来存储该地址。 * 请注意，这个字段在 x86 上可能是 16 位 ... ;) * * 在 asm/page.h 中定义了 WANT_PAGE_VIRTUAL * 的体系结构具有较慢的乘法。 */ #if defined(WANT_PAGE_VIRTUAL) void *virtual; /* 内核虚拟地址（如果未 kmapped，则为 NULL，即 highmem） */ #endif /* WANT_PAGE_VIRTUAL */ #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS int _last_cpupid; #endif } _struct_page_alignment;

struct { /* 复合页面（compound page）的第一个页面 */ unsigned long compound_head; /* 第 0 位被置位 */ unsigned char compound_dtor; unsigned char compound_order; atomic_t compound_mapcount; atomic_t compound_pincount; #ifdef CONFIG_64BIT unsigned int compound_nr; /* 1 << compound_order */ #endif };

struct { /* 复合页面（compound page）的尾页面 */ unsigned long _compound_pad_1; /* compound_head */ unsigned long _compound_pad_2; /* 用于 global 和 memcg */ struct list_head deferred_list; };

struct { /* 由 netstack 使用的 page_pool */ /*@pp_magic: 用于避免回收非 page_pool 分配的页面。*/ unsigned long pp_magic; struct page_pool *pp; unsigned long _pp_mapping_pad; unsigned long dma_addr; union { /* dma_addr_upper：在 32 位体系结构上可能需要 64 位值。*/ unsigned long dma_addr_upper; /* 用于 frag 页面支持，在带有 64 位 DMA 的 32 位体系结构上不支持。*/ atomic_long_t pp_frag_count; }; };

struct { /* 页表页 */ unsigned long _pt_pad_1; /* compound_head */ pgtable_t pmd_huge_pte; /* protected by page->ptl */ unsigned long _pt_pad_2; /* mapping */ union { struct mm_struct *pt_mm; /* x86 pgds only */ atomic_t pt_frag_refcount; /* powerpc */ }; #if ALLOC_SPLIT_PTLOCKS spinlock_t *ptl; #else spinlock_t ptl; #endif };

#if MAX_NR_ZONES < 2 #define ZONES_SHIFT 0 #elif MAX_NR_ZONES <= 2 #define ZONES_SHIFT 1 #elif MAX_NR_ZONES <= 4 #define ZONES_SHIFT 2 #elif MAX_NR_ZONES <= 8 #define ZONES_SHIFT 3 #else #error ZONES_SHIFT "Too many zones conpd" #endif #define ZONES_WIDTH ZONES_SHIFT // 通常为 3

#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) #define KASAN_TAG_WIDTH 8 #else #define KASAN_TAG_WIDTH 0 #endif

config NODES_SHIFT int "Maximum NUMA Nodes (as a power of 2)" if !MAXSMP range 1 10 default "10" if MAXSMP default "6" if X86_64 default "3" depends on NUMA help Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables. #ifdef CONFIG_NODES_SHIFT #define NODES_SHIFT CONFIG_NODES_SHIFT // default "6" if X86_64 #else #define NODES_SHIFT 0 #endif #if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS #define NODES_WIDTH NODES_SHIFT // default "6" if X86_64 #elif defined(CONFIG_SPARSEMEM_VMEMMAP) #error "Vmemmap: No space for nodes field in page flags" #else #define NODES_WIDTH 0 #endif

#ifdef CONFIG_NUMA_BALANCING #define LAST__PID_SHIFT 8 #define LAST__PID_MASK ((1 << LAST__PID_SHIFT)-1) #define LAST__CPU_SHIFT NR_CPUS_BITS #define LAST__CPU_MASK ((1 << LAST__CPU_SHIFT)-1) #define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT) #else #define LAST_CPUPID_SHIFT 0 #endif #if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ <= BITS_PER_LONG - NR_PAGEFLAGS #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT #else #define LAST_CPUPID_WIDTH 0 #endif

#ifdef CONFIG_SPARSEMEM #include <asm/sparsemem.h> #define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS) //46 - 27 = 19 #else #define SECTIONS_SHIFT 0 #endif #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) #define SECTIONS_WIDTH SECTIONS_SHIFT // 19 #else #define SECTIONS_WIDTH 0 #endif

static inline unsigned long page_to_section(const struct page *page) { return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; }

/* * Don't use the pageflags directly. Use the PageFoo macros. * * The page flags field is split into two parts, the main flags area * which extends from the low bits upwards, and the fields area which * extends from the high bits downwards. * * | FIELD | ... | FLAGS | * N-1 ^ 0 * (NR_PAGEFLAGS) * * The fields area is reserved for fields mapping zone, node (for NUMA) and * SPARSEMEM section (for variants of SPARSEMEM that require section ids like * SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP). */ enum pageflags { PG_locked, /* Page is locked. Don't touch. */ PG_referenced, PG_uptodate, PG_dirty, PG_lru, PG_active, PG_workingset, PG_waiters, /* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_locked" */ PG_error, PG_slab, PG_owner_priv_1, /* Owner use. If pagecache, fs may use*/ PG_arch_1, PG_reserved, PG_private, /* If pagecache, has fs-private data */ PG_private_2, /* If pagecache, has fs aux data */ PG_writeback, /* Page is under writeback */ PG_head, /* A head page */ PG_mappedtodisk, /* Has blocks allocated on-disk */ PG_reclaim, /* To be reclaimed asap */ PG_swapbacked, /* Page is backed by RAM/swap */ PG_unevictable, /* Page is "unevictable" */ #ifdef CONFIG_MMU PG_mlocked, /* Page is vma mlocked */ #endif #ifdef CONFIG_ARCH_USES_PG_UNCACHED PG_uncached, /* Page has been mapped as uncached */ #endif #ifdef CONFIG_MEMORY_FAILURE PG_hwpoison, /* hardware poisoned page. Don't touch */ #endif #if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) PG_young, PG_idle, #endif #ifdef CONFIG_ARCH_USES_PG_ARCH_X PG_arch_2, PG_arch_3, #endif __NR_PAGEFLAGS, PG_readahead = PG_reclaim, PG_anon_exclusive = PG_mappedtodisk, /* Filesystems */ PG_checked = PG_owner_priv_1, /* SwapBacked */ PG_swapcache = PG_owner_priv_1, /* Swap page: swp_entry_t in private */ PG_fscache = PG_private_2, /* page backed by cache */ /* XEN */ /* Pinned in Xen as a read-only pagetable page. */ PG_pinned = PG_owner_priv_1, /* Pinned as part of domain save (see xen_mm_pin_all()). */ PG_savepinned = PG_dirty, /* Has a grant mapping of another (foreign) domain's page. */ PG_foreign = PG_owner_priv_1, /* Remapped by swiotlb-xen. */ PG_xen_remapped = PG_owner_priv_1, #ifdef CONFIG_MEMORY_FAILURE PG_has_hwpoisoned = PG_error, #endif /* non-lru isolated movable page */ PG_isolated = PG_reclaim, /* Only valid for buddy pages. Used to track pages that are reported */ PG_reported = PG_uptodate, #ifdef CONFIG_MEMORY_HOTPLUG /* For self-hosted memmap pages */ PG_vmemmap_self_hosted = PG_owner_priv_1, #endif }; #define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1)

$ cat /proc/zoneinfo ··· Node 0, zone Normal pages free 329518 min 8346 low 10432 high 12518 spanned 786432 present 786432 managed 751697 protection: (0, 0, 0, 0, 0) nr_free_pages 329518 nr_zone_inactive_anon 109932 //匿名页的 inactive 链表的长度 nr_zone_active_anon 17338 //匿名页的 active 链表的长度 nr_zone_inactive_file 141904 //文件页的 inactive 链表的长度 nr_zone_active_file 119306 //文件页的 active 链表的长度 nr_mlock 0 // mlock 链表的长度 ···

[root@929533e6b321 /]# cat /proc/sys/vm/swappiness 60

sysctl -w vm.swappiness=80

[root@929533e6b321 test]# cat /proc/iomem 00000000-7fffffffff : PCI Bus 0000:00 // 512GB 00000000-00000fff : Reserved // 4KB 00001000-0009fbff : System RAM // 635KB 000f0000-000fffff : System ROM // 64KB 00100000-bfffffff : System RAM // 2.99GB c0000000-c0001fff : 0000:00:01.0 // 8KB c0002000-c0003fff : 0000:00:02.0 // 8KB c0004000-c0005fff : 0000:00:03.0 // 8KB c0006000-c0007fff : 0000:00:04.0 // 8KB c0008000-c00087ff : 0000:00:01.0 // 2KB c0008800-c0008fff : 0000:00:02.0 // 2KB c0009000-c00097ff : 0000:00:03.0 // 2KB c0009800-c0009fff : 0000:00:04.0 // 2KB c000a000-c000a7ff : 0000:00:1f.0 // 2KB e0000000-efffffff : PCI MMCONFIG 0000 [bus 00-ff] // 256MB e0000000-efffffff : pnp 00:00 // 256MB fec00000-fec003ff : IOAPIC 0 // 1KB fed00000-fed003ff : HPET 0 // 1KB fed00000-fed003ff : PNP0103:00 // 1KB fee00000-fee00fff : Local APIC // 4KB 100000000-1bfffffff : System RAM // 3GB 11a000000-11ae03316 : Kernel code // 14.01MB 11b000000-11b36bfff : Kernel rodata // 3.42MB 11b400000-11b641cbf : Kernel data // 144.45KB 11bb2c000-11bdfffff : Kernel bss // 2.83MB

#!/bin/bash if [ "$#" -ne 1 ]; then echo "用法: $0 <PID>" exit 1 fi pid="$1" # Read the pagemap file in binary and convert to hex using xxd xxd -p -c 8 -g 8 "/proc/$pid/pagemap" | while read entry; do echo "Page Entry: 0x$entry" done

#!/usr/bin/env python3 import os import sys def read_pagemap(pid): with open("/proc/{}/pagemap".format(pid), "rb") as f: while True: data = f.read(8) if not data: break entry = int(data.hex(), 16) print("Page Entry: 0x{:016x}".format(entry)) if __name__ == "__main__": pid = int(sys.argv[1]) # 获取命令行参数作为PID read_pagemap(pid)

#!/usr/bin/env python # -*- coding: utf-8 -*- import os import sys def read_pagemap(pid): with open("/proc/{}/pagemap".format(pid), "rb") as f: while True: data = f.read(8) if not data: break entry = int(data.encode('hex'), 16) print("Page Entry: 0x{:016x}".format(entry)) if __name__ == "__main__": pid = int(sys.argv[1]) # 获取命令行参数作为PID read_pagemap(pid)

[root@929533e6b321 test]# sh read_pagemap.sh 123 |grep -v 0x0000000000000000 Page Entry: 0x74ce1600000000a1 Page Entry: 0x7a59170000000081 Page Entry: 0x0156170000000081 Page Entry: 0x2462170000000081

cat /proc/kpagecount 6

$ cat /proc/kpageflags

$ cat /proc/kpagecgroup