Details | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
422 | giacomo | 1 | #ifndef _LINUX_MMZONE_H |
2 | #define _LINUX_MMZONE_H |
||
3 | |||
4 | #ifdef __KERNEL__ |
||
5 | #ifndef __ASSEMBLY__ |
||
6 | |||
7 | #include <linux/config.h> |
||
8 | #include <linux/spinlock.h> |
||
9 | #include <linux/list.h> |
||
10 | #include <linux/wait.h> |
||
11 | #include <linux/cache.h> |
||
12 | #include <linux/threads.h> |
||
13 | #include <linux/numa.h> |
||
14 | #include <asm/atomic.h> |
||
15 | |||
16 | /* Free memory management - zoned buddy allocator. */ |
||
17 | #ifndef CONFIG_FORCE_MAX_ZONEORDER |
||
18 | #define MAX_ORDER 11 |
||
19 | #else |
||
20 | #define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER |
||
21 | #endif |
||
22 | |||
23 | struct free_area { |
||
24 | struct list_head free_list; |
||
25 | unsigned long *map; |
||
26 | }; |
||
27 | |||
28 | struct pglist_data; |
||
29 | |||
30 | /* |
||
31 | * zone->lock and zone->lru_lock are two of the hottest locks in the kernel. |
||
32 | * So add a wild amount of padding here to ensure that they fall into separate |
||
33 | * cachelines. There are very few zone structures in the machine, so space |
||
34 | * consumption is not a concern here. |
||
35 | */ |
||
36 | #if defined(CONFIG_SMP) |
||
37 | struct zone_padding { |
||
38 | int x; |
||
39 | } ____cacheline_maxaligned_in_smp; |
||
40 | #define ZONE_PADDING(name) struct zone_padding name; |
||
41 | #else |
||
42 | #define ZONE_PADDING(name) |
||
43 | #endif |
||
44 | |||
45 | struct per_cpu_pages { |
||
46 | int count; /* number of pages in the list */ |
||
47 | int low; /* low watermark, refill needed */ |
||
48 | int high; /* high watermark, emptying needed */ |
||
49 | int batch; /* chunk size for buddy add/remove */ |
||
50 | struct list_head list; /* the list of pages */ |
||
51 | }; |
||
52 | |||
53 | struct per_cpu_pageset { |
||
54 | struct per_cpu_pages pcp[2]; /* 0: hot. 1: cold */ |
||
55 | } ____cacheline_aligned_in_smp; |
||
56 | |||
57 | /* |
||
58 | * On machines where it is needed (eg PCs) we divide physical memory |
||
59 | * into multiple physical zones. On a PC we have 3 zones: |
||
60 | * |
||
61 | * ZONE_DMA < 16 MB ISA DMA capable memory |
||
62 | * ZONE_NORMAL 16-896 MB direct mapped by the kernel |
||
63 | * ZONE_HIGHMEM > 896 MB only page cache and user processes |
||
64 | */ |
||
65 | |||
66 | struct zone { |
||
67 | /* |
||
68 | * Commonly accessed fields: |
||
69 | */ |
||
70 | spinlock_t lock; |
||
71 | unsigned long free_pages; |
||
72 | unsigned long pages_min, pages_low, pages_high; |
||
73 | |||
74 | ZONE_PADDING(_pad1_) |
||
75 | |||
76 | spinlock_t lru_lock; |
||
77 | struct list_head active_list; |
||
78 | struct list_head inactive_list; |
||
79 | atomic_t refill_counter; |
||
80 | unsigned long nr_active; |
||
81 | unsigned long nr_inactive; |
||
82 | int all_unreclaimable; /* All pages pinned */ |
||
83 | unsigned long pages_scanned; /* since last reclaim */ |
||
84 | |||
85 | ZONE_PADDING(_pad2_) |
||
86 | |||
87 | /* |
||
88 | * prev_priority holds the scanning priority for this zone. It is |
||
89 | * defined as the scanning priority at which we achieved our reclaim |
||
90 | * target at the previous try_to_free_pages() or balance_pgdat() |
||
91 | * invokation. |
||
92 | * |
||
93 | * We use prev_priority as a measure of how much stress page reclaim is |
||
94 | * under - it drives the swappiness decision: whether to unmap mapped |
||
95 | * pages. |
||
96 | * |
||
97 | * temp_priority is used to remember the scanning priority at which |
||
98 | * this zone was successfully refilled to free_pages == pages_high. |
||
99 | * |
||
100 | * Access to both these fields is quite racy even on uniprocessor. But |
||
101 | * it is expected to average out OK. |
||
102 | */ |
||
103 | int temp_priority; |
||
104 | int prev_priority; |
||
105 | |||
106 | /* |
||
107 | * free areas of different sizes |
||
108 | */ |
||
109 | struct free_area free_area[MAX_ORDER]; |
||
110 | |||
111 | /* |
||
112 | * wait_table -- the array holding the hash table |
||
113 | * wait_table_size -- the size of the hash table array |
||
114 | * wait_table_bits -- wait_table_size == (1 << wait_table_bits) |
||
115 | * |
||
116 | * The purpose of all these is to keep track of the people |
||
117 | * waiting for a page to become available and make them |
||
118 | * runnable again when possible. The trouble is that this |
||
119 | * consumes a lot of space, especially when so few things |
||
120 | * wait on pages at a given time. So instead of using |
||
121 | * per-page waitqueues, we use a waitqueue hash table. |
||
122 | * |
||
123 | * The bucket discipline is to sleep on the same queue when |
||
124 | * colliding and wake all in that wait queue when removing. |
||
125 | * When something wakes, it must check to be sure its page is |
||
126 | * truly available, a la thundering herd. The cost of a |
||
127 | * collision is great, but given the expected load of the |
||
128 | * table, they should be so rare as to be outweighed by the |
||
129 | * benefits from the saved space. |
||
130 | * |
||
131 | * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the |
||
132 | * primary users of these fields, and in mm/page_alloc.c |
||
133 | * free_area_init_core() performs the initialization of them. |
||
134 | */ |
||
135 | wait_queue_head_t * wait_table; |
||
136 | unsigned long wait_table_size; |
||
137 | unsigned long wait_table_bits; |
||
138 | |||
139 | ZONE_PADDING(_pad3_) |
||
140 | |||
141 | struct per_cpu_pageset pageset[NR_CPUS]; |
||
142 | |||
143 | /* |
||
144 | * Discontig memory support fields. |
||
145 | */ |
||
146 | struct pglist_data *zone_pgdat; |
||
147 | struct page *zone_mem_map; |
||
148 | /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ |
||
149 | unsigned long zone_start_pfn; |
||
150 | |||
151 | /* |
||
152 | * rarely used fields: |
||
153 | */ |
||
154 | char *name; |
||
155 | unsigned long spanned_pages; /* total size, including holes */ |
||
156 | unsigned long present_pages; /* amount of memory (excluding holes) */ |
||
157 | } ____cacheline_maxaligned_in_smp; |
||
158 | |||
159 | #define ZONE_DMA 0 |
||
160 | #define ZONE_NORMAL 1 |
||
161 | #define ZONE_HIGHMEM 2 |
||
162 | #define MAX_NR_ZONES 3 |
||
163 | #define GFP_ZONEMASK 0x03 |
||
164 | |||
165 | /* |
||
166 | * One allocation request operates on a zonelist. A zonelist |
||
167 | * is a list of zones, the first one is the 'goal' of the |
||
168 | * allocation, the other zones are fallback zones, in decreasing |
||
169 | * priority. |
||
170 | * |
||
171 | * Right now a zonelist takes up less than a cacheline. We never |
||
172 | * modify it apart from boot-up, and only a few indices are used, |
||
173 | * so despite the zonelist table being relatively big, the cache |
||
174 | * footprint of this construct is very small. |
||
175 | */ |
||
176 | struct zonelist { |
||
177 | struct zone *zones[MAX_NUMNODES * MAX_NR_ZONES + 1]; // NULL delimited |
||
178 | }; |
||
179 | |||
180 | |||
181 | /* |
||
182 | * The pg_data_t structure is used in machines with CONFIG_DISCONTIGMEM |
||
183 | * (mostly NUMA machines?) to denote a higher-level memory zone than the |
||
184 | * zone denotes. |
||
185 | * |
||
186 | * On NUMA machines, each NUMA node would have a pg_data_t to describe |
||
187 | * it's memory layout. |
||
188 | * |
||
189 | * Memory statistics and page replacement data structures are maintained on a |
||
190 | * per-zone basis. |
||
191 | */ |
||
192 | struct bootmem_data; |
||
193 | typedef struct pglist_data { |
||
194 | struct zone node_zones[MAX_NR_ZONES]; |
||
195 | struct zonelist node_zonelists[MAX_NR_ZONES]; |
||
196 | int nr_zones; |
||
197 | struct page *node_mem_map; |
||
198 | unsigned long *valid_addr_bitmap; |
||
199 | struct bootmem_data *bdata; |
||
200 | unsigned long node_start_pfn; |
||
201 | unsigned long node_present_pages; /* total number of physical pages */ |
||
202 | unsigned long node_spanned_pages; /* total size of physical page |
||
203 | range, including holes */ |
||
204 | int node_id; |
||
205 | struct pglist_data *pgdat_next; |
||
206 | wait_queue_head_t kswapd_wait; |
||
207 | } pg_data_t; |
||
208 | |||
209 | #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) |
||
210 | #define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages) |
||
211 | |||
212 | extern int numnodes; |
||
213 | extern struct pglist_data *pgdat_list; |
||
214 | |||
215 | void get_zone_counts(unsigned long *active, unsigned long *inactive, |
||
216 | unsigned long *free); |
||
217 | void build_all_zonelists(void); |
||
218 | void wakeup_kswapd(struct zone *zone); |
||
219 | |||
220 | /** |
||
221 | * for_each_pgdat - helper macro to iterate over all nodes |
||
222 | * @pgdat - pointer to a pg_data_t variable |
||
223 | * |
||
224 | * Meant to help with common loops of the form |
||
225 | * pgdat = pgdat_list; |
||
226 | * while(pgdat) { |
||
227 | * ... |
||
228 | * pgdat = pgdat->pgdat_next; |
||
229 | * } |
||
230 | */ |
||
231 | #define for_each_pgdat(pgdat) \ |
||
232 | for (pgdat = pgdat_list; pgdat; pgdat = pgdat->pgdat_next) |
||
233 | |||
234 | /* |
||
235 | * next_zone - helper magic for for_each_zone() |
||
236 | * Thanks to William Lee Irwin III for this piece of ingenuity. |
||
237 | */ |
||
238 | static inline struct zone *next_zone(struct zone *zone) |
||
239 | { |
||
240 | pg_data_t *pgdat = zone->zone_pgdat; |
||
241 | |||
242 | if (zone - pgdat->node_zones < MAX_NR_ZONES - 1) |
||
243 | zone++; |
||
244 | else if (pgdat->pgdat_next) { |
||
245 | pgdat = pgdat->pgdat_next; |
||
246 | zone = pgdat->node_zones; |
||
247 | } else |
||
248 | zone = NULL; |
||
249 | |||
250 | return zone; |
||
251 | } |
||
252 | |||
253 | /** |
||
254 | * for_each_zone - helper macro to iterate over all memory zones |
||
255 | * @zone - pointer to struct zone variable |
||
256 | * |
||
257 | * The user only needs to declare the zone variable, for_each_zone |
||
258 | * fills it in. This basically means for_each_zone() is an |
||
259 | * easier to read version of this piece of code: |
||
260 | * |
||
261 | * for (pgdat = pgdat_list; pgdat; pgdat = pgdat->node_next) |
||
262 | * for (i = 0; i < MAX_NR_ZONES; ++i) { |
||
263 | * struct zone * z = pgdat->node_zones + i; |
||
264 | * ... |
||
265 | * } |
||
266 | * } |
||
267 | */ |
||
268 | #define for_each_zone(zone) \ |
||
269 | for (zone = pgdat_list->node_zones; zone; zone = next_zone(zone)) |
||
270 | |||
271 | /** |
||
272 | * is_highmem - helper function to quickly check if a struct zone is a |
||
273 | * highmem zone or not. This is an attempt to keep references |
||
274 | * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum. |
||
275 | * @zone - pointer to struct zone variable |
||
276 | */ |
||
277 | static inline int is_highmem(struct zone *zone) |
||
278 | { |
||
279 | return (zone - zone->zone_pgdat->node_zones == ZONE_HIGHMEM); |
||
280 | } |
||
281 | |||
282 | /* These two functions are used to setup the per zone pages min values */ |
||
283 | struct ctl_table; |
||
284 | struct file; |
||
285 | int min_free_kbytes_sysctl_handler(struct ctl_table *, int, struct file *, |
||
286 | void *, size_t *); |
||
287 | extern void setup_per_zone_pages_min(void); |
||
288 | |||
289 | |||
290 | #ifdef CONFIG_NUMA |
||
291 | #define MAX_NR_MEMBLKS BITS_PER_LONG /* Max number of Memory Blocks */ |
||
292 | #else /* !CONFIG_NUMA */ |
||
293 | #define MAX_NR_MEMBLKS 1 |
||
294 | #endif /* CONFIG_NUMA */ |
||
295 | |||
296 | #include <linux/topology.h> |
||
297 | /* Returns the number of the current Node. */ |
||
298 | #define numa_node_id() (cpu_to_node(smp_processor_id())) |
||
299 | |||
300 | #ifndef CONFIG_DISCONTIGMEM |
||
301 | |||
302 | extern struct pglist_data contig_page_data; |
||
303 | #define NODE_DATA(nid) (&contig_page_data) |
||
304 | #define NODE_MEM_MAP(nid) mem_map |
||
305 | #define MAX_NODES_SHIFT 0 |
||
306 | |||
307 | #else /* CONFIG_DISCONTIGMEM */ |
||
308 | |||
309 | #include <asm/mmzone.h> |
||
310 | |||
311 | #if BITS_PER_LONG == 32 |
||
312 | /* |
||
313 | * with 32 bit flags field, page->zone is currently 8 bits. |
||
314 | * there are 3 zones (2 bits) and this leaves 8-2=6 bits for nodes. |
||
315 | */ |
||
316 | #define MAX_NODES_SHIFT 6 |
||
317 | #elif BITS_PER_LONG == 64 |
||
318 | /* |
||
319 | * with 64 bit flags field, there's plenty of room. |
||
320 | */ |
||
321 | #define MAX_NODES_SHIFT 10 |
||
322 | #endif |
||
323 | |||
324 | #endif /* !CONFIG_DISCONTIGMEM */ |
||
325 | |||
326 | #if NODES_SHIFT > MAX_NODES_SHIFT |
||
327 | #error NODES_SHIFT > MAX_NODES_SHIFT |
||
328 | #endif |
||
329 | |||
330 | extern DECLARE_BITMAP(node_online_map, MAX_NUMNODES); |
||
331 | extern DECLARE_BITMAP(memblk_online_map, MAX_NR_MEMBLKS); |
||
332 | |||
333 | #if defined(CONFIG_DISCONTIGMEM) || defined(CONFIG_NUMA) |
||
334 | |||
335 | #define node_online(node) test_bit(node, node_online_map) |
||
336 | #define node_set_online(node) set_bit(node, node_online_map) |
||
337 | #define node_set_offline(node) clear_bit(node, node_online_map) |
||
338 | static inline unsigned int num_online_nodes(void) |
||
339 | { |
||
340 | int i, num = 0; |
||
341 | |||
342 | for(i = 0; i < MAX_NUMNODES; i++){ |
||
343 | if (node_online(i)) |
||
344 | num++; |
||
345 | } |
||
346 | return num; |
||
347 | } |
||
348 | |||
349 | #define memblk_online(memblk) test_bit(memblk, memblk_online_map) |
||
350 | #define memblk_set_online(memblk) set_bit(memblk, memblk_online_map) |
||
351 | #define memblk_set_offline(memblk) clear_bit(memblk, memblk_online_map) |
||
352 | static inline unsigned int num_online_memblks(void) |
||
353 | { |
||
354 | int i, num = 0; |
||
355 | |||
356 | for(i = 0; i < MAX_NR_MEMBLKS; i++){ |
||
357 | if (memblk_online(i)) |
||
358 | num++; |
||
359 | } |
||
360 | return num; |
||
361 | } |
||
362 | |||
363 | #else /* !CONFIG_DISCONTIGMEM && !CONFIG_NUMA */ |
||
364 | |||
365 | #define node_online(node) \ |
||
366 | ({ BUG_ON((node) != 0); test_bit(node, node_online_map); }) |
||
367 | #define node_set_online(node) \ |
||
368 | ({ BUG_ON((node) != 0); set_bit(node, node_online_map); }) |
||
369 | #define node_set_offline(node) \ |
||
370 | ({ BUG_ON((node) != 0); clear_bit(node, node_online_map); }) |
||
371 | #define num_online_nodes() 1 |
||
372 | |||
373 | #define memblk_online(memblk) \ |
||
374 | ({ BUG_ON((memblk) != 0); test_bit(memblk, memblk_online_map); }) |
||
375 | #define memblk_set_online(memblk) \ |
||
376 | ({ BUG_ON((memblk) != 0); set_bit(memblk, memblk_online_map); }) |
||
377 | #define memblk_set_offline(memblk) \ |
||
378 | ({ BUG_ON((memblk) != 0); clear_bit(memblk, memblk_online_map); }) |
||
379 | #define num_online_memblks() 1 |
||
380 | |||
381 | #endif /* CONFIG_DISCONTIGMEM || CONFIG_NUMA */ |
||
382 | #endif /* !__ASSEMBLY__ */ |
||
383 | #endif /* __KERNEL__ */ |
||
384 | #endif /* _LINUX_MMZONE_H */ |