Index: linux-2.6.16-rc2/mm/vmscan.c =================================================================== --- linux-2.6.16-rc2.orig/mm/vmscan.c 2006-02-04 20:24:00.000000000 -0800 +++ linux-2.6.16-rc2/mm/vmscan.c 2006-02-04 20:55:33.000000000 -0800 @@ -58,14 +58,6 @@ struct scan_control { /* Incremented by the number of pages reclaimed */ unsigned long nr_reclaimed; - /* This context's GFP mask */ - gfp_t gfp_mask; - - int may_writepage; - - /* Can pages be swapped as part of reclaim? */ - int may_swap; - /* This context's SWAP_CLUSTER_MAX. If freeing memory for * suspend, we effectively ignore SWAP_CLUSTER_MAX. * In this context, it doesn't matter that we scan the @@ -74,6 +66,28 @@ struct scan_control { }; /* + * Some additional flags to be added to gfp_t in the context + * of swap processing. + */ +#define MAY_SWAP (1 << __GFP_BITS_SHIFT) +#define MAY_WRITEPAGE (1 << (__GFP_BITS_SHIFT + 1)) + +/* + * init_scan_control prepares scan control with the proper settings + * + * nr_pages is the minimum number of pages we intend to free in the scan. + */ +static inline void init_scan_control(struct scan_control *sc, int nr_pages) +{ + sc->nr_scanned = 0; + sc->nr_reclaimed = 0; + if (nr_pages > SWAP_CLUSTER_MAX) + sc->swap_cluster_max = nr_pages; + else + sc->swap_cluster_max = SWAP_CLUSTER_MAX; +} + +/* * The list of shrinker callbacks used by to apply pressure to * ageable caches. */ @@ -408,7 +422,8 @@ cannot_free: /* * shrink_list adds the number of reclaimed pages to sc->nr_reclaimed */ -static int shrink_list(struct list_head *page_list, struct scan_control *sc) +static int shrink_list(struct list_head *page_list, gfp_t gfp_mask, + struct scan_control *sc) { LIST_HEAD(ret_pages); struct pagevec freed_pvec; @@ -453,7 +468,7 @@ static int shrink_list(struct list_head * Try to allocate it some swap space here. */ if (PageAnon(page) && !PageSwapCache(page)) { - if (!sc->may_swap) + if (!(gfp_mask & MAY_SWAP)) goto keep_locked; if (!add_to_swap(page, GFP_ATOMIC)) goto activate_locked; @@ -461,8 +476,8 @@ static int shrink_list(struct list_head #endif /* CONFIG_SWAP */ mapping = page_mapping(page); - may_enter_fs = (sc->gfp_mask & __GFP_FS) || - (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO)); + may_enter_fs = (gfp_mask & __GFP_FS) || + (PageSwapCache(page) && (gfp_mask & __GFP_IO)); /* * The page is mapped into the page tables of one or more @@ -472,7 +487,7 @@ static int shrink_list(struct list_head /* * No unmapping if we do not swap */ - if (!sc->may_swap) + if (!(gfp_mask & MAY_SWAP)) goto keep_locked; switch (try_to_unmap(page, 0)) { @@ -490,7 +505,7 @@ static int shrink_list(struct list_head goto keep_locked; if (!may_enter_fs) goto keep_locked; - if (!sc->may_writepage) + if (!(gfp_mask & MAY_WRITEPAGE)) goto keep_locked; /* Page is dirty, try to write it out here */ @@ -538,7 +553,7 @@ static int shrink_list(struct list_head * Otherwise, leave the page on the LRU so it is swappable. */ if (PagePrivate(page)) { - if (!try_to_release_page(page, sc->gfp_mask)) + if (!try_to_release_page(page, gfp_mask)) goto activate_locked; if (!mapping && page_count(page) == 1) goto free_it; @@ -1076,7 +1091,8 @@ static int isolate_lru_pages(int nr_to_s /* * shrink_cache() adds the number of pages reclaimed to sc->nr_reclaimed */ -static void shrink_cache(int max_scan, struct zone *zone, struct scan_control *sc) +static void shrink_cache(int max_scan, struct zone *zone, gfp_t gfp_mask, + struct scan_control *sc) { LIST_HEAD(page_list); struct pagevec pvec; @@ -1102,7 +1118,7 @@ static void shrink_cache(int max_scan, s goto done; max_scan -= nr_scan; - nr_freed = shrink_list(&page_list, sc); + nr_freed = shrink_list(&page_list, gfp_mask, sc); local_irq_disable(); if (current_is_kswapd()) { @@ -1286,7 +1302,7 @@ refill_inactive_zone(int nr_pages, struc * This is a basic per-zone page freer. Used by both kswapd and direct reclaim. */ static void -shrink_zone(int priority, struct zone *zone, struct scan_control *sc) +shrink_zone(int priority, struct zone *zone, gfp_t gfp_mask, struct scan_control *sc) { unsigned long nr_active; unsigned long nr_inactive; @@ -1324,7 +1340,7 @@ shrink_zone(int priority, struct zone *z nr_to_scan = min(nr_inactive, (unsigned long)sc->swap_cluster_max); nr_inactive -= nr_to_scan; - shrink_cache(nr_to_scan, zone, sc); + shrink_cache(nr_to_scan, zone, gfp_mask, sc); } } @@ -1350,7 +1366,8 @@ shrink_zone(int priority, struct zone *z * scan then give up on it. */ static void -shrink_caches(int priority, struct zone **zones, struct scan_control *sc) +shrink_caches(int priority, struct zone **zones, gfp_t gfp_mask, + struct scan_control *sc) { int i; @@ -1370,7 +1387,7 @@ shrink_caches(int priority, struct zone if (zone->all_unreclaimable && priority != DEF_PRIORITY) continue; /* Let kswapd poll it */ - shrink_zone(priority, zone, sc); + shrink_zone(priority, zone, gfp_mask, sc); } } @@ -1393,13 +1410,12 @@ int try_to_free_pages(struct zone **zone int ret = 0; int total_scanned = 0, total_reclaimed = 0; struct reclaim_state *reclaim_state = current->reclaim_state; - struct scan_control sc; unsigned long lru_pages = 0; int i; - sc.gfp_mask = gfp_mask; - sc.may_writepage = !laptop_mode; - sc.may_swap = 1; + gfp_mask |= MAY_SWAP; + if (!laptop_mode) + gfp_mask |= MAY_WRITEPAGE; inc_page_state(allocstall); @@ -1414,12 +1430,13 @@ int try_to_free_pages(struct zone **zone } for (priority = DEF_PRIORITY; priority >= 0; priority--) { - sc.nr_scanned = 0; - sc.nr_reclaimed = 0; - sc.swap_cluster_max = SWAP_CLUSTER_MAX; + struct scan_control sc; + + init_scan_control(&sc, SWAP_CLUSTER_MAX); + if (!priority) disable_swap_token(); - shrink_caches(priority, zones, &sc); + shrink_caches(priority, zones, gfp_mask, &sc); shrink_slab(sc.nr_scanned, gfp_mask, lru_pages); if (reclaim_state) { sc.nr_reclaimed += reclaim_state->reclaimed_slab; @@ -1441,7 +1458,7 @@ int try_to_free_pages(struct zone **zone */ if (total_scanned > sc.swap_cluster_max + sc.swap_cluster_max/2) { wakeup_pdflush(laptop_mode ? 0 : total_scanned); - sc.may_writepage = 1; + gfp_mask |= MAY_WRITEPAGE; } /* Take a nap, wait for some writeback to complete */ @@ -1493,14 +1510,14 @@ static int balance_pgdat(pg_data_t *pgda int i; int total_scanned, total_reclaimed; struct reclaim_state *reclaim_state = current->reclaim_state; - struct scan_control sc; + gfp_t gfp_mask; loop_again: total_scanned = 0; total_reclaimed = 0; - sc.gfp_mask = GFP_KERNEL; - sc.may_writepage = !laptop_mode; - sc.may_swap = 1; + gfp_mask = GFP_KERNEL | MAY_SWAP; + if (!laptop_mode) + gfp_mask |= MAY_WRITEPAGE; inc_page_state(pageoutrun); @@ -1563,6 +1580,7 @@ scan: */ for (i = 0; i <= end_zone; i++) { struct zone *zone = pgdat->node_zones + i; + struct scan_control sc; int nr_slab; if (!populated_zone(zone)) @@ -1579,11 +1597,11 @@ scan: zone->temp_priority = priority; if (zone->prev_priority > priority) zone->prev_priority = priority; - sc.nr_scanned = 0; - sc.nr_reclaimed = 0; - sc.swap_cluster_max = nr_pages? nr_pages : SWAP_CLUSTER_MAX; + + init_scan_control(&sc, nr_pages); + atomic_inc(&zone->reclaim_in_progress); - shrink_zone(priority, zone, &sc); + shrink_zone(priority, zone, gfp_mask, &sc); atomic_dec(&zone->reclaim_in_progress); reclaim_state->reclaimed_slab = 0; nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL, @@ -1603,7 +1621,7 @@ scan: */ if (total_scanned > SWAP_CLUSTER_MAX * 2 && total_scanned > total_reclaimed+total_reclaimed/2) - sc.may_writepage = 1; + gfp_mask |= MAY_WRITEPAGE; } if (nr_pages && to_free > total_reclaimed) continue; /* swsusp: need to do more work */ @@ -1855,19 +1873,15 @@ int zone_reclaim(struct zone *zone, gfp_ if (!cpus_empty(mask) && node_id != numa_node_id()) return 0; - sc.may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE); - sc.may_swap = !!(zone_reclaim_mode & RECLAIM_SWAP); - sc.nr_scanned = 0; - sc.nr_reclaimed = 0; - sc.gfp_mask = gfp_mask; + if (zone_reclaim_mode & RECLAIM_WRITE) + gfp_mask |= MAY_WRITEPAGE; + if (zone_reclaim_mode & RECLAIM_SWAP) + gfp_mask |= MAY_SWAP; disable_swap_token(); nr_pages = 1 << order; - if (nr_pages > SWAP_CLUSTER_MAX) - sc.swap_cluster_max = nr_pages; - else - sc.swap_cluster_max = SWAP_CLUSTER_MAX; + init_scan_control(&sc, nr_pages); cond_resched(); p->flags |= PF_MEMALLOC; @@ -1881,7 +1895,7 @@ int zone_reclaim(struct zone *zone, gfp_ for (priority = ZONE_RECLAIM_PRIORITY; priority >=0 && sc.nr_reclaimed < nr_pages; priority--) - shrink_zone(priority, zone, &sc); + shrink_zone(priority, zone, gfp_mask, &sc); if (sc.nr_reclaimed < nr_pages && (zone_reclaim_mode & RECLAIM_SLAB)) {