From 9a468453aa78258840370481dbc618eb5b0a2e97 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 26 Jul 2007 14:13:09 -0700 Subject: [PATCH] vmstat: Use local operations instead of disabling / enabling interrupts Signed-off-by: Christoph Lameter --- include/asm-i386/local.h | 1 + mm/vmstat.c | 97 ++++++++++++++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 33 deletions(-) diff --git a/include/asm-i386/local.h b/include/asm-i386/local.h index f38a9a3..78cf5ab 100644 --- a/include/asm-i386/local.h +++ b/include/asm-i386/local.h @@ -196,6 +196,7 @@ static __inline__ long local_sub_return(long i, local_t *l) #define local_begin(__flags) \ { \ + __flags = 0; /* Compiler will optimize this away */ \ preempt_disable(); \ } diff --git a/mm/vmstat.c b/mm/vmstat.c index fadf791..ccc477a 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -158,15 +158,25 @@ void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, { struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); s8 *p = pcp->vm_stat_diff + item; - long x; + s8 o; + unsigned long n; + unsigned long add; + +redo: + add = 0; + o = *p; + n = delta + *p; + + if (unlikely(n > pcp->stat_threshold || n < -pcp->stat_threshold)) { + add = n; + n = 0; + } - x = delta + *p; + if (cmpxchg_local(p, o, n) != o) + goto redo; - if (unlikely(x > pcp->stat_threshold || x < -pcp->stat_threshold)) { - zone_page_state_add(x, zone, item); - x = 0; - } - *p = x; + if (add) + zone_page_state_add(add, zone, item); } EXPORT_SYMBOL(__mod_zone_page_state); @@ -178,9 +188,9 @@ void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, { unsigned long flags; - local_irq_save(flags); + local_begin(flags); __mod_zone_page_state(zone, item, delta); - local_irq_restore(flags); + local_end(flags); } EXPORT_SYMBOL(mod_zone_page_state); @@ -211,15 +221,25 @@ void __inc_zone_state(struct zone *zone, enum zone_stat_item item) { struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); s8 *p = pcp->vm_stat_diff + item; + int overstep; + s8 o,n; - (*p)++; - - if (unlikely(*p > pcp->stat_threshold)) { - int overstep = pcp->stat_threshold / 2; +redo: + overstep = 0; + o = *p; + n = o + 1; - zone_page_state_add(*p + overstep, zone, item); - *p = -overstep; + if (unlikely(n > pcp->stat_threshold)) { + overstep = pcp->stat_threshold / 2; + n = -overstep; } + + if (cmpxchg_local(p, o, n) != o) + goto redo; + + if (overstep) + zone_page_state_add(o + 1 + overstep, + zone, item); } void __inc_zone_page_state(struct page *page, enum zone_stat_item item) @@ -232,15 +252,25 @@ void __dec_zone_state(struct zone *zone, enum zone_stat_item item) { struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); s8 *p = pcp->vm_stat_diff + item; + int overstep; + s8 o,n; - (*p)--; - - if (unlikely(*p < - pcp->stat_threshold)) { - int overstep = pcp->stat_threshold / 2; +redo: + overstep = 0; + o = *p; + n = o - 1; - zone_page_state_add(*p - overstep, zone, item); - *p = overstep; + if (unlikely(n < - pcp->stat_threshold)) { + overstep = pcp->stat_threshold / 2; + n = overstep; } + + if (cmpxchg_local(p, o, n) != 0) + goto redo; + + if (overstep) + zone_page_state_add(o - 1 - overstep, + zone, item); } void __dec_zone_page_state(struct page *page, enum zone_stat_item item) @@ -253,20 +283,20 @@ void inc_zone_state(struct zone *zone, enum zone_stat_item item) { unsigned long flags; - local_irq_save(flags); + local_begin(flags); __inc_zone_state(zone, item); - local_irq_restore(flags); + local_end(flags); } void inc_zone_page_state(struct page *page, enum zone_stat_item item) { - unsigned long flags; struct zone *zone; + unsigned long flags; zone = page_zone(page); - local_irq_save(flags); + local_begin(flags); __inc_zone_state(zone, item); - local_irq_restore(flags); + local_end(flags); } EXPORT_SYMBOL(inc_zone_page_state); @@ -274,9 +304,9 @@ void dec_zone_page_state(struct page *page, enum zone_stat_item item) { unsigned long flags; - local_irq_save(flags); + local_begin(flags); __dec_zone_page_state(page, item); - local_irq_restore(flags); + local_end(flags); } EXPORT_SYMBOL(dec_zone_page_state); @@ -310,15 +340,16 @@ void refresh_cpu_vm_stats(int cpu) for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) if (p->vm_stat_diff[i]) { - local_irq_save(flags); - zone_page_state_add(p->vm_stat_diff[i], - zone, i); - p->vm_stat_diff[i] = 0; + int diff; + + local_begin(flags); + diff = __local_xchg(&p->vm_stat_diff[i], 0); + zone_page_state_add(diff, zone, i); #ifdef CONFIG_NUMA /* 3 seconds idle till flush */ p->expire = 3; #endif - local_irq_restore(flags); + local_end(flags); } #ifdef CONFIG_NUMA /* -- 1.4.4.4