From 44421ea3dec5500b6a73a921948d1ab0b4223fce Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 26 Jul 2007 18:26:19 -0700 Subject: [PATCH] Misc --- include/asm-generic/local.h | 30 ++++++++++++----- mm/slub.c | 77 ++++++++++++++++++++++++++++++------------ 2 files changed, 76 insertions(+), 31 deletions(-) diff --git a/include/asm-generic/local.h b/include/asm-generic/local.h index 0c966f1..e715fcf 100644 --- a/include/asm-generic/local.h +++ b/include/asm-generic/local.h @@ -46,6 +46,23 @@ typedef struct #define local_add_unless(l, a, u) atomic_long_add_unless((&(l)->a), (a), (u)) #define local_inc_not_zero(l) atomic_long_inc_not_zero(&(l)->a) +#ifdef ARCH_HAS_CMPXCHG + +#define local_begin(flags) \ +{ flags = 0; \ + preempt_enable(); \ +} + +static inline void local_end(unsigned long flags) +{ + preempt_disable(); +} + +#define local_up_irq(flags) local_irq_disable(flags); +#define local_down_irq(flags) local_irq_enable(flags); +#define cmpxchg_local cmpxchg +#else + /* * Establish a state necessary for __local_xx functions to work. */ @@ -68,10 +85,13 @@ static inline void local_end(unsigned long flags) /* * Back from interrupt to local ops */ -static inline void #define local_down_irq(unsigned long flags) +static inline void local_down_irq(unsigned long flags) { } +#define cmpxchg_local(v, o, n) (*(v) = (n), (o)) +#endif + /* * Non-atomic variants, ie. within local_begin() / local_end() or * preempt_disable / enable() and won't be touched in interrupt, etc. @@ -82,14 +102,6 @@ static inline void #define local_down_irq(unsigned long flags) #define __local_add(i,l) local_set((l), local_read(l) + (i)) #define __local_sub(i,l) local_set((l), local_read(l) - (i)) -#define __local_cmpxchg((v), (o), (n)) (*(v) = (n), (o)) -#define __local_xchg((v), (n)) \ -({ \ - __typeof(v) x = *(v); \ - *(v) = (n); \ - x; \ -)} - /* Use these for per-cpu local_t variables: on some archs they are * much more efficient than these naive implementations. Note they take * a variable (eg. mystruct.foo), not an address. diff --git a/mm/slub.c b/mm/slub.c index 9b2d617..5fce282 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1452,16 +1452,20 @@ static void flush_all(struct kmem_cache *s) * And if we were unable to get a new slab from the partial slab lists then * we need to allocate a new slab. This is slowest path since we may sleep. */ -static void *__slab_alloc(struct kmem_cache *s, - gfp_t gfpflags, int node, void *addr, struct page *page) +static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, + void *addr, struct page *page, unsigned long pflags) { void **object; int cpu = smp_processor_id(); + unsigned long flags; + local_irq_disable(); if (!page) goto new_slab; slab_lock(page); + if (page->lockless_freelist) + goto another_slab; if (unlikely(node != -1 && page_to_nid(page) != node)) goto another_slab; load_freelist: @@ -1475,7 +1479,11 @@ load_freelist: page->lockless_freelist = object[page->offset]; page->inuse = s->objects; page->freelist = NULL; +out: slab_unlock(page); + local_end(pflags); + if (unlikely(gfpflags & __GFP_ZERO)) + memset(object, 0, s->objsize); return object; another_slab: @@ -1518,6 +1526,7 @@ new_slab: s->cpu_slab[cpu] = page; goto load_freelist; } + local_end(pflags); return NULL; debug: object = page->freelist; @@ -1526,8 +1535,7 @@ debug: page->inuse++; page->freelist = object[page->offset]; - slab_unlock(page); - return object; + goto out; } /* @@ -1547,23 +1555,34 @@ static void __always_inline *slab_alloc(struct kmem_cache *s, void **object; unsigned long flags; - local_irq_save(flags); + local_begin(flags); +redo: page = s->cpu_slab[smp_processor_id()]; - if (unlikely(!page || !page->lockless_freelist || - (node != -1 && page_to_nid(page) != node))) + if (unlikely(!page)) + goto slow; - object = __slab_alloc(s, gfpflags, node, addr, page); + object = page->lockless_freelist; + if (unlikely(!object)) + goto slow; - else { - object = page->lockless_freelist; - page->lockless_freelist = object[page->offset]; - } - local_irq_restore(flags); +#ifdef CONFIG_NUMA + if (node != -1 && page_to_nid(page) != node) + goto slow; +#endif + + if (cmpxchg_local(&page->lockless_freelist, object, + object[page->offset]) != object) + goto redo; + + local_end(flags); - if (unlikely((gfpflags & __GFP_ZERO) && object)) + if (unlikely((gfpflags & __GFP_ZERO))) memset(object, 0, s->objsize); return object; + +slow: + return __slab_alloc(s, gfpflags, node, addr, page, flags); } void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) @@ -1593,7 +1612,9 @@ static void __slab_free(struct kmem_cache *s, struct page *page, { void *prior; void **object = (void *)x; + unsigned long flags; + local_up_irq(flags); slab_lock(page); if (unlikely(SlabDebug(page))) @@ -1619,6 +1640,7 @@ checks_ok: out_unlock: slab_unlock(page); + local_down_irq(flags); return; slab_empty: @@ -1629,6 +1651,7 @@ slab_empty: remove_partial(s, page); slab_unlock(page); + local_down_irq(flags); discard_slab(s, page); return; @@ -1654,16 +1677,26 @@ static void __always_inline slab_free(struct kmem_cache *s, { void **object = (void *)x; unsigned long flags; + void **ll; - local_irq_save(flags); - if (likely(page == s->cpu_slab[smp_processor_id()] && - !SlabDebug(page))) { - object[page->offset] = page->lockless_freelist; - page->lockless_freelist = object; - } else - __slab_free(s, page, x, addr); + local_begin(flags); +redo: + if (unlikely(page != s->cpu_slab[smp_processor_id()])) + goto slow; + if (unlikely(SlabDebug(page))) + goto slow; - local_irq_restore(flags); + ll = page->lockless_freelist; + object[page->offset] = ll; + if (cmpxchg_local(&page->lockless_freelist, ll, object) != object) + goto redo; + +out: + local_end(flags); + return; +slow: + __slab_free(s, page, x, addr); + goto out; } void kmem_cache_free(struct kmem_cache *s, void *x) -- 1.4.4.4