--- include/linux/slub_def.h | 1 + mm/slub.c | 32 ++++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 10 deletions(-) Index: linux-2.6/include/linux/slub_def.h =================================================================== --- linux-2.6.orig/include/linux/slub_def.h 2008-02-07 23:36:54.259439791 -0800 +++ linux-2.6/include/linux/slub_def.h 2008-02-07 23:39:16.561015495 -0800 @@ -34,6 +34,7 @@ enum stat_item { struct kmem_cache_cpu { void **freelist; /* Pointer to first free per cpu object */ struct page *page; /* The slab from which we are allocating */ + unsigned long mask; /* mask to get page from object */ int node; /* The node of the page (or -1 for debug) */ unsigned int offset; /* Freepointer offset (in word units) */ unsigned int objsize; /* Size of an object (from kmem_cache) */ Index: linux-2.6/mm/slub.c =================================================================== --- linux-2.6.orig/mm/slub.c 2008-02-07 23:36:54.271439913 -0800 +++ linux-2.6/mm/slub.c 2008-02-07 23:38:40.984639388 -0800 @@ -1542,7 +1542,6 @@ load_freelist: c->freelist = object[c->offset]; c->page->inuse = s->objects; c->page->freelist = c->page->end; - c->node = page_to_nid(c->page); unlock_out: slab_unlock(c->page); stat(c, ALLOC_SLOWPATH); @@ -1558,9 +1557,8 @@ another_slab: new_slab: new = get_partial(s, gfpflags, node); if (new) { - c->page = new; stat(c, ALLOC_FROM_PARTIAL); - goto load_freelist; + goto loadnew; } if (gfpflags & __GFP_WAIT) @@ -1578,7 +1576,9 @@ new_slab: flush_slab(s, c); slab_lock(new); SetSlabFrozen(new); +loadnew: c->page = new; + c->node = page_to_nid(c->page); goto load_freelist; } object = NULL; @@ -1693,6 +1693,9 @@ static void __slab_free(struct kmem_cach local_irq_save(flags); #endif + if (!page) + page = virt_to_head_page(x); + c = get_cpu_slab(s, raw_smp_processor_id()); stat(c, FREE_SLOWPATH); slab_lock(page); @@ -1752,6 +1755,18 @@ debug: } /* + * Determine if a given address matches the current cpu slab + */ +static inline int match_addr(struct kmem_cache_cpu *c, void *x) +{ + if (!c->page) + return 0; + + return (((unsigned long)x ^ (unsigned long)c->page->end) + & c->mask) == 0; +} + +/* * Fastpath with forced inlining to produce a kfree and kmem_cache_free that * can perform fastpath freeing without additional function calls. * @@ -1787,7 +1802,7 @@ static __always_inline void slab_free(st * then any change of cpu_slab will cause the cmpxchg to fail * since the freelist pointers are unique per slab. */ - if (unlikely(page != c->page || c->node < 0)) { + if (unlikely(!match_addr(c, x) || c->node < 0)) { __slab_free(s, page, x, addr, c->offset); break; } @@ -1800,7 +1815,7 @@ static __always_inline void slab_free(st local_irq_save(flags); debug_check_no_locks_freed(object, s->objsize); c = get_cpu_slab(s, smp_processor_id()); - if (likely(page == c->page && c->node >= 0)) { + if (likely(match_addr(c, x) && c->node >= 0)) { object[c->offset] = c->freelist; c->freelist = object; stat(c, FREE_FASTPATH); @@ -1813,11 +1828,7 @@ static __always_inline void slab_free(st void kmem_cache_free(struct kmem_cache *s, void *x) { - struct page *page; - - page = virt_to_head_page(x); - - slab_free(s, page, x, __builtin_return_address(0)); + slab_free(s, NULL, x, __builtin_return_address(0)); } EXPORT_SYMBOL(kmem_cache_free); @@ -1989,6 +2000,7 @@ static void init_kmem_cache_cpu(struct k c->node = 0; c->offset = s->offset / sizeof(void *); c->objsize = s->objsize; + c->mask = ~(PAGE_SIZE << s->order) - 1; } static void init_kmem_cache_node(struct kmem_cache_node *n)