[RFC,PATCH] optimize fixed size kmalloc calls

Manfred Spraul (manfred@colorfullife.com)
Mon, 09 Jun 2003 21:38:38 +0200


This is a MIME-formatted message. If you see this text it means that your
E-mail software does not support MIME-formatted messages.

--=_courier-25814-1055187578-0001-2
Content-Type: text/plain; charset=iso-8859-1; format=flowed
Content-Transfer-Encoding: 7bit

Hi,

kmalloc(constant,GFP_KERNEL) spends a singificant amount of time in the
initial loop that finds the correct cache. For constant allocations, the
correct cache can be indentified at compile time.

What do you think about the attached patch? It's not pretty, but it
should work will all gcc versions. Any other ideas?
Just FYI: The function that forced me to use use switch/case instead of
if is interrupts_open in fs/proc/proc_misc.c.

--
    Manfred

--=_courier-25814-1055187578-0001-2 Content-Type: text/plain; name=patch-kmalloc-fixed; charset=iso-8859-1 Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="patch-kmalloc-fixed"

// $Header$ // Kernel Version: // VERSION = 2 // PATCHLEVEL = 5 // SUBLEVEL = 70 // EXTRAVERSION = -mm4 --- 2.5/include/linux/slab.h 2003-06-08 23:32:03.000000000 +0200 +++ build-2.5/include/linux/slab.h 2003-06-08 23:30:41.000000000 +0200 @@ -62,7 +62,52 @@ extern void kmem_cache_free(kmem_cache_t *, void *); extern unsigned int kmem_cache_size(kmem_cache_t *); -extern void *kmalloc(size_t, int); +/* Size description struct for general caches. */ +struct cache_sizes { + size_t cs_size; + kmem_cache_t *cs_cachep; + kmem_cache_t *cs_dmacachep; +}; +extern struct cache_sizes malloc_sizes[]; +extern void *__kmalloc(size_t, int); + +/* + * gcc's brain is lossy: is forgets that a number is known at compile + * time after a few accesses and produces bogus code if a sequence of + * if clauses is used. This is avoided by using select. + */ +static inline void * kmalloc(size_t size, int flags) +{ + if (__builtin_constant_p(size)) { +extern void __you_cannot_kmalloc_that_much(void); + unsigned int i,j; + j = 0; + switch(size) { + case 0 ... +#define CACHE(x) \ + (x): j++; \ + case (x+1) ... +#define LCACHE(x) \ + (x): j++; break; +#include "kmalloc_sizes.h" +#undef CACHE +#undef LCACHE + default: + __you_cannot_kmalloc_that_much(); + } + i = 0; +#define CACHE(x) \ + i++; +#include "kmalloc_sizes.h" +#undef CACHE + return kmem_cache_alloc( (flags & GFP_DMA)? + malloc_sizes[i-j].cs_dmacachep + : malloc_sizes[i-j].cs_cachep, + flags); + } + return __kmalloc(size,flags); +} + extern void kfree(const void *); extern unsigned int ksize(const void *); --- 2.5/mm/slab.c 2003-06-08 23:32:06.000000000 +0200 +++ build-2.5/mm/slab.c 2003-06-07 21:57:59.000000000 +0200 @@ -388,11 +388,7 @@ #define GET_PAGE_SLAB(pg) ((struct slab *)(pg)->list.prev) /* These are the default caches for kmalloc. Custom caches can have other sizes. */ -static struct cache_sizes { - size_t cs_size; - kmem_cache_t *cs_cachep; - kmem_cache_t *cs_dmacachep; -} malloc_sizes[] = { +struct cache_sizes malloc_sizes[] = { #define CACHE(x) { .cs_size = (x) }, #include <linux/kmalloc_sizes.h> { 0, } @@ -2039,7 +2035,7 @@ * platforms. For example, on i386, it means that the memory must come * from the first 16MB. */ -void * kmalloc (size_t size, int flags) +void * __kmalloc (size_t size, int flags) { struct cache_sizes *csizep = malloc_sizes; --- 2.5/kernel/ksyms.c 2003-06-08 23:32:04.000000000 +0200 +++ build-2.5/kernel/ksyms.c 2003-06-07 21:56:55.000000000 +0200 @@ -95,7 +95,8 @@ EXPORT_SYMBOL(kmem_cache_size); EXPORT_SYMBOL(set_shrinker); EXPORT_SYMBOL(remove_shrinker); -EXPORT_SYMBOL(kmalloc); +EXPORT_SYMBOL(malloc_sizes); +EXPORT_SYMBOL(__kmalloc); EXPORT_SYMBOL(kfree); #ifdef CONFIG_SMP EXPORT_SYMBOL(__alloc_percpu); --- 2.5/include/linux/kmalloc_sizes.h 2003-06-05 19:01:42.000000000 +0200 +++ build-2.5/include/linux/kmalloc_sizes.h 2003-06-08 23:23:11.000000000 +0200 @@ -1,3 +1,14 @@ +/* + * kmalloc cache sizes. + * - CACHE(x) is called for every entry except the last + * - for the last entry, LCACHE is called. LCACHE defaults + * to CACHE. + */ +#ifndef LCACHE +#define LCACHE(x) CACHE(x) +#define __LCACHE_DEFINED +#endif + #if (PAGE_SIZE == 4096) CACHE(32) #endif @@ -18,16 +29,25 @@ CACHE(16384) CACHE(32768) CACHE(65536) +#ifdef CONFIG_MMU + LCACHE(131072) +#else CACHE(131072) -#ifndef CONFIG_MMU CACHE(262144) CACHE(524288) +#ifndef CONFIG_LARGE_ALLOCS + LCACHE(1048576) +#else CACHE(1048576) -#ifdef CONFIG_LARGE_ALLOCS CACHE(2097152) CACHE(4194304) CACHE(8388608) CACHE(16777216) - CACHE(33554432) + LCACHE(33554432) #endif /* CONFIG_LARGE_ALLOCS */ #endif /* CONFIG_MMU */ + +#ifdef __LCACHE_DEFINED +#undef __LCACHE_DEFINIED +#undef LCACHE +#endif

--=_courier-25814-1055187578-0001-2--