--------------------- PatchSet 1990 Date: 2001/04/19 23:53:39 Author: akroonmaa Branch: chunked_mempools Tag: (none) Log: Major change. Implemented freelist cache that traverses all chunks as linklist. major performance improvement. Pooled alloc/free are now within 100-200 cpu ticks on average. Skipped all overhead of finding and returning freed objects to their own chunk, this is left for regular event cleanup handler, meant to run every 15-30 secs and imposing notable overhead only if we are over idle mempool limits. Cleanup handler not yet implemented, so mempools do not shrink, yet. So, we have an allocator with near-zero ram overhead and very fast. Members: include/MemPool.h:1.1.2.4->1.1.2.5 include/util.h:1.7.16.1->1.7.16.2 lib/MemPool.c:1.1.2.3->1.1.2.4 src/MemPoolStats.c:1.1.2.5->1.1.2.6 src/stat.c:1.9.8.1->1.9.8.2 Index: squid/include/MemPool.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/Attic/MemPool.h,v retrieving revision 1.1.2.4 retrieving revision 1.1.2.5 diff -u -r1.1.2.4 -r1.1.2.5 --- squid/include/MemPool.h 18 Apr 2001 18:17:43 -0000 1.1.2.4 +++ squid/include/MemPool.h 19 Apr 2001 23:53:39 -0000 1.1.2.5 @@ -53,8 +53,13 @@ int capacity; int memPID; int chunkCount; + size_t alloc_calls; + size_t free_calls; + size_t inuse; + size_t idle; + void *freeCache; + MemChunk *nextFreeChunk; MemChunk *Chunks; - MemChunk *nextbest; MemChunk *lastchunk; MemPoolMeter meter; splayNode *allChunks; @@ -64,6 +69,7 @@ void *freeList; void *objCache; int count; + MemChunk *nextFreeChunk; MemChunk *next; MemChunk *prev; time_t lastref; @@ -105,6 +111,7 @@ extern void memPoolClean(MemPool * pool, time_t maxage); extern void memPoolCleanIdlePools(void *unused); extern void memPoolShrink(MemPool * pool, ssize_t new_limit); +extern void memPoolFlushMeters(MemPool * pool); /* MemMeter */ Index: squid/include/util.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/util.h,v retrieving revision 1.7.16.1 retrieving revision 1.7.16.2 diff -u -r1.7.16.1 -r1.7.16.2 --- squid/include/util.h 3 Apr 2001 22:35:48 -0000 1.7.16.1 +++ squid/include/util.h 19 Apr 2001 23:53:39 -0000 1.7.16.2 @@ -1,5 +1,5 @@ /* - * $Id: util.h,v 1.7.16.1 2001/04/03 22:35:48 akroonmaa Exp $ + * $Id: util.h,v 1.7.16.2 2001/04/19 23:53:39 akroonmaa Exp $ * * AUTHOR: Harvest Derived * @@ -136,6 +136,7 @@ /* gb_type operations */ #define gb_flush_limit (0x3FFFFFFF) #define gb_inc(gb, delta) { if ((gb)->bytes > gb_flush_limit || delta > gb_flush_limit) gb_flush(gb); (gb)->bytes += delta; (gb)->count++; } +#define gb_incc(gb, delta) { if ((gb)->bytes > gb_flush_limit || delta > gb_flush_limit) gb_flush(gb); (gb)->count+= delta; } extern double gb_to_double(const gb_t *); extern const char *gb_to_str(const gb_t *); extern void gb_flush(gb_t *); /* internal, do not use this */ Index: squid/lib/MemPool.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/Attic/MemPool.c,v retrieving revision 1.1.2.3 retrieving revision 1.1.2.4 diff -u -r1.1.2.3 -r1.1.2.4 --- squid/lib/MemPool.c 18 Apr 2001 18:17:43 -0000 1.1.2.3 +++ squid/lib/MemPool.c 19 Apr 2001 23:53:39 -0000 1.1.2.4 @@ -78,26 +78,8 @@ * Andres Kroonmaa. */ -/* - * Debug assist: - * PARANOID - Scan freelist to detect double-freed and tampered free objects - * PARANOID_LEVEL - skip pools with more than that items, for speedup - * - * DISABLE_POOLS - no mempooling, revert back to good-old xcalloc/xfree - * might be useful when debugging memory issues. - * - * MEM_PROTECT - mprotect() freed pool objects that are pagealigned and - * size of multiples of VMpage. Helps you to detect buffer writes after - * being freed. Needs mmap support. Also, redefine in dlmalloc.c: - * #define DEFAULT_MMAP_MAX (2048) - * #define DEFAULT_MMAP_THRESHOLD (16 * 1024) - */ - #define SPLAY 1 #define DISABLE_POOLS 0 -#define PARANOID 0 /* NB: slows down alot! */ -#define PARANOID_LEVEL 300 -#define MEM_PROTECT 0 #include #include @@ -134,6 +116,7 @@ void memArrayAppend(Array * s, void *obj); void memArrayDestroy(Array * s); void memPoolClean(MemPool * pool, time_t maxage); +void memPoolCreateChunk(MemPool * pool); static MemPool *lastPool; int @@ -168,10 +151,14 @@ *Free = (void *) Free + pool->obj_size; Free = *Free; } + chunk->nextFreeChunk = pool->nextFreeChunk; + pool->nextFreeChunk = chunk; + memMeterAdd(pool->meter.alloc, pool->capacity); memMeterAdd(TheMeter.alloc, pool->capacity * pool->obj_size); memMeterAdd(pool->meter.idle, pool->capacity); memMeterAdd(TheMeter.idle, pool->capacity * pool->obj_size); + pool->idle += pool->capacity; pool->chunkCount++; chunk->lastref = squid_curtime; lastPool = pool; @@ -186,6 +173,7 @@ memMeterDel(TheMeter.alloc, pool->capacity * pool->obj_size); memMeterDel(pool->meter.idle, pool->capacity); memMeterDel(TheMeter.idle, pool->capacity * pool->obj_size); + pool->idle -= pool->capacity; pool->chunkCount--; if (pool->lastchunk == chunk) pool->lastchunk = chunk->prev; @@ -195,99 +183,87 @@ xfree(chunk); } -void +inline void memPoolPush(MemPool * pool, void *obj) { - MemChunk *chunk; void **Free; - assert(pool->Chunks); - lastPool = pool; - pool->allChunks = splay_splay(obj, pool->allChunks, (SPLAYCMP *) memCompObjChunks); - if (splayLastResult == 0) { - chunk = pool->allChunks->data; - assert(chunk->count > 0); - chunk->count--; - if ((pool->obj_size % 2048) != 0) - memset(obj, 0, pool->obj_size); - Free = obj; - *Free = chunk->freeList; - chunk->freeList = obj; - chunk->lastref = squid_curtime; - if (pool->nextbest) - if (chunk->objCache < pool->nextbest->objCache) /* prefer chunks in lowest ram */ - pool->nextbest = chunk; - if (TheMeter.idle.level > mem_idle_limit) { - if (chunk->count == 0) - memPoolClean(pool, 0); /* free chunk, over idle limits, so cleanup */ - } - return; - } - /* obj does NOT belong to this pool!! */ - assert("memPoolFree: pool->Chunks" == "obj does NOT belong to this pool!"); - return; /* not reached */ + if ((pool->obj_size % 2048) != 0) + memset(obj, 0, pool->obj_size); + Free = obj; + *Free = pool->freeCache; + pool->freeCache = obj; + return; } /* - * Find a chunk with a free item, starting from "nextbest" for speedups - * nextbest is tracked to prefer chunk that has 1) free item, 2) is in lowest ram - * Create new chunk on demand if no chunk with frees found, point nextbest at it + * Find a chunk with a free item. + * Create new chunk on demand if no chunk with frees found. * Insert new chunk in front of lowest ram chunk, making it preferred in future, * and resulting slow compaction towards lowest ram area. */ void * memPoolGet(MemPool * pool) { - MemChunk *chunk, *new; - void *newObj; + MemChunk *chunk; void **Free; - chunk = pool->Chunks; - if (pool->nextbest) /* start looking from chunk known to have frees */ - chunk = pool->nextbest; - while (chunk) { - if (chunk->count < pool->capacity) { - pool->nextbest = chunk; - assert(chunk->freeList); - Free = chunk->freeList; - chunk->freeList = *Free; - *Free = NULL; - chunk->count++; - chunk->lastref = squid_curtime; - return Free; - } - if (chunk->next == NULL) { - pool->nextbest = NULL; - break; - } - chunk = chunk->next; + /* first, try cache */ + if (pool->freeCache) { + Free = pool->freeCache; + pool->freeCache = *Free; + *Free = NULL; + return Free; } + /* then try perchunk freelist chain */ - new = memPoolChunkNew(pool); - pool->nextbest = new; /* no chunk with frees, so we are the best */ - Free = new->freeList; - new->freeList = *Free; + if (pool->nextFreeChunk == NULL) { + /* no chunk with frees, so create new one */ + memPoolCreateChunk(pool); + } + /* now we have some in perchunk freelist chain */ + chunk = pool->nextFreeChunk; + + Free = chunk->freeList; + chunk->freeList = *Free; *Free = NULL; - newObj = Free; - new->count++; + chunk->count++; + chunk->lastref = squid_curtime; + + if (chunk->freeList == NULL) { + /* last free in this chunk, so remove us from perchunk freelist chain */ + pool->nextFreeChunk = chunk->nextFreeChunk; + } + return Free; +} + +/* just create a new chunk and place it into a good spot in the chunk chain */ +void +memPoolCreateChunk(MemPool * pool) +{ + MemChunk *chunk, *new; + + new = memPoolChunkNew(pool); chunk = pool->Chunks; if (chunk == NULL) { /* first chunk in pool */ pool->Chunks = new; - return newObj; + return; } - if (new->objCache < chunk->objCache) { /* we are lowest ram chunk, insert as first chunk */ + if (new->objCache < chunk->objCache) { + /* we are lowest ram chunk, insert as first chunk */ new->next = chunk; chunk->prev = new; pool->Chunks = new; - return newObj; + return; } while (chunk->next) { - if (new->objCache < chunk->next->objCache) { /* new chunk is in lower ram, insert here */ + if (new->objCache < chunk->next->objCache) { + /* new chunk is in lower ram, insert here */ new->next = chunk->next; chunk->next = new; new->prev = chunk; new->next->prev = new; - return newObj; + return; } chunk = chunk->next; } @@ -295,7 +271,7 @@ chunk->next = new; new->prev = chunk; pool->lastchunk = new; - return newObj; + return; } void @@ -352,28 +328,60 @@ assert(pool); } +#define FLUSH_LIMIT 1000 /* Flush memPool counters to memMeters after flush limit calls */ + +void +memPoolFlushMeters(MemPool * pool) +{ + size_t calls; + size_t bytes; + + calls = pool->alloc_calls; + if (calls) { + bytes = pool->obj_size * pool->alloc_calls; + + memMeterAdd(pool->meter.inuse, calls); + gb_incc(&pool->meter.saved, calls); + gb_incc(&pool->meter.total, calls); + mem_pool_alloc_calls += calls; + + memMeterAdd(TheMeter.inuse, bytes); + gb_inc(&TheMeter.saved, bytes); + gb_inc(&TheMeter.total, bytes); + gb_inc(&mem_traffic_volume, bytes); + + memMeterDel(TheMeter.idle, bytes); + memMeterDel(pool->meter.idle, calls); + pool->alloc_calls = 0; + } + calls = pool->free_calls; + if (calls) { + bytes = pool->obj_size * pool->free_calls; + memMeterDel(pool->meter.inuse, calls); + memMeterAdd(pool->meter.idle, calls); + mem_pool_free_calls += calls; + + memMeterDel(TheMeter.inuse, bytes); + memMeterAdd(TheMeter.idle, bytes); + pool->free_calls = 0; + } +} + void * memPoolAlloc(MemPool * pool) { void *p; assert(pool); #if !DISABLE_POOLS - memMeterInc(pool->meter.inuse); - gb_inc(&pool->meter.total, 1); - gb_inc(&TheMeter.total, pool->obj_size); - memMeterAdd(TheMeter.inuse, pool->obj_size); - gb_inc(&mem_traffic_volume, pool->obj_size); - mem_pool_alloc_calls++; p = memPoolGet(pool); - assert(pool->meter.idle.level); -#if XMALLOC_DEBUG2 - check_malloc(p, pool->obj_size); -#endif - memMeterDec(pool->meter.idle); - memMeterDel(TheMeter.idle, pool->obj_size); - gb_inc(&pool->meter.saved, 1); - gb_inc(&TheMeter.saved, pool->obj_size); - assert(pool->meter.inuse.level <= pool->meter.alloc.level); + assert(pool->idle); + pool->idle--; + pool->inuse++; + + if (++pool->alloc_calls == FLUSH_LIMIT) + memPoolFlushMeters(pool); + + assert(pool->inuse <= pool->meter.alloc.level); #else p = xcalloc(1, pool->obj_size); #endif @@ -385,17 +393,15 @@ { assert(pool && obj); #if !DISABLE_POOLS - memMeterDec(pool->meter.inuse); - memMeterDel(TheMeter.inuse, pool->obj_size); - memMeterInc(pool->meter.idle); - memMeterAdd(TheMeter.idle, pool->obj_size); - mem_pool_free_calls++; -#if XMALLOC_DEBUG2 - if (obj != NULL) - check_free(obj); -#endif + memPoolPush(pool, obj); - assert(pool->meter.idle.level <= pool->meter.alloc.level); + pool->idle++; + pool->inuse--; + + if (++pool->free_calls == FLUSH_LIMIT) + memPoolFlushMeters(pool); + + assert(pool->idle <= pool->meter.alloc.level); #else xfree(obj); #endif @@ -408,15 +414,15 @@ MemChunk *chunk, *freechunk; time_t age; chunk = pool->Chunks; - if (!chunk) - return; +// if (!chunk) + return; /* we start from pool->Chunks->next, so first chunk is never released */ /* We skip chunk that is nextbest, so lowest chunk with frees is not released */ while ((freechunk = chunk->next) != NULL) { age = squid_curtime - freechunk->lastref; - if ((freechunk != pool->nextbest) && /* Skip chunk if its nextbest */ + if ((freechunk != pool->nextFreeChunk) && /* Skip chunk if its nextbest */ (freechunk->count == 0) && (age >= maxage) ) { chunk->next = freechunk->next; Index: squid/src/MemPoolStats.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/MemPoolStats.c,v retrieving revision 1.1.2.5 retrieving revision 1.1.2.6 diff -u -r1.1.2.5 -r1.1.2.6 --- squid/src/MemPoolStats.c 18 Apr 2001 18:17:43 -0000 1.1.2.5 +++ squid/src/MemPoolStats.c 19 Apr 2001 23:53:39 -0000 1.1.2.6 @@ -108,6 +108,7 @@ for (i = 0; i < Pools.count; i++) { MemPool *pool = Pools.items[i]; + memPoolFlushMeters(pool); if (pool->meter.idle.level > (pool->capacity << 1) ) { memPoolClean(pool, (time_t) clean_interval); cleanable++; @@ -287,6 +288,7 @@ xm_time = current_dtime; for (i = 0; i < Pools.count ; i++) { const MemPool *pool = Pools.items[i]; + memPoolFlushMeters(Pools.items[i]); if (memPoolWasUsed(pool)) { memPoolReport(pool, e); free_count += freecount; Index: squid/src/stat.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stat.c,v retrieving revision 1.9.8.1 retrieving revision 1.9.8.2 diff -u -r1.9.8.1 -r1.9.8.2 --- squid/src/stat.c 18 Apr 2001 18:17:43 -0000 1.9.8.1 +++ squid/src/stat.c 19 Apr 2001 23:53:39 -0000 1.9.8.2 @@ -1,6 +1,6 @@ /* - * $Id: stat.c,v 1.9.8.1 2001/04/18 18:17:43 akroonmaa Exp $ + * $Id: stat.c,v 1.9.8.2 2001/04/19 23:53:39 akroonmaa Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -596,11 +596,11 @@ storeAppendPrintf(sentry, "Memory accounted for:\n"); storeAppendPrintf(sentry, "\tTotal accounted: %6d KB\n", statMemoryAccounted() >> 10); - storeAppendPrintf(sentry, "\tmemPool accounted: %6d KB\n", - memTotalAllocated() >> 10); - storeAppendPrintf(sentry, "\tmemPoolAlloc calls: %d\n", + storeAppendPrintf(sentry, "\tmemPool accounted: %6d KB %3d%%\n", + memTotalAllocated() >> 10, percent(memTotalAllocated(), t)); + storeAppendPrintf(sentry, "\tmemPoolAlloc calls: %9d\n", mem_pool_alloc_calls); - storeAppendPrintf(sentry, "\tmemPoolFree calls: %d\n", + storeAppendPrintf(sentry, "\tmemPoolFree calls: %9d\n", mem_pool_free_calls); storeAppendPrintf(sentry, "File descriptor usage for %s:\n", appname);