--------------------- PatchSet 2164 Date: 2001/04/30 19:35:30 Author: akroonmaa Branch: chunked_mempools Tag: (none) Log: reduce exported symbols reduce stats structures used and stop being global (eg. TheMeter, Pools) make cumulative counters doubles remove unused funcs and declarations review stats Reporting Members: include/MemPool.h:1.1.2.9->1.1.2.10 lib/MemPool.c:1.1.2.9->1.1.2.10 src/MemPoolStats.c:1.1.2.11->1.1.2.12 src/protos.h:1.23.4.5->1.23.4.6 Index: squid/include/MemPool.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/Attic/MemPool.h,v retrieving revision 1.1.2.9 retrieving revision 1.1.2.10 diff -u -r1.1.2.9 -r1.1.2.10 --- squid/include/MemPool.h 29 Apr 2001 12:44:27 -0000 1.1.2.9 +++ squid/include/MemPool.h 30 Apr 2001 19:35:30 -0000 1.1.2.10 @@ -22,8 +22,6 @@ extern void memArrayInit(Array * s); -#define CHUNKS 1 - /* object to track per-action memory usage (e.g. #idle objects) */ struct _MemMeter { ssize_t level; /* current level (count or volume) */ @@ -31,17 +29,21 @@ time_t hwater_stamp; /* timestamp of last high water mark change */ }; +/* object to track per-pool cumulative counters */ +typedef struct { + double count; + double bytes; +} mgb_t; + /* object to track per-pool memory usage (alloc = inuse+idle) */ struct _MemPoolMeter { MemMeter alloc; MemMeter inuse; MemMeter idle; - gb_t osaved; - gb_t saved; - gb_t total; + mgb_t gb_saved; + mgb_t gb_osaved; }; -#ifdef CHUNKS /* a pool is a [growing] space for objects of the same size */ struct _MemPool { const char *label; @@ -70,22 +72,8 @@ time_t lastref; }; -#else -/* a pool is a [growing] space for objects of the same size */ -struct _MemPool { - const char *label; - size_t obj_size; - Stack pstack; /* stack for free pointers */ - MemPoolMeter meter; -}; - -#endif - #define SIZEOF_CHUNK ( ( sizeof(MemChunk) + sizeof(double) -1) / sizeof(double) ) * sizeof(double); -extern MemPoolMeter TheMeter; -extern Stack Pools; -extern gb_t mem_traffic_volume; extern unsigned int mem_pool_alloc_calls; extern unsigned int mem_pool_free_calls; @@ -98,13 +86,9 @@ extern void memPoolDestroy(MemPool * pool); extern void *memPoolAlloc(MemPool * pool); extern void memPoolFree(MemPool * pool, void *obj); -extern int memPoolWasUsed(const MemPool * pool); extern int memPoolInUseCount(const MemPool * pool); -extern size_t memPoolInUseSize(const MemPool * pool); -extern int memPoolUsedCount(const MemPool * pool); extern void memPoolClean(MemPool * pool, time_t maxage); extern void memPoolCleanIdlePools(void *unused); -extern void memPoolShrink(MemPool * pool, ssize_t new_limit); extern void memPoolFlushMeters(MemPool * pool); extern size_t memPoolsTotalAllocated(void); @@ -117,5 +101,4 @@ #define memMeterAdd(m, sz) { (m).level += (sz); memMeterCheckHWater(m); } #define memMeterDel(m, sz) { (m).level -= (sz); } - #endif /* _MEM_POOLS_H_ */ Index: squid/lib/MemPool.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/Attic/MemPool.c,v retrieving revision 1.1.2.9 retrieving revision 1.1.2.10 diff -u -r1.1.2.9 -r1.1.2.10 --- squid/lib/MemPool.c 29 Apr 2001 12:44:27 -0000 1.1.2.9 +++ squid/lib/MemPool.c 30 Apr 2001 19:35:30 -0000 1.1.2.10 @@ -99,12 +99,9 @@ unsigned int mem_pool_alloc_calls = 0; unsigned int mem_pool_free_calls = 0; -/* memory pool accounting */ -MemPoolMeter TheMeter; -gb_t mem_traffic_volume = -{0, 0}; +/* locals */ -Array Pools; +static Array *PoolArray = NULL; static int Pool_id_counter = 0; static int bounds; @@ -122,8 +119,8 @@ { assert(m); if (m->hwater_level < m->level) { - m->hwater_level = m->level; - m->hwater_stamp = squid_curtime; + m->hwater_level = m->level; + m->hwater_stamp = squid_curtime; } } @@ -163,9 +160,7 @@ pool->nextFreeChunk = chunk; memMeterAdd(pool->meter.alloc, pool->capacity); - memMeterAdd(TheMeter.alloc, pool->capacity * pool->obj_size); memMeterAdd(pool->meter.idle, pool->capacity); - memMeterAdd(TheMeter.idle, pool->capacity * pool->obj_size); pool->idle += pool->capacity; pool->chunkCount++; chunk->lastref = squid_curtime; @@ -178,9 +173,7 @@ memPoolChunkDestroy(MemPool * pool, MemChunk * chunk) { memMeterDel(pool->meter.alloc, pool->capacity); - memMeterDel(TheMeter.alloc, pool->capacity * pool->obj_size); memMeterDel(pool->meter.idle, pool->capacity); - memMeterDel(TheMeter.idle, pool->capacity * pool->obj_size); pool->idle -= pool->capacity; pool->chunkCount--; lastPool = pool; @@ -314,7 +307,7 @@ memPoolTune(pool, MEM_CHUNK_SIZE, 0); pool->memPID = ++Pool_id_counter; - memArrayAppend(&Pools, pool); + memArrayAppend(PoolArray, pool); return pool; } @@ -341,9 +334,7 @@ memMeterDel(pool->meter.inuse, calls); #if !DISABLE_POOLS memMeterAdd(pool->meter.idle, calls); - memMeterAdd(TheMeter.idle, bytes); #endif - memMeterDel(TheMeter.inuse, bytes); pool->free_calls = 0; } calls = pool->alloc_calls; @@ -351,19 +342,10 @@ bytes = pool->obj_size * pool->alloc_calls; memMeterAdd(pool->meter.inuse, calls); - gb_incc(&pool->meter.saved, calls); - gb_incc(&pool->meter.total, calls); - gb_incc(&TheMeter.saved, calls); - gb_incc(&TheMeter.total, calls); - gb_incc(&mem_traffic_volume, calls); + pool->meter.gb_saved.count += calls; mem_pool_alloc_calls += calls; - memMeterAdd(TheMeter.inuse, bytes); - gb_incb(&TheMeter.saved, bytes); - gb_incb(&TheMeter.total, bytes); - gb_incb(&mem_traffic_volume, bytes); #if !DISABLE_POOLS - memMeterDel(TheMeter.idle, bytes); memMeterDel(pool->meter.idle, calls); #endif pool->alloc_calls = 0; @@ -503,37 +485,19 @@ } int -memPoolWasUsed(const MemPool * pool) -{ - assert(pool); - return pool->meter.inuse.hwater_level > 0; -} - -int memPoolInUseCount(const MemPool * pool) { assert(pool); return pool->meter.inuse.level; } -size_t -memPoolInUseSize(const MemPool * pool) -{ - assert(pool); - return pool->obj_size * pool->meter.inuse.level; -} - -size_t -memPoolsTotalAllocated(void) -{ - return TheMeter.alloc.level; -} - void memArrayInit(Array * a) { assert(a); memset(a, 0, sizeof(Array)); + assert(PoolArray == NULL); + PoolArray = a; } #if UNUSED_CODE @@ -545,6 +509,7 @@ xfree(a->items); a->items = NULL; } + #endif #if UNUSED_CODE @@ -555,6 +520,7 @@ memArrayClean(a); xfree(a); } + #endif static void Index: squid/src/MemPoolStats.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/MemPoolStats.c,v retrieving revision 1.1.2.11 retrieving revision 1.1.2.12 diff -u -r1.1.2.11 -r1.1.2.12 --- squid/src/MemPoolStats.c 29 Apr 2001 12:44:27 -0000 1.1.2.11 +++ squid/src/MemPoolStats.c 30 Apr 2001 19:35:30 -0000 1.1.2.12 @@ -40,6 +40,8 @@ /* module globals */ +/* module locals */ +static Array Pools; static double clean_interval = 15.0; /* time to live of idle chunk before release */ static double xm_time = 0; static double xm_deltat = 0; @@ -47,6 +49,7 @@ /* huge constant to set mem_idle_limit to "unlimited" */ static const size_t mem_unlimited_size = 2 * 1024 * MB; static size_t mem_idle_limit = 0; +static MemPoolMeter TheMeter; /* local prototypes */ static void memPoolDescribe(const MemPool * pool); @@ -65,6 +68,12 @@ return (size + 1024 - 1) / 1024; } +size_t +memPoolsTotalAllocated(void) +{ + return TheMeter.alloc.level; +} + /* Initialization */ /* XXX This is NOT statistics. Does NOT belong here. */ @@ -86,29 +95,53 @@ /* shrink memory pools if needed */ if (TheMeter.idle.level > new_pool_limit) { debug(63, 1) ("Shrinking idle mem pools to %.2f MB\n", toMB(new_pool_limit)); - memShrink(new_pool_limit); +// memShrink(new_pool_limit); } mem_idle_limit = new_pool_limit; } +/* updates all pool counters, and recreates TheMeter totals from all pools */ +static void +memPoolFlushMetersAllPools(void) +{ + int i; + MemPool *pool; + TheMeter.alloc.level = 0; + TheMeter.inuse.level = 0; + TheMeter.idle.level = 0; + TheMeter.gb_saved.count = 0; + TheMeter.gb_saved.bytes = 0; + for (i = 0; i < Pools.count; i++) { + pool = Pools.items[i]; + memPoolFlushMeters(pool); + memMeterAdd(TheMeter.alloc, pool->meter.alloc.level * pool->obj_size); + memMeterAdd(TheMeter.inuse, pool->meter.inuse.level * pool->obj_size); + memMeterAdd(TheMeter.idle, pool->meter.idle.level * pool->obj_size); + TheMeter.gb_saved.count += pool->meter.gb_saved.count; + TheMeter.gb_saved.bytes += pool->meter.gb_saved.count * pool->obj_size; + pool->meter.gb_saved.bytes = pool->meter.gb_saved.count * pool->obj_size; + } +} + /* XXX This is NOT statistics. Does NOT belong here. */ void memPoolCleanIdlePools(void *unused) { int i; + MemPool *pool; int cleanable = 0; int shift = 1; int maxage = clean_interval; + memPoolFlushMetersAllPools(); if (TheMeter.idle.level > mem_idle_limit) maxage = shift = 0; for (i = 0; i < Pools.count; i++) { - MemPool *pool = Pools.items[i]; - memPoolFlushMeters(pool); - if (pool->meter.idle.level > (pool->capacity << shift) ) { - memPoolClean(pool, (time_t) maxage); + pool = Pools.items[i]; + if (pool->meter.idle.level > (pool->capacity << shift)) { + memPoolClean(pool, (time_t) maxage); cleanable++; - } + } } eventAdd("memPoolCleanIdlePools", memPoolCleanIdlePools, NULL, clean_interval, 1); } @@ -119,7 +152,7 @@ memset(&TheMeter, 0, sizeof(TheMeter)); memArrayInit(&Pools); debug(63, 1) ("Memory pools are '%s'; limit: %.2f MB\n", - (Config.onoff.mem_pools ? "on" : "off"), toMB(mem_idle_limit)); + (Config.onoff.mem_pools ? "on" : "off"), toMB(mem_idle_limit)); } void @@ -128,19 +161,21 @@ int i; int dirty_count = 0; for (i = 0; i < Pools.count; i++) { - MemPool *pool = Pools.items[i]; - if (memPoolInUseCount(pool)) { - memPoolDescribe(pool); - dirty_count++; - } else { - memPoolDestroy(pool); - } + MemPool *pool = Pools.items[i]; + if (memPoolInUseCount(pool)) { + memPoolDescribe(pool); + dirty_count++; + } else { + memPoolDestroy(pool); + } } if (dirty_count) - debug(63, 2) ("memCleanModule: %d pools are left dirty\n", dirty_count); + debug(63, 2) ("memCleanModule: %d pools are left dirty\n", dirty_count); /* we clean the stack anyway */ } +#if UNUSED_CODE +/* XXX need to rewrite */ static void memShrink(ssize_t new_limit) { @@ -148,19 +183,20 @@ int i; assert(start_limit >= 0 && new_limit >= 0); debug(63, 1) ("memShrink: started with %d KB goal: %d KB\n", - toKB(TheMeter.idle.level), toKB(new_limit)); + toKB(TheMeter.idle.level), toKB(new_limit)); /* first phase: cut proportionally to the pool idle size */ for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) { - MemPool *pool = Pools.items[i]; - const ssize_t target_pool_size = (size_t) ((double) pool->meter.idle.level * new_limit) / start_limit; - memPoolShrink(pool, target_pool_size); + MemPool *pool = Pools.items[i]; + const ssize_t target_pool_size = (size_t) ((double) pool->meter.idle.level * new_limit) / start_limit; + memPoolShrink(pool, target_pool_size); } debug(63, 1) ("memShrink: 1st phase done with %d KB left\n", toKB(TheMeter.idle.level)); /* second phase: cut to 0 */ - for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) - memPoolShrink(Pools.items[i], 0); + for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) + memPoolShrink(Pools.items[i], 0); debug(63, 1) ("memShrink: 2nd phase done with %d KB left\n", toKB(TheMeter.idle.level)); } +#endif /* MemPoolMeter */ @@ -169,7 +205,11 @@ int alloc_count, int inuse_count, int idle_count, StoreEntry * e) { assert(pm); - storeAppendPrintf(e, "%d\t %d\t %d\t %.2f\t %d\t %d\t %d\t %d\t %d\t %d\t %d\t %d\t %d\t %.1f\t %.1f\t %.1f\n", + storeAppendPrintf(e, + "%d\t %d\t %d\t %.2f\t %d\t" /* alloc */ + "%d\t %d\t %d\t %d\t" /* in use */ + "%d\t %d\t %d\t" /* idle */ + "%.0f\t %.1f\t %.1f\t %.1f\n", /* saved */ /* alloc */ alloc_count, toKB(obj_size * pm->alloc.level), @@ -187,11 +227,11 @@ toKB(obj_size * pm->idle.hwater_level), /* (int)rint(xpercent(pm->idle.level, pm->alloc.level)), */ /* saved */ - pm->saved.count, - xpercent(pm->saved.count, mem_traffic_volume.count), - xpercent(obj_size * gb_to_double(&pm->saved), gb_to_double(&mem_traffic_volume)), - xdiv(pm->saved.count - pm->osaved.count, xm_deltat)); - ((MemPoolMeter *)pm)->osaved.count = pm->saved.count; + pm->gb_saved.count, + xpercent(pm->gb_saved.count, TheMeter.gb_saved.count), + xpercent(pm->gb_saved.bytes, TheMeter.gb_saved.bytes), + xdiv(pm->gb_saved.count - pm->gb_osaved.count, xm_deltat)); + ((MemPoolMeter *) pm)->gb_osaved.count = pm->gb_saved.count; } /* to-do: make debug level a parameter? */ @@ -201,58 +241,56 @@ assert(pool); debug(63, 2) ("%-20s: %6d x %4d bytes = %5d KB\n", pool->label, memPoolInUseCount(pool), pool->obj_size, - toKB(memPoolInUseSize(pool))); + toKB(pool->obj_size * pool->meter.inuse.level)); } /* These should be moved into the MemPool statistics... */ -static int freecount; /* need these outside memPoolReport() */ +static int freecount; /* need these outside memPoolReport() */ static int partial; static int inuse; static void memPoolReport(const MemPool * pool, StoreEntry * e) { - int excess,needed; + int excess, needed; MemChunk *chunk; assert(pool); storeAppendPrintf(e, "%-20s\t %4d\t ", - pool->label, pool->obj_size); -#if CHUNKS + pool->label, pool->obj_size); storeAppendPrintf(e, "%4d\t %4d\t ", - toKB(pool->obj_size * pool->capacity), pool->capacity); - freecount=partial=0; + toKB(pool->obj_size * pool->capacity), pool->capacity); + freecount = partial = 0; chunk = pool->Chunks; while (chunk) { if (chunk->inuse_count == 0) - freecount++; + freecount++; else if (chunk->inuse_count < pool->capacity) - partial++; + partial++; chunk = chunk->next; } inuse = pool->chunkCount - freecount; - if ( pool->meter.inuse.level % pool->capacity ) + if (pool->meter.inuse.level % pool->capacity) needed = pool->meter.inuse.level / pool->capacity + 1; else needed = pool->meter.inuse.level / pool->capacity; - excess = inuse - needed ; + excess = inuse - needed; storeAppendPrintf(e, "%4d\t %4d\t %4d\t %4d\t %.1f\t ", - pool->chunkCount, inuse, freecount, partial, - xpercent( excess, needed ) ); + pool->chunkCount, inuse, freecount, partial, + xpercent(excess, needed)); /* - Fragmentation calculation: - needed = inuse.level / capacity - excess = used - needed - fragmentation = excess / needed * 100% - - Fragm = (alloced - (inuse / obj_ch) ) / alloced -*/ -#endif + * Fragmentation calculation: + * needed = inuse.level / capacity + * excess = used - needed + * fragmentation = excess / needed * 100% + * + * Fragm = (alloced - (inuse / obj_ch) ) / alloced + */ memPoolMeterReport(&pool->meter, pool->obj_size, - pool->meter.alloc.level, pool->meter.inuse.level, pool->meter.idle.level, - e); + pool->meter.alloc.level, pool->meter.inuse.level, pool->meter.idle.level, + e); } void @@ -270,9 +308,15 @@ /* caption */ storeAppendPrintf(e, "Current memory usage:\n"); /* heading */ - storeAppendPrintf(e, "Pool\t Obj Size\t" - " Chunks\t\t\t\t\t\t\t" - "Allocated\t\t\t\t\t In Use\t\t\t\t Idle\t\t\t Allocations Saved\t\t\t Hit Rate\t\n" + storeAppendPrintf(e, + "Pool\t Obj Size\t" + "Chunks\t\t\t\t\t\t\t" + "Allocated\t\t\t\t\t" + "In Use\t\t\t\t" + "Idle\t\t\t" + "Allocations Saved\t\t\t" + "Hit Rate\t" + "\n" " \t (bytes)\t" "KB/ch\t obj/ch\t" "(#)\t used\t free\t part\t %Frag\t " @@ -285,11 +329,11 @@ /* main table */ xm_deltat = current_dtime - xm_time; xm_time = current_dtime; - for (i = 0; i < Pools.count ; i++) { + memPoolFlushMetersAllPools(); + for (i = 0; i < Pools.count; i++) { MemPool *pool = Pools.items[i]; - memPoolFlushMeters(Pools.items[i]); memPoolClean(pool, (time_t) 555555); /* don't want to get chunks released before reporting */ - if (memPoolWasUsed(pool)) { + if (pool->meter.inuse.hwater_level > 0) { /* this pool has been used */ memPoolReport(pool, e); free_count += freecount; part_count += partial; @@ -310,11 +354,10 @@ chunk_count, nuse_count, free_count, part_count); memPoolMeterReport(&TheMeter, 1, alloc_count, inuse_count, idle_count, e); - storeAppendPrintf(e, "Cumulative allocated volume: %s\n", gb_to_str(&mem_traffic_volume)); + storeAppendPrintf(e, "Cumulative allocated volume: %.0f\n", TheMeter.gb_saved.bytes); /* overhead */ storeAppendPrintf(e, "Current overhead: %d bytes (%.3f%%)\n", overhd_size, xpercent(overhd_size, TheMeter.inuse.level)); /* limits */ storeAppendPrintf(e, "Idle pool limit: %.2f MB\n", toMB(mem_idle_limit)); } - Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.23.4.5 retrieving revision 1.23.4.6 diff -u -r1.23.4.5 -r1.23.4.6 --- squid/src/protos.h 29 Apr 2001 12:44:27 -0000 1.23.4.5 +++ squid/src/protos.h 30 Apr 2001 19:35:30 -0000 1.23.4.6 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.23.4.5 2001/04/29 12:44:27 hno Exp $ + * $Id: protos.h,v 1.23.4.6 2001/04/30 19:35:30 akroonmaa Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -837,10 +837,7 @@ extern void memPoolDestroy(MemPool * pool); extern void *memPoolAlloc(MemPool * pool); extern void memPoolFree(MemPool * pool, void *obj); -extern int memPoolWasUsed(const MemPool * pool); extern int memPoolInUseCount(const MemPool * pool); -extern size_t memPoolInUseSize(const MemPool * pool); -extern int memPoolUsedCount(const MemPool * pool); /* Mem */ extern void memReport(StoreEntry * e);