--------------------- PatchSet 1848 Date: 2001/04/09 06:49:47 Author: akroonmaa Branch: cpuProfiling Tag: (none) Log: undo my stupid merge with chunked_mempools Members: include/MemPool.h:1.1.4.1->1.1.4.2(DEAD) include/util.h:1.7.20.1->1.7.20.2 lib/MemPool.c:1.1.4.1->1.1.4.2(DEAD) lib/util.c:1.7.24.3->1.7.24.4 src/MemPoolStats.c:1.1.4.1->1.1.4.2(DEAD) src/defines.h:1.10.8.1->1.10.8.2 src/protos.h:1.23.6.1->1.23.6.2 src/squid.h:1.11.10.2->1.11.10.3 src/tools.c:1.11.12.1->1.11.12.2 src/typedefs.h:1.17.12.1->1.17.12.2 --- squid/include/MemPool.h Wed Feb 14 00:51:50 2007 +++ /dev/null Wed Feb 14 00:51:37 2007 @@ -1,114 +0,0 @@ - -#ifndef _MEM_POOLS_H_ -#define _MEM_POOLS_H_ - -#include "config.h" -#include "Stack.h" -#include "util.h" - -#define MB ((size_t)1024*1024) - -#define MEM_PAGE_SIZE 4096 -#define MEM_CHUNK_SIZE 4096 * 4 -#define MEM_MAX_SIZE 262144 /* 64 4K pages */ -#define MEM_MIN_FREE 32 -#define MEM_MAX_FREE 65535 /* ushort is max number of items per chunk */ - -typedef struct _MemPoolMeter MemPoolMeter; -typedef struct _MemPool MemPool; -typedef struct _MemMeter MemMeter; -typedef struct _MemChunk MemChunk; - -extern void memArrayInit(Array * s); -extern void memArrayClean(Array * s); -extern void memArrayAppend(Array *s, void *obj); -extern void memArrayDestroy(Array *s); - -#define CHUNKS 1 - -/* object to track per-action memory usage (e.g. #idle objects) */ -struct _MemMeter { - ssize_t level; /* current level (count or volume) */ - ssize_t hwater_level; /* high water mark */ - time_t hwater_stamp; /* timestamp of last high water mark change */ -}; - -/* object to track per-pool memory usage (alloc = inuse+idle) */ -struct _MemPoolMeter { - MemMeter alloc; - MemMeter inuse; - MemMeter idle; - gb_t saved; - gb_t total; -}; - -#ifdef CHUNKS -/* a pool is a [growing] space for objects of the same size */ -struct _MemPool { - const char *label; - size_t obj_size; - size_t chunk_size; - int capacity; - int memPID; - int chunkCount; - MemChunk *Chunks; - MemChunk *nextbest; - MemChunk *lastchunk; - MemPoolMeter meter; -}; - -struct _MemChunk { - void *freeList; - void *objCache; - int count; - MemChunk *next; - MemChunk *prev; - time_t lastref; -}; -#else -/* a pool is a [growing] space for objects of the same size */ -struct _MemPool { - const char *label; - size_t obj_size; - Stack pstack; /* stack for free pointers */ - MemPoolMeter meter; -}; -#endif - -#define SIZEOF_CHUNK ( ( sizeof(MemChunk) + sizeof(double) -1) / sizeof(double) ) * sizeof(double); - -extern size_t mem_idle_limit; -extern MemPoolMeter TheMeter; -extern Stack Pools; -extern gb_t mem_traffic_volume; -extern unsigned int mem_pool_alloc_calls; -extern unsigned int mem_pool_free_calls; - -extern void memConfigure(void); -extern void memInitModule(void); - -/* memPools */ -extern void memPoolTune(MemPool * pool, size_t chunksize, int unused); -extern MemPool *memPoolCreate(const char *label, size_t obj_size); -extern void memPoolDestroy(MemPool * pool); -extern void *memPoolAlloc(MemPool * pool); -extern void memPoolFree(MemPool * pool, void *obj); -extern int memPoolWasUsed(const MemPool * pool); -extern int memPoolInUseCount(const MemPool * pool); -extern size_t memPoolInUseSize(const MemPool * pool); -extern int memPoolUsedCount(const MemPool * pool); -extern void memPoolClean(MemPool * pool, time_t maxage); -extern void memPoolCleanIdlePools(void *unused); -extern void memPoolShrink(MemPool * pool, ssize_t new_limit); - -/* MemMeter */ - -extern void memMeterSyncHWater(MemMeter * m); -#define memMeterCheckHWater(m) { if ((m).hwater_level < (m).level) memMeterSyncHWater(&(m)); } -#define memMeterInc(m) { (m).level++; memMeterCheckHWater(m); } -#define memMeterDec(m) { (m).level--; } -#define memMeterAdd(m, sz) { (m).level += (sz); memMeterCheckHWater(m); } -#define memMeterDel(m, sz) { (m).level -= (sz); } - - -#endif /* _MEM_POOLS_H_ */ Index: squid/include/util.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/util.h,v retrieving revision 1.7.20.1 retrieving revision 1.7.20.2 diff -u -r1.7.20.1 -r1.7.20.2 --- squid/include/util.h 6 Apr 2001 20:51:20 -0000 1.7.20.1 +++ squid/include/util.h 9 Apr 2001 06:49:47 -0000 1.7.20.2 @@ -1,5 +1,5 @@ /* - * $Id: util.h,v 1.7.20.1 2001/04/06 20:51:20 akroonmaa Exp $ + * $Id: util.h,v 1.7.20.2 2001/04/09 06:49:47 akroonmaa Exp $ * * AUTHOR: Harvest Derived * @@ -127,19 +127,6 @@ double drand48(void); #endif -typedef struct { - size_t count; - size_t bytes; - size_t gb; -} gb_t; - -/* gb_type operations */ -#define gb_flush_limit (0x3FFFFFFF) -#define gb_inc(gb, delta) { if ((gb)->bytes > gb_flush_limit || delta > gb_flush_limit) gb_flush(gb); (gb)->bytes += delta; (gb)->count++; } -extern double gb_to_double(const gb_t *); -extern const char *gb_to_str(const gb_t *); -extern void gb_flush(gb_t *); /* internal, do not use this */ - /* * Returns the amount of known allocated memory */ --- squid/lib/MemPool.c Wed Feb 14 00:51:50 2007 +++ /dev/null Wed Feb 14 00:51:37 2007 @@ -1,563 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 63 Low Level Memory Pool Management - * AUTHOR: Alex Rousskov - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -/* - Old way: - xmalloc each item separately, upon free stack into idle pool array. - each item is individually malloc()ed from system, imposing libmalloc - overhead, and additionally we add our overhead of pointer size per item - as we keep a list of pointer to free items. - - Chunking: - xmalloc Chunk that fits at least MEM_MIN_FREE (32) items in an array, but - limit Chunk size to MEM_MAX_SIZE (256K). Chunk size is rounded up to - MEM_PAGE_SIZE (4K), trying to have chunks in multiples of VM_PAGE size. - Minimum Chunk size is MEM_CHUNK_SIZE (16K). - A number of items fits into a single chunk, depending on item size. - Maximum number of items per chunk is limited to MEM_MAX_FREE (65535). - - We populate Chunk with a linkedlist, each node at first word of item, - and pointing at next free item. Chunk->FreeList is pointing at first - free node. Thus we stuff free housekeeping into the Chunk itself, and - omit pointer overhead per item. - - Chunks are created on demand, and new chunks are inserted into linklist - of chunks so that Chunks with smaller pointer value are placed closer - to the linklist head. Head is a hotspot, servicing most of requests, so - slow sorting occurs and Chunks in highest memory tend to become idle - and freeable. - - event is registered that runs every 15 secs and checks reference time - of each idle chunk. If a chunk is not referenced for 15 secs, it is - released. - - If mem_idle_limit is exceeded with pools, every chunk that becomes - idle is immediately considered for release, unless this is the only - chunk with free items in it. - - In cachemgr output, there are new columns for chunking. Special item, - Fragm, is shown to estimate approximately fragmentation of chunked - pools. Fragmentation is calculated by taking amount of items in use, - calculating needed amount of chunks to fit all, and then comparing to - actual amount of chunks in use. Fragm number, in percent, is showing - how many percent of chunks are in use excessively. 100% meaning that - twice the needed amount of chunks are in use. - - Andres Kroonmaa. - */ - -/* - Debug assist: - PARANOID - Scan freelist to detect double-freed and tampered free objects - PARANOID_LEVEL - skip pools with more than that items, for speedup - - DISABLE_POOLS - no mempooling, revert back to good-old xcalloc/xfree - might be useful when debugging memory issues. - - MEM_PROTECT - mprotect() freed pool objects that are pagealigned and - size of multiples of VMpage. Helps you to detect buffer writes after - being freed. Needs mmap support. Also, redefine in dlmalloc.c: - #define DEFAULT_MMAP_MAX (2048) - #define DEFAULT_MMAP_THRESHOLD (16 * 1024) -*/ - -#define DISABLE_POOLS 0 -#define PARANOID 0 /* NB: slows down alot! */ -#define PARANOID_LEVEL 300 -#define MEM_PROTECT 0 - -#include -#include - -#include "config.h" -#include "Stack.h" -#include "MemPool.h" - -extern time_t squid_curtime; - -/* exported */ -unsigned int mem_pool_alloc_calls = 0; -unsigned int mem_pool_free_calls = 0; - -/* module globals */ -/* we cannot keep idle more than this limit */ -size_t mem_idle_limit = 0; -time_t idletime = 30; - -/* memory pool accounting */ -MemPoolMeter TheMemPools; -MemPoolMeter TheChunks; -MemPoolMeter TheMeter; -gb_t mem_traffic_volume = { 0, 0 }; - -Array Pools; -static int Pool_id_counter = 0; -static int bounds; - -static void memArrayGrow(Array * a, int min_capacity); -void memArrayInit(Array * s); -void memArrayClean(Array * s); -void memArrayAppend(Array * s, void *obj); -void memArrayDestroy(Array * s); -void memPoolClean(MemPool * pool, time_t maxage); - -MemChunk * -memPoolChunkNew(MemPool * pool) -{ - int i; - void **Free; - MemChunk *chunk; - - chunk = xcalloc(1, sizeof(MemChunk)); /* should have a pool for this too */ -#if MEM_PROTECT - chunk->objCache = xvalloc(pool->chunk_size); - memset(chunk->objCache, 0, pool->chunk_size); -#else - chunk->objCache = xcalloc(1, pool->chunk_size); -#endif - Free = chunk->freeList = chunk->objCache; - - for (i = 1; i < pool->capacity; i++) { - *Free = (void *) Free + pool->obj_size; - Free = *Free; - } - memMeterAdd(pool->meter.alloc, pool->capacity); - memMeterAdd(TheMeter.alloc, pool->capacity * pool->obj_size); - memMeterAdd(pool->meter.idle, pool->capacity); - memMeterAdd(TheMeter.idle, pool->capacity * pool->obj_size); - pool->chunkCount++; - chunk->lastref = squid_curtime; - return chunk; -} - -void -memPoolChunkDestroy(MemPool * pool, MemChunk * chunk) -{ - memMeterDel(pool->meter.alloc, pool->capacity); - memMeterDel(TheMeter.alloc, pool->capacity * pool->obj_size); - memMeterDel(pool->meter.idle, pool->capacity); - memMeterDel(TheMeter.idle, pool->capacity * pool->obj_size); - pool->chunkCount--; - if (pool->lastchunk == chunk) - pool->lastchunk = chunk->prev; -#if PARANOID - if (pool->nextbest == chunk) /* shouldn't happen */ - pool->nextbest = NULL; -#endif - xfree(chunk->objCache); - xfree(chunk); -} - -void -memPoolPush(MemPool * pool, void *obj) -{ - MemChunk *chunk; - void **Free; -#if PARANOID - void *last=NULL; - void *prelast=NULL; -#endif - assert(pool->Chunks); - chunk = pool->Chunks; - while (1) { - bounds = obj - chunk->objCache; - if ((bounds >= 0) && (bounds < pool->chunk_size)) { /* does it belong to this chunk? */ -#if PARANOID - Free = chunk->freeList; -#if PARANOID_LEVEL - if (pool->meter.inuse.level < PARANOID_LEVEL) /* Omit larger pools for speedup */ -#endif - while (Free) { - assert(obj != Free); - bounds = (void *) Free - chunk->objCache; - assert(PARANOID && bounds >= 0 && bounds < pool->chunk_size); - prelast = last; - last = Free; /* assist debugging */ - Free = *Free; - } -#endif - assert(chunk->count > 0); - chunk->count--; - memset(obj, 0, pool->obj_size); - Free = obj; - *Free = chunk->freeList; -#if MEM_PROTECT - if ((pool->obj_size % MEM_PAGE_SIZE) == 0) { - if ( ( (int)obj % MEM_PAGE_SIZE) == 0) { - mprotect(obj, pool->obj_size, PROT_READ); - } - } -#endif - chunk->freeList = obj; - chunk->lastref = squid_curtime; - if (pool->nextbest) - if (chunk->objCache < pool->nextbest->objCache) /* prefer chunks in lowest ram */ - pool->nextbest = chunk; - if (TheMeter.idle.level <= mem_idle_limit) - return; - if (chunk->count == 0) - memPoolClean(pool, 0); /* free chunk, over idle limits, so cleanup */ - return; - } - if (chunk->next == NULL) - break; - chunk = chunk->next; - } - /* obj does NOT belong to this pool!! */ - assert(pool->Chunks == obj); - /* not reached */ - - /* Perhaps we should assume we are over pool_limits and revert to xfree() ? */ - return; -} - -/* - * Find a chunk with a free item, starting from "nextbest" for speedups - * nextbest is tracked to prefer chunk that has 1) free item, 2) is in lowest ram - * Create new chunk on demand if no chunk with frees found, point nextbest at it - * Insert new chunk in front of lowest ram chunk, making it preferred in future, - * and resulting slow compaction towards lowest ram area. - */ -void * -memPoolGet(MemPool * pool) -{ - MemChunk *chunk, *new; - void *newObj; - void **Free; - chunk = pool->Chunks; - - if (pool->nextbest) /* start looking from chunk known to have frees */ - chunk = pool->nextbest; - - while (chunk) { - if (chunk->count < pool->capacity) { - pool->nextbest = chunk; - assert(chunk->freeList); - Free = chunk->freeList; -#if PARANOID - if (*Free) { - bounds = (void *) *Free - chunk->objCache; - assert(*Free && (bounds >= 0) && (bounds < pool->chunk_size)); - } -#endif - chunk->freeList = *Free; -#if MEM_PROTECT - if ((pool->obj_size % MEM_PAGE_SIZE) == 0) { - if ( ( (int)Free % MEM_PAGE_SIZE) == 0) { - mprotect((void *) Free, pool->obj_size, PROT_READ | PROT_WRITE | PROT_EXEC); - } - } -#endif - *Free = NULL; -#if PARANOID - bounds = (void *) Free - chunk->objCache; - assert(Free && (bounds >= 0) && (bounds < pool->chunk_size)); - if (chunk->freeList) { - bounds = chunk->freeList - chunk->objCache; - assert(chunk->freeList && (bounds >= 0) - && (bounds < pool->chunk_size)); - } -#endif - chunk->count++; - chunk->lastref = squid_curtime; - return Free; - } - if (chunk->next == NULL) { - pool->nextbest = NULL; - break; - } - chunk = chunk->next; - } - - new = memPoolChunkNew(pool); - pool->nextbest = new; /* no chunk with frees, so we are the best */ - Free = new->freeList; - new->freeList = *Free; - *Free = NULL; - newObj = Free; - new->count++; - - chunk = pool->Chunks; - if (chunk == NULL) { /* first chunk in pool */ - pool->Chunks = new; - return newObj; - } - - if (new->objCache < chunk->objCache) { /* we are lowest ram chunk, insert as first chunk */ - new->next = chunk; - chunk->prev = new; - pool->Chunks = new; - return newObj; - } - - while (chunk->next) { - if (new->objCache < chunk->next->objCache) { /* new chunk is in lower ram, insert here */ - new->next = chunk->next; - chunk->next = new; - new->prev = chunk; - new->next->prev = new; - return newObj; - } - chunk = chunk->next; - } - /* we are the worst chunk in chain, add as last */ - chunk->next = new; - new->prev = chunk; - pool->lastchunk = new; - return newObj; -} - -void -memPoolTune(MemPool * pool, size_t chunksize, int unused) -{ - int cap; - size_t csize = chunksize; - - if (pool->Chunks) /* unsafe to tamper */ - return; - - csize = ((csize + MEM_PAGE_SIZE - 1) / MEM_PAGE_SIZE) * MEM_PAGE_SIZE; /* round up to page size */ - cap = csize / pool->obj_size; - - if (cap < MEM_MIN_FREE) - cap = MEM_MIN_FREE; - if (cap * pool->obj_size > MEM_MAX_SIZE) - cap = MEM_MAX_SIZE / pool->obj_size; - if (cap > MEM_MAX_FREE) - cap = MEM_MAX_FREE; - - csize = cap * pool->obj_size; - csize = ((csize + MEM_PAGE_SIZE - 1) / MEM_PAGE_SIZE) * MEM_PAGE_SIZE; /* round up to page size */ - cap = csize / pool->obj_size; - - pool->capacity = cap; - pool->chunk_size = csize; -} - -MemPool * -memPoolCreate(const char *label, size_t obj_size) -{ - MemPool *pool = xcalloc(1, sizeof(MemPool)); - assert(label && obj_size); - pool->label = label; - pool->obj_size = obj_size; - pool->obj_size = - ((obj_size + sizeof(void *) - 1) / sizeof(void *)) * sizeof(void *); - - memPoolTune(pool, MEM_CHUNK_SIZE, 0); - - pool->memPID = ++Pool_id_counter; - memArrayAppend(&Pools, pool); - return pool; -} - -/* - * warning: we do not clean this entry from Pools assuming memPoolDestroy - * is used at the end of the program only - */ -void -memPoolDestroy(MemPool * pool) -{ - assert(pool); -} - -void * -memPoolAlloc(MemPool * pool) -{ - void *p; - assert(pool); -#if !DISABLE_POOLS - memMeterInc(pool->meter.inuse); - gb_inc(&pool->meter.total, 1); - gb_inc(&TheMeter.total, pool->obj_size); - memMeterAdd(TheMeter.inuse, pool->obj_size); - gb_inc(&mem_traffic_volume, pool->obj_size); - mem_pool_alloc_calls++; - p = memPoolGet(pool); - assert(pool->meter.idle.level); -#if XMALLOC_DEBUG2 - check_malloc(p, pool->obj_size); -#endif - memMeterDec(pool->meter.idle); - memMeterDel(TheMeter.idle, pool->obj_size); - gb_inc(&pool->meter.saved, 1); - gb_inc(&TheMeter.saved, pool->obj_size); - assert(pool->meter.inuse.level <= pool->meter.alloc.level); -#else - p = xcalloc(1, pool->obj_size); -#endif - return p; -} - -void -memPoolFree(MemPool * pool, void *obj) -{ - assert(pool && obj); -#if !DISABLE_POOLS - memMeterDec(pool->meter.inuse); - memMeterDel(TheMeter.inuse, pool->obj_size); - memMeterInc(pool->meter.idle); - memMeterAdd(TheMeter.idle, pool->obj_size); - mem_pool_free_calls++; -#if XMALLOC_DEBUG2 - if (obj != NULL) - check_free(obj); -#endif - memPoolPush(pool, obj); - assert(pool->meter.idle.level <= pool->meter.alloc.level); -#else - xfree(obj); -#endif -} - -/* removes empty Chunks from pool */ -void -memPoolClean(MemPool * pool, time_t maxage) -{ - MemChunk *chunk, *freechunk; - time_t age; - chunk = pool->Chunks; - if (!chunk) - return; - - /* we start from pool->Chunks->next, so first chunk is never released */ - /* We skip chunk that is nextbest, so lowest chunk with frees is not released */ - - while ((freechunk = chunk->next) != NULL) { - age = squid_curtime - freechunk->lastref; - if ((freechunk != pool->nextbest) && /* Skip chunk if its nextbest */ - (freechunk->count == 0) && (age >= maxage) - ) { - chunk->next = freechunk->next; - if (freechunk->next) - freechunk->next->prev = chunk; - memPoolChunkDestroy(pool, freechunk); - } - if (chunk->next == NULL) - break; - chunk = chunk->next; - } - return; -} - -void -memPoolShrink(MemPool * pool, ssize_t new_limit) -{ - assert(pool); - assert(new_limit >= 0); - memPoolClean(pool, 0); -} - -int -memPoolWasUsed(const MemPool * pool) -{ - assert(pool); - return pool->meter.alloc.hwater_level > 0; -} - -int -memPoolInUseCount(const MemPool * pool) -{ - assert(pool); - return pool->meter.inuse.level; -} - -size_t -memPoolInUseSize(const MemPool * pool) -{ - assert(pool); - return pool->obj_size * pool->meter.inuse.level; -} - -size_t -memTotalAllocated(void) -{ - return TheMeter.alloc.level; -} - -void -memArrayInit(Array * a) -{ - assert(a); - memset(a, 0, sizeof(Array)); -} - -void -memArrayClean(Array * a) -{ - assert(a); - /* could also warn if some objects are left */ - xfree(a->items); - a->items = NULL; -} - -void -memArrayDestroy(Array * a) -{ - assert(a); - memArrayClean(a); - xfree(a); -} - -void -memArrayAppend(Array * a, void *obj) -{ - assert(a); - if (a->count >= a->capacity) - memArrayGrow(a, a->count + 1); - a->items[a->count++] = obj; -} - -/* grows internal buffer to satisfy required minimal capacity */ -static void -memArrayGrow(Array * a, int min_capacity) -{ - const int min_delta = 32; - int delta; - assert(a->capacity < min_capacity); - delta = min_capacity; - /* make delta a multiple of min_delta */ - delta += min_delta - 1; - delta /= min_delta; - delta *= min_delta; - /* actual grow */ - assert(delta > 0); - a->capacity += delta; - a->items = a->items ? - xrealloc(a->items, a->capacity * sizeof(void *)) : - xmalloc(a->capacity * sizeof(void *)); - /* reset, just in case */ - memset(a->items + a->count, 0, (a->capacity - a->count) * sizeof(void *)); -} Index: squid/lib/util.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/util.c,v retrieving revision 1.7.24.3 retrieving revision 1.7.24.4 diff -u -r1.7.24.3 -r1.7.24.4 --- squid/lib/util.c 6 Apr 2001 20:51:21 -0000 1.7.24.3 +++ squid/lib/util.c 9 Apr 2001 06:49:47 -0000 1.7.24.4 @@ -1,6 +1,6 @@ /* - * $Id: util.c,v 1.7.24.3 2001/04/06 20:51:21 akroonmaa Exp $ + * $Id: util.c,v 1.7.24.4 2001/04/09 06:49:47 akroonmaa Exp $ * * DEBUG: * AUTHOR: Harvest Derived @@ -779,40 +779,3 @@ write(2, "\n", 1); abort(); } - -void -gb_flush(gb_t * g) -{ - g->gb += (g->bytes >> 30); - g->bytes &= (1 << 30) - 1; -} - -double -gb_to_double(const gb_t * g) -{ - return ((double) g->gb) * ((double) (1 << 30)) + ((double) g->bytes); -} - -const char * -gb_to_str(const gb_t * g) -{ - /* - * it is often convenient to call gb_to_str several times for _one_ printf - */ -#define max_cc_calls 5 - typedef char GbBuf[32]; - static GbBuf bufs[max_cc_calls]; - static int call_id = 0; - double value = gb_to_double(g); - char *buf = bufs[call_id++]; - if (call_id >= max_cc_calls) - call_id = 0; - /* select format */ - if (value < 1e9) - snprintf(buf, sizeof(GbBuf), "%.2f MB", value / 1e6); - else if (value < 1e12) - snprintf(buf, sizeof(GbBuf), "%.2f GB", value / 1e9); - else - snprintf(buf, sizeof(GbBuf), "%.2f TB", value / 1e12); - return buf; -} --- squid/src/MemPoolStats.c Wed Feb 14 00:51:50 2007 +++ /dev/null Wed Feb 14 00:51:37 2007 @@ -1,311 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 63 Low Level Memory Pool Management - * AUTHOR: Alex Rousskov - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - - -#include "squid.h" -#include "Stack.h" -#include "MemPool.h" - -/* module globals */ - -static double clean_interval = 15.0; /* time to live of idle chunk before release */ - -/* huge constant to set mem_idle_limit to "unlimited" */ -static const size_t mem_unlimited_size = 2 * 1024 * MB; - -/* local prototypes */ -static void memPoolDescribe(const MemPool * pool); -static void memShrink(ssize_t new_limit); - -static double -toMB(size_t size) -{ - return ((double) size) / MB; -} - -static size_t -toKB(size_t size) -{ - return (size + 1024 - 1) / 1024; -} - -/* MemMeter */ - -void -memMeterSyncHWater(MemMeter * m) -{ - assert(m); - if (m->hwater_level < m->level) { - m->hwater_level = m->level; - m->hwater_stamp = squid_curtime; - } -} - -/* Initialization */ - -void -memConfigure(void) -{ - size_t new_pool_limit = mem_idle_limit; - /* set to configured value first */ -#if PURIFY - debug(63, 1) ("Disabling Memory pools under purify\n"); - Config.onoff.mem_pools = 0; -#endif - if (!Config.onoff.mem_pools) - new_pool_limit = 0; - else if (Config.MemPools.limit > 0) - new_pool_limit = Config.MemPools.limit; - else - new_pool_limit = mem_unlimited_size; - /* shrink memory pools if needed */ - if (TheMeter.idle.level > new_pool_limit) { - debug(63, 1) ("Shrinking idle mem pools to %.2f MB\n", toMB(new_pool_limit)); - memShrink(new_pool_limit); - } - mem_idle_limit = new_pool_limit; -} - -void -memPoolCleanIdlePools(void *unused) -{ - int i; - int cleanable = 0; - - for (i = 0; i < Pools.count; i++) { - MemPool *pool = Pools.items[i]; - if (pool->meter.idle.level > (pool->capacity << 1) ) { - memPoolClean(pool, (time_t) clean_interval); - cleanable++; - } - } - - eventAdd("memPoolCleanIdlePools", memPoolCleanIdlePools, NULL, clean_interval, 1); -} - -void -memInitModule(void) -{ - memset(&TheMeter, 0, sizeof(TheMeter)); - memArrayInit(&Pools); - debug(63, 1) ("Memory pools are '%s'; limit: %.2f MB\n", - (Config.onoff.mem_pools ? "on" : "off"), toMB(mem_idle_limit)); -} - -void -memCleanModule(void) -{ - int i; - int dirty_count = 0; - for (i = 0; i < Pools.count; i++) { - MemPool *pool = Pools.items[i]; - if (memPoolInUseCount(pool)) { - memPoolDescribe(pool); - dirty_count++; - } else { - memPoolDestroy(pool); - } - } - if (dirty_count) - debug(63, 2) ("memCleanModule: %d pools are left dirty\n", dirty_count); - /* we clean the stack anyway */ -} - -static void -memShrink(ssize_t new_limit) -{ - ssize_t start_limit = TheMeter.idle.level; - int i; - assert(start_limit >= 0 && new_limit >= 0); - debug(63, 1) ("memShrink: started with %d KB goal: %d KB\n", - toKB(TheMeter.idle.level), toKB(new_limit)); - /* first phase: cut proportionally to the pool idle size */ - for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) { - MemPool *pool = Pools.items[i]; - const ssize_t target_pool_size = (size_t) ((double) pool->meter.idle.level * new_limit) / start_limit; - memPoolShrink(pool, target_pool_size); - } - debug(63, 1) ("memShrink: 1st phase done with %d KB left\n", toKB(TheMeter.idle.level)); - /* second phase: cut to 0 */ - for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) - memPoolShrink(Pools.items[i], 0); - debug(63, 1) ("memShrink: 2nd phase done with %d KB left\n", toKB(TheMeter.idle.level)); -} - -/* MemPoolMeter */ - -static void -memPoolMeterReport(const MemPoolMeter * pm, size_t obj_size, - int alloc_count, int inuse_count, int idle_count, StoreEntry * e) -{ - assert(pm); - storeAppendPrintf(e, "%d\t %d\t %d\t %.2f\t %d\t %d\t %d\t %d\t %d\t %d\t %d\t %d\t %d\t %.1f\t %.1f\t %.1f\n", - /* alloc */ - alloc_count, - toKB(obj_size * pm->alloc.level), - toKB(obj_size * pm->alloc.hwater_level), - (double) ((squid_curtime - pm->alloc.hwater_stamp) / 3600.), - xpercentInt(obj_size * pm->alloc.level, TheMeter.alloc.level), - /* in use */ - inuse_count, - toKB(obj_size * pm->inuse.level), - toKB(obj_size * pm->inuse.hwater_level), - xpercentInt(pm->inuse.level, pm->alloc.level), - /* idle */ - idle_count, - toKB(obj_size * pm->idle.level), - toKB(obj_size * pm->idle.hwater_level), - /* (int)rint(xpercent(pm->idle.level, pm->alloc.level)), */ - /* saved */ - pm->saved.count, - xpercent(pm->saved.count, mem_traffic_volume.count), - xpercent(obj_size * gb_to_double(&pm->saved), gb_to_double(&mem_traffic_volume)), - xpercent(pm->saved.count, pm->total.count)); -} - -/* to-do: make debug level a parameter? */ -static void -memPoolDescribe(const MemPool * pool) -{ - assert(pool); - debug(63, 2) ("%-20s: %6d x %4d bytes = %5d KB\n", - pool->label, memPoolInUseCount(pool), pool->obj_size, - toKB(memPoolInUseSize(pool))); -} - -static int freecount; /* need these outside memPoolReport() */ -static int inuse; - -void -memPoolReport(const MemPool * pool, StoreEntry * e) -{ - int excess,needed; - MemChunk *chunk; - assert(pool); - storeAppendPrintf(e, "%-20s\t %4d\t ", - pool->label, pool->obj_size); -#if CHUNKS - storeAppendPrintf(e, "%4d\t %4d\t ", - toKB(pool->obj_size * pool->capacity), pool->capacity); - freecount=0; - chunk = pool->Chunks; - while (chunk) { - if (chunk->count == 0) - freecount++; - chunk = chunk->next; - } - inuse = pool->chunkCount - freecount; - - if ( pool->meter.inuse.level % pool->capacity ) - needed = pool->meter.inuse.level / pool->capacity + 1; - else - needed = pool->meter.inuse.level / pool->capacity; - - excess = inuse - needed ; - - storeAppendPrintf(e, "%4d\t %4d\t %4d\t %.1f\t ", - pool->chunkCount, inuse, freecount, - xpercent( excess, needed ) ); -/* - Fragmentation calculation: - needed = inuse.level / capacity - excess = used - needed - fragmentation = excess / needed * 100% - - Fragm = (alloced - (inuse / obj_ch) ) / alloced -*/ -#endif - memPoolMeterReport(&pool->meter, pool->obj_size, - pool->meter.alloc.level, pool->meter.inuse.level, pool->meter.idle.level, - e); -} - -void -memReport(StoreEntry * e) -{ - size_t overhd_size = 0; - int alloc_count = 0; - int inuse_count = 0; - int idle_count = 0; - int chunk_count = 0; - int nuse_count = 0; - int free_count = 0; - int i; - /* caption */ - storeAppendPrintf(e, "Current memory usage:\n"); - /* heading */ - storeAppendPrintf(e, "Pool\t Obj Size\t" - " Chunks\t\t\t\t\t\t" - "Allocated\t\t\t\t\t In Use\t\t\t\t Idle\t\t\t Allocations Saved\t\t\t Hit Rate\t\n" - " \t (bytes)\t" - "KB/ch\t obj/ch\t" - "(#)\t used\t free\t %Fragm\t " - "(#)\t (KB)\t high (KB)\t high (hrs)\t %%Tot\t" - "(#)\t (KB)\t high (KB)\t %%alloc\t" - "(#)\t (KB)\t high (KB)\t" - "(count)\t %%cnt\t %%vol\t" - "%%num" - "\n"); - /* main table */ - for (i = 0; i < Pools.count ; i++) { - const MemPool *pool = Pools.items[i]; - if (memPoolWasUsed(pool)) { - memPoolReport(pool, e); - free_count += freecount; - nuse_count += inuse; - chunk_count += pool->chunkCount; - alloc_count += pool->meter.alloc.level; - inuse_count += pool->meter.inuse.level; - idle_count += pool->meter.idle.level; - } - overhd_size += sizeof(MemPool) + sizeof(MemPool *) + - strlen(pool->label) + 1 + - pool->chunkCount * sizeof(MemChunk); - } - overhd_size += sizeof(Pools) + Pools.capacity * sizeof(MemPool *); - /* totals */ - storeAppendPrintf(e, "%-20s\t %-4s\t ", "Total", "-"); - storeAppendPrintf(e, "- \t -\t %4d\t %4d\t %4d\t -\t ", - chunk_count, nuse_count, free_count); - - memPoolMeterReport(&TheMeter, 1, alloc_count, inuse_count, idle_count, e); - storeAppendPrintf(e, "Cumulative allocated volume: %s\n", gb_to_str(&mem_traffic_volume)); - /* overhead */ - storeAppendPrintf(e, "Current overhead: %d bytes (%.3f%%)\n", - overhd_size, xpercent(overhd_size, TheMeter.inuse.level)); - /* limits */ - storeAppendPrintf(e, "Idle pool limit: %.2f MB\n", toMB(mem_idle_limit)); -} - Index: squid/src/defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/defines.h,v retrieving revision 1.10.8.1 retrieving revision 1.10.8.2 diff -u -r1.10.8.1 -r1.10.8.2 --- squid/src/defines.h 6 Apr 2001 20:51:21 -0000 1.10.8.1 +++ squid/src/defines.h 9 Apr 2001 06:49:47 -0000 1.10.8.2 @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.10.8.1 2001/04/06 20:51:21 akroonmaa Exp $ + * $Id: defines.h,v 1.10.8.2 2001/04/09 06:49:47 akroonmaa Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -219,6 +219,10 @@ /* were to look for errors if config path fails */ #define DEFAULT_SQUID_ERROR_DIR "/usr/local/squid/etc/errors" +/* gb_type operations */ +#define gb_flush_limit (0x3FFFFFFF) +#define gb_inc(gb, delta) { if ((gb)->bytes > gb_flush_limit || delta > gb_flush_limit) gb_flush(gb); (gb)->bytes += delta; (gb)->count++; } + /* iteration for HttpHdrRange */ #define HttpHdrRangeInitPos (-1) Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.23.6.1 retrieving revision 1.23.6.2 diff -u -r1.23.6.1 -r1.23.6.2 --- squid/src/protos.h 6 Apr 2001 20:51:21 -0000 1.23.6.1 +++ squid/src/protos.h 9 Apr 2001 06:49:47 -0000 1.23.6.2 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.23.6.1 2001/04/06 20:51:21 akroonmaa Exp $ + * $Id: protos.h,v 1.23.6.2 2001/04/09 06:49:47 akroonmaa Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1131,6 +1131,9 @@ extern dlink_node *dlinkNodeNew(void); extern void kb_incr(kb_t *, size_t); +extern double gb_to_double(const gb_t *); +extern const char *gb_to_str(const gb_t *); +extern void gb_flush(gb_t *); /* internal, do not use this */ extern int stringHasWhitespace(const char *); extern int stringHasCntl(const char *); extern void linklistPush(link_list **, void *); Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.11.10.2 retrieving revision 1.11.10.3 diff -u -r1.11.10.2 -r1.11.10.3 --- squid/src/squid.h 6 Apr 2001 20:51:21 -0000 1.11.10.2 +++ squid/src/squid.h 9 Apr 2001 06:49:48 -0000 1.11.10.3 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.11.10.2 2001/04/06 20:51:21 akroonmaa Exp $ + * $Id: squid.h,v 1.11.10.3 2001/04/09 06:49:48 akroonmaa Exp $ * * AUTHOR: Duane Wessels * Index: squid/src/tools.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/tools.c,v retrieving revision 1.11.12.1 retrieving revision 1.11.12.2 diff -u -r1.11.12.1 -r1.11.12.2 --- squid/src/tools.c 6 Apr 2001 20:51:21 -0000 1.11.12.1 +++ squid/src/tools.c 9 Apr 2001 06:49:48 -0000 1.11.12.2 @@ -1,6 +1,6 @@ /* - * $Id: tools.c,v 1.11.12.1 2001/04/06 20:51:21 akroonmaa Exp $ + * $Id: tools.c,v 1.11.12.2 2001/04/09 06:49:48 akroonmaa Exp $ * * DEBUG: section 21 Misc Functions * AUTHOR: Harvest Derived @@ -831,6 +831,43 @@ } void +gb_flush(gb_t * g) +{ + g->gb += (g->bytes >> 30); + g->bytes &= (1 << 30) - 1; +} + +double +gb_to_double(const gb_t * g) +{ + return ((double) g->gb) * ((double) (1 << 30)) + ((double) g->bytes); +} + +const char * +gb_to_str(const gb_t * g) +{ + /* + * it is often convenient to call gb_to_str several times for _one_ printf + */ +#define max_cc_calls 5 + typedef char GbBuf[32]; + static GbBuf bufs[max_cc_calls]; + static int call_id = 0; + double value = gb_to_double(g); + char *buf = bufs[call_id++]; + if (call_id >= max_cc_calls) + call_id = 0; + /* select format */ + if (value < 1e9) + snprintf(buf, sizeof(GbBuf), "%.2f MB", value / 1e6); + else if (value < 1e12) + snprintf(buf, sizeof(GbBuf), "%.2f GB", value / 1e9); + else + snprintf(buf, sizeof(GbBuf), "%.2f TB", value / 1e12); + return buf; +} + +void debugObj(int section, int level, const char *label, void *obj, ObjPackMethod pm) { MemBuf mb; Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.17.12.1 retrieving revision 1.17.12.2 diff -u -r1.17.12.1 -r1.17.12.2 --- squid/src/typedefs.h 6 Apr 2001 20:51:21 -0000 1.17.12.1 +++ squid/src/typedefs.h 9 Apr 2001 06:49:48 -0000 1.17.12.2 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.17.12.1 2001/04/06 20:51:21 akroonmaa Exp $ + * $Id: typedefs.h,v 1.17.12.2 2001/04/09 06:49:48 akroonmaa Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -46,6 +46,12 @@ size_t kb; } kb_t; +typedef struct { + size_t count; + size_t bytes; + size_t gb; +} gb_t; + /* * grep '^struct' structs.h \ * | perl -ne '($a,$b)=split;$c=$b;$c=~s/^_//; print "typedef struct $b $c;\n";'