--------------------- PatchSet 10197 Date: 2007/12/06 06:34:37 Author: adri Branch: s27_adri Tag: (none) Log: Bring Alex's MemPools code out of src/ and into libmem/. * leave the reporting stuff in src/; it needs to know about StoreEntry's and such. * move some of the global variables and Config related stuff into local structures so the libmem/ code doesn't need to know about the Config struct. Members: Makefile.am:1.20.36.2->1.20.36.3 configure.in:1.184.2.1.4.2->1.184.2.1.4.3 libbuf/String.c:1.1.2.3->1.1.2.4 libbuf/buf.h:1.1.2.2->1.1.2.3 libmem/Makefile.am:1.1->1.1.2.1 libmem/MemPool.c:1.1->1.1.2.1 libmem/MemPool.h:1.1->1.1.2.1 libmem/mem.c:1.1->1.1.2.1 libmem/mem.h:1.1->1.1.2.1 src/HttpHeader.c:1.28.6.1.4.1->1.28.6.1.4.2 src/Makefile.am:1.54.2.1.4.2->1.54.2.1.4.3 src/MemPool.c:1.15->1.15.8.1 src/cache_cf.c:1.102.2.1->1.102.2.1.4.1 src/defines.h:1.43.8.1->1.43.8.2 src/protos.h:1.146.2.4.4.3->1.146.2.4.4.4 src/squid.h:1.36.24.4->1.36.24.5 src/stat.c:1.42.2.3->1.42.2.3.4.1 src/structs.h:1.158.2.5.4.2->1.158.2.5.4.3 src/tools.c:1.62.2.3.4.2->1.62.2.3.4.3 src/typedefs.h:1.43.2.3.4.2->1.43.2.3.4.3 Index: squid/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/Makefile.am,v retrieving revision 1.20.36.2 retrieving revision 1.20.36.3 diff -u -r1.20.36.2 -r1.20.36.3 --- squid/Makefile.am 2 Dec 2007 09:34:13 -0000 1.20.36.2 +++ squid/Makefile.am 6 Dec 2007 06:34:37 -0000 1.20.36.3 @@ -4,8 +4,8 @@ # AUTOMAKE_OPTIONS = dist-bzip2 subdir-objects 1.5 -DIST_SUBDIRS = lib libcore libbuf snmplib scripts src icons errors contrib doc helpers tools -SUBDIRS = lib libcore libbuf @makesnmplib@ scripts src icons errors doc helpers tools +DIST_SUBDIRS = lib libcore libmem libbuf snmplib scripts src icons errors contrib doc helpers tools +SUBDIRS = lib libcore libmem libbuf @makesnmplib@ scripts src icons errors doc helpers tools DISTCLEANFILES = include/stamp-h include/stamp-h[0-9]* DEFAULT_PINGER = $(libexecdir)/`echo pinger | sed '$(transform);s/$$/$(EXEEXT)/'` Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.184.2.1.4.2 retrieving revision 1.184.2.1.4.3 diff -u -r1.184.2.1.4.2 -r1.184.2.1.4.3 --- squid/configure.in 2 Dec 2007 09:34:13 -0000 1.184.2.1.4.2 +++ squid/configure.in 6 Dec 2007 06:34:37 -0000 1.184.2.1.4.3 @@ -1,7 +1,7 @@ dnl dnl Configuration input file for Squid dnl -dnl $Id: configure.in,v 1.184.2.1.4.2 2007/12/02 09:34:13 adri Exp $ +dnl $Id: configure.in,v 1.184.2.1.4.3 2007/12/06 06:34:37 adri Exp $ dnl dnl dnl @@ -10,7 +10,7 @@ AM_CONFIG_HEADER(include/autoconf.h) AC_CONFIG_AUX_DIR(cfgaux) AM_INIT_AUTOMAKE -AC_REVISION($Revision: 1.184.2.1.4.2 $)dnl +AC_REVISION($Revision: 1.184.2.1.4.3 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AM_MAINTAINER_MODE @@ -3174,6 +3174,7 @@ AC_OUTPUT([\ Makefile \ libcore/Makefile \ + libmem/Makefile \ libbuf/Makefile \ lib/Makefile \ scripts/Makefile \ Index: squid/libbuf/String.c =================================================================== RCS file: /cvsroot/squid-sf//squid/libbuf/Attic/String.c,v retrieving revision 1.1.2.3 retrieving revision 1.1.2.4 diff -u -r1.1.2.3 -r1.1.2.4 --- squid/libbuf/String.c 5 Dec 2007 05:35:29 -0000 1.1.2.3 +++ squid/libbuf/String.c 6 Dec 2007 06:34:38 -0000 1.1.2.4 @@ -1,6 +1,6 @@ /* - * $Id: String.c,v 1.1.2.3 2007/12/05 05:35:29 adri Exp $ + * $Id: String.c,v 1.1.2.4 2007/12/06 06:34:38 adri Exp $ * * DEBUG: section 67 String * AUTHOR: Duane Wessels @@ -90,9 +90,10 @@ String stringDup(const String * s) { - String dup; + String dup = StringNull; assert(s); - stringInitStr(&dup, s); + dup.len = s->len; + dup.nb = buf_ref(s->nb); return dup; } @@ -123,6 +124,8 @@ return; } + stringMakePrivate(s); + /* The buffer will have a \0 at the end; so make sure we're not on it */ /* XXX breaks layering again! */ if (buf_len(s->nb) > 0) @@ -134,5 +137,19 @@ void stringMakePrivate(String *s) { + buf_t *nb; + + /* If the reference count is one then we don't bother; its ours. */ + if (buf_refcnt(s->nb) == 1) + return; + + debug(1, 1) ("stringMakePrivate: making %p private (%s)\n", s, strBuf(*s)); + + /* If the reference count is >1 then we create a duplicate and deref the old one */ + assert(buf_refcnt(s->nb) > 1); + nb = buf_create_size(buf_len(s->nb)); + buf_append(nb, buf_buf(s->nb), buf_len(s->nb)); + s->nb = buf_deref(s->nb); + s->nb = nb; /* no need to ref it, its already created with the ref intended for us */ } Index: squid/libbuf/buf.h =================================================================== RCS file: /cvsroot/squid-sf//squid/libbuf/Attic/buf.h,v retrieving revision 1.1.2.2 retrieving revision 1.1.2.3 diff -u -r1.1.2.2 -r1.1.2.3 --- squid/libbuf/buf.h 5 Dec 2007 05:35:29 -0000 1.1.2.2 +++ squid/libbuf/buf.h 6 Dec 2007 06:34:38 -0000 1.1.2.3 @@ -34,6 +34,7 @@ */ #define buf_buf(buf) ((buf)->b) #define buf_len(buf) ((buf)->len) +#define buf_refcnt(buf) ((buf)->nref) static inline int buf_get_chr(const buf_t *buf, int offset) { --- /dev/null Fri Dec 7 01:18:28 2007 +++ squid/libmem/Makefile.am Fri Dec 7 01:18:28 2007 @@ -0,0 +1,13 @@ +## Process this file with automake to produce Makefile.in +# +# $Id: Makefile.am,v 1.1.2.1 2007/12/06 06:34:38 adri Exp $ +# + +libmem_a_SOURCES = \ + MemPool.c + +libmem_a_LIBADD = \ + MemPool.o + +noinst_LIBRARIES = \ + libmem.a --- /dev/null Fri Dec 7 01:18:28 2007 +++ squid/libmem/MemPool.c Fri Dec 7 01:18:28 2007 @@ -0,0 +1,380 @@ + +/* + * $Id: MemPool.c,v 1.1.2.1 2007/12/06 06:34:38 adri Exp $ + * + * DEBUG: section 63 Low Level Memory Pool Management + * AUTHOR: Alex Rousskov + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include +#include +#include +#include +#include + +#include "../include/config.h" +#include "../include/util.h" +#include "../libcore/valgrind.h" +#include "../libcore/tools.h" +#include "../libcore/varargs.h" +#include "../libcore/debug.h" + +#include "../include/Stack.h" + +#include "MemPool.h" + +extern time_t squid_curtime; + +#define MB ((size_t)1024*1024) + +MemPoolCfgStruct MemPoolCfg = { 0, 0, 0, 0 }; + +/* exported */ +MemPoolStatsStruct MemPoolStats = { 0, 0 }; + +/* module globals */ + +/* huge constant to set MemPoolCfg.idle_limit to "unlimited" */ +static const size_t mem_unlimited_size = 2 * 1024 * MB - 1; + +/* memory pool accounting */ +MemPoolMeter TheMeter; +gb_t mem_traffic_volume = {0, 0}; +Stack Pools; + +/* local prototypes */ +static void memShrink(size_t new_limit); +static void memPoolDescribe(const MemPool * pool); +static void memPoolShrink(MemPool * pool, size_t new_limit); + + +static double +toMB(size_t size) +{ + return ((double) size) / MB; +} + +static size_t +toKB(size_t size) +{ + return (size + 1024 - 1) / 1024; +} + +/* Initialization */ + +void +memConfigure(int enable, int limit, int dozero) +{ + size_t new_pool_limit = MemPoolCfg.idle_limit; + /* set to configured value first */ + + MemPoolCfg.enable = enable; + MemPoolCfg.limit = limit; + MemPoolCfg.zero_buffers = dozero; + +#if LEAK_CHECK_MODE +#if PURIFY + if (1) { +#else + if (RUNNING_ON_VALGRIND) { +#endif + debug(63, 1) ("Disabling Memory pools for accurate leak checks\n"); + MemPoolCfg.enable = 0; + } +#endif + if (!MemPoolCfg.enable) + new_pool_limit = 0; + else if (MemPoolCfg.limit > 0) + new_pool_limit = MemPoolCfg.limit; + else + new_pool_limit = mem_unlimited_size; + /* shrink memory pools if needed */ + if (TheMeter.idle.level > new_pool_limit) { + debug(63, 1) ("Shrinking idle mem pools to %.2f MB\n", toMB(new_pool_limit)); + memShrink(new_pool_limit); + } + assert(TheMeter.idle.level <= new_pool_limit); + MemPoolCfg.idle_limit = new_pool_limit; +} + +void +memInitModule(void) +{ + memset(&TheMeter, 0, sizeof(TheMeter)); + stackInit(&Pools); + debug(63, 1) ("Memory pools are '%s'; limit: %.2f MB\n", + (MemPoolCfg.enable ? "on" : "off"), toMB(MemPoolCfg.idle_limit)); +} + +void +memCleanModule(void) +{ + int i; + int dirty_count = 0; + for (i = 0; i < Pools.count; i++) { + MemPool *pool = Pools.items[i]; + if (!pool) + continue; + if (memPoolInUseCount(pool)) { + memPoolDescribe(pool); + dirty_count++; + } else { + memPoolDestroy(pool); + } + } + if (dirty_count) + debug(63, 2) ("memCleanModule: %d pools are left dirty\n", dirty_count); + /* we clean the stack anyway */ + stackClean(&Pools); +} + + +static void +memShrink(size_t new_limit) +{ + size_t start_limit = TheMeter.idle.level; + int i; + debug(63, 1) ("memShrink: started with %ld KB goal: %ld KB\n", + (long int) toKB(TheMeter.idle.level), (long int) toKB(new_limit)); + /* first phase: cut proportionally to the pool idle size */ + for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) { + MemPool *pool = Pools.items[i]; + const size_t target_pool_size = (size_t) ((double) pool->meter.idle.level * new_limit) / start_limit; + memPoolShrink(pool, target_pool_size); + } + debug(63, 1) ("memShrink: 1st phase done with %ld KB left\n", (long int) toKB(TheMeter.idle.level)); + /* second phase: cut to 0 */ + for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) + memPoolShrink(Pools.items[i], 0); + debug(63, 1) ("memShrink: 2nd phase done with %ld KB left\n", (long int) toKB(TheMeter.idle.level)); + assert(TheMeter.idle.level <= new_limit); /* paranoid */ +} + +/* MemMeter */ + +void +memMeterSyncHWater(MemMeter * m) +{ + assert(m); + if (m->hwater_level < m->level) { + m->hwater_level = m->level; + m->hwater_stamp = squid_curtime; + } +} + +/* MemPool */ + +MemPool * +memPoolCreate(const char *label, size_t obj_size) +{ + MemPool *pool = xcalloc(1, sizeof(MemPool)); + assert(label && obj_size); + pool->label = label; + pool->obj_size = obj_size; +#if DEBUG_MEMPOOL + pool->real_obj_size = (obj_size & 7) ? (obj_size | 7) + 1 : obj_size; +#endif + pool->flags.dozero = 1; + stackInit(&pool->pstack); + /* other members are set to 0 */ + stackPush(&Pools, pool); + return pool; +} + +void +memPoolNonZero(MemPool * p) +{ + p->flags.dozero = 0; +} + +void +memPoolDestroy(MemPool * pool) +{ + int i; + assert(pool); + for (i = 0; i < Pools.count; i++) { + if (Pools.items[i] == pool) { + Pools.items[i] = NULL; + break; + } + } + stackClean(&pool->pstack); + xfree(pool); +} + +#if DEBUG_MEMPOOL +#define MEMPOOL_COOKIE(p) ((void *)((unsigned long)(p) ^ 0xDEADBEEF)) +struct mempool_cookie { + MemPool *pool; + void *cookie; +}; + +#endif + +void * +memPoolAlloc(MemPool * pool) +{ + void *obj; + assert(pool); + memMeterInc(pool->meter.inuse); + gb_inc(&pool->meter.total, 1); + gb_inc(&TheMeter.total, pool->obj_size); + memMeterAdd(TheMeter.inuse, pool->obj_size); + gb_inc(&mem_traffic_volume, pool->obj_size); + MemPoolStats.mem_pool_alloc_calls++; + if (pool->pstack.count) { + assert(pool->meter.idle.level); + memMeterDec(pool->meter.idle); + memMeterDel(TheMeter.idle, pool->obj_size); + gb_inc(&pool->meter.saved, 1); + gb_inc(&TheMeter.saved, pool->obj_size); + obj = stackPop(&pool->pstack); +#if DEBUG_MEMPOOL + (void) VALGRIND_MAKE_READABLE(obj, pool->real_obj_size + sizeof(struct mempool_cookie)); +#else + (void) VALGRIND_MAKE_READABLE(obj, pool->obj_size); +#endif +#if DEBUG_MEMPOOL + { + struct mempool_cookie *cookie = (void *) (((unsigned char *) obj) + pool->real_obj_size); + assert(cookie->cookie == MEMPOOL_COOKIE(obj)); + assert(cookie->pool == pool); + (void) VALGRIND_MAKE_NOACCESS(cookie, sizeof(cookie)); + } +#endif + } else { + assert(!pool->meter.idle.level); + memMeterInc(pool->meter.alloc); + memMeterAdd(TheMeter.alloc, pool->obj_size); +#if DEBUG_MEMPOOL + { + struct mempool_cookie *cookie; + obj = xcalloc(1, pool->real_obj_size + sizeof(struct mempool_cookie)); + cookie = (struct mempool_cookie *) (((unsigned char *) obj) + pool->real_obj_size); + cookie->cookie = MEMPOOL_COOKIE(obj); + cookie->pool = pool; + (void) VALGRIND_MAKE_NOACCESS(cookie, sizeof(cookie)); + } +#else + if (MemPoolCfg.zero_buffers || pool->flags.dozero) + obj = xcalloc(1, pool->obj_size); + else + obj = xmalloc(pool->obj_size); +#endif + } + return obj; +} + +void +memPoolFree(MemPool * pool, void *obj) +{ + assert(pool && obj); + memMeterDec(pool->meter.inuse); + memMeterDel(TheMeter.inuse, pool->obj_size); + MemPoolStats.mem_pool_free_calls++; + (void) VALGRIND_CHECK_WRITABLE(obj, pool->obj_size); +#if DEBUG_MEMPOOL + { + struct mempool_cookie *cookie = (void *) (((unsigned char *) obj) + pool->real_obj_size); + (void) VALGRIND_MAKE_READABLE(cookie, sizeof(cookie)); + assert(cookie->cookie == MEMPOOL_COOKIE(obj)); + assert(cookie->pool == pool); + } +#endif + if (TheMeter.idle.level + pool->obj_size <= MemPoolCfg.idle_limit) { + memMeterInc(pool->meter.idle); + memMeterAdd(TheMeter.idle, pool->obj_size); + if (MemPoolCfg.zero_buffers || pool->flags.dozero) + memset(obj, 0, pool->obj_size); +#if DEBUG_MEMPOOL + (void) VALGRIND_MAKE_NOACCESS(obj, pool->real_obj_size + sizeof(struct mempool_cookie)); +#else + (void) VALGRIND_MAKE_NOACCESS(obj, pool->obj_size); +#endif + stackPush(&pool->pstack, obj); + } else { + memMeterDec(pool->meter.alloc); + memMeterDel(TheMeter.alloc, pool->obj_size); + xfree(obj); + } + assert(pool->meter.idle.level <= pool->meter.alloc.level); +} + +static void +memPoolShrink(MemPool * pool, size_t new_limit) +{ + assert(pool); + while (pool->meter.idle.level > new_limit && pool->pstack.count > 0) { + memMeterDec(pool->meter.alloc); + memMeterDec(pool->meter.idle); + memMeterDel(TheMeter.idle, pool->obj_size); + memMeterDel(TheMeter.alloc, pool->obj_size); + xfree(stackPop(&pool->pstack)); + } + assert(pool->meter.idle.level <= new_limit); /* paranoid */ +} + +int +memPoolWasUsed(const MemPool * pool) +{ + assert(pool); + return pool->meter.alloc.hwater_level > 0; +} + +int +memPoolInUseCount(const MemPool * pool) +{ + assert(pool); + return pool->meter.inuse.level; +} + +size_t +memPoolInUseSize(const MemPool * pool) +{ + assert(pool); + return (pool->obj_size * pool->meter.inuse.level); +} + +/* to-do: make debug level a parameter? */ +static void +memPoolDescribe(const MemPool * pool) +{ + assert(pool); + debug(63, 2) ("%-20s: %6d x %4d bytes = %5ld KB\n", + pool->label, memPoolInUseCount(pool), (int) pool->obj_size, + (long int) toKB(memPoolInUseSize(pool))); +} + +size_t +memTotalAllocated(void) +{ + return TheMeter.alloc.level; +} + --- /dev/null Fri Dec 7 01:18:28 2007 +++ squid/libmem/MemPool.h Fri Dec 7 01:18:28 2007 @@ -0,0 +1,82 @@ +#ifndef __LIBCORE_MEMPOOL_H__ +#define __LIBCORE_MEMPOOL_H__ + +typedef struct _MemMeter MemMeter; +typedef struct _MemPoolMeter MemPoolMeter; +typedef struct _MemPool MemPool; + +/* object to track per-action memory usage (e.g. #idle objects) */ +struct _MemMeter { + ssize_t level; /* current level (count or volume) */ + ssize_t hwater_level; /* high water mark */ + time_t hwater_stamp; /* timestamp of last high water mark change */ +}; + +/* object to track per-pool memory usage (alloc = inuse+idle) */ +struct _MemPoolMeter { + MemMeter alloc; + MemMeter inuse; + MemMeter idle; + gb_t saved; + gb_t total; +}; + +/* a pool is a [growing] space for objects of the same size */ +struct _MemPool { + const char *label; + size_t obj_size; +#if DEBUG_MEMPOOL + size_t real_obj_size; /* with alignment */ +#endif + struct { + int dozero:1; + } flags; + Stack pstack; /* stack for free pointers */ + MemPoolMeter meter; +#if DEBUG_MEMPOOL + MemPoolMeter diff_meter; +#endif +}; + +typedef struct { + int enable; + int limit; + int zero_buffers; + int idle_limit; +} MemPoolCfgStruct; + +extern MemPoolCfgStruct MemPoolCfg; + +typedef struct { + unsigned int mem_pool_alloc_calls; + unsigned int mem_pool_free_calls; +} MemPoolStatsStruct; + +extern MemPoolStatsStruct MemPoolStats; + +/* XXX these should be moved into a MemPool* prefixed name ! */ +extern MemPoolMeter TheMeter; +extern gb_t mem_traffic_volume; +extern Stack Pools; + + +extern void memConfigure(int enable, int limit, int dozero); +extern MemPool *memPoolCreate(const char *label, size_t obj_size); +extern void memPoolDestroy(MemPool * pool); +extern void memPoolNonZero(MemPool * p); +extern void *memPoolAlloc(MemPool * pool); +extern void memPoolFree(MemPool * pool, void *obj); +extern int memPoolWasUsed(const MemPool * pool); +extern int memPoolInUseCount(const MemPool * pool); +extern size_t memPoolInUseSize(const MemPool * pool); +extern int memPoolUsedCount(const MemPool * pool); + +/* MemMeter */ +extern void memMeterSyncHWater(MemMeter * m); +#define memMeterCheckHWater(m) { if ((m).hwater_level < (m).level) memMeterSyncHWater(&(m)); } +#define memMeterInc(m) { (m).level++; memMeterCheckHWater(m); } +#define memMeterDec(m) { (m).level--; } +#define memMeterAdd(m, sz) { (m).level += (sz); memMeterCheckHWater(m); } +#define memMeterDel(m, sz) { (m).level -= (sz); } + +#endif /* __LIBCORE_MEMPOOL_H__ */ --- /dev/null Fri Dec 7 01:18:28 2007 +++ squid/libmem/mem.c Fri Dec 7 01:18:29 2007 @@ -0,0 +1,469 @@ + +/* + * $Id: mem.c,v 1.1.2.1 2007/12/06 06:34:38 adri Exp $ + * + * DEBUG: section 13 High Level Memory Pool Management + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +/* module globals */ + +static MemPool *MemPools[MEM_MAX]; + +/* string pools */ +#define mem_str_pool_count 3 +static const struct { + const char *name; + size_t obj_size; +} StrPoolsAttrs[mem_str_pool_count] = { + + { + "Short Strings", 36, + }, /* to fit rfc1123 and similar */ + { + "Medium Strings", 128, + }, /* to fit most urls */ + { + "Long Strings", 512 + } /* other */ +}; +static struct { + MemPool *pool; +} StrPools[mem_str_pool_count]; +static MemMeter StrCountMeter; +static MemMeter StrVolumeMeter; + +static MemMeter HugeBufCountMeter; +static MemMeter HugeBufVolumeMeter; + +/* local routines */ + +static void +memStringStats(StoreEntry * sentry) +{ + const char *pfmt = "%-20s\t %d\t %d\n"; + int i; + int pooled_count = 0; + size_t pooled_volume = 0; + /* heading */ + storeAppendPrintf(sentry, + "String Pool\t Impact\t\t\n" + " \t (%%strings)\t (%%volume)\n"); + /* table body */ + for (i = 0; i < mem_str_pool_count; i++) { + const MemPool *pool = StrPools[i].pool; + const int plevel = pool->meter.inuse.level; + storeAppendPrintf(sentry, pfmt, + pool->label, + xpercentInt(plevel, StrCountMeter.level), + xpercentInt(plevel * pool->obj_size, StrVolumeMeter.level)); + pooled_count += plevel; + pooled_volume += plevel * pool->obj_size; + } + /* malloc strings */ + storeAppendPrintf(sentry, pfmt, + "Other Strings", + xpercentInt(StrCountMeter.level - pooled_count, StrCountMeter.level), + xpercentInt(StrVolumeMeter.level - pooled_volume, StrVolumeMeter.level)); + storeAppendPrintf(sentry, "\n"); +} + +static void +memBufStats(StoreEntry * sentry) +{ + storeAppendPrintf(sentry, "Large buffers: %d (%d KB)\n", + (int) HugeBufCountMeter.level, + (int) HugeBufVolumeMeter.level / 1024); +} + +static void +memStats(StoreEntry * sentry) +{ + storeBuffer(sentry); + memReport(sentry); + memStringStats(sentry); + memBufStats(sentry); + storeBufferFlush(sentry); +#if WITH_VALGRIND + if (RUNNING_ON_VALGRIND) { + long int leaked = 0, dubious = 0, reachable = 0, suppressed = 0; + storeAppendPrintf(sentry, "Valgrind Report:\n"); + storeAppendPrintf(sentry, "Type\tAmount\n"); + debug(13, 1) ("Asking valgrind for memleaks\n"); + VALGRIND_DO_LEAK_CHECK; + debug(13, 1) ("Getting valgrind statistics\n"); + VALGRIND_COUNT_LEAKS(leaked, dubious, reachable, suppressed); + storeAppendPrintf(sentry, "Leaked\t%ld\n", leaked); + storeAppendPrintf(sentry, "Dubious\t%ld\n", dubious); + storeAppendPrintf(sentry, "Reachable\t%ld\n", reachable); + storeAppendPrintf(sentry, "Suppressed\t%ld\n", suppressed); + } +#endif +} + + +/* + * public routines + */ + +/* + * we have a limit on _total_ amount of idle memory so we ignore + * max_pages for now + */ +void +memDataInit(mem_type type, const char *name, size_t size, int max_pages_notused) +{ + assert(name && size); + MemPools[type] = memPoolCreate(name, size); +} + +static void +memDataNonZero(mem_type type) +{ + memPoolNonZero(MemPools[type]); +} + + +/* find appropriate pool and use it (pools always init buffer with 0s) */ +void * +memAllocate(mem_type type) +{ + return memPoolAlloc(MemPools[type]); +} + +/* give memory back to the pool */ +void +memFree(void *p, int type) +{ + memPoolFree(MemPools[type], p); +} + +/* allocate a variable size buffer using best-fit pool */ +void * +memAllocString(size_t net_size, size_t * gross_size) +{ + int i; + MemPool *pool = NULL; + assert(gross_size); + for (i = 0; i < mem_str_pool_count; i++) { + if (net_size <= StrPoolsAttrs[i].obj_size) { + pool = StrPools[i].pool; + break; + } + } + *gross_size = pool ? pool->obj_size : net_size; + assert(*gross_size >= net_size); + memMeterInc(StrCountMeter); + memMeterAdd(StrVolumeMeter, *gross_size); + return pool ? memPoolAlloc(pool) : xcalloc(1, net_size); +} + +/* free buffer allocated with memAllocString() */ +void +memFreeString(size_t size, void *buf) +{ + int i; + MemPool *pool = NULL; + assert(size && buf); + for (i = 0; i < mem_str_pool_count; i++) { + if (size <= StrPoolsAttrs[i].obj_size) { + assert(size == StrPoolsAttrs[i].obj_size); + pool = StrPools[i].pool; + break; + } + } + memMeterDec(StrCountMeter); + memMeterDel(StrVolumeMeter, size); + pool ? memPoolFree(pool, buf) : xfree(buf); +} + +/* Find the best fit MEM_X_BUF type */ +static mem_type +memFindBufSizeType(size_t net_size, size_t * gross_size) +{ + mem_type type; + size_t size; + if (net_size <= 2 * 1024) { + type = MEM_2K_BUF; + size = 2 * 1024; + } else if (net_size <= 4 * 1024) { + type = MEM_4K_BUF; + size = 4 * 1024; + } else if (net_size <= 8 * 1024) { + type = MEM_8K_BUF; + size = 8 * 1024; + } else if (net_size <= 16 * 1024) { + type = MEM_16K_BUF; + size = 16 * 1024; + } else if (net_size <= 32 * 1024) { + type = MEM_32K_BUF; + size = 32 * 1024; + } else if (net_size <= 64 * 1024) { + type = MEM_64K_BUF; + size = 64 * 1024; + } else { + type = MEM_NONE; + size = net_size; + } + if (gross_size) + *gross_size = size; + return type; +} + +/* allocate a variable size buffer using best-fit pool */ +void * +memAllocBuf(size_t net_size, size_t * gross_size) +{ + mem_type type = memFindBufSizeType(net_size, gross_size); + if (type != MEM_NONE) + return memAllocate(type); + else { + memMeterInc(HugeBufCountMeter); + memMeterAdd(HugeBufVolumeMeter, *gross_size); + return xcalloc(1, net_size); + } +} + +/* resize a variable sized buffer using best-fit pool */ +void * +memReallocBuf(void *oldbuf, size_t net_size, size_t * gross_size) +{ + /* XXX This can be optimized on very large buffers to use realloc() */ + size_t new_gross_size; + void *newbuf = memAllocBuf(net_size, &new_gross_size); + if (oldbuf) { + int data_size = *gross_size; + if (data_size > net_size) + data_size = net_size; + memcpy(newbuf, oldbuf, data_size); + memFreeBuf(*gross_size, oldbuf); + } + *gross_size = new_gross_size; + return newbuf; +} + +/* free buffer allocated with memAllocBuf() */ +void +memFreeBuf(size_t size, void *buf) +{ + mem_type type = memFindBufSizeType(size, NULL); + if (type != MEM_NONE) + memFree(buf, type); + else { + xfree(buf); + memMeterDec(HugeBufCountMeter); + memMeterDel(HugeBufVolumeMeter, size); + } +} + + +void +memInit(void) +{ + int i; + memInitModule(); + /* set all pointers to null */ + memset(MemPools, '\0', sizeof(MemPools)); + /* + * it does not hurt much to have a lot of pools since sizeof(MemPool) is + * small; someday we will figure out what to do with all the entries here + * that are never used or used only once; perhaps we should simply use + * malloc() for those? @?@ + */ + memDataInit(MEM_2K_BUF, "2K Buffer", 2048, 10); + memDataNonZero(MEM_2K_BUF); + memDataInit(MEM_4K_BUF, "4K Buffer", 4096, 10); + memDataNonZero(MEM_4K_BUF); + memDataInit(MEM_8K_BUF, "8K Buffer", 8192, 10); + memDataNonZero(MEM_8K_BUF); + memDataInit(MEM_16K_BUF, "16K Buffer", 16384, 10); + memDataNonZero(MEM_16K_BUF); + memDataInit(MEM_32K_BUF, "32K Buffer", 32768, 10); + memDataNonZero(MEM_32K_BUF); + memDataInit(MEM_64K_BUF, "64K Buffer", 65536, 10); + memDataNonZero(MEM_64K_BUF); + memDataInit(MEM_STORE_CLIENT_BUF, "Store Client Buffer", STORE_CLIENT_BUF_SZ, 0); + memDataNonZero(MEM_STORE_CLIENT_BUF); + memDataInit(MEM_ACL, "acl", sizeof(acl), 0); + memDataInit(MEM_ACL_DENY_INFO_LIST, "acl_deny_info_list", + sizeof(acl_deny_info_list), 0); + memDataInit(MEM_ACL_IP_DATA, "acl_ip_data", sizeof(acl_ip_data), 0); + memDataInit(MEM_ACL_LIST, "acl_list", sizeof(acl_list), 0); + memDataInit(MEM_ACL_NAME_LIST, "acl_name_list", sizeof(acl_name_list), 0); +#if USE_SSL + memDataInit(MEM_ACL_CERT_DATA, "acl_cert_data", sizeof(acl_cert_data), 0); +#endif + memDataInit(MEM_ACL_TIME_DATA, "acl_time_data", sizeof(acl_time_data), 0); + memDataInit(MEM_ACL_REQUEST_TYPE, "acl_request_type", sizeof(acl_request_type), 0); + memDataInit(MEM_AUTH_USER_T, "auth_user_t", + sizeof(auth_user_t), 0); + memDataInit(MEM_AUTH_USER_HASH, "auth_user_hash_pointer", + sizeof(auth_user_hash_pointer), 0); + memDataInit(MEM_ACL_PROXY_AUTH_MATCH, "acl_proxy_auth_match_cache", + sizeof(acl_proxy_auth_match_cache), 0); + memDataInit(MEM_ACL_USER_DATA, "acl_user_data", + sizeof(acl_user_data), 0); +#if USE_CACHE_DIGESTS + memDataInit(MEM_CACHE_DIGEST, "CacheDigest", sizeof(CacheDigest), 0); +#endif + memDataInit(MEM_LINK_LIST, "link_list", sizeof(link_list), 10); + memDataInit(MEM_DLINK_NODE, "dlink_node", sizeof(dlink_node), 10); + memDataInit(MEM_DREAD_CTRL, "dread_ctrl", sizeof(dread_ctrl), 0); + memDataInit(MEM_DWRITE_Q, "dwrite_q", sizeof(dwrite_q), 0); + memDataInit(MEM_FWD_SERVER, "FwdServer", sizeof(FwdServer), 0); + memDataInit(MEM_HTTP_REPLY, "HttpReply", sizeof(HttpReply), 0); + memDataInit(MEM_HTTP_HDR_ENTRY, "HttpHeaderEntry", sizeof(HttpHeaderEntry), 0); + memDataInit(MEM_HTTP_HDR_CC, "HttpHdrCc", sizeof(HttpHdrCc), 0); + memDataInit(MEM_HTTP_HDR_RANGE_SPEC, "HttpHdrRangeSpec", sizeof(HttpHdrRangeSpec), 0); + memDataInit(MEM_HTTP_HDR_RANGE, "HttpHdrRange", sizeof(HttpHdrRange), 0); + memDataInit(MEM_HTTP_HDR_CONTENT_RANGE, "HttpHdrContRange", sizeof(HttpHdrContRange), 0); + memDataInit(MEM_INTLIST, "intlist", sizeof(intlist), 0); + memDataInit(MEM_MEMOBJECT, "MemObject", sizeof(MemObject), + Squid_MaxFD >> 3); + memDataInit(MEM_MEM_NODE, "mem_node", sizeof(mem_node), 0); + memDataNonZero(MEM_MEM_NODE); + memDataInit(MEM_NETDBENTRY, "netdbEntry", sizeof(netdbEntry), 0); + memDataInit(MEM_NET_DB_NAME, "net_db_name", sizeof(net_db_name), 0); + memDataInit(MEM_RELIST, "relist", sizeof(relist), 0); + memDataInit(MEM_REQUEST_T, "request_t", sizeof(request_t), + Squid_MaxFD >> 3); + memDataInit(MEM_STOREENTRY, "StoreEntry", sizeof(StoreEntry), 0); + memDataInit(MEM_WORDLIST, "wordlist", sizeof(wordlist), 0); + memDataInit(MEM_CLIENT_INFO, "ClientInfo", sizeof(ClientInfo), 0); + memDataInit(MEM_MD5_DIGEST, "MD5 digest", MD5_DIGEST_CHARS, 0); + memDataInit(MEM_HELPER_REQUEST, "helper_request", + sizeof(helper_request), 0); + memDataInit(MEM_HELPER_STATEFUL_REQUEST, "helper_stateful_request", + sizeof(helper_stateful_request), 0); + memDataInit(MEM_TLV, "storeSwapTLV", sizeof(tlv), 0); + memDataInit(MEM_SWAP_LOG_DATA, "storeSwapLogData", sizeof(storeSwapLogData), 0); + + /* init string pools */ + for (i = 0; i < mem_str_pool_count; i++) { + StrPools[i].pool = memPoolCreate(StrPoolsAttrs[i].name, StrPoolsAttrs[i].obj_size); + memPoolNonZero(StrPools[i].pool); + } + cachemgrRegister("mem", + "Memory Utilization", + memStats, 0, 1); +} + +/* + * Test that all entries are initialized + */ +void +memCheckInit(void) +{ + mem_type t; + for (t = MEM_NONE, t++; t < MEM_MAX; t++) { + if (MEM_DONTFREE == t) + continue; + /* + * If you hit this assertion, then you forgot to add a + * memDataInit() line for type 't'. + */ + assert(MemPools[t]); + } +} + +void +memClean(void) +{ + memCleanModule(); +} + +int +memInUse(mem_type type) +{ + return memPoolInUseCount(MemPools[type]); +} + +/* ick */ + +static void +memFree2K(void *p) +{ + memFree(p, MEM_2K_BUF); +} + +void +memFree4K(void *p) +{ + memFree(p, MEM_4K_BUF); +} + +void +memFree8K(void *p) +{ + memFree(p, MEM_8K_BUF); +} + +static void +memFree16K(void *p) +{ + memFree(p, MEM_16K_BUF); +} + +static void +memFree32K(void *p) +{ + memFree(p, MEM_32K_BUF); +} + +static void +memFree64K(void *p) +{ + memFree(p, MEM_64K_BUF); +} + +FREE * +memFreeBufFunc(size_t size) +{ + switch (size) { + case 2 * 1024: + return memFree2K; + case 4 * 1024: + return memFree4K; + case 8 * 1024: + return memFree8K; + case 16 * 1024: + return memFree16K; + case 32 * 1024: + return memFree32K; + case 64 * 1024: + return memFree64K; + default: + memMeterDec(HugeBufCountMeter); + memMeterDel(HugeBufVolumeMeter, size); + return xfree; + } +} --- /dev/null Fri Dec 7 01:18:28 2007 +++ squid/libmem/mem.h Fri Dec 7 01:18:29 2007 @@ -0,0 +1,6 @@ +#ifndef __LIBCORE_MEM_H__ +#define __LIBCORE_MEM_H__ + + + +#endif Index: squid/src/HttpHeader.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpHeader.c,v retrieving revision 1.28.6.1.4.1 retrieving revision 1.28.6.1.4.2 diff -u -r1.28.6.1.4.1 -r1.28.6.1.4.2 --- squid/src/HttpHeader.c 3 Dec 2007 08:54:28 -0000 1.28.6.1.4.1 +++ squid/src/HttpHeader.c 6 Dec 2007 06:34:38 -0000 1.28.6.1.4.2 @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.c,v 1.28.6.1.4.1 2007/12/03 08:54:28 adri Exp $ + * $Id: HttpHeader.c,v 1.28.6.1.4.2 2007/12/06 06:34:38 adri Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -807,7 +807,7 @@ return httpHeaderGetList(hdr, id); if ((e = httpHeaderFindEntry(hdr, id))) { String s; - stringLimitInit(&s, strBuf(e->value), strLen(e->value)); + s = stringDup(&e->value); return s; } return StringNull; Index: squid/src/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v retrieving revision 1.54.2.1.4.2 retrieving revision 1.54.2.1.4.3 diff -u -r1.54.2.1.4.2 -r1.54.2.1.4.3 --- squid/src/Makefile.am 2 Dec 2007 09:34:15 -0000 1.54.2.1.4.2 +++ squid/src/Makefile.am 6 Dec 2007 06:34:39 -0000 1.54.2.1.4.3 @@ -272,6 +272,7 @@ squid_LDADD = \ -L../lib \ + -L../libmem \ -L../libcore \ -L../libbuf \ @XTRA_OBJS@ \ @@ -286,6 +287,7 @@ @LIB_EPOLL@ \ -lmiscutil \ -lcore \ + -lmem \ -lbuf \ @XTRA_LIBS@ \ $(MINGWEXLIB) @@ -324,7 +326,7 @@ data_DATA = \ mib.txt -LDADD = -L../lib -L../libcore -L../libbuf -lcore -lbuf -lmiscutil @XTRA_LIBS@ +LDADD = -L../lib -L../libcore -L../libmem -L../libbuf -lcore -lbuf -lmiscutil @XTRA_LIBS@ EXTRA_DIST = \ cf_gen_defines \ Index: squid/src/MemPool.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/MemPool.c,v retrieving revision 1.15 retrieving revision 1.15.8.1 diff -u -r1.15 -r1.15.8.1 --- squid/src/MemPool.c 21 Sep 2007 09:52:23 -0000 1.15 +++ squid/src/MemPool.c 6 Dec 2007 06:34:39 -0000 1.15.8.1 @@ -1,6 +1,6 @@ /* - * $Id: MemPool.c,v 1.15 2007/09/21 09:52:23 squidadm Exp $ + * $Id: MemPool.c,v 1.15.8.1 2007/12/06 06:34:39 adri Exp $ * * DEBUG: section 63 Low Level Memory Pool Management * AUTHOR: Alex Rousskov @@ -37,32 +37,8 @@ #include "squid.h" #include "Stack.h" +/* XXX these should be in libcore! */ #define MB ((size_t)1024*1024) - -/* exported */ -unsigned int mem_pool_alloc_calls = 0; -unsigned int mem_pool_free_calls = 0; - -/* module globals */ - -/* huge constant to set mem_idle_limit to "unlimited" */ -static const size_t mem_unlimited_size = 2 * 1024 * MB - 1; - -/* we cannot keep idle more than this limit */ -static size_t mem_idle_limit = 0; - -/* memory pool accounting */ -static MemPoolMeter TheMeter; -static gb_t mem_traffic_volume = -{0, 0}; -static Stack Pools; - -/* local prototypes */ -static void memShrink(size_t new_limit); -static void memPoolDescribe(const MemPool * pool); -static void memPoolShrink(MemPool * pool, size_t new_limit); - - static double toMB(size_t size) { @@ -76,91 +52,6 @@ } -/* Initialization */ - -void -memConfigure(void) -{ - size_t new_pool_limit = mem_idle_limit; - /* set to configured value first */ -#if LEAK_CHECK_MODE -#if PURIFY - if (1) { -#else - if (RUNNING_ON_VALGRIND) { -#endif - debug(63, 1) ("Disabling Memory pools for accurate leak checks\n"); - Config.onoff.mem_pools = 0; - } -#endif - if (!Config.onoff.mem_pools) - new_pool_limit = 0; - else if (Config.MemPools.limit > 0) - new_pool_limit = Config.MemPools.limit; - else - new_pool_limit = mem_unlimited_size; - /* shrink memory pools if needed */ - if (TheMeter.idle.level > new_pool_limit) { - debug(63, 1) ("Shrinking idle mem pools to %.2f MB\n", toMB(new_pool_limit)); - memShrink(new_pool_limit); - } - assert(TheMeter.idle.level <= new_pool_limit); - mem_idle_limit = new_pool_limit; -} - -void -memInitModule(void) -{ - memset(&TheMeter, 0, sizeof(TheMeter)); - stackInit(&Pools); - debug(63, 1) ("Memory pools are '%s'; limit: %.2f MB\n", - (Config.onoff.mem_pools ? "on" : "off"), toMB(mem_idle_limit)); -} - -void -memCleanModule(void) -{ - int i; - int dirty_count = 0; - for (i = 0; i < Pools.count; i++) { - MemPool *pool = Pools.items[i]; - if (!pool) - continue; - if (memPoolInUseCount(pool)) { - memPoolDescribe(pool); - dirty_count++; - } else { - memPoolDestroy(pool); - } - } - if (dirty_count) - debug(63, 2) ("memCleanModule: %d pools are left dirty\n", dirty_count); - /* we clean the stack anyway */ - stackClean(&Pools); -} - - -static void -memShrink(size_t new_limit) -{ - size_t start_limit = TheMeter.idle.level; - int i; - debug(63, 1) ("memShrink: started with %ld KB goal: %ld KB\n", - (long int) toKB(TheMeter.idle.level), (long int) toKB(new_limit)); - /* first phase: cut proportionally to the pool idle size */ - for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) { - MemPool *pool = Pools.items[i]; - const size_t target_pool_size = (size_t) ((double) pool->meter.idle.level * new_limit) / start_limit; - memPoolShrink(pool, target_pool_size); - } - debug(63, 1) ("memShrink: 1st phase done with %ld KB left\n", (long int) toKB(TheMeter.idle.level)); - /* second phase: cut to 0 */ - for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) - memPoolShrink(Pools.items[i], 0); - debug(63, 1) ("memShrink: 2nd phase done with %ld KB left\n", (long int) toKB(TheMeter.idle.level)); - assert(TheMeter.idle.level <= new_limit); /* paranoid */ -} - /* MemPoolMeter */ static void @@ -195,205 +86,6 @@ /* MemMeter */ -void -memMeterSyncHWater(MemMeter * m) -{ - assert(m); - if (m->hwater_level < m->level) { - m->hwater_level = m->level; - m->hwater_stamp = squid_curtime; - } -} - -/* MemPool */ - -MemPool * -memPoolCreate(const char *label, size_t obj_size) -{ - MemPool *pool = xcalloc(1, sizeof(MemPool)); - assert(label && obj_size); - pool->label = label; - pool->obj_size = obj_size; -#if DEBUG_MEMPOOL - pool->real_obj_size = (obj_size & 7) ? (obj_size | 7) + 1 : obj_size; -#endif - pool->flags.dozero = 1; - stackInit(&pool->pstack); - /* other members are set to 0 */ - stackPush(&Pools, pool); - return pool; -} - -void -memPoolNonZero(MemPool * p) -{ - p->flags.dozero = 0; -} - -void -memPoolDestroy(MemPool * pool) -{ - int i; - assert(pool); - for (i = 0; i < Pools.count; i++) { - if (Pools.items[i] == pool) { - Pools.items[i] = NULL; - break; - } - } - stackClean(&pool->pstack); - xfree(pool); -} - -#if DEBUG_MEMPOOL -#define MEMPOOL_COOKIE(p) ((void *)((unsigned long)(p) ^ 0xDEADBEEF)) -struct mempool_cookie { - MemPool *pool; - void *cookie; -}; - -#endif - -void * -memPoolAlloc(MemPool * pool) -{ - void *obj; - assert(pool); - memMeterInc(pool->meter.inuse); - gb_inc(&pool->meter.total, 1); - gb_inc(&TheMeter.total, pool->obj_size); - memMeterAdd(TheMeter.inuse, pool->obj_size); - gb_inc(&mem_traffic_volume, pool->obj_size); - mem_pool_alloc_calls++; - if (pool->pstack.count) { - assert(pool->meter.idle.level); - memMeterDec(pool->meter.idle); - memMeterDel(TheMeter.idle, pool->obj_size); - gb_inc(&pool->meter.saved, 1); - gb_inc(&TheMeter.saved, pool->obj_size); - obj = stackPop(&pool->pstack); -#if DEBUG_MEMPOOL - (void) VALGRIND_MAKE_READABLE(obj, pool->real_obj_size + sizeof(struct mempool_cookie)); -#else - (void) VALGRIND_MAKE_READABLE(obj, pool->obj_size); -#endif -#if DEBUG_MEMPOOL - { - struct mempool_cookie *cookie = (void *) (((unsigned char *) obj) + pool->real_obj_size); - assert(cookie->cookie == MEMPOOL_COOKIE(obj)); - assert(cookie->pool == pool); - (void) VALGRIND_MAKE_NOACCESS(cookie, sizeof(cookie)); - } -#endif - } else { - assert(!pool->meter.idle.level); - memMeterInc(pool->meter.alloc); - memMeterAdd(TheMeter.alloc, pool->obj_size); -#if DEBUG_MEMPOOL - { - struct mempool_cookie *cookie; - obj = xcalloc(1, pool->real_obj_size + sizeof(struct mempool_cookie)); - cookie = (struct mempool_cookie *) (((unsigned char *) obj) + pool->real_obj_size); - cookie->cookie = MEMPOOL_COOKIE(obj); - cookie->pool = pool; - (void) VALGRIND_MAKE_NOACCESS(cookie, sizeof(cookie)); - } -#else - if (Config.onoff.zero_buffers || pool->flags.dozero) - obj = xcalloc(1, pool->obj_size); - else - obj = xmalloc(pool->obj_size); -#endif - } - return obj; -} - -void -memPoolFree(MemPool * pool, void *obj) -{ - assert(pool && obj); - memMeterDec(pool->meter.inuse); - memMeterDel(TheMeter.inuse, pool->obj_size); - mem_pool_free_calls++; - (void) VALGRIND_CHECK_WRITABLE(obj, pool->obj_size); -#if DEBUG_MEMPOOL - { - struct mempool_cookie *cookie = (void *) (((unsigned char *) obj) + pool->real_obj_size); - (void) VALGRIND_MAKE_READABLE(cookie, sizeof(cookie)); - assert(cookie->cookie == MEMPOOL_COOKIE(obj)); - assert(cookie->pool == pool); - } -#endif - if (TheMeter.idle.level + pool->obj_size <= mem_idle_limit) { - memMeterInc(pool->meter.idle); - memMeterAdd(TheMeter.idle, pool->obj_size); - if (Config.onoff.zero_buffers || pool->flags.dozero) - memset(obj, 0, pool->obj_size); -#if DEBUG_MEMPOOL - (void) VALGRIND_MAKE_NOACCESS(obj, pool->real_obj_size + sizeof(struct mempool_cookie)); -#else - (void) VALGRIND_MAKE_NOACCESS(obj, pool->obj_size); -#endif - stackPush(&pool->pstack, obj); - } else { - memMeterDec(pool->meter.alloc); - memMeterDel(TheMeter.alloc, pool->obj_size); - xfree(obj); - } - assert(pool->meter.idle.level <= pool->meter.alloc.level); -} - -static void -memPoolShrink(MemPool * pool, size_t new_limit) -{ - assert(pool); - while (pool->meter.idle.level > new_limit && pool->pstack.count > 0) { - memMeterDec(pool->meter.alloc); - memMeterDec(pool->meter.idle); - memMeterDel(TheMeter.idle, pool->obj_size); - memMeterDel(TheMeter.alloc, pool->obj_size); - xfree(stackPop(&pool->pstack)); - } - assert(pool->meter.idle.level <= new_limit); /* paranoid */ -} - -int -memPoolWasUsed(const MemPool * pool) -{ - assert(pool); - return pool->meter.alloc.hwater_level > 0; -} - -int -memPoolInUseCount(const MemPool * pool) -{ - assert(pool); - return pool->meter.inuse.level; -} - -size_t -memPoolInUseSize(const MemPool * pool) -{ - assert(pool); - return (pool->obj_size * pool->meter.inuse.level); -} - -/* to-do: make debug level a parameter? */ -static void -memPoolDescribe(const MemPool * pool) -{ - assert(pool); - debug(63, 2) ("%-20s: %6d x %4d bytes = %5ld KB\n", - pool->label, memPoolInUseCount(pool), (int) pool->obj_size, - (long int) toKB(memPoolInUseSize(pool))); -} - -size_t -memTotalAllocated(void) -{ - return TheMeter.alloc.level; -} - #if DEBUG_MEMPOOL static void memPoolDiffReport(const MemPool * pool, StoreEntry * e) @@ -485,7 +177,7 @@ storeAppendPrintf(e, "Current overhead: %ld bytes (%.3f%%)\n", (long int) overhd_size, xpercent(overhd_size, TheMeter.inuse.level)); /* limits */ - storeAppendPrintf(e, "Idle pool limit: %.2f MB\n", toMB(mem_idle_limit)); - storeAppendPrintf(e, "memPoolAlloc calls: %d\n", mem_pool_alloc_calls); - storeAppendPrintf(e, "memPoolFree calls: %d\n", mem_pool_free_calls); + storeAppendPrintf(e, "Idle pool limit: %.2f MB\n", toMB(MemPoolCfg.idle_limit)); + storeAppendPrintf(e, "memPoolAlloc calls: %d\n", MemPoolStats.mem_pool_alloc_calls); + storeAppendPrintf(e, "memPoolFree calls: %d\n", MemPoolStats.mem_pool_free_calls); } Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.102.2.1 retrieving revision 1.102.2.1.4.1 diff -u -r1.102.2.1 -r1.102.2.1.4.1 --- squid/src/cache_cf.c 27 Nov 2007 08:12:24 -0000 1.102.2.1 +++ squid/src/cache_cf.c 6 Dec 2007 06:34:39 -0000 1.102.2.1.4.1 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.102.2.1 2007/11/27 08:12:24 adri Exp $ + * $Id: cache_cf.c,v 1.102.2.1.4.1 2007/12/06 06:34:39 adri Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -394,7 +394,7 @@ { memset(&Config2, '\0', sizeof(SquidConfig2)); /* init memory as early as possible */ - memConfigure(); + memConfigure(Config.onoff.mem_pools, Config.MemPools.limit, Config.onoff.zero_buffers); /* Sanity checks */ if (Config.cacheSwap.swapDirs == NULL) fatal("No cache_dir's specified in config file"); Index: squid/src/defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/defines.h,v retrieving revision 1.43.8.1 retrieving revision 1.43.8.2 diff -u -r1.43.8.1 -r1.43.8.2 --- squid/src/defines.h 2 Dec 2007 02:01:34 -0000 1.43.8.1 +++ squid/src/defines.h 6 Dec 2007 06:34:40 -0000 1.43.8.2 @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.43.8.1 2007/12/02 02:01:34 adri Exp $ + * $Id: defines.h,v 1.43.8.2 2007/12/06 06:34:40 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -256,10 +256,6 @@ #define DEFAULT_SQUID_ERROR_DIR "/usr/local/squid/etc/errors" #endif -/* gb_type operations */ -#define gb_flush_limit (0x3FFFFFFF) -#define gb_inc(gb, delta) { if ((gb)->bytes > gb_flush_limit || delta > gb_flush_limit) gb_flush(gb); (gb)->bytes += delta; (gb)->count++; } - /* iteration for HttpHdrRange */ #define HttpHdrRangeInitPos (-1) Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.146.2.4.4.3 retrieving revision 1.146.2.4.4.4 diff -u -r1.146.2.4.4.3 -r1.146.2.4.4.4 --- squid/src/protos.h 3 Dec 2007 07:59:25 -0000 1.146.2.4.4.3 +++ squid/src/protos.h 6 Dec 2007 06:34:40 -0000 1.146.2.4.4.4 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.146.2.4.4.3 2007/12/03 07:59:25 adri Exp $ + * $Id: protos.h,v 1.146.2.4.4.4 2007/12/06 06:34:40 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -855,7 +855,6 @@ extern void memClean(void); extern void memInitModule(void); extern void memCleanModule(void); -extern void memConfigure(void); extern void *memAllocate(mem_type); extern void *memAllocString(size_t net_size, size_t * gross_size); extern void *memAllocBuf(size_t net_size, size_t * gross_size); @@ -1195,9 +1194,6 @@ extern dlink_node *dlinkNodeNew(void); extern void kb_incr(kb_t *, squid_off_t); -extern double gb_to_double(const gb_t *); -extern const char *gb_to_str(const gb_t *); -extern void gb_flush(gb_t *); /* internal, do not use this */ extern int stringHasWhitespace(const char *); extern int stringHasCntl(const char *); extern void linklistPush(link_list **, void *); Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.36.24.4 retrieving revision 1.36.24.5 diff -u -r1.36.24.4 -r1.36.24.5 --- squid/src/squid.h 5 Dec 2007 05:35:30 -0000 1.36.24.4 +++ squid/src/squid.h 6 Dec 2007 06:34:45 -0000 1.36.24.5 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.36.24.4 2007/12/05 05:35:30 adri Exp $ + * $Id: squid.h,v 1.36.24.5 2007/12/06 06:34:45 adri Exp $ * * AUTHOR: Duane Wessels * @@ -374,6 +374,7 @@ #include "../libcore/debug.h" #include "../libcore/ctx.h" #include "../libcore/syslog.h" +#include "../libcore/valgrind.h" /* libbuf */ @@ -382,6 +383,11 @@ #include "Stack.h" +/* libmem */ +#include "../libmem/MemPool.h" +#include "../libmem/mem.h" + + /* Needed for poll() on Linux at least */ #if USE_POLL #ifndef POLLRDNORM @@ -508,20 +514,4 @@ #error Your platform does not support large integers. Can not build with --enable-large-cache-files #endif -/* - * valgrind debug support - */ -#if WITH_VALGRIND -#include -#else -#define VALGRIND_MAKE_NOACCESS(a,b) (0) -#define VALGRIND_MAKE_WRITABLE(a,b) (0) -#define VALGRIND_MAKE_READABLE(a,b) (0) -#define VALGRIND_CHECK_WRITABLE(a,b) (0) -#define VALGRIND_CHECK_READABLE(a,b) (0) -#define VALGRIND_MALLOCLIKE_BLOCK(a,b,c,d) -#define VALGRIND_FREELIKE_BLOCK(a,b) -#define RUNNING_ON_VALGRIND 0 -#endif /* WITH_VALGRIND */ - #endif /* SQUID_H */ Index: squid/src/stat.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stat.c,v retrieving revision 1.42.2.3 retrieving revision 1.42.2.3.4.1 diff -u -r1.42.2.3 -r1.42.2.3.4.1 --- squid/src/stat.c 27 Nov 2007 08:12:28 -0000 1.42.2.3 +++ squid/src/stat.c 6 Dec 2007 06:34:46 -0000 1.42.2.3.4.1 @@ -1,6 +1,6 @@ /* - * $Id: stat.c,v 1.42.2.3 2007/11/27 08:12:28 adri Exp $ + * $Id: stat.c,v 1.42.2.3.4.1 2007/12/06 06:34:46 adri Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -657,9 +657,9 @@ storeAppendPrintf(sentry, "\tTotal accounted: %6d KB\n", (int) (statMemoryAccounted() >> 10)); storeAppendPrintf(sentry, "\tmemPoolAlloc calls: %u\n", - mem_pool_alloc_calls); + MemPoolStats.mem_pool_alloc_calls); storeAppendPrintf(sentry, "\tmemPoolFree calls: %u\n", - mem_pool_free_calls); + MemPoolStats.mem_pool_free_calls); storeAppendPrintf(sentry, "File descriptor usage for %s:\n", appname); storeAppendPrintf(sentry, "\tMaximum number of file descriptors: %4d\n", Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.158.2.5.4.2 retrieving revision 1.158.2.5.4.3 diff -u -r1.158.2.5.4.2 -r1.158.2.5.4.3 --- squid/src/structs.h 3 Dec 2007 07:59:25 -0000 1.158.2.5.4.2 +++ squid/src/structs.h 6 Dec 2007 06:34:46 -0000 1.158.2.5.4.3 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.158.2.5.4.2 2007/12/03 07:59:25 adri Exp $ + * $Id: structs.h,v 1.158.2.5.4.3 2007/12/06 06:34:46 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -2184,40 +2184,6 @@ #endif - -/* object to track per-action memory usage (e.g. #idle objects) */ -struct _MemMeter { - ssize_t level; /* current level (count or volume) */ - ssize_t hwater_level; /* high water mark */ - time_t hwater_stamp; /* timestamp of last high water mark change */ -}; - -/* object to track per-pool memory usage (alloc = inuse+idle) */ -struct _MemPoolMeter { - MemMeter alloc; - MemMeter inuse; - MemMeter idle; - gb_t saved; - gb_t total; -}; - -/* a pool is a [growing] space for objects of the same size */ -struct _MemPool { - const char *label; - size_t obj_size; -#if DEBUG_MEMPOOL - size_t real_obj_size; /* with alignment */ -#endif - struct { - int dozero:1; - } flags; - Stack pstack; /* stack for free pointers */ - MemPoolMeter meter; -#if DEBUG_MEMPOOL - MemPoolMeter diff_meter; -#endif -}; - struct _ClientInfo { hash_link hash; /* must be first */ struct in_addr addr; Index: squid/src/tools.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/tools.c,v retrieving revision 1.62.2.3.4.2 retrieving revision 1.62.2.3.4.3 diff -u -r1.62.2.3.4.2 -r1.62.2.3.4.3 --- squid/src/tools.c 2 Dec 2007 06:46:42 -0000 1.62.2.3.4.2 +++ squid/src/tools.c 6 Dec 2007 06:34:46 -0000 1.62.2.3.4.3 @@ -1,6 +1,6 @@ /* - * $Id: tools.c,v 1.62.2.3.4.2 2007/12/02 06:46:42 adri Exp $ + * $Id: tools.c,v 1.62.2.3.4.3 2007/12/06 06:34:46 adri Exp $ * * DEBUG: section 21 Misc Functions * AUTHOR: Harvest Derived @@ -967,43 +967,6 @@ } void -gb_flush(gb_t * g) -{ - g->gb += (g->bytes >> 30); - g->bytes &= (1 << 30) - 1; -} - -double -gb_to_double(const gb_t * g) -{ - return ((double) g->gb) * ((double) (1 << 30)) + ((double) g->bytes); -} - -const char * -gb_to_str(const gb_t * g) -{ - /* - * it is often convenient to call gb_to_str several times for _one_ printf - */ -#define max_cc_calls 5 - typedef char GbBuf[32]; - static GbBuf bufs[max_cc_calls]; - static int call_id = 0; - double value = gb_to_double(g); - char *buf = bufs[call_id++]; - if (call_id >= max_cc_calls) - call_id = 0; - /* select format */ - if (value < 1e9) - snprintf(buf, sizeof(GbBuf), "%.2f MB", value / 1e6); - else if (value < 1e12) - snprintf(buf, sizeof(GbBuf), "%.2f GB", value / 1e9); - else - snprintf(buf, sizeof(GbBuf), "%.2f TB", value / 1e12); - return buf; -} - -void debugObj(int section, int level, const char *label, void *obj, ObjPackMethod pm) { MemBuf mb; Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.43.2.3.4.2 retrieving revision 1.43.2.3.4.3 diff -u -r1.43.2.3.4.2 -r1.43.2.3.4.3 --- squid/src/typedefs.h 3 Dec 2007 07:59:25 -0000 1.43.2.3.4.2 +++ squid/src/typedefs.h 6 Dec 2007 06:34:46 -0000 1.43.2.3.4.3 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.43.2.3.4.2 2007/12/03 07:59:25 adri Exp $ + * $Id: typedefs.h,v 1.43.2.3.4.3 2007/12/06 06:34:46 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -76,12 +76,6 @@ squid_off_t kb; } kb_t; -typedef struct { - size_t count; - size_t bytes; - size_t gb; -} gb_t; - /* * grep '^struct' structs.h \ * | perl -ne '($a,$b)=split;$c=$b;$c=~s/^_//; print "typedef struct $b $c;\n";' @@ -199,9 +193,6 @@ typedef struct _authConfig authConfig; typedef struct _cacheSwap cacheSwap; typedef struct _StatHist StatHist; -typedef struct _MemMeter MemMeter; -typedef struct _MemPoolMeter MemPoolMeter; -typedef struct _MemPool MemPool; typedef struct _ClientInfo ClientInfo; typedef struct _cd_guess_stats cd_guess_stats; typedef struct _CacheDigest CacheDigest;