--------------------- PatchSet 10199 Date: 2007/12/06 08:10:26 Author: adri Branch: s27_adri Tag: (none) Log: * Move the string-related stuff out of src/mem.c and into libmem/memstr.[ch] * Use the memstr stuff in libbuf/buf.c as the "backing" for strings. Ideally I'd like to use the same logic for allocating/freeing buffers as I would with strings, as strings are "just" a smaller version of a buffer, but this isn't important at the present. The buf_t stuff is destined to be used for client send/receive buffers as well as store buffers - str_t / String will then reference memory reigons inside -that-. The majority of this "best fit" buffer logic will then go away for almost all of the current execution paths in Squid - it'll only hang about if strings are modified copy-on-write or new strings are generated. Members: libbuf/buf.c:1.1.2.2->1.1.2.3 libmem/Makefile.am:1.1.2.1->1.1.2.2 libmem/memstr.c:1.1->1.1.2.1 libmem/memstr.h:1.1->1.1.2.1 src/main.c:1.80.2.1.4.2->1.80.2.1.4.3 src/mem.c:1.31->1.31.8.1 src/protos.h:1.146.2.4.4.4->1.146.2.4.4.5 src/squid.h:1.36.24.5->1.36.24.6 Index: squid/libbuf/buf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/libbuf/Attic/buf.c,v retrieving revision 1.1.2.2 retrieving revision 1.1.2.3 diff -u -r1.1.2.2 -r1.1.2.3 --- squid/libbuf/buf.c 6 Dec 2007 07:37:30 -0000 1.1.2.2 +++ squid/libbuf/buf.c 6 Dec 2007 08:10:26 -0000 1.1.2.3 @@ -10,6 +10,7 @@ #include "../libcore/debug.h" #include "../libcore/tools.h" #include "../libmem/MemPool.h" +#include "../libmem/memstr.h" #include "buf.h" static int buf_configured = 0; @@ -30,6 +31,8 @@ buf_changesize(buf_t *b, int newsize) { char *p; + size_t ns; + /* * buffer shouldn't ever be smaller than 'len', but we'll try * to handle it.. @@ -41,12 +44,21 @@ if (b->flags.isfinal != 0) return 0; - p = realloc(b->b, newsize); - if (p == NULL) - return 0; + /* if its empty then allocate it */ + if (b->b == NULL) { + b->b = memAllocString(newsize, &ns); + if (! b->b) + return 0; + } else if (b->size < newsize) { + p = memAllocString(newsize, &ns); + if (! p) + return 0; + memcpy(p, b->b, b->size); + memFreeString(b->size, b->b); + b->b = p; + } - b->b = p; - b->size = newsize; + b->size = ns; /* truncate buffer length if/where possible */ if (b->len > b->size) { b->len = b->size; @@ -130,7 +142,7 @@ if (b->nref == 0) { debug (85, 5) ("buf_deref: %p: FREEing\n", b); if (!b->flags.isconst) { - free(b->b); + memFreeString(b->size, b->b); } memPoolFree(buf_pool, b); return NULL; Index: squid/libmem/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/libmem/Attic/Makefile.am,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/libmem/Makefile.am 6 Dec 2007 06:34:38 -0000 1.1.2.1 +++ squid/libmem/Makefile.am 6 Dec 2007 08:10:26 -0000 1.1.2.2 @@ -1,13 +1,15 @@ ## Process this file with automake to produce Makefile.in # -# $Id: Makefile.am,v 1.1.2.1 2007/12/06 06:34:38 adri Exp $ +# $Id: Makefile.am,v 1.1.2.2 2007/12/06 08:10:26 adri Exp $ # libmem_a_SOURCES = \ - MemPool.c + MemPool.c \ + memstr.c libmem_a_LIBADD = \ - MemPool.o + MemPool.o \ + memstr.o noinst_LIBRARIES = \ libmem.a --- /dev/null Fri Dec 7 01:18:29 2007 +++ squid/libmem/memstr.c Fri Dec 7 01:18:29 2007 @@ -0,0 +1,128 @@ + +/* + * $Id: memstr.c,v 1.1.2.1 2007/12/06 08:10:26 adri Exp $ + * + * DEBUG: section 13 High Level Memory Pool Management + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include +#include +#include +#include +#include + +#include "../include/config.h" +#include "../include/util.h" +#include "../include/Stack.h" +#include "../libcore/valgrind.h" +#include "../libcore/tools.h" +#include "../libcore/varargs.h" +#include "../libcore/debug.h" + +#include "MemPool.h" +#include "memstr.h" + +/* module globals */ + +/* string pools */ +#define MEM_STR_POOL_COUNT 3 +static const struct { + const char *name; + size_t obj_size; +} StrPoolsAttrs[MEM_STR_POOL_COUNT] = { + + { + "Short Strings", 36, + }, /* to fit rfc1123 and similar */ + { + "Medium Strings", 128, + }, /* to fit most urls */ + { + "Long Strings", 512 + } /* other */ +}; + +StrPoolsStruct StrPools[MEM_STR_POOL_COUNT]; +MemMeter StrCountMeter; +MemMeter StrVolumeMeter; + +/* local routines */ + +/* allocate a variable size buffer using best-fit pool */ +void * +memAllocString(size_t net_size, size_t * gross_size) +{ + int i; + MemPool *pool = NULL; + assert(gross_size); + for (i = 0; i < MEM_STR_POOL_COUNT; i++) { + if (net_size <= StrPoolsAttrs[i].obj_size) { + pool = StrPools[i].pool; + break; + } + } + *gross_size = pool ? pool->obj_size : net_size; + assert(*gross_size >= net_size); + memMeterInc(StrCountMeter); + memMeterAdd(StrVolumeMeter, *gross_size); + return pool ? memPoolAlloc(pool) : xmalloc(net_size); +} + +/* free buffer allocated with memAllocString() */ +void +memFreeString(size_t size, void *buf) +{ + int i; + MemPool *pool = NULL; + assert(size && buf); + for (i = 0; i < MEM_STR_POOL_COUNT; i++) { + if (size <= StrPoolsAttrs[i].obj_size) { + assert(size == StrPoolsAttrs[i].obj_size); + pool = StrPools[i].pool; + break; + } + } + memMeterDec(StrCountMeter); + memMeterDel(StrVolumeMeter, size); + pool ? memPoolFree(pool, buf) : xfree(buf); +} + +void +memStringInit(void) +{ + int i; + /* init string pools */ + for (i = 0; i < MEM_STR_POOL_COUNT; i++) { + StrPools[i].pool = memPoolCreate(StrPoolsAttrs[i].name, StrPoolsAttrs[i].obj_size); + memPoolNonZero(StrPools[i].pool); + } +} + --- /dev/null Fri Dec 7 01:18:29 2007 +++ squid/libmem/memstr.h Fri Dec 7 01:18:29 2007 @@ -0,0 +1,21 @@ +#ifndef __LIBMEM_MEMSTR_H__ +#define __LIBMEM_MEMSTR_H__ + +#define MEM_STR_POOL_COUNT 3 + +typedef struct { + MemPool *pool; +} StrPoolsStruct; + +extern MemMeter StrCountMeter; +extern MemMeter StrVolumeMeter; +extern StrPoolsStruct StrPools[]; + +extern void * memAllocString(size_t net_size, size_t * gross_size); +extern void memFreeString(size_t size, void *buf); +extern void memStringInit(void); + + + + +#endif Index: squid/src/main.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/main.c,v retrieving revision 1.80.2.1.4.2 retrieving revision 1.80.2.1.4.3 diff -u -r1.80.2.1.4.2 -r1.80.2.1.4.3 --- squid/src/main.c 6 Dec 2007 07:37:30 -0000 1.80.2.1.4.2 +++ squid/src/main.c 6 Dec 2007 08:10:26 -0000 1.80.2.1.4.3 @@ -1,6 +1,6 @@ /* - * $Id: main.c,v 1.80.2.1.4.2 2007/12/06 07:37:30 adri Exp $ + * $Id: main.c,v 1.80.2.1.4.3 2007/12/06 08:10:26 adri Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -720,6 +720,7 @@ /* Initialise the buffer code early on so things like Strings can be used */ memInit(); memConfigure(1, 0, 0); + memStringInit(); buf_init(); getCurrentTime(); Index: squid/src/mem.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/mem.c,v retrieving revision 1.31 retrieving revision 1.31.8.1 diff -u -r1.31 -r1.31.8.1 --- squid/src/mem.c 24 Sep 2007 13:52:09 -0000 1.31 +++ squid/src/mem.c 6 Dec 2007 08:10:26 -0000 1.31.8.1 @@ -1,6 +1,6 @@ /* - * $Id: mem.c,v 1.31 2007/09/24 13:52:09 squidadm Exp $ + * $Id: mem.c,v 1.31.8.1 2007/12/06 08:10:26 adri Exp $ * * DEBUG: section 13 High Level Memory Pool Management * AUTHOR: Harvest Derived @@ -40,27 +40,6 @@ static MemPool *MemPools[MEM_MAX]; /* string pools */ -#define mem_str_pool_count 3 -static const struct { - const char *name; - size_t obj_size; -} StrPoolsAttrs[mem_str_pool_count] = { - - { - "Short Strings", 36, - }, /* to fit rfc1123 and similar */ - { - "Medium Strings", 128, - }, /* to fit most urls */ - { - "Long Strings", 512 - } /* other */ -}; -static struct { - MemPool *pool; -} StrPools[mem_str_pool_count]; -static MemMeter StrCountMeter; -static MemMeter StrVolumeMeter; static MemMeter HugeBufCountMeter; static MemMeter HugeBufVolumeMeter; @@ -79,7 +58,7 @@ "String Pool\t Impact\t\t\n" " \t (%%strings)\t (%%volume)\n"); /* table body */ - for (i = 0; i < mem_str_pool_count; i++) { + for (i = 0; i < MEM_STR_POOL_COUNT; i++) { const MemPool *pool = StrPools[i].pool; const int plevel = pool->meter.inuse.level; storeAppendPrintf(sentry, pfmt, @@ -167,45 +146,6 @@ memPoolFree(MemPools[type], p); } -/* allocate a variable size buffer using best-fit pool */ -void * -memAllocString(size_t net_size, size_t * gross_size) -{ - int i; - MemPool *pool = NULL; - assert(gross_size); - for (i = 0; i < mem_str_pool_count; i++) { - if (net_size <= StrPoolsAttrs[i].obj_size) { - pool = StrPools[i].pool; - break; - } - } - *gross_size = pool ? pool->obj_size : net_size; - assert(*gross_size >= net_size); - memMeterInc(StrCountMeter); - memMeterAdd(StrVolumeMeter, *gross_size); - return pool ? memPoolAlloc(pool) : xcalloc(1, net_size); -} - -/* free buffer allocated with memAllocString() */ -void -memFreeString(size_t size, void *buf) -{ - int i; - MemPool *pool = NULL; - assert(size && buf); - for (i = 0; i < mem_str_pool_count; i++) { - if (size <= StrPoolsAttrs[i].obj_size) { - assert(size == StrPoolsAttrs[i].obj_size); - pool = StrPools[i].pool; - break; - } - } - memMeterDec(StrCountMeter); - memMeterDel(StrVolumeMeter, size); - pool ? memPoolFree(pool, buf) : xfree(buf); -} - /* Find the best fit MEM_X_BUF type */ static mem_type memFindBufSizeType(size_t net_size, size_t * gross_size) @@ -289,7 +229,6 @@ void memInit(void) { - int i; memInitModule(); /* set all pointers to null */ memset(MemPools, '\0', sizeof(MemPools)); @@ -367,11 +306,6 @@ memDataInit(MEM_TLV, "storeSwapTLV", sizeof(tlv), 0); memDataInit(MEM_SWAP_LOG_DATA, "storeSwapLogData", sizeof(storeSwapLogData), 0); - /* init string pools */ - for (i = 0; i < mem_str_pool_count; i++) { - StrPools[i].pool = memPoolCreate(StrPoolsAttrs[i].name, StrPoolsAttrs[i].obj_size); - memPoolNonZero(StrPools[i].pool); - } cachemgrRegister("mem", "Memory Utilization", memStats, 0, 1); Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.146.2.4.4.4 retrieving revision 1.146.2.4.4.5 diff -u -r1.146.2.4.4.4 -r1.146.2.4.4.5 --- squid/src/protos.h 6 Dec 2007 06:34:40 -0000 1.146.2.4.4.4 +++ squid/src/protos.h 6 Dec 2007 08:10:26 -0000 1.146.2.4.4.5 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.146.2.4.4.4 2007/12/06 06:34:40 adri Exp $ + * $Id: protos.h,v 1.146.2.4.4.5 2007/12/06 08:10:26 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -842,27 +842,17 @@ extern StatHistBinDumper statHistIntDumper; -/* MemMeter */ -extern void memMeterSyncHWater(MemMeter * m); -#define memMeterCheckHWater(m) { if ((m).hwater_level < (m).level) memMeterSyncHWater(&(m)); } -#define memMeterInc(m) { (m).level++; memMeterCheckHWater(m); } -#define memMeterDec(m) { (m).level--; } -#define memMeterAdd(m, sz) { (m).level += (sz); memMeterCheckHWater(m); } -#define memMeterDel(m, sz) { (m).level -= (sz); } - /* mem */ extern void memInit(void); extern void memClean(void); extern void memInitModule(void); extern void memCleanModule(void); extern void *memAllocate(mem_type); -extern void *memAllocString(size_t net_size, size_t * gross_size); extern void *memAllocBuf(size_t net_size, size_t * gross_size); extern void *memReallocBuf(void *buf, size_t net_size, size_t * gross_size); extern void memFree(void *, int type); extern void memFree4K(void *); extern void memFree8K(void *); -extern void memFreeString(size_t size, void *); extern void memFreeBuf(size_t size, void *); extern FREE *memFreeBufFunc(size_t size); extern int memInUse(mem_type); @@ -870,17 +860,6 @@ extern void memDataInit(mem_type, const char *, size_t, int); extern void memCheckInit(void); -/* MemPool */ -extern MemPool *memPoolCreate(const char *label, size_t obj_size); -extern void memPoolDestroy(MemPool * pool); -extern void memPoolNonZero(MemPool * p); -extern void *memPoolAlloc(MemPool * pool); -extern void memPoolFree(MemPool * pool, void *obj); -extern int memPoolWasUsed(const MemPool * pool); -extern int memPoolInUseCount(const MemPool * pool); -extern size_t memPoolInUseSize(const MemPool * pool); -extern int memPoolUsedCount(const MemPool * pool); - /* Mem */ extern void memReport(StoreEntry * e); Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.36.24.5 retrieving revision 1.36.24.6 diff -u -r1.36.24.5 -r1.36.24.6 --- squid/src/squid.h 6 Dec 2007 06:34:45 -0000 1.36.24.5 +++ squid/src/squid.h 6 Dec 2007 08:10:27 -0000 1.36.24.6 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.36.24.5 2007/12/06 06:34:45 adri Exp $ + * $Id: squid.h,v 1.36.24.6 2007/12/06 08:10:27 adri Exp $ * * AUTHOR: Duane Wessels * @@ -385,6 +385,7 @@ /* libmem */ #include "../libmem/MemPool.h" +#include "../libmem/memstr.h" #include "../libmem/mem.h"