This patch is generated from the etag-2_5 branch of s2_5 in squid Sun May 21 00:14:51 2006 GMT See http://devel.squid-cache.org/ Index: squid/src/ETag.c diff -u squid/src/ETag.c:1.4 squid/src/ETag.c:removed --- squid/src/ETag.c:1.4 Fri Jan 12 00:20:32 2001 +++ squid/src/ETag.c Sat May 20 17:14:57 2006 @@ -1,68 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: none ETag parsing support - * AUTHOR: Alex Rousskov - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -/* - * Note: ETag is not an http "field" like, for example HttpHdrRange. ETag is a - * field-value that maybe used in many http fields. - */ - -/* parses a string as weak or strong entity-tag; returns true on success */ -/* note: we do not duplicate "str"! */ -int -etagParseInit(ETag * etag, const char *str) -{ - int len; - assert(etag && str); - etag->str = NULL; - etag->weak = !strncmp(str, "W/", 2); - if (etag->weak) - str += 2; - /* check format (quoted-string) */ - len = strlen(str); - if (len >= 2 && str[0] == '"' && str[len - 1] == '"') - etag->str = str; - return etag->str != NULL; -} - -/* returns true if etags are equal */ -int -etagIsEqual(const ETag * tag1, const ETag * tag2) -{ - assert(tag1 && tag2); - assert(!tag1->weak && !tag2->weak); /* weak comparison not implemented yet */ - return !strcmp(tag1->str, tag2->str); -} Index: squid/src/HttpHeader.c diff -u squid/src/HttpHeader.c:1.10.6.26 squid/src/HttpHeader.c:1.6.18.2.2.7 --- squid/src/HttpHeader.c:1.10.6.26 Fri May 12 09:51:33 2006 +++ squid/src/HttpHeader.c Sat May 20 12:15:10 2006 @@ -87,7 +87,7 @@ {"Content-Type", HDR_CONTENT_TYPE, ftStr}, {"Cookie", HDR_COOKIE, ftStr}, {"Date", HDR_DATE, ftDate_1123}, - {"ETag", HDR_ETAG, ftETag}, + {"ETag", HDR_ETAG, ftStr}, {"Expires", HDR_EXPIRES, ftDate_1123}, {"From", HDR_FROM, ftStr}, {"Host", HDR_HOST, ftStr}, @@ -1101,18 +1101,6 @@ return base64_decode(field); } -ETag -httpHeaderGetETag(const HttpHeader * hdr, http_hdr_type id) -{ - ETag etag = - {NULL, -1}; - HttpHeaderEntry *e; - assert(Headers[id].type == ftETag); /* must be of an appropriate type */ - if ((e = httpHeaderFindEntry(hdr, id))) - etagParseInit(&etag, strBuf(e->value)); - return etag; -} - TimeOrTag httpHeaderGetTimeOrTag(const HttpHeader * hdr, http_hdr_type id) { @@ -1123,17 +1111,20 @@ if ((e = httpHeaderFindEntry(hdr, id))) { const char *str = strBuf(e->value); /* try as an ETag */ - if (etagParseInit(&tot.tag, str)) { - tot.valid = tot.tag.str != NULL; + if (*str == '"' || (str[0] == 'W' && str[1] == '/')) { + tot.tag = str; tot.time = -1; + tot.valid = 1; } else { /* or maybe it is time? */ tot.time = parse_rfc1123(str); - tot.valid = tot.time >= 0; - tot.tag.str = NULL; + if (tot.time >= 0) + tot.valid = 1; + tot.tag = NULL; } + } else { + tot.time = -1; } - assert(tot.time < 0 || !tot.tag.str); /* paranoid */ return tot; } Index: squid/src/HttpReply.c diff -u squid/src/HttpReply.c:1.10.6.6 squid/src/HttpReply.c:1.7.14.3.2.3 --- squid/src/HttpReply.c:1.10.6.6 Sat Feb 25 19:13:56 2006 +++ squid/src/HttpReply.c Sat May 20 12:15:10 2006 @@ -212,7 +212,7 @@ httpPacked304Reply(const HttpReply * rep) { static const http_hdr_type ImsEntries[] = - {HDR_DATE, HDR_CONTENT_TYPE, HDR_EXPIRES, HDR_LAST_MODIFIED, /* eof */ HDR_OTHER}; + {HDR_DATE, HDR_CONTENT_TYPE, HDR_EXPIRES, HDR_LAST_MODIFIED, HDR_ETAG, /* eof */ HDR_OTHER}; int t; MemBuf mb; Packer p; Index: squid/src/HttpRequest.c diff -u squid/src/HttpRequest.c:1.7.36.4 squid/src/HttpRequest.c:1.7.14.2.2.2 --- squid/src/HttpRequest.c:1.7.36.4 Fri Mar 10 19:16:30 2006 +++ squid/src/HttpRequest.c Sat May 20 12:15:11 2006 @@ -67,6 +67,13 @@ httpHdrCcDestroy(req->cache_control); if (req->range) httpHdrRangeDestroy(req->range); + if (req->vary) { + if (req->etags == &req->vary->etags) + req->etags = NULL; + storeLocateVaryDone(req->vary); + } + assert(req->etags == NULL); + safe_free(req->etag); memFree(req, MEM_REQUEST_T); } Index: squid/src/Makefile.am diff -u squid/src/Makefile.am:1.13.2.11 squid/src/Makefile.am:1.1.10.5.2.8 --- squid/src/Makefile.am:1.13.2.11 Wed Sep 28 19:13:47 2005 +++ squid/src/Makefile.am Sat May 20 12:15:11 2006 @@ -128,7 +128,6 @@ $(DNSSOURCE) \ enums.h \ errorpage.c \ - ETag.c \ event.c \ external_acl.c \ fd.c \ Index: squid/src/MemBuf.c diff -u squid/src/MemBuf.c:1.5.30.3 squid/src/MemBuf.c:1.5.8.1.2.5 --- squid/src/MemBuf.c:1.5.30.3 Fri Mar 25 19:15:54 2005 +++ squid/src/MemBuf.c Sat May 20 12:20:54 2006 @@ -139,7 +139,7 @@ mb->size = 0; mb->max_capacity = szMax; mb->capacity = 0; - mb->freefunc = NULL; + mb->stolen = 0; memBufGrow(mb, szInit); } @@ -153,10 +153,9 @@ { assert(mb); assert(mb->buf); - assert(mb->freefunc); /* not frozen */ + assert(!mb->stolen); /* not frozen */ - (*mb->freefunc) (mb->buf); /* free */ - mb->freefunc = NULL; /* freeze */ + memFreeBuf(mb->capacity, mb->buf); mb->buf = NULL; mb->size = mb->capacity = mb->max_capacity = 0; } @@ -171,7 +170,7 @@ if (memBufIsNull(mb)) { memBufDefInit(mb); } else { - assert(mb->freefunc); /* not frozen */ + assert(!mb->stolen); /* not frozen */ /* reset */ memset(mb->buf, 0, mb->capacity); mb->size = 0; @@ -196,7 +195,7 @@ { assert(mb && buf && sz >= 0); assert(mb->buf); - assert(mb->freefunc); /* not frozen */ + assert(!mb->stolen); /* not frozen */ if (sz > 0) { if (mb->size + sz > mb->capacity) @@ -242,7 +241,7 @@ int sz = 0; assert(mb && fmt); assert(mb->buf); - assert(mb->freefunc); /* not frozen */ + assert(!mb->stolen); /* not frozen */ /* assert in Grow should quit first, but we do not want to have a scary infinite loop */ while (mb->capacity <= mb->max_capacity) { mb_size_t free_space = mb->capacity - mb->size; @@ -290,10 +289,10 @@ FREE *ff; assert(mb); assert(mb->buf); - assert(mb->freefunc); /* not frozen */ + assert(!mb->stolen); /* not frozen */ - ff = mb->freefunc; - mb->freefunc = NULL; /* freeze */ + ff = memFreeBufFunc((size_t) mb->capacity); + mb->stolen = 1; /* freeze */ return ff; } @@ -301,79 +300,34 @@ static void memBufGrow(MemBuf * mb, mb_size_t min_cap) { - mb_size_t new_cap; - MemBuf old_mb; + size_t new_cap; + size_t buf_cap; assert(mb); + assert(!mb->stolen); assert(mb->capacity < min_cap); /* determine next capacity */ - new_cap = mb->capacity; - if (new_cap > 0) - while (new_cap < min_cap) - new_cap *= 2; /* double */ - else - new_cap = min_cap; + if (min_cap > 64 * 1024) { + new_cap = 64 * 1024; + while (new_cap < (size_t) min_cap) + new_cap += 64 * 1024; /* increase in reasonable steps */ + } else { + new_cap = (size_t) min_cap; + } /* last chance to fit before we assert(!overflow) */ - if (new_cap > mb->max_capacity) - new_cap = mb->max_capacity; + if (new_cap > (size_t) mb->max_capacity) + new_cap = (size_t) mb->max_capacity; - assert(new_cap <= mb->max_capacity); /* no overflow */ - assert(new_cap > mb->capacity); /* progress */ + assert(new_cap <= (size_t) mb->max_capacity); /* no overflow */ + assert(new_cap > (size_t) mb->capacity); /* progress */ - old_mb = *mb; - - /* allocate new memory */ - switch (new_cap) { - case 2048: - mb->buf = memAllocate(MEM_2K_BUF); - mb->freefunc = &memFree2K; - break; - case 4096: - mb->buf = memAllocate(MEM_4K_BUF); - mb->freefunc = &memFree4K; - break; - case 8192: - mb->buf = memAllocate(MEM_8K_BUF); - mb->freefunc = &memFree8K; - break; - case 16384: - mb->buf = memAllocate(MEM_16K_BUF); - mb->freefunc = &memFree16K; - break; - case 32768: - mb->buf = memAllocate(MEM_32K_BUF); - mb->freefunc = &memFree32K; - break; - case 65536: - mb->buf = memAllocate(MEM_64K_BUF); - mb->freefunc = &memFree64K; - break; - default: - /* recycle if old buffer was not "pool"ed */ - if (old_mb.freefunc == &xfree) { - mb->buf = xrealloc(old_mb.buf, new_cap); - old_mb.buf = NULL; - old_mb.freefunc = NULL; - /* init tail, just in case */ - memset(mb->buf + mb->size, 0, new_cap - mb->size); - } else { - mb->buf = xcalloc(1, new_cap); - mb->freefunc = &xfree; - } - } - - /* copy and free old buffer if needed */ - if (old_mb.buf && old_mb.freefunc) { - xmemcpy(mb->buf, old_mb.buf, old_mb.size); - (*old_mb.freefunc) (old_mb.buf); - } else { - assert(!old_mb.buf && !old_mb.freefunc); - } + buf_cap = (size_t) mb->capacity; + mb->buf = memReallocBuf(mb->buf, new_cap, &buf_cap); /* done */ - mb->capacity = new_cap; + mb->capacity = (mb_size_t) buf_cap; } Index: squid/src/String.c diff -u squid/src/String.c:1.4 squid/src/String.c:1.4.46.1.2.2 --- squid/src/String.c:1.4 Fri Jan 12 00:20:32 2001 +++ squid/src/String.c Mon Apr 1 16:24:56 2002 @@ -38,7 +38,7 @@ static void stringInitBuf(String * s, size_t sz) { - s->buf = memAllocBuf(sz, &sz); + s->buf = memAllocString(sz, &sz); assert(sz < 65536); s->size = sz; } @@ -77,7 +77,7 @@ { assert(s); if (s->buf) - memFreeBuf(s->size, s->buf); + memFreeString(s->size, s->buf); *s = StringNull; } Index: squid/src/client_side.c diff -u squid/src/client_side.c:1.47.2.76 squid/src/client_side.c:1.29.2.22.2.18 --- squid/src/client_side.c:1.47.2.76 Fri Mar 10 19:16:31 2006 +++ squid/src/client_side.c Sat May 20 12:15:15 2006 @@ -150,6 +150,7 @@ static void clientEatRequestBody(clientHttpRequest *); static BODY_HANDLER clientReadBody; static void clientAbortBody(request_t * req); +static int varyEvaluateMatch(StoreEntry * entry, request_t * request); static int checkAccelOnly(clientHttpRequest * http) @@ -420,11 +421,139 @@ } static void +clientHandleETagMiss(clientHttpRequest * http) +{ + StoreEntry *entry = http->entry; + MemObject *mem = entry->mem_obj; + request_t *request = http->request; + + if (mem->reply) { + const char *etag = httpHeaderGetStr(&mem->reply->header, HDR_ETAG); + if (etag) { + /* This has to match storeSetPublicKey, except for the key which is NULL */ + String vary = StringNull; + String varyhdr; + varyhdr = httpHeaderGetList(&mem->reply->header, HDR_VARY); + if (strBuf(varyhdr)) + strListAdd(&vary, strBuf(varyhdr), ','); + stringClean(&varyhdr); +#if X_ACCELERATOR_VARY + /* This needs to match the order in http.c:httpMakeVaryMark */ + varyhdr = httpHeaderGetList(&mem->reply->header, HDR_X_ACCELERATOR_VARY); + if (strBuf(varyhdr)) + strListAdd(&vary, strBuf(varyhdr), ','); + stringClean(&varyhdr); +#endif + storeAddVary(mem->url, mem->log_url, mem->method, NULL, httpHeaderGetStr(&mem->reply->header, HDR_ETAG), strBuf(vary), httpMakeVaryMark(request, mem->reply)); + stringClean(&vary); + } + } + request->done_etag = 1; + if (request->vary) { + storeLocateVaryDone(request->vary); + request->vary = NULL; + request->etags = NULL; /* pointed into request->vary */ + } + safe_free(request->etag); + safe_free(request->vary_headers); + storeUnregister(http->sc, entry, http); + storeUnlockObject(entry); + clientProcessRequest(http); +} + +static void +clientHandleETagReply(void *data, char *buf, ssize_t size) +{ + clientHttpRequest *http = data; + StoreEntry *entry = http->entry; + MemObject *mem; + const char *url = storeUrl(entry); + http_status status; + debug(33, 3) ("clientHandleETagReply: %s, %d bytes\n", url, (int) size); + if (entry == NULL) { + /* client aborted */ + memFree(buf, MEM_CLIENT_SOCK_BUF); + return; + } + if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientHandleETagMiss(http); + return; + } + mem = entry->mem_obj; + status = mem->reply->sline.status; + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + debug(33, 3) ("clientHandleETagReply: ABORTED '%s'\n", url); + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientHandleETagMiss(http); + return; + } + if (STORE_PENDING == entry->store_status && 0 == status) { + debug(33, 3) ("clientHandleETagReply: Incomplete headers for '%s'\n", url); + if (size >= CLIENT_SOCK_SZ) { + /* will not get any bigger than that */ + debug(33, 3) ("clientHandleETagReply: Reply is too large '%s'\n", url); + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientHandleETagMiss(http); + } else { + storeClientCopy(http->sc, entry, + http->out.offset + size, + http->out.offset, + CLIENT_SOCK_SZ, + buf, + clientHandleETagReply, + http); + } + return; + } + if (HTTP_NOT_MODIFIED == mem->reply->sline.status) { + /* Remember the ETag and restart */ + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientHandleETagMiss(http); + return; + } + /* Send the new object to the client */ + clientSendMoreData(data, buf, size); + return; +} + +static void +clientProcessETag(clientHttpRequest * http) +{ + char *url = http->uri; + StoreEntry *entry = NULL; + debug(33, 3) ("clientProcessETag: '%s'\n", http->uri); + entry = storeCreateEntry(url, + http->log_uri, + http->request->flags, + http->request->method); + http->sc = storeClientListAdd(entry, http); +#if DELAY_POOLS + /* delay_id is already set on original store client */ + delaySetStoreClient(http->sc, delayClient(http)); +#endif + http->entry = entry; + http->out.offset = 0; + fwdStart(http->conn->fd, http->entry, http->request); + /* Register with storage manager to receive updates when data comes in. */ + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) + debug(33, 0) ("clientProcessETag: found ENTRY_ABORTED object\n"); + storeClientCopy(http->sc, entry, + http->out.offset, + http->out.offset, + CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), + clientHandleETagReply, + http); +} + +static void clientProcessExpired(void *data) { clientHttpRequest *http = data; char *url = http->uri; StoreEntry *entry = NULL; + const char *etag; debug(33, 3) ("clientProcessExpired: '%s'\n", http->uri); assert(http->entry->lastmod >= 0); /* @@ -459,6 +588,9 @@ debug(33, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod); http->entry = entry; http->out.offset = 0; + etag = httpHeaderGetStr(&http->old_entry->mem_obj->reply->header, HDR_ETAG); + if (etag) + http->request->etag = xstrdup(etag); fwdStart(http->conn->fd, http->entry, http->request); /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) @@ -492,6 +624,20 @@ debug(33, 5) ("clientGetsOldEntry: NO, reply=%d\n", status); return 0; } + /* If the ETag matches the clients If-None-Match, then return + * the servers 304 reply + */ + if (httpHeaderHas(&new_entry->mem_obj->reply->header, HDR_ETAG) && + httpHeaderHas(&request->header, HDR_IF_NONE_MATCH)) { + const char *etag = httpHeaderGetStr(&new_entry->mem_obj->reply->header, HDR_ETAG); + String etags = httpHeaderGetList(&request->header, HDR_IF_NONE_MATCH); + int etag_match = strListIsMember(&etags, etag, ','); + stringClean(&etags); + if (etag_match) { + debug(33, 5) ("clientGetsOldEntry: NO, client If-None-Match\n"); + return 0; + } + } /* If the client did not send IMS in the request, then it * must get the old object, not this "Not Modified" reply */ if (!request->flags.ims) { @@ -957,12 +1103,7 @@ authenticateAuthUserRequestUnlock(connState->auth_user_request); connState->auth_user_request = NULL; authenticateOnCloseConnection(connState); - if (connState->in.size == CLIENT_REQ_BUF_SZ) { - memFree(connState->in.buf, MEM_CLIENT_REQ_BUF); - connState->in.buf = NULL; - } else - safe_free(connState->in.buf); - /* XXX account connState->in.buf */ + memFreeBuf(connState->in.size, connState->in.buf); pconnHistCount(0, connState->nrequests); cbdataFree(connState); #ifdef _SQUID_LINUX_ @@ -1220,22 +1361,22 @@ if (!spec.valid) return 0; /* got an ETag? */ - if (spec.tag.str) { - ETag rep_tag = httpHeaderGetETag(&rep->header, HDR_ETAG); + if (spec.tag) { + const char *rep_tag = httpHeaderGetStr(&rep->header, HDR_ETAG); debug(33, 3) ("clientIfRangeMatch: ETags: %s and %s\n", - spec.tag.str, rep_tag.str ? rep_tag.str : ""); - if (!rep_tag.str) + spec.tag, rep_tag ? rep_tag : ""); + if (!rep_tag) return 0; /* entity has no etag to compare with! */ - if (spec.tag.weak || rep_tag.weak) { + if (spec.tag[0] == 'W' || rep_tag[0] == 'W') { debug(33, 1) ("clientIfRangeMatch: Weak ETags are not allowed in If-Range: %s ? %s\n", - spec.tag.str, rep_tag.str); + spec.tag, rep_tag); return 0; /* must use strong validator for sub-range requests */ } - return etagIsEqual(&rep_tag, &spec.tag); + return strcmp(rep_tag, spec.tag) == 0; } /* got modification time? */ - if (spec.time >= 0) { - return http->entry->lastmod <= spec.time; + else if (spec.time >= 0) { + return http->entry->lastmod == spec.time; } assert(0); /* should not happen */ return 0; @@ -1518,6 +1659,31 @@ } /* + * clientProcessVary is called when it is detected that a object + * varies and we need to get the correct variant + */ +static void +clientProcessVary(VaryData * vary, void *data) +{ + clientHttpRequest *http = data; + if (!vary) { + clientProcessRequest(http); + return; + } + if (vary->key) { + debug(33, 2) ("clientProcessVary: HIT key=%s etag=%s\n", vary->key, vary->etag); + } else { + int i; + debug(33, 2) ("clientProcessVary MISS\n"); + for (i = 0; i < vary->etags.count; i++) { + debug(33, 3) ("ETag: %s\n", (char *) vary->etags.items[i]); + } + } + http->request->vary = vary; + clientProcessRequest(http); +} + +/* * clientCacheHit should only be called until the HTTP reply headers * have been parsed. Normally this should be a single call, but * it might take more than one. As soon as we have the headers, @@ -1531,6 +1697,7 @@ StoreEntry *e = http->entry; MemObject *mem; request_t *r = http->request; + int is_modified = -1; debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); http->flags.hit = 0; if (http->entry == NULL) { @@ -1590,20 +1757,28 @@ debug(33, 2) ("clientProcessHit: Vary MATCH!\n"); break; case VARY_OTHER: - /* This is not the correct entity for this request. We need - * to requery the cache. - */ - memFree(buf, MEM_CLIENT_SOCK_BUF); - http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; - storeUnlockObject(e); - /* Note: varyEvalyateMatch updates the request with vary information - * so we only get here once. (it also takes care of cancelling loops) - */ - debug(33, 2) ("clientProcessHit: Vary detected!\n"); - clientProcessRequest(http); - return; + { + /* This is not the correct entity for this request. We need + * to requery the cache. + */ + store_client *sc = http->sc; + http->entry = NULL; /* saved in e */ + memFree(buf, MEM_CLIENT_SOCK_BUF); + /* Warning: storeUnregister may abort the object so we must + * call storeLocateVary before unregistering, and + * storeLocateVary may complete immediately so we cannot + * rely on the http structure for this... + */ + http->sc = NULL; + storeLocateVary(e, e->mem_obj->reply->hdr_sz, r->vary_headers, clientProcessVary, http); + storeUnregister(sc, e, http); + storeUnlockObject(e); + /* Note: varyEvalyateMatch updates the request with vary information + * so we only get here once. (it also takes care of cancelling loops) + */ + debug(33, 2) ("clientProcessHit: Vary detected!\n"); + return; + } case VARY_CANCEL: /* varyEvaluateMatch found a object loop. Process as miss */ debug(33, 1) ("clientProcessHit: Vary object loop!\n"); @@ -1628,7 +1803,35 @@ ) { http->log_type = LOG_TCP_NEGATIVE_HIT; clientSendMoreData(data, buf, size); - } else if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) { + return; + } + if (httpHeaderHas(&r->header, HDR_IF_MATCH)) { + String req_etags; + const char *rep_etag = httpHeaderGetStr(&e->mem_obj->reply->header, HDR_ETAG); + int has_etag; + if (!rep_etag) { + /* The cached object does not have a entity tag. This cannot + * be a hit for the requested object. + */ + http->log_type = LOG_TCP_MISS; + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientProcessMiss(http); + return; + } + req_etags = httpHeaderGetList(&http->request->header, HDR_IF_MATCH); + has_etag = strListIsMember(&req_etags, rep_etag, ','); + stringClean(&req_etags); + if (!has_etag) { + /* The entity tags does not match. This cannot be a + * hit for this object. Qyery the origin. + */ + http->log_type = LOG_TCP_MISS; + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientProcessMiss(http); + return; + } + } + if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) { debug(33, 5) ("clientCacheHit: in refreshCheck() block\n"); /* * We hold a stale copy; it needs to be validated @@ -1671,7 +1874,39 @@ clientProcessMiss(http); } memFree(buf, MEM_CLIENT_SOCK_BUF); - } else if (r->flags.ims) { + return; + } + if (httpHeaderHas(&r->header, HDR_IF_NONE_MATCH)) { + String req_etags; + const char *rep_etag = httpHeaderGetStr(&e->mem_obj->reply->header, HDR_ETAG); + int has_etag; + if (mem->reply->sline.status != HTTP_OK) { + debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", + mem->reply->sline.status); + memFree(buf, MEM_CLIENT_SOCK_BUF); + http->log_type = LOG_TCP_MISS; + clientProcessMiss(http); + return; + } + if (!rep_etag) { + /* The cached object does not have a entity tag, but the client + * obviously thinks there should be one... Query the origin to + * be on the safe side. + */ + http->log_type = LOG_TCP_MISS; + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientProcessMiss(http); + return; + } + req_etags = httpHeaderGetList(&http->request->header, HDR_IF_NONE_MATCH); + has_etag = strListIsMember(&req_etags, rep_etag, ','); + stringClean(&req_etags); + if (has_etag) { + http->log_type = LOG_TCP_IMS_HIT; + is_modified = 0; + } + } + if (is_modified != 0 && r->flags.ims) { /* * Handle If-Modified-Since requests from the client */ @@ -1681,41 +1916,45 @@ memFree(buf, MEM_CLIENT_SOCK_BUF); http->log_type = LOG_TCP_MISS; clientProcessMiss(http); + return; } else if (modifiedSince(e, http->request)) { http->log_type = LOG_TCP_IMS_HIT; clientSendMoreData(data, buf, size); - } else { - time_t timestamp = e->timestamp; - MemBuf mb = httpPacked304Reply(e->mem_obj->reply); - http->log_type = LOG_TCP_IMS_HIT; - memFree(buf, MEM_CLIENT_SOCK_BUF); - storeUnregister(http->sc, e, http); - http->sc = NULL; - storeUnlockObject(e); - e = clientCreateStoreEntry(http, http->request->method, null_request_flags); - /* - * Copy timestamp from the original entry so the 304 - * reply has a meaningful Age: header. - */ - e->timestamp = timestamp; - http->entry = e; - httpReplyParse(e->mem_obj->reply, mb.buf, mb.size); - storeAppend(e, mb.buf, mb.size); - memBufClean(&mb); - storeComplete(e); + return; } - } else { + is_modified = 0; + } + if (is_modified == 0) { + time_t timestamp = e->timestamp; + MemBuf mb = httpPacked304Reply(e->mem_obj->reply); + http->log_type = LOG_TCP_IMS_HIT; + memFree(buf, MEM_CLIENT_SOCK_BUF); + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + e = clientCreateStoreEntry(http, http->request->method, null_request_flags); /* - * plain ol' cache hit + * Copy timestamp from the original entry so the 304 + * reply has a meaningful Age: header. */ - if (e->store_status != STORE_OK) - http->log_type = LOG_TCP_MISS; - else if (e->mem_status == IN_MEMORY) - http->log_type = LOG_TCP_MEM_HIT; - else if (Config.onoff.offline) - http->log_type = LOG_TCP_OFFLINE_HIT; - clientSendMoreData(data, buf, size); + e->timestamp = timestamp; + http->entry = e; + httpReplyParse(e->mem_obj->reply, mb.buf, mb.size); + storeAppend(e, mb.buf, mb.size); + memBufClean(&mb); + storeComplete(e); + return; } + /* + * plain ol' cache hit + */ + if (e->store_status != STORE_OK) + http->log_type = LOG_TCP_MISS; + else if (e->mem_status == IN_MEMORY) + http->log_type = LOG_TCP_MEM_HIT; + else if (Config.onoff.offline) + http->log_type = LOG_TCP_OFFLINE_HIT; + clientSendMoreData(data, buf, size); } /* put terminating boundary for multiparts */ @@ -2397,6 +2636,16 @@ if (NULL == e) { /* this object isn't in the cache */ debug(33, 3) ("clientProcessRequest2: storeGet() MISS\n"); + if (r->vary) { + if (r->done_etag) { + debug(33, 1) ("clientProcessRequest2: ETag loop\n"); + } else if (r->etags) { + debug(33, 2) ("clientProcessRequest2: ETag miss\n"); + r->etags = NULL; + } else if (r->vary->etags.count > 0) { + r->etags = &r->vary->etags; + } + } return LOG_TCP_MISS; } if (Config.onoff.offline) { @@ -2559,9 +2808,9 @@ return; } assert(http->out.offset == 0); - http->entry = clientCreateStoreEntry(http, r->method, r->flags); if (http->redirect.status) { HttpReply *rep = httpReplyCreate(); + http->entry = clientCreateStoreEntry(http, r->method, r->flags); #if LOG_TCP_REDIRECTS http->log_type = LOG_TCP_REDIRECT; #endif @@ -2572,6 +2821,11 @@ storeComplete(http->entry); return; } + if (r->etags) { + clientProcessETag(http); + return; + } + http->entry = clientCreateStoreEntry(http, r->method, r->flags); if (http->flags.internal) r->protocol = PROTO_INTERNAL; fwdStart(http->conn->fd, http->entry, r); @@ -2976,7 +3230,6 @@ int parser_return_code = 0; request_t *request = NULL; int size; - void *p; method_t method; clientHttpRequest *http = NULL; clientHttpRequest **H = NULL; @@ -2988,15 +3241,7 @@ commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, conn, 0); if (len == 0) { /* Grow the request memory area to accomodate for a large request */ - conn->in.size += CLIENT_REQ_BUF_SZ; - if (conn->in.size == 2 * CLIENT_REQ_BUF_SZ) { - p = conn->in.buf; /* get rid of fixed size Pooled buffer */ - conn->in.buf = xcalloc(2, CLIENT_REQ_BUF_SZ); - xmemcpy(conn->in.buf, p, CLIENT_REQ_BUF_SZ); - memFree(p, MEM_CLIENT_REQ_BUF); - } else - conn->in.buf = xrealloc(conn->in.buf, conn->in.size); - /* XXX account conn->in.buf */ + conn->in.buf = memReallocBuf(conn->in.buf, conn->in.size * 2, &conn->in.size); debug(33, 2) ("growing request buffer: offset=%ld size=%ld\n", (long) conn->in.offset, (long) conn->in.size); len = conn->in.size - conn->in.offset - 1; @@ -3538,9 +3783,7 @@ connState->log_addr.s_addr &= Config.Addrs.client_netmask.s_addr; connState->me = me; connState->fd = fd; - connState->in.size = CLIENT_REQ_BUF_SZ; - connState->in.buf = memAllocate(MEM_CLIENT_REQ_BUF); - /* XXX account connState->in.buf */ + connState->in.buf = memAllocBuf(CLIENT_REQ_BUF_SZ, &connState->in.size); comm_add_close_handler(fd, connStateFree, connState); if (Config.onoff.log_fqdn) fqdncache_gethostbyaddr(peer.sin_addr, FQDN_LOOKUP_IF_MISS); @@ -3656,8 +3899,7 @@ connState->log_addr.s_addr &= Config.Addrs.client_netmask.s_addr; connState->me = me; connState->fd = fd; - connState->in.size = CLIENT_REQ_BUF_SZ; - connState->in.buf = memAllocate(MEM_CLIENT_REQ_BUF); + connState->in.buf = memAllocBuf(CLIENT_REQ_BUF_SZ, &connState->in.size); /* XXX account connState->in.buf */ comm_add_close_handler(fd, connStateFree, connState); if (Config.onoff.log_fqdn) @@ -3902,7 +4144,7 @@ NHttpSockets = 0; } -int +static int varyEvaluateMatch(StoreEntry * entry, request_t * request) { const char *vary = request->vary_headers; @@ -3944,6 +4186,7 @@ /* Ouch.. we cannot handle this kind of variance */ /* XXX This cannot really happen, but just to be complete */ return VARY_CANCEL; +#if NOT_FOR_ETAGS } else if (strcmp(vary, entry->mem_obj->vary_headers) == 0) { return VARY_MATCH; } else { @@ -3953,6 +4196,10 @@ debug(33, 1) ("varyEvaluateMatch: Oops. Not a Vary match on second attempt, '%s' '%s'\n", entry->mem_obj->url, vary); return VARY_CANCEL; +#else + } else { + return VARY_MATCH; +#endif } } } Index: squid/src/defines.h diff -u squid/src/defines.h:1.15.6.4 squid/src/defines.h:1.12.4.5.2.6 --- squid/src/defines.h:1.15.6.4 Fri Mar 25 19:15:56 2005 +++ squid/src/defines.h Sat May 20 12:15:15 2006 @@ -237,7 +237,7 @@ #define countof(arr) (sizeof(arr)/sizeof(*arr)) /* to initialize static variables (see also MemBufNull) */ -#define MemBufNULL { NULL, 0, 0, 0, NULL } +#define MemBufNULL { NULL, 0, 0, 0, 0 } /* * Max number of ICP messages to receive per call to icpHandleUdp Index: squid/src/enums.h diff -u squid/src/enums.h:1.29.2.18 squid/src/enums.h:1.21.8.6.2.12 --- squid/src/enums.h:1.29.2.18 Fri Nov 11 19:13:48 2005 +++ squid/src/enums.h Sat May 20 12:15:15 2006 @@ -609,7 +609,6 @@ MEM_EVENT, MEM_TLV, MEM_SWAP_LOG_DATA, - MEM_CLIENT_REQ_BUF, MEM_MAX } mem_type; Index: squid/src/ftp.c diff -u squid/src/ftp.c:1.18.6.27 squid/src/ftp.c:1.13.14.4.2.12 --- squid/src/ftp.c:1.18.6.27 Sat Feb 25 19:13:57 2006 +++ squid/src/ftp.c Sat May 20 12:15:16 2006 @@ -114,7 +114,6 @@ char *buf; size_t size; size_t offset; - FREE *freefunc; wordlist *message; char *last_command; char *last_reply; @@ -125,7 +124,6 @@ char *buf; size_t size; size_t offset; - FREE *freefunc; char *host; u_short port; } data; @@ -288,21 +286,15 @@ storeUnlockObject(ftpState->entry); if (ftpState->reply_hdr) { memFree(ftpState->reply_hdr, MEM_8K_BUF); - /* this seems unnecessary, but people report SEGV's - * when freeing memory in this function */ ftpState->reply_hdr = NULL; } requestUnlink(ftpState->request); if (ftpState->ctrl.buf) { - ftpState->ctrl.freefunc(ftpState->ctrl.buf); - /* this seems unnecessary, but people report SEGV's - * when freeing memory in this function */ + memFreeBuf(ftpState->ctrl.size, ftpState->ctrl.buf); ftpState->ctrl.buf = NULL; } if (ftpState->data.buf) { - ftpState->data.freefunc(ftpState->data.buf); - /* this seems unnecessary, but people report SEGV's - * when freeing memory in this function */ + memFreeBuf(ftpState->data.size, ftpState->data.buf); ftpState->data.buf = NULL; } if (ftpState->pathcomps) @@ -1160,13 +1152,9 @@ ftpState->user, ftpState->password); ftpState->state = BEGIN; ftpState->ctrl.last_command = xstrdup("Connect to server"); - ftpState->ctrl.buf = memAllocate(MEM_4K_BUF); - ftpState->ctrl.freefunc = memFree4K; - ftpState->ctrl.size = 4096; + ftpState->ctrl.buf = memAllocBuf(4096, &ftpState->ctrl.size); ftpState->ctrl.offset = 0; - ftpState->data.buf = xmalloc(SQUID_TCP_SO_RCVBUF); - ftpState->data.size = SQUID_TCP_SO_RCVBUF; - ftpState->data.freefunc = xfree; + ftpState->data.buf = memAllocBuf(SQUID_TCP_SO_RCVBUF, &ftpState->data.size); ftpScheduleReadControlReply(ftpState, 0); } @@ -1389,7 +1377,6 @@ static void ftpHandleControlReply(FtpStateData * ftpState) { - char *oldbuf; wordlist **W; int bytes_used = 0; wordlistDestroy(&ftpState->ctrl.message); @@ -1398,12 +1385,7 @@ if (ftpState->ctrl.message == NULL) { /* didn't get complete reply yet */ if (ftpState->ctrl.offset == ftpState->ctrl.size) { - oldbuf = ftpState->ctrl.buf; - ftpState->ctrl.buf = xcalloc(ftpState->ctrl.size << 1, 1); - xmemcpy(ftpState->ctrl.buf, oldbuf, ftpState->ctrl.size); - ftpState->ctrl.size <<= 1; - ftpState->ctrl.freefunc(oldbuf); - ftpState->ctrl.freefunc = xfree; + ftpState->ctrl.buf = memReallocBuf(ftpState->ctrl.buf, ftpState->ctrl.size << 1, &ftpState->ctrl.size); } ftpScheduleReadControlReply(ftpState, 0); return; Index: squid/src/http.c diff -u squid/src/http.c:1.17.6.32 squid/src/http.c:1.13.14.10.2.9 --- squid/src/http.c:1.17.6.32 Tue Oct 18 19:13:21 2005 +++ squid/src/http.c Sat May 20 12:15:16 2006 @@ -137,8 +137,6 @@ int remove = 0; int forbidden = 0; StoreEntry *pe; - if (!EBIT_TEST(e->flags, KEY_PRIVATE)) - return; switch (status) { case HTTP_OK: case HTTP_NON_AUTHORITATIVE_INFORMATION: @@ -170,14 +168,16 @@ } if (!remove && !forbidden) return; - assert(e->mem_obj); - if (e->mem_obj->request) - pe = storeGetPublicByRequest(e->mem_obj->request); - else - pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method); - if (pe != NULL) { - assert(e != pe); - storeRelease(pe); + if (EBIT_TEST(e->flags, KEY_PRIVATE)) { + assert(e->mem_obj); + if (e->mem_obj->request) + pe = storeGetPublicByRequest(e->mem_obj->request); + else + pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method); + if (pe != NULL) { + assert(e != pe); + storeRelease(pe); + } } /* * Also remove any cached HEAD response in case the object has @@ -187,8 +187,7 @@ pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_HEAD); else pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD); - if (pe != NULL) { - assert(e != pe); + if (pe != NULL && e != pe) { storeRelease(pe); } if (forbidden) @@ -330,7 +329,7 @@ * Returns false if the variance cannot be stored */ const char * -httpMakeVaryMark(request_t * request, HttpReply * reply) +httpMakeVaryMark(const request_t * request, HttpReply * reply) { String vary, hdr; const char *pos = NULL; @@ -457,8 +456,6 @@ storeTimestampsSet(entry); /* Check if object is cacheable or not based on reply code */ debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); - if (neighbors_do_private_keys) - httpMaybeRemovePublic(entry, reply->sline.status); if (httpHeaderHas(&reply->header, HDR_VARY) #if X_ACCELERATOR_VARY || httpHeaderHas(&reply->header, HDR_X_ACCELERATOR_VARY) @@ -495,6 +492,8 @@ else if (EBIT_TEST(reply->cache_control->mask, CC_MUST_REVALIDATE)) EBIT_SET(entry->flags, ENTRY_REVALIDATE); } + if (neighbors_do_private_keys) + httpMaybeRemovePublic(entry, reply->sline.status); if (httpState->flags.keepalive) if (httpState->peer) httpState->peer->stats.n_keepalives_sent++; @@ -843,7 +842,13 @@ /* append our IMS header */ if (request->lastmod > -1) httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); - + if (request->etag) + httpHeaderPutStr(hdr_out, HDR_IF_NONE_MATCH, request->etag); + else if (request->etags) { + int i; + for (i = 0; i < request->etags->count; i++) + httpHeaderPutStr(hdr_out, HDR_IF_NONE_MATCH, request->etags->items[i]); + } /* decide if we want to do Ranges ourselves * (and fetch the whole object now) * We want to handle Ranges ourselves iff Index: squid/src/mem.c diff -u squid/src/mem.c:1.13 squid/src/mem.c:1.12.8.3.2.5 --- squid/src/mem.c:1.13 Fri Sep 7 16:55:49 2001 +++ squid/src/mem.c Sat May 20 13:27:30 2006 @@ -62,6 +62,8 @@ static MemMeter StrCountMeter; static MemMeter StrVolumeMeter; +static MemMeter HugeBufCountMeter; +static MemMeter HugeBufVolumeMeter; /* local routines */ @@ -92,6 +94,15 @@ "Other Strings", xpercentInt(StrCountMeter.level - pooled_count, StrCountMeter.level), xpercentInt(StrVolumeMeter.level - pooled_volume, StrVolumeMeter.level)); + storeAppendPrintf(sentry, "\n"); +} + +static void +memBufStats(StoreEntry * sentry) +{ + storeAppendPrintf(sentry, "Large buffers: %d (%d KB)\n", + (int)HugeBufCountMeter.level, + (int)HugeBufVolumeMeter.level / 1024); } static void @@ -100,6 +111,7 @@ storeBuffer(sentry); memReport(sentry); memStringStats(sentry); + memBufStats(sentry); storeBufferFlush(sentry); } @@ -136,7 +148,7 @@ /* allocate a variable size buffer using best-fit pool */ void * -memAllocBuf(size_t net_size, size_t * gross_size) +memAllocString(size_t net_size, size_t * gross_size) { int i; MemPool *pool = NULL; @@ -154,9 +166,9 @@ return pool ? memPoolAlloc(pool) : xcalloc(1, net_size); } -/* free buffer allocated with memAllocBuf() */ +/* free buffer allocated with memAllocString() */ void -memFreeBuf(size_t size, void *buf) +memFreeString(size_t size, void *buf) { int i; MemPool *pool = NULL; @@ -173,6 +185,86 @@ pool ? memPoolFree(pool, buf) : xfree(buf); } +/* Find the best fit MEM_X_BUF type */ +static mem_type +memFindBufSizeType(size_t net_size, size_t * gross_size) +{ + mem_type type; + size_t size; + if (net_size <= 2 * 1024) { + type = MEM_2K_BUF; + size = 2 * 1024; + } else if (net_size <= 4 * 1024) { + type = MEM_4K_BUF; + size = 4 * 1024; + } else if (net_size <= 8 * 1024) { + type = MEM_8K_BUF; + size = 8 * 1024; + } else if (net_size <= 16 * 1024) { + type = MEM_16K_BUF; + size = 16 * 1024; + } else if (net_size <= 32 * 1024) { + type = MEM_32K_BUF; + size = 32 * 1024; + } else if (net_size <= 64 * 1024) { + type = MEM_64K_BUF; + size = 64 * 1024; + } else { + type = MEM_NONE; + size = net_size; + } + if (gross_size) + *gross_size = size; + return type; +} + +/* allocate a variable size buffer using best-fit pool */ +void * +memAllocBuf(size_t net_size, size_t * gross_size) +{ + mem_type type = memFindBufSizeType(net_size, gross_size); + if (type != MEM_NONE) + return memAllocate(type); + else { + memMeterInc(HugeBufCountMeter); + memMeterAdd(HugeBufVolumeMeter, *gross_size); + return xcalloc(1, net_size); + } +} + +/* resize a variable sized buffer using best-fit pool */ +void * +memReallocBuf(void *oldbuf, size_t net_size, size_t * gross_size) +{ + /* XXX This can be optimized on very large buffers to use realloc() */ + size_t new_gross_size; + void *newbuf = memAllocBuf(net_size, &new_gross_size); + if (oldbuf) { + int data_size = *gross_size; + if (data_size > net_size) + data_size = net_size; + memcpy(newbuf, oldbuf, data_size); + memFreeBuf(*gross_size, oldbuf); + } + *gross_size = new_gross_size; + return newbuf; +} + +/* free buffer allocated with memAllocBuf() */ +void +memFreeBuf(size_t size, void *buf) +{ + mem_type type = memFindBufSizeType(size, NULL); + if (type != MEM_NONE) + memFree(buf, type); + else { + xfree(buf); + memMeterDec(HugeBufCountMeter); + memMeterDel(HugeBufVolumeMeter, size); + } +} + + void memInit(void) { @@ -240,7 +332,6 @@ memDataInit(MEM_HELPER_STATEFUL_REQUEST, "helper_stateful_request", sizeof(helper_stateful_request), 0); memDataInit(MEM_TLV, "storeSwapTLV", sizeof(tlv), 0); - memDataInit(MEM_CLIENT_REQ_BUF, "clientRequestBuffer", CLIENT_REQ_BUF_SZ, 0); memDataInit(MEM_SWAP_LOG_DATA, "storeSwapLogData", sizeof(storeSwapLogData), 0); /* init string pools */ @@ -284,7 +375,7 @@ /* ick */ -void +static void memFree2K(void *p) { memFree(p, MEM_2K_BUF); @@ -302,20 +393,43 @@ memFree(p, MEM_8K_BUF); } -void +static void memFree16K(void *p) { memFree(p, MEM_16K_BUF); } -void +static void memFree32K(void *p) { memFree(p, MEM_32K_BUF); } -void +static void memFree64K(void *p) { memFree(p, MEM_64K_BUF); } + +FREE * +memFreeBufFunc(size_t size) +{ + switch (size) { + case 2 * 1024: + return memFree2K; + case 4 * 1024: + return memFree4K; + case 8 * 1024: + return memFree8K; + case 16 * 1024: + return memFree16K; + case 32 * 1024: + return memFree32K; + case 64 * 1024: + return memFree64K; + default: + memMeterDec(HugeBufCountMeter); + memMeterDel(HugeBufVolumeMeter, size); + return xfree; + } +} Index: squid/src/protos.h diff -u squid/src/protos.h:1.41.6.34 squid/src/protos.h:1.29.2.12.2.13 --- squid/src/protos.h:1.41.6.34 Sat Feb 25 19:13:57 2006 +++ squid/src/protos.h Sat May 20 12:20:54 2006 @@ -302,11 +302,7 @@ extern int httpAnonHdrDenied(http_hdr_type hdr_id); extern void httpBuildRequestHeader(request_t *, request_t *, StoreEntry *, HttpHeader *, http_state_flags); extern void httpBuildVersion(http_version_t * version, unsigned int major, unsigned int minor); -extern const char *httpMakeVaryMark(request_t * request, HttpReply * reply); - -/* ETag */ -extern int etagParseInit(ETag * etag, const char *str); -extern int etagIsEqual(const ETag * tag1, const ETag * tag2); +extern const char *httpMakeVaryMark(const request_t * request, HttpReply * reply); /* Http Status Line */ /* init/clean */ @@ -436,7 +432,6 @@ extern time_t httpHeaderGetTime(const HttpHeader * hdr, http_hdr_type id); extern TimeOrTag httpHeaderGetTimeOrTag(const HttpHeader * hdr, http_hdr_type id); extern HttpHdrCc *httpHeaderGetCc(const HttpHeader * hdr); -extern ETag httpHeaderGetETag(const HttpHeader * hdr, http_hdr_type id); extern HttpHdrRange *httpHeaderGetRange(const HttpHeader * hdr); extern HttpHdrContRange *httpHeaderGetContRange(const HttpHeader * hdr); extern const char *httpHeaderGetStr(const HttpHeader * hdr, http_hdr_type id); @@ -834,15 +829,15 @@ extern void memCleanModule(void); extern void memConfigure(void); extern void *memAllocate(mem_type); +extern void *memAllocString(size_t net_size, size_t * gross_size); extern void *memAllocBuf(size_t net_size, size_t * gross_size); +extern void *memReallocBuf(void *buf, size_t net_size, size_t * gross_size); extern void memFree(void *, int type); -extern void memFreeBuf(size_t size, void *); -extern void memFree2K(void *); extern void memFree4K(void *); extern void memFree8K(void *); -extern void memFree16K(void *); -extern void memFree32K(void *); -extern void memFree64K(void *); +extern void memFreeString(size_t size, void *); +extern void memFreeBuf(size_t size, void *); +extern FREE *memFreeBufFunc(size_t size); extern int memInUse(mem_type); extern int memTotalAllocated(void); extern void memDataInit(mem_type, const char *, size_t, int); @@ -1316,9 +1311,6 @@ */ extern StatCounters *snmpStatGet(int); -/* Vary support functions */ -int varyEvaluateMatch(StoreEntry * entry, request_t * req); - /* CygWin & Windows NT Port */ /* win32.c */ #if defined(_SQUID_MSWIN_) || defined(_SQUID_CYGWIN_) @@ -1341,4 +1333,9 @@ extern int externalAclRequiresAuth(void *acl_data); extern char *strtokFile(void); +/* ETag support */ +void storeLocateVaryDone(VaryData * data); +void storeLocateVary(StoreEntry * e, int offset, const char *vary_data, STLVCB * callback, void *cbdata); +void storeAddVary(const char *url, const char *log_url, const method_t method, const cache_key * key, const char *etag, const char *vary, const char *vary_headers); + #endif /* SQUID_PROTOS_H */ Index: squid/src/store.c diff -u squid/src/store.c:1.16.6.9 squid/src/store.c:1.12.14.20.2.11 --- squid/src/store.c:1.16.6.9 Thu Sep 1 19:13:43 2005 +++ squid/src/store.c Sat May 20 13:27:30 2006 @@ -330,6 +330,13 @@ StoreEntry * storeGetPublicByRequestMethod(request_t * req, const method_t method) { + if (req->vary) { + /* Varying objects... */ + if (req->vary->key) + return storeGet(storeKeyScan(req->vary->key)); + else + return NULL; + } return storeGet(storeKeyPublicByRequestMethod(req, method)); } @@ -375,6 +382,447 @@ storeHashInsert(e, newkey); } +typedef struct { + StoreEntry *oe; + StoreEntry *e; + store_client *sc; + char *url; + char *key; + char *vary_headers; + char *etag; + int offset; + int seen_offset; + char *buf; + size_t buf_size; + int done:1; + struct { + char *key; + char *etag; + int this_key:1; + int key_used:1; + int ignore:1; + } current; +} AddVaryState; + +CBDATA_TYPE(AddVaryState); +static void +free_AddVaryState(void *data) +{ + AddVaryState *state = data; + debug(11, 2) ("free_AddVaryState: %p\n", data); + if (!state->done && state->key) { + storeAppendPrintf(state->e, "Key: %s\n", state->key); + if (state->etag) + storeAppendPrintf(state->e, "ETag: %s\n", state->etag); + storeAppendPrintf(state->e, "VaryData: %s\n", state->vary_headers); + } + storeBufferFlush(state->e); + storeTimestampsSet(state->e); + storeComplete(state->e); + storeTimestampsSet(state->e); + storeUnlockObject(state->e); + state->e = NULL; + if (state->oe) { + storeUnlockObject(state->oe); + state->oe = NULL; + } + safe_free(state->url); + safe_free(state->key); + safe_free(state->vary_headers); + safe_free(state->etag); + safe_free(state->current.key); + safe_free(state->current.etag); + if (state->buf) { + memFreeBuf(state->buf_size, state->buf); + state->buf = NULL; + } +} + +static int inline +strmatchbeg(const char *search, const char *match, int maxlen) +{ + int mlen = strlen(match); + if (maxlen < mlen) + return -1; + return strncmp(search, match, mlen); +} + +static int inline +strmatch(const char *search, const char *match, int maxlen) +{ + int mlen = strlen(match); + if (maxlen < mlen) + return -1; + return strncmp(search, match, maxlen); +} + +static void +storeAddVaryFlush(AddVaryState * state) +{ + if (state->current.ignore || state->current.key_used) { + /* do nothing */ + } else if (state->current.this_key) { + if (state->current.key) + storeAppendPrintf(state->e, "Key: %s\n", state->current.key); + else + storeAppendPrintf(state->e, "Key: %s\n", state->key); + if (state->etag) + storeAppendPrintf(state->e, "ETag: %s\n", state->etag); + storeAppendPrintf(state->e, "VaryData: %s\n", state->vary_headers); + state->done = 1; + state->current.key_used = 1; + } else if (state->current.key) { + storeAppendPrintf(state->e, "Key: %s\n", state->current.key); + safe_free(state->current.key); + if (state->current.etag) { + storeAppendPrintf(state->e, "ETag: %s\n", state->current.etag); + safe_free(state->current.etag); + } + state->current.key_used = 1; + } +} + +static void +storeAddVaryReadOld(void *data, char *buf, ssize_t size) +{ + AddVaryState *state = data; + int l = size; + char *e; + char *p = buf; + debug(11, 3) ("storeAddVaryReadOld: %p offset=%d seen_offset=%d size=%d\n", data, state->offset, state->seen_offset, (int)size); + if (size <= 0) { + storeUnregister(state->sc, state->oe, state); + state->sc = NULL; + cbdataFree(state); + debug(11, 2) ("storeAddVaryReadOld: DONE\n"); + return; + } + state->seen_offset = state->offset + size; + while ((e = memchr(p, '\n', l)) != NULL) { + int l2; + char *p2; + if (strmatchbeg(p, "Key: ", l) == 0) { + /* key field */ + p2 = p + 5; + l2 = e - p2; + if (state->current.this_key) { + storeAddVaryFlush(state); + } + safe_free(state->current.key); + safe_free(state->current.etag); + memset(&state->current, 0, sizeof(state->current)); + state->current.key = xmalloc(l2 + 1); + memcpy(state->current.key, p2, l2); + state->current.key[l2] = '\0'; + if (state->key) { + if (strcmp(state->current.key, state->key) == 0) { + state->current.this_key = 1; + } + } + debug(11, 3) ("storeAddVaryReadOld: Key: %s%s\n", state->current.key, state->current.this_key ? " (THIS)" : ""); + } else if (strmatchbeg(p, "ETag: ", l) == 0) { + /* etag field */ + p2 = p + 6; + l2 = e - p2; + safe_free(state->current.etag); + state->current.etag = xmalloc(l2 + 1); + memcpy(state->current.etag, p2, l2); + state->current.etag[l2] = '\0'; + if (state->etag && strcmp(state->current.etag, state->etag) == 0) { + if (!state->key) { + state->current.this_key = 1; + } else { + const cache_key *oldkey = storeKeyScan(state->current.key); + if (strmatch(p2, state->key, l) != 0) { + StoreEntry *old_e = storeGet(oldkey); + if (old_e) + storeRelease(old_e); + safe_free(state->current.key); + state->current.key = xstrdup(state->key); + state->current.this_key = 1; + } + } + } else if (state->current.this_key) { + state->current.ignore = 1; + } + debug(11, 2) ("storeAddVaryReadOld: ETag: %s%s%s\n", state->current.etag, state->current.this_key ? " (THIS)" : "", state->current.ignore ? " (IGNORE)" : ""); + } else if (!state->current.ignore && strmatchbeg(p, "VaryData: ", l) == 0) { + /* vary field */ + p2 = p + 10; + l2 = e - p2; + storeAddVaryFlush(state); + if (strmatch(p2, state->vary_headers, l2) != 0) { + storeAppend(state->e, p, e - p + 1); + debug(11, 3) ("storeAddVaryReadOld: %s\n", p); + } + } + e += 1; + l -= e - p; + p = e; + if (l == 0) + break; + assert(l > 0); + assert(p < (buf + size)); + } + if (p == state->buf && size == state->buf_size) { + /* Oops.. the buffer size is not sufficient. Grow */ + if (state->buf_size < 65536) { + debug(11, 2) ("storeAddVaryReadOld: Increasing entry buffer size to %d\n", (int)state->buf_size * 2); + state->buf = memReallocBuf(state->buf, state->buf_size * 2, &state->buf_size); + } else { + /* This does not look good. Bail out. This should match the size <= 0 case above */ + debug(11, 1) ("storeAddVaryReadOld: Buffer very large and still can't fit the data.. bailing out\n"); + storeUnregister(state->sc, state->oe, state); + state->sc = NULL; + cbdataFree(state); + return; + } + } + state->offset += p - buf; + debug(11, 3) ("storeAddVaryReadOld: %p offset=%d seen_offset=%d\n", data, state->offset, state->seen_offset); + storeClientCopy(state->sc, state->oe, + state->seen_offset, + state->offset, + state->buf_size, + state->buf, + storeAddVaryReadOld, + state); +} + +/* + * Adds/updates a Vary record. + * For updates only one of key or etag needs to be specified + * At leas one of key or etag must be specified, preferably both. + */ +void +storeAddVary(const char *url, const char *log_url, const method_t method, const cache_key * key, const char *etag, const char *vary, const char *vary_headers) +{ + AddVaryState *state; + http_version_t version; + request_flags flags = null_request_flags; + CBDATA_INIT_TYPE_FREECB(AddVaryState, free_AddVaryState); + state = cbdataAlloc(AddVaryState); + state->url = xstrdup(url); + if (key) + state->key = xstrdup(storeKeyText(key)); + state->vary_headers = xstrdup(vary_headers); + if (etag) + state->etag = xstrdup(etag); + state->oe = storeGetPublic(url, method); + debug(11, 2) ("storeAddVary: %s (%s) %s %s\n", + state->url, state->key, state->vary_headers, state->etag); + if (state->oe) + storeLockObject(state->oe); + flags.cachable = 1; + state->e = storeCreateEntry(url, log_url, flags, method); + httpBuildVersion(&version, 1, 0); + httpReplySetHeaders(state->e->mem_obj->reply, version, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); + httpHeaderPutStr(&state->e->mem_obj->reply->header, HDR_VARY, vary); + storeSetPublicKey(state->e); + storeBuffer(state->e); + httpReplySwapOut(state->e->mem_obj->reply, state->e); + storeBufferFlush(state->e); + if (state->oe) { + /* Here we need to tack on the old etag/vary information, and we should + * merge, clean up etc + * + * Suggestion: + * swap in the old file, looking for ETag, Key and VaryData. If a match is + * found then + * - on ETag, update the key, and expire the old object if different + * - on Key, drop the old data if ETag is different, else nothing + * - on VaryData, remove the line if a different key. If this makes + * the searched key "empty" then expire it and remove it from the + * map + * - VaryData is added last in the Key record it corresponds to (after + * modifications above) + */ + /* Swap in the dummy Vary object */ + if (!state->oe->mem_obj) { + storeCreateMemObject(state->oe, state->url, log_url); + state->oe->mem_obj->method = method; + } + state->sc = storeClientListAdd(state->oe, state); + state->buf = memAllocBuf(4096, &state->buf_size); + debug(11, 3) ("storeAddVary: %p\n", state); + storeClientCopy(state->sc, state->oe, 0, 0, + state->buf_size, + state->buf, + storeAddVaryReadOld, + state); + return; + } else { + cbdataFree(state); + } +} + +static MemPool *VaryData_pool = NULL; + +void +storeLocateVaryDone(VaryData * data) +{ + int i; + safe_free(data->key); + data->etag = NULL; /* points to an entry in etags */ + for (i = 0; i < data->etags.count; i++) { + safe_free(data->etags.items[i]); + } + arrayClean(&data->etags); + memPoolFree(VaryData_pool, data); +} + +typedef struct { + VaryData *data; + STLVCB *callback; + void *callback_data; + StoreEntry *e; + store_client *sc; + char *buf; + size_t buf_size; + char *vary_data; + int offset, seen_offset; + struct { + char *key; + char *etag; + } current; +} LocateVaryState; + +CBDATA_TYPE(LocateVaryState); + +static void +storeLocateVaryCallback(LocateVaryState * state) +{ + if (cbdataValid(state->callback_data)) { + VaryData *data = state->data; + if (data->key || data->etags.count) { + state->callback(data, state->callback_data); + state->data = NULL; /* now owned by the caller */ + } else { + state->callback(NULL, state->callback_data); + } + } + cbdataUnlock(state->callback_data); + if (state->data) { + storeLocateVaryDone(state->data); + state->data = NULL; + } + state->current.etag = NULL; /* shared by data->entries[x] */ + safe_free(state->vary_data); + safe_free(state->current.key); + if (state->sc) { + storeUnregister(state->sc, state->e, state); + state->sc = NULL; + } + if (state->e) { + storeUnlockObject(state->e); + state->e = NULL; + } + if (state->buf) { + memFreeBuf(state->buf_size, state->buf); + state->buf = NULL; + } + cbdataFree(state); + debug(11, 2) ("storeLocateVary: DONE\n"); +} + +static void +storeLocateVaryRead(void *data, char *buf, ssize_t size) +{ + LocateVaryState *state = data; + char *e; + char *p = buf; + int l = size; + debug(11, 3) ("storeLocateVaryRead: %s %p offset=%d seen_offset=%d size=%d\n", state->vary_data, data, state->offset, state->seen_offset, (int)size); + if (size <= 0) { + storeLocateVaryCallback(state); + return; + } + state->seen_offset = state->offset + size; + while ((e = memchr(p, '\n', l)) != NULL) { + int l2; + char *p2; + if (strmatchbeg(p, "Key: ", l) == 0) { + /* key field */ + p2 = p + 5; + l2 = e - p2; + safe_free(state->current.key); + state->current.etag = NULL; + safe_free(state->current.etag); + memset(&state->current, 0, sizeof(state->current)); + state->current.key = xmalloc(l2 + 1); + memcpy(state->current.key, p2, l2); + state->current.key[l2] = '\0'; + debug(11, 3) ("storeLocateVaryRead: Key: %s\n", state->current.key); + } else if (strmatchbeg(p, "ETag: ", l) == 0) { + /* etag field */ + char *etag; + p2 = p + 6; + l2 = e - p2; + etag = xmalloc(l2 + 1); + memcpy(etag, p2, l2); + etag[l2] = '\0'; + state->current.etag = etag; + arrayAppend(&state->data->etags, etag); + debug(11, 3) ("storeLocateVaryRead: ETag: %s\n", etag); + } else if (strmatchbeg(p, "VaryData: ", l) == 0) { + /* vary field */ + p2 = p + 10; + l2 = e - p2; + if (strmatch(p2, state->vary_data, l2) == 0) { + /* A matching vary header found */ + safe_free(state->data->key); + state->data->key = xstrdup(state->current.key); + state->data->etag = state->current.etag; + debug(11, 2) ("storeLocateVaryRead: MATCH! %s %s\n", state->current.key, state->current.etag); + } + } + e += 1; + l -= e - p; + p = e; + if (l == 0) + break; + assert(l > 0); + assert(p < (buf + size)); + } + state->offset += p - buf; + if (p == state->buf && size == state->buf_size) { + /* Oops.. the buffer size is not sufficient. Grow */ + if (state->buf_size < 65536) { + debug(11, 2) ("storeLocateVaryRead: Increasing entry buffer size to %d\n", (int)state->buf_size * 2); + state->buf = memReallocBuf(state->buf, state->buf_size * 2, &state->buf_size); + } else { + /* This does not look good. Bail out. This should match the size <= 0 case above */ + debug(11, 1) ("storeLocateVaryRead: Buffer very large and still can't fit the data.. bailing out\n"); + storeLocateVaryCallback(state); + return; + } + } + debug(11, 3) ("storeLocateVaryRead: %p offset=%d seen_offset=%d\n", data, state->offset, state->seen_offset); + storeClientCopy(state->sc, state->e, state->seen_offset, state->offset, state->buf_size, state->buf, storeLocateVaryRead, state); +} + +void +storeLocateVary(StoreEntry * e, int offset, const char *vary_data, STLVCB * callback, void *cbdata) +{ + LocateVaryState *state; + debug(11, 2) ("storeLocateVary: %s\n", vary_data); + CBDATA_INIT_TYPE(LocateVaryState); + if (!VaryData_pool) + VaryData_pool = memPoolCreate("VaryData", sizeof(VaryData)); + state = cbdataAlloc(LocateVaryState); + state->vary_data = xstrdup(vary_data); + state->data = memPoolAlloc(VaryData_pool); + state->e = e; + storeLockObject(state->e); + state->callback_data = cbdata; + cbdataLock(cbdata); + state->callback = callback; + state->buf = memAllocBuf(4096, &state->buf_size); + state->sc = storeClientListAdd(state->e, state); + state->offset = state->seen_offset = offset; + storeClientCopy(state->sc, state->e, state->seen_offset, state->offset, state->buf_size, state->buf, storeLocateVaryRead, state); +} + void storeSetPublicKey(StoreEntry * e) { @@ -423,36 +871,27 @@ request->vary_headers = xstrdup(vary); } } - if (mem->vary_headers && !storeGetPublic(mem->url, mem->method)) { - /* Create "vary" base object */ - http_version_t version; - String vary; - pe = storeCreateEntry(mem->url, mem->log_url, request->flags, request->method); - httpBuildVersion(&version, 1, 0); - httpReplySetHeaders(pe->mem_obj->reply, version, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); - vary = httpHeaderGetList(&mem->reply->header, HDR_VARY); - if (strBuf(vary)) { - httpHeaderPutStr(&pe->mem_obj->reply->header, HDR_VARY, strBuf(vary)); - stringClean(&vary); - } + newkey = storeKeyPublicByRequest(mem->request); + if (mem->vary_headers) { + String vary = StringNull; + String varyhdr; + varyhdr = httpHeaderGetList(&mem->reply->header, HDR_VARY); + if (strBuf(varyhdr)) + strListAdd(&vary, strBuf(varyhdr), ','); + stringClean(&varyhdr); #if X_ACCELERATOR_VARY - vary = httpHeaderGetList(&mem->reply->header, HDR_X_ACCELERATOR_VARY); - if (strBuf(vary)) { - httpHeaderPutStr(&pe->mem_obj->reply->header, HDR_X_ACCELERATOR_VARY, strBuf(vary)); - stringClean(&vary); - } + /* This needs to match the order in http.c:httpMakeVaryMark */ + varyhdr = httpHeaderGetList(&mem->reply->header, HDR_X_ACCELERATOR_VARY); + if (strBuf(varyhdr)) + strListAdd(&vary, strBuf(varyhdr), ','); + stringClean(&varyhdr); #endif - storeSetPublicKey(pe); - storeBuffer(pe); - httpReplySwapOut(pe->mem_obj->reply, pe); - storeBufferFlush(pe); - storeTimestampsSet(pe); - storeComplete(pe); - storeUnlockObject(pe); + storeAddVary(mem->url, mem->log_url, mem->method, newkey, httpHeaderGetStr(&mem->reply->header, HDR_ETAG), strBuf(vary), mem->vary_headers); + stringClean(&vary); } - newkey = storeKeyPublicByRequest(mem->request); - } else + } else { newkey = storeKeyPublic(mem->url, mem->method); + } if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); storeSetPrivateKey(e2); Index: squid/src/structs.h diff -u squid/src/structs.h:1.48.2.46 squid/src/structs.h:1.33.4.10.2.18 --- squid/src/structs.h:1.48.2.46 Fri Mar 10 19:16:31 2006 +++ squid/src/structs.h Sat May 20 13:27:30 2006 @@ -834,7 +834,7 @@ /* private, stay away; use interface function instead */ mb_size_t max_capacity; /* when grows: assert(new_capacity <= max_capacity) */ mb_size_t capacity; /* allocated space */ - FREE *freefunc; /* what to use to free the buffer, NULL after memBufFreeFunc() is called */ + unsigned stolen:1; /* the buffer has been stolen for use by someone else */ }; /* see Packer.c for description */ @@ -899,7 +899,7 @@ /* some fields can hold either time or etag specs (e.g. If-Range) */ struct _TimeOrTag { - ETag tag; /* entity tag */ + const char *tag; /* entity tag */ time_t time; int valid; /* true if struct is usable */ }; @@ -1684,7 +1684,11 @@ err_type err_type; char *peer_login; /* Configured peer login:password */ time_t lastmod; /* Used on refreshes */ - const char *vary_headers; /* Used when varying entities are detected. Changes how the store key is calculated */ + char *vary_headers; /* Used when varying entities are detected. Changes how the store key is calculated */ + VaryData *vary; + Array *etags; /* possible known entity tags (Vary MISS) */ + char *etag; /* current entity tag, cache validation */ + unsigned int done_etag:1; /* We have done clientProcessETag on this, don't attempt it again */ BODY_HANDLER *body_reader; void *body_reader_data; }; @@ -2220,4 +2224,10 @@ void (*dump) (StoreEntry * e, const char *option, SwapDir * sd); }; +struct _VaryData { + char *key; + char *etag; + Array etags; +}; + #endif /* SQUID_STRUCTS_H */ Index: squid/src/typedefs.h diff -u squid/src/typedefs.h:1.25.6.8 squid/src/typedefs.h:1.21.4.3.2.7 --- squid/src/typedefs.h:1.25.6.8 Sat Mar 26 18:16:17 2005 +++ squid/src/typedefs.h Sat May 20 12:20:54 2006 @@ -386,4 +386,7 @@ typedef struct _external_acl external_acl; typedef struct _external_acl_entry external_acl_entry; +typedef struct _VaryData VaryData; +typedef void STLVCB(VaryData * vary, void *cbdata); + #endif /* SQUID_TYPEDEFS_H */