This patch is generated from the vary-2_4 branch of s2_4 in squid Sun Jan 25 13:59:17 2004 GMT See http://devel.squid-cache.org/ Index: squid/acconfig.h diff -u squid/acconfig.h:1.3.4.1 squid/acconfig.h:1.3.4.1.4.1 --- squid/acconfig.h:1.3.4.1 Sat Jan 6 05:32:43 2001 +++ squid/acconfig.h Thu Feb 1 05:59:31 2001 @@ -290,3 +290,8 @@ * Enable support for Transparent Proxy on Linux 2.4 systems */ #undef LINUX_NETFILTER + +/* + * Enable support for the X-Accelerator-Vary HTTP header + */ +#undef X_ACCELERATOR_VARY Index: squid/configure.in diff -u squid/configure.in:1.3.4.8 squid/configure.in:1.3.4.5.2.2.2.1 --- squid/configure.in:1.3.4.8 Mon Oct 1 15:04:09 2001 +++ squid/configure.in Tue Oct 16 14:57:07 2001 @@ -737,6 +737,19 @@ fi ]) +dnl Enable X-Accelerator-Vary for Vary support within an accelerator setup +AC_ARG_ENABLE(x_accelerator_vary, +[ --enable-x-accelerator-vary Enable support for the X-Accelerator-Vary + HTTP header. Can be used to indicate + variance within an accelerator setup. + Typically used together with other code + that adds custom HTTP headers to the requests.], +[ if test "$enableval" = "yes" ; then + echo "Enabling support for X-Accelerator-Vary" + AC_DEFINE(X_ACCELERATOR_VARY, 1) + fi +]) + # Force some compilers to use ANSI features # case "$host" in Index: squid/include/version.h diff -u squid/include/version.h:1.3.4.4 squid/include/version.h:1.3.4.1.4.4 --- squid/include/version.h:1.3.4.4 Sun Sep 9 13:18:35 2001 +++ squid/include/version.h Tue Oct 16 14:59:04 2001 @@ -4,7 +4,7 @@ * SQUID_VERSION - String for version id of this distribution */ #ifndef SQUID_VERSION -#define SQUID_VERSION "2.4.STABLE2" +#define SQUID_VERSION "2.4.STABLE2-vary" #endif #ifndef SQUID_RELEASE_TIME Index: squid/src/HttpHeader.c diff -u squid/src/HttpHeader.c:1.3.4.2 squid/src/HttpHeader.c:1.3.4.2.2.3.2.1 --- squid/src/HttpHeader.c:1.3.4.2 Mon Jan 15 14:49:17 2001 +++ squid/src/HttpHeader.c Sat Apr 6 16:45:59 2002 @@ -124,6 +124,10 @@ {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr}, {"X-Request-URI", HDR_X_REQUEST_URI, ftStr}, {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr}, + {"Negotiate", HDR_NEGOTIATE, ftStr}, +#if X_ACCELERATOR_VARY + {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr}, +#endif {"Other:", HDR_OTHER, ftStr} /* ':' will not allow matches */ }; static HttpHeaderFieldInfo *Headers = NULL; @@ -154,6 +158,9 @@ HDR_AUTHENTICATION_INFO, HDR_PROXY_AUTHENTICATION_INFO, /* HDR_EXPECT, HDR_TE, HDR_TRAILER */ +#if X_ACCELERATOR_VARY + HDR_X_ACCELERATOR_VARY, +#endif HDR_X_FORWARDED_FOR }; @@ -187,6 +194,9 @@ HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE, HDR_X_CACHE_LOOKUP, HDR_X_REQUEST_URI, +#if X_ACCELERATOR_VARY + HDR_X_ACCELERATOR_VARY, +#endif HDR_X_SQUID_ERROR }; @@ -606,6 +616,51 @@ return s; } +/* return a string or list of entries with the same id separated by ',' and ws */ +String +httpHeaderGetStrOrList(const HttpHeader * hdr, http_hdr_type id) +{ + HttpHeaderEntry *e; + + if (CBIT_TEST(ListHeadersMask, id)) + return httpHeaderGetList(hdr, id); + if ((e = httpHeaderFindEntry(hdr, id))) { + String s; + stringInit(&s, strBuf(e->value)); + return s; + } + return StringNull; +} + +/* + * returns a pointer to a specified entry if any + * note that we return one entry so it does not make much sense to ask for + * "list" headers + */ +String +httpHeaderGetByName(const HttpHeader * hdr, const char *name) +{ + http_hdr_type id; + HttpHeaderPos pos = HttpHeaderInitPos; + HttpHeaderEntry *e; + String result = StringNull; + + assert(hdr); + assert(name); + + /* First try the quick path */ + id = httpHeaderIdByNameDef(name, strlen(name)); + if (id != -1) + return httpHeaderGetStrOrList(hdr, id); + + /* Sorry, an unknown header name. Do linear search */ + while ((e = httpHeaderGetEntry(hdr, &pos))) { + if (e->id == HDR_OTHER && strCaseCmp(e->name, name) == 0) { + strListAdd(&result, strBuf(e->value), ','); + } + } + return result; +} /* test if a field is present */ int Index: squid/src/HttpReply.c diff -u squid/src/HttpReply.c:1.3.4.3 squid/src/HttpReply.c:1.3.4.3.2.1 --- squid/src/HttpReply.c:1.3.4.3 Mon Jan 15 14:49:17 2001 +++ squid/src/HttpReply.c Thu Jan 18 07:29:04 2001 @@ -316,6 +316,13 @@ return squid_curtime; } } + if (Config.onoff.vary_ignore_expire && + httpHeaderHas(&rep->header, HDR_VARY)) { + const time_t d = httpHeaderGetTime(&rep->header, HDR_DATE); + const time_t e = httpHeaderGetTime(&rep->header, HDR_EXPIRES); + if (d == e) + return -1; + } if (httpHeaderHas(&rep->header, HDR_EXPIRES)) { const time_t e = httpHeaderGetTime(&rep->header, HDR_EXPIRES); /* Index: squid/src/HttpRequest.c diff -u squid/src/HttpRequest.c:1.3.4.1 squid/src/HttpRequest.c:1.3.4.1.2.1 --- squid/src/HttpRequest.c:1.3.4.1 Mon Jan 15 14:49:17 2001 +++ squid/src/HttpRequest.c Tue Jan 16 09:07:10 2001 @@ -57,6 +57,7 @@ assert(req); safe_free(req->body); safe_free(req->canonical); + safe_free(req->vary_headers); stringClean(&req->urlpath); httpHeaderClean(&req->header); if (req->cache_control) Index: squid/src/cf.data.pre diff -u squid/src/cf.data.pre:1.4.4.7 squid/src/cf.data.pre:1.4.4.3.2.2.2.2 --- squid/src/cf.data.pre:1.4.4.7 Fri Apr 13 12:04:56 2001 +++ squid/src/cf.data.pre Fri Apr 13 12:10:53 2001 @@ -3358,5 +3358,20 @@ force fresh content. DOC_END +NAME: vary_ignore_expire +COMMENT: on|off +TYPE: onoff +LOC: Config.onoff.vary_ignore_expire +DEFAULT: off +DOC_START + Many HTTP servers supporting Vary gives such objects + immediate expiry time with no cache-control header + when requested by a HTTP/1.0 client. This option + enables Squid to ignore such expiry times until + HTTP/1.1 is fully implemented. + WARNING: This may eventually cause some varying + objects not intended for caching to get cached. +DOC_END + EOF Index: squid/src/client_side.c diff -u squid/src/client_side.c:1.5.2.8 squid/src/client_side.c:1.5.2.5.2.9.2.2 --- squid/src/client_side.c:1.5.2.8 Fri May 4 06:15:50 2001 +++ squid/src/client_side.c Thu May 31 08:24:16 2001 @@ -577,7 +577,7 @@ StoreEntry *entry; ErrorState *err = NULL; HttpReply *r; - http_status status; + http_status status = HTTP_NOT_FOUND; http_version_t version; debug(33, 3) ("Config2.onoff.enable_purge = %d\n", Config2.onoff.enable_purge); if (!Config2.onoff.enable_purge) { @@ -589,17 +589,66 @@ errorAppendEntry(http->entry, err); return; } - http->log_type = LOG_TCP_MISS; - /* Release both IP and object cache entries */ + /* Release both IP cache */ ipcacheInvalidate(http->request->host); - if ((entry = storeGetPublic(http->uri, METHOD_GET)) == NULL) { - status = HTTP_NOT_FOUND; - } else { + + if (!http->flags.purging) { + /* Try to find a base entry */ + http->flags.purging = 1; + entry = storeGetPublicByRequestMethod(http->request, METHOD_GET); + if (!entry) + entry = storeGetPublicByRequestMethod(http->request, METHOD_HEAD); + if (entry) { + /* Swap in the metadata */ + http->entry = entry; + storeLockObject(http->entry); + storeCreateMemObject(http->entry, http->uri, http->log_uri); + http->entry->mem_obj->method = http->request->method; + http->sc = storeClientListAdd(http->entry, http); + http->log_type = LOG_TCP_HIT; + storeClientCopy(http->sc, http->entry, + http->out.offset, + http->out.offset, + CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), + clientCacheHit, + http); + return; + } + } + http->log_type = LOG_TCP_MISS; + /* Release the cached URI */ + entry = storeGetPublicByRequestMethod(http->request, METHOD_GET); + if (entry) { + debug(33, 4) ("clientPurgeRequest: GET '%s'\n", + storeUrl(entry)); storeRelease(entry); status = HTTP_OK; } - debug(33, 4) ("clientPurgeRequest: Not modified '%s'\n", - storeUrl(entry)); + entry = storeGetPublicByRequestMethod(http->request, METHOD_HEAD); + if (entry) { + debug(33, 4) ("clientPurgeRequest: HEAD '%s'\n", + storeUrl(entry)); + storeRelease(entry); + status = HTTP_OK; + } + /* And for Vary, release the base URI if none of the headers was included in the request */ + if (http->request->vary_headers && !strstr(http->request->vary_headers,"=")) { + entry = storeGetPublic(urlCanonical(http->request), METHOD_GET); + if (entry) { + debug(33, 4) ("clientPurgeRequest: Vary GET '%s'\n", + storeUrl(entry)); + storeRelease(entry); + status = HTTP_OK; + } + entry = storeGetPublic(urlCanonical(http->request), METHOD_HEAD); + if (entry) { + debug(33, 4) ("clientPurgeRequest: Vary HEAD '%s'\n", + storeUrl(entry)); + storeRelease(entry); + status = HTTP_OK; + } + } /* * Make a new entry to hold the reply to be written * to the client. @@ -1390,6 +1439,45 @@ * Got the headers, now grok them */ assert(http->log_type == LOG_TCP_HIT); + switch (varyEvaluateMatch(e, r)) { + case VARY_NONE: + /* No variance detected. Continue as normal */ + break; + case VARY_MATCH: + /* This is the correct entity for this request. Continue */ + debug(33, 2) ("clientProcessHit: Vary MATCH!\n"); + break; + case VARY_OTHER: + /* This is not the correct entity for this request. We need + * to requery the cache. + */ + memFree(buf, MEM_CLIENT_SOCK_BUF); + http->entry = NULL; + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + /* Note: varyEvalyateMatch updates the request with vary information + * so we only get here once. (it also takes care of cancelling loops) + */ + debug(33, 2) ("clientProcessHit: Vary detected!\n"); + clientProcessRequest(http); + return; + case VARY_CANCEL: + /* varyEvaluateMatch found a object loop. Process as miss */ + debug(33, 1) ("clientProcessHit: Vary object loop!\n"); + memFree(buf, MEM_CLIENT_SOCK_BUF); + clientProcessMiss(http); + return; + } + if (r->method == METHOD_PURGE) { + memFree(buf, MEM_CLIENT_SOCK_BUF); + http->entry = NULL; + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + clientPurgeRequest(http); + return; + } if (checkNegativeHit(e)) { http->log_type = LOG_TCP_NEGATIVE_HIT; clientSendMoreData(data, buf, size); @@ -2019,11 +2107,7 @@ { request_t *r = http->request; StoreEntry *e; - e = http->entry = storeGetPublic(http->uri, r->method); - if (r->method == METHOD_HEAD && e == NULL) { - /* We can generate a HEAD reply from a cached GET object */ - e = http->entry = storeGetPublic(http->uri, METHOD_GET); - } + e = http->entry = storeGetPublicByRequest(r); /* Release negatively cached IP-cache entries on reload */ if (r->flags.nocache) ipcacheInvalidate(r->host); @@ -2194,6 +2278,10 @@ storeUnlockObject(http->entry); http->entry = NULL; } + if (r->method == METHOD_PURGE) { + clientPurgeRequest(http); + return; + } if (clientOnlyIfCached(http)) { clientProcessOnlyIfCachedMiss(http); return; @@ -3098,3 +3186,54 @@ } NHttpSockets = 0; } + +int +varyEvaluateMatch(StoreEntry *entry, request_t *request) +{ + const char *vary = request->vary_headers; + int has_vary = httpHeaderHas(&entry->mem_obj->reply->header, HDR_VARY); +#if X_ACCELERATOR_VARY + has_vary |= httpHeaderHas(&entry->mem_obj->reply->header, HDR_X_ACCELERATOR_VARY); +#endif + if (!has_vary || !entry->mem_obj->vary_headers) { + if (vary) { + /* Oops... something odd is going on here.. */ + debug(33, 1) ("varyEvaluateMatch: Oops. Not a Vary object on second attempt, '%s' '%s'\n", + entry->mem_obj->url, vary); + safe_free(request->vary_headers); + return VARY_CANCEL; + } + if (!has_vary) { + /* This is not a varying object */ + return VARY_NONE; + } + /* virtual "vary" object found. Calculate the vary key and + * continue the search + */ + vary = request->vary_headers = xstrdup(httpMakeVaryMark(request, entry->mem_obj->reply)); + if (vary) + return VARY_OTHER; + else { + /* Ouch.. we cannot handle this kind of variance */ + /* XXX This cannot really happen, but just to be complete */ + return VARY_CANCEL; + } + } else { + if (!vary) + vary = request->vary_headers = xstrdup(httpMakeVaryMark(request, entry->mem_obj->reply)); + if (!vary) { + /* Ouch.. we cannot handle this kind of variance */ + /* XXX This cannot really happen, but just to be complete */ + return VARY_CANCEL; + } else if (strcmp(vary, entry->mem_obj->vary_headers) == 0) { + return VARY_MATCH; + } else { + /* Oops.. we have already been here and still haven't + * found the requested variant. Bail out + */ + debug(33, 1) ("varyEvaluateMatch: Oops. Not a Vary match on second attempt, '%s' '%s'\n", + entry->mem_obj->url, vary); + return VARY_CANCEL; + } + } +} Index: squid/src/enums.h diff -u squid/src/enums.h:1.5.2.2 squid/src/enums.h:1.5.2.2.2.2 --- squid/src/enums.h:1.5.2.2 Mon Jan 15 14:49:19 2001 +++ squid/src/enums.h Thu Feb 1 05:59:32 2001 @@ -227,6 +227,10 @@ HDR_X_FORWARDED_FOR, HDR_X_REQUEST_URI, /* appended if ADD_X_REQUEST_URI is #defined */ HDR_X_SQUID_ERROR, + HDR_NEGOTIATE, +#if X_ACCELERATOR_VARY + HDR_X_ACCELERATOR_VARY, +#endif HDR_OTHER, HDR_ENUM_END } http_hdr_type; @@ -622,6 +626,7 @@ STORE_META_STD, /* standard metadata */ STORE_META_HITMETERING, /* reserved for hit metering */ STORE_META_VALID, + STORE_META_VARY_HEADERS, /* Stores Vary request headers */ STORE_META_END }; @@ -675,3 +680,16 @@ NETDB_EX_RTT, NETDB_EX_HOPS }; + +/* + * Return codes from checkVary(request) + */ +enum { + VARY_NONE, + VARY_MATCH, + VARY_OTHER, + VARY_CANCEL +}; + + + Index: squid/src/ftp.c diff -u squid/src/ftp.c:1.4.2.5 squid/src/ftp.c:1.4.2.4.2.3.2.1 --- squid/src/ftp.c:1.4.2.5 Mon Oct 1 15:03:58 2001 +++ squid/src/ftp.c Tue Oct 16 14:57:08 2001 @@ -1043,8 +1043,7 @@ const char *url = storeUrl(entry); FtpStateData *ftpState = xcalloc(1, sizeof(FtpStateData)); HttpReply *reply; - StoreEntry *pe = NULL; - const cache_key *key = NULL; + cbdataAdd(ftpState, cbdataXfree, 0); debug(9, 3) ("ftpStart: '%s'\n", url); statCounter.server.all.requests++; @@ -1075,10 +1074,6 @@ snprintf(realm, 8192, "ftp %s port %d", ftpState->user, request->port); } - /* eject any old cached object */ - key = storeKeyPublic(entry->mem_obj->url, entry->mem_obj->method); - if ((pe = storeGet(key)) != NULL) - storeRelease(pe); /* create reply */ reply = entry->mem_obj->reply; assert(reply != NULL); @@ -2468,9 +2463,9 @@ const char *filename = NULL; const char *t = NULL; StoreEntry *e = ftpState->entry; - StoreEntry *pe = NULL; http_reply *reply = e->mem_obj->reply; http_version_t version; + if (ftpState->flags.http_header_sent) return; ftpState->flags.http_header_sent = 1; @@ -2521,12 +2516,8 @@ storeTimestampsSet(e); if (ftpState->flags.authenticated) { /* - * Authenticated requests can't be cached. Eject any old cached - * object + * Authenticated requests can't be cached. */ - pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method); - if (pe) - storeRelease(pe); storeRelease(e); } else if (EBIT_TEST(e->flags, ENTRY_CACHABLE) && !ftpState->restarted_offset) { storeSetPublicKey(e); Index: squid/src/htcp.c diff -u squid/src/htcp.c:1.4.4.3 squid/src/htcp.c:1.4.4.2.2.1.2.1 --- squid/src/htcp.c:1.4.4.3 Fri Apr 13 12:04:56 2001 +++ squid/src/htcp.c Fri Apr 13 12:10:53 2001 @@ -614,16 +614,8 @@ { request_t *request; method_t m = urlParseMethod(s->method); - StoreEntry *e = storeGetPublic(s->uri, m); + StoreEntry *e = NULL, *hit = NULL; char *blk_end; - if (NULL == e) { - debug(31, 3) ("htcpCheckHit: NO; public object not found\n"); - return NULL; - } - if (!storeEntryValidToSend(e)) { - debug(31, 3) ("htcpCheckHit: NO; entry not valid to send\n"); - return NULL; - } request = urlParse(m, s->uri); if (NULL == request) { debug(31, 3) ("htcpCheckHit: NO; failed to parse URL\n"); @@ -632,15 +624,26 @@ blk_end = s->req_hdrs + strlen(s->req_hdrs); if (!httpHeaderParse(&request->header, s->req_hdrs, blk_end)) { debug(31, 3) ("htcpCheckHit: NO; failed to parse request headers\n"); - e = NULL; - } else if (refreshCheckHTCP(e, request)) { + goto miss; + } + e = storeGetPublicByRequest(request); + if (NULL == e) { + debug(31, 3) ("htcpCheckHit: NO; public object not found\n"); + goto miss; + } + if (!storeEntryValidToSend(e)) { + debug(31, 3) ("htcpCheckHit: NO; entry not valid to send\n"); + goto miss; + } + if (refreshCheckHTCP(e, request)) { debug(31, 3) ("htcpCheckHit: NO; cached response is stale\n"); - e = NULL; - } else { - debug(31, 3) ("htcpCheckHit: YES!?\n"); + goto miss; } + debug(31, 3) ("htcpCheckHit: YES!?\n"); + hit = e; +miss: requestDestroy(request); - return e; + return hit; } static void Index: squid/src/http.c diff -u squid/src/http.c:1.4.4.3 squid/src/http.c:1.4.4.2.2.9.2.3 --- squid/src/http.c:1.4.4.3 Thu Jul 26 17:33:32 2001 +++ squid/src/http.c Tue Oct 16 14:57:08 2001 @@ -168,7 +168,11 @@ if (!remove && !forbidden) return; assert(e->mem_obj); - if ((pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method)) != NULL) { + if (e->mem_obj->request) + pe = storeGetPublicByRequest(e->mem_obj->request); + else + pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method); + if (pe != NULL) { assert(e != pe); storeRelease(pe); } @@ -176,7 +180,11 @@ * Also remove any cached HEAD response in case the object has * changed. */ - if ((pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD)) != NULL) { + if (e->mem_obj->request) + pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_HEAD); + else + pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD); + if (pe != NULL) { assert(e != pe); storeRelease(pe); } @@ -194,7 +202,11 @@ * Remove any cached GET object if it is beleived that the * object may have changed as a result of other methods */ - if ((pe = storeGetPublic(e->mem_obj->url, METHOD_GET)) != NULL) { + if (e->mem_obj->request) + pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_GET); + else + pe = storeGetPublic(e->mem_obj->url, METHOD_GET); + if (pe != NULL) { assert(e != pe); storeRelease(pe); } @@ -224,12 +236,6 @@ if (!EBIT_TEST(cc_mask, CC_PUBLIC)) return 0; } - /* - * We don't properly deal with Vary features yet, so we can't - * cache these - */ - if (httpHeaderHas(hdr, HDR_VARY)) - return 0; /* Pragma: no-cache in _replies_ is not documented in HTTP, * but servers like "Active Imaging Webcast/2.0" sure do use it */ if (httpHeaderHas(hdr, HDR_PRAGMA)) { @@ -314,6 +320,65 @@ /* NOTREACHED */ } +/* + * For Vary, store the relevant request headers as + * virtual headers in the reply + * Returns false if the variance cannot be stored + */ +const char * +httpMakeVaryMark(request_t * request, HttpReply *reply) +{ + int ok = 1; + String vary, hdr; + const char *pos = NULL; + const char *item; + const char *value; + int ilen; + static String vstr = {0,0,NULL}; + + stringClean(&vstr); + vary = httpHeaderGetList(&reply->header, HDR_VARY); + while (strListGetItem(&vary, ',', &item, &ilen, &pos)) { + char *name = xmalloc(ilen+1); + xstrncpy(name, item, ilen+1); + Tolower(name); + strListAdd(&vstr, name, ','); + hdr = httpHeaderGetByName(&request->header, name); + safe_free(name); + value = strBuf(hdr); + if (value) { + value = rfc1738_escape(value); + stringAppend(&vstr, "=\"", 2); + stringAppend(&vstr, value, strlen(value)); + stringAppend(&vstr, "\"", 1); + } + stringClean(&hdr); + } + stringClean(&vary); +#if X_ACCELERATOR_VARY + vary = httpHeaderGetList(&reply->header, HDR_X_ACCELERATOR_VARY); + while (strListGetItem(&vary, ',', &item, &ilen, &pos)) { + char *name = xmalloc(ilen+1); + xstrncpy(name, item, ilen+1); + Tolower(name); + strListAdd(&vstr, name, ','); + hdr = httpHeaderGetByName(&request->header, name); + safe_free(name); + value = strBuf(hdr); + if (value) { + value = rfc1738_escape(value); + stringAppend(&vstr, "=\"", 2); + stringAppend(&vstr, value, strlen(value)); + stringAppend(&vstr, "\"", 1); + } + stringClean(&hdr); + } + stringClean(&vary); +#endif + debug(11, 0) ("httpMakeVaryMark: %d / %s\n", ok, strBuf(vstr)); + return strBuf(vstr); +} + /* rewrite this later using new interfaces @?@ */ void httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) @@ -366,7 +431,22 @@ httpMaybeRemovePublic(entry, reply->sline.status); switch (httpCachableReply(httpState)) { case 1: - httpMakePublic(entry); + if (httpHeaderHas(&reply->header, HDR_VARY) +#if X_ACCELERATOR_VARY + || httpHeaderHas(&reply->header, HDR_X_ACCELERATOR_VARY) +#endif + ) { + const char *vary = httpMakeVaryMark(httpState->orig_request, reply); + if (vary) { + entry->mem_obj->vary_headers = xstrdup(vary); + /* Kill the old base object if a change in variance is detected */ + httpMakePublic(entry); + } else { + httpMakePrivate(entry); + } + } else { + httpMakePublic(entry); + } break; case 0: httpMakePrivate(entry); Index: squid/src/neighbors.c diff -u squid/src/neighbors.c:1.5.2.4 squid/src/neighbors.c:1.5.2.2.2.1.2.2 --- squid/src/neighbors.c:1.5.2.4 Thu Jul 26 17:33:32 2001 +++ squid/src/neighbors.c Tue Oct 16 14:57:08 2001 @@ -559,10 +559,10 @@ /* lookup the digest of a given peer */ lookup_t -peerDigestLookup(peer * p, request_t * request, StoreEntry * entry) +peerDigestLookup(peer * p, request_t * request) { #if USE_CACHE_DIGESTS - const cache_key *key = request ? storeKeyPublic(storeUrl(entry), request->method) : NULL; + const cache_key *key = request ? storeKeyPublicByRequest(request) : NULL; assert(p); assert(request); debug(15, 5) ("peerDigestLookup: peer %s\n", p->host); @@ -598,7 +598,7 @@ /* select best peer based on cache digests */ peer * -neighborsDigestSelect(request_t * request, StoreEntry * entry) +neighborsDigestSelect(request_t * request) { peer *best_p = NULL; #if USE_CACHE_DIGESTS @@ -611,14 +611,14 @@ int i; if (!request->flags.hierarchical) return NULL; - key = storeKeyPublic(storeUrl(entry), request->method); + key = storeKeyPublicByRequest(request); for (i = 0, p = first_ping; i++ < Config.npeers; p = p->next) { lookup_t lookup; if (!p) p = Config.peers; if (i == 1) first_ping = p; - lookup = peerDigestLookup(p, request, entry); + lookup = peerDigestLookup(p, request); if (lookup == LOOKUP_NONE) continue; choice_count++; Index: squid/src/peer_digest.c diff -u squid/src/peer_digest.c:1.3.4.1 squid/src/peer_digest.c:1.3.4.1.2.1 --- squid/src/peer_digest.c:1.3.4.1 Mon Jan 15 14:49:19 2001 +++ squid/src/peer_digest.c Wed Jan 17 03:43:22 2001 @@ -281,10 +281,10 @@ url = internalRemoteUri(p->host, p->http_port, "/squid-internal-periodic/", StoreDigestFileName); - key = storeKeyPublic(url, METHOD_GET); - debug(72, 2) ("peerDigestRequest: %s key: %s\n", url, storeKeyText(key)); req = urlParse(METHOD_GET, url); assert(req); + key = storeKeyPublicByRequest(req); + debug(72, 2) ("peerDigestRequest: %s key: %s\n", url, storeKeyText(key)); /* add custom headers */ assert(!req->header.len); Index: squid/src/peer_select.c diff -u squid/src/peer_select.c:1.3.4.2 squid/src/peer_select.c:1.3.4.2.2.1 --- squid/src/peer_select.c:1.3.4.2 Mon Jan 15 14:49:19 2001 +++ squid/src/peer_select.c Wed Jan 17 03:43:22 2001 @@ -326,7 +326,7 @@ return; } #if USE_CACHE_DIGESTS - if ((p = neighborsDigestSelect(request, entry))) { + if ((p = neighborsDigestSelect(request))) { if (neighborType(p, request) == PEER_PARENT) code = CD_PARENT_HIT; else Index: squid/src/protos.h diff -u squid/src/protos.h:1.4.4.7 squid/src/protos.h:1.4.4.4.2.4.2.3 --- squid/src/protos.h:1.4.4.7 Sun Sep 9 13:18:35 2001 +++ squid/src/protos.h Tue Oct 16 14:57:08 2001 @@ -285,6 +285,7 @@ extern int httpAnonHdrDenied(http_hdr_type hdr_id); extern void httpBuildRequestHeader(request_t *, request_t *, StoreEntry *, HttpHeader *, int, http_state_flags); extern void httpBuildVersion(http_version_t * version, unsigned int major, unsigned int minor); +extern const char *httpMakeVaryMark(request_t *request, HttpReply *reply); /* ETag */ extern int etagParseInit(ETag * etag, const char *str); @@ -421,6 +422,8 @@ extern const char *httpHeaderGetLastStr(const HttpHeader * hdr, http_hdr_type id); extern const char *httpHeaderGetAuth(const HttpHeader * hdr, http_hdr_type id, const char *authScheme); extern String httpHeaderGetList(const HttpHeader * hdr, http_hdr_type id); +extern String httpHeaderGetStrOrList(const HttpHeader * hdr, http_hdr_type id); +extern String httpHeaderGetByName(const HttpHeader * hdr, const char *name); extern int httpHeaderDelByName(HttpHeader * hdr, const char *name); extern int httpHeaderDelById(HttpHeader * hdr, http_hdr_type id); extern void httpHeaderDelAt(HttpHeader * hdr, HttpHeaderPos pos); @@ -628,8 +631,8 @@ extern peer *getRoundRobinParent(request_t * request); EVH peerClearRR; extern peer *getAnyParent(request_t * request); -extern lookup_t peerDigestLookup(peer * p, request_t * request, StoreEntry * entry); -extern peer *neighborsDigestSelect(request_t * request, StoreEntry * entry); +extern lookup_t peerDigestLookup(peer * p, request_t * request); +extern peer *neighborsDigestSelect(request_t * request); extern void peerNoteDigestLookup(request_t * request, peer * p, lookup_t lookup); extern void peerNoteDigestGone(peer * p); extern int neighborUp(const peer * e); @@ -810,6 +813,8 @@ extern StoreEntry *new_StoreEntry(int, const char *, const char *); extern StoreEntry *storeGet(const cache_key *); extern StoreEntry *storeGetPublic(const char *uri, const method_t method); +extern StoreEntry *storeGetPublicByRequest(request_t *request); +extern StoreEntry *storeGetPublicByRequestMethod(request_t *request, const method_t method); extern StoreEntry *storeCreateEntry(const char *, const char *, request_flags, method_t); extern void storeSetPublicKey(StoreEntry *); extern void storeComplete(StoreEntry *); @@ -896,6 +901,8 @@ extern const cache_key *storeKeyScan(const char *); extern const char *storeKeyText(const cache_key *); extern const cache_key *storeKeyPublic(const char *, const method_t); +extern const cache_key *storeKeyPublicByRequest(request_t *); +extern const cache_key *storeKeyPublicByRequestMethod(request_t *, const method_t); extern const cache_key *storeKeyPrivate(const char *, method_t, int); extern int storeKeyHashBuckets(int); extern int storeKeyNull(const cache_key *); @@ -1222,3 +1229,7 @@ #if URL_CHECKSUM_DEBUG extern unsigned int url_checksum(const char *url); #endif + +/* Vary support functions */ +int varyEvaluateMatch(StoreEntry *entry, request_t *req); + Index: squid/src/store.c diff -u squid/src/store.c:1.4.4.5 squid/src/store.c:1.4.4.3.2.8.2.2 --- squid/src/store.c:1.4.4.5 Thu Mar 29 04:02:12 2001 +++ squid/src/store.c Thu Mar 29 04:25:58 2001 @@ -166,6 +166,7 @@ ctx_exit(ctx); /* must exit before we free mem->url */ safe_free(mem->url); safe_free(mem->log_url); /* XXX account log_url */ + safe_free((void *)mem->vary_headers); memFree(mem, MEM_MEMOBJECT); } @@ -325,6 +326,22 @@ return storeGet(storeKeyPublic(uri, method)); } +StoreEntry * +storeGetPublicByRequestMethod(request_t *req, const method_t method) +{ + return storeGet(storeKeyPublicByRequestMethod(req, method)); +} + +StoreEntry * +storeGetPublicByRequest(request_t *req) +{ + StoreEntry *e = storeGetPublicByRequestMethod(req, req->method); + if (e == NULL && req->method == METHOD_HEAD) + /* We can generate a HEAD reply from a cached GET object */ + e = storeGetPublicByRequestMethod(req, METHOD_GET); + return e; +} + static int getKeyCounter(void) { @@ -382,12 +399,64 @@ e->hash.key, mem->url); #endif assert(!EBIT_TEST(e->flags, RELEASE_REQUEST)); - newkey = storeKeyPublic(mem->url, mem->method); + if (mem->request) { + StoreEntry *pe; + request_t *request = mem->request; + if (!mem->vary_headers) { + /* First handle the case where the object no longer varies */ + safe_free(request->vary_headers); + } else { + if (request->vary_headers && strcmp(request->vary_headers, mem->vary_headers) != 0) { + /* Oops.. the variance has changed. Kill the base object + * to record the new variance key + */ + safe_free(request->vary_headers); /* free old "bad" variance key */ + pe = storeGetPublic(mem->url, mem->method); + if (pe) + storeRelease(pe); + } + /* Make sure the request knows the variance status */ + if (!request->vary_headers) + request->vary_headers = xstrdup(httpMakeVaryMark(request, mem->reply)); + } + if (mem->vary_headers && !storeGetPublic(mem->url, mem->method)) { + /* Create "vary" base object */ + http_version_t version; + String vary; + pe = storeCreateEntry(mem->url, mem->log_url, request->flags, request->method); + httpBuildVersion(&version, 1, 0); + httpReplySetHeaders(pe->mem_obj->reply, version, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); + vary = httpHeaderGetList(&mem->reply->header, HDR_VARY); + if (strBuf(vary)) { + httpHeaderPutStr(&pe->mem_obj->reply->header, HDR_VARY, strBuf(vary)); + stringClean(&vary); + } +#if X_ACCELERATOR_VARY + vary = httpHeaderGetList(&mem->reply->header, HDR_X_ACCELERATOR_VARY); + if (strBuf(vary)) { + httpHeaderPutStr(&pe->mem_obj->reply->header, HDR_X_ACCELERATOR_VARY, strBuf(vary)); + stringClean(&vary); + } +#endif + storeSetPublicKey(pe); + httpReplySwapOut(pe->mem_obj->reply, pe); + storeBufferFlush(pe); + storeTimestampsSet(pe); + storeComplete(pe); + storeUnlockObject(pe); + } + newkey = storeKeyPublicByRequest(mem->request); + } + else + newkey = storeKeyPublic(mem->url, mem->method); if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); storeSetPrivateKey(e2); storeRelease(e2); - newkey = storeKeyPublic(mem->url, mem->method); + if (mem->request) + newkey = storeKeyPublicByRequest(mem->request); + else + newkey = storeKeyPublic(mem->url, mem->method); } if (e->hash.key) storeHashDelete(e); Index: squid/src/store_client.c diff -u squid/src/store_client.c:1.4.4.1 squid/src/store_client.c:1.4.4.1.2.1 --- squid/src/store_client.c:1.4.4.1 Mon Jan 15 14:49:20 2001 +++ squid/src/store_client.c Tue Jan 16 09:07:10 2001 @@ -434,6 +434,15 @@ break; case STORE_META_STD: break; + case STORE_META_VARY_HEADERS: + if (mem->vary_headers) { + if (strcmp(mem->vary_headers, t->value) != 0) + swap_object_ok = 0; + } else { + /* Assume the object is OK.. remember the vary request headers */ + mem->vary_headers = xstrdup(t->value); + } + break; default: debug(20, 1) ("WARNING: got unused STORE_META type %d\n", t->type); break; Index: squid/src/store_key_md5.c diff -u squid/src/store_key_md5.c:1.4.4.1 squid/src/store_key_md5.c:1.4.4.1.2.2 --- squid/src/store_key_md5.c:1.4.4.1 Mon Jan 15 14:49:20 2001 +++ squid/src/store_key_md5.c Wed Jan 17 03:56:04 2001 @@ -120,6 +120,28 @@ return digest; } +const cache_key * +storeKeyPublicByRequest(request_t *request) +{ + return storeKeyPublicByRequestMethod(request, request->method); +} + +const cache_key * +storeKeyPublicByRequestMethod(request_t *request, const method_t method) +{ + static cache_key digest[MD5_DIGEST_CHARS]; + unsigned char m = (unsigned char) method; + const char *url = urlCanonical(request); + MD5_CTX M; + MD5Init(&M); + MD5Update(&M, &m, sizeof(m)); + MD5Update(&M, (unsigned char *) url, strlen(url)); + if (request->vary_headers) + MD5Update(&M, (unsigned char *) request->vary_headers, strlen(request->vary_headers)); + MD5Final(digest, &M); + return digest; +} + cache_key * storeKeyDup(const cache_key * key) { Index: squid/src/store_swapmeta.c diff -u squid/src/store_swapmeta.c:1.4.4.2 squid/src/store_swapmeta.c:1.4.4.1.2.2 --- squid/src/store_swapmeta.c:1.4.4.2 Wed Feb 7 11:12:56 2001 +++ squid/src/store_swapmeta.c Wed Feb 7 11:13:53 2001 @@ -67,6 +67,7 @@ tlv *TLV = NULL; /* we'll return this */ tlv **T = &TLV; const char *url; + const char *vary; assert(e->mem_obj != NULL); assert(e->swap_status == SWAPOUT_WRITING); url = storeUrl(e); @@ -74,6 +75,9 @@ T = storeSwapTLVAdd(STORE_META_KEY, e->hash.key, MD5_DIGEST_CHARS, T); T = storeSwapTLVAdd(STORE_META_STD, &e->timestamp, STORE_HDR_METASIZE, T); T = storeSwapTLVAdd(STORE_META_URL, url, strlen(url) + 1, T); + vary = e->mem_obj->vary_headers; + if (vary) + T = storeSwapTLVAdd(STORE_META_VARY_HEADERS, vary, strlen(vary) + 1, T); return TLV; } Index: squid/src/structs.h diff -u squid/src/structs.h:1.6.2.9 squid/src/structs.h:1.6.2.5.2.3.2.2 --- squid/src/structs.h:1.6.2.9 Fri Apr 13 12:04:56 2001 +++ squid/src/structs.h Fri Apr 13 12:10:53 2001 @@ -432,6 +432,7 @@ int log_ip_on_direct; int authenticateIpTTLStrict; int ie_refresh; + int vary_ignore_expire; int pipeline_prefetch; } onoff; acl *aclList; @@ -898,6 +899,7 @@ unsigned int accel:1; unsigned int internal:1; unsigned int done_copying:1; + unsigned int purging:1; } flags; struct { http_status status; @@ -1311,6 +1313,7 @@ #if URL_CHECKSUM_DEBUG unsigned int chksum; #endif + const char *vary_headers; }; struct _StoreEntry { @@ -1464,6 +1467,7 @@ err_type err_type; char *peer_login; /* Configured peer login:password */ time_t lastmod; /* Used on refreshes */ + char *vary_headers; /* Used when varying entities are detected. Changes how the store key is calculated */ }; struct _cachemgr_passwd { squid-vary-2_4-s2_4.new squid-vary-2_4-s2_4 differ: char 68, line 2