--------------------- PatchSet 1091 Date: 2004/10/17 22:04:53 Author: hno Branch: collapsed_forwarding Tag: (none) Log: Initial port of collapsed_forwarding-2.5 to Squid-3. Many thanks to Swelltech for contributing the port. Members: src/MemObject.cc:1.13->1.13.2.1 src/MemObject.h:1.9->1.9.2.1 src/cf.data.pre:1.55->1.55.2.1 src/client_side_reply.cc:1.52->1.52.2.1 src/http.cc:1.35->1.35.2.1 src/protos.h:1.42->1.42.2.1 src/refresh.cc:1.8->1.8.4.1 src/store.cc:1.26->1.26.2.1 src/structs.h:1.55->1.55.2.1 Index: squid3/src/MemObject.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/MemObject.cc,v retrieving revision 1.13 retrieving revision 1.13.2.1 diff -u -r1.13 -r1.13.2.1 --- squid3/src/MemObject.cc 31 Aug 2004 02:14:26 -0000 1.13 +++ squid3/src/MemObject.cc 17 Oct 2004 22:04:53 -0000 1.13.2.1 @@ -1,6 +1,6 @@ /* - * $Id: MemObject.cc,v 1.13 2004/08/31 02:14:26 squidadm Exp $ + * $Id: MemObject.cc,v 1.13.2.1 2004/10/17 22:04:53 hno Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Robert Collins @@ -110,6 +110,11 @@ #endif + if (ims_entry) { + storeUnlockObject(ims_entry); + ims_entry = NULL; + } + httpReplyDestroy((HttpReply *)_reply); requestUnlink(request); Index: squid3/src/MemObject.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/MemObject.h,v retrieving revision 1.9 retrieving revision 1.9.2.1 diff -u -r1.9 -r1.9.2.1 --- squid3/src/MemObject.h 31 Aug 2004 02:14:26 -0000 1.9 +++ squid3/src/MemObject.h 17 Oct 2004 22:05:04 -0000 1.9.2.1 @@ -1,6 +1,6 @@ /* - * $Id: MemObject.h,v 1.9 2004/08/31 02:14:26 squidadm Exp $ + * $Id: MemObject.h,v 1.9.2.1 2004/10/17 22:05:04 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -136,6 +136,9 @@ const char *vary_headers; + StoreEntry *ims_entry; + time_t refresh_timestamp; + void delayRead(DeferredRead const &); void kickReads(); Index: squid3/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/cf.data.pre,v retrieving revision 1.55 retrieving revision 1.55.2.1 diff -u -r1.55 -r1.55.2.1 --- squid3/src/cf.data.pre 11 Oct 2004 02:13:14 -0000 1.55 +++ squid3/src/cf.data.pre 17 Oct 2004 22:05:04 -0000 1.55.2.1 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.55 2004/10/11 02:13:14 squidadm Exp $ +# $Id: cf.data.pre,v 1.55.2.1 2004/10/17 22:05:04 hno Exp $ # # # SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -2225,6 +2225,32 @@ client requested. (default) DOC_END +NAME: collapsed_forwarding +COMMENT: (on|off) +TYPE: onoff +LOC: Config.onoff.collapsed_forwarding +DEFAULT: off +DOC_START + This option enables multiple requests for the same URI to be + processed as one request. Normally disabled to avoid increased + latency on dynamic content, but there can be benefit from enabling + this in accelerator setups where the web servers are the bottleneck + and reliable and returns mostly cacheable information. +DOC_END + +NAME: refresh_stale_hit +COMMENT: (time) +TYPE: time_t +DEFAULT: 0 seconds +LOC: Config.refresh_stale_window +DOC_START + This option changes the refresh algorithm to allow concurrent + requests while an object is being refreshed to be processed as + cache hits if the object expired less than X seconds ago. Default + is 0 to disable this feature. This option is mostly interesting + in accelerator setups where a few objects is accessed very + frequently. +DOC_END COMMENT_START TIMEOUTS Index: squid3/src/client_side_reply.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/client_side_reply.cc,v retrieving revision 1.52 retrieving revision 1.52.2.1 diff -u -r1.52 -r1.52.2.1 --- squid3/src/client_side_reply.cc 15 Oct 2004 02:12:58 -0000 1.52 +++ squid3/src/client_side_reply.cc 17 Oct 2004 22:05:04 -0000 1.52.2.1 @@ -1,6 +1,6 @@ /* - * $Id: client_side_reply.cc,v 1.52 2004/10/15 02:12:58 squidadm Exp $ + * $Id: client_side_reply.cc,v 1.52.2.1 2004/10/17 22:05:04 hno Exp $ * * DEBUG: section 88 Client-side Reply Routines * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c) @@ -70,6 +70,12 @@ * is still valid */ removeClientStoreReference(&sc, http); + + if (old_entry && old_entry->mem_obj && old_entry->mem_obj->ims_entry && old_entry->mem_obj->ims_entry == http->storeEntry()) { + storeUnlockObject(old_entry->mem_obj->ims_entry); + old_entry->mem_obj->ims_entry = NULL; + } + /* old_entry might still be set if we didn't yet get the reply * code in HandleIMSReply() */ removeStoreReference(&old_sc, &old_entry); @@ -230,6 +236,7 @@ { char *url = http->uri; StoreEntry *entry = NULL; + int hit = 0; debug(88, 3)("clientReplyContext::processExpired: '%s'", http->uri); assert(http->storeEntry()->lastmod >= 0); /* @@ -251,8 +258,40 @@ #endif /* Prepare to make a new temporary request */ saveState(); - entry = storeCreateEntry(url, - http->log_uri, http->request->flags, http->request->method); + + if (http->storeEntry()->mem_obj && http->storeEntry()->mem_obj->ims_entry) { + entry = http->storeEntry()->mem_obj->ims_entry; + debug(88, 5) ("clientProcessExpired: collapsed request\n"); + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + debug(88, 1) ("clientProcessExpired: collapsed request ABORTED!\n"); + entry = NULL; + } else if (http->storeEntry()->mem_obj->refresh_timestamp + 30 < squid_curtime) { + debug(88, 1) ("clientProcessExpired: collapsed request STALE!\n"); + entry = NULL; + } + if (entry) { + storeLockObject(entry); + hit = 1; + } else { + storeUnlockObject(http->storeEntry()->mem_obj->ims_entry); + http->storeEntry()->mem_obj->ims_entry = NULL; + } + } + if (!entry) { + entry = storeCreateEntry(url, + http->log_uri, + http->request->flags, + http->request->method); + if (http->storeEntry()->mem_obj) { + http->storeEntry()->mem_obj->refresh_timestamp = squid_curtime; + if (Config.onoff.collapsed_forwarding) { + debug(88, 1) ("clientProcessExpired: collapsed request forwarding\n"); /* # */ + http->storeEntry()->mem_obj->ims_entry = entry; + storeLockObject(http->storeEntry()->mem_obj->ims_entry); + } + } + } + /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ sc = storeClientListAdd(entry, this); #if DELAY_POOLS @@ -265,7 +304,12 @@ (long int) entry->lastmod); http->storeEntry(entry); assert(http->out.offset == 0); - fwdStart(http->getConn().getRaw() != NULL ? http->getConn()->fd : -1, http->storeEntry(), http->request); + + if (!hit) + fwdStart(http->getConn().getRaw() != NULL ? http->getConn()->fd : -1, http->storeEntry(), http->request); + else + debug(88, 0) ("clientReplyContext::processExpired: HIT2"); + /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) @@ -518,6 +562,11 @@ storeUrl(http->storeEntry()), (long unsigned) result.length); + if (old_entry && old_entry->mem_obj && old_entry->mem_obj->ims_entry) { + storeUnlockObject(old_entry->mem_obj->ims_entry); + old_entry->mem_obj->ims_entry = NULL; + } + if (http->storeEntry() == NULL) return; @@ -667,7 +716,15 @@ if (storeCheckNegativeHit(e)) { http->logType = LOG_TCP_NEGATIVE_HIT; sendMoreData(result); - } else if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) { + return; + } + + if (Config.refresh_stale_window > 0 && e->mem_obj && e->mem_obj->refresh_timestamp + Config.refresh_stale_window > squid_curtime && !refreshCheckHTTPStale(e, r)) { + debug(88, 2) ("clientProcessHit: refresh_stale HIT\n"); + goto hit; + } + + if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) { debug(88, 5) ("clientCacheHit: in refreshCheck() block\n"); /* * We hold a stale copy; it needs to be validated @@ -710,7 +767,12 @@ http->logType = LOG_TCP_MISS; processMiss(); } - } else if (r->flags.ims) { + + return; + } + +hit: + if (r->flags.ims) { /* * Handle If-Modified-Since requests from the client */ @@ -744,20 +806,22 @@ */ triggerInitialStoreRead(); } - } else { - /* - * plain ol' cache hit - */ - if (e->store_status != STORE_OK) - http->logType = LOG_TCP_MISS; - else if (e->mem_status == IN_MEMORY) - http->logType = LOG_TCP_MEM_HIT; - else if (Config.onoff.offline) - http->logType = LOG_TCP_OFFLINE_HIT; + return; + } - sendMoreData(result); - } + /* + * plain ol' cache hit + */ + + if (e->store_status != STORE_OK) + http->logType = LOG_TCP_MISS; + else if (e->mem_status == IN_MEMORY) + http->logType = LOG_TCP_MEM_HIT; + else if (Config.onoff.offline) + http->logType = LOG_TCP_OFFLINE_HIT; + + sendMoreData(result); } /* @@ -784,7 +848,11 @@ storeEntryDump(http->storeEntry(), 1); } - removeClientStoreReference(&sc, http); + /* touch timestamp for refresh_stale_hit */ + if (http->storeEntry()->mem_obj) + http->storeEntry()->mem_obj->refresh_timestamp = squid_curtime; + + removeClientStoreReference(&sc, http); } if (r->method == METHOD_PURGE) { @@ -811,8 +879,23 @@ return; } else { assert(http->out.offset == 0); + createStoreEntry(r->method, r->flags); - triggerInitialStoreRead(); + debug(88, 0) ("clientProcessMiss: createStoreEntry.\n"); + + if (Config.onoff.collapsed_forwarding && r->flags.cachable && !r->flags.need_validation && (METHOD_GET == r->method || METHOD_HEAD == r->method)) + { + /* touch timestamp for refresh_stale_hit */ + if (http->storeEntry()->mem_obj) + http->storeEntry()->mem_obj->refresh_timestamp = squid_curtime; + + storeSetPublicKey(http->storeEntry()); + + debug(88, 0) ("clientProcessMiss: storeSetPublicKey.\n"); + storeEntryDump(http->storeEntry(), 1); + } + + triggerInitialStoreRead(); if (http->redirect.status) { HttpReply *rep = httpReplyCreate(); @@ -1645,7 +1728,12 @@ StoreIOBuffer tempBuffer; storeLockObject(http->storeEntry()); - if (http->storeEntry()->mem_obj == NULL) { + if (http->storeEntry()->store_status == STORE_PENDING && http->storeEntry()->mem_obj) { + if (http->storeEntry()->mem_obj->request) + http->request->hier = http->storeEntry()->mem_obj->request->hier; + }; + + if (http->storeEntry()->mem_obj == NULL) { /* * This if-block exists because we don't want to clobber * a preexiting mem_obj->method value if the mem_obj Index: squid3/src/http.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/http.cc,v retrieving revision 1.35 retrieving revision 1.35.2.1 diff -u -r1.35 -r1.35.2.1 --- squid3/src/http.cc 11 Oct 2004 02:13:14 -0000 1.35 +++ squid3/src/http.cc 17 Oct 2004 22:05:04 -0000 1.35.2.1 @@ -1,6 +1,6 @@ /* - * $Id: http.cc,v 1.35 2004/10/11 02:13:14 squidadm Exp $ + * $Id: http.cc,v 1.35.2.1 2004/10/17 22:05:04 hno Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -683,7 +683,7 @@ /* Check if object is cacheable or not based on reply code */ debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", entry->getReply()->sline.status); - if (neighbors_do_private_keys) + if (neighbors_do_private_keys && !Config.onoff.collapsed_forwarding) httpMaybeRemovePublic(entry, entry->getReply()->sline.status); if (httpHeaderHas(&entry->getReply()->header, HDR_VARY) @@ -1191,16 +1191,18 @@ #endif /* append X-Forwarded-For */ - strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); + if (opt_forwarded_for) { + strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); - if (opt_forwarded_for && orig_request->client_addr.s_addr != no_addr.s_addr) - strListAdd(&strFwd, inet_ntoa(orig_request->client_addr), ','); - else - strListAdd(&strFwd, "unknown", ','); + if (orig_request->client_addr.s_addr != no_addr.s_addr) + strListAdd(&strFwd, inet_ntoa(orig_request->client_addr), ','); + else + strListAdd(&strFwd, "unknown", ','); - httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strFwd.buf()); + httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strFwd.buf()); - strFwd.clean(); + strFwd.clean(); + } /* append Host if not there already */ if (!httpHeaderHas(hdr_out, HDR_HOST)) { @@ -1440,6 +1442,11 @@ break; + case HDR_X_FORWARDED_FOR: + if (!opt_forwarded_for) + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + break; + case HDR_RANGE: case HDR_IF_RANGE: @@ -1454,8 +1461,6 @@ case HDR_CONNECTION: - case HDR_X_FORWARDED_FOR: - case HDR_CACHE_CONTROL: /* append these after the loop if needed */ break; Index: squid3/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/protos.h,v retrieving revision 1.42 retrieving revision 1.42.2.1 diff -u -r1.42 -r1.42.2.1 --- squid3/src/protos.h 31 Aug 2004 02:14:28 -0000 1.42 +++ squid3/src/protos.h 17 Oct 2004 22:05:04 -0000 1.42.2.1 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.42 2004/08/31 02:14:28 squidadm Exp $ + * $Id: protos.h,v 1.42.2.1 2004/10/17 22:05:04 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -619,6 +619,7 @@ extern void refreshAddToList(const char *, int, time_t, int, time_t); extern int refreshIsCachable(const StoreEntry *); extern int refreshCheckHTTP(const StoreEntry *, HttpRequest *); +extern int refreshCheckHTTPStale(const StoreEntry *, HttpRequest *); extern int refreshCheckICP(const StoreEntry *, HttpRequest *); extern int refreshCheckHTCP(const StoreEntry *, HttpRequest *); extern int refreshCheckDigest(const StoreEntry *, time_t delta); Index: squid3/src/refresh.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/refresh.cc,v retrieving revision 1.8 retrieving revision 1.8.4.1 diff -u -r1.8 -r1.8.4.1 --- squid3/src/refresh.cc 11 Aug 2003 02:13:03 -0000 1.8 +++ squid3/src/refresh.cc 17 Oct 2004 22:05:04 -0000 1.8.4.1 @@ -1,6 +1,6 @@ /* - * $Id: refresh.cc,v 1.8 2003/08/11 02:13:03 squidadm Exp $ + * $Id: refresh.cc,v 1.8.4.1 2004/10/17 22:05:04 hno Exp $ * * DEBUG: section 22 Refresh Calculation * AUTHOR: Harvest Derived @@ -439,6 +439,13 @@ } int +refreshCheckHTTPStale(const StoreEntry * entry, HttpRequest * request) +{ + int reason = refreshCheck(entry, request, -Config.refresh_stale_window); + return (reason < 200) ? 0 : 1; +} + +int refreshCheckICP(const StoreEntry * entry, HttpRequest * request) { int reason = refreshCheck(entry, request, 30); Index: squid3/src/store.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/store.cc,v retrieving revision 1.26 retrieving revision 1.26.2.1 diff -u -r1.26 -r1.26.2.1 --- squid3/src/store.cc 31 Aug 2004 02:14:28 -0000 1.26 +++ squid3/src/store.cc 17 Oct 2004 22:05:04 -0000 1.26.2.1 @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.26 2004/08/31 02:14:28 squidadm Exp $ + * $Id: store.cc,v 1.26.2.1 2004/10/17 22:05:04 hno Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -809,6 +809,8 @@ assert(len >= 0); assert(e->store_status == STORE_PENDING); + mem->refresh_timestamp = squid_curtime; + StoreIOBuffer tempBuffer; tempBuffer.data = (char *)buf; tempBuffer.length = len; @@ -1483,6 +1485,11 @@ if (EBIT_TEST(e->flags, ENTRY_ABORTED)) return 0; + /* Entries which seem to have got stuck are not valid to be sent to new clients */ + if ((e->store_status == STORE_PENDING) + && (!e->mem_obj || e->mem_obj->refresh_timestamp + 30 < squid_curtime)) + return 0; + return 1; } Index: squid3/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/structs.h,v retrieving revision 1.55 retrieving revision 1.55.2.1 diff -u -r1.55 -r1.55.2.1 --- squid3/src/structs.h 15 Oct 2004 02:12:58 -0000 1.55 +++ squid3/src/structs.h 17 Oct 2004 22:05:04 -0000 1.55.2.1 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.55 2004/10/15 02:12:58 squidadm Exp $ + * $Id: structs.h,v 1.55.2.1 2004/10/17 22:05:04 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -559,6 +559,8 @@ int check_hostnames; int via; int emailErrData; + + int collapsed_forwarding; } onoff; @@ -730,6 +732,8 @@ ssl_client; #endif + + time_t refresh_stale_window; }; struct _SquidConfig2