This patch is generated from the collapsed_forwarding branch of HEAD in squid3 Fri May 19 00:14:52 2006 GMT See http://devel.squid-cache.org/ Index: squid3/src/MemObject.cc diff -u squid3/src/MemObject.cc:1.20 squid3/src/MemObject.cc:1.13.2.4 --- squid3/src/MemObject.cc:1.20 Fri Feb 17 19:13:25 2006 +++ squid3/src/MemObject.cc Thu May 18 16:22:00 2006 @@ -117,6 +117,11 @@ #endif + if (ims_entry) { + storeUnlockObject(ims_entry); + ims_entry = NULL; + } + HTTPMSGUNLOCK(_reply); HTTPMSGUNLOCK(request); Index: squid3/src/MemObject.h diff -u squid3/src/MemObject.h:1.11 squid3/src/MemObject.h:1.9.2.4 --- squid3/src/MemObject.h:1.11 Mon Jan 23 19:13:27 2006 +++ squid3/src/MemObject.h Thu May 18 16:22:00 2006 @@ -137,6 +137,9 @@ const char *vary_headers; + StoreEntry *ims_entry; + time_t refresh_timestamp; + void delayRead(DeferredRead const &); void kickReads(); Index: squid3/src/cf.data.pre diff -u squid3/src/cf.data.pre:1.97 squid3/src/cf.data.pre:1.55.2.4 --- squid3/src/cf.data.pre:1.97 Sun May 14 05:50:37 2006 +++ squid3/src/cf.data.pre Thu May 18 15:35:11 2006 @@ -2343,6 +2343,32 @@ client requested. (default) DOC_END +NAME: collapsed_forwarding +COMMENT: (on|off) +TYPE: onoff +LOC: Config.onoff.collapsed_forwarding +DEFAULT: off +DOC_START + This option enables multiple requests for the same URI to be + processed as one request. Normally disabled to avoid increased + latency on dynamic content, but there can be benefit from enabling + this in accelerator setups where the web servers are the bottleneck + and reliable and returns mostly cacheable information. +DOC_END + +NAME: refresh_stale_hit +COMMENT: (time) +TYPE: time_t +DEFAULT: 0 seconds +LOC: Config.refresh_stale_window +DOC_START + This option changes the refresh algorithm to allow concurrent + requests while an object is being refreshed to be processed as + cache hits if the object expired less than X seconds ago. Default + is 0 to disable this feature. This option is mostly interesting + in accelerator setups where a few objects is accessed very + frequently. +DOC_END COMMENT_START TIMEOUTS Index: squid3/src/client_side_reply.cc diff -u squid3/src/client_side_reply.cc:1.80 squid3/src/client_side_reply.cc:1.52.2.5 --- squid3/src/client_side_reply.cc:1.80 Fri May 12 06:49:56 2006 +++ squid3/src/client_side_reply.cc Thu May 18 16:22:00 2006 @@ -75,11 +75,20 @@ * is still valid */ removeClientStoreReference(&sc, http); + + if (old_entry && old_entry->mem_obj && old_entry->mem_obj->ims_entry && old_entry->mem_obj->ims_entry == http->storeEntry()) { + storeUnlockObject(old_entry->mem_obj->ims_entry); + old_entry->mem_obj->ims_entry = NULL; + } + /* old_entry might still be set if we didn't yet get the reply * code in HandleIMSReply() */ removeStoreReference(&old_sc, &old_entry); + safe_free(tempBuffer.data); + cbdataReferenceDone(http); + HTTPMSGUNLOCK(reply); } @@ -233,6 +242,7 @@ { char *url = http->uri; StoreEntry *entry = NULL; + int hit = 0; debug(88, 3)("clientReplyContext::processExpired: '%s'", http->uri); assert(http->storeEntry()->lastmod >= 0); /* @@ -254,28 +264,75 @@ #endif /* Prepare to make a new temporary request */ saveState(); - entry = storeCreateEntry(url, - http->log_uri, http->request->flags, http->request->method); + + if (http->storeEntry()->mem_obj && http->storeEntry()->mem_obj->ims_entry) { + entry = http->storeEntry()->mem_obj->ims_entry; + debug(88, 5) ("clientProcessExpired: collapsed request\n"); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + debug(88, 1) ("clientProcessExpired: collapsed request ABORTED!\n"); + entry = NULL; + } else if (http->storeEntry()->mem_obj->refresh_timestamp + 30 < squid_curtime) { + debug(88, 1) ("clientProcessExpired: collapsed request STALE!\n"); + entry = NULL; + } + + if (entry) { + storeLockObject(entry); + hit = 1; + } else { + storeUnlockObject(http->storeEntry()->mem_obj->ims_entry); + http->storeEntry()->mem_obj->ims_entry = NULL; + } + } + + if (!entry) { + entry = storeCreateEntry(url, + http->log_uri, + http->request->flags, + http->request->method); + + if (http->storeEntry()->mem_obj) { + http->storeEntry()->mem_obj->refresh_timestamp = squid_curtime; + + if (Config.onoff.collapsed_forwarding) { + debug(88, 1) ("clientProcessExpired: collapsed request forwarding\n"); /* # */ + http->storeEntry()->mem_obj->ims_entry = entry; + storeLockObject(http->storeEntry()->mem_obj->ims_entry); + } + } + } + /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ sc = storeClientListAdd(entry, this); + #if DELAY_POOLS /* delay_id is already set on original store client */ sc->setDelayId(DelayId::DelayClient(http)); + #endif http->request->lastmod = old_entry->lastmod; + debug(88, 5)("clientReplyContext::processExpired : lastmod %ld", (long int) entry->lastmod); + http->storeEntry(entry); + assert(http->out.offset == 0); - /* - * A refcounted pointer so that FwdState stays around as long as - * this clientReplyContext does - */ - FwdState::fwdStart(http->getConn().getRaw() != NULL ? http->getConn()->fd : -1, - http->storeEntry(), - http->request); + if (!hit) { + /* + * A refcounted pointer so that FwdState stays around as long as + * this clientReplyContext does + */ + FwdState::fwdStart(http->getConn().getRaw() != NULL ? http->getConn()->fd : -1, + http->storeEntry(), + http->request); + } else { + debug(88, 0) ("clientReplyContext::processExpired: HIT2"); + } + /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) @@ -525,6 +582,11 @@ storeUrl(http->storeEntry()), (long unsigned) result.length); + if (old_entry && old_entry->mem_obj && old_entry->mem_obj->ims_entry) { + storeUnlockObject(old_entry->mem_obj->ims_entry); + old_entry->mem_obj->ims_entry = NULL; + } + if (http->storeEntry() == NULL) return; @@ -678,7 +740,15 @@ ) { http->logType = LOG_TCP_NEGATIVE_HIT; sendMoreData(result); - } else if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) { + return; + } + + if (Config.refresh_stale_window > 0 && e->mem_obj && e->mem_obj->refresh_timestamp + Config.refresh_stale_window > squid_curtime && !refreshCheckHTTPStale(e, r)) { + debug(88, 2) ("clientProcessHit: refresh_stale HIT\n"); + goto hit; + } + + if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) { debug(88, 5) ("clientCacheHit: in refreshCheck() block\n"); /* * We hold a stale copy; it needs to be validated @@ -721,7 +791,13 @@ http->logType = LOG_TCP_MISS; processMiss(); } - } else if (r->flags.ims) { + + return; + } + +hit: + + if (r->flags.ims) { /* * Handle If-Modified-Since requests from the client */ @@ -755,20 +831,22 @@ */ triggerInitialStoreRead(); } - } else { - /* - * plain ol' cache hit - */ - if (e->store_status != STORE_OK) - http->logType = LOG_TCP_MISS; - else if (e->mem_status == IN_MEMORY) - http->logType = LOG_TCP_MEM_HIT; - else if (Config.onoff.offline) - http->logType = LOG_TCP_OFFLINE_HIT; - - sendMoreData(result); + return; } + + /* + * plain ol' cache hit + */ + + if (e->store_status != STORE_OK) + http->logType = LOG_TCP_MISS; + else if (e->mem_status == IN_MEMORY) + http->logType = LOG_TCP_MEM_HIT; + else if (Config.onoff.offline) + http->logType = LOG_TCP_OFFLINE_HIT; + + sendMoreData(result); } /* @@ -795,6 +873,10 @@ storeEntryDump(http->storeEntry(), 1); } + /* touch timestamp for refresh_stale_hit */ + if (http->storeEntry()->mem_obj) + http->storeEntry()->mem_obj->refresh_timestamp = squid_curtime; + removeClientStoreReference(&sc, http); } @@ -822,7 +904,23 @@ return; } else { assert(http->out.offset == 0); + createStoreEntry(r->method, r->flags); + debug(88, 0) ("clientProcessMiss: createStoreEntry.\n"); + + if (Config.onoff.collapsed_forwarding && r->flags.cachable && !r->flags.need_validation && (METHOD_GET == r->method || METHOD_HEAD == r->method)) { + /* touch timestamp for refresh_stale_hit */ + + if (http->storeEntry()->mem_obj) + http->storeEntry()->mem_obj->refresh_timestamp = squid_curtime; + + storeSetPublicKey(http->storeEntry()); + + debug(88, 0) ("clientProcessMiss: storeSetPublicKey.\n"); + + storeEntryDump(http->storeEntry(), 1); + } + triggerInitialStoreRead(); if (http->redirect.status) { @@ -1682,6 +1780,11 @@ StoreIOBuffer tempBuffer; storeLockObject(http->storeEntry()); + if (http->storeEntry()->store_status == STORE_PENDING && http->storeEntry()->mem_obj) { + if (http->storeEntry()->mem_obj->request) + http->request->hier = http->storeEntry()->mem_obj->request->hier; + }; + if (http->storeEntry()->mem_obj == NULL) { /* * This if-block exists because we don't want to clobber Index: squid3/src/http.cc diff -u squid3/src/http.cc:1.83 squid3/src/http.cc:1.35.2.6 --- squid3/src/http.cc:1.83 Fri May 12 06:49:57 2006 +++ squid3/src/http.cc Thu May 18 16:22:00 2006 @@ -802,7 +802,7 @@ /* Check if object is cacheable or not based on reply code */ debug(11, 3) ("haveParsedReplyHeaders: HTTP CODE: %d\n", getReply()->sline.status); - if (neighbors_do_private_keys) + if (neighbors_do_private_keys && !Config.onoff.collapsed_forwarding) httpMaybeRemovePublic(entry, getReply()->sline.status); if (getReply()->header.has(HDR_VARY) @@ -1406,16 +1406,18 @@ #endif /* append X-Forwarded-For */ - strFwd = hdr_in->getList(HDR_X_FORWARDED_FOR); + if (opt_forwarded_for) { + strFwd = hdr_in->getList(HDR_X_FORWARDED_FOR); - if (opt_forwarded_for && orig_request->client_addr.s_addr != no_addr.s_addr) - strListAdd(&strFwd, inet_ntoa(orig_request->client_addr), ','); - else - strListAdd(&strFwd, "unknown", ','); + if (orig_request->client_addr.s_addr != no_addr.s_addr) + strListAdd(&strFwd, inet_ntoa(orig_request->client_addr), ','); + else + strListAdd(&strFwd, "unknown", ','); - hdr_out->putStr(HDR_X_FORWARDED_FOR, strFwd.buf()); + hdr_out->putStr(HDR_X_FORWARDED_FOR, strFwd.buf()); - strFwd.clean(); + strFwd.clean(); + } /* append Host if not there already */ if (!hdr_out->has(HDR_HOST)) { @@ -1655,6 +1657,12 @@ break; + case HDR_X_FORWARDED_FOR: + if (!opt_forwarded_for) + hdr_out->addEntry(e->clone()); + + break; + case HDR_RANGE: case HDR_IF_RANGE: @@ -1669,8 +1677,6 @@ case HDR_CONNECTION: - case HDR_X_FORWARDED_FOR: - case HDR_CACHE_CONTROL: /* append these after the loop if needed */ break; Index: squid3/src/protos.h diff -u squid3/src/protos.h:1.71 squid3/src/protos.h:1.42.2.4 --- squid3/src/protos.h:1.71 Fri May 12 06:49:57 2006 +++ squid3/src/protos.h Thu May 18 15:35:12 2006 @@ -500,6 +500,7 @@ extern void refreshAddToList(const char *, int, time_t, int, time_t); extern int refreshIsCachable(const StoreEntry *); extern int refreshCheckHTTP(const StoreEntry *, HttpRequest *); +extern int refreshCheckHTTPStale(const StoreEntry *, HttpRequest *); extern int refreshCheckICP(const StoreEntry *, HttpRequest *); extern int refreshCheckHTCP(const StoreEntry *, HttpRequest *); extern int refreshCheckDigest(const StoreEntry *, time_t delta); Index: squid3/src/refresh.cc diff -u squid3/src/refresh.cc:1.14 squid3/src/refresh.cc:1.8.4.4 --- squid3/src/refresh.cc:1.14 Fri May 12 06:49:57 2006 +++ squid3/src/refresh.cc Thu May 18 16:22:00 2006 @@ -437,6 +437,13 @@ } int +refreshCheckHTTPStale(const StoreEntry * entry, HttpRequest * request) +{ + int reason = refreshCheck(entry, request, -Config.refresh_stale_window); + return (reason < 200) ? 0 : 1; +} + +int refreshCheckICP(const StoreEntry * entry, HttpRequest * request) { int reason = refreshCheck(entry, request, 30); Index: squid3/src/store.cc diff -u squid3/src/store.cc:1.38 squid3/src/store.cc:1.26.2.5 --- squid3/src/store.cc:1.38 Fri May 12 06:49:57 2006 +++ squid3/src/store.cc Thu May 18 16:22:00 2006 @@ -892,6 +892,8 @@ assert(len >= 0); assert(store_status == STORE_PENDING); + mem_obj->refresh_timestamp = squid_curtime; + StoreIOBuffer tempBuffer; tempBuffer.data = (char *)buf; tempBuffer.length = len; @@ -1552,6 +1554,11 @@ if (EBIT_TEST(e->flags, ENTRY_ABORTED)) return 0; + /* Entries which seem to have got stuck are not valid to be sent to new clients */ + if ((e->store_status == STORE_PENDING) + && (!e->mem_obj || e->mem_obj->refresh_timestamp + 30 < squid_curtime)) + return 0; + return 1; } Index: squid3/src/structs.h diff -u squid3/src/structs.h:1.93 squid3/src/structs.h:1.55.2.5 --- squid3/src/structs.h:1.93 Fri May 12 06:49:57 2006 +++ squid3/src/structs.h Thu May 18 16:22:00 2006 @@ -547,6 +547,7 @@ int emailErrData; int httpd_suppress_version_string; int global_internal_static; + int collapsed_forwarding; } onoff; @@ -724,6 +725,8 @@ ssl_client; #endif + + time_t refresh_stale_window; }; struct _SquidConfig2 Index: squid3/src/fs/coss/store_dir_coss.cc diff -u squid3/src/fs/coss/store_dir_coss.cc:1.23 squid3/src/fs/coss/store_dir_coss.cc:1.18.2.2 --- squid3/src/fs/coss/store_dir_coss.cc:1.23 Sun Jul 3 19:14:11 2005 +++ squid3/src/fs/coss/store_dir_coss.cc Thu May 18 15:35:13 2006 @@ -1,7 +1,6 @@ /* * $Id$ - * vim: set et : * * DEBUG: section 47 Store COSS Directory Routines * AUTHOR: Eric Stern