This patch is generated from the collapsed_forwarding-2_6 branch of HEAD in squid
Fri Aug 18 22:56:58 2006 GMT
See http://devel.squid-cache.org/

Index: squid/src/cf.data.pre
diff -u squid/src/cf.data.pre:1.101 squid/src/cf.data.pre:1.101.2.1
--- squid/src/cf.data.pre:1.101	Wed May 17 16:51:08 2006
+++ squid/src/cf.data.pre	Wed May 17 17:07:11 2006
@@ -2296,6 +2296,32 @@
 	client requested. (default)
 DOC_END
 
+NAME: collapsed_forwarding
+COMMENT: (on|off)
+TYPE: onoff
+LOC: Config.onoff.collapsed_forwarding
+DEFAULT: off
+DOC_START
+	This option enables multiple requests for the same URI to be
+	processed as one request. Normally disabled to avoid increased
+	latency on dynamic content, but there can be benefit from enabling
+	this in accelerator setups where the web servers are the bottleneck
+	and reliable and returns mostly cacheable information.
+DOC_END
+
+NAME: refresh_stale_hit
+COMMENT: (time)
+TYPE: time_t
+DEFAULT: 0 seconds
+LOC: Config.refresh_stale_window
+DOC_START
+	This option changes the refresh algorithm to allow concurrent
+	requests while an object is being refreshed to be processed as
+	cache hits if the object expired less than X seconds ago. Default
+	is 0 to disable this feature. This option is mostly interesiting
+	in accelerator setups where a few objects is accessed very
+	frequently.
+DOC_END
 
 COMMENT_START
  TIMEOUTS
Index: squid/src/client_side.c
diff -u squid/src/client_side.c:1.92 squid/src/client_side.c:1.91.2.2
--- squid/src/client_side.c:1.92	Wed May 17 21:33:21 2006
+++ squid/src/client_side.c	Thu May 18 06:45:07 2006
@@ -503,6 +503,7 @@
     clientHttpRequest *http = data;
     char *url = http->uri;
     StoreEntry *entry = NULL;
+    int hit = 0;
     debug(33, 3) ("clientProcessExpired: '%s'\n", http->uri);
     assert(http->entry->lastmod >= 0);
     /*
@@ -523,10 +524,37 @@
      * freed from memory before we need to access it.
      */
     assert(http->sc->callback_data == http);
-    entry = storeCreateEntry(url,
-	http->log_uri,
-	http->request->flags,
-	http->request->method);
+    if (http->entry->mem_obj && http->entry->mem_obj->ims_entry) {
+	entry = http->entry->mem_obj->ims_entry;
+	debug(33, 5) ("clientProcessExpired: collapsed request\n");
+	if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
+	    debug(33, 1) ("clientProcessExpired: collapsed request ABORTED!\n");
+	    entry = NULL;
+	} else if (http->entry->mem_obj->refresh_timestamp + 30 < squid_curtime) {
+	    debug(33, 1) ("clientProcessExpired: collapsed request STALE!\n");
+	    entry = NULL;
+	}
+	if (entry) {
+	    storeLockObject(entry);
+	    hit = 1;
+	} else {
+	    storeUnlockObject(http->entry->mem_obj->ims_entry);
+	    http->entry->mem_obj->ims_entry = NULL;
+	}
+    }
+    if (!entry) {
+	entry = storeCreateEntry(url,
+	    http->log_uri,
+	    http->request->flags,
+	    http->request->method);
+	if (http->entry->mem_obj) {
+	    http->entry->mem_obj->refresh_timestamp = squid_curtime;
+	    if (Config.onoff.collapsed_forwarding) {
+		http->entry->mem_obj->ims_entry = entry;
+		storeLockObject(http->entry->mem_obj->ims_entry);
+	    }
+	}
+    }
     /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */
     http->sc = storeClientListAdd(entry, http);
 #if DELAY_POOLS
@@ -537,7 +565,8 @@
     debug(33, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod);
     http->entry = entry;
     http->out.offset = 0;
-    fwdStart(http->conn->fd, http->entry, http->request);
+    if (!hit)
+	fwdStart(http->conn->fd, http->entry, http->request);
     /* Register with storage manager to receive updates when data comes in. */
     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 	debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n");
@@ -600,6 +629,10 @@
     int recopy = 1;
     http_status status;
     debug(33, 3) ("clientHandleIMSReply: %s, %ld bytes\n", url, (long int) size);
+    if (http->old_entry && http->old_entry->mem_obj && http->old_entry->mem_obj->ims_entry) {
+	storeUnlockObject(http->old_entry->mem_obj->ims_entry);
+	http->old_entry->mem_obj->ims_entry = NULL;
+    }
     if (entry == NULL) {
 	memFree(buf, MEM_CLIENT_SOCK_BUF);
 	return;
@@ -997,6 +1030,10 @@
     http->al.request = NULL;
     safe_free(http->redirect.location);
     stringClean(&http->range_iter.boundary);
+    if (http->old_entry && http->old_entry->mem_obj && http->old_entry->mem_obj->ims_entry && http->old_entry->mem_obj->ims_entry == http->entry) {
+	storeUnlockObject(http->old_entry->mem_obj->ims_entry);
+	http->old_entry->mem_obj->ims_entry = NULL;
+    }
     if ((e = http->entry)) {
 	http->entry = NULL;
 	storeUnregister(http->sc, e, http);
@@ -1740,7 +1777,13 @@
 	    http->log_type = LOG_TCP_MISS;
 	    clientProcessMiss(http);
 	}
-    } else if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) {
+	return;
+    }
+    if (Config.refresh_stale_window > 0 && e->mem_obj && e->mem_obj->refresh_timestamp + Config.refresh_stale_window > squid_curtime && !refreshCheckHTTPStale(e, r)) {
+	debug(33, 2) ("clientProcessHit: refresh_stale HIT\n");
+	goto hit;
+    }
+    if (!Config.onoff.offline && refreshCheckHTTP(e, r) && !http->flags.internal) {
 	debug(33, 5) ("clientCacheHit: in refreshCheck() block\n");
 	/*
 	 * We hold a stale copy; it needs to be validated
@@ -1783,7 +1826,10 @@
 	    clientProcessMiss(http);
 	}
 	memFree(buf, MEM_CLIENT_SOCK_BUF);
-    } else if (r->flags.ims) {
+	return;
+    }
+  hit:
+    if (r->flags.ims) {
 	/*
 	 * Handle If-Modified-Since requests from the client
 	 */
@@ -1816,18 +1862,18 @@
 	    memBufClean(&mb);
 	    storeComplete(e);
 	}
-    } else {
-	/*
-	 * plain ol' cache hit
-	 */
-	if (e->store_status != STORE_OK)
-	    http->log_type = LOG_TCP_MISS;
-	else if (e->mem_status == IN_MEMORY)
-	    http->log_type = LOG_TCP_MEM_HIT;
-	else if (Config.onoff.offline)
-	    http->log_type = LOG_TCP_OFFLINE_HIT;
-	clientSendMoreHeaderData(data, buf, size);
+	return;
     }
+    /*
+     * plain ol' cache hit
+     */
+    if (e->store_status != STORE_OK)
+	http->log_type = LOG_TCP_MISS;
+    else if (e->mem_status == IN_MEMORY)
+	http->log_type = LOG_TCP_MEM_HIT;
+    else if (Config.onoff.offline)
+	http->log_type = LOG_TCP_OFFLINE_HIT;
+    clientSendMoreHeaderData(data, buf, size);
 }
 
 /* put terminating boundary for multiparts */
@@ -2875,6 +2921,10 @@
     http->out.offset = 0;
     if (NULL != http->entry) {
 	storeLockObject(http->entry);
+	if (http->entry->store_status == STORE_PENDING && http->entry->mem_obj) {
+	    if (http->entry->mem_obj->request)
+		r->hier = http->entry->mem_obj->request->hier;
+	}
 	storeCreateMemObject(http->entry, http->uri, http->log_uri);
 	http->entry->mem_obj->method = r->method;
 	http->sc = storeClientListAdd(http->entry, http);
@@ -2916,6 +2966,9 @@
 	    debug(33, 0) ("\tlog_type = %s\n", log_tags[http->log_type]);
 	    storeEntryDump(http->entry, 1);
 	}
+	/* touch timestamp for refresh_stale_hit */
+	if (http->entry->mem_obj)
+	    http->entry->mem_obj->refresh_timestamp = squid_curtime;
 	storeUnregister(http->sc, http->entry, http);
 	http->sc = NULL;
 	storeUnlockObject(http->entry);
@@ -2944,6 +2997,10 @@
     }
     assert(http->out.offset == 0);
     http->entry = clientCreateStoreEntry(http, r->method, r->flags);
+    if (Config.onoff.collapsed_forwarding && r->flags.cachable && !r->flags.need_validation && (r->method = METHOD_GET || r->method == METHOD_HEAD)) {
+	http->entry->mem_obj->refresh_timestamp = squid_curtime;
+	storeSetPublicKey(http->entry);
+    }
     if (http->redirect.status) {
 	HttpReply *rep = httpReplyCreate();
 #if LOG_TCP_REDIRECTS
Index: squid/src/forward.c
diff -u squid/src/forward.c:1.22 squid/src/forward.c:1.21.2.2
--- squid/src/forward.c:1.22	Wed May 17 21:33:24 2006
+++ squid/src/forward.c	Thu May 18 06:45:09 2006
@@ -592,6 +592,7 @@
     assert(entry->lock_count);
     EBIT_SET(entry->flags, ENTRY_DISPATCHED);
     netdbPingSite(request->host);
+    entry->mem_obj->refresh_timestamp = squid_curtime;
     if (fwdState->servers && (p = fwdState->servers->peer)) {
 	p->stats.fetches++;
 	fwdState->request->peer_login = p->login;
Index: squid/src/http.c
diff -u squid/src/http.c:1.29 squid/src/http.c:1.29.2.1
--- squid/src/http.c:1.29	Wed May 17 16:51:09 2006
+++ squid/src/http.c	Wed May 17 17:07:12 2006
@@ -457,7 +457,7 @@
     storeTimestampsSet(entry);
     /* Check if object is cacheable or not based on reply code */
     debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status);
-    if (neighbors_do_private_keys)
+    if (neighbors_do_private_keys && !Config.onoff.collapsed_forwarding)
 	httpMaybeRemovePublic(entry, reply->sline.status);
     if (httpHeaderHas(&reply->header, HDR_VARY)
 #if X_ACCELERATOR_VARY
@@ -935,6 +935,10 @@
 		    httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1);
 	    }
 	    break;
+	case HDR_X_FORWARDED_FOR:
+	    if (!opt_forwarded_for)
+		httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
+	    break;
 	case HDR_RANGE:
 	case HDR_IF_RANGE:
 	case HDR_REQUEST_RANGE:
@@ -948,7 +952,6 @@
 	    break;
 	case HDR_PROXY_CONNECTION:
 	case HDR_CONNECTION:
-	case HDR_X_FORWARDED_FOR:
 	case HDR_CACHE_CONTROL:
 	    /* append these after the loop if needed */
 	    break;
@@ -973,12 +976,14 @@
 	stringClean(&strVia);
     }
     /* append X-Forwarded-For */
-    strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR);
-    strListAdd(&strFwd,
-	(((orig_request->client_addr.s_addr != no_addr.s_addr) && opt_forwarded_for) ?
-	    inet_ntoa(orig_request->client_addr) : "unknown"), ',');
-    httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd));
-    stringClean(&strFwd);
+    if (opt_forwarded_for) {
+	strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR);
+	strListAdd(&strFwd,
+	    (((orig_request->client_addr.s_addr != no_addr.s_addr) && opt_forwarded_for) ?
+		inet_ntoa(orig_request->client_addr) : "unknown"), ',');
+	httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd));
+	stringClean(&strFwd);
+    }
 
     /* append Host if not there already */
     if (!httpHeaderHas(hdr_out, HDR_HOST)) {
Index: squid/src/protos.h
diff -u squid/src/protos.h:1.76 squid/src/protos.h:1.74.6.2
--- squid/src/protos.h:1.76	Thu May 18 05:51:31 2006
+++ squid/src/protos.h	Thu May 18 06:45:09 2006
@@ -792,6 +792,7 @@
 extern void refreshAddToList(const char *, int, time_t, int, time_t);
 extern int refreshIsCachable(const StoreEntry *);
 extern int refreshCheckHTTP(const StoreEntry *, request_t *);
+extern int refreshCheckHTTPStale(const StoreEntry *, request_t *);
 extern int refreshCheckICP(const StoreEntry *, request_t *);
 extern int refreshCheckHTCP(const StoreEntry *, request_t *);
 extern int refreshCheckDigest(const StoreEntry *, time_t delta);
Index: squid/src/refresh.c
diff -u squid/src/refresh.c:1.8 squid/src/refresh.c:1.8.30.1
--- squid/src/refresh.c:1.8	Sun Jun 16 01:50:41 2002
+++ squid/src/refresh.c	Wed May 17 17:07:12 2006
@@ -366,6 +366,13 @@
 }
 
 int
+refreshCheckHTTPStale(const StoreEntry * entry, request_t * request)
+{
+    int reason = refreshCheck(entry, request, -Config.refresh_stale_window);
+    return (reason < 200) ? 0 : 1;
+}
+
+int
 refreshCheckICP(const StoreEntry * entry, request_t * request)
 {
     int reason = refreshCheck(entry, request, 30);
Index: squid/src/store.c
diff -u squid/src/store.c:1.21 squid/src/store.c:1.21.12.1
--- squid/src/store.c:1.21	Fri Apr 28 04:10:53 2006
+++ squid/src/store.c	Wed May 17 17:07:12 2006
@@ -161,6 +161,10 @@
      */
     assert(mem->clients.head == NULL);
 #endif
+    if (mem->ims_entry) {
+	storeUnlockObject(mem->ims_entry);
+	mem->ims_entry = NULL;
+    }
     httpReplyDestroy(mem->reply);
     requestUnlink(mem->request);
     mem->request = NULL;
@@ -521,6 +525,7 @@
     assert(mem != NULL);
     assert(len >= 0);
     assert(e->store_status == STORE_PENDING);
+    mem->refresh_timestamp = squid_curtime;
     if (len) {
 	debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n",
 	    len,
@@ -1066,6 +1071,10 @@
 	    return 0;
     if (EBIT_TEST(e->flags, ENTRY_ABORTED))
 	return 0;
+    /* Entries which seem to have got stuck is not valid to send to new clients */
+    if (e->store_status == STORE_PENDING)
+       	if (!e->mem_obj || e->mem_obj->refresh_timestamp + 30 < squid_curtime)
+	    return 0;
     return 1;
 }
 
Index: squid/src/structs.h
diff -u squid/src/structs.h:1.85 squid/src/structs.h:1.82.2.2
--- squid/src/structs.h:1.85	Thu May 18 05:51:31 2006
+++ squid/src/structs.h	Thu May 18 06:45:10 2006
@@ -658,6 +658,7 @@
 	int request_entities;
 	int detect_broken_server_pconns;
 	int balance_on_multiple_ip;
+	int collapsed_forwarding;
 	int relaxed_header_parser;
 	int accel_no_pmtu_disc;
 	int global_internal_static;
@@ -784,6 +785,7 @@
 	SSL_CTX *sslContext;
     } ssl_client;
 #endif
+    time_t refresh_stale_window;
 };
 
 struct _SquidConfig2 {
@@ -1657,6 +1659,8 @@
     unsigned int chksum;
 #endif
     const char *vary_headers;
+    StoreEntry *ims_entry;
+    time_t refresh_timestamp;
 };
 
 struct _StoreEntry {