---------------------
PatchSet 1517 
Date: 2001/02/08 08:18:21
Author: rbcollins
Branch: rbcollins_filters
Tag: (none) 
Log:
 storeAppend now accepts flags

Members: 
	src/Packer.c:1.1.1.2.12.1.4.2->1.1.1.2.12.1.4.3 
	src/client_side.c:1.1.1.3.4.1.4.15.2.25->1.1.1.3.4.1.4.15.2.26 
	src/defines.h:1.1.1.3.8.7.2.4->1.1.1.3.8.7.2.5 
	src/forward.c:1.1.1.3.8.7.2.1->1.1.1.3.8.7.2.2 
	src/ftp.c:1.1.1.3.4.1.4.6.4.1->1.1.1.3.4.1.4.6.4.2 
	src/gopher.c:1.1.1.3.8.5.4.1->1.1.1.3.8.5.4.2 
	src/http.c:1.1.1.3.4.1.4.12.2.11->1.1.1.3.4.1.4.12.2.12 
	src/internal.c:1.1.1.3.8.2.4.1->1.1.1.3.8.2.4.2 
	src/mime.c:1.1.1.3.8.3.4.1->1.1.1.3.8.3.4.2 
	src/protos.h:1.1.1.3.8.11.2.11->1.1.1.3.8.11.2.12 
	src/store.c:1.1.1.3.8.7.4.1->1.1.1.3.8.7.4.2 
	src/structs.h:1.1.1.3.4.1.4.12.2.15->1.1.1.3.4.1.4.12.2.16 
	src/transfer-encoding.c:1.1.2.1.2.5->1.1.2.1.2.6 
	src/typedefs.h:1.1.1.3.8.7.4.12->1.1.1.3.8.7.4.13 
	src/wais.c:1.1.1.2.12.3.4.1->1.1.1.2.12.3.4.2 
	src/whois.c:1.1.1.2.12.4.4.1->1.1.1.2.12.4.4.2 
	src/modules/htmldemo/htmldemo.c:1.1.2.5->1.1.2.6 
	src/modules/spy/spy.c:1.1.2.5->1.1.2.6 
	src/modules/textreplace/textreplace.c:1.1.2.10->1.1.2.11 

Index: squid/src/Packer.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Packer.c,v
retrieving revision 1.1.1.2.12.1.4.2
retrieving revision 1.1.1.2.12.1.4.3
diff -u -r1.1.1.2.12.1.4.2 -r1.1.1.2.12.1.4.3
--- squid/src/Packer.c	6 Feb 2001 14:10:41 -0000	1.1.1.2.12.1.4.2
+++ squid/src/Packer.c	8 Feb 2001 08:18:21 -0000	1.1.1.2.12.1.4.3
@@ -1,6 +1,6 @@
 
 /*
- * $Id: Packer.c,v 1.1.1.2.12.1.4.2 2001/02/06 14:10:41 rbcollins Exp $
+ * $Id: Packer.c,v 1.1.1.2.12.1.4.3 2001/02/08 08:18:21 rbcollins Exp $
  *
  * DEBUG: section 60    Packer: A uniform interface to store-like modules
  * AUTHOR: Alex Rousskov
@@ -94,7 +94,7 @@
 
 
 /* append()'s */
-static void (*const store_append) (StoreEntry *, const char *, int) = &storeAppend;
+static void (*const store_append) (StoreEntry *, const char *, int) = &storeAppendOld;
 static void (*const memBuf_append) (MemBuf *, const char *, mb_size_t) = &memBufAppend;
 
 /* vprintf()'s */
Index: squid/src/client_side.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v
retrieving revision 1.1.1.3.4.1.4.15.2.25
retrieving revision 1.1.1.3.4.1.4.15.2.26
diff -u -r1.1.1.3.4.1.4.15.2.25 -r1.1.1.3.4.1.4.15.2.26
--- squid/src/client_side.c	7 Feb 2001 02:23:41 -0000	1.1.1.3.4.1.4.15.2.25
+++ squid/src/client_side.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.4.1.4.15.2.26
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side.c,v 1.1.1.3.4.1.4.15.2.25 2001/02/07 02:23:41 rbcollins Exp $
+ * $Id: client_side.c,v 1.1.1.3.4.1.4.15.2.26 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 33    Client-side Routines
  * AUTHOR: Duane Wessels
@@ -1642,7 +1642,7 @@
 	    e->timestamp = timestamp;
 	    http->entry = e;
 	    httpReplyParse(e->mem_obj->reply, mb.buf, mb.size);
-	    storeAppend(e, mb.buf, mb.size);
+	    storeAppend(e, mb.buf, mb.size, 0);
 	    memBufClean(&mb);
 	    storeComplete(e);
 	}
@@ -1864,7 +1864,7 @@
      */
     debug (33,8)("clientWriteReplyHeaders: Beginning\n");
     temp_filter=filters->node.next->data;
-    rvflags |= temp_filter->filter(buf,len,filter_list, temp_filter, flags | FILTER_HTTP_HEADER, temp_filter->data);
+    rvflags |= temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags | FILTER_HTTP_HEADER, temp_filter->data);
     dlinkDelete(&filters->node, filter_list);
     xfree(filters);
     http->flags.done_reply_headers=1;
@@ -1889,7 +1889,7 @@
 
     if (!http->flags.done_reply_headers) {
 	/* skip the header packet - we don't care about it */
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data);
     }
     if (http->request->range) {
         MemBuf mb;
@@ -1900,73 +1900,17 @@
         /* force the end of the transfer if we are done */
         if (!clientPackMoreRanges(http, buf, len, &mb))
             http->flags.done_copying = 1;
-	rvflags |= temp_filter->filter(mb.buf, mb.size, filter_list, temp_filter, flags, temp_filter->data);
+	rvflags |= temp_filter->filter(mb.buf, mb.size, offset, filter_list, temp_filter, flags, temp_filter->data);
 	memBufFreeFunc(&mb);
     } else {
 	debug(1,1)("**************\nBUG DoRangeReply called for a non range request\n************\n");
-        rvflags|= temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+        rvflags|= temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data);
 	dlinkDelete(&filters->node, filter_list);
 	xfree(filters);
     }
     debug (33,8)("clientDoRangeReply: Finished\n");
     return rvflags;
 }
-#if 0
-static void
-clientDoTEReply(const char *buf, size_t len, dlink_list *filter_list, FILTER_list *filters, unsigned int flags, void *data) {
-    clientHttpRequest *http = data;
-    FILTER_list *temp_filter;
-    /* adds any required TE codings to the data in buf and forwards it down the filter 
-     */
-    int free_te;
-    char * te_body_buf;
-    size_t te_body_size;
-
-    debug (33,8)("clientDoTEReply: Beginning\n");
-    temp_filter=filters->node.next->data;
-    if (!http->flags.done_reply_headers) {
-        /* skip the header packet - we don't care about it */
-	debug (33,8)("clientDoTEReply: skipping header block\n");
-        temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
-        return;
-    }
-#if 0
-    if (buf && len) {
-#endif
-#if 0
-        http->out.offset += len;
-        check_size += body_size;
-#endif
-
-	http->oldte_rv = free_te =
-              perform_te (http->te_translations,buf, len, &te_body_buf, &te_body_size);
-        
-	if (te_body_size != len)
-            debug(33,1) ("****TE changed body size %s:%d:%d\n",http->log_uri, len,te_body_size);
-	/* actually the next test needs to be if output and !EOF */
-	if (te_body_size)
-	    temp_filter->filter(te_body_buf, te_body_size, filter_list, temp_filter, flags, temp_filter->data);
-	/* if there's no output, don't call the rest of the chain */
-
-        if (free_te & TE_BUFFER_ALLOCATED)
-	    xfree (te_body_buf);
-
-	/* this should be later on... other filters may add data - for example
-	 * a postscript to the data ?! */
-
-        if (free_te & TE_CHUNK_B)
-            http->flags.done_copying = 1;
-#if 0
-    } else 
-	temp_filter->filter(NULL, 0, filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data);
-#endif
-#if 0
-    if (!http->request->range && http->request->method == METHOD_GET)
-        assert(check_size == size);
-#endif
-    debug (33,8)("clientDoTEReply: Finished\n");
-} 
-#endif
 
 DATAFILTER_FILTER(clientDoCommWriteMemBuf) {
     clientHttpRequest *http = data;
@@ -2078,7 +2022,7 @@
 	if (http->repfilters.head != NULL) {
 	    FILTER_list *temp_filter;
 	    temp_filter=http->repfilters.head->data;
-	    temp_filter->filter(NULL, 0, &http->repfilters, temp_filter, filter_flags, temp_filter->data);
+	    temp_filter->filter(NULL, 0, 0, &http->repfilters, temp_filter, filter_flags | FILTER_EOF, temp_filter->data);
 	} else 
 	    clientWriteComplete(fd, NULL, 0, COMM_OK, http);
 	memFree(buf, MEM_CLIENT_SOCK_BUF);
@@ -2276,7 +2220,7 @@
 	 */
 	assert(http->repfilters.head);
 	temp_filter=http->repfilters.head->data;
-	temp_filter->filter(mb.buf, mb.size, &http->repfilters, temp_filter, filter_flags, temp_filter->data);
+	temp_filter->filter(mb.buf, mb.size, -1, &http->repfilters, temp_filter, filter_flags, temp_filter->data);
 	memBufFreeFunc(&mb);
 	
     }
@@ -2305,7 +2249,7 @@
         filter_flags |= FILTER_EOF;
     assert(http->repfilters.head != NULL);
     temp_filter=http->repfilters.head->data;
-    temp_filter->filter(body_buf, body_size, &http->repfilters, temp_filter, filter_flags, temp_filter->data);
+    temp_filter->filter(body_buf, body_size, 0, &http->repfilters, temp_filter, filter_flags, temp_filter->data);
 #if 0
     http->out.offset+=body_size;
 #endif
@@ -2477,7 +2421,7 @@
         if (http->repfilters.head != NULL && size>0 && !http->flags.done_copying && (http->entry->mem_obj->reply->content_length < 0)) {
             FILTER_list *temp_filter;
             temp_filter=http->repfilters.head->data;
-            temp_filter->filter(NULL, 0, &http->repfilters, temp_filter, FILTER_EOF, temp_filter->data);	
+            temp_filter->filter(NULL, 0, 0, &http->repfilters, temp_filter, FILTER_EOF, temp_filter->data);	
 	return;
 	}
 
Index: squid/src/defines.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/defines.h,v
retrieving revision 1.1.1.3.8.7.2.4
retrieving revision 1.1.1.3.8.7.2.5
diff -u -r1.1.1.3.8.7.2.4 -r1.1.1.3.8.7.2.5
--- squid/src/defines.h	6 Feb 2001 14:10:42 -0000	1.1.1.3.8.7.2.4
+++ squid/src/defines.h	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.7.2.5
@@ -1,6 +1,6 @@
 
 /*
- * $Id: defines.h,v 1.1.1.3.8.7.2.4 2001/02/06 14:10:42 rbcollins Exp $
+ * $Id: defines.h,v 1.1.1.3.8.7.2.5 2001/02/08 08:18:22 rbcollins Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -335,6 +335,6 @@
 
 
 /* Prototypes for lazy coding */
-#define DATAFILTER_FILTER(func_name) static unsigned int func_name(const char *buf, size_t len, dlink_list * filter_list,FILTER_list * filters, unsigned int flags, void *data)
+#define DATAFILTER_FILTER(func_name) static unsigned int func_name(const char *buf, size_t len, size_t offset, dlink_list * filter_list,FILTER_list * filters, unsigned int flags, void *data)
 #endif
 
Index: squid/src/forward.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/forward.c,v
retrieving revision 1.1.1.3.8.7.2.1
retrieving revision 1.1.1.3.8.7.2.2
diff -u -r1.1.1.3.8.7.2.1 -r1.1.1.3.8.7.2.2
--- squid/src/forward.c	5 Feb 2001 13:37:11 -0000	1.1.1.3.8.7.2.1
+++ squid/src/forward.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.7.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: forward.c,v 1.1.1.3.8.7.2.1 2001/02/05 13:37:11 rbcollins Exp $
+ * $Id: forward.c,v 1.1.1.3.8.7.2.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 17    Request Forwarding
  * AUTHOR: Duane Wessels
@@ -450,7 +450,7 @@
     StoreEntry *e = fwdState->entry;
     FwdServer *fs = fwdState->servers;
     http_status s;
-    assert(e->store_status == STORE_PENDING);
+    assert(e->store_status == STORE_PENDING || e->store_status == STORE_OK);
     assert(e->mem_obj);
 #if URL_CHECKSUM_DEBUG
     assert(e->mem_obj->chksum == url_checksum(e->mem_obj->url));
@@ -632,15 +632,14 @@
 
 /*
  * server-side modules call fwdComplete() when they are done
- * downloading an object.  Then, we either 1) re-forward the
- * request somewhere else if needed, or 2) call storeComplete()
- * to finish it off
+ * downloading an object.  Then, we re-forward the request somewhere 
+ * else if needed. The server side module signals EOF directly to the store.
  */
 void
 fwdComplete(FwdState * fwdState)
 {
     StoreEntry *e = fwdState->entry;
-    assert(e->store_status == STORE_PENDING);
+    assert(e->store_status == STORE_PENDING || e->store_status == STORE_OK);
     debug(17, 3) ("fwdComplete: %s\n\tstatus %d\n", storeUrl(e),
 	e->mem_obj->reply->sline.status);
 #if URL_CHECKSUM_DEBUG
@@ -659,7 +658,9 @@
 	debug(17, 3) ("fwdComplete: not re-forwarding status %d\n",
 	    e->mem_obj->reply->sline.status);
 	EBIT_CLR(e->flags, ENTRY_FWD_HDR_WAIT);
+#if 0
 	storeComplete(e);
+#endif
 	/*
 	 * If fwdState isn't associated with a server FD, it
 	 * won't get freed unless we do it here.
Index: squid/src/ftp.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/ftp.c,v
retrieving revision 1.1.1.3.4.1.4.6.4.1
retrieving revision 1.1.1.3.4.1.4.6.4.2
diff -u -r1.1.1.3.4.1.4.6.4.1 -r1.1.1.3.4.1.4.6.4.2
--- squid/src/ftp.c	5 Feb 2001 13:37:11 -0000	1.1.1.3.4.1.4.6.4.1
+++ squid/src/ftp.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.4.1.4.6.4.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: ftp.c,v 1.1.1.3.4.1.4.6.4.1 2001/02/05 13:37:11 rbcollins Exp $
+ * $Id: ftp.c,v 1.1.1.3.4.1.4.6.4.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 9     File Transfer Protocol (FTP)
  * AUTHOR: Harvest Derived
@@ -410,7 +410,7 @@
     storeAppendPrintf(e, "</H2>\n");
     storeAppendPrintf(e, "<PRE>\n");
     dirup = ftpHtmlifyListEntry("<internal-dirup>", ftpState);
-    storeAppend(e, dirup, strlen(dirup));
+    storeAppend(e, dirup, strlen(dirup), 0);
     storeBufferFlush(e);
     ftpState->flags.html_header_sent = 1;
 }
@@ -824,7 +824,7 @@
 	    continue;
 	t = ftpHtmlifyListEntry(line, ftpState);
 	assert(t != NULL);
-	storeAppend(e, t, strlen(t));
+	storeAppend(e, t, strlen(t), 0);
     }
     storeBufferFlush(e);
     assert(usable <= len);
@@ -922,7 +922,7 @@
 	if (ftpState->flags.isdir) {
 	    ftpParseListing(ftpState);
 	} else {
-	    storeAppend(entry, ftpState->data.buf, len);
+	    storeAppend(entry, ftpState->data.buf, len, 0);
 	    ftpState->data.offset = 0;
 	}
 	commSetSelect(fd,
@@ -1083,6 +1083,8 @@
 	/* create appropriate reply */
 	ftpAuthRequired(reply, request, realm);
 	httpReplySwapOut(reply, entry);
+	/* hack until I get to work on the ftp code to make it filter based too */
+	storeComplete(entry);
 	fwdComplete(ftpState->fwd);
 	comm_close(fd);
 	return;
@@ -1656,8 +1658,11 @@
 	 * ftpReadControlReply() and before here, probably when
 	 * trying to write to the client.
 	 */
-	if (!EBIT_TEST(ftpState->entry->flags, ENTRY_ABORTED))
+	if (!EBIT_TEST(ftpState->entry->flags, ENTRY_ABORTED)) {
+	    /* hack until I get to work on the ftp code to make it filter based too */
+	    storeComplete(ftpState->entry);
 	    fwdComplete(ftpState->fwd);
+	}
 	ftpSendQuit(ftpState);
 	return;
     }
Index: squid/src/gopher.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/gopher.c,v
retrieving revision 1.1.1.3.8.5.4.1
retrieving revision 1.1.1.3.8.5.4.2
diff -u -r1.1.1.3.8.5.4.1 -r1.1.1.3.8.5.4.2
--- squid/src/gopher.c	5 Feb 2001 13:37:12 -0000	1.1.1.3.8.5.4.1
+++ squid/src/gopher.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.5.4.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: gopher.c,v 1.1.1.3.8.5.4.1 2001/02/05 13:37:12 rbcollins Exp $
+ * $Id: gopher.c,v 1.1.1.3.8.5.4.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 10    Gopher
  * AUTHOR: Harvest Derived
@@ -187,7 +187,7 @@
     }
     memBufPrintf(&mb, "\r\n");
     EBIT_CLR(gopherState->entry->flags, ENTRY_FWD_HDR_WAIT);
-    storeAppend(gopherState->entry, mb.buf, mb.size);
+    storeAppend(gopherState->entry, mb.buf, mb.size, 0);
     memBufClean(&mb);
 }
 
@@ -563,7 +563,7 @@
     }				/* while loop */
 
     if ((int) strlen(outbuf) > 0) {
-	storeAppend(entry, outbuf, strlen(outbuf));
+	storeAppend(entry, outbuf, strlen(outbuf), 0);
 	/* now let start sending stuff to client */
 	storeBufferFlush(entry);
     }
@@ -657,13 +657,15 @@
 	    gopherEndHTML(data);
 	storeTimestampsSet(entry);
 	storeBufferFlush(entry);
+	/* hack until I get to work on the ftp code to make it filter based too */
+	storeComplete(entry);
 	fwdComplete(gopherState->fwdState);
 	comm_close(fd);
     } else {
 	if (gopherState->conversion != NORMAL) {
 	    gopherToHTML(data, buf, len);
 	} else {
-	    storeAppend(entry, buf, len);
+	    storeAppend(entry, buf, len, 0);
 	}
 	commSetSelect(fd,
 	    COMM_SELECT_READ,
@@ -802,6 +804,8 @@
 	    }
 	}
 	gopherToHTML(gopherState, (char *) NULL, 0);
+	/* hack until I get to work on the ftp code to make it filter based too */
+	storeComplete(entry);
 	fwdComplete(fwdState);
 	comm_close(fd);
 	return;
Index: squid/src/http.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/http.c,v
retrieving revision 1.1.1.3.4.1.4.12.2.11
retrieving revision 1.1.1.3.4.1.4.12.2.12
diff -u -r1.1.1.3.4.1.4.12.2.11 -r1.1.1.3.4.1.4.12.2.12
--- squid/src/http.c	6 Feb 2001 14:10:42 -0000	1.1.1.3.4.1.4.12.2.11
+++ squid/src/http.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.4.1.4.12.2.12
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.c,v 1.1.1.3.4.1.4.12.2.11 2001/02/06 14:10:42 rbcollins Exp $
+ * $Id: http.c,v 1.1.1.3.4.1.4.12.2.12 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
  * AUTHOR: Harvest Derived
@@ -64,6 +64,7 @@
 #endif
     if (httpState == NULL)
 	return;
+    /* TODO free the filter chain here */
     storeUnlockObject(httpState->entry);
     if (httpState->reply_hdr) {
 	memFree(httpState->reply_hdr, MEM_8K_BUF);
@@ -427,13 +428,13 @@
 
    
     if (!httpState->flags.keepalive) 
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data);
 
     /* if we know that it's EOF, then keepalive is still ok (it TE signalled termination
      * of body
      */
     if (flags & FILTER_EOF) 
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data);
 
     /*
      * What does the reply have to say about keep-alive?
@@ -451,27 +452,27 @@
     mem = httpState->entry->mem_obj;
     reply = mem->reply;
     if (!reply->keep_alive) 
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data);
     debug(11, 5) ("httpPconnTransferDone: content_length=%d\n",
 	reply->content_length);
     /* If we haven't seen the end of reply headers, we are not done */
     if (httpState->reply_hdr_state < 2) 
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data);
     clen = httpReplyBodySize(httpState->request->method, reply);
     /* If there is no message body, we can be persistent */
     if (0 == clen) 
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data);
+	return (temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data) | FILTER_EOF);
     /* If the body size is unknown we must wait for EOF */
     if (clen < 0) 
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data);
     /* If the body size is known, we must wait until we've gotten all of it.  */
     if (mem->inmem_hi < reply->content_length + reply->hdr_sz) 
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data);
     /* We got it all */
-    return temp_filter->filter(buf,len,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data);
+    return (temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data) | FILTER_EOF);
 }
 
-void httpProcessData(const char *buf, size_t len, void *data);
+unsigned int httpProcessData(const char *buf, size_t len, void *data);
 
 static DATAFILTER httpDoAppend;
 /* This will be called when data is ready to be read from fd.  Read until
@@ -501,6 +502,13 @@
 	comm_close(fd);
 	return;
     }
+    /* Check for unexpected writes/protocol violations */
+    if (httpState->eof) {
+	/* eof has _already_ been reached. */
+	debug(11,1)("Server continued writing after End of message reached\n");
+	comm_close(fd);
+	return;
+    }
     /* check if we want to defer reading */
     errno = 0;
     read_sz = SQUID_TCP_SO_RCVBUF;
@@ -556,7 +564,12 @@
 	fwdFail(httpState->fwd, err);
 	httpState->eof = 1;
 	comm_close(fd);
+#if 0
     } else if (len == 0) {
+	unsigned int rvflags=0;
+	/* TODO: this is broken: if this was a persistent connection, EOF can mean
+	 * a failure - we should check we've recieved the expected amount of data
+	 */
 	/* Connection closed; retrieval done. */
 	httpState->eof = 1;
 	/* TODO: optimise this out: it becomes a call to the filter chain
@@ -571,11 +584,45 @@
 	    httpProcessReplyHeader(httpState, buf, len);
             te_build_decode_xlate_list(&entry->mem_obj->reply->header, &httpState->filters);
 	}
-	/* we should call the filter list to push out any buffered data */
+
+	rvflags = httpProcessData(NULL,0,data, FILTER_EOF);
+	
 	fwdComplete(httpState->fwd);
 	comm_close(fd);
+#endif
     } else {
-	httpProcessData(buf,len,data);
+        unsigned int rvflags=0;
+	rvflags = httpProcessData(buf,len,data);
+	if (len==0) {
+	    debug(11,1)("EOF by len=0 in httpHandleRead\n");
+	    httpState->eof = 1;
+	    fwdComplete(httpState->fwd);
+	    comm_close(fd);
+	}
+	else if (rvflags & FILTER_ABORT) {
+	    debug(11,1)("Entry abort detected in httpHandleRead\n");
+	}
+	else if (!(rvflags & FILTER_EOF)) {
+	    /* Read more data */
+	    commSetSelect(httpState->fd, COMM_SELECT_READ, httpReadReply, httpState, 0);
+	} else if (rvflags & FILTER_EOF)  {
+            debug (11,1)(" End of persistent connection in httpHandleRead\n");
+	    httpState->eof = 1;
+            /* yes we have to clear all these! */
+            commSetDefer(httpState->fd, NULL, NULL);
+            commSetTimeout(httpState->fd, -1, NULL, NULL);
+            commSetSelect(httpState->fd, COMM_SELECT_READ, NULL, NULL, 0);
+#if DELAY_POOLS
+            delayClearNoDelay(httpState->fd);
+#endif
+            comm_remove_close_handler(httpState->fd, httpStateFree, httpState);
+            fwdUnregister(httpState->fd, httpState->fwd);
+            pconnPush(httpState->fd, request->host, request->port);
+            fwdComplete(httpState->fwd);
+            fd=httpState->fd;
+            httpState->fd = -1;
+            httpStateFree(fd, httpState);
+        }
     }
 } 
 /* DATAFILTER */
@@ -599,7 +646,8 @@
 /* the calling pattern is (Buf, LEN, list_head, self_node, flags, statedata) */
 
 /* TODO: send two filter chunks: one with just the headers, one with the rest */
-void httpProcessData(const char *buf, size_t len, void *data)
+unsigned int
+ httpProcessData(const char *buf, size_t len, void *data)
 {
     HttpStateData *httpState = data;
     StoreEntry *entry = httpState->entry;
@@ -607,8 +655,10 @@
      * and thus, all filters, should not modify the request_t struct... hmmm */
     request_t *request = httpState->request;
     FILTER_list *temp_filter;
-    unsigned int rvflags = 0;
+    unsigned int rvflags = 0, flags=0;
     debug(1,1)("Processing data (len=%d\n",len);
+    if (len == 0) 
+	flags |= FILTER_EOF;
 	if (httpState->reply_hdr_state < 2) {
 	    /* we haven't seen the full headers yet */
 	    /* ProcessReplyHeader expects partial data. This can be cleaned up now */
@@ -673,11 +723,37 @@
 		httpState->headerlength+=len;
 		xfree(tempbuf);
 		hdr_len=httpState->headerlength;
+
+
+/* TODO: the below code may call the next filter with EOF set twice. This is a bad thing.
+ * The problem is that we don't test for body data and incoming eof yet
+ */
+		if (httpState->reply_hdr_state == 2) {
+		    debug(1,1)("hdr_size %d, headerlength %d\n",httpState->reply_hdr_size,
+		        httpState->headerlength);
+		    assert(httpState->reply_hdr_size == httpState->headerlength);
+		}
 		debug(1,1)("sending combined headers\n");
 		/* todo: split this into two calls like it is below */ 
-		        assert(httpState->filters.head);
-		        temp_filter=httpState->filters.head->data;
-		        rvflags = temp_filter->filter(httpState->headerstore, hdr_len, &httpState->filters, temp_filter, FILTER_HTTP_HEADER, temp_filter->data);
+		assert(httpState->filters.head);
+		temp_filter=httpState->filters.head->data;
+		rvflags |= temp_filter->filter(httpState->headerstore, httpState->reply_hdr_size, -1, &httpState->filters, temp_filter, flags | FILTER_HTTP_HEADER, temp_filter->data);
+
+		httpState->read_offset=0;
+		if (httpState->headerlength-httpState->reply_hdr_size) {
+		if (httpState->reply_hdr_state == 2) {
+		    debug(1,1)("hdr_size %d, headerlength %d\n",httpState->reply_hdr_size,
+		        httpState->headerlength);
+		    assert(httpState->reply_hdr_size == httpState->headerlength);
+		}
+		    debug(1,1)("headers (%d bytes) written, sending body (%d of %d)\n",
+    httpState->reply_hdr_size,
+    httpState->headerlength-httpState->reply_hdr_size,httpState->headerlength);
+		rvflags |= temp_filter->filter(httpState->headerstore+httpState->reply_hdr_size, httpState->headerlength-httpState->reply_hdr_size, -1, &httpState->filters, temp_filter, flags , temp_filter->data);
+		httpState->read_offset+=len-hdr_len;
+		} else 
+		                    debug(1,1)("headers (%d bytes) written, skipping body (%d of %d) due to filter return flags %d\n",httpState->reply_hdr_size,0,httpState->headerlength, rvflags);
+
 	    } else if (httpState->reply_hdr_state == 2){
 		size_t hdr_len=httpState->headerlength;
 				/* no partial headers, got them in one chunk */
@@ -685,15 +761,18 @@
 		    hdr_len);
 		assert(httpState->filters.head);
 		temp_filter=httpState->filters.head->data;
-		rvflags |= temp_filter->filter(buf, hdr_len, 
-		    &httpState->filters, temp_filter, FILTER_HTTP_HEADER, 
+		rvflags |= temp_filter->filter(buf, hdr_len, -1, 
+		    &httpState->filters, temp_filter, flags | FILTER_HTTP_HEADER, 
 		    temp_filter->data);
+		httpState->read_offset=0;
 		if (!(rvflags & (FILTER_EOF | FILTER_ABORT))) {
 		debug(1,1)("headers (%d bytes) written, sending body (%d of %d)\n",
 		    hdr_len,
 		    len-hdr_len,len);
-		if (len-hdr_len)
-		    temp_filter->filter(buf+hdr_len,len-hdr_len, &httpState->filters, temp_filter, 0, temp_filter->data);
+		if (len-hdr_len) {
+		    rvflags |= temp_filter->filter(buf+hdr_len,len-hdr_len, httpState->read_offset, &httpState->filters, temp_filter, flags , temp_filter->data);
+		    httpState->read_offset+=len-hdr_len;
+		}
 		} else
 		debug(1,1)("headers (%d bytes) written, skipping body (%d of %d) due to filter return flags %d\n",
 		    hdr_len,
@@ -705,97 +784,12 @@
 	debug(1,1)("headers previously written \n");
         assert(httpState->filters.head);
         temp_filter=httpState->filters.head->data;
-        temp_filter->filter(buf, len, &httpState->filters, temp_filter, 0, temp_filter->data);
-	}
-}
-# if 0
-void httpPerformTE(const char *buf, size_t len, dlink_list *filter_list, FILTER_list *filters, unsigned int flags, void *data)
-{
-    HttpStateData *httpState = data;
-    StoreEntry *entry = httpState->entry;
-    const request_t *request = httpState->request;
-    FILTER_list *temp_filter;
-
-    debug(1,1)("httpPerformTE: len = %d\n",len);
-
-    if (flags & FILTER_HTTP_HEADER) {
-	temp_filter=filters->node.next->data;
-	temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
-	return;
-    }
-
-        /* perform response data modifications here */
-
-    {
-	size_t k=0;
-	int terv=0;
-/* for now, client_side filters bad headers, we just know where they stop for te 
- * to start */
-/* TODO: recreate headers without te stuff */
- 	if (httpState->written==0) {
-	    k = headersEnd(buf, len);
-	    debug(1,1)("HttpReplyBody: First time, header length =%d\n",k);
-	    /* get the headers out of the system */
-	    /* we _should filter the headers we don't want here. ah well */
-	    temp_filter=filters->node.next->data;
-	    temp_filter->filter(buf,k,filter_list, temp_filter, flags, temp_filter->data);
-	    /* storeAppend(entry,buf,k); */
-	    httpState->written+=k;
-	    
+        rvflags |= temp_filter->filter(buf, len, httpState->read_offset, &httpState->filters, temp_filter, flags, temp_filter->data);
+	httpState->read_offset+=len;
 	}
-	debug(1,1)("k %d len %d\n",k,len);
-	if (k != len) {
-	    char *te_body_buf;
-	    size_t te_body_len;
-	    /* perform transfer encoding on the recieved buffer */
-            /* memory is allocated as needed by perform_te */
-	    debug(1,1)("httpPerformTE: calling pte %p %p %d %p %p\n",entry->mem_obj->reply->decode_translations,buf+k, len-k, &te_body_buf, &te_body_len);
-	    terv = perform_te (entry->mem_obj->reply->decode_translations,
-		buf+k, len-k, &te_body_buf, &te_body_len);
-  
-	    /* now te_body_buf is our output buffer, and te_body_len is the amount to
-		 write*/
-	    if (te_body_len) {
-	        /* stuff to write */
-	        /* storeAppend(entry, te_body_buf, te_body_len);*/
-		temp_filter=filters->node.next->data;
-		temp_filter->filter(te_body_buf,te_body_len,filter_list, temp_filter, flags, temp_filter->data);
-                httpState->bodysize+=te_body_len;
-	        debug(1,1)("interim content-length:%d\n",httpState->bodysize);
-	        httpState->written+=te_body_len;
-	    }
-	    if (terv & TE_BUFFER_ALLOCATED)
-	        xfree (te_body_buf);
-	    if (terv & TE_CHUNK_B){
-	        /* FIXME: I think this is te's EOF marker */
-	        debug(1,1)("*** EOF ???\n");
-                if (entry->mem_obj->reply->content_length==-1) {
-	            /* the current body size is unknown */
-		    /* TODO: fixme. The place for this is DoAppend when EOF is true */
-                    entry->mem_obj->reply->content_length=httpState->bodysize;
-	        }
-  	    }
-            if (te_body_len==0 && !(terv & TE_BUFFERING_OUTPUT)) {
-            /* some terminating condition takes place here */
-		debug(1,1)("figure OUT THIS TEST!?!\n");
-	    }
-        /* we should probably split off the processing of data
-         * so that a modification routine can alter the incoming data 
-         * by buffering it. */
-/*
- * Ie. before storeAppend, iCAP response (entry, buf, len, httpstoreReponse, httpstate)
- * which buffers what it sends off to the icap server,
- * and when a response is gottenm decides whether to forward the whole lot to the 
- * icap server, and storeAppend THAT response, or whether to 
- * storeAppend everything it gets
- */
-
-	}
-    }
-    debug(1,1)("written - %d\n",httpState->written);
+    debug(1,1)("*** HTTP RAW READ OFFSET NOW %d\n",httpState->read_offset);
+    return rvflags;
 }
-#endif
-
 
 /* empty filter. Must be the last in the filter chain */
 DATAFILTER_FILTER(httpDoAppend) {
@@ -808,8 +802,9 @@
     assert(filters->node.next==NULL);
 debug(1,1)("entry %p\n",entry);    
 
+    debug(1,1)("httpDoAppend: recieved %d bytes at offset %d with flags %d\n",len,offset,flags);
     if (!EBIT_TEST(entry->flags, ENTRY_ABORTED)) 
-            storeAppend(entry,buf,len);
+            storeAppend(entry,buf,len, flags);
 
     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
             /*
@@ -817,28 +812,8 @@
              * in that case, the server FD should already be closed.
              * there's nothing for us to do.
              */
-            (void) 0;
-        } else if (flags & FILTER_EOF)  {
-	    debug (11,1)(" End of persistent connection in httpDoAppend\n");
-            /* yes we have to clear all these! */
-            commSetDefer(httpState->fd, NULL, NULL);
-            commSetTimeout(httpState->fd, -1, NULL, NULL);
-            commSetSelect(httpState->fd, COMM_SELECT_READ, NULL, NULL, 0);
-#if DELAY_POOLS
-            delayClearNoDelay(httpState->fd);
-#endif
-            comm_remove_close_handler(httpState->fd, httpStateFree, httpState);
-            fwdUnregister(httpState->fd, httpState->fwd);
-            pconnPush(httpState->fd, request->host, request->port);
-            fwdComplete(httpState->fwd);
-	    fd=httpState->fd;
-            httpState->fd = -1;
-            httpStateFree(fd, httpState);
-	    rvflags |= FILTER_EOF; 
-        } else {
-            /* Read more data */
-            commSetSelect(httpState->fd, COMM_SELECT_READ, httpReadReply, httpState, 0);
-        }
+	rvflags |= FILTER_ABORT;
+    }
     return rvflags;
 }
 
@@ -1211,6 +1186,7 @@
     httpState->fwd = fwd;
     httpState->entry = fwd->entry;
     httpState->fd = fd;
+    httpState->read_offset=-1; /*-1 means we are waiting for a response, or in headers */
     if (fwd->servers)
 	httpState->peer = fwd->servers->peer;	/* might be NULL */
     if (httpState->peer) {
Index: squid/src/internal.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/internal.c,v
retrieving revision 1.1.1.3.8.2.4.1
retrieving revision 1.1.1.3.8.2.4.2
diff -u -r1.1.1.3.8.2.4.1 -r1.1.1.3.8.2.4.2
--- squid/src/internal.c	5 Feb 2001 13:37:12 -0000	1.1.1.3.8.2.4.1
+++ squid/src/internal.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.2.4.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: internal.c,v 1.1.1.3.8.2.4.1 2001/02/05 13:37:12 rbcollins Exp $
+ * $Id: internal.c,v 1.1.1.3.8.2.4.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 76    Internal Squid Object handling
  * AUTHOR: Duane, Alex, Henrik
@@ -65,7 +65,7 @@
 	    squid_curtime,
 	    -2);
 	httpReplySwapOut(entry->mem_obj->reply, entry);
-	storeAppend(entry, msgbuf, strlen(msgbuf));
+	storeAppend(entry, msgbuf, strlen(msgbuf), 0);
 	storeComplete(entry);
     } else {
 	debugObj(76, 1, "internalStart: unknown request:\n",
Index: squid/src/mime.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/mime.c,v
retrieving revision 1.1.1.3.8.3.4.1
retrieving revision 1.1.1.3.8.3.4.2
diff -u -r1.1.1.3.8.3.4.1 -r1.1.1.3.8.3.4.2
--- squid/src/mime.c	5 Feb 2001 13:37:12 -0000	1.1.1.3.8.3.4.1
+++ squid/src/mime.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.3.4.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: mime.c,v 1.1.1.3.8.3.4.1 2001/02/05 13:37:12 rbcollins Exp $
+ * $Id: mime.c,v 1.1.1.3.8.3.4.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 25    MIME Parsing
  * AUTHOR: Harvest Derived
@@ -440,7 +440,7 @@
     /* read the file into the buffer and append it to store */
     buf = memAllocate(MEM_4K_BUF);
     while ((n = read(fd, buf, 4096)) > 0)
-	storeAppend(e, buf, n);
+	storeAppend(e, buf, n, 0);
     file_close(fd);
     EBIT_SET(e->flags, ENTRY_SPECIAL);
     storeBufferFlush(e);
Index: squid/src/protos.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/protos.h,v
retrieving revision 1.1.1.3.8.11.2.11
retrieving revision 1.1.1.3.8.11.2.12
diff -u -r1.1.1.3.8.11.2.11 -r1.1.1.3.8.11.2.12
--- squid/src/protos.h	5 Feb 2001 13:37:33 -0000	1.1.1.3.8.11.2.11
+++ squid/src/protos.h	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.11.2.12
@@ -1,6 +1,6 @@
 
 /*
- * $Id: protos.h,v 1.1.1.3.8.11.2.11 2001/02/05 13:37:33 rbcollins Exp $
+ * $Id: protos.h,v 1.1.1.3.8.11.2.12 2001/02/08 08:18:22 rbcollins Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -871,7 +871,9 @@
 extern void storeInit(void);
 extern int storeClientWaiting(const StoreEntry *);
 extern void storeAbort(StoreEntry *);
-extern void storeAppend(StoreEntry *, const char *, int);
+extern void storeAppend(StoreEntry *, const char *, int, unsigned int);
+/* for packer.c as I don't plan to rewrite Membufs just yet!?! */
+extern void storeAppendOld(StoreEntry *, const char *, int);
 extern void storeLockObject(StoreEntry *);
 extern void storeRelease(StoreEntry *);
 extern int storeUnlockObject(StoreEntry *);
Index: squid/src/store.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/store.c,v
retrieving revision 1.1.1.3.8.7.4.1
retrieving revision 1.1.1.3.8.7.4.2
diff -u -r1.1.1.3.8.7.4.1 -r1.1.1.3.8.7.4.2
--- squid/src/store.c	5 Feb 2001 13:37:33 -0000	1.1.1.3.8.7.4.1
+++ squid/src/store.c	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.7.4.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: store.c,v 1.1.1.3.8.7.4.1 2001/02/05 13:37:33 rbcollins Exp $
+ * $Id: store.c,v 1.1.1.3.8.7.4.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 20    Storage Manager
  * AUTHOR: Harvest Derived
@@ -440,14 +440,28 @@
     e->expires = squid_curtime;
 }
 
+void
+storeAppendOld(StoreEntry * e, const char *buf, int len)
+{
+    storeAppend(e,buf,len,0);
+}
+
 /* Append incoming data from a primary server to an entry. */
 void
-storeAppend(StoreEntry * e, const char *buf, int len)
+storeAppend(StoreEntry * e, const char *buf, int len, unsigned int flags)
 {
     MemObject *mem = e->mem_obj;
     assert(mem != NULL);
     assert(len >= 0);
+    debug(20,1)("storeAppend, entry %p, len %d, flags %d\n",e,len,flags);
     assert(e->store_status == STORE_PENDING);
+    if (flags & FILTER_ABORT) {
+	/* the filter chain is aborting the request. Discard any attached data.  and 
+	 * perform the abort
+	 */
+	storeAbort(e);
+	return;
+    }
     if (len) {
 	debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n",
 	    len,
@@ -458,8 +472,13 @@
     }
     if (EBIT_TEST(e->flags, DELAY_SENDING))
 	return;
-    InvokeHandlers(e);
-    storeSwapOut(e);
+    if (flags & FILTER_EOF) {
+	/* this is the last write we will recieve. */
+	storeComplete(e);
+    } else {
+	InvokeHandlers(e);
+	storeSwapOut(e);
+    }
 }
 
 void
@@ -492,7 +511,7 @@
     LOCAL_ARRAY(char, buf, 4096);
     buf[0] = '\0';
     vsnprintf(buf, 4096, fmt, vargs);
-    storeAppend(e, buf, strlen(buf));
+    storeAppend(e, buf, strlen(buf), 0);
 }
 
 struct _store_check_cachable_hist {
Index: squid/src/structs.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/structs.h,v
retrieving revision 1.1.1.3.4.1.4.12.2.15
retrieving revision 1.1.1.3.4.1.4.12.2.16
diff -u -r1.1.1.3.4.1.4.12.2.15 -r1.1.1.3.4.1.4.12.2.16
--- squid/src/structs.h	5 Feb 2001 13:37:34 -0000	1.1.1.3.4.1.4.12.2.15
+++ squid/src/structs.h	8 Feb 2001 08:18:22 -0000	1.1.1.3.4.1.4.12.2.16
@@ -1,6 +1,6 @@
 
 /*
- * $Id: structs.h,v 1.1.1.3.4.1.4.12.2.15 2001/02/05 13:37:34 rbcollins Exp $
+ * $Id: structs.h,v 1.1.1.3.4.1.4.12.2.16 2001/02/08 08:18:22 rbcollins Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -973,6 +973,8 @@
     http_state_flags flags;
     FwdState *fwd;
     dlink_list filters;
+    /* How far into the network object are we? */
+    size_t read_offset;
     /* this should be for the ReplyHeaders filter */
     char *headerstore;
     size_t headerlength;
Index: squid/src/transfer-encoding.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Attic/transfer-encoding.c,v
retrieving revision 1.1.2.1.2.5
retrieving revision 1.1.2.1.2.6
diff -u -r1.1.2.1.2.5 -r1.1.2.1.2.6
--- squid/src/transfer-encoding.c	6 Feb 2001 14:10:42 -0000	1.1.2.1.2.5
+++ squid/src/transfer-encoding.c	8 Feb 2001 08:18:22 -0000	1.1.2.1.2.6
@@ -45,8 +45,15 @@
   int half_crlf;                /* 4 - reading data */
                                 /* 5 - seeking CRLF from data */
                                 /* 6 - looking for double CRLF */
+  size_t read_offset; /* how far through the object we have read data */
+  size_t write_offset; /* how far though we have written the data */
 } chunked_t;
 
+typedef struct dochunk_state_t {
+    size_t read_offset; /* how far through the data we are */
+    size_t write_offset; /* how far though we have written the data */
+} dochunk_state;
+
 #if 0
 int
 perform_te (TE_list *xlat, const char *ibuf, int ibufl,
@@ -95,6 +102,15 @@
 
 /* this is a TE filter */
 
+static void
+dochunked_remove (FILTER_list * filters, dlink_list * filter_list,
+                             void *data)
+{
+  dlinkDelete (&filters->node, filter_list);
+  xfree (filters);
+  xfree (data);
+}
+
 DATAFILTER_FILTER(dochunked) {
     FILTER_list *temp_filter;
     /* are we going to have more that 17 characters in the hex length? unlikely*/
@@ -106,20 +122,22 @@
     if (flags & FILTER_HTTP_HEADER) {
         /* skip the header packet - we don't care about it */
         debug (33,8)("dochunked: skipping header block\n");
-        return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+        return temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data);
     }
 
+    
+
     if (buf) {
 	snprintf (tempbuf,sizeof(tempbuf)-1,"%X\r\n", len);
 	/* TODO: test for strlen(tempbuf)<sizeof(tempbuf)-1 */
 	debug(82, 8) ("dochunked: creating chunk of %d\n", len);
-	rvflags |= temp_filter->filter(tempbuf,strlen(tempbuf),filter_list, temp_filter, flags & !FILTER_EOF, temp_filter->data);
+	rvflags |= temp_filter->filter(tempbuf, strlen(tempbuf), offset,filter_list, temp_filter, flags & !FILTER_EOF, temp_filter->data);
         if (rvflags & FILTER_ABORT)
 	    return rvflags;
-	rvflags |= temp_filter->filter(buf,len,filter_list, temp_filter, flags & !FILTER_EOF, temp_filter->data);
+	rvflags |= temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags & !FILTER_EOF, temp_filter->data);
 	if (rvflags & FILTER_ABORT)
 	    return rvflags;
-	rvflags |= temp_filter->filter("\r\n",2,filter_list, temp_filter, flags & !FILTER_EOF, temp_filter->data);
+	rvflags |= temp_filter->filter("\r\n",2, offset,filter_list ,temp_filter, flags & !FILTER_EOF, temp_filter->data);
 	if (rvflags & FILTER_ABORT)
 	    return rvflags;
     }
@@ -127,7 +145,7 @@
     if ((flags & FILTER_EOF) || !buf) {
 	/* write terminating 0 */
 	debug(82, 8) ("dochunked: creating termination chunk\n");
-	rvflags |= temp_filter->filter("0\r\n\r\n",5,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data);
+	rvflags |= temp_filter->filter("0\r\n\r\n",5, offset,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data);
 	rvflags |= FILTER_EOF;
     }
     debug(82,8)("dochunked: FINISHED\n");
@@ -149,8 +167,8 @@
     FILTER_list *temp_filter=NULL;
     chunked_t *h=data;
     char *tb=NULL, *w=NULL, *l=NULL;
-    int tbl=0, created,rv = 1,tr=0;
-    unsigned int newflags=0;
+    int tbl=0, created,tr=0;
+    unsigned int newflags=0, rvflags = 0;
 
 //    debug (33,8)("undochunked: Beginning\n");
     assert(filters!=NULL);
@@ -160,8 +178,8 @@
     temp_filter=filters->node.next->data;
     if (flags & FILTER_HTTP_HEADER) {
 	/* skip the header packet - we don't care about it */
-	debug (33,8)("undochunked: skipping header block\n");
-	return temp_filter->filter(buf,len,filter_list, temp_filter, flags, temp_filter->data);
+	debug (82,8)("undochunked: skipping header block\n");
+	return temp_filter->filter(buf,len, offset, filter_list, temp_filter, flags, temp_filter->data);
     }
 
   tbl = len;
@@ -170,7 +188,16 @@
   created = 0;
   w=(char *)buf;
   l = (char *) buf+len;
-  debug(82, 8)("undochunked: called with ibufl=%d\n",len);
+  debug(82, 8)("undochunked: called with ibufl=%d at offset %d expecting offset %d written offset %d\n",len,offset,h->read_offset,h->write_offset);
+
+ if (h->read_offset != offset) {
+	/* we haven't recieved contiguous data! can't unchunk this... */
+	debug(82,1)("*****************************************************************************************\nUndochunked recieved non contiguous data !\n****************************************************************************************\n");
+	/* send an abort downstream */
+	rvflags |= temp_filter->filter(NULL,0, h->write_offset, filter_list, temp_filter, flags | FILTER_ABORT, temp_filter->data);
+	/* return an abort */
+	return rvflags | FILTER_ABORT;
+    }
 
  state_machine_repeat:
 
@@ -297,6 +324,7 @@
         /* chunked complete */
         h->state = 7;
 	newflags = FILTER_EOF;
+	rvflags  |= FILTER_EOF;
 
       case 7:                   /* done, ignore */
         break;
@@ -305,11 +333,14 @@
     {
       xfree (tb);
 	debug (33,8)("undochunked: no input data\n");
-        return temp_filter->filter(NULL,0,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data);
+        return (temp_filter->filter(NULL,0, h->write_offset, filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data) | FILTER_EOF);
     }
   
-  debug (33,8)("undochunked: sending unchunked data, len %d flags %d\n",created, flags | newflags);
-  return temp_filter->filter(tb,created,filter_list, temp_filter, flags | newflags, temp_filter->data);
+  debug (33,8)("undochunked: sending unchunked data, len %d offset %d flags %d\n",created,h->write_offset, flags | newflags);
+  rvflags |= temp_filter->filter(created ? tb: NULL,created, h->write_offset , filter_list, temp_filter, flags | newflags, temp_filter->data);
+  h->write_offset+=created;
+  h->read_offset+=len;
+  return rvflags;
 }
 
 static void
@@ -368,6 +399,8 @@
 		debug(82, 7) ("removing chunked\n");
 		state=xmalloc(sizeof(chunked_t));
 		state->state=1;
+		state->read_offset=0;
+		state->write_offset=0;
 		new_xlat(&temp_list, 0, undochunked, undochunked_remove, state);
 	    }
 
@@ -418,6 +451,8 @@
     String s_te;
     dlink_list temp_list;
     dlink_node *temp_node,*temp_node2;
+    dochunk_state *state;
+
     temp_list.head=NULL;
     temp_list.tail=NULL;
   
@@ -445,7 +480,10 @@
 
             /* always add chunked to cover nonterminating encodings */
                 debug(82, 7) ("will add chunked\n");
-                new_xlat(&temp_list, 1, dochunked, NULL, NULL);
+		state=xmalloc(sizeof(dochunk_state));
+		state->read_offset=0;
+		state->write_offset=0;
+                new_xlat(&temp_list, 1, dochunked, dochunked_remove, state);
                 strcat(vlb, "chunked");
                 httpHeaderPutStr(hdr, HDR_TRANSFER_ENCODING, vlb);
                 httpHeaderDelById(hdr, HDR_CONTENT_LENGTH);
Index: squid/src/typedefs.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v
retrieving revision 1.1.1.3.8.7.4.12
retrieving revision 1.1.1.3.8.7.4.13
diff -u -r1.1.1.3.8.7.4.12 -r1.1.1.3.8.7.4.13
--- squid/src/typedefs.h	6 Feb 2001 14:10:42 -0000	1.1.1.3.8.7.4.12
+++ squid/src/typedefs.h	8 Feb 2001 08:18:22 -0000	1.1.1.3.8.7.4.13
@@ -1,6 +1,6 @@
 
 /*
- * $Id: typedefs.h,v 1.1.1.3.8.7.4.12 2001/02/06 14:10:42 rbcollins Exp $
+ * $Id: typedefs.h,v 1.1.1.3.8.7.4.13 2001/02/08 08:18:22 rbcollins Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -361,10 +361,12 @@
 typedef struct _filterAccess filterAccess;
 typedef struct _filterAccessConfig filterAccessConfig;
 
-/* buffer, data length, remaining filters pointer, flags, state data */
+/* buffer, data length, body offste. remaining filters pointer, flags, state data */
 /* returns flags, as per FILTER_*, allowing filters to abort connections and the like
+ * boyd_offset = -1 when we are not sending body content ie in headers. BUT CHECK THE 
+ * FLAGS 
  */
-typedef unsigned int DATAFILTER(const char *, size_t , dlink_list *, FILTER_list *, unsigned int, void *);
+typedef unsigned int DATAFILTER(const char *, size_t ,size_t, dlink_list *, FILTER_list *, unsigned int, void *);
 /* self, the list, the state */
 typedef void REMOVEFILTER(FILTER_list *, dlink_list *, void*);
 typedef void *FILTERMKSTATE(void *, clientHttpRequest *, HttpReply *, request_t *);
Index: squid/src/wais.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/wais.c,v
retrieving revision 1.1.1.2.12.3.4.1
retrieving revision 1.1.1.2.12.3.4.2
diff -u -r1.1.1.2.12.3.4.1 -r1.1.1.2.12.3.4.2
--- squid/src/wais.c	5 Feb 2001 13:37:34 -0000	1.1.1.2.12.3.4.1
+++ squid/src/wais.c	8 Feb 2001 08:18:22 -0000	1.1.1.2.12.3.4.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: wais.c,v 1.1.1.2.12.3.4.1 2001/02/05 13:37:34 rbcollins Exp $
+ * $Id: wais.c,v 1.1.1.2.12.3.4.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 24    WAIS Relay
  * AUTHOR: Harvest Derived
@@ -148,10 +148,12 @@
     } else if (len == 0) {
 	/* Connection closed; retrieval done. */
 	entry->expires = squid_curtime;
+	/* hack until I work on the other protocols to be filter based */
+	storeComplete(entry);
 	fwdComplete(waisState->fwd);
 	comm_close(fd);
     } else {
-	storeAppend(entry, buf, len);
+	storeAppend(entry, buf, len, 0);
 	commSetSelect(fd,
 	    COMM_SELECT_READ,
 	    waisReadReply,
Index: squid/src/whois.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/whois.c,v
retrieving revision 1.1.1.2.12.4.4.1
retrieving revision 1.1.1.2.12.4.4.2
diff -u -r1.1.1.2.12.4.4.1 -r1.1.1.2.12.4.4.2
--- squid/src/whois.c	5 Feb 2001 13:37:34 -0000	1.1.1.2.12.4.4.1
+++ squid/src/whois.c	8 Feb 2001 08:18:22 -0000	1.1.1.2.12.4.4.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: whois.c,v 1.1.1.2.12.4.4.1 2001/02/05 13:37:34 rbcollins Exp $
+ * $Id: whois.c,v 1.1.1.2.12.4.4.2 2001/02/08 08:18:22 rbcollins Exp $
  *
  * DEBUG: section 75    WHOIS protocol
  * AUTHOR: Duane Wessels, Kostas Anagnostakis
@@ -102,7 +102,7 @@
 	fd_bytes(fd, len, FD_READ);
 	kb_incr(&statCounter.server.all.kbytes_in, len);
 	kb_incr(&statCounter.server.http.kbytes_in, len);
-	storeAppend(entry, buf, len);
+	storeAppend(entry, buf, len, 0);
 	commSetSelect(fd, COMM_SELECT_READ, whoisReadReply, p, Config.Timeout.read);
     } else if (len < 0) {
 	debug(50, 2) ("whoisReadReply: FD %d: read failure: %s.\n",
@@ -119,6 +119,8 @@
 	    comm_close(fd);
 	}
     } else {
+	/* hack until I work on the other serverside protocols */
+	storeComplete(entry);
 	fwdComplete(p->fwd);
 	debug(75, 3) ("whoisReadReply: Done: %s\n", storeUrl(entry));
 	comm_close(fd);
Index: squid/src/modules/htmldemo/htmldemo.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/modules/htmldemo/Attic/htmldemo.c,v
retrieving revision 1.1.2.5
retrieving revision 1.1.2.6
diff -u -r1.1.2.5 -r1.1.2.6
--- squid/src/modules/htmldemo/htmldemo.c	6 Feb 2001 14:10:42 -0000	1.1.2.5
+++ squid/src/modules/htmldemo/htmldemo.c	8 Feb 2001 08:18:23 -0000	1.1.2.6
@@ -173,7 +173,7 @@
     FILTER_list *temp_filter;
     HtmlDemoState *state=data;
     HtmlDemoConfig *config=state->config;
-    unsigned int rvflags;
+    unsigned int rvflags=0;
 
     temp_filter=filters->node.next->data;
     debug(1,1)("******* clientFilterHtmlDemo (%p,%d,%d)\n",buf,len,flags);
@@ -181,7 +181,7 @@
     if (flags & FILTER_HTTP_HEADER)
     {
 	debug(1,1)("[%s]\n",buf);
-        return temp_filter->filter(buf,len,filter_list, temp_filter, flags,
+        return temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags,
                             temp_filter->data);
     };
 
@@ -212,7 +212,7 @@
                 state->in_tag=0;
         };
 
-        rvflags=temp_filter->filter(state->buf,state->buf_len,filter_list,
+        rvflags=temp_filter->filter(state->buf,state->buf_len, offset,filter_list,
                             temp_filter, flags, temp_filter->data);
         /*HtmlDemoWrite(state,"",1);
         debug(1,1)("[%s]\n",state->buf);*/
Index: squid/src/modules/spy/spy.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/modules/spy/Attic/spy.c,v
retrieving revision 1.1.2.5
retrieving revision 1.1.2.6
diff -u -r1.1.2.5 -r1.1.2.6
--- squid/src/modules/spy/spy.c	6 Feb 2001 14:10:43 -0000	1.1.2.5
+++ squid/src/modules/spy/spy.c	8 Feb 2001 08:18:23 -0000	1.1.2.6
@@ -1,6 +1,6 @@
 
 /*
- * $Id: spy.c,v 1.1.2.5 2001/02/06 14:10:43 rbcollins Exp $
+ * $Id: spy.c,v 1.1.2.6 2001/02/08 08:18:23 rbcollins Exp $
  *
  * DEBUG: section 83    Content Processing Filters
  * AUTHOR: Robert Collins
@@ -135,7 +135,7 @@
     OnUnloadState *state=data;
     FILTER_list *temp_filter=filters->node.next->data;
 
-    debug(1,1)("**** Spying my dummy filter (buf=%x,len=%d,flags=%d url=%s\n",buf,len,flags,state->url);
-    return temp_filter->filter(buf,len,filter_list,temp_filter,flags,temp_filter->data);
+    debug(1,1)("**** Spying my dummy filter (buf=%x,len=%d,offset=%d,flags=%d url=%s\n",buf,len,offset,flags,state->url);
+    return temp_filter->filter(buf,len, offset,filter_list,temp_filter,flags,temp_filter->data);
 }
 
Index: squid/src/modules/textreplace/textreplace.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/modules/textreplace/Attic/textreplace.c,v
retrieving revision 1.1.2.10
retrieving revision 1.1.2.11
diff -u -r1.1.2.10 -r1.1.2.11
--- squid/src/modules/textreplace/textreplace.c	6 Feb 2001 14:10:43 -0000	1.1.2.10
+++ squid/src/modules/textreplace/textreplace.c	8 Feb 2001 08:18:23 -0000	1.1.2.11
@@ -1,6 +1,6 @@
 
 /*
- * $Id: textreplace.c,v 1.1.2.10 2001/02/06 14:10:43 rbcollins Exp $
+ * $Id: textreplace.c,v 1.1.2.11 2001/02/08 08:18:23 rbcollins Exp $
  *
  * DEBUG: section 83    Content Processing Filters
  * AUTHOR: Robert Collins
@@ -227,14 +227,14 @@
     {
       /* skip the header packet - we don't care about it */
       debug (33, 8) ("clientFilterOnUnload: skipping header block\n");
-      return temp_filter->filter (buf, len, filter_list, temp_filter, flags,
+      return temp_filter->filter (buf, len, offset, filter_list, temp_filter, flags,
 			   temp_filter->data);
     }
 
   if (!buf) {
     /* called with no buffer - legacy EOF indicator */
     debug (33, 8) ("clientFilterOnUnload: EOF via empty block\n");
-    rvflags = temp_filter->filter (NULL, 0, filter_list, temp_filter, flags,
+    rvflags = temp_filter->filter (NULL, 0,  offset, filter_list, temp_filter, flags,
                      temp_filter->data);
     if (rvflags & FILTER_ABORT)
 	return rvflags;
@@ -257,7 +257,7 @@
 	      if (pos - buf >= len) {
 		  /* end of data */
 		  return temp_filter->filter (lastmatch, len - (lastmatch - buf),
-				       filter_list, temp_filter, flags,
+				        offset, filter_list, temp_filter, flags, 
 				       temp_filter->data);
 		}
 	      if (*chr == '\0')	{
@@ -265,11 +265,11 @@
 		  /* send the first set of data */
 		  if (startpos - lastmatch)
 		    temp_filter->filter (lastmatch, startpos - lastmatch,
-					 filter_list, temp_filter, flags & !FILTER_EOF,
+					  offset, filter_list, temp_filter, flags & !FILTER_EOF,
 					 temp_filter->data);
 		  /* send the filter set of data */
 		  temp_filter->filter (config->replace, replacelen,
-				       filter_list, temp_filter, flags & !FILTER_EOF,
+				        offset, filter_list, temp_filter, flags & !FILTER_EOF,
 				       temp_filter->data);
 		  lastmatch = startpos + searchlen;
 		  /* update the pointers */
@@ -298,6 +298,6 @@
     }
     if (len- (lastmatch-buf))
 	/* found nothing */
-	return temp_filter->filter(lastmatch,len - (lastmatch-buf) ,filter_list, temp_filter, flags, temp_filter->data);
+	return temp_filter->filter(lastmatch,len - (lastmatch-buf) , offset, filter_list, temp_filter, flags, temp_filter->data);
     return rvflags;
 }