---------------------
PatchSet 1319 
Date: 2001/01/16 15:31:59
Author: rbcollins
Branch: rbcollins_filters
Tag: (none) 
Log:
got it going then broke it :-[

Members: 
	doc/debug-sections.txt:1.1.1.2.8.2->1.1.1.2.8.2.4.1 
	src/http.c:1.1.1.3.4.1.4.12->1.1.1.3.4.1.4.12.2.1 
	src/structs.h:1.1.1.3.4.1.4.12->1.1.1.3.4.1.4.12.2.1 
	src/typedefs.h:1.1.1.3.8.7->1.1.1.3.8.7.4.1 

Index: squid/doc/debug-sections.txt
===================================================================
RCS file: /cvsroot/squid-sf//squid/doc/debug-sections.txt,v
retrieving revision 1.1.1.2.8.2
retrieving revision 1.1.1.2.8.2.4.1
diff -u -r1.1.1.2.8.2 -r1.1.1.2.8.2.4.1
--- squid/doc/debug-sections.txt	10 Jan 2001 13:09:16 -0000	1.1.1.2.8.2
+++ squid/doc/debug-sections.txt	16 Jan 2001 15:31:59 -0000	1.1.1.2.8.2.4.1
@@ -86,3 +86,4 @@
 section 79    HTTP Meter Header
 section 80    WCCP
 section 81    Store Removal/Replacement policy
+section 82    Transfer-encoding Routines
Index: squid/src/http.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/http.c,v
retrieving revision 1.1.1.3.4.1.4.12
retrieving revision 1.1.1.3.4.1.4.12.2.1
diff -u -r1.1.1.3.4.1.4.12 -r1.1.1.3.4.1.4.12.2.1
--- squid/src/http.c	13 Jan 2001 08:31:13 -0000	1.1.1.3.4.1.4.12
+++ squid/src/http.c	16 Jan 2001 15:31:59 -0000	1.1.1.3.4.1.4.12.2.1
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.c,v 1.1.1.3.4.1.4.12 2001/01/13 08:31:13 rbcollins Exp $
+ * $Id: http.c,v 1.1.1.3.4.1.4.12.2.1 2001/01/16 15:31:59 rbcollins Exp $
  *
  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
  * AUTHOR: Harvest Derived
@@ -450,6 +450,9 @@
     return 1;
 }
 
+DATAFILTER httpReplyHeaders;
+DATAFILTER httpPerformTE;
+DATAFILTER httpDoAppend;
 /* This will be called when data is ready to be read from fd.  Read until
  * error or connection closed. */
 /* XXX this function is too long! */
@@ -534,6 +537,8 @@
     } else if (len == 0) {
 	/* Connection closed; retrieval done. */
 	httpState->eof = 1;
+	/* TODO: optimise this out: it becomes a call to the filter chain
+	 * with FILTER_EOF set */
 	if (httpState->reply_hdr_state < 2)
 	    /*
 	     * Yes Henrik, there is a point to doing this.  When we
@@ -547,7 +552,98 @@
 	comm_close(fd);
     } else {
         /* we have data */
+/* TODO: move the chain init hack to the request creation code! */
+	if (!httpState->filters) {
+	    FILTER_list *temp_filter;
+	    /* the first entries added, end up being applied last */
+	    /* Send the data to the store manager */
+#if 0
+            httpState->filters=xmalloc(sizeof(FILTER_list));
+            httpState->filters->filter=httpDoAppend;
+            httpState->filters->data=httpState;
+            /* cbDataLock(httpState); ? */
+            httpState->filters->prev=NULL;
+            httpState->filters->next=NULL;
+#endif
+	    /* perform TE encoding */
+#if 0
+	    temp_filter=httpState->filters;
+            httpState->filters=xmalloc(sizeof(FILTER_list));
+            httpState->filters->filter=httpPerformTE;
+            httpState->filters->data=httpState;
+            /* cbDataLock(httpState); ? */
+            httpState->filters->prev=NULL;
+            httpState->filters->next=temp_filter;
+	    temp_filter->prev=httpState->filters;
+#endif
+	    /* process the headers */
+            temp_filter=httpState->filters;
+	    httpState->filters=xmalloc(sizeof(FILTER_list));
+	    httpState->filters->filter=httpReplyHeaders;
+	    httpState->filters->data=httpState;
+	    /* cbDataLock(httpState); ? */
+	    httpState->filters->prev=NULL;
+	    httpState->filters->next=NULL;//temp_filter;
+//            temp_filter->prev=httpState->filters;
+	}
+	/* call the first filter in our filter chain */
+	/* we use a filter no matter what */
+	assert(httpState->filters); 
+	httpState->filters->filter(buf, len, httpState->filters->next, 0, httpState->filters->data);
+        if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
+            /*
+             * the above storeAppend() call could ABORT this entry,
+             * in that case, the server FD should already be closed.
+             * there's nothing for us to do.
+             */
+            (void) 0;
+        } else if (httpPconnTransferDone(httpState)) {
+            /* yes we have to clear all these! */
+            commSetDefer(fd, NULL, NULL);
+            commSetTimeout(fd, -1, NULL, NULL);
+            commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0);
+#if DELAY_POOLS
+            delayClearNoDelay(fd);
+#endif
+            comm_remove_close_handler(fd, httpStateFree, httpState);
+            fwdUnregister(fd, httpState->fwd);
+            pconnPush(fd, request->host, request->port);
+            fwdComplete(httpState->fwd);
+            httpState->fd = -1;
+            httpStateFree(fd, httpState);
+        } else {
+            /* Wait for EOF condition */
+            commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0);
+        }
+    }
+} 
+/* DATAFILTER */
+/* data filters do not consider len==0 to mean EOF. Set EOF in flags */
+/* all filters still get called on EOF. The last one back here calls the normal write func */
+/* reentrance and datafilters: if a datafilter wants to 'block' it has to:
+ * copy off the buf it was given
+ * call a callback style i/o/processing/etc routine 
+ * return
+ * handle being called again, for the same request, before the callback has returned to
+ * the next step in processing 
+ */
+/* the calling code assumes the following: the datafilter has forwarded all the data to 
+ * the end of the chain. This MUST happen eventually. If can be deferred, but if
+ * DATA_EOF is set, the filter MUST flush and remove it self from the list and free it's
+ * private data
+ */
+/* A filter with no data to send for the moment, MAY call the filter chain but does not
+ * NEED to
+ */
+void httpReplyHeaders(char *buf, size_t len, FILTER_list *filters, unsigned int flags, void *data)
+{
+    HttpStateData *httpState = data;
+    StoreEntry *entry = httpState->entry;
+    const request_t *request = httpState->request;
+    {
 	if (httpState->reply_hdr_state < 2) {
+	    /* we haven't seen the full headers yet */
+	    /* ProcessReplyHeader expects partial data. This can be cleaned up now */
 	    httpProcessReplyHeader(httpState, buf, len);
 	    if (httpState->reply_hdr_state == 2) {
                 http_status s = entry->mem_obj->reply->sline.status;
@@ -561,57 +657,158 @@
 		 */
 		if (!fwdReforwardableStatus(s))
 		    EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
+	    } else {
+		/* why bother client_side & the store with a response that hasn't even 
+		 * got complete headers yet ? 
+		 */
+		char *tempbuf;
+		tempbuf=httpState->headerstore;
+		httpState->headerstore=xmalloc(httpState->headerlength+len);
+		if (tempbuf)
+		    memcpy(httpState->headerstore,tempbuf,httpState->headerlength);
+		memcpy(httpState->headerstore+httpState->headerlength,buf,len);
+		httpState->headerlength+=len;
+		if (tempbuf)
+		    xfree(tempbuf);
 	    }
+	    /* add the data into any partial headers we have here. We memcpy out of
+	     * courtesy to later fn's so they get all the headers at once. Aren't we 
+	     * nice
+	     */
+	    /* we don't do this above (yet) because ProcessHeaders expects the data bit
+	     * by bit. That can be fixed now by doing this test and merge earlier */
+	    if (httpState->headerstore && httpState->reply_hdr_state == 2) {
+		/* headers have been processed, but there's a last combine step to do */
+		char *tempbuf;
+		tempbuf=httpState->headerstore;
+		httpState->headerstore=xmalloc(httpState->headerlength+len);
+		memcpy(httpState->headerstore,tempbuf,httpState->headerlength);
+		memcpy(httpState->headerstore+httpState->headerlength,buf,len);
+		httpState->headerlength+=len;
+		xfree(tempbuf);
+debug(1,1)("sending combined headers\n");
+		if(filters->filter)
+		    filters->filter(httpState->headerstore,httpState->headerlength,filters->next, flags, filters->data);
+		xfree(httpState->headerstore);
+		httpState->headerstore=NULL;
+		httpState->headerlength=0;
+		/* remove from filter chain */
+		{
+		FILTER_list *temp;
+		temp=filters->prev;
+		filters->prev=filters->prev->prev;
+		if (temp->prev)
+		    temp->prev->next=filters;
+		/* if I alloced data for _me_ free it */
+		/* free the list entry */
+		xfree(temp);
+		}
+	    } else if (httpState->reply_hdr_state == 2){
+		/* no partials headers, got them in one chunk */
+debug(1,1)("headers in one packet... sending\n");
+		if(filters->filter)
+		    filters->filter(buf,len,filters->next, flags, filters->data);
+debug(1,1)("headers written \n");
+		{
+		FILTER_list *temp;
+		temp=filters->prev;
+		filters->prev=filters->prev->prev;
+		if (temp->prev)
+		    temp->prev->next=filters;
+		/* if I alloced data for _me_ free it */
+		/* free the list entry */
+		xfree(temp);
+		storeAppend(entry,buf,len);
+		}
+	    }
+	} else
+	/* headers have been seen. If we're still in the list for some reason, call the
+	 * next filter 
+	 */
+	filters->filter(buf,len,filters->next, flags, filters->data);
+    }
+}
+
+void httpPerformTE(char *buf, size_t len, FILTER_list *filters, unsigned int flags, void *data)
+{
+    HttpStateData *httpState = data;
+    StoreEntry *entry = httpState->entry;
+    const request_t *request = httpState->request;
+
+    debug(1,1)("httpPerformTE: len = %d\n",len);
+
+    if (!entry->mem_obj->reply->decode_translations) {
+	if(filters->filter)
+	    filters->filter(buf,len,filters->next, flags, filters->data);
+	{
+	/* why are we here? */
+	FILTER_list *temp;
+	temp=filters->prev;
+	filters->prev=filters->prev->prev;
+	if (temp->prev)
+	    temp->prev->next=filters;
+	/* if I alloced data for _me_ free it */
+	/* free the list entry */
+	xfree(temp);
 	}
+	return;
+    }
+
         /* perform response data modifications here */
 
-{
-size_t k=0;
-int terv=0;
-/* for now, client_side filters bad headers, we just know where they stop for te to start
-*/
-/* recreate headers without te stuff */
+    {
+	size_t k=0;
+	int terv=0;
+/* for now, client_side filters bad headers, we just know where they stop for te 
+ * to start */
+/* TODO: recreate headers without te stuff */
  	if (httpState->written==0) {
 	    k = headersEnd(buf, len);
-	    debug(1,1)("HttpReadReply: First time, header length =%d\n",k);
-	    storeAppend(entry,buf,k);
+	    debug(1,1)("HttpReplyBody: First time, header length =%d\n",k);
+	    /* get the headers out of the system */
+	    /* we _should filter the headers we don't want here. ah well */
+	    if(filters->filter)
+	        filters->filter(buf,k,filters->next, flags, filters->data);
+	    /* storeAppend(entry,buf,k); */
 	    httpState->written+=k;
 	    
 	}
-	if (k != len)
-	{
+	debug(1,1)("k %d len %d\n",k,len);
+	if (k != len) {
 	    char *te_body_buf;
 	    size_t te_body_len;
-	/* perform transfer encoding on the recieved buffer */
-        /* memory is allocated as needed by perform_te */
-	terv = perform_te (entry->mem_obj->reply->decode_translations,
+	    /* perform transfer encoding on the recieved buffer */
+            /* memory is allocated as needed by perform_te */
+	    debug(1,1)("httpPerformTE: calling pte %p %p %d %p %p\n",entry->mem_obj->reply->decode_translations,buf+k, len-k, &te_body_buf, &te_body_len);
+	    terv = perform_te (entry->mem_obj->reply->decode_translations,
 		buf+k, len-k, &te_body_buf, &te_body_len);
   
-	/* now te_body_buf is our output buffer, and te_body_len is the amount to write*/
-	if (te_body_len)
-	{
-	    /* stuff to write */
-	    storeAppend(entry, te_body_buf, te_body_len);
-            httpState->bodysize+=te_body_len;
-	    debug(1,1)("interim content-length:%d\n",httpState->bodysize);
-	    httpState->written+=te_body_len;
-	}
-	if (terv & TE_BUFFER_ALLOCATED)
-	    xfree (te_body_buf);
-	if (terv & TE_CHUNK_B){
-	    /* FIXME: I think this is te's EOF marker */
-	    debug(1,1)("*** EOF ???\n");
-            if (entry->mem_obj->reply->content_length==-1)
-	    {
-	         /* the current body size is unknown */
-                 entry->mem_obj->reply->content_length=httpState->bodysize;
+	    /* now te_body_buf is our output buffer, and te_body_len is the amount to
+		 write*/
+	    if (te_body_len) {
+	        /* stuff to write */
+	        /* storeAppend(entry, te_body_buf, te_body_len);*/
+	        if(filters->filter)
+		    filters->filter(te_body_buf,te_body_len,filters->next, flags, filters->data);
+                httpState->bodysize+=te_body_len;
+	        debug(1,1)("interim content-length:%d\n",httpState->bodysize);
+	        httpState->written+=te_body_len;
+	    }
+	    if (terv & TE_BUFFER_ALLOCATED)
+	        xfree (te_body_buf);
+	    if (terv & TE_CHUNK_B){
+	        /* FIXME: I think this is te's EOF marker */
+	        debug(1,1)("*** EOF ???\n");
+                if (entry->mem_obj->reply->content_length==-1) {
+	            /* the current body size is unknown */
+		    /* TODO: fixme. The place for this is DoAppend when EOF is true */
+                    entry->mem_obj->reply->content_length=httpState->bodysize;
+	        }
+  	    }
+            if (te_body_len==0 && !(terv & TE_BUFFERING_OUTPUT)) {
+            /* some terminating condition takes place here */
+		debug(1,1)("figure OUT THIS TEST!?!\n");
 	    }
-	}
-        if (te_body_len==0 && !(terv & TE_BUFFERING_OUTPUT))
-        /* some terminating condition takes place here */
-	{
-	    debug(1,1)("figure OUT THIS TEST!?!\n");
-	}
         /* we should probably split off the processing of data
          * so that a modification routine can alter the incoming data 
          * by buffering it. */
@@ -624,34 +821,23 @@
  */
 
 	}
-	}
-debug(1,1)("written - %d\n",httpState->written);
-	if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
-	    /*
-	     * the above storeAppend() call could ABORT this entry,
-	     * in that case, the server FD should already be closed.
-	     * there's nothing for us to do.
-	     */
-	    (void) 0;
-	} else if (httpPconnTransferDone(httpState)) {
-	    /* yes we have to clear all these! */
-	    commSetDefer(fd, NULL, NULL);
-	    commSetTimeout(fd, -1, NULL, NULL);
-	    commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0);
-#if DELAY_POOLS
-	    delayClearNoDelay(fd);
-#endif
-	    comm_remove_close_handler(fd, httpStateFree, httpState);
-	    fwdUnregister(fd, httpState->fwd);
-	    pconnPush(fd, request->host, request->port);
-	    fwdComplete(httpState->fwd);
-	    httpState->fd = -1;
-	    httpStateFree(fd, httpState);
-	} else {
-	    /* Wait for EOF condition */
-	    commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0);
-	}
     }
+    debug(1,1)("written - %d\n",httpState->written);
+}
+
+
+/* empty filter. Must be the last in the filter chain */
+void httpDoAppend(char *buf, size_t len, FILTER_list *filters, unsigned int flags, void *
+data)
+{
+    HttpStateData *httpState = data;
+    StoreEntry *entry = httpState->entry;
+    const request_t *request = httpState->request;
+
+    assert(filters==NULL);
+    
+    storeAppend(entry,buf,len);
+
 }
 
 /* This will be called when request write is complete. Schedule read of
Index: squid/src/structs.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/structs.h,v
retrieving revision 1.1.1.3.4.1.4.12
retrieving revision 1.1.1.3.4.1.4.12.2.1
diff -u -r1.1.1.3.4.1.4.12 -r1.1.1.3.4.1.4.12.2.1
--- squid/src/structs.h	13 Jan 2001 08:31:13 -0000	1.1.1.3.4.1.4.12
+++ squid/src/structs.h	16 Jan 2001 15:31:59 -0000	1.1.1.3.4.1.4.12.2.1
@@ -1,6 +1,6 @@
 
 /*
- * $Id: structs.h,v 1.1.1.3.4.1.4.12 2001/01/13 08:31:13 rbcollins Exp $
+ * $Id: structs.h,v 1.1.1.3.4.1.4.12.2.1 2001/01/16 15:31:59 rbcollins Exp $
  *
  *
  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
@@ -906,6 +906,13 @@
     unsigned int only_if_cached:1;
 };
 
+struct _FILTER_list {
+    DATAFILTER * filter;
+    void *data;
+    FILTER_list *next;
+    FILTER_list *prev;
+};
+
 struct _HttpStateData {
     StoreEntry *entry;
     request_t *request;
@@ -920,6 +927,10 @@
     int fd;
     http_state_flags flags;
     FwdState *fwd;
+    FILTER_list *filters;
+    /* this should be for the ReplyHeaders filter */
+    char *headerstore;
+    size_t headerlength;
 };
 
 struct _icpUdpData {
Index: squid/src/typedefs.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v
retrieving revision 1.1.1.3.8.7
retrieving revision 1.1.1.3.8.7.4.1
diff -u -r1.1.1.3.8.7 -r1.1.1.3.8.7.4.1
--- squid/src/typedefs.h	10 Jan 2001 13:09:41 -0000	1.1.1.3.8.7
+++ squid/src/typedefs.h	16 Jan 2001 15:32:07 -0000	1.1.1.3.8.7.4.1
@@ -1,6 +1,6 @@
 
 /*
- * $Id: typedefs.h,v 1.1.1.3.8.7 2001/01/10 13:09:41 rbcollins Exp $
+ * $Id: typedefs.h,v 1.1.1.3.8.7.4.1 2001/01/16 15:32:07 rbcollins Exp $
  *
  *
  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
@@ -352,4 +352,9 @@
 
 typedef int STDIRSELECT(const StoreEntry *);
 
+/* filters.c */
+typedef struct _FILTER_list FILTER_list;
+/* buffer, data length, remaining filters pointer, flags, state data */
+typedef void DATAFILTER(char *, size_t , FILTER_list *, unsigned int, void *);
+
 #endif				/* _TYPEDEFS_H_ */
