--------------------- PatchSet 2118 Date: 2001/04/27 14:03:09 Author: rbcollins Branch: rbcollins_filters Tag: (none) Log: Headers in different stream to body Members: src/client_side.c:1.1.1.3.4.1.4.15.2.32->1.1.1.3.4.1.4.15.2.33 src/defines.h:1.1.1.3.8.7.2.6->1.1.1.3.8.7.2.7 src/filters.c:1.1.2.11->1.1.2.12 src/http.c:1.1.1.3.4.1.4.12.2.14->1.1.1.3.4.1.4.12.2.15 src/protos.h:1.1.1.3.8.11.2.18->1.1.1.3.8.11.2.19 src/structs.h:1.1.1.3.4.1.4.12.2.23->1.1.1.3.4.1.4.12.2.24 src/transfer-encoding.c:1.1.2.1.2.7->1.1.2.1.2.8 src/typedefs.h:1.1.1.3.8.7.4.22->1.1.1.3.8.7.4.23 src/modules/htmldemo/htmldemo.c:1.1.2.8->1.1.2.9 src/modules/spy/spy.c:1.1.2.8->1.1.2.9 src/modules/textreplace/textreplace.c:1.1.2.14->1.1.2.15 Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.1.1.3.4.1.4.15.2.32 retrieving revision 1.1.1.3.4.1.4.15.2.33 diff -u -r1.1.1.3.4.1.4.15.2.32 -r1.1.1.3.4.1.4.15.2.33 --- squid/src/client_side.c 31 Mar 2001 11:28:54 -0000 1.1.1.3.4.1.4.15.2.32 +++ squid/src/client_side.c 27 Apr 2001 14:03:09 -0000 1.1.1.3.4.1.4.15.2.33 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.1.1.3.4.1.4.15.2.32 2001/03/31 11:28:54 rbcollins Exp $ + * $Id: client_side.c,v 1.1.1.3.4.1.4.15.2.33 2001/04/27 14:03:09 rbcollins Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -1229,7 +1229,7 @@ httpStatusLineSet(&rep->sline, rep->sline.version, HTTP_PARTIAL_CONTENT, NULL); /* add the ranges filter to the list */ - filterChainAddTail(&http->repfilters,clientDoRangeReply, NULL,http); + filterChainAddTail(&http->repfilters,clientDoRangeReply, identity_header, NULL,http); } } @@ -1273,6 +1273,7 @@ static DATAFILTER clientWriteReplyHeaders; static DATAFILTER clientDoCommWriteMemBuf; +static DATAFILTER_HDR clientDoCommWriteHeaders; void clientTEReplyheader(clientHttpRequest * http, HttpReply * rep, dlink_list *filter_list) { @@ -1325,7 +1326,7 @@ request_t *request = http->request; /* add the header check & forward filter */ - filterChainAdd(&http->repfilters,clientWriteReplyHeaders,NULL,http); + filterChainAdd(&http->repfilters,identity_body, clientWriteReplyHeaders, NULL,http); #if DONT_FILTER_THESE @@ -1463,7 +1464,7 @@ /* build reply header & install filters */ clientBuildReplyHeader(http, rep); /* add the terminating filter */ - filterChainAddTail(&http->repfilters,clientDoCommWriteMemBuf,NULL,http); + filterChainAddTail(&http->repfilters,clientDoCommWriteMemBuf,clientDoCommWriteHeaders, NULL,http); } else { /* parsing failure, get rid of the invalid reply */ httpReplyDestroy(rep); @@ -1946,6 +1947,68 @@ } +/* write the headers. Yes this is the same code as above. It is the same code for + * two reasons: + * + * 1: I haven't stripped it back yet. + * 2: pipelining may require buffering the headers + * It is the same code for Three, yes Three reasons + * 3: we don't recieve a header_t struct yet, so there's no reason to be different. + * When we do get that struct, this will create the headers to send to the client, + * But otherwise be the same + */ +DATAFILTER_FILTERHEADER(clientDoCommWriteHeaders) { + clientHttpRequest *http = data; + MemBuf mb; + memBufDefInit(&mb); + + /* allocate a membuf, put the data from range and te into it, and call comm_write + * membuf + */ + + /* the more efficient longterm design (say there are no ranges, and no te.. + * storeClientCopy doesn't copy the data (maybe there's another PIA call? + * we just write the buffer and when the write completes it auto frees. + * I wonder if membuffers could do that. Then + * It'd be trivial - if a later filter wants the original buffer, it just locks + * it. Hmmm. + */ + + /* we are the last of the mohicans */ + assert(filters->node.next==NULL); + assert((buf && len) || (flags & (FILTER_EOF | FILTER_ABORT))); + + debug (33,8)("clientDoCommWriteHeaders: buf %p len %d flags %d\n",buf,len,flags); + if (!buf || !len) { + /* No new data to write */ + /* if not EOF, then some error occured upstream... abort here TODO */ + clientWriteComplete(http->conn->fd, NULL, 0, 0, http); + /* TODO: when we start buffer blocks here, this is where we flush the buffer */ + /* Why? calling clientWriteComplete flushes the buffer too */ + return 0; + + } + /* there is data to write. Concept: buffer 4K/client sock buf/ whatever here + * and only call TCP/IP layer when EOF or a larger amount is present ??? */ + + if (flags & FILTER_EOF) + http->flags.done_copying=1; + + if (!http->flags.reply_write_in_progress) { + http->flags.reply_write_in_progress=1; + memBufAppend(&mb, buf, len); + comm_write_mbuf(http->conn->fd, mb, clientWriteComplete, http); + } else { + /* queue the data to the http membuffer */ + if (!http->mb.buf) + memBufDefInit(&http->mb); + memBufAppend(&http->mb, buf, len); + } +// comm_write(http->conn->fd, buf, len, clientWriteBodyComplete, http, NULL); + return 0; +} + + /* Responses with no body will not have a content-type header, * which breaks the rep_mime_type acl, which * coincidentally, is the most common acl for reply access lists. Index: squid/src/defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/defines.h,v retrieving revision 1.1.1.3.8.7.2.6 retrieving revision 1.1.1.3.8.7.2.7 diff -u -r1.1.1.3.8.7.2.6 -r1.1.1.3.8.7.2.7 --- squid/src/defines.h 31 Mar 2001 11:28:54 -0000 1.1.1.3.8.7.2.6 +++ squid/src/defines.h 27 Apr 2001 14:03:09 -0000 1.1.1.3.8.7.2.7 @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.1.1.3.8.7.2.6 2001/03/31 11:28:54 rbcollins Exp $ + * $Id: defines.h,v 1.1.1.3.8.7.2.7 2001/04/27 14:03:09 rbcollins Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -338,5 +338,7 @@ /* Prototypes for lazy coding */ #define DATAFILTER_FILTER(func_name) static unsigned int func_name(const char *buf, size_t len, size_t offset, dlink_list * filter_list,FILTER_list * filters, unsigned int flags, void *data) +#define DATAFILTER_FILTERHEADER(func_name) static unsigned int func_name(const char *buf, size_t len, size_t offset, dlink_list * filter_list,FILTER_list * filters, unsigned int flags, void *data) + #endif Index: squid/src/filters.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/filters.c,v retrieving revision 1.1.2.11 retrieving revision 1.1.2.12 diff -u -r1.1.2.11 -r1.1.2.12 --- squid/src/filters.c 31 Mar 2001 14:08:59 -0000 1.1.2.11 +++ squid/src/filters.c 27 Apr 2001 14:03:09 -0000 1.1.2.12 @@ -1,6 +1,6 @@ /* - * $Id: filters.c,v 1.1.2.11 2001/03/31 14:08:59 rbcollins Exp $ + * $Id: filters.c,v 1.1.2.12 2001/04/27 14:03:09 rbcollins Exp $ * * DEBUG: section 83 Content Processing Filters * AUTHOR: Robert Collins @@ -198,11 +198,12 @@ /* Add a filter to a filter chain. */ void -filterChainAdd(dlink_list *filter_list, DATAFILTER *filter, REMOVEFILTER *Remove, void *data) { +filterChainAdd(dlink_list *filter_list, DATAFILTER *filter, DATAFILTER_HDR *filter_hdr, REMOVEFILTER *Remove, void *data) { FILTER_list *temp_filter; /* add the header check & forward filter */ temp_filter=xmalloc(sizeof(FILTER_list)); temp_filter->filter=filter; + temp_filter->filter_hdr=filter_hdr; temp_filter->Remove=Remove; /* if this is to be locked, the caller does that */ temp_filter->data=data; @@ -212,17 +213,43 @@ /* Add a filter to a filter chain at the end of the chain */ void -filterChainAddTail(dlink_list *filter_list, DATAFILTER *filter, REMOVEFILTER *Remove, void *data) { +filterChainAddTail(dlink_list *filter_list, DATAFILTER *filter, + DATAFILTER_HDR *filter_hdr, REMOVEFILTER *Remove, void *data) { FILTER_list *temp_filter; /* add the header check & forward filter */ temp_filter=xmalloc(sizeof(FILTER_list)); temp_filter->filter=filter; + temp_filter->filter_hdr=filter_hdr; temp_filter->Remove=Remove; /* if this is to be locked, the caller does that */ temp_filter->data=data; dlinkAddTail(temp_filter, &temp_filter->node, filter_list); } +/* identity filter function: does nothing with the headers */ +unsigned int identity_header(const char *buf,size_t len, size_t offset, + dlink_list * filter_list,FILTER_list * filters, unsigned int flags, + void *data) { + FILTER_list *temp_filter; + temp_filter=filters->node.next->data; + debug (83, 8)("identity_header: skipping headers\n"); + return temp_filter->filter_hdr(buf,len, offset,filter_list, temp_filter, flags, + temp_filter->data); +} + +/* identity body function: does nothing with the body. (Use where the header filter will + * remove itself from the list. This _could_ be NULL, but if someone changes the header + * filter in the future so it doesn't remove itself, you'd kill squid */ +unsigned int identity_body(const char *buf,size_t len, size_t offset, + dlink_list * filter_list,FILTER_list * filters, unsigned int flags, + void *data) { + FILTER_list *temp_filter; + temp_filter=filters->node.next->data; + debug (83, 8)("identity_header: skipping headers\n"); + return temp_filter->filter_hdr(buf,len, offset,filter_list, temp_filter, flags, + temp_filter->data); +} + static void name_document_filter_add(void){ printf( Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.1.1.3.4.1.4.12.2.14 retrieving revision 1.1.1.3.4.1.4.12.2.15 diff -u -r1.1.1.3.4.1.4.12.2.14 -r1.1.1.3.4.1.4.12.2.15 --- squid/src/http.c 31 Mar 2001 11:28:54 -0000 1.1.1.3.4.1.4.12.2.14 +++ squid/src/http.c 27 Apr 2001 14:03:09 -0000 1.1.1.3.4.1.4.12.2.15 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.1.1.3.4.1.4.12.2.14 2001/03/31 11:28:54 rbcollins Exp $ + * $Id: http.c,v 1.1.1.3.4.1.4.12.2.15 2001/04/27 14:03:09 rbcollins Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -475,6 +475,8 @@ unsigned int httpProcessData(const char *buf, size_t len, void *data); static DATAFILTER httpDoAppend; +static DATAFILTER httpDoAppendHeaders; + /* This will be called when data is ready to be read from fd. Read until * error or connection closed. */ /* XXX this function is too long! */ @@ -668,14 +670,14 @@ http_status s = entry->mem_obj->reply->sline.status; te_build_decode_xlate_list(&entry->mem_obj->reply->header, &httpState->filters); /* test for EOF condition (has to happen after content decoding! */ - filterChainAddTail(&httpState->filters,httpPconnTransferDone,NULL,httpState); + filterChainAddTail(&httpState->filters,httpPconnTransferDone,identity_header, NULL,httpState); /* apply user configured filters */ filterBuildChain(Config.response_filters, &httpState->filters, NULL, entry->mem_obj->reply , request); /* Send the data to the store manager */ - filterChainAddTail(&httpState->filters,httpDoAppend,NULL,httpState); + filterChainAddTail(&httpState->filters,httpDoAppend,httpDoAppendHeaders, NULL,httpState); #if WIP_FWD_LOG fwdStatus(httpState->fwd, s); #endif @@ -810,6 +812,38 @@ return rvflags; } +/* empty filter. Must be the last in the filter chain + * this is the same as the above for the same reasons clientWriteMemBuf(Header) are the + * same. + * It should send the headers to the broker server side header function. (When this + * filter is hit, the server side filters have had the chance to remove/alter headers + * as needed. + */ +DATAFILTER_FILTERHEADER(httpDoAppendHeaders) { + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + int fd; + int rvflags = 0; + + assert(filters->node.next==NULL); +debug(1,1)("entry %p\n",entry); + + debug(1,1)("httpDoAppend: recieved %d bytes at offset %d with flags %d\n",len,offset,flags); + if (!EBIT_TEST(entry->flags, ENTRY_ABORTED)) + storeAppend(entry,buf,len, flags); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * the above storeAppend() call could ABORT this entry, + * in that case, the server FD should already be closed. + * there's nothing for us to do. + */ + rvflags |= FILTER_ABORT; + } + return rvflags; +} + /* This will be called when request write is complete. Schedule read of * reply. */ static void Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.1.1.3.8.11.2.18 retrieving revision 1.1.1.3.8.11.2.19 diff -u -r1.1.1.3.8.11.2.18 -r1.1.1.3.8.11.2.19 --- squid/src/protos.h 31 Mar 2001 14:08:59 -0000 1.1.1.3.8.11.2.18 +++ squid/src/protos.h 27 Apr 2001 14:03:09 -0000 1.1.1.3.8.11.2.19 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.1.1.3.8.11.2.18 2001/03/31 14:08:59 rbcollins Exp $ + * $Id: protos.h,v 1.1.1.3.8.11.2.19 2001/04/27 14:03:09 rbcollins Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1321,7 +1321,7 @@ * transfer_encoding.c */ extern void new_xlat (dlink_list *, int beg_or_end, - DATAFILTER *, REMOVEFILTER *, + DATAFILTER *, DATAFILTER_HDR *, REMOVEFILTER *, void *data); extern int perform_te (TE_list *xlat, const char *ibuf, int ibufl, char **obuf, int *obufl); @@ -1336,6 +1336,10 @@ extern void filterDeregisterModule(const char *); extern void filterBuildChain(filterAccess *, dlink_list *, clientHttpRequest * , HttpReply *, request_t *); extern void filterCleanChain(dlink_list *); -extern void filterChainAdd(dlink_list *, DATAFILTER *, REMOVEFILTER *, void *); -extern void filterChainAddTail(dlink_list *, DATAFILTER *, REMOVEFILTER *, void *); +extern void filterChainAdd(dlink_list *, DATAFILTER *, DATAFILTER_HDR *, + REMOVEFILTER *, void *); +extern void filterChainAddTail(dlink_list *, DATAFILTER *, DATAFILTER_HDR *, + REMOVEFILTER *, void *); extern void filterParserRegister(void); +extern DATAFILTER_HDR identity_header; +extern DATAFILTER identity_body; Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.1.1.3.4.1.4.12.2.23 retrieving revision 1.1.1.3.4.1.4.12.2.24 diff -u -r1.1.1.3.4.1.4.12.2.23 -r1.1.1.3.4.1.4.12.2.24 --- squid/src/structs.h 31 Mar 2001 14:08:59 -0000 1.1.1.3.4.1.4.12.2.23 +++ squid/src/structs.h 27 Apr 2001 14:03:09 -0000 1.1.1.3.4.1.4.12.2.24 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.1.1.3.4.1.4.12.2.23 2001/03/31 14:08:59 rbcollins Exp $ + * $Id: structs.h,v 1.1.1.3.4.1.4.12.2.24 2001/04/27 14:03:09 rbcollins Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -367,6 +367,7 @@ struct _FILTER_list { dlink_node node; DATAFILTER * filter; + DATAFILTER * filter_hdr; REMOVEFILTER *Remove; void *data; }; Index: squid/src/transfer-encoding.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/transfer-encoding.c,v retrieving revision 1.1.2.1.2.7 retrieving revision 1.1.2.1.2.8 diff -u -r1.1.2.1.2.7 -r1.1.2.1.2.8 --- squid/src/transfer-encoding.c 12 Feb 2001 13:31:24 -0000 1.1.2.1.2.7 +++ squid/src/transfer-encoding.c 27 Apr 2001 14:03:09 -0000 1.1.2.1.2.8 @@ -121,12 +121,6 @@ debug (82,8)("dochunked: Beginning len %d off %d flags %d\n",len,offset,flags); temp_filter=filters->node.next->data; - if (flags & FILTER_HTTP_HEADER) { - /* skip the header packet - we don't care about it */ - debug (33,8)("dochunked: skipping header block\n"); - return temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data); - } - if (buf) { snprintf (tempbuf,sizeof(tempbuf)-1,"%X\r\n", len); @@ -385,7 +379,7 @@ state->state=1; state->read_offset=0; state->write_offset=0; - filterChainAdd(&temp_list,undochunked,undochunked_remove,state); + filterChainAdd(&temp_list,undochunked,identity_header, undochunked_remove,state); } @@ -460,7 +454,7 @@ state=xmalloc(sizeof(dochunk_state)); state->read_offset=0; state->write_offset=0; - filterChainAddTail(&temp_list,dochunked,dochunked_remove,state); + filterChainAddTail(&temp_list,dochunked, identity_header, dochunked_remove,state); strcat(vlb, "chunked"); httpHeaderPutStr(hdr, HDR_TRANSFER_ENCODING, vlb); httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.1.1.3.8.7.4.22 retrieving revision 1.1.1.3.8.7.4.23 diff -u -r1.1.1.3.8.7.4.22 -r1.1.1.3.8.7.4.23 --- squid/src/typedefs.h 31 Mar 2001 11:28:55 -0000 1.1.1.3.8.7.4.22 +++ squid/src/typedefs.h 27 Apr 2001 14:03:09 -0000 1.1.1.3.8.7.4.23 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.1.1.3.8.7.4.22 2001/03/31 11:28:55 rbcollins Exp $ + * $Id: typedefs.h,v 1.1.1.3.8.7.4.23 2001/04/27 14:03:09 rbcollins Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -387,6 +387,7 @@ * FLAGS */ typedef unsigned int DATAFILTER(const char *, size_t ,size_t, dlink_list *, FILTER_list *, unsigned int, void *); +typedef unsigned int DATAFILTER_HDR(const char *, size_t ,size_t, dlink_list *, FILTER_list *, unsigned int, void *); /* self, the list, the state */ typedef void REMOVEFILTER(FILTER_list *, dlink_list *, void*); typedef void *FILTERMKSTATE(void *, clientHttpRequest *, HttpReply *, request_t *); Index: squid/src/modules/htmldemo/htmldemo.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/modules/htmldemo/Attic/htmldemo.c,v retrieving revision 1.1.2.8 retrieving revision 1.1.2.9 diff -u -r1.1.2.8 -r1.1.2.9 --- squid/src/modules/htmldemo/htmldemo.c 31 Mar 2001 14:08:59 -0000 1.1.2.8 +++ squid/src/modules/htmldemo/htmldemo.c 27 Apr 2001 14:03:09 -0000 1.1.2.9 @@ -97,7 +97,7 @@ static void HtmlDemo_Add(dlink_list *filters, void *filter_config) { - filterChainAddTail(filters,HtmlDemo_Filter,htmldemo_Remove,filter_config); + filterChainAddTail(filters,HtmlDemo_Filter, identity_header ,htmldemo_Remove,filter_config); #if 0 FILTER_list *temp_filter; temp_filter=xmalloc(sizeof(FILTER_list)); Index: squid/src/modules/spy/spy.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/modules/spy/Attic/spy.c,v retrieving revision 1.1.2.8 retrieving revision 1.1.2.9 diff -u -r1.1.2.8 -r1.1.2.9 --- squid/src/modules/spy/spy.c 31 Mar 2001 14:08:59 -0000 1.1.2.8 +++ squid/src/modules/spy/spy.c 27 Apr 2001 14:03:09 -0000 1.1.2.9 @@ -1,6 +1,6 @@ /* - * $Id: spy.c,v 1.1.2.8 2001/03/31 14:08:59 rbcollins Exp $ + * $Id: spy.c,v 1.1.2.9 2001/04/27 14:03:09 rbcollins Exp $ * * DEBUG: section 83 Content Processing Filters * AUTHOR: Robert Collins @@ -75,7 +75,7 @@ static void SpyFilter_Add (dlink_list * filters, void *filter_config) { - filterChainAddTail(filters,SpyFilter,SpyFilter_Remove,filter_config); + filterChainAddTail(filters,SpyFilter, SpyFilter, SpyFilter_Remove,filter_config); #if 0 FILTER_list *temp_filter; temp_filter = xmalloc (sizeof (FILTER_list)); Index: squid/src/modules/textreplace/textreplace.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/modules/textreplace/Attic/textreplace.c,v retrieving revision 1.1.2.14 retrieving revision 1.1.2.15 diff -u -r1.1.2.14 -r1.1.2.15 --- squid/src/modules/textreplace/textreplace.c 4 Apr 2001 22:30:16 -0000 1.1.2.14 +++ squid/src/modules/textreplace/textreplace.c 27 Apr 2001 14:03:09 -0000 1.1.2.15 @@ -1,6 +1,6 @@ /* - * $Id: textreplace.c,v 1.1.2.14 2001/04/04 22:30:16 rbcollins Exp $ + * $Id: textreplace.c,v 1.1.2.15 2001/04/27 14:03:09 rbcollins Exp $ * * DEBUG: section 83 Content Processing Filters * AUTHOR: Robert Collins @@ -125,7 +125,8 @@ static void clientFilterOnUnload_Add (dlink_list * filters, void *filter_config) { - filterChainAddTail(filters,clientFilterOnUnload,clientFilterOnUnload_Remove,filter_config); + filterChainAddTail(filters,clientFilterOnUnload,clientFilterOnUnload, + clientFilterOnUnload_Remove,filter_config); } static void *