--------------------- PatchSet 1319 Date: 2001/01/16 15:31:59 Author: rbcollins Branch: rbcollins_filters Tag: (none) Log: got it going then broke it :-[ Members: doc/debug-sections.txt:1.1.1.2.8.2->1.1.1.2.8.2.4.1 src/http.c:1.1.1.3.4.1.4.12->1.1.1.3.4.1.4.12.2.1 src/structs.h:1.1.1.3.4.1.4.12->1.1.1.3.4.1.4.12.2.1 src/typedefs.h:1.1.1.3.8.7->1.1.1.3.8.7.4.1 Index: squid/doc/debug-sections.txt =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/debug-sections.txt,v retrieving revision 1.1.1.2.8.2 retrieving revision 1.1.1.2.8.2.4.1 diff -u -r1.1.1.2.8.2 -r1.1.1.2.8.2.4.1 --- squid/doc/debug-sections.txt 10 Jan 2001 13:09:16 -0000 1.1.1.2.8.2 +++ squid/doc/debug-sections.txt 16 Jan 2001 15:31:59 -0000 1.1.1.2.8.2.4.1 @@ -86,3 +86,4 @@ section 79 HTTP Meter Header section 80 WCCP section 81 Store Removal/Replacement policy +section 82 Transfer-encoding Routines Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.1.1.3.4.1.4.12 retrieving revision 1.1.1.3.4.1.4.12.2.1 diff -u -r1.1.1.3.4.1.4.12 -r1.1.1.3.4.1.4.12.2.1 --- squid/src/http.c 13 Jan 2001 08:31:13 -0000 1.1.1.3.4.1.4.12 +++ squid/src/http.c 16 Jan 2001 15:31:59 -0000 1.1.1.3.4.1.4.12.2.1 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.1.1.3.4.1.4.12 2001/01/13 08:31:13 rbcollins Exp $ + * $Id: http.c,v 1.1.1.3.4.1.4.12.2.1 2001/01/16 15:31:59 rbcollins Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -450,6 +450,9 @@ return 1; } +DATAFILTER httpReplyHeaders; +DATAFILTER httpPerformTE; +DATAFILTER httpDoAppend; /* This will be called when data is ready to be read from fd. Read until * error or connection closed. */ /* XXX this function is too long! */ @@ -534,6 +537,8 @@ } else if (len == 0) { /* Connection closed; retrieval done. */ httpState->eof = 1; + /* TODO: optimise this out: it becomes a call to the filter chain + * with FILTER_EOF set */ if (httpState->reply_hdr_state < 2) /* * Yes Henrik, there is a point to doing this. When we @@ -547,7 +552,98 @@ comm_close(fd); } else { /* we have data */ +/* TODO: move the chain init hack to the request creation code! */ + if (!httpState->filters) { + FILTER_list *temp_filter; + /* the first entries added, end up being applied last */ + /* Send the data to the store manager */ +#if 0 + httpState->filters=xmalloc(sizeof(FILTER_list)); + httpState->filters->filter=httpDoAppend; + httpState->filters->data=httpState; + /* cbDataLock(httpState); ? */ + httpState->filters->prev=NULL; + httpState->filters->next=NULL; +#endif + /* perform TE encoding */ +#if 0 + temp_filter=httpState->filters; + httpState->filters=xmalloc(sizeof(FILTER_list)); + httpState->filters->filter=httpPerformTE; + httpState->filters->data=httpState; + /* cbDataLock(httpState); ? */ + httpState->filters->prev=NULL; + httpState->filters->next=temp_filter; + temp_filter->prev=httpState->filters; +#endif + /* process the headers */ + temp_filter=httpState->filters; + httpState->filters=xmalloc(sizeof(FILTER_list)); + httpState->filters->filter=httpReplyHeaders; + httpState->filters->data=httpState; + /* cbDataLock(httpState); ? */ + httpState->filters->prev=NULL; + httpState->filters->next=NULL;//temp_filter; +// temp_filter->prev=httpState->filters; + } + /* call the first filter in our filter chain */ + /* we use a filter no matter what */ + assert(httpState->filters); + httpState->filters->filter(buf, len, httpState->filters->next, 0, httpState->filters->data); + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * the above storeAppend() call could ABORT this entry, + * in that case, the server FD should already be closed. + * there's nothing for us to do. + */ + (void) 0; + } else if (httpPconnTransferDone(httpState)) { + /* yes we have to clear all these! */ + commSetDefer(fd, NULL, NULL); + commSetTimeout(fd, -1, NULL, NULL); + commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); +#if DELAY_POOLS + delayClearNoDelay(fd); +#endif + comm_remove_close_handler(fd, httpStateFree, httpState); + fwdUnregister(fd, httpState->fwd); + pconnPush(fd, request->host, request->port); + fwdComplete(httpState->fwd); + httpState->fd = -1; + httpStateFree(fd, httpState); + } else { + /* Wait for EOF condition */ + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + } + } +} +/* DATAFILTER */ +/* data filters do not consider len==0 to mean EOF. Set EOF in flags */ +/* all filters still get called on EOF. The last one back here calls the normal write func */ +/* reentrance and datafilters: if a datafilter wants to 'block' it has to: + * copy off the buf it was given + * call a callback style i/o/processing/etc routine + * return + * handle being called again, for the same request, before the callback has returned to + * the next step in processing + */ +/* the calling code assumes the following: the datafilter has forwarded all the data to + * the end of the chain. This MUST happen eventually. If can be deferred, but if + * DATA_EOF is set, the filter MUST flush and remove it self from the list and free it's + * private data + */ +/* A filter with no data to send for the moment, MAY call the filter chain but does not + * NEED to + */ +void httpReplyHeaders(char *buf, size_t len, FILTER_list *filters, unsigned int flags, void *data) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + { if (httpState->reply_hdr_state < 2) { + /* we haven't seen the full headers yet */ + /* ProcessReplyHeader expects partial data. This can be cleaned up now */ httpProcessReplyHeader(httpState, buf, len); if (httpState->reply_hdr_state == 2) { http_status s = entry->mem_obj->reply->sline.status; @@ -561,57 +657,158 @@ */ if (!fwdReforwardableStatus(s)) EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + } else { + /* why bother client_side & the store with a response that hasn't even + * got complete headers yet ? + */ + char *tempbuf; + tempbuf=httpState->headerstore; + httpState->headerstore=xmalloc(httpState->headerlength+len); + if (tempbuf) + memcpy(httpState->headerstore,tempbuf,httpState->headerlength); + memcpy(httpState->headerstore+httpState->headerlength,buf,len); + httpState->headerlength+=len; + if (tempbuf) + xfree(tempbuf); } + /* add the data into any partial headers we have here. We memcpy out of + * courtesy to later fn's so they get all the headers at once. Aren't we + * nice + */ + /* we don't do this above (yet) because ProcessHeaders expects the data bit + * by bit. That can be fixed now by doing this test and merge earlier */ + if (httpState->headerstore && httpState->reply_hdr_state == 2) { + /* headers have been processed, but there's a last combine step to do */ + char *tempbuf; + tempbuf=httpState->headerstore; + httpState->headerstore=xmalloc(httpState->headerlength+len); + memcpy(httpState->headerstore,tempbuf,httpState->headerlength); + memcpy(httpState->headerstore+httpState->headerlength,buf,len); + httpState->headerlength+=len; + xfree(tempbuf); +debug(1,1)("sending combined headers\n"); + if(filters->filter) + filters->filter(httpState->headerstore,httpState->headerlength,filters->next, flags, filters->data); + xfree(httpState->headerstore); + httpState->headerstore=NULL; + httpState->headerlength=0; + /* remove from filter chain */ + { + FILTER_list *temp; + temp=filters->prev; + filters->prev=filters->prev->prev; + if (temp->prev) + temp->prev->next=filters; + /* if I alloced data for _me_ free it */ + /* free the list entry */ + xfree(temp); + } + } else if (httpState->reply_hdr_state == 2){ + /* no partials headers, got them in one chunk */ +debug(1,1)("headers in one packet... sending\n"); + if(filters->filter) + filters->filter(buf,len,filters->next, flags, filters->data); +debug(1,1)("headers written \n"); + { + FILTER_list *temp; + temp=filters->prev; + filters->prev=filters->prev->prev; + if (temp->prev) + temp->prev->next=filters; + /* if I alloced data for _me_ free it */ + /* free the list entry */ + xfree(temp); + storeAppend(entry,buf,len); + } + } + } else + /* headers have been seen. If we're still in the list for some reason, call the + * next filter + */ + filters->filter(buf,len,filters->next, flags, filters->data); + } +} + +void httpPerformTE(char *buf, size_t len, FILTER_list *filters, unsigned int flags, void *data) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + + debug(1,1)("httpPerformTE: len = %d\n",len); + + if (!entry->mem_obj->reply->decode_translations) { + if(filters->filter) + filters->filter(buf,len,filters->next, flags, filters->data); + { + /* why are we here? */ + FILTER_list *temp; + temp=filters->prev; + filters->prev=filters->prev->prev; + if (temp->prev) + temp->prev->next=filters; + /* if I alloced data for _me_ free it */ + /* free the list entry */ + xfree(temp); } + return; + } + /* perform response data modifications here */ -{ -size_t k=0; -int terv=0; -/* for now, client_side filters bad headers, we just know where they stop for te to start -*/ -/* recreate headers without te stuff */ + { + size_t k=0; + int terv=0; +/* for now, client_side filters bad headers, we just know where they stop for te + * to start */ +/* TODO: recreate headers without te stuff */ if (httpState->written==0) { k = headersEnd(buf, len); - debug(1,1)("HttpReadReply: First time, header length =%d\n",k); - storeAppend(entry,buf,k); + debug(1,1)("HttpReplyBody: First time, header length =%d\n",k); + /* get the headers out of the system */ + /* we _should filter the headers we don't want here. ah well */ + if(filters->filter) + filters->filter(buf,k,filters->next, flags, filters->data); + /* storeAppend(entry,buf,k); */ httpState->written+=k; } - if (k != len) - { + debug(1,1)("k %d len %d\n",k,len); + if (k != len) { char *te_body_buf; size_t te_body_len; - /* perform transfer encoding on the recieved buffer */ - /* memory is allocated as needed by perform_te */ - terv = perform_te (entry->mem_obj->reply->decode_translations, + /* perform transfer encoding on the recieved buffer */ + /* memory is allocated as needed by perform_te */ + debug(1,1)("httpPerformTE: calling pte %p %p %d %p %p\n",entry->mem_obj->reply->decode_translations,buf+k, len-k, &te_body_buf, &te_body_len); + terv = perform_te (entry->mem_obj->reply->decode_translations, buf+k, len-k, &te_body_buf, &te_body_len); - /* now te_body_buf is our output buffer, and te_body_len is the amount to write*/ - if (te_body_len) - { - /* stuff to write */ - storeAppend(entry, te_body_buf, te_body_len); - httpState->bodysize+=te_body_len; - debug(1,1)("interim content-length:%d\n",httpState->bodysize); - httpState->written+=te_body_len; - } - if (terv & TE_BUFFER_ALLOCATED) - xfree (te_body_buf); - if (terv & TE_CHUNK_B){ - /* FIXME: I think this is te's EOF marker */ - debug(1,1)("*** EOF ???\n"); - if (entry->mem_obj->reply->content_length==-1) - { - /* the current body size is unknown */ - entry->mem_obj->reply->content_length=httpState->bodysize; + /* now te_body_buf is our output buffer, and te_body_len is the amount to + write*/ + if (te_body_len) { + /* stuff to write */ + /* storeAppend(entry, te_body_buf, te_body_len);*/ + if(filters->filter) + filters->filter(te_body_buf,te_body_len,filters->next, flags, filters->data); + httpState->bodysize+=te_body_len; + debug(1,1)("interim content-length:%d\n",httpState->bodysize); + httpState->written+=te_body_len; + } + if (terv & TE_BUFFER_ALLOCATED) + xfree (te_body_buf); + if (terv & TE_CHUNK_B){ + /* FIXME: I think this is te's EOF marker */ + debug(1,1)("*** EOF ???\n"); + if (entry->mem_obj->reply->content_length==-1) { + /* the current body size is unknown */ + /* TODO: fixme. The place for this is DoAppend when EOF is true */ + entry->mem_obj->reply->content_length=httpState->bodysize; + } + } + if (te_body_len==0 && !(terv & TE_BUFFERING_OUTPUT)) { + /* some terminating condition takes place here */ + debug(1,1)("figure OUT THIS TEST!?!\n"); } - } - if (te_body_len==0 && !(terv & TE_BUFFERING_OUTPUT)) - /* some terminating condition takes place here */ - { - debug(1,1)("figure OUT THIS TEST!?!\n"); - } /* we should probably split off the processing of data * so that a modification routine can alter the incoming data * by buffering it. */ @@ -624,34 +821,23 @@ */ } - } -debug(1,1)("written - %d\n",httpState->written); - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* - * the above storeAppend() call could ABORT this entry, - * in that case, the server FD should already be closed. - * there's nothing for us to do. - */ - (void) 0; - } else if (httpPconnTransferDone(httpState)) { - /* yes we have to clear all these! */ - commSetDefer(fd, NULL, NULL); - commSetTimeout(fd, -1, NULL, NULL); - commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); -#if DELAY_POOLS - delayClearNoDelay(fd); -#endif - comm_remove_close_handler(fd, httpStateFree, httpState); - fwdUnregister(fd, httpState->fwd); - pconnPush(fd, request->host, request->port); - fwdComplete(httpState->fwd); - httpState->fd = -1; - httpStateFree(fd, httpState); - } else { - /* Wait for EOF condition */ - commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - } } + debug(1,1)("written - %d\n",httpState->written); +} + + +/* empty filter. Must be the last in the filter chain */ +void httpDoAppend(char *buf, size_t len, FILTER_list *filters, unsigned int flags, void * +data) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + + assert(filters==NULL); + + storeAppend(entry,buf,len); + } /* This will be called when request write is complete. Schedule read of Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.1.1.3.4.1.4.12 retrieving revision 1.1.1.3.4.1.4.12.2.1 diff -u -r1.1.1.3.4.1.4.12 -r1.1.1.3.4.1.4.12.2.1 --- squid/src/structs.h 13 Jan 2001 08:31:13 -0000 1.1.1.3.4.1.4.12 +++ squid/src/structs.h 16 Jan 2001 15:31:59 -0000 1.1.1.3.4.1.4.12.2.1 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.1.1.3.4.1.4.12 2001/01/13 08:31:13 rbcollins Exp $ + * $Id: structs.h,v 1.1.1.3.4.1.4.12.2.1 2001/01/16 15:31:59 rbcollins Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -906,6 +906,13 @@ unsigned int only_if_cached:1; }; +struct _FILTER_list { + DATAFILTER * filter; + void *data; + FILTER_list *next; + FILTER_list *prev; +}; + struct _HttpStateData { StoreEntry *entry; request_t *request; @@ -920,6 +927,10 @@ int fd; http_state_flags flags; FwdState *fwd; + FILTER_list *filters; + /* this should be for the ReplyHeaders filter */ + char *headerstore; + size_t headerlength; }; struct _icpUdpData { Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.1.1.3.8.7 retrieving revision 1.1.1.3.8.7.4.1 diff -u -r1.1.1.3.8.7 -r1.1.1.3.8.7.4.1 --- squid/src/typedefs.h 10 Jan 2001 13:09:41 -0000 1.1.1.3.8.7 +++ squid/src/typedefs.h 16 Jan 2001 15:32:07 -0000 1.1.1.3.8.7.4.1 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.1.1.3.8.7 2001/01/10 13:09:41 rbcollins Exp $ + * $Id: typedefs.h,v 1.1.1.3.8.7.4.1 2001/01/16 15:32:07 rbcollins Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -352,4 +352,9 @@ typedef int STDIRSELECT(const StoreEntry *); +/* filters.c */ +typedef struct _FILTER_list FILTER_list; +/* buffer, data length, remaining filters pointer, flags, state data */ +typedef void DATAFILTER(char *, size_t , FILTER_list *, unsigned int, void *); + #endif /* _TYPEDEFS_H_ */