--------------------- PatchSet 4706 Date: 2002/08/18 11:09:22 Author: rbcollins Branch: esi Tag: (none) Log: esi:comment and esi:include support Members: configure.in:1.61.2.2->1.61.2.3 src/ESI.c:1.1.2.10->1.1.2.11 src/Makefile.am:1.23.2.1->1.23.2.2 src/client_side.c:1.65.2.15->1.65.2.16 src/client_side_reply.c:1.1.2.14->1.1.2.15 src/mem.c:1.20->1.20.2.1 Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.61.2.2 retrieving revision 1.61.2.3 diff -u -r1.61.2.2 -r1.61.2.3 --- squid/configure.in 11 Aug 2002 10:16:34 -0000 1.61.2.2 +++ squid/configure.in 18 Aug 2002 11:09:22 -0000 1.61.2.3 @@ -3,7 +3,7 @@ dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.61.2.2 2002/08/11 10:16:34 rbcollins Exp $ +dnl $Id: configure.in,v 1.61.2.3 2002/08/18 11:09:22 rbcollins Exp $ dnl dnl dnl @@ -11,7 +11,7 @@ AC_CONFIG_AUX_DIR(cfgaux) AM_INIT_AUTOMAKE(squid, 2.6-DEVEL) AM_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.61.2.2 $)dnl +AC_REVISION($Revision: 1.61.2.3 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AM_MAINTAINER_MODE @@ -460,6 +460,16 @@ fi ]) +AM_CONDITIONAL(USE_ESI, false) +AC_ARG_ENABLE(esi, + AC_HELP_STRING([--enable-esi],[Enable ESI for accelerators. Requires libexpat]), + ac_cv_use_esi=$enableval, ac_cv_use_esi=no) +AC_CACHE_CHECK(whether to enable ESI,ac_cv_use_esi, ac_cv_use_esi=no) +if test "$ac_cv_use_esi" = "yes" ; then + AM_CONDITIONAL(USE_ESI, true) + XTRA_LIBS="$XTRA_LIBS -lexpat" +fi + dnl This is a developer only option. Developers know how to set defines dnl dnl AC_ARG_ENABLE(mem-gen-trace, Index: squid/src/ESI.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/ESI.c,v retrieving revision 1.1.2.10 retrieving revision 1.1.2.11 diff -u -r1.1.2.10 -r1.1.2.11 --- squid/src/ESI.c 16 Aug 2002 08:33:00 -0000 1.1.2.10 +++ squid/src/ESI.c 18 Aug 2002 11:09:22 -0000 1.1.2.11 @@ -1,6 +1,6 @@ /* - * $Id: ESI.c,v 1.1.2.10 2002/08/16 08:33:00 rbcollins Exp $ + * $Id: ESI.c,v 1.1.2.11 2002/08/18 11:09:22 rbcollins Exp $ * * DEBUG: section 86 ESI processing * AUTHOR: Robert Collins @@ -23,7 +23,7 @@ * (at your option) any later version. * * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of + ; but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * @@ -34,6 +34,7 @@ */ #include "squid.h" +#include "expat.h" /* quick reference on behaviour here. * The ESI specification 1.0 requires the ESI processor to be able to @@ -54,9 +55,159 @@ } esiSegment; CBDATA_TYPE(esiSegment); -typedef struct _esiContext { - int remaining_esi_requests; - int completed_requests; +typedef struct _esiStreamContext esiStreamContext; +typedef struct _esiContext esiContext; +typedef struct _esiInclude esiInclude; + +/* TODO: split this out into separate files */ +/* Parsing: quick and dirty. ESI files are not valid XML, so a generic + * XML parser is not much use. Also we need a push parser not a pull + * parser, so LibXML is out. + * We don't have C++, but an interpreter pattern (possibly with a builder) + * is ideal for our needs + * so: we typedef each method, and manually build a vptr for each + * 'class'. Finally, we use a #define to call the 'methods'. + * TODO: make a builder for the parse tree. + * + * + * Interpreter methods: + * Render: May only ever be calle dafter Process returns PROCESS_COMPLETE. + * Renders the resulting content into a esiSegment chain. + * Process: returns the status of the node. + * COMPLETE - processing is complete, rendering may staret + * PENDING_WONTFAIL - process is incomplete, but the element *will* + * be able to be rendered given time. + * PENDING_MAYFAIL - processing is incomplete, and the element *may* + * fail to be able to rendered. + * FAILED - processing failed, return an error to the client. + */ + +/* TODO: remove invalid markup (silently swallow markup in invalid locations + * TODO: inline variables + * TODO: 3.3 choose when otherwise + * TODO: 3.4 try attempt except + * TODO: 3.6 remove + * TODO: 3.7 vars + * + * + * NOT TODO: esi:inline - out of scope. + */ + +typedef enum { + ESI_PROCESS_COMPLETE, + ESI_PROCESS_PENDING_WONTFAIL, + ESI_PROCESS_PENDING_MAYFAIL, + ESI_PROCESS_FAILED +} esiProcessResult_t; + +typedef enum { + ESI_KICK_FAILED, + ESI_KICK_PENDING, + ESI_KICK_SENT +} esiKick_t; + +typedef struct _esiElementType esiElementType; +typedef struct _esiElement esiElement; + +typedef void esiRender (void *, esiSegment *); /* render into the segment list */ +#define RENDER(foo, bar) foo->vptr->render(foo, bar) +typedef int esiAddElement (void *, esiElement *); /* add a child to an element */ +#define ADDELEMENT(foo, bar) foo->vptr->add(foo,bar) +typedef esiProcessResult_t esiProcessSelf (void *); /* process this element */ +#define PROCESS(foo) foo->vptr->process(foo) +struct _esiElementType { + esiRender *render; + esiAddElement *add; + esiProcessSelf *process; +}; +struct _esiElement { + esiElementType *vptr; +}; + +/* some core operators */ + +/* the methods for each type */ +static esiAddElement esiAddFail; +static esiProcessSelf esiProcessComplete; + +static esiRender esiCommentRender; + +static esiRender esiLiteralRender; + +static esiAddElement esiSequenceAdd; +static esiRender esiSequenceRender; +static esiProcessSelf esiSequenceProcess; + +static esiRender esiIncludeRender; +static esiProcessSelf esiIncludeProcess; +static void esiIncludeSubRequestDone (esiInclude *, esiStreamContext *, int); + +/* the types we have */ +typedef enum { + ESI_ELEMENT_NONE, + ESI_ELEMENT_INCLUDE, + ESI_ELEMENT_COMMENT +} esiElementType_t; + +static esiElementType _esiComment = {esiCommentRender,esiAddFail,esiProcessComplete}; +static esiElementType _esiLiteral = {esiLiteralRender,esiAddFail,esiProcessComplete}; +static esiElementType _esiSequence = {esiSequenceRender,esiSequenceAdd,esiSequenceProcess}; +static esiElementType _esiInclude = {esiIncludeRender, esiAddFail, esiIncludeProcess}; + +/* esiComment */ +typedef struct { + esiElementType *vptr; +} esiComment; +CBDATA_TYPE (esiComment); +static FREE esiCommentFree; +static esiElement * esiCommentNew(void); + +/* esiLiteral */ +typedef struct { + esiElementType *vptr; + /* optimise copies away later */ + esiSegment *buffer; +} esiLiteral; +CBDATA_TYPE (esiLiteral); +static FREE esiLiteralFree; +static esiElement * esiLiteralNew(const XML_Char *s, int len); + +/* esiSequence */ +typedef struct { + esiElementType *vptr; + esiElement ** elements; /* unprocessed or rendered nodes */ + int allocedcount; + size_t allocedsize; + int elementcount; + int processedcount; +} esiSequence; +CBDATA_TYPE (esiSequence); +static FREE esiSequenceFree; +static esiElement * esiSequenceNew(void); + +/* esiInclude */ +struct _esiInclude { + esiElementType *vptr; + struct { + int onerrorcontinue:1; /* on error return zero data */ + int failed:1; /* Failed to process completely */ + int finished:1; /* Finished getting subrequest data */ + } flags; + esiStreamContext *src; + esiStreamContext *alt; + esiSegment *srccontent; + esiSegment *altcontent; + esiContext *context; + char *srcurl, *alturl; +}; +CBDATA_TYPE (esiInclude); +static FREE esiIncludeFree; +static esiElement * esiIncludeNew (int attributes, const char **attr, esiContext *); + +struct _esiContext { + clientStreamNode *this; /* our stream node */ + clientHttpRequest *http; /* the request we are processing. HMM: cbdataReferencing this will result + in a circular reference, so we don't. Note: we are automatically freed when it is, so thats ok. */ struct { int passthrough:1; int oktosend:1; @@ -65,40 +216,61 @@ * regardless. Note that we don't fail midstream * because we buffer until we can not fail */ + int finishedtemplate:1; /* we've read the entire template */ + int parserinited:1; + int clientwantsdata:1; /* we need to satisfy a read request */ + int kicked:1; /* don't reenter the kick routine */ } flags; + err_type errorpage; /* if we error what page to use */ + http_status errorstatus; /* if we error, what code to return */ HttpReply *rep; /* buffered until we pass data downstream */ esiSegment *buffered; /* unprocessed data - for whatever reason */ esiSegment *incoming; esiSegment *outbound; /* processed data we are waiting to send, or for * potential errors to be resolved */ + esiSegment *outboundtail; /* our write segment */ size_t outbound_offset; /* the offset to the next character to send - * non zero if we haven't sent the entire segment * for some reason */ off_t readpos; /* the logical position we are reading from */ off_t pos; /* the logical position of outbound_offset in the data stream */ -} esiContext; + struct { + off_t unprocessed_start; /* the offset into the buffered chain of the + oldest data not processed + */ + off_t buffered_tokenstart; /* the offset into the buffered chain of the + parsers current symbol candidate + */ + esiElement *stack[10]; /* a stack of esi elements that are open */ + int stackdepth; /* self explanatory */ + XML_Parser p; /* our parser */ + } parserState; /* todo factor this off somewhere else; */ + esiElement *tree; +}; CBDATA_TYPE(esiContext); -typedef struct _esiStreamContext { - clientHttpRequest *http; - esiContext *esi; - esiSegment localbuf; -} esiStreamContext; +struct _esiStreamContext { + int finished; + esiInclude *include; + esiSegment localbuffer; + esiSegment *buffer; +}; CBDATA_TYPE (esiStreamContext); /* Local functions */ /* esiContext */ static FREE esiContextFree; -static esiContext *esiContextNew(HttpReply *); +static esiContext *esiContextNew(HttpReply *, clientStreamNode *, clientHttpRequest *); static void esiContextFreeResources (esiContext *); static size_t esiSend (clientStreamNode *this, esiContext *,clientHttpRequest *http); +static esiProcessResult_t esiProcess (esiContext *); /* esiStreamContext */ static FREE esiStreamContextFree; -static esiStreamContext *esiStreamContextNew (clientHttpRequest *, esiContext *); +static esiStreamContext *esiStreamContextNew (esiInclude *); /* esiSegment */ static void esiSegmentFreeList (esiSegment **head); @@ -106,11 +278,64 @@ static CSCB esiBufferRecipient; /* static void esiStartSub (void); */ static void esiFail (clientStreamNode *this, esiContext *context, clientHttpRequest *http); +static esiKick_t esiKick (esiContext *context); -/* ESI TODO: +/* ESI TO CONSIDER: * 1. retry failed upstream requests */ + +esiKick_t +esiKick (esiContext *context) +{ + assert (context); + if (context->flags.kicked) + return ESI_KICK_PENDING; + else + context->flags.kicked = 1; + /* Something has occured. Process any remaining nodes */ + if (!context->flags.finished) + /* Process some of our data */ + switch (esiProcess (context)) { + case ESI_PROCESS_COMPLETE: + debug (86,1)("esiKick: esiProcess OK\n");break; + case ESI_PROCESS_PENDING_WONTFAIL: + debug (86,1)("esiKick: esiProcess PENDING OK\n");break; + case ESI_PROCESS_PENDING_MAYFAIL: + debug (86,1)("esiKick: esiProcess PENDING UNKNOWN\n");break; + case ESI_PROCESS_FAILED: + debug (86,0)("esiKick: esiProcess FAILED\n"); + /* this can not happen - processing can't fail until we have data, + * and when we come here we have sent data to the client + */ + if (context->pos == 0) + esiFail (context->this, context, context->http); + context->flags.kicked = 0; + return ESI_KICK_FAILED; + } + + /* Render if we can to get maximal sent data */ + assert (context->tree || context->flags.error); + + if (!context->outbound) + context->outbound = context->outboundtail = cbdataAlloc (esiSegment); + if (!context->flags.error) + RENDER (context->tree, context->outboundtail); + /* TODO: fixup this outboundtail dross a little */ + while (context->outboundtail->next) + context->outboundtail = context->outboundtail->next; + /* Is there data to send? */ + if (esiSend (context->this, context, context->http)) { + /* some data was sent. we're finished until the next read */ + context->flags.kicked = 0; + return ESI_KICK_SENT; + } + + context->flags.kicked = 0; + /* nothing to send */ + return ESI_KICK_PENDING; +} + /* request from downstream for more data */ void @@ -134,40 +359,48 @@ cbdataReferenceDone (context); return; } - - /* Ok, not passing through. Is there data to send? */ - if (context->flags.oktosend && context->outbound && - context->outbound_offset < context->outbound->len) { - /* Yes! Send it without asking for more upstream */ - /* memcopying because the client provided the buffer */ - /* TODO: skip data until pos == next->readoff; */ - clientStreamNode *next = this->node.next->data; - size_t len = next->readlen < context->outbound->len - context->outbound_offset ? - next->readlen : context->outbound->len - context->outbound_offset; - assert (context->pos == next->readoff); - xmemcpy (next->readbuf, &context->outbound->buf[context->outbound_offset], len); - if (len + context->outbound_offset == context->outbound->len) { - esiSegment *temp = context->outbound->next; - /* remove the used buffer */ - context->outbound_offset = 0; - cbdataFree (context->outbound); - context->outbound = temp; - } - context->pos += len; - clientStreamCallback (this, http, context->rep, next->readbuf, len); + + context->flags.clientwantsdata = 1; + debug (86,5)("esiStreamRead: Client now wants data\n"); + + /* Ok, not passing through */ + switch (esiKick (context)) { + case ESI_KICK_FAILED: + /* this can not happen - processing can't fail until we have data, + * and when we come here we have sent data to the client + */ + cbdataReferenceDone (context); + return; + case ESI_KICK_SENT: cbdataReferenceDone (context); return; + case ESI_KICK_PENDING: + break; } + /* Nothing to send */ + + if (context->flags.oktosend && context->flags.finishedtemplate && + ! context->flags.finished) { + /* we've started sending, finished reading, but not finished + * processing. stop here, a callback will resume the stream + * flow + */ + debug (86,1) ("esiStreamRead: Waiting for async resume of esi processing\n"); + cbdataReferenceDone (context); + return; + } + if (context->flags.oktosend && context->flags.finished) { - /* We've finished process, and there is no more data buffered */ - debug (0,0)("Telling recipient EOF\n"); + assert (!context->outbound); + /* We've finished processing, and there is no more data buffered */ + debug (0,0)("Telling recipient EOF on READ\n"); clientStreamCallback (this, http, NULL, NULL, 0); cbdataReferenceDone (context); return; } - /* no data that is ready to send? well, lets get some */ + /* no data that is ready to send, and still reading? well, lets get some */ /* secure a buffer */ if (!context->incoming) { /* create a new buffer segment */ @@ -202,9 +435,11 @@ if (context->flags.oktosend && context->flags.finished && !(context->outbound && context->outbound_offset < context->outbound->len)) { cbdataReferenceDone (context); + debug (86,1) ("Telling recipient EOF on STATUS\n"); return STREAM_UNPLANNED_COMPLETE; /* we don't know lengths in advance */ } /* ?? RC: we can't be aborted / fail ? */ + cbdataReferenceDone (context); return STREAM_NONE; } @@ -232,8 +467,21 @@ esiSend (clientStreamNode *this, esiContext *context, clientHttpRequest *http) { /* send any processed data */ + + /* trim leading empty buffers ? */ + while (context->outbound && context->outbound->next && !context->outbound->len) { + esiSegment *temp = context->outbound->next; + cbdataFree (context->outbound); + context->outbound = temp; + } + + if (!context->flags.clientwantsdata) { + debug (86,5)("esiSend: Client does not want data - not sending anything\n"); + return 0; + } + if (context->flags.oktosend && (context->rep || (context->outbound && - context->outbound_offset <= context->outbound->len))) { + context->outbound->len && (context->outbound_offset <= context->outbound->len)))) { /* Yes! Send it without asking for more upstream */ /* memcopying because the client provided the buffer */ /* TODO: skip data until pos == next->readoff; */ @@ -241,10 +489,12 @@ size_t len = 0; cbdataReference (context); if (context->outbound) - len = next->readlen < context->outbound->len - context->outbound_offset ? - next->readlen : context->outbound->len - context->outbound_offset; - /* prevent corruption on range requests, even thought we don't support them yet */ + len = next->readlen < (context->outbound->len - context->outbound_offset) ? + next->readlen : (context->outbound->len - context->outbound_offset); + /* prevent corruption on range requests, even though we don't support them yet */ assert (context->pos == next->readoff); + /* We must send data or a reply */ + assert (len != 0 || context->rep != NULL); if (len) { xmemcpy (next->readbuf, &context->outbound->buf[context->outbound_offset], len); if (len + context->outbound_offset == context->outbound->len) { @@ -255,15 +505,27 @@ context->outbound = temp; } context->pos += len; + if (!context->outbound) + context->outboundtail = NULL; + /* trim leading empty buffers ? */ + while (context->outbound && context->outbound->next && !context->outbound->len) { + esiSegment *temp = context->outbound->next; + cbdataFree (context->outbound); + context->outbound = temp; + } } + context->flags.clientwantsdata = 0; + debug (86,5)("esiSend: Client no longer wants data \n"); clientStreamCallback (this, http, context->rep, next->readbuf, len); if (len == 0) len = -1; /* tell the caller we sent something */ if (context->rep) context->rep = NULL; /* freed downstream */ cbdataReferenceDone (context); + debug (86,5)("esiSend returning %d\n",len); return len; } + debug (86,5)("esiSend returning 0\n"); return 0; } @@ -296,20 +558,26 @@ assert (this->node.next != NULL); if (!this->data) /* setup ESI context from reply headers */ - this->data = esiContextNew(rep); + this->data = esiContextNew(rep, this, http); context = cbdataReference(this->data); - /* Finished all ESI processing. All remaining data gets untouched. - * Mainly used when an error has been detected to prevent ESI processing the error body + + /* Skipping all ESI processing. All remaining data gets untouched. + * Mainly used when an error or other non-ESI processable entity + * has been detected to prevent ESI processing the error body */ if (context->flags.passthrough) { cbdataReferenceDone(context); clientStreamCallback (this, http, rep, body_data, body_size); return; } + + /* once we finish the template, we *cannot* return here */ + assert (!context->flags.finishedtemplate); /* Can we generate any data ?*/ if (body_data) { + /* Increase our buffer area with incoming data */ assert (body_size <= HTTP_REQBUF_SZ); debug (86,1 )("ESIProcessStream found %u bytes of body data at offset %ld\n", body_size, this->readoff); /* secure the data for later use */ @@ -353,87 +621,42 @@ } } - debug(86, 0) ("esiProcessStream: Will be checking for ESI content here - %d sub requests remaining\n", context->remaining_esi_requests); - - /* HACK to test buffering */ /* EOF / Read error / aborted entry */ - if (rep == NULL && body_data == NULL && body_size == 0 && !context->flags.finished) { - /* Bit test the entry for aborts */ - /* flush the esi processor */ - debug (0,0)("Flushing now \n"); - context->flags.oktosend = 1; - context->flags.finished = 1; - context->outbound = context->buffered; - context->buffered = NULL; - context->incoming = NULL; - cbdataReferenceDone(context); - esiProcessStream (this, http, rep, body_data, body_size); - return; - } - - /* send any processed data */ - if (esiSend (this, context, http)) { - /* some data was sent. we're finished until the next read */ - cbdataReferenceDone (context); - return; + if (rep == NULL && body_data == NULL && body_size == 0 && !context->flags.finishedtemplate) { + /* TODO: get stream status to test the entry for aborts */ + /* else flush the esi processor */ + debug (86,1)("Finished reading upstream data\n"); + /* This is correct */ + context->flags.finishedtemplate = 1; } + switch (esiKick (context)) { + case ESI_KICK_FAILED: + /* this can not happen - processing can't fail until we have data, + * and when we come here we have sent data to the client + */ + cbdataReferenceDone (context); + return; + case ESI_KICK_SENT: + cbdataReferenceDone (context); + return; + case ESI_KICK_PENDING: + break; + } + /* ok.. no data sent, try to pull more data in from upstream. * FIXME: Don't try this if we have finished reading the template */ - assert (context->incoming && context->incoming->len < HTTP_REQBUF_SZ); - clientStreamRead (this, http, context->readpos, HTTP_REQBUF_SZ - context->incoming->len, - &context->incoming->buf[context->incoming->len]); - -// esiFail (this, context, http); - - cbdataReferenceDone (context); -#if 0 - - /* HACK to test buffering */ - /* EOF / Read error / aborted entry */ - if (rep == NULL && body_data == NULL && body_size == 0) { - /* Bit test the entry for aborts */ - /* flush the esi processor */ - context->flags.oktosend = 1; - cbdataReferenceDone(context); - esiProcessStream (this, http, rep, body_data, body_size); + if (!context->flags.finishedtemplate) { + assert (context->incoming && context->incoming->len < HTTP_REQBUF_SZ); + clientStreamRead (this, http, context->readpos, HTTP_REQBUF_SZ - context->incoming->len, + &context->incoming->buf[context->incoming->len]); + cbdataReferenceDone (context); return; } -#endif - - - if (0) - /* Kick off a request */ - if (context->remaining_esi_requests--) - { - HttpHeader tempheaders; - esiStreamContext *esiStream = esiStreamContextNew (http, context); - assert (esiStream != NULL); - httpHeaderInit (&tempheaders, hoRequest); - httpHeaderPutStr (&tempheaders, HDR_SURROGATE_CAPABILITY, "Surrogate/1.0"); - - if (clientBeginRequest(METHOD_GET, "http://192.168.0.2/ESI_fragment_1.txt", esiBufferRecipient, esiStream, &tempheaders, esiStream->localbuf.buf, HTTP_REQBUF_SZ)) { - debug (86,0 ) ("starting new ESI subrequest failed\n"); - } - httpHeaderClean (&tempheaders); - } - - /* EOF / Read error / aborted entry */ - if (rep == NULL && body_data == NULL && body_size == 0) { - /* Bit test the entry for aborts */ - /* flush the esi processor */ - } - - if (0) - // send an error - { - esiFail (this, context, http); - cbdataReferenceDone(context); - return; - } -// cbdataReferenceDone(context); + debug (86,1)("esiProcessStream: no data to send, no data to read, awaiting a callback\n"); + cbdataReferenceDone(context); } void @@ -445,23 +668,307 @@ } esiContext * -esiContextNew (HttpReply *rep) +esiContextNew (HttpReply *rep, clientStreamNode *this, clientHttpRequest *http) { esiContext *rv; CBDATA_INIT_TYPE_FREECB(esiContext, esiContextFree); rv = cbdataAlloc(esiContext); assert (rep); - /* ESI TODO: remove this hardcoded test parameter */ - rv->remaining_esi_requests = 2; rv->rep = rep; if (esiAlwaysPassthrough(rep->sline.status)) { rv->flags.passthrough = 1; } else { - /* TODO: remove Etag, content-length headers */ + /* remove specific headers for ESI to prevent + * downstream cache confusion */ + HttpHeader *hdr = &rep->header; + httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); + httpHeaderDelById(hdr, HDR_ETAG); + httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); + httpHeaderDelById(hdr, HDR_CONTENT_MD5); + /* Tree may use this */ + CBDATA_INIT_TYPE(esiSegment); + rv->tree = esiSequenceNew(); + rv->this = this; + rv->http = http; + rv->flags.clientwantsdata = 1; + debug (86,5)("esiContextNew: Client wants data (always created during reply cycle\n"); } return rv; } +typedef struct { + char *buf; /* not owned by us. */ + size_t len; +} inplacetoken; + +inplacetoken +esiFindToken (esiSegment *buffer, off_t startat); + +typedef enum { + TOKEN_START, /* looking for < */ + TOKEN_QUOTED, /* looking for a match to the start quote */ + TOKEN_UNQUOTED, /* looking for " or ' or > */ + TOKEN_COMPLETE /* finished */ +} tokenstate; + +int +esiAddLiteral (esiContext *context, const XML_Char *s, int len); +void start(void *data, const XML_Char *el, const char **attr); +void end(void *data, const XML_Char *el); +void esiExpatDefault (void *data, const XML_Char *s, int len); + +static esiElementType_t +esiIdentifyElement (const XML_Char *el) ; + +esiElementType_t +esiIdentifyElement (const XML_Char *el) +{ + assert (el); + if (strlen (el) < 5 || strncmp (el, "esi:", 4)) + return ESI_ELEMENT_NONE; + if (!strncmp (el + 4, "comment",7)) + return ESI_ELEMENT_COMMENT; + if (!strncmp (el + 4, "include",7)) + return ESI_ELEMENT_INCLUDE; + return ESI_ELEMENT_NONE; +} + +void +start(void *data,const XML_Char *el, const char **attr) +{ + int i; + int ellen = strlen (el); + char localbuf [HTTP_REQBUF_SZ]; + esiContext *context = data; + esiElement *element; + int specifiedattcount = XML_GetSpecifiedAttributeCount (context->parserState.p) * 2; + char *pos; + assert (ellen < sizeof (localbuf)); /* prevent unexpected overruns. */ + + debug (86, 5)("esiExpatState: element '%s' with %d tags\n", el, specifiedattcount); + + switch (esiIdentifyElement (el)) { + case ESI_ELEMENT_NONE: + /* Spit out elements we aren't interested in */ + localbuf[0] = '<'; + localbuf[1] = '\0'; + assert (xstrncpy (&localbuf[1], el, sizeof(localbuf) - 2)); + pos = localbuf + strlen (localbuf); + + for (i = 0; attr[i] && i < specifiedattcount; i += 2) { + *pos++ = ' '; + /* TODO: handle this gracefully */ + assert (xstrncpy (pos, attr[i], sizeof(localbuf) + (pos - localbuf))); + pos += strlen (pos); + *pos++ = '='; + *pos++ = '\''; + assert (xstrncpy (pos, attr[i + 1], sizeof(localbuf) + (pos - localbuf))); + pos += strlen (pos); + *pos++ = '\''; + } + *pos++ = '>'; + *pos = '\0'; + + if (esiAddLiteral (context, localbuf, pos - localbuf)) + context->flags.error = 1; + break; + case ESI_ELEMENT_COMMENT: + /* Put on the stack to allow skipping of 'invalid' markup */ + assert (context->parserState.stackdepth <11); + element = esiCommentNew (); + if (ADDELEMENT (context->parserState.stack[context->parserState.stackdepth-1], element)) { + debug (86,1)("esiProcess: failed to add esi node, probable error in ESI template\n"); + context->flags.error = 1; + cbdataFree (element); + break; + } else { + /* added ok, push onto the stack */ + context->parserState.stack[context->parserState.stackdepth++] = element; + } + break; + case ESI_ELEMENT_INCLUDE: + /* Put on the stack to allow skipping of 'invalid' markup */ + assert (context->parserState.stackdepth <11); + element = esiIncludeNew (specifiedattcount, attr, context); + if (ADDELEMENT (context->parserState.stack[context->parserState.stackdepth-1], element)) { + debug (86,1)("esiProcess: failed to add esi node, probable error in ESI template\n"); + context->flags.error = 1; + cbdataFree (element); + break; + } else { + /* added ok, push onto the stack */ + context->parserState.stack[context->parserState.stackdepth++] = element; + } + break; + } + + + debug (86,1)("esi stack depth %d\n",context->parserState.stackdepth); + +} /* End of start handler */ + +void +end(void *data, const char *el) +{ + int ellen = strlen (el); + char localbuf [HTTP_REQBUF_SZ]; + char *pos; + esiContext *context = data; + + switch (esiIdentifyElement (el)) { + case ESI_ELEMENT_NONE: + assert (ellen < sizeof (localbuf)); /* prevent unexpected overruns. */ + /* Add elements we aren't interested in */ localbuf[0] = '<'; + localbuf[1] = '/'; + assert (xstrncpy (&localbuf[2], el, sizeof(localbuf) - 3)); + pos = localbuf + strlen (localbuf); + *pos++ = '>'; + *pos = '\0'; + if (esiAddLiteral (context, localbuf, pos - localbuf)) + context->flags.error = 1; + break; + case ESI_ELEMENT_COMMENT: + /* pop of the stack */ + --context->parserState.stackdepth; + break; + case ESI_ELEMENT_INCLUDE: + /* pop of the stack */ + --context->parserState.stackdepth; + break; + } +} /* End of end handler */ + +void +esiExpatDefault (void *data, const XML_Char *s, int len) +{ + esiContext *context = data; + /* handle and skipped data */ + if (esiAddLiteral (context, s, len)) + context->flags.error = 1; +} + +int +esiAddLiteral (esiContext *context, const XML_Char *s, int len) +{ + /* handle any skipped data */ + esiElement *element; + assert (len); + debug (86,5)("literal length is %d\n", len); + /* give a literal to the current element */ + assert (context->parserState.stackdepth <11); + element = esiLiteralNew (s, len); + if (ADDELEMENT (context->parserState.stack[context->parserState.stackdepth-1], element)) { + debug (86,1)("esiProcess: failed to add esi node, probable error in ESI template\n"); + cbdataFree (element); + return -1; + } + return 0; +} + +esiProcessResult_t +esiProcess (esiContext *context) +{ + /* parsing: + * read through buffered, skipping plain text, and skipping any + * <...> entry that is not an flags.finished == 0); + + if (!context->parserState.stackdepth) { + debug (86,1)("empty parser stack, inserting the top level node\n"); + assert (context->tree); + context->parserState.stack[context->parserState.stackdepth++] = context->tree; + } + + if (context->rep && !context->flags.parserinited) + { + /* TODO: grab the document encoding from the headers */ + context->parserState.p = XML_ParserCreate(NULL); + XML_SetUserData (context->parserState.p, context); + XML_SetElementHandler(context->parserState.p, start, end); + XML_SetDefaultHandler(context->parserState.p, esiExpatDefault); + } + + /* we have data */ + if (context->buffered) { + /* we don't keep any data around */ + + while (context->buffered) { + esiSegment *temp; + if (context->buffered->len) { + if (! XML_Parse(context->parserState.p, context->buffered->buf, context->buffered->len, context->flags.finishedtemplate)) { + context->errorpage = ERR_UNSUP_REQ; + context->errorstatus = HTTP_INTERNAL_SERVER_ERROR; + /* TODO: transmit this error the user */ + debug (86,0)("esiProcess: Parse error at line %d:\n%s\n", + XML_GetCurrentLineNumber(context->parserState.p), + XML_ErrorString(XML_GetErrorCode(context->parserState.p))); + return ESI_PROCESS_FAILED; + } + if (context->flags.error) { + /* TODO" check if we should reset flags.failed */ + context->errorpage = ERR_UNSUP_REQ; + context->errorstatus = HTTP_INTERNAL_SERVER_ERROR; + return ESI_PROCESS_FAILED; + } + + } + temp = context->buffered; + context->buffered = temp->next; + cbdataFree (temp); + } + /* Tel the read code to allocate a new buffer */ + context->incoming = NULL; + } + + /* ok, we've done all we can with the data. What can we process now? + */ + { + esiProcessResult_t status = PROCESS(context->tree); + switch (status) { + case ESI_PROCESS_COMPLETE: + debug (86,1)("esiProcess: tree Processed OK\n"); + break; + case ESI_PROCESS_PENDING_WONTFAIL: + debug (86,1)("esiProcess: tree Processed PENDING OK\n"); + break; + case ESI_PROCESS_PENDING_MAYFAIL: + debug (86,1)("eseProcess: tree Processed PENDING UNKNOWN\n"); + break; + case ESI_PROCESS_FAILED: + debug (86,0)("esiProcess: tree Processed FAILED\n"); + /* ESI TODO: create a new error page */ + context->errorpage = ERR_UNSUP_REQ; + context->errorstatus = HTTP_INTERNAL_SERVER_ERROR; + return ESI_PROCESS_FAILED; + break; + } + if (status != ESI_PROCESS_PENDING_MAYFAIL && context->flags.finishedtemplate){ + /* We've read the entire template, and no nodes will + * return failure + */ + debug (86,1)("esiProcess, request will succeed\n"); + context->flags.oktosend = 1; + } + + if (status == ESI_PROCESS_COMPLETE && context->flags.finishedtemplate) { + /* we've finished all processing. Render and send. */ + debug (86,1)("esiProcess, processing complete\n"); + context->flags.finished = 1; + } + + } + + if (!context->flags.finishedtemplate && !context->incoming) { + context->incoming = context->buffered = cbdataAlloc (esiSegment); + } + + return ESI_PROCESS_COMPLETE; /* because we have no callbacks */ +} + void esiSegmentFreeList (esiSegment **head) { while (*head) { @@ -477,6 +984,11 @@ httpReplyDestroy (context->rep); context->rep = NULL; } + if (context->tree) + cbdataFree (context->tree); + if (context->flags.parserinited) { + XML_ParserFree (context->parserState.p); + } esiSegmentFreeList (&context->buffered); esiSegmentFreeList (&context->outbound); /* don't touch incoming, it's a pointer into buffered anyway */ @@ -500,7 +1012,7 @@ /* don't honour range requests - for errors we sent it all */ context->flags.error = 1; /* create an error object */ - err = clientBuildError(ERR_TOO_BIG, HTTP_FORBIDDEN, NULL, + err = clientBuildError(context->errorpage, context->errorstatus, NULL, http->conn ? &http->conn->peer.sin_addr : &no_addr, http->request); context->rep = errorBuildReply (err); errorprogress = context->rep->body.mb.size; @@ -510,23 +1022,19 @@ while (errorprogress < context->rep->body.mb.size) { size_t amount; - esiSegment *where; if (!context->outbound) { - context->outbound = cbdataAlloc (esiSegment); - where = context->outbound; + context->outboundtail = context->outbound = cbdataAlloc (esiSegment); } else { /* add to tail */ - where = context->outbound; - while (where->next) - where = where->next; - where->next = cbdataAlloc (esiSegment); - where = where->next; + assert (context->outboundtail->next == NULL); + context->outboundtail->next = cbdataAlloc (esiSegment); + context->outboundtail = context->outboundtail->next; } amount = context->rep->body.mb.size - errorprogress; if (amount > HTTP_REQBUF_SZ) amount = HTTP_REQBUF_SZ; - xmemcpy (where->buf, context->rep->body.mb.buf + errorprogress, amount); - where->len = amount; + xmemcpy (context->outboundtail->buf, context->rep->body.mb.buf + errorprogress, amount); + context->outboundtail->len = amount; errorprogress += amount; } /* the esiCode now thinks that the error is the outbound, and all processing has finished. */ @@ -559,61 +1067,75 @@ assert (node->data != NULL); assert (node->node.next == NULL); assert (http->conn == NULL); - - esiStream = cbdataReference (node->data); - debug (86,0) ("esiBufferRecipient\n"); + esiStream = cbdataReference (node->data); + /* If segments become more flexible, ignore this */ + assert (body_size <= sizeof(esiStream->localbuffer.buf)); + assert (!esiStream->finished); + + debug (86,5) ("esiBufferRecipient rep %p body %p len %d\n", rep, body_data, body_size); /* handle data - ESI TODO */ /* basic outline follows */ /* trivial case */ if (http->out.offset != 0) { assert(rep == NULL); - /* Avoid copying to MemBuf if we know "rep" is NULL, and we only have a body */ - http->out.offset += body_size; - /* give the data to the final user - the ESI processor in this case, - * once used, schedule another read (if apporpriate) via - * - */ - - /* As we are a non-blocking stream, we can simply call: - return; - */ } else { -/* push the headers and then the data through to -wherever-. Once the recipient has - * done it's thing, reschedule more data if appropriate */ -#if 0 - MemBuf mb; - /* write headers and/or body if any */ - assert(rep || (body_data && body_size)); - /* init mb; put status line and headers if any */ -#endif if (rep) { - //mb = httpReplyPack(rep); -// http->out.offset += rep->hdr_sz; + if (rep->sline.status != HTTP_OK) { + httpReplyDestroy(rep); + rep = NULL; + esiIncludeSubRequestDone (esiStream->include, esiStream, 0); + esiStream->finished = 1; + cbdataReferenceDone (esiStream); + httpRequestFree (http); + return; + } #if HEADERS_LOG - // should be done in the store rather than ever recipient? + /* should be done in the store rather than ever recipient? */ headersLog(0, 0, http->request->method, rep); #endif httpReplyDestroy(rep); rep = NULL; -// } else { -// memBufDefInit(&mb); - } - if (body_data && body_size) { - http->out.offset += body_size; -// memBufAppend(&mb, body_data, body_size); } - /* write */ -// comm_write_mbuf(fd, mb, clientWriteComplete, http); - /* if we don't do it, who will? */ } + if (body_data && body_size) { + http->out.offset += body_size; + + if (body_data >= esiStream->localbuffer.buf && + body_data < &esiStream->localbuffer.buf[sizeof(esiStream->localbuffer.buf)]) { + /* original static buffer */ + if (body_data != esiStream->localbuffer.buf) { + /* But not the start of it */ + xmemmove (esiStream->localbuffer.buf, body_data, body_size); + } + esiStream->localbuffer.len = body_size; + } else { + assert (esiStream->buffer != NULL); + esiStream->buffer->len = body_size; + } + } + + /* EOF / Read error / aborted entry */ + if (rep == NULL && body_data == NULL && body_size == 0) { + /* TODO: get stream status to test the entry for aborts */ + debug (86,1)("Finished reading upstream data in subrequest\n"); + esiIncludeSubRequestDone (esiStream->include, esiStream, 1); + esiStream->finished = 1; + cbdataReferenceDone (esiStream); + httpRequestFree (http); + return; + } + + /* after the write to the user occurs, (ie here, or in a callback) * we call */ if (clientHttpRequestStatus(-1, http)) { + /* TODO: Does this if block leak htto ? */ esiStreamContext *temp = esiStream; - /* ESI TODO: Tell the initiator we've failed */ + esiIncludeSubRequestDone (esiStream->include, esiStream, 0); + esiStream->finished = 1; cbdataReferenceDone (esiStream); cbdataFree (temp); /* free the request */ return; @@ -621,63 +1143,433 @@ switch (clientStreamStatus (node, http)) { case STREAM_UNPLANNED_COMPLETE: /* fallthru ok */ case STREAM_COMPLETE: /* ok */ - // usertell request finished - debug (86,0)("ESI subrequest finished OK\n"); + debug (86,1)("ESI subrequest finished OK\n"); + esiIncludeSubRequestDone (esiStream->include, esiStream, 1); + esiStream->finished = 1; cbdataReferenceDone (esiStream); httpRequestFree (http); return; case STREAM_FAILED: - // usertell request failed - debug (86,0)("ESI subrequest failed transfer\n"); + debug (86,1)("ESI subrequest failed transfer\n"); + esiIncludeSubRequestDone (esiStream->include, esiStream, 0); + esiStream->finished = 1; cbdataReferenceDone (esiStream); httpRequestFree (http); return; - case STREAM_NONE: - /* More data will be coming from primary server; register with - * storage manager. */ - clientStreamRead (node, - http, http->out.offset, - HTTP_REQBUF_SZ, - esiStream->localbuf.buf); + case STREAM_NONE: { + if (!esiStream->buffer) { + esiStream->buffer = &esiStream->localbuffer; + } + while (esiStream->buffer->next) esiStream->buffer = esiStream->buffer->next; + if (esiStream->buffer->len) { + esiStream->buffer->next = cbdataAlloc (esiSegment); + esiStream->buffer = esiStream->buffer->next; + } + + /* now just read into 'buffer' */ + clientStreamRead (node, + http, http->out.offset, + sizeof (esiStream->buffer->buf), + esiStream->buffer->buf); debug (86,0)("Requested more data for ESI subrequest\n"); + } break; default:fatal ("Hit unreachable code in esiBufferRecipient\n"); } cbdataReferenceDone (esiStream); } -#if 0 -void -esiStartSub (void) -{ - StoreEntry *e, *old_e; - char *url; - const cache_key *key; - request_t *req; - - - url = xstrdup("http://192.168.0.2:3128/ESI_fragment_1.txt"); -} -#endif - /* esiStream functions */ void esiStreamContextFree (void *data) { esiStreamContext *esiStream = data; assert (esiStream); - cbdataReferenceDone (esiStream->http); - cbdataReferenceDone (esiStream->esi); - debug (0,0)("Freeing stream context\n"); + cbdataReferenceDone (esiStream->include); + esiSegmentFreeList (&esiStream->localbuffer.next); + debug (86,1)("Freeing stream context\n"); } esiStreamContext * -esiStreamContextNew (clientHttpRequest *http, esiContext *esi) +esiStreamContextNew (esiInclude *include) { esiStreamContext *rv = NULL; CBDATA_INIT_TYPE_FREECB(esiStreamContext, esiStreamContextFree); rv = cbdataAlloc(esiStreamContext); - rv->http = cbdataReference (http); - rv->esi = cbdataReference (esi); + rv->include = cbdataReference (include); return rv; } + + +/* Implementation of esiElements */ + +/* 'inherited' methods */ + +/* Don't accept children */ +int +esiAddFail (void *data, esiElement *element) { + debug (86,5)("esiAddFail: Failed for %p\n",data); + return -1; +} + +/* always complete processing */ +esiProcessResult_t +esiProcessComplete (void *data) { + debug (86,5) ("esiProcessComplete: Processed %p\n",data); + return ESI_PROCESS_COMPLETE; +} + +/* esiComment */ +void +esiCommentFree (void *data) +{ + esiComment *this = data; + debug (86,5)("esiCommentFree %p\n", this); +} + +esiElement * +esiCommentNew () +{ + esiComment *rv; + CBDATA_INIT_TYPE_FREECB(esiComment, esiCommentFree); + rv = cbdataAlloc (esiComment); + rv->vptr = &_esiComment; + return (esiElement *) rv; +} + +void +esiCommentRender (void *data, esiSegment *output) +{ + /* Comments do nothing dude */ + debug (86, 5)("esiCommentRender: Rendering comment %p\n", data); +} + +/* esiLiteral */ +void +esiLiteralFree (void *data) +{ + esiLiteral *this = data; + debug (86, 5) ("esiLiteralFree %p\n", this); + esiSegmentFreeList (&this->buffer); +} + +/* precondition: the buffer chain has at least start + length bytes of data + */ +esiElement * +esiLiteralNew (const XML_Char *s, int length) +{ + esiLiteral *rv; + esiSegment *local; + off_t start; + CBDATA_INIT_TYPE_FREECB(esiLiteral, esiLiteralFree); + rv = cbdataAlloc (esiLiteral); + rv->vptr = &_esiLiteral; + assert (s); + rv->buffer = cbdataAlloc (esiSegment); + local = rv->buffer; + start = 0; + while (length > 0) { + off_t len = sizeof (local->buf) - local->len; + if (len > length) + len = length; + xmemcpy (&local->buf[local->len], &s[start], len); + local->len += len; + length -= len; + s += len; + if (local->len == sizeof (local->buf) && length){ + local->next = cbdataAlloc (esiSegment); + local=local->next; + } + } + return (esiElement *)rv; +} + +void +esiLiteralRender (void *data, esiSegment *output) +{ + /* append the entire chain */ + esiLiteral *this = data; + esiSegment *myout = output; + assert (myout->next == NULL); + myout->next = this->buffer; + this->buffer = NULL; +} + +/* esiSequence */ +void +esiSequenceFree (void *data) +{ + esiSequence *this = data; + int i; + debug (86,1)("esiSequenceFree %p\n", this); + for (i = 0; i < this->elementcount; ++i) { + cbdataFree(this->elements[i]); + } + if (this->elements) + memFreeBuf (this->allocedsize, this->elements); +} + +esiElement * +esiSequenceNew () +{ + esiSequence *rv; + CBDATA_INIT_TYPE_FREECB(esiSequence, esiSequenceFree); + rv = cbdataAlloc (esiSequence); + rv->vptr = &_esiSequence; + rv->elements = NULL; + return (esiElement *)rv; +} + +void +esiSequenceRender (void *data, esiSegment *output) +{ + /* append all processed elements, and trim processed and rendered elements */ + esiSequence *this = data; + int i; + assert (output->next == NULL); + debug (86,5)("esiSequenceRender: rendering %d elements\n", this->processedcount); + for (i = 0; i < this->processedcount; ++i) { + RENDER (this->elements[i], output); + /* FIXME: pass a esiSegment ** ? */ + while (output->next) + output = output->next; + cbdataFree (this->elements[i]); + } + if (this->processedcount) { + xmemmove (this->elements, &this->elements[this->processedcount], (this->elementcount - this->processedcount) * sizeof (esiElement *)); + } + this->elementcount -= this->processedcount; + this->processedcount = 0; + assert (output->next == NULL); +} + +int +esiSequenceAdd (void *data, esiElement *element) +{ + /* add an element to the output list */ + esiSequence *this = data; + this->elements = memReallocBuf (this->elements, ++this->elementcount * sizeof (esiElement *), + &this->allocedsize); + assert (this->elements); + this->allocedcount = this->elementcount; + this->elements[this->elementcount - 1] = element; + debug (86,3)("esiSequenceAdd: Added a new element, elements = %d\n", this->elementcount); + return 0; +} + +esiProcessResult_t +esiSequenceProcess (void *data) +{ + /* process as much of the list as we can, stopping only on + * faliures + */ + esiSequence *this = data; + int i; + esiProcessResult_t rv = ESI_PROCESS_COMPLETE; + for (i = this->processedcount; i < this->elementcount; ++i) { + switch (PROCESS(this->elements[i])){ + case ESI_PROCESS_COMPLETE: + debug (86,5)("esiSequenceProcess: element Processed OK\n"); + if (i == this->processedcount) + /* another completely ready */ + ++this->processedcount; + break; + case ESI_PROCESS_PENDING_WONTFAIL: + debug (86,5)("esiSequenceProcess: element Processed PENDING OK\n"); + if (rv < ESI_PROCESS_PENDING_WONTFAIL) + rv = ESI_PROCESS_PENDING_WONTFAIL; + break; + case ESI_PROCESS_PENDING_MAYFAIL: + debug (86,5)("eseSequenceProcess: element Processed PENDING UNKNOWN\n"); + if (rv < ESI_PROCESS_PENDING_MAYFAIL) + rv = ESI_PROCESS_PENDING_MAYFAIL; + break; + case ESI_PROCESS_FAILED: + debug (86,5)("esiSequenceProcess: elemen Processed FAILED\n"); + return ESI_PROCESS_FAILED; + break; + } + } + assert (rv != ESI_PROCESS_COMPLETE || this->processedcount == this->elementcount); + return rv; +} + +/* esiInclude */ +void +esiIncludeFree (void *data) +{ + esiInclude *this = data; + debug (86,5)("esiIncludeFree %p\n", this); + esiSegmentFreeList (&this->srccontent); + esiSegmentFreeList (&this->altcontent); + cbdataReferenceDone (this->context); + safe_free (this->srcurl); + safe_free (this->alturl); +} + +void +esiIncludeStart (esiStreamContext *, char const *); + +void +esiIncludeStart (esiStreamContext *stream, char const *url) +{ + HttpHeader tempheaders; + if (!stream) + return; + httpHeaderInit (&tempheaders, hoRequest); + httpHeaderPutStr (&tempheaders, HDR_SURROGATE_CAPABILITY, "Surrogate/1.0"); + /* TODO: evaluate variables */ + if (clientBeginRequest(METHOD_GET, url, esiBufferRecipient, stream, &tempheaders, stream->localbuffer.buf, HTTP_REQBUF_SZ)) { + debug (86,0 ) ("starting new ESI subrequest failed\n"); + } + httpHeaderClean (&tempheaders); + +} + +esiElement * +esiIncludeNew (int attrcount, char const **attr, esiContext *context) +{ + esiInclude *rv; + int i; + assert (context); + CBDATA_INIT_TYPE_FREECB(esiInclude, esiIncludeFree); + rv = cbdataAlloc (esiInclude); + rv->vptr = &_esiInclude; + + for (i = 0; attr[i] && i < attrcount; i += 2) { + if (!strcmp(attr[i],"src")) { + /* Start a request for this url */ + debug (86,5)("esiIncludeNew: Requesting source '%s'\n",attr[i+1]); + /* TODO: don't assert on this, ignore the duplicate */ + assert (rv->src == NULL); + rv->src = esiStreamContextNew (rv); + assert (rv->src != NULL); + rv->srcurl =xstrdup ( attr[i+1]); + } else if (!strcmp(attr[i],"alt")) { + /* Start a secondary request for this url */ + /* TODO: make a config parameter to wait on requesting alt's + * for the src to fail + */ + debug (86,5)("esiIncludeNew: Requesting alternate '%s'\n",attr[i+1]); + assert (rv->alt == NULL); /* TODO: FIXME */ + rv->alt = esiStreamContextNew (rv); + assert (rv->alt != NULL); + rv->alturl = xstrdup (attr[i+1]); + } else if (!strcmp(attr[i],"onerror")) { + if (!strcmp(attr[i+1], "continue")) { + rv->flags.onerrorcontinue = 1; + } else { + /* ignore mistyped attributes */ + debug (86, 1)("invalid value for onerror='%s'\n", attr[i+1]); + } + } else { + /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed + */ + } + } + rv->context = cbdataReference(context); + esiIncludeStart (rv->src, rv->srcurl); + esiIncludeStart (rv->alt, rv->alturl); + return (esiElement *)rv; +} + +void +esiIncludeRender (void *data, esiSegment *output) +{ + esiInclude *this = data; + esiSegment *myout = NULL; + debug (86, 5)("esiIncludeRender: Rendering include %p\n", this); + assert (this->flags.finished || (this->flags.failed && this->flags.onerrorcontinue)); + if (this->flags.failed && this->flags.onerrorcontinue) { + return; + } + /* Render the content */ + if (this->srccontent) { + myout = this->srccontent; + this->srccontent = NULL; + } else if (this->altcontent) { + myout = this->altcontent; + this->altcontent = NULL; + } else + fatal ("esiIncludeRender called with no content, and no failure!\n"); + assert (output->next == NULL); + output->next = myout; +} + +esiProcessResult_t +esiIncludeProcess (void *data) +{ + esiInclude *this = data; + debug (86, 5)("esiIncludeRender: Processing include %p\n", this); + if (this->flags.failed) { + if (this->flags.onerrorcontinue) + return ESI_PROCESS_COMPLETE; + else + return ESI_PROCESS_FAILED; + } + if (!this->flags.finished) { + if (this->flags.onerrorcontinue) + return ESI_PROCESS_PENDING_WONTFAIL; + else + return ESI_PROCESS_PENDING_MAYFAIL; + } + return ESI_PROCESS_COMPLETE; +} + +void +esiIncludeSubRequestDone(esiInclude *this, esiStreamContext *stream, int success) +{ + assert (this); + if (this->flags.finished || this->flags.failed) + /* Do nothing, we don't need the data */ + return; + if (stream == this->src) { + if (success) { + /* copy the lead segment */ + this->srccontent = cbdataAlloc (esiSegment); + xmemcpy (this->srccontent->buf, stream->localbuffer.buf, stream->localbuffer.len); + this->srccontent->len = stream->localbuffer.len; + this->srccontent->next = stream->localbuffer.next; + stream->localbuffer.next = NULL; + /* we're done! */ + this->flags.finished = 1; + } else { + /* Fail if there is no alt being retrieved */ + if (!(this->alt || this->altcontent)) { + this->flags.failed = 1; + } else if (this->altcontent) { + /* ALT was already retrieved, we are done */ + this->flags.finished = 1; + } + } + this->src = NULL; + } else if (stream == this->alt) { + if (success) { + /* copy the lead segment */ + this->altcontent = cbdataAlloc (esiSegment); + xmemcpy (this->altcontent->buf, stream->localbuffer.buf, stream->localbuffer.len); + this->altcontent->len = stream->localbuffer.len; + this->altcontent->next = stream->localbuffer.next; + stream->localbuffer.next = NULL; + /* we're done! */ + if (!(this->src || this->srccontent)) { + /* src already failed, kick ESI processor */ + this->flags.finished = 1; + } + } else { + if (!(this->src || this->srccontent)) { + /* src already failed */ + this->flags.failed = 1; + } + } + this->alt = NULL; + } else { + fatal ("esiIncludeSubRequestDone: non-owned stream found!\n"); + } + if (this->flags.finished || this->flags.failed) { + /* Kick ESI Processor */ + debug (86,1)("esiInclude %p SubRequest %p completed, kicking processor , status %s\n", this, stream, this->flags.finished ? "OK" : "FAILED"); + assert (this->context); + esiKick (this->context); + } +} Index: squid/src/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v retrieving revision 1.23.2.1 retrieving revision 1.23.2.2 diff -u -r1.23.2.1 -r1.23.2.2 --- squid/src/Makefile.am 9 Aug 2002 11:49:45 -0000 1.23.2.1 +++ squid/src/Makefile.am 18 Aug 2002 11:09:22 -0000 1.23.2.2 @@ -26,6 +26,12 @@ DELAY_POOL_SOURCE = endif +if USE_ESI + ESI_SOURCE = ESI.c +else + ESI_SOURCE = +endif + if ENABLE_HTCP HTCPSOURCE = htcp.c endif @@ -130,7 +136,7 @@ $(DNSSOURCE) \ enums.h \ errorpage.c \ - ESI.c \ + $(ESI_SOURCE) \ ETag.c \ event.c \ external_acl.c \ Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.65.2.15 retrieving revision 1.65.2.16 diff -u -r1.65.2.15 -r1.65.2.16 --- squid/src/client_side.c 16 Aug 2002 01:12:32 -0000 1.65.2.15 +++ squid/src/client_side.c 18 Aug 2002 11:09:22 -0000 1.65.2.16 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.65.2.15 2002/08/16 01:12:32 rbcollins Exp $ + * $Id: client_side.c,v 1.65.2.16 2002/08/18 11:09:22 rbcollins Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -477,7 +477,7 @@ clientSocketContext *context; /* Test preconditions */ assert (node != NULL); - /* ESI TODO: handle this rather than asserting - it should only ever happen if we cause an abort and + /* TODO: handle this rather than asserting - it should only ever happen if we cause an abort and * the callback chain loops back to here, so we can simply return. * However, that itself shouldn't happen, so it stays as an assert for now. */ Index: squid/src/client_side_reply.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/client_side_reply.c,v retrieving revision 1.1.2.14 retrieving revision 1.1.2.15 diff -u -r1.1.2.14 -r1.1.2.15 --- squid/src/client_side_reply.c 16 Aug 2002 08:33:00 -0000 1.1.2.14 +++ squid/src/client_side_reply.c 18 Aug 2002 11:09:23 -0000 1.1.2.15 @@ -1,6 +1,6 @@ /* - * $Id: client_side_reply.c,v 1.1.2.14 2002/08/16 08:33:00 rbcollins Exp $ + * $Id: client_side_reply.c,v 1.1.2.15 2002/08/18 11:09:23 rbcollins Exp $ * * DEBUG: section 88 Client-side Reply Routines * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c) @@ -75,6 +75,8 @@ static StoreEntry *clientCreateStoreEntry(clientReplyContext *, method_t, request_flags); static STCB clientSendMoreData; static void clientRemoveStoreReference (clientReplyContext *, store_client **, StoreEntry **); +static void clientReplyContextSaveState (clientReplyContext *, clientHttpRequest *); +static void clientReplyContextRestoreState (clientReplyContext *, clientHttpRequest *); extern CSS clientReplyStatus; extern ErrorState *clientBuildError (err_type, http_status, char const *, struct in_addr *, request_t *); @@ -143,6 +145,39 @@ } } +void +clientReplyContextSaveState (clientReplyContext *this, clientHttpRequest *http) +{ + assert (this->old_sc == NULL); + debug (88,1)("clientReplyContextSaveState: saving store context\n"); + http->old_entry = http->entry; + this->old_sc = this->sc; + this->old_reqsize = this->reqsize; + this->old_reqofs = this->reqofs; + /* Prevent accessing the now saved entries */ + http->entry = NULL; + this->sc = NULL; + this->reqsize = 0; + this->reqofs = 0; +} + +void +clientReplyContextRestoreState (clientReplyContext *this, clientHttpRequest *http) +{ + assert (this->old_sc != NULL); + debug (88,1)("clientReplyContextRestoreState: Restoring store context\n"); + http->entry = http->old_entry; + this->sc = this->old_sc; + this->reqsize = this->old_reqsize; + this->reqofs = this->old_reqofs; + /* Prevent accessed the old saved entries */ + http->old_entry = NULL; + this->old_sc = NULL; + this->old_reqsize = 0; + this->old_reqofs = 0; +} + + /* there is an expired entry in the store. * setup a temporary buffer area and perform an IMS to the origin */ @@ -164,18 +199,16 @@ return; } http->request->flags.refresh = 1; - http->old_entry = http->entry; - context->old_sc = context->sc; - context->old_reqsize = context->reqsize; - context->old_reqofs = context->reqofs; #if STORE_CLIENT_LIST_DEBUG /* * Assert that 'http' is already a client of old_entry. If * it is not, then the beginning of the object data might get * freed from memory before we need to access it. */ - assert(http->sc->owner == http); + assert(http->sc->owner == context); #endif + /* Prepare to make a new temporary request */ + clientReplyContextSaveState (context, http); entry = storeCreateEntry(url, http->log_uri, http->request->flags, @@ -189,13 +222,12 @@ http->request->lastmod = http->old_entry->lastmod; debug(88, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod); http->entry = entry; - http->out.offset = 0; + http->out.offset = 0; /* FIXME Not needed - we have not written anything anyway */ fwdStart(http->conn ? http->conn->fd : -1, http->entry, http->request); /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(88, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); /* start counting the length from 0 */ - context->reqofs = 0; storeClientCopy(context->sc, entry, 0, HTTP_REQBUF_SZ, @@ -303,10 +335,9 @@ /* Its okay to send the old one anyway */ http->log_type = LOG_TCP_REFRESH_FAIL_HIT; clientRemoveStoreReference (context, &context->sc, &entry); - entry = http->entry = http->old_entry; - context->sc = context->old_sc; - context->reqofs = context->old_reqofs; - context->reqsize = context->old_reqsize; + /* Get the old request back */ + clientReplyContextRestoreState (context, http); + entry = http->entry; } else if (STORE_PENDING == entry->store_status && 0 == status) { /* more headers needed to decide */ debug(88, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); @@ -317,15 +348,13 @@ http->log_type = LOG_TCP_REFRESH_FAIL_HIT; clientRemoveStoreReference (context, &context->sc, &entry); entry = http->entry = http->old_entry; - context->sc = context->old_sc; -// http->reqbuf = http->norm_reqbuf; - context->reqofs = context->old_reqofs; - context->reqsize = context->old_reqsize; + /* Get the old request back */ + clientReplyContextRestoreState (context, http); + entry = http->entry; /* continue */ } else { -// http->reqofs += size; storeClientCopy(context->sc, entry, - http->out.offset + context->reqofs, + context->reqofs, HTTP_REQBUF_SZ - context->reqofs, context->tempbuf + context->reqofs, clientHandleIMSReply, @@ -349,21 +378,15 @@ httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); storeTimestampsSet(oldentry); clientRemoveStoreReference (context, &context->sc, &entry); - context->sc = context->old_sc; - entry = http->entry = oldentry; - entry->timestamp = squid_curtime; + oldentry->timestamp = squid_curtime; if (unlink_request) { - requestUnlink(entry->mem_obj->request); - entry->mem_obj->request = NULL; + requestUnlink(oldentry->mem_obj->request); + oldentry->mem_obj->request = NULL; } -// http->reqbuf = http->norm_reqbuf; - context->reqofs = context->old_reqofs; - context->reqsize = context->old_reqsize; + /* Get the old request back */ + clientReplyContextRestoreState (context, http); + entry = http->entry; /* here the data to send is in the next nodes buffers already */ - http->old_entry = NULL; /* done with old_entry */ - context->old_sc = NULL; - context->old_reqofs = 0; - context->old_reqsize = 0; assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); clientSendMoreData(context, context->tempbuf, context->reqsize); } else { @@ -835,6 +858,7 @@ /* * Handle STORE_OK objects. * objectLen(entry) will be set proprely. + * RC: Does objectLen(entry) include the Headers? */ if (entry->store_status == STORE_OK) { if (http->out.offset >= objectLen(entry)) Index: squid/src/mem.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/mem.c,v retrieving revision 1.20 retrieving revision 1.20.2.1 diff -u -r1.20 -r1.20.2.1 --- squid/src/mem.c 21 Jul 2002 01:06:07 -0000 1.20 +++ squid/src/mem.c 18 Aug 2002 11:09:23 -0000 1.20.2.1 @@ -1,6 +1,6 @@ /* - * $Id: mem.c,v 1.20 2002/07/21 01:06:07 squidadm Exp $ + * $Id: mem.c,v 1.20.2.1 2002/08/18 11:09:23 rbcollins Exp $ * * DEBUG: section 13 High Level Memory Pool Management * AUTHOR: Harvest Derived @@ -263,6 +263,7 @@ memReallocBuf(void *oldbuf, size_t net_size, size_t * gross_size) { /* XXX This can be optimized on very large buffers to use realloc() */ + /* TODO: if the existing gross size is >= new gross size, do nothing */ int new_gross_size; void *newbuf = memAllocBuf(net_size, &new_gross_size); if (oldbuf) {