---------------------
PatchSet 4706 
Date: 2002/08/18 11:09:22
Author: rbcollins
Branch: esi
Tag: (none) 
Log:
esi:comment and esi:include support

Members: 
	configure.in:1.61.2.2->1.61.2.3 
	src/ESI.c:1.1.2.10->1.1.2.11 
	src/Makefile.am:1.23.2.1->1.23.2.2 
	src/client_side.c:1.65.2.15->1.65.2.16 
	src/client_side_reply.c:1.1.2.14->1.1.2.15 
	src/mem.c:1.20->1.20.2.1 

Index: squid/configure.in
===================================================================
RCS file: /cvsroot/squid-sf//squid/configure.in,v
retrieving revision 1.61.2.2
retrieving revision 1.61.2.3
diff -u -r1.61.2.2 -r1.61.2.3
--- squid/configure.in	11 Aug 2002 10:16:34 -0000	1.61.2.2
+++ squid/configure.in	18 Aug 2002 11:09:22 -0000	1.61.2.3
@@ -3,7 +3,7 @@
 dnl
 dnl  Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9)
 dnl
-dnl  $Id: configure.in,v 1.61.2.2 2002/08/11 10:16:34 rbcollins Exp $
+dnl  $Id: configure.in,v 1.61.2.3 2002/08/18 11:09:22 rbcollins Exp $
 dnl
 dnl
 dnl
@@ -11,7 +11,7 @@
 AC_CONFIG_AUX_DIR(cfgaux)
 AM_INIT_AUTOMAKE(squid, 2.6-DEVEL)
 AM_CONFIG_HEADER(include/autoconf.h)
-AC_REVISION($Revision: 1.61.2.2 $)dnl
+AC_REVISION($Revision: 1.61.2.3 $)dnl
 AC_PREFIX_DEFAULT(/usr/local/squid)
 AM_MAINTAINER_MODE
 
@@ -460,6 +460,16 @@
   fi
 ])
 
+AM_CONDITIONAL(USE_ESI, false)
+AC_ARG_ENABLE(esi,
+              AC_HELP_STRING([--enable-esi],[Enable ESI for accelerators. Requires libexpat]),
+	      ac_cv_use_esi=$enableval, ac_cv_use_esi=no)
+AC_CACHE_CHECK(whether to enable ESI,ac_cv_use_esi, ac_cv_use_esi=no)
+if test "$ac_cv_use_esi" = "yes" ; then
+  AM_CONDITIONAL(USE_ESI, true)
+  XTRA_LIBS="$XTRA_LIBS -lexpat"
+fi
+
 dnl This is a developer only option. Developers know how to set defines
 dnl
 dnl AC_ARG_ENABLE(mem-gen-trace,
Index: squid/src/ESI.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Attic/ESI.c,v
retrieving revision 1.1.2.10
retrieving revision 1.1.2.11
diff -u -r1.1.2.10 -r1.1.2.11
--- squid/src/ESI.c	16 Aug 2002 08:33:00 -0000	1.1.2.10
+++ squid/src/ESI.c	18 Aug 2002 11:09:22 -0000	1.1.2.11
@@ -1,6 +1,6 @@
 
 /*
- * $Id: ESI.c,v 1.1.2.10 2002/08/16 08:33:00 rbcollins Exp $
+ * $Id: ESI.c,v 1.1.2.11 2002/08/18 11:09:22 rbcollins Exp $
  *
  * DEBUG: section 86    ESI processing
  * AUTHOR: Robert Collins
@@ -23,7 +23,7 @@
  *  (at your option) any later version.
  *  
  *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ ;  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *  GNU General Public License for more details.
  *  
@@ -34,6 +34,7 @@
  */
 
 #include "squid.h"
+#include "expat.h"
 
 /* quick reference on behaviour here.
  * The ESI specification 1.0 requires the ESI processor to be able to 
@@ -54,9 +55,159 @@
 } esiSegment;
 CBDATA_TYPE(esiSegment);
 
-typedef struct _esiContext {
-    int remaining_esi_requests;
-    int completed_requests;
+typedef struct _esiStreamContext esiStreamContext;
+typedef struct _esiContext esiContext;
+typedef struct _esiInclude esiInclude;
+
+/* TODO: split this out into separate files */
+/* Parsing: quick and dirty. ESI files are not valid XML, so a generic 
+ * XML parser is not much use. Also we need a push parser not a pull 
+ * parser, so LibXML is out.
+ * We don't have C++, but an interpreter pattern (possibly with a builder)
+ * is ideal for our needs
+ * so: we typedef each method, and manually build a vptr for each 
+ * 'class'. Finally, we use a #define to call the 'methods'.
+ * TODO: make a builder for the parse tree. 
+ * 
+ *
+ * Interpreter methods: 
+ * Render: May only ever be calle dafter Process returns PROCESS_COMPLETE.
+ * Renders the resulting content into a esiSegment chain.
+ * Process: returns the status of the node.
+ * COMPLETE - processing is complete, rendering may staret
+ * PENDING_WONTFAIL - process is incomplete, but the element *will*
+ *   be able to be rendered given time.
+ * PENDING_MAYFAIL - processing is incomplete, and the element *may*
+ *   fail to be able to rendered.
+ * FAILED - processing failed, return an error to the client.
+ */
+
+/* TODO: remove invalid markup (silently swallow markup in invalid locations
+ * TODO: inline variables
+ * TODO: 3.3 choose when otherwise
+ * TODO: 3.4 try attempt except
+ * TODO: 3.6 remove
+ * TODO: 3.7 vars
+ *
+ *
+ * NOT TODO: esi:inline - out of scope.
+ */
+
+typedef enum {
+    ESI_PROCESS_COMPLETE,
+    ESI_PROCESS_PENDING_WONTFAIL,
+    ESI_PROCESS_PENDING_MAYFAIL,
+    ESI_PROCESS_FAILED
+} esiProcessResult_t;
+
+typedef enum {
+    ESI_KICK_FAILED,
+    ESI_KICK_PENDING,
+    ESI_KICK_SENT
+} esiKick_t;
+
+typedef struct _esiElementType esiElementType;
+typedef struct _esiElement esiElement;
+
+typedef void esiRender (void *, esiSegment *); /* render into the segment list */
+#define RENDER(foo, bar)  foo->vptr->render(foo, bar)
+typedef int esiAddElement (void *, esiElement *); /* add a child to an element */
+#define ADDELEMENT(foo, bar) foo->vptr->add(foo,bar)
+typedef esiProcessResult_t esiProcessSelf (void *); /* process this element */
+#define PROCESS(foo) foo->vptr->process(foo)
+struct _esiElementType {
+    esiRender *render;
+    esiAddElement *add;
+    esiProcessSelf *process;
+};
+struct _esiElement {
+    esiElementType *vptr;
+};
+
+/* some core operators */
+
+/* the methods for each type */
+static esiAddElement esiAddFail;
+static esiProcessSelf esiProcessComplete;
+
+static esiRender esiCommentRender;
+
+static esiRender esiLiteralRender;
+
+static esiAddElement esiSequenceAdd;
+static esiRender esiSequenceRender;
+static esiProcessSelf esiSequenceProcess;
+
+static esiRender esiIncludeRender;
+static esiProcessSelf esiIncludeProcess;
+static void esiIncludeSubRequestDone (esiInclude *, esiStreamContext *, int);
+
+/* the types we have */
+typedef enum {
+    ESI_ELEMENT_NONE,
+    ESI_ELEMENT_INCLUDE,
+    ESI_ELEMENT_COMMENT
+} esiElementType_t;
+
+static esiElementType _esiComment  = {esiCommentRender,esiAddFail,esiProcessComplete};
+static esiElementType _esiLiteral  = {esiLiteralRender,esiAddFail,esiProcessComplete};
+static esiElementType _esiSequence = {esiSequenceRender,esiSequenceAdd,esiSequenceProcess};
+static esiElementType _esiInclude =  {esiIncludeRender, esiAddFail, esiIncludeProcess};
+
+/* esiComment */
+typedef struct {
+    esiElementType *vptr;
+} esiComment;
+CBDATA_TYPE (esiComment);
+static FREE esiCommentFree;
+static esiElement * esiCommentNew(void);
+
+/* esiLiteral */
+typedef struct {
+    esiElementType *vptr;
+    /* optimise copies away later */
+    esiSegment *buffer;
+} esiLiteral;
+CBDATA_TYPE (esiLiteral);
+static FREE esiLiteralFree;
+static esiElement * esiLiteralNew(const XML_Char *s, int len);
+
+/* esiSequence */
+typedef struct {
+    esiElementType *vptr;
+    esiElement ** elements; /* unprocessed or rendered nodes */
+    int allocedcount;
+    size_t allocedsize;
+    int elementcount;
+    int processedcount;
+} esiSequence;
+CBDATA_TYPE (esiSequence);
+static FREE esiSequenceFree;
+static esiElement * esiSequenceNew(void);
+
+/* esiInclude */
+struct _esiInclude {
+    esiElementType *vptr;
+    struct {
+	int onerrorcontinue:1; /* on error return zero data */
+	int failed:1; /* Failed to process completely */
+	int finished:1; /* Finished getting subrequest data */
+    } flags;
+    esiStreamContext *src;
+    esiStreamContext *alt;
+    esiSegment *srccontent;
+    esiSegment *altcontent;
+    esiContext *context;
+    char *srcurl, *alturl;
+};
+CBDATA_TYPE (esiInclude);
+static FREE esiIncludeFree;
+static esiElement * esiIncludeNew (int attributes, const char **attr, esiContext *);
+
+struct _esiContext {
+    clientStreamNode *this; /* our stream node */
+    clientHttpRequest *http; /* the request we are processing. HMM: cbdataReferencing this will result
+    in a circular reference, so we don't. Note: we are automatically freed when it is, so thats ok. */
     struct {
 	int passthrough:1;
 	int oktosend:1;
@@ -65,40 +216,61 @@
 	              * regardless. Note that we don't fail midstream
 		      * because we buffer until we can not fail
 		      */
+	int finishedtemplate:1; /* we've read the entire template */
+	int parserinited:1;
+	int clientwantsdata:1; /* we need to satisfy a read request */
+	int kicked:1; /* don't reenter the kick routine */
     } flags;
+    err_type errorpage; /* if we error what page to use */
+    http_status errorstatus; /* if we error, what code to return */
     HttpReply *rep; /* buffered until we pass data downstream */
     esiSegment *buffered; /* unprocessed data - for whatever reason */
     esiSegment *incoming;
     esiSegment *outbound; /* processed data we are waiting to send, or for 
 			   * potential errors to be resolved 
 			   */
+    esiSegment *outboundtail; /* our write segment */
     size_t outbound_offset; /* the offset to the next character to send -
 				  * non zero if we haven't sent the entire segment 
 				  * for some reason 
 				  */
     off_t readpos; /* the logical position we are reading from */
     off_t pos; /* the logical position of outbound_offset in the data stream */
-} esiContext;
+    struct {
+	off_t unprocessed_start; /* the offset into the buffered chain of the
+				    oldest data not processed
+				    */
+	off_t buffered_tokenstart; /* the offset into the buffered chain of the
+				      parsers current symbol candidate 
+				      */
+	esiElement *stack[10]; /* a stack of esi elements that are open */
+	int stackdepth; /* self explanatory */
+	XML_Parser p; /* our parser */
+    } parserState; /* todo factor this off somewhere else; */
+    esiElement *tree;
+};
 
 CBDATA_TYPE(esiContext);
 
-typedef struct _esiStreamContext {
-    clientHttpRequest *http;
-    esiContext *esi;
-    esiSegment localbuf;
-} esiStreamContext;
+struct _esiStreamContext {
+    int finished;
+    esiInclude *include;
+    esiSegment localbuffer;
+    esiSegment *buffer;
+};
 
 CBDATA_TYPE (esiStreamContext);
 
 /* Local functions */
 /* esiContext */
 static FREE esiContextFree;
-static esiContext *esiContextNew(HttpReply *);
+static esiContext *esiContextNew(HttpReply *, clientStreamNode *, clientHttpRequest *);
 static void esiContextFreeResources (esiContext *);
 static size_t esiSend (clientStreamNode *this, esiContext *,clientHttpRequest *http);
+static esiProcessResult_t esiProcess (esiContext *);
 /* esiStreamContext */
 static FREE esiStreamContextFree;
-static esiStreamContext *esiStreamContextNew (clientHttpRequest *, esiContext *);
+static esiStreamContext *esiStreamContextNew (esiInclude *);
 /* esiSegment */
 static void esiSegmentFreeList (esiSegment **head);
 
@@ -106,11 +278,64 @@
 static CSCB esiBufferRecipient;
 /* static void esiStartSub (void); */
 static void esiFail (clientStreamNode *this, esiContext *context, clientHttpRequest *http);
+static esiKick_t esiKick (esiContext *context);
 
-/* ESI TODO:
+/* ESI TO CONSIDER:
  * 1. retry failed upstream requests
  */
 
+
+esiKick_t
+esiKick (esiContext *context)
+{
+    assert (context);
+    if (context->flags.kicked)
+	return ESI_KICK_PENDING;
+    else
+	context->flags.kicked = 1;
+    /* Something has occured. Process any remaining nodes */
+    if (!context->flags.finished)
+	/* Process some of our data */
+	switch (esiProcess (context)) {
+	  case ESI_PROCESS_COMPLETE: 
+	      debug (86,1)("esiKick: esiProcess OK\n");break;
+	  case ESI_PROCESS_PENDING_WONTFAIL:
+	      debug (86,1)("esiKick: esiProcess PENDING OK\n");break;
+	  case ESI_PROCESS_PENDING_MAYFAIL:
+	      debug (86,1)("esiKick: esiProcess PENDING UNKNOWN\n");break;
+	  case ESI_PROCESS_FAILED:
+	      debug (86,0)("esiKick: esiProcess FAILED\n");
+	      /* this can not happen - processing can't fail until we have data,
+	       * and when we come here we have sent data to the client
+	       */
+	      if (context->pos == 0)
+		  esiFail (context->this, context, context->http);
+	      context->flags.kicked = 0;
+	      return ESI_KICK_FAILED;
+	}
+    
+    /* Render if we can to get maximal sent data */
+    assert (context->tree || context->flags.error);
+    
+    if (!context->outbound)
+	context->outbound = context->outboundtail = cbdataAlloc (esiSegment);
+    if (!context->flags.error)
+    RENDER (context->tree, context->outboundtail);
+    /* TODO: fixup this outboundtail dross a little */
+    while (context->outboundtail->next)
+	context->outboundtail = context->outboundtail->next;
+    /* Is there data to send? */
+    if (esiSend (context->this, context, context->http)) {
+	/* some data was sent. we're finished until the next read */
+	context->flags.kicked = 0;
+	return ESI_KICK_SENT;
+    }
+
+    context->flags.kicked = 0;
+    /* nothing to send */
+    return ESI_KICK_PENDING;
+}
+
 /* request from downstream for more data
  */
 void
@@ -134,40 +359,48 @@
 	cbdataReferenceDone (context);
 	return;
     }
-    
-    /* Ok, not passing through. Is there data to send? */
-    if (context->flags.oktosend && context->outbound && 
-	context->outbound_offset < context->outbound->len) {
-	/* Yes! Send it without asking for more upstream */
-	/* memcopying because the client provided the buffer */
-	/* TODO: skip data until pos == next->readoff; */
-	clientStreamNode *next = this->node.next->data;
-	size_t len = next->readlen < context->outbound->len - context->outbound_offset ? 
-	   next->readlen : context->outbound->len - context->outbound_offset;
-	assert (context->pos == next->readoff);
-	xmemcpy (next->readbuf, &context->outbound->buf[context->outbound_offset], len);
-	if (len + context->outbound_offset == context->outbound->len) {
-	    esiSegment *temp = context->outbound->next;
-	    /* remove the used buffer */
-	    context->outbound_offset = 0;
-	    cbdataFree (context->outbound);
-	    context->outbound = temp;
-	}
-	context->pos += len;
-	clientStreamCallback (this, http, context->rep, next->readbuf, len);
+
+    context->flags.clientwantsdata = 1;
+    debug (86,5)("esiStreamRead: Client now wants data\n");
+
+    /* Ok, not passing through */
+    switch (esiKick (context)) {
+      case ESI_KICK_FAILED:
+	/* this can not happen - processing can't fail until we have data,
+	 * and when we come here we have sent data to the client
+	 */
+        cbdataReferenceDone (context);
+	return;
+      case ESI_KICK_SENT:
 	cbdataReferenceDone (context);
 	return;
+      case ESI_KICK_PENDING:
+	break;
     }
 
+    /* Nothing to send */
+
+    if (context->flags.oktosend && context->flags.finishedtemplate && 
+	! context->flags.finished) {
+	/* we've started sending, finished reading, but not finished
+	 * processing. stop here, a callback will resume the stream
+	 * flow
+	 */
+	debug (86,1) ("esiStreamRead: Waiting for async resume of esi processing\n");
+	cbdataReferenceDone (context);
+	return;
+    }
+ 
     if (context->flags.oktosend && context->flags.finished) {
-	/* We've finished process, and there is no more data buffered */
-	debug (0,0)("Telling recipient EOF\n");
+	assert (!context->outbound);
+	/* We've finished processing, and there is no more data buffered */
+	debug (0,0)("Telling recipient EOF on READ\n");
 	clientStreamCallback (this, http, NULL, NULL, 0);
 	cbdataReferenceDone (context);
 	return;
     }
  
-    /* no data that is ready to send? well, lets get some */
+    /* no data that is ready to send, and still reading? well, lets get some */
     /* secure a buffer */
     if (!context->incoming) {
 	/* create a new buffer segment */
@@ -202,9 +435,11 @@
     if (context->flags.oktosend && context->flags.finished &&
 	!(context->outbound && context->outbound_offset < context->outbound->len)) {
 	cbdataReferenceDone (context);
+	debug (86,1) ("Telling recipient EOF on STATUS\n");
 	return STREAM_UNPLANNED_COMPLETE; /* we don't know lengths in advance */
     }
     /* ?? RC: we can't be aborted / fail ? */
+    cbdataReferenceDone (context);
     return STREAM_NONE;
 }
 
@@ -232,8 +467,21 @@
 esiSend (clientStreamNode *this, esiContext *context, clientHttpRequest *http)
 {
     /* send any processed data */
+
+    /* trim leading empty buffers ? */
+    while (context->outbound && context->outbound->next && !context->outbound->len) {
+	esiSegment *temp = context->outbound->next;
+	cbdataFree (context->outbound);
+	context->outbound = temp;
+    }
+
+    if (!context->flags.clientwantsdata) {
+	debug (86,5)("esiSend: Client does not want data - not sending anything\n");
+	return 0;
+    }
+    
     if (context->flags.oktosend && (context->rep || (context->outbound &&
-	context->outbound_offset <= context->outbound->len))) {
+	context->outbound->len && (context->outbound_offset <= context->outbound->len)))) {
 	/* Yes! Send it without asking for more upstream */
 	/* memcopying because the client provided the buffer */
 	/* TODO: skip data until pos == next->readoff; */
@@ -241,10 +489,12 @@
 	size_t len = 0;
 	cbdataReference (context);
 	if (context->outbound)
-	    len = next->readlen < context->outbound->len - context->outbound_offset ?
-	      next->readlen : context->outbound->len - context->outbound_offset;
-	/* prevent corruption on range requests, even thought we don't support them yet */
+	    len = next->readlen < (context->outbound->len - context->outbound_offset) ?
+	      next->readlen : (context->outbound->len - context->outbound_offset);
+	/* prevent corruption on range requests, even though we don't support them yet */
 	assert (context->pos == next->readoff);
+	/* We must send data or a reply */
+	assert (len != 0 || context->rep != NULL);
 	if (len) {
 	    xmemcpy (next->readbuf, &context->outbound->buf[context->outbound_offset], len);
 	    if (len + context->outbound_offset == context->outbound->len) {
@@ -255,15 +505,27 @@
 		context->outbound = temp;
 	    }
 	    context->pos += len;
+	    if (!context->outbound)
+		context->outboundtail = NULL;
+	    /* trim leading empty buffers ? */
+	    while (context->outbound && context->outbound->next && !context->outbound->len) {
+		esiSegment *temp = context->outbound->next;
+		cbdataFree (context->outbound);
+		context->outbound = temp;
+	    }
 	}
+	context->flags.clientwantsdata = 0;
+	debug (86,5)("esiSend: Client no longer wants data \n");
 	clientStreamCallback (this, http, context->rep, next->readbuf, len);
 	if (len == 0)
 	    len = -1; /* tell the caller we sent something */
 	if (context->rep)
 	  context->rep = NULL; /* freed downstream */
 	cbdataReferenceDone (context);
+	debug (86,5)("esiSend returning %d\n",len);
 	return len;
     }
+    debug (86,5)("esiSend returning 0\n");
     return 0;
 }
 
@@ -296,20 +558,26 @@
     assert (this->node.next != NULL);
     if (!this->data)
 	/* setup ESI context from reply headers */
-	this->data = esiContextNew(rep);
+	this->data = esiContextNew(rep, this, http);
     context = cbdataReference(this->data);
-    /* Finished all ESI processing. All remaining data gets untouched. 
-     * Mainly used when an error has been detected to prevent ESI processing the error body
+
+    /* Skipping all ESI processing. All remaining data gets untouched. 
+     * Mainly used when an error or other non-ESI processable entity
+     * has been detected to prevent ESI processing the error body
      */
     if (context->flags.passthrough) {
 	cbdataReferenceDone(context);
 	clientStreamCallback (this, http, rep, body_data, body_size);
 	return;
     }
+
+    /* once we finish the template, we *cannot* return here */
+    assert (!context->flags.finishedtemplate);
     
     /* Can we generate any data ?*/
 
     if (body_data) {
+	/* Increase our buffer area with incoming data */
         assert (body_size <= HTTP_REQBUF_SZ);
 	debug (86,1 )("ESIProcessStream found %u bytes of body data at offset %ld\n", body_size, this->readoff);
         /* secure the data for later use */
@@ -353,87 +621,42 @@
         }
     }
 
-    debug(86, 0) ("esiProcessStream: Will be checking for ESI content here - %d sub requests remaining\n", context->remaining_esi_requests);
-    
-    /* HACK to test buffering */
     /* EOF / Read error /  aborted entry */
-    if (rep == NULL && body_data == NULL && body_size == 0 && !context->flags.finished) {
-	/* Bit test the entry for aborts */       
-	/* flush the esi processor */     
-	debug (0,0)("Flushing now \n");
-	context->flags.oktosend = 1;
-	context->flags.finished = 1;
-	context->outbound = context->buffered;
-	context->buffered = NULL;
-	context->incoming = NULL;
-	cbdataReferenceDone(context);
-	esiProcessStream (this, http, rep, body_data, body_size);
-	return;
-    }
- 
-    /* send any processed data */
-    if (esiSend (this, context, http)) {
-	/* some data was sent. we're finished until the next read */
-	cbdataReferenceDone (context);
-	return;
+    if (rep == NULL && body_data == NULL && body_size == 0 && !context->flags.finishedtemplate) {
+	/* TODO: get stream status to test the entry for aborts */
+	/* else flush the esi processor */  
+	debug (86,1)("Finished reading upstream data\n");
+	/* This is correct */
+	context->flags.finishedtemplate = 1;
     }
 
+    switch (esiKick (context)) {
+      case ESI_KICK_FAILED: 
+      /* this can not happen - processing can't fail until we have data,
+       * and when we come here we have sent data to the client
+       */ 
+      cbdataReferenceDone (context);
+      return;
+      case ESI_KICK_SENT:
+      cbdataReferenceDone (context);
+      return;
+      case ESI_KICK_PENDING:
+      break;
+    }
+	
     /* ok.. no data sent, try to pull more data in from upstream.
      * FIXME: Don't try this if we have finished reading the template
      */
-    assert (context->incoming && context->incoming->len < HTTP_REQBUF_SZ);
-    clientStreamRead (this, http, context->readpos, HTTP_REQBUF_SZ - context->incoming->len,
-		      &context->incoming->buf[context->incoming->len]);
-
-//		      esiFail (this, context, http);
-    
-    cbdataReferenceDone (context);
-#if 0
-
-    /* HACK to test buffering */        
-    /* EOF / Read error /  aborted entry */     
-    if (rep == NULL && body_data == NULL && body_size == 0) {
-	/* Bit test the entry for aborts */       
-	/* flush the esi processor */     
-	context->flags.oktosend = 1;       
-	cbdataReferenceDone(context);
-	esiProcessStream (this, http, rep, body_data, body_size);
+    if (!context->flags.finishedtemplate) {
+	assert (context->incoming && context->incoming->len < HTTP_REQBUF_SZ);
+	clientStreamRead (this, http, context->readpos, HTTP_REQBUF_SZ - context->incoming->len,
+			  &context->incoming->buf[context->incoming->len]);
+	cbdataReferenceDone (context);
 	return;
     }
     
-#endif
-    
-    
-    if (0)
-    /* Kick off a request */
-    if (context->remaining_esi_requests--)
-      {
-	HttpHeader tempheaders;
-	esiStreamContext *esiStream = esiStreamContextNew (http, context);
-	assert (esiStream != NULL);
-	httpHeaderInit (&tempheaders, hoRequest);
-	httpHeaderPutStr (&tempheaders, HDR_SURROGATE_CAPABILITY, "Surrogate/1.0");
-	
-	if (clientBeginRequest(METHOD_GET, "http://192.168.0.2/ESI_fragment_1.txt", esiBufferRecipient, esiStream, &tempheaders, esiStream->localbuf.buf, HTTP_REQBUF_SZ)) {
-	    debug (86,0 ) ("starting new ESI subrequest failed\n");
-	}
-	httpHeaderClean (&tempheaders);
-      }
-
-    /* EOF / Read error /  aborted entry */
-    if (rep == NULL && body_data == NULL && body_size == 0) {
-	/* Bit test the entry for aborts */
-	/* flush the esi processor */
-    }
-	
-    if (0)
-    // send an error
-    {
-	    esiFail (this, context, http);
-	    cbdataReferenceDone(context);
-	    return;
-      }
-//    cbdataReferenceDone(context);
+    debug (86,1)("esiProcessStream: no data to send, no data to read, awaiting a callback\n");
+    cbdataReferenceDone(context);
 }
 
 void
@@ -445,23 +668,307 @@
 }
 
 esiContext *
-esiContextNew (HttpReply *rep)
+esiContextNew (HttpReply *rep, clientStreamNode *this, clientHttpRequest *http)
 {
     esiContext *rv;
     CBDATA_INIT_TYPE_FREECB(esiContext, esiContextFree);
     rv = cbdataAlloc(esiContext);
     assert (rep);
-    /* ESI TODO: remove this hardcoded test parameter */
-    rv->remaining_esi_requests = 2;
     rv->rep = rep;
     if (esiAlwaysPassthrough(rep->sline.status)) {
 	rv->flags.passthrough = 1;
     } else {
-	/* TODO: remove Etag, content-length headers */
+	/* remove specific headers for ESI to prevent
+	 * downstream cache confusion */
+	HttpHeader *hdr = &rep->header;
+	httpHeaderDelById(hdr, HDR_ACCEPT_RANGES);
+	httpHeaderDelById(hdr, HDR_ETAG);
+	httpHeaderDelById(hdr, HDR_CONTENT_LENGTH);
+	httpHeaderDelById(hdr, HDR_CONTENT_MD5);
+	/* Tree may use this */
+	CBDATA_INIT_TYPE(esiSegment);
+	rv->tree = esiSequenceNew();
+	rv->this = this;
+	rv->http = http;
+	rv->flags.clientwantsdata = 1;
+	debug (86,5)("esiContextNew: Client wants data (always created during reply cycle\n");
     }
     return rv;
 }
 
+typedef struct {
+    char *buf; /* not owned by us. */
+    size_t len;
+} inplacetoken;
+
+inplacetoken
+esiFindToken (esiSegment *buffer, off_t startat);
+
+typedef enum { 
+    TOKEN_START, /* looking for < */
+    TOKEN_QUOTED, /* looking for a match to the start quote */
+    TOKEN_UNQUOTED, /* looking for " or ' or > */
+    TOKEN_COMPLETE /* finished */
+} tokenstate;
+
+int
+esiAddLiteral (esiContext *context, const XML_Char *s, int len);
+void start(void *data, const XML_Char *el, const char **attr);
+void end(void *data, const XML_Char *el);
+void esiExpatDefault (void *data, const XML_Char *s, int len);
+
+static esiElementType_t
+esiIdentifyElement (const XML_Char *el) ;
+
+esiElementType_t
+esiIdentifyElement (const XML_Char *el)
+{
+    assert (el);
+    if (strlen (el) < 5 || strncmp (el, "esi:", 4))
+	return ESI_ELEMENT_NONE;
+    if (!strncmp (el + 4, "comment",7))
+	return ESI_ELEMENT_COMMENT;
+    if (!strncmp (el + 4, "include",7))
+	return ESI_ELEMENT_INCLUDE;
+    return ESI_ELEMENT_NONE;
+}
+
+void
+start(void *data,const XML_Char *el, const char **attr) 
+{
+    int i;
+    int ellen = strlen (el);
+    char localbuf [HTTP_REQBUF_SZ]; 
+    esiContext *context = data;
+    esiElement *element;
+    int specifiedattcount = XML_GetSpecifiedAttributeCount (context->parserState.p) * 2;
+    char *pos;
+    assert (ellen < sizeof (localbuf)); /* prevent unexpected overruns. */
+    
+    debug (86, 5)("esiExpatState: element '%s' with %d tags\n", el, specifiedattcount);
+
+    switch (esiIdentifyElement (el)) {
+      case ESI_ELEMENT_NONE:
+        /* Spit out elements we aren't interested in */
+        localbuf[0] = '<';
+	localbuf[1] = '\0';
+	assert (xstrncpy (&localbuf[1], el, sizeof(localbuf) - 2));
+	pos = localbuf + strlen (localbuf);
+	
+	for (i = 0; attr[i] && i < specifiedattcount; i += 2) {
+	    *pos++ = ' ';
+	    /* TODO: handle this gracefully */
+	    assert (xstrncpy (pos, attr[i], sizeof(localbuf) + (pos - localbuf)));
+	    pos += strlen (pos);
+	    *pos++ = '=';
+	    *pos++ = '\'';
+	    assert (xstrncpy (pos, attr[i + 1], sizeof(localbuf) + (pos - localbuf)));
+	    pos += strlen (pos);
+	    *pos++ = '\'';
+	}
+	*pos++ = '>';
+	*pos = '\0';
+	
+	if (esiAddLiteral (context, localbuf, pos - localbuf))
+	    context->flags.error = 1;
+	break;
+      case ESI_ELEMENT_COMMENT:
+	/* Put on the stack to allow skipping of 'invalid' markup */
+	assert (context->parserState.stackdepth <11);
+	element = esiCommentNew ();
+	if (ADDELEMENT (context->parserState.stack[context->parserState.stackdepth-1], element)) {
+	    debug (86,1)("esiProcess: failed to add esi node, probable error in ESI template\n");
+	    context->flags.error = 1;
+	    cbdataFree (element);
+	    break;
+	} else {
+	    /* added ok, push onto the stack */
+	    context->parserState.stack[context->parserState.stackdepth++] = element;
+	}
+	break;
+      case ESI_ELEMENT_INCLUDE:
+	/* Put on the stack to allow skipping of 'invalid' markup */
+	assert (context->parserState.stackdepth <11);
+	element = esiIncludeNew (specifiedattcount, attr, context);
+	if (ADDELEMENT (context->parserState.stack[context->parserState.stackdepth-1], element)) {
+	    debug (86,1)("esiProcess: failed to add esi node, probable error in ESI template\n");
+	    context->flags.error = 1;
+	    cbdataFree (element);
+	    break;
+	} else {
+	    /* added ok, push onto the stack */
+	    context->parserState.stack[context->parserState.stackdepth++] = element;
+	}
+	break;
+    }
+   
+
+    debug (86,1)("esi stack depth %d\n",context->parserState.stackdepth);
+    
+}  /* End of start handler */
+
+void
+end(void *data, const char *el)
+{
+    int ellen = strlen (el);
+    char localbuf [HTTP_REQBUF_SZ];
+    char *pos;
+    esiContext *context = data;
+
+    switch (esiIdentifyElement (el)) {
+      case ESI_ELEMENT_NONE:
+        assert (ellen < sizeof (localbuf)); /* prevent unexpected overruns. */
+	/* Add elements we aren't interested in */    localbuf[0] = '<';
+	localbuf[1] = '/';
+	assert (xstrncpy (&localbuf[2], el, sizeof(localbuf) - 3));
+	pos = localbuf + strlen (localbuf);
+	*pos++ = '>';
+	*pos = '\0';
+	if (esiAddLiteral (context, localbuf, pos - localbuf))
+	    context->flags.error = 1;
+	break;
+      case ESI_ELEMENT_COMMENT:
+	/* pop of the stack */
+	--context->parserState.stackdepth;
+	break;
+      case ESI_ELEMENT_INCLUDE:
+	/* pop of the stack */
+	--context->parserState.stackdepth;
+	break;
+    }
+}  /* End of end handler */
+
+void
+esiExpatDefault (void *data, const XML_Char *s, int len)
+{
+    esiContext *context = data;
+    /* handle and skipped data */
+    if (esiAddLiteral (context, s, len))
+        context->flags.error = 1;
+}
+
+int
+esiAddLiteral (esiContext *context, const XML_Char *s, int len)
+{
+    /* handle any skipped data */
+    esiElement *element;
+    assert (len);
+    debug (86,5)("literal length is %d\n", len);
+    /* give a literal to the current element */
+    assert (context->parserState.stackdepth <11);
+    element = esiLiteralNew (s, len);
+    if (ADDELEMENT (context->parserState.stack[context->parserState.stackdepth-1], element)) {
+	debug (86,1)("esiProcess: failed to add esi node, probable error in ESI template\n");
+	cbdataFree (element);
+	return -1;
+    }
+    return 0;
+}
+
+esiProcessResult_t
+esiProcess (esiContext *context)
+{
+    /* parsing:
+     * read through buffered, skipping plain text, and skipping any
+     * <...> entry that is not an <esi: entry.
+     * when it's found, hand an esiLiteral of the preceeding data to our current
+     * context
+     */
+
+    assert (context->flags.finished == 0);
+
+    if (!context->parserState.stackdepth) {
+	debug (86,1)("empty parser stack, inserting the top level node\n");
+	assert (context->tree);
+	context->parserState.stack[context->parserState.stackdepth++] = context->tree;
+    }
+
+    if (context->rep && !context->flags.parserinited)
+      {
+	/* TODO: grab the document encoding from the headers */
+	context->parserState.p = XML_ParserCreate(NULL);
+	XML_SetUserData (context->parserState.p, context);
+	XML_SetElementHandler(context->parserState.p, start, end);
+	XML_SetDefaultHandler(context->parserState.p, esiExpatDefault);
+      }
+
+    /* we have data */
+    if (context->buffered) {
+	/* we don't keep any data around */
+	
+	while (context->buffered) {
+	    esiSegment *temp;
+	    if (context->buffered->len) {
+		if (! XML_Parse(context->parserState.p, context->buffered->buf, context->buffered->len, context->flags.finishedtemplate)) {
+		    context->errorpage = ERR_UNSUP_REQ;
+		    context->errorstatus = HTTP_INTERNAL_SERVER_ERROR;
+		    /* TODO: transmit this error the user */
+		    debug (86,0)("esiProcess: Parse error at line %d:\n%s\n",
+			   XML_GetCurrentLineNumber(context->parserState.p),
+			   XML_ErrorString(XML_GetErrorCode(context->parserState.p)));
+		    return ESI_PROCESS_FAILED;
+		}
+		if (context->flags.error) {
+		    /* TODO" check if we should reset flags.failed */
+		    context->errorpage = ERR_UNSUP_REQ;
+		    context->errorstatus = HTTP_INTERNAL_SERVER_ERROR;
+		    return ESI_PROCESS_FAILED;
+		}
+		
+	    }
+	    temp = context->buffered;
+	    context->buffered = temp->next;
+	    cbdataFree (temp);
+	}
+	/* Tel the read code to allocate a new buffer */
+	context->incoming = NULL;
+    }
+
+    /* ok, we've done all we can with the data. What can we process now?
+     */
+    {
+      esiProcessResult_t status = PROCESS(context->tree);
+      switch (status) {
+	case ESI_PROCESS_COMPLETE:
+	    debug (86,1)("esiProcess: tree Processed OK\n");
+	    break;
+	case ESI_PROCESS_PENDING_WONTFAIL:
+	    debug (86,1)("esiProcess: tree Processed PENDING OK\n");
+	    break;
+	case ESI_PROCESS_PENDING_MAYFAIL:
+	    debug (86,1)("eseProcess: tree Processed PENDING UNKNOWN\n");
+	    break;
+	case ESI_PROCESS_FAILED:
+	    debug (86,0)("esiProcess: tree Processed FAILED\n");
+	    /* ESI TODO: create a new error page */
+	    context->errorpage = ERR_UNSUP_REQ;
+	    context->errorstatus = HTTP_INTERNAL_SERVER_ERROR;
+	    return ESI_PROCESS_FAILED;
+	    break;
+        }
+      if (status != ESI_PROCESS_PENDING_MAYFAIL && context->flags.finishedtemplate){
+	  /* We've read the entire template, and no nodes will
+	   * return failure
+	   */
+	  debug (86,1)("esiProcess, request will succeed\n");
+	  context->flags.oktosend = 1;
+      }
+
+      if (status == ESI_PROCESS_COMPLETE && context->flags.finishedtemplate) {
+	  /* we've finished all processing. Render and send. */
+	  debug (86,1)("esiProcess, processing complete\n");
+	  context->flags.finished = 1;
+      }
+ 
+    }
+
+    if (!context->flags.finishedtemplate && !context->incoming) {
+	context->incoming = context->buffered = cbdataAlloc (esiSegment);
+    }
+    
+    return ESI_PROCESS_COMPLETE; /* because we have no callbacks */
+}
+
 void
 esiSegmentFreeList (esiSegment **head) {
     while (*head) {
@@ -477,6 +984,11 @@
 	httpReplyDestroy (context->rep);
 	context->rep = NULL;
     }
+    if (context->tree)
+	cbdataFree (context->tree);
+    if (context->flags.parserinited) {
+	XML_ParserFree (context->parserState.p);
+    }
     esiSegmentFreeList (&context->buffered);
     esiSegmentFreeList (&context->outbound);
     /* don't touch incoming, it's a pointer into buffered anyway */
@@ -500,7 +1012,7 @@
     /* don't honour range requests - for errors we sent it all */
     context->flags.error = 1;
     /* create an error object */
-    err = clientBuildError(ERR_TOO_BIG, HTTP_FORBIDDEN, NULL,
+    err = clientBuildError(context->errorpage, context->errorstatus, NULL,
 			   http->conn ? &http->conn->peer.sin_addr : &no_addr, http->request);
     context->rep = errorBuildReply (err);
     errorprogress = context->rep->body.mb.size;
@@ -510,23 +1022,19 @@
     while (errorprogress < context->rep->body.mb.size)
       {
 	size_t amount;
-	esiSegment *where;
 	if (!context->outbound) {
-	    context->outbound = cbdataAlloc (esiSegment);
-	    where = context->outbound;
+	    context->outboundtail = context->outbound = cbdataAlloc (esiSegment);
 	} else {
 	    /* add to tail */
-	    where = context->outbound;
-	    while (where->next)
-		where = where->next;
-	    where->next = cbdataAlloc (esiSegment);
-	    where = where->next;
+	    assert (context->outboundtail->next == NULL);
+	    context->outboundtail->next = cbdataAlloc (esiSegment);
+	    context->outboundtail = context->outboundtail->next;
 	}
 	amount = context->rep->body.mb.size - errorprogress;
 	if (amount > HTTP_REQBUF_SZ)
 	    amount = HTTP_REQBUF_SZ;
-	xmemcpy (where->buf, context->rep->body.mb.buf + errorprogress, amount);
-	where->len = amount;
+	xmemcpy (context->outboundtail->buf, context->rep->body.mb.buf + errorprogress, amount);
+	context->outboundtail->len = amount;
 	errorprogress += amount;
       }
     /* the esiCode now thinks that the error is the outbound, and all processing has finished. */
@@ -559,61 +1067,75 @@
     assert (node->data != NULL); 
     assert (node->node.next == NULL);
     assert (http->conn == NULL);
-    
-    esiStream = cbdataReference (node->data);
 
-    debug (86,0) ("esiBufferRecipient\n");
+    esiStream = cbdataReference (node->data);
+    /* If segments become more flexible, ignore this */
+    assert (body_size <= sizeof(esiStream->localbuffer.buf));
+    assert (!esiStream->finished);
+    
+    debug (86,5) ("esiBufferRecipient rep %p body %p len %d\n", rep, body_data, body_size);
 
     /* handle data - ESI TODO */
     /* basic outline follows */
     /* trivial case */
     if (http->out.offset != 0) {
         assert(rep == NULL);
-        /* Avoid copying to MemBuf if we know "rep" is NULL, and we only have a body */
-        http->out.offset += body_size;
-	/* give the data to the final user - the ESI processor in this case,
-	 * once used, schedule another read (if apporpriate) via
-	 *
-	 */
-	 
-	/* As we are a non-blocking stream, we can simply call:
-        return;
-	*/
     } else {
-/* push the headers and then the data through to -wherever-. Once the recipient has 
- * done it's thing, reschedule more data if appropriate */
-#if 0
-        MemBuf mb;
-        /* write headers and/or body if any */
-        assert(rep || (body_data && body_size));
-        /* init mb; put status line and headers if any */
-#endif
         if (rep) {
-            //mb = httpReplyPack(rep);
-//            http->out.offset += rep->hdr_sz;
+	    if (rep->sline.status != HTTP_OK) {
+		httpReplyDestroy(rep);
+		rep = NULL;
+		esiIncludeSubRequestDone (esiStream->include, esiStream, 0);
+		esiStream->finished = 1;
+		cbdataReferenceDone (esiStream);
+		httpRequestFree (http);
+		return;
+	    }
 #if HEADERS_LOG
-	    // should be done in the store rather than ever recipient? 
+	    /* should be done in the store rather than ever recipient?  */
             headersLog(0, 0, http->request->method, rep);
 #endif
             httpReplyDestroy(rep);
             rep = NULL;
-//        } else {
-//          memBufDefInit(&mb);
-        }
-        if (body_data && body_size) {
-            http->out.offset += body_size;
-//            memBufAppend(&mb, body_data, body_size);
         }
-        /* write */
-//        comm_write_mbuf(fd, mb, clientWriteComplete, http);
-        /* if we don't do it, who will? */
     }
 
+    if (body_data && body_size) {
+	http->out.offset += body_size;
+
+	if (body_data >= esiStream->localbuffer.buf &&
+	    body_data < &esiStream->localbuffer.buf[sizeof(esiStream->localbuffer.buf)]) {
+	    /* original static buffer */
+	    if (body_data != esiStream->localbuffer.buf) {
+		/* But not the start of it */
+		xmemmove (esiStream->localbuffer.buf, body_data, body_size);
+	    }
+	    esiStream->localbuffer.len = body_size;
+	} else {
+	    assert (esiStream->buffer != NULL);
+	    esiStream->buffer->len = body_size;
+	}
+    }
+
+    /* EOF / Read error /  aborted entry */
+    if (rep == NULL && body_data == NULL && body_size == 0) {
+	/* TODO: get stream status to test the entry for aborts */
+	debug (86,1)("Finished reading upstream data in subrequest\n");
+	esiIncludeSubRequestDone (esiStream->include, esiStream, 1);
+	esiStream->finished = 1;
+	cbdataReferenceDone (esiStream);
+	httpRequestFree (http);
+	return;
+    }
+	
+
     /* after the write to the user occurs, (ie here, or in a callback)
      * we call */
     if (clientHttpRequestStatus(-1, http)) {
+	/* TODO: Does this if block leak htto ? */
 	esiStreamContext *temp = esiStream;
-	/* ESI TODO: Tell the initiator we've failed */
+	esiIncludeSubRequestDone (esiStream->include, esiStream, 0);
+	esiStream->finished = 1;
 	cbdataReferenceDone (esiStream);
 	cbdataFree (temp); /* free the request */
 	return;
@@ -621,63 +1143,433 @@
     switch (clientStreamStatus (node, http)) {
         case STREAM_UNPLANNED_COMPLETE: /* fallthru ok */
         case STREAM_COMPLETE: /* ok */
-	  // usertell request finished 
-	  debug (86,0)("ESI subrequest finished OK\n");
+	  debug (86,1)("ESI subrequest finished OK\n");
+	  esiIncludeSubRequestDone (esiStream->include, esiStream, 1);
+	  esiStream->finished = 1;
 	  cbdataReferenceDone (esiStream);
 	  httpRequestFree (http);
 	  return;
         case STREAM_FAILED:
-	  // usertell request failed
-	  debug (86,0)("ESI subrequest failed transfer\n");
+	  debug (86,1)("ESI subrequest failed transfer\n");
+	  esiIncludeSubRequestDone (esiStream->include, esiStream, 0);
+	  esiStream->finished = 1;
 	  cbdataReferenceDone (esiStream);
 	  httpRequestFree (http);
           return;
-        case STREAM_NONE:
-               /* More data will be coming from primary server; register with
-                * storage manager. */
-	  clientStreamRead (node,
-			    http, http->out.offset,
-			    HTTP_REQBUF_SZ,
-			    esiStream->localbuf.buf);
+        case STREAM_NONE: {
+	    if (!esiStream->buffer) {
+		esiStream->buffer = &esiStream->localbuffer;
+	    }
+	    while (esiStream->buffer->next) esiStream->buffer = esiStream->buffer->next;
+	    if (esiStream->buffer->len) {
+		esiStream->buffer->next = cbdataAlloc (esiSegment);
+		esiStream->buffer = esiStream->buffer->next;
+	    }
+	    
+	    /* now just read into 'buffer' */
+	    clientStreamRead (node,
+			      http, http->out.offset,
+			      sizeof (esiStream->buffer->buf),
+			      esiStream->buffer->buf);
 	       debug (86,0)("Requested more data for ESI subrequest\n");
+			  }
 	       break;
         default:fatal ("Hit unreachable code in esiBufferRecipient\n");
     }
     cbdataReferenceDone (esiStream);
 }
 
-#if 0
-void
-esiStartSub (void)
-{
-    StoreEntry *e, *old_e;
-    char *url;
-    const cache_key *key;
-    request_t *req;
-
-
-    url = xstrdup("http://192.168.0.2:3128/ESI_fragment_1.txt");
-}
-#endif
-
 /* esiStream functions */
 void
 esiStreamContextFree (void *data)
 {
     esiStreamContext *esiStream = data;
     assert (esiStream);
-    cbdataReferenceDone (esiStream->http);
-    cbdataReferenceDone (esiStream->esi);
-    debug (0,0)("Freeing stream context\n");
+    cbdataReferenceDone (esiStream->include);
+    esiSegmentFreeList (&esiStream->localbuffer.next);
+    debug (86,1)("Freeing stream context\n");
 }
 
 esiStreamContext *
-esiStreamContextNew (clientHttpRequest *http, esiContext *esi)
+esiStreamContextNew (esiInclude *include)
 {
     esiStreamContext *rv = NULL;
     CBDATA_INIT_TYPE_FREECB(esiStreamContext, esiStreamContextFree);
     rv = cbdataAlloc(esiStreamContext);
-    rv->http = cbdataReference (http);
-    rv->esi = cbdataReference (esi);
+    rv->include = cbdataReference (include);
     return rv;
 }
+
+
+/* Implementation of esiElements */
+
+/* 'inherited' methods */
+
+/* Don't accept children */
+int
+esiAddFail (void *data, esiElement *element) {
+    debug (86,5)("esiAddFail: Failed for %p\n",data);
+    return -1;
+}
+
+/* always complete processing */
+esiProcessResult_t
+esiProcessComplete (void *data) {
+    debug (86,5) ("esiProcessComplete: Processed %p\n",data);
+    return ESI_PROCESS_COMPLETE;
+}
+
+/* esiComment */
+void
+esiCommentFree (void *data)
+{
+    esiComment *this = data;
+    debug (86,5)("esiCommentFree %p\n", this);
+}
+
+esiElement *
+esiCommentNew ()
+{
+    esiComment *rv;
+    CBDATA_INIT_TYPE_FREECB(esiComment, esiCommentFree);
+    rv = cbdataAlloc (esiComment);
+    rv->vptr = &_esiComment;
+    return (esiElement *) rv;
+}
+
+void
+esiCommentRender (void *data, esiSegment *output)
+{
+    /* Comments do nothing dude */
+    debug (86, 5)("esiCommentRender: Rendering comment %p\n", data);
+}
+
+/* esiLiteral */
+void
+esiLiteralFree (void *data)
+{
+    esiLiteral *this = data;
+    debug (86, 5) ("esiLiteralFree %p\n", this);
+    esiSegmentFreeList (&this->buffer);
+}
+
+/* precondition: the buffer chain has at least start + length bytes of data
+ */
+esiElement *
+esiLiteralNew (const XML_Char *s, int length)
+{
+    esiLiteral *rv;
+    esiSegment *local;
+    off_t start;
+    CBDATA_INIT_TYPE_FREECB(esiLiteral, esiLiteralFree);
+    rv = cbdataAlloc (esiLiteral);
+    rv->vptr = &_esiLiteral;
+    assert (s);
+    rv->buffer = cbdataAlloc (esiSegment);
+    local = rv->buffer;
+    start = 0;
+    while (length > 0) {
+	off_t len = sizeof (local->buf) - local->len;
+	if (len > length)
+	    len = length;
+	xmemcpy (&local->buf[local->len], &s[start], len);
+	local->len += len;
+	length -= len;
+	s += len;
+	if (local->len == sizeof (local->buf) && length){
+	    local->next = cbdataAlloc (esiSegment);
+	    local=local->next;
+	}
+    }
+    return (esiElement *)rv;
+}
+
+void
+esiLiteralRender (void *data, esiSegment *output)
+{
+    /* append the entire chain */
+    esiLiteral *this = data;
+    esiSegment *myout = output;
+    assert (myout->next == NULL);
+    myout->next = this->buffer;
+    this->buffer = NULL;
+}
+
+/* esiSequence */
+void
+esiSequenceFree (void *data)
+{
+    esiSequence *this = data;
+    int i;
+    debug (86,1)("esiSequenceFree %p\n", this);
+    for (i = 0; i < this->elementcount; ++i) {
+	cbdataFree(this->elements[i]);
+      }
+    if (this->elements)
+	memFreeBuf (this->allocedsize, this->elements);
+}
+
+esiElement *
+esiSequenceNew ()
+{
+    esiSequence *rv;
+    CBDATA_INIT_TYPE_FREECB(esiSequence, esiSequenceFree);
+    rv = cbdataAlloc (esiSequence);
+    rv->vptr = &_esiSequence;
+    rv->elements = NULL;
+    return (esiElement *)rv;
+}
+
+void
+esiSequenceRender (void *data, esiSegment *output)
+{
+    /* append all processed elements, and trim processed and rendered elements */
+    esiSequence *this = data;
+    int i;
+    assert (output->next == NULL);
+    debug (86,5)("esiSequenceRender: rendering %d elements\n", this->processedcount);
+    for (i = 0; i < this->processedcount; ++i) {
+	RENDER (this->elements[i], output);
+	/* FIXME: pass a esiSegment ** ? */
+	while (output->next)
+	    output = output->next;
+	cbdataFree (this->elements[i]);
+    }
+    if (this->processedcount) {
+	xmemmove (this->elements, &this->elements[this->processedcount], (this->elementcount - this->processedcount)  * sizeof (esiElement *));
+    }
+    this->elementcount -= this->processedcount;
+    this->processedcount = 0;
+    assert (output->next == NULL);
+}
+
+int
+esiSequenceAdd (void *data, esiElement *element)
+{
+    /* add an element to the output list */
+    esiSequence *this = data;
+    this->elements = memReallocBuf (this->elements, ++this->elementcount * sizeof (esiElement *),
+				    &this->allocedsize);
+    assert (this->elements);
+    this->allocedcount = this->elementcount;
+    this->elements[this->elementcount - 1] = element;
+    debug (86,3)("esiSequenceAdd: Added a new element, elements = %d\n", this->elementcount);
+    return 0;
+}
+
+esiProcessResult_t
+esiSequenceProcess (void *data)
+{
+    /* process as much of the list as we can, stopping only on
+     * faliures
+     */
+    esiSequence *this = data;
+    int i;
+    esiProcessResult_t rv = ESI_PROCESS_COMPLETE;
+    for (i = this->processedcount; i < this->elementcount; ++i) {
+	switch (PROCESS(this->elements[i])){
+	    case ESI_PROCESS_COMPLETE:
+	        debug (86,5)("esiSequenceProcess: element Processed OK\n");
+		if (i == this->processedcount)
+		    /* another completely ready */
+		    ++this->processedcount;
+		break;
+	    case ESI_PROCESS_PENDING_WONTFAIL:
+		debug (86,5)("esiSequenceProcess: element Processed PENDING OK\n");
+		if (rv < ESI_PROCESS_PENDING_WONTFAIL)
+		    rv = ESI_PROCESS_PENDING_WONTFAIL;
+		break;
+	    case ESI_PROCESS_PENDING_MAYFAIL:
+		debug (86,5)("eseSequenceProcess: element Processed PENDING UNKNOWN\n");
+		if (rv < ESI_PROCESS_PENDING_MAYFAIL)
+		    rv = ESI_PROCESS_PENDING_MAYFAIL;
+		break;
+	    case ESI_PROCESS_FAILED:
+		debug (86,5)("esiSequenceProcess: elemen Processed FAILED\n");
+		return ESI_PROCESS_FAILED;
+		break;
+	}
+    }
+    assert (rv != ESI_PROCESS_COMPLETE || this->processedcount == this->elementcount);
+    return rv;
+}
+
+/* esiInclude */
+void
+esiIncludeFree (void *data)
+{
+    esiInclude *this = data;
+    debug (86,5)("esiIncludeFree %p\n", this);
+    esiSegmentFreeList (&this->srccontent);
+    esiSegmentFreeList (&this->altcontent);
+    cbdataReferenceDone (this->context);
+    safe_free (this->srcurl);
+    safe_free (this->alturl);
+}
+
+void
+esiIncludeStart (esiStreamContext *, char const *);
+
+void
+esiIncludeStart (esiStreamContext *stream, char const *url)
+{
+    HttpHeader tempheaders;
+    if (!stream)
+	return;
+    httpHeaderInit (&tempheaders, hoRequest);
+    httpHeaderPutStr (&tempheaders, HDR_SURROGATE_CAPABILITY, "Surrogate/1.0");
+    /* TODO: evaluate variables */
+    if (clientBeginRequest(METHOD_GET, url, esiBufferRecipient, stream, &tempheaders, stream->localbuffer.buf, HTTP_REQBUF_SZ)) {
+	debug (86,0 ) ("starting new ESI subrequest failed\n");
+    }
+    httpHeaderClean (&tempheaders);
+    
+}
+
+esiElement *
+esiIncludeNew (int attrcount, char const **attr, esiContext *context)
+{
+    esiInclude *rv;
+    int i;
+    assert (context);
+    CBDATA_INIT_TYPE_FREECB(esiInclude, esiIncludeFree);
+    rv = cbdataAlloc (esiInclude);
+    rv->vptr = &_esiInclude;
+
+    for (i = 0; attr[i] && i < attrcount; i += 2) {
+	if (!strcmp(attr[i],"src")) {
+	    /* Start a request for this url */
+	    debug (86,5)("esiIncludeNew: Requesting source '%s'\n",attr[i+1]);
+	    /* TODO: don't assert on this, ignore the duplicate */
+	    assert (rv->src == NULL);
+	    rv->src = esiStreamContextNew (rv);
+	    assert (rv->src != NULL);
+	    rv->srcurl =xstrdup ( attr[i+1]);
+	} else if (!strcmp(attr[i],"alt")) {
+	    /* Start a secondary request for this url */
+	    /* TODO: make a config parameter to wait on requesting alt's 
+	     * for the src to fail
+	     */
+	    debug (86,5)("esiIncludeNew: Requesting alternate '%s'\n",attr[i+1]);
+	    assert (rv->alt == NULL); /* TODO: FIXME */
+	    rv->alt = esiStreamContextNew (rv);
+	    assert (rv->alt != NULL);
+	    rv->alturl = xstrdup (attr[i+1]);
+	} else if (!strcmp(attr[i],"onerror")) {
+	    if (!strcmp(attr[i+1], "continue")) {
+		rv->flags.onerrorcontinue = 1;
+	    } else {
+		/* ignore mistyped attributes */
+		debug (86, 1)("invalid value for onerror='%s'\n", attr[i+1]);
+	    }
+	} else {
+	    /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
+	     */
+	}
+      }
+    rv->context = cbdataReference(context); 
+    esiIncludeStart (rv->src, rv->srcurl);
+    esiIncludeStart (rv->alt, rv->alturl);
+    return (esiElement *)rv;
+}
+
+void
+esiIncludeRender (void *data, esiSegment *output)
+{
+    esiInclude *this = data;
+    esiSegment *myout = NULL;
+    debug (86, 5)("esiIncludeRender: Rendering include %p\n", this);
+    assert (this->flags.finished || (this->flags.failed && this->flags.onerrorcontinue));
+    if (this->flags.failed && this->flags.onerrorcontinue) {
+	return;
+    }
+    /* Render the content */
+    if (this->srccontent) {
+	myout = this->srccontent;
+	this->srccontent = NULL;
+    } else if (this->altcontent) {
+	myout = this->altcontent;
+	this->altcontent = NULL;
+    } else 
+	fatal ("esiIncludeRender called with no content, and no failure!\n");
+    assert (output->next == NULL);
+    output->next = myout;
+}
+
+esiProcessResult_t
+esiIncludeProcess (void *data)
+{
+    esiInclude *this = data;
+    debug (86, 5)("esiIncludeRender: Processing include %p\n", this);
+    if (this->flags.failed) {
+	if (this->flags.onerrorcontinue)
+	    return ESI_PROCESS_COMPLETE;
+	else
+	    return ESI_PROCESS_FAILED;
+    }
+    if (!this->flags.finished) {
+	if (this->flags.onerrorcontinue)
+	    return ESI_PROCESS_PENDING_WONTFAIL;
+	else
+	    return ESI_PROCESS_PENDING_MAYFAIL;
+    }
+    return ESI_PROCESS_COMPLETE;
+}
+
+void
+esiIncludeSubRequestDone(esiInclude *this, esiStreamContext *stream, int success)
+{
+    assert (this);
+    if (this->flags.finished || this->flags.failed)
+	/* Do nothing, we don't need the data */
+	return;
+    if (stream == this->src) {
+	if (success) {
+	    /* copy the lead segment */
+	    this->srccontent = cbdataAlloc (esiSegment);
+	    xmemcpy (this->srccontent->buf, stream->localbuffer.buf, stream->localbuffer.len);
+	    this->srccontent->len = stream->localbuffer.len;
+	    this->srccontent->next = stream->localbuffer.next;
+	    stream->localbuffer.next = NULL;
+	    /* we're done! */
+	    this->flags.finished = 1;
+	} else {
+	    /* Fail if there is no alt being retrieved */
+	    if (!(this->alt || this->altcontent)) {
+		this->flags.failed = 1;
+	    } else if (this->altcontent) {
+		/* ALT was already retrieved, we are done */
+		this->flags.finished = 1;
+	    }
+	}
+	this->src = NULL;
+    } else if (stream == this->alt) {
+	if (success) {
+	    /* copy the lead segment */
+	    this->altcontent = cbdataAlloc (esiSegment);
+	    xmemcpy (this->altcontent->buf, stream->localbuffer.buf, stream->localbuffer.len);
+	    this->altcontent->len = stream->localbuffer.len;
+	    this->altcontent->next = stream->localbuffer.next;
+	    stream->localbuffer.next = NULL;
+	    /* we're done! */
+	    if (!(this->src || this->srccontent)) {
+		/* src already failed, kick ESI processor */
+		this->flags.finished = 1;
+	    }
+	} else {
+	    if (!(this->src || this->srccontent)) {
+		/* src already failed */
+		this->flags.failed = 1;
+	    }
+	}
+	this->alt = NULL;
+    } else {
+	fatal ("esiIncludeSubRequestDone: non-owned stream found!\n");
+    }
+    if (this->flags.finished || this->flags.failed) {
+	/* Kick ESI Processor */
+	debug (86,1)("esiInclude %p SubRequest %p completed, kicking processor , status %s\n", this, stream, this->flags.finished ? "OK" : "FAILED");
+	assert (this->context);
+	esiKick (this->context);
+    }
+}
Index: squid/src/Makefile.am
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v
retrieving revision 1.23.2.1
retrieving revision 1.23.2.2
diff -u -r1.23.2.1 -r1.23.2.2
--- squid/src/Makefile.am	9 Aug 2002 11:49:45 -0000	1.23.2.1
+++ squid/src/Makefile.am	18 Aug 2002 11:09:22 -0000	1.23.2.2
@@ -26,6 +26,12 @@
 DELAY_POOL_SOURCE = 
 endif
 
+if USE_ESI
+  ESI_SOURCE = ESI.c
+else
+  ESI_SOURCE = 
+endif
+
 if ENABLE_HTCP
 HTCPSOURCE = htcp.c
 endif
@@ -130,7 +136,7 @@
 	$(DNSSOURCE) \
 	enums.h \
 	errorpage.c \
-	ESI.c \
+	$(ESI_SOURCE) \
 	ETag.c \
 	event.c \
 	external_acl.c \
Index: squid/src/client_side.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v
retrieving revision 1.65.2.15
retrieving revision 1.65.2.16
diff -u -r1.65.2.15 -r1.65.2.16
--- squid/src/client_side.c	16 Aug 2002 01:12:32 -0000	1.65.2.15
+++ squid/src/client_side.c	18 Aug 2002 11:09:22 -0000	1.65.2.16
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side.c,v 1.65.2.15 2002/08/16 01:12:32 rbcollins Exp $
+ * $Id: client_side.c,v 1.65.2.16 2002/08/18 11:09:22 rbcollins Exp $
  *
  * DEBUG: section 33    Client-side Routines
  * AUTHOR: Duane Wessels
@@ -477,7 +477,7 @@
     clientSocketContext *context;
     /* Test preconditions */
     assert (node != NULL);
-    /* ESI TODO: handle this rather than asserting - it should only ever happen if we cause an abort and 
+    /* TODO: handle this rather than asserting - it should only ever happen if we cause an abort and 
      * the callback chain loops back to here, so we can simply return. 
      * However, that itself shouldn't happen, so it stays as an assert for now. 
      */
Index: squid/src/client_side_reply.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Attic/client_side_reply.c,v
retrieving revision 1.1.2.14
retrieving revision 1.1.2.15
diff -u -r1.1.2.14 -r1.1.2.15
--- squid/src/client_side_reply.c	16 Aug 2002 08:33:00 -0000	1.1.2.14
+++ squid/src/client_side_reply.c	18 Aug 2002 11:09:23 -0000	1.1.2.15
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side_reply.c,v 1.1.2.14 2002/08/16 08:33:00 rbcollins Exp $
+ * $Id: client_side_reply.c,v 1.1.2.15 2002/08/18 11:09:23 rbcollins Exp $
  *
  * DEBUG: section 88    Client-side Reply Routines
  * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
@@ -75,6 +75,8 @@
 static StoreEntry *clientCreateStoreEntry(clientReplyContext *, method_t, request_flags);
 static STCB clientSendMoreData;
 static void clientRemoveStoreReference (clientReplyContext *, store_client **, StoreEntry **);
+static void clientReplyContextSaveState (clientReplyContext *, clientHttpRequest *);
+static void clientReplyContextRestoreState (clientReplyContext *, clientHttpRequest *);
 extern CSS clientReplyStatus;
 extern ErrorState *clientBuildError (err_type, http_status, char const *, struct in_addr *, request_t *);
 
@@ -143,6 +145,39 @@
     }
 }
 
+void
+clientReplyContextSaveState (clientReplyContext *this, clientHttpRequest *http)
+{
+   assert (this->old_sc == NULL);
+   debug (88,1)("clientReplyContextSaveState: saving store context\n");
+   http->old_entry = http->entry;
+   this->old_sc = this->sc;
+   this->old_reqsize = this->reqsize;
+   this->old_reqofs = this->reqofs;
+   /* Prevent accessing the now saved entries */
+   http->entry = NULL;
+   this->sc = NULL;
+   this->reqsize = 0;
+   this->reqofs = 0;
+}
+
+void
+clientReplyContextRestoreState (clientReplyContext *this, clientHttpRequest *http)
+{
+    assert (this->old_sc != NULL);
+    debug (88,1)("clientReplyContextRestoreState: Restoring store context\n");
+    http->entry = http->old_entry;
+    this->sc = this->old_sc;
+    this->reqsize = this->old_reqsize;
+    this->reqofs = this->old_reqofs;
+    /* Prevent accessed the old saved entries */
+    http->old_entry = NULL;
+    this->old_sc = NULL;
+    this->old_reqsize = 0;
+    this->old_reqofs = 0;
+}
+
+
 /* there is an expired entry in the store.
  * setup a temporary buffer area and perform an IMS to the origin
  */
@@ -164,18 +199,16 @@
         return;
     }
     http->request->flags.refresh = 1;
-    http->old_entry = http->entry;
-    context->old_sc = context->sc;
-    context->old_reqsize = context->reqsize;
-    context->old_reqofs = context->reqofs;
 #if STORE_CLIENT_LIST_DEBUG
     /*
      * Assert that 'http' is already a client of old_entry.  If 
      * it is not, then the beginning of the object data might get
      * freed from memory before we need to access it.
      */
-    assert(http->sc->owner == http);
+    assert(http->sc->owner == context);
 #endif
+    /* Prepare to make a new temporary request */
+    clientReplyContextSaveState (context, http);
     entry = storeCreateEntry(url,
         http->log_uri,
         http->request->flags,
@@ -189,13 +222,12 @@
     http->request->lastmod = http->old_entry->lastmod;
     debug(88, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod);
     http->entry = entry;
-    http->out.offset = 0;
+    http->out.offset = 0; /* FIXME Not needed - we have not written anything anyway */
     fwdStart(http->conn ? http->conn->fd : -1, http->entry, http->request);
     /* Register with storage manager to receive updates when data comes in. */
     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
         debug(88, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n");
     /* start counting the length from 0 */
-    context->reqofs = 0;
     storeClientCopy(context->sc, entry,
         0,
         HTTP_REQBUF_SZ,
@@ -303,10 +335,9 @@
         /* Its okay to send the old one anyway */
         http->log_type = LOG_TCP_REFRESH_FAIL_HIT;
 	clientRemoveStoreReference (context, &context->sc, &entry);
-        entry = http->entry = http->old_entry;
-        context->sc = context->old_sc;
-        context->reqofs = context->old_reqofs;
-        context->reqsize = context->old_reqsize;
+	/* Get the old request back */
+	clientReplyContextRestoreState (context, http);
+	entry = http->entry;
     } else if (STORE_PENDING == entry->store_status && 0 == status) {
 	/* more headers needed to decide */
         debug(88, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url);
@@ -317,15 +348,13 @@
             http->log_type = LOG_TCP_REFRESH_FAIL_HIT;
 	    clientRemoveStoreReference (context, &context->sc, &entry);
             entry = http->entry = http->old_entry;
-            context->sc = context->old_sc;
-//            http->reqbuf = http->norm_reqbuf;
-            context->reqofs = context->old_reqofs;
-            context->reqsize = context->old_reqsize;
+	    /* Get the old request back */
+	    clientReplyContextRestoreState (context, http);
+	    entry = http->entry;
             /* continue */
         } else {
-//            http->reqofs += size;
             storeClientCopy(context->sc, entry,
-                http->out.offset + context->reqofs,
+                context->reqofs,
                 HTTP_REQBUF_SZ - context->reqofs,
                 context->tempbuf + context->reqofs,
                 clientHandleIMSReply,
@@ -349,21 +378,15 @@
         httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply);
         storeTimestampsSet(oldentry);
 	clientRemoveStoreReference (context, &context->sc, &entry);
-        context->sc = context->old_sc;
-        entry = http->entry = oldentry;
-        entry->timestamp = squid_curtime;
+        oldentry->timestamp = squid_curtime;
         if (unlink_request) {
-            requestUnlink(entry->mem_obj->request);
-            entry->mem_obj->request = NULL;
+            requestUnlink(oldentry->mem_obj->request);
+            oldentry->mem_obj->request = NULL;
         }
-//        http->reqbuf = http->norm_reqbuf;
-        context->reqofs = context->old_reqofs;
-        context->reqsize = context->old_reqsize;
+	/* Get the old request back */
+	clientReplyContextRestoreState (context, http);
+	entry = http->entry;
 	/* here the data to send is in the next nodes buffers already */
-	http->old_entry = NULL;     /* done with old_entry */
-	context->old_sc = NULL;
-	context->old_reqofs = 0;
-	context->old_reqsize = 0;
 	assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED));
 	clientSendMoreData(context, context->tempbuf, context->reqsize);
     } else {
@@ -835,6 +858,7 @@
     /*
      * Handle STORE_OK objects.
      * objectLen(entry) will be set proprely.
+     * RC: Does objectLen(entry) include the Headers? 
      */
     if (entry->store_status == STORE_OK) {
         if (http->out.offset >= objectLen(entry))
Index: squid/src/mem.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/mem.c,v
retrieving revision 1.20
retrieving revision 1.20.2.1
diff -u -r1.20 -r1.20.2.1
--- squid/src/mem.c	21 Jul 2002 01:06:07 -0000	1.20
+++ squid/src/mem.c	18 Aug 2002 11:09:23 -0000	1.20.2.1
@@ -1,6 +1,6 @@
 
 /*
- * $Id: mem.c,v 1.20 2002/07/21 01:06:07 squidadm Exp $
+ * $Id: mem.c,v 1.20.2.1 2002/08/18 11:09:23 rbcollins Exp $
  *
  * DEBUG: section 13    High Level Memory Pool Management
  * AUTHOR: Harvest Derived
@@ -263,6 +263,7 @@
 memReallocBuf(void *oldbuf, size_t net_size, size_t * gross_size)
 {
     /* XXX This can be optimized on very large buffers to use realloc() */
+    /* TODO: if the existing gross size is >= new gross size, do nothing */
     int new_gross_size;
     void *newbuf = memAllocBuf(net_size, &new_gross_size);
     if (oldbuf) {
