---------------------
PatchSet 1311 
Date: 2001/01/15 23:53:03
Author: rbcollins
Branch: te_modules
Tag: (none) 
Log:
deflate seems good, gzip work started, rproxy experimentation (#ifdefed)

Members: 
	acconfig.h:1.1.1.3.8.6->1.1.1.3.8.6.2.1 
	configure.in:1.1.1.3.8.10.2.1->1.1.1.3.8.10.2.2 
	src/http.c:1.1.1.3.4.1.4.10.2.2->1.1.1.3.4.1.4.10.2.3 
	src/protos.h:1.1.1.3.8.10.2.2->1.1.1.3.8.10.2.3 
	src/transfer-encoding.c:1.1.4.3->1.1.4.4 

Index: squid/acconfig.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/Attic/acconfig.h,v
retrieving revision 1.1.1.3.8.6
retrieving revision 1.1.1.3.8.6.2.1
diff -u -r1.1.1.3.8.6 -r1.1.1.3.8.6.2.1
--- squid/acconfig.h	10 Jan 2001 13:09:13 -0000	1.1.1.3.8.6
+++ squid/acconfig.h	15 Jan 2001 23:53:03 -0000	1.1.1.3.8.6.2.1
@@ -20,7 +20,7 @@
  *
  */
 @ TOP @
-/* $Id: acconfig.h,v 1.1.1.3.8.6 2001/01/10 13:09:13 rbcollins Exp $ */
+/* $Id: acconfig.h,v 1.1.1.3.8.6.2.1 2001/01/15 23:53:03 rbcollins Exp $ */
 
 /*********************************
  * START OF CONFIGURABLE OPTIONS *
@@ -275,6 +275,11 @@
 #undef ATHY_COMPRESSION
 
 /*
+ * Use the freely available zlib compression/decompression library
+ */
+#undef ZLIB_COMPRESSION
+
+/*
  * message type for message queues
  */
 #undef mtyp_t
Index: squid/configure.in
===================================================================
RCS file: /cvsroot/squid-sf//squid/configure.in,v
retrieving revision 1.1.1.3.8.10.2.1
retrieving revision 1.1.1.3.8.10.2.2
diff -u -r1.1.1.3.8.10.2.1 -r1.1.1.3.8.10.2.2
--- squid/configure.in	11 Jan 2001 12:14:59 -0000	1.1.1.3.8.10.2.1
+++ squid/configure.in	15 Jan 2001 23:53:03 -0000	1.1.1.3.8.10.2.2
@@ -3,13 +3,13 @@
 dnl
 dnl  Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9)
 dnl
-dnl  $Id: configure.in,v 1.1.1.3.8.10.2.1 2001/01/11 12:14:59 rbcollins Exp $
+dnl  $Id: configure.in,v 1.1.1.3.8.10.2.2 2001/01/15 23:53:03 rbcollins Exp $
 dnl
 dnl
 dnl
 AC_INIT(src/main.c)
 AC_CONFIG_HEADER(include/autoconf.h)
-AC_REVISION($Revision: 1.1.1.3.8.10.2.1 $)dnl
+AC_REVISION($Revision: 1.1.1.3.8.10.2.2 $)dnl
 AC_PREFIX_DEFAULT(/usr/local/squid)
 AC_CONFIG_AUX_DIR(cfgaux)
 
@@ -688,6 +688,41 @@
   fi
 ])
 
+AC_ARG_WITH(pthreads,
+[  --with-pthreads         Use POSIX Threads],
+[ if test "$enableval" = "yes"; then
+    with_pthreads=yes
+  fi
+])
+if test "$with_pthreads"; then
+    echo "With pthreads"
+    SQUID_PTHREAD_LIB='$(PTHREADLIB)'
+    CFLAGS="$CFLAGS -D_REENTRANT"
+    case "$host" in
+    i386-unknown-freebsd*)
+        if test "$GCC" = "yes" ; then
+            if test -z "$PRESET_LDFLAGS"; then
+                LDFLAGS="$LDFLAGS -pthread"
+            fi
+        fi
+    ;;
+    esac
+fi
+
+dnl Enable zlib_compression
+AC_ARG_WITH(zlib,
+[  --with-zlib            This option makes use of libz (zlib) to handle
+                          compressed transfer encodings.],
+[ if test "$enableval" = "yes" ; then
+    with_zlib=yes
+  fi
+])
+if test "$with_zlib"; then
+    echo "Enabling ZLIB_COMPRESSION"
+    AC_DEFINE(ZLIB_COMPRESSION, 1)
+    LIBS="-lz $LIBS"
+fi
+
 dnl Select auth schemes modules to build
 AC_ARG_ENABLE(auth,
 [  --enable-auth=\"list of auth scheme modules\"
Index: squid/src/http.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/http.c,v
retrieving revision 1.1.1.3.4.1.4.10.2.2
retrieving revision 1.1.1.3.4.1.4.10.2.3
diff -u -r1.1.1.3.4.1.4.10.2.2 -r1.1.1.3.4.1.4.10.2.3
--- squid/src/http.c	15 Jan 2001 09:38:17 -0000	1.1.1.3.4.1.4.10.2.2
+++ squid/src/http.c	15 Jan 2001 23:53:04 -0000	1.1.1.3.4.1.4.10.2.3
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.c,v 1.1.1.3.4.1.4.10.2.2 2001/01/15 09:38:17 rbcollins Exp $
+ * $Id: http.c,v 1.1.1.3.4.1.4.10.2.3 2001/01/15 23:53:04 rbcollins Exp $
  *
  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
  * AUTHOR: Harvest Derived
@@ -837,22 +837,8 @@
     if (Config.fake_ua && !httpHeaderHas(hdr_out, HDR_USER_AGENT))
 	httpHeaderPutStr(hdr_out, HDR_USER_AGENT, Config.fake_ua);
 #endif
-    /* append TE */
-    {
-      char strTE[128];
-      
-      httpHeaderDelById(hdr_out, HDR_TE); /* hop by hop.. that's what last
-				       hop could do*/
-#if ATHY_COMPRESSION || ZLIB_COMPRESSION
-      strcpy (strTE,"biff;q=0.0, gzip;q=1.0, deflate;q=1.0, jones;q=0.0, chunked;q=1.0");
-#else
-      strcpy (strTE,"biff;q=1.0, gzip;q=0.0, deflate;q=0.0, jones;q=1.0, chunked;q=1.0");
-#endif
 
-      httpHeaderPutStr (hdr_out,HDR_TE,strTE);
-      httpHeaderPutStr(hdr_out,HDR_CONNECTION, "TE"); /*  its hop by hop */
-
-    }
+    te_generate_request(hdr_out);
     
     /* append Via */
     strVia = httpHeaderGetList(hdr_in, HDR_VIA);
Index: squid/src/protos.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/protos.h,v
retrieving revision 1.1.1.3.8.10.2.2
retrieving revision 1.1.1.3.8.10.2.3
diff -u -r1.1.1.3.8.10.2.2 -r1.1.1.3.8.10.2.3
--- squid/src/protos.h	14 Jan 2001 13:11:21 -0000	1.1.1.3.8.10.2.2
+++ squid/src/protos.h	15 Jan 2001 23:53:04 -0000	1.1.1.3.8.10.2.3
@@ -1,6 +1,6 @@
 
 /*
- * $Id: protos.h,v 1.1.1.3.8.10.2.2 2001/01/14 13:11:21 rbcollins Exp $
+ * $Id: protos.h,v 1.1.1.3.8.10.2.3 2001/01/15 23:53:04 rbcollins Exp $
  *
  *
  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
@@ -1297,4 +1297,4 @@
 /* alters hdr as it build the list */
 extern void te_build_decode_xlate_list(HttpHeader * hdr, TE_list **translations);
 extern void te_build_encode_xlate_list(const HttpHeader *ohdr, HttpHeader *hdr, TE_list **translations, http_version_t http_ver, request_flags *flags);
-
+extern void te_generate_request(HttpHeader * hdrout);
Index: squid/src/transfer-encoding.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Attic/transfer-encoding.c,v
retrieving revision 1.1.4.3
retrieving revision 1.1.4.4
diff -u -r1.1.4.3 -r1.1.4.4
--- squid/src/transfer-encoding.c	15 Jan 2001 09:38:18 -0000	1.1.4.3
+++ squid/src/transfer-encoding.c	15 Jan 2001 23:53:04 -0000	1.1.4.4
@@ -48,6 +48,11 @@
 #endif
 #endif
 
+#define HAVE_H_HSYNC
+#ifdef HAVE_H_HSYNC
+#include <hsync.h>
+#endif
+
 /* NOTES: TEFILTERS MUST set obuf and obufl */
 /*        TEFILTERS use obuf for output IFF the outpub <= the input size */
 /*       ibuf always == obuf */
@@ -762,7 +767,7 @@
             output+= DEFLATEBLOCKSIZE-zdata->avail_out;
             if (zdata->avail_in) {
                 /* ran out of space */
-                debug(82,1)("undodeflate allocating another %d bytes - buffer now %d\n",DEFLATEBLOCKSIZE,output+DEFLATEBLOCKSIZE);
+                debug(82,1)("undodeflate allocating another %d bytes - buffer now %d - remaining bytes to be process %d\n",DEFLATEBLOCKSIZE,output+DEFLATEBLOCKSIZE,zdata->avail_in);
                 tempbuf=*obuf;
                 *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
                 memcpy(*obuf,tempbuf,output);
@@ -827,12 +832,526 @@
     return TE_BUFFER_ALLOCATED;  /* & TE_BUFFERED_OUTPUT */
 }
 
+
+static int gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */
+/* gzip flag byte */
+#define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
+#define HEAD_CRC     0x02 /* bit 1 set: header CRC present */
+#define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
+#define ORIG_NAME    0x08 /* bit 3 set: original file name present */
+#define COMMENT      0x10 /* bit 4 set: file comment present */
+#define RESERVED     0xE0 /* bits 5..7: reserved */
+
+typedef struct _gzipstate {
+    z_stream stream;
+    uLong crc;
+    } gzipstate;
+
+/* this is a TE filter */
+/* look at gzio.c in zlib's source for the inspiration code */
+static int dogzip (char *ibuf, size_t ibufl, char **obuf, size_t *obufl, void **d)
+{
+/* 4K. TODO: lookup membuf 4k thingy code */
+/* this hsould be large enough to handle the gzip header */
+#define DEFLATEBLOCKSIZE 4096
+    gzipstate * zdata;
+    size_t output=0;
+    int rv;
+    /* if we run out of output space, we have to store the already output data */
+    char *tempbuf=NULL;
+
+    if (!ibufl && !*d) {
+        /* we haven't been inited and there's nothing to do */
+        *obufl=0;
+        *obuf=NULL;
+        return 1;
+    }
+    zdata=*d;
+    assert (!((ibufl==0) && ibuf != NULL));
+    if (!*d) {
+        int rv;
+        /* first call - no data in progress */
+        zdata=xmalloc(sizeof(gzipstate));
+        *d=zdata;
+        assert(zdata!= NULL);
+        zdata->stream.zalloc=compcalloc;
+        zdata->stream.zfree=compfree;
+        zdata->stream.opaque=NULL;
+	zdata->crc=crc32(0L, Z_NULL, 0);
+	zdata->stream.next_in=zdata->stream.next_out=NULL;
+	zdata->stream.avail_in=zdata->stream.avail_out=0;	
+
+        rv=deflateInit2(&(zdata->stream), Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS,
+		8, Z_DEFAULT_STRATEGY);
+        debug (82,1)("dogzip: initialising compression library: rv is %d\n",rv);
+        if (rv != Z_OK) {
+            /* error initing compressiong */
+            debug(82,1)("dogzip: compression library returned: %s\n",zdata->stream.msg);
+            return 0;
+        }
+	*obuf= xmalloc (11);
+	snprintf(*obuf,11, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1],
+	    Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/, 0x03);
+	output=10;
+	tempbuf=*obuf;
+	*obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+	memcpy(*obuf,tempbuf,output);
+	xfree(tempbuf);
+    } else {
+	*obuf = xmalloc( DEFLATEBLOCKSIZE);
+    }
+    zdata->stream.next_in=ibuf;
+    zdata->stream.avail_in=ibufl;
+    debug(82,1)("available buffer = %d ibufl  = %d\n", DEFLATEBLOCKSIZE,ibufl);
+    if (ibufl >0) {
+        debug(82,1)("dogzip compressing %d bytes\n",ibufl);
+        do {
+            zdata->stream.next_out=*obuf+output;
+            zdata->stream.avail_out=DEFLATEBLOCKSIZE;
+            rv=deflate(&(zdata->stream),Z_NO_FLUSH);
+            output+= DEFLATEBLOCKSIZE-zdata->stream.avail_out;
+            if (zdata->stream.avail_in) {
+                /* ran out of space */
+                debug(82,1)("dogzip allocating another %d bytes\n",DEFLATEBLOCKSIZE);
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (rv != Z_OK) {
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                deflateEnd(&(zdata->stream));
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (zdata->stream.avail_in);
+	zdata->crc = crc32(zdata->crc, ibuf, ibufl);
+    }
+    else {
+        debug(82,1)("dogzip flushing buffer at EOF\n");
+        /* flush the remaining data */
+        do {
+            zdata->stream.avail_out= DEFLATEBLOCKSIZE;
+            zdata->stream.next_out=*obuf+output;
+            rv=deflate (&(zdata->stream), Z_FINISH);
+            output+= DEFLATEBLOCKSIZE-zdata->stream.avail_out;
+            if (rv==Z_OK && (zdata->stream.avail_out == 0)) {
+                /* ran out of space */
+                debug(82,1)("dogzip allocating another %d bytes\n",DEFLATEBLOCKSIZE);
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (rv != Z_STREAM_END) {
+                /* wasn't out of space, and not all finished */
+                debug(82,1)("dogzip: error flushing data\n");
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                deflateEnd(&(zdata->stream));
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (rv != Z_STREAM_END);
+	if (zdata->stream.avail_out<9) {
+	    /* ran out of space */
+	    debug(82,1)("dogzip allocating another %d bytes\n",DEFLATEBLOCKSIZE);
+	    tempbuf=*obuf;
+	    *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+	    memcpy(*obuf,tempbuf,output);
+	    xfree(tempbuf);
+	}
+	/* write checksum and content length */
+	{
+	    int n;
+	    for (n = 0; n < 4; n++) {
+	        *(*obuf+output++)=(unsigned char)(zdata->crc & 0xff);
+	        zdata->crc >>= 8;
+	    }
+	}
+	{
+            int n;
+	    for (n = 0; n < 4; n++) {
+	    *(*obuf+output++)=(unsigned char)(zdata->stream.total_in & 0xff);
+	    zdata->stream.total_in >>= 8;
+	    }
+	}
+        deflateEnd(&(zdata->stream));
+        xfree(zdata);
+        *d=NULL;
+
+    }
+    debug(82,1)("avail in %d, out %d\n",zdata->stream.avail_in, zdata->stream.avail_out);
+    debug(82,1)("created block of %d bytes\n",output);
+    *obufl=output;
+    return TE_BUFFER_ALLOCATED;  /* & TE_BUFFERED_OUTPUT */
+}
+
+/* this is a TE filter */
+static int undogzip(char *ibuf, size_t ibufl, char **obuf, size_t *obufl, void **d)
+{
+#if 0
+/* 4K. TODO: lookup membuf 4k thingy code */
+#define DEFLATEBLOCKSIZE 4096
+    gzipstate *zdata;
+    size_t output=0;
+    int rv;
+    /* if we run out of output space, we have to store the already output data */
+    char *tempbuf=NULL;
+
+    if (!ibufl && !*d) {
+        /* we haven't been inited and there's nothing to do */
+        *obufl=0;
+        *obuf=NULL;
+        return 0;
+    }
+    if (!*d) {
+        int rv;
+        /* first call - no data in progress */
+        zdata=xmalloc(sizeof(gzipstate));
+        *d=zdata;
+        assert(zdata!= NULL);
+        zdata->zalloc=compcalloc;
+        zdata->zfree=compfree;
+        zdata->opaque=NULL;
+        zdata->next_in=ibuf;
+        /* defer header reading */
+        zdata->avail_in=0;
+
+        rv=inflateInit(zdata);
+        debug (82,1)("undogzip: initialising compression library: rv is %d\n",rv);
+        if (rv != Z_OK) {
+            /* error initing compressiong */
+            debug(82,1)("undogzip: compression library returned: %s\n",zdata->msg);
+            return 0;
+        }
+
+    }
+    zdata=*d;
+    assert (!((ibufl==0) && ibuf != NULL));
+    zdata->next_in=ibuf;
+    zdata->avail_in=ibufl;
+    debug(82,1)("available buffer = %d ibufl  = %d\n", DEFLATEBLOCKSIZE,ibufl);
+    *obuf = xmalloc( DEFLATEBLOCKSIZE);
+    if (ibufl >0) {
+        debug(82,1)("undogzip uncompressing %d bytes\n",ibufl);
+        do {
+            zdata->next_out=*obuf+output;
+            zdata->avail_out=DEFLATEBLOCKSIZE;
+            rv=inflate(zdata,Z_NO_FLUSH);
+            output+= DEFLATEBLOCKSIZE-zdata->avail_out;
+            if (zdata->avail_in) {
+                /* ran out of space */
+                debug(82,1)("undogzip allocating another %d bytes - buffer now %d - rem
+aining bytes to be process %d\n",DEFLATEBLOCKSIZE,output+DEFLATEBLOCKSIZE,zdata->avail_in)
+;
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (!((rv == Z_OK) || (rv == Z_STREAM_END))) {
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                inflateEnd(zdata);
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (zdata->avail_in);
+        if (rv== Z_STREAM_END) {
+            inflateEnd(zdata);
+            xfree(zdata);
+            *d=NULL;
+        }
+    }
+    else {
+        /* errors in decoding are trapped above. Successful decoding frees t*d,
+         * so the !*d && !ibufl test runs above. */
+        fatal("undogzip: inconsistent decompression state reached\n");
+#if 0
+        debug(82,1)("undodeflate flushing buffer at EOF\n");
+        /* flush the remaining data */
+        do {
+            zdata->avail_out= DEFLATEBLOCKSIZE;
+            zdata->next_out=*obuf+output;
+            rv=deflate (zdata, Z_FINISH);
+            output+= DEFLATEBLOCKSIZE-zdata->avail_out;
+            if (rv==Z_OK && (zdata->avail_out == 0)) {
+                /* ran out of space */
+                debug(82,1)("undodeflate allocating another %d bytes\n",DEFLATEBLOCKSIZE);
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (rv != Z_STREAM_END) {
+                /* wasn't out of space, and not all finished */
+                debug(82,1)("undodeflate: error flushing data\n");
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                deflateEnd(zdata);
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (rv != Z_STREAM_END);
+        deflateEnd(zdata);
+        xfree(zdata);
+        *d=NULL;
+        /* return TE_BUFFER_ALLOCATED; */ /* & TE_BUFFERED_OUTPUT */
+#endif
+    }
+    if (zdata)
+        debug(82,1)("avail in %d, out %d\n",zdata->avail_in, zdata->avail_out);
+    debug(82,1)("created block of %d bytes\n",output);
+    *obufl=output;
+    return TE_BUFFER_ALLOCATED;  /* & TE_BUFFERED_OUTPUT */
+#endif
+}
+
 #endif /*ZLIB_COMPRESSION */
 #endif /*!ATHY_COMPRESSION*/
 
+
+#ifdef HAVE_H_HSYNC
+/* this is a TE filter */
+static int dorsyncdelta (char *ibuf, size_t ibufl, char **obuf, size_t *obufl, void **d)
+{
+/* 4K. TODO: lookup membuf 4k thingy code */
+#define DEFLATEBLOCKSIZE 4096
+    z_streamp zdata;
+    size_t output=0;
+    int rv;
+    /* if we run out of output space, we have to store the already output data */
+    char *tempbuf=NULL;
+
+    if (!ibufl && !*d) {
+        /* we haven't been inited and there's nothing to do */
+        *obufl=0;
+        *obuf=NULL;
+        return 1;
+    }
+    if (!*d) {
+        int rv;
+        /* first call - no data in progress */
+        zdata=xmalloc(sizeof(z_stream));
+        *d=zdata;
+        assert(zdata!= NULL);
+        zdata->zalloc=compcalloc;
+        zdata->zfree=compfree;
+        zdata->opaque=NULL;
+
+        rv=deflateInit(zdata, Z_DEFAULT_COMPRESSION);
+        debug (82,1)("dodeflate: initialising compression library: rv is %d\n",rv);
+        if (rv != Z_OK) {
+            /* error initing compressiong */
+            debug(82,1)("dodeflate: compression library returned: %s\n",zdata->msg);
+            return 0;
+        }
+
+    }
+    zdata=*d;
+    assert (!((ibufl==0) && ibuf != NULL));
+    zdata->next_in=ibuf;
+    zdata->avail_in=ibufl;
+    debug(82,1)("available buffer = %d ibufl  = %d\n", DEFLATEBLOCKSIZE,ibufl);
+    *obuf = xmalloc( DEFLATEBLOCKSIZE);
+    if (ibufl >0) {
+        debug(82,1)("dodeflate compressing %d bytes\n",ibufl);
+        do {
+            zdata->next_out=*obuf+output;
+            zdata->avail_out=DEFLATEBLOCKSIZE;
+            rv=deflate(zdata,Z_NO_FLUSH);
+            output+= DEFLATEBLOCKSIZE-zdata->avail_out;
+            if (zdata->avail_in) {
+                /* ran out of space */
+                debug(82,1)("dodeflate allocating another %d bytes\n",DEFLATEBLOCKSIZE);
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (rv != Z_OK) {
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                deflateEnd(zdata);
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (zdata->avail_in);
+    }
+    else {
+        debug(82,1)("dodeflate flushing buffer at EOF\n");
+        /* flush the remaining data */
+        do {
+            zdata->avail_out= DEFLATEBLOCKSIZE;
+            zdata->next_out=*obuf+output;
+            rv=deflate (zdata, Z_FINISH);
+            output+= DEFLATEBLOCKSIZE-zdata->avail_out;
+            if (rv==Z_OK && (zdata->avail_out == 0)) {
+                /* ran out of space */
+                debug(82,1)("dodeflate allocating another %d bytes\n",DEFLATEBLOCKSIZE);
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (rv != Z_STREAM_END) {
+                /* wasn't out of space, and not all finished */
+                debug(82,1)("dodeflate: error flushing data\n");
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                deflateEnd(zdata);
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (rv != Z_STREAM_END);
+        deflateEnd(zdata);
+        xfree(zdata);
+        *d=NULL;
+        /* return TE_BUFFER_ALLOCATED; */ /* & TE_BUFFERED_OUTPUT */
+
+    }
+    debug(82,1)("avail in %d, out %d\n",zdata->avail_in, zdata->avail_out);
+    debug(82,1)("created block of %d bytes\n",output);
+    *obufl=output;
+    return TE_BUFFER_ALLOCATED;  /* & TE_BUFFERED_OUTPUT */
+}
+
+/* this is a TE filter */
+static int applyrsyncdelta(char *ibuf, size_t ibufl, char **obuf, size_t *obufl, void **d)
+{
+/* 4K. TODO: lookup membuf 4k thingy code */
+#define DEFLATEBLOCKSIZE 4096
+    z_streamp zdata;
+    size_t output=0;
+    int rv;
+    /* if we run out of output space, we have to store the already output data */
+    char *tempbuf=NULL;
+
+    if (!ibufl && !*d) {
+        /* we haven't been inited and there's nothing to do */
+        *obufl=0;
+        *obuf=NULL;
+        return 0;
+    }
+    if (!*d) {
+        int rv;
+        /* first call - no data in progress */
+        zdata=xmalloc(sizeof(z_stream));
+        *d=zdata;
+        assert(zdata!= NULL);
+        zdata->zalloc=compcalloc;
+        zdata->zfree=compfree;
+        zdata->opaque=NULL;
+        zdata->next_in=ibuf;
+        /* defer header reading */
+        zdata->avail_in=0;
+
+        rv=inflateInit(zdata);
+        debug (82,1)("undodeflate: initialising compression library: rv is %d\n",rv);
+        if (rv != Z_OK) {
+            /* error initing compressiong */
+            debug(82,1)("undodeflate: compression library returned: %s\n",zdata->msg);
+            return 0;
+        }
+
+    }
+    zdata=*d;
+    assert (!((ibufl==0) && ibuf != NULL));
+    zdata->next_in=ibuf;
+    zdata->avail_in=ibufl;
+    debug(82,1)("available buffer = %d ibufl  = %d\n", DEFLATEBLOCKSIZE,ibufl);
+    *obuf = xmalloc( DEFLATEBLOCKSIZE);
+    if (ibufl >0) {
+        debug(82,1)("undodeflate uncompressing %d bytes\n",ibufl);
+        do {
+            zdata->next_out=*obuf+output;
+            zdata->avail_out=DEFLATEBLOCKSIZE;
+            rv=inflate(zdata,Z_NO_FLUSH);
+            output+= DEFLATEBLOCKSIZE-zdata->avail_out;
+            if (zdata->avail_in) {
+                /* ran out of space */
+                debug(82,1)("undodeflate allocating another %d bytes - buffer now %d\n",DEFLATEBLOCKSIZE,output+DEFLATEBLOCKSIZE);
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (!((rv == Z_OK) || (rv == Z_STREAM_END))) {
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                inflateEnd(zdata);
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (zdata->avail_in);
+        if (rv== Z_STREAM_END) {
+            inflateEnd(zdata);
+            xfree(zdata);
+            *d=NULL;
+        }
+    }
+    else {
+        /* errors in decoding are trapped above. Successful decoding frees t*d,
+         * so the !*d && !ibufl test runs above. */
+        fatal("undodeflate: inconsistent decompression state reached\n");
+#if 0
+        debug(82,1)("undodeflate flushing buffer at EOF\n");
+        /* flush the remaining data */
+        do {
+            zdata->avail_out= DEFLATEBLOCKSIZE;
+            zdata->next_out=*obuf+output;
+            rv=deflate (zdata, Z_FINISH);
+            output+= DEFLATEBLOCKSIZE-zdata->avail_out;
+            if (rv==Z_OK && (zdata->avail_out == 0)) {
+                /* ran out of space */
+                debug(82,1)("undodeflate allocating another %d bytes\n",DEFLATEBLOCKSIZE);
+                tempbuf=*obuf;
+                *obuf=xmalloc(output+DEFLATEBLOCKSIZE);
+                memcpy(*obuf,tempbuf,output);
+                xfree(tempbuf);
+            } else if (rv != Z_STREAM_END) {
+                /* wasn't out of space, and not all finished */
+                debug(82,1)("undodeflate: error flushing data\n");
+                xfree(*obuf);
+                *obuf=0;
+                *obufl=0;
+                deflateEnd(zdata);
+                xfree(zdata);
+                *d=NULL;
+                return 0;
+            }
+        } while (rv != Z_STREAM_END);
+        deflateEnd(zdata);
+        xfree(zdata);
+        *d=NULL;
+        /* return TE_BUFFER_ALLOCATED; */ /* & TE_BUFFERED_OUTPUT */
+#endif
+    }
+    if (zdata)
+        debug(82,1)("avail in %d, out %d\n",zdata->avail_in, zdata->avail_out);
+    debug(82,1)("created block of %d bytes\n",output);
+    *obufl=output;
+    return TE_BUFFER_ALLOCATED;  /* & TE_BUFFERED_OUTPUT */
+}
+
+static int dorsyncsignature()
+{
+}
+
+#endif
+
 static te_coding_t *codecs=NULL;
 
-void teAddCoding(char *token, TEFILTER *coder, TEFILTER *decoder) {
+void teAddCoding(char *token, void *makerequest, TEFILTER *coder, TEFILTER *decoder) {
     te_coding_t * codec;
     codec=codecs;
     while (codec){
@@ -897,11 +1416,15 @@
     int y;
 
     if (!codecs) {
-        teAddCoding("chunked", dochunk, undochunked);
-        teAddCoding("biff", biff, unbiff);
-        teAddCoding("jones", jones, unjones);
+        teAddCoding("chunked", NULL, dochunk, undochunked);
+        teAddCoding("biff", NULL, biff, unbiff);
+        teAddCoding("jones", NULL, jones, unjones);
 #if ZLIB_COMPRESSION || ATHY_COMPRESSION
-	teAddCoding("deflate", dodeflate, undodeflate);
+	teAddCoding("deflate", NULL, dodeflate, undodeflate);
+	teAddCoding("gzip", NULL, dogzip, undogzip);
+#endif
+#ifdef HAVE_H_HSYNC
+        teAddCoding("rsync", dorsyncsignature, dorsyncdelta, applyrsyncdelta);
 #endif
     }
 
@@ -962,6 +1485,10 @@
 		/* need to remove the TE.. and rewrite the header */
 		debug(33, 7) ("removing deflate\n");
 		new_xlat(translations, 0, "deflate", 1, NULL);
+	    } else if (!strncasecmp(item, "gzip", y)) {
+		/* need to remove the TE.. and rewrite the header */
+		debug(33, 7) ("removing gzip\n");
+		new_xlat(translations, 0, "gzip", 1, NULL);
 #endif
 	    } else {
 		debug(1,1)("unknown transfer encoding in response\n");
@@ -1019,11 +1546,15 @@
 */
 
     if (!codecs) {
-	teAddCoding("chunked", dochunk, undochunked);
-	teAddCoding("biff", biff, unbiff);
-	teAddCoding("jones", jones, unjones);
+        teAddCoding("chunked", NULL, dochunk, undochunked);
+        teAddCoding("biff", NULL, biff, unbiff);
+        teAddCoding("jones", NULL, jones, unjones);
 #if ZLIB_COMPRESSION || ATHY_COMPRESSION
-	teAddCoding("deflate", dodeflate, undodeflate);
+        teAddCoding("deflate", NULL, dodeflate, undodeflate);
+	teAddCoding("gzip", NULL, dogzip, undogzip);
+#endif
+#ifdef HAVE_H_HSYNC
+        teAddCoding("rsync", dorsyncsignature, dorsyncdelta, applyrsyncdelta);
 #endif
     }
 
@@ -1088,6 +1619,7 @@
 #endif
 
 
+
     if ((httpHeaderHas(ohdr, HDR_TE)
         || ((http_ver.major == 1) && (http_ver.minor >=1)))) {
         char vlb[128];
@@ -1105,7 +1637,6 @@
 		/* strListIsMember_q returns the value of param q.. but it's broken */
 		/* what is used elsewhere in squid to parse headers like type;param=value;param=value.. */
             if (strListIsMember_q(&s_te, "biff", ',')) {
-                debug(33, 7) ("will add biff\n");
                 new_xlat(translations, 1, "biff", 0, NULL);
 
                 if (added_te)
@@ -1115,7 +1646,6 @@
                 strcat(vlb, "biff");
             } else
             if (strListIsMember_q(&s_te, "jones", ',')) {
-                debug(33, 7) ("will add jones\n");
                 new_xlat(translations, 1, "jones", 0, NULL);
 
                 if (added_te)
@@ -1125,12 +1655,23 @@
                 strcat(vlb, "jones");
             }
 
+#ifdef HAVE_H_HSYNC
+	    if ((strListIsMember_q(&s_te, "rsync", ','))){
+		new_xlat(translations, 1, "rsync", 0, NULL);
+		if (added_te)
+		    strcat(vlb, ", ");
+		else
+		    added_te = 1;
+		strcat(vlb, "rsync");
+	    }
+#endif
+
+#if ATHY_COMPRESSION || ZLIB_COMPRESSION
             if ((strListIsMember_q(&s_te, "deflate", ','))){
 #if 0
    /* this needs to be an acl test against content-encoding */
                 && (!(may_apply_tes & 0x02))) { /* the ce: compressed */
 #endif
-                debug(33, 7) ("will add deflate\n");
                 new_xlat(translations, 1, "deflate", 0, NULL);
 
                 if (added_te)
@@ -1138,10 +1679,17 @@
                 else
                     added_te = 1;
                 strcat(vlb, "deflate");
-            }
+            } else if ((strListIsMember_q(&s_te, "gzip", ','))){
+		new_xlat(translations, 1, "gzip", 0, NULL);
+		if (added_te)
+		    strcat(vlb, ", ");
+		else
+		    added_te = 1;
+		strcat(vlb, "gzip");
+	    }
+#endif
 
             /* always add chunked to cover nonterminating encodings */
-            debug(82, 7) ("will add chunked\n");
             new_xlat(translations, 1, "chunked", 0, NULL);
 
 	    if (added_te)
@@ -1161,3 +1709,26 @@
     }
     /* add transfer-encoding to the connection header */
 }
+
+void
+te_generate_request(HttpHeader *hdr_out)
+{
+    /* append TE */
+      char strTE[128];
+
+      httpHeaderDelById(hdr_out, HDR_TE); /* hop by hop.. that's what last
+                                       hop could do*/
+
+#if 0 && defined(HAVE_H_HSYNC)
+	strcpy (strTE,"chunked;q=1.0, rsync;q=1.0");
+#endif
+#if ATHY_COMPRESSION || ZLIB_COMPRESSION
+      strcpy (strTE,"biff;q=0.0, gzip;q=1.0, deflate;q=1.0, jones;q=0.0, chunked;q=1.0");
+#else
+      strcpy (strTE,"biff;q=1.0, gzip;q=0.0, deflate;q=0.0, jones;q=1.0, chunked;q=1.0");
+#endif
+
+      httpHeaderPutStr (hdr_out,HDR_TE,strTE);
+      httpHeaderPutStr(hdr_out,HDR_CONNECTION, "TE"); /*  its hop by hop */
+
+}
