This patch is generated from the gzip branch of HEAD in squid3 Wed Feb 6 01:22:15 2008 GMT See http://devel.squid-cache.org/ Index: squid3/configure.in diff -u squid3/configure.in:1.175 squid3/configure.in:1.112.2.7 --- squid3/configure.in:1.175 Wed Jan 23 13:51:00 2008 +++ squid3/configure.in Mon Feb 4 21:36:55 2008 @@ -681,6 +681,18 @@ fi ]) +AM_CONDITIONAL(ENABLE_CONTENT_ENCODING, false) +AC_ARG_ENABLE(content-encoding, +[ --enable-content-encoding Enable content-encoding to compress content], +[ if test "$enableval" = "yes" ; then + echo "Content encoding enabled" + AC_CHECK_LIB(z, deflate) + XTRA_LIBS="$XTRA_LIBS -lz" + AC_DEFINE([CONTENT_ENCODING],1,[content encoding support.]) + AM_CONDITIONAL(ENABLE_CONTENT_ENCODING, true,) + fi +]) + AM_CONDITIONAL(USE_ESI, false) AC_ARG_ENABLE(esi, AC_HELP_STRING([--enable-esi],[Enable ESI for accelerators. Requires libexpat. Enabling ESI will cause squid to follow the Edge Acceleration Specification (www.esi.org). This causes squid to IGNORE client Cache-Control headers. DO NOT use this in a squid configured as a web proxy, ONLY use it in a squid configured for webserver acceleration.]), Index: squid3/include/profiling.h diff -u squid3/include/profiling.h:1.24 squid3/include/profiling.h:1.12.2.5 --- squid3/include/profiling.h:1.24 Thu Jan 24 12:51:00 2008 +++ squid3/include/profiling.h Mon Feb 4 21:36:58 2008 @@ -136,6 +136,9 @@ XPROF_MemObject_write, XPROF_storeWriteComplete, XPROF_mem_hdr_write, +#ifdef CONTENT_ENCODING + XPROF_zlib, +#endif XPROF_headersEnd, XPROF_parseHttpRequest, XPROF_HttpStateData_processReplyHeader, Index: squid3/src/HttpContentCoder.cc diff -u /dev/null squid3/src/HttpContentCoder.cc:1.1.2.2 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/HttpContentCoder.cc Fri Sep 1 16:44:54 2006 @@ -0,0 +1,177 @@ +/* + * $Id$ + * + * DEBUG: section 93 HTTP Content-Encoding + * AUTHOR: Gonzalo Arana + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#include "ACLChecklist.h" +#include "HttpContentCoder.h" +#include "HttpReply.h" +#include "client_side_request.h" +#include "typedefs.h" + +const char* coderStat_ntoa(int status) { + switch (status) { + case CC_ERROR: return "CC_ERROR"; + case CC_OK: return "CC_OK"; + case CC_PENDING: return "CC_PENDING"; + case CC_DONE: return "CC_DONE"; + default: return "INVALID_VALUE"; + } + return "INTERNAL_ERROR"; +} + +CBDATA_CLASS_INIT(HttpContentCoderAcl); + +HttpContentCoder::HttpContentCoder() { +} + +HttpContentCoder::~HttpContentCoder() { +} + +void* +HttpContentCoderAcl::operator new (size_t size) +{ + CBDATA_INIT_TYPE(HttpContentCoderAcl); + return cbdataAlloc(HttpContentCoderAcl); +} + +void +HttpContentCoderAcl::operator delete (void* address) +{ + cbdataFree(address); +} + +HttpContentCoderAcl::HttpContentCoderAcl(acl_access *acl) +{ + encodeAcl = acl; + negotiate_ce_access = encode_access = NULL; + access_answer = -1; // it han't been checked + http = NULL; + reply = NULL; +} + +HttpContentCoderAcl::~HttpContentCoderAcl() { + //FIXME: is this OK? + delete negotiate_ce_access; + delete encode_access; +} + +/* + * Used internally aclChecklist creation. + */ +static ACLChecklist * +codeAclChecklistCreate(acl_access* acl, ClientHttpRequest* http, HttpReply* reply) { + ConnStateData::Pointer conn = http->getConn(); + //FIXME: is this OK? + ACLChecklist* dev = + aclChecklistCreate(acl, http->request, + conn.getRaw() != NULL ? conn->rfc931 : dash_str); + dev->reply = HTTPMSGLOCK(reply); + + /* Check client_side.cc/clientACLChecklistCreate comments */ + if (conn.getRaw() != NULL) dev->conn(conn); + + return dev; +} + +/* + * Acl check for encoding. + * FIXME: the only way to ensure that negotiate_ce_access will not be + * evaluated more than once, is to place negotiate_ce_access in http*, + * and use the last returned value. + */ +void +codeAclCheckDone(int answer, void* data) { + debug(93, 4)("codeAclCheckDone(%d, %p)\n", answer, data); + HttpContentCoderAcl* coder = (HttpContentCoderAcl*)data; + coder->access_answer = answer; + //This is called from AclChecklist::checkCallback, which does a delete this. + coder->encode_access = NULL; +} + +void +negotiateCodeAclCheckDone(int answer, void* data) { + debug(93, 4)("negotiateCodeAclCheckDone(%d, %p)\n", answer, data); + HttpContentCoderAcl* coder = (HttpContentCoderAcl*)data; + // idem codeAclCheckDone + coder->negotiate_ce_access = NULL; + if (answer != ACCESS_ALLOWED) { + codeAclCheckDone(answer, coder); + } else { + coder->codeAclCheck1(); + } +} + +void +HttpContentCoderAcl::codeAclCheck(ClientHttpRequest* http, HttpReply* reply) { + debug(93, 4)("HttpContentCoderAcl::codeAclCheck(%p, %p)\n", http, reply); + if (negotiate_ce_access) { /* to avoid repeated unnecesary checks */ + codeAclCheckDone(access_answer, this); //FIXME: is this ok? + return; + } + + this->http = http; + this->reply = reply; + debug(93, 9)("HttpContentCoderAcl::codeAclCheck: " + "creating checklist for negotiate_ce_access\n"); + negotiate_ce_access = + codeAclChecklistCreate(Config.accessList.negotiate_ce, http, reply); + + debug(93, 9)("HttpContentCoderAcl::codeAclCheck: " + "calling (negotiate_ce_access:%p)->nonBlockingCheck\n", + negotiate_ce_access); + negotiate_ce_access->nonBlockingCheck(negotiateCodeAclCheckDone, this); +} + +void +HttpContentCoderAcl::codeAclCheck1() { + debug(93, 4)("HttpContentCoderAcl::codeAclCheck1()\n"); + if (!encodeAcl) { + codeAclCheckDone(ACCESS_ALLOWED, this); + } else { + encode_access = codeAclChecklistCreate(encodeAcl, http, reply); + encode_access->nonBlockingCheck(codeAclCheckDone, this); + } +} + + +bool +HttpContentCoderAcl::canCode() { + if (access_answer == ACCESS_ALLOWED) + return true; + + debug(93, 2) ("HttpContentCoderAcl::canCode: access_answer = %d\n", + access_answer); + return false; +} Index: squid3/src/HttpContentCoder.h diff -u /dev/null squid3/src/HttpContentCoder.h:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/HttpContentCoder.h Thu Aug 31 16:11:31 2006 @@ -0,0 +1,117 @@ +/* + * $Id$ + * + * DEBUG: section 93 HTTP Content-Encoding + * AUTHOR: Jon Kay + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#ifndef __HTTPCONTENTCODER_H_ +#define __HTTPCONTENTCODER_H_ + +#include "squid.h" +#include "client_side.h" +/* Needed for ClientHttpRequest */ +#include "ACL.h" +/* acl_access */ + +#define CC_OK 0 +#define CC_PENDING 1 +#define CC_DONE 2 +#define CC_ERROR -1 + +extern const char* coderStat_ntoa(int status); + +class HttpHdrContCode; + +/* + * All methods return CC_OK or CC_ERR, except: + * decodeChunk who may return CC_DONE as well + * (de|en)codeEnd who may return CC_PENDING + * + * ibuff: input buffer (unmodiffied) + * obuff: output buffer (these methods may write up to *obufflen bytes). + * *ibufflen: + * input: number of bytes available in ibuff. + * output: number of bytes processed by coder. + * *obufflen: + * input: number of bytes available in output buffer, + * output: number of bytes written to obuff by coder. + */ +class HttpContentCoder { + + public: + HttpContentCoder(); + virtual ~HttpContentCoder(); + virtual int encodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen) = 0; + virtual int encodeEnd(char* obuff, int* obufflen) = 0; + virtual int encodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen) = 0; + virtual int decodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen) = 0; + virtual int decodeEnd(char* obuff, int* obufflen) = 0; + virtual int decodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen) = 0; + virtual int totalIn() const = 0; + virtual int totalOut() const = 0; + + private: + void codeAclCheck1(); + friend void codeAclCheckDone(int answer, void* data); + friend void negotiateCodeAclCheckDone(int answer, void* data); +}; + +class HttpContentCoderAcl { + + private: + CBDATA_CLASS(HttpContentCoderAcl); + ACLChecklist *negotiate_ce_access; + ACLChecklist *encode_access; + acl_access *encodeAcl; + int access_answer; + HttpHdrContCode *code; + // if (!getEncodeACL()) means allow all :( + ClientHttpRequest* http; + HttpReply* reply; + public: + void* operator new (size_t size); + void operator delete(void* address); + HttpContentCoderAcl(acl_access *acl); + ~HttpContentCoderAcl(); + bool canCode(); + void codeAclCheck(ClientHttpRequest* http, HttpReply* reply); + + private: + void codeAclCheck1(); + friend void codeAclCheckDone(int answer, void* data); + friend void negotiateCodeAclCheckDone(int answer, void* data); + public: + allow_t AccessAnswer() const { return allow_t(access_answer); } +}; + +#endif Index: squid3/src/HttpHdrContCode.cc diff -u /dev/null squid3/src/HttpHdrContCode.cc:1.1.2.2 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/HttpHdrContCode.cc Fri Sep 1 16:44:54 2006 @@ -0,0 +1,838 @@ + +/* + * $Id$ + * + * DEBUG: section 93 HTTP Content-Encoding + * AUTHOR: Jon Kay + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "Store.h" +#include "RefCount.h" +#include "clientStream.h" +#include "MemBuf.h" +#include "http.h" +#include "HttpRequest.h" +#include "client_side_request.h" +#include "client_side_reply.h" +#include "HttpReply.h" +#include "HttpContentCoder.h" +#include "HttpHdrContCode.h" +#include "deflateContentCoder.h" +#include "gzipContentCoder.h" +#include "noneContentCoder.h" + +/* Twice the size of the 4k input buffer, to allow room for encoding growth */ +#define ENCODE_CHUNK_SIZE 8192 + +static void codeStreamRead(clientStreamNode *, + ClientHttpRequest *); +static clientStream_status_t codeStreamStatus(clientStreamNode *, + ClientHttpRequest *); +static void codeStreamCallback(clientStreamNode *, + ClientHttpRequest *, + HttpReply *, StoreIOBuffer); +static void codeStreamDetach(clientStreamNode *, + ClientHttpRequest *); +static void codeStreamOvertime(clientStreamNode *, + HttpHdrContCoder *, + ClientHttpRequest *, + HttpReply *reply, + int codestat); + +static bool codeListIn(char *list, int len, http_code_type type); +static bool codeListContained(char *list1, int len1, char* list2, int len2); + +static const char *codeName(http_code_type code); + +static int codeParseField(String *field, char codelist[]); + + +void +HttpHdrContCoder::chooseCoder(const http_code_type code) +{ + switch(code) { + case CODE_GZIP: + coder = new gzipContentCoder(); + break; + case CODE_DEFLATE: + coder = new deflateContentCoder(); + break; + case CODE_NONE: + coder = new noneContentCoder(); + break; + default: + coder = new noneContentCoder(); + debug(93, 2) + ("HttpHdrContCode::chooseCoder: unknown encoding %d, " + "using identity encoding\n", code); + break; + } +} + + +HttpHdrContCoder::HttpHdrContCoder(dlink_list *stream, ClientHttpRequest *http) +{ + coder = new noneContentCoder(); /* Default to identity coder */ + offset = 0; + uncoded = 0; + ignore = false; + doneended = false; + codedBuf = NULL; + contentLen = -1; + codedEntry = NULL; + + codeClientStreamInit(stream); +} + +HttpHdrContCoder::~HttpHdrContCoder() +{ + if (coder) delete coder; + +} + +void +HttpHdrContCoder::initEntry(ClientHttpRequest *http, HttpReply *reply, + char *codeList, int ncoders) +{ + StoreEntry *entry = http->storeEntry(); + + if (!http->request->flags.nocache && entry && !EBIT_TEST(entry->flags, KEY_PRIVATE)) { + request_flags rflags; + rflags.cachable = 1; + codedEntry = storeCreateEntry(http->uri, http->uri, rflags, METHOD_GET); + EBIT_SET(codedEntry->flags, ENTRY_CODED); + + /* XXX - use Etag functions when real Etag support arrives */ + char ztag[80]; + const cache_key *key; + key = storeKeyPublicByEncoding(http->request, METHOD_GET, + codeList, ncoders); + + snprintf(ztag, sizeof(ztag), "\"sq3tag%s/%d-%x%x\"", getMyHostname(), + Config.Sockaddr.http->s.sin_port, + *(unsigned int *) key, + *(unsigned int *) (key + 4)); + debug (93, 4) ("HttpHdrContCoder::initEntry: using etag %s\n", ztag); + + reply->header.delById(HDR_ETAG); + reply->header.putStr(HDR_ETAG, ztag); + } +} + +void +HttpHdrContCoder::codingKnown(http_code_type code, bool codebit, bool ignbit) +{ + delete coder; + chooseCoder(code); + amencoding = codebit; + ignore = ignbit; + +} + + +/* clientStream stuff */ +bool +HttpHdrContCoder::codeClientStreamInit(dlink_list *stream) +{ + debug(93, 5) ("HttpHdrContCoder::codeClientStreamInit: Yo!\n"); + clientStreamInsertHead(stream, codeStreamRead, codeStreamCallback, + codeStreamDetach, codeStreamStatus, + this); + return true; +} + +/* Encodes from sbuf to myNode->readBuffer */ +static void +codeStreamCallback(clientStreamNode *myNode, + ClientHttpRequest *http, + HttpReply *reply, + StoreIOBuffer sbuf) +{ + HttpHdrContCoder *hcoder = + dynamic_cast(myNode->data.getRaw()); + + /* Remember and use content length if we ever see it; sometimes we + only see it on the first packet */ + int clen = reply != NULL ? reply->content_length : -1; + if (clen != -1) + hcoder->contentLen = clen; + clen = hcoder->contentLen; + + int hlen = reply != NULL ? reply->header.len : -1; + debug (93, 2) ("HttpHdrContCoder::codeClientStreamCallback: %s\n", + http->uri); + debug (93, 2) ("\t%d at %d / %d, hlen = %d (reply = %p)\n", + (int) sbuf.length, (int) sbuf.offset, clen, hlen, + reply); + int codestat = CC_OK; + debug (93, 5) ("\tsbuf.offset=%lu\n", sbuf.offset); + debug (93, 5) ("\tsbuf.length=%lu\n", sbuf.length); + debug (93, 5) ("\tmyNode->readBuffer.offset=%lu\n", + myNode->readBuffer.offset); + debug (93, 5) ("\tmyNode->readBuffer.length=%lu\n", + myNode->readBuffer.length); + /* compiler bug bait - dumps core on gcc 3.3.3 (egcs) in + client_side.cc:clientSocketRecipient if the next statement isn't there (!) */ + ClientSocketContext::Pointer context = + dynamic_cast(myNode->next()->data.getRaw()); + HttpContentCoder *coder = hcoder->coder; + StoreIOBuffer *iobuf = &myNode->readBuffer; + + /* Allocate our buffer, to put coded data to.*/ + if (hcoder->codedBuf == NULL) { + hcoder->codedBuf = (char *) xmalloc(ENCODE_CHUNK_SIZE); + } + /* Always need to reset buffer here, because codeStreamRead call + must always overwrite it with read-to buffer pointer */ + iobuf->data = hcoder->codedBuf; + + /* always need to reset this to 8k, since was changed in previous call. */ + iobuf->length = ENCODE_CHUNK_SIZE; /* 2x input buflen */ + + if (hcoder->ignore || hcoder->doneended) { + debug (93, 2) + ("HttpHdrContCoder::codeClientStreamCallback: ignoring " + "(ign=%d, doneended=%d)\n", int(hcoder->ignore), + int(hcoder->doneended)); + //iobuf->data = sbuf.data; + xmemcpy(iobuf->data, sbuf.data, sbuf.length); + iobuf->length = sbuf.length; + hcoder->uncoded += iobuf->length; + iobuf->offset = hcoder->uncoded; + clientStreamCallback(myNode, http, reply, sbuf); + return; + } + + /* Offset should be nondecreasing */ + assert(sbuf.offset >= hcoder->uncoded); + + char *mybuf = iobuf->data; + char *frombuf = sbuf.data; + int ilen = sbuf.length; + int olen = iobuf->length; + int processed = 0; // number of input bytes processed + int encoded = 0; // number of output bytes encoded + + int at_start = 0; + + debug (93, 5) + ("HttpHdrContCoder::codeStreamCallback: have to encode %d bytes " + "into (at most) %d bytes\n", ilen, olen); + + /* The start */ + if (sbuf.offset == 0) { + int _olen = olen; + int _ilen = ilen; + at_start = 1; + if (hcoder->amencoding) + codestat = coder->encodeStart(frombuf, &ilen, mybuf, &olen); + else + codestat = coder->decodeStart(frombuf, &ilen, mybuf, &olen); + debug (93, 5) + ("HttpHdrContCoder::codeStreamCallback: encodeStart: %d -> " + "%d bytes (codestat=%s)\n", ilen, olen, coderStat_ntoa(codestat)); + processed = ilen; + frombuf += ilen; + encoded = olen; + mybuf += olen; + olen = _olen - olen; /* setup this variables for encodeEnd */ + ilen = _ilen - ilen; + } + + /* Not at start && have data to process*/ + if (!processed && ilen && codestat == CC_OK) { + /* Not at the end => somewhere in the middle */ + int _olen = olen; + int _ilen = ilen; + if (hcoder->amencoding) + codestat = coder->encodeChunk(frombuf, &ilen, mybuf, &olen); + else + codestat = coder->decodeChunk(frombuf, &ilen, mybuf, &olen); + + debug(93, 5) + ("HttpHdrContCoder::codeStreamCallback: encodeChunk: %d -> %d " + "bytes (codestat=%s)\n", ilen, olen, coderStat_ntoa(codestat)); + processed += ilen; + frombuf += ilen; + encoded += olen; + mybuf += olen; + olen = _olen - olen; + ilen = _ilen - ilen; + } + + /* sbuf.length == 0 => end of input stream */ + /* decodeChunk may return CC_DONE */ + if (olen && (codestat == CC_OK || codestat == CC_DONE) && + (sbuf.length == 0 || + (clen != -1 && hcoder->uncoded + (int) sbuf.length >= clen))) { + /* No more input data to process */ + /* FIXME:garana: Handle output buffer unerrun */ + if (hcoder->amencoding) + codestat = coder->encodeEnd(mybuf, &olen); + else + codestat = coder->decodeEnd(mybuf, &olen); + debug(93, 2) + ("HttpHdrContCoder::codeStreamCallback: encodeEnd: ilen = %d, " + "olen = %d, (codestat=%s)\n", ilen, olen, + coderStat_ntoa(codestat)); + hcoder->doneended = codestat != CC_PENDING ? true : false; + encoded += olen; + } + + iobuf->length = encoded; + iobuf->offset = hcoder->offset; + + hcoder->offset += encoded; + hcoder->uncoded += processed; + + debug(93, 2) + ("HttpHdrContCoder::codeStreamCallback: offset=%d, uncoded=%lu, " + "contentlen=%d, processed=%d encoded=%d (codestat=%s)\n", + (int) iobuf->offset, hcoder->uncoded, clen, processed, encoded, + coderStat_ntoa(codestat)); + + if (codestat == CC_PENDING) + codeStreamOvertime(myNode, hcoder, http, reply, codestat); + + /* Write encoded data to storeEntry, if we have one */ + if (hcoder->codedEntry != NULL) { + debug(93,3) ("HttpHdrContCode:codeStreamCallback:writing codedEntry: off=%d,len=%d\n", + (int)iobuf->offset, (int)iobuf->length); + hcoder->codedEntry->write(*iobuf); + } + + /* pass control to rest of streamRead */ + clientStreamCallback(myNode, http, reply, *iobuf); +} + +static void +codeStreamRead(clientStreamNode *myNode, ClientHttpRequest *http) +{ + debug (93, 4) ("HttpHdrContCoder::codeStreamRead: next = %p, prev = %p\n", + myNode->next(), myNode->prev()); + /* compiler bug bait - dumps core on gcc 3.3.3 (egcs) in + client_side.cc:clientSocketRecipient if the next statement isn't there (!) */ + ClientSocketContext::Pointer context = + dynamic_cast(myNode->next()->data.getRaw()); + HttpHdrContCoder *hcoder = + dynamic_cast(myNode->data.getRaw()); + clientStreamNode *next = myNode->next(); + + myNode->readBuffer = next->readBuffer; + if (myNode->readBuffer.offset > 0 && !hcoder->ignore) + /* Need to set offset to uncoded offset */ + myNode->readBuffer.offset = hcoder->uncoded; + + debug (93, 4) ("\treadbuffer(offset=%d, length=%d, data=%p)\n", + (int)myNode->readBuffer.offset, (int) myNode->readBuffer.length, + myNode->readBuffer.data); + clientStreamRead(myNode, http, myNode->readBuffer); +} + +static void +codeStreamDetach(clientStreamNode *myNode, + ClientHttpRequest *http) +{ + debug(93, 4) ("HttpHdrContCoder::codeClientStreamDetach: Yo!\n"); + HttpHdrContCoder *hcoder = + dynamic_cast(myNode->data.getRaw()); + + /* Not always true even when working properly, but sometimes + useful for debugging */ + /* assert(hcoder->doneended || hcoder->ignore); */ + + if (hcoder->codedBuf) + xfree(hcoder->codedBuf); + + /* Make storeEntry public */ + if (hcoder->codedEntry != NULL){ + if (EBIT_TEST(http->storeEntry()->flags, KEY_PRIVATE) + || EBIT_TEST(hcoder->codedEntry->flags, ENTRY_CODED)) { + if (!EBIT_TEST(hcoder->codedEntry->flags, KEY_PRIVATE)) + storeSetPrivateKey(hcoder->codedEntry); + storeReleaseRequest(hcoder->codedEntry); + hcoder->codedEntry->unlock(); + } + else { + EBIT_CLR(hcoder->codedEntry->flags, RELEASE_REQUEST); + storeSetPublicKey(hcoder->codedEntry); + hcoder->codedEntry = NULL; + } + } + + clientStreamDetach(myNode, http); +} + +static clientStream_status_t +codeStreamStatus(clientStreamNode *myNode, + ClientHttpRequest *request) +{ + debug(93, 4) ("HttpHdrContCoder::codeClientStreamStatus: Yo!\n"); + return clientStreamStatus(myNode, request); +} + +/* Handler for case where there's too much overflow */ +static void +codeStreamOvertime(clientStreamNode *myNode, + HttpHdrContCoder *hcoder, + ClientHttpRequest *http, + HttpReply *reply, int codestat) +{ + HttpContentCoder *coder = hcoder->coder; + + debug(93, 3) + ("HttpHdrContCoder::codeStreamOvertime: Hi!\n"); + + char *mybuf = myNode->readBuffer.data; + + if (codestat == CC_ERROR) { + debug(93, 1) ("HttpHdrContCode.cc:codeStreamCallback: encoding error\n"); + return; + } + /* must be CC_PENDING */ + assert (codestat == CC_PENDING); + + do { + + /* Write encoded data to storeEntry, if appropriate */ + if (hcoder->codedEntry != NULL) { + hcoder->codedEntry->write(myNode->readBuffer); + } + + /* Put it to the rest of the stack*/ + clientStreamCallback(myNode, http, reply, myNode->readBuffer); + int olen = myNode->readBuffer.length; //GARANA + + if (hcoder->amencoding) + codestat = coder->encodeEnd(mybuf, &olen); + else + codestat = coder->decodeEnd(mybuf, &olen); + debug(93, 5) + ("HttpHdrContCoder::codeStreamOvertime: encodeEnd: generated %d bytes\n", + olen); + + hcoder->offset += olen; + myNode->readBuffer.offset += olen; + if (!olen) break; + + } while (codestat == CC_PENDING); + + /* after returning, codeStreamCallback will call clientStreamCallback + with current buffer contents */ +} + + +/* + * Content-Coding + */ +HttpHdrContCode::HttpHdrContCode(ClientHttpRequest *http) +{ + clientStream = &http->client_stream; + + ceAcl = new HttpContentCoderAcl(Config.accessList.negotiate_ce); + deflateAcl = new HttpContentCoderAcl(Config.accessList.encode_deflate); + gzipAcl = new HttpContentCoderAcl(Config.accessList.encode_gzip); + + mkCoders(http); + + /* Extract Accept-Encoding field */ + HttpHeader *hdr = &http->request->header; + aelist = CodeList(hdr, &naecoders, HDR_ACCEPT_ENCODING); +} + +HttpHdrContCode::~HttpHdrContCode() +{ + // These are released by ~cbdata + //while (coders.size()) + // delete coders.pop_back(); + + delete ceAcl; + delete deflateAcl; + delete gzipAcl; + + safe_free(aelist); +} + +/* create first coder for this object */ +void +HttpHdrContCode::mkCoders(ClientHttpRequest *http) +{ + debug (93, 4) ("HttpHdrContCode::mkCoders: creating coder\n"); + HttpHdrContCoder *coder = new HttpHdrContCoder(clientStream, http); + coders.push_back(coder); +} + +HttpHdrContCode::iterator +HttpHdrContCode::begin() +{ + return coders.begin(); +} + +HttpHdrContCode::iterator +HttpHdrContCode::end() +{ + return coders.end(); +} + +HttpHdrContCode::const_iterator +HttpHdrContCode::begin() const +{ + return coders.begin(); +} + +HttpHdrContCode::const_iterator +HttpHdrContCode::end() const +{ + return coders.end(); +} + +/* Turn encoding list into HTTP field list suitable for Content-Encoding + or Accept-Encoding headers */ +char *codeField(char *toenclist, int ntoencode) +{ + static char field[80]; + char *fi = field; + int i; + + for (i = 0; i < ntoencode; ++i) { + strcpy(fi, codeName((http_code_type) toenclist[i])); + fi += strlen(fi); + *fi++ = ','; + } + if (ntoencode) + fi[-1] = '\0'; + return field; +} + +/* Need to add check against reply status code later! */ +bool +HttpHdrContCode::codeInitReply(ClientHttpRequest *http, HttpReply *reply) +{ + debug (93, 3) + ("HttpHdrContCode.cc::CodeInitReply: reply = %p\n", reply); + bool amencoding = true; + + HttpHeader *hdr = &reply->header; + char *celist; /* Content-Encoding list */ + int ncecoders; + + /* XXX - for now assume just one active encoding for any objeect */ + HttpHdrContCoder *hcoder = *coders.begin(); + + if (reply->sline.status != 200) { + /* Only want to code correct content */ + debug(93, 4) + ("HttpHdrContCode.cc::CodeInitReply: status %d, no coding needed\n", + reply->sline.status); + hcoder->codingKnown(CODE_NONE, false, true); + return false; + } + + celist = CodeList(hdr, &ncecoders, HDR_CONTENT_ENCODING); + + /* Is there a request to encode? */ + if (naecoders <= 0) { + if (ncecoders <= 0) { + debug(93, 2) + ("HttpHdrContCode.cc::CodeInitReply: no encoding in request or reply, no encode needed\n"); + hcoder->ignore = true; + return false; + } + else /* ncecoders > 0 */ { + debug(93, 2) ("HttpHdrContCode.cc::CodeInitReply: Need to DEcode\n"); + amencoding = false; + } + } + else /* naecoders > 0 */ { + if (ncecoders <= 0) { + debug(93, 2) ("HttpHdrContCode.cc::CodeInitReply: Need to ENcode\n"); + amencoding = true; + } + else /* ncecoders > 0 */ { + if (ncecoders == naecoders && !memcmp(aelist, celist, ncecoders)) { + hcoder->ignore = true; + return false; + } + else if (codeListContained(celist, ncecoders, aelist, naecoders)) { + hcoder->ignore = true; + return false; + } + else { + debug(93, 2) + ("HttpHdrContCode.cc::CodeInitReply: Coder mismatch - need to DEcode\n"); + amencoding = false; + } + } + } + debug(93, 2) + ("HttpHdrContCode.cc::CodeInitReply: have content coding, encoding = %d\n", + amencoding); + + assert(!hcoder->ignore); + int ntoencode = 1; +#if 1 + char toenclist[1]; /* list of encodings to actually use here */ + + /* Can only have one active coder so far - use ifs instead of loop */ + /* for now */ + ceAcl->codeAclCheck(http, reply); + gzipAcl->codeAclCheck(http, reply); + deflateAcl->codeAclCheck(http, reply); + + if (!ceAcl->canCode()) { + /* CE Negotiation disabled, ignore */ + hcoder->ignore = true; + return false; + } + /* FIXME: this is incomplete: it should check aelist against celist, on each member */ + if (gzipAcl->canCode() && codeListIn(aelist, naecoders, CODE_GZIP)) + toenclist[0] = CODE_GZIP; + else if (deflateAcl->canCode() && codeListIn(aelist, naecoders, CODE_DEFLATE)) + toenclist[0] = CODE_DEFLATE; + else + toenclist[0] = CODE_NONE; +#else + char *toenclist; /* list of encodings to actually use here */ + + /* preACL test code for determining coder */ + toenclist = toEncodeList(hdr, &ntoencode, &amencoding, + aelist, naecoders, celist, ncecoders); +#endif + + hcoder->codingKnown((http_code_type) toenclist[0], amencoding, false); + + /* Set up StoreEntry to copy to, if appropriate */ + hcoder->initEntry(http, reply, toenclist, ntoencode); + + /* Put encoding in header */ + reply->header.delById(HDR_CONTENT_ENCODING); + if (amencoding) { + char *field = codeField(toenclist, ntoencode); + reply->header.putStr(HDR_CONTENT_ENCODING, field); + } + /* We won't know the length before sending header. Delete + Content-Length field and disable persistence for this connection */ + reply->header.delById(HDR_CONTENT_LENGTH); + + /* If I'm going to decode, no content-encoding will be present */ + /* FIXME: when [de]coder list selection logic is fixed, this should be + removed or at least modified */ + if (!hcoder->amencoding) { + reply->header.delById(HDR_CONTENT_ENCODING); + } + + return true; +} + +char * +CodeList(HttpHeader *hdr, int *ncodersp, http_hdr_type htype) +{ + String field = hdr->getList(htype); + int ncoders; + + if (field.size() == 0) { + if (ncodersp) + *ncodersp = 0; + return NULL; + } + + char *codelist = (char *) xmalloc(field.size() / 2); + ncoders = codeParseField(&field, codelist); + if (ncodersp) + *ncodersp = ncoders; + return codelist; +} + +/* codeList membership fn. True iff type in list somewhere */ +static bool +codeListIn(char *list, int len, http_code_type type) +{ + int i; + + for (i = len; i--; ) { + if (list[i] == type) + return true; + } + return false; +} + +/* codeList inclusion. True iff big contains small */ +static bool +codeListContained(char *list1, int len1, char* list2, int len2) { + /* This way avoids an O(x^2) algorithm */ + unsigned mask1 = 0, mask2 = 0; + for (int i = len1; i--;) mask1 |= (1 << list1[i]); + for (int i = len2; i--;) mask2 |= (1 << list2[i]); + return (mask1 & mask2) == mask1 ? true : false; +} + +/* Debugging code only, to call from codeInitReply */ +char * +HttpHdrContCode::toEncodeList(HttpHeader *hdr, int *ncodersp, bool *amencodingsp, + char *aelist, int naecoders, + char *celist, int ncecoders) +{ + /* Assume that aelist differs from celist */ + + static char toenclist[1]; + + + /* + Should work intelligently, based on acl entries + + Algorithm - Decide on what encoding(s) to use based on + Content-Type. Need to have provision to specify in acl + whether multiple encodings are allowed. + */ + /* XXX - Right now, only support one encoding or decoding. + If celist != "", then decode */ + assert(ncecoders <= 1); + if (ncecoders == 1) { + *amencodingsp = false; + *ncodersp = ncecoders; + return celist; + } + + /* No content-encoding, just allow-encodings */ + assert(naecoders > 0); /* ...since aecoders != cecoders... */ + /* For now, give priority to gzip, until acl does this riught + Otherwise, just take the first encoding on the list */ + if (strchr(aelist, CODE_GZIP)) + toenclist[0] = CODE_GZIP; + else + toenclist[0] = aelist[0]; + *ncodersp = 1; + return toenclist; +} + + +/* Assorted fns with no class atall */ + +static http_code_type nameCode(String *codename) +{ + if (!codename->cmp("gzip")) + return CODE_GZIP; + else if (!codename->cmp("x-gzip")) + return CODE_GZIP; + else if (!codename->cmp("deflate")) + return CODE_DEFLATE; + else { + debug(93, 2) + ("HttpHdrContCode::nameCode: unknown encoding %s\n", codename->buf()); + } + return CODE_ERR; +} + +static const char *codeName(http_code_type code) +{ + if (code == CODE_GZIP) + return "gzip"; + else if (code== CODE_DEFLATE) + return "deflate"; + else if (code== CODE_NONE) + return "identity"; + else if (code == CODE_ERR) { + debug(93, 1) + ("HttpHdrContCode::nameCode: translating CODE_ERR\n"); + return "identity"; + } + else { + debug(93, 1) + ("HttpHdrContCode::nameCode: unknown encoding %d\n", code); + return "identity"; + } + /* NOTREACHED */ +} + + +/* Parse *-Encoding field */ +static int +codeParseField(String *field, char codelist[]) +{ + char *fstr = xstrdup(field->buf()); + http_code_type code; + char *next, *qptr; + int c = 0; + + for (next = fstr; next; fstr = next) + { + /* skip spaces */ + int nspaces = strspn(fstr, " \t"); + fstr += nspaces; + + /* look for next element and zero */ + next = strchr(fstr, ','); + if (next) + *next++ = '\0'; + + /* Ignore semicolon-delimited q-factor by zeroing semicolon */ + qptr = strchr(fstr, ';'); + if (qptr) + *qptr = '\0'; + + code = nameCode(new String(fstr)); + debug(93, 4) ("HttpHdrContCode::CodeParseField: %s->%d\n", fstr, code); + if (code != CODE_ERR) + codelist[c++] = code; + } + { + int i; + debug(93, 4) + ("HttpHdrContCode.cc:CodeParseField: ncoders = %d, coders are:\n", c); + for (i = 0; i < c; ++i) + debug(93, 2) (" %d\n", codelist[i]); + } + + return c; +} + +/* External interface functions to set up encoding if appropriate */ +HttpHdrContCode * +CodeParseRequest(ClientHttpRequest *http) +{ + debug(93, 3) ("HttpHdrContCode.cc::CodeParseRequest: url %s http = %p\n", + http->request->urlpath.buf(), http); + if (http->request->method != METHOD_GET) { + debug(93, 2) + ("HttpHdrContCode.cc::CodeParseRequest: no encoding for %s method\n", + RequestMethodStr[http->request->method]); + return NULL; + } + + /* This connection can't be persistent because we can't figure out amounts + of encoded data in advance. For now.*/ + http->request->flags.proxy_keepalive = 0; + + assert(http->request->codings == NULL); + HttpHdrContCode *code = new HttpHdrContCode::HttpHdrContCode(http); + http->request->codings = code; + return(code); +} Index: squid3/src/HttpHdrContCode.h diff -u /dev/null squid3/src/HttpHdrContCode.h:1.1.2.2 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/HttpHdrContCode.h Fri Sep 1 16:44:54 2006 @@ -0,0 +1,121 @@ + +/* + * $Id$ + * + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_HTTPCONTCODE_H +#define SQUID_HTTPCONTCODE_H + +#include "Array.h" + +class HttpReply; +class HttpStateData; +class ClientHttpRequest; +class HttpContentCoder; +class HttpContentCoderAcl; + +class HttpHdrContCoder: public RefCountable +{ + +public: + MEMPROXY_CLASS(HttpHdrContCoder); + + HttpHdrContCoder(dlink_list *stream, ClientHttpRequest *http); + ~HttpHdrContCoder(); + + void codingKnown(http_code_type code, bool amencoding, + bool ignore); + void initEntry(ClientHttpRequest *http, HttpReply *reply, + char *codeList, int ncoders); + + HttpContentCoder *coder; + + /* Encoding control */ + int contentLen; + char *codedBuf; /* Buffer in which to put encoded data */ + ssize_t offset; /* Offset in encoded bytes */ + ssize_t uncoded; /* Offset in uncoded bytes */ + bool ignore; /* If true, then don't encode anything */ + bool amencoding; /* true if encoding; false if decoding */ + bool doneended; /* called codeEnd */ + + /* buffer-initted bit */ + bool bufinitted; + + StoreEntry *codedEntry; + +private: + void chooseCoder(const http_code_type code); + bool codeClientStreamInit(dlink_list *stream); +}; +MEMPROXY_CLASS_INLINE(HttpHdrContCoder); + +class HttpHdrContCode +{ + +public: + MEMPROXY_CLASS(HttpHdrContCode); + HttpHdrContCode(ClientHttpRequest *http); + ~HttpHdrContCode(); + + bool codeInitReply(ClientHttpRequest *http, HttpReply *reply); + + typedef Vector::iterator iterator; + typedef Vector::const_iterator const_iterator; + iterator begin(); + const_iterator begin () const; + iterator end(); + const_iterator end() const; + Vector coders; + + /* Codelist */ + char *aelist; /* Accept-Encodings list */ + int naecoders; + +private: + void mkCoders(ClientHttpRequest *http); + char *toEncodeList(HttpHeader *hdr, int *ncodersp, bool *amencodingp, + char *aelist, int naecoders, + char *celist, int ncecoders); + + dlink_list *clientStream; + + HttpContentCoderAcl *ceAcl; + HttpContentCoderAcl *deflateAcl; + HttpContentCoderAcl *gzipAcl; +}; +MEMPROXY_CLASS_INLINE(HttpHdrContCode); + +SQUIDCEXTERN HttpHdrContCode *CodeParseRequest(ClientHttpRequest *http); +SQUIDCEXTERN char *CodeList(HttpHeader *hdr, int *ncodersp, http_hdr_type htype); + +#endif /* SQUID_HTTPCONTCODE_H */ + Index: squid3/src/HttpHeader.cc diff -u squid3/src/HttpHeader.cc:1.52 squid3/src/HttpHeader.cc:1.35.2.6 --- squid3/src/HttpHeader.cc:1.52 Tue Jan 22 12:50:40 2008 +++ squid3/src/HttpHeader.cc Mon Feb 4 21:37:00 2008 @@ -1078,7 +1078,12 @@ HttpHeader::putStr(http_hdr_type id, const char *str) { assert_eid(id); +#ifdef CONTENT_ENCODING + /*XXX delete me when real ETag support appears! */ + assert(Headers[id].type == ftStr || Headers[id].type == ftETag); +#else assert(Headers[id].type == ftStr); /* must be of an appropriate type */ +#endif assert(str); addEntry(new HttpHeaderEntry(id, NULL, str)); } Index: squid3/src/HttpRequest.cc diff -u squid3/src/HttpRequest.cc:1.46 squid3/src/HttpRequest.cc:1.33.2.7 --- squid3/src/HttpRequest.cc:1.46 Sun Feb 3 02:50:54 2008 +++ squid3/src/HttpRequest.cc Mon Feb 4 21:37:00 2008 @@ -38,6 +38,9 @@ #include "HttpRequest.h" #include "AuthUserRequest.h" #include "HttpHeaderRange.h" +#ifdef CONTENT_ENCODING +#include "HttpHdrContCode.h" +#endif #include "MemBuf.h" #include "Store.h" @@ -94,6 +97,9 @@ extacl_user = null_string; extacl_passwd = null_string; extacl_log = null_string; +#ifdef CONTENT_ENCODING + codings = NULL; +#endif pstate = psReadyToParseStartLine; } @@ -131,6 +137,13 @@ extacl_passwd.clean(); extacl_log.clean(); + +#ifdef CONTENT_ENCODING + if (codings) + delete codings; + codings = NULL; +#endif + } void Index: squid3/src/HttpRequest.h diff -u squid3/src/HttpRequest.h:1.32 squid3/src/HttpRequest.h:1.23.2.6 --- squid3/src/HttpRequest.h:1.32 Sun Jan 20 01:50:56 2008 +++ squid3/src/HttpRequest.h Mon Feb 4 21:37:00 2008 @@ -46,6 +46,9 @@ class HttpHdrRange; +#ifdef CONTENT_ENCODING +class HttpHdrContCode; +#endif class HttpRequest: public HttpMsg { @@ -144,6 +147,10 @@ String extacl_log; /* String to be used for access.log purposes */ +#ifdef CONTENT_ENCODING + HttpHdrContCode *codings; +#endif + public: bool multipartRangeRequest() const; Index: squid3/src/Makefile.am diff -u squid3/src/Makefile.am:1.138 squid3/src/Makefile.am:1.101.2.6 --- squid3/src/Makefile.am:1.138 Wed Jan 23 13:51:00 2008 +++ squid3/src/Makefile.am Mon Feb 4 21:37:02 2008 @@ -30,6 +30,21 @@ SUBDIRS = fs repl auth +if ENABLE_CONTENT_ENCODING +CONT_CODE_SOURCE = HttpContentCoder.h \ + HttpContentCoder.cc \ + deflateContentCoder.cc \ + deflateContentCoder.h \ + gzipContentCoder.cc \ + gzipContentCoder.h \ + noneContentCoder.cc \ + noneContentCoder.h \ + HttpHdrContCode.cc \ + HttpHdrContCode.h +else +CONT_CODE_SOURCE = +endif + DELAY_POOL_ALL_SOURCE = \ CommonPool.h \ CompositePoolNode.h \ @@ -457,6 +472,7 @@ ConfigParser.cc \ ConfigParser.h \ ConnectionDetail.h \ + $(CONT_CODE_SOURCE) \ debug.cc \ Debug.h \ defines.h \ @@ -519,6 +535,7 @@ HttpRequestMethod.cc \ HttpRequestMethod.h \ HttpVersion.h \ + $(CONTENT_ENCODING_SOURCE) \ $(ICMPSOURCE) \ ICP.h \ icp_v2.cc \ @@ -798,6 +815,7 @@ $(squid_COMMSOURCES) \ ConfigOption.cc \ defines.h \ + $(CONT_CODE_SOURCE) \ $(DELAY_POOL_SOURCE) \ disk.cc \ $(DNSSOURCE) \ Index: squid3/src/Store.h diff -u squid3/src/Store.h:1.41 squid3/src/Store.h:1.24.2.6 --- squid3/src/Store.h:1.41 Sun Jan 20 01:50:56 2008 +++ squid3/src/Store.h Mon Feb 4 21:37:03 2008 @@ -153,6 +153,7 @@ public: static size_t inUseCount(); + static void getPublicByEncoding(StoreClient * aClient, HttpRequest * request); static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method); static void getPublicByRequest(StoreClient * aClient, HttpRequest * request); static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method); Index: squid3/src/cf.data.pre diff -u squid3/src/cf.data.pre:1.171 squid3/src/cf.data.pre:1.106.2.7 --- squid3/src/cf.data.pre:1.171 Wed Jan 23 13:51:00 2008 +++ squid3/src/cf.data.pre Mon Feb 4 21:37:04 2008 @@ -5521,4 +5521,36 @@ rounded to 1000. DOC_END +NAME: negotiate_ce_access +IFDEF: CONTENT_ENCODING +TYPE: acl_access +LOC: Config.accessList.negotiate_ce +DEFAULT: none +DEFAULT_IF_NONE: deny all +DOC_START + Enable or disable content encoding negotiation. This + allows squid to do [de]compression of response bodies + before they are returned to clients. +DOC_END + +NAME: encode_gzip_access +IFDEF: CONTENT_ENCODING +TYPE: acl_access +LOC: Config.accessList.encode_gzip +DEFAULT: none +DEFAULT_IF_NONE: deny all +DOC_START + Enable or disable gzip compression. +DOC_END + +NAME: encode_deflate_access +IFDEF: CONTENT_ENCODING +TYPE: acl_access +LOC: Config.accessList.encode_deflate +DEFAULT: none +DEFAULT_IF_NONE: deny all +DOC_START + Enable or disable deflate compression. +DOC_END + EOF Index: squid3/src/cf_gen_defines diff -u squid3/src/cf_gen_defines:1.6 squid3/src/cf_gen_defines:1.4.2.1 --- squid3/src/cf_gen_defines:1.6 Wed Jan 23 13:51:00 2008 +++ squid3/src/cf_gen_defines Thu Aug 31 16:11:33 2006 @@ -20,6 +20,7 @@ define["USE_WCCP"]="--enable-wccp" define["USE_WCCPv2"]="--enable-wccpv2" define["ESI"]="--enable-esi" + define["CONTENT_ENCODING"]="--enable-content-encoding" } /^IFDEF:/ { if (define[$2] != "") Index: squid3/src/clientStream.cc diff -u squid3/src/clientStream.cc:1.11 squid3/src/clientStream.cc:1.10.6.2 --- squid3/src/clientStream.cc:1.11 Sat Apr 28 15:51:49 2007 +++ squid3/src/clientStream.cc Thu May 10 15:39:02 2007 @@ -44,7 +44,7 @@ * An alternative approach is to pass each node in the pipe the call- * back to use on each IO call. This allows the callbacks to be changed * very easily by a participating node, but requires more maintenance - * in each node (store the call back to the msot recent IO request in + * in each node (store the call back to the most recent IO request in * the nodes context.) Such an approach also prevents dynamically * changing the pipeline from outside without an additional interface * method to extract the callback and context from the next node. Index: squid3/src/client_side_reply.cc diff -u squid3/src/client_side_reply.cc:1.124 squid3/src/client_side_reply.cc:1.86.2.7 --- squid3/src/client_side_reply.cc:1.124 Sun Feb 3 02:50:55 2008 +++ squid3/src/client_side_reply.cc Mon Feb 4 21:37:04 2008 @@ -34,6 +34,7 @@ */ #include "squid.h" +#include "http.h" #include "client_side_reply.h" #include "errorpage.h" #include "StoreClient.h" @@ -55,6 +56,10 @@ #endif #include "client_side.h" #include "SquidTime.h" +#ifdef CONTENT_ENCODING +#include "HttpContentCoder.h" +#include "HttpHdrContCode.h" +#endif CBDATA_CLASS_INIT(clientReplyContext); @@ -1422,6 +1427,10 @@ /* do header conversions */ buildReplyHeader(); +#ifdef CONTENT_ENCODING + if (http->request->codings != NULL) + http->request->codings->codeInitReply(http, reply); +#endif } void @@ -1431,7 +1440,7 @@ if (r->flags.cachable || r->flags.internal) { lookingforstore = 5; - StoreEntry::getPublicByRequest (this, r); + StoreEntry::getPublicByEncoding (this, r); } else { identifyFoundObject (NullStoreEntry::getInstance()); } @@ -1691,8 +1700,13 @@ StoreIOBuffer tempBuffer; flags.complete = 1; tempBuffer.flags.error = result.flags.error; +#if CONTENT_ENCODING_WANTS_REPLY + clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, + reply, tempBuffer); +#else clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL, tempBuffer); +#endif } void @@ -1712,8 +1726,13 @@ if (tempBuffer.length) tempBuffer.data = source; +#if CONTENT_ENCODING_WANTS_REPLY + clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, + reply, tempBuffer); +#else clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL, tempBuffer); +#endif } clientStreamNode * Index: squid3/src/client_side_reply.h diff -u squid3/src/client_side_reply.h:1.20 squid3/src/client_side_reply.h:1.15.2.4 --- squid3/src/client_side_reply.h:1.20 Sun Jan 20 01:50:56 2008 +++ squid3/src/client_side_reply.h Mon Feb 4 21:37:04 2008 @@ -116,7 +116,7 @@ flags; clientStreamNode *ourNode; /* This will go away if/when this file gets refactored some more */ - + private: CBDATA_CLASS(clientReplyContext); clientStreamNode *getNextNode() const; Index: squid3/src/client_side_request.cc diff -u squid3/src/client_side_request.cc:1.88 squid3/src/client_side_request.cc:1.57.2.7 --- squid3/src/client_side_request.cc:1.88 Sun Feb 3 02:50:55 2008 +++ squid3/src/client_side_request.cc Mon Feb 4 21:37:04 2008 @@ -55,6 +55,10 @@ #include "HttpReply.h" #include "MemObject.h" #include "ClientRequestContext.h" +#ifdef CONTENT_ENCODING +#include "HttpContentCoder.h" +#include "HttpHdrContCode.h" +#endif #include "SquidTime.h" #include "wordlist.h" @@ -904,6 +908,9 @@ logType = LOG_TAG_NONE; debugs(85, 4, "ClientHttpRequest::httpStart: " << log_tags[logType] << " for '" << uri << "'"); +#ifdef CONTENT_ENCODING + request->codings = CodeParseRequest(this); +#endif /* no one should have touched this */ assert(out.offset == 0); /* Use the Stream Luke */ Index: squid3/src/deflateContentCoder.cc diff -u /dev/null squid3/src/deflateContentCoder.cc:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/deflateContentCoder.cc Thu Aug 31 16:11:35 2006 @@ -0,0 +1,382 @@ +/* + * $Id$ + * + * DEBUG: section 93 HTTP Content-Encoding + * AUTHOR: Gonzalo Arana + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#include "deflateContentCoder.h" +#include +#include "squid.h" + +acl_access * +deflateContentCoder::getEncodeACL() { + return Config.accessList.encode_deflate; +} +CBDATA_CLASS_INIT(deflateContentCoder); + +// /* +// * deflateContentCoder::MemPool routines +// */ +// MemPool* deflateContentCoder::Pool = NULL; + +void* +deflateContentCoder::operator new (size_t size) +{ + assert (size == sizeof(deflateContentCoder)); + CBDATA_INIT_TYPE(deflateContentCoder); + return cbdataAlloc(deflateContentCoder); +// if (!Pool) +// Pool = memPoolCreate("deflateContentCoder", sizeof(deflateContentCoder)); + +// return memPoolAlloc(Pool); +} + +void +deflateContentCoder::operator delete (void *address) +{ + deflateContentCoder* tmp = (deflateContentCoder*)address; + cbdataFree(tmp); +// memPoolFree(Pool, address); +} + + + +inline static int min(int x, int y) { return x < y ? x : y; } + +deflateContentCoder::deflateContentCoder() { + memset(&zlibState, 0, sizeof(zlibState)); + state = NONE; +} + +static void __zstream_init(z_stream* z) { + z->next_out = z->next_in = NULL; + z->avail_out = z->avail_in = 0; + z->zalloc = NULL; + z->zfree = NULL; + z->opaque = NULL; +} + +static void __prepare(z_stream* zlibState, + char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + zlibState->next_in = (Bytef*) ibuff; + zlibState->avail_in = *ibufflen; + zlibState->next_out = (Bytef*) obuff; + zlibState->avail_out = *obufflen; +} + +#if defined(TEST_DEFLATE) || defined(TEST_GZIP) +static const char* zlib_status_ntoa (int status) { + switch (status) { + case Z_OK: + return "Z_OK"; + case Z_STREAM_END: + return "Z_STREAM_END"; + case Z_NEED_DICT: + return "Z_NEED_DICT"; + case Z_ERRNO: + return "Z_ERRNO"; + case Z_STREAM_ERROR: + return "Z_STREAM_ERROR"; + case Z_DATA_ERROR: + return "Z_DATA_ERROR"; + case Z_MEM_ERROR: + return "Z_MEM_ERROR"; + case Z_BUF_ERROR: + return "Z_BUF_ERROR"; + case Z_VERSION_ERROR: + return "Z_VERSION_ERROR"; + default: + return "unknown"; + } + return NULL; +} +#endif + +int deflateContentCoder::encodeStart(char* ibuff, int *ibufflen, + char* obuff, int* obufflen) { + assert(state == NONE); + debug (93, 4) + ("deflateContentCoder::encodeStart(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + __zstream_init(&zlibState); + debug (93, 5)("deflateContentCoder::encodeStart: calling deflateInit2\n"); + statCounter.libcalls.zlib.deflateInit++; + PROF_start(zlib); + if (Z_OK != deflateInit2(&zlibState, 9, + Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY)) { + state = ERR; + } + PROF_stop(zlib); + if (state == ERR) { + *ibufflen = *obufflen = 0; + return CC_ERROR; + } + state = DEFLATING; + + /* + * FIXME: This condition check is due to gzipContentCoder's difficulty + * to handle empty input & output buffers upon initializing. + * Should be fixed. + */ + if (*ibufflen && *obufflen) + return encodeChunk(ibuff, ibufflen, obuff, obufflen); + else { + *obufflen = *ibufflen = 0; + } + return CC_OK; +} + + +int deflateContentCoder::encodeChunk(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + debug (93, 4) + ("deflateContentCoder::encodeChunk(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + + int status; + if (state == ERR) { return CC_ERROR; } + + assert(state == DEFLATING); + + if (*obufflen) { + __prepare(&zlibState, ibuff, ibufflen, obuff, obufflen); + //status = deflate(&zlibState, Z_NO_FLUSH); + + debug(93, 5) + ("deflateContentCoder::encodeChunk: calling " + "deflate(avail_in=%d, avail_out=%d, " + "total_in=%ld, total_out=%ld)\n", + zlibState.avail_in, zlibState.avail_out, + zlibState.total_in, zlibState.total_out); + PROF_start(zlib); + statCounter.libcalls.zlib.deflate++; + status = deflate(&zlibState, Z_SYNC_FLUSH); + PROF_stop(zlib); + debug(93, 5) + ("deflateContentCoder::encodeChunk: called " + "deflate(avail_in=%d, avail_out=%d, " + "total_in=%ld, total_out=%ld)\n", + zlibState.avail_in, zlibState.avail_out, + zlibState.total_in, zlibState.total_out); + if (Z_OK != status) { + debug (93, 4)("deflateContentCoder::encodeChunk Error\n"); + state = ERR; + return CC_ERROR; + } + *ibufflen = *ibufflen - zlibState.avail_in; + *obufflen = *obufflen - zlibState.avail_out; + } else + *ibufflen = 0; + debug (93, 4) + ("deflateContentCoder::encodeChunk: generated %d zipped bytes\n", + *obufflen); + return CC_OK; +} + +int deflateContentCoder::encodeEnd(char* obuff, int* obufflen) { + debug (93, 4) + ("deflateContentCoder::encodeEnd(obuff=%p, obufflen=%d)\n", + obuff, *obufflen); + int deflate_status; + int _zero = 0; + char ibuff; + int ospace = *obufflen; // number of bytes available in obuff + *obufflen = 0; + //assert(state == DEFLATING || state == ERR || state == WRITING_TRAILER); + assert(state == DEFLATING); + + __prepare(&zlibState, &ibuff, &_zero, obuff, &ospace); + statCounter.libcalls.zlib.deflate++; + PROF_start(zlib); + deflate_status = deflate(&zlibState, Z_FINISH); + PROF_stop(zlib); + *obufflen = ospace - zlibState.avail_out; + // number of bytes written to obuff + ospace = zlibState.avail_out; + // number of bytes available in obuff + obuff = (char*) zlibState.next_out; + if (deflate_status == Z_OK) return CC_PENDING; + if (deflate_status != Z_STREAM_END) { state = ERR; return CC_ERROR; } + PROF_start(zlib); + statCounter.libcalls.zlib.deflateEnd++; + PROF_stop(zlib); + if (Z_OK != deflateEnd(&zlibState)) { + state = ERR; + return CC_ERROR; + } + /* Must write crc & total_in */ + state = NONE; + debug (93, 4) + ("deflateContentCoder::encodeEnd: generated %d zipped bytes \n", + *obufflen); + return CC_DONE; +} + +int deflateContentCoder::decodeStart(char* ibuff, int *ibufflen, + char* obuff, int* obufflen) { + debug (93, 4) + ("deflateContentCoder::decodeStart(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + assert(state == NONE); + __zstream_init(&zlibState); + zlibState.next_in = Z_NULL; + debug (93, 5)("deflateContentCoder::encodeStart: calling inflateInit2\n"); + statCounter.libcalls.zlib.inflateInit++; + PROF_start(zlib); + if (Z_OK != inflateInit2(&zlibState, -MAX_WBITS)) { + PROF_stop(zlib); + state = ERR; + return CC_ERROR; + } + PROF_stop(zlib); + state = INFLATING; + return decodeChunk(ibuff, ibufflen, obuff, obufflen); +} + +int deflateContentCoder::decodeChunk(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + debug (93, 4) + ("deflateContentCoder::decodeChunk(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + + if (state == ERR) return CC_ERROR; + assert(state == INFLATING); + if (!*obufflen || !*ibufflen) { + *ibufflen = *obufflen = 0; + return CC_OK; + } + __prepare(&zlibState, ibuff, ibufflen, obuff, obufflen); + debug(93, 5) + ("deflateContentCoder::encodeChunk: calling " + "inflate(avail_in=%d, avail_out=%d, " + "total_in=%ld, total_out=%ld)\n", + zlibState.avail_in, zlibState.avail_out, + zlibState.total_in, zlibState.total_out); + PROF_start(zlib); + statCounter.libcalls.zlib.inflate++; + int status = inflate(&zlibState, Z_SYNC_FLUSH); + PROF_stop(zlib); + debug(93, 5) + ("deflateContentCoder::encodeChunk: called " + "inflate(avail_in=%d, avail_out=%d, " + "total_in=%ld, total_out=%ld)\n", + zlibState.avail_in, zlibState.avail_out, + zlibState.total_in, zlibState.total_out); + + *ibufflen = *ibufflen - zlibState.avail_in; + *obufflen = *obufflen - zlibState.avail_out; + + debug(93, 4) + ("deflateContentCoder::encodeChunk: generated %d plain bytes\n", + *obufflen); + + if (Z_OK == status) return CC_OK; + if (Z_STREAM_END == status) { state = ENDING; return CC_DONE; } + + state = ERR; + debug (93, 4)("deflateContentCoder::encodeChunk Error\n"); + return CC_ERROR; +} + +int deflateContentCoder::decodeEnd(char* obuff, int* obufflen) { + debug (93, 4) + ("deflateContentCoder::decodeEnd(obuff=%p, obufflen=%d)\n", + obuff, *obufflen); + int _zero = 0, inflate_status; + switch (state) { + case INFLATING: + case ERR: + __prepare(&zlibState, obuff, &_zero, obuff, obufflen); + PROF_start(zlib); + statCounter.libcalls.zlib.inflate++; + inflate_status = inflate(&zlibState, Z_SYNC_FLUSH); + PROF_stop(zlib); + *obufflen -= zlibState.avail_out; + switch (inflate_status) { + case Z_BUF_ERROR: /* no progress is possible (already at the end) */ + case Z_STREAM_ERROR: /* used incorrectly */ + assert(0/* incorrect usage */); + return CC_ERROR; + case Z_STREAM_END: /* end of compressed data */ + PROF_start(zlib); + statCounter.libcalls.zlib.inflateEnd++; + if (Z_OK == inflateEnd(&zlibState)) { + PROF_stop(zlib); + state = NONE; + return CC_DONE; + } else { + PROF_stop(zlib); + state = ERR; + return CC_ERROR; + } + break; + case Z_OK: /* progress has been made */ + state = INFLATING; + return CC_PENDING; + default: /* unknown error */ + assert(0); + break; + } + assert(0); + break; + case ENDING: + *obufflen = 0; + state = ENDING; + PROF_start(zlib); + statCounter.libcalls.zlib.inflateEnd++; + if (Z_OK == inflateEnd(&zlibState)) { + PROF_stop(zlib); + /* must append crc */ + state = NONE; + return CC_DONE; + } else { + PROF_stop(zlib); + state = ERR; + return CC_ERROR; + } + return CC_DONE; + default: + assert(0); + } + assert(0); + return CC_ERROR; +} Index: squid3/src/deflateContentCoder.h diff -u /dev/null squid3/src/deflateContentCoder.h:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/deflateContentCoder.h Thu Aug 31 16:11:38 2006 @@ -0,0 +1,77 @@ +/* + * $Id$ + * + * DEBUG: section 93 HTTP Content-Encoding + * AUTHOR: Gonzalo Arana + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#ifndef __DEFLATECONTENTCODER_H_ +#define __DEFLATECONTENTCODER_H_ + +#include "HttpContentCoder.h" +#include "MemPool.h" +#include + +class deflateContentCoder : public virtual HttpContentCoder { + + public: + enum STATE { + NONE = 0, PARSING_HEADER, DEFLATING, + INFLATING, ENDING, WRITING_TRAILER, ERR + }; + + protected: + enum STATE state; + z_stream zlibState; + + public: + + deflateContentCoder(); + void* operator new (size_t); + void operator delete(void*); + int encodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen); + int encodeEnd(char* obuff, int* obufflen); + int encodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen); + int decodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen); + int decodeEnd(char* obuff, int* obufflen); + int decodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen); + + int totalIn() const { return zlibState.total_in; } + int totalOut() const { return zlibState.total_out; } + + private: + acl_access* getEncodeACL(); + CBDATA_CLASS(deflateContentCoder); +/* static MemPool* Pool; */ +}; + +#endif Index: squid3/src/enums.h diff -u squid3/src/enums.h:1.40 squid3/src/enums.h:1.34.2.4 --- squid3/src/enums.h:1.40 Fri Dec 14 12:51:16 2007 +++ squid3/src/enums.h Mon Feb 4 21:37:06 2008 @@ -324,7 +324,8 @@ ENTRY_NEGCACHED, ENTRY_VALIDATED, ENTRY_BAD_LENGTH, - ENTRY_ABORTED + ENTRY_ABORTED, + ENTRY_CODED }; /* @@ -563,4 +564,11 @@ DISABLE_PMTU_TRANSPARENT }; +typedef enum { + CODE_ERR, + CODE_NONE, + CODE_GZIP, + CODE_DEFLATE +} http_code_type; + #endif /* SQUID_ENUMS_H */ Index: squid3/src/gzipContentCoder.cc diff -u /dev/null squid3/src/gzipContentCoder.cc:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/gzipContentCoder.cc Thu Aug 31 16:11:38 2006 @@ -0,0 +1,476 @@ +/* + * $Id$ + * + * DEBUG: section 93 HTTP Content-Encoding + * AUTHOR: Gonzalo Arana + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#include "squid.h" +#include "gzipContentCoder.h" + +/* + * Defines for gzip headers. + */ + +/* magics known to process */ +#define PACK_MAGIC "\037\036" /* Magic header for packed files */ +#define GZIP_MAGIC "\037\213" /* Magic header for gzip files, 1F 8B */ +#define OLD_GZIP_MAGIC "\037\236" /* Magic header for gzip 0.5 = freeze 1.x */ +#define LZH_MAGIC "\037\240" /* Magic header for SCO LZH Compress files*/ +#define PKZIP_MAGIC "\120\113\003\004" /* Magic header for pkzip files */ + +/* flags in header[3] */ +#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ +#define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */ +#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ +#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ +#define COMMENT 0x10 /* bit 4 set: file comment present */ +#define ENCRYPTED 0x20 /* bit 5 set: file is encrypted */ +#define RESERVED 0xC0 /* bit 6,7: reserved */ + +/* Compression methods (see algorithm.doc) */ +#define STORED 0 +#define COMPRESSED 1 +#define PACKED 2 +#define LZHED 3 +/* methods 4 to 7 reserved */ +#define DEFLATED 8 +#define MAX_METHODS 9 + +acl_access * +gzipContentCoder::getEncodeACL() { + return Config.accessList.encode_gzip; +} +CBDATA_CLASS_INIT(gzipContentCoder); + +// /* +// * gzipContentCoder::MemPool routines +// */ +// MemPool* gzipContentCoder::Pool = NULL; + +void* +gzipContentCoder::operator new (size_t size) +{ + assert (size == sizeof(gzipContentCoder)); + CBDATA_INIT_TYPE(gzipContentCoder); + return cbdataAlloc(gzipContentCoder); +// assert (size == sizeof(gzipContentCoder)); + +// if (!Pool) +// Pool = memPoolCreate("gzipContentCoder", sizeof(gzipContentCoder)); + +// return memPoolAlloc(Pool); +} + +void +gzipContentCoder::operator delete (void *address) +{ + gzipContentCoder* tmp = (gzipContentCoder*) address; + cbdataFree(tmp); +// memPoolFree(Pool, address); +} + +/* + * Internal routines + */ +inline static int min(int x, int y) { return x < y ? x : y; } + +/* + * gzip's internals + */ +int gzipContentCoder::mtime() const { + int dev = header[4]; + dev |= header[5] << 8; + dev |= header[6] << 16; + dev |= header[7] << 24; + return dev; +} + +static const char gzip_header[10] = { + 0x1f, 0x8b, // gzip magic + 0x08, // compression method: deflate + 0, // flags: is ascii, etc. + 0, 0, 0, 0, // mtime of file (now) + 0x02, // extra flags: max compression (ignored on reading) + 0x03 // os of compression: UNIX +}; + +/* + * [de]compression methods. + */ +int gzipContentCoder::encodeStart(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + assert(state == NONE); + debug (93, 4) + ("gzipContentCoder::encodeStart(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + header_bytes_sent = min(10, *obufflen); + crc = 0L; + plainBytes = 0; + wrote_tail_bytes = 0; + + /* + * call deflate::encodeStart with no data to let deflateContentCoder + * initialize itself. Then, copy as much as gzip header as possible, + * and call encodeChunk with input data. + */ + int _zero = 0, olen = *obufflen; + /* + * Since no input is given to deflate::encodeStart, no output will be + * generated, so discard olen value. + */ + if (deflateContentCoder::encodeStart(ibuff, &_zero, obuff, &olen) != CC_OK) { + /* On error, no input is processed nor output generated. */ + *ibufflen = *obufflen = 0; + return CC_ERROR; + } + + xmemcpy((void*)obuff, (void*)gzip_header, header_bytes_sent); + /* I have fewer bytes available in output buffer. */ + *obufflen -= header_bytes_sent; + obuff += header_bytes_sent; + debug (93, 5)("gzipContentCoder::encodeStart: wrote %d header bytes\n", + header_bytes_sent); + int dev = encodeChunk(ibuff, ibufflen, obuff, obufflen); + *obufflen += header_bytes_sent; + debug (93, 4) + ("gzipContentCoder::encodeStart %d plain bytes -> %d zipped bytes\n", + *ibufflen, *obufflen); + return dev; +} + +int gzipContentCoder::encodeChunk(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + int ncopy = 0; + assert(state == DEFLATING); + debug (93, 4) + ("gzipContentCoder::encodeChunk(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + + /* Check for pending header */ + if (header_bytes_sent < 10) { + ncopy = min(*obufflen, 10 - header_bytes_sent); + xmemcpy((void*)obuff, (void*)(gzip_header + header_bytes_sent), ncopy); + header_bytes_sent += ncopy; + *obufflen -= ncopy; + obuff += ncopy; + } + + /* silly assert to check pending header logic */ + if (header_bytes_sent < 10) { + assert(!*obufflen); + *obufflen = ncopy; + // ncopy bytes has been written obuff + *ibufflen = 0; + // no input has been processed due to insuficient output space + return CC_OK; + } else { + debug (93, 5)("gzipContentCoder::encodeChunk: wrote %d header bytes " + "(so far)\n", header_bytes_sent); + } + + int dev = + deflateContentCoder::encodeChunk(ibuff, ibufflen, obuff, obufflen); + + *obufflen += ncopy; + crc = crc32(crc, (Bytef*)ibuff, *ibufflen); + plainBytes += *ibufflen; + + debug (93, 4) + ("gzipContentCoder::encodeChunk %d plain bytes -> %d zipped bytes\n", + *ibufflen, *obufflen); + + return dev; +} + +int gzipContentCoder::encodeEnd(char* obuf, int* obufflen) { + int ospace = *obufflen; + if (state == ERR || state == DEFLATING) { + int dev = deflateContentCoder::encodeEnd(obuf, obufflen); + if (dev != CC_DONE) return dev; + state = WRITING_TRAILER; + xmemcpy(trailer, &crc, sizeof(crc)); + xmemcpy(trailer+4, &plainBytes, sizeof(plainBytes)); + ospace -= *obufflen; + obuf += *obufflen; + } + if (state == WRITING_TRAILER) { + assert(wrote_tail_bytes < sizeof(trailer)); + int nout = min(ospace, sizeof(trailer)-wrote_tail_bytes); + if (nout) { + xmemcpy(obuf, trailer + wrote_tail_bytes, nout); + wrote_tail_bytes += nout; + *obufflen += nout; + debug (93, 4) + ("gzipContentCoder::encodeEnd: generated %d tail bytes\n", nout); + if (wrote_tail_bytes == sizeof(trailer)) { + state = NONE; + return CC_DONE; + } + } + return CC_PENDING; + } + state = ERR; + return CC_ERROR; +} + +#define _pending_skip(must,did) ((must) && !(did)) +#define _done_skipping(must,did) (!_pending_skip(must,did)) +#define DidSkip(x) _done_skipping(must_skip_##x,did_skip_##x) +#define SkipPending(x) _pending_skip(must_skip_##x,did_skip_##x) + +int gzipContentCoder::done_skipping_header() const { + return _done_skipping(must_skip_extra_field, did_skip_extra_field) && + _done_skipping(must_skip_orig_name, did_skip_orig_name) && + _done_skipping(must_skip_comment, did_skip_comment); + /*&& _done_skipping(must_skip_crc16, did_skip_crc16)*/ +} + +int gzipContentCoder::decodeStart(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + debug (93, 4) + ("gzipContentCoder::decodeStart(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + assert(state == NONE); + /* + * must parse gzip header first, so initializde decompressor + * with no input. + */ + header_bytes_read = 0; + trailer_bytes_read = 0; + skip_bytes = -1; // mark it as uninitialized + did_skip_extra_field = 0; + must_skip_extra_field = 0; + did_skip_orig_name = 0; + must_skip_orig_name = 0; + did_skip_comment = 0; + must_skip_comment = 0; + wrote_tail_bytes = 0; + int _zero = 0; + int _unused = 0; + if (deflateContentCoder::decodeStart(ibuff, &_zero, obuff, &_unused) != + CC_OK) { + *ibufflen = *obufflen = 0; + return CC_ERROR; + } + + state = PARSING_HEADER; + return decodeChunk(ibuff, ibufflen, obuff, obufflen); +} + +int gzipContentCoder::decodeChunk(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + /* + * Ugh, I have to check gzip header ... + */ + int skipped = 0; + int ispace = *ibufflen; // unprocessed bytes of ibuff + debug (93, 4) + ("gzipContentCoder::decodeChunk(ibuff=%p, ibufflen=%d, " + "obuff=%p, obufflen=%d)\n", + ibuff, *ibufflen, + obuff, *obufflen); + assert(state == PARSING_HEADER || state == INFLATING); + + if (state == PARSING_HEADER && header_bytes_read < 10) { + int nbytes = min(sizeof(header)-header_bytes_read, *ibufflen); + debug(93, 4) + ("gzipContentCoder::decodeChunk: skipping %d header bytes\n", + nbytes); + xmemcpy(header+header_bytes_read, ibuff, nbytes); + header_bytes_read += nbytes; + *ibufflen = nbytes; + ispace -= nbytes; + ibuff += nbytes; + skipped += nbytes; + + if (header_bytes_read == 10) { // Got full header? + /* I can only deflate, not decompress, de-lzh, + * decrypt, nor deflate splitted files (zlib can't). + * Reserved flags are unsupported as well. + */ + if (method() != DEFLATED || + flags() & (ENCRYPTED|CONTINUATION|RESERVED) != 0) { + state = ERR; + *obufflen = 0; + return CC_ERROR; + } + + if (flags() & EXTRA_FIELD) must_skip_extra_field = 1; + if (flags() & ORIG_NAME) must_skip_orig_name = 1; + if (flags() & COMMENT) must_skip_comment = 1; + } + } + + if (state == PARSING_HEADER) { + // skip extra_field + if (SkipPending(extra_field)) { + if (skip_bytes < 0) { + if (ispace < 2) return CC_OK; // need more bytes + skip_bytes = getWord(ibuff); + debug(93, 4) + ("gzipContentCoder::decodeChunk: must skip %d extra field bytes\n", + skip_bytes); + ibuff += 2; *ibufflen += 2; ispace -= 2; skipped += 2; + } + if (skip_bytes > 0) { + int nbytes = min(ispace, skip_bytes); + debug(93, 4) + ("gzipContentCoder::decodeChunk: skipped %d extra field bytes\n", + skip_bytes); + ispace -= skip_bytes; + *ibufflen += skip_bytes; + ibuff += skip_bytes; + skip_bytes -= nbytes; + skipped += nbytes; + } + if (!skip_bytes) { + skip_bytes = -1; + did_skip_extra_field = 1; + } + } + + if (DidSkip(extra_field) && SkipPending(orig_name)) { + debug(93, 4) + ("gzipContentCoder::decodeChunk: must skip orig_name\n"); + while (ispace && ibuff[0]) { + /* fname is null terminated */ + ++ibuff; ++*ibufflen; --ispace; ++skipped; + } + if (ispace && !ibuff[0]) { + did_skip_orig_name = 1; + debug(93, 4) + ("gzipContentCoder::decodeChunk: finished skipping orig_name\n"); + } + } + + if (DidSkip(orig_name) && SkipPending(comment)) { + debug(93, 4) + ("gzipContentCoder::decodeChunk: must skip comment\n"); + while (ispace && ibuff[0]) { + /* comment is null terminated */ + ++ibuff; ++*ibufflen; --ispace; ++skipped; + } + if (ispace && !ibuff[0]) { + did_skip_comment = 1; + skip_bytes = -1; + debug(93, 4) + ("gzipContentCoder::decodeChunk: finished skipping comment\n"); + } + } + +// if (DidSkip(comment) && SkipPending(crc16)) { +// debug(93, 4) +// ("gzipContentCoder::decodeChunk: must skip crc16\n"); +// if (skip_bytes < 0) skip_bytes = 2; +// while (ispace && skip_bytes) { +// ++ibuff; ++*ibufflen; --ispace; +// } +// if (!skip_bytes) { +// did_skip_crc16 = 1; +// skip_bytes = -1; +// debug(93, 4) +// ("gzipContentCoder::decodeChunk: finish skipping crc16\n"); +// } +// } + } + + if (state == PARSING_HEADER && + header_bytes_read == 10 && + done_skipping_header()) { + debug (93, 4) + ("gzipContentCoder::decodeChunk: header skipped\n"); + state = INFLATING; + } + + *ibufflen = skipped; + + /* need 8 bytes at least for trailer */ + if (ispace <= 8 || state != INFLATING) { + *obufflen = 0; + return CC_OK; + } + + int ospace = *obufflen; + + if (trailer_bytes_read) { + char new_trailer[8]; + xmemcpy(new_trailer, ibuff + ispace - 8, 8); + xmemmove(ibuff + 8, ibuff, ispace - 8); /* FIXME: big memmove */ + xmemcpy(ibuff, trailer, 8); + xmemcpy(trailer, new_trailer, 8); + } else { + xmemcpy(trailer, ibuff + ispace - 8, 8); + ispace -= 8; + *ibufflen += 8; /* FIXME: this should be done if all available input has been processed by deflate */ + trailer_bytes_read = 8; + } + int dev = deflateContentCoder::decodeChunk(ibuff, &ispace, obuff, &ospace); + *ibufflen += ispace; + *obufflen = ospace; + trailer_bytes_read = 8; + if (dev == CC_DONE) { + state = INFLATING; + int _zero = 0; + char buff; + return decodeEnd(&buff, &_zero); + } + return dev; +} + +int gzipContentCoder::decodeEnd(char* obuff, int* obufflen) { + char _null; + int dev = deflateContentCoder::decodeEnd(&_null, obufflen); + switch (dev) { + case CC_PENDING: return dev; + case CC_ERROR: return CC_ERROR; + case CC_DONE: + if ((lengthTrailer() != zlibState.total_out) || + (crc32Trailer() != crc)) { + state = ERR; + return CC_ERROR; + } + state = NONE; + return CC_DONE; + default: + assert(0); + break; + } + assert(0); + return CC_ERROR; +} Index: squid3/src/gzipContentCoder.h diff -u /dev/null squid3/src/gzipContentCoder.h:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/gzipContentCoder.h Thu Aug 31 16:11:38 2006 @@ -0,0 +1,94 @@ +/* + * $Id$ + * + * DEBUG: section 93 HTTP Content-Encoding + * AUTHOR: Gonzalo Arana + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#ifndef __GZIPCONTENTCODER_H_ +#define __GZIPCONTENTCODER_H_ + +#include "deflateContentCoder.h" +#include "MemPool.h" +#include + +class gzipContentCoder : public deflateContentCoder { + + protected: + int header_bytes_sent; + + /* Used for inflation (decompression) */ + int header_bytes_read; + int trailer_bytes_read; + char header[10]; + char trailer[8]; + int skip_bytes; + unsigned long crc; + unsigned long plainBytes; + unsigned short wrote_tail_bytes; + + unsigned did_skip_extra_field: 1; + unsigned must_skip_extra_field: 1; + unsigned did_skip_orig_name: 1; + unsigned must_skip_orig_name: 1; + unsigned did_skip_comment: 1; + unsigned must_skip_comment: 1; +/* unsigned did_skip_crc16: 1; */ +/* unsigned must_skip_crc16: 1; */ + int done_skipping_header() const; + + inline char method() const { return header[2]; } + inline char flags() const { return header[3]; } + int mtime() const; + inline int getWord(char* buff) { return unsigned(buff[0]) | (unsigned(buff[1])<<8); } + + int decodeHeader(char* obuff, int* obufflen); + unsigned lengthTrailer() const { return *(unsigned*)trailer; } + unsigned crc32Trailer() const { return *(unsigned*)&trailer[4]; } + + public: + void* operator new (size_t); + void operator delete(void*); + int encodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen); + int encodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen); + int encodeEnd(char* obuff, int* obufflen); + int decodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen); + int decodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen); + int decodeEnd(char* obuff, int* obufflen); + + private: + acl_access* getEncodeACL(); + CBDATA_CLASS(gzipContentCoder); +/* static MemPool* Pool; */ +}; + +#endif Index: squid3/src/http.cc diff -u squid3/src/http.cc:1.130 squid3/src/http.cc:1.91.2.8 --- squid3/src/http.cc:1.130 Sun Feb 3 02:50:56 2008 +++ squid3/src/http.cc Mon Feb 4 21:37:08 2008 @@ -76,6 +76,10 @@ static void httpMaybeRemovePublic(StoreEntry *, http_status); static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, String strConnection, HttpRequest * request, HttpRequest * orig_request, HttpHeader * hdr_out, int we_do_ranges, http_state_flags); +#ifdef CONTENT_ENCODING +#include "HttpContentCoder.h" +#include "HttpHdrContCode.h" +#endif HttpStateData::HttpStateData(FwdState *theFwdState) : ServerStateData(theFwdState), header_bytes_read(0), reply_bytes_read(0), httpChunkDecoder(NULL) @@ -757,6 +761,10 @@ processSurrogateControl (vrep); +#ifdef CONTENT_ENCODING + /*request->codings->codeInitReply(this, reply);*/ +#endif + /* TODO: IF the reply is a 1.0 reply, AND it has a Connection: Header * Parse the header and remove all referenced headers */ Index: squid3/src/noneContentCoder.cc diff -u /dev/null squid3/src/noneContentCoder.cc:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/noneContentCoder.cc Thu Aug 31 16:11:39 2006 @@ -0,0 +1,111 @@ +/* + * $Id$ + * + * DEBUG: section 28 Access Control + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#include "squid.h" +#include "noneContentCoder.h" + +#if 0 +conflicts with sqiud.h defn +static int min(int x, int y) { + return x < y ? x : y; +} +#endif + +acl_access* +noneContentCoder::getEncodeACL() { + return NULL; //FIXME: what should be done here? +} +CBDATA_CLASS_INIT(noneContentCoder); + +void* +noneContentCoder::operator new(size_t size) { + assert(size = sizeof(noneContentCoder)); + CBDATA_INIT_TYPE(noneContentCoder); + return cbdataAlloc(noneContentCoder); +} + +void +noneContentCoder::operator delete(void* address) { + noneContentCoder* tmp = (noneContentCoder*)address; + cbdataFree(tmp); +} + +noneContentCoder::noneContentCoder() { ; } + +noneContentCoder::~noneContentCoder() { ; } + +int +noneContentCoder::encodeStart(char* ibuff, int *ibufflen, + char* obuff, int* obufflen) { + state = ENCODING; + return encodeChunk(ibuff, ibufflen, obuff, obufflen); +} + +int +noneContentCoder::encodeChunk(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + assert(state == ENCODING); + int ncopy = min(*obufflen, *ibufflen); + xmemcpy((void *) obuff, (void *) ibuff, ncopy); + *obufflen = ncopy; + *ibufflen = ncopy; + return CC_OK; +} + +int +noneContentCoder::encodeEnd(char* obuff, int* obufflen) { + state = NONE; + *obufflen = 0; + return CC_OK; +} + +int +noneContentCoder::decodeStart(char *ibuff, int* ibufflen, + char* obuff, int* obufflen) { + return encodeStart(ibuff, ibufflen, obuff, obufflen); +} + +int +noneContentCoder::decodeChunk(char* ibuff, int* ibufflen, + char* obuff, int* obufflen) { + return encodeChunk(ibuff, ibufflen, obuff, obufflen); +} + +int +noneContentCoder::decodeEnd(char* obuff, int* obufflen) { + return encodeEnd(obuff, obufflen); +} + Index: squid3/src/noneContentCoder.h diff -u /dev/null squid3/src/noneContentCoder.h:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/src/noneContentCoder.h Thu Aug 31 16:11:39 2006 @@ -0,0 +1,72 @@ +/* + * $Id$ + * + * DEBUG: section 28 Access Control + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * + * Copyright (c) 2003, Robert Collins + */ + +#ifndef __NONECONTENTCODER_H_ +#define __NONECONTENTCODER_H_ + +#include "HttpContentCoder.h" +#include "zlib.h" + +class noneContentCoder : public HttpContentCoder { + + public: + enum STATE { NONE = 0, ENCODING, ERR }; + + protected: + enum STATE state; + + public: + + void* operator new(size_t size); + void operator delete(void*); + noneContentCoder(); + ~noneContentCoder(); + int encodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen); + int encodeEnd(char* obuff, int* obufflen); + int encodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen); + int decodeStart(char* ibuff, int *ibufflen, char* obuff, int* obufflen); + int decodeEnd(char* obuff, int* obufflen); + int decodeChunk(char* ibuff, int *ibuff, char* obuff, int* obufflen); + + int totalIn() const { return 0; } + int totalOut() const { return 0; } + + private: + acl_access* getEncodeACL(); + CBDATA_CLASS(noneContentCoder); +}; + +#endif Index: squid3/src/protos.h diff -u squid3/src/protos.h:1.94 squid3/src/protos.h:1.76.2.5 --- squid3/src/protos.h:1.94 Thu Jan 24 11:51:10 2008 +++ squid3/src/protos.h Mon Feb 4 21:37:09 2008 @@ -528,6 +528,7 @@ SQUIDCEXTERN const char *storeKeyText(const cache_key *); SQUIDCEXTERN const cache_key *storeKeyPublic(const char *, const HttpRequestMethod&); SQUIDCEXTERN const cache_key *storeKeyPublicByRequest(HttpRequest *); +SQUIDCEXTERN const cache_key *storeKeyPublicByEncoding(HttpRequest *,const method_t, char encodings[], int); SQUIDCEXTERN const cache_key *storeKeyPublicByRequestMethod(HttpRequest *, const HttpRequestMethod&); SQUIDCEXTERN const cache_key *storeKeyPrivate(const char *, const HttpRequestMethod&, int); SQUIDCEXTERN int storeKeyHashBuckets(int); Index: squid3/src/stat.cc diff -u squid3/src/stat.cc:1.47 squid3/src/stat.cc:1.32.2.5 --- squid3/src/stat.cc:1.47 Mon Jan 7 07:51:08 2008 +++ squid3/src/stat.cc Mon Feb 4 21:37:09 2008 @@ -1007,6 +1007,15 @@ storeAppendPrintf(sentry, "syscalls.sock.recvfroms = %f/sec\n", XAVG(syscalls.sock.recvfroms)); storeAppendPrintf(sentry, "syscalls.sock.sendtos = %f/sec\n", XAVG(syscalls.sock.sendtos)); +#ifdef CONTENT_ENCODING + storeAppendPrintf(sentry, "libcalls.zlib.inflateInit = %f/sec\n", XAVG(libcalls.zlib.inflateInit)); + storeAppendPrintf(sentry, "libcalls.zlib.deflateInit = %f/sec\n", XAVG(libcalls.zlib.deflateInit)); + storeAppendPrintf(sentry, "libcalls.zlib.inflate = %f/sec\n", XAVG(libcalls.zlib.inflate)); + storeAppendPrintf(sentry, "libcalls.zlib.deflate = %f/sec\n", XAVG(libcalls.zlib.deflate)); + storeAppendPrintf(sentry, "libcalls.zlib.inflateEnd = %f/sec\n", XAVG(libcalls.zlib.inflateEnd)); + storeAppendPrintf(sentry, "libcalls.zlib.deflateEnd = %f/sec\n", XAVG(libcalls.zlib.deflateEnd)); +#endif + storeAppendPrintf(sentry, "cpu_time = %f seconds\n", ct); storeAppendPrintf(sentry, "wall_time = %f seconds\n", dt); storeAppendPrintf(sentry, "cpu_usage = %f%%\n", dpercent(ct, dt)); Index: squid3/src/store.cc diff -u squid3/src/store.cc:1.67 squid3/src/store.cc:1.46.2.6 --- squid3/src/store.cc:1.67 Sun Jan 20 01:50:57 2008 +++ squid3/src/store.cc Mon Feb 4 21:37:09 2008 @@ -50,6 +50,10 @@ #endif #include "Stack.h" #include "SquidTime.h" +#ifdef CONTENT_ENCODING +#include "HttpContentCoder.h" +#include "HttpHdrContCode.h" +#endif static STMCB storeWriteComplete; @@ -544,6 +548,34 @@ } void +StoreEntry::getPublicByEncoding (StoreClient *aClient, HttpRequest * request) +{ + StoreEntry *result; + + assert (aClient); + +#ifdef CONTENT_ENCODING + HttpHdrContCode *codings = request->codings; + if (codings) + result = Store::Root().get(storeKeyPublicByEncoding(request, request->method, + codings->aelist, + codings->naecoders)); + else + result = NULL; + if (!result) { +#endif + result = storeGetPublicByRequest (request); +#ifdef CONTENT_ENCODING + } +#endif + + if (!result) + result = NullStoreEntry::getInstance(); + + aClient->created (result); +} + +void StoreEntry::getPublic (StoreClient *aClient, const char *uri, const HttpRequestMethod& method) { assert (aClient); @@ -631,6 +663,10 @@ { StoreEntry *e2 = NULL; const cache_key *newkey; +#ifdef CONTENT_ENCODING + int ncodings; + char *codelist = NULL; +#endif if (key && !EBIT_TEST(flags, KEY_PRIVATE)) return; /* is already public */ @@ -683,6 +719,14 @@ } } +#ifdef CONTENT_ENCODING + /* Content-Encoding header changes key to use */ + if (mem_obj->getReply() != NULL && request->method == METHOD_GET) { + codelist = CodeList((HttpHeader *) &mem_obj->getReply()->header, + &ncodings, HDR_CONTENT_ENCODING); + } +#endif + if (mem_obj->vary_headers && !storeGetPublic(mem_obj->url, mem_obj->method)) { /* Create "vary" base object */ String vary; @@ -721,6 +765,11 @@ } newkey = storeKeyPublicByRequest(mem_obj->request); +#ifdef CONTENT_ENCODING + } else if (codelist != NULL) { + newkey = storeKeyPublicByEncoding(mem_obj->request, mem_obj->method, + codelist, ncodings); +#endif } else newkey = storeKeyPublic(mem_obj->url, mem_obj->method); Index: squid3/src/store_key_md5.cc diff -u squid3/src/store_key_md5.cc:1.12 squid3/src/store_key_md5.cc:1.7.2.3 --- squid3/src/store_key_md5.cc:1.12 Sun Feb 3 02:50:57 2008 +++ squid3/src/store_key_md5.cc Mon Feb 4 21:37:10 2008 @@ -151,6 +151,27 @@ return digest; } +const cache_key * +storeKeyPublicByEncoding(HttpRequest * request, const method_t method, + char encodings[], int nencodings) +{ + static cache_key digest[MD5_DIGEST_CHARS]; + unsigned char m = (unsigned char) method; + const char *url = urlCanonical(request); + MD5_CTX M; + MD5Init(&M); + MD5Update(&M, &m, sizeof(m)); + MD5Update(&M, (unsigned char *) url, strlen(url)); + MD5Update(&M, (unsigned char *) encodings, nencodings * sizeof(encodings[0])); + + if (request->vary_headers) + MD5Update(&M, (unsigned char *) request->vary_headers, strlen(request->vary_headers)); + + MD5Final(digest, &M); + + return digest; +} + cache_key * storeKeyDup(const cache_key * key) { Index: squid3/src/structs.h diff -u squid3/src/structs.h:1.126 squid3/src/structs.h:1.99.2.7 --- squid3/src/structs.h:1.126 Wed Jan 23 13:51:00 2008 +++ squid3/src/structs.h Mon Feb 4 21:37:10 2008 @@ -585,7 +585,11 @@ acl_access *htcp; acl_access *htcp_clr; #endif - +#if CONTENT_ENCODING + acl_access *negotiate_ce; + acl_access *encode_gzip; + acl_access *encode_deflate; +#endif } accessList; @@ -1541,6 +1545,28 @@ } swap; + +#ifdef CONTENT_ENCODING + struct + { + + struct + { + int inflate; + int deflate; + int deflateInit; + int inflateInit; + int deflateEnd; + int inflateEnd; + } + + zlib; + + //StatHist deflate_ratios; + } + + libcalls; +#endif }; /* per header statistics */ Index: squid3/test-suite/Makefile.am diff -u squid3/test-suite/Makefile.am:1.18 squid3/test-suite/Makefile.am:1.17.2.2 --- squid3/test-suite/Makefile.am:1.18 Thu Jan 4 13:52:48 2007 +++ squid3/test-suite/Makefile.am Wed Feb 28 02:43:10 2007 @@ -46,7 +46,9 @@ splay \ StackTest \ syntheticoperators \ - VirtualDeleteOperator + VirtualDeleteOperator \ + encode \ + encoding_remote LDADD =$(top_builddir)/src/globals.o $(top_builddir)/src/time.o -L$(top_builddir)/lib -lmiscutil @@ -91,3 +93,10 @@ ## ##$(TARGLIB): $(LIBOBJS) ## $(AR_R) $(TARGLIB) $(LIBOBJS) + +encode_SOURCES = encode.cc ../src/gzipContentCoder.cc ../src/deflateContentCoder.cc $(DEBUG_SOURCE) +encode_LDADD = -lz $(LDADD) + +encoding_remote_SOURCES = encoding_remote.cc ../src/gzipContentCoder.cc ../src/deflateContentCoder.cc $(DEBUG_SOURCE) +encoding_remote_LDADD = -lz $(LDADD) + Index: squid3/test-suite/ce-local.sh diff -u /dev/null squid3/test-suite/ce-local.sh:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/test-suite/ce-local.sh Thu Aug 31 16:11:44 2006 @@ -0,0 +1,22 @@ +#!/bin/sh + +# No output should be generated by commands +# If any output is seen, there must be an error. + +# $Id$ + +FILE=$1 + +if [ ! -f "${FILE}" ]; then + echo Usage: `basename $0` file + echo Where file is any readable file + exit 1 +fi + +echo Testing compression against gzip +./encode -g -c <${FILE} | gunzip -dc | diff -bu - ${FILE} +echo Testing decompression against gzip +gzip -c <${FILE} | ./encode -g -d | diff -bu - ${FILE} +echo Testing deflate against itself +./encode -d -c <${FILE} | ./encode -d -d | diff -bu - ${FILE} + Index: squid3/test-suite/encode.cc diff -u /dev/null squid3/test-suite/encode.cc:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/test-suite/encode.cc Thu Aug 31 16:11:44 2006 @@ -0,0 +1,191 @@ +/* + * $Id$ + * + * AUTHOR: Gonzalo Arana + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * Copyright (c) 2003 Robert Collins + */ + +extern "C" { +#include +#include +#include +#include +}; + +#include "gzipContentCoder.h" +#include "deflateContentCoder.h" + +extern "C" { +#undef assert +#include +}; + +void compress(HttpContentCoder& coder, int ifd, int ofd, int debug); +void decompress(HttpContentCoder& coder, int ifd, int ofd, int debug); +time_t squid_curtime = 0; + + +int main(int argc, char* argv[]) { + int debug = 0; + gzipContentCoder gzipCoder; + deflateContentCoder deflateCoder; + HttpContentCoder* coder = NULL; + + /* FIXME: use getopt */ + if (argc != 3) { + fprintf(stderr, "Usage: %s -[dg] -[cd] output_file\n" + " First argument is algorithm: 'd': deflate, 'g': gzip\n" + " Second argument is action: 'c': inflate, 'd': deflate\n", + argv[0]); + return 0; + } + + if (argv[1][1] == 'd') coder = &deflateCoder; + else if (argv[1][1] == 'g') coder = &gzipCoder; + + if (argv[2][1] == 'd') decompress(*coder, 0, 1, debug); + else if (argv[2][1] == 'c') compress(*coder, 0, 1, debug); + + return 0; +} + +/* Decompresion */ +void decompress(HttpContentCoder& coder, int ifd, int ofd, int debug) { + char ichar; + char ochar; + int olen = 1; + int ilen = 0; + int nin = 0; + int nout = 0; + int status; + status = coder.decodeStart(&ichar, &nin, &ochar, &olen); + assert(status == CC_OK); + if (olen) { write(1, &ochar, 1); ++nout; } + if (read(0, &ichar, 1)) { + ++nin; + olen = ilen = 1; + while (1) { + status = coder.decodeChunk(&ichar, &ilen, &ochar, &olen); + assert(status == CC_OK || status == CC_DONE); + if (olen) { + ++nout; + write(1, &ochar, 1); + } + olen = 1; + if (ilen) { + if (read(0, &ichar, 1) != 1) { break; } /* done with input */ + ++nin; + } + ilen = 1; + if (debug && !(coder.totalIn() % 10000)) { + fprintf(stderr, "in=%d(%d) out=%d(%d)\n", coder.totalIn(), nin, + coder.totalOut(), nout); + } + if (status == CC_DONE) break; + } + } + /* now write the rest of output data */ + while (1) { + olen = 1; + status = coder.decodeEnd(&ochar, &olen); + if (olen) { + write(1, &ochar, 1); + ++nout; + } + if (status == CC_OK) break; + if (status != CC_PENDING) { + fprintf(stderr, "Error\n"); + break; + } + if (debug && !(coder.totalOut() % 10000)) { + fprintf(stderr, "in=%d(%d) out=%d(%d)\n", coder.totalIn(), nin, + coder.totalOut(), nout); + } + } + if (debug) + fprintf(stderr, "in=%d(%d) out=%d(%d)\n", coder.totalIn(), nin, + coder.totalOut(), nout); +} + +/* Compression */ +void compress(HttpContentCoder& coder, int ifd, int ofd, int debug) { + char ichar; + char ochar; + int olen = 1; + int ilen = 0; + int status; + int nin = 0; + int nout = 0; + status = coder.encodeStart(&ichar, &ilen, &ochar, &olen); + assert(status == CC_OK); + if (olen) { write(1, &ochar, 1); ++nout; } + olen = 1; + if (read(0, &ichar, 1)) { + ilen = 1; + while (1) { + status = coder.encodeChunk(&ichar, &ilen, &ochar, &olen); + if (ilen) { + if (read(0, &ichar, 1) != 1) { break; } /* done with input */ + ++nin; + } + ilen = 1; + assert(status == CC_OK); + if (olen) { + write(1, &ochar, 1); + ++nout; + } + olen = 1; + if (debug && !(coder.totalIn() % 10000)) { + fprintf(stderr, "in=%d(%d) out=%d(%d)\n", coder.totalIn(), nin, + coder.totalOut(), nout); + } + } + } + + /* need to write rest of output data */ + while (1) { + olen = 1; + status = coder.encodeEnd(&ochar, &olen); + if (olen) { + write(1, &ochar, 1); + ++nout; + } + if (status == CC_OK) break; + assert(status != CC_ERROR); + assert(status == CC_PENDING); + if (debug && !(coder.totalOut() % 10000)) { + fprintf(stderr, "in=%d(%d) out=%d(%d)\n", coder.totalIn(), nin, + coder.totalOut(), nout); + } + } + if (debug) + fprintf(stderr, "in=%d(%d) out=%d(%d)=\n", coder.totalIn(), nin, + coder.totalOut(), nout); +} Index: squid3/test-suite/encoding_remote.cc diff -u /dev/null squid3/test-suite/encoding_remote.cc:1.1.2.1 --- /dev/null Thu Jan 1 01:00:00 1970 +++ squid3/test-suite/encoding_remote.cc Thu Aug 31 16:11:45 2006 @@ -0,0 +1,515 @@ +/* + * $Id$ + * + * DEBUG: section 19 Store Memory Primitives + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * Copyright (c) 2003 Robert Collins + */ + +/* + * Test program to validate content-encoding desition & compression algorithm. + */ + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +}; + +#include "gzipContentCoder.h" + +struct testControl { + const char* description; /* textual description of test */ + const char* path; /* path in request */ + int send_ae; /* send Accept-Encoding in request */ + int send_nt; /* send Cache-Control: no-transform in request */ + int send_gzip; /* send gzipped content or not in reply */ + int expect_gz; /* reply read from squid should be gziped */ + int send_nc; /* append no-cache directive in Cache-Control */ +}; + +struct testInfo { + char host_header[128]; + struct sockaddr_in remote, local; + char sremote[32], slocal[32]; +}; + +const char USAGE[] = +"encoding-remote -l -p \n"; + +char PLAIN_BODY[1024]; +int plain_blen; +char ZIPPED_BODY[1024]; +int zipped_blen; + +int inetaddr_aton(const char* a, struct sockaddr_in* n) { + char ip[16]; + n->sin_family = AF_INET; + if (sscanf(a, "%15[^:]:%hu", ip, &n->sin_port) != 2) + return -1; + n->sin_port = htons(n->sin_port); + if (!inet_aton(ip, &n->sin_addr)) + return -2; + return 0; +} + +char* inetaddr_ntoa(struct sockaddr_in* n) { + static char ibuff[22]; + sprintf(ibuff, "%s:%hu", inet_ntoa(n->sin_addr), ntohs(n->sin_port)); + return ibuff; +} + +int parse_args(struct sockaddr_in* local, + struct sockaddr_in* remote, + int argc, char* argv[]) { + const char* slocal = NULL; + const char* sremote = NULL; + char c; + while ((c = getopt(argc, argv, "hl:p:")) != -1) { + switch (c) { + case 'l': + slocal = optarg; + break; + case 'p': + sremote = optarg; + break; + case 'h': + printf(USAGE); + exit(0); + break; + case '?': + break; + default: + fprintf(stderr, "Unknown option %c\n", c); + exit(1); + } + } + + if (!slocal) { + fprintf(stderr, "Must define local endpoint (ip:port)\n"); + return 1; + } + if (!sremote) { + fprintf(stderr, "Must define squid endpoint (ip:port)\n"); + return 2; + } + + if (inetaddr_aton(slocal, local)) { + printf("Error: invalid local endpoint \"%s\"\n", slocal); + return 3; + } + + if (inetaddr_aton(sremote, remote)) { + printf("Error: invalid remote endpoint: \"%s\"\n", sremote); + return 4; + } + + return 0; +} + +int gzipCoderCompress(char* ZIPPED_BODY, int zipped_blen, + char* PLAIN_BODY, int plain_blen) { + int zipped_size = zipped_blen; + gzipContentCoder gzip; + gzip.encodeStart(PLAIN_BODY, &plain_blen, ZIPPED_BODY, &zipped_blen); + assert(plain_blen == 0); // insufficient output memory + gzip.encodeEnd(ZIPPED_BODY + zipped_blen, &zipped_blen); + return zipped_size - zipped_blen; +} + +int gzipCoderDecompress(char* ZIPPED_BODY, int zipped_blen, + char* PLAIN_BODY, int plain_blen) { + int plain_size = plain_blen; + gzipContentCoder gzip; + if (gzip.decodeStart(ZIPPED_BODY, &zipped_blen, PLAIN_BODY, &plain_blen) != + CC_OK) return 0; + assert(plain_blen == 0); + gzip.decodeEnd(PLAIN_BODY + plain_blen, &plain_blen); + return plain_size - plain_blen; +} + +int build_request(char* obuff, const char* host_header, + struct testControl* control) { + static char ae[] = "Accept-Encoding: gzip\r\n"; + static const char* cache_controlv[4] = { + "", + "Cache-Control: no-cache\r\n", + "Cache-Control: no-transform\r\n", + "Cache-Control: no-transform, no-cache\r\n" + }; + const char *cache_control = + cache_controlv[(control->send_nt << 1) | control->send_nc]; + + /* request adds time in seconds in order to force allways a miss */ + sprintf(obuff, + "GET http://%s%s HTTP/1.1\r\n" + "Host: %s\r\n" + "%s" + "%s" + "Proxy-Connection: close\r\n" + "\r\n", + host_header, control->path, + host_header, + control->send_ae ? ae : "", + cache_control); + return strlen(obuff); +} + +/* + * Fills a piece of memory with random data. + */ +int memrnd(char* obuff, int obufflen) { + // compressable base + static char base[] = "aaabbbccceeedddlllkkkjjjaaaabbb"; + int nbase = strlen(base); + int i = 0; + assert(obufflen > nbase); + for (i = 0; i < obufflen - nbase; i += nbase) + xmemcpy(obuff + i, base, nbase); + return i; +// int fd = open("/dev/urandom", O_RDONLY); +// if (fd < 0) +// for (int i = 0; +// i < int(obufflen - sizeof(int)); +// ++i) +// *(int*)(obuff++) ^= rand(); +// else { +// read(fd, obuff, obufflen); +// close(fd); +// } +// return obufflen; +} + +/* + * Fills a piece of memory with compressed random data + */ +int memzrnd(char* obuff, int obufflen) { + static char tmpbuf[1024]; + int zero = 0; + int itmpbuf = sizeof(tmpbuf); + gzipContentCoder gz; + int dev = 0; + gz.encodeStart(obuff, &zero, obuff, &dev); + memrnd(tmpbuf, sizeof(tmpbuf)); + obufflen -= dev; + while (itmpbuf) { + int nbuff = obufflen; + gz.encodeChunk(tmpbuf + sizeof(tmpbuf) - itmpbuf, &itmpbuf, + obuff + dev, &obufflen); + dev += nbuff - obufflen; + } + int nbuff = obufflen; + gz.encodeEnd(obuff + dev, &obufflen); + dev += nbuff - obufflen; + return dev; +} + +int copy_reply_body(char* dst, int dstlen, char* src, int srclen) { + if (srclen > dstlen) { + printf("%s: %d -> %d\n", + "ERROR: Insufficient output space to copy reply body", + srclen, dstlen); + exit(1); + } + xmemcpy(dst, src, srclen); + return srclen; +} + +int build_reply(char* obuff, int obufflen, const char* host_header, + struct testControl* control) { + int dev; + sprintf(obuff, + "HTTP/1.1 200 OK\r\n" + "Host: %s\r\n" + "Connection: close\r\n" + "%s" + "\r\n", + host_header, + control->send_gzip ? "Content-Encoding: gzip\r\n" : ""); + dev = strlen(obuff); + obuff += dev; + obufflen -= dev; + plain_blen = memrnd(PLAIN_BODY, sizeof(PLAIN_BODY)); + zipped_blen = sizeof(ZIPPED_BODY); + zipped_blen = gzipCoderCompress(ZIPPED_BODY, zipped_blen, + PLAIN_BODY, plain_blen); + if (control->send_gzip) { + dev += copy_reply_body(obuff, obufflen, ZIPPED_BODY, zipped_blen); + } else { + dev += copy_reply_body(obuff, obufflen, PLAIN_BODY, plain_blen); + } + return dev; +} + +void add_fd(int fd, fd_set* in, fd_set* out, fd_set* err) { + if (fd < 0) return; + if (in) FD_SET(fd, in); + if (out) FD_SET(fd, out); + if (err) FD_SET(fd, err); +} + +int xnor(int x, int y) { + return (x && y) || (!x && !y); +} + +#define HAS_CE_HEADER 0x01 +#define HAS_GZIP_CE 0x02 +#define CONTENT_VALID_GZIP 0x04 +#define GZ_STATUS_MASK 0x07 + +const char* gzipped_status_ntoa(int st) { + static const char* statusv[] = { + "(no content-encoding header nor compressed content)", + "(CE_HEADER)", + "(HAS_GZIP_CE)", + "(CE_HEADER|HAS_GZIP_CE)", + "(CONTENT_VALID_GZIP)", + "(CONTENT_VALID_GZIP|CE_HEADER)", + "(CONTENT_VALID_GZIP|HAS_GZIP_CE)", + "(CONTENT_VALID_GZIP|CE_HEADER|CONTENT_GZIP)", + }; + return statusv[st & GZ_STATUS_MASK]; +} + +/* + * Since content might be compressed by squid, i have to decompress it + * and verify uncompressed content. + */ +int reply_is_gzipped(char* buff, int ibuff) { + int dev = 0; + char* body = strstr(buff, "\r\n\r\n"); + if (!body) { + printf(" (Response does not have a body) "); + return 0; + } + + body[0] = 0; + body += 4; + + char* header = strstr(buff, "Content-Encoding: "); + if (!header) return 0; + char* endh = strstr(header, "\r\n"); + if (!endh) { + printf(" (Response has invalid header) "); + return 0; + } + dev |= HAS_CE_HEADER; + endh[0] = 0; + static char UNZIPPED[1024]; + if ((gzipCoderDecompress(UNZIPPED, sizeof(UNZIPPED), (endh+1), + ibuff - (endh+1-buff)) == plain_blen) && + !memcmp(PLAIN_BODY, UNZIPPED, plain_blen)) + dev |= CONTENT_VALID_GZIP; + dev |= strstr(header, "gzip") ? HAS_GZIP_CE : 0; + return dev; +} + +//TODO: add q=x.y in Accept-Encoding +//TODO: validate connection re-use +/* + * RunTest: runs a complete test: + * 1) connects to a squid using clifd + * 2) binds using servfd + * 3) issue a request to squid + * 4) squid connects to servfd + * 5) connfd = accept() + * 6) sends to connfd the reply as directed in control + * 7) reads reply from squid + * 8) validates the reply as directed in control + */ +int RunTest (struct testInfo* info, + struct testControl* control) { + static char ireq[4096]; int nireq = 0; int direq = 0; + static char irep[4096]; int nirep = 0; int direp = 0; + static char oreq[4096]; int noreq = 0; int doreq = 0; int ioreq = 0; + static char orep[4096]; int norep = 0; int dorep = 0; int iorep = 0; + struct sockaddr peer; + socklen_t peer_addr_len = sizeof(peer); + /* + * ireq <-- newfd + * irep <-- clifd + * orep --> newfd + * oreq --> clifd + */ + int clifd = socket(PF_INET, SOCK_STREAM, 0); + int serfd = socket(PF_INET, SOCK_STREAM, 0); + int newfd = -1; + int nfds = 1; + int dev = 0; + setsockopt(serfd, SOL_SOCKET, SO_REUSEADDR, &nfds, sizeof(nfds)); + nfds = serfd+1; + printf("Test %s (no-cache: %d): ", control->description, control->send_nc); + if (bind(serfd, (struct sockaddr*)&info->local, sizeof(info->local))) { + printf("Error: %s\n", strerror(errno)); + dev = errno; + goto close; + } + if (listen(serfd, 1)) { + printf("Error: %s\n", strerror(errno)); + dev = errno; + goto close; + } + + noreq = build_request(oreq, info->host_header, control); + norep = build_reply(orep, sizeof(orep)/2, info->host_header, control); + + if (connect(clifd, (struct sockaddr*)(&info->remote), + sizeof(info->remote))) { + printf("Error: Can't connect to \"%s\": %s\n", + inetaddr_ntoa(&info->remote), strerror(errno)); + dev = errno; + goto close; + } + + /* Send request, wait for connection from squid, read request, send reply + and read reaply */ + while (clifd >= 0) { + fd_set in, out, err; + FD_ZERO(&in); + FD_ZERO(&out); + FD_ZERO(&err); + struct timeval timeout = { 1, 0 }; + add_fd(newfd, direq ? NULL : &in, direq && !dorep ? &out : NULL, &err); + add_fd(clifd, &in, &out, &err); + add_fd(serfd, &in, &out, &err); + select(nfds, &in, &out, &err, &timeout); + + /* serfd: accept new connection */ + if (serfd >= 0 && FD_ISSET(serfd, &in)) { + newfd = accept(serfd, &peer, &peer_addr_len); + //puts("new connection"); + close(serfd); + serfd = -1; + nfds = newfd + 1; + } + + /* newfd: read request and send reply */ + if (newfd >= 0 && !direq && FD_ISSET(newfd, &in)) { + int nread = read(newfd, ireq + nireq, sizeof(ireq)-nireq); + //printf("read req from squid: %d\n", nread); + nireq += nread; + if (!nread || strstr(ireq, "\n\n") || strstr(ireq, "\r\n\r\n")) { + direq = 1; + ireq[nireq] = 0; + } + } + + if (newfd >= 0 && !dorep && FD_ISSET(newfd, &out)) { + int nwrite = write(newfd, orep + iorep, norep - iorep); + //printf("write rep to squid: %d\n", nwrite); + iorep += nwrite; + if (iorep >= norep) { + dorep = 1; + close(newfd); + newfd = -1; + } + } + + /* clifd: send request and read reply */ + if (clifd >= 0 && !doreq && FD_ISSET(clifd, &out)) { + int nwrite = write(clifd, oreq + ioreq, noreq - ioreq); + //printf("write req to squid: %d\n", nwrite); + ioreq += nwrite; + if (ioreq >= noreq) doreq = 1; + } + + if (clifd >= 0 && !direp && FD_ISSET(clifd, &in)) { + int nread = read(clifd, irep + nirep, sizeof(irep) - nirep); + nirep += nread; + if (!nread) { + direp = 1; + close(clifd); + clifd = -1; + } + } + } + + close: + close(serfd); + close(clifd); + close(newfd); + if (!dev) { + int isgz = reply_is_gzipped(irep, nirep); + dev = !xnor(control->expect_gz, (isgz & GZ_STATUS_MASK) == GZ_STATUS_MASK); + if (dev) + printf("Failed: %s\n", gzipped_status_ntoa(isgz)); + else puts("OK"); + } + + return dev; +} + +struct testControl tests[] = { + { "plain -> plain", "/plain/plain", 0, 0, 0, 0 }, + { "gzipped -> plain", "/gzipped/plain", 0, 0, 1, 0 }, + { "gzipped -> gzipped (web server gzip)", "/gzip/gzip/ws", 1, 0, 1, 1 }, + { "plain -> gzipped", "/plain/gzip", 1, 0, 0, 1 }, + { "plain -> plain (no-transform)", "/plain/plain/no-transform", 0, 1, 0, 0, }, + { "tricky one", "/tricky/one", 0, 1, 1, 0, }, //GUAT!? + { "plain -> gzipped (squid gzip)", "/plain/gzip/by/squid", 1, 1, 0, 1 }, + { "gzipped -> gzipped (web server gzip)", "/gzip/gzip/ws/cc", 1, 1, 1, 1 } +}; +#define NTESTS (sizeof(tests)/sizeof(tests[0])) +int squid_curtime; + +int main(int argc, char* argv[]) { + struct testInfo info; + memset(&info, 0, sizeof(info)); + int dev = parse_args(&info.local, &info.remote, argc, argv); + if (dev) return dev; + strcpy(info.host_header, inetaddr_ntoa(&info.local)); + + dev = 0; + for (int i = 0; i < int(NTESTS); ++i) { + tests[i].send_nc = 0; + if (RunTest(&info, tests+i)) { + dev |= 1 << i; + } + } + for (int i = 0; i < int(NTESTS); ++i) { + tests[i].send_nc = 1; + if (RunTest(&info, tests+i)) { + dev |= 1 << (i+NTESTS); + } + } + + return dev; +} +