This patch is generated from the te branch of HEAD in squid Mon Jan 26 12:58:52 2004 GMT See http://devel.squid-cache.org/ Index: squid/acconfig.h diff -u squid/acconfig.h:1.6 squid/acconfig.h:1.1.1.3.8.7 --- squid/acconfig.h:1.6 Fri Mar 9 16:58:29 2001 +++ squid/acconfig.h Wed Mar 21 02:09:41 2001 @@ -276,6 +276,11 @@ #undef HEAP_REPLACEMENT /* + * Use the AppliedTheory compression/decompression library + */ +#undef ATHY_COMPRESSION + +/* * message type for message queues */ #undef mtyp_t Index: squid/configure.in diff -u squid/configure.in:1.19 squid/configure.in:1.1.1.3.8.13 --- squid/configure.in:1.19 Fri Mar 9 16:58:29 2001 +++ squid/configure.in Wed Mar 21 02:09:41 2001 @@ -674,6 +674,18 @@ fi ]) +dnl With ATHY_COMPRESSION +AC_ARG_WITH(athy-compression, +[ --with-athy-compression + This option makes use of a AppliedTheory compression/ + decompression library to handle compressed transfer + encodings. + ], +[ if test "$enableval" = "yes" ; then + echo "Enabling ATHY_COMPRESSION" + AC_DEFINE(ATHY_COMPRESSION, 1) + fi +]) dnl Select auth schemes modules to build AC_ARG_ENABLE(auth, Index: squid/doc/debug-sections.txt diff -u squid/doc/debug-sections.txt:1.3 squid/doc/debug-sections.txt:1.1.1.2.8.3 --- squid/doc/debug-sections.txt:1.3 Sun Jan 7 15:53:36 2001 +++ squid/doc/debug-sections.txt Mon Feb 5 04:06:36 2001 @@ -86,3 +86,4 @@ section 79 HTTP Meter Header section 80 WCCP section 81 Store Removal/Replacement policy +section 82 Transfer-encoding Routines Index: squid/include/version.h diff -u squid/include/version.h:1.4 squid/include/version.h:1.1.1.3.4.1.4.4 --- squid/include/version.h:1.4 Tue Nov 14 05:03:47 2000 +++ squid/include/version.h Wed Jan 10 05:09:17 2001 @@ -4,7 +4,7 @@ * SQUID_VERSION - String for version id of this distribution */ #ifndef SQUID_VERSION -#define SQUID_VERSION "2.5.DEVEL" +#define SQUID_VERSION "2.5.DEVEL-te" #endif #ifndef SQUID_RELEASE_TIME Index: squid/src/HttpHeader.c diff -u squid/src/HttpHeader.c:1.5 squid/src/HttpHeader.c:1.1.1.3.8.5 --- squid/src/HttpHeader.c:1.5 Fri Jan 12 00:20:32 2001 +++ squid/src/HttpHeader.c Mon Feb 5 04:20:39 2001 @@ -111,7 +111,9 @@ {"Retry-After", HDR_RETRY_AFTER, ftStr}, /* for now (ftDate_1123 or ftInt!) */ {"Server", HDR_SERVER, ftStr}, {"Set-Cookie", HDR_SET_COOKIE, ftStr}, + {"TE",HDR_TE, ftStr}, {"Title", HDR_TITLE, ftStr}, + {"Transfer-Encoding", HDR_TRANSFER_ENCODING,ftStr}, {"Upgrade", HDR_UPGRADE, ftStr}, /* for now */ {"User-Agent", HDR_USER_AGENT, ftStr}, {"Vary", HDR_VARY, ftStr}, /* for now */ @@ -145,7 +147,9 @@ HDR_IF_MATCH, HDR_IF_NONE_MATCH, HDR_LINK, HDR_PRAGMA, HDR_PROXY_CONNECTION, - /* HDR_TRANSFER_ENCODING, */ + /* HDR_EXPECT */ + HDR_TE, + HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_VARY, HDR_VIA, @@ -153,7 +157,7 @@ HDR_WWW_AUTHENTICATE, HDR_AUTHENTICATION_INFO, HDR_PROXY_AUTHENTICATION_INFO, - /* HDR_EXPECT, HDR_TE, HDR_TRAILER */ + /* HDR_TRAILER */ HDR_X_FORWARDED_FOR }; @@ -161,7 +165,7 @@ static http_hdr_type GeneralHeadersArr[] = { HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA, - /* HDR_TRANSFER_ENCODING, */ + HDR_TRANSFER_ENCODING, HDR_UPGRADE, /* HDR_TRAILER, */ HDR_VIA @@ -197,6 +201,7 @@ HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH, HDR_IF_RANGE, HDR_MAX_FORWARDS, HDR_PROXY_CONNECTION, HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE, + HDR_TE, HDR_USER_AGENT, HDR_X_FORWARDED_FOR }; Index: squid/src/HttpHeaderTools.c diff -u squid/src/HttpHeaderTools.c:1.6 squid/src/HttpHeaderTools.c:1.1.1.3.8.5 --- squid/src/HttpHeaderTools.c:1.6 Wed Feb 7 11:11:47 2001 +++ squid/src/HttpHeaderTools.c Tue Feb 13 06:41:14 2001 @@ -190,6 +190,38 @@ return 0; } +int +strListIsMember_q(const String * list, const char *m, char del) +{ + int rv; + const char *pos = NULL; + const char *item, *l; + int matchsize, len_m; + + assert(list && m); + len_m = strlen(m); + rv = 1000; + + while (strListGetItem(list, del, &item, &matchsize, &pos)) { + if (((l = strchr(item, ';'))) && ((l - item) < matchsize)) { + matchsize = l - item; + /* rtrim again */ + for (l--; (l >= item) && xisspace(*l); matchsize--); + l = strchr(item + matchsize, 'q'); + if (l) + l++; + for (; l && *l && xisspace(*l); l++); + if (l && (*l == '=')) + rv = ((double) atof(l + 1)) * 1000; + + } + + if ((len_m == matchsize) && (!strncasecmp(item, m, matchsize))) + return rv; + } + return 0; +} + /* returns true iff "s" is a substring of a member of the list */ int strListIsSubstr(const String * list, const char *s, char del) Index: squid/src/HttpMsg.c diff -u squid/src/HttpMsg.c:1.6 squid/src/HttpMsg.c:1.1.1.1.14.4 --- squid/src/HttpMsg.c:1.6 Fri Jan 12 00:20:32 2001 +++ squid/src/HttpMsg.c Mon Feb 5 04:20:39 2001 @@ -88,15 +88,19 @@ /* returns true if connection should be "persistent" * after processing this message */ +/* return 0 or one only: we are used in a flag */ int httpMsgIsPersistent(http_version_t http_ver, const HttpHeader * hdr) { if ((http_ver.major >= 1) && (http_ver.minor >= 1)) { + int rv; /* * for modern versions of HTTP: persistent unless there is * a "Connection: close" header. */ - return !httpHeaderHasConnDir(hdr, "close"); + rv=((!httpHeaderHasConnDir(hdr, "close")) ? 1: 0); + debug(74,1)("httpMsgIsPersistent: HTTP 1.1 or greater, returning %d\n",rv); + return rv; } else { /* * Persistent connections in Netscape 3.x are allegedly broken, @@ -111,6 +115,6 @@ return 0; } /* for old versions of HTTP: persistent if has "keep-alive" */ - return httpHeaderHasConnDir(hdr, "keep-alive"); + return httpHeaderHasConnDir(hdr, "keep-alive") ? 1:0; } } Index: squid/src/Makefile.in diff -u squid/src/Makefile.in:1.7 squid/src/Makefile.in:1.1.1.3.8.10 --- squid/src/Makefile.in:1.7 Sat Feb 10 08:49:03 2001 +++ squid/src/Makefile.in Tue Feb 13 06:41:14 2001 @@ -176,6 +176,7 @@ store_swapout.o \ string_arrays.o \ tools.o \ + transfer-encoding.o \ @UNLINKD_OBJS@ \ url.o \ urn.o \ Index: squid/src/client_side.c diff -u squid/src/client_side.c:1.24 squid/src/client_side.c:1.1.1.3.4.1.4.19 --- squid/src/client_side.c:1.24 Sun Mar 4 05:45:04 2001 +++ squid/src/client_side.c Wed Mar 21 02:09:43 2001 @@ -66,7 +66,6 @@ #include #endif - #if LINGERING_CLOSE #define comm_close comm_lingering_close #endif @@ -766,6 +765,11 @@ } if (http->acl_checklist) aclChecklistFree(http->acl_checklist); + while (http->te_translations) { + TE_list *tf = http->te_translations; + http->te_translations = http->te_translations->next; + xfree (tf); + } if (request) checkFailureRatio(request->err_type, http->al.hier.code); safe_free(http->uri); @@ -1213,6 +1217,43 @@ } } +/* Transfer Encoding Filters: They work like this.. + + filter (a,b,c,d,e) + a - is a character pointer to the data to be filtered + b - is length of a + c - is address of a character pointer where the resulting buffer should be stored + d - pointer to integer to store length of c in + e - pointer to a general data pointer that the filter may use to maintain state, + it begins life as NULL. + +the return value is a 4 bit mask + TE_BUFFER_ALLOCATED - c is a newly allocated buffer and should be freed when the + calling function has completed the filter + TE_CHUNK_A - reserved internally for chunking MAGIC + TE_CHUNK_B - reserved internally for chunking MAGIC + TE_BUFFERING_OUTPUT - set this bit if your funciton was called with input, but did not + produce any (for instance if you're buffering it in the context provided by e.) + prevents squid from thinking EOF has been reached. + + call sequence + first call: *e is NULL but a is not and b>0 + body calls : a is not null, b>0 and value of *e determined by previous calls + last call: b==0.. good time to clean up *e if you've stored stuff there.. you + may produce output if necessary, but last call will be repeated. +*/ + +#ifndef _TE_RVALUES_ +#define _TE_RVALUES_ +#define TE_BUFFER_ALLOCATED 0x01 +/* TE_CHUNK_A = allocated temp buffer with empty response because input was length 0 + & no data had previous been processed - empty buffer case */ +/* could also be just that chunking was done */ +#define TE_CHUNK_A 0x02 +#define TE_CHUNK_B 0x04 +#define TE_BUFFERING_OUTPUT 0x08 +#endif + /* * filters out unwanted entries from original reply header * adds extra entries if we have more info than origin server @@ -1224,6 +1265,14 @@ HttpHeader *hdr = &rep->header; int is_hit = isTcpHit(http->log_type); request_t *request = http->request; + String s_transfer_encoding, s_te, s_ce; + int y, may_apply_tes; + HttpHeader *ohdr; + const char *s_ct; + char added_te = 0; + + ohdr = &request->header; + #if DONT_FILTER_THESE /* but you might want to if you run Squid as an HTTP accelerator */ /* httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); */ @@ -1235,6 +1284,67 @@ /* remove Set-Cookie if a hit */ if (is_hit) httpHeaderDelById(hdr, HDR_SET_COOKIE); + may_apply_tes = 0; + + if (httpHeaderHas(hdr, HDR_CONTENT_TYPE)) { + s_ct = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE); + y = strlen(s_ct); + /* TODO: allow an acl for encoding by content-type */ + if (((y >= 5) && !strncasecmp(s_ct, "text/", 5)) + || strstr(s_ct, "postscript")) { + may_apply_tes = 1; + debug(33, 8) ("Content-Type of Application: %s\n", s_ct); + } + } + if (httpHeaderHas(hdr, HDR_CONTENT_ENCODING)) { + /* TODO: check logic here & make this an acl test */ + /* oh, base entity might be compressed.. don't want + * to double *that* */ + s_ce = httpHeaderGetList(hdr, HDR_CONTENT_ENCODING); + if ((strListIsMember(&s_ce, "gzip", ',')) || + (strListIsMember(&s_ce, "deflate", ',')) || + (strListIsMember(&s_ce, "x-deflate", ',')) || + (strListIsMember(&s_ce, "x-gzip", ',')) || + (strListIsMember(&s_ce, "x-compress", ',')) || + (strListIsMember(&s_ce, "compress", ','))) + may_apply_tes |= 0x02; /* 02 is the compress exception */ + stringClean(&s_ce); + } + +/* we see this here in hdr, because WE JUST REPARSE THE HEADERS ?!?! + te_build_decode_xlate_list(hdr, &http->te_translations); + + + we should be handling the connection semantics in http.c +*/ + + /* handle Connection header */ + if (httpHeaderHas(hdr, HDR_CONNECTION)) { + /* anything that matches Connection list member will be deleted */ + String strConnection = httpHeaderGetList(hdr, HDR_CONNECTION); + const HttpHeaderEntry *e; + HttpHeaderPos pos = HttpHeaderInitPos; + /* + * think: on-average-best nesting of the two loops (hdrEntry + * and strListItem) @?@ + */ + /* + * maybe we should delete standard stuff ("keep-alive","close") + * from strConnection first? + */ + while ((e = httpHeaderGetEntry(hdr, &pos))) { + + if (strListIsMember(&strConnection, strBuf(e->name), ',')) + httpHeaderDelAt(hdr, pos); + } + httpHeaderDelById(hdr, HDR_CONNECTION); + stringClean(&strConnection); + } + + + te_build_encode_xlate_list(ohdr, hdr, &http->te_translations, request->http_ver, &request->flags); + +#if wrong_order /* handle Connection header */ if (httpHeaderHas(hdr, HDR_CONNECTION)) { /* anything that matches Connection list member will be deleted */ @@ -1250,12 +1360,14 @@ * from strConnection first? */ while ((e = httpHeaderGetEntry(hdr, &pos))) { + if (strListIsMember(&strConnection, strBuf(e->name), ',')) httpHeaderDelAt(hdr, pos); } httpHeaderDelById(hdr, HDR_CONNECTION); stringClean(&strConnection); } +#endif /* Handle Ranges */ if (request->range) clientBuildRangeHeader(http, rep); @@ -1300,14 +1412,23 @@ http->lookup_type ? http->lookup_type : "NONE", getMyHostname(), ntohs(Config.Sockaddr.http->s.sin_port)); #endif - if (httpReplyBodySize(request->method, rep) < 0) { + if ((httpReplyBodySize(request->method, rep)) < 0 && !(request->flags.te_encoding)) { debug(33, 3) ("clientBuildReplyHeader: can't keep-alive, unknown body size\n"); request->flags.proxy_keepalive = 0; - } + } else if ((httpReplyBodySize(request->method, rep)) < 0 && (request->flags.te_encoding)) + debug(33, 3) ("clientBuildReplyHeader: can keep-alive, unknown body size with te\n"); /* Signal keep-alive if needed */ httpHeaderPutStr(hdr, http->flags.accel ? HDR_CONNECTION : HDR_PROXY_CONNECTION, request->flags.proxy_keepalive ? "keep-alive" : "close"); +#if CODEFORADDINGCONNECTIONVARS + s_te = httpHeaderGetList(hdr, HDR_CONNECTION); + strListAdd(&s_te, "Transfer-Encoding", ','); + httpHeaderDelById(hdr, HDR_CONNECTION); + httpHeaderPutStr(hdr, HDR_CONNECTION, strBuf(s_te)); + stringClean(&s_te); +#endif + #if ADD_X_REQUEST_URI /* * Knowing the URI of the request is useful when debugging persistent @@ -1325,10 +1446,13 @@ clientBuildReply(clientHttpRequest * http, const char *buf, size_t size) { HttpReply *rep = httpReplyCreate(); + /* why are we parsing this again ? */ + /* we should be given a oldreply with the upstream headers already parsed, + * connection header processed, etc etc */ size_t k = headersEnd(buf, size); if (k && httpReplyParse(rep, buf, k)) { /* enforce 1.0 reply version */ - httpBuildVersion(&rep->sline.version, 1, 0); + httpBuildVersion(&rep->sline.version, 1, 1); /* do header conversions */ clientBuildReplyHeader(http, rep); /* if we do ranges, change status to "Partial Content" */ @@ -1739,7 +1863,9 @@ int fd = conn->fd; HttpReply *rep = NULL; const char *body_buf = buf; - ssize_t body_size = size; + char *te_body_buf; + int body_size = size, te_body_size; + int free_te; MemBuf mb; ssize_t check_size = 0; debug(33, 5) ("clientSendMoreData: %s, %d bytes\n", http->uri, (int) size); @@ -1749,6 +1875,13 @@ dlinkAdd(http, &http->active, &ClientActiveRequests); debug(33, 5) ("clientSendMoreData: FD %d '%s', out.offset=%d \n", fd, storeUrl(entry), (int) http->out.offset); + + /* RBC There is a problem with all this logic: + * when we are sending encoded data, receiving no data does not imply + * that we have no data to send. IFF WriteComplete sends the final TE code + * then this is ok + */ + if (conn->chr != http) { /* there is another object in progress, defer this one */ debug(33, 1) ("clientSendMoreData: Deferring %s\n", storeUrl(entry)); @@ -1756,6 +1889,8 @@ return; } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* call clientWriteComplete so the client socket gets closed */ + /* broken for TE - we can simply send a chunk-length of 0, and + * a trailer error (I think). */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); memFree(buf, MEM_CLIENT_SOCK_BUF); return; @@ -1827,6 +1962,7 @@ aclChecklistFree(ch); } else if (size < CLIENT_SOCK_SZ && entry->store_status == STORE_PENDING) { /* wait for more to arrive */ + debug(1,1)("**************** THIS SHOULDN'T BE REACHED ***************\n"); storeClientCopy(http->sc, entry, http->out.offset + size, http->out.offset, @@ -1838,9 +1974,15 @@ } /* reset range iterator */ http->range_iter.pos = HttpHdrRangeInitPos; - } else if (!http->request->range) { + /* we can't avoid this to use TE. Tuff. + * Later on we can consider: decode in store object, then pass here + * so end clients that don't support TE aren't performance affected + * For now, because I'm lazy, I'm just disabling this + */ + } else if (!http->request->range && !(http->request->flags.te_encoding)) { /* Avoid copying to MemBuf for non-range requests */ /* Note, if we're here, then 'rep' is known to be NULL */ + debug(33,8)("clientSendMoreData: writing through - no te taking place \n"); http->out.offset += body_size; comm_write(fd, buf, size, clientWriteBodyComplete, http, NULL); /* NULL because clientWriteBodyComplete frees it */ @@ -1886,10 +2028,28 @@ /* force the end of the transfer if we are done */ if (!clientPackMoreRanges(http, body_buf, body_size, &mb)) http->flags.done_copying = 1; + /* yet more possible brokeness - te takes place regardless */ } else if (body_buf && body_size) { http->out.offset += body_size; check_size += body_size; - memBufAppend(&mb, body_buf, body_size); + if (body_size) + { + http->oldte_rv = free_te = + perform_te (http->te_translations,(char *)body_buf, + body_size, &te_body_buf, &te_body_size); + + + if (te_body_size != body_size) + debug(33,1) ("****TE changed body size %s:%d:%d\n",http->log_uri, body_size,te_body_size); + if (te_body_size) + memBufAppend(&mb, te_body_buf, te_body_size); + } + + if (free_te & TE_BUFFER_ALLOCATED) + xfree (te_body_buf); + + if (free_te & TE_CHUNK_B) + http->flags.done_copying = 1; } if (!http->request->range && http->request->method == METHOD_GET) assert(check_size == size); @@ -1899,6 +2059,7 @@ memFree(buf, MEM_CLIENT_SOCK_BUF); } + /* * clientWriteBodyComplete is called for MEM_CLIENT_SOCK_BUF's * written directly to the client socket, versus copying to a MemBuf @@ -1974,7 +2135,11 @@ { clientHttpRequest *http = data; StoreEntry *entry = http->entry; - int done; + int done, stuckdatasize,free_te; + char *stuck_data; + MemBuf mb; + + stuckdatasize = 0; http->out.size += size; debug(33, 5) ("clientWriteComplete: FD %d, sz %d, err %d, off %d, len %d\n", fd, size, errflag, (int) http->out.offset, entry ? objectLen(entry) : 0); @@ -1988,13 +2153,41 @@ * just close the socket, httpRequestFree will abort if needed */ comm_close(fd); + /* what does NULL=entry really test for XXX */ } else if (NULL == entry) { comm_close(fd); /* yuk */ } else if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { comm_close(fd); - } else if ((done = clientCheckTransferDone(http)) != 0 || size == 0) { + } else if ((done = clientCheckTransferDone(http)) != 0 || + (size == 0 && !(http->oldte_rv & TE_BUFFERING_OUTPUT))) { debug(33, 5) ("clientWriteComplete: FD %d transfer is DONE\n", fd); /* We're finished case */ + http->oldte_rv = free_te = perform_te (http->te_translations,NULL,0, + &stuck_data, &stuckdatasize); + if (stuckdatasize > 0) + { + debug(33,1) ("%s:0:%d\n",http->log_uri,stuckdatasize); + + memBufDefInit(&mb); + memBufAppend(&mb, stuck_data, stuckdatasize); + + comm_write_mbuf(fd, mb, clientWriteComplete, http); + + /* don't need this, as write may happen async, it's dangerous + too.. plus, it is done after the write but the hook function + within it */ + /* memBufClean(&mb); */ + + + if (free_te & TE_BUFFER_ALLOCATED) + xfree (stuck_data); + return; + } + if (free_te & TE_CHUNK_A) /* signal from dochunk() that + data has been chunked */ + { + http->flags.done_copying = 1; + } if (httpReplyBodySize(http->request->method, entry->mem_obj->reply) < 0) { debug(33, 5) ("clientWriteComplete: closing, content_length < 0\n"); comm_close(fd); @@ -2015,6 +2208,7 @@ } else { /* More data will be coming from primary server; register with * storage manager. */ + http->oldte_rv = 0x00; if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientWriteComplete 2: ENTRY_ABORTED\n"); storeClientCopy(http->sc, entry, @@ -2160,6 +2354,7 @@ * request. We would have to return "200 OK" for a _complex_ * Range request that is also a HIT. Thus, let's prevent HITs * on complex Range requests + * TODO: Fix this. RFC 2616 covers this case */ debug(33, 3) ("clientProcessRequest2: complex range MISS\n"); http->entry = NULL; @@ -2217,6 +2412,7 @@ http->uri); http->out.offset = 0; if (NULL != http->entry) { + /* not a miss, or attaching to a in process entry */ storeLockObject(http->entry); storeCreateMemObject(http->entry, http->uri, http->log_uri); http->entry->mem_obj->method = r->method; @@ -2790,6 +2986,10 @@ http->request = requestLink(request); clientSetKeepaliveFlag(http); /* Do we expect a request-body? */ + /* we might receive reqest-bodies encoded with transfer-encoding. + * RFC 2616 is vague on this. For now, we detect it and log it to cache.log */ + if (httpHeaderHas(&http->request->header, HDR_TRANSFER_ENCODING)) + debug(33,0)("client_side has recieved a transfer encoded request entity - we cannot handle this yet. REPORT THIS to Squid-dev@squid-cache.org\n"); if (request->content_length > 0) { conn->body.size_left = request->content_length; request->body_connection = conn; @@ -2972,6 +3172,9 @@ } /* general lifetime handler for HTTP requests */ +/* TODO FIXME: RFC2616 covers this: we should send a close message + * to http/1.1 clients + */ static void requestTimeout(int fd, void *data) { @@ -3104,6 +3307,7 @@ #define SENDING_BODY 0 #define SENDING_HDRSONLY 1 +/* checks status of sending to original client */ static int clientCheckTransferDone(clientHttpRequest * http) { Index: squid/src/defines.h diff -u squid/src/defines.h:1.10 squid/src/defines.h:1.1.1.3.8.9 --- squid/src/defines.h:1.10 Mon Mar 19 17:16:29 2001 +++ squid/src/defines.h Wed Mar 21 02:09:43 2001 @@ -294,3 +294,15 @@ #ifndef O_BINARY #define O_BINARY 0 #endif + +/* te macros */ +#ifndef _TE_RVALUES_ +#define _TE_RVALUES_ +#define TE_BUFFER_ALLOCATED 0x01 +/* TE_CHUNK_A = allocated temp buffer with empty response because input was length 0 + & no data had previous been processed - empty buffer case */ +/* could also be just that chunking was done */ +#define TE_CHUNK_A 0x02 +#define TE_CHUNK_B 0x04 +#define TE_BUFFERING_OUTPUT 0x08 +#endif Index: squid/src/enums.h diff -u squid/src/enums.h:1.17 squid/src/enums.h:1.1.1.3.8.12 --- squid/src/enums.h:1.17 Sat Mar 3 02:44:32 2001 +++ squid/src/enums.h Wed Mar 21 02:09:43 2001 @@ -221,7 +221,9 @@ HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, + HDR_TE, HDR_TITLE, + HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_USER_AGENT, HDR_VARY, Index: squid/src/forward.c diff -u squid/src/forward.c:1.10 squid/src/forward.c:1.1.1.3.8.9 --- squid/src/forward.c:1.10 Sat Mar 3 02:44:32 2001 +++ squid/src/forward.c Wed Mar 21 02:09:43 2001 @@ -424,6 +424,9 @@ * some Netscape browsers have a bug that sends CONNECT * requests as GET's over persistent connections. */ + /* + * XXX TODO MAGIC. This should be an ACL test for the browser + */ request->flags.proxy_keepalive = 0; /* * Set the dont_retry flag becuase this is not a Index: squid/src/http.c diff -u squid/src/http.c:1.12 squid/src/http.c:1.1.1.3.4.1.4.14 --- squid/src/http.c:1.12 Sat Mar 3 02:44:32 2001 +++ squid/src/http.c Wed Mar 21 02:09:43 2001 @@ -350,8 +350,10 @@ } *t = '\0'; httpState->reply_hdr_state++; + /* in progess state */ assert(httpState->reply_hdr_state == 1); ctx = ctx_enter(entry->mem_obj->url); + /* state becomes 2? whats different? */ httpState->reply_hdr_state++; debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", httpState->reply_hdr); @@ -484,6 +486,7 @@ statCounter.syscalls.sock.reads++; len = read(fd, buf, read_sz); debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); + if (len > 0) { fd_bytes(fd, len, FD_READ); #if DELAY_POOLS @@ -539,13 +542,16 @@ * we want to process the reply headers. */ httpProcessReplyHeader(httpState, buf, len); + te_build_decode_xlate_list(&entry->mem_obj->reply->header, &entry->mem_obj->reply->decode_translations); fwdComplete(httpState->fwd); comm_close(fd); } else { + /* we have data */ if (httpState->reply_hdr_state < 2) { httpProcessReplyHeader(httpState, buf, len); if (httpState->reply_hdr_state == 2) { - http_status s = entry->mem_obj->reply->sline.status; + http_status s = entry->mem_obj->reply->sline.status; + te_build_decode_xlate_list(&entry->mem_obj->reply->header, &entry->mem_obj->reply->decode_translations); #if WIP_FWD_LOG fwdStatus(httpState->fwd, s); #endif @@ -557,7 +563,69 @@ EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); } } - storeAppend(entry, buf, len); + /* perform response data modifications here */ + +{ +size_t k=0; +int terv=0; +/* for now, client_side filters bad headers, we just know where they stop for te to start +*/ +/* recreate headers without te stuff */ + if (httpState->written==0) { + k = headersEnd(buf, len); + debug(1,1)("HttpReadReply: First time, header length =%d\n",k); + storeAppend(entry,buf,k); + httpState->written+=k; + + } + if (k != len) + { + char *te_body_buf; + size_t te_body_len; + /* perform transfer encoding on the recieved buffer */ + /* memory is allocated as needed by perform_te */ + terv = perform_te (entry->mem_obj->reply->decode_translations, + buf+k, len-k, &te_body_buf, &te_body_len); + + /* now te_body_buf is our output buffer, and te_body_len is the amount to write*/ + if (te_body_len) + { + /* stuff to write */ + storeAppend(entry, te_body_buf, te_body_len); + httpState->bodysize+=te_body_len; + debug(1,1)("interim content-length:%d\n",httpState->bodysize); + httpState->written+=te_body_len; + } + if (terv & TE_BUFFER_ALLOCATED) + xfree (te_body_buf); + if (terv & TE_CHUNK_B){ + /* FIXME: I think this is te's EOF marker */ + debug(1,1)("*** EOF ???\n"); + if (entry->mem_obj->reply->content_length==-1) + { + /* the current body size is unknown */ + entry->mem_obj->reply->content_length=httpState->bodysize; + } + } + if (te_body_len==0 && !(terv & TE_BUFFERING_OUTPUT)) + /* some terminating condition takes place here */ + { + debug(1,1)("figure OUT THIS TEST!?!\n"); + } + /* we should probably split off the processing of data + * so that a modification routine can alter the incoming data + * by buffering it. */ +/* + * Ie. before storeAppend, iCAP response (entry, buf, len, httpstoreReponse, httpstate) + * which buffers what it sends off to the icap server, + * and when a response is gottenm decides whether to forward the whole lot to the + * icap server, and storeAppend THAT response, or whether to + * storeAppend everything it gets + */ + + } + } +debug(1,1)("written - %d\n",httpState->written); if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* * the above storeAppend() call could ABORT this entry, @@ -756,6 +824,28 @@ } } +#if UNKNOWN_CODE +/* This wasn't in head - should it be here */ + /* append fake user agent if configured and + * the real one is not supplied by the client */ + if (Config.fake_ua && !httpHeaderHas(hdr_out, HDR_USER_AGENT)) + httpHeaderPutStr(hdr_out, HDR_USER_AGENT, Config.fake_ua); +#endif + /* append TE */ + { + char strTE[128]; + + httpHeaderDelById(hdr_out, HDR_TE); /* hop by hop.. that's what last + hop could do*/ + /* TODO: this should be built from a list of known types & a acl allowing type + * to be used in requests vs responses */ + strcpy (strTE,"chunked;q=1.0"); + + httpHeaderPutStr (hdr_out,HDR_TE,strTE); + httpHeaderPutStr(hdr_out,HDR_CONNECTION, "TE"); /* its hop by hop */ + + } + /* append Via */ strVia = httpHeaderGetList(hdr_in, HDR_VIA); snprintf(bbuf, BBUF_SZ, "%d.%d %s", Index: squid/src/protos.h diff -u squid/src/protos.h:1.21 squid/src/protos.h:1.1.1.3.8.14 --- squid/src/protos.h:1.21 Mon Mar 19 17:16:29 2001 +++ squid/src/protos.h Wed Mar 21 02:09:44 2001 @@ -394,6 +394,7 @@ extern void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, ssize_t); extern void strListAdd(String * str, const char *item, char del); extern int strListIsMember(const String * str, const char *item, char del); +extern int strListIsMember_q (const String * list, const char *m, char del); extern int strListIsSubstr(const String * list, const char *s, char del); extern int strListGetItem(const String * str, char del, const char **item, int *ilen, const char **pos); extern const char *getStringPrefix(const char *str, const char *end); @@ -1140,6 +1141,7 @@ extern void *linklistShift(link_list **); extern int xrename(const char *from, const char *to); extern int isPowTen(int); +extern short hexvalue(char i); extern void parseEtcHosts(void); #if USE_HTCP @@ -1293,3 +1295,15 @@ * hack to allow snmp access to the statistics counters */ extern StatCounters *snmpStatGet(int); + +/* + * transfer_encoding.c + */ +extern void new_xlat (TE_list **translations, int beg_or_end, + int (*func)(char *, int , char **, int *, void **), + void *data); +extern int perform_te (TE_list *xlat, char *ibuf, int ibufl, + char **obuf, int *obufl); +/* alters hdr as it build the list */ +extern void te_build_decode_xlate_list(HttpHeader * hdr, TE_list **translations); +extern void te_build_encode_xlate_list(const HttpHeader *ohdr, HttpHeader *hdr, TE_list **translations, http_version_t http_ver, request_flags *flags); Index: squid/src/structs.h diff -u squid/src/structs.h:1.26 squid/src/structs.h:1.1.1.3.4.1.4.16 --- squid/src/structs.h:1.26 Tue Feb 20 16:10:13 2001 +++ squid/src/structs.h Wed Mar 21 02:09:44 2001 @@ -869,6 +869,16 @@ int len; /* length when packed, not counting terminating '\0' */ }; + +typedef struct _TE_list +{ + int (*func)(char *ibuf, int ibufl, char **obuf, int *obufl, void **data); + void *data; + struct _TE_list *next; +} +TE_list; + + struct _HttpReply { /* unsupported, writable, may disappear/change in the future */ int hdr_sz; /* sums _stored_ status-line, headers, and */ @@ -890,6 +900,9 @@ HttpStatusLine sline; HttpHeader header; HttpBody body; /* for small constant memory-resident text bodies only */ + + /* private for http* files */ + TE_list *decode_translations; }; struct _http_state_flags { @@ -904,6 +917,8 @@ char *reply_hdr; size_t reply_hdr_size; int reply_hdr_state; + size_t written; + size_t bodysize; peer *peer; /* peer request made to */ int eof; /* reached end-of-object? */ request_t *orig_request; @@ -978,6 +993,7 @@ HierarchyLogEntry hier; }; + struct _clientHttpRequest { ConnStateData *conn; request_t *request; /* Parsed URL ... */ @@ -1013,6 +1029,9 @@ char *location; } redirect; dlink_node active; + /* what will we be encoding with ? */ + TE_list *te_translations; + int oldte_rv; }; struct _ConnStateData { @@ -1531,6 +1550,7 @@ unsigned int accelerated:1; unsigned int internal:1; unsigned int body_sent:1; + unsigned int te_encoding:1; }; struct _link_list { @@ -2035,6 +2055,36 @@ int zero_object_sz; }; +typedef struct chunked_tag +{ + int state; /* 1 - in between */ + int pos; /* 2 - reading hexl */ + int toread; /* 3 - seeking CRLF from hexl */ + int half_crlf; /* 4 - reading data */ + /* 5 - seeking CRLF from data */ + /* 6 - looking for double CRLF */ +} chunked_t; + +#if removedfromhead +struct _PumpStateData { + FwdState *fwd; + request_t *req; + store_client *sc; /* The store client we're using */ + int c_fd; /* client fd */ + int s_fd; /* server end */ + int rcvd; /* bytes received from client */ + int sent; /* bytes sent to server */ + StoreEntry *request_entry; /* the request entry */ + StoreEntry *reply_entry; /* the reply entry */ + CWCB *callback; /* what to do when we finish sending */ + void *cbdata; /* callback data passed to callback func */ + struct { + int closing:1; + } flags; + struct _PumpStateData *next; +}; +#endif + /* * This defines an fs type */ Index: squid/src/tools.c diff -u squid/src/tools.c:1.11 squid/src/tools.c:1.1.1.3.8.10 --- squid/src/tools.c:1.11 Thu Feb 15 13:09:17 2001 +++ squid/src/tools.c Sat Feb 17 16:55:18 2001 @@ -955,6 +955,16 @@ return 1; } +short +hexvalue (char i) +{ + if ((i >= '0') && (i<='9')) + i= i - '0'; + else + i = toupper (i) - 'A' + 10; + return i; +} + void parseEtcHosts(void) { Index: squid/src/transfer-encoding.c diff -u /dev/null squid/src/transfer-encoding.c:1.1.2.1 --- /dev/null Mon Jan 26 04:58:16 2004 +++ squid/src/transfer-encoding.c Sat Jan 13 00:31:13 2001 @@ -0,0 +1,431 @@ + +/* + * $Id: squid-te-HEAD,v 1.1 2004/08/17 20:55:13 hno Exp $ + * + * DEBUG: section 82 Transfer-encoding Routines + * AUTHOR: Robert Collins + * + * Based on original work by Patrick McManus + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * the Regents of the University of California. Please see the + * COPYRIGHT file for full details. Squid incorporates software + * developed and/or copyrighted by other sources. Please see the + * CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +int +perform_te (TE_list *xlat, char *ibuf, int ibufl, + char **obuf, int *obufl) +{ + int rv = 0, made_new, orig_len; + + *obuf = ibuf; + *obufl = ibufl; + orig_len = ibufl; + + made_new = 0; + + while (xlat) + { + /* xlat func is likely to be unbiff or dogzip() or something */ + + rv &= 0xFE; /* turn off low bit, for reuse, while + preserving chunk indicators */ + + rv |= (*(xlat->func)) ((char *)ibuf, ibufl, obuf, obufl,&(xlat->data)); + + /* FREE SOME STUFF HERE if (rv & TE_BUFFER_ALLOCATED) */ + /* we don't free the handed buffer, this is if a xlate in a chain allocs a buffer */ + if ((made_new) && (rv & TE_BUFFER_ALLOCATED)) + xfree (ibuf); + + made_new = made_new | (rv & TE_BUFFER_ALLOCATED); + + ibuf = *obuf; + ibufl = *obufl; + xlat = xlat->next; + } + + if ((rv & TE_BUFFERING_OUTPUT) /* a 0 length chunk was created */ + && (orig_len > 0)) /* and there was actually input */ + { + /* is this a mem leak ? */ + *obufl = 0; /* don't write the chunk, as it means EOT */ + debug(82,8) ("EOT PREVENTION\n"); + } + + return rv | made_new; +} + +/* this is a TE filter */ +static int dochunk (char *ibuf, int ibufl, char **obuf, int *obufl, void **d) +{ + char *tb, *w; + int tbl, created, rv; + + tbl = ibufl+128; + tb = xmalloc (tbl); + rv = 0x00; + + if (ibufl > 0) + { + created = 0; + snprintf (tb,tbl,"%X\r\n", ibufl); + debug(82, 8) ("created chunk of %d\n",ibufl); + + w = tb + strlen (tb); + memcpy (w,ibuf,ibufl); + created = ibufl + (w-tb) + 2; + strcpy (w+ibufl,"\r\n"); + *d = (void *) 0x01; + } + else /* end signal */ + if (*d) + { + strcpy (tb,"0\r\n\r\n"); + created = 5; + debug(82,8) ("created chunk of %d\n",ibufl); + *d = NULL; + rv = TE_BUFFERING_OUTPUT; + } + else + { + /* allocate a buffer of length 128, and return it with length 0 */ + *obuf = tb; + *obufl = 0; + return TE_CHUNK_A | TE_BUFFER_ALLOCATED; /* signal that dochunk() is complete */ + } + + *obuf = tb; + *obufl = created; + + return rv | TE_BUFFER_ALLOCATED; +} + +static int undochunked + (char *ibuf, int ibufl, char **obuf, int *obufl, void **d) +{ + chunked_t *h; + char *tb, *w, *l; + int tbl, created,rv = 1,tr; + + h = *((chunked_t **) d); + + if (!h) + { + h = xmalloc (sizeof (chunked_t)); + h->state = 1; + *d = (void *) h; + } + + tbl = ibufl; + tb = xmalloc (tbl); + + created = 0; + w=ibuf; + l = ibuf+ibufl; + debug(82, 8)("undochunked: called with ibufl=%d\n",ibufl); + + state_machine_repeat: + + if (ibufl) + switch (h->state) + { + case 1: /* new chunk */ + h->pos = 0; + h->toread = 0; + h->half_crlf = 0; + + for (; (w < l) && !isxdigit (*w); w++); + if (w>=l) + break; + /* no default break! */ + h->state = 2; /* for 1 fallthru */ + + case 2: /* reading hexl */ + for (; (w < l) && isxdigit (*w); w++) + { + h->toread = h->toread << 4; + h->toread += hexvalue(*w); + } + if (w>=l) + break; + h->state = 3; + h->pos = h->toread; + debug(82,8) ("read chunk of %d\n",h->toread); + + + case 3: /* searching for CRLF */ + do + { + if (!h->half_crlf) + { + for (; (w < l) && (*w != '\r') ; w++); + if (w>=l) + break; + h->half_crlf = 1; + w++; + if (w>=l) + break; + } + if (*w != '\n') + h->half_crlf = 0; + else + { + h->state = 4; + } + w++; + if (w>=l) + break; + }while (h->state == 3); + + case 4: + /* gotta read h->toread bytes */ + if (h->toread) + { + tr = min (h->toread, l-w); + memcpy (tb+created, w, tr); + w += tr; + h->toread -= tr; + created += tr; + } + if (h->toread == 0) + { + h->state = 5; + h->half_crlf = 0; + } + if (w>=l) + break; + + case 5: /* seeking CRLF */ + do + { + if (!h->half_crlf) + { + for (; (w < l) && (*w != '\r') ; w++); + if (w>=l) + break; + h->half_crlf = 1; + w++; + if (w>=l) + break; + } + if (*(w++) != '\n') + h->half_crlf = 0; + else + { + debug(82,8) ("completed %d chunk\n",h->pos); + if (h->pos) + { + h->state = 1; + goto state_machine_repeat; + } + else + { + h->state =6; + h->half_crlf = 2; + } + } + + + + if (w>=l) + break; + }while (h->state == 5); + + case 6: + while (h->half_crlf < 4) + { + if (w>=l) + break; + if (!(h->half_crlf % 2)) + if (*w == '\r') + h->half_crlf++; + else + h->half_crlf =0; + else + if (*w == '\n') + h->half_crlf++; + else + h->half_crlf =0; + w++; + } + + /* chunked complete */ + h->state = 7; + rv = TE_CHUNK_B | TE_BUFFER_ALLOCATED; /* signal that undochunk() is complete */ + + case 7: /* done, ignore */ + break; + } + else + { + xfree (h); + xfree (tb); + *d = NULL; + *obufl = 0; + return 0; + } + + + *obuf = tb; + *obufl = created; + + return rv; +} + +void +new_xlat (TE_list **translations, int beg_or_end, + int (*func)(char *, int , char **, int *, void **), + void *data) +{ + TE_list *tp,*tx; + + tp = xmalloc (sizeof (TE_list)); + tp->func = func; + tp->data = data; + + if (beg_or_end == 0) + { /* beginning */ + tp->next = *translations; + *translations = tp; + } + else + { + /* end */ + tp->next = NULL; + for (tx = *translations; tx && tx->next; tx =tx->next); + if (!tx) /* root node */ + *translations = tp; + else + tx->next = tp; + } + + return; +} + +void +te_build_decode_xlate_list(HttpHeader * hdr, TE_list **translations) +{ + String s_transfer_encoding; + const char *pos = NULL; + const char *item; + int y; + if (httpHeaderHas(hdr, HDR_TRANSFER_ENCODING)) { + s_transfer_encoding = httpHeaderGetList(hdr, HDR_TRANSFER_ENCODING); + + /* REMOVE ALL TRANSFER ENCODINGS */ + /* order is impt here.. want those entries at end of transfer-encoding + * list to be undone first and they are listed in the order they are + * applied .. TE's are HOP BY HOP */ + + pos = NULL; + while (strListGetItem(&s_transfer_encoding, ',', &item, &y, &pos)) + if (!strncasecmp(item, "chunked", y)) { + /* need to remove the TE.. and rewrite the header */ + + debug(82, 7) ("removing chunked\n"); + new_xlat(translations, 0, undochunked, NULL); + } + + + httpHeaderDelById(hdr, HDR_TRANSFER_ENCODING); + stringClean(&s_transfer_encoding); + } + /* we've handles the incoming transfer encoding types */ + /* now we remove tansfer-encoding from the connection header */ + /* handle Connection header */ + if (httpHeaderHas(hdr, HDR_CONNECTION)) { + /* remove transfer-encoding from the Connection list. If it's not present + * log that + */ +#if FIXME + /* We need to be able to delete a list item from the connection hdr */ + String strConnection = httpHeaderGetList(hdr, HDR_CONNECTION); + const HttpHeaderEntry *e; + HttpHeaderPos pos = HttpHeaderInitPos; + /* + * think: on-average-best nesting of the two loops (hdrEntry + * and strListItem) @?@ + */ + while ((e = httpHeaderGetEntry(hdr, &pos))) { + + if (strListIsMember(&strConnection, strBuf(e->name), ',')) + httpHeaderDelAt(hdr, pos); + } + httpHeaderDelById(hdr, HDR_CONNECTION); + stringClean(&strConnection); +#endif + } + else + debug (82,1)("build_remove_xlate_list: RFC 2616 violation: Transfer-encoding present without a CONNECTION header\n"); +} + +void +te_build_encode_xlate_list(const HttpHeader *ohdr, HttpHeader *hdr, TE_list **translations, http_version_t http_ver, request_flags *flags) +{ + String s_te; + /* ADD NEW TRANSFER ENCODINGS.. check announcements */ + /* ALWAYS CHUNK. Chunking adds significant performance on busy links + * by allowing pipelining + */ + + if ((httpHeaderHas(ohdr, HDR_TE) + || ((http_ver.major == 1) && (http_ver.minor >=1)))) { + char vlb[128]; + vlb[0] = '\0'; + s_te = httpHeaderGetList(ohdr, HDR_TE); + +/* loop /choose compression te's */ +/* finish with chunked */ + +#if TEMPORARY_HACK + if ((strListIsMember_q(&s_te, "chunked", ',')) || + ((http_ver.major == 1) && (http_ver.minor >=1))) +#endif + { + + /* chunked is known.. we can add te's. */ + + /* always add chunked to cover nonterminating encodings */ + debug(82, 7) ("will add chunked\n"); + new_xlat(translations, 1, dochunk, NULL); + strcat(vlb, "chunked"); + httpHeaderPutStr(hdr, HDR_TRANSFER_ENCODING, vlb); + httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); + + } + stringClean(&s_te); + flags->te_encoding=1; + s_te = httpHeaderGetList(hdr, HDR_CONNECTION); + strListAdd(&s_te, "Transfer-Encoding", ','); + httpHeaderDelById(hdr, HDR_CONNECTION); + httpHeaderPutStr(hdr, HDR_CONNECTION, strBuf(s_te)); + stringClean(&s_te); + } + /* add transfer-encoding to the connection header */ +} squid-te-HEAD.new squid-te-HEAD differ: char 61, line 2