--------------------- PatchSet 8815 Date: 2006/12/19 01:03:13 Author: hno Branch: http11 Tag: (none) Log: Minimal HTTP/1.1 support, allowing advertising HTTP/1.1 upstream towards servers. Client-side HTTP/1.1 support still completely missing. - Implements decoding of chunked encoding in replies - HTTP server side connection management aware of chunked encoding - Don't forward HTTP/1.1 headers singalling capabilities we don't support (i.e. the Expect header signalling the use of 100-continue) Members: src/HttpHeader.c:1.24->1.24.6.1 src/HttpMsg.c:1.8->1.8.14.1 src/HttpReply.c:1.16->1.16.14.1 src/client_side.c:1.154->1.154.2.1 src/enums.h:1.57->1.57.6.1 src/ftp.c:1.36->1.36.6.1 src/http.c:1.49->1.49.6.1 src/structs.h:1.136->1.136.6.1 tools/squidclient.c:1.5->1.5.20.1 Index: squid/src/HttpHeader.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpHeader.c,v retrieving revision 1.24 retrieving revision 1.24.6.1 diff -u -r1.24 -r1.24.6.1 --- squid/src/HttpHeader.c 29 Nov 2006 05:51:38 -0000 1.24 +++ squid/src/HttpHeader.c 19 Dec 2006 01:03:13 -0000 1.24.6.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.c,v 1.24 2006/11/29 05:51:38 squidadm Exp $ + * $Id: HttpHeader.c,v 1.24.6.1 2006/12/19 01:03:13 hno Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -113,12 +113,14 @@ {"Retry-After", HDR_RETRY_AFTER, ftStr}, /* for now (ftDate_1123 or ftInt!) */ {"Server", HDR_SERVER, ftStr}, {"Set-Cookie", HDR_SET_COOKIE, ftStr}, - {"Title", HDR_TITLE, ftStr}, {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr}, + {"Te", HDR_TE, ftStr}, + {"Trailer", HDR_TRAILER, ftStr}, {"Upgrade", HDR_UPGRADE, ftStr}, /* for now */ {"User-Agent", HDR_USER_AGENT, ftStr}, {"Vary", HDR_VARY, ftStr}, /* for now */ {"Via", HDR_VIA, ftStr}, /* for now */ + {"Expect", HDR_EXPECT, ftStr}, {"Warning", HDR_WARNING, ftStr}, /* for now */ {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr}, {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr}, @@ -157,6 +159,8 @@ HDR_LINK, HDR_PRAGMA, HDR_PROXY_CONNECTION, HDR_PROXY_SUPPORT, + HDR_TE, + HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_VARY, @@ -165,7 +169,7 @@ HDR_WWW_AUTHENTICATE, HDR_AUTHENTICATION_INFO, HDR_PROXY_AUTHENTICATION_INFO, - /* HDR_EXPECT, HDR_TE, HDR_TRAILER */ + HDR_EXPECT, #if X_ACCELERATOR_VARY HDR_X_ACCELERATOR_VARY, #endif @@ -177,8 +181,8 @@ { HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA, HDR_TRANSFER_ENCODING, + HDR_TRAILER, HDR_UPGRADE, - /* HDR_TRAILER, */ HDR_VIA }; @@ -216,7 +220,7 @@ HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH, HDR_IF_RANGE, HDR_MAX_FORWARDS, HDR_PROXY_CONNECTION, HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE, - HDR_USER_AGENT, HDR_X_FORWARDED_FOR + HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_TE, HDR_EXPECT }; /* header accounting */ Index: squid/src/HttpMsg.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpMsg.c,v retrieving revision 1.8 retrieving revision 1.8.14.1 diff -u -r1.8 -r1.8.14.1 --- squid/src/HttpMsg.c 13 Jun 2006 14:53:53 -0000 1.8 +++ squid/src/HttpMsg.c 19 Dec 2006 01:03:13 -0000 1.8.14.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpMsg.c,v 1.8 2006/06/13 14:53:53 squidadm Exp $ + * $Id: HttpMsg.c,v 1.8.14.1 2006/12/19 01:03:13 hno Exp $ * * DEBUG: section 74 HTTP Message * AUTHOR: Alex Rousskov @@ -93,17 +93,13 @@ { if (httpHeaderHasConnDir(hdr, "close")) return 0; -#if WHEN_SQUID_IS_HTTP1_1 - if ((http_ver.major >= 1) && (http_ver.minor >= 1)) { + if ((http_ver.major >= 1 && http_ver.minor >= 1) || http_ver.major > 1) { /* * for modern versions of HTTP: persistent unless there is * a "Connection: close" header. */ return 1; } else { -#else - { -#endif /* * Persistent connections in Netscape 3.x are allegedly broken, * return false if it is a browser connection. If there is a Index: squid/src/HttpReply.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpReply.c,v retrieving revision 1.16 retrieving revision 1.16.14.1 diff -u -r1.16 -r1.16.14.1 --- squid/src/HttpReply.c 11 Jun 2006 00:50:43 -0000 1.16 +++ squid/src/HttpReply.c 19 Dec 2006 01:03:13 -0000 1.16.14.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpReply.c,v 1.16 2006/06/11 00:50:43 squidadm Exp $ + * $Id: HttpReply.c,v 1.16.14.1 2006/12/19 01:03:13 hno Exp $ * * DEBUG: section 58 HTTP Reply (Response) * AUTHOR: Alex Rousskov @@ -269,7 +269,7 @@ HttpHeader *hdr; http_version_t ver; assert(reply); - httpBuildVersion(&ver, 1, 0); + httpBuildVersion(&ver, 1, 1); httpStatusLineSet(&reply->sline, ver, status, httpStatusString(status)); hdr = &reply->header; httpHeaderPutStr(hdr, HDR_SERVER, full_appname_string); Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.154 retrieving revision 1.154.2.1 diff -u -r1.154 -r1.154.2.1 --- squid/src/client_side.c 11 Dec 2006 00:53:11 -0000 1.154 +++ squid/src/client_side.c 19 Dec 2006 01:03:13 -0000 1.154.2.1 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.154 2006/12/11 00:53:11 squidadm Exp $ + * $Id: client_side.c,v 1.154.2.1 2006/12/19 01:03:13 hno Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -1148,7 +1148,7 @@ */ http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); httpReplyReset(r = http->entry->mem_obj->reply); - httpBuildVersion(&version, 1, 0); + httpBuildVersion(&version, 1, 1); httpReplySetHeaders(r, version, status, NULL, NULL, 0, 0, -1); httpReplySwapOut(r, http->entry); storeComplete(http->entry); @@ -1568,9 +1568,7 @@ debug(33, 3) ("clientSetKeepaliveFlag: method = %s\n", RequestMethodStr[request->method]); { - http_version_t http_ver; - httpBuildVersion(&http_ver, 1, 0); /* we are HTTP/1.0, no matter what the client requests... */ - if (httpMsgIsPersistent(http_ver, req_hdr)) + if (httpMsgIsPersistent(request->http_ver, req_hdr)) request->flags.proxy_keepalive = 1; } } @@ -1836,6 +1834,9 @@ /* remove Set-Cookie if a hit */ if (http->flags.hit) httpHeaderDelById(hdr, HDR_SET_COOKIE); + httpHeaderDelById(hdr, HDR_TRAILER); + httpHeaderDelById(hdr, HDR_TRANSFER_ENCODING); + httpHeaderDelById(hdr, HDR_UPGRADE); /* handle Connection header */ if (httpHeaderHas(hdr, HDR_CONNECTION)) { /* anything that matches Connection list member will be deleted */ @@ -1894,6 +1895,10 @@ } else if (http->entry->timestamp < squid_curtime) httpHeaderPutInt(hdr, HDR_AGE, squid_curtime - http->entry->timestamp); + if (!httpHeaderHas(hdr, HDR_CONTENT_LENGTH) && http->entry->mem_obj && http->entry->store_status == STORE_OK) { + rep->content_length = contentLen(http->entry); + httpHeaderPutSize(hdr, HDR_CONTENT_LENGTH, rep->content_length); + } } /* Filter unproxyable authentication types */ if (http->log_type != LOG_TCP_DENIED && @@ -1961,7 +1966,7 @@ } /* Append Via */ { - LOCAL_ARRAY(char, bbuf, MAX_URL + 32); + char bbuf[MAX_URL + 32]; String strVia = httpHeaderGetList(hdr, HDR_VIA); snprintf(bbuf, sizeof(bbuf), "%d.%d %s", rep->sline.version.major, @@ -1972,9 +1977,13 @@ stringClean(&strVia); } /* Signal keep-alive if needed */ - httpHeaderPutStr(hdr, - (http->flags.accel || http->flags.transparent) ? HDR_CONNECTION : HDR_PROXY_CONNECTION, - request->flags.proxy_keepalive ? "keep-alive" : "close"); + if (!request->flags.proxy_keepalive) + httpHeaderPutStr(hdr, HDR_CONNECTION, "close"); + else if (request->http_ver.major == 1 && request->http_ver.minor == 0) { + httpHeaderPutStr(hdr, HDR_CONNECTION, "keep-alive"); + if (!(http->flags.accel || http->flags.transparent)) + httpHeaderPutStr(hdr, HDR_PROXY_CONNECTION, "keep-alive"); + } #if ADD_X_REQUEST_URI /* * Knowing the URI of the request is useful when debugging persistent @@ -1995,7 +2004,7 @@ size_t k = headersEnd(buf, size); if (k && httpReplyParse(rep, buf, k)) { /* enforce 1.0 reply version */ - httpBuildVersion(&rep->sline.version, 1, 0); + httpBuildVersion(&rep->sline.version, 1, 1); /* do header conversions */ clientBuildReplyHeader(http, rep); /* if we do ranges, change status to "Partial Content" */ @@ -3109,10 +3118,7 @@ } else if ((done = clientCheckTransferDone(http)) != 0 || size == 0) { debug(33, 5) ("clientWriteComplete: FD %d transfer is DONE\n", fd); /* We're finished case */ - if (httpReplyBodySize(http->request->method, entry->mem_obj->reply) < 0) { - debug(33, 5) ("clientWriteComplete: closing, content_length < 0\n"); - comm_close(fd); - } else if (!done) { + if (!done) { debug(33, 5) ("clientWriteComplete: closing, !done\n"); comm_close(fd); } else if (clientGotNotEnough(http)) { @@ -3340,7 +3346,7 @@ storeReleaseRequest(http->entry); storeBuffer(http->entry); rep = httpReplyCreate(); - httpBuildVersion(&version, 1, 0); + httpBuildVersion(&version, 1, 1); httpReplySetHeaders(rep, version, HTTP_OK, NULL, "text/plain", httpRequestPrefixLen(r), 0, squid_curtime); httpReplySwapOut(rep, http->entry); @@ -4780,7 +4786,8 @@ { squid_off_t cl = httpReplyBodySize(http->request->method, http->entry->mem_obj->reply); int hs = http->entry->mem_obj->reply->hdr_sz; - assert(cl >= 0); + if (cl < 0) + return 0; if (http->out.offset != cl + hs) return 1; return 0; Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.57 retrieving revision 1.57.6.1 diff -u -r1.57 -r1.57.6.1 --- squid/src/enums.h 30 Sep 2006 21:52:28 -0000 1.57 +++ squid/src/enums.h 19 Dec 2006 01:03:18 -0000 1.57.6.1 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.57 2006/09/30 21:52:28 squidadm Exp $ + * $Id: enums.h,v 1.57.6.1 2006/12/19 01:03:18 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -208,6 +208,9 @@ HDR_CONTENT_MD5, HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, + HDR_TE, + HDR_TRANSFER_ENCODING, + HDR_TRAILER, HDR_COOKIE, HDR_DATE, HDR_ETAG, @@ -235,12 +238,11 @@ HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, - HDR_TITLE, - HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_USER_AGENT, HDR_VARY, HDR_VIA, + HDR_EXPECT, HDR_WARNING, HDR_WWW_AUTHENTICATE, HDR_AUTHENTICATION_INFO, Index: squid/src/ftp.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ftp.c,v retrieving revision 1.36 retrieving revision 1.36.6.1 diff -u -r1.36 -r1.36.6.1 --- squid/src/ftp.c 18 Sep 2006 23:52:31 -0000 1.36 +++ squid/src/ftp.c 19 Dec 2006 01:03:18 -0000 1.36.6.1 @@ -1,6 +1,6 @@ /* - * $Id: ftp.c,v 1.36 2006/09/18 23:52:31 squidadm Exp $ + * $Id: ftp.c,v 1.36.6.1 2006/12/19 01:03:18 hno Exp $ * * DEBUG: section 9 File Transfer Protocol (FTP) * AUTHOR: Harvest Derived @@ -2624,7 +2624,7 @@ ftpUrlWith2f(const request_t * request) { LOCAL_ARRAY(char, buf, MAX_URL); - LOCAL_ARRAY(char, loginbuf, MAX_LOGIN_SZ + 1); + LOCAL_ARRAY(char, loginbuf, MAX_LOGIN_SZ + 2); LOCAL_ARRAY(char, portbuf, 32); char *t; portbuf[0] = '\0'; @@ -2634,7 +2634,7 @@ snprintf(portbuf, 32, ":%d", request->port); loginbuf[0] = '\0'; if ((int) strlen(request->login) > 0) { - xstrncpy(loginbuf, request->login, sizeof(loginbuf) - 2); + xstrncpy(loginbuf, request->login, MAX_LOGIN_SZ); if ((t = strchr(loginbuf, ':'))) *t = '\0'; strcat(loginbuf, "@"); Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.49 retrieving revision 1.49.6.1 diff -u -r1.49 -r1.49.6.1 --- squid/src/http.c 23 Oct 2006 21:53:15 -0000 1.49 +++ squid/src/http.c 19 Dec 2006 01:03:18 -0000 1.49.6.1 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.49 2006/10/23 21:53:15 squidadm Exp $ + * $Id: http.c,v 1.49.6.1 2006/12/19 01:03:18 hno Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -409,12 +409,14 @@ } /* rewrite this later using new interfaces @?@ */ -static void +static size_t httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) { StoreEntry *entry = httpState->entry; size_t hdr_len; size_t hdr_size; + size_t old_size; + size_t done; HttpReply *reply = entry->mem_obj->reply; Ctx ctx = ctx_enter(entry->mem_obj->url); debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", @@ -422,6 +424,7 @@ if (memBufIsNull(&httpState->reply_hdr)) memBufDefInit(&httpState->reply_hdr); assert(httpState->reply_hdr_state == 0); + old_size = httpState->reply_hdr.size; memBufAppend(&httpState->reply_hdr, buf, size); hdr_len = httpState->reply_hdr.size; if (hdr_len > 4 && strncmp(httpState->reply_hdr.buf, "HTTP/", 5)) { @@ -431,19 +434,19 @@ httpBuildVersion(&reply->sline.version, 0, 9); reply->sline.status = HTTP_INVALID_HEADER; ctx_exit(ctx); - return; + return 0; } hdr_size = headersEnd(httpState->reply_hdr.buf, hdr_len); if (hdr_size) hdr_len = hdr_size; if (hdr_len > Config.maxReplyHeaderSize) { debug(11, 1) ("httpProcessReplyHeader: Too large reply header\n"); - if (!memBufIsNull(&httpState->reply_hdr)) - memBufClean(&httpState->reply_hdr); + storeAppend(entry, httpState->reply_hdr.buf, httpState->reply_hdr.size); + memBufClean(&httpState->reply_hdr); reply->sline.status = HTTP_HEADER_TOO_LARGE; httpState->reply_hdr_state += 2; ctx_exit(ctx); - return; + return 0; } /* headers can be incomplete only if object still arriving */ if (!hdr_size) { @@ -451,7 +454,7 @@ hdr_size = hdr_len; else { ctx_exit(ctx); - return; /* headers not complete */ + return 0; /* headers not complete */ } } safe_free(entry->mem_obj->vary_headers); @@ -467,18 +470,50 @@ /* Parse headers into reply structure */ /* what happens if we fail to parse here? */ httpReplyParse(reply, httpState->reply_hdr.buf, hdr_size); + storeAppend(entry, httpState->reply_hdr.buf, hdr_size); + done = hdr_size - old_size; if (reply->sline.status >= HTTP_INVALID_HEADER) { debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr.buf); memBufClean(&httpState->reply_hdr); ctx_exit(ctx); - return; + return 0; } if (!peer_supports_connection_pinning(httpState)) httpState->orig_request->flags.no_connection_auth = 1; storeTimestampsSet(entry); /* Check if object is cacheable or not based on reply code */ debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); - if (httpHeaderHas(&reply->header, HDR_VARY) + if (httpHeaderHas(&reply->header, HDR_TRANSFER_ENCODING)) { + String tr = httpHeaderGetList(&reply->header, HDR_TRANSFER_ENCODING); + const char *pos = NULL; + const char *item = NULL; + int ilen = 0; + if (strListGetItem(&tr, ',', &item, &ilen, &pos)) { + if (ilen == 7 && strncasecmp(item, "chunked", ilen) == 0) { + httpState->flags.chunked = 1; + if (!strListGetItem(&tr, ',', &item, &ilen, &pos)) + item = NULL; + } + if (item) { + /* Can't handle other transfer-encodings */ + debug(11, 1) ("Unexpected transfer encoding '%s'\n", strBuf(tr)); + reply->sline.status = HTTP_INVALID_HEADER; + return 0; + } + } + httpState->flags.chunked = strListIsMember(&tr, "chunked", ','); + stringClean(&tr); + if (reply->content_length >= 0) { + /* Can't have a content-length in chunked encoding */ + reply->sline.status = HTTP_INVALID_HEADER; + return 0; + } + } + if (!httpState->flags.chunked) { + /* non-chunked, but known length. Handle as one single big chunk */ + httpState->chunk_size = httpReplyBodySize(httpState->orig_request->method, reply); + } + if (httpHeaderHas(&reply->header, HDR_VARY) #if X_ACCELERATOR_VARY || httpHeaderHas(&reply->header, HDR_X_ACCELERATOR_VARY) #endif @@ -488,7 +523,7 @@ vary = httpMakeVaryMark(httpState->orig_request, reply); if (!vary) { httpMakePrivate(entry); - goto no_cache; + goto no_cache; /* XXX Would be better if this was used by the swicht statement below */ } entry->mem_obj->vary_headers = xstrdup(vary); if (strBuf(httpState->orig_request->vary_encoding)) @@ -543,36 +578,7 @@ #if HEADERS_LOG headersLog(1, 0, httpState->request->method, reply); #endif -} - -static int -httpPconnTransferDone(HttpStateData * httpState) -{ - /* return 1 if we got the last of the data on a persistent connection */ - MemObject *mem = httpState->entry->mem_obj; - HttpReply *reply = mem->reply; - squid_off_t clen; - debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); - debug(11, 5) ("httpPconnTransferDone: content_length=%" PRINTF_OFF_T "\n", - reply->content_length); - /* If we haven't seen the end of reply headers, we are not done */ - if (httpState->reply_hdr_state < 2) - return 0; - clen = httpReplyBodySize(httpState->request->method, reply); - /* If the body size is unknown we must wait for EOF */ - if (clen < 0) - return 0; - /* Barf if we got more than we asked for */ - if (mem->inmem_hi > clen + reply->hdr_sz) - return -1; - /* If there is no message body, we can be persistent */ - if (0 == clen) - return 1; - /* If the body size is known, we must wait until we've gotten all of it. */ - if (mem->inmem_hi < clen + reply->hdr_sz) - return 0; - /* We got it all */ - return 1; + return done; } /* Small helper function to verify if connection pinning is supported or not @@ -615,6 +621,187 @@ return rc; } +static void +httpAppendBody(HttpStateData * httpState, const char *buf, ssize_t len, int buffer_filled) +{ + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + const request_t *orig_request = httpState->orig_request; + struct in_addr *client_addr = NULL; + u_short client_port = 0; + int fd = httpState->fd; + int complete = httpState->eof; + int keep_alive = !httpState->eof; + while (len > 0) { + if (httpState->chunk_size > 0) { + size_t size = len; + if (size > httpState->chunk_size) + size = httpState->chunk_size; + httpState->chunk_size -= size; + storeAppend(httpState->entry, buf, size); + buf += size; + len -= size; + } else if (httpState->chunk_size < 0) { + /* non-chunked without content-length */ + storeAppend(httpState->entry, buf, len); + len = 0; + } else if (httpState->flags.chunked) { + char *eol = memchr(buf, '\n', len); + size_t size = eol - buf + 1; + if (!eol) + size = len; + stringAppend(&httpState->chunkhdr, buf, size); + buf += size; + len -= size; + if (eol) { + if (!httpState->flags.trailer) { + /* chunk header */ + char *end = NULL; + int badchunk = 0; + debug(11, 3) ("Chunk header '%s'\n", strBuf(httpState->chunkhdr)); + httpState->chunk_size = strto_off_t(strBuf(httpState->chunkhdr), &end, 16); + if (end == strBuf(httpState->chunkhdr)) + badchunk = 1; + while (end && (*end == '\r' || *end == ' ' || *end == '\t')) + end++; + if (httpState->chunk_size < 0 || !end || (*end != '\n' && *end == ';')) { + debug(11, 0) ("Invalid chunk header '%s'\n", strBuf(httpState->chunkhdr)); + comm_close(fd); + return; + } + if (badchunk) + continue; /* Skip blank lines */ + debug(11, 2) ("Chunk size %" PRINTF_OFF_T "\n", httpState->chunk_size); + if (httpState->chunk_size == 0) { + debug(11, 3) ("Processing trailer\n"); + httpState->flags.trailer = 1; + } + } else { + /* trailer */ + const char *p = strBuf(httpState->chunkhdr); + while (*p == '\r') + p++; + if (*p == '\n') { + complete = 1; + debug(11, 2) ("Chunked response complete\n"); + } + } + stringReset(&httpState->chunkhdr, NULL); + } + } else { + /* Don't know what to do with this data. Bail out */ + break; + } + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * the above storeAppend() call could ABORT this entry, + * in that case, the server FD should already be closed. + * there's nothing for us to do. + */ + return; + } + } + if (!httpState->chunk_size && !httpState->flags.chunked) + complete = 1; + if (!complete && len == 0) { + /* Wait for more data or EOF condition */ + if (httpState->flags.keepalive_broken) { + commSetTimeout(fd, 10, NULL, NULL); + } else { + commSetTimeout(fd, Config.Timeout.read, NULL, NULL); + } + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + return; + } + /* Is it a incomplete reply? */ + if (httpState->chunk_size > 0) { + debug(11, 1) ("Short response on port %d. Expecting %" PRINTF_OFF_T " octets more\n", comm_local_port(fd), httpState->chunk_size); + comm_close(fd); + return; + } + /* + * Verify that the connection is clean + */ + if (len == 0 && buffer_filled >= 0) { + char buf2[4]; + statCounter.syscalls.sock.reads++; + len = FD_READ_METHOD(fd, buf2, sizeof(buf2)); + if ((len < 0 && !ignoreErrno(errno)) || len == 0) { + keep_alive = 0; + } + } + if (len > 0) { + debug(11, Config.onoff.relaxed_header_parser <= 0 || keep_alive ? 1 : 2) + ("httpReadReply: Excess data from \"%s %s\"\n", + RequestMethodStr[orig_request->method], + storeUrl(entry)); + comm_close(fd); + return; + } + /* + * Verified and done with the reply + */ + fwdComplete(httpState->fwd); + + /* + * If we didn't send a keep-alive request header, then this + * can not be a persistent connection. + */ + if (!httpState->flags.keepalive) + keep_alive = 0; + /* + * If we haven't sent the whole request then this can not be a persistent + * connection. + */ + if (!httpState->flags.request_sent) { + debug(11, 1) ("httpReadReply: Request not yet fully sent \"%s %s\"\n", + RequestMethodStr[orig_request->method], + storeUrl(entry)); + keep_alive = 0; + } + /* + * What does the reply have to say about keep-alive? + */ + if (!entry->mem_obj->reply->keep_alive) + keep_alive = 0; + if (keep_alive) { + int pinned = 0; +#if LINUX_TPROXY + if (orig_request->flags.tproxy) { + client_addr = &httpState->request->client_addr; + } +#endif + /* yes we have to clear all these! */ + commSetDefer(fd, NULL, NULL); + commSetTimeout(fd, -1, NULL, NULL); + commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); +#if DELAY_POOLS + delayClearNoDelay(fd); +#endif + comm_remove_close_handler(fd, httpStateFree, httpState); + fwdUnregister(fd, httpState->fwd); + if (request->flags.pinned) { + pinned = 1; + } else if (request->flags.connection_auth && request->flags.auth_sent) { + pinned = 1; + } + if (orig_request->pinned_connection && pinned) { + clientPinConnection(orig_request->pinned_connection, fd, orig_request, httpState->peer, request->flags.connection_auth); + } else if (httpState->peer) { + if (httpState->peer->options.originserver) + pconnPush(fd, httpState->peer->name, httpState->peer->http_port, httpState->orig_request->host, client_addr, client_port); + else + pconnPush(fd, httpState->peer->name, httpState->peer->http_port, NULL, client_addr, client_port); + } else { + pconnPush(fd, request->host, request->port, NULL, client_addr, client_port); + } + httpState->fd = -1; + httpStateFree(fd, httpState); + } else { + comm_close(fd); + } +} + /* This will be called when data is ready to be read from fd. Read until * error or connection closed. */ /* XXX this function is too long! */ @@ -622,19 +809,17 @@ httpReadReply(int fd, void *data) { HttpStateData *httpState = data; - LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); + LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF + 1); StoreEntry *entry = httpState->entry; - const request_t *request = httpState->request; - const request_t *orig_request = httpState->orig_request; - int len; + ssize_t len; int bin; int clen; + int done = 0; size_t read_sz = SQUID_TCP_SO_RCVBUF; - struct in_addr *client_addr = NULL; - u_short client_port = 0; #if DELAY_POOLS delay_id delay_id; #endif + int buffer_filled; if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { comm_close(fd); @@ -651,6 +836,7 @@ errno = 0; statCounter.syscalls.sock.reads++; len = FD_READ_METHOD(fd, buf, read_sz); + buffer_filled = len == read_sz; debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); if (len > 0) { fd_bytes(fd, len, FD_READ); @@ -663,6 +849,7 @@ for (clen = len - 1, bin = 0; clen; bin++) clen >>= 1; IOStats.Http.read_hist[bin]++; + buf[len] = '\0'; } if (!httpState->reply_hdr.size && len > 0 && fd_table[fd].uses > 1) { /* Skip whitespace */ @@ -711,13 +898,14 @@ fwdFail(httpState->fwd, errorCon(ERR_INVALID_RESP, HTTP_BAD_GATEWAY, httpState->fwd->request)); httpState->fwd->flags.dont_retry = 1; } else { - fwdComplete(httpState->fwd); + httpAppendBody(httpState, NULL, 0, -1); /* EOF */ + return; } comm_close(fd); return; } else { if (httpState->reply_hdr_state < 2) { - httpProcessReplyHeader(httpState, buf, len); + done = httpProcessReplyHeader(httpState, buf, len); if (httpState->reply_hdr_state == 2) { http_status s = entry->mem_obj->reply->sline.status; if (s == HTTP_HEADER_TOO_LARGE) { @@ -746,119 +934,8 @@ EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); } } - storeAppend(entry, buf, len); - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* - * the above storeAppend() call could ABORT this entry, - * in that case, the server FD should already be closed. - * there's nothing for us to do. - */ - return; - } - switch (httpPconnTransferDone(httpState)) { - case 1: - { - int keep_alive = 1; - /* - * If we didn't send a keep-alive request header, then this - * can not be a persistent connection. - */ - if (!httpState->flags.keepalive) - keep_alive = 0; - /* - * If we haven't sent the whole request then this can not be a persistent - * connection. - */ - if (!httpState->flags.request_sent) { - debug(11, 1) ("httpReadReply: Request not yet fully sent \"%s %s\"\n", - RequestMethodStr[orig_request->method], - storeUrl(entry)); - keep_alive = 0; - } - /* - * What does the reply have to say about keep-alive? - */ - if (!entry->mem_obj->reply->keep_alive) - keep_alive = 0; - /* - * Verify that the connection is clean - */ - if (len == read_sz) { - statCounter.syscalls.sock.reads++; - len = FD_READ_METHOD(fd, buf, SQUID_TCP_SO_RCVBUF); - if ((len < 0 && !ignoreErrno(errno)) || len == 0) { - keep_alive = 0; - } else if (len > 0) { - debug(11, Config.onoff.relaxed_header_parser <= 0 || keep_alive ? 1 : 2) - ("httpReadReply: Excess data from \"%s %s\"\n", - RequestMethodStr[orig_request->method], - storeUrl(entry)); - storeAppend(entry, buf, len); - keep_alive = 0; - } - } - if (keep_alive) { - int pinned = 0; -#if LINUX_TPROXY - if (orig_request->flags.tproxy) { - client_addr = &httpState->request->client_addr; - } -#endif - /* yes we have to clear all these! */ - commSetDefer(fd, NULL, NULL); - commSetTimeout(fd, -1, NULL, NULL); - commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); -#if DELAY_POOLS - delayClearNoDelay(fd); -#endif - comm_remove_close_handler(fd, httpStateFree, httpState); - fwdUnregister(fd, httpState->fwd); - if (request->flags.pinned) { - pinned = 1; - } else if (request->flags.connection_auth && request->flags.auth_sent) { - pinned = 1; - } - if (orig_request->pinned_connection && pinned) { - clientPinConnection(orig_request->pinned_connection, fd, orig_request, httpState->peer, request->flags.connection_auth); - } else if (httpState->peer) { - if (httpState->peer->options.originserver) - pconnPush(fd, httpState->peer->name, httpState->peer->http_port, httpState->orig_request->host, client_addr, client_port); - else - pconnPush(fd, httpState->peer->name, httpState->peer->http_port, NULL, client_addr, client_port); - } else { - pconnPush(fd, request->host, request->port, NULL, client_addr, client_port); - } - fwdComplete(httpState->fwd); - httpState->fd = -1; - httpStateFree(fd, httpState); - } else { - fwdComplete(httpState->fwd); - comm_close(fd); - } - } - return; - case 0: - /* Wait for more data or EOF condition */ - if (httpState->flags.keepalive_broken) { - commSetTimeout(fd, 10, NULL, NULL); - } else { - commSetTimeout(fd, Config.Timeout.read, NULL, NULL); - } - commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - return; - case -1: - /* Server is nasty on us. Shut down */ - debug(11, Config.onoff.relaxed_header_parser <= 0 || entry->mem_obj->reply->keep_alive ? 1 : 2) - ("httpReadReply: Excess data from \"%s %s\"\n", - RequestMethodStr[orig_request->method], - storeUrl(entry)); - fwdComplete(httpState->fwd); - comm_close(fd); - return; - default: - fatal("Unexpected httpPconnTransferDone() status\n"); - break; - } + httpAppendBody(httpState, buf + done, len - done, buffer_filled); + return; } } @@ -970,7 +1047,7 @@ debug(11, 5) ("httpBuildRequestHeader: %s: %s\n", strBuf(e->name), strBuf(e->value)); if (!httpRequestHdrAllowed(e, &strConnection)) { - debug(11, 2) ("'%s' header denied by anonymize_headers configuration\n", + debug(11, 2) ("'%s' header is a hop-by-hop connections header\n", strBuf(e->name)); continue; } @@ -1059,8 +1136,18 @@ if (!Config.onoff.via) httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); break; - case HDR_PROXY_CONNECTION: case HDR_CONNECTION: + case HDR_KEEP_ALIVE: + /* case HDR_PROXY_AUTHORIZATION: is special and handled above */ + case HDR_PROXY_AUTHENTICATE: + case HDR_TE: + case HDR_TRAILER: + case HDR_TRANSFER_ENCODING: + case HDR_UPGRADE: + case HDR_PROXY_CONNECTION: + case HDR_EXPECT: + /* hop-by-hop headers. Don't forward */ + break; case HDR_CACHE_CONTROL: /* append these after the loop if needed */ break; @@ -1232,7 +1319,7 @@ http_state_flags flags) { const int offset = mb->size; - memBufPrintf(mb, "%s %s HTTP/1.0\r\n", + memBufPrintf(mb, "%s %s HTTP/1.1\r\n", RequestMethodStr[request->method], strLen(request->urlpath) ? strBuf(request->urlpath) : "/"); /* build and pack headers */ Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.136 retrieving revision 1.136.6.1 diff -u -r1.136 -r1.136.6.1 --- squid/src/structs.h 29 Nov 2006 16:52:51 -0000 1.136 +++ squid/src/structs.h 19 Dec 2006 01:03:19 -0000 1.136.6.1 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.136 2006/11/29 16:52:51 squidadm Exp $ + * $Id: structs.h,v 1.136.6.1 2006/12/19 01:03:19 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1093,6 +1093,8 @@ unsigned int request_sent:1; unsigned int front_end_https:2; unsigned int originpeer:1; + unsigned int chunked:1; + unsigned int trailer:1; }; struct _HttpStateData { @@ -1108,6 +1110,8 @@ FwdState *fwd; char *body_buf; int body_buf_sz; + squid_off_t chunk_size; + String chunkhdr; }; struct _icpUdpData { Index: squid/tools/squidclient.c =================================================================== RCS file: /cvsroot/squid-sf//squid/tools/squidclient.c,v retrieving revision 1.5 retrieving revision 1.5.20.1 diff -u -r1.5 -r1.5.20.1 --- squid/tools/squidclient.c 4 Jun 2006 09:52:32 -0000 1.5 +++ squid/tools/squidclient.c 19 Dec 2006 01:03:20 -0000 1.5.20.1 @@ -1,6 +1,6 @@ /* - * $Id: squidclient.c,v 1.5 2006/06/04 09:52:32 squidadm Exp $ + * $Id: squidclient.c,v 1.5.20.1 2006/12/19 01:03:20 hno Exp $ * * DEBUG: section 0 WWW Client * AUTHOR: Harvest Derived @@ -142,6 +142,7 @@ " -i IMS If-Modified-Since time (in Epoch seconds).\n" " -h host Retrieve URL from cache on hostname. Default is localhost.\n" " -l host Specify a local IP address to bind to. Default is none.\n" + " -h hosthdr Host header content\n" " -p port Port number of cache. Default is %d.\n" " -m method Request method, default is GET.\n" " -t count Trace count cache-hops\n" @@ -152,7 +153,8 @@ " -u user Proxy authentication username\n" " -w password Proxy authentication password\n" " -U user WWW authentication username\n" - " -W password WWW authentication password\n", + " -W password WWW authentication password\n" + " -V version HTTP Version\n", progname, CACHE_HTTP_PORT); exit(1); } @@ -178,10 +180,12 @@ int i = 0, loops; long ping_int; long ping_min = 0, ping_max = 0, ping_sum = 0, ping_mean = 0; - char *proxy_user = NULL; - char *proxy_password = NULL; - char *www_user = NULL; - char *www_password = NULL; + const char *proxy_user = NULL; + const char *proxy_password = NULL; + const char *www_user = NULL; + const char *www_password = NULL; + const char *host = NULL; + const char *version = "1.1"; /* set the defaults */ hostname = "localhost"; @@ -201,18 +205,22 @@ url[BUFSIZ - 1] = '\0'; if (url[0] == '-') usage(argv[0]); - while ((c = getopt(argc, argv, "ah:l:P:i:km:p:rsvt:g:p:I:H:T:u:U:w:W:?")) != -1) + while ((c = getopt(argc, argv, "ah:j:V:l:P:i:km:p:rsvt:g:p:I:H:T:u:U:w:W:?")) != -1) switch (c) { case 'a': opt_noaccept = 1; break; case 'h': /* remote host */ - if (optarg != NULL) - hostname = optarg; + hostname = optarg; + break; + case 'j': + host = optarg; + break; + case 'V': + version = optarg; break; case 'l': /* local host */ - if (optarg != NULL) - localhost = optarg; + localhost = optarg; break; case 's': /* silent */ to_stdout = 0; @@ -308,7 +316,26 @@ #endif fstat(put_fd, &sb); } - snprintf(msg, BUFSIZ, "%s %s HTTP/1.0\r\n", method, url); + if (!host) { + char *newhost = strstr(url, "://"); + if (newhost) { + char *t; + newhost += 3; + newhost = strdup(newhost); + t = newhost + strcspn(newhost, "@/?"); + if (*t == '@') { + newhost = t + 1; + t = newhost + strcspn(newhost, "@/?"); + } + *t = '\0'; + host = newhost; + } + } + if (host) + snprintf(msg, BUFSIZ, "%s %s HTTP/%s\r\nHost: %s\r\n", method, url, version, host); + else + snprintf(msg, BUFSIZ, "%s %s HTTP/%s\r\n", method, url, version); + if (reload) { snprintf(buf, BUFSIZ, "Pragma: no-cache\r\n"); strcat(msg, buf); @@ -330,8 +357,8 @@ strcat(msg, buf); } if (proxy_user) { - char *user = proxy_user; - char *password = proxy_password; + const char *user = proxy_user; + const char *password = proxy_password; #if HAVE_GETPASS if (!password) password = getpass("Proxy password: "); @@ -345,8 +372,8 @@ strcat(msg, buf); } if (www_user) { - char *user = www_user; - char *password = www_password; + const char *user = www_user; + const char *password = www_password; #if HAVE_GETPASS if (!password) password = getpass("WWW password: "); @@ -359,16 +386,19 @@ snprintf(buf, BUFSIZ, "Authorization: Basic %s\r\n", base64_encode(buf)); strcat(msg, buf); } - if (keep_alive) { - if (port != 80) - snprintf(buf, BUFSIZ, "Proxy-Connection: keep-alive\r\n"); - else - snprintf(buf, BUFSIZ, "Connection: keep-alive\r\n"); - strcat(msg, buf); + if (strcmp(version, "1.0") == 0) { + if (keep_alive) { + if (strstr(url, "://")) + strcat(msg, "Proxy-Connection: keep-alive\r\n"); + else + strcat(msg, "Connection: keep-alive\r\n"); + } + } else { + if (!keep_alive) + strcat(msg, "Connection: close\r\n"); } strcat(msg, extra_hdrs); - snprintf(buf, BUFSIZ, "\r\n"); - strcat(msg, buf); + strcat(msg, "\r\n"); if (opt_verbose) fprintf(stderr, "headers: '%s'\n", msg);