--------------------- PatchSet 2287 Date: 2001/05/09 12:22:14 Author: rbcollins Branch: newhttp Tag: (none) Log: started modularising the broker Members: configure.in:1.1.1.3.8.11.2.8->1.1.1.3.8.11.2.8.2.1 src/Makefile.in:1.1.1.3.8.8.2.10.2.2->1.1.1.3.8.8.2.10.2.3 src/broker.h:1.1.2.2->1.1.2.3 src/client_side.c:1.1.1.3.4.1.4.15.2.34.2.7->1.1.1.3.4.1.4.15.2.34.2.8(DEAD) src/http.c:1.1.1.3.4.1.4.12.2.16.2.8->1.1.1.3.4.1.4.12.2.16.2.9(DEAD) src/protos.h:1.1.1.3.8.11.2.20.2.5->1.1.1.3.8.11.2.20.2.6 src/modules/http_client/Makefile.in:1.1->1.1.2.1 src/modules/http_client/client_side.c:1.1->1.1.2.1 src/modules/http_upstream/Makefile.in:1.1->1.1.2.1 src/modules/http_upstream/http.c:1.1->1.1.2.1 Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.1.1.3.8.11.2.8 retrieving revision 1.1.1.3.8.11.2.8.2.1 diff -u -r1.1.1.3.8.11.2.8 -r1.1.1.3.8.11.2.8.2.1 --- squid/configure.in 27 Apr 2001 14:38:37 -0000 1.1.1.3.8.11.2.8 +++ squid/configure.in 9 May 2001 12:22:14 -0000 1.1.1.3.8.11.2.8.2.1 @@ -3,13 +3,13 @@ dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.1.1.3.8.11.2.8 2001/04/27 14:38:37 rbcollins Exp $ +dnl $Id: configure.in,v 1.1.1.3.8.11.2.8.2.1 2001/05/09 12:22:14 rbcollins Exp $ dnl dnl dnl AC_INIT(src/main.c) AC_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.1.1.3.8.11.2.8 $)dnl +AC_REVISION($Revision: 1.1.1.3.8.11.2.8.2.1 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AC_CONFIG_AUX_DIR(cfgaux) @@ -893,6 +893,9 @@ CONF_MODULES="$CONF_MODULES `echo $enableval|sed -e 's/,/ /g;s/ */ /g'`" esac ]) +if test -z "$CONF_MODULES"; then + CONF_MODULES="http_upstream http_client" +fi if test -n "$CONF_MODULES"; then echo "Generic modules built: $CONF_MODULES" AC_SUBST(CONF_MODULES) Index: squid/src/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/Makefile.in,v retrieving revision 1.1.1.3.8.8.2.10.2.2 retrieving revision 1.1.1.3.8.8.2.10.2.3 diff -u -r1.1.1.3.8.8.2.10.2.2 -r1.1.1.3.8.8.2.10.2.3 --- squid/src/Makefile.in 7 May 2001 13:23:40 -0000 1.1.1.3.8.8.2.10.2.2 +++ squid/src/Makefile.in 9 May 2001 12:22:14 -0000 1.1.1.3.8.8.2.10.2.3 @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.1.1.3.8.8.2.10.2.2 2001/05/07 13:23:40 rbcollins Exp $ +# $Id: Makefile.in,v 1.1.1.3.8.8.2.10.2.3 2001/05/09 12:22:14 rbcollins Exp $ # # Uncomment and customize the following to suit your needs: # @@ -105,7 +105,6 @@ carp.o \ cbdata.o \ client_db.o \ - client_side.o \ comm.o \ comm_select.o \ debug.o \ @@ -125,7 +124,6 @@ gopher.o \ helper.o \ @HTCP_OBJS@ \ - http.o \ HttpStatusLine.o \ HttpHdrCc.o \ HttpHdrRange.o \ Index: squid/src/broker.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/broker.h,v retrieving revision 1.1.2.2 retrieving revision 1.1.2.3 diff -u -r1.1.2.2 -r1.1.2.3 --- squid/src/broker.h 1 May 2001 23:32:18 -0000 1.1.2.2 +++ squid/src/broker.h 9 May 2001 12:22:14 -0000 1.1.2.3 @@ -1,6 +1,6 @@ /* - * $Id: broker.h,v 1.1.2.2 2001/05/01 23:32:18 rbcollins Exp $ + * $Id: broker.h,v 1.1.2.3 2001/05/09 12:22:14 rbcollins Exp $ * * DEBUG: section 33 Protocol broker. * AUTHOR: Robert Collins @@ -45,3 +45,6 @@ extern void clientHttpInit(void); extern void serverHttpInit(void); +/* register an upstream protocol handler with the broker. */ +/* FIXME: how to tell cache vs peer vs upstream only apart? */ +extern void brokerRegisterUpstreamEntry(const char *, DATAFILTER_HDR *); --- squid/src/client_side.c Wed Feb 14 00:53:08 2007 +++ /dev/null Wed Feb 14 00:52:54 2007 @@ -1,3034 +0,0 @@ - -/* - * $Id: client_side.c,v 1.1.1.3.4.1.4.15.2.34.2.7 2001/05/07 23:09:15 rbcollins Exp $ - * - * DEBUG: section 33 Client-side Routines - * AUTHOR: Duane Wessels - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -#if IPF_TRANSPARENT -#if HAVE_SYS_IOCTL_H -#include -#endif -#include -#include -#if HAVE_IP_FIL_COMPAT_H -#include -#elif HAVE_NETINET_IP_FIL_COMPAT_H -#include -#elif HAVE_IP_COMPAT_H -#include -#elif HAVE_NETINET_IP_COMPAT_H -#include -#endif -#if HAVE_IP_FIL_H -#include -#elif HAVE_NETINET_IP_FIL_H -#include -#endif -#if HAVE_IP_NAT_H -#include -#elif HAVE_NETINET_IP_NAT_H -#include -#endif -#endif - -#if LINUX_NETFILTER -#include -#endif - -#if LINGERING_CLOSE -#define comm_close comm_lingering_close -#endif - -#include "broker.h" - -static const char *const crlf = "\r\n"; - -#define FAILURE_MODE_TIME 300 - -/* temp function */ -void -clientHttpInit(void) -{ -} - -/* Local functions */ - -static CWCB newclientWriteComplete; -static CWCB clientWriteBodyComplete; -static PF clientReadRequest; -static PF connStateFree; -static PF requestTimeout; -static PF clientLifetimeTimeout; -static int clientCheckTransferDone(clientHttpRequest *); -static void checkFailureRatio(err_type, hier_code); -static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri); -static clientHttpRequest *parseHttpRequest(ConnStateData *, method_t *, int *, size_t *); -static STCB clientHandleIMSReply; -static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); -static int checkAccelOnly(clientHttpRequest *); -#if USE_IDENT -static IDCB clientIdentDone; -#endif -static int clientOnlyIfCached(clientHttpRequest * http); -static void clientSetKeepaliveFlag(clientHttpRequest *); -static void clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb); -static void clientPackTermBound(String boundary, MemBuf * mb); -static DATAFILTER_HDR clientInterpretRequestHeaders; -static void clientProcessExpired(void *data); -static void clientProcessOnlyIfCachedMiss(clientHttpRequest * http); -static int clientCachable(clientHttpRequest * http); -static int clientHierarchical(clientHttpRequest * http); -static int clientCheckContentLength(request_t * r); -static DEFER httpAcceptDefer; -static int clientReplyBodyTooLarge(int clen); -static int clientRequestBodyTooLarge(int clen); -static void clientProcessBody(ConnStateData * conn); - - -static DATAFILTER_HDR clientTemp; -#if HEADERS_LOG -static DATAFILTER_HDR clientHeadersLog; -#endif -static DATAFILTER_HDR clientfdnote; -static DATAFILTER_HDR httplocalmethods; - -static int -checkAccelOnly(clientHttpRequest * http) -{ - /* return TRUE if someone makes a proxy request to us and - * we are in httpd-accel only mode */ - if (!Config2.Accel.on) - return 0; - if (Config.onoff.accel_with_proxy) - return 0; - if (http->request->protocol == PROTO_CACHEOBJ) - return 0; - if (http->flags.accel) - return 0; - return 1; -} - -#if USE_IDENT -static void -clientIdentDone(const char *ident, void *data) -{ - ConnStateData *conn = data; - xstrncpy(conn->rfc931, ident ? ident : dash_str, USER_IDENT_SZ); -} - -#endif - -static aclCheck_t * -clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http) -{ - aclCheck_t *ch; - ConnStateData *conn = http->conn; - ch = aclChecklistCreate(acl, - http->request, - conn->rfc931); - - /* - * hack for ident ACL. It needs to get full addresses, and a - * place to store the ident result on persistent connections... - */ - /* connection oriented auth also needs these two lines for it's operation. */ - ch->conn = conn; - cbdataLock(ch->conn); - - return ch; -} - -typedef struct _accessstate { - HttpReply *rep; - clientHttpRequest *request; - dlink_list * filter_list; - FILTER_list * filters; - unsigned int flags; - void *data; -} accessstate; - -CBDATA_TYPE(accessstate); - -DATAFILTER_FILTERHEADER(clientAccessCheck) -{ - clientHttpRequest *http = data; - accessstate *AccessState = NULL; - CBDATA_INIT_TYPE(accessstate); - AccessState = cbdataAlloc(accessstate); - AccessState->rep=rep; - AccessState->request=request; - AccessState->filter_list=filter_list; - AccessState->filters=filters; - AccessState->flags=flags; - AccessState->data=data; - cbdataLock(AccessState); - if (checkAccelOnly(http)) { - /* deny proxy requests in accel_only mode */ - debug(33, 1) ("clientAccessCheck: proxy request denied in accel_only mode\n"); - clientAccessCheckDone(ACCESS_DENIED, AccessState); - return FILTER_ABORT; - } - - http->acl_checklist = clientAclChecklistCreate(Config.accessList.http, http); - aclNBCheck(http->acl_checklist, clientAccessCheckDone, AccessState); - return 0; -} - -/* - * returns true if client specified that the object must come from the cache - * without contacting origin server - */ -static int -clientOnlyIfCached(clientHttpRequest * http) -{ - const request_t *r = http->request; - assert(r); - return r->cache_control && - EBIT_TEST(r->cache_control->mask, CC_ONLY_IF_CACHED); -} - -StoreEntry * -clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags) -{ - StoreEntry *e; - /* - * For erroneous requests, we might not have a h->request, - * so make a fake one. - */ - if (h->request == NULL) - h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); - e = storeCreateEntry(h->uri, h->log_uri, flags, m); - h->sc = storeClientListAdd(e, h); -#if DELAY_POOLS - delaySetStoreClient(h->sc, delayClient(h->request)); -#endif -/* storeClientCopy(h->sc, e, 0, 0, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, h); - */ return e; -} - -void -clientAccessCheckDone(int answer, void *data) -{ - accessstate *AccessState = data; - clientHttpRequest *http = AccessState->data; - err_type page_id; - http_status status; - ErrorState *err = NULL; - char *proxy_auth_msg = NULL; - FILTER_list *temp_filter; - temp_filter=AccessState->filters->node.next->data; - debug(33, 2) ("The request %s %s is %s, because it matched '%s'\n", - RequestMethodStr[http->request->method], http->uri, - answer == ACCESS_ALLOWED ? "ALLOWED" : "DENIED", - AclMatchedName ? AclMatchedName : "NO ACL's"); - proxy_auth_msg = authenticateAuthUserRequestMessage(http->conn->auth_user_request ? http->conn->auth_user_request : http->request->auth_user_request); - http->acl_checklist = NULL; - if (answer == ACCESS_ALLOWED) { -/* WHY DO THIS? We just created http->uri before - safe_free(http->uri); - http->uri = xstrdup(urlCanonical(http->request)); -*/ - temp_filter->filter_hdr(AccessState->rep, AccessState->request, AccessState->filter_list, temp_filter, AccessState->flags, temp_filter->data); - } else { - /* FIXME: tell the filter chain we are aborting */ - - debug(33, 5) ("Access Denied: %s\n", http->uri); - debug(33, 5) ("AclMatchedName = %s\n", - AclMatchedName ? AclMatchedName : ""); - debug(33, 5) ("Proxy Auth Message = %s\n", - proxy_auth_msg ? proxy_auth_msg : ""); - /* - * NOTE: get page_id here, based on AclMatchedName because - * if USE_DELAY_POOLS is enabled, then AclMatchedName gets - * clobbered in the clientCreateStoreEntry() call - * just below. Pedro Ribeiro - */ - page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName); - http->log_type = LOG_TCP_DENIED; - http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); - if (answer == ACCESS_REQ_PROXY_AUTH ) { -/* The test above was - * if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) { - * - * which is broken - the aclIs funciton didn't match proxy_auth_regex. - * But it also seems redundant - ACCESS_REQ_PROXY_AUTH is always set. - * Implementing aclIsProxyAuth is less efficient that checking a binary value. - * FIXME: if things fall through reimplementaclIsProxyAuth - */ - if (!http->flags.accel) { - /* Proxy authorisation needed */ - status = HTTP_PROXY_AUTHENTICATION_REQUIRED; - } else { - /* WWW authorisation needed */ - status = HTTP_UNAUTHORIZED; - } - if (page_id == ERR_NONE) - page_id = ERR_CACHE_ACCESS_DENIED; - } else { - status = HTTP_FORBIDDEN; - if (page_id == ERR_NONE) - page_id = ERR_ACCESS_DENIED; - } - err = errorCon(page_id, status); - err->request = requestLink(http->request); - err->src_addr = http->conn->peer.sin_addr; - if (http->conn->auth_user_request) - err->auth_user_request = http->conn->auth_user_request; - else if (http->request->auth_user_request) - err->auth_user_request = http->request->auth_user_request; - /* lock for the error state */ - if (err->auth_user_request) - authenticateAuthUserRequestLock(err->auth_user_request); - err->callback_data = NULL; - errorAppendEntry(http->entry, err); - } - dlinkDelete(&AccessState->filters->node, AccessState->filter_list); - xfree(AccessState->filters); - cbdataUnlock(AccessState); -} - -static void -clientUpdateCounters(clientHttpRequest * http) -{ - int svc_time = tvSubMsec(http->start, current_time); - ping_data *i; - HierarchyLogEntry *H; - statCounter.client_http.requests++; - if (isTcpHit(http->log_type)) - statCounter.client_http.hits++; - if (http->log_type == LOG_TCP_HIT) - statCounter.client_http.disk_hits++; - else if (http->log_type == LOG_TCP_MEM_HIT) - statCounter.client_http.mem_hits++; - if (http->request->err_type != ERR_NONE) - statCounter.client_http.errors++; - statHistCount(&statCounter.client_http.all_svc_time, svc_time); - /* - * The idea here is not to be complete, but to get service times - * for only well-defined types. For example, we don't include - * LOG_TCP_REFRESH_FAIL_HIT because its not really a cache hit - * (we *tried* to validate it, but failed). - */ - switch (http->log_type) { - case LOG_TCP_REFRESH_HIT: - statHistCount(&statCounter.client_http.nh_svc_time, svc_time); - break; - case LOG_TCP_IMS_HIT: - statHistCount(&statCounter.client_http.nm_svc_time, svc_time); - break; - case LOG_TCP_HIT: - case LOG_TCP_MEM_HIT: - case LOG_TCP_OFFLINE_HIT: - statHistCount(&statCounter.client_http.hit_svc_time, svc_time); - break; - case LOG_TCP_MISS: - case LOG_TCP_CLIENT_REFRESH_MISS: - statHistCount(&statCounter.client_http.miss_svc_time, svc_time); - break; - default: - /* make compiler warnings go away */ - break; - } - H = &http->request->hier; - switch (H->alg) { - case PEER_SA_DIGEST: - statCounter.cd.times_used++; - break; - case PEER_SA_ICP: - statCounter.icp.times_used++; - i = &H->ping; - if (0 != i->stop.tv_sec && 0 != i->start.tv_sec) - statHistCount(&statCounter.icp.query_svc_time, - tvSubUsec(i->start, i->stop)); - if (i->timeout) - statCounter.icp.query_timeouts++; - break; - case PEER_SA_NETDB: - statCounter.netdb.times_used++; - break; - default: - break; - } -} - -static void -httpRequestFree(void *data) -{ - clientHttpRequest *http = data; - clientHttpRequest **H; - ConnStateData *conn = http->conn; - StoreEntry *e; - request_t *request = http->request; - MemObject *mem = NULL; - debug(33, 3) ("httpRequestFree: %s\n", storeUrl(http->entry)); - if (!clientCheckTransferDone(http)) { - if (request && request->body_connection) - clientAbortBody(request); /* abort body transter */ -#if MYSTERIOUS_CODE - /* - * DW: this seems odd here, is it really needed? It causes - * incomplete transfers to get logged with "000" status - * code because http->entry becomes NULL. - */ - if ((e = http->entry)) { - http->entry = NULL; - storeUnregister(http->sc, e, http); - storeUnlockObject(e); - } -#endif - if (http->entry && http->entry->ping_status == PING_WAITING) - storeReleaseRequest(http->entry); - } - assert(http->log_type < LOG_TYPE_MAX); - if (http->entry) - mem = http->entry->mem_obj; - if (http->out.size || http->log_type) { - http->al.icp.opcode = ICP_INVALID; - http->al.url = http->log_uri; - debug(33, 9) ("httpRequestFree: al.url='%s'\n", http->al.url); - if (mem) { - http->al.http.code = mem->reply->sline.status; - http->al.http.content_type = strBuf(mem->reply->content_type); - } - http->al.cache.caddr = conn->log_addr; - http->al.cache.size = http->out.size; - http->al.cache.code = http->log_type; - http->al.cache.msec = tvSubMsec(http->start, current_time); - if (request) { - Packer p; - MemBuf mb; - memBufDefInit(&mb); - packerToMemInit(&p, &mb); - httpHeaderPackInto(&request->header, &p); - http->al.http.method = request->method; - http->al.http.version = request->http_ver; - http->al.headers.request = xstrdup(mb.buf); - http->al.hier = request->hier; - if (request->auth_user_request) { - http->al.cache.authuser = xstrdup(authenticateUserRequestUsername(request->auth_user_request)); - authenticateAuthUserRequestUnlock(request->auth_user_request); - request->auth_user_request = NULL; - } - if (conn->rfc931[0]) - http->al.cache.rfc931 = conn->rfc931; - packerClean(&p); - memBufClean(&mb); - } - accessLogLog(&http->al); - clientUpdateCounters(http); - clientdbUpdate(conn->peer.sin_addr, http->log_type, PROTO_HTTP, http->out.size); - } - if (http->acl_checklist) - aclChecklistFree(http->acl_checklist); - filterCleanChain(&http->repfilters); - if (request) - checkFailureRatio(request->err_type, http->al.hier.code); - safe_free(http->uri); - safe_free(http->log_uri); - safe_free(http->al.headers.request); - safe_free(http->al.headers.reply); - safe_free(http->redirect.location); - stringClean(&http->range_iter.boundary); - if ((e = http->entry)) { - http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; - storeUnlockObject(e); - } - /* old_entry might still be set if we didn't yet get the reply - * code in clientHandleIMSReply() */ - if ((e = http->old_entry)) { - http->old_entry = NULL; - storeUnregister(http->old_sc, e, http); - http->old_sc = NULL; - storeUnlockObject(e); - } - requestUnlink(http->request); - assert(http != http->next); - assert(http->conn->chr != NULL); - /* Unlink us from the clients request list */ - H = &http->conn->chr; - while (*H) { - if (*H == http) - break; - H = &(*H)->next; - } - assert(*H != NULL); - *H = http->next; - http->next = NULL; - dlinkDelete(&http->active, &ClientActiveRequests); - cbdataFree(http); -} - -/* This is a handler normally called by comm_close() */ -static void -connStateFree(int fd, void *data) -{ - ConnStateData *connState = data; - clientHttpRequest *http; - debug(33, 3) ("connStateFree: FD %d\n", fd); - assert(connState != NULL); - authenticateOnCloseConnection(connState); - clientdbEstablished(connState->peer.sin_addr, -1); /* decrement */ - while ((http = connState->chr) != NULL) { - assert(http->conn == connState); - assert(connState->chr != connState->chr->next); - httpRequestFree(http); - } - if (connState->in.size == CLIENT_REQ_BUF_SZ) - memFree(connState->in.buf, MEM_CLIENT_REQ_BUF); - else - safe_free(connState->in.buf); - /* XXX account connState->in.buf */ - pconnHistCount(0, connState->nrequests); - cbdataFree(connState); -#ifdef _SQUID_LINUX_ - /* prevent those nasty RST packets */ - { - char buf[SQUID_TCP_SO_RCVBUF]; - while (FD_READ_METHOD(fd, buf, SQUID_TCP_SO_RCVBUF) > 0); - } -#endif -} - - -DATAFILTER_FILTERHEADER (clientInterpretRequestHeaders) -{ - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; - clientHttpRequest * http=data; -// request_t *request = http->request; - const HttpHeader *req_hdr = &request->request->header; - int no_cache = 0; - const char *str; - request->request->imslen = -1; - request->request->ims = httpHeaderGetTime(req_hdr, HDR_IF_MODIFIED_SINCE); - if (request->request->ims > 0) - request->request->flags.ims = 1; - if (httpHeaderHas(req_hdr, HDR_PRAGMA)) { - String s = httpHeaderGetList(req_hdr, HDR_PRAGMA); - if (strListIsMember(&s, "no-cache", ',')) - no_cache++; - stringClean(&s); - } - request->request->cache_control = httpHeaderGetCc(req_hdr); - if (request->request->cache_control) - if (EBIT_TEST(request->request->cache_control->mask, CC_NO_CACHE)) - no_cache++; - /* Work around for supporting the Reload button in IE browsers - * when Squid is used as an accelerator or transparent proxy, - * by turning accelerated IMS request to no-cache requests. - * Now knows about IE 5.5 fix (is actually only fixed in SP1, - * but we can't tell whether we are talking to SP1 or not so - * all 5.5 versions are treated 'normally'). - */ - if (Config.onoff.ie_refresh) { - if (http->flags.accel && request->request->flags.ims) { - if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) { - if (strstr(str, "MSIE 5.01") != NULL) - no_cache++; - else if (strstr(str, "MSIE 5.0") != NULL) - no_cache++; - else if (strstr(str, "MSIE 4.") != NULL) - no_cache++; - else if (strstr(str, "MSIE 3.") != NULL) - no_cache++; - } - } - } - if (no_cache) { -#if HTTP_VIOLATIONS - if (Config.onoff.reload_into_ims) - request->request->flags.nocache_hack = 1; - else if (refresh_nocache_hack) - request->request->flags.nocache_hack = 1; - else -#endif - request->request->flags.nocache = 1; - } - /* ignore range header in non-GETs */ - if (request->request->method == METHOD_GET) { - request->request->range = httpHeaderGetRange(req_hdr); - if (request->request->range) - request->request->flags.range = 1; - } - if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) - request->request->flags.auth = 1; - if (request->request->login[0] != '\0') - request->request->flags.auth = 1; - if (httpHeaderHas(req_hdr, HDR_VIA)) { - String s = httpHeaderGetList(req_hdr, HDR_VIA); - /* - * ThisCache cannot be a member of Via header, "1.0 ThisCache" can. - * Note ThisCache2 has a space prepended to the hostname so we don't - * accidentally match super-domains. - */ - if (strListIsSubstr(&s, ThisCache2, ',')) { - debugObj(33, 1, "WARNING: Forwarding loop detected for:\n", - request->request, (ObjPackMethod) & httpRequestPack); - request->request->flags.loopdetect = 1; - } -#if FORW_VIA_DB - fvdbCountVia(strBuf(s)); -#endif - stringClean(&s); - } -#if USE_USERAGENT_LOG - if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) - logUserAgent(fqdnFromAddr(http->conn->peer.sin_addr), str); -#endif -#if USE_REFERER_LOG - if ((str = httpHeaderGetStr(req_hdr, HDR_REFERER))) - logReferer(fqdnFromAddr(http->conn->peer.sin_addr), str, - http->log_uri); -#endif -#if FORW_VIA_DB - if (httpHeaderHas(req_hdr, HDR_X_FORWARDED_FOR)) { - String s = httpHeaderGetList(req_hdr, HDR_X_FORWARDED_FOR); - fvdbCountForw(strBuf(s)); - stringClean(&s); - } -#endif - if (request->request->method == METHOD_TRACE) { - request->request->max_forwards = httpHeaderGetInt(req_hdr, HDR_MAX_FORWARDS); - } - if (clientCachable(http)) - request->request->flags.cachable = 1; - if (clientHierarchical(http)) - request->request->flags.hierarchical = 1; - debug(33, 5) ("clientInterpretRequestHeaders: REQ_NOCACHE = %s\n", - request->request->flags.nocache ? "SET" : "NOT SET"); - debug(33, 5) ("clientInterpretRequestHeaders: REQ_CACHABLE = %s\n", - request->request->flags.cachable ? "SET" : "NOT SET"); - debug(33, 5) ("clientInterpretRequestHeaders: REQ_HIERARCHICAL = %s\n", - request->request->flags.hierarchical ? "SET" : "NOT SET"); - rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); - dlinkDelete(&filters->node, filter_list); - xfree(filters); - return rvflags; -} - -/* - * clientSetKeepaliveFlag() sets request->flags.proxy_keepalive. - * This is the client-side persistent connection flag. We need - * to set this relatively early in the request processing - * to handle hacks for broken servers and clients. - */ -static void -clientSetKeepaliveFlag(clientHttpRequest * http) -{ - request_t *request = http->request; - const HttpHeader *req_hdr = &request->header; - debug(33, 3) ("clientSetKeepaliveFlag: http_ver = %d.%d\n", - request->http_ver.major, request->http_ver.minor); - debug(33, 3) ("clientSetKeepaliveFlag: method = %s\n", - RequestMethodStr[request->method]); - if (!Config.onoff.client_pconns) - request->flags.proxy_keepalive = 0; - else if (httpMsgIsPersistent(request->http_ver, req_hdr)) - request->flags.proxy_keepalive = 1; -} - -static int -clientCheckContentLength(request_t * r) -{ - switch (r->method) { - case METHOD_PUT: - case METHOD_POST: - /* PUT/POST requires a request entity */ - return (r->content_length >= 0); - case METHOD_GET: - case METHOD_HEAD: - /* We do not want to see a request entity on GET/HEAD requests */ - return (r->content_length <= 0); - default: - /* For other types of requests we don't care */ - return 1; - } - /* NOT REACHED */ -} - -static int -clientCachable(clientHttpRequest * http) -{ - const char *url = http->uri; - request_t *req = http->request; - method_t method = req->method; - if (req->protocol == PROTO_HTTP) - return httpCachable(method); - /* FTP is always cachable */ - if (req->protocol == PROTO_WAIS) - return 0; - if (method == METHOD_CONNECT) - return 0; - if (method == METHOD_TRACE) - return 0; - if (method == METHOD_PUT) - return 0; - if (method == METHOD_POST) - return 0; /* XXX POST may be cached sometimes.. ignored for now */ - if (req->protocol == PROTO_GOPHER) - return gopherCachable(url); - if (req->protocol == PROTO_CACHEOBJ) - return 0; - return 1; -} - -/* Return true if we can query our neighbors for this object */ -static int -clientHierarchical(clientHttpRequest * http) -{ - const char *url = http->uri; - request_t *request = http->request; - method_t method = request->method; - const wordlist *p = NULL; - - /* IMS needs a private key, so we can use the hierarchy for IMS only - * if our neighbors support private keys */ - if (request->flags.ims && !neighbors_do_private_keys) - return 0; - if (request->flags.auth) - return 0; - if (method == METHOD_TRACE) - return 1; - if (method != METHOD_GET) - return 0; - /* scan hierarchy_stoplist */ - for (p = Config.hierarchy_stoplist; p; p = p->next) - if (strstr(url, p->key)) - return 0; - if (request->flags.loopdetect) - return 0; - if (request->protocol == PROTO_HTTP) - return httpCachable(method); - if (request->protocol == PROTO_GOPHER) - return gopherCachable(url); - if (request->protocol == PROTO_WAIS) - return 0; - if (request->protocol == PROTO_CACHEOBJ) - return 0; - return 1; -} - -int -isTcpHit(log_type code) -{ - /* this should be a bitmap for better optimization */ - if (code == LOG_TCP_HIT) - return 1; - if (code == LOG_TCP_IMS_HIT) - return 1; - if (code == LOG_TCP_REFRESH_FAIL_HIT) - return 1; - if (code == LOG_TCP_REFRESH_HIT) - return 1; - if (code == LOG_TCP_NEGATIVE_HIT) - return 1; - if (code == LOG_TCP_MEM_HIT) - return 1; - if (code == LOG_TCP_OFFLINE_HIT) - return 1; - return 0; -} - -/* - * returns true if If-Range specs match reply, false otherwise - */ -static int -clientIfRangeMatch(clientHttpRequest * http, HttpReply * rep) -{ - const TimeOrTag spec = httpHeaderGetTimeOrTag(&http->request->header, HDR_IF_RANGE); - /* check for parsing falure */ - if (!spec.valid) - return 0; - /* got an ETag? */ - if (spec.tag.str) { - ETag rep_tag = httpHeaderGetETag(&rep->header, HDR_ETAG); - debug(33, 3) ("clientIfRangeMatch: ETags: %s and %s\n", - spec.tag.str, rep_tag.str ? rep_tag.str : ""); - if (!rep_tag.str) - return 0; /* entity has no etag to compare with! */ - if (spec.tag.weak || rep_tag.weak) { - debug(33, 1) ("clientIfRangeMatch: Weak ETags are not allowed in If-Range: %s ? %s\n", - spec.tag.str, rep_tag.str); - return 0; /* must use strong validator for sub-range requests */ - } - return etagIsEqual(&rep_tag, &spec.tag); - } - /* got modification time? */ - if (spec.time >= 0) { - return http->entry->lastmod <= spec.time; - } - assert(0); /* should not happen */ - return 0; -} - -/* returns expected content length for multi-range replies - * note: assumes that httpHdrRangeCanonize has already been called - * warning: assumes that HTTP headers for individual ranges at the - * time of the actuall assembly will be exactly the same as - * the headers when clientMRangeCLen() is called */ -static int -clientMRangeCLen(clientHttpRequest * http) -{ - int clen = 0; - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec; - MemBuf mb; - - assert(http->entry->mem_obj); - - memBufDefInit(&mb); - while ((spec = httpHdrRangeGetSpec(http->request->range, &pos))) { - - /* account for headers for this range */ - memBufReset(&mb); - clientPackRangeHdr(http->entry->mem_obj->reply, - spec, http->range_iter.boundary, &mb); - clen += mb.size; - - /* account for range content */ - clen += spec->length; - - debug(33, 6) ("clientMRangeCLen: (clen += %d + %d) == %d\n", - mb.size, spec->length, clen); - } - /* account for the terminating boundary */ - memBufReset(&mb); - clientPackTermBound(http->range_iter.boundary, &mb); - clen += mb.size; - - memBufClean(&mb); - return clen; -} - -static DATAFILTER clientDoRangeReply; - -/* adds appropriate Range headers if needed */ -static void -clientBuildRangeHeader(clientHttpRequest * http, HttpReply * rep) -{ - HttpHeader *hdr = rep ? &rep->header : 0; - const char *range_err = NULL; - request_t *request = http->request; - int is_hit = isTcpHit(http->log_type); - assert(request->range); - /* check if we still want to do ranges */ - if (!rep) - range_err = "no [parse-able] reply"; - else if (rep->sline.status != HTTP_OK) - range_err = "wrong status code"; - else if (httpHeaderHas(hdr, HDR_CONTENT_RANGE)) - range_err = "origin server does ranges"; - else if (rep->content_length < 0) - range_err = "unknown length"; - else if (rep->content_length != http->entry->mem_obj->reply->content_length) - range_err = "INCONSISTENT length"; /* a bug? */ - else if (httpHeaderHas(&http->request->header, HDR_IF_RANGE) && !clientIfRangeMatch(http, rep)) - range_err = "If-Range match failed"; - else if (!httpHdrRangeCanonize(http->request->range, rep->content_length)) - range_err = "canonization failed"; - else if (httpHdrRangeIsComplex(http->request->range)) - range_err = "too complex range header"; - else if (!request->flags.cachable) /* from we_do_ranges in http.c */ - range_err = "non-cachable request"; - else if (!is_hit && httpHdrRangeOffsetLimit(http->request->range)) - range_err = "range outside range_offset_limit"; - /* get rid of our range specs on error */ - if (range_err) { - debug(33, 3) ("clientBuildRangeHeader: will not do ranges: %s.\n", range_err); - httpHdrRangeDestroy(http->request->range); - http->request->range = NULL; - } else { - const int spec_count = http->request->range->specs.count; - int actual_clen = -1; - - debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %d\n", - spec_count, rep->content_length); - assert(spec_count > 0); - /* ETags should not be returned with Partial Content replies? */ - httpHeaderDelById(hdr, HDR_ETAG); - /* append appropriate header(s) */ - if (spec_count == 1) { - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec = httpHdrRangeGetSpec(http->request->range, &pos); - assert(spec); - /* append Content-Range */ - httpHeaderAddContRange(hdr, *spec, rep->content_length); - /* set new Content-Length to the actual number of bytes - * transmitted in the message-body */ - actual_clen = spec->length; - } else { - /* multipart! */ - /* generate boundary string */ - http->range_iter.boundary = httpHdrRangeBoundaryStr(http); - /* delete old Content-Type, add ours */ - httpHeaderDelById(hdr, HDR_CONTENT_TYPE); - httpHeaderPutStrf(hdr, HDR_CONTENT_TYPE, - "multipart/byteranges; boundary=\"%s\"", - strBuf(http->range_iter.boundary)); - /* Content-Length is not required in multipart responses - * but it is always nice to have one */ - actual_clen = clientMRangeCLen(http); - } - - /* replace Content-Length header */ - assert(actual_clen >= 0); - httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); - httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, actual_clen); - debug(33, 3) ("clientBuildRangeHeader: actual content length: %d\n", actual_clen); - - /* replace the status line */ - httpStatusLineSet(&rep->sline, rep->sline.version, HTTP_PARTIAL_CONTENT, NULL); - - /* add the ranges filter to the list */ - filterChainAddTail(&http->repfilters,clientDoRangeReply, identity_header, NULL,http); - } -} - -/* Transfer Encoding Filters: They work like this.. - - filter (a,b,c,d,e) - a - is a character pointer to the data to be filtered - b - is length of a - c - is address of a character pointer where the resulting buffer should be stored - d - pointer to integer to store length of c in - e - pointer to a general data pointer that the filter may use to maintain state, - it begins life as NULL. - -the return value is a 4 bit mask - TE_BUFFER_ALLOCATED - c is a newly allocated buffer and should be freed when the - calling function has completed the filter - TE_CHUNK_A - reserved internally for chunking MAGIC - TE_CHUNK_B - reserved internally for chunking MAGIC - TE_BUFFERING_OUTPUT - set this bit if your funciton was called with input, but did not - produce any (for instance if you're buffering it in the context provided by e.) - prevents squid from thinking EOF has been reached. - - call sequence - first call: *e is NULL but a is not and b>0 - body calls : a is not null, b>0 and value of *e determined by previous calls - last call: b==0.. good time to clean up *e if you've stored stuff there.. you - may produce output if necessary, but last call will be repeated. -*/ - -#ifndef _TE_RVALUES_ -#define _TE_RVALUES_ -#define TE_BUFFER_ALLOCATED 0x01 -/* TE_CHUNK_A = allocated temp buffer with empty response because input was length 0 - & no data had previous been processed - empty buffer case */ -/* could also be just that chunking was done */ -#define TE_CHUNK_A 0x02 -#define TE_CHUNK_B 0x04 -#define TE_BUFFERING_OUTPUT 0x08 -#endif - - -static DATAFILTER_HDR clientWriteReplyHeaders; -static DATAFILTER clientDoCommWriteMemBuf; -static DATAFILTER_HDR clientDoCommWriteHeaders; -static DATAFILTER_HDR newClientWH; -static DATAFILTER newClientW; -static DATAFILTER http_client_body; -static DATAFILTER_HDR http_client_hdr; - -void -clientTEReplyheader(clientHttpRequest * http, HttpReply * rep, dlink_list *filter_list) { - HttpHeader *hdr = &rep->header; - int y, may_apply_tes; - const char *s_ct; - String s_ce; - - /* begin list of modules here */ - may_apply_tes = 0; - - if (httpHeaderHas(hdr, HDR_CONTENT_TYPE)) { - s_ct = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE); - y = strlen(s_ct); - /* TODO: allow an acl for encoding by content-type */ - if (((y >= 5) && !strncasecmp(s_ct, "text/", 5)) - || strstr(s_ct, "postscript")) { - may_apply_tes = 1; - debug(33, 8) ("Content-Type of Application: %s\n", s_ct); - } - } - if (httpHeaderHas(hdr, HDR_CONTENT_ENCODING)) { - /* TODO: check logic here & make this an acl test */ - /* oh, base entity might be compressed.. don't want - * to double *that* */ - s_ce = httpHeaderGetList(hdr, HDR_CONTENT_ENCODING); - if ((strListIsMember(&s_ce, "gzip", ',')) || - (strListIsMember(&s_ce, "deflate", ',')) || - (strListIsMember(&s_ce, "x-deflate", ',')) || - (strListIsMember(&s_ce, "x-gzip", ',')) || - (strListIsMember(&s_ce, "x-compress", ',')) || - (strListIsMember(&s_ce, "compress", ','))) - may_apply_tes |= 0x02; /* 02 is the compress exception */ - stringClean(&s_ce); - } - - te_build_encode_xlate_list(&http->request->header, hdr, filter_list, http->request->http_ver, &http->request->flags); -} - -/* put terminating boundary for multiparts */ -static void -clientPackTermBound(String boundary, MemBuf * mb) -{ - memBufPrintf(mb, "\r\n--%s--\r\n", strBuf(boundary)); - debug(33, 6) ("clientPackTermBound: buf offset: %d\n", mb->size); -} - -/* appends a "part" HTTP header (as in a multi-part/range reply) to the buffer */ -static void -clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb) -{ - HttpHeader hdr; - Packer p; - assert(rep); - assert(spec); - - /* put boundary */ - debug(33, 5) ("clientPackRangeHdr: appending boundary: %s\n", strBuf(boundary)); - /* rfc2046 requires to _prepend_ boundary with ! */ - memBufPrintf(mb, "\r\n--%s\r\n", strBuf(boundary)); - - /* stuff the header with required entries and pack it */ - httpHeaderInit(&hdr, hoReply); - if (httpHeaderHas(&rep->header, HDR_CONTENT_TYPE)) - httpHeaderPutStr(&hdr, HDR_CONTENT_TYPE, httpHeaderGetStr(&rep->header, HDR_CONTENT_TYPE)); - httpHeaderAddContRange(&hdr, *spec, rep->content_length); - packerToMemInit(&p, mb); - httpHeaderPackInto(&hdr, &p); - packerClean(&p); - httpHeaderClean(&hdr); - - /* append (we packed a header, not a reply) */ - memBufPrintf(mb, crlf); -} - -/* - * extracts a "range" from *buf and appends them to mb, updating - * all offsets and such. - */ -static void -clientPackRange(clientHttpRequest * http, - HttpHdrRangeIter * i, - const char **buf, - ssize_t * size, - MemBuf * mb) -{ - const ssize_t copy_sz = i->debt_size <= *size ? i->debt_size : *size; - off_t body_off = http->out.offset - i->prefix_size; - assert(*size > 0); - assert(i->spec); - /* - * intersection of "have" and "need" ranges must not be empty - */ - assert(body_off < i->spec->offset + i->spec->length); - assert(body_off + *size > i->spec->offset); - /* - * put boundary and headers at the beginning of a range in a - * multi-range - */ - if (http->request->range->specs.count > 1 && i->debt_size == i->spec->length) { - assert(http->entry->mem_obj); - clientPackRangeHdr( - http->entry->mem_obj->reply, /* original reply */ - i->spec, /* current range */ - i->boundary, /* boundary, the same for all */ - mb - ); - } - /* - * append content - */ - debug(33, 3) ("clientPackRange: appending %d bytes\n", copy_sz); - memBufAppend(mb, *buf, copy_sz); - /* - * update offsets - */ - *size -= copy_sz; - i->debt_size -= copy_sz; - body_off += copy_sz; - *buf += copy_sz; - http->out.offset = body_off + i->prefix_size; /* sync */ - /* - * paranoid check - */ - assert(*size >= 0 && i->debt_size >= 0); -} - -/* returns true if there is still data available to pack more ranges - * increments iterator "i" - * used by clientPackMoreRanges */ -static int -clientCanPackMoreRanges(const clientHttpRequest * http, HttpHdrRangeIter * i, ssize_t size) -{ - /* first update "i" if needed */ - if (!i->debt_size) { - if ((i->spec = httpHdrRangeGetSpec(http->request->range, &i->pos))) - i->debt_size = i->spec->length; - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - /* continue condition: need_more_data && have_more_data */ - return i->spec && size > 0; -} - -/* extracts "ranges" from buf and appends them to mb, updating all offsets and such */ -/* returns true if we need more data */ -static int -clientPackMoreRanges(clientHttpRequest * http, const char *buf, ssize_t size, MemBuf * mb) -{ - HttpHdrRangeIter *i = &http->range_iter; - /* offset in range specs does not count the prefix of an http msg */ - off_t body_off = http->out.offset - i->prefix_size; - assert(size >= 0); - /* check: reply was parsed and range iterator was initialized */ - assert(i->prefix_size > 0); - /* filter out data according to range specs */ - while (clientCanPackMoreRanges(http, i, size)) { - off_t start; /* offset of still missing data */ - assert(i->spec); - start = i->spec->offset + i->spec->length - i->debt_size; - debug(33, 3) ("clientPackMoreRanges: in: offset: %d size: %d\n", - (int) body_off, size); - debug(33, 3) ("clientPackMoreRanges: out: start: %d spec[%d]: [%d, %d), len: %d debt: %d\n", - (int) start, (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length, i->debt_size); - assert(body_off <= start); /* we did not miss it */ - /* skip up to start */ - if (body_off + size > start) { - const size_t skip_size = start - body_off; - body_off = start; - size -= skip_size; - buf += skip_size; - } else { - /* has not reached start yet */ - body_off += size; - size = 0; - buf = NULL; - } - /* put next chunk if any */ - if (size) { - http->out.offset = body_off + i->prefix_size; /* sync */ - clientPackRange(http, i, &buf, &size, mb); - body_off = http->out.offset - i->prefix_size; /* sync */ - } - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - debug(33, 3) ("clientPackMoreRanges: buf exhausted: in: offset: %d size: %d need_more: %d\n", - (int) body_off, size, i->debt_size); - if (i->debt_size) { - debug(33, 3) ("clientPackMoreRanges: need more: spec[%d]: [%d, %d), len: %d\n", - (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length); - /* skip the data we do not need if possible */ - if (i->debt_size == i->spec->length) /* at the start of the cur. spec */ - body_off = i->spec->offset; - else - assert(body_off == i->spec->offset + i->spec->length - i->debt_size); - } else if (http->request->range->specs.count > 1) { - /* put terminating boundary for multiparts */ - clientPackTermBound(i->boundary, mb); - } - http->out.offset = body_off + i->prefix_size; /* sync */ - return i->debt_size > 0; -} - -static int -clientReplyBodyTooLarge(int clen) -{ - if (0 == Config.maxReplyBodySize) - return 0; /* disabled */ - if (clen < 0) - return 0; /* unknown */ - if (clen > Config.maxReplyBodySize) - return 1; /* too large */ - return 0; -} - -static int -clientRequestBodyTooLarge(int clen) -{ - if (0 == Config.maxRequestBodySize) - return 0; /* disabled */ - if (clen < 0) - return 0; /* unknown, bug? */ - if (clen > Config.maxRequestBodySize) - return 1; /* too large */ - return 0; -} - - -DATAFILTER clientDoCommWrite; - -DATAFILTER_FILTERHEADER(clientWriteReplyHeaders) { - clientHttpRequest *http = data; - FILTER_list *temp_filter; - unsigned int rvflags=0; - /* at this point, we are the first called filter, and only the reply headers - * are in buf. - * we _know_ buf is a membuf. - * we _don't know_ if other filters plan to touch the headers - * so all we do is pass them on, remove self from the list. - */ - - /* other filters can _depend_ on the test, http->request.out_offset==0 - * to mean that they are recieving a block that has _nothing but headers_ - */ - debug (33,8)("clientWriteReplyHeaders: Beginning\n"); - temp_filter=filters->node.next->data; - rvflags |= temp_filter->filter_hdr(http->entry->mem_obj->reply,http,filter_list, temp_filter, flags | FILTER_HTTP_HEADER, temp_filter->data); - dlinkDelete(&filters->node, filter_list); - xfree(filters); - http->flags.done_reply_headers=1; - debug (33,8)("clientWriteReplyHeaders: Finished\n"); - return rvflags; -} - -DATAFILTER_FILTER(clientDoRangeReply) { - clientHttpRequest *http = data; - FILTER_list *temp_filter; - unsigned int rvflags=0; - /* while we are only put here when expecting range requests.. I don't know if the - * request-> range variable ever gets altered. For safety this test stays (for now). - */ - - /* the membuf is temporary. Next step is to pack the ranges direct as a series of - * calls to the next filter - */ - - debug (33,8)("clientDoRangeReply: Beginning\n"); - temp_filter=filters->node.next->data; - - if (!http->flags.done_reply_headers) { - /* skip the header packet - we don't care about it */ - return temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data); - } - if (http->request->range) { - MemBuf mb; - memBufDefInit(&mb); - /* Only GET requests should have ranges */ - assert(http->request->method == METHOD_GET); - /* clientPackMoreRanges() updates http->out.offset */ - /* force the end of the transfer if we are done */ - if (!clientPackMoreRanges(http, buf, len, &mb)) - http->flags.done_copying = 1; - rvflags |= temp_filter->filter(mb.buf, mb.size, offset, filter_list, temp_filter, flags, temp_filter->data); - memBufFreeFunc(&mb); - } else { - debug(1,1)("**************\nBUG DoRangeReply called for a non range request\n************\n"); - rvflags|= temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data); - dlinkDelete(&filters->node, filter_list); - xfree(filters); - } - debug (33,8)("clientDoRangeReply: Finished\n"); - return rvflags; -} - -/* Responses with no body will not have a content-type header, - * which breaks the rep_mime_type acl, which - * coincidentally, is the most common acl for reply access lists. - * A better long term fix for this is to allow acl matchs on the various - * status codes, and then supply a default ruleset that puts these - * codes before any user defines access entries. That way the user - * can choose to block these responses where appropriate, but won't get - * mysterious breakages. - */ -static int -clientAlwaysAllowResponse(http_status sline) -{ - switch (sline) { - case HTTP_CONTINUE: - case HTTP_SWITCHING_PROTOCOLS: - case HTTP_PROCESSING: - case HTTP_NO_CONTENT: - case HTTP_NOT_MODIFIED: - return 1; - /* unreached */ - break; - default: - return 0; - } -} - - -static void -clientKeepaliveNextRequest(clientHttpRequest * http) -{ - ConnStateData *conn = http->conn; - StoreEntry *entry; - debug(33, 3) ("clientKeepaliveNextRequest: FD %d\n", conn->fd); - conn->defer.until = 0; /* Kick it to read a new request */ - httpRequestFree(http); - if ((http = conn->chr) == NULL) { - debug(33, 5) ("clientKeepaliveNextRequest: FD %d reading next req\n", - conn->fd); - fd_note(conn->fd, "Waiting for next request"); - -/* - * Set the timeout BEFORE calling clientReadRequest(). - */ - commSetTimeout(conn->fd, Config.Timeout.pconn, requestTimeout, conn); - /* - * CYGWIN has a problem and is blocking on read() requests when there - * is no data present. - * This hack may hit performance a little, but it's better than - * blocking!. - */ -#ifdef _SQUID_CYGWIN_ - commSetSelect(conn->fd, COMM_SELECT_READ, clientReadRequest, conn, 0); -#else - clientReadRequest(conn->fd, conn); /* Read next request */ -#endif - /* - * Note, the FD may be closed at this point. - */ - } else if ((entry = http->entry) == NULL) { - /* - * this request is in progress, maybe doing an ACL or a redirect, - * execution will resume after the operation completes. - */ - } else { - debug(33, 1) ("clientKeepaliveNextRequest: FD %d Sending next\n", - conn->fd); - assert(entry); - if (0 == storeClientCopyPending(http->sc, entry, http)) { - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) - debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); -/* storeClientCopy(http->sc, entry, - http->out.offset, - http->out.offset, - CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), - clientSendMoreData, - http); -*/ } - } -} - -/* - * client issued a request with an only-if-cached cache-control directive; - * we did not find a cached object that can be returned without - * contacting other servers; - * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068] - */ -static void -clientProcessOnlyIfCachedMiss(clientHttpRequest * http) -{ - char *url = http->uri; - request_t *r = http->request; - ErrorState *err = NULL; - debug(33, 4) ("clientProcessOnlyIfCachedMiss: '%s %s'\n", - RequestMethodStr[r->method], url); - http->al.http.code = HTTP_GATEWAY_TIMEOUT; - err = errorCon(ERR_ONLY_IF_CACHED_MISS, HTTP_GATEWAY_TIMEOUT); - err->request = requestLink(r); - err->src_addr = http->conn->peer.sin_addr; - if (http->entry) { - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; - storeUnlockObject(http->entry); - } - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); - errorAppendEntry(http->entry, err); -} - -/* - * Return true if we should force a cache miss on this range request. - * entry must be non-NULL. - */ -static int -clientCheckRangeForceMiss(StoreEntry * entry, HttpHdrRange * range) -{ - /* - * If the range_offset_limit is NOT in effect, there - * is no reason to force a miss. - */ - if (0 == httpHdrRangeOffsetLimit(range)) - return 0; - /* - * Here, we know it's possibly a hit. If we already have the - * whole object cached, we won't force a miss. - */ - if (STORE_OK == entry->store_status) - return 0; /* we have the whole object */ - /* - * Now we have a hit on a PENDING object. We need to see - * if the part we want is already cached. If so, we don't - * force a miss. - */ - assert(NULL != entry->mem_obj); - if (httpHdrRangeFirstOffset(range) <= entry->mem_obj->inmem_hi) - return 0; - /* - * Even though we have a PENDING copy of the object, we - * don't want to wait to reach the first range offset, - * so we force a miss for a new range request to the - * origin. - */ - return 1; -} - -static clientHttpRequest * -parseHttpRequestAbort(ConnStateData * conn, const char *uri) -{ - clientHttpRequest *http; - http = cbdataAlloc(clientHttpRequest); - http->conn = conn; - http->start = current_time; - http->req_sz = conn->in.offset; - http->uri = xstrdup(uri); - http->log_uri = xstrndup(uri, MAX_URL); - http->range_iter.boundary = StringNull; - dlinkAdd(http, &http->active, &ClientActiveRequests); - return http; -} - -/* - * parseHttpRequest() - * - * Returns - * NULL on error or incomplete request - * a clientHttpRequest structure on success - */ -static clientHttpRequest * -parseHttpRequest(ConnStateData * conn, method_t * method_p, int *status, - size_t * req_line_sz_p) -{ - char *inbuf = NULL; - char *mstr = NULL; - char *url = NULL; - char *req_hdr = NULL; - http_version_t http_ver; - char *token = NULL; - char *t = NULL; - char *lprefix; - char **prefix_p=&lprefix; - char *end; - size_t header_sz; /* size of headers, not including first line */ - size_t prefix_sz; /* size of whole request (req-line + headers) */ - size_t url_sz; - size_t req_sz; - method_t method; - clientHttpRequest *http = NULL; - HttpHeader hdr; - request_t *request = NULL; -#if IPF_TRANSPARENT - struct natlookup natLookup; - static int natfd = -1; - static int siocgnatl_cmd = SIOCGNATL & 0xff; - int x; -#endif -#if LINUX_NETFILTER - size_t sock_sz = sizeof(conn->me); -#endif - - if ((req_sz = headersEnd(conn->in.buf, conn->in.offset)) == 0) { - debug(33, 5) ("Incomplete request, waiting for end of headers\n"); - *status = 0; - *prefix_p = NULL; - *method_p = METHOD_NONE; - return NULL; - } - assert(req_sz <= conn->in.offset); - /* Use memcpy, not strdup! */ - inbuf = xmalloc(req_sz + 1); -/* FIXME: wecopy here because we alter the data! */ - xmemcpy(inbuf, conn->in.buf, req_sz); - *(inbuf + req_sz) = '\0'; - - /* pre-set these values to make aborting simpler */ - *prefix_p = inbuf; - *method_p = METHOD_NONE; - *status = -1; - - /* Barf on NULL characters in the headers */ - if (strlen(inbuf) != req_sz) { - debug(33, 1) ("parseHttpRequest: Requestheader contains NULL characters\n"); - return parseHttpRequestAbort(conn, "error:invalid-request"); - } - /* Look for request method */ - if ((mstr = strtok(inbuf, "\t ")) == NULL) { - debug(33, 1) ("parseHttpRequest: Can't get request method\n"); - return parseHttpRequestAbort(conn, "error:invalid-request-method"); - } - method = urlParseMethod(mstr); - if (method == METHOD_NONE) { - debug(33, 1) ("parseHttpRequest: Unsupported method '%s'\n", mstr); - return parseHttpRequestAbort(conn, "error:unsupported-request-method"); - } - debug(33, 5) ("parseHttpRequest: Method is '%s'\n", mstr); - *method_p = method; - - /* look for URL+HTTP/x.x */ - if ((url = strtok(NULL, "\n")) == NULL) { - debug(33, 1) ("parseHttpRequest: Missing URL\n"); - return parseHttpRequestAbort(conn, "error:missing-url"); - } - while (xisspace(*url)) - url++; - t = url + strlen(url); - assert(*t == '\0'); - token = NULL; - while (t > url) { - t--; - if (xisspace(*t) && !strncmp(t + 1, "HTTP/", 5)) { - token = t + 1; - break; - } - } - while (t > url && xisspace(*t)) - *(t--) = '\0'; - debug(33, 5) ("parseHttpRequest: URI is '%s'\n", url); - if (token == NULL) { - debug(33, 3) ("parseHttpRequest: Missing HTTP identifier\n"); -#if RELAXED_HTTP_PARSER - httpBuildVersion(&http_ver, 0, 9); /* wild guess */ -#else - return parseHttpRequestAbort(conn, "error:missing-http-ident"); -#endif - } else { - if (sscanf(token + 5, "%d.%d", &http_ver.major, &http_ver.minor) != 2) { - debug(33, 3) ("parseHttpRequest: Invalid HTTP identifier.\n"); - return parseHttpRequestAbort(conn, "error: invalid HTTP-ident"); - } - debug(33, 6) ("parseHttpRequest: Client HTTP version %d.%d.\n", http_ver.major, http_ver.minor); - } - - /* - * Process headers after request line - */ - req_hdr = strtok(NULL, null_string); - header_sz = req_sz - (req_hdr - inbuf); - - httpHeaderInit(&hdr, hoRequest); - /* httpRequestParseHeader uses MsgIsolateHeaders ..*/ - if (!httpHeaderParse(&hdr, req_hdr, req_hdr + header_sz)) { - debug(33, 3) ("parseHttpRequest: couldn't parse header\n"); - return NULL; - } - -/* Dump the headers */ -{ - HttpHeaderPos pos = HttpHeaderInitPos; - const HttpHeaderEntry *e; - debug(55, 7) ("packing hdr: (%p)\n", &hdr); - /* pack all entries one by one */ - while ((e = httpHeaderGetEntry(&hdr, &pos))) -printf("%s: %s\r\n",strBuf(e->name),strBuf(e->value)); -} - - - if (0 == header_sz) { - debug(33, 3) ("parseHttpRequest: header_sz == 0\n"); - *status = 0; - return NULL; - } - assert(header_sz > 0); - - debug(33, 3) ("parseHttpRequest: req_hdr = {%s}\n", req_hdr); - end = req_hdr + header_sz; - debug(33, 3) ("parseHttpRequest: end = {%s}\n", end); - - prefix_sz = end - inbuf; - *req_line_sz_p = req_hdr - inbuf; - debug(33, 3) ("parseHttpRequest: prefix_sz = %d, req_line_sz = %d\n", - (int) prefix_sz, (int) *req_line_sz_p); - assert(prefix_sz <= conn->in.offset); - - /* Ok, all headers are received */ - http = cbdataAlloc(clientHttpRequest); - http->http_ver = http_ver; - http->conn = conn; - http->start = current_time; - http->req_sz = prefix_sz; - http->range_iter.boundary = StringNull; - *prefix_p = xmalloc(prefix_sz + 1); - xmemcpy(*prefix_p, conn->in.buf, prefix_sz); - *(*prefix_p + prefix_sz) = '\0'; - dlinkAdd(http, &http->active, &ClientActiveRequests); - -// debug(33, 5) ("parseHttpRequest: Request Header is\n%s\n", (*prefix_p) + *req_line_sz_p); - if ((t = strchr(url, '#'))) /* remove HTML anchors */ - *t = '\0'; - - /* handle internal objects */ - if (internalCheck(url)) { - /* prepend our name & port */ - http->uri = xstrdup(internalLocalUri(NULL, url)); - http->flags.internal = 1; - http->flags.accel = 1; - } - /* see if we running in Config2.Accel.on, if so got to convert it to URL */ - else if (Config2.Accel.on && *url == '/') { - /* prepend the accel prefix */ -// if (opt_accel_uses_host && (t = mime_get_header(req_hdr, "Host"))) { - if (opt_accel_uses_host && (httpHeaderHas(&hdr, HDR_HOST))) { - int vport; - char *q; - char *protocol_name = "http"; - String host = httpHeaderGetStrOrList(&hdr, HDR_HOST); - if (vport_mode) - vport = (int) ntohs(http->conn->me.sin_port); - else - vport = (int) Config.Accel.port; - /* If a Host: header was specified, use it to build the URL - * instead of the one in the Config file. */ - /* - * XXX Use of the Host: header here opens a potential - * security hole. There are no checks that the Host: value - * corresponds to one of your servers. It might, for example, - * refer to www.playboy.com. The 'dst' and/or 'dst_domain' ACL - * types should be used to prevent httpd-accelerators - * handling requests for non-local servers */ - -/* FIXME: use string tools on the header host */ - t = xstrdup (strBuf(host)); - strtok(t, " /;@"); - if ((q = strchr(t, ':'))) { - *q++ = '\0'; - if (vport_mode) - vport = atoi(q); - } - url_sz = strlen(url) + 32 + Config.appendDomainLen + - strlen(t); - http->uri = xcalloc(url_sz, 1); - -#if SSL_FORWARDING_NOT_YET_DONE - if (Config.Sockaddr.https->s.sin_port == http->conn->me.sin_port) { - protocol_name = "https"; - vport = ntohs(http->conn->me.sin_port); - } -#endif - snprintf(http->uri, url_sz, "%s://%s:%d%s", - protocol_name, t, vport, url); - safe_free(t); - } else if (vhost_mode) { - int vport; - /* Put the local socket IP address as the hostname */ - url_sz = strlen(url) + 32 + Config.appendDomainLen; - http->uri = xcalloc(url_sz, 1); - if (vport_mode) - vport = (int) ntohs(http->conn->me.sin_port); - else - vport = (int) Config.Accel.port; -#if IPF_TRANSPARENT - natLookup.nl_inport = http->conn->me.sin_port; - natLookup.nl_outport = http->conn->peer.sin_port; - natLookup.nl_inip = http->conn->me.sin_addr; - natLookup.nl_outip = http->conn->peer.sin_addr; - natLookup.nl_flags = IPN_TCP; - if (natfd < 0) - natfd = open(IPL_NAT, O_RDONLY, 0); - if (natfd < 0) { - debug(50, 1) ("parseHttpRequest: NAT open failed: %s\n", - xstrerror()); - return parseHttpRequestAbort(conn, "error:nat-open-failed"); - } - /* - * IP-Filter changed the type for SIOCGNATL between - * 3.3 and 3.4. It also changed the cmd value for - * SIOCGNATL, so at least we can detect it. We could - * put something in configure and use ifdefs here, but - * this seems simpler. - */ - if (63 == siocgnatl_cmd) { - struct natlookup *nlp = &natLookup; - x = ioctl(natfd, SIOCGNATL, &nlp); - } else { - x = ioctl(natfd, SIOCGNATL, &natLookup); - } - if (x < 0) { - if (errno != ESRCH) { - debug(50, 1) ("parseHttpRequest: NAT lookup failed: ioctl(SIOCGNATL)\n"); - close(natfd); - natfd = -1; - return parseHttpRequestAbort(conn, "error:nat-lookup-failed"); - } else - snprintf(http->uri, url_sz, "http://%s:%d%s", - inet_ntoa(http->conn->me.sin_addr), - vport, url); - } else { - if (vport_mode) - vport = natLookup.nl_realport; - snprintf(http->uri, url_sz, "http://%s:%d%s", - inet_ntoa(natLookup.nl_realip), - vport, url); - } -#else -#if LINUX_NETFILTER - /* If the call fails the address structure will be unchanged */ - getsockopt(conn->fd, SOL_IP, SO_ORIGINAL_DST, &conn->me, &sock_sz); - debug(33, 5) ("parseHttpRequest: addr = %s", inet_ntoa(conn->me.sin_addr)); - if (vport_mode) - vport = (int) ntohs(http->conn->me.sin_port); -#endif - snprintf(http->uri, url_sz, "http://%s:%d%s", - inet_ntoa(http->conn->me.sin_addr), - vport, url); -#endif - debug(33, 5) ("VHOST REWRITE: '%s'\n", http->uri); - } else { - url_sz = strlen(Config2.Accel.prefix) + strlen(url) + - Config.appendDomainLen + 1; - http->uri = xcalloc(url_sz, 1); - snprintf(http->uri, url_sz, "%s%s", Config2.Accel.prefix, url); - } - http->flags.accel = 1; - } else { - /* URL may be rewritten later, so make extra room */ - url_sz = strlen(url) + Config.appendDomainLen + 5; - http->uri = xcalloc(url_sz, 1); - strcpy(http->uri, url); - http->flags.accel = 0; - } - if (!stringHasCntl(http->uri)) - http->log_uri = xstrndup(http->uri, MAX_URL); - else - http->log_uri = xstrndup(rfc1738_escape_unescaped(http->uri), MAX_URL); - debug(33, 5) ("parseHttpRequest: Complete request received\n"); - xfree(inbuf); - /* Now we know the correct URI */ - if ((request = urlParse(method, http->uri)) == NULL) { - fatal("failed to parse uri after succesful early parse\n"); - } - request->flags.accelerated = http->flags.accel; - request->header=hdr; - if (!http->flags.internal) { - if (internalCheck(strBuf(request->urlpath))) { - if (internalHostnameIs(request->host) && - request->port == ntohs(Config.Sockaddr.http->s.sin_port)) { - http->flags.internal = 1; - } else if (internalStaticCheck(strBuf(request->urlpath))) { - xstrncpy(request->host, internalHostname(), SQUIDHOSTNAMELEN); - request->port = ntohs(Config.Sockaddr.http->s.sin_port); - http->flags.internal = 1; - } - } - } - - /* cache some common data */ - request->content_length = httpHeaderGetInt(&request->header, - HDR_CONTENT_LENGTH); - request->flags.internal = http->flags.internal; - - /* FIXME: how many times do we do this? */ - safe_free(http->log_uri); - http->log_uri = xstrdup(urlCanonicalClean(request)); - /* fill in some more request state data. */ - request->client_addr = conn->peer.sin_addr; - request->my_addr = conn->me.sin_addr; - request->my_port = ntohs(conn->me.sin_port); - request->http_ver = http->http_ver; - /* FIXME: broker function */ - - if (!urlCheckRequest(request)) { - *status = -1; - return parseHttpRequestAbort(conn, "error: non support method"); - } - - /* this is protocol specific. It's good. */ - if (!clientCheckContentLength(request)) { - *status = -2; - return parseHttpRequestAbort(conn, "error: non support method"); - } - - http->request = requestLink(request); - /* FIXME: we're doing this way early... or are we? */ - clientSetKeepaliveFlag(http); - - /* Do we expect a request-body? */ - /* we might receive reqest-bodies encoded with transfer-encoding. - * RFC 2616 is vague on this. For now, we detect it and log it to cache.log */ - if (httpHeaderHas(&http->request->header, HDR_TRANSFER_ENCODING)) - debug(33,0)("client_side has recieved a transfer encoded request entity -we cannot handle this yet. REPORT THIS to Squid-dev@squid-cache.org\n"); - - /* FIXME: we've got potential state variable headaches */ -// filterChainAddTail(&http->reqfilters, identity_body, clientAccessCheck, NULL,http); - filterChainAddTail(&http->reqfilters, identity_body,clientInterpretRequestHeaders, NULL,http); -#if HEADERS_LOG - filterChainAddTail(&http->reqfilters, identity_body, clientHeadersLog, NULL,http); -#endif - filterChainAddTail(&http->reqfilters, identity_body, clientfdnote, NULL,http); - filterChainAddTail(&http->reqfilters, identity_body, httplocalmethods, NULL, http); - filterChainAddTail(&http->reqfilters, identity_body, broker_ClientEntry_hdr, NULL,http); - - /* this is the entry point for the reply to write data to the socket */ - filterChainAddTail(&http->repfilters, http_client_body, http_client_hdr, NULL,http); -// filterChainAddTail(&http->repfilters, newClientW, newClientWH, NULL,http); - - - - - - *status = 1; /* success */ - - return http; -} - -#if HEADERS_LOG -DATAFILTER_FILTERHEADER(clientHeadersLog) { - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; - headersLog(0, 1, request->request->method, request->request); - rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); - dlinkDelete(&filters->node, filter_list); - xfree(filters); - return rvflags; -} -#endif - -DATAFILTER_FILTERHEADER(clientfdnote) { - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; - clientHttpRequest *http=data; - fd_note(http->conn->fd, http->uri); - rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); - dlinkDelete(&filters->node, filter_list); - xfree(filters); - return rvflags; -} - -/* data for this is the http struct. It doesn't need to be though */ - -DATAFILTER_FILTERHEADER(http_trace_hdr) -{ - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; - http_version_t version; -// clientHttpRequest *http=data; - /* we are a source. confirm it the hard way */ - assert(filter_list->head->data==filters); - rep = httpReplyCreate(); - httpBuildVersion(&version, 1, 0); - /* the below _may_ be wrong: prefix length but we use the parsed headers */ - httpReplySetHeaders(rep, version, HTTP_OK, NULL, "text/plain", - httpRequestPrefixLen(request->request), 0, squid_curtime); - rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); - debug (3,3)("http_trace_hdr got rvflags of %0x\n",rvflags); - return rvflags; -} - -DATAFILTER_FILTER(http_trace_body) -{ - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; - clientHttpRequest *http=data; - request_t *request=http->request; - MemBuf mb; - /* we are a source. confirm it the hard way */ - assert(filter_list->head->data==filters); - /* now send the body component */ - mb = HttpRequestPackMemBuf(request); - rvflags = temp_filter->filter(mb.buf, mb.size, 0 ,filter_list,temp_filter, flags | FILTER_EOF, temp_filter->data); - return rvflags | FILTER_EOF; -} - - -/* handle local only methods - that is methods that don't apply to generic protocols - * or to the broker/store interface - * - there are none today however. - * this is a request-only filter. - */ -DATAFILTER_FILTERHEADER(httplocalmethods) { - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; - clientHttpRequest *http=data; - request_t *r = http->request; - /* Trace with a hop count of 0 is a local method */ - if (r->method == METHOD_TRACE && r->max_forwards < 1) { - - /* set the source and start the flow */ - filterChainAdd(&http->repfilters, http_trace_body,http_trace_hdr,NULL,http); - temp_filter=http->repfilters.head->data; - rvflags=temp_filter->filter_hdr(NULL, http, &http->repfilters, temp_filter,0, temp_filter->data); - - debug (33,3)("Process request got flags %0x from rep_header\n", rvflags); - if (!(rvflags & (FILTER_ABORT | FILTER_EOF))) { - rvflags=temp_filter->filter(NULL,0,0,&http->repfilters, temp_filter, - 0,temp_filter->data); - debug (33,3)("Process request got flags %0x from rep_filter\n", rvflags); - } - - return rvflags; - } - - rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); - dlinkDelete(&filters->node, filter_list); - xfree(filters); - return rvflags; -} - -DATAFILTER_FILTERHEADER(clientTemp) { - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; -// clientHttpRequest *http=data; - rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); - return rvflags; -} - - -/* this is the default entry point to client side functions */ -DATAFILTER_FILTER(http_client_body) -{ - FILTER_list *temp_filter=filters->node.next->data; - unsigned int rvflags; -// clientHttpRequest *http=data; - fatal("Unreachable code !!!\n"); - rvflags = temp_filter->filter(buf, len, offset,filter_list,temp_filter, flags, temp_filter->data); - return rvflags; -} - -/* this is the first time we see whats coming down at us from the source */ -/* Note: if this is cancelled we clean up gracefully, that means we still call most - * things. - * However, if FILTER_ABORT is set, we are not guaranteed valid headers... - * so be prepared! - */ -DATAFILTER_FILTERHEADER(http_client_hdr) -{ - FILTER_list *temp_filter; - unsigned int rvflags; - clientHttpRequest *http=data; - HttpHeader *hdr = &rep->header; - int is_hit = isTcpHit(http->log_type); - if (!rep) - /* the source hasn't created a reply for us. */ - /* this should _never_ happen */ - fatal("NO REPLY \n"); - /* Errors: we need a clean way to replace the source and it's related filters. - * a) two filter chains. - * b) some funky cold medina. - */ - if (clientReplyBodyTooLarge(rep->content_length)) - fatal("Body too large\n"); - - #if DONT_FILTER_THESE - /* but you might want to if you run Squid as an HTTP accelerator */ - /* httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); */ - httpHeaderDelById(hdr, HDR_ETAG); - #endif - /* this code is mich simpler than the old clientBuildReplyHeader because we assume - * A) only relevant headers are given to us. The server side functions are responsible - * for that. - * B) only fresh headers are provided. - */ - if (is_hit) - httpHeaderDelById(hdr, HDR_SET_COOKIE); - - /* Add user requested filters. */ - filterBuildChain(Config.reply_filters, &http->repfilters, http, rep, http->request); - - /* Handle Ranges (Note that these are _after_ the user specified filters. This means - * that if those filters inconsistently alter data length, we're screwed - */ -// filterChainAddTail(&http->repfilters, newClientRangeHdr, newClientRangeBody, NULL, http); - - /* handle TE */ - clientTEReplyheader(http,rep, &http->repfilters); - - /* - * Add a estimated Age header on cache hits. - */ - if (is_hit) { - /* - * Remove any existing Age header sent by upstream caches - * (note that the existing header is passed along unmodified - * on cache misses) - */ - httpHeaderDelById(hdr, HDR_AGE); - /* - * This adds the calculated object age. Note that the details of the - * age calculation is performed by adjusting the timestamp in - * storeTimestampsSet(), not here. - * - * BROWSER WORKAROUND: IE sometimes hangs when receiving a 0 Age - * header, so don't use it unless there is a age to report. Please - * note that Age is only used to make a conservative estimation of - * the objects age, so a Age: 0 header does not add any useful - * information to the reply in any case. - */ - if (NULL == http->entry) - (void) 0; - else if (http->entry->timestamp < 0) - (void) 0; - else if (http->entry->timestamp < squid_curtime) - httpHeaderPutInt(hdr, HDR_AGE, - squid_curtime - http->entry->timestamp); - } - - /* Handle authentication headers */ - if (request->request->auth_user_request) - authenticateFixHeader(rep, request->request->auth_user_request, request->request, http->flags.accel); - -#if 0 -/* this stuff does not belong here! - its a) not http specific and b) not client side - * driven - */ - /* Append X-Cache */ - httpHeaderPutStrf(hdr, HDR_X_CACHE, "%s from %s", - is_hit ? "HIT" : "MISS", getMyHostname()); -#if USE_CACHE_DIGESTS - /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */ - httpHeaderPutStrf(hdr, HDR_X_CACHE_LOOKUP, "%s from %s:%d", - http->lookup_type ? http->lookup_type : "NONE", - getMyHostname(), ntohs(Config.Sockaddr.http->s.sin_port)); -#endif -#endif - if ((httpReplyBodySize(request->request->method, rep)) < 0 && !(request->request->flags.te_encoding)) { - debug(33, 3) ("clientBuildReplyHeader: can't keep-alive, unknown body size\n"); - request->request->flags.proxy_keepalive = 0; - } else if ((httpReplyBodySize(request->request->method, rep)) < 0 && (request->request->flags.te_encoding)) - debug(33, 3) ("clientBuildReplyHeader: can keep-alive, unknown body size with te\n"); - /* Signal keep-alive if needed */ - httpHeaderPutStr(hdr, http->flags.accel ? HDR_CONNECTION : HDR_PROXY_CONNECTION, - request->request->flags.proxy_keepalive ? "keep-alive" : "close"); - -#if ADD_X_REQUEST_URI - /* - * Knowing the URI of the request is useful when debugging persistent - * connections in a client; we cannot guarantee the order of http headers, - * but X-Request-URI is likely to be the very last header to ease use from a - * debugger [hdr->entries.count-1]. - */ - httpHeaderPutStr(hdr, HDR_X_REQUEST_URI, - http->entry->mem_obj->url ? http->entry->mem_obj->url : http->uri); -#endif - httpHdrMangleList(hdr, request->request); - - - /* And add the IO filters */ - filterChainAddTail(&http->repfilters, newClientW, newClientWH, NULL,http); - /* process what we've just setup */ - temp_filter=filters->node.next->data; - /* but we're not needed anymore */ - dlinkDelete(&filters->node, filter_list); - xfree(filters); - rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); - return rvflags; -} - -static void -newclientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) -{ - clientHttpRequest *http = data; - StoreEntry *entry = http->entry; - - http->out.size += size; - http->flags.reply_write_in_progress=0; - /* objectlen as the test is broken becuase it records headers in the length - * calculation - */ - debug(33, 5) ("newclientWriteComplete: FD %d, sz %d, err %d, off %d, len %d\n", - fd, size, errflag, (int) http->out.offset, entry ? objectLen(entry) : 0); - if (size > 0) { - kb_incr(&statCounter.client_http.kbytes_out, size); - if (isTcpHit(http->log_type)) - kb_incr(&statCounter.client_http.hit_kbytes_out, size); - } - if (errflag) { - /* This is a socket error on the client socket. - * just close the socket, httpRequestFree will abort if needed - */ - debug(33,1)("newclientWriteComplete: error writing\n"); - comm_close(fd); - return; - } - if (http->mb.size) { - /* there is queued data in the membuffer. - * write it. - */ - debug(33,8)("newclientWriteComplete: Pushing queued data\n"); - http->flags.reply_write_in_progress=1; - comm_write_mbuf(http->conn->fd, http->mb, newclientWriteComplete, http); - /* zero the membuf - the data is untouched */ - memBufDefInit(&http->mb); - return; - } else if (http->flags.done_copying) { - debug(33, 5) ("newclientWriteComplete: FD %d transfer is DONE\n", fd); - /* why are we finished? */ - if (http->request->flags.proxy_keepalive) { - debug(33, 5) ("newclientWriteComplete: FD %d Keeping Alive\n", fd); - clientKeepaliveNextRequest(http); - } else { - comm_close(fd); - } - return; - } else if (clientReplyBodyTooLarge((int) http->out.offset)) { - debug(33,1)("newclientWriteComplete: size overrun\n"); - comm_close(fd); - return; - } - - /* call the "I want more data function". Probably the head of the reply function - * data filter, with no buffer. - */ - http->serverread(http->serverreaddata); -} - -DATAFILTER_FILTER(newClientW) { - clientHttpRequest *http = data; - MemBuf mb; - memBufDefInit(&mb); - - /* allocate a membuf, put the data from range and te into it, and call comm_write - * membuf - */ - - /* the more efficient longterm design (say there are no ranges, and no te.. - * storeClientCopy doesn't copy the data (maybe there's another PIA call? - * we just write the buffer and when the write completes it auto frees. - * I wonder if membuffers could do that. Then - * It'd be trivial - if a later filter wants the original buffer, it just locks - * it. Hmmm. - */ - - /* we are the last of the mohicans */ - assert(filters->node.next==NULL); - assert((buf && len) || (flags & (FILTER_EOF | FILTER_ABORT))); - - debug (33,8)("clientDoCommWriteMemBuf: buf %p len %d flags %d\n",buf,len,flags); - if (!buf || !len) { - /* No new data to write */ - /* if not EOF, then some error occured upstream... abort here TODO */ - newclientWriteComplete(http->conn->fd, NULL, 0, 0, http); - /* TODO: when we start buffer blocks here, this is where we flush the buffer */ - /* Why? calling clientWriteComplete flushes the buffer too */ - return 0; - - } - /* there is data to write. Concept: buffer 4K/client sock buf/ whatever here - * and only call TCP/IP layer when EOF or a larger amount is present ??? */ - - if (flags & FILTER_EOF) { - debug(33,1) ("newClientW called with EOF\n"); - http->flags.done_copying=1; - } - - if (!http->flags.reply_write_in_progress) { - http->flags.reply_write_in_progress=1; - memBufAppend(&mb, buf, len); - comm_write_mbuf(http->conn->fd, mb, newclientWriteComplete, http); - } else { - /* queue the data to the http membuffer */ - if (!http->mb.buf) - memBufDefInit(&http->mb); - memBufAppend(&http->mb, buf, len); - } -// comm_write(http->conn->fd, buf, len, clientWriteBodyComplete, http, NULL); - return 0; -} - -DATAFILTER_FILTERHEADER(newClientWH) { - clientHttpRequest *http = data; - MemBuf mb; -// size_t len; - -// memBufDefInit(&mb); - - /* allocate a membuf, put the data from range and te into it, and call comm_write - * membuf - */ - - /* the more efficient longterm design (say there are no ranges, and no te.. - * storeClientCopy doesn't copy the data (maybe there's another PIA call? - * we just write the buffer and when the write completes it auto frees. - * I wonder if membuffers could do that. Then - * It'd be trivial - if a later filter wants the original buffer, it just locks - * it. Hmmm. - */ - - /* we are the last of the mohicans */ - assert(filters->node.next==NULL); - - debug (33,1)("new client ** ** WriteHeaders: reply %p flags %d\n",rep,flags); - /* there is data to write. Concept: buffer 4K/client sock buf/ whatever here - * and only call TCP/IP layer when EOF or a larger amount is present ??? */ - - mb = httpReplyPack(rep); - - if (flags & FILTER_EOF) { - debug(33,1) ("newClientWH called with EOF\n"); - http->flags.done_copying=1; - } - - if (!http->flags.reply_write_in_progress) { - http->flags.reply_write_in_progress=1; - /* FIXME: just call commWrite */ - comm_write_mbuf(http->conn->fd, mb, newclientWriteComplete, http); - } else { /* FIXME: just call commWrite */ - /* queue the data to the http membuffer */ - if (!http->mb.buf) - memBufDefInit(&http->mb); - memBufAppend(&http->mb, mb.buf, mb.size); - memBufClean(&mb); - } -// comm_write(http->conn->fd, buf, len, clientWriteBodyComplete, http, NULL); - return 0; -} - -static int -clientReadDefer(int fdnotused, void *data) -{ - ConnStateData *conn = data; - if (conn->body.size_left) - return conn->in.offset >= conn->in.size; - else - return conn->defer.until > squid_curtime; -} - -static void -clientReadRequest(int fd, void *data) -{ - ConnStateData *conn = data; - int parser_return_code = 0; - int k; -// request_t *request = NULL; - int size; - void *p; - method_t method; - clientHttpRequest *http = NULL; - clientHttpRequest **H = NULL; - ErrorState *err = NULL; - fde *F = &fd_table[fd]; - int len = conn->in.size - conn->in.offset - 1; - debug(33, 4) ("clientReadRequest: FD %d: reading request...\n", fd); - /* FIXME: this bit is for adrian. I don't know what the modio/event network stuff - * can do or is capable of - */ - statCounter.syscalls.sock.reads++; - size = FD_READ_METHOD(fd, conn->in.buf + conn->in.offset, len); - if (size > 0) { - fd_bytes(fd, size, FD_READ); - kb_incr(&statCounter.client_http.kbytes_in, size); - } - /* - * Don't reset the timeout value here. The timeout value will be - * set to Config.Timeout.request by httpAccept() and - * clientWriteComplete(), and should apply to the request as a - * whole, not individual read() calls. Plus, it breaks our - * lame half-close detection - */ - if (size > 0) { - conn->in.offset += size; - conn->in.buf[conn->in.offset] = '\0'; /* Terminate the string */ - } else if (size == 0 && len > 0) { - if (conn->chr == NULL && conn->in.offset == 0) { - /* no current or pending requests */ - debug(33, 4) ("clientReadRequest: FD %d closed\n", fd); - comm_close(fd); - return; - } else if (!Config.onoff.half_closed_clients) { - /* admin doesn't want to support half-closed client sockets */ - debug(33, 3) ("clientReadRequest: FD %d aborted (half_closed_clients disabled)\n", fd); - comm_close(fd); - return; - } - /* It might be half-closed, we can't tell */ - debug(33, 5) ("clientReadRequest: FD %d closed?\n", fd); - F->flags.socket_eof = 1; - conn->defer.until = squid_curtime + 1; - conn->defer.n++; - fd_note(fd, "half-closed"); - /* There is one more close check at the end, to detect aborted - * (partial) requests. At this point we can't tell if the request - * is partial. - */ - /* Continue to process previously read data */ - } else if (size < 0) { - if (!ignoreErrno(errno)) { - debug(50, 2) ("clientReadRequest: FD %d: %s\n", fd, xstrerror()); - comm_close(fd); - return; - } else if (conn->in.offset == 0) { - debug(50, 2) ("clientReadRequest: FD %d: no data to process (%s)\n", fd, xstrerror()); - return; - } - /* Continue to process previously read data */ - } - - /* DON'T DO THIS FIXME */ - - commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, conn, 0); - - - /* FIXME this is the first thing to change. We need to change the select to call - * our initiating filter when a request is being read - */ - - /* Process request body if any */ - if (conn->in.offset > 0 && conn->body.callback != NULL) - // { temp_filter = http->reqfilters.head; rvflags = temp_filter->filter_hdr(NULL, http->request, &http->reqfilters, temp_filter, 0, temp_filter->data); } - clientProcessBody(conn); - - - - - /* Process next request */ - while (conn->in.offset > 0 && conn->body.size_left == 0) { - int nrequests; - size_t req_line_sz; - /* Skip leading (and trailing) whitespace */ - /* FIXME: this buffer fiddle is bad bad bad */ - while (conn->in.offset > 0 && xisspace(conn->in.buf[0])) { - xmemmove(conn->in.buf, conn->in.buf + 1, conn->in.offset - 1); - conn->in.offset--; - } - conn->in.buf[conn->in.offset] = '\0'; /* Terminate the string */ - if (conn->in.offset == 0) - break; - /* Limit the number of concurrent requests to 2 */ - for (H = &conn->chr, nrequests = 0; *H; H = &(*H)->next, nrequests++); - if (nrequests >= (Config.onoff.pipeline_prefetch ? 2 : 1)) { - debug(33, 3) ("clientReadRequest: FD %d max concurrent requests reached\n", fd); - debug(33, 5) ("clientReadRequest: FD %d defering new request until one is done\n", fd); - conn->defer.until = squid_curtime + 100; /* Reset when a request is complete */ - break; - } - conn->in.buf[conn->in.offset] = '\0'; /* Terminate the string */ - if (nrequests == 0) - fd_note(conn->fd, "Reading next request"); - - - - - /* Process request */ - http = parseHttpRequest(conn, - &method, - &parser_return_code, - &req_line_sz); - - -// we have a parsed request, with headers in - - if (http) { - FILTER_list *temp_filter; - unsigned int rvflags; - assert(http->req_sz > 0); - conn->in.offset -= http->req_sz; - assert(conn->in.offset >= 0); - debug(33, 5) ("conn->in.offset = %d\n", (int) conn->in.offset); - /* - * If we read past the end of this request, move the remaining - * data to the beginning - */ - if (conn->in.offset > 0) - xmemmove(conn->in.buf, conn->in.buf + http->req_sz, conn->in.offset); - /* add to the client request queue */ - for (H = &conn->chr; *H; H = &(*H)->next); - *H = http; - conn->nrequests++; - /* - * I wanted to lock 'http' here since its callback data for - * clientLifetimeTimeout(), but there's no logical place to - * cbdataUnlock if the timeout never happens. Maybe its safe - * enough to assume that if the FD is open, and the timeout - * triggers, that 'http' is valid. - */ - commSetTimeout(fd, Config.Timeout.lifetime, clientLifetimeTimeout, http); - if (parser_return_code < 0) { - //FIXME: case of return values */ - debug(33, 1) ("clientReadRequest: FD %d Invalid Request\n", fd); - err = errorCon(ERR_INVALID_REQ, HTTP_BAD_REQUEST); - err->request_hdrs = xstrdup(conn->in.buf); - http->entry = clientCreateStoreEntry(http, method, null_request_flags); - errorAppendEntry(http->entry, err); - break; - } - /* Do we expect a request-body? */ - if (http->request->content_length > 0) { - conn->body.size_left = http->request->content_length; - http->request->body_connection = conn; - /* Is it too large? */ - if (clientRequestBodyTooLarge(http->request->content_length)) { - err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); - err->request = requestLink(http->request); - http->entry = clientCreateStoreEntry(http, - METHOD_NONE, null_request_flags); - errorAppendEntry(http->entry, err); - break; - } - } - temp_filter = http->reqfilters.head->data; - rvflags = temp_filter->filter_hdr(NULL, http, &http->reqfilters, temp_filter, 0, temp_filter->data); - continue; /* while offset > 0 && body.size_left == 0 */ - } else if (parser_return_code == 0) { - /* - * Partial request received; reschedule until parseHttpRequest() - * is happy with the input - */ - k = conn->in.size - 1 - conn->in.offset; - if (k == 0) { - if (conn->in.offset >= Config.maxRequestHeaderSize) { - /* The request is too large to handle */ - debug(33, 1) ("Request header is too large (%d bytes)\n", - (int) conn->in.offset); - debug(33, 1) ("Config 'request_header_max_size'= %d bytes.\n", - Config.maxRequestHeaderSize); - err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); - http = parseHttpRequestAbort(conn, "error:request-too-large"); - /* add to the client request queue */ - for (H = &conn->chr; *H; H = &(*H)->next); - *H = http; - http->entry = clientCreateStoreEntry(http, METHOD_NONE, null_request_flags); - errorAppendEntry(http->entry, err); - return; - } - /* Grow the request memory area to accomodate for a large request */ - conn->in.size += CLIENT_REQ_BUF_SZ; - if (conn->in.size == 2 * CLIENT_REQ_BUF_SZ) { - p = conn->in.buf; /* get rid of fixed size Pooled buffer */ - conn->in.buf = xcalloc(2, CLIENT_REQ_BUF_SZ); - xmemcpy(conn->in.buf, p, CLIENT_REQ_BUF_SZ); - memFree(p, MEM_CLIENT_REQ_BUF); - } else - conn->in.buf = xrealloc(conn->in.buf, conn->in.size); - /* XXX account conn->in.buf */ - debug(33, 3) ("Handling a large request, offset=%d inbufsize=%d\n", - (int) conn->in.offset, conn->in.size); - k = conn->in.size - 1 - conn->in.offset; - } - break; - } - } /* while offset > 0 && conn->body.size_left == 0 */ - /* Check if a half-closed connection was aborted in the middle */ - if (F->flags.socket_eof) { - if (conn->in.offset != conn->body.size_left) { /* != 0 when no request body */ - /* Partial request received. Abort client connection! */ - debug(33, 3) ("clientReadRequest: FD %d aborted\n", fd); - // temp_filter = http->reqfilters.head; rvflags = temp_filter->filter_hdr(NULL, http->request, &http->reqfilters, temp_filter, FILTER_ABORT, temp_filter->data); - comm_close(fd); - return; - } - } -} - -/* file_read like function, for reading body content */ -void -clientReadBody(request_t * request, char *buf, size_t size, CBCB * callback, void *cbdata) -{ - ConnStateData *conn = request->body_connection; - if (!conn) { - debug(33, 5) ("clientReadBody: no body to read, request=%p\n", request); - callback(buf, 0, cbdata); /* Signal end of body */ - return; - } - debug(33, 2) ("clientReadBody: start fd=%d body_size=%d in.offset=%d cb=%p req=%p\n", conn->fd, conn->body.size_left, conn->in.offset, callback, request); - conn->body.callback = callback; - conn->body.cbdata = cbdata; - conn->body.buf = buf; - conn->body.bufsize = size; - conn->body.request = requestLink(request); - if (conn->in.offset) { - /* Data available */ - clientProcessBody(conn); - } else { - debug(33, 2) ("clientReadBody: fd %d wait for clientReadRequest\n", conn->fd); - } -} - -/* Called by clientReadRequest to process body content */ -static void -clientProcessBody(ConnStateData * conn) -{ - int size; - char *buf = conn->body.buf; - void *cbdata = conn->body.cbdata; - CBCB *callback = conn->body.callback; - request_t *request = conn->body.request; - /* Note: request is null while eating "aborted" transfers */ - debug(33, 2) ("clientProcessBody: start fd=%d body_size=%d in.offset=%d cb=%p req=%p\n", conn->fd, conn->body.size_left, conn->in.offset, callback, request); - /* Some sanity checks... */ - assert(conn->body.size_left > 0); - assert(conn->in.offset > 0); - assert(callback != NULL); - assert(buf != NULL); - /* How much do we have to process? */ - size = conn->in.offset; - if (size > conn->body.size_left) /* only process the body part */ - size = conn->body.size_left; - if (size > conn->body.bufsize) /* don't copy more than requested */ - size = conn->body.bufsize; - xmemcpy(buf, conn->in.buf, size); - conn->body.size_left -= size; - /* Move any remaining data */ - conn->in.offset -= size; - if (conn->in.offset > 0) - xmemmove(conn->in.buf, conn->in.buf + size, conn->in.offset); - /* Remove request link if this is the last part of the body, as - * clientReadRequest automatically continues to process next request */ - if (conn->body.size_left <= 0 && request != NULL) - request->body_connection = NULL; - /* Remove clientReadBody arguments (the call is completed) */ - conn->body.request = NULL; - conn->body.callback = NULL; - conn->body.buf = NULL; - conn->body.bufsize = 0; - /* Remember that we have touched the body, not restartable */ - if (request != NULL) - request->flags.body_sent = 1; - /* Invoke callback function */ - callback(buf, size, cbdata); - if (request != NULL) - requestUnlink(request); /* Linked in clientReadBody */ - debug(33, 2) ("clientProcessBody: end fd=%d size=%d body_size=%d in.offset=%d cb=%p req=%p\n", conn->fd, size, conn->body.size_left, conn->in.offset, callback, request); - return; -} - -/* A dummy handler that throws away a request-body */ -static char bodyAbortBuf[SQUID_TCP_SO_RCVBUF]; -static void -clientReadBodyAbortHandler(char *buf, size_t size, void *data) -{ - ConnStateData *conn = (ConnStateData *) data; - debug(33, 2) ("clientReadBodyAbortHandler: fd=%d body_size=%d in.offset=%d\n", conn->fd, conn->body.size_left, conn->in.offset); - if (size != 0 && conn->body.size_left != 0) { - debug(33, 3) ("clientReadBodyAbortHandler: fd=%d shedule next read\n", conn->fd); - conn->body.callback = clientReadBodyAbortHandler; - conn->body.buf = bodyAbortBuf; - conn->body.bufsize = sizeof(bodyAbortBuf); - conn->body.cbdata = data; - } -} - -/* Abort a body request */ -int -clientAbortBody(request_t * request) -{ - ConnStateData *conn = request->body_connection; - char *buf; - CBCB *callback; - void *cbdata; - request->body_connection = NULL; - if (!conn || conn->body.size_left <= 0) - return 0; /* No body to abort */ - if (conn->body.callback != NULL) { - buf = conn->body.buf; - callback = conn->body.callback; - cbdata = conn->body.cbdata; - assert(request == conn->body.request); - conn->body.buf = NULL; - conn->body.callback = NULL; - conn->body.cbdata = NULL; - conn->body.request = NULL; - callback(buf, -1, cbdata); /* Signal abort to clientReadBody caller */ - requestUnlink(request); - } - clientReadBodyAbortHandler(NULL, -1, conn); /* Install abort handler */ - /* clientProcessBody() */ - return 1; /* Aborted */ -} - -/* general lifetime handler for HTTP requests */ -/* TODO FIXME: RFC2616 covers this: we should send a close message - * to http/1.1 clients - */ -static void -requestTimeout(int fd, void *data) -{ -#if THIS_CONFUSES_PERSISTENT_CONNECTION_AWARE_BROWSERS_AND_USERS - ConnStateData *conn = data; - ErrorState *err; - debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd); - if (fd_table[fd].rwstate) { - /* - * Some data has been sent to the client, just close the FD - */ - comm_close(fd); - } else if (conn->nrequests) { - /* - * assume its a persistent connection; just close it - */ - comm_close(fd); - } else { - /* - * Generate an error - */ - err = errorCon(ERR_LIFETIME_EXP, HTTP_REQUEST_TIMEOUT); - err->url = xstrdup("N/A"); - /* - * Normally we shouldn't call errorSend() in client_side.c, but - * it should be okay in this case. Presumably if we get here - * this is the first request for the connection, and no data - * has been written yet - */ - assert(conn->chr == NULL); - errorSend(fd, err); - /* - * if we don't close() here, we still need a timeout handler! - */ - commSetTimeout(fd, 30, requestTimeout, conn); - /* - * Aha, but we don't want a read handler! - */ - commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); - } -#else - /* - * Just close the connection to not confuse browsers - * using persistent connections. Some browsers opens - * an connection and then does not use it until much - * later (presumeably because the request triggering - * the open has already been completed on another - * connection) - */ - debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd); - comm_close(fd); -#endif -} - -static void -clientLifetimeTimeout(int fd, void *data) -{ - clientHttpRequest *http = data; - ConnStateData *conn = http->conn; - debug(33, 1) ("WARNING: Closing client %s connection due to lifetime timeout\n", - inet_ntoa(conn->peer.sin_addr)); - debug(33, 1) ("\t%s\n", http->uri); - comm_close(fd); -} - -static int -httpAcceptDefer(int fdunused, void *dataunused) -{ - static time_t last_warn = 0; - if (fdNFree() >= RESERVED_FD) - return 0; - if (last_warn + 15 < squid_curtime) { - debug(33, 0) ("WARNING! Your cache is running out of filedescriptors\n"); - last_warn = squid_curtime; - } - return 1; -} - -/* Handle a new connection on HTTP socket. */ -void -httpAccept(int sock, void *data) -{ - int *N = data; - int fd = -1; - ConnStateData *connState = NULL; - struct sockaddr_in peer; - struct sockaddr_in me; - int max = INCOMING_HTTP_MAX; -#if USE_IDENT - static aclCheck_t identChecklist; -#endif - commSetSelect(sock, COMM_SELECT_READ, httpAccept, NULL, 0); - while (max-- && !httpAcceptDefer(sock, NULL)) { - memset(&peer, '\0', sizeof(struct sockaddr_in)); - memset(&me, '\0', sizeof(struct sockaddr_in)); - if ((fd = comm_accept(sock, &peer, &me)) < 0) { - if (!ignoreErrno(errno)) - debug(50, 1) ("httpAccept: FD %d: accept failure: %s\n", - sock, xstrerror()); - break; - } - debug(33, 4) ("httpAccept: FD %d: accepted\n", fd); - connState = cbdataAlloc(ConnStateData); - connState->peer = peer; - connState->log_addr = peer.sin_addr; - connState->log_addr.s_addr &= Config.Addrs.client_netmask.s_addr; - connState->me = me; - connState->fd = fd; - connState->in.size = CLIENT_REQ_BUF_SZ; - connState->in.buf = memAllocate(MEM_CLIENT_REQ_BUF); - /* XXX account connState->in.buf */ - comm_add_close_handler(fd, connStateFree, connState); - if (Config.onoff.log_fqdn) - fqdncache_gethostbyaddr(peer.sin_addr, FQDN_LOOKUP_IF_MISS); - commSetTimeout(fd, Config.Timeout.request, requestTimeout, connState); -#if USE_IDENT - identChecklist.src_addr = peer.sin_addr; - identChecklist.my_addr = me.sin_addr; - identChecklist.my_port = ntohs(me.sin_port); - if (aclCheckFast(Config.accessList.identLookup, &identChecklist)) - identStart(&me, &peer, clientIdentDone, connState); -#endif - commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, connState, 0); - commSetDefer(fd, clientReadDefer, connState); - clientdbEstablished(peer.sin_addr, 1); - assert(N); - (*N)++; - } -} - -#if USE_SSL - -/* negotiate an SSL connection */ -static void -clientNegotiateSSL(int fd, void *data) -{ - ConnStateData *conn = data; - X509 *client_cert; - int ret; - - if ((ret = SSL_accept(fd_table[fd].ssl)) <= 0) { - if (BIO_sock_should_retry(ret)) { - commSetSelect(fd, COMM_SELECT_READ, clientNegotiateSSL, conn, 0); - return; - } - ret = ERR_get_error(); - debug(81, 1) ("clientNegotiateSSL: Error negotiating SSL connection on FD %d: %s\n", - fd, ERR_error_string(ret, NULL)); - comm_close(fd); - return; - } - debug(81, 5) ("clientNegotiateSSL: FD %d negotiated cipher %s\n", fd, - SSL_get_cipher(fd_table[fd].ssl)); - - client_cert = SSL_get_peer_certificate(fd_table[fd].ssl); - if (client_cert != NULL) { - debug(81, 5) ("clientNegotiateSSL: FD %d client certificate: subject: %s\n", fd, - X509_NAME_oneline(X509_get_subject_name(client_cert), 0, 0)); - - debug(81, 5) ("clientNegotiateSSL: FD %d client certificate: issuer: %s\n", fd, - X509_NAME_oneline(X509_get_issuer_name(client_cert), 0, 0)); - - X509_free(client_cert); - } else { - debug(81, 5) ("clientNegotiateSSL: FD %d has no certificate.\n", fd); - } - - commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, conn, 0); -} - -/* handle a new HTTPS connection */ -static void -httpsAccept(int sock, void *data) -{ - int *N = data; - int fd = -1; - ConnStateData *connState = NULL; - struct sockaddr_in peer; - struct sockaddr_in me; - int max = INCOMING_HTTP_MAX; - SSL *ssl; - int ssl_error; -#if USE_IDENT - static aclCheck_t identChecklist; -#endif - commSetSelect(sock, COMM_SELECT_READ, httpsAccept, NULL, 0); - while (max-- && !httpAcceptDefer(sock, NULL)) { - memset(&peer, '\0', sizeof(struct sockaddr_in)); - memset(&me, '\0', sizeof(struct sockaddr_in)); - if ((fd = comm_accept(sock, &peer, &me)) < 0) { - if (!ignoreErrno(errno)) - debug(50, 1) ("httpsAccept: FD %d: accept failure: %s\n", - sock, xstrerror()); - break; - } - if ((ssl = SSL_new(sslContext)) == NULL) { - ssl_error = ERR_get_error(); - debug(81, 1) ("httpsAccept: Error allocating handle: %s\n", - ERR_error_string(ssl_error, NULL)); - break; - } - SSL_set_fd(ssl, fd); - fd_table[fd].ssl = ssl; - fd_table[fd].read_method = &ssl_read_method; - fd_table[fd].write_method = &ssl_write_method; - debug(50, 5) ("httpsAccept: FD %d accepted, starting SSL negotiation.\n", fd); - - connState = cbdataAlloc(ConnStateData); - connState->peer = peer; - connState->log_addr = peer.sin_addr; - connState->log_addr.s_addr &= Config.Addrs.client_netmask.s_addr; - connState->me = me; - connState->fd = fd; - connState->in.size = CLIENT_REQ_BUF_SZ; - connState->in.buf = memAllocate(MEM_CLIENT_REQ_BUF); - /* XXX account connState->in.buf */ - comm_add_close_handler(fd, connStateFree, connState); - if (Config.onoff.log_fqdn) - fqdncache_gethostbyaddr(peer.sin_addr, FQDN_LOOKUP_IF_MISS); - commSetTimeout(fd, Config.Timeout.request, requestTimeout, connState); -#if USE_IDENT - identChecklist.src_addr = peer.sin_addr; - identChecklist.my_addr = me.sin_addr; - identChecklist.my_port = ntohs(me.sin_port); - if (aclCheckFast(Config.accessList.identLookup, &identChecklist)) - identStart(&me, &peer, clientIdentDone, connState); -#endif - commSetSelect(fd, COMM_SELECT_READ, clientNegotiateSSL, connState, 0); - commSetDefer(fd, clientReadDefer, connState); - clientdbEstablished(peer.sin_addr, 1); - (*N)++; - } -} - -#endif /* USE_SSL */ - -#define SENDING_BODY 0 -#define SENDING_HDRSONLY 1 -/* checks status of sending to original client */ -static int -clientCheckTransferDone(clientHttpRequest * http) -{ -/* this code defies easy abstraction. Come back to it */ -#if 0 - int sending = SENDING_BODY; - StoreEntry *entry = http->entry; - MemObject *mem; - http_reply *reply; - int sendlen; - if (entry == NULL) - return 0; - /* - * For now, 'done_copying' is used for special cases like - * Range and HEAD & TE requests. - */ - if (http->flags.done_copying) - return 1; - /* - * Handle STORE_OK objects. - * objectLen(entry) will be set proprely. - */ - if (entry->store_status == STORE_OK) { - if (http->out.offset >= objectLen(entry)) - return 1; - else - return 0; - } - /* - * Now, handle STORE_PENDING objects - */ - mem = entry->mem_obj; - assert(mem != NULL); - assert(http->request != NULL); - reply = mem->reply; - if (reply->hdr_sz == 0) - return 0; /* haven't found end of headers yet */ - else if (reply->sline.status == HTTP_OK) - sending = SENDING_BODY; - else if (reply->sline.status == HTTP_NO_CONTENT) - sending = SENDING_HDRSONLY; - else if (reply->sline.status == HTTP_NOT_MODIFIED) - sending = SENDING_HDRSONLY; - else if (reply->sline.status < HTTP_OK) - sending = SENDING_HDRSONLY; - else if (http->request->method == METHOD_HEAD) - sending = SENDING_HDRSONLY; - else - sending = SENDING_BODY; - /* - * Figure out how much data we are supposed to send. - * If we are sending a body and we don't have a content-length, - * then we must wait for the object to become STORE_OK. - */ - if (sending == SENDING_HDRSONLY) - sendlen = reply->hdr_sz; - else if (reply->content_length < 0) - return 0; - else - sendlen = reply->content_length + reply->hdr_sz; - /* - * Now that we have the expected length, did we send it all? - */ - if (http->out.offset < sendlen) - return 0; - else -#endif - return 1; -} - -/* - * This function is designed to serve a fairly specific purpose. - * Occasionally our vBNS-connected caches can talk to each other, but not - * the rest of the world. Here we try to detect frequent failures which - * make the cache unusable (e.g. DNS lookup and connect() failures). If - * the failure:success ratio goes above 1.0 then we go into "hit only" - * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors - * will only fetch HITs from us if they are using the ICP protocol. We - * stay in this mode for 5 minutes. - * - * Duane W., Sept 16, 1996 - */ - -static void -checkFailureRatio(err_type etype, hier_code hcode) -{ - static double magic_factor = 100.0; - double n_good; - double n_bad; - if (hcode == HIER_NONE) - return; - n_good = magic_factor / (1.0 + request_failure_ratio); - n_bad = magic_factor - n_good; - switch (etype) { - case ERR_DNS_FAIL: - case ERR_CONNECT_FAIL: - case ERR_READ_ERROR: - n_bad++; - break; - default: - n_good++; - } - request_failure_ratio = n_bad / n_good; - if (hit_only_mode_until > squid_curtime) - return; - if (request_failure_ratio < 1.0) - return; - debug(33, 0) ("Failure Ratio at %4.2f\n", request_failure_ratio); - debug(33, 0) ("Going into hit-only-mode for %d minutes...\n", - FAILURE_MODE_TIME / 60); - hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME; - request_failure_ratio = 0.8; /* reset to something less than 1.0 */ -} - -void -clientHttpConnectionsOpen(void) -{ - sockaddr_in_list *s; - int fd; - for (s = Config.Sockaddr.http; s; s = s->next) { - if (MAXHTTPPORTS == NHttpSockets) { - debug(1, 1) ("WARNING: You have too many 'http_port' lines.\n"); - debug(1, 1) (" The limit is %d\n", MAXHTTPPORTS); - continue; - } - enter_suid(); - fd = comm_open(SOCK_STREAM, - 0, - s->s.sin_addr, - ntohs(s->s.sin_port), - COMM_NONBLOCKING, - "HTTP Socket"); - leave_suid(); - if (fd < 0) - continue; - comm_listen(fd); - commSetSelect(fd, COMM_SELECT_READ, httpAccept, NULL, 0); - /* - * We need to set a defer handler here so that we don't - * peg the CPU with select() when we hit the FD limit. - */ - commSetDefer(fd, httpAcceptDefer, NULL); - debug(1, 1) ("Accepting HTTP connections at %s, port %d, FD %d.\n", - inet_ntoa(s->s.sin_addr), - (int) ntohs(s->s.sin_port), - fd); - HttpSockets[NHttpSockets++] = fd; - } -#ifdef USE_SSL - for (s = Config.Sockaddr.https; s; s = s->next) { - enter_suid(); - fd = comm_open(SOCK_STREAM, - 0, - s->s.sin_addr, - ntohs(s->s.sin_port), - COMM_NONBLOCKING, - "HTTPS Socket"); - leave_suid(); - if (fd < 0) - continue; - comm_listen(fd); - commSetSelect(fd, COMM_SELECT_READ, httpsAccept, NULL, 0); - /*commSetDefer(fd, httpAcceptDefer, NULL); */ - debug(1, 1) ("Accepting HTTPS connections at %s, port %d, FD %d.\n", - inet_ntoa(s->s.sin_addr), - (int) ntohs(s->s.sin_port), - fd); - HttpSockets[NHttpSockets++] = fd; - } -#endif - if (NHttpSockets < 1) - fatal("Cannot open HTTP Port"); -} - -void -clientHttpConnectionsClose(void) -{ - int i; - for (i = 0; i < NHttpSockets; i++) { - if (HttpSockets[i] >= 0) { - debug(1, 1) ("FD %d Closing HTTP connection\n", HttpSockets[i]); - comm_close(HttpSockets[i]); - HttpSockets[i] = -1; - } - } - NHttpSockets = 0; -} - -int -varyEvaluateMatch(StoreEntry * entry, request_t * request) -{ - const char *vary = request->vary_headers; - int has_vary = httpHeaderHas(&entry->mem_obj->reply->header, HDR_VARY); -#if X_ACCELERATOR_VARY - has_vary |= httpHeaderHas(&entry->mem_obj->reply->header, HDR_X_ACCELERATOR_VARY); -#endif - if (!has_vary || !entry->mem_obj->vary_headers) { - if (vary) { - /* Oops... something odd is going on here.. */ - debug(33, 1) ("varyEvaluateMatch: Oops. Not a Vary object on second attempt, '%s' '%s'\n", - entry->mem_obj->url, vary); - safe_free(request->vary_headers); - return VARY_CANCEL; - } - if (!has_vary) { - /* This is not a varying object */ - return VARY_NONE; - } - /* virtual "vary" object found. Calculate the vary key and - * continue the search - */ - vary = request->vary_headers = xstrdup(httpMakeVaryMark(request, entry->mem_obj->reply)); - if (vary) - return VARY_OTHER; - else { - /* Ouch.. we cannot handle this kind of variance */ - /* XXX This cannot really happen, but just to be complete */ - return VARY_CANCEL; - } - } else { - if (!vary) - vary = request->vary_headers = xstrdup(httpMakeVaryMark(request, entry->mem_obj->reply)); - if (!vary) { - /* Ouch.. we cannot handle this kind of variance */ - /* XXX This cannot really happen, but just to be complete */ - return VARY_CANCEL; - } else if (strcmp(vary, entry->mem_obj->vary_headers) == 0) { - return VARY_MATCH; - } else { - /* Oops.. we have already been here and still haven't - * found the requested variant. Bail out - */ - debug(33, 1) ("varyEvaluateMatch: Oops. Not a Vary match on second attempt, '%s' '%s'\n", - entry->mem_obj->url, vary); - return VARY_CANCEL; - } - } -} --- squid/src/http.c Wed Feb 14 00:53:08 2007 +++ /dev/null Wed Feb 14 00:52:54 2007 @@ -1,2005 +0,0 @@ - -/* - * $Id: http.c,v 1.1.1.3.4.1.4.12.2.16.2.8 2001/05/07 13:23:41 rbcollins Exp $ - * - * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -/* - * Anonymizing patch by lutz@as-node.jena.thur.de - * have a look into http-anon.c to get more informations. - */ - -#include "squid.h" -#include "iobuf.h" - -static const char *const crlf = "\r\n"; - -static CWCB httpSendComplete; -static CWCB httpSendRequestEntry; - -static PF httpReadReply; -static void httpSendRequest(HttpStateData *); -static PF httpStateFree; -static PF httpTimeout; -static void httpCacheNegatively(StoreEntry *); -static void httpMakePrivate(StoreEntry *); -static void httpMakePublic(StoreEntry *); -static int httpCachableReply(HttpStateData *); -static void httpMaybeRemovePublic(StoreEntry *, http_status); - -/* this will go in a header when this file becomes a module */ - -struct _HttpStateData { - StoreEntry *entry; - request_t *request; - char *reply_hdr; - size_t reply_hdr_size; - int reply_hdr_state; - peer *peer; /* peer request made to */ - int eof; /* reached end-of-object? */ - request_t *orig_request; - int fd; - http_state_flags flags; - FwdState *fwd; - /* filter_hdr variables */ - HttpReply *rep; - clientHttpRequest *hcrequest; - /* filter data variables */ - /* buf is not needed - immediately used */ - /* offset is needed. */ - /* size isn't - its returned by the CWCB */ - /* filter variables */ - dlink_list * filter_list; - FILTER_list * filters; - unsigned int filterflags; - dlink_list oldfilters; - /* How far into the network object are we? */ - size_t read_offset; - /* this should be for the ReplyHeaders filter */ - char *headerstore; - size_t headerlength; - iobuf *readbuf; -}; - -CBDATA_TYPE(HttpStateData); - -/* temporary function */ -void -serverHttpInit(void) -{ -} - -/* this interface needs thought. */ -unsigned int -http_broker_entry(HttpReply *rep, clientHttpRequest *request, - dlink_list * filter_list,FILTER_list * filters, unsigned int flags, - void *data) -{ - FILTER_list *temp_filter; - fatal("http broker entry point. \n"); -/* - temp_filter = filters->node.next->data; - return temp_filter->filter_hdr(rep, request, filter_list, - temp_filter, flags, temp_filter->data); -*/ -} - -static void -httpStateFree(int fd, void *data) -{ - HttpStateData *httpState = data; -#if DELAY_POOLS - delayClearNoDelay(fd); -#endif - if (httpState == NULL) - return; - filterCleanChain(&httpState->oldfilters); - storeUnlockObject(httpState->entry); - if (httpState->reply_hdr) { - memFree(httpState->reply_hdr, MEM_8K_BUF); - httpState->reply_hdr = NULL; - } - requestUnlink(httpState->request); - requestUnlink(httpState->orig_request); - httpState->request = NULL; - httpState->orig_request = NULL; - cbdataFree(httpState); -} - -int -httpCachable(method_t method) -{ - /* GET and HEAD are cachable. Others are not. */ - if (method != METHOD_GET && method != METHOD_HEAD) - return 0; - /* else cachable */ - return 1; -} - -static void -httpTimeout(int fd, void *data) -{ - HttpStateData *httpState = data; - StoreEntry *entry = httpState->entry; - debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); - if (entry->store_status == STORE_PENDING) { - if (entry->mem_obj->inmem_hi == 0) { - fwdFail(httpState->fwd, - errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); - } - } - comm_close(fd); -} - -/* This object can be cached for a long time */ -static void -httpMakePublic(StoreEntry * entry) -{ - if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) - storeSetPublicKey(entry); -} - -/* This object should never be cached at all */ -static void -httpMakePrivate(StoreEntry * entry) -{ - storeExpireNow(entry); - storeReleaseRequest(entry); /* delete object when not used */ - /* storeReleaseRequest clears ENTRY_CACHABLE flag */ -} - -/* This object may be negatively cached */ -static void -httpCacheNegatively(StoreEntry * entry) -{ - storeNegativeCache(entry); - if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) - storeSetPublicKey(entry); -} - -static void -httpMaybeRemovePublic(StoreEntry * e, http_status status) -{ - int remove = 0; - int forbidden = 0; - StoreEntry *pe; - if (!EBIT_TEST(e->flags, KEY_PRIVATE)) - return; - switch (status) { - case HTTP_OK: - case HTTP_NON_AUTHORITATIVE_INFORMATION: - case HTTP_MULTIPLE_CHOICES: - case HTTP_MOVED_PERMANENTLY: - case HTTP_MOVED_TEMPORARILY: - case HTTP_GONE: - case HTTP_NOT_FOUND: - remove = 1; - break; - case HTTP_FORBIDDEN: - case HTTP_METHOD_NOT_ALLOWED: - forbidden = 1; - break; -#if WORK_IN_PROGRESS - case HTTP_UNAUTHORIZED: - forbidden = 1; - break; -#endif - default: -#if QUESTIONABLE - /* - * Any 2xx response should eject previously cached entities... - */ - if (status >= 200 && status < 300) - remove = 1; -#endif - break; - } - if (!remove && !forbidden) - return; - assert(e->mem_obj); - if (e->mem_obj->request) - pe = storeGetPublicByRequest(e->mem_obj->request); - else - pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method); - if (pe != NULL) { - assert(e != pe); - storeRelease(pe); - } - /* - * Also remove any cached HEAD response in case the object has - * changed. - */ - if (e->mem_obj->request) - pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_HEAD); - else - pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD); - if (pe != NULL) { - assert(e != pe); - storeRelease(pe); - } - if (forbidden) - return; - switch (e->mem_obj->method) { - case METHOD_PUT: - case METHOD_DELETE: - case METHOD_PROPPATCH: - case METHOD_MKCOL: - case METHOD_MOVE: - case METHOD_BMOVE: - case METHOD_BDELETE: - /* - * Remove any cached GET object if it is beleived that the - * object may have changed as a result of other methods - */ - if (e->mem_obj->request) - pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_GET); - else - pe = storeGetPublic(e->mem_obj->url, METHOD_GET); - if (pe != NULL) { - assert(e != pe); - storeRelease(pe); - } - break; - } -} - -static int -httpCachableReply(HttpStateData * httpState) -{ - HttpReply *rep = httpState->entry->mem_obj->reply; - HttpHeader *hdr = &rep->header; - const int cc_mask = (rep->cache_control) ? rep->cache_control->mask : 0; - const char *v; - if (EBIT_TEST(cc_mask, CC_PRIVATE)) - return 0; - if (EBIT_TEST(cc_mask, CC_NO_CACHE)) - return 0; - if (EBIT_TEST(cc_mask, CC_NO_STORE)) - return 0; - if (httpState->request->flags.auth) { - /* - * Responses to requests with authorization may be cached - * only if a Cache-Control: public reply header is present. - * RFC 2068, sec 14.9.4 - */ - if (!EBIT_TEST(cc_mask, CC_PUBLIC)) - return 0; - } - /* Pragma: no-cache in _replies_ is not documented in HTTP, - * but servers like "Active Imaging Webcast/2.0" sure do use it */ - if (httpHeaderHas(hdr, HDR_PRAGMA)) { - String s = httpHeaderGetList(hdr, HDR_PRAGMA); - const int no_cache = strListIsMember(&s, "no-cache", ','); - stringClean(&s); - if (no_cache) - return 0; - } - /* - * The "multipart/x-mixed-replace" content type is used for - * continuous push replies. These are generally dynamic and - * probably should not be cachable - */ - if ((v = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE))) - if (!strncasecmp(v, "multipart/x-mixed-replace", 25)) - return 0; - switch (httpState->entry->mem_obj->reply->sline.status) { - /* Responses that are cacheable */ - case HTTP_OK: - case HTTP_NON_AUTHORITATIVE_INFORMATION: - case HTTP_MULTIPLE_CHOICES: - case HTTP_MOVED_PERMANENTLY: - case HTTP_GONE: - /* - * Don't cache objects that need to be refreshed on next request, - * unless we know how to refresh it. - */ - if (!refreshIsCachable(httpState->entry)) - return 0; - /* don't cache objects from peers w/o LMT, Date, or Expires */ - /* check that is it enough to check headers @?@ */ - if (rep->date > -1) - return 1; - else if (rep->last_modified > -1) - return 1; - else if (!httpState->peer) - return 1; - /* @?@ (here and 302): invalid expires header compiles to squid_curtime */ - else if (rep->expires > -1) - return 1; - else - return 0; - /* NOTREACHED */ - break; - /* Responses that only are cacheable if the server says so */ - case HTTP_MOVED_TEMPORARILY: - if (rep->expires > -1) - return 1; - else - return 0; - /* NOTREACHED */ - break; - /* Errors can be negatively cached */ - case HTTP_NO_CONTENT: - case HTTP_USE_PROXY: - case HTTP_BAD_REQUEST: - case HTTP_FORBIDDEN: - case HTTP_NOT_FOUND: - case HTTP_METHOD_NOT_ALLOWED: - case HTTP_REQUEST_URI_TOO_LARGE: - case HTTP_INTERNAL_SERVER_ERROR: - case HTTP_NOT_IMPLEMENTED: - case HTTP_BAD_GATEWAY: - case HTTP_SERVICE_UNAVAILABLE: - case HTTP_GATEWAY_TIMEOUT: - return -1; - /* NOTREACHED */ - break; - /* Some responses can never be cached */ - case HTTP_PARTIAL_CONTENT: /* Not yet supported */ - case HTTP_SEE_OTHER: - case HTTP_NOT_MODIFIED: - case HTTP_UNAUTHORIZED: - case HTTP_PROXY_AUTHENTICATION_REQUIRED: - case HTTP_INVALID_HEADER: /* Squid header parsing error */ - default: /* Unknown status code */ - return 0; - /* NOTREACHED */ - break; - } - /* NOTREACHED */ -} - -/* - * For Vary, store the relevant request headers as - * virtual headers in the reply - * Returns false if the variance cannot be stored - */ -const char * -httpMakeVaryMark(request_t * request, HttpReply * reply) -{ - int ok = 1; - String vary, hdr; - const char *pos = NULL; - const char *item; - const char *value; - int ilen; - static String vstr = - {0, 0, NULL}; - - stringClean(&vstr); - vary = httpHeaderGetList(&reply->header, HDR_VARY); - while (strListGetItem(&vary, ',', &item, &ilen, &pos)) { - char *name = xmalloc(ilen + 1); - xstrncpy(name, item, ilen + 1); - Tolower(name); - strListAdd(&vstr, name, ','); - hdr = httpHeaderGetByName(&request->header, name); - safe_free(name); - value = strBuf(hdr); - if (value) { - value = rfc1738_escape(value); - stringAppend(&vstr, "=\"", 2); - stringAppend(&vstr, value, strlen(value)); - stringAppend(&vstr, "\"", 1); - } - stringClean(&hdr); - } - stringClean(&vary); -#if X_ACCELERATOR_VARY - vary = httpHeaderGetList(&reply->header, HDR_X_ACCELERATOR_VARY); - while (strListGetItem(&vary, ',', &item, &ilen, &pos)) { - char *name = xmalloc(ilen + 1); - xstrncpy(name, item, ilen + 1); - Tolower(name); - strListAdd(&vstr, name, ','); - hdr = httpHeaderGetByName(&request->header, name); - safe_free(name); - value = strBuf(hdr); - if (value) { - value = rfc1738_escape(value); - stringAppend(&vstr, "=\"", 2); - stringAppend(&vstr, value, strlen(value)); - stringAppend(&vstr, "\"", 1); - } - stringClean(&hdr); - } - stringClean(&vary); -#endif - debug(11, 0) ("httpMakeVaryMark: %d / %s\n", ok, strBuf(vstr)); - return strBuf(vstr); -} - -/* rewrite this later using new interfaces @?@ */ -void -httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) -{ - char *t = NULL; - StoreEntry *entry = httpState->entry; - int room; - size_t hdr_len; - HttpReply *reply = entry->mem_obj->reply; - Ctx ctx; - debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", - storeKeyText(entry->hash.key)); - if (httpState->reply_hdr == NULL) - httpState->reply_hdr = memAllocate(MEM_8K_BUF); - assert(httpState->reply_hdr_state == 0); - hdr_len = httpState->reply_hdr_size; - room = 8191 - hdr_len; - xmemcpy(httpState->reply_hdr + hdr_len, buf, room < size ? room : size); - hdr_len += room < size ? room : size; - httpState->reply_hdr[hdr_len] = '\0'; - httpState->reply_hdr_size = hdr_len; - if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { - debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr); - httpState->reply_hdr_state += 2; - reply->sline.status = HTTP_INVALID_HEADER; - return; - } - t = httpState->reply_hdr + hdr_len; - /* headers can be incomplete only if object still arriving */ - if (!httpState->eof) { - size_t k = headersEnd(httpState->reply_hdr, 8192); - if (0 == k) - return; /* headers not complete */ - t = httpState->reply_hdr + k; - httpState->headerlength=k; - } - *t = '\0'; - httpState->reply_hdr_state++; - /* in progess state */ - assert(httpState->reply_hdr_state == 1); - ctx = ctx_enter(entry->mem_obj->url); - /* state becomes 2? whats different? */ - httpState->reply_hdr_state++; - debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", - httpState->reply_hdr); - /* Parse headers into reply structure */ - /* what happens if we fail to parse here? */ - httpReplyParse(reply, httpState->reply_hdr, hdr_len); - storeTimestampsSet(entry); - /* Check if object is cacheable or not based on reply code */ - debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); - if (neighbors_do_private_keys) - httpMaybeRemovePublic(entry, reply->sline.status); - switch (httpCachableReply(httpState)) { - case 1: - if (httpHeaderHas(&reply->header, HDR_VARY) -#if X_ACCELERATOR_VARY - || httpHeaderHas(&reply->header, HDR_X_ACCELERATOR_VARY) -#endif - ) { - const char *vary = httpMakeVaryMark(httpState->request, reply); - if (vary) { - entry->mem_obj->vary_headers = xstrdup(vary); - /* Kill the old base object if a change in variance is detected */ - httpMakePublic(entry); - } else { - httpMakePrivate(entry); - } - } else { - httpMakePublic(entry); - } - break; - case 0: - httpMakePrivate(entry); - break; - case -1: - httpCacheNegatively(entry); - break; - default: - assert(0); - break; - } - if (reply->cache_control) { - if (EBIT_TEST(reply->cache_control->mask, CC_PROXY_REVALIDATE)) - EBIT_SET(entry->flags, ENTRY_REVALIDATE); - else if (EBIT_TEST(reply->cache_control->mask, CC_MUST_REVALIDATE)) - EBIT_SET(entry->flags, ENTRY_REVALIDATE); - } - if (httpState->flags.keepalive) - if (httpState->peer) - httpState->peer->stats.n_keepalives_sent++; - if (reply->keep_alive) - if (httpState->peer) - httpState->peer->stats.n_keepalives_recv++; - if (reply->date > -1 && !httpState->peer) { - int skew = abs(reply->date - squid_curtime); - if (skew > 86400) - debug(11, 3) ("%s's clock is skewed by %d seconds!\n", - httpState->request->host, skew); - } - ctx_exit(ctx); -#if HEADERS_LOG - headersLog(1, 0, httpState->request->method, reply); -#endif -} - - -DATAFILTER_FILTER(httpPconnTransferDone) -{ - HttpStateData *httpState=data; - MemObject *mem; - HttpReply *reply; - int clen; - - FILTER_list *temp_filter; - temp_filter=filters->node.next->data; - - debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); - /* - * If we didn't send a keep-alive request header, then this - * can not be a persistent connection. - */ - /* - * TODO: if we recieved an http/1.1 response then it will be persisent by default - */ - - - - if (!httpState->flags.keepalive) - return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); - - /* if we know that it's EOF, then keepalive is still ok (it TE signalled termination - * of body - */ - if (flags & FILTER_EOF) - return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); - - /* - * What does the reply have to say about keep-alive? - */ - /* - * XXX BUG? - * If the origin server (HTTP/1.0) does not send a keep-alive - * header, but keeps the connection open anyway, what happens? - * We'll return here and http.c waits for an EOF before changing - * store_status to STORE_OK. Combine this with ENTRY_FWD_HDR_WAIT - * and an error status code, and we might have to wait until - * the server times out the socket. - */ - - mem = httpState->entry->mem_obj; - reply = mem->reply; - if (!reply->keep_alive) - return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); - debug(11, 5) ("httpPconnTransferDone: content_length=%d\n", - reply->content_length); - /* If we haven't seen the end of reply headers, we are not done */ - if (httpState->reply_hdr_state < 2) - return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); - clen = httpReplyBodySize(httpState->request->method, reply); - /* If there is no message body, we can be persistent */ - if (0 == clen) - return (temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data) | FILTER_EOF); - /* If the body size is unknown we must wait for EOF */ - if (clen < 0) - return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); - /* If the body size is known, we must wait until we've gotten all of it. */ - if (mem->inmem_hi < reply->content_length + reply->hdr_sz) - return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); - /* We got it all */ - return (temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data) | FILTER_EOF); -} - -unsigned int httpProcessData(const char *buf, size_t len, void *data); - -static DATAFILTER httpDoAppend; -static DATAFILTER_HDR httpDoAppendHeaders; - -/* This will be called when data is ready to be read from fd. Read until - * error or connection closed. */ -/* XXX this function is too long! */ -static void -httpReadReply(int fd, void *data) -{ - HttpStateData *httpState = data; - LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); - StoreEntry *entry = httpState->entry; - const request_t *request = httpState->request; - int len; - int bin; - int clen; - size_t read_sz; -#if DELAY_POOLS - delay_id delay_id; - - /* special "if" only for http (for nodelay proxy conns) */ - if (delayIsNoDelay(fd)) - delay_id = 0; - else - delay_id = delayMostBytesAllowed(entry->mem_obj); -#endif - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - comm_close(fd); - return; - } - /* Check for unexpected writes/protocol violations */ - if (httpState->eof) { - /* eof has _already_ been reached. */ - debug(11,1)("Server continued writing after End of message reached\n"); - comm_close(fd); - return; - } - /* check if we want to defer reading */ - errno = 0; - read_sz = SQUID_TCP_SO_RCVBUF; -#if DELAY_POOLS - read_sz = delayBytesWanted(delay_id, 1, read_sz); -#endif - statCounter.syscalls.sock.reads++; - len = FD_READ_METHOD(fd, buf, read_sz); - debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); - assert(fd==httpState->fd); - - if (len > 0) { - fd_bytes(fd, len, FD_READ); -#if DELAY_POOLS - delayBytesIn(delay_id, len); -#endif - kb_incr(&statCounter.server.all.kbytes_in, len); - kb_incr(&statCounter.server.http.kbytes_in, len); - commSetTimeout(fd, Config.Timeout.read, NULL, NULL); - IOStats.Http.reads++; - for (clen = len - 1, bin = 0; clen; bin++) - clen >>= 1; - IOStats.Http.read_hist[bin]++; - } - if (!httpState->reply_hdr && len > 0) { - /* Skip whitespace */ - while (len > 0 && xisspace(*buf)) - xmemmove(buf, buf + 1, len--); - if (len == 0) { - /* Continue to read... */ - commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - return; - } - } - if (len < 0) { - debug(50, 2) ("httpReadReply: FD %d: read failure: %s.\n", - fd, xstrerror()); - if (ignoreErrno(errno)) { - commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - } else if (entry->mem_obj->inmem_hi == 0) { - ErrorState *err; - err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); - err->xerrno = errno; - fwdFail(httpState->fwd, err); - comm_close(fd); - } else { - comm_close(fd); - } - } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { - ErrorState *err; - err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); - err->xerrno = errno; - fwdFail(httpState->fwd, err); - httpState->eof = 1; - comm_close(fd); -#if 0 - /* this isn;t needed anymore because its just a special case of filter response */ - } else if (len == 0) { - unsigned int rvflags=0; - /* TODO: this is broken: if this was a persistent connection, EOF can mean - * a failure - we should check we've recieved the expected amount of data - */ - /* Connection closed; retrieval done. */ - httpState->eof = 1; - /* TODO: optimise this out: it becomes a call to the filter chain - * with FILTER_EOF set */ - if (httpState->reply_hdr_state < 2) { - /* - * Yes Henrik, there is a point to doing this. When we - * called httpProcessReplyHeader() before, we didn't find - * the end of headers, but now we are definately at EOF, so - * we want to process the reply headers. - */ - httpProcessReplyHeader(httpState, buf, len); - te_build_decode_xlate_list(&entry->mem_obj->reply->header, &httpState->filters); - } - - rvflags = httpProcessData(NULL,0,data, FILTER_EOF); - - fwdComplete(httpState->fwd); - comm_close(fd); -#endif - } else { - unsigned int rvflags=0; - rvflags = httpProcessData(buf,len,data); - if (len==0) { - debug(11,1)("EOF by len=0 in httpHandleRead\n"); - httpState->eof = 1; - fwdComplete(httpState->fwd); - comm_close(fd); - } - else if (rvflags & FILTER_ABORT) { - debug(11,1)("Entry abort detected in httpHandleRead\n"); - } - else if (!(rvflags & FILTER_EOF)) { - /* Read more data */ - commSetSelect(httpState->fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - } else if (rvflags & FILTER_EOF) { - debug (11,1)(" End of persistent connection in httpHandleRead\n"); - httpState->eof = 1; - /* yes we have to clear all these! */ - commSetDefer(httpState->fd, NULL, NULL); - commSetTimeout(httpState->fd, -1, NULL, NULL); - commSetSelect(httpState->fd, COMM_SELECT_READ, NULL, NULL, 0); -#if DELAY_POOLS - delayClearNoDelay(httpState->fd); -#endif - comm_remove_close_handler(httpState->fd, httpStateFree, httpState); - fwdUnregister(httpState->fd, httpState->fwd); - pconnPush(httpState->fd, request->host, request->port); - fwdComplete(httpState->fwd); - fd=httpState->fd; - httpState->fd = -1; - httpStateFree(fd, httpState); - } - } -} -/* DATAFILTER */ -/* data filters do not consider len==0 to mean EOF. Set EOF in flags */ -/* all filters still get called on EOF. The last one back here calls the normal write func */ -/* reentrance and datafilters: if a datafilter wants to 'block' it has to: - * copy off the buf it was given - * call a callback style i/o/processing/etc routine - * return - * handle being called again, for the same request, before the callback has returned to - * the next step in processing - */ -/* the calling code assumes the following: the datafilter has forwarded all the data to - * the end of the chain. This MUST happen eventually. If can be deferred, but if - * DATA_EOF is set, the filter MUST flush and remove it self from the list and free it's - * private data - */ -/* A filter with no data to send for the moment, MAY call the filter chain but does not - * NEED to - */ -/* the calling pattern is (Buf, LEN, list_head, self_node, flags, statedata) */ - -/* TODO: send two filter chunks: one with just the headers, one with the rest */ -unsigned int - httpProcessData(const char *buf, size_t len, void *data) -{ - HttpStateData *httpState = data; - StoreEntry *entry = httpState->entry; - /* this was a const request_t... maybe it still should be, but the filterBuildChain - * and thus, all filters, should not modify the request_t struct... hmmm */ - request_t *request = httpState->request; - FILTER_list *temp_filter; - unsigned int rvflags = 0, flags=0; - debug(1,1)("Processing data (len=%d\n",len); - if (len == 0) - flags |= FILTER_EOF; - if (httpState->reply_hdr_state < 2) { - /* we haven't seen the full headers yet */ - /* ProcessReplyHeader expects partial data. This can be cleaned up now */ - httpProcessReplyHeader(httpState, buf, len); - if (httpState->reply_hdr_state == 2) { - http_status s = entry->mem_obj->reply->sline.status; - te_build_decode_xlate_list(&entry->mem_obj->reply->header, &httpState->oldfilters); - /* test for EOF condition (has to happen after content decoding! */ - filterChainAddTail(&httpState->oldfilters,httpPconnTransferDone,identity_header, NULL,httpState); - - /* apply user configured filters */ - filterBuildChain(Config.response_filters, &httpState->oldfilters, NULL, entry->mem_obj->reply , request); - - - /* Send the data to the store manager */ - filterChainAddTail(&httpState->oldfilters,httpDoAppend,identity_header, NULL, httpState); - - /* process the headers */ -// filterChainAddTail(&httpState->filters,httpDoAppendHeaders, identity_header, NULL,httpState); -#if WIP_FWD_LOG - fwdStatus(httpState->fwd, s); -#endif - /* - * If its not a reply that we will re-forward, then - * allow the client to get it. - */ - if (!fwdReforwardableStatus(s)) - EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); - } else { - /* why bother client_side & the store with a response that hasn't even - * got complete headers yet ? - */ - char *tempbuf; - tempbuf=httpState->headerstore; - httpState->headerstore=xmalloc(httpState->headerlength+len); - if (tempbuf) - memcpy(httpState->headerstore,tempbuf,httpState->headerlength); - memcpy(httpState->headerstore+httpState->headerlength,buf,len); - httpState->headerlength+=len; - if (tempbuf) - xfree(tempbuf); - } - /* add the data into any partial headers we have here. We memcpy out of - * courtesy to later fn's so they get all the headers at once. Aren't we - * nice - */ - /* we don't do this above (yet) because ProcessHeaders expects the data bit - * by bit. That can be fixed now by doing this test and merge earlier */ - if (httpState->headerstore && httpState->reply_hdr_state == 2) { - /* headers have been processed, but there's a last combine step to do */ - char *tempbuf; - size_t hdr_len; - tempbuf=httpState->headerstore; - httpState->headerstore=xmalloc(httpState->headerlength+len); - memcpy(httpState->headerstore,tempbuf,httpState->headerlength); - memcpy(httpState->headerstore+httpState->headerlength,buf,len); - httpState->headerlength+=len; - xfree(tempbuf); - hdr_len=httpState->headerlength; - - -/* TODO: the below code may call the next filter with EOF set twice. This is a bad thing. - * The problem is that we don't test for body data and incoming eof yet - */ - if (httpState->reply_hdr_state == 2) { - debug(1,1)("hdr_size %d, headerlength %d\n",httpState->reply_hdr_size, - httpState->headerlength); - assert(httpState->reply_hdr_size == httpState->headerlength); - } - debug(1,1)("sending combined headers\n"); - /* todo: split this into two calls like it is below */ - assert(httpState->oldfilters.head); - temp_filter=httpState->oldfilters.head->data; - rvflags |= temp_filter->filter(httpState->headerstore, httpState->reply_hdr_size, -1, &httpState->oldfilters, temp_filter, flags | FILTER_HTTP_HEADER, temp_filter->data); - - httpState->read_offset=0; - if (httpState->headerlength-httpState->reply_hdr_size) { - if (httpState->reply_hdr_state == 2) { - debug(1,1)("hdr_size %d, headerlength %d\n",httpState->reply_hdr_size, - httpState->headerlength); - assert(httpState->reply_hdr_size == httpState->headerlength); - } - debug(1,1)("headers (%d bytes) written, sending body (%d of %d)\n", - httpState->reply_hdr_size, - httpState->headerlength-httpState->reply_hdr_size,httpState->headerlength); - rvflags |= temp_filter->filter(httpState->headerstore+httpState->reply_hdr_size, httpState->headerlength-httpState->reply_hdr_size, -1, &httpState->oldfilters, temp_filter, flags , temp_filter->data); - httpState->read_offset+=len-hdr_len; - } else - debug(1,1)("headers (%d bytes) written, skipping body (%d of %d) due to filter return flags %d\n",httpState->reply_hdr_size,0,httpState->headerlength, rvflags); - - } else if (httpState->reply_hdr_state == 2){ - size_t hdr_len=httpState->headerlength; - /* no partial headers, got them in one chunk */ - debug(1,1)("headers in one packet... sending %d bytes\n", - hdr_len); - assert(httpState->oldfilters.head); - temp_filter=httpState->oldfilters.head->data; - rvflags |= temp_filter->filter(buf, hdr_len, -1, - &httpState->oldfilters, temp_filter, flags | FILTER_HTTP_HEADER, - temp_filter->data); - httpState->read_offset=0; - if (!(rvflags & (FILTER_EOF | FILTER_ABORT))) { - debug(1,1)("headers (%d bytes) written, sending body (%d of %d)\n", - hdr_len, - len-hdr_len,len); - if (len-hdr_len) { - rvflags |= temp_filter->filter(buf+hdr_len,len-hdr_len, httpState->read_offset, &httpState->oldfilters, temp_filter, flags , temp_filter->data); - httpState->read_offset+=len-hdr_len; - } - } else - debug(1,1)("headers (%d bytes) written, skipping body (%d of %d) due to filter return flags %d\n", - hdr_len, - len-hdr_len,len, rvflags); - } - } else { - /* headers have been seen. - */ - debug(1,1)("headers previously written \n"); - assert(httpState->oldfilters.head); - temp_filter=httpState->oldfilters.head->data; - rvflags |= temp_filter->filter(buf, len, httpState->read_offset, &httpState->oldfilters, temp_filter, flags, temp_filter->data); - httpState->read_offset+=len; - } - debug(1,1)("*** HTTP RAW READ OFFSET NOW %d\n",httpState->read_offset); - return rvflags; -} - -/* empty filter. Must be the last in the filter chain */ -DATAFILTER_FILTER(httpDoAppend) { - HttpStateData *httpState = data; - StoreEntry *entry = httpState->entry; - const request_t *request = httpState->request; - int fd; - int rvflags = 0; - - assert(filters->node.next==NULL); -debug(1,1)("entry %p\n",entry); - - debug(1,1)("httpDoAppend: recieved %d bytes at offset %d with flags %d\n",len,offset,flags); - if (!EBIT_TEST(entry->flags, ENTRY_ABORTED)) - storeAppend(entry,buf,len, flags); - - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* - * the above storeAppend() call could ABORT this entry, - * in that case, the server FD should already be closed. - * there's nothing for us to do. - */ - rvflags |= FILTER_ABORT; - } - return rvflags; -} - -/* empty filter. Must be the last in the filter chain - * this is the same as the above for the same reasons clientWriteMemBuf(Header) are the - * same. - * It should send the headers to the broker server side header function. (When this - * filter is hit, the server side filters have had the chance to remove/alter headers - * as needed. - */ -DATAFILTER_FILTERHEADER(httpDoAppendHeaders) { -#if 0 - HttpStateData *httpState = data; - StoreEntry *entry = httpState->entry; - const request_t *request = httpState->request; - int fd; - int rvflags = 0; - - assert(filters->node.next==NULL); -debug(1,1)("entry %p\n",entry); - - debug(1,1)("httpDoAppend: recieved %d bytes at offset %d with flags %d\n",len,offset,flags); - if (!EBIT_TEST(entry->flags, ENTRY_ABORTED)) - storeAppend(entry,buf,len, flags); - - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* - * the above storeAppend() call could ABORT this entry, - * in that case, the server FD should already be closed. - * there's nothing for us to do. - */ - rvflags |= FILTER_ABORT; - } - return rvflags; -#endif -} - -/* This will be called when request write is complete. Schedule read of - * reply. */ -static void -httpSendComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) -{ - HttpStateData *httpState = data; - StoreEntry *entry = httpState->entry; - ErrorState *err; - debug(11, 5) ("httpSendComplete: FD %d: size %d: errflag %d.\n", - fd, size, errflag); -#if URL_CHECKSUM_DEBUG - assert(entry->mem_obj->chksum == url_checksum(entry->mem_obj->url)); -#endif - if (size > 0) { - fd_bytes(fd, size, FD_WRITE); - kb_incr(&statCounter.server.all.kbytes_out, size); - kb_incr(&statCounter.server.http.kbytes_out, size); - } - if (errflag == COMM_ERR_CLOSING) - return; - if (errflag) { - err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); - err->xerrno = errno; - err->request = requestLink(httpState->orig_request); - errorAppendEntry(entry, err); - comm_close(fd); - return; - } else { - /* Schedule read reply. */ - commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - /* - * Set the read timeout here because it hasn't been set yet. - * We only set the read timeout after the request has been - * fully written to the server-side. If we start the timeout - * after connection establishment, then we are likely to hit - * the timeout for POST/PUT requests that have very large - * request bodies. - */ - commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState); - commSetDefer(fd, fwdCheckDeferRead, entry); - } -} - -/* - * build request headers and append them to a given MemBuf - * used by httpBuildRequestPrefix() - * note: calls httpHeaderInit(), the caller is responsible for Clean()-ing - */ -void -httpBuildRequestHeader(request_t * request, - request_t * orig_request, - StoreEntry * entry, - HttpHeader * hdr_out, - int cfd, - http_state_flags flags) -{ - /* building buffer for complex strings */ -#define BBUF_SZ (MAX_URL+32) - LOCAL_ARRAY(char, bbuf, BBUF_SZ); - String strConnection = StringNull; - const HttpHeader *hdr_in = &orig_request->header; - int we_do_ranges; - const HttpHeaderEntry *e; - String strVia; - String strFwd; - HttpHeaderPos pos = HttpHeaderInitPos; - httpHeaderInit(hdr_out, hoRequest); - /* append our IMS header */ - if (request->lastmod > -1 && request->method == METHOD_GET) - httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); - - /* decide if we want to do Ranges ourselves - * (and fetch the whole object now) - * We want to handle Ranges ourselves iff - * - we can actually parse client Range specs - * - the specs are expected to be simple enough (e.g. no out-of-order ranges) - * - reply will be cachable - * (If the reply will be uncachable we have to throw it away after - * serving this request, so it is better to forward ranges to - * the server and fetch only the requested content) - */ - if (NULL == orig_request->range) - we_do_ranges = 0; - else if (!orig_request->flags.cachable) - we_do_ranges = 0; - else if (httpHdrRangeOffsetLimit(orig_request->range)) - we_do_ranges = 0; - else - we_do_ranges = 1; - debug(11, 8) ("httpBuildRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d\n", - orig_request->range, orig_request->flags.cachable, we_do_ranges); - - strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); - while ((e = httpHeaderGetEntry(hdr_in, &pos))) { - debug(11, 5) ("httpBuildRequestHeader: %s: %s\n", - strBuf(e->name), strBuf(e->value)); - if (!httpRequestHdrAllowed(e, &strConnection)) { - debug(11, 2) ("'%s' header denied by anonymize_headers configuration\n", - strBuf(e->name)); - continue; - } - switch (e->id) { - case HDR_PROXY_AUTHORIZATION: - /* Only pass on proxy authentication to peers for which - * authentication forwarding is explicitly enabled - */ - if (request->flags.proxying && orig_request->peer_login && - strcmp(orig_request->peer_login, "PASS") == 0) { - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - } - break; - case HDR_AUTHORIZATION: - /* Pass on WWW authentication even if used locally. If this is - * not wanted in an accelerator then the header can be removed - * using the anonymization functions - */ - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - /* XXX Some accelerators might want to strip the header - * and regard the reply as cacheable, but authentication - * is not normally enabled for accelerators without reading - * the code, so there is not much use in adding logics here - * without first defining the concept of having authentication - * in the accelerator... - */ - break; - case HDR_HOST: - /* - * Normally Squid does not copy the Host: header from - * a client request into the forwarded request headers. - * However, there is one case when we do: If the URL - * went through our redirector and the admin configured - * 'redir_rewrites_host' to be off. - */ - if (request->flags.redirected) - if (!Config.onoff.redir_rewrites_host) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - break; - case HDR_IF_MODIFIED_SINCE: - /* append unless we added our own; - * note: at most one client's ims header can pass through */ - if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE)) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - break; - case HDR_MAX_FORWARDS: - if (orig_request->method == METHOD_TRACE) { - /* sacrificing efficiency over clarity, etc. */ - const int hops = httpHeaderGetInt(hdr_in, HDR_MAX_FORWARDS); - if (hops > 0) - httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); - } - break; - case HDR_RANGE: - case HDR_IF_RANGE: - case HDR_REQUEST_RANGE: - if (!we_do_ranges) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - break; - case HDR_PROXY_CONNECTION: - case HDR_CONNECTION: - case HDR_VIA: - case HDR_X_FORWARDED_FOR: - case HDR_CACHE_CONTROL: - /* append these after the loop if needed */ - break; - default: - /* pass on all other header fields */ - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - } - } - -#if UNKNOWN_CODE -/* This wasn't in head - should it be here */ - /* append fake user agent if configured and - * the real one is not supplied by the client */ - if (Config.fake_ua && !httpHeaderHas(hdr_out, HDR_USER_AGENT)) - httpHeaderPutStr(hdr_out, HDR_USER_AGENT, Config.fake_ua); -#endif - /* append TE */ - { - char strTE[128]; - - httpHeaderDelById(hdr_out, HDR_TE); /* hop by hop.. that's what last - hop could do*/ - /* TODO: this should be built from a list of known types & a acl allowing type - * to be used in requests vs responses */ - strcpy (strTE,"chunked;q=1.0"); - - httpHeaderPutStr (hdr_out,HDR_TE,strTE); - httpHeaderPutStr(hdr_out,HDR_CONNECTION, "TE"); /* its hop by hop */ - - } - - /* append Via */ - strVia = httpHeaderGetList(hdr_in, HDR_VIA); - snprintf(bbuf, BBUF_SZ, "%d.%d %s", - orig_request->http_ver.major, - orig_request->http_ver.minor, ThisCache); - strListAdd(&strVia, bbuf, ','); - httpHeaderPutStr(hdr_out, HDR_VIA, strBuf(strVia)); - stringClean(&strVia); - - /* append X-Forwarded-For */ - strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); - strListAdd(&strFwd, (cfd < 0 ? "unknown" : fd_table[cfd].ipaddr), ','); - httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd)); - stringClean(&strFwd); - - /* append Host if not there already */ - if (!httpHeaderHas(hdr_out, HDR_HOST)) { - /* use port# only if not default */ - if (orig_request->port == urlDefaultPort(orig_request->protocol)) { - httpHeaderPutStr(hdr_out, HDR_HOST, orig_request->host); - } else { - httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d", - orig_request->host, (int) orig_request->port); - } - } - /* append Authorization if known in URL, not in header and going direct */ - if (!httpHeaderHas(hdr_out, HDR_AUTHORIZATION)) { - if (!request->flags.proxying && *request->login) { - httpHeaderPutStrf(hdr_out, HDR_AUTHORIZATION, "Basic %s", - base64_encode(request->login)); - } - } - /* append Proxy-Authorization if configured for peer, and proxying */ - if (request->flags.proxying && orig_request->peer_login && - !httpHeaderHas(hdr_out, HDR_PROXY_AUTHORIZATION) && - strcmp(orig_request->peer_login, "PASS") != 0) { - if (*orig_request->peer_login == '*') { - /* Special mode, to pass the username to the upstream cache */ - char loginbuf[256]; - char *username = "-"; - if (orig_request->auth_user_request) - username = authenticateUserRequestUsername(orig_request->auth_user_request); - snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, orig_request->peer_login + 1); - httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s", - base64_encode(loginbuf)); - } else { - httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s", - base64_encode(orig_request->peer_login)); - } - } - /* append Cache-Control, add max-age if not there already */ - { - HttpHdrCc *cc = httpHeaderGetCc(hdr_in); - if (!cc) - cc = httpHdrCcCreate(); - if (!EBIT_TEST(cc->mask, CC_MAX_AGE)) { - const char *url = entry ? storeUrl(entry) : urlCanonical(orig_request); - httpHdrCcSetMaxAge(cc, getMaxAge(url)); - if (strLen(request->urlpath)) - assert(strstr(url, strBuf(request->urlpath))); - } - if (flags.only_if_cached) - EBIT_SET(cc->mask, CC_ONLY_IF_CACHED); - httpHeaderPutCc(hdr_out, cc); - httpHdrCcDestroy(cc); - } - /* maybe append Connection: keep-alive */ - if (flags.keepalive) { - if (flags.proxying) { - httpHeaderPutStr(hdr_out, HDR_PROXY_CONNECTION, "keep-alive"); - } else { - httpHeaderPutStr(hdr_out, HDR_CONNECTION, "keep-alive"); - } - } - /* Now mangle the headers. */ - httpHdrMangleList(hdr_out, request); - stringClean(&strConnection); -} - -/* build request prefix and append it to a given MemBuf; - * return the length of the prefix */ -mb_size_t -httpBuildRequestPrefix(request_t * request, - request_t * orig_request, - StoreEntry * entry, - MemBuf * mb, - int cfd, - http_state_flags flags) -{ - const int offset = mb->size; - memBufPrintf(mb, "%s %s HTTP/1.0\r\n", - RequestMethodStr[request->method], - strLen(request->urlpath) ? strBuf(request->urlpath) : "/"); - /* build and pack headers */ - { - HttpHeader hdr; - Packer p; - httpBuildRequestHeader(request, orig_request, entry, &hdr, cfd, flags); - packerToMemInit(&p, mb); - httpHeaderPackInto(&hdr, &p); - httpHeaderClean(&hdr); - packerClean(&p); - } - /* append header terminator */ - memBufAppend(mb, crlf, 2); - return mb->size - offset; -} -/* This will be called when connect completes. Write request. */ -static void -httpSendRequest(HttpStateData * httpState) -{ - MemBuf mb; - request_t *req = httpState->request; - StoreEntry *entry = httpState->entry; - int cfd; - peer *p = httpState->peer; - CWCB *sendHeaderDone; - - debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", httpState->fd, httpState); - - if (httpState->orig_request->body_connection) - sendHeaderDone = httpSendRequestEntry; - else - sendHeaderDone = httpSendComplete; - - if (!opt_forwarded_for) - cfd = -1; - else if (entry->mem_obj == NULL) - cfd = -1; - else - cfd = entry->mem_obj->fd; - assert(-1 == cfd || FD_SOCKET == fd_table[cfd].type); - if (p != NULL) - httpState->flags.proxying = 1; - else - httpState->flags.proxying = 0; - /* - * Is keep-alive okay for all request methods? - */ - if (!Config.onoff.server_pconns) - httpState->flags.keepalive = 0; - else if (p == NULL) - httpState->flags.keepalive = 1; - else if (p->stats.n_keepalives_sent < 10) - httpState->flags.keepalive = 1; - else if ((double) p->stats.n_keepalives_recv / (double) p->stats.n_keepalives_sent > 0.50) - httpState->flags.keepalive = 1; - if (httpState->peer) - if (neighborType(httpState->peer, httpState->request) == PEER_SIBLING && - !httpState->peer->options.allow_miss) - httpState->flags.only_if_cached = 1; - memBufDefInit(&mb); - httpBuildRequestPrefix(req, - httpState->orig_request, - entry, - &mb, - cfd, - httpState->flags); - debug(11, 6) ("httpSendRequest: FD %d:\n%s\n", httpState->fd, mb.buf); - comm_write_mbuf(httpState->fd, mb, sendHeaderDone, httpState); -} - -/* we've got data from the server. It might be a new request,or part of an existing - * request. Later on we can look at different read callbacks for in progress requests - */ -static void -httpCommReadComplete(int fd, char *buf, off_t offset, size_t size, int howmuch, int flags, - void *data) -{ - HttpStateData *http=data; - size_t headerlen; - assert(http->readbuf); - assert(http->readbuf->buffer); - assert(offset==http->readbuf->offset); - assert(size=http->readbuf->size); - /* we may have fun parsing pipelined responses without memcpy - we really need - length = how much is in the buffer from offset on - offset = our current start point in the buffer (incremented as we process responses - size = the buffer size - buf = the buffer data - or possibly, move size into buf... - */ - if (http->rep && http->hcrequest->repfilters.head) { - /* possibly just a case of call the filter chain... */ - FILTER_list *temp_filter; - unsigned int rvflags, eofflag; - iobuf *readbuf; - /* if we've already got a request, we are using clean buffers now */ - assert(offset==0); - temp_filter=http->hcrequest->repfilters.head->data; - http->read_offset+=howmuch; - /* check for trivial EOF status. */ - #define COMM_EOF 0 - eofflag = ((http->read_offset >= http->rep->content_length) || (flags & COMM_EOF)) ? FILTER_EOF : 0; - debug (33,0)("read offset is %d, content length %d\n",http->read_offset, http->rep->content_length); - /* FIXME: check for content overrun here */ - /* NOTE: however we do this, it must be callable by the TE code as well.. */ - /* Allow a new read to be scheduled */ - readbuf=http->readbuf; - http->readbuf=NULL; - /* if this looks hacky.. it is. comm_reads not finished, and these two code - * blocks are reversed */ - rvflags=temp_filter->filter(howmuch ? readbuf->buffer : NULL, - howmuch,http->read_offset-howmuch, - &http->hcrequest->repfilters, temp_filter,eofflag,temp_filter->data); - ioBufferUnlock(readbuf); - /* TODO: one of the filters needs to be a pipeline request handler that gets the next request handled correctly - setups this func up again */ - debug (33,3)("Process request got flags %0x from rep_filter\n", rvflags); - /* TODO: cancel the read in some fashion if EOF is signalled after a read is queued */ - if (rvflags & FILTER_EOF || eofflag) { - http->hcrequest->serverread=NULL; - http->hcrequest->serverreaddata=NULL; - cbdataUnlock(http); - } - return; - } - - headerlen = headersEnd(buf+offset, size); - if (headerlen) { - http->rep=httpReplyCreate(); - if (httpReplyParse(http->rep, buf+offset, headerlen)) { - /* process the request */ - /* TODO: filter the server-side only things (ie connection header...) */ - /* TODO 2: add the server-side filters (ie TE) */ - FILTER_list *temp_filter; - unsigned int rvflags, eofflag; - iobuf *readbuf; - temp_filter=http->hcrequest->repfilters.head->data; - http->readbuf->offset+=howmuch; - http->read_offset=0; - /* FIXME: this is broken if the server writes the headers in a different - *packet to the first body data: we need to check the status line */ - if (!http->rep->content_length) - eofflag = !(http->readbuf->offset - headerlen) ? FILTER_EOF : 0; - else - eofflag = 0; - /* Allow a new read to be scheduled */ - readbuf=http->readbuf; - http->readbuf=NULL; - rvflags = temp_filter->filter_hdr(http->rep, http->hcrequest ,&http->hcrequest->repfilters,temp_filter, eofflag, temp_filter->data); - debug (33,3)("Process request got flags %0x from rep_header\n", rvflags); - if ((readbuf->offset-headerlen) - && !(rvflags & (FILTER_ABORT | FILTER_EOF))) { - temp_filter=http->hcrequest->repfilters.head->data; - http->read_offset+=readbuf->offset-headerlen; - /* offset total length in buffer */ - rvflags=temp_filter->filter(readbuf->buffer+readbuf->offset+headerlen-howmuch,readbuf->offset-headerlen,0,&http->hcrequest->repfilters, - temp_filter,0,temp_filter->data); - debug (33,3)("Process request got flags %0x from rep_filter\n", rvflags); - } - /* all data written */ - ioBufferUnlock(readbuf); - - - } - } else { - /* couldn't parse the reply */ - http->readbuf->offset+=howmuch; - if (http->readbuf->offset >= http->readbuf->size) - fatalf("more than buffer size (%d) without a parsable response header\n",http->readbuf->size); - } - /* it's up to the filter chain to schedule another read */ -} - -/* push the body up to the server */ -DATAFILTER_FILTER(http_server_req_body) -{ - /* check for EOF or content length overrun */ - /* check offset is valid. (=httpstate offset) - update httpstate offset - it is not updated by the commcomplete routine */ - return 0; -} - -/* we could check for request->content_length... but what if that gets... corrupted? - */ -DATAFILTER_FILTER(http_server_unexpected_body) -{ - fatal("unexpected body.. abort request/reply cleanly here\n"); - return 0; -} - -/* the header has been called.. should never be again - * remove me once stable code - */ -DATAFILTER_FILTERHEADER(http_trap) -{ - fatal("We should never have been called\n"); - return 0; -} - -static CWCB httpCommWriteComplete; -static CRCB httpCommReadComplete; -static FILTERREAD httpServerRead; - -static void -httpServerRead(void * data) -{ - HttpStateData *http=data; - iobuf *readbuf; - /* allocate a buffer. This is annoying: could comm_read allocate it for us? - * (when we pass a NULL buf) */ - if (http->readbuf) - /* theres a write to the client in progress, or an attempt to queue multiple reads*/ - return; - readbuf = ioBufferAlloc(4096); - assert(readbuf); - http->readbuf=readbuf; - comm_read(http->fd, readbuf->buffer, readbuf->offset, readbuf->size, httpCommReadComplete, http); -} - -/* we are the last in the filter chain. Always. */ -/* Unknowns: how do we detect early server writes: for example if the server rejects our - * header (ie content_length too long) - */ -static void -httpCommWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) -{ - HttpStateData *http=data; - if (http->hcrequest->request->content_length && ~(http->filterflags & FILTER_EOF )) { - /* there is a body.. and we haven't finished */ -// hcrequest->read(http->hcrequest); - } - /* there is no more body */ - httpServerRead(data); -} - -/* helper function for http_server_req. - * We modify the original request. Why? speed. - * This raises a general issue: - * - * the common-to-all-upstream headers should be processed once. - * then we process the per-upstream headers on each forward. - * this either requires a filter for the one-time which we remove, or - * a flag in the httpstate/request variable. - * TODO: DO THIS BEFORE THE FATALS RECOMMENDED BELOW - */ -void -httpPrepareRequestHeader(request_t * request, - http_state_flags flags) -{ - /* building buffer for complex strings */ -#define BBUF_SZ (MAX_URL+32) - LOCAL_ARRAY(char, bbuf, BBUF_SZ); - String strConnection = StringNull; - int we_do_ranges; - const HttpHeaderEntry *e; - const int hops; - String strVia; - String strFwd; - HttpHeaderPos pos = HttpHeaderInitPos; - -#if 0 - httpHeaderInit(hdr_out, hoRequest); - /* append our IMS header */ - if (request->lastmod > -1 && request->method == METHOD_GET) - httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); -#endif - - /* decide if we want to do Ranges ourselves - * (and fetch the whole object now) - * We want to handle Ranges ourselves iff - * - we can actually parse client Range specs - * - the specs are expected to be simple enough (e.g. no out-of-order ranges) - * - reply will be cachable - * (If the reply will be uncachable we have to throw it away after - * serving this request, so it is better to forward ranges to - * the server and fetch only the requested content) - */ - -/* Dump the headers - debugging purposes only*/ -{ - HttpHeaderPos pos = HttpHeaderInitPos; - const HttpHeaderEntry *e; - debug(55, 7) ("packing hdr: (%p)\n", request->header); - /* pack all entries one by one */ - while ((e = httpHeaderGetEntry(&request->header, &pos))) -printf("%s: %s\r\n",strBuf(e->name),strBuf(e->value)); -} - - if (NULL == request->range) - we_do_ranges = 0; - else if (!request->flags.cachable) - we_do_ranges = 0; - else if (httpHdrRangeOffsetLimit(request->range)) - we_do_ranges = 0; - else - we_do_ranges = 1; - debug(11, 8) ("httpPrepareRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d -\n", - request->range, request->flags.cachable, we_do_ranges); - -/* remove any and all headers we don't want here - so that the loop isn't broken */ -if (httpHeaderHas(&request->header, HDR_MAX_FORWARDS)) { - hops = httpHeaderGetInt(&request->header, HDR_MAX_FORWARDS); - httpHeaderDelById(&request->header, HDR_MAX_FORWARDS); - /* sacrificing efficiency over clarity, etc. */ - /* why don't we check for 0? because we are never reached for 0 */ - httpHeaderPutInt(&request->header, HDR_MAX_FORWARDS, hops - 1); -} -if (we_do_ranges) { - httpHeaderDelById(&request->header, HDR_RANGE); - httpHeaderDelById(&request->header, HDR_IF_RANGE); - httpHeaderDelById(&request->header, HDR_REQUEST_RANGE); -} -httpHeaderDelById(&request->header, HDR_PROXY_CONNECTION); -httpHeaderDelById(&request->header, HDR_CONNECTION); - - strConnection = httpHeaderGetList(&request->header, HDR_CONNECTION); - while ((e = httpHeaderGetEntry(&request->header, &pos))) { - debug(11, 5) ("httpPrepareRequestHeader: %s: %s\n", - strBuf(e->name), strBuf(e->value)); - if (!httpRequestHdrAllowed(e, &strConnection)) { - debug(11, 2) ("'%s' header denied by anonymize_headers configuration\n", - strBuf(e->name)); - continue; - } - switch (e->id) { - case HDR_PROXY_AUTHORIZATION: - /* FIXME: client side todo: remove the auth if it's not needed. - * Then we simply test: is it present? no->add peer credentials */ - /* Only pass on proxy authentication to peers for which - * authentication forwarding is explicitly enabled - */ -#if 0 - if (request->flags.proxying && request->peer_login && - strcmp(request->peer_login, "PASS") == 0) { - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - } - -#endif - break; - case HDR_AUTHORIZATION: - /* FIXME: is there _ever_ anything to do here? */ - /* Pass on WWW authentication even if used locally. If this is - * not wanted in an accelerator then the header can be removed - * using the anonymization functions - */ -#if 0 - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); -#endif - /* XXX Some accelerators might want to strip the header - * and regard the reply as cacheable, but authentication - * is not normally enabled for accelerators without reading - * the code, so there is not much use in adding logics here - * without first defining the concept of having authentication - * in the accelerator... - */ - break; - case HDR_HOST: - /* - * Normally Squid does not copy the Host: header from - * a client request into the forwarded request headers. - * However, there is one case when we do: If the URL - * went through our redirector and the admin configured - * 'redir_rewrites_host' to be off. - */ - /* FIXME: this seems broken: host is a required header for - * HTTP/1.1 - */ -#if 0 - if (request->flags.redirected) - if (!Config.onoff.redir_rewrites_host) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); -#endif - break; - case HDR_IF_MODIFIED_SINCE: - /* append unless we added our own; - * note: at most one client's ims header can pass through */ - /* FIXME: nothing to see, move along. (The cache adds this header itself - */ -#if 0 - if (!httpHeaderHas(&request->header, HDR_IF_MODIFIED_SINCE)) - httpHeaderAddEntry(&request->header, httpHeaderEntryClone(e)); -#endif - break; - case HDR_MAX_FORWARDS: - break; - case HDR_RANGE: - case HDR_IF_RANGE: - case HDR_REQUEST_RANGE: - /* we inherit the client range request */ - /* if we do the header logic, we have to delete the existing header */ -#if 0 - if (!we_do_ranges) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); -#endif - break; - case HDR_PROXY_CONNECTION: - /* FIXME: update client side to clean this, then fatal here to indicate a - * broken client module - */ - httpHeaderDelById(&request->header, HDR_PROXY_CONNECTION); - break; - case HDR_CONNECTION: - /* DITTO */ - httpHeaderDelById(&request->header, HDR_CONNECTION); - break; - case HDR_VIA: - case HDR_X_FORWARDED_FOR: - case HDR_CACHE_CONTROL: - /* append these after the loop if needed */ - /* FIXME: check what of these we are willing to pass on */ - break; - default: - /* pass on all other header fields */ - } - } - - /* append TE */ - { - char strTE[128]; - - /* FIXME: update client_side, then fatal here if the header is present */ - httpHeaderDelById(&request->header, HDR_TE); /* hop by hop.. that's what last - hop could do*/ - /* TODO: this should be built from a list of known types & a acl allowing type - * to be used in requests vs responses - */ - strcpy (strTE,"chunked;q=1.0"); - - httpHeaderPutStr (&request->header,HDR_TE,strTE); - httpHeaderPutStr (&request->header,HDR_CONNECTION, "TE"); /* its hop by hop */ - - } - - /* append Via */ - /* DOES THIS WORK?! or do we need to delete and add the header ? */ - strVia = httpHeaderGetList(&request->header, HDR_VIA); - /* this is broken: we want out http version don't we? */ - snprintf(bbuf, BBUF_SZ, "%d.%d %s", - request->http_ver.major, - request->http_ver.minor, ThisCache); - strListAdd(&strVia, bbuf, ','); - stringClean(&strVia); -#if 0 - strVia = httpHeaderGetList(&request->header, HDR_VIA); - snprintf(bbuf, BBUF_SZ, "%d.%d %s", - orig_request->http_ver.major, - orig_request->http_ver.minor, ThisCache); - strListAdd(&strVia, bbuf, ','); - httpHeaderPutStr(hdr_out, HDR_VIA, strBuf(strVia)); - stringClean(&strVia); -#endif - -#if 0 - /* the client side should choose to add this. We cannot depend on having a - * client. (think timed cache refreshes for instance) - */ - - /* append X-Forwarded-For */ - strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); - strListAdd(&strFwd, (cfd < 0 ? "unknown" : fd_table[cfd].ipaddr), ','); - httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd)); - stringClean(&strFwd); -#endif - - /* append Host if not there already */ - if (!httpHeaderHas(&request->header, HDR_HOST)) { - /* use port# only if not default */ - if (request->port == urlDefaultPort(request->protocol)) { - httpHeaderPutStr(&request->header, HDR_HOST, request->host); - } else { - httpHeaderPutStrf(&request->header, HDR_HOST, "%s:%d", - request->host, (int) request->port); - } - } - /* append Authorization if known in URL, not in header and going direct */ - if (!httpHeaderHas(&request->header, HDR_AUTHORIZATION)) { - if (!request->flags.proxying && *request->login) { - httpHeaderPutStrf(&request->header, HDR_AUTHORIZATION, "Basic %s", - base64_encode(request->login)); - } - } - /* Remember: the client side strips this if not needed */ - /* append Proxy-Authorization if configured for peer, and proxying */ - if (request->flags.proxying && request->peer_login && - !httpHeaderHas(&request->header, HDR_PROXY_AUTHORIZATION) && - strcmp(request->peer_login, "PASS") != 0) { - if (*request->peer_login == '*') { - /* Special mode, to pass the username to the upstream cache */ - char loginbuf[256]; - char *username = "-"; - if (request->auth_user_request) - username = authenticateUserRequestUsername(request->auth_user_request); - snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, request->peer_login + 1); - httpHeaderPutStrf(&request->header, HDR_PROXY_AUTHORIZATION, "Basic %s", - base64_encode(loginbuf)); - } else { - httpHeaderPutStrf(&request->header, HDR_PROXY_AUTHORIZATION, "Basic %s", - base64_encode(request->peer_login)); - } - } - - /* maybe append Connection: keep-alive */ - /* what about connection: TE ? ? -rob*/ - if (flags.keepalive) { - if (flags.proxying) { - httpHeaderPutStr(&request->header, HDR_PROXY_CONNECTION, "keep-alive"); - } else { - httpHeaderPutStr(&request->header, HDR_CONNECTION, "keep-alive"); - } - } - /* Now mangle the headers. */ - httpHdrMangleList(&request->header, request); - stringClean(&strConnection); -} - - -/* we're attached to an fd, and have a request to make */ -unsigned int -http_server_req(HttpReply *rep, clientHttpRequest *request, - dlink_list * filter_list,FILTER_list * filters, unsigned int flags, - void *data) -{ - HttpStateData *http=data; - int cfd=-1; /* we don't care about the client fd */ - MemBuf mb; - /* FIXME: check we haven't been called to cleanup _before_ the write completed */ - httpPrepareRequestHeader(request->request, http->flags); - mb = HttpRequestPackMemBuf(request->request); - - if (request->request->content_length) - filterChainAddTail(filter_list, http_server_req_body, http_trap, NULL, data); - else - filterChainAddTail(filter_list, http_server_unexpected_body, http_trap, NULL, data); - comm_write_mbuf(http->fd, mb, httpCommWriteComplete, data); - printf("\n===\n%s===\n",mb.buf); - http->rep=rep; - http->hcrequest=request; - http->filter_list=filter_list; - http->filters=filters->node.next->data; - http->filterflags=flags; - http->hcrequest->serverread=httpServerRead; - http->hcrequest->serverreaddata=http; - cbdataLock(http->hcrequest->serverreaddata); - dlinkDelete(&filters->node, filter_list); - xfree(filters); - } - - -/* create a HttpStateData variable for the server side functions to reference. */ -void * -http_server_reqstate(FwdState * fwd) -{ - HttpStateData *httpState; - CBDATA_INIT_TYPE(HttpStateData); - httpState=cbdataAlloc(HttpStateData); - httpState->fwd = fwd; - httpState->fd = fwd->server_fd; - httpState->read_offset=-1; /*-1 means we are waiting for a response, or in headers */ - httpState->peer = fwd->servers ? fwd->servers->peer : NULL; /* might be NULL */ - httpState->flags.proxying = httpState->peer ? 1 : 0; - /* - * Is keep-alive okay for all request methods? - */ - /* global */ - if (!Config.onoff.server_pconns) - httpState->flags.keepalive = 0; - /* "proxying" mean hierachy - keep alive non hierarchy requests always */ - else if (httpState->flags.proxying == 0) - httpState->flags.keepalive = 1; - /* for this peer, less that 10, always keep alive */ - else if (httpState->peer->stats.n_keepalives_sent < 10) - httpState->flags.keepalive = 1; - /* whats this magic for. I no understand - robert C */ - else if ((double) httpState->peer->stats.n_keepalives_recv / (double) httpState->peer->stats.n_keepalives_sent > 0.50) - httpState->flags.keepalive = 1; - if (httpState->peer && - neighborType(httpState->peer, httpState->request) == PEER_SIBLING && - !httpState->peer->options.allow_miss) - httpState->flags.only_if_cached = 1; - - /* old code created a separate server-side request. uck. what we want is a clean - * two level structure - client side stuff in a list of attached clients, and server - * side stuff in the server request. - * It _looks_ like that was done to avoid having a peer host and port struct ! - * -Rob - */ - /* - * register the handler to free HTTP state data when the FD closes - */ - /* - * This may not be correct anymore: thanks to filters, one cancel should percolate - * through anyway. FIXME - */ - comm_add_close_handler(httpState->fd, httpStateFree, httpState); - statCounter.server.all.requests++; - statCounter.server.http.requests++; - return httpState; -} - -void -httpStart(FwdState * fwd) -{ - int fd = fwd->server_fd; - HttpStateData *httpState; - request_t *proxy_req; - request_t *orig_req = fwd->request; - debug(11, 3) ("httpStart: \"%s %s\"\n", - RequestMethodStr[orig_req->method], - storeUrl(fwd->entry)); - httpState = cbdataAlloc(HttpStateData); - storeLockObject(fwd->entry); - httpState->fwd = fwd; - httpState->entry = fwd->entry; - httpState->fd = fd; - httpState->read_offset=-1; /*-1 means we are waiting for a response, or in headers */ - if (fwd->servers) - httpState->peer = fwd->servers->peer; /* might be NULL */ - if (httpState->peer) { - proxy_req = requestCreate(orig_req->method, - PROTO_NONE, storeUrl(httpState->entry)); - xstrncpy(proxy_req->host, httpState->peer->host, SQUIDHOSTNAMELEN); - proxy_req->port = httpState->peer->http_port; - proxy_req->flags = orig_req->flags; - proxy_req->lastmod = orig_req->lastmod; - httpState->request = requestLink(proxy_req); - httpState->orig_request = requestLink(orig_req); - proxy_req->flags.proxying = 1; - /* - * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here. - * We might end up getting the object from somewhere else if, - * for example, the request to this neighbor fails. - */ - if (httpState->peer->options.proxy_only) - storeReleaseRequest(httpState->entry); -#if DELAY_POOLS - assert(delayIsNoDelay(fd) == 0); - if (httpState->peer->options.no_delay) - delaySetNoDelay(fd); -#endif - } else { - httpState->request = requestLink(orig_req); - httpState->orig_request = requestLink(orig_req); - } - /* - * register the handler to free HTTP state data when the FD closes - */ - comm_add_close_handler(fd, httpStateFree, httpState); - statCounter.server.all.requests++; - statCounter.server.http.requests++; - httpSendRequest(httpState); - /* - * We used to set the read timeout here, but not any more. - * Now its set in httpSendComplete() after the full request, - * including request body, has been written to the server. - */ -} - -static void -httpSendRequestEntryDone(int fd, void *data) -{ - HttpStateData *httpState = data; - aclCheck_t ch; - debug(11, 5) ("httpSendRequestEntryDone: FD %d\n", - fd); - memset(&ch, '\0', sizeof(ch)); - ch.request = httpState->request; - if (!Config.accessList.brokenPosts) { - debug(11, 5) ("httpSendRequestEntryDone: No brokenPosts list\n"); - httpSendComplete(fd, NULL, 0, 0, data); - } else if (!aclCheckFast(Config.accessList.brokenPosts, &ch)) { - debug(11, 5) ("httpSendRequestEntryDone: didn't match brokenPosts\n"); - httpSendComplete(fd, NULL, 0, 0, data); - } else { - debug(11, 2) ("httpSendRequestEntryDone: matched brokenPosts\n"); - comm_write(fd, "\r\n", 2, httpSendComplete, data, NULL); - } -} - -static void -httpRequestBodyHandler(char *buf, size_t size, void *data) -{ - HttpStateData *httpState = (HttpStateData *) data; - if (size > 0) { - comm_write(httpState->fd, buf, size, httpSendRequestEntry, data, memFree8K); - } else if (size == 0) { - /* End of body */ - memFree8K(buf); - httpSendRequestEntryDone(httpState->fd, data); - } else { - /* Failed to get whole body, probably aborted */ - memFree8K(buf); - httpSendComplete(httpState->fd, NULL, 0, COMM_ERR_CLOSING, data); - } -} - -static void -httpSendRequestEntry(int fd, char *bufnotused, size_t size, int errflag, void *data) -{ - HttpStateData *httpState = data; - StoreEntry *entry = httpState->entry; - ErrorState *err; - debug(11, 5) ("httpSendRequestEntry: FD %d: size %d: errflag %d.\n", - fd, size, errflag); - if (size > 0) { - fd_bytes(fd, size, FD_WRITE); - kb_incr(&statCounter.server.all.kbytes_out, size); - kb_incr(&statCounter.server.http.kbytes_out, size); - } - if (errflag == COMM_ERR_CLOSING) - return; - if (errflag) { - err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); - err->xerrno = errno; - err->request = requestLink(httpState->orig_request); - errorAppendEntry(entry, err); - comm_close(fd); - return; - } - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - comm_close(fd); - return; - } - clientReadBody(httpState->orig_request, memAllocate(MEM_8K_BUF), 8192, httpRequestBodyHandler, httpState); -} - -void -httpBuildVersion(http_version_t * version, unsigned int major, unsigned int minor) -{ - version->major = major; - version->minor = minor; -} Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.1.1.3.8.11.2.20.2.5 retrieving revision 1.1.1.3.8.11.2.20.2.6 diff -u -r1.1.1.3.8.11.2.20.2.5 -r1.1.1.3.8.11.2.20.2.6 --- squid/src/protos.h 7 May 2001 13:23:41 -0000 1.1.1.3.8.11.2.20.2.5 +++ squid/src/protos.h 9 May 2001 12:22:15 -0000 1.1.1.3.8.11.2.20.2.6 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.1.1.3.8.11.2.20.2.5 2001/05/07 13:23:41 rbcollins Exp $ + * $Id: protos.h,v 1.1.1.3.8.11.2.20.2.6 2001/05/09 12:22:15 rbcollins Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1333,5 +1333,3 @@ extern DATAFILTER_HDR identity_header; extern DATAFILTER identity_body; -/* Vary support functions */ -int varyEvaluateMatch(StoreEntry * entry, request_t * req); --- /dev/null Wed Feb 14 00:52:54 2007 +++ squid/src/modules/http_client/Makefile.in Wed Feb 14 00:53:08 2007 @@ -0,0 +1,70 @@ +# +# Makefile for the ntlm authentication scheme module for the Squid Object Cache server +# +# $Id: Makefile.in,v 1.1.2.1 2001/05/09 12:22:15 rbcollins Exp $ +# + +MODULE = http_client + +#SUBDIRS = helpers + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(MODULE).a + +OBJS = \ + client_side.o + + +all install: $(OUT) + @for dir in $(SUBDIRS); do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) $@" || exit 1; \ + fi; \ + done; + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(MODULE).a + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) clean"; \ + fi; \ + done + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) distclean"; \ + fi; \ + done + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c --- /dev/null Wed Feb 14 00:52:54 2007 +++ squid/src/modules/http_client/client_side.c Wed Feb 14 00:53:08 2007 @@ -0,0 +1,2781 @@ + +/* + * $Id: client_side.c,v 1.1.2.1 2001/05/09 12:22:15 rbcollins Exp $ + * + * DEBUG: section 33 Client-side Routines + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +#if IPF_TRANSPARENT +#if HAVE_SYS_IOCTL_H +#include +#endif +#include +#include +#if HAVE_IP_FIL_COMPAT_H +#include +#elif HAVE_NETINET_IP_FIL_COMPAT_H +#include +#elif HAVE_IP_COMPAT_H +#include +#elif HAVE_NETINET_IP_COMPAT_H +#include +#endif +#if HAVE_IP_FIL_H +#include +#elif HAVE_NETINET_IP_FIL_H +#include +#endif +#if HAVE_IP_NAT_H +#include +#elif HAVE_NETINET_IP_NAT_H +#include +#endif +#endif + +#if LINUX_NETFILTER +#include +#endif + +#if LINGERING_CLOSE +#define comm_close comm_lingering_close +#endif + +#include "broker.h" + +static const char *const crlf = "\r\n"; + +#define FAILURE_MODE_TIME 300 + +/* temp function */ +void +clientHttpInit(void) +{ +} + +/* Local functions */ + +static CWCB newclientWriteComplete; +static CWCB clientWriteBodyComplete; +static PF clientReadRequest; +static PF connStateFree; +static PF requestTimeout; +static PF clientLifetimeTimeout; +static int clientCheckTransferDone(clientHttpRequest *); +static void checkFailureRatio(err_type, hier_code); +static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri); +static clientHttpRequest *parseHttpRequest(ConnStateData *, method_t *, int *, size_t *); +static STCB clientHandleIMSReply; +static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); +static int checkAccelOnly(clientHttpRequest *); +#if USE_IDENT +static IDCB clientIdentDone; +#endif +static int clientOnlyIfCached(clientHttpRequest * http); +static void clientSetKeepaliveFlag(clientHttpRequest *); +static void clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb); +static void clientPackTermBound(String boundary, MemBuf * mb); +static DATAFILTER_HDR clientInterpretRequestHeaders; +static void clientProcessExpired(void *data); +static void clientProcessOnlyIfCachedMiss(clientHttpRequest * http); +static int clientHierarchical(clientHttpRequest * http); +static int clientCheckContentLength(request_t * r); +static DEFER httpAcceptDefer; +static int clientReplyBodyTooLarge(int clen); +static int clientRequestBodyTooLarge(int clen); +static void clientProcessBody(ConnStateData * conn); + + +static DATAFILTER_HDR clientTemp; +#if HEADERS_LOG +static DATAFILTER_HDR clientHeadersLog; +#endif +static DATAFILTER_HDR clientfdnote; +static DATAFILTER_HDR httplocalmethods; + + +/* register as a module */ +void +mod_install_http_client (const char *namestr) +{ +/* FIXME: register as a client side protocol, with a callback for ports to open etc */ + /* Register as a potential client_side reply filter */ +#if 0 + filterRegisterModule (namestr, SpyFilter_AddInstance, + SpyFilter_RemInstance); +#endif +} + +/* deregister as a module */ +void +mod_uninstall_http_client (const char *namestr) { +#if 0 + filterDeregisterModule(namestr); +#endif +} + +static int +checkAccelOnly(clientHttpRequest * http) +{ + /* return TRUE if someone makes a proxy request to us and + * we are in httpd-accel only mode */ + if (!Config2.Accel.on) + return 0; + if (Config.onoff.accel_with_proxy) + return 0; + if (http->request->protocol == PROTO_CACHEOBJ) + return 0; + if (http->flags.accel) + return 0; + return 1; +} + +#if USE_IDENT +static void +clientIdentDone(const char *ident, void *data) +{ + ConnStateData *conn = data; + xstrncpy(conn->rfc931, ident ? ident : dash_str, USER_IDENT_SZ); +} + +#endif + +static aclCheck_t * +clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http) +{ + aclCheck_t *ch; + ConnStateData *conn = http->conn; + ch = aclChecklistCreate(acl, + http->request, + conn->rfc931); + + /* + * hack for ident ACL. It needs to get full addresses, and a + * place to store the ident result on persistent connections... + */ + /* connection oriented auth also needs these two lines for it's operation. */ + ch->conn = conn; + cbdataLock(ch->conn); + + return ch; +} + +typedef struct _accessstate { + HttpReply *rep; + clientHttpRequest *request; + dlink_list * filter_list; + FILTER_list * filters; + unsigned int flags; + void *data; +} accessstate; + +CBDATA_TYPE(accessstate); + +DATAFILTER_FILTERHEADER(clientAccessCheck) +{ + clientHttpRequest *http = data; + accessstate *AccessState = NULL; + CBDATA_INIT_TYPE(accessstate); + AccessState = cbdataAlloc(accessstate); + AccessState->rep=rep; + AccessState->request=request; + AccessState->filter_list=filter_list; + AccessState->filters=filters; + AccessState->flags=flags; + AccessState->data=data; + cbdataLock(AccessState); + if (checkAccelOnly(http)) { + /* deny proxy requests in accel_only mode */ + debug(33, 1) ("clientAccessCheck: proxy request denied in accel_only mode\n"); + clientAccessCheckDone(ACCESS_DENIED, AccessState); + return FILTER_ABORT; + } + + http->acl_checklist = clientAclChecklistCreate(Config.accessList.http, http); + aclNBCheck(http->acl_checklist, clientAccessCheckDone, AccessState); + return 0; +} + +/* + * returns true if client specified that the object must come from the cache + * without contacting origin server + */ +static int +clientOnlyIfCached(clientHttpRequest * http) +{ + const request_t *r = http->request; + assert(r); + return r->cache_control && + EBIT_TEST(r->cache_control->mask, CC_ONLY_IF_CACHED); +} + +StoreEntry * +clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags) +{ + StoreEntry *e; + /* + * For erroneous requests, we might not have a h->request, + * so make a fake one. + */ + if (h->request == NULL) + h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); + e = storeCreateEntry(h->uri, h->log_uri, flags, m); + h->sc = storeClientListAdd(e, h); +#if DELAY_POOLS + delaySetStoreClient(h->sc, delayClient(h->request)); +#endif +/* storeClientCopy(h->sc, e, 0, 0, CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, h); + */ return e; +} + +void +clientAccessCheckDone(int answer, void *data) +{ + accessstate *AccessState = data; + clientHttpRequest *http = AccessState->data; + err_type page_id; + http_status status; + ErrorState *err = NULL; + char *proxy_auth_msg = NULL; + FILTER_list *temp_filter; + temp_filter=AccessState->filters->node.next->data; + debug(33, 2) ("The request %s %s is %s, because it matched '%s'\n", + RequestMethodStr[http->request->method], http->uri, + answer == ACCESS_ALLOWED ? "ALLOWED" : "DENIED", + AclMatchedName ? AclMatchedName : "NO ACL's"); + proxy_auth_msg = authenticateAuthUserRequestMessage(http->conn->auth_user_request ? http->conn->auth_user_request : http->request->auth_user_request); + http->acl_checklist = NULL; + if (answer == ACCESS_ALLOWED) { +/* WHY DO THIS? We just created http->uri before + safe_free(http->uri); + http->uri = xstrdup(urlCanonical(http->request)); +*/ + temp_filter->filter_hdr(AccessState->rep, AccessState->request, AccessState->filter_list, temp_filter, AccessState->flags, temp_filter->data); + } else { + /* FIXME: tell the filter chain we are aborting */ + + debug(33, 5) ("Access Denied: %s\n", http->uri); + debug(33, 5) ("AclMatchedName = %s\n", + AclMatchedName ? AclMatchedName : ""); + debug(33, 5) ("Proxy Auth Message = %s\n", + proxy_auth_msg ? proxy_auth_msg : ""); + /* + * NOTE: get page_id here, based on AclMatchedName because + * if USE_DELAY_POOLS is enabled, then AclMatchedName gets + * clobbered in the clientCreateStoreEntry() call + * just below. Pedro Ribeiro + */ + page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName); + http->log_type = LOG_TCP_DENIED; + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags); + if (answer == ACCESS_REQ_PROXY_AUTH ) { +/* The test above was + * if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) { + * + * which is broken - the aclIs funciton didn't match proxy_auth_regex. + * But it also seems redundant - ACCESS_REQ_PROXY_AUTH is always set. + * Implementing aclIsProxyAuth is less efficient that checking a binary value. + * FIXME: if things fall through reimplementaclIsProxyAuth + */ + if (!http->flags.accel) { + /* Proxy authorisation needed */ + status = HTTP_PROXY_AUTHENTICATION_REQUIRED; + } else { + /* WWW authorisation needed */ + status = HTTP_UNAUTHORIZED; + } + if (page_id == ERR_NONE) + page_id = ERR_CACHE_ACCESS_DENIED; + } else { + status = HTTP_FORBIDDEN; + if (page_id == ERR_NONE) + page_id = ERR_ACCESS_DENIED; + } + err = errorCon(page_id, status); + err->request = requestLink(http->request); + err->src_addr = http->conn->peer.sin_addr; + if (http->conn->auth_user_request) + err->auth_user_request = http->conn->auth_user_request; + else if (http->request->auth_user_request) + err->auth_user_request = http->request->auth_user_request; + /* lock for the error state */ + if (err->auth_user_request) + authenticateAuthUserRequestLock(err->auth_user_request); + err->callback_data = NULL; + errorAppendEntry(http->entry, err); + } + dlinkDelete(&AccessState->filters->node, AccessState->filter_list); + xfree(AccessState->filters); + cbdataUnlock(AccessState); +} + +static void +httpRequestFree(void *data) +{ + clientHttpRequest *http = data; + clientHttpRequest **H; + ConnStateData *conn = http->conn; + StoreEntry *e; + request_t *request = http->request; + MemObject *mem = NULL; + debug(33, 3) ("httpRequestFree: %s\n", storeUrl(http->entry)); + if (!clientCheckTransferDone(http)) { + if (request && request->body_connection) + clientAbortBody(request); /* abort body transter */ +#if MYSTERIOUS_CODE + /* + * DW: this seems odd here, is it really needed? It causes + * incomplete transfers to get logged with "000" status + * code because http->entry becomes NULL. + */ + if ((e = http->entry)) { + http->entry = NULL; + storeUnregister(http->sc, e, http); + storeUnlockObject(e); + } +#endif + if (http->entry && http->entry->ping_status == PING_WAITING) + storeReleaseRequest(http->entry); + } + assert(http->log_type < LOG_TYPE_MAX); + if (http->entry) + mem = http->entry->mem_obj; + if (http->out.size || http->log_type) { + http->al.icp.opcode = ICP_INVALID; + http->al.url = http->log_uri; + debug(33, 9) ("httpRequestFree: al.url='%s'\n", http->al.url); + if (mem) { + http->al.http.code = mem->reply->sline.status; + http->al.http.content_type = strBuf(mem->reply->content_type); + } + http->al.cache.caddr = conn->log_addr; + http->al.cache.size = http->out.size; + http->al.cache.code = http->log_type; + http->al.cache.msec = tvSubMsec(http->start, current_time); + if (request) { + Packer p; + MemBuf mb; + memBufDefInit(&mb); + packerToMemInit(&p, &mb); + httpHeaderPackInto(&request->header, &p); + http->al.http.method = request->method; + http->al.http.version = request->http_ver; + http->al.headers.request = xstrdup(mb.buf); + http->al.hier = request->hier; + if (request->auth_user_request) { + http->al.cache.authuser = xstrdup(authenticateUserRequestUsername(request->auth_user_request)); + authenticateAuthUserRequestUnlock(request->auth_user_request); + request->auth_user_request = NULL; + } + if (conn->rfc931[0]) + http->al.cache.rfc931 = conn->rfc931; + packerClean(&p); + memBufClean(&mb); + } + accessLogLog(&http->al); + clientdbUpdate(conn->peer.sin_addr, http->log_type, PROTO_HTTP, http->out.size); + } + if (http->acl_checklist) + aclChecklistFree(http->acl_checklist); + filterCleanChain(&http->repfilters); + if (request) + checkFailureRatio(request->err_type, http->al.hier.code); + safe_free(http->uri); + safe_free(http->log_uri); + safe_free(http->al.headers.request); + safe_free(http->al.headers.reply); + safe_free(http->redirect.location); + stringClean(&http->range_iter.boundary); + if ((e = http->entry)) { + http->entry = NULL; + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + } + /* old_entry might still be set if we didn't yet get the reply + * code in clientHandleIMSReply() */ + if ((e = http->old_entry)) { + http->old_entry = NULL; + storeUnregister(http->old_sc, e, http); + http->old_sc = NULL; + storeUnlockObject(e); + } + requestUnlink(http->request); + assert(http != http->next); + assert(http->conn->chr != NULL); + /* Unlink us from the clients request list */ + H = &http->conn->chr; + while (*H) { + if (*H == http) + break; + H = &(*H)->next; + } + assert(*H != NULL); + *H = http->next; + http->next = NULL; + dlinkDelete(&http->active, &ClientActiveRequests); + cbdataFree(http); +} + +/* This is a handler normally called by comm_close() */ +static void +connStateFree(int fd, void *data) +{ + ConnStateData *connState = data; + clientHttpRequest *http; + debug(33, 3) ("connStateFree: FD %d\n", fd); + assert(connState != NULL); + authenticateOnCloseConnection(connState); + clientdbEstablished(connState->peer.sin_addr, -1); /* decrement */ + while ((http = connState->chr) != NULL) { + assert(http->conn == connState); + assert(connState->chr != connState->chr->next); + httpRequestFree(http); + } + if (connState->in.size == CLIENT_REQ_BUF_SZ) + memFree(connState->in.buf, MEM_CLIENT_REQ_BUF); + else + safe_free(connState->in.buf); + /* XXX account connState->in.buf */ + pconnHistCount(0, connState->nrequests); + cbdataFree(connState); +#ifdef _SQUID_LINUX_ + /* prevent those nasty RST packets */ + { + char buf[SQUID_TCP_SO_RCVBUF]; + while (FD_READ_METHOD(fd, buf, SQUID_TCP_SO_RCVBUF) > 0); + } +#endif +} + + +DATAFILTER_FILTERHEADER (clientInterpretRequestHeaders) +{ + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; + clientHttpRequest * http=data; +// request_t *request = http->request; + const HttpHeader *req_hdr = &request->request->header; + int no_cache = 0; + const char *str; + request->request->imslen = -1; + request->request->ims = httpHeaderGetTime(req_hdr, HDR_IF_MODIFIED_SINCE); + if (request->request->ims > 0) + request->request->flags.ims = 1; + if (httpHeaderHas(req_hdr, HDR_PRAGMA)) { + String s = httpHeaderGetList(req_hdr, HDR_PRAGMA); + if (strListIsMember(&s, "no-cache", ',')) + no_cache++; + stringClean(&s); + } + request->request->cache_control = httpHeaderGetCc(req_hdr); + if (request->request->cache_control) + if (EBIT_TEST(request->request->cache_control->mask, CC_NO_CACHE)) + no_cache++; + /* Work around for supporting the Reload button in IE browsers + * when Squid is used as an accelerator or transparent proxy, + * by turning accelerated IMS request to no-cache requests. + * Now knows about IE 5.5 fix (is actually only fixed in SP1, + * but we can't tell whether we are talking to SP1 or not so + * all 5.5 versions are treated 'normally'). + */ + if (Config.onoff.ie_refresh) { + if (http->flags.accel && request->request->flags.ims) { + if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) { + if (strstr(str, "MSIE 5.01") != NULL) + no_cache++; + else if (strstr(str, "MSIE 5.0") != NULL) + no_cache++; + else if (strstr(str, "MSIE 4.") != NULL) + no_cache++; + else if (strstr(str, "MSIE 3.") != NULL) + no_cache++; + } + } + } + if (no_cache) { +#if HTTP_VIOLATIONS + if (Config.onoff.reload_into_ims) + request->request->flags.nocache_hack = 1; + else if (refresh_nocache_hack) + request->request->flags.nocache_hack = 1; + else +#endif + request->request->flags.nocache = 1; + } + /* ignore range header in non-GETs */ + if (request->request->method == METHOD_GET) { + request->request->range = httpHeaderGetRange(req_hdr); + if (request->request->range) + request->request->flags.range = 1; + } + if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) + request->request->flags.auth = 1; + if (request->request->login[0] != '\0') + request->request->flags.auth = 1; + if (httpHeaderHas(req_hdr, HDR_VIA)) { + String s = httpHeaderGetList(req_hdr, HDR_VIA); + /* + * ThisCache cannot be a member of Via header, "1.0 ThisCache" can. + * Note ThisCache2 has a space prepended to the hostname so we don't + * accidentally match super-domains. + */ + if (strListIsSubstr(&s, ThisCache2, ',')) { + debugObj(33, 1, "WARNING: Forwarding loop detected for:\n", + request->request, (ObjPackMethod) & httpRequestPack); + request->request->flags.loopdetect = 1; + } +#if FORW_VIA_DB + fvdbCountVia(strBuf(s)); +#endif + stringClean(&s); + } +#if USE_USERAGENT_LOG + if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) + logUserAgent(fqdnFromAddr(http->conn->peer.sin_addr), str); +#endif +#if USE_REFERER_LOG + if ((str = httpHeaderGetStr(req_hdr, HDR_REFERER))) + logReferer(fqdnFromAddr(http->conn->peer.sin_addr), str, + http->log_uri); +#endif +#if FORW_VIA_DB + if (httpHeaderHas(req_hdr, HDR_X_FORWARDED_FOR)) { + String s = httpHeaderGetList(req_hdr, HDR_X_FORWARDED_FOR); + fvdbCountForw(strBuf(s)); + stringClean(&s); + } +#endif + if (request->request->method == METHOD_TRACE) { + request->request->max_forwards = httpHeaderGetInt(req_hdr, HDR_MAX_FORWARDS); + } + /* allow caching of the request - doesn't _require_ caching. Other filters can + * override this + */ + request->request->flags.cachable = 1; + if (clientHierarchical(http)) + request->request->flags.hierarchical = 1; + debug(33, 5) ("clientInterpretRequestHeaders: REQ_NOCACHE = %s\n", + request->request->flags.nocache ? "SET" : "NOT SET"); + debug(33, 5) ("clientInterpretRequestHeaders: REQ_CACHABLE = %s\n", + request->request->flags.cachable ? "SET" : "NOT SET"); + debug(33, 5) ("clientInterpretRequestHeaders: REQ_HIERARCHICAL = %s\n", + request->request->flags.hierarchical ? "SET" : "NOT SET"); + rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); + dlinkDelete(&filters->node, filter_list); + xfree(filters); + return rvflags; +} + +/* + * clientSetKeepaliveFlag() sets request->flags.proxy_keepalive. + * This is the client-side persistent connection flag. We need + * to set this relatively early in the request processing + * to handle hacks for broken servers and clients. + */ +static void +clientSetKeepaliveFlag(clientHttpRequest * http) +{ + request_t *request = http->request; + const HttpHeader *req_hdr = &request->header; + debug(33, 3) ("clientSetKeepaliveFlag: http_ver = %d.%d\n", + request->http_ver.major, request->http_ver.minor); + debug(33, 3) ("clientSetKeepaliveFlag: method = %s\n", + RequestMethodStr[request->method]); + if (!Config.onoff.client_pconns) + request->flags.proxy_keepalive = 0; + else if (httpMsgIsPersistent(request->http_ver, req_hdr)) + request->flags.proxy_keepalive = 1; +} + +static int +clientCheckContentLength(request_t * r) +{ + switch (r->method) { + case METHOD_PUT: + case METHOD_POST: + /* PUT/POST requires a request entity */ + return (r->content_length >= 0); + case METHOD_GET: + case METHOD_HEAD: + /* We do not want to see a request entity on GET/HEAD requests */ + return (r->content_length <= 0); + default: + /* For other types of requests we don't care */ + return 1; + } + /* NOT REACHED */ +} + +/* Return true if we can query our neighbors for this object */ +static int +clientHierarchical(clientHttpRequest * http) +{ + const char *url = http->uri; + request_t *request = http->request; + method_t method = request->method; + const wordlist *p = NULL; + + /* IMS needs a private key, so we can use the hierarchy for IMS only + * if our neighbors support private keys */ + if (request->flags.ims && !neighbors_do_private_keys) + return 0; + if (request->flags.auth) + return 0; + if (method == METHOD_TRACE) + return 1; + if (method != METHOD_GET) + return 0; + /* scan hierarchy_stoplist */ + for (p = Config.hierarchy_stoplist; p; p = p->next) + if (strstr(url, p->key)) + return 0; + if (request->flags.loopdetect) + return 0; + if (request->protocol == PROTO_HTTP) + return httpCachable(method); + if (request->protocol == PROTO_GOPHER) + return gopherCachable(url); + if (request->protocol == PROTO_WAIS) + return 0; + if (request->protocol == PROTO_CACHEOBJ) + return 0; + return 1; +} + +int +isTcpHit(log_type code) +{ + /* this should be a bitmap for better optimization */ + if (code == LOG_TCP_HIT) + return 1; + if (code == LOG_TCP_IMS_HIT) + return 1; + if (code == LOG_TCP_REFRESH_FAIL_HIT) + return 1; + if (code == LOG_TCP_REFRESH_HIT) + return 1; + if (code == LOG_TCP_NEGATIVE_HIT) + return 1; + if (code == LOG_TCP_MEM_HIT) + return 1; + if (code == LOG_TCP_OFFLINE_HIT) + return 1; + return 0; +} + +/* + * returns true if If-Range specs match reply, false otherwise + */ +static int +clientIfRangeMatch(clientHttpRequest * http, HttpReply * rep) +{ + const TimeOrTag spec = httpHeaderGetTimeOrTag(&http->request->header, HDR_IF_RANGE); + /* check for parsing falure */ + if (!spec.valid) + return 0; + /* got an ETag? */ + if (spec.tag.str) { + ETag rep_tag = httpHeaderGetETag(&rep->header, HDR_ETAG); + debug(33, 3) ("clientIfRangeMatch: ETags: %s and %s\n", + spec.tag.str, rep_tag.str ? rep_tag.str : ""); + if (!rep_tag.str) + return 0; /* entity has no etag to compare with! */ + if (spec.tag.weak || rep_tag.weak) { + debug(33, 1) ("clientIfRangeMatch: Weak ETags are not allowed in If-Range: %s ? %s\n", + spec.tag.str, rep_tag.str); + return 0; /* must use strong validator for sub-range requests */ + } + return etagIsEqual(&rep_tag, &spec.tag); + } + /* got modification time? */ + if (spec.time >= 0) { + return http->entry->lastmod <= spec.time; + } + assert(0); /* should not happen */ + return 0; +} + +/* returns expected content length for multi-range replies + * note: assumes that httpHdrRangeCanonize has already been called + * warning: assumes that HTTP headers for individual ranges at the + * time of the actuall assembly will be exactly the same as + * the headers when clientMRangeCLen() is called */ +static int +clientMRangeCLen(clientHttpRequest * http) +{ + int clen = 0; + HttpHdrRangePos pos = HttpHdrRangeInitPos; + const HttpHdrRangeSpec *spec; + MemBuf mb; + + assert(http->entry->mem_obj); + + memBufDefInit(&mb); + while ((spec = httpHdrRangeGetSpec(http->request->range, &pos))) { + + /* account for headers for this range */ + memBufReset(&mb); + clientPackRangeHdr(http->entry->mem_obj->reply, + spec, http->range_iter.boundary, &mb); + clen += mb.size; + + /* account for range content */ + clen += spec->length; + + debug(33, 6) ("clientMRangeCLen: (clen += %d + %d) == %d\n", + mb.size, spec->length, clen); + } + /* account for the terminating boundary */ + memBufReset(&mb); + clientPackTermBound(http->range_iter.boundary, &mb); + clen += mb.size; + + memBufClean(&mb); + return clen; +} + +static DATAFILTER clientDoRangeReply; + +/* adds appropriate Range headers if needed */ +static void +clientBuildRangeHeader(clientHttpRequest * http, HttpReply * rep) +{ + HttpHeader *hdr = rep ? &rep->header : 0; + const char *range_err = NULL; + request_t *request = http->request; + int is_hit = isTcpHit(http->log_type); + assert(request->range); + /* check if we still want to do ranges */ + if (!rep) + range_err = "no [parse-able] reply"; + else if (rep->sline.status != HTTP_OK) + range_err = "wrong status code"; + else if (httpHeaderHas(hdr, HDR_CONTENT_RANGE)) + range_err = "origin server does ranges"; + else if (rep->content_length < 0) + range_err = "unknown length"; + else if (rep->content_length != http->entry->mem_obj->reply->content_length) + range_err = "INCONSISTENT length"; /* a bug? */ + else if (httpHeaderHas(&http->request->header, HDR_IF_RANGE) && !clientIfRangeMatch(http, rep)) + range_err = "If-Range match failed"; + else if (!httpHdrRangeCanonize(http->request->range, rep->content_length)) + range_err = "canonization failed"; + else if (httpHdrRangeIsComplex(http->request->range)) + range_err = "too complex range header"; + else if (!request->flags.cachable) /* from we_do_ranges in http.c */ + range_err = "non-cachable request"; + else if (!is_hit && httpHdrRangeOffsetLimit(http->request->range)) + range_err = "range outside range_offset_limit"; + /* get rid of our range specs on error */ + if (range_err) { + debug(33, 3) ("clientBuildRangeHeader: will not do ranges: %s.\n", range_err); + httpHdrRangeDestroy(http->request->range); + http->request->range = NULL; + } else { + const int spec_count = http->request->range->specs.count; + int actual_clen = -1; + + debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %d\n", + spec_count, rep->content_length); + assert(spec_count > 0); + /* ETags should not be returned with Partial Content replies? */ + httpHeaderDelById(hdr, HDR_ETAG); + /* append appropriate header(s) */ + if (spec_count == 1) { + HttpHdrRangePos pos = HttpHdrRangeInitPos; + const HttpHdrRangeSpec *spec = httpHdrRangeGetSpec(http->request->range, &pos); + assert(spec); + /* append Content-Range */ + httpHeaderAddContRange(hdr, *spec, rep->content_length); + /* set new Content-Length to the actual number of bytes + * transmitted in the message-body */ + actual_clen = spec->length; + } else { + /* multipart! */ + /* generate boundary string */ + http->range_iter.boundary = httpHdrRangeBoundaryStr(http); + /* delete old Content-Type, add ours */ + httpHeaderDelById(hdr, HDR_CONTENT_TYPE); + httpHeaderPutStrf(hdr, HDR_CONTENT_TYPE, + "multipart/byteranges; boundary=\"%s\"", + strBuf(http->range_iter.boundary)); + /* Content-Length is not required in multipart responses + * but it is always nice to have one */ + actual_clen = clientMRangeCLen(http); + } + + /* replace Content-Length header */ + assert(actual_clen >= 0); + httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); + httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, actual_clen); + debug(33, 3) ("clientBuildRangeHeader: actual content length: %d\n", actual_clen); + + /* replace the status line */ + httpStatusLineSet(&rep->sline, rep->sline.version, HTTP_PARTIAL_CONTENT, NULL); + + /* add the ranges filter to the list */ + filterChainAddTail(&http->repfilters,clientDoRangeReply, identity_header, NULL,http); + } +} + +/* Transfer Encoding Filters: They work like this.. + + filter (a,b,c,d,e) + a - is a character pointer to the data to be filtered + b - is length of a + c - is address of a character pointer where the resulting buffer should be stored + d - pointer to integer to store length of c in + e - pointer to a general data pointer that the filter may use to maintain state, + it begins life as NULL. + +the return value is a 4 bit mask + TE_BUFFER_ALLOCATED - c is a newly allocated buffer and should be freed when the + calling function has completed the filter + TE_CHUNK_A - reserved internally for chunking MAGIC + TE_CHUNK_B - reserved internally for chunking MAGIC + TE_BUFFERING_OUTPUT - set this bit if your funciton was called with input, but did not + produce any (for instance if you're buffering it in the context provided by e.) + prevents squid from thinking EOF has been reached. + + call sequence + first call: *e is NULL but a is not and b>0 + body calls : a is not null, b>0 and value of *e determined by previous calls + last call: b==0.. good time to clean up *e if you've stored stuff there.. you + may produce output if necessary, but last call will be repeated. +*/ + +#ifndef _TE_RVALUES_ +#define _TE_RVALUES_ +#define TE_BUFFER_ALLOCATED 0x01 +/* TE_CHUNK_A = allocated temp buffer with empty response because input was length 0 + & no data had previous been processed - empty buffer case */ +/* could also be just that chunking was done */ +#define TE_CHUNK_A 0x02 +#define TE_CHUNK_B 0x04 +#define TE_BUFFERING_OUTPUT 0x08 +#endif + + +static DATAFILTER_HDR clientWriteReplyHeaders; +static DATAFILTER clientDoCommWriteMemBuf; +static DATAFILTER_HDR clientDoCommWriteHeaders; +static DATAFILTER_HDR newClientWH; +static DATAFILTER newClientW; +static DATAFILTER http_client_body; +static DATAFILTER_HDR http_client_hdr; + +void +clientTEReplyheader(clientHttpRequest * http, HttpReply * rep, dlink_list *filter_list) { + HttpHeader *hdr = &rep->header; + int y, may_apply_tes; + const char *s_ct; + String s_ce; + + /* begin list of modules here */ + may_apply_tes = 0; + + if (httpHeaderHas(hdr, HDR_CONTENT_TYPE)) { + s_ct = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE); + y = strlen(s_ct); + /* TODO: allow an acl for encoding by content-type */ + if (((y >= 5) && !strncasecmp(s_ct, "text/", 5)) + || strstr(s_ct, "postscript")) { + may_apply_tes = 1; + debug(33, 8) ("Content-Type of Application: %s\n", s_ct); + } + } + if (httpHeaderHas(hdr, HDR_CONTENT_ENCODING)) { + /* TODO: check logic here & make this an acl test */ + /* oh, base entity might be compressed.. don't want + * to double *that* */ + s_ce = httpHeaderGetList(hdr, HDR_CONTENT_ENCODING); + if ((strListIsMember(&s_ce, "gzip", ',')) || + (strListIsMember(&s_ce, "deflate", ',')) || + (strListIsMember(&s_ce, "x-deflate", ',')) || + (strListIsMember(&s_ce, "x-gzip", ',')) || + (strListIsMember(&s_ce, "x-compress", ',')) || + (strListIsMember(&s_ce, "compress", ','))) + may_apply_tes |= 0x02; /* 02 is the compress exception */ + stringClean(&s_ce); + } + + te_build_encode_xlate_list(&http->request->header, hdr, filter_list, http->request->http_ver, &http->request->flags); +} + +/* put terminating boundary for multiparts */ +static void +clientPackTermBound(String boundary, MemBuf * mb) +{ + memBufPrintf(mb, "\r\n--%s--\r\n", strBuf(boundary)); + debug(33, 6) ("clientPackTermBound: buf offset: %d\n", mb->size); +} + +/* appends a "part" HTTP header (as in a multi-part/range reply) to the buffer */ +static void +clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb) +{ + HttpHeader hdr; + Packer p; + assert(rep); + assert(spec); + + /* put boundary */ + debug(33, 5) ("clientPackRangeHdr: appending boundary: %s\n", strBuf(boundary)); + /* rfc2046 requires to _prepend_ boundary with ! */ + memBufPrintf(mb, "\r\n--%s\r\n", strBuf(boundary)); + + /* stuff the header with required entries and pack it */ + httpHeaderInit(&hdr, hoReply); + if (httpHeaderHas(&rep->header, HDR_CONTENT_TYPE)) + httpHeaderPutStr(&hdr, HDR_CONTENT_TYPE, httpHeaderGetStr(&rep->header, HDR_CONTENT_TYPE)); + httpHeaderAddContRange(&hdr, *spec, rep->content_length); + packerToMemInit(&p, mb); + httpHeaderPackInto(&hdr, &p); + packerClean(&p); + httpHeaderClean(&hdr); + + /* append (we packed a header, not a reply) */ + memBufPrintf(mb, crlf); +} + +/* + * extracts a "range" from *buf and appends them to mb, updating + * all offsets and such. + */ +static void +clientPackRange(clientHttpRequest * http, + HttpHdrRangeIter * i, + const char **buf, + ssize_t * size, + MemBuf * mb) +{ + const ssize_t copy_sz = i->debt_size <= *size ? i->debt_size : *size; + off_t body_off = http->out.offset - i->prefix_size; + assert(*size > 0); + assert(i->spec); + /* + * intersection of "have" and "need" ranges must not be empty + */ + assert(body_off < i->spec->offset + i->spec->length); + assert(body_off + *size > i->spec->offset); + /* + * put boundary and headers at the beginning of a range in a + * multi-range + */ + if (http->request->range->specs.count > 1 && i->debt_size == i->spec->length) { + assert(http->entry->mem_obj); + clientPackRangeHdr( + http->entry->mem_obj->reply, /* original reply */ + i->spec, /* current range */ + i->boundary, /* boundary, the same for all */ + mb + ); + } + /* + * append content + */ + debug(33, 3) ("clientPackRange: appending %d bytes\n", copy_sz); + memBufAppend(mb, *buf, copy_sz); + /* + * update offsets + */ + *size -= copy_sz; + i->debt_size -= copy_sz; + body_off += copy_sz; + *buf += copy_sz; + http->out.offset = body_off + i->prefix_size; /* sync */ + /* + * paranoid check + */ + assert(*size >= 0 && i->debt_size >= 0); +} + +/* returns true if there is still data available to pack more ranges + * increments iterator "i" + * used by clientPackMoreRanges */ +static int +clientCanPackMoreRanges(const clientHttpRequest * http, HttpHdrRangeIter * i, ssize_t size) +{ + /* first update "i" if needed */ + if (!i->debt_size) { + if ((i->spec = httpHdrRangeGetSpec(http->request->range, &i->pos))) + i->debt_size = i->spec->length; + } + assert(!i->debt_size == !i->spec); /* paranoid sync condition */ + /* continue condition: need_more_data && have_more_data */ + return i->spec && size > 0; +} + +/* extracts "ranges" from buf and appends them to mb, updating all offsets and such */ +/* returns true if we need more data */ +static int +clientPackMoreRanges(clientHttpRequest * http, const char *buf, ssize_t size, MemBuf * mb) +{ + HttpHdrRangeIter *i = &http->range_iter; + /* offset in range specs does not count the prefix of an http msg */ + off_t body_off = http->out.offset - i->prefix_size; + assert(size >= 0); + /* check: reply was parsed and range iterator was initialized */ + assert(i->prefix_size > 0); + /* filter out data according to range specs */ + while (clientCanPackMoreRanges(http, i, size)) { + off_t start; /* offset of still missing data */ + assert(i->spec); + start = i->spec->offset + i->spec->length - i->debt_size; + debug(33, 3) ("clientPackMoreRanges: in: offset: %d size: %d\n", + (int) body_off, size); + debug(33, 3) ("clientPackMoreRanges: out: start: %d spec[%d]: [%d, %d), len: %d debt: %d\n", + (int) start, (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length, i->debt_size); + assert(body_off <= start); /* we did not miss it */ + /* skip up to start */ + if (body_off + size > start) { + const size_t skip_size = start - body_off; + body_off = start; + size -= skip_size; + buf += skip_size; + } else { + /* has not reached start yet */ + body_off += size; + size = 0; + buf = NULL; + } + /* put next chunk if any */ + if (size) { + http->out.offset = body_off + i->prefix_size; /* sync */ + clientPackRange(http, i, &buf, &size, mb); + body_off = http->out.offset - i->prefix_size; /* sync */ + } + } + assert(!i->debt_size == !i->spec); /* paranoid sync condition */ + debug(33, 3) ("clientPackMoreRanges: buf exhausted: in: offset: %d size: %d need_more: %d\n", + (int) body_off, size, i->debt_size); + if (i->debt_size) { + debug(33, 3) ("clientPackMoreRanges: need more: spec[%d]: [%d, %d), len: %d\n", + (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length); + /* skip the data we do not need if possible */ + if (i->debt_size == i->spec->length) /* at the start of the cur. spec */ + body_off = i->spec->offset; + else + assert(body_off == i->spec->offset + i->spec->length - i->debt_size); + } else if (http->request->range->specs.count > 1) { + /* put terminating boundary for multiparts */ + clientPackTermBound(i->boundary, mb); + } + http->out.offset = body_off + i->prefix_size; /* sync */ + return i->debt_size > 0; +} + +static int +clientReplyBodyTooLarge(int clen) +{ + if (0 == Config.maxReplyBodySize) + return 0; /* disabled */ + if (clen < 0) + return 0; /* unknown */ + if (clen > Config.maxReplyBodySize) + return 1; /* too large */ + return 0; +} + +static int +clientRequestBodyTooLarge(int clen) +{ + if (0 == Config.maxRequestBodySize) + return 0; /* disabled */ + if (clen < 0) + return 0; /* unknown, bug? */ + if (clen > Config.maxRequestBodySize) + return 1; /* too large */ + return 0; +} + + +DATAFILTER clientDoCommWrite; + +DATAFILTER_FILTERHEADER(clientWriteReplyHeaders) { + clientHttpRequest *http = data; + FILTER_list *temp_filter; + unsigned int rvflags=0; + /* at this point, we are the first called filter, and only the reply headers + * are in buf. + * we _know_ buf is a membuf. + * we _don't know_ if other filters plan to touch the headers + * so all we do is pass them on, remove self from the list. + */ + + /* other filters can _depend_ on the test, http->request.out_offset==0 + * to mean that they are recieving a block that has _nothing but headers_ + */ + debug (33,8)("clientWriteReplyHeaders: Beginning\n"); + temp_filter=filters->node.next->data; + rvflags |= temp_filter->filter_hdr(http->entry->mem_obj->reply,http,filter_list, temp_filter, flags | FILTER_HTTP_HEADER, temp_filter->data); + dlinkDelete(&filters->node, filter_list); + xfree(filters); + http->flags.done_reply_headers=1; + debug (33,8)("clientWriteReplyHeaders: Finished\n"); + return rvflags; +} + +DATAFILTER_FILTER(clientDoRangeReply) { + clientHttpRequest *http = data; + FILTER_list *temp_filter; + unsigned int rvflags=0; + /* while we are only put here when expecting range requests.. I don't know if the + * request-> range variable ever gets altered. For safety this test stays (for now). + */ + + /* the membuf is temporary. Next step is to pack the ranges direct as a series of + * calls to the next filter + */ + + debug (33,8)("clientDoRangeReply: Beginning\n"); + temp_filter=filters->node.next->data; + + if (!http->flags.done_reply_headers) { + /* skip the header packet - we don't care about it */ + return temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data); + } + if (http->request->range) { + MemBuf mb; + memBufDefInit(&mb); + /* Only GET requests should have ranges */ + assert(http->request->method == METHOD_GET); + /* clientPackMoreRanges() updates http->out.offset */ + /* force the end of the transfer if we are done */ + if (!clientPackMoreRanges(http, buf, len, &mb)) + http->flags.done_copying = 1; + rvflags |= temp_filter->filter(mb.buf, mb.size, offset, filter_list, temp_filter, flags, temp_filter->data); + memBufFreeFunc(&mb); + } else { + debug(1,1)("**************\nBUG DoRangeReply called for a non range request\n************\n"); + rvflags|= temp_filter->filter(buf,len, offset,filter_list, temp_filter, flags, temp_filter->data); + dlinkDelete(&filters->node, filter_list); + xfree(filters); + } + debug (33,8)("clientDoRangeReply: Finished\n"); + return rvflags; +} + +/* Responses with no body will not have a content-type header, + * which breaks the rep_mime_type acl, which + * coincidentally, is the most common acl for reply access lists. + * A better long term fix for this is to allow acl matchs on the various + * status codes, and then supply a default ruleset that puts these + * codes before any user defines access entries. That way the user + * can choose to block these responses where appropriate, but won't get + * mysterious breakages. + */ +static int +clientAlwaysAllowResponse(http_status sline) +{ + switch (sline) { + case HTTP_CONTINUE: + case HTTP_SWITCHING_PROTOCOLS: + case HTTP_PROCESSING: + case HTTP_NO_CONTENT: + case HTTP_NOT_MODIFIED: + return 1; + /* unreached */ + break; + default: + return 0; + } +} + + +static void +clientKeepaliveNextRequest(clientHttpRequest * http) +{ + ConnStateData *conn = http->conn; + StoreEntry *entry; + debug(33, 3) ("clientKeepaliveNextRequest: FD %d\n", conn->fd); + conn->defer.until = 0; /* Kick it to read a new request */ + httpRequestFree(http); + if ((http = conn->chr) == NULL) { + debug(33, 5) ("clientKeepaliveNextRequest: FD %d reading next req\n", + conn->fd); + fd_note(conn->fd, "Waiting for next request"); + +/* + * Set the timeout BEFORE calling clientReadRequest(). + */ + commSetTimeout(conn->fd, Config.Timeout.pconn, requestTimeout, conn); + /* + * CYGWIN has a problem and is blocking on read() requests when there + * is no data present. + * This hack may hit performance a little, but it's better than + * blocking!. + */ +#ifdef _SQUID_CYGWIN_ + commSetSelect(conn->fd, COMM_SELECT_READ, clientReadRequest, conn, 0); +#else + clientReadRequest(conn->fd, conn); /* Read next request */ +#endif + /* + * Note, the FD may be closed at this point. + */ + } else if ((entry = http->entry) == NULL) { + /* + * this request is in progress, maybe doing an ACL or a redirect, + * execution will resume after the operation completes. + */ + } else { + debug(33, 1) ("clientKeepaliveNextRequest: FD %d Sending next\n", + conn->fd); + assert(entry); + if (0 == storeClientCopyPending(http->sc, entry, http)) { + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) + debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); +/* storeClientCopy(http->sc, entry, + http->out.offset, + http->out.offset, + CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), + clientSendMoreData, + http); +*/ } + } +} + +/* + * client issued a request with an only-if-cached cache-control directive; + * we did not find a cached object that can be returned without + * contacting other servers; + * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068] + */ +static void +clientProcessOnlyIfCachedMiss(clientHttpRequest * http) +{ + char *url = http->uri; + request_t *r = http->request; + ErrorState *err = NULL; + debug(33, 4) ("clientProcessOnlyIfCachedMiss: '%s %s'\n", + RequestMethodStr[r->method], url); + http->al.http.code = HTTP_GATEWAY_TIMEOUT; + err = errorCon(ERR_ONLY_IF_CACHED_MISS, HTTP_GATEWAY_TIMEOUT); + err->request = requestLink(r); + err->src_addr = http->conn->peer.sin_addr; + if (http->entry) { + storeUnregister(http->sc, http->entry, http); + http->sc = NULL; + storeUnlockObject(http->entry); + } + http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + errorAppendEntry(http->entry, err); +} + +/* + * Return true if we should force a cache miss on this range request. + * entry must be non-NULL. + */ +static int +clientCheckRangeForceMiss(StoreEntry * entry, HttpHdrRange * range) +{ + /* + * If the range_offset_limit is NOT in effect, there + * is no reason to force a miss. + */ + if (0 == httpHdrRangeOffsetLimit(range)) + return 0; + /* + * Here, we know it's possibly a hit. If we already have the + * whole object cached, we won't force a miss. + */ + if (STORE_OK == entry->store_status) + return 0; /* we have the whole object */ + /* + * Now we have a hit on a PENDING object. We need to see + * if the part we want is already cached. If so, we don't + * force a miss. + */ + assert(NULL != entry->mem_obj); + if (httpHdrRangeFirstOffset(range) <= entry->mem_obj->inmem_hi) + return 0; + /* + * Even though we have a PENDING copy of the object, we + * don't want to wait to reach the first range offset, + * so we force a miss for a new range request to the + * origin. + */ + return 1; +} + +static clientHttpRequest * +parseHttpRequestAbort(ConnStateData * conn, const char *uri) +{ + clientHttpRequest *http; + http = cbdataAlloc(clientHttpRequest); + http->conn = conn; + http->start = current_time; + http->req_sz = conn->in.offset; + http->uri = xstrdup(uri); + http->log_uri = xstrndup(uri, MAX_URL); + http->range_iter.boundary = StringNull; + dlinkAdd(http, &http->active, &ClientActiveRequests); + return http; +} + +/* + * parseHttpRequest() + * + * Returns + * NULL on error or incomplete request + * a clientHttpRequest structure on success + */ +static clientHttpRequest * +parseHttpRequest(ConnStateData * conn, method_t * method_p, int *status, + size_t * req_line_sz_p) +{ + char *inbuf = NULL; + char *mstr = NULL; + char *url = NULL; + char *req_hdr = NULL; + http_version_t http_ver; + char *token = NULL; + char *t = NULL; + char *lprefix; + char **prefix_p=&lprefix; + char *end; + size_t header_sz; /* size of headers, not including first line */ + size_t prefix_sz; /* size of whole request (req-line + headers) */ + size_t url_sz; + size_t req_sz; + method_t method; + clientHttpRequest *http = NULL; + HttpHeader hdr; + request_t *request = NULL; +#if IPF_TRANSPARENT + struct natlookup natLookup; + static int natfd = -1; + static int siocgnatl_cmd = SIOCGNATL & 0xff; + int x; +#endif +#if LINUX_NETFILTER + size_t sock_sz = sizeof(conn->me); +#endif + + if ((req_sz = headersEnd(conn->in.buf, conn->in.offset)) == 0) { + debug(33, 5) ("Incomplete request, waiting for end of headers\n"); + *status = 0; + *prefix_p = NULL; + *method_p = METHOD_NONE; + return NULL; + } + assert(req_sz <= conn->in.offset); + /* Use memcpy, not strdup! */ + inbuf = xmalloc(req_sz + 1); +/* FIXME: wecopy here because we alter the data! */ + xmemcpy(inbuf, conn->in.buf, req_sz); + *(inbuf + req_sz) = '\0'; + + /* pre-set these values to make aborting simpler */ + *prefix_p = inbuf; + *method_p = METHOD_NONE; + *status = -1; + + /* Barf on NULL characters in the headers */ + if (strlen(inbuf) != req_sz) { + debug(33, 1) ("parseHttpRequest: Requestheader contains NULL characters\n"); + return parseHttpRequestAbort(conn, "error:invalid-request"); + } + /* Look for request method */ + if ((mstr = strtok(inbuf, "\t ")) == NULL) { + debug(33, 1) ("parseHttpRequest: Can't get request method\n"); + return parseHttpRequestAbort(conn, "error:invalid-request-method"); + } + method = urlParseMethod(mstr); + if (method == METHOD_NONE) { + debug(33, 1) ("parseHttpRequest: Unsupported method '%s'\n", mstr); + return parseHttpRequestAbort(conn, "error:unsupported-request-method"); + } + debug(33, 5) ("parseHttpRequest: Method is '%s'\n", mstr); + *method_p = method; + + /* look for URL+HTTP/x.x */ + if ((url = strtok(NULL, "\n")) == NULL) { + debug(33, 1) ("parseHttpRequest: Missing URL\n"); + return parseHttpRequestAbort(conn, "error:missing-url"); + } + while (xisspace(*url)) + url++; + t = url + strlen(url); + assert(*t == '\0'); + token = NULL; + while (t > url) { + t--; + if (xisspace(*t) && !strncmp(t + 1, "HTTP/", 5)) { + token = t + 1; + break; + } + } + while (t > url && xisspace(*t)) + *(t--) = '\0'; + debug(33, 5) ("parseHttpRequest: URI is '%s'\n", url); + if (token == NULL) { + debug(33, 3) ("parseHttpRequest: Missing HTTP identifier\n"); +#if RELAXED_HTTP_PARSER + httpBuildVersion(&http_ver, 0, 9); /* wild guess */ +#else + return parseHttpRequestAbort(conn, "error:missing-http-ident"); +#endif + } else { + if (sscanf(token + 5, "%d.%d", &http_ver.major, &http_ver.minor) != 2) { + debug(33, 3) ("parseHttpRequest: Invalid HTTP identifier.\n"); + return parseHttpRequestAbort(conn, "error: invalid HTTP-ident"); + } + debug(33, 6) ("parseHttpRequest: Client HTTP version %d.%d.\n", http_ver.major, http_ver.minor); + } + + /* + * Process headers after request line + */ + req_hdr = strtok(NULL, null_string); + header_sz = req_sz - (req_hdr - inbuf); + + httpHeaderInit(&hdr, hoRequest); + /* httpRequestParseHeader uses MsgIsolateHeaders ..*/ + if (!httpHeaderParse(&hdr, req_hdr, req_hdr + header_sz)) { + debug(33, 3) ("parseHttpRequest: couldn't parse header\n"); + return NULL; + } + +/* Dump the headers */ +{ + HttpHeaderPos pos = HttpHeaderInitPos; + const HttpHeaderEntry *e; + debug(55, 7) ("packing hdr: (%p)\n", &hdr); + /* pack all entries one by one */ + while ((e = httpHeaderGetEntry(&hdr, &pos))) +printf("%s: %s\r\n",strBuf(e->name),strBuf(e->value)); +} + + + if (0 == header_sz) { + debug(33, 3) ("parseHttpRequest: header_sz == 0\n"); + *status = 0; + return NULL; + } + assert(header_sz > 0); + + debug(33, 3) ("parseHttpRequest: req_hdr = {%s}\n", req_hdr); + end = req_hdr + header_sz; + debug(33, 3) ("parseHttpRequest: end = {%s}\n", end); + + prefix_sz = end - inbuf; + *req_line_sz_p = req_hdr - inbuf; + debug(33, 3) ("parseHttpRequest: prefix_sz = %d, req_line_sz = %d\n", + (int) prefix_sz, (int) *req_line_sz_p); + assert(prefix_sz <= conn->in.offset); + + /* Ok, all headers are received */ + http = cbdataAlloc(clientHttpRequest); + http->http_ver = http_ver; + http->conn = conn; + http->start = current_time; + http->req_sz = prefix_sz; + http->range_iter.boundary = StringNull; + *prefix_p = xmalloc(prefix_sz + 1); + xmemcpy(*prefix_p, conn->in.buf, prefix_sz); + *(*prefix_p + prefix_sz) = '\0'; + dlinkAdd(http, &http->active, &ClientActiveRequests); + +// debug(33, 5) ("parseHttpRequest: Request Header is\n%s\n", (*prefix_p) + *req_line_sz_p); + if ((t = strchr(url, '#'))) /* remove HTML anchors */ + *t = '\0'; + + /* handle internal objects */ + if (internalCheck(url)) { + /* prepend our name & port */ + http->uri = xstrdup(internalLocalUri(NULL, url)); + http->flags.internal = 1; + http->flags.accel = 1; + } + /* see if we running in Config2.Accel.on, if so got to convert it to URL */ + else if (Config2.Accel.on && *url == '/') { + /* prepend the accel prefix */ +// if (opt_accel_uses_host && (t = mime_get_header(req_hdr, "Host"))) { + if (opt_accel_uses_host && (httpHeaderHas(&hdr, HDR_HOST))) { + int vport; + char *q; + char *protocol_name = "http"; + String host = httpHeaderGetStrOrList(&hdr, HDR_HOST); + if (vport_mode) + vport = (int) ntohs(http->conn->me.sin_port); + else + vport = (int) Config.Accel.port; + /* If a Host: header was specified, use it to build the URL + * instead of the one in the Config file. */ + /* + * XXX Use of the Host: header here opens a potential + * security hole. There are no checks that the Host: value + * corresponds to one of your servers. It might, for example, + * refer to www.playboy.com. The 'dst' and/or 'dst_domain' ACL + * types should be used to prevent httpd-accelerators + * handling requests for non-local servers */ + +/* FIXME: use string tools on the header host */ + t = xstrdup (strBuf(host)); + strtok(t, " /;@"); + if ((q = strchr(t, ':'))) { + *q++ = '\0'; + if (vport_mode) + vport = atoi(q); + } + url_sz = strlen(url) + 32 + Config.appendDomainLen + + strlen(t); + http->uri = xcalloc(url_sz, 1); + + snprintf(http->uri, url_sz, "%s://%s:%d%s", + protocol_name, t, vport, url); + safe_free(t); + } else if (vhost_mode) { + int vport; + /* Put the local socket IP address as the hostname */ + url_sz = strlen(url) + 32 + Config.appendDomainLen; + http->uri = xcalloc(url_sz, 1); + if (vport_mode) + vport = (int) ntohs(http->conn->me.sin_port); + else + vport = (int) Config.Accel.port; +#if IPF_TRANSPARENT + natLookup.nl_inport = http->conn->me.sin_port; + natLookup.nl_outport = http->conn->peer.sin_port; + natLookup.nl_inip = http->conn->me.sin_addr; + natLookup.nl_outip = http->conn->peer.sin_addr; + natLookup.nl_flags = IPN_TCP; + if (natfd < 0) + natfd = open(IPL_NAT, O_RDONLY, 0); + if (natfd < 0) { + debug(50, 1) ("parseHttpRequest: NAT open failed: %s\n", + xstrerror()); + return parseHttpRequestAbort(conn, "error:nat-open-failed"); + } + /* + * IP-Filter changed the type for SIOCGNATL between + * 3.3 and 3.4. It also changed the cmd value for + * SIOCGNATL, so at least we can detect it. We could + * put something in configure and use ifdefs here, but + * this seems simpler. + */ + if (63 == siocgnatl_cmd) { + struct natlookup *nlp = &natLookup; + x = ioctl(natfd, SIOCGNATL, &nlp); + } else { + x = ioctl(natfd, SIOCGNATL, &natLookup); + } + if (x < 0) { + if (errno != ESRCH) { + debug(50, 1) ("parseHttpRequest: NAT lookup failed: ioctl(SIOCGNATL)\n"); + close(natfd); + natfd = -1; + return parseHttpRequestAbort(conn, "error:nat-lookup-failed"); + } else + snprintf(http->uri, url_sz, "http://%s:%d%s", + inet_ntoa(http->conn->me.sin_addr), + vport, url); + } else { + if (vport_mode) + vport = natLookup.nl_realport; + snprintf(http->uri, url_sz, "http://%s:%d%s", + inet_ntoa(natLookup.nl_realip), + vport, url); + } +#else +#if LINUX_NETFILTER + /* If the call fails the address structure will be unchanged */ + getsockopt(conn->fd, SOL_IP, SO_ORIGINAL_DST, &conn->me, &sock_sz); + debug(33, 5) ("parseHttpRequest: addr = %s", inet_ntoa(conn->me.sin_addr)); + if (vport_mode) + vport = (int) ntohs(http->conn->me.sin_port); +#endif + snprintf(http->uri, url_sz, "http://%s:%d%s", + inet_ntoa(http->conn->me.sin_addr), + vport, url); +#endif + debug(33, 5) ("VHOST REWRITE: '%s'\n", http->uri); + } else { + url_sz = strlen(Config2.Accel.prefix) + strlen(url) + + Config.appendDomainLen + 1; + http->uri = xcalloc(url_sz, 1); + snprintf(http->uri, url_sz, "%s%s", Config2.Accel.prefix, url); + } + http->flags.accel = 1; + } else { + /* URL may be rewritten later, so make extra room */ + url_sz = strlen(url) + Config.appendDomainLen + 5; + http->uri = xcalloc(url_sz, 1); + strcpy(http->uri, url); + http->flags.accel = 0; + } + if (!stringHasCntl(http->uri)) + http->log_uri = xstrndup(http->uri, MAX_URL); + else + http->log_uri = xstrndup(rfc1738_escape_unescaped(http->uri), MAX_URL); + debug(33, 5) ("parseHttpRequest: Complete request received\n"); + xfree(inbuf); + /* Now we know the correct URI */ + if ((request = urlParse(method, http->uri)) == NULL) { + fatal("failed to parse uri after succesful early parse\n"); + } + request->flags.accelerated = http->flags.accel; + request->header=hdr; + if (!http->flags.internal) { + if (internalCheck(strBuf(request->urlpath))) { + if (internalHostnameIs(request->host) && + request->port == ntohs(Config.Sockaddr.http->s.sin_port)) { + http->flags.internal = 1; + } else if (internalStaticCheck(strBuf(request->urlpath))) { + xstrncpy(request->host, internalHostname(), SQUIDHOSTNAMELEN); + request->port = ntohs(Config.Sockaddr.http->s.sin_port); + http->flags.internal = 1; + } + } + } + + /* cache some common data */ + request->content_length = httpHeaderGetInt(&request->header, + HDR_CONTENT_LENGTH); + request->flags.internal = http->flags.internal; + + /* FIXME: how many times do we do this? */ + safe_free(http->log_uri); + http->log_uri = xstrdup(urlCanonicalClean(request)); + /* fill in some more request state data. */ + request->client_addr = conn->peer.sin_addr; + request->my_addr = conn->me.sin_addr; + request->my_port = ntohs(conn->me.sin_port); + request->http_ver = http->http_ver; + /* FIXME: broker function */ + + if (!urlCheckRequest(request)) { + *status = -1; + return parseHttpRequestAbort(conn, "error: non support method"); + } + + /* this is protocol specific. It's good. */ + if (!clientCheckContentLength(request)) { + *status = -2; + return parseHttpRequestAbort(conn, "error: non support method"); + } + + http->request = requestLink(request); + /* FIXME: we're doing this way early... or are we? */ + clientSetKeepaliveFlag(http); + + /* Do we expect a request-body? */ + /* we might receive reqest-bodies encoded with transfer-encoding. + * RFC 2616 is vague on this. For now, we detect it and log it to cache.log */ + if (httpHeaderHas(&http->request->header, HDR_TRANSFER_ENCODING)) + debug(33,0)("client_side has recieved a transfer encoded request entity -we cannot handle this yet. REPORT THIS to Squid-dev@squid-cache.org\n"); + + /* FIXME: we've got potential state variable headaches */ +// filterChainAddTail(&http->reqfilters, identity_body, clientAccessCheck, NULL,http); + filterChainAddTail(&http->reqfilters, identity_body,clientInterpretRequestHeaders, NULL,http); +#if HEADERS_LOG + filterChainAddTail(&http->reqfilters, identity_body, clientHeadersLog, NULL,http); +#endif + filterChainAddTail(&http->reqfilters, identity_body, clientfdnote, NULL,http); + filterChainAddTail(&http->reqfilters, identity_body, httplocalmethods, NULL, http); + filterChainAddTail(&http->reqfilters, identity_body, broker_ClientEntry_hdr, NULL,http); + + /* this is the entry point for the reply to write data to the socket */ + filterChainAddTail(&http->repfilters, http_client_body, http_client_hdr, NULL,http); +// filterChainAddTail(&http->repfilters, newClientW, newClientWH, NULL,http); + + + + + + *status = 1; /* success */ + + return http; +} + +#if HEADERS_LOG +DATAFILTER_FILTERHEADER(clientHeadersLog) { + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; + headersLog(0, 1, request->request->method, request->request); + rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); + dlinkDelete(&filters->node, filter_list); + xfree(filters); + return rvflags; +} +#endif + +DATAFILTER_FILTERHEADER(clientfdnote) { + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; + clientHttpRequest *http=data; + fd_note(http->conn->fd, http->uri); + rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); + dlinkDelete(&filters->node, filter_list); + xfree(filters); + return rvflags; +} + +/* data for this is the http struct. It doesn't need to be though */ + +DATAFILTER_FILTERHEADER(http_trace_hdr) +{ + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; + http_version_t version; +// clientHttpRequest *http=data; + /* we are a source. confirm it the hard way */ + assert(filter_list->head->data==filters); + rep = httpReplyCreate(); + httpBuildVersion(&version, 1, 0); + /* the below _may_ be wrong: prefix length but we use the parsed headers */ + httpReplySetHeaders(rep, version, HTTP_OK, NULL, "text/plain", + httpRequestPrefixLen(request->request), 0, squid_curtime); + rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); + debug (3,3)("http_trace_hdr got rvflags of %0x\n",rvflags); + return rvflags; +} + +DATAFILTER_FILTER(http_trace_body) +{ + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; + clientHttpRequest *http=data; + request_t *request=http->request; + MemBuf mb; + /* we are a source. confirm it the hard way */ + assert(filter_list->head->data==filters); + /* now send the body component */ + mb = HttpRequestPackMemBuf(request); + rvflags = temp_filter->filter(mb.buf, mb.size, 0 ,filter_list,temp_filter, flags | FILTER_EOF, temp_filter->data); + return rvflags | FILTER_EOF; +} + + +/* handle local only methods - that is methods that don't apply to generic protocols + * or to the broker/store interface + * - there are none today however. + * this is a request-only filter. + */ +DATAFILTER_FILTERHEADER(httplocalmethods) { + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; + clientHttpRequest *http=data; + request_t *r = http->request; + /* Trace with a hop count of 0 is a local method */ + if (r->method == METHOD_TRACE && r->max_forwards < 1) { + + /* set the source and start the flow */ + filterChainAdd(&http->repfilters, http_trace_body,http_trace_hdr,NULL,http); + temp_filter=http->repfilters.head->data; + rvflags=temp_filter->filter_hdr(NULL, http, &http->repfilters, temp_filter,0, temp_filter->data); + + debug (33,3)("Process request got flags %0x from rep_header\n", rvflags); + if (!(rvflags & (FILTER_ABORT | FILTER_EOF))) { + rvflags=temp_filter->filter(NULL,0,0,&http->repfilters, temp_filter, + 0,temp_filter->data); + debug (33,3)("Process request got flags %0x from rep_filter\n", rvflags); + } + + return rvflags; + } + + rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); + dlinkDelete(&filters->node, filter_list); + xfree(filters); + return rvflags; +} + +DATAFILTER_FILTERHEADER(clientTemp) { + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; +// clientHttpRequest *http=data; + rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); + return rvflags; +} + + +/* this is the default entry point to client side functions */ +DATAFILTER_FILTER(http_client_body) +{ + FILTER_list *temp_filter=filters->node.next->data; + unsigned int rvflags; +// clientHttpRequest *http=data; + fatal("Unreachable code !!!\n"); + rvflags = temp_filter->filter(buf, len, offset,filter_list,temp_filter, flags, temp_filter->data); + return rvflags; +} + +/* this is the first time we see whats coming down at us from the source */ +/* Note: if this is cancelled we clean up gracefully, that means we still call most + * things. + * However, if FILTER_ABORT is set, we are not guaranteed valid headers... + * so be prepared! + */ +DATAFILTER_FILTERHEADER(http_client_hdr) +{ + FILTER_list *temp_filter; + unsigned int rvflags; + clientHttpRequest *http=data; + HttpHeader *hdr = &rep->header; + int is_hit = isTcpHit(http->log_type); + if (!rep) + /* the source hasn't created a reply for us. */ + /* this should _never_ happen */ + fatal("NO REPLY \n"); + /* Errors: we need a clean way to replace the source and it's related filters. + * a) two filter chains. + * b) some funky cold medina. + */ + if (clientReplyBodyTooLarge(rep->content_length)) + fatal("Body too large\n"); + + #if DONT_FILTER_THESE + /* but you might want to if you run Squid as an HTTP accelerator */ + /* httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); */ + httpHeaderDelById(hdr, HDR_ETAG); + #endif + /* this code is mich simpler than the old clientBuildReplyHeader because we assume + * A) only relevant headers are given to us. The server side functions are responsible + * for that. + * B) only fresh headers are provided. + */ + if (is_hit) + httpHeaderDelById(hdr, HDR_SET_COOKIE); + + /* Add user requested filters. */ + filterBuildChain(Config.reply_filters, &http->repfilters, http, rep, http->request); + + /* Handle Ranges (Note that these are _after_ the user specified filters. This means + * that if those filters inconsistently alter data length, we're screwed + */ +// filterChainAddTail(&http->repfilters, newClientRangeHdr, newClientRangeBody, NULL, http); + + /* handle TE */ + clientTEReplyheader(http,rep, &http->repfilters); + + /* + * Add a estimated Age header on cache hits. + */ + if (is_hit) { + /* + * Remove any existing Age header sent by upstream caches + * (note that the existing header is passed along unmodified + * on cache misses) + */ + httpHeaderDelById(hdr, HDR_AGE); + /* + * This adds the calculated object age. Note that the details of the + * age calculation is performed by adjusting the timestamp in + * storeTimestampsSet(), not here. + * + * BROWSER WORKAROUND: IE sometimes hangs when receiving a 0 Age + * header, so don't use it unless there is a age to report. Please + * note that Age is only used to make a conservative estimation of + * the objects age, so a Age: 0 header does not add any useful + * information to the reply in any case. + */ + if (NULL == http->entry) + (void) 0; + else if (http->entry->timestamp < 0) + (void) 0; + else if (http->entry->timestamp < squid_curtime) + httpHeaderPutInt(hdr, HDR_AGE, + squid_curtime - http->entry->timestamp); + } + + /* Handle authentication headers */ + if (request->request->auth_user_request) + authenticateFixHeader(rep, request->request->auth_user_request, request->request, http->flags.accel); + +#if 0 +/* this stuff does not belong here! - its a) not http specific and b) not client side + * driven + */ + /* Append X-Cache */ + httpHeaderPutStrf(hdr, HDR_X_CACHE, "%s from %s", + is_hit ? "HIT" : "MISS", getMyHostname()); +#if USE_CACHE_DIGESTS + /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */ + httpHeaderPutStrf(hdr, HDR_X_CACHE_LOOKUP, "%s from %s:%d", + http->lookup_type ? http->lookup_type : "NONE", + getMyHostname(), ntohs(Config.Sockaddr.http->s.sin_port)); +#endif +#endif + if ((httpReplyBodySize(request->request->method, rep)) < 0 && !(request->request->flags.te_encoding)) { + debug(33, 3) ("clientBuildReplyHeader: can't keep-alive, unknown body size\n"); + request->request->flags.proxy_keepalive = 0; + } else if ((httpReplyBodySize(request->request->method, rep)) < 0 && (request->request->flags.te_encoding)) + debug(33, 3) ("clientBuildReplyHeader: can keep-alive, unknown body size with te\n"); + /* Signal keep-alive if needed */ + httpHeaderPutStr(hdr, http->flags.accel ? HDR_CONNECTION : HDR_PROXY_CONNECTION, + request->request->flags.proxy_keepalive ? "keep-alive" : "close"); + +#if ADD_X_REQUEST_URI + /* + * Knowing the URI of the request is useful when debugging persistent + * connections in a client; we cannot guarantee the order of http headers, + * but X-Request-URI is likely to be the very last header to ease use from a + * debugger [hdr->entries.count-1]. + */ + httpHeaderPutStr(hdr, HDR_X_REQUEST_URI, + http->entry->mem_obj->url ? http->entry->mem_obj->url : http->uri); +#endif + httpHdrMangleList(hdr, request->request); + + + /* And add the IO filters */ + filterChainAddTail(&http->repfilters, newClientW, newClientWH, NULL,http); + /* process what we've just setup */ + temp_filter=filters->node.next->data; + /* but we're not needed anymore */ + dlinkDelete(&filters->node, filter_list); + xfree(filters); + rvflags = temp_filter->filter_hdr(rep,request,filter_list,temp_filter, flags, temp_filter->data); + return rvflags; +} + +static void +newclientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) +{ + clientHttpRequest *http = data; + StoreEntry *entry = http->entry; + + http->out.size += size; + http->flags.reply_write_in_progress=0; + /* objectlen as the test is broken becuase it records headers in the length + * calculation + */ + debug(33, 5) ("newclientWriteComplete: FD %d, sz %d, err %d, off %d, len %d\n", + fd, size, errflag, (int) http->out.offset, entry ? objectLen(entry) : 0); + if (size > 0) { + kb_incr(&statCounter.client_http.kbytes_out, size); + if (isTcpHit(http->log_type)) + kb_incr(&statCounter.client_http.hit_kbytes_out, size); + } + if (errflag) { + /* This is a socket error on the client socket. + * just close the socket, httpRequestFree will abort if needed + */ + debug(33,1)("newclientWriteComplete: error writing\n"); + comm_close(fd); + return; + } + if (http->mb.size) { + /* there is queued data in the membuffer. + * write it. + */ + debug(33,8)("newclientWriteComplete: Pushing queued data\n"); + http->flags.reply_write_in_progress=1; + comm_write_mbuf(http->conn->fd, http->mb, newclientWriteComplete, http); + /* zero the membuf - the data is untouched */ + memBufDefInit(&http->mb); + return; + } else if (http->flags.done_copying) { + debug(33, 5) ("newclientWriteComplete: FD %d transfer is DONE\n", fd); + /* why are we finished? */ + if (http->request->flags.proxy_keepalive) { + debug(33, 5) ("newclientWriteComplete: FD %d Keeping Alive\n", fd); + clientKeepaliveNextRequest(http); + } else { + comm_close(fd); + } + return; + } else if (clientReplyBodyTooLarge((int) http->out.offset)) { + debug(33,1)("newclientWriteComplete: size overrun\n"); + comm_close(fd); + return; + } + + /* call the "I want more data function". Probably the head of the reply function + * data filter, with no buffer. + */ + http->serverread(http->serverreaddata); +} + +DATAFILTER_FILTER(newClientW) { + clientHttpRequest *http = data; + MemBuf mb; + memBufDefInit(&mb); + + /* allocate a membuf, put the data from range and te into it, and call comm_write + * membuf + */ + + /* the more efficient longterm design (say there are no ranges, and no te.. + * storeClientCopy doesn't copy the data (maybe there's another PIA call? + * we just write the buffer and when the write completes it auto frees. + * I wonder if membuffers could do that. Then + * It'd be trivial - if a later filter wants the original buffer, it just locks + * it. Hmmm. + */ + + /* we are the last of the mohicans */ + assert(filters->node.next==NULL); + assert((buf && len) || (flags & (FILTER_EOF | FILTER_ABORT))); + + debug (33,8)("clientDoCommWriteMemBuf: buf %p len %d flags %d\n",buf,len,flags); + if (!buf || !len) { + /* No new data to write */ + /* if not EOF, then some error occured upstream... abort here TODO */ + newclientWriteComplete(http->conn->fd, NULL, 0, 0, http); + /* TODO: when we start buffer blocks here, this is where we flush the buffer */ + /* Why? calling clientWriteComplete flushes the buffer too */ + return 0; + + } + /* there is data to write. Concept: buffer 4K/client sock buf/ whatever here + * and only call TCP/IP layer when EOF or a larger amount is present ??? */ + + if (flags & FILTER_EOF) { + debug(33,1) ("newClientW called with EOF\n"); + http->flags.done_copying=1; + } + + if (!http->flags.reply_write_in_progress) { + http->flags.reply_write_in_progress=1; + memBufAppend(&mb, buf, len); + comm_write_mbuf(http->conn->fd, mb, newclientWriteComplete, http); + } else { + /* queue the data to the http membuffer */ + if (!http->mb.buf) + memBufDefInit(&http->mb); + memBufAppend(&http->mb, buf, len); + } +// comm_write(http->conn->fd, buf, len, clientWriteBodyComplete, http, NULL); + return 0; +} + +DATAFILTER_FILTERHEADER(newClientWH) { + clientHttpRequest *http = data; + MemBuf mb; +// size_t len; + +// memBufDefInit(&mb); + + /* allocate a membuf, put the data from range and te into it, and call comm_write + * membuf + */ + + /* the more efficient longterm design (say there are no ranges, and no te.. + * storeClientCopy doesn't copy the data (maybe there's another PIA call? + * we just write the buffer and when the write completes it auto frees. + * I wonder if membuffers could do that. Then + * It'd be trivial - if a later filter wants the original buffer, it just locks + * it. Hmmm. + */ + + /* we are the last of the mohicans */ + assert(filters->node.next==NULL); + + debug (33,1)("new client ** ** WriteHeaders: reply %p flags %d\n",rep,flags); + /* there is data to write. Concept: buffer 4K/client sock buf/ whatever here + * and only call TCP/IP layer when EOF or a larger amount is present ??? */ + + mb = httpReplyPack(rep); + + if (flags & FILTER_EOF) { + debug(33,1) ("newClientWH called with EOF\n"); + http->flags.done_copying=1; + } + + if (!http->flags.reply_write_in_progress) { + http->flags.reply_write_in_progress=1; + /* FIXME: just call commWrite */ + comm_write_mbuf(http->conn->fd, mb, newclientWriteComplete, http); + } else { /* FIXME: just call commWrite */ + /* queue the data to the http membuffer */ + if (!http->mb.buf) + memBufDefInit(&http->mb); + memBufAppend(&http->mb, mb.buf, mb.size); + memBufClean(&mb); + } +// comm_write(http->conn->fd, buf, len, clientWriteBodyComplete, http, NULL); + return 0; +} + +static int +clientReadDefer(int fdnotused, void *data) +{ + ConnStateData *conn = data; + if (conn->body.size_left) + return conn->in.offset >= conn->in.size; + else + return conn->defer.until > squid_curtime; +} + +static void +clientReadRequest(int fd, void *data) +{ + ConnStateData *conn = data; + int parser_return_code = 0; + int k; +// request_t *request = NULL; + int size; + void *p; + method_t method; + clientHttpRequest *http = NULL; + clientHttpRequest **H = NULL; + ErrorState *err = NULL; + fde *F = &fd_table[fd]; + int len = conn->in.size - conn->in.offset - 1; + debug(33, 4) ("clientReadRequest: FD %d: reading request...\n", fd); + /* FIXME: this bit is for adrian. I don't know what the modio/event network stuff + * can do or is capable of + */ + statCounter.syscalls.sock.reads++; + size = FD_READ_METHOD(fd, conn->in.buf + conn->in.offset, len); + if (size > 0) { + fd_bytes(fd, size, FD_READ); + kb_incr(&statCounter.client_http.kbytes_in, size); + } + /* + * Don't reset the timeout value here. The timeout value will be + * set to Config.Timeout.request by httpAccept() and + * clientWriteComplete(), and should apply to the request as a + * whole, not individual read() calls. Plus, it breaks our + * lame half-close detection + */ + if (size > 0) { + conn->in.offset += size; + conn->in.buf[conn->in.offset] = '\0'; /* Terminate the string */ + } else if (size == 0 && len > 0) { + if (conn->chr == NULL && conn->in.offset == 0) { + /* no current or pending requests */ + debug(33, 4) ("clientReadRequest: FD %d closed\n", fd); + comm_close(fd); + return; + } else if (!Config.onoff.half_closed_clients) { + /* admin doesn't want to support half-closed client sockets */ + debug(33, 3) ("clientReadRequest: FD %d aborted (half_closed_clients disabled)\n", fd); + comm_close(fd); + return; + } + /* It might be half-closed, we can't tell */ + debug(33, 5) ("clientReadRequest: FD %d closed?\n", fd); + F->flags.socket_eof = 1; + conn->defer.until = squid_curtime + 1; + conn->defer.n++; + fd_note(fd, "half-closed"); + /* There is one more close check at the end, to detect aborted + * (partial) requests. At this point we can't tell if the request + * is partial. + */ + /* Continue to process previously read data */ + } else if (size < 0) { + if (!ignoreErrno(errno)) { + debug(50, 2) ("clientReadRequest: FD %d: %s\n", fd, xstrerror()); + comm_close(fd); + return; + } else if (conn->in.offset == 0) { + debug(50, 2) ("clientReadRequest: FD %d: no data to process (%s)\n", fd, xstrerror()); + return; + } + /* Continue to process previously read data */ + } + + /* DON'T DO THIS FIXME */ + + commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, conn, 0); + + + /* FIXME this is the first thing to change. We need to change the select to call + * our initiating filter when a request is being read + */ + + /* Process request body if any */ + if (conn->in.offset > 0 && conn->body.callback != NULL) + // { temp_filter = http->reqfilters.head; rvflags = temp_filter->filter_hdr(NULL, http->request, &http->reqfilters, temp_filter, 0, temp_filter->data); } + clientProcessBody(conn); + + + + + /* Process next request */ + while (conn->in.offset > 0 && conn->body.size_left == 0) { + int nrequests; + size_t req_line_sz; + /* Skip leading (and trailing) whitespace */ + /* FIXME: this buffer fiddle is bad bad bad */ + while (conn->in.offset > 0 && xisspace(conn->in.buf[0])) { + xmemmove(conn->in.buf, conn->in.buf + 1, conn->in.offset - 1); + conn->in.offset--; + } + conn->in.buf[conn->in.offset] = '\0'; /* Terminate the string */ + if (conn->in.offset == 0) + break; + /* Limit the number of concurrent requests to 2 */ + for (H = &conn->chr, nrequests = 0; *H; H = &(*H)->next, nrequests++); + if (nrequests >= (Config.onoff.pipeline_prefetch ? 2 : 1)) { + debug(33, 3) ("clientReadRequest: FD %d max concurrent requests reached\n", fd); + debug(33, 5) ("clientReadRequest: FD %d defering new request until one is done\n", fd); + conn->defer.until = squid_curtime + 100; /* Reset when a request is complete */ + break; + } + conn->in.buf[conn->in.offset] = '\0'; /* Terminate the string */ + if (nrequests == 0) + fd_note(conn->fd, "Reading next request"); + + + + + /* Process request */ + http = parseHttpRequest(conn, + &method, + &parser_return_code, + &req_line_sz); + + +// we have a parsed request, with headers in + + if (http) { + FILTER_list *temp_filter; + unsigned int rvflags; + assert(http->req_sz > 0); + conn->in.offset -= http->req_sz; + assert(conn->in.offset >= 0); + debug(33, 5) ("conn->in.offset = %d\n", (int) conn->in.offset); + /* + * If we read past the end of this request, move the remaining + * data to the beginning + */ + if (conn->in.offset > 0) + xmemmove(conn->in.buf, conn->in.buf + http->req_sz, conn->in.offset); + /* add to the client request queue */ + for (H = &conn->chr; *H; H = &(*H)->next); + *H = http; + conn->nrequests++; + /* + * I wanted to lock 'http' here since its callback data for + * clientLifetimeTimeout(), but there's no logical place to + * cbdataUnlock if the timeout never happens. Maybe its safe + * enough to assume that if the FD is open, and the timeout + * triggers, that 'http' is valid. + */ + commSetTimeout(fd, Config.Timeout.lifetime, clientLifetimeTimeout, http); + if (parser_return_code < 0) { + //FIXME: case of return values */ + debug(33, 1) ("clientReadRequest: FD %d Invalid Request\n", fd); + err = errorCon(ERR_INVALID_REQ, HTTP_BAD_REQUEST); + err->request_hdrs = xstrdup(conn->in.buf); + http->entry = clientCreateStoreEntry(http, method, null_request_flags); + errorAppendEntry(http->entry, err); + break; + } + /* Do we expect a request-body? */ + if (http->request->content_length > 0) { + conn->body.size_left = http->request->content_length; + http->request->body_connection = conn; + /* Is it too large? */ + if (clientRequestBodyTooLarge(http->request->content_length)) { + err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); + err->request = requestLink(http->request); + http->entry = clientCreateStoreEntry(http, + METHOD_NONE, null_request_flags); + errorAppendEntry(http->entry, err); + break; + } + } + temp_filter = http->reqfilters.head->data; + rvflags = temp_filter->filter_hdr(NULL, http, &http->reqfilters, temp_filter, 0, temp_filter->data); + continue; /* while offset > 0 && body.size_left == 0 */ + } else if (parser_return_code == 0) { + /* + * Partial request received; reschedule until parseHttpRequest() + * is happy with the input + */ + k = conn->in.size - 1 - conn->in.offset; + if (k == 0) { + if (conn->in.offset >= Config.maxRequestHeaderSize) { + /* The request is too large to handle */ + debug(33, 1) ("Request header is too large (%d bytes)\n", + (int) conn->in.offset); + debug(33, 1) ("Config 'request_header_max_size'= %d bytes.\n", + Config.maxRequestHeaderSize); + err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); + http = parseHttpRequestAbort(conn, "error:request-too-large"); + /* add to the client request queue */ + for (H = &conn->chr; *H; H = &(*H)->next); + *H = http; + http->entry = clientCreateStoreEntry(http, METHOD_NONE, null_request_flags); + errorAppendEntry(http->entry, err); + return; + } + /* Grow the request memory area to accomodate for a large request */ + conn->in.size += CLIENT_REQ_BUF_SZ; + if (conn->in.size == 2 * CLIENT_REQ_BUF_SZ) { + p = conn->in.buf; /* get rid of fixed size Pooled buffer */ + conn->in.buf = xcalloc(2, CLIENT_REQ_BUF_SZ); + xmemcpy(conn->in.buf, p, CLIENT_REQ_BUF_SZ); + memFree(p, MEM_CLIENT_REQ_BUF); + } else + conn->in.buf = xrealloc(conn->in.buf, conn->in.size); + /* XXX account conn->in.buf */ + debug(33, 3) ("Handling a large request, offset=%d inbufsize=%d\n", + (int) conn->in.offset, conn->in.size); + k = conn->in.size - 1 - conn->in.offset; + } + break; + } + } /* while offset > 0 && conn->body.size_left == 0 */ + /* Check if a half-closed connection was aborted in the middle */ + if (F->flags.socket_eof) { + if (conn->in.offset != conn->body.size_left) { /* != 0 when no request body */ + /* Partial request received. Abort client connection! */ + debug(33, 3) ("clientReadRequest: FD %d aborted\n", fd); + // temp_filter = http->reqfilters.head; rvflags = temp_filter->filter_hdr(NULL, http->request, &http->reqfilters, temp_filter, FILTER_ABORT, temp_filter->data); + comm_close(fd); + return; + } + } +} + +/* file_read like function, for reading body content */ +void +clientReadBody(request_t * request, char *buf, size_t size, CBCB * callback, void *cbdata) +{ + ConnStateData *conn = request->body_connection; + if (!conn) { + debug(33, 5) ("clientReadBody: no body to read, request=%p\n", request); + callback(buf, 0, cbdata); /* Signal end of body */ + return; + } + debug(33, 2) ("clientReadBody: start fd=%d body_size=%d in.offset=%d cb=%p req=%p\n", conn->fd, conn->body.size_left, conn->in.offset, callback, request); + conn->body.callback = callback; + conn->body.cbdata = cbdata; + conn->body.buf = buf; + conn->body.bufsize = size; + conn->body.request = requestLink(request); + if (conn->in.offset) { + /* Data available */ + clientProcessBody(conn); + } else { + debug(33, 2) ("clientReadBody: fd %d wait for clientReadRequest\n", conn->fd); + } +} + +/* Called by clientReadRequest to process body content */ +static void +clientProcessBody(ConnStateData * conn) +{ + int size; + char *buf = conn->body.buf; + void *cbdata = conn->body.cbdata; + CBCB *callback = conn->body.callback; + request_t *request = conn->body.request; + /* Note: request is null while eating "aborted" transfers */ + debug(33, 2) ("clientProcessBody: start fd=%d body_size=%d in.offset=%d cb=%p req=%p\n", conn->fd, conn->body.size_left, conn->in.offset, callback, request); + /* Some sanity checks... */ + assert(conn->body.size_left > 0); + assert(conn->in.offset > 0); + assert(callback != NULL); + assert(buf != NULL); + /* How much do we have to process? */ + size = conn->in.offset; + if (size > conn->body.size_left) /* only process the body part */ + size = conn->body.size_left; + if (size > conn->body.bufsize) /* don't copy more than requested */ + size = conn->body.bufsize; + xmemcpy(buf, conn->in.buf, size); + conn->body.size_left -= size; + /* Move any remaining data */ + conn->in.offset -= size; + if (conn->in.offset > 0) + xmemmove(conn->in.buf, conn->in.buf + size, conn->in.offset); + /* Remove request link if this is the last part of the body, as + * clientReadRequest automatically continues to process next request */ + if (conn->body.size_left <= 0 && request != NULL) + request->body_connection = NULL; + /* Remove clientReadBody arguments (the call is completed) */ + conn->body.request = NULL; + conn->body.callback = NULL; + conn->body.buf = NULL; + conn->body.bufsize = 0; + /* Remember that we have touched the body, not restartable */ + if (request != NULL) + request->flags.body_sent = 1; + /* Invoke callback function */ + callback(buf, size, cbdata); + if (request != NULL) + requestUnlink(request); /* Linked in clientReadBody */ + debug(33, 2) ("clientProcessBody: end fd=%d size=%d body_size=%d in.offset=%d cb=%p req=%p\n", conn->fd, size, conn->body.size_left, conn->in.offset, callback, request); + return; +} + +/* A dummy handler that throws away a request-body */ +static char bodyAbortBuf[SQUID_TCP_SO_RCVBUF]; +static void +clientReadBodyAbortHandler(char *buf, size_t size, void *data) +{ + ConnStateData *conn = (ConnStateData *) data; + debug(33, 2) ("clientReadBodyAbortHandler: fd=%d body_size=%d in.offset=%d\n", conn->fd, conn->body.size_left, conn->in.offset); + if (size != 0 && conn->body.size_left != 0) { + debug(33, 3) ("clientReadBodyAbortHandler: fd=%d shedule next read\n", conn->fd); + conn->body.callback = clientReadBodyAbortHandler; + conn->body.buf = bodyAbortBuf; + conn->body.bufsize = sizeof(bodyAbortBuf); + conn->body.cbdata = data; + } +} + +/* Abort a body request */ +int +clientAbortBody(request_t * request) +{ + ConnStateData *conn = request->body_connection; + char *buf; + CBCB *callback; + void *cbdata; + request->body_connection = NULL; + if (!conn || conn->body.size_left <= 0) + return 0; /* No body to abort */ + if (conn->body.callback != NULL) { + buf = conn->body.buf; + callback = conn->body.callback; + cbdata = conn->body.cbdata; + assert(request == conn->body.request); + conn->body.buf = NULL; + conn->body.callback = NULL; + conn->body.cbdata = NULL; + conn->body.request = NULL; + callback(buf, -1, cbdata); /* Signal abort to clientReadBody caller */ + requestUnlink(request); + } + clientReadBodyAbortHandler(NULL, -1, conn); /* Install abort handler */ + /* clientProcessBody() */ + return 1; /* Aborted */ +} + +/* general lifetime handler for HTTP requests */ +/* TODO FIXME: RFC2616 covers this: we should send a close message + * to http/1.1 clients + */ +static void +requestTimeout(int fd, void *data) +{ +#if THIS_CONFUSES_PERSISTENT_CONNECTION_AWARE_BROWSERS_AND_USERS + ConnStateData *conn = data; + ErrorState *err; + debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd); + if (fd_table[fd].rwstate) { + /* + * Some data has been sent to the client, just close the FD + */ + comm_close(fd); + } else if (conn->nrequests) { + /* + * assume its a persistent connection; just close it + */ + comm_close(fd); + } else { + /* + * Generate an error + */ + err = errorCon(ERR_LIFETIME_EXP, HTTP_REQUEST_TIMEOUT); + err->url = xstrdup("N/A"); + /* + * Normally we shouldn't call errorSend() in client_side.c, but + * it should be okay in this case. Presumably if we get here + * this is the first request for the connection, and no data + * has been written yet + */ + assert(conn->chr == NULL); + errorSend(fd, err); + /* + * if we don't close() here, we still need a timeout handler! + */ + commSetTimeout(fd, 30, requestTimeout, conn); + /* + * Aha, but we don't want a read handler! + */ + commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); + } +#else + /* + * Just close the connection to not confuse browsers + * using persistent connections. Some browsers opens + * an connection and then does not use it until much + * later (presumeably because the request triggering + * the open has already been completed on another + * connection) + */ + debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd); + comm_close(fd); +#endif +} + +static void +clientLifetimeTimeout(int fd, void *data) +{ + clientHttpRequest *http = data; + ConnStateData *conn = http->conn; + debug(33, 1) ("WARNING: Closing client %s connection due to lifetime timeout\n", + inet_ntoa(conn->peer.sin_addr)); + debug(33, 1) ("\t%s\n", http->uri); + comm_close(fd); +} + +static int +httpAcceptDefer(int fdunused, void *dataunused) +{ + static time_t last_warn = 0; + if (fdNFree() >= RESERVED_FD) + return 0; + if (last_warn + 15 < squid_curtime) { + debug(33, 0) ("WARNING! Your cache is running out of filedescriptors\n"); + last_warn = squid_curtime; + } + return 1; +} + +/* Handle a new connection on HTTP socket. */ +void +httpAccept(int sock, void *data) +{ + int *N = data; + int fd = -1; + ConnStateData *connState = NULL; + struct sockaddr_in peer; + struct sockaddr_in me; + int max = INCOMING_HTTP_MAX; +#if USE_IDENT + static aclCheck_t identChecklist; +#endif + commSetSelect(sock, COMM_SELECT_READ, httpAccept, NULL, 0); + while (max-- && !httpAcceptDefer(sock, NULL)) { + memset(&peer, '\0', sizeof(struct sockaddr_in)); + memset(&me, '\0', sizeof(struct sockaddr_in)); + if ((fd = comm_accept(sock, &peer, &me)) < 0) { + if (!ignoreErrno(errno)) + debug(50, 1) ("httpAccept: FD %d: accept failure: %s\n", + sock, xstrerror()); + break; + } + debug(33, 4) ("httpAccept: FD %d: accepted\n", fd); + connState = cbdataAlloc(ConnStateData); + connState->peer = peer; + connState->log_addr = peer.sin_addr; + connState->log_addr.s_addr &= Config.Addrs.client_netmask.s_addr; + connState->me = me; + connState->fd = fd; + connState->in.size = CLIENT_REQ_BUF_SZ; + connState->in.buf = memAllocate(MEM_CLIENT_REQ_BUF); + /* XXX account connState->in.buf */ + comm_add_close_handler(fd, connStateFree, connState); + if (Config.onoff.log_fqdn) + fqdncache_gethostbyaddr(peer.sin_addr, FQDN_LOOKUP_IF_MISS); + commSetTimeout(fd, Config.Timeout.request, requestTimeout, connState); +#if USE_IDENT + identChecklist.src_addr = peer.sin_addr; + identChecklist.my_addr = me.sin_addr; + identChecklist.my_port = ntohs(me.sin_port); + if (aclCheckFast(Config.accessList.identLookup, &identChecklist)) + identStart(&me, &peer, clientIdentDone, connState); +#endif + commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, connState, 0); + commSetDefer(fd, clientReadDefer, connState); + clientdbEstablished(peer.sin_addr, 1); + assert(N); + (*N)++; + } +} + + +#define SENDING_BODY 0 +#define SENDING_HDRSONLY 1 +/* checks status of sending to original client */ +static int +clientCheckTransferDone(clientHttpRequest * http) +{ +/* this code defies easy abstraction. Come back to it */ +#if 0 + int sending = SENDING_BODY; + StoreEntry *entry = http->entry; + MemObject *mem; + http_reply *reply; + int sendlen; + if (entry == NULL) + return 0; + /* + * For now, 'done_copying' is used for special cases like + * Range and HEAD & TE requests. + */ + if (http->flags.done_copying) + return 1; + /* + * Handle STORE_OK objects. + * objectLen(entry) will be set proprely. + */ + if (entry->store_status == STORE_OK) { + if (http->out.offset >= objectLen(entry)) + return 1; + else + return 0; + } + /* + * Now, handle STORE_PENDING objects + */ + mem = entry->mem_obj; + assert(mem != NULL); + assert(http->request != NULL); + reply = mem->reply; + if (reply->hdr_sz == 0) + return 0; /* haven't found end of headers yet */ + else if (reply->sline.status == HTTP_OK) + sending = SENDING_BODY; + else if (reply->sline.status == HTTP_NO_CONTENT) + sending = SENDING_HDRSONLY; + else if (reply->sline.status == HTTP_NOT_MODIFIED) + sending = SENDING_HDRSONLY; + else if (reply->sline.status < HTTP_OK) + sending = SENDING_HDRSONLY; + else if (http->request->method == METHOD_HEAD) + sending = SENDING_HDRSONLY; + else + sending = SENDING_BODY; + /* + * Figure out how much data we are supposed to send. + * If we are sending a body and we don't have a content-length, + * then we must wait for the object to become STORE_OK. + */ + if (sending == SENDING_HDRSONLY) + sendlen = reply->hdr_sz; + else if (reply->content_length < 0) + return 0; + else + sendlen = reply->content_length + reply->hdr_sz; + /* + * Now that we have the expected length, did we send it all? + */ + if (http->out.offset < sendlen) + return 0; + else +#endif + return 1; +} + +/* + * This function is designed to serve a fairly specific purpose. + * Occasionally our vBNS-connected caches can talk to each other, but not + * the rest of the world. Here we try to detect frequent failures which + * make the cache unusable (e.g. DNS lookup and connect() failures). If + * the failure:success ratio goes above 1.0 then we go into "hit only" + * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors + * will only fetch HITs from us if they are using the ICP protocol. We + * stay in this mode for 5 minutes. + * + * Duane W., Sept 16, 1996 + */ + +static void +checkFailureRatio(err_type etype, hier_code hcode) +{ + static double magic_factor = 100.0; + double n_good; + double n_bad; + if (hcode == HIER_NONE) + return; + n_good = magic_factor / (1.0 + request_failure_ratio); + n_bad = magic_factor - n_good; + switch (etype) { + case ERR_DNS_FAIL: + case ERR_CONNECT_FAIL: + case ERR_READ_ERROR: + n_bad++; + break; + default: + n_good++; + } + request_failure_ratio = n_bad / n_good; + if (hit_only_mode_until > squid_curtime) + return; + if (request_failure_ratio < 1.0) + return; + debug(33, 0) ("Failure Ratio at %4.2f\n", request_failure_ratio); + debug(33, 0) ("Going into hit-only-mode for %d minutes...\n", + FAILURE_MODE_TIME / 60); + hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME; + request_failure_ratio = 0.8; /* reset to something less than 1.0 */ +} + +void +clientHttpConnectionsOpen(void) +{ + sockaddr_in_list *s; + int fd; + for (s = Config.Sockaddr.http; s; s = s->next) { + if (MAXHTTPPORTS == NHttpSockets) { + debug(1, 1) ("WARNING: You have too many 'http_port' lines.\n"); + debug(1, 1) (" The limit is %d\n", MAXHTTPPORTS); + continue; + } + enter_suid(); + fd = comm_open(SOCK_STREAM, + 0, + s->s.sin_addr, + ntohs(s->s.sin_port), + COMM_NONBLOCKING, + "HTTP Socket"); + leave_suid(); + if (fd < 0) + continue; + comm_listen(fd); + commSetSelect(fd, COMM_SELECT_READ, httpAccept, NULL, 0); + /* + * We need to set a defer handler here so that we don't + * peg the CPU with select() when we hit the FD limit. + */ + commSetDefer(fd, httpAcceptDefer, NULL); + debug(1, 1) ("Accepting HTTP connections at %s, port %d, FD %d.\n", + inet_ntoa(s->s.sin_addr), + (int) ntohs(s->s.sin_port), + fd); + HttpSockets[NHttpSockets++] = fd; + } + if (NHttpSockets < 1) + fatal("Cannot open HTTP Port"); +} + +void +clientHttpConnectionsClose(void) +{ + int i; + for (i = 0; i < NHttpSockets; i++) { + if (HttpSockets[i] >= 0) { + debug(1, 1) ("FD %d Closing HTTP connection\n", HttpSockets[i]); + comm_close(HttpSockets[i]); + HttpSockets[i] = -1; + } + } + NHttpSockets = 0; +} --- /dev/null Wed Feb 14 00:52:54 2007 +++ squid/src/modules/http_upstream/Makefile.in Wed Feb 14 00:53:08 2007 @@ -0,0 +1,70 @@ +# +# Makefile for the ntlm authentication scheme module for the Squid Object Cache server +# +# $Id: Makefile.in,v 1.1.2.1 2001/05/09 12:22:15 rbcollins Exp $ +# + +MODULE = http_upstream + +#SUBDIRS = helpers + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(MODULE).a + +OBJS = \ + http.o + + +all install: $(OUT) + @for dir in $(SUBDIRS); do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) $@" || exit 1; \ + fi; \ + done; + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(MODULE).a + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) clean"; \ + fi; \ + done + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) distclean"; \ + fi; \ + done + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c --- /dev/null Wed Feb 14 00:52:54 2007 +++ squid/src/modules/http_upstream/http.c Wed Feb 14 00:53:08 2007 @@ -0,0 +1,2027 @@ + +/* + * $Id: http.c,v 1.1.2.1 2001/05/09 12:22:15 rbcollins Exp $ + * + * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +/* + * Anonymizing patch by lutz@as-node.jena.thur.de + * have a look into http-anon.c to get more informations. + */ + +#include "squid.h" +#include "iobuf.h" + +static const char *const crlf = "\r\n"; + +static CWCB httpSendComplete; +static CWCB httpSendRequestEntry; + +static PF httpReadReply; +static void httpSendRequest(HttpStateData *); +static PF httpStateFree; +static PF httpTimeout; +static void httpCacheNegatively(StoreEntry *); +static void httpMakePrivate(StoreEntry *); +static void httpMakePublic(StoreEntry *); +static int httpCachableReply(HttpStateData *); +static void httpMaybeRemovePublic(StoreEntry *, http_status); + +/* this will go in a header when this file becomes a module */ + +struct _HttpStateData { + StoreEntry *entry; + request_t *request; + char *reply_hdr; + size_t reply_hdr_size; + int reply_hdr_state; + peer *peer; /* peer request made to */ + int eof; /* reached end-of-object? */ + request_t *orig_request; + int fd; + http_state_flags flags; + FwdState *fwd; + /* filter_hdr variables */ + HttpReply *rep; + clientHttpRequest *hcrequest; + /* filter data variables */ + /* buf is not needed - immediately used */ + /* offset is needed. */ + /* size isn't - its returned by the CWCB */ + /* filter variables */ + dlink_list * filter_list; + FILTER_list * filters; + unsigned int filterflags; + dlink_list oldfilters; + /* How far into the network object are we? */ + size_t read_offset; + /* this should be for the ReplyHeaders filter */ + char *headerstore; + size_t headerlength; + iobuf *readbuf; +}; + +CBDATA_TYPE(HttpStateData); + +/* temporary function */ +void +serverHttpInit(void) +{ +} + + +/* register as a module */ +void +mod_install_http_upstream (const char *namestr) +{ + /* FIXME: register with the broker as an upstream handler for http */ +#if 0 + /* Register as a potential client_side reply filter */ + filterRegisterModule (namestr, SpyFilter_AddInstance, + SpyFilter_RemInstance); +#endif +} + +/* deregister as a module */ +void +mod_uninstall_http_upstream (const char *namestr) { +#if 0 + filterDeregisterModule(namestr); +#endif +} + + +/* this interface needs thought. */ +unsigned int +http_broker_entry(HttpReply *rep, clientHttpRequest *request, + dlink_list * filter_list,FILTER_list * filters, unsigned int flags, + void *data) +{ + FILTER_list *temp_filter; + fatal("http broker entry point. \n"); +/* + temp_filter = filters->node.next->data; + return temp_filter->filter_hdr(rep, request, filter_list, + temp_filter, flags, temp_filter->data); +*/ +} + +static void +httpStateFree(int fd, void *data) +{ + HttpStateData *httpState = data; +#if DELAY_POOLS + delayClearNoDelay(fd); +#endif + if (httpState == NULL) + return; + filterCleanChain(&httpState->oldfilters); + storeUnlockObject(httpState->entry); + if (httpState->reply_hdr) { + memFree(httpState->reply_hdr, MEM_8K_BUF); + httpState->reply_hdr = NULL; + } + requestUnlink(httpState->request); + requestUnlink(httpState->orig_request); + httpState->request = NULL; + httpState->orig_request = NULL; + cbdataFree(httpState); +} + +int +httpCachable(method_t method) +{ + /* GET and HEAD are cachable. Others are not. */ + if (method != METHOD_GET && method != METHOD_HEAD) + return 0; + /* else cachable */ + return 1; +} + +static void +httpTimeout(int fd, void *data) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); + if (entry->store_status == STORE_PENDING) { + if (entry->mem_obj->inmem_hi == 0) { + fwdFail(httpState->fwd, + errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); + } + } + comm_close(fd); +} + +/* This object can be cached for a long time */ +static void +httpMakePublic(StoreEntry * entry) +{ + if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) + storeSetPublicKey(entry); +} + +/* This object should never be cached at all */ +static void +httpMakePrivate(StoreEntry * entry) +{ + storeExpireNow(entry); + storeReleaseRequest(entry); /* delete object when not used */ + /* storeReleaseRequest clears ENTRY_CACHABLE flag */ +} + +/* This object may be negatively cached */ +static void +httpCacheNegatively(StoreEntry * entry) +{ + storeNegativeCache(entry); + if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) + storeSetPublicKey(entry); +} + +static void +httpMaybeRemovePublic(StoreEntry * e, http_status status) +{ + int remove = 0; + int forbidden = 0; + StoreEntry *pe; + if (!EBIT_TEST(e->flags, KEY_PRIVATE)) + return; + switch (status) { + case HTTP_OK: + case HTTP_NON_AUTHORITATIVE_INFORMATION: + case HTTP_MULTIPLE_CHOICES: + case HTTP_MOVED_PERMANENTLY: + case HTTP_MOVED_TEMPORARILY: + case HTTP_GONE: + case HTTP_NOT_FOUND: + remove = 1; + break; + case HTTP_FORBIDDEN: + case HTTP_METHOD_NOT_ALLOWED: + forbidden = 1; + break; +#if WORK_IN_PROGRESS + case HTTP_UNAUTHORIZED: + forbidden = 1; + break; +#endif + default: +#if QUESTIONABLE + /* + * Any 2xx response should eject previously cached entities... + */ + if (status >= 200 && status < 300) + remove = 1; +#endif + break; + } + if (!remove && !forbidden) + return; + assert(e->mem_obj); + if (e->mem_obj->request) + pe = storeGetPublicByRequest(e->mem_obj->request); + else + pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method); + if (pe != NULL) { + assert(e != pe); + storeRelease(pe); + } + /* + * Also remove any cached HEAD response in case the object has + * changed. + */ + if (e->mem_obj->request) + pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_HEAD); + else + pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD); + if (pe != NULL) { + assert(e != pe); + storeRelease(pe); + } + if (forbidden) + return; + switch (e->mem_obj->method) { + case METHOD_PUT: + case METHOD_DELETE: + case METHOD_PROPPATCH: + case METHOD_MKCOL: + case METHOD_MOVE: + case METHOD_BMOVE: + case METHOD_BDELETE: + /* + * Remove any cached GET object if it is beleived that the + * object may have changed as a result of other methods + */ + if (e->mem_obj->request) + pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_GET); + else + pe = storeGetPublic(e->mem_obj->url, METHOD_GET); + if (pe != NULL) { + assert(e != pe); + storeRelease(pe); + } + break; + } +} + +static int +httpCachableReply(HttpStateData * httpState) +{ + HttpReply *rep = httpState->entry->mem_obj->reply; + HttpHeader *hdr = &rep->header; + const int cc_mask = (rep->cache_control) ? rep->cache_control->mask : 0; + const char *v; + if (EBIT_TEST(cc_mask, CC_PRIVATE)) + return 0; + if (EBIT_TEST(cc_mask, CC_NO_CACHE)) + return 0; + if (EBIT_TEST(cc_mask, CC_NO_STORE)) + return 0; + if (httpState->request->flags.auth) { + /* + * Responses to requests with authorization may be cached + * only if a Cache-Control: public reply header is present. + * RFC 2068, sec 14.9.4 + */ + if (!EBIT_TEST(cc_mask, CC_PUBLIC)) + return 0; + } + /* Pragma: no-cache in _replies_ is not documented in HTTP, + * but servers like "Active Imaging Webcast/2.0" sure do use it */ + if (httpHeaderHas(hdr, HDR_PRAGMA)) { + String s = httpHeaderGetList(hdr, HDR_PRAGMA); + const int no_cache = strListIsMember(&s, "no-cache", ','); + stringClean(&s); + if (no_cache) + return 0; + } + /* + * The "multipart/x-mixed-replace" content type is used for + * continuous push replies. These are generally dynamic and + * probably should not be cachable + */ + if ((v = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE))) + if (!strncasecmp(v, "multipart/x-mixed-replace", 25)) + return 0; + switch (httpState->entry->mem_obj->reply->sline.status) { + /* Responses that are cacheable */ + case HTTP_OK: + case HTTP_NON_AUTHORITATIVE_INFORMATION: + case HTTP_MULTIPLE_CHOICES: + case HTTP_MOVED_PERMANENTLY: + case HTTP_GONE: + /* + * Don't cache objects that need to be refreshed on next request, + * unless we know how to refresh it. + */ + if (!refreshIsCachable(httpState->entry)) + return 0; + /* don't cache objects from peers w/o LMT, Date, or Expires */ + /* check that is it enough to check headers @?@ */ + if (rep->date > -1) + return 1; + else if (rep->last_modified > -1) + return 1; + else if (!httpState->peer) + return 1; + /* @?@ (here and 302): invalid expires header compiles to squid_curtime */ + else if (rep->expires > -1) + return 1; + else + return 0; + /* NOTREACHED */ + break; + /* Responses that only are cacheable if the server says so */ + case HTTP_MOVED_TEMPORARILY: + if (rep->expires > -1) + return 1; + else + return 0; + /* NOTREACHED */ + break; + /* Errors can be negatively cached */ + case HTTP_NO_CONTENT: + case HTTP_USE_PROXY: + case HTTP_BAD_REQUEST: + case HTTP_FORBIDDEN: + case HTTP_NOT_FOUND: + case HTTP_METHOD_NOT_ALLOWED: + case HTTP_REQUEST_URI_TOO_LARGE: + case HTTP_INTERNAL_SERVER_ERROR: + case HTTP_NOT_IMPLEMENTED: + case HTTP_BAD_GATEWAY: + case HTTP_SERVICE_UNAVAILABLE: + case HTTP_GATEWAY_TIMEOUT: + return -1; + /* NOTREACHED */ + break; + /* Some responses can never be cached */ + case HTTP_PARTIAL_CONTENT: /* Not yet supported */ + case HTTP_SEE_OTHER: + case HTTP_NOT_MODIFIED: + case HTTP_UNAUTHORIZED: + case HTTP_PROXY_AUTHENTICATION_REQUIRED: + case HTTP_INVALID_HEADER: /* Squid header parsing error */ + default: /* Unknown status code */ + return 0; + /* NOTREACHED */ + break; + } + /* NOTREACHED */ +} + +/* + * For Vary, store the relevant request headers as + * virtual headers in the reply + * Returns false if the variance cannot be stored + */ +const char * +httpMakeVaryMark(request_t * request, HttpReply * reply) +{ + int ok = 1; + String vary, hdr; + const char *pos = NULL; + const char *item; + const char *value; + int ilen; + static String vstr = + {0, 0, NULL}; + + stringClean(&vstr); + vary = httpHeaderGetList(&reply->header, HDR_VARY); + while (strListGetItem(&vary, ',', &item, &ilen, &pos)) { + char *name = xmalloc(ilen + 1); + xstrncpy(name, item, ilen + 1); + Tolower(name); + strListAdd(&vstr, name, ','); + hdr = httpHeaderGetByName(&request->header, name); + safe_free(name); + value = strBuf(hdr); + if (value) { + value = rfc1738_escape(value); + stringAppend(&vstr, "=\"", 2); + stringAppend(&vstr, value, strlen(value)); + stringAppend(&vstr, "\"", 1); + } + stringClean(&hdr); + } + stringClean(&vary); +#if X_ACCELERATOR_VARY + vary = httpHeaderGetList(&reply->header, HDR_X_ACCELERATOR_VARY); + while (strListGetItem(&vary, ',', &item, &ilen, &pos)) { + char *name = xmalloc(ilen + 1); + xstrncpy(name, item, ilen + 1); + Tolower(name); + strListAdd(&vstr, name, ','); + hdr = httpHeaderGetByName(&request->header, name); + safe_free(name); + value = strBuf(hdr); + if (value) { + value = rfc1738_escape(value); + stringAppend(&vstr, "=\"", 2); + stringAppend(&vstr, value, strlen(value)); + stringAppend(&vstr, "\"", 1); + } + stringClean(&hdr); + } + stringClean(&vary); +#endif + debug(11, 0) ("httpMakeVaryMark: %d / %s\n", ok, strBuf(vstr)); + return strBuf(vstr); +} + +/* rewrite this later using new interfaces @?@ */ +void +httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) +{ + char *t = NULL; + StoreEntry *entry = httpState->entry; + int room; + size_t hdr_len; + HttpReply *reply = entry->mem_obj->reply; + Ctx ctx; + debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", + storeKeyText(entry->hash.key)); + if (httpState->reply_hdr == NULL) + httpState->reply_hdr = memAllocate(MEM_8K_BUF); + assert(httpState->reply_hdr_state == 0); + hdr_len = httpState->reply_hdr_size; + room = 8191 - hdr_len; + xmemcpy(httpState->reply_hdr + hdr_len, buf, room < size ? room : size); + hdr_len += room < size ? room : size; + httpState->reply_hdr[hdr_len] = '\0'; + httpState->reply_hdr_size = hdr_len; + if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { + debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr); + httpState->reply_hdr_state += 2; + reply->sline.status = HTTP_INVALID_HEADER; + return; + } + t = httpState->reply_hdr + hdr_len; + /* headers can be incomplete only if object still arriving */ + if (!httpState->eof) { + size_t k = headersEnd(httpState->reply_hdr, 8192); + if (0 == k) + return; /* headers not complete */ + t = httpState->reply_hdr + k; + httpState->headerlength=k; + } + *t = '\0'; + httpState->reply_hdr_state++; + /* in progess state */ + assert(httpState->reply_hdr_state == 1); + ctx = ctx_enter(entry->mem_obj->url); + /* state becomes 2? whats different? */ + httpState->reply_hdr_state++; + debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", + httpState->reply_hdr); + /* Parse headers into reply structure */ + /* what happens if we fail to parse here? */ + httpReplyParse(reply, httpState->reply_hdr, hdr_len); + storeTimestampsSet(entry); + /* Check if object is cacheable or not based on reply code */ + debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); + if (neighbors_do_private_keys) + httpMaybeRemovePublic(entry, reply->sline.status); + switch (httpCachableReply(httpState)) { + case 1: + if (httpHeaderHas(&reply->header, HDR_VARY) +#if X_ACCELERATOR_VARY + || httpHeaderHas(&reply->header, HDR_X_ACCELERATOR_VARY) +#endif + ) { + const char *vary = httpMakeVaryMark(httpState->request, reply); + if (vary) { + entry->mem_obj->vary_headers = xstrdup(vary); + /* Kill the old base object if a change in variance is detected */ + httpMakePublic(entry); + } else { + httpMakePrivate(entry); + } + } else { + httpMakePublic(entry); + } + break; + case 0: + httpMakePrivate(entry); + break; + case -1: + httpCacheNegatively(entry); + break; + default: + assert(0); + break; + } + if (reply->cache_control) { + if (EBIT_TEST(reply->cache_control->mask, CC_PROXY_REVALIDATE)) + EBIT_SET(entry->flags, ENTRY_REVALIDATE); + else if (EBIT_TEST(reply->cache_control->mask, CC_MUST_REVALIDATE)) + EBIT_SET(entry->flags, ENTRY_REVALIDATE); + } + if (httpState->flags.keepalive) + if (httpState->peer) + httpState->peer->stats.n_keepalives_sent++; + if (reply->keep_alive) + if (httpState->peer) + httpState->peer->stats.n_keepalives_recv++; + if (reply->date > -1 && !httpState->peer) { + int skew = abs(reply->date - squid_curtime); + if (skew > 86400) + debug(11, 3) ("%s's clock is skewed by %d seconds!\n", + httpState->request->host, skew); + } + ctx_exit(ctx); +#if HEADERS_LOG + headersLog(1, 0, httpState->request->method, reply); +#endif +} + + +DATAFILTER_FILTER(httpPconnTransferDone) +{ + HttpStateData *httpState=data; + MemObject *mem; + HttpReply *reply; + int clen; + + FILTER_list *temp_filter; + temp_filter=filters->node.next->data; + + debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); + /* + * If we didn't send a keep-alive request header, then this + * can not be a persistent connection. + */ + /* + * TODO: if we recieved an http/1.1 response then it will be persisent by default + */ + + + + if (!httpState->flags.keepalive) + return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); + + /* if we know that it's EOF, then keepalive is still ok (it TE signalled termination + * of body + */ + if (flags & FILTER_EOF) + return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); + + /* + * What does the reply have to say about keep-alive? + */ + /* + * XXX BUG? + * If the origin server (HTTP/1.0) does not send a keep-alive + * header, but keeps the connection open anyway, what happens? + * We'll return here and http.c waits for an EOF before changing + * store_status to STORE_OK. Combine this with ENTRY_FWD_HDR_WAIT + * and an error status code, and we might have to wait until + * the server times out the socket. + */ + + mem = httpState->entry->mem_obj; + reply = mem->reply; + if (!reply->keep_alive) + return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); + debug(11, 5) ("httpPconnTransferDone: content_length=%d\n", + reply->content_length); + /* If we haven't seen the end of reply headers, we are not done */ + if (httpState->reply_hdr_state < 2) + return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); + clen = httpReplyBodySize(httpState->request->method, reply); + /* If there is no message body, we can be persistent */ + if (0 == clen) + return (temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data) | FILTER_EOF); + /* If the body size is unknown we must wait for EOF */ + if (clen < 0) + return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); + /* If the body size is known, we must wait until we've gotten all of it. */ + if (mem->inmem_hi < reply->content_length + reply->hdr_sz) + return temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags, temp_filter->data); + /* We got it all */ + return (temp_filter->filter(buf,len,offset,filter_list, temp_filter, flags | FILTER_EOF, temp_filter->data) | FILTER_EOF); +} + +unsigned int httpProcessData(const char *buf, size_t len, void *data); + +static DATAFILTER httpDoAppend; +static DATAFILTER_HDR httpDoAppendHeaders; + +/* This will be called when data is ready to be read from fd. Read until + * error or connection closed. */ +/* XXX this function is too long! */ +static void +httpReadReply(int fd, void *data) +{ + HttpStateData *httpState = data; + LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + int len; + int bin; + int clen; + size_t read_sz; +#if DELAY_POOLS + delay_id delay_id; + + /* special "if" only for http (for nodelay proxy conns) */ + if (delayIsNoDelay(fd)) + delay_id = 0; + else + delay_id = delayMostBytesAllowed(entry->mem_obj); +#endif + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + comm_close(fd); + return; + } + /* Check for unexpected writes/protocol violations */ + if (httpState->eof) { + /* eof has _already_ been reached. */ + debug(11,1)("Server continued writing after End of message reached\n"); + comm_close(fd); + return; + } + /* check if we want to defer reading */ + errno = 0; + read_sz = SQUID_TCP_SO_RCVBUF; +#if DELAY_POOLS + read_sz = delayBytesWanted(delay_id, 1, read_sz); +#endif + statCounter.syscalls.sock.reads++; + len = FD_READ_METHOD(fd, buf, read_sz); + debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); + assert(fd==httpState->fd); + + if (len > 0) { + fd_bytes(fd, len, FD_READ); +#if DELAY_POOLS + delayBytesIn(delay_id, len); +#endif + kb_incr(&statCounter.server.all.kbytes_in, len); + kb_incr(&statCounter.server.http.kbytes_in, len); + commSetTimeout(fd, Config.Timeout.read, NULL, NULL); + IOStats.Http.reads++; + for (clen = len - 1, bin = 0; clen; bin++) + clen >>= 1; + IOStats.Http.read_hist[bin]++; + } + if (!httpState->reply_hdr && len > 0) { + /* Skip whitespace */ + while (len > 0 && xisspace(*buf)) + xmemmove(buf, buf + 1, len--); + if (len == 0) { + /* Continue to read... */ + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + return; + } + } + if (len < 0) { + debug(50, 2) ("httpReadReply: FD %d: read failure: %s.\n", + fd, xstrerror()); + if (ignoreErrno(errno)) { + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + } else if (entry->mem_obj->inmem_hi == 0) { + ErrorState *err; + err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); + err->xerrno = errno; + fwdFail(httpState->fwd, err); + comm_close(fd); + } else { + comm_close(fd); + } + } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { + ErrorState *err; + err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); + err->xerrno = errno; + fwdFail(httpState->fwd, err); + httpState->eof = 1; + comm_close(fd); +#if 0 + /* this isn;t needed anymore because its just a special case of filter response */ + } else if (len == 0) { + unsigned int rvflags=0; + /* TODO: this is broken: if this was a persistent connection, EOF can mean + * a failure - we should check we've recieved the expected amount of data + */ + /* Connection closed; retrieval done. */ + httpState->eof = 1; + /* TODO: optimise this out: it becomes a call to the filter chain + * with FILTER_EOF set */ + if (httpState->reply_hdr_state < 2) { + /* + * Yes Henrik, there is a point to doing this. When we + * called httpProcessReplyHeader() before, we didn't find + * the end of headers, but now we are definately at EOF, so + * we want to process the reply headers. + */ + httpProcessReplyHeader(httpState, buf, len); + te_build_decode_xlate_list(&entry->mem_obj->reply->header, &httpState->filters); + } + + rvflags = httpProcessData(NULL,0,data, FILTER_EOF); + + fwdComplete(httpState->fwd); + comm_close(fd); +#endif + } else { + unsigned int rvflags=0; + rvflags = httpProcessData(buf,len,data); + if (len==0) { + debug(11,1)("EOF by len=0 in httpHandleRead\n"); + httpState->eof = 1; + fwdComplete(httpState->fwd); + comm_close(fd); + } + else if (rvflags & FILTER_ABORT) { + debug(11,1)("Entry abort detected in httpHandleRead\n"); + } + else if (!(rvflags & FILTER_EOF)) { + /* Read more data */ + commSetSelect(httpState->fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + } else if (rvflags & FILTER_EOF) { + debug (11,1)(" End of persistent connection in httpHandleRead\n"); + httpState->eof = 1; + /* yes we have to clear all these! */ + commSetDefer(httpState->fd, NULL, NULL); + commSetTimeout(httpState->fd, -1, NULL, NULL); + commSetSelect(httpState->fd, COMM_SELECT_READ, NULL, NULL, 0); +#if DELAY_POOLS + delayClearNoDelay(httpState->fd); +#endif + comm_remove_close_handler(httpState->fd, httpStateFree, httpState); + fwdUnregister(httpState->fd, httpState->fwd); + pconnPush(httpState->fd, request->host, request->port); + fwdComplete(httpState->fwd); + fd=httpState->fd; + httpState->fd = -1; + httpStateFree(fd, httpState); + } + } +} +/* DATAFILTER */ +/* data filters do not consider len==0 to mean EOF. Set EOF in flags */ +/* all filters still get called on EOF. The last one back here calls the normal write func */ +/* reentrance and datafilters: if a datafilter wants to 'block' it has to: + * copy off the buf it was given + * call a callback style i/o/processing/etc routine + * return + * handle being called again, for the same request, before the callback has returned to + * the next step in processing + */ +/* the calling code assumes the following: the datafilter has forwarded all the data to + * the end of the chain. This MUST happen eventually. If can be deferred, but if + * DATA_EOF is set, the filter MUST flush and remove it self from the list and free it's + * private data + */ +/* A filter with no data to send for the moment, MAY call the filter chain but does not + * NEED to + */ +/* the calling pattern is (Buf, LEN, list_head, self_node, flags, statedata) */ + +/* TODO: send two filter chunks: one with just the headers, one with the rest */ +unsigned int + httpProcessData(const char *buf, size_t len, void *data) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + /* this was a const request_t... maybe it still should be, but the filterBuildChain + * and thus, all filters, should not modify the request_t struct... hmmm */ + request_t *request = httpState->request; + FILTER_list *temp_filter; + unsigned int rvflags = 0, flags=0; + debug(1,1)("Processing data (len=%d\n",len); + if (len == 0) + flags |= FILTER_EOF; + if (httpState->reply_hdr_state < 2) { + /* we haven't seen the full headers yet */ + /* ProcessReplyHeader expects partial data. This can be cleaned up now */ + httpProcessReplyHeader(httpState, buf, len); + if (httpState->reply_hdr_state == 2) { + http_status s = entry->mem_obj->reply->sline.status; + te_build_decode_xlate_list(&entry->mem_obj->reply->header, &httpState->oldfilters); + /* test for EOF condition (has to happen after content decoding! */ + filterChainAddTail(&httpState->oldfilters,httpPconnTransferDone,identity_header, NULL,httpState); + + /* apply user configured filters */ + filterBuildChain(Config.response_filters, &httpState->oldfilters, NULL, entry->mem_obj->reply , request); + + + /* Send the data to the store manager */ + filterChainAddTail(&httpState->oldfilters,httpDoAppend,identity_header, NULL, httpState); + + /* process the headers */ +// filterChainAddTail(&httpState->filters,httpDoAppendHeaders, identity_header, NULL,httpState); +#if WIP_FWD_LOG + fwdStatus(httpState->fwd, s); +#endif + /* + * If its not a reply that we will re-forward, then + * allow the client to get it. + */ + if (!fwdReforwardableStatus(s)) + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + } else { + /* why bother client_side & the store with a response that hasn't even + * got complete headers yet ? + */ + char *tempbuf; + tempbuf=httpState->headerstore; + httpState->headerstore=xmalloc(httpState->headerlength+len); + if (tempbuf) + memcpy(httpState->headerstore,tempbuf,httpState->headerlength); + memcpy(httpState->headerstore+httpState->headerlength,buf,len); + httpState->headerlength+=len; + if (tempbuf) + xfree(tempbuf); + } + /* add the data into any partial headers we have here. We memcpy out of + * courtesy to later fn's so they get all the headers at once. Aren't we + * nice + */ + /* we don't do this above (yet) because ProcessHeaders expects the data bit + * by bit. That can be fixed now by doing this test and merge earlier */ + if (httpState->headerstore && httpState->reply_hdr_state == 2) { + /* headers have been processed, but there's a last combine step to do */ + char *tempbuf; + size_t hdr_len; + tempbuf=httpState->headerstore; + httpState->headerstore=xmalloc(httpState->headerlength+len); + memcpy(httpState->headerstore,tempbuf,httpState->headerlength); + memcpy(httpState->headerstore+httpState->headerlength,buf,len); + httpState->headerlength+=len; + xfree(tempbuf); + hdr_len=httpState->headerlength; + + +/* TODO: the below code may call the next filter with EOF set twice. This is a bad thing. + * The problem is that we don't test for body data and incoming eof yet + */ + if (httpState->reply_hdr_state == 2) { + debug(1,1)("hdr_size %d, headerlength %d\n",httpState->reply_hdr_size, + httpState->headerlength); + assert(httpState->reply_hdr_size == httpState->headerlength); + } + debug(1,1)("sending combined headers\n"); + /* todo: split this into two calls like it is below */ + assert(httpState->oldfilters.head); + temp_filter=httpState->oldfilters.head->data; + rvflags |= temp_filter->filter(httpState->headerstore, httpState->reply_hdr_size, -1, &httpState->oldfilters, temp_filter, flags | FILTER_HTTP_HEADER, temp_filter->data); + + httpState->read_offset=0; + if (httpState->headerlength-httpState->reply_hdr_size) { + if (httpState->reply_hdr_state == 2) { + debug(1,1)("hdr_size %d, headerlength %d\n",httpState->reply_hdr_size, + httpState->headerlength); + assert(httpState->reply_hdr_size == httpState->headerlength); + } + debug(1,1)("headers (%d bytes) written, sending body (%d of %d)\n", + httpState->reply_hdr_size, + httpState->headerlength-httpState->reply_hdr_size,httpState->headerlength); + rvflags |= temp_filter->filter(httpState->headerstore+httpState->reply_hdr_size, httpState->headerlength-httpState->reply_hdr_size, -1, &httpState->oldfilters, temp_filter, flags , temp_filter->data); + httpState->read_offset+=len-hdr_len; + } else + debug(1,1)("headers (%d bytes) written, skipping body (%d of %d) due to filter return flags %d\n",httpState->reply_hdr_size,0,httpState->headerlength, rvflags); + + } else if (httpState->reply_hdr_state == 2){ + size_t hdr_len=httpState->headerlength; + /* no partial headers, got them in one chunk */ + debug(1,1)("headers in one packet... sending %d bytes\n", + hdr_len); + assert(httpState->oldfilters.head); + temp_filter=httpState->oldfilters.head->data; + rvflags |= temp_filter->filter(buf, hdr_len, -1, + &httpState->oldfilters, temp_filter, flags | FILTER_HTTP_HEADER, + temp_filter->data); + httpState->read_offset=0; + if (!(rvflags & (FILTER_EOF | FILTER_ABORT))) { + debug(1,1)("headers (%d bytes) written, sending body (%d of %d)\n", + hdr_len, + len-hdr_len,len); + if (len-hdr_len) { + rvflags |= temp_filter->filter(buf+hdr_len,len-hdr_len, httpState->read_offset, &httpState->oldfilters, temp_filter, flags , temp_filter->data); + httpState->read_offset+=len-hdr_len; + } + } else + debug(1,1)("headers (%d bytes) written, skipping body (%d of %d) due to filter return flags %d\n", + hdr_len, + len-hdr_len,len, rvflags); + } + } else { + /* headers have been seen. + */ + debug(1,1)("headers previously written \n"); + assert(httpState->oldfilters.head); + temp_filter=httpState->oldfilters.head->data; + rvflags |= temp_filter->filter(buf, len, httpState->read_offset, &httpState->oldfilters, temp_filter, flags, temp_filter->data); + httpState->read_offset+=len; + } + debug(1,1)("*** HTTP RAW READ OFFSET NOW %d\n",httpState->read_offset); + return rvflags; +} + +/* empty filter. Must be the last in the filter chain */ +DATAFILTER_FILTER(httpDoAppend) { + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + int fd; + int rvflags = 0; + + assert(filters->node.next==NULL); +debug(1,1)("entry %p\n",entry); + + debug(1,1)("httpDoAppend: recieved %d bytes at offset %d with flags %d\n",len,offset,flags); + if (!EBIT_TEST(entry->flags, ENTRY_ABORTED)) + storeAppend(entry,buf,len, flags); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * the above storeAppend() call could ABORT this entry, + * in that case, the server FD should already be closed. + * there's nothing for us to do. + */ + rvflags |= FILTER_ABORT; + } + return rvflags; +} + +/* empty filter. Must be the last in the filter chain + * this is the same as the above for the same reasons clientWriteMemBuf(Header) are the + * same. + * It should send the headers to the broker server side header function. (When this + * filter is hit, the server side filters have had the chance to remove/alter headers + * as needed. + */ +DATAFILTER_FILTERHEADER(httpDoAppendHeaders) { +#if 0 + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + int fd; + int rvflags = 0; + + assert(filters->node.next==NULL); +debug(1,1)("entry %p\n",entry); + + debug(1,1)("httpDoAppend: recieved %d bytes at offset %d with flags %d\n",len,offset,flags); + if (!EBIT_TEST(entry->flags, ENTRY_ABORTED)) + storeAppend(entry,buf,len, flags); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * the above storeAppend() call could ABORT this entry, + * in that case, the server FD should already be closed. + * there's nothing for us to do. + */ + rvflags |= FILTER_ABORT; + } + return rvflags; +#endif +} + +/* This will be called when request write is complete. Schedule read of + * reply. */ +static void +httpSendComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + ErrorState *err; + debug(11, 5) ("httpSendComplete: FD %d: size %d: errflag %d.\n", + fd, size, errflag); +#if URL_CHECKSUM_DEBUG + assert(entry->mem_obj->chksum == url_checksum(entry->mem_obj->url)); +#endif + if (size > 0) { + fd_bytes(fd, size, FD_WRITE); + kb_incr(&statCounter.server.all.kbytes_out, size); + kb_incr(&statCounter.server.http.kbytes_out, size); + } + if (errflag == COMM_ERR_CLOSING) + return; + if (errflag) { + err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); + err->xerrno = errno; + err->request = requestLink(httpState->orig_request); + errorAppendEntry(entry, err); + comm_close(fd); + return; + } else { + /* Schedule read reply. */ + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + /* + * Set the read timeout here because it hasn't been set yet. + * We only set the read timeout after the request has been + * fully written to the server-side. If we start the timeout + * after connection establishment, then we are likely to hit + * the timeout for POST/PUT requests that have very large + * request bodies. + */ + commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState); + commSetDefer(fd, fwdCheckDeferRead, entry); + } +} + +/* + * build request headers and append them to a given MemBuf + * used by httpBuildRequestPrefix() + * note: calls httpHeaderInit(), the caller is responsible for Clean()-ing + */ +void +httpBuildRequestHeader(request_t * request, + request_t * orig_request, + StoreEntry * entry, + HttpHeader * hdr_out, + int cfd, + http_state_flags flags) +{ + /* building buffer for complex strings */ +#define BBUF_SZ (MAX_URL+32) + LOCAL_ARRAY(char, bbuf, BBUF_SZ); + String strConnection = StringNull; + const HttpHeader *hdr_in = &orig_request->header; + int we_do_ranges; + const HttpHeaderEntry *e; + String strVia; + String strFwd; + HttpHeaderPos pos = HttpHeaderInitPos; + httpHeaderInit(hdr_out, hoRequest); + /* append our IMS header */ + if (request->lastmod > -1 && request->method == METHOD_GET) + httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); + + /* decide if we want to do Ranges ourselves + * (and fetch the whole object now) + * We want to handle Ranges ourselves iff + * - we can actually parse client Range specs + * - the specs are expected to be simple enough (e.g. no out-of-order ranges) + * - reply will be cachable + * (If the reply will be uncachable we have to throw it away after + * serving this request, so it is better to forward ranges to + * the server and fetch only the requested content) + */ + if (NULL == orig_request->range) + we_do_ranges = 0; + else if (!orig_request->flags.cachable) + we_do_ranges = 0; + else if (httpHdrRangeOffsetLimit(orig_request->range)) + we_do_ranges = 0; + else + we_do_ranges = 1; + debug(11, 8) ("httpBuildRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d\n", + orig_request->range, orig_request->flags.cachable, we_do_ranges); + + strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); + while ((e = httpHeaderGetEntry(hdr_in, &pos))) { + debug(11, 5) ("httpBuildRequestHeader: %s: %s\n", + strBuf(e->name), strBuf(e->value)); + if (!httpRequestHdrAllowed(e, &strConnection)) { + debug(11, 2) ("'%s' header denied by anonymize_headers configuration\n", + strBuf(e->name)); + continue; + } + switch (e->id) { + case HDR_PROXY_AUTHORIZATION: + /* Only pass on proxy authentication to peers for which + * authentication forwarding is explicitly enabled + */ + if (request->flags.proxying && orig_request->peer_login && + strcmp(orig_request->peer_login, "PASS") == 0) { + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + } + break; + case HDR_AUTHORIZATION: + /* Pass on WWW authentication even if used locally. If this is + * not wanted in an accelerator then the header can be removed + * using the anonymization functions + */ + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + /* XXX Some accelerators might want to strip the header + * and regard the reply as cacheable, but authentication + * is not normally enabled for accelerators without reading + * the code, so there is not much use in adding logics here + * without first defining the concept of having authentication + * in the accelerator... + */ + break; + case HDR_HOST: + /* + * Normally Squid does not copy the Host: header from + * a client request into the forwarded request headers. + * However, there is one case when we do: If the URL + * went through our redirector and the admin configured + * 'redir_rewrites_host' to be off. + */ + if (request->flags.redirected) + if (!Config.onoff.redir_rewrites_host) + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + break; + case HDR_IF_MODIFIED_SINCE: + /* append unless we added our own; + * note: at most one client's ims header can pass through */ + if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE)) + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + break; + case HDR_MAX_FORWARDS: + if (orig_request->method == METHOD_TRACE) { + /* sacrificing efficiency over clarity, etc. */ + const int hops = httpHeaderGetInt(hdr_in, HDR_MAX_FORWARDS); + if (hops > 0) + httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); + } + break; + case HDR_RANGE: + case HDR_IF_RANGE: + case HDR_REQUEST_RANGE: + if (!we_do_ranges) + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + break; + case HDR_PROXY_CONNECTION: + case HDR_CONNECTION: + case HDR_VIA: + case HDR_X_FORWARDED_FOR: + case HDR_CACHE_CONTROL: + /* append these after the loop if needed */ + break; + default: + /* pass on all other header fields */ + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + } + } + +#if UNKNOWN_CODE +/* This wasn't in head - should it be here */ + /* append fake user agent if configured and + * the real one is not supplied by the client */ + if (Config.fake_ua && !httpHeaderHas(hdr_out, HDR_USER_AGENT)) + httpHeaderPutStr(hdr_out, HDR_USER_AGENT, Config.fake_ua); +#endif + /* append TE */ + { + char strTE[128]; + + httpHeaderDelById(hdr_out, HDR_TE); /* hop by hop.. that's what last + hop could do*/ + /* TODO: this should be built from a list of known types & a acl allowing type + * to be used in requests vs responses */ + strcpy (strTE,"chunked;q=1.0"); + + httpHeaderPutStr (hdr_out,HDR_TE,strTE); + httpHeaderPutStr(hdr_out,HDR_CONNECTION, "TE"); /* its hop by hop */ + + } + + /* append Via */ + strVia = httpHeaderGetList(hdr_in, HDR_VIA); + snprintf(bbuf, BBUF_SZ, "%d.%d %s", + orig_request->http_ver.major, + orig_request->http_ver.minor, ThisCache); + strListAdd(&strVia, bbuf, ','); + httpHeaderPutStr(hdr_out, HDR_VIA, strBuf(strVia)); + stringClean(&strVia); + + /* append X-Forwarded-For */ + strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); + strListAdd(&strFwd, (cfd < 0 ? "unknown" : fd_table[cfd].ipaddr), ','); + httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd)); + stringClean(&strFwd); + + /* append Host if not there already */ + if (!httpHeaderHas(hdr_out, HDR_HOST)) { + /* use port# only if not default */ + if (orig_request->port == urlDefaultPort(orig_request->protocol)) { + httpHeaderPutStr(hdr_out, HDR_HOST, orig_request->host); + } else { + httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d", + orig_request->host, (int) orig_request->port); + } + } + /* append Authorization if known in URL, not in header and going direct */ + if (!httpHeaderHas(hdr_out, HDR_AUTHORIZATION)) { + if (!request->flags.proxying && *request->login) { + httpHeaderPutStrf(hdr_out, HDR_AUTHORIZATION, "Basic %s", + base64_encode(request->login)); + } + } + /* append Proxy-Authorization if configured for peer, and proxying */ + if (request->flags.proxying && orig_request->peer_login && + !httpHeaderHas(hdr_out, HDR_PROXY_AUTHORIZATION) && + strcmp(orig_request->peer_login, "PASS") != 0) { + if (*orig_request->peer_login == '*') { + /* Special mode, to pass the username to the upstream cache */ + char loginbuf[256]; + char *username = "-"; + if (orig_request->auth_user_request) + username = authenticateUserRequestUsername(orig_request->auth_user_request); + snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, orig_request->peer_login + 1); + httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s", + base64_encode(loginbuf)); + } else { + httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s", + base64_encode(orig_request->peer_login)); + } + } + /* append Cache-Control, add max-age if not there already */ + { + HttpHdrCc *cc = httpHeaderGetCc(hdr_in); + if (!cc) + cc = httpHdrCcCreate(); + if (!EBIT_TEST(cc->mask, CC_MAX_AGE)) { + const char *url = entry ? storeUrl(entry) : urlCanonical(orig_request); + httpHdrCcSetMaxAge(cc, getMaxAge(url)); + if (strLen(request->urlpath)) + assert(strstr(url, strBuf(request->urlpath))); + } + if (flags.only_if_cached) + EBIT_SET(cc->mask, CC_ONLY_IF_CACHED); + httpHeaderPutCc(hdr_out, cc); + httpHdrCcDestroy(cc); + } + /* maybe append Connection: keep-alive */ + if (flags.keepalive) { + if (flags.proxying) { + httpHeaderPutStr(hdr_out, HDR_PROXY_CONNECTION, "keep-alive"); + } else { + httpHeaderPutStr(hdr_out, HDR_CONNECTION, "keep-alive"); + } + } + /* Now mangle the headers. */ + httpHdrMangleList(hdr_out, request); + stringClean(&strConnection); +} + +/* build request prefix and append it to a given MemBuf; + * return the length of the prefix */ +mb_size_t +httpBuildRequestPrefix(request_t * request, + request_t * orig_request, + StoreEntry * entry, + MemBuf * mb, + int cfd, + http_state_flags flags) +{ + const int offset = mb->size; + memBufPrintf(mb, "%s %s HTTP/1.0\r\n", + RequestMethodStr[request->method], + strLen(request->urlpath) ? strBuf(request->urlpath) : "/"); + /* build and pack headers */ + { + HttpHeader hdr; + Packer p; + httpBuildRequestHeader(request, orig_request, entry, &hdr, cfd, flags); + packerToMemInit(&p, mb); + httpHeaderPackInto(&hdr, &p); + httpHeaderClean(&hdr); + packerClean(&p); + } + /* append header terminator */ + memBufAppend(mb, crlf, 2); + return mb->size - offset; +} +/* This will be called when connect completes. Write request. */ +static void +httpSendRequest(HttpStateData * httpState) +{ + MemBuf mb; + request_t *req = httpState->request; + StoreEntry *entry = httpState->entry; + int cfd; + peer *p = httpState->peer; + CWCB *sendHeaderDone; + + debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", httpState->fd, httpState); + + if (httpState->orig_request->body_connection) + sendHeaderDone = httpSendRequestEntry; + else + sendHeaderDone = httpSendComplete; + + if (!opt_forwarded_for) + cfd = -1; + else if (entry->mem_obj == NULL) + cfd = -1; + else + cfd = entry->mem_obj->fd; + assert(-1 == cfd || FD_SOCKET == fd_table[cfd].type); + if (p != NULL) + httpState->flags.proxying = 1; + else + httpState->flags.proxying = 0; + /* + * Is keep-alive okay for all request methods? + */ + if (!Config.onoff.server_pconns) + httpState->flags.keepalive = 0; + else if (p == NULL) + httpState->flags.keepalive = 1; + else if (p->stats.n_keepalives_sent < 10) + httpState->flags.keepalive = 1; + else if ((double) p->stats.n_keepalives_recv / (double) p->stats.n_keepalives_sent > 0.50) + httpState->flags.keepalive = 1; + if (httpState->peer) + if (neighborType(httpState->peer, httpState->request) == PEER_SIBLING && + !httpState->peer->options.allow_miss) + httpState->flags.only_if_cached = 1; + memBufDefInit(&mb); + httpBuildRequestPrefix(req, + httpState->orig_request, + entry, + &mb, + cfd, + httpState->flags); + debug(11, 6) ("httpSendRequest: FD %d:\n%s\n", httpState->fd, mb.buf); + comm_write_mbuf(httpState->fd, mb, sendHeaderDone, httpState); +} + +/* we've got data from the server. It might be a new request,or part of an existing + * request. Later on we can look at different read callbacks for in progress requests + */ +static void +httpCommReadComplete(int fd, char *buf, off_t offset, size_t size, int howmuch, int flags, + void *data) +{ + HttpStateData *http=data; + size_t headerlen; + assert(http->readbuf); + assert(http->readbuf->buffer); + assert(offset==http->readbuf->offset); + assert(size=http->readbuf->size); + /* we may have fun parsing pipelined responses without memcpy - we really need + length = how much is in the buffer from offset on + offset = our current start point in the buffer (incremented as we process responses + size = the buffer size + buf = the buffer data + or possibly, move size into buf... + */ + if (http->rep && http->hcrequest->repfilters.head) { + /* possibly just a case of call the filter chain... */ + FILTER_list *temp_filter; + unsigned int rvflags, eofflag; + iobuf *readbuf; + /* if we've already got a request, we are using clean buffers now */ + assert(offset==0); + temp_filter=http->hcrequest->repfilters.head->data; + http->read_offset+=howmuch; + /* check for trivial EOF status. */ + #define COMM_EOF 0 + eofflag = ((http->read_offset >= http->rep->content_length) || (flags & COMM_EOF)) ? FILTER_EOF : 0; + debug (33,0)("read offset is %d, content length %d\n",http->read_offset, http->rep->content_length); + /* FIXME: check for content overrun here */ + /* NOTE: however we do this, it must be callable by the TE code as well.. */ + /* Allow a new read to be scheduled */ + readbuf=http->readbuf; + http->readbuf=NULL; + /* if this looks hacky.. it is. comm_reads not finished, and these two code + * blocks are reversed */ + rvflags=temp_filter->filter(howmuch ? readbuf->buffer : NULL, + howmuch,http->read_offset-howmuch, + &http->hcrequest->repfilters, temp_filter,eofflag,temp_filter->data); + ioBufferUnlock(readbuf); + /* TODO: one of the filters needs to be a pipeline request handler that gets the next request handled correctly - setups this func up again */ + debug (33,3)("Process request got flags %0x from rep_filter\n", rvflags); + /* TODO: cancel the read in some fashion if EOF is signalled after a read is queued */ + if (rvflags & FILTER_EOF || eofflag) { + http->hcrequest->serverread=NULL; + http->hcrequest->serverreaddata=NULL; + cbdataUnlock(http); + } + return; + } + + headerlen = headersEnd(buf+offset, size); + if (headerlen) { + http->rep=httpReplyCreate(); + if (httpReplyParse(http->rep, buf+offset, headerlen)) { + /* process the request */ + /* TODO: filter the server-side only things (ie connection header...) */ + /* TODO 2: add the server-side filters (ie TE) */ + FILTER_list *temp_filter; + unsigned int rvflags, eofflag; + iobuf *readbuf; + temp_filter=http->hcrequest->repfilters.head->data; + http->readbuf->offset+=howmuch; + http->read_offset=0; + /* FIXME: this is broken if the server writes the headers in a different + *packet to the first body data: we need to check the status line */ + if (!http->rep->content_length) + eofflag = !(http->readbuf->offset - headerlen) ? FILTER_EOF : 0; + else + eofflag = 0; + /* Allow a new read to be scheduled */ + readbuf=http->readbuf; + http->readbuf=NULL; + rvflags = temp_filter->filter_hdr(http->rep, http->hcrequest ,&http->hcrequest->repfilters,temp_filter, eofflag, temp_filter->data); + debug (33,3)("Process request got flags %0x from rep_header\n", rvflags); + if ((readbuf->offset-headerlen) + && !(rvflags & (FILTER_ABORT | FILTER_EOF))) { + temp_filter=http->hcrequest->repfilters.head->data; + http->read_offset+=readbuf->offset-headerlen; + /* offset total length in buffer */ + rvflags=temp_filter->filter(readbuf->buffer+readbuf->offset+headerlen-howmuch,readbuf->offset-headerlen,0,&http->hcrequest->repfilters, + temp_filter,0,temp_filter->data); + debug (33,3)("Process request got flags %0x from rep_filter\n", rvflags); + } + /* all data written */ + ioBufferUnlock(readbuf); + + + } + } else { + /* couldn't parse the reply */ + http->readbuf->offset+=howmuch; + if (http->readbuf->offset >= http->readbuf->size) + fatalf("more than buffer size (%d) without a parsable response header\n",http->readbuf->size); + } + /* it's up to the filter chain to schedule another read */ +} + +/* push the body up to the server */ +DATAFILTER_FILTER(http_server_req_body) +{ + /* check for EOF or content length overrun */ + /* check offset is valid. (=httpstate offset) + update httpstate offset - it is not updated by the commcomplete routine */ + return 0; +} + +/* we could check for request->content_length... but what if that gets... corrupted? + */ +DATAFILTER_FILTER(http_server_unexpected_body) +{ + fatal("unexpected body.. abort request/reply cleanly here\n"); + return 0; +} + +/* the header has been called.. should never be again + * remove me once stable code + */ +DATAFILTER_FILTERHEADER(http_trap) +{ + fatal("We should never have been called\n"); + return 0; +} + +static CWCB httpCommWriteComplete; +static CRCB httpCommReadComplete; +static FILTERREAD httpServerRead; + +static void +httpServerRead(void * data) +{ + HttpStateData *http=data; + iobuf *readbuf; + /* allocate a buffer. This is annoying: could comm_read allocate it for us? + * (when we pass a NULL buf) */ + if (http->readbuf) + /* theres a write to the client in progress, or an attempt to queue multiple reads*/ + return; + readbuf = ioBufferAlloc(4096); + assert(readbuf); + http->readbuf=readbuf; + comm_read(http->fd, readbuf->buffer, readbuf->offset, readbuf->size, httpCommReadComplete, http); +} + +/* we are the last in the filter chain. Always. */ +/* Unknowns: how do we detect early server writes: for example if the server rejects our + * header (ie content_length too long) + */ +static void +httpCommWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) +{ + HttpStateData *http=data; + if (http->hcrequest->request->content_length && ~(http->filterflags & FILTER_EOF )) { + /* there is a body.. and we haven't finished */ +// hcrequest->read(http->hcrequest); + } + /* there is no more body */ + httpServerRead(data); +} + +/* helper function for http_server_req. + * We modify the original request. Why? speed. + * This raises a general issue: + * + * the common-to-all-upstream headers should be processed once. + * then we process the per-upstream headers on each forward. + * this either requires a filter for the one-time which we remove, or + * a flag in the httpstate/request variable. + * TODO: DO THIS BEFORE THE FATALS RECOMMENDED BELOW + */ +void +httpPrepareRequestHeader(request_t * request, + http_state_flags flags) +{ + /* building buffer for complex strings */ +#define BBUF_SZ (MAX_URL+32) + LOCAL_ARRAY(char, bbuf, BBUF_SZ); + String strConnection = StringNull; + int we_do_ranges; + const HttpHeaderEntry *e; + const int hops; + String strVia; + String strFwd; + HttpHeaderPos pos = HttpHeaderInitPos; + +#if 0 + httpHeaderInit(hdr_out, hoRequest); + /* append our IMS header */ + if (request->lastmod > -1 && request->method == METHOD_GET) + httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); +#endif + + /* decide if we want to do Ranges ourselves + * (and fetch the whole object now) + * We want to handle Ranges ourselves iff + * - we can actually parse client Range specs + * - the specs are expected to be simple enough (e.g. no out-of-order ranges) + * - reply will be cachable + * (If the reply will be uncachable we have to throw it away after + * serving this request, so it is better to forward ranges to + * the server and fetch only the requested content) + */ + +/* Dump the headers - debugging purposes only*/ +{ + HttpHeaderPos pos = HttpHeaderInitPos; + const HttpHeaderEntry *e; + debug(55, 7) ("packing hdr: (%p)\n", request->header); + /* pack all entries one by one */ + while ((e = httpHeaderGetEntry(&request->header, &pos))) +printf("%s: %s\r\n",strBuf(e->name),strBuf(e->value)); +} + + if (NULL == request->range) + we_do_ranges = 0; + else if (!request->flags.cachable) + we_do_ranges = 0; + else if (httpHdrRangeOffsetLimit(request->range)) + we_do_ranges = 0; + else + we_do_ranges = 1; + debug(11, 8) ("httpPrepareRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d +\n", + request->range, request->flags.cachable, we_do_ranges); + +/* remove any and all headers we don't want here - so that the loop isn't broken */ +if (httpHeaderHas(&request->header, HDR_MAX_FORWARDS)) { + hops = httpHeaderGetInt(&request->header, HDR_MAX_FORWARDS); + httpHeaderDelById(&request->header, HDR_MAX_FORWARDS); + /* sacrificing efficiency over clarity, etc. */ + /* why don't we check for 0? because we are never reached for 0 */ + httpHeaderPutInt(&request->header, HDR_MAX_FORWARDS, hops - 1); +} +if (we_do_ranges) { + httpHeaderDelById(&request->header, HDR_RANGE); + httpHeaderDelById(&request->header, HDR_IF_RANGE); + httpHeaderDelById(&request->header, HDR_REQUEST_RANGE); +} +httpHeaderDelById(&request->header, HDR_PROXY_CONNECTION); +httpHeaderDelById(&request->header, HDR_CONNECTION); + + strConnection = httpHeaderGetList(&request->header, HDR_CONNECTION); + while ((e = httpHeaderGetEntry(&request->header, &pos))) { + debug(11, 5) ("httpPrepareRequestHeader: %s: %s\n", + strBuf(e->name), strBuf(e->value)); + if (!httpRequestHdrAllowed(e, &strConnection)) { + debug(11, 2) ("'%s' header denied by anonymize_headers configuration\n", + strBuf(e->name)); + continue; + } + switch (e->id) { + case HDR_PROXY_AUTHORIZATION: + /* FIXME: client side todo: remove the auth if it's not needed. + * Then we simply test: is it present? no->add peer credentials */ + /* Only pass on proxy authentication to peers for which + * authentication forwarding is explicitly enabled + */ +#if 0 + if (request->flags.proxying && request->peer_login && + strcmp(request->peer_login, "PASS") == 0) { + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); + } + +#endif + break; + case HDR_AUTHORIZATION: + /* FIXME: is there _ever_ anything to do here? */ + /* Pass on WWW authentication even if used locally. If this is + * not wanted in an accelerator then the header can be removed + * using the anonymization functions + */ +#if 0 + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); +#endif + /* XXX Some accelerators might want to strip the header + * and regard the reply as cacheable, but authentication + * is not normally enabled for accelerators without reading + * the code, so there is not much use in adding logics here + * without first defining the concept of having authentication + * in the accelerator... + */ + break; + case HDR_HOST: + /* + * Normally Squid does not copy the Host: header from + * a client request into the forwarded request headers. + * However, there is one case when we do: If the URL + * went through our redirector and the admin configured + * 'redir_rewrites_host' to be off. + */ + /* FIXME: this seems broken: host is a required header for + * HTTP/1.1 + */ +#if 0 + if (request->flags.redirected) + if (!Config.onoff.redir_rewrites_host) + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); +#endif + break; + case HDR_IF_MODIFIED_SINCE: + /* append unless we added our own; + * note: at most one client's ims header can pass through */ + /* FIXME: nothing to see, move along. (The cache adds this header itself + */ +#if 0 + if (!httpHeaderHas(&request->header, HDR_IF_MODIFIED_SINCE)) + httpHeaderAddEntry(&request->header, httpHeaderEntryClone(e)); +#endif + break; + case HDR_MAX_FORWARDS: + break; + case HDR_RANGE: + case HDR_IF_RANGE: + case HDR_REQUEST_RANGE: + /* we inherit the client range request */ + /* if we do the header logic, we have to delete the existing header */ +#if 0 + if (!we_do_ranges) + httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); +#endif + break; + case HDR_PROXY_CONNECTION: + /* FIXME: update client side to clean this, then fatal here to indicate a + * broken client module + */ + httpHeaderDelById(&request->header, HDR_PROXY_CONNECTION); + break; + case HDR_CONNECTION: + /* DITTO */ + httpHeaderDelById(&request->header, HDR_CONNECTION); + break; + case HDR_VIA: + case HDR_X_FORWARDED_FOR: + case HDR_CACHE_CONTROL: + /* append these after the loop if needed */ + /* FIXME: check what of these we are willing to pass on */ + break; + default: + /* pass on all other header fields */ + } + } + + /* append TE */ + { + char strTE[128]; + + /* FIXME: update client_side, then fatal here if the header is present */ + httpHeaderDelById(&request->header, HDR_TE); /* hop by hop.. that's what last + hop could do*/ + /* TODO: this should be built from a list of known types & a acl allowing type + * to be used in requests vs responses + */ + strcpy (strTE,"chunked;q=1.0"); + + httpHeaderPutStr (&request->header,HDR_TE,strTE); + httpHeaderPutStr (&request->header,HDR_CONNECTION, "TE"); /* its hop by hop */ + + } + + /* append Via */ + /* DOES THIS WORK?! or do we need to delete and add the header ? */ + strVia = httpHeaderGetList(&request->header, HDR_VIA); + /* this is broken: we want out http version don't we? */ + snprintf(bbuf, BBUF_SZ, "%d.%d %s", + request->http_ver.major, + request->http_ver.minor, ThisCache); + strListAdd(&strVia, bbuf, ','); + stringClean(&strVia); +#if 0 + strVia = httpHeaderGetList(&request->header, HDR_VIA); + snprintf(bbuf, BBUF_SZ, "%d.%d %s", + orig_request->http_ver.major, + orig_request->http_ver.minor, ThisCache); + strListAdd(&strVia, bbuf, ','); + httpHeaderPutStr(hdr_out, HDR_VIA, strBuf(strVia)); + stringClean(&strVia); +#endif + +#if 0 + /* the client side should choose to add this. We cannot depend on having a + * client. (think timed cache refreshes for instance) + */ + + /* append X-Forwarded-For */ + strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); + strListAdd(&strFwd, (cfd < 0 ? "unknown" : fd_table[cfd].ipaddr), ','); + httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd)); + stringClean(&strFwd); +#endif + + /* append Host if not there already */ + if (!httpHeaderHas(&request->header, HDR_HOST)) { + /* use port# only if not default */ + if (request->port == urlDefaultPort(request->protocol)) { + httpHeaderPutStr(&request->header, HDR_HOST, request->host); + } else { + httpHeaderPutStrf(&request->header, HDR_HOST, "%s:%d", + request->host, (int) request->port); + } + } + /* append Authorization if known in URL, not in header and going direct */ + if (!httpHeaderHas(&request->header, HDR_AUTHORIZATION)) { + if (!request->flags.proxying && *request->login) { + httpHeaderPutStrf(&request->header, HDR_AUTHORIZATION, "Basic %s", + base64_encode(request->login)); + } + } + /* Remember: the client side strips this if not needed */ + /* append Proxy-Authorization if configured for peer, and proxying */ + if (request->flags.proxying && request->peer_login && + !httpHeaderHas(&request->header, HDR_PROXY_AUTHORIZATION) && + strcmp(request->peer_login, "PASS") != 0) { + if (*request->peer_login == '*') { + /* Special mode, to pass the username to the upstream cache */ + char loginbuf[256]; + char *username = "-"; + if (request->auth_user_request) + username = authenticateUserRequestUsername(request->auth_user_request); + snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, request->peer_login + 1); + httpHeaderPutStrf(&request->header, HDR_PROXY_AUTHORIZATION, "Basic %s", + base64_encode(loginbuf)); + } else { + httpHeaderPutStrf(&request->header, HDR_PROXY_AUTHORIZATION, "Basic %s", + base64_encode(request->peer_login)); + } + } + + /* maybe append Connection: keep-alive */ + /* what about connection: TE ? ? -rob*/ + if (flags.keepalive) { + if (flags.proxying) { + httpHeaderPutStr(&request->header, HDR_PROXY_CONNECTION, "keep-alive"); + } else { + httpHeaderPutStr(&request->header, HDR_CONNECTION, "keep-alive"); + } + } + /* Now mangle the headers. */ + httpHdrMangleList(&request->header, request); + stringClean(&strConnection); +} + + +/* we're attached to an fd, and have a request to make */ +unsigned int +http_server_req(HttpReply *rep, clientHttpRequest *request, + dlink_list * filter_list,FILTER_list * filters, unsigned int flags, + void *data) +{ + HttpStateData *http=data; + int cfd=-1; /* we don't care about the client fd */ + MemBuf mb; + /* FIXME: check we haven't been called to cleanup _before_ the write completed */ + httpPrepareRequestHeader(request->request, http->flags); + mb = HttpRequestPackMemBuf(request->request); + + if (request->request->content_length) + filterChainAddTail(filter_list, http_server_req_body, http_trap, NULL, data); + else + filterChainAddTail(filter_list, http_server_unexpected_body, http_trap, NULL, data); + comm_write_mbuf(http->fd, mb, httpCommWriteComplete, data); + printf("\n===\n%s===\n",mb.buf); + http->rep=rep; + http->hcrequest=request; + http->filter_list=filter_list; + http->filters=filters->node.next->data; + http->filterflags=flags; + http->hcrequest->serverread=httpServerRead; + http->hcrequest->serverreaddata=http; + cbdataLock(http->hcrequest->serverreaddata); + dlinkDelete(&filters->node, filter_list); + xfree(filters); + } + + +/* create a HttpStateData variable for the server side functions to reference. */ +void * +http_server_reqstate(FwdState * fwd) +{ + HttpStateData *httpState; + CBDATA_INIT_TYPE(HttpStateData); + httpState=cbdataAlloc(HttpStateData); + httpState->fwd = fwd; + httpState->fd = fwd->server_fd; + httpState->read_offset=-1; /*-1 means we are waiting for a response, or in headers */ + httpState->peer = fwd->servers ? fwd->servers->peer : NULL; /* might be NULL */ + httpState->flags.proxying = httpState->peer ? 1 : 0; + /* + * Is keep-alive okay for all request methods? + */ + /* global */ + if (!Config.onoff.server_pconns) + httpState->flags.keepalive = 0; + /* "proxying" mean hierachy - keep alive non hierarchy requests always */ + else if (httpState->flags.proxying == 0) + httpState->flags.keepalive = 1; + /* for this peer, less that 10, always keep alive */ + else if (httpState->peer->stats.n_keepalives_sent < 10) + httpState->flags.keepalive = 1; + /* whats this magic for. I no understand - robert C */ + else if ((double) httpState->peer->stats.n_keepalives_recv / (double) httpState->peer->stats.n_keepalives_sent > 0.50) + httpState->flags.keepalive = 1; + if (httpState->peer && + neighborType(httpState->peer, httpState->request) == PEER_SIBLING && + !httpState->peer->options.allow_miss) + httpState->flags.only_if_cached = 1; + + /* old code created a separate server-side request. uck. what we want is a clean + * two level structure - client side stuff in a list of attached clients, and server + * side stuff in the server request. + * It _looks_ like that was done to avoid having a peer host and port struct ! + * -Rob + */ + /* + * register the handler to free HTTP state data when the FD closes + */ + /* + * This may not be correct anymore: thanks to filters, one cancel should percolate + * through anyway. FIXME + */ + comm_add_close_handler(httpState->fd, httpStateFree, httpState); + statCounter.server.all.requests++; + statCounter.server.http.requests++; + return httpState; +} + +void +httpStart(FwdState * fwd) +{ + int fd = fwd->server_fd; + HttpStateData *httpState; + request_t *proxy_req; + request_t *orig_req = fwd->request; + debug(11, 3) ("httpStart: \"%s %s\"\n", + RequestMethodStr[orig_req->method], + storeUrl(fwd->entry)); + httpState = cbdataAlloc(HttpStateData); + storeLockObject(fwd->entry); + httpState->fwd = fwd; + httpState->entry = fwd->entry; + httpState->fd = fd; + httpState->read_offset=-1; /*-1 means we are waiting for a response, or in headers */ + if (fwd->servers) + httpState->peer = fwd->servers->peer; /* might be NULL */ + if (httpState->peer) { + proxy_req = requestCreate(orig_req->method, + PROTO_NONE, storeUrl(httpState->entry)); + xstrncpy(proxy_req->host, httpState->peer->host, SQUIDHOSTNAMELEN); + proxy_req->port = httpState->peer->http_port; + proxy_req->flags = orig_req->flags; + proxy_req->lastmod = orig_req->lastmod; + httpState->request = requestLink(proxy_req); + httpState->orig_request = requestLink(orig_req); + proxy_req->flags.proxying = 1; + /* + * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here. + * We might end up getting the object from somewhere else if, + * for example, the request to this neighbor fails. + */ + if (httpState->peer->options.proxy_only) + storeReleaseRequest(httpState->entry); +#if DELAY_POOLS + assert(delayIsNoDelay(fd) == 0); + if (httpState->peer->options.no_delay) + delaySetNoDelay(fd); +#endif + } else { + httpState->request = requestLink(orig_req); + httpState->orig_request = requestLink(orig_req); + } + /* + * register the handler to free HTTP state data when the FD closes + */ + comm_add_close_handler(fd, httpStateFree, httpState); + statCounter.server.all.requests++; + statCounter.server.http.requests++; + httpSendRequest(httpState); + /* + * We used to set the read timeout here, but not any more. + * Now its set in httpSendComplete() after the full request, + * including request body, has been written to the server. + */ +} + +static void +httpSendRequestEntryDone(int fd, void *data) +{ + HttpStateData *httpState = data; + aclCheck_t ch; + debug(11, 5) ("httpSendRequestEntryDone: FD %d\n", + fd); + memset(&ch, '\0', sizeof(ch)); + ch.request = httpState->request; + if (!Config.accessList.brokenPosts) { + debug(11, 5) ("httpSendRequestEntryDone: No brokenPosts list\n"); + httpSendComplete(fd, NULL, 0, 0, data); + } else if (!aclCheckFast(Config.accessList.brokenPosts, &ch)) { + debug(11, 5) ("httpSendRequestEntryDone: didn't match brokenPosts\n"); + httpSendComplete(fd, NULL, 0, 0, data); + } else { + debug(11, 2) ("httpSendRequestEntryDone: matched brokenPosts\n"); + comm_write(fd, "\r\n", 2, httpSendComplete, data, NULL); + } +} + +static void +httpRequestBodyHandler(char *buf, size_t size, void *data) +{ + HttpStateData *httpState = (HttpStateData *) data; + if (size > 0) { + comm_write(httpState->fd, buf, size, httpSendRequestEntry, data, memFree8K); + } else if (size == 0) { + /* End of body */ + memFree8K(buf); + httpSendRequestEntryDone(httpState->fd, data); + } else { + /* Failed to get whole body, probably aborted */ + memFree8K(buf); + httpSendComplete(httpState->fd, NULL, 0, COMM_ERR_CLOSING, data); + } +} + +static void +httpSendRequestEntry(int fd, char *bufnotused, size_t size, int errflag, void *data) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + ErrorState *err; + debug(11, 5) ("httpSendRequestEntry: FD %d: size %d: errflag %d.\n", + fd, size, errflag); + if (size > 0) { + fd_bytes(fd, size, FD_WRITE); + kb_incr(&statCounter.server.all.kbytes_out, size); + kb_incr(&statCounter.server.http.kbytes_out, size); + } + if (errflag == COMM_ERR_CLOSING) + return; + if (errflag) { + err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); + err->xerrno = errno; + err->request = requestLink(httpState->orig_request); + errorAppendEntry(entry, err); + comm_close(fd); + return; + } + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + comm_close(fd); + return; + } + clientReadBody(httpState->orig_request, memAllocate(MEM_8K_BUF), 8192, httpRequestBodyHandler, httpState); +} + +void +httpBuildVersion(http_version_t * version, unsigned int major, unsigned int minor) +{ + version->major = major; + version->minor = minor; +}