--------------------- PatchSet 1525 Date: 2005/08/18 22:41:00 Author: dwsquid Branch: squid3-icap Tag: (none) Log: update to HTTP <-> ICAP interaction Members: src/ICAPAnchor.cc:1.1.2.3->1.1.2.4 src/ICAPAnchor.h:1.1.2.3->1.1.2.4 src/http.cc:1.49.2.2->1.49.2.3 src/http.h:1.11.4.1->1.11.4.2 Index: squid3/src/ICAPAnchor.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/Attic/ICAPAnchor.cc,v retrieving revision 1.1.2.3 retrieving revision 1.1.2.4 diff -u -r1.1.2.3 -r1.1.2.4 --- squid3/src/ICAPAnchor.cc 17 Aug 2005 23:49:25 -0000 1.1.2.3 +++ squid3/src/ICAPAnchor.cc 18 Aug 2005 22:41:00 -0000 1.1.2.4 @@ -7,7 +7,11 @@ #include "ICAPAnchor.h" #include "ICAPModule.h" +CBDATA_TYPE(ICAPAnchor); + + ICAPAnchor::ICAPAnchor(): virgin(NULL), adapted(NULL) { +debug(0,0)("ICAPAnchor constructed, this=%p\n", this); } ICAPAnchor::~ICAPAnchor() { @@ -15,6 +19,23 @@ cbdataReferenceDone(httpState); } + +void * +ICAPAnchor::operator new(size_t byteCount) +{ + CBDATA_INIT_TYPE(ICAPAnchor); + ICAPAnchor *result = cbdataAlloc(ICAPAnchor); + return result; +} + +void +ICAPAnchor::operator delete (void *address) +{ + ICAPAnchor *t = static_cast(address); + cbdataFree(t); +} + + void ICAPAnchor::startRespMod(HttpStateData *anHttpState, HttpRequest *request, HttpReply *reply) { httpState = cbdataReference(anHttpState); @@ -23,7 +44,12 @@ virgin->data = new MsgPipeData; virgin->data->cause = request; virgin->data->header = reply; +#if MSG_PIPE_USES_MEM_HDR + virgin->data->body = new mem_hdr; +#else virgin->data->body = new MemBuf; + memBufDefInit(virgin->data->body); +#endif adapted = new MsgPipe; adapted->sink = this; @@ -49,6 +75,12 @@ // HttpStateData says we have the entire HTTP message void ICAPAnchor::doneSending() { debug(0,0)("ICAPAnchor::doneSending() called\n"); + + /* LOOPBACK */ + /* simple assignments are not the right way to do this */ + adapted->data->header = virgin->data->header; + adapted->data->body = virgin->data->body; + noteSourceStart(adapted); } // HttpStateData tells us to abort @@ -74,7 +106,7 @@ // ICAP client has received new HTTP headers (if any) at this point void ICAPAnchor::noteSourceStart(MsgPipe *p) { debug(0,0)("ICAPAnchor::noteSourceStart() called\n"); - //tell HttpStateData to prepare the response store + httpState->takeAdaptedHeaders(adapted->data->header); noteSourceProgress(p); } Index: squid3/src/ICAPAnchor.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/Attic/ICAPAnchor.h,v retrieving revision 1.1.2.3 retrieving revision 1.1.2.4 diff -u -r1.1.2.3 -r1.1.2.4 --- squid3/src/ICAPAnchor.h 17 Aug 2005 23:49:25 -0000 1.1.2.3 +++ squid3/src/ICAPAnchor.h 18 Aug 2005 22:41:02 -0000 1.1.2.4 @@ -1,6 +1,6 @@ /* - * $Id: ICAPAnchor.h,v 1.1.2.3 2005/08/17 23:49:25 dwsquid Exp $ + * $Id: ICAPAnchor.h,v 1.1.2.4 2005/08/18 22:41:02 dwsquid Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -53,6 +53,10 @@ ICAPAnchor(); virtual ~ICAPAnchor(); + // Overload new and delete for cbdata + void *operator new(size_t); + void operator delete(void *); + // synchronous calls called by HttpStateData void startRespMod(HttpStateData *anHttpState, HttpRequest *request, HttpReply *reply); void sendMoreData(StoreIOBuffer buf); Index: squid3/src/http.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/http.cc,v retrieving revision 1.49.2.2 retrieving revision 1.49.2.3 diff -u -r1.49.2.2 -r1.49.2.3 --- squid3/src/http.cc 17 Aug 2005 23:49:25 -0000 1.49.2.2 +++ squid3/src/http.cc 18 Aug 2005 22:41:02 -0000 1.49.2.3 @@ -1,6 +1,6 @@ /* - * $Id: http.cc,v 1.49.2.2 2005/08/17 23:49:25 dwsquid Exp $ + * $Id: http.cc,v 1.49.2.3 2005/08/18 22:41:02 dwsquid Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -56,7 +56,6 @@ #endif CBDATA_TYPE(HttpStateData); -CBDATA_TYPE(ICAPAnchor); static const char *const crlf = "\r\n"; @@ -80,6 +79,7 @@ httpStateFree(int fd, void *data) { HttpStateData *httpState = static_cast(data); + debug(11,5)("httpStateFree: FD %d, httpState=%p\n", fd, data); if (httpState == NULL) return; @@ -647,6 +647,37 @@ } } +void +HttpStateData::keepaliveAccounting(HttpReply *reply) +{ + if (flags.keepalive) + if (_peer) + _peer->stats.n_keepalives_sent++; + + if (reply->keep_alive) { + if (_peer) + _peer->stats.n_keepalives_recv++; + + if (Config.onoff.detect_broken_server_pconns && httpReplyBodySize(request->method, reply) == -1) { + debug(11, 1) ("httpProcessReplyHeader: Impossible keep-alive header from '%s'\n", storeUrl(entry)); + // debug(11, 2) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", reply_hdr.buf); + flags.keepalive_broken = 1; + } + } +} + +void +HttpStateData::checkDateSkew(HttpReply *reply) +{ + if (reply->date > -1 && !_peer) { + int skew = abs((int)(reply->date - squid_curtime)); + + if (skew > 86400) + debug(11, 3) ("%s's clock is skewed by %d seconds!\n", + request->host, skew); + } +} + /* rewrite this later using new interfaces @?@ * This creates the error page itself.. its likely * that the forward ported reply header max size patch @@ -739,10 +770,10 @@ return; } + + keepaliveAccounting(reply); + checkDateSkew(reply); processSurrogateControl (reply); -#if ICAP_CLIENT - icap->startRespMod(this, request, reply); -#endif /* TODO: we need our own reply * in the httpState, as we probably don't want to replace * the storeEntry with interim headers */ @@ -751,10 +782,31 @@ * Parse the header and remove all referenced headers */ +#if ICAP_CLIENT + icap->startRespMod(this, request, reply); +#else storeEntryReplaceObject(entry, reply); - /* DO NOT USE reply now */ + /* Note storeEntryReplaceObject() consumes reply, so we cannot use it */ reply = NULL; + haveParsedReplyHeaders(); +#endif + + if (eof == 1) { + fwdComplete(fwd); + comm_close(fd); + } + ctx_exit(ctx); +} + +/* + * This function used to be joined with processReplyHeader(), but + * we split it for ICAP. + */ +void +HttpStateData::haveParsedReplyHeaders() +{ + Ctx ctx = ctx_enter(entry->mem_obj->url); if (entry->getReply()->sline.status == HTTP_PARTIAL_CONTENT && entry->getReply()->content_range) currentOffset = entry->getReply()->content_range->spec.offset; @@ -817,39 +869,11 @@ EBIT_SET(entry->flags, ENTRY_REVALIDATE); } - if (flags.keepalive) - if (_peer) - _peer->stats.n_keepalives_sent++; - - if (entry->getReply()->keep_alive) { - if (_peer) - _peer->stats.n_keepalives_recv++; - - if (Config.onoff.detect_broken_server_pconns && httpReplyBodySize(request->method, reply) == -1) { - debug(11, 1) ("httpProcessReplyHeader: Impossible keep-alive header from '%s'\n", storeUrl(entry)); - debug(11, 2) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", reply_hdr.buf); - flags.keepalive_broken = 1; - } - } - - if (entry->getReply()->date > -1 && !_peer) { - int skew = abs((int)(entry->getReply()->date - squid_curtime)); - - if (skew > 86400) - debug(11, 3) ("%s's clock is skewed by %d seconds!\n", - request->host, skew); - } - ctx_exit(ctx); #if HEADERS_LOG headersLog(1, 0, request->method, entry->getReply()); #endif - - if (eof == 1) { - fwdComplete(fwd); - comm_close(fd); - } } HttpStateData::ConnectionStatus @@ -901,16 +925,18 @@ { HttpReply const *reply = entry->getReply(); int clen; - debug(11, 3) ("httpPconnTransferDone: FD %d\n", fd); + debug(11, 3) ("persistentConnStatus: FD %d\n", fd); ConnectionStatus result = statusIfComplete(); - debug(11, 5) ("httpPconnTransferDone: content_length=%d\n", + debug(11, 5) ("persistentConnStatus: content_length=%d\n", reply->content_length); /* If we haven't seen the end of reply headers, we are not done */ +debug(0,0)("persistentConnStatus: reply_hdr_state=%d\n", reply_hdr_state); if (reply_hdr_state < 2) return INCOMPLETE_MSG; clen = httpReplyBodySize(request->method, reply); +debug(0,0)("persistentConnStatus: clen=%d\n", clen); /* If there is no message body, we can be persistent */ if (0 == clen) @@ -921,7 +947,11 @@ return INCOMPLETE_MSG; /* If the body size is known, we must wait until we've gotten all of it. */ - if (entry->mem_obj->endOffset() < reply->content_length + reply->hdr_sz) + /* old technique: + * if (entry->mem_obj->endOffset() < reply->content_length + reply->hdr_sz) */ +debug(0,0)("persistentConnStatus: body_bytes_read=%d, content_length=%d\n", + body_bytes_read, reply->content_length); + if (body_bytes_read < reply->content_length) return INCOMPLETE_MSG; /* We got it all */ @@ -1029,7 +1059,7 @@ do_next_read = 0; comm_close(fd); } - } else if (flag == COMM_OK && len == 0 && entry->isEmpty()) { + } else if (flag == COMM_OK && len == 0 && body_bytes_read == 0) { ErrorState *err; err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_BAD_GATEWAY); err->xerrno = errno; @@ -1066,6 +1096,9 @@ fwdFail(fwd, err); fwd->flags.dont_retry = 1; } else { +#if ICAP_CLIENT + icap->doneSending(); +#endif fwdComplete(fwd); } @@ -1135,6 +1168,7 @@ tempBuffer.length = len - end; tempBuffer.offset = currentOffset; currentOffset += tempBuffer.length; + body_bytes_read += tempBuffer.length; #if ICAP_CLIENT icap->sendMoreData (tempBuffer); #else @@ -1148,6 +1182,7 @@ tempBuffer.length = len; tempBuffer.offset = currentOffset; currentOffset += len; + body_bytes_read += tempBuffer.length; #if ICAP_CLIENT icap->sendMoreData (tempBuffer); #else @@ -1166,6 +1201,7 @@ switch (persistentConnStatus()) { case INCOMPLETE_MSG: +debug(0,0)("processReplyData: INCOMPLETE_MSG\n"); /* Wait for more data or EOF condition */ if (flags.keepalive_broken) { @@ -1178,6 +1214,7 @@ break; case COMPLETE_PERSISTENT_MSG: +debug(0,0)("processReplyData: COMPLETE_PERSISTENT_MSG\n"); /* yes we have to clear all these! */ commSetTimeout(fd, -1, NULL, NULL); do_next_read = 0; @@ -1194,18 +1231,26 @@ pconnPush(fd, request->host, request->port, NULL); } + +#if ICAP_CLIENT + icap->doneSending(); +#endif fwdComplete(fwd); fd = -1; httpStateFree(fd, this); return; case COMPLETE_NONPERSISTENT_MSG: +debug(0,0)("processReplyData: COMPLETE_NONPERSISTENT_MSG\n"); /* close the connection ourselves */ /* yes - same as for a complete persistent conn here */ commSetTimeout(fd, -1, NULL, NULL); commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); comm_remove_close_handler(fd, httpStateFree, this); fwdUnregister(fd, fwd); +#if ICAP_CLIENT + icap->doneSending(); +#endif fwdComplete(fwd); /* TODO: check that fd is still open here */ comm_close (fd); @@ -1950,13 +1995,16 @@ HttpStateData::doIcap() { debug(0,0)("HttpStateData::doIcap() called\n"); - CBDATA_INIT_TYPE(ICAPAnchor); - icap = cbdataAlloc(ICAPAnchor); - (void)cbdataReference(icap); - - /* XXX */ - storeUnlockObject(entry); - entry = NULL; + icap = new ICAPAnchor; + (void) cbdataReference(icap); return 0; } + +void +HttpStateData::takeAdaptedHeaders(HttpReply *reply) +{ + storeEntryReplaceObject(entry, reply); + reply = NULL; + haveParsedReplyHeaders(); +} #endif Index: squid3/src/http.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/http.h,v retrieving revision 1.11.4.1 retrieving revision 1.11.4.2 diff -u -r1.11.4.1 -r1.11.4.2 --- squid3/src/http.h 17 Aug 2005 22:01:27 -0000 1.11.4.1 +++ squid3/src/http.h 18 Aug 2005 22:41:02 -0000 1.11.4.2 @@ -1,6 +1,6 @@ /* - * $Id: http.h,v 1.11.4.1 2005/08/17 22:01:27 dwsquid Exp $ + * $Id: http.h,v 1.11.4.2 2005/08/18 22:41:02 dwsquid Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -50,6 +50,9 @@ IOCB readReply; void maybeReadData(); int cacheableReply(); +#if ICAP_CLIENT + void takeAdaptedHeaders(HttpReply *); +#endif StoreEntry *entry; HttpRequest *request; @@ -65,6 +68,7 @@ off_t currentOffset; int do_next_read; size_t read_sz; + int body_bytes_read; /* to find end of response, independent of StoreEntry */ char buf[SQUID_TCP_SO_RCVBUF]; bool ignoreCacheControl; bool surrogateNoStore; @@ -82,7 +86,12 @@ ConnectionStatus statusIfComplete() const; ConnectionStatus persistentConnStatus() const; void failReply (HttpReply *reply, http_status const &status); + void keepaliveAccounting(HttpReply *); + void checkDateSkew(HttpReply *); + void haveParsedReplyHeaders(); +#if ICAP_CLIENT int doIcap(); +#endif }; #endif /* SQUID_HTTP_H */