--------------------- PatchSet 6056 Date: 2003/11/28 13:47:10 Author: rhorstmann Branch: icap-2_5 Tag: (none) Log: Implemented bypass for reqmod. When an icap service is not reachable it is marked as unreachable. The unreachable flag is checked in all (reqmod-) request to either bypass the icap service or send an error. The interval of options requests during service failure is set to the new configuration option icap_check_interval. Only a successfull options request resets the unreachable flag. Members: src/MemBuf.c:1.5.44.2->1.5.44.3 src/String.c:1.4->1.4.78.1 src/cf.data.pre:1.49.2.33.2.13->1.49.2.33.2.14 src/client_side.c:1.47.2.28.2.16->1.47.2.28.2.17 src/defines.h:1.15.6.3->1.15.6.3.6.1 src/icap_opt.c:1.1.2.7->1.1.2.8 src/icap_reqmod.c:1.1.2.23->1.1.2.24 src/icap_respmod.c:1.1.2.26->1.1.2.27 src/protos.h:1.41.6.13.2.19->1.41.6.13.2.20 src/structs.h:1.48.2.9.2.22->1.48.2.9.2.23 Index: squid/src/MemBuf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/MemBuf.c,v retrieving revision 1.5.44.2 retrieving revision 1.5.44.3 diff -u -r1.5.44.2 -r1.5.44.3 --- squid/src/MemBuf.c 6 Nov 2003 22:26:28 -0000 1.5.44.2 +++ squid/src/MemBuf.c 28 Nov 2003 13:47:10 -0000 1.5.44.3 @@ -1,6 +1,6 @@ /* - * $Id: MemBuf.c,v 1.5.44.2 2003/11/06 22:26:28 dwsquid Exp $ + * $Id: MemBuf.c,v 1.5.44.3 2003/11/28 13:47:10 rhorstmann Exp $ * * DEBUG: section 59 auto-growing Memory Buffer with printf * AUTHOR: Alex Rousskov @@ -367,13 +367,13 @@ } int -memBufRead(int fd, MemBuf *mb) +memBufRead(int fd, MemBuf * mb) { - int len; - if (mb->capacity == mb->size) - memBufGrow(mb, SQUID_TCP_SO_RCVBUF); - len = FD_READ_METHOD(fd, mb->buf + mb->size, mb->capacity - mb->size); - if (len) - mb->size += len; - return len; + int len; + if (mb->capacity == mb->size) + memBufGrow(mb, SQUID_TCP_SO_RCVBUF); + len = FD_READ_METHOD(fd, mb->buf + mb->size, mb->capacity - mb->size); + if (len) + mb->size += len; + return len; } Index: squid/src/String.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/String.c,v retrieving revision 1.4 retrieving revision 1.4.78.1 diff -u -r1.4 -r1.4.78.1 --- squid/src/String.c 12 Jan 2001 08:20:32 -0000 1.4 +++ squid/src/String.c 28 Nov 2003 13:47:11 -0000 1.4.78.1 @@ -1,6 +1,7 @@ + /* - * $Id: String.c,v 1.4 2001/01/12 08:20:32 hno Exp $ + * $Id: String.c,v 1.4.78.1 2003/11/28 13:47:11 rhorstmann Exp $ * * DEBUG: section 67 String * AUTHOR: Duane Wessels Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.49.2.33.2.13 retrieving revision 1.49.2.33.2.14 diff -u -r1.49.2.33.2.13 -r1.49.2.33.2.14 --- squid/src/cf.data.pre 18 Nov 2003 16:50:07 -0000 1.49.2.33.2.13 +++ squid/src/cf.data.pre 28 Nov 2003 13:47:11 -0000 1.49.2.33.2.14 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.49.2.33.2.13 2003/11/18 16:50:07 dwsquid Exp $ +# $Id: cf.data.pre,v 1.49.2.33.2.14 2003/11/28 13:47:11 rhorstmann Exp $ # # # SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -2560,7 +2560,7 @@ LOC: Config.icapcfg.onoff DEFAULT: off DOC_START - If you want to enable the icap client module, set this to on + If you want to enable the ICAP client module, set this to on. DOC_END NAME: icap_preview_enable @@ -2585,6 +2585,16 @@ basis by OPTIONS requests. DOC_END +NAME: icap_check_interval +TYPE: int +IFDEF: HS_FEAT_ICAP +LOC: Config.icapcfg.check_interval +DEFAULT: 300 +DOC_START + If an ICAP server does not respond, it gets marked as unreachable. Squid + will try again to reach it after this time. +DOC_END + NAME: icap_send_client_ip TYPE: onoff IFDEF: HS_FEAT_ICAP @@ -2601,16 +2611,16 @@ LOC: Config.icapcfg DEFAULT: none DOC_START - Defines a single icap service + Defines a single ICAP service icap_service servicename vectoring_point bypass service_url vectoring_point = reqmod_precache|reqmod_postcache|respmod_precache|respmod_postcache - this specifies, at which point of request processing the icap - service should be plugged in + This specifies at which point of request processing the ICAP + service should be plugged in. bypass = 1|0 - if set to 1 and the icap server can not be reached, the request will go - through without being processed by an icap server + If set to 1 and the ICAP server cannot be reached, the request will go + through without being processed by an ICAP server service_url = icap://servername:port/service Note: reqmod_precache and respmod_postcache is not yet implemented @@ -2642,7 +2652,7 @@ LOC: Config.icapcfg DEFAULT: none DOC_START - Redirects a request through an icap service class, depending + Redirects a request through an ICAP service class, depending on given acls icap_access classname allow|deny [!]aclname... @@ -2651,7 +2661,7 @@ this configuration file. If an access list matches, the processing stops. For an "allow" rule, the specified class is used for the request. A "deny" rule simply stops processing without using the class. You can also use the - special classname "None" there. + special classname "None". For backward compatibility, it is also possible to use services directly here. Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.47.2.28.2.16 retrieving revision 1.47.2.28.2.17 diff -u -r1.47.2.28.2.16 -r1.47.2.28.2.17 --- squid/src/client_side.c 30 Oct 2003 21:22:49 -0000 1.47.2.28.2.16 +++ squid/src/client_side.c 28 Nov 2003 13:47:16 -0000 1.47.2.28.2.17 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.47.2.28.2.16 2003/10/30 21:22:49 dwsquid Exp $ + * $Id: client_side.c,v 1.47.2.28.2.17 2003/11/28 13:47:16 rhorstmann Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -3834,6 +3834,7 @@ static int clientIcapReqMod(clientHttpRequest * http) { + ErrorState *err; if (http->flags.did_icap_reqmod) return 0; if (NULL == icapService(ICAP_SERVICE_REQMOD_PRECACHE, http->request)) @@ -3853,8 +3854,21 @@ http->start, http->conn->log_addr, (void *) http->conn); - if (NULL == http->icap_reqmod) + if (NULL == http->icap_reqmod) { return 0; + } else if (-1 == (int) http->icap_reqmod) { + /* produce error */ + http->icap_reqmod = NULL; + debug(33, 2) ("clientIcapReqMod: icap told us to send an error\n"); + http->log_type = LOG_TCP_DENIED; + err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); + err->xerrno = ETIMEDOUT; + err->request = requestLink(http->request); + err->src_addr = http->conn->peer.sin_addr; + http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + errorAppendEntry(http->entry, err); + return 1; + } cbdataLock(http->icap_reqmod); http->flags.did_icap_reqmod = 1; return 1; Index: squid/src/defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/defines.h,v retrieving revision 1.15.6.3 retrieving revision 1.15.6.3.6.1 diff -u -r1.15.6.3 -r1.15.6.3.6.1 --- squid/src/defines.h 8 Aug 2002 20:18:40 -0000 1.15.6.3 +++ squid/src/defines.h 28 Nov 2003 13:47:18 -0000 1.15.6.3.6.1 @@ -1,6 +1,7 @@ + /* - * $Id: defines.h,v 1.15.6.3 2002/08/08 20:18:40 squidadm Exp $ + * $Id: defines.h,v 1.15.6.3.6.1 2003/11/28 13:47:18 rhorstmann Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ Index: squid/src/icap_opt.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/icap_opt.c,v retrieving revision 1.1.2.7 retrieving revision 1.1.2.8 diff -u -r1.1.2.7 -r1.1.2.8 --- squid/src/icap_opt.c 25 Sep 2003 08:29:43 -0000 1.1.2.7 +++ squid/src/icap_opt.c 28 Nov 2003 13:47:18 -0000 1.1.2.8 @@ -1,6 +1,6 @@ /* - * $Id: icap_opt.c,v 1.1.2.7 2003/09/25 08:29:43 rhorstmann Exp $ + * $Id: icap_opt.c,v 1.1.2.8 2003/11/28 13:47:18 rhorstmann Exp $ * * DEBUG: section 81 Internet Content Adaptation Protocol (ICAP) Client OPTIONS * AUTHOR: Ralf Horstmann @@ -61,7 +61,6 @@ /*************************************************************/ -#define OPTIONS_DEFAULT_TTL 300 #define TIMEOUT 10 void @@ -91,6 +90,23 @@ } } +/* + * mark a service as unreachable + */ +void +icapOptSetUnreachable(icap_service * s) +{ + s->unreachable = 1; + /* + * if there is an options request scheduled, delete it and add + * it again to reset the time to the default check_interval. + */ + if (eventFind(icapOptStart, s)) { + eventDelete(icapOptStart, s); + eventAdd("icapOptStart", icapOptStart, s, Config.icapcfg.check_interval, 1); + } +} + static void icapOptStart(void *data) { @@ -108,7 +124,7 @@ "ICAP OPTIONS connection"); if (fd < 0) { debug(81, 4) ("icapConnectStart: %s\n", xstrerror()); - eventAdd("icapOptStart", icapOptStart, s, OPTIONS_DEFAULT_TTL, 1); + eventAdd("icapOptStart", icapOptStart, s, Config.icapcfg.check_interval, 1); return; } assert(s->opt == NULL); /* if not null, another options request might be running, which should not happen */ @@ -126,7 +142,7 @@ IcapOptData *i = s->opt; int valid; - debug(81, 3) ("icapOptConnectTimeout: fd=%d, service=%s\n", fd, s->uri); + debug(81, 4) ("icapOptConnectTimeout: fd=%d, service=%s\n", fd, s->uri); comm_close(fd); valid = cbdataValid(s); @@ -140,7 +156,8 @@ icapOptDataFree(i); s->opt = NULL; s->unreachable = 1; - eventAdd("icapOptStart", icapOptStart, s, OPTIONS_DEFAULT_TTL, 1); + debug(81, 8) ("icapOptConnectTimeout: unreachable=1, service=%s\n", s->uri); + eventAdd("icapOptStart", icapOptStart, s, Config.icapcfg.check_interval, 1); } @@ -161,12 +178,12 @@ return; } if (status != COMM_OK) { - debug(81, 3) ("icapOptConnectDone: Connection not ok\n"); + debug(81, 3) ("icapOptConnectDone: unreachable=1, service=%s\n", s->uri); comm_close(server_fd); icapOptDataFree(i); s->opt = NULL; s->unreachable = 1; - eventAdd("icapOptStart", icapOptStart, s, OPTIONS_DEFAULT_TTL, 1); + eventAdd("icapOptStart", icapOptStart, s, Config.icapcfg.check_interval, 1); return; } debug(81, 3) ("icapOptConnectDone: Connection ok. Sending Options request for %s\n", s->name); @@ -206,10 +223,11 @@ } if (errflag) { /* cancel this for now */ + debug(81, 5) ("icapOptWriteComplete: unreachable=1, service=%s\n", s->uri); icapOptDataFree(i); s->opt = NULL; s->unreachable = 1; - eventAdd("icapOptStart", icapOptStart, s, OPTIONS_DEFAULT_TTL, 1); + eventAdd("icapOptStart", icapOptStart, s, Config.icapcfg.check_interval, 1); comm_close(fd); return; } @@ -254,23 +272,25 @@ } if (size < 0) { debug(81, 3) ("icapOptReadReply: FD %d: read failure: %s.\n", fd, xstrerror()); + debug(81, 3) ("icapOptReadReply: unreachable=1, service=%s.\n", s->uri); s->unreachable = 1; icapOptDataFree(i); s->opt = NULL; - eventAdd("icapOptStart", icapOptStart, s, OPTIONS_DEFAULT_TTL, 1); + eventAdd("icapOptStart", icapOptStart, s, Config.icapcfg.check_interval, 1); comm_close(fd); } else if (size == 0) { /* no more data, now we can parse the reply */ debug(81, 3) ("icapOptReadReply: FD %d: connection closed\n", fd); i->buf[i->offset] = '\0'; /* for string functions */ + debug(81, 3) ("icapOptReadReply: reachable=0, service=%s\n", s->uri); s->unreachable = 0; if (icapOptParseReply(s, i->buf) && s->options_ttl > 0) { debug(81, 3) ("icapOptReadReply: OPTIONS request successful. scheduling again in %d seconds\n", s->options_ttl); eventAdd("icapOptStart", icapOptStart, s, s->options_ttl, 1); } else { /* use a default ttl */ - debug(81, 3) ("icapOptReadReply: OPTIONS request not successful. scheduling again in %d seconds\n", OPTIONS_DEFAULT_TTL); - eventAdd("icapOptStart", icapOptStart, s, OPTIONS_DEFAULT_TTL, 1); + debug(81, 3) ("icapOptReadReply: OPTIONS request not successful. scheduling again in %d seconds\n", Config.icapcfg.check_interval); + eventAdd("icapOptStart", icapOptStart, s, Config.icapcfg.check_interval, 1); } icapOptDataFree(i); s->opt = NULL; Index: squid/src/icap_reqmod.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/icap_reqmod.c,v retrieving revision 1.1.2.23 retrieving revision 1.1.2.24 diff -u -r1.1.2.23 -r1.1.2.24 --- squid/src/icap_reqmod.c 26 Nov 2003 08:42:45 -0000 1.1.2.23 +++ squid/src/icap_reqmod.c 28 Nov 2003 13:47:18 -0000 1.1.2.24 @@ -1,5 +1,6 @@ + /* - * $Id: icap_reqmod.c,v 1.1.2.23 2003/11/26 08:42:45 rhorstmann Exp $ + * $Id: icap_reqmod.c,v 1.1.2.24 2003/11/28 13:47:18 rhorstmann Exp $ * * DEBUG: section 81 Internet Content Adaptation Protocol (ICAP) Client * AUTHOR: Geetha Manjunath, Hewlett Packard Company @@ -78,7 +79,7 @@ * the original function strips the qery terms (everything * after a question-mark. this code is removed here. */ -char * +static char * icapUrlCanonicalClean(const request_t * request) { LOCAL_ARRAY(char, buf, MAX_URL); @@ -582,6 +583,8 @@ if (errflag == COMM_ERR_CLOSING) return; if (errflag) { + debug(81, 3) ("icapSendReqModDone: unreachable=1, service=%s\n", icap->current_service->uri); + icapOptSetUnreachable(icap->current_service); icapEntryError(icap, ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR, errno); comm_close(fd); return; @@ -626,6 +629,8 @@ icap->current_service->hostname, icap->current_service->port, xstrerror()); + debug(81, 3) ("icapSendReqMod: unreachable=1, service=%s\n", icap->current_service->uri); + icapOptSetUnreachable(icap->current_service); icapEntryError(icap, ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE, errno); comm_close(fd); return; @@ -701,6 +706,15 @@ break; } + if (service->unreachable) { + if (service->bypass) { + debug(81, 5) ("icapReqModStart: BYPASS because service unreachable: %s\n", service->uri); + return NULL; + } else { + debug(81, 5) ("icapReqModStart: ERROR because service unreachable: %s\n", service->uri); + return -1; + } + } icap = icapAllocate(); if (!icap) { debug(81, 3) ("icapReqModStart: icapAllocate() failed\n"); Index: squid/src/icap_respmod.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/icap_respmod.c,v retrieving revision 1.1.2.26 retrieving revision 1.1.2.27 diff -u -r1.1.2.26 -r1.1.2.27 --- squid/src/icap_respmod.c 14 Nov 2003 20:10:48 -0000 1.1.2.26 +++ squid/src/icap_respmod.c 28 Nov 2003 13:47:18 -0000 1.1.2.27 @@ -1,5 +1,6 @@ + /* - * $Id: icap_respmod.c,v 1.1.2.26 2003/11/14 20:10:48 dwsquid Exp $ + * $Id: icap_respmod.c,v 1.1.2.27 2003/11/28 13:47:18 rhorstmann Exp $ * * DEBUG: section 81 Internet Content Adaptation Protocol (ICAP) Client * AUTHOR: Geetha Manjunath, Hewlett Packard Company @@ -777,7 +778,7 @@ { StoreEntry *entry = icap->respmod.entry; const request_t *request = icap->request; - debug(81,3)("icapReadReply2\n"); + debug(81, 3) ("icapReadReply2\n"); if (icap->chunk_buf.size == 0 && entry->mem_obj->inmem_hi == 0) { ErrorState *err; err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); @@ -864,7 +865,7 @@ { StoreEntry *entry = icap->respmod.entry; int fd = icap->icap_fd; - debug(81,3)("icapReadReply3\n"); + debug(81, 3) ("icapReadReply3\n"); if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* * the above storeAppend() call could ABORT this entry, Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.41.6.13.2.19 retrieving revision 1.41.6.13.2.20 diff -u -r1.41.6.13.2.19 -r1.41.6.13.2.20 --- squid/src/protos.h 14 Nov 2003 02:09:21 -0000 1.41.6.13.2.19 +++ squid/src/protos.h 28 Nov 2003 13:47:18 -0000 1.41.6.13.2.20 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.41.6.13.2.19 2003/11/14 02:09:21 dwsquid Exp $ + * $Id: protos.h,v 1.41.6.13.2.20 2003/11/28 13:47:18 rhorstmann Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -622,7 +622,7 @@ extern FREE *memBufFreeFunc(MemBuf * mb); /* puts report on MemBuf _module_ usage into mb */ extern void memBufReport(MemBuf * mb); -extern int memBufRead(int fd, MemBuf *mb); +extern int memBufRead(int fd, MemBuf * mb); extern char *mime_get_header(const char *mime, const char *header); extern char *mime_get_header_field(const char *mime, const char *name, const char *prefix); @@ -1384,6 +1384,7 @@ /* icap_opt.c */ void icapOptInit(void); void icapOptShutdown(void); +void icapOptSetUnreachable(icap_service * s); /* for debugging purposes only */ void dump_icap_config(IcapConfig * cfg); #endif Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.48.2.9.2.22 retrieving revision 1.48.2.9.2.23 diff -u -r1.48.2.9.2.22 -r1.48.2.9.2.23 --- squid/src/structs.h 18 Nov 2003 16:50:08 -0000 1.48.2.9.2.22 +++ squid/src/structs.h 28 Nov 2003 13:47:18 -0000 1.48.2.9.2.23 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.48.2.9.2.22 2003/11/18 16:50:08 dwsquid Exp $ + * $Id: structs.h,v 1.48.2.9.2.23 2003/11/28 13:47:18 rhorstmann Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -382,6 +382,7 @@ icap_class *class_head; icap_access *access_head; int preview_size; + int check_interval; int send_client_ip; };