--------------------- PatchSet 5584 Date: 2003/01/03 12:28:53 Author: geetham Branch: icap Tag: (none) Log: Updated to include respmod, reqmod and options method handling, additional config statements and so on. TODO: If both respmod reqmod are on for a request - there seems to be some problems. Also needs to be integrated with Robert Collin's clientStreams. Members: configure.in:1.52.2.2->1.52.2.3 src/Makefile.am:1.19.2.2->1.19.2.3 src/cache_cf.c:1.44.2.1->1.44.2.2 src/cbdata.c:1.15.2.2->1.15.2.3 src/cf.data.pre:1.59.2.2->1.59.2.3 src/cf_gen_defines:1.5->1.5.20.1 src/client_side.c:1.58.2.2->1.58.2.3 src/client_side_reply.c:1.13.4.1->1.13.4.2 src/client_side_request.c:1.6.12.1->1.6.12.2 src/enums.h:1.32.6.2->1.32.6.3 src/forward.c:1.15.2.2->1.15.2.3 src/http.c:1.20.10.2->1.20.10.3 src/icap.c:1.1.2.1->1.1.2.2 src/main.c:1.32.4.2->1.32.4.3 src/mem.c:1.19.2.1->1.19.2.2 src/mk-string-arrays.pl:1.2->1.2.62.1 src/protos.h:1.52.2.1->1.52.2.2 src/squid.h:1.16.2.1->1.16.2.2 src/structs.h:1.55.2.2->1.55.2.3 src/typedefs.h:1.26.4.2->1.26.4.3 Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.52.2.2 retrieving revision 1.52.2.3 diff -u -r1.52.2.2 -r1.52.2.3 --- squid/configure.in 10 Dec 2002 08:16:59 -0000 1.52.2.2 +++ squid/configure.in 3 Jan 2003 12:28:53 -0000 1.52.2.3 @@ -3,7 +3,7 @@ dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.52.2.2 2002/12/10 08:16:59 geetham Exp $ +dnl $Id: configure.in,v 1.52.2.3 2003/01/03 12:28:53 geetham Exp $ dnl dnl dnl @@ -13,7 +13,7 @@ AC_CONFIG_AUX_DIR(cfgaux) AM_INIT_AUTOMAKE(squid, 2.6-DEVEL) AM_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.52.2.2 $)dnl +AC_REVISION($Revision: 1.52.2.3 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AM_MAINTAINER_MODE @@ -459,7 +459,7 @@ [ --enable-icap-support Enable iCAP client capability], [ if test "$enableval" = "yes" ; then echo "ICAP support enabled" - AC_DEFINE(HS_FEAT_ICAP) + AC_DEFINE([HS_FEAT_ICAP],1,[Content Adaptation using ICAP]) AM_CONDITIONAL(USE_ICAP, true) fi ]) Index: squid/src/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v retrieving revision 1.19.2.2 retrieving revision 1.19.2.3 diff -u -r1.19.2.2 -r1.19.2.3 --- squid/src/Makefile.am 10 Dec 2002 08:22:54 -0000 1.19.2.2 +++ squid/src/Makefile.am 3 Jan 2003 12:28:55 -0000 1.19.2.3 @@ -7,7 +7,7 @@ # if USE_ICAP -ICAPSOURCE = icap.c +ICAPSOURCE = icap.c icap_opt.c else ICAPSOURCE = endif @@ -113,6 +113,7 @@ dns_internal.c \ htcp.c \ icap.c \ + icap_opt.c\ leakfinder.c \ snmp_core.c \ snmp_agent.c \ @@ -147,6 +148,7 @@ $(DELAY_POOL_SOURCE) \ disk.c \ $(DNSSOURCE) \ + $(ICAPSOURCE) \ enums.h \ errorpage.c \ ETag.c \ @@ -172,7 +174,6 @@ HttpMsg.c \ HttpReply.c \ HttpRequest.c \ - $(ICAPSOURCE) \ icmp.c \ icp_v2.c \ icp_v3.c \ Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.44.2.1 retrieving revision 1.44.2.2 diff -u -r1.44.2.1 -r1.44.2.2 --- squid/src/cache_cf.c 10 Dec 2002 08:23:16 -0000 1.44.2.1 +++ squid/src/cache_cf.c 3 Jan 2003 12:28:55 -0000 1.44.2.2 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.44.2.1 2002/12/10 08:23:16 geetham Exp $ + * $Id: cache_cf.c,v 1.44.2.2 2003/01/03 12:28:55 geetham Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -2106,6 +2106,556 @@ return bodylist.head == NULL; } +#ifdef HS_FEAT_ICAP + +/*************************************************** + * prototypes + */ +static int icap_service_process(icap_service *s); +static void icap_service_init(icap_service *s); +static void icap_service_destroy(icap_service *s); +icap_service* icap_service_lookup(char *name); +static int icap_class_process(icap_class *c); +static void icap_class_destroy(icap_class *c); +static void icap_access_destroy(icap_access* a); +static void dump_wordlist(StoreEntry * entry, const char *name, wordlist * list); +static void icap_class_add(icap_class *c); + +/*************************************************** + * icap_service + */ + +/* + * example: + * icap_service reqmode_precache 0 icap://192.168.0.1:1344/respmod + */ + +static void +parse_icap_service_type(IcapConfig * cfg) +{ + icap_service *A = NULL; + icap_service *B = NULL; + icap_service **T = NULL; + + A = cbdataAlloc(icap_service); + icap_service_init(A); + parse_string(&A->name); + parse_string(&A->type_name); + parse_ushort(&A->bypass); + parse_string(&A->uri); + debug(3,5) ("parse_icap_service_type (line %d): %s %s %d %s\n", config_lineno, A->name, A->type_name, A->bypass, A->name); + + if (icap_service_process(A)) { + /* put into linked list */ + for (B = cfg->service_head, T = &cfg->service_head; B; T = &B->next, B = B->next); + *T = A; + } else { + /* clean up structure */ + debug(3,0) ("parse_icap_service_type (line %d): skipping %s\n", config_lineno, A->name); + icap_service_destroy(A); + cbdataFree(A); + } + +} + +static void +dump_icap_service_type(StoreEntry *e, const char *name, IcapConfig cfg) +{ + icap_service *current_node = NULL; + + if (!cfg.service_head) { + storeAppendPrintf(e, "%s 0\n", name); + return; + } + + current_node = cfg.service_head; + + while (current_node) { + storeAppendPrintf(e, "%s %s %s %d %s\n", name, current_node->name, current_node->type_name, current_node->bypass, current_node->uri); + current_node = current_node->next; + } + +} + +static void +free_icap_service_type(IcapConfig * cfg) +{ + icap_service *current_node = NULL; + icap_service *next_node = NULL; + if (cfg->service_head) { + current_node = cfg->service_head; + while (current_node) { + next_node = current_node->next; + icap_service_destroy(current_node); + cbdataFree(current_node); + current_node = next_node; + } + } +} + +/* + * parse the raw string and cache some parts that are needed later + * returns 1 if everything was ok + */ +static int +icap_service_process(icap_service *s) +{ + char *start, *end; + char *tailp; + unsigned int len; + int port_in_uri; + s->type = icapServiceToType(s->type_name); + if (s->type >= ICAP_SERVICE_MAX) { + debug(3,0) ("icap_service_process (line %d): wrong service type %s\n", config_lineno, s->type_name); + return 0; + } + + if (s->type == ICAP_SERVICE_REQMOD_PRECACHE || s->type == ICAP_SERVICE_REQMOD_POSTCACHE) { + s->method = ICAP_METHOD_REQMOD; + } else if(s->type == ICAP_SERVICE_RESPMOD_PRECACHE || s->type == ICAP_SERVICE_RESPMOD_POSTCACHE) { + s->method = ICAP_METHOD_RESPMOD; + } + + debug(3,5) ("icap_service_process (line %d): type=%s\n", config_lineno, icapServiceToStr(s->type)); + if (strncmp (s->uri,"icap://", 7) != 0) { + debug(3,0) ("icap_service_process (line %d): wrong uri: %s\n", config_lineno, s->uri); + return 0; + } + start = s->uri + 7; + if ((end = strchr(start, ':')) != NULL ) { + /* ok */ + port_in_uri=1; + debug(3,5) ("icap_service_process (line %d): port given\n", config_lineno); + } else if ((end = strchr(start,'/')) != NULL ) { + /* ok */ + port_in_uri=0; + debug(3,5) ("icap_service_process (line %d): no port given\n", config_lineno); + } else { + debug(3,0) ("icap_service_process (line %d): wrong service uri: %s\n", config_lineno, s->uri); + return 0; + } + len = end - start; + s->hostname = xstrndup(start, len + 1); + s->hostname[len] = 0; + start = end; + + if (port_in_uri) { + start++; /* skip ':' */ + if ((end = strchr(start, '/')) != NULL) { + s->port = strtoul(start, &tailp, 0) % 65536; + if (tailp != end) { + debug(3,0) ("icap_service_process (line %d): wrong service uri (port could not be parsed): %s\n", config_lineno, s->uri); + return 0; + } + debug(3,5) ("icap_service_process (line %d): port=%d\n", config_lineno, s->port); + start = end; + if (start[0] != '/') { + debug(3,0) ("icap_service_process (line %d): no '/' between port and resource found in %s\n", config_lineno, s->uri); + return 0; + } + } + } else { + /* no explicit ICAP port; first ask by getservbyname or default to + hardwired port 1344 per ICAP specification section 4.2 */ + struct servent *serv = getservbyname("icap", "tcp"); + if (serv) { + s->port = htons(serv->s_port); + debug(3,5) ("icap_service_process (line %d): default port=%d getservbyname(icap,tcp)\n", config_lineno, s->port); + } else { + s->port = 1344; + debug(3,5) ("icap_service_process (line %d): default hardwired port=%d\n", config_lineno, s->port); + } + } + + start++; /* skip '/' */ + /* the rest is resource name */ + end = strchr(start, '\0'); + len = end - start; + if (len > 1024) { + debug(3,0) ("icap_service_process (line %d): long resource name (>1024), probably wrong\n", config_lineno); + } + s->resource = xstrndup(start, len + 1); + s->resource[len] = 0; + /* check bypass */ + if ( (s->bypass != 0) && (s->bypass != 1)) { + debug(3,0) ("icap_service_process (line %d): invalid bypass value\n", config_lineno); + return 0; + } + return 1; +} + +/* + * constructor + */ +static void +icap_service_init(icap_service *s) +{ + s->type = ICAP_SERVICE_MAX; /* means undefined */ + s->preview = Config.icapcfg.preview_size; + s->opt = 0; + s->istag = StringNull; + s->transfer_preview = StringNull; + s->transfer_ignore = StringNull; + s->transfer_complete = StringNull; +} + +/* + * destructor + * frees only strings, but don't touch the linked list + */ +static void +icap_service_destroy(icap_service *s) +{ + xfree(s->name); + xfree(s->uri); + xfree(s->type_name); + xfree(s->hostname); + xfree(s->resource); + assert(s->opt == 0); /* there should be no opt request running now */ + stringClean(&s->istag); + stringClean(&s->transfer_preview); + stringClean(&s->transfer_ignore); + stringClean(&s->transfer_complete); +} + +icap_service * +icap_service_lookup(char *name) +{ + icap_service *iter; + for (iter = Config.icapcfg.service_head; iter; iter = iter->next) { + if (! strcmp(name, iter->name)) { + return iter; + } + } + return NULL; +} + +/*************************************************** + * icap_service_list + */ + +static void +icap_service_list_add(icap_service_list **isl, icap_service *service) +{ + icap_service_list **iter; + icap_service_list *new; + + new = memAllocate(MEM_ICAP_SERVICE_LIST); + new->service = service; + + if (*isl) { + iter = isl; + while ((*iter)->next) + iter = &((*iter)->next); + (*iter)->next = new; + } else { + *isl = new; + } +} + +/* + * free the linked list without touching references icap_service + */ +static void +icap_service_list_destroy(icap_service_list* isl) +{ + icap_service_list *current; + icap_service_list *next; + + current = isl; + while (current) { + next = current->next; + memFree(current, MEM_ICAP_SERVICE_LIST); + current = next; + } +} + +/*************************************************** + * icap_class + */ +static void +parse_icap_class_type(IcapConfig *cfg) +{ + icap_class *s = NULL; + + s = memAllocate(MEM_ICAP_CLASS); + parse_string(&s->name); + parse_wordlist(&s->services); + + if (icap_class_process(s)) { + /* if ok, put into linked list */ + icap_class_add(s); + } else { + /* clean up structure */ + debug(3,0) ("parse_icap_class_type (line %d): skipping %s\n", config_lineno, s->name); + icap_class_destroy(s); + memFree(s, MEM_ICAP_CLASS); + } +} + +static void +dump_icap_class_type(StoreEntry *e, const char *name, IcapConfig cfg) +{ + icap_class* current_node = NULL; + LOCAL_ARRAY(char, nom, 64); + + if (!cfg.class_head) { + storeAppendPrintf(e, "%s 0\n", name); + return; + } + + current_node = cfg.class_head; + + while (current_node) { + snprintf(nom, 64, "%s %s", name, current_node->name); + dump_wordlist(e, nom, current_node->services); + current_node = current_node->next; + } +} + +static void +free_icap_class_type(IcapConfig *cfg) +{ + icap_class *current_node = NULL; + icap_class *next_node = NULL; + if (cfg->class_head) { + current_node = cfg->class_head; + while (current_node) { + next_node = current_node->next; + icap_class_destroy(current_node); + memFree(current_node, MEM_ICAP_CLASS); + current_node = next_node; + } + } +} + +/* + * process services list, return 1, if at least one service was found + */ +static int +icap_class_process(icap_class *c) +{ + icap_service_list *isl = NULL; + wordlist *iter; + icap_service *service; + /* take services list and build icap_service_list from it */ + for (iter = c->services; iter; iter = iter->next) { + service = icap_service_lookup(iter->key); + if (service) { + icap_service_list_add(&isl, service); + } else { + debug(3,0) ("icap_class_process (line %d): skipping service %s in class %s\n", config_lineno, iter->key, c->name); + } + } + + if (isl) { + c->isl = isl; + return 1; + } + return 0; +} + +/* + * search for an icap_class in the global IcapConfig + * classes with hidden-flag are skipped + */ +icap_class * +icap_class_lookup(char *name) +{ + icap_class *iter; + for (iter = Config.icapcfg.class_head; iter; iter = iter->next) { + if ((!strcmp(name, iter->name)) && (!iter->hidden)) { + return iter; + } + } + return NULL; +} + +/* + * adds an icap_class to the global IcapConfig + */ +static void +icap_class_add(icap_class *c) +{ + icap_class *cp = NULL; + icap_class **t = NULL; + IcapConfig *cfg = &Config.icapcfg; + if (c) { + for (cp = cfg->class_head, t = &cfg->class_head; cp; t = &cp->next, cp = cp->next); + *t = c; + } +} + +/* + * free allocated memory inside icap_class + */ +static void +icap_class_destroy(icap_class *c) +{ + xfree(c->name); + wordlistDestroy(&c->services); + icap_service_list_destroy(c->isl); +} + +/*************************************************** + * icap_access + */ + +/* format: icap_access {allow|deny} acl, ... */ +static void +parse_icap_access_type(IcapConfig * cfg) +{ + icap_access *A = NULL; + icap_access *B = NULL; + icap_access **T = NULL; + icap_service *s = NULL; + icap_class *c = NULL; + ushort no_class = 0; + + A = memAllocate(MEM_ICAP_ACCESS); + parse_string(&A->service_name); + + /* + * try to find a class with the given name first. if not found, search + * the services. if a service is found, create a new hidden class with + * only this service. this is for backward compatibility. + * + * the special classname All is allowed only in deny rules, because + * the class is not used there. + */ + if (!strcmp(A->service_name, "None")) { + no_class = 1; + } else { + A->class = icap_class_lookup(A->service_name); + if (!A->class) { + s = icap_service_lookup(A->service_name); + if (s) { + c = memAllocate(MEM_ICAP_CLASS); + c->name = xstrdup("(hidden)"); + c->hidden = 1; + wordlistAdd(&c->services, A->service_name); + c->isl = memAllocate(MEM_ICAP_SERVICE_LIST); + c->isl->service = s; + icap_class_add(c); + A->class = c; + } else { + debug(3,0) ("parse_icap_access_type (line %d): servicename %s not found. skipping.\n", config_lineno, A->service_name); + memFree(A, MEM_ICAP_ACCESS); + return; + } + } + } + + aclParseAccessLine(&(A->access)); + debug(3,5) ("parse_icap_access_type (line %d): %s\n", config_lineno, A->service_name); + + /* check that All class is only used in deny rule */ + if (no_class && A->access->allow) { + memFree(A, MEM_ICAP_ACCESS); + debug(3,0) ("parse_icap_access (line %d): special class 'None' only allowed in deny rule. skipping.\n", config_lineno); + return; + } + + if (A->access) { + for (B = cfg->access_head, T = &cfg->access_head; B; T = &B->next, B = B->next); + *T = A; + } else { + debug(3,0) ("parse_icap_access_type (line %d): invalid line skipped\n", config_lineno); + memFree(A, MEM_ICAP_ACCESS); + } +} + +static void +dump_icap_access_type(StoreEntry *e, const char *name, IcapConfig cfg) +{ + icap_access* current_node = NULL; + LOCAL_ARRAY(char, nom, 64); + + if (!cfg.access_head) { + storeAppendPrintf(e, "%s 0\n", name); + return; + } + + current_node = cfg.access_head; + + while (current_node) { + snprintf(nom, 64, "%s %s", name, current_node->service_name); + dump_acl_access(e, nom, current_node->access); + current_node = current_node->next; + } +} + +static void +free_icap_access_type(IcapConfig * cfg) +{ + icap_access *current_node = NULL; + icap_access *next_node = NULL; + if (cfg->access_head) { + current_node = cfg->access_head; + while (current_node) { + next_node = current_node->next; + icap_access_destroy(current_node); + memFree(current_node, MEM_ICAP_ACCESS); + current_node = next_node; + } + } +} + +/* + * destructor + * frees everything but the linked list + */ +static void +icap_access_destroy(icap_access* a) +{ + xfree(a->service_name); + aclDestroyAccessList(&a->access); +} + +/*************************************************** + * for debugging purposes only + */ +void +dump_icap_config(IcapConfig* cfg) { + icap_service* s_iter; + icap_class* c_iter; + icap_access* a_iter; + icap_service_list *isl_iter; + acl_list* l; + debug(3,0)("IcapConfig: onoff = %d\n", cfg->onoff); + debug(3,0)("IcapConfig: service_head = %d\n", (int) cfg->service_head); + debug(3,0)("IcapConfig: class_head = %d\n", (int) cfg->class_head); + debug(3,0)("IcapConfig: access_head = %d\n", (int) cfg->access_head); + + debug(3,0) ("IcapConfig: services =\n"); + for (s_iter = cfg->service_head; s_iter; s_iter = s_iter->next) { + printf(" %s: \n", s_iter->name); + printf(" bypass = %d\n", s_iter->bypass); + printf(" hostname = %s\n", s_iter->hostname); + printf(" port = %d\n", s_iter->port); + printf(" resource = %s\n", s_iter->resource); + } + debug(3,0) ("IcapConfig: classes =\n"); + for (c_iter = cfg->class_head; c_iter; c_iter = c_iter->next) { + printf(" %s: \n", c_iter->name); + printf(" services = \n"); + for (isl_iter = c_iter->isl; isl_iter; isl_iter = isl_iter->next) { + printf(" %s\n", isl_iter->service->name); + } + } + debug(3,0)("IcapConfig: access =\n"); + for (a_iter = cfg->access_head; a_iter; a_iter = a_iter->next) { + printf(" service_name = %s\n", a_iter->service_name); + printf(" access = %s", a_iter->access->allow ? "allow" : "deny"); + for (l = a_iter->access->aclList; l != NULL; l = l->next) { + printf(" %s%s", + l->op ? null_string : "!", + l->_acl->name); + } + printf("\n"); + } +} +#endif /* HS_FEAT_ICAP */ static void parse_kb_size_t(size_t * var) Index: squid/src/cbdata.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cbdata.c,v retrieving revision 1.15.2.2 retrieving revision 1.15.2.3 diff -u -r1.15.2.2 -r1.15.2.3 --- squid/src/cbdata.c 10 Dec 2002 08:23:25 -0000 1.15.2.2 +++ squid/src/cbdata.c 3 Jan 2003 12:28:56 -0000 1.15.2.3 @@ -1,6 +1,6 @@ /* - * $Id: cbdata.c,v 1.15.2.2 2002/12/10 08:23:25 geetham Exp $ + * $Id: cbdata.c,v 1.15.2.3 2003/01/03 12:28:56 geetham Exp $ * * DEBUG: section 45 Callback Data Registry * ORIGINAL AUTHOR: Duane Wessels @@ -135,6 +135,7 @@ CREATE_CBDATA(HttpStateData); #ifdef HS_FEAT_ICAP CREATE_CBDATA(IcapStateData); + CREATE_CBDATA(icap_service); #endif CREATE_CBDATA_FREE(peer, peerDestroy); CREATE_CBDATA(ps_state); Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.59.2.2 retrieving revision 1.59.2.3 diff -u -r1.59.2.2 -r1.59.2.3 --- squid/src/cf.data.pre 10 Dec 2002 08:23:27 -0000 1.59.2.2 +++ squid/src/cf.data.pre 3 Jan 2003 12:28:56 -0000 1.59.2.3 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.59.2.2 2002/12/10 08:23:27 geetham Exp $ +# $Id: cf.data.pre,v 1.59.2.3 2003/01/03 12:28:56 geetham Exp $ # # # SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -2093,7 +2093,7 @@ TYPE: acl_access LOC: Config.accessList.http DEFAULT: none -DEFAULT_IF_NONE: allow all +DEFAULT_IF_NONE: deny all DOC_START Allowing or Denying access based on defined access lists @@ -2174,7 +2174,7 @@ TYPE: acl_access LOC: Config.accessList.icp DEFAULT: none -DEFAULT_IF_NONE: allow all +DEFAULT_IF_NONE: deny all DOC_START Allowing or Denying access to the ICP port based on defined access lists @@ -2574,6 +2574,104 @@ DOC_END COMMENT_START + ICAP OPTIONS + ----------------------------------------------------------------------------- +COMMENT_END + +NAME: icap_enable +TYPE: onoff +IFDEF: HS_FEAT_ICAP +COMMENT: on|off +LOC: Config.icapcfg.onoff +DEFAULT: off +DOC_START + If you want to enable the icap client module, set this to on +DOC_END + +NAME: icap_preview_size +TYPE: int +IFDEF: HS_FEAT_ICAP +LOC: Config.icapcfg.preview_size +DEFAULT: -1 +DOC_START + The default size of preview data to be sent to the ICAP server. + -1 means no preview. This value might be overwritten on a per server + basis by OPTIONS requests. +DOC_END + +NAME: icap_send_client_ip +TYPE: onoff +IFDEF: HS_FEAT_ICAP +COMMENT: on|off +LOC: Config.icapcfg.send_client_ip +DEFAULT: off +DOC_START + This adds the header "X-Client-IP" to ICAP requests. +DOC_END + +NAME: icap_service +TYPE: icap_service_type +IFDEF: HS_FEAT_ICAP +LOC: Config.icapcfg +DEFAULT: none +DOC_START + Defines a single icap service + + icap_service servicename vectoring_point bypass service_url + + vectoring_point = reqmod_precache|reqmod_postcache|respmod_precache|respmod_postcache + this specifies, at which point of request processing the icap + service should be plugged in + bypass = 1|0 + if set to 1 and the icap server can not be reached, the request will go + through without being processed by an icap server + service_url = icap://servername:port/service + +Example: +icap_service service_1 reqmod_precache 0 icap://icap1.mydomain.net:1344/reqmod +icap_service service_2 respmod_precache 0 icap://icap2.mydomain.net:1344/respmod +DOC_END + +NAME: icap_class +TYPE: icap_class_type +IFDEF: HS_FEAT_ICAP +LOC: Config.icapcfg +DEFAULT: none +DOC_START + Defines an ICAP service chain. If there are multiple services per + vectoring point, they are processed in the specified order. + + icap_class classname servicename... + +Example: +icap_class class_1 service_1 service_2 +icap class class_2 service_1 service_3 +DOC_END + +NAME: icap_access +TYPE: icap_access_type +IFDEF: HS_FEAT_ICAP +LOC: Config.icapcfg +DEFAULT: none +DOC_START + Redirects a request through an icap service class, depending + on given acls + + icap_access classname allow|deny [!]aclname... + + The icap_access statements are processed in the order they appear in + this configuration file. If an access list matches, the processing stops. + For an "allow" rule, the specified class is used for the request. A "deny" + rule simply stops processing without using the class. You can also use the + special classname "None" there. + + For backward compatibility, it is also possible to use services + directly here. +Example: +icap_access class_1 allow all +DOC_END + +COMMENT_START MISCELLANEOUS ----------------------------------------------------------------------------- COMMENT_END @@ -3944,13 +4042,4 @@ Squid will not service requests for some amount of time until all the child processes have been started. DOC_END - -NAME: icap_mode -TYPE: string -LOC: Config.icap_mode -DEFAULT: reqmod -DOC_START - Specify the mode of functioning for the ICAP client. - Can be one of "reqmod" or "respmod" -DOC_END EOF Index: squid/src/cf_gen_defines =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf_gen_defines,v retrieving revision 1.5 retrieving revision 1.5.20.1 diff -u -r1.5 -r1.5.20.1 --- squid/src/cf_gen_defines 3 Dec 2001 08:03:21 -0000 1.5 +++ squid/src/cf_gen_defines 3 Jan 2003 12:28:57 -0000 1.5.20.1 @@ -18,6 +18,7 @@ define["USE_UNLINKD"]="--enable-unlinkd" define["USE_USERAGENT_LOG"]="--enable-useragent-log" define["USE_WCCP"]="--enable-wccp" + define["HS_FEAT_ICAP"]="--enable-icap-support" } /^IFDEF:/ { if (define[$2] != "") Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.58.2.2 retrieving revision 1.58.2.3 diff -u -r1.58.2.2 -r1.58.2.3 --- squid/src/client_side.c 10 Dec 2002 08:23:33 -0000 1.58.2.2 +++ squid/src/client_side.c 3 Jan 2003 12:28:57 -0000 1.58.2.3 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.58.2.2 2002/12/10 08:23:33 geetham Exp $ + * $Id: client_side.c,v 1.58.2.3 2003/01/03 12:28:57 geetham Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -65,32 +65,6 @@ #else #define STATIC static #endif - -#if IPF_TRANSPARENT -#if HAVE_SYS_IOCTL_H -#include -#endif -#include -#include -#if HAVE_IP_FIL_COMPAT_H -#include -#elif HAVE_NETINET_IP_FIL_COMPAT_H -#include -#elif HAVE_IP_COMPAT_H -#include -#elif HAVE_NETINET_IP_COMPAT_H -#include -#endif -#if HAVE_IP_FIL_H -#include -#elif HAVE_NETINET_IP_FIL_H -#include -#endif -#if HAVE_IP_NAT_H -#include -#elif HAVE_NETINET_IP_NAT_H -#include -#endif #if LINGERING_CLOSE #define comm_close comm_lingering_close #endif @@ -142,24 +116,16 @@ static CWCB clientWriteBodyComplete; STATIC PF clientReadRequest; STATIC PF connStateFree; -STATIC PF requestTimeout; +static PF requestTimeout; static PF clientLifetimeTimeout; static void checkFailureRatio(err_type, hier_code); -static void clientProcessMiss(clientHttpRequest *); -static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri); -static clientHttpRequest *parseHttpRequest(ConnStateData *, method_t *, int *, char **, size_t *); -static RH clientRedirectDone; -static void clientCheckNoCache(clientHttpRequest *); -static void clientCheckNoCacheDone(int answer, void *data); -static STCB clientHandleIMSReply; -static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); -static int checkAccelOnly(clientHttpRequest *); static clientSocketContext *parseHttpRequestAbort(ConnStateData * conn, const char *uri); static clientSocketContext *parseHttpRequest(ConnStateData *, method_t *, int *, char **, size_t *); -static int clientOnlyIfCached(clientHttpRequest * http); -static STCB clientCacheHit; +#if USE_IDENT +static IDCB clientIdentDone; +#endif static CSCB clientSocketRecipient; static CSD clientSocketDetach; static void clientSetKeepaliveFlag(clientHttpRequest *); @@ -168,28 +134,6 @@ static int clientIsRequestBodyValid(int bodyLength); static int clientIsRequestBodyTooLargeForPolicy(int bodyLength); static void clientProcessBody(ConnStateData * conn); - -static int -checkAccelOnly(clientHttpRequest * http) -{ - /* return TRUE if someone makes a proxy request to us and - * we are in httpd-accel only mode */ - if (!Config2.Accel.on) - return 0; - if (Config.onoff.accel_with_proxy) - return 0; - if (http->request->protocol == PROTO_CACHEOBJ) - return 0; - if (http->flags.accel) - return 0; - if (http->request->method == METHOD_PURGE) - return 0; - return 1; -} - -#if USE_IDENT -STATIC void -clientIdentDone(const char *ident, void *data); static clientStreamNode *getTail(clientSocketContext *); static void clientSocketRemoveThisFromConnectionList(clientSocketContext *, ConnStateData *); @@ -232,10 +176,10 @@ static void connNoteUseOfBuffer(ConnStateData * conn, int byteCount); static int connKeepReadingIncompleteRequest(ConnStateData * conn); static void connCancelIncompleteRequests(ConnStateData * conn); -static ConnStateData *connStateCreate(struct sockaddr_in peer, struct sockaddr_in me, int fd); +STATIC ConnStateData *connStateCreate(struct sockaddr_in peer, struct sockaddr_in me, int fd); static clientStreamNode *getClientReplyContext(clientSocketContext * context); static int connAreAllContextsForThisConnection(ConnStateData * connState); -static void connFreeAllContexts(ConnStateData * connState); +STATIC void connFreeAllContexts(ConnStateData * connState); static void clientPullData(clientSocketContext * context); clientStreamNode * @@ -297,53 +241,12 @@ } #if USE_IDENT -STATIC void +static void clientIdentDone(const char *ident, void *data) { ConnStateData *conn = data; xstrncpy(conn->rfc931, ident ? ident : dash_str, USER_IDENT_SZ); } - -static void -clientProcessExpired(void *data) -{ - clientHttpRequest *http = data; - char *url = http->uri; - StoreEntry *entry = NULL; - debug(33, 3) ("clientProcessExpired: '%s'\n", http->uri); - assert(http->entry->lastmod >= 0); - /* - * check if we are allowed to contact other servers - * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return - * a stale entry *if* it matches client requirements - */ - if (clientOnlyIfCached(http)) { - clientProcessOnlyIfCachedMiss(http); - return; - } - http->request->flags.refresh = 1; - http->old_entry = http->entry; - http->old_sc = http->sc; - http->old_reqsize = http->reqsize; - http->old_reqofs = http->reqofs; - http->reqbuf = http->ims_reqbuf; - /* - * Assert that 'http' is already a client of old_entry. If - * it is not, then the beginning of the object data might get - * freed from memory before we need to access it. - */ -#ifndef HS_FEAT_ICAP - assert(http->sc->callback_data == http); -#endif - entry = storeCreateEntry(url, - http->log_uri, - http->request->flags, - http->request->method); - /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ - http->sc = storeClientListAdd(entry, http); -#if DELAY_POOLS - /* delay_id is already set on original store client */ - delaySetStoreClient(http->sc, delayClient(http->request)); #endif void @@ -665,6 +568,16 @@ return conn->currentobject; } +#ifdef HS_FEAT_ICAP +clientHttpRequest * +connGetCurrentHReq(ConnStateData * conn) +{ + clientSocketContext *cs = connGetCurrentContext(conn); + assert(cs); + return cs->http; +} +#endif + void contextDeferRecipientForLater(clientSocketContext * context, clientStreamNode * node, HttpReply * rep, StoreIOBuffer recievedData) { @@ -691,11 +604,6 @@ return context->http->out.size == 0 ? 1 : 0; } -/* - * filters out unwanted entries from original reply header - * adds extra entries if we have more info than origin server - * adds Squid specific entries - */ void contextSendBody(clientSocketContext * context, HttpReply * rep, StoreIOBuffer bodyData) { @@ -1259,7 +1167,7 @@ return result; } -STATIC int +static int clientReadDefer(int fdnotused, void *data) { ConnStateData *conn = data; @@ -1562,14 +1470,12 @@ } } #ifdef HS_FEAT_ICAP - /* I need to read the modified request header from icap server - - * allow me. If it has come through Accept go ahead and service.. - * else don;t */ - if (conn->me.sin_family) + /* I need to read the modified request header from icap server - + allow me. If it has come through Accept go ahead and service.. + else don;t */ + if (conn->me.sin_family) #endif - { - clientAccessCheck(http); - } + clientAccessCheck(http); continue; /* while offset > 0 && body.size_left == 0 */ } else if (parser_return_code == 0) { if (!connKeepReadingIncompleteRequest(conn)) @@ -1712,7 +1618,7 @@ } /* general lifetime handler for HTTP requests */ -STATIC void +static void requestTimeout(int fd, void *data) { #if THIS_CONFUSES_PERSISTENT_CONNECTION_AWARE_BROWSERS_AND_USERS @@ -1847,8 +1753,9 @@ commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, connState, 0); commSetDefer(fd, clientReadDefer, connState); clientdbEstablished(peer.sin_addr, 1); - assert(N); - (*N)++; + /* fix this later */ + assert(N); + (*N)++; } } @@ -2181,3 +2088,4 @@ return ch; } + Index: squid/src/client_side_reply.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/client_side_reply.c,v retrieving revision 1.13.4.1 retrieving revision 1.13.4.2 diff -u -r1.13.4.1 -r1.13.4.2 --- squid/src/client_side_reply.c 10 Dec 2002 08:23:37 -0000 1.13.4.1 +++ squid/src/client_side_reply.c 3 Jan 2003 12:28:58 -0000 1.13.4.2 @@ -1,6 +1,6 @@ /* - * $Id: client_side_reply.c,v 1.13.4.1 2002/12/10 08:23:37 geetham Exp $ + * $Id: client_side_reply.c,v 1.13.4.2 2003/01/03 12:28:58 geetham Exp $ * * DEBUG: section 88 Client-side Reply Routines * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c) @@ -77,14 +77,14 @@ static void clientProcessOnlyIfCachedMiss(clientReplyContext *); static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); -static STCB clientHandleIMSReply; +STATIC STCB clientHandleIMSReply; static int modifiedSince(StoreEntry *, request_t *); static log_type clientIdentifyStoreObject(clientReplyContext *); static void clientPurgeRequest(clientReplyContext *); static void clientTraceReply(clientStreamNode *, clientReplyContext *); static StoreEntry *clientCreateStoreEntry(clientReplyContext *, method_t, request_flags); -STATIC STCB clientSendMoreData; +static STCB clientSendMoreData; static void clientRemoveStoreReference(clientReplyContext *, store_client **, StoreEntry **); static void clientReplyContextSaveState(clientReplyContext *, @@ -1096,7 +1096,7 @@ * adds extra entries if we have more info than origin server * adds Squid specific entries */ -STATIC void +static void clientBuildReplyHeader(clientReplyContext * context, HttpReply * rep) { clientHttpRequest *http = context->http; @@ -1231,7 +1231,7 @@ } -STATIC HttpReply * +static HttpReply * clientBuildReply(clientReplyContext * context, const char *buf, size_t size) { HttpReply *rep = httpReplyCreate(); Index: squid/src/client_side_request.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/client_side_request.c,v retrieving revision 1.6.12.1 retrieving revision 1.6.12.2 diff -u -r1.6.12.1 -r1.6.12.2 --- squid/src/client_side_request.c 10 Dec 2002 08:23:38 -0000 1.6.12.1 +++ squid/src/client_side_request.c 3 Jan 2003 12:28:58 -0000 1.6.12.2 @@ -1,6 +1,6 @@ /* - * $Id: client_side_request.c,v 1.6.12.1 2002/12/10 08:23:38 geetham Exp $ + * $Id: client_side_request.c,v 1.6.12.2 2003/01/03 12:28:58 geetham Exp $ * * DEBUG: section 85 Client-side Request Routines AUTHOR: Robert Collins * (Originally Duane Wessels in client_side.c) @@ -524,6 +524,13 @@ http->request = requestLink(new_request); } clientInterpretRequestHeaders(http); +#if HS_FEAT_ICAP + /* only for testing, try to do aclCheck for icap */ + http->request->flags.do_icap = 0; + if (Config.icapcfg.onoff) + if (icapCheckAcl(http)) + http->request->flags.do_icap = 1; +#endif #if HEADERS_LOG headersLog(0, 1, request->method, request); #endif Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.32.6.2 retrieving revision 1.32.6.3 diff -u -r1.32.6.2 -r1.32.6.3 --- squid/src/enums.h 10 Dec 2002 08:23:43 -0000 1.32.6.2 +++ squid/src/enums.h 3 Jan 2003 12:28:58 -0000 1.32.6.3 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.32.6.2 2002/12/10 08:23:43 geetham Exp $ + * $Id: enums.h,v 1.32.6.3 2003/01/03 12:28:58 geetham Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -630,6 +630,12 @@ MEM_EVENT, MEM_TLV, MEM_SWAP_LOG_DATA, +#if HS_FEAT_ICAP + MEM_ICAP_OPT_DATA, + MEM_ICAP_SERVICE_LIST, + MEM_ICAP_CLASS, + MEM_ICAP_ACCESS, +#endif MEM_MAX } mem_type; @@ -726,25 +732,26 @@ CBDATA_RemovalPolicy, CBDATA_RemovalPolicyWalker, CBDATA_RemovalPurgeWalker, - CBDATA_store_client, #ifdef HS_FEAT_ICAP CBDATA_IcapStateData, + CBDATA_icap_service, #endif CBDATA_FIRST_CUSTOM_TYPE = 1000 } cbdata_type; - + /* - * ICAP states: Mainly required for Preview support for RESPMOD. + * ICAP states: Maily required for Preview support for RESPMOD. */ typedef enum { - ICAP_OK, - ICAP_RQ_DONE, /* Reqmod over */ - ICAP_PREVIEWED, /* Respmod preview over */ - ICAP_OVER, /* Last chunk has been sent */ - ICAP_RS_DONE /* Respmod over */ + ICAP_EndOfInputData = 1, /* No more data to be expected on HTTP connection */ + ICAP_PreviewDone = 2, /* Handling of preview data finished or no preview at all */ + ICAP_WaitForPreviewReply = 4, /* Expect reply from ICAP server after preview */ + ICAP_WaitForReply = 8, /* Expect final reply from ICAP server */ + ICAP_GotReply = 16, } icapState; + /* * Return codes from checkVary(request) */ @@ -799,4 +806,67 @@ #endif +#if HS_FEAT_ICAP +typedef enum { + ICAP_STATUS_NONE = 0, + ICAP_STATUS_CONTINUE = 100, + ICAP_STATUS_SWITCHING_PROTOCOLS = 101, + ICAP_STATUS_STATUS_OK = 200, + ICAP_CREATED = 201, + ICAP_STATUS_ACCEPTED = 202, + ICAP_STATUS_NON_AUTHORITATIVE_INFORMATION = 203, + ICAP_STATUS_NO_MODIFICATION_NEEDED = 204, + ICAP_STATUS_RESET_CONTENT = 205, + ICAP_STATUS_PARTIAL_CONTENT = 206, + ICAP_STATUS_MULTIPLE_CHOICES = 300, + ICAP_STATUS_MOVED_PERMANENTLY = 301, + ICAP_STATUS_MOVED_TEMPORARILY = 302, + ICAP_STATUS_SEE_OTHER = 303, + ICAP_STATUS_NOT_MODIFIED = 304, + ICAP_STATUS_USE_PROXY = 305, + ICAP_STATUS_BAD_REQUEST = 400, + ICAP_STATUS_UNAUTHORIZED = 401, + ICAP_STATUS_PAYMENT_REQUIRED = 402, + ICAP_STATUS_FORBIDDEN = 403, + ICAP_STATUS_SERVICE_NOT_FOUND = 404, + ICAP_STATUS_METHOD_NOT_ALLOWED = 405, + ICAP_STATUS_NOT_ACCEPTABLE = 406, + ICAP_STATUS_PROXY_AUTHENTICATION_REQUIRED = 407, + ICAP_STATUS_REQUEST_TIMEOUT = 408, + ICAP_STATUS_CONFLICT = 409, + ICAP_STATUS_GONE = 410, + ICAP_STATUS_LENGTH_REQUIRED = 411, + ICAP_STATUS_PRECONDITION_FAILED = 412, + ICAP_STATUS_REQUEST_ENTITY_TOO_LARGE = 413, + ICAP_STATUS_REQUEST_URI_TOO_LARGE = 414, + ICAP_STATUS_UNSUPPORTED_MEDIA_TYPE = 415, + ICAP_STATUS_INTERNAL_SERVER_ERROR = 500, + ICAP_STATUS_NOT_IMPLEMENTED = 501, + ICAP_STATUS_BAD_GATEWAY = 502, + ICAP_STATUS_SERVICE_OVERLOADED = 503, + ICAP_STATUS_GATEWAY_TIMEOUT = 504, + ICAP_STATUS_ICAP_VERSION_NOT_SUPPORTED = 505, + ICAP_STATUS_INVALID_HEADER = 600 +} icap_status; + +/* + * these values are used as index in an array, so it seems to be better to + * assign some numbers + */ +typedef enum { + ICAP_SERVICE_REQMOD_PRECACHE = 0, + ICAP_SERVICE_REQMOD_POSTCACHE = 1, + ICAP_SERVICE_RESPMOD_PRECACHE = 2, + ICAP_SERVICE_RESPMOD_POSTCACHE = 3, + ICAP_SERVICE_MAX = 4 +} icap_service_t; + +typedef enum { + ICAP_METHOD_NONE, + ICAP_METHOD_OPTION, + ICAP_METHOD_REQMOD, + ICAP_METHOD_RESPMOD +} icap_method_t; +#endif /* HS_FEAT_ICAP */ + #endif /* SQUID_ENUMS_H */ Index: squid/src/forward.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/forward.c,v retrieving revision 1.15.2.2 retrieving revision 1.15.2.3 diff -u -r1.15.2.2 -r1.15.2.3 --- squid/src/forward.c 10 Dec 2002 08:23:44 -0000 1.15.2.2 +++ squid/src/forward.c 3 Jan 2003 12:28:59 -0000 1.15.2.3 @@ -1,6 +1,6 @@ /* - * $Id: forward.c,v 1.15.2.2 2002/12/10 08:23:44 geetham Exp $ + * $Id: forward.c,v 1.15.2.3 2003/01/03 12:28:59 geetham Exp $ * * DEBUG: section 17 Request Forwarding * AUTHOR: Duane Wessels @@ -42,10 +42,9 @@ #else #define STATIC static #endif - static PSC fwdStartComplete; static void fwdDispatch(FwdState *); -static void fwdConnectStart(void *); /* should be same as EVH */ +STATIC void fwdConnectStart(void *); /* should be same as EVH */ STATIC void fwdStateFree(FwdState * fwdState); STATIC PF fwdConnectTimeout; static PF fwdServerClosed; @@ -324,7 +323,7 @@ return aclMapTOS(Config.accessList.outgoing_tos, &ch); } -static void +STATIC void fwdConnectStart(void *data) { FwdState *fwdState = data; @@ -687,6 +686,13 @@ fwdUnregister(int fd, FwdState * fwdState) { debug(17, 3) ("fwdUnregister: %s\n", storeUrl(fwdState->entry)); +#if defined (HS_FEAT_ICAP) && defined (ICAP_CHUNKED) + /* ugly hack to get not so much assertions + * problem is, that fd might be different from fwd fd, because + * of the icap connection + */ + fd = fwdState->server_fd; +#endif assert(fd == fwdState->server_fd); assert(fd > -1); comm_remove_close_handler(fd, fwdServerClosed, fwdState); Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.20.10.2 retrieving revision 1.20.10.3 diff -u -r1.20.10.2 -r1.20.10.3 --- squid/src/http.c 10 Dec 2002 08:23:47 -0000 1.20.10.2 +++ squid/src/http.c 3 Jan 2003 12:28:59 -0000 1.20.10.3 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.20.10.2 2002/12/10 08:23:47 geetham Exp $ + * $Id: http.c,v 1.20.10.3 2003/01/03 12:28:59 geetham Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -62,7 +62,7 @@ static void httpMakePublic(StoreEntry *); static int httpCachableReply(HttpStateData *); static void httpMaybeRemovePublic(StoreEntry *, http_status); -static mb_size_t httpBuildRequestPrefix(request_t * request, +STATIC mb_size_t httpBuildRequestPrefix(request_t * request, request_t * orig_request, StoreEntry * entry, MemBuf * mb, @@ -526,6 +526,13 @@ */ if (!reply->keep_alive) return 0; +#if defined (HS_FEAT_ICAP) && defined (ICAP_CHUNKED) + if (httpState->icap && httpState->icap->icap_fd == httpState->fd && httpState->icap->chunk_size == -2) { + /* zero end chunk reached */ + debug(11, 5) ("httpPconnTransferDone: zero end chunk on ICAP connection reached\n"); + return 1; + } +#endif debug(11, 5) ("httpPconnTransferDone: content_length=%d\n", reply->content_length); /* If we haven't seen the end of reply headers, we are not done */ @@ -546,8 +553,6 @@ if (mem->inmem_hi < reply->content_length + reply->hdr_sz) return 0; /* We got it all */ - debug(11, 3) ("GEE:httpPconnTransferDone: FD %d we got it all %d %d %d\n", - httpState->fd, mem->inmem_hi, reply->content_length, reply->hdr_sz); return 1; } @@ -558,6 +563,9 @@ httpReadReply(int fd, void *data) { HttpStateData *httpState = data; +#ifdef HS_FEAT_ICAP + IcapStateData *icap; +#endif LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); StoreEntry *entry = httpState->entry; const request_t *request = httpState->request; @@ -673,8 +681,31 @@ EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); } } - storeAppend(entry, buf, len); +#if defined (HS_FEAT_ICAP) && defined (ICAP_CHUNKED) + /* read data on ICAP connection in chunked encoding */ + if (httpState->icap && httpState->icap->icap_fd == fd) { + if (icapReadChunkedBody (fd, httpState, len, buf)) + return; + } else + if (fd == httpState->fd) +#endif + storeAppend(entry, buf, len); + debug(11, 5) ("httpReadReply: after storeAppend FD %d read %d\n", fd, len); + +#ifdef ICAP_CHUNKED + /* + * when headers are complete and no body is specified in encapsulated header + * we probably can finish this request + */ + icap = ICAP_FIELD(httpState); + if (icap && icap->reqmod && icap->enc_null_body >= 0 && httpState->reply_hdr_state == 2) { + fwdComplete(httpState->fwd); + comm_close(fd); + return; + } +#endif + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* * the above storeAppend() call could ABORT this entry, @@ -685,7 +716,8 @@ } else if (httpPconnTransferDone(httpState)) { /* yes we have to clear all these! */ #ifdef HS_FEAT_ICAP - if (ICAP_FIELD(httpState)) { + /* only enter respmod if we are not in reqmod mode */ + if (ICAP_FIELD(httpState) && !ICAP_FIELD(httpState)->reqmod && ICAP_FIELD(httpState)->http_fd == fd) { icapSendRespMod(httpState, buf, len, 1); return; } @@ -704,7 +736,8 @@ httpStateFree(fd, httpState); } else { #ifdef HS_FEAT_ICAP - if (ICAP_FIELD(httpState)) { + /* don't enter respmod for reqmod replies from icap server */ + if (ICAP_FIELD(httpState) && !ICAP_FIELD(httpState)->reqmod && ICAP_FIELD(httpState)->http_fd == fd) { icapSendRespMod(httpState, buf, len, 0); } else #endif @@ -743,6 +776,12 @@ return; } else { /* Schedule read reply. */ +#if defined (HS_FEAT_ICAP) + if (httpState->orig_request->flags.do_icap) { + if (startICAPrespmod(httpState) >= 0) + return; + } +#endif commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); /* * Set the read timeout here because it hasn't been set yet. @@ -1067,20 +1106,10 @@ httpState->orig_request = requestLink(orig_req); } #ifdef HS_FEAT_ICAP -#if 0 - if (Config.icap && !strcmp(Config.icap, "respmod")) { - startICAPrespmod(httpState); - } else { - if (startICAPreqmod(httpState) >= 0) + if (httpState->orig_request->flags.do_icap) { + if (startICAPreqmod(httpState) >= 0) return; - } -#else - { - extern int (*startICAPmod) (); - if (startICAPmod && startICAPmod(httpState) >= 0) - return; - } -#endif + } #endif /* * register the handler to free HTTP state data when the FD closes @@ -1171,3 +1200,4 @@ version->major = major; version->minor = minor; } + Index: squid/src/icap.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/icap.c,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/icap.c 18 Apr 2002 09:54:03 -0000 1.1.2.1 +++ squid/src/icap.c 3 Jan 2003 12:28:59 -0000 1.1.2.2 @@ -1,6 +1,5 @@ - /* - * $Id: icap.c,v 1.1.2.1 2002/04/18 09:54:03 geetham Exp $ + * $Id: icap.c,v 1.1.2.2 2003/01/03 12:28:59 geetham Exp $ * This file has the ICAP client code... * * Author: Geetha Manjunath, @@ -33,27 +32,18 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ + /******** TEMP!!!! ************/ + #define cbdataLock(cc) + #define cbdataUnlock(cc) + #define cbdataValid(cc) (1) #include "squid.h" -#define ICAP_FIELD(httpState) ((httpState)->icap) - -/* These has to be moved into the httpState data struct - * and default made configurable using squid.conf */ -/* static char * icap_server_host = "localhost"; */ -static char *icap_server_host = "15.10.45.222"; - -#define REQMOD_MODE 1 -#define RESPMOD_MODE 2 -static u_short icap_mode = REQMOD_MODE; -struct in_addr icap_server_addr; -static u_short icap_port = 1344; -/* - * static u_short icap_fd = -1; - */ +#include "clientStream.h" +#define ICAP_FIELD(httpState) (httpState)->icap static const char *const crlf = "\r\n"; -static int enable_icap_flag = 0; #define EXPECTED_ICAP_HEADER_LEN 256 +#define ICAP_OPTIONS_REQUEST PF httpReadReply; PF httpTimeout; @@ -65,24 +55,69 @@ PF connStateFree; PF requestTimeout; PF icapConnectTimeout; +extern clientHttpRequest * connGetCurrentHReq(ConnStateData * conn); void icapReadResponse(int fd, HttpStateData * httpState); void icapReqModReadReply(int fd, void *data); -void icapSendReqModDone(int fd, char *bufnotused, size_t size, int errflag, void *data); +void icapSendReqModDone(int fd, char *bufnotused, size_t size, int errflag, + void *data); void icapSendReqMod(int fd, int status, void *data); -void getICAPReqModString(MemBuf * mb, int method); -void getICAPRespModString(MemBuf * mb, int o1, int o2, int o3); +void getICAPReqModString(MemBuf * mb, int hdrLen, int bodyLen, char *service, char* client_addr); +void getICAPRespModString(MemBuf * mb, int o1, int o2, int o3, char *service, char* client_addr); int initICAP(); int startICAPrespmod(HttpStateData * httpState); int startICAPreqmod(HttpStateData * httpState); IcapStateData *startICAP(); void icapRespMod(int fd, int status, void *data); int icapSendRespMod(HttpStateData * httpState, void *buf, int len, int theEnd); -int sendRespModPreview(MemBuf * mb, HttpStateData * httpState, char *buf, ssize_t len, int theEnd); -void icapCheckContinue(HttpStateData * httpState); +int buildRespModHeader (MemBuf * mb, HttpStateData * httpState, char *buf, + ssize_t len, int theEnd); +void scheduleReadReply(int fd, void *data); +mb_size_t httpBuildRequestPrefix(request_t * request, request_t * orig_request, StoreEntry * entry, MemBuf * mb, http_state_flags); +void connFreeAllContexts(ConnStateData * connState); + +/* + * parse the contents of the encapsulated header (buffer between enc_start + * and enc_end) and put the result into IcapStateData + */ +static void +icapParseEncapsulated(IcapStateData * icap, const char *enc_start, + const char *enc_end) +{ + char *current, *end; -int (*startICAPmod) (); -#define ICAP_PREVIEW 1 + assert(icap); + assert(enc_start); + assert(enc_end); + + current = strchr(enc_start, ':'); + current++; + while (current < enc_end) { + while (isspace(*current)) + current++; + if (!strncmp(current, "res-hdr=", 8)) { + current += 8; + icap->enc_res_hdr = strtol(current, &end, 10); + } else if (!strncmp(current, "null-body=", 10)) { + current += 10; + icap->enc_null_body = strtol(current, &end, 10); + } else if (!strncmp(current, "res-body=", 9)) { + current += 9; + icap->enc_res_body = strtol(current, &end, 10); + } else { + /* invalid header */ + debug(21, 5) ("icapParseEncapsulated: error in: %s\n", current); + return; + } + current = end; + current = strchr(current, ','); + if (current == NULL) + break; + else + current++; /* skip ',' */ + } + +} /* This will be called when data is ready to be read from fd. Read until @@ -90,15 +125,16 @@ void icapReqModReadReply(int fd, void *data) { + void fwdConnectStart(void *); PF connStateFree; PF requestTimeout; HttpStateData *httpState = data; LOCAL_ARRAY(char, icap_header, SQUID_TCP_SO_RCVBUF); LOCAL_ARRAY(char, tmpbuf, SQUID_TCP_SO_RCVBUF); int read_sz, len, headlen; - char *tmp, *tmpicap; + char *tmpicap, *header_enc, *header_enc_end; StoreEntry *entry = httpState->entry; - int status, isIcap, directResponse; + int status, isIcap, directResponse = 0; float ver; ErrorState *err; @@ -111,21 +147,27 @@ * Note that I am peaking here: will read up the ICAP header part later. */ read_sz = EXPECTED_ICAP_HEADER_LEN; -/* To fix: If the len of ICAP header is exactly read_sz - we have problems */ - tmp = NULL; + /* To fix: If the len of ICAP header is exactly read_sz - we have problems */ + /* tmp = NULL; */ len = 0; headlen = 0; tmpicap = icap_header; isIcap = 0; - while (!tmp && len >= 0) { + while (!headlen && len >= 0) { + /* TODO: under some conditions we might cycle endless here, fix this */ /* len = read(fd, tmpbuf, read_sz); */ len = recv(fd, tmpbuf, read_sz, MSG_PEEK); - tmp = strstr(tmpbuf, "\r\n\r\n"); + /* tmp = strstr(tmpbuf, "\r\n\r\n"); */ if (len > 0) { - if (tmp) + headlen = headersEnd(tmpbuf, len); + if (headlen) /* End of ICAP header found - Read the \r\n\r\n too */ - len = (tmp - tmpbuf) + 4; - headlen += len; + len = headlen; + else { + /* Header is bigger */ + read_sz *=2; + if (read_sz < SQUID_TCP_SO_RCVBUF) continue; + } if (!isIcap) isIcap = !(!strstr(tmpbuf, "ICAP")); } @@ -148,60 +190,124 @@ /* Read the ICAP header only - leave the rest */ icap_header[headlen] = '\0'; debug(21, 3) ("Read icap header : <%s>\n", icap_header); - sscanf(icap_header, "ICAP/%f %d %s\r", &ver, &status, tmpbuf); + ver = -999.999; /* initalize the version to a bogus number. I + * think that we should parse it using 2 + * integers and a %d.%d scanf format - Basile + * june 2002 */ + if (sscanf(icap_header, "ICAP/%f %d %s\r", &ver, &status, tmpbuf) < 3 + || ver <= 0.0) { + debug(21, 1) ("BAD ICAP status line <%s>\n", icap_header); + /* is this correct in case of ICAP protocol error? */ + err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); + err->request = requestLink((request_t *) httpState->request); + err->xerrno = errno; + errorAppendEntry(entry, err); + comm_close(fd); + return; + }; /* printf("Version = %g, Status = %d, String = %s\n",ver,status,tmpbuf); */ /* directResponse = !(!strstr(icap_header,"res-body")); res-hdr or res-body */ + header_enc = strstr(icap_header, "Encapsulated:"); + if (header_enc) { + header_enc_end = header_enc + strcspn(header_enc, "\r\n"); + /* we should check if there is an Encapsulated - what happens + * if there is no such field? Basile + * + * it crashed. Ralf + */ + icapParseEncapsulated(ICAP_FIELD(httpState), header_enc, header_enc_end); + } directResponse = !(!strstr(icap_header, "res-")); } /* Check whether it is a direct reply - if so over to http part */ if (directResponse) { - /* printf ("ICAP server sent the response itself!\n"); */ - httpReadReply(fd, httpState); + debug(21, 3) ("icapReqModReadReply: fd %d, ICAP request satisfaction mode\n", fd); + /* got the reply, no need to come here again */ + httpState->icap->state &= ~ICAP_WaitForReply; + httpState->icap->state |= ICAP_GotReply; + /* Read the response here */ + icapReadResponse(fd, httpState); } else { + /* Read the new modified request from ICAP server and call httpSendRequest for that */ struct sockaddr_in peer, me; ConnStateData *connState = NULL; clientHttpRequest *http = NULL; + ConnStateData *connStateCreate(struct sockaddr_in peer, struct sockaddr_in me, int fd); + void icap_clientReadRequest(int fd, void *data); + memset(&peer, '\0', sizeof(struct sockaddr_in)); memset(&me, '\0', sizeof(struct sockaddr_in)); /* Code flicked from httpAccept */ - +#if 1 + connState = connStateCreate(peer, me, fd); +#else connState = cbdataAlloc(ConnStateData); connState->peer = peer; connState->log_addr = peer.sin_addr; connState->log_addr.s_addr &= Config.Addrs.client_netmask.s_addr; connState->me = me; connState->fd = fd; - connState->in.size = CLIENT_REQ_BUF_SZ; + connState->in.allocatedSize = CLIENT_REQ_BUF_SZ; /* * connState->in.buf = memAllocate(MEM_CLIENT_REQ_BUF); */ - connState->in.buf = memAllocBuf(CLIENT_REQ_BUF_SZ, &connState->in.size); + connState->in.buf = memAllocBuf(CLIENT_REQ_BUF_SZ, &connState->in.allocatedSize); +#endif /* XXX account connState->in.buf */ comm_add_close_handler(fd, connStateFree, connState); + cbdataLock(httpState); commSetTimeout(fd, Config.Timeout.request, icapConnectTimeout, connState); clientReadRequest(fd, connState); /* Now download the modified request just read */ if (connState->nrequests > 0) { - /* Completed handling */ - commSetTimeout(fd, -1, 0, connState); - http = connState->chr; - httpState->request = requestLink(http->request); - connState->nrequests = 0; - connState->chr = NULL; - - /* Change orig request's host too : See httpBuildRequestHeader - puts default Host header from orig_request */ - strcpy(httpState->orig_request->host, httpState->request->host); - - cbdataReference(httpState); - httpSendRequest(httpState); + request_t *hreq; + commSetTimeout(fd, -1, 0, connState); + http = connGetCurrentHReq(connState); + hreq = http->request; + /* + if ( ! (httpState->fwd->servers && httpState->fwd->servers->code == DIRECT)) + stringReset(&(hreq->urlpath), http->uri); + */ + httpState->request = requestLink(hreq); + connState->nrequests = 0; + /* connState->chr = NULL; */ + if (httpState->fwd->servers && httpState->fwd->servers->code == DIRECT) { + /* Direct connection to internet! Open a connection to the new host */ + httpState->fwd->request = requestLink(http->request); + httpState->fwd->server_fd = -1; + fwdConnectStart(httpState->fwd); + httpState->fd = httpState->fwd->server_fd; + comm_add_close_handler(httpState->fd, httpStateFree, httpState); + } else { + stringReset(&(hreq->urlpath), http->uri); + } + /* Change orig request's host too : See httpBuildRequestHeader + - puts default Host header from orig_request */ + strcpy(httpState->orig_request->host, httpState->request->host); + httpState->orig_request->port = httpState->request->port; + /** GEETHA: Added this to fix bugs related to header modifying reqmod filters. */ + httpState->orig_request->header = httpState->request->header; + + { /* UNTESTED CODE */ + /* connFreeAllContexts(connState); */ + extern void httpRequestFreeResources(clientHttpRequest * http); + /* + void * connGetCurrentContext(ConnStateData * conn); + extern void clientRequestContextFree(void *data); + clientRequestContextFree(connGetCurrentContext(connState)); + httpRequestFreeResources(http); + */ + connState->currentobject=NULL; + } + connStateFree(fd,connState); + httpSendRequest(httpState); } + icapStateFree(ICAP_FIELD(httpState)); + ICAP_FIELD(httpState) = NULL; } - cbdataReferenceDone(ICAP_FIELD(httpState)); - cbdataFree(ICAP_FIELD(httpState)); - ICAP_FIELD(httpState) = NULL; } /* Type should be CWCB */ @@ -235,7 +341,8 @@ return; } /* Schedule read reply. */ - commSetSelect(fd, COMM_SELECT_READ, icapReqModReadReply, cbdataReference(httpState), 0); + cbdataLock(httpState); + commSetSelect(fd, COMM_SELECT_READ, icapReqModReadReply, httpState, 0); /* * Set the read timeout here because it hasn't been set yet. * We only set the read timeout after the request has been @@ -256,18 +363,37 @@ icapSendReqMod(int fd, int status, void *data) { MemBuf mb; + MemBuf mb_hdr; HttpStateData *httpState = data; IcapStateData *icap = ICAP_FIELD(httpState); request_t *req = httpState->request; StoreEntry *entry = httpState->entry; - int cfd, blen; - void *b; + char *client_addr; + int cfd; + int blen = 0; + void *b = NULL; int icap_fd = icap->icap_fd; - char *icapHead; - /* TODO: - * POST to be tested. - */ + icap_service *service; + if (req->method == METHOD_POST) { + /* TODO: + * Data is usually missing. Sending of data has to be postponed until + * it is read. But we have to take care that some ICAP servers may not + * be able to start the ICAP response before the complete request body + * is read. In that case we do not have the request header to send to + * the HTTP server that the HTTP code needs before reading the body + * from the client. + * At least way, it could work that POSTs are sent to the ICAP server + * without body for now. + */ + ConnStateData *conn = httpState->orig_request->body_connection; + if (conn) { + blen = conn->body.bufsize; + b = conn->body.buf; + if ( ! b) + blen = 0; /* make sure this goes together */ + } + } /* cfd = client fd */ if (entry->mem_obj) cfd = entry->mem_obj->fd; @@ -276,6 +402,7 @@ assert(-1 == cfd || FD_SOCKET == fd_table[cfd].type); memBufDefInit(&mb); + memBufDefInit(&mb_hdr); /* memBufPrintf(&buf, ...); * * memBufAppend(MemBuf * mb, const char *buf, mb_size_t sz) @@ -283,82 +410,79 @@ * * -- memBufClean(&buf); * * memBufPrintf(mb, "%s %s HTTP/1.0\r\n", */ - getICAPReqModString(&mb, req->method); - memBufAppend(&mb, crlf, 2); - httpBuildRequestPrefix(req, - httpState->orig_request, - entry, - &mb, - cfd, - httpState->flags); + httpState->orig_request, entry, &mb_hdr, httpState->flags); + + service = icap->current_service; + assert(service); + client_addr = inet_ntoa(httpState->orig_request->client_addr); + + getICAPReqModString(&mb, mb_hdr.size, blen, service->uri, client_addr); + memBufAppend(&mb, "Connection: close\r\n", 19); memBufAppend(&mb, crlf, 2); + memBufAppend(&mb, mb_hdr.buf, mb_hdr.size); + memBufClean(&mb_hdr); - if (req->method == METHOD_POST) { -#if 1 - ConnStateData *conn = httpState->orig_request->body_connection; - if (conn && (blen = conn->body.bufsize) > 0 && - (b = conn->body.buf)) -#else - if ((blen = httpState->orig_request->body_sz) > 0 && - (b = httpState->orig_request->body)) -#endif - { + if (blen > 0) { memBufPrintf(&mb, "%x\r\n", blen); memBufAppend(&mb, b, blen); memBufAppend(&mb, crlf, 2); /* if (httpState->orig_request->content_length > 0 * Need to check and see if we have to put content length header */ - } } memBufAppend(&mb, crlf, 2); debug(21, 3) ("icapSendReqMod: FD %d:\n%s\n", httpState->fd, mb.buf); + cbdataLock(httpState); comm_write_mbuf(icap_fd, mb, icapSendReqModDone, httpState); } -#if 0 -int -startICAPmod(HttpStateData * httpState) -{ - if (icap_mode == REQMOD_MODE) - return startICAPreqmod(httpState); - else - return startICAPrespmod(httpState); -} -#endif - int startICAPreqmod(HttpStateData * httpState) { IcapStateData *icap; PF httpStateFree; + icap_service_list *isl_iter; + icap_service *service = NULL; + icap_service_t type; - if (ICAP_FIELD(httpState)) + if (ICAP_FIELD(httpState) + || !httpState->orig_request->flags.do_icap) return -1; - if (httpState->orig_request->flags.icap_done) - return -1; - else { - httpState->orig_request->flags.icap_done = 1; + isl_iter = httpState->orig_request->class->isl; + + while (isl_iter) { + type = isl_iter->service->type; + if (type == ICAP_SERVICE_REQMOD_PRECACHE + || type == ICAP_SERVICE_REQMOD_POSTCACHE) { + service = isl_iter->service; + break; + } + isl_iter = isl_iter->next; } - icap = startICAP(); + + if (!service) + return -1; /* no service found */ + + icap = startICAP(service->uri); if (!icap) - return -1; - ICAP_FIELD(httpState) = cbdataReference(icap); -#if 0 /* TEMP */ - EBIT_CLR(httpState->entry->flags, ENTRY_CACHABLE); -#else - /* Will be make public by httpMakePublic() later */ - storeSetPrivateKey(httpState->entry); -#endif + return -1; + + icap->current_service = service; + icap->reqmod = 1; + icap->http_fd = httpState->fd; + ICAP_FIELD(httpState) = icap; #if 1 /* register the handler to free HTTP state data when the FD closes */ comm_add_close_handler(httpState->fd, httpStateFree, httpState); #endif - commSetTimeout(icap->icap_fd, Config.Timeout.connect, icapConnectTimeout, httpState); - commConnectStart(icap->icap_fd, icap_server_host, icap_port, icapSendReqMod, httpState); + commSetTimeout(icap->icap_fd, Config.Timeout.connect, icapConnectTimeout, + httpState); + cbdataLock(httpState); + commConnectStart(icap->icap_fd, service->hostname, service->port, + icapSendReqMod, httpState); /* Calls icapSendReqMod after connection is started */ return 0; } @@ -374,39 +498,51 @@ LOCAL_ARRAY(char, icap_header, SQUID_TCP_SO_RCVBUF); LOCAL_ARRAY(char, tmpbuf, SQUID_TCP_SO_RCVBUF); int read_sz, len, headlen; - char *tmp, *tmpicap; + char *tmpicap; StoreEntry *entry = httpState->entry; - int status = 0, isIcap, directResponse; + int status = 0, isIcap, directResponse = 0; float ver; ErrorState *err; debug(21, 5) ("icapRespModReadReply: FD %d data = %p\n", fd, data); statCounter.syscalls.sock.reads++; + assert(httpState->orig_request); + /* Else it will loop */ + httpState->orig_request->flags.do_icap = 0; + read_sz = EXPECTED_ICAP_HEADER_LEN; -/* To fix: If the len of ICAP header is exactly read_sz - we have problems */ - tmp = NULL; + /* To fix: If the len of ICAP header is exactly read_sz - we have problems */ + /* tmp = NULL; */ len = 0; headlen = 0; tmpicap = icap_header; isIcap = 0; - while (!tmp && len >= 0) { + while (!headlen && len >= 0) { /* len = read(fd, tmpbuf, read_sz); */ len = recv(fd, tmpbuf, read_sz, MSG_PEEK); - tmp = strstr(tmpbuf, "\r\n\r\n"); + /* tmp = strstr(tmpbuf, "\r\n\r\n"); */ if (len > 0) { - if (tmp) + headlen = headersEnd(tmpbuf, len); + if (headlen) /* End of ICAP header found - Read the \r\n\r\n too */ - len = (tmp - tmpbuf) + 4; - headlen += len; + len = headlen; + else { + /* Header is bigger */ + read_sz *=2; + if (read_sz < SQUID_TCP_SO_RCVBUF) continue; + } if (!isIcap) isIcap = !(!strstr(tmpbuf, "ICAP")); } if (!len && !headlen) { /* No data */ - commSetSelect(fd, COMM_SELECT_READ, icapRespModReadReply, cbdataReference(httpState), 0); - commSetTimeout(fd, Config.Timeout.read, icapConnectTimeout, httpState); -/* commSetDefer(fd, fwdCheckDeferRead, entry); */ + cbdataLock(httpState); + commSetSelect(fd, COMM_SELECT_READ, icapRespModReadReply, httpState, + 0); + commSetTimeout(fd, Config.Timeout.read, icapConnectTimeout, + httpState); + /* commSetDefer(fd, fwdCheckDeferRead, entry); */ return; } } @@ -421,33 +557,55 @@ if (len > 0) { icap_header[headlen] = '\0'; debug(21, 3) ("Read icap header : <%s>\n", icap_header); - sscanf(icap_header, "ICAP/%f %d %s\r", &ver, &status, tmpbuf); + ver = -999.999; /* initalize the version to a bogus number. I + * think that we should parse it using 2 + * integers and a %d.%d scanf format - Basile + * june 2002 */ + if (sscanf(icap_header, "ICAP/%f %d %s\r", &ver, &status, + tmpbuf) < 3 || ver <= 0.0) { + debug(21, 1) ("BAD ICAP status line <%s>\n", icap_header); + /* is this correct in case of ICAP protocol error? */ + err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); + err->request = requestLink((request_t *) httpState->request); + err->xerrno = errno; + errorAppendEntry(entry, err); + comm_close(fd); + return; + }; /* printf("Version = %g, Status = %d, String = %s\n",ver,status,tmpbuf); */ } -#ifdef ICAP_PREVIEW - if (status == 100) { - icapCheckContinue(httpState); - return; - } else if (status == 204) { - IcapStateData *icap = httpState->icap; - comm_close(fd); - ICAP_FIELD(httpState) = NULL; - if (icap->state == ICAP_OVER) { - /* Reset is required to avoid duplicate stmemFreeDataUpto , - * but will I loose all info now ? */ - /* storeEntryReset(entry); */ - /* stmemFreeDataUpto(&(entry->mem_obj->data_hdr), -icap->sc); */ - fwdComplete(httpState->fwd); - } else { - commSetSelect(httpState->fd, COMM_SELECT_READ, httpReadReply, cbdataReference(httpState), 0); - } - cbdataFree(ICAP_FIELD(httpState)); - return; + + if ((httpState->icap->state & ICAP_WaitForPreviewReply) != 0) { + if (status == 100) { + debug(21, 5) ("icapRespModReadReply: 100 Continue received, httpState=%p\n", httpState); + httpState->icap->state &= ~ICAP_WaitForPreviewReply; + /* call again icapSendRespMod to handle data that was received while waiting fot this ICAP response */ + icapSendRespMod (httpState, NULL, 0, 0); + return; + } else if (status == 204) { + IcapStateData *icap = httpState->icap; + comm_close(fd); + ICAP_FIELD(httpState) = NULL; + if ((icap->state & ICAP_EndOfInputData) != 0) { + /* Reset is required to avoid duplicate stmemFreeDataUpto , + * but will I loose all info now ? */ + /* storeEntryReset(entry); */ + /* stmemFreeDataUpto(&(entry->mem_obj->data_hdr), -icap->sc); */ + fwdComplete(httpState->fwd); + } else { + cbdataLock(httpState); + commSetSelect(httpState->fd, COMM_SELECT_READ, httpReadReply, + httpState, 0); + } + icapStateFree(icap); + return; + } } -#endif - if (!headlen || len != headlen || status != 200) { + /* if (!headlen || len != headlen || status != 200) */ + if (!headlen || len != headlen || status < 200 || status > 300) { /* Did not find a proper ICAP response */ debug(21, 3) ("ICAP : Error path!\n"); + storeEntryReset(entry); err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); err->request = requestLink((request_t *) httpState->request); err->xerrno = errno; @@ -470,6 +628,9 @@ comm_close(fd); return; } + /* got the reply, no need to come here again */ + httpState->icap->state &= ~ICAP_WaitForReply; + httpState->icap->state |= ICAP_GotReply; /* Read the response here */ icapReadResponse(fd, httpState); } @@ -479,20 +640,30 @@ void icapReadResponse(int fd, HttpStateData * httpState) { - StoreEntry *entry, *nentry; + StoreEntry *entry; MemObject *mem; entry = httpState->entry; mem = entry->mem_obj; - cbdataFree(ICAP_FIELD(httpState)); + assert (httpState->icap); + debug(21, 5) ("icapReadResponse: fd %d, httpState %p, ICAP state flags %X\n", fd, httpState, httpState->icap->state); + /* do not delete the ICAP field here. + * it is needed to route data now still coming from the HTTP + * connection up to the ICAP server + icapStateFree(ICAP_FIELD(httpState)); ICAP_FIELD(httpState) = NULL; - httpState->fd = fd; + */ + httpState->fd = fd; /* now read the HTTP data from the ICAP connection */ comm_add_close_handler(fd, httpStateFree, httpState); /* stmemFreeDataUpto(&mem->data_hdr, -icap->sc); */ storeEntryReset(entry); - storeSetPublicKey(entry); - httpReadReply(fd, httpState); -/* cbdataReferenceDone(httpState); */ +#ifdef ICAP_CHUNKED + httpState->reply_hdr_state = 0; /* parse modified HTTP header */ + httpState->reply_hdr_size = 0; + httpState->icap->chunk_size = 0; /* body comes in chunked encoding */ +#endif + scheduleReadReply(fd, httpState); + cbdataUnlock(httpState); } @@ -504,55 +675,58 @@ ErrorState *err; StoreEntry *entry = httpState->entry; - if (cbdataReferenceValid(httpState)) { - if (cbdataReferenceValid(httpState->icap)) { - if (httpState->icap->state != ICAP_OVER) { - commSetSelect(httpState->fd, COMM_SELECT_READ, httpReadReply, cbdataReference(httpState), 0); - } + debug(21, 5) ("icapSendRespModDone: FD %d: size %d: errflag %d.\n", + fd, size, errflag); + if (httpState->icap) { + debug(21, 5) ("icapSendRespModDone: ICAP state flags %X\n", httpState->icap->state); } + if (cbdataValid(httpState)) { + if (httpState->icap && cbdataValid(httpState->icap)) { + if ((httpState->icap->state & ICAP_EndOfInputData) == 0) { + cbdataLock(httpState); + debug(21, 5) ("icapSendRespModDone: FD %d: commSetSelect on read httpReadReply get more orig data.\n", httpState->icap->http_fd); + commSetSelect(httpState->icap->http_fd, COMM_SELECT_READ, httpReadReply, + httpState, 0); + } + } } - debug(21, 5) ("icapSendRespModDone: FD %d: size %d: errflag %d.\n", - fd, size, errflag); if (size > 0) { - fd_bytes(fd, size, FD_WRITE); - kb_incr(&statCounter.server.all.kbytes_out, size); - kb_incr(&statCounter.server.http.kbytes_out, size); + fd_bytes(fd, size, FD_WRITE); + kb_incr(&statCounter.server.all.kbytes_out, size); + kb_incr(&statCounter.server.http.kbytes_out, size); } if (errflag == COMM_ERR_CLOSING) - return; + return; if (errflag) { - err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); - err->xerrno = errno; - if (cbdataReferenceValid(httpState)) - err->request = requestLink(httpState->request); - storeEntryReset(entry); - errorAppendEntry(entry, err); - comm_close(fd); - return; + err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); + err->xerrno = errno; + if (cbdataValid(httpState)) + err->request = requestLink(httpState->request); + storeEntryReset(entry); + errorAppendEntry(entry, err); + comm_close(fd); + return; } if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - comm_close(fd); - return; + comm_close(fd); + return; } -#ifdef ICAP_PREVIEW - if (httpState->icap->state < ICAP_PREVIEWED || - httpState->icap->state == ICAP_OVER) -#else - if (httpState->icap->state == ICAP_OVER) -#endif - { - /* Schedule read reply. */ - storeSetPrivateKey(httpState->entry); - commSetSelect(fd, COMM_SELECT_READ, icapRespModReadReply, httpState, 0); - /* - * Set the read timeout here because it hasn't been set yet. - * We only set the read timeout after the request has been - * fully written to the server-side. If we start the timeout - * after connection establishment, then we are likely to hit - * the timeout for POST/PUT requests that have very large - * request bodies. - */ - commSetTimeout(fd, Config.Timeout.read, icapConnectTimeout, httpState); + if (httpState->icap && + (httpState->icap->state & (ICAP_WaitForPreviewReply | ICAP_WaitForReply)) != 0) { + /* Schedule reading the ICAP response */ + debug(21, 3) ("icapSendRespModDone: FD %d: commSetSelect on read icapRespModReadReply.\n", fd); + commSetSelect(fd, COMM_SELECT_READ, icapRespModReadReply, httpState, 0); + cbdataLock(httpState); + if ((httpState->icap->state & (ICAP_WaitForPreviewReply | ICAP_EndOfInputData)) != 0) { + /* + * Set the read timeout only after all data has been sent + * or we are waiting for a preview response + * If the ICAP server does not return any data till all data + * has been sent, we are likely to hit the timeout for large + * HTTP bodies + */ + commSetTimeout(fd, Config.Timeout.read, icapConnectTimeout, httpState); + } } } @@ -567,160 +741,271 @@ err->xerrno = errno; err->request = requestLink(httpState->orig_request); errorAppendEntry(entry, err); + cbdataUnlock(httpState); ICAP_FIELD(httpState) = NULL; - /* cbdataReferenceDone(httpState); */ comm_close(fd); return; } - /* cbdataReferenceDone(httpState); */ + fd = httpState->fd; /* Now read from upstream server */ + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState); + /* Read fully - don't put limits on the read ahead count */ + /* Oh! I broke my head for this */ + commSetDefer(fd, NULL, NULL); } int icapSendRespMod(HttpStateData * httpState, void *buf, int len, int theEnd) { - MemBuf mb, mb_content, mb_hdr; + MemBuf mb; IcapStateData *icap = ICAP_FIELD(httpState); - request_t *req = httpState->request; - StoreEntry *entry = httpState->entry; - MemObject *mem = entry->mem_obj; - int sz, offset, contentLength; - int o2, o3; - CWCB *callback; - debug(21, 5) ("icapSendRespMod: httpState %p: len %d.\n", httpState, len); - + int size; + const int preview_size = icap->preview_size; + char ch; + debug(21, 5) ("icapSendRespMod: httpState %p: len %d, theEnd %d, icap state %X.\n", + httpState, len, theEnd, icap->state); memBufDefInit(&mb); - if (icap->sc == 0) { - /* Send the ICAP header too */ - sendRespModPreview(&mb, httpState, buf, len, theEnd); - } else if (icap->sc > 0) { - memBufPrintf(&mb, "%x\r\n", len); - memBufAppend(&mb, buf, len); - /* memBufAppend(&mb,crlf,2); */ - } - if (theEnd) { -#ifdef ICAP_PREVIEW - memBufAppend(&mb, "0; ieof\r\n", 9); -#else - memBufAppend(&mb, "0\r\n", 3); -#endif - /* stmemFreeDataUpto(&mem->data_hdr, icap->sc); */ - icap->state = ICAP_OVER; - } -#ifdef ICAP_PREVIEW - else if (icap->sc == 0) - /* we need to send this only when we reach the preview len */ - memBufAppend(&mb, "0\r\n", 3); -#endif + if (theEnd) + icap->state |= ICAP_EndOfInputData; + + if (icap->sc == 0) { + /* No data sent yet. Start with headers */ + icap->sc = buildRespModHeader (&mb, httpState, buf, len, theEnd); + buf = ((char*) buf) + icap->sc; + len -= icap->sc; + } - icap->sc += len; + if (preview_size < 0) /* preview feature off */ + icap->state |= ICAP_PreviewDone; + + if ((icap->state & ICAP_PreviewDone) == 0) { + /* preview not yet sent */ + if (icap->sc > 0 && icap->buffer.size <= preview_size && len > 0) { + /* Try to collect at least preview_size+1 bytes */ + /* By collecting one more byte than needed for preview we know best */ + /* whether we have to send the ieof chunk extension */ + size = icap->buffer.size + len; + if (size > preview_size+1) + size = preview_size+1; + size -= icap->buffer.size; + debug(21, 3) ("icapSendRespMod: FD %d: copy %d more bytes to preview buffer.\n", icap->http_fd, size); + memBufAppend (&icap->buffer, buf, size); + buf = ((char*) buf) + size; + len -= size; + } + + if (icap->buffer.size > preview_size || theEnd) { + /* we got enough bytes for preview or this is the last call */ + /* add preview preview now */ + if (icap->buffer.size > 0) { + size = icap->buffer.size; + if (size > preview_size) + size = preview_size; + memBufPrintf(&mb, "%x\r\n", size); + memBufAppend(&mb, icap->buffer.buf, size); + memBufAppend(&mb, crlf, 2); + icap->sc += size; + } + if (icap->buffer.size <= preview_size) { + /* content length is less than preview size+1 */ + memBufAppend(&mb, "0; ieof\r\n\r\n", 11); + memBufReset (&icap->buffer); /* will now be used for other data */ + } else { + memBufAppend(&mb, "0\r\n\r\n", 5); + /* end of preview, wait for continue or 204 signal */ + /* copy the extra byte and all other data to the icap buffer */ + /* so that it can be handled next time */ + ch = icap->buffer.buf[preview_size]; + memBufReset (&icap->buffer); /* will now be used for other data */ + memBufAppend(&icap->buffer, &ch, 1); + debug(21, 3) ("icapSendRespMod: FD %d: sending preview and keeping %d bytes in internal buf.\n", icap->http_fd, len+1); + if (len > 0) + memBufAppend(&icap->buffer, buf, len); + } + icap->state |= ICAP_PreviewDone | ICAP_WaitForPreviewReply; + } + } else if ((icap->state & ICAP_WaitForPreviewReply) != 0) { + /* received new data while waiting for preview response */ + /* add data to internal buffer and send later */ + debug(21, 3) ("icapSendRespMod: FD %d: add %d more bytes to internal buf while wiaiting for preview-response.\n", icap->http_fd, len); + if (len > 0) + memBufAppend (&icap->buffer, buf, len); + /* do not send any data now while waiting for preview response */ + /* but prepare for read more data on the HTTP connection */ + if ((icap->state & ICAP_EndOfInputData) == 0) { + cbdataLock (httpState); + debug(21, 3) ("icapSendRespMod: FD %d: commSetSelect on read httpReadReply waiting for preview response.\n", icap->http_fd); + commSetSelect(icap->http_fd, COMM_SELECT_READ, httpReadReply, + httpState, 0); + } + return 0; + } else { + /* after preview completed and ICAP preview response received*/ + /* there may still be some data in the buffer */ + if (icap->buffer.size > 0) { + memBufPrintf(&mb, "%x\r\n", icap->buffer.size); + memBufAppend(&mb, icap->buffer.buf, icap->buffer.size); + memBufAppend(&mb, crlf, 2); + icap->sc += icap->buffer.size; + memBufReset (&icap->buffer); + } + if (len > 0) { + memBufPrintf(&mb, "%x\r\n", len); + memBufAppend(&mb, buf, len); + memBufAppend(&mb, crlf, 2); + icap->sc += len; + } + if ((icap->state & ICAP_EndOfInputData) != 0) { + /* send zero end chunk */ + memBufAppend(&mb, "0\r\n\r\n", 5); + } + /* wait for data coming from ICAP server as soon as we sent something */ + /* but of course only until we got the response header */ + if ((icap->state & ICAP_GotReply) == 0) + icap->state |= ICAP_WaitForReply; + } commSetDefer(icap->icap_fd, NULL, NULL); commSetTimeout(icap->icap_fd, -1, NULL, NULL); + cbdataLock(icap); + cbdataLock(httpState); comm_write_mbuf(icap->icap_fd, mb, icapSendRespModDone, httpState); return 1; } int -sendRespModPreview(MemBuf * mb, HttpStateData * httpState, char *buf, ssize_t len, int theEnd) +buildRespModHeader(MemBuf * mb, HttpStateData * httpState, char *buf, + ssize_t len, int theEnd) { IcapStateData *icap = ICAP_FIELD(httpState); MemBuf mb_hdr; + char *client_addr; request_t *req = httpState->request; StoreEntry *entry = httpState->entry; int o2, o3, hlen = 0; - char *tchar; - int ret = 0; HttpReply *reply = httpState->entry->mem_obj->reply; + icap_service *service; - debug(21, 5) ("icapSendRespModPreview: httpState %p: len %d.\n", httpState, len); + debug(21, 5) ("icapbuildRespModHeader: httpState %p: len %d.\n", httpState, + len); /* Copy request header */ memBufDefInit(&mb_hdr); - httpBuildRequestPrefix(req, httpState->orig_request, entry, &mb_hdr, -1, - httpState->flags); + httpBuildRequestPrefix(req, httpState->orig_request, entry, &mb_hdr, httpState->flags); o2 = mb_hdr.size; /* Copy response header - Append to request header mbuffer */ /* I have to send the response header separately because the * response body goes with a len prefix, while buf has response as is . */ if (strlen(httpState->reply_hdr) > 0) { - memBufAppend(&mb_hdr, httpState->reply_hdr, strlen(httpState->reply_hdr)); + memBufAppend(&mb_hdr, httpState->reply_hdr, + strlen(httpState->reply_hdr)); o3 = mb_hdr.size; - tchar = strstr(buf, "\r\n\r\n"); + hlen = headersEnd(buf, len); #if DEBUG - if (len < strlen(httpState->reply_hdr) || !tchar) + if (len < strlen(httpState->reply_hdr) || !hlen) printf("Looks like there is some problem in respmod!\n"); #endif - hlen = (tchar - buf + 4); len -= hlen; } else { o3 = o2; o2 = -1; } + service = icap->current_service; + assert(service); + client_addr = inet_ntoa(httpState->orig_request->client_addr); + /* What if the buf contains only header info */ if ((len > 0 || reply->content_length > 0) || !theEnd) - getICAPRespModString(mb, 0, o2, o3); + getICAPRespModString(mb, 0, o2, o3, service->uri, client_addr); else - getICAPRespModString(mb, 0, o2, -1); -#ifdef ICAP_PREVIEW - /* Variable preview length for the time being */ - memBufPrintf(mb, "Preview: %d\r\n", len); +#ifdef ICAP_CHUNKED + getICAPRespModString(mb, 0, o2, -o3, service->uri, client_addr); +#else + getICAPRespModString(mb, 0, o2, -1, service->uri, client_addr); #endif + if (icap->preview_size >= 0 ) + memBufPrintf(mb, "Preview: %d\r\n", icap->preview_size); + memBufAppend(mb, "Connection: close\r\n", 19); memBufAppend(mb, crlf, 2); memBufAppend(mb, mb_hdr.buf, mb_hdr.size); - - /* - * if (len > Preview_length) - */ - if (len > 0) { - memBufPrintf(mb, "%x\r\n", len); - memBufAppend(mb, buf + hlen, len); - /* memBufAppend(mb, crlf, 2); */ - } memBufClean(&mb_hdr); - return ret; + + return hlen; } +/* Schedule read reply. */ void -icapCheckContinue(HttpStateData * httpState) +scheduleReadReply(int fd, void *data) { - debug(21, 5) ("icapCheckContinue: httpState %p\n", httpState); - /* Don't abort - just go ahead and read more */ - httpState->icap->state = ICAP_PREVIEWED; + HttpStateData *httpState = data; + if (httpState->orig_request->flags.do_icap) + if (startICAPrespmod(httpState) >= 0) + return; + debug(21, 3) ("scheduleReadReply: FD %d, httpState %p: call httpReadReply directly.\n", fd, httpState); + httpReadReply(fd, httpState); } + int startICAPrespmod(HttpStateData * httpState) { IcapStateData *icap; int fd; + icap_service_list *isl_iter; + icap_service *service = NULL; + icap_service_t type; - if (ICAP_FIELD(httpState)) + if (ICAP_FIELD(httpState) || + !httpState->orig_request->flags.do_icap) return -1; - icap = startICAP(); + + isl_iter = httpState->orig_request->class->isl; + + while (isl_iter) { + type = isl_iter->service->type; + if (type == ICAP_SERVICE_RESPMOD_PRECACHE + || type == ICAP_SERVICE_RESPMOD_POSTCACHE) { + service = isl_iter->service; + break; + } + isl_iter = isl_iter->next; + } + + if (!service) + return -1; /* no service found */ + + icap = startICAP(service->uri); if (!icap) return -1; - ICAP_FIELD(httpState) = cbdataReference(icap); + + icap->current_service = service; + icap->preview_size = service->preview; + ICAP_FIELD(httpState) = icap; fd = icap->icap_fd; + icap->http_fd = httpState->fd; /* Dont send to client just now */ storeBuffer(httpState->entry); -#if 0 /* TEMP */ +#if 1 /* TEMP */ EBIT_CLR(httpState->entry->flags, ENTRY_CACHABLE); #endif - commConnectStart(icap->icap_fd, icap_server_host, icap_port, icapConnectOver, httpState); - /* return 0; */ - return -1; + cbdataLock(httpState); + + commConnectStart(icap->icap_fd, service->hostname, service->port, + icapConnectOver, httpState); + commSetTimeout(fd, Config.Timeout.connect, icapConnectTimeout, httpState); + return 0; } /**********************************************************************/ IcapStateData * -startICAP() +startICAP(const char *note) { int fd; IcapStateData *icap; @@ -729,46 +1014,43 @@ * 0, COMM_NONBLOCKING, p->host); */ /* fd = comm_open(SOCK_STREAM, 0, icap_server_addr, 0, */ - if (!enable_icap_flag) + if (!Config.icapcfg.onoff) return 0; - fd = comm_open(SOCK_STREAM, 0, no_addr, 0, - COMM_NONBLOCKING, icap_server_host); + fd = comm_open(SOCK_STREAM, 0, no_addr, 0, COMM_NONBLOCKING, note); if (fd < 0) return NULL; + /* + * icap = memAllocate(MEM_ICAP_STATE_DATA); + * cbdataAdd(icap, memFree, MEM_ICAP_STATE_DATA); + */ icap = cbdataAlloc(IcapStateData); icap->icap_fd = fd; + icap->enc_res_hdr = -1; + icap->enc_null_body = -1; + icap->enc_res_body = -1; + icap->state = 0; + icap->chunk_size = -1; + memBufDefInit(&icap->buffer); debug(21, 3) ("New ICAP connection: fd %d\n", fd); return icap; } - -/* Along the lines of parse_address(struct in_addr *addr) */ -int -initICAP() +void icapStateFree(IcapStateData *icap) { - char *host = icap_server_host; - const struct hostent *hp; - - if (!enable_icap_flag) { - startICAPmod = NULL; - return 0; - } - if (Config.icap_mode && !strcmp(Config.icap_mode, "respmod")) { - icap_mode = RESPMOD_MODE; - startICAPmod = startICAPrespmod; - } else { - startICAPmod = startICAPreqmod; - icap_mode = REQMOD_MODE; - } + if (icap) { + memBufClean(&icap->buffer); + cbdataFree (icap); + } +} - if (safe_inet_addr(host, &icap_server_addr) != 1) { - if ((hp = gethostbyname(host))) /* dont use ipcache */ - icap_server_addr = inaddrFromHostent(hp); - else - fatal_dump("Cannot recognize ICAP SERVER \n"); - } +int +icapInit() +{ +#ifdef ICAP_OPTIONS_REQUEST + icapOptInit(); +#endif return 0; } @@ -780,9 +1062,9 @@ ErrorState *err; debug(17, 2) ("icapConnectTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); - if (cbdataReferenceValid(httpState)) { + if (cbdataValid(httpState)) { assert(fd == ICAP_FIELD(httpState)->icap_fd); - cbdataFree(httpState); + cbdataUnlock(httpState); } if (entry->mem_obj->inmem_hi == 0) { err = errorCon(ERR_CONNECT_FAIL, HTTP_GATEWAY_TIMEOUT); @@ -792,43 +1074,329 @@ } } -static -char *reqModString = "REQMOD icap://icap-server.net/ctc ICAP/1.0\r\nHost: icap-sever.net\r\nEncapsulated: req-hdr=0"; - -void -getICAPReqModString(MemBuf * mb, int method) +void +getICAPReqModString(MemBuf * mb, int hdrLen, int bodyLen, char *service, char* client_addr) { - memBufAppend(mb, reqModString, strlen(reqModString)); + /* memBufAppend(mb, reqModString,strlen(reqModString)); */ + memBufPrintf(mb, "REQMOD %s ICAP/1.0\r\n", service); + memBufPrintf(mb, "Encapsulated: req-hdr=0"); /* TODO: Change the offset using 'request' if needed */ - if (method == METHOD_POST) - memBufPrintf(mb, " req-body=0"); + if (bodyLen > 0) + memBufPrintf(mb, ", req-body=%d", hdrLen); + else + memBufPrintf(mb, ", null-body=%d", hdrLen); memBufAppend(mb, crlf, 2); + if (Config.icapcfg.send_client_ip) { + memBufPrintf(mb, "X-Client-IP: %s\r\n", client_addr); + } } -static -/* - * char * respModString= "RESPMOD icap://icap-server.net/respmod ICAP/1.0\r\nHost: icap-sever.net\r\nEncapsulated: req-hdr=%1d res-hdr=%1d res-body=%1d\r\n"; - */ -char *respModString = "RESPMOD icap://icap-server.net/respmod ICAP/1.0\r\nHost: icap-sever.net\r\nEncapsulated:"; - -void -getICAPRespModString(MemBuf * mb, int o1, int o2, int o3) +void +getICAPRespModString(MemBuf * mb, int o1, int o2, int o3, char *service, char* client_addr) { - memBufAppend(mb, respModString, strlen(respModString)); + /* memBufAppend(mb, respModString,strlen(respModString)); */ + memBufPrintf(mb, "RESPMOD %s ICAP/1.0\r\nEncapsulated:", service); if (o1 >= 0) memBufPrintf(mb, " req-hdr=%1d", o1); if (o2 >= 0) - memBufPrintf(mb, " res-hdr=%1d", o2); + memBufPrintf(mb, ", res-hdr=%1d", o2); if (o3 >= 0) - memBufPrintf(mb, " res-body=%1d", o3); + memBufPrintf(mb, ", res-body=%1d", o3); + else + memBufPrintf(mb, ", null-body=%1d", -o3); + memBufPrintf(mb, "\r\n"); + if (Config.icapcfg.send_client_ip) { + memBufPrintf(mb, "X-Client-IP: %s\r\n", client_addr); + } } +/* + * this is done now with a squid.conf statement + */ +#if 0 +int enable_icap(char *name) { enable_icap_flag = 1; + if (name) icap_server_host = xstrdup(name); return 0; } +#endif + + +/*************************************************** + * service_list + */ +void +icapServiceListDump(icap_service_list * isl[]) +{ + icap_service_list *isl_element; + int i; + + for (i = 0; i < ICAP_SERVICE_MAX; i++) { + isl_element = isl[i]; + /* debug(81,5) ("icapDumpServiceList: %s\n", connState->chr->uri); */ + while (isl_element) { + debug(81, 5) ("%30s * %s\n", + icapServiceToStr(isl_element->service->type), + isl_element->service->name); + isl_element = isl_element->next; + } + } +} + +void +icapServiceListFree(icap_service_list * isl[]) +{ + int i; + icap_service_list *next; + + for (i = 0; i < ICAP_SERVICE_MAX; i++) { + while (isl[i]) { + next = isl[i]->next; + memFree(isl[i], MEM_ICAP_SERVICE_LIST); + isl[i] = next; + } + } +} + +/*************************************************** + * icap_service + */ + +icap_service_t +icapServiceToType(const char *s) +{ + if (!strcmp(s, "reqmod_precache")) + return ICAP_SERVICE_REQMOD_PRECACHE; + if (!strcmp(s, "reqmod_postcache")) + return ICAP_SERVICE_REQMOD_POSTCACHE; + if (!strcmp(s, "respmod_precache")) + return ICAP_SERVICE_RESPMOD_PRECACHE; + if (!strcmp(s, "respmod_postcache")) + return ICAP_SERVICE_RESPMOD_POSTCACHE; + return ICAP_SERVICE_MAX; +} + +extern const char *icap_service_type_str[]; + +const char * +icapServiceToStr(const icap_service_t type) +{ + if (type >= 0 && type < ICAP_SERVICE_MAX) + return icap_service_type_str[type]; + else + return "error"; +} + +/* + * code copied from acl.c because we need to differentiate three cases: + * match with allow (1), match with deny (2), no match (0). + */ +int +icapCheckFast(const acl_access * A, aclCheck_t * checklist) +{ + allow_t allow = ACCESS_DENIED; + debug(81, 5) ("icapCheckFast: list: %p\n", A); + while (A) { + allow = A->allow; + if (aclMatchAclList(A->aclList, checklist)) + return allow == ACCESS_ALLOWED; + A = A->next; + } + debug(81, 5) ("icapCheckFast: no matches, returning: %d\n", + allow == ACCESS_DENIED); + return allow == ACCESS_DENIED; +} + + +/* + * check wether we do icap for a request + */ +int +icapCheckAcl(clientHttpRequest * http) +{ + icap_access *iter; + aclCheck_t icapChecklist; + + memset(&icapChecklist, '\0', sizeof(aclCheck_t)); + icapChecklist.src_addr = http->conn->peer.sin_addr; + icapChecklist.request = http->request; + for (iter = Config.icapcfg.access_head; iter; iter = iter->next) { + acl_access *A = iter->access; + if (aclMatchAclList(A->aclList, &icapChecklist)) { + debug(81, 5) ("icapCheckAcl: match for class=%s\n", + iter->class->name); + if (A->allow) { + /* allow rule, do icap and use associated class */ + http->request->class = iter->class; + return 1; + } else { + /* deny rule, stop processing */ + return 0; + } + } + } + return 0; +} + + +#if 0 +/* Made it a macro */ +getIcapService(HttpStateData * httpState, icap_service_t vpoint) +{ + icap_service_list **isl; + isl = httpState->orig_request->isl; + /* Selecting the first matching service */ + if (isl[vpoint]) + return isl[vpoint]->service; +} + +#endif + + +/* returns the amount of data until lineending ( \r or \r\n ) */ +static size_t +icapLineLength(char *start, int len) +{ + size_t lineLen = 0; + char *end = (char *) memchr(start, 0x0A, len); + if (end) { + lineLen = end - start + 1; + if (lineLen < len) { + if (end[1] == 0x0D) + lineLen++; + } + } + return lineLen; +} + +int +icapReadChunkSize(void *data, char *buf, int len) +{ + IcapStateData *icap = data; + int chunkSize = 0; + char c; + size_t start; + size_t end; + size_t nextStart = 0; + do { + start = nextStart; + if (len <= start) { + /* end of buffer, so far no lines or only empty lines, wait for more data */ + icap->chunk_size = 0; /* read chunk size with next buffer */ + return nextStart; + } + end = start + icapLineLength(buf + start, len - start); + nextStart = end; + if (end <= start) { + /* no line found, need more code here, now we are in deep trouble, buffer stops with */ + /* half a chunk size line. For now stop here. */ + icap->chunk_size = -2; // we are done + + return nextStart; + } + while (start < end) { + if (buf[start] != ' ' && buf[start] != '\t' && buf[start] != '\r' && buf[start] != '\n') + break; + start++; + } + while (start < end) { + if (buf[end - 1] != ' ' && buf[end - 1] != '\t' && buf[end - 1] != '\r' && buf[end - 1] != '\n') + break; + end--; + } + /* if now end <= start we got an empty line */ + /* The previous chunk data should stop with a CRLF. In case */ + /* that the other end does not follow the specs and sends no */ + /* CRLF or too many empty lines, just continue till we have */ + /* a non-empty line. */ + } while (end <= start); + + /* Non-empty line: Parse the chunk size */ + while (start < end) { + c = buf[start++]; + if (c >= 'a' && c <= 'f') { + chunkSize = chunkSize * 16 + c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + chunkSize = chunkSize * 16 + c - 'A' + 10; + } else if (c >= '0' && c <= '9') { + chunkSize = chunkSize * 16 + c - '0'; + } else { + if (!(c == ';' || c == ' ' || c == '\t')) { + /*Syntax error: Chunksize expected. */ + icap->chunk_size = -2; // we are done + + return nextStart; + } + /* Next comes a chunk extension */ + break; + } + } + /* if we read a zero chunk, we reached the end. Mark this for httpPconnTransferDone */ + icap->chunk_size = (chunkSize > 0) ? chunkSize : -2; + return nextStart; +} + +int +icapReadChunkedBody (int fd, void* data, int len, char* buf) +{ + HttpStateData *httpState = data; + StoreEntry *entry = httpState->entry; + int bufOffset; + + if (httpState->icap->chunk_size == -2) { + debug(21, 3) ("zero end chunk reached\n"); + len = 0; /* don't add anything else */ + } else if (httpState->reply_hdr_state == 2 && httpState->icap->chunk_size >= 0) { + /* read data which is transfered in chunked encoding */ + /* set bufOffset to beginning of body data */ + /* If header was already written to store, we can read from beginning */ + if (entry->mem_obj && entry->mem_obj->inmem_hi > 0) + bufOffset = 0; + else + bufOffset = headersEnd(httpState->reply_hdr, httpState->reply_hdr_size); + + /* Store the header if we didn't do so yet */ + assert (fd == httpState->fd); /* check the condition that the other storeAppends must fulfill */ + if (bufOffset > 0) + storeAppend(entry, buf, bufOffset); + + while (bufOffset < len) { + if (httpState->icap->chunk_size == 0) { + bufOffset += icapReadChunkSize(httpState->icap, buf + bufOffset, len - bufOffset); + debug(21, 3) ("got chunksize %d, new offset %d\n", httpState->icap->chunk_size, bufOffset); + if (httpState->icap->chunk_size == -2) { + debug(21, 3) ("zero end chunk reached\n"); + len = 0; /* don't add anything else */ + /* break; */ + /* let's try to close the connection here */ + fwdComplete(httpState->fwd); + comm_close(fd); + return 1; /* break from httpReadReply */ + } + } + if (httpState->icap->chunk_size > 0) { + if (httpState->icap->chunk_size >= len - bufOffset) { + storeAppend(entry, buf + bufOffset, len - bufOffset); + httpState->icap->chunk_size -= len - bufOffset; + bufOffset = len; + } else { + storeAppend(entry, buf + bufOffset, httpState->icap->chunk_size); + bufOffset += httpState->icap->chunk_size; + httpState->icap->chunk_size = 0; + } + } + } + } else { + if (fd == httpState->fd) + storeAppend(entry, buf, len); + } + return 0; +} + + Index: squid/src/main.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/main.c,v retrieving revision 1.32.4.2 retrieving revision 1.32.4.3 diff -u -r1.32.4.2 -r1.32.4.3 --- squid/src/main.c 10 Dec 2002 08:23:51 -0000 1.32.4.2 +++ squid/src/main.c 3 Jan 2003 12:29:00 -0000 1.32.4.3 @@ -1,6 +1,6 @@ /* - * $Id: main.c,v 1.32.4.2 2002/12/10 08:23:51 geetham Exp $ + * $Id: main.c,v 1.32.4.3 2003/01/03 12:29:00 geetham Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -111,7 +111,7 @@ extern char *optarg; int c; - while ((c = getopt(argc, argv, "CDFNRSVYXa:d:f:hi::k:m::su:vz?")) != -1) { + while ((c = getopt(argc, argv, "CDFNRSVYXa:d:f:hk:m::su:vz?")) != -1) { switch (c) { case 'C': opt_catch_signals = 0; @@ -154,11 +154,6 @@ case 'h': usage(); break; -#ifdef HS_FEAT_ICAP - case 'i': - enable_icap(optarg); - break; -#endif case 'k': if ((int) strlen(optarg) < 1) usage(); @@ -357,7 +352,7 @@ #endif redirectInit(); #ifdef HS_FEAT_ICAP - initICAP(); + icapInit(); #endif authenticateInit(&Config.authConfig); externalAclInit(); @@ -403,7 +398,7 @@ #endif redirectInit(); #ifdef HS_FEAT_ICAP - initICAP(); + icapInit(); #endif authenticateInit(&Config.authConfig); externalAclInit(); @@ -488,7 +483,7 @@ #endif redirectInit(); #ifdef HS_FEAT_ICAP - initICAP(); + icapInit(); #endif authenticateInit(&Config.authConfig); externalAclInit(); Index: squid/src/mem.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/mem.c,v retrieving revision 1.19.2.1 retrieving revision 1.19.2.2 diff -u -r1.19.2.1 -r1.19.2.2 --- squid/src/mem.c 10 Dec 2002 08:23:51 -0000 1.19.2.1 +++ squid/src/mem.c 3 Jan 2003 12:29:00 -0000 1.19.2.2 @@ -1,6 +1,6 @@ /* - * $Id: mem.c,v 1.19.2.1 2002/12/10 08:23:51 geetham Exp $ + * $Id: mem.c,v 1.19.2.2 2003/01/03 12:29:00 geetham Exp $ * * DEBUG: section 13 High Level Memory Pool Management * AUTHOR: Harvest Derived @@ -393,6 +393,13 @@ memDataInit(MEM_TLV, "storeSwapTLV", sizeof(tlv), 0); memDataInit(MEM_SWAP_LOG_DATA, "storeSwapLogData", sizeof(storeSwapLogData), 0); +#ifdef HS_FEAT_ICAP + memDataInit(MEM_ICAP_OPT_DATA, "IcapOptData", sizeof(IcapOptData), 0); + memDataInit(MEM_ICAP_SERVICE_LIST, "icap_service_list", sizeof(icap_service_list),0); + memDataInit(MEM_ICAP_CLASS, "icap_class", sizeof(icap_class), 0); + memDataInit(MEM_ICAP_ACCESS, "icap_access", sizeof(icap_access), 0); +#endif + /* init string pools */ for (i = 0; i < mem_str_pool_count; i++) { StrPools[i].pool = memPoolCreate(StrPoolsAttrs[i].name, StrPoolsAttrs[i].obj_size); Index: squid/src/mk-string-arrays.pl =================================================================== RCS file: /cvsroot/squid-sf//squid/src/mk-string-arrays.pl,v retrieving revision 1.2 retrieving revision 1.2.62.1 diff -u -r1.2 -r1.2.62.1 --- squid/src/mk-string-arrays.pl 23 Oct 2000 15:04:21 -0000 1.2 +++ squid/src/mk-string-arrays.pl 3 Jan 2003 12:29:00 -0000 1.2.62.1 @@ -1,5 +1,5 @@ #****************************************************************************** -# $Id: mk-string-arrays.pl,v 1.2 2000/10/23 15:04:21 hno Exp $ +# $Id: mk-string-arrays.pl,v 1.2.62.1 2003/01/03 12:29:00 geetham Exp $ # # File: mk-strs.pl # @@ -16,6 +16,7 @@ $pat{'icp_opcode'} = "icp_opcode_str"; $pat{'swap_log_op'} = "swap_log_op_str"; $pat{'lookup_t'} = "lookup_t_str"; +$pat{'icap_service_t'} = "icap_service_type_str"; $state = 0; # start state while (<>) { Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.52.2.1 retrieving revision 1.52.2.2 diff -u -r1.52.2.1 -r1.52.2.2 --- squid/src/protos.h 10 Dec 2002 08:23:54 -0000 1.52.2.1 +++ squid/src/protos.h 3 Jan 2003 12:29:00 -0000 1.52.2.2 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.52.2.1 2002/12/10 08:23:54 geetham Exp $ + * $Id: protos.h,v 1.52.2.2 2003/01/03 12:29:00 geetham Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1338,4 +1338,23 @@ extern void externalAclShutdown(void); extern char *strtokFile(void); +#ifdef HS_FEAT_ICAP +/* icap.c */ +int icapInit(); +const char* icapServiceToStr(const icap_service_t type); +icap_service_t icapServiceToType(const char *s); +void icapServiceListFree(icap_service_list* isl[]); +void icapServiceListDump(icap_service_list* isl[]); +int icapCheckAcl( clientHttpRequest *http ); +void icapStateFree (IcapStateData *icap); +int icapReadChunkSize(void *data, char *buf, int len); +int icapReadChunkedBody (int fd, void* data, int len, char* buf); +int icapSendRespMod(HttpStateData * httpState, void *buf, int len, int theEnd); +int startICAPrespmod(HttpStateData * httpState); +int startICAPreqmod(HttpStateData * httpState); +/* icap_opt.c */ +void icapOptInit(); +void icapOptShutdown(); +#endif + #endif /* SQUID_PROTOS_H */ Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.16.2.1 retrieving revision 1.16.2.2 diff -u -r1.16.2.1 -r1.16.2.2 --- squid/src/squid.h 10 Dec 2002 08:23:56 -0000 1.16.2.1 +++ squid/src/squid.h 3 Jan 2003 12:29:01 -0000 1.16.2.2 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.16.2.1 2002/12/10 08:23:56 geetham Exp $ + * $Id: squid.h,v 1.16.2.2 2003/01/03 12:29:01 geetham Exp $ * * AUTHOR: Duane Wessels * @@ -37,6 +37,10 @@ #include "config.h" +#ifdef HS_FEAT_ICAP +#define ICAP_CHUNKED 1 +#endif + #if PURIFY #define assert(EX) ((void)0) #elif defined(NODEBUG) @@ -412,3 +416,4 @@ #define FD_WRITE_METHOD(fd, buf, len) (*fd_table[fd].write_method)(fd, buf, len) #endif /* SQUID_H */ + Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.55.2.2 retrieving revision 1.55.2.3 diff -u -r1.55.2.2 -r1.55.2.3 --- squid/src/structs.h 10 Dec 2002 08:24:00 -0000 1.55.2.2 +++ squid/src/structs.h 3 Jan 2003 12:29:02 -0000 1.55.2.3 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.55.2.2 2002/12/10 08:24:00 geetham Exp $ + * $Id: structs.h,v 1.55.2.3 2003/01/03 12:29:02 geetham Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -373,6 +373,18 @@ wordlist *args; }; +#if HS_FEAT_ICAP +struct _IcapConfig { + int onoff; + icap_service *service_head; + /* icap_service_list *service_list_head; */ + icap_class *class_head; + icap_access *access_head; + int preview_size; + int send_client_ip; +}; +#endif /* HS_FEAT_ICAP */ + struct _SquidConfig { struct { size_t maxSize; @@ -698,6 +710,7 @@ int sleep_after_fork; /* microseconds */ #ifdef HS_FEAT_ICAP char *icap_mode; + IcapConfig icapcfg; #endif external_acl *externalAclHelperList; }; @@ -980,8 +993,69 @@ #ifdef HS_FEAT_ICAP struct _IcapStateData { int icap_fd; + int http_fd; int sc; icapState state; + icap_service *current_service; + int enc_res_hdr; + int enc_null_body; + int enc_res_body; + ushort reqmod; + MemBuf buffer; + int chunk_size; + int preview_size; +}; + +struct _icap_service { + icap_service *next; + char *name; /* name to be used when referencing ths service */ + char *uri; /* uri of server/service to use */ + char *type_name; /* {req|resp}mod_{pre|post}cache */ + + char *hostname; + unsigned short int port; + char *resource; + icap_service_t type; /* parsed type */ + icap_method_t method; + ushort bypass; /* flag: bypass allowed */ + ushort unreachable; /* flag: set to 1 if options request fails */ + IcapOptData *opt; /* temp data needed during opt request */ + struct { + unsigned int allow_204:1; + } flags; + int preview; + String istag; + String transfer_preview; + String transfer_ignore; + String transfer_complete; + int max_connections; + int options_ttl; +}; + +struct _icap_service_list { + icap_service_list* next; + icap_service* service; +}; + +struct _icap_class { + icap_class *next; + char* name; + wordlist* services; + icap_service_list *isl; + ushort hidden; /* for unnamed classes */ +}; + +struct _icap_access { + icap_access *next; + char *service_name; + icap_class *class; + acl_access *access; +}; + +struct _IcapOptData { + char *buf; + off_t offset; + size_t size; }; #endif @@ -1609,7 +1683,7 @@ unsigned int body_sent:1; unsigned int reset_tcp:1; #ifdef HS_FEAT_ICAP - unsigned int icap_done:1; + unsigned int do_icap:1; #endif }; @@ -1667,6 +1741,11 @@ char *peer_login; /* Configured peer login:password */ time_t lastmod; /* Used on refreshes */ const char *vary_headers; /* Used when varying entities are detected. Changes how the store key is calculated */ +#if HS_FEAT_ICAP + /* linked lists of pointers to icap_services to use */ + /* icap_service_list* isl[ICAP_SERVICE_MAX]; */ + icap_class *class; +#endif }; struct _cachemgr_passwd { Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.26.4.2 retrieving revision 1.26.4.3 diff -u -r1.26.4.2 -r1.26.4.3 --- squid/src/typedefs.h 10 Dec 2002 08:24:01 -0000 1.26.4.2 +++ squid/src/typedefs.h 3 Jan 2003 12:29:03 -0000 1.26.4.3 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.26.4.2 2002/12/10 08:24:01 geetham Exp $ + * $Id: typedefs.h,v 1.26.4.3 2003/01/03 12:29:03 geetham Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -109,6 +109,12 @@ typedef struct _HttpStateData HttpStateData; #ifdef HS_FEAT_ICAP typedef struct _IcapStateData IcapStateData; +typedef struct _IcapConfig IcapConfig; +typedef struct _icap_service icap_service; +typedef struct _icap_service_list icap_service_list; +typedef struct _icap_class icap_class; +typedef struct _icap_access icap_access; +typedef struct _IcapOptData IcapOptData; #endif typedef struct _icpUdpData icpUdpData; typedef struct _clientHttpRequest clientHttpRequest;