--------------------- PatchSet 3457 Date: 2001/12/07 23:25:31 Author: jkay Branch: push Tag: (none) Log: Initial port of diffs from Cuttlefish. Members: acconfig.h:1.12->1.12.2.1 configure.in:1.35->1.35.2.1 errors/list:1.1.1.1->1.1.1.1.120.1 src/HintCacheNet.c:1.1.2.1->1.1.2.2 src/Makefile.am:1.8.2.1->1.8.2.2 src/access_log.c:1.15->1.15.2.1 src/acl.c:1.42->1.42.2.1 src/cache_cf.c:1.38->1.38.2.1 src/cf.data.pre:1.43->1.43.2.1 src/client_side.c:1.45.2.1->1.45.2.2 src/comm.c:1.18->1.18.2.1 src/defines.h:1.15.2.1->1.15.2.2 src/dist.c:1.1.2.1->1.1.2.2 src/enums.h:1.27.2.1->1.27.2.2 src/globals.h:1.14.2.1->1.14.2.2 src/http.c:1.17->1.17.2.1 src/icp_v2.c:1.5->1.5.14.1 src/icp_v3.c:1.4->1.4.44.1 src/main.c:1.28.2.1->1.28.2.2 src/neighbors.c:1.14->1.14.2.1 src/protos.h:1.41.2.1->1.41.2.2 src/put.c:1.1.2.1->1.1.2.2 src/squid.h:1.13->1.13.2.1 src/stat.c:1.13->1.13.2.1 src/structs.h:1.47.2.1->1.47.2.2 src/typedefs.h:1.25.2.1->1.25.2.2 src/url.c:1.7->1.7.2.1 Index: squid/acconfig.h =================================================================== RCS file: /cvsroot/squid-sf//squid/Attic/acconfig.h,v retrieving revision 1.12 retrieving revision 1.12.2.1 diff -u -r1.12 -r1.12.2.1 --- squid/acconfig.h 24 Oct 2001 09:42:59 -0000 1.12 +++ squid/acconfig.h 7 Dec 2001 23:25:31 -0000 1.12.2.1 @@ -23,7 +23,7 @@ #ifndef __CONFIGURE_H__ #define __CONFIGURE_H__ @TOP@ -/* $Id: acconfig.h,v 1.12 2001/10/24 09:42:59 squidadm Exp $ */ +/* $Id: acconfig.h,v 1.12.2.1 2001/12/07 23:25:31 jkay Exp $ */ /* * configure command line used to configure Squid @@ -133,6 +133,21 @@ #undef USE_ARP_ACL /* + * Define this to include code for Hint Caching. + */ +#undef USE_HINT_CACHE + +/* + * Define this to include code for the Plaxton dynamic cache hierarchy. + */ +#undef USE_DYNAMIC_HIERARCHY + +/* + * Define this to include code for sending data by ICP + */ +#undef USE_ICPDATA + +/* * Define this to include code for the Hypertext Cache Protocol (HTCP) */ #undef USE_HTCP Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.35 retrieving revision 1.35.2.1 diff -u -r1.35 -r1.35.2.1 --- squid/configure.in 18 Nov 2001 08:55:16 -0000 1.35 +++ squid/configure.in 7 Dec 2001 23:25:31 -0000 1.35.2.1 @@ -3,7 +3,7 @@ dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.35 2001/11/18 08:55:16 squidadm Exp $ +dnl $Id: configure.in,v 1.35.2.1 2001/12/07 23:25:31 jkay Exp $ dnl dnl dnl @@ -11,7 +11,7 @@ AC_CONFIG_AUX_DIR(cfgaux) AM_INIT_AUTOMAKE(Squid, 2.5-DEVEL) AM_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.35 $)dnl +AC_REVISION($Revision: 1.35.2.1 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AM_MAINTAINER_MODE @@ -75,6 +75,11 @@ AC_DEFINE_UNQUOTED(SQUID_CONFIGURE_OPTIONS, "$ac_configure_args") +PRESET_CFLAGS="$CFLAGS" + +dnl Check for GNU cc +AC_PROG_CC + dnl Gerben Wierda case "$host" in mab-next-nextstep3) @@ -544,6 +549,36 @@ fi ]) +AM_CONDITIONAL(ENABLE_HINTS, false) +AC_ARG_ENABLE(hints, +[ --enable-hints Enable Hint Cache protocol], +[ if test "$enableval" = "yes" ; then + echo "Hint Cache enabled" + AC_DEFINE(USE_HINT_CACHE) + AM_CONDITIONAL(ENABLE_HINTS, true) + fi +]) + +AM_CONDITIONAL(ENABLE_PLAXTON, false) +AC_ARG_ENABLE(plaxton, +[ --enable-plaxton Enable Plaxton dynamic hierarchy protocol], +[ if test "$enableval" = "yes" ; then + echo "Plaxton dynamic hierarchy enabled" + AC_DEFINE(USE_DYNAMIC_HIERARCHY) + AM_CONDITIONAL(ENABLE_PLAXTON, true) + fi +]) + +AM_CONDITIONAL(ENABLE_ICPDATA, false) +AC_ARG_ENABLE(icpdata, +[ --enable-icpdata Enable sending data by ICP], +[ if test "$enableval" = "yes" ; then + echo "Push over ICP/UDP enabled" + AC_DEFINE(USE_ICPDATA) + AM_CONDITIONAL(ENABLE_ICPDATA, true) + fi +]) + AM_CONDITIONAL(ENABLE_HTCP, false) AC_ARG_ENABLE(htcp, [ --enable-htcp Enable HTCP protocol], @@ -1077,6 +1112,7 @@ strings.h \ sys/file.h \ sys/ioctl.h \ + sys/mman.h \ sys/mount.h \ sys/msg.h \ sys/param.h \ Index: squid/errors/list =================================================================== RCS file: /cvsroot/squid-sf//squid/errors/list,v retrieving revision 1.1.1.1 retrieving revision 1.1.1.1.120.1 diff -u -r1.1.1.1 -r1.1.1.1.120.1 --- squid/errors/list 26 Jan 2000 03:21:47 -0000 1.1.1.1 +++ squid/errors/list 7 Dec 2001 23:25:31 -0000 1.1.1.1.120.1 @@ -3,6 +3,7 @@ ERR_CACHE_MGR_ACCESS_DENIED ERR_CANNOT_FORWARD ERR_CLIENT_ABORT +ERR_CONFLICT ERR_CONNECT_FAIL ERR_DNS_FAIL ERR_FORWARDING_DENIED Index: squid/src/HintCacheNet.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/HintCacheNet.c,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/HintCacheNet.c 20 Nov 2001 06:53:20 -0000 1.1.2.1 +++ squid/src/HintCacheNet.c 7 Dec 2001 23:25:31 -0000 1.1.2.2 @@ -1,5 +1,5 @@ /* - * $Date: 2001/11/20 06:53:20 $ $Id: HintCacheNet.c,v 1.1.2.1 2001/11/20 06:53:20 jkay Exp $ + * $Date: 2001/12/07 23:25:31 $ $Id: HintCacheNet.c,v 1.1.2.2 2001/12/07 23:25:31 jkay Exp $ * * DEBUG: section 55 Hint cache networking. * AUTHOR: Mike Dahlin, UT Austin, 1998 @@ -77,9 +77,9 @@ assert(n); /* Set up entry with request */ - BIT_SET(flags, REQ_CACHABLE); - BIT_SET(flags, KEEP_INMEM); - BIT_SET(flags, DONT_FREE_FOR); + EBIT_SET(flags, REQ_CACHABLE); + EBIT_SET(flags, ENTRY_SPECIAL); /* formerly set KEEP_INMEM */ +/* EBIT_SET(flags, DONT_FREE_FOR); */ n->sendBuffer = storeCreateEntry(url, url, NULL, 0, flags, METHOD_GET); n->sendBuffer->mem_obj->request = requestLink(urlParse(METHOD_GET, (char *) url)); @@ -168,8 +168,8 @@ assert(n->state == HC_full); - BIT_RESET(entry->flag, KEEP_INMEM); - BIT_SET(entry->flag, RELEASE_REQUEST); + EBIT_CLR(entry->flag, KEEP_INMEM); + EBIT_SET(entry->flag, RELEASE_REQUEST); storeUnlockObject(entry); clear(n); Index: squid/src/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v retrieving revision 1.8.2.1 retrieving revision 1.8.2.2 diff -u -r1.8.2.1 -r1.8.2.2 --- squid/src/Makefile.am 20 Nov 2001 23:03:30 -0000 1.8.2.1 +++ squid/src/Makefile.am 7 Dec 2001 23:25:31 -0000 1.8.2.2 @@ -72,6 +72,12 @@ PLAXSOURCE = endif +if ENABLE_ICPDATA +ICPDATASOURCE = icpdata.c +else +ICPDATASOURCE = +endif + if ENABLE_WIN32SPECIFIC WIN32SOURCE = win32.c else @@ -166,6 +172,7 @@ icmp.c \ icp_v2.c \ icp_v3.c \ + $(ICPDATASOURCE) \ ident.c \ internal.c \ ipc.c \ @@ -299,6 +306,7 @@ DEFAULT_ICON_DIR = $(sysconfdir)/icons DEFAULT_ERROR_DIR = $(sysconfdir)/errors DEFAULT_MIB_PATH = $(sysconfdir)/mib.txt +DEFAULT_HINT_FILE = $(DEFAULT_SWAP_DIR)/hints DEFS = @DEFS@ -DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\" @@ -348,6 +356,7 @@ s%@DEFAULT_ICON_DIR@%$(DEFAULT_ICON_DIR)%g;\ s%@DEFAULT_MIB_PATH@%$(DEFAULT_MIB_PATH)%g;\ s%@DEFAULT_ERROR_DIR@%$(DEFAULT_ERROR_DIR)%g;\ + s%@DEFAULT_HINT_FILE@%$(DEFAULT_HINT_FILE)%g;\ s%@DEFAULT_PREFIX@%$(DEFAULT_PREFIX)%g;"\ < $(srcdir)/cf.data.pre >$@ Index: squid/src/access_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/access_log.c,v retrieving revision 1.15 retrieving revision 1.15.2.1 diff -u -r1.15 -r1.15.2.1 --- squid/src/access_log.c 24 Oct 2001 09:42:11 -0000 1.15 +++ squid/src/access_log.c 7 Dec 2001 23:25:31 -0000 1.15.2.1 @@ -1,6 +1,6 @@ /* - * $Id: access_log.c,v 1.15 2001/10/24 09:42:11 squidadm Exp $ + * $Id: access_log.c,v 1.15.2.1 2001/12/07 23:25:31 jkay Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -64,6 +64,7 @@ "TCP_MEM_HIT", "TCP_DENIED", "TCP_OFFLINE_HIT", + "TCP_PUT", #if LOG_TCP_REDIRECTS "TCP_REDIRECT", #endif @@ -72,6 +73,7 @@ "UDP_DENIED", "UDP_INVALID", "UDP_MISS_NOFETCH", + "UDP_PUT", "ICP_QUERY", "LOG_TYPE_MAX" }; Index: squid/src/acl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/acl.c,v retrieving revision 1.42 retrieving revision 1.42.2.1 diff -u -r1.42 -r1.42.2.1 --- squid/src/acl.c 13 Nov 2001 22:19:33 -0000 1.42 +++ squid/src/acl.c 7 Dec 2001 23:25:31 -0000 1.42.2.1 @@ -1,6 +1,6 @@ /* - * $Id: acl.c,v 1.42 2001/11/13 22:19:33 squidadm Exp $ + * $Id: acl.c,v 1.42.2.1 2001/12/07 23:25:31 jkay Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Duane Wessels @@ -351,7 +351,7 @@ protocol_t protocol; for (Tail = curlist; *Tail; Tail = &((*Tail)->next)); while ((t = strtokFile())) { - protocol = urlParseProtocol(t); + protocol = urlParseProtocol(t, 0, 0); q = memAllocate(MEM_INTLIST); q->i = (int) protocol; *(Tail) = q; Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.38 retrieving revision 1.38.2.1 diff -u -r1.38 -r1.38.2.1 --- squid/src/cache_cf.c 24 Oct 2001 09:42:12 -0000 1.38 +++ squid/src/cache_cf.c 7 Dec 2001 23:25:31 -0000 1.38.2.1 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.38 2001/10/24 09:42:12 squidadm Exp $ + * $Id: cache_cf.c,v 1.38.2.1 2001/12/07 23:25:31 jkay Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -1461,6 +1461,21 @@ p->options.allow_miss = 1; } else if (!strcasecmp(token, "max-conn=")) { p->max_conn = atoi(token + 9); + } else if (!strncasecmp(token, "dist", 4)) { + p->options.dist = 1; + } else if (!strncasecmp(token, "fwdpush", 7)) { + p->fwdpush = DistAllPeers; + DistAllPeers = p; +#if USE_ICPDATA + } else if (!strncasecmp(token, "icpdata", 7)) { + p->options.icpdata = 1; + } else if (!strncasecmp(token, "maxrate=", 8)) { + p->maxrate = atoi(token + 8); +#endif +#if USE_HINT_CACHE + } else if (!strncasecmp(token, "nohint", 6)) { + p->options.nohint = 1; +#endif } else { debug(3, 0) ("parse_peer: token='%s'\n", token); self_destruct(); Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.43 retrieving revision 1.43.2.1 diff -u -r1.43 -r1.43.2.1 --- squid/src/cf.data.pre 13 Nov 2001 16:17:30 -0000 1.43 +++ squid/src/cf.data.pre 7 Dec 2001 23:25:31 -0000 1.43.2.1 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.43 2001/11/13 16:17:30 squidadm Exp $ +# $Id: cf.data.pre,v 1.43.2.1 2001/12/07 23:25:31 jkay Exp $ # # # SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -278,6 +278,8 @@ digest-url=url allow-miss max-conn + dist + nohint use 'proxy-only' to specify that objects fetched from this cache should not be saved locally. @@ -365,6 +367,19 @@ use 'max-conn' to limit the amount of connections Squid may open to this peer. + 'dist' tells squid to ask for consistency updates + via httpdist on all queries to that peer. + + 'nohint' tells squid not to send hint cache updates + to that peer. The default is that updates are sent + if hints are active. + + 'icpdata' says to send pushes to this peer by the + UDP-based ICPDATA protocol instead of TCP. + + 'maxrate=n' specifies the maximum rate to send + ICPDATA data at. + NOTE: non-ICP neighbors must be specified as 'parent'. DOC_END @@ -3599,6 +3614,111 @@ time. By default it is set to 10% of the Cache Digest. DOC_END +NAME: hint_cache_listen +IFDEF: USE_HINT_CACHE +TYPE: onoff +LOC: Config.onoff.hint_cache_listen +DEFAULT: off +DOC_START + This controls whether hint cache is used to decide where to + forward misses. Hint caches exchange data to track nearest + location and metadata of all URLs in a hint cache cloud; + This variable also decides whether we listen to hints from + neighboring hint caches. +DOC_END + +NAME: hint_cache_advertise +IFDEF: USE_HINT_CACHE +TYPE: onoff +LOC: Config.onoff.hint_cache_advertise +DEFAULT: off +DOC_START + This controls whether we tell other caches about content in + our own cache. We only want to do that if we are in a + position to efficiently serve requests from other caches. + + We encourage you to turn this on, as participation in the + hint cloud as a full neighbor brings greater value to + everybody, including yourself. +DOC_END + +NAME: hint_cache_file +IFDEF: USE_HINT_CACHE +TYPE: string +LOC: Config.Hints.cache_file +DEFAULT: @DEFAULT_HINT_FILE@ +DOC_START + This is the name of the file holding the hint cache. +DOC_END + +NAME: hint_cache_size +COMMENT: bytes +IFDEF: USE_HINT_CACHE +TYPE: int +LOC: Config.Hints.size; +DEFAULT: 83886080 +DOC_START + This is the size of the hint cache, in bytes. It has to be + big enough to hold location and metadata for all objects + tracked. +DOC_END + +NAME: hint_cache_assoc +IFDEF: USE_HINT_CACHE +TYPE: int +LOC: Config.Hints.assoc; +DEFAULT: 4 +DOC_START + Hint cache associativity. +DOC_END + +NAME: hint_cache_interval +IFDEF: USE_HINT_CACHE +TYPE: int +LOC: Config.Hints.intvl +DEFAULT: 60 +DOC_START + Max interval between hint transmissions to neighbors, in + seconds. Actual interval is randomly, uniformly distributed + between 0 and hint_cache_interval. +DOC_END + +NAME: hint_cache_use_mmap +IFDEF: USE_HINT_CACHE +TYPE: onoff +LOC: Config.onoff.hint_cache_use_mmap +DEFAULT: off +DOC_START + This tells the hint cache to access the hint cache file + using mmap() instead of standard file I/O. A slight + speedup on machines with plenty of RAM. If you are not + so long on RAM for your cache, leave it off. It probably + isn't efficient unless you can hold almost the entire + cache in memory. See hint_cache_size, above, for how + big your hint cache is. +DOC_END + +NAME: hint_cache_join_interval +IFDEF: USE_HINT_CACHE +TYPE: int +LOC: Config.Hints.join_intvl +DEFAULT: 14400 +DOC_START + Max interval between join transmissions to neighbors, in + seconds. Actual interval is randomly, uniformly distributed + between 0 and hint_cache_join_interval. +DOC_END + +NAME: hint_cache_holddown +IFDEF: USE_HINT_CACHE +TYPE: int +LOC: Config.Hints.holddown +DEFAULT: 900 +DOC_START + Amount of time to pass before we can assume that a hint + has finished propagating. +DOC_END + NAME: chroot TYPE: string LOC: Config.chroot_dir Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.45.2.1 retrieving revision 1.45.2.2 diff -u -r1.45.2.1 -r1.45.2.2 --- squid/src/client_side.c 24 Nov 2001 05:55:17 -0000 1.45.2.1 +++ squid/src/client_side.c 7 Dec 2001 23:25:32 -0000 1.45.2.2 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.45.2.1 2001/11/24 05:55:17 jkay Exp $ + * $Id: client_side.c,v 1.45.2.2 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -2316,7 +2316,16 @@ storeCreateMemObject(http->entry, http->uri, http->log_uri); http->entry->mem_obj->method = r->method; http->sc = storeClientListAdd(http->entry, http); -#if DELAY_POOLS + if (r->dist_type != DIST_NONE) { + if (r->dist_type == DIST_DIST) + distNewUpdatee(http->entry, r->client_addr, r->dist_port); + else { + /* DIST_UNDIST */ + assert(r->dist_type == DIST_UNDIST); + distUndist(http->entry, r->client_addr, r->dist_port); + } + } + #if DELAY_POOLS delaySetStoreClient(http->sc, delayClient(r)); #endif storeClientCopy(http->sc, http->entry, Index: squid/src/comm.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/comm.c,v retrieving revision 1.18 retrieving revision 1.18.2.1 diff -u -r1.18 -r1.18.2.1 --- squid/src/comm.c 24 Oct 2001 09:42:12 -0000 1.18 +++ squid/src/comm.c 7 Dec 2001 23:25:32 -0000 1.18.2.1 @@ -1,6 +1,6 @@ /* - * $Id: comm.c,v 1.18 2001/10/24 09:42:12 squidadm Exp $ + * $Id: comm.c,v 1.18.2.1 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 5 Socket Functions * AUTHOR: Harvest Derived @@ -618,6 +618,9 @@ comm_close(int fd) { fde *F = NULL; +#if USE_ICPDATA + int isvfd; +#endif debug(5, 5) ("comm_close: FD %d\n", fd); assert(fd >= 0); @@ -635,6 +638,9 @@ if (F->ssl) ssl_shutdown_method(fd); #endif +#if USE_ICPDATA + isvfd = (conn->comm_type == SOCK_VIRTUAL); +#endif CommWriteStateCallbackAndFree(fd, COMM_ERR_CLOSING); commCallCloseHandlers(fd); if (F->uses) /* assume persistent connect count */ @@ -646,7 +652,18 @@ } #endif fd_close(fd); /* update fdstat */ +#if USE_ICPDATA + if (isvfd) { + /* flag for later reuse */ + conn->comm_type = SOCK_VIRTUAL; + } + else { + debug(5, 9) ("comm_close: really closing fd %d \n", fd); + close(fd); + } +#else /* USE_ICPDATA */ close(fd); +#endif /* USE_ICPDATA */ statCounter.syscalls.sock.closes++; } @@ -999,3 +1016,48 @@ } } } + +#if USE_ICPDATA +/* + * Get fake file descriptor suitable for indexing through fd_table. + */ +int +comm_virtual_fd() +{ + static int firstfd = -1; + struct in_addr nobody; + int vfd; + int i; + + if (firstfd == -1){ + nobody.s_addr = INADDR_ANY; + firstfd = comm_open(SOCK_DGRAM, 0, nobody, 0, COMM_NONBLOCKING, "virtual"); + vfd = firstfd; + } + else { + for (i = 0; i <= Biggest_FD; ++i) { + if (fd_table[i].comm_type == SOCK_VIRTUAL && !fd_table[i].openned) + break; + } + if (fd_table[i].comm_type != SOCK_VIRTUAL) { + nobody.s_addr = INADDR_ANY; + vfd = comm_open(SOCK_DGRAM, 0, nobody, 0, COMM_NONBLOCKING, "virtual"); + debug(5, 4, "comm_virtual_fd: opened new vfd %d (Biggest_FD = %d)\n", + vfd, Biggest_FD); + assert(vfd > -1); + } + else { + vfd = i; + } + } + + fdstat_open(vfd, FD_SOCKET); + memset((char *) &fd_table[vfd], '\0', sizeof(FD_ENTRY)); + fd_table[vfd].openned = 1; + fd_table[vfd].lifetime = -1; + fd_table[vfd].comm_type = SOCK_VIRTUAL; + debug(5, 7, "comm_virtual_fd: returning vfd %d \n", vfd); + return(vfd); +} + +#endif /* USE_ICPDATA */ Index: squid/src/defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/defines.h,v retrieving revision 1.15.2.1 retrieving revision 1.15.2.2 diff -u -r1.15.2.1 -r1.15.2.2 --- squid/src/defines.h 24 Nov 2001 05:53:17 -0000 1.15.2.1 +++ squid/src/defines.h 7 Dec 2001 23:25:32 -0000 1.15.2.2 @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.15.2.1 2001/11/24 05:53:17 jkay Exp $ + * $Id: defines.h,v 1.15.2.2 2001/12/07 23:25:32 jkay Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -299,7 +299,6 @@ /* put.c */ #define P_ACTIVE(p) ((p)->flags.connected | (p)->flags.connecting) - #if USE_HINT_CACHE /* Hint Cache Interface */ Index: squid/src/dist.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/dist.c,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/dist.c 20 Nov 2001 22:55:51 -0000 1.1.2.1 +++ squid/src/dist.c 7 Dec 2001 23:25:32 -0000 1.1.2.2 @@ -1,5 +1,5 @@ /* - * $Id: dist.c,v 1.1.2.1 2001/11/20 22:55:51 jkay Exp $ + * $Id: dist.c,v 1.1.2.2 2001/12/07 23:25:32 jkay Exp $ * * Distribution network algorithm. * @@ -36,14 +36,13 @@ static char urlbuf[MAX_URL + 48]; -/* Updatee management */ - /* Create new updatee ds & add to object's updatee list */ Updatee * distNewUpdatee(StoreEntry *entry, struct in_addr host, int dport) { Updatee *u; + ++statCounter.dist.dists; /* Don't add a client if it isn't interested in updates */ if (dport == 0) return(NULL); @@ -64,19 +63,22 @@ u->saddr.sin_port = dport; u->lastmod = entry->lastmod; entry->updatees = u; + ++statCounter.dist.updatees; return(u); } void -distDelUpdatee(StoreEntry *entry, struct sockaddr_in *sin) +distDelUpdatee(StoreEntry *entry, struct in_addr host, int dport) { Updatee *i, **optr; + ++statCounter.dist.undists; optr = &entry->updatees; for (i = entry->updatees; i; i = i->next) { - if (i->saddr.sin_addr.s_addr == sin->sin_addr.s_addr - && i->saddr.sin_port == sin->sin_port) { + if (i->saddr.sin_addr.s_addr == host.s_addr + && i->saddr.sin_port == dport) { (*optr) = i->next; + --statCounter.dist.updatees; xfree(i); } optr = &i->next; @@ -90,6 +92,7 @@ for ( ; u; u = nu) { nu = u->next; + --statCounter.dist.updatees; xfree(u); } } @@ -104,7 +107,9 @@ /* Already has this version */ return; u->lastmod = u->entry->lastmod; /* Mark version being xferred */ - ++statCounter.push.dists; + + ++statCounter.dist.updates; + kb_incr(&statCounter.dist.update_kbytes, u->entry->mem_obj->inmem_hi); u->p = putSend(u->entry, &u->saddr, (PF *) 0, 0, 1 /* dodist */); } @@ -116,10 +121,13 @@ struct sockaddr_in *me = &Config.Sockaddr.http->s; Updatee *u; + ++statCounter.dist.content_upd; + /* Open distribution socket to everybody interested */ for (u = entry->updatees; u; u = u->next) { - if (!req || !req->dport || req->dport != u->saddr.sin_port - || peer->sin_addr.s_addr != me->sin_addr.s_addr) + if (!req || !req->dist_port + || req->dist_port != u->saddr.sin_port + || peer->sin_addr.s_addr != me->sin_addr.s_addr) /* Send if got a dist and it isn't to where we got this update */ distUpdate(u); } @@ -133,7 +141,7 @@ char * distifyUrl(char *url) { - char distbuf[20]; /* dist string holder */ + char distbuf[48]; /* dist string holder */ int plen, poff; char *eop; @@ -155,3 +163,40 @@ return(urlbuf); } + +/* Report on push stats */ +void +statDistPrint(StoreEntry * sentry) +{ + statPutPrint(sentry); + + storeAppendPrintf(sentry, "\nDIST Statistics:\n\n"); + storeAppendPrintf(sentry, "DIST requests: %d\n", + statCounter.dist.dists); + storeAppendPrintf(sentry, "unDIST requests: %d\n", + statCounter.dist.undists); + storeAppendPrintf(sentry, "Updates Sent: %d\n", + statCounter.dist.updates); + storeAppendPrintf(sentry, "KBytes of Update Sent: %d\n", + statCounter.dist.update_kbytes); + storeAppendPrintf(sentry, "Content Updates: %d\n", + statCounter.dist.content_upd); + storeAppendPrintf(sentry, "Updatee Structures: %d\n", + statCounter.dist.updatees); + storeAppendPrintf(sentry, "Updatee Memory Consumption: %d\n", + statCounter.dist.updatees * sizeof(Updatee)); +} + + +/* Need this to make sure that reconfiguration gives correctly + updated FwdPushLinks */ +void +distInit() +{ + DistAllPeers = 0; + + cachemgrRegister("push", "Push Stats", + statDistPrint, 0, 1); +} + + Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.27.2.1 retrieving revision 1.27.2.2 diff -u -r1.27.2.1 -r1.27.2.2 --- squid/src/enums.h 24 Nov 2001 05:53:17 -0000 1.27.2.1 +++ squid/src/enums.h 7 Dec 2001 23:25:32 -0000 1.27.2.2 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.27.2.1 2001/11/24 05:53:17 jkay Exp $ + * $Id: enums.h,v 1.27.2.2 2001/12/07 23:25:32 jkay Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -431,6 +431,9 @@ PROTO_WHOIS, PROTO_INTERNAL, PROTO_HTTPS, +#if USE_HINT_CACHE + PROTO_ROUTE, +#endif PROTO_MAX } protocol_t; @@ -731,6 +734,12 @@ VARY_CANCEL }; +enum { + DIST_NONE, + DIST_DIST, + DIST_UNDIST +}; + #if USE_HINT_CACHE enum HintCacheNet_State {HC_uninitialized, HC_filling, HC_full, HC_destroyed}; Index: squid/src/globals.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/globals.h,v retrieving revision 1.14.2.1 retrieving revision 1.14.2.2 diff -u -r1.14.2.1 -r1.14.2.2 --- squid/src/globals.h 24 Nov 2001 05:53:17 -0000 1.14.2.1 +++ squid/src/globals.h 7 Dec 2001 23:25:32 -0000 1.14.2.2 @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.14.2.1 2001/11/24 05:53:17 jkay Exp $ + * $Id: globals.h,v 1.14.2.2 2001/12/07 23:25:32 jkay Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -157,6 +157,7 @@ extern RemovalPolicy *mem_policy; extern hash_table *proxy_auth_username_cache; /* NULL */ extern int incoming_sockets_accepted; +extern peer *DistAllPeers; #if USE_HINT_CACHE extern const URLKey INVALID_URL_KEY; extern HintCacheDisk *HCDisk; Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.17 retrieving revision 1.17.2.1 diff -u -r1.17 -r1.17.2.1 --- squid/src/http.c 24 Oct 2001 09:42:13 -0000 1.17 +++ squid/src/http.c 7 Dec 2001 23:25:32 -0000 1.17.2.1 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.17 2001/10/24 09:42:13 squidadm Exp $ + * $Id: http.c,v 1.17.2.1 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -45,9 +45,7 @@ static CWCB httpSendComplete; static CWCB httpSendRequestEntry; -static PF httpReadReply; static void httpSendRequest(HttpStateData *); -static PF httpStateFree; static PF httpTimeout; static void httpCacheNegatively(StoreEntry *); static void httpMakePrivate(StoreEntry *); @@ -55,7 +53,7 @@ static int httpCachableReply(HttpStateData *); static void httpMaybeRemovePublic(StoreEntry *, http_status); -static void +void httpStateFree(int fd, void *data) { HttpStateData *httpState = data; @@ -79,9 +77,11 @@ int httpCachable(method_t method) { - /* GET and HEAD are cachable. Others are not. */ - if (method != METHOD_GET && method != METHOD_HEAD) + /* GET, PUT, and HEAD are cachable. Others are not. */ + if (method != METHOD_GET && method != METHOD_HEAD && + method != METHOD_PUT) { return 0; + } /* else cachable */ return 1; } @@ -415,6 +415,7 @@ t = httpState->reply_hdr + k; } *t = '\0'; + httpState->body_remain -= hdr_len - (t - httpState->reply_hdr); httpState->reply_hdr_state++; assert(httpState->reply_hdr_state == 1); ctx = ctx_enter(entry->mem_obj->url); @@ -424,6 +425,9 @@ /* Parse headers into reply structure */ /* what happens if we fail to parse here? */ httpReplyParse(reply, httpState->reply_hdr, hdr_len); + if (reply->content_length > 0) { + httpState->body_remain = reply->content_length + reply->hdr_sz; + } storeTimestampsSet(entry); /* Check if object is cacheable or not based on reply code */ debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); @@ -532,7 +536,7 @@ /* This will be called when data is ready to be read from fd. Read until * error or connection closed. */ /* XXX this function is too long! */ -static void +void httpReadReply(int fd, void *data) { HttpStateData *httpState = data; @@ -611,7 +615,7 @@ fwdFail(httpState->fwd, err); httpState->eof = 1; comm_close(fd); - } else if (len == 0) { + } else if (len == 0 || len >= httpState->body_remain) { /* Connection closed; retrieval done. */ httpState->eof = 1; if (httpState->reply_hdr_state < 2) @@ -622,8 +626,23 @@ * we want to process the reply headers. */ httpProcessReplyHeader(httpState, buf, len); - fwdComplete(httpState->fwd); - comm_close(fd); + else + /* whole body is subtracted */ + httpState->body_remain -= len; + debug(50, 7) ("httpReadReply: body_remain = %d (len %d, state %d)\n", + httpState->body_remain, len, httpState->reply_hdr_state); + if (len > 0) + storeAppend(entry, buf, len); + if (entry->mem_obj->request && + entry->mem_obj->request->method == METHOD_PUT) { + putReplyAndClose(fd, entry); + debug(50, 8) ("httpReadReply: called putReplyAndClose\n"); + /* putReplyAndClose calls storeComplete() */ + } + else { + fwdComplete(httpState->fwd); + comm_close(fd); + } } else { if (httpState->reply_hdr_state < 2) { httpProcessReplyHeader(httpState, buf, len); @@ -640,6 +659,8 @@ EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); } } + httpState->body_remain -= len; + debug(50, 8) ("httpReadReply: %d remain\n", httpState->body_remain); storeAppend(entry, buf, len); if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* @@ -929,9 +950,16 @@ http_state_flags flags) { const int offset = mb->size; + const char *url; + + url = strLen(request->urlpath) ? strBuf(request->urlpath) : "/"; + if (request->flags.distify) { + /* Ask other cache for dist */ + url = (const char *) distifyUrl((char *) url); + } + memBufPrintf(mb, "%s %s HTTP/1.0\r\n", - RequestMethodStr[request->method], - strLen(request->urlpath) ? strBuf(request->urlpath) : "/"); + RequestMethodStr[request->method], url); /* build and pack headers */ { HttpHeader hdr; @@ -1008,6 +1036,7 @@ HttpStateData *httpState; request_t *proxy_req; request_t *orig_req = fwd->request; + peer *peer; debug(11, 3) ("httpStart: \"%s %s\"\n", RequestMethodStr[orig_req->method], storeUrl(fwd->entry)); @@ -1016,13 +1045,15 @@ httpState->fwd = fwd; httpState->entry = fwd->entry; httpState->fd = fd; + httpState->body_remain = 0x7fffffff; if (fwd->servers) httpState->peer = fwd->servers->peer; /* might be NULL */ - if (httpState->peer) { + peer = httpState->peer; + if (peer) { proxy_req = requestCreate(orig_req->method, PROTO_NONE, storeUrl(httpState->entry)); - xstrncpy(proxy_req->host, httpState->peer->host, SQUIDHOSTNAMELEN); - proxy_req->port = httpState->peer->http_port; + xstrncpy(proxy_req->host, peer->host, SQUIDHOSTNAMELEN); + proxy_req->port = peer->http_port; proxy_req->flags = orig_req->flags; proxy_req->lastmod = orig_req->lastmod; httpState->request = requestLink(proxy_req); @@ -1033,8 +1064,15 @@ * We might end up getting the object from somewhere else if, * for example, the request to this neighbor fails. */ - if (httpState->peer->options.proxy_only) + if (peer->options.proxy_only) storeReleaseRequest(httpState->entry); + /* Use consistency protocol for this request */ + if (peer->options.dist) { + distNewUpdatee(httpState->entry, + peer->in_addr.sin_addr, + peer->http_port); + httpState->request->flags.distify = 1; + } #if DELAY_POOLS assert(delayIsNoDelay(fd) == 0); if (httpState->peer->options.no_delay) @@ -1132,3 +1170,32 @@ version->major = major; version->minor = minor; } + +/* + * Create an httpState parallel to an existing clientHttpRequest + * Assumption: http->entry is either zero or locked. + */ +HttpStateData * +httpStateFromClient(clientHttpRequest *http) +{ + HttpStateData *httpState; + + /* Make me an httpState */ + httpState = cbdataAlloc(HttpStateData); + + /* Fill in the blanks */ + httpState->entry = http->entry; + httpState->request = http->request; + httpState->body_remain = 0x7fffffff; + + assert(http->entry->lock_count >= 1); + /* Lock entry for a second time to reflect presence in http */ + /* XXX - can't use storeLockObject() because is NOT_IN_MEMORY (!!!) */ + httpState->entry->lock_count++; + assert(http->entry->lock_count >= 2); + + /* Register new request pointer */ + requestLink(httpState->request); + + return(httpState); +} Index: squid/src/icp_v2.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/icp_v2.c,v retrieving revision 1.5 retrieving revision 1.5.14.1 diff -u -r1.5 -r1.5.14.1 --- squid/src/icp_v2.c 4 May 2001 13:39:12 -0000 1.5 +++ squid/src/icp_v2.c 7 Dec 2001 23:25:32 -0000 1.5.14.1 @@ -1,6 +1,6 @@ /* - * $Id: icp_v2.c,v 1.5 2001/05/04 13:39:12 squidadm Exp $ + * $Id: icp_v2.c,v 1.5.14.1 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 12 Internet Cache Protocol * AUTHOR: Duane Wessels @@ -302,6 +302,14 @@ neighborsUdpAck(key, &header, &from); break; +#if USE_ICPDATA + case ICP_OP_DATABEG: + case ICP_OP_DATA: + case ICP_OP_DATAEND: + icpRecvData(fd, &from, &header, buf + sizeof(header), len); + break; +#endif + case ICP_INVALID: case ICP_ERR: break; Index: squid/src/icp_v3.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/icp_v3.c,v retrieving revision 1.4 retrieving revision 1.4.44.1 diff -u -r1.4 -r1.4.44.1 --- squid/src/icp_v3.c 12 Jan 2001 08:20:33 -0000 1.4 +++ squid/src/icp_v3.c 7 Dec 2001 23:25:32 -0000 1.4.44.1 @@ -1,6 +1,6 @@ /* - * $Id: icp_v3.c,v 1.4 2001/01/12 08:20:33 hno Exp $ + * $Id: icp_v3.c,v 1.4.44.1 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 12 Internet Cache Protocol * AUTHOR: Duane Wessels @@ -143,6 +143,14 @@ neighborsUdpAck(key, &header, &from); break; +#if USE_ICPDATA + case ICP_OP_DATABEG: + case ICP_OP_DATA: + case ICP_OP_DATAEND: + icpRecvData(fd, &from, &header, buf + sizeof(header), len); + break; +#endif + case ICP_INVALID: case ICP_ERR: break; Index: squid/src/main.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/main.c,v retrieving revision 1.28.2.1 retrieving revision 1.28.2.2 diff -u -r1.28.2.1 -r1.28.2.2 --- squid/src/main.c 24 Nov 2001 05:53:17 -0000 1.28.2.1 +++ squid/src/main.c 7 Dec 2001 23:25:32 -0000 1.28.2.2 @@ -1,6 +1,6 @@ /* - * $Id: main.c,v 1.28.2.1 2001/11/24 05:53:17 jkay Exp $ + * $Id: main.c,v 1.28.2.2 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -353,6 +353,7 @@ fqdncache_restart(); /* sigh, fqdncache too */ parseEtcHosts(); errorInitialize(); /* reload error pages */ + distInit(); #if USE_DNSSERVERS dnsInit(); #else @@ -491,6 +492,7 @@ httpReplyInitModule(); /* must go before accepting replies */ errorInitialize(); accessLogInit(); + distInit(); #if USE_IDENT identInit(); #endif @@ -521,6 +523,9 @@ #if USE_HINT_CACHE hintCacheInit(); #endif +#if USE_ICPDATA + icpDataInit(); +#endif fwdInit(); } #if USE_WCCP Index: squid/src/neighbors.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/neighbors.c,v retrieving revision 1.14 retrieving revision 1.14.2.1 diff -u -r1.14 -r1.14.2.1 --- squid/src/neighbors.c 18 Oct 2001 20:52:11 -0000 1.14 +++ squid/src/neighbors.c 7 Dec 2001 23:25:32 -0000 1.14.2.1 @@ -1,6 +1,6 @@ /* - * $Id: neighbors.c,v 1.14 2001/10/18 20:52:11 squidadm Exp $ + * $Id: neighbors.c,v 1.14.2.1 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 15 Neighbor Routines * AUTHOR: Harvest Derived @@ -38,6 +38,8 @@ /* count mcast group peers every 15 minutes */ #define MCAST_COUNT_RATE 900 + + static int peerAllowedToUse(const peer *, request_t *); static int peerWouldBePinged(const peer *, request_t *); static void neighborRemove(peer *); Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.41.2.1 retrieving revision 1.41.2.2 diff -u -r1.41.2.1 -r1.41.2.2 --- squid/src/protos.h 24 Nov 2001 05:53:17 -0000 1.41.2.1 +++ squid/src/protos.h 7 Dec 2001 23:25:32 -0000 1.41.2.2 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.41.2.1 2001/11/24 05:53:17 jkay Exp $ + * $Id: protos.h,v 1.41.2.2 2001/12/07 23:25:32 jkay Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -174,6 +174,9 @@ extern void commSetDefer(int fd, DEFER * func, void *); extern int ignoreErrno(int); extern void commCloseAllSockets(void); +#if USE_ICPDATA +extern int comm_virtual_fd(void); +#endif /* USE_ICPDATA */ /* @@ -533,6 +536,12 @@ int reqnum, int pad); extern int icpUdpSend(int, const struct sockaddr_in *, icp_common_t *, log_type, int); +#if USE_ICPDATA +void icpSendData(int vfd, StoreEntry *entry, struct _pusher *p); +void icpRecvData(int fd, struct sockaddr_in *src, icp_common_t *hdr, + char *buf, int len); +void icpDataInit(); +#endif extern PF icpHandleUdp; extern PF icpUdpSendQueue; extern PF httpAccept; @@ -1108,7 +1117,7 @@ extern char *url_convert_hex(char *org_url, int allocate); extern char *url_escape(const char *url); -extern protocol_t urlParseProtocol(const char *, int *); +extern protocol_t urlParseProtocol(const char *, int *, int *); extern method_t urlParseMethod(const char *); extern void urlInitialize(void); extern request_t *urlParse(method_t, char *); @@ -1302,6 +1311,7 @@ int dodist); void putSendDone(int fd, Pusher *p); void putReplyAndClose(int, StoreEntry *); +void statPutPrint(StoreEntry * sentry); /* dist.c */ extern Updatee *distNewUpdatee(StoreEntry *entry, @@ -1311,6 +1321,7 @@ extern void distDelUpdatees(Updatee *u); extern char *distifyUrl(char *url); extern void distInputDone(StoreEntry *); +void distInit(); /* * Removal Policies Index: squid/src/put.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/put.c,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/put.c 20 Nov 2001 22:54:48 -0000 1.1.2.1 +++ squid/src/put.c 7 Dec 2001 23:25:32 -0000 1.1.2.2 @@ -1,5 +1,5 @@ /* - * $Id: put.c,v 1.1.2.1 2001/11/20 22:54:48 jkay Exp $ + * $Id: put.c,v 1.1.2.2 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 81 PUT transmission and reception * AUTHOR: Jon Kay, 1997 @@ -38,6 +38,7 @@ static PF putRecvFree; static void putReply(int fd, StoreEntry *entry); +static void putRecvError(clientHttpRequest *http, http_status entry); static void putProcessHeader(HttpStateData *, clientHttpRequest *); static void putHandleStore(void *data, char *buf, ssize_t len); static CNCB putConnected; @@ -55,6 +56,8 @@ StoreEntry *entry; const cache_key *pubkey; HttpReply *reply; + int lastmod; + peer *p; pubkey = storeKeyPublic(http->uri, request->method); http->request->flags.cachable = 1; @@ -99,34 +102,53 @@ reply = entry->mem_obj->reply; /* Update the timestamp */ - if (entry->mem_obj->reply->last_modified) + lastmod = entry->mem_obj->reply->last_modified; + if (lastmod) { + assert(lastmod >= 0 && entry->lastmod >= 0); + if (lastmod < entry->lastmod) { + /* We do not accept old versions */ + ++statCounter.put.conflicts; + putRecvError(http, HTTP_CONFLICT); + return; + } + /* Note: it's OK to overwrite the same version. It just won't be + propagated. */ entry->lastmod = entry->mem_obj->reply->last_modified; + } else /* Make it clear the version has changed even if no lastmod */ ++entry->lastmod; /* Make sure dist knows about the sender, and distribute the update */ - if (request->dport) - distNewUpdatee(entry, conn->peer.sin_addr, request->dport); + if (request->dist_type == DIST_DIST) + distNewUpdatee(entry, conn->peer.sin_addr, request->dist_port); + else if (request->dist_type == DIST_UNDIST) + distDelUpdatee(entry, conn->peer.sin_addr, request->dist_port); distEntryUpdate(entry, request, &conn->peer); -#ifdef USE_ICPDATA /* fwdpush support */ if (http->log_type == LOG_TCP_PUT) { - for (p = FwdPushLinks; p; p = p->fwdpush) { - ++ndistrib; - if (p->options & NEIGHBOR_ICPDATA) { + for (p = DistAllPeers; p; p = p->fwdpush) { +#if USE_ICPDATA + if (p->options.icpdata) { putSend(entry, &p->in_addr, NULL, NULL, P_ICPDATA, p->icp_data_q); } else { +#endif + struct sockaddr_in sin; + sin.sin_family = AF_INET; sin.sin_addr = p->in_addr.sin_addr; sin.sin_port = p->http_port; - putSend(entry, &sin, NULL, NULL, 0, 0); + putSend(entry, &sin, NULL, NULL, 0); +#if USE_ICPDATA } +#endif } } -#endif + + /* Update stats */ + ++statCounter.put.ins; /* XXX - Can we use httpReplyParse to simplify this? */ @@ -186,6 +208,8 @@ HttpStateData *httpState = data; StoreEntry *entry = httpState->entry; + kb_incr(&statCounter.put.kbytes_in, entry->mem_obj->inmem_hi); + putReply(fd, entry); httpStateFree(fd, httpState); assert(entry->lock_count >= 1); @@ -213,6 +237,19 @@ write(fd, PUT_REPLY_HDR(), sizeof(PUT_REPLY_HDR())); } +static void +putRecvError(clientHttpRequest *http, http_status entry) +{ + ErrorState *err; + + http->log_type = LOG_TCP_DENIED; + err = errorCon(ERR_ACCESS_DENIED, HTTP_CONFLICT); + err->request = requestLink(http->request); + err->src_addr = http->conn->peer.sin_addr; + http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + errorAppendEntry(http->entry, err); +} + /* Output PUTs */ @@ -246,7 +283,7 @@ p->entry = entry; p->handler = handler; p->arg = arg; -#ifdef USE_ICPDATA +#if USE_ICPDATA if (flags & P_ICPDATA) { p->fd = comm_virtual_fd(); p->q = q; @@ -270,6 +307,8 @@ /* Deal with socket closing */ comm_add_close_handler(fd, (PF *) putSendDone, entry); + ++statCounter.put.outs; + /* Connect to this updatee */ strcpy(p->hname, inet_ntoa(saddr->sin_addr)); commConnectStart(fd, p->hname, saddr->sin_port, putConnected, p); @@ -290,6 +329,7 @@ if (p->handler) (*p->handler)(fd, p->arg); + kb_incr(&statCounter.put.kbytes_out, p->off); storeUnregister(p->sc, entry, (void *) fd); storeUnlockObject(entry); @@ -410,3 +450,21 @@ comm_write(fd, buf, strlen(buf), putSendData, p, 0); } + + +/* Report on push stats */ +void +statPutPrint(StoreEntry * sentry) +{ + storeAppendPrintf(sentry, "PUT Statistics:\n\n"); + storeAppendPrintf(sentry, "PUTs Sent: %d\n", + statCounter.put.outs); + storeAppendPrintf(sentry, "PUT KBytes Sent: %d\n", + (int) statCounter.put.kbytes_out.kb); + storeAppendPrintf(sentry, "PUTs Received: %d\n", + statCounter.put.ins); + storeAppendPrintf(sentry, "PUT KBytes Received: %d\n", + (int) statCounter.put.kbytes_in.kb); + storeAppendPrintf(sentry, "Stale Objects Received: %d\n", + statCounter.put.conflicts); +} Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.13 retrieving revision 1.13.2.1 diff -u -r1.13 -r1.13.2.1 --- squid/src/squid.h 13 Nov 2001 22:19:33 -0000 1.13 +++ squid/src/squid.h 7 Dec 2001 23:25:32 -0000 1.13.2.1 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.13 2001/11/13 22:19:33 squidadm Exp $ + * $Id: squid.h,v 1.13.2.1 2001/12/07 23:25:32 jkay Exp $ * * AUTHOR: Duane Wessels * @@ -409,6 +409,10 @@ #include "snprintf.h" #endif +#if HAVE_SYS_MMAN_H +#include +#endif + #define XMIN(x,y) ((x)<(y)? (x) : (y)) #define XMAX(a,b) ((a)>(b)? (a) : (b)) Index: squid/src/stat.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stat.c,v retrieving revision 1.13 retrieving revision 1.13.2.1 diff -u -r1.13 -r1.13.2.1 --- squid/src/stat.c 24 Oct 2001 09:42:13 -0000 1.13 +++ squid/src/stat.c 7 Dec 2001 23:25:32 -0000 1.13.2.1 @@ -1,6 +1,6 @@ /* - * $Id: stat.c,v 1.13 2001/10/24 09:42:13 squidadm Exp $ + * $Id: stat.c,v 1.13.2.1 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -1184,6 +1184,32 @@ f->swap.files_cleaned); storeAppendPrintf(sentry, "aborted_requests = %d\n", f->aborted_requests); + + storeAppendPrintf(sentry, "dist.dists = %d\n", + statCounter.dist.dists); + storeAppendPrintf(sentry, "dist.undists = %d\n", + statCounter.dist.undists); + storeAppendPrintf(sentry, "dist.updates = %d\n", + statCounter.dist.updates); + storeAppendPrintf(sentry, "dist.update_kbytes = %d\n", + statCounter.dist.update_kbytes); + storeAppendPrintf(sentry, "dist.content_upd = %d\n", + statCounter.dist.content_upd); + storeAppendPrintf(sentry, "dist.updatees = %d\n", + statCounter.dist.updatees); + storeAppendPrintf(sentry, "dist.updatees = %d\n", + statCounter.dist.updatees * sizeof(Updatee)); + + storeAppendPrintf(sentry, "put.outs = %d\n", + statCounter.put.outs); + storeAppendPrintf(sentry, "put.kbytes_out = %d\n", + (int) statCounter.put.kbytes_out.kb); + storeAppendPrintf(sentry, "put.ins =: %d\n", + statCounter.put.ins); + storeAppendPrintf(sentry, "put.kbytes_in = %d\n", + (int) statCounter.put.kbytes_in.kb); + storeAppendPrintf(sentry, "put.conflicts = %d\n", + statCounter.put.conflicts); } void Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.47.2.1 retrieving revision 1.47.2.2 diff -u -r1.47.2.1 -r1.47.2.2 --- squid/src/structs.h 24 Nov 2001 05:53:17 -0000 1.47.2.1 +++ squid/src/structs.h 7 Dec 2001 23:25:32 -0000 1.47.2.2 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.47.2.1 2001/11/24 05:53:17 jkay Exp $ + * $Id: structs.h,v 1.47.2.2 2001/12/07 23:25:32 jkay Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -590,6 +590,11 @@ int ie_refresh; int vary_ignore_expire; int pipeline_prefetch; +#if USE_HINT_CACHE + int hint_cache_listen; + int hint_cache_advertise; + int hint_cache_use_mmap; +#endif } onoff; acl *aclList; struct { @@ -682,12 +687,11 @@ #endif #if USE_HINT_CACHE struct { - char *cachefile; + char *cache_file; int size; int assoc; int intvl; - int joinintvl; - int usemmap; + int join_intvl; int holddown; } Hints; #endif @@ -1280,6 +1284,13 @@ unsigned int no_delay:1; #endif unsigned int allow_miss:1; + unsigned int dist:1; +#if USE_ICPDATA + unsigned int icpdata:1; +#endif +#if USE_HINT_CACHE + unsigned int nohint:1; +#endif } options; int weight; struct { @@ -1314,6 +1325,8 @@ char *login; /* Proxy authorization */ time_t connect_timeout; int max_conn; + peer *fwdpush; /* next peer on FwdPushLinks */ + int max_rate; /* Max push rate to this peer */ }; struct _net_db_name { @@ -1619,6 +1632,7 @@ unsigned int accelerated:1; unsigned int internal:1; unsigned int body_sent:1; + unsigned int distify:1; /* Use dist reqs for consistency */ }; struct _link_list { @@ -1667,7 +1681,8 @@ struct in_addr client_addr; struct in_addr my_addr; unsigned short my_port; - unsigned short dport; + int dist_type; + unsigned short dist_port; HttpHeader header; ConnStateData *body_connection; /* used by clientReadBody() */ int content_length; @@ -1868,7 +1883,19 @@ } swap; struct { int dists; - } push; + int undists; + int updates; + kb_t update_kbytes; + int content_upd; + int updatees; + } dist; + struct { + int outs; + int ins; + kb_t kbytes_out; + kb_t kbytes_in; + int conflicts; + } put; }; /* per header statistics */ @@ -2241,48 +2268,48 @@ struct _Net_URLKey { - GenericKey mungedKey; + GenericKey mungedKey; }; struct _NodeKey { - List_Links links; /* MUST BE FIRST FIELD OF STRUCT */ - GenericKey key; - struct sockaddr_in addr; + List_Links links; /* MUST BE FIRST FIELD OF STRUCT */ + GenericKey key; + struct sockaddr_in addr; }; struct _HintCacheEntry { - URLKey key; - struct in_addr ipaddr; - unsigned short port; - short pad; /* Compiler padding space */ - unsigned int mtime; /* Last-modified time */ + URLKey key; + struct in_addr ipaddr; + unsigned short port; + short pad; /* Compiler padding space */ + unsigned int mtime; /* Last-modified time */ }; /* This is what goes onto the net */ struct _Net_HintCacheEntry { - Net_URLKey mungedKey; - struct in_addr mungedIpaddr; - unsigned short mungedPort; - short mungedPad; - unsigned int mungedMtime; + Net_URLKey mungedKey; + struct in_addr mungedIpaddr; + unsigned short mungedPort; + short mungedPad; + unsigned int mungedMtime; }; struct _HintCacheUpdate { - unsigned char action; - char hopcount; - char UNUSED; /* Remove this field sometime XXX */ - char pad1; - int pad2; - HintCacheEntry entry; + unsigned char action; + char hopcount; + char UNUSED; /* Remove this field sometime XXX */ + char pad1; + int pad2; + HintCacheEntry entry; }; struct _Net_HintCacheUpdate { - unsigned char mungedAction; - char mungedHopcount; - char mungedUNUSED; - char mungedPad1; - int mungedPad2; - Net_HintCacheEntry mungedEntry; + unsigned char mungedAction; + char mungedHopcount; + char mungedUNUSED; + char mungedPad1; + int mungedPad2; + Net_HintCacheEntry mungedEntry; }; /* Hint Cache Disk */ @@ -2292,19 +2319,19 @@ }; struct _HintCacheDisk { - unsigned int nbuckets; - int entriesPerBucket; - HintCacheDiskEntry *mmappedArray; - int fd; + unsigned int nbuckets; + int entriesPerBucket; + HintCacheDiskEntry *mmappedArray; + int fd; }; typedef struct HintCacheNetS{ - StoreEntry *sendBuffer; - int sendLength; - enum HintCacheNet_State state; + StoreEntry *sendBuffer; + int sendLength; + enum HintCacheNet_State state; #ifdef DOTEST - int testMode; - int testCount; + int testMode; + int testCount; #endif } HintCacheNet; @@ -2315,4 +2342,15 @@ #endif /* USE_HINT_CACHE */ +#if USE_ICPDATA +typedef struct _icpDataStats { + int nsends; + int nrecvs; + int nfrags; + int ndups; + int ntmofrags; + int nbadfrags; +} +#endif + #endif /* SQUID_STRUCTS_H */ Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.25.2.1 retrieving revision 1.25.2.2 diff -u -r1.25.2.1 -r1.25.2.2 --- squid/src/typedefs.h 24 Nov 2001 05:53:17 -0000 1.25.2.1 +++ squid/src/typedefs.h 7 Dec 2001 23:25:32 -0000 1.25.2.2 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.25.2.1 2001/11/24 05:53:17 jkay Exp $ + * $Id: typedefs.h,v 1.25.2.2 2001/12/07 23:25:32 jkay Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -386,4 +386,8 @@ typedef List_Links *HintCacheNodeListIter; #endif /* USE_HINT_CACHE */ +#if USE_ICPDATA +typedef struct _icpDataStats icpDataStats; +#endif + #endif /* SQUID_TYPEDEFS_H */ Index: squid/src/url.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/url.c,v retrieving revision 1.7 retrieving revision 1.7.2.1 diff -u -r1.7 -r1.7.2.1 --- squid/src/url.c 18 Nov 2001 01:15:42 -0000 1.7 +++ squid/src/url.c 7 Dec 2001 23:25:32 -0000 1.7.2.1 @@ -1,6 +1,6 @@ /* - * $Id: url.c,v 1.7 2001/11/18 01:15:42 squidadm Exp $ + * $Id: url.c,v 1.7.2.1 2001/12/07 23:25:32 jkay Exp $ * * DEBUG: section 23 URL Parsing * AUTHOR: Duane Wessels @@ -98,6 +98,9 @@ "whois", "internal", "https", +#if USE_HINT_CACHE + "route", +#endif "TOTAL" }; @@ -191,30 +194,84 @@ protocol_t -urlParseProtocol(const char *s) +urlParseProtocol(const char *s, int *dportp, int *dtypep) { + int proto; + /* test common stuff first */ - if (strcasecmp(s, "http") == 0) - return PROTO_HTTP; - if (strcasecmp(s, "ftp") == 0) - return PROTO_FTP; - if (strcasecmp(s, "https") == 0) - return PROTO_HTTPS; - if (strcasecmp(s, "file") == 0) - return PROTO_FTP; - if (strcasecmp(s, "gopher") == 0) - return PROTO_GOPHER; - if (strcasecmp(s, "wais") == 0) - return PROTO_WAIS; - if (strcasecmp(s, "cache_object") == 0) - return PROTO_CACHEOBJ; - if (strcasecmp(s, "urn") == 0) - return PROTO_URN; - if (strcasecmp(s, "whois") == 0) - return PROTO_WHOIS; - if (strcasecmp(s, "internal") == 0) - return PROTO_INTERNAL; - return PROTO_NONE; + if (strcasecmp(s, "http") == 0) { + s += 4; + proto = PROTO_HTTP; + } +#if USE_HINT_CACHE + else if (strcasecmp(s, "route") == 0) { + s += 5; + proto = PROTO_ROUTE; + } +#endif + else if (strcasecmp(s, "ftp") == 0) { + s += 3; + proto = PROTO_FTP; + } + else if (strcasecmp(s, "https") == 0) { + s += 5; + proto = PROTO_HTTPS; + } + else if (strcasecmp(s, "file") == 0) { + s += 4; + proto = PROTO_FTP; + } + else if (strcasecmp(s, "gopher") == 0) { + s += 6; + proto = PROTO_GOPHER; + } + else if (strcasecmp(s, "wais") == 0) { + s += 4; + proto =PROTO_WAIS; + } + else if (strcasecmp(s, "cache_object") == 0) { + s += 12; + proto = PROTO_CACHEOBJ; + } + else if (strcasecmp(s, "urn") == 0) { + s += 3; + proto = PROTO_URN; + } + else if (strcasecmp(s, "whois") == 0) { + s += 5; + proto = PROTO_WHOIS; + } + else if (strcasecmp(s, "internal") == 0) { + s += 8; + proto = PROTO_INTERNAL; + } + else + proto = PROTO_NONE; + + if (dportp != NULL) { + if (strncasecmp(s, "dist", 4) == 0) { + s += 4; + if (dportp != NULL) + *dportp = atoi(s); + if (dtypep) + *dtypep = DIST_DIST; + } + else if (strncasecmp(s, "undist", 6) == 0) { + s += 6; + if (dportp != NULL) + *dportp = atoi(s); + if (dtypep) + *dtypep = DIST_UNDIST; + } + else { + if (dportp) + *dportp = 0; + if (dtypep) + *dtypep = 0; + } + } + + return proto; } @@ -252,7 +309,7 @@ request_t *request = NULL; char *t = NULL; char *q = NULL; - int port; + int port, dport, dtype; protocol_t protocol = PROTO_NONE; int l; proto[0] = host[0] = urlpath[0] = login[0] = '\0'; @@ -272,7 +329,7 @@ } else { if (sscanf(url, "%[^:]://%[^/]%[^\r\n]", proto, host, urlpath) < 2) return NULL; - protocol = urlParseProtocol(proto); + protocol = urlParseProtocol(proto, &dport, &dtype); port = urlDefaultPort(protocol); /* Is there any login informaiton? */ if ((t = strrchr(host, '@'))) { @@ -353,6 +410,7 @@ xstrncpy(request->host, host, SQUIDHOSTNAMELEN); xstrncpy(request->login, login, MAX_LOGIN_SZ); request->port = (u_short) port; + request->dist_port = (u_short) dport; return request; }