--------------------- PatchSet 5338 Date: 2002/10/11 15:40:40 Author: rbcollins Branch: rbcollins_cxxtest Tag: (none) Log: more conversions Members: src/ICP.h:1.1.2.4->1.1.2.5 src/Makefile.am:1.29.2.23->1.29.2.24 src/acl.cc:1.1.2.2->1.1.2.3 src/cache_cf.cc:1.1.2.4->1.1.2.5 src/client_db.cc:1.1.2.1->1.1.2.2 src/client_side.cc:1.1.2.1->1.1.2.2 src/comm.c:1.27.2.1->1.27.2.2(DEAD) src/comm.cc:1.1->1.1.2.1 src/comm_kqueue.c:1.3.6.1->1.3.6.2(DEAD) src/comm_kqueue.cc:1.1->1.1.2.1 src/comm_poll.c:1.6.2.1->1.6.2.2(DEAD) src/comm_poll.cc:1.1->1.1.2.1 src/comm_select.c:1.15.2.1->1.15.2.2(DEAD) src/comm_select.cc:1.1->1.1.2.1 src/debug.c:1.10->1.10.6.1(DEAD) src/debug.cc:1.1->1.1.2.1 src/disk.c:1.12->1.12.2.1(DEAD) src/disk.cc:1.1->1.1.2.1 src/fqdncache.c:1.17.6.1->1.17.6.2(DEAD) src/fqdncache.cc:1.1->1.1.2.1 src/ftp.c:1.28.6.2->1.28.6.3(DEAD) src/ftp.cc:1.1->1.1.2.1 src/gopher.c:1.19.2.1->1.19.2.2(DEAD) src/gopher.cc:1.1->1.1.2.1 src/helper.c:1.26.2.1->1.26.2.2(DEAD) src/helper.cc:1.1->1.1.2.1 src/icp_v2.cc:1.1.2.17->1.1.2.18 src/internal.c:1.9.18.1->1.9.18.2(DEAD) src/internal.cc:1.1->1.1.2.1 src/ipc.c:1.9->1.9.20.1(DEAD) src/ipc.cc:1.1->1.1.2.1 src/net_db.cc:1.1.2.1->1.1.2.2 src/peer_digest.c:1.17.2.1->1.17.2.2(DEAD) src/peer_digest.cc:1.1->1.1.2.1 src/peer_select.c:1.16.6.2->1.16.6.3(DEAD) src/peer_select.cc:1.1->1.1.2.1 src/pinger.c:1.4.96.1->1.4.96.2(DEAD) src/pinger.cc:1.1->1.1.2.1 src/referer.c:1.4->1.4.60.1(DEAD) src/referer.cc:1.1->1.1.2.1 src/refresh.c:1.8.14.1->1.8.14.2(DEAD) src/refresh.cc:1.1->1.1.2.1 src/snmp_agent.cc:1.1.2.1->1.1.2.2 src/snmp_core.c:1.10->1.10.28.1(DEAD) src/snmp_core.cc:1.1->1.1.2.1 src/ssl_support.c:1.7->1.7.14.1(DEAD) src/ssl_support.cc:1.1->1.1.2.1 src/store_key_md5.c:1.6->1.6.66.1(DEAD) src/store_key_md5.cc:1.1->1.1.2.1 src/store_rebuild.c:1.9.44.2->1.9.44.3(DEAD) src/store_rebuild.cc:1.1->1.1.2.1 src/store_swapin.c:1.7.2.1->1.7.2.2(DEAD) src/store_swapin.cc:1.1->1.1.2.1 src/store_swapout.c:1.14.2.1->1.14.2.2(DEAD) src/store_swapout.cc:1.1->1.1.2.1 src/structs.h:1.70.2.8->1.70.2.9 Index: squid/src/ICP.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/ICP.h,v retrieving revision 1.1.2.4 retrieving revision 1.1.2.5 diff -u -r1.1.2.4 -r1.1.2.5 --- squid/src/ICP.h 9 Oct 2002 05:55:49 -0000 1.1.2.4 +++ squid/src/ICP.h 11 Oct 2002 15:40:40 -0000 1.1.2.5 @@ -1,6 +1,6 @@ /* - * $Id: ICP.h,v 1.1.2.4 2002/10/09 05:55:49 rbcollins Exp $ + * $Id: ICP.h,v 1.1.2.5 2002/10/11 15:40:40 rbcollins Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -56,6 +56,7 @@ _icp_common_t (char *buf, unsigned int len); void handleReply(char *buf, struct sockaddr_in *from); static _icp_common_t *createMessage(icp_opcode opcode, int flags, const char *url, int reqnum, int pad); + icp_opcode getOpCode() const; #endif }; Index: squid/src/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v retrieving revision 1.29.2.23 retrieving revision 1.29.2.24 diff -u -r1.29.2.23 -r1.29.2.24 --- squid/src/Makefile.am 9 Oct 2002 14:36:40 -0000 1.29.2.23 +++ squid/src/Makefile.am 11 Oct 2002 15:40:41 -0000 1.29.2.24 @@ -15,7 +15,7 @@ endif if USE_SNMP -SNMPSOURCE = snmp_core.c snmp_agent.cc +SNMPSOURCE = snmp_core.cc snmp_agent.cc else SNMPSOURCE = endif @@ -57,7 +57,7 @@ endif if ENABLE_SSL -SSLSOURCE = ssl_support.c +SSLSOURCE = ssl_support.cc else SSLSOURCE = endif @@ -109,10 +109,10 @@ dns_internal.cc \ htcp.cc \ leakfinder.c \ - snmp_core.c \ + snmp_core.cc \ snmp_agent.cc \ unlinkd.cc \ - ssl_support.c \ + ssl_support.cc \ ssl_support.h \ win32.cc @@ -133,14 +133,14 @@ client_side_request.h \ clientStream.cc \ clientStream.h \ - comm.c \ - comm_select.c \ - comm_poll.c \ - comm_kqueue.c \ - debug.c \ + comm.cc \ + comm_select.cc \ + comm_poll.cc \ + comm_kqueue.cc \ + debug.cc \ defines.h \ $(DELAY_POOL_SOURCE) \ - disk.c \ + disk.cc \ $(DNSSOURCE) \ enums.h \ errorpage.cc \ @@ -150,11 +150,11 @@ fd.cc \ filemap.cc \ forward.c \ - fqdncache.c \ - ftp.c \ + fqdncache.cc \ + ftp.cc \ globals.h \ - gopher.c \ - helper.c \ + gopher.cc \ + helper.cc \ $(HTCPSOURCE) \ http.cc \ HttpStatusLine.cc \ @@ -171,8 +171,8 @@ icp_v2.cc \ icp_v3.cc \ ident.cc \ - internal.c \ - ipc.c \ + internal.cc \ + ipc.cc \ ipcache.cc \ IPInterception.cc \ IPInterception.h \ @@ -188,12 +188,12 @@ Packer.cc \ $(XPROF_STATS_SOURCE) \ pconn.cc \ - peer_digest.c \ - peer_select.c \ + peer_digest.cc \ + peer_select.cc \ protos.h \ redirect.cc \ - referer.c \ - refresh.c \ + referer.cc \ + refresh.cc \ send-announce.cc \ $(SNMPSOURCE) \ squid.h \ @@ -210,12 +210,12 @@ StoreClient.h \ store_digest.cc \ store_dir.cc \ - store_key_md5.c \ + store_key_md5.cc \ store_log.cc \ - store_rebuild.c \ - store_swapin.c \ + store_rebuild.cc \ + store_swapin.cc \ store_swapmeta.cc \ - store_swapout.c \ + store_swapout.cc \ structs.h \ tools.cc \ typedefs.h \ @@ -257,8 +257,8 @@ $(COMPILE) -DUNLINK_DAEMON -c $(srcdir)/unlinkd.cc -o $@ pinger_SOURCES = \ - pinger.c \ - debug.c + pinger.cc \ + debug.cc dnsserver_SOURCES = dnsserver.cc recv_announce_SOURCES = recv-announce.cc Index: squid/src/acl.cc =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/acl.cc,v retrieving revision 1.1.2.2 retrieving revision 1.1.2.3 diff -u -r1.1.2.2 -r1.1.2.3 --- squid/src/acl.cc 9 Oct 2002 00:53:35 -0000 1.1.2.2 +++ squid/src/acl.cc 11 Oct 2002 15:40:42 -0000 1.1.2.3 @@ -1,5 +1,5 @@ /* - * $Id: acl.cc,v 1.1.2.2 2002/10/09 00:53:35 rbcollins Exp $ + * $Id: acl.cc,v 1.1.2.3 2002/10/11 15:40:42 rbcollins Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Duane Wessels @@ -610,7 +610,7 @@ { char *t = NULL; while ((t = strtokFile())) - wordlistAdd(curlist, t); + wordlistAdd((wordlist **)curlist, t); } #endif @@ -1630,7 +1630,7 @@ /* NOTREACHED */ #if SQUID_SNMP case ACL_SNMP_COMMUNITY: - return aclMatchWordList(ae->data, checklist->snmp_community); + return aclMatchWordList((wordlist *)ae->data, checklist->snmp_community); /* NOTREACHED */ #endif case ACL_SRC_ASN: @@ -2476,7 +2476,7 @@ return aclDumpDomainList(a->data); #if SQUID_SNMP case ACL_SNMP_COMMUNITY: - return wordlistDup(a->data); + return wordlistDup((wordlist *)a->data); #endif #if USE_IDENT case ACL_IDENT: Index: squid/src/cache_cf.cc =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/cache_cf.cc,v retrieving revision 1.1.2.4 retrieving revision 1.1.2.5 diff -u -r1.1.2.4 -r1.1.2.5 --- squid/src/cache_cf.cc 6 Oct 2002 06:07:44 -0000 1.1.2.4 +++ squid/src/cache_cf.cc 11 Oct 2002 15:40:42 -0000 1.1.2.5 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.cc,v 1.1.2.4 2002/10/06 06:07:44 rbcollins Exp $ + * $Id: cache_cf.cc,v 1.1.2.5 2002/10/11 15:40:42 rbcollins Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -2378,7 +2378,7 @@ } else { self_destruct(); } - s = xcalloc(1, sizeof(*s)); + s = (https_port_list *)xcalloc(1, sizeof(*s)); s->s.sin_port = htons(port); if (NULL == host) s->s.sin_addr = any_addr; Index: squid/src/client_db.cc =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/client_db.cc,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/client_db.cc 9 Oct 2002 00:23:16 -0000 1.1.2.1 +++ squid/src/client_db.cc 11 Oct 2002 15:40:43 -0000 1.1.2.2 @@ -1,6 +1,6 @@ /* - * $Id: client_db.cc,v 1.1.2.1 2002/10/09 00:23:16 rbcollins Exp $ + * $Id: client_db.cc,v 1.1.2.2 2002/10/11 15:40:43 rbcollins Exp $ * * DEBUG: section 0 Client Database * AUTHOR: Duane Wessels @@ -298,7 +298,7 @@ break; case MESH_CTBL_HTHITS: aggr = 0; - for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; l++) { + for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) { if (logTypeIsATcpHit(l)) aggr += c->Http.result_hist[l]; } Index: squid/src/client_side.cc =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/client_side.cc,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/client_side.cc 9 Oct 2002 14:14:26 -0000 1.1.2.1 +++ squid/src/client_side.cc 11 Oct 2002 15:40:43 -0000 1.1.2.2 @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.1.2.1 2002/10/09 14:14:26 rbcollins Exp $ + * $Id: client_side.cc,v 1.1.2.2 2002/10/11 15:40:43 rbcollins Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -1747,7 +1747,7 @@ static void clientNegotiateSSL(int fd, void *data) { - ConnStateData *conn = data; + ConnStateData *conn = (ConnStateData *)data; X509 *client_cert; int ret; @@ -1795,7 +1795,7 @@ httpsAccept(int sock, void *data) { int *N = &incoming_sockets_accepted; - https_port_data *https_port = data; + https_port_data *https_port = (https_port_data *)data; SSL_CTX *sslContext = https_port->sslContext; int fd = -1; ConnStateData *connState = NULL; --- squid/src/comm.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,1050 +0,0 @@ - -/* - * $Id: comm.c,v 1.27.2.1 2002/10/09 05:09:23 rbcollins Exp $ - * - * DEBUG: section 5 Socket Functions - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -#if defined(_SQUID_CYGWIN_) -#include -#endif -#ifdef HAVE_NETINET_TCP_H -#include -#endif - -typedef struct { - char *host; - u_short port; - struct sockaddr_in S; - CNCB *callback; - void *data; - struct in_addr in_addr; - int locks; - int fd; - int tries; - int addrcount; - int connstart; -} ConnectStateData; - -/* STATIC */ -static int commBind(int s, struct in_addr, u_short port); -static void commSetReuseAddr(int); -static void commSetNoLinger(int); -static void CommWriteStateCallbackAndFree(int fd, comm_err_t code); -#ifdef TCP_NODELAY -static void commSetTcpNoDelay(int); -#endif -static void commSetTcpRcvbuf(int, int); -static PF commConnectFree; -static PF commConnectHandle; -static PF commHandleWrite; -static IPH commConnectDnsHandle; -static void commConnectCallback(ConnectStateData * cs, comm_err_t status); -static int commResetFD(ConnectStateData * cs); -static int commRetryConnect(ConnectStateData * cs); -CBDATA_TYPE(ConnectStateData); - -static MemPool *comm_write_pool = NULL; -static MemPool *conn_close_pool = NULL; - -static void -CommWriteStateCallbackAndFree(int fd, comm_err_t code) -{ - CommWriteStateData *CommWriteState = fd_table[fd].rwstate; - CWCB *callback = NULL; - void *cbdata; - fd_table[fd].rwstate = NULL; - if (CommWriteState == NULL) - return; - if (CommWriteState->free_func) { - FREE *free_func = CommWriteState->free_func; - void *free_buf = CommWriteState->buf; - CommWriteState->free_func = NULL; - CommWriteState->buf = NULL; - free_func(free_buf); - } - callback = CommWriteState->handler; - CommWriteState->handler = NULL; - if (callback && cbdataReferenceValidDone(CommWriteState->handler_data, &cbdata)) - callback(fd, CommWriteState->buf, CommWriteState->offset, code, cbdata); - memPoolFree(comm_write_pool, CommWriteState); -} - -/* Return the local port associated with fd. */ -u_short -comm_local_port(int fd) -{ - struct sockaddr_in addr; - socklen_t addr_len = 0; - fde *F = &fd_table[fd]; - - /* If the fd is closed already, just return */ - if (!F->flags.open) { - debug(5, 0) ("comm_local_port: FD %d has been closed.\n", fd); - return 0; - } - if (F->local_port) - return F->local_port; - addr_len = sizeof(addr); - if (getsockname(fd, (struct sockaddr *) &addr, &addr_len)) { - debug(50, 1) ("comm_local_port: Failed to retrieve TCP/UDP port number for socket: FD %d: %s\n", fd, xstrerror()); - return 0; - } - F->local_port = ntohs(addr.sin_port); - debug(5, 6) ("comm_local_port: FD %d: port %d\n", fd, (int) F->local_port); - return F->local_port; -} - -static comm_err_t -commBind(int s, struct in_addr in_addr, u_short port) -{ - struct sockaddr_in S; - - memset(&S, '\0', sizeof(S)); - S.sin_family = AF_INET; - S.sin_port = htons(port); - S.sin_addr = in_addr; - statCounter.syscalls.sock.binds++; - if (bind(s, (struct sockaddr *) &S, sizeof(S)) == 0) - return COMM_OK; - debug(50, 0) ("commBind: Cannot bind socket FD %d to %s:%d: %s\n", - s, - S.sin_addr.s_addr == INADDR_ANY ? "*" : inet_ntoa(S.sin_addr), - (int) port, - xstrerror()); - return COMM_ERROR; -} - -/* Create a socket. Default is blocking, stream (TCP) socket. IO_TYPE - * is OR of flags specified in comm.h. Defaults TOS */ -int -comm_open(int sock_type, - int proto, - struct in_addr addr, - u_short port, - int flags, - const char *note) -{ - return comm_openex(sock_type, proto, addr, port, flags, 0, note); -} - - -/* Create a socket. Default is blocking, stream (TCP) socket. IO_TYPE - * is OR of flags specified in defines.h:COMM_* */ -int -comm_openex(int sock_type, - int proto, - struct in_addr addr, - u_short port, - int flags, - unsigned char TOS, - const char *note) -{ - int new_socket; - int tos = 0; - fde *F = NULL; - - PROF_start(comm_open); - /* Create socket for accepting new connections. */ - statCounter.syscalls.sock.sockets++; - if ((new_socket = socket(AF_INET, sock_type, proto)) < 0) { - /* Increase the number of reserved fd's if calls to socket() - * are failing because the open file table is full. This - * limits the number of simultaneous clients */ - switch (errno) { - case ENFILE: - case EMFILE: - debug(50, 1) ("comm_open: socket failure: %s\n", xstrerror()); - fdAdjustReserved(); - break; - default: - debug(50, 0) ("comm_open: socket failure: %s\n", xstrerror()); - } - PROF_stop(comm_open); - return -1; - } - /* set TOS if needed */ - if (TOS) { -#ifdef IP_TOS - tos = TOS; - if (setsockopt(new_socket, IPPROTO_IP, IP_TOS, (char *) &tos, sizeof(int)) < 0) - debug(50, 1) ("comm_open: setsockopt(IP_TOS) on FD %d: %s\n", - new_socket, xstrerror()); -#else - debug(50, 0) ("comm_open: setsockopt(IP_TOS) not supported on this platform\n"); -#endif - } - /* update fdstat */ - debug(5, 5) ("comm_open: FD %d is a new socket\n", new_socket); - fd_open(new_socket, FD_SOCKET, note); - F = &fd_table[new_socket]; - F->local_addr = addr; - F->tos = tos; - if (!(flags & COMM_NOCLOEXEC)) - commSetCloseOnExec(new_socket); - if ((flags & COMM_REUSEADDR)) - commSetReuseAddr(new_socket); - if (port > (u_short) 0) { - commSetNoLinger(new_socket); - if (opt_reuseaddr) - commSetReuseAddr(new_socket); - } - if (addr.s_addr != no_addr.s_addr) { - if (commBind(new_socket, addr, port) != COMM_OK) { - comm_close(new_socket); - return -1; - PROF_stop(comm_open); - } - } - F->local_port = port; - - if (flags & COMM_NONBLOCKING) - if (commSetNonBlocking(new_socket) == COMM_ERROR) { - return -1; - PROF_stop(comm_open); - } -#ifdef TCP_NODELAY - if (sock_type == SOCK_STREAM) - commSetTcpNoDelay(new_socket); -#endif - if (Config.tcpRcvBufsz > 0 && sock_type == SOCK_STREAM) - commSetTcpRcvbuf(new_socket, Config.tcpRcvBufsz); - PROF_stop(comm_open); - return new_socket; -} - -/* - * NOTE: set the listen queue to Squid_MaxFD/4 and rely on the kernel to - * impose an upper limit. Solaris' listen(3n) page says it has - * no limit on this parameter, but sys/socket.h sets SOMAXCONN - * to 5. HP-UX currently has a limit of 20. SunOS is 5 and - * OSF 3.0 is 8. - */ -int -comm_listen(int sock) -{ - int x; - if ((x = listen(sock, Squid_MaxFD >> 2)) < 0) { - debug(50, 0) ("comm_listen: listen(%d, %d): %s\n", - Squid_MaxFD >> 2, - sock, xstrerror()); - return x; - } - return sock; -} - -void -commConnectStart(int fd, const char *host, u_short port, CNCB * callback, void *data) -{ - ConnectStateData *cs; - debug(5, 3) ("commConnectStart: FD %d, %s:%d\n", fd, host, (int) port); - cs = cbdataAlloc(ConnectStateData); - cs->fd = fd; - cs->host = xstrdup(host); - cs->port = port; - cs->callback = callback; - cs->data = cbdataReference(data); - comm_add_close_handler(fd, commConnectFree, cs); - cs->locks++; - ipcache_nbgethostbyname(host, commConnectDnsHandle, cs); -} - -static void -commConnectDnsHandle(const ipcache_addrs * ia, void *data) -{ - ConnectStateData *cs = data; - assert(cs->locks == 1); - cs->locks--; - if (ia == NULL) { - debug(5, 3) ("commConnectDnsHandle: Unknown host: %s\n", cs->host); - if (!dns_error_message) { - dns_error_message = "Unknown DNS error"; - debug(5, 1) ("commConnectDnsHandle: Bad dns_error_message\n"); - } - assert(dns_error_message != NULL); - commConnectCallback(cs, COMM_ERR_DNS); - return; - } - assert(ia->cur < ia->count); - cs->in_addr = ia->in_addrs[ia->cur]; - ipcacheCycleAddr(cs->host, NULL); - cs->addrcount = ia->count; - cs->connstart = squid_curtime; - commConnectHandle(cs->fd, cs); -} - -static void -commConnectCallback(ConnectStateData * cs, comm_err_t status) -{ - CNCB *callback = cs->callback; - void *cbdata = cs->data; - int fd = cs->fd; - comm_remove_close_handler(fd, commConnectFree, cs); - cs->callback = NULL; - cs->data = NULL; - commSetTimeout(fd, -1, NULL, NULL); - commConnectFree(fd, cs); - if (cbdataReferenceValid(cbdata)) - callback(fd, status, cbdata); -} - -static void -commConnectFree(int fd, void *data) -{ - ConnectStateData *cs = data; - debug(5, 3) ("commConnectFree: FD %d\n", fd); - cbdataReferenceDone(cs->data); - safe_free(cs->host); - cbdataFree(cs); -} - -/* Reset FD so that we can connect() again */ -static int -commResetFD(ConnectStateData * cs) -{ - int fd2; - fde *F; - if (!cbdataReferenceValid(cs->data)) - return 0; - statCounter.syscalls.sock.sockets++; - fd2 = socket(AF_INET, SOCK_STREAM, 0); - statCounter.syscalls.sock.sockets++; - if (fd2 < 0) { - debug(5, 0) ("commResetFD: socket: %s\n", xstrerror()); - if (ENFILE == errno || EMFILE == errno) - fdAdjustReserved(); - return 0; - } - if (dup2(fd2, cs->fd) < 0) { - debug(5, 0) ("commResetFD: dup2: %s\n", xstrerror()); - if (ENFILE == errno || EMFILE == errno) - fdAdjustReserved(); - close(fd2); - return 0; - } - close(fd2); - F = &fd_table[cs->fd]; - fd_table[cs->fd].flags.called_connect = 0; - /* - * yuck, this has assumptions about comm_open() arguments for - * the original socket - */ - if (commBind(cs->fd, F->local_addr, F->local_port) != COMM_OK) { - debug(5, 0) ("commResetFD: bind: %s\n", xstrerror()); - return 0; - } -#ifdef IP_TOS - if (F->tos) { - int tos = F->tos; - if (setsockopt(cs->fd, IPPROTO_IP, IP_TOS, (char *) &tos, sizeof(int)) < 0) - debug(50, 1) ("commResetFD: setsockopt(IP_TOS) on FD %d: %s\n", cs->fd, xstrerror()); - } -#endif - if (F->flags.close_on_exec) - commSetCloseOnExec(cs->fd); - if (F->flags.nonblocking) - commSetNonBlocking(cs->fd); -#ifdef TCP_NODELAY - if (F->flags.nodelay) - commSetTcpNoDelay(cs->fd); -#endif - if (Config.tcpRcvBufsz > 0) - commSetTcpRcvbuf(cs->fd, Config.tcpRcvBufsz); - return 1; -} - -static int -commRetryConnect(ConnectStateData * cs) -{ - assert(cs->addrcount > 0); - if (cs->addrcount == 1) { - if (cs->tries >= Config.retry.maxtries) - return 0; - if (squid_curtime - cs->connstart > Config.Timeout.connect) - return 0; - } else { - if (cs->tries > cs->addrcount) - return 0; - } - return commResetFD(cs); -} - -/* Connect SOCK to specified DEST_PORT at DEST_HOST. */ -static void -commConnectHandle(int fd, void *data) -{ - ConnectStateData *cs = data; - if (cs->S.sin_addr.s_addr == 0) { - cs->S.sin_family = AF_INET; - cs->S.sin_addr = cs->in_addr; - cs->S.sin_port = htons(cs->port); - if (Config.onoff.log_fqdn) - fqdncache_gethostbyaddr(cs->S.sin_addr, FQDN_LOOKUP_IF_MISS); - } - switch (comm_connect_addr(fd, &cs->S)) { - case COMM_INPROGRESS: - debug(5, 5) ("commConnectHandle: FD %d: COMM_INPROGRESS\n", fd); - commSetSelect(fd, COMM_SELECT_WRITE, commConnectHandle, cs, 0); - break; - case COMM_OK: - ipcacheMarkGoodAddr(cs->host, cs->S.sin_addr); - commConnectCallback(cs, COMM_OK); - break; - default: - cs->tries++; - ipcacheMarkBadAddr(cs->host, cs->S.sin_addr); - if (Config.onoff.test_reachability) - netdbDeleteAddrNetwork(cs->S.sin_addr); - if (commRetryConnect(cs)) { - cs->locks++; - ipcache_nbgethostbyname(cs->host, commConnectDnsHandle, cs); - } else { - commConnectCallback(cs, COMM_ERR_CONNECT); - } - break; - } -} - -int -commSetTimeout(int fd, int timeout, PF * handler, void *data) -{ - fde *F; - debug(5, 3) ("commSetTimeout: FD %d timeout %d\n", fd, timeout); - assert(fd >= 0); - assert(fd < Squid_MaxFD); - F = &fd_table[fd]; - assert(F->flags.open); - if (timeout < 0) { - cbdataReferenceDone(F->timeout_data); - F->timeout_handler = NULL; - F->timeout = 0; - } else { - assert(handler || F->timeout_handler); - if (handler) { - cbdataReferenceDone(F->timeout_data); - F->timeout_handler = handler; - F->timeout_data = cbdataReference(data); - } - F->timeout = squid_curtime + (time_t) timeout; - } - return F->timeout; -} - -int -comm_connect_addr(int sock, const struct sockaddr_in *address) -{ - comm_err_t status = COMM_OK; - fde *F = &fd_table[sock]; - int x; - int err = 0; - socklen_t errlen; - assert(ntohs(address->sin_port) != 0); - PROF_start(comm_connect_addr); - /* Establish connection. */ - errno = 0; - if (!F->flags.called_connect) { - F->flags.called_connect = 1; - statCounter.syscalls.sock.connects++; - x = connect(sock, (struct sockaddr *) address, sizeof(*address)); - if (x < 0) - debug(5, 9) ("connect FD %d: %s\n", sock, xstrerror()); - } else { -#if defined(_SQUID_NEWSOS6_) - /* Makoto MATSUSHITA */ - connect(sock, (struct sockaddr *) address, sizeof(*address)); - if (errno == EINVAL) { - errlen = sizeof(err); - x = getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &errlen); - if (x >= 0) - errno = x; - } -#else - errlen = sizeof(err); - x = getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &errlen); - if (x == 0) - errno = err; -#if defined(_SQUID_SOLARIS_) - /* - * Solaris 2.4's socket emulation doesn't allow you - * to determine the error from a failed non-blocking - * connect and just returns EPIPE. Create a fake - * error message for connect. -- fenner@parc.xerox.com - */ - if (x < 0 && errno == EPIPE) - errno = ENOTCONN; -#endif -#endif - } - PROF_stop(comm_connect_addr); - if (errno == 0 || errno == EISCONN) - status = COMM_OK; - else if (ignoreErrno(errno)) - status = COMM_INPROGRESS; - else - return COMM_ERROR; - xstrncpy(F->ipaddr, inet_ntoa(address->sin_addr), 16); - F->remote_port = ntohs(address->sin_port); - if (status == COMM_OK) { - debug(5, 10) ("comm_connect_addr: FD %d connected to %s:%d\n", - sock, F->ipaddr, F->remote_port); - } else if (status == COMM_INPROGRESS) { - debug(5, 10) ("comm_connect_addr: FD %d connection pending\n", sock); - } - return status; -} - -/* Wait for an incoming connection on FD. FD should be a socket returned - * from comm_listen. */ -int -comm_accept(int fd, struct sockaddr_in *pn, struct sockaddr_in *me) -{ - int sock; - struct sockaddr_in P; - struct sockaddr_in M; - socklen_t Slen; - fde *F = NULL; - Slen = sizeof(P); - statCounter.syscalls.sock.accepts++; - PROF_start(comm_accept); - if ((sock = accept(fd, (struct sockaddr *) &P, &Slen)) < 0) { - PROF_stop(comm_accept); - if (ignoreErrno(errno)) { - debug(50, 5) ("comm_accept: FD %d: %s\n", fd, xstrerror()); - return COMM_NOMESSAGE; - } else if (ENFILE == errno || EMFILE == errno) { - debug(50, 3) ("comm_accept: FD %d: %s\n", fd, xstrerror()); - return COMM_ERROR; - } else { - debug(50, 1) ("comm_accept: FD %d: %s\n", fd, xstrerror()); - return COMM_ERROR; - } - } - if (pn) - *pn = P; - Slen = sizeof(M); - memset(&M, '\0', Slen); - getsockname(sock, (struct sockaddr *) &M, &Slen); - if (me) - *me = M; - commSetCloseOnExec(sock); - /* fdstat update */ - fd_open(sock, FD_SOCKET, "HTTP Request"); - F = &fd_table[sock]; - xstrncpy(F->ipaddr, inet_ntoa(P.sin_addr), 16); - F->remote_port = htons(P.sin_port); - F->local_port = htons(M.sin_port); - commSetNonBlocking(sock); - PROF_stop(comm_accept); - return sock; -} - -void -commCallCloseHandlers(int fd) -{ - fde *F = &fd_table[fd]; - close_handler *ch; - debug(5, 5) ("commCallCloseHandlers: FD %d\n", fd); - while ((ch = F->closeHandler) != NULL) { - F->closeHandler = ch->next; - debug(5, 5) ("commCallCloseHandlers: ch->handler=%p\n", ch->handler); - if (cbdataReferenceValid(ch->data)) - ch->handler(fd, ch->data); - cbdataReferenceDone(ch->data); - memPoolFree(conn_close_pool, ch); /* AAA */ - } -} - -#if LINGERING_CLOSE -static void -commLingerClose(int fd, void *unused) -{ - LOCAL_ARRAY(char, buf, 1024); - int n; - n = FD_READ_METHOD(fd, buf, 1024); - if (n < 0) - debug(5, 3) ("commLingerClose: FD %d read: %s\n", fd, xstrerror()); - comm_close(fd); -} - -static void -commLingerTimeout(int fd, void *unused) -{ - debug(5, 3) ("commLingerTimeout: FD %d\n", fd); - comm_close(fd); -} - -/* - * Inspired by apache - */ -void -comm_lingering_close(int fd) -{ -#if USE_SSL - if (fd_table[fd].ssl) - ssl_shutdown_method(fd); -#endif - if (shutdown(fd, 1) < 0) { - comm_close(fd); - return; - } - fd_note(fd, "lingering close"); - commSetTimeout(fd, 10, commLingerTimeout, NULL); - commSetSelect(fd, COMM_SELECT_READ, commLingerClose, NULL, 0); -} -#endif - -/* - * enable linger with time of 0 so that when the socket is - * closed, TCP generates a RESET - */ -void -comm_reset_close(int fd) -{ - struct linger L; - L.l_onoff = 1; - L.l_linger = 0; - if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char *) &L, sizeof(L)) < 0) - debug(50, 0) ("commResetTCPClose: FD %d: %s\n", fd, xstrerror()); - comm_close(fd); -} - -void -comm_close(int fd) -{ - fde *F = NULL; - - debug(5, 5) ("comm_close: FD %d\n", fd); - assert(fd >= 0); - assert(fd < Squid_MaxFD); - F = &fd_table[fd]; - - if (F->flags.closing) - return; - if (shutting_down && (!F->flags.open || F->type == FD_FILE)) - return; - assert(F->flags.open); - assert(F->type != FD_FILE); - PROF_start(comm_close); - F->flags.closing = 1; -#if USE_SSL - if (F->ssl) - ssl_shutdown_method(fd); -#endif - commSetTimeout(fd, -1, NULL, NULL); - CommWriteStateCallbackAndFree(fd, COMM_ERR_CLOSING); - commCallCloseHandlers(fd); - if (F->uses) /* assume persistent connect count */ - pconnHistCount(1, F->uses); -#if USE_SSL - if (F->ssl) { - SSL_free(F->ssl); - F->ssl = NULL; - } -#endif - fd_close(fd); /* update fdstat */ - close(fd); - statCounter.syscalls.sock.closes++; - PROF_stop(comm_close); -} - -/* Send a udp datagram to specified TO_ADDR. */ -int -comm_udp_sendto(int fd, - const struct sockaddr_in *to_addr, - int addr_len, - const void *buf, - int len) -{ - int x; - PROF_start(comm_udp_sendto); - statCounter.syscalls.sock.sendtos++; - x = sendto(fd, buf, len, 0, (struct sockaddr *) to_addr, addr_len); - PROF_stop(comm_udp_sendto); - if (x < 0) { -#ifdef _SQUID_LINUX_ - if (ECONNREFUSED != errno) -#endif - debug(50, 1) ("comm_udp_sendto: FD %d, %s, port %d: %s\n", - fd, - inet_ntoa(to_addr->sin_addr), - (int) htons(to_addr->sin_port), - xstrerror()); - return COMM_ERROR; - } - return x; -} - -void -commSetDefer(int fd, DEFER * func, void *data) -{ - fde *F = &fd_table[fd]; - F->defer_check = func; - F->defer_data = data; -} - -void -comm_add_close_handler(int fd, PF * handler, void *data) -{ - close_handler *new = memPoolAlloc(conn_close_pool); /* AAA */ - close_handler *c; - debug(5, 5) ("comm_add_close_handler: FD %d, handler=%p, data=%p\n", - fd, handler, data); - for (c = fd_table[fd].closeHandler; c; c = c->next) - assert(c->handler != handler || c->data != data); - new->handler = handler; - new->data = cbdataReference(data); - new->next = fd_table[fd].closeHandler; - fd_table[fd].closeHandler = new; -} - -void -comm_remove_close_handler(int fd, PF * handler, void *data) -{ - close_handler *p; - close_handler *last = NULL; - /* Find handler in list */ - debug(5, 5) ("comm_remove_close_handler: FD %d, handler=%p, data=%p\n", - fd, handler, data); - for (p = fd_table[fd].closeHandler; p != NULL; last = p, p = p->next) - if (p->handler == handler && p->data == data) - break; /* This is our handler */ - assert(p != NULL); - /* Remove list entry */ - if (last) - last->next = p->next; - else - fd_table[fd].closeHandler = p->next; - cbdataReferenceDone(p->data); - memPoolFree(conn_close_pool, p); -} - -static void -commSetNoLinger(int fd) -{ - struct linger L; - L.l_onoff = 0; /* off */ - L.l_linger = 0; - if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char *) &L, sizeof(L)) < 0) - debug(50, 0) ("commSetNoLinger: FD %d: %s\n", fd, xstrerror()); - fd_table[fd].flags.nolinger = 1; -} - -static void -commSetReuseAddr(int fd) -{ - int on = 1; - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on)) < 0) - debug(50, 1) ("commSetReuseAddr: FD %d: %s\n", fd, xstrerror()); -} - -static void -commSetTcpRcvbuf(int fd, int size) -{ - if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (char *) &size, sizeof(size)) < 0) - debug(50, 1) ("commSetTcpRcvbuf: FD %d, SIZE %d: %s\n", - fd, size, xstrerror()); -} - -int -commSetNonBlocking(int fd) -{ - int flags; - int dummy = 0; -#ifdef _SQUID_CYGWIN_ - int nonblocking = TRUE; - if (fd_table[fd].type != FD_PIPE) { - if (ioctl(fd, FIONBIO, &nonblocking) < 0) { - debug(50, 0) ("commSetNonBlocking: FD %d: %s %u\n", fd, xstrerror(), fd_table[fd].type); - return COMM_ERROR; - } - } else { -#endif - if ((flags = fcntl(fd, F_GETFL, dummy)) < 0) { - debug(50, 0) ("FD %d: fcntl F_GETFL: %s\n", fd, xstrerror()); - return COMM_ERROR; - } - if (fcntl(fd, F_SETFL, flags | SQUID_NONBLOCK) < 0) { - debug(50, 0) ("commSetNonBlocking: FD %d: %s\n", fd, xstrerror()); - return COMM_ERROR; - } -#ifdef _SQUID_CYGWIN_ - } -#endif - fd_table[fd].flags.nonblocking = 1; - return 0; -} - -int -commUnsetNonBlocking(int fd) -{ - int flags; - int dummy = 0; - if ((flags = fcntl(fd, F_GETFL, dummy)) < 0) { - debug(50, 0) ("FD %d: fcntl F_GETFL: %s\n", fd, xstrerror()); - return COMM_ERROR; - } - if (fcntl(fd, F_SETFL, flags & (~SQUID_NONBLOCK)) < 0) { - debug(50, 0) ("commUnsetNonBlocking: FD %d: %s\n", fd, xstrerror()); - return COMM_ERROR; - } - fd_table[fd].flags.nonblocking = 0; - return 0; -} - -void -commSetCloseOnExec(int fd) -{ -#ifdef FD_CLOEXEC - int flags; - int dummy = 0; - if ((flags = fcntl(fd, F_GETFL, dummy)) < 0) { - debug(50, 0) ("FD %d: fcntl F_GETFL: %s\n", fd, xstrerror()); - return; - } - if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) - debug(50, 0) ("FD %d: set close-on-exec failed: %s\n", fd, xstrerror()); - fd_table[fd].flags.close_on_exec = 1; -#endif -} - -#ifdef TCP_NODELAY -static void -commSetTcpNoDelay(int fd) -{ - int on = 1; - if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &on, sizeof(on)) < 0) - debug(50, 1) ("commSetTcpNoDelay: FD %d: %s\n", fd, xstrerror()); - fd_table[fd].flags.nodelay = 1; -} -#endif - - -void -comm_init(void) -{ - fd_table = xcalloc(Squid_MaxFD, sizeof(fde)); - /* XXX account fd_table */ - /* Keep a few file descriptors free so that we don't run out of FD's - * after accepting a client but before it opens a socket or a file. - * Since Squid_MaxFD can be as high as several thousand, don't waste them */ - RESERVED_FD = XMIN(100, Squid_MaxFD / 4); - CBDATA_INIT_TYPE(ConnectStateData); - comm_write_pool = memPoolCreate("CommWriteStateData", sizeof(CommWriteStateData)); - conn_close_pool = memPoolCreate("close_handler", sizeof(close_handler)); -} - -/* Write to FD. */ -static void -commHandleWrite(int fd, void *data) -{ - CommWriteStateData *state = data; - int len = 0; - int nleft; - - PROF_start(commHandleWrite); - debug(5, 5) ("commHandleWrite: FD %d: off %ld, sz %ld.\n", - fd, (long int) state->offset, (long int) state->size); - - nleft = state->size - state->offset; - len = FD_WRITE_METHOD(fd, state->buf + state->offset, nleft); - debug(5, 5) ("commHandleWrite: write() returns %d\n", len); - fd_bytes(fd, len, FD_WRITE); - statCounter.syscalls.sock.writes++; - - if (len == 0) { - /* Note we even call write if nleft == 0 */ - /* We're done */ - if (nleft != 0) - debug(5, 1) ("commHandleWrite: FD %d: write failure: connection closed with %d bytes remaining.\n", fd, nleft); - CommWriteStateCallbackAndFree(fd, nleft ? COMM_ERROR : COMM_OK); - } else if (len < 0) { - /* An error */ - if (fd_table[fd].flags.socket_eof) { - debug(50, 2) ("commHandleWrite: FD %d: write failure: %s.\n", - fd, xstrerror()); - CommWriteStateCallbackAndFree(fd, COMM_ERROR); - } else if (ignoreErrno(errno)) { - debug(50, 10) ("commHandleWrite: FD %d: write failure: %s.\n", - fd, xstrerror()); - commSetSelect(fd, - COMM_SELECT_WRITE, - commHandleWrite, - state, - 0); - } else { - debug(50, 2) ("commHandleWrite: FD %d: write failure: %s.\n", - fd, xstrerror()); - CommWriteStateCallbackAndFree(fd, COMM_ERROR); - } - } else { - /* A successful write, continue */ - state->offset += len; - if (state->offset < state->size) { - /* Not done, reinstall the write handler and write some more */ - commSetSelect(fd, - COMM_SELECT_WRITE, - commHandleWrite, - state, - 0); - } else { - CommWriteStateCallbackAndFree(fd, COMM_OK); - } - } - PROF_stop(commHandleWrite); -} - - - -/* - * Queue a write. handler/handler_data are called when the write - * completes, on error, or on file descriptor close. - * - * free_func is used to free the passed buffer when the write has completed. - */ -void -comm_write(int fd, const char *buf, int size, CWCB * handler, void *handler_data, FREE * free_func) -{ - CommWriteStateData *state = fd_table[fd].rwstate; - debug(5, 5) ("comm_write: FD %d: sz %d: hndl %p: data %p.\n", - fd, size, handler, handler_data); - if (NULL != state) { - debug(5, 1) ("comm_write: fd_table[%d].rwstate != NULL\n", fd); - memPoolFree(comm_write_pool, state); - fd_table[fd].rwstate = NULL; - } - fd_table[fd].rwstate = state = memPoolAlloc(comm_write_pool); - state->buf = (char *) buf; - state->size = size; - state->offset = 0; - state->handler = handler; - state->handler_data = cbdataReference(handler_data); - state->free_func = free_func; - commSetSelect(fd, COMM_SELECT_WRITE, commHandleWrite, state, 0); -} - -/* a wrapper around comm_write to allow for MemBuf to be comm_written in a snap */ -void -comm_write_mbuf(int fd, MemBuf mb, CWCB * handler, void *handler_data) -{ - comm_write(fd, mb.buf, mb.size, handler, handler_data, memBufFreeFunc(&mb)); -} - -/* - * hm, this might be too general-purpose for all the places we'd - * like to use it. - */ -int -ignoreErrno(int ierrno) -{ - switch (ierrno) { - case EINPROGRESS: - case EWOULDBLOCK: -#if EAGAIN != EWOULDBLOCK - case EAGAIN: -#endif - case EALREADY: - case EINTR: -#ifdef ERESTART - case ERESTART: -#endif - return 1; - default: - return 0; - } - /* NOTREACHED */ -} - -void -commCloseAllSockets(void) -{ - int fd; - fde *F = NULL; - for (fd = 0; fd <= Biggest_FD; fd++) { - F = &fd_table[fd]; - if (!F->flags.open) - continue; - if (F->type != FD_SOCKET) - continue; - if (F->flags.ipc) /* don't close inter-process sockets */ - continue; - if (F->timeout_handler) { - PF *callback = F->timeout_handler; - void *cbdata = NULL; - F->timeout_handler = NULL; - debug(5, 5) ("commCloseAllSockets: FD %d: Calling timeout handler\n", - fd); - if (cbdataReferenceValidDone(F->timeout_data, &cbdata)) - callback(fd, cbdata); - } else { - debug(5, 5) ("commCloseAllSockets: FD %d: calling comm_close()\n", fd); - comm_close(fd); - } - } -} - -void -checkTimeouts(void) -{ - int fd; - fde *F = NULL; - PF *callback; - for (fd = 0; fd <= Biggest_FD; fd++) { - F = &fd_table[fd]; - if (!F->flags.open) - continue; - if (F->timeout == 0) - continue; - if (F->timeout > squid_curtime) - continue; - debug(5, 5) ("checkTimeouts: FD %d Expired\n", fd); - if (F->timeout_handler) { - debug(5, 5) ("checkTimeouts: FD %d: Call timeout handler\n", fd); - callback = F->timeout_handler; - F->timeout_handler = NULL; - callback(fd, F->timeout_data); - } else { - debug(5, 5) ("checkTimeouts: FD %d: Forcing comm_close()\n", fd); - comm_close(fd); - } - } -} - - -int -commDeferRead(int fd) -{ - fde *F = &fd_table[fd]; - if (F->defer_check == NULL) - return 0; - return F->defer_check(fd, F->defer_data); -} --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/comm.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,1050 @@ + +/* + * $Id: comm.cc,v 1.1.2.1 2002/10/11 15:40:44 rbcollins Exp $ + * + * DEBUG: section 5 Socket Functions + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +#if defined(_SQUID_CYGWIN_) +#include +#endif +#ifdef HAVE_NETINET_TCP_H +#include +#endif + +typedef struct { + char *host; + u_short port; + struct sockaddr_in S; + CNCB *callback; + void *data; + struct in_addr in_addr; + int locks; + int fd; + int tries; + int addrcount; + int connstart; +} ConnectStateData; + +/* STATIC */ +static comm_err_t commBind(int s, struct in_addr, u_short port); +static void commSetReuseAddr(int); +static void commSetNoLinger(int); +static void CommWriteStateCallbackAndFree(int fd, comm_err_t code); +#ifdef TCP_NODELAY +static void commSetTcpNoDelay(int); +#endif +static void commSetTcpRcvbuf(int, int); +static PF commConnectFree; +static PF commConnectHandle; +static PF commHandleWrite; +static IPH commConnectDnsHandle; +static void commConnectCallback(ConnectStateData * cs, comm_err_t status); +static int commResetFD(ConnectStateData * cs); +static int commRetryConnect(ConnectStateData * cs); +CBDATA_TYPE(ConnectStateData); + +static MemPool *comm_write_pool = NULL; +static MemPool *conn_close_pool = NULL; + +static void +CommWriteStateCallbackAndFree(int fd, comm_err_t code) +{ + CommWriteStateData *CommWriteState = fd_table[fd].rwstate; + CWCB *callback = NULL; + void *cbdata; + fd_table[fd].rwstate = NULL; + if (CommWriteState == NULL) + return; + if (CommWriteState->free_func) { + FREE *free_func = CommWriteState->free_func; + void *free_buf = CommWriteState->buf; + CommWriteState->free_func = NULL; + CommWriteState->buf = NULL; + free_func(free_buf); + } + callback = CommWriteState->handler; + CommWriteState->handler = NULL; + if (callback && cbdataReferenceValidDone(CommWriteState->handler_data, &cbdata)) + callback(fd, CommWriteState->buf, CommWriteState->offset, code, cbdata); + memPoolFree(comm_write_pool, CommWriteState); +} + +/* Return the local port associated with fd. */ +u_short +comm_local_port(int fd) +{ + struct sockaddr_in addr; + socklen_t addr_len = 0; + fde *F = &fd_table[fd]; + + /* If the fd is closed already, just return */ + if (!F->flags.open) { + debug(5, 0) ("comm_local_port: FD %d has been closed.\n", fd); + return 0; + } + if (F->local_port) + return F->local_port; + addr_len = sizeof(addr); + if (getsockname(fd, (struct sockaddr *) &addr, &addr_len)) { + debug(50, 1) ("comm_local_port: Failed to retrieve TCP/UDP port number for socket: FD %d: %s\n", fd, xstrerror()); + return 0; + } + F->local_port = ntohs(addr.sin_port); + debug(5, 6) ("comm_local_port: FD %d: port %d\n", fd, (int) F->local_port); + return F->local_port; +} + +static comm_err_t +commBind(int s, struct in_addr in_addr, u_short port) +{ + struct sockaddr_in S; + + memset(&S, '\0', sizeof(S)); + S.sin_family = AF_INET; + S.sin_port = htons(port); + S.sin_addr = in_addr; + statCounter.syscalls.sock.binds++; + if (bind(s, (struct sockaddr *) &S, sizeof(S)) == 0) + return COMM_OK; + debug(50, 0) ("commBind: Cannot bind socket FD %d to %s:%d: %s\n", + s, + S.sin_addr.s_addr == INADDR_ANY ? "*" : inet_ntoa(S.sin_addr), + (int) port, + xstrerror()); + return COMM_ERROR; +} + +/* Create a socket. Default is blocking, stream (TCP) socket. IO_TYPE + * is OR of flags specified in comm.h. Defaults TOS */ +int +comm_open(int sock_type, + int proto, + struct in_addr addr, + u_short port, + int flags, + const char *note) +{ + return comm_openex(sock_type, proto, addr, port, flags, 0, note); +} + + +/* Create a socket. Default is blocking, stream (TCP) socket. IO_TYPE + * is OR of flags specified in defines.h:COMM_* */ +int +comm_openex(int sock_type, + int proto, + struct in_addr addr, + u_short port, + int flags, + unsigned char TOS, + const char *note) +{ + int new_socket; + int tos = 0; + fde *F = NULL; + + PROF_start(comm_open); + /* Create socket for accepting new connections. */ + statCounter.syscalls.sock.sockets++; + if ((new_socket = socket(AF_INET, sock_type, proto)) < 0) { + /* Increase the number of reserved fd's if calls to socket() + * are failing because the open file table is full. This + * limits the number of simultaneous clients */ + switch (errno) { + case ENFILE: + case EMFILE: + debug(50, 1) ("comm_open: socket failure: %s\n", xstrerror()); + fdAdjustReserved(); + break; + default: + debug(50, 0) ("comm_open: socket failure: %s\n", xstrerror()); + } + PROF_stop(comm_open); + return -1; + } + /* set TOS if needed */ + if (TOS) { +#ifdef IP_TOS + tos = TOS; + if (setsockopt(new_socket, IPPROTO_IP, IP_TOS, (char *) &tos, sizeof(int)) < 0) + debug(50, 1) ("comm_open: setsockopt(IP_TOS) on FD %d: %s\n", + new_socket, xstrerror()); +#else + debug(50, 0) ("comm_open: setsockopt(IP_TOS) not supported on this platform\n"); +#endif + } + /* update fdstat */ + debug(5, 5) ("comm_open: FD %d is a new socket\n", new_socket); + fd_open(new_socket, FD_SOCKET, note); + F = &fd_table[new_socket]; + F->local_addr = addr; + F->tos = tos; + if (!(flags & COMM_NOCLOEXEC)) + commSetCloseOnExec(new_socket); + if ((flags & COMM_REUSEADDR)) + commSetReuseAddr(new_socket); + if (port > (u_short) 0) { + commSetNoLinger(new_socket); + if (opt_reuseaddr) + commSetReuseAddr(new_socket); + } + if (addr.s_addr != no_addr.s_addr) { + if (commBind(new_socket, addr, port) != COMM_OK) { + comm_close(new_socket); + return -1; + PROF_stop(comm_open); + } + } + F->local_port = port; + + if (flags & COMM_NONBLOCKING) + if (commSetNonBlocking(new_socket) == COMM_ERROR) { + return -1; + PROF_stop(comm_open); + } +#ifdef TCP_NODELAY + if (sock_type == SOCK_STREAM) + commSetTcpNoDelay(new_socket); +#endif + if (Config.tcpRcvBufsz > 0 && sock_type == SOCK_STREAM) + commSetTcpRcvbuf(new_socket, Config.tcpRcvBufsz); + PROF_stop(comm_open); + return new_socket; +} + +/* + * NOTE: set the listen queue to Squid_MaxFD/4 and rely on the kernel to + * impose an upper limit. Solaris' listen(3n) page says it has + * no limit on this parameter, but sys/socket.h sets SOMAXCONN + * to 5. HP-UX currently has a limit of 20. SunOS is 5 and + * OSF 3.0 is 8. + */ +int +comm_listen(int sock) +{ + int x; + if ((x = listen(sock, Squid_MaxFD >> 2)) < 0) { + debug(50, 0) ("comm_listen: listen(%d, %d): %s\n", + Squid_MaxFD >> 2, + sock, xstrerror()); + return x; + } + return sock; +} + +void +commConnectStart(int fd, const char *host, u_short port, CNCB * callback, void *data) +{ + ConnectStateData *cs; + debug(5, 3) ("commConnectStart: FD %d, %s:%d\n", fd, host, (int) port); + cs = cbdataAlloc(ConnectStateData); + cs->fd = fd; + cs->host = xstrdup(host); + cs->port = port; + cs->callback = callback; + cs->data = cbdataReference(data); + comm_add_close_handler(fd, commConnectFree, cs); + cs->locks++; + ipcache_nbgethostbyname(host, commConnectDnsHandle, cs); +} + +static void +commConnectDnsHandle(const ipcache_addrs * ia, void *data) +{ + ConnectStateData *cs = (ConnectStateData *)data; + assert(cs->locks == 1); + cs->locks--; + if (ia == NULL) { + debug(5, 3) ("commConnectDnsHandle: Unknown host: %s\n", cs->host); + if (!dns_error_message) { + dns_error_message = "Unknown DNS error"; + debug(5, 1) ("commConnectDnsHandle: Bad dns_error_message\n"); + } + assert(dns_error_message != NULL); + commConnectCallback(cs, COMM_ERR_DNS); + return; + } + assert(ia->cur < ia->count); + cs->in_addr = ia->in_addrs[ia->cur]; + ipcacheCycleAddr(cs->host, NULL); + cs->addrcount = ia->count; + cs->connstart = squid_curtime; + commConnectHandle(cs->fd, cs); +} + +static void +commConnectCallback(ConnectStateData * cs, comm_err_t status) +{ + CNCB *callback = cs->callback; + void *cbdata = cs->data; + int fd = cs->fd; + comm_remove_close_handler(fd, commConnectFree, cs); + cs->callback = NULL; + cs->data = NULL; + commSetTimeout(fd, -1, NULL, NULL); + commConnectFree(fd, cs); + if (cbdataReferenceValid(cbdata)) + callback(fd, status, cbdata); +} + +static void +commConnectFree(int fd, void *data) +{ + ConnectStateData *cs = (ConnectStateData *)data; + debug(5, 3) ("commConnectFree: FD %d\n", fd); + cbdataReferenceDone(cs->data); + safe_free(cs->host); + cbdataFree(cs); +} + +/* Reset FD so that we can connect() again */ +static int +commResetFD(ConnectStateData * cs) +{ + int fd2; + fde *F; + if (!cbdataReferenceValid(cs->data)) + return 0; + statCounter.syscalls.sock.sockets++; + fd2 = socket(AF_INET, SOCK_STREAM, 0); + statCounter.syscalls.sock.sockets++; + if (fd2 < 0) { + debug(5, 0) ("commResetFD: socket: %s\n", xstrerror()); + if (ENFILE == errno || EMFILE == errno) + fdAdjustReserved(); + return 0; + } + if (dup2(fd2, cs->fd) < 0) { + debug(5, 0) ("commResetFD: dup2: %s\n", xstrerror()); + if (ENFILE == errno || EMFILE == errno) + fdAdjustReserved(); + close(fd2); + return 0; + } + close(fd2); + F = &fd_table[cs->fd]; + fd_table[cs->fd].flags.called_connect = 0; + /* + * yuck, this has assumptions about comm_open() arguments for + * the original socket + */ + if (commBind(cs->fd, F->local_addr, F->local_port) != COMM_OK) { + debug(5, 0) ("commResetFD: bind: %s\n", xstrerror()); + return 0; + } +#ifdef IP_TOS + if (F->tos) { + int tos = F->tos; + if (setsockopt(cs->fd, IPPROTO_IP, IP_TOS, (char *) &tos, sizeof(int)) < 0) + debug(50, 1) ("commResetFD: setsockopt(IP_TOS) on FD %d: %s\n", cs->fd, xstrerror()); + } +#endif + if (F->flags.close_on_exec) + commSetCloseOnExec(cs->fd); + if (F->flags.nonblocking) + commSetNonBlocking(cs->fd); +#ifdef TCP_NODELAY + if (F->flags.nodelay) + commSetTcpNoDelay(cs->fd); +#endif + if (Config.tcpRcvBufsz > 0) + commSetTcpRcvbuf(cs->fd, Config.tcpRcvBufsz); + return 1; +} + +static int +commRetryConnect(ConnectStateData * cs) +{ + assert(cs->addrcount > 0); + if (cs->addrcount == 1) { + if (cs->tries >= Config.retry.maxtries) + return 0; + if (squid_curtime - cs->connstart > Config.Timeout.connect) + return 0; + } else { + if (cs->tries > cs->addrcount) + return 0; + } + return commResetFD(cs); +} + +/* Connect SOCK to specified DEST_PORT at DEST_HOST. */ +static void +commConnectHandle(int fd, void *data) +{ + ConnectStateData *cs = (ConnectStateData *)data; + if (cs->S.sin_addr.s_addr == 0) { + cs->S.sin_family = AF_INET; + cs->S.sin_addr = cs->in_addr; + cs->S.sin_port = htons(cs->port); + if (Config.onoff.log_fqdn) + fqdncache_gethostbyaddr(cs->S.sin_addr, FQDN_LOOKUP_IF_MISS); + } + switch (comm_connect_addr(fd, &cs->S)) { + case COMM_INPROGRESS: + debug(5, 5) ("commConnectHandle: FD %d: COMM_INPROGRESS\n", fd); + commSetSelect(fd, COMM_SELECT_WRITE, commConnectHandle, cs, 0); + break; + case COMM_OK: + ipcacheMarkGoodAddr(cs->host, cs->S.sin_addr); + commConnectCallback(cs, COMM_OK); + break; + default: + cs->tries++; + ipcacheMarkBadAddr(cs->host, cs->S.sin_addr); + if (Config.onoff.test_reachability) + netdbDeleteAddrNetwork(cs->S.sin_addr); + if (commRetryConnect(cs)) { + cs->locks++; + ipcache_nbgethostbyname(cs->host, commConnectDnsHandle, cs); + } else { + commConnectCallback(cs, COMM_ERR_CONNECT); + } + break; + } +} + +int +commSetTimeout(int fd, int timeout, PF * handler, void *data) +{ + fde *F; + debug(5, 3) ("commSetTimeout: FD %d timeout %d\n", fd, timeout); + assert(fd >= 0); + assert(fd < Squid_MaxFD); + F = &fd_table[fd]; + assert(F->flags.open); + if (timeout < 0) { + cbdataReferenceDone(F->timeout_data); + F->timeout_handler = NULL; + F->timeout = 0; + } else { + assert(handler || F->timeout_handler); + if (handler) { + cbdataReferenceDone(F->timeout_data); + F->timeout_handler = handler; + F->timeout_data = cbdataReference(data); + } + F->timeout = squid_curtime + (time_t) timeout; + } + return F->timeout; +} + +int +comm_connect_addr(int sock, const struct sockaddr_in *address) +{ + comm_err_t status = COMM_OK; + fde *F = &fd_table[sock]; + int x; + int err = 0; + socklen_t errlen; + assert(ntohs(address->sin_port) != 0); + PROF_start(comm_connect_addr); + /* Establish connection. */ + errno = 0; + if (!F->flags.called_connect) { + F->flags.called_connect = 1; + statCounter.syscalls.sock.connects++; + x = connect(sock, (struct sockaddr *) address, sizeof(*address)); + if (x < 0) + debug(5, 9) ("connect FD %d: %s\n", sock, xstrerror()); + } else { +#if defined(_SQUID_NEWSOS6_) + /* Makoto MATSUSHITA */ + connect(sock, (struct sockaddr *) address, sizeof(*address)); + if (errno == EINVAL) { + errlen = sizeof(err); + x = getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &errlen); + if (x >= 0) + errno = x; + } +#else + errlen = sizeof(err); + x = getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &errlen); + if (x == 0) + errno = err; +#if defined(_SQUID_SOLARIS_) + /* + * Solaris 2.4's socket emulation doesn't allow you + * to determine the error from a failed non-blocking + * connect and just returns EPIPE. Create a fake + * error message for connect. -- fenner@parc.xerox.com + */ + if (x < 0 && errno == EPIPE) + errno = ENOTCONN; +#endif +#endif + } + PROF_stop(comm_connect_addr); + if (errno == 0 || errno == EISCONN) + status = COMM_OK; + else if (ignoreErrno(errno)) + status = COMM_INPROGRESS; + else + return COMM_ERROR; + xstrncpy(F->ipaddr, inet_ntoa(address->sin_addr), 16); + F->remote_port = ntohs(address->sin_port); + if (status == COMM_OK) { + debug(5, 10) ("comm_connect_addr: FD %d connected to %s:%d\n", + sock, F->ipaddr, F->remote_port); + } else if (status == COMM_INPROGRESS) { + debug(5, 10) ("comm_connect_addr: FD %d connection pending\n", sock); + } + return status; +} + +/* Wait for an incoming connection on FD. FD should be a socket returned + * from comm_listen. */ +int +comm_accept(int fd, struct sockaddr_in *pn, struct sockaddr_in *me) +{ + int sock; + struct sockaddr_in P; + struct sockaddr_in M; + socklen_t Slen; + fde *F = NULL; + Slen = sizeof(P); + statCounter.syscalls.sock.accepts++; + PROF_start(comm_accept); + if ((sock = accept(fd, (struct sockaddr *) &P, &Slen)) < 0) { + PROF_stop(comm_accept); + if (ignoreErrno(errno)) { + debug(50, 5) ("comm_accept: FD %d: %s\n", fd, xstrerror()); + return COMM_NOMESSAGE; + } else if (ENFILE == errno || EMFILE == errno) { + debug(50, 3) ("comm_accept: FD %d: %s\n", fd, xstrerror()); + return COMM_ERROR; + } else { + debug(50, 1) ("comm_accept: FD %d: %s\n", fd, xstrerror()); + return COMM_ERROR; + } + } + if (pn) + *pn = P; + Slen = sizeof(M); + memset(&M, '\0', Slen); + getsockname(sock, (struct sockaddr *) &M, &Slen); + if (me) + *me = M; + commSetCloseOnExec(sock); + /* fdstat update */ + fd_open(sock, FD_SOCKET, "HTTP Request"); + F = &fd_table[sock]; + xstrncpy(F->ipaddr, inet_ntoa(P.sin_addr), 16); + F->remote_port = htons(P.sin_port); + F->local_port = htons(M.sin_port); + commSetNonBlocking(sock); + PROF_stop(comm_accept); + return sock; +} + +void +commCallCloseHandlers(int fd) +{ + fde *F = &fd_table[fd]; + close_handler *ch; + debug(5, 5) ("commCallCloseHandlers: FD %d\n", fd); + while ((ch = F->closeHandler) != NULL) { + F->closeHandler = ch->next; + debug(5, 5) ("commCallCloseHandlers: ch->handler=%p\n", ch->handler); + if (cbdataReferenceValid(ch->data)) + ch->handler(fd, ch->data); + cbdataReferenceDone(ch->data); + memPoolFree(conn_close_pool, ch); /* AAA */ + } +} + +#if LINGERING_CLOSE +static void +commLingerClose(int fd, void *unused) +{ + LOCAL_ARRAY(char, buf, 1024); + int n; + n = FD_READ_METHOD(fd, buf, 1024); + if (n < 0) + debug(5, 3) ("commLingerClose: FD %d read: %s\n", fd, xstrerror()); + comm_close(fd); +} + +static void +commLingerTimeout(int fd, void *unused) +{ + debug(5, 3) ("commLingerTimeout: FD %d\n", fd); + comm_close(fd); +} + +/* + * Inspired by apache + */ +void +comm_lingering_close(int fd) +{ +#if USE_SSL + if (fd_table[fd].ssl) + ssl_shutdown_method(fd); +#endif + if (shutdown(fd, 1) < 0) { + comm_close(fd); + return; + } + fd_note(fd, "lingering close"); + commSetTimeout(fd, 10, commLingerTimeout, NULL); + commSetSelect(fd, COMM_SELECT_READ, commLingerClose, NULL, 0); +} +#endif + +/* + * enable linger with time of 0 so that when the socket is + * closed, TCP generates a RESET + */ +void +comm_reset_close(int fd) +{ + struct linger L; + L.l_onoff = 1; + L.l_linger = 0; + if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char *) &L, sizeof(L)) < 0) + debug(50, 0) ("commResetTCPClose: FD %d: %s\n", fd, xstrerror()); + comm_close(fd); +} + +void +comm_close(int fd) +{ + fde *F = NULL; + + debug(5, 5) ("comm_close: FD %d\n", fd); + assert(fd >= 0); + assert(fd < Squid_MaxFD); + F = &fd_table[fd]; + + if (F->flags.closing) + return; + if (shutting_down && (!F->flags.open || F->type == FD_FILE)) + return; + assert(F->flags.open); + assert(F->type != FD_FILE); + PROF_start(comm_close); + F->flags.closing = 1; +#if USE_SSL + if (F->ssl) + ssl_shutdown_method(fd); +#endif + commSetTimeout(fd, -1, NULL, NULL); + CommWriteStateCallbackAndFree(fd, COMM_ERR_CLOSING); + commCallCloseHandlers(fd); + if (F->uses) /* assume persistent connect count */ + pconnHistCount(1, F->uses); +#if USE_SSL + if (F->ssl) { + SSL_free(F->ssl); + F->ssl = NULL; + } +#endif + fd_close(fd); /* update fdstat */ + close(fd); + statCounter.syscalls.sock.closes++; + PROF_stop(comm_close); +} + +/* Send a udp datagram to specified TO_ADDR. */ +int +comm_udp_sendto(int fd, + const struct sockaddr_in *to_addr, + int addr_len, + const void *buf, + int len) +{ + int x; + PROF_start(comm_udp_sendto); + statCounter.syscalls.sock.sendtos++; + x = sendto(fd, buf, len, 0, (struct sockaddr *) to_addr, addr_len); + PROF_stop(comm_udp_sendto); + if (x < 0) { +#ifdef _SQUID_LINUX_ + if (ECONNREFUSED != errno) +#endif + debug(50, 1) ("comm_udp_sendto: FD %d, %s, port %d: %s\n", + fd, + inet_ntoa(to_addr->sin_addr), + (int) htons(to_addr->sin_port), + xstrerror()); + return COMM_ERROR; + } + return x; +} + +void +commSetDefer(int fd, DEFER * func, void *data) +{ + fde *F = &fd_table[fd]; + F->defer_check = func; + F->defer_data = data; +} + +void +comm_add_close_handler(int fd, PF * handler, void *data) +{ + close_handler *newHandler = (close_handler *)memPoolAlloc(conn_close_pool); /* AAA */ + close_handler *c; + debug(5, 5) ("comm_add_close_handler: FD %d, handler=%p, data=%p\n", + fd, handler, data); + for (c = fd_table[fd].closeHandler; c; c = c->next) + assert(c->handler != handler || c->data != data); + newHandler->handler = handler; + newHandler->data = cbdataReference(data); + newHandler->next = fd_table[fd].closeHandler; + fd_table[fd].closeHandler = newHandler; +} + +void +comm_remove_close_handler(int fd, PF * handler, void *data) +{ + close_handler *p; + close_handler *last = NULL; + /* Find handler in list */ + debug(5, 5) ("comm_remove_close_handler: FD %d, handler=%p, data=%p\n", + fd, handler, data); + for (p = fd_table[fd].closeHandler; p != NULL; last = p, p = p->next) + if (p->handler == handler && p->data == data) + break; /* This is our handler */ + assert(p != NULL); + /* Remove list entry */ + if (last) + last->next = p->next; + else + fd_table[fd].closeHandler = p->next; + cbdataReferenceDone(p->data); + memPoolFree(conn_close_pool, p); +} + +static void +commSetNoLinger(int fd) +{ + struct linger L; + L.l_onoff = 0; /* off */ + L.l_linger = 0; + if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char *) &L, sizeof(L)) < 0) + debug(50, 0) ("commSetNoLinger: FD %d: %s\n", fd, xstrerror()); + fd_table[fd].flags.nolinger = 1; +} + +static void +commSetReuseAddr(int fd) +{ + int on = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on)) < 0) + debug(50, 1) ("commSetReuseAddr: FD %d: %s\n", fd, xstrerror()); +} + +static void +commSetTcpRcvbuf(int fd, int size) +{ + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (char *) &size, sizeof(size)) < 0) + debug(50, 1) ("commSetTcpRcvbuf: FD %d, SIZE %d: %s\n", + fd, size, xstrerror()); +} + +int +commSetNonBlocking(int fd) +{ + int flags; + int dummy = 0; +#ifdef _SQUID_CYGWIN_ + int nonblocking = TRUE; + if (fd_table[fd].type != FD_PIPE) { + if (ioctl(fd, FIONBIO, &nonblocking) < 0) { + debug(50, 0) ("commSetNonBlocking: FD %d: %s %u\n", fd, xstrerror(), fd_table[fd].type); + return COMM_ERROR; + } + } else { +#endif + if ((flags = fcntl(fd, F_GETFL, dummy)) < 0) { + debug(50, 0) ("FD %d: fcntl F_GETFL: %s\n", fd, xstrerror()); + return COMM_ERROR; + } + if (fcntl(fd, F_SETFL, flags | SQUID_NONBLOCK) < 0) { + debug(50, 0) ("commSetNonBlocking: FD %d: %s\n", fd, xstrerror()); + return COMM_ERROR; + } +#ifdef _SQUID_CYGWIN_ + } +#endif + fd_table[fd].flags.nonblocking = 1; + return 0; +} + +int +commUnsetNonBlocking(int fd) +{ + int flags; + int dummy = 0; + if ((flags = fcntl(fd, F_GETFL, dummy)) < 0) { + debug(50, 0) ("FD %d: fcntl F_GETFL: %s\n", fd, xstrerror()); + return COMM_ERROR; + } + if (fcntl(fd, F_SETFL, flags & (~SQUID_NONBLOCK)) < 0) { + debug(50, 0) ("commUnsetNonBlocking: FD %d: %s\n", fd, xstrerror()); + return COMM_ERROR; + } + fd_table[fd].flags.nonblocking = 0; + return 0; +} + +void +commSetCloseOnExec(int fd) +{ +#ifdef FD_CLOEXEC + int flags; + int dummy = 0; + if ((flags = fcntl(fd, F_GETFL, dummy)) < 0) { + debug(50, 0) ("FD %d: fcntl F_GETFL: %s\n", fd, xstrerror()); + return; + } + if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) + debug(50, 0) ("FD %d: set close-on-exec failed: %s\n", fd, xstrerror()); + fd_table[fd].flags.close_on_exec = 1; +#endif +} + +#ifdef TCP_NODELAY +static void +commSetTcpNoDelay(int fd) +{ + int on = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &on, sizeof(on)) < 0) + debug(50, 1) ("commSetTcpNoDelay: FD %d: %s\n", fd, xstrerror()); + fd_table[fd].flags.nodelay = 1; +} +#endif + + +void +comm_init(void) +{ + fd_table = (fde *)xcalloc(Squid_MaxFD, sizeof(fde)); + /* XXX account fd_table */ + /* Keep a few file descriptors free so that we don't run out of FD's + * after accepting a client but before it opens a socket or a file. + * Since Squid_MaxFD can be as high as several thousand, don't waste them */ + RESERVED_FD = XMIN(100, Squid_MaxFD / 4); + CBDATA_INIT_TYPE(ConnectStateData); + comm_write_pool = memPoolCreate("CommWriteStateData", sizeof(CommWriteStateData)); + conn_close_pool = memPoolCreate("close_handler", sizeof(close_handler)); +} + +/* Write to FD. */ +static void +commHandleWrite(int fd, void *data) +{ + CommWriteStateData *state = (CommWriteStateData *)data; + int len = 0; + int nleft; + + PROF_start(commHandleWrite); + debug(5, 5) ("commHandleWrite: FD %d: off %ld, sz %ld.\n", + fd, (long int) state->offset, (long int) state->size); + + nleft = state->size - state->offset; + len = FD_WRITE_METHOD(fd, state->buf + state->offset, nleft); + debug(5, 5) ("commHandleWrite: write() returns %d\n", len); + fd_bytes(fd, len, FD_WRITE); + statCounter.syscalls.sock.writes++; + + if (len == 0) { + /* Note we even call write if nleft == 0 */ + /* We're done */ + if (nleft != 0) + debug(5, 1) ("commHandleWrite: FD %d: write failure: connection closed with %d bytes remaining.\n", fd, nleft); + CommWriteStateCallbackAndFree(fd, nleft ? COMM_ERROR : COMM_OK); + } else if (len < 0) { + /* An error */ + if (fd_table[fd].flags.socket_eof) { + debug(50, 2) ("commHandleWrite: FD %d: write failure: %s.\n", + fd, xstrerror()); + CommWriteStateCallbackAndFree(fd, COMM_ERROR); + } else if (ignoreErrno(errno)) { + debug(50, 10) ("commHandleWrite: FD %d: write failure: %s.\n", + fd, xstrerror()); + commSetSelect(fd, + COMM_SELECT_WRITE, + commHandleWrite, + state, + 0); + } else { + debug(50, 2) ("commHandleWrite: FD %d: write failure: %s.\n", + fd, xstrerror()); + CommWriteStateCallbackAndFree(fd, COMM_ERROR); + } + } else { + /* A successful write, continue */ + state->offset += len; + if (state->offset < (off_t)state->size) { + /* Not done, reinstall the write handler and write some more */ + commSetSelect(fd, + COMM_SELECT_WRITE, + commHandleWrite, + state, + 0); + } else { + CommWriteStateCallbackAndFree(fd, COMM_OK); + } + } + PROF_stop(commHandleWrite); +} + + + +/* + * Queue a write. handler/handler_data are called when the write + * completes, on error, or on file descriptor close. + * + * free_func is used to free the passed buffer when the write has completed. + */ +void +comm_write(int fd, const char *buf, int size, CWCB * handler, void *handler_data, FREE * free_func) +{ + CommWriteStateData *state = fd_table[fd].rwstate; + debug(5, 5) ("comm_write: FD %d: sz %d: hndl %p: data %p.\n", + fd, size, handler, handler_data); + if (NULL != state) { + debug(5, 1) ("comm_write: fd_table[%d].rwstate != NULL\n", fd); + memPoolFree(comm_write_pool, state); + fd_table[fd].rwstate = NULL; + } + fd_table[fd].rwstate = state = (CommWriteStateData *)memPoolAlloc(comm_write_pool); + state->buf = (char *) buf; + state->size = size; + state->offset = 0; + state->handler = handler; + state->handler_data = cbdataReference(handler_data); + state->free_func = free_func; + commSetSelect(fd, COMM_SELECT_WRITE, commHandleWrite, state, 0); +} + +/* a wrapper around comm_write to allow for MemBuf to be comm_written in a snap */ +void +comm_write_mbuf(int fd, MemBuf mb, CWCB * handler, void *handler_data) +{ + comm_write(fd, mb.buf, mb.size, handler, handler_data, memBufFreeFunc(&mb)); +} + +/* + * hm, this might be too general-purpose for all the places we'd + * like to use it. + */ +int +ignoreErrno(int ierrno) +{ + switch (ierrno) { + case EINPROGRESS: + case EWOULDBLOCK: +#if EAGAIN != EWOULDBLOCK + case EAGAIN: +#endif + case EALREADY: + case EINTR: +#ifdef ERESTART + case ERESTART: +#endif + return 1; + default: + return 0; + } + /* NOTREACHED */ +} + +void +commCloseAllSockets(void) +{ + int fd; + fde *F = NULL; + for (fd = 0; fd <= Biggest_FD; fd++) { + F = &fd_table[fd]; + if (!F->flags.open) + continue; + if (F->type != FD_SOCKET) + continue; + if (F->flags.ipc) /* don't close inter-process sockets */ + continue; + if (F->timeout_handler) { + PF *callback = F->timeout_handler; + void *cbdata = NULL; + F->timeout_handler = NULL; + debug(5, 5) ("commCloseAllSockets: FD %d: Calling timeout handler\n", + fd); + if (cbdataReferenceValidDone(F->timeout_data, &cbdata)) + callback(fd, cbdata); + } else { + debug(5, 5) ("commCloseAllSockets: FD %d: calling comm_close()\n", fd); + comm_close(fd); + } + } +} + +void +checkTimeouts(void) +{ + int fd; + fde *F = NULL; + PF *callback; + for (fd = 0; fd <= Biggest_FD; fd++) { + F = &fd_table[fd]; + if (!F->flags.open) + continue; + if (F->timeout == 0) + continue; + if (F->timeout > squid_curtime) + continue; + debug(5, 5) ("checkTimeouts: FD %d Expired\n", fd); + if (F->timeout_handler) { + debug(5, 5) ("checkTimeouts: FD %d: Call timeout handler\n", fd); + callback = F->timeout_handler; + F->timeout_handler = NULL; + callback(fd, F->timeout_data); + } else { + debug(5, 5) ("checkTimeouts: FD %d: Forcing comm_close()\n", fd); + comm_close(fd); + } + } +} + + +int +commDeferRead(int fd) +{ + fde *F = &fd_table[fd]; + if (F->defer_check == NULL) + return 0; + return F->defer_check(fd, F->defer_data); +} --- squid/src/comm_kqueue.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,286 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 5 Socket functions - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -/* - * This code was originally written by Benno Rice and hacked on quite - * a bit by Adrian. Adrian then took it to the hybrid-ircd project to use - * in their new IO subsystem. After a year of modifications and some - * rather interesting changes (event aggregation) its back in squid. - * Thanks to the ircd-hybrid guys. - */ - -/* - * XXX Currently not implemented / supported by this module XXX - * - * - delay pools - * - deferred reads - * - * So, its not entirely useful in a production setup since if a read - * is meant to be deferred it isn't (we're not even throwing the event - * away here). Eventually the rest of the code will be rewritten - * so deferred reads aren't required. - * -- adrian - */ - -#include "squid.h" -#include "Store.h" - -#ifdef USE_KQUEUE - -#include - - -#define KE_LENGTH 128 - -/* jlemon goofed up and didn't add EV_SET until fbsd 4.3 */ - -#ifndef EV_SET -#define EV_SET(kevp, a, b, c, d, e, f) do { \ - (kevp)->ident = (a); \ - (kevp)->filter = (b); \ - (kevp)->flags = (c); \ - (kevp)->fflags = (d); \ - (kevp)->data = (e); \ - (kevp)->udata = (f); \ -} while(0) -#endif - -static void kq_update_events(int, short, PF *); -static int kq; -static struct timespec zero_timespec; - -static struct kevent *kqlst; /* kevent buffer */ -static int kqmax; /* max structs to buffer */ -static int kqoff; /* offset into the buffer */ - - -/* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ -/* Private functions */ - -void -kq_update_events(int fd, short filter, PF * handler) -{ - PF *cur_handler; - int kep_flags; - -#if 0 - int retval; -#endif - - switch (filter) { - case EVFILT_READ: - cur_handler = fd_table[fd].read_handler; - break; - case EVFILT_WRITE: - cur_handler = fd_table[fd].write_handler; - break; - default: - /* XXX bad! -- adrian */ - return; - break; - } - - if ((cur_handler == NULL && handler != NULL) - || (cur_handler != NULL && handler == NULL)) { - struct kevent *kep; - - kep = kqlst + kqoff; - - if (handler != NULL) { - kep_flags = (EV_ADD | EV_ONESHOT); - } else { - kep_flags = EV_DELETE; - } - - EV_SET(kep, (uintptr_t) fd, filter, kep_flags, 0, 0, 0); - - if (kqoff == kqmax) { - int ret; - - ret = kevent(kq, kqlst, kqoff, NULL, 0, &zero_timespec); - /* jdc -- someone needs to do error checking... */ - if (ret == -1) { - perror("kq_update_events(): kevent()"); - return; - } - kqoff = 0; - } else { - kqoff++; - } -#if 0 - if (retval < 0) { - /* Error! */ - if (ke.flags & EV_ERROR) { - errno = ke.data; - } - } -#endif - } -} - - - -/* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ -/* Public functions */ - - -/* - * comm_select_init - * - * This is a needed exported function which will be called to initialise - * the network loop code. - */ -void -comm_select_init(void) -{ - kq = kqueue(); - if (kq < 0) { - fatal("comm_select_init: Couldn't open kqueue fd!\n"); - } - kqmax = getdtablesize(); - kqlst = xmalloc(sizeof(*kqlst) * kqmax); - zero_timespec.tv_sec = 0; - zero_timespec.tv_nsec = 0; -} - -/* - * comm_setselect - * - * This is a needed exported function which will be called to register - * and deregister interest in a pending IO state for a given FD. - */ -void -commSetSelect(int fd, unsigned int type, PF * handler, - void *client_data, time_t timeout) -{ - fde *F = &fd_table[fd]; - assert(fd >= 0); - assert(F->flags.open); - - if (type & COMM_SELECT_READ) { - kq_update_events(fd, EVFILT_READ, handler); - F->read_handler = handler; - F->read_data = client_data; - } - if (type & COMM_SELECT_WRITE) { - kq_update_events(fd, EVFILT_WRITE, handler); - F->write_handler = handler; - F->write_data = client_data; - } - if (timeout) - F->timeout = squid_curtime + timeout; - -} - -/* - * Check all connections for new connections and input data that is to be - * processed. Also check for connections with data queued and whether we can - * write it out. - */ - -/* - * comm_select - * - * Called to do the new-style IO, courtesy of of squid (like most of this - * new IO code). This routine handles the stuff we've hidden in - * comm_setselect and fd_table[] and calls callbacks for IO ready - * events. - */ - -comm_err_t -comm_select(int msec) -{ - int num, i; - static struct kevent ke[KE_LENGTH]; - struct timespec poll_time; - - do { - /* - * remember we are doing NANOseconds here, not micro/milli. God knows - * why jlemon used a timespec, but hey, he wrote the interface, not I - * -- Adrian - */ - poll_time.tv_sec = msec / 1000; - poll_time.tv_nsec = (msec % 1000) * 1000000; - for (;;) { - num = kevent(kq, kqlst, kqoff, ke, KE_LENGTH, &poll_time); - statCounter.select_loops++; - kqoff = 0; - if (num >= 0) - break; - if (ignoreErrno(errno)) - break; - getCurrentTime(); - return COMM_ERROR; - /* NOTREACHED */ - } - - getCurrentTime(); - if (num == 0) - continue; - - for (i = 0; i < num; i++) { - int fd = (int) ke[i].ident; - PF *hdl = NULL; - fde *F = &fd_table[fd]; - - if (ke[i].flags & EV_ERROR) { - errno = ke[i].data; - /* XXX error == bad! -- adrian */ - continue; /* XXX! */ - } - switch (ke[i].filter) { - case EVFILT_READ: - if ((hdl = F->read_handler) != NULL) { - F->read_handler = NULL; - hdl(fd, F->read_data); - } - case EVFILT_WRITE: - if ((hdl = F->write_handler) != NULL) { - F->write_handler = NULL; - hdl(fd, F->write_data); - } - default: - /* Bad! -- adrian */ - break; - } - } - return COMM_OK; - } - while (0); /* XXX should rip this out! -- adrian */ - /* XXX Get here, we broke! */ - return 0; -} - -#endif /* USE_KQUEUE */ --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/comm_kqueue.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,286 @@ + +/* + * $Id: comm_kqueue.cc,v 1.1.2.1 2002/10/11 15:40:44 rbcollins Exp $ + * + * DEBUG: section 5 Socket functions + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +/* + * This code was originally written by Benno Rice and hacked on quite + * a bit by Adrian. Adrian then took it to the hybrid-ircd project to use + * in their new IO subsystem. After a year of modifications and some + * rather interesting changes (event aggregation) its back in squid. + * Thanks to the ircd-hybrid guys. + */ + +/* + * XXX Currently not implemented / supported by this module XXX + * + * - delay pools + * - deferred reads + * + * So, its not entirely useful in a production setup since if a read + * is meant to be deferred it isn't (we're not even throwing the event + * away here). Eventually the rest of the code will be rewritten + * so deferred reads aren't required. + * -- adrian + */ + +#include "squid.h" +#include "Store.h" + +#ifdef USE_KQUEUE + +#include + + +#define KE_LENGTH 128 + +/* jlemon goofed up and didn't add EV_SET until fbsd 4.3 */ + +#ifndef EV_SET +#define EV_SET(kevp, a, b, c, d, e, f) do { \ + (kevp)->ident = (a); \ + (kevp)->filter = (b); \ + (kevp)->flags = (c); \ + (kevp)->fflags = (d); \ + (kevp)->data = (e); \ + (kevp)->udata = (f); \ +} while(0) +#endif + +static void kq_update_events(int, short, PF *); +static int kq; +static struct timespec zero_timespec; + +static struct kevent *kqlst; /* kevent buffer */ +static int kqmax; /* max structs to buffer */ +static int kqoff; /* offset into the buffer */ + + +/* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ +/* Private functions */ + +void +kq_update_events(int fd, short filter, PF * handler) +{ + PF *cur_handler; + int kep_flags; + +#if 0 + int retval; +#endif + + switch (filter) { + case EVFILT_READ: + cur_handler = fd_table[fd].read_handler; + break; + case EVFILT_WRITE: + cur_handler = fd_table[fd].write_handler; + break; + default: + /* XXX bad! -- adrian */ + return; + break; + } + + if ((cur_handler == NULL && handler != NULL) + || (cur_handler != NULL && handler == NULL)) { + struct kevent *kep; + + kep = kqlst + kqoff; + + if (handler != NULL) { + kep_flags = (EV_ADD | EV_ONESHOT); + } else { + kep_flags = EV_DELETE; + } + + EV_SET(kep, (uintptr_t) fd, filter, kep_flags, 0, 0, 0); + + if (kqoff == kqmax) { + int ret; + + ret = kevent(kq, kqlst, kqoff, NULL, 0, &zero_timespec); + /* jdc -- someone needs to do error checking... */ + if (ret == -1) { + perror("kq_update_events(): kevent()"); + return; + } + kqoff = 0; + } else { + kqoff++; + } +#if 0 + if (retval < 0) { + /* Error! */ + if (ke.flags & EV_ERROR) { + errno = ke.data; + } + } +#endif + } +} + + + +/* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ +/* Public functions */ + + +/* + * comm_select_init + * + * This is a needed exported function which will be called to initialise + * the network loop code. + */ +void +comm_select_init(void) +{ + kq = kqueue(); + if (kq < 0) { + fatal("comm_select_init: Couldn't open kqueue fd!\n"); + } + kqmax = getdtablesize(); + kqlst = xmalloc(sizeof(*kqlst) * kqmax); + zero_timespec.tv_sec = 0; + zero_timespec.tv_nsec = 0; +} + +/* + * comm_setselect + * + * This is a needed exported function which will be called to register + * and deregister interest in a pending IO state for a given FD. + */ +void +commSetSelect(int fd, unsigned int type, PF * handler, + void *client_data, time_t timeout) +{ + fde *F = &fd_table[fd]; + assert(fd >= 0); + assert(F->flags.open); + + if (type & COMM_SELECT_READ) { + kq_update_events(fd, EVFILT_READ, handler); + F->read_handler = handler; + F->read_data = client_data; + } + if (type & COMM_SELECT_WRITE) { + kq_update_events(fd, EVFILT_WRITE, handler); + F->write_handler = handler; + F->write_data = client_data; + } + if (timeout) + F->timeout = squid_curtime + timeout; + +} + +/* + * Check all connections for new connections and input data that is to be + * processed. Also check for connections with data queued and whether we can + * write it out. + */ + +/* + * comm_select + * + * Called to do the new-style IO, courtesy of of squid (like most of this + * new IO code). This routine handles the stuff we've hidden in + * comm_setselect and fd_table[] and calls callbacks for IO ready + * events. + */ + +comm_err_t +comm_select(int msec) +{ + int num, i; + static struct kevent ke[KE_LENGTH]; + struct timespec poll_time; + + do { + /* + * remember we are doing NANOseconds here, not micro/milli. God knows + * why jlemon used a timespec, but hey, he wrote the interface, not I + * -- Adrian + */ + poll_time.tv_sec = msec / 1000; + poll_time.tv_nsec = (msec % 1000) * 1000000; + for (;;) { + num = kevent(kq, kqlst, kqoff, ke, KE_LENGTH, &poll_time); + statCounter.select_loops++; + kqoff = 0; + if (num >= 0) + break; + if (ignoreErrno(errno)) + break; + getCurrentTime(); + return COMM_ERROR; + /* NOTREACHED */ + } + + getCurrentTime(); + if (num == 0) + continue; + + for (i = 0; i < num; i++) { + int fd = (int) ke[i].ident; + PF *hdl = NULL; + fde *F = &fd_table[fd]; + + if (ke[i].flags & EV_ERROR) { + errno = ke[i].data; + /* XXX error == bad! -- adrian */ + continue; /* XXX! */ + } + switch (ke[i].filter) { + case EVFILT_READ: + if ((hdl = F->read_handler) != NULL) { + F->read_handler = NULL; + hdl(fd, F->read_data); + } + case EVFILT_WRITE: + if ((hdl = F->write_handler) != NULL) { + F->write_handler = NULL; + hdl(fd, F->write_data); + } + default: + /* Bad! -- adrian */ + break; + } + } + return COMM_OK; + } + while (0); /* XXX should rip this out! -- adrian */ + /* XXX Get here, we broke! */ + return 0; +} + +#endif /* USE_KQUEUE */ --- squid/src/comm_poll.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,630 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 5 Socket Functions - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "Store.h" - -#ifdef USE_POLL - -static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */ - -#ifndef howmany -#define howmany(x, y) (((x)+((y)-1))/(y)) -#endif -#ifndef NBBY -#define NBBY 8 -#endif -#define FD_MASK_BYTES sizeof(fd_mask) -#define FD_MASK_BITS (FD_MASK_BYTES*NBBY) - -/* STATIC */ -static int fdIsHttp(int fd); -static int fdIsIcp(int fd); -static int fdIsDns(int fd); -static OBJH commIncomingStats; -static int comm_check_incoming_poll_handlers(int nfds, int *fds); -static void comm_poll_dns_incoming(void); -static void commUpdateReadBits(int fd, PF * handler); -static void commUpdateWriteBits(int fd, PF * handler); - -static fd_set global_readfds; -static fd_set global_writefds; -static int nreadfds; -static int nwritefds; - -/* - * Automatic tuning for incoming requests: - * - * INCOMING sockets are the ICP and HTTP ports. We need to check these - * fairly regularly, but how often? When the load increases, we - * want to check the incoming sockets more often. If we have a lot - * of incoming ICP, then we need to check these sockets more than - * if we just have HTTP. - * - * The variables 'incoming_icp_interval' and 'incoming_http_interval' - * determine how many normal I/O events to process before checking - * incoming sockets again. Note we store the incoming_interval - * multipled by a factor of (2^INCOMING_FACTOR) to have some - * pseudo-floating point precision. - * - * The variable 'icp_io_events' and 'http_io_events' counts how many normal - * I/O events have been processed since the last check on the incoming - * sockets. When io_events > incoming_interval, its time to check incoming - * sockets. - * - * Every time we check incoming sockets, we count how many new messages - * or connections were processed. This is used to adjust the - * incoming_interval for the next iteration. The new incoming_interval - * is calculated as the current incoming_interval plus what we would - * like to see as an average number of events minus the number of - * events just processed. - * - * incoming_interval = incoming_interval + target_average - number_of_events_processed - * - * There are separate incoming_interval counters for both HTTP and ICP events - * - * You can see the current values of the incoming_interval's, as well as - * a histogram of 'incoming_events' by asking the cache manager - * for 'comm_incoming', e.g.: - * - * % ./client mgr:comm_incoming - * - * Caveats: - * - * - We have MAX_INCOMING_INTEGER as a magic upper limit on - * incoming_interval for both types of sockets. At the - * largest value the cache will effectively be idling. - * - * - The higher the INCOMING_FACTOR, the slower the algorithm will - * respond to load spikes/increases/decreases in demand. A value - * between 3 and 8 is recommended. - */ - -#define MAX_INCOMING_INTEGER 256 -#define INCOMING_FACTOR 5 -#define MAX_INCOMING_INTERVAL (MAX_INCOMING_INTEGER << INCOMING_FACTOR) -static int icp_io_events = 0; -static int dns_io_events = 0; -static int http_io_events = 0; -static int incoming_icp_interval = 16 << INCOMING_FACTOR; -static int incoming_dns_interval = 16 << INCOMING_FACTOR; -static int incoming_http_interval = 16 << INCOMING_FACTOR; -#define commCheckICPIncoming (++icp_io_events > (incoming_icp_interval>> INCOMING_FACTOR)) -#define commCheckDNSIncoming (++dns_io_events > (incoming_dns_interval>> INCOMING_FACTOR)) -#define commCheckHTTPIncoming (++http_io_events > (incoming_http_interval>> INCOMING_FACTOR)) - - -void -commSetSelect(int fd, unsigned int type, PF * handler, void *client_data, - time_t timeout) -{ - fde *F = &fd_table[fd]; - assert(fd >= 0); - assert(F->flags.open); - debug(5, 5) ("commSetSelect: FD %d type %d\n", fd, type); - if (type & COMM_SELECT_READ) { - F->read_handler = handler; - F->read_data = client_data; - commUpdateReadBits(fd, handler); - } - if (type & COMM_SELECT_WRITE) { - F->write_handler = handler; - F->write_data = client_data; - commUpdateWriteBits(fd, handler); - } - if (timeout) - F->timeout = squid_curtime + timeout; -} - -static int -fdIsIcp(int fd) -{ - if (fd == theInIcpConnection) - return 1; - if (fd == theOutIcpConnection) - return 1; - return 0; -} - -static int -fdIsDns(int fd) -{ - if (fd == DnsSocket) - return 1; - return 0; -} - -static int -fdIsHttp(int fd) -{ - int j; - for (j = 0; j < NHttpSockets; j++) { - if (fd == HttpSockets[j]) - return 1; - } - return 0; -} - -#if DELAY_POOLS -static int slowfdcnt = 0; -static int slowfdarr[SQUID_MAXFD]; - -static void -commAddSlowFd(int fd) -{ - assert(slowfdcnt < SQUID_MAXFD); - slowfdarr[slowfdcnt++] = fd; -} - -static int -commGetSlowFd(void) -{ - int whichfd, retfd; - - if (!slowfdcnt) - return -1; - whichfd = squid_random() % slowfdcnt; - retfd = slowfdarr[whichfd]; - slowfdarr[whichfd] = slowfdarr[--slowfdcnt]; - return retfd; -} -#endif - -static int -comm_check_incoming_poll_handlers(int nfds, int *fds) -{ - int i; - int fd; - PF *hdl = NULL; - int npfds; - struct pollfd pfds[3 + MAXHTTPPORTS]; - PROF_start(comm_check_incoming); - incoming_sockets_accepted = 0; - for (i = npfds = 0; i < nfds; i++) { - int events; - fd = fds[i]; - events = 0; - if (fd_table[fd].read_handler) - events |= POLLRDNORM; - if (fd_table[fd].write_handler) - events |= POLLWRNORM; - if (events) { - pfds[npfds].fd = fd; - pfds[npfds].events = events; - pfds[npfds].revents = 0; - npfds++; - } - } - if (!nfds) { - PROF_stop(comm_check_incoming); - return -1; - } - getCurrentTime(); - statCounter.syscalls.polls++; - if (poll(pfds, npfds, 0) < 1) { - PROF_stop(comm_check_incoming); - return incoming_sockets_accepted; - } - for (i = 0; i < npfds; i++) { - int revents; - if (((revents = pfds[i].revents) == 0) || ((fd = pfds[i].fd) == -1)) - continue; - if (revents & (POLLRDNORM | POLLIN | POLLHUP | POLLERR)) { - if ((hdl = fd_table[fd].read_handler)) { - fd_table[fd].read_handler = NULL; - hdl(fd, fd_table[fd].read_data); - } else if (pfds[i].events & POLLRDNORM) - debug(5, 1) ("comm_poll_incoming: FD %d NULL read handler\n", - fd); - } - if (revents & (POLLWRNORM | POLLOUT | POLLHUP | POLLERR)) { - if ((hdl = fd_table[fd].write_handler)) { - fd_table[fd].write_handler = NULL; - hdl(fd, fd_table[fd].write_data); - } else if (pfds[i].events & POLLWRNORM) - debug(5, 1) ("comm_poll_incoming: FD %d NULL write_handler\n", - fd); - } - } - PROF_stop(comm_check_incoming); - return incoming_sockets_accepted; -} - -static void -comm_poll_icp_incoming(void) -{ - int nfds = 0; - int fds[2]; - int nevents; - icp_io_events = 0; - if (theInIcpConnection >= 0) - fds[nfds++] = theInIcpConnection; - if (theInIcpConnection != theOutIcpConnection) - if (theOutIcpConnection >= 0) - fds[nfds++] = theOutIcpConnection; - if (nfds == 0) - return; - nevents = comm_check_incoming_poll_handlers(nfds, fds); - incoming_icp_interval += Config.comm_incoming.icp_average - nevents; - if (incoming_icp_interval < Config.comm_incoming.icp_min_poll) - incoming_icp_interval = Config.comm_incoming.icp_min_poll; - if (incoming_icp_interval > MAX_INCOMING_INTERVAL) - incoming_icp_interval = MAX_INCOMING_INTERVAL; - if (nevents > INCOMING_ICP_MAX) - nevents = INCOMING_ICP_MAX; - statHistCount(&statCounter.comm_icp_incoming, nevents); -} - -static void -comm_poll_http_incoming(void) -{ - int nfds = 0; - int fds[MAXHTTPPORTS]; - int j; - int nevents; - http_io_events = 0; - for (j = 0; j < NHttpSockets; j++) { - if (HttpSockets[j] < 0) - continue; - if (commDeferRead(HttpSockets[j])) - continue; - fds[nfds++] = HttpSockets[j]; - } - nevents = comm_check_incoming_poll_handlers(nfds, fds); - incoming_http_interval = incoming_http_interval - + Config.comm_incoming.http_average - nevents; - if (incoming_http_interval < Config.comm_incoming.http_min_poll) - incoming_http_interval = Config.comm_incoming.http_min_poll; - if (incoming_http_interval > MAX_INCOMING_INTERVAL) - incoming_http_interval = MAX_INCOMING_INTERVAL; - if (nevents > INCOMING_HTTP_MAX) - nevents = INCOMING_HTTP_MAX; - statHistCount(&statCounter.comm_http_incoming, nevents); -} - -/* poll all sockets; call handlers for those that are ready. */ -comm_err_t -comm_select(int msec) -{ - struct pollfd pfds[SQUID_MAXFD]; -#if DELAY_POOLS - fd_set slowfds; -#endif - PF *hdl = NULL; - int fd; - int i; - int maxfd; - unsigned long nfds; - unsigned long npending; - int num; - int callicp = 0, callhttp = 0; - int calldns = 0; - static time_t last_timeout = 0; - double timeout = current_dtime + (msec / 1000.0); - do { - double start; - getCurrentTime(); - start = current_dtime; - /* Handle any fs callbacks that need doing */ - storeDirCallback(); -#if DELAY_POOLS - FD_ZERO(&slowfds); -#endif - if (commCheckICPIncoming) - comm_poll_icp_incoming(); - if (commCheckDNSIncoming) - comm_poll_dns_incoming(); - if (commCheckHTTPIncoming) - comm_poll_http_incoming(); - PROF_start(comm_poll_prep_pfds); - callicp = calldns = callhttp = 0; - nfds = 0; - npending = 0; - maxfd = Biggest_FD + 1; - for (i = 0; i < maxfd; i++) { - int events; - events = 0; - /* Check each open socket for a handler. */ - if (fd_table[i].read_handler) { - switch (commDeferRead(i)) { - case 0: - events |= POLLRDNORM; - break; - case 1: - break; -#if DELAY_POOLS - case -1: - events |= POLLRDNORM; - FD_SET(i, &slowfds); - break; -#endif - default: - fatalf("bad return value from commDeferRead(FD %d)\n", i); - } - } - if (fd_table[i].write_handler) - events |= POLLWRNORM; - if (events) { - pfds[nfds].fd = i; - pfds[nfds].events = events; - pfds[nfds].revents = 0; - nfds++; - if ((events & POLLRDNORM) && fd_table[i].flags.read_pending) - npending++; - } - } - PROF_stop(comm_poll_prep_pfds); - if (nfds == 0) { - assert(shutting_down); - return COMM_SHUTDOWN; - } - if (npending) - msec = 0; - if (msec > MAX_POLL_TIME) - msec = MAX_POLL_TIME; - for (;;) { - PROF_start(comm_poll_normal); - statCounter.syscalls.polls++; - num = poll(pfds, nfds, msec); - statCounter.select_loops++; - PROF_stop(comm_poll_normal); - if (num >= 0 || npending >= 0) - break; - if (ignoreErrno(errno)) - continue; - debug(5, 0) ("comm_poll: poll failure: %s\n", xstrerror()); - assert(errno != EINVAL); - return COMM_ERROR; - /* NOTREACHED */ - } - debug(5, num ? 5 : 8) ("comm_poll: %d+%ld FDs ready\n", num, npending); - statHistCount(&statCounter.select_fds_hist, num); - /* Check timeout handlers ONCE each second. */ - if (squid_curtime > last_timeout) { - last_timeout = squid_curtime; - checkTimeouts(); - } - if (num == 0 && npending == 0) - continue; - /* scan each socket but the accept socket. Poll this - * more frequently to minimize losses due to the 5 connect - * limit in SunOS */ - PROF_start(comm_handle_ready_fd); - for (i = 0; i < nfds; i++) { - fde *F; - int revents = pfds[i].revents; - fd = pfds[i].fd; - if (fd == -1) - continue; - if (fd_table[fd].flags.read_pending) - revents |= POLLIN; - if (revents == 0) - continue; - if (fdIsIcp(fd)) { - callicp = 1; - continue; - } - if (fdIsDns(fd)) { - calldns = 1; - continue; - } - if (fdIsHttp(fd)) { - callhttp = 1; - continue; - } - F = &fd_table[fd]; - if (revents & (POLLRDNORM | POLLIN | POLLHUP | POLLERR)) { - debug(5, 6) ("comm_poll: FD %d ready for reading\n", fd); - if (NULL == (hdl = F->read_handler)) - (void) 0; -#if DELAY_POOLS - else if (FD_ISSET(fd, &slowfds)) - commAddSlowFd(fd); -#endif - else { - PROF_start(comm_read_handler); - F->read_handler = NULL; - hdl(fd, F->read_data); - PROF_stop(comm_read_handler); - statCounter.select_fds++; - if (commCheckICPIncoming) - comm_poll_icp_incoming(); - if (commCheckDNSIncoming) - comm_poll_dns_incoming(); - if (commCheckHTTPIncoming) - comm_poll_http_incoming(); - } - } - if (revents & (POLLWRNORM | POLLOUT | POLLHUP | POLLERR)) { - debug(5, 5) ("comm_poll: FD %d ready for writing\n", fd); - if ((hdl = F->write_handler)) { - PROF_start(comm_write_handler); - F->write_handler = NULL; - hdl(fd, F->write_data); - PROF_stop(comm_write_handler); - statCounter.select_fds++; - if (commCheckICPIncoming) - comm_poll_icp_incoming(); - if (commCheckDNSIncoming) - comm_poll_dns_incoming(); - if (commCheckHTTPIncoming) - comm_poll_http_incoming(); - } - } - if (revents & POLLNVAL) { - close_handler *ch; - debug(5, 0) ("WARNING: FD %d has handlers, but it's invalid.\n", fd); - debug(5, 0) ("FD %d is a %s\n", fd, fdTypeStr[F->type]); - debug(5, 0) ("--> %s\n", F->desc); - debug(5, 0) ("tmout:%p read:%p write:%p\n", - F->timeout_handler, - F->read_handler, - F->write_handler); - for (ch = F->closeHandler; ch; ch = ch->next) - debug(5, 0) (" close handler: %p\n", ch->handler); - if (F->closeHandler) { - commCallCloseHandlers(fd); - } else if (F->timeout_handler) { - debug(5, 0) ("comm_poll: Calling Timeout Handler\n"); - F->timeout_handler(fd, F->timeout_data); - } - F->closeHandler = NULL; - F->timeout_handler = NULL; - F->read_handler = NULL; - F->write_handler = NULL; - if (F->flags.open) - fd_close(fd); - } - } - PROF_stop(comm_handle_ready_fd); - if (callicp) - comm_poll_icp_incoming(); - if (calldns) - comm_poll_dns_incoming(); - if (callhttp) - comm_poll_http_incoming(); -#if DELAY_POOLS - while ((fd = commGetSlowFd()) != -1) { - fde *F = &fd_table[fd]; - debug(5, 6) ("comm_select: slow FD %d selected for reading\n", fd); - if ((hdl = F->read_handler)) { - F->read_handler = NULL; - hdl(fd, F->read_data); - statCounter.select_fds++; - if (commCheckICPIncoming) - comm_poll_icp_incoming(); - if (commCheckDNSIncoming) - comm_poll_dns_incoming(); - if (commCheckHTTPIncoming) - comm_poll_http_incoming(); - } - } -#endif - getCurrentTime(); - statCounter.select_time += (current_dtime - start); - return COMM_OK; - } - while (timeout > current_dtime); - debug(5, 8) ("comm_poll: time out: %ld.\n", (long int) squid_curtime); - return COMM_TIMEOUT; -} - - -static void -comm_poll_dns_incoming(void) -{ - int nfds = 0; - int fds[2]; - int nevents; - dns_io_events = 0; - if (DnsSocket < 0) - return; - fds[nfds++] = DnsSocket; - nevents = comm_check_incoming_poll_handlers(nfds, fds); - if (nevents < 0) - return; - incoming_dns_interval += Config.comm_incoming.dns_average - nevents; - if (incoming_dns_interval < Config.comm_incoming.dns_min_poll) - incoming_dns_interval = Config.comm_incoming.dns_min_poll; - if (incoming_dns_interval > MAX_INCOMING_INTERVAL) - incoming_dns_interval = MAX_INCOMING_INTERVAL; - if (nevents > INCOMING_DNS_MAX) - nevents = INCOMING_DNS_MAX; - statHistCount(&statCounter.comm_dns_incoming, nevents); -} - -void -comm_select_init(void) -{ - cachemgrRegister("comm_incoming", - "comm_incoming() stats", - commIncomingStats, 0, 1); - FD_ZERO(&global_readfds); - FD_ZERO(&global_writefds); - nreadfds = nwritefds = 0; -} - - -static void -commIncomingStats(StoreEntry * sentry) -{ - StatCounters *f = &statCounter; - storeAppendPrintf(sentry, "Current incoming_icp_interval: %d\n", - incoming_icp_interval >> INCOMING_FACTOR); - storeAppendPrintf(sentry, "Current incoming_dns_interval: %d\n", - incoming_dns_interval >> INCOMING_FACTOR); - storeAppendPrintf(sentry, "Current incoming_http_interval: %d\n", - incoming_http_interval >> INCOMING_FACTOR); - storeAppendPrintf(sentry, "\n"); - storeAppendPrintf(sentry, "Histogram of events per incoming socket type\n"); - storeAppendPrintf(sentry, "ICP Messages handled per comm_poll_icp_incoming() call:\n"); - statHistDump(&f->comm_icp_incoming, sentry, statHistIntDumper); - storeAppendPrintf(sentry, "DNS Messages handled per comm_poll_dns_incoming() call:\n"); - statHistDump(&f->comm_dns_incoming, sentry, statHistIntDumper); - storeAppendPrintf(sentry, "HTTP Messages handled per comm_poll_http_incoming() call:\n"); - statHistDump(&f->comm_http_incoming, sentry, statHistIntDumper); -} - -void -commUpdateReadBits(int fd, PF * handler) -{ - if (handler && !FD_ISSET(fd, &global_readfds)) { - FD_SET(fd, &global_readfds); - nreadfds++; - } else if (!handler && FD_ISSET(fd, &global_readfds)) { - FD_CLR(fd, &global_readfds); - nreadfds--; - } -} - -void -commUpdateWriteBits(int fd, PF * handler) -{ - if (handler && !FD_ISSET(fd, &global_writefds)) { - FD_SET(fd, &global_writefds); - nwritefds++; - } else if (!handler && FD_ISSET(fd, &global_writefds)) { - FD_CLR(fd, &global_writefds); - nwritefds--; - } -} - -/* Called by async-io or diskd to speed up the polling */ -void -comm_quick_poll_required(void) -{ - MAX_POLL_TIME = 10; -} - -#endif /* USE_POLL */ --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/comm_poll.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,629 @@ + +/* + * $Id: comm_poll.cc,v 1.1.2.1 2002/10/11 15:40:47 rbcollins Exp $ + * + * DEBUG: section 5 Socket Functions + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "Store.h" + +#ifdef USE_POLL + +static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */ + +#ifndef howmany +#define howmany(x, y) (((x)+((y)-1))/(y)) +#endif +#ifndef NBBY +#define NBBY 8 +#endif +#define FD_MASK_BYTES sizeof(fd_mask) +#define FD_MASK_BITS (FD_MASK_BYTES*NBBY) + +/* STATIC */ +static int fdIsHttp(int fd); +static int fdIsIcp(int fd); +static int fdIsDns(int fd); +static OBJH commIncomingStats; +static int comm_check_incoming_poll_handlers(int nfds, int *fds); +static void comm_poll_dns_incoming(void); +static void commUpdateReadBits(int fd, PF * handler); +static void commUpdateWriteBits(int fd, PF * handler); + +static fd_set global_readfds; +static fd_set global_writefds; +static int nreadfds; +static int nwritefds; + +/* + * Automatic tuning for incoming requests: + * + * INCOMING sockets are the ICP and HTTP ports. We need to check these + * fairly regularly, but how often? When the load increases, we + * want to check the incoming sockets more often. If we have a lot + * of incoming ICP, then we need to check these sockets more than + * if we just have HTTP. + * + * The variables 'incoming_icp_interval' and 'incoming_http_interval' + * determine how many normal I/O events to process before checking + * incoming sockets again. Note we store the incoming_interval + * multipled by a factor of (2^INCOMING_FACTOR) to have some + * pseudo-floating point precision. + * + * The variable 'icp_io_events' and 'http_io_events' counts how many normal + * I/O events have been processed since the last check on the incoming + * sockets. When io_events > incoming_interval, its time to check incoming + * sockets. + * + * Every time we check incoming sockets, we count how many new messages + * or connections were processed. This is used to adjust the + * incoming_interval for the next iteration. The new incoming_interval + * is calculated as the current incoming_interval plus what we would + * like to see as an average number of events minus the number of + * events just processed. + * + * incoming_interval = incoming_interval + target_average - number_of_events_processed + * + * There are separate incoming_interval counters for both HTTP and ICP events + * + * You can see the current values of the incoming_interval's, as well as + * a histogram of 'incoming_events' by asking the cache manager + * for 'comm_incoming', e.g.: + * + * % ./client mgr:comm_incoming + * + * Caveats: + * + * - We have MAX_INCOMING_INTEGER as a magic upper limit on + * incoming_interval for both types of sockets. At the + * largest value the cache will effectively be idling. + * + * - The higher the INCOMING_FACTOR, the slower the algorithm will + * respond to load spikes/increases/decreases in demand. A value + * between 3 and 8 is recommended. + */ + +#define MAX_INCOMING_INTEGER 256 +#define INCOMING_FACTOR 5 +#define MAX_INCOMING_INTERVAL (MAX_INCOMING_INTEGER << INCOMING_FACTOR) +static int icp_io_events = 0; +static int dns_io_events = 0; +static int http_io_events = 0; +static int incoming_icp_interval = 16 << INCOMING_FACTOR; +static int incoming_dns_interval = 16 << INCOMING_FACTOR; +static int incoming_http_interval = 16 << INCOMING_FACTOR; +#define commCheckICPIncoming (++icp_io_events > (incoming_icp_interval>> INCOMING_FACTOR)) +#define commCheckDNSIncoming (++dns_io_events > (incoming_dns_interval>> INCOMING_FACTOR)) +#define commCheckHTTPIncoming (++http_io_events > (incoming_http_interval>> INCOMING_FACTOR)) + + +void +commSetSelect(int fd, unsigned int type, PF * handler, void *client_data, + time_t timeout) +{ + fde *F = &fd_table[fd]; + assert(fd >= 0); + assert(F->flags.open); + debug(5, 5) ("commSetSelect: FD %d type %d\n", fd, type); + if (type & COMM_SELECT_READ) { + F->read_handler = handler; + F->read_data = client_data; + commUpdateReadBits(fd, handler); + } + if (type & COMM_SELECT_WRITE) { + F->write_handler = handler; + F->write_data = client_data; + commUpdateWriteBits(fd, handler); + } + if (timeout) + F->timeout = squid_curtime + timeout; +} + +static int +fdIsIcp(int fd) +{ + if (fd == theInIcpConnection) + return 1; + if (fd == theOutIcpConnection) + return 1; + return 0; +} + +static int +fdIsDns(int fd) +{ + if (fd == DnsSocket) + return 1; + return 0; +} + +static int +fdIsHttp(int fd) +{ + int j; + for (j = 0; j < NHttpSockets; j++) { + if (fd == HttpSockets[j]) + return 1; + } + return 0; +} + +#if DELAY_POOLS +static int slowfdcnt = 0; +static int slowfdarr[SQUID_MAXFD]; + +static void +commAddSlowFd(int fd) +{ + assert(slowfdcnt < SQUID_MAXFD); + slowfdarr[slowfdcnt++] = fd; +} + +static int +commGetSlowFd(void) +{ + int whichfd, retfd; + + if (!slowfdcnt) + return -1; + whichfd = squid_random() % slowfdcnt; + retfd = slowfdarr[whichfd]; + slowfdarr[whichfd] = slowfdarr[--slowfdcnt]; + return retfd; +} +#endif + +static int +comm_check_incoming_poll_handlers(int nfds, int *fds) +{ + int i; + int fd; + PF *hdl = NULL; + int npfds; + struct pollfd pfds[3 + MAXHTTPPORTS]; + PROF_start(comm_check_incoming); + incoming_sockets_accepted = 0; + for (i = npfds = 0; i < nfds; i++) { + int events; + fd = fds[i]; + events = 0; + if (fd_table[fd].read_handler) + events |= POLLRDNORM; + if (fd_table[fd].write_handler) + events |= POLLWRNORM; + if (events) { + pfds[npfds].fd = fd; + pfds[npfds].events = events; + pfds[npfds].revents = 0; + npfds++; + } + } + if (!nfds) { + PROF_stop(comm_check_incoming); + return -1; + } + getCurrentTime(); + statCounter.syscalls.polls++; + if (poll(pfds, npfds, 0) < 1) { + PROF_stop(comm_check_incoming); + return incoming_sockets_accepted; + } + for (i = 0; i < npfds; i++) { + int revents; + if (((revents = pfds[i].revents) == 0) || ((fd = pfds[i].fd) == -1)) + continue; + if (revents & (POLLRDNORM | POLLIN | POLLHUP | POLLERR)) { + if ((hdl = fd_table[fd].read_handler)) { + fd_table[fd].read_handler = NULL; + hdl(fd, fd_table[fd].read_data); + } else if (pfds[i].events & POLLRDNORM) + debug(5, 1) ("comm_poll_incoming: FD %d NULL read handler\n", + fd); + } + if (revents & (POLLWRNORM | POLLOUT | POLLHUP | POLLERR)) { + if ((hdl = fd_table[fd].write_handler)) { + fd_table[fd].write_handler = NULL; + hdl(fd, fd_table[fd].write_data); + } else if (pfds[i].events & POLLWRNORM) + debug(5, 1) ("comm_poll_incoming: FD %d NULL write_handler\n", + fd); + } + } + PROF_stop(comm_check_incoming); + return incoming_sockets_accepted; +} + +static void +comm_poll_icp_incoming(void) +{ + int nfds = 0; + int fds[2]; + int nevents; + icp_io_events = 0; + if (theInIcpConnection >= 0) + fds[nfds++] = theInIcpConnection; + if (theInIcpConnection != theOutIcpConnection) + if (theOutIcpConnection >= 0) + fds[nfds++] = theOutIcpConnection; + if (nfds == 0) + return; + nevents = comm_check_incoming_poll_handlers(nfds, fds); + incoming_icp_interval += Config.comm_incoming.icp_average - nevents; + if (incoming_icp_interval < Config.comm_incoming.icp_min_poll) + incoming_icp_interval = Config.comm_incoming.icp_min_poll; + if (incoming_icp_interval > MAX_INCOMING_INTERVAL) + incoming_icp_interval = MAX_INCOMING_INTERVAL; + if (nevents > INCOMING_ICP_MAX) + nevents = INCOMING_ICP_MAX; + statHistCount(&statCounter.comm_icp_incoming, nevents); +} + +static void +comm_poll_http_incoming(void) +{ + int nfds = 0; + int fds[MAXHTTPPORTS]; + int j; + int nevents; + http_io_events = 0; + for (j = 0; j < NHttpSockets; j++) { + if (HttpSockets[j] < 0) + continue; + if (commDeferRead(HttpSockets[j])) + continue; + fds[nfds++] = HttpSockets[j]; + } + nevents = comm_check_incoming_poll_handlers(nfds, fds); + incoming_http_interval = incoming_http_interval + + Config.comm_incoming.http_average - nevents; + if (incoming_http_interval < Config.comm_incoming.http_min_poll) + incoming_http_interval = Config.comm_incoming.http_min_poll; + if (incoming_http_interval > MAX_INCOMING_INTERVAL) + incoming_http_interval = MAX_INCOMING_INTERVAL; + if (nevents > INCOMING_HTTP_MAX) + nevents = INCOMING_HTTP_MAX; + statHistCount(&statCounter.comm_http_incoming, nevents); +} + +/* poll all sockets; call handlers for those that are ready. */ +comm_err_t +comm_select(int msec) +{ + struct pollfd pfds[SQUID_MAXFD]; +#if DELAY_POOLS + fd_set slowfds; +#endif + PF *hdl = NULL; + int fd; + int maxfd; + unsigned long nfds; + unsigned long npending; + int num; + int callicp = 0, callhttp = 0; + int calldns = 0; + static time_t last_timeout = 0; + double timeout = current_dtime + (msec / 1000.0); + do { + double start; + getCurrentTime(); + start = current_dtime; + /* Handle any fs callbacks that need doing */ + storeDirCallback(); +#if DELAY_POOLS + FD_ZERO(&slowfds); +#endif + if (commCheckICPIncoming) + comm_poll_icp_incoming(); + if (commCheckDNSIncoming) + comm_poll_dns_incoming(); + if (commCheckHTTPIncoming) + comm_poll_http_incoming(); + PROF_start(comm_poll_prep_pfds); + callicp = calldns = callhttp = 0; + nfds = 0; + npending = 0; + maxfd = Biggest_FD + 1; + for (int i = 0; i < maxfd; i++) { + int events; + events = 0; + /* Check each open socket for a handler. */ + if (fd_table[i].read_handler) { + switch (commDeferRead(i)) { + case 0: + events |= POLLRDNORM; + break; + case 1: + break; +#if DELAY_POOLS + case -1: + events |= POLLRDNORM; + FD_SET(i, &slowfds); + break; +#endif + default: + fatalf("bad return value from commDeferRead(FD %d)\n", i); + } + } + if (fd_table[i].write_handler) + events |= POLLWRNORM; + if (events) { + pfds[nfds].fd = i; + pfds[nfds].events = events; + pfds[nfds].revents = 0; + nfds++; + if ((events & POLLRDNORM) && fd_table[i].flags.read_pending) + npending++; + } + } + PROF_stop(comm_poll_prep_pfds); + if (nfds == 0) { + assert(shutting_down); + return COMM_SHUTDOWN; + } + if (npending) + msec = 0; + if (msec > MAX_POLL_TIME) + msec = MAX_POLL_TIME; + for (;;) { + PROF_start(comm_poll_normal); + statCounter.syscalls.polls++; + num = poll(pfds, nfds, msec); + statCounter.select_loops++; + PROF_stop(comm_poll_normal); + if (num >= 0 || npending >= 0) + break; + if (ignoreErrno(errno)) + continue; + debug(5, 0) ("comm_poll: poll failure: %s\n", xstrerror()); + assert(errno != EINVAL); + return COMM_ERROR; + /* NOTREACHED */ + } + debug(5, num ? 5 : 8) ("comm_poll: %d+%ld FDs ready\n", num, npending); + statHistCount(&statCounter.select_fds_hist, num); + /* Check timeout handlers ONCE each second. */ + if (squid_curtime > last_timeout) { + last_timeout = squid_curtime; + checkTimeouts(); + } + if (num == 0 && npending == 0) + continue; + /* scan each socket but the accept socket. Poll this + * more frequently to minimize losses due to the 5 connect + * limit in SunOS */ + PROF_start(comm_handle_ready_fd); + for (size_t loopIndex = 0; loopIndex < nfds; loopIndex++) { + fde *F; + int revents = pfds[loopIndex].revents; + fd = pfds[loopIndex].fd; + if (fd == -1) + continue; + if (fd_table[fd].flags.read_pending) + revents |= POLLIN; + if (revents == 0) + continue; + if (fdIsIcp(fd)) { + callicp = 1; + continue; + } + if (fdIsDns(fd)) { + calldns = 1; + continue; + } + if (fdIsHttp(fd)) { + callhttp = 1; + continue; + } + F = &fd_table[fd]; + if (revents & (POLLRDNORM | POLLIN | POLLHUP | POLLERR)) { + debug(5, 6) ("comm_poll: FD %d ready for reading\n", fd); + if (NULL == (hdl = F->read_handler)) + (void) 0; +#if DELAY_POOLS + else if (FD_ISSET(fd, &slowfds)) + commAddSlowFd(fd); +#endif + else { + PROF_start(comm_read_handler); + F->read_handler = NULL; + hdl(fd, F->read_data); + PROF_stop(comm_read_handler); + statCounter.select_fds++; + if (commCheckICPIncoming) + comm_poll_icp_incoming(); + if (commCheckDNSIncoming) + comm_poll_dns_incoming(); + if (commCheckHTTPIncoming) + comm_poll_http_incoming(); + } + } + if (revents & (POLLWRNORM | POLLOUT | POLLHUP | POLLERR)) { + debug(5, 5) ("comm_poll: FD %d ready for writing\n", fd); + if ((hdl = F->write_handler)) { + PROF_start(comm_write_handler); + F->write_handler = NULL; + hdl(fd, F->write_data); + PROF_stop(comm_write_handler); + statCounter.select_fds++; + if (commCheckICPIncoming) + comm_poll_icp_incoming(); + if (commCheckDNSIncoming) + comm_poll_dns_incoming(); + if (commCheckHTTPIncoming) + comm_poll_http_incoming(); + } + } + if (revents & POLLNVAL) { + close_handler *ch; + debug(5, 0) ("WARNING: FD %d has handlers, but it's invalid.\n", fd); + debug(5, 0) ("FD %d is a %s\n", fd, fdTypeStr[F->type]); + debug(5, 0) ("--> %s\n", F->desc); + debug(5, 0) ("tmout:%p read:%p write:%p\n", + F->timeout_handler, + F->read_handler, + F->write_handler); + for (ch = F->closeHandler; ch; ch = ch->next) + debug(5, 0) (" close handler: %p\n", ch->handler); + if (F->closeHandler) { + commCallCloseHandlers(fd); + } else if (F->timeout_handler) { + debug(5, 0) ("comm_poll: Calling Timeout Handler\n"); + F->timeout_handler(fd, F->timeout_data); + } + F->closeHandler = NULL; + F->timeout_handler = NULL; + F->read_handler = NULL; + F->write_handler = NULL; + if (F->flags.open) + fd_close(fd); + } + } + PROF_stop(comm_handle_ready_fd); + if (callicp) + comm_poll_icp_incoming(); + if (calldns) + comm_poll_dns_incoming(); + if (callhttp) + comm_poll_http_incoming(); +#if DELAY_POOLS + while ((fd = commGetSlowFd()) != -1) { + fde *F = &fd_table[fd]; + debug(5, 6) ("comm_select: slow FD %d selected for reading\n", fd); + if ((hdl = F->read_handler)) { + F->read_handler = NULL; + hdl(fd, F->read_data); + statCounter.select_fds++; + if (commCheckICPIncoming) + comm_poll_icp_incoming(); + if (commCheckDNSIncoming) + comm_poll_dns_incoming(); + if (commCheckHTTPIncoming) + comm_poll_http_incoming(); + } + } +#endif + getCurrentTime(); + statCounter.select_time += (current_dtime - start); + return COMM_OK; + } + while (timeout > current_dtime); + debug(5, 8) ("comm_poll: time out: %ld.\n", (long int) squid_curtime); + return COMM_TIMEOUT; +} + + +static void +comm_poll_dns_incoming(void) +{ + int nfds = 0; + int fds[2]; + int nevents; + dns_io_events = 0; + if (DnsSocket < 0) + return; + fds[nfds++] = DnsSocket; + nevents = comm_check_incoming_poll_handlers(nfds, fds); + if (nevents < 0) + return; + incoming_dns_interval += Config.comm_incoming.dns_average - nevents; + if (incoming_dns_interval < Config.comm_incoming.dns_min_poll) + incoming_dns_interval = Config.comm_incoming.dns_min_poll; + if (incoming_dns_interval > MAX_INCOMING_INTERVAL) + incoming_dns_interval = MAX_INCOMING_INTERVAL; + if (nevents > INCOMING_DNS_MAX) + nevents = INCOMING_DNS_MAX; + statHistCount(&statCounter.comm_dns_incoming, nevents); +} + +void +comm_select_init(void) +{ + cachemgrRegister("comm_incoming", + "comm_incoming() stats", + commIncomingStats, 0, 1); + FD_ZERO(&global_readfds); + FD_ZERO(&global_writefds); + nreadfds = nwritefds = 0; +} + + +static void +commIncomingStats(StoreEntry * sentry) +{ + StatCounters *f = &statCounter; + storeAppendPrintf(sentry, "Current incoming_icp_interval: %d\n", + incoming_icp_interval >> INCOMING_FACTOR); + storeAppendPrintf(sentry, "Current incoming_dns_interval: %d\n", + incoming_dns_interval >> INCOMING_FACTOR); + storeAppendPrintf(sentry, "Current incoming_http_interval: %d\n", + incoming_http_interval >> INCOMING_FACTOR); + storeAppendPrintf(sentry, "\n"); + storeAppendPrintf(sentry, "Histogram of events per incoming socket type\n"); + storeAppendPrintf(sentry, "ICP Messages handled per comm_poll_icp_incoming() call:\n"); + statHistDump(&f->comm_icp_incoming, sentry, statHistIntDumper); + storeAppendPrintf(sentry, "DNS Messages handled per comm_poll_dns_incoming() call:\n"); + statHistDump(&f->comm_dns_incoming, sentry, statHistIntDumper); + storeAppendPrintf(sentry, "HTTP Messages handled per comm_poll_http_incoming() call:\n"); + statHistDump(&f->comm_http_incoming, sentry, statHistIntDumper); +} + +void +commUpdateReadBits(int fd, PF * handler) +{ + if (handler && !FD_ISSET(fd, &global_readfds)) { + FD_SET(fd, &global_readfds); + nreadfds++; + } else if (!handler && FD_ISSET(fd, &global_readfds)) { + FD_CLR(fd, &global_readfds); + nreadfds--; + } +} + +void +commUpdateWriteBits(int fd, PF * handler) +{ + if (handler && !FD_ISSET(fd, &global_writefds)) { + FD_SET(fd, &global_writefds); + nwritefds++; + } else if (!handler && FD_ISSET(fd, &global_writefds)) { + FD_CLR(fd, &global_writefds); + nwritefds--; + } +} + +/* Called by async-io or diskd to speed up the polling */ +void +comm_quick_poll_required(void) +{ + MAX_POLL_TIME = 10; +} + +#endif /* USE_POLL */ --- squid/src/comm_select.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,736 +0,0 @@ - -/* - * $Id: comm_select.c,v 1.15.2.1 2002/10/04 07:07:24 rbcollins Exp $ - * - * DEBUG: section 5 Socket Functions - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "Store.h" - -#ifdef USE_SELECT - -static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */ - -#ifndef howmany -#define howmany(x, y) (((x)+((y)-1))/(y)) -#endif -#ifndef NBBY -#define NBBY 8 -#endif -#define FD_MASK_BYTES sizeof(fd_mask) -#define FD_MASK_BITS (FD_MASK_BYTES*NBBY) - -/* STATIC */ -static int examine_select(fd_set *, fd_set *); -static int fdIsHttp(int fd); -static int fdIsIcp(int fd); -static int fdIsDns(int fd); -static OBJH commIncomingStats; -static int comm_check_incoming_select_handlers(int nfds, int *fds); -static void comm_select_dns_incoming(void); -static void commUpdateReadBits(int fd, PF * handler); -static void commUpdateWriteBits(int fd, PF * handler); - - -static struct timeval zero_tv; -static fd_set global_readfds; -static fd_set global_writefds; -static int nreadfds; -static int nwritefds; - -/* - * Automatic tuning for incoming requests: - * - * INCOMING sockets are the ICP and HTTP ports. We need to check these - * fairly regularly, but how often? When the load increases, we - * want to check the incoming sockets more often. If we have a lot - * of incoming ICP, then we need to check these sockets more than - * if we just have HTTP. - * - * The variables 'incoming_icp_interval' and 'incoming_http_interval' - * determine how many normal I/O events to process before checking - * incoming sockets again. Note we store the incoming_interval - * multipled by a factor of (2^INCOMING_FACTOR) to have some - * pseudo-floating point precision. - * - * The variable 'icp_io_events' and 'http_io_events' counts how many normal - * I/O events have been processed since the last check on the incoming - * sockets. When io_events > incoming_interval, its time to check incoming - * sockets. - * - * Every time we check incoming sockets, we count how many new messages - * or connections were processed. This is used to adjust the - * incoming_interval for the next iteration. The new incoming_interval - * is calculated as the current incoming_interval plus what we would - * like to see as an average number of events minus the number of - * events just processed. - * - * incoming_interval = incoming_interval + target_average - number_of_events_processed - * - * There are separate incoming_interval counters for both HTTP and ICP events - * - * You can see the current values of the incoming_interval's, as well as - * a histogram of 'incoming_events' by asking the cache manager - * for 'comm_incoming', e.g.: - * - * % ./client mgr:comm_incoming - * - * Caveats: - * - * - We have MAX_INCOMING_INTEGER as a magic upper limit on - * incoming_interval for both types of sockets. At the - * largest value the cache will effectively be idling. - * - * - The higher the INCOMING_FACTOR, the slower the algorithm will - * respond to load spikes/increases/decreases in demand. A value - * between 3 and 8 is recommended. - */ - -#define MAX_INCOMING_INTEGER 256 -#define INCOMING_FACTOR 5 -#define MAX_INCOMING_INTERVAL (MAX_INCOMING_INTEGER << INCOMING_FACTOR) -static int icp_io_events = 0; -static int dns_io_events = 0; -static int http_io_events = 0; -static int incoming_icp_interval = 16 << INCOMING_FACTOR; -static int incoming_dns_interval = 16 << INCOMING_FACTOR; -static int incoming_http_interval = 16 << INCOMING_FACTOR; -#define commCheckICPIncoming (++icp_io_events > (incoming_icp_interval>> INCOMING_FACTOR)) -#define commCheckDNSIncoming (++dns_io_events > (incoming_dns_interval>> INCOMING_FACTOR)) -#define commCheckHTTPIncoming (++http_io_events > (incoming_http_interval>> INCOMING_FACTOR)) - -void -commSetSelect(int fd, unsigned int type, PF * handler, void *client_data, - time_t timeout) -{ - fde *F = &fd_table[fd]; - assert(fd >= 0); - assert(F->flags.open); - debug(5, 5) ("commSetSelect: FD %d type %d\n", fd, type); - if (type & COMM_SELECT_READ) { - F->read_handler = handler; - F->read_data = client_data; - commUpdateReadBits(fd, handler); - } - if (type & COMM_SELECT_WRITE) { - F->write_handler = handler; - F->write_data = client_data; - commUpdateWriteBits(fd, handler); - } - if (timeout) - F->timeout = squid_curtime + timeout; -} - - -static int -fdIsIcp(int fd) -{ - if (fd == theInIcpConnection) - return 1; - if (fd == theOutIcpConnection) - return 1; - return 0; -} - -static int -fdIsDns(int fd) -{ - if (fd == DnsSocket) - return 1; - return 0; -} - -static int -fdIsHttp(int fd) -{ - int j; - for (j = 0; j < NHttpSockets; j++) { - if (fd == HttpSockets[j]) - return 1; - } - return 0; -} - -#if DELAY_POOLS -static int slowfdcnt = 0; -static int slowfdarr[SQUID_MAXFD]; - -static void -commAddSlowFd(int fd) -{ - assert(slowfdcnt < SQUID_MAXFD); - slowfdarr[slowfdcnt++] = fd; -} - -static int -commGetSlowFd(void) -{ - int whichfd, retfd; - - if (!slowfdcnt) - return -1; - whichfd = squid_random() % slowfdcnt; - retfd = slowfdarr[whichfd]; - slowfdarr[whichfd] = slowfdarr[--slowfdcnt]; - return retfd; -} -#endif - -static int -comm_check_incoming_select_handlers(int nfds, int *fds) -{ - int i; - int fd; - int maxfd = 0; - PF *hdl = NULL; - fd_set read_mask; - fd_set write_mask; - FD_ZERO(&read_mask); - FD_ZERO(&write_mask); - incoming_sockets_accepted = 0; - for (i = 0; i < nfds; i++) { - fd = fds[i]; - if (fd_table[fd].read_handler) { - FD_SET(fd, &read_mask); - if (fd > maxfd) - maxfd = fd; - } - if (fd_table[fd].write_handler) { - FD_SET(fd, &write_mask); - if (fd > maxfd) - maxfd = fd; - } - } - if (maxfd++ == 0) - return -1; - getCurrentTime(); - statCounter.syscalls.selects++; - if (select(maxfd, &read_mask, &write_mask, NULL, &zero_tv) < 1) - return incoming_sockets_accepted; - for (i = 0; i < nfds; i++) { - fd = fds[i]; - if (FD_ISSET(fd, &read_mask)) { - if ((hdl = fd_table[fd].read_handler) != NULL) { - fd_table[fd].read_handler = NULL; - commUpdateReadBits(fd, NULL); - hdl(fd, fd_table[fd].read_data); - } else { - debug(5, 1) ("comm_select_incoming: FD %d NULL read handler\n", - fd); - } - } - if (FD_ISSET(fd, &write_mask)) { - if ((hdl = fd_table[fd].write_handler) != NULL) { - fd_table[fd].write_handler = NULL; - commUpdateWriteBits(fd, NULL); - hdl(fd, fd_table[fd].write_data); - } else { - debug(5, 1) ("comm_select_incoming: FD %d NULL write handler\n", - fd); - } - } - } - return incoming_sockets_accepted; -} - -static void -comm_select_icp_incoming(void) -{ - int nfds = 0; - int fds[2]; - int nevents; - icp_io_events = 0; - if (theInIcpConnection >= 0) - fds[nfds++] = theInIcpConnection; - if (theInIcpConnection != theOutIcpConnection) - if (theOutIcpConnection >= 0) - fds[nfds++] = theOutIcpConnection; - if (nfds == 0) - return; - nevents = comm_check_incoming_select_handlers(nfds, fds); - incoming_icp_interval += Config.comm_incoming.icp_average - nevents; - if (incoming_icp_interval < 0) - incoming_icp_interval = 0; - if (incoming_icp_interval > MAX_INCOMING_INTERVAL) - incoming_icp_interval = MAX_INCOMING_INTERVAL; - if (nevents > INCOMING_ICP_MAX) - nevents = INCOMING_ICP_MAX; - statHistCount(&statCounter.comm_icp_incoming, nevents); -} - -static void -comm_select_http_incoming(void) -{ - int nfds = 0; - int fds[MAXHTTPPORTS]; - int j; - int nevents; - http_io_events = 0; - for (j = 0; j < NHttpSockets; j++) { - if (HttpSockets[j] < 0) - continue; - if (commDeferRead(HttpSockets[j])) - continue; - fds[nfds++] = HttpSockets[j]; - } - nevents = comm_check_incoming_select_handlers(nfds, fds); - incoming_http_interval += Config.comm_incoming.http_average - nevents; - if (incoming_http_interval < 0) - incoming_http_interval = 0; - if (incoming_http_interval > MAX_INCOMING_INTERVAL) - incoming_http_interval = MAX_INCOMING_INTERVAL; - if (nevents > INCOMING_HTTP_MAX) - nevents = INCOMING_HTTP_MAX; - statHistCount(&statCounter.comm_http_incoming, nevents); -} - -#define DEBUG_FDBITS 0 -/* Select on all sockets; call handlers for those that are ready. */ -comm_err_t -comm_select(int msec) -{ - fd_set readfds; - fd_set pendingfds; - fd_set writefds; -#if DELAY_POOLS - fd_set slowfds; -#endif - PF *hdl = NULL; - int fd; - int maxfd; - int num; - int pending; - int callicp = 0, callhttp = 0; - int calldns = 0; - int maxindex; - int k; - int j; -#if DEBUG_FDBITS - int i; -#endif - fd_mask *fdsp; - fd_mask *pfdsp; - fd_mask tmask; - static time_t last_timeout = 0; - struct timeval poll_time; - double timeout = current_dtime + (msec / 1000.0); - fde *F; - do { - getCurrentTime(); -#if DELAY_POOLS - FD_ZERO(&slowfds); -#endif - /* Handle any fs callbacks that need doing */ - storeDirCallback(); - if (commCheckICPIncoming) - comm_select_icp_incoming(); - if (commCheckDNSIncoming) - comm_select_dns_incoming(); - if (commCheckHTTPIncoming) - comm_select_http_incoming(); - callicp = calldns = callhttp = 0; - maxfd = Biggest_FD + 1; - xmemcpy(&readfds, &global_readfds, - howmany(maxfd, FD_MASK_BITS) * FD_MASK_BYTES); - xmemcpy(&writefds, &global_writefds, - howmany(maxfd, FD_MASK_BITS) * FD_MASK_BYTES); - /* remove stalled FDs, and deal with pending descriptors */ - pending = 0; - FD_ZERO(&pendingfds); - maxindex = howmany(maxfd, FD_MASK_BITS); - fdsp = (fd_mask *) & readfds; - for (j = 0; j < maxindex; j++) { - if ((tmask = fdsp[j]) == 0) - continue; /* no bits here */ - for (k = 0; k < FD_MASK_BITS; k++) { - if (!EBIT_TEST(tmask, k)) - continue; - /* Found a set bit */ - fd = (j * FD_MASK_BITS) + k; - switch (commDeferRead(fd)) { - case 0: - break; - case 1: - FD_CLR(fd, &readfds); - break; -#if DELAY_POOLS - case -1: - FD_SET(fd, &slowfds); - break; -#endif - default: - fatalf("bad return value from commDeferRead(FD %d)\n", fd); - } - if (FD_ISSET(fd, &readfds) && fd_table[fd].flags.read_pending) { - FD_SET(fd, &pendingfds); - pending++; - } - } - } -#if DEBUG_FDBITS - for (i = 0; i < maxfd; i++) { - /* Check each open socket for a handler. */ -#if DELAY_POOLS - if (fd_table[i].read_handler && commDeferRead(i) != 1) { -#else - if (fd_table[i].read_handler && !commDeferRead(i)) { -#endif - assert(FD_ISSET(i, &readfds)); - } - if (fd_table[i].write_handler) { - assert(FD_ISSET(i, &writefds)); - } - } -#endif - if (nreadfds + nwritefds == 0) { - assert(shutting_down); - return COMM_SHUTDOWN; - } - if (msec > MAX_POLL_TIME) - msec = MAX_POLL_TIME; -#ifdef _SQUID_OS2_ - if (msec < 0) - msec = MAX_POLL_TIME; -#endif - if (pending) - msec = 0; - for (;;) { - poll_time.tv_sec = msec / 1000; - poll_time.tv_usec = (msec % 1000) * 1000; - statCounter.syscalls.selects++; - num = select(maxfd, &readfds, &writefds, NULL, &poll_time); - statCounter.select_loops++; - if (num >= 0 || pending > 0) - break; - if (ignoreErrno(errno)) - break; - debug(50, 0) ("comm_select: select failure: %s\n", - xstrerror()); - examine_select(&readfds, &writefds); - return COMM_ERROR; - /* NOTREACHED */ - } - if (num < 0 && !pending) - continue; - debug(5, num ? 5 : 8) ("comm_select: %d+%d FDs ready at %d\n", - num, pending, (int) squid_curtime); - statHistCount(&statCounter.select_fds_hist, num); - /* Check lifetime and timeout handlers ONCE each second. - * Replaces brain-dead check every time through the loop! */ - if (squid_curtime > last_timeout) { - last_timeout = squid_curtime; - checkTimeouts(); - } - if (num == 0 && pending == 0) - continue; - /* Scan return fd masks for ready descriptors */ - fdsp = (fd_mask *) & readfds; - pfdsp = (fd_mask *) & pendingfds; - maxindex = howmany(maxfd, FD_MASK_BITS); - for (j = 0; j < maxindex; j++) { - if ((tmask = (fdsp[j] | pfdsp[j])) == 0) - continue; /* no bits here */ - for (k = 0; k < FD_MASK_BITS; k++) { - if (tmask == 0) - break; /* no more bits left */ - if (!EBIT_TEST(tmask, k)) - continue; - /* Found a set bit */ - fd = (j * FD_MASK_BITS) + k; - EBIT_CLR(tmask, k); /* this will be done */ -#if DEBUG_FDBITS - debug(5, 9) ("FD %d bit set for reading\n", fd); - assert(FD_ISSET(fd, &readfds)); -#endif - if (fdIsIcp(fd)) { - callicp = 1; - continue; - } - if (fdIsDns(fd)) { - calldns = 1; - continue; - } - if (fdIsHttp(fd)) { - callhttp = 1; - continue; - } - F = &fd_table[fd]; - debug(5, 6) ("comm_select: FD %d ready for reading\n", fd); - if (NULL == (hdl = F->read_handler)) - (void) 0; -#if DELAY_POOLS - else if (FD_ISSET(fd, &slowfds)) - commAddSlowFd(fd); -#endif - else { - F->read_handler = NULL; - commUpdateReadBits(fd, NULL); - hdl(fd, F->read_data); - statCounter.select_fds++; - if (commCheckICPIncoming) - comm_select_icp_incoming(); - if (commCheckDNSIncoming) - comm_select_dns_incoming(); - if (commCheckHTTPIncoming) - comm_select_http_incoming(); - } - } - } - fdsp = (fd_mask *) & writefds; - for (j = 0; j < maxindex; j++) { - if ((tmask = fdsp[j]) == 0) - continue; /* no bits here */ - for (k = 0; k < FD_MASK_BITS; k++) { - if (tmask == 0) - break; /* no more bits left */ - if (!EBIT_TEST(tmask, k)) - continue; - /* Found a set bit */ - fd = (j * FD_MASK_BITS) + k; - EBIT_CLR(tmask, k); /* this will be done */ -#if DEBUG_FDBITS - debug(5, 9) ("FD %d bit set for writing\n", fd); - assert(FD_ISSET(fd, &writefds)); -#endif - if (fdIsIcp(fd)) { - callicp = 1; - continue; - } - if (fdIsDns(fd)) { - calldns = 1; - continue; - } - if (fdIsHttp(fd)) { - callhttp = 1; - continue; - } - F = &fd_table[fd]; - debug(5, 5) ("comm_select: FD %d ready for writing\n", fd); - if ((hdl = F->write_handler)) { - F->write_handler = NULL; - commUpdateWriteBits(fd, NULL); - hdl(fd, F->write_data); - statCounter.select_fds++; - if (commCheckICPIncoming) - comm_select_icp_incoming(); - if (commCheckDNSIncoming) - comm_select_dns_incoming(); - if (commCheckHTTPIncoming) - comm_select_http_incoming(); - } - } - } - if (callicp) - comm_select_icp_incoming(); - if (calldns) - comm_select_dns_incoming(); - if (callhttp) - comm_select_http_incoming(); -#if DELAY_POOLS - while ((fd = commGetSlowFd()) != -1) { - F = &fd_table[fd]; - debug(5, 6) ("comm_select: slow FD %d selected for reading\n", fd); - if ((hdl = F->read_handler)) { - F->read_handler = NULL; - commUpdateReadBits(fd, NULL); - hdl(fd, F->read_data); - statCounter.select_fds++; - if (commCheckICPIncoming) - comm_select_icp_incoming(); - if (commCheckDNSIncoming) - comm_select_dns_incoming(); - if (commCheckHTTPIncoming) - comm_select_http_incoming(); - } - } -#endif - return COMM_OK; - } - while (timeout > current_dtime); - debug(5, 8) ("comm_select: time out: %d\n", (int) squid_curtime); - return COMM_TIMEOUT; -} - -static void -comm_select_dns_incoming(void) -{ - int nfds = 0; - int fds[2]; - int nevents; - dns_io_events = 0; - if (DnsSocket < 0) - return; - fds[nfds++] = DnsSocket; - nevents = comm_check_incoming_select_handlers(nfds, fds); - if (nevents < 0) - return; - incoming_dns_interval += Config.comm_incoming.dns_average - nevents; - if (incoming_dns_interval < Config.comm_incoming.dns_min_poll) - incoming_dns_interval = Config.comm_incoming.dns_min_poll; - if (incoming_dns_interval > MAX_INCOMING_INTERVAL) - incoming_dns_interval = MAX_INCOMING_INTERVAL; - if (nevents > INCOMING_DNS_MAX) - nevents = INCOMING_DNS_MAX; - statHistCount(&statCounter.comm_dns_incoming, nevents); -} - -void -comm_select_init(void) -{ - zero_tv.tv_sec = 0; - zero_tv.tv_usec = 0; - cachemgrRegister("comm_incoming", - "comm_incoming() stats", - commIncomingStats, 0, 1); - FD_ZERO(&global_readfds); - FD_ZERO(&global_writefds); - nreadfds = nwritefds = 0; -} - -/* - * examine_select - debug routine. - * - * I spend the day chasing this core dump that occurs when both the client - * and the server side of a cache fetch simultaneoulsy abort the - * connection. While I haven't really studied the code to figure out how - * it happens, the snippet below may prevent the cache from exitting: - * - * Call this from where the select loop fails. - */ -static int -examine_select(fd_set * readfds, fd_set * writefds) -{ - int fd = 0; - fd_set read_x; - fd_set write_x; - struct timeval tv; - close_handler *ch = NULL; - fde *F = NULL; - struct stat sb; - debug(5, 0) ("examine_select: Examining open file descriptors...\n"); - for (fd = 0; fd < Squid_MaxFD; fd++) { - FD_ZERO(&read_x); - FD_ZERO(&write_x); - tv.tv_sec = tv.tv_usec = 0; - if (FD_ISSET(fd, readfds)) - FD_SET(fd, &read_x); - else if (FD_ISSET(fd, writefds)) - FD_SET(fd, &write_x); - else - continue; - statCounter.syscalls.selects++; - errno = 0; - if (!fstat(fd, &sb)) { - debug(5, 5) ("FD %d is valid.\n", fd); - continue; - } - F = &fd_table[fd]; - debug(5, 0) ("FD %d: %s\n", fd, xstrerror()); - debug(5, 0) ("WARNING: FD %d has handlers, but it's invalid.\n", fd); - debug(5, 0) ("FD %d is a %s called '%s'\n", - fd, - fdTypeStr[F->type], - F->desc); - debug(5, 0) ("tmout:%p read:%p write:%p\n", - F->timeout_handler, - F->read_handler, - F->write_handler); - for (ch = F->closeHandler; ch; ch = ch->next) - debug(5, 0) (" close handler: %p\n", ch->handler); - if (F->closeHandler) { - commCallCloseHandlers(fd); - } else if (F->timeout_handler) { - debug(5, 0) ("examine_select: Calling Timeout Handler\n"); - F->timeout_handler(fd, F->timeout_data); - } - F->closeHandler = NULL; - F->timeout_handler = NULL; - F->read_handler = NULL; - F->write_handler = NULL; - FD_CLR(fd, readfds); - FD_CLR(fd, writefds); - } - return 0; -} - - -static void -commIncomingStats(StoreEntry * sentry) -{ - StatCounters *f = &statCounter; - storeAppendPrintf(sentry, "Current incoming_icp_interval: %d\n", - incoming_icp_interval >> INCOMING_FACTOR); - storeAppendPrintf(sentry, "Current incoming_dns_interval: %d\n", - incoming_dns_interval >> INCOMING_FACTOR); - storeAppendPrintf(sentry, "Current incoming_http_interval: %d\n", - incoming_http_interval >> INCOMING_FACTOR); - storeAppendPrintf(sentry, "\n"); - storeAppendPrintf(sentry, "Histogram of events per incoming socket type\n"); - storeAppendPrintf(sentry, "ICP Messages handled per comm_select_icp_incoming() call:\n"); - statHistDump(&f->comm_icp_incoming, sentry, statHistIntDumper); - storeAppendPrintf(sentry, "DNS Messages handled per comm_select_dns_incoming() call:\n"); - statHistDump(&f->comm_dns_incoming, sentry, statHistIntDumper); - storeAppendPrintf(sentry, "HTTP Messages handled per comm_select_http_incoming() call:\n"); - statHistDump(&f->comm_http_incoming, sentry, statHistIntDumper); -} - -void -commUpdateReadBits(int fd, PF * handler) -{ - if (handler && !FD_ISSET(fd, &global_readfds)) { - FD_SET(fd, &global_readfds); - nreadfds++; - } else if (!handler && FD_ISSET(fd, &global_readfds)) { - FD_CLR(fd, &global_readfds); - nreadfds--; - } -} - -void -commUpdateWriteBits(int fd, PF * handler) -{ - if (handler && !FD_ISSET(fd, &global_writefds)) { - FD_SET(fd, &global_writefds); - nwritefds++; - } else if (!handler && FD_ISSET(fd, &global_writefds)) { - FD_CLR(fd, &global_writefds); - nwritefds--; - } -} - -/* Called by async-io or diskd to speed up the polling */ -void -comm_quick_poll_required(void) -{ - MAX_POLL_TIME = 10; -} - -#endif /* USE_SELECT */ --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/comm_select.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,736 @@ + +/* + * $Id: comm_select.cc,v 1.1.2.1 2002/10/11 15:40:47 rbcollins Exp $ + * + * DEBUG: section 5 Socket Functions + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "Store.h" + +#ifdef USE_SELECT + +static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */ + +#ifndef howmany +#define howmany(x, y) (((x)+((y)-1))/(y)) +#endif +#ifndef NBBY +#define NBBY 8 +#endif +#define FD_MASK_BYTES sizeof(fd_mask) +#define FD_MASK_BITS (FD_MASK_BYTES*NBBY) + +/* STATIC */ +static int examine_select(fd_set *, fd_set *); +static int fdIsHttp(int fd); +static int fdIsIcp(int fd); +static int fdIsDns(int fd); +static OBJH commIncomingStats; +static int comm_check_incoming_select_handlers(int nfds, int *fds); +static void comm_select_dns_incoming(void); +static void commUpdateReadBits(int fd, PF * handler); +static void commUpdateWriteBits(int fd, PF * handler); + + +static struct timeval zero_tv; +static fd_set global_readfds; +static fd_set global_writefds; +static int nreadfds; +static int nwritefds; + +/* + * Automatic tuning for incoming requests: + * + * INCOMING sockets are the ICP and HTTP ports. We need to check these + * fairly regularly, but how often? When the load increases, we + * want to check the incoming sockets more often. If we have a lot + * of incoming ICP, then we need to check these sockets more than + * if we just have HTTP. + * + * The variables 'incoming_icp_interval' and 'incoming_http_interval' + * determine how many normal I/O events to process before checking + * incoming sockets again. Note we store the incoming_interval + * multipled by a factor of (2^INCOMING_FACTOR) to have some + * pseudo-floating point precision. + * + * The variable 'icp_io_events' and 'http_io_events' counts how many normal + * I/O events have been processed since the last check on the incoming + * sockets. When io_events > incoming_interval, its time to check incoming + * sockets. + * + * Every time we check incoming sockets, we count how many new messages + * or connections were processed. This is used to adjust the + * incoming_interval for the next iteration. The new incoming_interval + * is calculated as the current incoming_interval plus what we would + * like to see as an average number of events minus the number of + * events just processed. + * + * incoming_interval = incoming_interval + target_average - number_of_events_processed + * + * There are separate incoming_interval counters for both HTTP and ICP events + * + * You can see the current values of the incoming_interval's, as well as + * a histogram of 'incoming_events' by asking the cache manager + * for 'comm_incoming', e.g.: + * + * % ./client mgr:comm_incoming + * + * Caveats: + * + * - We have MAX_INCOMING_INTEGER as a magic upper limit on + * incoming_interval for both types of sockets. At the + * largest value the cache will effectively be idling. + * + * - The higher the INCOMING_FACTOR, the slower the algorithm will + * respond to load spikes/increases/decreases in demand. A value + * between 3 and 8 is recommended. + */ + +#define MAX_INCOMING_INTEGER 256 +#define INCOMING_FACTOR 5 +#define MAX_INCOMING_INTERVAL (MAX_INCOMING_INTEGER << INCOMING_FACTOR) +static int icp_io_events = 0; +static int dns_io_events = 0; +static int http_io_events = 0; +static int incoming_icp_interval = 16 << INCOMING_FACTOR; +static int incoming_dns_interval = 16 << INCOMING_FACTOR; +static int incoming_http_interval = 16 << INCOMING_FACTOR; +#define commCheckICPIncoming (++icp_io_events > (incoming_icp_interval>> INCOMING_FACTOR)) +#define commCheckDNSIncoming (++dns_io_events > (incoming_dns_interval>> INCOMING_FACTOR)) +#define commCheckHTTPIncoming (++http_io_events > (incoming_http_interval>> INCOMING_FACTOR)) + +void +commSetSelect(int fd, unsigned int type, PF * handler, void *client_data, + time_t timeout) +{ + fde *F = &fd_table[fd]; + assert(fd >= 0); + assert(F->flags.open); + debug(5, 5) ("commSetSelect: FD %d type %d\n", fd, type); + if (type & COMM_SELECT_READ) { + F->read_handler = handler; + F->read_data = client_data; + commUpdateReadBits(fd, handler); + } + if (type & COMM_SELECT_WRITE) { + F->write_handler = handler; + F->write_data = client_data; + commUpdateWriteBits(fd, handler); + } + if (timeout) + F->timeout = squid_curtime + timeout; +} + + +static int +fdIsIcp(int fd) +{ + if (fd == theInIcpConnection) + return 1; + if (fd == theOutIcpConnection) + return 1; + return 0; +} + +static int +fdIsDns(int fd) +{ + if (fd == DnsSocket) + return 1; + return 0; +} + +static int +fdIsHttp(int fd) +{ + int j; + for (j = 0; j < NHttpSockets; j++) { + if (fd == HttpSockets[j]) + return 1; + } + return 0; +} + +#if DELAY_POOLS +static int slowfdcnt = 0; +static int slowfdarr[SQUID_MAXFD]; + +static void +commAddSlowFd(int fd) +{ + assert(slowfdcnt < SQUID_MAXFD); + slowfdarr[slowfdcnt++] = fd; +} + +static int +commGetSlowFd(void) +{ + int whichfd, retfd; + + if (!slowfdcnt) + return -1; + whichfd = squid_random() % slowfdcnt; + retfd = slowfdarr[whichfd]; + slowfdarr[whichfd] = slowfdarr[--slowfdcnt]; + return retfd; +} +#endif + +static int +comm_check_incoming_select_handlers(int nfds, int *fds) +{ + int i; + int fd; + int maxfd = 0; + PF *hdl = NULL; + fd_set read_mask; + fd_set write_mask; + FD_ZERO(&read_mask); + FD_ZERO(&write_mask); + incoming_sockets_accepted = 0; + for (i = 0; i < nfds; i++) { + fd = fds[i]; + if (fd_table[fd].read_handler) { + FD_SET(fd, &read_mask); + if (fd > maxfd) + maxfd = fd; + } + if (fd_table[fd].write_handler) { + FD_SET(fd, &write_mask); + if (fd > maxfd) + maxfd = fd; + } + } + if (maxfd++ == 0) + return -1; + getCurrentTime(); + statCounter.syscalls.selects++; + if (select(maxfd, &read_mask, &write_mask, NULL, &zero_tv) < 1) + return incoming_sockets_accepted; + for (i = 0; i < nfds; i++) { + fd = fds[i]; + if (FD_ISSET(fd, &read_mask)) { + if ((hdl = fd_table[fd].read_handler) != NULL) { + fd_table[fd].read_handler = NULL; + commUpdateReadBits(fd, NULL); + hdl(fd, fd_table[fd].read_data); + } else { + debug(5, 1) ("comm_select_incoming: FD %d NULL read handler\n", + fd); + } + } + if (FD_ISSET(fd, &write_mask)) { + if ((hdl = fd_table[fd].write_handler) != NULL) { + fd_table[fd].write_handler = NULL; + commUpdateWriteBits(fd, NULL); + hdl(fd, fd_table[fd].write_data); + } else { + debug(5, 1) ("comm_select_incoming: FD %d NULL write handler\n", + fd); + } + } + } + return incoming_sockets_accepted; +} + +static void +comm_select_icp_incoming(void) +{ + int nfds = 0; + int fds[2]; + int nevents; + icp_io_events = 0; + if (theInIcpConnection >= 0) + fds[nfds++] = theInIcpConnection; + if (theInIcpConnection != theOutIcpConnection) + if (theOutIcpConnection >= 0) + fds[nfds++] = theOutIcpConnection; + if (nfds == 0) + return; + nevents = comm_check_incoming_select_handlers(nfds, fds); + incoming_icp_interval += Config.comm_incoming.icp_average - nevents; + if (incoming_icp_interval < 0) + incoming_icp_interval = 0; + if (incoming_icp_interval > MAX_INCOMING_INTERVAL) + incoming_icp_interval = MAX_INCOMING_INTERVAL; + if (nevents > INCOMING_ICP_MAX) + nevents = INCOMING_ICP_MAX; + statHistCount(&statCounter.comm_icp_incoming, nevents); +} + +static void +comm_select_http_incoming(void) +{ + int nfds = 0; + int fds[MAXHTTPPORTS]; + int j; + int nevents; + http_io_events = 0; + for (j = 0; j < NHttpSockets; j++) { + if (HttpSockets[j] < 0) + continue; + if (commDeferRead(HttpSockets[j])) + continue; + fds[nfds++] = HttpSockets[j]; + } + nevents = comm_check_incoming_select_handlers(nfds, fds); + incoming_http_interval += Config.comm_incoming.http_average - nevents; + if (incoming_http_interval < 0) + incoming_http_interval = 0; + if (incoming_http_interval > MAX_INCOMING_INTERVAL) + incoming_http_interval = MAX_INCOMING_INTERVAL; + if (nevents > INCOMING_HTTP_MAX) + nevents = INCOMING_HTTP_MAX; + statHistCount(&statCounter.comm_http_incoming, nevents); +} + +#define DEBUG_FDBITS 0 +/* Select on all sockets; call handlers for those that are ready. */ +comm_err_t +comm_select(int msec) +{ + fd_set readfds; + fd_set pendingfds; + fd_set writefds; +#if DELAY_POOLS + fd_set slowfds; +#endif + PF *hdl = NULL; + int fd; + int maxfd; + int num; + int pending; + int callicp = 0, callhttp = 0; + int calldns = 0; + int maxindex; + int k; + int j; +#if DEBUG_FDBITS + int i; +#endif + fd_mask *fdsp; + fd_mask *pfdsp; + fd_mask tmask; + static time_t last_timeout = 0; + struct timeval poll_time; + double timeout = current_dtime + (msec / 1000.0); + fde *F; + do { + getCurrentTime(); +#if DELAY_POOLS + FD_ZERO(&slowfds); +#endif + /* Handle any fs callbacks that need doing */ + storeDirCallback(); + if (commCheckICPIncoming) + comm_select_icp_incoming(); + if (commCheckDNSIncoming) + comm_select_dns_incoming(); + if (commCheckHTTPIncoming) + comm_select_http_incoming(); + callicp = calldns = callhttp = 0; + maxfd = Biggest_FD + 1; + xmemcpy(&readfds, &global_readfds, + howmany(maxfd, FD_MASK_BITS) * FD_MASK_BYTES); + xmemcpy(&writefds, &global_writefds, + howmany(maxfd, FD_MASK_BITS) * FD_MASK_BYTES); + /* remove stalled FDs, and deal with pending descriptors */ + pending = 0; + FD_ZERO(&pendingfds); + maxindex = howmany(maxfd, FD_MASK_BITS); + fdsp = (fd_mask *) & readfds; + for (j = 0; j < maxindex; j++) { + if ((tmask = fdsp[j]) == 0) + continue; /* no bits here */ + for (k = 0; k < FD_MASK_BITS; k++) { + if (!EBIT_TEST(tmask, k)) + continue; + /* Found a set bit */ + fd = (j * FD_MASK_BITS) + k; + switch (commDeferRead(fd)) { + case 0: + break; + case 1: + FD_CLR(fd, &readfds); + break; +#if DELAY_POOLS + case -1: + FD_SET(fd, &slowfds); + break; +#endif + default: + fatalf("bad return value from commDeferRead(FD %d)\n", fd); + } + if (FD_ISSET(fd, &readfds) && fd_table[fd].flags.read_pending) { + FD_SET(fd, &pendingfds); + pending++; + } + } + } +#if DEBUG_FDBITS + for (i = 0; i < maxfd; i++) { + /* Check each open socket for a handler. */ +#if DELAY_POOLS + if (fd_table[i].read_handler && commDeferRead(i) != 1) { +#else + if (fd_table[i].read_handler && !commDeferRead(i)) { +#endif + assert(FD_ISSET(i, &readfds)); + } + if (fd_table[i].write_handler) { + assert(FD_ISSET(i, &writefds)); + } + } +#endif + if (nreadfds + nwritefds == 0) { + assert(shutting_down); + return COMM_SHUTDOWN; + } + if (msec > MAX_POLL_TIME) + msec = MAX_POLL_TIME; +#ifdef _SQUID_OS2_ + if (msec < 0) + msec = MAX_POLL_TIME; +#endif + if (pending) + msec = 0; + for (;;) { + poll_time.tv_sec = msec / 1000; + poll_time.tv_usec = (msec % 1000) * 1000; + statCounter.syscalls.selects++; + num = select(maxfd, &readfds, &writefds, NULL, &poll_time); + statCounter.select_loops++; + if (num >= 0 || pending > 0) + break; + if (ignoreErrno(errno)) + break; + debug(50, 0) ("comm_select: select failure: %s\n", + xstrerror()); + examine_select(&readfds, &writefds); + return COMM_ERROR; + /* NOTREACHED */ + } + if (num < 0 && !pending) + continue; + debug(5, num ? 5 : 8) ("comm_select: %d+%d FDs ready at %d\n", + num, pending, (int) squid_curtime); + statHistCount(&statCounter.select_fds_hist, num); + /* Check lifetime and timeout handlers ONCE each second. + * Replaces brain-dead check every time through the loop! */ + if (squid_curtime > last_timeout) { + last_timeout = squid_curtime; + checkTimeouts(); + } + if (num == 0 && pending == 0) + continue; + /* Scan return fd masks for ready descriptors */ + fdsp = (fd_mask *) & readfds; + pfdsp = (fd_mask *) & pendingfds; + maxindex = howmany(maxfd, FD_MASK_BITS); + for (j = 0; j < maxindex; j++) { + if ((tmask = (fdsp[j] | pfdsp[j])) == 0) + continue; /* no bits here */ + for (k = 0; k < FD_MASK_BITS; k++) { + if (tmask == 0) + break; /* no more bits left */ + if (!EBIT_TEST(tmask, k)) + continue; + /* Found a set bit */ + fd = (j * FD_MASK_BITS) + k; + EBIT_CLR(tmask, k); /* this will be done */ +#if DEBUG_FDBITS + debug(5, 9) ("FD %d bit set for reading\n", fd); + assert(FD_ISSET(fd, &readfds)); +#endif + if (fdIsIcp(fd)) { + callicp = 1; + continue; + } + if (fdIsDns(fd)) { + calldns = 1; + continue; + } + if (fdIsHttp(fd)) { + callhttp = 1; + continue; + } + F = &fd_table[fd]; + debug(5, 6) ("comm_select: FD %d ready for reading\n", fd); + if (NULL == (hdl = F->read_handler)) + (void) 0; +#if DELAY_POOLS + else if (FD_ISSET(fd, &slowfds)) + commAddSlowFd(fd); +#endif + else { + F->read_handler = NULL; + commUpdateReadBits(fd, NULL); + hdl(fd, F->read_data); + statCounter.select_fds++; + if (commCheckICPIncoming) + comm_select_icp_incoming(); + if (commCheckDNSIncoming) + comm_select_dns_incoming(); + if (commCheckHTTPIncoming) + comm_select_http_incoming(); + } + } + } + fdsp = (fd_mask *) & writefds; + for (j = 0; j < maxindex; j++) { + if ((tmask = fdsp[j]) == 0) + continue; /* no bits here */ + for (k = 0; k < FD_MASK_BITS; k++) { + if (tmask == 0) + break; /* no more bits left */ + if (!EBIT_TEST(tmask, k)) + continue; + /* Found a set bit */ + fd = (j * FD_MASK_BITS) + k; + EBIT_CLR(tmask, k); /* this will be done */ +#if DEBUG_FDBITS + debug(5, 9) ("FD %d bit set for writing\n", fd); + assert(FD_ISSET(fd, &writefds)); +#endif + if (fdIsIcp(fd)) { + callicp = 1; + continue; + } + if (fdIsDns(fd)) { + calldns = 1; + continue; + } + if (fdIsHttp(fd)) { + callhttp = 1; + continue; + } + F = &fd_table[fd]; + debug(5, 5) ("comm_select: FD %d ready for writing\n", fd); + if ((hdl = F->write_handler)) { + F->write_handler = NULL; + commUpdateWriteBits(fd, NULL); + hdl(fd, F->write_data); + statCounter.select_fds++; + if (commCheckICPIncoming) + comm_select_icp_incoming(); + if (commCheckDNSIncoming) + comm_select_dns_incoming(); + if (commCheckHTTPIncoming) + comm_select_http_incoming(); + } + } + } + if (callicp) + comm_select_icp_incoming(); + if (calldns) + comm_select_dns_incoming(); + if (callhttp) + comm_select_http_incoming(); +#if DELAY_POOLS + while ((fd = commGetSlowFd()) != -1) { + F = &fd_table[fd]; + debug(5, 6) ("comm_select: slow FD %d selected for reading\n", fd); + if ((hdl = F->read_handler)) { + F->read_handler = NULL; + commUpdateReadBits(fd, NULL); + hdl(fd, F->read_data); + statCounter.select_fds++; + if (commCheckICPIncoming) + comm_select_icp_incoming(); + if (commCheckDNSIncoming) + comm_select_dns_incoming(); + if (commCheckHTTPIncoming) + comm_select_http_incoming(); + } + } +#endif + return COMM_OK; + } + while (timeout > current_dtime); + debug(5, 8) ("comm_select: time out: %d\n", (int) squid_curtime); + return COMM_TIMEOUT; +} + +static void +comm_select_dns_incoming(void) +{ + int nfds = 0; + int fds[2]; + int nevents; + dns_io_events = 0; + if (DnsSocket < 0) + return; + fds[nfds++] = DnsSocket; + nevents = comm_check_incoming_select_handlers(nfds, fds); + if (nevents < 0) + return; + incoming_dns_interval += Config.comm_incoming.dns_average - nevents; + if (incoming_dns_interval < Config.comm_incoming.dns_min_poll) + incoming_dns_interval = Config.comm_incoming.dns_min_poll; + if (incoming_dns_interval > MAX_INCOMING_INTERVAL) + incoming_dns_interval = MAX_INCOMING_INTERVAL; + if (nevents > INCOMING_DNS_MAX) + nevents = INCOMING_DNS_MAX; + statHistCount(&statCounter.comm_dns_incoming, nevents); +} + +void +comm_select_init(void) +{ + zero_tv.tv_sec = 0; + zero_tv.tv_usec = 0; + cachemgrRegister("comm_incoming", + "comm_incoming() stats", + commIncomingStats, 0, 1); + FD_ZERO(&global_readfds); + FD_ZERO(&global_writefds); + nreadfds = nwritefds = 0; +} + +/* + * examine_select - debug routine. + * + * I spend the day chasing this core dump that occurs when both the client + * and the server side of a cache fetch simultaneoulsy abort the + * connection. While I haven't really studied the code to figure out how + * it happens, the snippet below may prevent the cache from exitting: + * + * Call this from where the select loop fails. + */ +static int +examine_select(fd_set * readfds, fd_set * writefds) +{ + int fd = 0; + fd_set read_x; + fd_set write_x; + struct timeval tv; + close_handler *ch = NULL; + fde *F = NULL; + struct stat sb; + debug(5, 0) ("examine_select: Examining open file descriptors...\n"); + for (fd = 0; fd < Squid_MaxFD; fd++) { + FD_ZERO(&read_x); + FD_ZERO(&write_x); + tv.tv_sec = tv.tv_usec = 0; + if (FD_ISSET(fd, readfds)) + FD_SET(fd, &read_x); + else if (FD_ISSET(fd, writefds)) + FD_SET(fd, &write_x); + else + continue; + statCounter.syscalls.selects++; + errno = 0; + if (!fstat(fd, &sb)) { + debug(5, 5) ("FD %d is valid.\n", fd); + continue; + } + F = &fd_table[fd]; + debug(5, 0) ("FD %d: %s\n", fd, xstrerror()); + debug(5, 0) ("WARNING: FD %d has handlers, but it's invalid.\n", fd); + debug(5, 0) ("FD %d is a %s called '%s'\n", + fd, + fdTypeStr[F->type], + F->desc); + debug(5, 0) ("tmout:%p read:%p write:%p\n", + F->timeout_handler, + F->read_handler, + F->write_handler); + for (ch = F->closeHandler; ch; ch = ch->next) + debug(5, 0) (" close handler: %p\n", ch->handler); + if (F->closeHandler) { + commCallCloseHandlers(fd); + } else if (F->timeout_handler) { + debug(5, 0) ("examine_select: Calling Timeout Handler\n"); + F->timeout_handler(fd, F->timeout_data); + } + F->closeHandler = NULL; + F->timeout_handler = NULL; + F->read_handler = NULL; + F->write_handler = NULL; + FD_CLR(fd, readfds); + FD_CLR(fd, writefds); + } + return 0; +} + + +static void +commIncomingStats(StoreEntry * sentry) +{ + StatCounters *f = &statCounter; + storeAppendPrintf(sentry, "Current incoming_icp_interval: %d\n", + incoming_icp_interval >> INCOMING_FACTOR); + storeAppendPrintf(sentry, "Current incoming_dns_interval: %d\n", + incoming_dns_interval >> INCOMING_FACTOR); + storeAppendPrintf(sentry, "Current incoming_http_interval: %d\n", + incoming_http_interval >> INCOMING_FACTOR); + storeAppendPrintf(sentry, "\n"); + storeAppendPrintf(sentry, "Histogram of events per incoming socket type\n"); + storeAppendPrintf(sentry, "ICP Messages handled per comm_select_icp_incoming() call:\n"); + statHistDump(&f->comm_icp_incoming, sentry, statHistIntDumper); + storeAppendPrintf(sentry, "DNS Messages handled per comm_select_dns_incoming() call:\n"); + statHistDump(&f->comm_dns_incoming, sentry, statHistIntDumper); + storeAppendPrintf(sentry, "HTTP Messages handled per comm_select_http_incoming() call:\n"); + statHistDump(&f->comm_http_incoming, sentry, statHistIntDumper); +} + +void +commUpdateReadBits(int fd, PF * handler) +{ + if (handler && !FD_ISSET(fd, &global_readfds)) { + FD_SET(fd, &global_readfds); + nreadfds++; + } else if (!handler && FD_ISSET(fd, &global_readfds)) { + FD_CLR(fd, &global_readfds); + nreadfds--; + } +} + +void +commUpdateWriteBits(int fd, PF * handler) +{ + if (handler && !FD_ISSET(fd, &global_writefds)) { + FD_SET(fd, &global_writefds); + nwritefds++; + } else if (!handler && FD_ISSET(fd, &global_writefds)) { + FD_CLR(fd, &global_writefds); + nwritefds--; + } +} + +/* Called by async-io or diskd to speed up the polling */ +void +comm_quick_poll_required(void) +{ + MAX_POLL_TIME = 10; +} + +#endif /* USE_SELECT */ --- squid/src/debug.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,436 +0,0 @@ - -/* - * $Id: debug.c,v 1.10 2002/09/01 16:30:42 squidadm Exp $ - * - * DEBUG: section 0 Debug Routines - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -static char *debug_log_file = NULL; -static int Ctx_Lock = 0; -static const char *debugLogTime(time_t); -static void ctx_print(void); -#if HAVE_SYSLOG -static void _db_print_syslog(const char *format, va_list args); -#endif -static void _db_print_stderr(const char *format, va_list args); -static void _db_print_file(const char *format, va_list args); - -void -#if STDC_HEADERS -_db_print(const char *format,...) -{ -#else -_db_print(va_alist) - va_dcl -{ - const char *format = NULL; -#endif - LOCAL_ARRAY(char, f, BUFSIZ); - va_list args1; -#if STDC_HEADERS - va_list args2; - va_list args3; -#else -#define args2 args1 -#define args3 args1 -#endif - /* give a chance to context-based debugging to print current context */ - if (!Ctx_Lock) - ctx_print(); -#if STDC_HEADERS - va_start(args1, format); - va_start(args2, format); - va_start(args3, format); -#else - format = va_arg(args1, const char *); -#endif - snprintf(f, BUFSIZ, "%s| %s", - debugLogTime(squid_curtime), - format); - _db_print_file(f, args1); - _db_print_stderr(f, args2); -#if HAVE_SYSLOG - _db_print_syslog(format, args3); -#endif - va_end(args1); -#if STDC_HEADERS - va_end(args2); - va_end(args3); -#endif -} - -static void -_db_print_file(const char *format, va_list args) -{ - if (debug_log == NULL) - return; - /* give a chance to context-based debugging to print current context */ - if (!Ctx_Lock) - ctx_print(); - vfprintf(debug_log, format, args); - if (!Config.onoff.buffered_logs) - fflush(debug_log); -} - -static void -_db_print_stderr(const char *format, va_list args) -{ - if (opt_debug_stderr < _db_level) - return; - if (debug_log == stderr) - return; - vfprintf(stderr, format, args); -} - -#if HAVE_SYSLOG -static void -_db_print_syslog(const char *format, va_list args) -{ - LOCAL_ARRAY(char, tmpbuf, BUFSIZ); - /* level 0,1 go to syslog */ - if (_db_level > 1) - return; - if (0 == opt_syslog_enable) - return; - tmpbuf[0] = '\0'; - vsnprintf(tmpbuf, BUFSIZ, format, args); - tmpbuf[BUFSIZ - 1] = '\0'; - syslog(_db_level == 0 ? LOG_WARNING : LOG_NOTICE, "%s", tmpbuf); -} -#endif /* HAVE_SYSLOG */ - -static void -debugArg(const char *arg) -{ - int s = 0; - int l = 0; - int i; - if (!strncasecmp(arg, "ALL", 3)) { - s = -1; - arg += 4; - } else { - s = atoi(arg); - while (*arg && *arg++ != ','); - } - l = atoi(arg); - assert(s >= -1); - assert(s < MAX_DEBUG_SECTIONS); - if (l < 0) - l = 0; - if (l > 10) - l = 10; - if (s >= 0) { - debugLevels[s] = l; - return; - } - for (i = 0; i < MAX_DEBUG_SECTIONS; i++) - debugLevels[i] = l; -} - -static void -debugOpenLog(const char *logfile) -{ - if (logfile == NULL) { - debug_log = stderr; - return; - } - if (debug_log_file) - xfree(debug_log_file); - debug_log_file = xstrdup(logfile); /* keep a static copy */ - if (debug_log && debug_log != stderr) - fclose(debug_log); - debug_log = fopen(logfile, "a+"); - if (!debug_log) { - fprintf(stderr, "WARNING: Cannot write log file: %s\n", logfile); - perror(logfile); - fprintf(stderr, " messages will be sent to 'stderr'.\n"); - fflush(stderr); - debug_log = stderr; - } -#if defined(_SQUID_CYGWIN_)||defined(_SQUID_MSWIN_) - setmode(fileno(debug_log), O_TEXT); -#endif -} - -void -_db_init(const char *logfile, const char *options) -{ - int i; - char *p = NULL; - char *s = NULL; - - for (i = 0; i < MAX_DEBUG_SECTIONS; i++) - debugLevels[i] = -1; - - if (options) { - p = xstrdup(options); - for (s = strtok(p, w_space); s; s = strtok(NULL, w_space)) - debugArg(s); - xfree(p); - } - debugOpenLog(logfile); - -#if HAVE_SYSLOG && defined(LOG_LOCAL4) - if (opt_syslog_enable) - openlog(appname, LOG_PID | LOG_NDELAY | LOG_CONS, LOG_LOCAL4); -#endif /* HAVE_SYSLOG */ - -} - -void -_db_rotate_log(void) -{ - int i; - LOCAL_ARRAY(char, from, MAXPATHLEN); - LOCAL_ARRAY(char, to, MAXPATHLEN); -#ifdef S_ISREG - struct stat sb; -#endif - - if (debug_log_file == NULL) - return; -#ifdef S_ISREG - if (stat(debug_log_file, &sb) == 0) - if (S_ISREG(sb.st_mode) == 0) - return; -#endif - - /* - * NOTE: we cannot use xrename here without having it in a - * separate file -- tools.c has too many dependencies to be - * used everywhere debug.c is used. - */ - /* Rotate numbers 0 through N up one */ - for (i = Config.Log.rotateNumber; i > 1;) { - i--; - snprintf(from, MAXPATHLEN, "%s.%d", debug_log_file, i - 1); - snprintf(to, MAXPATHLEN, "%s.%d", debug_log_file, i); - rename(from, to); - } -/* - * You can't rename open files on Microsoft "operating systems" - * so we close before renaming. - */ -#ifdef _SQUID_MSWIN_ - if (debug_log != stderr) - fclose(debug_log); -#endif - /* Rotate the current log to .0 */ - if (Config.Log.rotateNumber > 0) { - snprintf(to, MAXPATHLEN, "%s.%d", debug_log_file, 0); - rename(debug_log_file, to); - } - /* Close and reopen the log. It may have been renamed "manually" - * before HUP'ing us. */ - if (debug_log != stderr) - debugOpenLog(Config.Log.log); -} - -static const char * -debugLogTime(time_t t) -{ - struct tm *tm; - static char buf[128]; - static time_t last_t = 0; - if (t != last_t) { - tm = localtime(&t); - strftime(buf, 127, "%Y/%m/%d %H:%M:%S", tm); - last_t = t; - } - return buf; -} - -void -xassert(const char *msg, const char *file, int line) -{ - debug(0, 0) ("assertion failed: %s:%d: \"%s\"\n", file, line, msg); - if (!shutting_down) - abort(); -} - -/* - * Context-based Debugging - * - * Rationale - * --------- - * - * When you have a long nested processing sequence, it is often impossible - * for low level routines to know in what larger context they operate. If a - * routine coredumps, one can restore the context using debugger trace. - * However, in many case you do not want to coredump, but just want to report - * a potential problem. A report maybe useless out of problem context. - * - * To solve this potential problem, use the following approach: - * - * int - * top_level_foo(const char *url) - * { - * // define current context - * // note: we stack but do not dup ctx descriptions! - * Ctx ctx = ctx_enter(url); - * ... - * // go down; middle_level_bar will eventually call bottom_level_boo - * middle_level_bar(method, protocol); - * ... - * // exit, clean after yourself - * ctx_exit(ctx); - * } - * - * void - * bottom_level_boo(int status, void *data) - * { - * // detect exceptional condition, and simply report it, the context - * // information will be available somewhere close in the log file - * if (status == STRANGE_STATUS) - * debug(13, 6) ("DOS attack detected, data: %p\n", data); - * ... - * } - * - * Current implementation is extremely simple but still very handy. It has a - * negligible overhead (descriptions are not duplicated). - * - * When the _first_ debug message for a given context is printed, it is - * prepended with the current context description. Context is printed with - * the same debugging level as the original message. - * - * Note that we do not print context every type you do ctx_enter(). This - * approach would produce too many useless messages. For the same reason, a - * context description is printed at most _once_ even if you have 10 - * debugging messages within one context. - * - * Contexts can be nested, of course. You must use ctx_enter() to enter a - * context (push it onto stack). It is probably safe to exit several nested - * contexts at _once_ by calling ctx_exit() at the top level (this will pop - * all context till current one). However, as in any stack, you cannot start - * in the middle. - * - * Analysis: - * i) locate debugging message, - * ii) locate current context by going _upstream_ in your log file, - * iii) hack away. - * - * - * To-Do: - * ----- - * - * decide if we want to dup() descriptions (adds overhead) but allows to - * add printf()-style interface - * - * implementation: - * --------------- - * - * descriptions for contexts over CTX_MAX_LEVEL limit are ignored, you probably - * have a bug if your nesting goes that deep. - */ - -#define CTX_MAX_LEVEL 255 - -/* - * produce a warning when nesting reaches this level and then double - * the level - */ -static int Ctx_Warn_Level = 32; -/* all descriptions has been printed up to this level */ -static int Ctx_Reported_Level = -1; -/* descriptions are still valid or active up to this level */ -static int Ctx_Valid_Level = -1; -/* current level, the number of nested ctx_enter() calls */ -static int Ctx_Current_Level = -1; -/* saved descriptions (stack) */ -static const char *Ctx_Descrs[CTX_MAX_LEVEL + 1]; -/* "safe" get secription */ -static const char *ctx_get_descr(Ctx ctx); - - -Ctx -ctx_enter(const char *descr) -{ - Ctx_Current_Level++; - - if (Ctx_Current_Level <= CTX_MAX_LEVEL) - Ctx_Descrs[Ctx_Current_Level] = descr; - - if (Ctx_Current_Level == Ctx_Warn_Level) { - debug(0, 0) ("# ctx: suspiciously deep (%d) nesting:\n", Ctx_Warn_Level); - Ctx_Warn_Level *= 2; - } - return Ctx_Current_Level; -} - -void -ctx_exit(Ctx ctx) -{ - assert(ctx >= 0); - Ctx_Current_Level = (ctx >= 0) ? ctx - 1 : -1; - if (Ctx_Valid_Level > Ctx_Current_Level) - Ctx_Valid_Level = Ctx_Current_Level; -} - -/* - * the idea id to print each context description at most once but provide enough - * info for deducing the current execution stack - */ -static void -ctx_print(void) -{ - /* lock so _db_print will not call us recursively */ - Ctx_Lock++; - /* ok, user saw [0,Ctx_Reported_Level] descriptions */ - /* first inform about entries popped since user saw them */ - if (Ctx_Valid_Level < Ctx_Reported_Level) { - if (Ctx_Reported_Level != Ctx_Valid_Level + 1) - _db_print("ctx: exit levels from %2d down to %2d\n", - Ctx_Reported_Level, Ctx_Valid_Level + 1); - else - _db_print("ctx: exit level %2d\n", Ctx_Reported_Level); - Ctx_Reported_Level = Ctx_Valid_Level; - } - /* report new contexts that were pushed since last report */ - while (Ctx_Reported_Level < Ctx_Current_Level) { - Ctx_Reported_Level++; - Ctx_Valid_Level++; - _db_print("ctx: enter level %2d: '%s'\n", Ctx_Reported_Level, - ctx_get_descr(Ctx_Reported_Level)); - } - /* unlock */ - Ctx_Lock--; -} - -/* checks for nulls and overflows */ -static const char * -ctx_get_descr(Ctx ctx) -{ - if (ctx < 0 || ctx > CTX_MAX_LEVEL) - return ""; - return Ctx_Descrs[ctx] ? Ctx_Descrs[ctx] : ""; -} --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/debug.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,436 @@ + +/* + * $Id: debug.cc,v 1.1.2.1 2002/10/11 15:40:48 rbcollins Exp $ + * + * DEBUG: section 0 Debug Routines + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +static char *debug_log_file = NULL; +static int Ctx_Lock = 0; +static const char *debugLogTime(time_t); +static void ctx_print(void); +#if HAVE_SYSLOG +static void _db_print_syslog(const char *format, va_list args); +#endif +static void _db_print_stderr(const char *format, va_list args); +static void _db_print_file(const char *format, va_list args); + +void +#if STDC_HEADERS +_db_print(const char *format,...) +{ +#else +_db_print(va_alist) + va_dcl +{ + const char *format = NULL; +#endif + LOCAL_ARRAY(char, f, BUFSIZ); + va_list args1; +#if STDC_HEADERS + va_list args2; + va_list args3; +#else +#define args2 args1 +#define args3 args1 +#endif + /* give a chance to context-based debugging to print current context */ + if (!Ctx_Lock) + ctx_print(); +#if STDC_HEADERS + va_start(args1, format); + va_start(args2, format); + va_start(args3, format); +#else + format = va_arg(args1, const char *); +#endif + snprintf(f, BUFSIZ, "%s| %s", + debugLogTime(squid_curtime), + format); + _db_print_file(f, args1); + _db_print_stderr(f, args2); +#if HAVE_SYSLOG + _db_print_syslog(format, args3); +#endif + va_end(args1); +#if STDC_HEADERS + va_end(args2); + va_end(args3); +#endif +} + +static void +_db_print_file(const char *format, va_list args) +{ + if (debug_log == NULL) + return; + /* give a chance to context-based debugging to print current context */ + if (!Ctx_Lock) + ctx_print(); + vfprintf(debug_log, format, args); + if (!Config.onoff.buffered_logs) + fflush(debug_log); +} + +static void +_db_print_stderr(const char *format, va_list args) +{ + if (opt_debug_stderr < _db_level) + return; + if (debug_log == stderr) + return; + vfprintf(stderr, format, args); +} + +#if HAVE_SYSLOG +static void +_db_print_syslog(const char *format, va_list args) +{ + LOCAL_ARRAY(char, tmpbuf, BUFSIZ); + /* level 0,1 go to syslog */ + if (_db_level > 1) + return; + if (0 == opt_syslog_enable) + return; + tmpbuf[0] = '\0'; + vsnprintf(tmpbuf, BUFSIZ, format, args); + tmpbuf[BUFSIZ - 1] = '\0'; + syslog(_db_level == 0 ? LOG_WARNING : LOG_NOTICE, "%s", tmpbuf); +} +#endif /* HAVE_SYSLOG */ + +static void +debugArg(const char *arg) +{ + int s = 0; + int l = 0; + int i; + if (!strncasecmp(arg, "ALL", 3)) { + s = -1; + arg += 4; + } else { + s = atoi(arg); + while (*arg && *arg++ != ','); + } + l = atoi(arg); + assert(s >= -1); + assert(s < MAX_DEBUG_SECTIONS); + if (l < 0) + l = 0; + if (l > 10) + l = 10; + if (s >= 0) { + debugLevels[s] = l; + return; + } + for (i = 0; i < MAX_DEBUG_SECTIONS; i++) + debugLevels[i] = l; +} + +static void +debugOpenLog(const char *logfile) +{ + if (logfile == NULL) { + debug_log = stderr; + return; + } + if (debug_log_file) + xfree(debug_log_file); + debug_log_file = xstrdup(logfile); /* keep a static copy */ + if (debug_log && debug_log != stderr) + fclose(debug_log); + debug_log = fopen(logfile, "a+"); + if (!debug_log) { + fprintf(stderr, "WARNING: Cannot write log file: %s\n", logfile); + perror(logfile); + fprintf(stderr, " messages will be sent to 'stderr'.\n"); + fflush(stderr); + debug_log = stderr; + } +#if defined(_SQUID_CYGWIN_)||defined(_SQUID_MSWIN_) + setmode(fileno(debug_log), O_TEXT); +#endif +} + +void +_db_init(const char *logfile, const char *options) +{ + int i; + char *p = NULL; + char *s = NULL; + + for (i = 0; i < MAX_DEBUG_SECTIONS; i++) + debugLevels[i] = -1; + + if (options) { + p = xstrdup(options); + for (s = strtok(p, w_space); s; s = strtok(NULL, w_space)) + debugArg(s); + xfree(p); + } + debugOpenLog(logfile); + +#if HAVE_SYSLOG && defined(LOG_LOCAL4) + if (opt_syslog_enable) + openlog(appname, LOG_PID | LOG_NDELAY | LOG_CONS, LOG_LOCAL4); +#endif /* HAVE_SYSLOG */ + +} + +void +_db_rotate_log(void) +{ + int i; + LOCAL_ARRAY(char, from, MAXPATHLEN); + LOCAL_ARRAY(char, to, MAXPATHLEN); +#ifdef S_ISREG + struct stat sb; +#endif + + if (debug_log_file == NULL) + return; +#ifdef S_ISREG + if (stat(debug_log_file, &sb) == 0) + if (S_ISREG(sb.st_mode) == 0) + return; +#endif + + /* + * NOTE: we cannot use xrename here without having it in a + * separate file -- tools.c has too many dependencies to be + * used everywhere debug.c is used. + */ + /* Rotate numbers 0 through N up one */ + for (i = Config.Log.rotateNumber; i > 1;) { + i--; + snprintf(from, MAXPATHLEN, "%s.%d", debug_log_file, i - 1); + snprintf(to, MAXPATHLEN, "%s.%d", debug_log_file, i); + rename(from, to); + } +/* + * You can't rename open files on Microsoft "operating systems" + * so we close before renaming. + */ +#ifdef _SQUID_MSWIN_ + if (debug_log != stderr) + fclose(debug_log); +#endif + /* Rotate the current log to .0 */ + if (Config.Log.rotateNumber > 0) { + snprintf(to, MAXPATHLEN, "%s.%d", debug_log_file, 0); + rename(debug_log_file, to); + } + /* Close and reopen the log. It may have been renamed "manually" + * before HUP'ing us. */ + if (debug_log != stderr) + debugOpenLog(Config.Log.log); +} + +static const char * +debugLogTime(time_t t) +{ + struct tm *tm; + static char buf[128]; + static time_t last_t = 0; + if (t != last_t) { + tm = localtime(&t); + strftime(buf, 127, "%Y/%m/%d %H:%M:%S", tm); + last_t = t; + } + return buf; +} + +void +xassert(const char *msg, const char *file, int line) +{ + debug(0, 0) ("assertion failed: %s:%d: \"%s\"\n", file, line, msg); + if (!shutting_down) + abort(); +} + +/* + * Context-based Debugging + * + * Rationale + * --------- + * + * When you have a long nested processing sequence, it is often impossible + * for low level routines to know in what larger context they operate. If a + * routine coredumps, one can restore the context using debugger trace. + * However, in many case you do not want to coredump, but just want to report + * a potential problem. A report maybe useless out of problem context. + * + * To solve this potential problem, use the following approach: + * + * int + * top_level_foo(const char *url) + * { + * // define current context + * // note: we stack but do not dup ctx descriptions! + * Ctx ctx = ctx_enter(url); + * ... + * // go down; middle_level_bar will eventually call bottom_level_boo + * middle_level_bar(method, protocol); + * ... + * // exit, clean after yourself + * ctx_exit(ctx); + * } + * + * void + * bottom_level_boo(int status, void *data) + * { + * // detect exceptional condition, and simply report it, the context + * // information will be available somewhere close in the log file + * if (status == STRANGE_STATUS) + * debug(13, 6) ("DOS attack detected, data: %p\n", data); + * ... + * } + * + * Current implementation is extremely simple but still very handy. It has a + * negligible overhead (descriptions are not duplicated). + * + * When the _first_ debug message for a given context is printed, it is + * prepended with the current context description. Context is printed with + * the same debugging level as the original message. + * + * Note that we do not print context every type you do ctx_enter(). This + * approach would produce too many useless messages. For the same reason, a + * context description is printed at most _once_ even if you have 10 + * debugging messages within one context. + * + * Contexts can be nested, of course. You must use ctx_enter() to enter a + * context (push it onto stack). It is probably safe to exit several nested + * contexts at _once_ by calling ctx_exit() at the top level (this will pop + * all context till current one). However, as in any stack, you cannot start + * in the middle. + * + * Analysis: + * i) locate debugging message, + * ii) locate current context by going _upstream_ in your log file, + * iii) hack away. + * + * + * To-Do: + * ----- + * + * decide if we want to dup() descriptions (adds overhead) but allows to + * add printf()-style interface + * + * implementation: + * --------------- + * + * descriptions for contexts over CTX_MAX_LEVEL limit are ignored, you probably + * have a bug if your nesting goes that deep. + */ + +#define CTX_MAX_LEVEL 255 + +/* + * produce a warning when nesting reaches this level and then double + * the level + */ +static int Ctx_Warn_Level = 32; +/* all descriptions has been printed up to this level */ +static int Ctx_Reported_Level = -1; +/* descriptions are still valid or active up to this level */ +static int Ctx_Valid_Level = -1; +/* current level, the number of nested ctx_enter() calls */ +static int Ctx_Current_Level = -1; +/* saved descriptions (stack) */ +static const char *Ctx_Descrs[CTX_MAX_LEVEL + 1]; +/* "safe" get secription */ +static const char *ctx_get_descr(Ctx ctx); + + +Ctx +ctx_enter(const char *descr) +{ + Ctx_Current_Level++; + + if (Ctx_Current_Level <= CTX_MAX_LEVEL) + Ctx_Descrs[Ctx_Current_Level] = descr; + + if (Ctx_Current_Level == Ctx_Warn_Level) { + debug(0, 0) ("# ctx: suspiciously deep (%d) nesting:\n", Ctx_Warn_Level); + Ctx_Warn_Level *= 2; + } + return Ctx_Current_Level; +} + +void +ctx_exit(Ctx ctx) +{ + assert(ctx >= 0); + Ctx_Current_Level = (ctx >= 0) ? ctx - 1 : -1; + if (Ctx_Valid_Level > Ctx_Current_Level) + Ctx_Valid_Level = Ctx_Current_Level; +} + +/* + * the idea id to print each context description at most once but provide enough + * info for deducing the current execution stack + */ +static void +ctx_print(void) +{ + /* lock so _db_print will not call us recursively */ + Ctx_Lock++; + /* ok, user saw [0,Ctx_Reported_Level] descriptions */ + /* first inform about entries popped since user saw them */ + if (Ctx_Valid_Level < Ctx_Reported_Level) { + if (Ctx_Reported_Level != Ctx_Valid_Level + 1) + _db_print("ctx: exit levels from %2d down to %2d\n", + Ctx_Reported_Level, Ctx_Valid_Level + 1); + else + _db_print("ctx: exit level %2d\n", Ctx_Reported_Level); + Ctx_Reported_Level = Ctx_Valid_Level; + } + /* report new contexts that were pushed since last report */ + while (Ctx_Reported_Level < Ctx_Current_Level) { + Ctx_Reported_Level++; + Ctx_Valid_Level++; + _db_print("ctx: enter level %2d: '%s'\n", Ctx_Reported_Level, + ctx_get_descr(Ctx_Reported_Level)); + } + /* unlock */ + Ctx_Lock--; +} + +/* checks for nulls and overflows */ +static const char * +ctx_get_descr(Ctx ctx) +{ + if (ctx < 0 || ctx > CTX_MAX_LEVEL) + return ""; + return Ctx_Descrs[ctx] ? Ctx_Descrs[ctx] : ""; +} --- squid/src/disk.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,418 +0,0 @@ - -/* - * $Id: disk.c,v 1.12 2002/10/02 11:10:47 squidadm Exp $ - * - * DEBUG: section 6 Disk I/O Routines - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -static PF diskHandleRead; -static PF diskHandleWrite; - -#if defined(_SQUID_MSWIN_) || defined(_SQUID_OS2_) || defined(_SQUID_CYGWIN_) -static int -diskWriteIsComplete(int fd) -{ - return fd_table[fd].disk.write_q ? 0 : 1; -} -#endif - -void -disk_init(void) -{ - (void) 0; -} - -/* - * opens a disk file specified by 'path'. This function always - * blocks! There is no callback. - */ -int -file_open(const char *path, int mode) -{ - int fd; - PROF_start(file_open); - if (FILE_MODE(mode) == O_WRONLY) - mode |= O_APPEND; - mode |= SQUID_NONBLOCK; - errno = 0; - fd = open(path, mode, 0644); - statCounter.syscalls.disk.opens++; - if (fd < 0) { - debug(50, 3) ("file_open: error opening file %s: %s\n", path, - xstrerror()); - fd = DISK_ERROR; - } else { - debug(6, 5) ("file_open: FD %d\n", fd); - commSetCloseOnExec(fd); - fd_open(fd, FD_FILE, path); - } - PROF_stop(file_open); - return fd; -} - - -/* close a disk file. */ -void -file_close(int fd) -{ - fde *F = &fd_table[fd]; - PF *read_callback; - PROF_start(file_close); - assert(fd >= 0); - assert(F->flags.open); - if ((read_callback = F->read_handler)) { - F->read_handler = NULL; - read_callback(-1, F->read_data); - } - if (F->flags.write_daemon) { -#if defined(_SQUID_MSWIN_) || defined(_SQUID_OS2_) || defined(_SQUID_CYGWIN_) - /* - * on some operating systems, you can not delete or rename - * open files, so we won't allow delayed close. - */ - while (!diskWriteIsComplete(fd)) - diskHandleWrite(fd, NULL); -#else - F->flags.close_request = 1; - debug(6, 2) ("file_close: FD %d, delaying close\n", fd); - PROF_stop(file_close); - return; -#endif - } - /* - * Assert there is no write callback. Otherwise we might be - * leaking write state data by closing the descriptor - */ - assert(F->write_handler == NULL); - F->flags.closing = 1; -#if CALL_FSYNC_BEFORE_CLOSE - fsync(fd); -#endif - close(fd); - debug(6, F->flags.close_request ? 2 : 5) - ("file_close: FD %d, really closing\n", fd); - fd_close(fd); - statCounter.syscalls.disk.closes++; - PROF_stop(file_close); -} - -/* - * This function has the purpose of combining multiple writes. This is - * to facilitate the ASYNC_IO option since it can only guarantee 1 - * write to a file per trip around the comm.c select() loop. That's bad - * because more than 1 write can be made to the access.log file per - * trip, and so this code is purely designed to help batch multiple - * sequential writes to the access.log file. Squid will never issue - * multiple writes for any other file type during 1 trip around the - * select() loop. --SLF - */ -static void -diskCombineWrites(struct _fde_disk *fdd) -{ - int len = 0; - dwrite_q *q = NULL; - dwrite_q *wq = NULL; - /* - * We need to combine multiple write requests on an FD's write - * queue But only if we don't need to seek() in between them, ugh! - * XXX This currently ignores any seeks (file_offset) - */ - if (fdd->write_q != NULL && fdd->write_q->next != NULL) { - len = 0; - for (q = fdd->write_q; q != NULL; q = q->next) - len += q->len - q->buf_offset; - wq = memAllocate(MEM_DWRITE_Q); - wq->buf = xmalloc(len); - wq->len = 0; - wq->buf_offset = 0; - wq->next = NULL; - wq->free_func = xfree; - do { - q = fdd->write_q; - len = q->len - q->buf_offset; - xmemcpy(wq->buf + wq->len, q->buf + q->buf_offset, len); - wq->len += len; - fdd->write_q = q->next; - if (q->free_func) - (q->free_func) (q->buf); - if (q) { - memFree(q, MEM_DWRITE_Q); - q = NULL; - } - } while (fdd->write_q != NULL); - fdd->write_q_tail = wq; - fdd->write_q = wq; - } -} - -/* write handler */ -static void -diskHandleWrite(int fd, void *notused) -{ - int len = 0; - fde *F = &fd_table[fd]; - struct _fde_disk *fdd = &F->disk; - dwrite_q *q = fdd->write_q; - int status = DISK_OK; - int do_close; - if (NULL == q) - return; - PROF_start(diskHandleWrite); - debug(6, 3) ("diskHandleWrite: FD %d\n", fd); - F->flags.write_daemon = 0; - assert(fdd->write_q != NULL); - assert(fdd->write_q->len > fdd->write_q->buf_offset); - debug(6, 3) ("diskHandleWrite: FD %d writing %d bytes\n", - fd, (int) (fdd->write_q->len - fdd->write_q->buf_offset)); - errno = 0; - if (fdd->write_q->file_offset != -1) - lseek(fd, fdd->write_q->file_offset, SEEK_SET); - len = FD_WRITE_METHOD(fd, - fdd->write_q->buf + fdd->write_q->buf_offset, - fdd->write_q->len - fdd->write_q->buf_offset); - debug(6, 3) ("diskHandleWrite: FD %d len = %d\n", fd, len); - statCounter.syscalls.disk.writes++; - fd_bytes(fd, len, FD_WRITE); - if (len < 0) { - if (!ignoreErrno(errno)) { - status = errno == ENOSPC ? DISK_NO_SPACE_LEFT : DISK_ERROR; - debug(50, 1) ("diskHandleWrite: FD %d: disk write error: %s\n", - fd, xstrerror()); - /* - * If there is no write callback, then this file is - * most likely something important like a log file, or - * an interprocess pipe. Its not a swapfile. We feel - * that a write failure on a log file is rather important, - * and Squid doesn't otherwise deal with this condition. - * So to get the administrators attention, we exit with - * a fatal message. - */ - if (fdd->wrt_handle == NULL) - fatal("Write failure -- check your disk space and cache.log"); - /* - * If there is a write failure, then we notify the - * upper layer via the callback, at the end of this - * function. Meanwhile, flush all pending buffers - * here. Let the upper layer decide how to handle the - * failure. This will prevent experiencing multiple, - * repeated write failures for the same FD because of - * the queued data. - */ - do { - fdd->write_q = q->next; - if (q->free_func) - (q->free_func) (q->buf); - if (q) { - memFree(q, MEM_DWRITE_Q); - q = NULL; - } - } while ((q = fdd->write_q)); - } - len = 0; - } - if (q != NULL) { - /* q might become NULL from write failure above */ - q->buf_offset += len; - if (q->buf_offset > q->len) - debug(50, 1) ("diskHandleWriteComplete: q->buf_offset > q->len (%p,%d, %d, %d FD %d)\n", - q, (int) q->buf_offset, q->len, len, fd); - assert(q->buf_offset <= q->len); - if (q->buf_offset == q->len) { - /* complete write */ - fdd->write_q = q->next; - if (q->free_func) - (q->free_func) (q->buf); - if (q) { - memFree(q, MEM_DWRITE_Q); - q = NULL; - } - } - } - if (fdd->write_q == NULL) { - /* no more data */ - fdd->write_q_tail = NULL; - } else { - /* another block is queued */ - diskCombineWrites(fdd); - commSetSelect(fd, COMM_SELECT_WRITE, diskHandleWrite, NULL, 0); - F->flags.write_daemon = 1; - } - do_close = F->flags.close_request; - if (fdd->wrt_handle) { - DWCB *callback = fdd->wrt_handle; - void *cbdata; - fdd->wrt_handle = NULL; - if (cbdataReferenceValidDone(fdd->wrt_handle_data, &cbdata)) { - callback(fd, status, len, cbdata); - /* - * NOTE, this callback can close the FD, so we must - * not touch 'F', 'fdd', etc. after this. - */ - PROF_stop(diskHandleWrite); - return; - /* XXX But what about close_request??? */ - } - } - if (do_close) - file_close(fd); - PROF_stop(diskHandleWrite); -} - - -/* write block to a file */ -/* write back queue. Only one writer at a time. */ -/* call a handle when writing is complete. */ -void -file_write(int fd, - off_t file_offset, - void *ptr_to_buf, - int len, - DWCB * handle, - void *handle_data, - FREE * free_func) -{ - dwrite_q *wq = NULL; - fde *F = &fd_table[fd]; - PROF_start(file_write); - assert(fd >= 0); - assert(F->flags.open); - /* if we got here. Caller is eligible to write. */ - wq = memAllocate(MEM_DWRITE_Q); - wq->file_offset = file_offset; - wq->buf = ptr_to_buf; - wq->len = len; - wq->buf_offset = 0; - wq->next = NULL; - wq->free_func = free_func; - if (!F->disk.wrt_handle_data) { - F->disk.wrt_handle = handle; - F->disk.wrt_handle_data = cbdataReference(handle_data); - } else { - /* Detect if there is multiple concurrent users of this fd.. we only support one callback */ - assert(F->disk.wrt_handle_data == handle_data && F->disk.wrt_handle == handle); - } - /* add to queue */ - if (F->disk.write_q == NULL) { - /* empty queue */ - F->disk.write_q = F->disk.write_q_tail = wq; - } else { - F->disk.write_q_tail->next = wq; - F->disk.write_q_tail = wq; - } - if (!F->flags.write_daemon) { - diskHandleWrite(fd, NULL); - } - PROF_stop(file_write); -} - -/* - * a wrapper around file_write to allow for MemBuf to be file_written - * in a snap - */ -void -file_write_mbuf(int fd, off_t off, MemBuf mb, DWCB * handler, void *handler_data) -{ - file_write(fd, off, mb.buf, mb.size, handler, handler_data, memBufFreeFunc(&mb)); -} - -/* Read from FD */ -static void -diskHandleRead(int fd, void *data) -{ - dread_ctrl *ctrl_dat = data; - fde *F = &fd_table[fd]; - int len; - int rc = DISK_OK; - /* - * FD < 0 indicates premature close; we just have to free - * the state data. - */ - if (fd < 0) { - memFree(ctrl_dat, MEM_DREAD_CTRL); - return; - } - PROF_start(diskHandleRead); - if (F->disk.offset != ctrl_dat->offset) { - debug(6, 3) ("diskHandleRead: FD %d seeking to offset %d\n", - fd, (int) ctrl_dat->offset); - lseek(fd, ctrl_dat->offset, SEEK_SET); /* XXX ignore return? */ - statCounter.syscalls.disk.seeks++; - F->disk.offset = ctrl_dat->offset; - } - errno = 0; - len = FD_READ_METHOD(fd, ctrl_dat->buf, ctrl_dat->req_len); - if (len > 0) - F->disk.offset += len; - statCounter.syscalls.disk.reads++; - fd_bytes(fd, len, FD_READ); - if (len < 0) { - if (ignoreErrno(errno)) { - commSetSelect(fd, COMM_SELECT_READ, diskHandleRead, ctrl_dat, 0); - PROF_stop(diskHandleRead); - return; - } - debug(50, 1) ("diskHandleRead: FD %d: %s\n", fd, xstrerror()); - len = 0; - rc = DISK_ERROR; - } else if (len == 0) { - rc = DISK_EOF; - } - if (cbdataReferenceValid(ctrl_dat->client_data)) - ctrl_dat->handler(fd, ctrl_dat->buf, len, rc, ctrl_dat->client_data); - cbdataReferenceDone(ctrl_dat->client_data); - memFree(ctrl_dat, MEM_DREAD_CTRL); - PROF_stop(diskHandleRead); -} - - -/* start read operation */ -/* buffer must be allocated from the caller. - * It must have at least req_len space in there. - * call handler when a reading is complete. */ -void -file_read(int fd, char *buf, int req_len, off_t offset, DRCB * handler, void *client_data) -{ - dread_ctrl *ctrl_dat; - PROF_start(file_read); - assert(fd >= 0); - ctrl_dat = memAllocate(MEM_DREAD_CTRL); - ctrl_dat->fd = fd; - ctrl_dat->offset = offset; - ctrl_dat->req_len = req_len; - ctrl_dat->buf = buf; - ctrl_dat->end_of_file = 0; - ctrl_dat->handler = handler; - ctrl_dat->client_data = cbdataReference(client_data); - diskHandleRead(fd, ctrl_dat); - PROF_stop(file_read); -} --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/disk.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,418 @@ + +/* + * $Id: disk.cc,v 1.1.2.1 2002/10/11 15:40:48 rbcollins Exp $ + * + * DEBUG: section 6 Disk I/O Routines + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +static PF diskHandleRead; +static PF diskHandleWrite; + +#if defined(_SQUID_MSWIN_) || defined(_SQUID_OS2_) || defined(_SQUID_CYGWIN_) +static int +diskWriteIsComplete(int fd) +{ + return fd_table[fd].disk.write_q ? 0 : 1; +} +#endif + +void +disk_init(void) +{ + (void) 0; +} + +/* + * opens a disk file specified by 'path'. This function always + * blocks! There is no callback. + */ +int +file_open(const char *path, int mode) +{ + int fd; + PROF_start(file_open); + if (FILE_MODE(mode) == O_WRONLY) + mode |= O_APPEND; + mode |= SQUID_NONBLOCK; + errno = 0; + fd = open(path, mode, 0644); + statCounter.syscalls.disk.opens++; + if (fd < 0) { + debug(50, 3) ("file_open: error opening file %s: %s\n", path, + xstrerror()); + fd = DISK_ERROR; + } else { + debug(6, 5) ("file_open: FD %d\n", fd); + commSetCloseOnExec(fd); + fd_open(fd, FD_FILE, path); + } + PROF_stop(file_open); + return fd; +} + + +/* close a disk file. */ +void +file_close(int fd) +{ + fde *F = &fd_table[fd]; + PF *read_callback; + PROF_start(file_close); + assert(fd >= 0); + assert(F->flags.open); + if ((read_callback = F->read_handler)) { + F->read_handler = NULL; + read_callback(-1, F->read_data); + } + if (F->flags.write_daemon) { +#if defined(_SQUID_MSWIN_) || defined(_SQUID_OS2_) || defined(_SQUID_CYGWIN_) + /* + * on some operating systems, you can not delete or rename + * open files, so we won't allow delayed close. + */ + while (!diskWriteIsComplete(fd)) + diskHandleWrite(fd, NULL); +#else + F->flags.close_request = 1; + debug(6, 2) ("file_close: FD %d, delaying close\n", fd); + PROF_stop(file_close); + return; +#endif + } + /* + * Assert there is no write callback. Otherwise we might be + * leaking write state data by closing the descriptor + */ + assert(F->write_handler == NULL); + F->flags.closing = 1; +#if CALL_FSYNC_BEFORE_CLOSE + fsync(fd); +#endif + close(fd); + debug(6, F->flags.close_request ? 2 : 5) + ("file_close: FD %d, really closing\n", fd); + fd_close(fd); + statCounter.syscalls.disk.closes++; + PROF_stop(file_close); +} + +/* + * This function has the purpose of combining multiple writes. This is + * to facilitate the ASYNC_IO option since it can only guarantee 1 + * write to a file per trip around the comm.c select() loop. That's bad + * because more than 1 write can be made to the access.log file per + * trip, and so this code is purely designed to help batch multiple + * sequential writes to the access.log file. Squid will never issue + * multiple writes for any other file type during 1 trip around the + * select() loop. --SLF + */ +static void +diskCombineWrites(struct _fde_disk *fdd) +{ + int len = 0; + dwrite_q *q = NULL; + dwrite_q *wq = NULL; + /* + * We need to combine multiple write requests on an FD's write + * queue But only if we don't need to seek() in between them, ugh! + * XXX This currently ignores any seeks (file_offset) + */ + if (fdd->write_q != NULL && fdd->write_q->next != NULL) { + len = 0; + for (q = fdd->write_q; q != NULL; q = q->next) + len += q->len - q->buf_offset; + wq = (dwrite_q *)memAllocate(MEM_DWRITE_Q); + wq->buf = (char *)xmalloc(len); + wq->len = 0; + wq->buf_offset = 0; + wq->next = NULL; + wq->free_func = xfree; + do { + q = fdd->write_q; + len = q->len - q->buf_offset; + xmemcpy(wq->buf + wq->len, q->buf + q->buf_offset, len); + wq->len += len; + fdd->write_q = q->next; + if (q->free_func) + (q->free_func) (q->buf); + if (q) { + memFree(q, MEM_DWRITE_Q); + q = NULL; + } + } while (fdd->write_q != NULL); + fdd->write_q_tail = wq; + fdd->write_q = wq; + } +} + +/* write handler */ +static void +diskHandleWrite(int fd, void *notused) +{ + int len = 0; + fde *F = &fd_table[fd]; + struct _fde_disk *fdd = &F->disk; + dwrite_q *q = fdd->write_q; + int status = DISK_OK; + int do_close; + if (NULL == q) + return; + PROF_start(diskHandleWrite); + debug(6, 3) ("diskHandleWrite: FD %d\n", fd); + F->flags.write_daemon = 0; + assert(fdd->write_q != NULL); + assert(fdd->write_q->len > fdd->write_q->buf_offset); + debug(6, 3) ("diskHandleWrite: FD %d writing %d bytes\n", + fd, (int) (fdd->write_q->len - fdd->write_q->buf_offset)); + errno = 0; + if (fdd->write_q->file_offset != -1) + lseek(fd, fdd->write_q->file_offset, SEEK_SET); + len = FD_WRITE_METHOD(fd, + fdd->write_q->buf + fdd->write_q->buf_offset, + fdd->write_q->len - fdd->write_q->buf_offset); + debug(6, 3) ("diskHandleWrite: FD %d len = %d\n", fd, len); + statCounter.syscalls.disk.writes++; + fd_bytes(fd, len, FD_WRITE); + if (len < 0) { + if (!ignoreErrno(errno)) { + status = errno == ENOSPC ? DISK_NO_SPACE_LEFT : DISK_ERROR; + debug(50, 1) ("diskHandleWrite: FD %d: disk write error: %s\n", + fd, xstrerror()); + /* + * If there is no write callback, then this file is + * most likely something important like a log file, or + * an interprocess pipe. Its not a swapfile. We feel + * that a write failure on a log file is rather important, + * and Squid doesn't otherwise deal with this condition. + * So to get the administrators attention, we exit with + * a fatal message. + */ + if (fdd->wrt_handle == NULL) + fatal("Write failure -- check your disk space and cache.log"); + /* + * If there is a write failure, then we notify the + * upper layer via the callback, at the end of this + * function. Meanwhile, flush all pending buffers + * here. Let the upper layer decide how to handle the + * failure. This will prevent experiencing multiple, + * repeated write failures for the same FD because of + * the queued data. + */ + do { + fdd->write_q = q->next; + if (q->free_func) + (q->free_func) (q->buf); + if (q) { + memFree(q, MEM_DWRITE_Q); + q = NULL; + } + } while ((q = fdd->write_q)); + } + len = 0; + } + if (q != NULL) { + /* q might become NULL from write failure above */ + q->buf_offset += len; + if (q->buf_offset > q->len) + debug(50, 1) ("diskHandleWriteComplete: q->buf_offset > q->len (%p,%d, %d, %d FD %d)\n", + q, (int) q->buf_offset, q->len, len, fd); + assert(q->buf_offset <= q->len); + if (q->buf_offset == q->len) { + /* complete write */ + fdd->write_q = q->next; + if (q->free_func) + (q->free_func) (q->buf); + if (q) { + memFree(q, MEM_DWRITE_Q); + q = NULL; + } + } + } + if (fdd->write_q == NULL) { + /* no more data */ + fdd->write_q_tail = NULL; + } else { + /* another block is queued */ + diskCombineWrites(fdd); + commSetSelect(fd, COMM_SELECT_WRITE, diskHandleWrite, NULL, 0); + F->flags.write_daemon = 1; + } + do_close = F->flags.close_request; + if (fdd->wrt_handle) { + DWCB *callback = fdd->wrt_handle; + void *cbdata; + fdd->wrt_handle = NULL; + if (cbdataReferenceValidDone(fdd->wrt_handle_data, &cbdata)) { + callback(fd, status, len, cbdata); + /* + * NOTE, this callback can close the FD, so we must + * not touch 'F', 'fdd', etc. after this. + */ + PROF_stop(diskHandleWrite); + return; + /* XXX But what about close_request??? */ + } + } + if (do_close) + file_close(fd); + PROF_stop(diskHandleWrite); +} + + +/* write block to a file */ +/* write back queue. Only one writer at a time. */ +/* call a handle when writing is complete. */ +void +file_write(int fd, + off_t file_offset, + void *ptr_to_buf, + int len, + DWCB * handle, + void *handle_data, + FREE * free_func) +{ + dwrite_q *wq = NULL; + fde *F = &fd_table[fd]; + PROF_start(file_write); + assert(fd >= 0); + assert(F->flags.open); + /* if we got here. Caller is eligible to write. */ + wq = (dwrite_q *)memAllocate(MEM_DWRITE_Q); + wq->file_offset = file_offset; + wq->buf = (char *)ptr_to_buf; + wq->len = len; + wq->buf_offset = 0; + wq->next = NULL; + wq->free_func = free_func; + if (!F->disk.wrt_handle_data) { + F->disk.wrt_handle = handle; + F->disk.wrt_handle_data = cbdataReference(handle_data); + } else { + /* Detect if there is multiple concurrent users of this fd.. we only support one callback */ + assert(F->disk.wrt_handle_data == handle_data && F->disk.wrt_handle == handle); + } + /* add to queue */ + if (F->disk.write_q == NULL) { + /* empty queue */ + F->disk.write_q = F->disk.write_q_tail = wq; + } else { + F->disk.write_q_tail->next = wq; + F->disk.write_q_tail = wq; + } + if (!F->flags.write_daemon) { + diskHandleWrite(fd, NULL); + } + PROF_stop(file_write); +} + +/* + * a wrapper around file_write to allow for MemBuf to be file_written + * in a snap + */ +void +file_write_mbuf(int fd, off_t off, MemBuf mb, DWCB * handler, void *handler_data) +{ + file_write(fd, off, mb.buf, mb.size, handler, handler_data, memBufFreeFunc(&mb)); +} + +/* Read from FD */ +static void +diskHandleRead(int fd, void *data) +{ + dread_ctrl *ctrl_dat = (dread_ctrl *)data; + fde *F = &fd_table[fd]; + int len; + int rc = DISK_OK; + /* + * FD < 0 indicates premature close; we just have to free + * the state data. + */ + if (fd < 0) { + memFree(ctrl_dat, MEM_DREAD_CTRL); + return; + } + PROF_start(diskHandleRead); + if (F->disk.offset != ctrl_dat->offset) { + debug(6, 3) ("diskHandleRead: FD %d seeking to offset %d\n", + fd, (int) ctrl_dat->offset); + lseek(fd, ctrl_dat->offset, SEEK_SET); /* XXX ignore return? */ + statCounter.syscalls.disk.seeks++; + F->disk.offset = ctrl_dat->offset; + } + errno = 0; + len = FD_READ_METHOD(fd, ctrl_dat->buf, ctrl_dat->req_len); + if (len > 0) + F->disk.offset += len; + statCounter.syscalls.disk.reads++; + fd_bytes(fd, len, FD_READ); + if (len < 0) { + if (ignoreErrno(errno)) { + commSetSelect(fd, COMM_SELECT_READ, diskHandleRead, ctrl_dat, 0); + PROF_stop(diskHandleRead); + return; + } + debug(50, 1) ("diskHandleRead: FD %d: %s\n", fd, xstrerror()); + len = 0; + rc = DISK_ERROR; + } else if (len == 0) { + rc = DISK_EOF; + } + if (cbdataReferenceValid(ctrl_dat->client_data)) + ctrl_dat->handler(fd, ctrl_dat->buf, len, rc, ctrl_dat->client_data); + cbdataReferenceDone(ctrl_dat->client_data); + memFree(ctrl_dat, MEM_DREAD_CTRL); + PROF_stop(diskHandleRead); +} + + +/* start read operation */ +/* buffer must be allocated from the caller. + * It must have at least req_len space in there. + * call handler when a reading is complete. */ +void +file_read(int fd, char *buf, int req_len, off_t offset, DRCB * handler, void *client_data) +{ + dread_ctrl *ctrl_dat; + PROF_start(file_read); + assert(fd >= 0); + ctrl_dat = (dread_ctrl *)memAllocate(MEM_DREAD_CTRL); + ctrl_dat->fd = fd; + ctrl_dat->offset = offset; + ctrl_dat->req_len = req_len; + ctrl_dat->buf = buf; + ctrl_dat->end_of_file = 0; + ctrl_dat->handler = handler; + ctrl_dat->client_data = cbdataReference(client_data); + diskHandleRead(fd, ctrl_dat); + PROF_stop(file_read); +} --- squid/src/fqdncache.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,650 +0,0 @@ - -/* - * $Id: fqdncache.c,v 1.17.6.1 2002/10/04 07:07:25 rbcollins Exp $ - * - * DEBUG: section 35 FQDN Cache - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "Store.h" - -#define FQDN_LOW_WATER 90 -#define FQDN_HIGH_WATER 95 - -typedef struct _fqdncache_entry fqdncache_entry; - -struct _fqdncache_entry { - hash_link hash; /* must be first */ - time_t lastref; - time_t expires; - unsigned char name_count; - char *names[FQDN_MAX_NAMES + 1]; - FQDNH *handler; - void *handlerData; - char *error_message; - struct timeval request_time; - dlink_node lru; - unsigned short locks; - struct { - unsigned int negcached:1; - unsigned int fromhosts:1; - } flags; -}; - -static struct { - int requests; - int replies; - int hits; - int misses; - int negative_hits; - int errors; - int ghba_calls; /* # calls to blocking gethostbyaddr() */ -} FqdncacheStats; - -static dlink_list lru_list; - -#if USE_DNSSERVERS -static HLPCB fqdncacheHandleReply; -static fqdncache_entry *fqdncacheParse(const char *buf); -#else -static IDNSCB fqdncacheHandleReply; -static fqdncache_entry *fqdncacheParse(rfc1035_rr *, int); -#endif -static void fqdncacheRelease(fqdncache_entry *); -static fqdncache_entry *fqdncacheCreateEntry(const char *name); -static void fqdncacheCallback(fqdncache_entry *); -static fqdncache_entry *fqdncache_get(const char *); -static FQDNH dummy_handler; -static int fqdncacheExpiredEntry(const fqdncache_entry *); -static void fqdncacheLockEntry(fqdncache_entry * f); -static void fqdncacheUnlockEntry(fqdncache_entry * f); -static FREE fqdncacheFreeEntry; -static void fqdncacheAddEntry(fqdncache_entry * f); - -static hash_table *fqdn_table = NULL; - -static long fqdncache_low = 180; -static long fqdncache_high = 200; - -/* removes the given fqdncache entry */ -static void -fqdncacheRelease(fqdncache_entry * f) -{ - int k; - hash_remove_link(fqdn_table, (hash_link *) f); - for (k = 0; k < (int) f->name_count; k++) - safe_free(f->names[k]); - debug(35, 5) ("fqdncacheRelease: Released FQDN record for '%s'.\n", - hashKeyStr(&f->hash)); - dlinkDelete(&f->lru, &lru_list); - safe_free(f->hash.key); - safe_free(f->error_message); - memFree(f, MEM_FQDNCACHE_ENTRY); -} - -/* return match for given name */ -static fqdncache_entry * -fqdncache_get(const char *name) -{ - hash_link *e; - static fqdncache_entry *f; - f = NULL; - if (fqdn_table) { - if ((e = hash_lookup(fqdn_table, name)) != NULL) - f = (fqdncache_entry *) e; - } - return f; -} - -static int -fqdncacheExpiredEntry(const fqdncache_entry * f) -{ - /* all static entries are locked, so this takes care of them too */ - if (f->locks != 0) - return 0; - if (f->expires > squid_curtime) - return 0; - return 1; -} - -void -fqdncache_purgelru(void *notused) -{ - dlink_node *m; - dlink_node *prev = NULL; - fqdncache_entry *f; - int removed = 0; - eventAdd("fqdncache_purgelru", fqdncache_purgelru, NULL, 10.0, 1); - for (m = lru_list.tail; m; m = prev) { - if (memInUse(MEM_FQDNCACHE_ENTRY) < fqdncache_low) - break; - prev = m->prev; - f = m->data; - if (f->locks != 0) - continue; - fqdncacheRelease(f); - removed++; - } - debug(35, 9) ("fqdncache_purgelru: removed %d entries\n", removed); -} - -static void -purge_entries_fromhosts(void) -{ - dlink_node *m = lru_list.head; - fqdncache_entry *i = NULL; - fqdncache_entry *t; - while (m) { - if (i != NULL) { /* need to delay deletion */ - fqdncacheRelease(i); /* we just override locks */ - i = NULL; - } - t = m->data; - if (t->flags.fromhosts) - i = t; - m = m->next; - } - if (i != NULL) - fqdncacheRelease(i); -} - -/* create blank fqdncache_entry */ -static fqdncache_entry * -fqdncacheCreateEntry(const char *name) -{ - static fqdncache_entry *f; - f = memAllocate(MEM_FQDNCACHE_ENTRY); - f->hash.key = xstrdup(name); - f->expires = squid_curtime + Config.negativeDnsTtl; - return f; -} - -static void -fqdncacheAddEntry(fqdncache_entry * f) -{ - hash_link *e = hash_lookup(fqdn_table, f->hash.key); - if (NULL != e) { - /* avoid colission */ - fqdncache_entry *q = (fqdncache_entry *) e; - fqdncacheRelease(q); - } - hash_join(fqdn_table, &f->hash); - dlinkAdd(f, &f->lru, &lru_list); - f->lastref = squid_curtime; -} - -/* walks down the pending list, calling handlers */ -static void -fqdncacheCallback(fqdncache_entry * f) -{ - FQDNH *callback; - void *cbdata; - f->lastref = squid_curtime; - if (!f->handler) - return; - fqdncacheLockEntry(f); - callback = f->handler; - f->handler = NULL; - if (cbdataReferenceValidDone(f->handlerData, &cbdata)) { - dns_error_message = f->error_message; - callback(f->flags.negcached ? NULL : f->names[0], cbdata); - } - fqdncacheUnlockEntry(f); -} - -static fqdncache_entry * -#if USE_DNSSERVERS -fqdncacheParse(const char *inbuf) -{ - LOCAL_ARRAY(char, buf, DNS_INBUF_SZ); - char *token; - static fqdncache_entry f; - int ttl; - memset(&f, '\0', sizeof(f)); - f.expires = squid_curtime; - f.flags.negcached = 1; - if (inbuf == NULL) { - debug(35, 1) ("fqdncacheParse: Got reply\n"); - return &f; - } - xstrncpy(buf, inbuf, DNS_INBUF_SZ); - debug(35, 5) ("fqdncacheParse: parsing: {%s}\n", buf); - token = strtok(buf, w_space); - if (NULL == token) { - debug(35, 1) ("fqdncacheParse: Got , expecting '$name'\n"); - return &f; - } - if (0 == strcmp(token, "$fail")) { - f.expires = squid_curtime + Config.negativeDnsTtl; - token = strtok(NULL, "\n"); - assert(NULL != token); - f.error_message = xstrdup(token); - return &f; - } - if (0 != strcmp(token, "$name")) { - debug(35, 1) ("fqdncacheParse: Got '%s', expecting '$name'\n", token); - return &f; - } - token = strtok(NULL, w_space); - if (NULL == token) { - debug(35, 1) ("fqdncacheParse: Got , expecting TTL\n"); - return &f; - } - f.flags.negcached = 0; - ttl = atoi(token); - if (ttl > 0) - f.expires = squid_curtime + ttl; - else - f.expires = squid_curtime + Config.positiveDnsTtl; - token = strtok(NULL, w_space); - if (NULL != token) { - f.names[0] = xstrdup(token); - f.name_count = 1; - } - return &f; -} -#else -fqdncacheParse(rfc1035_rr * answers, int nr) -{ - static fqdncache_entry f; - int k; - int na = 0; - memset(&f, '\0', sizeof(f)); - f.expires = squid_curtime; - f.flags.negcached = 1; - if (nr < 0) { - debug(35, 3) ("fqdncacheParse: Lookup failed (error %d)\n", - rfc1035_errno); - assert(rfc1035_error_message); - f.error_message = xstrdup(rfc1035_error_message); - return &f; - } - if (nr == 0) { - debug(35, 3) ("fqdncacheParse: No DNS records\n"); - f.error_message = xstrdup("No DNS records"); - return &f; - } - debug(35, 3) ("fqdncacheParse: %d answers\n", nr); - assert(answers); - for (k = 0; k < nr; k++) { - if (answers[k].type != RFC1035_TYPE_PTR) - continue; - if (answers[k]._class != RFC1035_CLASS_IN) - continue; - na++; - f.flags.negcached = 0; - f.names[0] = xstrdup(answers[k].rdata); - f.name_count = 1; - f.expires = squid_curtime + answers[k].ttl; - return &f; - } - debug(35, 1) ("fqdncacheParse: No PTR record\n"); - f.error_message = xstrdup("No PTR record"); - return &f; -} -#endif - -static void -#if USE_DNSSERVERS -fqdncacheHandleReply(void *data, char *reply) -#else -fqdncacheHandleReply(void *data, rfc1035_rr * answers, int na) -#endif -{ - int n; - generic_cbdata *c = data; - fqdncache_entry *f = c->data; - fqdncache_entry *x = NULL; - cbdataFree(c); - c = NULL; - n = ++FqdncacheStats.replies; - statHistCount(&statCounter.dns.svc_time, - tvSubMsec(f->request_time, current_time)); -#if USE_DNSSERVERS - x = fqdncacheParse(reply); -#else - x = fqdncacheParse(answers, na); -#endif - assert(x); - f->name_count = x->name_count; - for (n = 0; n < (int) f->name_count; n++) - f->names[n] = x->names[n]; - f->error_message = x->error_message; - f->expires = x->expires; - f->flags = x->flags; - fqdncacheAddEntry(f); - fqdncacheCallback(f); -} - -void -fqdncache_nbgethostbyaddr(struct in_addr addr, FQDNH * handler, void *handlerData) -{ - fqdncache_entry *f = NULL; - char *name = inet_ntoa(addr); - generic_cbdata *c; - assert(handler); - debug(35, 4) ("fqdncache_nbgethostbyaddr: Name '%s'.\n", name); - FqdncacheStats.requests++; - if (name == NULL || name[0] == '\0') { - debug(35, 4) ("fqdncache_nbgethostbyaddr: Invalid name!\n"); - handler(NULL, handlerData); - return; - } - f = fqdncache_get(name); - if (NULL == f) { - /* miss */ - (void) 0; - } else if (fqdncacheExpiredEntry(f)) { - /* hit, but expired -- bummer */ - fqdncacheRelease(f); - f = NULL; - } else { - /* hit */ - debug(35, 4) ("fqdncache_nbgethostbyaddr: HIT for '%s'\n", name); - if (f->flags.negcached) - FqdncacheStats.negative_hits++; - else - FqdncacheStats.hits++; - f->handler = handler; - f->handlerData = cbdataReference(handlerData); - fqdncacheCallback(f); - return; - } - - debug(35, 5) ("fqdncache_nbgethostbyaddr: MISS for '%s'\n", name); - FqdncacheStats.misses++; - f = fqdncacheCreateEntry(name); - f->handler = handler; - f->handlerData = cbdataReference(handlerData); - f->request_time = current_time; - c = cbdataAlloc(generic_cbdata); - c->data = f; -#if USE_DNSSERVERS - dnsSubmit(hashKeyStr(&f->hash), fqdncacheHandleReply, c); -#else - idnsPTRLookup(addr, fqdncacheHandleReply, c); -#endif -} - -/* initialize the fqdncache */ -void -fqdncache_init(void) -{ - int n; - if (fqdn_table) - return; - debug(35, 3) ("Initializing FQDN Cache...\n"); - memset(&FqdncacheStats, '\0', sizeof(FqdncacheStats)); - memset(&lru_list, '\0', sizeof(lru_list)); - fqdncache_high = (long) (((float) Config.fqdncache.size * - (float) FQDN_HIGH_WATER) / (float) 100); - fqdncache_low = (long) (((float) Config.fqdncache.size * - (float) FQDN_LOW_WATER) / (float) 100); - n = hashPrime(fqdncache_high / 4); - fqdn_table = hash_create((HASHCMP *) strcmp, n, hash4); - cachemgrRegister("fqdncache", - "FQDN Cache Stats and Contents", - fqdnStats, 0, 1); - memDataInit(MEM_FQDNCACHE_ENTRY, "fqdncache_entry", - sizeof(fqdncache_entry), 0); -} - -const char * -fqdncache_gethostbyaddr(struct in_addr addr, int flags) -{ - char *name = inet_ntoa(addr); - fqdncache_entry *f = NULL; - struct in_addr ip; - assert(name); - FqdncacheStats.requests++; - f = fqdncache_get(name); - if (NULL == f) { - (void) 0; - } else if (fqdncacheExpiredEntry(f)) { - fqdncacheRelease(f); - f = NULL; - } else if (f->flags.negcached) { - FqdncacheStats.negative_hits++; - dns_error_message = f->error_message; - return NULL; - } else { - FqdncacheStats.hits++; - f->lastref = squid_curtime; - return f->names[0]; - } - /* check if it's already a FQDN address in text form. */ - if (!safe_inet_addr(name, &ip)) - return name; - FqdncacheStats.misses++; - if (flags & FQDN_LOOKUP_IF_MISS) - fqdncache_nbgethostbyaddr(addr, dummy_handler, NULL); - return NULL; -} - - -/* process objects list */ -void -fqdnStats(StoreEntry * sentry) -{ - fqdncache_entry *f = NULL; - int k; - int ttl; - if (fqdn_table == NULL) - return; - storeAppendPrintf(sentry, "FQDN Cache Statistics:\n"); - storeAppendPrintf(sentry, "FQDNcache Entries: %d\n", - memInUse(MEM_FQDNCACHE_ENTRY)); - storeAppendPrintf(sentry, "FQDNcache Requests: %d\n", - FqdncacheStats.requests); - storeAppendPrintf(sentry, "FQDNcache Hits: %d\n", - FqdncacheStats.hits); - storeAppendPrintf(sentry, "FQDNcache Negative Hits: %d\n", - FqdncacheStats.negative_hits); - storeAppendPrintf(sentry, "FQDNcache Misses: %d\n", - FqdncacheStats.misses); - storeAppendPrintf(sentry, "Blocking calls to gethostbyaddr(): %d\n", - FqdncacheStats.ghba_calls); - storeAppendPrintf(sentry, "FQDN Cache Contents:\n\n"); - storeAppendPrintf(sentry, "%-15.15s %3s %3s %3s %s\n", - "Address", "Flg", "TTL", "Cnt", "Hostnames"); - hash_first(fqdn_table); - while ((f = (fqdncache_entry *) hash_next(fqdn_table))) { - ttl = (f->flags.fromhosts ? -1 : (f->expires - squid_curtime)); - storeAppendPrintf(sentry, "%-15.15s %c%c %3.3d % 3d", - hashKeyStr(&f->hash), - f->flags.negcached ? 'N' : ' ', - f->flags.fromhosts ? 'H' : ' ', - ttl, - (int) f->name_count); - for (k = 0; k < (int) f->name_count; k++) - storeAppendPrintf(sentry, " %s", f->names[k]); - storeAppendPrintf(sentry, "\n"); - } -} - -static void -dummy_handler(const char *bufnotused, void *datanotused) -{ - return; -} - -const char * -fqdnFromAddr(struct in_addr addr) -{ - const char *n; - static char buf[32]; - if (Config.onoff.log_fqdn && (n = fqdncache_gethostbyaddr(addr, 0))) - return n; - xstrncpy(buf, inet_ntoa(addr), 32); - return buf; -} - -static void -fqdncacheLockEntry(fqdncache_entry * f) -{ - if (f->locks++ == 0) { - dlinkDelete(&f->lru, &lru_list); - dlinkAdd(f, &f->lru, &lru_list); - } -} - -static void -fqdncacheUnlockEntry(fqdncache_entry * f) -{ - assert(f->locks > 0); - f->locks--; - if (fqdncacheExpiredEntry(f)) - fqdncacheRelease(f); -} - -static void -fqdncacheFreeEntry(void *data) -{ - fqdncache_entry *f = data; - int k; - for (k = 0; k < (int) f->name_count; k++) - safe_free(f->names[k]); - safe_free(f->hash.key); - safe_free(f->error_message); - memFree(f, MEM_FQDNCACHE_ENTRY); -} - -void -fqdncacheFreeMemory(void) -{ - hashFreeItems(fqdn_table, fqdncacheFreeEntry); - hashFreeMemory(fqdn_table); - fqdn_table = NULL; -} - -/* Recalculate FQDN cache size upon reconfigure */ -void -fqdncache_restart(void) -{ - fqdncache_high = (long) (((float) Config.fqdncache.size * - (float) FQDN_HIGH_WATER) / (float) 100); - fqdncache_low = (long) (((float) Config.fqdncache.size * - (float) FQDN_LOW_WATER) / (float) 100); - purge_entries_fromhosts(); -} - -/* - * adds a "static" entry from /etc/hosts. the worldist is to be - * managed by the caller, including pointed-to strings - */ -void -fqdncacheAddEntryFromHosts(char *addr, wordlist * hostnames) -{ - fqdncache_entry *fce; - int j = 0; - if ((fce = fqdncache_get(addr))) { - if (1 == fce->flags.fromhosts) { - fqdncacheUnlockEntry(fce); - } else if (fce->locks > 0) { - debug(35, 1) ("fqdncacheAddEntryFromHosts: can't add static entry for locked address '%s'\n", addr); - return; - } else { - fqdncacheRelease(fce); - } - } - fce = fqdncacheCreateEntry(addr); - while (hostnames) { - fce->names[j] = xstrdup(hostnames->key); - j++; - hostnames = hostnames->next; - if (j >= FQDN_MAX_NAMES) - break; - } - fce->name_count = j; - fce->names[j] = NULL; /* it's safe */ - fce->flags.fromhosts = 1; - fqdncacheAddEntry(fce); - fqdncacheLockEntry(fce); -} - - -#ifdef SQUID_SNMP -/* - * The function to return the fqdn statistics via SNMP - */ - -variable_list * -snmp_netFqdnFn(variable_list * Var, snint * ErrP) -{ - variable_list *Answer = NULL; - debug(49, 5) ("snmp_netFqdnFn: Processing request:\n"); - snmpDebugOid(5, Var->name, Var->name_length); - *ErrP = SNMP_ERR_NOERROR; - switch (Var->name[LEN_SQ_NET + 1]) { - case FQDN_ENT: - Answer = snmp_var_new_integer(Var->name, Var->name_length, - memInUse(MEM_FQDNCACHE_ENTRY), - SMI_GAUGE32); - break; - case FQDN_REQ: - Answer = snmp_var_new_integer(Var->name, Var->name_length, - FqdncacheStats.requests, - SMI_COUNTER32); - break; - case FQDN_HITS: - Answer = snmp_var_new_integer(Var->name, Var->name_length, - FqdncacheStats.hits, - SMI_COUNTER32); - break; - case FQDN_PENDHIT: - /* this is now worthless */ - Answer = snmp_var_new_integer(Var->name, Var->name_length, - 0, - SMI_GAUGE32); - break; - case FQDN_NEGHIT: - Answer = snmp_var_new_integer(Var->name, Var->name_length, - FqdncacheStats.negative_hits, - SMI_COUNTER32); - break; - case FQDN_MISS: - Answer = snmp_var_new_integer(Var->name, Var->name_length, - FqdncacheStats.misses, - SMI_COUNTER32); - break; - case FQDN_GHBN: - Answer = snmp_var_new_integer(Var->name, Var->name_length, - FqdncacheStats.ghba_calls, - SMI_COUNTER32); - break; - default: - *ErrP = SNMP_ERR_NOSUCHNAME; - break; - } - return Answer; -} - -#endif /*SQUID_SNMP */ --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/fqdncache.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,650 @@ + +/* + * $Id: fqdncache.cc,v 1.1.2.1 2002/10/11 15:40:48 rbcollins Exp $ + * + * DEBUG: section 35 FQDN Cache + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "Store.h" + +#define FQDN_LOW_WATER 90 +#define FQDN_HIGH_WATER 95 + +typedef struct _fqdncache_entry fqdncache_entry; + +struct _fqdncache_entry { + hash_link hash; /* must be first */ + time_t lastref; + time_t expires; + unsigned char name_count; + char *names[FQDN_MAX_NAMES + 1]; + FQDNH *handler; + void *handlerData; + char *error_message; + struct timeval request_time; + dlink_node lru; + unsigned short locks; + struct { + unsigned int negcached:1; + unsigned int fromhosts:1; + } flags; +}; + +static struct { + int requests; + int replies; + int hits; + int misses; + int negative_hits; + int errors; + int ghba_calls; /* # calls to blocking gethostbyaddr() */ +} FqdncacheStats; + +static dlink_list lru_list; + +#if USE_DNSSERVERS +static HLPCB fqdncacheHandleReply; +static fqdncache_entry *fqdncacheParse(const char *buf); +#else +static IDNSCB fqdncacheHandleReply; +static fqdncache_entry *fqdncacheParse(rfc1035_rr *, int); +#endif +static void fqdncacheRelease(fqdncache_entry *); +static fqdncache_entry *fqdncacheCreateEntry(const char *name); +static void fqdncacheCallback(fqdncache_entry *); +static fqdncache_entry *fqdncache_get(const char *); +static FQDNH dummy_handler; +static int fqdncacheExpiredEntry(const fqdncache_entry *); +static void fqdncacheLockEntry(fqdncache_entry * f); +static void fqdncacheUnlockEntry(fqdncache_entry * f); +static FREE fqdncacheFreeEntry; +static void fqdncacheAddEntry(fqdncache_entry * f); + +static hash_table *fqdn_table = NULL; + +static long fqdncache_low = 180; +static long fqdncache_high = 200; + +/* removes the given fqdncache entry */ +static void +fqdncacheRelease(fqdncache_entry * f) +{ + int k; + hash_remove_link(fqdn_table, (hash_link *) f); + for (k = 0; k < (int) f->name_count; k++) + safe_free(f->names[k]); + debug(35, 5) ("fqdncacheRelease: Released FQDN record for '%s'.\n", + hashKeyStr(&f->hash)); + dlinkDelete(&f->lru, &lru_list); + safe_free(f->hash.key); + safe_free(f->error_message); + memFree(f, MEM_FQDNCACHE_ENTRY); +} + +/* return match for given name */ +static fqdncache_entry * +fqdncache_get(const char *name) +{ + hash_link *e; + static fqdncache_entry *f; + f = NULL; + if (fqdn_table) { + if ((e = (hash_link *)hash_lookup(fqdn_table, name)) != NULL) + f = (fqdncache_entry *) e; + } + return f; +} + +static int +fqdncacheExpiredEntry(const fqdncache_entry * f) +{ + /* all static entries are locked, so this takes care of them too */ + if (f->locks != 0) + return 0; + if (f->expires > squid_curtime) + return 0; + return 1; +} + +void +fqdncache_purgelru(void *notused) +{ + dlink_node *m; + dlink_node *prev = NULL; + fqdncache_entry *f; + int removed = 0; + eventAdd("fqdncache_purgelru", fqdncache_purgelru, NULL, 10.0, 1); + for (m = lru_list.tail; m; m = prev) { + if (memInUse(MEM_FQDNCACHE_ENTRY) < fqdncache_low) + break; + prev = m->prev; + f = (fqdncache_entry *)m->data; + if (f->locks != 0) + continue; + fqdncacheRelease(f); + removed++; + } + debug(35, 9) ("fqdncache_purgelru: removed %d entries\n", removed); +} + +static void +purge_entries_fromhosts(void) +{ + dlink_node *m = lru_list.head; + fqdncache_entry *i = NULL; + fqdncache_entry *t; + while (m) { + if (i != NULL) { /* need to delay deletion */ + fqdncacheRelease(i); /* we just override locks */ + i = NULL; + } + t = (fqdncache_entry *)m->data; + if (t->flags.fromhosts) + i = t; + m = m->next; + } + if (i != NULL) + fqdncacheRelease(i); +} + +/* create blank fqdncache_entry */ +static fqdncache_entry * +fqdncacheCreateEntry(const char *name) +{ + static fqdncache_entry *f; + f = (fqdncache_entry *)memAllocate(MEM_FQDNCACHE_ENTRY); + f->hash.key = xstrdup(name); + f->expires = squid_curtime + Config.negativeDnsTtl; + return f; +} + +static void +fqdncacheAddEntry(fqdncache_entry * f) +{ + hash_link *e = (hash_link *)hash_lookup(fqdn_table, f->hash.key); + if (NULL != e) { + /* avoid colission */ + fqdncache_entry *q = (fqdncache_entry *) e; + fqdncacheRelease(q); + } + hash_join(fqdn_table, &f->hash); + dlinkAdd(f, &f->lru, &lru_list); + f->lastref = squid_curtime; +} + +/* walks down the pending list, calling handlers */ +static void +fqdncacheCallback(fqdncache_entry * f) +{ + FQDNH *callback; + void *cbdata; + f->lastref = squid_curtime; + if (!f->handler) + return; + fqdncacheLockEntry(f); + callback = f->handler; + f->handler = NULL; + if (cbdataReferenceValidDone(f->handlerData, &cbdata)) { + dns_error_message = f->error_message; + callback(f->flags.negcached ? NULL : f->names[0], cbdata); + } + fqdncacheUnlockEntry(f); +} + +static fqdncache_entry * +#if USE_DNSSERVERS +fqdncacheParse(const char *inbuf) +{ + LOCAL_ARRAY(char, buf, DNS_INBUF_SZ); + char *token; + static fqdncache_entry f; + int ttl; + memset(&f, '\0', sizeof(f)); + f.expires = squid_curtime; + f.flags.negcached = 1; + if (inbuf == NULL) { + debug(35, 1) ("fqdncacheParse: Got reply\n"); + return &f; + } + xstrncpy(buf, inbuf, DNS_INBUF_SZ); + debug(35, 5) ("fqdncacheParse: parsing: {%s}\n", buf); + token = strtok(buf, w_space); + if (NULL == token) { + debug(35, 1) ("fqdncacheParse: Got , expecting '$name'\n"); + return &f; + } + if (0 == strcmp(token, "$fail")) { + f.expires = squid_curtime + Config.negativeDnsTtl; + token = strtok(NULL, "\n"); + assert(NULL != token); + f.error_message = xstrdup(token); + return &f; + } + if (0 != strcmp(token, "$name")) { + debug(35, 1) ("fqdncacheParse: Got '%s', expecting '$name'\n", token); + return &f; + } + token = strtok(NULL, w_space); + if (NULL == token) { + debug(35, 1) ("fqdncacheParse: Got , expecting TTL\n"); + return &f; + } + f.flags.negcached = 0; + ttl = atoi(token); + if (ttl > 0) + f.expires = squid_curtime + ttl; + else + f.expires = squid_curtime + Config.positiveDnsTtl; + token = strtok(NULL, w_space); + if (NULL != token) { + f.names[0] = xstrdup(token); + f.name_count = 1; + } + return &f; +} +#else +fqdncacheParse(rfc1035_rr * answers, int nr) +{ + static fqdncache_entry f; + int k; + int na = 0; + memset(&f, '\0', sizeof(f)); + f.expires = squid_curtime; + f.flags.negcached = 1; + if (nr < 0) { + debug(35, 3) ("fqdncacheParse: Lookup failed (error %d)\n", + rfc1035_errno); + assert(rfc1035_error_message); + f.error_message = xstrdup(rfc1035_error_message); + return &f; + } + if (nr == 0) { + debug(35, 3) ("fqdncacheParse: No DNS records\n"); + f.error_message = xstrdup("No DNS records"); + return &f; + } + debug(35, 3) ("fqdncacheParse: %d answers\n", nr); + assert(answers); + for (k = 0; k < nr; k++) { + if (answers[k].type != RFC1035_TYPE_PTR) + continue; + if (answers[k]._class != RFC1035_CLASS_IN) + continue; + na++; + f.flags.negcached = 0; + f.names[0] = xstrdup(answers[k].rdata); + f.name_count = 1; + f.expires = squid_curtime + answers[k].ttl; + return &f; + } + debug(35, 1) ("fqdncacheParse: No PTR record\n"); + f.error_message = xstrdup("No PTR record"); + return &f; +} +#endif + +static void +#if USE_DNSSERVERS +fqdncacheHandleReply(void *data, char *reply) +#else +fqdncacheHandleReply(void *data, rfc1035_rr * answers, int na) +#endif +{ + int n; + generic_cbdata *c = (generic_cbdata *)data; + fqdncache_entry *f = (fqdncache_entry *)c->data; + fqdncache_entry *x = NULL; + cbdataFree(c); + c = NULL; + n = ++FqdncacheStats.replies; + statHistCount(&statCounter.dns.svc_time, + tvSubMsec(f->request_time, current_time)); +#if USE_DNSSERVERS + x = fqdncacheParse(reply); +#else + x = fqdncacheParse(answers, na); +#endif + assert(x); + f->name_count = x->name_count; + for (n = 0; n < (int) f->name_count; n++) + f->names[n] = x->names[n]; + f->error_message = x->error_message; + f->expires = x->expires; + f->flags = x->flags; + fqdncacheAddEntry(f); + fqdncacheCallback(f); +} + +void +fqdncache_nbgethostbyaddr(struct in_addr addr, FQDNH * handler, void *handlerData) +{ + fqdncache_entry *f = NULL; + char *name = inet_ntoa(addr); + generic_cbdata *c; + assert(handler); + debug(35, 4) ("fqdncache_nbgethostbyaddr: Name '%s'.\n", name); + FqdncacheStats.requests++; + if (name == NULL || name[0] == '\0') { + debug(35, 4) ("fqdncache_nbgethostbyaddr: Invalid name!\n"); + handler(NULL, handlerData); + return; + } + f = fqdncache_get(name); + if (NULL == f) { + /* miss */ + (void) 0; + } else if (fqdncacheExpiredEntry(f)) { + /* hit, but expired -- bummer */ + fqdncacheRelease(f); + f = NULL; + } else { + /* hit */ + debug(35, 4) ("fqdncache_nbgethostbyaddr: HIT for '%s'\n", name); + if (f->flags.negcached) + FqdncacheStats.negative_hits++; + else + FqdncacheStats.hits++; + f->handler = handler; + f->handlerData = cbdataReference(handlerData); + fqdncacheCallback(f); + return; + } + + debug(35, 5) ("fqdncache_nbgethostbyaddr: MISS for '%s'\n", name); + FqdncacheStats.misses++; + f = fqdncacheCreateEntry(name); + f->handler = handler; + f->handlerData = cbdataReference(handlerData); + f->request_time = current_time; + c = cbdataAlloc(generic_cbdata); + c->data = f; +#if USE_DNSSERVERS + dnsSubmit(hashKeyStr(&f->hash), fqdncacheHandleReply, c); +#else + idnsPTRLookup(addr, fqdncacheHandleReply, c); +#endif +} + +/* initialize the fqdncache */ +void +fqdncache_init(void) +{ + int n; + if (fqdn_table) + return; + debug(35, 3) ("Initializing FQDN Cache...\n"); + memset(&FqdncacheStats, '\0', sizeof(FqdncacheStats)); + memset(&lru_list, '\0', sizeof(lru_list)); + fqdncache_high = (long) (((float) Config.fqdncache.size * + (float) FQDN_HIGH_WATER) / (float) 100); + fqdncache_low = (long) (((float) Config.fqdncache.size * + (float) FQDN_LOW_WATER) / (float) 100); + n = hashPrime(fqdncache_high / 4); + fqdn_table = hash_create((HASHCMP *) strcmp, n, hash4); + cachemgrRegister("fqdncache", + "FQDN Cache Stats and Contents", + fqdnStats, 0, 1); + memDataInit(MEM_FQDNCACHE_ENTRY, "fqdncache_entry", + sizeof(fqdncache_entry), 0); +} + +const char * +fqdncache_gethostbyaddr(struct in_addr addr, int flags) +{ + char *name = inet_ntoa(addr); + fqdncache_entry *f = NULL; + struct in_addr ip; + assert(name); + FqdncacheStats.requests++; + f = fqdncache_get(name); + if (NULL == f) { + (void) 0; + } else if (fqdncacheExpiredEntry(f)) { + fqdncacheRelease(f); + f = NULL; + } else if (f->flags.negcached) { + FqdncacheStats.negative_hits++; + dns_error_message = f->error_message; + return NULL; + } else { + FqdncacheStats.hits++; + f->lastref = squid_curtime; + return f->names[0]; + } + /* check if it's already a FQDN address in text form. */ + if (!safe_inet_addr(name, &ip)) + return name; + FqdncacheStats.misses++; + if (flags & FQDN_LOOKUP_IF_MISS) + fqdncache_nbgethostbyaddr(addr, dummy_handler, NULL); + return NULL; +} + + +/* process objects list */ +void +fqdnStats(StoreEntry * sentry) +{ + fqdncache_entry *f = NULL; + int k; + int ttl; + if (fqdn_table == NULL) + return; + storeAppendPrintf(sentry, "FQDN Cache Statistics:\n"); + storeAppendPrintf(sentry, "FQDNcache Entries: %d\n", + memInUse(MEM_FQDNCACHE_ENTRY)); + storeAppendPrintf(sentry, "FQDNcache Requests: %d\n", + FqdncacheStats.requests); + storeAppendPrintf(sentry, "FQDNcache Hits: %d\n", + FqdncacheStats.hits); + storeAppendPrintf(sentry, "FQDNcache Negative Hits: %d\n", + FqdncacheStats.negative_hits); + storeAppendPrintf(sentry, "FQDNcache Misses: %d\n", + FqdncacheStats.misses); + storeAppendPrintf(sentry, "Blocking calls to gethostbyaddr(): %d\n", + FqdncacheStats.ghba_calls); + storeAppendPrintf(sentry, "FQDN Cache Contents:\n\n"); + storeAppendPrintf(sentry, "%-15.15s %3s %3s %3s %s\n", + "Address", "Flg", "TTL", "Cnt", "Hostnames"); + hash_first(fqdn_table); + while ((f = (fqdncache_entry *) hash_next(fqdn_table))) { + ttl = (f->flags.fromhosts ? -1 : (f->expires - squid_curtime)); + storeAppendPrintf(sentry, "%-15.15s %c%c %3.3d % 3d", + hashKeyStr(&f->hash), + f->flags.negcached ? 'N' : ' ', + f->flags.fromhosts ? 'H' : ' ', + ttl, + (int) f->name_count); + for (k = 0; k < (int) f->name_count; k++) + storeAppendPrintf(sentry, " %s", f->names[k]); + storeAppendPrintf(sentry, "\n"); + } +} + +static void +dummy_handler(const char *bufnotused, void *datanotused) +{ + return; +} + +const char * +fqdnFromAddr(struct in_addr addr) +{ + const char *n; + static char buf[32]; + if (Config.onoff.log_fqdn && (n = fqdncache_gethostbyaddr(addr, 0))) + return n; + xstrncpy(buf, inet_ntoa(addr), 32); + return buf; +} + +static void +fqdncacheLockEntry(fqdncache_entry * f) +{ + if (f->locks++ == 0) { + dlinkDelete(&f->lru, &lru_list); + dlinkAdd(f, &f->lru, &lru_list); + } +} + +static void +fqdncacheUnlockEntry(fqdncache_entry * f) +{ + assert(f->locks > 0); + f->locks--; + if (fqdncacheExpiredEntry(f)) + fqdncacheRelease(f); +} + +static void +fqdncacheFreeEntry(void *data) +{ + fqdncache_entry *f = (fqdncache_entry *)data; + int k; + for (k = 0; k < (int) f->name_count; k++) + safe_free(f->names[k]); + safe_free(f->hash.key); + safe_free(f->error_message); + memFree(f, MEM_FQDNCACHE_ENTRY); +} + +void +fqdncacheFreeMemory(void) +{ + hashFreeItems(fqdn_table, fqdncacheFreeEntry); + hashFreeMemory(fqdn_table); + fqdn_table = NULL; +} + +/* Recalculate FQDN cache size upon reconfigure */ +void +fqdncache_restart(void) +{ + fqdncache_high = (long) (((float) Config.fqdncache.size * + (float) FQDN_HIGH_WATER) / (float) 100); + fqdncache_low = (long) (((float) Config.fqdncache.size * + (float) FQDN_LOW_WATER) / (float) 100); + purge_entries_fromhosts(); +} + +/* + * adds a "static" entry from /etc/hosts. the worldist is to be + * managed by the caller, including pointed-to strings + */ +void +fqdncacheAddEntryFromHosts(char *addr, wordlist * hostnames) +{ + fqdncache_entry *fce; + int j = 0; + if ((fce = fqdncache_get(addr))) { + if (1 == fce->flags.fromhosts) { + fqdncacheUnlockEntry(fce); + } else if (fce->locks > 0) { + debug(35, 1) ("fqdncacheAddEntryFromHosts: can't add static entry for locked address '%s'\n", addr); + return; + } else { + fqdncacheRelease(fce); + } + } + fce = fqdncacheCreateEntry(addr); + while (hostnames) { + fce->names[j] = xstrdup(hostnames->key); + j++; + hostnames = hostnames->next; + if (j >= FQDN_MAX_NAMES) + break; + } + fce->name_count = j; + fce->names[j] = NULL; /* it's safe */ + fce->flags.fromhosts = 1; + fqdncacheAddEntry(fce); + fqdncacheLockEntry(fce); +} + + +#ifdef SQUID_SNMP +/* + * The function to return the fqdn statistics via SNMP + */ + +variable_list * +snmp_netFqdnFn(variable_list * Var, snint * ErrP) +{ + variable_list *Answer = NULL; + debug(49, 5) ("snmp_netFqdnFn: Processing request:\n"); + snmpDebugOid(5, Var->name, Var->name_length); + *ErrP = SNMP_ERR_NOERROR; + switch (Var->name[LEN_SQ_NET + 1]) { + case FQDN_ENT: + Answer = snmp_var_new_integer(Var->name, Var->name_length, + memInUse(MEM_FQDNCACHE_ENTRY), + SMI_GAUGE32); + break; + case FQDN_REQ: + Answer = snmp_var_new_integer(Var->name, Var->name_length, + FqdncacheStats.requests, + SMI_COUNTER32); + break; + case FQDN_HITS: + Answer = snmp_var_new_integer(Var->name, Var->name_length, + FqdncacheStats.hits, + SMI_COUNTER32); + break; + case FQDN_PENDHIT: + /* this is now worthless */ + Answer = snmp_var_new_integer(Var->name, Var->name_length, + 0, + SMI_GAUGE32); + break; + case FQDN_NEGHIT: + Answer = snmp_var_new_integer(Var->name, Var->name_length, + FqdncacheStats.negative_hits, + SMI_COUNTER32); + break; + case FQDN_MISS: + Answer = snmp_var_new_integer(Var->name, Var->name_length, + FqdncacheStats.misses, + SMI_COUNTER32); + break; + case FQDN_GHBN: + Answer = snmp_var_new_integer(Var->name, Var->name_length, + FqdncacheStats.ghba_calls, + SMI_COUNTER32); + break; + default: + *ErrP = SNMP_ERR_NOSUCHNAME; + break; + } + return Answer; +} + +#endif /*SQUID_SNMP */ --- squid/src/ftp.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,2615 +0,0 @@ - -/* - * $Id: ftp.c,v 1.28.6.2 2002/10/09 14:14:26 rbcollins Exp $ - * - * DEBUG: section 9 File Transfer Protocol (FTP) - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "Store.h" - - -static const char *const crlf = "\r\n"; -static char cbuf[1024]; - -typedef enum { - BEGIN, - SENT_USER, - SENT_PASS, - SENT_TYPE, - SENT_MDTM, - SENT_SIZE, - SENT_PORT, - SENT_PASV, - SENT_CWD, - SENT_LIST, - SENT_NLST, - SENT_REST, - SENT_RETR, - SENT_STOR, - SENT_QUIT, - READING_DATA, - WRITING_DATA, - SENT_MKDIR -} ftp_state_t; - -struct _ftp_flags { - unsigned int isdir:1; - unsigned int pasv_supported:1; - unsigned int skip_whitespace:1; - unsigned int rest_supported:1; - unsigned int pasv_only:1; - unsigned int authenticated:1; - unsigned int http_header_sent:1; - unsigned int tried_nlst:1; - unsigned int need_base_href:1; - unsigned int root_dir:1; - unsigned int no_dotdot:1; - unsigned int html_header_sent:1; - unsigned int binary:1; - unsigned int try_slash_hack:1; - unsigned int put:1; - unsigned int put_mkdir:1; - unsigned int listformat_unknown:1; - unsigned int datachannel_hack:1; -}; - -typedef struct _Ftpdata { - StoreEntry *entry; - request_t *request; - char user[MAX_URL]; - char password[MAX_URL]; - int password_url; - char *reply_hdr; - int reply_hdr_state; - String title_url; - String base_href; - int conn_att; - int login_att; - ftp_state_t state; - time_t mdtm; - int size; - wordlist *pathcomps; - char *filepath; - int restart_offset; - int restarted_offset; - int rest_att; - char *proxy_host; - size_t list_width; - wordlist *cwd_message; - char *old_request; - char *old_reply; - char *old_filepath; - char typecode; - struct { - int fd; - char *buf; - size_t size; - off_t offset; - wordlist *message; - char *last_command; - char *last_reply; - int replycode; - } ctrl; - struct { - int fd; - char *buf; - size_t size; - off_t offset; - char *host; - u_short port; - } data; - struct _ftp_flags flags; - FwdState *fwd; -} FtpStateData; - -typedef struct { - char type; - int size; - char *date; - char *name; - char *showname; - char *link; -} ftpListParts; - -typedef void (FTPSM) (FtpStateData *); - -#define FTP_LOGIN_ESCAPED 1 -#define FTP_LOGIN_NOT_ESCAPED 0 - -/* Local functions */ -static CNCB ftpPasvCallback; -static PF ftpDataRead; -static PF ftpDataWrite; -static CWCB ftpDataWriteCallback; -static PF ftpStateFree; -static PF ftpTimeout; -static PF ftpReadControlReply; -static CWCB ftpWriteCommandCallback; -static void ftpLoginParser(const char *, FtpStateData *, int escaped); -static wordlist *ftpParseControlReply(char *, size_t, int *, int *); -static int ftpRestartable(FtpStateData * ftpState); -static void ftpAppendSuccessHeader(FtpStateData * ftpState); -static void ftpAuthRequired(HttpReply * reply, request_t * request, const char *realm); -static void ftpHackShortcut(FtpStateData * ftpState, FTPSM * nextState); -static void ftpUnhack(FtpStateData * ftpState); -static void ftpScheduleReadControlReply(FtpStateData *, int); -static void ftpHandleControlReply(FtpStateData *); -static char *ftpHtmlifyListEntry(const char *line, FtpStateData * ftpState); -static void ftpFailed(FtpStateData *, err_type); -static void ftpFailedErrorMessage(FtpStateData *, err_type); - -/* - * State machine functions - * send == state transition - * read == wait for response, and select next state transition - * other == Transition logic - */ -static FTPSM ftpReadWelcome; -static FTPSM ftpSendUser; -static FTPSM ftpReadUser; -static FTPSM ftpSendPass; -static FTPSM ftpReadPass; -static FTPSM ftpSendType; -static FTPSM ftpReadType; -static FTPSM ftpSendMdtm; -static FTPSM ftpReadMdtm; -static FTPSM ftpSendSize; -static FTPSM ftpReadSize; -static FTPSM ftpSendPort; -static FTPSM ftpReadPort; -static FTPSM ftpSendPasv; -static FTPSM ftpReadPasv; -static FTPSM ftpTraverseDirectory; -static FTPSM ftpListDir; -static FTPSM ftpGetFile; -static FTPSM ftpSendCwd; -static FTPSM ftpReadCwd; -static FTPSM ftpRestOrList; -static FTPSM ftpSendList; -static FTPSM ftpSendNlst; -static FTPSM ftpReadList; -static FTPSM ftpSendRest; -static FTPSM ftpReadRest; -static FTPSM ftpSendRetr; -static FTPSM ftpReadRetr; -static FTPSM ftpReadTransferDone; -static FTPSM ftpSendStor; -static FTPSM ftpReadStor; -static FTPSM ftpWriteTransferDone; -static FTPSM ftpSendReply; -static FTPSM ftpSendMkdir; -static FTPSM ftpReadMkdir; -static FTPSM ftpFail; -static FTPSM ftpSendQuit; -static FTPSM ftpReadQuit; -/************************************************ -** State Machine Description (excluding hacks) ** -************************************************* -From To ---------------------------------------- -Welcome User -User Pass -Pass Type -Type TraverseDirectory / GetFile -TraverseDirectory Cwd / GetFile / ListDir -Cwd TraverseDirectory / Mkdir -GetFile Mdtm -Mdtm Size -Size Pasv -ListDir Pasv -Pasv FileOrList -FileOrList Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d) -Rest Retr -Retr / Nlst / List DataRead* (on datachannel) -DataRead* ReadTransferDone -ReadTransferDone DataTransferDone -Stor DataWrite* (on datachannel) -DataWrite* RequestPutBody** (from client) -RequestPutBody** DataWrite* / WriteTransferDone -WriteTransferDone DataTransferDone -DataTransferDone Quit -Quit - -************************************************/ - -FTPSM *FTP_SM_FUNCS[] = -{ - ftpReadWelcome, /* BEGIN */ - ftpReadUser, /* SENT_USER */ - ftpReadPass, /* SENT_PASS */ - ftpReadType, /* SENT_TYPE */ - ftpReadMdtm, /* SENT_MDTM */ - ftpReadSize, /* SENT_SIZE */ - ftpReadPort, /* SENT_PORT */ - ftpReadPasv, /* SENT_PASV */ - ftpReadCwd, /* SENT_CWD */ - ftpReadList, /* SENT_LIST */ - ftpReadList, /* SENT_NLST */ - ftpReadRest, /* SENT_REST */ - ftpReadRetr, /* SENT_RETR */ - ftpReadStor, /* SENT_STOR */ - ftpReadQuit, /* SENT_QUIT */ - ftpReadTransferDone, /* READING_DATA (RETR,LIST,NLST) */ - ftpWriteTransferDone, /* WRITING_DATA (STOR) */ - ftpReadMkdir /* SENT_MKDIR */ -}; - -static void -ftpStateFree(int fdnotused, void *data) -{ - FtpStateData *ftpState = data; - if (ftpState == NULL) - return; - debug(9, 3) ("ftpStateFree: %s\n", storeUrl(ftpState->entry)); - storeUnregisterAbort(ftpState->entry); - storeUnlockObject(ftpState->entry); - if (ftpState->reply_hdr) { - memFree(ftpState->reply_hdr, MEM_8K_BUF); - ftpState->reply_hdr = NULL; - } - requestUnlink(ftpState->request); - if (ftpState->ctrl.buf) { - memFreeBuf(ftpState->ctrl.size, ftpState->ctrl.buf); - ftpState->ctrl.buf = NULL; - } - if (ftpState->data.buf) { - memFreeBuf(ftpState->data.size, ftpState->data.buf); - ftpState->data.buf = NULL; - } - if (ftpState->pathcomps) - wordlistDestroy(&ftpState->pathcomps); - if (ftpState->ctrl.message) - wordlistDestroy(&ftpState->ctrl.message); - if (ftpState->cwd_message) - wordlistDestroy(&ftpState->cwd_message); - safe_free(ftpState->ctrl.last_reply); - safe_free(ftpState->ctrl.last_command); - safe_free(ftpState->old_request); - safe_free(ftpState->old_reply); - safe_free(ftpState->old_filepath); - stringClean(&ftpState->title_url); - stringClean(&ftpState->base_href); - safe_free(ftpState->filepath); - safe_free(ftpState->data.host); - if (ftpState->data.fd > -1) { - comm_close(ftpState->data.fd); - ftpState->data.fd = -1; - } - cbdataFree(ftpState); -} - -static void -ftpLoginParser(const char *login, FtpStateData * ftpState, int escaped) -{ - char *s = NULL; - xstrncpy(ftpState->user, login, MAX_URL); - if ((s = strchr(ftpState->user, ':'))) { - *s = 0; - xstrncpy(ftpState->password, s + 1, MAX_URL); - if (escaped) - rfc1738_unescape(ftpState->password); - ftpState->password_url = 1; - } else { - xstrncpy(ftpState->password, null_string, MAX_URL); - } - if (escaped) - rfc1738_unescape(ftpState->user); - if (ftpState->user[0] || ftpState->password[0]) - return; - xstrncpy(ftpState->user, "anonymous", MAX_URL); - xstrncpy(ftpState->password, Config.Ftp.anon_user, MAX_URL); -} - -static void -ftpTimeout(int fd, void *data) -{ - FtpStateData *ftpState = data; - StoreEntry *entry = ftpState->entry; - debug(9, 4) ("ftpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); - if (SENT_PASV == ftpState->state && fd == ftpState->data.fd) { - /* stupid ftp.netscape.com */ - ftpState->fwd->flags.dont_retry = 0; - ftpState->fwd->flags.ftp_pasv_failed = 1; - debug(9, 1) ("ftpTimeout: timeout in SENT_PASV state\n"); - } - ftpFailed(ftpState, ERR_READ_TIMEOUT); - /* ftpFailed closes ctrl.fd and frees ftpState */ -} - -static void -ftpListingStart(FtpStateData * ftpState) -{ - StoreEntry *e = ftpState->entry; - wordlist *w; - char *dirup; - int i, j, k; - char *title; - storeBuffer(e); - storeAppendPrintf(e, "\n", - version_string); - storeAppendPrintf(e, "\n", mkrfc1123(squid_curtime)); - storeAppendPrintf(e, "\n"); - storeAppendPrintf(e, "\n"); - storeAppendPrintf(e, "FTP Directory: %s\n", - html_quote(strBuf(ftpState->title_url))); - storeAppendPrintf(e, "\n"); - storeAppendPrintf(e, "\n"); - if (ftpState->flags.need_base_href) - storeAppendPrintf(e, "\n", - html_quote(strBuf(ftpState->base_href))); - storeAppendPrintf(e, "\n"); - if (ftpState->cwd_message) { - storeAppendPrintf(e, "
\n");
-	for (w = ftpState->cwd_message; w; w = w->next)
-	    storeAppendPrintf(e, "%s\n", html_quote(w->key));
-	storeAppendPrintf(e, "
\n"); - storeAppendPrintf(e, "
\n"); - wordlistDestroy(&ftpState->cwd_message); - } - storeAppendPrintf(e, "

\n"); - storeAppendPrintf(e, "FTP Directory: "); - /* "ftp://" == 6 characters */ - assert(strLen(ftpState->title_url) >= 6); - title = html_quote(strBuf(ftpState->title_url)); - for (i = 6, j = 0; title[i]; j = i) { - storeAppendPrintf(e, ""); - for (k = j; k < i - 1; k++) - storeAppendPrintf(e, "%c", title[k]); - if (strBuf(ftpState->title_url)[k] != '/') - storeAppendPrintf(e, "%c", title[k++]); - storeAppendPrintf(e, ""); - if (k < i) - storeAppendPrintf(e, "%c", title[k++]); - if (i == j) { - /* Error guard, or "assert" */ - storeAppendPrintf(e, "ERROR: Failed to parse URL: %s\n", - html_quote(strBuf(ftpState->title_url))); - debug(9, 0) ("Failed to parse URL: %s\n", strBuf(ftpState->title_url)); - break; - } - } - storeAppendPrintf(e, "

\n"); - storeAppendPrintf(e, "
\n");
-    dirup = ftpHtmlifyListEntry("", ftpState);
-    storeAppend(e, dirup, strlen(dirup));
-    storeBufferFlush(e);
-    ftpState->flags.html_header_sent = 1;
-}
-
-static void
-ftpListingFinish(FtpStateData * ftpState)
-{
-    StoreEntry *e = ftpState->entry;
-    storeBuffer(e);
-    storeAppendPrintf(e, "
\n"); - if (ftpState->flags.listformat_unknown && !ftpState->flags.tried_nlst) { - storeAppendPrintf(e, "[As plain directory]\n"); - } else if (ftpState->typecode == 'D') { - storeAppendPrintf(e, "[As extended directory]\n"); - } - storeAppendPrintf(e, "
\n"); - storeAppendPrintf(e, "
\n"); - storeAppendPrintf(e, "Generated %s by %s (%s)\n", - mkrfc1123(squid_curtime), - getMyHostname(), - full_appname_string); - storeAppendPrintf(e, "
\n"); - storeBufferFlush(e); -} - -static const char *Month[] = -{ - "Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" -}; - -static int -is_month(const char *buf) -{ - int i; - for (i = 0; i < 12; i++) - if (!strcasecmp(buf, Month[i])) - return 1; - return 0; -} - - -static void -ftpListPartsFree(ftpListParts ** parts) -{ - safe_free((*parts)->date); - safe_free((*parts)->name); - safe_free((*parts)->showname); - safe_free((*parts)->link); - safe_free(*parts); -} - -#define MAX_TOKENS 64 - -static ftpListParts * -ftpListParseParts(const char *buf, struct _ftp_flags flags) -{ - ftpListParts *p = NULL; - char *t = NULL; - const char *ct = NULL; - char *tokens[MAX_TOKENS]; - int i; - int n_tokens; - static char tbuf[128]; - char *xbuf = NULL; - static int scan_ftp_initialized = 0; - static regex_t scan_ftp_integer; - static regex_t scan_ftp_time; - static regex_t scan_ftp_dostime; - static regex_t scan_ftp_dosdate; - - if (!scan_ftp_initialized) { - scan_ftp_initialized = 1; - regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB); - regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB); - regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB); - regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE); - } - if (buf == NULL) - return NULL; - if (*buf == '\0') - return NULL; - p = xcalloc(1, sizeof(ftpListParts)); - n_tokens = 0; - memset(tokens, 0, sizeof(tokens)); - xbuf = xstrdup(buf); - if (flags.tried_nlst) { - /* Machine readable format, one name per line */ - p->name = xbuf; - p->type = '\0'; - return p; - } - for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) - tokens[n_tokens++] = xstrdup(t); - xfree(xbuf); - /* locate the Month field */ - for (i = 3; i < n_tokens - 2; i++) { - char *size = tokens[i - 1]; - char *month = tokens[i]; - char *day = tokens[i + 1]; - char *year = tokens[i + 2]; - if (!is_month(month)) - continue; - if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0) - continue; - if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0) - continue; - if (regexec(&scan_ftp_time, day, 0, NULL, 0) != 0) /* Yr | hh:mm */ - continue; - snprintf(tbuf, 128, "%s %2s %5s", - month, day, year); - if (!strstr(buf, tbuf)) - snprintf(tbuf, 128, "%s %2s %-5s", - month, day, year); - if ((t = strstr(buf, tbuf))) { - p->type = *tokens[0]; - p->size = atoi(size); - p->date = xstrdup(tbuf); - if (flags.skip_whitespace) { - t += strlen(tbuf); - while (strchr(w_space, *t)) - t++; - } else { - /* XXX assumes a single space between date and filename - * suggested by: Nathan.Bailey@cc.monash.edu.au and - * Mike Battersby */ - t += strlen(tbuf) + 1; - } - p->name = xstrdup(t); - if ((t = strstr(p->name, " -> "))) { - *t = '\0'; - p->link = xstrdup(t + 4); - } - goto found; - } - break; - } - /* try it as a DOS listing, 04-05-70 09:33PM ... */ - if (n_tokens > 3 && - regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 && - regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) { - if (!strcasecmp(tokens[2], "")) { - p->type = 'd'; - } else { - p->type = '-'; - p->size = atoi(tokens[2]); - } - snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]); - p->date = xstrdup(tbuf); - if (p->type == 'd') { - /* Directory.. name begins with first printable after */ - ct = strstr(buf, tokens[2]); - ct += strlen(tokens[2]); - while (xisspace(*ct)) - ct++; - if (!*ct) - ct = NULL; - } else { - /* A file. Name begins after size, with a space in between */ - snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]); - ct = strstr(buf, tbuf); - if (ct) { - ct += strlen(tokens[2]) + 2; - } - } - p->name = xstrdup(ct ? ct : tokens[3]); - goto found; - } - /* Try EPLF format; carson@lehman.com */ - if (buf[0] == '+') { - ct = buf + 1; - p->type = 0; - while (ct && *ct) { - time_t t; - int l = strcspn(ct + 1, ","); - char *tmp; - if (l < 1) - goto blank; - switch (*ct) { - case '\t': - p->name = xstrndup(ct + 1, l + 1); - break; - case 's': - p->size = atoi(ct + 1); - break; - case 'm': - t = (time_t) strtol(ct + 1, &tmp, 0); - if (*tmp || (tmp == ct + 1)) - break; /* not a valid integer */ - p->date = xstrdup(ctime(&t)); - *(strstr(p->date, "\n")) = '\0'; - break; - case '/': - p->type = 'd'; - break; - case 'r': - p->type = '-'; - break; - case 'i': - break; - default: - break; - } - blank: - ct = strstr(ct, ","); - if (ct) { - ct++; - } - } - if (p->type == 0) { - p->type = '-'; - } - if (p->name) - goto found; - else - safe_free(p->date); - } - found: - for (i = 0; i < n_tokens; i++) - xfree(tokens[i]); - if (!p->name) - ftpListPartsFree(&p); /* cleanup */ - return p; -} - -static const char * -dots_fill(size_t len) -{ - static char buf[256]; - int i = 0; - if (len > Config.Ftp.list_width) { - memset(buf, ' ', 256); - buf[0] = '\n'; - buf[Config.Ftp.list_width + 4] = '\0'; - return buf; - } - for (i = (int) len; i < Config.Ftp.list_width; i++) - buf[i - len] = (i % 2) ? '.' : ' '; - buf[i - len] = '\0'; - return buf; -} - -static char * -ftpHtmlifyListEntry(const char *line, FtpStateData * ftpState) -{ - LOCAL_ARRAY(char, icon, 2048); - LOCAL_ARRAY(char, href, 2048 + 40); - LOCAL_ARRAY(char, text, 2048); - LOCAL_ARRAY(char, size, 2048); - LOCAL_ARRAY(char, chdir, 2048 + 40); - LOCAL_ARRAY(char, view, 2048 + 40); - LOCAL_ARRAY(char, download, 2048 + 40); - LOCAL_ARRAY(char, link, 2048 + 40); - LOCAL_ARRAY(char, html, 8192); - size_t width = Config.Ftp.list_width; - ftpListParts *parts; - *icon = *href = *text = *size = *chdir = *view = *download = *link = *html = '\0'; - if ((int) strlen(line) > 1024) { - snprintf(html, 8192, "%s\n", line); - return html; - } - /* Handle builtin */ - if (strcmp(line, "") == 0) { - /* {icon} {text} {link} */ - snprintf(icon, 2048, "\"%-6s\"", - mimeGetIconURL("internal-dirup"), - "[DIRUP]"); - if (!ftpState->flags.no_dotdot && !ftpState->flags.root_dir) { - /* Normal directory */ - strcpy(href, "../"); - strcpy(text, "Parent Directory"); - } else if (!ftpState->flags.no_dotdot && ftpState->flags.root_dir) { - /* "Top level" directory */ - strcpy(href, "%2e%2e/"); - strcpy(text, "Parent Directory"); - snprintf(link, 2048, "(%s)", - "%2f/", - "Root Directory"); - } else if (ftpState->flags.no_dotdot && !ftpState->flags.root_dir) { - /* Normal directory where last component is / or .. */ - strcpy(href, "%2e%2e/"); - strcpy(text, "Parent Directory"); - snprintf(link, 2048, "(%s)", - "../", - "Back"); - } else { /* NO_DOTDOT && ROOT_DIR */ - /* "UNIX Root" directory */ - strcpy(href, "../"); - strcpy(text, "Home Directory"); - } - snprintf(html, 8192, "%s %s %s\n", - href, icon, href, text, link); - return html; - } - if ((parts = ftpListParseParts(line, ftpState->flags)) == NULL) { - const char *p; - snprintf(html, 8192, "%s\n", line); - for (p = line; *p && xisspace(*p); p++); - if (*p && !xisspace(*p)) - ftpState->flags.listformat_unknown = 1; - return html; - } - if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) { - *html = '\0'; - ftpListPartsFree(&parts); - return html; - } - parts->size += 1023; - parts->size >>= 10; - parts->showname = xstrdup(parts->name); - if (!Config.Ftp.list_wrap) { - if (strlen(parts->showname) > width - 1) { - *(parts->showname + width - 1) = '>'; - *(parts->showname + width - 0) = '\0'; - } - } - /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n */ - xstrncpy(href, rfc1738_escape_part(parts->name), 2048); - xstrncpy(text, parts->showname, 2048); - switch (parts->type) { - case 'd': - snprintf(icon, 2048, "\"%-6s\"", - mimeGetIconURL("internal-dir"), - "[DIR]"); - strcat(href, "/"); /* margin is allocated above */ - break; - case 'l': - snprintf(icon, 2048, "\"%-6s\"", - mimeGetIconURL("internal-link"), - "[LINK]"); - /* sometimes there is an 'l' flag, but no "->" link */ - if (parts->link) { - char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link))); - snprintf(link, 2048, " -> %s", - link2, - html_quote(parts->link)); - safe_free(link2); - } - break; - case '\0': - snprintf(icon, 2048, "\"%-6s\"", - mimeGetIconURL(parts->name), - "[UNKNOWN]"); - snprintf(chdir, 2048, " ", - rfc1738_escape_part(parts->name), - mimeGetIconURL("internal-dir")); - break; - case '-': - default: - snprintf(icon, 2048, "\"%-6s\"", - mimeGetIconURL(parts->name), - "[FILE]"); - snprintf(size, 2048, " %6dk", parts->size); - break; - } - if (parts->type != 'd') { - if (mimeGetViewOption(parts->name)) { - snprintf(view, 2048, " ", - href, mimeGetIconURL("internal-view")); - } - if (mimeGetDownloadOption(parts->name)) { - snprintf(download, 2048, " ", - href, mimeGetIconURL("internal-download")); - } - } - /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n */ - if (parts->type != '\0') { - snprintf(html, 8192, "%s %s%s " - "%s%8s%s%s%s%s\n", - href, icon, href, html_quote(text), dots_fill(strlen(text)), - parts->date, size, chdir, view, download, link); - } else { - /* Plain listing. {icon} {text} ... {chdir}{view}{download} */ - snprintf(html, 8192, "%s %s%s " - "%s%s%s%s\n", - href, icon, href, html_quote(text), dots_fill(strlen(text)), - chdir, view, download, link); - } - ftpListPartsFree(&parts); - return html; -} - -static void -ftpParseListing(FtpStateData * ftpState) -{ - char *buf = ftpState->data.buf; - char *sbuf; /* NULL-terminated copy of buf */ - char *end; - char *line; - char *s; - char *t; - size_t linelen; - size_t usable; - StoreEntry *e = ftpState->entry; - int len = ftpState->data.offset; - /* - * We need a NULL-terminated buffer for scanning, ick - */ - sbuf = xmalloc(len + 1); - xstrncpy(sbuf, buf, len + 1); - end = sbuf + len - 1; - while (*end != '\r' && *end != '\n' && end > sbuf) - end--; - usable = end - sbuf; - debug(9, 3) ("ftpParseListing: usable = %d\n", (int) usable); - if (usable == 0) { - debug(9, 3) ("ftpParseListing: didn't find end for %s\n", storeUrl(e)); - xfree(sbuf); - return; - } - debug(9, 3) ("ftpParseListing: %d bytes to play with\n", len); - line = memAllocate(MEM_4K_BUF); - end++; - storeBuffer(e); - s = sbuf; - s += strspn(s, crlf); - for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { - debug(9, 3) ("ftpParseListing: s = {%s}\n", s); - linelen = strcspn(s, crlf) + 1; - if (linelen < 2) - break; - if (linelen > 4096) - linelen = 4096; - xstrncpy(line, s, linelen); - debug(9, 7) ("ftpParseListing: {%s}\n", line); - if (!strncmp(line, "total", 5)) - continue; - t = ftpHtmlifyListEntry(line, ftpState); - assert(t != NULL); - storeAppend(e, t, strlen(t)); - } - storeBufferFlush(e); - assert(usable <= len); - if (usable < len) { - /* must copy partial line to beginning of buf */ - linelen = len - usable; - if (linelen > 4096) - linelen = 4096; - xstrncpy(line, end, linelen); - xstrncpy(ftpState->data.buf, line, ftpState->data.size); - ftpState->data.offset = strlen(ftpState->data.buf); - } - memFree(line, MEM_4K_BUF); - xfree(sbuf); -} - -static void -ftpDataComplete(FtpStateData * ftpState) -{ - debug(9, 3) ("ftpDataComplete\n"); - /* Connection closed; transfer done. */ - if (ftpState->data.fd > -1) { - /* - * close data socket so it does not occupy resources while - * we wait - */ - comm_close(ftpState->data.fd); - ftpState->data.fd = -1; - } - /* expect the "transfer complete" message on the control socket */ - ftpScheduleReadControlReply(ftpState, 1); -} - -static void -ftpDataRead(int fd, void *data) -{ - FtpStateData *ftpState = data; - int len; - int j; - int bin; - StoreEntry *entry = ftpState->entry; - size_t read_sz; -#if DELAY_POOLS - MemObject *mem = entry->mem_obj; - delay_id delay_id = delayMostBytesAllowed(mem); -#endif - assert(fd == ftpState->data.fd); - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - comm_close(ftpState->ctrl.fd); - return; - } - errno = 0; - read_sz = ftpState->data.size - ftpState->data.offset; -#if DELAY_POOLS - read_sz = delayBytesWanted(delay_id, 1, read_sz); -#endif - memset(ftpState->data.buf + ftpState->data.offset, '\0', read_sz); - statCounter.syscalls.sock.reads++; - len = FD_READ_METHOD(fd, ftpState->data.buf + ftpState->data.offset, read_sz); - if (len > 0) { - fd_bytes(fd, len, FD_READ); -#if DELAY_POOLS - delayBytesIn(delay_id, len); -#endif - kb_incr(&statCounter.server.all.kbytes_in, len); - kb_incr(&statCounter.server.ftp.kbytes_in, len); - ftpState->data.offset += len; - } - debug(9, 5) ("ftpDataRead: FD %d, Read %d bytes\n", fd, len); - if (len > 0) { - IOStats.Ftp.reads++; - for (j = len - 1, bin = 0; j; bin++) - j >>= 1; - IOStats.Ftp.read_hist[bin]++; - } - if (ftpState->flags.isdir && !ftpState->flags.html_header_sent && len >= 0) { - ftpListingStart(ftpState); - } - if (len < 0) { - debug(50, ignoreErrno(errno) ? 3 : 1) ("ftpDataRead: read error: %s\n", xstrerror()); - if (ignoreErrno(errno)) { - commSetSelect(fd, - COMM_SELECT_READ, - ftpDataRead, - data, - Config.Timeout.read); - } else { - ftpFailed(ftpState, ERR_READ_ERROR); - /* ftpFailed closes ctrl.fd and frees ftpState */ - return; - } - } else if (len == 0) { - ftpDataComplete(ftpState); - } else { - if (ftpState->flags.isdir) { - ftpParseListing(ftpState); - } else { - storeAppend(entry, ftpState->data.buf, len); - ftpState->data.offset = 0; - } - commSetSelect(fd, - COMM_SELECT_READ, - ftpDataRead, - data, - Config.Timeout.read); - } -} - -/* - * ftpCheckAuth - * - * Return 1 if we have everything needed to complete this request. - * Return 0 if something is missing. - */ -static int -ftpCheckAuth(FtpStateData * ftpState, const HttpHeader * req_hdr) -{ - char *orig_user; - const char *auth; - ftpLoginParser(ftpState->request->login, ftpState, FTP_LOGIN_ESCAPED); - if (!ftpState->user[0]) - return 1; /* no name */ - if (ftpState->password_url || ftpState->password[0]) - return 1; /* passwd provided in URL */ - /* URL has name, but no passwd */ - if (!(auth = httpHeaderGetAuth(req_hdr, HDR_AUTHORIZATION, "Basic"))) - return 0; /* need auth header */ - ftpState->flags.authenticated = 1; - orig_user = xstrdup(ftpState->user); - ftpLoginParser(auth, ftpState, FTP_LOGIN_NOT_ESCAPED); - if (strcmp(orig_user, ftpState->user) == 0) { - xfree(orig_user); - return 1; /* same username */ - } - xstrncpy(ftpState->user, orig_user, sizeof(ftpState->user)); - xfree(orig_user); - return 0; /* different username */ -} - -static void -ftpCheckUrlpath(FtpStateData * ftpState) -{ - request_t *request = ftpState->request; - int l; - const char *t; - if ((t = strRChr(request->urlpath, ';')) != NULL) { - if (strncasecmp(t + 1, "type=", 5) == 0) { - ftpState->typecode = (char) toupper((int) *(t + 6)); - strCutPtr(request->urlpath, t); - } - } - l = strLen(request->urlpath); - ftpState->flags.need_base_href = 1; - /* check for null path */ - if (!l) { - ftpState->flags.isdir = 1; - ftpState->flags.root_dir = 1; - } else if (!strCmp(request->urlpath, "/%2f/")) { - /* UNIX root directory */ - ftpState->flags.need_base_href = 0; - ftpState->flags.isdir = 1; - ftpState->flags.root_dir = 1; - } else if ((l >= 1) && (*(strBuf(request->urlpath) + l - 1) == '/')) { - /* Directory URL, ending in / */ - ftpState->flags.isdir = 1; - ftpState->flags.need_base_href = 0; - if (l == 1) - ftpState->flags.root_dir = 1; - } -} - -static void -ftpBuildTitleUrl(FtpStateData * ftpState) -{ - request_t *request = ftpState->request; - - stringReset(&ftpState->title_url, "ftp://"); - if (strcmp(ftpState->user, "anonymous")) { - strCat(ftpState->title_url, ftpState->user); - strCat(ftpState->title_url, "@"); - } - strCat(ftpState->title_url, request->host); - if (request->port != urlDefaultPort(PROTO_FTP)) { - strCat(ftpState->title_url, ":"); - strCat(ftpState->title_url, xitoa(request->port)); - } - strCat(ftpState->title_url, strBuf(request->urlpath)); - - stringReset(&ftpState->base_href, "ftp://"); - if (strcmp(ftpState->user, "anonymous") != 0) { - strCat(ftpState->base_href, rfc1738_escape_part(ftpState->user)); - if (ftpState->password_url) { - strCat(ftpState->base_href, ":"); - strCat(ftpState->base_href, rfc1738_escape_part(ftpState->password)); - } - strCat(ftpState->base_href, "@"); - } - strCat(ftpState->base_href, request->host); - if (request->port != urlDefaultPort(PROTO_FTP)) { - strCat(ftpState->base_href, ":"); - strCat(ftpState->base_href, xitoa(request->port)); - } - strCat(ftpState->base_href, strBuf(request->urlpath)); - strCat(ftpState->base_href, "/"); -} - -CBDATA_TYPE(FtpStateData); -void -ftpStart(FwdState * fwd) -{ - request_t *request = fwd->request; - StoreEntry *entry = fwd->entry; - int fd = fwd->server_fd; - LOCAL_ARRAY(char, realm, 8192); - const char *url = storeUrl(entry); - FtpStateData *ftpState; - HttpReply *reply; - - CBDATA_INIT_TYPE(FtpStateData); - ftpState = cbdataAlloc(FtpStateData); - debug(9, 3) ("ftpStart: '%s'\n", url); - statCounter.server.all.requests++; - statCounter.server.ftp.requests++; - storeLockObject(entry); - ftpState->entry = entry; - ftpState->request = requestLink(request); - ftpState->ctrl.fd = fd; - ftpState->data.fd = -1; - ftpState->size = -1; - ftpState->mdtm = -1; - if (!Config.Ftp.passive) - ftpState->flags.rest_supported = 0; - else if (fwd->flags.ftp_pasv_failed) - ftpState->flags.pasv_supported = 0; - else - ftpState->flags.pasv_supported = 1; - ftpState->flags.rest_supported = 1; - ftpState->fwd = fwd; - comm_add_close_handler(fd, ftpStateFree, ftpState); - if (ftpState->request->method == METHOD_PUT) - ftpState->flags.put = 1; - if (!ftpCheckAuth(ftpState, &request->header)) { - /* This request is not fully authenticated */ - if (request->port == 21) { - snprintf(realm, 8192, "ftp %s", ftpState->user); - } else { - snprintf(realm, 8192, "ftp %s port %d", - ftpState->user, request->port); - } - /* create reply */ - reply = entry->mem_obj->reply; - assert(reply != NULL); - /* create appropriate reply */ - ftpAuthRequired(reply, request, realm); - httpReplySwapOut(reply, entry); - fwdComplete(ftpState->fwd); - comm_close(fd); - return; - } - ftpCheckUrlpath(ftpState); - ftpBuildTitleUrl(ftpState); - debug(9, 5) ("ftpStart: host=%s, path=%s, user=%s, passwd=%s\n", - ftpState->request->host, strBuf(ftpState->request->urlpath), - ftpState->user, ftpState->password); - ftpState->state = BEGIN; - ftpState->ctrl.last_command = xstrdup("Connect to server"); - ftpState->ctrl.buf = memAllocBuf(4096, &ftpState->ctrl.size); - ftpState->ctrl.offset = 0; - ftpState->data.buf = memAllocBuf(SQUID_TCP_SO_RCVBUF, &ftpState->data.size); - ftpScheduleReadControlReply(ftpState, 0); -} - -/* ====================================================================== */ - -static void -ftpWriteCommand(const char *buf, FtpStateData * ftpState) -{ - debug(9, 5) ("ftpWriteCommand: %s\n", buf); - safe_free(ftpState->ctrl.last_command); - safe_free(ftpState->ctrl.last_reply); - ftpState->ctrl.last_command = xstrdup(buf); - comm_write(ftpState->ctrl.fd, - xstrdup(buf), - strlen(buf), - ftpWriteCommandCallback, - ftpState, - xfree); - ftpScheduleReadControlReply(ftpState, 0); -} - -static void -ftpWriteCommandCallback(int fd, char *bufnotused, size_t size, comm_err_t errflag, void *data) -{ - FtpStateData *ftpState = data; - debug(9, 7) ("ftpWriteCommandCallback: wrote %d bytes\n", (int) size); - if (size > 0) { - fd_bytes(fd, size, FD_WRITE); - kb_incr(&statCounter.server.all.kbytes_out, size); - kb_incr(&statCounter.server.ftp.kbytes_out, size); - } - if (errflag == COMM_ERR_CLOSING) - return; - if (errflag) { - debug(9, 1) ("ftpWriteCommandCallback: FD %d: %s\n", fd, xstrerror()); - ftpFailed(ftpState, ERR_WRITE_ERROR); - /* ftpFailed closes ctrl.fd and frees ftpState */ - return; - } -} - -static wordlist * -ftpParseControlReply(char *buf, size_t len, int *codep, int *used) -{ - char *s; - char *sbuf; - char *end; - int usable; - int complete = 0; - wordlist *head = NULL; - wordlist *list; - wordlist **tail = &head; - off_t offset; - size_t linelen; - int code = -1; - debug(9, 5) ("ftpParseControlReply\n"); - /* - * We need a NULL-terminated buffer for scanning, ick - */ - sbuf = xmalloc(len + 1); - xstrncpy(sbuf, buf, len + 1); - end = sbuf + len - 1; - while (*end != '\r' && *end != '\n' && end > sbuf) - end--; - usable = end - sbuf; - debug(9, 3) ("ftpParseControlReply: usable = %d\n", usable); - if (usable == 0) { - debug(9, 3) ("ftpParseControlReply: didn't find end of line\n"); - safe_free(sbuf); - return NULL; - } - debug(9, 3) ("ftpParseControlReply: %d bytes to play with\n", (int) len); - end++; - s = sbuf; - s += strspn(s, crlf); - for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { - if (complete) - break; - debug(9, 3) ("ftpParseControlReply: s = {%s}\n", s); - linelen = strcspn(s, crlf) + 1; - if (linelen < 2) - break; - if (linelen > 3) - complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' '); - if (complete) - code = atoi(s); - offset = 0; - if (linelen > 3) - if (*s >= '0' && *s <= '9' && (*(s + 3) == '-' || *(s + 3) == ' ')) - offset = 4; - list = memAllocate(MEM_WORDLIST); - list->key = xmalloc(linelen - offset); - xstrncpy(list->key, s + offset, linelen - offset); - debug(9, 7) ("%d %s\n", code, list->key); - *tail = list; - tail = &list->next; - } - *used = (int) (s - sbuf); - safe_free(sbuf); - if (!complete) - wordlistDestroy(&head); - if (codep) - *codep = code; - return head; -} - -static void -ftpScheduleReadControlReply(FtpStateData * ftpState, int buffered_ok) -{ - debug(9, 3) ("ftpScheduleReadControlReply: FD %d\n", ftpState->ctrl.fd); - if (buffered_ok && ftpState->ctrl.offset > 0) { - /* We've already read some reply data */ - ftpHandleControlReply(ftpState); - } else { - commSetSelect(ftpState->ctrl.fd, - COMM_SELECT_READ, - ftpReadControlReply, - ftpState, - Config.Timeout.read); - /* - * Cancel the timeout on the Data socket (if any) and - * establish one on the control socket. - */ - if (ftpState->data.fd > -1) - commSetTimeout(ftpState->data.fd, -1, NULL, NULL); - commSetTimeout(ftpState->ctrl.fd, Config.Timeout.read, ftpTimeout, - ftpState); - } -} - -static void -ftpReadControlReply(int fd, void *data) -{ - FtpStateData *ftpState = data; - StoreEntry *entry = ftpState->entry; - int len; - debug(9, 5) ("ftpReadControlReply\n"); - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - comm_close(ftpState->ctrl.fd); - return; - } - assert(ftpState->ctrl.offset < ftpState->ctrl.size); - statCounter.syscalls.sock.reads++; - len = FD_READ_METHOD(fd, - ftpState->ctrl.buf + ftpState->ctrl.offset, - ftpState->ctrl.size - ftpState->ctrl.offset); - if (len > 0) { - fd_bytes(fd, len, FD_READ); - kb_incr(&statCounter.server.all.kbytes_in, len); - kb_incr(&statCounter.server.ftp.kbytes_in, len); - } - debug(9, 5) ("ftpReadControlReply: FD %d, Read %d bytes\n", fd, len); - if (len < 0) { - debug(50, ignoreErrno(errno) ? 3 : 1) ("ftpReadControlReply: read error: %s\n", xstrerror()); - if (ignoreErrno(errno)) { - ftpScheduleReadControlReply(ftpState, 0); - } else { - ftpFailed(ftpState, ERR_READ_ERROR); - /* ftpFailed closes ctrl.fd and frees ftpState */ - return; - } - return; - } - if (len == 0) { - if (entry->store_status == STORE_PENDING) { - ftpFailed(ftpState, ERR_FTP_FAILURE); - /* ftpFailed closes ctrl.fd and frees ftpState */ - return; - } - comm_close(ftpState->ctrl.fd); - return; - } - len += ftpState->ctrl.offset; - ftpState->ctrl.offset = len; - assert(len <= ftpState->ctrl.size); - ftpHandleControlReply(ftpState); -} - -static void -ftpHandleControlReply(FtpStateData * ftpState) -{ - wordlist **W; - int bytes_used = 0; - wordlistDestroy(&ftpState->ctrl.message); - ftpState->ctrl.message = ftpParseControlReply(ftpState->ctrl.buf, - ftpState->ctrl.offset, &ftpState->ctrl.replycode, &bytes_used); - if (ftpState->ctrl.message == NULL) { - /* didn't get complete reply yet */ - if (ftpState->ctrl.offset == ftpState->ctrl.size) { - ftpState->ctrl.buf = memReallocBuf(ftpState->ctrl.buf, ftpState->ctrl.size << 1, &ftpState->ctrl.size); - } - ftpScheduleReadControlReply(ftpState, 0); - return; - } else if (ftpState->ctrl.offset == bytes_used) { - /* used it all up */ - ftpState->ctrl.offset = 0; - } else { - /* Got some data past the complete reply */ - assert(bytes_used < ftpState->ctrl.offset); - ftpState->ctrl.offset -= bytes_used; - xmemmove(ftpState->ctrl.buf, ftpState->ctrl.buf + bytes_used, - ftpState->ctrl.offset); - } - /* Move the last line of the reply message to ctrl.last_reply */ - for (W = &ftpState->ctrl.message; (*W)->next; W = &(*W)->next); - safe_free(ftpState->ctrl.last_reply); - ftpState->ctrl.last_reply = xstrdup((*W)->key); - wordlistDestroy(W); - /* Copy the rest of the message to cwd_message to be printed in - * error messages - */ - wordlistAddWl(&ftpState->cwd_message, ftpState->ctrl.message); - debug(9, 8) ("ftpHandleControlReply: state=%d, code=%d\n", ftpState->state, - ftpState->ctrl.replycode); - FTP_SM_FUNCS[ftpState->state] (ftpState); -} - -/* ====================================================================== */ - -static void -ftpReadWelcome(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("ftpReadWelcome\n"); - if (ftpState->flags.pasv_only) - ftpState->login_att++; - /* Dont retry if the FTP server accepted the connection */ - ftpState->fwd->flags.dont_retry = 1; - if (code == 220) { - if (ftpState->ctrl.message) { - if (strstr(ftpState->ctrl.message->key, "NetWare")) - ftpState->flags.skip_whitespace = 1; - } - ftpSendUser(ftpState); - } else if (code == 120) { - if (NULL != ftpState->ctrl.message) - debug(9, 3) ("FTP server is busy: %s\n", - ftpState->ctrl.message->key); - return; - } else { - ftpFail(ftpState); - } -} - -static void -ftpSendUser(FtpStateData * ftpState) -{ - if (ftpState->proxy_host != NULL) - snprintf(cbuf, 1024, "USER %s@%s\r\n", - ftpState->user, - ftpState->request->host); - else - snprintf(cbuf, 1024, "USER %s\r\n", ftpState->user); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_USER; -} - -static void -ftpReadUser(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("ftpReadUser\n"); - if (code == 230) { - ftpReadPass(ftpState); - } else if (code == 331) { - ftpSendPass(ftpState); - } else { - ftpFail(ftpState); - } -} - -static void -ftpSendPass(FtpStateData * ftpState) -{ - snprintf(cbuf, 1024, "PASS %s\r\n", ftpState->password); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_PASS; -} - -static void -ftpReadPass(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("ftpReadPass\n"); - if (code == 230) { - ftpSendType(ftpState); - } else { - ftpFail(ftpState); - } -} - -static void -ftpSendType(FtpStateData * ftpState) -{ - const char *t; - const char *filename; - char mode; - /* - * Ref section 3.2.2 of RFC 1738 - */ - mode = ftpState->typecode; - switch (mode) { - case 'D': - mode = 'A'; - break; - case 'A': - case 'I': - break; - default: - if (ftpState->flags.isdir) { - mode = 'A'; - } else { - t = strRChr(ftpState->request->urlpath, '/'); - filename = t ? t + 1 : strBuf(ftpState->request->urlpath); - mode = mimeGetTransferMode(filename); - } - break; - } - if (mode == 'I') - ftpState->flags.binary = 1; - else - ftpState->flags.binary = 0; - snprintf(cbuf, 1024, "TYPE %c\r\n", mode); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_TYPE; -} - -static void -ftpReadType(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - char *path; - char *d, *p; - debug(9, 3) ("This is ftpReadType\n"); - if (code == 200) { - p = path = xstrdup(strBuf(ftpState->request->urlpath)); - if (*p == '/') - p++; - while (*p) { - d = p; - p += strcspn(p, "/"); - if (*p) - *p++ = '\0'; - rfc1738_unescape(d); - wordlistAdd(&ftpState->pathcomps, d); - } - xfree(path); - if (ftpState->pathcomps) - ftpTraverseDirectory(ftpState); - else - ftpListDir(ftpState); - } else { - ftpFail(ftpState); - } -} - -static void -ftpTraverseDirectory(FtpStateData * ftpState) -{ - wordlist *w; - debug(9, 4) ("ftpTraverseDirectory %s\n", - ftpState->filepath ? ftpState->filepath : ""); - - safe_free(ftpState->filepath); - /* Done? */ - if (ftpState->pathcomps == NULL) { - debug(9, 3) ("the final component was a directory\n"); - ftpListDir(ftpState); - return; - } - /* Go to next path component */ - w = ftpState->pathcomps; - ftpState->filepath = w->key; - ftpState->pathcomps = w->next; - memFree(w, MEM_WORDLIST); - /* Check if we are to CWD or RETR */ - if (ftpState->pathcomps != NULL || ftpState->flags.isdir) { - ftpSendCwd(ftpState); - } else { - debug(9, 3) ("final component is probably a file\n"); - ftpGetFile(ftpState); - return; - } -} - -static void -ftpSendCwd(FtpStateData * ftpState) -{ - char *path = ftpState->filepath; - debug(9, 3) ("ftpSendCwd\n"); - if (!strcmp(path, "..") || !strcmp(path, "/")) { - ftpState->flags.no_dotdot = 1; - } else { - ftpState->flags.no_dotdot = 0; - } - if (*path) - snprintf(cbuf, 1024, "CWD %s\r\n", path); - else - snprintf(cbuf, 1024, "CWD\r\n"); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_CWD; -} - -static void -ftpReadCwd(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadCwd\n"); - if (code >= 200 && code < 300) { - /* CWD OK */ - ftpUnhack(ftpState); - /* Reset cwd_message to only include the last message */ - if (ftpState->cwd_message) - wordlistDestroy(&ftpState->cwd_message); - ftpState->cwd_message = ftpState->ctrl.message; - ftpState->ctrl.message = NULL; - /* Continue to traverse the path */ - ftpTraverseDirectory(ftpState); - } else { - /* CWD FAILED */ - if (!ftpState->flags.put) - ftpFail(ftpState); - else - ftpSendMkdir(ftpState); - } -} - -static void -ftpSendMkdir(FtpStateData * ftpState) -{ - char *path = ftpState->filepath; - debug(9, 3) ("ftpSendMkdir: with path=%s\n", path); - snprintf(cbuf, 1024, "MKD %s\r\n", path); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_MKDIR; -} - -static void -ftpReadMkdir(FtpStateData * ftpState) -{ - char *path = ftpState->filepath; - int code = ftpState->ctrl.replycode; - - debug(9, 3) ("ftpReadMkdir: path %s, code %d\n", path, code); - if (code == 257) { /* success */ - ftpSendCwd(ftpState); - } else if (code == 550) { /* dir exists */ - if (ftpState->flags.put_mkdir) { - ftpState->flags.put_mkdir = 1; - ftpSendCwd(ftpState); - } else - ftpSendReply(ftpState); - } else - ftpSendReply(ftpState); -} - -static void -ftpGetFile(FtpStateData * ftpState) -{ - assert(*ftpState->filepath != '\0'); - ftpState->flags.isdir = 0; - ftpSendMdtm(ftpState); -} - -static void -ftpListDir(FtpStateData * ftpState) -{ - if (!ftpState->flags.isdir) { - debug(9, 3) ("Directory path did not end in /\n"); - strCat(ftpState->title_url, "/"); - ftpState->flags.isdir = 1; - ftpState->flags.need_base_href = 1; - } - ftpSendPasv(ftpState); -} - -static void -ftpSendMdtm(FtpStateData * ftpState) -{ - assert(*ftpState->filepath != '\0'); - snprintf(cbuf, 1024, "MDTM %s\r\n", ftpState->filepath); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_MDTM; -} - -static void -ftpReadMdtm(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadMdtm\n"); - if (code == 213) { - ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply); - ftpUnhack(ftpState); - } else if (code < 0) { - ftpFail(ftpState); - } - ftpSendSize(ftpState); -} - -static void -ftpSendSize(FtpStateData * ftpState) -{ - /* Only send SIZE for binary transfers. The returned size - * is useless on ASCII transfers */ - if (ftpState->flags.binary) { - assert(ftpState->filepath != NULL); - assert(*ftpState->filepath != '\0'); - snprintf(cbuf, 1024, "SIZE %s\r\n", ftpState->filepath); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_SIZE; - } else - /* Skip to next state no non-binary transfers */ - ftpSendPasv(ftpState); -} - -static void -ftpReadSize(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadSize\n"); - if (code == 213) { - ftpUnhack(ftpState); - ftpState->size = atoi(ftpState->ctrl.last_reply); - if (ftpState->size == 0) { - debug(9, 2) ("ftpReadSize: SIZE reported %s on %s\n", - ftpState->ctrl.last_reply, - strBuf(ftpState->title_url)); - ftpState->size = -1; - } - } else if (code < 0) { - ftpFail(ftpState); - } - ftpSendPasv(ftpState); -} - -static void -ftpSendPasv(FtpStateData * ftpState) -{ - int fd; - struct sockaddr_in addr; - socklen_t addr_len; - if (ftpState->request->method == METHOD_HEAD) { - /* Terminate here for HEAD requests */ - ftpAppendSuccessHeader(ftpState); - storeTimestampsSet(ftpState->entry); - /* - * On rare occasions I'm seeing the entry get aborted after - * ftpReadControlReply() and before here, probably when - * trying to write to the client. - */ - if (!EBIT_TEST(ftpState->entry->flags, ENTRY_ABORTED)) - fwdComplete(ftpState->fwd); - ftpSendQuit(ftpState); - return; - } - if (ftpState->data.fd >= 0) { - if (!ftpState->flags.datachannel_hack) { - /* We are already connected, reuse this connection. */ - ftpRestOrList(ftpState); - return; - } else { - /* Close old connection */ - comm_close(ftpState->data.fd); - ftpState->data.fd = -1; - } - } - if (!ftpState->flags.pasv_supported) { - ftpSendPort(ftpState); - return; - } - addr_len = sizeof(addr); - if (getsockname(ftpState->ctrl.fd, (struct sockaddr *) &addr, &addr_len)) { - debug(9, 0) ("ftpSendPasv: getsockname(%d,..): %s\n", - ftpState->ctrl.fd, xstrerror()); - ftpFail(ftpState); - return; - } - /* Open data channel with the same local address as control channel */ - fd = comm_open(SOCK_STREAM, - 0, - addr.sin_addr, - 0, - COMM_NONBLOCKING, - storeUrl(ftpState->entry)); - debug(9, 3) ("ftpSendPasv: Unconnected data socket created on FD %d\n", fd); - if (fd < 0) { - ftpFail(ftpState); - return; - } - /* - * No comm_add_close_handler() here. If we have both ctrl and - * data FD's call ftpStateFree() upon close, then we have - * to delete the close handler which did NOT get called - * to prevent ftpStateFree() getting called twice. - * Instead we'll always call comm_close() on the ctrl FD. - */ - ftpState->data.fd = fd; - snprintf(cbuf, 1024, "PASV\r\n"); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_PASV; - /* - * ugly hack for ftp servers like ftp.netscape.com that sometimes - * dont acknowledge PORT commands. - */ - commSetTimeout(ftpState->data.fd, 15, ftpTimeout, ftpState); -} - -static void -ftpReadPasv(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - int h1, h2, h3, h4; - int p1, p2; - int n; - u_short port; - int fd = ftpState->data.fd; - char *buf; - LOCAL_ARRAY(char, ipaddr, 1024); - debug(9, 3) ("This is ftpReadPasv\n"); - if (code != 227) { - debug(9, 3) ("PASV not supported by remote end\n"); - ftpSendPort(ftpState); - return; - } - /* 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). */ - /* ANSI sez [^0-9] is undefined, it breaks on Watcom cc */ - debug(9, 5) ("scanning: %s\n", ftpState->ctrl.last_reply); - buf = strstr(ftpState->ctrl.last_reply, "("); - if (!buf) { - debug(9, 1) ("Unsafe PASV reply from %s: '%s'\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); - ftpSendPort(ftpState); - return; - } - buf++; /* skip ( */ - n = sscanf(buf, "%d,%d,%d,%d,%d,%d", &h1, &h2, &h3, &h4, &p1, &p2); - if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) { - debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); - ftpSendPort(ftpState); - return; - } - snprintf(ipaddr, 1024, "%d.%d.%d.%d", h1, h2, h3, h4); - if (!safe_inet_addr(ipaddr, NULL)) { - debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); - ftpSendPort(ftpState); - return; - } - port = ((p1 << 8) + p2); - if (0 == port) { - debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); - ftpSendPort(ftpState); - return; - } - if (Config.Ftp.sanitycheck) { - if (strcmp(fd_table[ftpState->ctrl.fd].ipaddr, ipaddr) != 0) { - debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); - ftpSendPort(ftpState); - return; - } - if (port < 1024) { - debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); - ftpSendPort(ftpState); - return; - } - } - debug(9, 5) ("ftpReadPasv: connecting to %s, port %d\n", ipaddr, port); - ftpState->data.port = port; - ftpState->data.host = xstrdup(ipaddr); - safe_free(ftpState->ctrl.last_command); - safe_free(ftpState->ctrl.last_reply); - ftpState->ctrl.last_command = xstrdup("Connect to server data port"); - commConnectStart(fd, ipaddr, port, ftpPasvCallback, ftpState); -} - -static void -ftpPasvCallback(int fd, int status, void *data) -{ - FtpStateData *ftpState = data; - debug(9, 3) ("ftpPasvCallback\n"); - if (status != COMM_OK) { - debug(9, 2) ("ftpPasvCallback: failed to connect. Retrying without PASV.\n"); - ftpState->fwd->flags.dont_retry = 0; /* this is a retryable error */ - ftpState->fwd->flags.ftp_pasv_failed = 1; - ftpFailed(ftpState, ERR_NONE); - /* ftpFailed closes ctrl.fd and frees ftpState */ - return; - } - ftpRestOrList(ftpState); -} - -static int -ftpOpenListenSocket(FtpStateData * ftpState, int fallback) -{ - int fd; - struct sockaddr_in addr; - socklen_t addr_len; - int on = 1; - u_short port = 0; - /* - * Tear down any old data connection if any. We are about to - * establish a new one. - */ - if (ftpState->data.fd > 0) { - comm_close(ftpState->data.fd); - ftpState->data.fd = -1; - } - /* - * Set up a listen socket on the same local address as the - * control connection. - */ - addr_len = sizeof(addr); - if (getsockname(ftpState->ctrl.fd, (struct sockaddr *) &addr, &addr_len)) { - debug(9, 0) ("ftpOpenListenSocket: getsockname(%d,..): %s\n", - ftpState->ctrl.fd, xstrerror()); - return -1; - } - /* - * REUSEADDR is needed in fallback mode, since the same port is - * used for both control and data. - */ - if (fallback) { - setsockopt(ftpState->ctrl.fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on)); - port = ntohs(addr.sin_port); - } - fd = comm_open(SOCK_STREAM, - 0, - addr.sin_addr, - port, - COMM_NONBLOCKING | (fallback ? COMM_REUSEADDR : 0), - storeUrl(ftpState->entry)); - debug(9, 3) ("ftpOpenListenSocket: Unconnected data socket created on FD %d\n", fd); - if (fd < 0) { - debug(9, 0) ("ftpOpenListenSocket: comm_open failed\n"); - return -1; - } - if (comm_listen(fd) < 0) { - comm_close(fd); - return -1; - } - ftpState->data.fd = fd; - ftpState->data.port = comm_local_port(fd); - ftpState->data.host = NULL; - return fd; -} - -static void -ftpSendPort(FtpStateData * ftpState) -{ - int fd; - struct sockaddr_in addr; - socklen_t addr_len; - unsigned char *addrptr; - unsigned char *portptr; - debug(9, 3) ("This is ftpSendPort\n"); - ftpState->flags.pasv_supported = 0; - fd = ftpOpenListenSocket(ftpState, 0); - addr_len = sizeof(addr); - if (getsockname(fd, (struct sockaddr *) &addr, &addr_len)) { - debug(9, 0) ("ftpSendPort: getsockname(%d,..): %s\n", fd, xstrerror()); - /* XXX Need to set error message */ - ftpFail(ftpState); - return; - } - addrptr = (unsigned char *) &addr.sin_addr.s_addr; - portptr = (unsigned char *) &addr.sin_port; - snprintf(cbuf, 1024, "PORT %d,%d,%d,%d,%d,%d\r\n", - addrptr[0], addrptr[1], addrptr[2], addrptr[3], - portptr[0], portptr[1]); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_PORT; -} - -static void -ftpReadPort(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadPort\n"); - if (code != 200) { - /* Fall back on using the same port as the control connection */ - debug(9, 3) ("PORT not supported by remote end\n"); - ftpOpenListenSocket(ftpState, 1); - } - ftpRestOrList(ftpState); -} - -/* "read" handler to accept data connection */ -static void -ftpAcceptDataConnection(int fd, void *data) -{ - FtpStateData *ftpState = data; - struct sockaddr_in my_peer, me; - debug(9, 3) ("ftpAcceptDataConnection\n"); - - if (EBIT_TEST(ftpState->entry->flags, ENTRY_ABORTED)) { - comm_close(ftpState->ctrl.fd); - return; - } - fd = comm_accept(fd, &my_peer, &me); - if (Config.Ftp.sanitycheck) { - char *ipaddr = inet_ntoa(my_peer.sin_addr); - if (strcmp(fd_table[ftpState->ctrl.fd].ipaddr, ipaddr) != 0) { - debug(9, 1) ("FTP data connection from unexpected server (%s:%d), expecting %s\n", ipaddr, (int) ntohs(my_peer.sin_port), fd_table[ftpState->ctrl.fd].ipaddr); - comm_close(fd); - commSetSelect(ftpState->data.fd, - COMM_SELECT_READ, - ftpAcceptDataConnection, - ftpState, - 0); - return; - } - } - if (fd < 0) { - debug(9, 1) ("ftpHandleDataAccept: comm_accept(%d): %s", fd, xstrerror()); - /* XXX Need to set error message */ - ftpFail(ftpState); - return; - } - /* Replace the Listen socket with the accepted data socket */ - comm_close(ftpState->data.fd); - debug(9, 3) ("ftpAcceptDataConnection: Connected data socket on FD %d\n", fd); - ftpState->data.fd = fd; - ftpState->data.port = ntohs(my_peer.sin_port); - ftpState->data.host = xstrdup(inet_ntoa(my_peer.sin_addr)); - commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); - commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, - ftpState); - /* XXX We should have a flag to track connect state... - * host NULL -> not connected, port == local port - * host set -> connected, port == remote port - */ - /* Restart state (SENT_NLST/LIST/RETR) */ - FTP_SM_FUNCS[ftpState->state] (ftpState); -} - -static void -ftpRestOrList(FtpStateData * ftpState) -{ - debug(9, 3) ("This is ftpRestOrList\n"); - if (ftpState->typecode == 'D') { - ftpState->flags.isdir = 1; - ftpState->flags.need_base_href = 1; - if (ftpState->flags.put) { - ftpSendMkdir(ftpState); /* PUT name;type=d */ - } else { - ftpSendNlst(ftpState); /* GET name;type=d sec 3.2.2 of RFC 1738 */ - } - } else if (ftpState->flags.put) { - debug(9, 3) ("ftpRestOrList: Sending STOR request...\n"); - ftpSendStor(ftpState); - } else if (ftpState->flags.isdir) - ftpSendList(ftpState); - else if (ftpRestartable(ftpState)) - ftpSendRest(ftpState); - else - ftpSendRetr(ftpState); -} - -static void -ftpSendStor(FtpStateData * ftpState) -{ - if (ftpState->filepath != NULL) { - /* Plain file upload */ - snprintf(cbuf, 1024, "STOR %s\r\n", ftpState->filepath); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_STOR; - } else if (httpHeaderGetInt(&ftpState->request->header, HDR_CONTENT_LENGTH) > 0) { - /* File upload without a filename. use STOU to generate one */ - snprintf(cbuf, 1024, "STOU\r\n"); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_STOR; - } else { - /* No file to transfer. Only create directories if needed */ - ftpSendReply(ftpState); - } -} - -static void -ftpReadStor(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadStor\n"); - if (code == 125 || (code == 150 && ftpState->data.host)) { - /* Begin data transfer */ - debug(9, 3) ("ftpReadStor: starting data transfer\n"); - commSetSelect(ftpState->data.fd, - COMM_SELECT_WRITE, - ftpDataWrite, - ftpState, - Config.Timeout.read); - /* - * Cancel the timeout on the Control socket and - * establish one on the data socket. - */ - commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); - commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, - ftpState); - ftpState->state = WRITING_DATA; - debug(9, 3) ("ftpReadStor: writing data channel\n"); - } else if (code == 150) { - /* Accept data channel */ - debug(9, 3) ("ftpReadStor: accepting data channel\n"); - commSetSelect(ftpState->data.fd, - COMM_SELECT_READ, - ftpAcceptDataConnection, - ftpState, - 0); - } else { - debug(9, 3) ("ftpReadStor: Unexpected reply code %03d\n", code); - ftpFail(ftpState); - } -} - -static void -ftpSendRest(FtpStateData * ftpState) -{ - snprintf(cbuf, 1024, "REST %d\r\n", ftpState->restart_offset); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_REST; -} - -static int -ftpRestartable(FtpStateData * ftpState) -{ - if (ftpState->restart_offset > 0) - return 1; - if (!ftpState->request->range) - return 0; - if (!ftpState->flags.binary) - return 0; - if (ftpState->size <= 0) - return 0; - - ftpState->restart_offset = httpHdrRangeLowestOffset(ftpState->request->range, (size_t) ftpState->size); - if (ftpState->restart_offset <= 0) - return 0; - return 1; -} - -static void -ftpReadRest(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadRest\n"); - assert(ftpState->restart_offset > 0); - if (code == 350) { - ftpState->restarted_offset = ftpState->restart_offset; - ftpSendRetr(ftpState); - } else if (code > 0) { - debug(9, 3) ("ftpReadRest: REST not supported\n"); - ftpState->flags.rest_supported = 0; - ftpSendRetr(ftpState); - } else { - ftpFail(ftpState); - } -} - -static void -ftpSendList(FtpStateData * ftpState) -{ - if (ftpState->filepath) { - ftpState->flags.need_base_href = 1; - snprintf(cbuf, 1024, "LIST %s\r\n", ftpState->filepath); - } else { - snprintf(cbuf, 1024, "LIST\r\n"); - } - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_LIST; -} - -static void -ftpSendNlst(FtpStateData * ftpState) -{ - ftpState->flags.tried_nlst = 1; - if (ftpState->filepath) { - ftpState->flags.need_base_href = 1; - snprintf(cbuf, 1024, "NLST %s\r\n", ftpState->filepath); - } else { - snprintf(cbuf, 1024, "NLST\r\n"); - } - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_NLST; -} - -static void -ftpReadList(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadList\n"); - if (code == 125 || (code == 150 && ftpState->data.host)) { - /* Begin data transfer */ - ftpAppendSuccessHeader(ftpState); - commSetSelect(ftpState->data.fd, - COMM_SELECT_READ, - ftpDataRead, - ftpState, - Config.Timeout.read); - commSetDefer(ftpState->data.fd, fwdCheckDeferRead, ftpState->entry); - ftpState->state = READING_DATA; - /* - * Cancel the timeout on the Control socket and establish one - * on the data socket - */ - commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); - commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, ftpState); - return; - } else if (code == 150) { - /* Accept data channel */ - commSetSelect(ftpState->data.fd, - COMM_SELECT_READ, - ftpAcceptDataConnection, - ftpState, - 0); - /* - * Cancel the timeout on the Control socket and establish one - * on the data socket - */ - commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); - commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, ftpState); - return; - } else if (!ftpState->flags.tried_nlst && code > 300) { - ftpSendNlst(ftpState); - } else { - ftpFail(ftpState); - return; - } -} - -static void -ftpSendRetr(FtpStateData * ftpState) -{ - assert(ftpState->filepath != NULL); - snprintf(cbuf, 1024, "RETR %s\r\n", ftpState->filepath); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_RETR; -} - -static void -ftpReadRetr(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadRetr\n"); - if (code == 125 || (code == 150 && ftpState->data.host)) { - /* Begin data transfer */ - debug(9, 3) ("ftpReadRetr: reading data channel\n"); - ftpAppendSuccessHeader(ftpState); - commSetSelect(ftpState->data.fd, - COMM_SELECT_READ, - ftpDataRead, - ftpState, - Config.Timeout.read); - commSetDefer(ftpState->data.fd, fwdCheckDeferRead, ftpState->entry); - ftpState->state = READING_DATA; - /* - * Cancel the timeout on the Control socket and establish one - * on the data socket - */ - commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); - commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, - ftpState); - } else if (code == 150) { - /* Accept data channel */ - commSetSelect(ftpState->data.fd, - COMM_SELECT_READ, - ftpAcceptDataConnection, - ftpState, - 0); - /* - * Cancel the timeout on the Control socket and establish one - * on the data socket - */ - commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); - commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, - ftpState); - } else if (code >= 300) { - if (!ftpState->flags.try_slash_hack) { - /* Try this as a directory missing trailing slash... */ - ftpHackShortcut(ftpState, ftpSendCwd); - } else { - ftpFail(ftpState); - } - } else { - ftpFail(ftpState); - } -} - -static void -ftpReadTransferDone(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpReadTransferDone\n"); - if (code == 226) { - /* Connection closed; retrieval done. */ - if (ftpState->flags.html_header_sent) - ftpListingFinish(ftpState); - fwdUnregister(ftpState->ctrl.fd, ftpState->fwd); - fwdComplete(ftpState->fwd); - ftpSendQuit(ftpState); - } else { /* != 226 */ - debug(9, 1) ("ftpReadTransferDone: Got code %d after reading data\n", - code); - ftpFailed(ftpState, ERR_FTP_FAILURE); - /* ftpFailed closes ctrl.fd and frees ftpState */ - return; - } -} - -/* This will be called when there is data available to put */ -static void -ftpRequestBody(char *buf, ssize_t size, void *data) -{ - FtpStateData *ftpState = (FtpStateData *) data; - debug(9, 3) ("ftpRequestBody: buf=%p size=%d ftpState=%p\n", buf, (int) size, data); - ftpState->data.offset = size; - if (size > 0) { - /* DataWrite */ - comm_write(ftpState->data.fd, buf, size, ftpDataWriteCallback, data, NULL); - } else if (size < 0) { - /* Error */ - debug(9, 1) ("ftpRequestBody: request aborted"); - ftpFailed(ftpState, ERR_READ_ERROR); - } else if (size == 0) { - /* End of transfer */ - ftpDataComplete(ftpState); - } -} - -/* This will be called when the put write is completed */ -static void -ftpDataWriteCallback(int fd, char *buf, size_t size, int err, void *data) -{ - FtpStateData *ftpState = (FtpStateData *) data; - if (!err) { - /* Shedule the rest of the request */ - clientReadBody(ftpState->request, ftpState->data.buf, ftpState->data.size, ftpRequestBody, ftpState); - } else { - debug(9, 1) ("ftpDataWriteCallback: write error: %s\n", xstrerror()); - ftpFailed(ftpState, ERR_WRITE_ERROR); - } -} - -static void -ftpDataWrite(int ftp, void *data) -{ - FtpStateData *ftpState = (FtpStateData *) data; - debug(9, 3) ("ftpDataWrite\n"); - /* This starts the body transfer */ - clientReadBody(ftpState->request, ftpState->data.buf, ftpState->data.size, ftpRequestBody, ftpState); -} - -static void -ftpWriteTransferDone(FtpStateData * ftpState) -{ - int code = ftpState->ctrl.replycode; - debug(9, 3) ("This is ftpWriteTransferDone\n"); - if (code != 226) { - debug(9, 1) ("ftpReadTransferDone: Got code %d after sending data\n", - code); - ftpFailed(ftpState, ERR_FTP_PUT_ERROR); - return; - } - storeTimestampsSet(ftpState->entry); /* XXX Is this needed? */ - ftpSendReply(ftpState); -} - -static void -ftpSendQuit(FtpStateData * ftpState) -{ - assert(ftpState->ctrl.fd > -1); - snprintf(cbuf, 1024, "QUIT\r\n"); - ftpWriteCommand(cbuf, ftpState); - ftpState->state = SENT_QUIT; -} - -static void -ftpReadQuit(FtpStateData * ftpState) -{ - comm_close(ftpState->ctrl.fd); -} - -static void -ftpTrySlashHack(FtpStateData * ftpState) -{ - char *path; - ftpState->flags.try_slash_hack = 1; - /* Free old paths */ - if (ftpState->pathcomps) - wordlistDestroy(&ftpState->pathcomps); - safe_free(ftpState->filepath); - /* Build the new path (urlpath begins with /) */ - path = xstrdup(strBuf(ftpState->request->urlpath)); - rfc1738_unescape(path); - ftpState->filepath = path; - /* And off we go */ - ftpGetFile(ftpState); -} - -static void -ftpTryDatachannelHack(FtpStateData * ftpState) -{ - ftpState->flags.datachannel_hack = 1; - /* we have to undo some of the slash hack... */ - if (ftpState->old_filepath != NULL) { - ftpState->flags.try_slash_hack = 0; - safe_free(ftpState->filepath); - ftpState->filepath = ftpState->old_filepath; - ftpState->old_filepath = NULL; - } - ftpState->flags.tried_nlst = 0; - /* And off we go */ - if (ftpState->flags.isdir) { - ftpListDir(ftpState); - } else { - ftpGetFile(ftpState); - } - return; -} - -/* Forget hack status. Next error is shown to the user */ -static void -ftpUnhack(FtpStateData * ftpState) -{ - if (ftpState->old_request != NULL) { - safe_free(ftpState->old_request); - safe_free(ftpState->old_reply); - } -} - -static void -ftpHackShortcut(FtpStateData * ftpState, FTPSM * nextState) -{ - /* Clear some unwanted state */ - ftpState->restarted_offset = 0; - ftpState->restart_offset = 0; - /* Save old error message & some state info */ - if (ftpState->old_request == NULL) { - ftpState->old_request = ftpState->ctrl.last_command; - ftpState->ctrl.last_command = NULL; - ftpState->old_reply = ftpState->ctrl.last_reply; - ftpState->ctrl.last_reply = NULL; - if (ftpState->pathcomps == NULL && ftpState->filepath != NULL) - ftpState->old_filepath = xstrdup(ftpState->filepath); - } - /* Jump to the "hack" state */ - nextState(ftpState); -} - -static void -ftpFail(FtpStateData * ftpState) -{ - debug(9, 3) ("ftpFail\n"); - /* Try the / hack to support "Netscape" FTP URL's for retreiving files */ - if (!ftpState->flags.isdir && /* Not a directory */ - !ftpState->flags.try_slash_hack && /* Not in slash hack */ - ftpState->mdtm <= 0 && ftpState->size < 0 && /* Not known as a file */ - strNCaseCmp(ftpState->request->urlpath, "/%2f", 4) != 0) { /* No slash encoded */ - switch (ftpState->state) { - case SENT_CWD: - case SENT_RETR: - /* Try the / hack */ - ftpHackShortcut(ftpState, ftpTrySlashHack); - return; - default: - break; - } - } - /* Try to reopen datachannel */ - if (!ftpState->flags.datachannel_hack && - ftpState->pathcomps == NULL) { - switch (ftpState->state) { - case SENT_RETR: - case SENT_LIST: - case SENT_NLST: - /* Try to reopen datachannel */ - ftpHackShortcut(ftpState, ftpTryDatachannelHack); - return; - default: - break; - } - } - ftpFailed(ftpState, ERR_NONE); - /* ftpFailed closes ctrl.fd and frees ftpState */ -} - -static void -ftpFailed(FtpStateData * ftpState, err_type error) -{ - StoreEntry *entry = ftpState->entry; - if (entry->mem_obj->inmem_hi == 0) - ftpFailedErrorMessage(ftpState, error); - if (ftpState->data.fd > -1) { - comm_close(ftpState->data.fd); - ftpState->data.fd = -1; - } - comm_close(ftpState->ctrl.fd); -} - -static void -ftpFailedErrorMessage(FtpStateData * ftpState, err_type error) -{ - ErrorState *err; - const char *command, *reply; - /* Translate FTP errors into HTTP errors */ - err = NULL; - switch (error) { - case ERR_NONE: - switch (ftpState->state) { - case SENT_USER: - case SENT_PASS: - if (ftpState->ctrl.replycode > 500) - err = errorCon(ERR_FTP_FORBIDDEN, HTTP_FORBIDDEN); - else if (ftpState->ctrl.replycode == 421) - err = errorCon(ERR_FTP_UNAVAILABLE, HTTP_SERVICE_UNAVAILABLE); - break; - case SENT_CWD: - case SENT_RETR: - if (ftpState->ctrl.replycode == 550) - err = errorCon(ERR_FTP_NOT_FOUND, HTTP_NOT_FOUND); - break; - default: - break; - } - break; - case ERR_READ_TIMEOUT: - err = errorCon(error, HTTP_GATEWAY_TIMEOUT); - break; - default: - err = errorCon(error, HTTP_BAD_GATEWAY); - break; - } - if (err == NULL) - err = errorCon(ERR_FTP_FAILURE, HTTP_BAD_GATEWAY); - err->xerrno = errno; - err->request = requestLink(ftpState->request); - err->ftp.server_msg = ftpState->ctrl.message; - ftpState->ctrl.message = NULL; - if (ftpState->old_request) - command = ftpState->old_request; - else - command = ftpState->ctrl.last_command; - if (command && strncmp(command, "PASS", 4) == 0) - command = "PASS "; - if (ftpState->old_reply) - reply = ftpState->old_reply; - else - reply = ftpState->ctrl.last_reply; - if (command) - err->ftp.request = xstrdup(command); - if (reply) - err->ftp.reply = xstrdup(reply); - fwdFail(ftpState->fwd, err); -} - -static void -ftpSendReply(FtpStateData * ftpState) -{ - ErrorState *err; - int code = ftpState->ctrl.replycode; - http_status http_code; - err_type err_code = ERR_NONE; - debug(9, 5) ("ftpSendReply: %s, code %d\n", - storeUrl(ftpState->entry), code); - if (cbdataReferenceValid(ftpState)) - debug(9, 5) ("ftpSendReply: ftpState (%p) is valid!\n", ftpState); - if (code == 226) { - err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED; - http_code = (ftpState->mdtm > 0) ? HTTP_ACCEPTED : HTTP_CREATED; - } else if (code == 227) { - err_code = ERR_FTP_PUT_CREATED; - http_code = HTTP_CREATED; - } else { - err_code = ERR_FTP_PUT_ERROR; - http_code = HTTP_INTERNAL_SERVER_ERROR; - } - err = errorCon(err_code, http_code); - err->request = requestLink(ftpState->request); - if (ftpState->old_request) - err->ftp.request = xstrdup(ftpState->old_request); - else - err->ftp.request = xstrdup(ftpState->ctrl.last_command); - if (ftpState->old_reply) - err->ftp.reply = xstrdup(ftpState->old_reply); - else if (ftpState->ctrl.last_reply) - err->ftp.reply = xstrdup(ftpState->ctrl.last_reply); - else - err->ftp.reply = xstrdup(""); - errorAppendEntry(ftpState->entry, err); - storeBufferFlush(ftpState->entry); - ftpSendQuit(ftpState); -} - -static void -ftpAppendSuccessHeader(FtpStateData * ftpState) -{ - const char *mime_type = NULL; - const char *mime_enc = NULL; - String urlpath = ftpState->request->urlpath; - const char *filename = NULL; - const char *t = NULL; - StoreEntry *e = ftpState->entry; - http_reply *reply = e->mem_obj->reply; - http_version_t version; - - if (ftpState->flags.http_header_sent) - return; - ftpState->flags.http_header_sent = 1; - assert(e->mem_obj->inmem_hi == 0); - EBIT_CLR(e->flags, ENTRY_FWD_HDR_WAIT); - filename = (t = strRChr(urlpath, '/')) ? t + 1 : strBuf(urlpath); - if (ftpState->flags.isdir) { - mime_type = "text/html"; - } else { - switch (ftpState->typecode) { - case 'I': - mime_type = "application/octet-stream"; - mime_enc = mimeGetContentEncoding(filename); - break; - case 'A': - mime_type = "text/plain"; - break; - default: - mime_type = mimeGetContentType(filename); - mime_enc = mimeGetContentEncoding(filename); - break; - } - } - storeBuffer(e); - httpReplyReset(reply); - /* set standard stuff */ - if (ftpState->restarted_offset) { - /* Partial reply */ - HttpHdrRangeSpec range_spec; - range_spec.offset = ftpState->restarted_offset; - range_spec.length = ftpState->size - ftpState->restarted_offset; - httpBuildVersion(&version, 1, 0); - httpReplySetHeaders(reply, version, HTTP_PARTIAL_CONTENT, "Gatewaying", - mime_type, ftpState->size - ftpState->restarted_offset, ftpState->mdtm, -2); - httpHeaderAddContRange(&reply->header, range_spec, ftpState->size); - } else { - /* Full reply */ - httpBuildVersion(&version, 1, 0); - httpReplySetHeaders(reply, version, HTTP_OK, "Gatewaying", - mime_type, ftpState->size, ftpState->mdtm, -2); - } - /* additional info */ - if (mime_enc) - httpHeaderPutStr(&reply->header, HDR_CONTENT_ENCODING, mime_enc); - httpReplySwapOut(reply, e); - storeBufferFlush(e); - reply->hdr_sz = e->mem_obj->inmem_hi; - storeTimestampsSet(e); - if (ftpState->flags.authenticated) { - /* - * Authenticated requests can't be cached. - */ - storeRelease(e); - } else if (EBIT_TEST(e->flags, ENTRY_CACHABLE) && !ftpState->restarted_offset) { - storeSetPublicKey(e); - } else { - storeRelease(e); - } -} - -static void -ftpAuthRequired(HttpReply * old_reply, request_t * request, const char *realm) -{ - ErrorState *err = errorCon(ERR_CACHE_ACCESS_DENIED, HTTP_UNAUTHORIZED); - HttpReply *rep; - err->request = requestLink(request); - rep = errorBuildReply(err); - errorStateFree(err); - /* add Authenticate header */ - httpHeaderPutAuth(&rep->header, "Basic", realm); - /* move new reply to the old one */ - httpReplyAbsorb(old_reply, rep); -} - -char * -ftpUrlWith2f(const request_t * request) -{ - LOCAL_ARRAY(char, buf, MAX_URL); - LOCAL_ARRAY(char, loginbuf, MAX_LOGIN_SZ + 1); - LOCAL_ARRAY(char, portbuf, 32); - char *t; - portbuf[0] = '\0'; - if (request->protocol != PROTO_FTP) - return NULL; - if (request->port != urlDefaultPort(request->protocol)) - snprintf(portbuf, 32, ":%d", request->port); - loginbuf[0] = '\0'; - if ((int) strlen(request->login) > 0) { - xstrncpy(loginbuf, request->login, sizeof(loginbuf) - 2); - if ((t = strchr(loginbuf, ':'))) - *t = '\0'; - strcat(loginbuf, "@"); - } - snprintf(buf, MAX_URL, "%s://%s%s%s%s%s", - ProtocolStr[request->protocol], - loginbuf, - request->host, - portbuf, - "/%2f", - strBuf(request->urlpath)); - if ((t = strchr(buf, '?'))) - *t = '\0'; - return buf; -} --- /dev/null Wed Feb 14 01:07:22 2007 +++ squid/src/ftp.cc Wed Feb 14 01:07:40 2007 @@ -0,0 +1,2615 @@ + +/* + * $Id: ftp.cc,v 1.1.2.1 2002/10/11 15:40:49 rbcollins Exp $ + * + * DEBUG: section 9 File Transfer Protocol (FTP) + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "Store.h" + + +static const char *const crlf = "\r\n"; +static char cbuf[1024]; + +typedef enum { + BEGIN, + SENT_USER, + SENT_PASS, + SENT_TYPE, + SENT_MDTM, + SENT_SIZE, + SENT_PORT, + SENT_PASV, + SENT_CWD, + SENT_LIST, + SENT_NLST, + SENT_REST, + SENT_RETR, + SENT_STOR, + SENT_QUIT, + READING_DATA, + WRITING_DATA, + SENT_MKDIR +} ftp_state_t; + +struct _ftp_flags { + unsigned int isdir:1; + unsigned int pasv_supported:1; + unsigned int skip_whitespace:1; + unsigned int rest_supported:1; + unsigned int pasv_only:1; + unsigned int authenticated:1; + unsigned int http_header_sent:1; + unsigned int tried_nlst:1; + unsigned int need_base_href:1; + unsigned int root_dir:1; + unsigned int no_dotdot:1; + unsigned int html_header_sent:1; + unsigned int binary:1; + unsigned int try_slash_hack:1; + unsigned int put:1; + unsigned int put_mkdir:1; + unsigned int listformat_unknown:1; + unsigned int datachannel_hack:1; +}; + +typedef struct _Ftpdata { + StoreEntry *entry; + request_t *request; + char user[MAX_URL]; + char password[MAX_URL]; + int password_url; + char *reply_hdr; + int reply_hdr_state; + String title_url; + String base_href; + int conn_att; + int login_att; + ftp_state_t state; + time_t mdtm; + int size; + wordlist *pathcomps; + char *filepath; + int restart_offset; + int restarted_offset; + int rest_att; + char *proxy_host; + size_t list_width; + wordlist *cwd_message; + char *old_request; + char *old_reply; + char *old_filepath; + char typecode; + struct { + int fd; + char *buf; + size_t size; + off_t offset; + wordlist *message; + char *last_command; + char *last_reply; + int replycode; + } ctrl; + struct { + int fd; + char *buf; + size_t size; + off_t offset; + char *host; + u_short port; + } data; + struct _ftp_flags flags; + FwdState *fwd; +} FtpStateData; + +typedef struct { + char type; + int size; + char *date; + char *name; + char *showname; + char *link; +} ftpListParts; + +typedef void (FTPSM) (FtpStateData *); + +#define FTP_LOGIN_ESCAPED 1 +#define FTP_LOGIN_NOT_ESCAPED 0 + +/* Local functions */ +static CNCB ftpPasvCallback; +static PF ftpDataRead; +static PF ftpDataWrite; +static CWCB ftpDataWriteCallback; +static PF ftpStateFree; +static PF ftpTimeout; +static PF ftpReadControlReply; +static CWCB ftpWriteCommandCallback; +static void ftpLoginParser(const char *, FtpStateData *, int escaped); +static wordlist *ftpParseControlReply(char *, size_t, int *, int *); +static int ftpRestartable(FtpStateData * ftpState); +static void ftpAppendSuccessHeader(FtpStateData * ftpState); +static void ftpAuthRequired(HttpReply * reply, request_t * request, const char *realm); +static void ftpHackShortcut(FtpStateData * ftpState, FTPSM * nextState); +static void ftpUnhack(FtpStateData * ftpState); +static void ftpScheduleReadControlReply(FtpStateData *, int); +static void ftpHandleControlReply(FtpStateData *); +static char *ftpHtmlifyListEntry(const char *line, FtpStateData * ftpState); +static void ftpFailed(FtpStateData *, err_type); +static void ftpFailedErrorMessage(FtpStateData *, err_type); + +/* + * State machine functions + * send == state transition + * read == wait for response, and select next state transition + * other == Transition logic + */ +static FTPSM ftpReadWelcome; +static FTPSM ftpSendUser; +static FTPSM ftpReadUser; +static FTPSM ftpSendPass; +static FTPSM ftpReadPass; +static FTPSM ftpSendType; +static FTPSM ftpReadType; +static FTPSM ftpSendMdtm; +static FTPSM ftpReadMdtm; +static FTPSM ftpSendSize; +static FTPSM ftpReadSize; +static FTPSM ftpSendPort; +static FTPSM ftpReadPort; +static FTPSM ftpSendPasv; +static FTPSM ftpReadPasv; +static FTPSM ftpTraverseDirectory; +static FTPSM ftpListDir; +static FTPSM ftpGetFile; +static FTPSM ftpSendCwd; +static FTPSM ftpReadCwd; +static FTPSM ftpRestOrList; +static FTPSM ftpSendList; +static FTPSM ftpSendNlst; +static FTPSM ftpReadList; +static FTPSM ftpSendRest; +static FTPSM ftpReadRest; +static FTPSM ftpSendRetr; +static FTPSM ftpReadRetr; +static FTPSM ftpReadTransferDone; +static FTPSM ftpSendStor; +static FTPSM ftpReadStor; +static FTPSM ftpWriteTransferDone; +static FTPSM ftpSendReply; +static FTPSM ftpSendMkdir; +static FTPSM ftpReadMkdir; +static FTPSM ftpFail; +static FTPSM ftpSendQuit; +static FTPSM ftpReadQuit; +/************************************************ +** State Machine Description (excluding hacks) ** +************************************************* +From To +--------------------------------------- +Welcome User +User Pass +Pass Type +Type TraverseDirectory / GetFile +TraverseDirectory Cwd / GetFile / ListDir +Cwd TraverseDirectory / Mkdir +GetFile Mdtm +Mdtm Size +Size Pasv +ListDir Pasv +Pasv FileOrList +FileOrList Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d) +Rest Retr +Retr / Nlst / List DataRead* (on datachannel) +DataRead* ReadTransferDone +ReadTransferDone DataTransferDone +Stor DataWrite* (on datachannel) +DataWrite* RequestPutBody** (from client) +RequestPutBody** DataWrite* / WriteTransferDone +WriteTransferDone DataTransferDone +DataTransferDone Quit +Quit - +************************************************/ + +FTPSM *FTP_SM_FUNCS[] = +{ + ftpReadWelcome, /* BEGIN */ + ftpReadUser, /* SENT_USER */ + ftpReadPass, /* SENT_PASS */ + ftpReadType, /* SENT_TYPE */ + ftpReadMdtm, /* SENT_MDTM */ + ftpReadSize, /* SENT_SIZE */ + ftpReadPort, /* SENT_PORT */ + ftpReadPasv, /* SENT_PASV */ + ftpReadCwd, /* SENT_CWD */ + ftpReadList, /* SENT_LIST */ + ftpReadList, /* SENT_NLST */ + ftpReadRest, /* SENT_REST */ + ftpReadRetr, /* SENT_RETR */ + ftpReadStor, /* SENT_STOR */ + ftpReadQuit, /* SENT_QUIT */ + ftpReadTransferDone, /* READING_DATA (RETR,LIST,NLST) */ + ftpWriteTransferDone, /* WRITING_DATA (STOR) */ + ftpReadMkdir /* SENT_MKDIR */ +}; + +static void +ftpStateFree(int fdnotused, void *data) +{ + FtpStateData *ftpState = (FtpStateData *)data; + if (ftpState == NULL) + return; + debug(9, 3) ("ftpStateFree: %s\n", storeUrl(ftpState->entry)); + storeUnregisterAbort(ftpState->entry); + storeUnlockObject(ftpState->entry); + if (ftpState->reply_hdr) { + memFree(ftpState->reply_hdr, MEM_8K_BUF); + ftpState->reply_hdr = NULL; + } + requestUnlink(ftpState->request); + if (ftpState->ctrl.buf) { + memFreeBuf(ftpState->ctrl.size, ftpState->ctrl.buf); + ftpState->ctrl.buf = NULL; + } + if (ftpState->data.buf) { + memFreeBuf(ftpState->data.size, ftpState->data.buf); + ftpState->data.buf = NULL; + } + if (ftpState->pathcomps) + wordlistDestroy(&ftpState->pathcomps); + if (ftpState->ctrl.message) + wordlistDestroy(&ftpState->ctrl.message); + if (ftpState->cwd_message) + wordlistDestroy(&ftpState->cwd_message); + safe_free(ftpState->ctrl.last_reply); + safe_free(ftpState->ctrl.last_command); + safe_free(ftpState->old_request); + safe_free(ftpState->old_reply); + safe_free(ftpState->old_filepath); + stringClean(&ftpState->title_url); + stringClean(&ftpState->base_href); + safe_free(ftpState->filepath); + safe_free(ftpState->data.host); + if (ftpState->data.fd > -1) { + comm_close(ftpState->data.fd); + ftpState->data.fd = -1; + } + cbdataFree(ftpState); +} + +static void +ftpLoginParser(const char *login, FtpStateData * ftpState, int escaped) +{ + char *s = NULL; + xstrncpy(ftpState->user, login, MAX_URL); + if ((s = strchr(ftpState->user, ':'))) { + *s = 0; + xstrncpy(ftpState->password, s + 1, MAX_URL); + if (escaped) + rfc1738_unescape(ftpState->password); + ftpState->password_url = 1; + } else { + xstrncpy(ftpState->password, null_string, MAX_URL); + } + if (escaped) + rfc1738_unescape(ftpState->user); + if (ftpState->user[0] || ftpState->password[0]) + return; + xstrncpy(ftpState->user, "anonymous", MAX_URL); + xstrncpy(ftpState->password, Config.Ftp.anon_user, MAX_URL); +} + +static void +ftpTimeout(int fd, void *data) +{ + FtpStateData *ftpState = (FtpStateData *)data; + StoreEntry *entry = ftpState->entry; + debug(9, 4) ("ftpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); + if (SENT_PASV == ftpState->state && fd == ftpState->data.fd) { + /* stupid ftp.netscape.com */ + ftpState->fwd->flags.dont_retry = 0; + ftpState->fwd->flags.ftp_pasv_failed = 1; + debug(9, 1) ("ftpTimeout: timeout in SENT_PASV state\n"); + } + ftpFailed(ftpState, ERR_READ_TIMEOUT); + /* ftpFailed closes ctrl.fd and frees ftpState */ +} + +static void +ftpListingStart(FtpStateData * ftpState) +{ + StoreEntry *e = ftpState->entry; + wordlist *w; + char *dirup; + int i, j, k; + char *title; + storeBuffer(e); + storeAppendPrintf(e, "\n", + version_string); + storeAppendPrintf(e, "\n", mkrfc1123(squid_curtime)); + storeAppendPrintf(e, "\n"); + storeAppendPrintf(e, "\n"); + storeAppendPrintf(e, "FTP Directory: %s\n", + html_quote(strBuf(ftpState->title_url))); + storeAppendPrintf(e, "\n"); + storeAppendPrintf(e, "\n"); + if (ftpState->flags.need_base_href) + storeAppendPrintf(e, "\n", + html_quote(strBuf(ftpState->base_href))); + storeAppendPrintf(e, "\n"); + if (ftpState->cwd_message) { + storeAppendPrintf(e, "
\n");
+	for (w = ftpState->cwd_message; w; w = w->next)
+	    storeAppendPrintf(e, "%s\n", html_quote(w->key));
+	storeAppendPrintf(e, "
\n"); + storeAppendPrintf(e, "
\n"); + wordlistDestroy(&ftpState->cwd_message); + } + storeAppendPrintf(e, "

\n"); + storeAppendPrintf(e, "FTP Directory: "); + /* "ftp://" == 6 characters */ + assert(strLen(ftpState->title_url) >= 6); + title = html_quote(strBuf(ftpState->title_url)); + for (i = 6, j = 0; title[i]; j = i) { + storeAppendPrintf(e, ""); + for (k = j; k < i - 1; k++) + storeAppendPrintf(e, "%c", title[k]); + if (strBuf(ftpState->title_url)[k] != '/') + storeAppendPrintf(e, "%c", title[k++]); + storeAppendPrintf(e, ""); + if (k < i) + storeAppendPrintf(e, "%c", title[k++]); + if (i == j) { + /* Error guard, or "assert" */ + storeAppendPrintf(e, "ERROR: Failed to parse URL: %s\n", + html_quote(strBuf(ftpState->title_url))); + debug(9, 0) ("Failed to parse URL: %s\n", strBuf(ftpState->title_url)); + break; + } + } + storeAppendPrintf(e, "

\n"); + storeAppendPrintf(e, "
\n");
+    dirup = ftpHtmlifyListEntry("", ftpState);
+    storeAppend(e, dirup, strlen(dirup));
+    storeBufferFlush(e);
+    ftpState->flags.html_header_sent = 1;
+}
+
+static void
+ftpListingFinish(FtpStateData * ftpState)
+{
+    StoreEntry *e = ftpState->entry;
+    storeBuffer(e);
+    storeAppendPrintf(e, "
\n"); + if (ftpState->flags.listformat_unknown && !ftpState->flags.tried_nlst) { + storeAppendPrintf(e, "[As plain directory]\n"); + } else if (ftpState->typecode == 'D') { + storeAppendPrintf(e, "[As extended directory]\n"); + } + storeAppendPrintf(e, "
\n"); + storeAppendPrintf(e, "
\n"); + storeAppendPrintf(e, "Generated %s by %s (%s)\n", + mkrfc1123(squid_curtime), + getMyHostname(), + full_appname_string); + storeAppendPrintf(e, "
\n"); + storeBufferFlush(e); +} + +static const char *Month[] = +{ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" +}; + +static int +is_month(const char *buf) +{ + int i; + for (i = 0; i < 12; i++) + if (!strcasecmp(buf, Month[i])) + return 1; + return 0; +} + + +static void +ftpListPartsFree(ftpListParts ** parts) +{ + safe_free((*parts)->date); + safe_free((*parts)->name); + safe_free((*parts)->showname); + safe_free((*parts)->link); + safe_free(*parts); +} + +#define MAX_TOKENS 64 + +static ftpListParts * +ftpListParseParts(const char *buf, struct _ftp_flags flags) +{ + ftpListParts *p = NULL; + char *t = NULL; + const char *ct = NULL; + char *tokens[MAX_TOKENS]; + int i; + int n_tokens; + static char tbuf[128]; + char *xbuf = NULL; + static int scan_ftp_initialized = 0; + static regex_t scan_ftp_integer; + static regex_t scan_ftp_time; + static regex_t scan_ftp_dostime; + static regex_t scan_ftp_dosdate; + + if (!scan_ftp_initialized) { + scan_ftp_initialized = 1; + regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB); + regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB); + regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB); + regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE); + } + if (buf == NULL) + return NULL; + if (*buf == '\0') + return NULL; + p = (ftpListParts *)xcalloc(1, sizeof(ftpListParts)); + n_tokens = 0; + memset(tokens, 0, sizeof(tokens)); + xbuf = xstrdup(buf); + if (flags.tried_nlst) { + /* Machine readable format, one name per line */ + p->name = xbuf; + p->type = '\0'; + return p; + } + for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) + tokens[n_tokens++] = xstrdup(t); + xfree(xbuf); + /* locate the Month field */ + for (i = 3; i < n_tokens - 2; i++) { + char *size = tokens[i - 1]; + char *month = tokens[i]; + char *day = tokens[i + 1]; + char *year = tokens[i + 2]; + if (!is_month(month)) + continue; + if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0) + continue; + if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0) + continue; + if (regexec(&scan_ftp_time, day, 0, NULL, 0) != 0) /* Yr | hh:mm */ + continue; + snprintf(tbuf, 128, "%s %2s %5s", + month, day, year); + if (!strstr(buf, tbuf)) + snprintf(tbuf, 128, "%s %2s %-5s", + month, day, year); + if ((t = strstr(buf, tbuf))) { + p->type = *tokens[0]; + p->size = atoi(size); + p->date = xstrdup(tbuf); + if (flags.skip_whitespace) { + t += strlen(tbuf); + while (strchr(w_space, *t)) + t++; + } else { + /* XXX assumes a single space between date and filename + * suggested by: Nathan.Bailey@cc.monash.edu.au and + * Mike Battersby */ + t += strlen(tbuf) + 1; + } + p->name = xstrdup(t); + if ((t = strstr(p->name, " -> "))) { + *t = '\0'; + p->link = xstrdup(t + 4); + } + goto found; + } + break; + } + /* try it as a DOS listing, 04-05-70 09:33PM ... */ + if (n_tokens > 3 && + regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 && + regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) { + if (!strcasecmp(tokens[2], "")) { + p->type = 'd'; + } else { + p->type = '-'; + p->size = atoi(tokens[2]); + } + snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]); + p->date = xstrdup(tbuf); + if (p->type == 'd') { + /* Directory.. name begins with first printable after */ + ct = strstr(buf, tokens[2]); + ct += strlen(tokens[2]); + while (xisspace(*ct)) + ct++; + if (!*ct) + ct = NULL; + } else { + /* A file. Name begins after size, with a space in between */ + snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]); + ct = strstr(buf, tbuf); + if (ct) { + ct += strlen(tokens[2]) + 2; + } + } + p->name = xstrdup(ct ? ct : tokens[3]); + goto found; + } + /* Try EPLF format; carson@lehman.com */ + if (buf[0] == '+') { + ct = buf + 1; + p->type = 0; + while (ct && *ct) { + time_t t; + int l = strcspn(ct + 1, ","); + char *tmp; + if (l < 1) + goto blank; + switch (*ct) { + case '\t': + p->name = xstrndup(ct + 1, l + 1); + break; + case 's': + p->size = atoi(ct + 1); + break; + case 'm': + t = (time_t) strtol(ct + 1, &tmp, 0); + if (*tmp || (tmp == ct + 1)) + break; /* not a valid integer */ + p->date = xstrdup(ctime(&t)); + *(strstr(p->date, "\n")) = '\0'; + break; + case '/': + p->type = 'd'; + break; + case 'r': + p->type = '-'; + break; + case 'i': + break; + default: + break; + } + blank: + ct = strstr(ct, ","); + if (ct) { + ct++; + } + } + if (p->type == 0) { + p->type = '-'; + } + if (p->name) + goto found; + else + safe_free(p->date); + } + found: + for (i = 0; i < n_tokens; i++) + xfree(tokens[i]); + if (!p->name) + ftpListPartsFree(&p); /* cleanup */ + return p; +} + +static const char * +dots_fill(size_t len) +{ + static char buf[256]; + size_t i = 0; + if (len > Config.Ftp.list_width) { + memset(buf, ' ', 256); + buf[0] = '\n'; + buf[Config.Ftp.list_width + 4] = '\0'; + return buf; + } + for (i = len; i < Config.Ftp.list_width; i++) + buf[i - len] = (i % 2) ? '.' : ' '; + buf[i - len] = '\0'; + return buf; +} + +static char * +ftpHtmlifyListEntry(const char *line, FtpStateData * ftpState) +{ + LOCAL_ARRAY(char, icon, 2048); + LOCAL_ARRAY(char, href, 2048 + 40); + LOCAL_ARRAY(char, text, 2048); + LOCAL_ARRAY(char, size, 2048); + LOCAL_ARRAY(char, chdir, 2048 + 40); + LOCAL_ARRAY(char, view, 2048 + 40); + LOCAL_ARRAY(char, download, 2048 + 40); + LOCAL_ARRAY(char, link, 2048 + 40); + LOCAL_ARRAY(char, html, 8192); + size_t width = Config.Ftp.list_width; + ftpListParts *parts; + *icon = *href = *text = *size = *chdir = *view = *download = *link = *html = '\0'; + if ((int) strlen(line) > 1024) { + snprintf(html, 8192, "%s\n", line); + return html; + } + /* Handle builtin */ + if (strcmp(line, "") == 0) { + /* {icon} {text} {link} */ + snprintf(icon, 2048, "\"%-6s\"", + mimeGetIconURL("internal-dirup"), + "[DIRUP]"); + if (!ftpState->flags.no_dotdot && !ftpState->flags.root_dir) { + /* Normal directory */ + strcpy(href, "../"); + strcpy(text, "Parent Directory"); + } else if (!ftpState->flags.no_dotdot && ftpState->flags.root_dir) { + /* "Top level" directory */ + strcpy(href, "%2e%2e/"); + strcpy(text, "Parent Directory"); + snprintf(link, 2048, "(%s)", + "%2f/", + "Root Directory"); + } else if (ftpState->flags.no_dotdot && !ftpState->flags.root_dir) { + /* Normal directory where last component is / or .. */ + strcpy(href, "%2e%2e/"); + strcpy(text, "Parent Directory"); + snprintf(link, 2048, "(%s)", + "../", + "Back"); + } else { /* NO_DOTDOT && ROOT_DIR */ + /* "UNIX Root" directory */ + strcpy(href, "../"); + strcpy(text, "Home Directory"); + } + snprintf(html, 8192, "%s %s %s\n", + href, icon, href, text, link); + return html; + } + if ((parts = ftpListParseParts(line, ftpState->flags)) == NULL) { + const char *p; + snprintf(html, 8192, "%s\n", line); + for (p = line; *p && xisspace(*p); p++); + if (*p && !xisspace(*p)) + ftpState->flags.listformat_unknown = 1; + return html; + } + if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) { + *html = '\0'; + ftpListPartsFree(&parts); + return html; + } + parts->size += 1023; + parts->size >>= 10; + parts->showname = xstrdup(parts->name); + if (!Config.Ftp.list_wrap) { + if (strlen(parts->showname) > width - 1) { + *(parts->showname + width - 1) = '>'; + *(parts->showname + width - 0) = '\0'; + } + } + /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n */ + xstrncpy(href, rfc1738_escape_part(parts->name), 2048); + xstrncpy(text, parts->showname, 2048); + switch (parts->type) { + case 'd': + snprintf(icon, 2048, "\"%-6s\"", + mimeGetIconURL("internal-dir"), + "[DIR]"); + strcat(href, "/"); /* margin is allocated above */ + break; + case 'l': + snprintf(icon, 2048, "\"%-6s\"", + mimeGetIconURL("internal-link"), + "[LINK]"); + /* sometimes there is an 'l' flag, but no "->" link */ + if (parts->link) { + char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link))); + snprintf(link, 2048, " -> %s", + link2, + html_quote(parts->link)); + safe_free(link2); + } + break; + case '\0': + snprintf(icon, 2048, "\"%-6s\"", + mimeGetIconURL(parts->name), + "[UNKNOWN]"); + snprintf(chdir, 2048, " ", + rfc1738_escape_part(parts->name), + mimeGetIconURL("internal-dir")); + break; + case '-': + default: + snprintf(icon, 2048, "\"%-6s\"", + mimeGetIconURL(parts->name), + "[FILE]"); + snprintf(size, 2048, " %6dk", parts->size); + break; + } + if (parts->type != 'd') { + if (mimeGetViewOption(parts->name)) { + snprintf(view, 2048, " ", + href, mimeGetIconURL("internal-view")); + } + if (mimeGetDownloadOption(parts->name)) { + snprintf(download, 2048, " ", + href, mimeGetIconURL("internal-download")); + } + } + /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n */ + if (parts->type != '\0') { + snprintf(html, 8192, "%s %s%s " + "%s%8s%s%s%s%s\n", + href, icon, href, html_quote(text), dots_fill(strlen(text)), + parts->date, size, chdir, view, download, link); + } else { + /* Plain listing. {icon} {text} ... {chdir}{view}{download} */ + snprintf(html, 8192, "%s %s%s " + "%s%s%s%s\n", + href, icon, href, html_quote(text), dots_fill(strlen(text)), + chdir, view, download, link); + } + ftpListPartsFree(&parts); + return html; +} + +static void +ftpParseListing(FtpStateData * ftpState) +{ + char *buf = ftpState->data.buf; + char *sbuf; /* NULL-terminated copy of buf */ + char *end; + char *line; + char *s; + char *t; + size_t linelen; + size_t usable; + StoreEntry *e = ftpState->entry; + size_t len = ftpState->data.offset; + /* + * We need a NULL-terminated buffer for scanning, ick + */ + sbuf = (char *)xmalloc(len + 1); + xstrncpy(sbuf, buf, len + 1); + end = sbuf + len - 1; + while (*end != '\r' && *end != '\n' && end > sbuf) + end--; + usable = end - sbuf; + debug(9, 3) ("ftpParseListing: usable = %d\n", (int) usable); + if (usable == 0) { + debug(9, 3) ("ftpParseListing: didn't find end for %s\n", storeUrl(e)); + xfree(sbuf); + return; + } + debug(9, 3) ("ftpParseListing: %d bytes to play with\n", len); + line = (char *)memAllocate(MEM_4K_BUF); + end++; + storeBuffer(e); + s = sbuf; + s += strspn(s, crlf); + for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { + debug(9, 3) ("ftpParseListing: s = {%s}\n", s); + linelen = strcspn(s, crlf) + 1; + if (linelen < 2) + break; + if (linelen > 4096) + linelen = 4096; + xstrncpy(line, s, linelen); + debug(9, 7) ("ftpParseListing: {%s}\n", line); + if (!strncmp(line, "total", 5)) + continue; + t = ftpHtmlifyListEntry(line, ftpState); + assert(t != NULL); + storeAppend(e, t, strlen(t)); + } + storeBufferFlush(e); + assert(usable <= len); + if (usable < len) { + /* must copy partial line to beginning of buf */ + linelen = len - usable; + if (linelen > 4096) + linelen = 4096; + xstrncpy(line, end, linelen); + xstrncpy(ftpState->data.buf, line, ftpState->data.size); + ftpState->data.offset = strlen(ftpState->data.buf); + } + memFree(line, MEM_4K_BUF); + xfree(sbuf); +} + +static void +ftpDataComplete(FtpStateData * ftpState) +{ + debug(9, 3) ("ftpDataComplete\n"); + /* Connection closed; transfer done. */ + if (ftpState->data.fd > -1) { + /* + * close data socket so it does not occupy resources while + * we wait + */ + comm_close(ftpState->data.fd); + ftpState->data.fd = -1; + } + /* expect the "transfer complete" message on the control socket */ + ftpScheduleReadControlReply(ftpState, 1); +} + +static void +ftpDataRead(int fd, void *data) +{ + FtpStateData *ftpState = (FtpStateData *)data; + int len; + int j; + int bin; + StoreEntry *entry = ftpState->entry; + size_t read_sz; +#if DELAY_POOLS + MemObject *mem = entry->mem_obj; + delay_id delay_id = delayMostBytesAllowed(mem); +#endif + assert(fd == ftpState->data.fd); + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + comm_close(ftpState->ctrl.fd); + return; + } + errno = 0; + read_sz = ftpState->data.size - ftpState->data.offset; +#if DELAY_POOLS + read_sz = delayBytesWanted(delay_id, 1, read_sz); +#endif + memset(ftpState->data.buf + ftpState->data.offset, '\0', read_sz); + statCounter.syscalls.sock.reads++; + len = FD_READ_METHOD(fd, ftpState->data.buf + ftpState->data.offset, read_sz); + if (len > 0) { + fd_bytes(fd, len, FD_READ); +#if DELAY_POOLS + delayBytesIn(delay_id, len); +#endif + kb_incr(&statCounter.server.all.kbytes_in, len); + kb_incr(&statCounter.server.ftp.kbytes_in, len); + ftpState->data.offset += len; + } + debug(9, 5) ("ftpDataRead: FD %d, Read %d bytes\n", fd, len); + if (len > 0) { + IOStats.Ftp.reads++; + for (j = len - 1, bin = 0; j; bin++) + j >>= 1; + IOStats.Ftp.read_hist[bin]++; + } + if (ftpState->flags.isdir && !ftpState->flags.html_header_sent && len >= 0) { + ftpListingStart(ftpState); + } + if (len < 0) { + debug(50, ignoreErrno(errno) ? 3 : 1) ("ftpDataRead: read error: %s\n", xstrerror()); + if (ignoreErrno(errno)) { + commSetSelect(fd, + COMM_SELECT_READ, + ftpDataRead, + ftpState, + Config.Timeout.read); + } else { + ftpFailed(ftpState, ERR_READ_ERROR); + /* ftpFailed closes ctrl.fd and frees ftpState */ + return; + } + } else if (len == 0) { + ftpDataComplete(ftpState); + } else { + if (ftpState->flags.isdir) { + ftpParseListing(ftpState); + } else { + storeAppend(entry, ftpState->data.buf, len); + ftpState->data.offset = 0; + } + commSetSelect(fd, + COMM_SELECT_READ, + ftpDataRead, + ftpState, + Config.Timeout.read); + } +} + +/* + * ftpCheckAuth + * + * Return 1 if we have everything needed to complete this request. + * Return 0 if something is missing. + */ +static int +ftpCheckAuth(FtpStateData * ftpState, const HttpHeader * req_hdr) +{ + char *orig_user; + const char *auth; + ftpLoginParser(ftpState->request->login, ftpState, FTP_LOGIN_ESCAPED); + if (!ftpState->user[0]) + return 1; /* no name */ + if (ftpState->password_url || ftpState->password[0]) + return 1; /* passwd provided in URL */ + /* URL has name, but no passwd */ + if (!(auth = httpHeaderGetAuth(req_hdr, HDR_AUTHORIZATION, "Basic"))) + return 0; /* need auth header */ + ftpState->flags.authenticated = 1; + orig_user = xstrdup(ftpState->user); + ftpLoginParser(auth, ftpState, FTP_LOGIN_NOT_ESCAPED); + if (strcmp(orig_user, ftpState->user) == 0) { + xfree(orig_user); + return 1; /* same username */ + } + xstrncpy(ftpState->user, orig_user, sizeof(ftpState->user)); + xfree(orig_user); + return 0; /* different username */ +} + +static void +ftpCheckUrlpath(FtpStateData * ftpState) +{ + request_t *request = ftpState->request; + int l; + const char *t; + if ((t = strRChr(request->urlpath, ';')) != NULL) { + if (strncasecmp(t + 1, "type=", 5) == 0) { + ftpState->typecode = (char) toupper((int) *(t + 6)); + strCutPtr(request->urlpath, t); + } + } + l = strLen(request->urlpath); + ftpState->flags.need_base_href = 1; + /* check for null path */ + if (!l) { + ftpState->flags.isdir = 1; + ftpState->flags.root_dir = 1; + } else if (!strCmp(request->urlpath, "/%2f/")) { + /* UNIX root directory */ + ftpState->flags.need_base_href = 0; + ftpState->flags.isdir = 1; + ftpState->flags.root_dir = 1; + } else if ((l >= 1) && (*(strBuf(request->urlpath) + l - 1) == '/')) { + /* Directory URL, ending in / */ + ftpState->flags.isdir = 1; + ftpState->flags.need_base_href = 0; + if (l == 1) + ftpState->flags.root_dir = 1; + } +} + +static void +ftpBuildTitleUrl(FtpStateData * ftpState) +{ + request_t *request = ftpState->request; + + stringReset(&ftpState->title_url, "ftp://"); + if (strcmp(ftpState->user, "anonymous")) { + strCat(ftpState->title_url, ftpState->user); + strCat(ftpState->title_url, "@"); + } + strCat(ftpState->title_url, request->host); + if (request->port != urlDefaultPort(PROTO_FTP)) { + strCat(ftpState->title_url, ":"); + strCat(ftpState->title_url, xitoa(request->port)); + } + strCat(ftpState->title_url, strBuf(request->urlpath)); + + stringReset(&ftpState->base_href, "ftp://"); + if (strcmp(ftpState->user, "anonymous") != 0) { + strCat(ftpState->base_href, rfc1738_escape_part(ftpState->user)); + if (ftpState->password_url) { + strCat(ftpState->base_href, ":"); + strCat(ftpState->base_href, rfc1738_escape_part(ftpState->password)); + } + strCat(ftpState->base_href, "@"); + } + strCat(ftpState->base_href, request->host); + if (request->port != urlDefaultPort(PROTO_FTP)) { + strCat(ftpState->base_href, ":"); + strCat(ftpState->base_href, xitoa(request->port)); + } + strCat(ftpState->base_href, strBuf(request->urlpath)); + strCat(ftpState->base_href, "/"); +} + +CBDATA_TYPE(FtpStateData); +void +ftpStart(FwdState * fwd) +{ + request_t *request = fwd->request; + StoreEntry *entry = fwd->entry; + int fd = fwd->server_fd; + LOCAL_ARRAY(char, realm, 8192); + const char *url = storeUrl(entry); + FtpStateData *ftpState; + HttpReply *reply; + + CBDATA_INIT_TYPE(FtpStateData); + ftpState = cbdataAlloc(FtpStateData); + debug(9, 3) ("ftpStart: '%s'\n", url); + statCounter.server.all.requests++; + statCounter.server.ftp.requests++; + storeLockObject(entry); + ftpState->entry = entry; + ftpState->request = requestLink(request); + ftpState->ctrl.fd = fd; + ftpState->data.fd = -1; + ftpState->size = -1; + ftpState->mdtm = -1; + if (!Config.Ftp.passive) + ftpState->flags.rest_supported = 0; + else if (fwd->flags.ftp_pasv_failed) + ftpState->flags.pasv_supported = 0; + else + ftpState->flags.pasv_supported = 1; + ftpState->flags.rest_supported = 1; + ftpState->fwd = fwd; + comm_add_close_handler(fd, ftpStateFree, ftpState); + if (ftpState->request->method == METHOD_PUT) + ftpState->flags.put = 1; + if (!ftpCheckAuth(ftpState, &request->header)) { + /* This request is not fully authenticated */ + if (request->port == 21) { + snprintf(realm, 8192, "ftp %s", ftpState->user); + } else { + snprintf(realm, 8192, "ftp %s port %d", + ftpState->user, request->port); + } + /* create reply */ + reply = entry->mem_obj->reply; + assert(reply != NULL); + /* create appropriate reply */ + ftpAuthRequired(reply, request, realm); + httpReplySwapOut(reply, entry); + fwdComplete(ftpState->fwd); + comm_close(fd); + return; + } + ftpCheckUrlpath(ftpState); + ftpBuildTitleUrl(ftpState); + debug(9, 5) ("ftpStart: host=%s, path=%s, user=%s, passwd=%s\n", + ftpState->request->host, strBuf(ftpState->request->urlpath), + ftpState->user, ftpState->password); + ftpState->state = BEGIN; + ftpState->ctrl.last_command = xstrdup("Connect to server"); + ftpState->ctrl.buf = (char *)memAllocBuf(4096, &ftpState->ctrl.size); + ftpState->ctrl.offset = 0; + ftpState->data.buf = (char *)memAllocBuf(SQUID_TCP_SO_RCVBUF, &ftpState->data.size); + ftpScheduleReadControlReply(ftpState, 0); +} + +/* ====================================================================== */ + +static void +ftpWriteCommand(const char *buf, FtpStateData * ftpState) +{ + debug(9, 5) ("ftpWriteCommand: %s\n", buf); + safe_free(ftpState->ctrl.last_command); + safe_free(ftpState->ctrl.last_reply); + ftpState->ctrl.last_command = xstrdup(buf); + comm_write(ftpState->ctrl.fd, + xstrdup(buf), + strlen(buf), + ftpWriteCommandCallback, + ftpState, + xfree); + ftpScheduleReadControlReply(ftpState, 0); +} + +static void +ftpWriteCommandCallback(int fd, char *bufnotused, size_t size, comm_err_t errflag, void *data) +{ + FtpStateData *ftpState = (FtpStateData *)data; + debug(9, 7) ("ftpWriteCommandCallback: wrote %d bytes\n", (int) size); + if (size > 0) { + fd_bytes(fd, size, FD_WRITE); + kb_incr(&statCounter.server.all.kbytes_out, size); + kb_incr(&statCounter.server.ftp.kbytes_out, size); + } + if (errflag == COMM_ERR_CLOSING) + return; + if (errflag) { + debug(9, 1) ("ftpWriteCommandCallback: FD %d: %s\n", fd, xstrerror()); + ftpFailed(ftpState, ERR_WRITE_ERROR); + /* ftpFailed closes ctrl.fd and frees ftpState */ + return; + } +} + +static wordlist * +ftpParseControlReply(char *buf, size_t len, int *codep, int *used) +{ + char *s; + char *sbuf; + char *end; + int usable; + int complete = 0; + wordlist *head = NULL; + wordlist *list; + wordlist **tail = &head; + off_t offset; + size_t linelen; + int code = -1; + debug(9, 5) ("ftpParseControlReply\n"); + /* + * We need a NULL-terminated buffer for scanning, ick + */ + sbuf = (char *)xmalloc(len + 1); + xstrncpy(sbuf, buf, len + 1); + end = sbuf + len - 1; + while (*end != '\r' && *end != '\n' && end > sbuf) + end--; + usable = end - sbuf; + debug(9, 3) ("ftpParseControlReply: usable = %d\n", usable); + if (usable == 0) { + debug(9, 3) ("ftpParseControlReply: didn't find end of line\n"); + safe_free(sbuf); + return NULL; + } + debug(9, 3) ("ftpParseControlReply: %d bytes to play with\n", (int) len); + end++; + s = sbuf; + s += strspn(s, crlf); + for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { + if (complete) + break; + debug(9, 3) ("ftpParseControlReply: s = {%s}\n", s); + linelen = strcspn(s, crlf) + 1; + if (linelen < 2) + break; + if (linelen > 3) + complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' '); + if (complete) + code = atoi(s); + offset = 0; + if (linelen > 3) + if (*s >= '0' && *s <= '9' && (*(s + 3) == '-' || *(s + 3) == ' ')) + offset = 4; + list = (wordlist *)memAllocate(MEM_WORDLIST); + list->key = (char *)xmalloc(linelen - offset); + xstrncpy(list->key, s + offset, linelen - offset); + debug(9, 7) ("%d %s\n", code, list->key); + *tail = list; + tail = &list->next; + } + *used = (int) (s - sbuf); + safe_free(sbuf); + if (!complete) + wordlistDestroy(&head); + if (codep) + *codep = code; + return head; +} + +static void +ftpScheduleReadControlReply(FtpStateData * ftpState, int buffered_ok) +{ + debug(9, 3) ("ftpScheduleReadControlReply: FD %d\n", ftpState->ctrl.fd); + if (buffered_ok && ftpState->ctrl.offset > 0) { + /* We've already read some reply data */ + ftpHandleControlReply(ftpState); + } else { + commSetSelect(ftpState->ctrl.fd, + COMM_SELECT_READ, + ftpReadControlReply, + ftpState, + Config.Timeout.read); + /* + * Cancel the timeout on the Data socket (if any) and + * establish one on the control socket. + */ + if (ftpState->data.fd > -1) + commSetTimeout(ftpState->data.fd, -1, NULL, NULL); + commSetTimeout(ftpState->ctrl.fd, Config.Timeout.read, ftpTimeout, + ftpState); + } +} + +static void +ftpReadControlReply(int fd, void *data) +{ + FtpStateData *ftpState = (FtpStateData *)data; + StoreEntry *entry = ftpState->entry; + size_t len; + debug(9, 5) ("ftpReadControlReply\n"); + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + comm_close(ftpState->ctrl.fd); + return; + } + assert(ftpState->ctrl.offset < (off_t)ftpState->ctrl.size); + statCounter.syscalls.sock.reads++; + len = FD_READ_METHOD(fd, + ftpState->ctrl.buf + ftpState->ctrl.offset, + ftpState->ctrl.size - ftpState->ctrl.offset); + if (len > 0) { + fd_bytes(fd, len, FD_READ); + kb_incr(&statCounter.server.all.kbytes_in, len); + kb_incr(&statCounter.server.ftp.kbytes_in, len); + } + debug(9, 5) ("ftpReadControlReply: FD %d, Read %d bytes\n", fd, len); + if (len < 0) { + debug(50, ignoreErrno(errno) ? 3 : 1) ("ftpReadControlReply: read error: %s\n", xstrerror()); + if (ignoreErrno(errno)) { + ftpScheduleReadControlReply(ftpState, 0); + } else { + ftpFailed(ftpState, ERR_READ_ERROR); + /* ftpFailed closes ctrl.fd and frees ftpState */ + return; + } + return; + } + if (len == 0) { + if (entry->store_status == STORE_PENDING) { + ftpFailed(ftpState, ERR_FTP_FAILURE); + /* ftpFailed closes ctrl.fd and frees ftpState */ + return; + } + comm_close(ftpState->ctrl.fd); + return; + } + len += ftpState->ctrl.offset; + ftpState->ctrl.offset = len; + assert(len <= ftpState->ctrl.size); + ftpHandleControlReply(ftpState); +} + +static void +ftpHandleControlReply(FtpStateData * ftpState) +{ + wordlist **W; + int bytes_used = 0; + wordlistDestroy(&ftpState->ctrl.message); + ftpState->ctrl.message = ftpParseControlReply(ftpState->ctrl.buf, + ftpState->ctrl.offset, &ftpState->ctrl.replycode, &bytes_used); + if (ftpState->ctrl.message == NULL) { + /* didn't get complete reply yet */ + if (ftpState->ctrl.offset == (off_t)ftpState->ctrl.size) { + ftpState->ctrl.buf = (char *)memReallocBuf(ftpState->ctrl.buf, ftpState->ctrl.size << 1, &ftpState->ctrl.size); + } + ftpScheduleReadControlReply(ftpState, 0); + return; + } else if (ftpState->ctrl.offset == bytes_used) { + /* used it all up */ + ftpState->ctrl.offset = 0; + } else { + /* Got some data past the complete reply */ + assert(bytes_used < ftpState->ctrl.offset); + ftpState->ctrl.offset -= bytes_used; + xmemmove(ftpState->ctrl.buf, ftpState->ctrl.buf + bytes_used, + ftpState->ctrl.offset); + } + /* Move the last line of the reply message to ctrl.last_reply */ + for (W = &ftpState->ctrl.message; (*W)->next; W = &(*W)->next); + safe_free(ftpState->ctrl.last_reply); + ftpState->ctrl.last_reply = xstrdup((*W)->key); + wordlistDestroy(W); + /* Copy the rest of the message to cwd_message to be printed in + * error messages + */ + wordlistAddWl(&ftpState->cwd_message, ftpState->ctrl.message); + debug(9, 8) ("ftpHandleControlReply: state=%d, code=%d\n", ftpState->state, + ftpState->ctrl.replycode); + FTP_SM_FUNCS[ftpState->state] (ftpState); +} + +/* ====================================================================== */ + +static void +ftpReadWelcome(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("ftpReadWelcome\n"); + if (ftpState->flags.pasv_only) + ftpState->login_att++; + /* Dont retry if the FTP server accepted the connection */ + ftpState->fwd->flags.dont_retry = 1; + if (code == 220) { + if (ftpState->ctrl.message) { + if (strstr(ftpState->ctrl.message->key, "NetWare")) + ftpState->flags.skip_whitespace = 1; + } + ftpSendUser(ftpState); + } else if (code == 120) { + if (NULL != ftpState->ctrl.message) + debug(9, 3) ("FTP server is busy: %s\n", + ftpState->ctrl.message->key); + return; + } else { + ftpFail(ftpState); + } +} + +static void +ftpSendUser(FtpStateData * ftpState) +{ + if (ftpState->proxy_host != NULL) + snprintf(cbuf, 1024, "USER %s@%s\r\n", + ftpState->user, + ftpState->request->host); + else + snprintf(cbuf, 1024, "USER %s\r\n", ftpState->user); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_USER; +} + +static void +ftpReadUser(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("ftpReadUser\n"); + if (code == 230) { + ftpReadPass(ftpState); + } else if (code == 331) { + ftpSendPass(ftpState); + } else { + ftpFail(ftpState); + } +} + +static void +ftpSendPass(FtpStateData * ftpState) +{ + snprintf(cbuf, 1024, "PASS %s\r\n", ftpState->password); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_PASS; +} + +static void +ftpReadPass(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("ftpReadPass\n"); + if (code == 230) { + ftpSendType(ftpState); + } else { + ftpFail(ftpState); + } +} + +static void +ftpSendType(FtpStateData * ftpState) +{ + const char *t; + const char *filename; + char mode; + /* + * Ref section 3.2.2 of RFC 1738 + */ + mode = ftpState->typecode; + switch (mode) { + case 'D': + mode = 'A'; + break; + case 'A': + case 'I': + break; + default: + if (ftpState->flags.isdir) { + mode = 'A'; + } else { + t = strRChr(ftpState->request->urlpath, '/'); + filename = t ? t + 1 : strBuf(ftpState->request->urlpath); + mode = mimeGetTransferMode(filename); + } + break; + } + if (mode == 'I') + ftpState->flags.binary = 1; + else + ftpState->flags.binary = 0; + snprintf(cbuf, 1024, "TYPE %c\r\n", mode); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_TYPE; +} + +static void +ftpReadType(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + char *path; + char *d, *p; + debug(9, 3) ("This is ftpReadType\n"); + if (code == 200) { + p = path = xstrdup(strBuf(ftpState->request->urlpath)); + if (*p == '/') + p++; + while (*p) { + d = p; + p += strcspn(p, "/"); + if (*p) + *p++ = '\0'; + rfc1738_unescape(d); + wordlistAdd(&ftpState->pathcomps, d); + } + xfree(path); + if (ftpState->pathcomps) + ftpTraverseDirectory(ftpState); + else + ftpListDir(ftpState); + } else { + ftpFail(ftpState); + } +} + +static void +ftpTraverseDirectory(FtpStateData * ftpState) +{ + wordlist *w; + debug(9, 4) ("ftpTraverseDirectory %s\n", + ftpState->filepath ? ftpState->filepath : ""); + + safe_free(ftpState->filepath); + /* Done? */ + if (ftpState->pathcomps == NULL) { + debug(9, 3) ("the final component was a directory\n"); + ftpListDir(ftpState); + return; + } + /* Go to next path component */ + w = ftpState->pathcomps; + ftpState->filepath = w->key; + ftpState->pathcomps = w->next; + memFree(w, MEM_WORDLIST); + /* Check if we are to CWD or RETR */ + if (ftpState->pathcomps != NULL || ftpState->flags.isdir) { + ftpSendCwd(ftpState); + } else { + debug(9, 3) ("final component is probably a file\n"); + ftpGetFile(ftpState); + return; + } +} + +static void +ftpSendCwd(FtpStateData * ftpState) +{ + char *path = ftpState->filepath; + debug(9, 3) ("ftpSendCwd\n"); + if (!strcmp(path, "..") || !strcmp(path, "/")) { + ftpState->flags.no_dotdot = 1; + } else { + ftpState->flags.no_dotdot = 0; + } + if (*path) + snprintf(cbuf, 1024, "CWD %s\r\n", path); + else + snprintf(cbuf, 1024, "CWD\r\n"); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_CWD; +} + +static void +ftpReadCwd(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadCwd\n"); + if (code >= 200 && code < 300) { + /* CWD OK */ + ftpUnhack(ftpState); + /* Reset cwd_message to only include the last message */ + if (ftpState->cwd_message) + wordlistDestroy(&ftpState->cwd_message); + ftpState->cwd_message = ftpState->ctrl.message; + ftpState->ctrl.message = NULL; + /* Continue to traverse the path */ + ftpTraverseDirectory(ftpState); + } else { + /* CWD FAILED */ + if (!ftpState->flags.put) + ftpFail(ftpState); + else + ftpSendMkdir(ftpState); + } +} + +static void +ftpSendMkdir(FtpStateData * ftpState) +{ + char *path = ftpState->filepath; + debug(9, 3) ("ftpSendMkdir: with path=%s\n", path); + snprintf(cbuf, 1024, "MKD %s\r\n", path); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_MKDIR; +} + +static void +ftpReadMkdir(FtpStateData * ftpState) +{ + char *path = ftpState->filepath; + int code = ftpState->ctrl.replycode; + + debug(9, 3) ("ftpReadMkdir: path %s, code %d\n", path, code); + if (code == 257) { /* success */ + ftpSendCwd(ftpState); + } else if (code == 550) { /* dir exists */ + if (ftpState->flags.put_mkdir) { + ftpState->flags.put_mkdir = 1; + ftpSendCwd(ftpState); + } else + ftpSendReply(ftpState); + } else + ftpSendReply(ftpState); +} + +static void +ftpGetFile(FtpStateData * ftpState) +{ + assert(*ftpState->filepath != '\0'); + ftpState->flags.isdir = 0; + ftpSendMdtm(ftpState); +} + +static void +ftpListDir(FtpStateData * ftpState) +{ + if (!ftpState->flags.isdir) { + debug(9, 3) ("Directory path did not end in /\n"); + strCat(ftpState->title_url, "/"); + ftpState->flags.isdir = 1; + ftpState->flags.need_base_href = 1; + } + ftpSendPasv(ftpState); +} + +static void +ftpSendMdtm(FtpStateData * ftpState) +{ + assert(*ftpState->filepath != '\0'); + snprintf(cbuf, 1024, "MDTM %s\r\n", ftpState->filepath); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_MDTM; +} + +static void +ftpReadMdtm(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadMdtm\n"); + if (code == 213) { + ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply); + ftpUnhack(ftpState); + } else if (code < 0) { + ftpFail(ftpState); + } + ftpSendSize(ftpState); +} + +static void +ftpSendSize(FtpStateData * ftpState) +{ + /* Only send SIZE for binary transfers. The returned size + * is useless on ASCII transfers */ + if (ftpState->flags.binary) { + assert(ftpState->filepath != NULL); + assert(*ftpState->filepath != '\0'); + snprintf(cbuf, 1024, "SIZE %s\r\n", ftpState->filepath); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_SIZE; + } else + /* Skip to next state no non-binary transfers */ + ftpSendPasv(ftpState); +} + +static void +ftpReadSize(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadSize\n"); + if (code == 213) { + ftpUnhack(ftpState); + ftpState->size = atoi(ftpState->ctrl.last_reply); + if (ftpState->size == 0) { + debug(9, 2) ("ftpReadSize: SIZE reported %s on %s\n", + ftpState->ctrl.last_reply, + strBuf(ftpState->title_url)); + ftpState->size = -1; + } + } else if (code < 0) { + ftpFail(ftpState); + } + ftpSendPasv(ftpState); +} + +static void +ftpSendPasv(FtpStateData * ftpState) +{ + int fd; + struct sockaddr_in addr; + socklen_t addr_len; + if (ftpState->request->method == METHOD_HEAD) { + /* Terminate here for HEAD requests */ + ftpAppendSuccessHeader(ftpState); + storeTimestampsSet(ftpState->entry); + /* + * On rare occasions I'm seeing the entry get aborted after + * ftpReadControlReply() and before here, probably when + * trying to write to the client. + */ + if (!EBIT_TEST(ftpState->entry->flags, ENTRY_ABORTED)) + fwdComplete(ftpState->fwd); + ftpSendQuit(ftpState); + return; + } + if (ftpState->data.fd >= 0) { + if (!ftpState->flags.datachannel_hack) { + /* We are already connected, reuse this connection. */ + ftpRestOrList(ftpState); + return; + } else { + /* Close old connection */ + comm_close(ftpState->data.fd); + ftpState->data.fd = -1; + } + } + if (!ftpState->flags.pasv_supported) { + ftpSendPort(ftpState); + return; + } + addr_len = sizeof(addr); + if (getsockname(ftpState->ctrl.fd, (struct sockaddr *) &addr, &addr_len)) { + debug(9, 0) ("ftpSendPasv: getsockname(%d,..): %s\n", + ftpState->ctrl.fd, xstrerror()); + ftpFail(ftpState); + return; + } + /* Open data channel with the same local address as control channel */ + fd = comm_open(SOCK_STREAM, + 0, + addr.sin_addr, + 0, + COMM_NONBLOCKING, + storeUrl(ftpState->entry)); + debug(9, 3) ("ftpSendPasv: Unconnected data socket created on FD %d\n", fd); + if (fd < 0) { + ftpFail(ftpState); + return; + } + /* + * No comm_add_close_handler() here. If we have both ctrl and + * data FD's call ftpStateFree() upon close, then we have + * to delete the close handler which did NOT get called + * to prevent ftpStateFree() getting called twice. + * Instead we'll always call comm_close() on the ctrl FD. + */ + ftpState->data.fd = fd; + snprintf(cbuf, 1024, "PASV\r\n"); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_PASV; + /* + * ugly hack for ftp servers like ftp.netscape.com that sometimes + * dont acknowledge PORT commands. + */ + commSetTimeout(ftpState->data.fd, 15, ftpTimeout, ftpState); +} + +static void +ftpReadPasv(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + int h1, h2, h3, h4; + int p1, p2; + int n; + u_short port; + int fd = ftpState->data.fd; + char *buf; + LOCAL_ARRAY(char, ipaddr, 1024); + debug(9, 3) ("This is ftpReadPasv\n"); + if (code != 227) { + debug(9, 3) ("PASV not supported by remote end\n"); + ftpSendPort(ftpState); + return; + } + /* 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). */ + /* ANSI sez [^0-9] is undefined, it breaks on Watcom cc */ + debug(9, 5) ("scanning: %s\n", ftpState->ctrl.last_reply); + buf = strstr(ftpState->ctrl.last_reply, "("); + if (!buf) { + debug(9, 1) ("Unsafe PASV reply from %s: '%s'\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); + ftpSendPort(ftpState); + return; + } + buf++; /* skip ( */ + n = sscanf(buf, "%d,%d,%d,%d,%d,%d", &h1, &h2, &h3, &h4, &p1, &p2); + if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) { + debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); + ftpSendPort(ftpState); + return; + } + snprintf(ipaddr, 1024, "%d.%d.%d.%d", h1, h2, h3, h4); + if (!safe_inet_addr(ipaddr, NULL)) { + debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); + ftpSendPort(ftpState); + return; + } + port = ((p1 << 8) + p2); + if (0 == port) { + debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); + ftpSendPort(ftpState); + return; + } + if (Config.Ftp.sanitycheck) { + if (strcmp(fd_table[ftpState->ctrl.fd].ipaddr, ipaddr) != 0) { + debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); + ftpSendPort(ftpState); + return; + } + if (port < 1024) { + debug(9, 1) ("Unsafe PASV reply from %s: %s\n", fd_table[ftpState->ctrl.fd].ipaddr, ftpState->ctrl.last_reply); + ftpSendPort(ftpState); + return; + } + } + debug(9, 5) ("ftpReadPasv: connecting to %s, port %d\n", ipaddr, port); + ftpState->data.port = port; + ftpState->data.host = xstrdup(ipaddr); + safe_free(ftpState->ctrl.last_command); + safe_free(ftpState->ctrl.last_reply); + ftpState->ctrl.last_command = xstrdup("Connect to server data port"); + commConnectStart(fd, ipaddr, port, ftpPasvCallback, ftpState); +} + +static void +ftpPasvCallback(int fd, comm_err_t status, void *data) +{ + FtpStateData *ftpState = (FtpStateData *)data; + debug(9, 3) ("ftpPasvCallback\n"); + if (status != COMM_OK) { + debug(9, 2) ("ftpPasvCallback: failed to connect. Retrying without PASV.\n"); + ftpState->fwd->flags.dont_retry = 0; /* this is a retryable error */ + ftpState->fwd->flags.ftp_pasv_failed = 1; + ftpFailed(ftpState, ERR_NONE); + /* ftpFailed closes ctrl.fd and frees ftpState */ + return; + } + ftpRestOrList(ftpState); +} + +static int +ftpOpenListenSocket(FtpStateData * ftpState, int fallback) +{ + int fd; + struct sockaddr_in addr; + socklen_t addr_len; + int on = 1; + u_short port = 0; + /* + * Tear down any old data connection if any. We are about to + * establish a new one. + */ + if (ftpState->data.fd > 0) { + comm_close(ftpState->data.fd); + ftpState->data.fd = -1; + } + /* + * Set up a listen socket on the same local address as the + * control connection. + */ + addr_len = sizeof(addr); + if (getsockname(ftpState->ctrl.fd, (struct sockaddr *) &addr, &addr_len)) { + debug(9, 0) ("ftpOpenListenSocket: getsockname(%d,..): %s\n", + ftpState->ctrl.fd, xstrerror()); + return -1; + } + /* + * REUSEADDR is needed in fallback mode, since the same port is + * used for both control and data. + */ + if (fallback) { + setsockopt(ftpState->ctrl.fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on)); + port = ntohs(addr.sin_port); + } + fd = comm_open(SOCK_STREAM, + 0, + addr.sin_addr, + port, + COMM_NONBLOCKING | (fallback ? COMM_REUSEADDR : 0), + storeUrl(ftpState->entry)); + debug(9, 3) ("ftpOpenListenSocket: Unconnected data socket created on FD %d\n", fd); + if (fd < 0) { + debug(9, 0) ("ftpOpenListenSocket: comm_open failed\n"); + return -1; + } + if (comm_listen(fd) < 0) { + comm_close(fd); + return -1; + } + ftpState->data.fd = fd; + ftpState->data.port = comm_local_port(fd); + ftpState->data.host = NULL; + return fd; +} + +static void +ftpSendPort(FtpStateData * ftpState) +{ + int fd; + struct sockaddr_in addr; + socklen_t addr_len; + unsigned char *addrptr; + unsigned char *portptr; + debug(9, 3) ("This is ftpSendPort\n"); + ftpState->flags.pasv_supported = 0; + fd = ftpOpenListenSocket(ftpState, 0); + addr_len = sizeof(addr); + if (getsockname(fd, (struct sockaddr *) &addr, &addr_len)) { + debug(9, 0) ("ftpSendPort: getsockname(%d,..): %s\n", fd, xstrerror()); + /* XXX Need to set error message */ + ftpFail(ftpState); + return; + } + addrptr = (unsigned char *) &addr.sin_addr.s_addr; + portptr = (unsigned char *) &addr.sin_port; + snprintf(cbuf, 1024, "PORT %d,%d,%d,%d,%d,%d\r\n", + addrptr[0], addrptr[1], addrptr[2], addrptr[3], + portptr[0], portptr[1]); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_PORT; +} + +static void +ftpReadPort(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadPort\n"); + if (code != 200) { + /* Fall back on using the same port as the control connection */ + debug(9, 3) ("PORT not supported by remote end\n"); + ftpOpenListenSocket(ftpState, 1); + } + ftpRestOrList(ftpState); +} + +/* "read" handler to accept data connection */ +static void +ftpAcceptDataConnection(int fd, void *data) +{ + FtpStateData *ftpState = (FtpStateData *)data; + struct sockaddr_in my_peer, me; + debug(9, 3) ("ftpAcceptDataConnection\n"); + + if (EBIT_TEST(ftpState->entry->flags, ENTRY_ABORTED)) { + comm_close(ftpState->ctrl.fd); + return; + } + fd = comm_accept(fd, &my_peer, &me); + if (Config.Ftp.sanitycheck) { + char *ipaddr = inet_ntoa(my_peer.sin_addr); + if (strcmp(fd_table[ftpState->ctrl.fd].ipaddr, ipaddr) != 0) { + debug(9, 1) ("FTP data connection from unexpected server (%s:%d), expecting %s\n", ipaddr, (int) ntohs(my_peer.sin_port), fd_table[ftpState->ctrl.fd].ipaddr); + comm_close(fd); + commSetSelect(ftpState->data.fd, + COMM_SELECT_READ, + ftpAcceptDataConnection, + ftpState, + 0); + return; + } + } + if (fd < 0) { + debug(9, 1) ("ftpHandleDataAccept: comm_accept(%d): %s", fd, xstrerror()); + /* XXX Need to set error message */ + ftpFail(ftpState); + return; + } + /* Replace the Listen socket with the accepted data socket */ + comm_close(ftpState->data.fd); + debug(9, 3) ("ftpAcceptDataConnection: Connected data socket on FD %d\n", fd); + ftpState->data.fd = fd; + ftpState->data.port = ntohs(my_peer.sin_port); + ftpState->data.host = xstrdup(inet_ntoa(my_peer.sin_addr)); + commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); + commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, + ftpState); + /* XXX We should have a flag to track connect state... + * host NULL -> not connected, port == local port + * host set -> connected, port == remote port + */ + /* Restart state (SENT_NLST/LIST/RETR) */ + FTP_SM_FUNCS[ftpState->state] (ftpState); +} + +static void +ftpRestOrList(FtpStateData * ftpState) +{ + debug(9, 3) ("This is ftpRestOrList\n"); + if (ftpState->typecode == 'D') { + ftpState->flags.isdir = 1; + ftpState->flags.need_base_href = 1; + if (ftpState->flags.put) { + ftpSendMkdir(ftpState); /* PUT name;type=d */ + } else { + ftpSendNlst(ftpState); /* GET name;type=d sec 3.2.2 of RFC 1738 */ + } + } else if (ftpState->flags.put) { + debug(9, 3) ("ftpRestOrList: Sending STOR request...\n"); + ftpSendStor(ftpState); + } else if (ftpState->flags.isdir) + ftpSendList(ftpState); + else if (ftpRestartable(ftpState)) + ftpSendRest(ftpState); + else + ftpSendRetr(ftpState); +} + +static void +ftpSendStor(FtpStateData * ftpState) +{ + if (ftpState->filepath != NULL) { + /* Plain file upload */ + snprintf(cbuf, 1024, "STOR %s\r\n", ftpState->filepath); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_STOR; + } else if (httpHeaderGetInt(&ftpState->request->header, HDR_CONTENT_LENGTH) > 0) { + /* File upload without a filename. use STOU to generate one */ + snprintf(cbuf, 1024, "STOU\r\n"); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_STOR; + } else { + /* No file to transfer. Only create directories if needed */ + ftpSendReply(ftpState); + } +} + +static void +ftpReadStor(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadStor\n"); + if (code == 125 || (code == 150 && ftpState->data.host)) { + /* Begin data transfer */ + debug(9, 3) ("ftpReadStor: starting data transfer\n"); + commSetSelect(ftpState->data.fd, + COMM_SELECT_WRITE, + ftpDataWrite, + ftpState, + Config.Timeout.read); + /* + * Cancel the timeout on the Control socket and + * establish one on the data socket. + */ + commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); + commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, + ftpState); + ftpState->state = WRITING_DATA; + debug(9, 3) ("ftpReadStor: writing data channel\n"); + } else if (code == 150) { + /* Accept data channel */ + debug(9, 3) ("ftpReadStor: accepting data channel\n"); + commSetSelect(ftpState->data.fd, + COMM_SELECT_READ, + ftpAcceptDataConnection, + ftpState, + 0); + } else { + debug(9, 3) ("ftpReadStor: Unexpected reply code %03d\n", code); + ftpFail(ftpState); + } +} + +static void +ftpSendRest(FtpStateData * ftpState) +{ + snprintf(cbuf, 1024, "REST %d\r\n", ftpState->restart_offset); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_REST; +} + +static int +ftpRestartable(FtpStateData * ftpState) +{ + if (ftpState->restart_offset > 0) + return 1; + if (!ftpState->request->range) + return 0; + if (!ftpState->flags.binary) + return 0; + if (ftpState->size <= 0) + return 0; + + ftpState->restart_offset = httpHdrRangeLowestOffset(ftpState->request->range, (size_t) ftpState->size); + if (ftpState->restart_offset <= 0) + return 0; + return 1; +} + +static void +ftpReadRest(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadRest\n"); + assert(ftpState->restart_offset > 0); + if (code == 350) { + ftpState->restarted_offset = ftpState->restart_offset; + ftpSendRetr(ftpState); + } else if (code > 0) { + debug(9, 3) ("ftpReadRest: REST not supported\n"); + ftpState->flags.rest_supported = 0; + ftpSendRetr(ftpState); + } else { + ftpFail(ftpState); + } +} + +static void +ftpSendList(FtpStateData * ftpState) +{ + if (ftpState->filepath) { + ftpState->flags.need_base_href = 1; + snprintf(cbuf, 1024, "LIST %s\r\n", ftpState->filepath); + } else { + snprintf(cbuf, 1024, "LIST\r\n"); + } + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_LIST; +} + +static void +ftpSendNlst(FtpStateData * ftpState) +{ + ftpState->flags.tried_nlst = 1; + if (ftpState->filepath) { + ftpState->flags.need_base_href = 1; + snprintf(cbuf, 1024, "NLST %s\r\n", ftpState->filepath); + } else { + snprintf(cbuf, 1024, "NLST\r\n"); + } + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_NLST; +} + +static void +ftpReadList(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadList\n"); + if (code == 125 || (code == 150 && ftpState->data.host)) { + /* Begin data transfer */ + ftpAppendSuccessHeader(ftpState); + commSetSelect(ftpState->data.fd, + COMM_SELECT_READ, + ftpDataRead, + ftpState, + Config.Timeout.read); + commSetDefer(ftpState->data.fd, fwdCheckDeferRead, ftpState->entry); + ftpState->state = READING_DATA; + /* + * Cancel the timeout on the Control socket and establish one + * on the data socket + */ + commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); + commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, ftpState); + return; + } else if (code == 150) { + /* Accept data channel */ + commSetSelect(ftpState->data.fd, + COMM_SELECT_READ, + ftpAcceptDataConnection, + ftpState, + 0); + /* + * Cancel the timeout on the Control socket and establish one + * on the data socket + */ + commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); + commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, ftpState); + return; + } else if (!ftpState->flags.tried_nlst && code > 300) { + ftpSendNlst(ftpState); + } else { + ftpFail(ftpState); + return; + } +} + +static void +ftpSendRetr(FtpStateData * ftpState) +{ + assert(ftpState->filepath != NULL); + snprintf(cbuf, 1024, "RETR %s\r\n", ftpState->filepath); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_RETR; +} + +static void +ftpReadRetr(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadRetr\n"); + if (code == 125 || (code == 150 && ftpState->data.host)) { + /* Begin data transfer */ + debug(9, 3) ("ftpReadRetr: reading data channel\n"); + ftpAppendSuccessHeader(ftpState); + commSetSelect(ftpState->data.fd, + COMM_SELECT_READ, + ftpDataRead, + ftpState, + Config.Timeout.read); + commSetDefer(ftpState->data.fd, fwdCheckDeferRead, ftpState->entry); + ftpState->state = READING_DATA; + /* + * Cancel the timeout on the Control socket and establish one + * on the data socket + */ + commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); + commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, + ftpState); + } else if (code == 150) { + /* Accept data channel */ + commSetSelect(ftpState->data.fd, + COMM_SELECT_READ, + ftpAcceptDataConnection, + ftpState, + 0); + /* + * Cancel the timeout on the Control socket and establish one + * on the data socket + */ + commSetTimeout(ftpState->ctrl.fd, -1, NULL, NULL); + commSetTimeout(ftpState->data.fd, Config.Timeout.read, ftpTimeout, + ftpState); + } else if (code >= 300) { + if (!ftpState->flags.try_slash_hack) { + /* Try this as a directory missing trailing slash... */ + ftpHackShortcut(ftpState, ftpSendCwd); + } else { + ftpFail(ftpState); + } + } else { + ftpFail(ftpState); + } +} + +static void +ftpReadTransferDone(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpReadTransferDone\n"); + if (code == 226) { + /* Connection closed; retrieval done. */ + if (ftpState->flags.html_header_sent) + ftpListingFinish(ftpState); + fwdUnregister(ftpState->ctrl.fd, ftpState->fwd); + fwdComplete(ftpState->fwd); + ftpSendQuit(ftpState); + } else { /* != 226 */ + debug(9, 1) ("ftpReadTransferDone: Got code %d after reading data\n", + code); + ftpFailed(ftpState, ERR_FTP_FAILURE); + /* ftpFailed closes ctrl.fd and frees ftpState */ + return; + } +} + +/* This will be called when there is data available to put */ +static void +ftpRequestBody(char *buf, ssize_t size, void *data) +{ + FtpStateData *ftpState = (FtpStateData *) data; + debug(9, 3) ("ftpRequestBody: buf=%p size=%d ftpState=%p\n", buf, (int) size, data); + ftpState->data.offset = size; + if (size > 0) { + /* DataWrite */ + comm_write(ftpState->data.fd, buf, size, ftpDataWriteCallback, ftpState, NULL); + } else if (size < 0) { + /* Error */ + debug(9, 1) ("ftpRequestBody: request aborted"); + ftpFailed(ftpState, ERR_READ_ERROR); + } else if (size == 0) { + /* End of transfer */ + ftpDataComplete(ftpState); + } +} + +/* This will be called when the put write is completed */ +static void +ftpDataWriteCallback(int fd, char *buf, size_t size, comm_err_t err, void *data) +{ + FtpStateData *ftpState = (FtpStateData *) data; + if (!err) { + /* Shedule the rest of the request */ + clientReadBody(ftpState->request, ftpState->data.buf, ftpState->data.size, ftpRequestBody, ftpState); + } else { + debug(9, 1) ("ftpDataWriteCallback: write error: %s\n", xstrerror()); + ftpFailed(ftpState, ERR_WRITE_ERROR); + } +} + +static void +ftpDataWrite(int ftp, void *data) +{ + FtpStateData *ftpState = (FtpStateData *) data; + debug(9, 3) ("ftpDataWrite\n"); + /* This starts the body transfer */ + clientReadBody(ftpState->request, ftpState->data.buf, ftpState->data.size, ftpRequestBody, ftpState); +} + +static void +ftpWriteTransferDone(FtpStateData * ftpState) +{ + int code = ftpState->ctrl.replycode; + debug(9, 3) ("This is ftpWriteTransferDone\n"); + if (code != 226) { + debug(9, 1) ("ftpReadTransferDone: Got code %d after sending data\n", + code); + ftpFailed(ftpState, ERR_FTP_PUT_ERROR); + return; + } + storeTimestampsSet(ftpState->entry); /* XXX Is this needed? */ + ftpSendReply(ftpState); +} + +static void +ftpSendQuit(FtpStateData * ftpState) +{ + assert(ftpState->ctrl.fd > -1); + snprintf(cbuf, 1024, "QUIT\r\n"); + ftpWriteCommand(cbuf, ftpState); + ftpState->state = SENT_QUIT; +} + +static void +ftpReadQuit(FtpStateData * ftpState) +{ + comm_close(ftpState->ctrl.fd); +} + +static void +ftpTrySlashHack(FtpStateData * ftpState) +{ + char *path; + ftpState->flags.try_slash_hack = 1; + /* Free old paths */ + if (ftpState->pathcomps) + wordlistDestroy(&ftpState->pathcomps); + safe_free(ftpState->filepath); + /* Build the new path (urlpath begins with /) */ + path = xstrdup(strBuf(ftpState->request->urlpath)); + rfc1738_unescape(path); + ftpState->filepath = path; + /* And off we go */ + ftpGetFile(ftpState); +} + +static void +ftpTryDatachannelHack(FtpStateData * ftpState) +{ + ftpState->flags.datachannel_hack = 1; + /* we have to undo some of the slash hack... */ + if (ftpState->old_filepath != NULL) { + ftpState->flags.try_slash_hack = 0; + safe_free(ftpState->filepath); + ftpState->filepath = ftpState->old_filepath; + ftpState->old_filepath = NULL; + } + ftpState->flags.tried_nlst = 0; + /* And off we go */ + if (ftpState->flags.isdir) { + ftpListDir(ftpState); + } else { + ftpGetFile(ftpState); + } + return; +} + +/* Forget hack status. Next error is shown to the user */ +static void +ftpUnhack(FtpStateData * ftpState) +{ + if (ftpState->old_request != NULL) { + safe_free(ftpState->old_request); + safe_free(ftpState->old_reply); + } +} + +static void +ftpHackShortcut(FtpStateData * ftpState, FTPSM * nextState) +{ + /* Clear some unwanted state */ + ftpState->restarted_offset = 0; + ftpState->restart_offset = 0; + /* Save old error message & some state info */ + if (ftpState->old_request == NULL) { + ftpState->old_request = ftpState->ctrl.last_command; + ftpState->ctrl.last_command = NULL; + ftpState->old_reply = ftpState->ctrl.last_reply; + ftpState->ctrl.last_reply = NULL; + if (ftpState->pathcomps == NULL && ftpState->filepath != NULL) + ftpState->old_filepath = xstrdup(ftpState->filepath); + } + /* Jump to the "hack" state */ + nextState(ftpState); +} + +static void +ftpFail(FtpStateData * ftpState) +{ + debug(9, 3) ("ftpFail\n"); + /* Try the / hack to support "Netscape" FTP URL's for retreiving files */ + if (!ftpState->flags.isdir && /* Not a directory */ + !ftpState->flags.try_slash_hack && /* Not in slash hack */ + ftpState->mdtm <= 0 && ftpState->size < 0 && /* Not known as a file */ + strNCaseCmp(ftpState->request->urlpath, "/%2f", 4) != 0) { /* No slash encoded */ + switch (ftpState->state) { + case SENT_CWD: + case SENT_RETR: + /* Try the / hack */ + ftpHackShortcut(ftpState, ftpTrySlashHack); + return; + default: + break; + } + } + /* Try to reopen datachannel */ + if (!ftpState->flags.datachannel_hack && + ftpState->pathcomps == NULL) { + switch (ftpState->state) { + case SENT_RETR: + case SENT_LIST: + case SENT_NLST: + /* Try to reopen datachannel */ + ftpHackShortcut(ftpState, ftpTryDatachannelHack); + return; + default: + break; + } + } + ftpFailed(ftpState, ERR_NONE); + /* ftpFailed closes ctrl.fd and frees ftpState */ +} + +static void +ftpFailed(FtpStateData * ftpState, err_type error) +{ + StoreEntry *entry = ftpState->entry; + if (entry->mem_obj->inmem_hi == 0) + ftpFailedErrorMessage(ftpState, error); + if (ftpState->data.fd > -1) { + comm_close(ftpState->data.fd); + ftpState->data.fd = -1; + } + comm_close(ftpState->ctrl.fd); +} + +static void +ftpFailedErrorMessage(FtpStateData * ftpState, err_type error) +{ + ErrorState *err; + const char *command, *reply; + /* Translate FTP errors into HTTP errors */ + err = NULL; + switch (error) { + case ERR_NONE: + switch (ftpState->state) { + case SENT_USER: + case SENT_PASS: + if (ftpState->ctrl.replycode > 500) + err = errorCon(ERR_FTP_FORBIDDEN, HTTP_FORBIDDEN); + else if (ftpState->ctrl.replycode == 421) + err = errorCon(ERR_FTP_UNAVAILABLE, HTTP_SERVICE_UNAVAILABLE); + break; + case SENT_CWD: + case SENT_RETR: + if (ftpState->ctrl.replycode == 550) + err = errorCon(ERR_FTP_NOT_FOUND, HTTP_NOT_FOUND); + break; + default: + break; + } + break; + case ERR_READ_TIMEOUT: + err = errorCon(error, HTTP_GATEWAY_TIMEOUT); + break; + default: + err = errorCon(error, HTTP_BAD_GATEWAY); + break; + } + if (err == NULL) + err = errorCon(ERR_FTP_FAILURE, HTTP_BAD_GATEWAY); + err->xerrno = errno; + err->request = requestLink(ftpState->request); + err->ftp.server_msg = ftpState->ctrl.message; + ftpState->ctrl.message = NULL; + if (ftpState->old_request) + command = ftpState->old_request; + else + command = ftpState->ctrl.last_command; + if (command && strncmp(command, "PASS", 4) == 0) + command = "PASS "; + if (ftpState->old_reply) + reply = ftpState->old_reply; + else + reply = ftpState->ctrl.last_reply; + if (command) + err->ftp.request = xstrdup(command); + if (reply) + err->ftp.reply = xstrdup(reply); + fwdFail(ftpState->fwd, err); +} + +static void +ftpSendReply(FtpStateData * ftpState) +{ + ErrorState *err; + int code = ftpState->ctrl.replycode; + http_status http_code; + err_type err_code = ERR_NONE; + debug(9, 5) ("ftpSendReply: %s, code %d\n", + storeUrl(ftpState->entry), code); + if (cbdataReferenceValid(ftpState)) + debug(9, 5) ("ftpSendReply: ftpState (%p) is valid!\n", ftpState); + if (code == 226) { + err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED; + http_code = (ftpState->mdtm > 0) ? HTTP_ACCEPTED : HTTP_CREATED; + } else if (code == 227) { + err_code = ERR_FTP_PUT_CREATED; + http_code = HTTP_CREATED; + } else { + err_code = ERR_FTP_PUT_ERROR; + http_code = HTTP_INTERNAL_SERVER_ERROR; + } + err = errorCon(err_code, http_code); + err->request = requestLink(ftpState->request); + if (ftpState->old_request) + err->ftp.request = xstrdup(ftpState->old_request); + else + err->ftp.request = xstrdup(ftpState->ctrl.last_command); + if (ftpState->old_reply) + err->ftp.reply = xstrdup(ftpState->old_reply); + else if (ftpState->ctrl.last_reply) + err->ftp.reply = xstrdup(ftpState->ctrl.last_reply); + else + err->ftp.reply = xstrdup(""); + errorAppendEntry(ftpState->entry, err); + storeBufferFlush(ftpState->entry); + ftpSendQuit(ftpState); +} + +static void +ftpAppendSuccessHeader(FtpStateData * ftpState) +{ + const char *mime_type = NULL; + const char *mime_enc = NULL; + String urlpath = ftpState->request->urlpath; + const char *filename = NULL; + const char *t = NULL; + StoreEntry *e = ftpState->entry; + http_reply *reply = e->mem_obj->reply; + http_version_t version; + + if (ftpState->flags.http_header_sent) + return; + ftpState->flags.http_header_sent = 1; + assert(e->mem_obj->inmem_hi == 0); + EBIT_CLR(e->flags, ENTRY_FWD_HDR_WAIT); + filename = (t = strRChr(urlpath, '/')) ? t + 1 : strBuf(urlpath); + if (ftpState->flags.isdir) { + mime_type = "text/html"; + } else { + switch (ftpState->typecode) { + case 'I': + mime_type = "application/octet-stream"; + mime_enc = mimeGetContentEncoding(filename); + break; + case 'A': + mime_type = "text/plain"; + break; + default: + mime_type = mimeGetContentType(filename); + mime_enc = mimeGetContentEncoding(filename); + break; + } + } + storeBuffer(e); + httpReplyReset(reply); + /* set standard stuff */ + if (ftpState->restarted_offset) { + /* Partial reply */ + HttpHdrRangeSpec range_spec; + range_spec.offset = ftpState->restarted_offset; + range_spec.length = ftpState->size - ftpState->restarted_offset; + httpBuildVersion(&version, 1, 0); + httpReplySetHeaders(reply, version, HTTP_PARTIAL_CONTENT, "Gatewaying", + mime_type, ftpState->size - ftpState->restarted_offset, ftpState->mdtm, -2); + httpHeaderAddContRange(&reply->header, range_spec, ftpState->size); + } else { + /* Full reply */ + httpBuildVersion(&version, 1, 0); + httpReplySetHeaders(reply, version, HTTP_OK, "Gatewaying", + mime_type, ftpState->size, ftpState->mdtm, -2); + } + /* additional info */ + if (mime_enc) + httpHeaderPutStr(&reply->header, HDR_CONTENT_ENCODING, mime_enc); + httpReplySwapOut(reply, e); + storeBufferFlush(e); + reply->hdr_sz = e->mem_obj->inmem_hi; + storeTimestampsSet(e); + if (ftpState->flags.authenticated) { + /* + * Authenticated requests can't be cached. + */ + storeRelease(e); + } else if (EBIT_TEST(e->flags, ENTRY_CACHABLE) && !ftpState->restarted_offset) { + storeSetPublicKey(e); + } else { + storeRelease(e); + } +} + +static void +ftpAuthRequired(HttpReply * old_reply, request_t * request, const char *realm) +{ + ErrorState *err = errorCon(ERR_CACHE_ACCESS_DENIED, HTTP_UNAUTHORIZED); + HttpReply *rep; + err->request = requestLink(request); + rep = errorBuildReply(err); + errorStateFree(err); + /* add Authenticate header */ + httpHeaderPutAuth(&rep->header, "Basic", realm); + /* move new reply to the old one */ + httpReplyAbsorb(old_reply, rep); +} + +char * +ftpUrlWith2f(const request_t * request) +{ + LOCAL_ARRAY(char, buf, MAX_URL); + LOCAL_ARRAY(char, loginbuf, MAX_LOGIN_SZ + 1); + LOCAL_ARRAY(char, portbuf, 32); + char *t; + portbuf[0] = '\0'; + if (request->protocol != PROTO_FTP) + return NULL; + if (request->port != urlDefaultPort(request->protocol)) + snprintf(portbuf, 32, ":%d", request->port); + loginbuf[0] = '\0'; + if ((int) strlen(request->login) > 0) { + xstrncpy(loginbuf, request->login, sizeof(loginbuf) - 2); + if ((t = strchr(loginbuf, ':'))) + *t = '\0'; + strcat(loginbuf, "@"); + } + snprintf(buf, MAX_URL, "%s://%s%s%s%s%s", + ProtocolStr[request->protocol], + loginbuf, + request->host, + portbuf, + "/%2f", + strBuf(request->urlpath)); + if ((t = strchr(buf, '?'))) + *t = '\0'; + return buf; +} --- squid/src/gopher.c Wed Feb 14 01:07:40 2007 +++ /dev/null Wed Feb 14 01:07:22 2007 @@ -1,852 +0,0 @@ - -/* - * $Id: gopher.c,v 1.19.2.1 2002/10/04 07:07:25 rbcollins Exp $ - * - * DEBUG: section 10 Gopher - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "Store.h" - -/* gopher type code from rfc. Anawat. */ -#define GOPHER_FILE '0' -#define GOPHER_DIRECTORY '1' -#define GOPHER_CSO '2' -#define GOPHER_ERROR '3' -#define GOPHER_MACBINHEX '4' -#define GOPHER_DOSBIN '5' -#define GOPHER_UUENCODED '6' -#define GOPHER_INDEX '7' -#define GOPHER_TELNET '8' -#define GOPHER_BIN '9' -#define GOPHER_REDUNT '+' -#define GOPHER_3270 'T' -#define GOPHER_GIF 'g' -#define GOPHER_IMAGE 'I' - -#define GOPHER_HTML 'h' /* HTML */ -#define GOPHER_INFO 'i' -#define GOPHER_WWW 'w' /* W3 address */ -#define GOPHER_SOUND 's' - -#define GOPHER_PLUS_IMAGE ':' -#define GOPHER_PLUS_MOVIE ';' -#define GOPHER_PLUS_SOUND '<' - -#define GOPHER_PORT 70 - -#define TAB '\t' -#define TEMP_BUF_SIZE 4096 -#define MAX_CSO_RESULT 1024 - -typedef struct gopher_ds { - StoreEntry *entry; - enum { - NORMAL, - HTML_DIR, - HTML_INDEX_RESULT, - HTML_CSO_RESULT, - HTML_INDEX_PAGE, - HTML_CSO_PAGE - } conversion; - int HTML_header_added; - char type_id; - char request[MAX_URL]; - int data_in; - int cso_recno; - int len; - char *buf; /* pts to a 4k page */ - int fd; - request_t *req; - FwdState *fwdState; -} GopherStateData; - -static PF gopherStateFree; -static void gopher_mime_content(MemBuf * mb, const char *name, const char *def); -static void gopherMimeCreate(GopherStateData *); -static void gopher_request_parse(const request_t * req, - char *type_id, - char *request); -static void gopherEndHTML(GopherStateData *); -static void gopherToHTML(GopherStateData *, char *inbuf, int len); -static PF gopherTimeout; -static PF gopherReadReply; -static CWCB gopherSendComplete; -static PF gopherSendRequest; - -static char def_gopher_bin[] = "www/unknown"; -static char def_gopher_text[] = "text/plain"; - -static void -gopherStateFree(int fdnotused, void *data) -{ - GopherStateData *gopherState = data; - if (gopherState == NULL) - return; - if (gopherState->entry) { - storeUnlockObject(gopherState->entry); - } - if (gopherState->req) { - requestUnlink(gopherState->req); - } - memFree(gopherState->buf, MEM_4K_BUF); - gopherState->buf = NULL; - cbdataFree(gopherState); -} - - -/* figure out content type from file extension */ -static void -gopher_mime_content(MemBuf * mb, const char *name, const char *def_ctype) -{ - char *ctype = mimeGetContentType(name); - char *cenc = mimeGetContentEncoding(name); - if (cenc) - memBufPrintf(mb, "Content-Encoding: %s\r\n", cenc); - memBufPrintf(mb, "Content-Type: %s\r\n", - ctype ? ctype : def_ctype); -} - - - -/* create MIME Header for Gopher Data */ -static void -gopherMimeCreate(GopherStateData * gopherState) -{ - MemBuf mb; - - memBufDefInit(&mb); - - memBufPrintf(&mb, - "HTTP/1.0 200 OK Gatewaying\r\n" - "Server: Squid/%s\r\n" - "Date: %s\r\n", - version_string, mkrfc1123(squid_curtime)); - - switch (gopherState->type_id) { - - case GOPHER_DIRECTORY: - case GOPHER_INDEX: - case GOPHER_HTML: - case GOPHER_WWW: - case GOPHER_CSO: - memBufPrintf(&mb, "Content-Type: text/html\r\n"); - break; - case GOPHER_GIF: - case GOPHER_IMAGE: - case GOPHER_PLUS_IMAGE: - memBufPrintf(&mb, "Content-Type: image/gif\r\n"); - break; - case GOPHER_SOUND: - case GOPHER_PLUS_SOUND: - memBufPrintf(&mb, "Content-Type: audio/basic\r\n"); - break; - case GOPHER_PLUS_MOVIE: - memBufPrintf(&mb, "Content-Type: video/mpeg\r\n"); - break; - case GOPHER_MACBINHEX: - case GOPHER_DOSBIN: - case GOPHER_UUENCODED: - case GOPHER_BIN: - /* Rightnow We have no idea what it is. */ - gopher_mime_content(&mb, gopherState->request, def_gopher_bin); - break; - case GOPHER_FILE: - default: - gopher_mime_content(&mb, gopherState->request, def_gopher_text); - break; - } - memBufPrintf(&mb, "\r\n"); - EBIT_CLR(gopherState->entry->flags, ENTRY_FWD_HDR_WAIT); - storeAppend(gopherState->entry, mb.buf, mb.size); - memBufClean(&mb); -} - -/* Parse a gopher request into components. By Anawat. */ -static void -gopher_request_parse(const request_t * req, char *type_id, char *request) -{ - const char *path = strBuf(req->urlpath); - - if (request) - request[0] = '\0'; - - if (path && (*path == '/')) - path++; - - if (!path || !*path) { - *type_id = GOPHER_DIRECTORY; - return; - } - *type_id = path[0]; - - if (request) { - xstrncpy(request, path + 1, MAX_URL); - /* convert %xx to char */ - url_convert_hex(request, 0); - } -} - -int -gopherCachable(const request_t * req) -{ - int cachable = 1; - char type_id; - /* parse to see type */ - gopher_request_parse(req, - &type_id, - NULL); - switch (type_id) { - case GOPHER_INDEX: - case GOPHER_CSO: - case GOPHER_TELNET: - case GOPHER_3270: - cachable = 0; - break; - default: - cachable = 1; - } - return cachable; -} - -static void -gopherHTMLHeader(StoreEntry * e, const char *title, const char *substring) -{ - storeAppendPrintf(e, "\n"); - storeAppendPrintf(e, ""); - storeAppendPrintf(e, title, substring); - storeAppendPrintf(e, ""); - storeAppendPrintf(e, "\n"); - storeAppendPrintf(e, "\n

"); - storeAppendPrintf(e, title, substring); - storeAppendPrintf(e, "

\n"); -} - -static void -gopherHTMLFooter(StoreEntry * e) -{ - storeAppendPrintf(e, "
\n"); - storeAppendPrintf(e, "
\n"); - storeAppendPrintf(e, "Generated %s by %s (%s)\n", - mkrfc1123(squid_curtime), - getMyHostname(), - full_appname_string); - storeAppendPrintf(e, "
\n"); -} - -static void -gopherEndHTML(GopherStateData * gopherState) -{ - StoreEntry *e = gopherState->entry; - if (!gopherState->data_in) { - gopherHTMLHeader(e, "Server Return Nothing", NULL); - storeAppendPrintf(e, "

The Gopher query resulted in a blank response

"); - } else { - storeAppendPrintf(e, "\n"); - } - gopherHTMLFooter(e); -} - - -/* Convert Gopher to HTML */ -/* Borrow part of code from libwww2 came with Mosaic distribution */ -static void -gopherToHTML(GopherStateData * gopherState, char *inbuf, int len) -{ - char *pos = inbuf; - char *lpos = NULL; - char *tline = NULL; - LOCAL_ARRAY(char, line, TEMP_BUF_SIZE); - LOCAL_ARRAY(char, tmpbuf, TEMP_BUF_SIZE); - String outbuf = StringNull; - char *name = NULL; - char *selector = NULL; - char *host = NULL; - char *port = NULL; - char *escaped_selector = NULL; - const char *icon_url = NULL; - char gtype; - StoreEntry *entry = NULL; - - memset(tmpbuf, '\0', TEMP_BUF_SIZE); - memset(line, '\0', TEMP_BUF_SIZE); - - entry = gopherState->entry; - - if (gopherState->conversion == HTML_INDEX_PAGE) { - char *html_url = html_quote(storeUrl(entry)); - gopherHTMLHeader(entry, "Gopher Index %s", html_url); - storeAppendPrintf(entry, - "

This is a searchable Gopher index. Use the search\n" - "function of your browser to enter search terms.\n" - "\n"); - gopherHTMLFooter(entry); - /* now let start sending stuff to client */ - storeBufferFlush(entry); - gopherState->data_in = 1; - - return; - } - if (gopherState->conversion == HTML_CSO_PAGE) { - char *html_url = html_quote(storeUrl(entry)); - gopherHTMLHeader(entry, "CSO Search of %s", html_url); - storeAppendPrintf(entry, - "

A CSO database usually contains a phonebook or\n" - "directory. Use the search function of your browser to enter\n" - "search terms.

\n"); - gopherHTMLFooter(entry); - /* now let start sending stuff to client */ - storeBufferFlush(entry); - gopherState->data_in = 1; - - return; - } - inbuf[len] = '\0'; - - if (!gopherState->HTML_header_added) { - if (gopherState->conversion == HTML_CSO_RESULT) - gopherHTMLHeader(entry, "CSO Search Result", NULL); - else - gopherHTMLHeader(entry, "Gopher Menu", NULL); - strCat(outbuf, "
");
-	gopherState->HTML_header_added = 1;
-    }
-    while ((pos != NULL) && (pos < inbuf + len)) {
-
-	if (gopherState->len != 0) {
-	    /* there is something left from last tx. */
-	    xstrncpy(line, gopherState->buf, gopherState->len + 1);
-	    lpos = (char *) memccpy(line + gopherState->len, inbuf, '\n', len);
-	    if (lpos)
-		*lpos = '\0';
-	    else {
-		/* there is no complete line in inbuf */
-		/* copy it to temp buffer */
-		if (gopherState->len + len > TEMP_BUF_SIZE) {
-		    debug(10, 1) ("GopherHTML: Buffer overflow. Lost some data on URL: %s\n",
-			storeUrl(entry));
-		    len = TEMP_BUF_SIZE - gopherState->len;
-		}
-		xmemcpy(gopherState->buf + gopherState->len, inbuf, len);
-		gopherState->len += len;
-		return;
-	    }
-
-	    /* skip one line */
-	    pos = (char *) memchr(pos, '\n', len);
-	    if (pos)
-		pos++;
-
-	    /* we're done with the remain from last tx. */
-	    gopherState->len = 0;
-	    *(gopherState->buf) = '\0';
-	} else {
-
-	    lpos = (char *) memccpy(line, pos, '\n', len - (pos - inbuf));
-	    if (lpos)
-		*lpos = '\0';
-	    else {
-		/* there is no complete line in inbuf */
-		/* copy it to temp buffer */
-		if ((len - (pos - inbuf)) > TEMP_BUF_SIZE) {
-		    debug(10, 1) ("GopherHTML: Buffer overflow. Lost some data on URL: %s\n",
-			storeUrl(entry));
-		    len = TEMP_BUF_SIZE;
-		}
-		if (len > (pos - inbuf)) {
-		    xmemcpy(gopherState->buf, pos, len - (pos - inbuf));
-		    gopherState->len = len - (pos - inbuf);
-		}
-		break;
-	    }
-
-	    /* skip one line */
-	    pos = (char *) memchr(pos, '\n', len);
-	    if (pos)
-		pos++;
-
-	}
-
-	/* at this point. We should have one line in buffer to process */
-
-	if (*line == '.') {
-	    /* skip it */
-	    memset(line, '\0', TEMP_BUF_SIZE);
-	    continue;
-	}
-	switch (gopherState->conversion) {
-
-	case HTML_INDEX_RESULT:
-	case HTML_DIR:{
-		tline = line;
-		gtype = *tline++;
-		name = tline;
-		selector = strchr(tline, TAB);
-		if (selector) {
-		    *selector++ = '\0';
-		    host = strchr(selector, TAB);
-		    if (host) {
-			*host++ = '\0';
-			port = strchr(host, TAB);
-			if (port) {
-			    char *junk;
-			    port[0] = ':';
-			    junk = strchr(host, TAB);
-			    if (junk)
-				*junk++ = 0;	/* Chop port */
-			    else {
-				junk = strchr(host, '\r');
-				if (junk)
-				    *junk++ = 0;	/* Chop port */
-				else {
-				    junk = strchr(host, '\n');
-				    if (junk)
-					*junk++ = 0;	/* Chop port */
-				}
-			    }
-			    if ((port[1] == '0') && (!port[2]))
-				port[0] = 0;	/* 0 means none */
-			}
-			/* escape a selector here */
-			escaped_selector = xstrdup(rfc1738_escape_part(selector));
-
-			switch (gtype) {
-			case GOPHER_DIRECTORY:
-			    icon_url = mimeGetIconURL("internal-menu");
-			    break;
-			case GOPHER_HTML:
-			case GOPHER_FILE:
-			    icon_url = mimeGetIconURL("internal-text");
-			    break;
-			case GOPHER_INDEX:
-			case GOPHER_CSO:
-			    icon_url = mimeGetIconURL("internal-index");
-			    break;
-			case GOPHER_IMAGE:
-			case GOPHER_GIF:
-			case GOPHER_PLUS_IMAGE:
-			    icon_url = mimeGetIconURL("internal-image");
-			    break;
-			case GOPHER_SOUND:
-			case GOPHER_PLUS_SOUND:
-			    icon_url = mimeGetIconURL("internal-sound");
-			    break;
-			case GOPHER_PLUS_MOVIE:
-			    icon_url = mimeGetIconURL("internal-movie");
-			    break;
-			case GOPHER_TELNET:
-			case GOPHER_3270:
-			    icon_url = mimeGetIconURL("internal-telnet");
-			    break;
-			case GOPHER_BIN:
-			case GOPHER_MACBINHEX:
-			case GOPHER_DOSBIN:
-			case GOPHER_UUENCODED:
-			    icon_url = mimeGetIconURL("internal-binary");
-			    break;
-			case GOPHER_INFO:
-			    icon_url = NULL;
-			    break;
-			default:
-			    icon_url = mimeGetIconURL("internal-unknown");
-			    break;
-			}
-
-			memset(tmpbuf, '\0', TEMP_BUF_SIZE);
-			if ((gtype == GOPHER_TELNET) || (gtype == GOPHER_3270)) {
-			    if (strlen(escaped_selector) != 0)
-				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
-				    icon_url, escaped_selector, rfc1738_escape_part(host),
-				    *port ? ":" : "", port, html_quote(name));
-			    else
-				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
-				    icon_url, rfc1738_escape_part(host), *port ? ":" : "",
-				    port, html_quote(name));
-
-			} else if (gtype == GOPHER_INFO) {
-			    snprintf(tmpbuf, TEMP_BUF_SIZE, "\t%s\n", html_quote(name));
-			} else {
-			    if (strncmp(selector, "GET /", 5) == 0) {
-				/* WWW link */
-				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
-				    icon_url, host, rfc1738_escape_unescaped(selector + 5), html_quote(name));
-			    } else {
-				/* Standard link */
-				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
-				    icon_url, host, gtype, escaped_selector, html_quote(name));
-			    }
-			}
-			safe_free(escaped_selector);
-			strCat(outbuf, tmpbuf);
-			gopherState->data_in = 1;
-		    } else {
-			memset(line, '\0', TEMP_BUF_SIZE);
-			continue;
-		    }
-		} else {
-		    memset(line, '\0', TEMP_BUF_SIZE);
-		    continue;
-		}
-		break;
-	    }			/* HTML_DIR, HTML_INDEX_RESULT */
-
-
-	case HTML_CSO_RESULT:{
-		if (line[0] == '-') {
-		    int code, recno;
-		    char *s_code, *s_recno, *result;
-
-		    s_code = strtok(line + 1, ":\n");
-		    s_recno = strtok(NULL, ":\n");
-		    result = strtok(NULL, "\n");
-
-		    if (!result)
-			break;
-
-		    code = atoi(s_code);
-		    recno = atoi(s_recno);
-
-		    if (code != 200)
-			break;
-
-		    if (gopherState->cso_recno != recno) {
-			snprintf(tmpbuf, TEMP_BUF_SIZE, "

Record# %d
%s

\n
", recno, html_quote(result));
-			gopherState->cso_recno = recno;
-		    } else {
-			snprintf(tmpbuf, TEMP_BUF_SIZE, "%s\n", html_quote(result));
-		    }
-		    strCat(outbuf, tmpbuf);
-		    gopherState->data_in = 1;
-		    break;
-		} else {
-		    int code;
-		    char *s_code, *result;
-
-		    s_code = strtok(line, ":");
-		    result = strtok(NULL, "\n");
-
-		    if (!result)
-			break;
-
-		    code = atoi(s_code);
-		    switch (code) {
-
-		    case 200:{
-			    /* OK */
-			    /* Do nothing here */
-			    break;
-			}
-
-		    case 102:	/* Number of matches */
-		    case 501:	/* No Match */
-		    case 502:	/* Too Many Matches */
-			{
-			    /* Print the message the server returns */
-			    snprintf(tmpbuf, TEMP_BUF_SIZE, "

%s

\n
", html_quote(result));
-			    strCat(outbuf, tmpbuf);
-			    gopherState->data_in = 1;
-			    break;
-			}
-
-
-		    }
-		}
-
-	    }			/* HTML_CSO_RESULT */
-	default:
-	    break;		/* do nothing */
-
-	}			/* switch */
-
-    }				/* while loop */
-
-    if (strLen(outbuf) > 0) {
-	storeAppend(entry, strBuf(outbuf), strLen(outbuf));
-	/* now let start sending stuff to client */
-	storeBufferFlush(entry);
-    }
-    stringClean(&outbuf);
-    return;
-}
-
-static void
-gopherTimeout(int fd, void *data)
-{
-    GopherStateData *gopherState = data;
-    StoreEntry *entry = gopherState->entry;
-    debug(10, 4) ("gopherTimeout: FD %d: '%s'\n", fd, storeUrl(entry));
-    if (entry->store_status == STORE_PENDING) {
-	if (entry->mem_obj->inmem_hi == 0) {
-	    fwdFail(gopherState->fwdState,
-		errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT));
-	}
-    }
-    comm_close(fd);
-}
-
-/* This will be called when data is ready to be read from fd.  Read until
- * error or connection closed. */
-static void
-gopherReadReply(int fd, void *data)
-{
-    GopherStateData *gopherState = data;
-    StoreEntry *entry = gopherState->entry;
-    char *buf = NULL;
-    int len;
-    int clen;
-    int bin;
-    size_t read_sz;
-#if DELAY_POOLS
-    delay_id delay_id = delayMostBytesAllowed(entry->mem_obj);
-#endif
-    if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
-	comm_close(fd);
-	return;
-    }
-    errno = 0;
-    buf = memAllocate(MEM_4K_BUF);
-    read_sz = 4096 - 1;		/* leave room for termination */
-#if DELAY_POOLS
-    read_sz = delayBytesWanted(delay_id, 1, read_sz);
-#endif
-    /* leave one space for \0 in gopherToHTML */
-    statCounter.syscalls.sock.reads++;
-    len = FD_READ_METHOD(fd, buf, read_sz);
-    if (len > 0) {
-	fd_bytes(fd, len, FD_READ);
-#if DELAY_POOLS
-	delayBytesIn(delay_id, len);
-#endif
-	kb_incr(&statCounter.server.all.kbytes_in, len);
-	kb_incr(&statCounter.server.other.kbytes_in, len);
-    }
-    debug(10, 5) ("gopherReadReply: FD %d read len=%d\n", fd, len);
-    if (len > 0) {
-	commSetTimeout(fd, Config.Timeout.read, NULL, NULL);
-	IOStats.Gopher.reads++;
-	for (clen = len - 1, bin = 0; clen; bin++)
-	    clen >>= 1;
-	IOStats.Gopher.read_hist[bin]++;
-    }
-    if (len < 0) {
-	debug(50, 1) ("gopherReadReply: error reading: %s\n", xstrerror());
-	if (ignoreErrno(errno)) {
-	    commSetSelect(fd, COMM_SELECT_READ, gopherReadReply, data, 0);
-	} else if (entry->mem_obj->inmem_hi == 0) {
-	    ErrorState *err;
-	    err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR);
-	    err->xerrno = errno;
-	    err->url = xstrdup(storeUrl(entry));
-	    errorAppendEntry(entry, err);
-	    comm_close(fd);
-	} else {
-	    comm_close(fd);
-	}
-    } else if (len == 0 && entry->mem_obj->inmem_hi == 0) {
-	ErrorState *err;
-	err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE);
-	err->xerrno = errno;
-	err->url = xstrdup(gopherState->request);
-	errorAppendEntry(entry, err);
-	comm_close(fd);
-    } else if (len == 0) {
-	/* Connection closed; retrieval done. */
-	/* flush the rest of data in temp buf if there is one. */
-	if (gopherState->conversion != NORMAL)
-	    gopherEndHTML(data);
-	storeTimestampsSet(entry);
-	storeBufferFlush(entry);
-	fwdComplete(gopherState->fwdState);
-	comm_close(fd);
-    } else {
-	if (gopherState->conversion != NORMAL) {
-	    gopherToHTML(data, buf, len);
-	} else {
-	    storeAppend(entry, buf, len);
-	}
-	commSetSelect(fd,
-	    COMM_SELECT_READ,
-	    gopherReadReply,
-	    data, 0);
-    }
-    memFree(buf, MEM_4K_BUF);
-    return;
-}
-
-/* This will be called when request write is complete. Schedule read of
- * reply. */
-static void
-gopherSendComplete(int fd, char *buf, size_t size, comm_err_t errflag, void *data)
-{
-    GopherStateData *gopherState = (GopherStateData *) data;
-    StoreEntry *entry = gopherState->entry;
-    debug(10, 5) ("gopherSendComplete: FD %d size: %d errflag: %d\n",
-	fd, (int) size, errflag);
-    if (size > 0) {
-	fd_bytes(fd, size, FD_WRITE);
-	kb_incr(&statCounter.server.all.kbytes_out, size);
-	kb_incr(&statCounter.server.other.kbytes_out, size);
-    }
-    if (errflag) {
-	ErrorState *err;
-	err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE);
-	err->xerrno = errno;
-	err->host = xstrdup(gopherState->req->host);
-	err->port = gopherState->req->port;
-	err->url = xstrdup(storeUrl(entry));
-	errorAppendEntry(entry, err);
-	comm_close(fd);
-	if (buf)
-	    memFree(buf, MEM_4K_BUF);	/* Allocated by gopherSendRequest. */
-	return;
-    }
-    /* 
-     * OK. We successfully reach remote site.  Start MIME typing
-     * stuff.  Do it anyway even though request is not HTML type.
-     */
-    gopherMimeCreate(gopherState);
-    switch (gopherState->type_id) {
-    case GOPHER_DIRECTORY:
-	/* we got to convert it first */
-	storeBuffer(entry);
-	gopherState->conversion = HTML_DIR;
-	gopherState->HTML_header_added = 0;
-	break;
-    case GOPHER_INDEX:
-	/* we got to convert it first */
-	storeBuffer(entry);
-	gopherState->conversion = HTML_INDEX_RESULT;
-	gopherState->HTML_header_added = 0;
-	break;
-    case GOPHER_CSO:
-	/* we got to convert it first */
-	storeBuffer(entry);
-	gopherState->conversion = HTML_CSO_RESULT;
-	gopherState->cso_recno = 0;
-	gopherState->HTML_header_added = 0;
-	break;
-    default:
-	gopherState->conversion = NORMAL;
-    }
-    /* Schedule read reply. */
-    commSetSelect(fd, COMM_SELECT_READ, gopherReadReply, gopherState, 0);
-    commSetDefer(fd, fwdCheckDeferRead, entry);
-    if (buf)
-	memFree(buf, MEM_4K_BUF);	/* Allocated by gopherSendRequest. */
-}
-
-/* This will be called when connect completes. Write request. */
-static void
-gopherSendRequest(int fd, void *data)
-{
-    GopherStateData *gopherState = data;
-    char *buf = memAllocate(MEM_4K_BUF);
-    if (gopherState->type_id == GOPHER_CSO) {
-	const char *t = strchr(gopherState->request, '?');
-	if (t != NULL)
-	    t++;		/* skip the ? */
-	else
-	    t = "";
-	snprintf(buf, 4096, "query %s\r\nquit\r\n", t);
-    } else if (gopherState->type_id == GOPHER_INDEX) {
-	char *t = strchr(gopherState->request, '?');
-	if (t != NULL)
-	    *t = '\t';
-	snprintf(buf, 4096, "%s\r\n", gopherState->request);
-    } else {
-	snprintf(buf, 4096, "%s\r\n", gopherState->request);
-    }
-    debug(10, 5) ("gopherSendRequest: FD %d\n", fd);
-    comm_write(fd,
-	buf,
-	strlen(buf),
-	gopherSendComplete,
-	data,
-	memFree4K);
-    if (EBIT_TEST(gopherState->entry->flags, ENTRY_CACHABLE))
-	storeSetPublicKey(gopherState->entry);	/* Make it public */
-}
-
-CBDATA_TYPE(GopherStateData);
-
-void
-gopherStart(FwdState * fwdState)
-{
-    int fd = fwdState->server_fd;
-    StoreEntry *entry = fwdState->entry;
-    GopherStateData *gopherState;
-    CBDATA_INIT_TYPE(GopherStateData);
-    gopherState = cbdataAlloc(GopherStateData);
-    gopherState->buf = memAllocate(MEM_4K_BUF);
-    storeLockObject(entry);
-    gopherState->entry = entry;
-    gopherState->fwdState = fwdState;
-    debug(10, 3) ("gopherStart: %s\n", storeUrl(entry));
-    statCounter.server.all.requests++;
-    statCounter.server.other.requests++;
-    /* Parse url. */
-    gopher_request_parse(fwdState->request,
-	&gopherState->type_id, gopherState->request);
-#if OLD_PARSE_ERROR_CODE
-    if (...) {
-	ErrorState *err;
-	err = errorCon(ERR_INVALID_URL, HTTP_BAD_REQUEST);
-	err->url = xstrdup(storeUrl(entry));
-	errorAppendEntry(entry, err);
-	gopherStateFree(-1, gopherState);
-	return;
-    }
-#endif
-    comm_add_close_handler(fd, gopherStateFree, gopherState);
-    if (((gopherState->type_id == GOPHER_INDEX) || (gopherState->type_id == GOPHER_CSO))
-	&& (strchr(gopherState->request, '?') == NULL)) {
-	/* Index URL without query word */
-	/* We have to generate search page back to client. No need for connection */
-	gopherMimeCreate(gopherState);
-	if (gopherState->type_id == GOPHER_INDEX) {
-	    gopherState->conversion = HTML_INDEX_PAGE;
-	} else {
-	    if (gopherState->type_id == GOPHER_CSO) {
-		gopherState->conversion = HTML_CSO_PAGE;
-	    } else {
-		gopherState->conversion = HTML_INDEX_PAGE;
-	    }
-	}
-	gopherToHTML(gopherState, (char *) NULL, 0);
-	fwdComplete(fwdState);
-	comm_close(fd);
-	return;
-    }
-    gopherState->fd = fd;
-    gopherState->fwdState = fwdState;
-    commSetSelect(fd, COMM_SELECT_WRITE, gopherSendRequest, gopherState, 0);
-    commSetTimeout(fd, Config.Timeout.read, gopherTimeout, gopherState);
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/gopher.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,852 @@
+
+/*
+ * $Id: gopher.cc,v 1.1.2.1 2002/10/11 15:40:50 rbcollins Exp $
+ *
+ * DEBUG: section 10    Gopher
+ * AUTHOR: Harvest Derived
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "Store.h"
+
+/* gopher type code from rfc. Anawat. */
+#define GOPHER_FILE         '0'
+#define GOPHER_DIRECTORY    '1'
+#define GOPHER_CSO          '2'
+#define GOPHER_ERROR        '3'
+#define GOPHER_MACBINHEX    '4'
+#define GOPHER_DOSBIN       '5'
+#define GOPHER_UUENCODED    '6'
+#define GOPHER_INDEX        '7'
+#define GOPHER_TELNET       '8'
+#define GOPHER_BIN          '9'
+#define GOPHER_REDUNT       '+'
+#define GOPHER_3270         'T'
+#define GOPHER_GIF          'g'
+#define GOPHER_IMAGE        'I'
+
+#define GOPHER_HTML         'h'	/* HTML */
+#define GOPHER_INFO         'i'
+#define GOPHER_WWW          'w'	/* W3 address */
+#define GOPHER_SOUND        's'
+
+#define GOPHER_PLUS_IMAGE   ':'
+#define GOPHER_PLUS_MOVIE   ';'
+#define GOPHER_PLUS_SOUND   '<'
+
+#define GOPHER_PORT         70
+
+#define TAB                 '\t'
+#define TEMP_BUF_SIZE       4096
+#define MAX_CSO_RESULT      1024
+
+typedef struct gopher_ds {
+    StoreEntry *entry;
+    enum {
+	NORMAL,
+	HTML_DIR,
+	HTML_INDEX_RESULT,
+	HTML_CSO_RESULT,
+	HTML_INDEX_PAGE,
+	HTML_CSO_PAGE
+    } conversion;
+    int HTML_header_added;
+    char type_id;
+    char request[MAX_URL];
+    int data_in;
+    int cso_recno;
+    int len;
+    char *buf;			/* pts to a 4k page */
+    int fd;
+    request_t *req;
+    FwdState *fwdState;
+} GopherStateData;
+
+static PF gopherStateFree;
+static void gopher_mime_content(MemBuf * mb, const char *name, const char *def);
+static void gopherMimeCreate(GopherStateData *);
+static void gopher_request_parse(const request_t * req,
+    char *type_id,
+    char *request);
+static void gopherEndHTML(GopherStateData *);
+static void gopherToHTML(GopherStateData *, char *inbuf, int len);
+static PF gopherTimeout;
+static PF gopherReadReply;
+static CWCB gopherSendComplete;
+static PF gopherSendRequest;
+
+static char def_gopher_bin[] = "www/unknown";
+static char def_gopher_text[] = "text/plain";
+
+static void
+gopherStateFree(int fdnotused, void *data)
+{
+    GopherStateData *gopherState = (GopherStateData *)data;
+    if (gopherState == NULL)
+	return;
+    if (gopherState->entry) {
+	storeUnlockObject(gopherState->entry);
+    }
+    if (gopherState->req) {
+	requestUnlink(gopherState->req);
+    }
+    memFree(gopherState->buf, MEM_4K_BUF);
+    gopherState->buf = NULL;
+    cbdataFree(gopherState);
+}
+
+
+/* figure out content type from file extension */
+static void
+gopher_mime_content(MemBuf * mb, const char *name, const char *def_ctype)
+{
+    char *ctype = mimeGetContentType(name);
+    char *cenc = mimeGetContentEncoding(name);
+    if (cenc)
+	memBufPrintf(mb, "Content-Encoding: %s\r\n", cenc);
+    memBufPrintf(mb, "Content-Type: %s\r\n",
+	ctype ? ctype : def_ctype);
+}
+
+
+
+/* create MIME Header for Gopher Data */
+static void
+gopherMimeCreate(GopherStateData * gopherState)
+{
+    MemBuf mb;
+
+    memBufDefInit(&mb);
+
+    memBufPrintf(&mb,
+	"HTTP/1.0 200 OK Gatewaying\r\n"
+	"Server: Squid/%s\r\n"
+	"Date: %s\r\n",
+	version_string, mkrfc1123(squid_curtime));
+
+    switch (gopherState->type_id) {
+
+    case GOPHER_DIRECTORY:
+    case GOPHER_INDEX:
+    case GOPHER_HTML:
+    case GOPHER_WWW:
+    case GOPHER_CSO:
+	memBufPrintf(&mb, "Content-Type: text/html\r\n");
+	break;
+    case GOPHER_GIF:
+    case GOPHER_IMAGE:
+    case GOPHER_PLUS_IMAGE:
+	memBufPrintf(&mb, "Content-Type: image/gif\r\n");
+	break;
+    case GOPHER_SOUND:
+    case GOPHER_PLUS_SOUND:
+	memBufPrintf(&mb, "Content-Type: audio/basic\r\n");
+	break;
+    case GOPHER_PLUS_MOVIE:
+	memBufPrintf(&mb, "Content-Type: video/mpeg\r\n");
+	break;
+    case GOPHER_MACBINHEX:
+    case GOPHER_DOSBIN:
+    case GOPHER_UUENCODED:
+    case GOPHER_BIN:
+	/* Rightnow We have no idea what it is. */
+	gopher_mime_content(&mb, gopherState->request, def_gopher_bin);
+	break;
+    case GOPHER_FILE:
+    default:
+	gopher_mime_content(&mb, gopherState->request, def_gopher_text);
+	break;
+    }
+    memBufPrintf(&mb, "\r\n");
+    EBIT_CLR(gopherState->entry->flags, ENTRY_FWD_HDR_WAIT);
+    storeAppend(gopherState->entry, mb.buf, mb.size);
+    memBufClean(&mb);
+}
+
+/* Parse a gopher request into components.  By Anawat. */
+static void
+gopher_request_parse(const request_t * req, char *type_id, char *request)
+{
+    const char *path = strBuf(req->urlpath);
+
+    if (request)
+	request[0] = '\0';
+
+    if (path && (*path == '/'))
+	path++;
+
+    if (!path || !*path) {
+	*type_id = GOPHER_DIRECTORY;
+	return;
+    }
+    *type_id = path[0];
+
+    if (request) {
+	xstrncpy(request, path + 1, MAX_URL);
+	/* convert %xx to char */
+	url_convert_hex(request, 0);
+    }
+}
+
+int
+gopherCachable(const request_t * req)
+{
+    int cachable = 1;
+    char type_id;
+    /* parse to see type */
+    gopher_request_parse(req,
+	&type_id,
+	NULL);
+    switch (type_id) {
+    case GOPHER_INDEX:
+    case GOPHER_CSO:
+    case GOPHER_TELNET:
+    case GOPHER_3270:
+	cachable = 0;
+	break;
+    default:
+	cachable = 1;
+    }
+    return cachable;
+}
+
+static void
+gopherHTMLHeader(StoreEntry * e, const char *title, const char *substring)
+{
+    storeAppendPrintf(e, "\n");
+    storeAppendPrintf(e, "");
+    storeAppendPrintf(e, title, substring);
+    storeAppendPrintf(e, "");
+    storeAppendPrintf(e, "\n");
+    storeAppendPrintf(e, "\n
");
+    storeAppendPrintf(e, title, substring);
+    storeAppendPrintf(e, "
\n");
+}
+
+static void
+gopherHTMLFooter(StoreEntry * e)
+{
+    storeAppendPrintf(e, "
\n");
+    storeAppendPrintf(e, "
\n");
+    storeAppendPrintf(e, "Generated %s by %s (%s)\n",
+	mkrfc1123(squid_curtime),
+	getMyHostname(),
+	full_appname_string);
+    storeAppendPrintf(e, "
\n");
+}
+
+static void
+gopherEndHTML(GopherStateData * gopherState)
+{
+    StoreEntry *e = gopherState->entry;
+    if (!gopherState->data_in) {
+	gopherHTMLHeader(e, "Server Return Nothing", NULL);
+	storeAppendPrintf(e, "
The Gopher query resulted in a blank response
");
+    } else {
+	storeAppendPrintf(e, "
\n"); + } + gopherHTMLFooter(e); +} + + +/* Convert Gopher to HTML */ +/* Borrow part of code from libwww2 came with Mosaic distribution */ +static void +gopherToHTML(GopherStateData * gopherState, char *inbuf, int len) +{ + char *pos = inbuf; + char *lpos = NULL; + char *tline = NULL; + LOCAL_ARRAY(char, line, TEMP_BUF_SIZE); + LOCAL_ARRAY(char, tmpbuf, TEMP_BUF_SIZE); + String outbuf = StringNull; + char *name = NULL; + char *selector = NULL; + char *host = NULL; + char *port = NULL; + char *escaped_selector = NULL; + const char *icon_url = NULL; + char gtype; + StoreEntry *entry = NULL; + + memset(tmpbuf, '\0', TEMP_BUF_SIZE); + memset(line, '\0', TEMP_BUF_SIZE); + + entry = gopherState->entry; + + if (gopherState->conversion == gopher_ds::HTML_INDEX_PAGE) { + char *html_url = html_quote(storeUrl(entry)); + gopherHTMLHeader(entry, "Gopher Index %s", html_url); + storeAppendPrintf(entry, + "

This is a searchable Gopher index. Use the search\n" + "function of your browser to enter search terms.\n" + "\n"); + gopherHTMLFooter(entry); + /* now let start sending stuff to client */ + storeBufferFlush(entry); + gopherState->data_in = 1; + + return; + } + if (gopherState->conversion == gopher_ds::HTML_CSO_PAGE) { + char *html_url = html_quote(storeUrl(entry)); + gopherHTMLHeader(entry, "CSO Search of %s", html_url); + storeAppendPrintf(entry, + "

A CSO database usually contains a phonebook or\n" + "directory. Use the search function of your browser to enter\n" + "search terms.

\n"); + gopherHTMLFooter(entry); + /* now let start sending stuff to client */ + storeBufferFlush(entry); + gopherState->data_in = 1; + + return; + } + inbuf[len] = '\0'; + + if (!gopherState->HTML_header_added) { + if (gopherState->conversion == gopher_ds::HTML_CSO_RESULT) + gopherHTMLHeader(entry, "CSO Search Result", NULL); + else + gopherHTMLHeader(entry, "Gopher Menu", NULL); + strCat(outbuf, "
");
+	gopherState->HTML_header_added = 1;
+    }
+    while ((pos != NULL) && (pos < inbuf + len)) {
+
+	if (gopherState->len != 0) {
+	    /* there is something left from last tx. */
+	    xstrncpy(line, gopherState->buf, gopherState->len + 1);
+	    lpos = (char *) memccpy(line + gopherState->len, inbuf, '\n', len);
+	    if (lpos)
+		*lpos = '\0';
+	    else {
+		/* there is no complete line in inbuf */
+		/* copy it to temp buffer */
+		if (gopherState->len + len > TEMP_BUF_SIZE) {
+		    debug(10, 1) ("GopherHTML: Buffer overflow. Lost some data on URL: %s\n",
+			storeUrl(entry));
+		    len = TEMP_BUF_SIZE - gopherState->len;
+		}
+		xmemcpy(gopherState->buf + gopherState->len, inbuf, len);
+		gopherState->len += len;
+		return;
+	    }
+
+	    /* skip one line */
+	    pos = (char *) memchr(pos, '\n', len);
+	    if (pos)
+		pos++;
+
+	    /* we're done with the remain from last tx. */
+	    gopherState->len = 0;
+	    *(gopherState->buf) = '\0';
+	} else {
+
+	    lpos = (char *) memccpy(line, pos, '\n', len - (pos - inbuf));
+	    if (lpos)
+		*lpos = '\0';
+	    else {
+		/* there is no complete line in inbuf */
+		/* copy it to temp buffer */
+		if ((len - (pos - inbuf)) > TEMP_BUF_SIZE) {
+		    debug(10, 1) ("GopherHTML: Buffer overflow. Lost some data on URL: %s\n",
+			storeUrl(entry));
+		    len = TEMP_BUF_SIZE;
+		}
+		if (len > (pos - inbuf)) {
+		    xmemcpy(gopherState->buf, pos, len - (pos - inbuf));
+		    gopherState->len = len - (pos - inbuf);
+		}
+		break;
+	    }
+
+	    /* skip one line */
+	    pos = (char *) memchr(pos, '\n', len);
+	    if (pos)
+		pos++;
+
+	}
+
+	/* at this point. We should have one line in buffer to process */
+
+	if (*line == '.') {
+	    /* skip it */
+	    memset(line, '\0', TEMP_BUF_SIZE);
+	    continue;
+	}
+	switch (gopherState->conversion) {
+
+	case gopher_ds::HTML_INDEX_RESULT:
+	case gopher_ds::HTML_DIR:{
+		tline = line;
+		gtype = *tline++;
+		name = tline;
+		selector = strchr(tline, TAB);
+		if (selector) {
+		    *selector++ = '\0';
+		    host = strchr(selector, TAB);
+		    if (host) {
+			*host++ = '\0';
+			port = strchr(host, TAB);
+			if (port) {
+			    char *junk;
+			    port[0] = ':';
+			    junk = strchr(host, TAB);
+			    if (junk)
+				*junk++ = 0;	/* Chop port */
+			    else {
+				junk = strchr(host, '\r');
+				if (junk)
+				    *junk++ = 0;	/* Chop port */
+				else {
+				    junk = strchr(host, '\n');
+				    if (junk)
+					*junk++ = 0;	/* Chop port */
+				}
+			    }
+			    if ((port[1] == '0') && (!port[2]))
+				port[0] = 0;	/* 0 means none */
+			}
+			/* escape a selector here */
+			escaped_selector = xstrdup(rfc1738_escape_part(selector));
+
+			switch (gtype) {
+			case GOPHER_DIRECTORY:
+			    icon_url = mimeGetIconURL("internal-menu");
+			    break;
+			case GOPHER_HTML:
+			case GOPHER_FILE:
+			    icon_url = mimeGetIconURL("internal-text");
+			    break;
+			case GOPHER_INDEX:
+			case GOPHER_CSO:
+			    icon_url = mimeGetIconURL("internal-index");
+			    break;
+			case GOPHER_IMAGE:
+			case GOPHER_GIF:
+			case GOPHER_PLUS_IMAGE:
+			    icon_url = mimeGetIconURL("internal-image");
+			    break;
+			case GOPHER_SOUND:
+			case GOPHER_PLUS_SOUND:
+			    icon_url = mimeGetIconURL("internal-sound");
+			    break;
+			case GOPHER_PLUS_MOVIE:
+			    icon_url = mimeGetIconURL("internal-movie");
+			    break;
+			case GOPHER_TELNET:
+			case GOPHER_3270:
+			    icon_url = mimeGetIconURL("internal-telnet");
+			    break;
+			case GOPHER_BIN:
+			case GOPHER_MACBINHEX:
+			case GOPHER_DOSBIN:
+			case GOPHER_UUENCODED:
+			    icon_url = mimeGetIconURL("internal-binary");
+			    break;
+			case GOPHER_INFO:
+			    icon_url = NULL;
+			    break;
+			default:
+			    icon_url = mimeGetIconURL("internal-unknown");
+			    break;
+			}
+
+			memset(tmpbuf, '\0', TEMP_BUF_SIZE);
+			if ((gtype == GOPHER_TELNET) || (gtype == GOPHER_3270)) {
+			    if (strlen(escaped_selector) != 0)
+				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
+				    icon_url, escaped_selector, rfc1738_escape_part(host),
+				    *port ? ":" : "", port, html_quote(name));
+			    else
+				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
+				    icon_url, rfc1738_escape_part(host), *port ? ":" : "",
+				    port, html_quote(name));
+
+			} else if (gtype == GOPHER_INFO) {
+			    snprintf(tmpbuf, TEMP_BUF_SIZE, "\t%s\n", html_quote(name));
+			} else {
+			    if (strncmp(selector, "GET /", 5) == 0) {
+				/* WWW link */
+				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
+				    icon_url, host, rfc1738_escape_unescaped(selector + 5), html_quote(name));
+			    } else {
+				/* Standard link */
+				snprintf(tmpbuf, TEMP_BUF_SIZE, " %s\n",
+				    icon_url, host, gtype, escaped_selector, html_quote(name));
+			    }
+			}
+			safe_free(escaped_selector);
+			strCat(outbuf, tmpbuf);
+			gopherState->data_in = 1;
+		    } else {
+			memset(line, '\0', TEMP_BUF_SIZE);
+			continue;
+		    }
+		} else {
+		    memset(line, '\0', TEMP_BUF_SIZE);
+		    continue;
+		}
+		break;
+	    }			/* HTML_DIR, HTML_INDEX_RESULT */
+
+
+	case gopher_ds::HTML_CSO_RESULT:{
+		if (line[0] == '-') {
+		    int code, recno;
+		    char *s_code, *s_recno, *result;
+
+		    s_code = strtok(line + 1, ":\n");
+		    s_recno = strtok(NULL, ":\n");
+		    result = strtok(NULL, "\n");
+
+		    if (!result)
+			break;
+
+		    code = atoi(s_code);
+		    recno = atoi(s_recno);
+
+		    if (code != 200)
+			break;
+
+		    if (gopherState->cso_recno != recno) {
+			snprintf(tmpbuf, TEMP_BUF_SIZE, "

Record# %d
%s

\n
", recno, html_quote(result));
+			gopherState->cso_recno = recno;
+		    } else {
+			snprintf(tmpbuf, TEMP_BUF_SIZE, "%s\n", html_quote(result));
+		    }
+		    strCat(outbuf, tmpbuf);
+		    gopherState->data_in = 1;
+		    break;
+		} else {
+		    int code;
+		    char *s_code, *result;
+
+		    s_code = strtok(line, ":");
+		    result = strtok(NULL, "\n");
+
+		    if (!result)
+			break;
+
+		    code = atoi(s_code);
+		    switch (code) {
+
+		    case 200:{
+			    /* OK */
+			    /* Do nothing here */
+			    break;
+			}
+
+		    case 102:	/* Number of matches */
+		    case 501:	/* No Match */
+		    case 502:	/* Too Many Matches */
+			{
+			    /* Print the message the server returns */
+			    snprintf(tmpbuf, TEMP_BUF_SIZE, "

%s

\n
", html_quote(result));
+			    strCat(outbuf, tmpbuf);
+			    gopherState->data_in = 1;
+			    break;
+			}
+
+
+		    }
+		}
+
+	    }			/* HTML_CSO_RESULT */
+	default:
+	    break;		/* do nothing */
+
+	}			/* switch */
+
+    }				/* while loop */
+
+    if (strLen(outbuf) > 0) {
+	storeAppend(entry, strBuf(outbuf), strLen(outbuf));
+	/* now let start sending stuff to client */
+	storeBufferFlush(entry);
+    }
+    stringClean(&outbuf);
+    return;
+}
+
+static void
+gopherTimeout(int fd, void *data)
+{
+    GopherStateData *gopherState = (GopherStateData *)data;
+    StoreEntry *entry = gopherState->entry;
+    debug(10, 4) ("gopherTimeout: FD %d: '%s'\n", fd, storeUrl(entry));
+    if (entry->store_status == STORE_PENDING) {
+	if (entry->mem_obj->inmem_hi == 0) {
+	    fwdFail(gopherState->fwdState,
+		errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT));
+	}
+    }
+    comm_close(fd);
+}
+
+/* This will be called when data is ready to be read from fd.  Read until
+ * error or connection closed. */
+static void
+gopherReadReply(int fd, void *data)
+{
+    GopherStateData *gopherState = (GopherStateData *)data;
+    StoreEntry *entry = gopherState->entry;
+    char *buf = NULL;
+    int len;
+    int clen;
+    int bin;
+    size_t read_sz;
+#if DELAY_POOLS
+    delay_id delay_id = delayMostBytesAllowed(entry->mem_obj);
+#endif
+    if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
+	comm_close(fd);
+	return;
+    }
+    errno = 0;
+    buf = (char *)memAllocate(MEM_4K_BUF);
+    read_sz = 4096 - 1;		/* leave room for termination */
+#if DELAY_POOLS
+    read_sz = delayBytesWanted(delay_id, 1, read_sz);
+#endif
+    /* leave one space for \0 in gopherToHTML */
+    statCounter.syscalls.sock.reads++;
+    len = FD_READ_METHOD(fd, buf, read_sz);
+    if (len > 0) {
+	fd_bytes(fd, len, FD_READ);
+#if DELAY_POOLS
+	delayBytesIn(delay_id, len);
+#endif
+	kb_incr(&statCounter.server.all.kbytes_in, len);
+	kb_incr(&statCounter.server.other.kbytes_in, len);
+    }
+    debug(10, 5) ("gopherReadReply: FD %d read len=%d\n", fd, len);
+    if (len > 0) {
+	commSetTimeout(fd, Config.Timeout.read, NULL, NULL);
+	IOStats.Gopher.reads++;
+	for (clen = len - 1, bin = 0; clen; bin++)
+	    clen >>= 1;
+	IOStats.Gopher.read_hist[bin]++;
+    }
+    if (len < 0) {
+	debug(50, 1) ("gopherReadReply: error reading: %s\n", xstrerror());
+	if (ignoreErrno(errno)) {
+	    commSetSelect(fd, COMM_SELECT_READ, gopherReadReply, gopherState, 0);
+	} else if (entry->mem_obj->inmem_hi == 0) {
+	    ErrorState *err;
+	    err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR);
+	    err->xerrno = errno;
+	    err->url = xstrdup(storeUrl(entry));
+	    errorAppendEntry(entry, err);
+	    comm_close(fd);
+	} else {
+	    comm_close(fd);
+	}
+    } else if (len == 0 && entry->mem_obj->inmem_hi == 0) {
+	ErrorState *err;
+	err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE);
+	err->xerrno = errno;
+	err->url = xstrdup(gopherState->request);
+	errorAppendEntry(entry, err);
+	comm_close(fd);
+    } else if (len == 0) {
+	/* Connection closed; retrieval done. */
+	/* flush the rest of data in temp buf if there is one. */
+	if (gopherState->conversion != gopher_ds::NORMAL)
+	    gopherEndHTML(gopherState);
+	storeTimestampsSet(entry);
+	storeBufferFlush(entry);
+	fwdComplete(gopherState->fwdState);
+	comm_close(fd);
+    } else {
+	if (gopherState->conversion != gopher_ds::NORMAL) {
+	    gopherToHTML(gopherState, buf, len);
+	} else {
+	    storeAppend(entry, buf, len);
+	}
+	commSetSelect(fd,
+	    COMM_SELECT_READ,
+	    gopherReadReply,
+	    gopherState, 0);
+    }
+    memFree(buf, MEM_4K_BUF);
+    return;
+}
+
+/* This will be called when request write is complete. Schedule read of
+ * reply. */
+static void
+gopherSendComplete(int fd, char *buf, size_t size, comm_err_t errflag, void *data)
+{
+    GopherStateData *gopherState = (GopherStateData *) data;
+    StoreEntry *entry = gopherState->entry;
+    debug(10, 5) ("gopherSendComplete: FD %d size: %d errflag: %d\n",
+	fd, (int) size, errflag);
+    if (size > 0) {
+	fd_bytes(fd, size, FD_WRITE);
+	kb_incr(&statCounter.server.all.kbytes_out, size);
+	kb_incr(&statCounter.server.other.kbytes_out, size);
+    }
+    if (errflag) {
+	ErrorState *err;
+	err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE);
+	err->xerrno = errno;
+	err->host = xstrdup(gopherState->req->host);
+	err->port = gopherState->req->port;
+	err->url = xstrdup(storeUrl(entry));
+	errorAppendEntry(entry, err);
+	comm_close(fd);
+	if (buf)
+	    memFree(buf, MEM_4K_BUF);	/* Allocated by gopherSendRequest. */
+	return;
+    }
+    /* 
+     * OK. We successfully reach remote site.  Start MIME typing
+     * stuff.  Do it anyway even though request is not HTML type.
+     */
+    gopherMimeCreate(gopherState);
+    switch (gopherState->type_id) {
+    case GOPHER_DIRECTORY:
+	/* we got to convert it first */
+	storeBuffer(entry);
+	gopherState->conversion = gopher_ds::HTML_DIR;
+	gopherState->HTML_header_added = 0;
+	break;
+    case GOPHER_INDEX:
+	/* we got to convert it first */
+	storeBuffer(entry);
+	gopherState->conversion = gopher_ds::HTML_INDEX_RESULT;
+	gopherState->HTML_header_added = 0;
+	break;
+    case GOPHER_CSO:
+	/* we got to convert it first */
+	storeBuffer(entry);
+	gopherState->conversion = gopher_ds::HTML_CSO_RESULT;
+	gopherState->cso_recno = 0;
+	gopherState->HTML_header_added = 0;
+	break;
+    default:
+	gopherState->conversion = gopher_ds::NORMAL;
+    }
+    /* Schedule read reply. */
+    commSetSelect(fd, COMM_SELECT_READ, gopherReadReply, gopherState, 0);
+    commSetDefer(fd, fwdCheckDeferRead, entry);
+    if (buf)
+	memFree(buf, MEM_4K_BUF);	/* Allocated by gopherSendRequest. */
+}
+
+/* This will be called when connect completes. Write request. */
+static void
+gopherSendRequest(int fd, void *data)
+{
+    GopherStateData *gopherState = (GopherStateData *)data;
+    char *buf = (char *)memAllocate(MEM_4K_BUF);
+    if (gopherState->type_id == GOPHER_CSO) {
+	const char *t = strchr(gopherState->request, '?');
+	if (t != NULL)
+	    t++;		/* skip the ? */
+	else
+	    t = "";
+	snprintf(buf, 4096, "query %s\r\nquit\r\n", t);
+    } else if (gopherState->type_id == GOPHER_INDEX) {
+	char *t = strchr(gopherState->request, '?');
+	if (t != NULL)
+	    *t = '\t';
+	snprintf(buf, 4096, "%s\r\n", gopherState->request);
+    } else {
+	snprintf(buf, 4096, "%s\r\n", gopherState->request);
+    }
+    debug(10, 5) ("gopherSendRequest: FD %d\n", fd);
+    comm_write(fd,
+	buf,
+	strlen(buf),
+	gopherSendComplete,
+	gopherState,
+	memFree4K);
+    if (EBIT_TEST(gopherState->entry->flags, ENTRY_CACHABLE))
+	storeSetPublicKey(gopherState->entry);	/* Make it public */
+}
+
+CBDATA_TYPE(GopherStateData);
+
+void
+gopherStart(FwdState * fwdState)
+{
+    int fd = fwdState->server_fd;
+    StoreEntry *entry = fwdState->entry;
+    GopherStateData *gopherState;
+    CBDATA_INIT_TYPE(GopherStateData);
+    gopherState = cbdataAlloc(GopherStateData);
+    gopherState->buf = (char *)memAllocate(MEM_4K_BUF);
+    storeLockObject(entry);
+    gopherState->entry = entry;
+    gopherState->fwdState = fwdState;
+    debug(10, 3) ("gopherStart: %s\n", storeUrl(entry));
+    statCounter.server.all.requests++;
+    statCounter.server.other.requests++;
+    /* Parse url. */
+    gopher_request_parse(fwdState->request,
+	&gopherState->type_id, gopherState->request);
+#if OLD_PARSE_ERROR_CODE
+    if (...) {
+	ErrorState *err;
+	err = errorCon(ERR_INVALID_URL, HTTP_BAD_REQUEST);
+	err->url = xstrdup(storeUrl(entry));
+	errorAppendEntry(entry, err);
+	gopherStateFree(-1, gopherState);
+	return;
+    }
+#endif
+    comm_add_close_handler(fd, gopherStateFree, gopherState);
+    if (((gopherState->type_id == GOPHER_INDEX) || (gopherState->type_id == GOPHER_CSO))
+	&& (strchr(gopherState->request, '?') == NULL)) {
+	/* Index URL without query word */
+	/* We have to generate search page back to client. No need for connection */
+	gopherMimeCreate(gopherState);
+	if (gopherState->type_id == GOPHER_INDEX) {
+	    gopherState->conversion = gopher_ds::HTML_INDEX_PAGE;
+	} else {
+	    if (gopherState->type_id == GOPHER_CSO) {
+		gopherState->conversion = gopher_ds::HTML_CSO_PAGE;
+	    } else {
+		gopherState->conversion = gopher_ds::HTML_INDEX_PAGE;
+	    }
+	}
+	gopherToHTML(gopherState, (char *) NULL, 0);
+	fwdComplete(fwdState);
+	comm_close(fd);
+	return;
+    }
+    gopherState->fd = fd;
+    gopherState->fwdState = fwdState;
+    commSetSelect(fd, COMM_SELECT_WRITE, gopherSendRequest, gopherState, 0);
+    commSetTimeout(fd, Config.Timeout.read, gopherTimeout, gopherState);
+}
--- squid/src/helper.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,1132 +0,0 @@
-
-/*
- * $Id: helper.c,v 1.26.2.1 2002/10/04 07:07:25 rbcollins Exp $
- *
- * DEBUG: section 84    Helper process maintenance
- * AUTHOR: Harvest Derived?
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-#include "Store.h"
-
-#define HELPER_MAX_ARGS 64
-
-static PF helperHandleRead;
-static PF helperStatefulHandleRead;
-static PF helperServerFree;
-static PF helperStatefulServerFree;
-static void Enqueue(helper * hlp, helper_request *);
-static helper_request *Dequeue(helper * hlp);
-static helper_stateful_request *StatefulDequeue(statefulhelper * hlp);
-static helper_server *GetFirstAvailable(helper * hlp);
-static helper_stateful_server *StatefulGetFirstAvailable(statefulhelper * hlp);
-static void helperDispatch(helper_server * srv, helper_request * r);
-static void helperStatefulDispatch(helper_stateful_server * srv, helper_stateful_request * r);
-static void helperKickQueue(helper * hlp);
-static void helperStatefulKickQueue(statefulhelper * hlp);
-static void helperRequestFree(helper_request * r);
-static void helperStatefulRequestFree(helper_stateful_request * r);
-static void StatefulEnqueue(statefulhelper * hlp, helper_stateful_request * r);
-static helper_stateful_request *StatefulServerDequeue(helper_stateful_server * srv);
-static void StatefulServerEnqueue(helper_stateful_server * srv, helper_stateful_request * r);
-static void helperStatefulServerKickQueue(helper_stateful_server * srv);
-
-void
-helperOpenServers(helper * hlp)
-{
-    char *s;
-    char *progname;
-    char *shortname;
-    char *procname;
-    const char *args[HELPER_MAX_ARGS];
-    char fd_note_buf[FD_DESC_SZ];
-    helper_server *srv;
-    int nargs = 0;
-    int k;
-    int x;
-    int rfd;
-    int wfd;
-    wordlist *w;
-    if (hlp->cmdline == NULL)
-	return;
-    progname = hlp->cmdline->key;
-    if ((s = strrchr(progname, '/')))
-	shortname = xstrdup(s + 1);
-    else
-	shortname = xstrdup(progname);
-    debug(84, 1) ("helperOpenServers: Starting %d '%s' processes\n",
-	hlp->n_to_start, shortname);
-    procname = xmalloc(strlen(shortname) + 3);
-    snprintf(procname, strlen(shortname) + 3, "(%s)", shortname);
-    args[nargs++] = procname;
-    for (w = hlp->cmdline->next; w && nargs < HELPER_MAX_ARGS; w = w->next)
-	args[nargs++] = w->key;
-    args[nargs++] = NULL;
-    assert(nargs <= HELPER_MAX_ARGS);
-    for (k = 0; k < hlp->n_to_start; k++) {
-	getCurrentTime();
-	rfd = wfd = -1;
-	x = ipcCreate(hlp->ipc_type,
-	    progname,
-	    args,
-	    shortname,
-	    &rfd,
-	    &wfd);
-	if (x < 0) {
-	    debug(84, 1) ("WARNING: Cannot run '%s' process.\n", progname);
-	    continue;
-	}
-	hlp->n_running++;
-	srv = cbdataAlloc(helper_server);
-	srv->pid = x;
-	srv->flags.alive = 1;
-	srv->index = k;
-	srv->rfd = rfd;
-	srv->wfd = wfd;
-	srv->buf = memAllocate(MEM_8K_BUF);
-	srv->buf_sz = 8192;
-	srv->offset = 0;
-	srv->parent = cbdataReference(hlp);
-	dlinkAddTail(srv, &srv->link, &hlp->servers);
-	if (rfd == wfd) {
-	    snprintf(fd_note_buf, FD_DESC_SZ, "%s #%d", shortname, k + 1);
-	    fd_note(rfd, fd_note_buf);
-	} else {
-	    snprintf(fd_note_buf, FD_DESC_SZ, "reading %s #%d", shortname, k + 1);
-	    fd_note(rfd, fd_note_buf);
-	    snprintf(fd_note_buf, FD_DESC_SZ, "writing %s #%d", shortname, k + 1);
-	    fd_note(wfd, fd_note_buf);
-	}
-	commSetNonBlocking(rfd);
-	if (wfd != rfd)
-	    commSetNonBlocking(wfd);
-	comm_add_close_handler(rfd, helperServerFree, srv);
-    }
-    safe_free(shortname);
-    safe_free(procname);
-    helperKickQueue(hlp);
-}
-
-void
-helperStatefulOpenServers(statefulhelper * hlp)
-{
-    char *s;
-    char *progname;
-    char *shortname;
-    char *procname;
-    const char *args[HELPER_MAX_ARGS];
-    char fd_note_buf[FD_DESC_SZ];
-    helper_stateful_server *srv;
-    int nargs = 0;
-    int k;
-    int x;
-    int rfd;
-    int wfd;
-    wordlist *w;
-    if (hlp->cmdline == NULL)
-	return;
-    progname = hlp->cmdline->key;
-    if ((s = strrchr(progname, '/')))
-	shortname = xstrdup(s + 1);
-    else
-	shortname = xstrdup(progname);
-    debug(84, 1) ("helperStatefulOpenServers: Starting %d '%s' processes\n",
-	hlp->n_to_start, shortname);
-    procname = xmalloc(strlen(shortname) + 3);
-    snprintf(procname, strlen(shortname) + 3, "(%s)", shortname);
-    args[nargs++] = procname;
-    for (w = hlp->cmdline->next; w && nargs < HELPER_MAX_ARGS; w = w->next)
-	args[nargs++] = w->key;
-    args[nargs++] = NULL;
-    assert(nargs <= HELPER_MAX_ARGS);
-    for (k = 0; k < hlp->n_to_start; k++) {
-	getCurrentTime();
-	rfd = wfd = -1;
-	x = ipcCreate(hlp->ipc_type,
-	    progname,
-	    args,
-	    shortname,
-	    &rfd,
-	    &wfd);
-	if (x < 0) {
-	    debug(84, 1) ("WARNING: Cannot run '%s' process.\n", progname);
-	    continue;
-	}
-	hlp->n_running++;
-	srv = cbdataAlloc(helper_stateful_server);
-	srv->pid = x;
-	srv->flags.alive = 1;
-	srv->flags.reserved = S_HELPER_FREE;
-	srv->deferred_requests = 0;
-	srv->stats.deferbyfunc = 0;
-	srv->stats.deferbycb = 0;
-	srv->stats.submits = 0;
-	srv->stats.releases = 0;
-	srv->index = k;
-	srv->rfd = rfd;
-	srv->wfd = wfd;
-	srv->buf = memAllocate(MEM_8K_BUF);
-	srv->buf_sz = 8192;
-	srv->offset = 0;
-	srv->parent = cbdataReference(hlp);
-	if (hlp->datapool != NULL)
-	    srv->data = memPoolAlloc(hlp->datapool);
-	dlinkAddTail(srv, &srv->link, &hlp->servers);
-	if (rfd == wfd) {
-	    snprintf(fd_note_buf, FD_DESC_SZ, "%s #%d", shortname, k + 1);
-	    fd_note(rfd, fd_note_buf);
-	} else {
-	    snprintf(fd_note_buf, FD_DESC_SZ, "reading %s #%d", shortname, k + 1);
-	    fd_note(rfd, fd_note_buf);
-	    snprintf(fd_note_buf, FD_DESC_SZ, "writing %s #%d", shortname, k + 1);
-	    fd_note(wfd, fd_note_buf);
-	}
-	commSetNonBlocking(rfd);
-	if (wfd != rfd)
-	    commSetNonBlocking(wfd);
-	comm_add_close_handler(rfd, helperStatefulServerFree, srv);
-    }
-    safe_free(shortname);
-    safe_free(procname);
-    helperStatefulKickQueue(hlp);
-}
-
-
-void
-helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data)
-{
-    helper_request *r = memAllocate(MEM_HELPER_REQUEST);
-    helper_server *srv;
-    if (hlp == NULL) {
-	debug(84, 3) ("helperSubmit: hlp == NULL\n");
-	callback(data, NULL);
-	return;
-    }
-    r->callback = callback;
-    r->data = cbdataReference(data);
-    r->buf = xstrdup(buf);
-    if ((srv = GetFirstAvailable(hlp)))
-	helperDispatch(srv, r);
-    else
-	Enqueue(hlp, r);
-    debug(84, 9) ("helperSubmit: %s\n", buf);
-}
-
-/* lastserver = "server last used as part of a deferred or reserved
- * request sequence"
- */
-void
-helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPSCB * callback, void *data, helper_stateful_server * lastserver)
-{
-    helper_stateful_request *r = memAllocate(MEM_HELPER_STATEFUL_REQUEST);
-    helper_stateful_server *srv;
-    if (hlp == NULL) {
-	debug(84, 3) ("helperStatefulSubmit: hlp == NULL\n");
-	callback(data, 0, NULL);
-	return;
-    }
-    r->callback = callback;
-    r->data = cbdataReference(data);
-    if (buf != NULL) {
-	r->buf = xstrdup(buf);
-	r->placeholder = 0;
-    } else {
-	r->buf = NULL;
-	r->placeholder = 1;
-    }
-    if ((buf != NULL) && lastserver) {
-	debug(84, 5) ("StatefulSubmit with lastserver %p\n", lastserver);
-	/* the queue doesn't count for this assert because queued requests
-	 * have already gone through here and been tested.
-	 * It's legal to have deferred_requests == 0 and queue entries 
-	 * and status of S_HELPEER_DEFERRED.
-	 * BUT:  It's not legal to submit a new request w/lastserver in
-	 * that state.
-	 */
-	assert(!(lastserver->deferred_requests == 0 &&
-		lastserver->flags.reserved == S_HELPER_DEFERRED));
-	if (lastserver->flags.reserved != S_HELPER_RESERVED) {
-	    lastserver->stats.submits++;
-	    lastserver->deferred_requests--;
-	}
-	if (!(lastserver->request)) {
-	    debug(84, 5) ("StatefulSubmit dispatching\n");
-	    helperStatefulDispatch(lastserver, r);
-	} else {
-	    debug(84, 5) ("StatefulSubmit queuing\n");
-	    StatefulServerEnqueue(lastserver, r);
-	}
-    } else {
-	if ((srv = StatefulGetFirstAvailable(hlp))) {
-	    helperStatefulDispatch(srv, r);
-	} else
-	    StatefulEnqueue(hlp, r);
-    }
-    debug(84, 9) ("helperStatefulSubmit: placeholder: '%d', buf '%s'.\n", r->placeholder, buf);
-}
-
-helper_stateful_server *
-helperStatefulDefer(statefulhelper * hlp)
-/* find and add a deferred request to a server */
-{
-    dlink_node *n;
-    helper_stateful_server *srv = NULL, *rv = NULL;
-    if (hlp == NULL) {
-	debug(84, 3) ("helperStatefulReserve: hlp == NULL\n");
-	return NULL;
-    }
-    debug(84, 5) ("helperStatefulDefer: Running servers %d.\n", hlp->n_running);
-    if (hlp->n_running == 0) {
-	debug(84, 1) ("helperStatefulDefer: No running servers!. \n");
-	return NULL;
-    }
-    srv = StatefulGetFirstAvailable(hlp);
-    /* all currently busy:loop through servers and find server with the shortest queue */
-    rv = srv;
-    if (rv == NULL)
-	for (n = hlp->servers.head; n != NULL; n = n->next) {
-	    srv = n->data;
-	    if (srv->flags.reserved == S_HELPER_RESERVED)
-		continue;
-	    if (!srv->flags.alive)
-		continue;
-	    if ((hlp->IsAvailable != NULL) && (srv->data != NULL) &&
-		!(hlp->IsAvailable(srv->data)))
-		continue;
-	    if ((rv != NULL) && (rv->deferred_requests < srv->deferred_requests))
-		continue;
-	    rv = srv;
-	}
-    if (rv == NULL) {
-	debug(84, 1) ("helperStatefulDefer: None available.\n");
-	return NULL;
-    }
-    /* consistency check:
-     * when the deferred count is 0,
-     *   submits + releases == deferbyfunc + deferbycb
-     * Or in english, when there are no deferred requests, the amount
-     * we have submitted to the queue or cancelled must equal the amount
-     * we have said we wanted to be able to submit or cancel
-     */
-    if (rv->deferred_requests == 0)
-	assert(rv->stats.submits + rv->stats.releases ==
-	    rv->stats.deferbyfunc + rv->stats.deferbycb);
-
-    rv->flags.reserved = S_HELPER_DEFERRED;
-    rv->deferred_requests++;
-    rv->stats.deferbyfunc++;
-    return rv;
-}
-
-void
-helperStatefulReset(helper_stateful_server * srv)
-/* puts this helper back in the queue. the calling app is required to 
- * manage the state in the helper.
- */
-{
-    statefulhelper *hlp = srv->parent;
-    helper_stateful_request *r;
-    r = srv->request;
-    if (r != NULL) {
-	/* reset attempt DURING an outstaning request */
-	debug(84, 1) ("helperStatefulReset: RESET During request %s \n",
-	    hlp->id_name);
-	srv->flags.busy = 0;
-	srv->offset = 0;
-	helperStatefulRequestFree(r);
-	srv->request = NULL;
-    }
-    srv->flags.busy = 0;
-    if (srv->queue.head) {
-	srv->flags.reserved = S_HELPER_DEFERRED;
-	helperStatefulServerKickQueue(srv);
-    } else {
-	srv->flags.reserved = S_HELPER_FREE;
-	if ((srv->parent->OnEmptyQueue != NULL) && (srv->data))
-	    srv->parent->OnEmptyQueue(srv->data);
-	helperStatefulKickQueue(hlp);
-    }
-}
-
-void
-helperStatefulReleaseServer(helper_stateful_server * srv)
-/*decrease the number of 'waiting' clients that set the helper to be DEFERRED */
-{
-    srv->stats.releases++;
-    if (srv->flags.reserved == S_HELPER_DEFERRED) {
-	assert(srv->deferred_requests);
-	srv->deferred_requests--;
-    }
-    if (!(srv->deferred_requests) && (srv->flags.reserved == S_HELPER_DEFERRED) && !(srv->queue.head)) {
-	srv->flags.reserved = S_HELPER_FREE;
-	if ((srv->parent->OnEmptyQueue != NULL) && (srv->data))
-	    srv->parent->OnEmptyQueue(srv->data);
-    }
-}
-
-void *
-helperStatefulServerGetData(helper_stateful_server * srv)
-/* return a pointer to the stateful routines data area */
-{
-    return srv->data;
-}
-
-void
-helperStats(StoreEntry * sentry, helper * hlp)
-{
-    helper_server *srv;
-    dlink_node *link;
-    double tt;
-    storeAppendPrintf(sentry, "number running: %d of %d\n",
-	hlp->n_running, hlp->n_to_start);
-    storeAppendPrintf(sentry, "requests sent: %d\n",
-	hlp->stats.requests);
-    storeAppendPrintf(sentry, "replies received: %d\n",
-	hlp->stats.replies);
-    storeAppendPrintf(sentry, "queue length: %d\n",
-	hlp->stats.queue_size);
-    storeAppendPrintf(sentry, "avg service time: %.2f msec\n",
-	(double) hlp->stats.avg_svc_time / 1000.0);
-    storeAppendPrintf(sentry, "\n");
-    storeAppendPrintf(sentry, "%7s\t%7s\t%7s\t%11s\t%s\t%7s\t%7s\t%7s\n",
-	"#",
-	"FD",
-	"PID",
-	"# Requests",
-	"Flags",
-	"Time",
-	"Offset",
-	"Request");
-    for (link = hlp->servers.head; link; link = link->next) {
-	srv = link->data;
-	tt = 0.001 * tvSubMsec(srv->dispatch_time,
-	    srv->flags.busy ? current_time : srv->answer_time);
-	storeAppendPrintf(sentry, "%7d\t%7d\t%7d\t%11d\t%c%c%c%c\t%7.3f\t%7d\t%s\n",
-	    srv->index + 1,
-	    srv->rfd,
-	    srv->pid,
-	    srv->stats.uses,
-	    srv->flags.alive ? 'A' : ' ',
-	    srv->flags.busy ? 'B' : ' ',
-	    srv->flags.closing ? 'C' : ' ',
-	    srv->flags.shutdown ? 'S' : ' ',
-	    tt < 0.0 ? 0.0 : tt,
-	    (int) srv->offset,
-	    srv->request ? log_quote(srv->request->buf) : "(none)");
-    }
-    storeAppendPrintf(sentry, "\nFlags key:\n\n");
-    storeAppendPrintf(sentry, "   A = ALIVE\n");
-    storeAppendPrintf(sentry, "   B = BUSY\n");
-    storeAppendPrintf(sentry, "   C = CLOSING\n");
-    storeAppendPrintf(sentry, "   S = SHUTDOWN\n");
-}
-
-void
-helperStatefulStats(StoreEntry * sentry, statefulhelper * hlp)
-{
-    helper_stateful_server *srv;
-    dlink_node *link;
-    double tt;
-    storeAppendPrintf(sentry, "number running: %d of %d\n",
-	hlp->n_running, hlp->n_to_start);
-    storeAppendPrintf(sentry, "requests sent: %d\n",
-	hlp->stats.requests);
-    storeAppendPrintf(sentry, "replies received: %d\n",
-	hlp->stats.replies);
-    storeAppendPrintf(sentry, "queue length: %d\n",
-	hlp->stats.queue_size);
-    storeAppendPrintf(sentry, "avg service time: %d msec\n",
-	hlp->stats.avg_svc_time);
-    storeAppendPrintf(sentry, "\n");
-    storeAppendPrintf(sentry, "%7s\t%7s\t%7s\t%11s\t%s\t%7s\t%7s\t%7s\t%7s\n",
-	"#",
-	"FD",
-	"PID",
-	"# Requests",
-	"# Deferred Requests",
-	"Flags",
-	"Time",
-	"Offset",
-	"Request");
-    for (link = hlp->servers.head; link; link = link->next) {
-	srv = link->data;
-	tt = 0.001 * tvSubMsec(srv->dispatch_time, current_time);
-	storeAppendPrintf(sentry, "%7d\t%7d\t%7d\t%11d\t%11d\t%c%c%c%c%c%c\t%7.3f\t%7d\t%s\n",
-	    srv->index + 1,
-	    srv->rfd,
-	    srv->pid,
-	    srv->stats.uses,
-	    (int) srv->deferred_requests,
-	    srv->flags.alive ? 'A' : ' ',
-	    srv->flags.busy ? 'B' : ' ',
-	    srv->flags.closing ? 'C' : ' ',
-	    srv->flags.reserved != S_HELPER_FREE ? 'R' : ' ',
-	    srv->flags.shutdown ? 'S' : ' ',
-	    srv->request ? (srv->request->placeholder ? 'P' : ' ') : ' ',
-	    tt < 0.0 ? 0.0 : tt,
-	    (int) srv->offset,
-	    srv->request ? log_quote(srv->request->buf) : "(none)");
-    }
-    storeAppendPrintf(sentry, "\nFlags key:\n\n");
-    storeAppendPrintf(sentry, "   A = ALIVE\n");
-    storeAppendPrintf(sentry, "   B = BUSY\n");
-    storeAppendPrintf(sentry, "   C = CLOSING\n");
-    storeAppendPrintf(sentry, "   R = RESERVED or DEFERRED\n");
-    storeAppendPrintf(sentry, "   S = SHUTDOWN\n");
-    storeAppendPrintf(sentry, "   P = PLACEHOLDER\n");
-}
-
-void
-helperShutdown(helper * hlp)
-{
-    dlink_node *link = hlp->servers.head;
-    while (link) {
-	helper_server *srv;
-	srv = link->data;
-	link = link->next;
-	if (!srv->flags.alive) {
-	    debug(34, 3) ("helperShutdown: %s #%d is NOT ALIVE.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	srv->flags.shutdown = 1;	/* request it to shut itself down */
-	if (srv->flags.busy) {
-	    debug(34, 3) ("helperShutdown: %s #%d is BUSY.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	if (srv->flags.closing) {
-	    debug(34, 3) ("helperShutdown: %s #%d is CLOSING.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	srv->flags.closing = 1;
-	/* the rest of the details is dealt with in the helperServerFree
-	 * close handler
-	 */
-	comm_close(srv->rfd);
-    }
-}
-
-void
-helperStatefulShutdown(statefulhelper * hlp)
-{
-    dlink_node *link = hlp->servers.head;
-    helper_stateful_server *srv;
-    while (link) {
-	srv = link->data;
-	link = link->next;
-	if (!srv->flags.alive) {
-	    debug(34, 3) ("helperStatefulShutdown: %s #%d is NOT ALIVE.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	srv->flags.shutdown = 1;	/* request it to shut itself down */
-	if (srv->flags.busy) {
-	    debug(34, 3) ("helperStatefulShutdown: %s #%d is BUSY.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	if (srv->flags.closing) {
-	    debug(34, 3) ("helperStatefulShutdown: %s #%d is CLOSING.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	if (srv->flags.reserved != S_HELPER_FREE) {
-	    debug(34, 3) ("helperStatefulShutdown: %s #%d is RESERVED.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	if (srv->deferred_requests) {
-	    debug(34, 3) ("helperStatefulShutdown: %s #%d has DEFERRED requests.\n",
-		hlp->id_name, srv->index + 1);
-	    continue;
-	}
-	srv->flags.closing = 1;
-	/* the rest of the details is dealt with in the helperStatefulServerFree
-	 * close handler
-	 */
-	comm_close(srv->rfd);
-    }
-}
-
-
-helper *
-helperCreate(const char *name)
-{
-    helper *hlp;
-    hlp = cbdataAlloc(helper);
-    hlp->id_name = name;
-    return hlp;
-}
-
-statefulhelper *
-helperStatefulCreate(const char *name)
-{
-    statefulhelper *hlp;
-    hlp = cbdataAlloc(statefulhelper);
-    hlp->id_name = name;
-    return hlp;
-}
-
-
-void
-helperFree(helper * hlp)
-{
-    if (!hlp)
-	return;
-    /* note, don't free hlp->name, it probably points to static memory */
-    if (hlp->queue.head)
-	debug(84, 0) ("WARNING: freeing %s helper with %d requests queued\n",
-	    hlp->id_name, hlp->stats.queue_size);
-    cbdataFree(hlp);
-}
-
-void
-helperStatefulFree(statefulhelper * hlp)
-{
-    if (!hlp)
-	return;
-    /* note, don't free hlp->name, it probably points to static memory */
-    if (hlp->queue.head)
-	debug(84, 0) ("WARNING: freeing %s helper with %d requests queued\n",
-	    hlp->id_name, hlp->stats.queue_size);
-    cbdataFree(hlp);
-}
-
-
-/* ====================================================================== */
-/* LOCAL FUNCTIONS */
-/* ====================================================================== */
-
-static void
-helperServerFree(int fd, void *data)
-{
-    helper_server *srv = data;
-    helper *hlp = srv->parent;
-    helper_request *r;
-    assert(srv->rfd == fd);
-    if (srv->buf) {
-	memFree(srv->buf, MEM_8K_BUF);
-	srv->buf = NULL;
-    }
-    if ((r = srv->request)) {
-	void *cbdata;
-	if (cbdataReferenceValidDone(r->data, &cbdata))
-	    r->callback(cbdata, srv->buf);
-	helperRequestFree(r);
-	srv->request = NULL;
-    }
-    if (srv->wfd != srv->rfd && srv->wfd != -1)
-	comm_close(srv->wfd);
-    dlinkDelete(&srv->link, &hlp->servers);
-    hlp->n_running--;
-    assert(hlp->n_running >= 0);
-    if (!srv->flags.shutdown) {
-	debug(34, 0) ("WARNING: %s #%d (FD %d) exited\n",
-	    hlp->id_name, srv->index + 1, fd);
-	if (hlp->n_running < hlp->n_to_start / 2)
-	    fatalf("Too few %s processes are running", hlp->id_name);
-    }
-    cbdataReferenceDone(srv->parent);
-    cbdataFree(srv);
-}
-
-static void
-helperStatefulServerFree(int fd, void *data)
-{
-    helper_stateful_server *srv = data;
-    statefulhelper *hlp = srv->parent;
-    helper_stateful_request *r;
-    assert(srv->rfd == fd);
-    if (srv->buf) {
-	memFree(srv->buf, MEM_8K_BUF);
-	srv->buf = NULL;
-    }
-    if ((r = srv->request)) {
-	void *cbdata;
-	if (cbdataReferenceValidDone(r->data, &cbdata))
-	    r->callback(cbdata, srv, srv->buf);
-	helperStatefulRequestFree(r);
-	srv->request = NULL;
-    }
-    /* TODO: walk the local queue of requests and carry them all out */
-    if (srv->wfd != srv->rfd && srv->wfd != -1)
-	comm_close(srv->wfd);
-    dlinkDelete(&srv->link, &hlp->servers);
-    hlp->n_running--;
-    assert(hlp->n_running >= 0);
-    if (!srv->flags.shutdown) {
-	debug(34, 0) ("WARNING: %s #%d (FD %d) exited\n",
-	    hlp->id_name, srv->index + 1, fd);
-	if (hlp->n_running < hlp->n_to_start / 2)
-	    fatalf("Too few %s processes are running", hlp->id_name);
-    }
-    if (srv->data != NULL)
-	memPoolFree(hlp->datapool, srv->data);
-    cbdataReferenceDone(srv->parent);
-    cbdataFree(srv);
-}
-
-
-static void
-helperHandleRead(int fd, void *data)
-{
-    int len;
-    char *t = NULL;
-    helper_server *srv = data;
-    helper_request *r;
-    helper *hlp = srv->parent;
-    assert(fd == srv->rfd);
-    assert(cbdataReferenceValid(data));
-    statCounter.syscalls.sock.reads++;
-    len = FD_READ_METHOD(fd, srv->buf + srv->offset, srv->buf_sz - srv->offset);
-    fd_bytes(fd, len, FD_READ);
-    debug(84, 5) ("helperHandleRead: %d bytes from %s #%d.\n",
-	len, hlp->id_name, srv->index + 1);
-    if (len <= 0) {
-	if (len < 0)
-	    debug(50, 1) ("helperHandleRead: FD %d read: %s\n", fd, xstrerror());
-	comm_close(fd);
-	return;
-    }
-    srv->offset += len;
-    srv->buf[srv->offset] = '\0';
-    r = srv->request;
-    if (r == NULL) {
-	/* someone spoke without being spoken to */
-	debug(84, 1) ("helperHandleRead: unexpected read from %s #%d, %d bytes\n",
-	    hlp->id_name, srv->index + 1, len);
-	srv->offset = 0;
-    } else if ((t = strchr(srv->buf, '\n'))) {
-	/* end of reply found */
-	HLPCB *callback;
-	void *cbdata;
-	debug(84, 3) ("helperHandleRead: end of reply found\n");
-	*t = '\0';
-	callback = r->callback;
-	r->callback = NULL;
-	if (cbdataReferenceValidDone(r->data, &cbdata))
-	    callback(cbdata, srv->buf);
-	srv->flags.busy = 0;
-	srv->offset = 0;
-	helperRequestFree(r);
-	srv->request = NULL;
-	hlp->stats.replies++;
-	srv->answer_time = current_time;
-	hlp->stats.avg_svc_time =
-	    intAverage(hlp->stats.avg_svc_time,
-	    tvSubUsec(srv->dispatch_time, current_time),
-	    hlp->stats.replies, REDIRECT_AV_FACTOR);
-	if (srv->flags.shutdown) {
-	    int wfd = srv->wfd;
-	    srv->wfd = -1;
-	    comm_close(wfd);
-	} else
-	    helperKickQueue(hlp);
-    } else {
-	commSetSelect(srv->rfd, COMM_SELECT_READ, helperHandleRead, srv, 0);
-    }
-}
-
-static void
-helperStatefulHandleRead(int fd, void *data)
-{
-    int len;
-    char *t = NULL;
-    helper_stateful_server *srv = data;
-    helper_stateful_request *r;
-    statefulhelper *hlp = srv->parent;
-    assert(fd == srv->rfd);
-    assert(cbdataReferenceValid(data));
-    statCounter.syscalls.sock.reads++;
-    len = FD_READ_METHOD(fd, srv->buf + srv->offset, srv->buf_sz - srv->offset);
-    fd_bytes(fd, len, FD_READ);
-    debug(84, 5) ("helperStatefulHandleRead: %d bytes from %s #%d.\n",
-	len, hlp->id_name, srv->index + 1);
-    if (len <= 0) {
-	if (len < 0)
-	    debug(50, 1) ("helperStatefulHandleRead: FD %d read: %s\n", fd, xstrerror());
-	comm_close(fd);
-	return;
-    }
-    srv->offset += len;
-    srv->buf[srv->offset] = '\0';
-    r = srv->request;
-    if (r == NULL) {
-	/* someone spoke without being spoken to */
-	debug(84, 1) ("helperStatefulHandleRead: unexpected read from %s #%d, %d bytes\n",
-	    hlp->id_name, srv->index + 1, len);
-	srv->offset = 0;
-    } else if ((t = strchr(srv->buf, '\n'))) {
-	/* end of reply found */
-	debug(84, 3) ("helperStatefulHandleRead: end of reply found\n");
-	*t = '\0';
-	if (cbdataReferenceValid(r->data)) {
-	    switch ((r->callback(r->data, srv, srv->buf))) {	/*if non-zero reserve helper */
-	    case S_HELPER_UNKNOWN:
-		fatal("helperStatefulHandleRead: either a non-state aware callback was give to the stateful helper routines, or an uninitialised callback response was recieved.\n");
-		break;
-	    case S_HELPER_RELEASE:	/* helper finished with */
-		if (!srv->deferred_requests && !srv->queue.head) {
-		    srv->flags.reserved = S_HELPER_FREE;
-		    if ((srv->parent->OnEmptyQueue != NULL) && (srv->data))
-			srv->parent->OnEmptyQueue(srv->data);
-		    debug(84, 5) ("StatefulHandleRead: releasing %s #%d\n", hlp->id_name, srv->index + 1);
-		} else {
-		    srv->flags.reserved = S_HELPER_DEFERRED;
-		    debug(84, 5) ("StatefulHandleRead: outstanding deferred requests on %s #%d. reserving for deferred requests.\n", hlp->id_name, srv->index + 1);
-		}
-		break;
-	    case S_HELPER_RESERVE:	/* 'pin' this helper for the caller */
-		if (!srv->queue.head) {
-		    assert(srv->deferred_requests == 0);
-		    srv->flags.reserved = S_HELPER_RESERVED;
-		    debug(84, 5) ("StatefulHandleRead: reserving %s #%d\n", hlp->id_name, srv->index + 1);
-		} else {
-		    fatal("StatefulHandleRead: Callback routine attempted to reserve a stateful helper with deferred requests. This can lead to deadlock.\n");
-		}
-		break;
-	    case S_HELPER_DEFER:
-		/* the helper is still needed, but can
-		 * be used for other requests in the meantime.
-		 */
-		srv->flags.reserved = S_HELPER_DEFERRED;
-		srv->deferred_requests++;
-		srv->stats.deferbycb++;
-		debug(84, 5) ("StatefulHandleRead: reserving %s #%d for deferred requests.\n", hlp->id_name, srv->index + 1);
-		break;
-	    default:
-		fatal("helperStatefulHandleRead: unknown stateful helper callback result.\n");
-	    }
-
-	} else {
-	    debug(84, 1) ("StatefulHandleRead: no callback data registered\n");
-	}
-	srv->flags.busy = 0;
-	srv->offset = 0;
-	helperStatefulRequestFree(r);
-	srv->request = NULL;
-	hlp->stats.replies++;
-	hlp->stats.avg_svc_time =
-	    intAverage(hlp->stats.avg_svc_time,
-	    tvSubMsec(srv->dispatch_time, current_time),
-	    hlp->stats.replies, REDIRECT_AV_FACTOR);
-	if (srv->flags.shutdown
-	    && srv->flags.reserved == S_HELPER_FREE
-	    && !srv->deferred_requests) {
-	    int wfd = srv->wfd;
-	    srv->wfd = -1;
-	    comm_close(wfd);
-	} else {
-	    if (srv->queue.head)
-		helperStatefulServerKickQueue(srv);
-	    else
-		helperStatefulKickQueue(hlp);
-	}
-    } else {
-	commSetSelect(srv->rfd, COMM_SELECT_READ, helperStatefulHandleRead, srv, 0);
-    }
-}
-
-static void
-Enqueue(helper * hlp, helper_request * r)
-{
-    dlink_node *link = memAllocate(MEM_DLINK_NODE);
-    dlinkAddTail(r, link, &hlp->queue);
-    hlp->stats.queue_size++;
-    if (hlp->stats.queue_size < hlp->n_running)
-	return;
-    if (squid_curtime - hlp->last_queue_warn < 600)
-	return;
-    if (shutting_down || reconfiguring)
-	return;
-    hlp->last_queue_warn = squid_curtime;
-    debug(84, 0) ("WARNING: All %s processes are busy.\n", hlp->id_name);
-    debug(84, 0) ("WARNING: %d pending requests queued\n", hlp->stats.queue_size);
-    if (hlp->stats.queue_size > hlp->n_running * 2)
-	fatalf("Too many queued %s requests", hlp->id_name);
-    debug(84, 1) ("Consider increasing the number of %s processes in your config file.\n", hlp->id_name);
-}
-
-static void
-StatefulEnqueue(statefulhelper * hlp, helper_stateful_request * r)
-{
-    dlink_node *link = memAllocate(MEM_DLINK_NODE);
-    dlinkAddTail(r, link, &hlp->queue);
-    hlp->stats.queue_size++;
-    if (hlp->stats.queue_size < hlp->n_running)
-	return;
-    if (hlp->stats.queue_size > hlp->n_running * 2)
-	fatalf("Too many queued %s requests", hlp->id_name);
-    if (squid_curtime - hlp->last_queue_warn < 600)
-	return;
-    if (shutting_down || reconfiguring)
-	return;
-    hlp->last_queue_warn = squid_curtime;
-    debug(84, 0) ("WARNING: All %s processes are busy.\n", hlp->id_name);
-    debug(84, 0) ("WARNING: %d pending requests queued\n", hlp->stats.queue_size);
-    debug(84, 1) ("Consider increasing the number of %s processes in your config file.\n", hlp->id_name);
-}
-
-static void
-StatefulServerEnqueue(helper_stateful_server * srv, helper_stateful_request * r)
-{
-    dlink_node *link = memAllocate(MEM_DLINK_NODE);
-    dlinkAddTail(r, link, &srv->queue);
-/* TODO: warning if the queue on this server is more than X
- * We don't check the queue size at the moment, because
- * requests hitting here are deferrable 
- */
-/*    hlp->stats.queue_size++;
- * if (hlp->stats.queue_size < hlp->n_running)
- * return;
- * if (squid_curtime - hlp->last_queue_warn < 600)
- * return;
- * if (shutting_down || reconfiguring)
- * return;
- * hlp->last_queue_warn = squid_curtime;
- * debug(84, 0) ("WARNING: All %s processes are busy.\n", hlp->id_name);
- * debug(84, 0) ("WARNING: %d pending requests queued\n", hlp->stats.queue_size);
- * if (hlp->stats.queue_size > hlp->n_running * 2)
- * fatalf("Too many queued %s requests", hlp->id_name);
- * debug(84, 1) ("Consider increasing the number of %s processes in your config file.\n", hlp->id_name);  */
-}
-
-
-static helper_request *
-Dequeue(helper * hlp)
-{
-    dlink_node *link;
-    helper_request *r = NULL;
-    if ((link = hlp->queue.head)) {
-	r = link->data;
-	dlinkDelete(link, &hlp->queue);
-	memFree(link, MEM_DLINK_NODE);
-	hlp->stats.queue_size--;
-    }
-    return r;
-}
-
-static helper_stateful_request *
-StatefulServerDequeue(helper_stateful_server * srv)
-{
-    dlink_node *link;
-    helper_stateful_request *r = NULL;
-    if ((link = srv->queue.head)) {
-	r = link->data;
-	dlinkDelete(link, &srv->queue);
-	memFree(link, MEM_DLINK_NODE);
-    }
-    return r;
-}
-
-static helper_stateful_request *
-StatefulDequeue(statefulhelper * hlp)
-{
-    dlink_node *link;
-    helper_stateful_request *r = NULL;
-    if ((link = hlp->queue.head)) {
-	r = link->data;
-	dlinkDelete(link, &hlp->queue);
-	memFree(link, MEM_DLINK_NODE);
-	hlp->stats.queue_size--;
-    }
-    return r;
-}
-
-static helper_server *
-GetFirstAvailable(helper * hlp)
-{
-    dlink_node *n;
-    helper_server *srv = NULL;
-    if (hlp->n_running == 0)
-	return NULL;
-    for (n = hlp->servers.head; n != NULL; n = n->next) {
-	srv = n->data;
-	if (srv->flags.busy)
-	    continue;
-	if (!srv->flags.alive)
-	    continue;
-	return srv;
-    }
-    return NULL;
-}
-
-static helper_stateful_server *
-StatefulGetFirstAvailable(statefulhelper * hlp)
-{
-    dlink_node *n;
-    helper_stateful_server *srv = NULL;
-    debug(84, 5) ("StatefulGetFirstAvailable: Running servers %d.\n", hlp->n_running);
-    if (hlp->n_running == 0)
-	return NULL;
-    for (n = hlp->servers.head; n != NULL; n = n->next) {
-	srv = n->data;
-	if (srv->flags.busy)
-	    continue;
-	if (srv->flags.reserved == S_HELPER_RESERVED)
-	    continue;
-	if (!srv->flags.alive)
-	    continue;
-	if ((hlp->IsAvailable != NULL) && (srv->data != NULL) && !(hlp->IsAvailable(srv->data)))
-	    continue;
-	return srv;
-    }
-    debug(84, 5) ("StatefulGetFirstAvailable: None available.\n");
-    return NULL;
-}
-
-
-static void
-helperDispatch(helper_server * srv, helper_request * r)
-{
-    helper *hlp = srv->parent;
-    if (!cbdataReferenceValid(r->data)) {
-	debug(84, 1) ("helperDispatch: invalid callback data\n");
-	helperRequestFree(r);
-	return;
-    }
-    assert(!srv->flags.busy);
-    srv->flags.busy = 1;
-    srv->request = r;
-    srv->dispatch_time = current_time;
-    comm_write(srv->wfd,
-	r->buf,
-	strlen(r->buf),
-	NULL,			/* Handler */
-	NULL,			/* Handler-data */
-	NULL);			/* free */
-    commSetSelect(srv->rfd,
-	COMM_SELECT_READ,
-	helperHandleRead,
-	srv, 0);
-    debug(84, 5) ("helperDispatch: Request sent to %s #%d, %d bytes\n",
-	hlp->id_name, srv->index + 1, (int) strlen(r->buf));
-    srv->stats.uses++;
-    hlp->stats.requests++;
-}
-
-static void
-helperStatefulDispatch(helper_stateful_server * srv, helper_stateful_request * r)
-{
-    statefulhelper *hlp = srv->parent;
-    if (!cbdataReferenceValid(r->data)) {
-	debug(84, 1) ("helperStatefulDispatch: invalid callback data\n");
-	helperStatefulRequestFree(r);
-	return;
-    }
-    debug(84, 9) ("helperStatefulDispatch busying helper %s #%d\n", hlp->id_name, srv->index + 1);
-    if (r->placeholder == 1) {
-	/* a callback is needed before this request can _use_ a helper. */
-	/* we don't care about releasing/deferring this helper. The request NEVER
-	 * gets to the helper. So we throw away the return code */
-	r->callback(r->data, srv, NULL);
-	/* throw away the placeholder */
-	helperStatefulRequestFree(r);
-	/* and push the queue. Note that the callback may have submitted a new 
-	 * request to the helper which is why we test for the request*/
-	if (srv->request == NULL) {
-	    if (srv->flags.shutdown
-		&& srv->flags.reserved == S_HELPER_FREE
-		&& !srv->deferred_requests) {
-		int wfd = srv->wfd;
-		srv->wfd = -1;
-		comm_close(wfd);
-	    } else {
-		if (srv->queue.head)
-		    helperStatefulServerKickQueue(srv);
-		else
-		    helperStatefulKickQueue(hlp);
-	    }
-	}
-	return;
-    }
-    srv->flags.busy = 1;
-    srv->request = r;
-    srv->dispatch_time = current_time;
-    comm_write(srv->wfd,
-	r->buf,
-	strlen(r->buf),
-	NULL,			/* Handler */
-	NULL,			/* Handler-data */
-	NULL);			/* free */
-    commSetSelect(srv->rfd,
-	COMM_SELECT_READ,
-	helperStatefulHandleRead,
-	srv, 0);
-    debug(84, 5) ("helperStatefulDispatch: Request sent to %s #%d, %d bytes\n",
-	hlp->id_name, srv->index + 1, (int) strlen(r->buf));
-    srv->stats.uses++;
-    hlp->stats.requests++;
-}
-
-
-static void
-helperKickQueue(helper * hlp)
-{
-    helper_request *r;
-    helper_server *srv;
-    while ((srv = GetFirstAvailable(hlp)) && (r = Dequeue(hlp)))
-	helperDispatch(srv, r);
-}
-
-static void
-helperStatefulKickQueue(statefulhelper * hlp)
-{
-    helper_stateful_request *r;
-    helper_stateful_server *srv;
-    while ((srv = StatefulGetFirstAvailable(hlp)) && (r = StatefulDequeue(hlp)))
-	helperStatefulDispatch(srv, r);
-}
-
-static void
-helperStatefulServerKickQueue(helper_stateful_server * srv)
-{
-    helper_stateful_request *r;
-    if ((r = StatefulServerDequeue(srv)))
-	helperStatefulDispatch(srv, r);
-}
-
-static void
-helperRequestFree(helper_request * r)
-{
-    cbdataReferenceDone(r->data);
-    xfree(r->buf);
-    memFree(r, MEM_HELPER_REQUEST);
-}
-
-static void
-helperStatefulRequestFree(helper_stateful_request * r)
-{
-    cbdataReferenceDone(r->data);
-    xfree(r->buf);
-    memFree(r, MEM_HELPER_STATEFUL_REQUEST);
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/helper.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,1132 @@
+
+/*
+ * $Id: helper.cc,v 1.1.2.1 2002/10/11 15:40:51 rbcollins Exp $
+ *
+ * DEBUG: section 84    Helper process maintenance
+ * AUTHOR: Harvest Derived?
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "Store.h"
+
+#define HELPER_MAX_ARGS 64
+
+static PF helperHandleRead;
+static PF helperStatefulHandleRead;
+static PF helperServerFree;
+static PF helperStatefulServerFree;
+static void Enqueue(helper * hlp, helper_request *);
+static helper_request *Dequeue(helper * hlp);
+static helper_stateful_request *StatefulDequeue(statefulhelper * hlp);
+static helper_server *GetFirstAvailable(helper * hlp);
+static helper_stateful_server *StatefulGetFirstAvailable(statefulhelper * hlp);
+static void helperDispatch(helper_server * srv, helper_request * r);
+static void helperStatefulDispatch(helper_stateful_server * srv, helper_stateful_request * r);
+static void helperKickQueue(helper * hlp);
+static void helperStatefulKickQueue(statefulhelper * hlp);
+static void helperRequestFree(helper_request * r);
+static void helperStatefulRequestFree(helper_stateful_request * r);
+static void StatefulEnqueue(statefulhelper * hlp, helper_stateful_request * r);
+static helper_stateful_request *StatefulServerDequeue(helper_stateful_server * srv);
+static void StatefulServerEnqueue(helper_stateful_server * srv, helper_stateful_request * r);
+static void helperStatefulServerKickQueue(helper_stateful_server * srv);
+
+void
+helperOpenServers(helper * hlp)
+{
+    char *s;
+    char *progname;
+    char *shortname;
+    char *procname;
+    const char *args[HELPER_MAX_ARGS];
+    char fd_note_buf[FD_DESC_SZ];
+    helper_server *srv;
+    int nargs = 0;
+    int k;
+    int x;
+    int rfd;
+    int wfd;
+    wordlist *w;
+    if (hlp->cmdline == NULL)
+	return;
+    progname = hlp->cmdline->key;
+    if ((s = strrchr(progname, '/')))
+	shortname = xstrdup(s + 1);
+    else
+	shortname = xstrdup(progname);
+    debug(84, 1) ("helperOpenServers: Starting %d '%s' processes\n",
+	hlp->n_to_start, shortname);
+    procname = (char *)xmalloc(strlen(shortname) + 3);
+    snprintf(procname, strlen(shortname) + 3, "(%s)", shortname);
+    args[nargs++] = procname;
+    for (w = hlp->cmdline->next; w && nargs < HELPER_MAX_ARGS; w = w->next)
+	args[nargs++] = w->key;
+    args[nargs++] = NULL;
+    assert(nargs <= HELPER_MAX_ARGS);
+    for (k = 0; k < hlp->n_to_start; k++) {
+	getCurrentTime();
+	rfd = wfd = -1;
+	x = ipcCreate(hlp->ipc_type,
+	    progname,
+	    args,
+	    shortname,
+	    &rfd,
+	    &wfd);
+	if (x < 0) {
+	    debug(84, 1) ("WARNING: Cannot run '%s' process.\n", progname);
+	    continue;
+	}
+	hlp->n_running++;
+	srv = cbdataAlloc(helper_server);
+	srv->pid = x;
+	srv->flags.alive = 1;
+	srv->index = k;
+	srv->rfd = rfd;
+	srv->wfd = wfd;
+	srv->buf = (char *)memAllocate(MEM_8K_BUF);
+	srv->buf_sz = 8192;
+	srv->offset = 0;
+	srv->parent = cbdataReference(hlp);
+	dlinkAddTail(srv, &srv->link, &hlp->servers);
+	if (rfd == wfd) {
+	    snprintf(fd_note_buf, FD_DESC_SZ, "%s #%d", shortname, k + 1);
+	    fd_note(rfd, fd_note_buf);
+	} else {
+	    snprintf(fd_note_buf, FD_DESC_SZ, "reading %s #%d", shortname, k + 1);
+	    fd_note(rfd, fd_note_buf);
+	    snprintf(fd_note_buf, FD_DESC_SZ, "writing %s #%d", shortname, k + 1);
+	    fd_note(wfd, fd_note_buf);
+	}
+	commSetNonBlocking(rfd);
+	if (wfd != rfd)
+	    commSetNonBlocking(wfd);
+	comm_add_close_handler(rfd, helperServerFree, srv);
+    }
+    safe_free(shortname);
+    safe_free(procname);
+    helperKickQueue(hlp);
+}
+
+void
+helperStatefulOpenServers(statefulhelper * hlp)
+{
+    char *s;
+    char *progname;
+    char *shortname;
+    char *procname;
+    const char *args[HELPER_MAX_ARGS];
+    char fd_note_buf[FD_DESC_SZ];
+    helper_stateful_server *srv;
+    int nargs = 0;
+    int k;
+    int x;
+    int rfd;
+    int wfd;
+    wordlist *w;
+    if (hlp->cmdline == NULL)
+	return;
+    progname = hlp->cmdline->key;
+    if ((s = strrchr(progname, '/')))
+	shortname = xstrdup(s + 1);
+    else
+	shortname = xstrdup(progname);
+    debug(84, 1) ("helperStatefulOpenServers: Starting %d '%s' processes\n",
+	hlp->n_to_start, shortname);
+    procname = (char *)xmalloc(strlen(shortname) + 3);
+    snprintf(procname, strlen(shortname) + 3, "(%s)", shortname);
+    args[nargs++] = procname;
+    for (w = hlp->cmdline->next; w && nargs < HELPER_MAX_ARGS; w = w->next)
+	args[nargs++] = w->key;
+    args[nargs++] = NULL;
+    assert(nargs <= HELPER_MAX_ARGS);
+    for (k = 0; k < hlp->n_to_start; k++) {
+	getCurrentTime();
+	rfd = wfd = -1;
+	x = ipcCreate(hlp->ipc_type,
+	    progname,
+	    args,
+	    shortname,
+	    &rfd,
+	    &wfd);
+	if (x < 0) {
+	    debug(84, 1) ("WARNING: Cannot run '%s' process.\n", progname);
+	    continue;
+	}
+	hlp->n_running++;
+	srv = cbdataAlloc(helper_stateful_server);
+	srv->pid = x;
+	srv->flags.alive = 1;
+	srv->flags.reserved = S_HELPER_FREE;
+	srv->deferred_requests = 0;
+	srv->stats.deferbyfunc = 0;
+	srv->stats.deferbycb = 0;
+	srv->stats.submits = 0;
+	srv->stats.releases = 0;
+	srv->index = k;
+	srv->rfd = rfd;
+	srv->wfd = wfd;
+	srv->buf = (char *)memAllocate(MEM_8K_BUF);
+	srv->buf_sz = 8192;
+	srv->offset = 0;
+	srv->parent = cbdataReference(hlp);
+	if (hlp->datapool != NULL)
+	    srv->data = memPoolAlloc(hlp->datapool);
+	dlinkAddTail(srv, &srv->link, &hlp->servers);
+	if (rfd == wfd) {
+	    snprintf(fd_note_buf, FD_DESC_SZ, "%s #%d", shortname, k + 1);
+	    fd_note(rfd, fd_note_buf);
+	} else {
+	    snprintf(fd_note_buf, FD_DESC_SZ, "reading %s #%d", shortname, k + 1);
+	    fd_note(rfd, fd_note_buf);
+	    snprintf(fd_note_buf, FD_DESC_SZ, "writing %s #%d", shortname, k + 1);
+	    fd_note(wfd, fd_note_buf);
+	}
+	commSetNonBlocking(rfd);
+	if (wfd != rfd)
+	    commSetNonBlocking(wfd);
+	comm_add_close_handler(rfd, helperStatefulServerFree, srv);
+    }
+    safe_free(shortname);
+    safe_free(procname);
+    helperStatefulKickQueue(hlp);
+}
+
+
+void
+helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data)
+{
+    helper_request *r = (helper_request *)memAllocate(MEM_HELPER_REQUEST);
+    helper_server *srv;
+    if (hlp == NULL) {
+	debug(84, 3) ("helperSubmit: hlp == NULL\n");
+	callback(data, NULL);
+	return;
+    }
+    r->callback = callback;
+    r->data = cbdataReference(data);
+    r->buf = xstrdup(buf);
+    if ((srv = GetFirstAvailable(hlp)))
+	helperDispatch(srv, r);
+    else
+	Enqueue(hlp, r);
+    debug(84, 9) ("helperSubmit: %s\n", buf);
+}
+
+/* lastserver = "server last used as part of a deferred or reserved
+ * request sequence"
+ */
+void
+helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPSCB * callback, void *data, helper_stateful_server * lastserver)
+{
+    helper_stateful_request *r = (helper_stateful_request *)memAllocate(MEM_HELPER_STATEFUL_REQUEST);
+    helper_stateful_server *srv;
+    if (hlp == NULL) {
+	debug(84, 3) ("helperStatefulSubmit: hlp == NULL\n");
+	callback(data, 0, NULL);
+	return;
+    }
+    r->callback = callback;
+    r->data = cbdataReference(data);
+    if (buf != NULL) {
+	r->buf = xstrdup(buf);
+	r->placeholder = 0;
+    } else {
+	r->buf = NULL;
+	r->placeholder = 1;
+    }
+    if ((buf != NULL) && lastserver) {
+	debug(84, 5) ("StatefulSubmit with lastserver %p\n", lastserver);
+	/* the queue doesn't count for this assert because queued requests
+	 * have already gone through here and been tested.
+	 * It's legal to have deferred_requests == 0 and queue entries 
+	 * and status of S_HELPEER_DEFERRED.
+	 * BUT:  It's not legal to submit a new request w/lastserver in
+	 * that state.
+	 */
+	assert(!(lastserver->deferred_requests == 0 &&
+		lastserver->flags.reserved == S_HELPER_DEFERRED));
+	if (lastserver->flags.reserved != S_HELPER_RESERVED) {
+	    lastserver->stats.submits++;
+	    lastserver->deferred_requests--;
+	}
+	if (!(lastserver->request)) {
+	    debug(84, 5) ("StatefulSubmit dispatching\n");
+	    helperStatefulDispatch(lastserver, r);
+	} else {
+	    debug(84, 5) ("StatefulSubmit queuing\n");
+	    StatefulServerEnqueue(lastserver, r);
+	}
+    } else {
+	if ((srv = StatefulGetFirstAvailable(hlp))) {
+	    helperStatefulDispatch(srv, r);
+	} else
+	    StatefulEnqueue(hlp, r);
+    }
+    debug(84, 9) ("helperStatefulSubmit: placeholder: '%d', buf '%s'.\n", r->placeholder, buf);
+}
+
+helper_stateful_server *
+helperStatefulDefer(statefulhelper * hlp)
+/* find and add a deferred request to a server */
+{
+    dlink_node *n;
+    helper_stateful_server *srv = NULL, *rv = NULL;
+    if (hlp == NULL) {
+	debug(84, 3) ("helperStatefulReserve: hlp == NULL\n");
+	return NULL;
+    }
+    debug(84, 5) ("helperStatefulDefer: Running servers %d.\n", hlp->n_running);
+    if (hlp->n_running == 0) {
+	debug(84, 1) ("helperStatefulDefer: No running servers!. \n");
+	return NULL;
+    }
+    srv = StatefulGetFirstAvailable(hlp);
+    /* all currently busy:loop through servers and find server with the shortest queue */
+    rv = srv;
+    if (rv == NULL)
+	for (n = hlp->servers.head; n != NULL; n = n->next) {
+	    srv = (helper_stateful_server *)n->data;
+	    if (srv->flags.reserved == S_HELPER_RESERVED)
+		continue;
+	    if (!srv->flags.alive)
+		continue;
+	    if ((hlp->IsAvailable != NULL) && (srv->data != NULL) &&
+		!(hlp->IsAvailable(srv->data)))
+		continue;
+	    if ((rv != NULL) && (rv->deferred_requests < srv->deferred_requests))
+		continue;
+	    rv = srv;
+	}
+    if (rv == NULL) {
+	debug(84, 1) ("helperStatefulDefer: None available.\n");
+	return NULL;
+    }
+    /* consistency check:
+     * when the deferred count is 0,
+     *   submits + releases == deferbyfunc + deferbycb
+     * Or in english, when there are no deferred requests, the amount
+     * we have submitted to the queue or cancelled must equal the amount
+     * we have said we wanted to be able to submit or cancel
+     */
+    if (rv->deferred_requests == 0)
+	assert(rv->stats.submits + rv->stats.releases ==
+	    rv->stats.deferbyfunc + rv->stats.deferbycb);
+
+    rv->flags.reserved = S_HELPER_DEFERRED;
+    rv->deferred_requests++;
+    rv->stats.deferbyfunc++;
+    return rv;
+}
+
+void
+helperStatefulReset(helper_stateful_server * srv)
+/* puts this helper back in the queue. the calling app is required to 
+ * manage the state in the helper.
+ */
+{
+    statefulhelper *hlp = srv->parent;
+    helper_stateful_request *r;
+    r = srv->request;
+    if (r != NULL) {
+	/* reset attempt DURING an outstaning request */
+	debug(84, 1) ("helperStatefulReset: RESET During request %s \n",
+	    hlp->id_name);
+	srv->flags.busy = 0;
+	srv->offset = 0;
+	helperStatefulRequestFree(r);
+	srv->request = NULL;
+    }
+    srv->flags.busy = 0;
+    if (srv->queue.head) {
+	srv->flags.reserved = S_HELPER_DEFERRED;
+	helperStatefulServerKickQueue(srv);
+    } else {
+	srv->flags.reserved = S_HELPER_FREE;
+	if ((srv->parent->OnEmptyQueue != NULL) && (srv->data))
+	    srv->parent->OnEmptyQueue(srv->data);
+	helperStatefulKickQueue(hlp);
+    }
+}
+
+void
+helperStatefulReleaseServer(helper_stateful_server * srv)
+/*decrease the number of 'waiting' clients that set the helper to be DEFERRED */
+{
+    srv->stats.releases++;
+    if (srv->flags.reserved == S_HELPER_DEFERRED) {
+	assert(srv->deferred_requests);
+	srv->deferred_requests--;
+    }
+    if (!(srv->deferred_requests) && (srv->flags.reserved == S_HELPER_DEFERRED) && !(srv->queue.head)) {
+	srv->flags.reserved = S_HELPER_FREE;
+	if ((srv->parent->OnEmptyQueue != NULL) && (srv->data))
+	    srv->parent->OnEmptyQueue(srv->data);
+    }
+}
+
+void *
+helperStatefulServerGetData(helper_stateful_server * srv)
+/* return a pointer to the stateful routines data area */
+{
+    return srv->data;
+}
+
+void
+helperStats(StoreEntry * sentry, helper * hlp)
+{
+    helper_server *srv;
+    dlink_node *link;
+    double tt;
+    storeAppendPrintf(sentry, "number running: %d of %d\n",
+	hlp->n_running, hlp->n_to_start);
+    storeAppendPrintf(sentry, "requests sent: %d\n",
+	hlp->stats.requests);
+    storeAppendPrintf(sentry, "replies received: %d\n",
+	hlp->stats.replies);
+    storeAppendPrintf(sentry, "queue length: %d\n",
+	hlp->stats.queue_size);
+    storeAppendPrintf(sentry, "avg service time: %.2f msec\n",
+	(double) hlp->stats.avg_svc_time / 1000.0);
+    storeAppendPrintf(sentry, "\n");
+    storeAppendPrintf(sentry, "%7s\t%7s\t%7s\t%11s\t%s\t%7s\t%7s\t%7s\n",
+	"#",
+	"FD",
+	"PID",
+	"# Requests",
+	"Flags",
+	"Time",
+	"Offset",
+	"Request");
+    for (link = hlp->servers.head; link; link = link->next) {
+	srv = (helper_server*)link->data;
+	tt = 0.001 * tvSubMsec(srv->dispatch_time,
+	    srv->flags.busy ? current_time : srv->answer_time);
+	storeAppendPrintf(sentry, "%7d\t%7d\t%7d\t%11d\t%c%c%c%c\t%7.3f\t%7d\t%s\n",
+	    srv->index + 1,
+	    srv->rfd,
+	    srv->pid,
+	    srv->stats.uses,
+	    srv->flags.alive ? 'A' : ' ',
+	    srv->flags.busy ? 'B' : ' ',
+	    srv->flags.closing ? 'C' : ' ',
+	    srv->flags.shutdown ? 'S' : ' ',
+	    tt < 0.0 ? 0.0 : tt,
+	    (int) srv->offset,
+	    srv->request ? log_quote(srv->request->buf) : "(none)");
+    }
+    storeAppendPrintf(sentry, "\nFlags key:\n\n");
+    storeAppendPrintf(sentry, "   A = ALIVE\n");
+    storeAppendPrintf(sentry, "   B = BUSY\n");
+    storeAppendPrintf(sentry, "   C = CLOSING\n");
+    storeAppendPrintf(sentry, "   S = SHUTDOWN\n");
+}
+
+void
+helperStatefulStats(StoreEntry * sentry, statefulhelper * hlp)
+{
+    helper_stateful_server *srv;
+    dlink_node *link;
+    double tt;
+    storeAppendPrintf(sentry, "number running: %d of %d\n",
+	hlp->n_running, hlp->n_to_start);
+    storeAppendPrintf(sentry, "requests sent: %d\n",
+	hlp->stats.requests);
+    storeAppendPrintf(sentry, "replies received: %d\n",
+	hlp->stats.replies);
+    storeAppendPrintf(sentry, "queue length: %d\n",
+	hlp->stats.queue_size);
+    storeAppendPrintf(sentry, "avg service time: %d msec\n",
+	hlp->stats.avg_svc_time);
+    storeAppendPrintf(sentry, "\n");
+    storeAppendPrintf(sentry, "%7s\t%7s\t%7s\t%11s\t%s\t%7s\t%7s\t%7s\t%7s\n",
+	"#",
+	"FD",
+	"PID",
+	"# Requests",
+	"# Deferred Requests",
+	"Flags",
+	"Time",
+	"Offset",
+	"Request");
+    for (link = hlp->servers.head; link; link = link->next) {
+	srv = (helper_stateful_server *)link->data;
+	tt = 0.001 * tvSubMsec(srv->dispatch_time, current_time);
+	storeAppendPrintf(sentry, "%7d\t%7d\t%7d\t%11d\t%11d\t%c%c%c%c%c%c\t%7.3f\t%7d\t%s\n",
+	    srv->index + 1,
+	    srv->rfd,
+	    srv->pid,
+	    srv->stats.uses,
+	    (int) srv->deferred_requests,
+	    srv->flags.alive ? 'A' : ' ',
+	    srv->flags.busy ? 'B' : ' ',
+	    srv->flags.closing ? 'C' : ' ',
+	    srv->flags.reserved != S_HELPER_FREE ? 'R' : ' ',
+	    srv->flags.shutdown ? 'S' : ' ',
+	    srv->request ? (srv->request->placeholder ? 'P' : ' ') : ' ',
+	    tt < 0.0 ? 0.0 : tt,
+	    (int) srv->offset,
+	    srv->request ? log_quote(srv->request->buf) : "(none)");
+    }
+    storeAppendPrintf(sentry, "\nFlags key:\n\n");
+    storeAppendPrintf(sentry, "   A = ALIVE\n");
+    storeAppendPrintf(sentry, "   B = BUSY\n");
+    storeAppendPrintf(sentry, "   C = CLOSING\n");
+    storeAppendPrintf(sentry, "   R = RESERVED or DEFERRED\n");
+    storeAppendPrintf(sentry, "   S = SHUTDOWN\n");
+    storeAppendPrintf(sentry, "   P = PLACEHOLDER\n");
+}
+
+void
+helperShutdown(helper * hlp)
+{
+    dlink_node *link = hlp->servers.head;
+    while (link) {
+	helper_server *srv;
+	srv = (helper_server *)link->data;
+	link = link->next;
+	if (!srv->flags.alive) {
+	    debug(34, 3) ("helperShutdown: %s #%d is NOT ALIVE.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	srv->flags.shutdown = 1;	/* request it to shut itself down */
+	if (srv->flags.busy) {
+	    debug(34, 3) ("helperShutdown: %s #%d is BUSY.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	if (srv->flags.closing) {
+	    debug(34, 3) ("helperShutdown: %s #%d is CLOSING.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	srv->flags.closing = 1;
+	/* the rest of the details is dealt with in the helperServerFree
+	 * close handler
+	 */
+	comm_close(srv->rfd);
+    }
+}
+
+void
+helperStatefulShutdown(statefulhelper * hlp)
+{
+    dlink_node *link = hlp->servers.head;
+    helper_stateful_server *srv;
+    while (link) {
+	srv = (helper_stateful_server *)link->data;
+	link = link->next;
+	if (!srv->flags.alive) {
+	    debug(34, 3) ("helperStatefulShutdown: %s #%d is NOT ALIVE.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	srv->flags.shutdown = 1;	/* request it to shut itself down */
+	if (srv->flags.busy) {
+	    debug(34, 3) ("helperStatefulShutdown: %s #%d is BUSY.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	if (srv->flags.closing) {
+	    debug(34, 3) ("helperStatefulShutdown: %s #%d is CLOSING.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	if (srv->flags.reserved != S_HELPER_FREE) {
+	    debug(34, 3) ("helperStatefulShutdown: %s #%d is RESERVED.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	if (srv->deferred_requests) {
+	    debug(34, 3) ("helperStatefulShutdown: %s #%d has DEFERRED requests.\n",
+		hlp->id_name, srv->index + 1);
+	    continue;
+	}
+	srv->flags.closing = 1;
+	/* the rest of the details is dealt with in the helperStatefulServerFree
+	 * close handler
+	 */
+	comm_close(srv->rfd);
+    }
+}
+
+
+helper *
+helperCreate(const char *name)
+{
+    helper *hlp;
+    hlp = cbdataAlloc(helper);
+    hlp->id_name = name;
+    return hlp;
+}
+
+statefulhelper *
+helperStatefulCreate(const char *name)
+{
+    statefulhelper *hlp;
+    hlp = cbdataAlloc(statefulhelper);
+    hlp->id_name = name;
+    return hlp;
+}
+
+
+void
+helperFree(helper * hlp)
+{
+    if (!hlp)
+	return;
+    /* note, don't free hlp->name, it probably points to static memory */
+    if (hlp->queue.head)
+	debug(84, 0) ("WARNING: freeing %s helper with %d requests queued\n",
+	    hlp->id_name, hlp->stats.queue_size);
+    cbdataFree(hlp);
+}
+
+void
+helperStatefulFree(statefulhelper * hlp)
+{
+    if (!hlp)
+	return;
+    /* note, don't free hlp->name, it probably points to static memory */
+    if (hlp->queue.head)
+	debug(84, 0) ("WARNING: freeing %s helper with %d requests queued\n",
+	    hlp->id_name, hlp->stats.queue_size);
+    cbdataFree(hlp);
+}
+
+
+/* ====================================================================== */
+/* LOCAL FUNCTIONS */
+/* ====================================================================== */
+
+static void
+helperServerFree(int fd, void *data)
+{
+    helper_server *srv = (helper_server *)data;
+    helper *hlp = srv->parent;
+    helper_request *r;
+    assert(srv->rfd == fd);
+    if (srv->buf) {
+	memFree(srv->buf, MEM_8K_BUF);
+	srv->buf = NULL;
+    }
+    if ((r = srv->request)) {
+	void *cbdata;
+	if (cbdataReferenceValidDone(r->data, &cbdata))
+	    r->callback(cbdata, srv->buf);
+	helperRequestFree(r);
+	srv->request = NULL;
+    }
+    if (srv->wfd != srv->rfd && srv->wfd != -1)
+	comm_close(srv->wfd);
+    dlinkDelete(&srv->link, &hlp->servers);
+    hlp->n_running--;
+    assert(hlp->n_running >= 0);
+    if (!srv->flags.shutdown) {
+	debug(34, 0) ("WARNING: %s #%d (FD %d) exited\n",
+	    hlp->id_name, srv->index + 1, fd);
+	if (hlp->n_running < hlp->n_to_start / 2)
+	    fatalf("Too few %s processes are running", hlp->id_name);
+    }
+    cbdataReferenceDone(srv->parent);
+    cbdataFree(srv);
+}
+
+static void
+helperStatefulServerFree(int fd, void *data)
+{
+    helper_stateful_server *srv = (helper_stateful_server *)data;
+    statefulhelper *hlp = srv->parent;
+    helper_stateful_request *r;
+    assert(srv->rfd == fd);
+    if (srv->buf) {
+	memFree(srv->buf, MEM_8K_BUF);
+	srv->buf = NULL;
+    }
+    if ((r = srv->request)) {
+	void *cbdata;
+	if (cbdataReferenceValidDone(r->data, &cbdata))
+	    r->callback(cbdata, srv, srv->buf);
+	helperStatefulRequestFree(r);
+	srv->request = NULL;
+    }
+    /* TODO: walk the local queue of requests and carry them all out */
+    if (srv->wfd != srv->rfd && srv->wfd != -1)
+	comm_close(srv->wfd);
+    dlinkDelete(&srv->link, &hlp->servers);
+    hlp->n_running--;
+    assert(hlp->n_running >= 0);
+    if (!srv->flags.shutdown) {
+	debug(34, 0) ("WARNING: %s #%d (FD %d) exited\n",
+	    hlp->id_name, srv->index + 1, fd);
+	if (hlp->n_running < hlp->n_to_start / 2)
+	    fatalf("Too few %s processes are running", hlp->id_name);
+    }
+    if (srv->data != NULL)
+	memPoolFree(hlp->datapool, srv->data);
+    cbdataReferenceDone(srv->parent);
+    cbdataFree(srv);
+}
+
+
+static void
+helperHandleRead(int fd, void *data)
+{
+    int len;
+    char *t = NULL;
+    helper_server *srv = (helper_server *)data;
+    helper_request *r;
+    helper *hlp = srv->parent;
+    assert(fd == srv->rfd);
+    assert(cbdataReferenceValid(data));
+    statCounter.syscalls.sock.reads++;
+    len = FD_READ_METHOD(fd, srv->buf + srv->offset, srv->buf_sz - srv->offset);
+    fd_bytes(fd, len, FD_READ);
+    debug(84, 5) ("helperHandleRead: %d bytes from %s #%d.\n",
+	len, hlp->id_name, srv->index + 1);
+    if (len <= 0) {
+	if (len < 0)
+	    debug(50, 1) ("helperHandleRead: FD %d read: %s\n", fd, xstrerror());
+	comm_close(fd);
+	return;
+    }
+    srv->offset += len;
+    srv->buf[srv->offset] = '\0';
+    r = srv->request;
+    if (r == NULL) {
+	/* someone spoke without being spoken to */
+	debug(84, 1) ("helperHandleRead: unexpected read from %s #%d, %d bytes\n",
+	    hlp->id_name, srv->index + 1, len);
+	srv->offset = 0;
+    } else if ((t = strchr(srv->buf, '\n'))) {
+	/* end of reply found */
+	HLPCB *callback;
+	void *cbdata;
+	debug(84, 3) ("helperHandleRead: end of reply found\n");
+	*t = '\0';
+	callback = r->callback;
+	r->callback = NULL;
+	if (cbdataReferenceValidDone(r->data, &cbdata))
+	    callback(cbdata, srv->buf);
+	srv->flags.busy = 0;
+	srv->offset = 0;
+	helperRequestFree(r);
+	srv->request = NULL;
+	hlp->stats.replies++;
+	srv->answer_time = current_time;
+	hlp->stats.avg_svc_time =
+	    intAverage(hlp->stats.avg_svc_time,
+	    tvSubUsec(srv->dispatch_time, current_time),
+	    hlp->stats.replies, REDIRECT_AV_FACTOR);
+	if (srv->flags.shutdown) {
+	    int wfd = srv->wfd;
+	    srv->wfd = -1;
+	    comm_close(wfd);
+	} else
+	    helperKickQueue(hlp);
+    } else {
+	commSetSelect(srv->rfd, COMM_SELECT_READ, helperHandleRead, srv, 0);
+    }
+}
+
+static void
+helperStatefulHandleRead(int fd, void *data)
+{
+    int len;
+    char *t = NULL;
+    helper_stateful_server *srv = (helper_stateful_server *)data;
+    helper_stateful_request *r;
+    statefulhelper *hlp = srv->parent;
+    assert(fd == srv->rfd);
+    assert(cbdataReferenceValid(data));
+    statCounter.syscalls.sock.reads++;
+    len = FD_READ_METHOD(fd, srv->buf + srv->offset, srv->buf_sz - srv->offset);
+    fd_bytes(fd, len, FD_READ);
+    debug(84, 5) ("helperStatefulHandleRead: %d bytes from %s #%d.\n",
+	len, hlp->id_name, srv->index + 1);
+    if (len <= 0) {
+	if (len < 0)
+	    debug(50, 1) ("helperStatefulHandleRead: FD %d read: %s\n", fd, xstrerror());
+	comm_close(fd);
+	return;
+    }
+    srv->offset += len;
+    srv->buf[srv->offset] = '\0';
+    r = srv->request;
+    if (r == NULL) {
+	/* someone spoke without being spoken to */
+	debug(84, 1) ("helperStatefulHandleRead: unexpected read from %s #%d, %d bytes\n",
+	    hlp->id_name, srv->index + 1, len);
+	srv->offset = 0;
+    } else if ((t = strchr(srv->buf, '\n'))) {
+	/* end of reply found */
+	debug(84, 3) ("helperStatefulHandleRead: end of reply found\n");
+	*t = '\0';
+	if (cbdataReferenceValid(r->data)) {
+	    switch ((r->callback(r->data, srv, srv->buf))) {	/*if non-zero reserve helper */
+	    case S_HELPER_UNKNOWN:
+		fatal("helperStatefulHandleRead: either a non-state aware callback was give to the stateful helper routines, or an uninitialised callback response was recieved.\n");
+		break;
+	    case S_HELPER_RELEASE:	/* helper finished with */
+		if (!srv->deferred_requests && !srv->queue.head) {
+		    srv->flags.reserved = S_HELPER_FREE;
+		    if ((srv->parent->OnEmptyQueue != NULL) && (srv->data))
+			srv->parent->OnEmptyQueue(srv->data);
+		    debug(84, 5) ("StatefulHandleRead: releasing %s #%d\n", hlp->id_name, srv->index + 1);
+		} else {
+		    srv->flags.reserved = S_HELPER_DEFERRED;
+		    debug(84, 5) ("StatefulHandleRead: outstanding deferred requests on %s #%d. reserving for deferred requests.\n", hlp->id_name, srv->index + 1);
+		}
+		break;
+	    case S_HELPER_RESERVE:	/* 'pin' this helper for the caller */
+		if (!srv->queue.head) {
+		    assert(srv->deferred_requests == 0);
+		    srv->flags.reserved = S_HELPER_RESERVED;
+		    debug(84, 5) ("StatefulHandleRead: reserving %s #%d\n", hlp->id_name, srv->index + 1);
+		} else {
+		    fatal("StatefulHandleRead: Callback routine attempted to reserve a stateful helper with deferred requests. This can lead to deadlock.\n");
+		}
+		break;
+	    case S_HELPER_DEFER:
+		/* the helper is still needed, but can
+		 * be used for other requests in the meantime.
+		 */
+		srv->flags.reserved = S_HELPER_DEFERRED;
+		srv->deferred_requests++;
+		srv->stats.deferbycb++;
+		debug(84, 5) ("StatefulHandleRead: reserving %s #%d for deferred requests.\n", hlp->id_name, srv->index + 1);
+		break;
+	    default:
+		fatal("helperStatefulHandleRead: unknown stateful helper callback result.\n");
+	    }
+
+	} else {
+	    debug(84, 1) ("StatefulHandleRead: no callback data registered\n");
+	}
+	srv->flags.busy = 0;
+	srv->offset = 0;
+	helperStatefulRequestFree(r);
+	srv->request = NULL;
+	hlp->stats.replies++;
+	hlp->stats.avg_svc_time =
+	    intAverage(hlp->stats.avg_svc_time,
+	    tvSubMsec(srv->dispatch_time, current_time),
+	    hlp->stats.replies, REDIRECT_AV_FACTOR);
+	if (srv->flags.shutdown
+	    && srv->flags.reserved == S_HELPER_FREE
+	    && !srv->deferred_requests) {
+	    int wfd = srv->wfd;
+	    srv->wfd = -1;
+	    comm_close(wfd);
+	} else {
+	    if (srv->queue.head)
+		helperStatefulServerKickQueue(srv);
+	    else
+		helperStatefulKickQueue(hlp);
+	}
+    } else {
+	commSetSelect(srv->rfd, COMM_SELECT_READ, helperStatefulHandleRead, srv, 0);
+    }
+}
+
+static void
+Enqueue(helper * hlp, helper_request * r)
+{
+    dlink_node *link = (dlink_node *)memAllocate(MEM_DLINK_NODE);
+    dlinkAddTail(r, link, &hlp->queue);
+    hlp->stats.queue_size++;
+    if (hlp->stats.queue_size < hlp->n_running)
+	return;
+    if (squid_curtime - hlp->last_queue_warn < 600)
+	return;
+    if (shutting_down || reconfiguring)
+	return;
+    hlp->last_queue_warn = squid_curtime;
+    debug(84, 0) ("WARNING: All %s processes are busy.\n", hlp->id_name);
+    debug(84, 0) ("WARNING: %d pending requests queued\n", hlp->stats.queue_size);
+    if (hlp->stats.queue_size > hlp->n_running * 2)
+	fatalf("Too many queued %s requests", hlp->id_name);
+    debug(84, 1) ("Consider increasing the number of %s processes in your config file.\n", hlp->id_name);
+}
+
+static void
+StatefulEnqueue(statefulhelper * hlp, helper_stateful_request * r)
+{
+    dlink_node *link = (dlink_node *)memAllocate(MEM_DLINK_NODE);
+    dlinkAddTail(r, link, &hlp->queue);
+    hlp->stats.queue_size++;
+    if (hlp->stats.queue_size < hlp->n_running)
+	return;
+    if (hlp->stats.queue_size > hlp->n_running * 2)
+	fatalf("Too many queued %s requests", hlp->id_name);
+    if (squid_curtime - hlp->last_queue_warn < 600)
+	return;
+    if (shutting_down || reconfiguring)
+	return;
+    hlp->last_queue_warn = squid_curtime;
+    debug(84, 0) ("WARNING: All %s processes are busy.\n", hlp->id_name);
+    debug(84, 0) ("WARNING: %d pending requests queued\n", hlp->stats.queue_size);
+    debug(84, 1) ("Consider increasing the number of %s processes in your config file.\n", hlp->id_name);
+}
+
+static void
+StatefulServerEnqueue(helper_stateful_server * srv, helper_stateful_request * r)
+{
+    dlink_node *link = (dlink_node *)memAllocate(MEM_DLINK_NODE);
+    dlinkAddTail(r, link, &srv->queue);
+/* TODO: warning if the queue on this server is more than X
+ * We don't check the queue size at the moment, because
+ * requests hitting here are deferrable 
+ */
+/*    hlp->stats.queue_size++;
+ * if (hlp->stats.queue_size < hlp->n_running)
+ * return;
+ * if (squid_curtime - hlp->last_queue_warn < 600)
+ * return;
+ * if (shutting_down || reconfiguring)
+ * return;
+ * hlp->last_queue_warn = squid_curtime;
+ * debug(84, 0) ("WARNING: All %s processes are busy.\n", hlp->id_name);
+ * debug(84, 0) ("WARNING: %d pending requests queued\n", hlp->stats.queue_size);
+ * if (hlp->stats.queue_size > hlp->n_running * 2)
+ * fatalf("Too many queued %s requests", hlp->id_name);
+ * debug(84, 1) ("Consider increasing the number of %s processes in your config file.\n", hlp->id_name);  */
+}
+
+
+static helper_request *
+Dequeue(helper * hlp)
+{
+    dlink_node *link;
+    helper_request *r = NULL;
+    if ((link = hlp->queue.head)) {
+	r = (helper_request *)link->data;
+	dlinkDelete(link, &hlp->queue);
+	memFree(link, MEM_DLINK_NODE);
+	hlp->stats.queue_size--;
+    }
+    return r;
+}
+
+static helper_stateful_request *
+StatefulServerDequeue(helper_stateful_server * srv)
+{
+    dlink_node *link;
+    helper_stateful_request *r = NULL;
+    if ((link = srv->queue.head)) {
+	r = (helper_stateful_request *)link->data;
+	dlinkDelete(link, &srv->queue);
+	memFree(link, MEM_DLINK_NODE);
+    }
+    return r;
+}
+
+static helper_stateful_request *
+StatefulDequeue(statefulhelper * hlp)
+{
+    dlink_node *link;
+    helper_stateful_request *r = NULL;
+    if ((link = hlp->queue.head)) {
+	r = (helper_stateful_request *)link->data;
+	dlinkDelete(link, &hlp->queue);
+	memFree(link, MEM_DLINK_NODE);
+	hlp->stats.queue_size--;
+    }
+    return r;
+}
+
+static helper_server *
+GetFirstAvailable(helper * hlp)
+{
+    dlink_node *n;
+    helper_server *srv = NULL;
+    if (hlp->n_running == 0)
+	return NULL;
+    for (n = hlp->servers.head; n != NULL; n = n->next) {
+	srv = (helper_server *)n->data;
+	if (srv->flags.busy)
+	    continue;
+	if (!srv->flags.alive)
+	    continue;
+	return srv;
+    }
+    return NULL;
+}
+
+static helper_stateful_server *
+StatefulGetFirstAvailable(statefulhelper * hlp)
+{
+    dlink_node *n;
+    helper_stateful_server *srv = NULL;
+    debug(84, 5) ("StatefulGetFirstAvailable: Running servers %d.\n", hlp->n_running);
+    if (hlp->n_running == 0)
+	return NULL;
+    for (n = hlp->servers.head; n != NULL; n = n->next) {
+	srv = (helper_stateful_server *)n->data;
+	if (srv->flags.busy)
+	    continue;
+	if (srv->flags.reserved == S_HELPER_RESERVED)
+	    continue;
+	if (!srv->flags.alive)
+	    continue;
+	if ((hlp->IsAvailable != NULL) && (srv->data != NULL) && !(hlp->IsAvailable(srv->data)))
+	    continue;
+	return srv;
+    }
+    debug(84, 5) ("StatefulGetFirstAvailable: None available.\n");
+    return NULL;
+}
+
+
+static void
+helperDispatch(helper_server * srv, helper_request * r)
+{
+    helper *hlp = srv->parent;
+    if (!cbdataReferenceValid(r->data)) {
+	debug(84, 1) ("helperDispatch: invalid callback data\n");
+	helperRequestFree(r);
+	return;
+    }
+    assert(!srv->flags.busy);
+    srv->flags.busy = 1;
+    srv->request = r;
+    srv->dispatch_time = current_time;
+    comm_write(srv->wfd,
+	r->buf,
+	strlen(r->buf),
+	NULL,			/* Handler */
+	NULL,			/* Handler-data */
+	NULL);			/* free */
+    commSetSelect(srv->rfd,
+	COMM_SELECT_READ,
+	helperHandleRead,
+	srv, 0);
+    debug(84, 5) ("helperDispatch: Request sent to %s #%d, %d bytes\n",
+	hlp->id_name, srv->index + 1, (int) strlen(r->buf));
+    srv->stats.uses++;
+    hlp->stats.requests++;
+}
+
+static void
+helperStatefulDispatch(helper_stateful_server * srv, helper_stateful_request * r)
+{
+    statefulhelper *hlp = srv->parent;
+    if (!cbdataReferenceValid(r->data)) {
+	debug(84, 1) ("helperStatefulDispatch: invalid callback data\n");
+	helperStatefulRequestFree(r);
+	return;
+    }
+    debug(84, 9) ("helperStatefulDispatch busying helper %s #%d\n", hlp->id_name, srv->index + 1);
+    if (r->placeholder == 1) {
+	/* a callback is needed before this request can _use_ a helper. */
+	/* we don't care about releasing/deferring this helper. The request NEVER
+	 * gets to the helper. So we throw away the return code */
+	r->callback(r->data, srv, NULL);
+	/* throw away the placeholder */
+	helperStatefulRequestFree(r);
+	/* and push the queue. Note that the callback may have submitted a new 
+	 * request to the helper which is why we test for the request*/
+	if (srv->request == NULL) {
+	    if (srv->flags.shutdown
+		&& srv->flags.reserved == S_HELPER_FREE
+		&& !srv->deferred_requests) {
+		int wfd = srv->wfd;
+		srv->wfd = -1;
+		comm_close(wfd);
+	    } else {
+		if (srv->queue.head)
+		    helperStatefulServerKickQueue(srv);
+		else
+		    helperStatefulKickQueue(hlp);
+	    }
+	}
+	return;
+    }
+    srv->flags.busy = 1;
+    srv->request = r;
+    srv->dispatch_time = current_time;
+    comm_write(srv->wfd,
+	r->buf,
+	strlen(r->buf),
+	NULL,			/* Handler */
+	NULL,			/* Handler-data */
+	NULL);			/* free */
+    commSetSelect(srv->rfd,
+	COMM_SELECT_READ,
+	helperStatefulHandleRead,
+	srv, 0);
+    debug(84, 5) ("helperStatefulDispatch: Request sent to %s #%d, %d bytes\n",
+	hlp->id_name, srv->index + 1, (int) strlen(r->buf));
+    srv->stats.uses++;
+    hlp->stats.requests++;
+}
+
+
+static void
+helperKickQueue(helper * hlp)
+{
+    helper_request *r;
+    helper_server *srv;
+    while ((srv = GetFirstAvailable(hlp)) && (r = Dequeue(hlp)))
+	helperDispatch(srv, r);
+}
+
+static void
+helperStatefulKickQueue(statefulhelper * hlp)
+{
+    helper_stateful_request *r;
+    helper_stateful_server *srv;
+    while ((srv = StatefulGetFirstAvailable(hlp)) && (r = StatefulDequeue(hlp)))
+	helperStatefulDispatch(srv, r);
+}
+
+static void
+helperStatefulServerKickQueue(helper_stateful_server * srv)
+{
+    helper_stateful_request *r;
+    if ((r = StatefulServerDequeue(srv)))
+	helperStatefulDispatch(srv, r);
+}
+
+static void
+helperRequestFree(helper_request * r)
+{
+    cbdataReferenceDone(r->data);
+    xfree(r->buf);
+    memFree(r, MEM_HELPER_REQUEST);
+}
+
+static void
+helperStatefulRequestFree(helper_stateful_request * r)
+{
+    cbdataReferenceDone(r->data);
+    xfree(r->buf);
+    memFree(r, MEM_HELPER_STATEFUL_REQUEST);
+}
Index: squid/src/icp_v2.cc
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Attic/icp_v2.cc,v
retrieving revision 1.1.2.17
retrieving revision 1.1.2.18
diff -u -r1.1.2.17 -r1.1.2.18
--- squid/src/icp_v2.cc	9 Oct 2002 05:55:49 -0000	1.1.2.17
+++ squid/src/icp_v2.cc	11 Oct 2002 15:40:51 -0000	1.1.2.18
@@ -1,6 +1,6 @@
 
 /*
- * $Id: icp_v2.cc,v 1.1.2.17 2002/10/09 05:55:49 rbcollins Exp $
+ * $Id: icp_v2.cc,v 1.1.2.18 2002/10/11 15:40:51 rbcollins Exp $
  *
  * DEBUG: section 12    Internet Cache Protocol
  * AUTHOR: Duane Wessels
@@ -69,6 +69,14 @@
     pad = ntohl(pad);
 }
 
+icp_opcode
+_icp_common_t::getOpCode() const
+{
+    if (opcode > (char)ICP_END)
+	return ICP_INVALID;
+    return (icp_opcode)opcode;
+}
+
 /* ICPState */
 
 ICPState:: ICPState(icp_common_t & aHeader):header(aHeader)
--- squid/src/internal.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,153 +0,0 @@
-
-/*
- * $Id: internal.c,v 1.9.18.1 2002/10/04 07:07:26 rbcollins Exp $
- *
- * DEBUG: section 76    Internal Squid Object handling
- * AUTHOR: Duane, Alex, Henrik
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-#include "Store.h"
-
-/* called when we "miss" on an internal object;
- * generate known dynamic objects, 
- * return HTTP_NOT_FOUND for others
- */
-void
-internalStart(request_t * request, StoreEntry * entry)
-{
-    ErrorState *err;
-    const char *upath = strBuf(request->urlpath);
-    http_version_t version;
-    debug(76, 3) ("internalStart: %s requesting '%s'\n",
-	inet_ntoa(request->client_addr), upath);
-    if (0 == strcmp(upath, "/squid-internal-dynamic/netdb")) {
-	netdbBinaryExchange(entry);
-    } else if (0 == strcmp(upath, "/squid-internal-periodic/store_digest")) {
-#if USE_CACHE_DIGESTS
-	const char *msgbuf = "This cache is currently building its digest.\n";
-#else
-	const char *msgbuf = "This cache does not suport Cache Digests.\n";
-#endif
-	httpBuildVersion(&version, 1, 0);
-	httpReplySetHeaders(entry->mem_obj->reply,
-	    version,
-	    HTTP_NOT_FOUND,
-	    "Not Found",
-	    "text/plain",
-	    strlen(msgbuf),
-	    squid_curtime,
-	    -2);
-	httpReplySwapOut(entry->mem_obj->reply, entry);
-	storeAppend(entry, msgbuf, strlen(msgbuf));
-	storeComplete(entry);
-    } else {
-	debugObj(76, 1, "internalStart: unknown request:\n",
-	    request, (ObjPackMethod) & httpRequestPack);
-	err = errorCon(ERR_INVALID_REQ, HTTP_NOT_FOUND);
-	err->request = requestLink(request);
-	errorAppendEntry(entry, err);
-    }
-}
-
-int
-internalCheck(const char *urlpath)
-{
-    return (0 == strncmp(urlpath, "/squid-internal-", 16));
-}
-
-int
-internalStaticCheck(const char *urlpath)
-{
-    return (0 == strncmp(urlpath, "/squid-internal-static", 22));
-}
-
-/*
- * makes internal url with a given host and port (remote internal url)
- */
-char *
-internalRemoteUri(const char *host, u_short port, const char *dir, const char *name)
-{
-    static MemBuf mb = MemBufNULL;
-    static char lc_host[SQUIDHOSTNAMELEN];
-    assert(host && name);
-    /* convert host name to lower case */
-    xstrncpy(lc_host, host, SQUIDHOSTNAMELEN - 1);
-    Tolower(lc_host);
-    /*
-     * append the domain in order to mirror the requests with appended
-     * domains
-     */
-    if (Config.appendDomain && !strchr(lc_host, '.'))
-	strncat(lc_host, Config.appendDomain, SQUIDHOSTNAMELEN -
-	    strlen(lc_host) - 1);
-    /* build uri in mb */
-    memBufReset(&mb);
-    memBufPrintf(&mb, "http://%s", lc_host);
-    /* append port if not default */
-    if (port && port != urlDefaultPort(PROTO_HTTP))
-	memBufPrintf(&mb, ":%d", port);
-    if (dir)
-	memBufPrintf(&mb, "%s", dir);
-    memBufPrintf(&mb, "%s", name);
-    /* return a pointer to a local static buffer */
-    return mb.buf;
-}
-
-/*
- * makes internal url with local host and port
- */
-char *
-internalLocalUri(const char *dir, const char *name)
-{
-    return internalRemoteUri(getMyHostname(),
-	getMyPort(), dir, name);
-}
-
-const char *
-internalHostname(void)
-{
-    LOCAL_ARRAY(char, host, SQUIDHOSTNAMELEN + 1);
-    xstrncpy(host, getMyHostname(), SQUIDHOSTNAMELEN);
-    Tolower(host);
-    return host;
-}
-
-int
-internalHostnameIs(const char *arg)
-{
-    wordlist *w;
-    if (0 == strcmp(arg, internalHostname()))
-	return 1;
-    for (w = Config.hostnameAliases; w; w = w->next)
-	if (0 == strcmp(arg, w->key))
-	    return 1;
-    return 0;
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/internal.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,153 @@
+
+/*
+ * $Id: internal.cc,v 1.1.2.1 2002/10/11 15:40:53 rbcollins Exp $
+ *
+ * DEBUG: section 76    Internal Squid Object handling
+ * AUTHOR: Duane, Alex, Henrik
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "Store.h"
+
+/* called when we "miss" on an internal object;
+ * generate known dynamic objects, 
+ * return HTTP_NOT_FOUND for others
+ */
+void
+internalStart(request_t * request, StoreEntry * entry)
+{
+    ErrorState *err;
+    const char *upath = strBuf(request->urlpath);
+    http_version_t version;
+    debug(76, 3) ("internalStart: %s requesting '%s'\n",
+	inet_ntoa(request->client_addr), upath);
+    if (0 == strcmp(upath, "/squid-internal-dynamic/netdb")) {
+	netdbBinaryExchange(entry);
+    } else if (0 == strcmp(upath, "/squid-internal-periodic/store_digest")) {
+#if USE_CACHE_DIGESTS
+	const char *msgbuf = "This cache is currently building its digest.\n";
+#else
+	const char *msgbuf = "This cache does not suport Cache Digests.\n";
+#endif
+	httpBuildVersion(&version, 1, 0);
+	httpReplySetHeaders(entry->mem_obj->reply,
+	    version,
+	    HTTP_NOT_FOUND,
+	    "Not Found",
+	    "text/plain",
+	    strlen(msgbuf),
+	    squid_curtime,
+	    -2);
+	httpReplySwapOut(entry->mem_obj->reply, entry);
+	storeAppend(entry, msgbuf, strlen(msgbuf));
+	storeComplete(entry);
+    } else {
+	debugObj(76, 1, "internalStart: unknown request:\n",
+	    request, (ObjPackMethod) & httpRequestPack);
+	err = errorCon(ERR_INVALID_REQ, HTTP_NOT_FOUND);
+	err->request = requestLink(request);
+	errorAppendEntry(entry, err);
+    }
+}
+
+int
+internalCheck(const char *urlpath)
+{
+    return (0 == strncmp(urlpath, "/squid-internal-", 16));
+}
+
+int
+internalStaticCheck(const char *urlpath)
+{
+    return (0 == strncmp(urlpath, "/squid-internal-static", 22));
+}
+
+/*
+ * makes internal url with a given host and port (remote internal url)
+ */
+char *
+internalRemoteUri(const char *host, u_short port, const char *dir, const char *name)
+{
+    static MemBuf mb = MemBufNULL;
+    static char lc_host[SQUIDHOSTNAMELEN];
+    assert(host && name);
+    /* convert host name to lower case */
+    xstrncpy(lc_host, host, SQUIDHOSTNAMELEN - 1);
+    Tolower(lc_host);
+    /*
+     * append the domain in order to mirror the requests with appended
+     * domains
+     */
+    if (Config.appendDomain && !strchr(lc_host, '.'))
+	strncat(lc_host, Config.appendDomain, SQUIDHOSTNAMELEN -
+	    strlen(lc_host) - 1);
+    /* build uri in mb */
+    memBufReset(&mb);
+    memBufPrintf(&mb, "http://%s", lc_host);
+    /* append port if not default */
+    if (port && port != urlDefaultPort(PROTO_HTTP))
+	memBufPrintf(&mb, ":%d", port);
+    if (dir)
+	memBufPrintf(&mb, "%s", dir);
+    memBufPrintf(&mb, "%s", name);
+    /* return a pointer to a local static buffer */
+    return mb.buf;
+}
+
+/*
+ * makes internal url with local host and port
+ */
+char *
+internalLocalUri(const char *dir, const char *name)
+{
+    return internalRemoteUri(getMyHostname(),
+	getMyPort(), dir, name);
+}
+
+const char *
+internalHostname(void)
+{
+    LOCAL_ARRAY(char, host, SQUIDHOSTNAMELEN + 1);
+    xstrncpy(host, getMyHostname(), SQUIDHOSTNAMELEN);
+    Tolower(host);
+    return host;
+}
+
+int
+internalHostnameIs(const char *arg)
+{
+    wordlist *w;
+    if (0 == strcmp(arg, internalHostname()))
+	return 1;
+    for (w = Config.hostnameAliases; w; w = w->next)
+	if (0 == strcmp(arg, w->key))
+	    return 1;
+    return 0;
+}
--- squid/src/ipc.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,312 +0,0 @@
-
-/*
- * $Id: ipc.c,v 1.9 2002/04/06 11:34:50 squidadm Exp $
- *
- * DEBUG: section 54    Interprocess Communication
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-
-static const char *hello_string = "hi there\n";
-#define HELLO_BUF_SZ 32
-static char hello_buf[HELLO_BUF_SZ];
-
-static int
-ipcCloseAllFD(int prfd, int pwfd, int crfd, int cwfd)
-{
-    if (prfd >= 0)
-	comm_close(prfd);
-    if (prfd != pwfd)
-	if (pwfd >= 0)
-	    comm_close(pwfd);
-    if (crfd >= 0)
-	comm_close(crfd);
-    if (crfd != cwfd)
-	if (cwfd >= 0)
-	    comm_close(cwfd);
-    return -1;
-}
-
-int
-ipcCreate(int type, const char *prog, const char *const args[], const char *name, int *rfd, int *wfd)
-{
-    pid_t pid;
-    struct sockaddr_in CS;
-    struct sockaddr_in PS;
-    int crfd = -1;
-    int prfd = -1;
-    int cwfd = -1;
-    int pwfd = -1;
-    int fd;
-    int t1, t2, t3;
-    socklen_t len;
-    int tmp_s;
-#if HAVE_PUTENV
-    char *env_str;
-#endif
-    int x;
-
-#if USE_POLL && defined(_SQUID_OSF_)
-    assert(type != IPC_FIFO);
-#endif
-
-    if (rfd)
-	*rfd = -1;
-    if (wfd)
-	*wfd = -1;
-    if (type == IPC_TCP_SOCKET) {
-	crfd = cwfd = comm_open(SOCK_STREAM,
-	    0,
-	    local_addr,
-	    0,
-	    COMM_NOCLOEXEC,
-	    name);
-	prfd = pwfd = comm_open(SOCK_STREAM,
-	    0,			/* protocol */
-	    local_addr,
-	    0,			/* port */
-	    0,			/* blocking */
-	    name);
-    } else if (type == IPC_UDP_SOCKET) {
-	crfd = cwfd = comm_open(SOCK_DGRAM,
-	    0,
-	    local_addr,
-	    0,
-	    COMM_NOCLOEXEC,
-	    name);
-	prfd = pwfd = comm_open(SOCK_DGRAM,
-	    0,
-	    local_addr,
-	    0,
-	    0,
-	    name);
-    } else if (type == IPC_FIFO) {
-	int p2c[2];
-	int c2p[2];
-	if (pipe(p2c) < 0) {
-	    debug(50, 0) ("ipcCreate: pipe: %s\n", xstrerror());
-	    return -1;
-	}
-	if (pipe(c2p) < 0) {
-	    debug(50, 0) ("ipcCreate: pipe: %s\n", xstrerror());
-	    return -1;
-	}
-	fd_open(prfd = p2c[0], FD_PIPE, "IPC FIFO Parent Read");
-	fd_open(cwfd = p2c[1], FD_PIPE, "IPC FIFO Child Write");
-	fd_open(crfd = c2p[0], FD_PIPE, "IPC FIFO Child Read");
-	fd_open(pwfd = c2p[1], FD_PIPE, "IPC FIFO Parent Write");
-#if HAVE_SOCKETPAIR && defined(AF_UNIX)
-    } else if (type == IPC_UNIX_STREAM) {
-	int fds[2];
-	int buflen = 32768;
-	if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) < 0) {
-	    debug(50, 0) ("ipcCreate: socketpair: %s\n", xstrerror());
-	    return -1;
-	}
-	setsockopt(fds[0], SOL_SOCKET, SO_SNDBUF, (void *) &buflen, sizeof(buflen));
-	setsockopt(fds[0], SOL_SOCKET, SO_RCVBUF, (void *) &buflen, sizeof(buflen));
-	setsockopt(fds[1], SOL_SOCKET, SO_SNDBUF, (void *) &buflen, sizeof(buflen));
-	setsockopt(fds[1], SOL_SOCKET, SO_RCVBUF, (void *) &buflen, sizeof(buflen));
-	fd_open(prfd = pwfd = fds[0], FD_PIPE, "IPC UNIX STREAM Parent");
-	fd_open(crfd = cwfd = fds[1], FD_PIPE, "IPC UNIX STREAM Parent");
-    } else if (type == IPC_UNIX_DGRAM) {
-	int fds[2];
-	if (socketpair(AF_UNIX, SOCK_DGRAM, 0, fds) < 0) {
-	    debug(50, 0) ("ipcCreate: socketpair: %s\n", xstrerror());
-	    return -1;
-	}
-	fd_open(prfd = pwfd = fds[0], FD_PIPE, "IPC UNIX DGRAM Parent");
-	fd_open(crfd = cwfd = fds[1], FD_PIPE, "IPC UNIX DGRAM Parent");
-#endif
-    } else {
-	assert(IPC_NONE);
-    }
-    debug(54, 3) ("ipcCreate: prfd FD %d\n", prfd);
-    debug(54, 3) ("ipcCreate: pwfd FD %d\n", pwfd);
-    debug(54, 3) ("ipcCreate: crfd FD %d\n", crfd);
-    debug(54, 3) ("ipcCreate: cwfd FD %d\n", cwfd);
-
-    if (crfd < 0) {
-	debug(54, 0) ("ipcCreate: Failed to create child FD.\n");
-	return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-    }
-    if (pwfd < 0) {
-	debug(54, 0) ("ipcCreate: Failed to create server FD.\n");
-	return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-    }
-    if (type == IPC_TCP_SOCKET || type == IPC_UDP_SOCKET) {
-	len = sizeof(PS);
-	memset(&PS, '\0', len);
-	if (getsockname(pwfd, (struct sockaddr *) &PS, &len) < 0) {
-	    debug(50, 0) ("ipcCreate: getsockname: %s\n", xstrerror());
-	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-	}
-	debug(54, 3) ("ipcCreate: FD %d sockaddr %s:%d\n",
-	    pwfd, inet_ntoa(PS.sin_addr), ntohs(PS.sin_port));
-	len = sizeof(CS);
-	memset(&CS, '\0', len);
-	if (getsockname(crfd, (struct sockaddr *) &CS, &len) < 0) {
-	    debug(50, 0) ("ipcCreate: getsockname: %s\n", xstrerror());
-	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-	}
-	debug(54, 3) ("ipcCreate: FD %d sockaddr %s:%d\n",
-	    crfd, inet_ntoa(CS.sin_addr), ntohs(CS.sin_port));
-    }
-    if (type == IPC_TCP_SOCKET) {
-	if (listen(crfd, 1) < 0) {
-	    debug(50, 1) ("ipcCreate: listen FD %d: %s\n", crfd, xstrerror());
-	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-	}
-	debug(54, 3) ("ipcCreate: FD %d listening...\n", crfd);
-    }
-    /* flush or else we get dup data if unbuffered_logs is set */
-    logsFlush();
-    if ((pid = fork()) < 0) {
-	debug(50, 1) ("ipcCreate: fork: %s\n", xstrerror());
-	return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-    }
-    if (pid > 0) {		/* parent */
-	/* close shared socket with child */
-	comm_close(crfd);
-	if (cwfd != crfd)
-	    comm_close(cwfd);
-	cwfd = crfd = -1;
-	if (type == IPC_TCP_SOCKET || type == IPC_UDP_SOCKET) {
-	    if (comm_connect_addr(pwfd, &CS) == COMM_ERROR)
-		return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-	}
-	memset(hello_buf, '\0', HELLO_BUF_SZ);
-	if (type == IPC_UDP_SOCKET)
-	    x = recv(prfd, hello_buf, HELLO_BUF_SZ - 1, 0);
-	else
-	    x = read(prfd, hello_buf, HELLO_BUF_SZ - 1);
-	if (x < 0) {
-	    debug(50, 0) ("ipcCreate: PARENT: hello read test failed\n");
-	    debug(50, 0) ("--> read: %s\n", xstrerror());
-	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-	} else if (strcmp(hello_buf, hello_string)) {
-	    debug(54, 0) ("ipcCreate: PARENT: hello read test failed\n");
-	    debug(54, 0) ("--> read returned %d\n", x);
-	    debug(54, 0) ("--> got '%s'\n", rfc1738_escape(hello_buf));
-	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-	}
-	commSetTimeout(prfd, -1, NULL, NULL);
-	commSetNonBlocking(prfd);
-	commSetNonBlocking(pwfd);
-	if (rfd)
-	    *rfd = prfd;
-	if (wfd)
-	    *wfd = pwfd;
-	fd_table[prfd].flags.ipc = 1;
-	fd_table[pwfd].flags.ipc = 1;
-	if (Config.sleep_after_fork) {
-	    /* XXX emulation of usleep() */
-	    struct timeval sl;
-	    sl.tv_sec = Config.sleep_after_fork / 1000000;
-	    sl.tv_usec = Config.sleep_after_fork % 1000000;
-	    select(0, NULL, NULL, NULL, &sl);
-	}
-	return pid;
-    }
-    /* child */
-    no_suid();			/* give up extra priviliges */
-    /* close shared socket with parent */
-    close(prfd);
-    if (pwfd != prfd)
-	close(pwfd);
-    pwfd = prfd = -1;
-
-    if (type == IPC_TCP_SOCKET) {
-	debug(54, 3) ("ipcCreate: calling accept on FD %d\n", crfd);
-	if ((fd = accept(crfd, NULL, NULL)) < 0) {
-	    debug(50, 0) ("ipcCreate: FD %d accept: %s\n", crfd, xstrerror());
-	    _exit(1);
-	}
-	debug(54, 3) ("ipcCreate: CHILD accepted new FD %d\n", fd);
-	close(crfd);
-	cwfd = crfd = fd;
-    } else if (type == IPC_UDP_SOCKET) {
-	if (comm_connect_addr(crfd, &PS) == COMM_ERROR)
-	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
-    }
-    if (type == IPC_UDP_SOCKET) {
-	x = send(cwfd, hello_string, strlen(hello_string) + 1, 0);
-	if (x < 0) {
-	    debug(50, 0) ("sendto FD %d: %s\n", cwfd, xstrerror());
-	    debug(50, 0) ("ipcCreate: CHILD: hello write test failed\n");
-	    _exit(1);
-	}
-    } else {
-	if (write(cwfd, hello_string, strlen(hello_string) + 1) < 0) {
-	    debug(50, 0) ("write FD %d: %s\n", cwfd, xstrerror());
-	    debug(50, 0) ("ipcCreate: CHILD: hello write test failed\n");
-	    _exit(1);
-	}
-    }
-#if HAVE_PUTENV
-    env_str = xcalloc((tmp_s = strlen(Config.debugOptions) + 32), 1);
-    snprintf(env_str, tmp_s, "SQUID_DEBUG=%s", Config.debugOptions);
-    putenv(env_str);
-#endif
-    /*
-     * This double-dup stuff avoids problems when one of 
-     *  crfd, cwfd, or debug_log are in the rage 0-2.
-     */
-    do {
-	x = open(_PATH_DEVNULL, 0, 0444);
-	if (x > -1)
-	    commSetCloseOnExec(x);
-    } while (x < 3);
-    t1 = dup(crfd);
-    t2 = dup(cwfd);
-    t3 = dup(fileno(debug_log));
-    assert(t1 > 2 && t2 > 2 && t3 > 2);
-    close(crfd);
-    close(cwfd);
-    close(fileno(debug_log));
-    dup2(t1, 0);
-    dup2(t2, 1);
-    dup2(t3, 2);
-    close(t1);
-    close(t2);
-    close(t3);
-    /* Make sure all other filedescriptors are closed */
-    for (x = 3; x < SQUID_MAXFD; x++)
-	close(x);
-#if HAVE_SETSID
-    setsid();
-#endif
-    execvp(prog, (char *const *) args);
-    debug_log = fdopen(2, "a+");
-    debug(50, 0) ("ipcCreate: %s: %s\n", prog, xstrerror());
-    _exit(1);
-    return 0;
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/ipc.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,312 @@
+
+/*
+ * $Id: ipc.cc,v 1.1.2.1 2002/10/11 15:40:54 rbcollins Exp $
+ *
+ * DEBUG: section 54    Interprocess Communication
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+static const char *hello_string = "hi there\n";
+#define HELLO_BUF_SZ 32
+static char hello_buf[HELLO_BUF_SZ];
+
+static int
+ipcCloseAllFD(int prfd, int pwfd, int crfd, int cwfd)
+{
+    if (prfd >= 0)
+	comm_close(prfd);
+    if (prfd != pwfd)
+	if (pwfd >= 0)
+	    comm_close(pwfd);
+    if (crfd >= 0)
+	comm_close(crfd);
+    if (crfd != cwfd)
+	if (cwfd >= 0)
+	    comm_close(cwfd);
+    return -1;
+}
+
+int
+ipcCreate(int type, const char *prog, const char *const args[], const char *name, int *rfd, int *wfd)
+{
+    pid_t pid;
+    struct sockaddr_in CS;
+    struct sockaddr_in PS;
+    int crfd = -1;
+    int prfd = -1;
+    int cwfd = -1;
+    int pwfd = -1;
+    int fd;
+    int t1, t2, t3;
+    socklen_t len;
+    int tmp_s;
+#if HAVE_PUTENV
+    char *env_str;
+#endif
+    int x;
+
+#if USE_POLL && defined(_SQUID_OSF_)
+    assert(type != IPC_FIFO);
+#endif
+
+    if (rfd)
+	*rfd = -1;
+    if (wfd)
+	*wfd = -1;
+    if (type == IPC_TCP_SOCKET) {
+	crfd = cwfd = comm_open(SOCK_STREAM,
+	    0,
+	    local_addr,
+	    0,
+	    COMM_NOCLOEXEC,
+	    name);
+	prfd = pwfd = comm_open(SOCK_STREAM,
+	    0,			/* protocol */
+	    local_addr,
+	    0,			/* port */
+	    0,			/* blocking */
+	    name);
+    } else if (type == IPC_UDP_SOCKET) {
+	crfd = cwfd = comm_open(SOCK_DGRAM,
+	    0,
+	    local_addr,
+	    0,
+	    COMM_NOCLOEXEC,
+	    name);
+	prfd = pwfd = comm_open(SOCK_DGRAM,
+	    0,
+	    local_addr,
+	    0,
+	    0,
+	    name);
+    } else if (type == IPC_FIFO) {
+	int p2c[2];
+	int c2p[2];
+	if (pipe(p2c) < 0) {
+	    debug(50, 0) ("ipcCreate: pipe: %s\n", xstrerror());
+	    return -1;
+	}
+	if (pipe(c2p) < 0) {
+	    debug(50, 0) ("ipcCreate: pipe: %s\n", xstrerror());
+	    return -1;
+	}
+	fd_open(prfd = p2c[0], FD_PIPE, "IPC FIFO Parent Read");
+	fd_open(cwfd = p2c[1], FD_PIPE, "IPC FIFO Child Write");
+	fd_open(crfd = c2p[0], FD_PIPE, "IPC FIFO Child Read");
+	fd_open(pwfd = c2p[1], FD_PIPE, "IPC FIFO Parent Write");
+#if HAVE_SOCKETPAIR && defined(AF_UNIX)
+    } else if (type == IPC_UNIX_STREAM) {
+	int fds[2];
+	int buflen = 32768;
+	if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) < 0) {
+	    debug(50, 0) ("ipcCreate: socketpair: %s\n", xstrerror());
+	    return -1;
+	}
+	setsockopt(fds[0], SOL_SOCKET, SO_SNDBUF, (void *) &buflen, sizeof(buflen));
+	setsockopt(fds[0], SOL_SOCKET, SO_RCVBUF, (void *) &buflen, sizeof(buflen));
+	setsockopt(fds[1], SOL_SOCKET, SO_SNDBUF, (void *) &buflen, sizeof(buflen));
+	setsockopt(fds[1], SOL_SOCKET, SO_RCVBUF, (void *) &buflen, sizeof(buflen));
+	fd_open(prfd = pwfd = fds[0], FD_PIPE, "IPC UNIX STREAM Parent");
+	fd_open(crfd = cwfd = fds[1], FD_PIPE, "IPC UNIX STREAM Parent");
+    } else if (type == IPC_UNIX_DGRAM) {
+	int fds[2];
+	if (socketpair(AF_UNIX, SOCK_DGRAM, 0, fds) < 0) {
+	    debug(50, 0) ("ipcCreate: socketpair: %s\n", xstrerror());
+	    return -1;
+	}
+	fd_open(prfd = pwfd = fds[0], FD_PIPE, "IPC UNIX DGRAM Parent");
+	fd_open(crfd = cwfd = fds[1], FD_PIPE, "IPC UNIX DGRAM Parent");
+#endif
+    } else {
+	assert(IPC_NONE);
+    }
+    debug(54, 3) ("ipcCreate: prfd FD %d\n", prfd);
+    debug(54, 3) ("ipcCreate: pwfd FD %d\n", pwfd);
+    debug(54, 3) ("ipcCreate: crfd FD %d\n", crfd);
+    debug(54, 3) ("ipcCreate: cwfd FD %d\n", cwfd);
+
+    if (crfd < 0) {
+	debug(54, 0) ("ipcCreate: Failed to create child FD.\n");
+	return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+    }
+    if (pwfd < 0) {
+	debug(54, 0) ("ipcCreate: Failed to create server FD.\n");
+	return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+    }
+    if (type == IPC_TCP_SOCKET || type == IPC_UDP_SOCKET) {
+	len = sizeof(PS);
+	memset(&PS, '\0', len);
+	if (getsockname(pwfd, (struct sockaddr *) &PS, &len) < 0) {
+	    debug(50, 0) ("ipcCreate: getsockname: %s\n", xstrerror());
+	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+	}
+	debug(54, 3) ("ipcCreate: FD %d sockaddr %s:%d\n",
+	    pwfd, inet_ntoa(PS.sin_addr), ntohs(PS.sin_port));
+	len = sizeof(CS);
+	memset(&CS, '\0', len);
+	if (getsockname(crfd, (struct sockaddr *) &CS, &len) < 0) {
+	    debug(50, 0) ("ipcCreate: getsockname: %s\n", xstrerror());
+	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+	}
+	debug(54, 3) ("ipcCreate: FD %d sockaddr %s:%d\n",
+	    crfd, inet_ntoa(CS.sin_addr), ntohs(CS.sin_port));
+    }
+    if (type == IPC_TCP_SOCKET) {
+	if (listen(crfd, 1) < 0) {
+	    debug(50, 1) ("ipcCreate: listen FD %d: %s\n", crfd, xstrerror());
+	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+	}
+	debug(54, 3) ("ipcCreate: FD %d listening...\n", crfd);
+    }
+    /* flush or else we get dup data if unbuffered_logs is set */
+    logsFlush();
+    if ((pid = fork()) < 0) {
+	debug(50, 1) ("ipcCreate: fork: %s\n", xstrerror());
+	return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+    }
+    if (pid > 0) {		/* parent */
+	/* close shared socket with child */
+	comm_close(crfd);
+	if (cwfd != crfd)
+	    comm_close(cwfd);
+	cwfd = crfd = -1;
+	if (type == IPC_TCP_SOCKET || type == IPC_UDP_SOCKET) {
+	    if (comm_connect_addr(pwfd, &CS) == COMM_ERROR)
+		return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+	}
+	memset(hello_buf, '\0', HELLO_BUF_SZ);
+	if (type == IPC_UDP_SOCKET)
+	    x = recv(prfd, hello_buf, HELLO_BUF_SZ - 1, 0);
+	else
+	    x = read(prfd, hello_buf, HELLO_BUF_SZ - 1);
+	if (x < 0) {
+	    debug(50, 0) ("ipcCreate: PARENT: hello read test failed\n");
+	    debug(50, 0) ("--> read: %s\n", xstrerror());
+	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+	} else if (strcmp(hello_buf, hello_string)) {
+	    debug(54, 0) ("ipcCreate: PARENT: hello read test failed\n");
+	    debug(54, 0) ("--> read returned %d\n", x);
+	    debug(54, 0) ("--> got '%s'\n", rfc1738_escape(hello_buf));
+	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+	}
+	commSetTimeout(prfd, -1, NULL, NULL);
+	commSetNonBlocking(prfd);
+	commSetNonBlocking(pwfd);
+	if (rfd)
+	    *rfd = prfd;
+	if (wfd)
+	    *wfd = pwfd;
+	fd_table[prfd].flags.ipc = 1;
+	fd_table[pwfd].flags.ipc = 1;
+	if (Config.sleep_after_fork) {
+	    /* XXX emulation of usleep() */
+	    struct timeval sl;
+	    sl.tv_sec = Config.sleep_after_fork / 1000000;
+	    sl.tv_usec = Config.sleep_after_fork % 1000000;
+	    select(0, NULL, NULL, NULL, &sl);
+	}
+	return pid;
+    }
+    /* child */
+    no_suid();			/* give up extra priviliges */
+    /* close shared socket with parent */
+    close(prfd);
+    if (pwfd != prfd)
+	close(pwfd);
+    pwfd = prfd = -1;
+
+    if (type == IPC_TCP_SOCKET) {
+	debug(54, 3) ("ipcCreate: calling accept on FD %d\n", crfd);
+	if ((fd = accept(crfd, NULL, NULL)) < 0) {
+	    debug(50, 0) ("ipcCreate: FD %d accept: %s\n", crfd, xstrerror());
+	    _exit(1);
+	}
+	debug(54, 3) ("ipcCreate: CHILD accepted new FD %d\n", fd);
+	close(crfd);
+	cwfd = crfd = fd;
+    } else if (type == IPC_UDP_SOCKET) {
+	if (comm_connect_addr(crfd, &PS) == COMM_ERROR)
+	    return ipcCloseAllFD(prfd, pwfd, crfd, cwfd);
+    }
+    if (type == IPC_UDP_SOCKET) {
+	x = send(cwfd, hello_string, strlen(hello_string) + 1, 0);
+	if (x < 0) {
+	    debug(50, 0) ("sendto FD %d: %s\n", cwfd, xstrerror());
+	    debug(50, 0) ("ipcCreate: CHILD: hello write test failed\n");
+	    _exit(1);
+	}
+    } else {
+	if (write(cwfd, hello_string, strlen(hello_string) + 1) < 0) {
+	    debug(50, 0) ("write FD %d: %s\n", cwfd, xstrerror());
+	    debug(50, 0) ("ipcCreate: CHILD: hello write test failed\n");
+	    _exit(1);
+	}
+    }
+#if HAVE_PUTENV
+    env_str = (char *)xcalloc((tmp_s = strlen(Config.debugOptions) + 32), 1);
+    snprintf(env_str, tmp_s, "SQUID_DEBUG=%s", Config.debugOptions);
+    putenv(env_str);
+#endif
+    /*
+     * This double-dup stuff avoids problems when one of 
+     *  crfd, cwfd, or debug_log are in the rage 0-2.
+     */
+    do {
+	x = open(_PATH_DEVNULL, 0, 0444);
+	if (x > -1)
+	    commSetCloseOnExec(x);
+    } while (x < 3);
+    t1 = dup(crfd);
+    t2 = dup(cwfd);
+    t3 = dup(fileno(debug_log));
+    assert(t1 > 2 && t2 > 2 && t3 > 2);
+    close(crfd);
+    close(cwfd);
+    close(fileno(debug_log));
+    dup2(t1, 0);
+    dup2(t2, 1);
+    dup2(t3, 2);
+    close(t1);
+    close(t2);
+    close(t3);
+    /* Make sure all other filedescriptors are closed */
+    for (x = 3; x < SQUID_MAXFD; x++)
+	close(x);
+#if HAVE_SETSID
+    setsid();
+#endif
+    execvp(prog, (char *const *) args);
+    debug_log = fdopen(2, "a+");
+    debug(50, 0) ("ipcCreate: %s: %s\n", prog, xstrerror());
+    _exit(1);
+    return 0;
+}
Index: squid/src/net_db.cc
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Attic/net_db.cc,v
retrieving revision 1.1.2.1
retrieving revision 1.1.2.2
diff -u -r1.1.2.1 -r1.1.2.2
--- squid/src/net_db.cc	9 Oct 2002 05:21:29 -0000	1.1.2.1
+++ squid/src/net_db.cc	11 Oct 2002 15:40:54 -0000	1.1.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: net_db.cc,v 1.1.2.1 2002/10/09 05:21:29 rbcollins Exp $
+ * $Id: net_db.cc,v 1.1.2.2 2002/10/11 15:40:54 rbcollins Exp $
  *
  * DEBUG: section 38    Network Measurement Database
  * AUTHOR: Duane Wessels
@@ -109,7 +109,7 @@
 static void
 netdbHashDelete(const char *key)
 {
-    hash_link *hptr = hash_lookup(addr_table, key);
+    hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
     if (hptr == NULL) {
 	debug_trap("netdbHashDelete: key not found");
 	return;
@@ -120,7 +120,7 @@
 static void
 netdbHostInsert(netdbEntry * n, const char *hostname)
 {
-    net_db_name *x = memAllocate(MEM_NET_DB_NAME);
+    net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
     x->hash.key = xstrdup(hostname);
     x->next = n->hosts;
     n->hosts = x;
@@ -180,8 +180,8 @@
 static int
 netdbLRU(const void *A, const void *B)
 {
-    const netdbEntry *const *n1 = A;
-    const netdbEntry *const *n2 = B;
+    const netdbEntry *const *n1 = (const netdbEntry *const *)A;
+    const netdbEntry *const *n2 = (const netdbEntry *const *)B;
     if ((*n1)->last_use_time > (*n2)->last_use_time)
 	return (1);
     if ((*n1)->last_use_time < (*n2)->last_use_time)
@@ -197,7 +197,7 @@
     int k = 0;
     int list_count = 0;
     int removed = 0;
-    list = xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
+    list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
     hash_first(addr_table);
     while ((n = (netdbEntry *) hash_next(addr_table))) {
 	assert(list_count < memInUse(MEM_NETDBENTRY));
@@ -233,7 +233,7 @@
     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 	netdbPurgeLRU();
     if ((n = netdbLookupAddr(addr)) == NULL) {
-	n = memAllocate(MEM_NETDBENTRY);
+	n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 	netdbHashInsert(n, addr);
     }
     return n;
@@ -243,7 +243,7 @@
 netdbSendPing(const ipcache_addrs * ia, void *data)
 {
     struct in_addr addr;
-    char *hostname = ((generic_cbdata *) data)->data;
+    char *hostname = (char *)((generic_cbdata *) data)->data;
     netdbEntry *n;
     netdbEntry *na;
     net_db_name *x;
@@ -320,8 +320,8 @@
 static int
 sortByRtt(const void *A, const void *B)
 {
-    const netdbEntry *const *n1 = A;
-    const netdbEntry *const *n2 = B;
+    const netdbEntry *const *n1 = (const netdbEntry *const *)A;
+    const netdbEntry *const *n2 = (const netdbEntry *const *)B;
     if ((*n1)->rtt > (*n2)->rtt)
 	return 1;
     else if ((*n1)->rtt < (*n2)->rtt)
@@ -358,7 +358,7 @@
 	    n->n_peers_alloc <<= 1;
 	debug(38, 3) ("netdbPeerAdd: Growing peer list for '%s' to %d\n",
 	    n->network, n->n_peers_alloc);
-	n->peers = xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
+	n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 	for (i = 0; i < osize; i++)
 	    *(n->peers + i) = *(o + i);
 	if (osize) {
@@ -374,8 +374,8 @@
 static int
 sortPeerByRtt(const void *A, const void *B)
 {
-    const net_db_peer *p1 = A;
-    const net_db_peer *p2 = B;
+    const net_db_peer *p1 = (net_db_peer *)A;
+    const net_db_peer *p2 = (net_db_peer *)B;
     if (p1->rtt > p2->rtt)
 	return 1;
     else if (p1->rtt < p2->rtt)
@@ -462,7 +462,7 @@
 	file_close(fd);
 	return;
     }
-    t = buf = xcalloc(1, sb.st_size + 1);
+    t = buf = (char *)xcalloc(1, sb.st_size + 1);
     l = FD_READ_METHOD(fd, buf, sb.st_size);
     file_close(fd);
     if (l <= 0)
@@ -503,7 +503,7 @@
 	if ((q = strtok(NULL, w_space)) == NULL)
 	    continue;
 	N.last_use_time = (time_t) atoi(q);
-	n = memAllocate(MEM_NETDBENTRY);
+	n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 	xmemcpy(n, &N, sizeof(netdbEntry));
 	netdbHashInsert(n, addr);
 	while ((q = strtok(NULL, w_space)) != NULL) {
@@ -533,7 +533,7 @@
 static void
 netdbFreeNetdbEntry(void *data)
 {
-    netdbEntry *n = data;
+    netdbEntry *n = (netdbEntry *)data;
     safe_free(n->peers);
     memFree(n, MEM_NETDBENTRY);
 }
@@ -541,7 +541,7 @@
 static void
 netdbFreeNameEntry(void *data)
 {
-    net_db_name *x = data;
+    net_db_name *x = (net_db_name *)data;
     xfree(x->hash.key);
     memFree(x, MEM_NET_DB_NAME);
 }
@@ -550,7 +550,7 @@
 static void
 netdbExchangeHandleReply(void *data, StoreIOBuffer recievedData)
 {
-    netdbExchangeState *ex = data;
+    netdbExchangeState *ex = (netdbExchangeState *)data;
     int rec_sz = 0;
     off_t o;
     struct in_addr addr;
@@ -598,7 +598,7 @@
 		netdbExchangeDone(ex);
 		return;
 	    }
-	    assert(ex->buf_ofs >= hdr_sz);
+	    assert((size_t)ex->buf_ofs >= hdr_sz);
 
 	    /*
 	     * Now, point p to the part of the buffer where the data
@@ -725,7 +725,7 @@
 static void
 netdbExchangeDone(void *data)
 {
-    netdbExchangeState *ex = data;
+    netdbExchangeState *ex = (netdbExchangeState *)data;
     debug(38, 3) ("netdbExchangeDone: %s\n", storeUrl(ex->e));
     requestUnlink(ex->r);
     storeUnregister(ex->sc, ex->e, ex);
@@ -784,8 +784,8 @@
     N = ++n->pings_recv;
     if (N > 5)
 	N = 5;
-    if (rtt < 1.0)
-	rtt = 1.0;
+    if (rtt < 1)
+	rtt = 1;
     n->hops = ((n->hops * (N - 1)) + hops) / N;
     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
     debug(38, 3) ("netdbHandlePingReply: %s; rtt=%5.1f  hops=%4.1f\n",
@@ -841,7 +841,7 @@
 	"RTT",
 	"Hops",
 	"Hostnames");
-    list = xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
+    list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
     i = 0;
     hash_first(addr_table);
     while ((n = (netdbEntry *) hash_next(addr_table)))
@@ -1020,7 +1020,7 @@
     rec_sz += 1 + sizeof(addr.s_addr);
     rec_sz += 1 + sizeof(int);
     rec_sz += 1 + sizeof(int);
-    buf = memAllocate(MEM_4K_BUF);
+    buf = (char *)memAllocate(MEM_4K_BUF);
     i = 0;
     hash_first(addr_table);
     while ((n = (netdbEntry *) hash_next(addr_table))) {
@@ -1071,7 +1071,7 @@
 netdbExchangeStart(void *data)
 {
 #if USE_ICMP
-    peer *p = data;
+    peer *p = (peer *)data;
     char *uri;
     netdbExchangeState *ex;
     StoreIOBuffer tempBuffer = EMPTYIOBUFFER;
--- squid/src/peer_digest.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,982 +0,0 @@
-
-/*
- * $Id: peer_digest.c,v 1.17.2.1 2002/10/04 07:07:27 rbcollins Exp $
- *
- * DEBUG: section 72    Peer Digest Routines
- * AUTHOR: Alex Rousskov
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-#include "Store.h"
-
-#if USE_CACHE_DIGESTS
-
-#include "StoreClient.h"
-
-/* local types */
-
-/* local prototypes */
-static time_t peerDigestIncDelay(const PeerDigest * pd);
-static time_t peerDigestNewDelay(const StoreEntry * e);
-static void peerDigestSetCheck(PeerDigest * pd, time_t delay);
-static void peerDigestClean(PeerDigest *);
-static EVH peerDigestCheck;
-static void peerDigestRequest(PeerDigest * pd);
-static STCB peerDigestHandleReply;
-static int peerDigestFetchReply(void *, char *, ssize_t);
-static int peerDigestSwapInHeaders(void *, char *, ssize_t);
-static int peerDigestSwapInCBlock(void *, char *, ssize_t);
-static int peerDigestSwapInMask(void *, char *, ssize_t);
-static int peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const char *step_name);
-static void peerDigestFetchStop(DigestFetchState * fetch, char *buf, const char *reason);
-static void peerDigestFetchAbort(DigestFetchState * fetch, char *buf, const char *reason);
-static void peerDigestReqFinish(DigestFetchState * fetch, char *buf, int, int, int, const char *reason, int err);
-static void peerDigestPDFinish(DigestFetchState * fetch, int pcb_valid, int err);
-static void peerDigestFetchFinish(DigestFetchState * fetch, int err);
-static void peerDigestFetchSetStats(DigestFetchState * fetch);
-static int peerDigestSetCBlock(PeerDigest * pd, const char *buf);
-static int peerDigestUseful(const PeerDigest * pd);
-
-
-/* local constants */
-
-#define StoreDigestCBlockSize sizeof(StoreDigestCBlock)
-
-/* min interval for requesting digests from a given peer */
-static const time_t PeerDigestReqMinGap = 5 * 60;	/* seconds */
-/* min interval for requesting digests (cumulative request stream) */
-static const time_t GlobDigestReqMinGap = 1 * 60;	/* seconds */
-
-/* local vars */
-
-static time_t pd_last_req_time = 0;	/* last call to Check */
-
-/* initialize peer digest */
-static void
-peerDigestInit(PeerDigest * pd, peer * p)
-{
-    assert(pd && p);
-
-    memset(pd, 0, sizeof(*pd));
-    pd->peer = p;
-    /* if peer disappears, we will know it's name */
-    stringInit(&pd->host, p->host);
-
-    pd->times.initialized = squid_curtime;
-}
-
-static void
-peerDigestClean(PeerDigest * pd)
-{
-    assert(pd);
-    if (pd->cd)
-	cacheDigestDestroy(pd->cd);
-    stringClean(&pd->host);
-}
-
-CBDATA_TYPE(PeerDigest);
-
-/* allocate new peer digest, call Init, and lock everything */
-PeerDigest *
-peerDigestCreate(peer * p)
-{
-    PeerDigest *pd;
-    assert(p);
-
-    CBDATA_INIT_TYPE(PeerDigest);
-    pd = cbdataAlloc(PeerDigest);
-    peerDigestInit(pd, p);
-
-    /* XXX This does not look right, and the same thing again in the caller */
-    return cbdataReference(pd);
-}
-
-/* call Clean and free/unlock everything */
-static void
-peerDigestDestroy(PeerDigest * pd)
-{
-    peer *p;
-    assert(pd);
-
-    /* inform peer (if any) that we are gone */
-    if (cbdataReferenceValidDone(pd->peer, (void **) &p))
-	peerNoteDigestGone(p);
-
-    peerDigestClean(pd);
-    cbdataFree(pd);
-}
-
-/* called by peer to indicate that somebody actually needs this digest */
-void
-peerDigestNeeded(PeerDigest * pd)
-{
-    assert(pd);
-    assert(!pd->flags.needed);
-    assert(!pd->cd);
-
-    pd->flags.needed = 1;
-    pd->times.needed = squid_curtime;
-    peerDigestSetCheck(pd, 0);	/* check asap */
-}
-
-/* currently we do not have a reason to disable without destroying */
-#if FUTURE_CODE
-/* disables peer for good */
-static void
-peerDigestDisable(PeerDigest * pd)
-{
-    debug(72, 2) ("peerDigestDisable: peer %s disabled for good\n",
-	strBuf(pd->host));
-    pd->times.disabled = squid_curtime;
-    pd->times.next_check = -1;	/* never */
-    pd->flags.usable = 0;
-
-    if (pd->cd) {
-	cacheDigestDestroy(pd->cd);
-	pd->cd = NULL;
-    }
-    /* we do not destroy the pd itself to preserve its "history" and stats */
-}
-#endif
-
-/* increment retry delay [after an unsuccessful attempt] */
-static time_t
-peerDigestIncDelay(const PeerDigest * pd)
-{
-    assert(pd);
-    return pd->times.retry_delay > 0 ?
-	2 * pd->times.retry_delay :	/* exponential backoff */
-	PeerDigestReqMinGap;	/* minimal delay */
-}
-
-/* artificially increases Expires: setting to avoid race conditions 
- * returns the delay till that [increased] expiration time */
-static time_t
-peerDigestNewDelay(const StoreEntry * e)
-{
-    assert(e);
-    if (e->expires > 0)
-	return e->expires + PeerDigestReqMinGap - squid_curtime;
-    return PeerDigestReqMinGap;
-}
-
-/* registers next digest verification */
-static void
-peerDigestSetCheck(PeerDigest * pd, time_t delay)
-{
-    eventAdd("peerDigestCheck", peerDigestCheck, pd, (double) delay, 1);
-    pd->times.next_check = squid_curtime + delay;
-    debug(72, 3) ("peerDigestSetCheck: will check peer %s in %d secs\n",
-	strBuf(pd->host), (int) delay);
-}
-
-/*
- * called when peer is about to disappear or have already disappeared
- */
-void
-peerDigestNotePeerGone(PeerDigest * pd)
-{
-    if (pd->flags.requested) {
-	debug(72, 2) ("peerDigest: peer %s gone, will destroy after fetch.\n",
-	    strBuf(pd->host));
-	/* do nothing now, the fetching chain will notice and take action */
-    } else {
-	debug(72, 2) ("peerDigest: peer %s is gone, destroying now.\n",
-	    strBuf(pd->host));
-	peerDigestDestroy(pd);
-    }
-}
-
-/* callback for eventAdd() (with peer digest locked)
- * request new digest if our copy is too old or if we lack one; 
- * schedule next check otherwise */
-static void
-peerDigestCheck(void *data)
-{
-    PeerDigest *pd = data;
-    time_t req_time;
-
-    assert(!pd->flags.requested);
-
-    pd->times.next_check = 0;	/* unknown */
-
-    if (!cbdataReferenceValid(pd->peer)) {
-	peerDigestNotePeerGone(pd);
-	return;
-    }
-    debug(72, 3) ("peerDigestCheck: peer %s:%d\n", pd->peer->host, pd->peer->http_port);
-    debug(72, 3) ("peerDigestCheck: time: %ld, last received: %ld (%+d)\n",
-	(long int) squid_curtime, (long int) pd->times.received, (int) (squid_curtime - pd->times.received));
-
-    /* decide when we should send the request:
-     * request now unless too close to other requests */
-    req_time = squid_curtime;
-
-    /* per-peer limit */
-    if (req_time - pd->times.received < PeerDigestReqMinGap) {
-	debug(72, 2) ("peerDigestCheck: %s, avoiding close peer requests (%d < %d secs).\n",
-	    strBuf(pd->host), (int) (req_time - pd->times.received),
-	    (int) PeerDigestReqMinGap);
-	req_time = pd->times.received + PeerDigestReqMinGap;
-    }
-    /* global limit */
-    if (req_time - pd_last_req_time < GlobDigestReqMinGap) {
-	debug(72, 2) ("peerDigestCheck: %s, avoiding close requests (%d < %d secs).\n",
-	    strBuf(pd->host), (int) (req_time - pd_last_req_time),
-	    (int) GlobDigestReqMinGap);
-	req_time = pd_last_req_time + GlobDigestReqMinGap;
-    }
-    if (req_time <= squid_curtime)
-	peerDigestRequest(pd);	/* will set pd->flags.requested */
-    else
-	peerDigestSetCheck(pd, req_time - squid_curtime);
-}
-
-CBDATA_TYPE(DigestFetchState);
-
-/* ask store for a digest */
-static void
-peerDigestRequest(PeerDigest * pd)
-{
-    peer *p = pd->peer;
-    StoreEntry *e, *old_e;
-    char *url;
-    const cache_key *key;
-    request_t *req;
-    DigestFetchState *fetch = NULL;
-    StoreIOBuffer tempBuffer = EMPTYIOBUFFER;
-
-    pd->req_result = NULL;
-    pd->flags.requested = 1;
-
-    /* compute future request components */
-    if (p->digest_url)
-	url = xstrdup(p->digest_url);
-    else
-	url = internalRemoteUri(p->host, p->http_port,
-	    "/squid-internal-periodic/", StoreDigestFileName);
-
-    req = urlParse(METHOD_GET, url);
-    assert(req);
-    key = storeKeyPublicByRequest(req);
-    debug(72, 2) ("peerDigestRequest: %s key: %s\n", url, storeKeyText(key));
-
-    /* add custom headers */
-    assert(!req->header.len);
-    httpHeaderPutStr(&req->header, HDR_ACCEPT, StoreDigestMimeStr);
-    httpHeaderPutStr(&req->header, HDR_ACCEPT, "text/html");
-    if (p->login)
-	xstrncpy(req->login, p->login, MAX_LOGIN_SZ);
-    /* create fetch state structure */
-    CBDATA_INIT_TYPE(DigestFetchState);
-    fetch = cbdataAlloc(DigestFetchState);
-    fetch->request = requestLink(req);
-    fetch->pd = cbdataReference(pd);
-    fetch->offset = 0;
-    fetch->state = DIGEST_READ_REPLY;
-
-    /* update timestamps */
-    fetch->start_time = squid_curtime;
-    pd->times.requested = squid_curtime;
-    pd_last_req_time = squid_curtime;
-
-    req->flags.cachable = 1;
-    /* the rest is based on clientProcessExpired() */
-    req->flags.refresh = 1;
-    old_e = fetch->old_entry = storeGet(key);
-    if (old_e) {
-	debug(72, 5) ("peerDigestRequest: found old entry\n");
-	storeLockObject(old_e);
-	storeCreateMemObject(old_e, url, url);
-	fetch->old_sc = storeClientListAdd(old_e, fetch);
-    }
-    e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method);
-    assert(EBIT_TEST(e->flags, KEY_PRIVATE));
-    fetch->sc = storeClientListAdd(e, fetch);
-    /* set lastmod to trigger IMS request if possible */
-    if (old_e)
-	e->lastmod = old_e->lastmod;
-
-    /* push towards peer cache */
-    debug(72, 3) ("peerDigestRequest: forwarding to fwdStart...\n");
-    fwdStart(-1, e, req);
-    tempBuffer.offset = 0;
-    tempBuffer.length =  SM_PAGE_SIZE;
-    tempBuffer.data = fetch->buf;
-    storeClientCopy(fetch->sc, e, tempBuffer,
-	peerDigestHandleReply, fetch);
-}
-
-
-/* Handle the data copying .. */
-
-/*
- * This routine handles the copy data and then redirects the
- * copy to a bunch of subfunctions depending upon the copy state.
- * It also tracks the buffer offset and "seen", since I'm actually
- * not interested in rewriting everything to suit my little idea.
- */
-static void
-peerDigestHandleReply(void *data, StoreIOBuffer recievedData)
-{
-    DigestFetchState *fetch = data;
-    PeerDigest *pd = fetch->pd;
-    int retsize = -1;
-    digest_read_state_t prevstate;
-    int newsize;
-
-    assert(pd && recievedData.data);
-    /* The existing code assumes that the recieved pointer is
-     * where we asked the data to be put
-     */
-    assert(fetch->buf + fetch->bufofs == recievedData.data);
-
-    /* Update the buffer size */
-    fetch->bufofs += recievedData.length;
-
-    assert(fetch->bufofs <= SM_PAGE_SIZE);
-
-    /* If we've fetched enough, return */
-    if (peerDigestFetchedEnough(fetch, fetch->buf, fetch->bufofs, "peerDigestHandleReply"))
-	return;
-
-    /* Call the right function based on the state */
-    /* (Those functions will update the state if needed) */
-
-    /* Give us a temporary reference. Some of the calls we make may
-     * try to destroy the fetch structure, and we like to know if they
-     * do
-     */
-    fetch = cbdataReference(fetch);
-
-    /* Repeat this loop until we're out of data OR the state changes */
-    /* (So keep going if the state has changed and we still have data */
-    do {
-	prevstate = fetch->state;
-	switch (fetch->state) {
-	case DIGEST_READ_REPLY:
-	    retsize = peerDigestFetchReply(data, fetch->buf, fetch->bufofs);
-	    break;
-	case DIGEST_READ_HEADERS:
-	    retsize = peerDigestSwapInHeaders(data, fetch->buf, fetch->bufofs);
-	    break;
-	case DIGEST_READ_CBLOCK:
-	    retsize = peerDigestSwapInCBlock(data, fetch->buf, fetch->bufofs);
-	    break;
-	case DIGEST_READ_MASK:
-	    retsize = peerDigestSwapInMask(data, fetch->buf, fetch->bufofs);
-	    break;
-	case DIGEST_READ_NONE:
-	    break;
-	case DIGEST_READ_DONE:
-	    goto finish;
-	    break;
-	default:
-	    fatal("Bad digest transfer mode!\n");
-	}
-
-	if (retsize < 0)
-	    goto finish;
-	/*
-	 * The returned size indicates how much of the buffer was read -
-	 * so move the remainder of the buffer to the beginning
-	 * and update the bufofs / bufsize
-	 */
-	newsize = fetch->bufofs - retsize;
-	xmemmove(fetch->buf, fetch->buf + retsize, fetch->bufofs - newsize);
-	fetch->bufofs = newsize;
-
-    } while (cbdataReferenceValid(fetch) && prevstate != fetch->state && fetch->bufofs > 0);
-
-    /* Update the copy offset */
-    fetch->offset += recievedData.length;
-
-    /* Schedule another copy */
-    if (cbdataReferenceValid(fetch)) {
-	StoreIOBuffer tempBuffer = EMPTYIOBUFFER;
-	tempBuffer.offset = fetch->offset;
-	tempBuffer.length = SM_PAGE_SIZE - fetch->bufofs;
-	tempBuffer.data = fetch->buf + fetch->bufofs;
-	storeClientCopy(fetch->sc, fetch->entry, tempBuffer,
-	    peerDigestHandleReply, fetch);
-    }
-  finish:
-    /* Get rid of our reference, we've finished with it for now */
-    cbdataReferenceDone(fetch);
-}
-
-
-
-/* wait for full http headers to be received then parse them */
-/*
- * This routine handles parsing the reply line.
- * If the reply line indicates an OK, the same data is thrown
- * to SwapInHeaders(). If the reply line is a NOT_MODIFIED,
- * we simply stop parsing.
- */
-static int
-peerDigestFetchReply(void *data, char *buf, ssize_t size)
-{
-    DigestFetchState *fetch = data;
-    PeerDigest *pd = fetch->pd;
-    size_t hdr_size;
-    assert(pd && buf);
-    assert(!fetch->offset);
-
-    assert(fetch->state == DIGEST_READ_REPLY);
-    if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestFetchReply"))
-	return -1;
-
-    if ((hdr_size = headersEnd(buf, size))) {
-	http_status status;
-	HttpReply *reply = fetch->entry->mem_obj->reply;
-	assert(reply);
-	httpReplyParse(reply, buf, hdr_size);
-	status = reply->sline.status;
-	debug(72, 3) ("peerDigestFetchReply: %s status: %d, expires: %ld (%+d)\n",
-	    strBuf(pd->host), status,
-	    (long int) reply->expires, (int) (reply->expires - squid_curtime));
-
-	/* this "if" is based on clientHandleIMSReply() */
-	if (status == HTTP_NOT_MODIFIED) {
-	    request_t *r = NULL;
-	    /* our old entry is fine */
-	    assert(fetch->old_entry);
-	    if (!fetch->old_entry->mem_obj->request)
-		fetch->old_entry->mem_obj->request = r =
-		    requestLink(fetch->entry->mem_obj->request);
-	    assert(fetch->old_entry->mem_obj->request);
-	    httpReplyUpdateOnNotModified(fetch->old_entry->mem_obj->reply, reply);
-	    storeTimestampsSet(fetch->old_entry);
-	    /* get rid of 304 reply */
-	    storeUnregister(fetch->sc, fetch->entry, fetch);
-	    storeUnlockObject(fetch->entry);
-	    fetch->entry = fetch->old_entry;
-	    fetch->old_entry = NULL;
-	    /* preserve request -- we need its size to update counters */
-	    /* requestUnlink(r); */
-	    /* fetch->entry->mem_obj->request = NULL; */
-	} else if (status == HTTP_OK) {
-	    /* get rid of old entry if any */
-	    if (fetch->old_entry) {
-		debug(72, 3) ("peerDigestFetchReply: got new digest, releasing old one\n");
-		storeUnregister(fetch->old_sc, fetch->old_entry, fetch);
-		storeReleaseRequest(fetch->old_entry);
-		storeUnlockObject(fetch->old_entry);
-		fetch->old_entry = NULL;
-	    }
-	} else {
-	    /* some kind of a bug */
-	    peerDigestFetchAbort(fetch, buf, httpStatusLineReason(&reply->sline));
-	    return -1;		/* XXX -1 will abort stuff in ReadReply! */
-	}
-	/* must have a ready-to-use store entry if we got here */
-	/* can we stay with the old in-memory digest? */
-	if (status == HTTP_NOT_MODIFIED && fetch->pd->cd) {
-	    peerDigestFetchStop(fetch, buf, "Not modified");
-	    fetch->state = DIGEST_READ_DONE;
-	} else {
-	    fetch->state = DIGEST_READ_HEADERS;
-	}
-    } else {
-	/* need more data, do we have space? */
-	if (size >= SM_PAGE_SIZE)
-	    peerDigestFetchAbort(fetch, buf, "reply header too big");
-    }
-
-    /* We don't want to actually ack that we've handled anything,
-     * otherwise SwapInHeaders() won't get the reply line .. */
-    return 0;
-}
-
-/* fetch headers from disk, pass on to SwapInCBlock */
-static int
-peerDigestSwapInHeaders(void *data, char *buf, ssize_t size)
-{
-    DigestFetchState *fetch = data;
-    size_t hdr_size;
-
-    assert(fetch->state == DIGEST_READ_HEADERS);
-    if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInHeaders"))
-	return -1;
-
-    assert(!fetch->offset);
-    if ((hdr_size = headersEnd(buf, size))) {
-	assert(fetch->entry->mem_obj->reply);
-	if (!fetch->entry->mem_obj->reply->sline.status)
-	    httpReplyParse(fetch->entry->mem_obj->reply, buf, hdr_size);
-	if (fetch->entry->mem_obj->reply->sline.status != HTTP_OK) {
-	    debug(72, 1) ("peerDigestSwapInHeaders: %s status %d got cached!\n",
-		strBuf(fetch->pd->host), fetch->entry->mem_obj->reply->sline.status);
-	    peerDigestFetchAbort(fetch, buf, "internal status error");
-	    return -1;
-	}
-	fetch->state = DIGEST_READ_CBLOCK;
-	return hdr_size;	/* Say how much data we read */
-    } else {
-	/* need more data, do we have space? */
-	if (size >= SM_PAGE_SIZE) {
-	    peerDigestFetchAbort(fetch, buf, "stored header too big");
-	    return -1;
-	} else {
-	    return 0;		/* We need to read more to parse .. */
-	}
-    }
-    fatal("peerDigestSwapInHeaders() - shouldn't get here!\n");
-}
-
-static int
-peerDigestSwapInCBlock(void *data, char *buf, ssize_t size)
-{
-    DigestFetchState *fetch = data;
-
-    assert(fetch->state == DIGEST_READ_CBLOCK);
-    if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInCBlock"))
-	return -1;
-
-    if (size >= StoreDigestCBlockSize) {
-	PeerDigest *pd = fetch->pd;
-	HttpReply *rep = fetch->entry->mem_obj->reply;
-
-	assert(pd && rep);
-	if (peerDigestSetCBlock(pd, buf)) {
-	    /* XXX: soon we will have variable header size */
-	    /* switch to CD buffer and fetch digest guts */
-	    buf = NULL;
-	    assert(pd->cd->mask);
-	    fetch->state = DIGEST_READ_MASK;
-	    return StoreDigestCBlockSize;
-	} else {
-	    peerDigestFetchAbort(fetch, buf, "invalid digest cblock");
-	    return -1;
-	}
-    } else {
-	/* need more data, do we have space? */
-	if (size >= SM_PAGE_SIZE) {
-	    peerDigestFetchAbort(fetch, buf, "digest cblock too big");
-	    return -1;
-	} else {
-	    return 0;		/* We need more data */
-	}
-    }
-    fatal("peerDigestSwapInCBlock(): shouldn't get here!\n");
-}
-
-static int
-peerDigestSwapInMask(void *data, char *buf, ssize_t size)
-{
-    DigestFetchState *fetch = data;
-    PeerDigest *pd;
-
-    pd = fetch->pd;
-    assert(pd->cd && pd->cd->mask);
-
-    /*
-     * NOTENOTENOTENOTENOTE: buf doesn't point to pd->cd->mask anymore!
-     * we need to do the copy ourselves!
-     */
-    xmemcpy(pd->cd->mask + fetch->mask_offset, buf, size);
-
-    /* NOTE! buf points to the middle of pd->cd->mask! */
-    if (peerDigestFetchedEnough(fetch, NULL, size, "peerDigestSwapInMask"))
-	return -1;
-
-    fetch->mask_offset += size;
-    if (fetch->mask_offset >= pd->cd->mask_size) {
-	debug(72, 2) ("peerDigestSwapInMask: Done! Got %d, expected %d\n",
-	    fetch->mask_offset, pd->cd->mask_size);
-	assert(fetch->mask_offset == pd->cd->mask_size);
-	assert(peerDigestFetchedEnough(fetch, NULL, 0, "peerDigestSwapInMask"));
-	return -1;		/* XXX! */
-    } else {
-	/* We always read everything, so return so */
-	return size;
-    }
-    fatal("peerDigestSwapInMask(): shouldn't get here!\n");
-}
-
-static int
-peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const char *step_name)
-{
-    PeerDigest *pd = NULL;
-    const char *host = "";	/* peer host */
-    const char *reason = NULL;	/* reason for completion */
-    const char *no_bug = NULL;	/* successful completion if set */
-    const int pdcb_valid = cbdataReferenceValid(fetch->pd);
-    const int pcb_valid = cbdataReferenceValid(fetch->pd->peer);
-
-    /* test possible exiting conditions (the same for most steps!)
-     * cases marked with '?!' should not happen */
-
-    if (!reason) {
-	if (!(pd = fetch->pd))
-	    reason = "peer digest disappeared?!";
-#if DONT			/* WHY NOT? /HNO */
-	else if (!cbdataReferenceValid(pd))
-	    reason = "invalidated peer digest?!";
-#endif
-	else
-	    host = strBuf(pd->host);
-    }
-    debug(72, 6) ("%s: peer %s, offset: %d size: %d.\n",
-	step_name, host,
-	fetch->offset, size);
-
-    /* continue checking (with pd and host known and valid) */
-    if (!reason) {
-	if (!cbdataReferenceValid(pd->peer))
-	    reason = "peer disappeared";
-	else if (size < 0)
-	    reason = "swap failure";
-	else if (!fetch->entry)
-	    reason = "swap aborted?!";
-	else if (EBIT_TEST(fetch->entry->flags, ENTRY_ABORTED))
-	    reason = "swap aborted";
-    }
-    /* continue checking (maybe-successful eof case) */
-    if (!reason && !size) {
-	if (!pd->cd)
-	    reason = "null digest?!";
-	else if (fetch->mask_offset != pd->cd->mask_size)
-	    reason = "premature end of digest?!";
-	else if (!peerDigestUseful(pd))
-	    reason = "useless digest";
-	else
-	    reason = no_bug = "success";
-    }
-    /* finish if we have a reason */
-    if (reason) {
-	const int level = strstr(reason, "?!") ? 1 : 3;
-	debug(72, level) ("%s: peer %s, exiting after '%s'\n",
-	    step_name, host, reason);
-	peerDigestReqFinish(fetch, buf,
-	    1, pdcb_valid, pcb_valid, reason, !no_bug);
-    } else {
-	/* paranoid check */
-	assert(pdcb_valid && pcb_valid);
-    }
-    return reason != NULL;
-}
-
-/* call this when all callback data is valid and fetch must be stopped but
- * no error has occurred (e.g. we received 304 reply and reuse old digest) */
-static void
-peerDigestFetchStop(DigestFetchState * fetch, char *buf, const char *reason)
-{
-    assert(reason);
-    debug(72, 2) ("peerDigestFetchStop: peer %s, reason: %s\n",
-	strBuf(fetch->pd->host), reason);
-    peerDigestReqFinish(fetch, buf, 1, 1, 1, reason, 0);
-}
-
-/* call this when all callback data is valid but something bad happened */
-static void
-peerDigestFetchAbort(DigestFetchState * fetch, char *buf, const char *reason)
-{
-    assert(reason);
-    debug(72, 2) ("peerDigestFetchAbort: peer %s, reason: %s\n",
-	strBuf(fetch->pd->host), reason);
-    peerDigestReqFinish(fetch, buf, 1, 1, 1, reason, 1);
-}
-
-/* complete the digest transfer, update stats, unlock/release everything */
-static void
-peerDigestReqFinish(DigestFetchState * fetch, char *buf,
-    int fcb_valid, int pdcb_valid, int pcb_valid,
-    const char *reason, int err)
-{
-    assert(reason);
-
-    /* must go before peerDigestPDFinish */
-    if (pdcb_valid) {
-	fetch->pd->flags.requested = 0;
-	fetch->pd->req_result = reason;
-    }
-    /* schedule next check if peer is still out there */
-    if (pcb_valid) {
-	PeerDigest *pd = fetch->pd;
-	if (err) {
-	    pd->times.retry_delay = peerDigestIncDelay(pd);
-	    peerDigestSetCheck(pd, pd->times.retry_delay);
-	} else {
-	    pd->times.retry_delay = 0;
-	    peerDigestSetCheck(pd, peerDigestNewDelay(fetch->entry));
-	}
-    }
-    /* note: order is significant */
-    if (fcb_valid)
-	peerDigestFetchSetStats(fetch);
-    if (pdcb_valid)
-	peerDigestPDFinish(fetch, pcb_valid, err);
-    if (fcb_valid)
-	peerDigestFetchFinish(fetch, err);
-}
-
-
-/* destroys digest if peer disappeared
- * must be called only when fetch and pd cbdata are valid */
-static void
-peerDigestPDFinish(DigestFetchState * fetch, int pcb_valid, int err)
-{
-    PeerDigest *pd = fetch->pd;
-    const char *host = strBuf(pd->host);
-
-    pd->times.received = squid_curtime;
-    pd->times.req_delay = fetch->resp_time;
-    kb_incr(&pd->stats.sent.kbytes, (size_t) fetch->sent.bytes);
-    kb_incr(&pd->stats.recv.kbytes, (size_t) fetch->recv.bytes);
-    pd->stats.sent.msgs += fetch->sent.msg;
-    pd->stats.recv.msgs += fetch->recv.msg;
-
-    if (err) {
-	debug(72, 1) ("%sdisabling (%s) digest from %s\n",
-	    pcb_valid ? "temporary " : "",
-	    pd->req_result, host);
-
-	if (pd->cd) {
-	    cacheDigestDestroy(pd->cd);
-	    pd->cd = NULL;
-	}
-	pd->flags.usable = 0;
-
-	if (!pcb_valid)
-	    peerDigestNotePeerGone(pd);
-    } else {
-	assert(pcb_valid);
-
-	pd->flags.usable = 1;
-
-	/* XXX: ugly condition, but how? */
-	if (fetch->entry->store_status == STORE_OK)
-	    debug(72, 2) ("re-used old digest from %s\n", host);
-	else
-	    debug(72, 2) ("received valid digest from %s\n", host);
-    }
-    cbdataReferenceDone(fetch->pd);
-}
-
-/* free fetch state structures
- * must be called only when fetch cbdata is valid */
-static void
-peerDigestFetchFinish(DigestFetchState * fetch, int err)
-{
-    assert(fetch->entry && fetch->request);
-
-    if (fetch->old_entry) {
-	debug(72, 2) ("peerDigestFetchFinish: deleting old entry\n");
-	storeUnregister(fetch->sc, fetch->old_entry, fetch);
-	storeReleaseRequest(fetch->old_entry);
-	storeUnlockObject(fetch->old_entry);
-	fetch->old_entry = NULL;
-    }
-    /* update global stats */
-    kb_incr(&statCounter.cd.kbytes_sent, (size_t) fetch->sent.bytes);
-    kb_incr(&statCounter.cd.kbytes_recv, (size_t) fetch->recv.bytes);
-    statCounter.cd.msgs_sent += fetch->sent.msg;
-    statCounter.cd.msgs_recv += fetch->recv.msg;
-
-    /* unlock everything */
-    storeUnregister(fetch->sc, fetch->entry, fetch);
-    storeUnlockObject(fetch->entry);
-    requestUnlink(fetch->request);
-    fetch->entry = NULL;
-    fetch->request = NULL;
-    assert(fetch->pd == NULL);
-    cbdataFree(fetch);
-}
-
-/* calculate fetch stats after completion */
-static void
-peerDigestFetchSetStats(DigestFetchState * fetch)
-{
-    MemObject *mem;
-    assert(fetch->entry && fetch->request);
-
-    mem = fetch->entry->mem_obj;
-    assert(mem);
-
-    /* XXX: outgoing numbers are not precise */
-    /* XXX: we must distinguish between 304 hits and misses here */
-    fetch->sent.bytes = httpRequestPrefixLen(fetch->request);
-    fetch->recv.bytes = fetch->entry->store_status == STORE_PENDING ?
-	mem->inmem_hi : mem->object_sz;
-    fetch->sent.msg = fetch->recv.msg = 1;
-    fetch->expires = fetch->entry->expires;
-    fetch->resp_time = squid_curtime - fetch->start_time;
-
-    debug(72, 3) ("peerDigestFetchFinish: recv %d bytes in %d secs\n",
-	fetch->recv.bytes, (int) fetch->resp_time);
-    debug(72, 3) ("peerDigestFetchFinish: expires: %ld (%+d), lmt: %ld (%+d)\n",
-	(long int) fetch->expires, (int) (fetch->expires - squid_curtime),
-	(long int) fetch->entry->lastmod, (int) (fetch->entry->lastmod - squid_curtime));
-}
-
-
-static int
-peerDigestSetCBlock(PeerDigest * pd, const char *buf)
-{
-    StoreDigestCBlock cblock;
-    int freed_size = 0;
-    const char *host = strBuf(pd->host);
-
-    xmemcpy(&cblock, buf, sizeof(cblock));
-    /* network -> host conversions */
-    cblock.ver.current = ntohs(cblock.ver.current);
-    cblock.ver.required = ntohs(cblock.ver.required);
-    cblock.capacity = ntohl(cblock.capacity);
-    cblock.count = ntohl(cblock.count);
-    cblock.del_count = ntohl(cblock.del_count);
-    cblock.mask_size = ntohl(cblock.mask_size);
-    debug(72, 2) ("got digest cblock from %s; ver: %d (req: %d)\n",
-	host, (int) cblock.ver.current, (int) cblock.ver.required);
-    debug(72, 2) ("\t size: %d bytes, e-cnt: %d, e-util: %d%%\n",
-	cblock.mask_size, cblock.count,
-	xpercentInt(cblock.count, cblock.capacity));
-    /* check version requirements (both ways) */
-    if (cblock.ver.required > CacheDigestVer.current) {
-	debug(72, 1) ("%s digest requires version %d; have: %d\n",
-	    host, cblock.ver.required, CacheDigestVer.current);
-	return 0;
-    }
-    if (cblock.ver.current < CacheDigestVer.required) {
-	debug(72, 1) ("%s digest is version %d; we require: %d\n",
-	    host, cblock.ver.current, CacheDigestVer.required);
-	return 0;
-    }
-    /* check consistency */
-    if (cblock.ver.required > cblock.ver.current ||
-	cblock.mask_size <= 0 || cblock.capacity <= 0 ||
-	cblock.bits_per_entry <= 0 || cblock.hash_func_count <= 0) {
-	debug(72, 0) ("%s digest cblock is corrupted.\n", host);
-	return 0;
-    }
-    /* check consistency further */
-    if (cblock.mask_size != cacheDigestCalcMaskSize(cblock.capacity, cblock.bits_per_entry)) {
-	debug(72, 0) ("%s digest cblock is corrupted (mask size mismatch: %d ? %d).\n",
-	    host, cblock.mask_size, cacheDigestCalcMaskSize(cblock.capacity, cblock.bits_per_entry));
-	return 0;
-    }
-    /* there are some things we cannot do yet */
-    if (cblock.hash_func_count != CacheDigestHashFuncCount) {
-	debug(72, 0) ("%s digest: unsupported #hash functions: %d ? %d.\n",
-	    host, cblock.hash_func_count, CacheDigestHashFuncCount);
-	return 0;
-    }
-    /*
-     * no cblock bugs below this point
-     */
-    /* check size changes */
-    if (pd->cd && cblock.mask_size != pd->cd->mask_size) {
-	debug(72, 2) ("%s digest changed size: %d -> %d\n",
-	    host, cblock.mask_size, pd->cd->mask_size);
-	freed_size = pd->cd->mask_size;
-	cacheDigestDestroy(pd->cd);
-	pd->cd = NULL;
-    }
-    if (!pd->cd) {
-	debug(72, 2) ("creating %s digest; size: %d (%+d) bytes\n",
-	    host, cblock.mask_size, (int) (cblock.mask_size - freed_size));
-	pd->cd = cacheDigestCreate(cblock.capacity, cblock.bits_per_entry);
-	if (cblock.mask_size >= freed_size)
-	    kb_incr(&statCounter.cd.memory, cblock.mask_size - freed_size);
-    }
-    assert(pd->cd);
-    /* these assignments leave us in an inconsistent state until we finish reading the digest */
-    pd->cd->count = cblock.count;
-    pd->cd->del_count = cblock.del_count;
-    return 1;
-}
-
-static int
-peerDigestUseful(const PeerDigest * pd)
-{
-    /* TODO: we should calculate the prob of a false hit instead of bit util */
-    const int bit_util = cacheDigestBitUtil(pd->cd);
-    if (bit_util > 65) {
-	debug(72, 0) ("Warning: %s peer digest has too many bits on (%d%%).\n",
-	    strBuf(pd->host), bit_util);
-	return 0;
-    }
-    return 1;
-}
-
-static int
-saneDiff(time_t diff)
-{
-    return abs(diff) > squid_curtime / 2 ? 0 : diff;
-}
-
-void
-peerDigestStatsReport(const PeerDigest * pd, StoreEntry * e)
-{
-#define f2s(flag) (pd->flags.flag ? "yes" : "no")
-#define appendTime(tm) storeAppendPrintf(e, "%s\t %10ld\t %+d\t %+d\n", \
-    ""#tm, (long int)pd->times.tm, \
-    saneDiff(pd->times.tm - squid_curtime), \
-    saneDiff(pd->times.tm - pd->times.initialized))
-
-    const char *host = pd ? strBuf(pd->host) : NULL;
-    assert(pd);
-
-    storeAppendPrintf(e, "\npeer digest from %s\n", host);
-
-    cacheDigestGuessStatsReport(&pd->stats.guess, e, host);
-
-    storeAppendPrintf(e, "\nevent\t timestamp\t secs from now\t secs from init\n");
-    appendTime(initialized);
-    appendTime(needed);
-    appendTime(requested);
-    appendTime(received);
-    appendTime(next_check);
-
-    storeAppendPrintf(e, "peer digest state:\n");
-    storeAppendPrintf(e, "\tneeded: %3s, usable: %3s, requested: %3s\n",
-	f2s(needed), f2s(usable), f2s(requested));
-    storeAppendPrintf(e, "\n\tlast retry delay: %d secs\n",
-	(int) pd->times.retry_delay);
-    storeAppendPrintf(e, "\tlast request response time: %d secs\n",
-	(int) pd->times.req_delay);
-    storeAppendPrintf(e, "\tlast request result: %s\n",
-	pd->req_result ? pd->req_result : "(none)");
-
-    storeAppendPrintf(e, "\npeer digest traffic:\n");
-    storeAppendPrintf(e, "\trequests sent: %d, volume: %d KB\n",
-	pd->stats.sent.msgs, (int) pd->stats.sent.kbytes.kb);
-    storeAppendPrintf(e, "\treplies recv:  %d, volume: %d KB\n",
-	pd->stats.recv.msgs, (int) pd->stats.recv.kbytes.kb);
-
-    storeAppendPrintf(e, "\npeer digest structure:\n");
-    if (pd->cd)
-	cacheDigestReport(pd->cd, host, e);
-    else
-	storeAppendPrintf(e, "\tno in-memory copy\n");
-}
-
-#endif
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/peer_digest.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,982 @@
+
+/*
+ * $Id: peer_digest.cc,v 1.1.2.1 2002/10/11 15:40:58 rbcollins Exp $
+ *
+ * DEBUG: section 72    Peer Digest Routines
+ * AUTHOR: Alex Rousskov
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "Store.h"
+
+#if USE_CACHE_DIGESTS
+
+#include "StoreClient.h"
+
+/* local types */
+
+/* local prototypes */
+static time_t peerDigestIncDelay(const PeerDigest * pd);
+static time_t peerDigestNewDelay(const StoreEntry * e);
+static void peerDigestSetCheck(PeerDigest * pd, time_t delay);
+static void peerDigestClean(PeerDigest *);
+static EVH peerDigestCheck;
+static void peerDigestRequest(PeerDigest * pd);
+static STCB peerDigestHandleReply;
+static int peerDigestFetchReply(void *, char *, ssize_t);
+static int peerDigestSwapInHeaders(void *, char *, ssize_t);
+static int peerDigestSwapInCBlock(void *, char *, ssize_t);
+static int peerDigestSwapInMask(void *, char *, ssize_t);
+static int peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const char *step_name);
+static void peerDigestFetchStop(DigestFetchState * fetch, char *buf, const char *reason);
+static void peerDigestFetchAbort(DigestFetchState * fetch, char *buf, const char *reason);
+static void peerDigestReqFinish(DigestFetchState * fetch, char *buf, int, int, int, const char *reason, int err);
+static void peerDigestPDFinish(DigestFetchState * fetch, int pcb_valid, int err);
+static void peerDigestFetchFinish(DigestFetchState * fetch, int err);
+static void peerDigestFetchSetStats(DigestFetchState * fetch);
+static int peerDigestSetCBlock(PeerDigest * pd, const char *buf);
+static int peerDigestUseful(const PeerDigest * pd);
+
+
+/* local constants */
+
+#define StoreDigestCBlockSize sizeof(StoreDigestCBlock)
+
+/* min interval for requesting digests from a given peer */
+static const time_t PeerDigestReqMinGap = 5 * 60;	/* seconds */
+/* min interval for requesting digests (cumulative request stream) */
+static const time_t GlobDigestReqMinGap = 1 * 60;	/* seconds */
+
+/* local vars */
+
+static time_t pd_last_req_time = 0;	/* last call to Check */
+
+/* initialize peer digest */
+static void
+peerDigestInit(PeerDigest * pd, peer * p)
+{
+    assert(pd && p);
+
+    memset(pd, 0, sizeof(*pd));
+    pd->peer = p;
+    /* if peer disappears, we will know it's name */
+    stringInit(&pd->host, p->host);
+
+    pd->times.initialized = squid_curtime;
+}
+
+static void
+peerDigestClean(PeerDigest * pd)
+{
+    assert(pd);
+    if (pd->cd)
+	cacheDigestDestroy(pd->cd);
+    stringClean(&pd->host);
+}
+
+CBDATA_TYPE(PeerDigest);
+
+/* allocate new peer digest, call Init, and lock everything */
+PeerDigest *
+peerDigestCreate(peer * p)
+{
+    PeerDigest *pd;
+    assert(p);
+
+    CBDATA_INIT_TYPE(PeerDigest);
+    pd = cbdataAlloc(PeerDigest);
+    peerDigestInit(pd, p);
+
+    /* XXX This does not look right, and the same thing again in the caller */
+    return cbdataReference(pd);
+}
+
+/* call Clean and free/unlock everything */
+static void
+peerDigestDestroy(PeerDigest * pd)
+{
+    peer *p;
+    assert(pd);
+
+    /* inform peer (if any) that we are gone */
+    if (cbdataReferenceValidDone(pd->peer, (void **) &p))
+	peerNoteDigestGone(p);
+
+    peerDigestClean(pd);
+    cbdataFree(pd);
+}
+
+/* called by peer to indicate that somebody actually needs this digest */
+void
+peerDigestNeeded(PeerDigest * pd)
+{
+    assert(pd);
+    assert(!pd->flags.needed);
+    assert(!pd->cd);
+
+    pd->flags.needed = 1;
+    pd->times.needed = squid_curtime;
+    peerDigestSetCheck(pd, 0);	/* check asap */
+}
+
+/* currently we do not have a reason to disable without destroying */
+#if FUTURE_CODE
+/* disables peer for good */
+static void
+peerDigestDisable(PeerDigest * pd)
+{
+    debug(72, 2) ("peerDigestDisable: peer %s disabled for good\n",
+	strBuf(pd->host));
+    pd->times.disabled = squid_curtime;
+    pd->times.next_check = -1;	/* never */
+    pd->flags.usable = 0;
+
+    if (pd->cd) {
+	cacheDigestDestroy(pd->cd);
+	pd->cd = NULL;
+    }
+    /* we do not destroy the pd itself to preserve its "history" and stats */
+}
+#endif
+
+/* increment retry delay [after an unsuccessful attempt] */
+static time_t
+peerDigestIncDelay(const PeerDigest * pd)
+{
+    assert(pd);
+    return pd->times.retry_delay > 0 ?
+	2 * pd->times.retry_delay :	/* exponential backoff */
+	PeerDigestReqMinGap;	/* minimal delay */
+}
+
+/* artificially increases Expires: setting to avoid race conditions 
+ * returns the delay till that [increased] expiration time */
+static time_t
+peerDigestNewDelay(const StoreEntry * e)
+{
+    assert(e);
+    if (e->expires > 0)
+	return e->expires + PeerDigestReqMinGap - squid_curtime;
+    return PeerDigestReqMinGap;
+}
+
+/* registers next digest verification */
+static void
+peerDigestSetCheck(PeerDigest * pd, time_t delay)
+{
+    eventAdd("peerDigestCheck", peerDigestCheck, pd, (double) delay, 1);
+    pd->times.next_check = squid_curtime + delay;
+    debug(72, 3) ("peerDigestSetCheck: will check peer %s in %d secs\n",
+	strBuf(pd->host), (int) delay);
+}
+
+/*
+ * called when peer is about to disappear or have already disappeared
+ */
+void
+peerDigestNotePeerGone(PeerDigest * pd)
+{
+    if (pd->flags.requested) {
+	debug(72, 2) ("peerDigest: peer %s gone, will destroy after fetch.\n",
+	    strBuf(pd->host));
+	/* do nothing now, the fetching chain will notice and take action */
+    } else {
+	debug(72, 2) ("peerDigest: peer %s is gone, destroying now.\n",
+	    strBuf(pd->host));
+	peerDigestDestroy(pd);
+    }
+}
+
+/* callback for eventAdd() (with peer digest locked)
+ * request new digest if our copy is too old or if we lack one; 
+ * schedule next check otherwise */
+static void
+peerDigestCheck(void *data)
+{
+    PeerDigest *pd = (PeerDigest *)data;
+    time_t req_time;
+
+    assert(!pd->flags.requested);
+
+    pd->times.next_check = 0;	/* unknown */
+
+    if (!cbdataReferenceValid(pd->peer)) {
+	peerDigestNotePeerGone(pd);
+	return;
+    }
+    debug(72, 3) ("peerDigestCheck: peer %s:%d\n", pd->peer->host, pd->peer->http_port);
+    debug(72, 3) ("peerDigestCheck: time: %ld, last received: %ld (%+d)\n",
+	(long int) squid_curtime, (long int) pd->times.received, (int) (squid_curtime - pd->times.received));
+
+    /* decide when we should send the request:
+     * request now unless too close to other requests */
+    req_time = squid_curtime;
+
+    /* per-peer limit */
+    if (req_time - pd->times.received < PeerDigestReqMinGap) {
+	debug(72, 2) ("peerDigestCheck: %s, avoiding close peer requests (%d < %d secs).\n",
+	    strBuf(pd->host), (int) (req_time - pd->times.received),
+	    (int) PeerDigestReqMinGap);
+	req_time = pd->times.received + PeerDigestReqMinGap;
+    }
+    /* global limit */
+    if (req_time - pd_last_req_time < GlobDigestReqMinGap) {
+	debug(72, 2) ("peerDigestCheck: %s, avoiding close requests (%d < %d secs).\n",
+	    strBuf(pd->host), (int) (req_time - pd_last_req_time),
+	    (int) GlobDigestReqMinGap);
+	req_time = pd_last_req_time + GlobDigestReqMinGap;
+    }
+    if (req_time <= squid_curtime)
+	peerDigestRequest(pd);	/* will set pd->flags.requested */
+    else
+	peerDigestSetCheck(pd, req_time - squid_curtime);
+}
+
+CBDATA_TYPE(DigestFetchState);
+
+/* ask store for a digest */
+static void
+peerDigestRequest(PeerDigest * pd)
+{
+    peer *p = pd->peer;
+    StoreEntry *e, *old_e;
+    char *url;
+    const cache_key *key;
+    request_t *req;
+    DigestFetchState *fetch = NULL;
+    StoreIOBuffer tempBuffer = EMPTYIOBUFFER;
+
+    pd->req_result = NULL;
+    pd->flags.requested = 1;
+
+    /* compute future request components */
+    if (p->digest_url)
+	url = xstrdup(p->digest_url);
+    else
+	url = internalRemoteUri(p->host, p->http_port,
+	    "/squid-internal-periodic/", StoreDigestFileName);
+
+    req = urlParse(METHOD_GET, url);
+    assert(req);
+    key = storeKeyPublicByRequest(req);
+    debug(72, 2) ("peerDigestRequest: %s key: %s\n", url, storeKeyText(key));
+
+    /* add custom headers */
+    assert(!req->header.len);
+    httpHeaderPutStr(&req->header, HDR_ACCEPT, StoreDigestMimeStr);
+    httpHeaderPutStr(&req->header, HDR_ACCEPT, "text/html");
+    if (p->login)
+	xstrncpy(req->login, p->login, MAX_LOGIN_SZ);
+    /* create fetch state structure */
+    CBDATA_INIT_TYPE(DigestFetchState);
+    fetch = cbdataAlloc(DigestFetchState);
+    fetch->request = requestLink(req);
+    fetch->pd = cbdataReference(pd);
+    fetch->offset = 0;
+    fetch->state = DIGEST_READ_REPLY;
+
+    /* update timestamps */
+    fetch->start_time = squid_curtime;
+    pd->times.requested = squid_curtime;
+    pd_last_req_time = squid_curtime;
+
+    req->flags.cachable = 1;
+    /* the rest is based on clientProcessExpired() */
+    req->flags.refresh = 1;
+    old_e = fetch->old_entry = storeGet(key);
+    if (old_e) {
+	debug(72, 5) ("peerDigestRequest: found old entry\n");
+	storeLockObject(old_e);
+	storeCreateMemObject(old_e, url, url);
+	fetch->old_sc = storeClientListAdd(old_e, fetch);
+    }
+    e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method);
+    assert(EBIT_TEST(e->flags, KEY_PRIVATE));
+    fetch->sc = storeClientListAdd(e, fetch);
+    /* set lastmod to trigger IMS request if possible */
+    if (old_e)
+	e->lastmod = old_e->lastmod;
+
+    /* push towards peer cache */
+    debug(72, 3) ("peerDigestRequest: forwarding to fwdStart...\n");
+    fwdStart(-1, e, req);
+    tempBuffer.offset = 0;
+    tempBuffer.length =  SM_PAGE_SIZE;
+    tempBuffer.data = fetch->buf;
+    storeClientCopy(fetch->sc, e, tempBuffer,
+	peerDigestHandleReply, fetch);
+}
+
+
+/* Handle the data copying .. */
+
+/*
+ * This routine handles the copy data and then redirects the
+ * copy to a bunch of subfunctions depending upon the copy state.
+ * It also tracks the buffer offset and "seen", since I'm actually
+ * not interested in rewriting everything to suit my little idea.
+ */
+static void
+peerDigestHandleReply(void *data, StoreIOBuffer recievedData)
+{
+    DigestFetchState *fetch = (DigestFetchState *)data;
+    PeerDigest *pd = fetch->pd;
+    int retsize = -1;
+    digest_read_state_t prevstate;
+    int newsize;
+
+    assert(pd && recievedData.data);
+    /* The existing code assumes that the recieved pointer is
+     * where we asked the data to be put
+     */
+    assert(fetch->buf + fetch->bufofs == recievedData.data);
+
+    /* Update the buffer size */
+    fetch->bufofs += recievedData.length;
+
+    assert(fetch->bufofs <= SM_PAGE_SIZE);
+
+    /* If we've fetched enough, return */
+    if (peerDigestFetchedEnough(fetch, fetch->buf, fetch->bufofs, "peerDigestHandleReply"))
+	return;
+
+    /* Call the right function based on the state */
+    /* (Those functions will update the state if needed) */
+
+    /* Give us a temporary reference. Some of the calls we make may
+     * try to destroy the fetch structure, and we like to know if they
+     * do
+     */
+    fetch = cbdataReference(fetch);
+
+    /* Repeat this loop until we're out of data OR the state changes */
+    /* (So keep going if the state has changed and we still have data */
+    do {
+	prevstate = fetch->state;
+	switch (fetch->state) {
+	case DIGEST_READ_REPLY:
+	    retsize = peerDigestFetchReply(fetch, fetch->buf, fetch->bufofs);
+	    break;
+	case DIGEST_READ_HEADERS:
+	    retsize = peerDigestSwapInHeaders(fetch, fetch->buf, fetch->bufofs);
+	    break;
+	case DIGEST_READ_CBLOCK:
+	    retsize = peerDigestSwapInCBlock(fetch, fetch->buf, fetch->bufofs);
+	    break;
+	case DIGEST_READ_MASK:
+	    retsize = peerDigestSwapInMask(fetch, fetch->buf, fetch->bufofs);
+	    break;
+	case DIGEST_READ_NONE:
+	    break;
+	case DIGEST_READ_DONE:
+	    goto finish;
+	    break;
+	default:
+	    fatal("Bad digest transfer mode!\n");
+	}
+
+	if (retsize < 0)
+	    goto finish;
+	/*
+	 * The returned size indicates how much of the buffer was read -
+	 * so move the remainder of the buffer to the beginning
+	 * and update the bufofs / bufsize
+	 */
+	newsize = fetch->bufofs - retsize;
+	xmemmove(fetch->buf, fetch->buf + retsize, fetch->bufofs - newsize);
+	fetch->bufofs = newsize;
+
+    } while (cbdataReferenceValid(fetch) && prevstate != fetch->state && fetch->bufofs > 0);
+
+    /* Update the copy offset */
+    fetch->offset += recievedData.length;
+
+    /* Schedule another copy */
+    if (cbdataReferenceValid(fetch)) {
+	StoreIOBuffer tempBuffer = EMPTYIOBUFFER;
+	tempBuffer.offset = fetch->offset;
+	tempBuffer.length = SM_PAGE_SIZE - fetch->bufofs;
+	tempBuffer.data = fetch->buf + fetch->bufofs;
+	storeClientCopy(fetch->sc, fetch->entry, tempBuffer,
+	    peerDigestHandleReply, fetch);
+    }
+  finish:
+    /* Get rid of our reference, we've finished with it for now */
+    cbdataReferenceDone(fetch);
+}
+
+
+
+/* wait for full http headers to be received then parse them */
+/*
+ * This routine handles parsing the reply line.
+ * If the reply line indicates an OK, the same data is thrown
+ * to SwapInHeaders(). If the reply line is a NOT_MODIFIED,
+ * we simply stop parsing.
+ */
+static int
+peerDigestFetchReply(void *data, char *buf, ssize_t size)
+{
+    DigestFetchState *fetch = (DigestFetchState *)data;
+    PeerDigest *pd = fetch->pd;
+    size_t hdr_size;
+    assert(pd && buf);
+    assert(!fetch->offset);
+
+    assert(fetch->state == DIGEST_READ_REPLY);
+    if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestFetchReply"))
+	return -1;
+
+    if ((hdr_size = headersEnd(buf, size))) {
+	http_status status;
+	HttpReply *reply = fetch->entry->mem_obj->reply;
+	assert(reply);
+	httpReplyParse(reply, buf, hdr_size);
+	status = reply->sline.status;
+	debug(72, 3) ("peerDigestFetchReply: %s status: %d, expires: %ld (%+d)\n",
+	    strBuf(pd->host), status,
+	    (long int) reply->expires, (int) (reply->expires - squid_curtime));
+
+	/* this "if" is based on clientHandleIMSReply() */
+	if (status == HTTP_NOT_MODIFIED) {
+	    request_t *r = NULL;
+	    /* our old entry is fine */
+	    assert(fetch->old_entry);
+	    if (!fetch->old_entry->mem_obj->request)
+		fetch->old_entry->mem_obj->request = r =
+		    requestLink(fetch->entry->mem_obj->request);
+	    assert(fetch->old_entry->mem_obj->request);
+	    httpReplyUpdateOnNotModified(fetch->old_entry->mem_obj->reply, reply);
+	    storeTimestampsSet(fetch->old_entry);
+	    /* get rid of 304 reply */
+	    storeUnregister(fetch->sc, fetch->entry, fetch);
+	    storeUnlockObject(fetch->entry);
+	    fetch->entry = fetch->old_entry;
+	    fetch->old_entry = NULL;
+	    /* preserve request -- we need its size to update counters */
+	    /* requestUnlink(r); */
+	    /* fetch->entry->mem_obj->request = NULL; */
+	} else if (status == HTTP_OK) {
+	    /* get rid of old entry if any */
+	    if (fetch->old_entry) {
+		debug(72, 3) ("peerDigestFetchReply: got new digest, releasing old one\n");
+		storeUnregister(fetch->old_sc, fetch->old_entry, fetch);
+		storeReleaseRequest(fetch->old_entry);
+		storeUnlockObject(fetch->old_entry);
+		fetch->old_entry = NULL;
+	    }
+	} else {
+	    /* some kind of a bug */
+	    peerDigestFetchAbort(fetch, buf, httpStatusLineReason(&reply->sline));
+	    return -1;		/* XXX -1 will abort stuff in ReadReply! */
+	}
+	/* must have a ready-to-use store entry if we got here */
+	/* can we stay with the old in-memory digest? */
+	if (status == HTTP_NOT_MODIFIED && fetch->pd->cd) {
+	    peerDigestFetchStop(fetch, buf, "Not modified");
+	    fetch->state = DIGEST_READ_DONE;
+	} else {
+	    fetch->state = DIGEST_READ_HEADERS;
+	}
+    } else {
+	/* need more data, do we have space? */
+	if (size >= SM_PAGE_SIZE)
+	    peerDigestFetchAbort(fetch, buf, "reply header too big");
+    }
+
+    /* We don't want to actually ack that we've handled anything,
+     * otherwise SwapInHeaders() won't get the reply line .. */
+    return 0;
+}
+
+/* fetch headers from disk, pass on to SwapInCBlock */
+static int
+peerDigestSwapInHeaders(void *data, char *buf, ssize_t size)
+{
+    DigestFetchState *fetch = (DigestFetchState *)data;
+    size_t hdr_size;
+
+    assert(fetch->state == DIGEST_READ_HEADERS);
+    if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInHeaders"))
+	return -1;
+
+    assert(!fetch->offset);
+    if ((hdr_size = headersEnd(buf, size))) {
+	assert(fetch->entry->mem_obj->reply);
+	if (!fetch->entry->mem_obj->reply->sline.status)
+	    httpReplyParse(fetch->entry->mem_obj->reply, buf, hdr_size);
+	if (fetch->entry->mem_obj->reply->sline.status != HTTP_OK) {
+	    debug(72, 1) ("peerDigestSwapInHeaders: %s status %d got cached!\n",
+		strBuf(fetch->pd->host), fetch->entry->mem_obj->reply->sline.status);
+	    peerDigestFetchAbort(fetch, buf, "internal status error");
+	    return -1;
+	}
+	fetch->state = DIGEST_READ_CBLOCK;
+	return hdr_size;	/* Say how much data we read */
+    } else {
+	/* need more data, do we have space? */
+	if (size >= SM_PAGE_SIZE) {
+	    peerDigestFetchAbort(fetch, buf, "stored header too big");
+	    return -1;
+	} else {
+	    return 0;		/* We need to read more to parse .. */
+	}
+    }
+    fatal("peerDigestSwapInHeaders() - shouldn't get here!\n");
+}
+
+static int
+peerDigestSwapInCBlock(void *data, char *buf, ssize_t size)
+{
+    DigestFetchState *fetch = (DigestFetchState *)data;
+
+    assert(fetch->state == DIGEST_READ_CBLOCK);
+    if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInCBlock"))
+	return -1;
+
+    if (size >= (ssize_t)StoreDigestCBlockSize) {
+	PeerDigest *pd = fetch->pd;
+	HttpReply *rep = fetch->entry->mem_obj->reply;
+
+	assert(pd && rep);
+	if (peerDigestSetCBlock(pd, buf)) {
+	    /* XXX: soon we will have variable header size */
+	    /* switch to CD buffer and fetch digest guts */
+	    buf = NULL;
+	    assert(pd->cd->mask);
+	    fetch->state = DIGEST_READ_MASK;
+	    return StoreDigestCBlockSize;
+	} else {
+	    peerDigestFetchAbort(fetch, buf, "invalid digest cblock");
+	    return -1;
+	}
+    } else {
+	/* need more data, do we have space? */
+	if (size >= SM_PAGE_SIZE) {
+	    peerDigestFetchAbort(fetch, buf, "digest cblock too big");
+	    return -1;
+	} else {
+	    return 0;		/* We need more data */
+	}
+    }
+    fatal("peerDigestSwapInCBlock(): shouldn't get here!\n");
+}
+
+static int
+peerDigestSwapInMask(void *data, char *buf, ssize_t size)
+{
+    DigestFetchState *fetch = (DigestFetchState *)data;
+    PeerDigest *pd;
+
+    pd = fetch->pd;
+    assert(pd->cd && pd->cd->mask);
+
+    /*
+     * NOTENOTENOTENOTENOTE: buf doesn't point to pd->cd->mask anymore!
+     * we need to do the copy ourselves!
+     */
+    xmemcpy(pd->cd->mask + fetch->mask_offset, buf, size);
+
+    /* NOTE! buf points to the middle of pd->cd->mask! */
+    if (peerDigestFetchedEnough(fetch, NULL, size, "peerDigestSwapInMask"))
+	return -1;
+
+    fetch->mask_offset += size;
+    if (fetch->mask_offset >= (off_t)pd->cd->mask_size) {
+	debug(72, 2) ("peerDigestSwapInMask: Done! Got %d, expected %d\n",
+	    fetch->mask_offset, pd->cd->mask_size);
+	assert(fetch->mask_offset == (off_t)pd->cd->mask_size);
+	assert(peerDigestFetchedEnough(fetch, NULL, 0, "peerDigestSwapInMask"));
+	return -1;		/* XXX! */
+    } else {
+	/* We always read everything, so return so */
+	return size;
+    }
+    fatal("peerDigestSwapInMask(): shouldn't get here!\n");
+}
+
+static int
+peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const char *step_name)
+{
+    PeerDigest *pd = NULL;
+    const char *host = "";	/* peer host */
+    const char *reason = NULL;	/* reason for completion */
+    const char *no_bug = NULL;	/* successful completion if set */
+    const int pdcb_valid = cbdataReferenceValid(fetch->pd);
+    const int pcb_valid = cbdataReferenceValid(fetch->pd->peer);
+
+    /* test possible exiting conditions (the same for most steps!)
+     * cases marked with '?!' should not happen */
+
+    if (!reason) {
+	if (!(pd = fetch->pd))
+	    reason = "peer digest disappeared?!";
+#if DONT			/* WHY NOT? /HNO */
+	else if (!cbdataReferenceValid(pd))
+	    reason = "invalidated peer digest?!";
+#endif
+	else
+	    host = strBuf(pd->host);
+    }
+    debug(72, 6) ("%s: peer %s, offset: %d size: %d.\n",
+	step_name, host,
+	fetch->offset, size);
+
+    /* continue checking (with pd and host known and valid) */
+    if (!reason) {
+	if (!cbdataReferenceValid(pd->peer))
+	    reason = "peer disappeared";
+	else if (size < 0)
+	    reason = "swap failure";
+	else if (!fetch->entry)
+	    reason = "swap aborted?!";
+	else if (EBIT_TEST(fetch->entry->flags, ENTRY_ABORTED))
+	    reason = "swap aborted";
+    }
+    /* continue checking (maybe-successful eof case) */
+    if (!reason && !size) {
+	if (!pd->cd)
+	    reason = "null digest?!";
+	else if (fetch->mask_offset != (off_t)pd->cd->mask_size)
+	    reason = "premature end of digest?!";
+	else if (!peerDigestUseful(pd))
+	    reason = "useless digest";
+	else
+	    reason = no_bug = "success";
+    }
+    /* finish if we have a reason */
+    if (reason) {
+	const int level = strstr(reason, "?!") ? 1 : 3;
+	debug(72, level) ("%s: peer %s, exiting after '%s'\n",
+	    step_name, host, reason);
+	peerDigestReqFinish(fetch, buf,
+	    1, pdcb_valid, pcb_valid, reason, !no_bug);
+    } else {
+	/* paranoid check */
+	assert(pdcb_valid && pcb_valid);
+    }
+    return reason != NULL;
+}
+
+/* call this when all callback data is valid and fetch must be stopped but
+ * no error has occurred (e.g. we received 304 reply and reuse old digest) */
+static void
+peerDigestFetchStop(DigestFetchState * fetch, char *buf, const char *reason)
+{
+    assert(reason);
+    debug(72, 2) ("peerDigestFetchStop: peer %s, reason: %s\n",
+	strBuf(fetch->pd->host), reason);
+    peerDigestReqFinish(fetch, buf, 1, 1, 1, reason, 0);
+}
+
+/* call this when all callback data is valid but something bad happened */
+static void
+peerDigestFetchAbort(DigestFetchState * fetch, char *buf, const char *reason)
+{
+    assert(reason);
+    debug(72, 2) ("peerDigestFetchAbort: peer %s, reason: %s\n",
+	strBuf(fetch->pd->host), reason);
+    peerDigestReqFinish(fetch, buf, 1, 1, 1, reason, 1);
+}
+
+/* complete the digest transfer, update stats, unlock/release everything */
+static void
+peerDigestReqFinish(DigestFetchState * fetch, char *buf,
+    int fcb_valid, int pdcb_valid, int pcb_valid,
+    const char *reason, int err)
+{
+    assert(reason);
+
+    /* must go before peerDigestPDFinish */
+    if (pdcb_valid) {
+	fetch->pd->flags.requested = 0;
+	fetch->pd->req_result = reason;
+    }
+    /* schedule next check if peer is still out there */
+    if (pcb_valid) {
+	PeerDigest *pd = fetch->pd;
+	if (err) {
+	    pd->times.retry_delay = peerDigestIncDelay(pd);
+	    peerDigestSetCheck(pd, pd->times.retry_delay);
+	} else {
+	    pd->times.retry_delay = 0;
+	    peerDigestSetCheck(pd, peerDigestNewDelay(fetch->entry));
+	}
+    }
+    /* note: order is significant */
+    if (fcb_valid)
+	peerDigestFetchSetStats(fetch);
+    if (pdcb_valid)
+	peerDigestPDFinish(fetch, pcb_valid, err);
+    if (fcb_valid)
+	peerDigestFetchFinish(fetch, err);
+}
+
+
+/* destroys digest if peer disappeared
+ * must be called only when fetch and pd cbdata are valid */
+static void
+peerDigestPDFinish(DigestFetchState * fetch, int pcb_valid, int err)
+{
+    PeerDigest *pd = fetch->pd;
+    const char *host = strBuf(pd->host);
+
+    pd->times.received = squid_curtime;
+    pd->times.req_delay = fetch->resp_time;
+    kb_incr(&pd->stats.sent.kbytes, (size_t) fetch->sent.bytes);
+    kb_incr(&pd->stats.recv.kbytes, (size_t) fetch->recv.bytes);
+    pd->stats.sent.msgs += fetch->sent.msg;
+    pd->stats.recv.msgs += fetch->recv.msg;
+
+    if (err) {
+	debug(72, 1) ("%sdisabling (%s) digest from %s\n",
+	    pcb_valid ? "temporary " : "",
+	    pd->req_result, host);
+
+	if (pd->cd) {
+	    cacheDigestDestroy(pd->cd);
+	    pd->cd = NULL;
+	}
+	pd->flags.usable = 0;
+
+	if (!pcb_valid)
+	    peerDigestNotePeerGone(pd);
+    } else {
+	assert(pcb_valid);
+
+	pd->flags.usable = 1;
+
+	/* XXX: ugly condition, but how? */
+	if (fetch->entry->store_status == STORE_OK)
+	    debug(72, 2) ("re-used old digest from %s\n", host);
+	else
+	    debug(72, 2) ("received valid digest from %s\n", host);
+    }
+    cbdataReferenceDone(fetch->pd);
+}
+
+/* free fetch state structures
+ * must be called only when fetch cbdata is valid */
+static void
+peerDigestFetchFinish(DigestFetchState * fetch, int err)
+{
+    assert(fetch->entry && fetch->request);
+
+    if (fetch->old_entry) {
+	debug(72, 2) ("peerDigestFetchFinish: deleting old entry\n");
+	storeUnregister(fetch->sc, fetch->old_entry, fetch);
+	storeReleaseRequest(fetch->old_entry);
+	storeUnlockObject(fetch->old_entry);
+	fetch->old_entry = NULL;
+    }
+    /* update global stats */
+    kb_incr(&statCounter.cd.kbytes_sent, (size_t) fetch->sent.bytes);
+    kb_incr(&statCounter.cd.kbytes_recv, (size_t) fetch->recv.bytes);
+    statCounter.cd.msgs_sent += fetch->sent.msg;
+    statCounter.cd.msgs_recv += fetch->recv.msg;
+
+    /* unlock everything */
+    storeUnregister(fetch->sc, fetch->entry, fetch);
+    storeUnlockObject(fetch->entry);
+    requestUnlink(fetch->request);
+    fetch->entry = NULL;
+    fetch->request = NULL;
+    assert(fetch->pd == NULL);
+    cbdataFree(fetch);
+}
+
+/* calculate fetch stats after completion */
+static void
+peerDigestFetchSetStats(DigestFetchState * fetch)
+{
+    MemObject *mem;
+    assert(fetch->entry && fetch->request);
+
+    mem = fetch->entry->mem_obj;
+    assert(mem);
+
+    /* XXX: outgoing numbers are not precise */
+    /* XXX: we must distinguish between 304 hits and misses here */
+    fetch->sent.bytes = httpRequestPrefixLen(fetch->request);
+    fetch->recv.bytes = fetch->entry->store_status == STORE_PENDING ?
+	mem->inmem_hi : mem->object_sz;
+    fetch->sent.msg = fetch->recv.msg = 1;
+    fetch->expires = fetch->entry->expires;
+    fetch->resp_time = squid_curtime - fetch->start_time;
+
+    debug(72, 3) ("peerDigestFetchFinish: recv %d bytes in %d secs\n",
+	fetch->recv.bytes, (int) fetch->resp_time);
+    debug(72, 3) ("peerDigestFetchFinish: expires: %ld (%+d), lmt: %ld (%+d)\n",
+	(long int) fetch->expires, (int) (fetch->expires - squid_curtime),
+	(long int) fetch->entry->lastmod, (int) (fetch->entry->lastmod - squid_curtime));
+}
+
+
+static int
+peerDigestSetCBlock(PeerDigest * pd, const char *buf)
+{
+    StoreDigestCBlock cblock;
+    int freed_size = 0;
+    const char *host = strBuf(pd->host);
+
+    xmemcpy(&cblock, buf, sizeof(cblock));
+    /* network -> host conversions */
+    cblock.ver.current = ntohs(cblock.ver.current);
+    cblock.ver.required = ntohs(cblock.ver.required);
+    cblock.capacity = ntohl(cblock.capacity);
+    cblock.count = ntohl(cblock.count);
+    cblock.del_count = ntohl(cblock.del_count);
+    cblock.mask_size = ntohl(cblock.mask_size);
+    debug(72, 2) ("got digest cblock from %s; ver: %d (req: %d)\n",
+	host, (int) cblock.ver.current, (int) cblock.ver.required);
+    debug(72, 2) ("\t size: %d bytes, e-cnt: %d, e-util: %d%%\n",
+	cblock.mask_size, cblock.count,
+	xpercentInt(cblock.count, cblock.capacity));
+    /* check version requirements (both ways) */
+    if (cblock.ver.required > CacheDigestVer.current) {
+	debug(72, 1) ("%s digest requires version %d; have: %d\n",
+	    host, cblock.ver.required, CacheDigestVer.current);
+	return 0;
+    }
+    if (cblock.ver.current < CacheDigestVer.required) {
+	debug(72, 1) ("%s digest is version %d; we require: %d\n",
+	    host, cblock.ver.current, CacheDigestVer.required);
+	return 0;
+    }
+    /* check consistency */
+    if (cblock.ver.required > cblock.ver.current ||
+	cblock.mask_size <= 0 || cblock.capacity <= 0 ||
+	cblock.bits_per_entry <= 0 || cblock.hash_func_count <= 0) {
+	debug(72, 0) ("%s digest cblock is corrupted.\n", host);
+	return 0;
+    }
+    /* check consistency further */
+    if ((size_t)cblock.mask_size != cacheDigestCalcMaskSize(cblock.capacity, cblock.bits_per_entry)) {
+	debug(72, 0) ("%s digest cblock is corrupted (mask size mismatch: %d ? %d).\n",
+	    host, cblock.mask_size, cacheDigestCalcMaskSize(cblock.capacity, cblock.bits_per_entry));
+	return 0;
+    }
+    /* there are some things we cannot do yet */
+    if (cblock.hash_func_count != CacheDigestHashFuncCount) {
+	debug(72, 0) ("%s digest: unsupported #hash functions: %d ? %d.\n",
+	    host, cblock.hash_func_count, CacheDigestHashFuncCount);
+	return 0;
+    }
+    /*
+     * no cblock bugs below this point
+     */
+    /* check size changes */
+    if (pd->cd && cblock.mask_size != (ssize_t)pd->cd->mask_size) {
+	debug(72, 2) ("%s digest changed size: %d -> %d\n",
+	    host, cblock.mask_size, pd->cd->mask_size);
+	freed_size = pd->cd->mask_size;
+	cacheDigestDestroy(pd->cd);
+	pd->cd = NULL;
+    }
+    if (!pd->cd) {
+	debug(72, 2) ("creating %s digest; size: %d (%+d) bytes\n",
+	    host, cblock.mask_size, (int) (cblock.mask_size - freed_size));
+	pd->cd = cacheDigestCreate(cblock.capacity, cblock.bits_per_entry);
+	if (cblock.mask_size >= freed_size)
+	    kb_incr(&statCounter.cd.memory, cblock.mask_size - freed_size);
+    }
+    assert(pd->cd);
+    /* these assignments leave us in an inconsistent state until we finish reading the digest */
+    pd->cd->count = cblock.count;
+    pd->cd->del_count = cblock.del_count;
+    return 1;
+}
+
+static int
+peerDigestUseful(const PeerDigest * pd)
+{
+    /* TODO: we should calculate the prob of a false hit instead of bit util */
+    const int bit_util = cacheDigestBitUtil(pd->cd);
+    if (bit_util > 65) {
+	debug(72, 0) ("Warning: %s peer digest has too many bits on (%d%%).\n",
+	    strBuf(pd->host), bit_util);
+	return 0;
+    }
+    return 1;
+}
+
+static int
+saneDiff(time_t diff)
+{
+    return abs(diff) > squid_curtime / 2 ? 0 : diff;
+}
+
+void
+peerDigestStatsReport(const PeerDigest * pd, StoreEntry * e)
+{
+#define f2s(flag) (pd->flags.flag ? "yes" : "no")
+#define appendTime(tm) storeAppendPrintf(e, "%s\t %10ld\t %+d\t %+d\n", \
+    ""#tm, (long int)pd->times.tm, \
+    saneDiff(pd->times.tm - squid_curtime), \
+    saneDiff(pd->times.tm - pd->times.initialized))
+
+    const char *host = pd ? strBuf(pd->host) : NULL;
+    assert(pd);
+
+    storeAppendPrintf(e, "\npeer digest from %s\n", host);
+
+    cacheDigestGuessStatsReport(&pd->stats.guess, e, host);
+
+    storeAppendPrintf(e, "\nevent\t timestamp\t secs from now\t secs from init\n");
+    appendTime(initialized);
+    appendTime(needed);
+    appendTime(requested);
+    appendTime(received);
+    appendTime(next_check);
+
+    storeAppendPrintf(e, "peer digest state:\n");
+    storeAppendPrintf(e, "\tneeded: %3s, usable: %3s, requested: %3s\n",
+	f2s(needed), f2s(usable), f2s(requested));
+    storeAppendPrintf(e, "\n\tlast retry delay: %d secs\n",
+	(int) pd->times.retry_delay);
+    storeAppendPrintf(e, "\tlast request response time: %d secs\n",
+	(int) pd->times.req_delay);
+    storeAppendPrintf(e, "\tlast request result: %s\n",
+	pd->req_result ? pd->req_result : "(none)");
+
+    storeAppendPrintf(e, "\npeer digest traffic:\n");
+    storeAppendPrintf(e, "\trequests sent: %d, volume: %d KB\n",
+	pd->stats.sent.msgs, (int) pd->stats.sent.kbytes.kb);
+    storeAppendPrintf(e, "\treplies recv:  %d, volume: %d KB\n",
+	pd->stats.recv.msgs, (int) pd->stats.recv.kbytes.kb);
+
+    storeAppendPrintf(e, "\npeer digest structure:\n");
+    if (pd->cd)
+	cacheDigestReport(pd->cd, host, e);
+    else
+	storeAppendPrintf(e, "\tno in-memory copy\n");
+}
+
+#endif
--- squid/src/peer_select.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,667 +0,0 @@
-
-/*
- * $Id: peer_select.c,v 1.16.6.2 2002/10/08 02:25:37 rbcollins Exp $
- *
- * DEBUG: section 44    Peer Selection Algorithm
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-#include "Store.h"
-#include "ICP.h"
-
-const char *hier_strings[] =
-{
-    "NONE",
-    "DIRECT",
-    "SIBLING_HIT",
-    "PARENT_HIT",
-    "DEFAULT_PARENT",
-    "SINGLE_PARENT",
-    "FIRST_UP_PARENT",
-    "FIRST_PARENT_MISS",
-    "CLOSEST_PARENT_MISS",
-    "CLOSEST_PARENT",
-    "CLOSEST_DIRECT",
-    "NO_DIRECT_FAIL",
-    "SOURCE_FASTEST",
-    "ROUNDROBIN_PARENT",
-#if USE_CACHE_DIGESTS
-    "CD_PARENT_HIT",
-    "CD_SIBLING_HIT",
-#endif
-#if USE_CARP
-    "CARP",
-#endif
-    "ANY_PARENT",
-    "INVALID CODE"
-};
-
-static struct {
-    int timeouts;
-} PeerStats;
-
-static const char *DirectStr[] =
-{
-    "DIRECT_UNKNOWN",
-    "DIRECT_NO",
-    "DIRECT_MAYBE",
-    "DIRECT_YES"
-};
-
-static void peerSelectFoo(ps_state *);
-static void peerPingTimeout(void *data);
-static void peerSelectCallback(ps_state * psstate);
-static IRCB peerHandlePingReply;
-static void peerSelectStateFree(ps_state * psstate);
-static void peerIcpParentMiss(peer *, icp_common_t *, ps_state *);
-#if USE_HTCP
-static void peerHtcpParentMiss(peer *, htcpReplyData *, ps_state *);
-static void peerHandleHtcpReply(peer *, peer_t, htcpReplyData *, void *);
-#endif
-static int peerCheckNetdbDirect(ps_state * psstate);
-static void peerGetSomeNeighbor(ps_state *);
-static void peerGetSomeNeighborReplies(ps_state *);
-static void peerGetSomeDirect(ps_state *);
-static void peerGetSomeParent(ps_state *);
-static void peerGetAllParents(ps_state *);
-static void peerAddFwdServer(FwdServer **, peer *, hier_code);
-
-static void
-peerSelectStateFree(ps_state * psstate)
-{
-    if (psstate->acl_checklist) {
-	debug(44, 1) ("calling aclChecklistFree() from peerSelectStateFree\n");
-	aclChecklistFree(psstate->acl_checklist);
-    }
-    requestUnlink(psstate->request);
-    psstate->request = NULL;
-    if (psstate->entry) {
-	assert(psstate->entry->ping_status != PING_WAITING);
-	storeUnlockObject(psstate->entry);
-	psstate->entry = NULL;
-    }
-    cbdataFree(psstate);
-}
-
-static int
-peerSelectIcpPing(request_t * request, int direct, StoreEntry * entry)
-{
-    int n;
-    assert(entry);
-    assert(entry->ping_status == PING_NONE);
-    assert(direct != DIRECT_YES);
-    debug(44, 3) ("peerSelectIcpPing: %s\n", storeUrl(entry));
-    if (!request->flags.hierarchical && direct != DIRECT_NO)
-	return 0;
-    if (EBIT_TEST(entry->flags, KEY_PRIVATE) && !neighbors_do_private_keys)
-	if (direct != DIRECT_NO)
-	    return 0;
-    n = neighborsCount(request);
-    debug(44, 3) ("peerSelectIcpPing: counted %d neighbors\n", n);
-    return n;
-}
-
-
-void
-peerSelect(request_t * request,
-    StoreEntry * entry,
-    PSC * callback,
-    void *callback_data)
-{
-    ps_state *psstate;
-    if (entry)
-	debug(44, 3) ("peerSelect: %s\n", storeUrl(entry));
-    else
-	debug(44, 3) ("peerSelect: %s\n", RequestMethodStr[request->method]);
-    psstate = cbdataAlloc(ps_state);
-    psstate->request = requestLink(request);
-    psstate->entry = entry;
-    psstate->callback = callback;
-    psstate->callback_data = cbdataReference(callback_data);
-    psstate->direct = DIRECT_UNKNOWN;
-#if USE_CACHE_DIGESTS
-    request->hier.peer_select_start = current_time;
-#endif
-    if (psstate->entry)
-	storeLockObject(psstate->entry);
-    peerSelectFoo(psstate);
-}
-
-static void
-peerCheckNeverDirectDone(int answer, void *data)
-{
-    ps_state *psstate = data;
-    psstate->acl_checklist = NULL;
-    debug(44, 3) ("peerCheckNeverDirectDone: %d\n", answer);
-    psstate->never_direct = answer ? 1 : -1;
-    peerSelectFoo(psstate);
-}
-
-static void
-peerCheckAlwaysDirectDone(int answer, void *data)
-{
-    ps_state *psstate = data;
-    psstate->acl_checklist = NULL;
-    debug(44, 3) ("peerCheckAlwaysDirectDone: %d\n", answer);
-    psstate->always_direct = answer ? 1 : -1;
-    peerSelectFoo(psstate);
-}
-
-static void
-peerSelectCallback(ps_state * psstate)
-{
-    StoreEntry *entry = psstate->entry;
-    FwdServer *fs = psstate->servers;
-    PSC *callback;
-    void *cbdata;
-    if (entry) {
-	debug(44, 3) ("peerSelectCallback: %s\n", storeUrl(entry));
-	if (entry->ping_status == PING_WAITING)
-	    eventDelete(peerPingTimeout, psstate);
-	entry->ping_status = PING_DONE;
-    }
-    if (fs == NULL) {
-	debug(44, 1) ("Failed to select source for '%s'\n", storeUrl(entry));
-	debug(44, 1) ("  always_direct = %d\n", psstate->always_direct);
-	debug(44, 1) ("   never_direct = %d\n", psstate->never_direct);
-	debug(44, 1) ("       timedout = %d\n", psstate->ping.timedout);
-    }
-    psstate->ping.stop = current_time;
-    psstate->request->hier.ping = psstate->ping;
-    callback = psstate->callback;
-    psstate->callback = NULL;
-    if (cbdataReferenceValidDone(psstate->callback_data, &cbdata)) {
-	psstate->servers = NULL;
-	callback(fs, cbdata);
-    }
-    peerSelectStateFree(psstate);
-}
-
-static int
-peerCheckNetdbDirect(ps_state * psstate)
-{
-    peer *p;
-    int myrtt;
-    int myhops;
-    if (psstate->direct == DIRECT_NO)
-	return 0;
-    myrtt = netdbHostRtt(psstate->request->host);
-    debug(44, 3) ("peerCheckNetdbDirect: MY RTT = %d msec\n", myrtt);
-    debug(44, 3) ("peerCheckNetdbDirect: minimum_direct_rtt = %d msec\n",
-	Config.minDirectRtt);
-    if (myrtt && myrtt <= Config.minDirectRtt)
-	return 1;
-    myhops = netdbHostHops(psstate->request->host);
-    debug(44, 3) ("peerCheckNetdbDirect: MY hops = %d\n", myhops);
-    debug(44, 3) ("peerCheckNetdbDirect: minimum_direct_hops = %d\n",
-	Config.minDirectHops);
-    if (myhops && myhops <= Config.minDirectHops)
-	return 1;
-    p = whichPeer(&psstate->closest_parent_miss);
-    if (p == NULL)
-	return 0;
-    debug(44, 3) ("peerCheckNetdbDirect: closest_parent_miss RTT = %d msec\n",
-	psstate->ping.p_rtt);
-    if (myrtt && myrtt <= psstate->ping.p_rtt)
-	return 1;
-    return 0;
-}
-
-static void
-peerSelectFoo(ps_state * ps)
-{
-    StoreEntry *entry = ps->entry;
-    request_t *request = ps->request;
-    debug(44, 3) ("peerSelectFoo: '%s %s'\n",
-	RequestMethodStr[request->method],
-	request->host);
-    if (ps->direct == DIRECT_UNKNOWN) {
-	if (ps->always_direct == 0 && Config.accessList.AlwaysDirect) {
-	    ps->acl_checklist = aclChecklistCreate(
-		Config.accessList.AlwaysDirect,
-		request,
-		NULL);		/* ident */
-	    aclNBCheck(ps->acl_checklist,
-		peerCheckAlwaysDirectDone,
-		ps);
-	    return;
-	} else if (ps->always_direct > 0) {
-	    ps->direct = DIRECT_YES;
-	} else if (ps->never_direct == 0 && Config.accessList.NeverDirect) {
-	    ps->acl_checklist = aclChecklistCreate(
-		Config.accessList.NeverDirect,
-		request,
-		NULL);		/* ident */
-	    aclNBCheck(ps->acl_checklist,
-		peerCheckNeverDirectDone,
-		ps);
-	    return;
-	} else if (ps->never_direct > 0) {
-	    ps->direct = DIRECT_NO;
-	} else if (request->flags.loopdetect) {
-	    ps->direct = DIRECT_YES;
-	} else if (peerCheckNetdbDirect(ps)) {
-	    ps->direct = DIRECT_YES;
-	} else {
-	    ps->direct = DIRECT_MAYBE;
-	}
-	debug(44, 3) ("peerSelectFoo: direct = %s\n",
-	    DirectStr[ps->direct]);
-    }
-    if (entry == NULL) {
-	(void) 0;
-    } else if (entry->ping_status == PING_NONE) {
-	peerGetSomeNeighbor(ps);
-	if (entry->ping_status == PING_WAITING)
-	    return;
-    } else if (entry->ping_status == PING_WAITING) {
-	peerGetSomeNeighborReplies(ps);
-	entry->ping_status = PING_DONE;
-    }
-    switch (ps->direct) {
-    case DIRECT_YES:
-	peerGetSomeDirect(ps);
-	break;
-    case DIRECT_NO:
-	peerGetSomeParent(ps);
-	peerGetAllParents(ps);
-	break;
-    default:
-	if (Config.onoff.prefer_direct)
-	    peerGetSomeDirect(ps);
-	if (request->flags.hierarchical || !Config.onoff.nonhierarchical_direct)
-	    peerGetSomeParent(ps);
-	if (!Config.onoff.prefer_direct)
-	    peerGetSomeDirect(ps);
-	break;
-    }
-    peerSelectCallback(ps);
-}
-
-/*
- * peerGetSomeNeighbor
- * 
- * Selects a neighbor (parent or sibling) based on one of the
- * following methods:
- *      Cache Digests
- *      CARP
- *      Netdb RTT estimates
- *      ICP/HTCP queries
- */
-static void
-peerGetSomeNeighbor(ps_state * ps)
-{
-    StoreEntry *entry = ps->entry;
-    request_t *request = ps->request;
-    peer *p;
-    hier_code code = HIER_NONE;
-    assert(entry->ping_status == PING_NONE);
-    if (ps->direct == DIRECT_YES) {
-	entry->ping_status = PING_DONE;
-	return;
-    }
-#if USE_CACHE_DIGESTS
-    if ((p = neighborsDigestSelect(request))) {
-	if (neighborType(p, request) == PEER_PARENT)
-	    code = CD_PARENT_HIT;
-	else
-	    code = CD_SIBLING_HIT;
-    } else
-#endif
-    if ((p = netdbClosestParent(request))) {
-	code = CLOSEST_PARENT;
-    } else if (peerSelectIcpPing(request, ps->direct, entry)) {
-	debug(44, 3) ("peerSelect: Doing ICP pings\n");
-	ps->ping.start = current_time;
-	ps->ping.n_sent = neighborsUdpPing(request,
-	    entry,
-	    peerHandlePingReply,
-	    ps,
-	    &ps->ping.n_replies_expected,
-	    &ps->ping.timeout);
-	if (ps->ping.n_sent == 0)
-	    debug(44, 0) ("WARNING: neighborsUdpPing returned 0\n");
-	debug(44, 3) ("peerSelect: %d ICP replies expected, RTT %d msec\n",
-	    ps->ping.n_replies_expected, ps->ping.timeout);
-	if (ps->ping.n_replies_expected > 0) {
-	    entry->ping_status = PING_WAITING;
-	    eventAdd("peerPingTimeout",
-		peerPingTimeout,
-		ps,
-		0.001 * ps->ping.timeout,
-		0);
-	    return;
-	}
-    }
-    if (code != HIER_NONE) {
-	assert(p);
-	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], p->host);
-	peerAddFwdServer(&ps->servers, p, code);
-    }
-    entry->ping_status = PING_DONE;
-}
-
-/*
- * peerGetSomeNeighborReplies
- * 
- * Selects a neighbor (parent or sibling) based on ICP/HTCP replies.
- */
-static void
-peerGetSomeNeighborReplies(ps_state * ps)
-{
-    request_t *request = ps->request;
-    peer *p = NULL;
-    hier_code code = HIER_NONE;
-    assert(ps->entry->ping_status == PING_WAITING);
-    assert(ps->direct != DIRECT_YES);
-    if (peerCheckNetdbDirect(ps)) {
-	code = CLOSEST_DIRECT;
-	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], request->host);
-	peerAddFwdServer(&ps->servers, NULL, code);
-	return;
-    }
-    if ((p = ps->hit)) {
-	code = ps->hit_type == PEER_PARENT ? PARENT_HIT : SIBLING_HIT;
-    } else
-#if ALLOW_SOURCE_PING
-    if ((p = ps->secho)) {
-	code = SOURCE_FASTEST;
-    } else
-#endif
-    if (ps->closest_parent_miss.sin_addr.s_addr != any_addr.s_addr) {
-	p = whichPeer(&ps->closest_parent_miss);
-	code = CLOSEST_PARENT_MISS;
-    } else if (ps->first_parent_miss.sin_addr.s_addr != any_addr.s_addr) {
-	p = whichPeer(&ps->first_parent_miss);
-	code = FIRST_PARENT_MISS;
-    }
-    if (p && code != HIER_NONE) {
-	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], p->host);
-	peerAddFwdServer(&ps->servers, p, code);
-    }
-}
-
-
-/*
- * peerGetSomeDirect
- * 
- * Simply adds a 'direct' entry to the FwdServers list if this
- * request can be forwarded directly to the origin server
- */
-static void
-peerGetSomeDirect(ps_state * ps)
-{
-    if (ps->direct == DIRECT_NO)
-	return;
-    if (ps->request->protocol == PROTO_WAIS)
-	/* Its not really DIRECT, now is it? */
-	peerAddFwdServer(&ps->servers, Config.Wais._peer, DIRECT);
-    else
-	peerAddFwdServer(&ps->servers, NULL, DIRECT);
-}
-
-static void
-peerGetSomeParent(ps_state * ps)
-{
-    peer *p;
-    request_t *request = ps->request;
-    hier_code code = HIER_NONE;
-    debug(44, 3) ("peerGetSomeParent: %s %s\n",
-	RequestMethodStr[request->method],
-	request->host);
-    if (ps->direct == DIRECT_YES)
-	return;
-    if ((p = getDefaultParent(request))) {
-	code = DEFAULT_PARENT;
-#if USE_CARP
-    } else if ((p = carpSelectParent(request))) {
-	code = CARP;
-#endif
-    } else if ((p = getRoundRobinParent(request))) {
-	code = ROUNDROBIN_PARENT;
-    } else if ((p = getWeightedRoundRobinParent(request))) {
-	code = ROUNDROBIN_PARENT;
-    } else if ((p = getFirstUpParent(request))) {
-	code = FIRSTUP_PARENT;
-    } else if ((p = getAnyParent(request))) {
-	code = ANY_OLD_PARENT;
-    }
-    if (code != HIER_NONE) {
-	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], p->host);
-	peerAddFwdServer(&ps->servers, p, code);
-    }
-}
-
-/* Adds alive parents. Used as a last resort for never_direct.
- */
-static void
-peerGetAllParents(ps_state * ps)
-{
-    peer *p;
-    request_t *request = ps->request;
-    /* Add all alive parents */
-    for (p = Config.peers; p; p = p->next) {
-	/* XXX: neighbors.c lacks a public interface for enumerating
-	 * parents to a request so we have to dig some here..
-	 */
-	if (neighborType(p, request) != PEER_PARENT)
-	    continue;
-	if (!peerHTTPOkay(p, request))
-	    continue;
-	debug(15, 3) ("peerGetAllParents: adding alive parent %s\n", p->host);
-	peerAddFwdServer(&ps->servers, p, ANY_OLD_PARENT);
-    }
-    /* XXX: should add dead parents here, but it is currently
-     * not possible to find out which parents are dead or which
-     * simply are not configured to handle the request.
-     */
-    /* Add default parent as a last resort */
-    if ((p = getDefaultParent(request))) {
-	peerAddFwdServer(&ps->servers, p, DEFAULT_PARENT);
-    }
-}
-
-static void
-peerPingTimeout(void *data)
-{
-    ps_state *psstate = data;
-    StoreEntry *entry = psstate->entry;
-    if (entry)
-	debug(44, 3) ("peerPingTimeout: '%s'\n", storeUrl(entry));
-    if (!cbdataReferenceValid(psstate->callback_data)) {
-	/* request aborted */
-	entry->ping_status = PING_DONE;
-	cbdataReferenceDone(psstate->callback_data);
-	peerSelectStateFree(psstate);
-	return;
-    }
-    PeerStats.timeouts++;
-    psstate->ping.timedout = 1;
-    peerSelectFoo(psstate);
-}
-
-void
-peerSelectInit(void)
-{
-    memset(&PeerStats, '\0', sizeof(PeerStats));
-    assert(sizeof(hier_strings) == (HIER_MAX + 1) * sizeof(char *));
-}
-
-static void
-peerIcpParentMiss(peer * p, icp_common_t * header, ps_state * ps)
-{
-    int rtt;
-    int hops;
-    if (Config.onoff.query_icmp) {
-	if (header->flags & ICP_FLAG_SRC_RTT) {
-	    rtt = header->pad & 0xFFFF;
-	    hops = (header->pad >> 16) & 0xFFFF;
-	    if (rtt > 0 && rtt < 0xFFFF)
-		netdbUpdatePeer(ps->request, p, rtt, hops);
-	    if (rtt && (ps->ping.p_rtt == 0 || rtt < ps->ping.p_rtt)) {
-		ps->closest_parent_miss = p->in_addr;
-		ps->ping.p_rtt = rtt;
-	    }
-	}
-    }
-    /* if closest-only is set, then don't allow FIRST_PARENT_MISS */
-    if (p->options.closest_only)
-	return;
-    /* set FIRST_MISS if there is no CLOSEST parent */
-    if (ps->closest_parent_miss.sin_addr.s_addr != any_addr.s_addr)
-	return;
-    rtt = (tvSubMsec(ps->ping.start, current_time) - p->basetime) / p->weight;
-    if (rtt < 1)
-	rtt = 1;
-    if (ps->first_parent_miss.sin_addr.s_addr == any_addr.s_addr ||
-	rtt < ps->ping.w_rtt) {
-	ps->first_parent_miss = p->in_addr;
-	ps->ping.w_rtt = rtt;
-    }
-}
-
-static void
-peerHandleIcpReply(peer * p, peer_t type, icp_common_t * header, void *data)
-{
-    ps_state *psstate = data;
-    icp_opcode op = header->opcode;
-    debug(44, 3) ("peerHandleIcpReply: %s %s\n",
-	icp_opcode_str[op],
-	storeUrl(psstate->entry));
-#if USE_CACHE_DIGESTS && 0
-    /* do cd lookup to count false misses */
-    if (p && request)
-	peerNoteDigestLookup(request, p,
-	    peerDigestLookup(p, request, psstate->entry));
-#endif
-    psstate->ping.n_recv++;
-    if (op == ICP_MISS || op == ICP_DECHO) {
-	if (type == PEER_PARENT)
-	    peerIcpParentMiss(p, header, psstate);
-    } else if (op == ICP_HIT) {
-	psstate->hit = p;
-	psstate->hit_type = type;
-	peerSelectFoo(psstate);
-	return;
-    }
-#if ALLOW_SOURCE_PING
-    else if (op == ICP_SECHO) {
-	psstate->secho = p;
-	peerSelectFoo(psstate);
-	return;
-    }
-#endif
-    if (psstate->ping.n_recv < psstate->ping.n_replies_expected)
-	return;
-    peerSelectFoo(psstate);
-}
-
-#if USE_HTCP
-static void
-peerHandleHtcpReply(peer * p, peer_t type, htcpReplyData * htcp, void *data)
-{
-    ps_state *psstate = data;
-    debug(44, 3) ("peerHandleIcpReply: %s %s\n",
-	htcp->hit ? "HIT" : "MISS",
-	storeUrl(psstate->entry));
-    psstate->ping.n_recv++;
-    if (htcp->hit) {
-	psstate->hit = p;
-	psstate->hit_type = type;
-	peerSelectFoo(psstate);
-	return;
-    }
-    if (type == PEER_PARENT)
-	peerHtcpParentMiss(p, htcp, psstate);
-    if (psstate->ping.n_recv < psstate->ping.n_replies_expected)
-	return;
-    peerSelectFoo(psstate);
-}
-
-static void
-peerHtcpParentMiss(peer * p, htcpReplyData * htcp, ps_state * ps)
-{
-    int rtt;
-    int hops;
-    if (Config.onoff.query_icmp) {
-	if (htcp->cto.rtt > 0) {
-	    rtt = (int) htcp->cto.rtt * 1000;
-	    hops = (int) htcp->cto.hops * 1000;
-	    netdbUpdatePeer(ps->request, p, rtt, hops);
-	    if (rtt && (ps->ping.p_rtt == 0 || rtt < ps->ping.p_rtt)) {
-		ps->closest_parent_miss = p->in_addr;
-		ps->ping.p_rtt = rtt;
-	    }
-	}
-    }
-    /* if closest-only is set, then don't allow FIRST_PARENT_MISS */
-    if (p->options.closest_only)
-	return;
-    /* set FIRST_MISS if there is no CLOSEST parent */
-    if (ps->closest_parent_miss.sin_addr.s_addr != any_addr.s_addr)
-	return;
-    rtt = (tvSubMsec(ps->ping.start, current_time) - p->basetime) / p->weight;
-    if (rtt < 1)
-	rtt = 1;
-    if (ps->first_parent_miss.sin_addr.s_addr == any_addr.s_addr ||
-	rtt < ps->ping.w_rtt) {
-	ps->first_parent_miss = p->in_addr;
-	ps->ping.w_rtt = rtt;
-    }
-}
-#endif
-
-static void
-peerHandlePingReply(peer * p, peer_t type, protocol_t proto, void *pingdata, void *data)
-{
-    if (proto == PROTO_ICP)
-	peerHandleIcpReply(p, type, pingdata, data);
-#if USE_HTCP
-    else if (proto == PROTO_HTCP)
-	peerHandleHtcpReply(p, type, pingdata, data);
-#endif
-    else
-	debug(44, 1) ("peerHandlePingReply: unknown protocol_t %d\n", (int) proto);
-}
-
-static void
-peerAddFwdServer(FwdServer ** FS, peer * p, hier_code code)
-{
-    FwdServer *fs = memAllocate(MEM_FWD_SERVER);
-    debug(44, 5) ("peerAddFwdServer: adding %s %s\n",
-	p ? p->host : "DIRECT",
-	hier_strings[code]);
-    fs->_peer = cbdataReference(p);
-    fs->code = code;
-    while (*FS)
-	FS = &(*FS)->next;
-    *FS = fs;
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/peer_select.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,667 @@
+
+/*
+ * $Id: peer_select.cc,v 1.1.2.1 2002/10/11 15:40:59 rbcollins Exp $
+ *
+ * DEBUG: section 44    Peer Selection Algorithm
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "Store.h"
+#include "ICP.h"
+
+const char *hier_strings[] =
+{
+    "NONE",
+    "DIRECT",
+    "SIBLING_HIT",
+    "PARENT_HIT",
+    "DEFAULT_PARENT",
+    "SINGLE_PARENT",
+    "FIRST_UP_PARENT",
+    "FIRST_PARENT_MISS",
+    "CLOSEST_PARENT_MISS",
+    "CLOSEST_PARENT",
+    "CLOSEST_DIRECT",
+    "NO_DIRECT_FAIL",
+    "SOURCE_FASTEST",
+    "ROUNDROBIN_PARENT",
+#if USE_CACHE_DIGESTS
+    "CD_PARENT_HIT",
+    "CD_SIBLING_HIT",
+#endif
+#if USE_CARP
+    "CARP",
+#endif
+    "ANY_PARENT",
+    "INVALID CODE"
+};
+
+static struct {
+    int timeouts;
+} PeerStats;
+
+static const char *DirectStr[] =
+{
+    "DIRECT_UNKNOWN",
+    "DIRECT_NO",
+    "DIRECT_MAYBE",
+    "DIRECT_YES"
+};
+
+static void peerSelectFoo(ps_state *);
+static void peerPingTimeout(void *data);
+static void peerSelectCallback(ps_state * psstate);
+static IRCB peerHandlePingReply;
+static void peerSelectStateFree(ps_state * psstate);
+static void peerIcpParentMiss(peer *, icp_common_t *, ps_state *);
+#if USE_HTCP
+static void peerHtcpParentMiss(peer *, htcpReplyData *, ps_state *);
+static void peerHandleHtcpReply(peer *, peer_t, htcpReplyData *, void *);
+#endif
+static int peerCheckNetdbDirect(ps_state * psstate);
+static void peerGetSomeNeighbor(ps_state *);
+static void peerGetSomeNeighborReplies(ps_state *);
+static void peerGetSomeDirect(ps_state *);
+static void peerGetSomeParent(ps_state *);
+static void peerGetAllParents(ps_state *);
+static void peerAddFwdServer(FwdServer **, peer *, hier_code);
+
+static void
+peerSelectStateFree(ps_state * psstate)
+{
+    if (psstate->acl_checklist) {
+	debug(44, 1) ("calling aclChecklistFree() from peerSelectStateFree\n");
+	aclChecklistFree(psstate->acl_checklist);
+    }
+    requestUnlink(psstate->request);
+    psstate->request = NULL;
+    if (psstate->entry) {
+	assert(psstate->entry->ping_status != PING_WAITING);
+	storeUnlockObject(psstate->entry);
+	psstate->entry = NULL;
+    }
+    cbdataFree(psstate);
+}
+
+static int
+peerSelectIcpPing(request_t * request, int direct, StoreEntry * entry)
+{
+    int n;
+    assert(entry);
+    assert(entry->ping_status == PING_NONE);
+    assert(direct != DIRECT_YES);
+    debug(44, 3) ("peerSelectIcpPing: %s\n", storeUrl(entry));
+    if (!request->flags.hierarchical && direct != DIRECT_NO)
+	return 0;
+    if (EBIT_TEST(entry->flags, KEY_PRIVATE) && !neighbors_do_private_keys)
+	if (direct != DIRECT_NO)
+	    return 0;
+    n = neighborsCount(request);
+    debug(44, 3) ("peerSelectIcpPing: counted %d neighbors\n", n);
+    return n;
+}
+
+
+void
+peerSelect(request_t * request,
+    StoreEntry * entry,
+    PSC * callback,
+    void *callback_data)
+{
+    ps_state *psstate;
+    if (entry)
+	debug(44, 3) ("peerSelect: %s\n", storeUrl(entry));
+    else
+	debug(44, 3) ("peerSelect: %s\n", RequestMethodStr[request->method]);
+    psstate = cbdataAlloc(ps_state);
+    psstate->request = requestLink(request);
+    psstate->entry = entry;
+    psstate->callback = callback;
+    psstate->callback_data = cbdataReference(callback_data);
+    psstate->direct = DIRECT_UNKNOWN;
+#if USE_CACHE_DIGESTS
+    request->hier.peer_select_start = current_time;
+#endif
+    if (psstate->entry)
+	storeLockObject(psstate->entry);
+    peerSelectFoo(psstate);
+}
+
+static void
+peerCheckNeverDirectDone(int answer, void *data)
+{
+    ps_state *psstate = (ps_state *) data;
+    psstate->acl_checklist = NULL;
+    debug(44, 3) ("peerCheckNeverDirectDone: %d\n", answer);
+    psstate->never_direct = answer ? 1 : -1;
+    peerSelectFoo(psstate);
+}
+
+static void
+peerCheckAlwaysDirectDone(int answer, void *data)
+{
+    ps_state *psstate = (ps_state *)data;
+    psstate->acl_checklist = NULL;
+    debug(44, 3) ("peerCheckAlwaysDirectDone: %d\n", answer);
+    psstate->always_direct = answer ? 1 : -1;
+    peerSelectFoo(psstate);
+}
+
+static void
+peerSelectCallback(ps_state * psstate)
+{
+    StoreEntry *entry = psstate->entry;
+    FwdServer *fs = psstate->servers;
+    PSC *callback;
+    void *cbdata;
+    if (entry) {
+	debug(44, 3) ("peerSelectCallback: %s\n", storeUrl(entry));
+	if (entry->ping_status == PING_WAITING)
+	    eventDelete(peerPingTimeout, psstate);
+	entry->ping_status = PING_DONE;
+    }
+    if (fs == NULL) {
+	debug(44, 1) ("Failed to select source for '%s'\n", storeUrl(entry));
+	debug(44, 1) ("  always_direct = %d\n", psstate->always_direct);
+	debug(44, 1) ("   never_direct = %d\n", psstate->never_direct);
+	debug(44, 1) ("       timedout = %d\n", psstate->ping.timedout);
+    }
+    psstate->ping.stop = current_time;
+    psstate->request->hier.ping = psstate->ping;
+    callback = psstate->callback;
+    psstate->callback = NULL;
+    if (cbdataReferenceValidDone(psstate->callback_data, &cbdata)) {
+	psstate->servers = NULL;
+	callback(fs, cbdata);
+    }
+    peerSelectStateFree(psstate);
+}
+
+static int
+peerCheckNetdbDirect(ps_state * psstate)
+{
+    peer *p;
+    int myrtt;
+    int myhops;
+    if (psstate->direct == DIRECT_NO)
+	return 0;
+    myrtt = netdbHostRtt(psstate->request->host);
+    debug(44, 3) ("peerCheckNetdbDirect: MY RTT = %d msec\n", myrtt);
+    debug(44, 3) ("peerCheckNetdbDirect: minimum_direct_rtt = %d msec\n",
+	Config.minDirectRtt);
+    if (myrtt && myrtt <= Config.minDirectRtt)
+	return 1;
+    myhops = netdbHostHops(psstate->request->host);
+    debug(44, 3) ("peerCheckNetdbDirect: MY hops = %d\n", myhops);
+    debug(44, 3) ("peerCheckNetdbDirect: minimum_direct_hops = %d\n",
+	Config.minDirectHops);
+    if (myhops && myhops <= Config.minDirectHops)
+	return 1;
+    p = whichPeer(&psstate->closest_parent_miss);
+    if (p == NULL)
+	return 0;
+    debug(44, 3) ("peerCheckNetdbDirect: closest_parent_miss RTT = %d msec\n",
+	psstate->ping.p_rtt);
+    if (myrtt && myrtt <= psstate->ping.p_rtt)
+	return 1;
+    return 0;
+}
+
+static void
+peerSelectFoo(ps_state * ps)
+{
+    StoreEntry *entry = ps->entry;
+    request_t *request = ps->request;
+    debug(44, 3) ("peerSelectFoo: '%s %s'\n",
+	RequestMethodStr[request->method],
+	request->host);
+    if (ps->direct == DIRECT_UNKNOWN) {
+	if (ps->always_direct == 0 && Config.accessList.AlwaysDirect) {
+	    ps->acl_checklist = aclChecklistCreate(
+		Config.accessList.AlwaysDirect,
+		request,
+		NULL);		/* ident */
+	    aclNBCheck(ps->acl_checklist,
+		peerCheckAlwaysDirectDone,
+		ps);
+	    return;
+	} else if (ps->always_direct > 0) {
+	    ps->direct = DIRECT_YES;
+	} else if (ps->never_direct == 0 && Config.accessList.NeverDirect) {
+	    ps->acl_checklist = aclChecklistCreate(
+		Config.accessList.NeverDirect,
+		request,
+		NULL);		/* ident */
+	    aclNBCheck(ps->acl_checklist,
+		peerCheckNeverDirectDone,
+		ps);
+	    return;
+	} else if (ps->never_direct > 0) {
+	    ps->direct = DIRECT_NO;
+	} else if (request->flags.loopdetect) {
+	    ps->direct = DIRECT_YES;
+	} else if (peerCheckNetdbDirect(ps)) {
+	    ps->direct = DIRECT_YES;
+	} else {
+	    ps->direct = DIRECT_MAYBE;
+	}
+	debug(44, 3) ("peerSelectFoo: direct = %s\n",
+	    DirectStr[ps->direct]);
+    }
+    if (entry == NULL) {
+	(void) 0;
+    } else if (entry->ping_status == PING_NONE) {
+	peerGetSomeNeighbor(ps);
+	if (entry->ping_status == PING_WAITING)
+	    return;
+    } else if (entry->ping_status == PING_WAITING) {
+	peerGetSomeNeighborReplies(ps);
+	entry->ping_status = PING_DONE;
+    }
+    switch (ps->direct) {
+    case DIRECT_YES:
+	peerGetSomeDirect(ps);
+	break;
+    case DIRECT_NO:
+	peerGetSomeParent(ps);
+	peerGetAllParents(ps);
+	break;
+    default:
+	if (Config.onoff.prefer_direct)
+	    peerGetSomeDirect(ps);
+	if (request->flags.hierarchical || !Config.onoff.nonhierarchical_direct)
+	    peerGetSomeParent(ps);
+	if (!Config.onoff.prefer_direct)
+	    peerGetSomeDirect(ps);
+	break;
+    }
+    peerSelectCallback(ps);
+}
+
+/*
+ * peerGetSomeNeighbor
+ * 
+ * Selects a neighbor (parent or sibling) based on one of the
+ * following methods:
+ *      Cache Digests
+ *      CARP
+ *      Netdb RTT estimates
+ *      ICP/HTCP queries
+ */
+static void
+peerGetSomeNeighbor(ps_state * ps)
+{
+    StoreEntry *entry = ps->entry;
+    request_t *request = ps->request;
+    peer *p;
+    hier_code code = HIER_NONE;
+    assert(entry->ping_status == PING_NONE);
+    if (ps->direct == DIRECT_YES) {
+	entry->ping_status = PING_DONE;
+	return;
+    }
+#if USE_CACHE_DIGESTS
+    if ((p = neighborsDigestSelect(request))) {
+	if (neighborType(p, request) == PEER_PARENT)
+	    code = CD_PARENT_HIT;
+	else
+	    code = CD_SIBLING_HIT;
+    } else
+#endif
+    if ((p = netdbClosestParent(request))) {
+	code = CLOSEST_PARENT;
+    } else if (peerSelectIcpPing(request, ps->direct, entry)) {
+	debug(44, 3) ("peerSelect: Doing ICP pings\n");
+	ps->ping.start = current_time;
+	ps->ping.n_sent = neighborsUdpPing(request,
+	    entry,
+	    peerHandlePingReply,
+	    ps,
+	    &ps->ping.n_replies_expected,
+	    &ps->ping.timeout);
+	if (ps->ping.n_sent == 0)
+	    debug(44, 0) ("WARNING: neighborsUdpPing returned 0\n");
+	debug(44, 3) ("peerSelect: %d ICP replies expected, RTT %d msec\n",
+	    ps->ping.n_replies_expected, ps->ping.timeout);
+	if (ps->ping.n_replies_expected > 0) {
+	    entry->ping_status = PING_WAITING;
+	    eventAdd("peerPingTimeout",
+		peerPingTimeout,
+		ps,
+		0.001 * ps->ping.timeout,
+		0);
+	    return;
+	}
+    }
+    if (code != HIER_NONE) {
+	assert(p);
+	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], p->host);
+	peerAddFwdServer(&ps->servers, p, code);
+    }
+    entry->ping_status = PING_DONE;
+}
+
+/*
+ * peerGetSomeNeighborReplies
+ * 
+ * Selects a neighbor (parent or sibling) based on ICP/HTCP replies.
+ */
+static void
+peerGetSomeNeighborReplies(ps_state * ps)
+{
+    request_t *request = ps->request;
+    peer *p = NULL;
+    hier_code code = HIER_NONE;
+    assert(ps->entry->ping_status == PING_WAITING);
+    assert(ps->direct != DIRECT_YES);
+    if (peerCheckNetdbDirect(ps)) {
+	code = CLOSEST_DIRECT;
+	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], request->host);
+	peerAddFwdServer(&ps->servers, NULL, code);
+	return;
+    }
+    if ((p = ps->hit)) {
+	code = ps->hit_type == PEER_PARENT ? PARENT_HIT : SIBLING_HIT;
+    } else
+#if ALLOW_SOURCE_PING
+    if ((p = ps->secho)) {
+	code = SOURCE_FASTEST;
+    } else
+#endif
+    if (ps->closest_parent_miss.sin_addr.s_addr != any_addr.s_addr) {
+	p = whichPeer(&ps->closest_parent_miss);
+	code = CLOSEST_PARENT_MISS;
+    } else if (ps->first_parent_miss.sin_addr.s_addr != any_addr.s_addr) {
+	p = whichPeer(&ps->first_parent_miss);
+	code = FIRST_PARENT_MISS;
+    }
+    if (p && code != HIER_NONE) {
+	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], p->host);
+	peerAddFwdServer(&ps->servers, p, code);
+    }
+}
+
+
+/*
+ * peerGetSomeDirect
+ * 
+ * Simply adds a 'direct' entry to the FwdServers list if this
+ * request can be forwarded directly to the origin server
+ */
+static void
+peerGetSomeDirect(ps_state * ps)
+{
+    if (ps->direct == DIRECT_NO)
+	return;
+    if (ps->request->protocol == PROTO_WAIS)
+	/* Its not really DIRECT, now is it? */
+	peerAddFwdServer(&ps->servers, Config.Wais._peer, DIRECT);
+    else
+	peerAddFwdServer(&ps->servers, NULL, DIRECT);
+}
+
+static void
+peerGetSomeParent(ps_state * ps)
+{
+    peer *p;
+    request_t *request = ps->request;
+    hier_code code = HIER_NONE;
+    debug(44, 3) ("peerGetSomeParent: %s %s\n",
+	RequestMethodStr[request->method],
+	request->host);
+    if (ps->direct == DIRECT_YES)
+	return;
+    if ((p = getDefaultParent(request))) {
+	code = DEFAULT_PARENT;
+#if USE_CARP
+    } else if ((p = carpSelectParent(request))) {
+	code = CARP;
+#endif
+    } else if ((p = getRoundRobinParent(request))) {
+	code = ROUNDROBIN_PARENT;
+    } else if ((p = getWeightedRoundRobinParent(request))) {
+	code = ROUNDROBIN_PARENT;
+    } else if ((p = getFirstUpParent(request))) {
+	code = FIRSTUP_PARENT;
+    } else if ((p = getAnyParent(request))) {
+	code = ANY_OLD_PARENT;
+    }
+    if (code != HIER_NONE) {
+	debug(44, 3) ("peerSelect: %s/%s\n", hier_strings[code], p->host);
+	peerAddFwdServer(&ps->servers, p, code);
+    }
+}
+
+/* Adds alive parents. Used as a last resort for never_direct.
+ */
+static void
+peerGetAllParents(ps_state * ps)
+{
+    peer *p;
+    request_t *request = ps->request;
+    /* Add all alive parents */
+    for (p = Config.peers; p; p = p->next) {
+	/* XXX: neighbors.c lacks a public interface for enumerating
+	 * parents to a request so we have to dig some here..
+	 */
+	if (neighborType(p, request) != PEER_PARENT)
+	    continue;
+	if (!peerHTTPOkay(p, request))
+	    continue;
+	debug(15, 3) ("peerGetAllParents: adding alive parent %s\n", p->host);
+	peerAddFwdServer(&ps->servers, p, ANY_OLD_PARENT);
+    }
+    /* XXX: should add dead parents here, but it is currently
+     * not possible to find out which parents are dead or which
+     * simply are not configured to handle the request.
+     */
+    /* Add default parent as a last resort */
+    if ((p = getDefaultParent(request))) {
+	peerAddFwdServer(&ps->servers, p, DEFAULT_PARENT);
+    }
+}
+
+static void
+peerPingTimeout(void *data)
+{
+    ps_state *psstate = (ps_state *)data;
+    StoreEntry *entry = psstate->entry;
+    if (entry)
+	debug(44, 3) ("peerPingTimeout: '%s'\n", storeUrl(entry));
+    if (!cbdataReferenceValid(psstate->callback_data)) {
+	/* request aborted */
+	entry->ping_status = PING_DONE;
+	cbdataReferenceDone(psstate->callback_data);
+	peerSelectStateFree(psstate);
+	return;
+    }
+    PeerStats.timeouts++;
+    psstate->ping.timedout = 1;
+    peerSelectFoo(psstate);
+}
+
+void
+peerSelectInit(void)
+{
+    memset(&PeerStats, '\0', sizeof(PeerStats));
+    assert(sizeof(hier_strings) == (HIER_MAX + 1) * sizeof(char *));
+}
+
+static void
+peerIcpParentMiss(peer * p, icp_common_t * header, ps_state * ps)
+{
+    int rtt;
+    int hops;
+    if (Config.onoff.query_icmp) {
+	if (header->flags & ICP_FLAG_SRC_RTT) {
+	    rtt = header->pad & 0xFFFF;
+	    hops = (header->pad >> 16) & 0xFFFF;
+	    if (rtt > 0 && rtt < 0xFFFF)
+		netdbUpdatePeer(ps->request, p, rtt, hops);
+	    if (rtt && (ps->ping.p_rtt == 0 || rtt < ps->ping.p_rtt)) {
+		ps->closest_parent_miss = p->in_addr;
+		ps->ping.p_rtt = rtt;
+	    }
+	}
+    }
+    /* if closest-only is set, then don't allow FIRST_PARENT_MISS */
+    if (p->options.closest_only)
+	return;
+    /* set FIRST_MISS if there is no CLOSEST parent */
+    if (ps->closest_parent_miss.sin_addr.s_addr != any_addr.s_addr)
+	return;
+    rtt = (tvSubMsec(ps->ping.start, current_time) - p->basetime) / p->weight;
+    if (rtt < 1)
+	rtt = 1;
+    if (ps->first_parent_miss.sin_addr.s_addr == any_addr.s_addr ||
+	rtt < ps->ping.w_rtt) {
+	ps->first_parent_miss = p->in_addr;
+	ps->ping.w_rtt = rtt;
+    }
+}
+
+static void
+peerHandleIcpReply(peer * p, peer_t type, icp_common_t * header, void *data)
+{
+    ps_state *psstate = (ps_state *)data;
+    icp_opcode op = header->getOpCode();
+    debug(44, 3) ("peerHandleIcpReply: %s %s\n",
+	icp_opcode_str[op],
+	storeUrl(psstate->entry));
+#if USE_CACHE_DIGESTS && 0
+    /* do cd lookup to count false misses */
+    if (p && request)
+	peerNoteDigestLookup(request, p,
+	    peerDigestLookup(p, request, psstate->entry));
+#endif
+    psstate->ping.n_recv++;
+    if (op == ICP_MISS || op == ICP_DECHO) {
+	if (type == PEER_PARENT)
+	    peerIcpParentMiss(p, header, psstate);
+    } else if (op == ICP_HIT) {
+	psstate->hit = p;
+	psstate->hit_type = type;
+	peerSelectFoo(psstate);
+	return;
+    }
+#if ALLOW_SOURCE_PING
+    else if (op == ICP_SECHO) {
+	psstate->secho = p;
+	peerSelectFoo(psstate);
+	return;
+    }
+#endif
+    if (psstate->ping.n_recv < psstate->ping.n_replies_expected)
+	return;
+    peerSelectFoo(psstate);
+}
+
+#if USE_HTCP
+static void
+peerHandleHtcpReply(peer * p, peer_t type, htcpReplyData * htcp, void *data)
+{
+    ps_state *psstate = (ps_state *)data;
+    debug(44, 3) ("peerHandleIcpReply: %s %s\n",
+	htcp->hit ? "HIT" : "MISS",
+	storeUrl(psstate->entry));
+    psstate->ping.n_recv++;
+    if (htcp->hit) {
+	psstate->hit = p;
+	psstate->hit_type = type;
+	peerSelectFoo(psstate);
+	return;
+    }
+    if (type == PEER_PARENT)
+	peerHtcpParentMiss(p, htcp, psstate);
+    if (psstate->ping.n_recv < psstate->ping.n_replies_expected)
+	return;
+    peerSelectFoo(psstate);
+}
+
+static void
+peerHtcpParentMiss(peer * p, htcpReplyData * htcp, ps_state * ps)
+{
+    int rtt;
+    int hops;
+    if (Config.onoff.query_icmp) {
+	if (htcp->cto.rtt > 0) {
+	    rtt = (int) htcp->cto.rtt * 1000;
+	    hops = (int) htcp->cto.hops * 1000;
+	    netdbUpdatePeer(ps->request, p, rtt, hops);
+	    if (rtt && (ps->ping.p_rtt == 0 || rtt < ps->ping.p_rtt)) {
+		ps->closest_parent_miss = p->in_addr;
+		ps->ping.p_rtt = rtt;
+	    }
+	}
+    }
+    /* if closest-only is set, then don't allow FIRST_PARENT_MISS */
+    if (p->options.closest_only)
+	return;
+    /* set FIRST_MISS if there is no CLOSEST parent */
+    if (ps->closest_parent_miss.sin_addr.s_addr != any_addr.s_addr)
+	return;
+    rtt = (tvSubMsec(ps->ping.start, current_time) - p->basetime) / p->weight;
+    if (rtt < 1)
+	rtt = 1;
+    if (ps->first_parent_miss.sin_addr.s_addr == any_addr.s_addr ||
+	rtt < ps->ping.w_rtt) {
+	ps->first_parent_miss = p->in_addr;
+	ps->ping.w_rtt = rtt;
+    }
+}
+#endif
+
+static void
+peerHandlePingReply(peer * p, peer_t type, protocol_t proto, void *pingdata, void *data)
+{
+    if (proto == PROTO_ICP)
+	peerHandleIcpReply(p, type, (icp_common_t *)pingdata, data);
+#if USE_HTCP
+    else if (proto == PROTO_HTCP)
+	peerHandleHtcpReply(p, type, (htcpReplyData *)pingdata, data);
+#endif
+    else
+	debug(44, 1) ("peerHandlePingReply: unknown protocol_t %d\n", (int) proto);
+}
+
+static void
+peerAddFwdServer(FwdServer ** FS, peer * p, hier_code code)
+{
+    FwdServer *fs = (FwdServer *)memAllocate(MEM_FWD_SERVER);
+    debug(44, 5) ("peerAddFwdServer: adding %s %s\n",
+	p ? p->host : "DIRECT",
+	hier_strings[code]);
+    fs->_peer = cbdataReference(p);
+    fs->code = code;
+    while (*FS)
+	FS = &(*FS)->next;
+    *FS = fs;
+}
--- squid/src/pinger.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,412 +0,0 @@
-
-/*
- * $Id: pinger.c,v 1.4.96.1 2002/10/06 04:01:14 rbcollins Exp $
- *
- * DEBUG: section 42    ICMP Pinger program
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-
-#if USE_ICMP
-
-#include 
-#include 
-#include 
-#include 
-
-#ifndef _SQUID_LINUX_
-#ifndef _SQUID_CYGWIN_
-#define icmphdr icmp
-#define iphdr ip
-#endif
-#endif
-
-#if defined (_SQUID_LINUX_) || defined (_SQUID_CYGWIN_)
-#ifdef icmp_id
-#undef icmp_id
-#endif
-#ifdef icmp_seq
-#undef icmp_seq
-#endif
-#define icmp_type type
-#define icmp_code code
-#define icmp_cksum checksum
-#define icmp_id un.echo.id
-#define icmp_seq un.echo.sequence
-#define ip_hl ihl
-#define ip_v version
-#define ip_tos tos
-#define ip_len tot_len
-#define ip_id id
-#define ip_off frag_off
-#define ip_ttl ttl
-#define ip_p protocol
-#define ip_sum check
-#define ip_src saddr
-#define ip_dst daddr
-#endif
-
-#if ALLOW_SOURCE_PING
-#define MAX_PKT_SZ 8192
-#define MAX_PAYLOAD (MAX_PKT_SZ - sizeof(struct icmphdr) - sizeof (char) - sizeof(struct timeval) - 1)
-#else
-#define MAX_PAYLOAD SQUIDHOSTNAMELEN
-#define MAX_PKT_SZ (MAX_PAYLOAD + sizeof(struct timeval) + sizeof (char) + sizeof(struct icmphdr) + 1)
-#endif
-
-typedef struct {
-    struct timeval tv;
-    unsigned char opcode;
-    char payload[MAX_PAYLOAD];
-} icmpEchoData;
-
-int icmp_ident = -1;
-int icmp_pkts_sent = 0;
-
-static const char *icmpPktStr[] =
-{
-    "Echo Reply",
-    "ICMP 1",
-    "ICMP 2",
-    "Destination Unreachable",
-    "Source Quench",
-    "Redirect",
-    "ICMP 6",
-    "ICMP 7",
-    "Echo",
-    "ICMP 9",
-    "ICMP 10",
-    "Time Exceeded",
-    "Parameter Problem",
-    "Timestamp",
-    "Timestamp Reply",
-    "Info Request",
-    "Info Reply",
-    "Out of Range Type"
-};
-
-static int in_cksum(unsigned short *ptr, int size);
-static void pingerRecv(void);
-static void pingerLog(struct icmphdr *, struct in_addr, int, int);
-static int ipHops(int ttl);
-static void pingerSendtoSquid(pingerReplyData * preply);
-
-void
-pingerOpen(void)
-{
-    struct protoent *proto = NULL;
-    if ((proto = getprotobyname("icmp")) == 0) {
-	debug(42, 0) ("pingerOpen: unknown protocol: icmp\n");
-	exit(1);
-    }
-    icmp_sock = socket(PF_INET, SOCK_RAW, proto->p_proto);
-    if (icmp_sock < 0) {
-	debug(50, 0) ("pingerOpen: icmp_sock: %s\n", xstrerror());
-	exit(1);
-    }
-    icmp_ident = getpid() & 0xffff;
-    debug(42, 0) ("pinger: ICMP socket opened\n");
-}
-
-void
-pingerClose(void)
-{
-    close(icmp_sock);
-    icmp_sock = -1;
-    icmp_ident = 0;
-}
-
-static void
-pingerSendEcho(struct in_addr to, int opcode, char *payload, int len)
-{
-    LOCAL_ARRAY(char, pkt, MAX_PKT_SZ);
-    struct icmphdr *icmp = NULL;
-    icmpEchoData *echo;
-    int icmp_pktsize = sizeof(struct icmphdr);
-    struct sockaddr_in S;
-    memset(pkt, '\0', MAX_PKT_SZ);
-    icmp = (struct icmphdr *) (void *) pkt;
-
-    /*
-     * cevans - beware signed/unsigned issues in untrusted data from
-     * the network!!
-     */
-    if (len < 0) {
-	len = 0;
-    }
-    icmp->icmp_type = ICMP_ECHO;
-    icmp->icmp_code = 0;
-    icmp->icmp_cksum = 0;
-    icmp->icmp_id = icmp_ident;
-    icmp->icmp_seq = (u_short) icmp_pkts_sent++;
-    echo = (icmpEchoData *) (icmp + 1);
-    echo->opcode = (unsigned char) opcode;
-    echo->tv = current_time;
-    icmp_pktsize += sizeof(icmpEchoData) - MAX_PAYLOAD;
-    if (payload) {
-	if (len > MAX_PAYLOAD)
-	    len = MAX_PAYLOAD;
-	xmemcpy(echo->payload, payload, len);
-	icmp_pktsize += len;
-    }
-    icmp->icmp_cksum = in_cksum((u_short *) icmp, icmp_pktsize);
-    S.sin_family = AF_INET;
-    /*
-     * cevans: alert: trusting to-host, was supplied in network packet
-     */
-    S.sin_addr = to;
-    S.sin_port = 0;
-    assert(icmp_pktsize <= MAX_PKT_SZ);
-    sendto(icmp_sock,
-	pkt,
-	icmp_pktsize,
-	0,
-	(struct sockaddr *) &S,
-	sizeof(struct sockaddr_in));
-    pingerLog(icmp, to, 0, 0);
-}
-
-static void
-pingerRecv(void)
-{
-    int n;
-    socklen_t fromlen;
-    struct sockaddr_in from;
-    int iphdrlen = 20;
-    struct iphdr *ip = NULL;
-    struct icmphdr *icmp = NULL;
-    static char *pkt = NULL;
-    struct timeval now;
-    icmpEchoData *echo;
-    static pingerReplyData preply;
-
-    if (pkt == NULL)
-	pkt = xmalloc(MAX_PKT_SZ);
-    fromlen = sizeof(from);
-    n = recvfrom(icmp_sock,
-	pkt,
-	MAX_PKT_SZ,
-	0,
-	(struct sockaddr *) &from,
-	&fromlen);
-#if GETTIMEOFDAY_NO_TZP
-    gettimeofday(&now);
-#else
-    gettimeofday(&now, NULL);
-#endif
-    debug(42, 9) ("pingerRecv: %d bytes from %s\n", n, inet_ntoa(from.sin_addr));
-    ip = (struct iphdr *) (void *) pkt;
-#if HAVE_STRUCT_IPHDR_IP_HL
-    iphdrlen = ip->ip_hl << 2;
-#else /* HAVE_STRUCT_IPHDR_IP_HL */
-#if WORDS_BIGENDIAN
-    iphdrlen = (ip->ip_vhl >> 4) << 2;
-#else
-    iphdrlen = (ip->ip_vhl & 0xF) << 2;
-#endif
-#endif /* HAVE_STRUCT_IPHDR_IP_HL */
-    icmp = (struct icmphdr *) (void *) (pkt + iphdrlen);
-    if (icmp->icmp_type != ICMP_ECHOREPLY)
-	return;
-    if (icmp->icmp_id != icmp_ident)
-	return;
-    echo = (icmpEchoData *) (void *) (icmp + 1);
-    preply.from = from.sin_addr;
-    preply.opcode = echo->opcode;
-    preply.hops = ipHops(ip->ip_ttl);
-    preply.rtt = tvSubMsec(echo->tv, now);
-    preply.psize = n - iphdrlen - (sizeof(icmpEchoData) - MAX_PKT_SZ);
-    pingerSendtoSquid(&preply);
-    pingerLog(icmp, from.sin_addr, preply.rtt, preply.hops);
-}
-
-
-static int
-in_cksum(unsigned short *ptr, int size)
-{
-    long sum;
-    unsigned short oddbyte;
-    unsigned short answer;
-    sum = 0;
-    while (size > 1) {
-	sum += *ptr++;
-	size -= 2;
-    }
-    if (size == 1) {
-	oddbyte = 0;
-	*((unsigned char *) &oddbyte) = *(unsigned char *) ptr;
-	sum += oddbyte;
-    }
-    sum = (sum >> 16) + (sum & 0xffff);
-    sum += (sum >> 16);
-    answer = ~sum;
-    return (answer);
-}
-
-static void
-pingerLog(struct icmphdr *icmp, struct in_addr addr, int rtt, int hops)
-{
-    debug(42, 2) ("pingerLog: %9d.%06d %-16s %d %-15.15s %dms %d hops\n",
-	(int) current_time.tv_sec,
-	(int) current_time.tv_usec,
-	inet_ntoa(addr),
-	(int) icmp->icmp_type,
-	icmpPktStr[icmp->icmp_type],
-	rtt,
-	hops);
-}
-
-static int
-ipHops(int ttl)
-{
-    if (ttl < 33)
-	return 33 - ttl;
-    if (ttl < 63)
-	return 63 - ttl;	/* 62 = (64+60)/2 */
-    if (ttl < 65)
-	return 65 - ttl;	/* 62 = (64+60)/2 */
-    if (ttl < 129)
-	return 129 - ttl;
-    if (ttl < 193)
-	return 193 - ttl;
-    return 256 - ttl;
-}
-
-static int
-pingerReadRequest(void)
-{
-    static pingerEchoData pecho;
-    int n;
-    int guess_size;
-    memset(&pecho, '\0', sizeof(pecho));
-    n = recv(0, (char *) &pecho, sizeof(pecho), 0);
-    if (n < 0)
-	return n;
-    if (0 == n) {
-	/* EOF indicator */
-	fprintf(stderr, "EOF encountered\n");
-	errno = 0;
-	return -1;
-    }
-    guess_size = n - (sizeof(pingerEchoData) - PINGER_PAYLOAD_SZ);
-    if (guess_size != pecho.psize) {
-	fprintf(stderr, "size mismatch, guess=%d psize=%d\n",
-	    guess_size, pecho.psize);
-	/* don't process this message, but keep running */
-	return 0;
-    }
-    pingerSendEcho(pecho.to,
-	pecho.opcode,
-	pecho.payload,
-	pecho.psize);
-    return n;
-}
-
-static void
-pingerSendtoSquid(pingerReplyData * preply)
-{
-    int len = sizeof(pingerReplyData) - MAX_PKT_SZ + preply->psize;
-    if (send(1, (char *) preply, len, 0) < 0) {
-	debug(50, 0) ("pinger: send: %s\n", xstrerror());
-	exit(1);
-    }
-}
-
-time_t
-getCurrentTime(void)
-{
-#if GETTIMEOFDAY_NO_TZP
-    gettimeofday(¤t_time);
-#else
-    gettimeofday(¤t_time, NULL);
-#endif
-    return squid_curtime = current_time.tv_sec;
-}
-
-
-int
-main(int argc, char *argv[])
-{
-    fd_set R;
-    int x;
-    struct timeval tv;
-    const char *debug_args = "ALL,1";
-    char *t;
-    time_t last_check_time = 0;
-
-/*
- * cevans - do this first. It grabs a raw socket. After this we can
- * drop privs
- */
-    pingerOpen();
-    setgid(getgid());
-    setuid(getuid());
-
-    if ((t = getenv("SQUID_DEBUG")))
-	debug_args = xstrdup(t);
-    getCurrentTime();
-    _db_init(NULL, debug_args);
-
-    for (;;) {
-	tv.tv_sec = 10;
-	tv.tv_usec = 0;
-	FD_ZERO(&R);
-	FD_SET(0, &R);
-	FD_SET(icmp_sock, &R);
-	x = select(icmp_sock + 1, &R, NULL, NULL, &tv);
-	getCurrentTime();
-	if (x < 0)
-	    exit(1);
-	if (FD_ISSET(0, &R))
-	    if (pingerReadRequest() < 0) {
-		debug(42, 0) ("Pinger exiting.\n");
-		exit(1);
-	    }
-	if (FD_ISSET(icmp_sock, &R))
-	    pingerRecv();
-	if (10 + last_check_time < squid_curtime) {
-	    if (send(1, (char *) &tv, 0, 0) < 0)
-		exit(1);
-	    last_check_time = squid_curtime;
-	}
-    }
-    /* NOTREACHED */
-}
-
-#else
-#include 
-int
-main(int argc, char *argv[])
-{
-    fprintf(stderr, "%s: ICMP support not compiled in.\n", argv[0]);
-    return 1;
-}
-#endif /* USE_ICMP */
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/pinger.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,412 @@
+
+/*
+ * $Id: pinger.cc,v 1.1.2.1 2002/10/11 15:41:00 rbcollins Exp $
+ *
+ * DEBUG: section 42    ICMP Pinger program
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+#if USE_ICMP
+
+#include 
+#include 
+#include 
+#include 
+
+#ifndef _SQUID_LINUX_
+#ifndef _SQUID_CYGWIN_
+#define icmphdr icmp
+#define iphdr ip
+#endif
+#endif
+
+#if defined (_SQUID_LINUX_) || defined (_SQUID_CYGWIN_)
+#ifdef icmp_id
+#undef icmp_id
+#endif
+#ifdef icmp_seq
+#undef icmp_seq
+#endif
+#define icmp_type type
+#define icmp_code code
+#define icmp_cksum checksum
+#define icmp_id un.echo.id
+#define icmp_seq un.echo.sequence
+#define ip_hl ihl
+#define ip_v version
+#define ip_tos tos
+#define ip_len tot_len
+#define ip_id id
+#define ip_off frag_off
+#define ip_ttl ttl
+#define ip_p protocol
+#define ip_sum check
+#define ip_src saddr
+#define ip_dst daddr
+#endif
+
+#if ALLOW_SOURCE_PING
+#define MAX_PKT_SZ 8192
+#define MAX_PAYLOAD (MAX_PKT_SZ - sizeof(struct icmphdr) - sizeof (char) - sizeof(struct timeval) - 1)
+#else
+#define MAX_PAYLOAD SQUIDHOSTNAMELEN
+#define MAX_PKT_SZ (MAX_PAYLOAD + sizeof(struct timeval) + sizeof (char) + sizeof(struct icmphdr) + 1)
+#endif
+
+typedef struct {
+    struct timeval tv;
+    unsigned char opcode;
+    char payload[MAX_PAYLOAD];
+} icmpEchoData;
+
+int icmp_ident = -1;
+int icmp_pkts_sent = 0;
+
+static const char *icmpPktStr[] =
+{
+    "Echo Reply",
+    "ICMP 1",
+    "ICMP 2",
+    "Destination Unreachable",
+    "Source Quench",
+    "Redirect",
+    "ICMP 6",
+    "ICMP 7",
+    "Echo",
+    "ICMP 9",
+    "ICMP 10",
+    "Time Exceeded",
+    "Parameter Problem",
+    "Timestamp",
+    "Timestamp Reply",
+    "Info Request",
+    "Info Reply",
+    "Out of Range Type"
+};
+
+static int in_cksum(unsigned short *ptr, int size);
+static void pingerRecv(void);
+static void pingerLog(struct icmphdr *, struct in_addr, int, int);
+static int ipHops(int ttl);
+static void pingerSendtoSquid(pingerReplyData * preply);
+
+void
+pingerOpen(void)
+{
+    struct protoent *proto = NULL;
+    if ((proto = getprotobyname("icmp")) == 0) {
+	debug(42, 0) ("pingerOpen: unknown protocol: icmp\n");
+	exit(1);
+    }
+    icmp_sock = socket(PF_INET, SOCK_RAW, proto->p_proto);
+    if (icmp_sock < 0) {
+	debug(50, 0) ("pingerOpen: icmp_sock: %s\n", xstrerror());
+	exit(1);
+    }
+    icmp_ident = getpid() & 0xffff;
+    debug(42, 0) ("pinger: ICMP socket opened\n");
+}
+
+void
+pingerClose(void)
+{
+    close(icmp_sock);
+    icmp_sock = -1;
+    icmp_ident = 0;
+}
+
+static void
+pingerSendEcho(struct in_addr to, int opcode, char *payload, int len)
+{
+    LOCAL_ARRAY(char, pkt, MAX_PKT_SZ);
+    struct icmphdr *icmp = NULL;
+    icmpEchoData *echo;
+    size_t icmp_pktsize = sizeof(struct icmphdr);
+    struct sockaddr_in S;
+    memset(pkt, '\0', MAX_PKT_SZ);
+    icmp = (struct icmphdr *) (void *) pkt;
+
+    /*
+     * cevans - beware signed/unsigned issues in untrusted data from
+     * the network!!
+     */
+    if (len < 0) {
+	len = 0;
+    }
+    icmp->icmp_type = ICMP_ECHO;
+    icmp->icmp_code = 0;
+    icmp->icmp_cksum = 0;
+    icmp->icmp_id = icmp_ident;
+    icmp->icmp_seq = (u_short) icmp_pkts_sent++;
+    echo = (icmpEchoData *) (icmp + 1);
+    echo->opcode = (unsigned char) opcode;
+    echo->tv = current_time;
+    icmp_pktsize += sizeof(icmpEchoData) - MAX_PAYLOAD;
+    if (payload) {
+	if (len > MAX_PAYLOAD)
+	    len = MAX_PAYLOAD;
+	xmemcpy(echo->payload, payload, len);
+	icmp_pktsize += len;
+    }
+    icmp->icmp_cksum = in_cksum((u_short *) icmp, icmp_pktsize);
+    S.sin_family = AF_INET;
+    /*
+     * cevans: alert: trusting to-host, was supplied in network packet
+     */
+    S.sin_addr = to;
+    S.sin_port = 0;
+    assert(icmp_pktsize <= MAX_PKT_SZ);
+    sendto(icmp_sock,
+	pkt,
+	icmp_pktsize,
+	0,
+	(struct sockaddr *) &S,
+	sizeof(struct sockaddr_in));
+    pingerLog(icmp, to, 0, 0);
+}
+
+static void
+pingerRecv(void)
+{
+    int n;
+    socklen_t fromlen;
+    struct sockaddr_in from;
+    int iphdrlen = 20;
+    struct iphdr *ip = NULL;
+    struct icmphdr *icmp = NULL;
+    static char *pkt = NULL;
+    struct timeval now;
+    icmpEchoData *echo;
+    static pingerReplyData preply;
+
+    if (pkt == NULL)
+	pkt = (char *)xmalloc(MAX_PKT_SZ);
+    fromlen = sizeof(from);
+    n = recvfrom(icmp_sock,
+	pkt,
+	MAX_PKT_SZ,
+	0,
+	(struct sockaddr *) &from,
+	&fromlen);
+#if GETTIMEOFDAY_NO_TZP
+    gettimeofday(&now);
+#else
+    gettimeofday(&now, NULL);
+#endif
+    debug(42, 9) ("pingerRecv: %d bytes from %s\n", n, inet_ntoa(from.sin_addr));
+    ip = (struct iphdr *) (void *) pkt;
+#if HAVE_STRUCT_IPHDR_IP_HL
+    iphdrlen = ip->ip_hl << 2;
+#else /* HAVE_STRUCT_IPHDR_IP_HL */
+#if WORDS_BIGENDIAN
+    iphdrlen = (ip->ip_vhl >> 4) << 2;
+#else
+    iphdrlen = (ip->ip_vhl & 0xF) << 2;
+#endif
+#endif /* HAVE_STRUCT_IPHDR_IP_HL */
+    icmp = (struct icmphdr *) (void *) (pkt + iphdrlen);
+    if (icmp->icmp_type != ICMP_ECHOREPLY)
+	return;
+    if (icmp->icmp_id != icmp_ident)
+	return;
+    echo = (icmpEchoData *) (void *) (icmp + 1);
+    preply.from = from.sin_addr;
+    preply.opcode = echo->opcode;
+    preply.hops = ipHops(ip->ip_ttl);
+    preply.rtt = tvSubMsec(echo->tv, now);
+    preply.psize = n - iphdrlen - (sizeof(icmpEchoData) - MAX_PKT_SZ);
+    pingerSendtoSquid(&preply);
+    pingerLog(icmp, from.sin_addr, preply.rtt, preply.hops);
+}
+
+
+static int
+in_cksum(unsigned short *ptr, int size)
+{
+    long sum;
+    unsigned short oddbyte;
+    unsigned short answer;
+    sum = 0;
+    while (size > 1) {
+	sum += *ptr++;
+	size -= 2;
+    }
+    if (size == 1) {
+	oddbyte = 0;
+	*((unsigned char *) &oddbyte) = *(unsigned char *) ptr;
+	sum += oddbyte;
+    }
+    sum = (sum >> 16) + (sum & 0xffff);
+    sum += (sum >> 16);
+    answer = ~sum;
+    return (answer);
+}
+
+static void
+pingerLog(struct icmphdr *icmp, struct in_addr addr, int rtt, int hops)
+{
+    debug(42, 2) ("pingerLog: %9d.%06d %-16s %d %-15.15s %dms %d hops\n",
+	(int) current_time.tv_sec,
+	(int) current_time.tv_usec,
+	inet_ntoa(addr),
+	(int) icmp->icmp_type,
+	icmpPktStr[icmp->icmp_type],
+	rtt,
+	hops);
+}
+
+static int
+ipHops(int ttl)
+{
+    if (ttl < 33)
+	return 33 - ttl;
+    if (ttl < 63)
+	return 63 - ttl;	/* 62 = (64+60)/2 */
+    if (ttl < 65)
+	return 65 - ttl;	/* 62 = (64+60)/2 */
+    if (ttl < 129)
+	return 129 - ttl;
+    if (ttl < 193)
+	return 193 - ttl;
+    return 256 - ttl;
+}
+
+static int
+pingerReadRequest(void)
+{
+    static pingerEchoData pecho;
+    int n;
+    int guess_size;
+    memset(&pecho, '\0', sizeof(pecho));
+    n = recv(0, (char *) &pecho, sizeof(pecho), 0);
+    if (n < 0)
+	return n;
+    if (0 == n) {
+	/* EOF indicator */
+	fprintf(stderr, "EOF encountered\n");
+	errno = 0;
+	return -1;
+    }
+    guess_size = n - (sizeof(pingerEchoData) - PINGER_PAYLOAD_SZ);
+    if (guess_size != pecho.psize) {
+	fprintf(stderr, "size mismatch, guess=%d psize=%d\n",
+	    guess_size, pecho.psize);
+	/* don't process this message, but keep running */
+	return 0;
+    }
+    pingerSendEcho(pecho.to,
+	pecho.opcode,
+	pecho.payload,
+	pecho.psize);
+    return n;
+}
+
+static void
+pingerSendtoSquid(pingerReplyData * preply)
+{
+    int len = sizeof(pingerReplyData) - MAX_PKT_SZ + preply->psize;
+    if (send(1, (char *) preply, len, 0) < 0) {
+	debug(50, 0) ("pinger: send: %s\n", xstrerror());
+	exit(1);
+    }
+}
+
+time_t
+getCurrentTime(void)
+{
+#if GETTIMEOFDAY_NO_TZP
+    gettimeofday(¤t_time);
+#else
+    gettimeofday(¤t_time, NULL);
+#endif
+    return squid_curtime = current_time.tv_sec;
+}
+
+
+int
+main(int argc, char *argv[])
+{
+    fd_set R;
+    int x;
+    struct timeval tv;
+    const char *debug_args = "ALL,1";
+    char *t;
+    time_t last_check_time = 0;
+
+/*
+ * cevans - do this first. It grabs a raw socket. After this we can
+ * drop privs
+ */
+    pingerOpen();
+    setgid(getgid());
+    setuid(getuid());
+
+    if ((t = getenv("SQUID_DEBUG")))
+	debug_args = xstrdup(t);
+    getCurrentTime();
+    _db_init(NULL, debug_args);
+
+    for (;;) {
+	tv.tv_sec = 10;
+	tv.tv_usec = 0;
+	FD_ZERO(&R);
+	FD_SET(0, &R);
+	FD_SET(icmp_sock, &R);
+	x = select(icmp_sock + 1, &R, NULL, NULL, &tv);
+	getCurrentTime();
+	if (x < 0)
+	    exit(1);
+	if (FD_ISSET(0, &R))
+	    if (pingerReadRequest() < 0) {
+		debug(42, 0) ("Pinger exiting.\n");
+		exit(1);
+	    }
+	if (FD_ISSET(icmp_sock, &R))
+	    pingerRecv();
+	if (10 + last_check_time < squid_curtime) {
+	    if (send(1, (char *) &tv, 0, 0) < 0)
+		exit(1);
+	    last_check_time = squid_curtime;
+	}
+    }
+    /* NOTREACHED */
+}
+
+#else
+#include 
+int
+main(int argc, char *argv[])
+{
+    fprintf(stderr, "%s: ICMP support not compiled in.\n", argv[0]);
+    return 1;
+}
+#endif /* USE_ICMP */
--- squid/src/referer.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,79 +0,0 @@
-
-/*
- * $Id$
- *
- * DEBUG: section 40    User-Agent and Referer logging
- * AUTHOR: Joe Ramey  (useragent)
- *         Jens-S. Vöckler  (mod 4 referer)
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-
-#if USE_REFERER_LOG
-static Logfile *refererlog = NULL;
-#endif
-
-void
-refererOpenLog(void)
-{
-#if USE_REFERER_LOG
-    assert(NULL == refererlog);
-    if (!Config.Log.referer || (0 == strcmp(Config.Log.referer, "none"))) {
-	debug(40, 1) ("Referer logging is disabled.\n");
-	return;
-    }
-    refererlog = logfileOpen(Config.Log.referer, 0, 1);
-#endif
-}
-
-void
-refererRotateLog(void)
-{
-#if USE_REFERER_LOG
-    if (NULL == refererlog)
-	return;
-    logfileRotate(refererlog);
-#endif
-}
-
-void
-logReferer(const char *client, const char *referer, const char *uri)
-{
-#if USE_REFERER_LOG
-    if (NULL == refererlog)
-	return;
-    logfilePrintf(refererlog, "%9d.%03d %s %s %s\n",
-	(int) current_time.tv_sec,
-	(int) current_time.tv_usec / 1000,
-	client,
-	referer,
-	uri ? uri : "-");
-#endif
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/referer.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,79 @@
+
+/*
+ * $Id: referer.cc,v 1.1.2.1 2002/10/11 15:41:00 rbcollins Exp $
+ *
+ * DEBUG: section 40    User-Agent and Referer logging
+ * AUTHOR: Joe Ramey  (useragent)
+ *         Jens-S. Vöckler  (mod 4 referer)
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+#if USE_REFERER_LOG
+static Logfile *refererlog = NULL;
+#endif
+
+void
+refererOpenLog(void)
+{
+#if USE_REFERER_LOG
+    assert(NULL == refererlog);
+    if (!Config.Log.referer || (0 == strcmp(Config.Log.referer, "none"))) {
+	debug(40, 1) ("Referer logging is disabled.\n");
+	return;
+    }
+    refererlog = logfileOpen(Config.Log.referer, 0, 1);
+#endif
+}
+
+void
+refererRotateLog(void)
+{
+#if USE_REFERER_LOG
+    if (NULL == refererlog)
+	return;
+    logfileRotate(refererlog);
+#endif
+}
+
+void
+logReferer(const char *client, const char *referer, const char *uri)
+{
+#if USE_REFERER_LOG
+    if (NULL == refererlog)
+	return;
+    logfilePrintf(refererlog, "%9d.%03d %s %s %s\n",
+	(int) current_time.tv_sec,
+	(int) current_time.tv_usec / 1000,
+	client,
+	referer,
+	uri ? uri : "-");
+#endif
+}
--- squid/src/refresh.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,514 +0,0 @@
-
-/*
- * $Id: refresh.c,v 1.8.14.1 2002/10/04 07:07:27 rbcollins Exp $
- *
- * DEBUG: section 22    Refresh Calculation
- * AUTHOR: Harvest Derived
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#ifndef USE_POSIX_REGEX
-#define USE_POSIX_REGEX		/* put before includes; always use POSIX */
-#endif
-
-#include "squid.h"
-#include "Store.h"
-
-typedef enum {
-    rcHTTP,
-    rcICP,
-#if USE_HTCP
-    rcHTCP,
-#endif
-#if USE_CACHE_DIGESTS
-    rcCDigest,
-#endif
-    rcStore,
-    rcCount
-} refreshCountsEnum;
-
-typedef struct {
-    unsigned int expires:1;
-    unsigned int min:1;
-    unsigned int lmfactor:1;
-    unsigned int max;
-} stale_flags;
-
-/*
- * This enumerated list assigns specific values, ala HTTP/FTP status
- * codes.  All Fresh codes are in the range 100-199 and all stale
- * codes are 200-299.  We might want to use these codes in logging,
- * so best to keep them consistent over time.
- */
-enum {
-    FRESH_REQUEST_MAX_STALE_ALL = 100,
-    FRESH_REQUEST_MAX_STALE_VALUE,
-    FRESH_EXPIRES,
-    FRESH_LMFACTOR_RULE,
-    FRESH_MIN_RULE,
-    FRESH_OVERRIDE_EXPIRES,
-    FRESH_OVERRIDE_LASTMOD,
-    STALE_MUST_REVALIDATE = 200,
-    STALE_RELOAD_INTO_IMS,
-    STALE_FORCED_RELOAD,
-    STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
-    STALE_EXPIRES,
-    STALE_MAX_RULE,
-    STALE_LMFACTOR_RULE,
-    STALE_DEFAULT = 299
-};
-
-static struct RefreshCounts {
-    const char *proto;
-    int total;
-    int status[STALE_DEFAULT + 1];
-} refreshCounts[rcCount];
-
-/*
- * Defaults:
- *      MIN     NONE
- *      PCT     20%
- *      MAX     3 days
- */
-#define REFRESH_DEFAULT_MIN	(time_t)0
-#define REFRESH_DEFAULT_PCT	0.20
-#define REFRESH_DEFAULT_MAX	(time_t)259200
-
-static const refresh_t *refreshLimits(const char *);
-static const refresh_t *refreshUncompiledPattern(const char *);
-static OBJH refreshStats;
-static int refreshStaleness(const StoreEntry *, time_t, time_t, const refresh_t *, stale_flags *);
-
-static refresh_t DefaultRefresh;
-
-static const refresh_t *
-refreshLimits(const char *url)
-{
-    const refresh_t *R;
-    for (R = Config.Refresh; R; R = R->next) {
-	if (!regexec(&(R->compiled_pattern), url, 0, 0, 0))
-	    return R;
-    }
-    return NULL;
-}
-
-static const refresh_t *
-refreshUncompiledPattern(const char *pat)
-{
-    const refresh_t *R;
-    for (R = Config.Refresh; R; R = R->next) {
-	if (0 == strcmp(R->pattern, pat))
-	    return R;
-    }
-    return NULL;
-}
-
-/*
- * Calculate how stale the response is (or will be at the check_time).
- * Staleness calculation is based on the following: (1) response
- * expiration time, (2) age greater than configured maximum, (3)
- * last-modified factor, and (4) age less than configured minimum.
- *
- * If the response is fresh, return -1.  Otherwise return its
- * staleness.  NOTE return value of 0 means the response is stale.
- *
- * The 'stale_flags' structure is used to tell the calling function
- * _why_ this response is fresh or stale.  Its used, for example,
- * when the admin wants to override expiration and last-modified
- * times.
- */
-static int
-refreshStaleness(const StoreEntry * entry, time_t check_time, time_t age, const refresh_t * R, stale_flags * sf)
-{
-    /*
-     * Check for an explicit expiration time.
-     */
-    if (entry->expires > -1) {
-	sf->expires = 1;
-	if (entry->expires > check_time) {
-	    debug(22, 3) ("FRESH: expires %d >= check_time %d \n",
-		(int) entry->expires, (int) check_time);
-	    return -1;
-	} else {
-	    debug(22, 3) ("STALE: expires %d < check_time %d \n",
-		(int) entry->expires, (int) check_time);
-	    return (check_time - entry->expires);
-	}
-    }
-    assert(age >= 0);
-    /*
-     * Use local heuristics to determine staleness.  Start with the
-     * max age from the refresh_pattern rule.
-     */
-    if (age > R->max) {
-	debug(22, 3) ("STALE: age %d > max %d \n", (int) age, (int) R->max);
-	sf->max = 1;
-	return (age - R->max);
-    }
-    /*
-     * Try the last-modified factor algorithm.
-     */
-    if (entry->lastmod > -1 && entry->timestamp > entry->lastmod) {
-	/*
-	 * stale_age is the Age of the response when it became/becomes
-	 * stale according to the last-modified factor algorithm.
-	 */
-	time_t stale_age = (entry->timestamp - entry->lastmod) * R->pct;
-	sf->lmfactor = 1;
-	if (age >= stale_age) {
-	    debug(22, 3) ("STALE: age %d > stale_age %d\n",
-		(int) age, (int) stale_age);
-	    return (age - stale_age);
-	} else {
-	    debug(22, 3) ("FRESH: age %d <= stale_age %d\n",
-		(int) age, (int) stale_age);
-	    return -1;
-	}
-    }
-    /*
-     * If we are here, staleness is determined by the refresh_pattern
-     * configured minimum age.
-     */
-    if (age <= R->min) {
-	debug(22, 3) ("FRESH: age %d <= min %d\n", (int) age, (int) R->min);
-	sf->min = 1;
-	return -1;
-    }
-    debug(22, 3) ("STALE: age %d > min %d\n", (int) age, (int) R->min);
-    return (age - R->min);
-}
-
-/*  return 1 if the entry must be revalidated within delta seconds
- *         0 otherwise
- *
- *  note: request maybe null (e.g. for cache digests build)
- */
-static int
-refreshCheck(const StoreEntry * entry, request_t * request, time_t delta)
-{
-    const refresh_t *R;
-    const char *uri = NULL;
-    time_t age = 0;
-    time_t check_time = squid_curtime + delta;
-    int staleness;
-    stale_flags sf;
-    if (entry->mem_obj)
-	uri = entry->mem_obj->url;
-    else if (request)
-	uri = urlCanonical(request);
-
-    debug(22, 3) ("refreshCheck: '%s'\n", uri ? uri : "");
-
-    if (check_time > entry->timestamp)
-	age = check_time - entry->timestamp;
-    R = uri ? refreshLimits(uri) : refreshUncompiledPattern(".");
-    if (NULL == R)
-	R = &DefaultRefresh;
-    memset(&sf, '\0', sizeof(sf));
-    staleness = refreshStaleness(entry, check_time, age, R, &sf);
-    debug(22, 3) ("Staleness = %d\n", staleness);
-
-    debug(22, 3) ("refreshCheck: Matched '%s %d %d%% %d'\n",
-	R->pattern, (int) R->min, (int) (100.0 * R->pct), (int) R->max);
-    debug(22, 3) ("refreshCheck: age = %d\n", (int) age);
-    debug(22, 3) ("\tcheck_time:\t%s\n", mkrfc1123(check_time));
-    debug(22, 3) ("\tentry->timestamp:\t%s\n", mkrfc1123(entry->timestamp));
-
-    if (EBIT_TEST(entry->flags, ENTRY_REVALIDATE) && staleness > -1) {
-	debug(22, 3) ("refreshCheck: YES: Must revalidate stale response\n");
-	return STALE_MUST_REVALIDATE;
-    }
-    /* request-specific checks */
-    if (request) {
-	HttpHdrCc *cc = request->cache_control;
-#if HTTP_VIOLATIONS
-	if (!request->flags.nocache_hack) {
-	    (void) 0;
-	} else if (R->flags.ignore_reload) {
-	    /* The clients no-cache header is ignored */
-	    debug(22, 3) ("refreshCheck: MAYBE: ignore-reload\n");
-	} else if (R->flags.reload_into_ims || Config.onoff.reload_into_ims) {
-	    /* The clients no-cache header is changed into a IMS query */
-	    debug(22, 3) ("refreshCheck: YES: reload-into-ims\n");
-	    return STALE_RELOAD_INTO_IMS;
-	} else {
-	    /* The clients no-cache header is not overridden on this request */
-	    debug(22, 3) ("refreshCheck: YES: client reload\n");
-	    request->flags.nocache = 1;
-	    return STALE_FORCED_RELOAD;
-	}
-#endif
-	if (NULL != cc) {
-	    if (cc->max_age > -1) {
-#if HTTP_VIOLATIONS
-		if (R->flags.ignore_reload && cc->max_age == 0) {
-		} else
-#endif
-		if (age > cc->max_age) {
-		    debug(22, 3) ("refreshCheck: YES: age > client-max-age\n");
-		    return STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE;
-		}
-	    }
-	    if (EBIT_TEST(cc->mask, CC_MAX_STALE) && staleness > -1) {
-		if (cc->max_stale < 0) {
-		    /* max-stale directive without a value */
-		    debug(22, 3) ("refreshCheck: NO: max-stale wildcard\n");
-		    return FRESH_REQUEST_MAX_STALE_ALL;
-		} else if (staleness < cc->max_stale) {
-		    debug(22, 3) ("refreshCheck: NO: staleness < max-stale\n");
-		    return FRESH_REQUEST_MAX_STALE_VALUE;
-		}
-	    }
-	}
-    }
-    if (-1 == staleness) {
-	if (sf.expires)
-	    return FRESH_EXPIRES;
-	assert(!sf.max);
-	if (sf.lmfactor)
-	    return FRESH_LMFACTOR_RULE;
-	assert(sf.min);
-	return FRESH_MIN_RULE;
-    }
-    /*
-     * At this point the response is stale, unless one of
-     * the override options kicks in.
-     */
-    if (sf.expires) {
-#if HTTP_VIOLATIONS
-	if (R->flags.override_expire && age < R->min) {
-	    debug(22, 3) ("refreshCheck: NO: age < min && override-expire\n");
-	    return FRESH_OVERRIDE_EXPIRES;
-	}
-#endif
-	return STALE_EXPIRES;
-    }
-    if (sf.max)
-	return STALE_MAX_RULE;
-    if (sf.lmfactor) {
-#if HTTP_VIOLATIONS
-	if (R->flags.override_lastmod && age < R->min) {
-	    debug(22, 3) ("refreshCheck: NO: age < min && override-lastmod\n");
-	    return FRESH_OVERRIDE_LASTMOD;
-	}
-#endif
-	return STALE_LMFACTOR_RULE;
-    }
-    return STALE_DEFAULT;
-}
-
-int
-refreshIsCachable(const StoreEntry * entry)
-{
-    /*
-     * Don't look at the request to avoid no-cache and other nuisances.
-     * the object should have a mem_obj so the URL will be found there.
-     * 60 seconds delta, to avoid objects which expire almost
-     * immediately, and which can't be refreshed.
-     */
-    int reason = refreshCheck(entry, NULL, 60);
-    refreshCounts[rcStore].total++;
-    refreshCounts[rcStore].status[reason]++;
-    if (reason < 200)
-	/* Does not need refresh. This is certainly cachable */
-	return 1;
-    if (entry->lastmod < 0)
-	/* Last modified is needed to do a refresh */
-	return 0;
-    if (entry->mem_obj == NULL)
-	/* no mem_obj? */
-	return 1;
-    if (entry->mem_obj->reply == NULL)
-	/* no reply? */
-	return 1;
-    if (entry->mem_obj->reply->content_length == 0)
-	/* No use refreshing (caching?) 0 byte objects */
-	return 0;
-    /* This seems to be refreshable. Cache it */
-    return 1;
-}
-
-/* refreshCheck... functions below are protocol-specific wrappers around
- * refreshCheck() function above */
-
-int
-refreshCheckHTTP(const StoreEntry * entry, request_t * request)
-{
-    int reason = refreshCheck(entry, request, 0);
-    refreshCounts[rcHTTP].total++;
-    refreshCounts[rcHTTP].status[reason]++;
-    return (reason < 200) ? 0 : 1;
-}
-
-int
-refreshCheckICP(const StoreEntry * entry, request_t * request)
-{
-    int reason = refreshCheck(entry, request, 30);
-    refreshCounts[rcICP].total++;
-    refreshCounts[rcICP].status[reason]++;
-    return (reason < 200) ? 0 : 1;
-}
-
-#if USE_HTCP
-int
-refreshCheckHTCP(const StoreEntry * entry, request_t * request)
-{
-    int reason = refreshCheck(entry, request, 10);
-    refreshCounts[rcHTCP].total++;
-    refreshCounts[rcHTCP].status[reason]++;
-    return (reason < 200) ? 0 : 1;
-}
-#endif
-
-#if USE_CACHE_DIGESTS
-int
-refreshCheckDigest(const StoreEntry * entry, time_t delta)
-{
-    int reason = refreshCheck(entry,
-	entry->mem_obj ? entry->mem_obj->request : NULL,
-	delta);
-    refreshCounts[rcCDigest].total++;
-    refreshCounts[rcCDigest].status[reason]++;
-    return (reason < 200) ? 0 : 1;
-}
-#endif
-
-time_t
-getMaxAge(const char *url)
-{
-    const refresh_t *R;
-    debug(22, 3) ("getMaxAge: '%s'\n", url);
-    if ((R = refreshLimits(url)))
-	return R->max;
-    else
-	return REFRESH_DEFAULT_MAX;
-}
-
-static void
-refreshCountsStats(StoreEntry * sentry, struct RefreshCounts *rc)
-{
-    int sum = 0;
-    int tot = rc->total;
-
-    storeAppendPrintf(sentry, "\n\n%s histogram:\n", rc->proto);
-    storeAppendPrintf(sentry, "Count\t%%Total\tCategory\n");
-
-#define refreshCountsStatsEntry(code,desc) { \
-	storeAppendPrintf(sentry, "%6d\t%6.2f\t%s\n", \
-	    rc->status[code], xpercent(rc->status[code], tot), desc); \
-    sum += rc->status[code]; \
-}
-
-    refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_ALL,
-	"Fresh: request max-stale wildcard");
-    refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_VALUE,
-	"Fresh: request max-stale value");
-    refreshCountsStatsEntry(FRESH_EXPIRES,
-	"Fresh: expires time not reached");
-    refreshCountsStatsEntry(FRESH_LMFACTOR_RULE,
-	"Fresh: refresh_pattern last-mod factor percentage");
-    refreshCountsStatsEntry(FRESH_MIN_RULE,
-	"Fresh: refresh_pattern min value");
-    refreshCountsStatsEntry(FRESH_OVERRIDE_EXPIRES,
-	"Fresh: refresh_pattern override expires");
-    refreshCountsStatsEntry(FRESH_OVERRIDE_LASTMOD,
-	"Fresh: refresh_pattern override lastmod");
-    refreshCountsStatsEntry(STALE_MUST_REVALIDATE,
-	"Stale: response has must-revalidate");
-    refreshCountsStatsEntry(STALE_RELOAD_INTO_IMS,
-	"Stale: changed reload into IMS");
-    refreshCountsStatsEntry(STALE_FORCED_RELOAD,
-	"Stale: request has no-cache directive");
-    refreshCountsStatsEntry(STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
-	"Stale: age exceeds request max-age value");
-    refreshCountsStatsEntry(STALE_EXPIRES,
-	"Stale: expires time reached");
-    refreshCountsStatsEntry(STALE_MAX_RULE,
-	"Stale: refresh_pattern max age rule");
-    refreshCountsStatsEntry(STALE_LMFACTOR_RULE,
-	"Stale: refresh_pattern last-mod factor percentage");
-    refreshCountsStatsEntry(STALE_DEFAULT,
-	"Stale: by default");
-
-    tot = sum;			/* paranoid: "total" line shows 100% if we forgot nothing */
-    storeAppendPrintf(sentry, "%6d\t%6.2f\tTOTAL\n",
-	rc->total, xpercent(rc->total, tot));
-    \
-	storeAppendPrintf(sentry, "\n");
-}
-
-static void
-refreshStats(StoreEntry * sentry)
-{
-    int i;
-    int total = 0;
-
-    /* get total usage count */
-    for (i = 0; i < rcCount; ++i)
-	total += refreshCounts[i].total;
-
-    /* protocol usage histogram */
-    storeAppendPrintf(sentry, "\nRefreshCheck calls per protocol\n\n");
-    storeAppendPrintf(sentry, "Protocol\t#Calls\t%%Calls\n");
-    for (i = 0; i < rcCount; ++i)
-	storeAppendPrintf(sentry, "%10s\t%6d\t%6.2f\n",
-	    refreshCounts[i].proto,
-	    refreshCounts[i].total,
-	    xpercent(refreshCounts[i].total, total));
-
-    /* per protocol histograms */
-    storeAppendPrintf(sentry, "\n\nRefreshCheck histograms for various protocols\n");
-    for (i = 0; i < rcCount; ++i)
-	refreshCountsStats(sentry, &refreshCounts[i]);
-}
-
-void
-refreshInit(void)
-{
-    memset(refreshCounts, 0, sizeof(refreshCounts));
-    refreshCounts[rcHTTP].proto = "HTTP";
-    refreshCounts[rcICP].proto = "ICP";
-#if USE_HTCP
-    refreshCounts[rcHTCP].proto = "HTCP";
-#endif
-    refreshCounts[rcStore].proto = "On Store";
-#if USE_CACHE_DIGESTS
-    refreshCounts[rcCDigest].proto = "Cache Digests";
-#endif
-    cachemgrRegister("refresh",
-	"Refresh Algorithm Statistics",
-	refreshStats,
-	0,
-	1);
-    memset(&DefaultRefresh, '\0', sizeof(DefaultRefresh));
-    DefaultRefresh.pattern = "";
-    DefaultRefresh.min = REFRESH_DEFAULT_MIN;
-    DefaultRefresh.pct = REFRESH_DEFAULT_PCT;
-    DefaultRefresh.max = REFRESH_DEFAULT_MAX;
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/refresh.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,514 @@
+
+/*
+ * $Id: refresh.cc,v 1.1.2.1 2002/10/11 15:41:01 rbcollins Exp $
+ *
+ * DEBUG: section 22    Refresh Calculation
+ * AUTHOR: Harvest Derived
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#ifndef USE_POSIX_REGEX
+#define USE_POSIX_REGEX		/* put before includes; always use POSIX */
+#endif
+
+#include "squid.h"
+#include "Store.h"
+
+typedef enum {
+    rcHTTP,
+    rcICP,
+#if USE_HTCP
+    rcHTCP,
+#endif
+#if USE_CACHE_DIGESTS
+    rcCDigest,
+#endif
+    rcStore,
+    rcCount
+} refreshCountsEnum;
+
+typedef struct {
+    unsigned int expires:1;
+    unsigned int min:1;
+    unsigned int lmfactor:1;
+    unsigned int max;
+} stale_flags;
+
+/*
+ * This enumerated list assigns specific values, ala HTTP/FTP status
+ * codes.  All Fresh codes are in the range 100-199 and all stale
+ * codes are 200-299.  We might want to use these codes in logging,
+ * so best to keep them consistent over time.
+ */
+enum {
+    FRESH_REQUEST_MAX_STALE_ALL = 100,
+    FRESH_REQUEST_MAX_STALE_VALUE,
+    FRESH_EXPIRES,
+    FRESH_LMFACTOR_RULE,
+    FRESH_MIN_RULE,
+    FRESH_OVERRIDE_EXPIRES,
+    FRESH_OVERRIDE_LASTMOD,
+    STALE_MUST_REVALIDATE = 200,
+    STALE_RELOAD_INTO_IMS,
+    STALE_FORCED_RELOAD,
+    STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
+    STALE_EXPIRES,
+    STALE_MAX_RULE,
+    STALE_LMFACTOR_RULE,
+    STALE_DEFAULT = 299
+};
+
+static struct RefreshCounts {
+    const char *proto;
+    int total;
+    int status[STALE_DEFAULT + 1];
+} refreshCounts[rcCount];
+
+/*
+ * Defaults:
+ *      MIN     NONE
+ *      PCT     20%
+ *      MAX     3 days
+ */
+#define REFRESH_DEFAULT_MIN	(time_t)0
+#define REFRESH_DEFAULT_PCT	0.20
+#define REFRESH_DEFAULT_MAX	(time_t)259200
+
+static const refresh_t *refreshLimits(const char *);
+static const refresh_t *refreshUncompiledPattern(const char *);
+static OBJH refreshStats;
+static int refreshStaleness(const StoreEntry *, time_t, time_t, const refresh_t *, stale_flags *);
+
+static refresh_t DefaultRefresh;
+
+static const refresh_t *
+refreshLimits(const char *url)
+{
+    const refresh_t *R;
+    for (R = Config.Refresh; R; R = R->next) {
+	if (!regexec(&(R->compiled_pattern), url, 0, 0, 0))
+	    return R;
+    }
+    return NULL;
+}
+
+static const refresh_t *
+refreshUncompiledPattern(const char *pat)
+{
+    const refresh_t *R;
+    for (R = Config.Refresh; R; R = R->next) {
+	if (0 == strcmp(R->pattern, pat))
+	    return R;
+    }
+    return NULL;
+}
+
+/*
+ * Calculate how stale the response is (or will be at the check_time).
+ * Staleness calculation is based on the following: (1) response
+ * expiration time, (2) age greater than configured maximum, (3)
+ * last-modified factor, and (4) age less than configured minimum.
+ *
+ * If the response is fresh, return -1.  Otherwise return its
+ * staleness.  NOTE return value of 0 means the response is stale.
+ *
+ * The 'stale_flags' structure is used to tell the calling function
+ * _why_ this response is fresh or stale.  Its used, for example,
+ * when the admin wants to override expiration and last-modified
+ * times.
+ */
+static int
+refreshStaleness(const StoreEntry * entry, time_t check_time, time_t age, const refresh_t * R, stale_flags * sf)
+{
+    /*
+     * Check for an explicit expiration time.
+     */
+    if (entry->expires > -1) {
+	sf->expires = 1;
+	if (entry->expires > check_time) {
+	    debug(22, 3) ("FRESH: expires %d >= check_time %d \n",
+		(int) entry->expires, (int) check_time);
+	    return -1;
+	} else {
+	    debug(22, 3) ("STALE: expires %d < check_time %d \n",
+		(int) entry->expires, (int) check_time);
+	    return (check_time - entry->expires);
+	}
+    }
+    assert(age >= 0);
+    /*
+     * Use local heuristics to determine staleness.  Start with the
+     * max age from the refresh_pattern rule.
+     */
+    if (age > R->max) {
+	debug(22, 3) ("STALE: age %d > max %d \n", (int) age, (int) R->max);
+	sf->max = 1;
+	return (age - R->max);
+    }
+    /*
+     * Try the last-modified factor algorithm.
+     */
+    if (entry->lastmod > -1 && entry->timestamp > entry->lastmod) {
+	/*
+	 * stale_age is the Age of the response when it became/becomes
+	 * stale according to the last-modified factor algorithm.
+	 */
+	time_t stale_age = static_cast((entry->timestamp - entry->lastmod) * R->pct);
+	sf->lmfactor = 1;
+	if (age >= stale_age) {
+	    debug(22, 3) ("STALE: age %d > stale_age %d\n",
+		(int) age, (int) stale_age);
+	    return (age - stale_age);
+	} else {
+	    debug(22, 3) ("FRESH: age %d <= stale_age %d\n",
+		(int) age, (int) stale_age);
+	    return -1;
+	}
+    }
+    /*
+     * If we are here, staleness is determined by the refresh_pattern
+     * configured minimum age.
+     */
+    if (age <= R->min) {
+	debug(22, 3) ("FRESH: age %d <= min %d\n", (int) age, (int) R->min);
+	sf->min = 1;
+	return -1;
+    }
+    debug(22, 3) ("STALE: age %d > min %d\n", (int) age, (int) R->min);
+    return (age - R->min);
+}
+
+/*  return 1 if the entry must be revalidated within delta seconds
+ *         0 otherwise
+ *
+ *  note: request maybe null (e.g. for cache digests build)
+ */
+static int
+refreshCheck(const StoreEntry * entry, request_t * request, time_t delta)
+{
+    const refresh_t *R;
+    const char *uri = NULL;
+    time_t age = 0;
+    time_t check_time = squid_curtime + delta;
+    int staleness;
+    stale_flags sf;
+    if (entry->mem_obj)
+	uri = entry->mem_obj->url;
+    else if (request)
+	uri = urlCanonical(request);
+
+    debug(22, 3) ("refreshCheck: '%s'\n", uri ? uri : "");
+
+    if (check_time > entry->timestamp)
+	age = check_time - entry->timestamp;
+    R = uri ? refreshLimits(uri) : refreshUncompiledPattern(".");
+    if (NULL == R)
+	R = &DefaultRefresh;
+    memset(&sf, '\0', sizeof(sf));
+    staleness = refreshStaleness(entry, check_time, age, R, &sf);
+    debug(22, 3) ("Staleness = %d\n", staleness);
+
+    debug(22, 3) ("refreshCheck: Matched '%s %d %d%% %d'\n",
+	R->pattern, (int) R->min, (int) (100.0 * R->pct), (int) R->max);
+    debug(22, 3) ("refreshCheck: age = %d\n", (int) age);
+    debug(22, 3) ("\tcheck_time:\t%s\n", mkrfc1123(check_time));
+    debug(22, 3) ("\tentry->timestamp:\t%s\n", mkrfc1123(entry->timestamp));
+
+    if (EBIT_TEST(entry->flags, ENTRY_REVALIDATE) && staleness > -1) {
+	debug(22, 3) ("refreshCheck: YES: Must revalidate stale response\n");
+	return STALE_MUST_REVALIDATE;
+    }
+    /* request-specific checks */
+    if (request) {
+	HttpHdrCc *cc = request->cache_control;
+#if HTTP_VIOLATIONS
+	if (!request->flags.nocache_hack) {
+	    (void) 0;
+	} else if (R->flags.ignore_reload) {
+	    /* The clients no-cache header is ignored */
+	    debug(22, 3) ("refreshCheck: MAYBE: ignore-reload\n");
+	} else if (R->flags.reload_into_ims || Config.onoff.reload_into_ims) {
+	    /* The clients no-cache header is changed into a IMS query */
+	    debug(22, 3) ("refreshCheck: YES: reload-into-ims\n");
+	    return STALE_RELOAD_INTO_IMS;
+	} else {
+	    /* The clients no-cache header is not overridden on this request */
+	    debug(22, 3) ("refreshCheck: YES: client reload\n");
+	    request->flags.nocache = 1;
+	    return STALE_FORCED_RELOAD;
+	}
+#endif
+	if (NULL != cc) {
+	    if (cc->max_age > -1) {
+#if HTTP_VIOLATIONS
+		if (R->flags.ignore_reload && cc->max_age == 0) {
+		} else
+#endif
+		if (age > cc->max_age) {
+		    debug(22, 3) ("refreshCheck: YES: age > client-max-age\n");
+		    return STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE;
+		}
+	    }
+	    if (EBIT_TEST(cc->mask, CC_MAX_STALE) && staleness > -1) {
+		if (cc->max_stale < 0) {
+		    /* max-stale directive without a value */
+		    debug(22, 3) ("refreshCheck: NO: max-stale wildcard\n");
+		    return FRESH_REQUEST_MAX_STALE_ALL;
+		} else if (staleness < cc->max_stale) {
+		    debug(22, 3) ("refreshCheck: NO: staleness < max-stale\n");
+		    return FRESH_REQUEST_MAX_STALE_VALUE;
+		}
+	    }
+	}
+    }
+    if (-1 == staleness) {
+	if (sf.expires)
+	    return FRESH_EXPIRES;
+	assert(!sf.max);
+	if (sf.lmfactor)
+	    return FRESH_LMFACTOR_RULE;
+	assert(sf.min);
+	return FRESH_MIN_RULE;
+    }
+    /*
+     * At this point the response is stale, unless one of
+     * the override options kicks in.
+     */
+    if (sf.expires) {
+#if HTTP_VIOLATIONS
+	if (R->flags.override_expire && age < R->min) {
+	    debug(22, 3) ("refreshCheck: NO: age < min && override-expire\n");
+	    return FRESH_OVERRIDE_EXPIRES;
+	}
+#endif
+	return STALE_EXPIRES;
+    }
+    if (sf.max)
+	return STALE_MAX_RULE;
+    if (sf.lmfactor) {
+#if HTTP_VIOLATIONS
+	if (R->flags.override_lastmod && age < R->min) {
+	    debug(22, 3) ("refreshCheck: NO: age < min && override-lastmod\n");
+	    return FRESH_OVERRIDE_LASTMOD;
+	}
+#endif
+	return STALE_LMFACTOR_RULE;
+    }
+    return STALE_DEFAULT;
+}
+
+int
+refreshIsCachable(const StoreEntry * entry)
+{
+    /*
+     * Don't look at the request to avoid no-cache and other nuisances.
+     * the object should have a mem_obj so the URL will be found there.
+     * 60 seconds delta, to avoid objects which expire almost
+     * immediately, and which can't be refreshed.
+     */
+    int reason = refreshCheck(entry, NULL, 60);
+    refreshCounts[rcStore].total++;
+    refreshCounts[rcStore].status[reason]++;
+    if (reason < 200)
+	/* Does not need refresh. This is certainly cachable */
+	return 1;
+    if (entry->lastmod < 0)
+	/* Last modified is needed to do a refresh */
+	return 0;
+    if (entry->mem_obj == NULL)
+	/* no mem_obj? */
+	return 1;
+    if (entry->mem_obj->reply == NULL)
+	/* no reply? */
+	return 1;
+    if (entry->mem_obj->reply->content_length == 0)
+	/* No use refreshing (caching?) 0 byte objects */
+	return 0;
+    /* This seems to be refreshable. Cache it */
+    return 1;
+}
+
+/* refreshCheck... functions below are protocol-specific wrappers around
+ * refreshCheck() function above */
+
+int
+refreshCheckHTTP(const StoreEntry * entry, request_t * request)
+{
+    int reason = refreshCheck(entry, request, 0);
+    refreshCounts[rcHTTP].total++;
+    refreshCounts[rcHTTP].status[reason]++;
+    return (reason < 200) ? 0 : 1;
+}
+
+int
+refreshCheckICP(const StoreEntry * entry, request_t * request)
+{
+    int reason = refreshCheck(entry, request, 30);
+    refreshCounts[rcICP].total++;
+    refreshCounts[rcICP].status[reason]++;
+    return (reason < 200) ? 0 : 1;
+}
+
+#if USE_HTCP
+int
+refreshCheckHTCP(const StoreEntry * entry, request_t * request)
+{
+    int reason = refreshCheck(entry, request, 10);
+    refreshCounts[rcHTCP].total++;
+    refreshCounts[rcHTCP].status[reason]++;
+    return (reason < 200) ? 0 : 1;
+}
+#endif
+
+#if USE_CACHE_DIGESTS
+int
+refreshCheckDigest(const StoreEntry * entry, time_t delta)
+{
+    int reason = refreshCheck(entry,
+	entry->mem_obj ? entry->mem_obj->request : NULL,
+	delta);
+    refreshCounts[rcCDigest].total++;
+    refreshCounts[rcCDigest].status[reason]++;
+    return (reason < 200) ? 0 : 1;
+}
+#endif
+
+time_t
+getMaxAge(const char *url)
+{
+    const refresh_t *R;
+    debug(22, 3) ("getMaxAge: '%s'\n", url);
+    if ((R = refreshLimits(url)))
+	return R->max;
+    else
+	return REFRESH_DEFAULT_MAX;
+}
+
+static void
+refreshCountsStats(StoreEntry * sentry, struct RefreshCounts *rc)
+{
+    int sum = 0;
+    int tot = rc->total;
+
+    storeAppendPrintf(sentry, "\n\n%s histogram:\n", rc->proto);
+    storeAppendPrintf(sentry, "Count\t%%Total\tCategory\n");
+
+#define refreshCountsStatsEntry(code,desc) { \
+	storeAppendPrintf(sentry, "%6d\t%6.2f\t%s\n", \
+	    rc->status[code], xpercent(rc->status[code], tot), desc); \
+    sum += rc->status[code]; \
+}
+
+    refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_ALL,
+	"Fresh: request max-stale wildcard");
+    refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_VALUE,
+	"Fresh: request max-stale value");
+    refreshCountsStatsEntry(FRESH_EXPIRES,
+	"Fresh: expires time not reached");
+    refreshCountsStatsEntry(FRESH_LMFACTOR_RULE,
+	"Fresh: refresh_pattern last-mod factor percentage");
+    refreshCountsStatsEntry(FRESH_MIN_RULE,
+	"Fresh: refresh_pattern min value");
+    refreshCountsStatsEntry(FRESH_OVERRIDE_EXPIRES,
+	"Fresh: refresh_pattern override expires");
+    refreshCountsStatsEntry(FRESH_OVERRIDE_LASTMOD,
+	"Fresh: refresh_pattern override lastmod");
+    refreshCountsStatsEntry(STALE_MUST_REVALIDATE,
+	"Stale: response has must-revalidate");
+    refreshCountsStatsEntry(STALE_RELOAD_INTO_IMS,
+	"Stale: changed reload into IMS");
+    refreshCountsStatsEntry(STALE_FORCED_RELOAD,
+	"Stale: request has no-cache directive");
+    refreshCountsStatsEntry(STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
+	"Stale: age exceeds request max-age value");
+    refreshCountsStatsEntry(STALE_EXPIRES,
+	"Stale: expires time reached");
+    refreshCountsStatsEntry(STALE_MAX_RULE,
+	"Stale: refresh_pattern max age rule");
+    refreshCountsStatsEntry(STALE_LMFACTOR_RULE,
+	"Stale: refresh_pattern last-mod factor percentage");
+    refreshCountsStatsEntry(STALE_DEFAULT,
+	"Stale: by default");
+
+    tot = sum;			/* paranoid: "total" line shows 100% if we forgot nothing */
+    storeAppendPrintf(sentry, "%6d\t%6.2f\tTOTAL\n",
+	rc->total, xpercent(rc->total, tot));
+    \
+	storeAppendPrintf(sentry, "\n");
+}
+
+static void
+refreshStats(StoreEntry * sentry)
+{
+    int i;
+    int total = 0;
+
+    /* get total usage count */
+    for (i = 0; i < rcCount; ++i)
+	total += refreshCounts[i].total;
+
+    /* protocol usage histogram */
+    storeAppendPrintf(sentry, "\nRefreshCheck calls per protocol\n\n");
+    storeAppendPrintf(sentry, "Protocol\t#Calls\t%%Calls\n");
+    for (i = 0; i < rcCount; ++i)
+	storeAppendPrintf(sentry, "%10s\t%6d\t%6.2f\n",
+	    refreshCounts[i].proto,
+	    refreshCounts[i].total,
+	    xpercent(refreshCounts[i].total, total));
+
+    /* per protocol histograms */
+    storeAppendPrintf(sentry, "\n\nRefreshCheck histograms for various protocols\n");
+    for (i = 0; i < rcCount; ++i)
+	refreshCountsStats(sentry, &refreshCounts[i]);
+}
+
+void
+refreshInit(void)
+{
+    memset(refreshCounts, 0, sizeof(refreshCounts));
+    refreshCounts[rcHTTP].proto = "HTTP";
+    refreshCounts[rcICP].proto = "ICP";
+#if USE_HTCP
+    refreshCounts[rcHTCP].proto = "HTCP";
+#endif
+    refreshCounts[rcStore].proto = "On Store";
+#if USE_CACHE_DIGESTS
+    refreshCounts[rcCDigest].proto = "Cache Digests";
+#endif
+    cachemgrRegister("refresh",
+	"Refresh Algorithm Statistics",
+	refreshStats,
+	0,
+	1);
+    memset(&DefaultRefresh, '\0', sizeof(DefaultRefresh));
+    DefaultRefresh.pattern = "";
+    DefaultRefresh.min = REFRESH_DEFAULT_MIN;
+    DefaultRefresh.pct = REFRESH_DEFAULT_PCT;
+    DefaultRefresh.max = REFRESH_DEFAULT_MAX;
+}
Index: squid/src/snmp_agent.cc
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/Attic/snmp_agent.cc,v
retrieving revision 1.1.2.1
retrieving revision 1.1.2.2
diff -u -r1.1.2.1 -r1.1.2.2
--- squid/src/snmp_agent.cc	9 Oct 2002 00:23:16 -0000	1.1.2.1
+++ squid/src/snmp_agent.cc	11 Oct 2002 15:41:01 -0000	1.1.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: snmp_agent.cc,v 1.1.2.1 2002/10/09 00:23:16 rbcollins Exp $
+ * $Id: snmp_agent.cc,v 1.1.2.2 2002/10/11 15:41:01 rbcollins Exp $
  *
  * DEBUG: section 49     SNMP Interface
  * AUTHOR: Kostas Anagnostakis
@@ -36,6 +36,7 @@
 
 #include "squid.h"
 #include "cache_snmp.h"
+#include "Store.h"
 
 /************************************************************************
 
@@ -63,7 +64,7 @@
 	break;
     case SYS_UPTIME:
 	Answer = snmp_var_new_integer(Var->name, Var->name_length,
-	    tvSubDsec(squid_start, current_time) * 100,
+	    (int)(tvSubDsec(squid_start, current_time) * 100),
 	    SMI_TIMETICKS);
 	break;
     default:
@@ -299,7 +300,7 @@
 	break;
     case PERF_SYS_NUMOBJCNT:
 	Answer = snmp_var_new_integer(Var->name, Var->name_length,
-	    (snint) memInUse(MEM_STOREENTRY),
+	    (snint) _StoreEntry::inUseCount(),
 	    SMI_COUNTER32);
 	break;
     default:
--- squid/src/snmp_core.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,1071 +0,0 @@
-
-/*
- * $Id: snmp_core.c,v 1.10 2002/02/13 17:24:42 squidadm Exp $
- *
- * DEBUG: section 49    SNMP support
- * AUTHOR: Glenn Chisholm
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-#include "squid.h"
-#include "cache_snmp.h"
-
-#define SNMP_REQUEST_SIZE 4096
-#define MAX_PROTOSTAT 5
-
-typedef struct _mib_tree_entry mib_tree_entry;
-typedef oid *(instance_Fn) (oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
-
-struct _mib_tree_entry {
-    oid *name;
-    int len;
-    oid_ParseFn *parsefunction;
-    instance_Fn *instancefunction;
-    int children;
-    struct _mib_tree_entry **leaves;
-    struct _mib_tree_entry *parent;
-};
-
-mib_tree_entry *mib_tree_head;
-mib_tree_entry *mib_tree_last;
-
-#if STDC_HEADERS
-static mib_tree_entry *snmpAddNode(oid * name, int len, oid_ParseFn * parsefunction, instance_Fn * instancefunction, int children,...);
-static oid *snmpCreateOid(int length,...);
-#else
-static mib_tree_entry *snmpAddNode();
-static oid *snmpCreateOid();
-#endif
-extern void (*snmplib_debug_hook) (int, char *);
-static oid *static_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
-static oid *time_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
-static oid *peer_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
-static oid *client_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
-static void snmpDecodePacket(snmp_request_t * rq);
-static void snmpConstructReponse(snmp_request_t * rq);
-static struct snmp_pdu *snmpAgentResponse(struct snmp_pdu *PDU);
-static oid_ParseFn *snmpTreeNext(oid * Current, snint CurrentLen, oid ** Next, snint * NextLen);
-static oid_ParseFn *snmpTreeGet(oid * Current, snint CurrentLen);
-static mib_tree_entry *snmpTreeEntry(oid entry, snint len, mib_tree_entry * current);
-static mib_tree_entry *snmpTreeSiblingEntry(oid entry, snint len, mib_tree_entry * current);
-static void snmpSnmplibDebug(int lvl, char *buf);
-
-
-/*
- * The functions used during startup:
- * snmpInit
- * snmpConnectionOpen
- * snmpConnectionShutdown
- * snmpConnectionClose
- */
-
-/*
- * Turns the MIB into a Tree structure. Called during the startup process.
- */
-void
-snmpInit(void)
-{
-    debug(49, 5) ("snmpInit: Called.\n");
-
-    debug(49, 5) ("snmpInit: Building SNMP mib tree structure\n");
-
-    snmplib_debug_hook = snmpSnmplibDebug;
-
-    mib_tree_head = snmpAddNode(snmpCreateOid(1, 1),
-	1, NULL, NULL, 1,
-	snmpAddNode(snmpCreateOid(2, 1, 3),
-	    2, NULL, NULL, 1,
-	    snmpAddNode(snmpCreateOid(3, 1, 3, 6),
-		3, NULL, NULL, 1,
-		snmpAddNode(snmpCreateOid(4, 1, 3, 6, 1),
-		    4, NULL, NULL, 1,
-		    snmpAddNode(snmpCreateOid(5, 1, 3, 6, 1, 4),
-			5, NULL, NULL, 1,
-			snmpAddNode(snmpCreateOid(6, 1, 3, 6, 1, 4, 1),
-			    6, NULL, NULL, 1,
-			    snmpAddNode(snmpCreateOid(7, 1, 3, 6, 1, 4, 1, 3495),
-				7, NULL, NULL, 1,
-				snmpAddNode(snmpCreateOid(LEN_SQUIDMIB, SQUIDMIB),
-				    8, NULL, NULL, 5,
-				    snmpAddNode(snmpCreateOid(LEN_SQ_SYS, SQ_SYS),
-					LEN_SQ_SYS, NULL, NULL, 3,
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_SYS, SYSVMSIZ),
-					    LEN_SYS, snmp_sysFn, static_Inst, 0),
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_SYS, SYSSTOR),
-					    LEN_SYS, snmp_sysFn, static_Inst, 0),
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_SYS, SYS_UPTIME),
-					    LEN_SYS, snmp_sysFn, static_Inst, 0)),
-				    snmpAddNode(snmpCreateOid(LEN_SQ_CONF, SQ_CONF),
-					LEN_SQ_CONF, NULL, NULL, 5,
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_ADMIN),
-					    LEN_SYS, snmp_confFn, static_Inst, 0),
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_VERSION),
-					    LEN_SYS, snmp_confFn, static_Inst, 0),
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_VERSION_ID),
-					    LEN_SYS, snmp_confFn, static_Inst, 0),
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_LOG_FAC),
-					    LEN_SYS, snmp_confFn, static_Inst, 0),
-					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_STORAGE),
-					    LEN_SYS, NULL, NULL, 4,
-					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_MMAXSZ),
-						LEN_CONF_ST, snmp_confFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_SWMAXSZ),
-						LEN_CONF_ST, snmp_confFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_SWHIWM),
-						LEN_CONF_ST, snmp_confFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_SWLOWM),
-						LEN_CONF_ST, snmp_confFn, static_Inst, 0))),
-				    snmpAddNode(snmpCreateOid(LEN_SQ_PRF, SQ_PRF),
-					LEN_SQ_PRF, NULL, NULL, 2,
-					snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 1, SQ_PRF, PERF_SYS),
-					    LEN_SQ_PRF + 1, NULL, NULL, 11,
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 1),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 2),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 3),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 4),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 5),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 6),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 7),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 8),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 9),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 10),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 11),
-						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0)),
-					snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 1, SQ_PRF, PERF_PROTO),
-					    LEN_SQ_PRF + 1, NULL, NULL, 2,
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_PROTO, 1),
-						LEN_SQ_PRF + 2, NULL, NULL, 15,
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 1),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 2),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 3),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 4),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 5),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 6),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 7),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 8),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 9),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 10),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 11),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 12),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 13),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 14),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 15),
-						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0)),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_PROTO, 2),
-						LEN_SQ_PRF + 2, NULL, NULL, 1,
-						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 2, 1),
-						    LEN_SQ_PRF + 3, NULL, NULL, 10,
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 1),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 2),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 3),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 4),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 5),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 6),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 7),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 8),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 9),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
-						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 10),
-							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0))))),
-				    snmpAddNode(snmpCreateOid(LEN_SQ_NET, SQ_NET),
-					LEN_SQ_NET, NULL, NULL, 3,
-					snmpAddNode(snmpCreateOid(LEN_SQ_NET + 1, SQ_NET, NET_IP_CACHE),
-					    LEN_SQ_NET + 1, NULL, NULL, 8,
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_ENT),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_REQ),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_HITS),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_PENDHIT),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_NEGHIT),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_MISS),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_GHBN),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_LOC),
-						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0)),
-					snmpAddNode(snmpCreateOid(LEN_SQ_NET + 1, SQ_NET, NET_FQDN_CACHE),
-					    LEN_SQ_NET + 1, NULL, NULL, 7,
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_ENT),
-						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_REQ),
-						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_HITS),
-						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_PENDHIT),
-						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_NEGHIT),
-						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_MISS),
-						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_GHBN),
-						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0)),
-					snmpAddNode(snmpCreateOid(LEN_SQ_NET + 1, SQ_NET, NET_DNS_CACHE),
-					    LEN_SQ_NET + 1, NULL, NULL, 3,
-#if USE_DNSSERVERS
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REQ),
-						LEN_SQ_NET + 2, snmp_netDnsFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REP),
-						LEN_SQ_NET + 2, snmp_netDnsFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_SERVERS),
-						LEN_SQ_NET + 2, snmp_netDnsFn, static_Inst, 0))),
-#else
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REQ),
-						LEN_SQ_NET + 2, snmp_netIdnsFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REP),
-						LEN_SQ_NET + 2, snmp_netIdnsFn, static_Inst, 0),
-					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_SERVERS),
-						LEN_SQ_NET + 2, snmp_netIdnsFn, static_Inst, 0))),
-#endif
-				    snmpAddNode(snmpCreateOid(LEN_SQ_MESH, SQ_MESH),
-					LEN_SQ_MESH, NULL, NULL, 2,
-					snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 1, SQ_MESH, 1),
-					    LEN_SQ_MESH + 1, NULL, NULL, 1,
-					    snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 2, SQ_MESH, 1, 1),
-						LEN_SQ_MESH + 2, NULL, NULL, 13,
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 1),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 2),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 3),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 4),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 5),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 6),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 7),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 8),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 9),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 10),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 11),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 12),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 13),
-						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0))),
-					snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 1, SQ_MESH, 2),
-					    LEN_SQ_MESH + 1, NULL, NULL, 1,
-					    snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 2, SQ_MESH, 2, 1),
-						LEN_SQ_MESH + 2, NULL, NULL, 9,
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 1),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 2),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 3),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 4),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 5),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 6),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 7),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 8),
-						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
-						(mib_tree_last = snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 9),
-							LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0)))))
-				)
-			    )
-			)
-		    )
-		)
-	    )
-	)
-	);
-
-    debug(49, 9) ("snmpInit: Completed SNMP mib tree structure\n");
-}
-
-void
-snmpConnectionOpen(void)
-{
-    u_short port;
-    struct sockaddr_in xaddr;
-    socklen_t len;
-    int x;
-
-    debug(49, 5) ("snmpConnectionOpen: Called\n");
-    if ((port = Config.Port.snmp) > (u_short) 0) {
-	enter_suid();
-	theInSnmpConnection = comm_open(SOCK_DGRAM,
-	    0,
-	    Config.Addrs.snmp_incoming,
-	    port,
-	    COMM_NONBLOCKING,
-	    "SNMP Port");
-	leave_suid();
-	if (theInSnmpConnection < 0)
-	    fatal("Cannot open snmp Port");
-	commSetSelect(theInSnmpConnection, COMM_SELECT_READ, snmpHandleUdp, NULL, 0);
-	debug(1, 1) ("Accepting SNMP messages on port %d, FD %d.\n",
-	    (int) port, theInSnmpConnection);
-	if (Config.Addrs.snmp_outgoing.s_addr != no_addr.s_addr) {
-	    enter_suid();
-	    theOutSnmpConnection = comm_open(SOCK_DGRAM,
-		0,
-		Config.Addrs.snmp_outgoing,
-		port,
-		COMM_NONBLOCKING,
-		"SNMP Port");
-	    leave_suid();
-	    if (theOutSnmpConnection < 0)
-		fatal("Cannot open Outgoing SNMP Port");
-	    commSetSelect(theOutSnmpConnection,
-		COMM_SELECT_READ,
-		snmpHandleUdp,
-		NULL, 0);
-	    debug(1, 1) ("Outgoing SNMP messages on port %d, FD %d.\n",
-		(int) port, theOutSnmpConnection);
-	    fd_note(theOutSnmpConnection, "Outgoing SNMP socket");
-	    fd_note(theInSnmpConnection, "Incoming SNMP socket");
-	} else {
-	    theOutSnmpConnection = theInSnmpConnection;
-	}
-	memset(&theOutSNMPAddr, '\0', sizeof(struct in_addr));
-	len = sizeof(struct sockaddr_in);
-	memset(&xaddr, '\0', len);
-	x = getsockname(theOutSnmpConnection,
-	    (struct sockaddr *) &xaddr, &len);
-	if (x < 0)
-	    debug(51, 1) ("theOutSnmpConnection FD %d: getsockname: %s\n",
-		theOutSnmpConnection, xstrerror());
-	else
-	    theOutSNMPAddr = xaddr.sin_addr;
-    }
-}
-
-void
-snmpConnectionShutdown(void)
-{
-    if (theInSnmpConnection < 0)
-	return;
-    if (theInSnmpConnection != theOutSnmpConnection) {
-	debug(49, 1) ("FD %d Closing SNMP socket\n", theInSnmpConnection);
-	comm_close(theInSnmpConnection);
-    }
-    /*
-     * Here we set 'theInSnmpConnection' to -1 even though the SNMP 'in'
-     * and 'out' sockets might be just one FD.  This prevents this
-     * function from executing repeatedly.  When we are really ready to
-     * exit or restart, main will comm_close the 'out' descriptor.
-     */ theInSnmpConnection = -1;
-    /*
-     * Normally we only write to the outgoing SNMP socket, but we
-     * also have a read handler there to catch messages sent to that
-     * specific interface.  During shutdown, we must disable reading
-     * on the outgoing socket.
-     */
-    assert(theOutSnmpConnection > -1);
-    commSetSelect(theOutSnmpConnection, COMM_SELECT_READ, NULL, NULL, 0);
-}
-
-void
-snmpConnectionClose(void)
-{
-    snmpConnectionShutdown();
-    if (theOutSnmpConnection > -1) {
-	debug(49, 1) ("FD %d Closing SNMP socket\n", theOutSnmpConnection);
-	comm_close(theOutSnmpConnection);
-    }
-}
-
-/*
- * Functions for handling the requests.
- */
-
-/*
- * Accept the UDP packet
- */
-void
-snmpHandleUdp(int sock, void *not_used)
-{
-    LOCAL_ARRAY(char, buf, SNMP_REQUEST_SIZE);
-    struct sockaddr_in from;
-    socklen_t from_len;
-    snmp_request_t *snmp_rq;
-    int len;
-
-    debug(49, 5) ("snmpHandleUdp: Called.\n");
-
-    commSetSelect(sock, COMM_SELECT_READ, snmpHandleUdp, NULL, 0);
-    from_len = sizeof(struct sockaddr_in);
-    memset(&from, '\0', from_len);
-    memset(buf, '\0', SNMP_REQUEST_SIZE);
-
-    statCounter.syscalls.sock.recvfroms++;
-
-    len = recvfrom(sock,
-	buf,
-	SNMP_REQUEST_SIZE,
-	0,
-	(struct sockaddr *) &from,
-	&from_len);
-
-    if (len > 0) {
-	buf[len] = '\0';
-	debug(49, 3) ("snmpHandleUdp: FD %d: received %d bytes from %s.\n",
-	    sock,
-	    len,
-	    inet_ntoa(from.sin_addr));
-
-	snmp_rq = xcalloc(1, sizeof(snmp_request_t));
-	snmp_rq->buf = (u_char *) buf;
-	snmp_rq->len = len;
-	snmp_rq->sock = sock;
-	snmp_rq->outbuf = xmalloc(snmp_rq->outlen = SNMP_REQUEST_SIZE);
-	xmemcpy(&snmp_rq->from, &from, sizeof(struct sockaddr_in));
-	snmpDecodePacket(snmp_rq);
-	xfree(snmp_rq->outbuf);
-	xfree(snmp_rq);
-    } else {
-	debug(49, 1) ("snmpHandleUdp: FD %d recvfrom: %s\n", sock, xstrerror());
-    }
-}
-
-/*
- * Turn SNMP packet into a PDU, check available ACL's
- */
-static void
-snmpDecodePacket(snmp_request_t * rq)
-{
-    struct snmp_pdu *PDU;
-    struct snmp_session Session;
-    aclCheck_t checklist;
-    u_char *Community;
-    u_char *buf = rq->buf;
-    int len = rq->len;
-    int allow = 0;
-
-    debug(49, 5) ("snmpDecodePacket: Called.\n");
-    /* Now that we have the data, turn it into a PDU */
-    PDU = snmp_pdu_create(0);
-    Session.Version = SNMP_VERSION_1;
-    Community = snmp_parse(&Session, PDU, buf, len);
-    memset(&checklist, '\0', sizeof(checklist));
-    checklist.src_addr = rq->from.sin_addr;
-    checklist.snmp_community = (char *) Community;
-
-    if (Community)
-	allow = aclCheckFast(Config.accessList.snmp, &checklist);
-    if ((snmp_coexist_V2toV1(PDU)) && (Community) && (allow)) {
-	rq->community = Community;
-	rq->PDU = PDU;
-	debug(49, 5) ("snmpAgentParse: reqid=[%d]\n", PDU->reqid);
-	snmpConstructReponse(rq);
-    } else {
-	debug(49, 1) ("Failed SNMP agent query from : %s.\n",
-	    inet_ntoa(rq->from.sin_addr));
-	snmp_free_pdu(PDU);
-    }
-    if (Community)
-	xfree(Community);
-}
-
-/*
- * Packet OK, ACL Check OK, Create reponse.
- */
-static void
-snmpConstructReponse(snmp_request_t * rq)
-{
-    struct snmp_session Session;
-    struct snmp_pdu *RespPDU;
-
-    debug(49, 5) ("snmpConstructReponse: Called.\n");
-    RespPDU = snmpAgentResponse(rq->PDU);
-    snmp_free_pdu(rq->PDU);
-    if (RespPDU != NULL) {
-	Session.Version = SNMP_VERSION_1;
-	Session.community = rq->community;
-	Session.community_len = strlen((char *) rq->community);
-	snmp_build(&Session, RespPDU, rq->outbuf, &rq->outlen);
-	sendto(rq->sock, rq->outbuf, rq->outlen, 0, (struct sockaddr *) &rq->from, sizeof(rq->from));
-	snmp_free_pdu(RespPDU);
-    }
-}
-
-/*
- * Decide how to respond to the request, construct a response and
- * return the response to the requester.
- */
-static struct snmp_pdu *
-snmpAgentResponse(struct snmp_pdu *PDU)
-{
-    struct snmp_pdu *Answer = NULL;
-    oid_ParseFn *ParseFn = NULL;
-
-    variable_list *VarPtr, *VarNew = NULL, **VarPtrP;
-    int index = 0;
-
-    debug(49, 5) ("snmpAgentResponse: Called.\n");
-
-    if ((Answer = snmp_pdu_create(SNMP_PDU_RESPONSE))) {
-	Answer->reqid = PDU->reqid;
-	Answer->errindex = 0;
-	if (PDU->command == SNMP_PDU_GET) {
-	    variable_list **RespVars;
-
-	    RespVars = &(Answer->variables);
-	    /* Loop through all variables */
-	    for (VarPtrP = &(PDU->variables);
-		*VarPtrP;
-		VarPtrP = &((*VarPtrP)->next_variable)) {
-		VarPtr = *VarPtrP;
-
-		index++;
-
-		/* Find the parsing function for this variable */
-		ParseFn = snmpTreeGet(VarPtr->name, VarPtr->name_length);
-
-		if (ParseFn == NULL) {
-		    Answer->errstat = SNMP_ERR_NOSUCHNAME;
-		    debug(49, 5) ("snmpAgentResponse: No such oid. ");
-		} else
-		    VarNew = (*ParseFn) (VarPtr, (snint *) & (Answer->errstat));
-
-		/* Was there an error? */
-		if ((Answer->errstat != SNMP_ERR_NOERROR) ||
-		    (VarNew == NULL)) {
-		    Answer->errindex = index;
-		    debug(49, 5) ("snmpAgentParse: successful.\n");
-		    /* Just copy the rest of the variables.  Quickly. */
-		    *RespVars = VarPtr;
-		    *VarPtrP = NULL;
-		    return (Answer);
-		}
-		/* No error.  Insert this var at the end, and move on to the next.
-		 */
-		*RespVars = VarNew;
-		RespVars = &(VarNew->next_variable);
-	    }
-	    return (Answer);
-	} else if (PDU->command == SNMP_PDU_GETNEXT) {
-	    oid *NextOidName = NULL;
-	    int NextOidNameLen = 0;
-
-	    ParseFn = snmpTreeNext(PDU->variables->name, PDU->variables->name_length,
-		&(NextOidName), (snint *) & NextOidNameLen);
-
-	    if (ParseFn == NULL) {
-		Answer->errstat = SNMP_ERR_NOSUCHNAME;
-		debug(49, 5) ("snmpAgentResponse: No such oid: ");
-		snmpDebugOid(5, PDU->variables->name, PDU->variables->name_length);
-	    } else {
-		xfree(PDU->variables->name);
-		PDU->variables->name = NextOidName;
-		PDU->variables->name_length = NextOidNameLen;
-		VarNew = (*ParseFn) (PDU->variables, (snint *) & Answer->errstat);
-	    }
-
-	    /* Was there an error? */
-	    if (Answer->errstat != SNMP_ERR_NOERROR) {
-		Answer->errindex = 1;
-		Answer->variables = PDU->variables;
-		PDU->variables = NULL;
-	    } else {
-		Answer->variables = VarNew;
-	    }
-
-	} else {
-	    snmp_free_pdu(Answer);
-	    Answer = NULL;
-	}
-    }
-    return (Answer);
-}
-
-static oid_ParseFn *
-snmpTreeGet(oid * Current, snint CurrentLen)
-{
-    oid_ParseFn *Fn = NULL;
-    mib_tree_entry *mibTreeEntry = NULL;
-    int count = 0;
-
-    debug(49, 5) ("snmpTreeGet: Called\n");
-
-    debug(49, 6) ("snmpTreeGet: Current : \n");
-    snmpDebugOid(6, Current, CurrentLen);
-
-    mibTreeEntry = mib_tree_head;
-    if (Current[count] == mibTreeEntry->name[count]) {
-	count++;
-	while ((mibTreeEntry) && (count < CurrentLen) && (!mibTreeEntry->parsefunction)) {
-	    mibTreeEntry = snmpTreeEntry(Current[count], count, mibTreeEntry);
-	    count++;
-	}
-    }
-    if (mibTreeEntry && mibTreeEntry->parsefunction)
-	Fn = mibTreeEntry->parsefunction;
-    debug(49, 5) ("snmpTreeGet: return\n");
-    return (Fn);
-}
-
-static oid_ParseFn *
-snmpTreeNext(oid * Current, snint CurrentLen, oid ** Next, snint * NextLen)
-{
-    oid_ParseFn *Fn = NULL;
-    mib_tree_entry *mibTreeEntry = NULL, *nextoid = NULL;
-    int count = 0;
-
-    debug(49, 5) ("snmpTreeNext: Called\n");
-
-    debug(49, 6) ("snmpTreeNext: Current : \n");
-    snmpDebugOid(6, Current, CurrentLen);
-
-    mibTreeEntry = mib_tree_head;
-    if (Current[count] == mibTreeEntry->name[count]) {
-	count++;
-	while ((mibTreeEntry) && (count < CurrentLen) && (!mibTreeEntry->parsefunction)) {
-	    mibTreeEntry = snmpTreeEntry(Current[count], count, mibTreeEntry);
-	    count++;
-	}
-	debug(49, 5) ("snmpTreeNext: Recursed down to requested object\n");
-    } else {
-	return NULL;
-    }
-    if (mibTreeEntry == mib_tree_last)
-	return (Fn);
-    if ((mibTreeEntry) && (mibTreeEntry->parsefunction)) {
-	*NextLen = CurrentLen;
-	*Next = (*mibTreeEntry->instancefunction) (Current, NextLen, mibTreeEntry, &Fn);
-	if (*Next)
-	    return (Fn);
-    }
-    if ((mibTreeEntry) && (mibTreeEntry->parsefunction)) {
-	count--;
-	nextoid = snmpTreeSiblingEntry(Current[count], count, mibTreeEntry->parent);
-	if (nextoid) {
-	    debug(49, 5) ("snmpTreeNext: Next OID found for sibling\n");
-	    mibTreeEntry = nextoid;
-	    count++;
-	} else {
-	    debug(49, 5) ("snmpTreeNext: Attempting to recurse up for next object\n");
-	    while (!nextoid) {
-		count--;
-		if (mibTreeEntry->parent->parent) {
-		    nextoid = mibTreeEntry->parent;
-		    mibTreeEntry = snmpTreeEntry(Current[count] + 1, count, nextoid->parent);
-		    if (!mibTreeEntry) {
-			mibTreeEntry = nextoid;
-			nextoid = NULL;
-		    }
-		} else {
-		    nextoid = mibTreeEntry;
-		    mibTreeEntry = NULL;
-		}
-	    }
-	}
-    }
-    while ((mibTreeEntry) && (!mibTreeEntry->parsefunction)) {
-	mibTreeEntry = mibTreeEntry->leaves[0];
-    }
-    if (mibTreeEntry) {
-	*NextLen = mibTreeEntry->len;
-	*Next = (*mibTreeEntry->instancefunction) (mibTreeEntry->name, NextLen, mibTreeEntry, &Fn);
-    }
-    if (*Next)
-	return (Fn);
-    else
-	return NULL;
-}
-
-static oid *
-static_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
-{
-    oid *instance = NULL;
-
-    if (*len <= current->len) {
-	instance = xmalloc(sizeof(name) * (*len + 1));
-	xmemcpy(instance, name, (sizeof(name) * *len));
-	instance[*len] = 0;
-	*len += 1;
-    }
-    *Fn = current->parsefunction;
-    return (instance);
-}
-
-static oid *
-time_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
-{
-    oid *instance = NULL;
-    int identifier = 0, loop = 0;
-    int index[TIME_INDEX_LEN] =
-    {TIME_INDEX};
-
-    if (*len <= current->len) {
-	instance = xmalloc(sizeof(name) * (*len + 1));
-	xmemcpy(instance, name, (sizeof(name) * *len));
-	instance[*len] = *index;
-	*len += 1;
-    } else {
-	identifier = name[*len - 1];
-	while ((identifier != index[loop]) && (loop < TIME_INDEX_LEN))
-	    loop++;
-	if (loop < TIME_INDEX_LEN - 1) {
-	    instance = xmalloc(sizeof(name) * (*len));
-	    xmemcpy(instance, name, (sizeof(name) * *len));
-	    instance[*len - 1] = index[++loop];
-	}
-    }
-    *Fn = current->parsefunction;
-    return (instance);
-}
-
-static oid *
-peer_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
-{
-    oid *instance = NULL;
-    u_char *cp = NULL;
-    peer *peers = Config.peers;
-    struct in_addr *laddr = NULL;
-    char *host_addr = NULL, *current_addr = NULL, *last_addr = NULL;
-
-    if (peers == NULL) {
-	current = current->parent->parent->parent->leaves[1];
-	while ((current) && (!current->parsefunction))
-	    current = current->leaves[0];
-	instance = client_Inst(current->name, len, current, Fn);
-    } else if (*len <= current->len) {
-	instance = xmalloc(sizeof(name) * (*len + 4));
-	xmemcpy(instance, name, (sizeof(name) * *len));
-	cp = (u_char *) & (peers->in_addr.sin_addr.s_addr);
-	instance[*len] = *cp++;
-	instance[*len + 1] = *cp++;
-	instance[*len + 2] = *cp++;
-	instance[*len + 3] = *cp++;
-	*len += 4;
-    } else {
-	laddr = oid2addr(&name[*len - 4]);
-	host_addr = inet_ntoa(*laddr);
-	last_addr = xmalloc(strlen(host_addr));
-	strncpy(last_addr, host_addr, strlen(host_addr));
-	current_addr = inet_ntoa(peers->in_addr.sin_addr);
-	while ((peers) && (strncmp(last_addr, current_addr, strlen(current_addr)))) {
-	    if (peers->next) {
-		peers = peers->next;
-		current_addr = inet_ntoa(peers->in_addr.sin_addr);
-	    } else {
-		peers = NULL;
-	    }
-	}
-	xfree(last_addr);
-	if (peers) {
-	    if (peers->next) {
-		peers = peers->next;
-		instance = xmalloc(sizeof(name) * (*len));
-		xmemcpy(instance, name, (sizeof(name) * *len));
-		cp = (u_char *) & (peers->in_addr.sin_addr.s_addr);
-		instance[*len - 4] = *cp++;
-		instance[*len - 3] = *cp++;
-		instance[*len - 2] = *cp++;
-		instance[*len - 1] = *cp++;
-	    } else {
-		return (instance);
-	    }
-	} else {
-	    return (instance);
-	}
-    }
-    *Fn = current->parsefunction;
-    return (instance);
-}
-
-static oid *
-client_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
-{
-    oid *instance = NULL;
-    u_char *cp = NULL;
-    struct in_addr *laddr = NULL;
-
-    if (*len <= current->len) {
-	instance = xmalloc(sizeof(name) * (*len + 4));
-	xmemcpy(instance, name, (sizeof(name) * *len));
-	laddr = client_entry(NULL);
-	if (laddr) {
-	    cp = (u_char *) & (laddr->s_addr);
-	    instance[*len] = *cp++;
-	    instance[*len + 1] = *cp++;
-	    instance[*len + 2] = *cp++;
-	    instance[*len + 3] = *cp++;
-	    *len += 4;
-	}
-    } else {
-	laddr = oid2addr(&name[*len - 4]);
-	laddr = client_entry(laddr);
-	if (laddr) {
-	    instance = xmalloc(sizeof(name) * (*len));
-	    xmemcpy(instance, name, (sizeof(name) * *len));
-	    cp = (u_char *) & (laddr->s_addr);
-	    instance[*len - 4] = *cp++;
-	    instance[*len - 3] = *cp++;
-	    instance[*len - 2] = *cp++;
-	    instance[*len - 1] = *cp++;
-	}
-    }
-    *Fn = current->parsefunction;
-    return (instance);
-}
-
-
-/*
- * Utility functions
- */
-
-/*
- * Tree utility functions. 
- */
-
-/* 
- * Returns a the sibling object in the tree
- */
-static mib_tree_entry *
-snmpTreeSiblingEntry(oid entry, snint len, mib_tree_entry * current)
-{
-    mib_tree_entry *next = NULL;
-    int count = 0;
-
-    while ((!next) && (count < current->children)) {
-	if (current->leaves[count]->name[len] == entry) {
-	    next = current->leaves[count];
-	}
-	count++;
-    }
-    if (count < current->children) {
-	next = current->leaves[count];
-    } else {
-	next = NULL;
-    }
-    return (next);
-}
-
-/* 
- * Returns the requested child object or NULL if it does not exist
- */
-static mib_tree_entry *
-snmpTreeEntry(oid entry, snint len, mib_tree_entry * current)
-{
-    mib_tree_entry *next = NULL;
-    int count = 0;
-
-    while ((!next) && (count < current->children)) {
-	if (current->leaves[count]->name[len] == entry) {
-	    next = current->leaves[count];
-	}
-	count++;
-    }
-    return (next);
-}
-
-/*
- * Adds a node to the MIB tree structure and adds the appropriate children
- */
-static mib_tree_entry *
-#if STDC_HEADERS
-snmpAddNode(oid * name, int len, oid_ParseFn * parsefunction, instance_Fn * instancefunction, int children,...)
-#else
-snmpAddNode(va_alist)
-     va_dcl
-#endif
-{
-#if STDC_HEADERS
-    va_list args;
-    int loop;
-    mib_tree_entry *entry = NULL;
-    va_start(args, children);
-#else
-    va_list args;
-    oid *name = NULL;
-    int len = 0, children = 0, loop;
-    oid_ParseFn *parsefunction = NULL;
-    instance_Fn *instancefunction = NULL;
-    mib_tree_entry *entry = NULL;
-    va_start(args);
-    name = va_arg(args, oid *);
-    len = va_arg(args, int);
-    parsefunction = va_arg(args, oid_ParseFn *);
-    instancefunction = va_arg(args, instance_Fn *);
-    children = va_arg(args, int);
-#endif
-
-    debug(49, 6) ("snmpAddNode: Children : %d, Oid : \n", children);
-    snmpDebugOid(6, name, len);
-
-    va_start(args, children);
-    entry = xmalloc(sizeof(mib_tree_entry));
-    entry->name = name;
-    entry->len = len;
-    entry->parsefunction = parsefunction;
-    entry->instancefunction = instancefunction;
-    entry->children = children;
-
-    if (children > 0) {
-	entry->leaves = xmalloc(sizeof(mib_tree_entry *) * children);
-	for (loop = 0; loop < children; loop++) {
-	    entry->leaves[loop] = va_arg(args, mib_tree_entry *);
-	    entry->leaves[loop]->parent = entry;
-	}
-    }
-    return (entry);
-}
-/* End of tree utility functions */
-
-/* 
- * Returns the list of parameters in an oid
- */
-static oid *
-#if STDC_HEADERS
-snmpCreateOid(int length,...)
-#else
-snmpCreateOid(va_alist)
-     va_dcl
-#endif
-{
-#if STDC_HEADERS
-    va_list args;
-    oid *new_oid;
-    int loop;
-    va_start(args, length);
-#else
-    va_list args;
-    int length = 0, loop;
-    oid *new_oid;
-    va_start(args);
-    length va_arg(args, int);
-#endif
-
-    new_oid = xmalloc(sizeof(oid) * length);
-
-    if (length > 0) {
-	for (loop = 0; loop < length; loop++) {
-	    new_oid[loop] = va_arg(args, int);
-	}
-    }
-    return (new_oid);
-}
-
-#if UNUSED_CODE
-/*
- * Allocate space for, and copy, an OID.  Returns new oid.
- */
-static oid *
-snmpOidDup(oid * A, snint ALen)
-{
-    oid *Ans = xmalloc(sizeof(oid) * ALen);
-    xmemcpy(Ans, A, (sizeof(oid) * ALen));
-    return Ans;
-}
-#endif
-
-/*
- * Debug calls, prints out the OID for debugging purposes.
- */
-void
-snmpDebugOid(int lvl, oid * Name, snint Len)
-{
-    char mbuf[16], objid[1024];
-    int x;
-    objid[0] = '\0';
-
-    for (x = 0; x < Len; x++) {
-	snprintf(mbuf, sizeof(mbuf), ".%u", (unsigned int) Name[x]);
-	strncat(objid, mbuf, sizeof(objid));
-    }
-
-    debug(49, lvl) ("   oid = %s\n", objid);
-}
-
-static void
-snmpSnmplibDebug(int lvl, char *buf)
-{
-    debug(49, lvl) ("%s", buf);
-}
-
-void
-addr2oid(struct in_addr addr, oid * Dest)
-{
-    u_char *cp;
-    cp = (u_char *) & (addr.s_addr);
-    Dest[0] = *cp++;
-    Dest[1] = *cp++;
-    Dest[2] = *cp++;
-    Dest[3] = *cp++;
-}
-
-struct in_addr
-       *
-oid2addr(oid * id)
-{
-    static struct in_addr laddr;
-    u_char *cp = (u_char *) & (laddr.s_addr);
-    cp[0] = id[0];
-    cp[1] = id[1];
-    cp[2] = id[2];
-    cp[3] = id[3];
-    return &laddr;
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/snmp_core.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,1071 @@
+
+/*
+ * $Id: snmp_core.cc,v 1.1.2.1 2002/10/11 15:41:01 rbcollins Exp $
+ *
+ * DEBUG: section 49    SNMP support
+ * AUTHOR: Glenn Chisholm
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+#include "squid.h"
+#include "cache_snmp.h"
+
+#define SNMP_REQUEST_SIZE 4096
+#define MAX_PROTOSTAT 5
+
+typedef struct _mib_tree_entry mib_tree_entry;
+typedef oid *(instance_Fn) (oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
+
+struct _mib_tree_entry {
+    oid *name;
+    int len;
+    oid_ParseFn *parsefunction;
+    instance_Fn *instancefunction;
+    int children;
+    struct _mib_tree_entry **leaves;
+    struct _mib_tree_entry *parent;
+};
+
+mib_tree_entry *mib_tree_head;
+mib_tree_entry *mib_tree_last;
+
+#if STDC_HEADERS
+static mib_tree_entry *snmpAddNode(oid * name, int len, oid_ParseFn * parsefunction, instance_Fn * instancefunction, int children,...);
+static oid *snmpCreateOid(int length,...);
+#else
+static mib_tree_entry *snmpAddNode();
+static oid *snmpCreateOid();
+#endif
+extern void (*snmplib_debug_hook) (int, char *);
+static oid *static_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
+static oid *time_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
+static oid *peer_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
+static oid *client_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn);
+static void snmpDecodePacket(snmp_request_t * rq);
+static void snmpConstructReponse(snmp_request_t * rq);
+static struct snmp_pdu *snmpAgentResponse(struct snmp_pdu *PDU);
+static oid_ParseFn *snmpTreeNext(oid * Current, snint CurrentLen, oid ** Next, snint * NextLen);
+static oid_ParseFn *snmpTreeGet(oid * Current, snint CurrentLen);
+static mib_tree_entry *snmpTreeEntry(oid entry, snint len, mib_tree_entry * current);
+static mib_tree_entry *snmpTreeSiblingEntry(oid entry, snint len, mib_tree_entry * current);
+static void snmpSnmplibDebug(int lvl, char *buf);
+
+
+/*
+ * The functions used during startup:
+ * snmpInit
+ * snmpConnectionOpen
+ * snmpConnectionShutdown
+ * snmpConnectionClose
+ */
+
+/*
+ * Turns the MIB into a Tree structure. Called during the startup process.
+ */
+void
+snmpInit(void)
+{
+    debug(49, 5) ("snmpInit: Called.\n");
+
+    debug(49, 5) ("snmpInit: Building SNMP mib tree structure\n");
+
+    snmplib_debug_hook = snmpSnmplibDebug;
+
+    mib_tree_head = snmpAddNode(snmpCreateOid(1, 1),
+	1, NULL, NULL, 1,
+	snmpAddNode(snmpCreateOid(2, 1, 3),
+	    2, NULL, NULL, 1,
+	    snmpAddNode(snmpCreateOid(3, 1, 3, 6),
+		3, NULL, NULL, 1,
+		snmpAddNode(snmpCreateOid(4, 1, 3, 6, 1),
+		    4, NULL, NULL, 1,
+		    snmpAddNode(snmpCreateOid(5, 1, 3, 6, 1, 4),
+			5, NULL, NULL, 1,
+			snmpAddNode(snmpCreateOid(6, 1, 3, 6, 1, 4, 1),
+			    6, NULL, NULL, 1,
+			    snmpAddNode(snmpCreateOid(7, 1, 3, 6, 1, 4, 1, 3495),
+				7, NULL, NULL, 1,
+				snmpAddNode(snmpCreateOid(LEN_SQUIDMIB, SQUIDMIB),
+				    8, NULL, NULL, 5,
+				    snmpAddNode(snmpCreateOid(LEN_SQ_SYS, SQ_SYS),
+					LEN_SQ_SYS, NULL, NULL, 3,
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_SYS, SYSVMSIZ),
+					    LEN_SYS, snmp_sysFn, static_Inst, 0),
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_SYS, SYSSTOR),
+					    LEN_SYS, snmp_sysFn, static_Inst, 0),
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_SYS, SYS_UPTIME),
+					    LEN_SYS, snmp_sysFn, static_Inst, 0)),
+				    snmpAddNode(snmpCreateOid(LEN_SQ_CONF, SQ_CONF),
+					LEN_SQ_CONF, NULL, NULL, 5,
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_ADMIN),
+					    LEN_SYS, snmp_confFn, static_Inst, 0),
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_VERSION),
+					    LEN_SYS, snmp_confFn, static_Inst, 0),
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_VERSION_ID),
+					    LEN_SYS, snmp_confFn, static_Inst, 0),
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_LOG_FAC),
+					    LEN_SYS, snmp_confFn, static_Inst, 0),
+					snmpAddNode(snmpCreateOid(LEN_SYS, SQ_CONF, CONF_STORAGE),
+					    LEN_SYS, NULL, NULL, 4,
+					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_MMAXSZ),
+						LEN_CONF_ST, snmp_confFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_SWMAXSZ),
+						LEN_CONF_ST, snmp_confFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_SWHIWM),
+						LEN_CONF_ST, snmp_confFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_CONF_ST, SQ_CONF, CONF_STORAGE, CONF_ST_SWLOWM),
+						LEN_CONF_ST, snmp_confFn, static_Inst, 0))),
+				    snmpAddNode(snmpCreateOid(LEN_SQ_PRF, SQ_PRF),
+					LEN_SQ_PRF, NULL, NULL, 2,
+					snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 1, SQ_PRF, PERF_SYS),
+					    LEN_SQ_PRF + 1, NULL, NULL, 11,
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 1),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 2),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 3),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 4),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 5),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 6),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 7),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 8),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 9),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 10),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_SYS, 11),
+						LEN_SQ_PRF + 2, snmp_prfSysFn, static_Inst, 0)),
+					snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 1, SQ_PRF, PERF_PROTO),
+					    LEN_SQ_PRF + 1, NULL, NULL, 2,
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_PROTO, 1),
+						LEN_SQ_PRF + 2, NULL, NULL, 15,
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 1),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 2),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 3),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 4),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 5),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 6),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 7),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 8),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 9),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 10),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 11),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 12),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 13),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 14),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 1, 15),
+						    LEN_SQ_PRF + 3, snmp_prfProtoFn, static_Inst, 0)),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 2, SQ_PRF, PERF_PROTO, 2),
+						LEN_SQ_PRF + 2, NULL, NULL, 1,
+						snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 3, SQ_PRF, PERF_PROTO, 2, 1),
+						    LEN_SQ_PRF + 3, NULL, NULL, 10,
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 1),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 2),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 3),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 4),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 5),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 6),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 7),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 8),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 9),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0),
+						    snmpAddNode(snmpCreateOid(LEN_SQ_PRF + 4, SQ_PRF, PERF_PROTO, 2, 1, 10),
+							LEN_SQ_PRF + 4, snmp_prfProtoFn, time_Inst, 0))))),
+				    snmpAddNode(snmpCreateOid(LEN_SQ_NET, SQ_NET),
+					LEN_SQ_NET, NULL, NULL, 3,
+					snmpAddNode(snmpCreateOid(LEN_SQ_NET + 1, SQ_NET, NET_IP_CACHE),
+					    LEN_SQ_NET + 1, NULL, NULL, 8,
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_ENT),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_REQ),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_HITS),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_PENDHIT),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_NEGHIT),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_MISS),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_GHBN),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_IP_CACHE, IP_LOC),
+						LEN_SQ_NET + 2, snmp_netIpFn, static_Inst, 0)),
+					snmpAddNode(snmpCreateOid(LEN_SQ_NET + 1, SQ_NET, NET_FQDN_CACHE),
+					    LEN_SQ_NET + 1, NULL, NULL, 7,
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_ENT),
+						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_REQ),
+						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_HITS),
+						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_PENDHIT),
+						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_NEGHIT),
+						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_MISS),
+						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_FQDN_CACHE, FQDN_GHBN),
+						LEN_SQ_NET + 2, snmp_netFqdnFn, static_Inst, 0)),
+					snmpAddNode(snmpCreateOid(LEN_SQ_NET + 1, SQ_NET, NET_DNS_CACHE),
+					    LEN_SQ_NET + 1, NULL, NULL, 3,
+#if USE_DNSSERVERS
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REQ),
+						LEN_SQ_NET + 2, snmp_netDnsFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REP),
+						LEN_SQ_NET + 2, snmp_netDnsFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_SERVERS),
+						LEN_SQ_NET + 2, snmp_netDnsFn, static_Inst, 0))),
+#else
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REQ),
+						LEN_SQ_NET + 2, snmp_netIdnsFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_REP),
+						LEN_SQ_NET + 2, snmp_netIdnsFn, static_Inst, 0),
+					    snmpAddNode(snmpCreateOid(LEN_SQ_NET + 2, SQ_NET, NET_DNS_CACHE, DNS_SERVERS),
+						LEN_SQ_NET + 2, snmp_netIdnsFn, static_Inst, 0))),
+#endif
+				    snmpAddNode(snmpCreateOid(LEN_SQ_MESH, SQ_MESH),
+					LEN_SQ_MESH, NULL, NULL, 2,
+					snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 1, SQ_MESH, 1),
+					    LEN_SQ_MESH + 1, NULL, NULL, 1,
+					    snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 2, SQ_MESH, 1, 1),
+						LEN_SQ_MESH + 2, NULL, NULL, 13,
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 1),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 2),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 3),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 4),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 5),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 6),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 7),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 8),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 9),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 10),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 11),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 12),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 1, 1, 13),
+						    LEN_SQ_MESH + 3, snmp_meshPtblFn, peer_Inst, 0))),
+					snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 1, SQ_MESH, 2),
+					    LEN_SQ_MESH + 1, NULL, NULL, 1,
+					    snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 2, SQ_MESH, 2, 1),
+						LEN_SQ_MESH + 2, NULL, NULL, 9,
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 1),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 2),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 3),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 4),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 5),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 6),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 7),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 8),
+						    LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0),
+						(mib_tree_last = snmpAddNode(snmpCreateOid(LEN_SQ_MESH + 3, SQ_MESH, 2, 1, 9),
+							LEN_SQ_MESH + 3, snmp_meshCtblFn, client_Inst, 0)))))
+				)
+			    )
+			)
+		    )
+		)
+	    )
+	)
+	);
+
+    debug(49, 9) ("snmpInit: Completed SNMP mib tree structure\n");
+}
+
+void
+snmpConnectionOpen(void)
+{
+    u_short port;
+    struct sockaddr_in xaddr;
+    socklen_t len;
+    int x;
+
+    debug(49, 5) ("snmpConnectionOpen: Called\n");
+    if ((port = Config.Port.snmp) > (u_short) 0) {
+	enter_suid();
+	theInSnmpConnection = comm_open(SOCK_DGRAM,
+	    0,
+	    Config.Addrs.snmp_incoming,
+	    port,
+	    COMM_NONBLOCKING,
+	    "SNMP Port");
+	leave_suid();
+	if (theInSnmpConnection < 0)
+	    fatal("Cannot open snmp Port");
+	commSetSelect(theInSnmpConnection, COMM_SELECT_READ, snmpHandleUdp, NULL, 0);
+	debug(1, 1) ("Accepting SNMP messages on port %d, FD %d.\n",
+	    (int) port, theInSnmpConnection);
+	if (Config.Addrs.snmp_outgoing.s_addr != no_addr.s_addr) {
+	    enter_suid();
+	    theOutSnmpConnection = comm_open(SOCK_DGRAM,
+		0,
+		Config.Addrs.snmp_outgoing,
+		port,
+		COMM_NONBLOCKING,
+		"SNMP Port");
+	    leave_suid();
+	    if (theOutSnmpConnection < 0)
+		fatal("Cannot open Outgoing SNMP Port");
+	    commSetSelect(theOutSnmpConnection,
+		COMM_SELECT_READ,
+		snmpHandleUdp,
+		NULL, 0);
+	    debug(1, 1) ("Outgoing SNMP messages on port %d, FD %d.\n",
+		(int) port, theOutSnmpConnection);
+	    fd_note(theOutSnmpConnection, "Outgoing SNMP socket");
+	    fd_note(theInSnmpConnection, "Incoming SNMP socket");
+	} else {
+	    theOutSnmpConnection = theInSnmpConnection;
+	}
+	memset(&theOutSNMPAddr, '\0', sizeof(struct in_addr));
+	len = sizeof(struct sockaddr_in);
+	memset(&xaddr, '\0', len);
+	x = getsockname(theOutSnmpConnection,
+	    (struct sockaddr *) &xaddr, &len);
+	if (x < 0)
+	    debug(51, 1) ("theOutSnmpConnection FD %d: getsockname: %s\n",
+		theOutSnmpConnection, xstrerror());
+	else
+	    theOutSNMPAddr = xaddr.sin_addr;
+    }
+}
+
+void
+snmpConnectionShutdown(void)
+{
+    if (theInSnmpConnection < 0)
+	return;
+    if (theInSnmpConnection != theOutSnmpConnection) {
+	debug(49, 1) ("FD %d Closing SNMP socket\n", theInSnmpConnection);
+	comm_close(theInSnmpConnection);
+    }
+    /*
+     * Here we set 'theInSnmpConnection' to -1 even though the SNMP 'in'
+     * and 'out' sockets might be just one FD.  This prevents this
+     * function from executing repeatedly.  When we are really ready to
+     * exit or restart, main will comm_close the 'out' descriptor.
+     */ theInSnmpConnection = -1;
+    /*
+     * Normally we only write to the outgoing SNMP socket, but we
+     * also have a read handler there to catch messages sent to that
+     * specific interface.  During shutdown, we must disable reading
+     * on the outgoing socket.
+     */
+    assert(theOutSnmpConnection > -1);
+    commSetSelect(theOutSnmpConnection, COMM_SELECT_READ, NULL, NULL, 0);
+}
+
+void
+snmpConnectionClose(void)
+{
+    snmpConnectionShutdown();
+    if (theOutSnmpConnection > -1) {
+	debug(49, 1) ("FD %d Closing SNMP socket\n", theOutSnmpConnection);
+	comm_close(theOutSnmpConnection);
+    }
+}
+
+/*
+ * Functions for handling the requests.
+ */
+
+/*
+ * Accept the UDP packet
+ */
+void
+snmpHandleUdp(int sock, void *not_used)
+{
+    LOCAL_ARRAY(char, buf, SNMP_REQUEST_SIZE);
+    struct sockaddr_in from;
+    socklen_t from_len;
+    snmp_request_t *snmp_rq;
+    int len;
+
+    debug(49, 5) ("snmpHandleUdp: Called.\n");
+
+    commSetSelect(sock, COMM_SELECT_READ, snmpHandleUdp, NULL, 0);
+    from_len = sizeof(struct sockaddr_in);
+    memset(&from, '\0', from_len);
+    memset(buf, '\0', SNMP_REQUEST_SIZE);
+
+    statCounter.syscalls.sock.recvfroms++;
+
+    len = recvfrom(sock,
+	buf,
+	SNMP_REQUEST_SIZE,
+	0,
+	(struct sockaddr *) &from,
+	&from_len);
+
+    if (len > 0) {
+	buf[len] = '\0';
+	debug(49, 3) ("snmpHandleUdp: FD %d: received %d bytes from %s.\n",
+	    sock,
+	    len,
+	    inet_ntoa(from.sin_addr));
+
+	snmp_rq = (snmp_request_t *)xcalloc(1, sizeof(snmp_request_t));
+	snmp_rq->buf = (u_char *) buf;
+	snmp_rq->len = len;
+	snmp_rq->sock = sock;
+	snmp_rq->outbuf = (unsigned char *)xmalloc(snmp_rq->outlen = SNMP_REQUEST_SIZE);
+	xmemcpy(&snmp_rq->from, &from, sizeof(struct sockaddr_in));
+	snmpDecodePacket(snmp_rq);
+	xfree(snmp_rq->outbuf);
+	xfree(snmp_rq);
+    } else {
+	debug(49, 1) ("snmpHandleUdp: FD %d recvfrom: %s\n", sock, xstrerror());
+    }
+}
+
+/*
+ * Turn SNMP packet into a PDU, check available ACL's
+ */
+static void
+snmpDecodePacket(snmp_request_t * rq)
+{
+    struct snmp_pdu *PDU;
+    struct snmp_session Session;
+    aclCheck_t checklist;
+    u_char *Community;
+    u_char *buf = rq->buf;
+    int len = rq->len;
+    int allow = 0;
+
+    debug(49, 5) ("snmpDecodePacket: Called.\n");
+    /* Now that we have the data, turn it into a PDU */
+    PDU = snmp_pdu_create(0);
+    Session.Version = SNMP_VERSION_1;
+    Community = snmp_parse(&Session, PDU, buf, len);
+    memset(&checklist, '\0', sizeof(checklist));
+    checklist.src_addr = rq->from.sin_addr;
+    checklist.snmp_community = (char *) Community;
+
+    if (Community)
+	allow = aclCheckFast(Config.accessList.snmp, &checklist);
+    if ((snmp_coexist_V2toV1(PDU)) && (Community) && (allow)) {
+	rq->community = Community;
+	rq->PDU = PDU;
+	debug(49, 5) ("snmpAgentParse: reqid=[%d]\n", PDU->reqid);
+	snmpConstructReponse(rq);
+    } else {
+	debug(49, 1) ("Failed SNMP agent query from : %s.\n",
+	    inet_ntoa(rq->from.sin_addr));
+	snmp_free_pdu(PDU);
+    }
+    if (Community)
+	xfree(Community);
+}
+
+/*
+ * Packet OK, ACL Check OK, Create reponse.
+ */
+static void
+snmpConstructReponse(snmp_request_t * rq)
+{
+    struct snmp_session Session;
+    struct snmp_pdu *RespPDU;
+
+    debug(49, 5) ("snmpConstructReponse: Called.\n");
+    RespPDU = snmpAgentResponse(rq->PDU);
+    snmp_free_pdu(rq->PDU);
+    if (RespPDU != NULL) {
+	Session.Version = SNMP_VERSION_1;
+	Session.community = rq->community;
+	Session.community_len = strlen((char *) rq->community);
+	snmp_build(&Session, RespPDU, rq->outbuf, &rq->outlen);
+	sendto(rq->sock, rq->outbuf, rq->outlen, 0, (struct sockaddr *) &rq->from, sizeof(rq->from));
+	snmp_free_pdu(RespPDU);
+    }
+}
+
+/*
+ * Decide how to respond to the request, construct a response and
+ * return the response to the requester.
+ */
+static struct snmp_pdu *
+snmpAgentResponse(struct snmp_pdu *PDU)
+{
+    struct snmp_pdu *Answer = NULL;
+    oid_ParseFn *ParseFn = NULL;
+
+    variable_list *VarPtr, *VarNew = NULL, **VarPtrP;
+    int index = 0;
+
+    debug(49, 5) ("snmpAgentResponse: Called.\n");
+
+    if ((Answer = snmp_pdu_create(SNMP_PDU_RESPONSE))) {
+	Answer->reqid = PDU->reqid;
+	Answer->errindex = 0;
+	if (PDU->command == SNMP_PDU_GET) {
+	    variable_list **RespVars;
+
+	    RespVars = &(Answer->variables);
+	    /* Loop through all variables */
+	    for (VarPtrP = &(PDU->variables);
+		*VarPtrP;
+		VarPtrP = &((*VarPtrP)->next_variable)) {
+		VarPtr = *VarPtrP;
+
+		index++;
+
+		/* Find the parsing function for this variable */
+		ParseFn = snmpTreeGet(VarPtr->name, VarPtr->name_length);
+
+		if (ParseFn == NULL) {
+		    Answer->errstat = SNMP_ERR_NOSUCHNAME;
+		    debug(49, 5) ("snmpAgentResponse: No such oid. ");
+		} else
+		    VarNew = (*ParseFn) (VarPtr, (snint *) & (Answer->errstat));
+
+		/* Was there an error? */
+		if ((Answer->errstat != SNMP_ERR_NOERROR) ||
+		    (VarNew == NULL)) {
+		    Answer->errindex = index;
+		    debug(49, 5) ("snmpAgentParse: successful.\n");
+		    /* Just copy the rest of the variables.  Quickly. */
+		    *RespVars = VarPtr;
+		    *VarPtrP = NULL;
+		    return (Answer);
+		}
+		/* No error.  Insert this var at the end, and move on to the next.
+		 */
+		*RespVars = VarNew;
+		RespVars = &(VarNew->next_variable);
+	    }
+	    return (Answer);
+	} else if (PDU->command == SNMP_PDU_GETNEXT) {
+	    oid *NextOidName = NULL;
+	    int NextOidNameLen = 0;
+
+	    ParseFn = snmpTreeNext(PDU->variables->name, PDU->variables->name_length,
+		&(NextOidName), (snint *) & NextOidNameLen);
+
+	    if (ParseFn == NULL) {
+		Answer->errstat = SNMP_ERR_NOSUCHNAME;
+		debug(49, 5) ("snmpAgentResponse: No such oid: ");
+		snmpDebugOid(5, PDU->variables->name, PDU->variables->name_length);
+	    } else {
+		xfree(PDU->variables->name);
+		PDU->variables->name = NextOidName;
+		PDU->variables->name_length = NextOidNameLen;
+		VarNew = (*ParseFn) (PDU->variables, (snint *) & Answer->errstat);
+	    }
+
+	    /* Was there an error? */
+	    if (Answer->errstat != SNMP_ERR_NOERROR) {
+		Answer->errindex = 1;
+		Answer->variables = PDU->variables;
+		PDU->variables = NULL;
+	    } else {
+		Answer->variables = VarNew;
+	    }
+
+	} else {
+	    snmp_free_pdu(Answer);
+	    Answer = NULL;
+	}
+    }
+    return (Answer);
+}
+
+static oid_ParseFn *
+snmpTreeGet(oid * Current, snint CurrentLen)
+{
+    oid_ParseFn *Fn = NULL;
+    mib_tree_entry *mibTreeEntry = NULL;
+    int count = 0;
+
+    debug(49, 5) ("snmpTreeGet: Called\n");
+
+    debug(49, 6) ("snmpTreeGet: Current : \n");
+    snmpDebugOid(6, Current, CurrentLen);
+
+    mibTreeEntry = mib_tree_head;
+    if (Current[count] == mibTreeEntry->name[count]) {
+	count++;
+	while ((mibTreeEntry) && (count < CurrentLen) && (!mibTreeEntry->parsefunction)) {
+	    mibTreeEntry = snmpTreeEntry(Current[count], count, mibTreeEntry);
+	    count++;
+	}
+    }
+    if (mibTreeEntry && mibTreeEntry->parsefunction)
+	Fn = mibTreeEntry->parsefunction;
+    debug(49, 5) ("snmpTreeGet: return\n");
+    return (Fn);
+}
+
+static oid_ParseFn *
+snmpTreeNext(oid * Current, snint CurrentLen, oid ** Next, snint * NextLen)
+{
+    oid_ParseFn *Fn = NULL;
+    mib_tree_entry *mibTreeEntry = NULL, *nextoid = NULL;
+    int count = 0;
+
+    debug(49, 5) ("snmpTreeNext: Called\n");
+
+    debug(49, 6) ("snmpTreeNext: Current : \n");
+    snmpDebugOid(6, Current, CurrentLen);
+
+    mibTreeEntry = mib_tree_head;
+    if (Current[count] == mibTreeEntry->name[count]) {
+	count++;
+	while ((mibTreeEntry) && (count < CurrentLen) && (!mibTreeEntry->parsefunction)) {
+	    mibTreeEntry = snmpTreeEntry(Current[count], count, mibTreeEntry);
+	    count++;
+	}
+	debug(49, 5) ("snmpTreeNext: Recursed down to requested object\n");
+    } else {
+	return NULL;
+    }
+    if (mibTreeEntry == mib_tree_last)
+	return (Fn);
+    if ((mibTreeEntry) && (mibTreeEntry->parsefunction)) {
+	*NextLen = CurrentLen;
+	*Next = (*mibTreeEntry->instancefunction) (Current, NextLen, mibTreeEntry, &Fn);
+	if (*Next)
+	    return (Fn);
+    }
+    if ((mibTreeEntry) && (mibTreeEntry->parsefunction)) {
+	count--;
+	nextoid = snmpTreeSiblingEntry(Current[count], count, mibTreeEntry->parent);
+	if (nextoid) {
+	    debug(49, 5) ("snmpTreeNext: Next OID found for sibling\n");
+	    mibTreeEntry = nextoid;
+	    count++;
+	} else {
+	    debug(49, 5) ("snmpTreeNext: Attempting to recurse up for next object\n");
+	    while (!nextoid) {
+		count--;
+		if (mibTreeEntry->parent->parent) {
+		    nextoid = mibTreeEntry->parent;
+		    mibTreeEntry = snmpTreeEntry(Current[count] + 1, count, nextoid->parent);
+		    if (!mibTreeEntry) {
+			mibTreeEntry = nextoid;
+			nextoid = NULL;
+		    }
+		} else {
+		    nextoid = mibTreeEntry;
+		    mibTreeEntry = NULL;
+		}
+	    }
+	}
+    }
+    while ((mibTreeEntry) && (!mibTreeEntry->parsefunction)) {
+	mibTreeEntry = mibTreeEntry->leaves[0];
+    }
+    if (mibTreeEntry) {
+	*NextLen = mibTreeEntry->len;
+	*Next = (*mibTreeEntry->instancefunction) (mibTreeEntry->name, NextLen, mibTreeEntry, &Fn);
+    }
+    if (*Next)
+	return (Fn);
+    else
+	return NULL;
+}
+
+static oid *
+static_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
+{
+    oid *instance = NULL;
+
+    if (*len <= current->len) {
+	instance = (oid *)xmalloc(sizeof(name) * (*len + 1));
+	xmemcpy(instance, name, (sizeof(name) * *len));
+	instance[*len] = 0;
+	*len += 1;
+    }
+    *Fn = current->parsefunction;
+    return (instance);
+}
+
+static oid *
+time_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
+{
+    oid *instance = NULL;
+    int identifier = 0, loop = 0;
+    int index[TIME_INDEX_LEN] =
+    {TIME_INDEX};
+
+    if (*len <= current->len) {
+	instance = (oid *)xmalloc(sizeof(name) * (*len + 1));
+	xmemcpy(instance, name, (sizeof(name) * *len));
+	instance[*len] = *index;
+	*len += 1;
+    } else {
+	identifier = name[*len - 1];
+	while ((identifier != index[loop]) && (loop < TIME_INDEX_LEN))
+	    loop++;
+	if (loop < TIME_INDEX_LEN - 1) {
+	    instance = (oid *)xmalloc(sizeof(name) * (*len));
+	    xmemcpy(instance, name, (sizeof(name) * *len));
+	    instance[*len - 1] = index[++loop];
+	}
+    }
+    *Fn = current->parsefunction;
+    return (instance);
+}
+
+static oid *
+peer_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
+{
+    oid *instance = NULL;
+    u_char *cp = NULL;
+    peer *peers = Config.peers;
+    struct in_addr *laddr = NULL;
+    char *host_addr = NULL, *current_addr = NULL, *last_addr = NULL;
+
+    if (peers == NULL) {
+	current = current->parent->parent->parent->leaves[1];
+	while ((current) && (!current->parsefunction))
+	    current = current->leaves[0];
+	instance = client_Inst(current->name, len, current, Fn);
+    } else if (*len <= current->len) {
+	instance = (oid *)xmalloc(sizeof(name) * (*len + 4));
+	xmemcpy(instance, name, (sizeof(name) * *len));
+	cp = (u_char *) & (peers->in_addr.sin_addr.s_addr);
+	instance[*len] = *cp++;
+	instance[*len + 1] = *cp++;
+	instance[*len + 2] = *cp++;
+	instance[*len + 3] = *cp++;
+	*len += 4;
+    } else {
+	laddr = oid2addr(&name[*len - 4]);
+	host_addr = inet_ntoa(*laddr);
+	last_addr = (char *)xmalloc(strlen(host_addr));
+	strncpy(last_addr, host_addr, strlen(host_addr));
+	current_addr = inet_ntoa(peers->in_addr.sin_addr);
+	while ((peers) && (strncmp(last_addr, current_addr, strlen(current_addr)))) {
+	    if (peers->next) {
+		peers = peers->next;
+		current_addr = inet_ntoa(peers->in_addr.sin_addr);
+	    } else {
+		peers = NULL;
+	    }
+	}
+	xfree(last_addr);
+	if (peers) {
+	    if (peers->next) {
+		peers = peers->next;
+		instance = (oid *)xmalloc(sizeof(name) * (*len));
+		xmemcpy(instance, name, (sizeof(name) * *len));
+		cp = (u_char *) & (peers->in_addr.sin_addr.s_addr);
+		instance[*len - 4] = *cp++;
+		instance[*len - 3] = *cp++;
+		instance[*len - 2] = *cp++;
+		instance[*len - 1] = *cp++;
+	    } else {
+		return (instance);
+	    }
+	} else {
+	    return (instance);
+	}
+    }
+    *Fn = current->parsefunction;
+    return (instance);
+}
+
+static oid *
+client_Inst(oid * name, snint * len, mib_tree_entry * current, oid_ParseFn ** Fn)
+{
+    oid *instance = NULL;
+    u_char *cp = NULL;
+    struct in_addr *laddr = NULL;
+
+    if (*len <= current->len) {
+	instance = (oid *)xmalloc(sizeof(name) * (*len + 4));
+	xmemcpy(instance, name, (sizeof(name) * *len));
+	laddr = client_entry(NULL);
+	if (laddr) {
+	    cp = (u_char *) & (laddr->s_addr);
+	    instance[*len] = *cp++;
+	    instance[*len + 1] = *cp++;
+	    instance[*len + 2] = *cp++;
+	    instance[*len + 3] = *cp++;
+	    *len += 4;
+	}
+    } else {
+	laddr = oid2addr(&name[*len - 4]);
+	laddr = client_entry(laddr);
+	if (laddr) {
+	    instance = (oid *)xmalloc(sizeof(name) * (*len));
+	    xmemcpy(instance, name, (sizeof(name) * *len));
+	    cp = (u_char *) & (laddr->s_addr);
+	    instance[*len - 4] = *cp++;
+	    instance[*len - 3] = *cp++;
+	    instance[*len - 2] = *cp++;
+	    instance[*len - 1] = *cp++;
+	}
+    }
+    *Fn = current->parsefunction;
+    return (instance);
+}
+
+
+/*
+ * Utility functions
+ */
+
+/*
+ * Tree utility functions. 
+ */
+
+/* 
+ * Returns a the sibling object in the tree
+ */
+static mib_tree_entry *
+snmpTreeSiblingEntry(oid entry, snint len, mib_tree_entry * current)
+{
+    mib_tree_entry *next = NULL;
+    int count = 0;
+
+    while ((!next) && (count < current->children)) {
+	if (current->leaves[count]->name[len] == entry) {
+	    next = current->leaves[count];
+	}
+	count++;
+    }
+    if (count < current->children) {
+	next = current->leaves[count];
+    } else {
+	next = NULL;
+    }
+    return (next);
+}
+
+/* 
+ * Returns the requested child object or NULL if it does not exist
+ */
+static mib_tree_entry *
+snmpTreeEntry(oid entry, snint len, mib_tree_entry * current)
+{
+    mib_tree_entry *next = NULL;
+    int count = 0;
+
+    while ((!next) && (count < current->children)) {
+	if (current->leaves[count]->name[len] == entry) {
+	    next = current->leaves[count];
+	}
+	count++;
+    }
+    return (next);
+}
+
+/*
+ * Adds a node to the MIB tree structure and adds the appropriate children
+ */
+static mib_tree_entry *
+#if STDC_HEADERS
+snmpAddNode(oid * name, int len, oid_ParseFn * parsefunction, instance_Fn * instancefunction, int children,...)
+#else
+snmpAddNode(va_alist)
+     va_dcl
+#endif
+{
+#if STDC_HEADERS
+    va_list args;
+    int loop;
+    mib_tree_entry *entry = NULL;
+    va_start(args, children);
+#else
+    va_list args;
+    oid *name = NULL;
+    int len = 0, children = 0, loop;
+    oid_ParseFn *parsefunction = NULL;
+    instance_Fn *instancefunction = NULL;
+    mib_tree_entry *entry = NULL;
+    va_start(args);
+    name = va_arg(args, oid *);
+    len = va_arg(args, int);
+    parsefunction = va_arg(args, oid_ParseFn *);
+    instancefunction = va_arg(args, instance_Fn *);
+    children = va_arg(args, int);
+#endif
+
+    debug(49, 6) ("snmpAddNode: Children : %d, Oid : \n", children);
+    snmpDebugOid(6, name, len);
+
+    va_start(args, children);
+    entry = (mib_tree_entry *)xmalloc(sizeof(mib_tree_entry));
+    entry->name = name;
+    entry->len = len;
+    entry->parsefunction = parsefunction;
+    entry->instancefunction = instancefunction;
+    entry->children = children;
+
+    if (children > 0) {
+	entry->leaves = (mib_tree_entry **)xmalloc(sizeof(mib_tree_entry *) * children);
+	for (loop = 0; loop < children; loop++) {
+	    entry->leaves[loop] = va_arg(args, mib_tree_entry *);
+	    entry->leaves[loop]->parent = entry;
+	}
+    }
+    return (entry);
+}
+/* End of tree utility functions */
+
+/* 
+ * Returns the list of parameters in an oid
+ */
+static oid *
+#if STDC_HEADERS
+snmpCreateOid(int length,...)
+#else
+snmpCreateOid(va_alist)
+     va_dcl
+#endif
+{
+#if STDC_HEADERS
+    va_list args;
+    oid *new_oid;
+    int loop;
+    va_start(args, length);
+#else
+    va_list args;
+    int length = 0, loop;
+    oid *new_oid;
+    va_start(args);
+    length va_arg(args, int);
+#endif
+
+    new_oid = (oid *)xmalloc(sizeof(oid) * length);
+
+    if (length > 0) {
+	for (loop = 0; loop < length; loop++) {
+	    new_oid[loop] = va_arg(args, int);
+	}
+    }
+    return (new_oid);
+}
+
+#if UNUSED_CODE
+/*
+ * Allocate space for, and copy, an OID.  Returns new oid.
+ */
+static oid *
+snmpOidDup(oid * A, snint ALen)
+{
+    oid *Ans = xmalloc(sizeof(oid) * ALen);
+    xmemcpy(Ans, A, (sizeof(oid) * ALen));
+    return Ans;
+}
+#endif
+
+/*
+ * Debug calls, prints out the OID for debugging purposes.
+ */
+void
+snmpDebugOid(int lvl, oid * Name, snint Len)
+{
+    char mbuf[16], objid[1024];
+    int x;
+    objid[0] = '\0';
+
+    for (x = 0; x < Len; x++) {
+	snprintf(mbuf, sizeof(mbuf), ".%u", (unsigned int) Name[x]);
+	strncat(objid, mbuf, sizeof(objid));
+    }
+
+    debug(49, lvl) ("   oid = %s\n", objid);
+}
+
+static void
+snmpSnmplibDebug(int lvl, char *buf)
+{
+    debug(49, lvl) ("%s", buf);
+}
+
+void
+addr2oid(struct in_addr addr, oid * Dest)
+{
+    u_char *cp;
+    cp = (u_char *) & (addr.s_addr);
+    Dest[0] = *cp++;
+    Dest[1] = *cp++;
+    Dest[2] = *cp++;
+    Dest[3] = *cp++;
+}
+
+struct in_addr
+       *
+oid2addr(oid * id)
+{
+    static struct in_addr laddr;
+    u_char *cp = (u_char *) & (laddr.s_addr);
+    cp[0] = id[0];
+    cp[1] = id[1];
+    cp[2] = id[2];
+    cp[3] = id[3];
+    return &laddr;
+}
--- squid/src/ssl_support.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,356 +0,0 @@
-
-/*
- * $Id$
- *
- * AUTHOR: Benno Rice
- * DEBUG: section 83    SSL accelerator support
- *
- * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from the
- *  Internet community.  Development is led by Duane Wessels of the
- *  National Laboratory for Applied Network Research and funded by the
- *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
- *  Duane Wessels and the University of California San Diego.  Please
- *  see the COPYRIGHT file for full details.  Squid incorporates
- *  software developed and/or copyrighted by other sources.  Please see
- *  the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-
-extern int commUnsetNonBlocking(int fd);
-extern int commSetNonBlocking(int fd);
-
-void clientNegotiateSSL(int fd, void *data);
-void clientReadSSLRequest(int fd, void *data);
-
-static RSA *
-ssl_temp_rsa_cb(SSL * ssl, int export, int keylen)
-{
-    static RSA *rsa = NULL;
-
-    if (rsa == NULL)
-	rsa = RSA_generate_key(512, RSA_F4, NULL, NULL);
-    return rsa;
-}
-
-static int
-ssl_verify_cb(int ok, X509_STORE_CTX * ctx)
-{
-    char buffer[256];
-
-    X509_NAME_oneline(X509_get_issuer_name(ctx->current_cert), buffer,
-	sizeof(buffer));
-    if (ok)
-	debug(83, 5) ("SSL Certificate OK: %s\n", buffer);
-    else {
-	switch (ctx->error) {
-	case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT:
-	    debug(83, 5) ("SSL Certficate error: CA not known: %s\n", buffer);
-	    break;
-	case X509_V_ERR_CERT_NOT_YET_VALID:
-	    debug(83, 5) ("SSL Certficate not yet valid: %s\n", buffer);
-	    break;
-	case X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD:
-	    debug(83, 5) ("SSL Certificate has illegal \'not before\' field: %s\n", buffer);
-	    break;
-	case X509_V_ERR_CERT_HAS_EXPIRED:
-	    debug(83, 5) ("SSL Certificate expired: %s\n", buffer);
-	    break;
-	case X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD:
-	    debug(83, 5) ("SSL Certificate has invalid \'not after\' field: %s\n", buffer);
-	    break;
-	default:
-	    debug(83, 5) ("SSL unknown certificate error %d in %s\n",
-		ctx->error, buffer);
-	    break;
-	}
-    }
-    return ok;
-}
-
-static struct ssl_option {
-    const char *name;
-    long value;
-} ssl_options[] = {
-
-    {
-	"MICROSOFT_SESS_ID_BUG", SSL_OP_MICROSOFT_SESS_ID_BUG
-    },
-    {
-	"NETSCAPE_CHALLENGE_BUG", SSL_OP_NETSCAPE_CHALLENGE_BUG
-    },
-    {
-	"NETSCAPE_REUSE_CIPHER_CHANGE_BUG", SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG
-    },
-    {
-	"SSLREF2_REUSE_CERT_TYPE_BUG", SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG
-    },
-    {
-	"MICROSOFT_BIG_SSLV3_BUFFER", SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER
-    },
-    {
-	"MSIE_SSLV2_RSA_PADDING", SSL_OP_MSIE_SSLV2_RSA_PADDING
-    },
-    {
-	"SSLEAY_080_CLIENT_DH_BUG", SSL_OP_SSLEAY_080_CLIENT_DH_BUG
-    },
-    {
-	"TLS_D5_BUG", SSL_OP_TLS_D5_BUG
-    },
-    {
-	"TLS_BLOCK_PADDING_BUG", SSL_OP_TLS_BLOCK_PADDING_BUG
-    },
-    {
-	"TLS_ROLLBACK_BUG", SSL_OP_TLS_ROLLBACK_BUG
-    },
-    {
-	"SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE
-    },
-    {
-	"EPHEMERAL_RSA", SSL_OP_EPHEMERAL_RSA
-    },
-    {
-	"PKCS1_CHECK_1", SSL_OP_PKCS1_CHECK_1
-    },
-    {
-	"PKCS1_CHECK_2", SSL_OP_PKCS1_CHECK_2
-    },
-    {
-	"NETSCAPE_CA_DN_BUG", SSL_OP_NETSCAPE_CA_DN_BUG
-    },
-    {
-	"NON_EXPORT_FIRST", SSL_OP_NON_EXPORT_FIRST
-    },
-    {
-	"NETSCAPE_DEMO_CIPHER_CHANGE_BUG", SSL_OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG
-    },
-    {
-	"ALL", SSL_OP_ALL
-    },
-    {
-	"NO_SSLv2", SSL_OP_NO_SSLv2
-    },
-    {
-	"NO_SSLv3", SSL_OP_NO_SSLv3
-    },
-    {
-	"NO_TLSv1", SSL_OP_NO_TLSv1
-    },
-    {
-	"", 0
-    },
-    {
-	NULL, 0
-    }
-};
-
-static long
-ssl_parse_options(const char *options)
-{
-    long op = SSL_OP_ALL;
-    char *tmp;
-    char *option;
-
-    if (!options)
-	goto no_options;
-
-    tmp = xstrdup(options);
-    option = strtok(tmp, ":,");
-    while (option) {
-	struct ssl_option *opt = NULL, *opttmp;
-	long value = 0;
-	enum {
-	    MODE_ADD, MODE_REMOVE
-	} mode;
-	switch (*option) {
-	case '!':
-	case '-':
-	    mode = MODE_REMOVE;
-	    option++;
-	    break;
-	case '+':
-	    mode = MODE_ADD;
-	    option++;
-	    break;
-	default:
-	    mode = MODE_ADD;
-	    break;
-	}
-	for (opttmp = ssl_options; opttmp->name; opttmp++) {
-	    if (strcmp(opttmp->name, option) == 0) {
-		opt = opttmp;
-		break;
-	    }
-	}
-	if (opt)
-	    value = opt->value;
-	else if (strncmp(option, "0x", 2) == 0) {
-	    /* Special case.. hex specification */
-	    value = strtol(option + 2, NULL, 16);
-	} else {
-	    fatalf("Unknown SSL option '%s'", option);
-	    value = 0;		/* Keep GCC happy */
-	}
-	switch (mode) {
-	case MODE_ADD:
-	    op |= value;
-	    break;
-	case MODE_REMOVE:
-	    op &= ~value;
-	    break;
-	}
-	option = strtok(NULL, ":,");
-    }
-
-    safe_free(tmp);
-  no_options:
-    return op;
-}
-
-SSL_CTX *
-sslCreateContext(const char *certfile, const char *keyfile, int version, const char *cipher, const char *options)
-{
-    int ssl_error;
-    SSL_METHOD *method;
-    SSL_CTX *sslContext;
-    static int ssl_initialized = 0;
-    if (!ssl_initialized) {
-	ssl_initialized = 1;
-	SSL_load_error_strings();
-	SSLeay_add_ssl_algorithms();
-    }
-    if (!keyfile)
-	keyfile = certfile;
-    if (!certfile)
-	certfile = keyfile;
-
-    debug(83, 1) ("Initialising SSL.\n");
-    switch (version) {
-    case 2:
-	debug(83, 5) ("Using SSLv2.\n");
-	method = SSLv2_server_method();
-	break;
-    case 3:
-	debug(83, 5) ("Using SSLv3.\n");
-	method = SSLv3_server_method();
-	break;
-    case 4:
-	debug(83, 5) ("Using TLSv1.\n");
-	method = TLSv1_server_method();
-	break;
-    case 1:
-    default:
-	debug(83, 5) ("Using SSLv2/SSLv3.\n");
-	method = SSLv23_server_method();
-	break;
-    }
-
-    sslContext = SSL_CTX_new(method);
-    if (sslContext == NULL) {
-	ssl_error = ERR_get_error();
-	fatalf("Failed to allocate SSL context: %s\n",
-	    ERR_error_string(ssl_error, NULL));
-    }
-    SSL_CTX_set_options(sslContext, ssl_parse_options(options));
-
-    if (cipher) {
-	debug(83, 5) ("Using chiper suite %s.\n", cipher);
-	if (!SSL_CTX_set_cipher_list(sslContext, cipher)) {
-	    ssl_error = ERR_get_error();
-	    fatalf("Failed to set SSL cipher suite: %s\n",
-		ERR_error_string(ssl_error, NULL));
-	}
-    }
-    debug(83, 1) ("Using certificate in %s\n", certfile);
-    if (!SSL_CTX_use_certificate_file(sslContext, certfile, SSL_FILETYPE_PEM)) {
-	ssl_error = ERR_get_error();
-	fatalf("Failed to acquire SSL certificate: %s\n",
-	    ERR_error_string(ssl_error, NULL));
-    }
-    debug(83, 1) ("Using private key in %s\n", keyfile);
-    if (!SSL_CTX_use_PrivateKey_file(sslContext, keyfile, SSL_FILETYPE_PEM)) {
-	ssl_error = ERR_get_error();
-	fatalf("Failed to acquire SSL private key: %s\n",
-	    ERR_error_string(ssl_error, NULL));
-    }
-    debug(83, 5) ("Comparing private and public SSL keys.\n");
-    if (!SSL_CTX_check_private_key(sslContext))
-	fatal("SSL private key does not match public key: %s\n");
-
-    debug(83, 9) ("Setting RSA key generation callback.\n");
-    SSL_CTX_set_tmp_rsa_callback(sslContext, ssl_temp_rsa_cb);
-
-    debug(83, 9) ("Setting certificate verification callback.\n");
-    SSL_CTX_set_verify(sslContext, SSL_VERIFY_NONE, ssl_verify_cb);
-
-    debug(83, 9) ("Setting default CA certificate location.\n");
-    if (!SSL_CTX_set_default_verify_paths(sslContext)) {
-	ssl_error = ERR_get_error();
-	debug(83, 1) ("Error error setting default CA certificate location: %s\n",
-	    ERR_error_string(ssl_error, NULL));
-	debug(83, 1) ("continuing anyway...\n");
-    }
-    debug(83, 9) ("Set client certifying authority list.\n");
-    SSL_CTX_set_client_CA_list(sslContext, SSL_load_client_CA_file(certfile));
-    return sslContext;
-}
-
-int
-ssl_read_method(fd, buf, len)
-     int fd;
-     char *buf;
-     int len;
-{
-    int i;
-
-    i = SSL_read(fd_table[fd].ssl, buf, len);
-
-    if (i > 0 && SSL_pending(fd_table[fd].ssl) > 0) {
-	debug(83, 2) ("SSL fd %d is pending\n", fd);
-	fd_table[fd].flags.read_pending = 1;
-    } else
-	fd_table[fd].flags.read_pending = 0;
-
-    return i;
-}
-
-int
-ssl_write_method(fd, buf, len)
-     int fd;
-     const char *buf;
-     int len;
-{
-    return (SSL_write(fd_table[fd].ssl, buf, len));
-}
-
-void
-ssl_shutdown_method(fd)
-{
-    SSL *ssl = fd_table[fd].ssl;
-    if (!fd_table[fd].ssl_shutdown) {
-	fd_table[fd].ssl_shutdown = 1;
-	if (Config.SSL.unclean_shutdown)
-	    SSL_set_shutdown(ssl, SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN);
-	else
-	    SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN);
-    }
-    SSL_shutdown(ssl);
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/ssl_support.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,350 @@
+
+/*
+ * $Id: ssl_support.cc,v 1.1.2.1 2002/10/11 15:41:02 rbcollins Exp $
+ *
+ * AUTHOR: Benno Rice
+ * DEBUG: section 83    SSL accelerator support
+ *
+ * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from the
+ *  Internet community.  Development is led by Duane Wessels of the
+ *  National Laboratory for Applied Network Research and funded by the
+ *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
+ *  Duane Wessels and the University of California San Diego.  Please
+ *  see the COPYRIGHT file for full details.  Squid incorporates
+ *  software developed and/or copyrighted by other sources.  Please see
+ *  the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+extern int commUnsetNonBlocking(int fd);
+extern int commSetNonBlocking(int fd);
+
+void clientNegotiateSSL(int fd, void *data);
+void clientReadSSLRequest(int fd, void *data);
+
+static RSA *
+ssl_temp_rsa_cb(SSL * ssl, int anInt, int keylen)
+{
+    static RSA *rsa = NULL;
+
+    if (rsa == NULL)
+	rsa = RSA_generate_key(512, RSA_F4, NULL, NULL);
+    return rsa;
+}
+
+static int
+ssl_verify_cb(int ok, X509_STORE_CTX * ctx)
+{
+    char buffer[256];
+
+    X509_NAME_oneline(X509_get_issuer_name(ctx->current_cert), buffer,
+	sizeof(buffer));
+    if (ok)
+	debug(83, 5) ("SSL Certificate OK: %s\n", buffer);
+    else {
+	switch (ctx->error) {
+	case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT:
+	    debug(83, 5) ("SSL Certficate error: CA not known: %s\n", buffer);
+	    break;
+	case X509_V_ERR_CERT_NOT_YET_VALID:
+	    debug(83, 5) ("SSL Certficate not yet valid: %s\n", buffer);
+	    break;
+	case X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD:
+	    debug(83, 5) ("SSL Certificate has illegal \'not before\' field: %s\n", buffer);
+	    break;
+	case X509_V_ERR_CERT_HAS_EXPIRED:
+	    debug(83, 5) ("SSL Certificate expired: %s\n", buffer);
+	    break;
+	case X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD:
+	    debug(83, 5) ("SSL Certificate has invalid \'not after\' field: %s\n", buffer);
+	    break;
+	default:
+	    debug(83, 5) ("SSL unknown certificate error %d in %s\n",
+		ctx->error, buffer);
+	    break;
+	}
+    }
+    return ok;
+}
+
+static struct ssl_option {
+    const char *name;
+    long value;
+} ssl_options[] = {
+
+    {
+	"MICROSOFT_SESS_ID_BUG", SSL_OP_MICROSOFT_SESS_ID_BUG
+    },
+    {
+	"NETSCAPE_CHALLENGE_BUG", SSL_OP_NETSCAPE_CHALLENGE_BUG
+    },
+    {
+	"NETSCAPE_REUSE_CIPHER_CHANGE_BUG", SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG
+    },
+    {
+	"SSLREF2_REUSE_CERT_TYPE_BUG", SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG
+    },
+    {
+	"MICROSOFT_BIG_SSLV3_BUFFER", SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER
+    },
+    {
+	"MSIE_SSLV2_RSA_PADDING", SSL_OP_MSIE_SSLV2_RSA_PADDING
+    },
+    {
+	"SSLEAY_080_CLIENT_DH_BUG", SSL_OP_SSLEAY_080_CLIENT_DH_BUG
+    },
+    {
+	"TLS_D5_BUG", SSL_OP_TLS_D5_BUG
+    },
+    {
+	"TLS_BLOCK_PADDING_BUG", SSL_OP_TLS_BLOCK_PADDING_BUG
+    },
+    {
+	"TLS_ROLLBACK_BUG", SSL_OP_TLS_ROLLBACK_BUG
+    },
+    {
+	"SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE
+    },
+    {
+	"EPHEMERAL_RSA", SSL_OP_EPHEMERAL_RSA
+    },
+    {
+	"PKCS1_CHECK_1", SSL_OP_PKCS1_CHECK_1
+    },
+    {
+	"PKCS1_CHECK_2", SSL_OP_PKCS1_CHECK_2
+    },
+    {
+	"NETSCAPE_CA_DN_BUG", SSL_OP_NETSCAPE_CA_DN_BUG
+    },
+    {
+	"NON_EXPORT_FIRST", SSL_OP_NON_EXPORT_FIRST
+    },
+    {
+	"NETSCAPE_DEMO_CIPHER_CHANGE_BUG", SSL_OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG
+    },
+    {
+	"ALL", SSL_OP_ALL
+    },
+    {
+	"NO_SSLv2", SSL_OP_NO_SSLv2
+    },
+    {
+	"NO_SSLv3", SSL_OP_NO_SSLv3
+    },
+    {
+	"NO_TLSv1", SSL_OP_NO_TLSv1
+    },
+    {
+	"", 0
+    },
+    {
+	NULL, 0
+    }
+};
+
+static long
+ssl_parse_options(const char *options)
+{
+    long op = SSL_OP_ALL;
+    char *tmp;
+    char *option;
+
+    if (!options)
+	goto no_options;
+
+    tmp = xstrdup(options);
+    option = strtok(tmp, ":,");
+    while (option) {
+	struct ssl_option *opt = NULL, *opttmp;
+	long value = 0;
+	enum {
+	    MODE_ADD, MODE_REMOVE
+	} mode;
+	switch (*option) {
+	case '!':
+	case '-':
+	    mode = MODE_REMOVE;
+	    option++;
+	    break;
+	case '+':
+	    mode = MODE_ADD;
+	    option++;
+	    break;
+	default:
+	    mode = MODE_ADD;
+	    break;
+	}
+	for (opttmp = ssl_options; opttmp->name; opttmp++) {
+	    if (strcmp(opttmp->name, option) == 0) {
+		opt = opttmp;
+		break;
+	    }
+	}
+	if (opt)
+	    value = opt->value;
+	else if (strncmp(option, "0x", 2) == 0) {
+	    /* Special case.. hex specification */
+	    value = strtol(option + 2, NULL, 16);
+	} else {
+	    fatalf("Unknown SSL option '%s'", option);
+	    value = 0;		/* Keep GCC happy */
+	}
+	switch (mode) {
+	case MODE_ADD:
+	    op |= value;
+	    break;
+	case MODE_REMOVE:
+	    op &= ~value;
+	    break;
+	}
+	option = strtok(NULL, ":,");
+    }
+
+    safe_free(tmp);
+  no_options:
+    return op;
+}
+
+SSL_CTX *
+sslCreateContext(const char *certfile, const char *keyfile, int version, const char *cipher, const char *options)
+{
+    int ssl_error;
+    SSL_METHOD *method;
+    SSL_CTX *sslContext;
+    static int ssl_initialized = 0;
+    if (!ssl_initialized) {
+	ssl_initialized = 1;
+	SSL_load_error_strings();
+	SSLeay_add_ssl_algorithms();
+    }
+    if (!keyfile)
+	keyfile = certfile;
+    if (!certfile)
+	certfile = keyfile;
+
+    debug(83, 1) ("Initialising SSL.\n");
+    switch (version) {
+    case 2:
+	debug(83, 5) ("Using SSLv2.\n");
+	method = SSLv2_server_method();
+	break;
+    case 3:
+	debug(83, 5) ("Using SSLv3.\n");
+	method = SSLv3_server_method();
+	break;
+    case 4:
+	debug(83, 5) ("Using TLSv1.\n");
+	method = TLSv1_server_method();
+	break;
+    case 1:
+    default:
+	debug(83, 5) ("Using SSLv2/SSLv3.\n");
+	method = SSLv23_server_method();
+	break;
+    }
+
+    sslContext = SSL_CTX_new(method);
+    if (sslContext == NULL) {
+	ssl_error = ERR_get_error();
+	fatalf("Failed to allocate SSL context: %s\n",
+	    ERR_error_string(ssl_error, NULL));
+    }
+    SSL_CTX_set_options(sslContext, ssl_parse_options(options));
+
+    if (cipher) {
+	debug(83, 5) ("Using chiper suite %s.\n", cipher);
+	if (!SSL_CTX_set_cipher_list(sslContext, cipher)) {
+	    ssl_error = ERR_get_error();
+	    fatalf("Failed to set SSL cipher suite: %s\n",
+		ERR_error_string(ssl_error, NULL));
+	}
+    }
+    debug(83, 1) ("Using certificate in %s\n", certfile);
+    if (!SSL_CTX_use_certificate_file(sslContext, certfile, SSL_FILETYPE_PEM)) {
+	ssl_error = ERR_get_error();
+	fatalf("Failed to acquire SSL certificate: %s\n",
+	    ERR_error_string(ssl_error, NULL));
+    }
+    debug(83, 1) ("Using private key in %s\n", keyfile);
+    if (!SSL_CTX_use_PrivateKey_file(sslContext, keyfile, SSL_FILETYPE_PEM)) {
+	ssl_error = ERR_get_error();
+	fatalf("Failed to acquire SSL private key: %s\n",
+	    ERR_error_string(ssl_error, NULL));
+    }
+    debug(83, 5) ("Comparing private and public SSL keys.\n");
+    if (!SSL_CTX_check_private_key(sslContext))
+	fatal("SSL private key does not match public key: %s\n");
+
+    debug(83, 9) ("Setting RSA key generation callback.\n");
+    SSL_CTX_set_tmp_rsa_callback(sslContext, ssl_temp_rsa_cb);
+
+    debug(83, 9) ("Setting certificate verification callback.\n");
+    SSL_CTX_set_verify(sslContext, SSL_VERIFY_NONE, ssl_verify_cb);
+
+    debug(83, 9) ("Setting default CA certificate location.\n");
+    if (!SSL_CTX_set_default_verify_paths(sslContext)) {
+	ssl_error = ERR_get_error();
+	debug(83, 1) ("Error error setting default CA certificate location: %s\n",
+	    ERR_error_string(ssl_error, NULL));
+	debug(83, 1) ("continuing anyway...\n");
+    }
+    debug(83, 9) ("Set client certifying authority list.\n");
+    SSL_CTX_set_client_CA_list(sslContext, SSL_load_client_CA_file(certfile));
+    return sslContext;
+}
+
+int
+ssl_read_method(int fd, char *buf, int len)
+{
+    int i;
+
+    i = SSL_read(fd_table[fd].ssl, buf, len);
+
+    if (i > 0 && SSL_pending(fd_table[fd].ssl) > 0) {
+	debug(83, 2) ("SSL fd %d is pending\n", fd);
+	fd_table[fd].flags.read_pending = 1;
+    } else
+	fd_table[fd].flags.read_pending = 0;
+
+    return i;
+}
+
+int
+ssl_write_method(int fd, const char *buf, int len)
+{
+    return (SSL_write(fd_table[fd].ssl, buf, len));
+}
+
+void
+ssl_shutdown_method(int fd)
+{
+    SSL *ssl = fd_table[fd].ssl;
+    if (!fd_table[fd].ssl_shutdown) {
+	fd_table[fd].ssl_shutdown = 1;
+	if (Config.SSL.unclean_shutdown)
+	    SSL_set_shutdown(ssl, SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN);
+	else
+	    SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN);
+    }
+    SSL_shutdown(ssl);
+}
--- squid/src/store_key_md5.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,188 +0,0 @@
-
-/*
- * $Id: store_key_md5.c,v 1.6 2001/04/14 00:31:02 squidadm Exp $
- *
- * DEBUG: section 20    Storage Manager MD5 Cache Keys
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-
-static cache_key null_key[MD5_DIGEST_CHARS];
-
-const char *
-storeKeyText(const unsigned char *key)
-{
-    static MemBuf mb = MemBufNULL;
-    int i;
-    memBufReset(&mb);
-    for (i = 0; i < MD5_DIGEST_CHARS; i++)
-	memBufPrintf(&mb, "%02X", *(key + i));
-    return mb.buf;
-}
-
-const cache_key *
-storeKeyScan(const char *buf)
-{
-    static unsigned char digest[MD5_DIGEST_CHARS];
-    int i;
-    int j = 0;
-    char t[3];
-    for (i = 0; i < MD5_DIGEST_CHARS; i++) {
-	t[0] = *(buf + (j++));
-	t[1] = *(buf + (j++));
-	t[2] = '\0';
-	*(digest + i) = (unsigned char) strtol(t, NULL, 16);
-    }
-    return digest;
-}
-
-int
-storeKeyHashCmp(const void *a, const void *b)
-{
-    const unsigned char *A = a;
-    const unsigned char *B = b;
-    int i;
-    for (i = 0; i < MD5_DIGEST_CHARS; i++) {
-	if (A[i] < B[i])
-	    return -1;
-	if (A[i] > B[i])
-	    return 1;
-    }
-    return 0;
-}
-
-unsigned int
-storeKeyHashHash(const void *key, unsigned int n)
-{
-    /* note, n must be a power of 2! */
-    const unsigned char *digest = key;
-    unsigned int i = digest[0]
-    | digest[1] << 8
-    | digest[2] << 16
-    | digest[3] << 24;
-    return (i & (--n));
-}
-
-const cache_key *
-storeKeyPrivate(const char *url, method_t method, int id)
-{
-    static cache_key digest[MD5_DIGEST_CHARS];
-    MD5_CTX M;
-    assert(id > 0);
-    debug(20, 3) ("storeKeyPrivate: %s %s\n",
-	RequestMethodStr[method], url);
-    MD5Init(&M);
-    MD5Update(&M, (unsigned char *) &id, sizeof(id));
-    MD5Update(&M, (unsigned char *) &method, sizeof(method));
-    MD5Update(&M, (unsigned char *) url, strlen(url));
-    MD5Final(digest, &M);
-    return digest;
-}
-
-const cache_key *
-storeKeyPublic(const char *url, const method_t method)
-{
-    static cache_key digest[MD5_DIGEST_CHARS];
-    unsigned char m = (unsigned char) method;
-    MD5_CTX M;
-    MD5Init(&M);
-    MD5Update(&M, &m, sizeof(m));
-    MD5Update(&M, (unsigned char *) url, strlen(url));
-    MD5Final(digest, &M);
-    return digest;
-}
-
-const cache_key *
-storeKeyPublicByRequest(request_t * request)
-{
-    return storeKeyPublicByRequestMethod(request, request->method);
-}
-
-const cache_key *
-storeKeyPublicByRequestMethod(request_t * request, const method_t method)
-{
-    static cache_key digest[MD5_DIGEST_CHARS];
-    unsigned char m = (unsigned char) method;
-    const char *url = urlCanonical(request);
-    MD5_CTX M;
-    MD5Init(&M);
-    MD5Update(&M, &m, sizeof(m));
-    MD5Update(&M, (unsigned char *) url, strlen(url));
-    if (request->vary_headers)
-	MD5Update(&M, (unsigned char *) request->vary_headers, strlen(request->vary_headers));
-    MD5Final(digest, &M);
-    return digest;
-}
-
-cache_key *
-storeKeyDup(const cache_key * key)
-{
-    cache_key *dup = memAllocate(MEM_MD5_DIGEST);
-    xmemcpy(dup, key, MD5_DIGEST_CHARS);
-    return dup;
-}
-
-cache_key *
-storeKeyCopy(cache_key * dst, const cache_key * src)
-{
-    xmemcpy(dst, src, MD5_DIGEST_CHARS);
-    return dst;
-}
-
-void
-storeKeyFree(const cache_key * key)
-{
-    memFree((void *) key, MEM_MD5_DIGEST);
-}
-
-int
-storeKeyHashBuckets(int nbuckets)
-{
-    int n = 0x2000;
-    while (n < nbuckets)
-	n <<= 1;
-    return n;
-}
-
-int
-storeKeyNull(const cache_key * key)
-{
-    if (memcmp(key, null_key, MD5_DIGEST_CHARS) == 0)
-	return 1;
-    else
-	return 0;
-}
-
-void
-storeKeyInit(void)
-{
-    memset(null_key, '\0', MD5_DIGEST_CHARS);
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/store_key_md5.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,188 @@
+
+/*
+ * $Id: store_key_md5.cc,v 1.1.2.1 2002/10/11 15:41:02 rbcollins Exp $
+ *
+ * DEBUG: section 20    Storage Manager MD5 Cache Keys
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+static cache_key null_key[MD5_DIGEST_CHARS];
+
+const char *
+storeKeyText(const unsigned char *key)
+{
+    static MemBuf mb = MemBufNULL;
+    int i;
+    memBufReset(&mb);
+    for (i = 0; i < MD5_DIGEST_CHARS; i++)
+	memBufPrintf(&mb, "%02X", *(key + i));
+    return mb.buf;
+}
+
+const cache_key *
+storeKeyScan(const char *buf)
+{
+    static unsigned char digest[MD5_DIGEST_CHARS];
+    int i;
+    int j = 0;
+    char t[3];
+    for (i = 0; i < MD5_DIGEST_CHARS; i++) {
+	t[0] = *(buf + (j++));
+	t[1] = *(buf + (j++));
+	t[2] = '\0';
+	*(digest + i) = (unsigned char) strtol(t, NULL, 16);
+    }
+    return digest;
+}
+
+int
+storeKeyHashCmp(const void *a, const void *b)
+{
+    const unsigned char *A = (const unsigned char *)a;
+    const unsigned char *B = (const unsigned char *)b;
+    int i;
+    for (i = 0; i < MD5_DIGEST_CHARS; i++) {
+	if (A[i] < B[i])
+	    return -1;
+	if (A[i] > B[i])
+	    return 1;
+    }
+    return 0;
+}
+
+unsigned int
+storeKeyHashHash(const void *key, unsigned int n)
+{
+    /* note, n must be a power of 2! */
+    const unsigned char *digest = (const unsigned char *)key;
+    unsigned int i = digest[0]
+    | digest[1] << 8
+    | digest[2] << 16
+    | digest[3] << 24;
+    return (i & (--n));
+}
+
+const cache_key *
+storeKeyPrivate(const char *url, method_t method, int id)
+{
+    static cache_key digest[MD5_DIGEST_CHARS];
+    MD5_CTX M;
+    assert(id > 0);
+    debug(20, 3) ("storeKeyPrivate: %s %s\n",
+	RequestMethodStr[method], url);
+    MD5Init(&M);
+    MD5Update(&M, (unsigned char *) &id, sizeof(id));
+    MD5Update(&M, (unsigned char *) &method, sizeof(method));
+    MD5Update(&M, (unsigned char *) url, strlen(url));
+    MD5Final(digest, &M);
+    return digest;
+}
+
+const cache_key *
+storeKeyPublic(const char *url, const method_t method)
+{
+    static cache_key digest[MD5_DIGEST_CHARS];
+    unsigned char m = (unsigned char) method;
+    MD5_CTX M;
+    MD5Init(&M);
+    MD5Update(&M, &m, sizeof(m));
+    MD5Update(&M, (unsigned char *) url, strlen(url));
+    MD5Final(digest, &M);
+    return digest;
+}
+
+const cache_key *
+storeKeyPublicByRequest(request_t * request)
+{
+    return storeKeyPublicByRequestMethod(request, request->method);
+}
+
+const cache_key *
+storeKeyPublicByRequestMethod(request_t * request, const method_t method)
+{
+    static cache_key digest[MD5_DIGEST_CHARS];
+    unsigned char m = (unsigned char) method;
+    const char *url = urlCanonical(request);
+    MD5_CTX M;
+    MD5Init(&M);
+    MD5Update(&M, &m, sizeof(m));
+    MD5Update(&M, (unsigned char *) url, strlen(url));
+    if (request->vary_headers)
+	MD5Update(&M, (unsigned char *) request->vary_headers, strlen(request->vary_headers));
+    MD5Final(digest, &M);
+    return digest;
+}
+
+cache_key *
+storeKeyDup(const cache_key * key)
+{
+    cache_key *dup = (cache_key *)memAllocate(MEM_MD5_DIGEST);
+    xmemcpy(dup, key, MD5_DIGEST_CHARS);
+    return dup;
+}
+
+cache_key *
+storeKeyCopy(cache_key * dst, const cache_key * src)
+{
+    xmemcpy(dst, src, MD5_DIGEST_CHARS);
+    return dst;
+}
+
+void
+storeKeyFree(const cache_key * key)
+{
+    memFree((void *) key, MEM_MD5_DIGEST);
+}
+
+int
+storeKeyHashBuckets(int nbuckets)
+{
+    int n = 0x2000;
+    while (n < nbuckets)
+	n <<= 1;
+    return n;
+}
+
+int
+storeKeyNull(const cache_key * key)
+{
+    if (memcmp(key, null_key, MD5_DIGEST_CHARS) == 0)
+	return 1;
+    else
+	return 0;
+}
+
+void
+storeKeyInit(void)
+{
+    memset(null_key, '\0', MD5_DIGEST_CHARS);
+}
--- squid/src/store_rebuild.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,197 +0,0 @@
-
-/*
- * $Id: store_rebuild.c,v 1.9.44.2 2002/10/04 21:57:52 rbcollins Exp $
- *
- * DEBUG: section 20    Store Rebuild Routines
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-#include "Store.h"
-
-static struct _store_rebuild_data counts;
-static struct timeval rebuild_start;
-static void storeCleanup(void *);
-
-typedef struct {
-    /* total number of "swap.state" entries that will be read */
-    int total;
-    /* number of entries read so far */
-    int scanned;
-} store_rebuild_progress;
-
-static store_rebuild_progress *RebuildProgress = NULL;
-
-static int
-storeCleanupDoubleCheck(StoreEntry * e)
-{
-    SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn];
-    return (SD->dblcheck(SD, e));
-}
-
-static void
-storeCleanup(void *datanotused)
-{
-    static int bucketnum = -1;
-    static int validnum = 0;
-    static int store_errors = 0;
-    int validnum_start;
-    StoreEntry *e;
-    hash_link *link_ptr = NULL;
-    hash_link *link_next = NULL;
-    validnum_start = validnum;
-    while (validnum - validnum_start < 500) {
-	if (++bucketnum >= store_hash_buckets) {
-	    debug(20, 1) ("  Completed Validation Procedure\n");
-	    debug(20, 1) ("  Validated %d Entries\n", validnum);
-	    debug(20, 1) ("  store_swap_size = %luk\n", store_swap_size);
-	    store_dirs_rebuilding--;
-	    assert(0 == store_dirs_rebuilding);
-	    if (opt_store_doublecheck)
-		assert(store_errors == 0);
-	    if (store_digest)
-		storeDigestNoteStoreReady();
-	    return;
-	}
-	link_next = hash_get_bucket(store_table, bucketnum);
-	while (NULL != (link_ptr = link_next)) {
-	    link_next = link_ptr->next;
-	    e = (StoreEntry *) link_ptr;
-	    if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
-		continue;
-	    /*
-	     * Calling storeRelease() has no effect because we're
-	     * still in 'store_rebuilding' state
-	     */
-	    if (e->swap_filen < 0)
-		continue;
-	    if (opt_store_doublecheck)
-		if (storeCleanupDoubleCheck(e))
-		    store_errors++;
-	    EBIT_SET(e->flags, ENTRY_VALIDATED);
-	    /*
-	     * Only set the file bit if we know its a valid entry
-	     * otherwise, set it in the validation procedure
-	     */
-	    storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1);
-	    if ((++validnum & 0x3FFFF) == 0)
-		debug(20, 1) ("  %7d Entries Validated so far.\n", validnum);
-	}
-    }
-    eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
-}
-
-/* meta data recreated from disk image in swap directory */
-void
-storeRebuildComplete(struct _store_rebuild_data *dc)
-{
-    double dt;
-    counts.objcount += dc->objcount;
-    counts.expcount += dc->expcount;
-    counts.scancount += dc->scancount;
-    counts.clashcount += dc->clashcount;
-    counts.dupcount += dc->dupcount;
-    counts.cancelcount += dc->cancelcount;
-    counts.invalid += dc->invalid;
-    counts.badflags += dc->badflags;
-    counts.bad_log_op += dc->bad_log_op;
-    counts.zero_object_sz += dc->zero_object_sz;
-    /*
-     * When store_dirs_rebuilding == 1, it means we are done reading
-     * or scanning all cache_dirs.  Now report the stats and start
-     * the validation (storeCleanup()) thread.
-     */
-    if (store_dirs_rebuilding > 1)
-	return;
-    dt = tvSubDsec(rebuild_start, current_time);
-    debug(20, 1) ("Finished rebuilding storage from disk.\n");
-    debug(20, 1) ("  %7d Entries scanned\n", counts.scancount);
-    debug(20, 1) ("  %7d Invalid entries.\n", counts.invalid);
-    debug(20, 1) ("  %7d With invalid flags.\n", counts.badflags);
-    debug(20, 1) ("  %7d Objects loaded.\n", counts.objcount);
-    debug(20, 1) ("  %7d Objects expired.\n", counts.expcount);
-    debug(20, 1) ("  %7d Objects cancelled.\n", counts.cancelcount);
-    debug(20, 1) ("  %7d Duplicate URLs purged.\n", counts.dupcount);
-    debug(20, 1) ("  %7d Swapfile clashes avoided.\n", counts.clashcount);
-    debug(20, 1) ("  Took %3.1f seconds (%6.1f objects/sec).\n", dt,
-	(double) counts.objcount / (dt > 0.0 ? dt : 1.0));
-    debug(20, 1) ("Beginning Validation Procedure\n");
-    eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
-    xfree(RebuildProgress);
-    RebuildProgress = NULL;
-}
-
-/*
- * this is ugly.  We don't actually start any rebuild threads here,
- * but only initialize counters, etc.  The rebuild threads are
- * actually started by the filesystem "fooDirInit" function.
- */
-void
-storeRebuildStart(void)
-{
-    memset(&counts, '\0', sizeof(counts));
-    rebuild_start = current_time;
-    /*
-     * Note: store_dirs_rebuilding is initialized to 1 in globals.c.
-     * This prevents us from trying to write clean logs until we
-     * finished rebuilding for sure.  The corresponding decrement
-     * occurs in storeCleanup(), when it is finished.
-     */
-    RebuildProgress = xcalloc(Config.cacheSwap.n_configured,
-	sizeof(store_rebuild_progress));
-}
-
-/*
- * A fs-specific rebuild procedure periodically reports its
- * progress.
- */
-void
-storeRebuildProgress(int sd_index, int total, int sofar)
-{
-    static time_t last_report = 0;
-    double n = 0.0;
-    double d = 0.0;
-    if (sd_index < 0)
-	return;
-    if (sd_index >= Config.cacheSwap.n_configured)
-	return;
-    if (NULL == RebuildProgress)
-	return;
-    RebuildProgress[sd_index].total = total;
-    RebuildProgress[sd_index].scanned = sofar;
-    if (squid_curtime - last_report < 15)
-	return;
-    for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; sd_index++) {
-	n += (double) RebuildProgress[sd_index].scanned;
-	d += (double) RebuildProgress[sd_index].total;
-    }
-    debug(20, 1) ("Store rebuilding is %4.1f%% complete\n", 100.0 * n / d);
-    last_report = squid_curtime;
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/store_rebuild.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,197 @@
+
+/*
+ * $Id: store_rebuild.cc,v 1.1.2.1 2002/10/11 15:41:02 rbcollins Exp $
+ *
+ * DEBUG: section 20    Store Rebuild Routines
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "Store.h"
+
+static struct _store_rebuild_data counts;
+static struct timeval rebuild_start;
+static void storeCleanup(void *);
+
+typedef struct {
+    /* total number of "swap.state" entries that will be read */
+    int total;
+    /* number of entries read so far */
+    int scanned;
+} store_rebuild_progress;
+
+static store_rebuild_progress *RebuildProgress = NULL;
+
+static int
+storeCleanupDoubleCheck(StoreEntry * e)
+{
+    SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn];
+    return (SD->dblcheck(SD, e));
+}
+
+static void
+storeCleanup(void *datanotused)
+{
+    static int bucketnum = -1;
+    static int validnum = 0;
+    static int store_errors = 0;
+    int validnum_start;
+    StoreEntry *e;
+    hash_link *link_ptr = NULL;
+    hash_link *link_next = NULL;
+    validnum_start = validnum;
+    while (validnum - validnum_start < 500) {
+	if (++bucketnum >= store_hash_buckets) {
+	    debug(20, 1) ("  Completed Validation Procedure\n");
+	    debug(20, 1) ("  Validated %d Entries\n", validnum);
+	    debug(20, 1) ("  store_swap_size = %luk\n", store_swap_size);
+	    store_dirs_rebuilding--;
+	    assert(0 == store_dirs_rebuilding);
+	    if (opt_store_doublecheck)
+		assert(store_errors == 0);
+	    if (store_digest)
+		storeDigestNoteStoreReady();
+	    return;
+	}
+	link_next = hash_get_bucket(store_table, bucketnum);
+	while (NULL != (link_ptr = link_next)) {
+	    link_next = link_ptr->next;
+	    e = (StoreEntry *) link_ptr;
+	    if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
+		continue;
+	    /*
+	     * Calling storeRelease() has no effect because we're
+	     * still in 'store_rebuilding' state
+	     */
+	    if (e->swap_filen < 0)
+		continue;
+	    if (opt_store_doublecheck)
+		if (storeCleanupDoubleCheck(e))
+		    store_errors++;
+	    EBIT_SET(e->flags, ENTRY_VALIDATED);
+	    /*
+	     * Only set the file bit if we know its a valid entry
+	     * otherwise, set it in the validation procedure
+	     */
+	    storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1);
+	    if ((++validnum & 0x3FFFF) == 0)
+		debug(20, 1) ("  %7d Entries Validated so far.\n", validnum);
+	}
+    }
+    eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
+}
+
+/* meta data recreated from disk image in swap directory */
+void
+storeRebuildComplete(struct _store_rebuild_data *dc)
+{
+    double dt;
+    counts.objcount += dc->objcount;
+    counts.expcount += dc->expcount;
+    counts.scancount += dc->scancount;
+    counts.clashcount += dc->clashcount;
+    counts.dupcount += dc->dupcount;
+    counts.cancelcount += dc->cancelcount;
+    counts.invalid += dc->invalid;
+    counts.badflags += dc->badflags;
+    counts.bad_log_op += dc->bad_log_op;
+    counts.zero_object_sz += dc->zero_object_sz;
+    /*
+     * When store_dirs_rebuilding == 1, it means we are done reading
+     * or scanning all cache_dirs.  Now report the stats and start
+     * the validation (storeCleanup()) thread.
+     */
+    if (store_dirs_rebuilding > 1)
+	return;
+    dt = tvSubDsec(rebuild_start, current_time);
+    debug(20, 1) ("Finished rebuilding storage from disk.\n");
+    debug(20, 1) ("  %7d Entries scanned\n", counts.scancount);
+    debug(20, 1) ("  %7d Invalid entries.\n", counts.invalid);
+    debug(20, 1) ("  %7d With invalid flags.\n", counts.badflags);
+    debug(20, 1) ("  %7d Objects loaded.\n", counts.objcount);
+    debug(20, 1) ("  %7d Objects expired.\n", counts.expcount);
+    debug(20, 1) ("  %7d Objects cancelled.\n", counts.cancelcount);
+    debug(20, 1) ("  %7d Duplicate URLs purged.\n", counts.dupcount);
+    debug(20, 1) ("  %7d Swapfile clashes avoided.\n", counts.clashcount);
+    debug(20, 1) ("  Took %3.1f seconds (%6.1f objects/sec).\n", dt,
+	(double) counts.objcount / (dt > 0.0 ? dt : 1.0));
+    debug(20, 1) ("Beginning Validation Procedure\n");
+    eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
+    xfree(RebuildProgress);
+    RebuildProgress = NULL;
+}
+
+/*
+ * this is ugly.  We don't actually start any rebuild threads here,
+ * but only initialize counters, etc.  The rebuild threads are
+ * actually started by the filesystem "fooDirInit" function.
+ */
+void
+storeRebuildStart(void)
+{
+    memset(&counts, '\0', sizeof(counts));
+    rebuild_start = current_time;
+    /*
+     * Note: store_dirs_rebuilding is initialized to 1 in globals.c.
+     * This prevents us from trying to write clean logs until we
+     * finished rebuilding for sure.  The corresponding decrement
+     * occurs in storeCleanup(), when it is finished.
+     */
+    RebuildProgress = (store_rebuild_progress *)xcalloc(Config.cacheSwap.n_configured,
+	sizeof(store_rebuild_progress));
+}
+
+/*
+ * A fs-specific rebuild procedure periodically reports its
+ * progress.
+ */
+void
+storeRebuildProgress(int sd_index, int total, int sofar)
+{
+    static time_t last_report = 0;
+    double n = 0.0;
+    double d = 0.0;
+    if (sd_index < 0)
+	return;
+    if (sd_index >= Config.cacheSwap.n_configured)
+	return;
+    if (NULL == RebuildProgress)
+	return;
+    RebuildProgress[sd_index].total = total;
+    RebuildProgress[sd_index].scanned = sofar;
+    if (squid_curtime - last_report < 15)
+	return;
+    for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; sd_index++) {
+	n += (double) RebuildProgress[sd_index].scanned;
+	d += (double) RebuildProgress[sd_index].total;
+    }
+    debug(20, 1) ("Store rebuilding is %4.1f%% complete\n", 100.0 * n / d);
+    last_report = squid_curtime;
+}
--- squid/src/store_swapin.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,102 +0,0 @@
-
-/*
- * $Id: store_swapin.c,v 1.7.2.1 2002/10/04 07:07:29 rbcollins Exp $
- *
- * DEBUG: section 20    Storage Manager Swapin Functions
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-#include "StoreClient.h"
-#include "Store.h"
-
-static STIOCB storeSwapInFileClosed;
-static STFNCB storeSwapInFileNotify;
-
-void
-storeSwapInStart(store_client * sc)
-{
-    StoreEntry *e = sc->entry;
-    storeIOState *sio;
-    assert(e->mem_status == NOT_IN_MEMORY);
-    if (!EBIT_TEST(e->flags, ENTRY_VALIDATED)) {
-	/* We're still reloading and haven't validated this entry yet */
-	return;
-    }
-    debug(20, 3) ("storeSwapInStart: called for %d %08X %s \n",
-	e->swap_dirn, e->swap_filen, storeKeyText(e->hash.key));
-    if (e->swap_status != SWAPOUT_WRITING && e->swap_status != SWAPOUT_DONE) {
-	debug(20, 1) ("storeSwapInStart: bad swap_status (%s)\n",
-	    swapStatusStr[e->swap_status]);
-	return;
-    }
-    if (e->swap_filen < 0) {
-	debug(20, 1) ("storeSwapInStart: swap_filen < 0\n");
-	return;
-    }
-    assert(e->mem_obj != NULL);
-    debug(20, 3) ("storeSwapInStart: Opening fileno %08X\n",
-	e->swap_filen);
-    sio = storeOpen(e, storeSwapInFileNotify, storeSwapInFileClosed, sc);
-    sc->swapin_sio = cbdataReference(sio);
-}
-
-static void
-storeSwapInFileClosed(void *data, int errflag, storeIOState * sio)
-{
-    store_client *sc = data;
-    StoreIOBuffer result =
-    {
-	{0}, 0, 0, sc->copyInto.data};
-    STCB *callback;
-    debug(20, 3) ("storeSwapInFileClosed: sio=%p, errflag=%d\n",
-	sio, errflag);
-    if (errflag)
-	result.flags.error = 1;
-    cbdataReferenceDone(sc->swapin_sio);
-    if ((callback = sc->callback)) {
-	assert(errflag <= 0);
-	sc->callback = NULL;
-	callback(sc->callback_data, result);
-    }
-    statCounter.swap.ins++;
-}
-
-static void
-storeSwapInFileNotify(void *data, int errflag, storeIOState * sio)
-{
-    store_client *sc = data;
-    StoreEntry *e = sc->entry;
-
-    debug(1, 3) ("storeSwapInFileNotify: changing %d/%d to %d/%d\n", e->swap_filen, e->swap_dirn, sio->swap_filen, sio->swap_dirn);
-
-    e->swap_filen = sio->swap_filen;
-    e->swap_dirn = sio->swap_dirn;
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/store_swapin.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,102 @@
+
+/*
+ * $Id: store_swapin.cc,v 1.1.2.1 2002/10/11 15:41:03 rbcollins Exp $
+ *
+ * DEBUG: section 20    Storage Manager Swapin Functions
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "StoreClient.h"
+#include "Store.h"
+
+static STIOCB storeSwapInFileClosed;
+static STFNCB storeSwapInFileNotify;
+
+void
+storeSwapInStart(store_client * sc)
+{
+    StoreEntry *e = sc->entry;
+    storeIOState *sio;
+    assert(e->mem_status == NOT_IN_MEMORY);
+    if (!EBIT_TEST(e->flags, ENTRY_VALIDATED)) {
+	/* We're still reloading and haven't validated this entry yet */
+	return;
+    }
+    debug(20, 3) ("storeSwapInStart: called for %d %08X %s \n",
+	e->swap_dirn, e->swap_filen, storeKeyText((const cache_key *)e->hash.key));
+    if (e->swap_status != SWAPOUT_WRITING && e->swap_status != SWAPOUT_DONE) {
+	debug(20, 1) ("storeSwapInStart: bad swap_status (%s)\n",
+	    swapStatusStr[e->swap_status]);
+	return;
+    }
+    if (e->swap_filen < 0) {
+	debug(20, 1) ("storeSwapInStart: swap_filen < 0\n");
+	return;
+    }
+    assert(e->mem_obj != NULL);
+    debug(20, 3) ("storeSwapInStart: Opening fileno %08X\n",
+	e->swap_filen);
+    sio = storeOpen(e, storeSwapInFileNotify, storeSwapInFileClosed, sc);
+    sc->swapin_sio = cbdataReference(sio);
+}
+
+static void
+storeSwapInFileClosed(void *data, int errflag, storeIOState * sio)
+{
+    store_client *sc = (store_client *)data;
+    StoreIOBuffer result =
+    {
+	{0}, 0, 0, sc->copyInto.data};
+    STCB *callback;
+    debug(20, 3) ("storeSwapInFileClosed: sio=%p, errflag=%d\n",
+	sio, errflag);
+    if (errflag)
+	result.flags.error = 1;
+    cbdataReferenceDone(sc->swapin_sio);
+    if ((callback = sc->callback)) {
+	assert(errflag <= 0);
+	sc->callback = NULL;
+	callback(sc->callback_data, result);
+    }
+    statCounter.swap.ins++;
+}
+
+static void
+storeSwapInFileNotify(void *data, int errflag, storeIOState * sio)
+{
+    store_client *sc = (store_client *)data;
+    StoreEntry *e = sc->entry;
+
+    debug(1, 3) ("storeSwapInFileNotify: changing %d/%d to %d/%d\n", e->swap_filen, e->swap_dirn, sio->swap_filen, sio->swap_dirn);
+
+    e->swap_filen = sio->swap_filen;
+    e->swap_dirn = sio->swap_dirn;
+}
--- squid/src/store_swapout.c	Wed Feb 14 01:07:40 2007
+++ /dev/null	Wed Feb 14 01:07:22 2007
@@ -1,386 +0,0 @@
-
-/*
- * $Id: store_swapout.c,v 1.14.2.1 2002/10/04 07:07:29 rbcollins Exp $
- *
- * DEBUG: section 20    Storage Manager Swapout Functions
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache          http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- *  Squid is the result of efforts by numerous individuals from
- *  the Internet community; see the CONTRIBUTORS file for full
- *  details.   Many organizations have provided support for Squid's
- *  development; see the SPONSORS file for full details.  Squid is
- *  Copyrighted (C) 2001 by the Regents of the University of
- *  California; see the COPYRIGHT file for full details.  Squid
- *  incorporates software developed and/or copyrighted by other
- *  sources; see the CREDITS file for full details.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *  
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *  
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
- *
- */
-
-#include "squid.h"
-#include "StoreClient.h"
-#include "Store.h"
-
-static off_t storeSwapOutObjectBytesOnDisk(const MemObject *);
-static void storeSwapOutStart(StoreEntry * e);
-static STIOCB storeSwapOutFileClosed;
-static STIOCB storeSwapOutFileNotify;
-
-/* start swapping object to disk */
-static void
-storeSwapOutStart(StoreEntry * e)
-{
-    generic_cbdata *c;
-    MemObject *mem = e->mem_obj;
-    int swap_hdr_sz = 0;
-    tlv *tlv_list;
-    char *buf;
-    storeIOState *sio;
-    assert(mem);
-    /* Build the swap metadata, so the filesystem will know how much
-     * metadata there is to store
-     */
-    debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to dirno %d, fileno %08X\n",
-	storeUrl(e), e->swap_dirn, e->swap_filen);
-    e->swap_status = SWAPOUT_WRITING;
-    tlv_list = storeSwapMetaBuild(e);
-    buf = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
-    storeSwapTLVFree(tlv_list);
-    mem->swap_hdr_sz = (size_t) swap_hdr_sz;
-    /* Create the swap file */
-    c = cbdataAlloc(generic_cbdata);
-    c->data = e;
-    sio = storeCreate(e, storeSwapOutFileNotify, storeSwapOutFileClosed, c);
-    mem->swapout.sio = cbdataReference(sio);
-    if (NULL == mem->swapout.sio) {
-	e->swap_status = SWAPOUT_NONE;
-	cbdataFree(c);
-	xfree(buf);
-	storeLog(STORE_LOG_SWAPOUTFAIL, e);
-	return;
-    }
-    storeLockObject(e);		/* Don't lock until after create, or the replacement
-				 * code might get confused */
-    /* Pick up the file number if it was assigned immediately */
-    e->swap_filen = mem->swapout.sio->swap_filen;
-    e->swap_dirn = mem->swapout.sio->swap_dirn;
-    /* write out the swap metadata */
-    storeWrite(mem->swapout.sio, buf, mem->swap_hdr_sz, 0, xfree);
-}
-
-static void
-storeSwapOutFileNotify(void *data, int errflag, storeIOState * sio)
-{
-    generic_cbdata *c = data;
-    StoreEntry *e = c->data;
-    MemObject *mem = e->mem_obj;
-    assert(e->swap_status == SWAPOUT_WRITING);
-    assert(mem);
-    assert(mem->swapout.sio == sio);
-    assert(errflag == 0);
-    e->swap_filen = mem->swapout.sio->swap_filen;
-    e->swap_dirn = mem->swapout.sio->swap_dirn;
-}
-
-void
-storeSwapOut(StoreEntry * e)
-{
-    MemObject *mem = e->mem_obj;
-    off_t lowest_offset;
-    off_t new_mem_lo;
-    off_t on_disk = 0;
-    ssize_t swapout_size;
-    ssize_t swap_buf_len;
-    if (mem == NULL)
-	return;
-    /* should we swap something out to disk? */
-    debug(20, 7) ("storeSwapOut: %s\n", storeUrl(e));
-    debug(20, 7) ("storeSwapOut: store_status = %s\n",
-	storeStatusStr[e->store_status]);
-    if (EBIT_TEST(e->flags, ENTRY_ABORTED)) {
-	assert(EBIT_TEST(e->flags, RELEASE_REQUEST));
-	storeSwapOutFileClose(e);
-	return;
-    }
-    if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
-	debug(20, 3) ("storeSwapOut: %s SPECIAL\n", storeUrl(e));
-	return;
-    }
-    debug(20, 7) ("storeSwapOut: mem->inmem_lo = %d\n",
-	(int) mem->inmem_lo);
-    debug(20, 7) ("storeSwapOut: mem->inmem_hi = %d\n",
-	(int) mem->inmem_hi);
-    debug(20, 7) ("storeSwapOut: swapout.queue_offset = %d\n",
-	(int) mem->swapout.queue_offset);
-    if (mem->swapout.sio)
-	debug(20, 7) ("storeSwapOut: storeOffset() = %d\n",
-	    (int) storeOffset(mem->swapout.sio));
-    assert(mem->inmem_hi >= mem->swapout.queue_offset);
-    lowest_offset = storeLowestMemReaderOffset(e);
-    debug(20, 7) ("storeSwapOut: lowest_offset = %d\n",
-	(int) lowest_offset);
-    /*
-     * Grab the swapout_size and check to see whether we're going to defer
-     * the swapout based upon size
-     */
-    swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset);
-    if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) {
-	/*
-	 * NOTE: the store_maxobjsize here is the max of optional
-	 * max-size values from 'cache_dir' lines.  It is not the
-	 * same as 'maximum_object_size'.  By default, store_maxobjsize
-	 * will be set to -1.  However, I am worried that this
-	 * deferance may consume a lot of memory in some cases.
-	 * It would be good to make this decision based on reply
-	 * content-length, rather than wait to accumulate huge
-	 * amounts of object data in memory.
-	 */
-	debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n");
-	return;
-    }
-    /*
-     * Careful.  lowest_offset can be greater than inmem_hi, such
-     * as in the case of a range request.
-     */
-    if (mem->inmem_hi < lowest_offset)
-	new_mem_lo = lowest_offset;
-    else if (mem->inmem_hi - mem->inmem_lo > Config.Store.maxInMemObjSize)
-	new_mem_lo = lowest_offset;
-    else
-	new_mem_lo = mem->inmem_lo;
-    assert(new_mem_lo >= mem->inmem_lo);
-    if (storeSwapOutAble(e)) {
-	/*
-	 * We should only free up to what we know has been written
-	 * to disk, not what has been queued for writing.  Otherwise
-	 * there will be a chunk of the data which is not in memory
-	 * and is not yet on disk.
-	 * The -1 makes sure the page isn't freed until storeSwapOut has
-	 * walked to the next page. (mem->swapout.memnode)
-	 */
-	if ((on_disk = storeSwapOutObjectBytesOnDisk(mem)) - 1 < new_mem_lo)
-	    new_mem_lo = on_disk - 1;
-	if (new_mem_lo == -1)
-	    new_mem_lo = 0;	/* the above might become -1 */
-    } else if (new_mem_lo > 0) {
-	/*
-	 * Its not swap-able, and we're about to delete a chunk,
-	 * so we must make it PRIVATE.  This is tricky/ugly because
-	 * for the most part, we treat swapable == cachable here.
-	 */
-	storeReleaseRequest(e);
-    }
-    stmemFreeDataUpto(&mem->data_hdr, new_mem_lo);
-    mem->inmem_lo = new_mem_lo;
-#if SIZEOF_OFF_T == 4
-    if (mem->inmem_hi > 0x7FFF0000) {
-	debug(20, 0) ("WARNING: preventing off_t overflow for %s\n", storeUrl(e));
-	storeAbort(e);
-	return;
-    }
-#endif
-    if (e->swap_status == SWAPOUT_WRITING)
-	assert(mem->inmem_lo <= on_disk);
-    if (!storeSwapOutAble(e))
-	return;
-    debug(20, 7) ("storeSwapOut: swapout_size = %d\n",
-	(int) swapout_size);
-    if (swapout_size == 0) {
-	if (e->store_status == STORE_OK)
-	    storeSwapOutFileClose(e);
-	return;			/* Nevermore! */
-    }
-    if (e->store_status == STORE_PENDING) {
-	/* wait for a full block to write */
-	if (swapout_size < SM_PAGE_SIZE)
-	    return;
-	/*
-	 * Wait until we are below the disk FD limit, only if the
-	 * next server-side read won't be deferred.
-	 */
-	if (storeTooManyDiskFilesOpen() && !fwdCheckDeferRead(-1, e))
-	    return;
-    }
-    /* Ok, we have stuff to swap out.  Is there a swapout.sio open? */
-    if (e->swap_status == SWAPOUT_NONE) {
-	assert(mem->swapout.sio == NULL);
-	assert(mem->inmem_lo == 0);
-	if (storeCheckCachable(e))
-	    storeSwapOutStart(e);
-	else
-	    return;
-	/* ENTRY_CACHABLE will be cleared and we'll never get here again */
-    }
-    if (NULL == mem->swapout.sio)
-	return;
-    do {
-	/*
-	 * Evil hack time.
-	 * We are paging out to disk in page size chunks. however, later on when
-	 * we update the queue position, we might not have a page (I *think*),
-	 * so we do the actual page update here.
-	 */
-
-	if (mem->swapout.memnode == NULL) {
-	    /* We need to swap out the first page */
-	    mem->swapout.memnode = mem->data_hdr.head;
-	} else {
-	    /* We need to swap out the next page */
-	    mem->swapout.memnode = mem->swapout.memnode->next;
-	}
-	/*
-	 * Get the length of this buffer. We are assuming(!) that the buffer
-	 * length won't change on this buffer, or things are going to be very
-	 * strange. I think that after the copy to a buffer is done, the buffer
-	 * size should stay fixed regardless so that this code isn't confused,
-	 * but we can look at this at a later date or whenever the code results
-	 * in bad swapouts, whichever happens first. :-)
-	 */
-	swap_buf_len = mem->swapout.memnode->len;
-
-	debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len);
-	assert(swap_buf_len > 0);
-	debug(20, 3) ("storeSwapOut: swapping out %ld bytes from %ld\n",
-	    (long int) swap_buf_len, (long int) mem->swapout.queue_offset);
-	mem->swapout.queue_offset += swap_buf_len;
-	storeWrite(mem->swapout.sio, mem->swapout.memnode->data, swap_buf_len, -1, NULL);
-	/* the storeWrite() call might generate an error */
-	if (e->swap_status != SWAPOUT_WRITING)
-	    break;
-	swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset);
-	if (e->store_status == STORE_PENDING)
-	    if (swapout_size < SM_PAGE_SIZE)
-		break;
-    } while (swapout_size > 0);
-    if (NULL == mem->swapout.sio)
-	/* oops, we're not swapping out any more */
-	return;
-    if (e->store_status == STORE_OK) {
-	/*
-	 * If the state is STORE_OK, then all data must have been given
-	 * to the filesystem at this point because storeSwapOut() is
-	 * not going to be called again for this entry.
-	 */
-	assert(mem->inmem_hi == mem->swapout.queue_offset);
-	storeSwapOutFileClose(e);
-    }
-}
-
-void
-storeSwapOutFileClose(StoreEntry * e)
-{
-    MemObject *mem = e->mem_obj;
-    assert(mem != NULL);
-    debug(20, 3) ("storeSwapOutFileClose: %s\n", storeKeyText(e->hash.key));
-    debug(20, 3) ("storeSwapOutFileClose: sio = %p\n", mem->swapout.sio);
-    if (mem->swapout.sio == NULL)
-	return;
-    storeClose(mem->swapout.sio);
-}
-
-static void
-storeSwapOutFileClosed(void *data, int errflag, storeIOState * sio)
-{
-    generic_cbdata *c = data;
-    StoreEntry *e = c->data;
-    MemObject *mem = e->mem_obj;
-    assert(e->swap_status == SWAPOUT_WRITING);
-    cbdataFree(c);
-    if (errflag) {
-	debug(20, 1) ("storeSwapOutFileClosed: dirno %d, swapfile %08X, errflag=%d\n\t%s\n",
-	    e->swap_dirn, e->swap_filen, errflag, xstrerror());
-	if (errflag == DISK_NO_SPACE_LEFT) {
-	    storeDirDiskFull(e->swap_dirn);
-	    storeDirConfigure();
-	    storeConfigure();
-	}
-	if (e->swap_filen > 0)
-	    storeUnlink(e);
-	e->swap_filen = -1;
-	e->swap_dirn = -1;
-	e->swap_status = SWAPOUT_NONE;
-	storeReleaseRequest(e);
-    } else {
-	/* swapping complete */
-	debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %d, %08X\n",
-	    storeUrl(e), e->swap_dirn, e->swap_filen);
-	e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz;
-	e->swap_status = SWAPOUT_DONE;
-	storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1);
-	if (storeCheckCachable(e)) {
-	    storeLog(STORE_LOG_SWAPOUT, e);
-	    storeDirSwapLog(e, SWAP_LOG_ADD);
-	}
-	statCounter.swap.outs++;
-    }
-    debug(20, 3) ("storeSwapOutFileClosed: %s:%d\n", __FILE__, __LINE__);
-    cbdataReferenceDone(mem->swapout.sio);
-    storeUnlockObject(e);
-}
-
-/*
- * How much of the object data is on the disk?
- */
-static off_t
-storeSwapOutObjectBytesOnDisk(const MemObject * mem)
-{
-    /*
-     * NOTE: storeOffset() represents the disk file size,
-     * not the amount of object data on disk.
-     * 
-     * If we don't have at least 'swap_hdr_sz' bytes
-     * then none of the object data is on disk.
-     *
-     * This should still be safe if swap_hdr_sz == 0,
-     * meaning we haven't even opened the swapout file
-     * yet.
-     */
-    off_t nwritten;
-    if (mem->swapout.sio == NULL)
-	return 0;
-    nwritten = storeOffset(mem->swapout.sio);
-    if (nwritten <= mem->swap_hdr_sz)
-	return 0;
-    return nwritten - mem->swap_hdr_sz;
-}
-
-/*
- * Is this entry a candidate for writing to disk?
- */
-int
-storeSwapOutAble(const StoreEntry * e)
-{
-    dlink_node *node;
-    if (e->mem_obj->swapout.sio != NULL)
-	return 1;
-    if (e->mem_obj->inmem_lo > 0)
-	return 0;
-    /*
-     * If there are DISK clients, we must write to disk
-     * even if its not cachable
-     */
-    for (node = e->mem_obj->clients.head; node; node = node->next) {
-	if (((store_client *) node->data)->type == STORE_DISK_CLIENT)
-	    return 1;
-    }
-    /* Don't pollute the disk with icons and other special entries */
-    if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
-	return 0;
-    return EBIT_TEST(e->flags, ENTRY_CACHABLE);
-}
--- /dev/null	Wed Feb 14 01:07:22 2007
+++ squid/src/store_swapout.cc	Wed Feb 14 01:07:40 2007
@@ -0,0 +1,386 @@
+
+/*
+ * $Id: store_swapout.cc,v 1.1.2.1 2002/10/11 15:41:05 rbcollins Exp $
+ *
+ * DEBUG: section 20    Storage Manager Swapout Functions
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "StoreClient.h"
+#include "Store.h"
+
+static off_t storeSwapOutObjectBytesOnDisk(const MemObject *);
+static void storeSwapOutStart(StoreEntry * e);
+static STIOCB storeSwapOutFileClosed;
+static STIOCB storeSwapOutFileNotify;
+
+/* start swapping object to disk */
+static void
+storeSwapOutStart(StoreEntry * e)
+{
+    generic_cbdata *c;
+    MemObject *mem = e->mem_obj;
+    int swap_hdr_sz = 0;
+    tlv *tlv_list;
+    char *buf;
+    storeIOState *sio;
+    assert(mem);
+    /* Build the swap metadata, so the filesystem will know how much
+     * metadata there is to store
+     */
+    debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to dirno %d, fileno %08X\n",
+	storeUrl(e), e->swap_dirn, e->swap_filen);
+    e->swap_status = SWAPOUT_WRITING;
+    tlv_list = storeSwapMetaBuild(e);
+    buf = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
+    storeSwapTLVFree(tlv_list);
+    mem->swap_hdr_sz = (size_t) swap_hdr_sz;
+    /* Create the swap file */
+    c = cbdataAlloc(generic_cbdata);
+    c->data = e;
+    sio = storeCreate(e, storeSwapOutFileNotify, storeSwapOutFileClosed, c);
+    mem->swapout.sio = cbdataReference(sio);
+    if (NULL == mem->swapout.sio) {
+	e->swap_status = SWAPOUT_NONE;
+	cbdataFree(c);
+	xfree(buf);
+	storeLog(STORE_LOG_SWAPOUTFAIL, e);
+	return;
+    }
+    storeLockObject(e);		/* Don't lock until after create, or the replacement
+				 * code might get confused */
+    /* Pick up the file number if it was assigned immediately */
+    e->swap_filen = mem->swapout.sio->swap_filen;
+    e->swap_dirn = mem->swapout.sio->swap_dirn;
+    /* write out the swap metadata */
+    storeWrite(mem->swapout.sio, buf, mem->swap_hdr_sz, 0, xfree);
+}
+
+static void
+storeSwapOutFileNotify(void *data, int errflag, storeIOState * sio)
+{
+    generic_cbdata *c = (generic_cbdata *)data;
+    StoreEntry *e = (StoreEntry *)c->data;
+    MemObject *mem = e->mem_obj;
+    assert(e->swap_status == SWAPOUT_WRITING);
+    assert(mem);
+    assert(mem->swapout.sio == sio);
+    assert(errflag == 0);
+    e->swap_filen = mem->swapout.sio->swap_filen;
+    e->swap_dirn = mem->swapout.sio->swap_dirn;
+}
+
+void
+storeSwapOut(StoreEntry * e)
+{
+    MemObject *mem = e->mem_obj;
+    off_t lowest_offset;
+    off_t new_mem_lo;
+    off_t on_disk = 0;
+    ssize_t swapout_size;
+    ssize_t swap_buf_len;
+    if (mem == NULL)
+	return;
+    /* should we swap something out to disk? */
+    debug(20, 7) ("storeSwapOut: %s\n", storeUrl(e));
+    debug(20, 7) ("storeSwapOut: store_status = %s\n",
+	storeStatusStr[e->store_status]);
+    if (EBIT_TEST(e->flags, ENTRY_ABORTED)) {
+	assert(EBIT_TEST(e->flags, RELEASE_REQUEST));
+	storeSwapOutFileClose(e);
+	return;
+    }
+    if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+	debug(20, 3) ("storeSwapOut: %s SPECIAL\n", storeUrl(e));
+	return;
+    }
+    debug(20, 7) ("storeSwapOut: mem->inmem_lo = %d\n",
+	(int) mem->inmem_lo);
+    debug(20, 7) ("storeSwapOut: mem->inmem_hi = %d\n",
+	(int) mem->inmem_hi);
+    debug(20, 7) ("storeSwapOut: swapout.queue_offset = %d\n",
+	(int) mem->swapout.queue_offset);
+    if (mem->swapout.sio)
+	debug(20, 7) ("storeSwapOut: storeOffset() = %d\n",
+	    (int) storeOffset(mem->swapout.sio));
+    assert(mem->inmem_hi >= mem->swapout.queue_offset);
+    lowest_offset = storeLowestMemReaderOffset(e);
+    debug(20, 7) ("storeSwapOut: lowest_offset = %d\n",
+	(int) lowest_offset);
+    /*
+     * Grab the swapout_size and check to see whether we're going to defer
+     * the swapout based upon size
+     */
+    swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset);
+    if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) {
+	/*
+	 * NOTE: the store_maxobjsize here is the max of optional
+	 * max-size values from 'cache_dir' lines.  It is not the
+	 * same as 'maximum_object_size'.  By default, store_maxobjsize
+	 * will be set to -1.  However, I am worried that this
+	 * deferance may consume a lot of memory in some cases.
+	 * It would be good to make this decision based on reply
+	 * content-length, rather than wait to accumulate huge
+	 * amounts of object data in memory.
+	 */
+	debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n");
+	return;
+    }
+    /*
+     * Careful.  lowest_offset can be greater than inmem_hi, such
+     * as in the case of a range request.
+     */
+    if (mem->inmem_hi < lowest_offset)
+	new_mem_lo = lowest_offset;
+    else if (mem->inmem_hi - mem->inmem_lo > (ssize_t)Config.Store.maxInMemObjSize)
+	new_mem_lo = lowest_offset;
+    else
+	new_mem_lo = mem->inmem_lo;
+    assert(new_mem_lo >= mem->inmem_lo);
+    if (storeSwapOutAble(e)) {
+	/*
+	 * We should only free up to what we know has been written
+	 * to disk, not what has been queued for writing.  Otherwise
+	 * there will be a chunk of the data which is not in memory
+	 * and is not yet on disk.
+	 * The -1 makes sure the page isn't freed until storeSwapOut has
+	 * walked to the next page. (mem->swapout.memnode)
+	 */
+	if ((on_disk = storeSwapOutObjectBytesOnDisk(mem)) - 1 < new_mem_lo)
+	    new_mem_lo = on_disk - 1;
+	if (new_mem_lo == -1)
+	    new_mem_lo = 0;	/* the above might become -1 */
+    } else if (new_mem_lo > 0) {
+	/*
+	 * Its not swap-able, and we're about to delete a chunk,
+	 * so we must make it PRIVATE.  This is tricky/ugly because
+	 * for the most part, we treat swapable == cachable here.
+	 */
+	storeReleaseRequest(e);
+    }
+    stmemFreeDataUpto(&mem->data_hdr, new_mem_lo);
+    mem->inmem_lo = new_mem_lo;
+#if SIZEOF_OFF_T == 4
+    if (mem->inmem_hi > 0x7FFF0000) {
+	debug(20, 0) ("WARNING: preventing off_t overflow for %s\n", storeUrl(e));
+	storeAbort(e);
+	return;
+    }
+#endif
+    if (e->swap_status == SWAPOUT_WRITING)
+	assert(mem->inmem_lo <= on_disk);
+    if (!storeSwapOutAble(e))
+	return;
+    debug(20, 7) ("storeSwapOut: swapout_size = %d\n",
+	(int) swapout_size);
+    if (swapout_size == 0) {
+	if (e->store_status == STORE_OK)
+	    storeSwapOutFileClose(e);
+	return;			/* Nevermore! */
+    }
+    if (e->store_status == STORE_PENDING) {
+	/* wait for a full block to write */
+	if (swapout_size < SM_PAGE_SIZE)
+	    return;
+	/*
+	 * Wait until we are below the disk FD limit, only if the
+	 * next server-side read won't be deferred.
+	 */
+	if (storeTooManyDiskFilesOpen() && !fwdCheckDeferRead(-1, e))
+	    return;
+    }
+    /* Ok, we have stuff to swap out.  Is there a swapout.sio open? */
+    if (e->swap_status == SWAPOUT_NONE) {
+	assert(mem->swapout.sio == NULL);
+	assert(mem->inmem_lo == 0);
+	if (storeCheckCachable(e))
+	    storeSwapOutStart(e);
+	else
+	    return;
+	/* ENTRY_CACHABLE will be cleared and we'll never get here again */
+    }
+    if (NULL == mem->swapout.sio)
+	return;
+    do {
+	/*
+	 * Evil hack time.
+	 * We are paging out to disk in page size chunks. however, later on when
+	 * we update the queue position, we might not have a page (I *think*),
+	 * so we do the actual page update here.
+	 */
+
+	if (mem->swapout.memnode == NULL) {
+	    /* We need to swap out the first page */
+	    mem->swapout.memnode = mem->data_hdr.head;
+	} else {
+	    /* We need to swap out the next page */
+	    mem->swapout.memnode = mem->swapout.memnode->next;
+	}
+	/*
+	 * Get the length of this buffer. We are assuming(!) that the buffer
+	 * length won't change on this buffer, or things are going to be very
+	 * strange. I think that after the copy to a buffer is done, the buffer
+	 * size should stay fixed regardless so that this code isn't confused,
+	 * but we can look at this at a later date or whenever the code results
+	 * in bad swapouts, whichever happens first. :-)
+	 */
+	swap_buf_len = mem->swapout.memnode->len;
+
+	debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len);
+	assert(swap_buf_len > 0);
+	debug(20, 3) ("storeSwapOut: swapping out %ld bytes from %ld\n",
+	    (long int) swap_buf_len, (long int) mem->swapout.queue_offset);
+	mem->swapout.queue_offset += swap_buf_len;
+	storeWrite(mem->swapout.sio, mem->swapout.memnode->data, swap_buf_len, -1, NULL);
+	/* the storeWrite() call might generate an error */
+	if (e->swap_status != SWAPOUT_WRITING)
+	    break;
+	swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset);
+	if (e->store_status == STORE_PENDING)
+	    if (swapout_size < SM_PAGE_SIZE)
+		break;
+    } while (swapout_size > 0);
+    if (NULL == mem->swapout.sio)
+	/* oops, we're not swapping out any more */
+	return;
+    if (e->store_status == STORE_OK) {
+	/*
+	 * If the state is STORE_OK, then all data must have been given
+	 * to the filesystem at this point because storeSwapOut() is
+	 * not going to be called again for this entry.
+	 */
+	assert(mem->inmem_hi == mem->swapout.queue_offset);
+	storeSwapOutFileClose(e);
+    }
+}
+
+void
+storeSwapOutFileClose(StoreEntry * e)
+{
+    MemObject *mem = e->mem_obj;
+    assert(mem != NULL);
+    debug(20, 3) ("storeSwapOutFileClose: %s\n", storeKeyText((const cache_key *)e->hash.key));
+    debug(20, 3) ("storeSwapOutFileClose: sio = %p\n", mem->swapout.sio);
+    if (mem->swapout.sio == NULL)
+	return;
+    storeClose(mem->swapout.sio);
+}
+
+static void
+storeSwapOutFileClosed(void *data, int errflag, storeIOState * sio)
+{
+    generic_cbdata *c = (generic_cbdata *)data;
+    StoreEntry *e = (StoreEntry *)c->data;
+    MemObject *mem = e->mem_obj;
+    assert(e->swap_status == SWAPOUT_WRITING);
+    cbdataFree(c);
+    if (errflag) {
+	debug(20, 1) ("storeSwapOutFileClosed: dirno %d, swapfile %08X, errflag=%d\n\t%s\n",
+	    e->swap_dirn, e->swap_filen, errflag, xstrerror());
+	if (errflag == DISK_NO_SPACE_LEFT) {
+	    storeDirDiskFull(e->swap_dirn);
+	    storeDirConfigure();
+	    storeConfigure();
+	}
+	if (e->swap_filen > 0)
+	    storeUnlink(e);
+	e->swap_filen = -1;
+	e->swap_dirn = -1;
+	e->swap_status = SWAPOUT_NONE;
+	storeReleaseRequest(e);
+    } else {
+	/* swapping complete */
+	debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %d, %08X\n",
+	    storeUrl(e), e->swap_dirn, e->swap_filen);
+	e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz;
+	e->swap_status = SWAPOUT_DONE;
+	storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1);
+	if (storeCheckCachable(e)) {
+	    storeLog(STORE_LOG_SWAPOUT, e);
+	    storeDirSwapLog(e, SWAP_LOG_ADD);
+	}
+	statCounter.swap.outs++;
+    }
+    debug(20, 3) ("storeSwapOutFileClosed: %s:%d\n", __FILE__, __LINE__);
+    cbdataReferenceDone(mem->swapout.sio);
+    storeUnlockObject(e);
+}
+
+/*
+ * How much of the object data is on the disk?
+ */
+static off_t
+storeSwapOutObjectBytesOnDisk(const MemObject * mem)
+{
+    /*
+     * NOTE: storeOffset() represents the disk file size,
+     * not the amount of object data on disk.
+     * 
+     * If we don't have at least 'swap_hdr_sz' bytes
+     * then none of the object data is on disk.
+     *
+     * This should still be safe if swap_hdr_sz == 0,
+     * meaning we haven't even opened the swapout file
+     * yet.
+     */
+    off_t nwritten;
+    if (mem->swapout.sio == NULL)
+	return 0;
+    nwritten = storeOffset(mem->swapout.sio);
+    if (nwritten <= (off_t)mem->swap_hdr_sz)
+	return 0;
+    return nwritten - mem->swap_hdr_sz;
+}
+
+/*
+ * Is this entry a candidate for writing to disk?
+ */
+int
+storeSwapOutAble(const StoreEntry * e)
+{
+    dlink_node *node;
+    if (e->mem_obj->swapout.sio != NULL)
+	return 1;
+    if (e->mem_obj->inmem_lo > 0)
+	return 0;
+    /*
+     * If there are DISK clients, we must write to disk
+     * even if its not cachable
+     */
+    for (node = e->mem_obj->clients.head; node; node = node->next) {
+	if (((store_client *) node->data)->type == STORE_DISK_CLIENT)
+	    return 1;
+    }
+    /* Don't pollute the disk with icons and other special entries */
+    if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
+	return 0;
+    return EBIT_TEST(e->flags, ENTRY_CACHABLE);
+}
Index: squid/src/structs.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/structs.h,v
retrieving revision 1.70.2.8
retrieving revision 1.70.2.9
diff -u -r1.70.2.8 -r1.70.2.9
--- squid/src/structs.h	9 Oct 2002 14:14:30 -0000	1.70.2.8
+++ squid/src/structs.h	11 Oct 2002 15:41:06 -0000	1.70.2.9
@@ -1,6 +1,6 @@
 
 /*
- * $Id: structs.h,v 1.70.2.8 2002/10/09 14:14:30 rbcollins Exp $
+ * $Id: structs.h,v 1.70.2.9 2002/10/11 15:41:06 rbcollins Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -658,6 +658,14 @@
     int weak;			/* true if it is a weak validator */
 };
 
+struct _fde_disk {
+    DWCB *wrt_handle;
+    void *wrt_handle_data;
+    dwrite_q *write_q;
+    dwrite_q *write_q_tail;
+    off_t offset;
+};
+
 struct _fde {
     unsigned int type;
     u_short local_port;
@@ -683,13 +691,7 @@
     int bytes_read;
     int bytes_written;
     int uses;			/* ie # req's over persistent conn */
-    struct _fde_disk {
-	DWCB *wrt_handle;
-	void *wrt_handle_data;
-	dwrite_q *write_q;
-	dwrite_q *write_q_tail;
-	off_t offset;
-    } disk;
+    struct _fde_disk disk;
     PF *read_handler;
     void *read_data;
     PF *write_handler;