--------------------- PatchSet 10170 Date: 2007/11/27 08:12:17 Author: adri Branch: store_copy Tag: (none) Log: Merge changes from Squid-2.HEAD. Members: ChangeLog:1.93->1.93.2.1 configure.in:1.184->1.184.2.1 doc/release-notes/release-2.6.html:1.47->1.47.2.1 doc/release-notes/release-2.6.sgml:1.59->1.59.2.1 helpers/basic_auth/DB/.cvsignore:1.2->1.2.2.1 helpers/basic_auth/DB/squid_db_auth.in:1.2->1.2.4.1 helpers/basic_auth/squid_radius_auth/ChangeLog:1.1->1.1.4.1(DEAD) helpers/basic_auth/squid_radius_auth/Makefile.am:1.1->1.1.4.1 helpers/basic_auth/squid_radius_auth/md5.c:1.1->1.1.4.1(DEAD) helpers/basic_auth/squid_radius_auth/md5.h:1.1->1.1.4.1(DEAD) helpers/basic_auth/squid_radius_auth/squid_rad_auth.c:1.1->1.1.4.1 helpers/basic_auth/squid_radius_auth/util.c:1.1->1.1.4.1 helpers/basic_auth/squid_radius_auth/util.h:1.1->1.1.4.1 include/Array.h:1.7->1.7.52.1 include/md5.h:1.12->1.12.34.1 lib/Array.c:1.6->1.6.52.1 lib/Makefile.am:1.10->1.10.18.1 lib/md5.c:1.12->1.12.30.1 src/HttpHeader.c:1.28->1.28.6.1 src/HttpReply.c:1.23->1.23.6.1 src/HttpRequest.c:1.17->1.17.10.1 src/Makefile.am:1.54->1.54.2.1 src/access_log.c:1.43->1.43.2.1 src/cache_cf.c:1.102->1.102.2.1 src/cf.data.depend:1.1->1.1.4.1 src/cf.data.pre:1.209->1.209.2.1 src/client_side.c:1.202.2.6->1.202.2.7 src/client_side_rewrite.c:1.2.4.1->1.2.4.2 src/client_side_storeurl_rewrite.c:1.3.2.1->1.3.2.2 src/enums.h:1.62->1.62.2.1 src/errorpage.c:1.37->1.37.6.1 src/fqdncache.c:1.22->1.22.2.1 src/helper.c:1.43->1.43.6.1 src/http.c:1.63->1.63.2.1 src/ipcache.c:1.25->1.25.2.1 src/main.c:1.80->1.80.2.1 src/mib.txt:1.9->1.9.4.1 src/protos.h:1.146.2.3->1.146.2.4 src/redirect.c:1.16->1.16.10.1 src/refresh.c:1.15->1.15.6.1 src/refresh_check.c:1.2.4.1->1.2.4.2 src/stat.c:1.42.2.2->1.42.2.3 src/store.c:1.49->1.49.4.1 src/store_client.c:1.25.10.4->1.25.10.5 src/store_key_md5.c:1.10->1.10.10.1 src/store_rewrite.c:1.1.4.1->1.1.4.2 src/store_swapmeta.c:1.12->1.12.22.1 src/store_update.c:1.2.4.1->1.2.4.2 src/structs.h:1.158.2.4->1.158.2.5 src/typedefs.h:1.43.2.2->1.43.2.3 src/fs/aufs/aiops.c:1.28->1.28.18.1 src/fs/aufs/aiops_win32.c:1.4->1.4.28.1 src/fs/aufs/store_dir_aufs.c:1.50->1.50.6.1 Index: squid/ChangeLog =================================================================== RCS file: /cvsroot/squid-sf//squid/ChangeLog,v retrieving revision 1.93 retrieving revision 1.93.2.1 diff -u -r1.93 -r1.93.2.1 --- squid/ChangeLog 23 Sep 2007 16:52:56 -0000 1.93 +++ squid/ChangeLog 27 Nov 2007 08:12:17 -0000 1.93.2.1 @@ -39,6 +39,45 @@ - Implementation of an external logfile daemon (ie, logging to a pipe); UDP logging (based in part on work done by Tim Starling) +Changes to squid-2.6.STABLE17 (26 Nov 2007) + + - Fix compile error with old GCC 2.x or other ANSI-C compilers before + C99 + - Mention the login= cache_peer option in release notes + - Fix bad cache_peer example in squid.conf + - Bug #2086: Fix a compile-time memory corruption error causing cf_gen + to fail + - Bug #2048: Clarify high_memory_warning usage + - Reject DNS responses which result in no data + - Fix version number in configuration manual + - Move cache and request/reply_header_max_size to their proper + sections + - Bug #2088: sbrk statistics broken when process size >2GB + - Move logopen() much earlier to have fatal startup errors sent to the + proper syslog facility + - Fix HTTP/0.9 responses + - Correct bad example config for tos_outgoing_tos + - Fix grammar in description of mail_program squid.conf option + - Ignore Content-Length in chunked responses instead of rejecting the + response as invalid + - Documented that http_port no longer have a default + - Cleanup of cache digest documentation + - Make aufs store rebuilding back off a little if I/O load too high + - Bug #2100: Respect DNS ttl=0 + - Update udp_(incoming|outgoing)_address documentation to reflect + current bahaviour. + - Update HTCP documentation + - Document the overlapping helper request format + - Change priority of proxy auth and extacl provided username in + login=*:pass + - pack header entries on cache updates + - Make squid_db_auth reopen the database connection on each query by + default + - Improve helper debug ouput, including the channel number + - Update cachePeerEntry MIB description to mention what is used as + index key + - Import squid_radius_auth for authenticating to RADIUS + Changes to squid-2.6.STABLE16 (5 Sep 2007) - Test for sys/capability.h linux include file to avoid failing on Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.184 retrieving revision 1.184.2.1 diff -u -r1.184 -r1.184.2.1 --- squid/configure.in 5 Sep 2007 22:52:14 -0000 1.184 +++ squid/configure.in 27 Nov 2007 08:12:17 -0000 1.184.2.1 @@ -1,7 +1,7 @@ dnl dnl Configuration input file for Squid dnl -dnl $Id: configure.in,v 1.184 2007/09/05 22:52:14 squidadm Exp $ +dnl $Id: configure.in,v 1.184.2.1 2007/11/27 08:12:17 adri Exp $ dnl dnl dnl @@ -10,7 +10,7 @@ AM_CONFIG_HEADER(include/autoconf.h) AC_CONFIG_AUX_DIR(cfgaux) AM_INIT_AUTOMAKE -AC_REVISION($Revision: 1.184 $)dnl +AC_REVISION($Revision: 1.184.2.1 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AM_MAINTAINER_MODE @@ -686,11 +686,13 @@ fi ]) +dnl SSL is not enabled by default. AM_CONDITIONAL(ENABLE_SSL, false) SSLLIB='' SSLFLAGS='' +dnl Default is to use OpenSSL when available AC_ARG_ENABLE(ssl, [ --enable-ssl Enable ssl gatewaying support using OpenSSL], [ if test "$enableval" != "no"; then @@ -709,8 +711,7 @@ fi ]) -AM_CONDITIONAL(NEED_OWN_MD5, true) - +dnl User may specify OpenSSL is needed from a non-standard location AC_ARG_WITH(openssl, [ --with-openssl[=prefix] Compile with the OpenSSL libraries. The path to @@ -739,7 +740,6 @@ AC_DEFINE(USE_OPENSSL, 1, [Define this to make use of the OpenSSL libraries for MD5 calculation rather than Squid's own MD5 implementation or if building with SSL encryption (USE_SSL)]) - AM_CONDITIONAL(NEED_OWN_MD5, false) if test -z "$SSLLIB"; then SSLLIB="-lcrypto" # for MD5 routines fi @@ -751,6 +751,7 @@ fi AC_SUBST(SSLLIB) + AC_ARG_ENABLE(forw-via-db, [ --enable-forw-via-db Enable Forw/Via database], [ if test "$enableval" = "yes" ; then @@ -1635,6 +1636,7 @@ sys/file.h \ sys/ioctl.h \ sys/mount.h \ + sys/md5.h \ sys/msg.h \ sys/prctl.h \ sys/resource.h \ @@ -2103,7 +2105,11 @@ dnl Check for libcrypt dnl Some of our helpers use crypt(3) which may be in libc, or in dnl libcrypt (eg FreeBSD) -AC_CHECK_LIB(crypt, crypt, [CRYPTLIB="-lcrypt"]) +AC_CHECK_LIB(crypt, crypt, [CRYPTLIB=" -lcrypt "]) + +dnl Solaris10 provides MD5 natively through libmd5 +AC_CHECK_LIB(md5, MD5Init, [CRYPTLIB+=" -lmd5 "]) + AC_SUBST(CRYPTLIB) dnl Check for libdl, used by auth_modules/PAM Index: squid/doc/release-notes/release-2.6.html =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/release-notes/release-2.6.html,v retrieving revision 1.47 retrieving revision 1.47.2.1 diff -u -r1.47 -r1.47.2.1 --- squid/doc/release-notes/release-2.6.html 5 Sep 2007 22:52:15 -0000 1.47 +++ squid/doc/release-notes/release-2.6.html 27 Nov 2007 08:12:19 -0000 1.47.2.1 @@ -2,12 +2,12 @@ - Squid 2.6.STABLE16 release notes + Squid 2.6.STABLE17 release notes -

Squid 2.6.STABLE16 release notes

+

Squid 2.6.STABLE17 release notes

-

Squid Developers

$Id: release-2.6.html,v 1.47 2007/09/05 22:52:15 squidadm Exp $ +

Squid Developers

$Id: release-2.6.html,v 1.47.2.1 2007/11/27 08:12:19 adri Exp $
This document contains the release notes for version 2.6 of Squid. Squid is a WWW Cache application developed by the Web Caching community. @@ -75,6 +75,9 @@

21. Key changes squid-2.6.STABLE15 to 2.6.STABLE16

+

+

22. Key changes squid-2.6.STABLE16 to 2.6.STABLE17

+

1. Key changes from squid 2.5

@@ -153,7 +156,7 @@

-
http_port

Now takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual.

+
http_port

Now takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual. Also there is no longer a default port and you must include a http_port directive in your squid.conf if you want Squid to listen for http requests.

httpd_accel_* for transparent proxy

Now implemented by the "transparent" http_port option

httpd_accel_* for accelerator mode

Now implemented by other options and the http_port accel option. See individual directives below.

httpd_accel_host

Replaced by defaultsite http_port option and cache_peer originserver option.

@@ -207,6 +210,7 @@
check_hostnames

Control if Squid should check the sanity of host names before trying to look them up in DNS

allow_underscores

Control if _ is to be considered a valid character in hostnames or not

cache_peer carp-load-factor

Option removed. CARP now uses the weight parameter instead.

+
cache_peer login=

Controls how Squid forwards login information to peer caches and servers.

@@ -743,5 +747,20 @@

+

22. Key changes squid-2.6.STABLE16 to 2.6.STABLE17

+ +

+

+

+ Index: squid/doc/release-notes/release-2.6.sgml =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/release-notes/release-2.6.sgml,v retrieving revision 1.59 retrieving revision 1.59.2.1 diff -u -r1.59 -r1.59.2.1 --- squid/doc/release-notes/release-2.6.sgml 9 Sep 2007 09:53:06 -0000 1.59 +++ squid/doc/release-notes/release-2.6.sgml 27 Nov 2007 08:12:19 -0000 1.59.2.1 @@ -1,8 +1,8 @@
-Squid 2.6.STABLE16 release notes +Squid 2.6.STABLE17 release notes Squid Developers -$Id: release-2.6.sgml,v 1.59 2007/09/09 09:53:06 squidadm Exp $ +$Id: release-2.6.sgml,v 1.59.2.1 2007/11/27 08:12:19 adri Exp $ This document contains the release notes for version 2.6 of Squid. @@ -82,7 +82,7 @@ Changes to squid.conf

-http_portNow takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual. +http_portNow takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual. Also there is no longer a default port and you must include a http_port directive in your squid.conf if you want Squid to listen for http requests. httpd_accel_* for transparent proxyNow implemented by the "transparent" http_port option httpd_accel_* for accelerator modeNow implemented by other options and the http_port accel option. See individual directives below. httpd_accel_hostReplaced by defaultsite http_port option and cache_peer originserver option. @@ -638,5 +638,20 @@ name="ChangeLog"> file for details. +Key changes squid-2.6.STABLE16 to 2.6.STABLE17 + +

+ + A temporary memory leak fixed on cache updates + New squid_radius_auth RADIUS authentication helper + Documentation cleanups + Code cleanups and portability fixes + Several minor bugfixes + See also the list of and the file for details. + +

Index: squid/helpers/basic_auth/DB/.cvsignore =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/DB/.cvsignore,v retrieving revision 1.2 retrieving revision 1.2.2.1 diff -u -r1.2 -r1.2.2.1 --- squid/helpers/basic_auth/DB/.cvsignore 3 Sep 2007 04:51:58 -0000 1.2 +++ squid/helpers/basic_auth/DB/.cvsignore 27 Nov 2007 08:12:20 -0000 1.2.2.1 @@ -2,4 +2,4 @@ Makefile Makefile.in squid_db_auth -squid_db_auth.8Makefile.in +squid_db_auth.8 Index: squid/helpers/basic_auth/DB/squid_db_auth.in =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/DB/squid_db_auth.in,v retrieving revision 1.2 retrieving revision 1.2.4.1 diff -u -r1.2 -r1.2.4.1 --- squid/helpers/basic_auth/DB/squid_db_auth.in 22 Aug 2007 06:52:10 -0000 1.2 +++ squid/helpers/basic_auth/DB/squid_db_auth.in 27 Nov 2007 08:12:20 -0000 1.2.4.1 @@ -21,6 +21,7 @@ my $db_passwdcol = "password"; my $db_cond = "enabled = 1"; my $plaintext = 0; +my $persist = 0; =pod @@ -66,6 +67,10 @@ Database contains plain-text passwords +=item B<--persist> + +Keep a persistent database connection open between queries. + =back =cut @@ -79,13 +84,30 @@ 'passwdcol=s' => \$db_passwdcol, 'cond=s' => \$db_cond, 'plaintext' => \$plaintext, + 'persist' => \$persist, ); -my $dbh = DBI->connect($dsn, $db_user, $db_passwd) || die ("Could not connect to $dsn\n"); +my ($_dbh, $_sth); -my ($sth) = $dbh->prepare("SELECT $db_passwdcol FROM $db_table WHERE $db_usercol = ?" . ($db_cond ne "" ? " AND $db_cond" : "")) || die; +sub close_db() +{ + return if !defined($_dbh); + $_dbh->disconnect(); + undef $_dbh; + undef $_sth; +} -my $status; +sub open_db() +{ + return $_sth if defined $_sth; + $_dbh = DBI->connect($dsn, $db_user, $db_passwd); + if (!defined $_dbh) { + warn ("Could not connect to $dsn\n"); + return undef; + } + $_sth = $_dbh->prepare("SELECT $db_passwdcol FROM $db_table WHERE $db_usercol = ?" . ($db_cond ne "" ? " AND $db_cond" : "")) || die; + return $_sth; +} sub check_password($$) { @@ -97,20 +119,34 @@ return 0; } + +sub query_db($) { + my ($user) = @_; + my ($sth) = open_db() || return undef; + if (!$sth->execute($user)) { + close_db(); + open_db() || return undef; + $sth->execute($user) || return undef;; + } + return $sth; +} +my $status; + while (<>) { my ($user, $password) = split; $status = "ERR"; $user =~ s/%(..)/pack("H*", $1)/ge; $password =~ s/%(..)/pack("H*", $1)/ge; - $status = "ERR internal error"; - $sth->execute($user) || next; + $status = "ERR database error"; + my $sth = query_db($user) || next; $status = "ERR unknown login"; - my ($row) = $sth->fetchrow_arrayref() || next; + my $row = $sth->fetchrow_arrayref() || next; $status = "ERR login failure"; next if (!check_password($password, @$row[0])); $status = "OK"; } continue { + close_db() if (!$persist); print $status . "\n"; } --- squid/helpers/basic_auth/squid_radius_auth/ChangeLog Wed Nov 28 01:20:24 2007 +++ /dev/null Wed Nov 28 01:20:24 2007 @@ -1,47 +0,0 @@ -2007-08-27 02:10 hno - - * Makefile.in: Bootstrapped - -2007-08-24 14:31 hno - - * Makefile.in: Bootstrapped - -2007-08-24 04:36 amosjeffries - - * Makefile.in: Bootstrapped. - -2007-08-18 02:11 hno - - * Makefile.in: Bootstrapped - -2007-08-17 20:58 serassio - - * Makefile.in: Bootstrapped - -2007-08-15 02:11 hno - - * Makefile.in: Bootstrapped - -2007-08-14 19:07 serassio - - * Makefile.in: Bootstrapped - -2007-08-13 10:28 serassio - - * .cvsignore: Added/updated .cvsignore files - -2007-06-25 15:57 hno - - * Makefile.in: Bootstrapped - -2007-06-25 13:02 hno - - * COPYRIGHT, Changelog, Make.inc, Makefile.am, Makefile.default, - Makefile.in, Makefile.solaris, README, md5.c, md5.h, radius.h, - squid_rad_auth.c, squid_radius_auth.8, util.c, util.h: Import - squid_radius_auth helper for basic authentication to RADIUS - -2004-08-07 20:09 hno - - * etc/squid_radius_auth.conf: Import of version 1.07 - Index: squid/helpers/basic_auth/squid_radius_auth/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/Makefile.am,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/Makefile.am 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/Makefile.am 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -1,7 +1,7 @@ # # Makefile for the Squid LDAP authentication helper # -# $Id: Makefile.am,v 1.1 2007/09/05 22:52:16 squidadm Exp $ +# $Id: Makefile.am,v 1.1.4.1 2007/11/27 08:12:20 adri Exp $ # # Uncomment and customize the following to suit your needs: # @@ -11,8 +11,8 @@ EXTRA_DIST = squid_radius_auth.8 squid_radius_auth_SOURCES = \ squid_rad_auth.c \ - md5.c md5.h \ radius.h \ util.c util.h -LDADD = $(XTRA_LIBS) +LDADD = -L$(top_builddir)/lib -lmiscutil $(SSLLIB) $(XTRA_LIBS) +INCLUDES = -I$(top_srcdir)/include --- squid/helpers/basic_auth/squid_radius_auth/md5.c Wed Nov 28 01:20:24 2007 +++ /dev/null Wed Nov 28 01:20:24 2007 @@ -1,333 +0,0 @@ -/* MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm - */ - -/* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All -rights reserved. - -License to copy and use this software is granted provided that it -is identified as the "RSA Data Security, Inc. MD5 Message-Digest -Algorithm" in all material mentioning or referencing this software -or this function. - -License is also granted to make and use derivative works provided -that such works are identified as "derived from the RSA Data -Security, Inc. MD5 Message-Digest Algorithm" in all material -mentioning or referencing the derived work. - -RSA Data Security, Inc. makes no representations concerning either -the merchantability of this software or the suitability of this -software for any particular purpose. It is provided "as is" -without express or implied warranty of any kind. - -These notices must be retained in any copies of any part of this -documentation and/or software. - */ - -#include "md5.h" - -/* Constants for MD5Transform routine. - */ -#define S11 7 -#define S12 12 -#define S13 17 -#define S14 22 -#define S21 5 -#define S22 9 -#define S23 14 -#define S24 20 -#define S31 4 -#define S32 11 -#define S33 16 -#define S34 23 -#define S41 6 -#define S42 10 -#define S43 15 -#define S44 21 - -static void MD5Transform PROTO_LIST ((UINT4 [4], unsigned char [64])); -static void Encode PROTO_LIST - ((unsigned char *, UINT4 *, unsigned int)); -static void Decode PROTO_LIST - ((UINT4 *, unsigned char *, unsigned int)); -static void MD5_memcpy PROTO_LIST ((POINTER, POINTER, unsigned int)); -static void MD5_memset PROTO_LIST ((POINTER, int, unsigned int)); - -static unsigned char PADDING[64] = { - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; - -/* F, G, H and I are basic MD5 functions. - */ -#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) -#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) -#define H(x, y, z) ((x) ^ (y) ^ (z)) -#define I(x, y, z) ((y) ^ ((x) | (~z))) - -/* ROTATE_LEFT rotates x left n bits. - */ -#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n)))) - -/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. -Rotation is separate from addition to prevent recomputation. - */ -#define FF(a, b, c, d, x, s, ac) { \ - (a) += F ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } -#define GG(a, b, c, d, x, s, ac) { \ - (a) += G ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } -#define HH(a, b, c, d, x, s, ac) { \ - (a) += H ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } -#define II(a, b, c, d, x, s, ac) { \ - (a) += I ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } - -/* MD5 initialization. Begins an MD5 operation, writing a new context. - */ -void MD5Init (context) -MD5_CTX *context; /* context */ -{ - context->count[0] = context->count[1] = 0; - /* Load magic initialization constants. -*/ - context->state[0] = 0x67452301; - context->state[1] = 0xefcdab89; - context->state[2] = 0x98badcfe; - context->state[3] = 0x10325476; -} - -/* MD5 block update operation. Continues an MD5 message-digest - operation, processing another message block, and updating the - context. - */ -void MD5Update (context, input, inputLen) -MD5_CTX *context; /* context */ -unsigned char *input; /* input block */ -unsigned int inputLen; /* length of input block */ -{ - unsigned int i, index, partLen; - - /* Compute number of bytes mod 64 */ - index = (unsigned int)((context->count[0] >> 3) & 0x3F); - - /* Update number of bits */ - if ((context->count[0] += ((UINT4)inputLen << 3)) - < ((UINT4)inputLen << 3)) - context->count[1]++; - context->count[1] += ((UINT4)inputLen >> 29); - - partLen = 64 - index; - - /* Transform as many times as possible. -*/ - if (inputLen >= partLen) { - MD5_memcpy - ((POINTER)&context->buffer[index], (POINTER)input, partLen); - MD5Transform (context->state, context->buffer); - - for (i = partLen; i + 63 < inputLen; i += 64) - MD5Transform (context->state, &input[i]); - - index = 0; - } - else - i = 0; - - /* Buffer remaining input */ - MD5_memcpy - ((POINTER)&context->buffer[index], (POINTER)&input[i], - inputLen-i); -} - -/* MD5 finalization. Ends an MD5 message-digest operation, writing the - the message digest and zeroizing the context. - */ -void MD5Final (digest, context) -unsigned char digest[16]; /* message digest */ -MD5_CTX *context; /* context */ -{ - unsigned char bits[8]; - unsigned int index, padLen; - - /* Save number of bits */ - Encode (bits, context->count, 8); - - /* Pad out to 56 mod 64. -*/ - index = (unsigned int)((context->count[0] >> 3) & 0x3f); - padLen = (index < 56) ? (56 - index) : (120 - index); - MD5Update (context, PADDING, padLen); - - /* Append length (before padding) */ - MD5Update (context, bits, 8); - - /* Store state in digest */ - Encode (digest, context->state, 16); - - /* Zeroize sensitive information. -*/ - MD5_memset ((POINTER)context, 0, sizeof (*context)); -} - -/* MD5 basic transformation. Transforms state based on block. - */ -static void MD5Transform (state, block) -UINT4 state[4]; -unsigned char block[64]; -{ - UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; - - Decode (x, block, 64); - - /* Round 1 */ - FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */ - FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */ - FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */ - FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */ - FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */ - FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */ - FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */ - FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */ - FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */ - FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */ - FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ - FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ - FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ - FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ - FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ - FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ - - /* Round 2 */ - GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */ - GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */ - GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ - GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */ - GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */ - GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */ - GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ - GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */ - GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */ - GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ - GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */ - GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */ - GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ - GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */ - GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */ - GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ - - /* Round 3 */ - HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */ - HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */ - HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ - HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ - HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */ - HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */ - HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */ - HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ - HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ - HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */ - HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */ - HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */ - HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */ - HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ - HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ - HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */ - - /* Round 4 */ - II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */ - II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */ - II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ - II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */ - II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ - II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */ - II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ - II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */ - II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */ - II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ - II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */ - II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ - II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */ - II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ - II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */ - II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */ - - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - - /* Zeroize sensitive information. -*/ - MD5_memset ((POINTER)x, 0, sizeof (x)); -} - -/* Encodes input (UINT4) into output (unsigned char). Assumes len is - a multiple of 4. - */ -static void Encode (output, input, len) -unsigned char *output; -UINT4 *input; -unsigned int len; -{ - unsigned int i, j; - - for (i = 0, j = 0; j < len; i++, j += 4) { - output[j] = (unsigned char)(input[i] & 0xff); - output[j+1] = (unsigned char)((input[i] >> 8) & 0xff); - output[j+2] = (unsigned char)((input[i] >> 16) & 0xff); - output[j+3] = (unsigned char)((input[i] >> 24) & 0xff); - } -} - -/* Decodes input (unsigned char) into output (UINT4). Assumes len is - a multiple of 4. - */ -static void Decode (output, input, len) -UINT4 *output; -unsigned char *input; -unsigned int len; -{ - unsigned int i, j; - - for (i = 0, j = 0; j < len; i++, j += 4) - output[i] = ((UINT4)input[j]) | (((UINT4)input[j+1]) << 8) | - (((UINT4)input[j+2]) << 16) | (((UINT4)input[j+3]) << 24); -} - -/* Note: Replace "for loop" with standard memcpy if possible. - */ - -static void MD5_memcpy (output, input, len) -POINTER output; -POINTER input; -unsigned int len; -{ - unsigned int i; - - for (i = 0; i < len; i++) - output[i] = input[i]; -} - -/* Note: Replace "for loop" with standard memset if possible. - */ -static void MD5_memset (output, value, len) -POINTER output; -int value; -unsigned int len; -{ - unsigned int i; - - for (i = 0; i < len; i++) - ((char *)output)[i] = (char)value; -} --- squid/helpers/basic_auth/squid_radius_auth/md5.h Wed Nov 28 01:20:24 2007 +++ /dev/null Wed Nov 28 01:20:24 2007 @@ -1,77 +0,0 @@ -/* GLOBAL.H - RSAREF types and constants - */ - -/* PROTOTYPES should be set to one if and only if the compiler supports - function argument prototyping. - The following makes PROTOTYPES default to 0 if it has not already - been defined with C compiler flags. - */ -#ifndef PROTOTYPES -# if __STDC__ -# define PROTOTYPES 1 -# else -# define PROTOTYPES 0 -# endif -#endif - -/* POINTER defines a generic pointer type */ -typedef unsigned char *POINTER; - -/* UINT2 defines a two byte word */ -typedef unsigned short int UINT2; - -/* UINT4 defines a four byte word */ -#if defined(__alpha) && (defined(__osf__) || defined(__linux__)) -typedef unsigned int UINT4; -#else -typedef unsigned long int UINT4; -#endif - -/* PROTO_LIST is defined depending on how PROTOTYPES is defined above. - If using PROTOTYPES, then PROTO_LIST returns the list, otherwise it - returns an empty list. - */ -#if PROTOTYPES -#define PROTO_LIST(list) list -#else -#define PROTO_LIST(list) () -#endif - -/* MD5.H - header file for MD5C.C - */ - -/* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All -rights reserved. - -License to copy and use this software is granted provided that it -is identified as the "RSA Data Security, Inc. MD5 Message-Digest -Algorithm" in all material mentioning or referencing this software -or this function. - -License is also granted to make and use derivative works provided -that such works are identified as "derived from the RSA Data -Security, Inc. MD5 Message-Digest Algorithm" in all material -mentioning or referencing the derived work. - -RSA Data Security, Inc. makes no representations concerning either -the merchantability of this software or the suitability of this -software for any particular purpose. It is provided "as is" -without express or implied warranty of any kind. - -These notices must be retained in any copies of any part of this -documentation and/or software. - */ - -/* MD5 context. */ -typedef struct { - UINT4 state[4]; /* state (ABCD) */ - UINT4 count[2]; /* number of bits, modulo 2^64 (lsb first) */ - unsigned char buffer[64]; /* input buffer */ -} MD5_CTX; - -void MD5Init PROTO_LIST ((MD5_CTX *)); -void MD5Update PROTO_LIST - ((MD5_CTX *, unsigned char *, unsigned int)); -void MD5Final PROTO_LIST ((unsigned char [16], MD5_CTX *)); -void md5_calc PROTO_LIST ((unsigned char *output, unsigned char *input, unsigned int inlen)); - Index: squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -45,21 +45,51 @@ * and many others */ +#include "config.h" + +#if HAVE_SYS_TYPES_H #include +#endif +#if HAVE_SYS_SOCKET_H #include +#endif +#if HAVE_NETINET_IN_H #include +#endif +#if HAVE_SYS_TIME_H #include +#endif +#if HAVE_UNISTD_H #include +#endif +#if HAVE_FCNTL_H #include +#endif +#if HAVE_CTYPE_H #include +#endif +#if HAVE_STDIO_H #include +#endif +#if HAVE_UNISTD_H #include +#endif +#if HAVE_NETDB_H #include +#endif +#if HAVE_PWD_H #include +#endif +#if HAVE_STDLIB_H #include +#endif +#if HAVE_TIME_H #include +#endif +#if HAVE_STRING_H #include +#endif #include "md5.h" #include "radius.h" @@ -83,8 +113,8 @@ static char svc_name[MAXLINE] = "radius"; static int nasport = 111; static int nasporttype = 0; -static UINT4 nas_ipaddr; -static UINT4 auth_ipaddr; +static u_int32_t nas_ipaddr; +static u_int32_t auth_ipaddr; static int retries = 30; char *progname = "squid_rad_auth"; @@ -111,10 +141,22 @@ } /* + * MD5 digest + */ +static void +md5_calc(uint8_t out[16], void *in, size_t len) +{ + MD5_CTX ctx; + MD5Init(&ctx); + MD5Update(&ctx, in, len); + MD5Final(out, &ctx); +} + +/* * Receive and verify the result. */ static int -result_recv(UINT4 host, u_short udp_port, char *buffer, int length) +result_recv(u_int32_t host, u_short udp_port, char *buffer, int length) { AUTH_HDR *auth; int totallen; @@ -241,7 +283,7 @@ int secretlen; u_char cbc[AUTH_VECTOR_LEN]; int i, j; - UINT4 ui; + u_int32_t ui; struct sockaddr_in saremote; fd_set readfds; socklen_t salen; Index: squid/helpers/basic_auth/squid_radius_auth/util.c =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/util.c,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/util.c 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/util.c 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -35,18 +35,42 @@ "@(#)util.c 1.5 Copyright 1992 Livingston Enterprises Inc\n" " 2.1 Copyright 1997 Cistron Internet Services B.V."; +#include "config.h" + +#if HAVE_SYS_TYES_H #include +#endif +#if HAVE_SYS_SOCKET_H #include +#endif +#if HAVE_SYS_TIME_H #include +#endif +#if HAVE_NETINET_IN_H #include +#endif +#if HAVE_STDIO_H #include +#endif +#if HAVE_STDLIB_H #include +#endif +#if HAVE_NETDB_H #include +#endif +#if HAVE_PWD_H #include +#endif +#if HAVE_TIME_H #include +#endif +#if HAVE_CTYPE_H #include +#endif +#if HAVE_SIGNAL_H #include +#endif #include "md5.h" #include "util.h" @@ -89,37 +113,37 @@ * Return an IP address in host long notation from * one supplied in standard dot notation. */ -static UINT4 ipstr2long(char *ip_str) +static u_int32_t ipstr2long(char *ip_str) { char buf[6]; char *ptr; int i; int count; - UINT4 ipaddr; + u_int32_t ipaddr; int cur_byte; - ipaddr = (UINT4)0; + ipaddr = (u_int32_t)0; for(i = 0;i < 4;i++) { ptr = buf; count = 0; *ptr = '\0'; while(*ip_str != '.' && *ip_str != '\0' && count < 4) { if(!isdigit(*ip_str)) { - return((UINT4)0); + return((u_int32_t)0); } *ptr++ = *ip_str++; count++; } if(count >= 4 || count == 0) { - return((UINT4)0); + return((u_int32_t)0); } *ptr = '\0'; cur_byte = atoi(buf); if(cur_byte < 0 || cur_byte > 255) { - return((UINT4)0); + return((u_int32_t)0); } ip_str++; - ipaddr = ipaddr << 8 | (UINT4)cur_byte; + ipaddr = ipaddr << 8 | (u_int32_t)cur_byte; } return(ipaddr); } @@ -128,7 +152,7 @@ * Return an IP address in host long notation from a host * name or address in dot notation. */ -UINT4 get_ipaddr(char *host) +u_int32_t get_ipaddr(char *host) { struct hostent *hp; @@ -136,9 +160,9 @@ return(ipstr2long(host)); } else if((hp = gethostbyname(host)) == (struct hostent *)NULL) { - return((UINT4)0); + return((u_int32_t)0); } - return(ntohl(*(UINT4 *)hp->h_addr)); + return(ntohl(*(u_int32_t *)hp->h_addr)); } Index: squid/helpers/basic_auth/squid_radius_auth/util.h =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/util.h,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/util.h 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/util.h 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -1,2 +1,2 @@ /* util.c */ -UINT4 get_ipaddr (char *); +u_int32_t get_ipaddr (char *); Index: squid/include/Array.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/Array.h,v retrieving revision 1.7 retrieving revision 1.7.52.1 diff -u -r1.7 -r1.7.52.1 --- squid/include/Array.h 28 Apr 2006 11:10:49 -0000 1.7 +++ squid/include/Array.h 27 Nov 2007 08:12:20 -0000 1.7.52.1 @@ -1,5 +1,5 @@ /* - * $Id: Array.h,v 1.7 2006/04/28 11:10:49 squidadm Exp $ + * $Id: Array.h,v 1.7.52.1 2007/11/27 08:12:20 adri Exp $ * * AUTHOR: Alex Rousskov * @@ -50,6 +50,8 @@ extern void arrayAppend(Array * s, void *obj); extern void arrayInsert(Array * s, void *obj, int position); extern void arrayPreAppend(Array * s, int app_count); +extern void arrayShrink(Array *a, int new_count); + #endif /* SQUID_ARRAY_H */ Index: squid/include/md5.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/md5.h,v retrieving revision 1.12 retrieving revision 1.12.34.1 diff -u -r1.12 -r1.12.34.1 --- squid/include/md5.h 29 May 2006 00:14:40 -0000 1.12 +++ squid/include/md5.h 27 Nov 2007 08:12:20 -0000 1.12.34.1 @@ -1,18 +1,18 @@ #ifndef SQUID_MD5_H #define SQUID_MD5_H -#if USE_OPENSSL +#if HAVE_SYS_TYPES_H +#include +#endif /* * If Squid is compiled with OpenSSL then we use the MD5 routines * from there via some wrapper macros, and the rest of this file is ignored.. */ +#define USE_SQUID_MD5 0 -#if HAVE_OPENSSL_MD5_H +#if USE_OPENSSL && HAVE_OPENSSL_MD5_H #include -#else -#error Cannot find OpenSSL headers -#endif /* Hack to adopt Squid to the OpenSSL syntax */ #define MD5_DIGEST_CHARS MD5_DIGEST_LENGTH @@ -21,7 +21,28 @@ #define MD5Update MD5_Update #define MD5Final MD5_Final -#else /* USE_OPENSSL */ +#elif USE_OPENSSL && !HAVE_OPENSSL_MD5_H +#error Cannot find OpenSSL MD5 headers + +#elif HAVE_SYS_MD5_H +/* + * Solaris 10 provides MD5 as part of the system. + */ +#include + +/* + * They also define MD5_CTX with different field names + * fortunately we do not access it directly in the squid code. + */ + +/* Hack to adopt Squid to the OpenSSL syntax */ +#define MD5_DIGEST_CHARS MD5_DIGEST_LENGTH + +#else /* NEED_OWN_MD5 */ + + /* Turn on internal MD5 code */ +#undef USE_SQUID_MD5 +#define USE_SQUID_MD5 1 /* * This is the header file for the MD5 message-digest algorithm. Index: squid/lib/Array.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/Array.c,v retrieving revision 1.6 retrieving revision 1.6.52.1 diff -u -r1.6 -r1.6.52.1 --- squid/lib/Array.c 28 Apr 2006 11:10:49 -0000 1.6 +++ squid/lib/Array.c 27 Nov 2007 08:12:20 -0000 1.6.52.1 @@ -1,5 +1,5 @@ /* - * $Id: Array.c,v 1.6 2006/04/28 11:10:49 squidadm Exp $ + * $Id: Array.c,v 1.6.52.1 2007/11/27 08:12:20 adri Exp $ * * AUTHOR: Alex Rousskov * @@ -138,3 +138,11 @@ /* reset, just in case */ memset(a->items + a->count, 0, (a->capacity - a->count) * sizeof(void *)); } + +void +arrayShrink(Array *a, int new_count) +{ + assert(new_count < a->capacity); + assert(new_count >= 0); + a->count = new_count; +} Index: squid/lib/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/Makefile.am,v retrieving revision 1.10 retrieving revision 1.10.18.1 diff -u -r1.10 -r1.10.18.1 --- squid/lib/Makefile.am 8 Sep 2006 19:50:55 -0000 1.10 +++ squid/lib/Makefile.am 27 Nov 2007 08:12:20 -0000 1.10.18.1 @@ -13,11 +13,6 @@ else STRSEPSOURCE= endif -if NEED_OWN_MD5 -MD5SOURCE=md5.c -else -MD5SOURCE= -endif if ENABLE_WIN32SPECIFIC LIBSSPWIN32=libsspwin32.a else @@ -41,7 +36,6 @@ @LIBREGEX@ \ $(LIBSSPWIN32) EXTRA_libmiscutil_a_SOURCES = \ - md5.c \ snprintf.c \ strsep.c \ win32lib.c @@ -53,7 +47,7 @@ heap.c \ html_quote.c \ iso3307.c \ - $(MD5SOURCE) \ + md5.c \ radix.c \ rfc1035.c \ rfc1123.c \ Index: squid/lib/md5.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/md5.c,v retrieving revision 1.12 retrieving revision 1.12.30.1 diff -u -r1.12 -r1.12.30.1 --- squid/lib/md5.c 31 May 2006 11:51:11 -0000 1.12 +++ squid/lib/md5.c 27 Nov 2007 08:12:21 -0000 1.12.30.1 @@ -26,11 +26,16 @@ */ #include "config.h" -/* MS VisualStudio Projects are monolithic, so we need the following - * #if to exclude the MD5 code from compile process when we are - * building the SSL support. +#include "md5.h" + +/* + * Now that we have several alternatives the MD5 files are + * passed in by default. But a header-selection decides whether + * this provided version is to be built. + * TODO: may obsolete the MSV #if below. */ -#if !USE_SSL +#if USE_SQUID_MD5 + #if HAVE_STRING_H #include /* for memcpy() */ #endif @@ -41,8 +46,6 @@ #include /* for ntohl() */ #endif -#include "md5.h" - #ifdef WORDS_BIGENDIAN void byteSwap(uint32_t * buf, unsigned words) @@ -255,5 +258,5 @@ buf[3] += d; } -#endif -#endif +#endif /* !ASM_MD5 */ +#endif /* !USE_SQUID_MD5 */ Index: squid/src/HttpHeader.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpHeader.c,v retrieving revision 1.28 retrieving revision 1.28.6.1 diff -u -r1.28 -r1.28.6.1 --- squid/src/HttpHeader.c 11 Apr 2007 22:52:54 -0000 1.28 +++ squid/src/HttpHeader.c 27 Nov 2007 08:12:23 -0000 1.28.6.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.c,v 1.28 2007/04/11 22:52:54 squidadm Exp $ + * $Id: HttpHeader.c,v 1.28.6.1 2007/11/27 08:12:23 adri Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -381,12 +381,34 @@ } } +static void +httpHeaderRepack(HttpHeader * hdr) +{ + HttpHeaderPos dp = HttpHeaderInitPos; + HttpHeaderPos pos = HttpHeaderInitPos; + + /* XXX breaks layering for now! ie, getting grubby fingers in without httpHeaderEntryGet() */ + dp = 0; + pos = 0; + while (dp < hdr->entries.count) { + for (; dp < hdr->entries.count && hdr->entries.items[dp] == NULL; dp++); + assert(dp < hdr->entries.count); + hdr->entries.items[pos] = hdr->entries.items[dp]; + if (dp != pos) + hdr->entries.items[dp] = NULL; + pos++; + dp++; + } + arrayShrink(&hdr->entries, pos); +} + /* use fresh entries to replace old ones */ void httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask) { const HttpHeaderEntry *e; HttpHeaderPos pos = HttpHeaderInitPos; + assert(old && fresh); assert(old != fresh); debug(55, 7) ("updating hdr: %p <- %p\n", old, fresh); @@ -399,8 +421,17 @@ httpHeaderDelById(old, e->id); else httpHeaderDelByName(old, strBuf(e->name)); + } + pos = HttpHeaderInitPos; + while ((e = httpHeaderGetEntry(fresh, &pos))) { + /* deny bad guys (ok to check for HDR_OTHER) here */ + if (denied_mask && CBIT_TEST(*denied_mask, e->id)) + continue; httpHeaderAddClone(old, e); } + + /* And now, repack the array to "fill in the holes" */ + httpHeaderRepack(old); } /* just handy in parsing: resets and returns false */ Index: squid/src/HttpReply.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpReply.c,v retrieving revision 1.23 retrieving revision 1.23.6.1 diff -u -r1.23 -r1.23.6.1 --- squid/src/HttpReply.c 2 Jun 2007 19:52:18 -0000 1.23 +++ squid/src/HttpReply.c 27 Nov 2007 08:12:23 -0000 1.23.6.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpReply.c,v 1.23 2007/06/02 19:52:18 squidadm Exp $ + * $Id: HttpReply.c,v 1.23.6.1 2007/11/27 08:12:23 adri Exp $ * * DEBUG: section 58 HTTP Reply (Response) * AUTHOR: Alex Rousskov @@ -42,9 +42,8 @@ static HttpHeaderMask Denied304HeadersMask; static http_hdr_type Denied304HeadersArr[] = { - HDR_ALLOW, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE, HDR_CONTENT_LENGTH, - HDR_CONTENT_LOCATION, HDR_CONTENT_RANGE, HDR_LAST_MODIFIED, HDR_LINK, - HDR_OTHER + HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE, HDR_CONTENT_LENGTH, + HDR_CONTENT_LOCATION, HDR_CONTENT_RANGE }; /* local routines */ @@ -274,6 +273,7 @@ /* clean cache */ httpReplyHdrCacheClean(rep); /* update raw headers */ + httpHeaderDelById(&rep->header, HDR_AGE); httpHeaderUpdate(&rep->header, &freshRep->header, (const HttpHeaderMask *) &Denied304HeadersMask); /* init cache */ Index: squid/src/HttpRequest.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpRequest.c,v retrieving revision 1.17 retrieving revision 1.17.10.1 diff -u -r1.17 -r1.17.10.1 --- squid/src/HttpRequest.c 21 Jan 2007 14:03:34 -0000 1.17 +++ squid/src/HttpRequest.c 27 Nov 2007 08:12:24 -0000 1.17.10.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpRequest.c,v 1.17 2007/01/21 14:03:34 squidadm Exp $ + * $Id: HttpRequest.c,v 1.17.10.1 2007/11/27 08:12:24 adri Exp $ * * DEBUG: section 73 HTTP Request * AUTHOR: Duane Wessels @@ -59,6 +59,7 @@ requestAbortBody(req); if (req->auth_user_request) authenticateAuthUserRequestUnlock(req->auth_user_request); + safe_free(req->store_url); safe_free(req->canonical); safe_free(req->vary_hdr); safe_free(req->vary_headers); Index: squid/src/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v retrieving revision 1.54 retrieving revision 1.54.2.1 diff -u -r1.54 -r1.54.2.1 --- squid/src/Makefile.am 23 Sep 2007 14:51:42 -0000 1.54 +++ squid/src/Makefile.am 27 Nov 2007 08:12:24 -0000 1.54.2.1 @@ -156,6 +156,8 @@ cbdata.c \ client_db.c \ client_side.c \ + client_side_rewrite.c \ + client_side_storeurl_rewrite.c \ comm.c \ $(COMMLOOP_SOURCE) \ debug.c \ @@ -223,8 +225,10 @@ peer_userhash.c \ protos.h \ redirect.c \ + store_rewrite.c \ referer.c \ refresh.c \ + refresh_check.c \ send-announce.c \ $(SNMPSOURCE) \ squid.h \ @@ -245,6 +249,7 @@ store_swapin.c \ store_swapmeta.c \ store_swapout.c \ + store_update.c \ structs.h \ tools.c \ typedefs.h \ Index: squid/src/access_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/access_log.c,v retrieving revision 1.43 retrieving revision 1.43.2.1 diff -u -r1.43 -r1.43.2.1 --- squid/src/access_log.c 23 Sep 2007 14:51:42 -0000 1.43 +++ squid/src/access_log.c 27 Nov 2007 08:12:24 -0000 1.43.2.1 @@ -1,6 +1,6 @@ /* - * $Id: access_log.c,v 1.43 2007/09/23 14:51:42 squidadm Exp $ + * $Id: access_log.c,v 1.43.2.1 2007/11/27 08:12:24 adri Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -320,6 +320,8 @@ LFT_SEQUENCE_NUMBER, + LFT_EXT_FRESHNESS, + LFT_PERCENT /* special string cases for escaped chars */ } logformat_bcode_t; @@ -428,6 +430,8 @@ {"ea", LFT_EXT_LOG}, + {"ef", LFT_EXT_FRESHNESS}, + {"%", LFT_PERCENT}, {NULL, LFT_NONE} /* this must be last */ @@ -691,6 +695,12 @@ doint = 1; break; + case LFT_EXT_FRESHNESS: + out = al->ext_refresh; + + quote = 1; + break; + case LFT_PERCENT: out = "%"; break; Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.102 retrieving revision 1.102.2.1 diff -u -r1.102 -r1.102.2.1 --- squid/src/cache_cf.c 23 Sep 2007 14:51:42 -0000 1.102 +++ squid/src/cache_cf.c 27 Nov 2007 08:12:24 -0000 1.102.2.1 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.102 2007/09/23 14:51:42 squidadm Exp $ + * $Id: cache_cf.c,v 1.102.2.1 2007/11/27 08:12:24 adri Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -479,6 +479,7 @@ requirePathnameExists("Error Directory", Config.errorDirectory); authenticateConfigure(&Config.authConfig); externalAclConfigure(); + refreshCheckConfigure(); #if HTTP_VIOLATIONS { const refresh_t *R; @@ -494,6 +495,12 @@ debug(22, 1) ("WARNING: use of 'override-lastmod' in 'refresh_pattern' violates HTTP\n"); break; } + for (R = Config.Refresh; R; R = R->next) { + if (R->stale_while_revalidate <= 0) + continue; + debug(22, 1) ("WARNING: use of 'stale-while-revalidate' in 'refresh_pattern' violates HTTP\n"); + break; + } } #endif #if !HTTP_VIOLATIONS @@ -1610,6 +1617,34 @@ } } +static u_short +GetService(const char *proto) +{ + struct servent *port = NULL; + char *token = strtok(NULL, w_space); + if (token == NULL) { + self_destruct(); + return -1; /* NEVER REACHED */ + } + port = getservbyname(token, proto); + if (port != NULL) { + return ntohs((u_short) port->s_port); + } + return xatos(token); +} + +static u_short +GetTcpService(void) +{ + return GetService("tcp"); +} + +static u_short +GetUdpService(void) +{ + return GetService("udp"); +} + static void parse_peer(peer ** head) { @@ -1634,10 +1669,10 @@ p->options.no_digest = 1; p->options.no_netdb_exchange = 1; } - p->http_port = GetShort(); + p->http_port = GetTcpService(); if (!p->http_port) self_destruct(); - p->icp.port = GetShort(); + p->icp.port = GetUdpService(); p->connection_auth = -1; /* auto */ while ((token = strtok(NULL, w_space))) { if (!strcasecmp(token, "proxy-only")) { @@ -2146,7 +2181,13 @@ storeAppendPrintf(entry, " ignore-private"); if (head->flags.ignore_auth) storeAppendPrintf(entry, " ignore-auth"); + if (head->stale_while_revalidate > 0) + storeAppendPrintf(entry, " stale-while-revalidate=%d", head->stale_while_revalidate); #endif + if (head->max_stale >= 0) + storeAppendPrintf(entry, " max-stale=%d", head->max_stale); + if (head->negative_ttl >= 0) + storeAppendPrintf(entry, " negative-ttl=%d", head->negative_ttl); storeAppendPrintf(entry, "\n"); head = head->next; } @@ -2169,6 +2210,9 @@ int ignore_private = 0; int ignore_auth = 0; #endif + int stale_while_revalidate = -1; + int max_stale = -1; + int negative_ttl = -1; int i; refresh_t *t; regex_t comp; @@ -2213,10 +2257,18 @@ ignore_reload = 1; refresh_nocache_hack = 1; /* tell client_side.c that this is used */ + } else if (!strncmp(token, "stale-while-revalidate=", 23)) { + stale_while_revalidate = atoi(token + 23); } else #endif + if (!strncmp(token, "max-stale=", 10)) { + max_stale = atoi(token + 10); + } else if (!strncmp(token, "negative-ttl=", 13)) { + negative_ttl = atoi(token + 13); + } else { debug(22, 0) ("redreshAddToList: Unknown option '%s': %s\n", pattern, token); + } } if ((errcode = regcomp(&comp, pattern, flags)) != 0) { char errbuf[256]; @@ -2253,6 +2305,9 @@ if (ignore_auth) t->flags.ignore_auth = 1; #endif + t->stale_while_revalidate = stale_while_revalidate; + t->max_stale = max_stale; + t->negative_ttl = negative_ttl; t->next = NULL; while (*head) head = &(*head)->next; Index: squid/src/cf.data.depend =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.depend,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/src/cf.data.depend 5 Sep 2007 21:51:54 -0000 1.1 +++ squid/src/cf.data.depend 27 Nov 2007 08:12:24 -0000 1.1.4.1 @@ -52,3 +52,4 @@ programline extension_method errormap +refreshCheckHelper Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.209 retrieving revision 1.209.2.1 diff -u -r1.209 -r1.209.2.1 --- squid/src/cf.data.pre 23 Sep 2007 14:51:43 -0000 1.209 +++ squid/src/cf.data.pre 27 Nov 2007 08:12:24 -0000 1.209.2.1 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.209 2007/09/23 14:51:43 squidadm Exp $ +# $Id: cf.data.pre,v 1.209.2.1 2007/11/27 08:12:24 adri Exp $ # # SQUID Web Proxy Cache http://www.squid-cache.org/ # ---------------------------------------------------------- @@ -626,10 +626,18 @@ NOCOMMENT_START #Recommended minimum configuration: -acl all src 0.0.0.0/0.0.0.0 +acl all src all acl manager proto cache_object -acl localhost src 127.0.0.1/255.255.255.255 +acl localhost src 127.0.0.1/32 acl to_localhost dst 127.0.0.0/8 +# +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 10.0.0.0/8 # RFC1918 possible internal network +acl localnet src 172.16.0.0/12 # RFC1918 possible internal network +acl localnet src 192.168.0.0/16 # RFC1918 possible internal network +# acl SSL_ports port 443 acl Safe_ports port 80 # http acl Safe_ports port 21 # ftp @@ -686,11 +694,10 @@ # # INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS -# Example rule allowing access from your local networks. Adapt -# to list your (internal) IP networks from where browsing should -# be allowed -#acl our_networks src 192.168.1.0/24 192.168.2.0/24 -#http_access allow our_networks +# Example rule allowing access from your local networks. +# Adapt localnet in the ACL section to list your (internal) IP networks +# from where browsing should be allowed +http_access allow localnet # And finally deny all other access to this proxy http_access deny all @@ -740,8 +747,9 @@ See http_access for details NOCOMMENT_START -#Allow ICP queries from everyone -icp_access allow all +#Allow ICP queries from local networks only +icp_access allow localnet +icp_access deny all NOCOMMENT_END DOC_END @@ -759,8 +767,15 @@ See http_access for details -#Allow HTCP queries from everyone -htcp_access allow all + NOTE: The default if no htcp_access lines are present is to + deny all traffic. This default may cause problems with peers + using the htcp or htcp-oldsquid options. + +NOCOMMENT_START +#Allow HTCP queries from local networks only +htcp_access allow localnet +htcp_access deny all +NOCOMMENT_END DOC_END NAME: htcp_clr_access @@ -1161,7 +1176,7 @@ acl normal_service_net src 10.0.0.0/255.255.255.0 acl good_service_net src 10.0.1.0/255.255.255.0 - tcp_outgoing_tos 0x00 normal_service_net 0x00 + tcp_outgoing_tos 0x00 normal_service_net tcp_outgoing_tos 0x20 good_service_net TOS/DSCP values really only have local significance - so you should @@ -1515,8 +1530,12 @@ use 'htcp' to send HTCP, instead of ICP, queries to the neighbor. You probably also want to set the "icp port" to 4827 instead of 3130. + You must also allow this Squid htcp_access and + http_access in the peer Squid configuration. use 'htcp-oldsquid' to send HTCP to old Squid versions + You must also allow this Squid htcp_access and + http_access in the peer Squid configuration. 'originserver' causes this parent peer to be contacted as a origin server. Meant to be used in accelerator setups. @@ -2064,6 +2083,20 @@ numbers closer together. DOC_END +NAME: update_headers +IFDEF: HTTP_VIOLATIONS +COMMENT: on|off +TYPE: onoff +DEFAULT: on +LOC: Config.onoff.update_headers +DOC_START + By default Squid updates stored HTTP headers when receiving + a 304 response. Set this to off if you want to disable this + for disk I/O performance reasons. Disabling this VIOLATES the + HTTP standard, and could make you liable for problems which it + causes. +DOC_END + COMMENT_START LOGFILE OPTIONS ----------------------------------------------------------------------------- @@ -2529,6 +2562,30 @@ ----------------------------------------------------------------------------- COMMENT_END +NAME: storeurl_rewrite_program +TYPE: programline +LOC: Config.Program.store_rewrite.command +DEFAULT: none +DOC_START + +DOC_END + +NAME: storeurl_rewrite_children +TYPE: int +DEFAULT: 5 +LOC: Config.Program.store_rewrite.children +DOC_START + +DOC_END + +NAME: storeurl_rewrite_concurrency +TYPE: int +DEFAULT: 0 +LOC: Config.Program.store_rewrite.concurrency +DOC_START + +DOC_END + NAME: url_rewrite_program redirect_program TYPE: programline LOC: Config.Program.url_rewrite.command @@ -2539,7 +2596,13 @@ For each requested URL rewriter will receive on line with the format - URL client_ip "/" fqdn user method urlgroup + URL client_ip "/" fqdn user method urlgroup + [ kvpairs] + + In the future, the rewriter interface will be extended with + key=value pairs ("kvpairs" shown above). Rewriter programs + should be prepared to receive and possibly ignore additional + whitespace-separated tokens on each input line. And the rewriter may return a rewritten URL. The other components of the request line does not need to be returned (ignored if they are). @@ -2550,7 +2613,7 @@ It can also return a "urlgroup" that can subsequently be matched in cache_peer_access and similar ACL driven rules. An urlgroup is - returned by prefixing the returned url with "!urlgroup!" + returned by prefixing the returned URL with "!urlgroup!". By default, a URL rewriter is not used. DOC_END @@ -2574,6 +2637,12 @@ The number of requests each redirector helper can handle in parallel. Defaults to 0 which indicates the redirector is a old-style single threaded redirector. + + When this directive is set to a value >= 1 then the protocol + used to communicate with the helper is modified to include + a request ID in front of the request/response. The request + ID from the request must be echoed back with the response + to that request. DOC_END NAME: url_rewrite_host_header redirect_rewrites_host_header @@ -2599,6 +2668,14 @@ are sent. DOC_END +NAME: storeurl_access +TYPE: acl_access +DEFAULT: none +LOC: Config.accessList.storeurl_rewrite +DOC_START + +DOC_END + NAME: redirector_bypass TYPE: onoff LOC: Config.onoff.redirector_bypass @@ -2692,6 +2769,16 @@ NOCOMMENT_END DOC_END +NAME: max_stale +COMMENT: time-units +TYPE: time_t +LOC: Config.maxStale +DEFAULT: 1 week +DOC_START + This option puts an upper limit on how stale content Squid + will serve from the cache if cache validation fails. +DOC_END + NAME: refresh_pattern TYPE: refreshpattern LOC: Config.Refresh @@ -2722,6 +2809,9 @@ ignore-no-cache ignore-private ignore-auth + stale-while-revalidate=NN + max-stale=NN + negative-ttl=NN override-expire enforces min age even if the server sent a Expires: header. Doing this VIOLATES the HTTP @@ -2758,6 +2848,19 @@ Enabling this feature could make you liable for problems which it causes. + stale-while-revalidate=NN makes Squid perform an asyncronous + cache validation if the object isn't more stale than NN. + Doing this VIOLATES the HTTP standard. Enabling this + feature could make you liable for problems which it + causes. + + max-stale=NN provided a maximum staleness factor. Squid won't + serve objects more stale than this even if it failed to + validate the object. + + negative-ttl=NN overrides the global negative_ttl parameter + selectively for URLs matching this pattern (in seconds). + Basically a cached object is: FRESH if expires < now, else STALE @@ -3232,6 +3335,69 @@ or response to be rejected. DOC_END +NAME: external_refresh_check +TYPE: refreshCheckHelper +DEFAULT: none +LOC: Config.Program.refresh_check +DOC_START + This option defines an external helper for determining whether to + refresh a stale response. It will be called when Squid receives a + request for a cached response that is stale; the helper can either + confirm that the response is stale with a STALE response, or + extend the freshness of the response (thereby avoiding a refresh + check) with a FRESH response, along with a freshness=nnn keyword. + + external_refresh_check [options] FORMAT.. /path/to/helper [helper_args] + + If present, helper_args will be passed to the helper on the command + line verbatim. + + Options: + + children=n Number of processes to spawn to service external + refresh checks (default 5). + concurrency=n Concurrency level per process. Only used with + helpers capable of processing more than one query + at a time. + + When using the concurrency option, the protocol is changed by introducing + a query channel tag infront of the request/response. The query channel + tag is a number between 0 and concurrency−1. + + FORMAT specifications: + + %CACHE_URI The URI of the cached response + %RES{Header} HTTP response header value + %AGE The age of the cached response + + The request sent to the helper consists of the data in the format + specification in the order specified. + + The helper receives lines per the above format specification, and + returns lines starting with OK or ERR indicating the validity of + the request and optionally followed by additional keywords with + more details. URL escaping is used to protect each value in both + requests and responses. + + General result syntax: + + FRESH / STALE keyword=value ... + + Defined keywords: + + freshness=nnn The number of seconds to extend the freshness of + the response by. + log=string String to be logged in access.log. Available as + %ef in logformat specifications. + res{Header}=value + Value to update response headers with. If already + present, the supplied value completely replaces + the cached value. + + In the event of a helper−related error (e.g., overload), Squid + will always default to STALE. +DOC_END + COMMENT_START TIMEOUTS ----------------------------------------------------------------------------- @@ -3404,9 +3570,10 @@ LOC: Config.EmailProgram DOC_START Email program used to send mail if the cache dies. - The default is "mail". The specified program must complain + The default is "mail". The specified program must comply with the standard Unix mail syntax: - mail_program recipient < mailfile + mail-program recipient < mailfile + Optional command line options can be specified. DOC_END @@ -3989,8 +4156,7 @@ DEFAULT: on DOC_START This controls whether the server will generate a Cache Digest - of its contents. By default, Cache Digest generation is - enabled if Squid is compiled with USE_CACHE_DIGESTS defined. + of its contents. DOC_END NAME: digest_bits_per_entry @@ -4011,7 +4177,7 @@ LOC: Config.digest.rebuild_period DEFAULT: 1 hour DOC_START - This is the number of seconds between Cache Digest rebuilds. + This is the wait time between Cache Digest rebuilds. DOC_END NAME: digest_rewrite_period @@ -4021,8 +4187,7 @@ LOC: Config.digest.rewrite_period DEFAULT: 1 hour DOC_START - This is the number of seconds between Cache Digest writes to - disk. + This is the wait time between Cache Digest writes to disk. DOC_END NAME: digest_swapout_chunk_size @@ -4155,31 +4320,46 @@ TYPE: address LOC:Config.Addrs.udp_incoming DEFAULT: 0.0.0.0 -DOC_NONE +DOC_START + udp_incoming_address is used for UDP packets received from other + caches. + + The default behavior is to not bind to any specific address. + + Only change this if you want to have all UDP queries received on + a specific interface/address. + + NOTE: udp_incoming_address is used by the ICP, HTCP, and DNS + modules. Altering it will affect all of them in the same manner. + + see also; udp_outgoing_address + + NOTE, udp_incoming_address and udp_outgoing_address can not + have the same value since they both use the same port. +DOC_END NAME: udp_outgoing_address TYPE: address LOC: Config.Addrs.udp_outgoing DEFAULT: 255.255.255.255 DOC_START - udp_incoming_address is used for the ICP socket receiving packets - from other caches. - udp_outgoing_address is used for ICP packets sent out to other + udp_outgoing_address is used for UDP packets sent out to other caches. The default behavior is to not bind to any specific address. - A udp_incoming_address value of 0.0.0.0 indicates Squid - should listen for UDP messages on all available interfaces. - - If udp_outgoing_address is set to 255.255.255.255 (the default) - it will use the same socket as udp_incoming_address. Only - change this if you want to have ICP queries sent using another - address than where this Squid listens for ICP queries from other + Instead it will use the same socket as udp_incoming_address. + Only change this if you want to have UDP queries sent using another + address than where this Squid listens for UDP queries from other caches. + NOTE: udp_outgoing_address is used by the ICP, HTCP, and DNS + modules. Altering it will affect all of them in the same manner. + + see also; udp_incoming_address + NOTE, udp_incoming_address and udp_outgoing_address can not - have the same value since they both use port 3130. + have the same value since they both use the same port. DOC_END NAME: icp_hit_stale @@ -4580,6 +4760,18 @@ acts on cacheable requests. DOC_END +NAME: ignore_ims_on_miss +IFDEF: HTTP_VIOLATIONS +COMMENT: on|off +TYPE: onoff +DEFAULT: off +LOC: Config.onoff.ignore_ims_on_miss +DOC_START + This options makes Squid ignore If-Modified-Since on + cache misses. This is useful while the cache is + mostly empty to more quickly have the cache populated. +DOC_END + NAME: always_direct TYPE: acl_access LOC: Config.accessList.AlwaysDirect Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.202.2.6 retrieving revision 1.202.2.7 diff -u -r1.202.2.6 -r1.202.2.7 --- squid/src/client_side.c 29 Sep 2007 10:38:27 -0000 1.202.2.6 +++ squid/src/client_side.c 27 Nov 2007 08:12:25 -0000 1.202.2.7 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.202.2.6 2007/09/29 10:38:27 adri Exp $ + * $Id: client_side.c,v 1.202.2.7 2007/11/27 08:12:25 adri Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -116,14 +116,14 @@ static int clientGotNotEnough(clientHttpRequest *); static void checkFailureRatio(err_type, hier_code); static void clientProcessMiss(clientHttpRequest *); +static void clientProcessHit(clientHttpRequest * http); static void clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep); static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri); static clientHttpRequest *parseHttpRequest(ConnStateData *, HttpMsgBuf *, method_t *, int *); -static void clientRedirectStart(clientHttpRequest * http); -static RH clientRedirectDone; + static void clientCheckNoCache(clientHttpRequest *); static void clientCheckNoCacheDone(int answer, void *data); -static STNCB clientHandleIMSReply; +static STHCB clientHandleIMSReply; static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); #if USE_IDENT static IDCB clientIdentDone; @@ -135,14 +135,15 @@ #endif /* FOLLOW_X_FORWARDED_FOR */ static int clientOnlyIfCached(clientHttpRequest * http); static STNCB clientSendMoreData; -static STNCB clientSendMoreHeaderData; -static STNCB clientCacheHit; +static STHCB clientSendHeaders; +static STHCB clientCacheHit; static void clientSetKeepaliveFlag(clientHttpRequest *); static void clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb); static void clientPackTermBound(String boundary, MemBuf * mb); -static void clientInterpretRequestHeaders(clientHttpRequest *); static void clientProcessRequest(clientHttpRequest *); -static void clientProcessExpired(void *data); +static void clientProcessExpired(clientHttpRequest *); +static void clientRefreshCheck(clientHttpRequest *); +static REFRESHCHECK clientRefreshCheckDone; static void clientProcessOnlyIfCachedMiss(clientHttpRequest * http); static int clientCachable(clientHttpRequest * http); static int clientHierarchical(clientHttpRequest * http); @@ -155,7 +156,6 @@ static void clientEatRequestBody(clientHttpRequest *); static void clientAccessCheck(void *data); static void clientAccessCheckDone(int answer, void *data); -static void clientAccessCheck2(void *data); static void clientAccessCheckDone2(int answer, void *data); static BODY_HANDLER clientReadBody; static void clientAbortBody(request_t * req); @@ -167,6 +167,31 @@ static StoreEntry *clientCreateStoreEntry(clientHttpRequest *, method_t, request_flags); static inline int clientNatLookup(ConnStateData * conn); +/* Temporary here while restructuring stuff */ +static void +storeClientCopyHeadersCB(void *data, char *buf, ssize_t size) +{ + clientHttpRequest *http = data; + assert(http->header_callback); + assert(http->header_entry); + memFree(buf, MEM_STORE_CLIENT_BUF); + if (!http->header_entry) + return; + if (size < 0 || !memHaveHeaders(http->header_entry->mem_obj)) { + http->header_callback(data, NULL); + return; + } + http->header_callback(data, http->header_entry->mem_obj->reply); +} +void +storeClientCopyHeaders(store_client * sc, StoreEntry * e, STHCB * callback, void *callback_data) +{ + clientHttpRequest *http = callback_data; + http->header_callback = callback; + http->header_entry = e; + storeClientCopy(http->sc, e, 0, 0, STORE_CLIENT_BUF_SZ, memAllocate(MEM_STORE_CLIENT_BUF), storeClientCopyHeadersCB, http); +} + #if USE_IDENT static void clientIdentDone(const char *ident, void *data) @@ -177,7 +202,7 @@ #endif -static aclCheck_t * +aclCheck_t * clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http) { aclCheck_t *ch; @@ -358,7 +383,7 @@ aclNBCheck(http->acl_checklist, clientAccessCheckDone, http); } -static void +void clientAccessCheck2(void *data) { clientHttpRequest *http = data; @@ -394,12 +419,14 @@ if (h->request == NULL) h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); e = storeCreateEntry(h->uri, flags, m); + if (h->request->store_url) + storeEntrySetStoreUrl(e, h->request->store_url); h->sc = storeClientRegister(e, h); #if DELAY_POOLS if (h->log_type != LOG_TCP_DENIED) delaySetStoreClient(h->sc, delayClient(h)); #endif - storeClientRef(h->sc, e, 0, 0, SM_PAGE_SIZE, clientSendMoreHeaderData, h); + storeClientCopyHeaders(h->sc, e, clientSendHeaders, h); return e; } @@ -531,123 +558,6 @@ } static void -clientRedirectAccessCheckDone(int answer, void *data) -{ - clientHttpRequest *http = data; - http->acl_checklist = NULL; - if (answer == ACCESS_ALLOWED) - redirectStart(http, clientRedirectDone, http); - else - clientRedirectDone(http, NULL); -} - -static void -clientRedirectStart(clientHttpRequest * http) -{ - debug(33, 5) ("clientRedirectStart: '%s'\n", http->uri); - if (Config.Program.url_rewrite.command == NULL) { - clientRedirectDone(http, NULL); - return; - } - if (Config.accessList.url_rewrite) { - http->acl_checklist = clientAclChecklistCreate(Config.accessList.url_rewrite, http); - aclNBCheck(http->acl_checklist, clientRedirectAccessCheckDone, http); - } else { - redirectStart(http, clientRedirectDone, http); - } -} - -static void -clientRedirectDone(void *data, char *result) -{ - clientHttpRequest *http = data; - request_t *new_request = NULL; - request_t *old_request = http->request; - const char *urlgroup = http->conn->port->urlgroup; - debug(33, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri, - result ? result : "NULL"); - assert(http->redirect_state == REDIRECT_PENDING); - http->redirect_state = REDIRECT_DONE; - if (result) { - http_status status; - if (*result == '!') { - char *t; - if ((t = strchr(result + 1, '!')) != NULL) { - urlgroup = result + 1; - *t++ = '\0'; - result = t; - } else { - debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); - } - } - status = (http_status) atoi(result); - if (status == HTTP_MOVED_PERMANENTLY - || status == HTTP_MOVED_TEMPORARILY - || status == HTTP_SEE_OTHER - || status == HTTP_TEMPORARY_REDIRECT) { - char *t = result; - if ((t = strchr(result, ':')) != NULL) { - http->redirect.status = status; - http->redirect.location = xstrdup(t + 1); - goto redirect_parsed; - } else { - debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); - } - } else if (strcmp(result, http->uri)) - new_request = urlParse(old_request->method, result); - } - redirect_parsed: - if (new_request) { - safe_free(http->uri); - http->uri = xstrdup(urlCanonical(new_request)); - new_request->http_ver = old_request->http_ver; - httpHeaderAppend(&new_request->header, &old_request->header); - new_request->client_addr = old_request->client_addr; - new_request->client_port = old_request->client_port; -#if FOLLOW_X_FORWARDED_FOR - new_request->indirect_client_addr = old_request->indirect_client_addr; -#endif /* FOLLOW_X_FORWARDED_FOR */ - new_request->my_addr = old_request->my_addr; - new_request->my_port = old_request->my_port; - new_request->flags = old_request->flags; - new_request->flags.redirected = 1; - if (old_request->auth_user_request) { - new_request->auth_user_request = old_request->auth_user_request; - authenticateAuthUserRequestLock(new_request->auth_user_request); - } - if (old_request->body_reader) { - new_request->body_reader = old_request->body_reader; - new_request->body_reader_data = old_request->body_reader_data; - old_request->body_reader = NULL; - old_request->body_reader_data = NULL; - } - new_request->content_length = old_request->content_length; - if (strBuf(old_request->extacl_log)) - new_request->extacl_log = stringDup(&old_request->extacl_log); - if (old_request->extacl_user) - new_request->extacl_user = xstrdup(old_request->extacl_user); - if (old_request->extacl_passwd) - new_request->extacl_passwd = xstrdup(old_request->extacl_passwd); - requestUnlink(old_request); - http->request = requestLink(new_request); - } else { - /* Don't mess with urlgroup on internal request */ - if (old_request->flags.internal) - urlgroup = NULL; - } - safe_free(http->request->urlgroup); /* only paranoia. should not happen */ - if (urlgroup && *urlgroup) - http->request->urlgroup = xstrdup(urlgroup); - clientInterpretRequestHeaders(http); -#if HEADERS_LOG - headersLog(0, 1, request->method, request); -#endif - /* XXX This really should become a ref-counted string type pointer, not a copy! */ - fd_note(http->conn->fd, http->uri); - clientAccessCheck2(http); -} - -static void clientCheckNoCache(clientHttpRequest * http) { if (Config.accessList.noCache && http->request->flags.cachable) { @@ -689,74 +599,49 @@ } static void -clientHandleETagReply(void *data, mem_node_ref ref, ssize_t size) +clientHandleETagReply(void *data, HttpReply * rep) { //const char *buf = ref.node->data + ref.offset; clientHttpRequest *http = data; StoreEntry *entry = http->entry; - MemObject *mem; - const char *url = storeUrl(entry); - http_status status; - debug(33, 3) ("clientHandleETagReply: %s, %d bytes\n", url, (int) size); + const char *url = storeLookupUrl(entry); if (entry == NULL) { /* client aborted */ - stmemNodeUnref(&ref); return; } - if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - stmemNodeUnref(&ref); + if (!rep) { + debug(33, 3) ("clientHandleETagReply: FAILED '%s'\n", url); clientHandleETagMiss(http); return; } - mem = entry->mem_obj; - status = mem->reply->sline.status; if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { debug(33, 3) ("clientHandleETagReply: ABORTED '%s'\n", url); - stmemNodeUnref(&ref); clientHandleETagMiss(http); return; } - if (STORE_PENDING == entry->store_status && 0 == status) { - debug(33, 3) ("clientHandleETagReply: Incomplete headers for '%s'\n", url); - if (size >= SM_PAGE_SIZE) { - /* will not get any bigger than that */ - debug(33, 3) ("clientHandleETagReply: Reply is too large '%s'\n", url); - stmemNodeUnref(&ref); - clientHandleETagMiss(http); - } else { - stmemNodeUnref(&ref); - storeClientRef(http->sc, entry, - http->out.offset + size, - http->out.offset, - SM_PAGE_SIZE, - clientHandleETagReply, - http); - } - return; - } - if (HTTP_NOT_MODIFIED == mem->reply->sline.status) { + debug(33, 3) ("clientHandleETagReply: %s = %d\n", url, (int) rep->sline.status); + if (HTTP_NOT_MODIFIED == rep->sline.status) { /* Remember the ETag and restart */ - stmemNodeUnref(&ref); - if (mem->reply) { + if (rep) { request_t *request = http->request; - const char *etag = httpHeaderGetStr(&mem->reply->header, HDR_ETAG); + const char *etag = httpHeaderGetStr(&rep->header, HDR_ETAG); const char *vary = request->vary_headers; - int has_vary = httpHeaderHas(&entry->mem_obj->reply->header, HDR_VARY); + int has_vary = httpHeaderHas(&rep->header, HDR_VARY); #if X_ACCELERATOR_VARY - has_vary |= httpHeaderHas(&entry->mem_obj->reply->header, HDR_X_ACCELERATOR_VARY); + has_vary |= httpHeaderHas(&rep->header, HDR_X_ACCELERATOR_VARY); #endif if (has_vary) - vary = httpMakeVaryMark(request, mem->reply); + vary = httpMakeVaryMark(request, rep); if (etag && vary) { - storeAddVary(mem->url, mem->method, NULL, httpHeaderGetStr(&mem->reply->header, HDR_ETAG), request->vary_hdr, request->vary_headers, strBuf(request->vary_encoding)); + storeAddVary(url, entry->mem_obj->method, NULL, httpHeaderGetStr(&rep->header, HDR_ETAG), request->vary_hdr, request->vary_headers, strBuf(request->vary_encoding)); } } clientHandleETagMiss(http); return; } /* Send the new object to the client */ - clientSendMoreHeaderData(data, ref, size); + clientSendHeaders(data, rep); return; } @@ -767,6 +652,8 @@ StoreEntry *entry = NULL; debug(33, 3) ("clientProcessETag: '%s'\n", http->uri); entry = storeCreateEntry(url, http->request->flags, http->request->method); + if (http->request->store_url) + storeEntrySetStoreUrl(entry, http->request->store_url); http->sc = storeClientRegister(entry, http); #if DELAY_POOLS /* delay_id is already set on original store client */ @@ -778,18 +665,34 @@ /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientProcessETag: found ENTRY_ABORTED object\n"); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, entry, clientHandleETagReply, http); } static void -clientProcessExpired(void *data) +clientRefreshCheck(clientHttpRequest * http) +{ + refreshCheckSubmit(http->entry, clientRefreshCheckDone, http); +} + +static void +clientRefreshCheckDone(void *data, int fresh, const char *log) { clientHttpRequest *http = data; + if (log) { + safe_free(http->al.ext_refresh); + http->al.ext_refresh = xstrdup(log); + } + if (fresh) + clientProcessHit(http); + else + clientProcessExpired(http); +} + +static void +clientProcessExpired(clientHttpRequest * http) +{ char *url = http->uri; StoreEntry *entry = NULL; int hit = 0; @@ -828,6 +731,8 @@ } if (!entry) { entry = storeCreateEntry(url, http->request->flags, http->request->method); + if (http->request->store_url) + storeEntrySetStoreUrl(entry, http->request->store_url); if (http->entry->mem_obj) { http->entry->mem_obj->refresh_timestamp = squid_curtime; if (Config.onoff.collapsed_forwarding) { @@ -836,7 +741,12 @@ } } } - /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ + if (entry->mem_obj->old_entry) { + storeUnlockObject(entry->mem_obj->old_entry); + entry->mem_obj->old_entry = NULL; + } + entry->mem_obj->old_entry = http->old_entry; + storeLockObject(entry->mem_obj->old_entry); http->sc = storeClientRegister(entry, http); #if DELAY_POOLS /* delay_id is already set on original store client */ @@ -848,6 +758,7 @@ } else http->request->lastmod = -1; debug(33, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod); + /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ http->entry = entry; http->out.offset = 0; if (can_revalidate) { @@ -862,10 +773,7 @@ /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, entry, clientHandleIMSReply, http); } @@ -881,8 +789,16 @@ /* If the reply is a failure then send the old object as a last * resort */ if (status >= 500 && status < 600) { - debug(33, 3) ("clientGetsOldEntry: YES, failure reply=%d\n", status); - return 1; + if (EBIT_TEST(new_entry->flags, ENTRY_NEGCACHED)) { + debug(33, 3) ("clientGetsOldEntry: NO, negatively cached failure reply=%d\n", status); + return 0; + } + if (refreshCheckStaleOK(old_entry, request)) { + debug(33, 3) ("clientGetsOldEntry: YES, failure reply=%d and old acceptable to send\n", status); + return 1; + } + debug(33, 3) ("clientGetsOldEntry: NO, failure reply=%d and old NOT acceptable to send\n", status); + return 0; } /* If the reply is not to a cache validation conditional then * we should forward it to the client */ @@ -929,7 +845,7 @@ static void -clientHandleIMSReply(void *data, mem_node_ref ref, ssize_t size) +clientHandleIMSReply(void *data, HttpReply * rep) { clientHttpRequest *http = data; StoreEntry *entry = http->entry; @@ -938,23 +854,20 @@ int unlink_request = 0; StoreEntry *oldentry; int recopy = 1; - http_status status; - debug(33, 3) ("clientHandleIMSReply: %s, %ld bytes\n", url, (long int) size); + debug(33, 3) ("clientHandleIMSReply: %s\n", url); if (http->old_entry && http->old_entry->mem_obj && http->old_entry->mem_obj->ims_entry) { storeUnlockObject(http->old_entry->mem_obj->ims_entry); http->old_entry->mem_obj->ims_entry = NULL; } if (entry == NULL) { - stmemNodeUnref(&ref); return; } - if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - stmemNodeUnref(&ref); - return; + if (entry->mem_obj->old_entry) { + storeUnlockObject(entry->mem_obj->old_entry); + entry->mem_obj->old_entry = NULL; } mem = entry->mem_obj; - status = mem->reply->sline.status; - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + if (!rep) { debug(33, 3) ("clientHandleIMSReply: ABORTED '%s'\n", url); /* We have an existing entry, but failed to validate it */ /* Its okay to send the old one anyway */ @@ -963,45 +876,26 @@ storeUnlockObject(entry); entry = http->entry = http->old_entry; http->sc = http->old_sc; - } else if (STORE_PENDING == entry->store_status && 0 == status) { - debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); - if (size >= SM_PAGE_SIZE) { - /* will not get any bigger than that */ - debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); - /* use old entry, this repeats the code abovez */ - http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeClientUnregister(http->sc, entry, http); - storeUnlockObject(entry); - entry = http->entry = http->old_entry; - http->sc = http->old_sc; - /* continue */ - } else { - stmemNodeUnref(&ref); - storeClientRef(http->sc, entry, - http->out.offset + size, - http->out.offset, - SM_PAGE_SIZE, - clientHandleIMSReply, - http); - return; - } } else if (clientGetsOldEntry(entry, http->old_entry, http->request)) { /* We initiated the IMS request, the client is not expecting * 304, so put the good one back. First, make sure the old entry * headers have been loaded from disk. */ oldentry = http->old_entry; - http->log_type = LOG_TCP_REFRESH_HIT; if (oldentry->mem_obj->request == NULL) { oldentry->mem_obj->request = requestLink(mem->request); unlink_request = 1; } - if (mem->reply->sline.status == HTTP_NOT_MODIFIED) { + if (rep->sline.status == HTTP_NOT_MODIFIED) { /* Don't memcpy() the whole reply structure here. For example, * www.thegist.com (Netscape/1.13) returns a content-length for * 304's which seems to be the length of the 304 HEADERS!!! and * not the body they refer to. */ - httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); + httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, rep); storeTimestampsSet(oldentry); + storeUpdate(oldentry, http->request); + http->log_type = LOG_TCP_REFRESH_HIT; + } else { + http->log_type = LOG_TCP_REFRESH_FAIL_HIT; } storeClientUnregister(http->sc, entry, http); http->sc = http->old_sc; @@ -1015,11 +909,14 @@ /* the client can handle this reply, whatever it is */ http->flags.hit = 0; http->log_type = LOG_TCP_REFRESH_MISS; - if (HTTP_NOT_MODIFIED == mem->reply->sline.status) { + if (HTTP_NOT_MODIFIED == rep->sline.status) { httpReplyUpdateOnNotModified(http->old_entry->mem_obj->reply, - mem->reply); + rep); storeTimestampsSet(http->old_entry); - http->log_type = LOG_TCP_REFRESH_HIT; + if (!EBIT_TEST(http->old_entry->flags, REFRESH_FAILURE)) + http->log_type = LOG_TCP_REFRESH_HIT; + else + http->log_type = LOG_TCP_REFRESH_FAIL_HIT; } /* Get rid of the old entry if not a cache validation */ if (!http->request->flags.cache_validation) @@ -1032,16 +929,11 @@ http->old_sc = NULL; assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); if (recopy) { - stmemNodeUnref(&ref); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, - clientSendMoreHeaderData, + storeClientCopyHeaders(http->sc, entry, + clientSendHeaders, http); } else { - clientSendMoreHeaderData(data, stmemNodeRef(&ref), size); - stmemNodeUnref(&ref); + clientSendHeaders(data, rep); } } @@ -1051,7 +943,7 @@ squid_off_t object_length; MemObject *mem = entry->mem_obj; time_t mod_time = entry->lastmod; - debug(33, 3) ("modifiedSince: '%s'\n", storeUrl(entry)); + debug(33, 3) ("modifiedSince: '%s'\n", storeLookupUrl(entry)); debug(33, 3) ("modifiedSince: mod_time = %ld\n", (long int) mod_time); if (mod_time < 0) return 1; @@ -1116,10 +1008,7 @@ http->entry->mem_obj->method = http->request->method; http->sc = storeClientRegister(http->entry, http); http->log_type = LOG_TCP_HIT; - storeClientRef(http->sc, http->entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, http->entry, clientCacheHit, http); return; @@ -1407,7 +1296,7 @@ #endif } -static void +void clientInterpretRequestHeaders(clientHttpRequest * http) { request_t *request = http->request; @@ -1680,6 +1569,8 @@ return 1; if (code == LOG_TCP_STALE_HIT) return 1; + if (code == LOG_TCP_ASYNC_HIT) + return 1; if (code == LOG_TCP_IMS_HIT) return 1; if (code == LOG_TCP_REFRESH_FAIL_HIT) @@ -2032,7 +1923,7 @@ httpHdrMangleList(hdr, request); } -/* Used exclusively by clientBuildReply() during failure cases only */ +/* Used exclusively by clientCloneReply() during failure cases only */ static void clientUnwindReply(clientHttpRequest * http, HttpReply * rep) { @@ -2054,7 +1945,7 @@ * be. */ static HttpReply * -clientBuildReply(clientHttpRequest * http) +clientCloneReply(clientHttpRequest * http, HttpReply * orig_rep) { HttpReply *rep = NULL; /* If we don't have a memobj / reply by now then we're stuffed */ @@ -2063,7 +1954,7 @@ return NULL; } /* try to grab the already-parsed header */ - rep = httpReplyClone(http->sc->entry->mem_obj->reply); + rep = httpReplyClone(orig_rep); if (rep->pstate == psParsed) { /* enforce 1.0 reply version */ httpBuildVersion(&rep->sline.version, 1, 0); @@ -2108,6 +1999,163 @@ } /* + * Perform an async refresh of an object + */ +typedef struct _clientAsyncRefreshRequest { + request_t *request; + StoreEntry *entry; + StoreEntry *old_entry; + store_client *sc; + squid_off_t offset; + size_t buf_in_use; + char readbuf[STORE_CLIENT_BUF_SZ]; + struct timeval start; +} clientAsyncRefreshRequest; + +CBDATA_TYPE(clientAsyncRefreshRequest); + +static void +clientAsyncDone(clientAsyncRefreshRequest * async) +{ + AccessLogEntry al; + static aclCheck_t *ch; + MemObject *mem = async->entry->mem_obj; + request_t *request = async->request; + memset(&al, 0, sizeof(al)); + al.icp.opcode = ICP_INVALID; + al.url = mem->url; + debug(33, 9) ("clientAsyncDone: url='%s'\n", al.url); + al.http.code = mem->reply->sline.status; + al.http.content_type = strBuf(mem->reply->content_type); + al.cache.size = async->offset; + if (async->old_entry->mem_obj) + async->old_entry->mem_obj->refresh_timestamp = 0; + if (mem->reply->sline.status == 304) { + /* Don't memcpy() the whole reply structure here. For example, + * www.thegist.com (Netscape/1.13) returns a content-length for + * 304's which seems to be the length of the 304 HEADERS!!! and + * not the body they refer to. */ + httpReplyUpdateOnNotModified(async->old_entry->mem_obj->reply, async->entry->mem_obj->reply); + storeTimestampsSet(async->old_entry); + storeUpdate(async->old_entry, async->request); + al.cache.code = LOG_TCP_ASYNC_HIT; + } else + al.cache.code = LOG_TCP_ASYNC_MISS; + al.cache.msec = tvSubMsec(async->start, current_time); + if (Config.onoff.log_mime_hdrs) { + Packer p; + MemBuf mb; + memBufDefInit(&mb); + packerToMemInit(&p, &mb); + httpHeaderPackInto(&request->header, &p); + al.headers.request = xstrdup(mb.buf); + packerClean(&p); + memBufClean(&mb); + } + al.http.method = request->method; + al.http.version = request->http_ver; + al.hier = request->hier; + if (request->auth_user_request) { + if (authenticateUserRequestUsername(request->auth_user_request)) + al.cache.authuser = xstrdup(authenticateUserRequestUsername(request->auth_user_request)); + authenticateAuthUserRequestUnlock(request->auth_user_request); + request->auth_user_request = NULL; + } else if (request->extacl_user) { + al.cache.authuser = xstrdup(request->extacl_user); + } + al.request = request; + al.reply = mem->reply; + ch = aclChecklistCreate(Config.accessList.http, request, NULL); + ch->reply = mem->reply; + if (!Config.accessList.log || aclCheckFast(Config.accessList.log, ch)) + accessLogLog(&al, ch); + aclChecklistFree(ch); + storeClientUnregister(async->sc, async->entry, async); + storeUnlockObject(async->entry); + storeUnlockObject(async->old_entry); + requestUnlink(async->request); + safe_free(al.headers.request); + safe_free(al.headers.reply); + safe_free(al.cache.authuser); + cbdataFree(async); +} + +static void +clientHandleAsyncReply(void *data, char *buf, ssize_t size) +{ + clientAsyncRefreshRequest *async = data; + StoreEntry *e = async->entry; + if (EBIT_TEST(e->flags, ENTRY_ABORTED)) { + clientAsyncDone(async); + return; + } + if (size <= 0) { + clientAsyncDone(async); + return; + } + async->offset += size; + if (e->mem_obj->reply->sline.status == 304) { + clientAsyncDone(async); + return; + } + storeClientCopy(async->sc, async->entry, + async->offset, + async->offset, + STORE_CLIENT_BUF_SZ, async->readbuf, + clientHandleAsyncReply, + async); +} + +static void +clientAsyncRefresh(clientHttpRequest * http) +{ + char *url = http->uri; + clientAsyncRefreshRequest *async; + request_t *request = http->request; + debug(33, 3) ("clientAsyncRefresh: '%s'\n", http->uri); + CBDATA_INIT_TYPE(clientAsyncRefreshRequest); + http->entry->mem_obj->refresh_timestamp = squid_curtime; + async = cbdataAlloc(clientAsyncRefreshRequest); + async->start = current_time; + async->request = requestLink(request); + async->old_entry = http->entry; + storeLockObject(async->old_entry); + async->entry = storeCreateEntry(url, + request->flags, + request->method); + async->entry->mem_obj->old_entry = async->old_entry; + storeLockObject(async->entry->mem_obj->old_entry); + async->sc = storeClientRegister(async->entry, async); + request->etags = NULL; /* Should always be null as this was a cache hit, but just in case.. */ + httpHeaderDelById(&request->header, HDR_RANGE); + httpHeaderDelById(&request->header, HDR_IF_RANGE); + httpHeaderDelById(&request->header, HDR_IF_NONE_MATCH); + httpHeaderDelById(&request->header, HDR_IF_MATCH); + if (async->old_entry->lastmod > 0) + request->lastmod = async->old_entry->lastmod; + else if (async->old_entry->mem_obj && async->old_entry->mem_obj->reply) + request->lastmod = async->old_entry->mem_obj->reply->date; + else + request->lastmod = -1; + if (!request->etag) { + const char *etag = httpHeaderGetStr(&async->old_entry->mem_obj->reply->header, HDR_ETAG); + if (etag) + async->request->etag = xstrdup(etag); + } +#if DELAY_POOLS + /* delay_id is already set on original store client */ + delaySetStoreClient(async->sc, delayClient(http)); +#endif + fwdStart(-1, async->entry, async->request); + storeClientCopy(async->sc, async->entry, + async->offset, + async->offset, + STORE_CLIENT_BUF_SZ, async->readbuf, + clientHandleAsyncReply, + async); +} + +/* * clientCacheHit should only be called until the HTTP reply headers * have been parsed. Normally this should be a single call, but * it might take more than one. As soon as we have the headers, @@ -2115,7 +2163,7 @@ * clientProcessMiss. */ static void -clientCacheHit(void *data, mem_node_ref ref, ssize_t size) +clientCacheHit(void *data, HttpReply * rep) { clientHttpRequest *http = data; StoreEntry *e = http->entry; @@ -2123,32 +2171,31 @@ request_t *r = http->request; int is_modified = -1; int stale; - debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); http->flags.hit = 0; if (http->entry == NULL) { - stmemNodeUnref(&ref); debug(33, 3) ("clientCacheHit: request aborted\n"); return; - } else if (size <= 0) { + } else if (!rep) { /* swap in failure */ - stmemNodeUnref(&ref); debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); http->log_type = LOG_TCP_SWAPFAIL_MISS; clientProcessMiss(http); return; } - assert(size > 0); mem = e->mem_obj; + debug(33, 3) ("clientCacheHit: %s = %d\n", http->uri, rep->sline.status); assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); - if (!memHaveHeaders(mem)) { - debug(33, 1) ("clientCacheHit: No reply headers in '%s'?\n", e->mem_obj->url); - stmemNodeUnref(&ref); + + if (r->store_url && mem->store_url == NULL && strcmp(r->store_url, mem->url) != 0) { + debug(33, 1) ("clientCacheHit: request has store_url '%s'; mem object in hit doesn't and doesn't match the url '%s'!\n", r->store_url, mem->url); clientProcessMiss(http); return; - } - if (strcmp(mem->url, urlCanonical(r)) != 0) { - debug(33, 1) ("clientCacheHit: URL mismatch '%s' != '%s'?\n", e->mem_obj->url, urlCanonical(r)); - stmemNodeUnref(&ref); + } else if (r->store_url && strcmp(mem->store_url, r->store_url) != 0) { + debug(33, 1) ("clientCacheHit: store URL mismatch '%s' != '%s'?\n", mem->store_url, r->store_url); + clientProcessMiss(http); + return; + } else if ((!r->store_url) && (!mem->store_url) && strcmp(mem->url, urlCanonical(r)) != 0) { + debug(33, 1) ("clientCacheHit: (store url '%s'); URL mismatch '%s' != '%s'?\n", r->store_url, e->mem_obj->url, urlCanonical(r)); clientProcessMiss(http); return; } @@ -2184,7 +2231,6 @@ * so we only get here once. (it also takes care of cancelling loops) */ debug(33, 2) ("clientProcessHit: Vary detected!\n"); - stmemNodeUnref(&ref); return; } case VARY_RESTART: @@ -2194,13 +2240,11 @@ safe_free(r->vary_hdr); safe_free(r->vary_headers); clientProcessRequest(http); - stmemNodeUnref(&ref); return; case VARY_CANCEL: /* varyEvaluateMatch found a object loop. Process as miss */ debug(33, 1) ("clientProcessHit: Vary object loop!\n"); clientProcessMiss(http); - stmemNodeUnref(&ref); return; } if (r->method == METHOD_PURGE) { @@ -2209,7 +2253,6 @@ http->sc = NULL; storeUnlockObject(e); clientPurgeRequest(http); - stmemNodeUnref(&ref); return; } http->flags.hit = 1; @@ -2220,11 +2263,9 @@ #endif ) { http->log_type = LOG_TCP_NEGATIVE_HIT; - clientSendMoreHeaderData(data, stmemNodeRef(&ref), size); - stmemNodeUnref(&ref); + clientSendHeaders(data, rep); } else { http->log_type = LOG_TCP_MISS; - stmemNodeUnref(&ref); clientProcessMiss(http); } return; @@ -2242,7 +2283,6 @@ * Query the origin to see what should be done. */ http->log_type = LOG_TCP_MISS; - stmemNodeUnref(&ref); clientProcessMiss(http); return; } @@ -2255,7 +2295,6 @@ debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", mem->reply->sline.status); http->log_type = LOG_TCP_MISS; - stmemNodeUnref(&ref); clientProcessMiss(http); return; } @@ -2290,6 +2329,10 @@ debug(33, 2) ("clientProcessHit: refresh_stale HIT\n"); http->log_type = LOG_TCP_STALE_HIT; stale = 0; + } else if (stale == -2 && e->mem_obj->refresh_timestamp + e->mem_obj->stale_while_revalidate >= squid_curtime) { + debug(33, 2) ("clientProcessHit: stale-while-revalidate HIT\n"); + http->log_type = LOG_TCP_STALE_HIT; + stale = 0; } else if (stale && http->flags.internal) { debug(33, 2) ("clientProcessHit: internal HIT\n"); stale = 0; @@ -2297,7 +2340,13 @@ debug(33, 2) ("clientProcessHit: offline HIT\n"); http->log_type = LOG_TCP_OFFLINE_HIT; stale = 0; + } else if (stale == -2) { + debug(33, 2) ("clientProcessHit: stale-while-revalidate needs revalidation\n"); + clientAsyncRefresh(http); + http->log_type = LOG_TCP_STALE_HIT; + stale = 0; } + http->is_modified = is_modified; if (stale) { debug(33, 5) ("clientCacheHit: in refreshCheck() block\n"); /* @@ -2318,12 +2367,20 @@ */ http->log_type = LOG_TCP_CLIENT_REFRESH_MISS; clientProcessMiss(http); - } else { - clientProcessExpired(http); + return; } - stmemNodeUnref(&ref); + clientRefreshCheck(http); return; } + clientProcessHit(http); +} + +static void +clientProcessHit(clientHttpRequest * http) +{ + int is_modified = http->is_modified; + StoreEntry *e = http->entry; + if (is_modified == 0) { time_t timestamp = e->timestamp; MemBuf mb = httpPacked304Reply(e->mem_obj->reply); @@ -2343,18 +2400,18 @@ storeAppend(e, mb.buf, mb.size); memBufClean(&mb); storeComplete(e); - stmemNodeUnref(&ref); return; } /* * plain ol' cache hit */ + if (EBIT_TEST(e->flags, REFRESH_FAILURE)) + http->log_type = LOG_TCP_NEGATIVE_HIT; if (e->store_status != STORE_OK) http->log_type = LOG_TCP_MISS; else if (http->log_type == LOG_TCP_HIT && e->mem_status == IN_MEMORY) http->log_type = LOG_TCP_MEM_HIT; - clientSendMoreHeaderData(data, stmemNodeRef(&ref), size); - stmemNodeUnref(&ref); + clientSendHeaders(http, e->mem_obj->reply); } /* put terminating boundary for multiparts */ @@ -2612,49 +2669,32 @@ * such, writes processed message to the client's socket */ static void -clientSendMoreHeaderData(void *data, mem_node_ref ref, ssize_t size) +clientSendHeaders(void *data, HttpReply * rep) { //const char *buf = ref.node->data + ref.offset; clientHttpRequest *http = data; StoreEntry *entry = http->entry; ConnStateData *conn = http->conn; int fd = conn->fd; - HttpReply *rep = NULL; - debug(33, 5) ("clientSendMoreHeaderData: %s, %d bytes\n", http->uri, (int) size); - assert(size <= SM_PAGE_SIZE); assert(http->request != NULL); dlinkDelete(&http->active, &ClientActiveRequests); dlinkAdd(http, &http->active, &ClientActiveRequests); - debug(33, 5) ("clientSendMoreHeaderData: FD %d '%s', out.offset=%ld \n", - fd, storeUrl(entry), (long int) http->out.offset); + debug(33, 5) ("clientSendMoreHeaderData: FD %d '%s'\n", fd, storeUrl(entry)); assert(conn->reqs.head != NULL); if (DLINK_HEAD(conn->reqs) != http) { /* there is another object in progress, defer this one */ debug(33, 2) ("clientSendMoreHeaderData: Deferring %s\n", storeUrl(entry)); - stmemNodeUnref(&ref); return; } else if (http->request->flags.reset_tcp) { comm_reset_close(fd); - stmemNodeUnref(&ref); - return; - } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* call clientWriteComplete so the client socket gets closed */ - stmemNodeUnref(&ref); - clientWriteComplete(fd, NULL, 0, COMM_OK, http); - return; - } else if (size < 0) { - /* call clientWriteComplete so the client socket gets closed */ - clientWriteComplete(fd, NULL, 0, COMM_OK, http); - stmemNodeUnref(&ref); return; - } else if (size == 0) { + } else if (!rep) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - stmemNodeUnref(&ref); return; } assert(http->out.offset == 0); - rep = http->reply = clientBuildReply(http); + rep = http->reply = clientCloneReply(http, rep); if (!rep) { ErrorState *err = errorCon(ERR_INVALID_RESP, HTTP_BAD_GATEWAY, http->orig_request); storeClientUnregister(http->sc, http->entry, http); @@ -2663,7 +2703,6 @@ http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); errorAppendEntry(http->entry, err); - stmemNodeUnref(&ref); return; } clientMaxBodySize(http->request, http, rep); @@ -2676,7 +2715,6 @@ http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); errorAppendEntry(http->entry, err); - stmemNodeUnref(&ref); return; } /* @@ -2691,7 +2729,6 @@ */ http->range_iter.prefix_size = rep->hdr_sz; debug(33, 3) ("clientSendMoreHeaderData: %d bytes of headers\n", rep->hdr_sz); - stmemNodeUnref(&ref); clientHttpLocationRewriteCheck(http); } @@ -2956,7 +2993,7 @@ * If we are here, then store_status == STORE_OK and it * seems we have a HEAD repsponse which is missing the * empty end-of-headers line (home.mira.net, phttpd/0.99.72 - * does this). Because clientBuildReply() fails we just + * does this). Because clientCloneReply() fails we just * call this reply a body, set the done_copying flag and * continue... */ @@ -3047,11 +3084,8 @@ if (0 == storeClientCopyPending(http->sc, entry, http)) { if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, - clientSendMoreHeaderData, + storeClientCopyHeaders(http->sc, entry, + clientSendHeaders, http); } } @@ -3357,10 +3391,7 @@ #if DELAY_POOLS delaySetStoreClient(http->sc, delayClient(http)); #endif - storeClientRef(http->sc, http->entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, http->entry, clientCacheHit, http); } else { --- /dev/null Wed Nov 28 01:20:24 2007 +++ squid/src/client_side_rewrite.c Wed Nov 28 01:20:25 2007 @@ -0,0 +1,154 @@ + +/* + * $Id: client_side_rewrite.c,v 1.2.4.2 2007/11/27 08:12:27 adri Exp $ + * + * DEBUG: section 33 Client-side Routines - URL Rewriter + * AUTHOR: Duane Wessels; Adrian Chadd + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + + +/* Local functions */ + +void +clientRedirectAccessCheckDone(int answer, void *data) +{ + clientHttpRequest *http = data; + http->acl_checklist = NULL; + if (answer == ACCESS_ALLOWED) + redirectStart(http, clientRedirectDone, http); + else + clientRedirectDone(http, NULL); +} + +void +clientRedirectStart(clientHttpRequest * http) +{ + debug(33, 5) ("clientRedirectStart: '%s'\n", http->uri); + if (Config.Program.url_rewrite.command == NULL) { + clientRedirectDone(http, NULL); + return; + } + if (Config.accessList.url_rewrite) { + http->acl_checklist = clientAclChecklistCreate(Config.accessList.url_rewrite, http); + aclNBCheck(http->acl_checklist, clientRedirectAccessCheckDone, http); + } else { + redirectStart(http, clientRedirectDone, http); + } +} + +void +clientRedirectDone(void *data, char *result) +{ + clientHttpRequest *http = data; + request_t *new_request = NULL; + request_t *old_request = http->request; + const char *urlgroup = http->conn->port->urlgroup; + debug(33, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri, + result ? result : "NULL"); + assert(http->redirect_state == REDIRECT_PENDING); + http->redirect_state = REDIRECT_DONE; + if (result) { + http_status status; + if (*result == '!') { + char *t; + if ((t = strchr(result + 1, '!')) != NULL) { + urlgroup = result + 1; + *t++ = '\0'; + result = t; + } else { + debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); + } + } + status = (http_status) atoi(result); + if (status == HTTP_MOVED_PERMANENTLY + || status == HTTP_MOVED_TEMPORARILY + || status == HTTP_SEE_OTHER + || status == HTTP_TEMPORARY_REDIRECT) { + char *t = result; + if ((t = strchr(result, ':')) != NULL) { + http->redirect.status = status; + http->redirect.location = xstrdup(t + 1); + goto redirect_parsed; + } else { + debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); + } + } else if (strcmp(result, http->uri)) + new_request = urlParse(old_request->method, result); + } + redirect_parsed: + if (new_request) { + safe_free(http->uri); + http->uri = xstrdup(urlCanonical(new_request)); + new_request->http_ver = old_request->http_ver; + httpHeaderAppend(&new_request->header, &old_request->header); + new_request->client_addr = old_request->client_addr; + new_request->client_port = old_request->client_port; +#if FOLLOW_X_FORWARDED_FOR + new_request->indirect_client_addr = old_request->indirect_client_addr; +#endif /* FOLLOW_X_FORWARDED_FOR */ + new_request->my_addr = old_request->my_addr; + new_request->my_port = old_request->my_port; + new_request->flags = old_request->flags; + new_request->flags.redirected = 1; + if (old_request->auth_user_request) { + new_request->auth_user_request = old_request->auth_user_request; + authenticateAuthUserRequestLock(new_request->auth_user_request); + } + if (old_request->body_reader) { + new_request->body_reader = old_request->body_reader; + new_request->body_reader_data = old_request->body_reader_data; + old_request->body_reader = NULL; + old_request->body_reader_data = NULL; + } + new_request->content_length = old_request->content_length; + if (strBuf(old_request->extacl_log)) + new_request->extacl_log = stringDup(&old_request->extacl_log); + if (old_request->extacl_user) + new_request->extacl_user = xstrdup(old_request->extacl_user); + if (old_request->extacl_passwd) + new_request->extacl_passwd = xstrdup(old_request->extacl_passwd); + requestUnlink(old_request); + http->request = requestLink(new_request); + } else { + /* Don't mess with urlgroup on internal request */ + if (old_request->flags.internal) + urlgroup = NULL; + } + safe_free(http->request->urlgroup); /* only paranoia. should not happen */ + if (urlgroup && *urlgroup) + http->request->urlgroup = xstrdup(urlgroup); + clientInterpretRequestHeaders(http); + /* XXX This really should become a ref-counted string type pointer, not a copy! */ + fd_note(http->conn->fd, http->uri); + + clientStoreURLRewriteStart(http); +} --- /dev/null Wed Nov 28 01:20:24 2007 +++ squid/src/client_side_storeurl_rewrite.c Wed Nov 28 01:20:25 2007 @@ -0,0 +1,92 @@ + +/* + * $Id: client_side_storeurl_rewrite.c,v 1.3.2.2 2007/11/27 08:12:27 adri Exp $ + * + * DEBUG: section 85 Client-side Routines - Store URL Rewriter + * AUTHOR: Duane Wessels; Adrian Chadd + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + + +/* Local functions */ + +void +clientStoreURLRewriteAccessCheckDone(int answer, void *data) +{ + clientHttpRequest *http = data; + http->acl_checklist = NULL; + if (answer == ACCESS_ALLOWED) + storeurlStart(http, clientStoreURLRewriteDone, http); + else + clientStoreURLRewriteDone(http, NULL); +} + +void +clientStoreURLRewriteStart(clientHttpRequest * http) +{ + debug(85, 5) ("clientStoreURLRewriteStart: '%s'\n", http->uri); + if (Config.Program.store_rewrite.command == NULL) { + clientStoreURLRewriteDone(http, NULL); + return; + } + if (Config.accessList.storeurl_rewrite) { + http->acl_checklist = clientAclChecklistCreate(Config.accessList.storeurl_rewrite, http); + aclNBCheck(http->acl_checklist, clientStoreURLRewriteAccessCheckDone, http); + } else { + storeurlStart(http, clientStoreURLRewriteDone, http); + } +} + +void +clientStoreURLRewriteDone(void *data, char *result) +{ + clientHttpRequest *http = data; + + debug(85, 3) ("clientStoreURLRewriteDone: '%s' result=%s\n", http->uri, + result ? result : "NULL"); +#if 0 + assert(http->redirect_state == REDIRECT_PENDING); + http->redirect_state = REDIRECT_DONE; +#endif + + if (result) { + http->request->store_url = xstrdup(result); + debug(85, 3) ("Rewrote to %s\n", http->request->store_url); + /* XXX is this actually the right spot to do this? How about revalidation? */ + //storeEntrySetStoreUrl(http->entry, result); + } + /* This is the final part of the rewrite chain - this should be broken out! */ + clientInterpretRequestHeaders(http); +#if HEADERS_LOG + headersLog(0, 1, request->method, request); +#endif + clientAccessCheck2(http); +} Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.62 retrieving revision 1.62.2.1 diff -u -r1.62 -r1.62.2.1 --- squid/src/enums.h 24 Sep 2007 13:52:08 -0000 1.62 +++ squid/src/enums.h 27 Nov 2007 08:12:27 -0000 1.62.2.1 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.62 2007/09/24 13:52:08 squidadm Exp $ + * $Id: enums.h,v 1.62.2.1 2007/11/27 08:12:27 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -52,6 +52,8 @@ LOG_TCP_REDIRECT, #endif LOG_TCP_STALE_HIT, + LOG_TCP_ASYNC_HIT, + LOG_TCP_ASYNC_MISS, LOG_UDP_HIT, LOG_UDP_MISS, LOG_UDP_DENIED, @@ -525,7 +527,7 @@ ENTRY_REVALIDATE, DELAY_SENDING, RELEASE_REQUEST, - REFRESH_REQUEST, + REFRESH_FAILURE, ENTRY_CACHABLE, ENTRY_DISPATCHED, KEY_PRIVATE, @@ -654,6 +656,7 @@ STORE_META_VARY_HEADERS, /* Stores Vary request headers */ STORE_META_STD_LFS, /* standard metadata in lfs format */ STORE_META_OBJSIZE, /* object size, if its known */ + STORE_META_STOREURL, /* the store url, if different to the normal URL */ STORE_META_END }; Index: squid/src/errorpage.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/errorpage.c,v retrieving revision 1.37 retrieving revision 1.37.6.1 diff -u -r1.37 -r1.37.6.1 --- squid/src/errorpage.c 26 Feb 2007 09:51:31 -0000 1.37 +++ squid/src/errorpage.c 27 Nov 2007 08:12:27 -0000 1.37.6.1 @@ -1,6 +1,6 @@ /* - * $Id: errorpage.c,v 1.37 2007/02/26 09:51:31 squidadm Exp $ + * $Id: errorpage.c,v 1.37.6.1 2007/11/27 08:12:27 adri Exp $ * * DEBUG: section 4 Error Generation * AUTHOR: Duane Wessels @@ -324,7 +324,7 @@ authenticateFixHeader(rep, err->auth_user_request, err->request, 0, 1); httpReplySwapOut(rep, entry); EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); - storeNegativeCache(entry); + storeExpireNow(entry); storeReleaseRequest(entry); storeBufferFlush(entry); storeComplete(entry); Index: squid/src/fqdncache.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fqdncache.c,v retrieving revision 1.22 retrieving revision 1.22.2.1 diff -u -r1.22 -r1.22.2.1 --- squid/src/fqdncache.c 17 Sep 2007 09:51:44 -0000 1.22 +++ squid/src/fqdncache.c 27 Nov 2007 08:12:27 -0000 1.22.2.1 @@ -1,6 +1,6 @@ /* - * $Id: fqdncache.c,v 1.22 2007/09/17 09:51:44 squidadm Exp $ + * $Id: fqdncache.c,v 1.22.2.1 2007/11/27 08:12:27 adri Exp $ * * DEBUG: section 35 FQDN Cache * AUTHOR: Harvest Derived @@ -316,7 +316,7 @@ f->error_message = xstrdup("No PTR record"); return f; } - if (ttl == 0 || ttl > Config.positiveDnsTtl) + if (ttl > Config.positiveDnsTtl) ttl = Config.positiveDnsTtl; if (ttl < Config.negativeDnsTtl) ttl = Config.negativeDnsTtl; Index: squid/src/helper.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/helper.c,v retrieving revision 1.43 retrieving revision 1.43.6.1 diff -u -r1.43 -r1.43.6.1 --- squid/src/helper.c 1 Apr 2007 15:08:17 -0000 1.43 +++ squid/src/helper.c 27 Nov 2007 08:12:27 -0000 1.43.6.1 @@ -1,6 +1,6 @@ /* - * $Id: helper.c,v 1.43 2007/04/01 15:08:17 squidadm Exp $ + * $Id: helper.c,v 1.43.6.1 2007/11/27 08:12:27 adri Exp $ * * DEBUG: section 84 Helper process maintenance * AUTHOR: Harvest Derived? @@ -1026,7 +1026,7 @@ { helper *hlp = srv->parent; helper_request **ptr = NULL; - int slot; + int slot = -1; if (!cbdataValid(r->data)) { debug(84, 1) ("helperDispatch: invalid callback data\n"); helperRequestFree(r); @@ -1057,8 +1057,8 @@ helperDispatch_done, /* Handler */ srv); } - debug(84, 5) ("helperDispatch: Request sent to %s #%d, %d bytes\n", - hlp->id_name, srv->index + 1, (int) strlen(r->buf)); + debug(84, 5) ("helperDispatch: Request sent to %s #%d[%d], %d bytes\n", + hlp->id_name, srv->index + 1, slot, (int) strlen(r->buf)); srv->stats.uses++; hlp->stats.requests++; } Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.63 retrieving revision 1.63.2.1 diff -u -r1.63 -r1.63.2.1 --- squid/src/http.c 3 Sep 2007 04:49:47 -0000 1.63 +++ squid/src/http.c 27 Nov 2007 08:12:28 -0000 1.63.2.1 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.63 2007/09/03 04:49:47 squidadm Exp $ + * $Id: http.c,v 1.63.2.1 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -131,6 +131,8 @@ storeNegativeCache(entry); if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) storeSetPublicKey(entry); + if (entry->expires <= squid_curtime) + storeRelease(entry); } static void @@ -432,7 +434,7 @@ debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr.buf); httpState->reply_hdr_state += 2; httpState->chunk_size = -1; /* Terminated by EOF */ - memBufClean(&httpState->reply_hdr); + httpState->reply_hdr.size = old_size; httpBuildVersion(&reply->sline.version, 0, 9); reply->sline.status = HTTP_INVALID_HEADER; ctx_exit(ctx); @@ -506,8 +508,8 @@ stringClean(&tr); if (httpState->flags.chunked && reply->content_length >= 0) { /* Can't have a content-length in chunked encoding */ - reply->sline.status = HTTP_INVALID_HEADER; - return done; + reply->content_length = -1; + httpHeaderDelById(&reply->header, HDR_CONTENT_LENGTH); } } if (!httpState->flags.chunked) { @@ -530,6 +532,8 @@ if (strBuf(httpState->orig_request->vary_encoding)) entry->mem_obj->vary_encoding = xstrdup(strBuf(httpState->orig_request->vary_encoding)); } + if (entry->mem_obj->old_entry) + EBIT_CLR(entry->mem_obj->old_entry->flags, REFRESH_FAILURE); switch (httpCachableReply(httpState)) { case 1: httpMakePublic(entry); @@ -538,10 +542,7 @@ httpMakePrivate(entry); break; case -1: - if (Config.negativeTtl > 0) - httpCacheNegatively(entry); - else - httpMakePrivate(entry); + httpCacheNegatively(entry); break; default: assert(0); @@ -955,6 +956,8 @@ httpHeaderPutTime(&reply->header, HDR_DATE, squid_curtime); mb = httpReplyPack(reply); storeAppend(entry, mb.buf, mb.size); + storeAppend(entry, httpState->reply_hdr.buf, httpState->reply_hdr.size); + memBufClean(&httpState->reply_hdr); httpReplyReset(reply); httpReplyParse(reply, mb.buf, mb.size); memBufClean(&mb); @@ -1145,13 +1148,14 @@ /* append unless we added our own; * note: at most one client's ims header can pass through */ if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE)) - httpHeaderAddClone(hdr_out, e); + if (!Config.onoff.ignore_ims_on_miss || !orig_request->flags.cachable || orig_request->flags.auth) + httpHeaderAddClone(hdr_out, e); break; case HDR_IF_NONE_MATCH: - /* append unless we added our own; - * note: at most one client's ims header can pass through */ + /* append unless ignore_ims_on_miss is in effect */ if (!httpHeaderHas(hdr_out, HDR_IF_NONE_MATCH)) - httpHeaderAddClone(hdr_out, e); + if (!Config.onoff.ignore_ims_on_miss || !orig_request->flags.cachable || orig_request->flags.auth) + httpHeaderAddClone(hdr_out, e); break; case HDR_MAX_FORWARDS: if (orig_request->method == METHOD_TRACE) { @@ -1253,10 +1257,10 @@ /* Special mode, to pass the username to the upstream cache */ char loginbuf[256]; const char *username = "-"; - if (orig_request->auth_user_request) - username = authenticateUserRequestUsername(orig_request->auth_user_request); - else if (orig_request->extacl_user) + if (orig_request->extacl_user) username = orig_request->extacl_user; + else if (orig_request->auth_user_request) + username = authenticateUserRequestUsername(orig_request->auth_user_request); snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, orig_request->peer_login + 1); httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s", base64_encode(loginbuf)); Index: squid/src/ipcache.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ipcache.c,v retrieving revision 1.25 retrieving revision 1.25.2.1 diff -u -r1.25 -r1.25.2.1 --- squid/src/ipcache.c 17 Sep 2007 09:51:44 -0000 1.25 +++ squid/src/ipcache.c 27 Nov 2007 08:12:28 -0000 1.25.2.1 @@ -1,6 +1,6 @@ /* - * $Id: ipcache.c,v 1.25 2007/09/17 09:51:44 squidadm Exp $ + * $Id: ipcache.c,v 1.25.2.1 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 14 IP Cache * AUTHOR: Harvest Derived @@ -369,7 +369,7 @@ i->addrs.count = (unsigned char) na; else i->addrs.count = 255; - if (ttl == 0 || ttl > Config.positiveDnsTtl) + if (ttl > Config.positiveDnsTtl) ttl = Config.positiveDnsTtl; if (ttl < Config.negativeDnsTtl) ttl = Config.negativeDnsTtl; Index: squid/src/main.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/main.c,v retrieving revision 1.80 retrieving revision 1.80.2.1 diff -u -r1.80 -r1.80.2.1 --- squid/src/main.c 24 Sep 2007 13:52:09 -0000 1.80 +++ squid/src/main.c 27 Nov 2007 08:12:28 -0000 1.80.2.1 @@ -1,6 +1,6 @@ /* - * $Id: main.c,v 1.80 2007/09/24 13:52:09 squidadm Exp $ + * $Id: main.c,v 1.80.2.1 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -396,9 +396,11 @@ idnsShutdown(); #endif redirectShutdown(); + storeurlShutdown(); locationRewriteShutdown(); authenticateShutdown(); externalAclShutdown(); + refreshCheckShutdown(); storeDirCloseSwapLogs(); storeLogClose(); accessLogClose(); @@ -425,9 +427,11 @@ idnsInit(); #endif redirectInit(); + storeurlInit(); locationRewriteInit(); authenticateInit(&Config.authConfig); externalAclInit(); + refreshCheckInit(); #if USE_WCCP wccpInit(); #endif @@ -463,6 +467,7 @@ locationRewriteShutdown(); authenticateShutdown(); externalAclShutdown(); + refreshCheckShutdown(); _db_rotate_log(); /* cache.log */ storeDirWriteCleanLogs(1); storeDirSync(); /* Flush pending I/O ops */ @@ -478,9 +483,11 @@ dnsInit(); #endif redirectInit(); + storeurlInit(); locationRewriteInit(); authenticateInit(&Config.authConfig); externalAclInit(); + refreshCheckInit(); } static void @@ -576,10 +583,12 @@ idnsInit(); #endif redirectInit(); + storeurlInit(); locationRewriteInit(); errorMapInit(); authenticateInit(&Config.authConfig); externalAclInit(); + refreshCheckInit(); useragentOpenLog(); refererOpenLog(); httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ @@ -715,7 +724,9 @@ #endif mainParseOptions(argc, argv); +#if HAVE_SYSLOG && defined(LOG_LOCAL4) openlog(appname, LOG_PID | LOG_NDELAY | LOG_CONS, syslog_facility); +#endif #if defined(USE_WIN32_SERVICE) && defined(_SQUID_WIN32_) if (opt_install_service) { @@ -1087,6 +1098,7 @@ #endif redirectShutdown(); externalAclShutdown(); + refreshCheckShutdown(); locationRewriteShutdown(); icpConnectionClose(); #if USE_HTCP Index: squid/src/mib.txt =================================================================== RCS file: /cvsroot/squid-sf//squid/src/mib.txt,v retrieving revision 1.9 retrieving revision 1.9.4.1 diff -u -r1.9 -r1.9.4.1 --- squid/src/mib.txt 27 Aug 2007 13:53:41 -0000 1.9 +++ squid/src/mib.txt 27 Nov 2007 08:12:28 -0000 1.9.4.1 @@ -2,7 +2,7 @@ SQUID-MIB DEFINITIONS ::= BEGIN -- --- $Id: mib.txt,v 1.9 2007/08/27 13:53:41 squidadm Exp $ +-- $Id: mib.txt,v 1.9.4.1 2007/11/27 08:12:28 adri Exp $ -- IMPORTS @@ -715,7 +715,7 @@ MAX-ACCESS not-accessible STATUS current DESCRIPTION - " An entry in cachePeerTable " + " cache_peer indexed by IP address " INDEX { cachePeerAddr } ::= { cachePeerTable 1 } @@ -724,7 +724,7 @@ MAX-ACCESS not-accessible STATUS current DESCRIPTION - " An entry in cachePeerTable " + " cache_peer indexed by position in squid.conf " INDEX { cachePeerIndex } ::= { cachePeerTable 2 } Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.146.2.3 retrieving revision 1.146.2.4 diff -u -r1.146.2.3 -r1.146.2.4 --- squid/src/protos.h 27 Sep 2007 15:09:17 -0000 1.146.2.3 +++ squid/src/protos.h 27 Nov 2007 08:12:28 -0000 1.146.2.4 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.146.2.3 2007/09/27 15:09:17 adri Exp $ + * $Id: protos.h,v 1.146.2.4 2007/11/27 08:12:28 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -747,6 +747,10 @@ extern void redirectInit(void); extern void redirectShutdown(void); +extern void storeurlStart(clientHttpRequest *, RH *, void *); +extern void storeurlInit(void); +extern void storeurlShutdown(void); + extern void locationRewriteStart(HttpReply *, clientHttpRequest *, RH *, void *); extern void locationRewriteInit(void); extern void locationRewriteShutdown(void); @@ -796,9 +800,11 @@ extern int refreshIsCachable(const StoreEntry *); extern int refreshCheckHTTP(const StoreEntry *, request_t *); extern int refreshCheckHTTPStale(const StoreEntry *, request_t *); +extern int refreshCheckStaleOK(const StoreEntry *, request_t *); extern int refreshCheckICP(const StoreEntry *, request_t *); extern int refreshCheckHTCP(const StoreEntry *, request_t *); extern int refreshCheckDigest(const StoreEntry *, time_t delta); +extern refresh_cc refreshCC(const StoreEntry *, request_t *); extern time_t getMaxAge(const char *url); extern void refreshInit(void); extern const refresh_t *refreshLimits(const char *url); @@ -911,6 +917,7 @@ * store.c */ extern StoreEntry *new_StoreEntry(int, const char *); +extern void storeEntrySetStoreUrl(StoreEntry * e, const char *store_url); extern StoreEntry *storeGet(const cache_key *); extern StoreEntry *storeGetPublic(const char *uri, const method_t method); extern StoreEntry *storeGetPublicByRequest(request_t * request); @@ -924,6 +931,7 @@ extern void storeLockObjectDebug(StoreEntry *, const char *file, const int line); extern void storeRelease(StoreEntry *); extern int storeUnlockObjectDebug(StoreEntry *, const char *file, const int line); +extern const char *storeLookupUrl(const StoreEntry * e); #define storeLockObject(a) storeLockObjectDebug(a, __FILE__, __LINE__); #define storeUnlockObject(a) storeUnlockObjectDebug(a, __FILE__, __LINE__); extern EVH storeMaintainSwapSpace; @@ -970,6 +978,7 @@ void storeResumeRead(StoreEntry *); void storeResetDefer(StoreEntry *); extern int memHaveHeaders(const MemObject * mem); +extern void storeUpdate(StoreEntry * e, request_t *); /* store_modules.c */ @@ -1091,6 +1100,7 @@ extern store_client *storeClientRegister(StoreEntry * e, void *data); extern void storeClientCopy(store_client *, StoreEntry *, squid_off_t, squid_off_t, size_t, char *, STCB *, void *); extern void storeClientRef(store_client *, StoreEntry *, squid_off_t, squid_off_t, size_t, STNCB *, void *); +extern void storeClientCopyHeaders(store_client *, StoreEntry *, STHCB *, void *); extern int storeClientCopyPending(store_client *, StoreEntry * e, void *data); extern int storeClientUnregister(store_client * sc, StoreEntry * e, void *data); extern squid_off_t storeLowestMemReaderOffset(const StoreEntry * entry); @@ -1415,6 +1425,15 @@ const char *externalAclMessage(external_acl_entry * entry); +/* refresh_check.c */ +extern void parse_refreshCheckHelper(refresh_check_helper **); +extern void dump_refreshCheckHelper(StoreEntry * sentry, const char *name, const refresh_check_helper *); +extern void free_refreshCheckHelper(refresh_check_helper **); +extern void refreshCheckSubmit(StoreEntry * entry, REFRESHCHECK * callback, void *data); +extern void refreshCheckInit(void); +extern void refreshCheckConfigure(void); +extern void refreshCheckShutdown(void); + #if USE_WCCPv2 extern void parse_wccp2_service(void *v); extern void free_wccp2_service(void *v); @@ -1448,5 +1467,20 @@ extern const char *xinet_ntoa(const struct in_addr addr); +/* client_side.c */ +extern aclCheck_t *clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http); +extern void clientInterpretRequestHeaders(clientHttpRequest * http); +extern void clientAccessCheck2(void *data); + +/* client_side_redirect.c */ +extern void clientRedirectAccessCheckDone(int answer, void *data); +extern void clientRedirectStart(clientHttpRequest * http); +extern void clientRedirectDone(void *data, char *result); + +/* client_side_storeurl_rewrite.c */ +extern void clientStoreURLRewriteAccessCheckDone(int answer, void *data); +extern void clientStoreURLRewriteStart(clientHttpRequest * http); +extern void clientStoreURLRewriteDone(void *data, char *result); + #endif /* SQUID_PROTOS_H */ Index: squid/src/redirect.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/redirect.c,v retrieving revision 1.16 retrieving revision 1.16.10.1 diff -u -r1.16 -r1.16.10.1 --- squid/src/redirect.c 1 Feb 2007 09:51:50 -0000 1.16 +++ squid/src/redirect.c 27 Nov 2007 08:12:28 -0000 1.16.10.1 @@ -1,9 +1,9 @@ /* - * $Id: redirect.c,v 1.16 2007/02/01 09:51:50 squidadm Exp $ + * $Id: redirect.c,v 1.16.10.1 2007/11/27 08:12:28 adri Exp $ * - * DEBUG: section 61 Redirector - * AUTHOR: Duane Wessels + * DEBUG: section 85 Store URL Redirector + * AUTHOR: Adrian Chadd; based on redirect.c by Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -98,6 +98,8 @@ const char *fqdn; char *urlgroup = conn->port->urlgroup; char buf[8192]; + char claddr[20]; + char myaddr[20]; assert(http); assert(handler); debug(61, 5) ("redirectStart: '%s'\n", http->uri); @@ -130,13 +132,17 @@ cbdataLock(r->data); if ((fqdn = fqdncache_gethostbyaddr(r->client_addr, 0)) == NULL) fqdn = dash_str; - snprintf(buf, 8191, "%s %s/%s %s %s %s", + xstrncpy(claddr, inet_ntoa(r->client_addr), 20); + xstrncpy(myaddr, inet_ntoa(http->request->my_addr), 20); + snprintf(buf, 8191, "%s %s/%s %s %s %s myip=%s myport=%d", r->orig_url, - inet_ntoa(r->client_addr), + claddr, fqdn, r->client_ident[0] ? rfc1738_escape(r->client_ident) : dash_str, r->method_s, - urlgroup ? urlgroup : "-"); + urlgroup ? urlgroup : "-", + myaddr, + http->request->my_port); debug(61, 6) ("redirectStart: sending '%s' to the helper\n", buf); strcat(buf, "\n"); helperSubmit(redirectors, buf, redirectHandleReply, r); Index: squid/src/refresh.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/refresh.c,v retrieving revision 1.15 retrieving revision 1.15.6.1 diff -u -r1.15 -r1.15.6.1 --- squid/src/refresh.c 24 May 2007 01:51:31 -0000 1.15 +++ squid/src/refresh.c 27 Nov 2007 08:12:28 -0000 1.15.6.1 @@ -1,6 +1,6 @@ /* - * $Id: refresh.c,v 1.15 2007/05/24 01:51:31 squidadm Exp $ + * $Id: refresh.c,v 1.15.6.1 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 22 Refresh Calculation * AUTHOR: Harvest Derived @@ -81,6 +81,8 @@ STALE_MAX_RULE, STALE_LMFACTOR_RULE, STALE_WITHIN_DELTA, + STALE_ASYNC_REFRESH, + STALE_MAX_STALE, STALE_DEFAULT = 299 }; @@ -128,6 +130,30 @@ return NULL; } +refresh_cc +refreshCC(const StoreEntry * entry, request_t * request) +{ + const refresh_t *R; + const char *uri = NULL; + refresh_cc cc; + cc.negative_ttl = Config.negativeTtl; + cc.max_stale = Config.maxStale; + if (entry->mem_obj) + uri = entry->mem_obj->url; + else if (request) + uri = urlCanonical(request); + + R = uri ? refreshLimits(uri) : refreshUncompiledPattern("."); + + if (R && R->negative_ttl >= 0) + cc.negative_ttl = R->negative_ttl; + + if (R && R->max_stale >= 0) + cc.max_stale = R->max_stale; + + return cc; +} + /* * Calculate how stale the response is (or will be at the check_time). * Staleness calculation is based on the following: (1) response @@ -308,6 +334,20 @@ * At this point the response is stale, unless one of * the override options kicks in. */ + if (entry->mem_obj) { + int stale_while_revalidate = R->stale_while_revalidate; + if (staleness < stale_while_revalidate) { + debug(22, 3) ("stale-while-revalidate: age=%d, staleness=%d, stale_while_revalidate=%d\n", (int) age, staleness, stale_while_revalidate); + entry->mem_obj->stale_while_revalidate = stale_while_revalidate; + return STALE_ASYNC_REFRESH; + } + } { + int max_stale = Config.maxStale; + if (R->max_stale >= 0) + max_stale = R->max_stale; + if (max_stale >= 0 && staleness >= max_stale) + return STALE_MAX_STALE; + } if (delta < 0 && staleness + delta < 0) { return STALE_WITHIN_DELTA; } @@ -382,10 +422,25 @@ int reason = refreshCheck(entry, request, -Config.refresh_stale_window); if (reason == STALE_WITHIN_DELTA) return -1; + if (reason == STALE_ASYNC_REFRESH) + return -2; + if (reason == STALE_MAX_STALE) + return 3; return (reason < 200) ? 0 : 1; } int +refreshCheckStaleOK(const StoreEntry * entry, request_t * request) +{ + int reason = refreshCheck(entry, request, 0); + if (reason == STALE_MUST_REVALIDATE) + return 0; + if (reason == STALE_MAX_STALE) + return 0; + return 1; +} + +int refreshCheckICP(const StoreEntry * entry, request_t * request) { int reason = refreshCheck(entry, request, 30); @@ -472,6 +527,12 @@ "Stale: refresh_pattern max age rule"); refreshCountsStatsEntry(STALE_LMFACTOR_RULE, "Stale: refresh_pattern last-mod factor percentage"); + refreshCountsStatsEntry(STALE_WITHIN_DELTA, + "Staĺe: but within a certain delta"); + refreshCountsStatsEntry(STALE_ASYNC_REFRESH, + "Stale: suitable for an async refresh"); + refreshCountsStatsEntry(STALE_MAX_STALE, + "Stale: max-stale"); refreshCountsStatsEntry(STALE_DEFAULT, "Stale: by default"); --- /dev/null Wed Nov 28 01:20:24 2007 +++ squid/src/refresh_check.c Wed Nov 28 01:20:25 2007 @@ -0,0 +1,550 @@ + +/* + * $Id: refresh_check.c,v 1.2.4.2 2007/11/27 08:12:28 adri Exp $ + * + * DEBUG: section 84 Refresh Check Helper + * AUTHOR: Henrik Nordstrom + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * The contents of this file is Copyright (C) 2007 by Henrik Nordstrom, + * Sweden, unless otherwise is indicated in the specific function. The + * author gives his full permission to include this file into the Squid + * software product under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + * Partially based on the External ACL implementation. + */ + +#include "squid.h" + +#ifndef DEFAULT_REFRESH_CHECK_CHILDREN +#define DEFAULT_REFRESH_CHECK_CHILDREN 5 +#endif + +/****************************************************************** + * external_refresh_check parser + */ +typedef struct _refresh_check_format refresh_check_format; + +struct _refresh_check_helper { + refresh_check_format *format; + wordlist *cmdline; + int children; + int concurrency; + helper *helper; + dlink_list queue; +}; + +struct _refresh_check_format { + enum { + REFRESH_CHECK_UNKNOWN, + REFRESH_CHECK_URI, + REFRESH_CHECK_RESP_HEADER, + REFRESH_CHECK_RESP_HEADER_MEMBER, + REFRESH_CHECK_RESP_HEADER_ID, + REFRESH_CHECK_RESP_HEADER_ID_MEMBER, + REFRESH_CHECK_AGE, + REFRESH_CHECK_END + } type; + refresh_check_format *next; + char *header; + char *member; + char separator; + http_hdr_type header_id; +}; + +/* FIXME: These are not really cbdata, but it is an easy way + * to get them pooled, refcounted, accounted and freed properly... + */ +CBDATA_TYPE(refresh_check_helper); +CBDATA_TYPE(refresh_check_format); + +static void +free_refresh_check_format(void *data) +{ + refresh_check_format *p = data; + safe_free(p->header); +} + +static void +free_refresh_check_helper(void *data) +{ + refresh_check_helper *p = data; + while (p->format) { + refresh_check_format *f = p->format; + p->format = f->next; + cbdataFree(f); + } + wordlistDestroy(&p->cmdline); + if (p->helper) { + helperShutdown(p->helper); + helperFree(p->helper); + p->helper = NULL; + } +} + +void +parse_refreshCheckHelper(refresh_check_helper ** ptr) +{ + refresh_check_helper *a; + char *token; + refresh_check_format **p; + + if (*ptr) + self_destruct(); + + CBDATA_INIT_TYPE_FREECB(refresh_check_helper, free_refresh_check_helper); + CBDATA_INIT_TYPE_FREECB(refresh_check_format, free_refresh_check_format); + + a = cbdataAlloc(refresh_check_helper); + + a->children = DEFAULT_REFRESH_CHECK_CHILDREN; + + /* Parse options */ + while ((token = strtok(NULL, w_space)) != NULL) { + if (strncmp(token, "children=", 9) == 0) { + a->children = atoi(token + 9); + } else if (strncmp(token, "concurrency=", 12) == 0) { + a->concurrency = atoi(token + 12); + } else { + break; + } + } + + /* Parse format */ + p = &a->format; + while (token) { + refresh_check_format *format; + + /* stop on first non-format token found */ + if (*token != '%') + break; + + format = cbdataAlloc(refresh_check_format); + + if (strncmp(token, "%RES{", 5) == 0) { + /* header format */ + char *header, *member, *end; + header = token + 5; + end = strchr(header, '}'); + /* cut away the terminating } */ + if (end && strlen(end) == 1) + *end = '\0'; + else + self_destruct(); + + member = strchr(header, ':'); + if (member) { + /* Split in header and member */ + *member++ = '\0'; + if (!xisalnum(*member)) + format->separator = *member++; + else + format->separator = ','; + format->member = xstrdup(member); + format->type = REFRESH_CHECK_RESP_HEADER_MEMBER; + } else { + format->type = REFRESH_CHECK_RESP_HEADER; + } + format->header = xstrdup(header); + format->header_id = httpHeaderIdByNameDef(header, strlen(header)); + if (format->header_id != -1) { + if (member) + format->type = REFRESH_CHECK_RESP_HEADER_ID_MEMBER; + else + format->type = REFRESH_CHECK_RESP_HEADER_ID; + } + } else if (strcmp(token, "%URI") == 0) + format->type = REFRESH_CHECK_URI; + else if (strcmp(token, "%URL") == 0) + format->type = REFRESH_CHECK_URI; + else if (strcmp(token, "%CACHE_URI") == 0) + format->type = REFRESH_CHECK_URI; + else if (strcmp(token, "%AGE") == 0) + format->type = REFRESH_CHECK_AGE; + else { + self_destruct(); + } + *p = format; + p = &format->next; + token = strtok(NULL, w_space); + } + /* There must be at least one format token */ + if (!a->format) + self_destruct(); + + /* helper */ + if (!token) + self_destruct(); + wordlistAdd(&a->cmdline, token); + + /* arguments */ + parse_wordlist(&a->cmdline); + + *ptr = a; +} + +void +dump_refreshCheckHelper(StoreEntry * sentry, const char *name, const refresh_check_helper * list) +{ + const refresh_check_helper *node = list; + const refresh_check_format *format; + const wordlist *word; + if (node) { + storeAppendPrintf(sentry, "%s", name); + if (node->children != DEFAULT_REFRESH_CHECK_CHILDREN) + storeAppendPrintf(sentry, " children=%d", node->children); + if (node->concurrency) + storeAppendPrintf(sentry, " concurrency=%d", node->concurrency); + for (format = node->format; format; format = format->next) { + switch (format->type) { + case REFRESH_CHECK_RESP_HEADER: + case REFRESH_CHECK_RESP_HEADER_ID: + storeAppendPrintf(sentry, " %%{%s}", format->header); + break; + case REFRESH_CHECK_RESP_HEADER_MEMBER: + case REFRESH_CHECK_RESP_HEADER_ID_MEMBER: + storeAppendPrintf(sentry, " %%{%s:%s}", format->header, format->member); + break; +#define DUMP_REFRESH_CHECK_TYPE(a) \ + case REFRESH_CHECK_##a: \ + storeAppendPrintf(sentry, " %%%s", #a); \ + break + DUMP_REFRESH_CHECK_TYPE(URI); + DUMP_REFRESH_CHECK_TYPE(AGE); + case REFRESH_CHECK_UNKNOWN: + case REFRESH_CHECK_END: + fatal("unknown refresh_check format error"); + break; + } + } + for (word = node->cmdline; word; word = word->next) + storeAppendPrintf(sentry, " %s", word->key); + storeAppendPrintf(sentry, "\n"); + } +} + +void +free_refreshCheckHelper(refresh_check_helper ** list) +{ + while (*list) { + refresh_check_helper *node = *list; + *list = NULL; + cbdataFree(node); + } +} + +/****************************************************************** + * refresh_check runtime + */ + +static inline int +refreshCheckOverload(refresh_check_helper * def) +{ + return def->helper->stats.queue_size > def->helper->n_running; +} + +static char * +makeRefreshCheckRequest(StoreEntry * entry, refresh_check_format * format) +{ + static MemBuf mb = MemBufNULL; + int first = 1; + HttpReply *reply; + String sb = StringNull; + + if (!entry->mem_obj) + return NULL; + + reply = entry->mem_obj->reply; + memBufReset(&mb); + for (; format; format = format->next) { + char buf[256]; + const char *str = NULL; + const char *quoted; + switch (format->type) { + case REFRESH_CHECK_URI: + str = entry->mem_obj->url; + break; + case REFRESH_CHECK_AGE: + snprintf(buf, sizeof(buf), "%ld", (long int) (squid_curtime - entry->timestamp)); + str = buf; + break; + case REFRESH_CHECK_RESP_HEADER: + sb = httpHeaderGetByName(&reply->header, format->header); + str = strBuf(sb); + break; + case REFRESH_CHECK_RESP_HEADER_ID: + sb = httpHeaderGetStrOrList(&reply->header, format->header_id); + str = strBuf(sb); + break; + case REFRESH_CHECK_RESP_HEADER_MEMBER: + sb = httpHeaderGetByNameListMember(&reply->header, format->header, format->member, format->separator); + str = strBuf(sb); + break; + case REFRESH_CHECK_RESP_HEADER_ID_MEMBER: + sb = httpHeaderGetListMember(&reply->header, format->header_id, format->member, format->separator); + str = strBuf(sb); + break; + + case REFRESH_CHECK_UNKNOWN: + case REFRESH_CHECK_END: + fatal("unknown refresh_check_program format error"); + break; + } + if (!str || !*str) + str = "-"; + if (!first) + memBufAppend(&mb, " ", 1); + quoted = rfc1738_escape(str); + memBufAppend(&mb, quoted, strlen(quoted)); + stringClean(&sb); + first = 0; + } + return mb.buf; +} + +/****************************************************************** + * refresh_check requests + */ + +typedef struct _refreshCheckState refreshCheckState; +struct _refreshCheckState { + REFRESHCHECK *callback; + void *callback_data; + StoreEntry *entry; + refresh_check_helper *def; + dlink_node list; + refreshCheckState *queue; +}; + +CBDATA_TYPE(refreshCheckState); +static void +free_refreshCheckState(void *data) +{ + refreshCheckState *state = data; + storeUnlockObject(state->entry); + cbdataUnlock(state->callback_data); + cbdataUnlock(state->def); +} + +/* + * The helper program receives queries on stdin, one + * per line, and must return the result on on stdout + */ +static void +refreshCheckHandleReply(void *data, char *reply) +{ + refreshCheckState *state = data; + refreshCheckState *next; + int freshness = -1; + char *log = NULL; + MemBuf hdrs = MemBufNULL; + + + debug(84, 2) ("refreshCheckHandleReply: reply=\"%s\"\n", reply); + + if (reply) { + char *t = NULL; + char *token = strwordtok(reply, &t); + if (token && strcmp(token, "FRESH") == 0) + freshness = 0; + else if (token && strcmp(token, "OK") == 0) + freshness = 0; + + while ((token = strwordtok(NULL, &t))) { + char *value = strchr(token, '='); + if (value) { + *value++ = '\0'; /* terminate the token, and move up to the value */ + rfc1738_unescape(value); + if (strcmp(token, "freshness") == 0) + freshness = atoi(value); + else if (strcmp(token, "log") == 0) + log = value; + else if (strncmp(token, "res{", 4) == 0) { + char *header, *t; + header = token + 4; + t = strrchr(header, '}'); + if (!t) + continue; + *t = '\0'; + if (!hdrs.buf) + memBufDefInit(&hdrs); + memBufPrintf(&hdrs, "%s: %s\r\n", header, value); + } + } + } + } + if (freshness >= 0) { + if (hdrs.size) { + HttpReply *rep = httpReplyCreate(); + httpHeaderParse(&rep->header, hdrs.buf, hdrs.buf + hdrs.size); + httpReplyUpdateOnNotModified(state->entry->mem_obj->reply, rep); + storeTimestampsSet(state->entry); + if (!httpHeaderHas(&rep->header, HDR_DATE)) { + state->entry->timestamp = squid_curtime; + state->entry->expires = squid_curtime + freshness; + } else if (freshness) { + state->entry->expires = squid_curtime + freshness; + } + httpReplyDestroy(rep); + storeUpdate(state->entry, NULL); + } else { + state->entry->timestamp = squid_curtime; + state->entry->expires = squid_curtime + freshness; + } + } + if (hdrs.buf) + memBufClean(&hdrs); + dlinkDelete(&state->list, &state->def->queue); + do { + cbdataUnlock(state->def); + state->def = NULL; + + if (state->callback && cbdataValid(state->callback_data)) + state->callback(state->callback_data, freshness >= 0, log); + cbdataUnlock(state->callback_data); + state->callback_data = NULL; + + next = state->queue; + cbdataFree(state); + state = next; + } while (state); +} + +void +refreshCheckSubmit(StoreEntry * entry, REFRESHCHECK * callback, void *callback_data) +{ + MemBuf buf; + const char *key; + refresh_check_helper *def = Config.Program.refresh_check; + refreshCheckState *state; + dlink_node *node; + refreshCheckState *oldstate = NULL; + + if (!def) { + callback(callback_data, 0, NULL); + return; + } + key = makeRefreshCheckRequest(entry, def->format); + if (!key) { + callback(callback_data, 0, NULL); + return; + } + debug(84, 2) ("refreshCheckLookup: for '%s'\n", key); + + /* Check for a pending lookup to hook into */ + for (node = def->queue.head; node; node = node->next) { + refreshCheckState *oldstatetmp = node->data; + if (entry == oldstatetmp->entry) { + oldstate = oldstatetmp; + break; + } + } + + state = cbdataAlloc(refreshCheckState); + state->def = def; + cbdataLock(state->def); + state->entry = entry; + storeLockObject(entry); + state->callback = callback; + state->callback_data = callback_data; + cbdataLock(state->callback_data); + if (oldstate) { + /* Hook into pending lookup */ + state->queue = oldstate->queue; + oldstate->queue = state; + } else { + /* No pending lookup found. Sumbit to helper */ + /* Check for queue overload */ + if (refreshCheckOverload(def)) { + debug(84, 1) ("refreshCheckLookup: queue overload\n"); + cbdataFree(state); + callback(callback_data, 0, "Overload"); + return; + } + /* Send it off to the helper */ + memBufDefInit(&buf); + memBufPrintf(&buf, "%s\n", key); + helperSubmit(def->helper, buf.buf, refreshCheckHandleReply, state); + dlinkAdd(state, &state->list, &def->queue); + memBufClean(&buf); + } +} + +static void +refreshCheckStats(StoreEntry * sentry) +{ + refresh_check_helper *p = Config.Program.refresh_check; + + if (p) { + storeAppendPrintf(sentry, "Refresh Check helper statistics:\n"); + helperStats(sentry, p->helper); + storeAppendPrintf(sentry, "\n"); + } +} +void +refreshCheckConfigure(void) +{ + refresh_check_helper *p = Config.Program.refresh_check; + if (p) { + requirePathnameExists("external_refresh_check", p->cmdline->key); + } +} +void +refreshCheckInit(void) +{ + static int firstTimeInit = 1; + refresh_check_helper *p = Config.Program.refresh_check; + + if (p) { + if (!p->helper) + p->helper = helperCreate("external_refresh_check"); + p->helper->cmdline = p->cmdline; + p->helper->n_to_start = p->children; + p->helper->concurrency = p->concurrency; + p->helper->ipc_type = IPC_STREAM; + helperOpenServers(p->helper); + if (firstTimeInit) { + firstTimeInit = 0; + cachemgrRegister("refresh_check", + "External ACL stats", + refreshCheckStats, 0, 1); + } + CBDATA_INIT_TYPE_FREECB(refreshCheckState, free_refreshCheckState); + } +} + +void +refreshCheckShutdown(void) +{ + refresh_check_helper *p = Config.Program.refresh_check; + if (p) { + helperShutdown(p->helper); + } +} Index: squid/src/stat.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stat.c,v retrieving revision 1.42.2.2 retrieving revision 1.42.2.3 diff -u -r1.42.2.2 -r1.42.2.3 --- squid/src/stat.c 26 Sep 2007 07:33:50 -0000 1.42.2.2 +++ squid/src/stat.c 27 Nov 2007 08:12:28 -0000 1.42.2.3 @@ -1,6 +1,6 @@ /* - * $Id: stat.c,v 1.42.2.2 2007/09/26 07:33:50 adri Exp $ + * $Id: stat.c,v 1.42.2.3 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -204,8 +204,8 @@ strcat(buf, "DELAY_SENDING,"); if (EBIT_TEST(flags, RELEASE_REQUEST)) strcat(buf, "RELEASE_REQUEST,"); - if (EBIT_TEST(flags, REFRESH_REQUEST)) - strcat(buf, "REFRESH_REQUEST,"); + if (EBIT_TEST(flags, REFRESH_FAILURE)) + strcat(buf, "REFRESH_FAILURE,"); if (EBIT_TEST(flags, ENTRY_CACHABLE)) strcat(buf, "CACHABLE,"); if (EBIT_TEST(flags, ENTRY_DISPATCHED)) @@ -251,6 +251,8 @@ if (mem) memBufPrintf(mb, "\t%s %s\n", RequestMethods[mem->method].str, mem->url); + if (mem && mem->store_url) + memBufPrintf(mb, "\tStore lookup URL: %s\n", mem->store_url); memBufPrintf(mb, "\t%s\n", describeStatuses(e)); memBufPrintf(mb, "\t%s\n", storeEntryFlags(e)); memBufPrintf(mb, "\t%s\n", describeTimestamps(e)); @@ -421,7 +423,15 @@ LOCAL_ARRAY(char, buf, 32); if (f->type != FD_SOCKET) return null_string; - snprintf(buf, 32, "%s.%d", f->ipaddr, (int) f->remote_port); + if (*f->ipaddr) + snprintf(buf, 32, "%s.%d", f->ipaddr, (int) f->remote_port); + else { + if (f->local_addr.s_addr != any_addr.s_addr) { + snprintf(buf, 32, "%s.%d", inet_ntoa(f->local_addr), (int) f->local_port); + } else { + snprintf(buf, 32, "*.%d", (int) f->local_port); + } + } return buf; } Index: squid/src/store.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store.c,v retrieving revision 1.49 retrieving revision 1.49.4.1 diff -u -r1.49 -r1.49.4.1 --- squid/src/store.c 29 Aug 2007 21:52:57 -0000 1.49 +++ squid/src/store.c 27 Nov 2007 08:12:29 -0000 1.49.4.1 @@ -1,6 +1,6 @@ /* - * $Id: store.c,v 1.49 2007/08/29 21:52:57 squidadm Exp $ + * $Id: store.c,v 1.49.4.1 2007/11/27 08:12:29 adri Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -128,7 +128,9 @@ int memHaveHeaders(const MemObject * mem) { - if (mem->reply == NULL) + if (!mem) + return 0; + if (!mem->reply) return 0; if (mem->reply->pstate != psParsed) return 0; @@ -150,6 +152,16 @@ return e; } +void +storeEntrySetStoreUrl(StoreEntry * e, const char *store_url) +{ + /* XXX eww, another strdup! */ + if (!e->mem_obj) + return; + safe_free(e->mem_obj->store_url); + e->mem_obj->store_url = xstrdup(store_url); +} + static void destroy_MemObject(StoreEntry * e) { @@ -175,6 +187,10 @@ storeUnlockObject(mem->ims_entry); mem->ims_entry = NULL; } + if (mem->old_entry) { + storeUnlockObject(mem->old_entry); + mem->old_entry = NULL; + } httpReplyDestroy(mem->reply); requestUnlink(mem->request); mem->request = NULL; @@ -1010,7 +1026,7 @@ * to record the new variance key */ safe_free(request->vary_headers); /* free old "bad" variance key */ - pe = storeGetPublic(mem->url, mem->method); + pe = storeGetPublic(storeLookupUrl(e), mem->method); if (pe) storeRelease(pe); } @@ -1042,7 +1058,7 @@ stringClean(&vary); } } else { - newkey = storeKeyPublic(mem->url, mem->method); + newkey = storeKeyPublic(storeLookupUrl(e), mem->method); } if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); @@ -1051,7 +1067,7 @@ if (mem->request) newkey = storeKeyPublicByRequest(mem->request); else - newkey = storeKeyPublic(mem->url, mem->method); + newkey = storeKeyPublic(storeLookupUrl(e), mem->method); } if (e->hash.key) storeHashDelete(e); @@ -1314,7 +1330,11 @@ if (e->mem_obj->request) e->mem_obj->request->hier.store_complete_stop = current_time; #endif - e->mem_obj->refresh_timestamp = e->timestamp; + e->mem_obj->refresh_timestamp = 0; + if (e->mem_obj->old_entry) { + if (e->mem_obj->old_entry->mem_obj) + e->mem_obj->old_entry->mem_obj->refresh_timestamp = 0; + } /* * We used to call InvokeHandlers, then storeSwapOut. However, * Madhukar Reddy reported that @@ -1338,7 +1358,7 @@ assert(mem != NULL); debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->hash.key)); storeLockObject(e); /* lock while aborting */ - storeNegativeCache(e); + storeExpireNow(e); storeReleaseRequest(e); EBIT_SET(e->flags, ENTRY_ABORTED); storeSetMemStatus(e, NOT_IN_MEMORY); @@ -1627,7 +1647,34 @@ void storeNegativeCache(StoreEntry * e) { - e->expires = squid_curtime + Config.negativeTtl; + StoreEntry *oe = e->mem_obj->old_entry; + time_t expires = e->expires; + refresh_cc cc = refreshCC(e, e->mem_obj->request); + if (expires == -1) + expires = squid_curtime + cc.negative_ttl; + if (oe && !EBIT_TEST(oe->flags, KEY_PRIVATE) && !EBIT_TEST(oe->flags, ENTRY_REVALIDATE)) { + if (cc.max_stale >= 0) { + time_t max_expires; + storeTimestampsSet(oe); + max_expires = oe->expires + cc.max_stale; + /* Bail out if beyond the stale-if-error staleness limit */ + if (max_expires <= squid_curtime) + goto cache_error_response; + /* Limit expiry time to stale-if-error/max_stale */ + if (expires > max_expires) + expires = max_expires; + } + /* Block the new error from getting cached */ + EBIT_CLR(e->flags, ENTRY_CACHABLE); + /* And negatively cache the old one */ + if (oe->expires < expires) + oe->expires = expires; + EBIT_SET(oe->flags, REFRESH_FAILURE); + return; + } + cache_error_response: + if (e->expires < expires) + e->expires = expires; EBIT_SET(e->flags, ENTRY_NEGCACHED); } @@ -1819,6 +1866,19 @@ return e->mem_obj->url; } +const char * +storeLookupUrl(const StoreEntry * e) +{ + if (e == NULL) + return "[null_entry]"; + else if (e->mem_obj == NULL) + return "[null_mem_obj]"; + else if (e->mem_obj->store_url) + return e->mem_obj->store_url; + else + return e->mem_obj->url; +} + void storeCreateMemObject(StoreEntry * e, const char *url) { Index: squid/src/store_client.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_client.c,v retrieving revision 1.25.10.4 retrieving revision 1.25.10.5 diff -u -r1.25.10.4 -r1.25.10.5 --- squid/src/store_client.c 27 Sep 2007 15:09:17 -0000 1.25.10.4 +++ squid/src/store_client.c 27 Nov 2007 08:12:29 -0000 1.25.10.5 @@ -1,6 +1,6 @@ /* - * $Id: store_client.c,v 1.25.10.4 2007/09/27 15:09:17 adri Exp $ + * $Id: store_client.c,v 1.25.10.5 2007/11/27 08:12:29 adri Exp $ * * DEBUG: section 20 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -533,6 +533,18 @@ break; } break; + case STORE_META_STOREURL: + if (NULL == mem->store_url) + mem->store_url = xstrdup(t->value); + else if (0 == strcasecmp(mem->store_url, t->value)) + (void) 0; /* a match! */ + else { + debug(20, 1) ("storeClientReadHeader: store URL mismatch\n"); + debug(20, 1) ("\t{%s} != {%s}\n", (char *) t->value, mem->store_url); + swap_object_ok = 0; + break; + } + break; case STORE_META_OBJSIZE: break; case STORE_META_STD: Index: squid/src/store_key_md5.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_key_md5.c,v retrieving revision 1.10 retrieving revision 1.10.10.1 diff -u -r1.10 -r1.10.10.1 --- squid/src/store_key_md5.c 21 Jan 2007 14:04:07 -0000 1.10 +++ squid/src/store_key_md5.c 27 Nov 2007 08:12:29 -0000 1.10.10.1 @@ -1,6 +1,6 @@ /* - * $Id: store_key_md5.c,v 1.10 2007/01/21 14:04:07 squidadm Exp $ + * $Id: store_key_md5.c,v 1.10.10.1 2007/11/27 08:12:29 adri Exp $ * * DEBUG: section 20 Storage Manager MD5 Cache Keys * AUTHOR: Duane Wessels @@ -131,7 +131,14 @@ { static cache_key digest[MD5_DIGEST_CHARS]; unsigned char m = (unsigned char) method; - const char *url = urlCanonical(request); + const char *url; + + if (request->store_url) { + url = request->store_url; + } else { + url = urlCanonical(request); + } + MD5_CTX M; MD5Init(&M); MD5Update(&M, &m, sizeof(m)); --- /dev/null Wed Nov 28 01:20:24 2007 +++ squid/src/store_rewrite.c Wed Nov 28 01:20:25 2007 @@ -0,0 +1,183 @@ + +/* + * $Id: store_rewrite.c,v 1.1.4.2 2007/11/27 08:12:29 adri Exp $ + * + * DEBUG: section 61 Redirector + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +typedef struct { + void *data; + char *orig_url; + struct in_addr client_addr; + const char *client_ident; + const char *method_s; + RH *handler; +} storeurlStateData; + +static HLPCB storeurlHandleReply; +static void storeurlStateFree(storeurlStateData * r); +static helper *storeurlors = NULL; +static OBJH storeurlStats; +static int n_bypassed = 0; +CBDATA_TYPE(storeurlStateData); + +static void +storeurlHandleReply(void *data, char *reply) +{ + storeurlStateData *r = data; + int valid; + char *t; + debug(61, 5) ("storeurlHandleRead: {%s}\n", reply ? reply : ""); + if (reply) { + if ((t = strchr(reply, ' '))) + *t = '\0'; + if (*reply == '\0') + reply = NULL; + } + valid = cbdataValid(r->data); + cbdataUnlock(r->data); + if (valid) + r->handler(r->data, reply); + storeurlStateFree(r); +} + +static void +storeurlStateFree(storeurlStateData * r) +{ + safe_free(r->orig_url); + cbdataFree(r); +} + +static void +storeurlStats(StoreEntry * sentry) +{ + storeAppendPrintf(sentry, "Redirector Statistics:\n"); + helperStats(sentry, storeurlors); + if (Config.onoff.storeurl_bypass) + storeAppendPrintf(sentry, "\nNumber of requests bypassed " + "because all store url bypassers were busy: %d\n", n_bypassed); +} + +/**** PUBLIC FUNCTIONS ****/ + +void +storeurlStart(clientHttpRequest * http, RH * handler, void *data) +{ + ConnStateData *conn = http->conn; + storeurlStateData *r = NULL; + const char *fqdn; + char *urlgroup = conn->port->urlgroup; + char buf[8192]; + char claddr[20]; + char myaddr[20]; + assert(http); + assert(handler); + debug(61, 5) ("storeurlStart: '%s'\n", http->uri); + if (Config.onoff.storeurl_bypass && storeurlors->stats.queue_size) { + /* Skip storeurlor if there is one request queued */ + n_bypassed++; + handler(data, NULL); + return; + } + r = cbdataAlloc(storeurlStateData); + r->orig_url = xstrdup(http->uri); + r->client_addr = conn->log_addr; + r->client_ident = NULL; + if (http->request->auth_user_request) + r->client_ident = authenticateUserRequestUsername(http->request->auth_user_request); + else if (http->request->extacl_user) { + r->client_ident = http->request->extacl_user; + } + if (!r->client_ident && conn->rfc931[0]) + r->client_ident = conn->rfc931; +#if USE_SSL + if (!r->client_ident) + r->client_ident = sslGetUserEmail(fd_table[conn->fd].ssl); +#endif + if (!r->client_ident) + r->client_ident = dash_str; + r->method_s = RequestMethods[http->request->method].str; + r->handler = handler; + r->data = data; + cbdataLock(r->data); + if ((fqdn = fqdncache_gethostbyaddr(r->client_addr, 0)) == NULL) + fqdn = dash_str; + xstrncpy(claddr, inet_ntoa(r->client_addr), 20); + xstrncpy(myaddr, inet_ntoa(http->request->my_addr), 20); + snprintf(buf, 8191, "%s %s/%s %s %s %s myip=%s myport=%d", + r->orig_url, + claddr, + fqdn, + r->client_ident[0] ? rfc1738_escape(r->client_ident) : dash_str, + r->method_s, + urlgroup ? urlgroup : "-", + myaddr, + http->request->my_port); + debug(61, 6) ("storeurlStart: sending '%s' to the helper\n", buf); + strcat(buf, "\n"); + helperSubmit(storeurlors, buf, storeurlHandleReply, r); +} + +void +storeurlInit(void) +{ + static int init = 0; + if (!Config.Program.store_rewrite.command) + return; + if (storeurlors == NULL) + storeurlors = helperCreate("store_rewriter"); + storeurlors->cmdline = Config.Program.store_rewrite.command; + storeurlors->n_to_start = Config.Program.store_rewrite.children; + storeurlors->concurrency = Config.Program.store_rewrite.concurrency; + storeurlors->ipc_type = IPC_STREAM; + helperOpenServers(storeurlors); + if (!init) { + cachemgrRegister("store_rewriter", + "URL Rewriter Stats", + storeurlStats, 0, 1); + init = 1; + CBDATA_INIT_TYPE(storeurlStateData); + } +} + +void +storeurlShutdown(void) +{ + if (!storeurlors) + return; + helperShutdown(storeurlors); + if (!shutting_down) + return; + helperFree(storeurlors); + storeurlors = NULL; +} Index: squid/src/store_swapmeta.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_swapmeta.c,v retrieving revision 1.12 retrieving revision 1.12.22.1 diff -u -r1.12 -r1.12.22.1 --- squid/src/store_swapmeta.c 29 Jul 2006 14:53:20 -0000 1.12 +++ squid/src/store_swapmeta.c 27 Nov 2007 08:12:29 -0000 1.12.22.1 @@ -1,6 +1,6 @@ /* - * $Id: store_swapmeta.c,v 1.12 2006/07/29 14:53:20 squidadm Exp $ + * $Id: store_swapmeta.c,v 1.12.22.1 2007/11/27 08:12:29 adri Exp $ * * DEBUG: section 20 Storage Manager Swapfile Metadata * AUTHOR: Kostas Anagnostakis @@ -86,6 +86,8 @@ vary = e->mem_obj->vary_headers; if (vary) T = storeSwapTLVAdd(STORE_META_VARY_HEADERS, vary, strlen(vary) + 1, T); + if (e->mem_obj->store_url) + T = storeSwapTLVAdd(STORE_META_STOREURL, e->mem_obj->store_url, strlen(e->mem_obj->store_url) + 1, T); return TLV; } --- /dev/null Wed Nov 28 01:20:24 2007 +++ squid/src/store_update.c Wed Nov 28 01:20:25 2007 @@ -0,0 +1,187 @@ + +/* + * $Id: store_update.c,v 1.2.4.2 2007/11/27 08:12:29 adri Exp $ + * + * DEBUG: section 20 Storage Manager + * AUTHOR: Henrik Nordstrom + * + * Copyright (C) 2007 Henrik Nordstrom + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +/* Update the on-disk representation of the object to match + * current in-memory representation. This requires rewriting + * the object headers (bot meta and http) which isn't supported + * by the store layer today, so we rewrite the full object. + */ + +/* Local state */ +typedef struct { + StoreEntry *oldentry; + StoreEntry *newentry; + store_client *sc; + squid_off_t offset; + char buf[4096]; +} StoreUpdateState; + +CBDATA_TYPE(StoreUpdateState); + +static void +free_StoreUpdateState(void *data) +{ + StoreUpdateState *state = data; + if (state->sc) + storeClientUnregister(state->sc, state->oldentry, state); + if (state->oldentry) + storeUnlockObject(state->oldentry); + if (state->newentry) { + /* Set to NULL on completion, so we only get here in abnormal situations */ + storeRelease(state->newentry); + storeUnlockObject(state->newentry); + } +} + +static void +storeUpdateDone(StoreUpdateState * state) +{ + if (state->sc) { + store_client *sc = state->sc; + state->sc = NULL; + storeClientUnregister(sc, state->oldentry, state); + } + cbdataFree(state); +} +static void +storeUpdateAbort(void *data) +{ + StoreUpdateState *state = data; + storeUpdateDone(state); +} + +static void +storeUpdateCopy(void *data, char *buf, ssize_t size) +{ + StoreUpdateState *state = data; + + if (EBIT_TEST(state->newentry->flags, ENTRY_ABORTED)) { + debug(20, 1) ("storeUpdateCopy: Aborted at %d (%d)\n", (int) state->offset, (int) size); + /* the abort callback deals with the needed cleanup */ + return; + } + if (EBIT_TEST(state->newentry->flags, KEY_PRIVATE) && state->newentry->mem_obj->nclients == 0) { + debug(20, 2) ("storeUpdateCopy: Gone stale with no clients, skip copying of the rest\n"); + storeUpdateDone(state); + return; + } + if (size < 0) { + debug(20, 1) ("storeUpdateCopy: Error at %d (%d)\n", (int) state->offset, (int) size); + storeUpdateDone(state); + return; + } + if (size > 0) { + storeAppend(state->newentry, buf, size); + if (EBIT_TEST(state->newentry->flags, ENTRY_ABORTED)) { + debug(20, 1) ("storeUpdateCopy: Aborted on write at %d (%d)\n", (int) state->offset, (int) size); + return; + } + state->offset += size; + storeClientCopy(state->sc, state->oldentry, state->offset, state->offset, sizeof(state->buf), state->buf, storeUpdateCopy, state); + return; + } else { + storeComplete(state->newentry); + storeUnlockObject(state->newentry); + state->newentry = NULL; + storeUpdateDone(state); + } +} + +void +storeUpdate(StoreEntry * entry, request_t * request) +{ + StoreUpdateState *state; + request_flags flags = null_request_flags; + const char *vary; + + if (!request) + request = entry->mem_obj->request; + + if (EBIT_TEST(entry->flags, KEY_PRIVATE)) + return; /* Nothing to do here... */ + + if (!Config.onoff.update_headers) + return; /* Disabled */ + + CBDATA_INIT_TYPE_FREECB(StoreUpdateState, free_StoreUpdateState); + + if (entry->mem_obj) + entry->mem_obj->refresh_timestamp = 0; + state = cbdataAlloc(StoreUpdateState); + state->oldentry = entry; + storeLockObject(state->oldentry); + flags.cachable = 1; + state->newentry = storeCreateEntry(storeUrl(entry), flags, entry->mem_obj->method); + storeRegisterAbort(state->newentry, storeUpdateAbort, state); + state->sc = storeClientRegister(state->oldentry, state); + state->offset = entry->mem_obj->reply->hdr_sz; + storeBuffer(state->newentry); + httpReplySwapOut(httpReplyClone(entry->mem_obj->reply), state->newentry); + state->newentry->timestamp = entry->timestamp; + state->newentry->lastref = entry->lastref; + state->newentry->expires = entry->expires; + state->newentry->lastmod = entry->lastmod; + state->newentry->refcount = entry->refcount; + if (request) { + state->newentry->mem_obj->request = requestLink(request); + vary = httpMakeVaryMark(request, state->newentry->mem_obj->reply); + if (vary) { + state->newentry->mem_obj->vary_headers = xstrdup(vary); + if (strBuf(request->vary_encoding)) + entry->mem_obj->vary_encoding = xstrdup(strBuf(request->vary_encoding)); + } + } else { + if (entry->mem_obj->vary_headers) + state->newentry->mem_obj->vary_headers = xstrdup(entry->mem_obj->vary_headers); + if (entry->mem_obj->vary_encoding) + state->newentry->mem_obj->vary_encoding = xstrdup(entry->mem_obj->vary_encoding); + } + storeSetPublicKey(state->newentry); + storeBufferFlush(state->newentry); + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * the above storeBufferFlush() call could ABORT this entry, + * in that case, there's nothing for us to do. + */ + debug(20, 1) ("storeUpdate: Aborted on write\n"); + return; + } + storeClientCopy(state->sc, state->oldentry, state->offset, state->offset, sizeof(state->buf), state->buf, storeUpdateCopy, state); + return; +} Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.158.2.4 retrieving revision 1.158.2.5 diff -u -r1.158.2.4 -r1.158.2.5 --- squid/src/structs.h 26 Sep 2007 07:33:50 -0000 1.158.2.4 +++ squid/src/structs.h 27 Nov 2007 08:12:29 -0000 1.158.2.5 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.158.2.4 2007/09/26 07:33:50 adri Exp $ + * $Id: structs.h,v 1.158.2.5 2007/11/27 08:12:29 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -443,6 +443,7 @@ RemovalPolicySettings *replPolicy; RemovalPolicySettings *memPolicy; time_t negativeTtl; + time_t maxStale; time_t negativeDnsTtl; time_t positiveDnsTtl; time_t shutdownLifetime; @@ -549,7 +550,13 @@ wordlist *command; int children; int concurrency; + } store_rewrite; + struct { + wordlist *command; + int children; + int concurrency; } location_rewrite; + refresh_check_helper *refresh_check; #if USE_ICMP char *pinger; #endif @@ -652,6 +659,7 @@ int nonhierarchical_direct; int strip_query_terms; int redirector_bypass; + int storeurl_bypass; int ignore_unknown_nameservers; int client_pconns; int server_pconns; @@ -672,6 +680,7 @@ int global_internal_static; int httpd_suppress_version_string; int via; + int ignore_ims_on_miss; int check_hostnames; int allow_underscore; int cache_vary; @@ -680,6 +689,7 @@ int delay_pool_uses_indirect_client; int log_uses_indirect_client; #endif + int update_headers; } onoff; acl *aclList; struct { @@ -700,6 +710,7 @@ acl_access *identLookup; #endif acl_access *url_rewrite; + acl_access *storeurl_rewrite; acl_access *location_rewrite; acl_access *reply; acl_address *outgoing_address; @@ -1190,6 +1201,7 @@ HierarchyLogEntry hier; HttpReply *reply; request_t *request; + char *ext_refresh; }; struct _clientHttpRequest { @@ -1238,6 +1250,9 @@ dlink_node active; squid_off_t maxBodySize; mem_node_ref nr; + STHCB *header_callback; /* Temporarily here for storeClientCopyHeaders */ + StoreEntry *header_entry; /* Temporarily here for storeClientCopyHeaders */ + int is_modified; }; struct _ConnStateData { @@ -1698,6 +1713,7 @@ struct _MemObject { method_t method; char *url; + const char *store_url; mem_hdr data_hdr; squid_off_t inmem_hi; squid_off_t inmem_lo; @@ -1730,7 +1746,9 @@ const char *vary_headers; const char *vary_encoding; StoreEntry *ims_entry; + StoreEntry *old_entry; time_t refresh_timestamp; + time_t stale_while_revalidate; }; struct _StoreEntry { @@ -1885,6 +1903,7 @@ u_short port; String urlpath; char *canonical; + char *store_url; /* rewritten URL for store lookup/storage; if NULL use canonical */ int link_count; /* free when zero */ request_flags flags; HttpHdrCc *cache_control; @@ -1936,6 +1955,11 @@ cachemgr_passwd *next; }; +struct _refresh_cc { + int max_stale; + int negative_ttl; +}; + struct _refresh_t { const char *pattern; regex_t compiled_pattern; @@ -1955,6 +1979,9 @@ unsigned int ignore_auth:1; #endif } flags; + int max_stale; + int stale_while_revalidate; + int negative_ttl; }; struct _ErrorState { Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.43.2.2 retrieving revision 1.43.2.3 diff -u -r1.43.2.2 -r1.43.2.3 --- squid/src/typedefs.h 25 Sep 2007 00:29:28 -0000 1.43.2.2 +++ squid/src/typedefs.h 27 Nov 2007 08:12:29 -0000 1.43.2.3 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.43.2.2 2007/09/25 00:29:28 adri Exp $ + * $Id: typedefs.h,v 1.43.2.3 2007/11/27 08:12:29 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -188,6 +188,7 @@ typedef struct _AccessLogEntry AccessLogEntry; typedef struct _cachemgr_passwd cachemgr_passwd; typedef struct _refresh_t refresh_t; +typedef struct _refresh_cc refresh_cc; typedef struct _CommWriteStateData CommWriteStateData; typedef struct _ErrorState ErrorState; typedef struct _dlink_node dlink_node; @@ -289,6 +290,7 @@ typedef int QS(const void *, const void *); /* qsort */ typedef void STCB(void *, char *, ssize_t); /* store callback */ typedef void STNCB(void *, struct _mem_node_ref r, ssize_t); /* new store callback */ +typedef void STHCB(void *, HttpReply *); /* store callback */ typedef void STABH(void *); typedef void ERCB(int fd, void *, size_t); typedef void OBJH(StoreEntry *); @@ -429,4 +431,6 @@ typedef void LOGROTATE(Logfile *); typedef void LOGCLOSE(Logfile *); +typedef void REFRESHCHECK(void *data, int fresh, const char *log); +typedef struct _refresh_check_helper refresh_check_helper; #endif /* SQUID_TYPEDEFS_H */ Index: squid/src/fs/aufs/aiops.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/aiops.c,v retrieving revision 1.28 retrieving revision 1.28.18.1 diff -u -r1.28 -r1.28.18.1 --- squid/src/fs/aufs/aiops.c 23 Sep 2006 10:51:41 -0000 1.28 +++ squid/src/fs/aufs/aiops.c 27 Nov 2007 08:12:30 -0000 1.28.18.1 @@ -322,7 +322,7 @@ /* Create threads and get them to sit in their wait loop */ squidaio_thread_pool = memPoolCreate("aio_thread", sizeof(squidaio_thread_t)); if (squidaio_nthreads == 0) { - int j = 16; + int j = THREAD_FACTOR; for (i = 0; i < n_asyncufs_dirs; i++) { squidaio_nthreads += j; j = j * 2 / 3; @@ -338,7 +338,7 @@ #endif } if (squidaio_nthreads == 0) - squidaio_nthreads = 16; + squidaio_nthreads = THREAD_FACTOR; squidaio_magic1 = squidaio_nthreads * MAGIC1_FACTOR; squidaio_magic2 = squidaio_nthreads * MAGIC2_FACTOR; for (i = 0; i < squidaio_nthreads; i++) { Index: squid/src/fs/aufs/aiops_win32.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/aiops_win32.c,v retrieving revision 1.4 retrieving revision 1.4.28.1 diff -u -r1.4 -r1.4.28.1 --- squid/src/fs/aufs/aiops_win32.c 23 Sep 2006 10:51:41 -0000 1.4 +++ squid/src/fs/aufs/aiops_win32.c 27 Nov 2007 08:12:30 -0000 1.4.28.1 @@ -1,5 +1,5 @@ /* - * $Id: aiops_win32.c,v 1.4 2006/09/23 10:51:41 squidadm Exp $ + * $Id: aiops_win32.c,v 1.4.28.1 2007/11/27 08:12:30 adri Exp $ * * DEBUG: section 43 Windows AIOPS * AUTHOR: Stewart Forster @@ -322,7 +322,7 @@ /* Create threads and get them to sit in their wait loop */ squidaio_thread_pool = memPoolCreate("aio_thread", sizeof(squidaio_thread_t)); if (squidaio_nthreads == 0) { - int j = 16; + int j = THREAD_FACTOR; for (i = 0; i < n_asyncufs_dirs; i++) { squidaio_nthreads += j; j = j * 2 / 3; @@ -338,7 +338,7 @@ #endif } if (squidaio_nthreads == 0) - squidaio_nthreads = 16; + squidaio_nthreads = THREAD_FACTOR; squidaio_magic1 = squidaio_nthreads * MAGIC1_FACTOR; squidaio_magic2 = squidaio_nthreads * MAGIC2_FACTOR; for (i = 0; i < squidaio_nthreads; i++) { Index: squid/src/fs/aufs/store_dir_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_dir_aufs.c,v retrieving revision 1.50 retrieving revision 1.50.6.1 diff -u -r1.50 -r1.50.6.1 --- squid/src/fs/aufs/store_dir_aufs.c 1 Mar 2007 07:53:23 -0000 1.50 +++ squid/src/fs/aufs/store_dir_aufs.c 27 Nov 2007 08:12:30 -0000 1.50.6.1 @@ -1551,6 +1551,8 @@ break; /* no more objects */ removed++; storeRelease(e); + if (aioQueueSize() > MAGIC2) + break; } walker->Done(walker); debug(47, (removed ? 2 : 3)) ("storeAufsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",