--------------------- PatchSet 303 Date: 2000/05/14 23:22:19 Author: hno Branch: hno-devel Tag: (none) Log: Syncronized with modio Members: configure.in:1.1.1.3.6.7.2.6->1.1.1.3.6.7.2.7 doc/Programming-Guide/prog-guide.sgml:1.1.1.2.6.1.2.1->1.1.1.2.6.1.2.2 lib/rfc1035.c:1.1.1.1.10.2->1.1.1.1.10.2.2.1 src/Makefile.in:1.1.1.3.10.7.2.2->1.1.1.3.10.7.2.3 src/acl.c:1.1.1.3.10.5.2.2->1.1.1.3.10.5.2.3 src/asn.c:1.1.1.3.6.1.2.1->1.1.1.3.6.1.2.2 src/cache_cf.c:1.1.1.3.10.4.2.2->1.1.1.3.10.4.2.3 src/cf.data.pre:1.1.1.3.10.6.2.4->1.1.1.3.10.6.2.5 src/cf_gen.c:1.1.1.3.10.2.2.1->1.1.1.3.10.2.2.2 src/client_side.c:1.1.1.3.10.7.2.4->1.1.1.3.10.7.2.5 src/comm_select.c:1.1.1.3.10.2.2.2->1.1.1.3.10.2.2.3 src/delay_pools.c:1.1.1.3.10.3.2.1->1.1.1.3.10.3.2.2 src/disk.c:1.1.1.3.6.2->1.1.1.3.6.2.2.1 src/dns_internal.c:1.1.1.1.10.4.2.2->1.1.1.1.10.4.2.3 src/forward.c:1.1.1.3.10.3.2.1->1.1.1.3.10.3.2.2 src/globals.h:1.1.1.3.10.3->1.1.1.3.10.3.2.1 src/http.c:1.1.1.3.10.4->1.1.1.3.10.4.2.1 src/ipc.c:1.1.1.3.10.4->1.1.1.3.10.4.2.1 src/mem.c:1.1.1.3.10.3.2.1->1.1.1.3.10.3.2.2 src/neighbors.c:1.1.1.3.10.3->1.1.1.3.10.3.2.1 src/protos.h:1.1.1.3.10.8.2.4->1.1.1.3.10.8.2.5 src/repl_modules.sh:1.1->1.1.6.1 src/snmp_agent.c:1.1.1.3.6.2->1.1.1.3.6.2.2.1 src/squid.h:1.1.1.3.10.2.2.1->1.1.1.3.10.2.2.2 src/ssl.c:1.1.1.3.10.3.2.1->1.1.1.3.10.3.2.2 src/stat.c:1.1.1.3.10.4.2.1->1.1.1.3.10.4.2.2 src/store.c:1.1.1.3.10.8.2.3->1.1.1.3.10.8.2.4 src/store_client.c:1.1.1.3.10.4.2.2->1.1.1.3.10.4.2.3 src/store_digest.c:1.1.1.3.6.1.2.1->1.1.1.3.6.1.2.2 src/store_dir.c:1.1.1.3.10.6.2.1->1.1.1.3.10.6.2.2 src/store_heap_replacement.c:1.1.1.1.6.2->1.1.1.1.6.2.2.1(DEAD) src/store_io.c:1.1.1.1.10.6->1.1.1.1.10.6.2.1 src/store_log.c:1.1.1.2.10.2->1.1.1.2.10.2.2.1 src/store_swapin.c:1.1.1.3.10.4->1.1.1.3.10.4.2.1 src/store_swapout.c:1.1.1.3.10.5.2.1->1.1.1.3.10.5.2.2 src/structs.h:1.1.1.3.10.8.2.5->1.1.1.3.10.8.2.6 src/typedefs.h:1.1.1.3.10.6.2.1->1.1.1.3.10.6.2.2 src/url.c:1.1.1.3.10.2->1.1.1.3.10.2.2.1 src/fs/aufs/async_io.c:1.1.6.1.2.3->1.1.6.1.2.4 src/fs/aufs/store_asyncufs.h:1.1.6.3.2.1->1.1.6.3.2.2 src/fs/aufs/store_dir_aufs.c:1.1.6.6.2.2->1.1.6.6.2.3 src/fs/aufs/store_io_aufs.c:1.1.6.6.2.2->1.1.6.6.2.3 src/fs/coss/store_coss.h:1.1.4.4->1.1.4.4.2.1 src/fs/coss/store_dir_coss.c:1.1.4.6.2.1->1.1.4.6.2.2 src/fs/coss/store_io_coss.c:1.1.4.8.2.3->1.1.4.8.2.4 src/fs/diskd/diskd.c:1.1.6.1->1.1.6.2 src/fs/diskd/store_dir_diskd.c:1.1.6.1->1.1.6.2 src/fs/diskd/store_diskd.h:1.1.6.1->1.1.6.2 src/fs/diskd/store_io_diskd.c:1.1.6.1->1.1.6.2 src/fs/ufs/store_dir_ufs.c:1.1.6.6.2.2->1.1.6.6.2.3 src/fs/ufs/store_io_ufs.c:1.1.6.6.2.3->1.1.6.6.2.4 src/fs/ufs/store_ufs.h:1.1.6.3.2.1->1.1.6.3.2.2 src/repl/Makefile.in:1.1->1.1.6.1 src/repl/heap/Makefile.in:1.1->1.1.6.1 src/repl/heap/store_heap_replacement.c:1.1->1.1.6.1 src/repl/heap/store_heap_replacement.h:1.1->1.1.6.1 src/repl/heap/store_repl_heap.c:1.1->1.1.6.1 src/repl/lru/Makefile.in:1.1->1.1.6.1 src/repl/lru/store_repl_lru.c:1.1->1.1.6.1 Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.1.1.3.6.7.2.6 retrieving revision 1.1.1.3.6.7.2.7 diff -u -r1.1.1.3.6.7.2.6 -r1.1.1.3.6.7.2.7 --- squid/configure.in 8 May 2000 18:02:07 -0000 1.1.1.3.6.7.2.6 +++ squid/configure.in 14 May 2000 23:22:19 -0000 1.1.1.3.6.7.2.7 @@ -3,13 +3,13 @@ dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.1.1.3.6.7.2.6 2000/05/08 18:02:07 hno Exp $ +dnl $Id: configure.in,v 1.1.1.3.6.7.2.7 2000/05/14 23:22:19 hno Exp $ dnl dnl dnl AC_INIT(src/main.c) AC_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.1.1.3.6.7.2.6 $)dnl +AC_REVISION($Revision: 1.1.1.3.6.7.2.7 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AC_CONFIG_AUX_DIR(cfgaux) @@ -320,6 +320,49 @@ STORE_OBJS="fs/`echo $STORE_MODULES|sed -e's% %.a fs/%g'`.a" AC_SUBST(STORE_OBJS) +dnl --enable-heap-replacement compability option +AC_ARG_ENABLE(heap-replacement, +[ --enable-heap-replacement + Backwards compability option. Please use the + new --enable-removal-policies directive instead.], +[ if test "$enableval" = "yes" ; then + echo "--enable-heap-replacement is obsolete. please use the new" + echo "--enable-removal-policies directive instead" + sleep 5 + REPL_POLICIES="heap" + fi +]) + +AC_ARG_ENABLE(removal_policies, +[ --enable-removal-policies=\"list of policies\" + Build support for the list of removal policies. + The default is only to build the "lru" module. + See src/repl for a list of available modules, or + Programmers Guide section 9.9 for details on how + to build your custom policy], +[ case $enableval in + yes) + for module in $srcdir/src/repl/*; do + if test -f $module/Makefile.in; then + REPL_POLICIES="$REPL_POLICIES `basename $module`" + fi + done + ;; + no) + ;; + *) REPL_POLICIES="`echo $enableval| sed -e 's/,/ /g;s/ */ /g'`" + ;; + esac +], +[ if test -z "$REPL_POLICIES"; then + REPL_POLICIES="lru" + fi +]) +echo "Store modules built: $REPL_POLICIES" +AC_SUBST(REPL_POLICIES) +REPL_OBJS="repl/`echo $REPL_POLICIES|sed -e's% %.a repl/%g'`.a" +AC_SUBST(REPL_OBJS) + AC_ARG_ENABLE(icmp, [ --enable-icmp Enable ICMP pinging], [ if test "$enableval" = "yes" ; then @@ -592,18 +635,6 @@ fi ]) -dnl Enable HEAP_REPLACEMENT -AC_ARG_ENABLE(heap-replacement, -[ --enable-heap-replacement - This option allows you to use various cache - replacement algorithms, instead of the standard - LRU algorithm.], -[ if test "$enableval" = "yes" ; then - echo "Enabling HEAP_REPLACEMENT" - AC_DEFINE(HEAP_REPLACEMENT, 1) - fi -]) - dnl Select auth modules to build AUTH_MODULES= AC_ARG_ENABLE(auth-modules, @@ -1579,6 +1610,9 @@ ./src/fs/ufs/Makefile \ ./src/fs/aufs/Makefile \ ./src/fs/coss/Makefile \ + ./src/repl/Makefile \ + ./src/repl/lru/Makefile \ + ./src/repl/heap/Makefile \ ./src/fs/diskd/Makefile \ ./contrib/Makefile \ $SNMP_MAKEFILE \ Index: squid/doc/Programming-Guide/prog-guide.sgml =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/Programming-Guide/prog-guide.sgml,v retrieving revision 1.1.1.2.6.1.2.1 retrieving revision 1.1.1.2.6.1.2.2 diff -u -r1.1.1.2.6.1.2.1 -r1.1.1.2.6.1.2.2 --- squid/doc/Programming-Guide/prog-guide.sgml 2 May 2000 22:50:28 -0000 1.1.1.2.6.1.2.1 +++ squid/doc/Programming-Guide/prog-guide.sgml 14 May 2000 23:22:19 -0000 1.1.1.2.6.1.2.2 @@ -2,7 +2,7 @@
Squid Programmers Guide Duane Wessels, Squid Developers -$Id: prog-guide.sgml,v 1.1.1.2.6.1.2.1 2000/05/02 22:50:28 hno Exp $ +$Id: prog-guide.sgml,v 1.1.1.2.6.1.2.2 2000/05/14 23:22:19 hno Exp $ Squid is a WWW Cache application developed by the National Laboratory @@ -1492,16 +1492,16 @@ This feature may not be required by some storage systems and can be implemented as a null-function (no-op). - int - STLOGCLEANOPEN(SwapDir *); + STLOGCLEANSTART(SwapDir *);

- The + + StoreEntry * + STLOGCLEANNEXTENTRY(SwapDir *); + + +

+ Gets the next entry that is a candidate for the clean log. + +

+ Returns NULL when there is no more objects to log + void - STLOGCLEANWRITE(const StoreEntry *, SwapDir *); + STLOGCLEANWRITE(SwapDir *, const StoreEntry *);

The - A NULL + void + STLOGCLEANDONE(SwapDir *); + +

+ Indicates the end of the clean-writing process and signals + the storage system to close the clean log, and rename or + move them to become the official state-holding log ready + to be opened. Replacement policy implementation @@ -1545,10 +1566,7 @@ coupling between the storage layer and the replacement policy. -Future removal policy - -

- (replaces the above Replace policy) +Removal policy API

The removal policy is responsible for determining in which order @@ -1625,10 +1643,26 @@

- Tells the policy that a StoreEntry has been referenced. + Tells the policy that a StoreEntry is going to be referenced. Called + whenever a entry gets locked.

- datap is a pointer to where policy specific data is stored + node is a pointer to where policy specific data is stored + for the store entry, currently the size of one (void *) pointer. + +policy.Dereferenced() + +

+ + policy->Dereferenced(RemovalPolicy *policy, const StoreEntry *, RemovalPolicyNode *node) + + +

+ Tells the policy that a StoreEntry has been referenced. Called when + an access to the entry has finished. + +

+ node is a pointer to where policy specific data is stored for the store entry, currently the size of one (void *) pointer. policy.WalkInit() @@ -1822,6 +1856,32 @@ Prior to returning the created instance must be registered as callback-data by calling cbdataAdd(). +Design notes/bugs + +

+ The RemovalPolicyNode design is incomplete/insufficient. The intention + was to abstract the location of the index pointers from the policy + implementation to allow the policy to work on both on-disk and memory + caches, but unfortunately the purge method for HEAP based policies + needs to update this, and it is also preferable if the purge method + in general knows how to clear the information. I think the agreement + was that the current design of thightly coupling the two together + on one StoreEntry is not the best design possible. + +

+ It is debated if the design in having the policy index control the + clean index writes is the correct approach. Perhaps not. Perhaps a + more appropriate design is probably to do the store indexing + completely outside the policy implementation (i.e. using the hash + index), and only ask the policy to dump it's state somehow. + +

+ The Referenced/Dereferenced() calls is today mapped to lock/unlock + which is an approximation of when they are intended to be called. + However, the real intention is to have Referenced() called whenever + an object is referenced, and Dereferenced() only called when the + object has actually been used for anything good. + Forwarding Selection Index: squid/lib/rfc1035.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/rfc1035.c,v retrieving revision 1.1.1.1.10.2 retrieving revision 1.1.1.1.10.2.2.1 diff -u -r1.1.1.1.10.2 -r1.1.1.1.10.2.2.1 --- squid/lib/rfc1035.c 20 Apr 2000 18:14:21 -0000 1.1.1.1.10.2 +++ squid/lib/rfc1035.c 14 May 2000 23:22:19 -0000 1.1.1.1.10.2.2.1 @@ -1,6 +1,6 @@ /* - * $Id: rfc1035.c,v 1.1.1.1.10.2 2000/04/20 18:14:21 hno Exp $ + * $Id: rfc1035.c,v 1.1.1.1.10.2.2.1 2000/05/14 23:22:19 hno Exp $ * * Low level DNS protocol routines * AUTHOR: Duane Wessels @@ -93,6 +93,11 @@ unsigned short arcount; }; +static const char *Alphanum = +"abcdefghijklmnopqrstuvwxyz" +"ABCDEFGHIJKLMNOPQRSTUVWXYZ" +"0123456789"; + /* * rfc1035HeaderPack() * @@ -362,6 +367,39 @@ return qid; } +static void +rfc1035SetErrno(int n) +{ + switch (rfc1035_errno = n) { + case 0: + rfc1035_error_message = "No error condition"; + break; + case 1: + rfc1035_error_message = "Format Error: The name server was " + "unable to interpret the query."; + break; + case 2: + rfc1035_error_message = "Server Failure: The name server was " + "unable to process this query."; + break; + case 3: + rfc1035_error_message = "Name Error: The domain name does " + "not exist."; + break; + case 4: + rfc1035_error_message = "Not Implemented: The name server does " + "not support the requested kind of query."; + break; + case 5: + rfc1035_error_message = "Refused: The name server refuses to " + "perform the specified operation."; + break; + default: + rfc1035_error_message = "Unknown Error"; + break; + } +} + void rfc1035RRDestroy(rfc1035_rr * rr, int n) { @@ -393,35 +431,7 @@ rfc1035_errno = 0; rfc1035_error_message = NULL; if (hdr.rcode) { - rfc1035_errno = (int) hdr.rcode; - switch (rfc1035_errno) { - case 0: - rfc1035_error_message = "No error condition"; - break; - case 1: - rfc1035_error_message = "Format Error: The name server was " - "unable to interpret the query."; - break; - case 2: - rfc1035_error_message = "Server Failure: The name server was " - "unable to process this query."; - break; - case 3: - rfc1035_error_message = "Name Error: The domain name does " - "not exist."; - break; - case 4: - rfc1035_error_message = "Not Implemented: The name server does " - "not support the requested kind of query."; - break; - case 5: - rfc1035_error_message = "Refused: The name server refuses to " - "perform the specified operation."; - break; - default: - rfc1035_error_message = "Unknown Error"; - break; - } + rfc1035SetErrno((int) hdr.rcode); return -rfc1035_errno; } i = (int) hdr.qdcount; @@ -471,6 +481,11 @@ off_t offset = 0; size_t sz = *szp; memset(&h, '\0', sizeof(h)); + /* the first char of hostname must be alphanmeric */ + if (NULL == strchr(Alphanum, *hostname)) { + rfc1035SetErrno(3); + return 0; + } h.id = rfc1035Qid(); h.qr = 0; h.rd = 1; Index: squid/src/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/Makefile.in,v retrieving revision 1.1.1.3.10.7.2.2 retrieving revision 1.1.1.3.10.7.2.3 diff -u -r1.1.1.3.10.7.2.2 -r1.1.1.3.10.7.2.3 --- squid/src/Makefile.in 2 May 2000 22:50:28 -0000 1.1.1.3.10.7.2.2 +++ squid/src/Makefile.in 14 May 2000 23:22:19 -0000 1.1.1.3.10.7.2.3 @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.1.1.3.10.7.2.2 2000/05/02 22:50:28 hno Exp $ +# $Id: Makefile.in,v 1.1.1.3.10.7.2.3 2000/05/14 23:22:19 hno Exp $ # # Uncomment and customize the following to suit your needs: # @@ -65,6 +65,8 @@ XTRA_OBJS = @XTRA_OBJS@ STORE_OBJS = @STORE_OBJS@ STORE_MODULES = @STORE_MODULES@ +REPL_OBJS = @REPL_OBJS@ +REPL_POLICIES = @REPL_POLICIES@ MV = @MV@ RM = @RM@ SHELL = /bin/sh @@ -149,6 +151,7 @@ pump.o \ redirect.o \ refresh.o \ + repl_modules.o \ send-announce.o \ @SNMP_OBJS@ \ ssl.o \ @@ -168,7 +171,6 @@ store_swapin.o \ store_swapmeta.o \ store_swapout.o \ - store_heap_replacement.o \ string_arrays.o \ tools.o \ @UNLINKD_OBJS@ \ @@ -194,14 +196,14 @@ DEFAULTS = \ -DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\" -all: squid.conf store_modules $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS) +all: squid.conf store_modules repl_modules $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS) $(OBJS): $(top_srcdir)/include/version.h ../include/autoconf.h $(SNMP_OBJS): ../snmplib/libsnmp.a $(top_srcdir)/include/cache_snmp.h -$(SQUID_EXE): $(OBJS) $(STORE_OBJS) - $(CC) -o $@ $(LDFLAGS) $(OBJS) $(STORE_OBJS) $(SQUID_LIBS) +$(SQUID_EXE): $(OBJS) $(STORE_OBJS) $(REPL_OBJS) + $(CC) -o $@ $(LDFLAGS) $(OBJS) $(STORE_OBJS) $(REPL_OBJS) $(SQUID_LIBS) globals.o: globals.c Makefile $(CC) -c globals.c $(CFLAGS) -I$(srcdir) $(DEFAULTS) @@ -277,6 +279,17 @@ store_modules fs/stamp: @sh -c "cd fs && $(MAKE) all" +repl_modules.c: repl_modules.sh Makefile + sh $(srcdir)/repl_modules.sh $(REPL_POLICIES) >repl_modules.c + +repl_modules.o: repl_modules.c + $(CC) -c repl_modules.c $(CFLAGS) -I$(srcdir) + +$(REPL_OBJS): repl/stamp + +repl_modules repl/stamp: + @sh -c "cd repl && $(MAKE) all" + install-mkdirs: -@if test ! -d $(prefix); then \ echo "mkdir $(prefix)"; \ Index: squid/src/acl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/acl.c,v retrieving revision 1.1.1.3.10.5.2.2 retrieving revision 1.1.1.3.10.5.2.3 diff -u -r1.1.1.3.10.5.2.2 -r1.1.1.3.10.5.2.3 --- squid/src/acl.c 2 May 2000 23:21:53 -0000 1.1.1.3.10.5.2.2 +++ squid/src/acl.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.5.2.3 @@ -1,6 +1,6 @@ /* - * $Id: acl.c,v 1.1.1.3.10.5.2.2 2000/05/02 23:21:53 hno Exp $ + * $Id: acl.c,v 1.1.1.3.10.5.2.3 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Duane Wessels @@ -1075,7 +1075,7 @@ */ static int -aclMatchProxyAuth(void * data, const char *proxy_auth, acl_proxy_auth_user * auth_user, aclCheck_t * checklist, squid_acl acltype) +aclMatchProxyAuth(void *data, const char *proxy_auth, acl_proxy_auth_user * auth_user, aclCheck_t * checklist, squid_acl acltype) { /* checklist is used to register user name when identified, nothing else */ LOCAL_ARRAY(char, login_buf, USER_IDENT_SZ); @@ -1142,14 +1142,14 @@ auth_user->ipaddr = checklist->src_addr; /* copy username to request for logging on client-side */ xstrncpy(checklist->request->user_ident, user, USER_IDENT_SZ); - switch(acltype) { + switch (acltype) { case ACL_PROXY_AUTH: return aclMatchUser(data, user); case ACL_PROXY_AUTH_REGEX: return aclMatchRegex(data, user); default: fatal("aclMatchProxyAuth: unknown ACL type"); - return 0; /* NOTREACHED */ + return 0; /* NOTREACHED */ } } else { if (Config.onoff.authenticateIpTTLStrict) { @@ -2004,16 +2004,16 @@ const char *d1; const char *d2; int ret; - d1=b; - d2=a; + d1 = b; + d2 = a; ret = aclHostDomainCompare(d1, d2); if (ret != 0) { - d1=a; - d2=b; + d1 = a; + d2 = b; ret = aclHostDomainCompare(d1, d2); } if (ret == 0) { - debug(28,0 ) ("WARNING: '%s' is a subdomain of '%s'\n", d1, d2); + debug(28, 0) ("WARNING: '%s' is a subdomain of '%s'\n", d1, d2); debug(28, 0) ("WARNING: because of this '%s' is ignored to keep splay tree searching predictable\n", a); debug(28, 0) ("WARNING: You should probably remove '%s' from the ACL named '%s'\n", d1, AclMatchedName); } Index: squid/src/asn.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/asn.c,v retrieving revision 1.1.1.3.6.1.2.1 retrieving revision 1.1.1.3.6.1.2.2 diff -u -r1.1.1.3.6.1.2.1 -r1.1.1.3.6.1.2.2 --- squid/src/asn.c 8 May 2000 18:02:07 -0000 1.1.1.3.6.1.2.1 +++ squid/src/asn.c 14 May 2000 23:22:19 -0000 1.1.1.3.6.1.2.2 @@ -1,5 +1,5 @@ /* - * $Id: asn.c,v 1.1.1.3.6.1.2.1 2000/05/08 18:02:07 hno Exp $ + * $Id: asn.c,v 1.1.1.3.6.1.2.2 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 53 AS Number handling * AUTHOR: Duane Wessels, Kostas Anagnostakis @@ -206,7 +206,7 @@ asState->seen = 0; asState->offset = 0; storeClientCopy(asState->sc, - e, + e, asState->seen, asState->offset, 4096, @@ -262,7 +262,7 @@ if (e->store_status == STORE_PENDING) { debug(53, 3) ("asHandleReply: store_status == STORE_PENDING: %s\n", storeUrl(e)); storeClientCopy(asState->sc, - e, + e, asState->seen, asState->offset, SM_PAGE_SIZE, @@ -272,7 +272,7 @@ } else if (asState->seen < e->mem_obj->inmem_hi) { debug(53, 3) ("asHandleReply: asState->seen < e->mem_obj->inmem_hi %s\n", storeUrl(e)); storeClientCopy(asState->sc, - e, + e, asState->seen, asState->offset, SM_PAGE_SIZE, Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.1.1.3.10.4.2.2 retrieving revision 1.1.1.3.10.4.2.3 diff -u -r1.1.1.3.10.4.2.2 -r1.1.1.3.10.4.2.3 --- squid/src/cache_cf.c 23 Apr 2000 09:55:37 -0000 1.1.1.3.10.4.2.2 +++ squid/src/cache_cf.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.4.2.3 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.1.1.3.10.4.2.2 2000/04/23 09:55:37 hno Exp $ + * $Id: cache_cf.c,v 1.1.1.3.10.4.2.3 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -113,7 +113,7 @@ } void -wordlistJoin(wordlist ** list, wordlist **wl) +wordlistJoin(wordlist ** list, wordlist ** wl) { while (*list) list = &(*list)->next; @@ -122,14 +122,14 @@ } void -wordlistAddWl(wordlist ** list, wordlist *wl) +wordlistAddWl(wordlist ** list, wordlist * wl) { while (*list) list = &(*list)->next; - for(;wl;wl=wl->next, list = &(*list)->next) { + for (; wl; wl = wl->next, list = &(*list)->next) { *list = memAllocate(MEM_WORDLIST); (*list)->key = xstrdup(wl->key); - (*list)->next = NULL; + (*list)->next = NULL; } } @@ -200,8 +200,8 @@ size_t ms = -1; for (i = 0; i < Config.cacheSwap.n_configured; i++) { - if (Config.cacheSwap.swapDirs[i].max_objsize > ms) - ms = Config.cacheSwap.swapDirs[i].max_objsize; + if (Config.cacheSwap.swapDirs[i].max_objsize > ms) + ms = Config.cacheSwap.swapDirs[i].max_objsize; } store_maxobjsize = ms; } @@ -348,14 +348,6 @@ debug(3, 0) ("WARNING: resetting 'maximum_single_addr_tries to 1\n"); Config.retry.maxtries = 1; } -#if HEAP_REPLACEMENT - /* The non-LRU policies do not use referenceAge */ -#else - if (Config.referenceAge < 300) { - debug(3, 0) ("WARNING: resetting 'reference_age' to 1 week\n"); - Config.referenceAge = 86400 * 7; - } -#endif requirePathnameExists("MIME Config Table", Config.mimeTablePathname); #if USE_DNSSERVERS requirePathnameExists("cache_dns_program", Config.Program.dnsserver); @@ -816,7 +808,7 @@ int i; for (i = 0; i < swap.n_configured; i++) { s = swap.swapDirs + i; - s->dump(entry, name, s); + s->dump(entry, name, s); } } @@ -854,7 +846,7 @@ { int i; for (i = 0; storefs_list[i].typestr != NULL; i++) { - if (strcasecmp(type, storefs_list[i].typestr) == 0) { + if (strcasecmp(type, storefs_list[i].typestr) == 0) { return i; } } @@ -874,7 +866,7 @@ if ((type_str = strtok(NULL, w_space)) == NULL) self_destruct(); - maxobjsize = (size_t)GetInteger(); + maxobjsize = (size_t) GetInteger(); if ((path_str = strtok(NULL, w_space)) == NULL) self_destruct(); @@ -898,26 +890,25 @@ */ for (i = 0; i < swap->n_configured; i++) { - if (0 == strcasecmp(path_str, swap->swapDirs[i].path)) { + if (0 == strcasecmp(path_str, swap->swapDirs[i].path)) { /* This is a little weird, you'll appreciate it later */ fs = find_fstype(type_str); if (fs < 0) { - fatalf("Unknown cache_dir type '%s'\n", type_str); + fatalf("Unknown cache_dir type '%s'\n", type_str); } sd = swap->swapDirs + i; storefs_list[fs].reconfigurefunc(sd, i, path_str); - sd->max_objsize = maxobjsize; - update_maxobjsize(); - return; + sd->max_objsize = maxobjsize; + update_maxobjsize(); + return; } } fs = find_fstype(type_str); if (fs < 0) { - /* If we get here, we didn't find a matching cache_dir type */ - fatalf("Unknown cache_dir type '%s'\n", type_str); + /* If we get here, we didn't find a matching cache_dir type */ + fatalf("Unknown cache_dir type '%s'\n", type_str); } - allocate_new_swapdir(swap); sd = swap->swapDirs + swap->n_configured; storefs_list[fs].parsefunc(sd, swap->n_configured, path_str); @@ -1494,7 +1485,7 @@ } static int -check_null_refreshpattern(refresh_t *data) +check_null_refreshpattern(refresh_t * data) { return data != NULL; } @@ -1705,6 +1696,40 @@ storeAppendPrintf(entry, "%s %s\n", name, s); } +static void +free_removalpolicy(RemovalPolicySettings **settings) +{ + if (!*settings) + return; + free_string(&(*settings)->type); + free_wordlist(&(*settings)->args); + xfree(*settings); + *settings = NULL; +} + +static void +parse_removalpolicy(RemovalPolicySettings **settings) +{ + if (*settings) + free_removalpolicy(settings); + *settings = xcalloc(1, sizeof(**settings)); + parse_string(&(*settings)->type); + parse_wordlist(&(*settings)->args); +} + +static void +dump_removalpolicy(StoreEntry * entry, const char *name, RemovalPolicySettings *settings) +{ + wordlist *args; + storeAppendPrintf(entry, "%s %s", name, settings->type); + args = settings->args; + while (args) { + storeAppendPrintf(entry, " %s", args->key); + args = args->next; + } +} + + #include "cf_parser.c" peer_t @@ -1806,3 +1831,5 @@ if (stat(path, &sb) < 0) fatalf("%s: %s", path, xstrerror()); } + + Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.1.1.3.10.6.2.4 retrieving revision 1.1.1.3.10.6.2.5 diff -u -r1.1.1.3.10.6.2.4 -r1.1.1.3.10.6.2.5 --- squid/src/cf.data.pre 8 May 2000 18:02:07 -0000 1.1.1.3.10.6.2.4 +++ squid/src/cf.data.pre 14 May 2000 23:22:19 -0000 1.1.1.3.10.6.2.5 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.1.1.3.10.6.2.4 2000/05/08 18:02:07 hno Exp $ +# $Id: cf.data.pre,v 1.1.1.3.10.6.2.5 2000/05/14 23:22:19 hno Exp $ # # # SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -580,6 +580,58 @@ Maximum number of FQDN cache entries. DOC_END +NAME: cache_replacement_policy +TYPE: removalpolicy +LOC: Config.replPolicy +DEFAULT: lru +DOC_START + The cache replacement policy parameter determines which + objects are evicted (replaced) when disk space is needed. + + lru : Squid's original list based LRU policy + heap GDSF : Greedy-Dual Size Frequency + heap LFUDA: Least Frequently Used with Dynamic Aging + heap LRU : LRU policy implemented using a heap + + Applies to any cache_dir lines listed below this. + + The LRU policies keeps recently referenced objects. + + The heap GDSF policy optimizes object hit rate by keeping smaller + popular objects in cache so it has a better chance of getting a + hit. It achieves a lower byte hit rate than LFUDA though since + it evicts larger (possibly popular) objects. + + The heap LFUDA policy keeps popular objects in cache regardless of + their size and thus optimizes byte hit rate at the expense of + hit rate since one large, popular object will prevent many + smaller, slightly less popular objects from being cached. + + Both policies utilize a dynamic aging mechanism that prevents + cache pollution that can otherwise occur with frequency-based + replacement policies. + + NOTE: if using the LFUDA replacement policy you should increase + the value of maximum_object_size above its default of 4096 KB to + to maximize the potential byte hit rate improvement of LFUDA. + + For more information about the GDSF and LFUDA cache replacement + policies see http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html + and http://fog.hpl.external.hp.com/techreports/98/HPL-98-173.html. +DOC_END + +NAME: memory_replacement_policy +TYPE: removalpolicy +LOC: Config.memPolicy +DEFAULT: lru +DOC_START + The memory replacement policy parameter determines which + objects are purged from memory when memory space is needed. + + See cache_replacement_policy for details. +DOC_END + + COMMENT_START LOGFILE PATHNAMES AND CACHE DIRECTORIES ----------------------------------------------------------------------------- @@ -593,7 +645,11 @@ DOC_START Usage: - cache_dir Type Maxobjsize Directory-Name Mbytes Level-1 Level2 + cache_dir Type Maxobjsize Directory-Name Mbytes Level-1 Level2 [...] + + DISKD Usage: + + cache_dir diskd Maxobjsize Directory-Name MB L1 L2 Q1 Q2 You can specify multiple cache_dir lines to spread the cache among different disk partitions. @@ -627,6 +683,13 @@ 'Level-2' is the number of second-level subdirectories which will be created under each first-level directory. The default is 256. + + For the diskd type, Q1 specifies the number of unacknowledged + I/O requests when Squid stops opening new files. If this + many messages are in the queues, Squid won't open new files. + Q2 specifies the number of unacknowledged messages when Squid + starts blocking. If this many messages are in the queues, + Squid blocks until it recevies some replies. DOC_END @@ -1194,49 +1257,6 @@ used. DOC_END - -NAME: replacement_policy -TYPE: string -LOC: Config.replPolicy -DEFAULT: LFUDA -IFDEF: HEAP_REPLACEMENT -DOC_START - The cache replacement policy parameter determines which - objects are evicted (replaced) when disk space is needed. - Squid used to have only a single replacement policy, LRU. - But when built with -DHEAP_REPLACEMENT you can choose - between two new, enhanced policies: - - GDSF: Greedy-Dual Size Frequency - LFUDA: Least Frequently Used with Dynamic Aging - - Both of these policies are frequency based rather than recency - based, and perform better than LRU. - - The GDSF policy optimizes object hit rate by keeping smaller - popular objects in cache so it has a better chance of getting a - hit. It achieves a lower byte hit rate than LFUDA though since - it evicts larger (possibly popular) objects. - - The LFUDA policy keeps popular objects in cache regardless of - their size and thus optimizes byte hit rate at the expense of - hit rate since one large, popular object will prevent many - smaller, slightly less popular objects from being cached. - - Both policies utilize a dynamic aging mechanism that prevents - cache pollution that can otherwise occur with frequency-based - replacement policies. - - NOTE: if using the LFUDA replacement policy you should increase - the value of maximum_object_size above its default of 4096 KB to - to maximize the potential byte hit rate improvement of LFUDA. - - For more information about these cache replacement policies see - http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html and - http://fog.hpl.external.hp.com/techreports/98/HPL-98-173.html. -DOC_END - - NAME: reference_age TYPE: time_t LOC: Config.referenceAge @@ -3130,21 +3150,5 @@ disable persistent connections with clients and/or servers. DOC_END -NAME: diskd_magic1 -IFDEF: USE_DISKD -TYPE: int -LOC: Config.diskd.magic1 -DEFAULT: 64 -DOC_NONE - -NAME: diskd_magic2 -IFDEF: USE_DISKD -TYPE: int -LOC: Config.diskd.magic2 -DEFAULT: 72 -DOC_START - Macic constants for the diskd daemon.. -DOC_END - EOF Index: squid/src/cf_gen.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf_gen.c,v retrieving revision 1.1.1.3.10.2.2.1 retrieving revision 1.1.1.3.10.2.2.2 diff -u -r1.1.1.3.10.2.2.1 -r1.1.1.3.10.2.2.2 --- squid/src/cf_gen.c 2 May 2000 22:50:28 -0000 1.1.1.3.10.2.2.1 +++ squid/src/cf_gen.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.2.2.2 @@ -1,6 +1,6 @@ /* - * $Id: cf_gen.c,v 1.1.1.3.10.2.2.1 2000/05/02 22:50:28 hno Exp $ + * $Id: cf_gen.c,v 1.1.1.3.10.2.2.2 2000/05/14 23:22:19 hno Exp $ * * DEBUG: none * AUTHOR: Max Okumoto @@ -108,11 +108,12 @@ static void gen_conf(Entry *, FILE *); static void gen_default_if_none(Entry *, FILE *); -static void lineAdd(Line **L, char *str) +static void +lineAdd(Line ** L, char *str) { - while(*L) - L = &(*L)->next; - *L=xcalloc(1, sizeof(Line)); + while (*L) + L = &(*L)->next; + *L = xcalloc(1, sizeof(Line)); (*L)->data = xstrdup(str); } @@ -432,7 +433,7 @@ fprintf(fp, "\t}\n"); } if (entry->ifdef) - fprintf(fp, "#endif\n"); + fprintf(fp, "#endif\n"); } fprintf(fp, "}\n\n"); } @@ -536,25 +537,27 @@ fprintf(fp, "}\n\n"); } -static int defined(char *name) +static int +defined(char *name) { - int i=0; - if (!name) - return 1; - for(i=0;strcmp(defines[i].name, name) != 0; i++) { - assert(defines[i].name); - } - return defines[i].defined; + int i = 0; + if (!name) + return 1; + for (i = 0; strcmp(defines[i].name, name) != 0; i++) { + assert(defines[i].name); + } + return defines[i].defined; } -static const char *available_if(char *name) +static const char * +available_if(char *name) { - int i=0; - assert(name); - for(i=0;strcmp(defines[i].name, name) != 0; i++) { - assert(defines[i].name); - } - return defines[i].enable; + int i = 0; + assert(name); + for (i = 0; strcmp(defines[i].name, name) != 0; i++) { + assert(defines[i].name); + } + return defines[i].enable; } static void @@ -582,7 +585,7 @@ for (line = entry->doc; line != NULL; line = line->next) { fprintf(fp, "#%s\n", line->data); } - if (entry->default_value && strcmp(entry->default_value,"none") != 0) { + if (entry->default_value && strcmp(entry->default_value, "none") != 0) { sprintf(buf, "%s %s", entry->name, entry->default_value); lineAdd(&def, buf); } @@ -595,7 +598,7 @@ if (entry->nocomment) blank = 0; if (!def && entry->doc && !entry->nocomment && - strcmp(entry->name, "comment") != 0) + strcmp(entry->name, "comment") != 0) lineAdd(&def, "none"); if (def && (entry->doc || entry->nocomment)) { if (blank) @@ -608,7 +611,7 @@ free(line->data); free(line); } - blank=1; + blank = 1; } if (entry->nocomment && blank) fprintf(fp, "#\n"); Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.1.1.3.10.7.2.4 retrieving revision 1.1.1.3.10.7.2.5 diff -u -r1.1.1.3.10.7.2.4 -r1.1.1.3.10.7.2.5 --- squid/src/client_side.c 8 May 2000 18:02:08 -0000 1.1.1.3.10.7.2.4 +++ squid/src/client_side.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.7.2.5 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.1.1.3.10.7.2.4 2000/05/08 18:02:08 hno Exp $ + * $Id: client_side.c,v 1.1.1.3.10.7.2.5 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -141,7 +141,7 @@ #endif static aclCheck_t * -clientAclChecklistCreate(const acl_access *acl, const clientHttpRequest *http) +clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http) { aclCheck_t *ch; ConnStateData *conn = http->conn; @@ -347,14 +347,13 @@ } http->request->flags.refresh = 1; http->old_entry = http->entry; + http->old_sc = http->sc; /* * Assert that 'http' is already a client of old_entry. If * it is not, then the beginning of the object data might get * freed from memory before we need to access it. */ -#if STORE_CLIENT_LIST_SEARCH - assert(storeClientListSearch(http->old_entry->mem_obj, http)); -#endif + assert(http->sc->callback_data == http); entry = storeCreateEntry(url, http->log_uri, http->request->flags, @@ -450,6 +449,7 @@ storeUnregister(http->sc, entry, http); storeUnlockObject(entry); entry = http->entry = http->old_entry; + http->sc = http->old_sc; } else if (STORE_PENDING == entry->store_status && 0 == status) { debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); if (size >= CLIENT_SOCK_SZ) { @@ -460,6 +460,7 @@ storeUnregister(http->sc, entry, http); storeUnlockObject(entry); entry = http->entry = http->old_entry; + http->sc = http->old_sc; /* continue */ } else { storeClientCopy(http->sc, entry, @@ -488,6 +489,7 @@ httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); storeTimestampsSet(oldentry); storeUnregister(http->sc, entry, http); + http->sc = http->old_sc; storeUnlockObject(entry); entry = http->entry = oldentry; entry->timestamp = squid_curtime; @@ -504,11 +506,12 @@ storeTimestampsSet(http->old_entry); http->log_type = LOG_TCP_REFRESH_HIT; } - storeUnregister(http->sc, http->old_entry, http); + storeUnregister(http->old_sc, http->old_entry, http); storeUnlockObject(http->old_entry); recopy = 0; } http->old_entry = NULL; /* done with old_entry */ + http->old_sc = NULL; assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); if (recopy) { storeClientCopy(http->sc, entry, @@ -1106,10 +1109,10 @@ range_err = "canonization failed"; else if (httpHdrRangeIsComplex(http->request->range)) range_err = "too complex range header"; - else if (!request->flags.cachable) /* from we_do_ranges in http.c */ + else if (!request->flags.cachable) /* from we_do_ranges in http.c */ range_err = "non-cachable request"; else if (!is_hit && Config.rangeOffsetLimit < httpHdrRangeFirstOffset(request->range) - && Config.rangeOffsetLimit != -1) /* from we_do_ranges in http.c */ + && Config.rangeOffsetLimit != -1) /* from we_do_ranges in http.c */ range_err = "range outside range_offset_limit"; /* get rid of our range specs on error */ if (range_err) { @@ -2478,6 +2481,13 @@ } /* reset range iterator */ http->range_iter.pos = HttpHdrRangeInitPos; + } else if (!http->request->range) { + /* Avoid copying to MemBuf for non-range requests */ + /* Note, if we're here, then 'rep' is known to be NULL */ + http->out.offset += body_size; + comm_write(fd, buf, size, clientWriteBodyComplete, http, NULL); + /* NULL because clientWriteBodyComplete frees it */ + return; } if (http->request->method == METHOD_HEAD) { if (rep) { @@ -2549,6 +2559,23 @@ memFree(buf, MEM_CLIENT_SOCK_BUF); } +/* + * clientWriteBodyComplete is called for MEM_CLIENT_SOCK_BUF's + * written directly to the client socket, versus copying to a MemBuf + * and going through comm_write_mbuf. Most non-range responses after + * the headers probably go through here. + */ +static void +clientWriteBodyComplete(int fd, char *buf, size_t size, int errflag, void *data) +{ + /* + * NOTE: clientWriteComplete doesn't currently use its "buf" + * (second) argument, so we pass in NULL. + */ + clientWriteComplete(fd, NULL, size, errflag, data); + memFree(buf, MEM_CLIENT_SOCK_BUF); +} + static void clientKeepaliveNextRequest(clientHttpRequest * http) { @@ -2593,14 +2620,6 @@ } static void -clientWriteBodyComplete(int fd, char *buf, size_t size, int errflag, -void *data) -{ - memFree(buf, MEM_CLIENT_SOCK_BUF); - clientWriteComplete(fd, buf, size, errflag, data); -} - -static void clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) { clientHttpRequest *http = data; @@ -3517,6 +3536,7 @@ commSetSelect(fd, COMM_SELECT_READ, clientReadRequest, connState, 0); commSetDefer(fd, clientReadDefer, connState); clientdbEstablished(peer.sin_addr, 1); + assert(N); (*N)++; } } Index: squid/src/comm_select.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/comm_select.c,v retrieving revision 1.1.1.3.10.2.2.2 retrieving revision 1.1.1.3.10.2.2.3 diff -u -r1.1.1.3.10.2.2.2 -r1.1.1.3.10.2.2.3 --- squid/src/comm_select.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.2.2.2 +++ squid/src/comm_select.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.2.2.3 @@ -1,6 +1,6 @@ /* - * $Id: comm_select.c,v 1.1.1.3.10.2.2.2 2000/05/02 22:50:29 hno Exp $ + * $Id: comm_select.c,v 1.1.1.3.10.2.2.3 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 5 Socket Functions * @@ -34,7 +34,7 @@ #include "squid.h" -static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */ +static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */ #ifndef howmany #define howmany(x, y) (((x)+((y)-1))/(y)) @@ -326,8 +326,8 @@ getCurrentTime(); start = current_dtime; #endif - /* Handle any fs callbacks that need doing */ - storeDirCallback(); + /* Handle any fs callbacks that need doing */ + storeDirCallback(); #if DELAY_POOLS FD_ZERO(&slowfds); #endif @@ -661,8 +661,8 @@ #if DELAY_POOLS FD_ZERO(&slowfds); #endif - /* Handle any fs callbacks that need doing */ - storeDirCallback(); + /* Handle any fs callbacks that need doing */ + storeDirCallback(); if (commCheckICPIncoming) comm_select_icp_incoming(); if (commCheckDNSIncoming) Index: squid/src/delay_pools.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/delay_pools.c,v retrieving revision 1.1.1.3.10.3.2.1 retrieving revision 1.1.1.3.10.3.2.2 diff -u -r1.1.1.3.10.3.2.1 -r1.1.1.3.10.3.2.2 --- squid/src/delay_pools.c 8 May 2000 18:02:08 -0000 1.1.1.3.10.3.2.1 +++ squid/src/delay_pools.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.3.2.2 @@ -1,6 +1,6 @@ /* - * $Id: delay_pools.c,v 1.1.1.3.10.3.2.1 2000/05/08 18:02:08 hno Exp $ + * $Id: delay_pools.c,v 1.1.1.3.10.3.2.2 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 77 Delay Pools * AUTHOR: David Luyer @@ -222,18 +222,18 @@ */ switch (class) { case 1: - delay_data[pool].class1->aggregate = (int)(((double)rates->aggregate.max_bytes * - Config.Delay.initial) / 100); + delay_data[pool].class1->aggregate = (int) (((double) rates->aggregate.max_bytes * + Config.Delay.initial) / 100); break; case 2: - delay_data[pool].class2->aggregate = (int)(((double)rates->aggregate.max_bytes * - Config.Delay.initial) / 100); + delay_data[pool].class2->aggregate = (int) (((double) rates->aggregate.max_bytes * + Config.Delay.initial) / 100); delay_data[pool].class2->individual_map[0] = 255; delay_data[pool].class2->individual_255_used = 0; break; case 3: - delay_data[pool].class3->aggregate = (int)(((double)rates->aggregate.max_bytes * - Config.Delay.initial) / 100); + delay_data[pool].class3->aggregate = (int) (((double) rates->aggregate.max_bytes * + Config.Delay.initial) / 100); delay_data[pool].class3->network_map[0] = 255; delay_data[pool].class3->network_255_used = 0; memset(&delay_data[pool].class3->individual_255_used, '\0', @@ -307,8 +307,8 @@ if (!delay_data[pool].class2->individual_255_used) { delay_data[pool].class2->individual_255_used = 1; delay_data[pool].class2->individual[IND_MAP_SZ - 1] = - (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100); + (int) (((double) Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); } return delayId(pool + 1, 255); } @@ -320,8 +320,8 @@ assert(i < (IND_MAP_SZ - 1)); delay_data[pool].class2->individual_map[i + 1] = 255; delay_data[pool].class2->individual[i] = - (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100); + (int) (((double) Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); break; } } @@ -336,8 +336,8 @@ if (!delay_data[pool].class3->network_255_used) { delay_data[pool].class3->network_255_used = 1; delay_data[pool].class3->network[255] = - (int)(((double)Config.Delay.rates[pool]->network.max_bytes * - Config.Delay.initial) / 100); + (int) (((double) Config.Delay.rates[pool]->network.max_bytes * + Config.Delay.initial) / 100); } } else { for (i = 0; i < NET_MAP_SZ; i++) { @@ -349,8 +349,8 @@ assert(i < (NET_MAP_SZ - 1)); delay_data[pool].class3->network_map[i + 1] = 255; delay_data[pool].class3->network[i] = - (int)(((double)Config.Delay.rates[pool]->network.max_bytes * - Config.Delay.initial) / 100); + (int) (((double) Config.Delay.rates[pool]->network.max_bytes * + Config.Delay.initial) / 100); break; } } @@ -362,8 +362,8 @@ delay_data[pool].class3->individual_255_used[i / 8] |= (1 << (i % 8)); assert(position < C3_IND_SZ); delay_data[pool].class3->individual[position] = - (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100); + (int) (((double) Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); } return delayId(pool + 1, position); } @@ -380,8 +380,8 @@ position |= j; assert(position < C3_IND_SZ); delay_data[pool].class3->individual[position] = - (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100); + (int) (((double) Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); break; } } @@ -611,7 +611,7 @@ store_client *sc; dlink_node *node; for (node = mem->clients.head; node; node = node->next) { - sc = (store_client *)node->data; + sc = (store_client *) node->data; if (sc->callback_data == NULL) /* open slot */ continue; if (sc->type != STORE_MEM_CLIENT) @@ -631,7 +631,7 @@ dlink_node *node; delay_id d = 0; for (node = mem->clients.head; node; node = node->next) { - sc = (store_client *)node->data; + sc = (store_client *) node->data; if (sc->callback_data == NULL) /* open slot */ continue; if (sc->type != STORE_MEM_CLIENT) @@ -646,7 +646,7 @@ } void -delaySetStoreClient(store_client *sc, delay_id delay_id) +delaySetStoreClient(store_client * sc, delay_id delay_id) { assert(sc != NULL); sc->delay_id = delay_id; Index: squid/src/disk.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/disk.c,v retrieving revision 1.1.1.3.6.2 retrieving revision 1.1.1.3.6.2.2.1 diff -u -r1.1.1.3.6.2 -r1.1.1.3.6.2.2.1 --- squid/src/disk.c 20 Apr 2000 18:14:22 -0000 1.1.1.3.6.2 +++ squid/src/disk.c 14 May 2000 23:22:19 -0000 1.1.1.3.6.2.2.1 @@ -1,6 +1,6 @@ /* - * $Id: disk.c,v 1.1.1.3.6.2 2000/04/20 18:14:22 hno Exp $ + * $Id: disk.c,v 1.1.1.3.6.2.2.1 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 6 Disk I/O Routines * AUTHOR: Harvest Derived @@ -188,7 +188,7 @@ fd, (int) (fdd->write_q->len - fdd->write_q->buf_offset)); errno = 0; if (fdd->write_q->file_offset != -1) - lseek(fd, fdd->write_q->file_offset, SEEK_SET); + lseek(fd, fdd->write_q->file_offset, SEEK_SET); len = write(fd, fdd->write_q->buf + fdd->write_q->buf_offset, fdd->write_q->len - fdd->write_q->buf_offset); Index: squid/src/dns_internal.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/dns_internal.c,v retrieving revision 1.1.1.1.10.4.2.2 retrieving revision 1.1.1.1.10.4.2.3 diff -u -r1.1.1.1.10.4.2.2 -r1.1.1.1.10.4.2.3 --- squid/src/dns_internal.c 2 May 2000 23:01:48 -0000 1.1.1.1.10.4.2.2 +++ squid/src/dns_internal.c 14 May 2000 23:22:19 -0000 1.1.1.1.10.4.2.3 @@ -1,6 +1,6 @@ /* - * $Id: dns_internal.c,v 1.1.1.1.10.4.2.2 2000/05/02 23:01:48 hno Exp $ + * $Id: dns_internal.c,v 1.1.1.1.10.4.2.3 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 78 DNS lookups; interacts with lib/rfc1035.c * AUTHOR: Duane Wessels @@ -206,7 +206,7 @@ assert(nns > 0); assert(q->lru.next == NULL); assert(q->lru.prev == NULL); -try_again: + try_again: ns = q->nsends % nns; x = comm_udp_sendto(DnsSocket, &nameservers[ns].S, @@ -323,6 +323,7 @@ break; } fd_bytes(DnsSocket, len, FD_READ); + assert(N); (*N)++; debug(78, 3) ("idnsRead: FD %d: received %d bytes from %s.\n", fd, @@ -355,7 +356,7 @@ event_queued = 0; for (n = lru_list.tail; n; n = p) { q = n->data; - if (tvSubDsec(q->sent_t, current_time) < Config.Timeout.idns_retransmit * ( 1 << q->nsends % nns)) + if (tvSubDsec(q->sent_t, current_time) < Config.Timeout.idns_retransmit * (1 << q->nsends % nns)) break; debug(78, 3) ("idnsCheckQueue: ID %#04x timeout\n", q->id); @@ -424,6 +425,12 @@ idns_query *q = memAllocate(MEM_IDNS_QUERY); q->sz = sizeof(q->buf); q->id = rfc1035BuildAQuery(name, q->buf, &q->sz); + if (0 == q->id) { + /* problem with query data -- query not sent */ + callback(data, NULL, 0); + memFree(q, MEM_IDNS_QUERY); + return; + } debug(78, 3) ("idnsALookup: buf is %d bytes for %s, id = %#hx\n", (int) q->sz, name, q->id); q->callback = callback; Index: squid/src/forward.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/forward.c,v retrieving revision 1.1.1.3.10.3.2.1 retrieving revision 1.1.1.3.10.3.2.2 diff -u -r1.1.1.3.10.3.2.1 -r1.1.1.3.10.3.2.2 --- squid/src/forward.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.3.2.1 +++ squid/src/forward.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.3.2.2 @@ -1,6 +1,6 @@ /* - * $Id: forward.c,v 1.1.1.3.10.3.2.1 2000/05/02 22:50:29 hno Exp $ + * $Id: forward.c,v 1.1.1.3.10.3.2.2 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 17 Request Forwarding * AUTHOR: Duane Wessels @@ -137,8 +137,8 @@ FwdServer *fs = fwdState->servers; FwdServer **T, *T2 = NULL; fwdState->servers = fs->next; - for (T = &fwdState->servers; *T; T2=*T, T = &(*T)->next); - if (T2 && T2->peer) { + for (T = &fwdState->servers; *T; T2 = *T, T = &(*T)->next); + if (T2 && T2->peer) { /* cycle */ *T = fs; fs->next = NULL; @@ -251,7 +251,7 @@ ctimeout = fs->peer->connect_timeout > 0 ? fs->peer->connect_timeout : Config.Timeout.peer_connect; } else if (fwdState->request->flags.accelerated && - Config.Accel.single_host && Config.Accel.host) { + Config.Accel.single_host && Config.Accel.host) { host = Config.Accel.host; port = Config.Accel.port; ctimeout = Config.Timeout.connect; Index: squid/src/globals.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/globals.h,v retrieving revision 1.1.1.3.10.3 retrieving revision 1.1.1.3.10.3.2.1 diff -u -r1.1.1.3.10.3 -r1.1.1.3.10.3.2.1 --- squid/src/globals.h 20 Apr 2000 18:14:22 -0000 1.1.1.3.10.3 +++ squid/src/globals.h 14 May 2000 23:22:19 -0000 1.1.1.3.10.3.2.1 @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.1.1.3.10.3 2000/04/20 18:14:22 hno Exp $ + * $Id: globals.h,v 1.1.1.3.10.3.2.1 2000/05/14 23:22:19 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -125,10 +125,6 @@ extern double current_dtime; extern int store_hash_buckets; /* 0 */ extern hash_table *store_table; /* NULL */ -#if HEAP_REPLACEMENT -extern heap *inmem_heap; -#else -#endif extern dlink_list ClientActiveRequests; extern const String StringNull; /* { 0, 0, NULL } */ extern const MemBuf MemBufNull; /* MemBufNULL */ @@ -149,8 +145,9 @@ extern request_flags null_request_flags; extern int store_open_disk_fd; /* 0 */ extern const char *SwapDirType[]; -extern storefs_entry_t *storefs_list; /* NULL */ +extern storefs_entry_t *storefs_list; /* NULL */ extern int store_swap_low; extern int store_swap_high; extern int store_pages_max; extern size_t store_maxobjsize; +extern RemovalPolicy *mem_policy; Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.1.1.3.10.4 retrieving revision 1.1.1.3.10.4.2.1 diff -u -r1.1.1.3.10.4 -r1.1.1.3.10.4.2.1 --- squid/src/http.c 20 Apr 2000 18:14:22 -0000 1.1.1.3.10.4 +++ squid/src/http.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.4.2.1 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.1.1.3.10.4 2000/04/20 18:14:22 hno Exp $ + * $Id: http.c,v 1.1.1.3.10.4.2.1 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -897,7 +897,7 @@ httpState->flags.keepalive = 1; if (httpState->peer) if (neighborType(httpState->peer, httpState->request) == PEER_SIBLING && - !httpState->peer->options.allow_miss) + !httpState->peer->options.allow_miss) httpState->flags.only_if_cached = 1; memBufDefInit(&mb); httpBuildRequestPrefix(req, Index: squid/src/ipc.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ipc.c,v retrieving revision 1.1.1.3.10.4 retrieving revision 1.1.1.3.10.4.2.1 diff -u -r1.1.1.3.10.4 -r1.1.1.3.10.4.2.1 --- squid/src/ipc.c 20 Apr 2000 18:14:22 -0000 1.1.1.3.10.4 +++ squid/src/ipc.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.4.2.1 @@ -1,6 +1,6 @@ /* - * $Id: ipc.c,v 1.1.1.3.10.4 2000/04/20 18:14:22 hno Exp $ + * $Id: ipc.c,v 1.1.1.3.10.4.2.1 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 54 Interprocess Communication * AUTHOR: Duane Wessels @@ -292,7 +292,7 @@ close(t2); close(t3); /* Make sure all other filedescriptors are closed */ - for(x=3;xrr_count < 0) p->rr_count = 0; p->rr_lastcount = p->rr_count; - eventAdd("peerClearRR",peerClearRR, p, 5*60, 0); + eventAdd("peerClearRR", peerClearRR, p, 5 * 60, 0); } peer * @@ -933,13 +933,13 @@ neighborUp(const peer * p) { if (!p->tcp_up) { - peerProbeConnect((peer *)p); + peerProbeConnect((peer *) p); return 0; } if (p->options.no_query) return 1; if (p->stats.probe_start != 0 && - squid_curtime - p->stats.probe_start > Config.Timeout.deadPeer) + squid_curtime - p->stats.probe_start > Config.Timeout.deadPeer) return 0; return 1; } @@ -1036,7 +1036,7 @@ } void -peerConnectFailed(peer *p) +peerConnectFailed(peer * p) { p->stats.last_connect_failure = squid_curtime; if (!p->tcp_up) { @@ -1054,7 +1054,7 @@ } void -peerConnectSucceded(peer *p) +peerConnectSucceded(peer * p) { if (!p->tcp_up) { debug(15, 2) ("TCP connection to %s/%d succeded\n", p->host, p->http_port); @@ -1070,13 +1070,13 @@ * peerProbeConnect will be called on dead peers by neighborUp */ static void -peerProbeConnect(peer *p) +peerProbeConnect(peer * p) { int fd; if (p->test_fd != -1) - return; /* probe already running */ + return; /* probe already running */ if (squid_curtime - p->stats.last_connect_probe < Config.Timeout.connect) - return; /* don't probe to often */ + return; /* don't probe to often */ fd = comm_open(SOCK_STREAM, 0, Config.Addrs.tcp_outgoing, 0, COMM_NONBLOCKING, p->host); if (fd < 0) Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.1.1.3.10.8.2.4 retrieving revision 1.1.1.3.10.8.2.5 diff -u -r1.1.1.3.10.8.2.4 -r1.1.1.3.10.8.2.5 --- squid/src/protos.h 8 May 2000 18:02:08 -0000 1.1.1.3.10.8.2.4 +++ squid/src/protos.h 14 May 2000 23:22:19 -0000 1.1.1.3.10.8.2.5 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.1.1.3.10.8.2.4 2000/05/08 18:02:08 hno Exp $ + * $Id: protos.h,v 1.1.1.3.10.8.2.5 2000/05/14 23:22:19 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -201,12 +201,12 @@ extern void disk_init(void); /* diskd.c */ -extern diskd_queue * afile_create_queue(void); +extern diskd_queue *afile_create_queue(void); extern void afile_destroy_queue(diskd_queue *); extern void afile_sync_queue(diskd_queue *); extern void afile_sync(void); extern void afile_open(const char *path, int mode, DOCB *, void *); -extern void afile_close(int fd, DCCB *callback, void *data); +extern void afile_close(int fd, DCCB * callback, void *data); extern void afile_write(int, off_t, void *, int len, DWCB *, void *, FREE *); extern void afile_write_mbuf(int fd, off_t, MemBuf, DWCB *, void *); extern void afile_read(int, char *, int, off_t, DRCB *, void *); @@ -851,19 +851,12 @@ extern void storeFsDone(void); extern void storeFsAdd(char *, STSETUP *); -/* store_heap_replacement.c */ -#ifdef HEAP_REPLACEMENT -extern heap_key HeapKeyGen_StoreEntry_LFUDA(void *, double); -extern heap_key HeapKeyGen_StoreEntry_GDSF(void *, double); -extern heap_key HeapKeyGen_StoreEntry_LRU(void *, double); -#endif - /* store_modules.c */ extern void storeFsSetup(void); /* store_io.c */ -extern storeIOState * storeCreate(StoreEntry *, STFNCB *, STIOCB *, void *); -extern storeIOState * storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *); +extern storeIOState *storeCreate(StoreEntry *, STFNCB *, STIOCB *, void *); +extern storeIOState *storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *); extern void storeClose(storeIOState *); extern void storeRead(storeIOState *, char *, size_t, off_t, STRCB *, void *); extern void storeWrite(storeIOState *, char *, size_t, off_t, FREE *); @@ -965,7 +958,7 @@ extern store_client *storeClientListAdd(StoreEntry * e, void *data); extern void storeClientCopy(store_client *, StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); extern int storeClientCopyPending(store_client *, StoreEntry * e, void *data); -extern int storeUnregister(store_client *sc, StoreEntry * e, void *data); +extern int storeUnregister(store_client * sc, StoreEntry * e, void *data); extern off_t storeLowestMemReaderOffset(const StoreEntry * entry); extern void InvokeHandlers(StoreEntry * e); extern int storePendingNClients(const StoreEntry * e); @@ -1153,7 +1146,7 @@ extern void delayBytesIn(delay_id, int qty); extern int delayMostBytesWanted(const MemObject * mem, int max); extern delay_id delayMostBytesAllowed(const MemObject * mem); -extern void delaySetStoreClient(store_client *sc, delay_id delay_id); +extern void delaySetStoreClient(store_client * sc, delay_id delay_id); extern void delayRegisterDelayIdPtr(delay_id * loc); extern void delayUnregisterDelayIdPtr(delay_id * loc); #endif @@ -1185,6 +1178,10 @@ extern void logfilePrintf(va_alist); #endif +/* Removal Policies */ +RemovalPolicy * +createRemovalPolicy(RemovalPolicySettings *settings); + /* * prototypes for system functions missing from system includes */ --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl_modules.sh Wed Feb 14 00:44:12 2007 @@ -0,0 +1,19 @@ +#!/bin/sh +echo "/* automatically generated `date` by +echo * $0 $*" +echo " * do not edit" +echo " */" +echo "#include \"squid.h\"" +echo "" +for module in "$@"; do + echo "REMOVALPOLICYCREATE createRemovalPolicy_${module};" +done +echo "RemovalPolicy * createRemovalPolicy(RemovalPolicySettings *settings)" +echo "{" +for module in "$@"; do + echo " if (strcmp(settings->type, \"${module}\") == 0)" + echo " return createRemovalPolicy_${module}(settings->args);" +done + echo " debug(20,1)(\"Unknown policy %s\n\", settings->type);" + echo " return NULL;" +echo "}" Index: squid/src/snmp_agent.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/snmp_agent.c,v retrieving revision 1.1.1.3.6.2 retrieving revision 1.1.1.3.6.2.2.1 diff -u -r1.1.1.3.6.2 -r1.1.1.3.6.2.2.1 --- squid/src/snmp_agent.c 20 Apr 2000 18:14:23 -0000 1.1.1.3.6.2 +++ squid/src/snmp_agent.c 14 May 2000 23:22:19 -0000 1.1.1.3.6.2.2.1 @@ -1,6 +1,6 @@ /* - * $Id: snmp_agent.c,v 1.1.1.3.6.2 2000/04/20 18:14:23 hno Exp $ + * $Id: snmp_agent.c,v 1.1.1.3.6.2.2.1 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 49 SNMP Interface * AUTHOR: Kostas Anagnostakis @@ -279,9 +279,9 @@ ASN_INTEGER); break; case PERF_SYS_CURLRUEXP: - /* No global LRU info anymore */ + /* No global LRU info anymore */ Answer = snmp_var_new_integer(Var->name, Var->name_length, - 0, + 0, SMI_TIMETICKS); break; case PERF_SYS_CURUNLREQ: Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.1.1.3.10.2.2.1 retrieving revision 1.1.1.3.10.2.2.2 diff -u -r1.1.1.3.10.2.2.1 -r1.1.1.3.10.2.2.2 --- squid/src/squid.h 22 Apr 2000 20:02:55 -0000 1.1.1.3.10.2.2.1 +++ squid/src/squid.h 14 May 2000 23:22:19 -0000 1.1.1.3.10.2.2.2 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.1.1.3.10.2.2.1 2000/04/22 20:02:55 hno Exp $ + * $Id: squid.h,v 1.1.1.3.10.2.2.2 2000/05/14 23:22:19 hno Exp $ * * AUTHOR: Duane Wessels * @@ -359,9 +359,6 @@ #include "hash.h" #include "rfc1035.h" -#if HEAP_REPLACEMENT -#include "heap.h" -#endif #include "defines.h" #include "enums.h" Index: squid/src/ssl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ssl.c,v retrieving revision 1.1.1.3.10.3.2.1 retrieving revision 1.1.1.3.10.3.2.2 diff -u -r1.1.1.3.10.3.2.1 -r1.1.1.3.10.3.2.2 --- squid/src/ssl.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.3.2.1 +++ squid/src/ssl.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.3.2.2 @@ -1,6 +1,6 @@ /* - * $Id: ssl.c,v 1.1.1.3.10.3.2.1 2000/05/02 22:50:29 hno Exp $ + * $Id: ssl.c,v 1.1.1.3.10.3.2.2 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 26 Secure Sockets Layer Proxy * AUTHOR: Duane Wessels @@ -382,10 +382,10 @@ sslState->servers->peer->host); else if (Config.onoff.log_ip_on_direct) hierarchyNote(&sslState->request->hier, sslState->servers->code, - fd_table[sslState->server.fd].ipaddr); + fd_table[sslState->server.fd].ipaddr); else hierarchyNote(&sslState->request->hier, sslState->servers->code, - sslState->host); + sslState->host); if (status == COMM_ERR_DNS) { debug(26, 4) ("sslConnect: Unknown host: %s\n", sslState->host); err = errorCon(ERR_DNS_FAIL, HTTP_NOT_FOUND); @@ -428,14 +428,14 @@ aclCheck_t ch; int answer; /* - * client_addr == no_addr indicates this is an "internal" request - * from peer_digest.c, asn.c, netdb.c, etc and should always - * be allowed. yuck, I know. - */ + * client_addr == no_addr indicates this is an "internal" request + * from peer_digest.c, asn.c, netdb.c, etc and should always + * be allowed. yuck, I know. + */ if (request->client_addr.s_addr != no_addr.s_addr) { /* - * Check if this host is allowed to fetch MISSES from us (miss_access) - */ + * Check if this host is allowed to fetch MISSES from us (miss_access) + */ memset(&ch, '\0', sizeof(aclCheck_t)); ch.src_addr = request->client_addr; ch.my_addr = request->my_addr; Index: squid/src/stat.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stat.c,v retrieving revision 1.1.1.3.10.4.2.1 retrieving revision 1.1.1.3.10.4.2.2 diff -u -r1.1.1.3.10.4.2.1 -r1.1.1.3.10.4.2.2 --- squid/src/stat.c 8 May 2000 18:02:08 -0000 1.1.1.3.10.4.2.1 +++ squid/src/stat.c 14 May 2000 23:22:19 -0000 1.1.1.3.10.4.2.2 @@ -1,6 +1,6 @@ /* - * $Id: stat.c,v 1.1.1.3.10.4.2.1 2000/05/08 18:02:08 hno Exp $ + * $Id: stat.c,v 1.1.1.3.10.4.2.2 2000/05/14 23:22:19 hno Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -277,8 +277,8 @@ if (mem->swapout.sio) storeAppendPrintf(s, "\tswapout: %d bytes written\n", (int) storeOffset(mem->swapout.sio)); - for (i = 0, node = mem->clients.head; node; node = node->next, i++) { - sc = (store_client *)node->data; + for (i = 0, node = mem->clients.head; node; node = node->next, i++) { + sc = (store_client *) node->data; if (sc->callback_data == NULL) continue; storeAppendPrintf(s, "\tClient #%d, %p\n", i, sc->callback_data); Index: squid/src/store.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store.c,v retrieving revision 1.1.1.3.10.8.2.3 retrieving revision 1.1.1.3.10.8.2.4 diff -u -r1.1.1.3.10.8.2.3 -r1.1.1.3.10.8.2.4 --- squid/src/store.c 8 May 2000 18:02:08 -0000 1.1.1.3.10.8.2.3 +++ squid/src/store.c 14 May 2000 23:22:20 -0000 1.1.1.3.10.8.2.4 @@ -1,6 +1,6 @@ /* - * $Id: store.c,v 1.1.1.3.10.8.2.3 2000/05/08 18:02:08 hno Exp $ + * $Id: store.c,v 1.1.1.3.10.8.2.4 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -81,6 +81,8 @@ static void destroy_MemObject(StoreEntry *); static FREE destroy_StoreEntry; static void storePurgeMem(StoreEntry *); +static void storeEntryReferenced(StoreEntry *); +static void storeEntryDereferenced(StoreEntry *); static int getKeyCounter(void); static int storeKeepInMemory(const StoreEntry *); static OBJH storeCheckCachableStats; @@ -89,14 +91,6 @@ /* * local variables */ -#if HEAP_REPLACEMENT -/* - * The heap equivalent of inmem_list, inmem_heap, is in globals.c so other - * modules can access it when updating object metadata (e.g., refcount) - */ -#else -static dlink_list inmem_list; -#endif int store_pages_max = 0; int store_swap_high = 0; int store_swap_low = 0; @@ -226,23 +220,50 @@ storeRelease(e); } -void -storeLockObject(StoreEntry * e) +static void +storeEntryReferenced(StoreEntry *e) { SwapDir *SD; - if (e->swap_dirn > -1) - SD = INDEXSD(e->swap_dirn); - else - SD = NULL; + /* Notify the fs that we're referencing this object again */ + if (e->swap_dirn > -1) { + SD = INDEXSD(e->swap_dirn); + if (SD->refobj) + SD->refobj(SD, e); + } + /* Notify the memory cache that we're referencing this object again */ + if(e->mem_obj) { + if (mem_policy->Referenced) + mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl); + } +} + +static void +storeEntryDereferenced(StoreEntry *e) +{ + SwapDir *SD; + /* Notify the fs that we're not referencing this object any more */ + if (e->swap_filen > -1) { + SD = INDEXSD(e->swap_dirn); + if (SD->unrefobj != NULL) + SD->unrefobj(SD, e); + } + /* Notify the memory cache that we're not referencing this object any more */ + if(e->mem_obj) { + if (mem_policy->Dereferenced) + mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl); + } +} + +void +storeLockObject(StoreEntry * e) +{ e->lock_count++; debug(20, 3) ("storeLockObject: key '%s' count=%d\n", storeKeyText(e->key), (int) e->lock_count); e->lastref = squid_curtime; - /* Notify the fs that we're referencing this object again */ - if (SD != NULL && SD->refobj != NULL) - SD->refobj(SD, e); + storeEntryReferenced(e); } void @@ -266,7 +287,6 @@ int storeUnlockObject(StoreEntry * e) { - SwapDir *SD; e->lock_count--; debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", storeKeyText(e->key), e->lock_count); @@ -275,30 +295,16 @@ if (e->store_status == STORE_PENDING) EBIT_SET(e->flags, RELEASE_REQUEST); assert(storePendingNClients(e) == 0); - /* Notify the fs that we're not referencing this object any more */ - if (e->swap_filen > -1) - SD = INDEXSD(e->swap_dirn); - else - SD = NULL; - if (SD != NULL && SD->unrefobj != NULL) - SD->unrefobj(SD, e); -#if HEAP_REPLACEMENT - storeHeapPositionUpdate(e, SD); -#else -#if 0 - /* Note: From 2.4. Not sure how this relates to the unrefobj() call above */ - storeDirLRUDelete(e); - storeDirLRUAdd(e); -#endif -#endif if (EBIT_TEST(e->flags, RELEASE_REQUEST)) storeRelease(e); else if (storeKeepInMemory(e)) { + storeEntryDereferenced(e); storeSetMemStatus(e, IN_MEMORY); requestUnlink(e->mem_obj->request); e->mem_obj->request = NULL; } else { storePurgeMem(e); + storeEntryDereferenced(e); if (EBIT_TEST(e->flags, KEY_PRIVATE)) debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__); } @@ -371,7 +377,7 @@ * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not * be set, and storeSetPublicKey() should not be called. */ -#if HEAP_REPLACEMENT +#if MORE_DEBUG_OUTPUT if (EBIT_TEST(e->flags, RELEASE_REQUEST)) debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n", e->key, mem->url); @@ -541,7 +547,7 @@ return 0; /* avoid release call below */ } else if ((e->mem_obj->reply->content_length > 0 && e->mem_obj->reply->content_length > Config.Store.maxObjectSize) || - e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { + e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { debug(20, 2) ("storeCheckCachable: NO: too big\n"); store_check_cachable_hist.no.too_big++; } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) { @@ -676,16 +682,7 @@ int released = 0; static time_t last_check = 0; int pages_needed; - int locked = 0; -#if !HEAP_REPLACEMENT - dlink_node *head; - dlink_node *m; - dlink_node *prev = NULL; -#else - heap_key age; - heap_key min_age = 0.0; - link_list *locked_entries = NULL; -#endif + RemovalPurgeWalker *walker; if (squid_curtime == last_check) return; last_check = squid_curtime; @@ -693,57 +690,15 @@ if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max) return; debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed); -#if HEAP_REPLACEMENT - while (heap_nodes(inmem_heap) > 0) { - age = heap_peepminkey(inmem_heap); - e = heap_extractmin(inmem_heap); - e->mem_obj->node = NULL; /* no longer in the heap */ - if (storeEntryLocked(e)) { - locked++; - debug(20, 5) ("storeGetMemSpace: locked key %s\n", - storeKeyText(e->key)); - linklistPush(&locked_entries, e); - continue; - } - released++; - debug(20, 3) ("Released memory object with key %f size %d refs %d url %s\n", - age, e->swap_file_sz, e->refcount, e->mem_obj->url); - min_age = age; + /* XXX what to set as max_scan here? */ + walker = mem_policy->PurgeInit(mem_policy, 100000); + while((e = walker->Next(walker))) { storePurgeMem(e); - if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max) - break; - } - /* - * Increase the heap age factor. - */ - if (min_age > 0) - inmem_heap->age = min_age; - /* - * Reinsert all bumped locked entries back into heap... - */ - while ((e = linklistShift(&locked_entries))) - e->mem_obj->node = heap_insert(inmem_heap, e); -#else - head = inmem_list.head; - for (m = inmem_list.tail; m; m = prev) { - if (m == head) - break; - prev = m->prev; - e = m->data; - if (storeEntryLocked(e)) { - locked++; - dlinkDelete(m, &inmem_list); - dlinkAdd(e, m, &inmem_list); - continue; - } released++; - storePurgeMem(e); if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max) break; } -#endif - debug(20, 3) ("storeGetMemSpace: released %d/%d locked %d\n", - released, hot_obj_count, locked); + walker->Done(walker); debug(20, 3) ("storeGetMemSpace stats:\n"); debug(20, 3) (" %6d HOT objects\n", hot_obj_count); debug(20, 3) (" %6d were released\n", released); @@ -765,15 +720,25 @@ { int i; SwapDir *SD; + static time_t last_warn_time = 0; /* walk each fs */ for (i = 0; i < Config.cacheSwap.n_configured; i++) { /* call the maintain function .. */ SD = INDEXSD(i); + /* XXX FixMe: This should be done "in parallell" on the different + * cache_dirs, not one at a time. + */ if (SD->maintainfs != NULL) SD->maintainfs(SD); } - + if (store_swap_size > Config.Swap.maxSize) { + if (squid_curtime - last_warn_time > 10) { + debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", + store_swap_size, Config.Swap.maxSize); + last_warn_time = squid_curtime; + } + } /* Reregister a maintain event .. */ eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1); } @@ -938,11 +903,9 @@ storeInitHashValues(); store_table = hash_create(storeKeyHashCmp, store_hash_buckets, storeKeyHashHash); + mem_policy = createRemovalPolicy(Config.memPolicy); storeDigestInit(); storeLogOpen(); -#if HEAP_REPLACEMENT - inmem_heap = new_heap(1000, HeapKeyGen_StoreEntry_GDSF); -#endif stackInit(&LateReleaseStack); eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); storeDirInit(); @@ -1126,35 +1089,24 @@ assert(mem != NULL); if (new_status == IN_MEMORY) { assert(mem->inmem_lo == 0); -#if HEAP_REPLACEMENT - if (mem->node == NULL) { - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - debug(20, 4) ("storeSetMemStatus: not inserting special %s\n", - mem->url); - } else { - mem->node = heap_insert(inmem_heap, e); - debug(20, 4) ("storeSetMemStatus: inserted mem node %p\n", - mem->node); - } + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n", + mem->url); + } else { + mem_policy->Add(mem_policy, e, &mem->repl); + debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n", + mem->url); } -#else - dlinkAdd(e, &mem->lru, &inmem_list); -#endif hot_obj_count++; } else { -#if HEAP_REPLACEMENT - /* - * It's being removed from the memory heap; is it already gone? - */ - if (mem->node) { - heap_delete(inmem_heap, mem->node); - debug(20, 4) ("storeSetMemStatus: deleted mem node %p\n", - mem->node); - mem->node = NULL; + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + debug(20, 4) ("storeSetMemStatus: special entry %s\n", + mem->url); + } else { + mem_policy->Remove(mem_policy, e, &mem->repl); + debug(20, 4) ("storeSetMemStatus: removed mem node %s\n", + mem->url); } -#else - dlinkDelete(&mem->lru, &inmem_list); -#endif hot_obj_count--; } e->mem_status = new_status; @@ -1233,18 +1185,6 @@ e->expires = e->lastmod = e->timestamp = -1; } -#if HEAP_REPLACEMENT -/* - * This routine only handles memory updates these days - */ -void -storeHeapPositionUpdate(StoreEntry * e, SwapDir * SD) -{ - if (e->mem_obj && e->mem_obj->node) - heap_update(inmem_heap, e->mem_obj->node, e); -} -#endif - /* * storeFsInit * @@ -1269,7 +1209,7 @@ int i = 0; while (storefs_list[i].typestr != NULL) { - storefs_list[i].donefunc(); + storefs_list[i].donefunc(); i++; } } @@ -1277,16 +1217,17 @@ /* * called to add another store fs module */ -void storeFsAdd(char *type, STSETUP *setup) +void +storeFsAdd(char *type, STSETUP * setup) { int i; /* find the number of currently known storefs types */ for (i = 0; storefs_list && storefs_list[i].typestr; i++) { - assert(strcmp(storefs_list[i].typestr, type)!=0); + assert(strcmp(storefs_list[i].typestr, type) != 0); } /* add the new type */ storefs_list = xrealloc(storefs_list, (i + 2) * sizeof(storefs_entry_t)); - memset(&storefs_list[i+1],0,sizeof(storefs_entry_t)); + memset(&storefs_list[i + 1], 0, sizeof(storefs_entry_t)); storefs_list[i].typestr = type; /* Call the FS to set up capabilities and initialize the FS driver */ setup(&storefs_list[i]); Index: squid/src/store_client.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_client.c,v retrieving revision 1.1.1.3.10.4.2.2 retrieving revision 1.1.1.3.10.4.2.3 diff -u -r1.1.1.3.10.4.2.2 -r1.1.1.3.10.4.2.3 --- squid/src/store_client.c 8 May 2000 18:02:08 -0000 1.1.1.3.10.4.2.2 +++ squid/src/store_client.c 14 May 2000 23:22:20 -0000 1.1.1.3.10.4.2.3 @@ -1,6 +1,6 @@ /* - * $Id: store_client.c,v 1.1.1.3.10.4.2.2 2000/05/08 18:02:08 hno Exp $ + * $Id: store_client.c,v 1.1.1.3.10.4.2.3 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 20 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -59,7 +59,7 @@ dlink_node *node; store_client *sc; for (node = mem->clients.head; node; node = node->next) { - sc = (store_client *)node->data; + sc = (store_client *) node->data; if (sc->callback_data != NULL) return 1; } @@ -73,7 +73,7 @@ dlink_node *node; store_client *sc = NULL; for (node = mem->clients.head; node; node = node->next) { - sc = (store_client *)node->data; + sc = (store_client *) node->data; if (sc->callback_data == data) return sc; } @@ -129,7 +129,7 @@ assert(mem); #if STORE_CLIENT_LIST_DEBUG if (storeClientListSearch(mem, data) != NULL) - assert(1==0); /* XXX die! */ + assert(1 == 0); /* XXX die! */ #endif e->refcount++; mem->nclients++; @@ -178,7 +178,7 @@ /* copy bytes requested by the client */ void -storeClientCopy(store_client *sc, +storeClientCopy(store_client * sc, StoreEntry * e, off_t seen_offset, off_t copy_offset, @@ -317,9 +317,9 @@ /* What the client wants is in memory */ debug(20, 3) ("storeClientCopy3: Copying from memory\n"); sz = stmemCopy(&mem->data_hdr, - sc->copy_offset, sc->copy_buf, sc->copy_size); - storeClientCallback(sc, sz); - return; + sc->copy_offset, sc->copy_buf, sc->copy_size); + storeClientCallback(sc, sz); + return; } /* What the client wants is not in memory. Schedule a disk read */ assert(STORE_DISK_CLIENT == sc->type); @@ -467,7 +467,7 @@ } int -storeClientCopyPending(store_client *sc, StoreEntry * e, void *data) +storeClientCopyPending(store_client * sc, StoreEntry * e, void *data) { #if STORE_CLIENT_LIST_DEBUG assert(sc == storeClientListSearch(e->mem_obj, data)); @@ -485,50 +485,50 @@ * passed sc. Yet. */ int -storeUnregister(store_client *sc, StoreEntry * e, void *data) +storeUnregister(store_client * sc, StoreEntry * e, void *data) { MemObject *mem = e->mem_obj; #if STORE_CLIENT_LIST_DEBUG assert(sc == storeClientListSearch(e->mem_obj, data)); #endif if (mem == NULL) - return 0; + return 0; debug(20, 3) ("storeUnregister: called for '%s'\n", storeKeyText(e->key)); if (sc == NULL) - return 0; + return 0; if (mem->clients.head == NULL) - return 0; - if (sc == (store_client *)mem->clients.head->data) { - /* - * If we are unregistering the _first_ client for this - * entry, then we have to reset the client FD to -1. - */ - mem->fd = -1; + return 0; + if (sc == (store_client *) mem->clients.head->data) { + /* + * If we are unregistering the _first_ client for this + * entry, then we have to reset the client FD to -1. + */ + mem->fd = -1; } dlinkDelete(&sc->node, &mem->clients); mem->nclients--; if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) - storeSwapOut(e); + storeSwapOut(e); if (sc->swapin_sio) { - storeClose(sc->swapin_sio); - cbdataUnlock(sc->swapin_sio); - sc->swapin_sio = NULL; + storeClose(sc->swapin_sio); + cbdataUnlock(sc->swapin_sio); + sc->swapin_sio = NULL; } if (NULL != sc->callback) { - /* callback with ssize = -1 to indicate unexpected termination */ - debug(20, 3) ("storeUnregister: store_client for %s has a callback\n", - mem->url); - storeClientCallback(sc, -1); + /* callback with ssize = -1 to indicate unexpected termination */ + debug(20, 3) ("storeUnregister: store_client for %s has a callback\n", + mem->url); + storeClientCallback(sc, -1); } #if DELAY_POOLS delayUnregisterDelayIdPtr(&sc->delay_id); #endif - cbdataUnlock(sc->callback_data); /* we're done with it now */ + cbdataUnlock(sc->callback_data); /* we're done with it now */ /*assert(!sc->flags.disk_io_pending); */ cbdataFree(sc); assert(e->lock_count > 0); if (mem->nclients == 0) - CheckQuickAbort(e); + CheckQuickAbort(e); return 1; } @@ -542,7 +542,7 @@ dlink_node *node; for (node = mem->clients.head; node; node = nx) { - sc = (store_client *)node->data; + sc = (store_client *) node->data; nx = node->next; if (sc->callback_data == NULL) /* open slot */ continue; @@ -570,7 +570,7 @@ debug(20, 3) ("InvokeHandlers: %s\n", storeKeyText(e->key)); /* walk the entire list looking for valid callbacks */ for (node = mem->clients.head; node; node = nx) { - sc = (store_client *)node->data; + sc = (store_client *) node->data; nx = node->next; debug(20, 3) ("InvokeHandlers: checking client #%d\n", i++); if (sc->callback_data == NULL) Index: squid/src/store_digest.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_digest.c,v retrieving revision 1.1.1.3.6.1.2.1 retrieving revision 1.1.1.3.6.1.2.2 diff -u -r1.1.1.3.6.1.2.1 -r1.1.1.3.6.1.2.2 --- squid/src/store_digest.c 8 May 2000 18:02:08 -0000 1.1.1.3.6.1.2.1 +++ squid/src/store_digest.c 14 May 2000 23:22:20 -0000 1.1.1.3.6.1.2.2 @@ -1,5 +1,5 @@ /* - * $Id: store_digest.c,v 1.1.1.3.6.1.2.1 2000/05/08 18:02:08 hno Exp $ + * $Id: store_digest.c,v 1.1.1.3.6.1.2.2 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 71 Store Digest Manager * AUTHOR: Alex Rousskov @@ -230,12 +230,10 @@ * idea: skip objects that are going to be purged before the next * update. */ -#if 0 /* This code isn't applicable anymore, we can't fix it atm either :( */ -#if !HEAP_REPLACEMENT +#if OLD_UNUSED_CODE /* This code isn't applicable anymore, we can't fix it atm either :( */ if ((squid_curtime + Config.digest.rebuild_period) - e->lastref > storeExpiredReferenceAge()) return 0; #endif -#endif return 1; } Index: squid/src/store_dir.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_dir.c,v retrieving revision 1.1.1.3.10.6.2.1 retrieving revision 1.1.1.3.10.6.2.2 diff -u -r1.1.1.3.10.6.2.1 -r1.1.1.3.10.6.2.2 --- squid/src/store_dir.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.6.2.1 +++ squid/src/store_dir.c 14 May 2000 23:22:20 -0000 1.1.1.3.10.6.2.2 @@ -1,6 +1,6 @@ /* - * $Id: store_dir.c,v 1.1.1.3.10.6.2.1 2000/05/02 22:50:29 hno Exp $ + * $Id: store_dir.c,v 1.1.1.3.10.6.2.2 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -36,8 +36,6 @@ #include "squid.h" static int storeDirValidSwapDirSize(int, size_t); -static void storeDirLRUWalkInitHead(SwapDir * sd); -static void *storeDirLRUWalkNext(SwapDir * sd); void storeDirInit(void) @@ -88,18 +86,18 @@ * If the swapdir's max_obj_size is -1, then it definitely can */ if (Config.cacheSwap.swapDirs[swapdir].max_objsize == -1) - return 1; + return 1; /* * Else, make sure that the max object size is larger than objsize */ if (Config.cacheSwap.swapDirs[swapdir].max_objsize > objsize) - return 1; + return 1; else - return 0; + return 0; } -#if UNUSED /* Squid-2..4.DEVEL3 code */ +#if UNUSED /* Squid-2..4.DEVEL3 code */ /* * This new selection scheme simply does round-robin on all SwapDirs. * A SwapDir is skipped if it is over the max_size (100%) limit. If @@ -179,7 +177,7 @@ * we sort out the real usefulness of this algorithm. */ int -storeDirSelectSwapDir(const StoreEntry *e) +storeDirSelectSwapDir(const StoreEntry * e) { size_t objsize; size_t least_size; @@ -198,28 +196,28 @@ least_size = Config.cacheSwap.swapDirs[0].cur_size; least_objsize = Config.cacheSwap.swapDirs[0].max_objsize; for (i = 0; i < Config.cacheSwap.n_configured; i++) { - SD = &Config.cacheSwap.swapDirs[i]; + SD = &Config.cacheSwap.swapDirs[i]; SD->flags.selected = 0; - if (SD->flags.read_only) - continue; - /* Valid for object size check */ - if (!storeDirValidSwapDirSize(i, objsize)) - continue; - load = SD->checkobj(SD, e); - if (load < 0) - continue; - if (SD->cur_size > SD->max_size) - continue; - if (load > least_load) - continue; - if ((least_objsize > 0) && (objsize > least_objsize)) - continue; - /* Only use leastsize if the load is equal */ - if ((load == least_load) && (SD->cur_size > least_size)) - continue; - least_load = load; - least_size = SD->cur_size; - dirn = i; + if (SD->flags.read_only) + continue; + /* Valid for object size check */ + if (!storeDirValidSwapDirSize(i, objsize)) + continue; + load = SD->checkobj(SD, e); + if (load < 0) + continue; + if (SD->cur_size > SD->max_size) + continue; + if (load > least_load) + continue; + if ((least_objsize > 0) && (objsize > least_objsize)) + continue; + /* Only use leastsize if the load is equal */ + if ((load == least_load) && (SD->cur_size > least_size)) + continue; + least_load = load; + least_size = SD->cur_size; + dirn = i; } if (dirn >= 0) @@ -261,14 +259,14 @@ debug(20, 3) ("storeDirSwapLog: %s %s %d %08X\n", swap_log_op_str[op], storeKeyText(e->key), - e->swap_dirn, + e->swap_dirn, e->swap_filen); sd = &Config.cacheSwap.swapDirs[e->swap_dirn]; sd->log.write(sd, e, op); } void -storeDirUpdateSwapSize(SwapDir *SD, size_t size, int sign) +storeDirUpdateSwapSize(SwapDir * SD, size_t size, int sign) { int k = ((size + 1023) >> 10) * sign; SD->cur_size += k; @@ -298,11 +296,11 @@ /* Now go through each swapdir, calling its statfs routine */ for (i = 0; i < Config.cacheSwap.n_configured; i++) { - storeAppendPrintf(sentry, "\n"); - SD = &(Config.cacheSwap.swapDirs[i]); - storeAppendPrintf(sentry, "Store Directory #%d (%s): %s\n", i, SD->type, - storeSwapDir(i)); - SD->statfs(SD, sentry); + storeAppendPrintf(sentry, "\n"); + SD = &(Config.cacheSwap.swapDirs[i]); + storeAppendPrintf(sentry, "Store Directory #%d (%s): %s\n", i, SD->type, + storeSwapDir(i)); + SD->statfs(SD, sentry); } } @@ -370,10 +368,9 @@ struct timeval start; double dt; SwapDir *sd; + RemovalPolicyWalker **walkers; int dirn; -#if HEAP_REPLACEMENT - int node; -#endif + int notdone = 1; if (store_dirs_rebuilding) { debug(20, 1) ("Not currently OK to rewrite swap log.\n"); debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n"); @@ -382,28 +379,24 @@ debug(20, 1) ("storeDirWriteCleanLogs: Starting...\n"); getCurrentTime(); start = current_time; + walkers = xcalloc(Config.cacheSwap.n_configured, sizeof *walkers); for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { sd = &Config.cacheSwap.swapDirs[dirn]; - if (sd->log.clean.open(sd) < 0) { - debug(20, 1) ("log.clean.open() failed for dir #%d\n", sd->index); + if (sd->log.clean.start(sd) < 0) { + debug(20, 1) ("log.clean.start() failed for dir #%d\n", sd->index); continue; } - if (NULL == sd->log.clean.write) - continue; -#if HEAP_REPLACEMENT - if (NULL == sd->repl.heap.heap) - continue; -#endif -#if HEAP_REPLACEMENT - for (node = 0; node < heap_nodes(sd->repl.heap.heap); node++) -#else - storeDirLRUWalkInitHead(sd); - while ((e = storeDirLRUWalkNext(sd)) != NULL) -#endif - { -#if HEAP_REPLACEMENT - e = (StoreEntry *) heap_peep(sd->repl.heap.heap, node); -#endif + } + while(notdone) { + notdone = 0; + for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { + sd = &Config.cacheSwap.swapDirs[dirn]; + if (NULL == sd->log.clean.write) + continue; + e = sd->log.clean.nextentry(sd); + if (!e) + continue; + notdone = 1; if (e->swap_filen < 0) continue; if (e->swap_status != SWAPOUT_DONE) @@ -416,14 +409,17 @@ continue; if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) continue; - sd->log.clean.write(e, sd); + sd->log.clean.write(sd, e); if ((++n & 0xFFFF) == 0) { getCurrentTime(); debug(20, 1) (" %7d entries written so far.\n", n); } } - /* Flush */ - sd->log.clean.write(NULL, sd); + } + /* Flush */ + for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { + sd = &Config.cacheSwap.swapDirs[dirn]; + sd->log.clean.done(sd); } if (reopen) storeDirOpenSwapLogs(); @@ -446,9 +442,9 @@ SwapDir *SD; for (i = 0; i < Config.cacheSwap.n_configured; i++) { - SD = &Config.cacheSwap.swapDirs[i]; - if (SD->sync != NULL) - SD->sync(SD); + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->sync != NULL) + SD->sync(SD); } } @@ -462,47 +458,8 @@ SwapDir *SD; for (i = 0; i < Config.cacheSwap.n_configured; i++) { - SD = &Config.cacheSwap.swapDirs[i]; - if (SD->callback != NULL) - SD->callback(SD); + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->callback != NULL) + SD->callback(SD); } } - -#if 0 /* from Squid-2.4.DEVEL3 */ -void -storeDirLRUDelete(StoreEntry * e) -{ - SwapDir *sd; - if (e->swap_filen < 0) - return; - sd = &Config.cacheSwap.swapDirs[e->swap_dirn]; - dlinkDelete(&e->lru, &sd->repl.lru.list); -} - -void -storeDirLRUAdd(StoreEntry * e) -{ - SwapDir *sd; - if (e->swap_file_number < 0) - return; - sd = &Config.cacheSwap.swapDirs[e->swap_dirn]; - dlinkAdd(e, &e->lru, &sd->repl.lru.list); -} -#endif /* from Squid-2.4.DEVEL3 */ - -static void -storeDirLRUWalkInitHead(SwapDir * sd) -{ - sd->repl.lru.walker = sd->repl.lru.list.head; -} - -static void * -storeDirLRUWalkNext(SwapDir * sd) -{ - void *p; - if (NULL == sd->repl.lru.walker) - return NULL; - p = sd->repl.lru.walker->data; - sd->repl.lru.walker = sd->repl.lru.walker->next; - return p; -} --- squid/src/store_heap_replacement.c Wed Feb 14 00:44:13 2007 +++ /dev/null Wed Feb 14 00:44:00 2007 @@ -1,111 +0,0 @@ - -/* - * $Id: store_heap_replacement.c,v 1.1.1.1.6.2 2000/04/20 18:14:23 hno Exp $ - * - * DEBUG: section 20 Storage Manager Heap-based replacement - * AUTHOR: John Dilley - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -/* - * The code in this file is Copyrighted (C) 1999 by Hewlett Packard. - * - * - * For a description of these cache replacement policies see -- - * http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html - */ - -#include "squid.h" - -#if HEAP_REPLACEMENT - -/* - * Key generation function to implement the LFU-DA policy (Least - * Frequently Used with Dynamic Aging). Similar to classical LFU - * but with aging to handle turnover of the popular document set. - * Maximizes byte hit rate by keeping more currently popular objects - * in cache regardless of size. Achieves lower hit rate than GDS - * because there are more large objects in cache (so less room for - * smaller popular objects). - * - * This version implements a tie-breaker based upon recency - * (e->lastref): for objects that have the same reference count - * the most recent object wins (gets a higher key value). - */ -heap_key -HeapKeyGen_StoreEntry_LFUDA(void *entry, double age) -{ - StoreEntry *e = entry; - double tie; - if (e->lastref <= 0) - tie = 0.0; - else if (squid_curtime <= e->lastref) - tie = 0.0; - else - tie = 1.0 - exp((double) (e->lastref - squid_curtime) / 86400.0); - return age + (double) e->refcount - tie; -} - - -/* - * Key generation function to implement the GDS-Frequency policy. - * Similar to Greedy Dual-Size Hits policy, but adds aging of - * documents to prevent pollution. Maximizes object hit rate by - * keeping more small, popular objects in cache. Achieves lower - * byte hit rate than LFUDA because there are fewer large objects - * in cache. - * - * This version implements a tie-breaker based upon recency - * (e->lastref): for objects that have the same reference count - * the most recent object wins (gets a higher key value). - */ -heap_key -HeapKeyGen_StoreEntry_GDSF(void *entry, double age) -{ - StoreEntry *e = entry; - double size = e->swap_file_sz ? (double) e->swap_file_sz : 1.0; - double tie = (e->lastref > 1) ? (1.0 / e->lastref) : 1.0; - return age + ((double) e->refcount / size) - tie; -} - -/* - * Key generation function to implement the LRU policy. Normally - * one would not do this with a heap -- use the linked list instead. - * For testing and performance characterization it was useful. - * Don't use it unless you are trying to compare performance among - * heap-based replacement policies... - */ -heap_key -HeapKeyGen_StoreEntry_LRU(void *entry, double age) -{ - StoreEntry *e = entry; - return (heap_key) e->lastref; -} - -#endif Index: squid/src/store_io.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_io.c,v retrieving revision 1.1.1.1.10.6 retrieving revision 1.1.1.1.10.6.2.1 diff -u -r1.1.1.1.10.6 -r1.1.1.1.10.6.2.1 --- squid/src/store_io.c 11 Apr 2000 18:18:40 -0000 1.1.1.1.10.6 +++ squid/src/store_io.c 14 May 2000 23:22:20 -0000 1.1.1.1.10.6.2.1 @@ -8,7 +8,7 @@ * to select different polices depending on object size or type. */ storeIOState * -storeCreate(StoreEntry *e, STIOCB *file_callback, STIOCB *close_callback, void *callback_data) +storeCreate(StoreEntry * e, STIOCB * file_callback, STIOCB * close_callback, void *callback_data) { size_t objsize; sdirno dirn; @@ -17,7 +17,7 @@ /* This is just done for logging purposes */ objsize = objectLen(e); if (objsize != -1) - objsize += e->mem_obj->swap_hdr_sz; + objsize += e->mem_obj->swap_hdr_sz; /* * Pick the swapdir @@ -25,14 +25,14 @@ */ dirn = storeDirSelectSwapDir(e); if (dirn == -1) { - debug(20, 2) ("storeCreate: no valid swapdirs for this object\n"); - return NULL; + debug(20, 2) ("storeCreate: no valid swapdirs for this object\n"); + return NULL; } - debug (20, 2) ("storeCreate: Selected dir '%d' for obj size '%d'\n", dirn, objsize); + debug(20, 2) ("storeCreate: Selected dir '%d' for obj size '%d'\n", dirn, objsize); SD = &Config.cacheSwap.swapDirs[dirn]; /* Now that we have a fs to use, call its storeCreate function */ - return(SD->obj.create(SD, e, file_callback, close_callback, callback_data)); + return (SD->obj.create(SD, e, file_callback, close_callback, callback_data)); /* Done */ } @@ -42,8 +42,8 @@ * storeOpen() is purely for reading .. */ storeIOState * -storeOpen(StoreEntry *e, STFNCB * file_callback, STIOCB * callback, - void *callback_data) +storeOpen(StoreEntry * e, STFNCB * file_callback, STIOCB * callback, + void *callback_data) { SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn]; return SD->obj.open(SD, e, file_callback, callback, callback_data); @@ -74,7 +74,7 @@ } void -storeUnlink(StoreEntry *e) +storeUnlink(StoreEntry * e) { SwapDir *SD = INDEXSD(e->swap_dirn); SD->obj.unlink(SD, e); Index: squid/src/store_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_log.c,v retrieving revision 1.1.1.2.10.2 retrieving revision 1.1.1.2.10.2.2.1 diff -u -r1.1.1.2.10.2 -r1.1.1.2.10.2.2.1 --- squid/src/store_log.c 20 Apr 2000 18:14:23 -0000 1.1.1.2.10.2 +++ squid/src/store_log.c 14 May 2000 23:22:20 -0000 1.1.1.2.10.2.2.1 @@ -1,6 +1,6 @@ /* - * $Id: store_log.c,v 1.1.1.2.10.2 2000/04/20 18:14:23 hno Exp $ + * $Id: store_log.c,v 1.1.1.2.10.2.2.1 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 20 Storage Manager Logging Functions * AUTHOR: Duane Wessels @@ -71,7 +71,7 @@ (int) current_time.tv_sec, (int) current_time.tv_usec / 1000, storeLogTags[tag], - e->swap_dirn, + e->swap_dirn, e->swap_filen, reply->sline.status, (int) reply->date, Index: squid/src/store_swapin.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_swapin.c,v retrieving revision 1.1.1.3.10.4 retrieving revision 1.1.1.3.10.4.2.1 diff -u -r1.1.1.3.10.4 -r1.1.1.3.10.4.2.1 --- squid/src/store_swapin.c 20 Apr 2000 18:14:23 -0000 1.1.1.3.10.4 +++ squid/src/store_swapin.c 14 May 2000 23:22:20 -0000 1.1.1.3.10.4.2.1 @@ -1,6 +1,6 @@ /* - * $Id: store_swapin.c,v 1.1.1.3.10.4 2000/04/20 18:14:23 hno Exp $ + * $Id: store_swapin.c,v 1.1.1.3.10.4.2.1 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 20 Storage Manager Swapin Functions * AUTHOR: Duane Wessels @@ -62,7 +62,7 @@ debug(20, 3) ("storeSwapInStart: Opening fileno %08X\n", e->swap_filen); sc->swapin_sio = storeOpen(e, storeSwapInFileNotify, storeSwapInFileClosed, - sc); + sc); cbdataLock(sc->swapin_sio); } @@ -85,11 +85,11 @@ static void storeSwapInFileNotify(void *data, int errflag, storeIOState * sio) { - store_client *sc = data; - StoreEntry *e = sc->entry; - - debug(1, 3) ("storeSwapInFileNotify: changing %d/%d to %d/%d\n", e->swap_filen, e->swap_dirn, sio->swap_filen, sio->swap_dirn); + store_client *sc = data; + StoreEntry *e = sc->entry; + + debug(1, 3) ("storeSwapInFileNotify: changing %d/%d to %d/%d\n", e->swap_filen, e->swap_dirn, sio->swap_filen, sio->swap_dirn); - e->swap_filen = sio->swap_filen; - e->swap_dirn = sio->swap_dirn; + e->swap_filen = sio->swap_filen; + e->swap_dirn = sio->swap_dirn; } Index: squid/src/store_swapout.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_swapout.c,v retrieving revision 1.1.1.3.10.5.2.1 retrieving revision 1.1.1.3.10.5.2.2 diff -u -r1.1.1.3.10.5.2.1 -r1.1.1.3.10.5.2.2 --- squid/src/store_swapout.c 8 May 2000 18:02:08 -0000 1.1.1.3.10.5.2.1 +++ squid/src/store_swapout.c 14 May 2000 23:22:20 -0000 1.1.1.3.10.5.2.2 @@ -1,6 +1,6 @@ /* - * $Id: store_swapout.c,v 1.1.1.3.10.5.2.1 2000/05/08 18:02:08 hno Exp $ + * $Id: store_swapout.c,v 1.1.1.3.10.5.2.2 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 20 Storage Manager Swapout Functions * AUTHOR: Duane Wessels @@ -71,8 +71,8 @@ xfree(buf); return; } - storeLockObject(e); /* Don't lock until after create, or the replacement - * code might get confused */ + storeLockObject(e); /* Don't lock until after create, or the replacement + * code might get confused */ /* Pick up the file number if it was assigned immediately */ e->swap_filen = mem->swapout.sio->swap_filen; e->swap_dirn = mem->swapout.sio->swap_dirn; @@ -134,8 +134,8 @@ */ swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) { - debug (20, 5) ("storeSwapOut: Deferring starting swapping out\n"); - return; + debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n"); + return; } /* * Careful. lowest_offset can be greater than inmem_hi, such @@ -194,8 +194,8 @@ * than the maximum object size (so we pick a -1 maxobjsize fs) */ if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) { - debug (20, 5) ("storeSwapOut: Deferring starting swapping out\n"); - return; + debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n"); + return; } /* Ok, we have stuff to swap out. Is there a swapout.sio open? */ if (e->swap_status == SWAPOUT_NONE) { @@ -210,18 +210,18 @@ if (NULL == mem->swapout.sio) return; do { - /* + /* * Evil hack time. * We are paging out to disk in page size chunks. however, later on when * we update the queue position, we might not have a page (I *think*), * so we do the actual page update here. - */ + */ - if (mem->swapout.memnode == NULL) { - /* We need to swap out the first page */ + if (mem->swapout.memnode == NULL) { + /* We need to swap out the first page */ mem->swapout.memnode = mem->data_hdr.head; } else { - /* We need to swap out the next page */ + /* We need to swap out the next page */ mem->swapout.memnode = mem->swapout.memnode->next; } /* @@ -292,8 +292,8 @@ } if (e->swap_filen > 0) storeUnlink(e); - e->swap_filen = -1; - e->swap_dirn = -1; + e->swap_filen = -1; + e->swap_dirn = -1; e->swap_status = SWAPOUT_NONE; storeReleaseRequest(e); } else { @@ -356,8 +356,8 @@ * even if its not cachable */ for (node = e->mem_obj->clients.head; node; node = node->next) { - if (((store_client *)node->data)->type == STORE_DISK_CLIENT) - return 1; + if (((store_client *) node->data)->type == STORE_DISK_CLIENT) + return 1; } if (store_dirs_rebuilding) if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.1.1.3.10.8.2.5 retrieving revision 1.1.1.3.10.8.2.6 diff -u -r1.1.1.3.10.8.2.5 -r1.1.1.3.10.8.2.6 --- squid/src/structs.h 8 May 2000 18:02:08 -0000 1.1.1.3.10.8.2.5 +++ squid/src/structs.h 14 May 2000 23:22:20 -0000 1.1.1.3.10.8.2.6 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.1.1.3.10.8.2.5 2000/05/08 18:02:08 hno Exp $ + * $Id: structs.h,v 1.1.1.3.10.8.2.6 2000/05/14 23:22:20 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -210,6 +210,11 @@ #endif +struct _RemovalPolicySettings { + char *type; + wordlist *args; +}; + struct _SquidConfig { struct { size_t maxSize; @@ -227,14 +232,8 @@ int pct; size_t max; } quickAbort; -#if HEAP_REPLACEMENT - char *replPolicy; -#else - /* - * Note: the non-LRU policies do not use referenceAge, but we cannot - * remove it until we find out how to implement #else for cf_parser.c - */ -#endif + RemovalPolicySettings *replPolicy; + RemovalPolicySettings *memPolicy; time_t referenceAge; time_t negativeTtl; time_t negativeDnsTtl; @@ -494,12 +493,6 @@ int rebuild_chunk_percentage; } digest; #endif -#if USE_DISKD - struct { - int magic1; - int magic2; - } diskd; -#endif }; struct _SquidConfig2 { @@ -861,6 +854,7 @@ ConnStateData *conn; request_t *request; /* Parsed URL ... */ store_client *sc; /* The store_client we're using */ + store_client *old_sc; /* ... for entry to be validated */ char *uri; char *log_uri; struct { @@ -1225,6 +1219,7 @@ int psize; char payload[PINGER_PAYLOAD_SZ]; }; + #endif struct _icp_common_t { @@ -1282,6 +1277,39 @@ }; +/* Removal policies */ + +struct _RemovalPolicyNode { + void *data; +}; + +struct _RemovalPolicy { + char *_type; + void *_data; + void (*Free)(RemovalPolicy *policy); + void (*Add)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node); + void (*Remove)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node); + void (*Referenced)(RemovalPolicy *policy, const StoreEntry *entry, RemovalPolicyNode *node); + void (*Dereferenced)(RemovalPolicy *policy, const StoreEntry *entry, RemovalPolicyNode *node); + RemovalPolicyWalker *(*WalkInit)(RemovalPolicy *policy); + RemovalPurgeWalker *(*PurgeInit)(RemovalPolicy *policy, int max_scan); +}; + +struct _RemovalPolicyWalker { + RemovalPolicy *_policy; + void *_data; + const StoreEntry *(*Next)(RemovalPolicyWalker *walker); + void (*Done)(RemovalPolicyWalker *walker); +}; + +struct _RemovalPurgeWalker { + RemovalPolicy *_policy; + void *_data; + int scanned, max_scan, locked; + StoreEntry *(*Next)(RemovalPurgeWalker *walker); + void (*Done)(RemovalPurgeWalker *walker); +}; + /* This structure can be freed while object is purged out from memory */ struct _MemObject { method_t method; @@ -1293,7 +1321,7 @@ int nclients; struct { off_t queue_offset; /* relative to in-mem data */ - mem_node *memnode; /* which node we're currently paging out */ + mem_node *memnode; /* which node we're currently paging out */ storeIOState *sio; } swapout; HttpReply *reply; @@ -1307,14 +1335,7 @@ void *data; } abort; char *log_url; -#if HEAP_REPLACEMENT - /* - * A MemObject knows where it is in the in-memory heap. - */ - heap_node *node; -#else - dlink_node lru; -#endif + RemovalPolicyNode repl; int id; ssize_t object_sz; size_t swap_hdr_sz; @@ -1337,12 +1358,7 @@ u_short flags; sdirno swap_dirn; sfileno swap_filen; - union { -#ifdef HEAP_REPLACEMENT - heap_node *node; -#endif - dlink_node lru; - } repl; + RemovalPolicyNode repl; u_short lock_count; /* Assume < 65536! */ mem_status_t mem_status:3; ping_status_t ping_status:3; @@ -1359,17 +1375,7 @@ int index; /* This entry's index into the swapDirs array */ int suggest; size_t max_objsize; - union { -#ifdef HEAP_REPLACEMENT - struct { - heap *heap; - } heap; -#endif - struct { - dlink_list list; - dlink_node *walker; - } lru; - } repl; + RemovalPolicy *repl; int removals; int scanned; struct { @@ -1390,7 +1396,7 @@ STCALLBACK *callback; /* Handle pending callbacks */ STSYNC *sync; /* Sync the directory */ struct { - STOBJCREATE *create; + STOBJCREATE *create; STOBJOPEN *open; STOBJCLOSE *close; STOBJREAD *read; @@ -1402,8 +1408,10 @@ STLOGCLOSE *close; STLOGWRITE *write; struct { - STLOGCLEANOPEN *open; + STLOGCLEANSTART *start; + STLOGCLEANNEXTENTRY *nextentry; STLOGCLEANWRITE *write; + STLOGCLEANDONE *done; void *state; } clean; } log; @@ -1905,7 +1913,7 @@ int recv_count; /* number of messages received */ struct { char *buf; /* shm buffer */ - link_list *stack; + link_list *stack; int id; /* sysvshm id */ } shm; }; @@ -1917,3 +1925,4 @@ size_t bufsz; off_t offset; }; + Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.1.1.3.10.6.2.1 retrieving revision 1.1.1.3.10.6.2.2 diff -u -r1.1.1.3.10.6.2.1 -r1.1.1.3.10.6.2.2 --- squid/src/typedefs.h 2 May 2000 22:50:29 -0000 1.1.1.3.10.6.2.1 +++ squid/src/typedefs.h 14 May 2000 23:22:20 -0000 1.1.1.3.10.6.2.2 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.1.1.3.10.6.2.1 2000/05/02 22:50:29 hno Exp $ + * $Id: typedefs.h,v 1.1.1.3.10.6.2.2 2000/05/14 23:22:20 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -171,6 +171,11 @@ typedef struct _storefs_entry storefs_entry_t; typedef struct _diskd_queue diskd_queue; typedef struct _Logfile Logfile; +typedef struct _RemovalPolicy RemovalPolicy; +typedef struct _RemovalPolicyWalker RemovalPolicyWalker; +typedef struct _RemovalPurgeWalker RemovalPurgeWalker; +typedef struct _RemovalPolicyNode RemovalPolicyNode; +typedef struct _RemovalPolicySettings RemovalPolicySettings; #if SQUID_SNMP typedef variable_list *(oid_ParseFn) (variable_list *, snint *); @@ -195,11 +200,11 @@ /* disk.c / diskd.c callback typedefs */ typedef void DRCB(int, const char *buf, int size, int errflag, void *data); /* Disk read CB */ -typedef void DWCB(int, int, size_t, void *); /* disk write CB */ +typedef void DWCB(int, int, size_t, void *); /* disk write CB */ typedef void DOCB(int, int errflag, void *data); /* disk open CB */ typedef void DCCB(int, int errflag, void *data); /* disk close CB */ -typedef void DUCB(int errflag, void *data); /* disk unlink CB */ -typedef void DTCB(int errflag, void *data); /* disk trunc CB */ +typedef void DUCB(int errflag, void *data); /* disk unlink CB */ +typedef void DTCB(int errflag, void *data); /* disk trunc CB */ typedef void FQDNH(const char *, void *); typedef void IDCB(const char *ident, void *data); @@ -251,8 +256,10 @@ typedef void STLOGOPEN(SwapDir *); typedef void STLOGCLOSE(SwapDir *); typedef void STLOGWRITE(const SwapDir *, const StoreEntry *, int); -typedef int STLOGCLEANOPEN(SwapDir *); -typedef void STLOGCLEANWRITE(const StoreEntry *, SwapDir *); +typedef int STLOGCLEANSTART(SwapDir *); +typedef const StoreEntry *STLOGCLEANNEXTENTRY(SwapDir *); +typedef void STLOGCLEANWRITE(SwapDir *, const StoreEntry *); +typedef void STLOGCLEANDONE(SwapDir *); /* Store dir configuration routines */ /* SwapDir *sd, char *path ( + char *opt later when the strtok mess is gone) */ @@ -301,4 +308,5 @@ typedef struct _htcpReplyData htcpReplyData; #endif +typedef RemovalPolicy *REMOVALPOLICYCREATE(wordlist *args); #endif /* _TYPEDEFS_H_ */ Index: squid/src/url.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/url.c,v retrieving revision 1.1.1.3.10.2 retrieving revision 1.1.1.3.10.2.2.1 diff -u -r1.1.1.3.10.2 -r1.1.1.3.10.2.2.1 --- squid/src/url.c 20 Apr 2000 18:14:23 -0000 1.1.1.3.10.2 +++ squid/src/url.c 14 May 2000 23:22:20 -0000 1.1.1.3.10.2.2.1 @@ -1,6 +1,6 @@ /* - * $Id: url.c,v 1.1.1.3.10.2 2000/04/20 18:14:23 hno Exp $ + * $Id: url.c,v 1.1.1.3.10.2.2.1 2000/05/14 23:22:20 hno Exp $ * * DEBUG: section 23 URL Parsing * AUTHOR: Duane Wessels @@ -243,10 +243,10 @@ port = urlDefaultPort(protocol); /* Is there any login informaiton? */ if ((t = strrchr(host, '@'))) { - strcpy((char *)login, (char *)host); + strcpy((char *) login, (char *) host); t = strrchr(login, '@'); *t = 0; - strcpy((char *)host, t + 1); + strcpy((char *) host, t + 1); } if ((t = strrchr(host, ':'))) { *t++ = '\0'; Index: squid/src/fs/aufs/async_io.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/async_io.c,v retrieving revision 1.1.6.1.2.3 retrieving revision 1.1.6.1.2.4 diff -u -r1.1.6.1.2.3 -r1.1.6.1.2.4 --- squid/src/fs/aufs/async_io.c 2 May 2000 22:50:29 -0000 1.1.6.1.2.3 +++ squid/src/fs/aufs/async_io.c 14 May 2000 23:22:20 -0000 1.1.6.1.2.4 @@ -78,7 +78,7 @@ static MemPool *aio_ctrl_pool; static void aioFDWasClosed(int fd); -MemPool * aio_state_pool; +MemPool *aio_state_pool; static void aioFDWasClosed(int fd) @@ -221,7 +221,7 @@ if (callback) (callback) (fd, callback_data, -1, errno); if (free_func) - free_func(bufp); + free_func(bufp); return; } ctrlp = memPoolAlloc(aio_ctrl_pool); @@ -351,7 +351,7 @@ void -aioCheckCallbacks(SwapDir *SD) +aioCheckCallbacks(SwapDir * SD) { aio_result_t *resultp; aio_ctrl_t *ctrlp; @@ -409,7 +409,7 @@ /* Flush all pending I/O */ void -aioSync(SwapDir *SD) +aioSync(SwapDir * SD) { if (!initialised) return; /* nothing to do then */ Index: squid/src/fs/aufs/store_asyncufs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_asyncufs.h,v retrieving revision 1.1.6.3.2.1 retrieving revision 1.1.6.3.2.2 diff -u -r1.1.6.3.2.1 -r1.1.6.3.2.2 --- squid/src/fs/aufs/store_asyncufs.h 2 May 2000 22:50:29 -0000 1.1.6.3.2.1 +++ squid/src/fs/aufs/store_asyncufs.h 14 May 2000 23:22:20 -0000 1.1.6.3.2.2 @@ -40,8 +40,8 @@ void aioInit(void); void aioDone(void); -void aioCancel(int); -void aioOpen(const char *, int, mode_t, AIOCB *, void *); +void aioCancel(int); +void aioOpen(const char *, int, mode_t, AIOCB *, void *); void aioClose(int); void aioWrite(int, int offset, char *, int size, AIOCB *, void *, FREE *); void aioRead(int, int offset, char *, int size, AIOCB *, void *); @@ -49,7 +49,7 @@ void aioUnlink(const char *, AIOCB *, void *); void aioCheckCallbacks(SwapDir *); void aioSync(SwapDir *); -int aioQueueSize(void); +int aioQueueSize(void); struct _aioinfo_t { int swaplog_fd; @@ -76,14 +76,14 @@ typedef struct _aiostate_t aiostate_t; /* The aio_state memory pool */ -extern MemPool * aio_state_pool; +extern MemPool *aio_state_pool; extern void storeAufsDirMapBitReset(SwapDir *, sfileno); extern int storeAufsDirMapBitAllocate(SwapDir *); -extern char * storeAufsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); +extern char *storeAufsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath); extern void storeAufsDirUnlinkFile(SwapDir *, sfileno); -extern void storeAufsDirReplAdd(SwapDir *SD, StoreEntry *); +extern void storeAufsDirReplAdd(SwapDir * SD, StoreEntry *); extern void storeAufsDirReplRemove(StoreEntry *); /* Index: squid/src/fs/aufs/store_dir_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_dir_aufs.c,v retrieving revision 1.1.6.6.2.2 retrieving revision 1.1.6.6.2.3 diff -u -r1.1.6.6.2.2 -r1.1.6.6.2.3 --- squid/src/fs/aufs/store_dir_aufs.c 2 May 2000 22:50:29 -0000 1.1.6.6.2.2 +++ squid/src/fs/aufs/store_dir_aufs.c 14 May 2000 23:22:20 -0000 1.1.6.6.2.3 @@ -81,7 +81,7 @@ static EVH storeAufsDirRebuildFromDirectory; static EVH storeAufsDirRebuildFromSwapLog; static int storeAufsDirGetNextFile(RebuildState *, int *sfileno, int *size); -static StoreEntry *storeAufsDirAddDiskRestore(SwapDir *SD, const cache_key * key, +static StoreEntry *storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -97,9 +97,10 @@ static STLOGOPEN storeAufsDirOpenSwapLog; static STINIT storeAufsDirInit; static STFREE storeAufsDirFree; -static STLOGCLEANOPEN storeAufsDirWriteCleanOpen; -static void storeAufsDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANSTART storeAufsDirWriteCleanStart; +static STLOGCLEANNEXTENTRY storeAufsDirCleanLogNextEntry; static STLOGCLEANWRITE storeAufsDirWriteCleanEntry; +static STLOGCLEANDONE storeAufsDirWriteCleanDone; static STLOGCLOSE storeAufsDirCloseSwapLog; static STLOGWRITE storeAufsDirSwapLog; static STNEWFS storeAufsDirNewfs; @@ -117,10 +118,6 @@ static void storeAufsDirStats(SwapDir *, StoreEntry *); static void storeAufsDirInitBitmap(SwapDir *); static int storeAufsDirValidFileno(SwapDir *, sfileno); -static int storeAufsDirCheckExpired(SwapDir *, StoreEntry *); -#if !HEAP_REPLACEMENT -static time_t storeAufsDirExpiredReferenceAge(SwapDir *); -#endif /* * These functions were ripped straight out of the heart of store_dir.c. @@ -130,36 +127,36 @@ */ int -storeAufsDirMapBitTest(SwapDir *SD, int fn) +storeAufsDirMapBitTest(SwapDir * SD, int fn) { sfileno filn = fn; aioinfo_t *aioinfo; - aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo = (aioinfo_t *) SD->fsdata; return file_map_bit_test(aioinfo->map, filn); } - + void -storeAufsDirMapBitSet(SwapDir *SD, int fn) -{ +storeAufsDirMapBitSet(SwapDir * SD, int fn) +{ sfileno filn = fn; aioinfo_t *aioinfo; - aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo = (aioinfo_t *) SD->fsdata; file_map_bit_set(aioinfo->map, filn); } - + void -storeAufsDirMapBitReset(SwapDir *SD, int fn) -{ +storeAufsDirMapBitReset(SwapDir * SD, int fn) +{ sfileno filn = fn; aioinfo_t *aioinfo; - aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo = (aioinfo_t *) SD->fsdata; file_map_bit_reset(aioinfo->map, filn); } int -storeAufsDirMapBitAllocate(SwapDir *SD) +storeAufsDirMapBitAllocate(SwapDir * SD) { - aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) SD->fsdata; int fn; fn = file_map_allocate(aioinfo->map, aioinfo->suggest); file_map_bit_set(aioinfo->map, fn); @@ -167,7 +164,7 @@ return fn; } - + /* * Initialise the asyncufs bitmap * @@ -175,16 +172,16 @@ * configured, we allocate a new bitmap and 'grow' the old one into it. */ static void -storeAufsDirInitBitmap(SwapDir *sd) +storeAufsDirInitBitmap(SwapDir * sd) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; if (aioinfo->map == NULL) { - /* First time */ + /* First time */ aioinfo->map = file_map_create(); } else if (aioinfo->map->max_n_files) { - /* it grew, need to expand */ - /* XXX We don't need it anymore .. */ + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ } /* else it shrunk, and we leave the old one in place */ } @@ -192,7 +189,7 @@ static char * storeAufsDirSwapSubDir(SwapDir * sd, int subdirn) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); assert(0 <= subdirn && subdirn < aioinfo->l1); @@ -245,7 +242,7 @@ static int storeAufsDirVerifyCacheDirs(SwapDir * sd) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; int j; const char *path = sd->path; @@ -262,7 +259,7 @@ static void storeAufsDirCreateSwapSubDirs(SwapDir * sd) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; int i, k; int should_exist; LOCAL_ARRAY(char, name, MAXPATHLEN); @@ -288,31 +285,31 @@ LOCAL_ARRAY(char, digit, 32); char *pathtmp2; if (Config.Log.swap) { - xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); - while (index(pathtmp,'/')) - *index(pathtmp,'/')='.'; - while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') - pathtmp[strlen(pathtmp)-1]= '\0'; - for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); - snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); - if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); - } + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp, '/')) + *index(pathtmp, '/') = '.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') + pathtmp[strlen(pathtmp) - 1] = '\0'; + for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } } else { - xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); - strcat(path, "/swap.state"); + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); } if (ext) - strncat(path, ext, 16); + strncat(path, ext, 16); return path; } static void storeAufsDirOpenSwapLog(SwapDir * sd) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; char *path; int fd; path = storeAufsDirSwapLogFile(sd, NULL); @@ -332,7 +329,7 @@ static void storeAufsDirCloseSwapLog(SwapDir * sd) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; if (aioinfo->swaplog_fd < 0) /* not open */ return; file_close(aioinfo->swaplog_fd); @@ -350,9 +347,9 @@ { static int started_clean_event = 0; static const char *errmsg = - "\tFailed to verify one of the swap directories, Check cache.log\n" - "\tfor details. Run 'squid -z' to create swap directories\n" - "\tif needed, or if running Squid for the first time."; + "\tFailed to verify one of the swap directories, Check cache.log\n" + "\tfor details. Run 'squid -z' to create swap directories\n" + "\tif needed, or if running Squid for the first time."; storeAufsDirInitBitmap(sd); if (storeAufsDirVerifyCacheDirs(sd) < 0) fatal(errmsg); @@ -551,14 +548,14 @@ * because adding to store_swap_size happens in * the cleanup procedure. */ - storeExpireNow(e); - storeReleaseRequest(e); - storeAufsDirReplRemove(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeAufsDirReplRemove(e); storeRelease(e); if (e->swap_filen > -1) { /* Fake a unlink here, this is a bad hack :( */ e->swap_filen = -1; - e->swap_dirn = -1; + e->swap_dirn = -1; } rb->counts.objcount--; rb->counts.cancelcount++; @@ -602,10 +599,7 @@ e->lastmod = s.lastmod; e->flags = s.flags; e->refcount += s.refcount; -#if HEAP_REPLACEMENT - storeHeapPositionUpdate(e, SD); storeAufsDirUnrefObj(SD, e); -#endif } else { debug_trap("storeAufsDirRebuildFromSwapLog: bad condition"); debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); @@ -639,14 +633,14 @@ } else if (e) { /* key already exists, this swapfile not being used */ /* junk old, load new */ - storeExpireNow(e); - storeReleaseRequest(e); - storeAufsDirReplRemove(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeAufsDirReplRemove(e); storeRelease(e); if (e->swap_filen > -1) { /* Fake a unlink here, this is a bad hack :( */ e->swap_filen = -1; - e->swap_dirn = -1; + e->swap_dirn = -1; } rb->counts.dupcount++; } else { @@ -675,7 +669,7 @@ storeAufsDirGetNextFile(RebuildState * rb, int *sfileno, int *size) { SwapDir *SD = rb->sd; - aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) SD->fsdata; int fd = -1; int used = 0; int dirs_opened = 0; @@ -761,7 +755,7 @@ /* Add a new object to the cache with empty memory copy and pointer to disk * use to rebuild store from disk. */ static StoreEntry * -storeAufsDirAddDiskRestore(SwapDir *SD, const cache_key * key, +storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -784,9 +778,6 @@ e->swap_dirn = SD->index; e->swap_file_sz = swap_file_sz; e->lock_count = 0; -#if !HEAP_REPLACEMENT - e->refcount = 0; -#endif e->lastref = lastref; e->timestamp = timestamp; e->expires = expires; @@ -842,7 +833,7 @@ static void storeAufsDirCloseTmpSwapLog(SwapDir * sd) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new")); int fd; @@ -870,7 +861,7 @@ static FILE * storeAufsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); char *clean_path = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean")); char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new")); @@ -923,6 +914,7 @@ char *outbuf; off_t outbuf_offset; int fd; + RemovalPolicyWalker *walker; }; #define CLEAN_BUF_SZ 16384 @@ -932,7 +924,7 @@ * we succeed, and assign the 'func' and 'data' return pointers. */ static int -storeAufsDirWriteCleanOpen(SwapDir * sd) +storeAufsDirWriteCleanStart(SwapDir * sd) { struct _clean_state *state = xcalloc(1, sizeof(*state)); struct stat sb; @@ -943,6 +935,7 @@ state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean")); state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); state->outbuf_offset = 0; + state->walker = sd->repl->WalkInit(sd->repl); unlink(state->new); unlink(state->cln); state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); @@ -960,18 +953,27 @@ } /* + * Get the next entry that is a candidate for clean log writing + */ +const StoreEntry * +storeAufsDirCleanLogNextEntry(SwapDir * sd) +{ + const StoreEntry *entry = NULL; + struct _clean_state *state = sd->log.clean.state; + if (state->walker) + entry = state->walker->Next(state->walker); + return entry; +} + +/* * "write" an entry to the clean log file. */ static void -storeAufsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +storeAufsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e) { storeSwapLogData s; static size_t ss = sizeof(storeSwapLogData); struct _clean_state *state = sd->log.clean.state; - if (NULL == e) { - storeAufsDirWriteCleanClose(sd); - return; - } memset(&s, '\0', ss); s.op = (char) SWAP_LOG_ADD; s.swap_filen = e->swap_filen; @@ -1003,11 +1005,12 @@ } static void -storeAufsDirWriteCleanClose(SwapDir * sd) +storeAufsDirWriteCleanDone(SwapDir * sd) { struct _clean_state *state = sd->log.clean.state; if (state->fd < 0) return; + state->walker->Done(state->walker); if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", state->new, xstrerror()); @@ -1056,7 +1059,7 @@ static void storeAufsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) { - aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) sd->fsdata; storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); s->op = (char) op; s->swap_filen = e->swap_filen; @@ -1115,7 +1118,7 @@ N0 = n_asyncufs_dirs; D0 = asyncufs_dir_index[swap_index % N0]; SD = &Config.cacheSwap.swapDirs[D0]; - aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo = (aioinfo_t *) SD->fsdata; N1 = aioinfo->l1; D1 = (swap_index / N0) % N1; N2 = aioinfo->l2; @@ -1137,7 +1140,7 @@ while ((de = readdir(dp)) != NULL && k < 20) { if (sscanf(de->d_name, "%X", &swapfileno) != 1) continue; - fn = swapfileno; /* XXX should remove this cruft ! */ + fn = swapfileno; /* XXX should remove this cruft ! */ if (storeAufsDirValidFileno(SD, fn)) if (storeAufsDirMapBitTest(SD, fn)) if (storeAufsFilenoBelongsHere(fn, D0, D1, D2)) @@ -1183,7 +1186,7 @@ assert(n_asyncufs_dirs); if (NULL == asyncufs_dir_index) { SwapDir *sd; - aioinfo_t *aioinfo; + aioinfo_t *aioinfo; /* * Initialize the little array that translates ASYNCUFS cache_dir * number into the Config.cacheSwap.swapDirs array index. @@ -1194,7 +1197,7 @@ if (!storeAufsDirIs(sd)) continue; asyncufs_dir_index[n++] = i; - aioinfo = (aioinfo_t *)sd->fsdata; + aioinfo = (aioinfo_t *) sd->fsdata; j += (aioinfo->l1 * aioinfo->l2); } assert(n == n_asyncufs_dirs); @@ -1233,7 +1236,7 @@ int filn = fn; aioinfo_t *aioinfo; assert(F0 < Config.cacheSwap.n_configured); - aioinfo = (aioinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata; + aioinfo = (aioinfo_t *) Config.cacheSwap.swapDirs[F0].fsdata; L1 = aioinfo->l1; L2 = aioinfo->l2; D1 = ((filn / L2) / L2) % L1; @@ -1245,167 +1248,60 @@ return 1; } -int -storeAufsDirValidFileno(SwapDir *SD, sfileno filn) +int +storeAufsDirValidFileno(SwapDir * SD, sfileno filn) { - aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) SD->fsdata; if (filn < 0) - return 0; - if (filn > aioinfo->map->max_n_files) - return 0; + return 0; + if (filn > aioinfo->map->max_n_files) + return 0; return 1; } void -storeAufsDirMaintain(SwapDir *SD) +storeAufsDirMaintain(SwapDir * SD) { StoreEntry *e = NULL; - int scanned = 0; - int locked = 0; - int expired = 0; + int removed = 0; int max_scan; int max_remove; double f; - static time_t last_warn_time = 0; -#if !HEAP_REPLACEMENT - dlink_node *m; - dlink_node *prev = NULL; -#else - heap_key age; - heap_key min_age = 0.0; - link_list *locked_entries = NULL; -#if HEAP_REPLACEMENT_DEBUG - if (!verify_heap_property(SD->repl.heap.heap)) { - debug(20, 1) ("Heap property violated!\n"); - } -#endif -#endif + RemovalPurgeWalker *walker; /* We can't delete objects while rebuilding swap */ if (store_dirs_rebuilding) { - return; + return; } else { - f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); - f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; - max_scan = (int) (f * 400.0 + 100.0); - max_remove = (int) (f * 70.0 + 10.0); + /* XXX FixMe: This should use the cache_dir hig/low values, not the + * global ones + */ + f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + max_scan = (int) (f * 400.0 + 100.0); + max_remove = (int) (f * 70.0 + 10.0); /* * This is kinda cheap, but so we need this priority hack? - */ -#if 0 - eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); -#endif - } - debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove); -#if HEAP_REPLACEMENT - while (heap_nodes(SD->repl.heap.heap) > 0) { - if (store_swap_size < store_swap_low) - break; - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; - age = heap_peepminkey(SD->repl.heap.heap); - e = heap_extractmin(SD->repl.heap.heap); - e->repl.node = NULL; /* no longer in the heap */ - scanned++; - if (storeEntryLocked(e)) { - /* - * Entry is in use ... put it in a linked list to ignore it. - */ - if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - /* - * If this was a "SPECIAL" do not add it back into the heap. - * It will always be "SPECIAL" and therefore never removed. - */ - debug(20, 4) ("storeAufsDirMaintain: locked url %s\n", - (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e-> -key)); - linklistPush(&locked_entries, e); - } - locked++; - continue; - } else if (storeAufsDirCheckExpired(SD, e)) { - /* - * Note: This will not check the reference age ifdef - * HEAP_REPLACEMENT, but it does some other useful - * checks... - */ - expired++; - debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", - age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); - min_age = age; - storeRelease(e); - } else { - /* - * Did not expire the object so we need to add it back - * into the heap! - */ - debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", - storeKeyText(e->key)); - linklistPush(&locked_entries, e); - continue; - } - if (store_swap_size < store_swap_low) - break; - else if (expired >= max_remove) - break; - else if (scanned >= max_scan) - break; + */ } - /* - * Bump the heap age factor. - */ - if (min_age > 0.0) - SD->repl.heap.heap->age = min_age; - /* - * Reinsert all bumped locked entries back into heap... - */ - while ((e = linklistShift(&locked_entries))) - e->repl.node = heap_insert(SD->repl.heap.heap, e); -#else - for (m = SD->repl.lru.list.tail; m; m = prev) { - prev = m->prev; - e = m->data; - scanned++; - if (storeEntryLocked(e)) { - /* - * If there is a locked entry at the tail of the LRU list, - * move it to the beginning to get it out of the way. - * Theoretically, we might have all locked objects at the - * tail, and then we'll never remove anything here and the - * LRU age will go to zero. - */ - if (memInUse(MEM_STOREENTRY) > max_scan) { - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); - } - locked++; - - } else if (storeAufsDirCheckExpired(SD, e)) { - expired++; - storeRelease(e); - } - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; + debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", + f, max_scan, max_remove); + walker = SD->repl->PurgeInit(SD->repl, max_scan); + while (1) { + /* XXX FixMe: This should use the cache_dir hig/low values, not the + * global ones + */ + if (store_swap_size < store_swap_low) + break; + if (removed >= max_remove) + break; + e = walker->Next(walker); + if (!e) + break; /* no more objects */ + removed++; + storeRelease(e); } -#endif - debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l -ocked %d f=%.03f\n", - scanned, max_scan, expired, max_remove, locked, f); - debug(20, 3) ("storeMaintainSwapSpace stats:\n"); - debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); - debug(20, 3) (" %6d were scanned\n", scanned); - debug(20, 3) (" %6d were locked\n", locked); - debug(20, 3) (" %6d were expired\n", expired); - if (store_swap_size < Config.Swap.maxSize) - return; - if (squid_curtime - last_warn_time < 10) - return; - debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", - store_swap_size, Config.Swap.maxSize); - last_warn_time = squid_curtime; + debug(20, (removed ? 2 : 3)) ("storeUfsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n", + SD->path, removed, max_remove, f, max_scan); } /* @@ -1416,12 +1312,12 @@ * happily store anything as long as the LRU time isn't too small. */ int -storeAufsDirCheckObj(SwapDir *SD, const StoreEntry *e) +storeAufsDirCheckObj(SwapDir * SD, const StoreEntry * e) { int loadav; int ql; -#if !HEAP_REPLACEMENT +#if OLD_UNUSED_CODE if (storeAufsDirExpiredReferenceAge(SD) < 300) { debug(20, 3) ("storeAufsDirCheckObj: NO: LRU Age = %d\n", storeAufsDirExpiredReferenceAge(SD)); @@ -1431,11 +1327,11 @@ #endif ql = aioQueueSize(); if (ql == 0) - loadav = 0; - else if (ql >= MAGIC1) /* Queue is too long, don't even consider it */ - loadav = -1; + loadav = 0; + else if (ql >= MAGIC1) /* Queue is too long, don't even consider it */ + loadav = -1; else - loadav = MAGIC1 * 1000 / ql; + loadav = MAGIC1 * 1000 / ql; return loadav; } @@ -1446,20 +1342,12 @@ * maintain replacement information within the storage fs. */ void -storeAufsDirRefObj(SwapDir *SD, StoreEntry *e) +storeAufsDirRefObj(SwapDir * SD, StoreEntry * e) { debug(1, 3) ("storeAufsDirRefObj: referencing %d/%d\n", e->swap_dirn, - e->swap_filen); -#if HEAP_REPLACEMENT - /* Nothing to do here */ -#else - /* Reference the object */ - if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && - !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); - } -#endif + e->swap_filen); + if (SD->repl->Referenced) + SD->repl->Referenced(SD->repl, e, &e->repl); } /* @@ -1468,14 +1356,12 @@ * removed, to maintain replacement information within the storage fs. */ void -storeAufsDirUnrefObj(SwapDir *SD, StoreEntry *e) +storeAufsDirUnrefObj(SwapDir * SD, StoreEntry * e) { debug(1, 3) ("storeAufsDirUnrefObj: referencing %d/%d\n", e->swap_dirn, - e->swap_filen); -#if HEAP_REPLACEMENT - if (e->repl.node) - heap_update(SD->repl.heap.heap, e->repl.node, e); -#endif + e->swap_filen); + if (SD->repl->Dereferenced) + SD->repl->Dereferenced(SD->repl, e, &e->repl); } /* @@ -1486,175 +1372,85 @@ * forced this bit of code here. Eeek. */ void -storeAufsDirUnlinkFile(SwapDir *SD, sfileno f) +storeAufsDirUnlinkFile(SwapDir * SD, sfileno f) { debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f); storeAufsDirMapBitReset(SD, f); aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL); } -#if !HEAP_REPLACEMENT -/* - * storeAufsDirExpiredReferenceAge - * - * The LRU age is scaled exponentially between 1 minute and - * Config.referenceAge , when store_swap_low < store_swap_size < - * store_swap_high. This keeps store_swap_size within the low and high - * water marks. If the cache is very busy then store_swap_size stays - * closer to the low water mark, if it is not busy, then it will stay - * near the high water mark. The LRU age value can be examined on the - * cachemgr 'info' page. - */ -static time_t -storeAufsDirExpiredReferenceAge(SwapDir *SD) -{ - double x; - double z; - time_t age; - long store_high, store_low; - - store_high = (long) (((float) SD->max_size * - (float) Config.Swap.highWaterMark) / (float) 100); - store_low = (long) (((float) SD->max_size * - (float) Config.Swap.lowWaterMark) / (float) 100); - debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size); - - x = (double) (store_high - SD->cur_size) / - (store_high - store_low); - x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; - z = pow((double) (Config.referenceAge / 60), x); - age = (time_t) (z * 60.0); - if (age < 60) - age = 60; - else if (age > Config.referenceAge) - age = Config.referenceAge; - return age; -} -#endif - -/* - * storeAufsDirCheckExpired - * - * Check whether the given object is expired or not - * It breaks layering a little by calling the upper layers to find - * out whether the object is locked or not, but we can't help this - * right now. - */ -static int -storeAufsDirCheckExpired(SwapDir *SD, StoreEntry *e) -{ - if (storeEntryLocked(e)) - return 0; - if (EBIT_TEST(e->flags, RELEASE_REQUEST)) - return 1; - if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) - return 1; - -#if HEAP_REPLACEMENT - /* - * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap - * controls the replacement of objects. - */ - return 1; -#else - if (squid_curtime - e->lastref > storeAufsDirExpiredReferenceAge(SD)) - return 1; - return 0; -#endif -} - /* * Add and remove the given StoreEntry from the replacement policy in * use. */ void -storeAufsDirReplAdd(SwapDir *SD, StoreEntry *e) +storeAufsDirReplAdd(SwapDir * SD, StoreEntry * e) { debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e, - SD->index); -#if HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->repl.node = heap_insert(SD->repl.heap.heap, e); - debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node); - } -#else - /* Shouldn't we not throw special objects into the lru ? */ - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); -#endif + SD->index); + SD->repl->Add(SD->repl, e, &e->repl); } void -storeAufsDirReplRemove(StoreEntry *e) +storeAufsDirReplRemove(StoreEntry * e) { SwapDir *SD = INDEXSD(e->swap_dirn); debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e, - SD->index); -#if HEAP_REPLACEMENT - /* And now, release the object from the replacement policy */ - if (e->repl.node) { - debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n", - e->repl.node); - heap_delete(SD->repl.heap.heap, e->repl.node); - e->repl.node = NULL; - } -#else - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); -#endif + SD->index); + SD->repl->Remove(SD->repl, e, &e->repl); } /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ void -storeAufsDirStats(SwapDir *SD, StoreEntry * sentry) +storeAufsDirStats(SwapDir * SD, StoreEntry * sentry) { aioinfo_t *aioinfo; #if HAVE_STATVFS struct statvfs sfs; #endif - aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo = (aioinfo_t *) SD->fsdata; storeAppendPrintf(sentry, "First level subdirectories: %d\n", aioinfo->l1); storeAppendPrintf(sentry, "Second level subdirectories: %d\n", aioinfo->l2); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", - 100.0 * SD->cur_size / SD->max_size); + 100.0 * SD->cur_size / SD->max_size); storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", - aioinfo->map->n_files_in_map, aioinfo->map->max_n_files, - percent(aioinfo->map->n_files_in_map, aioinfo->map->max_n_files)); + aioinfo->map->n_files_in_map, aioinfo->map->max_n_files, + percent(aioinfo->map->n_files_in_map, aioinfo->map->max_n_files)); #if HAVE_STATVFS #define fsbtoblk(num, fsbs, bs) \ (((fsbs) != 0 && (fsbs) < (bs)) ? \ (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) - if (!statvfs(SD->path, &sfs)) { - storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", - fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), - fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), - percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); - storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", - sfs.f_files - sfs.f_ffree, sfs.f_files, - percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); } #endif storeAppendPrintf(sentry, "Flags:"); if (SD->flags.selected) - storeAppendPrintf(sentry, " SELECTED"); + storeAppendPrintf(sentry, " SELECTED"); if (SD->flags.read_only) - storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, " READ-ONLY"); storeAppendPrintf(sentry, "\n"); +#if OLD_UNUSED_CODE #if !HEAP_REPLACEMENT storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", (double) storeAufsDirExpiredReferenceAge(SD) / 86400.0); #else -#if 0 storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", heap_peepminkey(sd.repl.heap.heap)); #endif -#endif +#endif /* OLD_UNUSED_CODE */ } /* @@ -1663,7 +1459,7 @@ * This routine is called when the given swapdir needs reconfiguring */ void -storeAufsDirReconfigure(SwapDir *sd, int index, char *path) +storeAufsDirReconfigure(SwapDir * sd, int index, char *path) { char *token; int i; @@ -1706,10 +1502,10 @@ void storeAufsDirDump(StoreEntry * entry, const char *name, SwapDir * s) { - aioinfo_t *aioinfo = (aioinfo_t *)s->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) s->fsdata; storeAppendPrintf(entry, "%s %s %s %d %d %d\n", name, - "asyncufs", + "asyncufs", s->path, s->max_size >> 10, aioinfo->l1, @@ -1722,22 +1518,21 @@ static void storeAufsDirFree(SwapDir * s) { - aioinfo_t *aioinfo = (aioinfo_t *)s->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) s->fsdata; if (aioinfo->swaplog_fd > -1) { file_close(aioinfo->swaplog_fd); aioinfo->swaplog_fd = -1; } filemapFreeMemory(aioinfo->map); xfree(aioinfo); - s->fsdata = NULL; /* Will aid debugging... */ - + s->fsdata = NULL; /* Will aid debugging... */ } char * -storeAufsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath) +storeAufsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath) { LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + aioinfo_t *aioinfo = (aioinfo_t *) SD->fsdata; int L1 = aioinfo->l1; int L2 = aioinfo->l2; if (!fullpath) @@ -1757,28 +1552,28 @@ * This is called by storeCleanup() if -S was given on the command line. */ static int -storeAufsCleanupDoubleCheck(SwapDir *sd, StoreEntry *e) +storeAufsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e) { struct stat sb; if (stat(storeAufsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { - debug(20, 0) ("storeAufsCleanupDoubleCheck: MISSING SWAP FILE\n"); - debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", - storeAufsDirFullPath(sd, e->swap_filen, NULL)); - storeEntryDump(e, 0); - return -1; - } + debug(20, 0) ("storeAufsCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", + storeAufsDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } if (e->swap_file_sz != sb.st_size) { - debug(20, 0) ("storeAufsCleanupDoubleCheck: SIZE MISMATCH\n"); - debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", - storeAufsDirFullPath(sd, e->swap_filen, NULL)); - debug(20, 0) ("storeAufsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", - e->swap_file_sz, (int) sb.st_size); - storeEntryDump(e, 0); - return -1; - } + debug(20, 0) ("storeAufsCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", + storeAufsDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeAufsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } return 0; } @@ -1788,7 +1583,7 @@ * Called when a *new* fs is being setup. */ void -storeAufsDirParse(SwapDir *sd, int index, char *path) +storeAufsDirParse(SwapDir * sd, int index, char *path) { char *token; int i; @@ -1816,7 +1611,7 @@ aioinfo = xmalloc(sizeof(aioinfo_t)); if (aioinfo == NULL) - fatal("storeAufsDirParse: couldn't xmalloc() aioinfo_t!\n"); + fatal("storeAufsDirParse: couldn't xmalloc() aioinfo_t!\n"); sd->index = index; sd->path = xstrdup(path); @@ -1825,7 +1620,7 @@ aioinfo->l1 = l1; aioinfo->l2 = l2; aioinfo->swaplog_fd = -1; - aioinfo->map = NULL; /* Debugging purposes */ + aioinfo->map = NULL; /* Debugging purposes */ aioinfo->suggest = 0; sd->flags.read_only = read_only; sd->init = storeAufsDirInit; @@ -1849,45 +1644,16 @@ sd->log.open = storeAufsDirOpenSwapLog; sd->log.close = storeAufsDirCloseSwapLog; sd->log.write = storeAufsDirSwapLog; - sd->log.clean.open = storeAufsDirWriteCleanOpen; + sd->log.clean.start = storeAufsDirWriteCleanStart; + sd->log.clean.nextentry = storeAufsDirCleanLogNextEntry; + sd->log.clean.done = storeAufsDirWriteCleanDone; /* Initialise replacement policy stuff */ -#if HEAP_REPLACEMENT - /* - * Create new heaps with cache replacement policies attached to them. - * The cache replacement policy is specified as either GDSF or LFUDA in - * the squid.conf configuration file. Note that the replacement policy - * applies only to the disk replacement algorithm. Memory replacement - * always uses GDSF since we want to maximize object hit rate. - */ - if (Config.replPolicy) { - if (tolower(Config.replPolicy[0]) == 'g') { - debug(20, 1) ("Using GDSF disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } else if (tolower(Config.replPolicy[0]) == 'l') { - if (tolower(Config.replPolicy[1]) == 'f') { - debug(20, 1) ("Using LFUDA disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); - } else if (tolower(Config.replPolicy[1]) == 'r') { - debug(20, 1) ("Using LRU heap disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); - } - } else { - debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } - } else { - debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } -#else - sd->repl.lru.list.head = NULL; - sd->repl.lru.list.tail = NULL; -#endif + sd->repl = createRemovalPolicy(Config.replPolicy); } void -storeFsSetup_aufs(storefs_entry_t *storefs) +storeFsSetup_aufs(storefs_entry_t * storefs) { storefs->parsefunc = storeAufsDirParse; storefs->reconfigurefunc = storeAufsDirReconfigure; Index: squid/src/fs/aufs/store_io_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_io_aufs.c,v retrieving revision 1.1.6.6.2.2 retrieving revision 1.1.6.6.2.3 diff -u -r1.1.6.6.2.2 -r1.1.6.6.2.3 --- squid/src/fs/aufs/store_io_aufs.c 2 May 2000 22:50:29 -0000 1.1.6.6.2.2 +++ squid/src/fs/aufs/store_io_aufs.c 14 May 2000 23:22:20 -0000 1.1.6.6.2.3 @@ -33,8 +33,8 @@ /* open for reading */ storeIOState * -storeAufsOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback, - STIOCB * callback, void *callback_data) +storeAufsOpen(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) { sfileno f = e->swap_filen; char *path = storeAufsDirFullPath(SD, f, NULL); @@ -49,8 +49,8 @@ sio = memAllocate(MEM_STORE_IO); cbdataAdd(sio, storeAufsIOFreeEntry, MEM_STORE_IO); sio->fsstate = memPoolAlloc(aio_state_pool); - ((aiostate_t *)(sio->fsstate))->fd = -1; - ((aiostate_t *)(sio->fsstate))->flags.opening = 1; + ((aiostate_t *) (sio->fsstate))->fd = -1; + ((aiostate_t *) (sio->fsstate))->flags.opening = 1; sio->swap_filen = f; sio->swap_dirn = SD->index; sio->mode = O_RDONLY; @@ -66,9 +66,9 @@ /* open for creating */ storeIOState * -storeAufsCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data) +storeAufsCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data) { - char *path ; + char *path; storeIOState *sio; sfileno filn; sdirno dirn; @@ -88,8 +88,8 @@ sio = memAllocate(MEM_STORE_IO); cbdataAdd(sio, storeAufsIOFreeEntry, MEM_STORE_IO); sio->fsstate = memPoolAlloc(aio_state_pool); - ((aiostate_t *)(sio->fsstate))->fd = -1; - ((aiostate_t *)(sio->fsstate))->flags.opening = 1; + ((aiostate_t *) (sio->fsstate))->fd = -1; + ((aiostate_t *) (sio->fsstate))->flags.opening = 1; sio->swap_filen = filn; sio->swap_dirn = dirn; sio->mode = O_WRONLY; @@ -111,9 +111,9 @@ /* Close */ void -storeAufsClose(SwapDir *SD, storeIOState * sio) +storeAufsClose(SwapDir * SD, storeIOState * sio) { - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; debug(78, 3) ("storeAufsClose: dirno %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, aiostate->fd); if (storeAufsSomethingPending(sio)) { @@ -126,9 +126,9 @@ /* Read */ void -storeAufsRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +storeAufsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) { - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; assert(sio->read.callback == NULL); assert(sio->read.callback_data == NULL); assert(!aiostate->flags.reading); @@ -160,9 +160,9 @@ /* Write */ void -storeAufsWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +storeAufsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) { - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; debug(78, 3) ("storeAufsWrite: dirno %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, aiostate->fd); if (aiostate->fd < 0) { @@ -200,7 +200,7 @@ /* Unlink */ void -storeAufsUnlink(SwapDir *SD, StoreEntry *e) +storeAufsUnlink(SwapDir * SD, StoreEntry * e) { debug(78, 3) ("storeAufsUnlink: dirno %d, fileno %08X\n", SD->index, e->swap_filen); storeAufsDirReplRemove(e); @@ -212,7 +212,7 @@ static int storeAufsKickWriteQueue(storeIOState * sio) { - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; struct _queued_write *q = linklistShift(&aiostate->pending_writes); if (NULL == q) return 0; @@ -226,7 +226,7 @@ static int storeAufsKickReadQueue(storeIOState * sio) { - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; struct _queued_read *q = linklistShift(&(aiostate->pending_reads)); if (NULL == q) return 0; @@ -241,7 +241,7 @@ storeAufsOpenDone(int unused, void *my_data, int fd, int errflag) { storeIOState *sio = my_data; - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; debug(78, 3) ("storeAufsOpenDone: FD %d, errflag %d\n", fd, errflag); Opening_FD--; aiostate->flags.opening = 0; @@ -266,7 +266,7 @@ storeAufsReadDone(int fd, void *my_data, int len, int errflag) { storeIOState *sio = my_data; - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; STRCB *callback = sio->read.callback; void *their_data = sio->read.callback_data; ssize_t rlen; @@ -293,7 +293,7 @@ * says it fixes his FD leaks, with no side effects. */ if (aiostate->flags.close_request) - storeAufsIOCallback(sio, DISK_OK); + storeAufsIOCallback(sio, DISK_OK); } /* @@ -307,7 +307,7 @@ { static int loop_detect = 0; storeIOState *sio = my_data; - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; debug(78, 3) ("storeAufsWriteDone: dirno %d, fileno %08X, FD %d, len %d\n", sio->swap_dirn, sio->swap_filen, fd, len); assert(++loop_detect < 10); @@ -331,7 +331,7 @@ { STIOCB *callback = sio->callback; void *their_data = sio->callback_data; - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; int fd = aiostate->fd; debug(78, 3) ("storeAufsIOCallback: errflag=%d\n", errflag); sio->callback = NULL; @@ -357,7 +357,7 @@ static int storeAufsSomethingPending(storeIOState * sio) { - aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + aiostate_t *aiostate = (aiostate_t *) sio->fsstate; if (aiostate->flags.reading) return 1; if (aiostate->flags.writing) @@ -371,11 +371,10 @@ /* * We can't pass memFree() as a free function here, because we need to free * the fsstate variable .. - */ + */ static void storeAufsIOFreeEntry(void *sio, int foo) { - memPoolFree(aio_state_pool, ((storeIOState *)sio)->fsstate); - memFree(sio, MEM_STORE_IO); + memPoolFree(aio_state_pool, ((storeIOState *) sio)->fsstate); + memFree(sio, MEM_STORE_IO); } - Index: squid/src/fs/coss/store_coss.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/coss/store_coss.h,v retrieving revision 1.1.4.4 retrieving revision 1.1.4.4.2.1 diff -u -r1.1.4.4 -r1.1.4.4.2.1 --- squid/src/fs/coss/store_coss.h 11 Apr 2000 18:18:47 -0000 1.1.4.4 +++ squid/src/fs/coss/store_coss.h 14 May 2000 23:22:20 -0000 1.1.4.4.2.1 @@ -16,8 +16,8 @@ int lockcount; char buffer[COSS_MEMBUF_SZ]; struct _cossmembuf_flags { - unsigned int full:1; - unsigned int writing:1; + unsigned int full:1; + unsigned int writing:1; } flags; struct _cossmembuf *next; }; @@ -31,6 +31,16 @@ int fd; int swaplog_fd; int numcollisions; + dlink_list index; + int count; + dlink_node *walk_current; +}; + +struct _cossindex { + /* Note: the dlink_node MUST be the first member of the structure. + * This member is later pointer typecasted to coss_index_node *. + */ + dlink_node node; }; @@ -42,20 +52,21 @@ size_t requestoffset; sfileno reqdiskoffset; struct { - unsigned int reading:1; - unsigned int writing:1; + unsigned int reading:1; + unsigned int writing:1; } flags; }; typedef struct _cossmembuf CossMemBuf; typedef struct _cossinfo CossInfo; typedef struct _cossstate CossState; +typedef struct _cossindex CossIndexNode; /* Whether the coss system has been setup or not */ extern int coss_initialised; -extern MemPool * coss_membuf_pool; -extern MemPool * coss_state_pool; - +extern MemPool *coss_membuf_pool; +extern MemPool *coss_state_pool; +extern MemPool *coss_index_pool; /* * Store IO stuff @@ -67,13 +78,15 @@ extern STOBJWRITE storeCossWrite; extern STOBJUNLINK storeCossUnlink; -extern off_t storeCossAllocate(SwapDir *SD, const StoreEntry *e, int which); -extern void storeCossFree(StoreEntry *e); -extern void storeCossMaintainSwapSpace(SwapDir *SD); +extern off_t storeCossAllocate(SwapDir * SD, const StoreEntry * e, int which); +extern void storeCossFree(StoreEntry * e); +extern void storeCossMaintainSwapSpace(SwapDir * SD); extern void storeCossDirStats(SwapDir *, StoreEntry *); -extern void storeCossDirDump(StoreEntry * entry, const char *name, SwapDir* s); +extern void storeCossDirDump(StoreEntry * entry, const char *name, SwapDir * s); extern void storeCossDirFree(SwapDir *); extern SwapDir *storeCossDirPick(void); +extern void storeCossAdd(SwapDir *, StoreEntry *); +extern void storeCossRemove(SwapDir *, StoreEntry *); void storeFsSetup_ufs(storefs_entry_t *); Index: squid/src/fs/coss/store_dir_coss.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/coss/store_dir_coss.c,v retrieving revision 1.1.4.6.2.1 retrieving revision 1.1.4.6.2.2 diff -u -r1.1.4.6.2.1 -r1.1.4.6.2.2 --- squid/src/fs/coss/store_dir_coss.c 2 May 2000 22:50:29 -0000 1.1.4.6.2.1 +++ squid/src/fs/coss/store_dir_coss.c 14 May 2000 23:22:20 -0000 1.1.4.6.2.2 @@ -1,4 +1,4 @@ - + /* * $Id$ * @@ -39,13 +39,14 @@ #define STORE_META_BUFSZ 4096 -extern void storeCossFlushMemBufs(SwapDir *SD); +extern void storeCossFlushMemBufs(SwapDir * SD); int n_coss_dirs = 0; /* static int last_coss_pick_index = -1; */ int coss_initialised = 0; -MemPool * coss_membuf_pool = NULL; -MemPool * coss_state_pool = NULL; +MemPool *coss_membuf_pool = NULL; +MemPool *coss_state_pool = NULL; +MemPool *coss_index_pool = NULL; typedef struct _RebuildState RebuildState; struct _RebuildState { @@ -72,7 +73,7 @@ static char *storeCossDirSwapLogFile(SwapDir *, const char *); static EVH storeCossRebuildFromSwapLog; -static StoreEntry *storeCossAddDiskRestore(SwapDir *SD,const cache_key *key, +static StoreEntry *storeCossAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -87,9 +88,10 @@ static FILE *storeCossDirOpenTmpSwapLog(SwapDir *, int *, int *); static STLOGOPEN storeCossDirOpenSwapLog; static STINIT storeCossDirInit; -static STLOGCLEANOPEN storeCossDirWriteCleanOpen; -static void storeCossDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANSTART storeCossDirWriteCleanStart; +static STLOGCLEANNEXTENTRY storeCossDirCleanLogNextEntry; static STLOGCLEANWRITE storeCossDirWriteCleanEntry; +static STLOGCLEANDONE storeCossDirWriteCleanDone; static STLOGCLOSE storeCossDirCloseSwapLog; static STLOGWRITE storeCossDirSwapLog; static STNEWFS storeCossDirNewfs; @@ -106,31 +108,31 @@ LOCAL_ARRAY(char, digit, 32); char *pathtmp2; if (Config.Log.swap) { - xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); - while (index(pathtmp,'/')) - *index(pathtmp,'/')='.'; - while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') - pathtmp[strlen(pathtmp)-1]= '\0'; - for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); - snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); - if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); - } + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp, '/')) + *index(pathtmp, '/') = '.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') + pathtmp[strlen(pathtmp) - 1] = '\0'; + for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } } else { - xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); - strcat(path, "/swap.state"); + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); } if (ext) - strncat(path, ext, 16); + strncat(path, ext, 16); return path; } static void storeCossDirOpenSwapLog(SwapDir * sd) { - CossInfo *cs = (CossInfo *)sd->fsdata; + CossInfo *cs = (CossInfo *) sd->fsdata; char *path; int fd; path = storeCossDirSwapLogFile(sd, NULL); @@ -146,7 +148,7 @@ static void storeCossDirCloseSwapLog(SwapDir * sd) { - CossInfo *cs = (CossInfo *)sd->fsdata; + CossInfo *cs = (CossInfo *) sd->fsdata; if (cs->swaplog_fd < 0) /* not open */ return; file_close(cs->swaplog_fd); @@ -158,17 +160,40 @@ static void storeCossDirInit(SwapDir * sd) { - CossInfo *cs = (CossInfo *)sd->fsdata; + CossInfo *cs = (CossInfo *) sd->fsdata; storeCossDirOpenSwapLog(sd); storeCossDirRebuild(sd); cs->fd = file_open(sd->path, O_RDWR | O_CREAT); n_coss_dirs++; } +void +storeCossRemove(SwapDir *sd, StoreEntry *e) +{ + CossInfo *cs = (CossInfo *) sd->fsdata; + CossIndexNode *coss_node = e->repl.data; + e->repl.data = NULL; + dlinkDelete(&coss_node->node, &cs->index); + memPoolFree(coss_index_pool, coss_node); + cs->count -= 1; +} + +void +storeCossAdd(SwapDir *sd, StoreEntry *e) +{ + CossInfo *cs = (CossInfo *) sd->fsdata; + CossIndexNode *coss_node = memPoolAlloc(coss_index_pool); + assert(!e->repl.data); + e->repl.data = coss_node; + dlinkAdd(e, &coss_node->node, &cs->index); + cs->count += 1; +} + static void storeCossRebuildFromSwapLog(void *data) { RebuildState *rb = data; + SwapDir *sd = rb->sd; StoreEntry *e = NULL; storeSwapLogData s; size_t ss = sizeof(storeSwapLogData); @@ -213,9 +238,9 @@ if (e->swap_filen > -1) { e->swap_filen = -1; } - storeRelease(e); - /* Fake an unlink here, this is a bad hack :( */ - dlinkDelete(&e->repl.lru, &rb->sd->repl.lru.list); + storeRelease(e); + /* Fake an unlink here, this is a bad hack :( */ + storeCossRemove(sd, e); rb->counts.objcount--; rb->counts.cancelcount++; } @@ -244,7 +269,7 @@ } /* update store_swap_size */ rb->counts.objcount++; - e = storeCossAddDiskRestore(rb->sd,s.key, + e = storeCossAddDiskRestore(rb->sd, s.key, s.swap_filen, s.swap_file_sz, s.expires, @@ -262,7 +287,7 @@ /* Add a new object to the cache with empty memory copy and pointer to disk * use to rebuild store from disk. */ static StoreEntry * -storeCossAddDiskRestore(SwapDir *SD, const cache_key * key, +storeCossAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -285,9 +310,6 @@ e->swap_filen = file_number; e->swap_file_sz = swap_file_sz; e->lock_count = 0; -#if !HEAP_REPLACEMENT - e->refcount = 0; -#endif e->lastref = lastref; e->timestamp = timestamp; e->expires = expires; @@ -300,7 +322,7 @@ e->ping_status = PING_NONE; EBIT_CLR(e->flags, ENTRY_VALIDATED); storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + storeCossAdd(SD, e); e->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY); return e; } @@ -323,7 +345,7 @@ debug(20, 1) ("Rebuilding COSS storage in %s (%s)\n", sd->path, clean ? "CLEAN" : "DIRTY"); if (!clean || fp == NULL) { - /* COSS cannot yet rebuild from a dirty state. If the log + /* COSS cannot yet rebuild from a dirty state. If the log * is dirty then the COSS contents is thrown away. * Why? I guess it is because some contents will be lost, * and COSS cannot verify this.. @@ -331,17 +353,17 @@ if (fp != NULL) fclose(fp); storeCossDirCloseTmpSwapLog(rb->sd); - /* + /* * XXX Make sure we don't trigger an assertion if this is the first * storedir, since if we are, this call will cause storeRebuildComplete * to prematurely complete the rebuild process, and then some other * storedir will try to rebuild and eventually die. */ - store_dirs_rebuilding++; - storeRebuildComplete(&rb->counts); - store_dirs_rebuilding--; + store_dirs_rebuilding++; + storeRebuildComplete(&rb->counts); + store_dirs_rebuilding--; xfree(rb); - return; + return; } func = storeCossRebuildFromSwapLog; rb->log = fp; @@ -354,7 +376,7 @@ static void storeCossDirCloseTmpSwapLog(SwapDir * sd) { - CossInfo *cs = (CossInfo *)sd->fsdata; + CossInfo *cs = (CossInfo *) sd->fsdata; char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); int fd; @@ -382,7 +404,7 @@ static FILE * storeCossDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) { - CossInfo *cs = (CossInfo *)sd->fsdata; + CossInfo *cs = (CossInfo *) sd->fsdata; char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); char *clean_path = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean")); char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); @@ -435,6 +457,7 @@ char *outbuf; off_t outbuf_offset; int fd; + dlink_node *current; }; #define CLEAN_BUF_SZ 16384 @@ -444,8 +467,9 @@ * we succeed, and assign the 'func' and 'data' return pointers. */ static int -storeCossDirWriteCleanOpen(SwapDir * sd) +storeCossDirWriteCleanStart(SwapDir * sd) { + CossInfo *cs = (CossInfo *) sd->fsdata; struct _clean_state *state = xcalloc(1, sizeof(*state)); struct stat sb; sd->log.clean.write = NULL; @@ -460,6 +484,7 @@ state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); if (state->fd < 0) return -1; + state->current = cs->index.tail; debug(20, 3) ("storeCOssDirWriteCleanLogs: opened %s, FD %d\n", state->new, state->fd); #if HAVE_FCHMOD @@ -468,22 +493,33 @@ #endif sd->log.clean.write = storeCossDirWriteCleanEntry; sd->log.clean.state = state; + return 0; } +static const StoreEntry * +storeCossDirCleanLogNextEntry(SwapDir * sd) +{ + struct _clean_state *state = sd->log.clean.state; + const StoreEntry *entry; + if (!state) + return NULL; + if (!state->current) + return NULL; + entry = (const StoreEntry *)state->current->data; + state->current = state->current->next; + return entry; +} + /* * "write" an entry to the clean log file. */ static void -storeCossDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +storeCossDirWriteCleanEntry(SwapDir *sd, const StoreEntry * e) { storeSwapLogData s; static size_t ss = sizeof(storeSwapLogData); struct _clean_state *state = sd->log.clean.state; - if (NULL == e) { - storeCossDirWriteCleanClose(sd); - return; - } memset(&s, '\0', ss); s.op = (char) SWAP_LOG_ADD; s.swap_filen = e->swap_filen; @@ -515,7 +551,7 @@ } static void -storeCossDirWriteCleanClose(SwapDir * sd) +storeCossDirWriteCleanDone(SwapDir * sd) { struct _clean_state *state = sd->log.clean.state; if (state->fd < 0) @@ -568,7 +604,7 @@ static void storeCossDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) { - CossInfo *cs = (CossInfo *)sd->fsdata; + CossInfo *cs = (CossInfo *) sd->fsdata; storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); s->op = (char) op; s->swap_filen = e->swap_filen; @@ -595,17 +631,17 @@ debug(47, 3) ("Creating swap space in %s\n", sd->path); } -// we are shutting down, flush all membufs to disk +/* we are shutting down, flush all membufs to disk */ static void -storeCossDirShutdown(SwapDir *SD) +storeCossDirShutdown(SwapDir * SD) { - CossInfo *cs = (CossInfo *)SD->fsdata; + CossInfo *cs = (CossInfo *) SD->fsdata; CossMemBuf *t; - // we need to do this synchronously! + /* we need to do this synchronously! */ for (t = cs->membufs; t; t = t->next) { - lseek(cs->fd, t->diskstart, SEEK_SET); - write(cs->fd, t->buffer, COSS_MEMBUF_SZ); + lseek(cs->fd, t->diskstart, SEEK_SET); + write(cs->fd, t->buffer, COSS_MEMBUF_SZ); } file_close(cs->fd); cs->fd = -1; @@ -614,7 +650,6 @@ file_close(cs->swaplog_fd); cs->swaplog_fd = -1; } - n_coss_dirs--; } @@ -627,46 +662,46 @@ * done by the upper layers. */ int -storeCossDirCheckObj(SwapDir *SD, const StoreEntry *e) +storeCossDirCheckObj(SwapDir * SD, const StoreEntry * e) { - /* Check if the object is a special object, we can't cache these */ - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) - return -1; + /* Check if the object is a special object, we can't cache these */ + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) + return -1; - /* Otherwise, we're ok */ - /* Return 900 (90%) load */ - return 900; + /* Otherwise, we're ok */ + /* Return 900 (90%) load */ + return 900; } - + /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ void -storeCossDirStats(SwapDir *SD, StoreEntry * sentry) +storeCossDirStats(SwapDir * SD, StoreEntry * sentry) { - CossInfo *cs = (CossInfo *)SD->fsdata; - + CossInfo *cs = (CossInfo *) SD->fsdata; + storeAppendPrintf(sentry, "\n"); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", - 100.0 * SD->cur_size / SD->max_size); - storeAppendPrintf(sentry, "Number of object collisions: %d\n", (int)cs->numcollisions); + 100.0 * SD->cur_size / SD->max_size); + storeAppendPrintf(sentry, "Number of object collisions: %d\n", (int) cs->numcollisions); #if 0 /* is this applicable? I Hope not .. */ storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", - SD->map->n_files_in_map, SD->map->max_n_files, - percent(SD->map->n_files_in_map, SD->map->max_n_files)); + SD->map->n_files_in_map, SD->map->max_n_files, + percent(SD->map->n_files_in_map, SD->map->max_n_files)); #endif storeAppendPrintf(sentry, "Flags:"); if (SD->flags.selected) - storeAppendPrintf(sentry, " SELECTED"); + storeAppendPrintf(sentry, " SELECTED"); if (SD->flags.read_only) - storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, " READ-ONLY"); storeAppendPrintf(sentry, "\n"); } static void -storeCossDirParse(SwapDir *sd, int index, char *path) +storeCossDirParse(SwapDir * sd, int index, char *path) { char *token; unsigned int i; @@ -703,10 +738,10 @@ sd->statfs = storeCossDirStats; sd->maintainfs = NULL; sd->checkobj = storeCossDirCheckObj; - sd->refobj = NULL; /* LRU is done in storeCossRead */ + sd->refobj = NULL; /* LRU is done in storeCossRead */ sd->unrefobj = NULL; sd->callback = NULL; - sd->sync = NULL; /* should we make it call the coss sync? */ + sd->sync = NULL; /* should we make it call the coss sync? */ sd->obj.create = storeCossCreate; sd->obj.open = storeCossOpen; @@ -718,8 +753,10 @@ sd->log.open = storeCossDirOpenSwapLog; sd->log.close = storeCossDirCloseSwapLog; sd->log.write = storeCossDirSwapLog; - sd->log.clean.open = storeCossDirWriteCleanOpen; + sd->log.clean.start = storeCossDirWriteCleanStart; sd->log.clean.write = storeCossDirWriteCleanEntry; + sd->log.clean.nextentry = storeCossDirCleanLogNextEntry; + sd->log.clean.done = storeCossDirWriteCleanDone; cs->current_offset = 0; cs->fd = -1; @@ -734,14 +771,13 @@ cs->membufs->next = NULL; cs->membufs->SD = sd; cs->current_membuf = cs->membufs; - - sd->repl.lru.list.head = NULL; - sd->repl.lru.list.tail = NULL; + cs->index.head = NULL; + cs->index.tail = NULL; } static void -storeCossDirReconfigure(SwapDir *sd, int index, char *path) +storeCossDirReconfigure(SwapDir * sd, int index, char *path) { char *token; unsigned int i; @@ -757,15 +793,15 @@ read_only = 1; if (size == sd->max_size) - debug (3, 1) ("Cache COSS dir '%s' size remains unchanged at %d KB\n", path, size); + debug(3, 1) ("Cache COSS dir '%s' size remains unchanged at %d KB\n", path, size); else { - debug (3, 1) ("Cache COSS dir '%s' size changed to %d KB\n", path, size); - sd->max_size = size; + debug(3, 1) ("Cache COSS dir '%s' size changed to %d KB\n", path, size); + sd->max_size = size; } if (read_only != sd->flags.read_only) { - debug (3, 1) ("Cache COSS dir '%s' now %s\n", path, read_only ? "Read-Only" : "Read-Write"); - sd->flags.read_only = read_only; + debug(3, 1) ("Cache COSS dir '%s' now %s\n", path, read_only ? "Read-Only" : "Read-Write"); + sd->flags.read_only = read_only; } } @@ -774,7 +810,7 @@ { storeAppendPrintf(entry, "%s %s %s %d\n", name, - s->type, + s->type, s->path, s->max_size >> 20); } @@ -783,40 +819,40 @@ SwapDir * storeCossDirPick(void) { - int i,choosenext = 0; + int i, choosenext = 0; SwapDir *SD; - + if (n_coss_dirs == 0) - return NULL; + return NULL; for (i = 0; i < Config.cacheSwap.n_configured; i++) { SD = &Config.cacheSwap.swapDirs[i]; - if (SD->type == SWAPDIR_COSS) { - if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { - last_coss_pick_index = i; - return SD; - } else if (choosenext) { - last_coss_pick_index = i; - return SD; - } else if (last_coss_pick_index == i) { - choosenext = 1; - } - } + if (SD->type == SWAPDIR_COSS) { + if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { + last_coss_pick_index = i; + return SD; + } else if (choosenext) { + last_coss_pick_index = i; + return SD; + } else if (last_coss_pick_index == i) { + choosenext = 1; + } + } } for (i = 0; i < Config.cacheSwap.n_configured; i++) { SD = &Config.cacheSwap.swapDirs[i]; - if (SD->type == SWAPDIR_COSS) { - if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { - last_coss_pick_index = i; - return SD; - } else if (choosenext) { - last_coss_pick_index = i; - return SD; - } else if (last_coss_pick_index == i) { - choosenext = 1; - } - } + if (SD->type == SWAPDIR_COSS) { + if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { + last_coss_pick_index = i; + return SD; + } else if (choosenext) { + last_coss_pick_index = i; + return SD; + } else if (last_coss_pick_index == i) { + choosenext = 1; + } + } } - return NULL; + return NULL; } #endif @@ -832,7 +868,7 @@ } void -storeFsSetup_coss(storefs_entry_t *storefs) +storeFsSetup_coss(storefs_entry_t * storefs) { assert(!coss_initialised); @@ -841,6 +877,6 @@ storefs->donefunc = storeCossDirDone; coss_membuf_pool = memPoolCreate("COSS Membuf data", sizeof(CossMemBuf)); coss_state_pool = memPoolCreate("COSS IO State data", sizeof(CossState)); + coss_index_pool = memPoolCreate("COSS index data", sizeof(CossIndexNode)); coss_initialised = 1; } - Index: squid/src/fs/coss/store_io_coss.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/coss/store_io_coss.c,v retrieving revision 1.1.4.8.2.3 retrieving revision 1.1.4.8.2.4 diff -u -r1.1.4.8.2.3 -r1.1.4.8.2.4 --- squid/src/fs/coss/store_io_coss.c 2 May 2000 22:50:29 -0000 1.1.4.8.2.3 +++ squid/src/fs/coss/store_io_coss.c 14 May 2000 23:22:20 -0000 1.1.4.8.2.4 @@ -39,13 +39,13 @@ static DWCB storeCossWriteMemBufDone; static DRCB storeCossReadDone; static void storeCossIOCallback(storeIOState * sio, int errflag); -static char *storeCossMemPointerFromDiskOffset(SwapDir *SD, size_t offset, CossMemBuf **mb); -static void storeCossMemBufLock(SwapDir *SD, storeIOState *e); -static void storeCossMemBufUnlock(SwapDir *SD, storeIOState *e); -static void storeCossWriteMemBuf(SwapDir *SD,CossMemBuf *t); +static char *storeCossMemPointerFromDiskOffset(SwapDir * SD, size_t offset, CossMemBuf ** mb); +static void storeCossMemBufLock(SwapDir * SD, storeIOState * e); +static void storeCossMemBufUnlock(SwapDir * SD, storeIOState * e); +static void storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t); static void storeCossWriteMemBufDone(int fd, int errflag, size_t len, void *my_data); -static CossMemBuf *storeCossCreateMemBuf(SwapDir *SD, size_t start, - sfileno curfn, int *collision); +static CossMemBuf *storeCossCreateMemBuf(SwapDir * SD, size_t start, + sfileno curfn, int *collision); static void storeCossIOFreeEntry(void *, int); static void storeCossMembufFree(void *, int); @@ -59,68 +59,71 @@ * -- Adrian */ off_t -storeCossAllocate(SwapDir *SD, const StoreEntry *e, int which) +storeCossAllocate(SwapDir * SD, const StoreEntry * e, int which) { - CossInfo *cs = (CossInfo *)SD->fsdata; + CossInfo *cs = (CossInfo *) SD->fsdata; CossMemBuf *newmb; off_t retofs; size_t allocsize; int coll = 0; sfileno checkf; - /* Make sure we chcek collisions if reallocating */ + /* Make sure we chcek collisions if reallocating */ if (which == COSS_ALLOC_REALLOC) checkf = e->swap_filen; else checkf = -1; - retofs = e->swap_filen; /* Just for defaults */ + retofs = e->swap_filen; /* Just for defaults */ if (e->swap_file_sz > 0) - allocsize = e->swap_file_sz; + allocsize = e->swap_file_sz; else - allocsize = objectLen(e) + e->mem_obj->swap_hdr_sz; + allocsize = objectLen(e) + e->mem_obj->swap_hdr_sz; if (which != COSS_ALLOC_NOTIFY) { - if ((cs->current_offset + allocsize) > (SD->max_size << 10)) { - // tried to allocate past the end of the disk, so wrap - // back to the beginning - cs->current_membuf->flags.full = 1; - cs->current_membuf->diskend = cs->current_offset-1; - cs->current_offset = 0; // wrap back to beginning - newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll); - cs->current_membuf = newmb; - } else if ((cs->current_offset + allocsize) > cs->current_membuf->diskend) { - cs->current_membuf->flags.full = 1; - cs->current_membuf->diskend = cs->current_offset-1; - newmb = storeCossCreateMemBuf(SD, cs->current_offset, - checkf, &coll); - cs->current_membuf = newmb; + if ((cs->current_offset + allocsize) > (SD->max_size << 10)) { + /* + * tried to allocate past the end of the disk, so wrap + * back to the beginning + */ + cs->current_membuf->flags.full = 1; + cs->current_membuf->diskend = cs->current_offset - 1; + cs->current_offset = 0; /* wrap back to beginning */ + + newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll); + cs->current_membuf = newmb; + } else if ((cs->current_offset + allocsize) > cs->current_membuf->diskend) { + cs->current_membuf->flags.full = 1; + cs->current_membuf->diskend = cs->current_offset - 1; + newmb = storeCossCreateMemBuf(SD, cs->current_offset, + checkf, &coll); + cs->current_membuf = newmb; + } + if (coll == 0) { + retofs = cs->current_offset; + } else { + debug(81, 3) ("storeCossAllocate: Collision\n"); } - if (coll == 0) { - retofs = cs->current_offset; - } else { - debug(81, 3) ("storeCossAllocate: Collision\n"); - } } if (coll == 0) { - cs->current_offset += allocsize; - return retofs; + cs->current_offset += allocsize; + return retofs; } else { - return -1; + return -1; } } void -storeCossUnlink(SwapDir *SD, StoreEntry *e) +storeCossUnlink(SwapDir * SD, StoreEntry * e) { debug(81, 3) ("storeCossUnlink: offset %d\n", e->swap_filen); - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + storeCossRemove(SD, e); } storeIOState * -storeCossCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data) +storeCossCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data) { CossState *cstate; storeIOState *sio; @@ -144,30 +147,30 @@ sio->callback = callback; sio->callback_data = callback_data; cbdataLock(callback_data); - sio->e = (StoreEntry *)e; - sio->st_size = -1; // we won't know this until we read the metadata + sio->e = (StoreEntry *) e; + sio->st_size = -1; /* we won't know this until we read the metadata */ cstate->flags.writing = 0; cstate->flags.reading = 0; cstate->readbuffer = NULL; cstate->reqdiskoffset = -1; - /* Now add it into the LRU */ - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + /* Now add it into the index list */ + storeCossAdd(SD, e); storeCossMemBufLock(SD, sio); return sio; } storeIOState * -storeCossOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback, - STIOCB * callback, void *callback_data) +storeCossOpen(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) { storeIOState *sio; char *p; CossState *cstate; sfileno f = e->swap_filen; - CossInfo *cs = (CossInfo *)SD->fsdata; + CossInfo *cs = (CossInfo *) SD->fsdata; debug(81, 3) ("storeCossOpen: offset %d\n", f); @@ -192,71 +195,71 @@ cstate->readbuffer = NULL; cstate->reqdiskoffset = -1; p = storeCossMemPointerFromDiskOffset(SD, f, NULL); - // make local copy so we don't have to lock membuf + /* make local copy so we don't have to lock membuf */ if (p) { - cstate->readbuffer = xmalloc(sio->st_size); - memcpy(cstate->readbuffer, p, sio->st_size); + cstate->readbuffer = xmalloc(sio->st_size); + memcpy(cstate->readbuffer, p, sio->st_size); } else { - /* Do the allocation */ + /* Do the allocation */ /* this is the first time we've been called on a new sio - read the whole object into memory, then return the - requested amount - */ - /* - * This bit of code actually does the LRU disk thing - we realloc - * a place for the object here, and the file_read() reads the object - * into the cossmembuf for later writing .. - */ - cstate->reqdiskoffset = sio->swap_filen; - sio->swap_filen = -1; - sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC); + * read the whole object into memory, then return the + * requested amount + */ + /* + * This bit of code actually does the LRU disk thing - we realloc + * a place for the object here, and the file_read() reads the object + * into the cossmembuf for later writing .. + */ + cstate->reqdiskoffset = sio->swap_filen; + sio->swap_filen = -1; + sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC); if (sio->swap_filen == -1) { - /* We have to clean up neatly .. */ + /* We have to clean up neatly .. */ cbdataFree(sio); cs->numcollisions++; debug(81, 2) ("storeCossOpen: Reallocation of %d/%d failed\n", e->swap_dirn, e->swap_filen); - /* XXX XXX XXX Will squid call storeUnlink for this object? */ + /* XXX XXX XXX Will squid call storeUnlink for this object? */ return NULL; } - /* Notify the upper levels that we've changed file number */ + /* Notify the upper levels that we've changed file number */ sio->file_callback(sio->callback_data, 0, sio); - - /* - * lock the buffer so it doesn't get swapped out on us - * this will get unlocked in storeCossReadDone - */ - storeCossMemBufLock(SD, sio); - - /* - * Do the LRU magic to keep the disk and memory LRUs identical - */ - dlinkDelete(&sio->e->repl.lru, &SD->repl.lru.list); - dlinkAdd(sio->e, &sio->e->repl.lru, &SD->repl.lru.list); - - /* - * Since we've reallocated a spot for this object, we need to - * write it to the cossmembuf *and* return it in the read .. - */ - cstate->readbuffer = NULL; + + /* + * lock the buffer so it doesn't get swapped out on us + * this will get unlocked in storeCossReadDone + */ + storeCossMemBufLock(SD, sio); + + /* + * Do the index magic to keep the disk and memory LRUs identical + */ + storeCossRemove(SD, e); + storeCossAdd(SD, e); + + /* + * Since we've reallocated a spot for this object, we need to + * write it to the cossmembuf *and* return it in the read .. + */ + cstate->readbuffer = NULL; } return sio; } void -storeCossClose(SwapDir *SD, storeIOState * sio) +storeCossClose(SwapDir * SD, storeIOState * sio) { - debug(81, 3) ("storeCossClose: offset %d\n",sio->swap_filen); + debug(81, 3) ("storeCossClose: offset %d\n", sio->swap_filen); if (sio->mode == O_WRONLY) - storeCossMemBufUnlock(SD, sio); + storeCossMemBufUnlock(SD, sio); storeCossIOCallback(sio, 0); } void -storeCossRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +storeCossRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) { char *p; - CossState *cstate = (CossState *)sio->fsstate; - CossInfo *cs = (CossInfo *)SD->fsdata; + CossState *cstate = (CossState *) sio->fsstate; + CossInfo *cs = (CossInfo *) SD->fsdata; assert(sio->read.callback == NULL); assert(sio->read.callback_data == NULL); @@ -267,43 +270,43 @@ sio->offset = offset; cstate->flags.reading = 1; if ((offset + size) > sio->st_size) - size = sio->st_size - offset; + size = sio->st_size - offset; cstate->requestlen = size; cstate->requestbuf = buf; cstate->requestoffset = offset; if (cstate->readbuffer == NULL) { - p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); - file_read(cs->fd, + p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); + file_read(cs->fd, p, sio->st_size, cstate->reqdiskoffset, storeCossReadDone, sio); - cstate->reqdiskoffset = 0; /* XXX */ + cstate->reqdiskoffset = 0; /* XXX */ } else { - storeCossReadDone(cs->fd, - cstate->readbuffer, - sio->st_size, - 0, - sio); - } + storeCossReadDone(cs->fd, + cstate->readbuffer, + sio->st_size, + 0, + sio); + } } void -storeCossWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +storeCossWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) { char *dest; CossMemBuf *membuf; off_t diskoffset; - - debug(81, 3) ("storeCossWrite: offset %d, len %d\n",sio->offset, size); + + debug(81, 3) ("storeCossWrite: offset %d, len %d\n", sio->offset, size); diskoffset = sio->swap_filen + sio->offset; dest = storeCossMemPointerFromDiskOffset(SD, diskoffset, &membuf); assert(dest != NULL); memcpy(dest, buf, size); sio->offset += size; if (free_func) - (free_func)(buf); + (free_func) (buf); } @@ -317,7 +320,7 @@ STRCB *callback = sio->read.callback; void *their_data = sio->read.callback_data; SwapDir *SD = INDEXSD(sio->swap_dirn); - CossState *cstate = (CossState *)sio->fsstate; + CossState *cstate = (CossState *) sio->fsstate; size_t rlen; debug(81, 3) ("storeCossReadDone: fileno %d, FD %d, len %d\n", @@ -325,18 +328,18 @@ cstate->flags.reading = 0; if (errflag) { debug(81, 3) ("storeCossReadDone: got failure (%d)\n", errflag); - rlen = -1; + rlen = -1; } else { - if (cstate->readbuffer == NULL) { - cstate->readbuffer = xmalloc(sio->st_size); - p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); - memcpy(cstate->readbuffer, p, sio->st_size); - storeCossMemBufUnlock(SD, sio); - } - sio->offset += len; - memcpy(cstate->requestbuf, &cstate->readbuffer[cstate->requestoffset], - cstate->requestlen); - rlen = (size_t) cstate->requestlen; + if (cstate->readbuffer == NULL) { + cstate->readbuffer = xmalloc(sio->st_size); + p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); + memcpy(cstate->readbuffer, p, sio->st_size); + storeCossMemBufUnlock(SD, sio); + } + sio->offset += len; + memcpy(cstate->requestbuf, &cstate->readbuffer[cstate->requestoffset], + cstate->requestlen); + rlen = (size_t) cstate->requestlen; } assert(callback); assert(their_data); @@ -350,7 +353,7 @@ static void storeCossIOCallback(storeIOState * sio, int errflag) { - CossState *cstate = (CossState *)sio->fsstate; + CossState *cstate = (CossState *) sio->fsstate; debug(81, 3) ("storeCossIOCallback: errflag=%d\n", errflag); xfree(cstate->readbuffer); if (cbdataValid(sio->callback_data)) @@ -360,63 +363,63 @@ } static char * -storeCossMemPointerFromDiskOffset(SwapDir *SD, size_t offset, CossMemBuf **mb) +storeCossMemPointerFromDiskOffset(SwapDir * SD, size_t offset, CossMemBuf ** mb) { CossMemBuf *t; - CossInfo *cs = (CossInfo *)SD->fsdata; + CossInfo *cs = (CossInfo *) SD->fsdata; - for (t=cs->membufs; t; t = t->next) - if ((offset >= t->diskstart) && (offset <= t->diskend)) { - if (mb) - *mb = t; - return &t->buffer[offset - t->diskstart]; - } + for (t = cs->membufs; t; t = t->next) + if ((offset >= t->diskstart) && (offset <= t->diskend)) { + if (mb) + *mb = t; + return &t->buffer[offset - t->diskstart]; + } if (mb) - *mb = NULL; + *mb = NULL; return NULL; } static void -storeCossMemBufLock(SwapDir *SD, storeIOState *e) +storeCossMemBufLock(SwapDir * SD, storeIOState * e) { CossMemBuf *t; - CossInfo *cs = (CossInfo *)SD->fsdata; + CossInfo *cs = (CossInfo *) SD->fsdata; for (t = cs->membufs; t; t = t->next) - if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { - debug(81, 3) ("storeCossMemBufLock: locking %08X, lockcount %d\n",t, t->lockcount); - t->lockcount++; - return; - } - debug(81, 3) ("storeCossMemBufLock: FAILED to lock %08X\n",e); + if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { + debug(81, 3) ("storeCossMemBufLock: locking %08X, lockcount %d\n", t, t->lockcount); + t->lockcount++; + return; + } + debug(81, 3) ("storeCossMemBufLock: FAILED to lock %08X\n", e); } static void -storeCossMemBufUnlock(SwapDir *SD, storeIOState *e) +storeCossMemBufUnlock(SwapDir * SD, storeIOState * e) { CossMemBuf *t; - CossInfo *cs = (CossInfo *)SD->fsdata; + CossInfo *cs = (CossInfo *) SD->fsdata; for (t = cs->membufs; t; t = t->next) { - if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { - t->lockcount--; - debug(81, 3) ("storeCossMemBufUnlock: unlocking %08X, lockcount %d\n",t, t->lockcount); - } - if (t->flags.full && !t->flags.writing && !t->lockcount) - storeCossWriteMemBuf(SD, t); + if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { + t->lockcount--; + debug(81, 3) ("storeCossMemBufUnlock: unlocking %08X, lockcount %d\n", t, t->lockcount); + } + if (t->flags.full && !t->flags.writing && !t->lockcount) + storeCossWriteMemBuf(SD, t); } } static void -storeCossWriteMemBuf(SwapDir *SD, CossMemBuf *t) +storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t) { - CossInfo *cs = (CossInfo *)SD->fsdata; + CossInfo *cs = (CossInfo *) SD->fsdata; debug(81, 3) ("storeCossWriteMemBuf: offset %d, len %d\n", t->diskstart, t->diskend - t->diskstart); cbdataAdd(t, storeCossMembufFree, 0); file_write(cs->fd, t->diskstart, &t->buffer, - t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL); + t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL); t->flags.writing = 1; } @@ -425,45 +428,45 @@ storeCossWriteMemBufDone(int fd, int errflag, size_t len, void *my_data) { CossMemBuf *t = my_data; - CossMemBuf *p,*prev; - CossInfo *cs = (CossInfo *)t->SD->fsdata; + CossMemBuf *p, *prev; + CossInfo *cs = (CossInfo *) t->SD->fsdata; - debug(81, 3) ("storeCossWriteMemBufDone: len %d\n",len); + debug(81, 3) ("storeCossWriteMemBufDone: len %d\n", len); if (errflag) { debug(81, 0) ("storeCossMemBufWriteDone: got failure (%d)\n", errflag); - cbdataFree(t); + cbdataFree(t); return; } - if (t == cs->membufs) { - cs->membufs = t->next; - cbdataFree(t); - return; + cs->membufs = t->next; + cbdataFree(t); + return; } prev = t; for (p = cs->membufs; p; p = p->next) { - if (t == p) { - prev->next = t->next; - cbdataFree(t); - return; - } - prev = p; - } + if (t == p) { + prev->next = t->next; + cbdataFree(t); + return; + } + prev = p; + } cbdataFree(t); } -static CossMemBuf *storeCossCreateMemBuf(SwapDir *SD, size_t start, - sfileno curfn, int *collision) +static CossMemBuf * +storeCossCreateMemBuf(SwapDir * SD, size_t start, + sfileno curfn, int *collision) { - CossMemBuf *newmb,*t; + CossMemBuf *newmb, *t; StoreEntry *e; - dlink_node *m,*prev; - int numreleased=0; - CossInfo *cs = (CossInfo *)SD->fsdata; + dlink_node *m, *prev; + int numreleased = 0; + CossInfo *cs = (CossInfo *) SD->fsdata; newmb = memPoolAlloc(coss_membuf_pool); newmb->diskstart = start; - debug(81, 3) ("storeCossCreateMemBuf: creating new membuf at %d\n",newmb->diskstart); + debug(81, 3) ("storeCossCreateMemBuf: creating new membuf at %d\n", newmb->diskstart); newmb->diskend = newmb->diskstart + COSS_MEMBUF_SZ - 1; newmb->flags.full = 0; newmb->flags.writing = 0; @@ -471,27 +474,27 @@ newmb->SD = SD; newmb->next = cs->membufs; cs->membufs = newmb; - for (t = cs->membufs; t; t = t->next) - debug(81, 3) ("storeCossCreateMemBuf: membuflist %d lockcount %d\n",t->diskstart,t->lockcount); + for (t = cs->membufs; t; t = t->next) + debug(81, 3) ("storeCossCreateMemBuf: membuflist %d lockcount %d\n", t->diskstart, t->lockcount); /* - * XXX more evil LRU specific code. This needs to be replaced. + * Kill objects from the tail to make space for a new chunk */ - for (m = SD->repl.lru.list.tail; m; m = prev) { - prev = m->prev; - e = m->data; - if (curfn == e->swap_filen) - *collision = 1; /* Mark an object alloc collision */ - if ((e->swap_filen >= newmb->diskstart) && - (e->swap_filen <= newmb->diskend)) { - storeRelease(e); - numreleased++; - } else - break; + for (m = cs->index.tail; m; m = prev) { + prev = m->prev; + e = m->data; + if (curfn == e->swap_filen) + *collision = 1; /* Mark an object alloc collision */ + if ((e->swap_filen >= newmb->diskstart) && + (e->swap_filen <= newmb->diskend)) { + storeRelease(e); + numreleased++; + } else + break; } if (numreleased > 0) - debug(81, 3) ("storeCossCreateMemBuf: this allocation released %d storeEntries\n",numreleased); - return newmb; + debug(81, 3) ("storeCossCreateMemBuf: this allocation released %d storeEntries\n", numreleased); + return newmb; } /* @@ -501,8 +504,8 @@ static void storeCossIOFreeEntry(void *sio, int foo) { - memPoolFree(coss_state_pool, ((storeIOState *)sio)->fsstate); - memFree(sio, MEM_STORE_IO); + memPoolFree(coss_state_pool, ((storeIOState *) sio)->fsstate); + memFree(sio, MEM_STORE_IO); } /* @@ -515,4 +518,3 @@ { memPoolFree(coss_membuf_pool, mb); } - Index: squid/src/fs/diskd/diskd.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/diskd/diskd.c,v retrieving revision 1.1.6.1 retrieving revision 1.1.6.2 diff -u -r1.1.6.1 -r1.1.6.2 --- squid/src/fs/diskd/diskd.c 2 May 2000 22:50:29 -0000 1.1.6.1 +++ squid/src/fs/diskd/diskd.c 14 May 2000 23:22:20 -0000 1.1.6.2 @@ -307,4 +307,3 @@ perror("shmctl IPC_RMID"); return 0; } - Index: squid/src/fs/diskd/store_dir_diskd.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/diskd/store_dir_diskd.c,v retrieving revision 1.1.6.1 retrieving revision 1.1.6.2 diff -u -r1.1.6.1 -r1.1.6.2 --- squid/src/fs/diskd/store_dir_diskd.c 2 May 2000 22:50:29 -0000 1.1.6.1 +++ squid/src/fs/diskd/store_dir_diskd.c 14 May 2000 23:22:20 -0000 1.1.6.2 @@ -81,7 +81,7 @@ static int n_diskd_dirs = 0; static int *diskd_dir_index = NULL; -MemPool * diskd_state_pool = NULL; +MemPool *diskd_state_pool = NULL; static int diskd_initialised = 0; static char *storeDiskdDirSwapSubDir(SwapDir *, int subdirn); @@ -93,7 +93,7 @@ static EVH storeDiskdDirRebuildFromDirectory; static EVH storeDiskdDirRebuildFromSwapLog; static int storeDiskdDirGetNextFile(RebuildState *, int *sfileno, int *size); -static StoreEntry *storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key, +static StoreEntry *storeDiskdDirAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -109,9 +109,10 @@ static STLOGOPEN storeDiskdDirOpenSwapLog; static STINIT storeDiskdDirInit; static STFREE storeDiskdDirFree; -static STLOGCLEANOPEN storeDiskdDirWriteCleanOpen; -static void storeDiskdDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANSTART storeDiskdDirWriteCleanStart; +static STLOGCLEANNEXTENTRY storeDiskdDirCleanLogNextEntry; static STLOGCLEANWRITE storeDiskdDirWriteCleanEntry; +static STLOGCLEANDONE storeDiskdDirWriteCleanDone; static STLOGCLOSE storeDiskdDirCloseSwapLog; static STLOGWRITE storeDiskdDirSwapLog; static STNEWFS storeDiskdDirNewfs; @@ -129,13 +130,8 @@ static void storeDiskdDirStats(SwapDir *, StoreEntry *); static void storeDiskdDirInitBitmap(SwapDir *); static int storeDiskdDirValidFileno(SwapDir *, sfileno); -static int storeDiskdDirCheckExpired(SwapDir *, StoreEntry *); -#if !HEAP_REPLACEMENT -static time_t storeDiskdDirExpiredReferenceAge(SwapDir *); -#endif static void storeDiskdStats(StoreEntry * sentry); static void storeDiskdDirSync(SwapDir *); -static void storeDiskdDirCallback(SwapDir *); /* @@ -146,43 +142,43 @@ */ int -storeDiskdDirMapBitTest(SwapDir *SD, int fn) +storeDiskdDirMapBitTest(SwapDir * SD, int fn) { sfileno filn = fn; diskdinfo_t *diskdinfo; - diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo = (diskdinfo_t *) SD->fsdata; return file_map_bit_test(diskdinfo->map, filn); } - + void -storeDiskdDirMapBitSet(SwapDir *SD, int fn) -{ +storeDiskdDirMapBitSet(SwapDir * SD, int fn) +{ sfileno filn = fn; diskdinfo_t *diskdinfo; - diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo = (diskdinfo_t *) SD->fsdata; file_map_bit_set(diskdinfo->map, filn); } - + void -storeDiskdDirMapBitReset(SwapDir *SD, int fn) -{ +storeDiskdDirMapBitReset(SwapDir * SD, int fn) +{ sfileno filn = fn; diskdinfo_t *diskdinfo; - diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo = (diskdinfo_t *) SD->fsdata; file_map_bit_reset(diskdinfo->map, filn); } int -storeDiskdDirMapBitAllocate(SwapDir *SD) +storeDiskdDirMapBitAllocate(SwapDir * SD) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; int fn; fn = file_map_allocate(diskdinfo->map, diskdinfo->suggest); file_map_bit_set(diskdinfo->map, fn); diskdinfo->suggest = fn + 1; return fn; } - + /* * Initialise the diskd bitmap * @@ -190,16 +186,16 @@ * configured, we allocate a new bitmap and 'grow' the old one into it. */ static void -storeDiskdDirInitBitmap(SwapDir *sd) +storeDiskdDirInitBitmap(SwapDir * sd) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; if (diskdinfo->map == NULL) { - /* First time */ + /* First time */ diskdinfo->map = file_map_create(); } else if (diskdinfo->map->max_n_files) { - /* it grew, need to expand */ - /* XXX We don't need it anymore .. */ + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ } /* else it shrunk, and we leave the old one in place */ } @@ -207,7 +203,7 @@ static char * storeDiskdDirSwapSubDir(SwapDir * sd, int subdirn) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); assert(0 <= subdirn && subdirn < diskdinfo->l1); @@ -260,7 +256,7 @@ static int storeDiskdDirVerifyCacheDirs(SwapDir * sd) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; int j; const char *path = sd->path; @@ -277,7 +273,7 @@ static void storeDiskdDirCreateSwapSubDirs(SwapDir * sd) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; int i, k; int should_exist; LOCAL_ARRAY(char, name, MAXPATHLEN); @@ -303,31 +299,31 @@ LOCAL_ARRAY(char, digit, 32); char *pathtmp2; if (Config.Log.swap) { - xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); - while (index(pathtmp,'/')) - *index(pathtmp,'/')='.'; - while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') - pathtmp[strlen(pathtmp)-1]= '\0'; - for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); - snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); - if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); - } + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp, '/')) + *index(pathtmp, '/') = '.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') + pathtmp[strlen(pathtmp) - 1] = '\0'; + for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } } else { - xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); - strcat(path, "/swap.state"); + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); } if (ext) - strncat(path, ext, 16); + strncat(path, ext, 16); return path; } static void storeDiskdDirOpenSwapLog(SwapDir * sd) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; char *path; int fd; path = storeDiskdDirSwapLogFile(sd, NULL); @@ -347,7 +343,7 @@ static void storeDiskdDirCloseSwapLog(SwapDir * sd) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; if (diskdinfo->swaplog_fd < 0) /* not open */ return; file_close(diskdinfo->swaplog_fd); @@ -380,28 +376,28 @@ diskdinfo->smsgid = msgget((key_t) ikey, 0700 | IPC_CREAT); if (diskdinfo->smsgid < 0) { - debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); - fatal("msgget failed"); + debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); + fatal("msgget failed"); } diskdinfo->rmsgid = msgget((key_t) (ikey + 1), 0700 | IPC_CREAT); if (diskdinfo->rmsgid < 0) { - debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); - fatal("msgget failed"); + debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); + fatal("msgget failed"); } diskdinfo->shm.id = shmget((key_t) (ikey + 2), - SHMBUFS * SHMBUF_BLKSZ, 0600 | IPC_CREAT); + SHMBUFS * SHMBUF_BLKSZ, 0600 | IPC_CREAT); if (diskdinfo->shm.id < 0) { - debug(50, 0) ("storeDiskdInit: shmget: %s\n", xstrerror()); - fatal("shmget failed"); + debug(50, 0) ("storeDiskdInit: shmget: %s\n", xstrerror()); + fatal("shmget failed"); } diskdinfo->shm.buf = shmat(diskdinfo->shm.id, NULL, 0); if (diskdinfo->shm.buf == (void *) -1) { - debug(50, 0) ("storeDiskdInit: shmat: %s\n", xstrerror()); - fatal("shmat failed"); + debug(50, 0) ("storeDiskdInit: shmat: %s\n", xstrerror()); + fatal("shmat failed"); } diskd_stats.shmbuf_count += SHMBUFS; for (i = 0; i < SHMBUFS; i++) - storeDiskdShmPut(sd, i * SHMBUF_BLKSZ); + storeDiskdShmPut(sd, i * SHMBUF_BLKSZ); snprintf(skey1, 32, "%d", ikey); snprintf(skey2, 32, "%d", ikey + 1); snprintf(skey3, 32, "%d", ikey + 2); @@ -416,15 +412,15 @@ #else x = ipcCreate(IPC_FIFO, #endif - SQUID_PREFIX "/bin/diskd", - args, - "diskd", - &rfd, - &diskdinfo->wfd); + SQUID_PREFIX "/bin/diskd", + args, + "diskd", + &rfd, + &diskdinfo->wfd); if (x < 0) - fatal("execl " SQUID_PREFIX "/bin/diskd failed"); + fatal("execl " SQUID_PREFIX "/bin/diskd failed"); if (rfd != diskdinfo->wfd) - comm_close(rfd); + comm_close(rfd); fd_note(diskdinfo->wfd, "squid -> diskd"); commSetTimeout(diskdinfo->wfd, -1, NULL, NULL); commSetNonBlocking(diskdinfo->wfd); @@ -459,7 +455,7 @@ * until the data has finished writing. */ static void -storeDiskdDirSync(SwapDir *SD) +storeDiskdDirSync(SwapDir * SD) { /* XXX NOT DONE YET! */ #warning "storeDiskdSync() needs to be written" @@ -473,35 +469,34 @@ * until the queue is below magic2. Otherwise, we simply return when we * don't get a message. */ -static void -storeDiskdDirCallback(SwapDir *SD) +void +storeDiskdDirCallback(SwapDir * SD) { diomsg M; int x; - diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; if (diskdinfo->away >= diskdinfo->magic2) diskd_stats.block_queue_len++; if (diskd_stats.sent_count - diskd_stats.recv_count > - diskd_stats.max_away) { - diskd_stats.max_away = diskd_stats.sent_count - diskd_stats.recv_count; - diskd_stats.max_shmuse = diskd_stats.shmbuf_count; + diskd_stats.max_away) { + diskd_stats.max_away = diskd_stats.sent_count - diskd_stats.recv_count; + diskd_stats.max_shmuse = diskd_stats.shmbuf_count; } - /* if we are above magic2, we do not break under any reason */ while (1) { - memset(&M, '\0', sizeof(M)); - x = msgrcv(diskdinfo->rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT); + memset(&M, '\0', sizeof(M)); + x = msgrcv(diskdinfo->rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT); if (x < 0) { if (diskdinfo->away >= diskdinfo->magic2) continue; - else - break; + else + break; } else if (x != msg_snd_rcv_sz) { - debug(81, 1) ("storeDiskdReadIndividualQueue: msgget returns %d\n", - x); - break; + debug(81, 1) ("storeDiskdDirCallback: msgget returns %d\n", + x); + break; } diskd_stats.recv_count++; diskdinfo->away--; @@ -580,7 +575,7 @@ tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); if (tlv_list == NULL) { debug(20, 1) ("storeDiskdDirRebuildFromDirectory: failed to get meta data\n"); - /* XXX shouldn't this be a call to storeDiskdUnlink ? */ + /* XXX shouldn't this be a call to storeDiskdUnlink ? */ storeDiskdDirUnlinkFile(SD, sfileno); continue; } @@ -700,13 +695,13 @@ * because adding to store_swap_size happens in * the cleanup procedure. */ - storeExpireNow(e); - storeReleaseRequest(e); - storeDiskdDirReplRemove(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeDiskdDirReplRemove(e); if (e->swap_filen > -1) { storeDiskdDirMapBitReset(SD, e->swap_filen); e->swap_filen = -1; - e->swap_dirn = -1; + e->swap_dirn = -1; } storeRelease(e); rb->counts.objcount--; @@ -751,10 +746,7 @@ e->lastmod = s.lastmod; e->flags = s.flags; e->refcount += s.refcount; -#if HEAP_REPLACEMENT - storeHeapPositionUpdate(e, SD); - storeDiskdDirUnrefObj(SD, e); -#endif + storeDiskdDirUnrefObj(SD, e); } else { debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition"); debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); @@ -788,14 +780,14 @@ } else if (e) { /* key already exists, this swapfile not being used */ /* junk old, load new */ - storeExpireNow(e); - storeReleaseRequest(e); - storeDiskdDirReplRemove(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeDiskdDirReplRemove(e); if (e->swap_filen > -1) { - /* Make sure we don't actually unlink the file */ + /* Make sure we don't actually unlink the file */ storeDiskdDirMapBitReset(SD, e->swap_filen); e->swap_filen = -1; - e->swap_dirn = -1; + e->swap_dirn = -1; } storeRelease(e); rb->counts.dupcount++; @@ -825,7 +817,7 @@ storeDiskdDirGetNextFile(RebuildState * rb, int *sfileno, int *size) { SwapDir *SD = rb->sd; - diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; int fd = -1; int used = 0; int dirs_opened = 0; @@ -911,7 +903,7 @@ /* Add a new object to the cache with empty memory copy and pointer to disk * use to rebuild store from disk. */ static StoreEntry * -storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key, +storeDiskdDirAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -934,9 +926,6 @@ e->swap_dirn = SD->index; e->swap_file_sz = swap_file_sz; e->lock_count = 0; -#if !HEAP_REPLACEMENT - e->refcount = 0; -#endif e->lastref = lastref; e->timestamp = timestamp; e->expires = expires; @@ -992,7 +981,7 @@ static void storeDiskdDirCloseTmpSwapLog(SwapDir * sd) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new")); int fd; @@ -1020,7 +1009,7 @@ static FILE * storeDiskdDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); char *clean_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean")); char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new")); @@ -1073,6 +1062,7 @@ char *outbuf; off_t outbuf_offset; int fd; + RemovalPolicyWalker *walker; }; #define CLEAN_BUF_SZ 16384 @@ -1082,7 +1072,7 @@ * we succeed, and assign the 'func' and 'data' return pointers. */ static int -storeDiskdDirWriteCleanOpen(SwapDir * sd) +storeDiskdDirWriteCleanStart(SwapDir * sd) { struct _clean_state *state = xcalloc(1, sizeof(*state)); struct stat sb; @@ -1093,6 +1083,7 @@ state->cln = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean")); state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); state->outbuf_offset = 0; + state->walker = sd->repl->WalkInit(sd->repl); unlink(state->new); unlink(state->cln); state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); @@ -1110,18 +1101,27 @@ } /* + * Get the next entry that is a candidate for clean log writing + */ +const StoreEntry * +storeDiskdDirCleanLogNextEntry(SwapDir * sd) +{ + const StoreEntry *entry = NULL; + struct _clean_state *state = sd->log.clean.state; + if (state->walker) + entry = state->walker->Next(state->walker); + return entry; +} + +/* * "write" an entry to the clean log file. */ static void -storeDiskdDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +storeDiskdDirWriteCleanEntry(SwapDir *sd, const StoreEntry * e) { storeSwapLogData s; static size_t ss = sizeof(storeSwapLogData); struct _clean_state *state = sd->log.clean.state; - if (NULL == e) { - storeDiskdDirWriteCleanClose(sd); - return; - } memset(&s, '\0', ss); s.op = (char) SWAP_LOG_ADD; s.swap_filen = e->swap_filen; @@ -1153,11 +1153,12 @@ } static void -storeDiskdDirWriteCleanClose(SwapDir * sd) +storeDiskdDirWriteCleanDone(SwapDir * sd) { struct _clean_state *state = sd->log.clean.state; if (state->fd < 0) return; + state->walker->Done(state->walker); if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", state->new, xstrerror()); @@ -1206,7 +1207,7 @@ static void storeDiskdDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); s->op = (char) op; s->swap_filen = e->swap_filen; @@ -1265,7 +1266,7 @@ N0 = n_diskd_dirs; D0 = diskd_dir_index[swap_index % N0]; SD = &Config.cacheSwap.swapDirs[D0]; - diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo = (diskdinfo_t *) SD->fsdata; N1 = diskdinfo->l1; D1 = (swap_index / N0) % N1; N2 = diskdinfo->l2; @@ -1287,7 +1288,7 @@ while ((de = readdir(dp)) != NULL && k < 20) { if (sscanf(de->d_name, "%X", &swapfileno) != 1) continue; - fn = swapfileno; /* XXX should remove this cruft ! */ + fn = swapfileno; /* XXX should remove this cruft ! */ if (storeDiskdDirValidFileno(SD, fn)) if (storeDiskdDirMapBitTest(SD, fn)) if (storeDiskdFilenoBelongsHere(fn, D0, D1, D2)) @@ -1333,7 +1334,7 @@ assert(n_diskd_dirs); if (NULL == diskd_dir_index) { SwapDir *sd; - diskdinfo_t *diskdinfo; + diskdinfo_t *diskdinfo; /* * Initialize the little array that translates DISKD cache_dir * number into the Config.cacheSwap.swapDirs array index. @@ -1344,7 +1345,7 @@ if (!storeDiskdDirIs(sd)) continue; diskd_dir_index[n++] = i; - diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo = (diskdinfo_t *) sd->fsdata; j += (diskdinfo->l1 * diskdinfo->l2); } assert(n == n_diskd_dirs); @@ -1383,7 +1384,7 @@ int filn = fn; diskdinfo_t *diskdinfo; assert(F0 < Config.cacheSwap.n_configured); - diskdinfo = (diskdinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata; + diskdinfo = (diskdinfo_t *) Config.cacheSwap.swapDirs[F0].fsdata; L1 = diskdinfo->l1; L2 = diskdinfo->l2; D1 = ((filn / L2) / L2) % L1; @@ -1395,167 +1396,60 @@ return 1; } -int -storeDiskdDirValidFileno(SwapDir *SD, sfileno filn) +int +storeDiskdDirValidFileno(SwapDir * SD, sfileno filn) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; if (filn < 0) - return 0; - if (filn > diskdinfo->map->max_n_files) - return 0; + return 0; + if (filn > diskdinfo->map->max_n_files) + return 0; return 1; } void -storeDiskdDirMaintain(SwapDir *SD) +storeDiskdDirMaintain(SwapDir * SD) { StoreEntry *e = NULL; - int scanned = 0; - int locked = 0; - int expired = 0; + int removed = 0; int max_scan; int max_remove; double f; - static time_t last_warn_time = 0; -#if !HEAP_REPLACEMENT - dlink_node *m; - dlink_node *prev = NULL; -#else - heap_key age; - heap_key min_age = 0.0; - link_list *locked_entries = NULL; -#if HEAP_REPLACEMENT_DEBUG - if (!verify_heap_property(SD->repl.heap.heap)) { - debug(20, 1) ("Heap property violated!\n"); - } -#endif -#endif + RemovalPurgeWalker *walker; /* We can't delete objects while rebuilding swap */ if (store_dirs_rebuilding) { - return; + return; } else { - f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); - f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; - max_scan = (int) (f * 400.0 + 100.0); - max_remove = (int) (f * 70.0 + 10.0); + /* XXX FixMe: This should use the cache_dir hig/low values, not the + * global ones + */ + f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + max_scan = (int) (f * 400.0 + 100.0); + max_remove = (int) (f * 70.0 + 10.0); /* * This is kinda cheap, but so we need this priority hack? - */ -#if 0 - eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); -#endif + */ } debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove); -#if HEAP_REPLACEMENT - while (heap_nodes(SD->repl.heap.heap) > 0) { - if (store_swap_size < store_swap_low) - break; - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; - age = heap_peepminkey(SD->repl.heap.heap); - e = heap_extractmin(SD->repl.heap.heap); - e->repl.node = NULL; /* no longer in the heap */ - scanned++; - if (storeEntryLocked(e)) { - /* - * Entry is in use ... put it in a linked list to ignore it. - */ - if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - /* - * If this was a "SPECIAL" do not add it back into the heap. - * It will always be "SPECIAL" and therefore never removed. - */ - debug(20, 4) ("storeDiskdDirMaintain: locked url %s\n", - (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e-> -key)); - linklistPush(&locked_entries, e); - } - locked++; - continue; - } else if (storeDiskdDirCheckExpired(SD, e)) { - /* - * Note: This will not check the reference age ifdef - * HEAP_REPLACEMENT, but it does some other useful - * checks... - */ - expired++; - debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", - age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); - min_age = age; - storeRelease(e); - } else { - /* - * Did not expire the object so we need to add it back - * into the heap! - */ - debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", - storeKeyText(e->key)); - linklistPush(&locked_entries, e); - continue; - } - if (store_swap_size < store_swap_low) - break; - else if (expired >= max_remove) - break; - else if (scanned >= max_scan) - break; - } - /* - * Bump the heap age factor. - */ - if (min_age > 0.0) - SD->repl.heap.heap->age = min_age; - /* - * Reinsert all bumped locked entries back into heap... - */ - while ((e = linklistShift(&locked_entries))) - e->repl.node = heap_insert(SD->repl.heap.heap, e); -#else - for (m = SD->repl.lru.list.tail; m; m = prev) { - prev = m->prev; - e = m->data; - scanned++; - if (storeEntryLocked(e)) { - /* - * If there is a locked entry at the tail of the LRU list, - * move it to the beginning to get it out of the way. - * Theoretically, we might have all locked objects at the - * tail, and then we'll never remove anything here and the - * LRU age will go to zero. - */ - if (memInUse(MEM_STOREENTRY) > max_scan) { - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); - } - locked++; - - } else if (storeDiskdDirCheckExpired(SD, e)) { - expired++; - storeRelease(e); - } - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; - } -#endif - debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l -ocked %d f=%.03f\n", - scanned, max_scan, expired, max_remove, locked, f); - debug(20, 3) ("storeMaintainSwapSpace stats:\n"); - debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); - debug(20, 3) (" %6d were scanned\n", scanned); - debug(20, 3) (" %6d were locked\n", locked); - debug(20, 3) (" %6d were expired\n", expired); - if (store_swap_size < Config.Swap.maxSize) - return; - if (squid_curtime - last_warn_time < 10) - return; - debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", - store_swap_size, Config.Swap.maxSize); - last_warn_time = squid_curtime; + walker = SD->repl->PurgeInit(SD->repl, max_scan); + while (1) { + /* XXX FixMe: This should use the cache_dir hig/low values, not the + * global ones + */ + if (store_swap_size < store_swap_low) + break; + if (removed >= max_remove) + break; + e = walker->Next(walker); + if (!e) + break; /* no more objects */ + removed++; + storeRelease(e); + } + walker->Done(walker); + debug(20, (removed ? 2 : 3)) ("storeDiskdDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n", + SD->path, removed, max_remove, f, max_scan); } /* @@ -1566,29 +1460,29 @@ * happily store anything as long as the LRU time isn't too small. */ int -storeDiskdDirCheckObj(SwapDir *SD, const StoreEntry *e) +storeDiskdDirCheckObj(SwapDir * SD, const StoreEntry * e) { int loadav; - diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; -#if !HEAP_REPLACEMENT + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; +#if OLD_UNUSED_CODE if (storeDiskdDirExpiredReferenceAge(SD) < 300) { - debug(20, 3) ("storeDiskdDirCheckObj: NO: LRU Age = %d\n", - storeDiskdDirExpiredReferenceAge(SD)); - /* store_check_cachable_hist.no.lru_age_too_low++; */ - return -1; + debug(20, 3) ("storeDiskdDirCheckObj: NO: LRU Age = %d\n", + storeDiskdDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; } #endif /* Check the queue length */ if (diskdinfo->away >= diskdinfo->magic1) - return -1; + return -1; /* Calculate the storedir load relative to magic2 on a scale of 0 .. 1000 */ if (diskdinfo->away == 0) - loadav = 0; + loadav = 0; else - loadav = diskdinfo->magic2 * 1000 / diskdinfo->away; + loadav = diskdinfo->magic2 * 1000 / diskdinfo->away; return loadav; } @@ -1599,20 +1493,12 @@ * maintain replacement information within the storage fs. */ void -storeDiskdDirRefObj(SwapDir *SD, StoreEntry *e) +storeDiskdDirRefObj(SwapDir * SD, StoreEntry * e) { debug(1, 3) ("storeDiskdDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, - e->swap_filen); -#if HEAP_REPLACEMENT - /* Nothing to do here */ -#else - /* Reference the object */ - if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && - !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); - } -#endif + e->swap_filen); + if (SD->repl->Referenced) + SD->repl->Referenced(SD->repl, e, &e->repl); } /* @@ -1621,14 +1507,12 @@ * removed, to maintain replacement information within the storage fs. */ void -storeDiskdDirUnrefObj(SwapDir *SD, StoreEntry *e) +storeDiskdDirUnrefObj(SwapDir * SD, StoreEntry * e) { debug(1, 3) ("storeDiskdDirUnrefObj: referencing %p %d/%d\n", e, - e->swap_dirn, e->swap_filen); -#if HEAP_REPLACEMENT - if (e->repl.node) - heap_update(SD->repl.heap.heap, e->repl.node, e); -#endif + e->swap_dirn, e->swap_filen); + if (SD->repl->Dereferenced) + SD->repl->Dereferenced(SD->repl, e, &e->repl); } /* @@ -1639,124 +1523,34 @@ * uses storeDiskdUnlink() .. */ void -storeDiskdDirUnlinkFile(SwapDir *SD, sfileno f) +storeDiskdDirUnlinkFile(SwapDir * SD, sfileno f) { debug(79, 3) ("storeDiskdDirUnlinkFile: unlinking fileno %08X\n", f); storeDiskdDirMapBitReset(SD, f); unlinkdUnlink(storeDiskdDirFullPath(SD, f, NULL)); } -#if !HEAP_REPLACEMENT -/* - * storeDiskdDirExpiredReferenceAge - * - * The LRU age is scaled exponentially between 1 minute and - * Config.referenceAge , when store_swap_low < store_swap_size < - * store_swap_high. This keeps store_swap_size within the low and high - * water marks. If the cache is very busy then store_swap_size stays - * closer to the low water mark, if it is not busy, then it will stay - * near the high water mark. The LRU age value can be examined on the - * cachemgr 'info' page. - */ -static time_t -storeDiskdDirExpiredReferenceAge(SwapDir *SD) -{ - double x; - double z; - time_t age; - long store_high, store_low; - - store_high = (long) (((float) SD->max_size * - (float) Config.Swap.highWaterMark) / (float) 100); - store_low = (long) (((float) SD->max_size * - (float) Config.Swap.lowWaterMark) / (float) 100); - debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size); - - x = (double) (store_high - SD->cur_size) / - (store_high - store_low); - x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; - z = pow((double) (Config.referenceAge / 60), x); - age = (time_t) (z * 60.0); - if (age < 60) - age = 60; - else if (age > Config.referenceAge) - age = Config.referenceAge; - return age; -} -#endif - -/* - * storeDiskdDirCheckExpired - * - * Check whether the given object is expired or not - * It breaks layering a little by calling the upper layers to find - * out whether the object is locked or not, but we can't help this - * right now. - */ -static int -storeDiskdDirCheckExpired(SwapDir *SD, StoreEntry *e) -{ - if (storeEntryLocked(e)) - return 0; - if (EBIT_TEST(e->flags, RELEASE_REQUEST)) - return 1; - if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) - return 1; - -#if HEAP_REPLACEMENT - /* - * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap - * controls the replacement of objects. - */ - return 1; -#else - if (squid_curtime - e->lastref > storeDiskdDirExpiredReferenceAge(SD)) - return 1; - return 0; -#endif -} - /* * Add and remove the given StoreEntry from the replacement policy in * use. */ void -storeDiskdDirReplAdd(SwapDir *SD, StoreEntry *e) +storeDiskdDirReplAdd(SwapDir * SD, StoreEntry * e) { debug(20, 4) ("storeDiskdDirReplAdd: added node %p to dir %d\n", e, - SD->index); -#if HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->repl.node = heap_insert(SD->repl.heap.heap, e); - debug(20, 4) ("storeDiskdDirReplAdd: inserted node 0x%x\n", e->repl.node); - } -#else - /* Shouldn't we not throw special objects into the lru ? */ - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); -#endif + SD->index); + SD->repl->Add(SD->repl, e, &e->repl); } void -storeDiskdDirReplRemove(StoreEntry *e) +storeDiskdDirReplRemove(StoreEntry * e) { SwapDir *SD = INDEXSD(e->swap_dirn); debug(20, 4) ("storeDiskdDirReplRemove: remove node %p from dir %d\n", e, - SD->index); -#if HEAP_REPLACEMENT - /* And now, release the object from the replacement policy */ - if (e->repl.node) { - debug(20, 4) ("storeDiskdDirReplRemove: deleting node 0x%x\n", - e->repl.node); - heap_delete(SD->repl.heap.heap, e->repl.node); - e->repl.node = NULL; - } -#else - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); -#endif + SD->index); + SD->repl->Remove(SD->repl, e, &e->repl); } @@ -1769,7 +1563,7 @@ storeDiskdShmGet(SwapDir * sd, int *shm_offset) { char *buf; - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; buf = linklistShift(&diskdinfo->shm.stack); assert(buf); *shm_offset = buf - diskdinfo->shm.buf; @@ -1782,7 +1576,7 @@ storeDiskdShmPut(SwapDir * sd, int offset) { char *buf; - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; assert(offset >= 0); assert(offset < SHMBUFS * SHMBUF_BLKSZ); buf = diskdinfo->shm.buf + offset; @@ -1796,51 +1590,51 @@ /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ void -storeDiskdDirStats(SwapDir *SD, StoreEntry * sentry) +storeDiskdDirStats(SwapDir * SD, StoreEntry * sentry) { diskdinfo_t *diskdinfo; #if HAVE_STATVFS struct statvfs sfs; #endif - diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo = (diskdinfo_t *) SD->fsdata; storeAppendPrintf(sentry, "First level subdirectories: %d\n", diskdinfo->l1); storeAppendPrintf(sentry, "Second level subdirectories: %d\n", diskdinfo->l2); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", - 100.0 * SD->cur_size / SD->max_size); + 100.0 * SD->cur_size / SD->max_size); storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", - diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files, - percent(diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files)); + diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files, + percent(diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files)); #if HAVE_STATVFS #define fsbtoblk(num, fsbs, bs) \ (((fsbs) != 0 && (fsbs) < (bs)) ? \ (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) - if (!statvfs(SD->path, &sfs)) { - storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", - fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), - fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), - percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); - storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", - sfs.f_files - sfs.f_ffree, sfs.f_files, - percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); } #endif storeAppendPrintf(sentry, "Flags:"); if (SD->flags.selected) - storeAppendPrintf(sentry, " SELECTED"); + storeAppendPrintf(sentry, " SELECTED"); if (SD->flags.read_only) - storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, " READ-ONLY"); storeAppendPrintf(sentry, "\n"); +#if OLD_UNUSED_CODE #if !HEAP_REPLACEMENT storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", - (double) storeDiskdDirExpiredReferenceAge(SD) / 86400.0); + (double) storeDiskdDirExpiredReferenceAge(SD) / 86400.0); #else -#if 0 storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", - heap_peepminkey(sd.repl.heap.heap)); -#endif + heap_peepminkey(sd.repl.heap.heap)); #endif +#endif /* OLD_UNUSED_CODE */ storeAppendPrintf(sentry, "Pending operations: %d\n", diskdinfo->away); } @@ -1850,7 +1644,7 @@ * This routine is called when the given swapdir needs reconfiguring */ void -storeDiskdDirReconfigure(SwapDir *sd, int index, char *path) +storeDiskdDirReconfigure(SwapDir * sd, int index, char *path) { char *token; int i; @@ -1896,7 +1690,7 @@ if (sd->flags.read_only != read_only) debug(3, 1) ("Cache dir '%s' now %s\n", path, read_only ? "Read-Only" : "Read-Write"); - diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo = (diskdinfo_t *) sd->fsdata; diskdinfo->magic1 = magic1; diskdinfo->magic2 = magic2; sd->flags.read_only = read_only; @@ -1906,10 +1700,10 @@ void storeDiskdDirDump(StoreEntry * entry, const char *name, SwapDir * s) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) s->fsdata; storeAppendPrintf(entry, "%s %s %s %d %d %d\n", name, - "diskd", + "diskd", s->path, s->max_size >> 10, diskdinfo->l1, @@ -1922,22 +1716,22 @@ static void storeDiskdDirFree(SwapDir * s) { - diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) s->fsdata; if (diskdinfo->swaplog_fd > -1) { file_close(diskdinfo->swaplog_fd); diskdinfo->swaplog_fd = -1; } filemapFreeMemory(diskdinfo->map); xfree(diskdinfo); - s->fsdata = NULL; /* Will aid debugging... */ + s->fsdata = NULL; /* Will aid debugging... */ } char * -storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath) +storeDiskdDirFullPath(SwapDir * SD, sfileno filn, char *fullpath) { LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; int L1 = diskdinfo->l1; int L2 = diskdinfo->l2; if (!fullpath) @@ -1957,28 +1751,28 @@ * This is called by storeCleanup() if -S was given on the command line. */ static int -storeDiskdCleanupDoubleCheck(SwapDir *sd, StoreEntry *e) +storeDiskdCleanupDoubleCheck(SwapDir * sd, StoreEntry * e) { struct stat sb; if (stat(storeDiskdDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { - debug(20, 0) ("storeDiskdCleanupDoubleCheck: MISSING SWAP FILE\n"); - debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", - storeDiskdDirFullPath(sd, e->swap_filen, NULL)); - storeEntryDump(e, 0); - return -1; - } + debug(20, 0) ("storeDiskdCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", + storeDiskdDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } if (e->swap_file_sz != sb.st_size) { - debug(20, 0) ("storeDiskdCleanupDoubleCheck: SIZE MISMATCH\n"); - debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", - storeDiskdDirFullPath(sd, e->swap_filen, NULL)); - debug(20, 0) ("storeDiskdCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", - e->swap_file_sz, (int) sb.st_size); - storeEntryDump(e, 0); - return -1; - } + debug(20, 0) ("storeDiskdCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", + storeDiskdDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } return 0; } @@ -1988,7 +1782,7 @@ * Called when a *new* fs is being setup. */ void -storeDiskdDirParse(SwapDir *sd, int index, char *path) +storeDiskdDirParse(SwapDir * sd, int index, char *path) { char *token; int i; @@ -2027,7 +1821,7 @@ diskdinfo = xmalloc(sizeof(diskdinfo_t)); if (diskdinfo == NULL) - fatal("storeDiskdDirParse: couldn't xmalloc() diskdinfo_t!\n"); + fatal("storeDiskdDirParse: couldn't xmalloc() diskdinfo_t!\n"); sd->index = index; sd->path = xstrdup(path); @@ -2036,7 +1830,7 @@ diskdinfo->l1 = l1; diskdinfo->l2 = l2; diskdinfo->swaplog_fd = -1; - diskdinfo->map = NULL; /* Debugging purposes */ + diskdinfo->map = NULL; /* Debugging purposes */ diskdinfo->suggest = 0; diskdinfo->magic1 = magic1; diskdinfo->magic2 = magic2; @@ -2062,41 +1856,12 @@ sd->log.open = storeDiskdDirOpenSwapLog; sd->log.close = storeDiskdDirCloseSwapLog; sd->log.write = storeDiskdDirSwapLog; - sd->log.clean.open = storeDiskdDirWriteCleanOpen; + sd->log.clean.start = storeDiskdDirWriteCleanStart; + sd->log.clean.nextentry = storeDiskdDirCleanLogNextEntry; + sd->log.clean.done = storeDiskdDirWriteCleanDone; /* Initialise replacement policy stuff */ -#if HEAP_REPLACEMENT - /* - * Create new heaps with cache replacement policies attached to them. - * The cache replacement policy is specified as either GDSF or LFUDA in - * the squid.conf configuration file. Note that the replacement policy - * applies only to the disk replacement algorithm. Memory replacement - * always uses GDSF since we want to maximize object hit rate. - */ - if (Config.replPolicy) { - if (tolower(Config.replPolicy[0]) == 'g') { - debug(20, 1) ("Using GDSF disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } else if (tolower(Config.replPolicy[0]) == 'l') { - if (tolower(Config.replPolicy[1]) == 'f') { - debug(20, 1) ("Using LFUDA disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); - } else if (tolower(Config.replPolicy[1]) == 'r') { - debug(20, 1) ("Using LRU heap disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); - } - } else { - debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } - } else { - debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } -#else - sd->repl.lru.list.head = NULL; - sd->repl.lru.list.tail = NULL; -#endif + sd->repl = createRemovalPolicy(Config.replPolicy); } /* @@ -2110,7 +1875,7 @@ } void -storeFsSetup_diskd(storefs_entry_t *storefs) +storeFsSetup_diskd(storefs_entry_t * storefs) { assert(!diskd_initialised); storefs->parsefunc = storeDiskdDirParse; @@ -2122,4 +1887,3 @@ debug(81, 1) ("diskd started\n"); diskd_initialised = 1; } - Index: squid/src/fs/diskd/store_diskd.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/diskd/store_diskd.h,v retrieving revision 1.1.6.1 retrieving revision 1.1.6.2 diff -u -r1.1.6.1 -r1.1.6.2 --- squid/src/fs/diskd/store_diskd.h 2 May 2000 22:50:29 -0000 1.1.6.1 +++ squid/src/fs/diskd/store_diskd.h 14 May 2000 23:22:20 -0000 1.1.6.2 @@ -82,17 +82,18 @@ static const int msg_snd_rcv_sz = sizeof(diomsg) - sizeof(mtyp_t); /* The diskd_state memory pool */ -extern MemPool * diskd_state_pool; +extern MemPool *diskd_state_pool; extern void storeDiskdDirMapBitReset(SwapDir *, sfileno); extern int storeDiskdDirMapBitAllocate(SwapDir *); -extern char * storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); +extern char *storeDiskdDirFullPath(SwapDir * SD, sfileno filn, char *fullpath); extern void storeDiskdDirUnlinkFile(SwapDir *, sfileno); extern void storeDiskdDirReplAdd(SwapDir *, StoreEntry *); extern void storeDiskdDirReplRemove(StoreEntry *); extern void storeDiskdShmPut(SwapDir *, int); extern void *storeDiskdShmGet(SwapDir *, int *); -extern void storeDiskdHandle(diomsg *M); +extern void storeDiskdHandle(diomsg * M); +extern void storeDiskdDirCallback(SwapDir *); /* Index: squid/src/fs/diskd/store_io_diskd.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/diskd/store_io_diskd.c,v retrieving revision 1.1.6.1 retrieving revision 1.1.6.2 diff -u -r1.1.6.1 -r1.1.6.2 --- squid/src/fs/diskd/store_io_diskd.c 2 May 2000 22:50:29 -0000 1.1.6.1 +++ squid/src/fs/diskd/store_io_diskd.c 14 May 2000 23:22:20 -0000 1.1.6.2 @@ -31,8 +31,8 @@ /* === PUBLIC =========================================================== */ storeIOState * -storeDiskdOpen(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, - STIOCB *callback, void *callback_data) +storeDiskdOpen(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) { sfileno f = e->swap_filen; int x; @@ -46,7 +46,7 @@ * If there are too many requests queued. */ if (diskdinfo->away > diskdinfo->magic1) { - debug(81, 3) ("storeDiskdOpen: FAILING, too many requests away\n"); + debug(81, 3) ("storeDiskdOpen: FAILING, too many requests away\n"); diskd_stats.open_fail_queue_len++; return NULL; } @@ -62,17 +62,17 @@ sio->e = e; cbdataLock(callback_data); - ((diskdstate_t *)(sio->fsstate))->flags.writing = 0; - ((diskdstate_t *)(sio->fsstate))->flags.reading = 0; - ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0; - ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++; + ((diskdstate_t *) (sio->fsstate))->flags.writing = 0; + ((diskdstate_t *) (sio->fsstate))->flags.reading = 0; + ((diskdstate_t *) (sio->fsstate))->flags.close_request = 0; + ((diskdstate_t *) (sio->fsstate))->id = diskd_stats.sio_id++; buf = storeDiskdShmGet(SD, &shm_offset); /* XXX WRONG!!! :) */ strcpy(buf, storeDiskdDirFullPath(SD, f, NULL)); x = storeDiskdSend(_MQD_OPEN, SD, - ((diskdstate_t *)(sio->fsstate))->id, + ((diskdstate_t *) (sio->fsstate))->id, sio, strlen(buf) + 1, O_RDONLY, @@ -88,8 +88,8 @@ } storeIOState * -storeDiskdCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, - STIOCB *callback, void *callback_data) +storeDiskdCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) { sfileno f; int x; @@ -105,7 +105,6 @@ diskd_stats.open_fail_queue_len++; return NULL; } - /* Allocate a number */ f = storeDiskdDirMapBitAllocate(SD); debug(81, 3) ("storeDiskdCreate: fileno %08X\n", f); @@ -122,17 +121,17 @@ sio->e = e; cbdataLock(callback_data); - ((diskdstate_t *)(sio->fsstate))->flags.writing = 0; - ((diskdstate_t *)(sio->fsstate))->flags.reading = 0; - ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0; - ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++; + ((diskdstate_t *) (sio->fsstate))->flags.writing = 0; + ((diskdstate_t *) (sio->fsstate))->flags.reading = 0; + ((diskdstate_t *) (sio->fsstate))->flags.close_request = 0; + ((diskdstate_t *) (sio->fsstate))->id = diskd_stats.sio_id++; buf = storeDiskdShmGet(SD, &shm_offset); /* XXX WRONG!!! :) */ strcpy(buf, storeDiskdDirFullPath(SD, f, NULL)); x = storeDiskdSend(_MQD_OPEN, SD, - ((diskdstate_t *)(sio->fsstate))->id, + ((diskdstate_t *) (sio->fsstate))->id, sio, strlen(buf) + 1, sio->mode, @@ -150,12 +149,12 @@ void -storeDiskdClose(SwapDir *SD, storeIOState * sio) +storeDiskdClose(SwapDir * SD, storeIOState * sio) { int x; - diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + diskdstate_t *diskdstate = (diskdstate_t *) sio->fsstate; debug(81, 3) ("storeDiskdClose: dirno %d, fileno %08X\n", SD->index, - sio->swap_filen); + sio->swap_filen); x = storeDiskdSend(_MQD_CLOSE, SD, diskdstate->id, @@ -170,12 +169,12 @@ } void -storeDiskdRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +storeDiskdRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) { int x; int shm_offset; char *rbuf; - diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + diskdstate_t *diskdstate = (diskdstate_t *) sio->fsstate; if (!cbdataValid(sio)) return; if (diskdstate->flags.reading) { @@ -208,12 +207,12 @@ } void -storeDiskdWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +storeDiskdWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) { int x; char *sbuf; int shm_offset; - diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + diskdstate_t *diskdstate = (diskdstate_t *) sio->fsstate; debug(81, 3) ("storeDiskdWrite: dirno %d, fileno %08X\n", SD->index, sio->swap_filen); if (!cbdataValid(sio)) { free_func(buf); @@ -223,7 +222,7 @@ sbuf = storeDiskdShmGet(SD, &shm_offset); xmemcpy(sbuf, buf, size); if (free_func) - free_func(buf); + free_func(buf); x = storeDiskdSend(_MQD_WRITE, SD, diskdstate->id, @@ -239,24 +238,23 @@ } void -storeDiskdUnlink(SwapDir *SD, StoreEntry *e) +storeDiskdUnlink(SwapDir * SD, StoreEntry * e) { int x; int shm_offset; char *buf; - diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; debug(81, 3) ("storeDiskdUnlink: dirno %d, fileno %08X\n", SD->index, - e->swap_filen); + e->swap_filen); storeDiskdDirReplRemove(e); storeDiskdDirMapBitReset(SD, e->swap_filen); if (diskdinfo->away >= diskdinfo->magic1) { - /* Damn, we need to issue a sync unlink here :( */ - debug(50, 2) ("storeDiskUnlink: Out of queue space, sync unlink\n"); - storeDiskdDirUnlinkFile(SD, e->swap_filen); - return; + /* Damn, we need to issue a sync unlink here :( */ + debug(50, 2) ("storeDiskUnlink: Out of queue space, sync unlink\n"); + storeDiskdDirUnlinkFile(SD, e->swap_filen); + return; } - /* We can attempt a diskd unlink */ buf = storeDiskdShmGet(SD, &shm_offset); strcpy(buf, storeDiskdDirFullPath(SD, e->swap_filen, NULL)); @@ -269,7 +267,7 @@ shm_offset); if (x < 0) { debug(50, 1) ("storeDiskdSend UNLINK: %s\n", xstrerror()); - unlink(buf); /* XXX EWW! */ + unlink(buf); /* XXX EWW! */ storeDiskdShmPut(SD, shm_offset); } } @@ -342,7 +340,7 @@ storeDiskdWriteDone(diomsg * M) { storeIOState *sio = M->callback_data; - diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + diskdstate_t *diskdstate = (diskdstate_t *) sio->fsstate; Counter.syscalls.disk.writes++; diskdstate->flags.writing = 0; debug(81, 3) ("storeDiskdWriteDone: dirno %d, fileno %08x status %d\n", @@ -423,7 +421,7 @@ static int send_errors = 0; static int last_seq_no = 0; static int seq_no = 0; - diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; M.mtype = mtype; M.callback_data = sio; M.size = size; @@ -447,6 +445,13 @@ cbdataUnlock(M.callback_data); assert(++send_errors < 100); } + /* + * We have to drain the queue here if necessary. If we don't, + * then we can have a lot of messages in the queue (probably + * up to 2*magic1) and we can run out of shared memory buffers. + */ + if (diskdinfo->away > diskdinfo->magic2) + storeDiskdDirCallback(sd); return x; } @@ -458,7 +463,6 @@ static void storeDiskdIOFreeEntry(void *sio, int foo) { - memPoolFree(diskd_state_pool, ((storeIOState *)sio)->fsstate); + memPoolFree(diskd_state_pool, ((storeIOState *) sio)->fsstate); memFree(sio, MEM_STORE_IO); } - Index: squid/src/fs/ufs/store_dir_ufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/ufs/store_dir_ufs.c,v retrieving revision 1.1.6.6.2.2 retrieving revision 1.1.6.6.2.3 diff -u -r1.1.6.6.2.2 -r1.1.6.6.2.3 --- squid/src/fs/ufs/store_dir_ufs.c 2 May 2000 22:50:30 -0000 1.1.6.6.2.2 +++ squid/src/fs/ufs/store_dir_ufs.c 14 May 2000 23:22:20 -0000 1.1.6.6.2.3 @@ -71,7 +71,7 @@ static int n_ufs_dirs = 0; static int *ufs_dir_index = NULL; -MemPool * ufs_state_pool = NULL; +MemPool *ufs_state_pool = NULL; static int ufs_initialised = 0; static char *storeUfsDirSwapSubDir(SwapDir *, int subdirn); @@ -83,7 +83,7 @@ static EVH storeUfsDirRebuildFromDirectory; static EVH storeUfsDirRebuildFromSwapLog; static int storeUfsDirGetNextFile(RebuildState *, int *sfileno, int *size); -static StoreEntry *storeUfsDirAddDiskRestore(SwapDir *SD, const cache_key * key, +static StoreEntry *storeUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -99,9 +99,10 @@ static STLOGOPEN storeUfsDirOpenSwapLog; static STINIT storeUfsDirInit; static STFREE storeUfsDirFree; -static STLOGCLEANOPEN storeUfsDirWriteCleanOpen; -static void storeUfsDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANSTART storeUfsDirWriteCleanStart; +static STLOGCLEANNEXTENTRY storeUfsDirCleanLogNextEntry; static STLOGCLEANWRITE storeUfsDirWriteCleanEntry; +static STLOGCLEANDONE storeUfsDirWriteCleanDone; static STLOGCLOSE storeUfsDirCloseSwapLog; static STLOGWRITE storeUfsDirSwapLog; static STNEWFS storeUfsDirNewfs; @@ -119,10 +120,6 @@ static void storeUfsDirStats(SwapDir *, StoreEntry *); static void storeUfsDirInitBitmap(SwapDir *); static int storeUfsDirValidFileno(SwapDir *, sfileno); -static int storeUfsDirCheckExpired(SwapDir *, StoreEntry *); -#if !HEAP_REPLACEMENT -static time_t storeUfsDirExpiredReferenceAge(SwapDir *); -#endif /* * These functions were ripped straight out of the heart of store_dir.c. @@ -132,43 +129,43 @@ */ int -storeUfsDirMapBitTest(SwapDir *SD, int fn) +storeUfsDirMapBitTest(SwapDir * SD, int fn) { sfileno filn = fn; ufsinfo_t *ufsinfo; - ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo = (ufsinfo_t *) SD->fsdata; return file_map_bit_test(ufsinfo->map, filn); } - + void -storeUfsDirMapBitSet(SwapDir *SD, int fn) -{ +storeUfsDirMapBitSet(SwapDir * SD, int fn) +{ sfileno filn = fn; ufsinfo_t *ufsinfo; - ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo = (ufsinfo_t *) SD->fsdata; file_map_bit_set(ufsinfo->map, filn); } - + void -storeUfsDirMapBitReset(SwapDir *SD, int fn) -{ +storeUfsDirMapBitReset(SwapDir * SD, int fn) +{ sfileno filn = fn; ufsinfo_t *ufsinfo; - ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo = (ufsinfo_t *) SD->fsdata; file_map_bit_reset(ufsinfo->map, filn); } int -storeUfsDirMapBitAllocate(SwapDir *SD) +storeUfsDirMapBitAllocate(SwapDir * SD) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata; int fn; fn = file_map_allocate(ufsinfo->map, ufsinfo->suggest); file_map_bit_set(ufsinfo->map, fn); ufsinfo->suggest = fn + 1; return fn; } - + /* * Initialise the ufs bitmap * @@ -176,16 +173,16 @@ * configured, we allocate a new bitmap and 'grow' the old one into it. */ static void -storeUfsDirInitBitmap(SwapDir *sd) +storeUfsDirInitBitmap(SwapDir * sd) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; if (ufsinfo->map == NULL) { - /* First time */ + /* First time */ ufsinfo->map = file_map_create(); } else if (ufsinfo->map->max_n_files) { - /* it grew, need to expand */ - /* XXX We don't need it anymore .. */ + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ } /* else it shrunk, and we leave the old one in place */ } @@ -193,7 +190,7 @@ static char * storeUfsDirSwapSubDir(SwapDir * sd, int subdirn) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); assert(0 <= subdirn && subdirn < ufsinfo->l1); @@ -246,7 +243,7 @@ static int storeUfsDirVerifyCacheDirs(SwapDir * sd) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; int j; const char *path = sd->path; @@ -263,7 +260,7 @@ static void storeUfsDirCreateSwapSubDirs(SwapDir * sd) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; int i, k; int should_exist; LOCAL_ARRAY(char, name, MAXPATHLEN); @@ -290,12 +287,12 @@ char *pathtmp2; if (Config.Log.swap) { xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); - while (index(pathtmp,'/')) - *index(pathtmp,'/')='.'; - while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') - pathtmp[strlen(pathtmp)-1]= '\0'; - for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); - snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + while (index(pathtmp, '/')) + *index(pathtmp, '/') = '.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') + pathtmp[strlen(pathtmp) - 1] = '\0'; + for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { strcat(path, "."); snprintf(digit, 32, "%02d", sd->index); @@ -313,7 +310,7 @@ static void storeUfsDirOpenSwapLog(SwapDir * sd) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; char *path; int fd; path = storeUfsDirSwapLogFile(sd, NULL); @@ -333,7 +330,7 @@ static void storeUfsDirCloseSwapLog(SwapDir * sd) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; if (ufsinfo->swaplog_fd < 0) /* not open */ return; file_close(ufsinfo->swaplog_fd); @@ -351,9 +348,9 @@ { static int started_clean_event = 0; static const char *errmsg = - "\tFailed to verify one of the swap directories, Check cache.log\n" - "\tfor details. Run 'squid -z' to create swap directories\n" - "\tif needed, or if running Squid for the first time."; + "\tFailed to verify one of the swap directories, Check cache.log\n" + "\tfor details. Run 'squid -z' to create swap directories\n" + "\tif needed, or if running Squid for the first time."; storeUfsDirInitBitmap(sd); if (storeUfsDirVerifyCacheDirs(sd) < 0) fatal(errmsg); @@ -432,7 +429,7 @@ tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); if (tlv_list == NULL) { debug(20, 1) ("storeUfsDirRebuildFromDirectory: failed to get meta data\n"); - /* XXX shouldn't this be a call to storeUfsUnlink ? */ + /* XXX shouldn't this be a call to storeUfsUnlink ? */ storeUfsDirUnlinkFile(SD, sfileno); continue; } @@ -552,13 +549,13 @@ * because adding to store_swap_size happens in * the cleanup procedure. */ - storeExpireNow(e); - storeReleaseRequest(e); - storeUfsDirReplRemove(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeUfsDirReplRemove(e); if (e->swap_filen > -1) { storeUfsDirMapBitReset(SD, e->swap_filen); e->swap_filen = -1; - e->swap_dirn = -1; + e->swap_dirn = -1; } storeRelease(e); rb->counts.objcount--; @@ -603,10 +600,7 @@ e->lastmod = s.lastmod; e->flags = s.flags; e->refcount += s.refcount; -#if HEAP_REPLACEMENT - storeHeapPositionUpdate(e, SD); - storeUfsDirUnrefObj(SD, e); -#endif + storeUfsDirUnrefObj(SD, e); } else { debug_trap("storeUfsDirRebuildFromSwapLog: bad condition"); debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); @@ -640,14 +634,14 @@ } else if (e) { /* key already exists, this swapfile not being used */ /* junk old, load new */ - storeExpireNow(e); - storeReleaseRequest(e); - storeUfsDirReplRemove(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeUfsDirReplRemove(e); if (e->swap_filen > -1) { - /* Make sure we don't actually unlink the file */ + /* Make sure we don't actually unlink the file */ storeUfsDirMapBitReset(SD, e->swap_filen); e->swap_filen = -1; - e->swap_dirn = -1; + e->swap_dirn = -1; } storeRelease(e); rb->counts.dupcount++; @@ -677,15 +671,14 @@ storeUfsDirGetNextFile(RebuildState * rb, int *sfileno, int *size) { SwapDir *SD = rb->sd; - ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata; int fd = -1; int used = 0; int dirs_opened = 0; debug(20, 3) ("storeUfsDirGetNextFile: flag=%d, %d: /%02X/%02X\n", rb->flags.init, rb->sd->index, - rb->curlvl1, - rb->curlvl2); + rb->curlvl1, rb->curlvl2); if (rb->done) return -2; while (fd < 0 && rb->done == 0) { @@ -701,7 +694,8 @@ if (0 == rb->in_dir) { /* we need to read in a new directory */ snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", rb->sd->path, - rb->curlvl1, rb->curlvl2); + rb->curlvl1, + rb->curlvl2); if (rb->flags.init && rb->td != NULL) closedir(rb->td); rb->td = NULL; @@ -763,7 +757,7 @@ /* Add a new object to the cache with empty memory copy and pointer to disk * use to rebuild store from disk. */ static StoreEntry * -storeUfsDirAddDiskRestore(SwapDir *SD, const cache_key * key, +storeUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key, int file_number, size_t swap_file_sz, time_t expires, @@ -786,9 +780,6 @@ e->swap_dirn = SD->index; e->swap_file_sz = swap_file_sz; e->lock_count = 0; -#if !HEAP_REPLACEMENT - e->refcount = 0; -#endif e->lastref = lastref; e->timestamp = timestamp; e->expires = expires; @@ -844,7 +835,7 @@ static void storeUfsDirCloseTmpSwapLog(SwapDir * sd) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new")); int fd; @@ -872,7 +863,7 @@ static FILE * storeUfsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); char *clean_path = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean")); char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new")); @@ -925,6 +916,7 @@ char *outbuf; off_t outbuf_offset; int fd; + RemovalPolicyWalker *walker; }; #define CLEAN_BUF_SZ 16384 @@ -934,7 +926,7 @@ * we succeed, and assign the 'func' and 'data' return pointers. */ static int -storeUfsDirWriteCleanOpen(SwapDir * sd) +storeUfsDirWriteCleanStart(SwapDir * sd) { struct _clean_state *state = xcalloc(1, sizeof(*state)); struct stat sb; @@ -945,6 +937,7 @@ state->cln = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean")); state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); state->outbuf_offset = 0; + state->walker = sd->repl->WalkInit(sd->repl); unlink(state->new); unlink(state->cln); state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); @@ -962,18 +955,27 @@ } /* + * Get the next entry that is a candidate for clean log writing + */ +const StoreEntry * +storeUfsDirCleanLogNextEntry(SwapDir * sd) +{ + const StoreEntry *entry = NULL; + struct _clean_state *state = sd->log.clean.state; + if (state->walker) + entry = state->walker->Next(state->walker); + return entry; +} + +/* * "write" an entry to the clean log file. */ static void -storeUfsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +storeUfsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e) { storeSwapLogData s; static size_t ss = sizeof(storeSwapLogData); struct _clean_state *state = sd->log.clean.state; - if (NULL == e) { - storeUfsDirWriteCleanClose(sd); - return; - } memset(&s, '\0', ss); s.op = (char) SWAP_LOG_ADD; s.swap_filen = e->swap_filen; @@ -1005,11 +1007,12 @@ } static void -storeUfsDirWriteCleanClose(SwapDir * sd) +storeUfsDirWriteCleanDone(SwapDir * sd) { struct _clean_state *state = sd->log.clean.state; if (state->fd < 0) return; + state->walker->Done(state->walker); if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", state->new, xstrerror()); @@ -1058,7 +1061,7 @@ static void storeUfsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata; storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); s->op = (char) op; s->swap_filen = e->swap_filen; @@ -1117,7 +1120,7 @@ N0 = n_ufs_dirs; D0 = ufs_dir_index[swap_index % N0]; SD = &Config.cacheSwap.swapDirs[D0]; - ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo = (ufsinfo_t *) SD->fsdata; N1 = ufsinfo->l1; D1 = (swap_index / N0) % N1; N2 = ufsinfo->l2; @@ -1139,7 +1142,7 @@ while ((de = readdir(dp)) != NULL && k < 20) { if (sscanf(de->d_name, "%X", &swapfileno) != 1) continue; - fn = swapfileno; /* XXX should remove this cruft ! */ + fn = swapfileno; /* XXX should remove this cruft ! */ if (storeUfsDirValidFileno(SD, fn)) if (storeUfsDirMapBitTest(SD, fn)) if (storeUfsFilenoBelongsHere(fn, D0, D1, D2)) @@ -1185,7 +1188,7 @@ assert(n_ufs_dirs); if (NULL == ufs_dir_index) { SwapDir *sd; - ufsinfo_t *ufsinfo; + ufsinfo_t *ufsinfo; /* * Initialize the little array that translates UFS cache_dir * number into the Config.cacheSwap.swapDirs array index. @@ -1196,7 +1199,7 @@ if (!storeUfsDirIs(sd)) continue; ufs_dir_index[n++] = i; - ufsinfo = (ufsinfo_t *)sd->fsdata; + ufsinfo = (ufsinfo_t *) sd->fsdata; j += (ufsinfo->l1 * ufsinfo->l2); } assert(n == n_ufs_dirs); @@ -1235,7 +1238,7 @@ int filn = fn; ufsinfo_t *ufsinfo; assert(F0 < Config.cacheSwap.n_configured); - ufsinfo = (ufsinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata; + ufsinfo = (ufsinfo_t *) Config.cacheSwap.swapDirs[F0].fsdata; L1 = ufsinfo->l1; L2 = ufsinfo->l2; D1 = ((filn / L2) / L2) % L1; @@ -1247,167 +1250,60 @@ return 1; } -int -storeUfsDirValidFileno(SwapDir *SD, sfileno filn) +int +storeUfsDirValidFileno(SwapDir * SD, sfileno filn) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata; if (filn < 0) - return 0; - if (filn > ufsinfo->map->max_n_files) - return 0; + return 0; + if (filn > ufsinfo->map->max_n_files) + return 0; return 1; } void -storeUfsDirMaintain(SwapDir *SD) +storeUfsDirMaintain(SwapDir * SD) { StoreEntry *e = NULL; - int scanned = 0; - int locked = 0; - int expired = 0; + int removed = 0; int max_scan; int max_remove; double f; - static time_t last_warn_time = 0; -#if !HEAP_REPLACEMENT - dlink_node *m; - dlink_node *prev = NULL; -#else - heap_key age; - heap_key min_age = 0.0; - link_list *locked_entries = NULL; -#if HEAP_REPLACEMENT_DEBUG - if (!verify_heap_property(SD->repl.heap.heap)) { - debug(20, 1) ("Heap property violated!\n"); - } -#endif -#endif + RemovalPurgeWalker *walker; /* We can't delete objects while rebuilding swap */ if (store_dirs_rebuilding) { - return; + return; } else { - f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); - f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; - max_scan = (int) (f * 400.0 + 100.0); - max_remove = (int) (f * 70.0 + 10.0); + /* XXX FixMe: This should use the cache_dir hig/low values, not the + * global ones + */ + f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + max_scan = (int) (f * 400.0 + 100.0); + max_remove = (int) (f * 70.0 + 10.0); /* * This is kinda cheap, but so we need this priority hack? - */ -#if 0 - eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); -#endif + */ } debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove); -#if HEAP_REPLACEMENT - while (heap_nodes(SD->repl.heap.heap) > 0) { - if (store_swap_size < store_swap_low) - break; - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; - age = heap_peepminkey(SD->repl.heap.heap); - e = heap_extractmin(SD->repl.heap.heap); - e->repl.node = NULL; /* no longer in the heap */ - scanned++; - if (storeEntryLocked(e)) { - /* - * Entry is in use ... put it in a linked list to ignore it. - */ - if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - /* - * If this was a "SPECIAL" do not add it back into the heap. - * It will always be "SPECIAL" and therefore never removed. - */ - debug(20, 4) ("storeUfsDirMaintain: locked url %s\n", - (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e-> -key)); - linklistPush(&locked_entries, e); - } - locked++; - continue; - } else if (storeUfsDirCheckExpired(SD, e)) { - /* - * Note: This will not check the reference age ifdef - * HEAP_REPLACEMENT, but it does some other useful - * checks... - */ - expired++; - debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", - age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); - min_age = age; - storeRelease(e); - } else { - /* - * Did not expire the object so we need to add it back - * into the heap! - */ - debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", - storeKeyText(e->key)); - linklistPush(&locked_entries, e); - continue; - } - if (store_swap_size < store_swap_low) - break; - else if (expired >= max_remove) - break; - else if (scanned >= max_scan) - break; - } - /* - * Bump the heap age factor. - */ - if (min_age > 0.0) - SD->repl.heap.heap->age = min_age; - /* - * Reinsert all bumped locked entries back into heap... - */ - while ((e = linklistShift(&locked_entries))) - e->repl.node = heap_insert(SD->repl.heap.heap, e); -#else - for (m = SD->repl.lru.list.tail; m; m = prev) { - prev = m->prev; - e = m->data; - scanned++; - if (storeEntryLocked(e)) { - /* - * If there is a locked entry at the tail of the LRU list, - * move it to the beginning to get it out of the way. - * Theoretically, we might have all locked objects at the - * tail, and then we'll never remove anything here and the - * LRU age will go to zero. - */ - if (memInUse(MEM_STOREENTRY) > max_scan) { - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); - } - locked++; - - } else if (storeUfsDirCheckExpired(SD, e)) { - expired++; - storeRelease(e); - } - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; - } -#endif - debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l -ocked %d f=%.03f\n", - scanned, max_scan, expired, max_remove, locked, f); - debug(20, 3) ("storeMaintainSwapSpace stats:\n"); - debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); - debug(20, 3) (" %6d were scanned\n", scanned); - debug(20, 3) (" %6d were locked\n", locked); - debug(20, 3) (" %6d were expired\n", expired); - if (store_swap_size < Config.Swap.maxSize) - return; - if (squid_curtime - last_warn_time < 10) - return; - debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", - store_swap_size, Config.Swap.maxSize); - last_warn_time = squid_curtime; + walker = SD->repl->PurgeInit(SD->repl, max_scan); + while (1) { + /* XXX FixMe: This should use the cache_dir hig/low values, not the + * global ones + */ + if (store_swap_size < store_swap_low) + break; + if (removed >= max_remove) + break; + e = walker->Next(walker); + if (!e) + break; /* no more objects */ + removed++; + storeRelease(e); + } + walker->Done(walker); + debug(20, (removed ? 2 : 3)) ("storeUfsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n", + SD->path, removed, max_remove, f, max_scan); } /* @@ -1418,14 +1314,14 @@ * happily store anything as long as the LRU time isn't too small. */ int -storeUfsDirCheckObj(SwapDir *SD, const StoreEntry *e) +storeUfsDirCheckObj(SwapDir * SD, const StoreEntry * e) { -#if !HEAP_REPLACEMENT +#if OLD_UNUSED_CODE if (storeUfsDirExpiredReferenceAge(SD) < 300) { - debug(20, 3) ("storeUfsDirCheckObj: NO: LRU Age = %d\n", - storeUfsDirExpiredReferenceAge(SD)); - /* store_check_cachable_hist.no.lru_age_too_low++; */ - return -1; + debug(20, 3) ("storeUfsDirCheckObj: NO: LRU Age = %d\n", + storeUfsDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; } #endif /* Return 999 (99.9%) constant load */ @@ -1439,20 +1335,12 @@ * maintain replacement information within the storage fs. */ void -storeUfsDirRefObj(SwapDir *SD, StoreEntry *e) +storeUfsDirRefObj(SwapDir * SD, StoreEntry * e) { debug(1, 3) ("storeUfsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, - e->swap_filen); -#if HEAP_REPLACEMENT - /* Nothing to do here */ -#else - /* Reference the object */ - if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && - !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); - } -#endif + e->swap_filen); + if (SD->repl->Referenced) + SD->repl->Referenced(SD->repl, e, &e->repl); } /* @@ -1461,14 +1349,12 @@ * removed, to maintain replacement information within the storage fs. */ void -storeUfsDirUnrefObj(SwapDir *SD, StoreEntry *e) +storeUfsDirUnrefObj(SwapDir * SD, StoreEntry * e) { debug(1, 3) ("storeUfsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn, - e->swap_filen); -#if HEAP_REPLACEMENT - if (e->repl.node) - heap_update(SD->repl.heap.heap, e->repl.node, e); -#endif + e->swap_filen); + if (SD->repl->Dereferenced) + SD->repl->Dereferenced(SD->repl, e, &e->repl); } /* @@ -1479,124 +1365,34 @@ * forced this bit of code here. Eeek. */ void -storeUfsDirUnlinkFile(SwapDir *SD, sfileno f) +storeUfsDirUnlinkFile(SwapDir * SD, sfileno f) { debug(79, 3) ("storeUfsDirUnlinkFile: unlinking fileno %08X\n", f); storeUfsDirMapBitReset(SD, f); unlinkdUnlink(storeUfsDirFullPath(SD, f, NULL)); } -#if !HEAP_REPLACEMENT -/* - * storeUfsDirExpiredReferenceAge - * - * The LRU age is scaled exponentially between 1 minute and - * Config.referenceAge , when store_swap_low < store_swap_size < - * store_swap_high. This keeps store_swap_size within the low and high - * water marks. If the cache is very busy then store_swap_size stays - * closer to the low water mark, if it is not busy, then it will stay - * near the high water mark. The LRU age value can be examined on the - * cachemgr 'info' page. - */ -static time_t -storeUfsDirExpiredReferenceAge(SwapDir *SD) -{ - double x; - double z; - time_t age; - long store_high, store_low; - - store_high = (long) (((float) SD->max_size * - (float) Config.Swap.highWaterMark) / (float) 100); - store_low = (long) (((float) SD->max_size * - (float) Config.Swap.lowWaterMark) / (float) 100); - debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size); - - x = (double) (store_high - SD->cur_size) / - (store_high - store_low); - x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; - z = pow((double) (Config.referenceAge / 60), x); - age = (time_t) (z * 60.0); - if (age < 60) - age = 60; - else if (age > Config.referenceAge) - age = Config.referenceAge; - return age; -} -#endif - -/* - * storeUfsDirCheckExpired - * - * Check whether the given object is expired or not - * It breaks layering a little by calling the upper layers to find - * out whether the object is locked or not, but we can't help this - * right now. - */ -static int -storeUfsDirCheckExpired(SwapDir *SD, StoreEntry *e) -{ - if (storeEntryLocked(e)) - return 0; - if (EBIT_TEST(e->flags, RELEASE_REQUEST)) - return 1; - if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) - return 1; - -#if HEAP_REPLACEMENT - /* - * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap - * controls the replacement of objects. - */ - return 1; -#else - if (squid_curtime - e->lastref > storeUfsDirExpiredReferenceAge(SD)) - return 1; - return 0; -#endif -} - /* * Add and remove the given StoreEntry from the replacement policy in * use. */ void -storeUfsDirReplAdd(SwapDir *SD, StoreEntry *e) +storeUfsDirReplAdd(SwapDir * SD, StoreEntry * e) { debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e, - SD->index); -#if HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->repl.node = heap_insert(SD->repl.heap.heap, e); - debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node); - } -#else - /* Shouldn't we not throw special objects into the lru ? */ - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); -#endif + SD->index); + SD->repl->Add(SD->repl, e, &e->repl); } void -storeUfsDirReplRemove(StoreEntry *e) +storeUfsDirReplRemove(StoreEntry * e) { SwapDir *SD = INDEXSD(e->swap_dirn); debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e, - SD->index); -#if HEAP_REPLACEMENT - /* And now, release the object from the replacement policy */ - if (e->repl.node) { - debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n", - e->repl.node); - heap_delete(SD->repl.heap.heap, e->repl.node); - e->repl.node = NULL; - } -#else - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); -#endif + SD->index); + SD->repl->Remove(SD->repl, e, &e->repl); } @@ -1604,51 +1400,51 @@ /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ void -storeUfsDirStats(SwapDir *SD, StoreEntry * sentry) +storeUfsDirStats(SwapDir * SD, StoreEntry * sentry) { ufsinfo_t *ufsinfo; #if HAVE_STATVFS struct statvfs sfs; #endif - ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo = (ufsinfo_t *) SD->fsdata; storeAppendPrintf(sentry, "First level subdirectories: %d\n", ufsinfo->l1); storeAppendPrintf(sentry, "Second level subdirectories: %d\n", ufsinfo->l2); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", - 100.0 * SD->cur_size / SD->max_size); + 100.0 * SD->cur_size / SD->max_size); storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", - ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files, - percent(ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files)); + ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files, + percent(ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files)); #if HAVE_STATVFS #define fsbtoblk(num, fsbs, bs) \ (((fsbs) != 0 && (fsbs) < (bs)) ? \ (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) - if (!statvfs(SD->path, &sfs)) { - storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", - fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), - fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), - percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); - storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", - sfs.f_files - sfs.f_ffree, sfs.f_files, - percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); } #endif storeAppendPrintf(sentry, "Flags:"); if (SD->flags.selected) - storeAppendPrintf(sentry, " SELECTED"); + storeAppendPrintf(sentry, " SELECTED"); if (SD->flags.read_only) - storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, " READ-ONLY"); storeAppendPrintf(sentry, "\n"); +#if OLD_UNUSED_CODE #if !HEAP_REPLACEMENT storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", - (double) storeUfsDirExpiredReferenceAge(SD) / 86400.0); + (double) storeUfsDirExpiredReferenceAge(SD) / 86400.0); #else -#if 0 storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", - heap_peepminkey(sd.repl.heap.heap)); -#endif + heap_peepminkey(sd.repl.heap.heap)); #endif +#endif /* OLD_UNUSED_CODE */ } /* @@ -1657,7 +1453,7 @@ * This routine is called when the given swapdir needs reconfiguring */ void -storeUfsDirReconfigure(SwapDir *sd, int index, char *path) +storeUfsDirReconfigure(SwapDir * sd, int index, char *path) { char *token; int i; @@ -1700,10 +1496,10 @@ void storeUfsDirDump(StoreEntry * entry, const char *name, SwapDir * s) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)s->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) s->fsdata; storeAppendPrintf(entry, "%s %s %s %d %d %d\n", name, - "ufs", + "ufs", s->path, s->max_size >> 10, ufsinfo->l1, @@ -1716,22 +1512,22 @@ static void storeUfsDirFree(SwapDir * s) { - ufsinfo_t *ufsinfo = (ufsinfo_t *)s->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) s->fsdata; if (ufsinfo->swaplog_fd > -1) { file_close(ufsinfo->swaplog_fd); ufsinfo->swaplog_fd = -1; } filemapFreeMemory(ufsinfo->map); xfree(ufsinfo); - s->fsdata = NULL; /* Will aid debugging... */ + s->fsdata = NULL; /* Will aid debugging... */ } char * -storeUfsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath) +storeUfsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath) { LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata; int L1 = ufsinfo->l1; int L2 = ufsinfo->l2; if (!fullpath) @@ -1751,28 +1547,28 @@ * This is called by storeCleanup() if -S was given on the command line. */ static int -storeUfsCleanupDoubleCheck(SwapDir *sd, StoreEntry *e) +storeUfsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e) { struct stat sb; if (stat(storeUfsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { - debug(20, 0) ("storeUfsCleanupDoubleCheck: MISSING SWAP FILE\n"); - debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n", - storeUfsDirFullPath(sd, e->swap_filen, NULL)); - storeEntryDump(e, 0); - return -1; - } + debug(20, 0) ("storeUfsCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n", + storeUfsDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } if (e->swap_file_sz != sb.st_size) { - debug(20, 0) ("storeUfsCleanupDoubleCheck: SIZE MISMATCH\n"); - debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n", - storeUfsDirFullPath(sd, e->swap_filen, NULL)); - debug(20, 0) ("storeUfsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", - e->swap_file_sz, (int) sb.st_size); - storeEntryDump(e, 0); - return -1; - } + debug(20, 0) ("storeUfsCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n", + storeUfsDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeUfsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } return 0; } @@ -1782,7 +1578,7 @@ * Called when a *new* fs is being setup. */ void -storeUfsDirParse(SwapDir *sd, int index, char *path) +storeUfsDirParse(SwapDir * sd, int index, char *path) { char *token; int i; @@ -1810,7 +1606,7 @@ ufsinfo = xmalloc(sizeof(ufsinfo_t)); if (ufsinfo == NULL) - fatal("storeUfsDirParse: couldn't xmalloc() ufsinfo_t!\n"); + fatal("storeUfsDirParse: couldn't xmalloc() ufsinfo_t!\n"); sd->index = index; sd->path = xstrdup(path); @@ -1819,7 +1615,7 @@ ufsinfo->l1 = l1; ufsinfo->l2 = l2; ufsinfo->swaplog_fd = -1; - ufsinfo->map = NULL; /* Debugging purposes */ + ufsinfo->map = NULL; /* Debugging purposes */ ufsinfo->suggest = 0; sd->flags.read_only = read_only; sd->init = storeUfsDirInit; @@ -1843,41 +1639,12 @@ sd->log.open = storeUfsDirOpenSwapLog; sd->log.close = storeUfsDirCloseSwapLog; sd->log.write = storeUfsDirSwapLog; - sd->log.clean.open = storeUfsDirWriteCleanOpen; + sd->log.clean.start = storeUfsDirWriteCleanStart; + sd->log.clean.nextentry = storeUfsDirCleanLogNextEntry; + sd->log.clean.done = storeUfsDirWriteCleanDone; /* Initialise replacement policy stuff */ -#if HEAP_REPLACEMENT - /* - * Create new heaps with cache replacement policies attached to them. - * The cache replacement policy is specified as either GDSF or LFUDA in - * the squid.conf configuration file. Note that the replacement policy - * applies only to the disk replacement algorithm. Memory replacement - * always uses GDSF since we want to maximize object hit rate. - */ - if (Config.replPolicy) { - if (tolower(Config.replPolicy[0]) == 'g') { - debug(20, 1) ("Using GDSF disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } else if (tolower(Config.replPolicy[0]) == 'l') { - if (tolower(Config.replPolicy[1]) == 'f') { - debug(20, 1) ("Using LFUDA disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); - } else if (tolower(Config.replPolicy[1]) == 'r') { - debug(20, 1) ("Using LRU heap disk replacement policy\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); - } - } else { - debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } - } else { - debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); - sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } -#else - sd->repl.lru.list.head = NULL; - sd->repl.lru.list.tail = NULL; -#endif + sd->repl = createRemovalPolicy(Config.replPolicy); } /* @@ -1891,7 +1658,7 @@ } void -storeFsSetup_ufs(storefs_entry_t *storefs) +storeFsSetup_ufs(storefs_entry_t * storefs) { assert(!ufs_initialised); storefs->parsefunc = storeUfsDirParse; @@ -1900,4 +1667,3 @@ ufs_state_pool = memPoolCreate("UFS IO State data", sizeof(ufsstate_t)); ufs_initialised = 1; } - Index: squid/src/fs/ufs/store_io_ufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/ufs/store_io_ufs.c,v retrieving revision 1.1.6.6.2.3 retrieving revision 1.1.6.6.2.4 diff -u -r1.1.6.6.2.3 -r1.1.6.6.2.4 --- squid/src/fs/ufs/store_io_ufs.c 2 May 2000 22:50:30 -0000 1.1.6.6.2.3 +++ squid/src/fs/ufs/store_io_ufs.c 14 May 2000 23:22:20 -0000 1.1.6.6.2.4 @@ -45,8 +45,8 @@ /* === PUBLIC =========================================================== */ storeIOState * -storeUfsOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback, - STIOCB * callback, void *callback_data) +storeUfsOpen(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) { sfileno f = e->swap_filen; char *path = storeUfsDirFullPath(SD, f, NULL); @@ -71,12 +71,12 @@ sio->callback_data = callback_data; cbdataLock(callback_data); sio->e = e; - ((ufsstate_t *)(sio->fsstate))->fd = fd; - ((ufsstate_t *)(sio->fsstate))->flags.writing = 0; - ((ufsstate_t *)(sio->fsstate))->flags.reading = 0; - ((ufsstate_t *)(sio->fsstate))->flags.close_request = 0; + ((ufsstate_t *) (sio->fsstate))->fd = fd; + ((ufsstate_t *) (sio->fsstate))->flags.writing = 0; + ((ufsstate_t *) (sio->fsstate))->flags.reading = 0; + ((ufsstate_t *) (sio->fsstate))->flags.close_request = 0; if (fstat(fd, &sb) == 0) - sio->st_size = sb.st_size; + sio->st_size = sb.st_size; store_open_disk_fd++; /* We should update the heap/dlink position here ! */ @@ -84,13 +84,13 @@ } storeIOState * -storeUfsCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data) +storeUfsCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data) { storeIOState *sio; int fd; int mode = (O_WRONLY | O_CREAT | O_TRUNC); char *path; - ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata; sfileno filn; sdirno dirn; @@ -104,8 +104,8 @@ debug(79, 3) ("storeUfsCreate: fileno %08X\n", filn); fd = file_open(path, mode); if (fd < 0) { - debug(79, 3) ("storeUfsCreate: got failure (%d)\n", errno); - return NULL; + debug(79, 3) ("storeUfsCreate: got failure (%d)\n", errno); + return NULL; } debug(79, 3) ("storeUfsCreate: opened FD %d\n", fd); sio = memAllocate(MEM_STORE_IO); @@ -119,10 +119,10 @@ sio->callback_data = callback_data; cbdataLock(callback_data); sio->e = (StoreEntry *) e; - ((ufsstate_t *)(sio->fsstate))->fd = fd; - ((ufsstate_t *)(sio->fsstate))->flags.writing = 0; - ((ufsstate_t *)(sio->fsstate))->flags.reading = 0; - ((ufsstate_t *)(sio->fsstate))->flags.close_request = 0; + ((ufsstate_t *) (sio->fsstate))->fd = fd; + ((ufsstate_t *) (sio->fsstate))->flags.writing = 0; + ((ufsstate_t *) (sio->fsstate))->flags.reading = 0; + ((ufsstate_t *) (sio->fsstate))->flags.close_request = 0; store_open_disk_fd++; /* now insert into the replacement policy */ @@ -131,9 +131,9 @@ } void -storeUfsClose(SwapDir *SD, storeIOState * sio) +storeUfsClose(SwapDir * SD, storeIOState * sio) { - ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate; debug(79, 3) ("storeUfsClose: dirno %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, ufsstate->fd); @@ -145,9 +145,9 @@ } void -storeUfsRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +storeUfsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) { - ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate; assert(sio->read.callback == NULL); assert(sio->read.callback_data == NULL); @@ -167,9 +167,9 @@ } void -storeUfsWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +storeUfsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) { - ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate; debug(79, 3) ("storeUfsWrite: dirn %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, ufsstate->fd); ufsstate->flags.writing = 1; file_write(ufsstate->fd, @@ -182,7 +182,7 @@ } void -storeUfsUnlink(SwapDir *SD, StoreEntry *e) +storeUfsUnlink(SwapDir * SD, StoreEntry * e) { debug(79, 3) ("storeUfsUnlink: fileno %08X\n", e->swap_filen); storeUfsDirReplRemove(e); @@ -195,7 +195,7 @@ storeUfsReadDone(int fd, const char *buf, int len, int errflag, void *my_data) { storeIOState *sio = my_data; - ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate; STRCB *callback = sio->read.callback; void *their_data = sio->read.callback_data; ssize_t rlen; @@ -205,10 +205,10 @@ ufsstate->flags.reading = 0; if (errflag) { debug(79, 3) ("storeUfsReadDone: got failure (%d)\n", errflag); - rlen = -1; + rlen = -1; } else { - rlen = (ssize_t) len; - sio->offset += len; + rlen = (ssize_t) len; + sio->offset += len; } assert(callback); assert(their_data); @@ -223,7 +223,7 @@ storeUfsWriteDone(int fd, int errflag, size_t len, void *my_data) { storeIOState *sio = my_data; - ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate; debug(79, 3) ("storeUfsWriteDone: dirno %d, fileno %08X, FD %d, len %d\n", sio->swap_dirn, sio->swap_filen, fd, len); ufsstate->flags.writing = 0; @@ -240,7 +240,7 @@ static void storeUfsIOCallback(storeIOState * sio, int errflag) { - ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate; debug(79, 3) ("storeUfsIOCallback: errflag=%d\n", errflag); if (ufsstate->fd > -1) { file_close(ufsstate->fd); @@ -262,6 +262,6 @@ static void storeUfsIOFreeEntry(void *sio, int foo) { - memPoolFree(ufs_state_pool, ((storeIOState *)sio)->fsstate); + memPoolFree(ufs_state_pool, ((storeIOState *) sio)->fsstate); memFree(sio, MEM_STORE_IO); } Index: squid/src/fs/ufs/store_ufs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/ufs/store_ufs.h,v retrieving revision 1.1.6.3.2.1 retrieving revision 1.1.6.3.2.2 diff -u -r1.1.6.3.2.1 -r1.1.6.3.2.2 --- squid/src/fs/ufs/store_ufs.h 2 May 2000 22:50:30 -0000 1.1.6.3.2.1 +++ squid/src/fs/ufs/store_ufs.h 14 May 2000 23:22:20 -0000 1.1.6.3.2.2 @@ -28,13 +28,13 @@ typedef struct _ufsstate_t ufsstate_t; /* The ufs_state memory pool */ -extern MemPool * ufs_state_pool; +extern MemPool *ufs_state_pool; extern void storeUfsDirMapBitReset(SwapDir *, sfileno); extern int storeUfsDirMapBitAllocate(SwapDir *); -extern char * storeUfsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); +extern char *storeUfsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath); extern void storeUfsDirUnlinkFile(SwapDir *, sfileno); -extern void storeUfsDirReplAdd(SwapDir *SD, StoreEntry *); +extern void storeUfsDirReplAdd(SwapDir * SD, StoreEntry *); extern void storeUfsDirReplRemove(StoreEntry *); /* --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl/Makefile.in Wed Feb 14 00:44:13 2007 @@ -0,0 +1,35 @@ +# Makefile for storage modules in the Squid Object Cache server +# +# $Id$ +# + +SUBDIRS = @REPL_POLICIES@ + +all: + @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \ + sh -c "cd $$dir && $(MAKE) all" || exit 1; \ + done; \ + if [ ! -f stamp ]; then \ + touch stamp; \ + fi + +clean: + -rm -f *.a stamp + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) $@" || exit 1;\ + fi; \ + done + +distclean: + -rm -f *.a Makefile + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) distclean"; \ + fi; \ + done + +.DEFAULT: + @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \ + sh -c "cd $$dir && $(MAKE) $@" || exit 1; \ + done --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl/heap/Makefile.in Wed Feb 14 00:44:13 2007 @@ -0,0 +1,53 @@ +# +# Makefile for the UFS storage driver for the Squid Object Cache server +# +# $Id$ +# + +REPL = heap + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(REPL).a + +OBJS = \ + store_repl_$(REPL).o \ + store_heap_replacement.o + +all: $(OUT) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl/heap/store_heap_replacement.c Wed Feb 14 00:44:13 2007 @@ -0,0 +1,105 @@ + +/* + * $Id$ + * + * DEBUG: section 20 Storage Manager Heap-based replacement + * AUTHOR: John Dilley + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * the Regents of the University of California. Please see the + * COPYRIGHT file for full details. Squid incorporates software + * developed and/or copyrighted by other sources. Please see the + * CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +/* + * The code in this file is Copyrighted (C) 1999 by Hewlett Packard. + * + * + * For a description of these cache replacement policies see -- + * http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html + */ + +#include "squid.h" +#include "heap.h" + +/* + * Key generation function to implement the LFU-DA policy (Least + * Frequently Used with Dynamic Aging). Similar to classical LFU + * but with aging to handle turnover of the popular document set. + * Maximizes byte hit rate by keeping more currently popular objects + * in cache regardless of size. Achieves lower hit rate than GDS + * because there are more large objects in cache (so less room for + * smaller popular objects). + * + * This version implements a tie-breaker based upon recency + * (e->lastref): for objects that have the same reference count + * the most recent object wins (gets a higher key value). + */ +heap_key HeapKeyGen_StoreEntry_LFUDA(void *entry, double age) +{ + StoreEntry *e = entry; + double tie; + if (e->lastref <= 0) + tie = 0.0; + else if (squid_curtime <= e->lastref) + tie = 0.0; + else + tie = 1.0 - exp((double) (e->lastref - squid_curtime) / 86400.0); + return age + (double) e->refcount - tie; +} + + +/* + * Key generation function to implement the GDS-Frequency policy. + * Similar to Greedy Dual-Size Hits policy, but adds aging of + * documents to prevent pollution. Maximizes object hit rate by + * keeping more small, popular objects in cache. Achieves lower + * byte hit rate than LFUDA because there are fewer large objects + * in cache. + * + * This version implements a tie-breaker based upon recency + * (e->lastref): for objects that have the same reference count + * the most recent object wins (gets a higher key value). + */ +heap_key HeapKeyGen_StoreEntry_GDSF(void *entry, double age) +{ + StoreEntry *e = entry; + double size = e->swap_file_sz ? (double) e->swap_file_sz : 1.0; + double tie = (e->lastref > 1) ? (1.0 / e->lastref) : 1.0; + return age + ((double) e->refcount / size) - tie; +} + +/* + * Key generation function to implement the LRU policy. Normally + * one would not do this with a heap -- use the linked list instead. + * For testing and performance characterization it was useful. + * Don't use it unless you are trying to compare performance among + * heap-based replacement policies... + */ +heap_key HeapKeyGen_StoreEntry_LRU(void *entry, double age) +{ + StoreEntry *e = entry; + return (heap_key) e->lastref; +} --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl/heap/store_heap_replacement.h Wed Feb 14 00:44:13 2007 @@ -0,0 +1,3 @@ +extern heap_key HeapKeyGen_StoreEntry_LFUDA(void *entry, double age); +extern heap_key HeapKeyGen_StoreEntry_GDSF(void *entry, double age); +extern heap_key HeapKeyGen_StoreEntry_LRU(void *entry, double age); --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl/heap/store_repl_heap.c Wed Feb 14 00:44:13 2007 @@ -0,0 +1,301 @@ + +/* + * $Id$ + * + * DEBUG: section ? HEAP based removal policies + * AUTHOR: Henrik Nordstrom + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "heap.h" +#include "store_heap_replacement.h" + +REMOVALPOLICYCREATE createRemovalPolicy_heap; + +static int nr_heap_policies = 0; + +typedef struct _HeapPolicyData HeapPolicyData; +struct _HeapPolicyData { + RemovalPolicy *policy; + heap *heap; + heap_key_func *keyfunc; + int count; + int nwalkers; + enum heap_entry_type + { TYPE_UNKNOWN = 0, TYPE_STORE_ENTRY, TYPE_STORE_MEM } + type; +}; + +/* Hack to avoid having to remember the RemovalPolicyNode location. + * Needed by the purge walker. + */ +static enum heap_entry_type +heap_guessType(StoreEntry * entry, RemovalPolicyNode * node) +{ + if (node == &entry->repl) + return TYPE_STORE_ENTRY; + if (entry->mem_obj && node == &entry->mem_obj->repl) + return TYPE_STORE_MEM; + fatal("Heap Replacement: Unknown StoreEntry node type"); + return TYPE_UNKNOWN; +} +#define SET_POLICY_NODE(entry,value) \ + switch(heap->type) { \ + case TYPE_STORE_ENTRY: entry->repl.data = value; break ; \ + case TYPE_STORE_MEM: entry->mem_obj->repl.data = value ; break ; \ + default: break; \ + } + +static void +heap_add(RemovalPolicy * policy, StoreEntry * entry, RemovalPolicyNode * node) +{ + HeapPolicyData *heap = policy->_data; + assert(!node->data); + node->data = heap_insert(heap->heap, entry); + heap->count += 1; + if (!heap->type) + heap->type = heap_guessType(entry, node); +} + +static void +heap_remove(RemovalPolicy * policy, StoreEntry * entry, + RemovalPolicyNode * node) +{ + HeapPolicyData *heap = policy->_data; + heap_node *hnode = node->data; + if (!hnode) + return; + heap_delete(heap->heap, hnode); + node->data = NULL; + heap->count -= 1; +} + +static void +heap_referenced(RemovalPolicy * policy, const StoreEntry * entry, + RemovalPolicyNode * node) +{ + HeapPolicyData *heap = policy->_data; + heap_node *hnode = node->data; + if (!hnode) + return; + heap_update(heap->heap, hnode, (StoreEntry *) entry); +} + +/** RemovalPolicyWalker **/ + +typedef struct _HeapWalkData HeapWalkData; +struct _HeapWalkData +{ + int current; +}; + +const StoreEntry * +heap_walkNext(RemovalPolicyWalker * walker) +{ + HeapWalkData *heap_walk = walker->_data; + RemovalPolicy *policy = walker->_policy; + HeapPolicyData *heap = policy->_data; + StoreEntry *entry; + if (heap_walk->current >= heap_nodes(heap->heap)) + return NULL; /* done */ + entry = (StoreEntry *) heap_peep(heap->heap, heap_walk->current++); + return entry; +} + +static void +heap_walkDone(RemovalPolicyWalker * walker) +{ + RemovalPolicy *policy = walker->_policy; + HeapPolicyData *heap = policy->_data; + assert(strcmp(policy->_type, "heap") == 0); + assert(heap->nwalkers > 0); + heap->nwalkers -= 1; + safe_free(walker->_data); + cbdataFree(walker); +} + +static RemovalPolicyWalker * +heap_walkInit(RemovalPolicy * policy) +{ + HeapPolicyData *heap = policy->_data; + RemovalPolicyWalker *walker; + HeapWalkData *heap_walk; + heap->nwalkers += 1; + walker = xcalloc(1, sizeof(*walker)); + heap_walk = xcalloc(1, sizeof(*heap_walk)); + heap_walk->current = 0; + walker->_policy = policy; + walker->_data = heap_walk; + walker->Next = heap_walkNext; + walker->Done = heap_walkDone; + cbdataAdd(walker, cbdataXfree, 0); + return walker; +} + +/** RemovalPurgeWalker **/ + +typedef struct _HeapPurgeData HeapPurgeData; +struct _HeapPurgeData +{ + link_list *locked_entries; + heap_key min_age; +}; + +static StoreEntry * +heap_purgeNext(RemovalPurgeWalker * walker) +{ + HeapPurgeData *heap_walker = walker->_data; + RemovalPolicy *policy = walker->_policy; + HeapPolicyData *heap = policy->_data; + StoreEntry *entry; + heap_key age; + try_again: + if (!heap_nodes(heap->heap) > 0) + return NULL; /* done */ + age = heap_peepminkey(heap->heap); + entry = heap_extractmin(heap->heap); + if (storeEntryLocked(entry)) { + linklistPush(&heap_walker->locked_entries, entry); + goto try_again; + } + SET_POLICY_NODE(entry, NULL); + return entry; +} + +static void +heap_purgeDone(RemovalPurgeWalker * walker) +{ + HeapPurgeData *heap_walker = walker->_data; + RemovalPolicy *policy = walker->_policy; + HeapPolicyData *heap = policy->_data; + StoreEntry *entry; + assert(strcmp(policy->_type, "heap") == 0); + assert(heap->nwalkers > 0); + heap->nwalkers -= 1; + if (heap_walker->min_age > 0) + heap->heap->age = heap_walker->min_age; + /* + * Reinsert the locked entries + */ + while ((entry = linklistShift(&heap_walker->locked_entries))) { + heap_node *node = heap_insert(heap->heap, entry); + SET_POLICY_NODE(entry, node); + } + safe_free(walker->_data); + cbdataFree(walker); +} + +static RemovalPurgeWalker * +heap_purgeInit(RemovalPolicy * policy, int max_scan) +{ + HeapPolicyData *heap = policy->_data; + RemovalPurgeWalker *walker; + HeapPurgeData *heap_walk; + heap->nwalkers += 1; + walker = xcalloc(1, sizeof(*walker)); + heap_walk = xcalloc(1, sizeof(*heap_walk)); + heap_walk->min_age = 0.0; + heap_walk->locked_entries = NULL; + walker->_policy = policy; + walker->_data = heap_walk; + walker->max_scan = max_scan; + walker->Next = heap_purgeNext; + walker->Done = heap_purgeDone; + cbdataAdd(walker, cbdataXfree, 0); +#if HEAP_REPLACEMENT_DEBUG + if (!verify_heap_property(heap->heap)) { + debug(20, 1) ("Heap property violated!\n"); + } +#endif + return walker; +} + +static void +heap_free(RemovalPolicy * policy) +{ + HeapPolicyData *heap = policy->_data; + /* Make some verification of the policy state */ + assert(strcmp(policy->_type, "heap") == 0); + assert(heap->nwalkers); + assert(heap->count); + /* Ok, time to destroy this policy */ + safe_free(policy->_data); + memset(policy, 0, sizeof(*policy)); + cbdataFree(policy); +} + +RemovalPolicy * +createRemovalPolicy_heap(wordlist * args) +{ + RemovalPolicy *policy; + HeapPolicyData *heap_data; + char *keytype; + /* Allocate the needed structures */ + policy = xcalloc(1, sizeof(*policy)); + heap_data = xcalloc(1, sizeof(*heap_data)); + /* cbdata register the policy */ + cbdataAdd(policy, cbdataXfree, 0); + /* Initialize the policy data */ + heap_data->policy = policy; + if (args) { + keytype = args->key; + args = args->next; + } else { + debug(20, 1) ("createRemovalPolicy_heap: No key type specified. Using LRU\n"); + keytype = "LRU"; + } + if (!strcmp(keytype, "GDSF")) + heap_data->keyfunc = HeapKeyGen_StoreEntry_GDSF; + else if (!strcmp(keytype, "LFUDA")) + heap_data->keyfunc = HeapKeyGen_StoreEntry_LFUDA; + else if (!strcmp(keytype, "LRU")) + heap_data->keyfunc = HeapKeyGen_StoreEntry_LRU; + else { + debug(20, 0) ("createRemovalPolicy_heap: Unknown key type \"%s\". Using LRU\n", + keytype); + heap_data->keyfunc = HeapKeyGen_StoreEntry_LRU; + } + /* No additional arguments expected */ + assert(!args); + heap_data->heap = new_heap(1000, heap_data->keyfunc); + /* Populate the policy structure */ + policy->_type = "heap"; + policy->_data = heap_data; + policy->Free = heap_free; + policy->Add = heap_add; + policy->Remove = heap_remove; + policy->Referenced = NULL; + policy->Dereferenced = heap_referenced; + policy->WalkInit = heap_walkInit; + policy->PurgeInit = heap_purgeInit; + /* Increase policy usage count */ + nr_heap_policies += 0; + return policy; +} --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl/lru/Makefile.in Wed Feb 14 00:44:13 2007 @@ -0,0 +1,52 @@ +# +# Makefile for the UFS storage driver for the Squid Object Cache server +# +# $Id$ +# + +REPL = lru + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(REPL).a + +OBJS = \ + store_repl_$(REPL).o + +all: $(OUT) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c --- /dev/null Wed Feb 14 00:44:00 2007 +++ squid/src/repl/lru/store_repl_lru.c Wed Feb 14 00:44:13 2007 @@ -0,0 +1,358 @@ + +/* + * $Id$ + * + * DEBUG: section ? LRU Removal policy + * AUTHOR: Henrik Nordstrom + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +REMOVALPOLICYCREATE createRemovalPolicy_lru; + +typedef struct _LruPolicyData LruPolicyData; +struct _LruPolicyData { + RemovalPolicy *policy; + dlink_list list; + int count; + int nwalkers; + enum heap_entry_type + { TYPE_UNKNOWN = 0, TYPE_STORE_ENTRY, TYPE_STORE_MEM } + type; +}; + +/* Hack to avoid having to remember the RemovalPolicyNode location. + * Needed by the purge walker + */ +static enum heap_entry_type +repl_guessType(StoreEntry * entry, RemovalPolicyNode * node) +{ + if (node == &entry->repl) + return TYPE_STORE_ENTRY; + if (entry->mem_obj && node == &entry->mem_obj->repl) + return TYPE_STORE_MEM; + fatal("Heap Replacement: Unknown StoreEntry node type"); + return TYPE_UNKNOWN; +} +#define SET_POLICY_NODE(entry,value) \ + switch(lru->type) { \ + case TYPE_STORE_ENTRY: entry->repl.data = value; break ; \ + case TYPE_STORE_MEM: entry->mem_obj->repl.data = value ; break ; \ + default: break; \ + } + +typedef struct _LruNode LruNode; +struct _LruNode +{ + /* Note: the dlink_node MUST be the first member of the LruNode + * structure. This member is later pointer typecasted to LruNode *. + */ + dlink_node node; +}; + +static MemPool *lru_node_pool = NULL; +static int nr_lru_policies = 0; + +static void +lru_add(RemovalPolicy * policy, StoreEntry * entry, RemovalPolicyNode * node) +{ + LruPolicyData *lru = policy->_data; + LruNode *lru_node; + assert(!node->data); + node->data = lru_node = memPoolAlloc(lru_node_pool); + dlinkAddTail(entry, &lru_node->node, &lru->list); + lru->count += 1; + if (!lru->type) + lru->type = repl_guessType(entry, node); +} + +static void +lru_remove(RemovalPolicy * policy, StoreEntry * entry, RemovalPolicyNode * node) +{ + LruPolicyData *lru = policy->_data; + LruNode *lru_node = node->data; + if (!lru_node) + return; + assert(lru_node->node.data == entry); + node->data = NULL; + dlinkDelete(&lru_node->node, &lru->list); + memPoolFree(lru_node_pool, lru_node); + lru->count -= 1; +} + +static void +lru_referenced(RemovalPolicy * policy, const StoreEntry * entry, + RemovalPolicyNode * node) +{ + LruPolicyData *lru = policy->_data; + LruNode *lru_node = node->data; + if (!lru_node) + return; + dlinkDelete(&lru_node->node, &lru->list); + dlinkAdd((void *) entry, &lru_node->node, &lru->list); +} + +/** RemovalPolicyWalker **/ + +typedef struct _LruWalkData LruWalkData; +struct _LruWalkData +{ + LruNode *current; +}; + +const StoreEntry * +lru_walkNext(RemovalPolicyWalker * walker) +{ + LruWalkData *lru_walk = walker->_data; + LruNode *lru_node = lru_walk->current; + if (!lru_node) + return NULL; + lru_walk->current = (LruNode *) lru_node->node.next; + return (StoreEntry *) lru_node->node.data; +} + +static void +lru_walkDone(RemovalPolicyWalker * walker) +{ + RemovalPolicy *policy = walker->_policy; + LruPolicyData *lru = policy->_data; + assert(strcmp(policy->_type, "lru") == 0); + assert(lru->nwalkers > 0); + lru->nwalkers -= 1; + safe_free(walker->_data); + cbdataFree(walker); +} + +static RemovalPolicyWalker * +lru_walkInit(RemovalPolicy * policy) +{ + LruPolicyData *lru = policy->_data; + RemovalPolicyWalker *walker; + LruWalkData *lru_walk; + lru->nwalkers += 1; + walker = xcalloc(1, sizeof(*walker)); + lru_walk = xcalloc(1, sizeof(*lru_walk)); + walker->_policy = policy; + walker->_data = lru_walk; + walker->Next = lru_walkNext; + walker->Done = lru_walkDone; + lru_walk->current = (LruNode *) lru->list.head; + cbdataAdd(walker, cbdataXfree, 0); + return walker; +} + +/** RemovalPurgeWalker **/ + +typedef struct _LruPurgeData LruPurgeData; +struct _LruPurgeData +{ + LruNode *current; + LruNode *start; +}; + +static StoreEntry * +lru_purgeNext(RemovalPurgeWalker * walker) +{ + LruPurgeData *lru_walker = walker->_data; + RemovalPolicy *policy = walker->_policy; + LruPolicyData *lru = policy->_data; + LruNode *lru_node; + StoreEntry *entry; + try_again: + lru_node = lru_walker->current; + if (!lru_node || walker->scanned >= walker->max_scan) + return NULL; + walker->scanned += 1; + lru_walker->current = (LruNode *) lru_node->node.next; + if (lru_walker->current == lru_walker->start) { + /* Last node found */ + lru_walker->current = NULL; + } + entry = (StoreEntry *) lru_node->node.data; + dlinkDelete(&lru_node->node, &lru->list); + if (storeEntryLocked(entry)) { + /* Shit, it is locked. we can't return this one */ + walker->locked++; + dlinkAdd(entry, &lru_node->node, &lru->list); + goto try_again; + } + memPoolFree(lru_node_pool, lru_node); + lru->count -= 1; + SET_POLICY_NODE(entry, NULL); + return entry; +} + +static void +lru_purgeDone(RemovalPurgeWalker * walker) +{ + RemovalPolicy *policy = walker->_policy; + LruPolicyData *lru = policy->_data; + assert(strcmp(policy->_type, "lru") == 0); + assert(lru->nwalkers > 0); + lru->nwalkers -= 1; + safe_free(walker->_data); + cbdataFree(walker); +} + +static RemovalPurgeWalker * +lru_purgeInit(RemovalPolicy * policy, int max_scan) +{ + LruPolicyData *lru = policy->_data; + RemovalPurgeWalker *walker; + LruPurgeData *lru_walk; + lru->nwalkers += 1; + walker = xcalloc(1, sizeof(*walker)); + lru_walk = xcalloc(1, sizeof(*lru_walk)); + walker->_policy = policy; + walker->_data = lru_walk; + walker->max_scan = max_scan; + walker->Next = lru_purgeNext; + walker->Done = lru_purgeDone; + lru_walk->start = lru_walk->current = (LruNode *) lru->list.head; + cbdataAdd(walker, cbdataXfree, 0); + return walker; +} + +static void +lru_free(RemovalPolicy * policy) +{ + LruPolicyData *lru = policy->_data; + /* Make some verification of the policy state */ + assert(strcmp(policy->_type, "lru") == 0); + assert(lru->nwalkers); + assert(lru->count); + /* Ok, time to destroy this policy */ + safe_free(policy->_data); + memset(policy, 0, sizeof(*policy)); + cbdataFree(policy); +} + +RemovalPolicy * +createRemovalPolicy_lru(wordlist *args) +{ + RemovalPolicy *policy; + LruPolicyData *lru_data; + /* no arguments expected or understood */ + assert(!args); + /* Initialize */ + if (!lru_node_pool) + lru_node_pool = memPoolCreate("LRU policy node", sizeof(LruNode)); + /* Allocate the needed structures */ + policy = xcalloc(1, sizeof(*policy)); + lru_data = xcalloc(1, sizeof(*lru_data)); + /* cbdata register the policy */ + cbdataAdd(policy, cbdataXfree, 0); + /* Initialize the URL data */ + lru_data->policy = policy; + /* Populate the policy structure */ + policy->_type = "lru"; + policy->_data = lru_data; + policy->Free = lru_free; + policy->Add = lru_add; + policy->Remove = lru_remove; + policy->Referenced = lru_referenced; + policy->Dereferenced = lru_referenced; + policy->WalkInit = lru_walkInit; + policy->PurgeInit = lru_purgeInit; + /* Increase policy usage count */ + nr_lru_policies += 0; + return policy; +} + + +#if OLD_UNUSED_CODE +/* + * storeUfsDirCheckExpired + * + * Check whether the given object is expired or not + * It breaks layering a little by calling the upper layers to find + * out whether the object is locked or not, but we can't help this + * right now. + */ +static int +storeUfsDirCheckExpired(SwapDir * SD, StoreEntry * e) +{ + if (storeEntryLocked(e)) + return 0; + if (EBIT_TEST(e->flags, RELEASE_REQUEST)) + return 1; + if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) + return 1; + +#if HEAP_REPLACEMENT + /* + * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap + * controls the replacement of objects. + */ + return 1; +#else + if (squid_curtime - e->lastref > storeUfsDirExpiredReferenceAge(SD)) + return 1; + return 0; +#endif +} + +/* + * storeUfsDirExpiredReferenceAge + * + * The LRU age is scaled exponentially between 1 minute and + * Config.referenceAge , when store_swap_low < store_swap_size < + * store_swap_high. This keeps store_swap_size within the low and high + * water marks. If the cache is very busy then store_swap_size stays + * closer to the low water mark, if it is not busy, then it will stay + * near the high water mark. The LRU age value can be examined on the + * cachemgr 'info' page. + */ +static time_t +storeUfsDirExpiredReferenceAge(SwapDir * SD) +{ + double x; + double z; + time_t age; + long store_high, store_low; + + store_high = (long) (((float) SD->max_size * + (float) Config.Swap.highWaterMark) / (float) 100); + store_low = (long) (((float) SD->max_size * + (float) Config.Swap.lowWaterMark) / (float) 100); + debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, + store_low, SD->cur_size); + + x = (double) (store_high - SD->cur_size) / (store_high - store_low); + x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; + z = pow((double) (Config.referenceAge / 60), x); + age = (time_t) (z * 60.0); + if (age < 60) + age = 60; + else if (age > Config.referenceAge) + age = Config.referenceAge; + return age; +} +#endif /* OLD_UNUSED_CODE */