--------------------- PatchSet 260 Date: 2000/05/03 19:18:10 Author: hno Branch: ntlm Tag: (none) Log: Syncronized with devel Members: CONTRIBUTORS:1.1.1.3->1.1.1.3.10.1 ChangeLog:1.1.1.3.10.2->1.1.1.3.10.3 acconfig.h:1.1.1.3.10.2->1.1.1.3.10.3 configure.in:1.1.1.3.10.3->1.1.1.3.10.4 makefile.in:1.1.1.3.10.1->1.1.1.3.10.2 auth_modules/Makefile.in:1.1->1.1.10.1 auth_modules/dummy.in:1.1.1.1->1.1.1.1.14.1(DEAD) auth_modules/MSNT/Makefile:1.1.1.1->1.1.1.1.12.1(DEAD) auth_modules/MSNT/Makefile.in:1.1->1.1.8.1 auth_modules/SMB/Makefile.in:1.1.1.1->1.1.1.1.12.1 doc/Programming-Guide/Makefile:1.1.1.1->1.1.1.1.16.1 doc/Programming-Guide/prog-guide.sgml:1.1.1.2.10.1->1.1.1.2.10.2 icons/Makefile.in:1.1.1.3->1.1.1.3.12.1 src/HttpReply.c:1.1.1.3.10.1->1.1.1.3.10.2 src/Makefile.in:1.1.1.3.12.2->1.1.1.3.12.3 src/access_log.c:1.1.1.3.12.1->1.1.1.3.12.2 src/acl.c:1.1.1.3.12.4->1.1.1.3.12.5 src/aiops.c:1.1.1.3.12.1->1.1.1.3.12.2(DEAD) src/async_io.c:1.1.1.3.4.1.2.1->1.1.1.3.4.1.2.2(DEAD) src/cache_cf.c:1.1.1.3.4.1.2.1->1.1.1.3.4.1.2.2 src/cf.data.pre:1.1.1.3.4.1.2.5->1.1.1.3.4.1.2.6 src/cf_gen.c:1.1.1.3.12.1->1.1.1.3.12.2 src/cf_gen_defines:1.1->1.1.10.1 src/client_side.c:1.1.1.3.4.1.2.9->1.1.1.3.4.1.2.10 src/comm_select.c:1.1.1.3.12.1->1.1.1.3.12.2 src/defines.h:1.1.1.3.12.2->1.1.1.3.12.3 src/delay_pools.c:1.1.1.3.12.1->1.1.1.3.12.2 src/disk.c:1.1.1.3.10.1->1.1.1.3.10.2 src/diskd.c:1.1.8.1->1.1.8.2(DEAD) src/dns_internal.c:1.1.1.1.4.1.2.1->1.1.1.1.4.1.2.2 src/enums.h:1.1.1.3.12.2->1.1.1.3.12.3 src/event.c:1.1.1.3.10.1->1.1.1.3.10.2 src/fd.c:1.1.1.3.12.1->1.1.1.3.12.2 src/filemap.c:1.1.1.2.10.1->1.1.1.2.10.2 src/forward.c:1.1.1.3.12.2->1.1.1.3.12.3 src/ftp.c:1.1.1.3.4.1.2.2->1.1.1.3.4.1.2.3 src/globals.h:1.1.1.3.12.1->1.1.1.3.12.2 src/helper.c:1.1.1.3->1.1.1.3.12.1 src/http.c:1.1.1.3.4.1.2.2->1.1.1.3.4.1.2.3 src/ipc.c:1.1.1.3.12.1->1.1.1.3.12.2 src/main.c:1.1.1.3.4.1.2.1->1.1.1.3.4.1.2.2 src/mem.c:1.1.1.3.12.2->1.1.1.3.12.3 src/neighbors.c:1.1.1.3.12.1->1.1.1.3.12.2 src/net_db.c:1.1.1.3.12.1->1.1.1.3.12.2 src/peer_select.c:1.1.1.3.12.1->1.1.1.3.12.2 src/protos.h:1.1.1.3.12.4->1.1.1.3.12.5 src/snmp_agent.c:1.1.1.3.10.1->1.1.1.3.10.2 src/squid.h:1.1.1.3.12.1->1.1.1.3.12.2 src/ssl.c:1.1.1.3.12.1->1.1.1.3.12.2 src/stat.c:1.1.1.3.12.2->1.1.1.3.12.3 src/store.c:1.1.1.3.12.1->1.1.1.3.12.2 src/store_client.c:1.1.1.3.4.1.2.1->1.1.1.3.4.1.2.2 src/store_dir.c:1.1.1.3.12.1->1.1.1.3.12.2 src/store_dir_ufs.c:1.1.1.1.12.1->1.1.1.1.12.2(DEAD) src/store_heap_replacement.c:1.1.1.1.10.1->1.1.1.1.10.2 src/store_io.c:1.1.1.1->1.1.1.1.12.1 src/store_io_asyncufs.c:1.1.1.1.4.1.2.1->1.1.1.1.4.1.2.2(DEAD) src/store_io_ufs.c:1.1.1.1.12.1->1.1.1.1.12.2(DEAD) src/store_log.c:1.1.1.2.12.1->1.1.1.2.12.2 src/store_modules.sh:1.1->1.1.10.1 src/store_rebuild.c:1.1.1.3.12.1->1.1.1.3.12.2 src/store_swapin.c:1.1.1.3.12.1->1.1.1.3.12.2 src/store_swapout.c:1.1.1.3.4.1.2.1->1.1.1.3.4.1.2.2 src/structs.h:1.1.1.3.4.1.2.5->1.1.1.3.4.1.2.6 src/tools.c:1.1.1.3.10.1->1.1.1.3.10.2 src/typedefs.h:1.1.1.3.12.3->1.1.1.3.12.4 src/unlinkd.c:1.1.1.3.12.1->1.1.1.3.12.2 src/url.c:1.1.1.3.12.1->1.1.1.3.12.2 src/fs/Makefile.in:1.1->1.1.10.1 src/fs/aufs/Makefile.in:1.1->1.1.10.1 src/fs/aufs/aiops.c:1.1->1.1.10.1 src/fs/aufs/async_io.c:1.1->1.1.10.1 src/fs/aufs/store_asyncufs.h:1.1->1.1.10.1 src/fs/aufs/store_dir_aufs.c:1.1->1.1.10.1 src/fs/aufs/store_io_aufs.c:1.1->1.1.10.1 src/fs/coss/Makefile.in:1.1->1.1.8.1 src/fs/coss/store_coss.h:1.1->1.1.8.1 src/fs/coss/store_dir_coss.c:1.1->1.1.8.1 src/fs/coss/store_io_coss.c:1.1->1.1.8.1 src/fs/diskd/Makefile.in:1.1->1.1.10.1 src/fs/diskd/diskd.c:1.1->1.1.10.1 src/fs/diskd/store_dir_diskd.c:1.1->1.1.10.1 src/fs/diskd/store_diskd.h:1.1->1.1.10.1 src/fs/diskd/store_io_diskd.c:1.1->1.1.10.1 src/fs/ufs/Makefile.in:1.1->1.1.10.1 src/fs/ufs/store_dir_ufs.c:1.1->1.1.10.1 src/fs/ufs/store_io_ufs.c:1.1->1.1.10.1 src/fs/ufs/store_ufs.h:1.1->1.1.10.1 Index: squid/CONTRIBUTORS =================================================================== RCS file: /cvsroot/squid-sf//squid/CONTRIBUTORS,v retrieving revision 1.1.1.3 retrieving revision 1.1.1.3.10.1 diff -u -r1.1.1.3 -r1.1.1.3.10.1 --- squid/CONTRIBUTORS 26 Jan 2000 03:25:00 -0000 1.1.1.3 +++ squid/CONTRIBUTORS 3 May 2000 19:18:10 -0000 1.1.1.3.10.1 @@ -70,6 +70,7 @@ Niall Doherty Pedro Ribeiro John Dilley + Adrian Chadd Development of this caching software is funded by the National Science Foundation (grants NCR-9616602 and NCR-9521745). Paid staff members on Index: squid/ChangeLog =================================================================== RCS file: /cvsroot/squid-sf//squid/ChangeLog,v retrieving revision 1.1.1.3.10.2 retrieving revision 1.1.1.3.10.3 diff -u -r1.1.1.3.10.2 -r1.1.1.3.10.3 --- squid/ChangeLog 17 Apr 2000 00:13:07 -0000 1.1.1.3.10.2 +++ squid/ChangeLog 3 May 2000 19:18:10 -0000 1.1.1.3.10.3 @@ -1,12 +1,69 @@ - - Reworked how request bodies are passed down to the protocols. - Now all client side processing is inside client_side.c, and - the pass and pump modules is no longer used. Changes to Squid-2.4.DEVEL3 (): - Added Logfile module. - Added DISKD stats via cachemgr. - Added squid.conf options for DISKD magic constants. + - Added --enable-auth-modules=... configure option + - Improved ICP dead peer detection to also work when the workload + is low + - Improved TCP dead peer detection and recovery + - Squid is now a bit more persistent in trying to find a alive + parent when never_direct is used. + - nonhierarchical_direct squid.conf directive to make non-ICP + peer selection behave a bit more like ICP selection with respect + to hierarchy. + - Bugfix where netdb selection could override never_direct + - ICP timeout selection now prefers to use parents only when + calculating the dynamic timeout to compensate for common RTT + differences between parents and siblings. + - No longer starts to swap out objects which are known to be above + the maximum allowed size. + - allow-miss cache_peer option disabling the use of "only-if-cached". + Meant to be used in conjunction with icp_hit_stale. + - Delay pools tuned to allow large initial pool values + - cachemgr filesystem space information changed to show useable space + rather than raw space, and platform support somewhat extended. + - Logs destination IP in the hierarchy log tag when going direct. + (can be disabled by turning log_ip_on_direct off) + - Async-IO on linux now makes proper use of mutexes. This fixes some + odd pthread segfaults on SMP Linux machines, at a slight performance + penalty. + - %s can now be used in cache_swap_log and will be substituded with + the last path component of cache_dir. + - no_cache is now a full ACL check without, allowing most ACL types + to be used. + - The CONNECT method now obeys miss_access requirements + - proxy_auth_regex and ident_regex ACL types + - Fixed a StoreEntry memory leak during "dirty" rebuild + - Helper processes no longer hold unrelated filedescriptors open + - Helpers are now restarted when the logs are rotated + - Negatively cached DNS entries are now purged on "reload". + - PURGE now also purges the DNS cache + - HEAD on FTP objects no longer retreives the whole object + - More cleanups of the dstdomain ACL type + - Squid no longer tries to do Range internally if it is not supported + by the origin server. Doing so could cause bandwidth spikes and/or + negative hit ratio. + - httpd_accel_single_host squid.conf directive + - "round-robin" cache_peer counters are reset every 5 minutes to + compensate previously dead peers + - DNS retransmit parameters + - Show all FTP server messages + - squid.conf.default now indicates if a directive isn't enabled in + the installed binary, and what configure option to use for enabling it + - Fixed a temporary memory leak on persistent POSTs + - Fixed a temporary memory leak when the server response headers + includes NULL characters + - authenticate_ip_ttl_is_strict squid.conf option + - req_mime_type ACL type + - A reworked storage system that supports storage directories in + a more modular fashion. The object replacement and IO is now + responsibility of the storage directory, and not of the storage + manager. + - Reworked how request bodies are passed down to the protocols. + Now all client side processing is inside client_side.c, and + the pass and pump modules is no longer used. Changes to Squid-2.4.DEVEL2 (): Index: squid/acconfig.h =================================================================== RCS file: /cvsroot/squid-sf//squid/Attic/acconfig.h,v retrieving revision 1.1.1.3.10.2 retrieving revision 1.1.1.3.10.3 diff -u -r1.1.1.3.10.2 -r1.1.1.3.10.3 --- squid/acconfig.h 17 Apr 2000 00:13:07 -0000 1.1.1.3.10.2 +++ squid/acconfig.h 3 May 2000 19:18:10 -0000 1.1.1.3.10.3 @@ -20,7 +20,7 @@ * */ @ TOP @ -/* $Id: acconfig.h,v 1.1.1.3.10.2 2000/04/17 00:13:07 hno Exp $ */ +/* $Id: acconfig.h,v 1.1.1.3.10.3 2000/05/03 19:18:10 hno Exp $ */ /********************************* * START OF CONFIGURABLE OPTIONS * @@ -49,7 +49,7 @@ #undef USE_ASYNC_IO /* Defines how many threads to use for async I/O */ -#undef NUMTHREADS +#undef ASYNC_IO_THREADS /* * If you want to use Squid's ICMP features (highly recommended!) then @@ -276,3 +276,6 @@ * message type for message queues */ #undef mtyp_t + +/* Define if you want to set the COSS membuf size */ +#undef COSS_MEMBUF_SZ Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.1.1.3.10.3 retrieving revision 1.1.1.3.10.4 diff -u -r1.1.1.3.10.3 -r1.1.1.3.10.4 --- squid/configure.in 17 Apr 2000 00:13:07 -0000 1.1.1.3.10.3 +++ squid/configure.in 3 May 2000 19:18:10 -0000 1.1.1.3.10.4 @@ -3,13 +3,13 @@ dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.1.1.3.10.3 2000/04/17 00:13:07 hno Exp $ +dnl $Id: configure.in,v 1.1.1.3.10.4 2000/05/03 19:18:10 hno Exp $ dnl dnl dnl AC_INIT(src/main.c) AC_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.1.1.3.10.3 $)dnl +AC_REVISION($Revision: 1.1.1.3.10.4 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AC_CONFIG_AUX_DIR(cfgaux) @@ -236,46 +236,90 @@ fi ]) -AC_ARG_ENABLE(async_io, +AC_ARG_ENABLE(async-io, [ --enable-async-io[=N_THREADS] - Do ASYNC disk I/O using threads. - N_THREADS is the number of worker threads - defaults to 16. See also src/squid.h for - some additional platform tuning], -[ case "$enableval" in + Shorthand for + --with-aio-threads=N_THREADS + --with-pthreads + --enable-storeio=ufs,aufs], +[ case $enableval in yes) - async_io=yes + with_pthreads=yes + STORE_MODULES="ufs aufs" ;; no) - async_io='' ;; *) - async_io=yes - AC_DEFINE_UNQUOTED(NUMTHREADS, $enableval) + async_io_threads=$enableval + with_pthreads=yes + STORE_MODULES="ufs aufs" ;; esac ]) -if test -n "$async_io" ; then - echo "Async I/O enabled" - async_io=yes - AC_DEFINE(USE_ASYNC_IO) - ASYNC_OBJS='$(ASYNC_OBJS)' +AC_ARG_WITH(aio-threads, +[ --with-aio-threads=N_THREADS + Tune the number of worker threads for the aufs object + store.], +[ async_io_threads=$withval ]) +if test "$async_io_threads"; then + echo "With $async_io_threads AIO threads" + with_pthreads=yes + AC_DEFINE_UNQUOTED(ASYNC_IO_THREADS,$async_io_threads) +fi + +AC_ARG_WITH(pthreads, +[ --with-pthreads Use POSIX Threads], +[ if test "$enableval" = "yes"; then + with_pthreads=yes + fi +]) +if test "$with_pthreads"; then + echo "With pthreads" SQUID_PTHREAD_LIB='$(PTHREADLIB)' CFLAGS="$CFLAGS -D_REENTRANT" case "$host" in i386-unknown-freebsd*) - if test "$GCC" = "yes" ; then - if test -z "$PRESET_LDFLAGS"; then - LDFLAGS="$LDFLAGS -pthread" - fi - fi - ;; + if test "$GCC" = "yes" ; then + if test -z "$PRESET_LDFLAGS"; then + LDFLAGS="$LDFLAGS -pthread" + fi + fi + ;; esac fi -AC_SUBST(ASYNC_OBJS) AC_SUBST(SQUID_PTHREAD_LIB) +AC_ARG_ENABLE(storeio, +[ --enable-storeio=\"list of modules\" + Build support for the list of store I/O modules. + The default is only to build the "ufs" module. + See src/fs for a list of available modules, or + Programmers Guide section + for details on how to build your custom store module], +[ case $enableval in + yes) + for module in $srcdir/src/fs/*; do + if test -f $module/Makefile.in; then + STORE_MODULES="$STORE_MODULES `basename $module`" + fi + done + ;; + no) + ;; + *) STORE_MODULES="`echo $enableval| sed -e 's/,/ /g;s/ */ /g'`" + ;; + esac +], +[ if test -z "$STORE_MODULES"; then + STORE_MODULES="ufs" + fi +]) +echo "Store moules built: $STORE_MODULES" +AC_SUBST(STORE_MODULES) +STORE_OBJS="fs/`echo $STORE_MODULES|sed -e's% %.a fs/%g'`.a" +AC_SUBST(STORE_OBJS) + AC_ARG_ENABLE(icmp, [ --enable-icmp Enable ICMP pinging], [ if test "$enableval" = "yes" ; then @@ -428,6 +472,15 @@ ],[ERR_LANGUAGE="English"]) AC_SUBST(ERR_LANGUAGE) +dnl Size of COSS memory buffer +AC_ARG_WITH(coss-membuf-size, +[ --with-coss-membuf-size COSS membuf size (default 1048576 bytes) ], +[ if test "$with_coss_membuf_size"; then + echo "Setting COSS membuf size to $with_coss_membuf_size bytes" + AC_DEFINE_UNQUOTED(COSS_MEMBUF_SZ, $with_coss_membuf_size) + fi +]) + dnl Enable poll() AC_ARG_ENABLE(poll, [ --enable-poll Enable poll() instead of select(). Normally poll @@ -556,27 +609,39 @@ [ --enable-heap-replacement This option allows you to use various cache replacement algorithms, instead of the standard - LRU algorithm. - ], + LRU algorithm.], [ if test "$enableval" = "yes" ; then echo "Enabling HEAP_REPLACEMENT" AC_DEFINE(HEAP_REPLACEMENT, 1) fi ]) -dnl Enable "diskd" code -AC_ARG_ENABLE(diskd, -[ --enable-diskd Uses shared memory, message queues, and external - processes for disk I/O.], -[ if test "$enableval" = "yes" ; then - echo "DISKD enabled" - AC_DEFINE(USE_DISKD, 1) - OPT_DISKD_EXE='$(DISKD_EXE)' - AC_SUBST(OPT_DISKD_EXE) - DISKD_OBJS='diskd.o' - AC_SUBST(DISKD_OBJS) - fi +dnl Select auth modules to build +AUTH_MODULES= +AC_ARG_ENABLE(auth-modules, +[ --enable-auth-modules=\"list of modules\" + This option selects wich proxy_auth helper modules + to build and install as part of the normal build + process. For a list of available modules see + the auth_modules directory.], +[ case "$enableval" in + yes) + for module in $srcdir/auth_modules/*; do + if test -f $module/Makefile.in; then + AUTH_MODULES="$AUTH_MODULES `basename $module`" + fi + done + ;; + no) + ;; + *) + AUTH_MODULES="`echo $enableval| sed -e 's/,/ /g;s/ */ /g'`" + esac ]) +if test -n "$AUTH_MODULES"; then + echo "Auth moules built: $AUTH_MODULES" +fi +AC_SUBST(AUTH_MODULES) # Force some compilers to use ANSI features # @@ -677,6 +742,7 @@ strings.h \ sys/file.h \ sys/ioctl.h \ + sys/mount.h \ sys/msg.h \ sys/param.h \ sys/resource.h \ @@ -684,6 +750,7 @@ sys/socket.h \ sys/stat.h \ sys/statvfs.h \ + sys/vfs.h \ sys/syscall.h \ sys/time.h \ sys/types.h \ @@ -1003,6 +1070,9 @@ mktime \ mstats \ poll \ + pthread_attr_setscope \ + pthread_setschedparam \ + pthread_attr_setschedparam \ putenv \ random \ regcomp \ @@ -1020,20 +1090,13 @@ snprintf \ srand48 \ srandom \ + statfs \ sysconf \ syslog \ timegm \ vsnprintf \ ) -if test "$async_io" = "yes" ; then - AC_CHECK_FUNCS(\ - pthread_attr_setscope \ - pthread_setschedparam \ - pthread_attr_setschedparam \ - ) -fi - dnl Yay! Another Linux brokenness. Its not good enough dnl to know that setresuid() exists, because RedHat 5.0 declares dnl setresuid() but doesn't implement it. @@ -1486,14 +1549,20 @@ ./scripts/RunCache \ ./scripts/RunAccel \ ./src/Makefile \ + ./src/fs/Makefile \ + ./src/fs/ufs/Makefile \ + ./src/fs/aufs/Makefile \ + ./src/fs/coss/Makefile \ + ./src/fs/diskd/Makefile \ ./contrib/Makefile \ $SNMP_MAKEFILE \ ./icons/Makefile \ ./errors/Makefile \ - ./auth_modules/dummy \ + ./auth_modules/Makefile \ ./auth_modules/NCSA/Makefile \ ./auth_modules/PAM/Makefile \ - ./auth_modules/SMB/Makefile + ./auth_modules/SMB/Makefile \ + ./auth_modules/MSNT/Makefile \ ./auth_modules/getpwnam/Makefile \ ./auth_modules/LDAP/Makefile \ ) Index: squid/makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/Attic/makefile.in,v retrieving revision 1.1.1.3.10.1 retrieving revision 1.1.1.3.10.2 diff -u -r1.1.1.3.10.1 -r1.1.1.3.10.2 --- squid/makefile.in 17 Apr 2000 00:13:07 -0000 1.1.1.3.10.1 +++ squid/makefile.in 3 May 2000 19:18:10 -0000 1.1.1.3.10.2 @@ -1,4 +1,4 @@ -# $Id: makefile.in,v 1.1.1.3.10.1 2000/04/17 00:13:07 hno Exp $ +# $Id: makefile.in,v 1.1.1.3.10.2 2000/05/03 19:18:10 hno Exp $ # srcdir = @srcdir@ @@ -14,12 +14,7 @@ prefix = @prefix@ exec_prefix = @exec_prefix@ -SUBDIRS = lib @makesnmplib@ scripts src icons errors -AUTHDIRS = auth_modules/NCSA \ - auth_modules/getpwnam \ - auth_modules/PAM \ - auth_modules/SMB \ - auth_modules/LDAP +SUBDIRS = lib @makesnmplib@ scripts src icons errors auth_modules noargs: all @@ -43,7 +38,7 @@ rm -f config.log makefile rm -f include/paths.h include/autoconf.h include/config.h rm -f auth_modules/dummy - @for dir in $(SUBDIRS) $(AUTHDIRS) contrib; do \ + @for dir in $(SUBDIRS) contrib; do \ echo Making distclean in $$dir; \ (cd $$dir; $(MAKE) $(MFLAGS) prefix="$(prefix)" distclean); \ done --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/auth_modules/Makefile.in Wed Feb 14 00:43:58 2007 @@ -0,0 +1,31 @@ +# Makefile for storage modules in the Squid Object Cache server +# +# $Id$ +# + +SUBDIRS = @AUTH_MODULES@ + +all: + @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \ + sh -c "cd $$dir && $(MAKE) all" || exit 1; \ + done + +clean: + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) clean"; \ + fi; \ + done + +distclean: + -rm -f Makefile + -for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) distclean"; \ + fi; \ + done + +.DEFAULT: + @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \ + sh -c "cd $$dir && $(MAKE) $@" || exit 1; \ + done --- squid/auth_modules/dummy.in Wed Feb 14 00:43:58 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,4 +0,0 @@ -this file stops configure/make complaining when building outside -the source tree. - -Masashi Fujita --- squid/auth_modules/MSNT/Makefile Wed Feb 14 00:43:58 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,18 +0,0 @@ -# Makefile for smb_auth - -INSTALLBIN = /usr/local/squid/bin - -OBJECTS = md4.o rfcnb-io.o rfcnb-util.o session.o smbauth.o smbdes.o \ -smbencrypt.o smblib-util.o smblib.o valid.o - -CC = gcc -CFLAGS = -s -O2 - -msntauth: $(OBJECTS) - $(CC) $(CFLAGS) -o msntauth $(OBJECTS) - -install: msntauth - install msntauth $(INSTALLBIN) - -clean: - rm -f msntauth libsmbvalid.a $(OBJECTS) --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/auth_modules/MSNT/Makefile.in Wed Feb 14 00:43:58 2007 @@ -0,0 +1,97 @@ +# +# Makefile for the Squid Object Cache server +# +# $Id$ +# +# Uncomment and customize the following to suit your needs: +# + +prefix = @prefix@ +exec_prefix = @exec_prefix@ +exec_suffix = @exec_suffix@ +cgi_suffix = @cgi_suffix@ +top_srcdir = @top_srcdir@ +bindir = @bindir@ +libexecdir = @libexecdir@ +sysconfdir = @sysconfdir@ +localstatedir = @localstatedir@ +srcdir = @srcdir@ +VPATH = @srcdir@ + +# Gotta love the DOS legacy +# +AUTH_EXE = msnt_auth$(exec_suffix) + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +INSTALL = @INSTALL@ +INSTALL_BIN = @INSTALL_PROGRAM@ +INSTALL_FILE = @INSTALL_DATA@ +INSTALL_SUID = @INSTALL_PROGRAM@ -o root -m 4755 +RANLIB = @RANLIB@ +LN_S = @LN_S@ +PERL = @PERL@ +CRYPTLIB = @CRYPTLIB@ +REGEXLIB = @REGEXLIB@ +PTHREADLIB = @PTHREADLIB@ +SNMPLIB = @SNMPLIB@ +MALLOCLIB = @LIB_MALLOC@ +AC_CFLAGS = @CFLAGS@ +LDFLAGS = @LDFLAGS@ +XTRA_LIBS = @XTRA_LIBS@ +XTRA_OBJS = @XTRA_OBJS@ +MV = @MV@ +RM = @RM@ +SHELL = /bin/sh +DEFINES = + +INCLUDE = -I. -I../../include -I$(top_srcdir)/include +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) +AUTH_LIBS = + +LIBPROGS = $(AUTH_EXE) +OBJS = md4.o rfcnb-io.o rfcnb-util.o session.o smbauth.o smbdes.o \ + smbencrypt.o smblib-util.o smblib.o valid.o + +all: $(AUTH_EXE) + +$(AUTH_EXE): $(OBJS) + $(CC) $(LDFLAGS) $(OBJS) -o $@ $(AUTH_LIBS) + +install-mkdirs: + -@if test ! -d $(prefix); then \ + echo "mkdir $(prefix)"; \ + mkdir $(prefix); \ + fi + -@if test ! -d $(libexecdir); then \ + echo "mkdir $(libexecdir)"; \ + mkdir $(libexecdir); \ + fi + +# Michael Lupp wants to know about additions +# to the install target. +install: all install-mkdirs + @for f in $(LIBPROGS); do \ + if test -f $(libexecdir)/$$f; then \ + echo $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + fi; \ + echo $(INSTALL_BIN) $$f $(libexecdir); \ + $(INSTALL_BIN) $$f $(libexecdir); \ + if test -f $(libexecdir)/-$$f; then \ + echo $(RM) -f $(libexecdir)/-$$f; \ + $(RM) -f $(libexecdir)/-$$f; \ + fi; \ + done + +clean: + -rm -rf *.o *.a *pure_* core $(LIBPROGS) + +distclean: clean + -rm -f Makefile + +tags: + ctags *.[ch] + +depend: + $(MAKEDEPEND) -fMakefile *.c Index: squid/auth_modules/SMB/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/auth_modules/SMB/Attic/Makefile.in,v retrieving revision 1.1.1.1 retrieving revision 1.1.1.1.12.1 diff -u -r1.1.1.1 -r1.1.1.1.12.1 --- squid/auth_modules/SMB/Makefile.in 26 Jan 2000 03:25:00 -0000 1.1.1.1 +++ squid/auth_modules/SMB/Makefile.in 3 May 2000 19:18:11 -0000 1.1.1.1.12.1 @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.1.1.1 2000/01/26 03:25:00 hno Exp $ +# $Id: Makefile.in,v 1.1.1.1.12.1 2000/05/03 19:18:11 hno Exp $ # # Uncomment and customize the following to suit your needs: # @@ -58,10 +58,11 @@ CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) AUTH_LIBS = -LIBPROGS = $(SMB_AUTH_EXE) $(SMB_AUTH_HELPER) +LIBPROGS = $(SMB_AUTH_EXE) +LIBSCRIPTS = $(SMB_AUTH_HELPER) OBJS = smb_auth.o -all: $(SMB_AUTH_EXE) +all: $(LIBPROGS) $(SMB_AUTH_EXE): smb_auth.o $(CC) $(LDFLAGS) smb_auth.o -o $@ $(AUTH_LIBS) @@ -91,9 +92,21 @@ $(RM) -f $(libexecdir)/-$$f; \ fi; \ done + @for f in $(LIBSCRIPTS); do \ + if test -f $(libexecdir)/$$f; then \ + echo $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + fi; \ + echo $(INSTALL_BIN) $$f $(libexecdir); \ + $(INSTALL_BIN) $(srcdir)/$$f $(libexecdir); \ + if test -f $(libexecdir)/-$$f; then \ + echo $(RM) -f $(libexecdir)/-$$f; \ + $(RM) -f $(libexecdir)/-$$f; \ + fi; \ + done clean: - -rm -rf *.o *pure_* core $(PROGS) + -rm -rf *.o *pure_* core $(LIBPROGS) distclean: clean -rm -f Makefile Index: squid/doc/Programming-Guide/Makefile =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/Programming-Guide/Makefile,v retrieving revision 1.1.1.1 retrieving revision 1.1.1.1.16.1 diff -u -r1.1.1.1 -r1.1.1.1.16.1 --- squid/doc/Programming-Guide/Makefile 26 Jan 2000 03:23:10 -0000 1.1.1.1 +++ squid/doc/Programming-Guide/Makefile 3 May 2000 19:18:11 -0000 1.1.1.1.16.1 @@ -3,7 +3,7 @@ all: $(DOC).html $(DOC).ps $(DOC).ps: $(DOC).dvi - dvips $(DOC).dvi + dvips -o $(DOC).ps $(DOC).dvi $(DOC).dvi: $(DOC).sgml sgml2latex $(DOC) Index: squid/doc/Programming-Guide/prog-guide.sgml =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/Programming-Guide/prog-guide.sgml,v retrieving revision 1.1.1.2.10.1 retrieving revision 1.1.1.2.10.2 diff -u -r1.1.1.2.10.1 -r1.1.1.2.10.2 --- squid/doc/Programming-Guide/prog-guide.sgml 17 Apr 2000 00:13:07 -0000 1.1.1.2.10.1 +++ squid/doc/Programming-Guide/prog-guide.sgml 3 May 2000 19:18:11 -0000 1.1.1.2.10.2 @@ -2,7 +2,7 @@
Squid Programmers Guide Duane Wessels, Squid Developers -$Id: prog-guide.sgml,v 1.1.1.2.10.1 2000/04/17 00:13:07 hno Exp $ +$Id: prog-guide.sgml,v 1.1.1.2.10.2 2000/05/03 19:18:11 hno Exp $ Squid is a WWW Cache application developed by the National Laboratory @@ -54,7 +54,7 @@ Note that the Squid source changes rapidly, and some parts of this document may become out-of-date. If you find any inconsistencies, please feel free to notify . + url="mailto:squid-dev@squid-cache.org" name="the Squid Developers">. Conventions @@ -106,22 +106,22 @@ Squid can quickly locate cached objects because it keeps (in memory) a hash table of all - Objects are saved to disk in a two-level directory structure. - For each object the + For each object the Client-side requests register themselves with a Processing Client Requests +

+ To be written... + Storage Manager - -Filesystem Interface +Introduction + +

+ The Storage Manager is the glue between client and server + sides. Every object saved in the cache is allocated a + + Squid can quickly locate cached objects because it keeps + (in memory) a hash table of all + For each object the + Client-side requests register themselves with a Object storage + +

+ To be written... + +Object retreival + +

+ To be written... + + +Storage Interface Introduction @@ -673,134 +723,528 @@ We want to try out our own, customized filesystems with Squid. In order to do that, we need a well-defined interface for the bits of Squid that access the permanent storage - devices. + devices. We also require tigher control of the replacement + policy by each storage module, rather than a single global + replacement policy. -The Interface +Build structure -Data Structures +

+ The storage types live in squid/src/fs/ . Each subdirectory corresponds + to the name of the storage type. When a new storage type is implemented + configure.in must be updated to autogenerate a Makefile in + squid/src/fs/$type/ from a Makefile.in file. - + configure will take a list of storage types through the + + Each storage type must create an archive file + + Each storefs must export a function named - Every cache object that is ``opened'' for reading or writing - will have a - struct _storeIOState { - sfileno swap_file_number; - mode_t mode; - size_t st_size; /* do stat(2) after read open */ - off_t offset; /* current offset pointer */ - STIOCB *callback; - void *callback_data; - struct { - STRCB *callback; - void *callback_data; - } read; - struct { - unsigned int closing:1; /* debugging aid */ - } flags; - union { - struct { - int fd; - struct { - unsigned int close_request:1; - unsigned int reading:1; - unsigned int writing:1; - } flags; - } ufs; - } type; - }; - - - External Functions - -Object I/O - -

- These functions all relate to per-object I/O tasks: opening, - closing, reading, writing, and unlinking objects on disk. - These functions can all be found in - Note that the underlying storage system functions are - accessed through function pointers, kept in the - + + +Initialisation of a storage type + +

+ Each storage type initialises through the - struct _SwapDir { - .... - struct { - STOBJOPEN *open; - STOBJCLOSE *close; - STOBJREAD *read; - STOBJWRITE *write; - STOBJUNLINK *unlink; - } obj; - .... - }; + void + storeFsSetup_ufs(storefs_entry_t *storefs) + { + assert(!ufs_initialised); + storefs->parsefunc = storeUfsDirParse; + storefs->reconfigurefunc = storeUfsDirReconfigure; + storefs->donefunc = storeUfsDirDone; + ufs_state_pool = memPoolCreate("UFS IO State data", sizeof(ufsstate_t)); + ufs_initialised = 1; + } + + +

+ There are five function pointers in the storefs_entry which require + initialising. In this example, some protection is made against the + setup function being called twice, and a memory pool is initialised + for use inside the storage module. + +

+ Each function will be covered below. + + +done + +

+ + typedef void + STFSSHUTDOWN(void); + + +

+ This function is called whenever the storage system is to be shut down. + It should take care of deallocating any resources currently allocated. + + + + typedef void STFSPARSE(SwapDir *SD, int index, char *path); + typedef void STFSRECONFIGURE(SwapDir *SD, int index, char *path); + + +

+ These functions handle configuring and reconfiguring a storage + directory. Additional arguments from the cache_dir configuration + line can be retrieved through calls to strtok() and GetInteger(). + +

+ + + struct _SwapDir { + char *type; /* Pointer to the store dir type string */ + int cur_size; /* Current swapsize in kb */ + int low_size; /* ?? */ + int max_size; /* Maximum swapsize in kb */ + char *path; /* Path to store */ + int index; /* This entry's index into the swapDir array */ + int suggest; /* Suggestion for UFS style stores (??) */ + size_t max_objsize; /* Maximum object size for this store */ + union { /* Replacement policy-specific fields */ + #ifdef HEAP_REPLACEMENT + struct { + heap *heap; + } heap; + #endif + struct { + dlink_list list; + dlink_node *walker; + } lru; + } repl; + int removals; + int scanned; + struct { + unsigned int selected:1; /* Currently selected for write */ + unsigned int read_only:1; /* This store is read only */ + } flags; + STINIT *init; /* Initialise the fs */ + STNEWFS *newfs; /* Create a new fs */ + STDUMP *dump; /* Dump fs config snippet */ + STFREE *freefs; /* Free the fs data */ + STDBLCHECK *dblcheck; /* Double check the obj integrity */ + STSTATFS *statfs; /* Dump fs statistics */ + STMAINTAINFS *maintainfs; /* Replacement maintainence */ + STCHECKOBJ *checkob; /* Check if the fs will store an object, and get the FS load */ + /* These two are notifications */ + STREFOBJ *refobj; /* Reference this object */ + STUNREFOBJ *unrefobj; /* Unreference this object */ + STCALLBACK *callback; /* Handle pending callbacks */ + STSYNC *sync; /* Sync the directory */ + struct { + STOBJCREATE *create; /* Create a new object */ + STOBJOPEN *open; /* Open an existing object */ + STOBJCLOSE *close; /* Close an open object */ + STOBJREAD *read; /* Read from an open object */ + STOBJWRITE *write; /* Write to a created object */ + STOBJUNLINK *unlink; /* Remove the given object */ + } obj; + struct { + STLOGOPEN *open; /* Open the log */ + STLOGCLOSE *close; /* Close the log */ + STLOGWRITE *write; /* Write to the log */ + struct { + STLOGCLEANOPEN *open; /* Open a clean log */ + STLOGCLEANWRITE *write; /* Write to the log */ + void *state; /* Current state */ + } clean; + } log; + void *fsdata; /* FS-specific data */ + }; + + + + +Operation of a storage module + +

+ Squid understands the concept of multiple diverse storage directories. + Each storage directory provides a caching object store, with object + storage, retrieval, indexing and replacement. + +

+ Each open object has associated with it a + + struct _storeIOState { + sdirno swap_dirn; /* SwapDir index */ + sfileno swap_filen; /* Unique file index number */ + StoreEntry *e; /* Pointer to parent StoreEntry */ + mode_t mode; /* Mode - O_RDONLY or O_WRONLY */ + size_t st_size; /* Size of the object if known */ + off_t offset; /* current _on-disk_ offset pointer */ + STFNCB *file_callback; /* called on delayed sfileno assignments */ + STIOCB *callback; /* IO Error handler callback */ + void *callback_data; /* IO Error handler callback data */ + struct { + STRCB *callback; /* Read completion callback */ + void *callback_data; /* Read complation callback data */ + } read; + struct { + unsigned int closing:1; /* debugging aid */ + } flags; + void *fsstate; /* pointer to private fs state */ + }; + + +

+ Each + The specific filesystem operations listed in the SwapDir object are + covered below. + +initfs + +

+ + typedef void + STINIT(SwapDir *SD);

- Thus, a storage system must do something like this - when initializing its newfs + +

- SwapDir->obj.open = storeFooOpen; - SwapDir->obj.close = storeFooClose; - SwapDir->obj.read = storeFooRead; - SwapDir->obj.write = storeFooWrite; - SwapDir->obj.unlink = storeFooUnlink; + typedef void + STNEWFS(SwapDir *SD); - + Called for each configured dumpfs + +

+ + typedef void + STDUMP(StoreEntry *e, const char *path, SwapDir *SD); + + +

+ Dump the configuration of the current freefs + +

+ + typedef void + STFREE(SwapDir *SD); + + +

+ Free the fsdata/. + + +doublecheckfs + +

+ + typedef int + STDBLCHECK(SwapDir *SD, StoreEntry *e); + + +

+ Double-check the given object for validity. Called during rebuild if + the '-S' flag is given to squid on the command line. Returns 1 if the + object is indeed valid, and 0 if the object is found invalid. + + +statfs + +

+ + typedef void + STSTATFS(SwapDir *SD, StoreEntry *e); + + +

+ Called to retrieve filesystem statistics, such as usage, load and + errors. The information should be appended to the passed + maintainfs + +

+ + typedef void + STMAINTAINFS(SwapDir *SD); + + +

+ Called periodically to replace objects. The active replacement policy + should be used to timeout unused objects in order to make room for + new objects. + +callback + +

+ + typedef void + STCALLBACK(SwapDir *SD); + + +

+ This function is called inside the comm_select/comm_poll loop to handle + any callbacks pending. + + +sync + +

+ + typedef void + STSYNC(SwapDir *SD); + + +

+ This function is called whenever a sync to disk is required. This + function should not return until all pending data has been flushed to + disk. + + +parse/reconfigure + +

+ +checkobj + +

+ + typedef int + STCHECKOBJ(SwapDir *SD, const StoreEntry *e); + + +

+ Called by referenceobj + +

+ + typedef void + STREFOBJ(SwapDir *SD, StoreEntry *e); + + +

+ Called whenever an object is locked by unreferenceobj + +

+ + typedef void + STUNREFOBJ(SwapDir *SD, StoreEntry *e); + + +

+ Called whenever the object is unlocked by createobj + +

+ + typedef storeIOState * + STOBJCREATE(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *io_callback, void *io_callback_data); + + +

+ Create an object in the + The IO callback should be called when an error occurs and when the + object is closed. Once the IO callback is called, the + openobj + +

+ + typedef storeIOState * + STOBJOPEN(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *io_callback, void *io_callback_data); + + +

+ Open the closeobj + +

+ + typedef void + STOBJCLOSE(SwapDir *SD, storeIOState *sio); + + +

+ Close an opened object. The readobj + +

+ + typedef void + STOBJREAD(SwapDir *SD, storeIOState *sio, char *buf, size_t size, off_t offset, STRCB *read_callback, void *read_callback_data); + + +

+ Read part of the object of into + If a read operation fails, the filesystem layer notifies the + calling module by calling the writeobj + +

+ + typedef void + STOBJWRITE(SwapDir *SD, storeIOState *sio, char *buf, size_t size, off_t offset, FREE *freefunc); + + +

+ Write the given block of data to the given store object. + If a write operation fails, the filesystem layer notifies the + calling module by calling the unlinkobj + +

+ + typedef void STOBJUNLINK(SwapDir *, StoreEntry *); + + +

+ Remove the Store IO calls + +

+ These routines are used inside the storage manager to create and + retrieve objects from a storage directory. + +storeCreate()

storeIOState * - storeOpen(sfileno f, mode_t mode, STIOCB *callback, void *callback_data) + storeCreate(StoreEntry *e, STIOCB *file_callback, STIOCB *close_callback, void * callback_data)

- - storeOpen()

- void - storeClose(storeIOState *sio) + storeIOState * + storeOpen(StoreEntry *e, STFNCB * file_callback, STIOCB * callback, void *callback_data)

- + + storeRead() + +

void storeRead(storeIOState *sio, char *buf, size_t size, off_t offset, STRCB *callback, void *callback_data) @@ -842,7 +1297,7 @@

- The caller is responsible for allocating and freeing +storeWrite() + +

void storeWrite(storeIOState *sio, char *buf, size_t size, off_t offset, FREE *free_func) @@ -862,7 +1318,7 @@

+storeUnlink() + +

- void - storeUnlink(sfileno f) + void + storeUnlink(StoreEntry *e)

@@ -888,21 +1345,28 @@ does not need to be opened first. The filesystem layer will remove the object if it exists on the disk. - +storeOffset() + +

- off_t - storeOffset(storeIOState *sio) + off_t storeOffset(storeIOState *sio) + +

- Returns the current byte-offset of the cache object on - disk. For read-objects, this is the offset after the last - successful disk read operation. For write-objects, it is - the offset of the last successful disk write operation. + offset/ refers to the ondisk offset, or undefined + results will occur. For reads, this returns the current offset of + sucessfully read data, not including queued reads. + -Callbacks + + @@ -929,7 +1393,7 @@ Once the The @@ -946,43 +1410,11 @@ called if the read operation is successful. If it fails, then the Config file parsing - -

- There are three functions relating to the Squid configuration - file: parsing, dumping, and freeing. +State Logging

- The parse function is called at startup, and during a reconfigure, - for a - The ``dump'' function is used to output a configuration - file from the in-memory configuration structure. It is - called with a - The free function is called during a reconfigure (and at - exit) to free up (or un-initialize) any memory or structures - associated with the configuration line. The Filesystem Startup, Initialization, and State Logging - -

- These functions deal with initializing, state - logging, and related tasks for a squid storage system. + These functions deal with state + logging and related tasks for a squid storage system. These functions are used (called) in @@ -1010,36 +1442,7 @@ }; - - - void - STINIT(SwapDir *); - - -

- The - - void - STNEWFS(SwapDir *); - - -

- The @@ -1060,7 +1463,7 @@ the state log and then re-opens them. A @@ -1074,7 +1477,7 @@ the open state-holding log files (if any) for the storage system. - @@ -1089,7 +1492,7 @@ This feature may not be required by some storage systems and can be implemented as a null-function (no-op). - @@ -1111,7 +1514,7 @@ keep state information for the clean-writing process, but should not be accessed by upper layers. - @@ -1130,9 +1533,301 @@ to become the official state-holding log. +Replacement policy implementation + +

+The replacement policy can be updated during STOBJREAD/STOBJWRITE/STOBJOPEN/ +STOBJCLOSE as well as STREFOBJ and STUNREFOBJ. Care should be taken to +only modify the relevant replacement policy entries in the StoreEntry. +The responsibility of replacement policy maintainence has been moved into +each SwapDir so that the storage code can have tight control of the +replacement policy. Cyclic filesystems such as COSS require this tight +coupling between the storage layer and the replacement policy. + + +Future removal policy + +

+ (replaces the above Replace policy) + +

+ The removal policy is responsible for determining in which order + objects are deleted when Squid needs to reclaim space for new objects. + Such a policy is used by a object storage for maintaining the stored + objects and determining what to remove to reclaim space for new objects. + (together they implements a replacement policy) + +API +

+ It is implemented as a modular API where a storage directory or + memory creates a policy of choice for maintaining it's objects, + and modules registerering to be used by this API. + +createRemovalPolicy() + +

+ + RemovalPolicy policy = createRemovalPolicy(cons char *type, cons char *args) + + +

+ Creates a removal policy instance where object priority can be + maintained + +

+ The returned RemovalPolicy instance is cbdata registered + +policy.Free() + +

+ + policy->Free(RemovalPolicy *policy) + + +

+ Destroys the policy instance and frees all related memory. + +policy.Add() + +

+ + policy->Add(RemovalPolicy *policy, StoreEntry *, RemovalPolicyNode *node) + + +

+ Adds a StoreEntry to the policy instance. + +

+ datap is a pointer to where policy specific data can be stored + for the store entry, currently the size of one (void *) pointer. + +policy.Remove() + +

+ + policy->Remove(RemovalPolicy *policy, StoreEntry *, RemovalPolicyNode *node) + + +

+ Removes a StoreEntry from the policy instance out of + policy order. For example when an object is replaced + by a newer one or is manually purged from the store. + +

+ datap is a pointer to where policy specific data is stored + for the store entry, currently the size of one (void *) pointer. + +policy.Referenced() + +

+ + policy->Referenced(RemovalPolicy *policy, const StoreEntry *, RemovalPolicyNode *node) + + +

+ Tells the policy that a StoreEntry has been referenced. + +

+ datap is a pointer to where policy specific data is stored + for the store entry, currently the size of one (void *) pointer. + +policy.WalkInit() + +

+ + RemovalPolicyWalker walker = policy->WalkInit(RemovalPolicy *policy) + + +

+ Initiates a walk of all objects in the policy instance. + The objects is returned in an order suitable for using + as reinsertion order when rebuilding the policy. + +

+ The returned RemovalPolicyWalker instance is cbdata registered + +

+ Note: The walk must be performed as an atomic operation + with no other policy actions interveaning, or the outcome + will be undefined. + +walker.Next() + +

+ + const StoreEntry *entry = walker->Next(RemovalPolicyWalker *walker) + + +

+ Gets the next object in the walk chain + +

+ Return NULL when there is no further objects + +walker.Done() + +

+ + walker->Done(RemovalPolicyWalker *walker) + + +

+ Finishes a walk of the maintaned objects, destroys + walker. + +policy.PurgeInit() + +

+ + RemovalPurgeWalker purgewalker = policy->PurgeInit(RemovalPolicy *policy, int max_scan) + + +

+ Initiates a search for removal candidates. Seach depth is indicated + by max_scan. + +

+ The returned RemovalPurgeWalker instance is cbdata registered + +

+ Note: The walk must be performed as an atomic operation + with no other policy actions interveaning, or the outcome + will be undefined. + +purgewalker.Next() + +

+ + StoreEntry *entry = purgewalker->Next(RemovalPurgeWalker *purgewalker) + + +

+ Gets the next object to purge. The purgewalker will remove each + returned object from the policy. + +

It is the policys responsibility to verify that the object + isn't locked or otherwise prevented from being removed. What this + means is that the policy must not return objects where + storeEntryLocked() is true. + +

+ Return NULL when there is no further purgeable objects in the policy. + +purgewalker.Done() + +

+ + purgewalker->Done(RemovalPurgeWalker *purgewalker) + + +

+ Finishes a walk of the maintaned objects, destroys + walker and restores the policy to it's normal state. + +Future removal policy implementation + +Source layout + +

+ Policy implementations resides in src/repl/<name>/, and a make in + such a directory must result in a object archive src/repl/<name>.a + containing all the objects implementing the policy. + +Internal structures + +RemovalPolicy + +

+ + typedef struct _RemovalPolicy RemovalPolicy; + struct _RemovalPolicy { + char *_type; + void *_data; + void (*add)(RemovalPolicy *policy, StoreEntry *); + ... /* see the API definition above */ + }; + + +

+ The _type member is mainly for debugging and diagnostics purposes, and + should be a pointer to the name of the policy (same name as used for + creation) + +

+ The _data member is for storing policy specific information. + +RemovalPolicyWalker + +

+ + typedef struct _RemovalPolicyWalker RemovalPolicyWalker; + struct _RemovalPolicyWalker { + RemovalPolicy *_policy; + void *_data; + StoreEntry *(*next)(RemovalPolicyWalker *); + ... /* see the API definition above */ + }; + + +RemovalPolicyNode + +

+ + typedef struct _RemovalPolicyNode RemovalPolicyNode; + struct _RemovalPolicyNode { + void *data; + }; + + + Stores policy specific information about a entry. Currently + there is only space for a single pointer, but plans are to + maybe later provide more space here to allow simple policies + to store all their data "inline" to preserve some memory. + +Policy registration + +

+ Policies are automatically registered in the Squid binary from the + policy selection made by the user building Squid. In the future this + might get extended to support loadable modules. All registered + policies are available to object stores which wishes to use them. + +Policy instance creation + +

+ Each policy must implement a "create/new" function " + It should also populate the _data member with a pointer to policy + specific data. + +

+ Prior to returning the created instance must be registered as + callback-data by calling cbdataAdd(). + +Walker + +

+ When a walker is created the policy populates it with at least the API + methods supported. Currently all API calls are mandatory, but the + policy implementation must make sure to NULL fill the structure prior + to populating it in order to assure future API compability. + +

+ Prior to returning the created instance must be registered as + callback-data by calling cbdataAdd(). + Forwarding Selection +

+ To be written... + IP Cache and FQDN Cache @@ -1217,30 +1912,72 @@ Server Protocols HTTP + +

+ To be written... + FTP + +

+ To be written... + Gopher + +

+ To be written... + Wais + +

+ To be written... + SSL + +

+ To be written... + Passthrough +

+ To be written... + Timeouts +

+ To be written... + Events +

+ To be written... + Access Controls +

+ To be written... + ICP +

+ To be written... + Network Measurement Database +

+ To be written... + Error Pages +

+ To be written... + + Callback Data Database

@@ -1309,6 +2046,9 @@ Cache Manager +

+ To be written... + HTTP Headers Index: squid/icons/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/icons/Attic/Makefile.in,v retrieving revision 1.1.1.3 retrieving revision 1.1.1.3.12.1 diff -u -r1.1.1.3 -r1.1.1.3.12.1 --- squid/icons/Makefile.in 26 Jan 2000 03:25:00 -0000 1.1.1.3 +++ squid/icons/Makefile.in 3 May 2000 19:18:11 -0000 1.1.1.3.12.1 @@ -1,4 +1,4 @@ -# $Id: Makefile.in,v 1.1.1.3 2000/01/26 03:25:00 hno Exp $ +# $Id: Makefile.in,v 1.1.1.3.12.1 2000/05/03 19:18:11 hno Exp $ # prefix = @prefix@ exec_prefix = @exec_prefix@ @@ -56,7 +56,7 @@ :; \ else \ echo "$(INSTALL_FILE) $$f $(DEFAULT_ICON_DIR)"; \ - $(INSTALL_FILE) $$f $(DEFAULT_ICON_DIR); \ + $(INSTALL_FILE) $(srcdir)/$$f $(DEFAULT_ICON_DIR); \ fi; \ done Index: squid/src/HttpReply.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpReply.c,v retrieving revision 1.1.1.3.10.1 retrieving revision 1.1.1.3.10.2 diff -u -r1.1.1.3.10.1 -r1.1.1.3.10.2 --- squid/src/HttpReply.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.10.1 +++ squid/src/HttpReply.c 3 May 2000 19:18:12 -0000 1.1.1.3.10.2 @@ -1,6 +1,6 @@ /* - * $Id: HttpReply.c,v 1.1.1.3.10.1 2000/04/17 00:13:09 hno Exp $ + * $Id: HttpReply.c,v 1.1.1.3.10.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 58 HTTP Reply (Response) * AUTHOR: Alex Rousskov @@ -144,12 +144,12 @@ */ char *headers = memAllocate(MEM_4K_BUF); int success; + size_t s = XMIN(end + 1, 4096); /* reset current state, because we are not used in incremental fashion */ httpReplyReset(rep); - /* put a string terminator */ - xstrncpy(headers, buf, 4096); - if (end >= 0 && end < 4096) - *(headers + end) = '\0'; + /* put a string terminator. s is how many bytes to touch in + * 'buf' including the terminating NULL. */ + xstrncpy(headers, buf, s); success = httpReplyParseStep(rep, headers, 0); memFree(headers, MEM_4K_BUF); return success == 1; Index: squid/src/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/Makefile.in,v retrieving revision 1.1.1.3.12.2 retrieving revision 1.1.1.3.12.3 diff -u -r1.1.1.3.12.2 -r1.1.1.3.12.3 --- squid/src/Makefile.in 17 Apr 2000 00:13:09 -0000 1.1.1.3.12.2 +++ squid/src/Makefile.in 3 May 2000 19:18:12 -0000 1.1.1.3.12.3 @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.1.1.3.12.2 2000/04/17 00:13:09 hno Exp $ +# $Id: Makefile.in,v 1.1.1.3.12.3 2000/05/03 19:18:12 hno Exp $ # # Uncomment and customize the following to suit your needs: # @@ -18,6 +18,8 @@ srcdir = @srcdir@ VPATH = @srcdir@ +SUBDIRS = fs + # Gotta love the DOS legacy # SQUID_EXE = squid$(exec_suffix) @@ -26,8 +28,6 @@ UNLINKD_EXE = unlinkd$(exec_suffix) PINGER_EXE = pinger$(exec_suffix) CACHEMGR_EXE = cachemgr$(cgi_suffix) -DISKD_EXE = diskd$(exec_suffix) -OPT_DISKD_EXE = @OPT_DISKD_EXE@ DEFAULT_PREFIX = $(prefix) DEFAULT_CONFIG_FILE = $(sysconfdir)/squid.conf @@ -62,6 +62,8 @@ LDFLAGS = @LDFLAGS@ XTRA_LIBS = @XTRA_LIBS@ XTRA_OBJS = @XTRA_OBJS@ +STORE_OBJS = @STORE_OBJS@ +STORE_MODULES = @STORE_MODULES@ MV = @MV@ RM = @RM@ SHELL = /bin/sh @@ -77,14 +79,13 @@ STD_APP_LIBS = -L../lib -lmiscutil $(XTRA_LIBS) PROGS = $(SQUID_EXE) $(CLIENT_EXE) -UTILS = $(DNSSERVER_EXE) $(UNLINKD_EXE) $(OPT_DISKD_EXE) +UTILS = $(DNSSERVER_EXE) $(UNLINKD_EXE) $(DISKD_EXE) SUID_UTILS = $(PINGER_EXE) CGIPROGS = $(CACHEMGR_EXE) OBJS = \ access_log.o \ acl.o \ asn.o \ - @ASYNC_OBJS@ \ authenticate.o \ cache_cf.o \ CacheDigest.o \ @@ -155,18 +156,17 @@ stmem.o \ store.o \ store_io.o \ - store_io_ufs.o \ - @DISKD_OBJS@ \ store_client.o \ store_digest.o \ store_dir.o \ - store_dir_ufs.o \ store_key_md5.o \ store_log.o \ + store_modules.o \ store_rebuild.o \ store_swapin.o \ store_swapmeta.o \ store_swapout.o \ + store_heap_replacement.o \ string_arrays.o \ tools.o \ unlinkd.o \ @@ -186,28 +186,23 @@ DELAY_OBJS = delay_pools.o -ASYNC_OBJS = \ - aiops.o \ - async_io.o \ - store_io_asyncufs.o - LEAKFINDER_OBJS = \ leakfinder.o DEFAULTS = \ -DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\" -all: squid.conf $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS) +all: squid.conf store_modules $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS) $(OBJS): $(top_srcdir)/include/version.h $(SNMP_OBJS): ../snmplib/libsnmp.a $(top_srcdir)/include/cache_snmp.h -$(SQUID_EXE): $(OBJS) - $(CC) -o $@ $(LDFLAGS) $(OBJS) $(SQUID_LIBS) +$(SQUID_EXE): $(OBJS) $(STORE_OBJS) + $(CC) -o $@ $(LDFLAGS) $(OBJS) $(STORE_OBJS) $(SQUID_LIBS) globals.o: globals.c Makefile - $(CC) -c $< $(CFLAGS) -I$(srcdir) $(DEFAULTS) + $(CC) -c globals.c $(CFLAGS) -I$(srcdir) $(DEFAULTS) globals.c: globals.h mk-globals-c.pl $(PERL) $(srcdir)/mk-globals-c.pl < $(srcdir)/globals.h > $@ @@ -221,12 +216,6 @@ $(DNSSERVER_EXE): dnsserver.o $(CC) -o $@ $(LDFLAGS) dnsserver.o $(DNSSERVER_LIBS) -$(DISKD_EXE): diskd-daemon.o - $(CC) -o $@ $(LDFLAGS) diskd-daemon.o $(DISKD_LIBS) - -diskd-daemon.o: diskd.c - $(CC) -c $(CFLAGS) -DDISKD_DAEMON=1 $(srcdir)/diskd.c -o $@ - $(CACHEMGR_EXE): cachemgr.o $(CC) -o $@ $(LDFLAGS) cachemgr.o $(CLIENT_LIBS) @@ -253,6 +242,11 @@ cf_gen: cf_gen.o $(CC) -o $@ $(LDFLAGS) cf_gen.o $(STD_APP_LIBS) +cf_gen.o: cf_gen.c cf_gen_defines.h Makefile defines.h ../include/autoconf.h + +cf_gen_defines.h: $(srcdir)/cf_gen_defines $(srcdir)/cf.data.pre + awk -f $(srcdir)/cf_gen_defines <$(srcdir)/cf.data.pre >cf_gen_defines.h + cf.data: cf.data.pre Makefile sed "\ s%@DEFAULT_MIME_TABLE@%$(DEFAULT_MIME_TABLE)%g;\ @@ -270,6 +264,17 @@ s%@DEFAULT_PREFIX@%$(DEFAULT_PREFIX)%g;"\ < $(srcdir)/cf.data.pre >$@ +store_modules.c: store_modules.sh Makefile + @sh $(srcdir)/store_modules.sh $(STORE_MODULES) >store_modules.c + +store_modules.o: store_modules.c + $(CC) -c store_modules.c $(CFLAGS) -I$(srcdir) + +$(STORE_OBJS): fs/stamp + +store_modules fs/stamp: + @sh -c "cd fs && $(MAKE) all" + install-mkdirs: -@if test ! -d $(prefix); then \ echo "mkdir $(prefix)"; \ @@ -303,6 +308,10 @@ # Michael Lupp wants to know about additions # to the install target. install: all install-mkdirs + @for dir in $(SUBDIRS); do \ + echo "Making $@ in $$dir..."; \ + (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \ + done @for f in $(PROGS); do \ if test -f $(bindir)/$$f; then \ echo $(MV) $(bindir)/$$f $(bindir)/-$$f; \ @@ -371,14 +380,24 @@ clean: -rm -rf *.o *pure_* core $(PROGS) $(UTILS) $(CGIPROGS) $(SUID_UTILS) - -rm -f cf_gen cf_parser.c cf.data globals.c string_arrays.c + -rm -f cf_gen cf_gen_defines.h cf_parser.c cf.data globals.c string_arrays.c + -rm -f store_modules.c + @for dir in $(SUBDIRS); do \ + echo "Making $@ in $$dir..."; \ + (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \ + done distclean: clean -rm -f Makefile squid.conf squid.conf.pre -rm -f Makefile.bak + -rm -f tags + @for dir in $(SUBDIRS); do \ + echo "Making $@ in $$dir..."; \ + (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \ + done tags: ctags *.[ch] ../include/*.h ../lib/*.[ch] -depend: - $(MAKEDEPEND) -I../include -I. -fMakefile *.c +depend: cf_parser.c + $(MAKEDEPEND) $(INCLUDE) -fMakefile $(srcdir)/*.c Index: squid/src/access_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/access_log.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/access_log.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.12.1 +++ squid/src/access_log.c 3 May 2000 19:18:12 -0000 1.1.1.3.12.2 @@ -1,7 +1,7 @@ /* - * $Id: access_log.c,v 1.1.1.3.12.1 2000/04/17 00:13:09 hno Exp $ + * $Id: access_log.c,v 1.1.1.3.12.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -37,7 +37,6 @@ #include "squid.h" -static char *log_quote(const char *header); static void accessLogSquid(AccessLogEntry * al); static void accessLogCommon(AccessLogEntry * al); static Logfile *logfile = NULL; @@ -118,7 +117,7 @@ /* log_quote -- URL-style encoding on MIME headers. */ -static char * +char * log_quote(const char *header) { int c; Index: squid/src/acl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/acl.c,v retrieving revision 1.1.1.3.12.4 retrieving revision 1.1.1.3.12.5 diff -u -r1.1.1.3.12.4 -r1.1.1.3.12.5 --- squid/src/acl.c 3 May 2000 11:22:56 -0000 1.1.1.3.12.4 +++ squid/src/acl.c 3 May 2000 19:18:12 -0000 1.1.1.3.12.5 @@ -1,6 +1,6 @@ /* - * $Id: acl.c,v 1.1.1.3.12.4 2000/05/03 11:22:56 asd Exp $ + * $Id: acl.c,v 1.1.1.3.12.5 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Duane Wessels @@ -184,6 +184,8 @@ #if USE_IDENT if (!strcmp(s, "ident")) return ACL_IDENT; + if (!strcmp(s, "ident_regex")) + return ACL_IDENT_REGEX; #endif if (!strncmp(s, "proto", 5)) return ACL_PROTO; @@ -193,6 +195,8 @@ return ACL_BROWSER; if (!strcmp(s, "proxy_auth")) return ACL_PROXY_AUTH; + if (!strcmp(s, "proxy_auth_regex")) + return ACL_PROXY_AUTH_REGEX; if (!strcmp(s, "src_as")) return ACL_SRC_ASN; if (!strcmp(s, "dst_as")) @@ -207,6 +211,8 @@ if (!strcmp(s, "arp")) return ACL_SRC_ARP; #endif + if (!strcmp(s, "req_mime_type")) + return ACL_REQ_MIME_TYPE; return ACL_NONE; } @@ -242,6 +248,8 @@ #if USE_IDENT if (type == ACL_IDENT) return "ident"; + if (type == ACL_IDENT_REGEX) + return "ident_regex"; #endif if (type == ACL_PROTO) return "proto"; @@ -251,6 +259,8 @@ return "browser"; if (type == ACL_PROXY_AUTH) return "proxy_auth"; + if (type == ACL_PROXY_AUTH_REGEX) + return "proxy_auth_regex"; if (type == ACL_SRC_ASN) return "src_as"; if (type == ACL_DST_ASN) @@ -265,6 +275,8 @@ if (type == ACL_SRC_ARP) return "arp"; #endif + if (type == ACL_REQ_MIME_TYPE) + return "req_mime_type"; return "ERROR"; } @@ -747,6 +759,9 @@ case ACL_IDENT: aclParseWordList(&A->data); break; + case ACL_IDENT_REGEX: + aclParseRegexList(&A->data); + break; #endif case ACL_PROTO: aclParseProtoList(&A->data); @@ -766,6 +781,14 @@ assert(proxy_auth_cache); } break; + case ACL_PROXY_AUTH_REGEX: + aclParseRegexList(&A->data); + if (!proxy_auth_cache) { + /* First time around, 7921 should be big enough */ + proxy_auth_cache = hash_create((HASHCMP *) strcmp, 7921, hash_string); + assert(proxy_auth_cache); + } + break; #if SQUID_SNMP case ACL_SNMP_COMMUNITY: aclParseWordList(&A->data); @@ -776,6 +799,9 @@ aclParseArpList(&A->data); break; #endif + case ACL_REQ_MIME_TYPE: + aclParseWordList(&A->data); + break; case ACL_NONE: default: fatal("Bad ACL type"); @@ -1115,7 +1141,7 @@ */ static int -aclMatchProxyAuth(wordlist * data, const char *proxy_auth, acl_proxy_auth_user * auth_user, aclCheck_t * checklist) +aclMatchProxyAuth(void * data, const char *proxy_auth, acl_proxy_auth_user * auth_user, aclCheck_t * checklist, squid_acl acltype) { /* checklist is used to register user name when identified, nothing else */ LOCAL_ARRAY(char, login_buf, USER_IDENT_SZ); @@ -1192,23 +1218,40 @@ auth_user->ipaddr = checklist->src_addr; /* copy username to request for logging on client-side */ xstrncpy(checklist->request->user_ident, user, USER_IDENT_SZ); - rv = aclMatchUser(data, user); - if (checklist->conn) { - if (rv) - checklist->auth_state = AUTHENTICATE_STATE_DONE; + switch(acltype) { + case ACL_PROXY_AUTH: + rv = aclMatchUser(data, user); + if (checklist->conn) { + if (rv) + checklist->auth_state = AUTHENTICATE_STATE_DONE; #if USE_NTLM - xstrncpy(checklist->conn->ident, user, USER_IDENT_SZ); + xstrncpy(checklist->conn->ident, user, USER_IDENT_SZ); #endif + } + return rv; + break; + case ACL_PROXY_AUTH_REGEX: + return aclMatchRegex(data, user); + default: + fatal("aclMatchProxyAuth: unknown ACL type"); + return 0; /* NOTREACHED */ } - return rv; } else { - /* user has switched to another IP addr */ - debug(28, 1) ("aclMatchProxyAuth: user '%s' has changed IP address\n", user); - /* remove this user from the hash, making him unknown */ - hash_remove_link(proxy_auth_cache, (hash_link *) auth_user); - aclFreeProxyAuthUser(auth_user); - /* require the user to reauthenticate */ - return -2; + if (Config.onoff.authenticateIpTTLStrict) { + /* Access from some other IP address than the one owning + * this user ID. Deny access + */ + debug(28, 1) ("aclMatchProxyAuth: user '%s' tries to use multple IP addresses!\n", user); + return 0; + } else { + /* user has switched to another IP addr */ + debug(28, 1) ("aclMatchProxyAuth: user '%s' has changed IP address\n", user); + /* remove this user from the hash, making him unknown */ + hash_remove_link(proxy_auth_cache, (hash_link *) auth_user); + aclFreeProxyAuthUser(auth_user); + /* require the user to reauthenticate */ + return -2; + } } } else { /* password mismatch/timeout */ @@ -1473,6 +1516,14 @@ return 0; } /* NOTREACHED */ + case ACL_IDENT_REGEX: + if (checklist->ident[0]) { + return aclMatchRegex(ae->data, checklist->ident); + } else { + checklist->state[ACL_IDENT] = ACL_LOOKUP_NEEDED; + return 0; + } + /* NOTREACHED */ #endif case ACL_PROTO: return aclMatchInteger(ae->data, r->protocol); @@ -1487,6 +1538,7 @@ return aclMatchRegex(ae->data, browser); /* NOTREACHED */ case ACL_PROXY_AUTH: + case ACL_PROXY_AUTH_REGEX: if (NULL == r) { return -1; } else if (!r->flags.accelerated) { @@ -1516,7 +1568,8 @@ switch (aclMatchProxyAuth(ae->data, header, checklist->auth_user, - checklist)) { + checklist, + ae->type)) { case 0: /* Correct password, but was not allowed in this ACL */ return 0; @@ -1565,6 +1618,13 @@ case ACL_SRC_ARP: return aclMatchArp(&ae->data, checklist->src_addr); #endif + case ACL_REQ_MIME_TYPE: + header = httpHeaderGetStr(&checklist->request->header, + HDR_CONTENT_TYPE); + if (NULL == header) + header = ""; + return aclMatchRegex(ae->data, header); + /* NOTREACHED */ case ACL_NONE: default: debug(28, 0) ("aclMatchAcl: '%s' has bad type %d\n", @@ -1925,6 +1985,10 @@ case ACL_TIME: aclDestroyTimeList(a->data); break; +#if USE_IDENT + case ACL_IDENT_REGEX: +#endif + case ACL_PROXY_AUTH_REGEX: case ACL_URL_REGEX: case ACL_URLPATH_REGEX: case ACL_BROWSER: @@ -2023,37 +2087,23 @@ static int aclDomainCompare(const void *a, const void *b) { - const char *d1 = a; - const char *d2 = b; - int l1; - int l2; - while ('.' == *d1) - d1++; - while ('.' == *d2) - d2++; - l1 = strlen(d1); - l2 = strlen(d2); - while (d1[--l1] == d2[--l2]) { - if ((l1 == 0) && (l2 == 0)) - return 0; /* d1 == d2 */ - if (0 == l1) { - if ('.' == d2[l2 - 1]) { - debug(28, 0) ("WARNING: %s is a subdomain of %s\n", d2, d1); - debug(28, 0) ("WARNING: This may break Splay tree searching\n"); - debug(28, 0) ("WARNING: You should remove '%s' from the ACL named '%s'\n", d2, AclMatchedName); - } - return -1; /* d1 < d2 */ - } - if (0 == l2) { - if ('.' == d1[l1 - 1]) { - debug(28, 0) ("WARNING: %s is a subdomain of %s\n", d1, d2); - debug(28, 0) ("WARNING: This may break Splay tree searching\n"); - debug(28, 0) ("WARNING: You should remove '%s' from the ACL named '%s'\n", d1, AclMatchedName); - } - return 1; /* d1 > d2 */ - } + const char *d1; + const char *d2; + int ret; + d1=b; + d2=a; + ret = aclHostDomainCompare(d1, d2); + if (ret != 0) { + d1=a; + d2=b; + ret = aclHostDomainCompare(d1, d2); + } + if (ret == 0) { + debug(28,0 ) ("WARNING: '%s' is a subdomain of '%s'\n", d1, d2); + debug(28, 0) ("WARNING: because of this '%s' is ignored to keep splay tree searching predictable\n", a); + debug(28, 0) ("WARNING: You should probably remove '%s' from the ACL named '%s'\n", d1, AclMatchedName); } - return (d1[l1] - d2[l2]); + return ret; } /* compare a host and a domain */ @@ -2250,6 +2300,11 @@ #endif #if USE_IDENT case ACL_IDENT: + return wordlistDup(a->data); + break; + case ACL_IDENT_REGEX: + return aclDumpRegexList(a->data); + break; #endif case ACL_PROXY_AUTH: return wordlistDup(a->data); @@ -2257,6 +2312,7 @@ case ACL_TIME: return aclDumpTimeSpecList(a->data); break; + case ACL_PROXY_AUTH_REGEX: case ACL_URL_REGEX: case ACL_URLPATH_REGEX: case ACL_BROWSER: @@ -2339,9 +2395,14 @@ * * NOTE: Linux code by David Luyer . * Original (BSD-specific) code no longer works. + * Solaris code by R. Gancarz */ +#ifdef _SQUID_SOLARIS_ +#include +#else #include +#endif #ifdef _SQUID_LINUX_ #include #include @@ -2416,10 +2477,10 @@ /***************/ /* aclMatchArp */ /***************/ -#ifdef _SQUID_LINUX_ static int aclMatchArp(void *dataptr, struct in_addr c) { +#if defined(_SQUID_LINUX_) struct arpreq arpReq; struct sockaddr_in ipAddr; unsigned char ifbuffer[sizeof(struct ifreq) * 64]; @@ -2533,6 +2594,48 @@ * exist on multiple interfaces? */ } +#elif defined(_SQUID_SOLARIS_) + struct arpreq arpReq; + struct sockaddr_in ipAddr; + unsigned char ifbuffer[sizeof(struct ifreq) * 64]; + struct ifconf ifc; + struct ifreq *ifr; + int offset; + splayNode **Top = dataptr; + /* + * Set up structures for ARP lookup with blank interface name + */ + ipAddr.sin_family = AF_INET; + ipAddr.sin_port = 0; + ipAddr.sin_addr = c; + memset(&arpReq, '\0', sizeof(arpReq)); + memcpy(&arpReq.arp_pa, &ipAddr, sizeof(struct sockaddr_in)); + /* Query ARP table */ + if (ioctl(HttpSockets[0], SIOCGARP, &arpReq) != -1) { + /* + * Solaris (at least 2.6/x86) does not use arp_ha.sa_family - + * it returns 00:00:00:00:00:00 for non-ethernet media + */ + if (arpReq.arp_ha.sa_data[0] == 0 && + arpReq.arp_ha.sa_data[1] == 0 && + arpReq.arp_ha.sa_data[2] == 0 && + arpReq.arp_ha.sa_data[3] == 0 && + arpReq.arp_ha.sa_data[4] == 0 && + arpReq.arp_ha.sa_data[5] == 0) + return 0; + debug(28, 4) ("Got address %02x:%02x:%02x:%02x:%02x:%02x\n", + arpReq.arp_ha.sa_data[0] & 0xff, arpReq.arp_ha.sa_data[1] & 0xff, + arpReq.arp_ha.sa_data[2] & 0xff, arpReq.arp_ha.sa_data[3] & 0xff, + arpReq.arp_ha.sa_data[4] & 0xff, arpReq.arp_ha.sa_data[5] & 0xff); + /* Do lookup */ + *Top = splay_splay(&arpReq.arp_ha.sa_data, *Top, aclArpCompare); + debug(28, 3) ("aclMatchArp: '%s' %s\n", + inet_ntoa(c), splayLastResult ? "NOT found" : "found"); + return (0 == splayLastResult); + } +#else + WRITE ME; +#endif /* * Address was not found on any interface */ @@ -2543,6 +2646,7 @@ static int aclArpCompare(const void *a, const void *b) { +#if defined(_SQUID_LINUX_) const unsigned short *d1 = a; const unsigned short *d2 = b; if (d1[0] != d2[0]) @@ -2551,22 +2655,28 @@ return (d1[1] > d2[1]) ? 1 : -1; if (d1[2] != d2[2]) return (d1[2] > d2[2]) ? 1 : -1; - return 0; -} +#elif defined(_SQUID_SOLARIS_) + const unsigned char *d1 = a; + const unsigned char *d2 = b; + if (d1[0] != d2[0]) + return (d1[0] > d2[0]) ? 1 : -1; + if (d1[1] != d2[1]) + return (d1[1] > d2[1]) ? 1 : -1; + if (d1[2] != d2[2]) + return (d1[2] > d2[2]) ? 1 : -1; + if (d1[3] != d2[3]) + return (d1[3] > d2[3]) ? 1 : -1; + if (d1[4] != d2[4]) + return (d1[4] > d2[4]) ? 1 : -1; + if (d1[5] != d2[5]) + return (d1[5] > d2[5]) ? 1 : -1; #else - -static int -aclMatchArp(void *dataptr, struct in_addr c) -{ - WRITE ME; -} - -static int -aclArpCompare(const void *data, splayNode * n) -{ WRITE ME; +#endif + return 0; } +#if UNUSED_CODE /********************************************************************** * This is from the pre-splay-tree code for BSD * I suspect the Linux approach will work on most O/S and be much --- squid/src/aiops.c Wed Feb 14 00:43:58 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,917 +0,0 @@ -/* - * $Id: aiops.c,v 1.1.1.3.12.1 2000/04/17 00:13:09 hno Exp $ - * - * DEBUG: section 43 AIOPS - * AUTHOR: Stewart Forster - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SCHED_H -#include -#endif - -#define RIDICULOUS_LENGTH 4096 - -#if defined(_SQUID_LINUX_) -/* Linux requires proper use of mutexes or it will segfault deep in the - * thread libraries. Observed on Alpha SMP Linux 2.2.10-ac12. - */ -#define USE_PROPER_MUTEX 1 -#endif - -enum _aio_thread_status { - _THREAD_STARTING = 0, - _THREAD_WAITING, - _THREAD_BUSY, - _THREAD_FAILED, - _THREAD_DONE -}; - -enum _aio_request_type { - _AIO_OP_NONE = 0, - _AIO_OP_OPEN, - _AIO_OP_READ, - _AIO_OP_WRITE, - _AIO_OP_CLOSE, - _AIO_OP_UNLINK, - _AIO_OP_OPENDIR, - _AIO_OP_STAT -}; - -typedef struct aio_request_t { - enum _aio_request_type request_type; - int cancelled; - char *path; - int oflag; - mode_t mode; - int fd; - char *bufferp; - char *tmpbufp; - int buflen; - off_t offset; - int whence; - int ret; - int err; - struct stat *tmpstatp; - struct stat *statp; - aio_result_t *resultp; - struct aio_request_t *next; -} aio_request_t; - - -typedef struct aio_thread_t { - pthread_t thread; - enum _aio_thread_status status; - pthread_mutex_t mutex; /* Mutex for testing condition variable */ - pthread_cond_t cond; /* Condition variable */ - struct aio_request_t *volatile req; /* set by main, cleared by thread */ - struct aio_request_t *processed_req; /* reminder to main */ - struct aio_thread_t *next; -} aio_thread_t; - - -int aio_cancel(aio_result_t *); -int aio_open(const char *, int, mode_t, aio_result_t *); -int aio_read(int, char *, int, off_t, int, aio_result_t *); -int aio_write(int, char *, int, off_t, int, aio_result_t *); -int aio_close(int, aio_result_t *); -int aio_unlink(const char *, aio_result_t *); -int aio_opendir(const char *, aio_result_t *); -aio_result_t *aio_poll_done(); -int aio_sync(void); - -static void aio_init(void); -static void aio_queue_request(aio_request_t *); -static void aio_process_request_queue(void); -static void aio_cleanup_request(aio_request_t *); -static void *aio_thread_loop(void *); -static void aio_do_open(aio_request_t *); -static void aio_do_read(aio_request_t *); -static void aio_do_write(aio_request_t *); -static void aio_do_close(aio_request_t *); -static void aio_do_stat(aio_request_t *); -static void aio_do_unlink(aio_request_t *); -#if AIO_OPENDIR -static void *aio_do_opendir(aio_request_t *); -#endif -static void aio_debug(aio_request_t *); -static void aio_poll_threads(void); - -static aio_thread_t *threads; -static int aio_initialised = 0; - -static int request_queue_len = 0; -static MemPool *aio_request_pool = NULL; -static aio_request_t *request_queue_head = NULL; -static aio_request_t *request_queue_tail = NULL; -static aio_request_t *request_done_head = NULL; -static aio_request_t *request_done_tail = NULL; -static aio_thread_t *wait_threads = NULL; -static aio_thread_t *busy_threads_head = NULL; -static aio_thread_t *busy_threads_tail = NULL; -static pthread_attr_t globattr; -static struct sched_param globsched; -static pthread_t main_thread; - -static void -aio_init(void) -{ - int i; - aio_thread_t *threadp; - - if (aio_initialised) - return; - - pthread_attr_init(&globattr); -#if HAVE_PTHREAD_ATTR_SETSCOPE -#if defined(_SQUID_SGI_) - /* - * Erik Hofman suggests PTHREAD_SCOPE_PROCESS - * instead of PTHREAD_SCOPE_SYSTEM, esp for IRIX. - */ - pthread_attr_setscope(&globattr, PTHREAD_SCOPE_PROCESS); -#else - pthread_attr_setscope(&globattr, PTHREAD_SCOPE_SYSTEM); -#endif -#endif - globsched.sched_priority = 1; - main_thread = pthread_self(); -#if HAVE_PTHREAD_SETSCHEDPARAM - pthread_setschedparam(main_thread, SCHED_OTHER, &globsched); -#endif - globsched.sched_priority = 2; -#if HAVE_PTHREAD_ATTR_SETSCHEDPARAM - pthread_attr_setschedparam(&globattr, &globsched); -#endif -#if defined(_SQUID_SGI_) - pthread_setconcurrency(NUMTHREADS + 1); -#endif - - /* Create threads and get them to sit in their wait loop */ - threads = xcalloc(NUMTHREADS, sizeof(aio_thread_t)); - - for (i = 0; i < NUMTHREADS; i++) { - threadp = &threads[i]; - threadp->status = _THREAD_STARTING; - if (pthread_mutex_init(&(threadp->mutex), NULL)) { - threadp->status = _THREAD_FAILED; - continue; - } - if (pthread_cond_init(&(threadp->cond), NULL)) { - threadp->status = _THREAD_FAILED; - continue; - } - threadp->req = NULL; - threadp->processed_req = NULL; - if (pthread_create(&threadp->thread, &globattr, aio_thread_loop, threadp)) { - fprintf(stderr, "Thread creation failed\n"); - threadp->status = _THREAD_FAILED; - continue; - } - threadp->next = wait_threads; - wait_threads = threadp; -#if AIO_PROPER_MUTEX - pthread_mutex_lock(&threadp->mutex); -#endif - } - - /* Create request pool */ - aio_request_pool = memPoolCreate("aio_request", sizeof(aio_request_t)); - - aio_initialised = 1; -} - - -static void * -aio_thread_loop(void *ptr) -{ - aio_thread_t *threadp = ptr; - aio_request_t *request; - sigset_t new; -#if !AIO_PROPER_MUTEX - struct timespec wait_time; -#endif - - /* - * Make sure to ignore signals which may possibly get sent to - * the parent squid thread. Causes havoc with mutex's and - * condition waits otherwise - */ - - sigemptyset(&new); - sigaddset(&new, SIGPIPE); - sigaddset(&new, SIGCHLD); -#ifdef _SQUID_LINUX_THREADS_ - sigaddset(&new, SIGQUIT); - sigaddset(&new, SIGTRAP); -#else - sigaddset(&new, SIGUSR1); - sigaddset(&new, SIGUSR2); -#endif - sigaddset(&new, SIGHUP); - sigaddset(&new, SIGTERM); - sigaddset(&new, SIGINT); - sigaddset(&new, SIGALRM); - pthread_sigmask(SIG_BLOCK, &new, NULL); - - pthread_mutex_lock(&threadp->mutex); - while (1) { -#if AIO_PROPER_MUTEX - while (threadp->req == NULL) { - threadp->status = _THREAD_WAITING; - pthread_cond_wait(&threadp->cond, &threadp->mutex); - } -#else - /* The timeout is used to unlock the race condition where - * ->req is set between the check and pthread_cond_wait. - * The thread steps it's own clock on each timeout, to avoid a CPU - * spin situation if the main thread is suspended (paging), and - * squid_curtime is not being updated timely. - */ - wait_time.tv_sec = squid_curtime + 1; /* little quicker first time */ - wait_time.tv_nsec = 0; - while (threadp->req == NULL) { - threadp->status = _THREAD_WAITING; - pthread_cond_timedwait(&threadp->cond, &threadp->mutex, &wait_time); - wait_time.tv_sec += 3; /* then wait 3 seconds between each check */ - } -#endif - request = threadp->req; - errno = 0; - if (!request->cancelled) { - switch (request->request_type) { - case _AIO_OP_OPEN: - aio_do_open(request); - break; - case _AIO_OP_READ: - aio_do_read(request); - break; - case _AIO_OP_WRITE: - aio_do_write(request); - break; - case _AIO_OP_CLOSE: - aio_do_close(request); - break; - case _AIO_OP_UNLINK: - aio_do_unlink(request); - break; -#if AIO_OPENDIR /* Opendir not implemented yet */ - case _AIO_OP_OPENDIR: - aio_do_opendir(request); - break; -#endif - case _AIO_OP_STAT: - aio_do_stat(request); - break; - default: - request->ret = -1; - request->err = EINVAL; - break; - } - } else { /* cancelled */ - request->ret = -1; - request->err = EINTR; - } - threadp->req = NULL; /* tells main thread that we are done */ - } /* while */ - return NULL; -} /* aio_thread_loop */ - -static void -aio_do_request(aio_request_t * requestp) -{ - if (wait_threads == NULL && busy_threads_head == NULL) { - fprintf(stderr, "PANIC: No threads to service requests with!\n"); - exit(-1); - } - aio_queue_request(requestp); -} /* aio_do_request */ - - -static void -aio_queue_request(aio_request_t * requestp) -{ - aio_request_t *rp; - static int last_warn = 0; - static int high_start = 0; - static int queue_high, queue_low; - int i; - - /* Mark it as not executed (failing result, no error) */ - requestp->ret = -1; - requestp->err = 0; - /* Queue it on the request queue */ - if (request_queue_head == NULL) { - request_queue_head = requestp; - request_queue_tail = requestp; - } else { - request_queue_tail->next = requestp; - request_queue_tail = requestp; - } - requestp->next = NULL; - request_queue_len += 1; - /* Poll done threads if needed */ - if (wait_threads == NULL) - aio_poll_threads(); - /* Kick it rolling */ - aio_process_request_queue(); - /* Warn if out of threads */ - if (request_queue_len > (NUMTHREADS >> 1)) { - if (high_start == 0) { - high_start = squid_curtime; - queue_high = request_queue_len; - queue_low = request_queue_len; - } - if (request_queue_len > queue_high) - queue_high = request_queue_len; - if (request_queue_len < queue_low) - queue_low = request_queue_len; - if (squid_curtime >= (last_warn + 15) && - squid_curtime >= (high_start + 3)) { - debug(43, 1) ("aio_queue_request: WARNING - Running out of I/O threads\n"); - debug(43, 2) ("aio_queue_request: Queue Length: current=%d, high=%d, low=%d, duration=%d\n", - request_queue_len, queue_high, queue_low, squid_curtime - high_start); - debug(43, 1) ("aio_queue_request: Perhaps you should increase NUMTHREADS\n"); - debug(43, 1) ("aio_queue_request: Or install more disks to share the load\n"); - debug(43, 3) ("aio_queue_request: First %d items on request queue\n", NUMTHREADS); - rp = request_queue_head; - for (i = 1; i <= NUMTHREADS; i++) { - switch (rp->request_type) { - case _AIO_OP_OPEN: - debug(43, 3) ("aio_queue_request: %d : open -> %s\n", i, rp->path); - break; - case _AIO_OP_READ: - debug(43, 3) ("aio_queue_request: %d : read -> FD = %d\n", i, rp->fd); - break; - case _AIO_OP_WRITE: - debug(43, 3) ("aio_queue_request: %d : write -> FD = %d\n", i, rp->fd); - break; - case _AIO_OP_CLOSE: - debug(43, 3) ("aio_queue_request: %d : close -> FD = %d\n", i, rp->fd); - break; - case _AIO_OP_UNLINK: - debug(43, 3) ("aio_queue_request: %d : unlink -> %s\n", i, rp->path); - break; - case _AIO_OP_STAT: - debug(43, 3) ("aio_queue_request: %d : stat -> %s\n", i, rp->path); - break; - default: - debug(43, 1) ("aio_queue_request: %d : Unimplemented request type: %d\n", i, rp->request_type); - break; - } - if ((rp = rp->next) == NULL) - break; - } - last_warn = squid_curtime; - } - } else { - high_start = 0; - } - if (request_queue_len > RIDICULOUS_LENGTH) { - debug(43, 0) ("aio_queue_request: Async request queue growing uncontrollably!\n"); - debug(43, 0) ("aio_queue_request: Syncing pending I/O operations.. (blocking)\n"); - aio_sync(); - debug(43, 0) ("aio_queue_request: Synced\n"); - } -} /* aio_queue_request */ - - -static void -aio_process_request_queue(void) -{ - aio_thread_t *threadp; - aio_request_t *requestp; - - for (;;) { - if (wait_threads == NULL || request_queue_head == NULL) - return; - - requestp = request_queue_head; - if ((request_queue_head = requestp->next) == NULL) - request_queue_tail = NULL; - requestp->next = NULL; - request_queue_len--; - - if (requestp->cancelled) { - aio_cleanup_request(requestp); - continue; - } - threadp = wait_threads; - wait_threads = threadp->next; - threadp->next = NULL; - - if (busy_threads_head != NULL) - busy_threads_tail->next = threadp; - else - busy_threads_head = threadp; - busy_threads_tail = threadp; - - threadp->status = _THREAD_BUSY; - threadp->req = threadp->processed_req = requestp; - pthread_cond_signal(&(threadp->cond)); -#if AIO_PROPER_MUTEX - pthread_mutex_unlock(&threadp->mutex); -#endif - } -} /* aio_process_request_queue */ - - -static void -aio_cleanup_request(aio_request_t * requestp) -{ - aio_result_t *resultp = requestp->resultp; - int cancelled = requestp->cancelled; - - /* Free allocated structures and copy data back to user space if the */ - /* request hasn't been cancelled */ - switch (requestp->request_type) { - case _AIO_OP_STAT: - if (!cancelled && requestp->ret == 0) - xmemcpy(requestp->statp, requestp->tmpstatp, sizeof(struct stat)); - xfree(requestp->tmpstatp); - case _AIO_OP_OPEN: - if (cancelled && requestp->ret >= 0) - /* The open() was cancelled but completed */ - close(requestp->ret); - xfree(requestp->path); - break; - case _AIO_OP_CLOSE: - if (cancelled && requestp->ret < 0) - /* The close() was cancelled and never got executed */ - close(requestp->fd); - break; - case _AIO_OP_UNLINK: - case _AIO_OP_OPENDIR: - xfree(requestp->path); - break; - case _AIO_OP_READ: - if (!cancelled && requestp->ret > 0) - xmemcpy(requestp->bufferp, requestp->tmpbufp, requestp->ret); - case _AIO_OP_WRITE: - xfree(requestp->tmpbufp); - break; - default: - break; - } - if (resultp != NULL && !cancelled) { - resultp->aio_return = requestp->ret; - resultp->aio_errno = requestp->err; - } - memPoolFree(aio_request_pool, requestp); -} /* aio_cleanup_request */ - - -int -aio_cancel(aio_result_t * resultp) -{ - aio_thread_t *threadp; - aio_request_t *requestp; - - for (threadp = busy_threads_head; threadp != NULL; threadp = threadp->next) - if (threadp->processed_req->resultp == resultp) { - threadp->processed_req->cancelled = 1; - threadp->processed_req->resultp = NULL; - return 0; - } - for (requestp = request_queue_head; requestp != NULL; requestp = requestp->next) - if (requestp->resultp == resultp) { - requestp->cancelled = 1; - requestp->resultp = NULL; - return 0; - } - for (requestp = request_done_head; requestp != NULL; requestp = requestp->next) - if (requestp->resultp == resultp) { - requestp->cancelled = 1; - requestp->resultp = NULL; - return 0; - } - return 1; -} /* aio_cancel */ - - -int -aio_open(const char *path, int oflag, mode_t mode, aio_result_t * resultp) -{ - aio_request_t *requestp; - int len; - - if (!aio_initialised) - aio_init(); - if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { - errno = ENOMEM; - return -1; - } - len = strlen(path) + 1; - if ((requestp->path = (char *) xmalloc(len)) == NULL) { - memPoolFree(aio_request_pool, requestp); - errno = ENOMEM; - return -1; - } - strncpy(requestp->path, path, len); - requestp->oflag = oflag; - requestp->mode = mode; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_OPEN; - requestp->cancelled = 0; - - aio_do_request(requestp); - return 0; -} - - -static void -aio_do_open(aio_request_t * requestp) -{ - requestp->ret = open(requestp->path, requestp->oflag, requestp->mode); - requestp->err = errno; -} - - -int -aio_read(int fd, char *bufp, int bufs, off_t offset, int whence, aio_result_t * resultp) -{ - aio_request_t *requestp; - - if (!aio_initialised) - aio_init(); - if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { - errno = ENOMEM; - return -1; - } - requestp->fd = fd; - requestp->bufferp = bufp; - if ((requestp->tmpbufp = (char *) xmalloc(bufs)) == NULL) { - memPoolFree(aio_request_pool, requestp); - errno = ENOMEM; - return -1; - } - requestp->buflen = bufs; - requestp->offset = offset; - requestp->whence = whence; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_READ; - requestp->cancelled = 0; - - aio_do_request(requestp); - return 0; -} - - -static void -aio_do_read(aio_request_t * requestp) -{ - lseek(requestp->fd, requestp->offset, requestp->whence); - requestp->ret = read(requestp->fd, requestp->tmpbufp, requestp->buflen); - requestp->err = errno; -} - - -int -aio_write(int fd, char *bufp, int bufs, off_t offset, int whence, aio_result_t * resultp) -{ - aio_request_t *requestp; - - if (!aio_initialised) - aio_init(); - if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { - errno = ENOMEM; - return -1; - } - requestp->fd = fd; - if ((requestp->tmpbufp = (char *) xmalloc(bufs)) == NULL) { - memPoolFree(aio_request_pool, requestp); - errno = ENOMEM; - return -1; - } - xmemcpy(requestp->tmpbufp, bufp, bufs); - requestp->buflen = bufs; - requestp->offset = offset; - requestp->whence = whence; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_WRITE; - requestp->cancelled = 0; - - aio_do_request(requestp); - return 0; -} - - -static void -aio_do_write(aio_request_t * requestp) -{ - requestp->ret = write(requestp->fd, requestp->tmpbufp, requestp->buflen); - requestp->err = errno; -} - - -int -aio_close(int fd, aio_result_t * resultp) -{ - aio_request_t *requestp; - - if (!aio_initialised) - aio_init(); - if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { - errno = ENOMEM; - return -1; - } - requestp->fd = fd; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_CLOSE; - requestp->cancelled = 0; - - aio_do_request(requestp); - return 0; -} - - -static void -aio_do_close(aio_request_t * requestp) -{ - requestp->ret = close(requestp->fd); - requestp->err = errno; -} - - -int -aio_stat(const char *path, struct stat *sb, aio_result_t * resultp) -{ - aio_request_t *requestp; - int len; - - if (!aio_initialised) - aio_init(); - if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { - errno = ENOMEM; - return -1; - } - len = strlen(path) + 1; - if ((requestp->path = (char *) xmalloc(len)) == NULL) { - memPoolFree(aio_request_pool, requestp); - errno = ENOMEM; - return -1; - } - strncpy(requestp->path, path, len); - requestp->statp = sb; - if ((requestp->tmpstatp = (struct stat *) xmalloc(sizeof(struct stat))) == NULL) { - xfree(requestp->path); - memPoolFree(aio_request_pool, requestp); - errno = ENOMEM; - return -1; - } - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_STAT; - requestp->cancelled = 0; - - aio_do_request(requestp); - return 0; -} - - -static void -aio_do_stat(aio_request_t * requestp) -{ - requestp->ret = stat(requestp->path, requestp->tmpstatp); - requestp->err = errno; -} - - -int -aio_unlink(const char *path, aio_result_t * resultp) -{ - aio_request_t *requestp; - int len; - - if (!aio_initialised) - aio_init(); - if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { - errno = ENOMEM; - return -1; - } - len = strlen(path) + 1; - if ((requestp->path = (char *) xmalloc(len)) == NULL) { - memPoolFree(aio_request_pool, requestp); - errno = ENOMEM; - return -1; - } - strncpy(requestp->path, path, len); - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_UNLINK; - requestp->cancelled = 0; - - aio_do_request(requestp); - return 0; -} - - -static void -aio_do_unlink(aio_request_t * requestp) -{ - requestp->ret = unlink(requestp->path); - requestp->err = errno; -} - - -#if AIO_OPENDIR -/* XXX aio_opendir NOT implemented? */ - -int -aio_opendir(const char *path, aio_result_t * resultp) -{ - aio_request_t *requestp; - int len; - - if (!aio_initialised) - aio_init(); - if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { - errno = ENOMEM; - return -1; - } - return -1; -} - -static void -aio_do_opendir(aio_request_t * requestp) -{ - /* NOT IMPLEMENTED */ -} - -#endif - - -void -aio_poll_threads(void) -{ - aio_thread_t *prev; - aio_thread_t *threadp; - aio_request_t *requestp; - - do { /* while found completed thread */ - prev = NULL; - threadp = busy_threads_head; - while (threadp) { - debug(43, 9) ("aio_poll_threads: %p: request type %d -> status %d\n", - threadp, - threadp->processed_req->request_type, - threadp->status); -#if AIO_PROPER_MUTEX - if (threadp->req == NULL) - if (pthread_mutex_trylock(&threadp->mutex) == 0) - break; -#else - if (threadp->req == NULL) - break; -#endif - prev = threadp; - threadp = threadp->next; - } - if (threadp == NULL) - break; - - if (prev == NULL) - busy_threads_head = busy_threads_head->next; - else - prev->next = threadp->next; - - if (busy_threads_tail == threadp) - busy_threads_tail = prev; - - requestp = threadp->processed_req; - threadp->processed_req = NULL; - - threadp->next = wait_threads; - wait_threads = threadp; - - if (request_done_tail != NULL) - request_done_tail->next = requestp; - else - request_done_head = requestp; - request_done_tail = requestp; - } while (threadp); - - aio_process_request_queue(); -} /* aio_poll_threads */ - -aio_result_t * -aio_poll_done(void) -{ - aio_request_t *requestp; - aio_result_t *resultp; - int cancelled; - - AIO_REPOLL: - aio_poll_threads(); - if (request_done_head == NULL) { - return NULL; - } - requestp = request_done_head; - request_done_head = requestp->next; - if (!request_done_head) - request_done_tail = NULL; - - resultp = requestp->resultp; - cancelled = requestp->cancelled; - aio_debug(requestp); - debug(43, 5) ("DONE: %d -> %d\n", requestp->ret, requestp->err); - aio_cleanup_request(requestp); - if (cancelled) - goto AIO_REPOLL; - return resultp; -} /* aio_poll_done */ - -int -aio_operations_pending(void) -{ - return request_queue_len + (request_done_head != NULL) + (busy_threads_head != NULL); -} - -int -aio_overloaded(void) -{ - static time_t last_warn = 0; - if (aio_operations_pending() > RIDICULOUS_LENGTH / 4) { - if (squid_curtime >= (last_warn + 15)) { - debug(43, 0) ("Warning: Async-IO overloaded\n"); - last_warn = squid_curtime; - } - return 1; - } - return 0; -} - -int -aio_sync(void) -{ - int loop_count = 0; - do { - aio_poll_threads(); - assert(++loop_count < 10); - } while (request_queue_len > 0); - return aio_operations_pending(); -} - -int -aio_get_queue_len(void) -{ - return request_queue_len; -} - -static void -aio_debug(aio_request_t * requestp) -{ - switch (requestp->request_type) { - case _AIO_OP_OPEN: - debug(43, 5) ("OPEN of %s to FD %d\n", requestp->path, requestp->ret); - break; - case _AIO_OP_READ: - debug(43, 5) ("READ on fd: %d\n", requestp->fd); - break; - case _AIO_OP_WRITE: - debug(43, 5) ("WRITE on fd: %d\n", requestp->fd); - break; - case _AIO_OP_CLOSE: - debug(43, 5) ("CLOSE of fd: %d\n", requestp->fd); - break; - case _AIO_OP_UNLINK: - debug(43, 5) ("UNLINK of %s\n", requestp->path); - break; - default: - break; - } -} --- squid/src/async_io.c Wed Feb 14 00:43:58 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,415 +0,0 @@ - -/* - * $Id: async_io.c,v 1.1.1.3.4.1.2.1 2000/04/17 00:13:09 hno Exp $ - * - * DEBUG: section 32 Asynchronous Disk I/O - * AUTHOR: Pete Bentley - * AUTHOR: Stewart Forster - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -#if USE_ASYNC_IO - -#define _AIO_OPEN 0 -#define _AIO_READ 1 -#define _AIO_WRITE 2 -#define _AIO_CLOSE 3 -#define _AIO_UNLINK 4 -#define _AIO_OPENDIR 5 -#define _AIO_STAT 6 - -typedef struct aio_ctrl_t { - struct aio_ctrl_t *next; - int fd; - int operation; - AIOCB *done_handler; - void *done_handler_data; - aio_result_t result; -} aio_ctrl_t; - -struct { - int open; - int close; - int cancel; - int write; - int read; - int stat; - int unlink; - int check_callback; -} aio_counts; - -typedef struct aio_unlinkq_t { - char *path; - struct aio_unlinkq_t *next; -} aio_unlinkq_t; - -static aio_ctrl_t *used_list = NULL; -static int initialised = 0; -static OBJH aioStats; -static MemPool *aio_ctrl_pool; -static void aioFDWasClosed(int fd); - -static void -aioFDWasClosed(int fd) -{ - if (fd_table[fd].flags.closing) - fd_close(fd); -} - -void -aioInit(void) -{ - if (initialised) - return; - aio_ctrl_pool = memPoolCreate("aio_ctrl", sizeof(aio_ctrl_t)); - cachemgrRegister("aio_counts", "Async IO Function Counters", - aioStats, 0, 1); - initialised = 1; -} - -void -aioOpen(const char *path, int oflag, mode_t mode, AIOCB * callback, void *callback_data) -{ - aio_ctrl_t *ctrlp; - int ret; - - assert(initialised); - aio_counts.open++; - ctrlp = memPoolAlloc(aio_ctrl_pool); - ctrlp->fd = -2; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_OPEN; - cbdataLock(callback_data); - if (aio_open(path, oflag, mode, &ctrlp->result) < 0) { - ret = open(path, oflag, mode); - if (callback) - (callback) (ctrlp->fd, callback_data, ret, errno); - cbdataUnlock(callback_data); - memPoolFree(aio_ctrl_pool, ctrlp); - return; - } - ctrlp->next = used_list; - used_list = ctrlp; - return; -} - -void -aioClose(int fd) -{ - aio_ctrl_t *ctrlp; - - assert(initialised); - aio_counts.close++; - aioCancel(fd); - ctrlp = memPoolAlloc(aio_ctrl_pool); - ctrlp->fd = fd; - ctrlp->done_handler = NULL; - ctrlp->done_handler_data = NULL; - ctrlp->operation = _AIO_CLOSE; - if (aio_close(fd, &ctrlp->result) < 0) { - close(fd); /* Can't create thread - do a normal close */ - memPoolFree(aio_ctrl_pool, ctrlp); - aioFDWasClosed(fd); - return; - } - ctrlp->next = used_list; - used_list = ctrlp; - return; -} - -void -aioCancel(int fd) -{ - aio_ctrl_t *curr; - aio_ctrl_t *prev; - aio_ctrl_t *next; - AIOCB *done_handler; - void *their_data; - - assert(initialised); - aio_counts.cancel++; - prev = NULL; - curr = used_list; - for (curr = used_list;; curr = next) { - while (curr != NULL) { - if (curr->fd == fd) - break; - prev = curr; - curr = curr->next; - } - if (curr == NULL) - break; - - aio_cancel(&curr->result); - - if ((done_handler = curr->done_handler)) { - their_data = curr->done_handler_data; - curr->done_handler = NULL; - curr->done_handler_data = NULL; - debug(32, 2) ("this be aioCancel\n"); - if (cbdataValid(their_data)) - done_handler(fd, their_data, -2, -2); - cbdataUnlock(their_data); - } - next = curr->next; - if (prev == NULL) - used_list = next; - else - prev->next = next; - - memPoolFree(aio_ctrl_pool, curr); - } -} - - -void -aioWrite(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func) -{ - aio_ctrl_t *ctrlp; - int seekmode; - - assert(initialised); - aio_counts.write++; - for (ctrlp = used_list; ctrlp != NULL; ctrlp = ctrlp->next) - if (ctrlp->fd == fd) - break; - if (ctrlp != NULL) { - debug(32, 2) ("aioWrite: EWOULDBLOCK\n"); - errno = EWOULDBLOCK; - if (callback) - (callback) (fd, callback_data, -1, errno); - free_func(bufp); - return; - } - ctrlp = memPoolAlloc(aio_ctrl_pool); - ctrlp->fd = fd; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_WRITE; - if (offset >= 0) - seekmode = SEEK_SET; - else { - seekmode = SEEK_END; - offset = 0; - } - cbdataLock(callback_data); - if (aio_write(fd, bufp, len, offset, seekmode, &ctrlp->result) < 0) { - if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL) - errno = EWOULDBLOCK; - if (callback) - (callback) (fd, callback_data, -1, errno); - cbdataUnlock(callback_data); - memPoolFree(aio_ctrl_pool, ctrlp); - } else { - ctrlp->next = used_list; - used_list = ctrlp; - } - /* - * aio_write copies the buffer so we can free it here - */ - free_func(bufp); -} /* aioWrite */ - - -void -aioRead(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data) -{ - aio_ctrl_t *ctrlp; - int seekmode; - - assert(initialised); - aio_counts.read++; - for (ctrlp = used_list; ctrlp != NULL; ctrlp = ctrlp->next) - if (ctrlp->fd == fd) - break; - if (ctrlp != NULL) { - errno = EWOULDBLOCK; - if (callback) - (callback) (fd, callback_data, -1, errno); - return; - } - ctrlp = memPoolAlloc(aio_ctrl_pool); - ctrlp->fd = fd; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_READ; - if (offset >= 0) - seekmode = SEEK_SET; - else { - seekmode = SEEK_CUR; - offset = 0; - } - cbdataLock(callback_data); - if (aio_read(fd, bufp, len, offset, seekmode, &ctrlp->result) < 0) { - if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL) - errno = EWOULDBLOCK; - if (callback) - (callback) (fd, callback_data, -1, errno); - cbdataUnlock(callback_data); - memPoolFree(aio_ctrl_pool, ctrlp); - return; - } - ctrlp->next = used_list; - used_list = ctrlp; - return; -} /* aioRead */ - -void -aioStat(char *path, struct stat *sb, AIOCB * callback, void *callback_data) -{ - aio_ctrl_t *ctrlp; - - assert(initialised); - aio_counts.stat++; - ctrlp = memPoolAlloc(aio_ctrl_pool); - ctrlp->fd = -2; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_STAT; - cbdataLock(callback_data); - if (aio_stat(path, sb, &ctrlp->result) < 0) { - if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL) - errno = EWOULDBLOCK; - if (callback) - (callback) (ctrlp->fd, callback_data, -1, errno); - cbdataUnlock(callback_data); - memPoolFree(aio_ctrl_pool, ctrlp); - return; - } - ctrlp->next = used_list; - used_list = ctrlp; - return; -} /* aioStat */ - -void -aioUnlink(const char *pathname, AIOCB * callback, void *callback_data) -{ - aio_ctrl_t *ctrlp; - char *path; - assert(initialised); - aio_counts.unlink++; - ctrlp = memPoolAlloc(aio_ctrl_pool); - ctrlp->fd = -2; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_UNLINK; - path = xstrdup(pathname); - cbdataLock(callback_data); - if (aio_unlink(path, &ctrlp->result) < 0) { - int ret = unlink(path); - (callback) (ctrlp->fd, callback_data, ret, errno); - cbdataUnlock(callback_data); - memPoolFree(aio_ctrl_pool, ctrlp); - xfree(path); - return; - } - ctrlp->next = used_list; - used_list = ctrlp; - xfree(path); -} /* aioUnlink */ - - -void -aioCheckCallbacks(void) -{ - aio_result_t *resultp; - aio_ctrl_t *ctrlp; - aio_ctrl_t *prev; - AIOCB *done_handler; - void *their_data; - - assert(initialised); - aio_counts.check_callback++; - for (;;) { - if ((resultp = aio_poll_done()) == NULL) - break; - prev = NULL; - for (ctrlp = used_list; ctrlp != NULL; prev = ctrlp, ctrlp = ctrlp->next) - if (&ctrlp->result == resultp) - break; - if (ctrlp == NULL) - continue; - if (prev == NULL) - used_list = ctrlp->next; - else - prev->next = ctrlp->next; - if ((done_handler = ctrlp->done_handler)) { - their_data = ctrlp->done_handler_data; - ctrlp->done_handler = NULL; - ctrlp->done_handler_data = NULL; - if (cbdataValid(their_data)) - done_handler(ctrlp->fd, their_data, - ctrlp->result.aio_return, ctrlp->result.aio_errno); - cbdataUnlock(their_data); - } - if (ctrlp->operation == _AIO_CLOSE) - aioFDWasClosed(ctrlp->fd); - memPoolFree(aio_ctrl_pool, ctrlp); - } -} - -void -aioStats(StoreEntry * sentry) -{ - storeAppendPrintf(sentry, "ASYNC IO Counters:\n"); - storeAppendPrintf(sentry, "open\t%d\n", aio_counts.open); - storeAppendPrintf(sentry, "close\t%d\n", aio_counts.close); - storeAppendPrintf(sentry, "cancel\t%d\n", aio_counts.cancel); - storeAppendPrintf(sentry, "write\t%d\n", aio_counts.write); - storeAppendPrintf(sentry, "read\t%d\n", aio_counts.read); - storeAppendPrintf(sentry, "stat\t%d\n", aio_counts.stat); - storeAppendPrintf(sentry, "unlink\t%d\n", aio_counts.unlink); - storeAppendPrintf(sentry, "check_callback\t%d\n", aio_counts.check_callback); - storeAppendPrintf(sentry, "queue\t%d\n", aio_get_queue_len()); -} - -/* Flush all pending I/O */ -void -aioSync(void) -{ - if (!initialised) - return; /* nothing to do then */ - /* Flush all pending operations */ - debug(32, 1) ("aioSync: flushing pending I/O operations\n"); - do { - aioCheckCallbacks(); - } while (aio_sync()); - debug(32, 1) ("aioSync: done\n"); -} - -int -aioQueueSize(void) -{ - return memPoolInUseCount(aio_ctrl_pool); -} - -#endif /* USE_ASYNC_IO */ Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.1.1.3.4.1.2.1 retrieving revision 1.1.1.3.4.1.2.2 diff -u -r1.1.1.3.4.1.2.1 -r1.1.1.3.4.1.2.2 --- squid/src/cache_cf.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.4.1.2.1 +++ squid/src/cache_cf.c 3 May 2000 19:18:12 -0000 1.1.1.3.4.1.2.2 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.1.1.3.4.1.2.1 2000/04/17 00:13:09 hno Exp $ + * $Id: cache_cf.c,v 1.1.1.3.4.1.2.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -58,6 +58,7 @@ static int http_header_first; static int http_header_allowed = 0; +static void update_maxobjsize(void); static void configDoConfigure(void); static void parse_refreshpattern(refresh_t **); static int parseTimeUnits(const char *unit); @@ -100,7 +101,7 @@ *list = NULL; } -wordlist * +const char * wordlistAdd(wordlist ** list, const char *key) { while (*list) @@ -108,7 +109,28 @@ *list = memAllocate(MEM_WORDLIST); (*list)->key = xstrdup(key); (*list)->next = NULL; - return *list; + return (*list)->key; +} + +void +wordlistJoin(wordlist ** list, wordlist **wl) +{ + while (*list) + list = &(*list)->next; + *list = *wl; + *wl = NULL; +} + +void +wordlistAddWl(wordlist ** list, wordlist *wl) +{ + while (*list) + list = &(*list)->next; + for(;wl;wl=wl->next, list = &(*list)->next) { + *list = memAllocate(MEM_WORDLIST); + (*list)->key = xstrdup(wl->key); + (*list)->next = NULL; + } } void @@ -171,6 +193,19 @@ return i; } +static void +update_maxobjsize(void) +{ + int i; + size_t ms = -1; + + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + if (Config.cacheSwap.swapDirs[i].max_objsize > ms) + ms = Config.cacheSwap.swapDirs[i].max_objsize; + } + store_maxobjsize = ms; +} + int parseConfigFile(const char *file_name) { @@ -237,7 +272,7 @@ Config.Announce.period = 86400 * 365; /* one year */ Config.onoff.announce = 0; } -#if USE_DNSSERVER +#if USE_DNSSERVERS if (Config.dnsChildren < 1) fatal("No dnsservers allocated"); if (Config.dnsChildren > DefaultDnsChildrenMax) { @@ -322,7 +357,7 @@ } #endif requirePathnameExists("MIME Config Table", Config.mimeTablePathname); -#if USE_DNSSERVER +#if USE_DNSSERVERS requirePathnameExists("cache_dns_program", Config.Program.dnsserver); #endif requirePathnameExists("unlinkd_program", Config.Program.unlinkd); @@ -779,16 +814,7 @@ int i; for (i = 0; i < swap.n_configured; i++) { s = swap.swapDirs + i; - switch (s->type) { - case SWAPDIR_UFS: - case SWAPDIR_ASYNCUFS: - storeUfsDirDump(entry, name, s); - break; - default: - debug(0, 0) ("dump_cachedir doesn't know about type %d\n", - (int) s->type); - break; - } + s->dump(entry, name, s); } } @@ -821,25 +847,84 @@ } } +static int +find_fstype(char *type) +{ + int i; + for (i = 0; storefs_list[i].typestr != NULL; i++) { + if (strcasecmp(type, storefs_list[i].typestr) == 0) { + return i; + } + } + return (-1); +} + static void parse_cachedir(cacheSwap * swap) { char *type_str; + char *path_str; + SwapDir *sd; + int i; + int fs; + size_t maxobjsize; + if ((type_str = strtok(NULL, w_space)) == NULL) self_destruct(); - if (0 == strcasecmp(type_str, "ufs")) { - storeUfsDirParse(swap); -#if USE_ASYNC_IO - } else if (0 == strcasecmp(type_str, "asyncufs")) { - storeAufsDirParse(swap); -#endif -#if USE_DISKD - } else if (0 == strcasecmp(type_str, "diskd")) { - storeDiskdDirParse(swap); -#endif - } else { - fatalf("Unknown cache_dir type '%s'\n", type_str); + + maxobjsize = (size_t)GetInteger(); + + if ((path_str = strtok(NULL, w_space)) == NULL) + self_destruct(); + + /* + * This bit of code is a little strange. + * See, if we find a path and type match for a given line, then + * as long as we're reconfiguring, we can just call its reconfigure + * function. No harm there. + * + * Trouble is, if we find a path match, but not a type match, we have + * a dilemma - we could gracefully shut down the fs, kill it, and + * create a new one of a new type in its place, BUT at this stage the + * fs is meant to be the *NEW* one, and so things go very strange. :-) + * + * So, we'll assume the person isn't going to change the fs type for now, + * and XXX later on we will make sure that its picked up. + * + * (moving around cache_dir lines will be looked at later in a little + * more sane detail..) + */ + + for (i = 0; i < swap->n_configured; i++) { + if (0 == strcasecmp(path_str, swap->swapDirs[i].path)) { + /* This is a little weird, you'll appreciate it later */ + fs = find_fstype(type_str); + if (fs < 0) { + fatalf("Unknown cache_dir type '%s'\n", type_str); + } + sd = swap->swapDirs + i; + storefs_list[fs].reconfigurefunc(sd, i, path_str); + sd->max_objsize = maxobjsize; + update_maxobjsize(); + return; + } } + + fs = find_fstype(type_str); + if (fs < 0) { + /* If we get here, we didn't find a matching cache_dir type */ + fatalf("Unknown cache_dir type '%s'\n", type_str); + } + + allocate_new_swapdir(swap); + sd = swap->swapDirs + swap->n_configured; + storefs_list[fs].parsefunc(sd, swap->n_configured, path_str); + /* XXX should we dupe the string here, in case it gets trodden on? */ + sd->type = storefs_list[fs].typestr; + sd->max_objsize = maxobjsize; + swap->n_configured++; + /* Update the max object size */ + update_maxobjsize(); } static void @@ -852,18 +937,8 @@ return; for (i = 0; i < swap->n_configured; i++) { s = swap->swapDirs + i; - switch (s->type) { - case SWAPDIR_UFS: - case SWAPDIR_ASYNCUFS: - storeUfsDirFree(s); - break; - default: - debug(0, 0) ("dump_cachedir doesn't know about type %d\n", - (int) s->type); - break; - } + s->freefs(s); xfree(s->path); - filemapFreeMemory(s->map); } safe_free(swap->swapDirs); swap->swapDirs = NULL; @@ -1005,6 +1080,8 @@ } else if (!strncasecmp(token, "digest-url=", 11)) { p->digest_url = xstrdup(token + 11); #endif + } else if (!strcasecmp(token, "allow-miss")) { + p->options.allow_miss = 1; } else { debug(3, 0) ("parse_peer: token='%s'\n", token); self_destruct(); @@ -1014,6 +1091,7 @@ p->weight = 1; p->icp.version = ICP_VERSION_CURRENT; p->tcp_up = PEER_TCP_MAGIC_COUNT; + p->test_fd = -1; #if USE_CARP if (p->carp.load_factor) { /* calculate this peers hash for use in CARP */ @@ -1034,6 +1112,7 @@ head = &(*head)->next; *head = p; Config.npeers++; + peerClearRR(p); } static void @@ -1412,6 +1491,12 @@ safe_free(pattern); } +static int +check_null_refreshpattern(refresh_t *data) +{ + return data != NULL; +} + static void free_refreshpattern(refresh_t ** head) { Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.1.1.3.4.1.2.5 retrieving revision 1.1.1.3.4.1.2.6 diff -u -r1.1.1.3.4.1.2.5 -r1.1.1.3.4.1.2.6 --- squid/src/cf.data.pre 17 Apr 2000 13:42:06 -0000 1.1.1.3.4.1.2.5 +++ squid/src/cf.data.pre 3 May 2000 19:18:12 -0000 1.1.1.3.4.1.2.6 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.1.1.3.4.1.2.5 2000/04/17 13:42:06 asd Exp $ +# $Id: cf.data.pre,v 1.1.1.3.4.1.2.6 2000/05/03 19:18:12 hno Exp $ # # # SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -82,8 +82,6 @@ address, however. You may specify multiple socket addresses on multiple lines. - -http_port 3128 DOC_END @@ -95,8 +93,6 @@ The port number where Squid sends and receives ICP queries to and from neighbor caches. Default is 3130. To disable use "0". May be overridden with -u on the command line. - -icp_port 3130 DOC_END NAME: htcp_port @@ -111,7 +107,6 @@ To enable this option, you must use --enable-htcp with the configure script. -htcp_port 4827 DOC_END @@ -140,8 +135,6 @@ Usage: mcast_groups 239.128.16.128 224.0.1.20 By default, Squid doesn't listen on any multicast groups. - -mcast_groups 239.128.16.128 DOC_END @@ -174,16 +167,20 @@ The default behavior is to not bind to any specific address. + A *_incoming_address value of 0.0.0.0 indicates that Squid should + listen on all available interfaces. + + If udp_outgoing_address is set to 255.255.255.255 (the default) + then it will use the same socket as udp_incoming_address. Only + change this if you want to have ICP queries sent using another + address than where this Squid listens for ICP queries from other + caches. + NOTE, udp_incoming_address and udp_outgoing_address can not - have the same value (unless it is 0.0.0.0) since they both use - port 3130. + have the same value since they both use port 3130. NOTE, tcp_incoming_address has been removed. You can now specify IP addresses on the 'http_port' line. - -tcp_outgoing_address 0.0.0.0 -udp_incoming_address 0.0.0.0 -udp_outgoing_address 0.0.0.0 DOC_END COMMENT_START @@ -198,7 +195,7 @@ DOC_START To specify other caches in a hierarchy, use the format: - hostname type http_port icp_port + cache_peer hostname type http_port icp_port For example, @@ -234,6 +231,7 @@ login=user:password connect-timeout=nn digest-url=url + allow-miss use 'proxy-only' to specify that objects fetched from this cache should not be saved locally. @@ -291,12 +289,18 @@ the specified URL rather than the Squid default location. - NOTE: non-ICP neighbors must be specified as 'parent'. + use 'allow-miss' to disable Squid's use of only-if-cached + when forwarding requests to siblings. This is primarily + useful when icp_hit_stale is used by the sibling. To + extensive use of this option may result in forwarding + loops, and you should avoid having two-way peerings + with this option. (for example to deny peer usage on + requests from peer by denying cache_peer_access if the + source is a peer) -cache_peer hostname type 3128 3130 + NOTE: non-ICP neighbors must be specified as 'parent'. DOC_END - NAME: cache_peer_domain cache_host_domain TYPE: hostdomain DEFAULT: none @@ -364,8 +368,6 @@ timeout (the old default), you would write: icp_query_timeout 2000 - -icp_query_timeout 0 DOC_END NAME: maximum_icp_query_timeout @@ -382,7 +384,6 @@ If 'icp_query_timeout' is set to zero, then this value is ignored. -maximum_icp_query_timeout 2000 DOC_END NAME: mcast_icp_query_timeout @@ -396,8 +397,6 @@ address. This value specifies how long Squid should wait to count all the replies. The default is 2000 msec, or 2 seconds. - -mcast_icp_query_timeout 2000 DOC_END NAME: dead_peer_timeout @@ -420,15 +419,12 @@ your time between requests is greater than this timeout, you will see a lot of requests sent DIRECT to origin servers instead of to your parents. - -dead_peer_timeout 10 seconds DOC_END NAME: hierarchy_stoplist TYPE: wordlist DEFAULT: none -DEFAULT_IF_NONE: cgi-bin ? LOC: Config.hierarchy_stoplist DOC_START A list of words which, if found in a URL, cause the object to @@ -436,9 +432,10 @@ to not query neighbor caches for certain objects. You may list this option multiple times. - The default is to directly fetch URLs containing 'cgi-bin' or '?'. - +NOCOMMENT_START +#We recommend you to use at least the following line. hierarchy_stoplist cgi-bin ? +NOCOMMENT_END DOC_END @@ -454,11 +451,11 @@ You must use the word 'DENY' to indicate the ACL names which should NOT be cached. - There is no default. We recommend you uncomment the following - two lines. - +NOCOMMENT_START +#We recommend you to use the following two lines. acl QUERY urlpath_regex cgi-bin \? no_cache deny QUERY +NOCOMMENT_END DOC_END @@ -503,10 +500,6 @@ decreases, blocks will be freed until the high-water mark is reached. Thereafter, blocks will be used to store hot objects. - - The default is 8 Megabytes. - -cache_mem 8 MB DOC_END @@ -534,9 +527,6 @@ Defaults are 90% and 95%. If you have a large cache, 5% could be hundreds of MB. If this is the case you may wish to set these numbers closer together. - -cache_swap_low 90 -cache_swap_high 95 DOC_END NAME: maximum_object_size @@ -555,8 +545,6 @@ NOTE: if using the LFUDA replacement policy you should increase this value to maximize the byte hit rate improvement of LFUDA! See replacement_policy below for a discussion of this policy. - -maximum_object_size 4096 KB DOC_END @@ -581,10 +569,6 @@ LOC: Config.ipcache.high DOC_START The size, low-, and high-water marks for the IP cache. - -ipcache_size 1024 -ipcache_low 90 -ipcache_high 95 DOC_END NAME: fqdncache_size @@ -594,7 +578,6 @@ LOC: Config.fqdncache.size DOC_START Maximum number of FQDN cache entries. -fqdncache_size 1024 DOC_END COMMENT_START @@ -605,12 +588,12 @@ NAME: cache_dir TYPE: cachedir DEFAULT: none -DEFAULT_IF_NONE: ufs @DEFAULT_SWAP_DIR@ 100 16 256 +DEFAULT_IF_NONE: ufs -1 @DEFAULT_SWAP_DIR@ 100 16 256 LOC: Config.cacheSwap DOC_START Usage: - cache_dir Type Directory-Name Mbytes Level-1 Level2 + cache_dir Type Maxobjsize Directory-Name Mbytes Level-1 Level2 You can specify multiple cache_dir lines to spread the cache among different disk partitions. @@ -621,6 +604,10 @@ want to try "asyncufs" as the type. Async IO support may be buggy, however, so beware. + Maxobjsize refers to the max object size this storedir supports. + It is used to initially choose the storedir to dump the object. + -1 means 'any size'. + 'Directory' is a top-level directory where cache swap files will be stored. If you want to use an entire disk for caching, then this can be the mount-point directory. @@ -640,8 +627,6 @@ 'Level-2' is the number of second-level subdirectories which will be created under each first-level directory. The default is 256. - -cache_dir ufs @DEFAULT_SWAP_DIR@ 100 16 256 DOC_END @@ -652,8 +637,6 @@ DOC_START Logs the client request activity. Contains an entry for every HTTP and ICP queries received. - -cache_access_log @DEFAULT_ACCESS_LOG@ DOC_END @@ -665,8 +648,6 @@ Cache logging file. This is where general information about your cache's behavior goes. You can increase the amount of data logged to this file with the "debug_options" tag below. - -cache_log @DEFAULT_CACHE_LOG@ DOC_END @@ -680,8 +661,6 @@ saved and for how long. To disable, enter "none". There are not really utilities to analyze this data, so you can safely disable it. - -cache_store_log @DEFAULT_STORE_LOG@ DOC_END @@ -692,14 +671,19 @@ DOC_START Location for the cache "swap.log." This log file holds the metadata of objects saved on disk. It is used to rebuild the - cache during startup. Normally this file resides in the first + cache during startup. Normally this file resides in each 'cache_dir' directory, but you may specify an alternate pathname here. Note you must give a full filename, not just a directory. Since this is the index for the whole object list you CANNOT periodically rotate it! - If you have more than one 'cache_dir', these swap logs will - have names such as: + If %s can be used in the file name then it will be replaced with a + a representation of the cache_dir name where each / is replaced + with '.'. This is needed to allow adding/removing cache_dir + lines when cache_swap_log is being used. + + If have more than one 'cache_dir', and %s is not used in the name + then these swap logs will have names such as: cache_swap_log.00 cache_swap_log.01 @@ -712,8 +696,6 @@ the correct 'cache_dir' entry (unless you manually rename them). We recommend that you do NOT use this option. It is better to keep these log files in each 'cache_dir' directory. - -cache_swap_log DOC_END @@ -728,10 +710,18 @@ emulate_httpd_log to 'off' or 'on'. The default is to use the native log format since it includes useful information that Squid-specific log analyzers use. - -emulate_httpd_log off DOC_END +NAME: log_ip_on_direct +COMMENT: on|off +TYPE: onoff +DEFAULT: on +LOC: Config.onoff.log_ip_on_direct +DOC_START + Log the destination IP address in the hierarchy log tag when going + direct. Earlier Squid versions logged the hostname here. If you + prefer the old way set this to off. +DOC_END NAME: mime_table TYPE: string @@ -741,8 +731,6 @@ Pathname to Squid's MIME table. You shouldn't need to change this, but the default file contains examples and formatting information if you do. - -mime_table @DEFAULT_MIME_TABLE@ DOC_END @@ -757,8 +745,6 @@ safely and will appear as two bracketed fields at the end of the access log (for either the native or httpd-emulated log formats). To enable this logging set log_mime_hdrs to 'on'. - -log_mime_hdrs off DOC_END @@ -766,13 +752,11 @@ TYPE: string LOC: Config.Log.useragent DEFAULT: none +IFDEF: USE_USERAGENT_LOG DOC_START - If configured with the "--enable-useragent_log" configure - option, Squid will write the User-Agent field from HTTP - requests to the filename specified here. By default - useragent_log is disabled. - -useragent_log none + Squid will write the User-Agent field from HTTP requests + to the filename specified here. By default useragent_log + is disabled. DOC_END @@ -782,8 +766,6 @@ LOC: Config.pidFilename DOC_START A filename to write the process-id to. To disable, enter "none". - -pid_filename @DEFAULT_PID_FILE@ DOC_END @@ -798,8 +780,6 @@ log file, so be careful. The magic word "ALL" sets debugging levels for all sections. We recommend normally running with "ALL,1". - -debug_options ALL,1 DOC_END @@ -814,8 +794,6 @@ IP's connecting to it. This can (in some situations) increase latency, which makes your cache seem slower for interactive browsing. - -log_fqdn off DOC_END @@ -828,8 +806,6 @@ Change this to protect the privacy of your cache clients. A netmask of 255.255.255.0 will log all IP's in that range with the last digit set to '0'. - -client_netmask 255.255.255.255 DOC_END @@ -852,8 +828,6 @@ depending on how the cache is used. Some ftp server also validate that the email address is valid (for example perl.com). - -ftp_user Squid@ DOC_END NAME: ftp_list_width @@ -864,8 +838,6 @@ Sets the width of ftp listings. This should be set to fit in the width of a standard browser. Setting this too small can cut off long filenames when browsing ftp sites. - -ftp_list_width 32 DOC_END NAME: cache_dns_program @@ -875,13 +847,11 @@ LOC: Config.Program.dnsserver DOC_START Specify the location of the executable for dnslookup process. - -cache_dns_program @DEFAULT_DNSSERVER@ DOC_END NAME: dns_children TYPE: int -IFDEF: USE_DNSSERVER +IFDEF: USE_DNSSERVERS DEFAULT: 5 LOC: Config.dnsChildren DOC_START @@ -891,10 +861,29 @@ is 32. The default is 5. You must have at least one dnsserver process. +DOC_END + +NAME: dns_retransmit_interval +TYPE: time_t +DEFAULT: 5 seconds +LOC: Config.Timeout.idns_retransmit +IFDEF: !USE_DNSSERVERS +DOC_START + Initial retransmit interval for DNS queries. The interval is + doubled each time all configured DNS servers have been tried. -dns_children 5 DOC_END +NAME: dns_timeout +TYPE: time_t +DEFAULT: 5 minutes +LOC: Config.Timeout.idns_query +IFDEF: !USE_DNSSERVERS +DOC_START + DNS Query timeout. If no response is received to a DNS query + within this time then all DNS servers for the queried domain + is assumed to be unavailable. +DOC_END NAME: dns_defnames COMMENT: on|off @@ -902,14 +891,13 @@ TYPE: onoff DEFAULT: off LOC: Config.onoff.res_defnames +IFDEF: USE_DNSSERVERS DOC_START Normally the 'dnsserver' disables the RES_DEFNAMES resolver option (see res_init(3)). This prevents caches in a hierarchy from interpreting single-component hostnames locally. To allow dnsserver to handle single-component names, enable this option. - -dns_defnames off DOC_END NAME: dns_nameservers @@ -922,8 +910,6 @@ /etc/resolv.conf file. Example: dns_nameservers 10.0.0.1 192.172.0.4 - -dns_nameservers none DOC_END @@ -933,23 +919,15 @@ LOC: Config.Program.unlinkd DOC_START Specify the location of the executable for file deletion process. - This isn't needed if you are using async-io since it's handled by - a thread. - -unlinkd_program @DEFAULT_UNLINKD@ DOC_END - NAME: pinger_program TYPE: string DEFAULT: @DEFAULT_PINGER@ LOC: Config.Program.pinger +IFDEF: USE_ICMP DOC_START Specify the location of the executable for the pinger process. - This is only useful if you configured Squid (during compilation) - with the '--enable-icmp' option. - -pinger_program @DEFAULT_PINGER@ DOC_END @@ -962,8 +940,6 @@ Since they can perform almost any function there isn't one included. See the Release-Notes for information on how to write one. By default, a redirector is not used. - -redirect_program none DOC_END @@ -976,8 +952,6 @@ too few Squid will have to wait for them to process a backlog of URLs, slowing it down. If you start too many they will use RAM and other system resources. - -redirect_children 5 DOC_END NAME: redirect_rewrites_host_header @@ -988,7 +962,6 @@ By default Squid rewrites any Host: header in redirected requests. If you are running a accelerator then this may not be a wanted effect of a redirector. -redirect_rewrites_host_header on DOC_END NAME: redirector_access @@ -1022,8 +995,6 @@ Then, set this line to something like authenticate_program @DEFAULT_PREFIX@/bin/ncsa_auth @DEFAULT_PREFIX@/etc/passwd - -authenticate_program none DOC_END NAME: authenticate_children @@ -1036,27 +1007,22 @@ of usercode/password verifications, slowing it down. When password verifications are done via a (slow) network you are likely to need lots of authenticator processes. - -authenticate_children 5 DOC_END NAME: authenticate_ttl -TYPE: int -DEFAULT: 3600 +TYPE: time_t +DEFAULT: 1 hour LOC: Config.authenticateTTL DOC_START - The time a checked username/password combination remains cached - (default 3600). If a wrong password is given for a cached user, - the user gets removed from the username/password cache forcing - a revalidation. - -authenticate_ttl 3600 + The time a checked username/password combination remains cached. + If a wrong password is given for a cached user, the user gets + removed from the username/password cache forcing a revalidation. DOC_END NAME: authenticate_ip_ttl -TYPE: int +TYPE: time_t LOC: Config.authenticateIpTTL -DEFAULT: 0 +DEFAULT: 0 seconds DOC_START With this option you control how long a proxy authentication will be bound to a specific IP address. If a request using @@ -1068,11 +1034,11 @@ port. The default is 0 to disable the check. Recommended value - if you have dialup users are no more than 60 (seconds). If - all your users are stationary then higher values may be - used. + if you have dialup users are no more than 60 seconds to allow + the user to redial without hassle. If all your users are + stationary then higher values may be used. -authenticate_ip_ttl 0 + See also authenticate_ip_ttl_is_strict DOC_END NAME: authenticate_ntlm_domain @@ -1083,8 +1049,17 @@ DOC_START For NTLM authentication, the NT domain that this server purports to be part of. This must be set if you wish to use NTLM authentication. +DOC_END -authenticate_ntlm_domain MYDOMAIN +NAME: authenticate_ip_ttl_is_strict +TYPE: onoff +LOC: Config.onoff.authenticateIpTTLStrict +DEFAULT: on +DOC_START + This option makes authenticate_ip_ttl a bit stricted. With this + enabled authenticate_ip_ttl will deny all access from other IP + addresses until the TTL has expired, and the IP address "owning" + the userid will not be forced to reauthenticate. DOC_END COMMENT_START @@ -1104,9 +1079,6 @@ LOC: Config.Wais.relayPort DOC_START Relay WAIS request to host (1st arg) at port (2 arg). - -wais_relay_host localhost -wais_relay_port 8000 DOC_END @@ -1121,7 +1093,6 @@ Placing a limit on the request header size will catch certain bugs (for example with persistent connections) and possibly buffer-overflow or denial-of-service attacks. -request_header_max_size 10 KB DOC_END NAME: request_body_max_size @@ -1136,7 +1107,6 @@ than this limit receives an "Invalid Request" error message. If you set this parameter to a zero, there will be no limit imposed. -request_body_max_size 1 MB DOC_END NAME: reply_body_max_size @@ -1163,7 +1133,6 @@ If you set this parameter to zero (the default), there will be no limit imposed. -reply_body_max_size 0 DOC_END @@ -1171,6 +1140,9 @@ TYPE: refreshpattern LOC: Config.Refresh DEFAULT: none +DEFAULT_IF_NONE: ^ftp: 1440 20% 10080 +DEFAULT_IF_NONE: ^gopher: 1440 0% 1440 +DEFAULT_IF_NONE: . 0 20% 4320 DOC_START usage: refresh_pattern [-i] regex min percent max [options] @@ -1227,12 +1199,9 @@ The first entry which matches is used. If none of the entries match, then the default will be used. -Default: -NOCOMMENT_START -refresh_pattern ^ftp: 1440 20% 10080 -refresh_pattern ^gopher: 1440 0% 1440 -refresh_pattern . 0 20% 4320 -NOCOMMENT_END + Note, you must uncomment all the default lines if you want + to change one. The default setting is only active if none is + used. DOC_END @@ -1275,8 +1244,6 @@ For more information about these cache replacement policies see http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html and http://fog.hpl.external.hp.com/techreports/98/HPL-98-173.html. - -replacement_policy LFUDA DOC_END @@ -1304,8 +1271,6 @@ NOTE: this parameter is not used when using the enhanced replacement policies, GDSH or LFUDA. - -reference_age 1 year DOC_END @@ -1319,7 +1284,7 @@ NAME: quick_abort_max COMMENT: (KB) TYPE: kb_size_t -DEFAULT: 16 kb +DEFAULT: 16 KB LOC: Config.quickAbort.max DOC_NONE @@ -1348,10 +1313,6 @@ If more than 'quick_abort_pct' of the transfer has completed, it will finish the retrieval. - -quick_abort_min 16 KB -quick_abort_max 16 KB -quick_abort_pct 95 DOC_END @@ -1366,8 +1327,6 @@ negatively-cached for a configurable amount of time. The default is 5 minutes. Note that this is different from negative caching of DNS lookups. - -negative_ttl 5 minutes DOC_END @@ -1380,8 +1339,6 @@ Time-to-Live (TTL) for positive caching of successful DNS lookups. Default is 6 hours (360 minutes). If you want to minimize the use of Squid's ipcache, set this to 1, not 0. - -positive_dns_ttl 6 hours DOC_END @@ -1392,8 +1349,6 @@ DEFAULT: 5 minutes DOC_START Time-to-Live (TTL) for negative caching of failed DNS lookups. - -negative_dns_ttl 5 minutes DOC_END NAME: range_offset_limit @@ -1416,8 +1371,6 @@ A value of 0 causes Squid to never fetch more than the client client requested. (default) - -range_offset_limit 0 KB DOC_END @@ -1437,8 +1390,6 @@ enforces its own timeout on server connections. This parameter specifies how long to wait for the connect to complete. The default is two minutes (120 seconds). - -connect_timeout 120 seconds DOC_END NAME: peer_connect_timeout @@ -1451,7 +1402,6 @@ connection to a peer cache. The default is 30 seconds. You may also set different timeout values for individual neighbors with the 'connect-timeout' option on a 'cache_peer' line. -peer_connect_timeout 30 seconds DOC_END NAME: siteselect_timeout @@ -1461,8 +1411,6 @@ DEFAULT: 4 seconds DOC_START For URN to multiple URL's URL selection - -siteselect_timeout 4 seconds DOC_END NAME: read_timeout @@ -1476,8 +1424,6 @@ amount. If no data is read again after this amount of time, the request is aborted and logged with ERR_READ_TIMEOUT. The default is 15 minutes. - -read_timeout 15 minutes DOC_END @@ -1489,8 +1435,6 @@ How long to wait for an HTTP request after connection establishment. For persistent connections, wait this long after the previous request completes. - -request_timeout 30 seconds DOC_END @@ -1514,8 +1458,6 @@ If you seem to have many client connections tying up filedescriptors, we recommend first tuning the read_timeout, request_timeout, pconn_timeout and quick_abort values. - -client_lifetime 1 day DOC_END NAME: half_closed_clients @@ -1531,8 +1473,6 @@ socket returns an error. Change this option to 'off' and Squid will immediately close client connections when read(2) returns "no more data to read." - -half_closed_clients on DOC_END NAME: pconn_timeout @@ -1542,7 +1482,6 @@ DOC_START Timeout for idle persistent connections to servers and other proxies. -pconn_timeout 120 seconds DOC_END NAME: ident_timeout @@ -1562,7 +1501,6 @@ This option may be disabled by using --disable-ident with the configure script. -ident_timeout 10 seconds DOC_END @@ -1577,8 +1515,6 @@ This value is the lifetime to set for all open descriptors during shutdown mode. Any active clients after this many seconds will receive a 'timeout' message. - -shutdown_lifetime 30 seconds DOC_END COMMENT_START @@ -1637,6 +1573,7 @@ acl aclname browser [-i] regexp # pattern match on User-Agent header acl aclname ident username ... + acl aclname ident_regex [-i] pattern ... # string match on ident output. # use REQUIRED to accept any non-null ident. acl aclname src_as number ... @@ -1650,6 +1587,7 @@ # cache_peer_access mycache_mydomain.net deny all acl aclname proxy_auth username ... + acl aclname proxy_auth_regex [-i] pattern ... # list of valid usernames # use REQUIRED to accept any valid username. # @@ -1675,18 +1613,30 @@ # This will be matched when the client's IP address has # more than HTTP connections established. + acl req_mime_type mime-type1 ... + # regex match agains the mime type of the request generated + # by the client. Can be used to detect file upload or some + # types HTTP tunelling requests. + # NOTE: This does NOT match the reply. You cannot use this + # to match the returned file type. Examples: acl myexample dst_as 1241 acl password proxy_auth REQUIRED +acl fileupload req_mime_type -i ^multipart/form-data$ -Defaults: NOCOMMENT_START +#Recommended minimum configuration: acl all src 0.0.0.0/0.0.0.0 acl manager proto cache_object acl localhost src 127.0.0.1/255.255.255.255 acl SSL_ports port 443 563 -acl Safe_ports port 80 21 443 563 70 210 1025-65535 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 563 # https, snews +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports acl Safe_ports port 280 # http-mgmt acl Safe_ports port 488 # gss-http acl Safe_ports port 591 # filemaker @@ -1706,12 +1656,9 @@ Access to the HTTP port: http_access allow|deny [!]aclname ... - Access to the ICP port: - icp_access allow|deny [!]aclname ... - NOTE on default values: - If there are no "access" lines present, the default is to allow + If there are no "access" lines present, the default is to deny the request. If none of the "access" lines cause a match, the default is the @@ -1721,15 +1668,20 @@ good idea to have an "deny all" or "allow all" entry at the end of your access lists to avoid potential confusion. -Default configuration: NOCOMMENT_START +#Recommended minimum configuration: +# +# Only allow cachemgr access from localhost http_access allow manager localhost http_access deny manager +# Deny requests to unknown ports http_access deny !Safe_ports +# Deny CONNECT to other than SSL ports http_access deny CONNECT !SSL_ports # # INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS # +# And finally deny all other access to this proxy http_access deny all NOCOMMENT_END DOC_END @@ -1741,9 +1693,15 @@ DEFAULT: none DEFAULT_IF_NONE: deny all DOC_START - Reply to all ICP queries we receive + Allowing or Denying access to the ICP port based on defined + access lists + + icp_access allow|deny [!]aclname ... + + See http_access for details NOCOMMENT_START +#Allow ICP queries from eveyone icp_access allow all NOCOMMENT_END DOC_END @@ -1766,8 +1724,10 @@ By default, allow all clients who passed the http_access rules to fetch MISSES from us. + NOCOMMENT_START -miss_access allow all +#Default setting: +# miss_access allow all NOCOMMENT_END DOC_END @@ -1795,8 +1755,6 @@ Specifies the realm name which is to be reported to the client for proxy authentication (part of the text the user will see when prompted their username and password). - -proxy_auth_realm Squid proxy-caching web server DOC_END @@ -1823,7 +1781,6 @@ This option may be disabled by using --disable-ident with the configure script. -ident_lookup_access deny all DOC_END @@ -1839,8 +1796,6 @@ DOC_START Email-address of local cache manager who will receive mail if the cache dies. The default is "webmaster." - -cache_mgr webmaster DOC_END @@ -1863,9 +1818,6 @@ If Squid is not started as root, the default is to keep the current UID/GID. Note that if Squid is not started as root then you cannot set http_port to a value lower than 1024. - -cache_effective_user nobody -cache_effective_group nogroup DOC_END @@ -1879,8 +1831,6 @@ will be used. If you have multiple caches in a cluster and get errors about IP-forwarding you must set them to have individual names with this setting. - -visible_hostname www-cache.foo.org DOC_END @@ -1892,8 +1842,6 @@ If you want to have multiple machines with the same 'visible_hostname' then you must give each machine a different 'unique_hostname' so that forwarding loops can be detected. - -unique_hostname www-cache1.foo.org DOC_END @@ -1941,7 +1889,10 @@ To enable announcing your cache, just uncomment the line below. -announce_period 1 day +NOCOMMENT_START +#To enable announcing your cache, just uncomment the line below. +#announce_period 1 day +NOCOMMENT_END DOC_END @@ -1969,9 +1920,6 @@ default default to 3131. If the 'filename' argument is given, the contents of that file will be included in the announce message. - -announce_host tracker.ircache.net -announce_port 3131 DOC_END COMMENT_START @@ -1999,11 +1947,28 @@ NOTE: enabling httpd_accel_host disables proxy-caching and ICP. If you want these features enabled also, then set the 'httpd_accel_with_proxy' option. - -httpd_accel_host hostname -httpd_accel_port port DOC_END +NAME: httpd_accel_single_host +COMMENT: on|off +TYPE: onoff +LOC: Config.Accel.single_host +DEFAULT: on +DOC_START + If you are running Squid as a accelerator and have a single backend + server then set this to on. This causes Squid to forward the request + to this server irregardles of what any redirectors or Host headers + says. + + Set this to off if you have multiple backend servers, and use a + redirector (or host table) to map the requests to the + appropriate backend servers. Note that the mapping needs to be a + 1-1 mapping between requested and backend (from redirector) domain + names or caching will fail, as cacing is performed using the + URL returned from the redirector. + + See also redirect_rewrites_host_header. +DOC_END NAME: httpd_accel_with_proxy COMMENT: on|off @@ -2012,12 +1977,13 @@ LOC: Config.onoff.accel_with_proxy DOC_START If you want to use Squid as both a local httpd accelerator - and as a proxy, change this to 'on'. - -httpd_accel_with_proxy off + and as a proxy, change this to 'on'. Note however that your + proxy users may have trouble to reach the accelerated domains + unless their browsers are configured not to use this proxy for + those domains (for example via the no_proxy browser configuration + setting) DOC_END - NAME: httpd_accel_uses_host_header COMMENT: on|off TYPE: onoff @@ -2034,7 +2000,6 @@ However, you will need to enable this option if you run Squid as a transparent proxy. Otherwise, virtual servers which require the Host: header will not be properly cached. -httpd_accel_uses_host_header off DOC_END COMMENT_START @@ -2046,13 +2011,11 @@ TYPE: wordlist LOC: Config.dns_testname_list DEFAULT: none +DEFAULT_IF_NONE: netscape.com internic.net nlanr.net microsoft.com DOC_START The DNS tests exit as soon as the first site is successfully looked up - If you want to disable DNS tests, do not comment out or delete this - list. Instead use the -D command line option - -dns_testnames netscape.com internic.net nlanr.net microsoft.com + This test can be disabled with the -D command line option. DOC_END @@ -2074,8 +2037,6 @@ purposes, so -k rotate uses another signal. It is best to get in the habit of using 'squid -k rotate' instead of 'kill -USR1 '. - -logfile_rotate 10 DOC_END @@ -2087,7 +2048,8 @@ Appends local domain name to hostnames without any dots in them. append_domain must begin with a period. -append_domain .yourdomain.com +Example: + append_domain .yourdomain.com DOC_END @@ -2100,8 +2062,6 @@ Size of receive buffer to set for TCP sockets. Probably just as easy to change your kernel's default. Set to zero to use the default buffer size. - -tcp_recv_bufsize 0 bytes DOC_END NAME: err_html_text @@ -2117,7 +2077,6 @@ the error template files (found in the "errors" directory). Wherever you want the 'err_html_text' line to appear, insert a %L tag in the error template file. -err_html_text DOC_END @@ -2148,8 +2107,6 @@ available for future use. If memory is a premium on your system and you believe your malloc library outperforms Squid routines, disable this. - -memory_pools on DOC_END NAME: memory_pools_limit @@ -2197,8 +2154,6 @@ If you disable this, it will appear as X-Forwarded-For: unknown - -forwarded_for on DOC_END NAME: log_icp_queries @@ -2210,8 +2165,6 @@ If set, ICP queries are logged to access.log. You may wish do disable this if your ICP load is VERY high to speed things up or to simplify log analysis. - -log_icp_queries on DOC_END NAME: icp_hit_stale @@ -2225,8 +2178,6 @@ in other administrative domains, this should be 'off'. If you only have sibling relationships with caches under your control, then it is probably okay to set this to 'on'. - -icp_hit_stale off DOC_END @@ -2237,8 +2188,6 @@ DOC_START If using the ICMP pinging stuff, do direct fetches for sites which are no more than this many hops away. - -minimum_direct_hops 4 DOC_END @@ -2298,9 +2247,10 @@ Use the keyword "all" to set the same password for all actions. -cachemgr_passwd secret shutdown -cachemgr_passwd lesssssssecret info stats/objects -cachemgr_passwd disable all +Example: + cachemgr_passwd secret shutdown + cachemgr_passwd lesssssssecret info stats/objects + cachemgr_passwd disable all DOC_END NAME: store_avg_object_size @@ -2312,8 +2262,6 @@ Average object size, used to estimate number of objects your cache can hold. See doc/Release-Notes-1.1.txt. The default is 13 KB. - -store_avg_object_size 13 KB DOC_END NAME: store_objects_per_bucket @@ -2324,8 +2272,6 @@ Target number of objects per bucket in the store hash table. Lowering this value increases the total number of buckets and also the storage maintenance rate. The default is 50. - -store_objects_per_bucket 20 DOC_END NAME: client_db @@ -2336,8 +2282,6 @@ DOC_START If you want to disable collecting per-client statistics, then turn off client_db here. - -client_db on DOC_END @@ -2356,9 +2300,6 @@ database. These are counts, not percents. The defaults are 900 and 1000. When the high water mark is reached, database entries will be deleted until the low mark is reached. - -netdb_low 900 -netdb_high 1000 DOC_END @@ -2370,8 +2311,6 @@ The minimum period for measuring a site. There will be at least this much delay between successive pings to the same network. The default is five minutes. - -netdb_ping_period 5 minutes DOC_END @@ -2392,8 +2331,6 @@ the minimal RTT to the origin server. When this happens, the hierarchy field of the access.log will be "CLOSEST_PARENT_MISS". This option is off by default. - -query_icmp off DOC_END NAME: test_reachability @@ -2405,8 +2342,6 @@ When this is 'on', ICP MISS replies will be ICP_MISS_NOFETCH instead of ICP_MISS if the target host is NOT in the ICMP database, or has a zero RTT. - -test_reachability off DOC_END NAME: buffered_logs @@ -2420,7 +2355,6 @@ unbuffered. By default they will be unbuffered. Buffering them can speed up the writing slightly (though you are unlikely to need to worry). -buffered_logs off DOC_END NAME: reload_into_ims @@ -2440,7 +2374,6 @@ This option may be disabled by using --disable-http-violations with the configure script. -reload_into_ims off DOC_END NAME: always_direct @@ -2551,8 +2484,6 @@ By default, all headers are allowed (no anonymizing is performed). - -anonymize_headers DOC_END NAME: fake_user_agent @@ -2566,8 +2497,6 @@ fake_user_agent Nutscrape/1.0 (CP/M; 8-bit) (credit to Paul Southworth pauls@etext.org for this one!) - -fake_user_agent none DOC_END NAME: icon_directory @@ -2610,8 +2539,6 @@ minimum value, is five seconds, and the maximum value is sixty seconds, or half of connect_timeout, whichever is greater and less than connect_timeout. - -minimum_retry_timeout 5 seconds DOC_END NAME: maximum_single_addr_tries @@ -2626,8 +2553,6 @@ The default value is three tries, the (not recommended) maximum is 255 tries. A warning message will be generated if it is set to a value greater than ten. - -maximum_single_addr_tries 3 DOC_END NAME: snmp_port @@ -2642,7 +2567,6 @@ NOTE: SNMP support requires use the --enable-snmp configure command line option. -snmp_port 3401 DOC_END NAME: snmp_access @@ -2660,8 +2584,8 @@ snmp_access allow|deny [!]aclname ... Example: -snmp_access allow snmppublic localhost -snmp_access deny all + snmp_access allow snmppublic localhost + snmp_access deny all DOC_END NAME: snmp_incoming_address @@ -2683,13 +2607,16 @@ snmp_outgoing_address is used for SNMP packets returned to SNMP agents. - The default behavior is to not bind to any specific address. + The default snmp_incoming_address (0.0.0.0) is to listen on all + available network interfaces. - NOTE, snmp_incoming_address and snmp_outgoing_address can not have - the same value since they both use port 3130. + If snmp_outgoing_address is set to 255.255.255.255 (the default) + then it will use the same socket as snmp_incoming_address. Only + change this if you want to have SNMP replies sent using another + address than where this Squid listens for SNMP queries. -snmp_incoming_address 0.0.0.0 -snmp_outgoing_address 0.0.0.0 + NOTE, snmp_incoming_address and snmp_outgoing_address can not have + the same value since they both use port 3401. DOC_END NAME: as_whois_server @@ -2711,7 +2638,6 @@ Use this option to define your WCCP ``home'' router for Squid. Setting the 'wccp_router' to 0.0.0.0 (the default) disables WCCP. -wccp_router 0.0.0.0 DOC_END NAME: wccp_incoming_address @@ -2744,9 +2670,6 @@ NOTE, wccp_incoming_address and wccp_outgoing_address can not have the same value since they both use port 2048. - -wccp_incoming_address 0.0.0.0 -wccp_outgoing_address 0.0.0.0 DOC_END @@ -2767,7 +2690,6 @@ To enable this option, you must use --enable-delay-pools with the configure script. -delay_pools 0 DOC_END NAME: delay_class @@ -2781,9 +2703,10 @@ delay pools, one of class 2 and one of class 3, the settings above and here would be: -delay_pools 2 # 2 delay pools -delay_class 1 2 # pool 1 is a class 2 pool -delay_class 2 3 # pool 2 is a class 3 pool +Example: + delay_pools 2 # 2 delay pools + delay_class 1 2 # pool 1 is a class 2 pool + delay_class 2 3 # pool 2 is a class 3 pool The delay pool classes are: @@ -2819,10 +2742,11 @@ all been checked. For example, if you want some_big_clients in delay pool 1 and lotsa_little_clients in delay pool 2: -delay_access 1 allow some_big_clients -delay_access 1 deny all -delay_access 2 allow lotsa_little_clients -delay_access 2 deny all +Example: + delay_access 1 allow some_big_clients + delay_access 1 deny all + delay_access 2 allow lotsa_little_clients + delay_access 2 deny all DOC_END NAME: delay_parameters @@ -2898,8 +2822,6 @@ a host accessing it (in class 2 and class 3, individual hosts and networks only have buckets associated with them once they have been "seen" by squid). - -delay_initial_bucket_level 50 DOC_END NAME: incoming_icp_average @@ -2928,18 +2850,19 @@ Heavy voodoo here. I can't even believe you are reading this. Are you crazy? Don't even think about adjusting these unless you understand the algorithms in comm_select.c first! - -incoming_icp_average 6 -incoming_http_average 4 -min_icp_poll_cnt 8 -min_http_poll_cnt 8 DOC_END NAME: max_open_disk_fds TYPE: int LOC: Config.max_open_disk_fds DEFAULT: 0 -DOC_NONE +DOC_START + To avoid having disk as the I/O bottleneck Squid can optionally + bypass the on-disk cache if more than this amount of disk file + descriptors are open. + + A value of 0 indicates no limit. +DOC_END NAME: offline_mode TYPE: onoff @@ -2973,7 +2896,6 @@ chop: The request is allowed and the URI is chopped at the first whitespace. This might also be considered a violation. -uri_whitespace strip DOC_END NAME: broken_posts @@ -2994,8 +2916,9 @@ forbidden by the BNF, an HTTP/1.1 client must not preface or follow a request with an extra CRLF. -acl buggy_server url_regex ^http://.... -broken_posts allow buggy_server +Example: + acl buggy_server url_regex ^http://.... + broken_posts allow buggy_server DOC_END NAME: mcast_miss_addr @@ -3042,19 +2965,38 @@ encrypted. This is the encryption key. DOC_END +NAME: nonhierarchical_direct +TYPE: onoff +LOC: Config.onoff.nonhierarchical_direct +DEFAULT: on +DOC_START + By default, Squid will send any non-hierarchical requests + (matching hierarchy_stoplist or not cachable request type) direct + to origin servers. + + If you set this to off, then Squid will prefer to send these + requests to parents. + + Note that in most configurations, by turning this off you will only + add latency to these request without any improvement in global hit + ratio. + + If you are inside an firewall then see never_direct instead of + this directive. +DOC_END + NAME: prefer_direct TYPE: onoff LOC: Config.onoff.prefer_direct -DEFAULT: on +DEFAULT: off DOC_START - By default, if the ICP, HTCP, Cache Digest, etc. techniques - do not yield a parent cache, Squid gives higher preference - to forwarding the request direct to origin servers, rather - than selecting a parent cache anyway. - - If you want Squid to give higher precedence to a parent - cache, instead of going direct, then turn this option off. -prefer_direct on + Normally Squid tries to use parents for most requests. If you by some + reason like it to first try going direct and only use a parent if + going direct fails then set this to off. + + By combining nonhierarchical_direct off and prefer_direct on you + can set up Squid to use a parent as a backup path if going direct + fails. DOC_END NAME: strip_query_terms @@ -3064,7 +3006,6 @@ DOC_START By default, Squid strips query terms from requested URLs before logging. This protects your user's privacy. -strip_query_terms on DOC_END NAME: coredump_dir @@ -3104,7 +3045,6 @@ don't match, Squid ignores the response and writes a warning message to cache.log. You can allow responses from unknown nameservers by setting this option to 'off'. -ignore_unknown_nameservers on DOC_END NAME: digest_generation @@ -3116,7 +3056,6 @@ This controls whether the server will generate a Cache Digest of its contents. By default, Cache Digest generation is enabled if Squid is compiled with USE_CACHE_DIGESTS defined. -digest_generation on DOC_END NAME: digest_bits_per_entry @@ -3128,7 +3067,6 @@ This is the number of bits of the server's Cache Digest which will be associated with the Digest entry for a given HTTP Method and URL (public key) combination. The default is 5. -digest_bits_per_entry 5 DOC_END NAME: digest_rebuild_period @@ -3139,8 +3077,6 @@ DEFAULT: 1 hour DOC_START This is the number of seconds between Cache Digest rebuilds. - By default the server's Digest is rebuilt every hour. -digest_rebuild_period 1 hour DOC_END NAME: digest_rewrite_period @@ -3151,9 +3087,7 @@ DEFAULT: 1 hour DOC_START This is the number of seconds between Cache Digest writes to - disk. By default the server's Digest is written to disk every - hour. -digest_rewrite_period 1 hour + disk. DOC_END NAME: digest_swapout_chunk_size @@ -3166,7 +3100,6 @@ This is the number of bytes of the Cache Digest to write to disk at a time. It defaults to 4096 bytes (4KB), the Squid default swap page. -digest_swapout_chunk_size 4096 bytes DOC_END NAME: digest_rebuild_chunk_percentage @@ -3178,7 +3111,6 @@ DOC_START This is the percentage of the Cache Digest to be scanned at a time. By default it is set to 10% of the Cache Digest. -digest_rebuild_chunk_percentage 10 DOC_END NAME: chroot @@ -3208,8 +3140,6 @@ default, Squid uses persistent connections (when allowed) with its clients and servers. You can use these options to disable persistent connections with clients and/or servers. -client_persistent_connections on -server_persistent_connections on DOC_END NAME: diskd_magic1 @@ -3224,7 +3154,9 @@ TYPE: int LOC: Config.diskd.magic2 DEFAULT: 72 -DOC_NONE +DOC_START + Macic constants for the diskd daemon.. +DOC_END EOF Index: squid/src/cf_gen.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf_gen.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/cf_gen.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.12.1 +++ squid/src/cf_gen.c 3 May 2000 19:18:12 -0000 1.1.1.3.12.2 @@ -1,5 +1,5 @@ /* - * $Id: cf_gen.c,v 1.1.1.3.12.1 2000/04/17 00:13:09 hno Exp $ + * $Id: cf_gen.c,v 1.1.1.3.12.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: none * AUTHOR: Max Okumoto @@ -48,6 +48,7 @@ *****************************************************************************/ #include "config.h" +#include "cf_gen_defines.h" #if HAVE_STDIO_H #include @@ -89,7 +90,7 @@ char *type; char *loc; char *default_value; - char *default_if_none; + Line *default_if_none; char *comment; char *ifdef; Line *doc; @@ -106,6 +107,14 @@ static void gen_conf(Entry *, FILE *); static void gen_default_if_none(Entry *, FILE *); +static void lineAdd(Line **L, char *str) +{ + while(*L) + L = &(*L)->next; + *L=xcalloc(1, sizeof(Line)); + (*L)->data = xstrdup(str); +} + int main(int argc, char *argv[]) { @@ -189,7 +198,7 @@ ptr = buff + 16; while (xisspace(*ptr)) ptr++; - curr->default_if_none = xstrdup(ptr); + lineAdd(&curr->default_if_none, ptr); } else if (!strncmp(buff, "LOC:", 4)) { if ((ptr = strtok(buff + 4, WS)) == NULL) { printf("Error on line %d\n", linenum); @@ -396,6 +405,7 @@ gen_default_if_none(Entry * head, FILE * fp) { Entry *entry; + Line *line; fprintf(fp, "static void\n" "defaults_if_none(void)\n" @@ -408,15 +418,20 @@ continue; if (entry->ifdef) fprintf(fp, "#if %s\n", entry->ifdef); - fprintf(fp, - "\tif (check_null_%s(%s))\n" - "\t\tdefault_line(\"%s %s\");\n", - entry->type, - entry->loc, - entry->name, - entry->default_if_none); + if (entry->default_if_none) { + fprintf(fp, + "\tif (check_null_%s(%s)) {\n", + entry->type, + entry->loc); + for (line = entry->default_if_none; line; line = line->next) + fprintf(fp, + "\t\tdefault_line(\"%s %s\");\n", + entry->name, + line->data); + fprintf(fp, "\t}\n"); + } if (entry->ifdef) - fprintf(fp, "#endif\n"); + fprintf(fp, "#endif\n"); } fprintf(fp, "}\n\n"); } @@ -520,13 +535,37 @@ fprintf(fp, "}\n\n"); } +static int defined(char *name) +{ + int i=0; + if (!name) + return 1; + for(i=0;strcmp(defines[i].name, name) != 0; i++) { + assert(defines[i].name); + } + return defines[i].defined; +} + +static const char *available_if(char *name) +{ + int i=0; + assert(name); + for(i=0;strcmp(defines[i].name, name) != 0; i++) { + assert(defines[i].name); + } + return defines[i].enable; +} + static void gen_conf(Entry * head, FILE * fp) { Entry *entry; + char buf[8192]; + Line *def = NULL; for (entry = head; entry != NULL; entry = entry->next) { Line *line; + int blank = 1; if (!strcmp(entry->name, "comment")) (void) 0; @@ -535,9 +574,43 @@ if (entry->comment) fprintf(fp, "\t%s", entry->comment); fprintf(fp, "\n"); + if (!defined(entry->ifdef)) { + fprintf(fp, "# Note: This option is only available if Squid is rebuilt with the\n"); + fprintf(fp, "# %s option\n#\n", available_if(entry->ifdef)); + } for (line = entry->doc; line != NULL; line = line->next) { fprintf(fp, "#%s\n", line->data); } + if (entry->default_value && strcmp(entry->default_value,"none") != 0) { + sprintf(buf, "%s %s", entry->name, entry->default_value); + lineAdd(&def, buf); + } + if (entry->default_if_none) { + for (line = entry->default_if_none; line; line = line->next) { + sprintf(buf, "%s %s", entry->name, line->data); + lineAdd(&def, buf); + } + } + if (entry->nocomment) + blank = 0; + if (!def && entry->doc && !entry->nocomment && + strcmp(entry->name, "comment") != 0) + lineAdd(&def, "none"); + if (def && (entry->doc || entry->nocomment)) { + if (blank) + fprintf(fp, "#\n"); + fprintf(fp, "#Default:\n"); + while (def != NULL) { + line = def; + def = line->next; + fprintf(fp, "# %s\n", line->data); + free(line->data); + free(line); + } + blank=1; + } + if (entry->nocomment && blank) + fprintf(fp, "#\n"); for (line = entry->nocomment; line != NULL; line = line->next) { fprintf(fp, "%s\n", line->data); } --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/cf_gen_defines Wed Feb 14 00:43:58 2007 @@ -0,0 +1,31 @@ +#!/usr/bin/awk -f +BEGIN { + print "/* Generated automatically from cf.data.pre" + print " * DO NOT EDIT" + print "*/" + print "struct { char *name; char *enable; int defined;} defines[] = {" + define["DELAY_POOLS"]="--enable-delaypools" + define["HEAP_REPLACEMENT"]="--enable-heap-replacement" + define["HTTP_VIOLATIONS"]="--enable-http-violations" + define["SQUID_SNMP"]="--enable-snmp" + define["USE_CACHE_DIGESTS"]="--enable-cache-digests" + define["USE_DNSSERVERS"]="--disable-internal-dns" + define["!USE_DNSSERVERS"]="--enable-internal-dns" + define["USE_HTCP"]="--enable-htcp" + define["USE_ICMP"]="--enable-icmp" + define["USE_IDENT"]="--enable-ident-lookups" + define["USE_USERAGENT_LOG"]="--enable-useragent-log" + define["USE_WCCP"]="--enable-wccp" +} +/^IFDEF:/ { + print "{\"" $2 "\", \"" (define[$2] != "" ? define[$2] : ("-D" $2)) "\", " + print "#if " $2 + print "1" + print "#else" + print "0" + print "#endif" + print "}," +} +END { + print "{(void *)0L, 0}};" +} Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.1.1.3.4.1.2.9 retrieving revision 1.1.1.3.4.1.2.10 diff -u -r1.1.1.3.4.1.2.9 -r1.1.1.3.4.1.2.10 --- squid/src/client_side.c 3 May 2000 11:22:56 -0000 1.1.1.3.4.1.2.9 +++ squid/src/client_side.c 3 May 2000 19:18:12 -0000 1.1.1.3.4.1.2.10 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.1.1.3.4.1.2.9 2000/05/03 11:22:56 asd Exp $ + * $Id: client_side.c,v 1.1.1.3.4.1.2.10 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -83,6 +83,8 @@ static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri); static clientHttpRequest *parseHttpRequest(ConnStateData *, method_t *, int *, char **, size_t *); static RH clientRedirectDone; +static void clientCheckNoCache(clientHttpRequest *); +static void clientCheckNoCacheDone(int answer, void *data); static STCB clientHandleIMSReply; static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); static int checkAccelOnly(clientHttpRequest *); @@ -136,16 +138,12 @@ } #endif -void -clientAccessCheck(void *data) +static aclCheck_t * +clientAclChecklistCreate(const acl_access *acl, const clientHttpRequest *http) { - clientHttpRequest *http = data; + aclCheck_t *ch; ConnStateData *conn = http->conn; - if (checkAccelOnly(http)) { - clientAccessCheckDone(ACCESS_ALLOWED, http); - return; - } - http->acl_checklist = aclChecklistCreate(Config.accessList.http, + ch = aclChecklistCreate(acl, http->request, conn->ident); #if USE_IDENT @@ -153,9 +151,21 @@ * hack for ident ACL. It needs to get full addresses, and a * place to store the ident result on persistent connections... */ - http->acl_checklist->conn = conn; - cbdataLock(http->acl_checklist->conn); + ch->conn = conn; + cbdataLock(ch->conn); #endif + return ch; +} + +void +clientAccessCheck(void *data) +{ + clientHttpRequest *http = data; + if (checkAccelOnly(http)) { + clientAccessCheckDone(ACCESS_ALLOWED, http); + return; + } + http->acl_checklist = clientAclChecklistCreate(Config.accessList.http, http); aclNBCheck(http->acl_checklist, clientAccessCheckDone, http); } @@ -297,6 +307,26 @@ headersLog(0, 1, request->method, request); #endif fd_note(http->conn->fd, http->uri); + clientCheckNoCache(http); +} + +static void +clientCheckNoCache(clientHttpRequest * http) +{ + if (Config.accessList.noCache && http->request->flags.cachable) { + http->acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http); + aclNBCheck(http->acl_checklist, clientCheckNoCacheDone, http); + } else { + clientCheckNoCacheDone(http->request->flags.cachable, http); + } +} + +void +clientCheckNoCacheDone(int answer, void *data) +{ + clientHttpRequest *http = data; + http->request->flags.cachable = answer; + http->acl_checklist = NULL; clientProcessRequest(http); } @@ -544,6 +574,8 @@ return; } http->log_type = LOG_TCP_MISS; + /* Release both IP and object cache entries */ + ipcacheInvalidate(http->request->host); if ((entry = storeGetPublic(http->uri, METHOD_GET)) == NULL) { http->http_code = HTTP_NOT_FOUND; } else { @@ -652,11 +684,18 @@ if (!clientCheckTransferDone(http)) { if (request && request->body_connection) clientAbortBody(request); /* abort body transter */ +#if MYSTERIOUS_CODE + /* + * DW: this seems odd here, is it really needed? It causes + * incomplete transfers to get logged with "000" status + * code because http->entry becomes NULL. + */ if ((e = http->entry)) { http->entry = NULL; storeUnregister(e, http); storeUnlockObject(e); } +#endif if (http->entry && http->entry->ping_status == PING_WAITING) storeReleaseRequest(http->entry); } @@ -895,28 +934,6 @@ const char *url = http->uri; request_t *req = http->request; method_t method = req->method; - aclCheck_t ch; - memset(&ch, '\0', sizeof(ch)); - /* - * Hopefully, nobody really wants 'no_cache' by client's IP - * address, but if they do, this should work if they use IP - * addresses in their ACLs, or if the client's address is in - * the FQDN cache. - * - * This may not work yet for 'dst' and 'dst_domain' ACLs. - */ - ch.src_addr = http->conn->peer.sin_addr; - ch.my_addr = http->conn->me.sin_addr; - ch.my_port = ntohs(http->conn->me.sin_port); - ch.request = http->request; - /* - * aclCheckFast returns 1 for ALLOW and 0 for DENY. The default - * is ALLOW, so we require 'no_cache DENY foo' in squid.conf - * to indicate uncachable objects. - */ - if (Config.accessList.noCache) - if (!aclCheckFast(Config.accessList.noCache, &ch)) - return 0; if (req->protocol == PROTO_HTTP) return httpCachable(method); /* FTP is always cachable */ @@ -1071,7 +1088,9 @@ { HttpHeader *hdr = rep ? &rep->header : 0; const char *range_err = NULL; - assert(http->request->range); + request_t *request = http->request; + int is_hit = isTcpHit(http->log_type); + assert(request->range); /* check if we still want to do ranges */ if (!rep) range_err = "no [parse-able] reply"; @@ -1089,6 +1108,11 @@ range_err = "canonization failed"; else if (httpHdrRangeIsComplex(http->request->range)) range_err = "too complex range header"; + else if (!request->flags.cachable) /* from we_do_ranges in http.c */ + range_err = "non-cachable request"; + else if (!is_hit && Config.rangeOffsetLimit < httpHdrRangeFirstOffset(request->range) + && Config.rangeOffsetLimit != -1) /* from we_do_ranges in http.c */ + range_err = "range outside range_offset_limit"; /* get rid of our range specs on error */ if (range_err) { debug(33, 3) ("clientBuildRangeHeader: will not do ranges: %s.\n", range_err); @@ -1879,6 +1903,13 @@ /* We can generate a HEAD reply from a cached GET object */ e = http->entry = storeGetPublic(http->uri, METHOD_GET); } + /* Release negatively cached IP-cache entries on reload */ + if (r->flags.nocache) + ipcacheReleaseInvalid(r->host); +#if HTTP_VIOLATIONS + else if (r->flags.nocache_hack) + ipcacheReleaseInvalid(r->host); +#endif #if USE_CACHE_DIGESTS http->lookup_type = e ? "HIT" : "MISS"; #endif @@ -1904,12 +1935,6 @@ return LOG_TCP_HIT; } #if HTTP_VIOLATIONS - if (r->flags.nocache_hack) { - /* if nocache_hack is set, nocache should always be clear, right? */ - assert(!r->flags.nocache); - ipcacheReleaseInvalid(r->host); - /* continue! */ - } if (e->store_status == STORE_PENDING) { if (r->flags.nocache || r->flags.nocache_hack) { debug(33, 3) ("Clearing no-cache for STORE_PENDING request\n\t%s\n", Index: squid/src/comm_select.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/comm_select.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/comm_select.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.12.1 +++ squid/src/comm_select.c 3 May 2000 19:18:12 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: comm_select.c,v 1.1.1.3.12.1 2000/04/17 00:13:09 hno Exp $ + * $Id: comm_select.c,v 1.1.1.3.12.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 5 Socket Functions * @@ -34,13 +34,7 @@ #include "squid.h" -#if USE_ASYNC_IO -#define MAX_POLL_TIME 10 -#elif USE_DISKD -#define MAX_POLL_TIME 10 -#else -#define MAX_POLL_TIME 1000 -#endif +static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */ #ifndef howmany #define howmany(x, y) (((x)+((y)-1))/(y)) @@ -332,12 +326,8 @@ getCurrentTime(); start = current_dtime; #endif -#if USE_ASYNC_IO - aioCheckCallbacks(); -#endif -#if USE_DISKD - storeDiskdReadQueue(); -#endif + /* Handle any fs callbacks that need doing */ + storeDirCallback(); #if DELAY_POOLS FD_ZERO(&slowfds); #endif @@ -645,6 +635,7 @@ fd_set slowfds; #endif PF *hdl = NULL; + SwapDir *SD; int fd; int maxfd; int num; @@ -662,19 +653,16 @@ struct timeval poll_time; double timeout = current_dtime + (msec / 1000.0); fde *F; + int i; do { #if !ALARM_UPDATES_TIME getCurrentTime(); #endif -#if USE_ASYNC_IO - aioCheckCallbacks(); -#endif -#if USE_DISKD - storeDiskdReadQueue(); -#endif #if DELAY_POOLS FD_ZERO(&slowfds); #endif + /* Handle any fs callbacks that need doing */ + storeDirCallback(); if (commCheckICPIncoming) comm_select_icp_incoming(); if (commCheckDNSIncoming) @@ -1090,3 +1078,10 @@ nwritefds--; } } + +/* Called by async-io or diskd to speed up the polling */ +void +comm_quick_poll_required(void) +{ + MAX_POLL_TIME = 10; +} Index: squid/src/defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/defines.h,v retrieving revision 1.1.1.3.12.2 retrieving revision 1.1.1.3.12.3 diff -u -r1.1.1.3.12.2 -r1.1.1.3.12.3 --- squid/src/defines.h 17 Apr 2000 13:45:13 -0000 1.1.1.3.12.2 +++ squid/src/defines.h 3 May 2000 19:18:12 -0000 1.1.1.3.12.3 @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.1.1.3.12.2 2000/04/17 13:45:13 asd Exp $ + * $Id: defines.h,v 1.1.1.3.12.3 2000/05/03 19:18:12 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -151,7 +151,6 @@ #define LOG_DISABLE 0 #define SM_PAGE_SIZE 4096 -#define DISK_PAGE_SIZE 8192 #define EBIT_SET(flag, bit) ((void)((flag) |= ((1L<<(bit))))) #define EBIT_CLR(flag, bit) ((void)((flag) &= ~((1L<<(bit))))) @@ -274,8 +273,3 @@ #define _PATH_DEVNULL "/dev/null" #endif -#if USE_ASYNC_IO -#ifndef NUMTHREADS -#define NUMTHREADS 16 -#endif -#endif Index: squid/src/delay_pools.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/delay_pools.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/delay_pools.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.12.1 +++ squid/src/delay_pools.c 3 May 2000 19:18:12 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: delay_pools.c,v 1.1.1.3.12.1 2000/04/17 00:13:09 hno Exp $ + * $Id: delay_pools.c,v 1.1.1.3.12.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 77 Delay Pools * AUTHOR: David Luyer @@ -222,18 +222,18 @@ */ switch (class) { case 1: - delay_data[pool].class1->aggregate = (rates->aggregate.max_bytes * - Config.Delay.initial) / 100; + delay_data[pool].class1->aggregate = (int)(((double)rates->aggregate.max_bytes * + Config.Delay.initial) / 100); break; case 2: - delay_data[pool].class2->aggregate = (rates->aggregate.max_bytes * - Config.Delay.initial) / 100; + delay_data[pool].class2->aggregate = (int)(((double)rates->aggregate.max_bytes * + Config.Delay.initial) / 100); delay_data[pool].class2->individual_map[0] = 255; delay_data[pool].class2->individual_255_used = 0; break; case 3: - delay_data[pool].class3->aggregate = (rates->aggregate.max_bytes * - Config.Delay.initial) / 100; + delay_data[pool].class3->aggregate = (int)(((double)rates->aggregate.max_bytes * + Config.Delay.initial) / 100); delay_data[pool].class3->network_map[0] = 255; delay_data[pool].class3->network_255_used = 0; memset(&delay_data[pool].class3->individual_255_used, '\0', @@ -307,8 +307,8 @@ if (!delay_data[pool].class2->individual_255_used) { delay_data[pool].class2->individual_255_used = 1; delay_data[pool].class2->individual[IND_MAP_SZ - 1] = - (Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100; + (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); } return delayId(pool + 1, 255); } @@ -320,8 +320,8 @@ assert(i < (IND_MAP_SZ - 1)); delay_data[pool].class2->individual_map[i + 1] = 255; delay_data[pool].class2->individual[i] = - (Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100; + (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); break; } } @@ -336,8 +336,8 @@ if (!delay_data[pool].class3->network_255_used) { delay_data[pool].class3->network_255_used = 1; delay_data[pool].class3->network[255] = - (Config.Delay.rates[pool]->network.max_bytes * - Config.Delay.initial) / 100; + (int)(((double)Config.Delay.rates[pool]->network.max_bytes * + Config.Delay.initial) / 100); } } else { for (i = 0; i < NET_MAP_SZ; i++) { @@ -349,8 +349,8 @@ assert(i < (NET_MAP_SZ - 1)); delay_data[pool].class3->network_map[i + 1] = 255; delay_data[pool].class3->network[i] = - (Config.Delay.rates[pool]->network.max_bytes * - Config.Delay.initial) / 100; + (int)(((double)Config.Delay.rates[pool]->network.max_bytes * + Config.Delay.initial) / 100); break; } } @@ -362,8 +362,8 @@ delay_data[pool].class3->individual_255_used[i / 8] |= (1 << (i % 8)); assert(position < C3_IND_SZ); delay_data[pool].class3->individual[position] = - (Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100; + (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); } return delayId(pool + 1, position); } @@ -380,8 +380,8 @@ position |= j; assert(position < C3_IND_SZ); delay_data[pool].class3->individual[position] = - (Config.Delay.rates[pool]->individual.max_bytes * - Config.Delay.initial) / 100; + (int)(((double)Config.Delay.rates[pool]->individual.max_bytes * + Config.Delay.initial) / 100); break; } } Index: squid/src/disk.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/disk.c,v retrieving revision 1.1.1.3.10.1 retrieving revision 1.1.1.3.10.2 diff -u -r1.1.1.3.10.1 -r1.1.1.3.10.2 --- squid/src/disk.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.10.1 +++ squid/src/disk.c 3 May 2000 19:18:12 -0000 1.1.1.3.10.2 @@ -1,6 +1,6 @@ /* - * $Id: disk.c,v 1.1.1.3.10.1 2000/04/17 00:13:09 hno Exp $ + * $Id: disk.c,v 1.1.1.3.10.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 6 Disk I/O Routines * AUTHOR: Harvest Derived @@ -187,6 +187,8 @@ debug(6, 3) ("diskHandleWrite: FD %d writing %d bytes\n", fd, (int) (fdd->write_q->len - fdd->write_q->buf_offset)); errno = 0; + if (fdd->write_q->file_offset != -1) + lseek(fd, fdd->write_q->file_offset, SEEK_SET); len = write(fd, fdd->write_q->buf + fdd->write_q->buf_offset, fdd->write_q->len - fdd->write_q->buf_offset); --- squid/src/diskd.c Wed Feb 14 00:43:58 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,871 +0,0 @@ - -#include "config.h" -#include "squid.h" - - -#include -#include -#include - -#undef assert -#include - -enum { - _MQD_NOP, - _MQD_OPEN, - _MQD_CLOSE, - _MQD_READ, - _MQD_WRITE, - _MQD_UNLINK -}; - -typedef struct _diomsg { - mtyp_t mtype; - int id; - int seq_no; - void *callback_data; - int size; - int offset; - int status; - int shm_offset; -} diomsg; - -static const int msg_snd_rcv_sz = sizeof(diomsg) - sizeof(mtyp_t); - -#if DISKD_DAEMON - -#define STDERR_DEBUG 0 - -typedef struct _file_state file_state; - -struct _file_state { - void *key; - file_state *next; - int id; - int fd; - off_t offset; -}; - -static hash_table *hash = NULL; -static pid_t mypid; -static char *shmbuf; - -static int -do_open(diomsg * r, int len, const char *buf) -{ - int fd; - file_state *fs; - /* - * note r->offset holds open() flags - */ - fd = open(buf, r->offset, 0600); - if (fd < 0) { - fprintf(stderr, "%d %s: ", (int) mypid, buf); - perror("open"); - return -errno; - } - fs = xcalloc(1, sizeof(*fs)); - fs->id = r->id; - fs->key = &fs->id; /* gack */ - fs->fd = fd; - hash_join(hash, (hash_link *) fs); -#if STDERR_DEBUG - fprintf(stderr, "%d OPEN id %d, FD %d, fs %p\n", - (int) mypid, - fs->id, - fs->fd, - fs); -#endif - return fd; -} - -static int -do_close(diomsg * r, int len) -{ - int fd; - file_state *fs; - fs = (file_state *) hash_lookup(hash, &r->id); - if (NULL == fs) { - errno = EBADF; - fprintf(stderr, "%d CLOSE id %d: ", (int) mypid, r->id); - perror("do_close"); - return -EBADF; - } - fd = fs->fd; - hash_remove_link(hash, (hash_link *) fs); -#if STDERR_DEBUG - fprintf(stderr, "%d CLOSE id %d, FD %d, fs %p\n", - (int) mypid, - r->id, - fs->fd, - fs); -#endif - xfree(fs); - return close(fd); -} - -static int -do_read(diomsg * r, int len, char *buf) -{ - int x; - int readlen = r->size; - file_state *fs; - fs = (file_state *) hash_lookup(hash, &r->id); - if (NULL == fs) { - errno = EBADF; - fprintf(stderr, "%d READ id %d: ", (int) mypid, r->id); - perror("do_read"); - return -EBADF; - } - if (r->offset > -1 && r->offset != fs->offset) { -#if STDERR_DEBUG - fprintf(stderr, "seeking to %d\n", r->offset); -#endif - if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { - fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); - perror("lseek"); - } - } - x = read(fs->fd, buf, readlen); -#if STDERR_DEBUG - fprintf(stderr, "%d READ %d,%d,%d ret %d\n", (int) mypid, - fs->fd, readlen, r->offset, x); -#endif - if (x < 0) { - fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); - perror("read"); - return -errno; - } - fs->offset = r->offset + x; - return x; -} - -static int -do_write(diomsg * r, int len, const char *buf) -{ - int wrtlen = r->size; - int x; - file_state *fs; - fs = (file_state *) hash_lookup(hash, &r->id); - if (NULL == fs) { - errno = EBADF; - fprintf(stderr, "%d WRITE id %d: ", (int) mypid, r->id); - perror("do_write"); - return -EBADF; - } - if (r->offset > -1 && r->offset != fs->offset) { - if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { - fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); - perror("lseek"); - } - } -#if STDERR_DEBUG - fprintf(stderr, "%d WRITE %d,%d,%d\n", (int) mypid, - fs->fd, wrtlen, r->offset); -#endif - x = write(fs->fd, buf, wrtlen); - if (x < 0) { - fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); - perror("write"); - return -errno; - } - fs->offset = r->offset + x; - return x; -} - -static int -do_unlink(diomsg * r, int len, const char *buf) -{ - if (truncate(buf, 0) < 0) { - fprintf(stderr, "%d UNLNK id %d %s: ", (int) mypid, r->id, buf); - perror("truncate"); - return -errno; - } -#if STDERR_DEBUG - fprintf(stderr, "%d UNLNK %s\n", (int) mypid, buf); -#endif - return 0; -} - -static void -msg_handle(diomsg * r, int rl, diomsg * s) -{ - char *buf = NULL; - s->mtype = r->mtype; - s->callback_data = r->callback_data; - s->shm_offset = r->shm_offset; - s->id = r->id; - if (s->shm_offset > -1) - buf = shmbuf + s->shm_offset; - switch (r->mtype) { - case _MQD_OPEN: - s->status = do_open(r, rl, buf); - break; - case _MQD_CLOSE: - s->status = do_close(r, rl); - break; - case _MQD_READ: - s->status = do_read(r, rl, buf); - break; - case _MQD_WRITE: - s->status = do_write(r, rl, buf); - break; - case _MQD_UNLINK: - s->status = do_unlink(r, rl, buf); - break; - default: - assert(0); - break; - } -} - -int -fsCmp(const void *a, const void *b) -{ - const int *A = a; - const int *B = b; - return *A != *B; -} - -unsigned int -fsHash(const void *key, unsigned int n) -{ - /* note, n must be a power of 2! */ - const int *k = key; - return (*k & (--n)); -} - -static void -alarm_handler(int sig) -{ - (void) 0; -} - -int -main(int argc, char *argv[]) -{ - int key; - int rmsgid; - int smsgid; - int shmid; - diomsg rmsg; - diomsg smsg; - int rlen; - char rbuf[512]; - struct sigaction sa; - setbuf(stdout, NULL); - setbuf(stderr, NULL); - mypid = getpid(); - assert(4 == argc); - key = atoi(argv[1]); - rmsgid = msgget(key, 0600); - if (rmsgid < 0) { - perror("msgget"); - return 1; - } - key = atoi(argv[2]); - smsgid = msgget(key, 0600); - if (smsgid < 0) { - perror("msgget"); - return 1; - } - key = atoi(argv[3]); - shmid = shmget(key, 0, 0600); - if (shmid < 0) { - perror("shmget"); - return 1; - } - shmbuf = shmat(shmid, NULL, 0); - if (shmbuf == (void *) -1) { - perror("shmat"); - return 1; - } - hash = hash_create(fsCmp, 1 << 4, fsHash); - assert(hash); - fcntl(0, F_SETFL, SQUID_NONBLOCK); - memset(&sa, '\0', sizeof(sa)); - sa.sa_handler = alarm_handler; - sa.sa_flags = SA_RESTART; - sigaction(SIGALRM, &sa, NULL); - for (;;) { - alarm(1); - memset(&rmsg, '\0', sizeof(rmsg)); - rlen = msgrcv(rmsgid, &rmsg, msg_snd_rcv_sz, 0, 0); - if (rlen < 0) { - if (EINTR == errno) { - if (read(0, rbuf, 512) <= 0) { - if (EWOULDBLOCK == errno) - (void) 0; - else if (EAGAIN == errno) - (void) 0; - else - break; - } - } - if (EAGAIN == errno) { - continue; - } - perror("msgrcv"); - break; - } - alarm(0); - msg_handle(&rmsg, rlen, &smsg); - if (msgsnd(smsgid, &smsg, msg_snd_rcv_sz, 0) < 0) { - perror("msgsnd"); - break; - } - } -#if STDERR_DEBUG - fprintf(stderr, "%d diskd exiting\n", (int) mypid); -#endif - if (msgctl(rmsgid, IPC_RMID, 0) < 0) - perror("msgctl IPC_RMID"); - if (msgctl(smsgid, IPC_RMID, 0) < 0) - perror("msgctl IPC_RMID"); - if (shmdt(shmbuf) < 0) - perror("shmdt"); - if (shmctl(shmid, IPC_RMID, 0) < 0) - perror("shmctl IPC_RMID"); - return 0; -} - -#elif USE_DISKD - -/* - * DEBUG: section 81 Diskd Interface functions - */ - -static int sent_count = 0; -static int recv_count = 0; -static int shmbuf_count = 0; -static int sio_id = 0; - -static int storeDiskdSend(int, SwapDir *, int, storeIOState *, int, int, int); -static void storeDiskdShmPut(SwapDir *, int); -static void *storeDiskdShmGet(SwapDir *, int *); -static void storeDiskdHandle(diomsg * M); -static void storeDiskdIOCallback(storeIOState * sio, int errflag); -static int storeDiskdReadIndividualQueue(SwapDir * sd); -static SwapDir *swapDirFromFileno(sfileno f); -static OBJH storeDiskdStats; - -/* - * SHMBUFS is the number of shared memory buffers to allocate for - * Each SwapDir. - */ -#define SHMBUFS 96 -#define SHMBUF_BLKSZ DISK_PAGE_SIZE -/* - * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv. - * If a queue has MAGIC2 (or more) messages away, then we read the - * queue until the level falls below MAGIC2. Recommended value - * is 75% of SHMBUFS. - */ -#define MAGIC1 Config.diskd.magic1 -#define MAGIC2 Config.diskd.magic2 - -struct { - int open_fail_queue_len; - int block_queue_len; - int max_away; - int max_shmuse; -} diskd_stats; - - -/* === PUBLIC =========================================================== */ - -storeIOState * -storeDiskdOpen(sfileno f, mode_t mode, STIOCB * callback, void *callback_data) -{ - int x; - storeIOState *sio; - char *buf; - int shm_offset; - SwapDir *sd = swapDirFromFileno(f); - debug(81, 3) ("storeDiskdOpen: fileno %08X, mode %d\n", f, mode); - /* - * XXX Eventually there should be an option here to fail on open() - * If there are too many requests queued. - */ - if (sd->u.diskd.away > MAGIC1) { - diskd_stats.open_fail_queue_len++; - return NULL; - } - assert(mode == O_RDONLY || mode == O_WRONLY); - sio = memAllocate(MEM_STORE_IO); - cbdataAdd(sio, memFree, MEM_STORE_IO); - sio->swap_file_number = f; - sio->mode = mode; - sio->callback = callback; - sio->callback_data = callback_data; - cbdataLock(sio->callback_data); - sio->type.diskd.id = sio_id++; - if (mode == O_WRONLY) - mode |= (O_CREAT | O_TRUNC); - buf = storeDiskdShmGet(sd, &shm_offset); - storeUfsFullPath(f, buf); - x = storeDiskdSend(_MQD_OPEN, - sd, - sio->type.diskd.id, - sio, - strlen(buf) + 1, - mode, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror()); - storeDiskdShmPut(sd, shm_offset); - cbdataUnlock(sio->callback_data); - cbdataFree(sio); - return NULL; - } - return sio; -} - -void -storeDiskdClose(storeIOState * sio) -{ - int x; - debug(81, 3) ("storeDiskdClose: fileno %08X\n", sio->swap_file_number); - x = storeDiskdSend(_MQD_CLOSE, - swapDirFromFileno(sio->swap_file_number), - sio->type.diskd.id, - sio, - 0, - 0, - -1); - if (x < 0) { - debug(50, 1) ("storeDiskdSend CLOSE: %s\n", xstrerror()); - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -void -storeDiskdRead(storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *their_data) -{ - int x; - int shm_offset; - char *rbuf; - SwapDir *sd = swapDirFromFileno(sio->swap_file_number); - if (!cbdataValid(sio)) - return; - if (sio->type.diskd.flags.reading) { - debug(81, 1) ("storeDiskdRead: already reading!\n"); - return; - } - assert(sio->read.callback == NULL); - assert(sio->read.callback_data == NULL); - sio->read.callback = callback; - sio->read.callback_data = their_data; - sio->type.diskd.read_buf = buf; /* the one passed from above */ - cbdataLock(sio->read.callback_data); - debug(81, 3) ("storeDiskdRead: fileno %08X\n", sio->swap_file_number); - sio->offset = offset; - sio->type.diskd.flags.reading = 1; - rbuf = storeDiskdShmGet(sd, &shm_offset); - assert(rbuf); - x = storeDiskdSend(_MQD_READ, - sd, - sio->type.diskd.id, - sio, - (int) size, - (int) offset, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend READ: %s\n", xstrerror()); - storeDiskdShmPut(sd, shm_offset); - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -void -storeDiskdWrite(storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) -{ - int x; - char *sbuf; - int shm_offset; - SwapDir *sd = swapDirFromFileno(sio->swap_file_number); - debug(81, 3) ("storeDiskdWrite: fileno %08X\n", sio->swap_file_number); - if (!cbdataValid(sio)) { - free_func(buf); - return; - } - sio->type.diskd.flags.writing = 1; - sbuf = storeDiskdShmGet(sd, &shm_offset); - xmemcpy(sbuf, buf, size); - free_func(buf); - x = storeDiskdSend(_MQD_WRITE, - sd, - sio->type.diskd.id, - sio, - (int) size, - (int) offset, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend WRITE: %s\n", xstrerror()); - storeDiskdShmPut(sd, shm_offset); - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -void -storeDiskdUnlink(sfileno f) -{ - int x; - int shm_offset; - char *buf; - SwapDir *sd = swapDirFromFileno(f); - debug(81, 3) ("storeDiskdUnlink: fileno %08X\n", f); - buf = storeDiskdShmGet(sd, &shm_offset); - storeUfsFullPath(f, buf); - x = storeDiskdSend(_MQD_UNLINK, - sd, - f, - NULL, - 0, - 0, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend UNLINK: %s\n", xstrerror()); - unlink(buf); - storeDiskdShmPut(sd, shm_offset); - } -} - -void -storeDiskdInit(SwapDir * sd) -{ - static int first = 0; - int x; - int i; - int rfd; - int ikey = (getpid() << 16) + (sd->index << 4); - char *args[5]; - char skey1[32]; - char skey2[32]; - char skey3[32]; - storeUfsDirInit(sd); - sd->u.diskd.smsgid = msgget((key_t) ikey, 0700 | IPC_CREAT); - if (sd->u.diskd.smsgid < 0) { - debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); - fatal("msgget failed"); - } - sd->u.diskd.rmsgid = msgget((key_t) (ikey + 1), 0700 | IPC_CREAT); - if (sd->u.diskd.rmsgid < 0) { - debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); - fatal("msgget failed"); - } - sd->u.diskd.shm.id = shmget((key_t) (ikey + 2), - SHMBUFS * SHMBUF_BLKSZ, 0600 | IPC_CREAT); - if (sd->u.diskd.shm.id < 0) { - debug(50, 0) ("storeDiskdInit: shmget: %s\n", xstrerror()); - fatal("shmget failed"); - } - sd->u.diskd.shm.buf = shmat(sd->u.diskd.shm.id, NULL, 0); - if (sd->u.diskd.shm.buf == (void *) -1) { - debug(50, 0) ("storeDiskdInit: shmat: %s\n", xstrerror()); - fatal("shmat failed"); - } - shmbuf_count += SHMBUFS; - for (i = 0; i < SHMBUFS; i++) - storeDiskdShmPut(sd, i * SHMBUF_BLKSZ); - snprintf(skey1, 32, "%d", ikey); - snprintf(skey2, 32, "%d", ikey + 1); - snprintf(skey3, 32, "%d", ikey + 2); - args[0] = "diskd"; - args[1] = skey1; - args[2] = skey2; - args[3] = skey3; - args[4] = NULL; -#if HAVE_POLL && defined(_SQUID_OSF_) - /* pipes and poll() don't get along on DUNIX -DW */ - x = ipcCreate(IPC_TCP_SOCKET, -#else - x = ipcCreate(IPC_FIFO, -#endif - "/usr/local/squid/bin/diskd", - args, - "diskd", - &rfd, - &sd->u.diskd.wfd); - if (x < 0) - fatal("execl /usr/local/squid/bin/diskd failed"); - if (rfd != sd->u.diskd.wfd) - comm_close(rfd); - fd_note(sd->u.diskd.wfd, "squid -> diskd"); - commSetTimeout(sd->u.diskd.wfd, -1, NULL, NULL); - commSetNonBlocking(sd->u.diskd.wfd); - debug(81, 1) ("diskd started\n"); - if (0 == first) { - first++; - memset(&diskd_stats, '\0', sizeof(diskd_stats)); - cachemgrRegister("diskd", "DISKD Stats", storeDiskdStats, 0, 1); - } -} - -void -storeDiskdReadQueue(void) -{ - SwapDir *sd; - int i; - int j; - static int ndir = 0; - if (sent_count - recv_count > diskd_stats.max_away) { - diskd_stats.max_away = sent_count - recv_count; - diskd_stats.max_shmuse = shmbuf_count; - } - do { - j = 0; - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - if (ndir >= Config.cacheSwap.n_configured) - ndir = ndir % Config.cacheSwap.n_configured; - sd = &Config.cacheSwap.swapDirs[ndir++]; - if (sd->type != SWAPDIR_DISKD) - continue; - if (0 == sd->u.diskd.away) - continue; - /* stay in this loop if one queue is over the limit */ - if (MAGIC2 <= sd->u.diskd.away) - j++; - j += storeDiskdReadIndividualQueue(sd); - } - } while (j > 0); - ndir++; -} - - -/* === STATIC =========================================================== */ - -static void -storeDiskdOpenDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - Counter.syscalls.disk.opens++; - debug(81, 3) ("storeDiskdOpenDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -static void -storeDiskdCloseDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - Counter.syscalls.disk.closes++; - debug(81, 3) ("storeDiskdCloseDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - return; - } - storeDiskdIOCallback(sio, DISK_OK); -} - -static void -storeDiskdReadDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - STRCB *callback = sio->read.callback; - void *their_data = sio->read.callback_data; - char *their_buf = sio->type.diskd.read_buf; - char *sbuf; - size_t len; - SwapDir *sd = swapDirFromFileno(sio->swap_file_number); - int valid; - Counter.syscalls.disk.reads++; - sio->type.diskd.flags.reading = 0; - valid = cbdataValid(sio->read.callback_data); - cbdataUnlock(sio->read.callback_data); - debug(81, 3) ("storeDiskdReadDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - return; - } - sbuf = sd->u.diskd.shm.buf + M->shm_offset; - len = M->status; - xmemcpy(their_buf, sbuf, len); /* yucky copy */ - sio->offset += len; - assert(callback); - assert(their_data); - sio->read.callback = NULL; - sio->read.callback_data = NULL; - if (valid) - callback(their_data, their_buf, len); -} - -static void -storeDiskdWriteDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - Counter.syscalls.disk.writes++; - sio->type.diskd.flags.writing = 0; - debug(81, 3) ("storeDiskdWriteDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - return; - } - sio->offset += M->status; -} - -static void -storeDiskdUnlinkDone(diomsg * M) -{ - debug(81, 3) ("storeDiskdUnlinkDone: fileno %08x status %d\n", - M->id, M->status); - Counter.syscalls.disk.unlinks++; -} - -static void -storeDiskdHandle(diomsg * M) -{ - int valid = M->callback_data ? cbdataValid(M->callback_data) : 1; - if (M->callback_data) - cbdataUnlock(M->callback_data); - if (!valid) { - debug(81, 3) ("storeDiskdHandle: Invalid callback_data %p\n", - M->callback_data); - /* - * The read operation has its own callback. If we don't - * call storeDiskdReadDone(), then we must make sure the - * callback_data gets unlocked! - */ - if (_MQD_READ == M->mtype) { - storeIOState *sio = M->callback_data; - cbdataUnlock(sio->read.callback_data); - } - return; - } - switch (M->mtype) { - case _MQD_OPEN: - storeDiskdOpenDone(M); - break; - case _MQD_CLOSE: - storeDiskdCloseDone(M); - break; - case _MQD_READ: - storeDiskdReadDone(M); - break; - case _MQD_WRITE: - storeDiskdWriteDone(M); - break; - case _MQD_UNLINK: - storeDiskdUnlinkDone(M); - break; - default: - assert(0); - break; - } -} - -static void -storeDiskdIOCallback(storeIOState * sio, int errflag) -{ - int valid = cbdataValid(sio->callback_data); - debug(81, 3) ("storeUfsIOCallback: errflag=%d\n", errflag); - cbdataUnlock(sio->callback_data); - if (valid) - sio->callback(sio->callback_data, errflag, sio); - cbdataFree(sio); -} - -static int -storeDiskdSend(int mtype, SwapDir * sd, int id, storeIOState * sio, int size, int offset, int shm_offset) -{ - int x; - diomsg M; - static int send_errors = 0; - static int last_seq_no = 0; - static int seq_no = 0; - M.mtype = mtype; - M.callback_data = sio; - M.size = size; - M.offset = offset; - M.status = -1; - M.shm_offset = shm_offset; - M.id = id; - M.seq_no = ++seq_no; - if (M.callback_data) - cbdataLock(M.callback_data); - if (M.seq_no < last_seq_no) - debug(81, 1) ("WARNING: sequencing out of order\n"); - x = msgsnd(sd->u.diskd.smsgid, &M, msg_snd_rcv_sz, IPC_NOWAIT); - last_seq_no = M.seq_no; - if (0 == x) { - sent_count++; - sd->u.diskd.away++; - } else { - debug(50, 1) ("storeDiskdSend: msgsnd: %s\n", xstrerror()); - if (M.callback_data) - cbdataUnlock(M.callback_data); - assert(++send_errors < 100); - } - if (sd->u.diskd.away > MAGIC2) { - diskd_stats.block_queue_len++; - storeDiskdReadQueue(); - } - return x; -} - -static void * -storeDiskdShmGet(SwapDir * sd, int *shm_offset) -{ - char *buf; - buf = linklistShift(&sd->u.diskd.shm.stack); - assert(buf); - *shm_offset = buf - sd->u.diskd.shm.buf; - assert(0 <= *shm_offset && *shm_offset < SHMBUFS * SHMBUF_BLKSZ); - shmbuf_count++; - return buf; -} - -static void -storeDiskdShmPut(SwapDir * sd, int offset) -{ - char *buf; - assert(offset >= 0); - assert(offset < SHMBUFS * SHMBUF_BLKSZ); - buf = sd->u.diskd.shm.buf + offset; - linklistPush(&sd->u.diskd.shm.stack, buf); - shmbuf_count--; -} - -static int -storeDiskdReadIndividualQueue(SwapDir * sd) -{ - diomsg M; - int x; - memset(&M, '\0', sizeof(M)); - x = msgrcv(sd->u.diskd.rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT); - if (x < 0) - return 0; - if (x != msg_snd_rcv_sz) { - debug(81, 1) ("storeDiskdReadIndividualQueue: msgget returns %d\n", x); - return 0; - } - recv_count++; - sd->u.diskd.away--; - storeDiskdHandle(&M); - if (M.shm_offset > -1) - storeDiskdShmPut(sd, M.shm_offset); - return 1; -} - -static SwapDir * -swapDirFromFileno(sfileno f) -{ - return &Config.cacheSwap.swapDirs[f >> SWAP_DIR_SHIFT]; -} - -static void -storeDiskdStats(StoreEntry * sentry) -{ - storeAppendPrintf(sentry, "max_away: %d\n", diskd_stats.max_away); - storeAppendPrintf(sentry, "max_shmuse: %d\n", diskd_stats.max_shmuse); - storeAppendPrintf(sentry, "open_fail_queue_len: %d\n", diskd_stats.open_fail_queue_len); - storeAppendPrintf(sentry, "block_queue_len: %d\n", diskd_stats.block_queue_len); - diskd_stats.max_away = diskd_stats.max_shmuse = 0; -} - -#endif Index: squid/src/dns_internal.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/dns_internal.c,v retrieving revision 1.1.1.1.4.1.2.1 retrieving revision 1.1.1.1.4.1.2.2 diff -u -r1.1.1.1.4.1.2.1 -r1.1.1.1.4.1.2.2 --- squid/src/dns_internal.c 17 Apr 2000 00:13:09 -0000 1.1.1.1.4.1.2.1 +++ squid/src/dns_internal.c 3 May 2000 19:18:12 -0000 1.1.1.1.4.1.2.2 @@ -1,6 +1,6 @@ /* - * $Id: dns_internal.c,v 1.1.1.1.4.1.2.1 2000/04/17 00:13:09 hno Exp $ + * $Id: dns_internal.c,v 1.1.1.1.4.1.2.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 78 DNS lookups; interacts with lib/rfc1035.c * AUTHOR: Duane Wessels @@ -44,8 +44,6 @@ #define DOMAIN_PORT 53 #endif -#define IDNS_MAX_TRIES 20 - typedef struct _idns_query idns_query; typedef struct _ns ns; @@ -143,13 +141,13 @@ t = strtok(buf, w_space); if (NULL == t) continue; - if (strcasecmp(t, "nameserver")) - continue; - t = strtok(NULL, w_space); - if (t == NULL) - continue;; - debug(78, 1) ("Adding nameserver %s from %s\n", t, _PATH_RESOLV_CONF); - idnsAddNameserver(t); + if (strcasecmp(t, "nameserver") == 0) { + t = strtok(NULL, w_space); + if (t == NULL) + continue;; + debug(78, 1) ("Adding nameserver %s from %s\n", t, _PATH_RESOLV_CONF); + idnsAddNameserver(t); + } } fclose(fp); } @@ -207,21 +205,24 @@ assert(nns > 0); assert(q->lru.next == NULL); assert(q->lru.prev == NULL); +try_again: ns = q->nsends % nns; x = comm_udp_sendto(DnsSocket, &nameservers[ns].S, sizeof(nameservers[ns].S), q->buf, q->sz); + q->nsends++; + q->sent_t = current_time; if (x < 0) { debug(50, 1) ("idnsSendQuery: FD %d: sendto: %s\n", DnsSocket, xstrerror()); + if (q->nsends % nns != 0) + goto try_again; } else { fd_bytes(DnsSocket, x, FD_WRITE); commSetSelect(DnsSocket, COMM_SELECT_READ, idnsRead, NULL, 0); } - q->nsends++; - q->sent_t = current_time; nameservers[ns].nqueries++; dlinkAdd(q, &q->lru, &lru_list); idnsTickleQueue(); @@ -353,13 +354,13 @@ event_queued = 0; for (n = lru_list.tail; n; n = p) { q = n->data; - if (tvSubDsec(q->sent_t, current_time) < 5.0) + if (tvSubDsec(q->sent_t, current_time) < Config.Timeout.idns_retransmit * ( 1 << q->nsends % nns)) break; debug(78, 3) ("idnsCheckQueue: ID %#04x timeout\n", q->id); p = n->prev; dlinkDelete(&q->lru, &lru_list); - if (q->nsends < IDNS_MAX_TRIES) { + if (tvSubDsec(q->start_t, current_time) < Config.Timeout.idns_query) { idnsSendQuery(q); } else { int v = cbdataValid(q->callback_data); Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.1.1.3.12.2 retrieving revision 1.1.1.3.12.3 diff -u -r1.1.1.3.12.2 -r1.1.1.3.12.3 --- squid/src/enums.h 3 May 2000 11:22:56 -0000 1.1.1.3.12.2 +++ squid/src/enums.h 3 May 2000 19:18:12 -0000 1.1.1.3.12.3 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.1.1.3.12.2 2000/05/03 11:22:56 asd Exp $ + * $Id: enums.h,v 1.1.1.3.12.3 2000/05/03 19:18:12 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -108,17 +108,20 @@ ACL_MY_PORT, #if USE_IDENT ACL_IDENT, + ACL_IDENT_REGEX, #endif ACL_PROTO, ACL_METHOD, ACL_BROWSER, ACL_PROXY_AUTH, + ACL_PROXY_AUTH_REGEX, ACL_SRC_ASN, ACL_DST_ASN, ACL_SRC_ARP, ACL_SNMP_COMMUNITY, ACL_NETDB_SRC_RTT, ACL_MAXCONN, + ACL_REQ_MIME_TYPE, ACL_ENUM_MAX } squid_acl; @@ -505,7 +508,6 @@ MEM_ACL_PROXY_AUTH_DATA, MEM_ACL_PROXY_AUTH_USER, MEM_ACL_TIME_DATA, - MEM_AIO_RESULT_T, MEM_CACHEMGR_PASSWD, #if USE_CACHE_DIGESTS MEM_CACHE_DIGEST, @@ -519,7 +521,6 @@ #if USE_CACHE_DIGESTS MEM_DIGEST_FETCH_STATE, #endif - MEM_DISK_BUF, MEM_DLINK_LIST, MEM_DLINK_NODE, MEM_DNSSERVER_T, @@ -569,8 +570,10 @@ #if USE_CACHE_DIGESTS MEM_PEER_DIGEST, #endif +#if USE_ICMP MEM_PINGERECHODATA, MEM_PINGERREPLYDATA, +#endif MEM_PS_STATE, MEM_REFRESH_T, MEM_RELIST, @@ -656,12 +659,3 @@ NETDB_EX_RTT, NETDB_EX_HOPS }; - -typedef enum { - SWAPDIR_UFS, - SWAPDIR_ASYNCUFS, -#if USE_DISKD - SWAPDIR_DISKD, -#endif - SWAPDIR_MAX -} swapdir_t; Index: squid/src/event.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/event.c,v retrieving revision 1.1.1.3.10.1 retrieving revision 1.1.1.3.10.2 diff -u -r1.1.1.3.10.1 -r1.1.1.3.10.2 --- squid/src/event.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.10.1 +++ squid/src/event.c 3 May 2000 19:18:12 -0000 1.1.1.3.10.2 @@ -1,6 +1,6 @@ /* - * $Id: event.c,v 1.1.1.3.10.1 2000/04/17 00:13:09 hno Exp $ + * $Id: event.c,v 1.1.1.3.10.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 41 Event Processing * AUTHOR: Henrik Nordstrom @@ -191,6 +191,7 @@ { struct ev_entry *event; while ((event = tasks)) { + tasks = event->next; if (NULL != event->arg) cbdataUnlock(event->arg); memFree(event, MEM_EVENT); Index: squid/src/fd.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fd.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/fd.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.12.1 +++ squid/src/fd.c 3 May 2000 19:18:12 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: fd.c,v 1.1.1.3.12.1 2000/04/17 00:13:09 hno Exp $ + * $Id: fd.c,v 1.1.1.3.12.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 51 Filedescriptor Functions * AUTHOR: Duane Wessels @@ -95,12 +95,6 @@ { fde *F = &fd_table[fd]; assert(fd >= 0); -#if USE_ASYNC_IO - if (F->flags.closing) { - /* Reuse of a closed FD before we have noticed it is closed */ - fd_close(fd); - } -#endif if (F->flags.open) { debug(51, 1) ("WARNING: Closing open FD %4d\n", fd); fd_close(fd); Index: squid/src/filemap.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/filemap.c,v retrieving revision 1.1.1.2.10.1 retrieving revision 1.1.1.2.10.2 diff -u -r1.1.1.2.10.1 -r1.1.1.2.10.2 --- squid/src/filemap.c 17 Apr 2000 00:13:09 -0000 1.1.1.2.10.1 +++ squid/src/filemap.c 3 May 2000 19:18:12 -0000 1.1.1.2.10.2 @@ -1,6 +1,6 @@ /* - * $Id: filemap.c,v 1.1.1.2.10.1 2000/04/17 00:13:09 hno Exp $ + * $Id: filemap.c,v 1.1.1.2.10.2 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 8 Swap File Bitmap * AUTHOR: Harvest Derived @@ -133,7 +133,7 @@ for (bit = 0; bit < BITS_IN_A_LONG; bit++) { suggestion = ((unsigned long) word << LONG_BIT_SHIFT) | bit; if (!file_map_bit_test(fm, suggestion)) { - return file_map_bit_set(fm, suggestion); + return suggestion; } } debug(8, 3) ("growing from file_map_allocate\n"); Index: squid/src/forward.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/forward.c,v retrieving revision 1.1.1.3.12.2 retrieving revision 1.1.1.3.12.3 diff -u -r1.1.1.3.12.2 -r1.1.1.3.12.3 --- squid/src/forward.c 17 Apr 2000 00:13:09 -0000 1.1.1.3.12.2 +++ squid/src/forward.c 3 May 2000 19:18:12 -0000 1.1.1.3.12.3 @@ -1,6 +1,6 @@ /* - * $Id: forward.c,v 1.1.1.3.12.2 2000/04/17 00:13:09 hno Exp $ + * $Id: forward.c,v 1.1.1.3.12.3 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 17 Request Forwarding * AUTHOR: Duane Wessels @@ -132,13 +132,20 @@ fwdState->n_tries, (int) (squid_curtime - fwdState->start)); if (fwdState->servers->next) { - /* cycle */ + /* use next, or cycle if origin server isn't last */ FwdServer *fs = fwdState->servers; - FwdServer **T; + FwdServer **T, *T2 = NULL; fwdState->servers = fs->next; - for (T = &fwdState->servers; *T; T = &(*T)->next); - *T = fs; - fs->next = NULL; + for (T = &fwdState->servers; *T; T2=*T, T = &(*T)->next); + if (T2 && T2->peer) { + /* cycle */ + *T = fs; + fs->next = NULL; + } else { + /* Use next. The last "direct" entry is retried multiple times */ + fwdState->servers = fs->next; + fwdServerFree(fs); + } } /* use eventAdd to break potential call sequence loops */ eventAdd("fwdConnectStart", fwdConnectStart, fwdState, 0.0, 0); @@ -181,12 +188,20 @@ err->request = requestLink(request); fwdFail(fwdState, err); if (fs->peer) - peerCheckConnectStart(fs->peer); + peerConnectFailed(fs->peer); comm_close(server_fd); } else { debug(17, 3) ("fwdConnectDone: FD %d: '%s'\n", server_fd, storeUrl(fwdState->entry)); + if (fs->peer) + hierarchyNote(&fwdState->request->hier, fs->code, fs->peer->host); + else if (Config.onoff.log_ip_on_direct) + hierarchyNote(&fwdState->request->hier, fs->code, fd_table[server_fd].ipaddr); + else + hierarchyNote(&fwdState->request->hier, fs->code, request->host); fd_note(server_fd, storeUrl(fwdState->entry)); fd_table[server_fd].uses++; + if (fs->peer) + peerConnectSucceded(fs->peer); fwdDispatch(fwdState); } current = NULL; @@ -210,7 +225,7 @@ */ if (fwdState->servers) if (fwdState->servers->peer) - peerCheckConnectStart(fwdState->servers->peer); + peerConnectFailed(fwdState->servers->peer); } comm_close(fd); } @@ -234,12 +249,16 @@ port = fs->peer->http_port; ctimeout = fs->peer->connect_timeout > 0 ? fs->peer->connect_timeout : Config.Timeout.peer_connect; + } else if (fwdState->request->flags.accelerated && + Config.Accel.single_host && Config.Accel.host) { + host = Config.Accel.host; + port = Config.Accel.port; + ctimeout = Config.Timeout.connect; } else { host = fwdState->request->host; port = fwdState->request->port; ctimeout = Config.Timeout.connect; } - hierarchyNote(&fwdState->request->hier, fs->code, host); if ((fd = pconnPop(host, port)) >= 0) { debug(17, 3) ("fwdConnectStart: reusing pconn FD %d\n", fd); fwdState->server_fd = fd; Index: squid/src/ftp.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ftp.c,v retrieving revision 1.1.1.3.4.1.2.2 retrieving revision 1.1.1.3.4.1.2.3 diff -u -r1.1.1.3.4.1.2.2 -r1.1.1.3.4.1.2.3 --- squid/src/ftp.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.4.1.2.2 +++ squid/src/ftp.c 3 May 2000 19:18:12 -0000 1.1.1.3.4.1.2.3 @@ -1,6 +1,6 @@ /* - * $Id: ftp.c,v 1.1.1.3.4.1.2.2 2000/04/17 00:13:10 hno Exp $ + * $Id: ftp.c,v 1.1.1.3.4.1.2.3 2000/05/03 19:18:12 hno Exp $ * * DEBUG: section 9 File Transfer Protocol (FTP) * AUTHOR: Harvest Derived @@ -899,7 +899,7 @@ ftpListingStart(ftpState); } if (len < 0) { - debug(9, 1) ("ftpDataRead: read error: %s\n", xstrerror()); + debug(50, ignoreErrno(errno) ? 3 : 1) ("ftpDataRead: read error: %s\n", xstrerror()); if (ignoreErrno(errno)) { commSetSelect(fd, COMM_SELECT_READ, @@ -1247,7 +1247,7 @@ } debug(9, 5) ("ftpReadControlReply: FD %d, Read %d bytes\n", fd, len); if (len < 0) { - debug(9, 1) ("ftpReadControlReply: read error: %s\n", xstrerror()); + debug(50, ignoreErrno(errno) ? 3 : 1) ("ftpReadControlReply: read error: %s\n", xstrerror()); if (ignoreErrno(errno)) { ftpScheduleReadControlReply(ftpState, 0); } else { @@ -1303,11 +1303,15 @@ xmemmove(ftpState->ctrl.buf, ftpState->ctrl.buf + bytes_used, ftpState->ctrl.offset); } - /* Find the last line of the reply message */ + /* Move the last line of the reply message to ctrl.last_reply */ for (W = &ftpState->ctrl.message; (*W)->next; W = &(*W)->next); safe_free(ftpState->ctrl.last_reply); - ftpState->ctrl.last_reply = (*W)->key; - safe_free(*W); + ftpState->ctrl.last_reply = xstrdup((*W)->key); + wordlistDestroy(W); + /* Copy the rest of the message to cwd_message to be printed in + * error messages + */ + wordlistAddWl(&ftpState->cwd_message, ftpState->ctrl.message); debug(9, 8) ("ftpHandleControlReply: state=%d, code=%d\n", ftpState->state, ftpState->ctrl.replycode); FTP_SM_FUNCS[ftpState->state] (ftpState); @@ -1328,10 +1332,6 @@ if (ftpState->ctrl.message) { if (strstr(ftpState->ctrl.message->key, "NetWare")) ftpState->flags.skip_whitespace = 1; - if (ftpState->cwd_message) - wordlistDestroy(&ftpState->cwd_message); - ftpState->cwd_message = ftpState->ctrl.message; - ftpState->ctrl.message = NULL; } ftpSendUser(ftpState); } else if (code == 120) { @@ -1385,12 +1385,6 @@ int code = ftpState->ctrl.replycode; debug(9, 3) ("ftpReadPass\n"); if (code == 230) { - if (ftpState->ctrl.message) { - if (ftpState->cwd_message) - wordlistDestroy(&ftpState->cwd_message); - ftpState->cwd_message = ftpState->ctrl.message; - ftpState->ctrl.message = NULL; - } ftpSendType(ftpState); } else { ftpFail(ftpState); @@ -1516,6 +1510,7 @@ if (code >= 200 && code < 300) { /* CWD OK */ ftpUnhack(ftpState); + /* Reset cwd_message to only include the last message */ if (ftpState->cwd_message) wordlistDestroy(&ftpState->cwd_message); ftpState->cwd_message = ftpState->ctrl.message; @@ -1645,6 +1640,14 @@ int fd; struct sockaddr_in addr; socklen_t addr_len; + if (ftpState->request->method == METHOD_HEAD) { + /* Terminate here for HEAD requests */ + ftpAppendSuccessHeader(ftpState); + storeTimestampsSet(ftpState->entry); + fwdComplete(ftpState->fwd); + ftpSendQuit(ftpState); + return; + } if (ftpState->data.fd >= 0) { if (!ftpState->flags.datachannel_hack) { /* We are already connected, reuse this connection. */ @@ -2390,8 +2393,8 @@ err = errorCon(ERR_FTP_FAILURE, HTTP_BAD_GATEWAY); err->xerrno = errno; err->request = requestLink(ftpState->request); - err->ftp.server_msg = ftpState->ctrl.message; - ftpState->ctrl.message = NULL; + err->ftp.server_msg = ftpState->cwd_message; + ftpState->cwd_message = NULL; if (ftpState->old_request) command = ftpState->old_request; else Index: squid/src/globals.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/globals.h,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/globals.h 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/globals.h 3 May 2000 19:18:12 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: globals.h,v 1.1.1.3.12.2 2000/05/03 19:18:12 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -126,10 +126,8 @@ extern int store_hash_buckets; /* 0 */ extern hash_table *store_table; /* NULL */ #if HEAP_REPLACEMENT -extern heap *store_heap; extern heap *inmem_heap; #else -extern dlink_list store_list; #endif extern dlink_list ClientActiveRequests; extern const String StringNull; /* { 0, 0, NULL } */ @@ -151,3 +149,8 @@ extern request_flags null_request_flags; extern int store_open_disk_fd; /* 0 */ extern const char *SwapDirType[]; +extern storefs_entry_t *storefs_list; /* NULL */ +extern int store_swap_low; +extern int store_swap_high; +extern int store_pages_max; +extern size_t store_maxobjsize; Index: squid/src/helper.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/helper.c,v retrieving revision 1.1.1.3 retrieving revision 1.1.1.3.12.1 diff -u -r1.1.1.3 -r1.1.1.3.12.1 --- squid/src/helper.c 26 Jan 2000 03:25:01 -0000 1.1.1.3 +++ squid/src/helper.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.1 @@ -126,17 +126,18 @@ storeAppendPrintf(sentry, "avg service time: %d msec\n", hlp->stats.avg_svc_time); storeAppendPrintf(sentry, "\n"); - storeAppendPrintf(sentry, "%7s\t%7s\t%11s\t%s\t%7s\t%7s\n", + storeAppendPrintf(sentry, "%7s\t%7s\t%11s\t%s\t%7s\t%7s\t%7s\n", "#", "FD", "# Requests", "Flags", "Time", - "Offset"); + "Offset", + "Request"); for (link = hlp->servers.head; link; link = link->next) { srv = link->data; tt = 0.001 * tvSubMsec(srv->dispatch_time, current_time); - storeAppendPrintf(sentry, "%7d\t%7d\t%11d\t%c%c%c%c\t%7.3f\t%7d\n", + storeAppendPrintf(sentry, "%7d\t%7d\t%11d\t%c%c%c%c\t%7.3f\t%7d\t%s\n", srv->index + 1, srv->rfd, srv->stats.uses, @@ -145,7 +146,8 @@ srv->flags.closing ? 'C' : ' ', srv->flags.shutdown ? 'S' : ' ', tt < 0.0 ? 0.0 : tt, - (int) srv->offset); + (int) srv->offset, + srv->request ? log_quote(srv->request->buf) : "(none)"); } storeAppendPrintf(sentry, "\nFlags key:\n\n"); storeAppendPrintf(sentry, " A = ALIVE\n"); @@ -179,7 +181,8 @@ continue; } srv->flags.closing = 1; - comm_close(srv->rfd); + comm_close(srv->wfd); + srv->wfd = -1; } } @@ -223,7 +226,7 @@ helperRequestFree(r); srv->request = NULL; } - if (srv->wfd != srv->rfd) + if (srv->wfd != srv->rfd && srv->wfd != -1) comm_close(srv->wfd); dlinkDelete(&srv->link, &hlp->servers); hlp->n_running--; @@ -282,9 +285,10 @@ intAverage(hlp->stats.avg_svc_time, tvSubMsec(srv->dispatch_time, current_time), hlp->stats.replies, REDIRECT_AV_FACTOR); - if (srv->flags.shutdown) + if (srv->flags.shutdown) { comm_close(srv->wfd); - else + srv->wfd = -1; + } else helperKickQueue(hlp); } else { commSetSelect(srv->rfd, COMM_SELECT_READ, helperHandleRead, srv, 0); Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.1.1.3.4.1.2.2 retrieving revision 1.1.1.3.4.1.2.3 diff -u -r1.1.1.3.4.1.2.2 -r1.1.1.3.4.1.2.3 --- squid/src/http.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.4.1.2.2 +++ squid/src/http.c 3 May 2000 19:18:13 -0000 1.1.1.3.4.1.2.3 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.1.1.3.4.1.2.2 2000/04/17 00:13:10 hno Exp $ + * $Id: http.c,v 1.1.1.3.4.1.2.3 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -326,10 +326,12 @@ if (httpState->reply_hdr == NULL) httpState->reply_hdr = memAllocate(MEM_8K_BUF); assert(httpState->reply_hdr_state == 0); - hdr_len = strlen(httpState->reply_hdr); + hdr_len = httpState->reply_hdr_size; room = 8191 - hdr_len; - strncat(httpState->reply_hdr, buf, room < size ? room : size); + memcpy(httpState->reply_hdr + hdr_len, buf, room < size ? room : size); hdr_len += room < size ? room : size; + httpState->reply_hdr[hdr_len] = '\0'; + httpState->reply_hdr_size = hdr_len; if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr); httpState->reply_hdr_state += 2; @@ -339,9 +341,17 @@ t = httpState->reply_hdr + hdr_len; /* headers can be incomplete only if object still arriving */ if (!httpState->eof) { - size_t k = headersEnd(httpState->reply_hdr, 8192); - if (0 == k) - return; /* headers not complete */ + size_t k = headersEnd(httpState->reply_hdr, hdr_len); + if (0 == k) { + if (hdr_len >= 8191 || room == 0) { + debug(11, 3) ("httpProcessReplyHeader: Too large HTTP header: '%s'\n", httpState->reply_hdr); + httpState->reply_hdr_state += 2; + reply->sline.status = HTTP_INVALID_HEADER; + return; + } else { + return; /* headers not complete */ + } + } t = httpState->reply_hdr + k; } *t = '\0'; @@ -865,7 +875,8 @@ else if ((double) p->stats.n_keepalives_recv / (double) p->stats.n_keepalives_sent > 0.50) httpState->flags.keepalive = 1; if (httpState->peer) - if (neighborType(httpState->peer, httpState->request) == PEER_SIBLING) + if (neighborType(httpState->peer, httpState->request) == PEER_SIBLING && + !httpState->peer->options.allow_miss) httpState->flags.only_if_cached = 1; memBufDefInit(&mb); httpBuildRequestPrefix(req, Index: squid/src/ipc.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ipc.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/ipc.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/ipc.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: ipc.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: ipc.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 54 Interprocess Communication * AUTHOR: Duane Wessels @@ -228,14 +228,14 @@ return ipcCloseAllFD(prfd, pwfd, crfd, cwfd); } if (type == IPC_UDP_SOCKET) { - x = send(cwfd, hello_string, strlen(hello_string), 0); + x = send(cwfd, hello_string, strlen(hello_string) + 1, 0); if (x < 0) { debug(50, 0) ("sendto FD %d: %s\n", cwfd, xstrerror()); debug(50, 0) ("ipcCreate: CHILD: hello write test failed\n"); _exit(1); } } else { - if (write(cwfd, hello_string, strlen(hello_string)) < 0) { + if (write(cwfd, hello_string, strlen(hello_string) + 1) < 0) { debug(50, 0) ("write FD %d: %s\n", cwfd, xstrerror()); debug(50, 0) ("ipcCreate: CHILD: hello write test failed\n"); _exit(1); @@ -268,10 +268,14 @@ close(t1); close(t2); close(t3); + /* Make sure all other filedescriptors are closed */ + for(x=3;xoptions.mcast_responder) return 0; + if (p->n_addresses == 0) + return 0; /* the case below seems strange, but can happen if the * URL host is on the other side of a firewall */ if (p->type == PEER_SIBLING) if (!request->flags.hierarchical) return 0; - if (p->icp.port == echo_port) - if (!neighborUp(p)) - return 0; - if (p->n_addresses == 0) + /* Ping dead peers every timeout interval */ + if (squid_curtime - p->stats.last_query > Config.Timeout.deadPeer) + return 1; + if (!neighborUp(p)) return 0; return 1; } @@ -258,6 +260,18 @@ return q; } +/* This gets called every 5 minutes to clear the round-robin counter. */ +void +peerClearRR(void *data) +{ + peer *p = data; + p->rr_count -= p->rr_lastcount; + if (p->rr_count < 0) + p->rr_count = 0; + p->rr_lastcount = p->rr_count; + eventAdd("peerClearRR",peerClearRR, p, 5*60, 0); +} + peer * getDefaultParent(request_t * request) { @@ -372,6 +386,8 @@ icp_common_t *query; int queries_sent = 0; int peers_pinged = 0; + int parent_timeout = 0, parent_exprep = 0; + int sibling_timeout = 0, sibling_exprep = 0; if (Config.peers == NULL) return 0; @@ -381,7 +397,6 @@ mem->start_ping = current_time; mem->ping_reply_callback = callback; mem->ircb_data = callback_data; - *timeout = 0.0; reqnum = icpSetCacheKey(entry->key); for (i = 0, p = first_ping; i++ < Config.npeers; p = p->next) { if (p == NULL) @@ -434,28 +449,15 @@ */ p->stats.last_reply = squid_curtime; (*exprep) += p->mcast.n_replies_expected; - } else if (squid_curtime - p->stats.last_query > Config.Timeout.deadPeer) { - /* - * fake a recent reply if its been a long time since our - * last query - */ - p->stats.last_reply = squid_curtime; - /* - * We used to not expect a reply in this case; we assumed - * the peer was DEAD if we hadn't queried it in a long - * time. However, the number of people whining to - * squid-users that ICP is broken became unbearable. They - * tried a single request which, to their amazement, was - * forwarded directly to the origin server, even thought - * they KNEW it was in a neighbor cache. Ok, I give up, you - * win! - */ - (*exprep)++; - (*timeout) += 1000; } else if (neighborUp(p)) { /* its alive, expect a reply from it */ - (*exprep)++; - (*timeout) += p->stats.rtt; + if (neighborType(p, request) == PEER_PARENT) { + parent_exprep++; + parent_timeout += p->stats.rtt; + } else { + sibling_exprep++; + sibling_timeout += p->stats.rtt; + } } else { /* Neighbor is dead; ping it anyway, but don't expect a reply */ /* log it once at the threshold */ @@ -467,6 +469,8 @@ } } p->stats.last_query = squid_curtime; + if (p->stats.probe_start == 0) + p->stats.probe_start = squid_curtime; } if ((first_ping = first_ping->next) == NULL) first_ping = Config.peers; @@ -503,14 +507,22 @@ } #endif /* + * How many replies to expect? + */ + *exprep = parent_exprep + sibling_exprep; + + /* * If there is a configured timeout, use it */ if (Config.Timeout.icp_query) *timeout = Config.Timeout.icp_query; else { - if (*exprep > 0) - (*timeout) = 2 * (*timeout) / (*exprep); - else + if (*exprep > 0) { + if (parent_exprep) + *timeout = 2 * parent_timeout / parent_exprep; + else + *timeout = 2 * sibling_timeout / sibling_exprep; + } else *timeout = 2000; /* 2 seconds */ if (Config.Timeout.icp_query_max) if (*timeout > Config.Timeout.icp_query_max) @@ -633,6 +645,7 @@ p->stats.logged_state = PEER_ALIVE; } p->stats.last_reply = squid_curtime; + p->stats.probe_start = 0; p->stats.pings_acked++; if ((icp_opcode) header->opcode <= ICP_END) p->icp.counts[header->opcode]++; @@ -665,6 +678,7 @@ p->stats.logged_state = PEER_ALIVE; } p->stats.last_reply = squid_curtime; + p->stats.probe_start = 0; p->stats.pings_acked++; p->htcp.counts[htcp->hit ? 1 : 0]++; p->htcp.version = htcp->version; @@ -878,11 +892,14 @@ int neighborUp(const peer * p) { - if (!p->tcp_up) + if (!p->tcp_up) { + peerProbeConnect((peer *)p); return 0; - if (squid_curtime - p->stats.last_query > Config.Timeout.deadPeer) + } + if (p->options.no_query) return 1; - if (p->stats.last_query - p->stats.last_reply > Config.Timeout.deadPeer) + if (p->stats.probe_start != 0 && + squid_curtime - p->stats.probe_start > Config.Timeout.deadPeer) return 0; return 1; } @@ -978,62 +995,82 @@ eventAddIsh("peerRefreshDNS", peerRefreshDNS, NULL, 3600.0, 1); } +void +peerConnectFailed(peer *p) +{ + p->stats.last_connect_failure = squid_curtime; + if (!p->tcp_up) { + debug(15, 2) ("TCP connection to %s/%d dead\n", p->host, p->http_port); + return; + } + debug(15, 1) ("TCP connection to %s/%d failed\n", p->host, p->http_port); + p->tcp_up--; + if (!p->tcp_up) { + debug(15, 1) ("Detected DEAD %s: %s/%d/%d\n", + neighborTypeStr(p), + p->host, p->http_port, p->icp.port); + p->stats.logged_state = PEER_DEAD; + } +} + +void +peerConnectSucceded(peer *p) +{ + if (!p->tcp_up) { + debug(15, 2) ("TCP connection to %s/%d succeded\n", p->host, p->http_port); + debug(15, 1) ("Detected REVIVED %s: %s/%d/%d\n", + neighborTypeStr(p), + p->host, p->http_port, p->icp.port); + p->stats.logged_state = PEER_ALIVE; + } + p->tcp_up = PEER_TCP_MAGIC_COUNT; +} + /* - * peerCheckConnect will NOT be called by eventRun if the peer/data - * pointer becomes invalid. + * peerProbeConnect will be called on dead peers by neighborUp */ static void -peerCheckConnect(void *data) +peerProbeConnect(peer *p) { - peer *p = data; int fd; + if (p->test_fd != -1) + return; /* probe already running */ + if (squid_curtime - p->stats.last_connect_probe < Config.Timeout.connect) + return; /* don't probe to often */ fd = comm_open(SOCK_STREAM, 0, Config.Addrs.tcp_outgoing, 0, COMM_NONBLOCKING, p->host); if (fd < 0) return; p->test_fd = fd; - ipcache_nbgethostbyname(p->host, peerCheckConnect2, p); + p->stats.last_connect_probe = squid_curtime; + ipcache_nbgethostbyname(p->host, peerProbeConnect2, p); } static void -peerCheckConnect2(const ipcache_addrs * ianotused, void *data) +peerProbeConnect2(const ipcache_addrs * ianotused, void *data) { peer *p = data; commConnectStart(p->test_fd, p->host, p->http_port, - peerCheckConnectDone, + peerProbeConnectDone, p); } static void -peerCheckConnectDone(int fd, int status, void *data) +peerProbeConnectDone(int fd, int status, void *data) { peer *p = data; if (status == COMM_OK) { - p->tcp_up = PEER_TCP_MAGIC_COUNT; - debug(15, 1) ("TCP connection to %s/%d succeeded\n", - p->host, p->http_port); + peerConnectSucceded(p); } else { - eventAdd("peerCheckConnect", peerCheckConnect, p, 60.0, 1); + peerConnectFailed(p); } comm_close(fd); + p->test_fd = -1; return; } -void -peerCheckConnectStart(peer * p) -{ - if (!p->tcp_up) - return; - debug(15, 1) ("TCP connection to %s/%d failed\n", p->host, p->http_port); - p->tcp_up--; - if (p->tcp_up != (PEER_TCP_MAGIC_COUNT - 1)) - return; - p->last_fail_time = squid_curtime; - eventAdd("peerCheckConnect", peerCheckConnect, p, 30.0, 1); -} - static void peerCountMcastPeersSchedule(peer * p, time_t when) { @@ -1236,9 +1273,9 @@ #if USE_HTCP } #endif - if (e->last_fail_time) { + if (e->stats.last_connect_failure) { storeAppendPrintf(sentry, "Last failed connect() at: %s\n", - mkhttpdlogtime(&(e->last_fail_time))); + mkhttpdlogtime(&(e->stats.last_connect_failure))); } if (e->peer_domain != NULL) { storeAppendPrintf(sentry, "DOMAIN LIST: "); Index: squid/src/net_db.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/net_db.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/net_db.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/net_db.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: net_db.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: net_db.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 38 Network Measurement Database * AUTHOR: Duane Wessels @@ -60,7 +60,7 @@ static netdbEntry *netdbLookupHost(const char *key); static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *); static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e); -static char *netdbPeerName(const char *name); +static const char *netdbPeerName(const char *name); static IPH netdbSendPing; static QS sortPeerByRtt; static QS sortByRtt; @@ -467,16 +467,15 @@ count, tvSubMsec(start, current_time)); } -static char * +static const char * netdbPeerName(const char *name) { - wordlist *w; + const wordlist *w; for (w = peer_names; w; w = w->next) { if (!strcmp(w->key, name)) return w->key; } - w = wordlistAdd(&peer_names, name); - return w->key; + return wordlistAdd(&peer_names, name); } static void Index: squid/src/peer_select.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/peer_select.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/peer_select.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/peer_select.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: peer_select.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: peer_select.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 44 Peer Selection Algorithm * AUTHOR: Duane Wessels @@ -89,6 +89,7 @@ static void peerGetSomeNeighborReplies(ps_state *); static void peerGetSomeDirect(ps_state *); static void peerGetSomeParent(ps_state *); +static void peerGetAllParents(ps_state *); static void peerAddFwdServer(FwdServer **, peer *, hier_code); static void @@ -209,6 +210,8 @@ int myhops; if (p == NULL) return 0; + if (psstate->direct == DIRECT_NO) + return 0; myrtt = netdbHostRtt(psstate->request->host); debug(44, 3) ("peerCheckNetdbDirect: MY RTT = %d msec\n", myrtt); debug(44, 3) ("peerCheckNetdbDirect: closest_parent_miss RTT = %d msec\n", @@ -273,11 +276,23 @@ peerGetSomeNeighborReplies(ps); entry->ping_status = PING_DONE; } - if (Config.onoff.prefer_direct) - peerGetSomeDirect(ps); - peerGetSomeParent(ps); - if (!Config.onoff.prefer_direct) + switch (ps->direct) { + case DIRECT_YES: peerGetSomeDirect(ps); + break; + case DIRECT_NO: + peerGetSomeParent(ps); + peerGetAllParents(ps); + break; + default: + if (Config.onoff.prefer_direct) + peerGetSomeDirect(ps); + if (request->flags.hierarchical || !Config.onoff.nonhierarchical_direct) + peerGetSomeParent(ps); + if (!Config.onoff.prefer_direct) + peerGetSomeDirect(ps); + break; + } peerSelectCallback(ps); } @@ -434,6 +449,35 @@ } } +/* Adds alive parents. Used as a last resort for never_direct. + */ +static void +peerGetAllParents(ps_state * ps) +{ + peer *p; + request_t *request = ps->request; + /* Add all alive parents */ + for (p = Config.peers; p; p = p->next) { + /* XXX: neighbors.c lacks a public interface for enumerating + * parents to a request so we have to dig some here.. + */ + if (neighborType(p, request) != PEER_PARENT) + continue; + if (!peerHTTPOkay(p, request)) + continue; + debug(15, 3) ("peerGetAllParents: adding alive parent %s\n", p->host); + peerAddFwdServer(&ps->servers, p, ANY_OLD_PARENT); + } + /* XXX: should add dead parents here, but it is currently + * not possible to find out which parents are dead or which + * simply are not configured to handle the request. + */ + /* Add default parent as a last resort */ + if ((p = getDefaultParent(request))) { + peerAddFwdServer(&ps->servers, p, DEFAULT_PARENT); + } +} + static void peerPingTimeout(void *data) { Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.1.1.3.12.4 retrieving revision 1.1.1.3.12.5 diff -u -r1.1.1.3.12.4 -r1.1.1.3.12.5 --- squid/src/protos.h 3 May 2000 11:22:56 -0000 1.1.1.3.12.4 +++ squid/src/protos.h 3 May 2000 19:18:13 -0000 1.1.1.3.12.5 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.1.1.3.12.4 2000/05/03 11:22:56 asd Exp $ + * $Id: protos.h,v 1.1.1.3.12.5 2000/05/03 19:18:13 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -44,6 +44,7 @@ #if HEADERS_LOG extern void headersLog(int cs, int pq, method_t m, void *data); #endif +char *log_quote(const char *header); extern aclCheck_t *aclChecklistCreate(const struct _acl_access *, request_t *, @@ -67,41 +68,15 @@ extern wordlist *aclDumpGeneric(const acl *); extern int aclPurgeMethodInUse(acl_access *); -#if USE_ASYNC_IO -extern int aio_cancel(aio_result_t *); -extern int aio_open(const char *, int, mode_t, aio_result_t *); -extern int aio_read(int, char *, int, off_t, int, aio_result_t *); -extern int aio_write(int, char *, int, off_t, int, aio_result_t *); -extern int aio_close(int, aio_result_t *); -extern int aio_stat(const char *, struct stat *, aio_result_t *); -extern int aio_unlink(const char *, aio_result_t *); -extern int aio_opendir(const char *, aio_result_t *); -extern aio_result_t *aio_poll_done(void); -extern int aio_operations_pending(void); -extern int aio_overloaded(void); -extern int aio_sync(void); -extern int aio_get_queue_len(void); - -extern void aioInit(void); -extern void aioCancel(int); -extern void aioOpen(const char *, int, mode_t, AIOCB *, void *); -extern void aioClose(int); -extern void aioWrite(int, int offset, char *, int size, AIOCB *, void *, FREE *); -extern void aioRead(int, int offset, char *, int size, AIOCB *, void *); -extern void aioStat(char *, struct stat *, AIOCB *, void *); -extern void aioUnlink(const char *, AIOCB *, void *); -extern void aioCheckCallbacks(void); -extern void aioSync(void); -extern int aioQueueSize(void); -#endif - /* * cache_cf.c */ extern int parseConfigFile(const char *file_name); extern void intlistDestroy(intlist **); extern int intlistFind(intlist * list, int i); -extern wordlist *wordlistAdd(wordlist **, const char *); +extern const char *wordlistAdd(wordlist **, const char *); +extern void wordlistAddWl(wordlist **, wordlist *); +extern void wordlistJoin(wordlist **, wordlist **); extern wordlist *wordlistDup(const wordlist *); extern void wordlistDestroy(wordlist **); extern void configFreeMemory(void); @@ -189,6 +164,7 @@ #endif extern void commUpdateReadBits(int, PF *); extern void commUpdateWriteBits(int, PF *); +extern void comm_quick_poll_required(); extern void packerToStoreInit(Packer * p, StoreEntry * e); extern void packerToMemInit(Packer * p, MemBuf * mb); @@ -218,7 +194,7 @@ /* packs, then prints an object using debug() */ extern void debugObj(int section, int level, const char *label, void *obj, ObjPackMethod pm); - +/* disk.c */ extern int file_open(const char *path, int mode); extern void file_close(int fd); extern void file_write(int, off_t, void *, int len, DWCB *, void *, FREE *); @@ -226,6 +202,19 @@ extern void file_read(int, char *, int, off_t, DRCB *, void *); extern void disk_init(void); +/* diskd.c */ +extern diskd_queue * afile_create_queue(void); +extern void afile_destroy_queue(diskd_queue *); +extern void afile_sync_queue(diskd_queue *); +extern void afile_sync(void); +extern void afile_open(const char *path, int mode, DOCB *, void *); +extern void afile_close(int fd, DCCB *callback, void *data); +extern void afile_write(int, off_t, void *, int len, DWCB *, void *, FREE *); +extern void afile_write_mbuf(int fd, off_t, MemBuf, DWCB *, void *); +extern void afile_read(int, char *, int, off_t, DRCB *, void *); +extern void afile_unlink(const char *path, DUCB *, void *); +extern void afile_truncate(const char *path, DTCB *, void *); + extern void dnsShutdown(void); extern void dnsInit(void); extern void dnsSubmit(const char *lookup, HLPCB * callback, void *data); @@ -634,6 +623,7 @@ extern peer *peerFindByNameAndPort(const char *, unsigned short); extern peer *getDefaultParent(request_t * request); extern peer *getRoundRobinParent(request_t * request); +EVH peerClearRR; extern peer *getAnyParent(request_t * request); extern lookup_t peerDigestLookup(peer * p, request_t * request, StoreEntry * entry); extern peer *neighborsDigestSelect(request_t * request, StoreEntry * entry); @@ -643,7 +633,8 @@ extern CBDUNL peerDestroy; extern char *neighborTypeStr(const peer * e); extern peer_t neighborType(const peer *, const request_t *); -extern void peerCheckConnectStart(peer *); +extern void peerConnectFailed(peer *); +extern void peerConnectSucceded(peer *); extern void dump_peer_options(StoreEntry *, peer *); extern int peerHTTPOkay(const peer *, request_t *); extern peer *whichPeer(const struct sockaddr_in *from); @@ -738,7 +729,7 @@ extern double statRequestHitMemoryRatio(int minutes); extern double statRequestHitDiskRatio(int minutes); extern double statByteHitRatio(int minutes); - +extern int storeEntryLocked(const StoreEntry *); /* StatHist */ @@ -776,7 +767,6 @@ extern void memFree2K(void *); extern void memFree4K(void *); extern void memFree8K(void *); -extern void memFreeDISK(void *); extern int memInUse(mem_type); extern size_t memTotalAllocated(void); extern void memDataInit(mem_type, const char *, size_t, int); @@ -841,9 +831,6 @@ extern void InvokeHandlers(StoreEntry *); extern int storeEntryValidToSend(StoreEntry *); extern void storeTimestampsSet(StoreEntry *); -#if !HEAP_REPLACEMENT -extern time_t storeExpiredReferenceAge(void); -#endif extern void storeRegisterAbort(StoreEntry * e, STABH * cb, void *); extern void storeUnregisterAbort(StoreEntry * e); extern void storeMemObjectDump(MemObject * mem); @@ -868,51 +855,32 @@ extern HttpReply *storeEntryReply(StoreEntry *); extern int storeTooManyDiskFilesOpen(void); extern void storeEntryReset(StoreEntry *); -extern void storeHeapPositionUpdate(StoreEntry *); +extern void storeHeapPositionUpdate(StoreEntry *, SwapDir *); extern void storeSwapFileNumberSet(StoreEntry * e, sfileno filn); +extern void storeFsInit(void); +extern void storeFsDone(void); +extern void storeFsAdd(char *, STSETUP *); + +/* store_heap_replacement.c */ +#ifdef HEAP_REPLACEMENT +extern heap_key HeapKeyGen_StoreEntry_LFUDA(void *, double); +extern heap_key HeapKeyGen_StoreEntry_GDSF(void *, double); +extern heap_key HeapKeyGen_StoreEntry_LRU(void *, double); +#endif + +/* store_modules.c */ +extern void storeFsSetup(void); /* store_io.c */ -extern STOBJOPEN storeOpen; -extern STOBJCLOSE storeClose; -extern STOBJREAD storeRead; -extern STOBJWRITE storeWrite; -extern STOBJUNLINK storeUnlink; +extern storeIOState * storeCreate(StoreEntry *, STFNCB *, STIOCB *, void *); +extern storeIOState * storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *); +extern void storeClose(storeIOState *); +extern void storeRead(storeIOState *, char *, size_t, off_t, STRCB *, void *); +extern void storeWrite(storeIOState *, char *, size_t, off_t, FREE *); +extern void storeUnlink(StoreEntry *); extern off_t storeOffset(storeIOState *); /* - * store_io_ufs.c - */ -extern storeIOState *storeUfsOpen(sfileno, mode_t, STIOCB *, void *); -extern void storeUfsClose(storeIOState * sio); -extern void storeUfsRead(storeIOState *, char *, size_t, off_t, STRCB *, void *); -extern void storeUfsWrite(storeIOState *, char *, size_t, off_t, FREE *); -extern void storeUfsUnlink(int fileno); - -#if USE_ASYNC_IO -/* - * store_io_ufs.c - */ -extern storeIOState *storeAufsOpen(sfileno, mode_t, STIOCB *, void *); -extern void storeAufsClose(storeIOState * sio); -extern void storeAufsRead(storeIOState *, char *, size_t, off_t, STRCB *, void *); -extern void storeAufsWrite(storeIOState *, char *, size_t, off_t, FREE *); -extern void storeAufsUnlink(int fileno); -#endif - -#if USE_DISKD -/* - * diskd.c - */ -extern storeIOState *storeDiskdOpen(sfileno, mode_t, STIOCB *, void *); -extern void storeDiskdClose(storeIOState * sio); -extern void storeDiskdRead(storeIOState *, char *, size_t, off_t, STRCB *, void *); -extern void storeDiskdWrite(storeIOState *, char *, size_t, off_t, FREE *); -extern void storeDiskdUnlink(int fileno); -extern STINIT storeDiskdInit; -extern void storeDiskdReadQueue(void); -#endif - -/* * store_log.c */ extern void storeLog(int tag, const StoreEntry * e); @@ -955,46 +923,24 @@ extern char *storeSwapFullPath(int, char *); extern char *storeSwapSubSubDir(int, char *); extern const char *storeSwapPath(int); -extern int storeDirMapAllocate(void); -extern int storeDirMapBitTest(int fn); -extern int storeDirMapBitsInUse(void); -extern int storeDirNumber(int fileno); -extern int storeDirProperFileno(int dirn, int fn); -extern int storeDirValidFileno(int fn, int); extern int storeDirWriteCleanLogs(int reopen); +extern int storeDirSelectSwapDir(const StoreEntry *); extern int storeVerifySwapDirs(void); extern void storeCreateSwapDirectories(void); extern void storeDirCloseSwapLogs(void); extern void storeDirCloseTmpSwapLog(int dirn); extern void storeDirConfigure(void); -extern void storeDirDiskFull(int fn); +extern void storeDirDiskFull(sdirno); extern void storeDirInit(void); -extern void storeDirMapBitReset(int fn); -extern void storeDirMapBitSet(int fn); extern void storeDirOpenSwapLogs(void); extern void storeDirSwapLog(const StoreEntry *, int op); -extern void storeDirUpdateSwapSize(int fn, size_t size, int sign); +extern void storeDirUpdateSwapSize(SwapDir *, size_t size, int sign); +extern void storeDirSync(void); +extern void storeDirCallback(void); extern void storeDirLRUDelete(StoreEntry *); extern void storeDirLRUAdd(StoreEntry *); /* - * store_dir_ufs.c - */ -extern OBJH storeUfsDirStats; -extern void storeUfsDirParse(cacheSwap * swap); -extern void storeUfsDirDump(StoreEntry * entry, const char *name, SwapDir * s); -extern void storeUfsDirFree(SwapDir *); -extern char *storeUfsFullPath(sfileno fn, char *fullpath); -extern STINIT storeUfsDirInit; -#if USE_ASYNC_IO -extern void storeAufsDirParse(cacheSwap * swap); -#endif -#if USE_DISKD -extern void storeDiskdDirParse(cacheSwap *); -#endif - - -/* * store_swapmeta.c */ extern char *storeSwapMetaPack(tlv * tlv_list, int *length); Index: squid/src/snmp_agent.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/snmp_agent.c,v retrieving revision 1.1.1.3.10.1 retrieving revision 1.1.1.3.10.2 diff -u -r1.1.1.3.10.1 -r1.1.1.3.10.2 --- squid/src/snmp_agent.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.10.1 +++ squid/src/snmp_agent.c 3 May 2000 19:18:13 -0000 1.1.1.3.10.2 @@ -1,6 +1,6 @@ /* - * $Id: snmp_agent.c,v 1.1.1.3.10.1 2000/04/17 00:13:10 hno Exp $ + * $Id: snmp_agent.c,v 1.1.1.3.10.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 49 SNMP Interface * AUTHOR: Kostas Anagnostakis @@ -279,12 +279,9 @@ ASN_INTEGER); break; case PERF_SYS_CURLRUEXP: + /* No global LRU info anymore */ Answer = snmp_var_new_integer(Var->name, Var->name_length, -#if !HEAP_REPLACEMENT - (snint) (storeExpiredReferenceAge() * 100), -#else - 0, -#endif + 0, SMI_TIMETICKS); break; case PERF_SYS_CURUNLREQ: Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/squid.h 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/squid.h 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: squid.h,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * AUTHOR: Duane Wessels * @@ -340,12 +340,6 @@ #include #endif -#if USE_ASYNC_IO -#undef USE_UNLINKD -#else -#define USE_UNLINKD 1 -#endif - #include "md5.h" #include "Stack.h" @@ -426,7 +420,13 @@ #define SQUID_NONBLOCK O_NDELAY #endif -#define SWAP_DIR_SHIFT 24 -#define SWAP_FILE_MASK 0x00FFFFFF +#include +#include + +/* + * I'm sick of having to keep doing this .. + */ + +#define INDEXSD(i) (&Config.cacheSwap.swapDirs[(i)]) #endif /* SQUID_H */ Index: squid/src/ssl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ssl.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/ssl.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/ssl.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: ssl.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: ssl.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 26 Secure Sockets Layer Proxy * AUTHOR: Duane Wessels @@ -377,6 +377,15 @@ SslStateData *sslState = data; request_t *request = sslState->request; ErrorState *err = NULL; + if (sslState->servers->peer) + hierarchyNote(&sslState->request->hier, sslState->servers->code, + sslState->servers->peer->host); + else if (Config.onoff.log_ip_on_direct) + hierarchyNote(&sslState->request->hier, sslState->servers->code, + fd_table[sslState->server.fd].ipaddr); + else + hierarchyNote(&sslState->request->hier, sslState->servers->code, + sslState->host); if (status == COMM_ERR_DNS) { debug(26, 4) ("sslConnect: Unknown host: %s\n", sslState->host); err = errorCon(ERR_DNS_FAIL, HTTP_NOT_FOUND); @@ -416,6 +425,31 @@ SslStateData *sslState = NULL; int sock; ErrorState *err = NULL; + aclCheck_t ch; + int answer; + /* + * client_addr == no_addr indicates this is an "internal" request + * from peer_digest.c, asn.c, netdb.c, etc and should always + * be allowed. yuck, I know. + */ + if (request->client_addr.s_addr != no_addr.s_addr) { + /* + * Check if this host is allowed to fetch MISSES from us (miss_access) + */ + memset(&ch, '\0', sizeof(aclCheck_t)); + ch.src_addr = request->client_addr; + ch.my_addr = request->my_addr; + ch.my_port = request->my_port; + ch.request = request; + answer = aclCheckFast(Config.accessList.miss, &ch); + if (answer == 0) { + err = errorCon(ERR_FORWARDING_DENIED, HTTP_FORBIDDEN); + err->request = requestLink(request); + err->src_addr = request->client_addr; + errorSend(fd, err); + return; + } + } debug(26, 3) ("sslStart: '%s %s'\n", RequestMethodStr[request->method], url); Counter.server.all.requests++; @@ -546,9 +580,6 @@ sslState->delay_id = 0; } #endif - hierarchyNote(&sslState->request->hier, - fs->peer ? fs->code : DIRECT, - sslState->host); commConnectStart(sslState->server.fd, sslState->host, sslState->port, Index: squid/src/stat.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stat.c,v retrieving revision 1.1.1.3.12.2 retrieving revision 1.1.1.3.12.3 diff -u -r1.1.1.3.12.2 -r1.1.1.3.12.3 --- squid/src/stat.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.2 +++ squid/src/stat.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.3 @@ -1,6 +1,6 @@ /* - * $Id: stat.c,v 1.1.1.3.12.2 2000/04/17 00:13:10 hno Exp $ + * $Id: stat.c,v 1.1.1.3.12.3 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -264,8 +264,8 @@ (int) e->lock_count, storePendingNClients(e), (int) e->refcount); - storeAppendPrintf(s, "\tSwap File %#08X\n", - e->swap_file_number); + storeAppendPrintf(s, "\tSwap Dir %d, File %#08X\n", + e->swap_dirn, e->swap_filen); if (mem != NULL) { storeAppendPrintf(s, "\tinmem_lo: %d\n", (int) mem->inmem_lo); storeAppendPrintf(s, "\tinmem_hi: %d\n", (int) mem->inmem_hi); @@ -495,13 +495,6 @@ store_swap_size); storeAppendPrintf(sentry, "\tStorage Mem size:\t%d KB\n", (int) (store_mem_size >> 10)); -#if HEAP_REPLACEMENT - storeAppendPrintf(sentry, "\tStorage Replacement Threshold:\t%f\n", - heap_peepminkey(store_heap)); -#else - storeAppendPrintf(sentry, "\tStorage LRU Expiration Age:\t%6.2f days\n", - (double) storeExpiredReferenceAge() / 86400.0); -#endif storeAppendPrintf(sentry, "\tMean Object Size:\t%0.2f KB\n", n_disk_objects ? (double) store_swap_size / n_disk_objects : 0.0); storeAppendPrintf(sentry, "\tRequests given to unlinkd:\t%d\n", @@ -616,8 +609,6 @@ memInUse(MEM_MEMOBJECT)); storeAppendPrintf(sentry, "\t%6d Hot Object Cache Items\n", hot_obj_count); - storeAppendPrintf(sentry, "\t%6d Filemap bits set\n", - storeDirMapBitsInUse()); storeAppendPrintf(sentry, "\t%6d on-disk objects\n", n_disk_objects); Index: squid/src/store.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/store.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/store.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: store.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: store.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -74,8 +74,6 @@ /* * local function prototypes */ -static int storeCheckExpired(const StoreEntry *); -static int storeEntryLocked(const StoreEntry *); static int storeEntryValidLength(const StoreEntry *); static void storeGetMemSpace(int); static void storeHashDelete(StoreEntry *); @@ -87,11 +85,6 @@ static int storeKeepInMemory(const StoreEntry *); static OBJH storeCheckCachableStats; static EVH storeLateRelease; -#if HEAP_REPLACEMENT -static heap_key_func HeapKeyGen_StoreEntry_LFUDA; -static heap_key_func HeapKeyGen_StoreEntry_GDSF; -static heap_key_func HeapKeyGen_StoreEntry_LRU; -#endif /* * local variables @@ -104,9 +97,9 @@ #else static dlink_list inmem_list; #endif -static int store_pages_max = 0; -static int store_swap_high = 0; -static int store_swap_low = 0; +int store_pages_max = 0; +int store_swap_high = 0; +int store_swap_low = 0; static Stack LateReleaseStack; #if URL_CHECKSUM_DEBUG @@ -150,7 +143,8 @@ e->mem_obj = new_MemObject(url, log_url); debug(20, 3) ("new_StoreEntry: returning %p\n", e); e->expires = e->lastmod = e->lastref = e->timestamp = -1; - e->swap_file_number = -1; + e->swap_filen = -1; + e->swap_dirn = -1; return e; } @@ -204,27 +198,12 @@ e, storeKeyText(key)); e->key = storeKeyDup(key); hash_join(store_table, (hash_link *) e); -#if HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->node = heap_insert(store_heap, e); - debug(20, 4) ("storeHashInsert: inserted node %p\n", e->node); - } -#endif } static void storeHashDelete(StoreEntry * e) { hash_remove_link(store_table, (hash_link *) e); -#if HEAP_REPLACEMENT - if (e->node) { - debug(20, 4) ("storeHashDelete: deleting node %p\n", e->node); - heap_delete(store_heap, e->node); - e->node = NULL; - } -#endif storeKeyFree(e->key); e->key = NULL; } @@ -250,24 +229,20 @@ void storeLockObject(StoreEntry * e) { - if (e->lock_count++ == 0) { -#if HEAP_REPLACEMENT - /* - * There is no reason to take any action here. Squid by - * default is moving locked objects to the end of the LRU - * list to keep them from getting bumped into by the - * replacement algorithm. We can't do that so we will just - * have to handle them. - */ - debug(20, 4) ("storeLockObject: just locked node %p\n", e->node); -#else - storeDirLRUDelete(e); - storeDirLRUAdd(e); -#endif - } + SwapDir *SD; + + if (e->swap_dirn > -1) + SD = INDEXSD(e->swap_dirn); + else + SD = NULL; + + e->lock_count++; debug(20, 3) ("storeLockObject: key '%s' count=%d\n", storeKeyText(e->key), (int) e->lock_count); e->lastref = squid_curtime; + /* Notify the fs that we're referencing this object again */ + if (SD != NULL && SD->refobj != NULL) + SD->refobj(SD, e); } void @@ -291,6 +266,7 @@ int storeUnlockObject(StoreEntry * e) { + SwapDir *SD; e->lock_count--; debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", storeKeyText(e->key), e->lock_count); @@ -299,12 +275,22 @@ if (e->store_status == STORE_PENDING) EBIT_SET(e->flags, RELEASE_REQUEST); assert(storePendingNClients(e) == 0); + /* Notify the fs that we're not referencing this object any more */ + if (e->swap_filen > -1) + SD = INDEXSD(e->swap_dirn); + else + SD = NULL; + if (SD != NULL && SD->unrefobj != NULL) + SD->unrefobj(SD, e); #if HEAP_REPLACEMENT - storeHeapPositionUpdate(e); + storeHeapPositionUpdate(e, SD); #else +#if 0 + /* Note: From 2.4. Not sure how this relates to the unrefobj() call above */ storeDirLRUDelete(e); storeDirLRUAdd(e); #endif +#endif if (EBIT_TEST(e->flags, RELEASE_REQUEST)) storeRelease(e); else if (storeKeepInMemory(e)) { @@ -351,7 +337,7 @@ if (e->key && EBIT_TEST(e->flags, KEY_PRIVATE)) return; /* is already private */ if (e->key) { - if (e->swap_file_number > -1) + if (e->swap_filen > -1) storeDirSwapLog(e, SWAP_LOG_DEL); storeHashDelete(e); } @@ -402,7 +388,7 @@ storeHashDelete(e); EBIT_CLR(e->flags, KEY_PRIVATE); storeHashInsert(e, newkey); - if (e->swap_file_number > -1) + if (e->swap_filen > -1) storeDirSwapLog(e, SWAP_LOG_ADD); } @@ -431,7 +417,8 @@ e->store_status = STORE_PENDING; storeSetMemStatus(e, NOT_IN_MEMORY); e->swap_status = SWAPOUT_NONE; - e->swap_file_number = -1; + e->swap_filen = -1; + e->swap_dirn = -1; e->refcount = 0; e->lastref = squid_curtime; e->timestamp = 0; /* set in storeTimestampsSet() */ @@ -552,7 +539,9 @@ debug(20, 3) ("storeCheckCachable: NO: negative cached\n"); store_check_cachable_hist.no.negative_cached++; return 0; /* avoid release call below */ - } else if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { + } else if ((e->mem_obj->reply->content_length > 0 && + e->mem_obj->reply->content_length > Config.Store.maxObjectSize) || + e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { debug(20, 2) ("storeCheckCachable: NO: too big\n"); store_check_cachable_hist.no.too_big++; } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) { @@ -674,11 +663,8 @@ } /* Notify the client side */ InvokeHandlers(e); - /* Do we need to close the swapout file? */ - /* Not if we never started swapping out */ - if (e->swap_file_number > -1) { - storeSwapOutFileClose(e); - } + /* Close any swapout file */ + storeSwapOutFileClose(e); storeUnlockObject(e); /* unlock */ } @@ -777,173 +763,23 @@ void storeMaintainSwapSpace(void *datanotused) { - StoreEntry *e = NULL; - int scanned = 0; - int locked = 0; - int expired = 0; - int max_scan; - int max_remove; int i; - int j; - static int ndir = 0; - double f; - static time_t last_warn_time = 0; -#if !HEAP_REPLACEMENT - SwapDir *sd; -#else - heap_key age; - heap_key min_age = 0.0; - link_list *locked_entries = NULL; -#if HEAP_REPLACEMENT_DEBUG - if (!verify_heap_property(store_heap)) { - debug(20, 1) ("Heap property violated!\n"); - } -#endif -#endif - /* We can't delete objects while rebuilding swap */ - if (store_dirs_rebuilding) { - eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1); - return; - } else { - f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); - f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; - max_scan = (int) (f * 400.0 + 100.0); - if ((max_remove = stat5minClientRequests()) < 10) - max_remove = 10; - eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); - } - debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", - f, max_scan, max_remove); -#if HEAP_REPLACEMENT - while (heap_nodes(store_heap) > 0) { - if (store_swap_size < store_swap_low) - break; - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; - age = heap_peepminkey(store_heap); - e = heap_extractmin(store_heap); - e->node = NULL; /* no longer in the heap */ - scanned++; - if (storeEntryLocked(e)) { - /* - * Entry is in use ... put it in a linked list to ignore it. - */ - if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - /* - * If this was a "SPECIAL" do not add it back into the heap. - * It will always be "SPECIAL" and therefore never removed. - */ - debug(20, 4) ("storeMaintainSwapSpace: locked url %s\n", - (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->key)); - linklistPush(&locked_entries, e); - } - locked++; - continue; - } else if (storeCheckExpired(e)) { - /* - * Note: This will not check the reference age ifdef - * HEAP_REPLACEMENT, but it does some other useful - * checks... - */ - expired++; - debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", - age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); - min_age = age; - storeRelease(e); - } else { - /* - * Did not expire the object so we need to add it back - * into the heap! - */ - debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", - storeKeyText(e->key)); - linklistPush(&locked_entries, e); - continue; - } - if (store_swap_size < store_swap_low) - break; - else if (expired >= max_remove) - break; - else if (scanned >= max_scan) - break; - } - /* - * Bump the heap age factor. - */ - if (min_age > 0.0) - store_heap->age = min_age; - /* - * Reinsert all bumped locked entries back into heap... - */ - while ((e = linklistShift(&locked_entries))) - e->node = heap_insert(store_heap, e); -#else + SwapDir *SD; + + /* walk each fs */ for (i = 0; i < Config.cacheSwap.n_configured; i++) { - sd = &Config.cacheSwap.swapDirs[i]; - sd->lru_walker = sd->lru_list.tail; + /* call the maintain function .. */ + SD = INDEXSD(i); + if (SD->maintainfs != NULL) + SD->maintainfs(SD); } - do { - j = 0; - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - if (ndir >= Config.cacheSwap.n_configured) - ndir = ndir % Config.cacheSwap.n_configured; - sd = &Config.cacheSwap.swapDirs[ndir++]; - if (sd->cur_size < sd->low_size) - continue; - if (NULL == sd->lru_walker) - continue; - e = sd->lru_walker->data; - sd->lru_walker = sd->lru_walker->prev; - j++; - scanned++; - sd->scanned++; - if (storeEntryLocked(e)) { - /* - * If there is a locked entry at the tail of the LRU list, - * move it to the beginning to get it out of the way. - * Theoretically, we might have all locked objects at the - * tail, and then we'll never remove anything here and the - * LRU age will go to zero. - */ - if (memInUse(MEM_STOREENTRY) > max_scan) { - storeDirLRUDelete(e); - if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) - storeDirLRUAdd(e); - } - locked++; - } else if (storeCheckExpired(e)) { - expired++; - sd->removals++; - storeRelease(e); - } - if (expired >= max_remove) - break; - if (scanned >= max_scan) - break; - } - } while (j > 0 && expired < max_remove && scanned < max_scan); -#endif - debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d locked %d f=%.03f\n", - scanned, max_scan, expired, max_remove, locked, f); - debug(20, 3) ("storeMaintainSwapSpace stats:\n"); - debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); - debug(20, 3) (" %6d were scanned\n", scanned); - debug(20, 3) (" %6d were locked\n", locked); - debug(20, 3) (" %6d were expired\n", expired); - if (store_swap_size < Config.Swap.maxSize) - return; - if (squid_curtime - last_warn_time < 10) - return; - debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", - store_swap_size, Config.Swap.maxSize); - last_warn_time = squid_curtime; + + /* Reregister a maintain event .. */ + eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1); } /* release an object from a cache */ -/* return number of objects released. */ void storeRelease(StoreEntry * e) { @@ -956,30 +792,37 @@ storeReleaseRequest(e); return; } - if (store_dirs_rebuilding && e->swap_file_number > -1) { + if (store_dirs_rebuilding && e->swap_filen > -1) { storeSetPrivateKey(e); if (e->mem_obj) { storeSetMemStatus(e, NOT_IN_MEMORY); destroy_MemObject(e); } - /* - * Fake a call to storeLockObject(). When rebuilding is done, - * we'll just call storeUnlockObject() on these. - */ - e->lock_count++; - EBIT_SET(e->flags, RELEASE_REQUEST); - stackPush(&LateReleaseStack, e); - return; + if (e->swap_filen > -1) { + /* + * Fake a call to storeLockObject(). When rebuilding is done, + * we'll just call storeUnlockObject() on these. + */ + e->lock_count++; + EBIT_SET(e->flags, RELEASE_REQUEST); + stackPush(&LateReleaseStack, e); + return; + } else { + destroy_StoreEntry(e); + } } storeLog(STORE_LOG_RELEASE, e); - if (e->swap_file_number > -1) { - storeUnlink(e->swap_file_number); + if (e->swap_filen > -1) { + storeUnlink(e); if (e->swap_status == SWAPOUT_DONE) if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) - storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, -1); + storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, -1); if (!EBIT_TEST(e->flags, KEY_PRIVATE)) storeDirSwapLog(e, SWAP_LOG_DEL); +#if 0 + /* From 2.4. I think we do this in storeUnlink? */ storeSwapFileNumberSet(e, -1); +#endif } storeSetMemStatus(e, NOT_IN_MEMORY); destroy_StoreEntry(e); @@ -1009,7 +852,7 @@ } /* return 1 if a store entry is locked */ -static int +int storeEntryLocked(const StoreEntry * e) { if (e->lock_count) @@ -1088,10 +931,6 @@ debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize); } -#if HEAP_REPLACEMENT -#include "store_heap_replacement.c" -#endif - void storeInit(void) { @@ -1102,36 +941,7 @@ storeDigestInit(); storeLogOpen(); #if HEAP_REPLACEMENT - /* - * Create new heaps with cache replacement policies attached to them. - * The cache replacement policy is specified as either GDSF or LFUDA in - * the squid.conf configuration file. Note that the replacement policy - * applies only to the disk replacement algorithm. Memory replacement - * always uses GDSF since we want to maximize object hit rate. - */ inmem_heap = new_heap(1000, HeapKeyGen_StoreEntry_GDSF); - if (Config.replPolicy) { - if (tolower(Config.replPolicy[0]) == 'g') { - debug(20, 1) ("Using GDSF disk replacement policy\n"); - store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } else if (tolower(Config.replPolicy[0]) == 'l') { - if (tolower(Config.replPolicy[1]) == 'f') { - debug(20, 1) ("Using LFUDA disk replacement policy\n"); - store_heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); - } else if (tolower(Config.replPolicy[1]) == 'r') { - debug(20, 1) ("Using LRU heap disk replacement policy\n"); - store_heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); - } - } else { - debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); - store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } - } else { - debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); - store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); - } -#else - inmem_list.head = inmem_list.tail = NULL; #endif stackInit(&LateReleaseStack); eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); @@ -1166,48 +976,6 @@ return mem->inmem_lo == 0; } -static int -storeCheckExpired(const StoreEntry * e) -{ - if (storeEntryLocked(e)) - return 0; - if (EBIT_TEST(e->flags, RELEASE_REQUEST)) - return 1; - if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) - return 1; - return 1; -} - -#if !HEAP_REPLACEMENT -/* - * storeExpiredReferenceAge - * - * The LRU age is scaled exponentially between 1 minute and - * Config.referenceAge , when store_swap_low < store_swap_size < - * store_swap_high. This keeps store_swap_size within the low and high - * water marks. If the cache is very busy then store_swap_size stays - * closer to the low water mark, if it is not busy, then it will stay - * near the high water mark. The LRU age value can be examined on the - * cachemgr 'info' page. - */ -time_t -storeExpiredReferenceAge(void) -{ - double x; - double z; - time_t age; - x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low); - x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; - z = pow((double) (Config.referenceAge / 60), x); - age = (time_t) (z * 60.0); - if (age < 60) - age = 60; - else if (age > 31536000) - age = 31536000; - return age; -} -#endif - void storeNegativeCache(StoreEntry * e) { @@ -1337,7 +1105,8 @@ debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz); debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount); debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e)); - debug(20, l) ("StoreEntry->swap_file_number: %d\n", (int) e->swap_file_number); + debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn); + debug(20, l) ("StoreEntry->swap_filen: %d\n", (int) e->swap_filen); debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count); debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status); debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status); @@ -1465,16 +1234,65 @@ } #if HEAP_REPLACEMENT +/* + * This routine only handles memory updates these days + */ void -storeHeapPositionUpdate(StoreEntry * e) +storeHeapPositionUpdate(StoreEntry * e, SwapDir * SD) { - if (e->node) - heap_update(store_heap, e->node, e); if (e->mem_obj && e->mem_obj->node) heap_update(inmem_heap, e->mem_obj->node, e); } #endif +/* + * storeFsInit + * + * This routine calls the SETUP routine for each fs type. + * I don't know where the best place for this is, and I'm not going to shuffle + * around large chunks of code right now (that can be done once its working.) + */ +void +storeFsInit(void) +{ + storeFsSetup(); +} + + +/* + * similar to above, but is called when a graceful shutdown is to occur + * of each fs module. + */ +void +storeFsDone(void) +{ + int i = 0; + + while (storefs_list[i].typestr != NULL) { + storefs_list[i].donefunc(); + i++; + } +} + +/* + * called to add another store fs module + */ +void storeFsAdd(char *type, STSETUP *setup) +{ + int i; + /* find the number of currently known storefs types */ + for (i = 0; storefs_list && storefs_list[i].typestr; i++) { + assert(strcmp(storefs_list[i].typestr, type)!=0); + } + /* add the new type */ + storefs_list = xrealloc(storefs_list, (i + 2) * sizeof(storefs_entry_t)); + memset(&storefs_list[i+1],0,sizeof(storefs_entry_t)); + storefs_list[i].typestr = type; + /* Call the FS to set up capabilities and initialize the FS driver */ + setup(&storefs_list[i]); +} + +#if 0 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn) { @@ -1491,3 +1309,4 @@ storeDirLRUAdd(e); } } +#endif Index: squid/src/store_client.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_client.c,v retrieving revision 1.1.1.3.4.1.2.1 retrieving revision 1.1.1.3.4.1.2.2 diff -u -r1.1.1.3.4.1.2.1 -r1.1.1.3.4.1.2.2 --- squid/src/store_client.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.4.1.2.1 +++ squid/src/store_client.c 3 May 2000 19:18:13 -0000 1.1.1.3.4.1.2.2 @@ -1,6 +1,6 @@ /* - * $Id: store_client.c,v 1.1.1.3.4.1.2.1 2000/04/17 00:13:10 hno Exp $ + * $Id: store_client.c,v 1.1.1.3.4.1.2.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 20 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -138,7 +138,7 @@ if (sc->type == STORE_DISK_CLIENT) /* assert we'll be able to get the data we want */ /* maybe we should open swapin_fd here */ - assert(e->swap_file_number > -1 || storeSwapOutAble(e)); + assert(e->swap_filen > -1 || storeSwapOutAble(e)); for (T = &mem->clients; *T; T = &(*T)->next); *T = sc; #if DELAY_POOLS @@ -152,7 +152,6 @@ STCB *callback = sc->callback; char *buf = sc->copy_buf; assert(sc->callback); - assert(sc->copy_buf); sc->callback = NULL; sc->copy_buf = NULL; if (cbdataValid(sc->callback_data)) @@ -259,6 +258,7 @@ { MemObject *mem = e->mem_obj; size_t sz; + if (storeClientNoMoreToSend(e, sc)) { /* There is no more to send! */ storeClientCallback(sc, 0); @@ -307,10 +307,11 @@ /* What the client wants is in memory */ debug(20, 3) ("storeClientCopy3: Copying from memory\n"); sz = stmemCopy(&mem->data_hdr, - sc->copy_offset, sc->copy_buf, sc->copy_size); - storeClientCallback(sc, sz); - return; + sc->copy_offset, sc->copy_buf, sc->copy_size); + storeClientCallback(sc, sz); + return; } + /* What the client wants is not in memory. Schedule a disk read */ assert(STORE_DISK_CLIENT == sc->type); assert(!sc->flags.disk_io_pending); debug(20, 3) ("storeClientCopy3: reading from STORE\n"); @@ -386,7 +387,7 @@ return; } if (tlv_list == NULL) { - debug(20, 1) ("WRNING: failed to unpack meta data\n"); + debug(20, 1) ("WARNING: failed to unpack meta data\n"); storeClientCallback(sc, -1); return; } @@ -520,13 +521,15 @@ storeLowestMemReaderOffset(const StoreEntry * entry) { const MemObject *mem = entry->mem_obj; - off_t lowest = mem->inmem_hi; + off_t lowest = mem->inmem_hi + 1; store_client *sc; store_client *nx = NULL; for (sc = mem->clients; sc; sc = nx) { nx = sc->next; if (sc->callback_data == NULL) /* open slot */ continue; + if (sc->type != STORE_MEM_CLIENT) + continue; if (sc->type == STORE_DISK_CLIENT) if (NULL != sc->swapin_sio) continue; Index: squid/src/store_dir.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_dir.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/store_dir.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/store_dir.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: store_dir.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: store_dir.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -35,15 +35,10 @@ #include "squid.h" +static int storeDirValidSwapDirSize(int, size_t); static void storeDirLRUWalkInitHead(SwapDir * sd); static void *storeDirLRUWalkNext(SwapDir * sd); -const char *SwapDirType[] = -{ - "ufs", - "!ERROR!" -}; - void storeDirInit(void) { @@ -79,6 +74,33 @@ } /* + * Determine whether the given directory can handle this object + * size + * + * Note: if the object size is -1, then the only swapdirs that + * will return true here are ones that have max_obj_size = -1, + * ie any-sized-object swapdirs. This is a good thing. + */ +static int +storeDirValidSwapDirSize(int swapdir, size_t objsize) +{ + /* + * If the swapdir's max_obj_size is -1, then it definitely can + */ + if (Config.cacheSwap.swapDirs[swapdir].max_objsize == -1) + return 1; + /* + * Else, make sure that the max object size is larger than objsize + */ + if (Config.cacheSwap.swapDirs[swapdir].max_objsize > objsize) + return 1; + else + return 0; +} + + +#if UNUSED /* Squid-2..4.DEVEL3 code */ +/* * This new selection scheme simply does round-robin on all SwapDirs. * A SwapDir is skipped if it is over the max_size (100%) limit. If * all SwapDir's are above the limit, then the first dirn that we @@ -141,154 +163,72 @@ } #endif -#if OLD +#endif /* Squid-2.4.DEVEL3 code */ + /* - * This is Stew Forster's selection algorithm. - * Spread load across least 3/4 of the store directories + * Spread load across all of the store directories + * + * Note: We should modify this later on to prefer sticking objects + * in the *tightest fit* swapdir to conserve space, along with the + * actual swapdir usage. But for now, this hack will do while + * testing, so you should order your swapdirs in the config file + * from smallest maxobjsize to unlimited (-1) maxobjsize. + * + * We also have to choose nleast == nconf since we need to consider + * ALL swapdirs, regardless of state. Again, this is a hack while + * we sort out the real usefulness of this algorithm. */ -static int -storeDirSelectSwapDir(void) +int +storeDirSelectSwapDir(const StoreEntry *e) { - double least_used = 1.0; - double high = (double) Config.Swap.highWaterMark / 100.0; - double u; - int dirn; - int i, j; + size_t objsize; + size_t least_size; + size_t least_objsize; + int least_load = 1000; + int load; + int dirn = -1; + int i; SwapDir *SD; - static int nleast = 0; - static int nconf = 0; - static int *dirq = NULL; - static double *diru = NULL; - /* - * Handle simplest case of a single swap directory immediately - */ - if (Config.cacheSwap.n_configured == 1) - return 0; - /* - * Initialise dirq on the first call or on change of number of dirs - */ - if (nconf != Config.cacheSwap.n_configured) { - nconf = Config.cacheSwap.n_configured; - nleast = (nconf * 3) / 4; - safe_free(dirq); - dirq = (int *) xmalloc(sizeof(int) * nleast); - safe_free(diru); - diru = (double *) xmalloc(sizeof(double) * nconf); - for (j = 0; j < nleast; j++) - dirq[j] = -1; - } - /* - * Scan for a non-negative dirn in the dirq array and return that one - */ - dirn = -1; - for (j = 0; j < nleast; j++) { - dirn = dirq[j]; - if (dirn < 0) - continue; - dirq[j] = -1; - break; - } - /* - * If we found a valid dirn return it - */ - if (dirn >= 0) - return dirn; - /* - * Now for the real guts of the algorithm - building the dirq array - */ - for (i = 0; i < nconf; i++) { - diru[i] = 1.1; - SD = &Config.cacheSwap.swapDirs[i]; + + /* Calculate the object size */ + objsize = objectLen(e); + if (objsize != -1) + objsize += e->mem_obj->swap_hdr_sz; + /* Initial defaults */ + least_size = Config.cacheSwap.swapDirs[0].cur_size; + least_objsize = Config.cacheSwap.swapDirs[0].max_objsize; + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + SD = &Config.cacheSwap.swapDirs[i]; SD->flags.selected = 0; - if (SD->flags.read_only) - continue; - u = (double) SD->cur_size / SD->max_size; - if (u > high) - continue; - diru[i] = u; - } - for (j = 0; j < nleast; j++) { - dirq[j] = -1; - least_used = 1.0; - dirn = -1; - for (i = 0; i < nconf; i++) { - if (diru[i] < least_used) { - least_used = diru[i]; - dirn = i; - } - } - if (dirn < 0) - break; - dirq[j] = dirn; - diru[dirn] = 1.1; - /* set selected flag for debugging/cachemgr only */ - Config.cacheSwap.swapDirs[dirn].flags.selected = 1; + if (SD->flags.read_only) + continue; + /* Valid for object size check */ + if (!storeDirValidSwapDirSize(i, objsize)) + continue; + load = SD->checkobj(SD, e); + if (load < 0) + continue; + if (SD->cur_size > SD->max_size) + continue; + if (load > least_load) + continue; + if ((least_objsize > 0) && (objsize > least_objsize)) + continue; + /* Only use leastsize if the load is equal */ + if ((load == least_load) && (SD->cur_size > least_size)) + continue; + least_load = load; + least_size = SD->cur_size; + dirn = i; } - /* - * Setup default return of 0 if no least found - */ - if (dirq[0] < 0) - dirq[0] = 0; - dirn = dirq[0]; - dirq[0] = -1; - return dirn; -} -#endif -int -storeDirValidFileno(int fn, int flag) -{ - int dirn = fn >> SWAP_DIR_SHIFT; - int filn = fn & SWAP_FILE_MASK; - if (dirn > Config.cacheSwap.n_configured) - return 0; - if (dirn < 0) - return 0; - if (filn < 0) - return 0; - /* - * If flag is set it means out-of-range file number should - * be considered invalid. - */ - if (flag) - if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files) - return 0; - return 1; -} + if (dirn >= 0) + Config.cacheSwap.swapDirs[dirn].flags.selected = 1; -int -storeDirMapBitTest(int fn) -{ - int dirn = fn >> SWAP_DIR_SHIFT; - int filn = fn & SWAP_FILE_MASK; - return file_map_bit_test(Config.cacheSwap.swapDirs[dirn].map, filn); + return dirn; } -void -storeDirMapBitSet(int fn) -{ - int dirn = fn >> SWAP_DIR_SHIFT; - int filn = fn & SWAP_FILE_MASK; - file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn); -} -void -storeDirMapBitReset(int fn) -{ - int dirn = fn >> SWAP_DIR_SHIFT; - int filn = fn & SWAP_FILE_MASK; - file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn); -} - -int -storeDirMapAllocate(void) -{ - int dirn = storeDirSelectSwapDir(); - SwapDir *SD = &Config.cacheSwap.swapDirs[dirn]; - int filn = file_map_allocate(SD->map, SD->suggest); - SD->suggest = filn + 1; - return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK); -} char * storeSwapDir(int dirn) @@ -297,18 +237,6 @@ return Config.cacheSwap.swapDirs[dirn].path; } -int -storeDirNumber(int swap_file_number) -{ - return swap_file_number >> SWAP_DIR_SHIFT; -} - -int -storeDirProperFileno(int dirn, int fn) -{ - return (dirn << SWAP_DIR_SHIFT) | (fn & SWAP_FILE_MASK); -} - /* * An entry written to the swap log MUST have the following * properties. @@ -316,36 +244,34 @@ * a public ADD, change the key, then log a private * DEL. So we need to log a DEL before we change a * key from public to private. - * 2. It MUST have a valid (> -1) swap_file_number. + * 2. It MUST have a valid (> -1) swap_filen. */ void storeDirSwapLog(const StoreEntry * e, int op) { - int dirn = e->swap_file_number >> SWAP_DIR_SHIFT; SwapDir *sd; - assert(dirn < Config.cacheSwap.n_configured); assert(!EBIT_TEST(e->flags, KEY_PRIVATE)); - assert(e->swap_file_number >= 0); + assert(e->swap_filen >= 0); /* * icons and such; don't write them to the swap log */ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) return; assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX); - debug(20, 3) ("storeDirSwapLog: %s %s %08X\n", + debug(20, 3) ("storeDirSwapLog: %s %s %d %08X\n", swap_log_op_str[op], storeKeyText(e->key), - e->swap_file_number); - sd = &Config.cacheSwap.swapDirs[dirn]; + e->swap_dirn, + e->swap_filen); + sd = &Config.cacheSwap.swapDirs[e->swap_dirn]; sd->log.write(sd, e, op); } void -storeDirUpdateSwapSize(int fn, size_t size, int sign) +storeDirUpdateSwapSize(SwapDir *SD, size_t size, int sign) { - int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured; int k = ((size + 1023) >> 10) * sign; - Config.cacheSwap.swapDirs[dirn].cur_size += k; + SD->cur_size += k; store_swap_size += k; if (sign > 0) n_disk_objects++; @@ -356,6 +282,9 @@ void storeDirStats(StoreEntry * sentry) { + int i; + SwapDir *SD; + storeAppendPrintf(sentry, "Store Directory Statistics:\n"); storeAppendPrintf(sentry, "Store Entries : %d\n", memInUse(MEM_STOREENTRY)); @@ -366,17 +295,15 @@ storeAppendPrintf(sentry, "Current Capacity : %d%% used, %d%% free\n", percent((int) store_swap_size, (int) Config.Swap.maxSize), percent((int) (Config.Swap.maxSize - store_swap_size), (int) Config.Swap.maxSize)); - storeUfsDirStats(sentry); /* XXX */ -} -int -storeDirMapBitsInUse(void) -{ - int i; - int n = 0; - for (i = 0; i < Config.cacheSwap.n_configured; i++) - n += Config.cacheSwap.swapDirs[i].map->n_files_in_map; - return n; + /* Now go through each swapdir, calling its statfs routine */ + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + storeAppendPrintf(sentry, "\n"); + SD = &(Config.cacheSwap.swapDirs[i]); + storeAppendPrintf(sentry, "Store Directory #%d (%s): %s\n", i, SD->type, + storeSwapDir(i)); + SD->statfs(SD, sentry); + } } void @@ -388,17 +315,14 @@ for (i = 0; i < Config.cacheSwap.n_configured; i++) { SD = &Config.cacheSwap.swapDirs[i];; Config.Swap.maxSize += SD->max_size; - if (NULL == SD->map) - SD->map = file_map_create(); SD->low_size = (int) (((float) SD->max_size * (float) Config.Swap.lowWaterMark) / 100.0); } } void -storeDirDiskFull(int fn) +storeDirDiskFull(sdirno dirn) { - int dirn = fn >> SWAP_DIR_SHIFT; SwapDir *SD = &Config.cacheSwap.swapDirs[dirn]; assert(0 <= dirn && dirn < Config.cacheSwap.n_configured); SD->max_size = SD->cur_size; @@ -432,6 +356,10 @@ * storeDirWriteCleanLogs * * Writes a "clean" swap log file from in-memory metadata. + * This is a rewrite of the original function to troll each + * StoreDir and write the logs, and flush at the end of + * the run. Thanks goes to Eric Stern, since this solution + * came out of his COSS code. */ #define CLEAN_BUF_SZ 16384 int @@ -445,8 +373,6 @@ int dirn; #if HEAP_REPLACEMENT int node; -#else - int j; #endif if (store_dirs_rebuilding) { debug(20, 1) ("Not currently OK to rewrite swap log.\n"); @@ -462,58 +388,23 @@ debug(20, 1) ("log.clean.open() failed for dir #%d\n", sd->index); continue; } -#if !HEAP_REPLACEMENT - storeDirLRUWalkInitHead(sd); -#endif - } -#if HEAP_REPLACEMENT - if (NULL == store_heap) - return 0; - for (node = 0; node < heap_nodes(store_heap); node++) { - e = (StoreEntry *) heap_peep(store_heap, node); - if (e->swap_file_number < 0) - continue; - if (e->swap_status != SWAPOUT_DONE) - continue; - if (e->swap_file_sz <= 0) - continue; - if (EBIT_TEST(e->flags, RELEASE_REQUEST)) - continue; - if (EBIT_TEST(e->flags, KEY_PRIVATE)) - continue; - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) - continue; - dirn = storeDirNumber(e->swap_file_number); - sd = &Config.cacheSwap.swapDirs[dirn]; - if (NULL == sd->log.clean.write) - continue; - sd->log.clean.write(e, sd); - if ((++n & 0xFFFF) == 0) { - getCurrentTime(); - debug(20, 1) (" %7d entries written so far.\n", n); - } - } - /* flush */ - for (dirn = 0; dirn < N; dirn++) { - sd = &Config.cacheSwap.swapDirs[dirn]; if (NULL == sd->log.clean.write) continue; - sd->log.clean.write(NULL, sd); - } +#if HEAP_REPLACEMENT + if (NULL == sd->repl.heap.heap) + continue; +#endif +#if HEAP_REPLACEMENT + for (node = 0; node < heap_nodes(sd->repl.heap.heap); node++) #else - do { - j = 0; - for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { - sd = &Config.cacheSwap.swapDirs[dirn]; - if (NULL == sd->log.clean.write) - continue; - e = storeDirLRUWalkNext(sd); - if (NULL == e) { - sd->log.clean.write(NULL, sd); - continue; - } - j++; - if (e->swap_file_number < 0) + storeDirLRUWalkInitHead(sd); + while ((e = storeDirLRUWalkNext(sd)) != NULL) +#endif + { +#if HEAP_REPLACEMENT + e = (StoreEntry *) heap_peep(sd->repl.heap.heap, node); +#endif + if (e->swap_filen < 0) continue; if (e->swap_status != SWAPOUT_DONE) continue; @@ -531,8 +422,9 @@ debug(20, 1) (" %7d entries written so far.\n", n); } } - } while (j > 0); -#endif + /* Flush */ + sd->log.clean.write(NULL, sd); + } if (reopen) storeDirOpenSwapLogs(); getCurrentTime(); @@ -544,14 +436,47 @@ } #undef CLEAN_BUF_SZ +/* + * sync all avaliable fs'es .. + */ +void +storeDirSync(void) +{ + int i; + SwapDir *SD; + + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->sync != NULL) + SD->sync(SD); + } +} + +/* + * handle callbacks all avaliable fs'es .. + */ +void +storeDirCallback(void) +{ + int i; + SwapDir *SD; + + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->callback != NULL) + SD->callback(SD); + } +} + +#if 0 /* from Squid-2.4.DEVEL3 */ void storeDirLRUDelete(StoreEntry * e) { SwapDir *sd; - if (e->swap_file_number < 0) + if (e->swap_filen < 0) return; - sd = &Config.cacheSwap.swapDirs[e->swap_file_number >> SWAP_DIR_SHIFT]; - dlinkDelete(&e->lru, &sd->lru_list); + sd = &Config.cacheSwap.swapDirs[e->swap_dirn]; + dlinkDelete(&e->lru, &sd->repl.lru.list); } void @@ -560,23 +485,24 @@ SwapDir *sd; if (e->swap_file_number < 0) return; - sd = &Config.cacheSwap.swapDirs[e->swap_file_number >> SWAP_DIR_SHIFT]; - dlinkAdd(e, &e->lru, &sd->lru_list); + sd = &Config.cacheSwap.swapDirs[e->swap_dirn]; + dlinkAdd(e, &e->lru, &sd->repl.lru.list); } +#endif /* from Squid-2.4.DEVEL3 */ static void storeDirLRUWalkInitHead(SwapDir * sd) { - sd->lru_walker = sd->lru_list.head; + sd->repl.lru.walker = sd->repl.lru.list.head; } static void * storeDirLRUWalkNext(SwapDir * sd) { void *p; - if (NULL == sd->lru_walker) + if (NULL == sd->repl.lru.walker) return NULL; - p = sd->lru_walker->data; - sd->lru_walker = sd->lru_walker->next; + p = sd->repl.lru.walker->data; + sd->repl.lru.walker = sd->repl.lru.walker->next; return p; } --- squid/src/store_dir_ufs.c Wed Feb 14 00:43:59 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,1433 +0,0 @@ - -/* - * $Id: store_dir_ufs.c,v 1.1.1.1.12.1 2000/04/17 00:13:10 hno Exp $ - * - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Duane Wessels - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#if HAVE_STATVFS -#if HAVE_SYS_STATVFS_H -#include -#endif -#endif - -#define DefaultLevelOneDirs 16 -#define DefaultLevelTwoDirs 256 -#define STORE_META_BUFSZ 4096 - -typedef struct _RebuildState RebuildState; -struct _RebuildState { - SwapDir *sd; - int n_read; - FILE *log; - int speed; - int curlvl1; - int curlvl2; - struct { - unsigned int need_to_validate:1; - unsigned int clean:1; - unsigned int init:1; - } flags; - int done; - int in_dir; - int fn; - struct dirent *entry; - DIR *td; - char fullpath[SQUID_MAXPATHLEN]; - char fullfilename[SQUID_MAXPATHLEN]; - struct _store_rebuild_data counts; -}; - -static int n_ufs_dirs = 0; -static int *ufs_dir_index = NULL; - -static char *storeUfsSwapSubDir(SwapDir *, int subdirn); -static int storeUfsCreateDirectory(const char *path, int); -static int storeUfsVerifyCacheDirs(SwapDir *); -static int storeUfsVerifyDirectory(const char *path); -static void storeUfsCreateSwapSubDirs(SwapDir *); -static char *storeUfsDirSwapLogFile(SwapDir *, const char *); -static EVH storeRebuildFromDirectory; -static EVH storeRebuildFromSwapLog; -static int storeGetNextFile(RebuildState *, int *sfileno, int *size); -static StoreEntry *storeAddDiskRestore(const cache_key * key, - int file_number, - size_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_num32 refcount, - u_short flags, - int clean); -static void storeUfsDirRebuild(SwapDir * sd); -static void storeUfsDirCloseTmpSwapLog(SwapDir * sd); -static FILE *storeUfsDirOpenTmpSwapLog(SwapDir *, int *, int *); -static STLOGOPEN storeUfsDirOpenSwapLog; -static STLOGCLEANOPEN storeUfsDirWriteCleanOpen; -static void storeUfsDirWriteCleanClose(SwapDir * sd); -static STLOGCLEANWRITE storeUfsDirWriteCleanEntry; -static STLOGCLOSE storeUfsDirCloseSwapLog; -static STLOGWRITE storeUfsDirSwapLog; -static STNEWFS storeUfsDirNewfs; -static QS rev_int_sort; -static int storeUfsDirClean(int swap_index); -static EVH storeUfsDirCleanEvent; -static int storeUfsDirIs(SwapDir * sd); -static int storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2); - -static char * -storeUfsSwapSubDir(SwapDir * sd, int subdirn) -{ - LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - assert(0 <= subdirn && subdirn < sd->u.ufs.l1); - snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn); - return fullfilename; -} - -static int -storeUfsCreateDirectory(const char *path, int should_exist) -{ - int created = 0; - struct stat st; - getCurrentTime(); - if (0 == stat(path, &st)) { - if (S_ISDIR(st.st_mode)) { - debug(20, should_exist ? 3 : 1) ("%s exists\n", path); - } else { - fatalf("Swap directory %s is not a directory.", path); - } - } else if (0 == mkdir(path, 0755)) { - debug(20, should_exist ? 1 : 3) ("%s created\n", path); - created = 1; - } else { - fatalf("Failed to make swap directory %s: %s", - path, xstrerror()); - } - return created; -} - -static int -storeUfsVerifyDirectory(const char *path) -{ - struct stat sb; - if (stat(path, &sb) < 0) { - debug(20, 0) ("%s: %s\n", path, xstrerror()); - return -1; - } - if (S_ISDIR(sb.st_mode) == 0) { - debug(20, 0) ("%s is not a directory\n", path); - return -1; - } - return 0; -} - -/* - * This function is called by storeUfsDirInit(). If this returns < 0, - * then Squid exits, complains about swap directories not - * existing, and instructs the admin to run 'squid -z' - */ -static int -storeUfsVerifyCacheDirs(SwapDir * sd) -{ - int j; - const char *path = sd->path; - if (storeUfsVerifyDirectory(path) < 0) - return -1; - for (j = 0; j < sd->u.ufs.l1; j++) { - path = storeUfsSwapSubDir(sd, j); - if (storeUfsVerifyDirectory(path) < 0) - return -1; - } - return 0; -} - -static void -storeUfsCreateSwapSubDirs(SwapDir * sd) -{ - int i, k; - int should_exist; - LOCAL_ARRAY(char, name, MAXPATHLEN); - for (i = 0; i < sd->u.ufs.l1; i++) { - snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i); - if (storeUfsCreateDirectory(name, 0)) - should_exist = 0; - else - should_exist = 1; - debug(47, 1) ("Making directories in %s\n", name); - for (k = 0; k < sd->u.ufs.l2; k++) { - snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k); - storeUfsCreateDirectory(name, should_exist); - } - } -} - -static char * -storeUfsDirSwapLogFile(SwapDir * sd, const char *ext) -{ - LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); - LOCAL_ARRAY(char, digit, 32); - if (Config.Log.swap) { - xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64); - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); - } else { - xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); - strcat(path, "/swap.state"); - } - if (ext) - strncat(path, ext, 16); - return path; -} - -static void -storeUfsDirOpenSwapLog(SwapDir * sd) -{ - char *path; - int fd; - path = storeUfsDirSwapLogFile(sd, NULL); - fd = file_open(path, O_WRONLY | O_CREAT); - if (fd < 0) { - debug(50, 1) ("%s: %s\n", path, xstrerror()); - fatal("storeUfsDirOpenSwapLog: Failed to open swap log."); - } - debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); - sd->u.ufs.swaplog_fd = fd; - if (0 == n_ufs_dirs) - assert(NULL == ufs_dir_index); - n_ufs_dirs++; - assert(n_ufs_dirs <= Config.cacheSwap.n_configured); -} - -static void -storeUfsDirCloseSwapLog(SwapDir * sd) -{ - if (sd->u.ufs.swaplog_fd < 0) /* not open */ - return; - file_close(sd->u.ufs.swaplog_fd); - debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", - sd->index, sd->u.ufs.swaplog_fd); - sd->u.ufs.swaplog_fd = -1; - n_ufs_dirs--; - assert(n_ufs_dirs >= 0); - if (0 == n_ufs_dirs) - safe_free(ufs_dir_index); -} - -void -storeUfsDirInit(SwapDir * sd) -{ - static int started_clean_event = 0; - static const char *errmsg = - "\tFailed to verify one of the swap directories, Check cache.log\n" - "\tfor details. Run 'squid -z' to create swap directories\n" - "\tif needed, or if running Squid for the first time."; - if (storeUfsVerifyCacheDirs(sd) < 0) - fatal(errmsg); - storeUfsDirOpenSwapLog(sd); - storeUfsDirRebuild(sd); - if (!started_clean_event) { - eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL, 15.0, 1); - started_clean_event = 1; - } -} - -static void -storeRebuildFromDirectory(void *data) -{ - RebuildState *rb = data; - LOCAL_ARRAY(char, hdr_buf, DISK_PAGE_SIZE); - StoreEntry *e = NULL; - StoreEntry tmpe; - cache_key key[MD5_DIGEST_CHARS]; - int sfileno = 0; - int count; - int size; - struct stat sb; - int swap_hdr_len; - int fd = -1; - tlv *tlv_list; - tlv *t; - assert(rb != NULL); - if (opt_foreground_rebuild) - getCurrentTime(); - debug(20, 3) ("storeRebuildFromDirectory: DIR #%d\n", rb->sd->index); - for (count = 0; count < rb->speed; count++) { - assert(fd == -1); - fd = storeGetNextFile(rb, &sfileno, &size); - if (fd == -2) { - debug(20, 1) ("Done scanning %s swaplog (%d entries)\n", - rb->sd->path, rb->n_read); - store_dirs_rebuilding--; - storeUfsDirCloseTmpSwapLog(rb->sd); - storeRebuildComplete(&rb->counts); - cbdataFree(rb); - return; - } else if (fd < 0) { - continue; - } - assert(fd > -1); - /* lets get file stats here */ - if (fstat(fd, &sb) < 0) { - debug(20, 1) ("storeRebuildFromDirectory: fstat(FD %d): %s\n", - fd, xstrerror()); - file_close(fd); - store_open_disk_fd--; - fd = -1; - continue; - } - if ((++rb->counts.scancount & 0x3FFF) == 0) - debug(20, 1) (" %s %7d files opened so far.\n", - rb->sd->path, rb->counts.scancount); - debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno); - Counter.syscalls.disk.reads++; - if (read(fd, hdr_buf, DISK_PAGE_SIZE) < 0) { - debug(20, 1) ("storeRebuildFromDirectory: read(FD %d): %s\n", - fd, xstrerror()); - file_close(fd); - store_open_disk_fd--; - fd = -1; - continue; - } - file_close(fd); - store_open_disk_fd--; - fd = -1; - swap_hdr_len = 0; -#if USE_TRUNCATE - if (sb.st_size == 0) - continue; -#endif - tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); - if (tlv_list == NULL) { - debug(20, 1) ("storeRebuildFromDirectory: failed to get meta data\n"); - storeUnlink(sfileno); - continue; - } - debug(20, 3) ("storeRebuildFromDirectory: successful swap meta unpacking\n"); - memset(key, '\0', MD5_DIGEST_CHARS); - memset(&tmpe, '\0', sizeof(StoreEntry)); - for (t = tlv_list; t; t = t->next) { - switch (t->type) { - case STORE_META_KEY: - assert(t->length == MD5_DIGEST_CHARS); - xmemcpy(key, t->value, MD5_DIGEST_CHARS); - break; - case STORE_META_STD: - assert(t->length == STORE_HDR_METASIZE); - xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); - break; - default: - break; - } - } - storeSwapTLVFree(tlv_list); - tlv_list = NULL; - if (storeKeyNull(key)) { - debug(20, 1) ("storeRebuildFromDirectory: NULL key\n"); - storeUnlink(sfileno); - continue; - } - tmpe.key = key; - /* check sizes */ - if (tmpe.swap_file_sz == 0) { - tmpe.swap_file_sz = sb.st_size; - } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { - tmpe.swap_file_sz = sb.st_size; - } else if (tmpe.swap_file_sz != sb.st_size) { - debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d!=%d\n", - tmpe.swap_file_sz, (int) sb.st_size); - storeUnlink(sfileno); - continue; - } - if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { - storeUnlink(sfileno); - rb->counts.badflags++; - continue; - } - e = storeGet(key); - if (e && e->lastref >= tmpe.lastref) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - rb->counts.dupcount++; - continue; - } else if (NULL != e) { - /* URL already exists, this swapfile not being used */ - /* junk old, load new */ - storeRelease(e); /* release old entry */ - rb->counts.dupcount++; - } - rb->counts.objcount++; - storeEntryDump(&tmpe, 5); - e = storeAddDiskRestore(key, - sfileno, - tmpe.swap_file_sz, - tmpe.expires, - tmpe.timestamp, - tmpe.lastref, - tmpe.lastmod, - tmpe.refcount, /* refcount */ - tmpe.flags, /* flags */ - (int) rb->flags.clean); - } - eventAdd("storeRebuild", storeRebuildFromDirectory, rb, 0.0, 0); -} - -static void -storeRebuildFromSwapLog(void *data) -{ - RebuildState *rb = data; - StoreEntry *e = NULL; - storeSwapLogData s; - size_t ss = sizeof(storeSwapLogData); - int count; - int used; /* is swapfile already in use? */ - int disk_entry_newer; /* is the log entry newer than current entry? */ - double x; - assert(rb != NULL); - /* load a number of objects per invocation */ - for (count = 0; count < rb->speed; count++) { - if (fread(&s, ss, 1, rb->log) != 1) { - debug(20, 1) ("Done reading %s swaplog (%d entries)\n", - rb->sd->path, rb->n_read); - fclose(rb->log); - rb->log = NULL; - store_dirs_rebuilding--; - storeUfsDirCloseTmpSwapLog(rb->sd); - storeRebuildComplete(&rb->counts); - cbdataFree(rb); - return; - } - rb->n_read++; - if (s.op <= SWAP_LOG_NOP) - continue; - if (s.op >= SWAP_LOG_MAX) - continue; - s.swap_file_number = storeDirProperFileno(rb->sd->index, s.swap_file_number); - debug(20, 3) ("storeRebuildFromSwapLog: %s %s %08X\n", - swap_log_op_str[(int) s.op], - storeKeyText(s.key), - s.swap_file_number); - if (s.op == SWAP_LOG_ADD) { - (void) 0; - } else if (s.op == SWAP_LOG_DEL) { - if ((e = storeGet(s.key)) != NULL) { - /* - * Make sure we don't unlink the file, it might be - * in use by a subsequent entry. Also note that - * we don't have to subtract from store_swap_size - * because adding to store_swap_size happens in - * the cleanup procedure. - */ - storeExpireNow(e); - storeReleaseRequest(e); - storeSwapFileNumberSet(e, -1); - rb->counts.objcount--; - rb->counts.cancelcount++; - } - continue; - } else { - x = log(++rb->counts.bad_log_op) / log(10.0); - if (0.0 == x - (double) (int) x) - debug(20, 1) ("WARNING: %d invalid swap log entries found\n", - rb->counts.bad_log_op); - rb->counts.invalid++; - continue; - } - if ((++rb->counts.scancount & 0x3FFF) == 0) - debug(20, 1) (" %7d %s Entries read so far.\n", - rb->counts.scancount, rb->sd->path); - if (!storeDirValidFileno(s.swap_file_number, 0)) { - rb->counts.invalid++; - continue; - } - if (EBIT_TEST(s.flags, KEY_PRIVATE)) { - rb->counts.badflags++; - continue; - } - e = storeGet(s.key); - used = storeDirMapBitTest(s.swap_file_number); - /* If this URL already exists in the cache, does the swap log - * appear to have a newer entry? Compare 'lastref' from the - * swap log to e->lastref. */ - disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; - if (used && !disk_entry_newer) { - /* log entry is old, ignore it */ - rb->counts.clashcount++; - continue; - } else if (used && e && e->swap_file_number == s.swap_file_number) { - /* swapfile taken, same URL, newer, update meta */ - if (e->store_status == STORE_OK) { - e->lastref = s.timestamp; - e->timestamp = s.timestamp; - e->expires = s.expires; - e->lastmod = s.lastmod; - e->flags = s.flags; - e->refcount += s.refcount; -#if HEAP_REPLACEMENT - storeHeapPositionUpdate(e); -#endif - } else { - debug_trap("storeRebuildFromSwapLog: bad condition"); - debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); - } - continue; - } else if (used) { - /* swapfile in use, not by this URL, log entry is newer */ - /* This is sorta bad: the log entry should NOT be newer at this - * point. If the log is dirty, the filesize check should have - * caught this. If the log is clean, there should never be a - * newer entry. */ - debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n", - s.swap_file_number); - /* I'm tempted to remove the swapfile here just to be safe, - * but there is a bad race condition in the NOVM version if - * the swapfile has recently been opened for writing, but - * not yet opened for reading. Because we can't map - * swapfiles back to StoreEntrys, we don't know the state - * of the entry using that file. */ - /* We'll assume the existing entry is valid, probably because - * were in a slow rebuild and the the swap file number got taken - * and the validation procedure hasn't run. */ - assert(rb->flags.need_to_validate); - rb->counts.clashcount++; - continue; - } else if (e && !disk_entry_newer) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - rb->counts.dupcount++; - continue; - } else if (e) { - /* key already exists, this swapfile not being used */ - /* junk old, load new */ - storeExpireNow(e); - storeReleaseRequest(e); - storeSwapFileNumberSet(e, -1); - rb->counts.dupcount++; - } else { - /* URL doesnt exist, swapfile not in use */ - /* load new */ - (void) 0; - } - /* update store_swap_size */ - rb->counts.objcount++; - e = storeAddDiskRestore(s.key, - s.swap_file_number, - s.swap_file_sz, - s.expires, - s.timestamp, - s.lastref, - s.lastmod, - s.refcount, - s.flags, - (int) rb->flags.clean); - storeDirSwapLog(e, SWAP_LOG_ADD); - } - eventAdd("storeRebuild", storeRebuildFromSwapLog, rb, 0.0, 0); -} - -static int -storeGetNextFile(RebuildState * rb, int *sfileno, int *size) -{ - int fd = -1; - int used = 0; - int dirs_opened = 0; - debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n", - rb->flags.init, - rb->sd->index, - rb->curlvl1, - rb->curlvl2); - if (rb->done) - return -2; - while (fd < 0 && rb->done == 0) { - fd = -1; - if (0 == rb->flags.init) { /* initialize, open first file */ - rb->done = 0; - rb->curlvl1 = 0; - rb->curlvl2 = 0; - rb->in_dir = 0; - rb->flags.init = 1; - assert(Config.cacheSwap.n_configured > 0); - } - if (0 == rb->in_dir) { /* we need to read in a new directory */ - snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", - rb->sd->path, - rb->curlvl1, rb->curlvl2); - if (rb->flags.init && rb->td != NULL) - closedir(rb->td); - rb->td = NULL; - if (dirs_opened) - return -1; - rb->td = opendir(rb->fullpath); - dirs_opened++; - if (rb->td == NULL) { - debug(50, 1) ("storeGetNextFile: opendir: %s: %s\n", - rb->fullpath, xstrerror()); - } else { - rb->entry = readdir(rb->td); /* skip . and .. */ - rb->entry = readdir(rb->td); - if (rb->entry == NULL && errno == ENOENT) - debug(20, 1) ("storeGetNextFile: directory does not exist!.\n"); - debug(20, 3) ("storeGetNextFile: Directory %s\n", rb->fullpath); - } - } - if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) { - rb->in_dir++; - if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) { - debug(20, 3) ("storeGetNextFile: invalid %s\n", - rb->entry->d_name); - continue; - } - if (!storeUfsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) { - debug(20, 3) ("storeGetNextFile: %08X does not belong in %d/%d/%d\n", - rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2); - continue; - } - rb->fn = storeDirProperFileno(rb->sd->index, rb->fn); - used = storeDirMapBitTest(rb->fn); - if (used) { - debug(20, 3) ("storeGetNextFile: Locked, continuing with next.\n"); - continue; - } - snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s", - rb->fullpath, rb->entry->d_name); - debug(20, 3) ("storeGetNextFile: Opening %s\n", rb->fullfilename); - fd = file_open(rb->fullfilename, O_RDONLY); - if (fd < 0) - debug(50, 1) ("storeGetNextFile: %s: %s\n", rb->fullfilename, xstrerror()); - else - store_open_disk_fd++; - continue; - } - rb->in_dir = 0; - if (++rb->curlvl2 < rb->sd->u.ufs.l2) - continue; - rb->curlvl2 = 0; - if (++rb->curlvl1 < rb->sd->u.ufs.l1) - continue; - rb->curlvl1 = 0; - rb->done = 1; - } - *sfileno = rb->fn; - return fd; -} - -/* Add a new object to the cache with empty memory copy and pointer to disk - * use to rebuild store from disk. */ -static StoreEntry * -storeAddDiskRestore(const cache_key * key, - int file_number, - size_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_num32 refcount, - u_short flags, - int clean) -{ - StoreEntry *e = NULL; - debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); - /* if you call this you'd better be sure file_number is not - * already in use! */ - e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); - e->store_status = STORE_OK; - storeSetMemStatus(e, NOT_IN_MEMORY); - e->swap_status = SWAPOUT_DONE; - storeSwapFileNumberSet(e, file_number); - e->swap_file_sz = swap_file_sz; - e->lock_count = 0; -#if !HEAP_REPLACEMENT - e->refcount = 0; -#endif - e->lastref = lastref; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->refcount = refcount; - e->flags = flags; - EBIT_SET(e->flags, ENTRY_CACHABLE); - EBIT_CLR(e->flags, RELEASE_REQUEST); - EBIT_CLR(e->flags, KEY_PRIVATE); - e->ping_status = PING_NONE; - EBIT_CLR(e->flags, ENTRY_VALIDATED); - storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ - return e; -} - -static void -storeUfsDirRebuild(SwapDir * sd) -{ - RebuildState *rb = xcalloc(1, sizeof(*rb)); - int clean = 0; - int zero = 0; - FILE *fp; - EVH *func = NULL; - rb->sd = sd; - rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; - /* - * If the swap.state file exists in the cache_dir, then - * we'll use storeRebuildFromSwapLog(), otherwise we'll - * use storeRebuildFromDirectory() to open up each file - * and suck in the meta data. - */ - fp = storeUfsDirOpenTmpSwapLog(sd, &clean, &zero); - if (fp == NULL || zero) { - if (fp != NULL) - fclose(fp); - func = storeRebuildFromDirectory; - } else { - func = storeRebuildFromSwapLog; - rb->log = fp; - rb->flags.clean = (unsigned int) clean; - } - if (!clean) - rb->flags.need_to_validate = 1; - debug(20, 1) ("Rebuilding storage in %s (%s)\n", - sd->path, clean ? "CLEAN" : "DIRTY"); - store_dirs_rebuilding++; - cbdataAdd(rb, cbdataXfree, 0); - eventAdd("storeRebuild", func, rb, 0.0, 0); -} - -static void -storeUfsDirCloseTmpSwapLog(SwapDir * sd) -{ - char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); - char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new")); - int fd; - file_close(sd->u.ufs.swaplog_fd); -#ifdef _SQUID_OS2_ - if (unlink(swaplog_path) < 0) { - debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); - fatal("storeUfsDirCloseTmpSwapLog: unlink failed"); - } -#endif - if (xrename(new_path, swaplog_path) < 0) { - fatal("storeUfsDirCloseTmpSwapLog: rename failed"); - } - fd = file_open(swaplog_path, O_WRONLY | O_CREAT); - if (fd < 0) { - debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); - fatal("storeUfsDirCloseTmpSwapLog: Failed to open swap log."); - } - safe_free(swaplog_path); - safe_free(new_path); - sd->u.ufs.swaplog_fd = fd; - debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); -} - -static FILE * -storeUfsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) -{ - char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); - char *clean_path = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean")); - char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new")); - struct stat log_sb; - struct stat clean_sb; - FILE *fp; - int fd; - if (stat(swaplog_path, &log_sb) < 0) { - debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index); - safe_free(swaplog_path); - safe_free(clean_path); - safe_free(new_path); - return NULL; - } - *zero_flag = log_sb.st_size == 0 ? 1 : 0; - /* close the existing write-only FD */ - if (sd->u.ufs.swaplog_fd >= 0) - file_close(sd->u.ufs.swaplog_fd); - /* open a write-only FD for the new log */ - fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC); - if (fd < 0) { - debug(50, 1) ("%s: %s\n", new_path, xstrerror()); - fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); - } - sd->u.ufs.swaplog_fd = fd; - /* open a read-only stream of the old log */ - fp = fopen(swaplog_path, "r"); - if (fp == NULL) { - debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); - fatal("Failed to open swap log for reading"); - } - commSetCloseOnExec(fileno(fp)); - memset(&clean_sb, '\0', sizeof(struct stat)); - if (stat(clean_path, &clean_sb) < 0) - *clean_flag = 0; - else if (clean_sb.st_mtime < log_sb.st_mtime) - *clean_flag = 0; - else - *clean_flag = 1; - safeunlink(clean_path, 1); - safe_free(swaplog_path); - safe_free(clean_path); - safe_free(new_path); - return fp; -} - -struct _clean_state { - char *cur; - char *new; - char *cln; - char *outbuf; - off_t outbuf_offset; - int fd; -}; - -#define CLEAN_BUF_SZ 16384 -/* - * Begin the process to write clean cache state. For UFS this means - * opening some log files and allocating write buffers. Return 0 if - * we succeed, and assign the 'func' and 'data' return pointers. - */ -static int -storeUfsDirWriteCleanOpen(SwapDir * sd) -{ - struct _clean_state *state = xcalloc(1, sizeof(*state)); - struct stat sb; - sd->log.clean.write = NULL; - sd->log.clean.state = NULL; - state->cur = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); - state->new = xstrdup(storeUfsDirSwapLogFile(sd, ".clean")); - state->cln = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean")); - state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); - state->outbuf_offset = 0; - unlink(state->new); - unlink(state->cln); - state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); - if (state->fd < 0) - return -1; - debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n", - state->new, state->fd); -#if HAVE_FCHMOD - if (stat(state->cur, &sb) == 0) - fchmod(state->fd, sb.st_mode); -#endif - sd->log.clean.write = storeUfsDirWriteCleanEntry; - sd->log.clean.state = state; - return 0; -} - -/* - * "write" an entry to the clean log file. - */ -static void -storeUfsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) -{ - storeSwapLogData s; - static size_t ss = sizeof(storeSwapLogData); - struct _clean_state *state = sd->log.clean.state; - if (NULL == e) { - storeUfsDirWriteCleanClose(sd); - return; - } - memset(&s, '\0', ss); - s.op = (char) SWAP_LOG_ADD; - s.swap_file_number = e->swap_file_number; - s.timestamp = e->timestamp; - s.lastref = e->lastref; - s.expires = e->expires; - s.lastmod = e->lastmod; - s.swap_file_sz = e->swap_file_sz; - s.refcount = e->refcount; - s.flags = e->flags; - xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); - xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); - state->outbuf_offset += ss; - /* buffered write */ - if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { - if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", - state->new, xstrerror()); - debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); - file_close(state->fd); - state->fd = -1; - unlink(state->new); - safe_free(state); - sd->log.clean.state = NULL; - sd->log.clean.write = NULL; - } else { - state->outbuf_offset = 0; - } - } -} - -static void -storeUfsDirWriteCleanClose(SwapDir * sd) -{ - struct _clean_state *state = sd->log.clean.state; - if (state->fd < 0) - return; - if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", - state->new, xstrerror()); - debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile " - "not replaced.\n"); - file_close(state->fd); - state->fd = -1; - unlink(state->new); - } - safe_free(state->outbuf); - /* - * You can't rename open files on Microsoft "operating systems" - * so we have to close before renaming. - */ - storeUfsDirCloseSwapLog(sd); - /* rename */ - if (state->fd >= 0) { -#ifdef _SQUID_OS2_ - file_close(state->fd); - state->fd = -1; - if (unlink(cur) < 0) - debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n", - xstrerror(), cur); -#endif - xrename(state->new, state->cur); - } - /* touch a timestamp file if we're not still validating */ - if (store_dirs_rebuilding) - (void) 0; - else if (state->fd < 0) - (void) 0; - else - file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC)); - /* close */ - safe_free(state->cur); - safe_free(state->new); - safe_free(state->cln); - if (state->fd >= 0) - file_close(state->fd); - state->fd = -1; - safe_free(state); - sd->log.clean.state = NULL; - sd->log.clean.write = NULL; -} - -static void -storeUfsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) -{ - storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); - s->op = (char) op; - s->swap_file_number = e->swap_file_number; - s->timestamp = e->timestamp; - s->lastref = e->lastref; - s->expires = e->expires; - s->lastmod = e->lastmod; - s->swap_file_sz = e->swap_file_sz; - s->refcount = e->refcount; - s->flags = e->flags; - xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); - file_write(sd->u.ufs.swaplog_fd, - -1, - s, - sizeof(storeSwapLogData), - NULL, - NULL, - xfree); -} - -static void -storeUfsDirNewfs(SwapDir * sd) -{ - debug(47, 3) ("Creating swap space in %s\n", sd->path); - storeUfsCreateDirectory(sd->path, 0); - storeUfsCreateSwapSubDirs(sd); -} - -static int -rev_int_sort(const void *A, const void *B) -{ - const int *i1 = A; - const int *i2 = B; - return *i2 - *i1; -} - -static int -storeUfsDirClean(int swap_index) -{ - DIR *dp = NULL; - struct dirent *de = NULL; - LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); - LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); -#if USE_TRUNCATE - struct stat sb; -#endif - int files[20]; - int swapfileno; - int fn; /* same as swapfileno, but with dirn bits set */ - int n = 0; - int k = 0; - int N0, N1, N2; - int D0, D1, D2; - N0 = n_ufs_dirs; - D0 = ufs_dir_index[swap_index % N0]; - N1 = Config.cacheSwap.swapDirs[D0].u.ufs.l1; - D1 = (swap_index / N0) % N1; - N2 = Config.cacheSwap.swapDirs[D0].u.ufs.l2; - D2 = ((swap_index / N0) / N1) % N2; - snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X", - Config.cacheSwap.swapDirs[D0].path, D1, D2); - debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1); - dp = opendir(p1); - if (dp == NULL) { - if (errno == ENOENT) { - debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1); - if (mkdir(p1, 0777) == 0) - return 0; - } - debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror()); - safeunlink(p1, 1); - return 0; - } - while ((de = readdir(dp)) != NULL && k < 20) { - if (sscanf(de->d_name, "%X", &swapfileno) != 1) - continue; - fn = storeDirProperFileno(D0, swapfileno); - if (storeDirValidFileno(fn, 1)) - if (storeDirMapBitTest(fn)) - if (storeUfsFilenoBelongsHere(fn, D0, D1, D2)) - continue; -#if USE_TRUNCATE - if (!stat(de->d_name, &sb)) - if (sb.st_size == 0) - continue; -#endif - files[k++] = swapfileno; - } - closedir(dp); - if (k == 0) - return 0; - qsort(files, k, sizeof(int), rev_int_sort); - if (k > 10) - k = 10; - for (n = 0; n < k; n++) { - debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]); - snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); -#if USE_TRUNCATE - truncate(p2, 0); -#else - safeunlink(p2, 0); -#endif - Counter.swap_files_cleaned++; - } - debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1); - return k; -} - -static void -storeUfsDirCleanEvent(void *unused) -{ - static int swap_index = 0; - int i; - int j = 0; - int n = 0; - /* - * Assert that there are UFS cache_dirs configured, otherwise - * we should never be called. - */ - assert(n_ufs_dirs); - if (NULL == ufs_dir_index) { - SwapDir *sd; - /* - * Initialize the little array that translates UFS cache_dir - * number into the Config.cacheSwap.swapDirs array index. - */ - ufs_dir_index = xcalloc(n_ufs_dirs, sizeof(*ufs_dir_index)); - for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { - sd = &Config.cacheSwap.swapDirs[i]; - if (!storeUfsDirIs(sd)) - continue; - ufs_dir_index[n++] = i; - j += (sd->u.ufs.l1 * sd->u.ufs.l2); - } - assert(n == n_ufs_dirs); - /* - * Start the storeUfsDirClean() swap_index with a random - * value. j equals the total number of UFS level 2 - * swap directories - */ - swap_index = (int) (squid_random() % j); - } - if (0 == store_dirs_rebuilding) { - n = storeUfsDirClean(swap_index); - swap_index++; - } - eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL, - 15.0 * exp(-0.25 * n), 1); -} - -static int -storeUfsDirIs(SwapDir * sd) -{ - if (sd->type == SWAPDIR_UFS) - return 1; - if (sd->type == SWAPDIR_ASYNCUFS) - return 1; -#if USE_DISKD - if (sd->type == SWAPDIR_DISKD) - return 1; -#endif - return 0; -} - -/* - * Does swapfile number 'fn' belong in cachedir #F0, - * level1 dir #F1, level2 dir #F2? - * - * Don't check that (fn >> SWAP_DIR_SHIFT) == F0 because - * 'fn' may not have the directory bits set. - */ -static int -storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2) -{ - int D1, D2; - int L1, L2; - int filn = fn & SWAP_FILE_MASK; - assert(F0 < Config.cacheSwap.n_configured); - L1 = Config.cacheSwap.swapDirs[F0].u.ufs.l1; - L2 = Config.cacheSwap.swapDirs[F0].u.ufs.l2; - D1 = ((filn / L2) / L2) % L1; - if (F1 != D1) - return 0; - D2 = (filn / L2) % L2; - if (F2 != D2) - return 0; - return 1; -} - -/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ - -void -storeUfsDirStats(StoreEntry * sentry) -{ - int i; - SwapDir *SD; -#if HAVE_STATVFS - struct statvfs sfs; -#endif - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - SD = &Config.cacheSwap.swapDirs[i]; - storeAppendPrintf(sentry, "\n"); - storeAppendPrintf(sentry, "Store Directory #%d: %s\n", i, SD->path); - storeAppendPrintf(sentry, "First level subdirectories: %d\n", SD->u.ufs.l1); - storeAppendPrintf(sentry, "Second level subdirectories: %d\n", SD->u.ufs.l2); - storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); - storeAppendPrintf(sentry, "LoWater Size: %d KB\n", SD->low_size); - storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); - storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", - 100.0 * SD->cur_size / SD->max_size); - storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", - SD->map->n_files_in_map, SD->map->max_n_files, - percent(SD->map->n_files_in_map, SD->map->max_n_files)); - storeAppendPrintf(sentry, "Removals: %d\n", SD->removals); - storeAppendPrintf(sentry, " Scanned: %d\n", SD->scanned); -#if HAVE_STATVFS -#define fsbtoblk(num, fsbs, bs) \ - (((fsbs) != 0 && (fsbs) < (bs)) ? \ - (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) - if (!statvfs(SD->path, &sfs)) { - storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", - fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), - fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), - percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); - storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", - sfs.f_files - sfs.f_ffree, sfs.f_files, - percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); - } -#endif - storeAppendPrintf(sentry, "Flags:"); - if (SD->flags.selected) - storeAppendPrintf(sentry, " SELECTED"); - if (SD->flags.read_only) - storeAppendPrintf(sentry, " READ-ONLY"); - storeAppendPrintf(sentry, "\n"); - } -} - -void -storeUfsDirParse(cacheSwap * swap) -{ - char *token; - char *path; - int i; - int size; - int l1; - int l2; - unsigned int read_only = 0; - SwapDir *sd = NULL; - if ((path = strtok(NULL, w_space)) == NULL) - self_destruct(); - i = GetInteger(); - size = i << 10; /* Mbytes to kbytes */ - if (size <= 0) - fatal("storeUfsDirParse: invalid size value"); - i = GetInteger(); - l1 = i; - if (l1 <= 0) - fatal("storeUfsDirParse: invalid level 1 directories value"); - i = GetInteger(); - l2 = i; - if (l2 <= 0) - fatal("storeUfsDirParse: invalid level 2 directories value"); - if ((token = strtok(NULL, w_space))) - if (!strcasecmp(token, "read-only")) - read_only = 1; - for (i = 0; i < swap->n_configured; i++) { - sd = swap->swapDirs + i; - if (!strcmp(path, sd->path)) { - /* just reconfigure it */ - if (size == sd->max_size) - debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", - path, size); - else - debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", - path, size); - sd->max_size = size; - if (sd->flags.read_only != read_only) - debug(3, 1) ("Cache dir '%s' now %s\n", - path, read_only ? "Read-Only" : "Read-Write"); - sd->flags.read_only = read_only; - return; - } - } - allocate_new_swapdir(swap); - sd = swap->swapDirs + swap->n_configured; - sd->type = SWAPDIR_UFS; - sd->index = swap->n_configured; - sd->path = xstrdup(path); - sd->max_size = size; - sd->u.ufs.l1 = l1; - sd->u.ufs.l2 = l2; - sd->u.ufs.swaplog_fd = -1; - sd->flags.read_only = read_only; - sd->init = storeUfsDirInit; - sd->newfs = storeUfsDirNewfs; - sd->obj.open = storeUfsOpen; - sd->obj.close = storeUfsClose; - sd->obj.read = storeUfsRead; - sd->obj.write = storeUfsWrite; - sd->obj.unlink = storeUfsUnlink; - sd->log.open = storeUfsDirOpenSwapLog; - sd->log.close = storeUfsDirCloseSwapLog; - sd->log.write = storeUfsDirSwapLog; - sd->log.clean.open = storeUfsDirWriteCleanOpen; - swap->n_configured++; -} - -#if USE_ASYNC_IO -void -storeAufsDirParse(cacheSwap * swap) -{ - char *token; - char *path; - int i; - int size; - int l1; - int l2; - unsigned int read_only = 0; - SwapDir *sd = NULL; - if ((path = strtok(NULL, w_space)) == NULL) - self_destruct(); - i = GetInteger(); - size = i << 10; /* Mbytes to kbytes */ - if (size <= 0) - fatal("storeUfsDirParse: invalid size value"); - i = GetInteger(); - l1 = i; - if (l1 <= 0) - fatal("storeUfsDirParse: invalid level 1 directories value"); - i = GetInteger(); - l2 = i; - if (l2 <= 0) - fatal("storeUfsDirParse: invalid level 2 directories value"); - if ((token = strtok(NULL, w_space))) - if (!strcasecmp(token, "read-only")) - read_only = 1; - for (i = 0; i < swap->n_configured; i++) { - sd = swap->swapDirs + i; - if (!strcmp(path, sd->path)) { - /* just reconfigure it */ - if (size == sd->max_size) - debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", - path, size); - else - debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", - path, size); - sd->max_size = size; - if (sd->flags.read_only != read_only) - debug(3, 1) ("Cache dir '%s' now %s\n", - path, read_only ? "Read-Only" : "Read-Write"); - sd->flags.read_only = read_only; - return; - } - } - allocate_new_swapdir(swap); - sd = swap->swapDirs + swap->n_configured; - sd->type = SWAPDIR_ASYNCUFS; - sd->index = swap->n_configured; - sd->path = xstrdup(path); - sd->max_size = size; - sd->u.ufs.l1 = l1; - sd->u.ufs.l2 = l2; - sd->u.ufs.swaplog_fd = -1; - sd->flags.read_only = read_only; - sd->init = storeUfsDirInit; - sd->newfs = storeUfsDirNewfs; - sd->obj.open = storeAufsOpen; - sd->obj.close = storeAufsClose; - sd->obj.read = storeAufsRead; - sd->obj.write = storeAufsWrite; - sd->obj.unlink = storeAufsUnlink; - sd->log.open = storeUfsDirOpenSwapLog; - sd->log.close = storeUfsDirCloseSwapLog; - sd->log.write = storeUfsDirSwapLog; - sd->log.clean.open = storeUfsDirWriteCleanOpen; - swap->n_configured++; -} -#endif - -#if USE_DISKD -void -storeDiskdDirParse(cacheSwap * swap) -{ - char *token; - char *path; - int i; - int size; - int l1; - int l2; - unsigned int read_only = 0; - SwapDir *sd = NULL; - if ((path = strtok(NULL, w_space)) == NULL) - self_destruct(); - i = GetInteger(); - size = i << 10; /* Mbytes to kbytes */ - if (size <= 0) - fatal("storeUfsDirParse: invalid size value"); - i = GetInteger(); - l1 = i; - if (l1 <= 0) - fatal("storeUfsDirParse: invalid level 1 directories value"); - i = GetInteger(); - l2 = i; - if (l2 <= 0) - fatal("storeUfsDirParse: invalid level 2 directories value"); - if ((token = strtok(NULL, w_space))) - if (!strcasecmp(token, "read-only")) - read_only = 1; - for (i = 0; i < swap->n_configured; i++) { - sd = swap->swapDirs + i; - if (!strcmp(path, sd->path)) { - /* just reconfigure it */ - if (size == sd->max_size) - debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", - path, size); - else - debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", - path, size); - sd->max_size = size; - if (sd->flags.read_only != read_only) - debug(3, 1) ("Cache dir '%s' now %s\n", - path, read_only ? "Read-Only" : "Read-Write"); - sd->flags.read_only = read_only; - return; - } - } - allocate_new_swapdir(swap); - sd = swap->swapDirs + swap->n_configured; - sd->type = SWAPDIR_DISKD; - sd->index = swap->n_configured; - sd->path = xstrdup(path); - sd->max_size = size; - sd->u.ufs.l1 = l1; - sd->u.ufs.l2 = l2; - sd->u.ufs.swaplog_fd = -1; - sd->flags.read_only = read_only; - sd->init = storeDiskdInit; - sd->newfs = storeUfsDirNewfs; - sd->obj.open = storeDiskdOpen; - sd->obj.close = storeDiskdClose; - sd->obj.read = storeDiskdRead; - sd->obj.write = storeDiskdWrite; - sd->obj.unlink = storeDiskdUnlink; - sd->log.open = storeUfsDirOpenSwapLog; - sd->log.close = storeUfsDirCloseSwapLog; - sd->log.write = storeUfsDirSwapLog; - sd->log.clean.open = storeUfsDirWriteCleanOpen; - swap->n_configured++; -} -#endif - -void -storeUfsDirDump(StoreEntry * entry, const char *name, SwapDir * s) -{ - storeAppendPrintf(entry, "%s %s %s %d %d %d\n", - name, - SwapDirType[s->type], - s->path, - s->max_size >> 10, - s->u.ufs.l1, - s->u.ufs.l2); -} - -/* - * Only "free" the filesystem specific stuff here - */ -void -storeUfsDirFree(SwapDir * s) -{ - if (s->u.ufs.swaplog_fd > -1) { - file_close(s->u.ufs.swaplog_fd); - s->u.ufs.swaplog_fd = -1; - } -} - -char * -storeUfsFullPath(int fn, char *fullpath) -{ - LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured; - int filn = fn & SWAP_FILE_MASK; - SwapDir *SD = &Config.cacheSwap.swapDirs[dirn]; - int L1 = SD->u.ufs.l1; - int L2 = SD->u.ufs.l2; - if (!fullpath) - fullpath = fullfilename; - fullpath[0] = '\0'; - snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", - Config.cacheSwap.swapDirs[dirn].path, - ((filn / L2) / L2) % L1, - (filn / L2) % L2, - filn); - return fullpath; -} Index: squid/src/store_heap_replacement.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/store_heap_replacement.c,v retrieving revision 1.1.1.1.10.1 retrieving revision 1.1.1.1.10.2 diff -u -r1.1.1.1.10.1 -r1.1.1.1.10.2 --- squid/src/store_heap_replacement.c 17 Apr 2000 00:13:10 -0000 1.1.1.1.10.1 +++ squid/src/store_heap_replacement.c 3 May 2000 19:18:13 -0000 1.1.1.1.10.2 @@ -1,6 +1,6 @@ /* - * $Id: store_heap_replacement.c,v 1.1.1.1.10.1 2000/04/17 00:13:10 hno Exp $ + * $Id: store_heap_replacement.c,v 1.1.1.1.10.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 20 Storage Manager Heap-based replacement * AUTHOR: John Dilley @@ -41,6 +41,10 @@ * http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html */ +#include "squid.h" + +#if HEAP_REPLACEMENT + /* * Key generation function to implement the LFU-DA policy (Least * Frequently Used with Dynamic Aging). Similar to classical LFU @@ -54,7 +58,7 @@ * (e->lastref): for objects that have the same reference count * the most recent object wins (gets a higher key value). */ -static heap_key +heap_key HeapKeyGen_StoreEntry_LFUDA(void *entry, double age) { StoreEntry *e = entry; @@ -81,7 +85,7 @@ * (e->lastref): for objects that have the same reference count * the most recent object wins (gets a higher key value). */ -static heap_key +heap_key HeapKeyGen_StoreEntry_GDSF(void *entry, double age) { StoreEntry *e = entry; @@ -97,9 +101,11 @@ * Don't use it unless you are trying to compare performance among * heap-based replacement policies... */ -static heap_key +heap_key HeapKeyGen_StoreEntry_LRU(void *entry, double age) { StoreEntry *e = entry; return (heap_key) e->lastref; } + +#endif Index: squid/src/store_io.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_io.c,v retrieving revision 1.1.1.1 retrieving revision 1.1.1.1.12.1 diff -u -r1.1.1.1 -r1.1.1.1.12.1 --- squid/src/store_io.c 26 Jan 2000 03:25:01 -0000 1.1.1.1 +++ squid/src/store_io.c 3 May 2000 19:18:13 -0000 1.1.1.1.12.1 @@ -1,44 +1,83 @@ #include "squid.h" +/* + * submit a request to create a cache object for writing. + * The StoreEntry structure is sent as a hint to the filesystem + * to what will be stored in this object, to allow the filesystem + * to select different polices depending on object size or type. + */ +storeIOState * +storeCreate(StoreEntry *e, STIOCB *file_callback, STIOCB *close_callback, void *callback_data) +{ + size_t objsize; + sdirno dirn; + SwapDir *SD; + + /* This is just done for logging purposes */ + objsize = objectLen(e); + if (objsize != -1) + objsize += e->mem_obj->swap_hdr_sz; + + /* + * Pick the swapdir + * We assume that the header has been packed by now .. + */ + dirn = storeDirSelectSwapDir(e); + if (dirn == -1) { + debug(20, 2) ("storeCreate: no valid swapdirs for this object\n"); + return NULL; + } + debug (20, 2) ("storeCreate: Selected dir '%d' for obj size '%d'\n", dirn, objsize); + SD = &Config.cacheSwap.swapDirs[dirn]; + + /* Now that we have a fs to use, call its storeCreate function */ + return(SD->obj.create(SD, e, file_callback, close_callback, callback_data)); + + /* Done */ +} + +/* + * storeOpen() is purely for reading .. + */ storeIOState * -storeOpen(sfileno f, mode_t mode, STIOCB * callback, void *callback_data) +storeOpen(StoreEntry *e, STFNCB * file_callback, STIOCB * callback, + void *callback_data) { - SwapDir *SD = &Config.cacheSwap.swapDirs[f >> SWAP_DIR_SHIFT]; - assert(mode == O_RDONLY || mode == O_WRONLY); - return SD->obj.open(f, mode, callback, callback_data); + SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn]; + return SD->obj.open(SD, e, file_callback, callback, callback_data); } void storeClose(storeIOState * sio) { - SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_file_number >> SWAP_DIR_SHIFT]; + SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn]; if (sio->flags.closing) return; sio->flags.closing = 1; - SD->obj.close(sio); + SD->obj.close(SD, sio); } void storeRead(storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) { - SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_file_number >> SWAP_DIR_SHIFT]; - SD->obj.read(sio, buf, size, offset, callback, callback_data); + SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn]; + SD->obj.read(SD, sio, buf, size, offset, callback, callback_data); } void storeWrite(storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) { - SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_file_number >> SWAP_DIR_SHIFT]; - SD->obj.write(sio, buf, size, offset, free_func); + SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn]; + SD->obj.write(SD, sio, buf, size, offset, free_func); } void -storeUnlink(sfileno f) +storeUnlink(StoreEntry *e) { - SwapDir *SD = &Config.cacheSwap.swapDirs[f >> SWAP_DIR_SHIFT]; - SD->obj.unlink(f); + SwapDir *SD = INDEXSD(e->swap_dirn); + SD->obj.unlink(SD, e); } off_t --- squid/src/store_io_asyncufs.c Wed Feb 14 00:43:59 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,297 +0,0 @@ - -/* - * DEBUG 78 - */ - -#include "squid.h" - -#define SWAP_DIR_SHIFT 24 -#define SWAP_FILE_MASK 0x00FFFFFF -#define MAGIC1 (NUMTHREADS/Config.cacheSwap.n_configured/2) - -static AIOCB storeAufsReadDone; -static AIOCB storeAufsWriteDone; -static void storeAufsIOCallback(storeIOState * sio, int errflag); -static AIOCB storeAufsOpenDone; -static int storeAufsSomethingPending(storeIOState *); -static int storeAufsKickWriteQueue(storeIOState * sio); - -struct _queued_write { - char *buf; - size_t size; - off_t offset; - FREE *free_func; -}; - -struct _queued_read { - char *buf; - size_t size; - off_t offset; - STRCB *callback; - void *callback_data; -}; - -/* === PUBLIC =========================================================== */ - -storeIOState * -storeAufsOpen(sfileno f, mode_t mode, STIOCB * callback, void *callback_data) -{ - char *path = storeUfsFullPath(f, NULL); - storeIOState *sio; - debug(78, 3) ("storeAufsOpen: fileno %08X, mode %d\n", f, mode); - assert(mode == O_RDONLY || mode == O_WRONLY); - /* - * we should detect some 'too many files open' condition and return - * NULL here. - */ - sio = memAllocate(MEM_STORE_IO); - cbdataAdd(sio, memFree, MEM_STORE_IO); - sio->type.aufs.fd = -1; - sio->swap_file_number = f; - sio->mode = mode; - sio->callback = callback; - sio->callback_data = callback_data; - cbdataLock(callback_data); - if (mode == O_WRONLY) - mode |= (O_CREAT | O_TRUNC); - sio->type.aufs.flags.opening = 1; - aioOpen(path, mode, 0644, storeAufsOpenDone, sio); - Opening_FD++; - store_open_disk_fd++; - return sio; -} - -void -storeAufsClose(storeIOState * sio) -{ - debug(78, 3) ("storeAufsClose: fileno %08X, FD %d\n", - sio->swap_file_number, sio->type.aufs.fd); - if (storeAufsSomethingPending(sio)) { - sio->type.aufs.flags.close_request = 1; - return; - } - storeAufsIOCallback(sio, DISK_OK); -} - -void -storeAufsRead(storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) -{ - assert(sio->read.callback == NULL); - assert(sio->read.callback_data == NULL); - assert(!sio->type.aufs.flags.reading); - if (sio->type.aufs.fd < 0) { - struct _queued_read *q; - debug(78, 3) ("storeAufsRead: queueing read because FD < 0\n"); - assert(sio->type.aufs.flags.opening); - assert(sio->type.aufs.pending_reads == NULL); - q = xcalloc(1, sizeof(*q)); - q->buf = buf; - q->size = size; - q->offset = offset; - q->callback = callback; - q->callback_data = callback_data; - linklistPush(&sio->type.aufs.pending_reads, q); - return; - } - sio->read.callback = callback; - sio->read.callback_data = callback_data; - sio->type.aufs.read_buf = buf; - cbdataLock(callback_data); - debug(78, 3) ("storeAufsRead: fileno %08X, FD %d\n", - sio->swap_file_number, sio->type.aufs.fd); - sio->offset = offset; - sio->type.aufs.flags.reading = 1; - aioRead(sio->type.aufs.fd, offset, buf, size, storeAufsReadDone, sio); -} - -void -storeAufsWrite(storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) -{ - debug(78, 3) ("storeAufsWrite: fileno %08X, FD %d\n", - sio->swap_file_number, sio->type.aufs.fd); - if (sio->type.aufs.fd < 0) { - /* disk file not opened yet */ - struct _queued_write *q; - assert(sio->type.aufs.flags.opening); - q = xcalloc(1, sizeof(*q)); - q->buf = buf; - q->size = size; - q->offset = offset; - q->free_func = free_func; - linklistPush(&sio->type.aufs.pending_writes, q); - return; - } - if (sio->type.aufs.flags.writing) { - struct _queued_write *q; - debug(78, 3) ("storeAufsWrite: queuing write\n"); - q = xcalloc(1, sizeof(*q)); - q->buf = buf; - q->size = size; - q->offset = offset; - q->free_func = free_func; - linklistPush(&sio->type.aufs.pending_writes, q); - return; - } - sio->type.aufs.flags.writing = 1; - /* - * XXX it might be nice if aioWrite() gave is immediate - * feedback here about EWOULDBLOCK instead of in the - * callback function - */ - aioWrite(sio->type.aufs.fd, - offset, - buf, - size, - storeAufsWriteDone, - sio, - free_func); -} - -void -storeAufsUnlink(sfileno f) -{ - debug(78, 3) ("storeAufsUnlink: fileno %08X\n", f); - aioUnlink(storeUfsFullPath(f, NULL), NULL, NULL); -} - -/* === STATIC =========================================================== */ - -static int -storeAufsKickWriteQueue(storeIOState * sio) -{ - struct _queued_write *q = linklistShift(&sio->type.aufs.pending_writes); - if (NULL == q) - return 0; - debug(78, 3) ("storeAufsKickWriteQueue: writing queued chunk of %d bytes\n", - q->size); - storeAufsWrite(sio, q->buf, q->size, q->offset, q->free_func); - xfree(q); - return 1; -} - -static int -storeAufsKickReadQueue(storeIOState * sio) -{ - struct _queued_read *q = linklistShift(&sio->type.aufs.pending_reads); - if (NULL == q) - return 0; - debug(78, 3) ("storeAufsKickReadQueue: reading queued request of %d bytes\n", - q->size); - storeAufsRead(sio, q->buf, q->size, q->offset, q->callback, q->callback_data); - xfree(q); - return 1; -} - -static void -storeAufsOpenDone(int unused, void *my_data, int fd, int errflag) -{ - storeIOState *sio = my_data; - debug(78, 3) ("storeAufsOpenDone: FD %d, errflag %d\n", fd, errflag); - Opening_FD--; - sio->type.aufs.flags.opening = 0; - if (errflag || fd < 0) { - errno = errflag; - debug(78, 0) ("storeAufsOpenDone: %s\n", xstrerror()); - debug(78, 1) ("\t%s\n", storeUfsFullPath(sio->swap_file_number, NULL)); - storeAufsIOCallback(sio, DISK_ERROR); - return; - } - sio->type.aufs.fd = fd; - commSetCloseOnExec(fd); - fd_open(fd, FD_FILE, storeUfsFullPath(sio->swap_file_number, NULL)); - if (sio->mode == O_WRONLY) - storeAufsKickWriteQueue(sio); - else if (sio->mode == O_RDONLY) - storeAufsKickReadQueue(sio); - debug(78, 3) ("storeAufsOpenDone: exiting\n"); -} - -static void -storeAufsReadDone(int fd, void *my_data, int len, int errflag) -{ - storeIOState *sio = my_data; - STRCB *callback = sio->read.callback; - void *their_data = sio->read.callback_data; - ssize_t rlen; - debug(78, 3) ("storeAufsReadDone: fileno %08X, FD %d, len %d\n", - sio->swap_file_number, fd, len); - sio->type.aufs.flags.reading = 0; - if (errflag) { - debug(78, 3) ("storeAufsReadDone: got failure (%d)\n", errflag); - rlen = -1; - } else { - rlen = (ssize_t) len; - sio->offset += len; - } - assert(callback); - assert(their_data); - sio->read.callback = NULL; - sio->read.callback_data = NULL; - if (cbdataValid(their_data)) - callback(their_data, sio->type.aufs.read_buf, rlen); - cbdataUnlock(their_data); -} - -/* - * XXX TODO - * if errflag == EWOULDBLOCK, then we'll need to re-queue the - * chunk at the beginning of the write_pending list and try - * again later. - */ -static void -storeAufsWriteDone(int fd, void *my_data, int len, int errflag) -{ - static int loop_detect = 0; - storeIOState *sio = my_data; - debug(78, 3) ("storeAufsWriteDone: fileno %08X, FD %d, len %d\n", - sio->swap_file_number, fd, len); - assert(++loop_detect < 10); - sio->type.aufs.flags.writing = 0; - if (errflag) { - debug(78, 3) ("storeAufsWriteDone: got failure (%d)\n", errflag); - storeAufsIOCallback(sio, DISK_ERROR); - loop_detect--; - return; - } - sio->offset += len; - if (storeAufsKickWriteQueue(sio)) - (void) 0; - else if (sio->type.aufs.flags.close_request) - storeAufsIOCallback(sio, DISK_OK); - loop_detect--; -} - -static void -storeAufsIOCallback(storeIOState * sio, int errflag) -{ - STIOCB *callback = sio->callback; - void *their_data = sio->callback_data; - int fd = sio->type.aufs.fd; - debug(78, 3) ("storeAufsIOCallback: errflag=%d\n", errflag); - sio->callback = NULL; - sio->callback_data = NULL; - if (callback) - if (NULL == their_data || cbdataValid(their_data)) - callback(their_data, errflag, sio); - cbdataUnlock(their_data); - sio->type.aufs.fd = -1; - cbdataFree(sio); - if (fd < 0) - return; - aioClose(fd); - fd_close(fd); - store_open_disk_fd--; -} - - -static int -storeAufsSomethingPending(storeIOState * sio) -{ - if (sio->type.aufs.flags.reading) - return 1; - if (sio->type.aufs.flags.writing) - return 1; - if (sio->type.aufs.flags.opening) - return 1; - return 0; -} --- squid/src/store_io_ufs.c Wed Feb 14 00:43:59 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,185 +0,0 @@ - -/* - * $Id: store_io_ufs.c,v 1.1.1.1.12.1 2000/04/17 00:13:10 hno Exp $ - * - * DEBUG: section 79 Storage Manager UFS Interface - * AUTHOR: Duane Wessels - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -#define SWAP_DIR_SHIFT 24 -#define SWAP_FILE_MASK 0x00FFFFFF - -static DRCB storeUfsReadDone; -static DWCB storeUfsWriteDone; -static void storeUfsIOCallback(storeIOState * sio, int errflag); - -/* === PUBLIC =========================================================== */ - -storeIOState * -storeUfsOpen(sfileno f, mode_t mode, STIOCB * callback, void *callback_data) -{ - char *path = storeUfsFullPath(f, NULL); - storeIOState *sio; - struct stat sb; - int fd; - debug(79, 3) ("storeUfsOpen: fileno %08X, mode %d\n", f, mode); - assert(mode == O_RDONLY || mode == O_WRONLY); - if (mode == O_WRONLY) - mode |= (O_CREAT | O_TRUNC); - fd = file_open(path, mode); - if (fd < 0) { - debug(79, 3) ("storeUfsOpenDone: got failure (%d)\n", errno); - return NULL; - } - debug(79, 3) ("storeUfsOpen: opened FD %d\n", fd); - sio = memAllocate(MEM_STORE_IO); - cbdataAdd(sio, memFree, MEM_STORE_IO); - sio->swap_file_number = f; - sio->mode = mode; - sio->callback = callback; - sio->callback_data = callback_data; - sio->type.ufs.fd = fd; - sio->type.ufs.flags.writing = 0; - if (sio->mode == O_RDONLY) - if (fstat(fd, &sb) == 0) - sio->st_size = sb.st_size; - store_open_disk_fd++; - return sio; -} - -void -storeUfsClose(storeIOState * sio) -{ - debug(79, 3) ("storeUfsClose: fileno %08X, FD %d\n", - sio->swap_file_number, sio->type.ufs.fd); - if (sio->type.ufs.flags.reading || sio->type.ufs.flags.writing) { - sio->type.ufs.flags.close_request = 1; - return; - } - storeUfsIOCallback(sio, 0); -} - -void -storeUfsRead(storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) -{ - assert(sio->read.callback == NULL); - assert(sio->read.callback_data == NULL); - sio->read.callback = callback; - sio->read.callback_data = callback_data; - cbdataLock(callback_data); - debug(79, 3) ("storeUfsRead: fileno %08X, FD %d\n", - sio->swap_file_number, sio->type.ufs.fd); - sio->offset = offset; - sio->type.ufs.flags.reading = 1; - file_read(sio->type.ufs.fd, - buf, - size, - offset, - storeUfsReadDone, - sio); -} - -void -storeUfsWrite(storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) -{ - debug(79, 3) ("storeUfsWrite: fileno %08X, FD %d\n", sio->swap_file_number, sio->type.ufs.fd); - sio->type.ufs.flags.writing = 1; - file_write(sio->type.ufs.fd, - offset, - buf, - size, - storeUfsWriteDone, - sio, - free_func); -} - -void -storeUfsUnlink(sfileno f) -{ - debug(79, 3) ("storeUfsUnlink: fileno %08X\n", f); - unlinkdUnlink(storeUfsFullPath(f, NULL)); -} - -/* === STATIC =========================================================== */ - -static void -storeUfsReadDone(int fd, const char *buf, int len, int errflag, void *my_data) -{ - storeIOState *sio = my_data; - STRCB *callback = sio->read.callback; - void *their_data = sio->read.callback_data; - debug(79, 3) ("storeUfsReadDone: fileno %08X, FD %d, len %d\n", - sio->swap_file_number, fd, len); - sio->type.ufs.flags.reading = 0; - if (errflag) { - debug(79, 3) ("storeUfsReadDone: got failure (%d)\n", errflag); - storeUfsIOCallback(sio, errflag); - return; - } - sio->offset += len; - assert(callback); - assert(their_data); - sio->read.callback = NULL; - sio->read.callback_data = NULL; - if (cbdataValid(their_data)) - callback(their_data, buf, (size_t) len); - cbdataUnlock(their_data); -} - -static void -storeUfsWriteDone(int fd, int errflag, size_t len, void *my_data) -{ - storeIOState *sio = my_data; - debug(79, 3) ("storeUfsWriteDone: fileno %08X, FD %d, len %d\n", - sio->swap_file_number, fd, len); - sio->type.ufs.flags.writing = 0; - if (errflag) { - debug(79, 0) ("storeUfsWriteDone: got failure (%d)\n", errflag); - storeUfsIOCallback(sio, errflag); - return; - } - sio->offset += len; - if (sio->type.ufs.flags.close_request) - storeUfsIOCallback(sio, errflag); -} - -static void -storeUfsIOCallback(storeIOState * sio, int errflag) -{ - debug(79, 3) ("storeUfsIOCallback: errflag=%d\n", errflag); - if (sio->type.ufs.fd > -1) { - file_close(sio->type.ufs.fd); - store_open_disk_fd--; - } - sio->callback(sio->callback_data, errflag, sio); - cbdataFree(sio); -} Index: squid/src/store_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_log.c,v retrieving revision 1.1.1.2.12.1 retrieving revision 1.1.1.2.12.2 diff -u -r1.1.1.2.12.1 -r1.1.1.2.12.2 --- squid/src/store_log.c 17 Apr 2000 00:13:10 -0000 1.1.1.2.12.1 +++ squid/src/store_log.c 3 May 2000 19:18:13 -0000 1.1.1.2.12.2 @@ -1,6 +1,6 @@ /* - * $Id: store_log.c,v 1.1.1.2.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: store_log.c,v 1.1.1.2.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 20 Storage Manager Logging Functions * AUTHOR: Duane Wessels @@ -62,11 +62,17 @@ mem->log_url = xstrdup(mem->url); } reply = mem->reply; - logfilePrintf(storelog, "%9d.%03d %-7s %08X %4d %9d %9d %9d %s %d/%d %s %s\n", + /* + * XXX Ok, where should we print the dir number here? + * Because if we print it before the swap file number, it'll break + * the existing log format. + */ + logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %4d %9d %9d %9d %s %d/%d %s %s\n", (int) current_time.tv_sec, (int) current_time.tv_usec / 1000, storeLogTags[tag], - e->swap_file_number, + e->swap_dirn, + e->swap_filen, reply->sline.status, (int) reply->date, (int) reply->last_modified, --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/store_modules.sh Wed Feb 14 00:43:59 2007 @@ -0,0 +1,15 @@ +#!/bin/sh +echo "/* automatically generated by $0 $*" +echo " * do not edit" +echo " */" +echo "#include \"squid.h\"" +echo "" +for module in "$@"; do + echo "extern STSETUP storeFsSetup_${module};" +done +echo "void storeFsSetup(void)" +echo "{" +for module in "$@"; do + echo " storeFsAdd(\"$module\", storeFsSetup_${module});" +done +echo "}" Index: squid/src/store_rebuild.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_rebuild.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/store_rebuild.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/store_rebuild.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: store_rebuild.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: store_rebuild.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 20 Store Rebuild Routines * AUTHOR: Duane Wessels @@ -40,36 +40,10 @@ static void storeCleanup(void *); static int -storeCleanupDoubleCheck(const StoreEntry * e) +storeCleanupDoubleCheck(StoreEntry * e) { - /* XXX too UFS specific */ - struct stat sb; - int dirn = e->swap_file_number >> SWAP_DIR_SHIFT; - if (Config.cacheSwap.swapDirs[dirn].type == SWAPDIR_UFS) - (void) 0; - if (Config.cacheSwap.swapDirs[dirn].type == SWAPDIR_ASYNCUFS) - (void) 0; - else - return 0; - if (stat(storeUfsFullPath(e->swap_file_number, NULL), &sb) < 0) { - debug(20, 0) ("storeCleanup: MISSING SWAP FILE\n"); - debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number); - debug(20, 0) ("storeCleanup: PATH %s\n", - storeUfsFullPath(e->swap_file_number, NULL)); - storeEntryDump(e, 0); - return -1; - } - if (e->swap_file_sz != sb.st_size) { - debug(20, 0) ("storeCleanup: SIZE MISMATCH\n"); - debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number); - debug(20, 0) ("storeCleanup: PATH %s\n", - storeUfsFullPath(e->swap_file_number, NULL)); - debug(20, 0) ("storeCleanup: ENTRY SIZE: %d, FILE SIZE: %d\n", - e->swap_file_sz, (int) sb.st_size); - storeEntryDump(e, 0); - return -1; - } - return 0; + SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn]; + return (SD->dblcheck(SD, e)); } static void @@ -106,7 +80,7 @@ * Calling storeRelease() has no effect because we're * still in 'store_rebuilding' state */ - if (e->swap_file_number < 0) + if (e->swap_filen < 0) continue; if (opt_store_doublecheck) if (storeCleanupDoubleCheck(e)) @@ -116,7 +90,7 @@ * Only set the file bit if we know its a valid entry * otherwise, set it in the validation procedure */ - storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1); + storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1); if ((++validnum & 0x3FFFF) == 0) debug(20, 1) (" %7d Entries Validated so far.\n", validnum); } Index: squid/src/store_swapin.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_swapin.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/store_swapin.c 17 Apr 2000 00:13:10 -0000 1.1.1.3.12.1 +++ squid/src/store_swapin.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: store_swapin.c,v 1.1.1.3.12.1 2000/04/17 00:13:10 hno Exp $ + * $Id: store_swapin.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 20 Storage Manager Swapin Functions * AUTHOR: Duane Wessels @@ -36,6 +36,7 @@ #include "squid.h" static STIOCB storeSwapInFileClosed; +static STFNCB storeSwapInFileNotify; void storeSwapInStart(store_client * sc) @@ -46,24 +47,22 @@ /* We're still reloading and haven't validated this entry yet */ return; } - debug(20, 3) ("storeSwapInStart: called for %08X %s \n", - e->swap_file_number, storeKeyText(e->key)); + debug(20, 3) ("storeSwapInStart: called for %d %08X %s \n", + e->swap_dirn, e->swap_filen, storeKeyText(e->key)); if (e->swap_status != SWAPOUT_WRITING && e->swap_status != SWAPOUT_DONE) { debug(20, 1) ("storeSwapInStart: bad swap_status (%s)\n", swapStatusStr[e->swap_status]); return; } - if (e->swap_file_number < 0) { - debug(20, 1) ("storeSwapInStart: swap_file_number < 0\n"); + if (e->swap_filen < 0) { + debug(20, 1) ("storeSwapInStart: swap_filen < 0\n"); return; } assert(e->mem_obj != NULL); debug(20, 3) ("storeSwapInStart: Opening fileno %08X\n", - e->swap_file_number); - sc->swapin_sio = storeOpen(e->swap_file_number, - O_RDONLY, - storeSwapInFileClosed, - sc); + e->swap_filen); + sc->swapin_sio = storeOpen(e, storeSwapInFileNotify, storeSwapInFileClosed, + sc); cbdataLock(sc->swapin_sio); } @@ -82,3 +81,15 @@ callback(sc->callback_data, sc->copy_buf, errflag); } } + +static void +storeSwapInFileNotify(void *data, int errflag, storeIOState * sio) +{ + store_client *sc = data; + StoreEntry *e = sc->entry; + + debug(1, 3) ("storeSwapInFileNotify: changing %d/%d to %d/%d\n", e->swap_filen, e->swap_dirn, sio->swap_filen, sio->swap_dirn); + + e->swap_filen = sio->swap_filen; + e->swap_dirn = sio->swap_dirn; +} Index: squid/src/store_swapout.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_swapout.c,v retrieving revision 1.1.1.3.4.1.2.1 retrieving revision 1.1.1.3.4.1.2.2 diff -u -r1.1.1.3.4.1.2.1 -r1.1.1.3.4.1.2.2 --- squid/src/store_swapout.c 17 Apr 2000 00:13:11 -0000 1.1.1.3.4.1.2.1 +++ squid/src/store_swapout.c 3 May 2000 19:18:13 -0000 1.1.1.3.4.1.2.2 @@ -1,6 +1,6 @@ /* - * $Id: store_swapout.c,v 1.1.1.3.4.1.2.1 2000/04/17 00:13:11 hno Exp $ + * $Id: store_swapout.c,v 1.1.1.3.4.1.2.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 20 Storage Manager Swapout Functions * AUTHOR: Duane Wessels @@ -38,6 +38,7 @@ static off_t storeSwapOutObjectBytesOnDisk(const MemObject *); static void storeSwapOutStart(StoreEntry * e); static STIOCB storeSwapOutFileClosed; +static STIOCB storeSwapOutFileNotify; /* start swapping object to disk */ static void @@ -49,31 +50,51 @@ tlv *tlv_list; char *buf; assert(mem); - storeLockObject(e); - storeSwapFileNumberSet(e, storeDirMapAllocate()); + /* Build the swap metadata, so the filesystem will know how much + * metadata there is to store + */ + debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to dirno %d, fileno %08X\n", + storeUrl(e), e->swap_dirn, e->swap_filen); + e->swap_status = SWAPOUT_WRITING; + tlv_list = storeSwapMetaBuild(e); + buf = storeSwapMetaPack(tlv_list, &swap_hdr_sz); + storeSwapTLVFree(tlv_list); + mem->swap_hdr_sz = (size_t) swap_hdr_sz; + /* Create the swap file */ c = xcalloc(1, sizeof(*c)); c->data = e; cbdataAdd(c, cbdataXfree, 0); - mem->swapout.sio = storeOpen(e->swap_file_number, - O_WRONLY, storeSwapOutFileClosed, c); + mem->swapout.sio = storeCreate(e, storeSwapOutFileNotify, storeSwapOutFileClosed, c); if (NULL == mem->swapout.sio) { e->swap_status = SWAPOUT_NONE; - storeSwapFileNumberSet(e, -1); cbdataFree(c); - storeUnlockObject(e); + xfree(buf); return; } - e->swap_status = SWAPOUT_WRITING; + storeLockObject(e); /* Don't lock until after create, or the replacement + * code might get confused */ + /* Pick up the file number if it was assigned immediately */ + e->swap_filen = mem->swapout.sio->swap_filen; + e->swap_dirn = mem->swapout.sio->swap_dirn; + /* write out the swap metadata */ cbdataLock(mem->swapout.sio); - debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to fileno %08X\n", - storeUrl(e), e->swap_file_number); - tlv_list = storeSwapMetaBuild(e); - buf = storeSwapMetaPack(tlv_list, &swap_hdr_sz); - storeSwapTLVFree(tlv_list); - mem->swap_hdr_sz = (size_t) swap_hdr_sz; storeWrite(mem->swapout.sio, buf, mem->swap_hdr_sz, 0, xfree); } +static void +storeSwapOutFileNotify(void *data, int errflag, storeIOState * sio) +{ + generic_cbdata *c = data; + StoreEntry *e = c->data; + MemObject *mem = e->mem_obj; + assert(e->swap_status == SWAPOUT_WRITING); + assert(mem); + assert(mem->swapout.sio == sio); + assert(errflag == 0); + e->swap_filen = mem->swapout.sio->swap_filen; + e->swap_dirn = mem->swapout.sio->swap_dirn; +} + void storeSwapOut(StoreEntry * e) { @@ -82,7 +103,6 @@ off_t new_mem_lo; off_t on_disk = 0; size_t swapout_size; - char *swap_buf; ssize_t swap_buf_len; if (mem == NULL) return; @@ -108,10 +128,25 @@ lowest_offset = storeLowestMemReaderOffset(e); debug(20, 7) ("storeSwapOut: lowest_offset = %d\n", (int) lowest_offset); - if (mem->inmem_hi - lowest_offset > DISK_PAGE_SIZE) - new_mem_lo = lowest_offset; + /* + * Grab the swapout_size and check to see whether we're going to defer + * the swapout based upon size + */ + swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); + if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) { + debug (20, 5) ("storeSwapOut: Deferring starting swapping out\n"); + return; + } + /* + * Careful. lowest_offset can be greater than inmem_hi, such + * as in the case of a range request. + */ + if (mem->inmem_hi < lowest_offset) + new_mem_lo = lowest_offset; + else if (mem->inmem_hi - lowest_offset > SM_PAGE_SIZE) + new_mem_lo = lowest_offset; else - new_mem_lo = mem->inmem_lo; + new_mem_lo = mem->inmem_lo; assert(new_mem_lo >= mem->inmem_lo); if (storeSwapOutAble(e)) { /* @@ -136,7 +171,6 @@ assert(mem->inmem_lo <= on_disk); if (!storeSwapOutAble(e)) return; - swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); debug(20, 7) ("storeSwapOut: swapout_size = %d\n", (int) swapout_size); if (swapout_size == 0) { @@ -146,7 +180,7 @@ } if (e->store_status == STORE_PENDING) { /* wait for a full block to write */ - if (swapout_size < DISK_PAGE_SIZE) + if (swapout_size < SM_PAGE_SIZE) return; /* * Wait until we are below the disk FD limit, only if the @@ -155,6 +189,14 @@ if (storeTooManyDiskFilesOpen() && !fwdCheckDeferRead(-1, e)) return; } + /* + * Don't start swapping out until its either all in memory, or bigger + * than the maximum object size (so we pick a -1 maxobjsize fs) + */ + if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) { + debug (20, 5) ("storeSwapOut: Deferring starting swapping out\n"); + return; + } /* Ok, we have stuff to swap out. Is there a swapout.sio open? */ if (e->swap_status == SWAPOUT_NONE) { assert(mem->swapout.sio == NULL); @@ -168,35 +210,42 @@ if (NULL == mem->swapout.sio) return; do { - if (swapout_size > DISK_PAGE_SIZE) - swapout_size = DISK_PAGE_SIZE; - swap_buf = memAllocate(MEM_DISK_BUF); - swap_buf_len = stmemCopy(&mem->data_hdr, - mem->swapout.queue_offset, - swap_buf, - swapout_size); - if (swap_buf_len < 0) { - debug(20, 1) ("stmemCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key)); - storeUnlink(e->swap_file_number); - storeSwapFileNumberSet(e, -1); - e->swap_status = SWAPOUT_NONE; - memFree(swap_buf, MEM_DISK_BUF); - storeReleaseRequest(e); - storeSwapOutFileClose(e); - return; + /* + * Evil hack time. + * We are paging out to disk in page size chunks. however, later on when + * we update the queue position, we might not have a page (I *think*), + * so we do the actual page update here. + */ + + if (mem->swapout.memnode == NULL) { + /* We need to swap out the first page */ + mem->swapout.memnode = mem->data_hdr.head; + } else { + /* We need to swap out the next page */ + mem->swapout.memnode = mem->swapout.memnode->next; } + /* + * Get the length of this buffer. We are assuming(!) that the buffer + * length won't change on this buffer, or things are going to be very + * strange. I think that after the copy to a buffer is done, the buffer + * size should stay fixed regardless so that this code isn't confused, + * but we can look at this at a later date or whenever the code results + * in bad swapouts, whichever happens first. :-) + */ + swap_buf_len = mem->swapout.memnode->len; + debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); assert(swap_buf_len > 0); debug(20, 3) ("storeSwapOut: swapping out %d bytes from %d\n", swap_buf_len, (int) mem->swapout.queue_offset); mem->swapout.queue_offset += swap_buf_len; - storeWrite(mem->swapout.sio, swap_buf, swap_buf_len, -1, memFreeDISK); + storeWrite(mem->swapout.sio, mem->swapout.memnode->data, swap_buf_len, -1, NULL); /* the storeWrite() call might generate an error */ if (e->swap_status != SWAPOUT_WRITING) break; swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); if (e->store_status == STORE_PENDING) - if (swapout_size < DISK_PAGE_SIZE) + if (swapout_size < SM_PAGE_SIZE) break; } while (swapout_size > 0); if (NULL == mem->swapout.sio) @@ -234,30 +283,26 @@ assert(e->swap_status == SWAPOUT_WRITING); cbdataFree(c); if (errflag) { - sfileno bad = e->swap_file_number; - debug(20, 1) ("storeSwapOutFileClosed: swapfile %08X, errflag=%d\n\t%s\n", - bad, errflag, xstrerror()); - storeSwapFileNumberSet(e, -1); - /* - * yuck. re-set the filemap bit for some errors so that - * we don't try re-using it over and over - */ - if (errno == EPERM) - storeDirMapBitSet(bad); + debug(20, 1) ("storeSwapOutFileClosed: dirno %d, swapfile %08X, errflag=%d\n\t%s\n", + e->swap_dirn, e->swap_filen, errflag, xstrerror()); if (errflag == DISK_NO_SPACE_LEFT) { - storeDirDiskFull(bad); + storeDirDiskFull(e->swap_dirn); storeDirConfigure(); storeConfigure(); } - storeReleaseRequest(e); + if (e->swap_filen > 0) + storeUnlink(e); + e->swap_filen = -1; + e->swap_dirn = -1; e->swap_status = SWAPOUT_NONE; + storeReleaseRequest(e); } else { /* swapping complete */ - debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %08X\n", - storeUrl(e), e->swap_file_number); + debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %d, %08X\n", + storeUrl(e), e->swap_dirn, e->swap_filen); e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz; e->swap_status = SWAPOUT_DONE; - storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1); + storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1); if (storeCheckCachable(e)) { storeLog(STORE_LOG_SWAPOUT, e); storeDirSwapLog(e, SWAP_LOG_ADD); Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.1.1.3.4.1.2.5 retrieving revision 1.1.1.3.4.1.2.6 diff -u -r1.1.1.3.4.1.2.5 -r1.1.1.3.4.1.2.6 --- squid/src/structs.h 3 May 2000 11:22:56 -0000 1.1.1.3.4.1.2.5 +++ squid/src/structs.h 3 May 2000 19:18:13 -0000 1.1.1.3.4.1.2.6 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.1.1.3.4.1.2.5 2000/05/03 11:22:56 asd Exp $ + * $Id: structs.h,v 1.1.1.3.4.1.2.6 2000/05/03 19:18:13 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -49,12 +49,6 @@ acl_ip_data *next; /* used for parsing, not for storing */ }; -struct _acl_snmp_comm { - char *name; - void *community; - acl_snmp_comm *next; -}; - struct _acl_time_data { int weekbits; int start; @@ -172,11 +166,6 @@ void *callback_data; }; -struct _aio_result_t { - int aio_return; - int aio_errno; -}; - struct _wordlist { char *key; wordlist *next; @@ -279,6 +268,10 @@ #if USE_IDENT time_t ident; #endif +#if !USE_DNSSERVERS + time_t idns_retransmit; + time_t idns_query; +#endif } Timeout; size_t maxRequestHeaderSize; size_t maxRequestBodySize; @@ -314,32 +307,37 @@ char *access; char *store; char *swap; +#if USE_USERAGENT_LOG char *useragent; +#endif int rotateNumber; } Log; char *adminEmail; char *effectiveUser; char *effectiveGroup; struct { -#if USE_DNSSERVER +#if USE_DNSSERVERS char *dnsserver; #endif wordlist *redirect; wordlist *authenticate; +#if USE_ICMP char *pinger; +#endif char *unlinkd; } Program; -#if USE_DNSSERVER +#if USE_DNSSERVERS int dnsChildren; #endif int redirectChildren; int authenticateChildren; - int authenticateTTL; - int authenticateIpTTL; + time_t authenticateTTL; + time_t authenticateIpTTL; #ifdef USE_NTLM char *authenticate_ntlm_domain; #endif struct { + int single_host; char *host; u_short port; } Accel; @@ -398,7 +396,7 @@ } Netdb; struct { int log_udp; -#if USE_DNSSERVER +#if USE_DNSSERVERS int res_defnames; #endif int anonymizer; @@ -423,6 +421,7 @@ int offline; int redir_rewrites_host; int prefer_direct; + int nonhierarchical_direct; int strip_query_terms; int redirector_bypass; int ignore_unknown_nameservers; @@ -431,6 +430,8 @@ #if USE_CACHE_DIGESTS int digest_generation; #endif + int log_ip_on_direct; + int authenticateIpTTLStrict; } onoff; acl *aclList; struct { @@ -787,6 +788,7 @@ StoreEntry *entry; request_t *request; char *reply_hdr; + size_t reply_hdr_size; int reply_hdr_state; peer *peer; /* peer request made to */ int eof; /* reached end-of-object? */ @@ -1077,8 +1079,11 @@ int ignored_replies; int n_keepalives_sent; int n_keepalives_recv; + time_t probe_start; time_t last_query; time_t last_reply; + time_t last_connect_failure; + time_t last_connect_probe; int logged_state; /* so we can print dead/revived msgs */ } stats; struct { @@ -1112,6 +1117,7 @@ #if DELAY_POOLS unsigned int no_delay:1; #endif + unsigned int allow_miss:1; } options; int weight; struct { @@ -1130,10 +1136,10 @@ char *digest_url; #endif int tcp_up; /* 0 if a connect() fails */ - time_t last_fail_time; struct in_addr addresses[10]; int n_addresses; int rr_count; + int rr_lastcount; peer *next; int test_fd; #if USE_CARP @@ -1155,7 +1161,7 @@ }; struct _net_db_peer { - char *peername; + const char *peername; double hops; double rtt; time_t expires; @@ -1210,6 +1216,7 @@ aclCheck_t *acl_checklist; }; +#if USE_ICMP struct _pingerEchoData { struct in_addr to; unsigned char opcode; @@ -1225,6 +1232,7 @@ int psize; char payload[PINGER_PAYLOAD_SZ]; }; +#endif struct _icp_common_t { unsigned char opcode; /* opcode */ @@ -1292,6 +1300,7 @@ int nclients; struct { off_t queue_offset; /* relative to in-mem data */ + mem_node *memnode; /* which node we're currently paging out */ storeIOState *sio; } swapout; HttpReply *reply; @@ -1333,12 +1342,14 @@ size_t swap_file_sz; u_short refcount; u_short flags; - sfileno swap_file_number; -#if HEAP_REPLACEMENT - heap_node *node; -#else - dlink_node lru; + sdirno swap_dirn; + sfileno swap_filen; + union { +#ifdef HEAP_REPLACEMENT + heap_node *node; #endif + dlink_node lru; + } repl; u_short lock_count; /* Assume < 65536! */ mem_status_t mem_status:3; ping_status_t ping_status:3; @@ -1347,23 +1358,46 @@ }; struct _SwapDir { - swapdir_t type; - fileMap *map; + char *type; int cur_size; int low_size; int max_size; char *path; int index; /* This entry's index into the swapDirs array */ - sfileno suggest; + int suggest; + size_t max_objsize; + union { +#ifdef HEAP_REPLACEMENT + struct { + heap *heap; + } heap; +#endif + struct { + dlink_list list; + dlink_node *walker; + } lru; + } repl; int removals; int scanned; struct { unsigned int selected:1; unsigned int read_only:1; } flags; - STINIT *init; - STNEWFS *newfs; + STINIT *init; /* Initialise the fs */ + STNEWFS *newfs; /* Create a new fs */ + STDUMP *dump; /* Dump fs config snippet */ + STFREE *freefs; /* Free the fs data */ + STDBLCHECK *dblcheck; /* Double check the obj integrity */ + STSTATFS *statfs; /* Dump fs statistics */ + STMAINTAINFS *maintainfs; /* Replacement maintainence */ + STCHECKOBJ *checkobj; /* Check if the fs will store an object */ + /* These two are notifications */ + STREFOBJ *refobj; /* Reference this object */ + STUNREFOBJ *unrefobj; /* Unreference this object */ + STCALLBACK *callback; /* Handle pending callbacks */ + STSYNC *sync; /* Sync the directory */ struct { + STOBJCREATE *create; STOBJOPEN *open; STOBJCLOSE *close; STOBJREAD *read; @@ -1380,33 +1414,7 @@ void *state; } clean; } log; -#if !HEAP_REPLACEMENT - dlink_list lru_list; - dlink_node *lru_walker; -#endif - union { - struct { - int l1; - int l2; - int swaplog_fd; - } ufs; -#if USE_DISKD - struct { - int l1; - int l2; - int swaplog_fd; - int smsgid; - int rmsgid; - int wfd; - int away; - struct { - char *buf; - link_list *stack; - int id; - } shm; - } diskd; -#endif - } u; + void *fsdata; }; struct _request_flags { @@ -1437,10 +1445,13 @@ }; struct _storeIOState { - sfileno swap_file_number; + sdirno swap_dirn; + sfileno swap_filen; + StoreEntry *e; /* Need this so the FS layers can play god */ mode_t mode; size_t st_size; /* do stat(2) after read open */ - off_t offset; /* current offset pointer */ + off_t offset; /* current on-disk offset pointer */ + STFNCB *file_callback; /* called on delayed sfileno assignments */ STIOCB *callback; void *callback_data; struct { @@ -1450,38 +1461,7 @@ struct { unsigned int closing:1; /* debugging aid */ } flags; - union { - struct { - int fd; - struct { - unsigned int close_request:1; - unsigned int reading:1; - unsigned int writing:1; - } flags; - } ufs; - struct { - int fd; - struct { - unsigned int close_request:1; - unsigned int reading:1; - unsigned int writing:1; - unsigned int opening:1; - } flags; - const char *read_buf; - link_list *pending_writes; - link_list *pending_reads; - } aufs; -#if USE_DISKD - struct { - int id; - struct { - unsigned int reading:1; - unsigned int writing:1; - } flags; - char *read_buf; - } diskd; -#endif - } type; + void *fsstate; }; struct _request_t { @@ -1724,9 +1704,13 @@ tlv *next; }; +/* + * Do we need to have the dirn in here? I don't think so, since we already + * know the dirn .. + */ struct _storeSwapLogData { char op; - sfileno swap_file_number; + sfileno swap_filen; time_t timestamp; time_t lastref; time_t expires; @@ -1896,6 +1880,35 @@ int zero_object_sz; }; +/* + * This defines an fs type + */ + +struct _storefs_entry { + char *typestr; + STFSPARSE *parsefunc; + STFSRECONFIGURE *reconfigurefunc; + STFSSHUTDOWN *donefunc; +}; + +/* + * Async disk IO - this defines a async disk io queue + */ + +struct _diskd_queue { + int smsgid; /* send sysvmsg id */ + int rmsgid; /* recv sysvmsg id */ + int wfd; /* queue file descriptor ? */ + int away; /* number of requests away */ + int sent_count; /* number of messages sent */ + int recv_count; /* number of messages received */ + struct { + char *buf; /* shm buffer */ + link_list *stack; + int id; /* sysvshm id */ + } shm; +}; + struct _Logfile { int fd; char path[MAXPATHLEN]; Index: squid/src/tools.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/tools.c,v retrieving revision 1.1.1.3.10.1 retrieving revision 1.1.1.3.10.2 diff -u -r1.1.1.3.10.1 -r1.1.1.3.10.2 --- squid/src/tools.c 17 Apr 2000 00:13:11 -0000 1.1.1.3.10.1 +++ squid/src/tools.c 3 May 2000 19:18:13 -0000 1.1.1.3.10.2 @@ -1,6 +1,6 @@ /* - * $Id: tools.c,v 1.1.1.3.10.1 2000/04/17 00:13:11 hno Exp $ + * $Id: tools.c,v 1.1.1.3.10.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 21 Misc Functions * AUTHOR: Harvest Derived @@ -848,7 +848,7 @@ int stringHasWhitespace(const char *s) { - return (strcspn(s, w_space) != strlen(s)); + return strpbrk(s, w_space) != NULL; } void Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.1.1.3.12.3 retrieving revision 1.1.1.3.12.4 diff -u -r1.1.1.3.12.3 -r1.1.1.3.12.4 --- squid/src/typedefs.h 3 May 2000 11:22:56 -0000 1.1.1.3.12.3 +++ squid/src/typedefs.h 3 May 2000 19:18:13 -0000 1.1.1.3.12.4 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.1.1.3.12.3 2000/05/03 11:22:56 asd Exp $ + * $Id: typedefs.h,v 1.1.1.3.12.4 2000/05/03 19:18:13 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -39,6 +39,7 @@ typedef unsigned int ping_status_t; typedef unsigned int swap_status_t; typedef int sfileno; +typedef int sdirno; typedef struct { size_t bytes; @@ -69,7 +70,6 @@ typedef struct _acl_list acl_list; typedef struct _acl_access acl_access; typedef struct _aclCheck_t aclCheck_t; -typedef struct _aio_result_t aio_result_t; typedef struct _wordlist wordlist; typedef struct _intlist intlist; typedef struct _intrange intrange; @@ -169,6 +169,8 @@ typedef struct _generic_cbdata generic_cbdata; typedef struct _storeIOState storeIOState; typedef struct _link_list link_list; +typedef struct _storefs_entry storefs_entry_t; +typedef struct _diskd_queue diskd_queue; typedef struct _Logfile Logfile; #if SQUID_SNMP @@ -182,8 +184,6 @@ typedef struct _delaySpec delaySpec; #endif -/* define AIOCB even without USE_ASYNC_IO */ -typedef void AIOCB(int fd, void *, int aio_return, int aio_errno); typedef void CWCB(int fd, char *, size_t size, int flag, void *data); typedef void CNCB(int fd, int status, void *); @@ -192,8 +192,16 @@ typedef void FOCB(void *, int fd, int errcode); typedef void EVH(void *); typedef void PF(int, void *); -typedef void DRCB(int fd, const char *buf, int size, int errflag, void *data); -typedef void DWCB(int, int, size_t, void *); + +/* disk.c / diskd.c callback typedefs */ +typedef void DRCB(int, const char *buf, int size, int errflag, void *data); + /* Disk read CB */ +typedef void DWCB(int, int, size_t, void *); /* disk write CB */ +typedef void DOCB(int, int errflag, void *data); /* disk open CB */ +typedef void DCCB(int, int errflag, void *data); /* disk close CB */ +typedef void DUCB(int errflag, void *data); /* disk unlink CB */ +typedef void DTCB(int errflag, void *data); /* disk trunc CB */ + typedef void FQDNH(const char *, void *); typedef void IDCB(const char *ident, void *data); typedef void IPH(const ipcache_addrs *, void *); @@ -205,6 +213,7 @@ typedef void CBCB(char *buf, size_t size, void *data); typedef void STIOCB(void *their_data, int errflag, storeIOState *); +typedef void STFNCB(void *their_data, int errflag, storeIOState *); typedef void STRCB(void *their_data, const char *buf, ssize_t len); typedef void SIH(storeIOState *, void *); /* swap in */ @@ -221,17 +230,39 @@ typedef void STINIT(SwapDir *); typedef void STNEWFS(SwapDir *); -typedef storeIOState *STOBJOPEN(sfileno, mode_t, STIOCB *, void *); -typedef void STOBJCLOSE(storeIOState *); -typedef void STOBJREAD(storeIOState *, char *, size_t, off_t, STRCB *, void *); -typedef void STOBJWRITE(storeIOState *, char *, size_t, off_t, FREE *); -typedef void STOBJUNLINK(sfileno); +typedef void STDUMP(StoreEntry *, const char *, SwapDir *); +typedef void STFREE(SwapDir *); +typedef int STDBLCHECK(SwapDir *, StoreEntry *); +typedef void STSTATFS(SwapDir *, StoreEntry *); +typedef void STMAINTAINFS(SwapDir *); +typedef int STCHECKOBJ(SwapDir *, const StoreEntry *); +typedef void STREFOBJ(SwapDir *, StoreEntry *); +typedef void STUNREFOBJ(SwapDir *, StoreEntry *); +typedef void STSETUP(storefs_entry_t *); +typedef void STDONE(void); +typedef void STCALLBACK(SwapDir *); +typedef void STSYNC(SwapDir *); + +typedef storeIOState *STOBJCREATE(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *); +typedef storeIOState *STOBJOPEN(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *); +typedef void STOBJCLOSE(SwapDir *, storeIOState *); +typedef void STOBJREAD(SwapDir *, storeIOState *, char *, size_t, off_t, STRCB *, void *); +typedef void STOBJWRITE(SwapDir *, storeIOState *, char *, size_t, off_t, FREE *); +typedef void STOBJUNLINK(SwapDir *, StoreEntry *); + typedef void STLOGOPEN(SwapDir *); typedef void STLOGCLOSE(SwapDir *); typedef void STLOGWRITE(const SwapDir *, const StoreEntry *, int); typedef int STLOGCLEANOPEN(SwapDir *); typedef void STLOGCLEANWRITE(const StoreEntry *, SwapDir *); +/* Store dir configuration routines */ +/* SwapDir *sd, char *path ( + char *opt later when the strtok mess is gone) */ +typedef void STFSPARSE(SwapDir *, int, char *); +typedef void STFSRECONFIGURE(SwapDir *, int, char *); +typedef void STFSSTARTUP(void); +typedef void STFSSHUTDOWN(void); + typedef double hbase_f(double); typedef void StatHistBinDumper(StoreEntry *, int idx, double val, double size, int count); Index: squid/src/unlinkd.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/unlinkd.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/unlinkd.c 17 Apr 2000 00:13:11 -0000 1.1.1.3.12.1 +++ squid/src/unlinkd.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,5 +1,5 @@ /* - * $Id: unlinkd.c,v 1.1.1.3.12.1 2000/04/17 00:13:11 hno Exp $ + * $Id: unlinkd.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 12 Unlink Daemon * AUTHOR: Duane Wessels @@ -48,6 +48,8 @@ int x; setbuf(stdin, NULL); setbuf(stdout, NULL); + close(2); + open("/dev/null", O_RDWR); while (fgets(buf, UNLINK_BUF_LEN, stdin)) { if ((t = strchr(buf, '\n'))) *t = '\0'; @@ -68,17 +70,14 @@ /* This code gets linked to Squid */ -#if USE_UNLINKD static int unlinkd_wfd = -1; static int unlinkd_rfd = -1; -#endif #define UNLINKD_QUEUE_LIMIT 20 void unlinkdUnlink(const char *path) { -#if USE_UNLINKD char buf[MAXPATHLEN]; int l; int x; @@ -141,13 +140,11 @@ } Counter.unlink.requests++; queuelen++; -#endif } void unlinkdClose(void) { -#if USE_UNLINKD assert(unlinkd_wfd > -1); debug(12, 1) ("Closing unlinkd pipe on FD %d\n", unlinkd_wfd); file_close(unlinkd_wfd); @@ -155,13 +152,11 @@ file_close(unlinkd_rfd); unlinkd_wfd = -1; unlinkd_rfd = -1; -#endif } void unlinkdInit(void) { -#if USE_UNLINKD int x; char *args[2]; struct timeval slp; @@ -198,9 +193,6 @@ if (FD_PIPE == fd_table[unlinkd_wfd].type) commUnsetNonBlocking(unlinkd_wfd); debug(12, 1) ("Unlinkd pipe opened on FD %d\n", unlinkd_wfd); -#else - debug(12, 1) ("Unlinkd is disabled\n"); -#endif } #endif /* ndef UNLINK_DAEMON */ Index: squid/src/url.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/url.c,v retrieving revision 1.1.1.3.12.1 retrieving revision 1.1.1.3.12.2 diff -u -r1.1.1.3.12.1 -r1.1.1.3.12.2 --- squid/src/url.c 17 Apr 2000 00:13:11 -0000 1.1.1.3.12.1 +++ squid/src/url.c 3 May 2000 19:18:13 -0000 1.1.1.3.12.2 @@ -1,6 +1,6 @@ /* - * $Id: url.c,v 1.1.1.3.12.1 2000/04/17 00:13:11 hno Exp $ + * $Id: url.c,v 1.1.1.3.12.2 2000/05/03 19:18:13 hno Exp $ * * DEBUG: section 23 URL Parsing * AUTHOR: Duane Wessels @@ -127,7 +127,7 @@ * way we expect it to. */ assert(0 == matchDomainName("foo.com", "foo.com")); - assert(0 == matchDomainName(".foo.com", "foo.com")); + assert(0 < matchDomainName(".foo.com", "foo.com")); assert(0 == matchDomainName("foo.com", ".foo.com")); assert(0 == matchDomainName(".foo.com", ".foo.com")); assert(0 == matchDomainName("x.foo.com", ".foo.com")); @@ -140,6 +140,8 @@ assert(0 < matchDomainName("zzz.com", "foo.com")); assert(0 > matchDomainName("aaa.com", "foo.com")); assert(0 == matchDomainName("FOO.com", "foo.COM")); + assert(0 < matchDomainName("bfoo.com", "afoo.com")); + assert(0 > matchDomainName("afoo.com", "bfoo.com")); /* more cases? */ } @@ -241,10 +243,10 @@ port = urlDefaultPort(protocol); /* Is there any login informaiton? */ if ((t = strrchr(host, '@'))) { - strcpy(login, host); + strcpy((char *)login, (char *)host); t = strrchr(login, '@'); *t = 0; - strcpy(host, t + 1); + strcpy((char *)host, t + 1); } if ((t = strrchr(host, ':'))) { *t++ = '\0'; @@ -397,15 +399,12 @@ * HOST DOMAIN MATCH? * ------------- ------------- ------ * foo.com foo.com YES - * .foo.com foo.com YES + * .foo.com foo.com NO * x.foo.com foo.com NO * foo.com .foo.com YES * .foo.com .foo.com YES * x.foo.com .foo.com YES * - * We strip leading dots on hosts (but not domains!) so that - * ".foo.com" is is always the same as "foo.com". - * * Return values: * 0 means the host matches the domain * 1 means the host is greater than the domain @@ -417,8 +416,6 @@ { int dl; int hl; - while ('.' == *h) - h++; hl = strlen(h); dl = strlen(d); /* --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/Makefile.in Wed Feb 14 00:43:59 2007 @@ -0,0 +1,35 @@ +# Makefile for storage modules in the Squid Object Cache server +# +# $Id$ +# + +SUBDIRS = @STORE_MODULES@ + +all: + @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \ + sh -c "cd $$dir && $(MAKE) all" || exit 1; \ + done; \ + if [ ! -f stamp ]; then \ + touch stamp; \ + fi + +clean: + -rm -f *.a stamp + @for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) $@" || exit 1;\ + fi; \ + done + +distclean: + -rm -f *.a Makefile + @for dir in *; do \ + if [ -f $$dir/Makefile ]; then \ + sh -c "cd $$dir && $(MAKE) distclean"; \ + fi; \ + done + +.DEFAULT: + @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \ + sh -c "cd $$dir && $(MAKE) $@" || exit 1; \ + done --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/aufs/Makefile.in Wed Feb 14 00:43:59 2007 @@ -0,0 +1,58 @@ +# +# Makefile for the AUFS storage driver for the Squid Object Cache server +# +# $Id$ +# + +FS = aufs + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(FS).a + +OBJS = \ + aiops.o \ + async_io.o \ + store_dir_aufs.o \ + store_io_aufs.o + + +all: $(OUT) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +install: + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/aufs/aiops.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,907 @@ +/* + * $Id$ + * + * DEBUG: section 43 AIOPS + * AUTHOR: Stewart Forster + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "store_asyncufs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#if HAVE_SCHED_H +#include +#endif + +#define RIDICULOUS_LENGTH 4096 + +#if defined(_SQUID_LINUX_) +/* Linux requires proper use of mutexes or it will segfault deep in the + * thread libraries. Observed on Alpha SMP Linux 2.2.10-ac12. + */ +#define AIO_PROPER_MUTEX 1 +#endif + +enum _aio_thread_status { + _THREAD_STARTING = 0, + _THREAD_WAITING, + _THREAD_BUSY, + _THREAD_FAILED, + _THREAD_DONE +}; + +enum _aio_request_type { + _AIO_OP_NONE = 0, + _AIO_OP_OPEN, + _AIO_OP_READ, + _AIO_OP_WRITE, + _AIO_OP_CLOSE, + _AIO_OP_UNLINK, + _AIO_OP_OPENDIR, + _AIO_OP_STAT +}; + +typedef struct aio_request_t { + enum _aio_request_type request_type; + int cancelled; + char *path; + int oflag; + mode_t mode; + int fd; + char *bufferp; + char *tmpbufp; + int buflen; + off_t offset; + int whence; + int ret; + int err; + struct stat *tmpstatp; + struct stat *statp; + aio_result_t *resultp; + struct aio_request_t *next; +} aio_request_t; + + +typedef struct aio_thread_t { + pthread_t thread; + enum _aio_thread_status status; + pthread_mutex_t mutex; /* Mutex for testing condition variable */ + pthread_cond_t cond; /* Condition variable */ + struct aio_request_t *volatile req; /* set by main, cleared by thread */ + struct aio_request_t *processed_req; /* reminder to main */ + struct aio_thread_t *next; +} aio_thread_t; + + +int aio_cancel(aio_result_t *); +int aio_open(const char *, int, mode_t, aio_result_t *); +int aio_read(int, char *, int, off_t, int, aio_result_t *); +int aio_write(int, char *, int, off_t, int, aio_result_t *); +int aio_close(int, aio_result_t *); +int aio_unlink(const char *, aio_result_t *); +int aio_opendir(const char *, aio_result_t *); +aio_result_t *aio_poll_done(); +int aio_sync(void); + +static void aio_init(void); +static void aio_queue_request(aio_request_t *); +static void aio_process_request_queue(void); +static void aio_cleanup_request(aio_request_t *); +static void *aio_thread_loop(void *); +static void aio_do_open(aio_request_t *); +static void aio_do_read(aio_request_t *); +static void aio_do_write(aio_request_t *); +static void aio_do_close(aio_request_t *); +static void aio_do_stat(aio_request_t *); +static void aio_do_unlink(aio_request_t *); +#if AIO_OPENDIR +static void *aio_do_opendir(aio_request_t *); +#endif +static void aio_debug(aio_request_t *); +static void aio_poll_threads(void); + +static aio_thread_t *threads; +static int aio_initialised = 0; + +static int request_queue_len = 0; +static MemPool *aio_request_pool = NULL; +static aio_request_t *request_queue_head = NULL; +static aio_request_t *request_queue_tail = NULL; +static aio_request_t *request_done_head = NULL; +static aio_request_t *request_done_tail = NULL; +static aio_thread_t *wait_threads = NULL; +static aio_thread_t *busy_threads_head = NULL; +static aio_thread_t *busy_threads_tail = NULL; +static pthread_attr_t globattr; +static struct sched_param globsched; +static pthread_t main_thread; + +static void +aio_init(void) +{ + int i; + aio_thread_t *threadp; + + if (aio_initialised) + return; + + pthread_attr_init(&globattr); +#if HAVE_PTHREAD_ATTR_SETSCOPE + pthread_attr_setscope(&globattr, PTHREAD_SCOPE_SYSTEM); +#endif + globsched.sched_priority = 1; + main_thread = pthread_self(); +#if HAVE_PTHREAD_SETSCHEDPARAM + pthread_setschedparam(main_thread, SCHED_OTHER, &globsched); +#endif + globsched.sched_priority = 2; +#if HAVE_PTHREAD_ATTR_SETSCHEDPARAM + pthread_attr_setschedparam(&globattr, &globsched); +#endif + + /* Create threads and get them to sit in their wait loop */ + threads = xcalloc(NUMTHREADS, sizeof(aio_thread_t)); + + for (i = 0; i < NUMTHREADS; i++) { + threadp = &threads[i]; + threadp->status = _THREAD_STARTING; + if (pthread_mutex_init(&(threadp->mutex), NULL)) { + threadp->status = _THREAD_FAILED; + continue; + } + if (pthread_cond_init(&(threadp->cond), NULL)) { + threadp->status = _THREAD_FAILED; + continue; + } + threadp->req = NULL; + threadp->processed_req = NULL; + if (pthread_create(&threadp->thread, &globattr, aio_thread_loop, threadp)) { + fprintf(stderr, "Thread creation failed\n"); + threadp->status = _THREAD_FAILED; + continue; + } + threadp->next = wait_threads; + wait_threads = threadp; +#if AIO_PROPER_MUTEX + pthread_mutex_lock(&threadp->mutex); +#endif + } + + /* Create request pool */ + aio_request_pool = memPoolCreate("aio_request", sizeof(aio_request_t)); + + aio_initialised = 1; +} + + +static void * +aio_thread_loop(void *ptr) +{ + aio_thread_t *threadp = ptr; + aio_request_t *request; + sigset_t new; +#if !AIO_PROPER_MUTEX + struct timespec wait_time; +#endif + + /* + * Make sure to ignore signals which may possibly get sent to + * the parent squid thread. Causes havoc with mutex's and + * condition waits otherwise + */ + + sigemptyset(&new); + sigaddset(&new, SIGPIPE); + sigaddset(&new, SIGCHLD); +#ifdef _SQUID_LINUX_THREADS_ + sigaddset(&new, SIGQUIT); + sigaddset(&new, SIGTRAP); +#else + sigaddset(&new, SIGUSR1); + sigaddset(&new, SIGUSR2); +#endif + sigaddset(&new, SIGHUP); + sigaddset(&new, SIGTERM); + sigaddset(&new, SIGINT); + sigaddset(&new, SIGALRM); + pthread_sigmask(SIG_BLOCK, &new, NULL); + + pthread_mutex_lock(&threadp->mutex); + while (1) { +#if AIO_PROPER_MUTEX + while (threadp->req == NULL) { + threadp->status = _THREAD_WAITING; + pthread_cond_wait(&threadp->cond, &threadp->mutex); + } +#else + /* The timeout is used to unlock the race condition where + * ->req is set between the check and pthread_cond_wait. + * The thread steps it's own clock on each timeout, to avoid a CPU + * spin situation if the main thread is suspended (paging), and + * squid_curtime is not being updated timely. + */ + wait_time.tv_sec = squid_curtime + 1; /* little quicker first time */ + wait_time.tv_nsec = 0; + while (threadp->req == NULL) { + threadp->status = _THREAD_WAITING; + pthread_cond_timedwait(&threadp->cond, &threadp->mutex, &wait_time); + wait_time.tv_sec += 3; /* then wait 3 seconds between each check */ + } +#endif + request = threadp->req; + errno = 0; + if (!request->cancelled) { + switch (request->request_type) { + case _AIO_OP_OPEN: + aio_do_open(request); + break; + case _AIO_OP_READ: + aio_do_read(request); + break; + case _AIO_OP_WRITE: + aio_do_write(request); + break; + case _AIO_OP_CLOSE: + aio_do_close(request); + break; + case _AIO_OP_UNLINK: + aio_do_unlink(request); + break; +#if AIO_OPENDIR /* Opendir not implemented yet */ + case _AIO_OP_OPENDIR: + aio_do_opendir(request); + break; +#endif + case _AIO_OP_STAT: + aio_do_stat(request); + break; + default: + request->ret = -1; + request->err = EINVAL; + break; + } + } else { /* cancelled */ + request->ret = -1; + request->err = EINTR; + } + threadp->req = NULL; /* tells main thread that we are done */ + } /* while */ + return NULL; +} /* aio_thread_loop */ + +static void +aio_do_request(aio_request_t * requestp) +{ + if (wait_threads == NULL && busy_threads_head == NULL) { + fprintf(stderr, "PANIC: No threads to service requests with!\n"); + exit(-1); + } + aio_queue_request(requestp); +} /* aio_do_request */ + + +static void +aio_queue_request(aio_request_t * requestp) +{ + aio_request_t *rp; + static int last_warn = 0; + static int high_start = 0; + static int queue_high, queue_low; + int i; + + /* Mark it as not executed (failing result, no error) */ + requestp->ret = -1; + requestp->err = 0; + /* Queue it on the request queue */ + if (request_queue_head == NULL) { + request_queue_head = requestp; + request_queue_tail = requestp; + } else { + request_queue_tail->next = requestp; + request_queue_tail = requestp; + } + requestp->next = NULL; + request_queue_len += 1; + /* Poll done threads if needed */ + if (wait_threads == NULL) + aio_poll_threads(); + /* Kick it rolling */ + aio_process_request_queue(); + /* Warn if out of threads */ + if (request_queue_len > (NUMTHREADS >> 1)) { + if (high_start == 0) { + high_start = squid_curtime; + queue_high = request_queue_len; + queue_low = request_queue_len; + } + if (request_queue_len > queue_high) + queue_high = request_queue_len; + if (request_queue_len < queue_low) + queue_low = request_queue_len; + if (squid_curtime >= (last_warn + 15) && + squid_curtime >= (high_start + 3)) { + debug(43, 1) ("aio_queue_request: WARNING - Running out of I/O threads\n"); + debug(43, 2) ("aio_queue_request: Queue Length: current=%d, high=%d, low=%d, duration=%d\n", + request_queue_len, queue_high, queue_low, squid_curtime - high_start); + debug(43, 1) ("aio_queue_request: Perhaps you should increase NUMTHREADS\n"); + debug(43, 1) ("aio_queue_request: Or install more disks to share the load\n"); + debug(43, 3) ("aio_queue_request: First %d items on request queue\n", NUMTHREADS); + rp = request_queue_head; + for (i = 1; i <= NUMTHREADS; i++) { + switch (rp->request_type) { + case _AIO_OP_OPEN: + debug(43, 3) ("aio_queue_request: %d : open -> %s\n", i, rp->path); + break; + case _AIO_OP_READ: + debug(43, 3) ("aio_queue_request: %d : read -> FD = %d\n", i, rp->fd); + break; + case _AIO_OP_WRITE: + debug(43, 3) ("aio_queue_request: %d : write -> FD = %d\n", i, rp->fd); + break; + case _AIO_OP_CLOSE: + debug(43, 3) ("aio_queue_request: %d : close -> FD = %d\n", i, rp->fd); + break; + case _AIO_OP_UNLINK: + debug(43, 3) ("aio_queue_request: %d : unlink -> %s\n", i, rp->path); + break; + case _AIO_OP_STAT: + debug(43, 3) ("aio_queue_request: %d : stat -> %s\n", i, rp->path); + break; + default: + debug(43, 1) ("aio_queue_request: %d : Unimplemented request type: %d\n", i, rp->request_type); + break; + } + if ((rp = rp->next) == NULL) + break; + } + last_warn = squid_curtime; + } + } else { + high_start = 0; + } + if (request_queue_len > RIDICULOUS_LENGTH) { + debug(43, 0) ("aio_queue_request: Async request queue growing uncontrollably!\n"); + debug(43, 0) ("aio_queue_request: Syncing pending I/O operations.. (blocking)\n"); + aio_sync(); + debug(43, 0) ("aio_queue_request: Synced\n"); + } +} /* aio_queue_request */ + + +static void +aio_process_request_queue(void) +{ + aio_thread_t *threadp; + aio_request_t *requestp; + + for (;;) { + if (wait_threads == NULL || request_queue_head == NULL) + return; + + requestp = request_queue_head; + if ((request_queue_head = requestp->next) == NULL) + request_queue_tail = NULL; + requestp->next = NULL; + request_queue_len--; + + if (requestp->cancelled) { + aio_cleanup_request(requestp); + continue; + } + threadp = wait_threads; + wait_threads = threadp->next; + threadp->next = NULL; + + if (busy_threads_head != NULL) + busy_threads_tail->next = threadp; + else + busy_threads_head = threadp; + busy_threads_tail = threadp; + + threadp->status = _THREAD_BUSY; + threadp->req = threadp->processed_req = requestp; + pthread_cond_signal(&(threadp->cond)); +#if AIO_PROPER_MUTEX + pthread_mutex_unlock(&threadp->mutex); +#endif + } +} /* aio_process_request_queue */ + + +static void +aio_cleanup_request(aio_request_t * requestp) +{ + aio_result_t *resultp = requestp->resultp; + int cancelled = requestp->cancelled; + + /* Free allocated structures and copy data back to user space if the */ + /* request hasn't been cancelled */ + switch (requestp->request_type) { + case _AIO_OP_STAT: + if (!cancelled && requestp->ret == 0) + xmemcpy(requestp->statp, requestp->tmpstatp, sizeof(struct stat)); + xfree(requestp->tmpstatp); + case _AIO_OP_OPEN: + if (cancelled && requestp->ret >= 0) + /* The open() was cancelled but completed */ + close(requestp->ret); + xfree(requestp->path); + break; + case _AIO_OP_CLOSE: + if (cancelled && requestp->ret < 0) + /* The close() was cancelled and never got executed */ + close(requestp->fd); + break; + case _AIO_OP_UNLINK: + case _AIO_OP_OPENDIR: + xfree(requestp->path); + break; + case _AIO_OP_READ: + if (!cancelled && requestp->ret > 0) + xmemcpy(requestp->bufferp, requestp->tmpbufp, requestp->ret); + case _AIO_OP_WRITE: + xfree(requestp->tmpbufp); + break; + default: + break; + } + if (resultp != NULL && !cancelled) { + resultp->aio_return = requestp->ret; + resultp->aio_errno = requestp->err; + } + memPoolFree(aio_request_pool, requestp); +} /* aio_cleanup_request */ + + +int +aio_cancel(aio_result_t * resultp) +{ + aio_thread_t *threadp; + aio_request_t *requestp; + + for (threadp = busy_threads_head; threadp != NULL; threadp = threadp->next) + if (threadp->processed_req->resultp == resultp) { + threadp->processed_req->cancelled = 1; + threadp->processed_req->resultp = NULL; + return 0; + } + for (requestp = request_queue_head; requestp != NULL; requestp = requestp->next) + if (requestp->resultp == resultp) { + requestp->cancelled = 1; + requestp->resultp = NULL; + return 0; + } + for (requestp = request_done_head; requestp != NULL; requestp = requestp->next) + if (requestp->resultp == resultp) { + requestp->cancelled = 1; + requestp->resultp = NULL; + return 0; + } + return 1; +} /* aio_cancel */ + + +int +aio_open(const char *path, int oflag, mode_t mode, aio_result_t * resultp) +{ + aio_request_t *requestp; + int len; + + if (!aio_initialised) + aio_init(); + if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { + errno = ENOMEM; + return -1; + } + len = strlen(path) + 1; + if ((requestp->path = (char *) xmalloc(len)) == NULL) { + memPoolFree(aio_request_pool, requestp); + errno = ENOMEM; + return -1; + } + strncpy(requestp->path, path, len); + requestp->oflag = oflag; + requestp->mode = mode; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_OPEN; + requestp->cancelled = 0; + + aio_do_request(requestp); + return 0; +} + + +static void +aio_do_open(aio_request_t * requestp) +{ + requestp->ret = open(requestp->path, requestp->oflag, requestp->mode); + requestp->err = errno; +} + + +int +aio_read(int fd, char *bufp, int bufs, off_t offset, int whence, aio_result_t * resultp) +{ + aio_request_t *requestp; + + if (!aio_initialised) + aio_init(); + if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { + errno = ENOMEM; + return -1; + } + requestp->fd = fd; + requestp->bufferp = bufp; + if ((requestp->tmpbufp = (char *) xmalloc(bufs)) == NULL) { + memPoolFree(aio_request_pool, requestp); + errno = ENOMEM; + return -1; + } + requestp->buflen = bufs; + requestp->offset = offset; + requestp->whence = whence; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_READ; + requestp->cancelled = 0; + + aio_do_request(requestp); + return 0; +} + + +static void +aio_do_read(aio_request_t * requestp) +{ + lseek(requestp->fd, requestp->offset, requestp->whence); + requestp->ret = read(requestp->fd, requestp->tmpbufp, requestp->buflen); + requestp->err = errno; +} + + +int +aio_write(int fd, char *bufp, int bufs, off_t offset, int whence, aio_result_t * resultp) +{ + aio_request_t *requestp; + + if (!aio_initialised) + aio_init(); + if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { + errno = ENOMEM; + return -1; + } + requestp->fd = fd; + if ((requestp->tmpbufp = (char *) xmalloc(bufs)) == NULL) { + memPoolFree(aio_request_pool, requestp); + errno = ENOMEM; + return -1; + } + xmemcpy(requestp->tmpbufp, bufp, bufs); + requestp->buflen = bufs; + requestp->offset = offset; + requestp->whence = whence; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_WRITE; + requestp->cancelled = 0; + + aio_do_request(requestp); + return 0; +} + + +static void +aio_do_write(aio_request_t * requestp) +{ + requestp->ret = write(requestp->fd, requestp->tmpbufp, requestp->buflen); + requestp->err = errno; +} + + +int +aio_close(int fd, aio_result_t * resultp) +{ + aio_request_t *requestp; + + if (!aio_initialised) + aio_init(); + if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { + errno = ENOMEM; + return -1; + } + requestp->fd = fd; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_CLOSE; + requestp->cancelled = 0; + + aio_do_request(requestp); + return 0; +} + + +static void +aio_do_close(aio_request_t * requestp) +{ + requestp->ret = close(requestp->fd); + requestp->err = errno; +} + + +int +aio_stat(const char *path, struct stat *sb, aio_result_t * resultp) +{ + aio_request_t *requestp; + int len; + + if (!aio_initialised) + aio_init(); + if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { + errno = ENOMEM; + return -1; + } + len = strlen(path) + 1; + if ((requestp->path = (char *) xmalloc(len)) == NULL) { + memPoolFree(aio_request_pool, requestp); + errno = ENOMEM; + return -1; + } + strncpy(requestp->path, path, len); + requestp->statp = sb; + if ((requestp->tmpstatp = (struct stat *) xmalloc(sizeof(struct stat))) == NULL) { + xfree(requestp->path); + memPoolFree(aio_request_pool, requestp); + errno = ENOMEM; + return -1; + } + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_STAT; + requestp->cancelled = 0; + + aio_do_request(requestp); + return 0; +} + + +static void +aio_do_stat(aio_request_t * requestp) +{ + requestp->ret = stat(requestp->path, requestp->tmpstatp); + requestp->err = errno; +} + + +int +aio_unlink(const char *path, aio_result_t * resultp) +{ + aio_request_t *requestp; + int len; + + if (!aio_initialised) + aio_init(); + if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { + errno = ENOMEM; + return -1; + } + len = strlen(path) + 1; + if ((requestp->path = (char *) xmalloc(len)) == NULL) { + memPoolFree(aio_request_pool, requestp); + errno = ENOMEM; + return -1; + } + strncpy(requestp->path, path, len); + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_UNLINK; + requestp->cancelled = 0; + + aio_do_request(requestp); + return 0; +} + + +static void +aio_do_unlink(aio_request_t * requestp) +{ + requestp->ret = unlink(requestp->path); + requestp->err = errno; +} + + +#if AIO_OPENDIR +/* XXX aio_opendir NOT implemented? */ + +int +aio_opendir(const char *path, aio_result_t * resultp) +{ + aio_request_t *requestp; + int len; + + if (!aio_initialised) + aio_init(); + if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) { + errno = ENOMEM; + return -1; + } + return -1; +} + +static void +aio_do_opendir(aio_request_t * requestp) +{ + /* NOT IMPLEMENTED */ +} + +#endif + + +void +aio_poll_threads(void) +{ + aio_thread_t *prev; + aio_thread_t *threadp; + aio_request_t *requestp; + + do { /* while found completed thread */ + prev = NULL; + threadp = busy_threads_head; + while (threadp) { + debug(43, 9) ("aio_poll_threads: %p: request type %d -> status %d\n", + threadp, + threadp->processed_req->request_type, + threadp->status); +#if AIO_PROPER_MUTEX + if (threadp->req == NULL) + if (pthread_mutex_trylock(&threadp->mutex) == 0) + break; +#else + if (threadp->req == NULL) + break; +#endif + prev = threadp; + threadp = threadp->next; + } + if (threadp == NULL) + break; + + if (prev == NULL) + busy_threads_head = busy_threads_head->next; + else + prev->next = threadp->next; + + if (busy_threads_tail == threadp) + busy_threads_tail = prev; + + requestp = threadp->processed_req; + threadp->processed_req = NULL; + + threadp->next = wait_threads; + wait_threads = threadp; + + if (request_done_tail != NULL) + request_done_tail->next = requestp; + else + request_done_head = requestp; + request_done_tail = requestp; + } while (threadp); + + aio_process_request_queue(); +} /* aio_poll_threads */ + +aio_result_t * +aio_poll_done(void) +{ + aio_request_t *requestp; + aio_result_t *resultp; + int cancelled; + + AIO_REPOLL: + aio_poll_threads(); + if (request_done_head == NULL) { + return NULL; + } + requestp = request_done_head; + request_done_head = requestp->next; + if (!request_done_head) + request_done_tail = NULL; + + resultp = requestp->resultp; + cancelled = requestp->cancelled; + aio_debug(requestp); + debug(43, 5) ("DONE: %d -> %d\n", requestp->ret, requestp->err); + aio_cleanup_request(requestp); + if (cancelled) + goto AIO_REPOLL; + return resultp; +} /* aio_poll_done */ + +int +aio_operations_pending(void) +{ + return request_queue_len + (request_done_head != NULL) + (busy_threads_head != NULL); +} + +int +aio_overloaded(void) +{ + static time_t last_warn = 0; + if (aio_operations_pending() > RIDICULOUS_LENGTH / 4) { + if (squid_curtime >= (last_warn + 15)) { + debug(43, 0) ("Warning: Async-IO overloaded\n"); + last_warn = squid_curtime; + } + return 1; + } + return 0; +} + +int +aio_sync(void) +{ + int loop_count = 0; + do { + aio_poll_threads(); + assert(++loop_count < 10); + } while (request_queue_len > 0); + return aio_operations_pending(); +} + +int +aio_get_queue_len(void) +{ + return request_queue_len; +} + +static void +aio_debug(aio_request_t * requestp) +{ + switch (requestp->request_type) { + case _AIO_OP_OPEN: + debug(43, 5) ("OPEN of %s to FD %d\n", requestp->path, requestp->ret); + break; + case _AIO_OP_READ: + debug(43, 5) ("READ on fd: %d\n", requestp->fd); + break; + case _AIO_OP_WRITE: + debug(43, 5) ("WRITE on fd: %d\n", requestp->fd); + break; + case _AIO_OP_CLOSE: + debug(43, 5) ("CLOSE of fd: %d\n", requestp->fd); + break; + case _AIO_OP_UNLINK: + debug(43, 5) ("UNLINK of %s\n", requestp->path); + break; + default: + break; + } +} --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/aufs/async_io.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,426 @@ + +/* + * $Id$ + * + * DEBUG: section 32 Asynchronous Disk I/O + * AUTHOR: Pete Bentley + * AUTHOR: Stewart Forster + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "store_asyncufs.h" + +#define _AIO_OPEN 0 +#define _AIO_READ 1 +#define _AIO_WRITE 2 +#define _AIO_CLOSE 3 +#define _AIO_UNLINK 4 +#define _AIO_OPENDIR 5 +#define _AIO_STAT 6 + +typedef struct aio_ctrl_t { + struct aio_ctrl_t *next; + int fd; + int operation; + AIOCB *done_handler; + void *done_handler_data; + aio_result_t result; +} aio_ctrl_t; + +struct { + int open; + int close; + int cancel; + int write; + int read; + int stat; + int unlink; + int check_callback; +} aio_counts; + +typedef struct aio_unlinkq_t { + char *path; + struct aio_unlinkq_t *next; +} aio_unlinkq_t; + +static aio_ctrl_t *used_list = NULL; +static int initialised = 0; +static OBJH aioStats; +static MemPool *aio_ctrl_pool; +static void aioFDWasClosed(int fd); + +MemPool * aio_state_pool; + +static void +aioFDWasClosed(int fd) +{ + if (fd_table[fd].flags.closing) + fd_close(fd); +} + +void +aioInit(void) +{ + if (initialised) + return; + aio_ctrl_pool = memPoolCreate("aio_ctrl", sizeof(aio_ctrl_t)); + aio_state_pool = memPoolCreate("Async UFS IO State data", sizeof(aiostate_t)); + cachemgrRegister("aio_counts", "Async IO Function Counters", + aioStats, 0, 1); + initialised = 1; + comm_quick_poll_required(); +} + +void +aioDone(void) +{ + memPoolDestroy(aio_ctrl_pool); + memPoolDestroy(aio_state_pool); + initialised = 0; +} + +void +aioOpen(const char *path, int oflag, mode_t mode, AIOCB * callback, void *callback_data) +{ + aio_ctrl_t *ctrlp; + int ret; + + assert(initialised); + aio_counts.open++; + ctrlp = memPoolAlloc(aio_ctrl_pool); + ctrlp->fd = -2; + ctrlp->done_handler = callback; + ctrlp->done_handler_data = callback_data; + ctrlp->operation = _AIO_OPEN; + cbdataLock(callback_data); + if (aio_open(path, oflag, mode, &ctrlp->result) < 0) { + ret = open(path, oflag, mode); + if (callback) + (callback) (ctrlp->fd, callback_data, ret, errno); + cbdataUnlock(callback_data); + memPoolFree(aio_ctrl_pool, ctrlp); + return; + } + ctrlp->next = used_list; + used_list = ctrlp; + return; +} + +void +aioClose(int fd) +{ + aio_ctrl_t *ctrlp; + + assert(initialised); + aio_counts.close++; + aioCancel(fd); + ctrlp = memPoolAlloc(aio_ctrl_pool); + ctrlp->fd = fd; + ctrlp->done_handler = NULL; + ctrlp->done_handler_data = NULL; + ctrlp->operation = _AIO_CLOSE; + if (aio_close(fd, &ctrlp->result) < 0) { + close(fd); /* Can't create thread - do a normal close */ + memPoolFree(aio_ctrl_pool, ctrlp); + aioFDWasClosed(fd); + return; + } + ctrlp->next = used_list; + used_list = ctrlp; + return; +} + +void +aioCancel(int fd) +{ + aio_ctrl_t *curr; + aio_ctrl_t *prev; + aio_ctrl_t *next; + AIOCB *done_handler; + void *their_data; + + assert(initialised); + aio_counts.cancel++; + prev = NULL; + curr = used_list; + for (curr = used_list;; curr = next) { + while (curr != NULL) { + if (curr->fd == fd) + break; + prev = curr; + curr = curr->next; + } + if (curr == NULL) + break; + + aio_cancel(&curr->result); + + if ((done_handler = curr->done_handler)) { + their_data = curr->done_handler_data; + curr->done_handler = NULL; + curr->done_handler_data = NULL; + debug(0, 0) ("this be aioCancel\n"); + if (cbdataValid(their_data)) + done_handler(fd, their_data, -2, -2); + cbdataUnlock(their_data); + } + next = curr->next; + if (prev == NULL) + used_list = next; + else + prev->next = next; + + memPoolFree(aio_ctrl_pool, curr); + } +} + + +void +aioWrite(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func) +{ + aio_ctrl_t *ctrlp; + int seekmode; + + assert(initialised); + aio_counts.write++; + for (ctrlp = used_list; ctrlp != NULL; ctrlp = ctrlp->next) + if (ctrlp->fd == fd) + break; + if (ctrlp != NULL) { + debug(0, 0) ("aioWrite: EWOULDBLOCK\n"); + errno = EWOULDBLOCK; + if (callback) + (callback) (fd, callback_data, -1, errno); + if (free_func) + free_func(bufp); + return; + } + ctrlp = memPoolAlloc(aio_ctrl_pool); + ctrlp->fd = fd; + ctrlp->done_handler = callback; + ctrlp->done_handler_data = callback_data; + ctrlp->operation = _AIO_WRITE; + if (offset >= 0) + seekmode = SEEK_SET; + else { + seekmode = SEEK_END; + offset = 0; + } + cbdataLock(callback_data); + if (aio_write(fd, bufp, len, offset, seekmode, &ctrlp->result) < 0) { + if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL) + errno = EWOULDBLOCK; + if (callback) + (callback) (fd, callback_data, -1, errno); + cbdataUnlock(callback_data); + memPoolFree(aio_ctrl_pool, ctrlp); + } else { + ctrlp->next = used_list; + used_list = ctrlp; + } + /* + * aio_write copies the buffer so we can free it here + */ + if (free_func) + free_func(bufp); +} /* aioWrite */ + + +void +aioRead(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data) +{ + aio_ctrl_t *ctrlp; + int seekmode; + + assert(initialised); + aio_counts.read++; + for (ctrlp = used_list; ctrlp != NULL; ctrlp = ctrlp->next) + if (ctrlp->fd == fd) + break; + if (ctrlp != NULL) { + errno = EWOULDBLOCK; + if (callback) + (callback) (fd, callback_data, -1, errno); + return; + } + ctrlp = memPoolAlloc(aio_ctrl_pool); + ctrlp->fd = fd; + ctrlp->done_handler = callback; + ctrlp->done_handler_data = callback_data; + ctrlp->operation = _AIO_READ; + if (offset >= 0) + seekmode = SEEK_SET; + else { + seekmode = SEEK_CUR; + offset = 0; + } + cbdataLock(callback_data); + if (aio_read(fd, bufp, len, offset, seekmode, &ctrlp->result) < 0) { + if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL) + errno = EWOULDBLOCK; + if (callback) + (callback) (fd, callback_data, -1, errno); + cbdataUnlock(callback_data); + memPoolFree(aio_ctrl_pool, ctrlp); + return; + } + ctrlp->next = used_list; + used_list = ctrlp; + return; +} /* aioRead */ + +void +aioStat(char *path, struct stat *sb, AIOCB * callback, void *callback_data) +{ + aio_ctrl_t *ctrlp; + + assert(initialised); + aio_counts.stat++; + ctrlp = memPoolAlloc(aio_ctrl_pool); + ctrlp->fd = -2; + ctrlp->done_handler = callback; + ctrlp->done_handler_data = callback_data; + ctrlp->operation = _AIO_STAT; + cbdataLock(callback_data); + if (aio_stat(path, sb, &ctrlp->result) < 0) { + if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL) + errno = EWOULDBLOCK; + if (callback) + (callback) (ctrlp->fd, callback_data, -1, errno); + cbdataUnlock(callback_data); + memPoolFree(aio_ctrl_pool, ctrlp); + return; + } + ctrlp->next = used_list; + used_list = ctrlp; + return; +} /* aioStat */ + +void +aioUnlink(const char *pathname, AIOCB * callback, void *callback_data) +{ + aio_ctrl_t *ctrlp; + char *path; + assert(initialised); + aio_counts.unlink++; + ctrlp = memPoolAlloc(aio_ctrl_pool); + ctrlp->fd = -2; + ctrlp->done_handler = callback; + ctrlp->done_handler_data = callback_data; + ctrlp->operation = _AIO_UNLINK; + path = xstrdup(pathname); + cbdataLock(callback_data); + if (aio_unlink(path, &ctrlp->result) < 0) { + int ret = unlink(path); + (callback) (ctrlp->fd, callback_data, ret, errno); + cbdataUnlock(callback_data); + memPoolFree(aio_ctrl_pool, ctrlp); + xfree(path); + return; + } + ctrlp->next = used_list; + used_list = ctrlp; + xfree(path); +} /* aioUnlink */ + + +void +aioCheckCallbacks(SwapDir *SD) +{ + aio_result_t *resultp; + aio_ctrl_t *ctrlp; + aio_ctrl_t *prev; + AIOCB *done_handler; + void *their_data; + + assert(initialised); + aio_counts.check_callback++; + for (;;) { + if ((resultp = aio_poll_done()) == NULL) + break; + prev = NULL; + for (ctrlp = used_list; ctrlp != NULL; prev = ctrlp, ctrlp = ctrlp->next) + if (&ctrlp->result == resultp) + break; + if (ctrlp == NULL) + continue; + if (prev == NULL) + used_list = ctrlp->next; + else + prev->next = ctrlp->next; + if ((done_handler = ctrlp->done_handler)) { + their_data = ctrlp->done_handler_data; + ctrlp->done_handler = NULL; + ctrlp->done_handler_data = NULL; + if (cbdataValid(their_data)) + done_handler(ctrlp->fd, their_data, + ctrlp->result.aio_return, ctrlp->result.aio_errno); + cbdataUnlock(their_data); + } + if (ctrlp->operation == _AIO_CLOSE) + aioFDWasClosed(ctrlp->fd); + memPoolFree(aio_ctrl_pool, ctrlp); + } +} + +void +aioStats(StoreEntry * sentry) +{ + storeAppendPrintf(sentry, "ASYNC IO Counters:\n"); + storeAppendPrintf(sentry, "open\t%d\n", aio_counts.open); + storeAppendPrintf(sentry, "close\t%d\n", aio_counts.close); + storeAppendPrintf(sentry, "cancel\t%d\n", aio_counts.cancel); + storeAppendPrintf(sentry, "write\t%d\n", aio_counts.write); + storeAppendPrintf(sentry, "read\t%d\n", aio_counts.read); + storeAppendPrintf(sentry, "stat\t%d\n", aio_counts.stat); + storeAppendPrintf(sentry, "unlink\t%d\n", aio_counts.unlink); + storeAppendPrintf(sentry, "check_callback\t%d\n", aio_counts.check_callback); + storeAppendPrintf(sentry, "queue\t%d\n", aio_get_queue_len()); +} + +/* Flush all pending I/O */ +void +aioSync(SwapDir *SD) +{ + if (!initialised) + return; /* nothing to do then */ + /* Flush all pending operations */ + debug(32, 1) ("aioSync: flushing pending I/O operations\n"); + do { + aioCheckCallbacks(SD); + } while (aio_sync()); + debug(32, 1) ("aioSync: done\n"); +} + +int +aioQueueSize(void) +{ + return memPoolInUseCount(aio_ctrl_pool); +} --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/aufs/store_asyncufs.h Wed Feb 14 00:43:59 2007 @@ -0,0 +1,99 @@ +/* + * store_aufs.h + * + * Internal declarations for the aufs routines + */ + +#ifndef __STORE_ASYNCUFS_H__ +#define __STORE_ASYNCUFS_H__ + +#ifdef ASYNC_IO_THREADS +#define NUMTHREADS ASYNC_IO_THREADS +#else +#define NUMTHREADS 16 +#endif + +#define MAGIC1 (NUMTHREADS/Config.cacheSwap.n_configured/2) + +struct _aio_result_t { + int aio_return; + int aio_errno; +}; + +typedef struct _aio_result_t aio_result_t; + +typedef void AIOCB(int fd, void *, int aio_return, int aio_errno); + +int aio_cancel(aio_result_t *); +int aio_open(const char *, int, mode_t, aio_result_t *); +int aio_read(int, char *, int, off_t, int, aio_result_t *); +int aio_write(int, char *, int, off_t, int, aio_result_t *); +int aio_close(int, aio_result_t *); +int aio_stat(const char *, struct stat *, aio_result_t *); +int aio_unlink(const char *, aio_result_t *); +int aio_opendir(const char *, aio_result_t *); +aio_result_t *aio_poll_done(void); +int aio_operations_pending(void); +int aio_overloaded(void); +int aio_sync(void); +int aio_get_queue_len(void); + +void aioInit(void); +void aioDone(void); +void aioCancel(int); +void aioOpen(const char *, int, mode_t, AIOCB *, void *); +void aioClose(int); +void aioWrite(int, int offset, char *, int size, AIOCB *, void *, FREE *); +void aioRead(int, int offset, char *, int size, AIOCB *, void *); +void aioStat(char *, struct stat *, AIOCB *, void *); +void aioUnlink(const char *, AIOCB *, void *); +void aioCheckCallbacks(SwapDir *); +void aioSync(SwapDir *); +int aioQueueSize(void); + +struct _aioinfo_t { + int swaplog_fd; + int l1; + int l2; + fileMap *map; + int suggest; +}; + +struct _aiostate_t { + int fd; + struct { + unsigned int close_request:1; + unsigned int reading:1; + unsigned int writing:1; + unsigned int opening:1; + } flags; + const char *read_buf; + link_list *pending_writes; + link_list *pending_reads; +}; + +typedef struct _aioinfo_t aioinfo_t; +typedef struct _aiostate_t aiostate_t; + +/* The aio_state memory pool */ +extern MemPool * aio_state_pool; + +extern void storeAufsDirMapBitReset(SwapDir *, sfileno); +extern int storeAufsDirMapBitAllocate(SwapDir *); + +extern char * storeAufsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); +extern void storeAufsDirUnlinkFile(SwapDir *, sfileno); +extern void storeAufsDirReplAdd(SwapDir *SD, StoreEntry *); +extern void storeAufsDirReplRemove(StoreEntry *); + +/* + * Store IO stuff + */ +extern STOBJCREATE storeAufsCreate; +extern STOBJOPEN storeAufsOpen; +extern STOBJCLOSE storeAufsClose; +extern STOBJREAD storeAufsRead; +extern STOBJWRITE storeAufsWrite; +extern STOBJUNLINK storeAufsUnlink; + +#endif --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/aufs/store_dir_aufs.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,1896 @@ + +/* + * $Id$ + * + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Duane Wessels + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#if HAVE_STATVFS +#if HAVE_SYS_STATVFS_H +#include +#endif +#endif + +#include "store_asyncufs.h" + +#define DefaultLevelOneDirs 16 +#define DefaultLevelTwoDirs 256 +#define STORE_META_BASYNCUFSZ 4096 + +typedef struct _RebuildState RebuildState; +struct _RebuildState { + SwapDir *sd; + int n_read; + FILE *log; + int speed; + int curlvl1; + int curlvl2; + struct { + unsigned int need_to_validate:1; + unsigned int clean:1; + unsigned int init:1; + } flags; + int done; + int in_dir; + int fn; + struct dirent *entry; + DIR *td; + char fullpath[SQUID_MAXPATHLEN]; + char fullfilename[SQUID_MAXPATHLEN]; + struct _store_rebuild_data counts; +}; + +static int n_asyncufs_dirs = 0; +static int *asyncufs_dir_index = NULL; + +static char *storeAufsDirSwapSubDir(SwapDir *, int subdirn); +static int storeAufsDirCreateDirectory(const char *path, int); +static int storeAufsDirVerifyCacheDirs(SwapDir *); +static int storeAufsDirVerifyDirectory(const char *path); +static void storeAufsDirCreateSwapSubDirs(SwapDir *); +static char *storeAufsDirSwapLogFile(SwapDir *, const char *); +static EVH storeAufsDirRebuildFromDirectory; +static EVH storeAufsDirRebuildFromSwapLog; +static int storeAufsDirGetNextFile(RebuildState *, int *sfileno, int *size); +static StoreEntry *storeAufsDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean); +static void storeAufsDirRebuild(SwapDir * sd); +static void storeAufsDirCloseTmpSwapLog(SwapDir * sd); +static FILE *storeAufsDirOpenTmpSwapLog(SwapDir *, int *, int *); +static STLOGOPEN storeAufsDirOpenSwapLog; +static STINIT storeAufsDirInit; +static STFREE storeAufsDirFree; +static STLOGCLEANOPEN storeAufsDirWriteCleanOpen; +static void storeAufsDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANWRITE storeAufsDirWriteCleanEntry; +static STLOGCLOSE storeAufsDirCloseSwapLog; +static STLOGWRITE storeAufsDirSwapLog; +static STNEWFS storeAufsDirNewfs; +static STDUMP storeAufsDirDump; +static STMAINTAINFS storeAufsDirMaintain; +static STCHECKOBJ storeAufsDirCheckObj; +static STREFOBJ storeAufsDirRefObj; +static STUNREFOBJ storeAufsDirUnrefObj; +static QS rev_int_sort; +static int storeAufsDirClean(int swap_index); +static EVH storeAufsDirCleanEvent; +static int storeAufsDirIs(SwapDir * sd); +static int storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2); +static int storeAufsCleanupDoubleCheck(SwapDir *, StoreEntry *); +static void storeAufsDirStats(SwapDir *, StoreEntry *); +static void storeAufsDirInitBitmap(SwapDir *); +static int storeAufsDirValidFileno(SwapDir *, sfileno); +static int storeAufsDirCheckExpired(SwapDir *, StoreEntry *); +#if !HEAP_REPLACEMENT +static time_t storeAufsDirExpiredReferenceAge(SwapDir *); +#endif + +/* + * These functions were ripped straight out of the heart of store_dir.c. + * They assume that the given filenum is on a asyncufs partiton, which may or + * may not be true.. + * XXX this evilness should be tidied up at a later date! + */ + +int +storeAufsDirMapBitTest(SwapDir *SD, int fn) +{ + sfileno filn = fn; + aioinfo_t *aioinfo; + aioinfo = (aioinfo_t *)SD->fsdata; + return file_map_bit_test(aioinfo->map, filn); +} + +void +storeAufsDirMapBitSet(SwapDir *SD, int fn) +{ + sfileno filn = fn; + aioinfo_t *aioinfo; + aioinfo = (aioinfo_t *)SD->fsdata; + file_map_bit_set(aioinfo->map, filn); +} + +void +storeAufsDirMapBitReset(SwapDir *SD, int fn) +{ + sfileno filn = fn; + aioinfo_t *aioinfo; + aioinfo = (aioinfo_t *)SD->fsdata; + file_map_bit_reset(aioinfo->map, filn); +} + +int +storeAufsDirMapBitAllocate(SwapDir *SD) +{ + aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + int fn; + fn = file_map_allocate(aioinfo->map, aioinfo->suggest); + file_map_bit_set(aioinfo->map, fn); + aioinfo->suggest = fn + 1; + return fn; +} + + +/* + * Initialise the asyncufs bitmap + * + * If there already is a bitmap, and the numobjects is larger than currently + * configured, we allocate a new bitmap and 'grow' the old one into it. + */ +static void +storeAufsDirInitBitmap(SwapDir *sd) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + + if (aioinfo->map == NULL) { + /* First time */ + aioinfo->map = file_map_create(); + } else if (aioinfo->map->max_n_files) { + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ + } + /* else it shrunk, and we leave the old one in place */ +} + +static char * +storeAufsDirSwapSubDir(SwapDir * sd, int subdirn) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + assert(0 <= subdirn && subdirn < aioinfo->l1); + snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn); + return fullfilename; +} + +static int +storeAufsDirCreateDirectory(const char *path, int should_exist) +{ + int created = 0; + struct stat st; + getCurrentTime(); + if (0 == stat(path, &st)) { + if (S_ISDIR(st.st_mode)) { + debug(20, should_exist ? 3 : 1) ("%s exists\n", path); + } else { + fatalf("Swap directory %s is not a directory.", path); + } + } else if (0 == mkdir(path, 0755)) { + debug(20, should_exist ? 1 : 3) ("%s created\n", path); + created = 1; + } else { + fatalf("Failed to make swap directory %s: %s", + path, xstrerror()); + } + return created; +} + +static int +storeAufsDirVerifyDirectory(const char *path) +{ + struct stat sb; + if (stat(path, &sb) < 0) { + debug(20, 0) ("%s: %s\n", path, xstrerror()); + return -1; + } + if (S_ISDIR(sb.st_mode) == 0) { + debug(20, 0) ("%s is not a directory\n", path); + return -1; + } + return 0; +} + +/* + * This function is called by storeAufsDirInit(). If this returns < 0, + * then Squid exits, complains about swap directories not + * existing, and instructs the admin to run 'squid -z' + */ +static int +storeAufsDirVerifyCacheDirs(SwapDir * sd) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + int j; + const char *path = sd->path; + + if (storeAufsDirVerifyDirectory(path) < 0) + return -1; + for (j = 0; j < aioinfo->l1; j++) { + path = storeAufsDirSwapSubDir(sd, j); + if (storeAufsDirVerifyDirectory(path) < 0) + return -1; + } + return 0; +} + +static void +storeAufsDirCreateSwapSubDirs(SwapDir * sd) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + int i, k; + int should_exist; + LOCAL_ARRAY(char, name, MAXPATHLEN); + for (i = 0; i < aioinfo->l1; i++) { + snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i); + if (storeAufsDirCreateDirectory(name, 0)) + should_exist = 0; + else + should_exist = 1; + debug(47, 1) ("Making directories in %s\n", name); + for (k = 0; k < aioinfo->l2; k++) { + snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k); + storeAufsDirCreateDirectory(name, should_exist); + } + } +} + +static char * +storeAufsDirSwapLogFile(SwapDir * sd, const char *ext) +{ + LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; + if (Config.Log.swap) { + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } + } else { + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); + } + if (ext) + strncat(path, ext, 16); + return path; +} + +static void +storeAufsDirOpenSwapLog(SwapDir * sd) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + char *path; + int fd; + path = storeAufsDirSwapLogFile(sd, NULL); + fd = file_open(path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", path, xstrerror()); + fatal("storeAufsDirOpenSwapLog: Failed to open swap log."); + } + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); + aioinfo->swaplog_fd = fd; + if (0 == n_asyncufs_dirs) + assert(NULL == asyncufs_dir_index); + n_asyncufs_dirs++; + assert(n_asyncufs_dirs <= Config.cacheSwap.n_configured); +} + +static void +storeAufsDirCloseSwapLog(SwapDir * sd) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + if (aioinfo->swaplog_fd < 0) /* not open */ + return; + file_close(aioinfo->swaplog_fd); + debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", + sd->index, aioinfo->swaplog_fd); + aioinfo->swaplog_fd = -1; + n_asyncufs_dirs--; + assert(n_asyncufs_dirs >= 0); + if (0 == n_asyncufs_dirs) + safe_free(asyncufs_dir_index); +} + +static void +storeAufsDirInit(SwapDir * sd) +{ + static int started_clean_event = 0; + static const char *errmsg = + "\tFailed to verify one of the swap directories, Check cache.log\n" + "\tfor details. Run 'squid -z' to create swap directories\n" + "\tif needed, or if running Squid for the first time."; + storeAufsDirInitBitmap(sd); + if (storeAufsDirVerifyCacheDirs(sd) < 0) + fatal(errmsg); + storeAufsDirOpenSwapLog(sd); + storeAufsDirRebuild(sd); + if (!started_clean_event) { + eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL, 15.0, 1); + started_clean_event = 1; + } +} + +static void +storeAufsDirRebuildFromDirectory(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE); + StoreEntry *e = NULL; + StoreEntry tmpe; + cache_key key[MD5_DIGEST_CHARS]; + int sfileno = 0; + int count; + int size; + struct stat sb; + int swap_hdr_len; + int fd = -1; + tlv *tlv_list; + tlv *t; + assert(rb != NULL); + debug(20, 3) ("storeAufsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index); + for (count = 0; count < rb->speed; count++) { + assert(fd == -1); + fd = storeAufsDirGetNextFile(rb, &sfileno, &size); + if (fd == -2) { + debug(20, 1) ("Done scanning %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + store_dirs_rebuilding--; + storeAufsDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } else if (fd < 0) { + continue; + } + assert(fd > -1); + /* lets get file stats here */ + if (fstat(fd, &sb) < 0) { + debug(20, 1) ("storeAufsDirRebuildFromDirectory: fstat(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %s %7d files opened so far.\n", + rb->sd->path, rb->counts.scancount); + debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno); + Counter.syscalls.disk.reads++; + if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) { + debug(20, 1) ("storeAufsDirRebuildFromDirectory: read(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + file_close(fd); + store_open_disk_fd--; + fd = -1; + swap_hdr_len = 0; +#if USE_TRUNCATE + if (sb.st_size == 0) + continue; +#endif + tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); + if (tlv_list == NULL) { + debug(20, 1) ("storeAufsDirRebuildFromDirectory: failed to get meta data\n"); + /* XXX shouldn't this be a call to storeAufsUnlink ? */ + storeAufsDirUnlinkFile(SD, sfileno); + continue; + } + debug(20, 3) ("storeAufsDirRebuildFromDirectory: successful swap meta unpacking\n"); + memset(key, '\0', MD5_DIGEST_CHARS); + memset(&tmpe, '\0', sizeof(StoreEntry)); + for (t = tlv_list; t; t = t->next) { + switch (t->type) { + case STORE_META_KEY: + assert(t->length == MD5_DIGEST_CHARS); + xmemcpy(key, t->value, MD5_DIGEST_CHARS); + break; + case STORE_META_STD: + assert(t->length == STORE_HDR_METASIZE); + xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); + break; + default: + break; + } + } + storeSwapTLVFree(tlv_list); + tlv_list = NULL; + if (storeKeyNull(key)) { + debug(20, 1) ("storeAufsDirRebuildFromDirectory: NULL key\n"); + storeAufsDirUnlinkFile(SD, sfileno); + continue; + } + tmpe.key = key; + /* check sizes */ + if (tmpe.swap_file_sz == 0) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz != sb.st_size) { + debug(20, 1) ("storeAufsDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n", + tmpe.swap_file_sz, (int) sb.st_size); + storeAufsDirUnlinkFile(SD, sfileno); + continue; + } + if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { + storeAufsDirUnlinkFile(SD, sfileno); + rb->counts.badflags++; + continue; + } + e = storeGet(key); + if (e && e->lastref >= tmpe.lastref) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (NULL != e) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + rb->counts.dupcount++; + } + rb->counts.objcount++; + storeEntryDump(&tmpe, 5); + e = storeAufsDirAddDiskRestore(SD, key, + sfileno, + tmpe.swap_file_sz, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + tmpe.refcount, /* refcount */ + tmpe.flags, /* flags */ + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeAufsDirRebuildFromDirectory, rb, 0.0, 1); +} + +static void +storeAufsDirRebuildFromSwapLog(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + StoreEntry *e = NULL; + storeSwapLogData s; + size_t ss = sizeof(storeSwapLogData); + int count; + int used; /* is swapfile already in use? */ + int disk_entry_newer; /* is the log entry newer than current entry? */ + double x; + assert(rb != NULL); + /* load a number of objects per invocation */ + for (count = 0; count < rb->speed; count++) { + if (fread(&s, ss, 1, rb->log) != 1) { + debug(20, 1) ("Done reading %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + fclose(rb->log); + rb->log = NULL; + store_dirs_rebuilding--; + storeAufsDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } + rb->n_read++; + if (s.op <= SWAP_LOG_NOP) + continue; + if (s.op >= SWAP_LOG_MAX) + continue; + debug(20, 3) ("storeAufsDirRebuildFromSwapLog: %s %s %08X\n", + swap_log_op_str[(int) s.op], + storeKeyText(s.key), + s.swap_filen); + if (s.op == SWAP_LOG_ADD) { + (void) 0; + } else if (s.op == SWAP_LOG_DEL) { + if ((e = storeGet(s.key)) != NULL) { + /* + * Make sure we don't unlink the file, it might be + * in use by a subsequent entry. Also note that + * we don't have to subtract from store_swap_size + * because adding to store_swap_size happens in + * the cleanup procedure. + */ + storeExpireNow(e); + storeReleaseRequest(e); + storeAufsDirReplRemove(e); + storeRelease(e); + if (e->swap_filen > -1) { + /* Fake a unlink here, this is a bad hack :( */ + e->swap_filen = -1; + e->swap_dirn = -1; + } + rb->counts.objcount--; + rb->counts.cancelcount++; + } + continue; + } else { + x = log(++rb->counts.bad_log_op) / log(10.0); + if (0.0 == x - (double) (int) x) + debug(20, 1) ("WARNING: %d invalid swap log entries found\n", + rb->counts.bad_log_op); + rb->counts.invalid++; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %7d %s Entries read so far.\n", + rb->counts.scancount, rb->sd->path); + if (!storeAufsDirValidFileno(SD, s.swap_filen)) { + rb->counts.invalid++; + continue; + } + if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + rb->counts.badflags++; + continue; + } + e = storeGet(s.key); + used = storeAufsDirMapBitTest(SD, s.swap_filen); + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; + if (used && !disk_entry_newer) { + /* log entry is old, ignore it */ + rb->counts.clashcount++; + continue; + } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) { + /* swapfile taken, same URL, newer, update meta */ + if (e->store_status == STORE_OK) { + e->lastref = s.timestamp; + e->timestamp = s.timestamp; + e->expires = s.expires; + e->lastmod = s.lastmod; + e->flags = s.flags; + e->refcount += s.refcount; +#if HEAP_REPLACEMENT + storeHeapPositionUpdate(e, SD); + storeAufsDirUnrefObj(SD, e); +#endif + } else { + debug_trap("storeAufsDirRebuildFromSwapLog: bad condition"); + debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); + } + continue; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", + SD->index, s.swap_filen); + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + assert(rb->flags.need_to_validate); + rb->counts.clashcount++; + continue; + } else if (e && !disk_entry_newer) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (e) { + /* key already exists, this swapfile not being used */ + /* junk old, load new */ + storeExpireNow(e); + storeReleaseRequest(e); + storeAufsDirReplRemove(e); + storeRelease(e); + if (e->swap_filen > -1) { + /* Fake a unlink here, this is a bad hack :( */ + e->swap_filen = -1; + e->swap_dirn = -1; + } + rb->counts.dupcount++; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + /* update store_swap_size */ + rb->counts.objcount++; + e = storeAufsDirAddDiskRestore(SD, s.key, + s.swap_filen, + s.swap_file_sz, + s.expires, + s.timestamp, + s.lastref, + s.lastmod, + s.refcount, + s.flags, + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1); +} + +static int +storeAufsDirGetNextFile(RebuildState * rb, int *sfileno, int *size) +{ + SwapDir *SD = rb->sd; + aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + int fd = -1; + int used = 0; + int dirs_opened = 0; + debug(20, 3) ("storeAufsDirGetNextFile: flag=%d, %d: /%02X/%02X\n", + rb->flags.init, + rb->sd->index, + rb->curlvl1, + rb->curlvl2); + if (rb->done) + return -2; + while (fd < 0 && rb->done == 0) { + fd = -1; + if (0 == rb->flags.init) { /* initialize, open first file */ + rb->done = 0; + rb->curlvl1 = 0; + rb->curlvl2 = 0; + rb->in_dir = 0; + rb->flags.init = 1; + assert(Config.cacheSwap.n_configured > 0); + } + if (0 == rb->in_dir) { /* we need to read in a new directory */ + snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", + rb->sd->path, + rb->curlvl1, rb->curlvl2); + if (rb->flags.init && rb->td != NULL) + closedir(rb->td); + rb->td = NULL; + if (dirs_opened) + return -1; + rb->td = opendir(rb->fullpath); + dirs_opened++; + if (rb->td == NULL) { + debug(50, 1) ("storeAufsDirGetNextFile: opendir: %s: %s\n", + rb->fullpath, xstrerror()); + } else { + rb->entry = readdir(rb->td); /* skip . and .. */ + rb->entry = readdir(rb->td); + if (rb->entry == NULL && errno == ENOENT) + debug(20, 1) ("storeAufsDirGetNextFile: directory does not exist!.\n"); + debug(20, 3) ("storeAufsDirGetNextFile: Directory %s\n", rb->fullpath); + } + } + if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) { + rb->in_dir++; + if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) { + debug(20, 3) ("storeAufsDirGetNextFile: invalid %s\n", + rb->entry->d_name); + continue; + } + if (!storeAufsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) { + debug(20, 3) ("storeAufsDirGetNextFile: %08X does not belong in %d/%d/%d\n", + rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2); + continue; + } + used = storeAufsDirMapBitTest(SD, rb->fn); + if (used) { + debug(20, 3) ("storeAufsDirGetNextFile: Locked, continuing with next.\n"); + continue; + } + snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s", + rb->fullpath, rb->entry->d_name); + debug(20, 3) ("storeAufsDirGetNextFile: Opening %s\n", rb->fullfilename); + fd = file_open(rb->fullfilename, O_RDONLY); + if (fd < 0) + debug(50, 1) ("storeAufsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror()); + else + store_open_disk_fd++; + continue; + } + rb->in_dir = 0; + if (++rb->curlvl2 < aioinfo->l2) + continue; + rb->curlvl2 = 0; + if (++rb->curlvl1 < aioinfo->l1) + continue; + rb->curlvl1 = 0; + rb->done = 1; + } + *sfileno = rb->fn; + return fd; +} + +/* Add a new object to the cache with empty memory copy and pointer to disk + * use to rebuild store from disk. */ +static StoreEntry * +storeAufsDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean) +{ + StoreEntry *e = NULL; + debug(20, 5) ("storeAufsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); + /* if you call this you'd better be sure file_number is not + * already in use! */ + e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + e->store_status = STORE_OK; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; + e->swap_filen = file_number; + e->swap_dirn = SD->index; + e->swap_file_sz = swap_file_sz; + e->lock_count = 0; +#if !HEAP_REPLACEMENT + e->refcount = 0; +#endif + e->lastref = lastref; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->refcount = refcount; + e->flags = flags; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + storeAufsDirMapBitSet(SD, e->swap_filen); + storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ + storeAufsDirReplAdd(SD, e); + return e; +} + +static void +storeAufsDirRebuild(SwapDir * sd) +{ + RebuildState *rb = xcalloc(1, sizeof(*rb)); + int clean = 0; + int zero = 0; + FILE *fp; + EVH *func = NULL; + rb->sd = sd; + rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; + /* + * If the swap.state file exists in the cache_dir, then + * we'll use storeAufsDirRebuildFromSwapLog(), otherwise we'll + * use storeAufsDirRebuildFromDirectory() to open up each file + * and suck in the meta data. + */ + fp = storeAufsDirOpenTmpSwapLog(sd, &clean, &zero); + if (fp == NULL || zero) { + if (fp != NULL) + fclose(fp); + func = storeAufsDirRebuildFromDirectory; + } else { + func = storeAufsDirRebuildFromSwapLog; + rb->log = fp; + rb->flags.clean = (unsigned int) clean; + } + if (!clean) + rb->flags.need_to_validate = 1; + debug(20, 1) ("Rebuilding storage in %s (%s)\n", + sd->path, clean ? "CLEAN" : "DIRTY"); + store_dirs_rebuilding++; + cbdataAdd(rb, cbdataXfree, 0); + eventAdd("storeRebuild", func, rb, 0.0, 1); +} + +static void +storeAufsDirCloseTmpSwapLog(SwapDir * sd) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); + char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new")); + int fd; + file_close(aioinfo->swaplog_fd); +#ifdef _SQUID_OS2_ + if (unlink(swaplog_path) < 0) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeAufsDirCloseTmpSwapLog: unlink failed"); + } +#endif + if (xrename(new_path, swaplog_path) < 0) { + fatal("storeAufsDirCloseTmpSwapLog: rename failed"); + } + fd = file_open(swaplog_path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeAufsDirCloseTmpSwapLog: Failed to open swap log."); + } + safe_free(swaplog_path); + safe_free(new_path); + aioinfo->swaplog_fd = fd; + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); +} + +static FILE * +storeAufsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); + char *clean_path = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean")); + char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new")); + struct stat log_sb; + struct stat clean_sb; + FILE *fp; + int fd; + if (stat(swaplog_path, &log_sb) < 0) { + debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return NULL; + } + *zero_flag = log_sb.st_size == 0 ? 1 : 0; + /* close the existing write-only FD */ + if (aioinfo->swaplog_fd >= 0) + file_close(aioinfo->swaplog_fd); + /* open a write-only FD for the new log */ + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", new_path, xstrerror()); + fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); + } + aioinfo->swaplog_fd = fd; + /* open a read-only stream of the old log */ + fp = fopen(swaplog_path, "r"); + if (fp == NULL) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("Failed to open swap log for reading"); + } + memset(&clean_sb, '\0', sizeof(struct stat)); + if (stat(clean_path, &clean_sb) < 0) + *clean_flag = 0; + else if (clean_sb.st_mtime < log_sb.st_mtime) + *clean_flag = 0; + else + *clean_flag = 1; + safeunlink(clean_path, 1); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return fp; +} + +struct _clean_state { + char *cur; + char *new; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; +}; + +#define CLEAN_BUF_SZ 16384 +/* + * Begin the process to write clean cache state. For ASYNCUFS this means + * opening some log files and allocating write buffers. Return 0 if + * we succeed, and assign the 'func' and 'data' return pointers. + */ +static int +storeAufsDirWriteCleanOpen(SwapDir * sd) +{ + struct _clean_state *state = xcalloc(1, sizeof(*state)); + struct stat sb; + sd->log.clean.write = NULL; + sd->log.clean.state = NULL; + state->cur = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); + state->new = xstrdup(storeAufsDirSwapLogFile(sd, ".clean")); + state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean")); + state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); + state->outbuf_offset = 0; + unlink(state->new); + unlink(state->cln); + state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); + if (state->fd < 0) + return -1; + debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n", + state->new, state->fd); +#if HAVE_FCHMOD + if (stat(state->cur, &sb) == 0) + fchmod(state->fd, sb.st_mode); +#endif + sd->log.clean.write = storeAufsDirWriteCleanEntry; + sd->log.clean.state = state; + return 0; +} + +/* + * "write" an entry to the clean log file. + */ +static void +storeAufsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +{ + storeSwapLogData s; + static size_t ss = sizeof(storeSwapLogData); + struct _clean_state *state = sd->log.clean.state; + if (NULL == e) { + storeAufsDirWriteCleanClose(sd); + return; + } + memset(&s, '\0', ss); + s.op = (char) SWAP_LOG_ADD; + s.swap_filen = e->swap_filen; + s.timestamp = e->timestamp; + s.lastref = e->lastref; + s.expires = e->expires; + s.lastmod = e->lastmod; + s.swap_file_sz = e->swap_file_sz; + s.refcount = e->refcount; + s.flags = e->flags; + xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); + xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); + state->outbuf_offset += ss; + /* buffered write */ + if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; + } + state->outbuf_offset = 0; + } +} + +static void +storeAufsDirWriteCleanClose(SwapDir * sd) +{ + struct _clean_state *state = sd->log.clean.state; + if (state->fd < 0) + return; + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile " + "not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + } + safe_free(state->outbuf); + /* + * You can't rename open files on Microsoft "operating systems" + * so we have to close before renaming. + */ + storeAufsDirCloseSwapLog(sd); + /* rename */ + if (state->fd >= 0) { +#ifdef _SQUID_OS2_ + file_close(state->fd); + state->fd = -1; + if (unlink(cur) < 0) + debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n", + xstrerror(), cur); +#endif + xrename(state->new, state->cur); + } + /* touch a timestamp file if we're not still validating */ + if (store_dirs_rebuilding) + (void) 0; + else if (state->fd < 0) + (void) 0; + else + file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC)); + /* close */ + safe_free(state->cur); + safe_free(state->new); + safe_free(state->cln); + if (state->fd >= 0) + file_close(state->fd); + state->fd = -1; + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; +} + +static void +storeAufsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) +{ + aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata; + storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); + s->op = (char) op; + s->swap_filen = e->swap_filen; + s->timestamp = e->timestamp; + s->lastref = e->lastref; + s->expires = e->expires; + s->lastmod = e->lastmod; + s->swap_file_sz = e->swap_file_sz; + s->refcount = e->refcount; + s->flags = e->flags; + xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); + file_write(aioinfo->swaplog_fd, + -1, + s, + sizeof(storeSwapLogData), + NULL, + NULL, + xfree); +} + +static void +storeAufsDirNewfs(SwapDir * sd) +{ + debug(47, 3) ("Creating swap space in %s\n", sd->path); + storeAufsDirCreateDirectory(sd->path, 0); + storeAufsDirCreateSwapSubDirs(sd); +} + +static int +rev_int_sort(const void *A, const void *B) +{ + const int *i1 = A; + const int *i2 = B; + return *i2 - *i1; +} + +static int +storeAufsDirClean(int swap_index) +{ + DIR *dp = NULL; + struct dirent *de = NULL; + LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); + LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); +#if USE_TRUNCATE + struct stat sb; +#endif + int files[20]; + int swapfileno; + int fn; /* same as swapfileno, but with dirn bits set */ + int n = 0; + int k = 0; + int N0, N1, N2; + int D0, D1, D2; + SwapDir *SD; + aioinfo_t *aioinfo; + N0 = n_asyncufs_dirs; + D0 = asyncufs_dir_index[swap_index % N0]; + SD = &Config.cacheSwap.swapDirs[D0]; + aioinfo = (aioinfo_t *)SD->fsdata; + N1 = aioinfo->l1; + D1 = (swap_index / N0) % N1; + N2 = aioinfo->l2; + D2 = ((swap_index / N0) / N1) % N2; + snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X", + Config.cacheSwap.swapDirs[D0].path, D1, D2); + debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1); + dp = opendir(p1); + if (dp == NULL) { + if (errno == ENOENT) { + debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1); + if (mkdir(p1, 0777) == 0) + return 0; + } + debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror()); + safeunlink(p1, 1); + return 0; + } + while ((de = readdir(dp)) != NULL && k < 20) { + if (sscanf(de->d_name, "%X", &swapfileno) != 1) + continue; + fn = swapfileno; /* XXX should remove this cruft ! */ + if (storeAufsDirValidFileno(SD, fn)) + if (storeAufsDirMapBitTest(SD, fn)) + if (storeAufsFilenoBelongsHere(fn, D0, D1, D2)) + continue; +#if USE_TRUNCATE + if (!stat(de->d_name, &sb)) + if (sb.st_size == 0) + continue; +#endif + files[k++] = swapfileno; + } + closedir(dp); + if (k == 0) + return 0; + qsort(files, k, sizeof(int), rev_int_sort); + if (k > 10) + k = 10; + for (n = 0; n < k; n++) { + debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]); + snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); +#if USE_TRUNCATE + truncate(p2, 0); +#else + safeunlink(p2, 0); +#endif + Counter.swap_files_cleaned++; + } + debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1); + return k; +} + +static void +storeAufsDirCleanEvent(void *unused) +{ + static int swap_index = 0; + int i; + int j = 0; + int n = 0; + /* + * Assert that there are ASYNCUFS cache_dirs configured, otherwise + * we should never be called. + */ + assert(n_asyncufs_dirs); + if (NULL == asyncufs_dir_index) { + SwapDir *sd; + aioinfo_t *aioinfo; + /* + * Initialize the little array that translates ASYNCUFS cache_dir + * number into the Config.cacheSwap.swapDirs array index. + */ + asyncufs_dir_index = xcalloc(n_asyncufs_dirs, sizeof(*asyncufs_dir_index)); + for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { + sd = &Config.cacheSwap.swapDirs[i]; + if (!storeAufsDirIs(sd)) + continue; + asyncufs_dir_index[n++] = i; + aioinfo = (aioinfo_t *)sd->fsdata; + j += (aioinfo->l1 * aioinfo->l2); + } + assert(n == n_asyncufs_dirs); + /* + * Start the storeAufsDirClean() swap_index with a random + * value. j equals the total number of ASYNCUFS level 2 + * swap directories + */ + swap_index = (int) (squid_random() % j); + } + if (0 == store_dirs_rebuilding) { + n = storeAufsDirClean(swap_index); + swap_index++; + } + eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL, + 15.0 * exp(-0.25 * n), 1); +} + +static int +storeAufsDirIs(SwapDir * sd) +{ + if (strncmp(sd->type, "aufs", 3) == 0) + return 1; + return 0; +} + +/* + * Does swapfile number 'fn' belong in cachedir #F0, + * level1 dir #F1, level2 dir #F2? + */ +static int +storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2) +{ + int D1, D2; + int L1, L2; + int filn = fn; + aioinfo_t *aioinfo; + assert(F0 < Config.cacheSwap.n_configured); + aioinfo = (aioinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata; + L1 = aioinfo->l1; + L2 = aioinfo->l2; + D1 = ((filn / L2) / L2) % L1; + if (F1 != D1) + return 0; + D2 = (filn / L2) % L2; + if (F2 != D2) + return 0; + return 1; +} + +int +storeAufsDirValidFileno(SwapDir *SD, sfileno filn) +{ + aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + if (filn < 0) + return 0; + if (filn > aioinfo->map->max_n_files) + return 0; + return 1; +} + +void +storeAufsDirMaintain(SwapDir *SD) +{ + StoreEntry *e = NULL; + int scanned = 0; + int locked = 0; + int expired = 0; + int max_scan; + int max_remove; + double f; + static time_t last_warn_time = 0; +#if !HEAP_REPLACEMENT + dlink_node *m; + dlink_node *prev = NULL; +#else + heap_key age; + heap_key min_age = 0.0; + link_list *locked_entries = NULL; +#if HEAP_REPLACEMENT_DEBUG + if (!verify_heap_property(SD->repl.heap.heap)) { + debug(20, 1) ("Heap property violated!\n"); + } +#endif +#endif + /* We can't delete objects while rebuilding swap */ + if (store_dirs_rebuilding) { + return; + } else { + f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + max_scan = (int) (f * 400.0 + 100.0); + max_remove = (int) (f * 70.0 + 10.0); + /* + * This is kinda cheap, but so we need this priority hack? + */ +#if 0 + eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); +#endif + } + debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove); +#if HEAP_REPLACEMENT + while (heap_nodes(SD->repl.heap.heap) > 0) { + if (store_swap_size < store_swap_low) + break; + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + age = heap_peepminkey(SD->repl.heap.heap); + e = heap_extractmin(SD->repl.heap.heap); + e->repl.node = NULL; /* no longer in the heap */ + scanned++; + if (storeEntryLocked(e)) { + /* + * Entry is in use ... put it in a linked list to ignore it. + */ + if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + /* + * If this was a "SPECIAL" do not add it back into the heap. + * It will always be "SPECIAL" and therefore never removed. + */ + debug(20, 4) ("storeAufsDirMaintain: locked url %s\n", + (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e-> +key)); + linklistPush(&locked_entries, e); + } + locked++; + continue; + } else if (storeAufsDirCheckExpired(SD, e)) { + /* + * Note: This will not check the reference age ifdef + * HEAP_REPLACEMENT, but it does some other useful + * checks... + */ + expired++; + debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", + age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); + min_age = age; + storeRelease(e); + } else { + /* + * Did not expire the object so we need to add it back + * into the heap! + */ + debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", + storeKeyText(e->key)); + linklistPush(&locked_entries, e); + continue; + } + if (store_swap_size < store_swap_low) + break; + else if (expired >= max_remove) + break; + else if (scanned >= max_scan) + break; + } + /* + * Bump the heap age factor. + */ + if (min_age > 0.0) + SD->repl.heap.heap->age = min_age; + /* + * Reinsert all bumped locked entries back into heap... + */ + while ((e = linklistShift(&locked_entries))) + e->repl.node = heap_insert(SD->repl.heap.heap, e); +#else + for (m = SD->repl.lru.list.tail; m; m = prev) { + prev = m->prev; + e = m->data; + scanned++; + if (storeEntryLocked(e)) { + /* + * If there is a locked entry at the tail of the LRU list, + * move it to the beginning to get it out of the way. + * Theoretically, we might have all locked objects at the + * tail, and then we'll never remove anything here and the + * LRU age will go to zero. + */ + if (memInUse(MEM_STOREENTRY) > max_scan) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } + locked++; + + } else if (storeAufsDirCheckExpired(SD, e)) { + expired++; + storeRelease(e); + } + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + } +#endif + debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l +ocked %d f=%.03f\n", + scanned, max_scan, expired, max_remove, locked, f); + debug(20, 3) ("storeMaintainSwapSpace stats:\n"); + debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); + debug(20, 3) (" %6d were scanned\n", scanned); + debug(20, 3) (" %6d were locked\n", locked); + debug(20, 3) (" %6d were expired\n", expired); + if (store_swap_size < Config.Swap.maxSize) + return; + if (squid_curtime - last_warn_time < 10) + return; + debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", + store_swap_size, Config.Swap.maxSize); + last_warn_time = squid_curtime; +} + +/* + * storeAufsDirCheckObj + * + * This routine is called by storeDirSelectSwapDir to see if the given + * object is able to be stored on this filesystem. AUFS filesystems will + * happily store anything as long as the LRU time isn't too small. + */ +int +storeAufsDirCheckObj(SwapDir *SD, const StoreEntry *e) +{ + int loadav; + int ql; + +#if !HEAP_REPLACEMENT + if (storeAufsDirExpiredReferenceAge(SD) < 300) { + debug(20, 3) ("storeAufsDirCheckObj: NO: LRU Age = %d\n", + storeAufsDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; + } +#endif + ql = aioQueueSize(); + if (ql == 0) + loadav = 0; + else if (ql >= MAGIC1) /* Queue is too long, don't even consider it */ + loadav = -1; + else + loadav = MAGIC1 * 1000 / ql; + return loadav; +} + +/* + * storeAufsDirRefObj + * + * This routine is called whenever an object is referenced, so we can + * maintain replacement information within the storage fs. + */ +void +storeAufsDirRefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeAufsDirRefObj: referencing %d/%d\n", e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT + /* Nothing to do here */ +#else + /* Reference the object */ + if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && + !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } +#endif +} + +/* + * storeAufsDirUnrefObj + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ +void +storeAufsDirUnrefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeAufsDirUnrefObj: referencing %d/%d\n", e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT + if (e->repl.node) + heap_update(SD->repl.heap.heap, e->repl.node, e); +#endif +} + +/* + * storeAufsDirUnlinkFile + * + * This routine unlinks a file and pulls it out of the bitmap. + * It used to be in storeAufsUnlink(), however an interface change + * forced this bit of code here. Eeek. + */ +void +storeAufsDirUnlinkFile(SwapDir *SD, sfileno f) +{ + debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f); + storeAufsDirMapBitReset(SD, f); + aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL); +} + +#if !HEAP_REPLACEMENT +/* + * storeAufsDirExpiredReferenceAge + * + * The LRU age is scaled exponentially between 1 minute and + * Config.referenceAge , when store_swap_low < store_swap_size < + * store_swap_high. This keeps store_swap_size within the low and high + * water marks. If the cache is very busy then store_swap_size stays + * closer to the low water mark, if it is not busy, then it will stay + * near the high water mark. The LRU age value can be examined on the + * cachemgr 'info' page. + */ +static time_t +storeAufsDirExpiredReferenceAge(SwapDir *SD) +{ + double x; + double z; + time_t age; + long store_high, store_low; + + store_high = (long) (((float) SD->max_size * + (float) Config.Swap.highWaterMark) / (float) 100); + store_low = (long) (((float) SD->max_size * + (float) Config.Swap.lowWaterMark) / (float) 100); + debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size); + + x = (double) (store_high - SD->cur_size) / + (store_high - store_low); + x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; + z = pow((double) (Config.referenceAge / 60), x); + age = (time_t) (z * 60.0); + if (age < 60) + age = 60; + else if (age > Config.referenceAge) + age = Config.referenceAge; + return age; +} +#endif + +/* + * storeAufsDirCheckExpired + * + * Check whether the given object is expired or not + * It breaks layering a little by calling the upper layers to find + * out whether the object is locked or not, but we can't help this + * right now. + */ +static int +storeAufsDirCheckExpired(SwapDir *SD, StoreEntry *e) +{ + if (storeEntryLocked(e)) + return 0; + if (EBIT_TEST(e->flags, RELEASE_REQUEST)) + return 1; + if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) + return 1; + +#if HEAP_REPLACEMENT + /* + * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap + * controls the replacement of objects. + */ + return 1; +#else + if (squid_curtime - e->lastref > storeAufsDirExpiredReferenceAge(SD)) + return 1; + return 0; +#endif +} + +/* + * Add and remove the given StoreEntry from the replacement policy in + * use. + */ + +void +storeAufsDirReplAdd(SwapDir *SD, StoreEntry *e) +{ + debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + (void) 0; + } else { + e->repl.node = heap_insert(SD->repl.heap.heap, e); + debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node); + } +#else + /* Shouldn't we not throw special objects into the lru ? */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); +#endif +} + + +void +storeAufsDirReplRemove(StoreEntry *e) +{ + SwapDir *SD = INDEXSD(e->swap_dirn); + debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + /* And now, release the object from the replacement policy */ + if (e->repl.node) { + debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n", + e->repl.node); + heap_delete(SD->repl.heap.heap, e->repl.node); + e->repl.node = NULL; + } +#else + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); +#endif +} + + +/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ + +void +storeAufsDirStats(SwapDir *SD, StoreEntry * sentry) +{ + aioinfo_t *aioinfo; +#if HAVE_STATVFS + struct statvfs sfs; +#endif + aioinfo = (aioinfo_t *)SD->fsdata; + storeAppendPrintf(sentry, "First level subdirectories: %d\n", aioinfo->l1); + storeAppendPrintf(sentry, "Second level subdirectories: %d\n", aioinfo->l2); + storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); + storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); + storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", + 100.0 * SD->cur_size / SD->max_size); + storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", + aioinfo->map->n_files_in_map, aioinfo->map->max_n_files, + percent(aioinfo->map->n_files_in_map, aioinfo->map->max_n_files)); +#if HAVE_STATVFS +#define fsbtoblk(num, fsbs, bs) \ + (((fsbs) != 0 && (fsbs) < (bs)) ? \ + (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + } +#endif + storeAppendPrintf(sentry, "Flags:"); + if (SD->flags.selected) + storeAppendPrintf(sentry, " SELECTED"); + if (SD->flags.read_only) + storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, "\n"); +#if !HEAP_REPLACEMENT + storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", + (double) storeAufsDirExpiredReferenceAge(SD) / 86400.0); +#else +#if 0 + storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", + heap_peepminkey(sd.repl.heap.heap)); +#endif +#endif +} + +/* + * storeAufsDirReconfigure + * + * This routine is called when the given swapdir needs reconfiguring + */ +void +storeAufsDirReconfigure(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + unsigned int read_only = 0; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeAufsDirReconfigure: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeAufsDirReconfigure: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeAufsDirReconfigure: invalid level 2 directories value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + /* just reconfigure it */ + if (size == sd->max_size) + debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", + path, size); + else + debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", + path, size); + sd->max_size = size; + if (sd->flags.read_only != read_only) + debug(3, 1) ("Cache dir '%s' now %s\n", + path, read_only ? "Read-Only" : "Read-Write"); + sd->flags.read_only = read_only; + return; +} + +void +storeAufsDirDump(StoreEntry * entry, const char *name, SwapDir * s) +{ + aioinfo_t *aioinfo = (aioinfo_t *)s->fsdata; + storeAppendPrintf(entry, "%s %s %s %d %d %d\n", + name, + "asyncufs", + s->path, + s->max_size >> 10, + aioinfo->l1, + aioinfo->l2); +} + +/* + * Only "free" the filesystem specific stuff here + */ +static void +storeAufsDirFree(SwapDir * s) +{ + aioinfo_t *aioinfo = (aioinfo_t *)s->fsdata; + if (aioinfo->swaplog_fd > -1) { + file_close(aioinfo->swaplog_fd); + aioinfo->swaplog_fd = -1; + } + filemapFreeMemory(aioinfo->map); + xfree(aioinfo); + s->fsdata = NULL; /* Will aid debugging... */ + +} + +char * +storeAufsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath) +{ + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + int L1 = aioinfo->l1; + int L2 = aioinfo->l2; + if (!fullpath) + fullpath = fullfilename; + fullpath[0] = '\0'; + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", + SD->path, + ((filn / L2) / L2) % L1, + (filn / L2) % L2, + filn); + return fullpath; +} + +/* + * storeAufsCleanupDoubleCheck + * + * This is called by storeCleanup() if -S was given on the command line. + */ +static int +storeAufsCleanupDoubleCheck(SwapDir *sd, StoreEntry *e) +{ + struct stat sb; + + if (stat(storeAufsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { + debug(20, 0) ("storeAufsCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", + storeAufsDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } + if (e->swap_file_sz != sb.st_size) { + debug(20, 0) ("storeAufsCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", + storeAufsDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeAufsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } + return 0; +} + +/* + * storeAufsDirParse + * + * Called when a *new* fs is being setup. + */ +void +storeAufsDirParse(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + unsigned int read_only = 0; + aioinfo_t *aioinfo; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeAufsDirParse: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeAufsDirParse: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeAufsDirParse: invalid level 2 directories value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + aioinfo = xmalloc(sizeof(aioinfo_t)); + if (aioinfo == NULL) + fatal("storeAufsDirParse: couldn't xmalloc() aioinfo_t!\n"); + + sd->index = index; + sd->path = xstrdup(path); + sd->max_size = size; + sd->fsdata = aioinfo; + aioinfo->l1 = l1; + aioinfo->l2 = l2; + aioinfo->swaplog_fd = -1; + aioinfo->map = NULL; /* Debugging purposes */ + aioinfo->suggest = 0; + sd->flags.read_only = read_only; + sd->init = storeAufsDirInit; + sd->newfs = storeAufsDirNewfs; + sd->dump = storeAufsDirDump; + sd->freefs = storeAufsDirFree; + sd->dblcheck = storeAufsCleanupDoubleCheck; + sd->statfs = storeAufsDirStats; + sd->maintainfs = storeAufsDirMaintain; + sd->checkobj = storeAufsDirCheckObj; + sd->refobj = storeAufsDirRefObj; + sd->unrefobj = storeAufsDirUnrefObj; + sd->callback = aioCheckCallbacks; + sd->sync = aioSync; + sd->obj.create = storeAufsCreate; + sd->obj.open = storeAufsOpen; + sd->obj.close = storeAufsClose; + sd->obj.read = storeAufsRead; + sd->obj.write = storeAufsWrite; + sd->obj.unlink = storeAufsUnlink; + sd->log.open = storeAufsDirOpenSwapLog; + sd->log.close = storeAufsDirCloseSwapLog; + sd->log.write = storeAufsDirSwapLog; + sd->log.clean.open = storeAufsDirWriteCleanOpen; + + /* Initialise replacement policy stuff */ +#if HEAP_REPLACEMENT + /* + * Create new heaps with cache replacement policies attached to them. + * The cache replacement policy is specified as either GDSF or LFUDA in + * the squid.conf configuration file. Note that the replacement policy + * applies only to the disk replacement algorithm. Memory replacement + * always uses GDSF since we want to maximize object hit rate. + */ + if (Config.replPolicy) { + if (tolower(Config.replPolicy[0]) == 'g') { + debug(20, 1) ("Using GDSF disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } else if (tolower(Config.replPolicy[0]) == 'l') { + if (tolower(Config.replPolicy[1]) == 'f') { + debug(20, 1) ("Using LFUDA disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); + } else if (tolower(Config.replPolicy[1]) == 'r') { + debug(20, 1) ("Using LRU heap disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); + } + } else { + debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } + } else { + debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } +#else + sd->repl.lru.list.head = NULL; + sd->repl.lru.list.tail = NULL; +#endif +} + +void +storeFsSetup_aufs(storefs_entry_t *storefs) +{ + storefs->parsefunc = storeAufsDirParse; + storefs->reconfigurefunc = storeAufsDirReconfigure; + storefs->donefunc = aioDone; + aioInit(); +} --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/aufs/store_io_aufs.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,381 @@ + +/* + * DEBUG 78 + */ + +#include "squid.h" +#include "store_asyncufs.h" + +static AIOCB storeAufsReadDone; +static AIOCB storeAufsWriteDone; +static void storeAufsIOCallback(storeIOState * sio, int errflag); +static AIOCB storeAufsOpenDone; +static int storeAufsSomethingPending(storeIOState *); +static int storeAufsKickWriteQueue(storeIOState * sio); +static void storeAufsIOFreeEntry(void *, int); + +struct _queued_write { + char *buf; + size_t size; + off_t offset; + FREE *free_func; +}; + +struct _queued_read { + char *buf; + size_t size; + off_t offset; + STRCB *callback; + void *callback_data; +}; + +/* === PUBLIC =========================================================== */ + +/* open for reading */ +storeIOState * +storeAufsOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) +{ + sfileno f = e->swap_filen; + char *path = storeAufsDirFullPath(SD, f, NULL); + storeIOState *sio; + debug(78, 3) ("storeAufsOpen: fileno %08X\n", f); + /* + * we should detect some 'too many files open' condition and return + * NULL here. + */ + while (aioQueueSize() > MAGIC1) + return NULL; + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeAufsIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(aio_state_pool); + ((aiostate_t *)(sio->fsstate))->fd = -1; + ((aiostate_t *)(sio->fsstate))->flags.opening = 1; + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->mode = O_RDONLY; + sio->callback = callback; + sio->callback_data = callback_data; + sio->e = e; + cbdataLock(callback_data); + aioOpen(path, O_RDONLY, 0644, storeAufsOpenDone, sio); + Opening_FD++; + store_open_disk_fd++; + return sio; +} + +/* open for creating */ +storeIOState * +storeAufsCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data) +{ + char *path ; + storeIOState *sio; + sfileno filn; + sdirno dirn; + + /* Allocate a number */ + dirn = SD->index; + filn = storeAufsDirMapBitAllocate(SD); + path = storeAufsDirFullPath(SD, filn, NULL); + + debug(78, 3) ("storeAufsCreate: fileno %08X\n", filn); + /* + * we should detect some 'too many files open' condition and return + * NULL here. + */ + while (aioQueueSize() > MAGIC1) + return NULL; + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeAufsIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(aio_state_pool); + ((aiostate_t *)(sio->fsstate))->fd = -1; + ((aiostate_t *)(sio->fsstate))->flags.opening = 1; + sio->swap_filen = filn; + sio->swap_dirn = dirn; + sio->mode = O_WRONLY; + sio->callback = callback; + sio->callback_data = callback_data; + sio->e = (StoreEntry *) e; + cbdataLock(callback_data); + aioOpen(path, O_WRONLY | O_CREAT | O_TRUNC, 0644, storeAufsOpenDone, sio); + Opening_FD++; + store_open_disk_fd++; + + /* now insert into the replacement policy */ + storeAufsDirReplAdd(SD, e); + return sio; + +} + + + +/* Close */ +void +storeAufsClose(SwapDir *SD, storeIOState * sio) +{ + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + debug(78, 3) ("storeAufsClose: dirno %d, fileno %08X, FD %d\n", + sio->swap_dirn, sio->swap_filen, aiostate->fd); + if (storeAufsSomethingPending(sio)) { + aiostate->flags.close_request = 1; + return; + } + storeAufsIOCallback(sio, 0); +} + + +/* Read */ +void +storeAufsRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +{ + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + assert(sio->read.callback == NULL); + assert(sio->read.callback_data == NULL); + assert(!aiostate->flags.reading); + if (aiostate->fd < 0) { + struct _queued_read *q; + debug(78, 3) ("storeAufsRead: queueing read because FD < 0\n"); + assert(aiostate->flags.opening); + assert(aiostate->pending_reads == NULL); + q = xcalloc(1, sizeof(*q)); + q->buf = buf; + q->size = size; + q->offset = offset; + q->callback = callback; + q->callback_data = callback_data; + linklistPush(&(aiostate->pending_reads), q); + return; + } + sio->read.callback = callback; + sio->read.callback_data = callback_data; + aiostate->read_buf = buf; + cbdataLock(callback_data); + debug(78, 3) ("storeAufsRead: dirno %d, fileno %08X, FD %d\n", + sio->swap_dirn, sio->swap_filen, aiostate->fd); + sio->offset = offset; + aiostate->flags.reading = 1; + aioRead(aiostate->fd, offset, buf, size, storeAufsReadDone, sio); +} + + +/* Write */ +void +storeAufsWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +{ + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + debug(78, 3) ("storeAufsWrite: dirno %d, fileno %08X, FD %d\n", + sio->swap_dirn, sio->swap_filen, aiostate->fd); + if (aiostate->fd < 0) { + /* disk file not opened yet */ + struct _queued_write *q; + assert(aiostate->flags.opening); + q = xcalloc(1, sizeof(*q)); + q->buf = buf; + q->size = size; + q->offset = offset; + q->free_func = free_func; + linklistPush(&(aiostate->pending_writes), q); + return; + } + if (aiostate->flags.writing) { + struct _queued_write *q; + debug(78, 3) ("storeAufsWrite: queuing write\n"); + q = xcalloc(1, sizeof(*q)); + q->buf = buf; + q->size = size; + q->offset = offset; + q->free_func = free_func; + linklistPush(&(aiostate->pending_writes), q); + return; + } + aiostate->flags.writing = 1; + /* + * XXX it might be nice if aioWrite() gave is immediate + * feedback here about EWOULDBLOCK instead of in the + * callback function + */ + aioWrite(aiostate->fd, offset, buf, size, storeAufsWriteDone, sio, + free_func); +} + +/* Unlink */ +void +storeAufsUnlink(SwapDir *SD, StoreEntry *e) +{ + debug(78, 3) ("storeAufsUnlink: dirno %d, fileno %08X\n", SD->index, e->swap_filen); + storeAufsDirReplRemove(e); + storeAufsDirUnlinkFile(SD, e->swap_filen); +} + +/* === STATIC =========================================================== */ + +static int +storeAufsKickWriteQueue(storeIOState * sio) +{ + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + struct _queued_write *q = linklistShift(&aiostate->pending_writes); + if (NULL == q) + return 0; + debug(78, 3) ("storeAufsKickWriteQueue: writing queued chunk of %d bytes\n", + q->size); + storeAufsWrite(INDEXSD(sio->swap_dirn), sio, q->buf, q->size, q->offset, q->free_func); + xfree(q); + return 1; +} + +static int +storeAufsKickReadQueue(storeIOState * sio) +{ + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + struct _queued_read *q = linklistShift(&(aiostate->pending_reads)); + if (NULL == q) + return 0; + debug(78, 3) ("storeAufsKickReadQueue: reading queued request of %d bytes\n", + q->size); + storeAufsRead(INDEXSD(sio->swap_dirn), sio, q->buf, q->size, q->offset, q->callback, q->callback_data); + xfree(q); + return 1; +} + +static void +storeAufsOpenDone(int unused, void *my_data, int fd, int errflag) +{ + storeIOState *sio = my_data; + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + debug(78, 3) ("storeAufsOpenDone: FD %d, errflag %d\n", fd, errflag); + Opening_FD--; + aiostate->flags.opening = 0; + if (errflag || fd < 0) { + errno = errflag; + debug(78, 0) ("storeAufsOpenDone: %s\n", xstrerror()); + debug(78, 1) ("\t%s\n", storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL)); + storeAufsIOCallback(sio, errflag); + return; + } + aiostate->fd = fd; + commSetCloseOnExec(fd); + fd_open(fd, FD_FILE, storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL)); + if (sio->mode == O_WRONLY) + storeAufsKickWriteQueue(sio); + else if (sio->mode == O_RDONLY) + storeAufsKickReadQueue(sio); + debug(78, 3) ("storeAufsOpenDone: exiting\n"); +} + +static void +storeAufsReadDone(int fd, void *my_data, int len, int errflag) +{ + storeIOState *sio = my_data; + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + STRCB *callback = sio->read.callback; + void *their_data = sio->read.callback_data; + ssize_t rlen; + debug(78, 3) ("storeAufsReadDone: dirno %d, fileno %08X, FD %d, len %d\n", + sio->swap_dirn, sio->swap_filen, fd, len); + aiostate->flags.reading = 0; + if (errflag) { + debug(78, 3) ("storeAufsReadDone: got failure (%d)\n", errflag); + rlen = -1; + } else { + rlen = (ssize_t) len; + sio->offset += len; + } + assert(callback); + assert(their_data); + sio->read.callback = NULL; + sio->read.callback_data = NULL; + if (cbdataValid(their_data)) + callback(their_data, aiostate->read_buf, rlen); + cbdataUnlock(their_data); + /* + * XXX is this safe? The above callback may have caused sio + * to be freed/closed already? Philip Guenther + * says it fixes his FD leaks, with no side effects. + */ + if (aiostate->flags.close_request) + storeAufsIOCallback(sio, DISK_OK); +} + +/* + * XXX TODO + * if errflag == EWOULDBLOCK, then we'll need to re-queue the + * chunk at the beginning of the write_pending list and try + * again later. + */ +static void +storeAufsWriteDone(int fd, void *my_data, int len, int errflag) +{ + static int loop_detect = 0; + storeIOState *sio = my_data; + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + debug(78, 3) ("storeAufsWriteDone: dirno %d, fileno %08X, FD %d, len %d\n", + sio->swap_dirn, sio->swap_filen, fd, len); + assert(++loop_detect < 10); + aiostate->flags.writing = 0; + if (errflag) { + debug(78, 0) ("storeAufsWriteDone: got failure (%d)\n", errflag); + storeAufsIOCallback(sio, errflag); + loop_detect--; + return; + } + sio->offset += len; + if (storeAufsKickWriteQueue(sio)) + (void) 0; + else if (aiostate->flags.close_request) + storeAufsIOCallback(sio, errflag); + loop_detect--; +} + +static void +storeAufsIOCallback(storeIOState * sio, int errflag) +{ + STIOCB *callback = sio->callback; + void *their_data = sio->callback_data; + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + int fd = aiostate->fd; + debug(78, 3) ("storeAufsIOCallback: errflag=%d\n", errflag); + sio->callback = NULL; + sio->callback_data = NULL; + debug(78, 3) ("%s:%d\n", __FILE__, __LINE__); + if (callback) + if (NULL == their_data || cbdataValid(their_data)) + callback(their_data, errflag, sio); + debug(78, 3) ("%s:%d\n", __FILE__, __LINE__); + cbdataUnlock(their_data); + aiostate->fd = -1; + cbdataFree(sio); + if (fd < 0) + return; + debug(78, 3) ("%s:%d\n", __FILE__, __LINE__); + aioClose(fd); + fd_close(fd); + store_open_disk_fd--; + debug(78, 3) ("%s:%d\n", __FILE__, __LINE__); +} + + +static int +storeAufsSomethingPending(storeIOState * sio) +{ + aiostate_t *aiostate = (aiostate_t *)sio->fsstate; + if (aiostate->flags.reading) + return 1; + if (aiostate->flags.writing) + return 1; + if (aiostate->flags.opening) + return 1; + return 0; +} + + +/* + * We can't pass memFree() as a free function here, because we need to free + * the fsstate variable .. + */ +static void +storeAufsIOFreeEntry(void *sio, int foo) +{ + memPoolFree(aio_state_pool, ((storeIOState *)sio)->fsstate); + memFree(sio, MEM_STORE_IO); +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/coss/Makefile.in Wed Feb 14 00:43:59 2007 @@ -0,0 +1,56 @@ +# +# Makefile for the COSS storage driver for the Squid Object Cache server +# +# $Id$ +# + +FS = coss + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(FS).a + +OBJS = \ + store_dir_coss.o \ + store_io_coss.o + + +all: $(OUT) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +install: + +tags: + ctags *.[ch] ../../*.[ch] ../../../include/*.h ../lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/coss/store_coss.h Wed Feb 14 00:43:59 2007 @@ -0,0 +1,82 @@ +#ifndef __COSS_H__ +#define __COSS_H__ + +#ifndef COSS_MEMBUF_SZ +#define COSS_MEMBUF_SZ 1048576 +#endif + +#define COSS_ALLOC_NOTIFY 0 +#define COSS_ALLOC_ALLOCATE 1 +#define COSS_ALLOC_REALLOC 2 + +struct _cossmembuf { + size_t diskstart; + size_t diskend; + SwapDir *SD; + int lockcount; + char buffer[COSS_MEMBUF_SZ]; + struct _cossmembuf_flags { + unsigned int full:1; + unsigned int writing:1; + } flags; + struct _cossmembuf *next; +}; + + +/* Per-storedir info */ +struct _cossinfo { + struct _cossmembuf *membufs; + struct _cossmembuf *current_membuf; + size_t current_offset; + int fd; + int swaplog_fd; + int numcollisions; +}; + + +/* Per-storeiostate info */ +struct _cossstate { + char *readbuffer; + char *requestbuf; + size_t requestlen; + size_t requestoffset; + sfileno reqdiskoffset; + struct { + unsigned int reading:1; + unsigned int writing:1; + } flags; +}; + +typedef struct _cossmembuf CossMemBuf; +typedef struct _cossinfo CossInfo; +typedef struct _cossstate CossState; + +/* Whether the coss system has been setup or not */ +extern int coss_initialised; +extern MemPool * coss_membuf_pool; +extern MemPool * coss_state_pool; + + +/* + * Store IO stuff + */ +extern STOBJCREATE storeCossCreate; +extern STOBJOPEN storeCossOpen; +extern STOBJCLOSE storeCossClose; +extern STOBJREAD storeCossRead; +extern STOBJWRITE storeCossWrite; +extern STOBJUNLINK storeCossUnlink; + +extern off_t storeCossAllocate(SwapDir *SD, const StoreEntry *e, int which); +extern void storeCossFree(StoreEntry *e); +extern void storeCossMaintainSwapSpace(SwapDir *SD); +extern void storeCossDirStats(SwapDir *, StoreEntry *); +extern void storeCossDirDump(StoreEntry * entry, const char *name, SwapDir* s); +extern void storeCossDirFree(SwapDir *); +extern SwapDir *storeCossDirPick(void); + +void storeFsSetup_ufs(storefs_entry_t *); + + + +#endif --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/coss/store_dir_coss.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,846 @@ + +/* + * $Id$ + * + * DEBUG: section 81 Store COSS Directory Routines + * AUTHOR: Eric Stern + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +#include "store_coss.h" + +#define STORE_META_BUFSZ 4096 + +extern void storeCossFlushMemBufs(SwapDir *SD); + +int n_coss_dirs = 0; +/* static int last_coss_pick_index = -1; */ +int coss_initialised = 0; +MemPool * coss_membuf_pool = NULL; +MemPool * coss_state_pool = NULL; + +typedef struct _RebuildState RebuildState; +struct _RebuildState { + SwapDir *sd; + int n_read; + FILE *log; + int speed; + int curlvl1; + int curlvl2; + struct { + unsigned int need_to_validate:1; + unsigned int clean:1; + unsigned int init:1; + } flags; + int done; + int in_dir; + int fn; + struct dirent *entry; + DIR *td; + char fullpath[SQUID_MAXPATHLEN]; + char fullfilename[SQUID_MAXPATHLEN]; + struct _store_rebuild_data counts; +}; + +static char *storeCossDirSwapLogFile(SwapDir *, const char *); +static EVH storeCossRebuildFromSwapLog; +static StoreEntry *storeCossAddDiskRestore(SwapDir *SD,const cache_key *key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean); +static void storeCossDirRebuild(SwapDir * sd); +static void storeCossDirCloseTmpSwapLog(SwapDir * sd); +static FILE *storeCossDirOpenTmpSwapLog(SwapDir *, int *, int *); +static STLOGOPEN storeCossDirOpenSwapLog; +static STINIT storeCossDirInit; +static STLOGCLEANOPEN storeCossDirWriteCleanOpen; +static void storeCossDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANWRITE storeCossDirWriteCleanEntry; +static STLOGCLOSE storeCossDirCloseSwapLog; +static STLOGWRITE storeCossDirSwapLog; +static STNEWFS storeCossDirNewfs; +static STCHECKOBJ storeCossDirCheckObj; +static void storeCossDirShutdown(SwapDir * sd); +static void storeCossDirParse(SwapDir *, int, char *); +static void storeCossDirReconfigure(SwapDir *, int, char *); + +static char * +storeCossDirSwapLogFile(SwapDir * sd, const char *ext) +{ + LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; + if (Config.Log.swap) { + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } + } else { + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); + } + if (ext) + strncat(path, ext, 16); + return path; +} + +static void +storeCossDirOpenSwapLog(SwapDir * sd) +{ + CossInfo *cs = (CossInfo *)sd->fsdata; + char *path; + int fd; + path = storeCossDirSwapLogFile(sd, NULL); + fd = file_open(path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(81, 1) ("%s: %s\n", path, xstrerror()); + fatal("storeCossDirOpenSwapLog: Failed to open swap log."); + } + debug(81, 3) ("Cache COSS Dir #%d log opened on FD %d\n", sd->index, fd); + cs->swaplog_fd = fd; +} + +static void +storeCossDirCloseSwapLog(SwapDir * sd) +{ + CossInfo *cs = (CossInfo *)sd->fsdata; + if (cs->swaplog_fd < 0) /* not open */ + return; + file_close(cs->swaplog_fd); + debug(81, 3) ("Cache COSS Dir #%d log closed on FD %d\n", + sd->index, cs->swaplog_fd); + cs->swaplog_fd = -1; +} + +static void +storeCossDirInit(SwapDir * sd) +{ + CossInfo *cs = (CossInfo *)sd->fsdata; + storeCossDirOpenSwapLog(sd); + storeCossDirRebuild(sd); + cs->fd = file_open(sd->path, O_RDWR | O_CREAT); + n_coss_dirs++; +} + +static void +storeCossRebuildFromSwapLog(void *data) +{ + RebuildState *rb = data; + StoreEntry *e = NULL; + storeSwapLogData s; + size_t ss = sizeof(storeSwapLogData); + int count; + double x; + assert(rb != NULL); + /* load a number of objects per invocation */ + for (count = 0; count < rb->speed; count++) { + if (fread(&s, ss, 1, rb->log) != 1) { + debug(81, 1) ("Done reading %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + fclose(rb->log); + rb->log = NULL; + store_dirs_rebuilding--; + storeCossDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } + rb->n_read++; + if (s.op <= SWAP_LOG_NOP) + continue; + if (s.op >= SWAP_LOG_MAX) + continue; + debug(20, 3) ("storeCossRebuildFromSwapLog: %s %s %08X\n", + swap_log_op_str[(int) s.op], + storeKeyText(s.key), + s.swap_filen); + if (s.op == SWAP_LOG_ADD) { + (void) 0; + } else if (s.op == SWAP_LOG_DEL) { + if ((e = storeGet(s.key)) != NULL) { + /* + * Make sure we don't unlink the file, it might be + * in use by a subsequent entry. Also note that + * we don't have to subtract from store_swap_size + * because adding to store_swap_size happens in + * the cleanup procedure. + */ + storeExpireNow(e); + storeReleaseRequest(e); + if (e->swap_filen > -1) { + e->swap_filen = -1; + } + storeRelease(e); + /* Fake an unlink here, this is a bad hack :( */ + dlinkDelete(&e->repl.lru, &rb->sd->repl.lru.list); + rb->counts.objcount--; + rb->counts.cancelcount++; + } + continue; + } else { + x = log(++rb->counts.bad_log_op) / log(10.0); + if (0.0 == x - (double) (int) x) + debug(20, 1) ("WARNING: %d invalid swap log entries found\n", + rb->counts.bad_log_op); + rb->counts.invalid++; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %7d %s Entries read so far.\n", + rb->counts.scancount, rb->sd->path); + if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + rb->counts.badflags++; + continue; + } + e = storeGet(s.key); + if (e) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } + /* update store_swap_size */ + rb->counts.objcount++; + e = storeCossAddDiskRestore(rb->sd,s.key, + s.swap_filen, + s.swap_file_sz, + s.expires, + s.timestamp, + s.lastref, + s.lastmod, + s.refcount, + s.flags, + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeCossRebuild", storeCossRebuildFromSwapLog, rb, 0.0, 1); +} + +/* Add a new object to the cache with empty memory copy and pointer to disk + * use to rebuild store from disk. */ +static StoreEntry * +storeCossAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean) +{ + StoreEntry *e = NULL; + debug(20, 5) ("storeCossAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); + /* if you call this you'd better be sure file_number is not + * already in use! */ + e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + e->store_status = STORE_OK; + e->swap_dirn = SD->index; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; + e->swap_filen = file_number; + e->swap_file_sz = swap_file_sz; + e->lock_count = 0; +#if !HEAP_REPLACEMENT + e->refcount = 0; +#endif + e->lastref = lastref; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->refcount = refcount; + e->flags = flags; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + e->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY); + return e; +} + +static void +storeCossDirRebuild(SwapDir * sd) +{ + RebuildState *rb = xcalloc(1, sizeof(*rb)); + int clean = 0; + int zero = 0; + FILE *fp; + EVH *func = NULL; + rb->sd = sd; + rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; + /* + * If the swap.state file exists in the cache_dir, then + * we'll use storeCossRebuildFromSwapLog(). + */ + fp = storeCossDirOpenTmpSwapLog(sd, &clean, &zero); + debug(20, 1) ("Rebuilding COSS storage in %s (%s)\n", + sd->path, clean ? "CLEAN" : "DIRTY"); + if (!clean || fp == NULL) { + /* COSS cannot yet rebuild from a dirty state. If the log + * is dirty then the COSS contents is thrown away. + * Why? I guess it is because some contents will be lost, + * and COSS cannot verify this.. + */ + if (fp != NULL) + fclose(fp); + storeCossDirCloseTmpSwapLog(rb->sd); + /* + * XXX Make sure we don't trigger an assertion if this is the first + * storedir, since if we are, this call will cause storeRebuildComplete + * to prematurely complete the rebuild process, and then some other + * storedir will try to rebuild and eventually die. + */ + store_dirs_rebuilding++; + storeRebuildComplete(&rb->counts); + store_dirs_rebuilding--; + xfree(rb); + return; + } + func = storeCossRebuildFromSwapLog; + rb->log = fp; + rb->flags.clean = (unsigned int) clean; + store_dirs_rebuilding++; + cbdataAdd(rb, cbdataXfree, 0); + eventAdd("storeCossRebuild", func, rb, 0.0, 1); +} + +static void +storeCossDirCloseTmpSwapLog(SwapDir * sd) +{ + CossInfo *cs = (CossInfo *)sd->fsdata; + char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); + char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); + int fd; + file_close(cs->swaplog_fd); +#ifdef _SQUID_OS2_ + if (unlink(swaplog_path) < 0) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeCossDirCloseTmpSwapLog: unlink failed"); + } +#endif + if (xrename(new_path, swaplog_path) < 0) { + fatal("storeCossDirCloseTmpSwapLog: rename failed"); + } + fd = file_open(swaplog_path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeCossDirCloseTmpSwapLog: Failed to open swap log."); + } + safe_free(swaplog_path); + safe_free(new_path); + cs->swaplog_fd = fd; + debug(47, 3) ("Cache COSS Dir #%d log opened on FD %d\n", sd->index, fd); +} + +static FILE * +storeCossDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) +{ + CossInfo *cs = (CossInfo *)sd->fsdata; + char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); + char *clean_path = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean")); + char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); + struct stat log_sb; + struct stat clean_sb; + FILE *fp; + int fd; + if (stat(swaplog_path, &log_sb) < 0) { + debug(47, 1) ("Cache COSS Dir #%d: No log file\n", sd->index); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return NULL; + } + *zero_flag = log_sb.st_size == 0 ? 1 : 0; + /* close the existing write-only FD */ + if (cs->swaplog_fd >= 0) + file_close(cs->swaplog_fd); + /* open a write-only FD for the new log */ + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", new_path, xstrerror()); + fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); + } + cs->swaplog_fd = fd; + /* open a read-only stream of the old log */ + fp = fopen(swaplog_path, "r"); + if (fp == NULL) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("Failed to open swap log for reading"); + } + memset(&clean_sb, '\0', sizeof(struct stat)); + if (stat(clean_path, &clean_sb) < 0) + *clean_flag = 0; + else if (clean_sb.st_mtime < log_sb.st_mtime) + *clean_flag = 0; + else + *clean_flag = 1; + safeunlink(clean_path, 1); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return fp; +} + +struct _clean_state { + char *cur; + char *new; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; +}; + +#define CLEAN_BUF_SZ 16384 +/* + * Begin the process to write clean cache state. For COSS this means + * opening some log files and allocating write buffers. Return 0 if + * we succeed, and assign the 'func' and 'data' return pointers. + */ +static int +storeCossDirWriteCleanOpen(SwapDir * sd) +{ + struct _clean_state *state = xcalloc(1, sizeof(*state)); + struct stat sb; + sd->log.clean.write = NULL; + sd->log.clean.state = NULL; + state->cur = xstrdup(storeCossDirSwapLogFile(sd, NULL)); + state->new = xstrdup(storeCossDirSwapLogFile(sd, ".clean")); + state->cln = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean")); + state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); + state->outbuf_offset = 0; + unlink(state->new); + unlink(state->cln); + state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); + if (state->fd < 0) + return -1; + debug(20, 3) ("storeCOssDirWriteCleanLogs: opened %s, FD %d\n", + state->new, state->fd); +#if HAVE_FCHMOD + if (stat(state->cur, &sb) == 0) + fchmod(state->fd, sb.st_mode); +#endif + sd->log.clean.write = storeCossDirWriteCleanEntry; + sd->log.clean.state = state; + return 0; +} + +/* + * "write" an entry to the clean log file. + */ +static void +storeCossDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +{ + storeSwapLogData s; + static size_t ss = sizeof(storeSwapLogData); + struct _clean_state *state = sd->log.clean.state; + if (NULL == e) { + storeCossDirWriteCleanClose(sd); + return; + } + memset(&s, '\0', ss); + s.op = (char) SWAP_LOG_ADD; + s.swap_filen = e->swap_filen; + s.timestamp = e->timestamp; + s.lastref = e->lastref; + s.expires = e->expires; + s.lastmod = e->lastmod; + s.swap_file_sz = e->swap_file_sz; + s.refcount = e->refcount; + s.flags = e->flags; + xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); + xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); + state->outbuf_offset += ss; + /* buffered write */ + if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeCossDirWriteCleanLogs: Current swap logfile not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; + } + state->outbuf_offset = 0; + } +} + +static void +storeCossDirWriteCleanClose(SwapDir * sd) +{ + struct _clean_state *state = sd->log.clean.state; + if (state->fd < 0) + return; + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeCossDirWriteCleanLogs: Current swap logfile " + "not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + } + safe_free(state->outbuf); + /* + * You can't rename open files on Microsoft "operating systems" + * so we have to close before renaming. + */ + storeCossDirCloseSwapLog(sd); + /* rename */ + if (state->fd >= 0) { +#ifdef _SQUID_OS2_ + file_close(state->fd); + state->fd = -1; + if (unlink(cur) < 0) + debug(50, 0) ("storeCossDirWriteCleanLogs: unlinkd failed: %s, %s\n", + xstrerror(), cur); +#endif + xrename(state->new, state->cur); + } + /* touch a timestamp file if we're not still validating */ + if (store_dirs_rebuilding) + (void) 0; + else if (state->fd < 0) + (void) 0; + else + file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC)); + /* close */ + safe_free(state->cur); + safe_free(state->new); + safe_free(state->cln); + if (state->fd >= 0) + file_close(state->fd); + state->fd = -1; + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; +} + +static void +storeCossDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) +{ + CossInfo *cs = (CossInfo *)sd->fsdata; + storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); + s->op = (char) op; + s->swap_filen = e->swap_filen; + s->timestamp = e->timestamp; + s->lastref = e->lastref; + s->expires = e->expires; + s->lastmod = e->lastmod; + s->swap_file_sz = e->swap_file_sz; + s->refcount = e->refcount; + s->flags = e->flags; + xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); + file_write(cs->swaplog_fd, + -1, + s, + sizeof(storeSwapLogData), + NULL, + NULL, + xfree); +} + +static void +storeCossDirNewfs(SwapDir * sd) +{ + debug(47, 3) ("Creating swap space in %s\n", sd->path); +} + +// we are shutting down, flush all membufs to disk +static void +storeCossDirShutdown(SwapDir *SD) +{ + CossInfo *cs = (CossInfo *)SD->fsdata; + CossMemBuf *t; + + // we need to do this synchronously! + for (t = cs->membufs; t; t = t->next) { + lseek(cs->fd, t->diskstart, SEEK_SET); + write(cs->fd, t->buffer, COSS_MEMBUF_SZ); + } + file_close(cs->fd); + cs->fd = -1; + + if (cs->swaplog_fd > -1) { + file_close(cs->swaplog_fd); + cs->swaplog_fd = -1; + } + + n_coss_dirs--; +} + +/* + * storeCossDirCheckObj + * + * This routine is called by storeDirSelectSwapDir to see if the given + * object is able to be stored on this filesystem. COSS filesystems will + * not store everything. We don't check for maxobjsize here since its + * done by the upper layers. + */ +int +storeCossDirCheckObj(SwapDir *SD, const StoreEntry *e) +{ + /* Check if the object is a special object, we can't cache these */ + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) + return -1; + + /* Otherwise, we're ok */ + /* Return 900 (90%) load */ + return 900; +} + +/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ + +void +storeCossDirStats(SwapDir *SD, StoreEntry * sentry) +{ + CossInfo *cs = (CossInfo *)SD->fsdata; + + storeAppendPrintf(sentry, "\n"); + storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); + storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); + storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", + 100.0 * SD->cur_size / SD->max_size); + storeAppendPrintf(sentry, "Number of object collisions: %d\n", (int)cs->numcollisions); +#if 0 + /* is this applicable? I Hope not .. */ + storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", + SD->map->n_files_in_map, SD->map->max_n_files, + percent(SD->map->n_files_in_map, SD->map->max_n_files)); +#endif + storeAppendPrintf(sentry, "Flags:"); + if (SD->flags.selected) + storeAppendPrintf(sentry, " SELECTED"); + if (SD->flags.read_only) + storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, "\n"); +} + +static void +storeCossDirParse(SwapDir *sd, int index, char *path) +{ + char *token; + unsigned int i; + unsigned int size; + unsigned int read_only = 0; + CossInfo *cs; + + i = GetInteger(); + size = i << 10; /* Mbytes to Kbytes */ + if (size <= 0) + fatal("storeCossDirParse: invalid size value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + cs = xmalloc(sizeof(CossInfo)); + if (cs == NULL) + fatal("storeCossDirParse: couldn't xmalloc() CossInfo!\n"); + + sd->index = index; + sd->path = xstrdup(path); + sd->max_size = size; + sd->fsdata = cs; + + cs->fd = -1; + cs->swaplog_fd = -1; + sd->flags.read_only = read_only; + + sd->init = storeCossDirInit; + sd->newfs = storeCossDirNewfs; + sd->dump = storeCossDirDump; + sd->freefs = storeCossDirShutdown; + sd->dblcheck = NULL; + sd->statfs = storeCossDirStats; + sd->maintainfs = NULL; + sd->checkobj = storeCossDirCheckObj; + sd->refobj = NULL; /* LRU is done in storeCossRead */ + sd->unrefobj = NULL; + sd->callback = NULL; + sd->sync = NULL; /* should we make it call the coss sync? */ + + sd->obj.create = storeCossCreate; + sd->obj.open = storeCossOpen; + sd->obj.close = storeCossClose; + sd->obj.read = storeCossRead; + sd->obj.write = storeCossWrite; + sd->obj.unlink = storeCossUnlink; + + sd->log.open = storeCossDirOpenSwapLog; + sd->log.close = storeCossDirCloseSwapLog; + sd->log.write = storeCossDirSwapLog; + sd->log.clean.open = storeCossDirWriteCleanOpen; + sd->log.clean.write = storeCossDirWriteCleanEntry; + + cs->current_offset = 0; + cs->fd = -1; + cs->swaplog_fd = -1; + cs->numcollisions = 0; + cs->membufs = memPoolAlloc(coss_membuf_pool); + cs->membufs->diskstart = 0; + cs->membufs->diskend = COSS_MEMBUF_SZ; + cs->membufs->lockcount = 0; + cs->membufs->flags.full = 0; + cs->membufs->flags.writing = 0; + cs->membufs->next = NULL; + cs->membufs->SD = sd; + cs->current_membuf = cs->membufs; + + sd->repl.lru.list.head = NULL; + sd->repl.lru.list.tail = NULL; +} + + +static void +storeCossDirReconfigure(SwapDir *sd, int index, char *path) +{ + char *token; + unsigned int i; + unsigned int size; + unsigned int read_only = 0; + + i = GetInteger(); + size = i << 10; /* Mbytes to Kbytes */ + if (size <= 0) + fatal("storeCossDirParse: invalid size value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + if (size == sd->max_size) + debug (3, 1) ("Cache COSS dir '%s' size remains unchanged at %d KB\n", path, size); + else { + debug (3, 1) ("Cache COSS dir '%s' size changed to %d KB\n", path, size); + sd->max_size = size; + } + + if (read_only != sd->flags.read_only) { + debug (3, 1) ("Cache COSS dir '%s' now %s\n", path, read_only ? "Read-Only" : "Read-Write"); + sd->flags.read_only = read_only; + } +} + +void +storeCossDirDump(StoreEntry * entry, const char *name, SwapDir * s) +{ + storeAppendPrintf(entry, "%s %s %s %d\n", + name, + s->type, + s->path, + s->max_size >> 20); +} + +#if 0 +SwapDir * +storeCossDirPick(void) +{ + int i,choosenext = 0; + SwapDir *SD; + + if (n_coss_dirs == 0) + return NULL; + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->type == SWAPDIR_COSS) { + if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { + last_coss_pick_index = i; + return SD; + } else if (choosenext) { + last_coss_pick_index = i; + return SD; + } else if (last_coss_pick_index == i) { + choosenext = 1; + } + } + } + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->type == SWAPDIR_COSS) { + if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { + last_coss_pick_index = i; + return SD; + } else if (choosenext) { + last_coss_pick_index = i; + return SD; + } else if (last_coss_pick_index == i) { + choosenext = 1; + } + } + } + return NULL; +} +#endif + +/* + * initial setup/done code + */ +static void +storeCossDirDone(void) +{ + memPoolDestroy(coss_membuf_pool); + memPoolDestroy(coss_state_pool); + coss_initialised = 0; +} + +void +storeFsSetup_coss(storefs_entry_t *storefs) +{ + assert(!coss_initialised); + + storefs->parsefunc = storeCossDirParse; + storefs->reconfigurefunc = storeCossDirReconfigure; + storefs->donefunc = storeCossDirDone; + coss_membuf_pool = memPoolCreate("COSS Membuf data", sizeof(CossMemBuf)); + coss_state_pool = memPoolCreate("COSS IO State data", sizeof(CossState)); + coss_initialised = 1; +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/coss/store_io_coss.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,518 @@ + +/* + * $Id$ + * + * DEBUG: section 81 Storage Manager COSS Interface + * AUTHOR: Eric Stern + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "store_coss.h" + +static DWCB storeCossWriteMemBufDone; +static DRCB storeCossReadDone; +static void storeCossIOCallback(storeIOState * sio, int errflag); +static char *storeCossMemPointerFromDiskOffset(SwapDir *SD, size_t offset, CossMemBuf **mb); +static void storeCossMemBufLock(SwapDir *SD, storeIOState *e); +static void storeCossMemBufUnlock(SwapDir *SD, storeIOState *e); +static void storeCossWriteMemBuf(SwapDir *SD,CossMemBuf *t); +static void storeCossWriteMemBufDone(int fd, int errflag, size_t len, void *my_data); +static CossMemBuf *storeCossCreateMemBuf(SwapDir *SD, size_t start, + sfileno curfn, int *collision); +static void storeCossIOFreeEntry(void *, int); +static void storeCossMembufFree(void *, int); + +/* === PUBLIC =========================================================== */ + +/* + * This routine sucks. I want to rewrite it when possible, and I also think + * that we should check after creatmembuf() to see if the object has a + * RELEASE_REQUEST set on it (thanks Eric!) rather than this way which seems + * to work.. + * -- Adrian + */ +off_t +storeCossAllocate(SwapDir *SD, const StoreEntry *e, int which) +{ + CossInfo *cs = (CossInfo *)SD->fsdata; + CossMemBuf *newmb; + off_t retofs; + size_t allocsize; + int coll = 0; + sfileno checkf; + + /* Make sure we chcek collisions if reallocating */ + if (which == COSS_ALLOC_REALLOC) + checkf = e->swap_filen; + else + checkf = -1; + + retofs = e->swap_filen; /* Just for defaults */ + + if (e->swap_file_sz > 0) + allocsize = e->swap_file_sz; + else + allocsize = objectLen(e) + e->mem_obj->swap_hdr_sz; + + if (which != COSS_ALLOC_NOTIFY) { + if ((cs->current_offset + allocsize) > (SD->max_size << 10)) { + // tried to allocate past the end of the disk, so wrap + // back to the beginning + cs->current_membuf->flags.full = 1; + cs->current_membuf->diskend = cs->current_offset-1; + cs->current_offset = 0; // wrap back to beginning + newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll); + cs->current_membuf = newmb; + } else if ((cs->current_offset + allocsize) > cs->current_membuf->diskend) { + cs->current_membuf->flags.full = 1; + cs->current_membuf->diskend = cs->current_offset-1; + newmb = storeCossCreateMemBuf(SD, cs->current_offset, + checkf, &coll); + cs->current_membuf = newmb; + } + if (coll == 0) { + retofs = cs->current_offset; + } else { + debug(81, 3) ("storeCossAllocate: Collision\n"); + } + } + if (coll == 0) { + cs->current_offset += allocsize; + return retofs; + } else { + return -1; + } +} + +void +storeCossUnlink(SwapDir *SD, StoreEntry *e) +{ + debug(81, 3) ("storeCossUnlink: offset %d\n", e->swap_filen); + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); +} + + +storeIOState * +storeCossCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data) +{ + CossState *cstate; + storeIOState *sio; + + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeCossIOFreeEntry, MEM_STORE_IO); + cstate = memPoolAlloc(coss_state_pool); + sio->fsstate = cstate; + sio->offset = 0; + sio->mode = O_WRONLY; + + /* + * this one is kinda strange - Eric called storeCossAllocate(), then + * storeCossOpen(O_RDONLY) .. weird. Anyway, I'm allocating this now. + */ + sio->st_size = objectLen(e) + e->mem_obj->swap_hdr_sz; + sio->swap_dirn = SD->index; + sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_ALLOCATE); + debug(81, 3) ("storeCossCreate: offset %d, size %d, end %d\n", sio->swap_filen, sio->st_size, sio->swap_filen + sio->st_size); + + sio->callback = callback; + sio->callback_data = callback_data; + cbdataLock(callback_data); + sio->e = (StoreEntry *)e; + sio->st_size = -1; // we won't know this until we read the metadata + + cstate->flags.writing = 0; + cstate->flags.reading = 0; + cstate->readbuffer = NULL; + cstate->reqdiskoffset = -1; + + /* Now add it into the LRU */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + + storeCossMemBufLock(SD, sio); + return sio; +} + +storeIOState * +storeCossOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) +{ + storeIOState *sio; + char *p; + CossState *cstate; + sfileno f = e->swap_filen; + CossInfo *cs = (CossInfo *)SD->fsdata; + + debug(81, 3) ("storeCossOpen: offset %d\n", f); + + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeCossIOFreeEntry, MEM_STORE_IO); + cstate = memPoolAlloc(coss_state_pool); + + sio->fsstate = cstate; + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->offset = 0; + sio->mode = O_RDONLY; + sio->callback = callback; + sio->file_callback = file_callback; + sio->callback_data = callback_data; + cbdataLock(callback_data); + sio->st_size = e->swap_file_sz; + sio->e = e; + + cstate->flags.writing = 0; + cstate->flags.reading = 0; + cstate->readbuffer = NULL; + cstate->reqdiskoffset = -1; + p = storeCossMemPointerFromDiskOffset(SD, f, NULL); + // make local copy so we don't have to lock membuf + if (p) { + cstate->readbuffer = xmalloc(sio->st_size); + memcpy(cstate->readbuffer, p, sio->st_size); + } else { + /* Do the allocation */ + /* this is the first time we've been called on a new sio + read the whole object into memory, then return the + requested amount + */ + /* + * This bit of code actually does the LRU disk thing - we realloc + * a place for the object here, and the file_read() reads the object + * into the cossmembuf for later writing .. + */ + cstate->reqdiskoffset = sio->swap_filen; + sio->swap_filen = -1; + sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC); + if (sio->swap_filen == -1) { + /* We have to clean up neatly .. */ + cbdataFree(sio); + cs->numcollisions++; + debug(81, 2) ("storeCossOpen: Reallocation of %d/%d failed\n", e->swap_dirn, e->swap_filen); + /* XXX XXX XXX Will squid call storeUnlink for this object? */ + return NULL; + } + /* Notify the upper levels that we've changed file number */ + sio->file_callback(sio->callback_data, 0, sio); + + /* + * lock the buffer so it doesn't get swapped out on us + * this will get unlocked in storeCossReadDone + */ + storeCossMemBufLock(SD, sio); + + /* + * Do the LRU magic to keep the disk and memory LRUs identical + */ + dlinkDelete(&sio->e->repl.lru, &SD->repl.lru.list); + dlinkAdd(sio->e, &sio->e->repl.lru, &SD->repl.lru.list); + + /* + * Since we've reallocated a spot for this object, we need to + * write it to the cossmembuf *and* return it in the read .. + */ + cstate->readbuffer = NULL; + } + return sio; +} + +void +storeCossClose(SwapDir *SD, storeIOState * sio) +{ + debug(81, 3) ("storeCossClose: offset %d\n",sio->swap_filen); + if (sio->mode == O_WRONLY) + storeCossMemBufUnlock(SD, sio); + storeCossIOCallback(sio, 0); +} + +void +storeCossRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +{ + char *p; + CossState *cstate = (CossState *)sio->fsstate; + CossInfo *cs = (CossInfo *)SD->fsdata; + + assert(sio->read.callback == NULL); + assert(sio->read.callback_data == NULL); + sio->read.callback = callback; + sio->read.callback_data = callback_data; + cbdataLock(callback_data); + debug(81, 3) ("storeCossRead: offset %d\n", offset); + sio->offset = offset; + cstate->flags.reading = 1; + if ((offset + size) > sio->st_size) + size = sio->st_size - offset; + cstate->requestlen = size; + cstate->requestbuf = buf; + cstate->requestoffset = offset; + if (cstate->readbuffer == NULL) { + p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); + file_read(cs->fd, + p, + sio->st_size, + cstate->reqdiskoffset, + storeCossReadDone, + sio); + cstate->reqdiskoffset = 0; /* XXX */ + } else { + storeCossReadDone(cs->fd, + cstate->readbuffer, + sio->st_size, + 0, + sio); + } +} + +void +storeCossWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +{ + char *dest; + CossMemBuf *membuf; + off_t diskoffset; + + debug(81, 3) ("storeCossWrite: offset %d, len %d\n",sio->offset, size); + diskoffset = sio->swap_filen + sio->offset; + dest = storeCossMemPointerFromDiskOffset(SD, diskoffset, &membuf); + assert(dest != NULL); + memcpy(dest, buf, size); + sio->offset += size; + if (free_func) + (free_func)(buf); +} + + +/* === STATIC =========================================================== */ + +static void +storeCossReadDone(int fd, const char *buf, int len, int errflag, void *my_data) +{ + storeIOState *sio = my_data; + char *p; + STRCB *callback = sio->read.callback; + void *their_data = sio->read.callback_data; + SwapDir *SD = INDEXSD(sio->swap_dirn); + CossState *cstate = (CossState *)sio->fsstate; + size_t rlen; + + debug(81, 3) ("storeCossReadDone: fileno %d, FD %d, len %d\n", + sio->swap_filen, fd, len); + cstate->flags.reading = 0; + if (errflag) { + debug(81, 3) ("storeCossReadDone: got failure (%d)\n", errflag); + rlen = -1; + } else { + if (cstate->readbuffer == NULL) { + cstate->readbuffer = xmalloc(sio->st_size); + p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); + memcpy(cstate->readbuffer, p, sio->st_size); + storeCossMemBufUnlock(SD, sio); + } + sio->offset += len; + memcpy(cstate->requestbuf, &cstate->readbuffer[cstate->requestoffset], + cstate->requestlen); + rlen = (size_t) cstate->requestlen; + } + assert(callback); + assert(their_data); + sio->read.callback = NULL; + sio->read.callback_data = NULL; + if (cbdataValid(their_data)) + callback(their_data, cstate->requestbuf, rlen); + cbdataUnlock(their_data); +} + +static void +storeCossIOCallback(storeIOState * sio, int errflag) +{ + CossState *cstate = (CossState *)sio->fsstate; + debug(81, 3) ("storeCossIOCallback: errflag=%d\n", errflag); + xfree(cstate->readbuffer); + if (cbdataValid(sio->callback_data)) + sio->callback(sio->callback_data, errflag, sio); + sio->callback_data = NULL; + cbdataFree(sio); +} + +static char * +storeCossMemPointerFromDiskOffset(SwapDir *SD, size_t offset, CossMemBuf **mb) +{ + CossMemBuf *t; + CossInfo *cs = (CossInfo *)SD->fsdata; + + for (t=cs->membufs; t; t = t->next) + if ((offset >= t->diskstart) && (offset <= t->diskend)) { + if (mb) + *mb = t; + return &t->buffer[offset - t->diskstart]; + } + if (mb) + *mb = NULL; + return NULL; +} + +static void +storeCossMemBufLock(SwapDir *SD, storeIOState *e) +{ + CossMemBuf *t; + CossInfo *cs = (CossInfo *)SD->fsdata; + + for (t = cs->membufs; t; t = t->next) + if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { + debug(81, 3) ("storeCossMemBufLock: locking %08X, lockcount %d\n",t, t->lockcount); + t->lockcount++; + return; + } + debug(81, 3) ("storeCossMemBufLock: FAILED to lock %08X\n",e); +} + +static void +storeCossMemBufUnlock(SwapDir *SD, storeIOState *e) +{ + CossMemBuf *t; + CossInfo *cs = (CossInfo *)SD->fsdata; + + for (t = cs->membufs; t; t = t->next) { + if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { + t->lockcount--; + debug(81, 3) ("storeCossMemBufUnlock: unlocking %08X, lockcount %d\n",t, t->lockcount); + } + if (t->flags.full && !t->flags.writing && !t->lockcount) + storeCossWriteMemBuf(SD, t); + } +} + + +static void +storeCossWriteMemBuf(SwapDir *SD, CossMemBuf *t) +{ + CossInfo *cs = (CossInfo *)SD->fsdata; + debug(81, 3) ("storeCossWriteMemBuf: offset %d, len %d\n", + t->diskstart, t->diskend - t->diskstart); + cbdataAdd(t, storeCossMembufFree, 0); + file_write(cs->fd, t->diskstart, &t->buffer, + t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL); + t->flags.writing = 1; +} + + +static void +storeCossWriteMemBufDone(int fd, int errflag, size_t len, void *my_data) +{ + CossMemBuf *t = my_data; + CossMemBuf *p,*prev; + CossInfo *cs = (CossInfo *)t->SD->fsdata; + + debug(81, 3) ("storeCossWriteMemBufDone: len %d\n",len); + if (errflag) { + debug(81, 0) ("storeCossMemBufWriteDone: got failure (%d)\n", errflag); + cbdataFree(t); + return; + } + + if (t == cs->membufs) { + cs->membufs = t->next; + cbdataFree(t); + return; + } + prev = t; + for (p = cs->membufs; p; p = p->next) { + if (t == p) { + prev->next = t->next; + cbdataFree(t); + return; + } + prev = p; + } + cbdataFree(t); +} + +static CossMemBuf *storeCossCreateMemBuf(SwapDir *SD, size_t start, + sfileno curfn, int *collision) +{ + CossMemBuf *newmb,*t; + StoreEntry *e; + dlink_node *m,*prev; + int numreleased=0; + CossInfo *cs = (CossInfo *)SD->fsdata; + + newmb = memPoolAlloc(coss_membuf_pool); + newmb->diskstart = start; + debug(81, 3) ("storeCossCreateMemBuf: creating new membuf at %d\n",newmb->diskstart); + newmb->diskend = newmb->diskstart + COSS_MEMBUF_SZ - 1; + newmb->flags.full = 0; + newmb->flags.writing = 0; + newmb->lockcount = 0; + newmb->SD = SD; + newmb->next = cs->membufs; + cs->membufs = newmb; + for (t = cs->membufs; t; t = t->next) + debug(81, 3) ("storeCossCreateMemBuf: membuflist %d lockcount %d\n",t->diskstart,t->lockcount); + + /* + * XXX more evil LRU specific code. This needs to be replaced. + */ + for (m = SD->repl.lru.list.tail; m; m = prev) { + prev = m->prev; + e = m->data; + if (curfn == e->swap_filen) + *collision = 1; /* Mark an object alloc collision */ + if ((e->swap_filen >= newmb->diskstart) && + (e->swap_filen <= newmb->diskend)) { + storeRelease(e); + numreleased++; + } else + break; + } + if (numreleased > 0) + debug(81, 3) ("storeCossCreateMemBuf: this allocation released %d storeEntries\n",numreleased); + return newmb; +} + +/* + * We can't pass memFree() as a free function here, because we need to free + * the fsstate variable .. + */ +static void +storeCossIOFreeEntry(void *sio, int foo) +{ + memPoolFree(coss_state_pool, ((storeIOState *)sio)->fsstate); + memFree(sio, MEM_STORE_IO); +} + +/* + * We can't pass memFree() as a free function here, since we have to pass it + * an int to memFree(), and we aren't using static memory pool allocation here. + * So we have this hack here .. + */ +static void +storeCossMembufFree(void *mb, int foo) +{ + memPoolFree(coss_membuf_pool, mb); +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/Makefile.in Wed Feb 14 00:43:59 2007 @@ -0,0 +1,93 @@ +# +# Makefile for the DISKD storage driver for the Squid Object Cache server +# +# $Id$ +# + +FS = diskd + +prefix = @prefix@ +exec_prefix = @exec_prefix@ +exec_suffix = @exec_suffix@ +cgi_suffix = @cgi_suffix@ +top_srcdir = @top_srcdir@ +bindir = @bindir@ +libexecdir = @libexecdir@ +sysconfdir = @sysconfdir@ +localstatedir = @localstatedir@ +srcdir = @srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh +LDFLAGS = @LDFLAGS@ +INSTALL = @INSTALL@ +INSTALL_BIN = @INSTALL_PROGRAM@ +MV = @MV@ +RM = @RM@ + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(FS).a +DISKD_EXE = diskd$(exec_suffix) + + +OBJS = \ + store_dir_diskd.o \ + store_io_diskd.o + +UTILS = \ + $(DISKD_EXE) + +all: $(OUT) $(UTILS) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) -DSQUID_PREFIX=\"$(prefix)\" $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a diskd + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c + +diskd.o: $(srcdir)/diskd.c + $(CC) $(CFLAGS) $(srcdir)/diskd.c -c -o diskd.o + +$(DISKD_EXE): diskd.o + $(CC) $(LDFLAGS) -L../../../lib/ $(CFLAGS) diskd.o -o $(DISKD_EXE) -lmiscutil -lm + +install: $(UTILS) + @for f in $(UTILS); do \ + if test -f $(libexecdir)/$$f; then \ + echo $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + fi; \ + echo $(INSTALL_BIN) $$f $(libexecdir); \ + $(INSTALL_BIN) $$f $(libexecdir); \ + if test -f $(libexecdir)/-$$f; then \ + echo $(RM) -f $(libexecdir)/-$$f; \ + $(RM) -f $(libexecdir)/-$$f; \ + fi; \ + done + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/diskd.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,310 @@ + +#include "config.h" +#include "squid.h" + + +#include "store_diskd.h" + +#include +#include +#include + +#undef assert +#include + + +#define STDERR_DEBUG 0 + +typedef struct _file_state file_state; + +struct _file_state { + void *key; + file_state *next; + int id; + int fd; + off_t offset; +}; + +static hash_table *hash = NULL; +static pid_t mypid; +static char *shmbuf; + +static int +do_open(diomsg * r, int len, const char *buf) +{ + int fd; + file_state *fs; + /* + * note r->offset holds open() flags + */ + fd = open(buf, r->offset, 0600); + if (fd < 0) { + fprintf(stderr, "%d %s: ", (int) mypid, buf); + perror("open"); + return -errno; + } + fs = xcalloc(1, sizeof(*fs)); + fs->id = r->id; + fs->key = &fs->id; /* gack */ + fs->fd = fd; + hash_join(hash, (hash_link *) fs); +#if STDERR_DEBUG + fprintf(stderr, "%d OPEN id %d, FD %d, fs %p\n", + (int) mypid, + fs->id, + fs->fd, + fs); +#endif + return fd; +} + +static int +do_close(diomsg * r, int len) +{ + int fd; + file_state *fs; + fs = (file_state *) hash_lookup(hash, &r->id); + if (NULL == fs) { + errno = EBADF; + fprintf(stderr, "%d CLOSE id %d: ", (int) mypid, r->id); + perror("do_close"); + return -EBADF; + } + fd = fs->fd; + hash_remove_link(hash, (hash_link *) fs); +#if STDERR_DEBUG + fprintf(stderr, "%d CLOSE id %d, FD %d, fs %p\n", + (int) mypid, + r->id, + fs->fd, + fs); +#endif + xfree(fs); + return close(fd); +} + +static int +do_read(diomsg * r, int len, char *buf) +{ + int x; + int readlen = r->size; + file_state *fs; + fs = (file_state *) hash_lookup(hash, &r->id); + if (NULL == fs) { + errno = EBADF; + fprintf(stderr, "%d READ id %d: ", (int) mypid, r->id); + perror("do_read"); + return -EBADF; + } + if (r->offset > -1 && r->offset != fs->offset) { +#if STDERR_DEBUG + fprintf(stderr, "seeking to %d\n", r->offset); +#endif + if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { + fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); + perror("lseek"); + } + } + x = read(fs->fd, buf, readlen); +#if STDERR_DEBUG + fprintf(stderr, "%d READ %d,%d,%d ret %d\n", (int) mypid, + fs->fd, readlen, r->offset, x); +#endif + if (x < 0) { + fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); + perror("read"); + return -errno; + } + fs->offset = r->offset + x; + return x; +} + +static int +do_write(diomsg * r, int len, const char *buf) +{ + int wrtlen = r->size; + int x; + file_state *fs; + fs = (file_state *) hash_lookup(hash, &r->id); + if (NULL == fs) { + errno = EBADF; + fprintf(stderr, "%d WRITE id %d: ", (int) mypid, r->id); + perror("do_write"); + return -EBADF; + } + if (r->offset > -1 && r->offset != fs->offset) { + if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { + fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); + perror("lseek"); + } + } +#if STDERR_DEBUG + fprintf(stderr, "%d WRITE %d,%d,%d\n", (int) mypid, + fs->fd, wrtlen, r->offset); +#endif + x = write(fs->fd, buf, wrtlen); + if (x < 0) { + fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); + perror("write"); + return -errno; + } + fs->offset = r->offset + x; + return x; +} + +static int +do_unlink(diomsg * r, int len, const char *buf) +{ + if (truncate(buf, 0) < 0) { + fprintf(stderr, "%d UNLNK id %d %s: ", (int) mypid, r->id, buf); + perror("truncate"); + return -errno; + } +#if STDERR_DEBUG + fprintf(stderr, "%d UNLNK %s\n", (int) mypid, buf); +#endif + return 0; +} + +static void +msg_handle(diomsg * r, int rl, diomsg * s) +{ + char *buf = NULL; + s->mtype = r->mtype; + s->callback_data = r->callback_data; + s->shm_offset = r->shm_offset; + s->id = r->id; + if (s->shm_offset > -1) + buf = shmbuf + s->shm_offset; + switch (r->mtype) { + case _MQD_OPEN: + s->status = do_open(r, rl, buf); + break; + case _MQD_CLOSE: + s->status = do_close(r, rl); + break; + case _MQD_READ: + s->status = do_read(r, rl, buf); + break; + case _MQD_WRITE: + s->status = do_write(r, rl, buf); + break; + case _MQD_UNLINK: + s->status = do_unlink(r, rl, buf); + break; + default: + assert(0); + break; + } +} + +int +fsCmp(const void *a, const void *b) +{ + const int *A = a; + const int *B = b; + return *A != *B; +} + +unsigned int +fsHash(const void *key, unsigned int n) +{ + /* note, n must be a power of 2! */ + const int *k = key; + return (*k & (--n)); +} + +static void +alarm_handler(int sig) +{ + (void) 0; +} + +int +main(int argc, char *argv[]) +{ + int key; + int rmsgid; + int smsgid; + int shmid; + diomsg rmsg; + diomsg smsg; + int rlen; + char rbuf[512]; + struct sigaction sa; + setbuf(stdout, NULL); + setbuf(stderr, NULL); + mypid = getpid(); + assert(4 == argc); + key = atoi(argv[1]); + rmsgid = msgget(key, 0600); + if (rmsgid < 0) { + perror("msgget"); + return 1; + } + key = atoi(argv[2]); + smsgid = msgget(key, 0600); + if (smsgid < 0) { + perror("msgget"); + return 1; + } + key = atoi(argv[3]); + shmid = shmget(key, 0, 0600); + if (shmid < 0) { + perror("shmget"); + return 1; + } + shmbuf = shmat(shmid, NULL, 0); + if (shmbuf == (void *) -1) { + perror("shmat"); + return 1; + } + hash = hash_create(fsCmp, 1 << 4, fsHash); + assert(hash); + fcntl(0, F_SETFL, SQUID_NONBLOCK); + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = alarm_handler; + sa.sa_flags = SA_RESTART; + sigaction(SIGALRM, &sa, NULL); + for (;;) { + alarm(1); + memset(&rmsg, '\0', sizeof(rmsg)); + rlen = msgrcv(rmsgid, &rmsg, msg_snd_rcv_sz, 0, 0); + if (rlen < 0) { + if (EINTR == errno) { + if (read(0, rbuf, 512) <= 0) { + if (EWOULDBLOCK == errno) + (void) 0; + else if (EAGAIN == errno) + (void) 0; + else + break; + } + } + if (EAGAIN == errno) { + continue; + } + perror("msgrcv"); + break; + } + alarm(0); + msg_handle(&rmsg, rlen, &smsg); + if (msgsnd(smsgid, &smsg, msg_snd_rcv_sz, 0) < 0) { + perror("msgsnd"); + break; + } + } +#if STDERR_DEBUG + fprintf(stderr, "%d diskd exiting\n", (int) mypid); +#endif + if (msgctl(rmsgid, IPC_RMID, 0) < 0) + perror("msgctl IPC_RMID"); + if (msgctl(smsgid, IPC_RMID, 0) < 0) + perror("msgctl IPC_RMID"); + if (shmdt(shmbuf) < 0) + perror("shmdt"); + if (shmctl(shmid, IPC_RMID, 0) < 0) + perror("shmctl IPC_RMID"); + return 0; +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/store_dir_diskd.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,2125 @@ + +/* + * $Id$ + * + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Duane Wessels + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#if HAVE_STATVFS +#if HAVE_SYS_STATVFS_H +#include +#endif +#endif + +#include +#include +#include + +#include "store_diskd.h" + +#define DefaultLevelOneDirs 16 +#define DefaultLevelTwoDirs 256 +#define STORE_META_BDISKDZ 4096 + +#ifndef SQUID_PREFIX +#error "SQUID_PREFIX needs defining!" +#endif + +diskd_stats_t diskd_stats; + +typedef struct _RebuildState RebuildState; +struct _RebuildState { + SwapDir *sd; + int n_read; + FILE *log; + int speed; + int curlvl1; + int curlvl2; + struct { + unsigned int need_to_validate:1; + unsigned int clean:1; + unsigned int init:1; + } flags; + int done; + int in_dir; + int fn; + struct dirent *entry; + DIR *td; + char fullpath[SQUID_MAXPATHLEN]; + char fullfilename[SQUID_MAXPATHLEN]; + struct _store_rebuild_data counts; +}; + +static int n_diskd_dirs = 0; +static int *diskd_dir_index = NULL; +MemPool * diskd_state_pool = NULL; +static int diskd_initialised = 0; + +static char *storeDiskdDirSwapSubDir(SwapDir *, int subdirn); +static int storeDiskdDirCreateDirectory(const char *path, int); +static int storeDiskdDirVerifyCacheDirs(SwapDir *); +static int storeDiskdDirVerifyDirectory(const char *path); +static void storeDiskdDirCreateSwapSubDirs(SwapDir *); +static char *storeDiskdDirSwapLogFile(SwapDir *, const char *); +static EVH storeDiskdDirRebuildFromDirectory; +static EVH storeDiskdDirRebuildFromSwapLog; +static int storeDiskdDirGetNextFile(RebuildState *, int *sfileno, int *size); +static StoreEntry *storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean); +static void storeDiskdDirRebuild(SwapDir * sd); +static void storeDiskdDirCloseTmpSwapLog(SwapDir * sd); +static FILE *storeDiskdDirOpenTmpSwapLog(SwapDir *, int *, int *); +static STLOGOPEN storeDiskdDirOpenSwapLog; +static STINIT storeDiskdDirInit; +static STFREE storeDiskdDirFree; +static STLOGCLEANOPEN storeDiskdDirWriteCleanOpen; +static void storeDiskdDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANWRITE storeDiskdDirWriteCleanEntry; +static STLOGCLOSE storeDiskdDirCloseSwapLog; +static STLOGWRITE storeDiskdDirSwapLog; +static STNEWFS storeDiskdDirNewfs; +static STDUMP storeDiskdDirDump; +static STMAINTAINFS storeDiskdDirMaintain; +static STCHECKOBJ storeDiskdDirCheckObj; +static STREFOBJ storeDiskdDirRefObj; +static STUNREFOBJ storeDiskdDirUnrefObj; +static QS rev_int_sort; +static int storeDiskdDirClean(int swap_index); +static EVH storeDiskdDirCleanEvent; +static int storeDiskdDirIs(SwapDir * sd); +static int storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2); +static int storeDiskdCleanupDoubleCheck(SwapDir *, StoreEntry *); +static void storeDiskdDirStats(SwapDir *, StoreEntry *); +static void storeDiskdDirInitBitmap(SwapDir *); +static int storeDiskdDirValidFileno(SwapDir *, sfileno); +static int storeDiskdDirCheckExpired(SwapDir *, StoreEntry *); +#if !HEAP_REPLACEMENT +static time_t storeDiskdDirExpiredReferenceAge(SwapDir *); +#endif +static void storeDiskdStats(StoreEntry * sentry); +static void storeDiskdDirSync(SwapDir *); +static void storeDiskdDirCallback(SwapDir *); + + +/* + * These functions were ripped straight out of the heart of store_dir.c. + * They assume that the given filenum is on a diskd partiton, which may or + * may not be true.. + * XXX this evilness should be tidied up at a later date! + */ + +int +storeDiskdDirMapBitTest(SwapDir *SD, int fn) +{ + sfileno filn = fn; + diskdinfo_t *diskdinfo; + diskdinfo = (diskdinfo_t *)SD->fsdata; + return file_map_bit_test(diskdinfo->map, filn); +} + +void +storeDiskdDirMapBitSet(SwapDir *SD, int fn) +{ + sfileno filn = fn; + diskdinfo_t *diskdinfo; + diskdinfo = (diskdinfo_t *)SD->fsdata; + file_map_bit_set(diskdinfo->map, filn); +} + +void +storeDiskdDirMapBitReset(SwapDir *SD, int fn) +{ + sfileno filn = fn; + diskdinfo_t *diskdinfo; + diskdinfo = (diskdinfo_t *)SD->fsdata; + file_map_bit_reset(diskdinfo->map, filn); +} + +int +storeDiskdDirMapBitAllocate(SwapDir *SD) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + int fn; + fn = file_map_allocate(diskdinfo->map, diskdinfo->suggest); + file_map_bit_set(diskdinfo->map, fn); + diskdinfo->suggest = fn + 1; + return fn; +} + +/* + * Initialise the diskd bitmap + * + * If there already is a bitmap, and the numobjects is larger than currently + * configured, we allocate a new bitmap and 'grow' the old one into it. + */ +static void +storeDiskdDirInitBitmap(SwapDir *sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + + if (diskdinfo->map == NULL) { + /* First time */ + diskdinfo->map = file_map_create(); + } else if (diskdinfo->map->max_n_files) { + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ + } + /* else it shrunk, and we leave the old one in place */ +} + +static char * +storeDiskdDirSwapSubDir(SwapDir * sd, int subdirn) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + assert(0 <= subdirn && subdirn < diskdinfo->l1); + snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn); + return fullfilename; +} + +static int +storeDiskdDirCreateDirectory(const char *path, int should_exist) +{ + int created = 0; + struct stat st; + getCurrentTime(); + if (0 == stat(path, &st)) { + if (S_ISDIR(st.st_mode)) { + debug(20, should_exist ? 3 : 1) ("%s exists\n", path); + } else { + fatalf("Swap directory %s is not a directory.", path); + } + } else if (0 == mkdir(path, 0755)) { + debug(20, should_exist ? 1 : 3) ("%s created\n", path); + created = 1; + } else { + fatalf("Failed to make swap directory %s: %s", + path, xstrerror()); + } + return created; +} + +static int +storeDiskdDirVerifyDirectory(const char *path) +{ + struct stat sb; + if (stat(path, &sb) < 0) { + debug(20, 0) ("%s: %s\n", path, xstrerror()); + return -1; + } + if (S_ISDIR(sb.st_mode) == 0) { + debug(20, 0) ("%s is not a directory\n", path); + return -1; + } + return 0; +} + +/* + * This function is called by storeDiskdDirInit(). If this returns < 0, + * then Squid exits, complains about swap directories not + * existing, and instructs the admin to run 'squid -z' + */ +static int +storeDiskdDirVerifyCacheDirs(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + int j; + const char *path = sd->path; + + if (storeDiskdDirVerifyDirectory(path) < 0) + return -1; + for (j = 0; j < diskdinfo->l1; j++) { + path = storeDiskdDirSwapSubDir(sd, j); + if (storeDiskdDirVerifyDirectory(path) < 0) + return -1; + } + return 0; +} + +static void +storeDiskdDirCreateSwapSubDirs(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + int i, k; + int should_exist; + LOCAL_ARRAY(char, name, MAXPATHLEN); + for (i = 0; i < diskdinfo->l1; i++) { + snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i); + if (storeDiskdDirCreateDirectory(name, 0)) + should_exist = 0; + else + should_exist = 1; + debug(47, 1) ("Making directories in %s\n", name); + for (k = 0; k < diskdinfo->l2; k++) { + snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k); + storeDiskdDirCreateDirectory(name, should_exist); + } + } +} + +static char * +storeDiskdDirSwapLogFile(SwapDir * sd, const char *ext) +{ + LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; + if (Config.Log.swap) { + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } + } else { + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); + } + if (ext) + strncat(path, ext, 16); + return path; +} + +static void +storeDiskdDirOpenSwapLog(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + char *path; + int fd; + path = storeDiskdDirSwapLogFile(sd, NULL); + fd = file_open(path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", path, xstrerror()); + fatal("storeDiskdDirOpenSwapLog: Failed to open swap log."); + } + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); + diskdinfo->swaplog_fd = fd; + if (0 == n_diskd_dirs) + assert(NULL == diskd_dir_index); + n_diskd_dirs++; + assert(n_diskd_dirs <= Config.cacheSwap.n_configured); +} + +static void +storeDiskdDirCloseSwapLog(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + if (diskdinfo->swaplog_fd < 0) /* not open */ + return; + file_close(diskdinfo->swaplog_fd); + debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", + sd->index, diskdinfo->swaplog_fd); + diskdinfo->swaplog_fd = -1; + n_diskd_dirs--; + assert(n_diskd_dirs >= 0); + if (0 == n_diskd_dirs) + safe_free(diskd_dir_index); +} + +static void +storeDiskdDirInit(SwapDir * sd) +{ + static int started_clean_event = 0; + int x; + int i; + int rfd; + int ikey = (getpid() << 16) + (sd->index << 4); + char *args[5]; + char skey1[32]; + char skey2[32]; + char skey3[32]; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; + static const char *errmsg = + "\tFailed to verify one of the swap directories, Check cache.log\n" + "\tfor details. Run 'squid -z' to create swap directories\n" + "\tif needed, or if running Squid for the first time."; + + diskdinfo->smsgid = msgget((key_t) ikey, 0700 | IPC_CREAT); + if (diskdinfo->smsgid < 0) { + debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); + fatal("msgget failed"); + } + diskdinfo->rmsgid = msgget((key_t) (ikey + 1), 0700 | IPC_CREAT); + if (diskdinfo->rmsgid < 0) { + debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); + fatal("msgget failed"); + } + diskdinfo->shm.id = shmget((key_t) (ikey + 2), + SHMBUFS * SHMBUF_BLKSZ, 0600 | IPC_CREAT); + if (diskdinfo->shm.id < 0) { + debug(50, 0) ("storeDiskdInit: shmget: %s\n", xstrerror()); + fatal("shmget failed"); + } + diskdinfo->shm.buf = shmat(diskdinfo->shm.id, NULL, 0); + if (diskdinfo->shm.buf == (void *) -1) { + debug(50, 0) ("storeDiskdInit: shmat: %s\n", xstrerror()); + fatal("shmat failed"); + } + diskd_stats.shmbuf_count += SHMBUFS; + for (i = 0; i < SHMBUFS; i++) + storeDiskdShmPut(sd, i * SHMBUF_BLKSZ); + snprintf(skey1, 32, "%d", ikey); + snprintf(skey2, 32, "%d", ikey + 1); + snprintf(skey3, 32, "%d", ikey + 2); + args[0] = "diskd"; + args[1] = skey1; + args[2] = skey2; + args[3] = skey3; + args[4] = NULL; +#if HAVE_POLL && defined(_SQUID_OSF_) + /* pipes and poll() don't get along on DUNIX -DW */ + x = ipcCreate(IPC_TCP_SOCKET, +#else + x = ipcCreate(IPC_FIFO, +#endif + SQUID_PREFIX "/bin/diskd", + args, + "diskd", + &rfd, + &diskdinfo->wfd); + if (x < 0) + fatal("execl " SQUID_PREFIX "/bin/diskd failed"); + if (rfd != diskdinfo->wfd) + comm_close(rfd); + fd_note(diskdinfo->wfd, "squid -> diskd"); + commSetTimeout(diskdinfo->wfd, -1, NULL, NULL); + commSetNonBlocking(diskdinfo->wfd); + storeDiskdDirInitBitmap(sd); + if (storeDiskdDirVerifyCacheDirs(sd) < 0) + fatal(errmsg); + storeDiskdDirOpenSwapLog(sd); + storeDiskdDirRebuild(sd); + if (!started_clean_event) { + eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL, 15.0, 1); + started_clean_event = 1; + } +} + + +static void +storeDiskdStats(StoreEntry * sentry) +{ + storeAppendPrintf(sentry, "sent_count: %d\n", diskd_stats.sent_count); + storeAppendPrintf(sentry, "recv_count: %d\n", diskd_stats.recv_count); + storeAppendPrintf(sentry, "max_away: %d\n", diskd_stats.max_away); + storeAppendPrintf(sentry, "max_shmuse: %d\n", diskd_stats.max_shmuse); + storeAppendPrintf(sentry, "open_fail_queue_len: %d\n", diskd_stats.open_fail_queue_len); + storeAppendPrintf(sentry, "block_queue_len: %d\n", diskd_stats.block_queue_len); + diskd_stats.max_away = diskd_stats.max_shmuse = 0; +} + +/* + * storeDiskdDirSync + * + * Sync any pending data. We just sit around and read the queue + * until the data has finished writing. + */ +static void +storeDiskdDirSync(SwapDir *SD) +{ + /* XXX NOT DONE YET! */ +#warning "storeDiskdSync() needs to be written" +} + + +/* + * storeDiskdDirCallback + * + * Handle callbacks. If we have more than magic2 requests away, we block + * until the queue is below magic2. Otherwise, we simply return when we + * don't get a message. + */ +static void +storeDiskdDirCallback(SwapDir *SD) +{ + diomsg M; + int x; + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + + if (diskdinfo->away >= diskdinfo->magic2) + diskd_stats.block_queue_len++; + + if (diskd_stats.sent_count - diskd_stats.recv_count > + diskd_stats.max_away) { + diskd_stats.max_away = diskd_stats.sent_count - diskd_stats.recv_count; + diskd_stats.max_shmuse = diskd_stats.shmbuf_count; + } + + /* if we are above magic2, we do not break under any reason */ + while (1) { + memset(&M, '\0', sizeof(M)); + x = msgrcv(diskdinfo->rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT); + if (x < 0) { + if (diskdinfo->away >= diskdinfo->magic2) + continue; + else + break; + } else if (x != msg_snd_rcv_sz) { + debug(81, 1) ("storeDiskdReadIndividualQueue: msgget returns %d\n", + x); + break; + } + diskd_stats.recv_count++; + diskdinfo->away--; + storeDiskdHandle(&M); + if (M.shm_offset > -1) + storeDiskdShmPut(SD, M.shm_offset); + } +} + + + +static void +storeDiskdDirRebuildFromDirectory(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE); + StoreEntry *e = NULL; + StoreEntry tmpe; + cache_key key[MD5_DIGEST_CHARS]; + int sfileno = 0; + int count; + int size; + struct stat sb; + int swap_hdr_len; + int fd = -1; + tlv *tlv_list; + tlv *t; + assert(rb != NULL); + debug(20, 3) ("storeDiskdDirRebuildFromDirectory: DIR #%d\n", rb->sd->index); + for (count = 0; count < rb->speed; count++) { + assert(fd == -1); + fd = storeDiskdDirGetNextFile(rb, &sfileno, &size); + if (fd == -2) { + debug(20, 1) ("Done scanning %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + store_dirs_rebuilding--; + storeDiskdDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } else if (fd < 0) { + continue; + } + assert(fd > -1); + /* lets get file stats here */ + if (fstat(fd, &sb) < 0) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: fstat(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %s %7d files opened so far.\n", + rb->sd->path, rb->counts.scancount); + debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno); + Counter.syscalls.disk.reads++; + if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: read(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + file_close(fd); + store_open_disk_fd--; + fd = -1; + swap_hdr_len = 0; +#if USE_TRUNCATE + if (sb.st_size == 0) + continue; +#endif + tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); + if (tlv_list == NULL) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: failed to get meta data\n"); + /* XXX shouldn't this be a call to storeDiskdUnlink ? */ + storeDiskdDirUnlinkFile(SD, sfileno); + continue; + } + debug(20, 3) ("storeDiskdDirRebuildFromDirectory: successful swap meta unpacking\n"); + memset(key, '\0', MD5_DIGEST_CHARS); + memset(&tmpe, '\0', sizeof(StoreEntry)); + for (t = tlv_list; t; t = t->next) { + switch (t->type) { + case STORE_META_KEY: + assert(t->length == MD5_DIGEST_CHARS); + xmemcpy(key, t->value, MD5_DIGEST_CHARS); + break; + case STORE_META_STD: + assert(t->length == STORE_HDR_METASIZE); + xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); + break; + default: + break; + } + } + storeSwapTLVFree(tlv_list); + tlv_list = NULL; + if (storeKeyNull(key)) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: NULL key\n"); + storeDiskdDirUnlinkFile(SD, sfileno); + continue; + } + tmpe.key = key; + /* check sizes */ + if (tmpe.swap_file_sz == 0) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz != sb.st_size) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n", + tmpe.swap_file_sz, (int) sb.st_size); + storeDiskdDirUnlinkFile(SD, sfileno); + continue; + } + if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { + storeDiskdDirUnlinkFile(SD, sfileno); + rb->counts.badflags++; + continue; + } + e = storeGet(key); + if (e && e->lastref >= tmpe.lastref) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (NULL != e) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + rb->counts.dupcount++; + } + rb->counts.objcount++; + storeEntryDump(&tmpe, 5); + e = storeDiskdDirAddDiskRestore(SD, key, + sfileno, + tmpe.swap_file_sz, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + tmpe.refcount, /* refcount */ + tmpe.flags, /* flags */ + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeDiskdDirRebuildFromDirectory, rb, 0.0, 1); +} + +static void +storeDiskdDirRebuildFromSwapLog(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + StoreEntry *e = NULL; + storeSwapLogData s; + size_t ss = sizeof(storeSwapLogData); + int count; + int used; /* is swapfile already in use? */ + int disk_entry_newer; /* is the log entry newer than current entry? */ + double x; + assert(rb != NULL); + /* load a number of objects per invocation */ + for (count = 0; count < rb->speed; count++) { + if (fread(&s, ss, 1, rb->log) != 1) { + debug(20, 1) ("Done reading %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + fclose(rb->log); + rb->log = NULL; + store_dirs_rebuilding--; + storeDiskdDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } + rb->n_read++; + if (s.op <= SWAP_LOG_NOP) + continue; + if (s.op >= SWAP_LOG_MAX) + continue; + debug(20, 3) ("storeDiskdDirRebuildFromSwapLog: %s %s %08X\n", + swap_log_op_str[(int) s.op], + storeKeyText(s.key), + s.swap_filen); + if (s.op == SWAP_LOG_ADD) { + (void) 0; + } else if (s.op == SWAP_LOG_DEL) { + if ((e = storeGet(s.key)) != NULL) { + /* + * Make sure we don't unlink the file, it might be + * in use by a subsequent entry. Also note that + * we don't have to subtract from store_swap_size + * because adding to store_swap_size happens in + * the cleanup procedure. + */ + storeExpireNow(e); + storeReleaseRequest(e); + storeDiskdDirReplRemove(e); + if (e->swap_filen > -1) { + storeDiskdDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.objcount--; + rb->counts.cancelcount++; + } + continue; + } else { + x = log(++rb->counts.bad_log_op) / log(10.0); + if (0.0 == x - (double) (int) x) + debug(20, 1) ("WARNING: %d invalid swap log entries found\n", + rb->counts.bad_log_op); + rb->counts.invalid++; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %7d %s Entries read so far.\n", + rb->counts.scancount, rb->sd->path); + if (!storeDiskdDirValidFileno(SD, s.swap_filen)) { + rb->counts.invalid++; + continue; + } + if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + rb->counts.badflags++; + continue; + } + e = storeGet(s.key); + used = storeDiskdDirMapBitTest(SD, s.swap_filen); + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; + if (used && !disk_entry_newer) { + /* log entry is old, ignore it */ + rb->counts.clashcount++; + continue; + } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) { + /* swapfile taken, same URL, newer, update meta */ + if (e->store_status == STORE_OK) { + e->lastref = s.timestamp; + e->timestamp = s.timestamp; + e->expires = s.expires; + e->lastmod = s.lastmod; + e->flags = s.flags; + e->refcount += s.refcount; +#if HEAP_REPLACEMENT + storeHeapPositionUpdate(e, SD); + storeDiskdDirUnrefObj(SD, e); +#endif + } else { + debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition"); + debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); + } + continue; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", + SD->index, s.swap_filen); + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + assert(rb->flags.need_to_validate); + rb->counts.clashcount++; + continue; + } else if (e && !disk_entry_newer) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (e) { + /* key already exists, this swapfile not being used */ + /* junk old, load new */ + storeExpireNow(e); + storeReleaseRequest(e); + storeDiskdDirReplRemove(e); + if (e->swap_filen > -1) { + /* Make sure we don't actually unlink the file */ + storeDiskdDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.dupcount++; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + /* update store_swap_size */ + rb->counts.objcount++; + e = storeDiskdDirAddDiskRestore(SD, s.key, + s.swap_filen, + s.swap_file_sz, + s.expires, + s.timestamp, + s.lastref, + s.lastmod, + s.refcount, + s.flags, + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1); +} + +static int +storeDiskdDirGetNextFile(RebuildState * rb, int *sfileno, int *size) +{ + SwapDir *SD = rb->sd; + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + int fd = -1; + int used = 0; + int dirs_opened = 0; + debug(20, 3) ("storeDiskdDirGetNextFile: flag=%d, %d: /%02X/%02X\n", + rb->flags.init, + rb->sd->index, + rb->curlvl1, + rb->curlvl2); + if (rb->done) + return -2; + while (fd < 0 && rb->done == 0) { + fd = -1; + if (0 == rb->flags.init) { /* initialize, open first file */ + rb->done = 0; + rb->curlvl1 = 0; + rb->curlvl2 = 0; + rb->in_dir = 0; + rb->flags.init = 1; + assert(Config.cacheSwap.n_configured > 0); + } + if (0 == rb->in_dir) { /* we need to read in a new directory */ + snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", + rb->sd->path, + rb->curlvl1, rb->curlvl2); + if (rb->flags.init && rb->td != NULL) + closedir(rb->td); + rb->td = NULL; + if (dirs_opened) + return -1; + rb->td = opendir(rb->fullpath); + dirs_opened++; + if (rb->td == NULL) { + debug(50, 1) ("storeDiskdDirGetNextFile: opendir: %s: %s\n", + rb->fullpath, xstrerror()); + } else { + rb->entry = readdir(rb->td); /* skip . and .. */ + rb->entry = readdir(rb->td); + if (rb->entry == NULL && errno == ENOENT) + debug(20, 1) ("storeDiskdDirGetNextFile: directory does not exist!.\n"); + debug(20, 3) ("storeDiskdDirGetNextFile: Directory %s\n", rb->fullpath); + } + } + if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) { + rb->in_dir++; + if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) { + debug(20, 3) ("storeDiskdDirGetNextFile: invalid %s\n", + rb->entry->d_name); + continue; + } + if (!storeDiskdFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) { + debug(20, 3) ("storeDiskdDirGetNextFile: %08X does not belong in %d/%d/%d\n", + rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2); + continue; + } + used = storeDiskdDirMapBitTest(SD, rb->fn); + if (used) { + debug(20, 3) ("storeDiskdDirGetNextFile: Locked, continuing with next.\n"); + continue; + } + snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s", + rb->fullpath, rb->entry->d_name); + debug(20, 3) ("storeDiskdDirGetNextFile: Opening %s\n", rb->fullfilename); + fd = file_open(rb->fullfilename, O_RDONLY); + if (fd < 0) + debug(50, 1) ("storeDiskdDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror()); + else + store_open_disk_fd++; + continue; + } + rb->in_dir = 0; + if (++rb->curlvl2 < diskdinfo->l2) + continue; + rb->curlvl2 = 0; + if (++rb->curlvl1 < diskdinfo->l1) + continue; + rb->curlvl1 = 0; + rb->done = 1; + } + *sfileno = rb->fn; + return fd; +} + +/* Add a new object to the cache with empty memory copy and pointer to disk + * use to rebuild store from disk. */ +static StoreEntry * +storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean) +{ + StoreEntry *e = NULL; + debug(20, 5) ("storeDiskdAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); + /* if you call this you'd better be sure file_number is not + * already in use! */ + e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + e->store_status = STORE_OK; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; + e->swap_filen = file_number; + e->swap_dirn = SD->index; + e->swap_file_sz = swap_file_sz; + e->lock_count = 0; +#if !HEAP_REPLACEMENT + e->refcount = 0; +#endif + e->lastref = lastref; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->refcount = refcount; + e->flags = flags; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + storeDiskdDirMapBitSet(SD, e->swap_filen); + storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ + storeDiskdDirReplAdd(SD, e); + return e; +} + +static void +storeDiskdDirRebuild(SwapDir * sd) +{ + RebuildState *rb = xcalloc(1, sizeof(*rb)); + int clean = 0; + int zero = 0; + FILE *fp; + EVH *func = NULL; + rb->sd = sd; + rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; + /* + * If the swap.state file exists in the cache_dir, then + * we'll use storeDiskdDirRebuildFromSwapLog(), otherwise we'll + * use storeDiskdDirRebuildFromDirectory() to open up each file + * and suck in the meta data. + */ + fp = storeDiskdDirOpenTmpSwapLog(sd, &clean, &zero); + if (fp == NULL || zero) { + if (fp != NULL) + fclose(fp); + func = storeDiskdDirRebuildFromDirectory; + } else { + func = storeDiskdDirRebuildFromSwapLog; + rb->log = fp; + rb->flags.clean = (unsigned int) clean; + } + if (!clean) + rb->flags.need_to_validate = 1; + debug(20, 1) ("Rebuilding storage in %s (%s)\n", + sd->path, clean ? "CLEAN" : "DIRTY"); + store_dirs_rebuilding++; + cbdataAdd(rb, cbdataXfree, 0); + eventAdd("storeRebuild", func, rb, 0.0, 1); +} + +static void +storeDiskdDirCloseTmpSwapLog(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); + char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new")); + int fd; + file_close(diskdinfo->swaplog_fd); +#ifdef _SQUID_OS2_ + if (unlink(swaplog_path) < 0) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeDiskdDirCloseTmpSwapLog: unlink failed"); + } +#endif + if (xrename(new_path, swaplog_path) < 0) { + fatal("storeDiskdDirCloseTmpSwapLog: rename failed"); + } + fd = file_open(swaplog_path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeDiskdDirCloseTmpSwapLog: Failed to open swap log."); + } + safe_free(swaplog_path); + safe_free(new_path); + diskdinfo->swaplog_fd = fd; + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); +} + +static FILE * +storeDiskdDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); + char *clean_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean")); + char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new")); + struct stat log_sb; + struct stat clean_sb; + FILE *fp; + int fd; + if (stat(swaplog_path, &log_sb) < 0) { + debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return NULL; + } + *zero_flag = log_sb.st_size == 0 ? 1 : 0; + /* close the existing write-only FD */ + if (diskdinfo->swaplog_fd >= 0) + file_close(diskdinfo->swaplog_fd); + /* open a write-only FD for the new log */ + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", new_path, xstrerror()); + fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); + } + diskdinfo->swaplog_fd = fd; + /* open a read-only stream of the old log */ + fp = fopen(swaplog_path, "r"); + if (fp == NULL) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("Failed to open swap log for reading"); + } + memset(&clean_sb, '\0', sizeof(struct stat)); + if (stat(clean_path, &clean_sb) < 0) + *clean_flag = 0; + else if (clean_sb.st_mtime < log_sb.st_mtime) + *clean_flag = 0; + else + *clean_flag = 1; + safeunlink(clean_path, 1); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return fp; +} + +struct _clean_state { + char *cur; + char *new; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; +}; + +#define CLEAN_BUF_SZ 16384 +/* + * Begin the process to write clean cache state. For DISKD this means + * opening some log files and allocating write buffers. Return 0 if + * we succeed, and assign the 'func' and 'data' return pointers. + */ +static int +storeDiskdDirWriteCleanOpen(SwapDir * sd) +{ + struct _clean_state *state = xcalloc(1, sizeof(*state)); + struct stat sb; + sd->log.clean.write = NULL; + sd->log.clean.state = NULL; + state->cur = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); + state->new = xstrdup(storeDiskdDirSwapLogFile(sd, ".clean")); + state->cln = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean")); + state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); + state->outbuf_offset = 0; + unlink(state->new); + unlink(state->cln); + state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); + if (state->fd < 0) + return -1; + debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n", + state->new, state->fd); +#if HAVE_FCHMOD + if (stat(state->cur, &sb) == 0) + fchmod(state->fd, sb.st_mode); +#endif + sd->log.clean.write = storeDiskdDirWriteCleanEntry; + sd->log.clean.state = state; + return 0; +} + +/* + * "write" an entry to the clean log file. + */ +static void +storeDiskdDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +{ + storeSwapLogData s; + static size_t ss = sizeof(storeSwapLogData); + struct _clean_state *state = sd->log.clean.state; + if (NULL == e) { + storeDiskdDirWriteCleanClose(sd); + return; + } + memset(&s, '\0', ss); + s.op = (char) SWAP_LOG_ADD; + s.swap_filen = e->swap_filen; + s.timestamp = e->timestamp; + s.lastref = e->lastref; + s.expires = e->expires; + s.lastmod = e->lastmod; + s.swap_file_sz = e->swap_file_sz; + s.refcount = e->refcount; + s.flags = e->flags; + xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); + xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); + state->outbuf_offset += ss; + /* buffered write */ + if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; + } + state->outbuf_offset = 0; + } +} + +static void +storeDiskdDirWriteCleanClose(SwapDir * sd) +{ + struct _clean_state *state = sd->log.clean.state; + if (state->fd < 0) + return; + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile " + "not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + } + safe_free(state->outbuf); + /* + * You can't rename open files on Microsoft "operating systems" + * so we have to close before renaming. + */ + storeDiskdDirCloseSwapLog(sd); + /* rename */ + if (state->fd >= 0) { +#ifdef _SQUID_OS2_ + file_close(state->fd); + state->fd = -1; + if (unlink(cur) < 0) + debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n", + xstrerror(), cur); +#endif + xrename(state->new, state->cur); + } + /* touch a timestamp file if we're not still validating */ + if (store_dirs_rebuilding) + (void) 0; + else if (state->fd < 0) + (void) 0; + else + file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC)); + /* close */ + safe_free(state->cur); + safe_free(state->new); + safe_free(state->cln); + if (state->fd >= 0) + file_close(state->fd); + state->fd = -1; + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; +} + +static void +storeDiskdDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); + s->op = (char) op; + s->swap_filen = e->swap_filen; + s->timestamp = e->timestamp; + s->lastref = e->lastref; + s->expires = e->expires; + s->lastmod = e->lastmod; + s->swap_file_sz = e->swap_file_sz; + s->refcount = e->refcount; + s->flags = e->flags; + xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); + file_write(diskdinfo->swaplog_fd, + -1, + s, + sizeof(storeSwapLogData), + NULL, + NULL, + xfree); +} + +static void +storeDiskdDirNewfs(SwapDir * sd) +{ + debug(47, 3) ("Creating swap space in %s\n", sd->path); + storeDiskdDirCreateDirectory(sd->path, 0); + storeDiskdDirCreateSwapSubDirs(sd); +} + +static int +rev_int_sort(const void *A, const void *B) +{ + const int *i1 = A; + const int *i2 = B; + return *i2 - *i1; +} + +static int +storeDiskdDirClean(int swap_index) +{ + DIR *dp = NULL; + struct dirent *de = NULL; + LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); + LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); +#if USE_TRUNCATE + struct stat sb; +#endif + int files[20]; + int swapfileno; + int fn; /* same as swapfileno, but with dirn bits set */ + int n = 0; + int k = 0; + int N0, N1, N2; + int D0, D1, D2; + SwapDir *SD; + diskdinfo_t *diskdinfo; + N0 = n_diskd_dirs; + D0 = diskd_dir_index[swap_index % N0]; + SD = &Config.cacheSwap.swapDirs[D0]; + diskdinfo = (diskdinfo_t *)SD->fsdata; + N1 = diskdinfo->l1; + D1 = (swap_index / N0) % N1; + N2 = diskdinfo->l2; + D2 = ((swap_index / N0) / N1) % N2; + snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X", + Config.cacheSwap.swapDirs[D0].path, D1, D2); + debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1); + dp = opendir(p1); + if (dp == NULL) { + if (errno == ENOENT) { + debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1); + if (mkdir(p1, 0777) == 0) + return 0; + } + debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror()); + safeunlink(p1, 1); + return 0; + } + while ((de = readdir(dp)) != NULL && k < 20) { + if (sscanf(de->d_name, "%X", &swapfileno) != 1) + continue; + fn = swapfileno; /* XXX should remove this cruft ! */ + if (storeDiskdDirValidFileno(SD, fn)) + if (storeDiskdDirMapBitTest(SD, fn)) + if (storeDiskdFilenoBelongsHere(fn, D0, D1, D2)) + continue; +#if USE_TRUNCATE + if (!stat(de->d_name, &sb)) + if (sb.st_size == 0) + continue; +#endif + files[k++] = swapfileno; + } + closedir(dp); + if (k == 0) + return 0; + qsort(files, k, sizeof(int), rev_int_sort); + if (k > 10) + k = 10; + for (n = 0; n < k; n++) { + debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]); + snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); +#if USE_TRUNCATE + truncate(p2, 0); +#else + safeunlink(p2, 0); +#endif + Counter.swap_files_cleaned++; + } + debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1); + return k; +} + +static void +storeDiskdDirCleanEvent(void *unused) +{ + static int swap_index = 0; + int i; + int j = 0; + int n = 0; + /* + * Assert that there are DISKD cache_dirs configured, otherwise + * we should never be called. + */ + assert(n_diskd_dirs); + if (NULL == diskd_dir_index) { + SwapDir *sd; + diskdinfo_t *diskdinfo; + /* + * Initialize the little array that translates DISKD cache_dir + * number into the Config.cacheSwap.swapDirs array index. + */ + diskd_dir_index = xcalloc(n_diskd_dirs, sizeof(*diskd_dir_index)); + for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { + sd = &Config.cacheSwap.swapDirs[i]; + if (!storeDiskdDirIs(sd)) + continue; + diskd_dir_index[n++] = i; + diskdinfo = (diskdinfo_t *)sd->fsdata; + j += (diskdinfo->l1 * diskdinfo->l2); + } + assert(n == n_diskd_dirs); + /* + * Start the storeDiskdDirClean() swap_index with a random + * value. j equals the total number of DISKD level 2 + * swap directories + */ + swap_index = (int) (squid_random() % j); + } + if (0 == store_dirs_rebuilding) { + n = storeDiskdDirClean(swap_index); + swap_index++; + } + eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL, + 15.0 * exp(-0.25 * n), 1); +} + +static int +storeDiskdDirIs(SwapDir * sd) +{ + if (strncmp(sd->type, "diskd", 3) == 0) + return 1; + return 0; +} + +/* + * Does swapfile number 'fn' belong in cachedir #F0, + * level1 dir #F1, level2 dir #F2? + */ +static int +storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2) +{ + int D1, D2; + int L1, L2; + int filn = fn; + diskdinfo_t *diskdinfo; + assert(F0 < Config.cacheSwap.n_configured); + diskdinfo = (diskdinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata; + L1 = diskdinfo->l1; + L2 = diskdinfo->l2; + D1 = ((filn / L2) / L2) % L1; + if (F1 != D1) + return 0; + D2 = (filn / L2) % L2; + if (F2 != D2) + return 0; + return 1; +} + +int +storeDiskdDirValidFileno(SwapDir *SD, sfileno filn) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + if (filn < 0) + return 0; + if (filn > diskdinfo->map->max_n_files) + return 0; + return 1; +} + +void +storeDiskdDirMaintain(SwapDir *SD) +{ + StoreEntry *e = NULL; + int scanned = 0; + int locked = 0; + int expired = 0; + int max_scan; + int max_remove; + double f; + static time_t last_warn_time = 0; +#if !HEAP_REPLACEMENT + dlink_node *m; + dlink_node *prev = NULL; +#else + heap_key age; + heap_key min_age = 0.0; + link_list *locked_entries = NULL; +#if HEAP_REPLACEMENT_DEBUG + if (!verify_heap_property(SD->repl.heap.heap)) { + debug(20, 1) ("Heap property violated!\n"); + } +#endif +#endif + /* We can't delete objects while rebuilding swap */ + if (store_dirs_rebuilding) { + return; + } else { + f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + max_scan = (int) (f * 400.0 + 100.0); + max_remove = (int) (f * 70.0 + 10.0); + /* + * This is kinda cheap, but so we need this priority hack? + */ +#if 0 + eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); +#endif + } + debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove); +#if HEAP_REPLACEMENT + while (heap_nodes(SD->repl.heap.heap) > 0) { + if (store_swap_size < store_swap_low) + break; + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + age = heap_peepminkey(SD->repl.heap.heap); + e = heap_extractmin(SD->repl.heap.heap); + e->repl.node = NULL; /* no longer in the heap */ + scanned++; + if (storeEntryLocked(e)) { + /* + * Entry is in use ... put it in a linked list to ignore it. + */ + if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + /* + * If this was a "SPECIAL" do not add it back into the heap. + * It will always be "SPECIAL" and therefore never removed. + */ + debug(20, 4) ("storeDiskdDirMaintain: locked url %s\n", + (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e-> +key)); + linklistPush(&locked_entries, e); + } + locked++; + continue; + } else if (storeDiskdDirCheckExpired(SD, e)) { + /* + * Note: This will not check the reference age ifdef + * HEAP_REPLACEMENT, but it does some other useful + * checks... + */ + expired++; + debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", + age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); + min_age = age; + storeRelease(e); + } else { + /* + * Did not expire the object so we need to add it back + * into the heap! + */ + debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", + storeKeyText(e->key)); + linklistPush(&locked_entries, e); + continue; + } + if (store_swap_size < store_swap_low) + break; + else if (expired >= max_remove) + break; + else if (scanned >= max_scan) + break; + } + /* + * Bump the heap age factor. + */ + if (min_age > 0.0) + SD->repl.heap.heap->age = min_age; + /* + * Reinsert all bumped locked entries back into heap... + */ + while ((e = linklistShift(&locked_entries))) + e->repl.node = heap_insert(SD->repl.heap.heap, e); +#else + for (m = SD->repl.lru.list.tail; m; m = prev) { + prev = m->prev; + e = m->data; + scanned++; + if (storeEntryLocked(e)) { + /* + * If there is a locked entry at the tail of the LRU list, + * move it to the beginning to get it out of the way. + * Theoretically, we might have all locked objects at the + * tail, and then we'll never remove anything here and the + * LRU age will go to zero. + */ + if (memInUse(MEM_STOREENTRY) > max_scan) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } + locked++; + + } else if (storeDiskdDirCheckExpired(SD, e)) { + expired++; + storeRelease(e); + } + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + } +#endif + debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l +ocked %d f=%.03f\n", + scanned, max_scan, expired, max_remove, locked, f); + debug(20, 3) ("storeMaintainSwapSpace stats:\n"); + debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); + debug(20, 3) (" %6d were scanned\n", scanned); + debug(20, 3) (" %6d were locked\n", locked); + debug(20, 3) (" %6d were expired\n", expired); + if (store_swap_size < Config.Swap.maxSize) + return; + if (squid_curtime - last_warn_time < 10) + return; + debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", + store_swap_size, Config.Swap.maxSize); + last_warn_time = squid_curtime; +} + +/* + * storeDiskdDirCheckObj + * + * This routine is called by storeDirSelectSwapDir to see if the given + * object is able to be stored on this filesystem. DISKD filesystems will + * happily store anything as long as the LRU time isn't too small. + */ +int +storeDiskdDirCheckObj(SwapDir *SD, const StoreEntry *e) +{ + int loadav; + + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; +#if !HEAP_REPLACEMENT + if (storeDiskdDirExpiredReferenceAge(SD) < 300) { + debug(20, 3) ("storeDiskdDirCheckObj: NO: LRU Age = %d\n", + storeDiskdDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; + } +#endif + + /* Check the queue length */ + if (diskdinfo->away >= diskdinfo->magic1) + return -1; + + /* Calculate the storedir load relative to magic2 on a scale of 0 .. 1000 */ + if (diskdinfo->away == 0) + loadav = 0; + else + loadav = diskdinfo->magic2 * 1000 / diskdinfo->away; + return loadav; +} + +/* + * storeDiskdDirRefObj + * + * This routine is called whenever an object is referenced, so we can + * maintain replacement information within the storage fs. + */ +void +storeDiskdDirRefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeDiskdDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT + /* Nothing to do here */ +#else + /* Reference the object */ + if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && + !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } +#endif +} + +/* + * storeDiskdDirUnrefObj + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ +void +storeDiskdDirUnrefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeDiskdDirUnrefObj: referencing %p %d/%d\n", e, + e->swap_dirn, e->swap_filen); +#if HEAP_REPLACEMENT + if (e->repl.node) + heap_update(SD->repl.heap.heap, e->repl.node, e); +#endif +} + +/* + * storeDiskdDirUnlinkFile + * + * This is a *synchronous* unlink which is currently used in the rebuild + * process. This is bad, but it'll have to stay until the dir rebuild + * uses storeDiskdUnlink() .. + */ +void +storeDiskdDirUnlinkFile(SwapDir *SD, sfileno f) +{ + debug(79, 3) ("storeDiskdDirUnlinkFile: unlinking fileno %08X\n", f); + storeDiskdDirMapBitReset(SD, f); + unlinkdUnlink(storeDiskdDirFullPath(SD, f, NULL)); +} + +#if !HEAP_REPLACEMENT +/* + * storeDiskdDirExpiredReferenceAge + * + * The LRU age is scaled exponentially between 1 minute and + * Config.referenceAge , when store_swap_low < store_swap_size < + * store_swap_high. This keeps store_swap_size within the low and high + * water marks. If the cache is very busy then store_swap_size stays + * closer to the low water mark, if it is not busy, then it will stay + * near the high water mark. The LRU age value can be examined on the + * cachemgr 'info' page. + */ +static time_t +storeDiskdDirExpiredReferenceAge(SwapDir *SD) +{ + double x; + double z; + time_t age; + long store_high, store_low; + + store_high = (long) (((float) SD->max_size * + (float) Config.Swap.highWaterMark) / (float) 100); + store_low = (long) (((float) SD->max_size * + (float) Config.Swap.lowWaterMark) / (float) 100); + debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size); + + x = (double) (store_high - SD->cur_size) / + (store_high - store_low); + x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; + z = pow((double) (Config.referenceAge / 60), x); + age = (time_t) (z * 60.0); + if (age < 60) + age = 60; + else if (age > Config.referenceAge) + age = Config.referenceAge; + return age; +} +#endif + +/* + * storeDiskdDirCheckExpired + * + * Check whether the given object is expired or not + * It breaks layering a little by calling the upper layers to find + * out whether the object is locked or not, but we can't help this + * right now. + */ +static int +storeDiskdDirCheckExpired(SwapDir *SD, StoreEntry *e) +{ + if (storeEntryLocked(e)) + return 0; + if (EBIT_TEST(e->flags, RELEASE_REQUEST)) + return 1; + if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) + return 1; + +#if HEAP_REPLACEMENT + /* + * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap + * controls the replacement of objects. + */ + return 1; +#else + if (squid_curtime - e->lastref > storeDiskdDirExpiredReferenceAge(SD)) + return 1; + return 0; +#endif +} + +/* + * Add and remove the given StoreEntry from the replacement policy in + * use. + */ + +void +storeDiskdDirReplAdd(SwapDir *SD, StoreEntry *e) +{ + debug(20, 4) ("storeDiskdDirReplAdd: added node %p to dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + (void) 0; + } else { + e->repl.node = heap_insert(SD->repl.heap.heap, e); + debug(20, 4) ("storeDiskdDirReplAdd: inserted node 0x%x\n", e->repl.node); + } +#else + /* Shouldn't we not throw special objects into the lru ? */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); +#endif +} + + +void +storeDiskdDirReplRemove(StoreEntry *e) +{ + SwapDir *SD = INDEXSD(e->swap_dirn); + debug(20, 4) ("storeDiskdDirReplRemove: remove node %p from dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + /* And now, release the object from the replacement policy */ + if (e->repl.node) { + debug(20, 4) ("storeDiskdDirReplRemove: deleting node 0x%x\n", + e->repl.node); + heap_delete(SD->repl.heap.heap, e->repl.node); + e->repl.node = NULL; + } +#else + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); +#endif +} + + + +/* + * SHM manipulation routines + */ + +void * +storeDiskdShmGet(SwapDir * sd, int *shm_offset) +{ + char *buf; + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + buf = linklistShift(&diskdinfo->shm.stack); + assert(buf); + *shm_offset = buf - diskdinfo->shm.buf; + assert(0 <= *shm_offset && *shm_offset < SHMBUFS * SHMBUF_BLKSZ); + diskd_stats.shmbuf_count++; + return buf; +} + +void +storeDiskdShmPut(SwapDir * sd, int offset) +{ + char *buf; + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + assert(offset >= 0); + assert(offset < SHMBUFS * SHMBUF_BLKSZ); + buf = diskdinfo->shm.buf + offset; + linklistPush(&diskdinfo->shm.stack, buf); + diskd_stats.shmbuf_count--; +} + + + + +/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ + +void +storeDiskdDirStats(SwapDir *SD, StoreEntry * sentry) +{ + diskdinfo_t *diskdinfo; +#if HAVE_STATVFS + struct statvfs sfs; +#endif + diskdinfo = (diskdinfo_t *)SD->fsdata; + storeAppendPrintf(sentry, "First level subdirectories: %d\n", diskdinfo->l1); + storeAppendPrintf(sentry, "Second level subdirectories: %d\n", diskdinfo->l2); + storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); + storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); + storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", + 100.0 * SD->cur_size / SD->max_size); + storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", + diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files, + percent(diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files)); +#if HAVE_STATVFS +#define fsbtoblk(num, fsbs, bs) \ + (((fsbs) != 0 && (fsbs) < (bs)) ? \ + (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + } +#endif + storeAppendPrintf(sentry, "Flags:"); + if (SD->flags.selected) + storeAppendPrintf(sentry, " SELECTED"); + if (SD->flags.read_only) + storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, "\n"); +#if !HEAP_REPLACEMENT + storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", + (double) storeDiskdDirExpiredReferenceAge(SD) / 86400.0); +#else +#if 0 + storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", + heap_peepminkey(sd.repl.heap.heap)); +#endif +#endif + storeAppendPrintf(sentry, "Pending operations: %d\n", diskdinfo->away); +} + +/* + * storeDiskdDirReconfigure + * + * This routine is called when the given swapdir needs reconfiguring + */ +void +storeDiskdDirReconfigure(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + int magic1, magic2; + unsigned int read_only = 0; + diskdinfo_t *diskdinfo; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeDiskdDirReconfigure: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeDiskdDirReconfigure: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeDiskdDirReconfigure: invalid level 2 directories value"); + i = GetInteger(); + magic1 = i; + if (magic1 <= 0) + fatal("storeDiskdDirParse: invalid magic1 value"); + i = GetInteger(); + magic2 = i; + if (magic2 <= 0) + fatal("storeDiskdDirParse: invalid magic2 value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + /* just reconfigure it */ + if (size == sd->max_size) + debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", + path, size); + else + debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", + path, size); + sd->max_size = size; + if (sd->flags.read_only != read_only) + debug(3, 1) ("Cache dir '%s' now %s\n", + path, read_only ? "Read-Only" : "Read-Write"); + diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo->magic1 = magic1; + diskdinfo->magic2 = magic2; + sd->flags.read_only = read_only; + return; +} + +void +storeDiskdDirDump(StoreEntry * entry, const char *name, SwapDir * s) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata; + storeAppendPrintf(entry, "%s %s %s %d %d %d\n", + name, + "diskd", + s->path, + s->max_size >> 10, + diskdinfo->l1, + diskdinfo->l2); +} + +/* + * Only "free" the filesystem specific stuff here + */ +static void +storeDiskdDirFree(SwapDir * s) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata; + if (diskdinfo->swaplog_fd > -1) { + file_close(diskdinfo->swaplog_fd); + diskdinfo->swaplog_fd = -1; + } + filemapFreeMemory(diskdinfo->map); + xfree(diskdinfo); + s->fsdata = NULL; /* Will aid debugging... */ + +} + +char * +storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath) +{ + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + int L1 = diskdinfo->l1; + int L2 = diskdinfo->l2; + if (!fullpath) + fullpath = fullfilename; + fullpath[0] = '\0'; + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", + SD->path, + ((filn / L2) / L2) % L1, + (filn / L2) % L2, + filn); + return fullpath; +} + +/* + * storeDiskdCleanupDoubleCheck + * + * This is called by storeCleanup() if -S was given on the command line. + */ +static int +storeDiskdCleanupDoubleCheck(SwapDir *sd, StoreEntry *e) +{ + struct stat sb; + + if (stat(storeDiskdDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { + debug(20, 0) ("storeDiskdCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", + storeDiskdDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } + if (e->swap_file_sz != sb.st_size) { + debug(20, 0) ("storeDiskdCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", + storeDiskdDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } + return 0; +} + +/* + * storeDiskdDirParse + * + * Called when a *new* fs is being setup. + */ +void +storeDiskdDirParse(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + int magic1, magic2; + unsigned int read_only = 0; + diskdinfo_t *diskdinfo; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeDiskdDirParse: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeDiskdDirParse: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeDiskdDirParse: invalid level 2 directories value"); + i = GetInteger(); + magic1 = i; + if (magic1 <= 0) + fatal("storeDiskdDirParse: invalid magic1 value"); + i = GetInteger(); + magic2 = i; + if (magic2 <= 0) + fatal("storeDiskdDirParse: invalid magic2 value"); + + + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + diskdinfo = xmalloc(sizeof(diskdinfo_t)); + if (diskdinfo == NULL) + fatal("storeDiskdDirParse: couldn't xmalloc() diskdinfo_t!\n"); + + sd->index = index; + sd->path = xstrdup(path); + sd->max_size = size; + sd->fsdata = diskdinfo; + diskdinfo->l1 = l1; + diskdinfo->l2 = l2; + diskdinfo->swaplog_fd = -1; + diskdinfo->map = NULL; /* Debugging purposes */ + diskdinfo->suggest = 0; + diskdinfo->magic1 = magic1; + diskdinfo->magic2 = magic2; + sd->flags.read_only = read_only; + sd->init = storeDiskdDirInit; + sd->newfs = storeDiskdDirNewfs; + sd->dump = storeDiskdDirDump; + sd->freefs = storeDiskdDirFree; + sd->dblcheck = storeDiskdCleanupDoubleCheck; + sd->statfs = storeDiskdDirStats; + sd->maintainfs = storeDiskdDirMaintain; + sd->checkobj = storeDiskdDirCheckObj; + sd->refobj = storeDiskdDirRefObj; + sd->unrefobj = storeDiskdDirUnrefObj; + sd->callback = storeDiskdDirCallback; + sd->sync = storeDiskdDirSync; + sd->obj.create = storeDiskdCreate; + sd->obj.open = storeDiskdOpen; + sd->obj.close = storeDiskdClose; + sd->obj.read = storeDiskdRead; + sd->obj.write = storeDiskdWrite; + sd->obj.unlink = storeDiskdUnlink; + sd->log.open = storeDiskdDirOpenSwapLog; + sd->log.close = storeDiskdDirCloseSwapLog; + sd->log.write = storeDiskdDirSwapLog; + sd->log.clean.open = storeDiskdDirWriteCleanOpen; + + /* Initialise replacement policy stuff */ +#if HEAP_REPLACEMENT + /* + * Create new heaps with cache replacement policies attached to them. + * The cache replacement policy is specified as either GDSF or LFUDA in + * the squid.conf configuration file. Note that the replacement policy + * applies only to the disk replacement algorithm. Memory replacement + * always uses GDSF since we want to maximize object hit rate. + */ + if (Config.replPolicy) { + if (tolower(Config.replPolicy[0]) == 'g') { + debug(20, 1) ("Using GDSF disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } else if (tolower(Config.replPolicy[0]) == 'l') { + if (tolower(Config.replPolicy[1]) == 'f') { + debug(20, 1) ("Using LFUDA disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); + } else if (tolower(Config.replPolicy[1]) == 'r') { + debug(20, 1) ("Using LRU heap disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); + } + } else { + debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } + } else { + debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } +#else + sd->repl.lru.list.head = NULL; + sd->repl.lru.list.tail = NULL; +#endif +} + +/* + * Initial setup / end destruction + */ +void +storeDiskdDirDone(void) +{ + memPoolDestroy(diskd_state_pool); + diskd_initialised = 0; +} + +void +storeFsSetup_diskd(storefs_entry_t *storefs) +{ + assert(!diskd_initialised); + storefs->parsefunc = storeDiskdDirParse; + storefs->reconfigurefunc = storeDiskdDirReconfigure; + storefs->donefunc = storeDiskdDirDone; + diskd_state_pool = memPoolCreate("DISKD IO State data", sizeof(diskdstate_t)); + memset(&diskd_stats, '\0', sizeof(diskd_stats)); + cachemgrRegister("diskd", "DISKD Stats", storeDiskdStats, 0, 1); + debug(81, 1) ("diskd started\n"); + diskd_initialised = 1; +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/store_diskd.h Wed Feb 14 00:43:59 2007 @@ -0,0 +1,126 @@ +/* + * store_diskd.h + * + * Internal declarations for the diskd routines + */ + +#ifndef __STORE_DISKD_H__ +#define __STORE_DISKD_H__ + +/* + * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv. + * If a queue has MAGIC2 (or more) messages away, then we read the + * queue until the level falls below MAGIC2. Recommended value + * is 75% of SHMBUFS. MAGIC1 is the number of messages away which we + * stop allowing open/create for. + */ + +struct _diskdinfo_t { + int swaplog_fd; + int l1; + int l2; + fileMap *map; + int suggest; + int smsgid; + int rmsgid; + int wfd; + int away; + struct { + char *buf; + link_list *stack; + int id; + } shm; + int magic1; + int magic2; +}; + +struct _diskdstate_t { + int id; + struct { + unsigned int close_request:1; + unsigned int reading:1; + unsigned int writing:1; + } flags; + char *read_buf; +}; + +enum { + _MQD_NOP, + _MQD_OPEN, + _MQD_CLOSE, + _MQD_READ, + _MQD_WRITE, + _MQD_UNLINK +}; + +typedef struct _diomsg { + mtyp_t mtype; + int id; + int seq_no; + void *callback_data; + int size; + int offset; + int status; + int shm_offset; +} diomsg; + +struct _diskd_stats { + int open_fail_queue_len; + int block_queue_len; + int max_away; + int max_shmuse; + int shmbuf_count; + int sent_count; + int recv_count; + int sio_id; +}; + +typedef struct _diskd_stats diskd_stats_t; +typedef struct _diskdinfo_t diskdinfo_t; +typedef struct _diskdstate_t diskdstate_t; + +static const int msg_snd_rcv_sz = sizeof(diomsg) - sizeof(mtyp_t); + +/* The diskd_state memory pool */ +extern MemPool * diskd_state_pool; + +extern void storeDiskdDirMapBitReset(SwapDir *, sfileno); +extern int storeDiskdDirMapBitAllocate(SwapDir *); +extern char * storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); +extern void storeDiskdDirUnlinkFile(SwapDir *, sfileno); +extern void storeDiskdDirReplAdd(SwapDir *, StoreEntry *); +extern void storeDiskdDirReplRemove(StoreEntry *); +extern void storeDiskdShmPut(SwapDir *, int); +extern void *storeDiskdShmGet(SwapDir *, int *); +extern void storeDiskdHandle(diomsg *M); + + +/* + * Store IO stuff + */ +extern STOBJCREATE storeDiskdCreate; +extern STOBJOPEN storeDiskdOpen; +extern STOBJCLOSE storeDiskdClose; +extern STOBJREAD storeDiskdRead; +extern STOBJWRITE storeDiskdWrite; +extern STOBJUNLINK storeDiskdUnlink; + +/* + * SHMBUFS is the number of shared memory buffers to allocate for + * Each SwapDir. + */ +#define SHMBUFS 96 +#define SHMBUF_BLKSZ SM_PAGE_SIZE +/* + * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv. + * If a queue has MAGIC2 (or more) messages away, then we read the + * queue until the level falls below MAGIC2. Recommended value + * is 75% of SHMBUFS. + */ +#define MAGIC1 Config.diskd.magic1 +#define MAGIC2 Config.diskd.magic2 + + +extern diskd_stats_t diskd_stats; + +#endif --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/store_io_diskd.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,464 @@ + +#include "config.h" +#include "squid.h" +#include "store_diskd.h" + +#include +#include +#include + +#undef assert +#include + + +/* + * DEBUG: section 81 Diskd Interface functions + */ + + +static int storeDiskdSend(int, SwapDir *, int, storeIOState *, int, int, int); +static void storeDiskdIOCallback(storeIOState * sio, int errflag); +static void storeDiskdIOFreeEntry(void *sio, int foo); + +/* + * SHMBUFS is the number of shared memory buffers to allocate for + * Each SwapDir. + */ +#define SHMBUFS 96 +#define SHMBUF_BLKSZ SM_PAGE_SIZE + + +/* === PUBLIC =========================================================== */ + +storeIOState * +storeDiskdOpen(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, + STIOCB *callback, void *callback_data) +{ + sfileno f = e->swap_filen; + int x; + storeIOState *sio; + char *buf; + int shm_offset; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; + debug(81, 3) ("storeDiskdOpen: fileno %08X\n", f); + /* + * XXX Eventually there should be an option here to fail on open() + * If there are too many requests queued. + */ + if (diskdinfo->away > diskdinfo->magic1) { + debug(81, 3) ("storeDiskdOpen: FAILING, too many requests away\n"); + diskd_stats.open_fail_queue_len++; + return NULL; + } + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeDiskdIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(diskd_state_pool); + + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->mode = O_RDONLY; + sio->callback = callback; + sio->callback_data = callback_data; + sio->e = e; + cbdataLock(callback_data); + + ((diskdstate_t *)(sio->fsstate))->flags.writing = 0; + ((diskdstate_t *)(sio->fsstate))->flags.reading = 0; + ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0; + ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++; + + buf = storeDiskdShmGet(SD, &shm_offset); + /* XXX WRONG!!! :) */ + strcpy(buf, storeDiskdDirFullPath(SD, f, NULL)); + x = storeDiskdSend(_MQD_OPEN, + SD, + ((diskdstate_t *)(sio->fsstate))->id, + sio, + strlen(buf) + 1, + O_RDONLY, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + cbdataUnlock(sio->callback_data); + cbdataFree(sio); + return NULL; + } + return sio; +} + +storeIOState * +storeDiskdCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, + STIOCB *callback, void *callback_data) +{ + sfileno f; + int x; + storeIOState *sio; + char *buf; + int shm_offset; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; + /* + * XXX Eventually there should be an option here to fail on open() + * If there are too many requests queued. + */ + if (diskdinfo->away > diskdinfo->magic1) { + diskd_stats.open_fail_queue_len++; + return NULL; + } + + /* Allocate a number */ + f = storeDiskdDirMapBitAllocate(SD); + debug(81, 3) ("storeDiskdCreate: fileno %08X\n", f); + + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeDiskdIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(diskd_state_pool); + + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->mode = O_WRONLY | O_CREAT | O_TRUNC; + sio->callback = callback; + sio->callback_data = callback_data; + sio->e = e; + cbdataLock(callback_data); + + ((diskdstate_t *)(sio->fsstate))->flags.writing = 0; + ((diskdstate_t *)(sio->fsstate))->flags.reading = 0; + ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0; + ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++; + + buf = storeDiskdShmGet(SD, &shm_offset); + /* XXX WRONG!!! :) */ + strcpy(buf, storeDiskdDirFullPath(SD, f, NULL)); + x = storeDiskdSend(_MQD_OPEN, + SD, + ((diskdstate_t *)(sio->fsstate))->id, + sio, + strlen(buf) + 1, + sio->mode, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + cbdataUnlock(sio->callback_data); + cbdataFree(sio); + return NULL; + } + storeDiskdDirReplAdd(SD, e); + return sio; +} + + +void +storeDiskdClose(SwapDir *SD, storeIOState * sio) +{ + int x; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + debug(81, 3) ("storeDiskdClose: dirno %d, fileno %08X\n", SD->index, + sio->swap_filen); + x = storeDiskdSend(_MQD_CLOSE, + SD, + diskdstate->id, + sio, + 0, + 0, + -1); + if (x < 0) { + debug(50, 1) ("storeDiskdSend CLOSE: %s\n", xstrerror()); + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +void +storeDiskdRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +{ + int x; + int shm_offset; + char *rbuf; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + if (!cbdataValid(sio)) + return; + if (diskdstate->flags.reading) { + debug(81, 1) ("storeDiskdRead: already reading!\n"); + return; + } + assert(sio->read.callback == NULL); + assert(sio->read.callback_data == NULL); + sio->read.callback = callback; + sio->read.callback_data = callback_data; + diskdstate->read_buf = buf; /* the one passed from above */ + cbdataLock(sio->read.callback_data); + debug(81, 3) ("storeDiskdRead: dirno %d, fileno %08X\n", sio->swap_dirn, sio->swap_filen); + sio->offset = offset; + diskdstate->flags.reading = 1; + rbuf = storeDiskdShmGet(SD, &shm_offset); + assert(rbuf); + x = storeDiskdSend(_MQD_READ, + SD, + diskdstate->id, + sio, + (int) size, + (int) offset, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend READ: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +void +storeDiskdWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +{ + int x; + char *sbuf; + int shm_offset; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + debug(81, 3) ("storeDiskdWrite: dirno %d, fileno %08X\n", SD->index, sio->swap_filen); + if (!cbdataValid(sio)) { + free_func(buf); + return; + } + diskdstate->flags.writing = 1; + sbuf = storeDiskdShmGet(SD, &shm_offset); + xmemcpy(sbuf, buf, size); + if (free_func) + free_func(buf); + x = storeDiskdSend(_MQD_WRITE, + SD, + diskdstate->id, + sio, + (int) size, + (int) offset, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend WRITE: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +void +storeDiskdUnlink(SwapDir *SD, StoreEntry *e) +{ + int x; + int shm_offset; + char *buf; + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + + debug(81, 3) ("storeDiskdUnlink: dirno %d, fileno %08X\n", SD->index, + e->swap_filen); + storeDiskdDirReplRemove(e); + storeDiskdDirMapBitReset(SD, e->swap_filen); + if (diskdinfo->away >= diskdinfo->magic1) { + /* Damn, we need to issue a sync unlink here :( */ + debug(50, 2) ("storeDiskUnlink: Out of queue space, sync unlink\n"); + storeDiskdDirUnlinkFile(SD, e->swap_filen); + return; + } + + /* We can attempt a diskd unlink */ + buf = storeDiskdShmGet(SD, &shm_offset); + strcpy(buf, storeDiskdDirFullPath(SD, e->swap_filen, NULL)); + x = storeDiskdSend(_MQD_UNLINK, + SD, + e->swap_filen, + NULL, + 0, + 0, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend UNLINK: %s\n", xstrerror()); + unlink(buf); /* XXX EWW! */ + storeDiskdShmPut(SD, shm_offset); + } +} + + +/* === STATIC =========================================================== */ + +static void +storeDiskdOpenDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + Counter.syscalls.disk.opens++; + debug(81, 3) ("storeDiskdOpenDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +static void +storeDiskdCloseDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + Counter.syscalls.disk.closes++; + debug(81, 3) ("storeDiskdCloseDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + return; + } + storeDiskdIOCallback(sio, DISK_OK); +} + +static void +storeDiskdReadDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + STRCB *callback = sio->read.callback; + SwapDir *sd = INDEXSD(sio->swap_dirn); + diskdstate_t *diskdstate = sio->fsstate; + diskdinfo_t *diskdinfo = sd->fsdata; + void *their_data = sio->read.callback_data; + char *their_buf = diskdstate->read_buf; + char *sbuf; + size_t len; + int valid; + Counter.syscalls.disk.reads++; + diskdstate->flags.reading = 0; + valid = cbdataValid(sio->read.callback_data); + cbdataUnlock(sio->read.callback_data); + debug(81, 3) ("storeDiskdReadDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + return; + } + sbuf = diskdinfo->shm.buf + M->shm_offset; + len = M->status; + xmemcpy(their_buf, sbuf, len); /* yucky copy */ + sio->offset += len; + assert(callback); + assert(their_data); + sio->read.callback = NULL; + sio->read.callback_data = NULL; + if (valid) + callback(their_data, their_buf, len); +} + +static void +storeDiskdWriteDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + Counter.syscalls.disk.writes++; + diskdstate->flags.writing = 0; + debug(81, 3) ("storeDiskdWriteDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + return; + } + sio->offset += M->status; +} + +static void +storeDiskdUnlinkDone(diomsg * M) +{ + debug(81, 3) ("storeDiskdUnlinkDone: fileno %08x status %d\n", + M->id, M->status); + Counter.syscalls.disk.unlinks++; +} + +void +storeDiskdHandle(diomsg * M) +{ + int valid = M->callback_data ? cbdataValid(M->callback_data) : 1; + if (M->callback_data) + cbdataUnlock(M->callback_data); + if (!valid) { + debug(81, 3) ("storeDiskdHandle: Invalid callback_data %p\n", + M->callback_data); + /* + * The read operation has its own callback. If we don't + * call storeDiskdReadDone(), then we must make sure the + * callback_data gets unlocked! + */ + if (_MQD_READ == M->mtype) { + storeIOState *sio = M->callback_data; + cbdataUnlock(sio->read.callback_data); + } + return; + } + switch (M->mtype) { + case _MQD_OPEN: + storeDiskdOpenDone(M); + break; + case _MQD_CLOSE: + storeDiskdCloseDone(M); + break; + case _MQD_READ: + storeDiskdReadDone(M); + break; + case _MQD_WRITE: + storeDiskdWriteDone(M); + break; + case _MQD_UNLINK: + storeDiskdUnlinkDone(M); + break; + default: + assert(0); + break; + } +} + +static void +storeDiskdIOCallback(storeIOState * sio, int errflag) +{ + int valid = cbdataValid(sio->callback_data); + debug(81, 3) ("storeUfsIOCallback: errflag=%d\n", errflag); + cbdataUnlock(sio->callback_data); + if (valid) + sio->callback(sio->callback_data, errflag, sio); + cbdataFree(sio); +} + +static int +storeDiskdSend(int mtype, SwapDir * sd, int id, storeIOState * sio, int size, int offset, int shm_offset) +{ + int x; + diomsg M; + static int send_errors = 0; + static int last_seq_no = 0; + static int seq_no = 0; + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + M.mtype = mtype; + M.callback_data = sio; + M.size = size; + M.offset = offset; + M.status = -1; + M.shm_offset = shm_offset; + M.id = id; + M.seq_no = ++seq_no; + if (M.callback_data) + cbdataLock(M.callback_data); + if (M.seq_no < last_seq_no) + debug(81, 1) ("WARNING: sequencing out of order\n"); + x = msgsnd(diskdinfo->smsgid, &M, msg_snd_rcv_sz, IPC_NOWAIT); + last_seq_no = M.seq_no; + if (0 == x) { + diskd_stats.sent_count++; + diskdinfo->away++; + } else { + debug(50, 1) ("storeDiskdSend: msgsnd: %s\n", xstrerror()); + if (M.callback_data) + cbdataUnlock(M.callback_data); + assert(++send_errors < 100); + } + return x; +} + + +/* + * We can't pass memFree() as a free function here, because we need to free + * the fsstate variable .. + */ +static void +storeDiskdIOFreeEntry(void *sio, int foo) +{ + memPoolFree(diskd_state_pool, ((storeIOState *)sio)->fsstate); + memFree(sio, MEM_STORE_IO); +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/ufs/Makefile.in Wed Feb 14 00:43:59 2007 @@ -0,0 +1,56 @@ +# +# Makefile for the UFS storage driver for the Squid Object Cache server +# +# $Id$ +# + +FS = ufs + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(FS).a + +OBJS = \ + store_dir_ufs.o \ + store_io_ufs.o + + +all: $(OUT) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +install: + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/ufs/store_dir_ufs.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,1903 @@ + +/* + * $Id$ + * + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Duane Wessels + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#if HAVE_STATVFS +#if HAVE_SYS_STATVFS_H +#include +#endif +#endif + +#include "store_ufs.h" + +#define DefaultLevelOneDirs 16 +#define DefaultLevelTwoDirs 256 +#define STORE_META_BUFSZ 4096 + +typedef struct _RebuildState RebuildState; +struct _RebuildState { + SwapDir *sd; + int n_read; + FILE *log; + int speed; + int curlvl1; + int curlvl2; + struct { + unsigned int need_to_validate:1; + unsigned int clean:1; + unsigned int init:1; + } flags; + int done; + int in_dir; + int fn; + struct dirent *entry; + DIR *td; + char fullpath[SQUID_MAXPATHLEN]; + char fullfilename[SQUID_MAXPATHLEN]; + struct _store_rebuild_data counts; +}; + +static int n_ufs_dirs = 0; +static int *ufs_dir_index = NULL; +MemPool * ufs_state_pool = NULL; +static int ufs_initialised = 0; + +static char *storeUfsDirSwapSubDir(SwapDir *, int subdirn); +static int storeUfsDirCreateDirectory(const char *path, int); +static int storeUfsDirVerifyCacheDirs(SwapDir *); +static int storeUfsDirVerifyDirectory(const char *path); +static void storeUfsDirCreateSwapSubDirs(SwapDir *); +static char *storeUfsDirSwapLogFile(SwapDir *, const char *); +static EVH storeUfsDirRebuildFromDirectory; +static EVH storeUfsDirRebuildFromSwapLog; +static int storeUfsDirGetNextFile(RebuildState *, int *sfileno, int *size); +static StoreEntry *storeUfsDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean); +static void storeUfsDirRebuild(SwapDir * sd); +static void storeUfsDirCloseTmpSwapLog(SwapDir * sd); +static FILE *storeUfsDirOpenTmpSwapLog(SwapDir *, int *, int *); +static STLOGOPEN storeUfsDirOpenSwapLog; +static STINIT storeUfsDirInit; +static STFREE storeUfsDirFree; +static STLOGCLEANOPEN storeUfsDirWriteCleanOpen; +static void storeUfsDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANWRITE storeUfsDirWriteCleanEntry; +static STLOGCLOSE storeUfsDirCloseSwapLog; +static STLOGWRITE storeUfsDirSwapLog; +static STNEWFS storeUfsDirNewfs; +static STDUMP storeUfsDirDump; +static STMAINTAINFS storeUfsDirMaintain; +static STCHECKOBJ storeUfsDirCheckObj; +static STREFOBJ storeUfsDirRefObj; +static STUNREFOBJ storeUfsDirUnrefObj; +static QS rev_int_sort; +static int storeUfsDirClean(int swap_index); +static EVH storeUfsDirCleanEvent; +static int storeUfsDirIs(SwapDir * sd); +static int storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2); +static int storeUfsCleanupDoubleCheck(SwapDir *, StoreEntry *); +static void storeUfsDirStats(SwapDir *, StoreEntry *); +static void storeUfsDirInitBitmap(SwapDir *); +static int storeUfsDirValidFileno(SwapDir *, sfileno); +static int storeUfsDirCheckExpired(SwapDir *, StoreEntry *); +#if !HEAP_REPLACEMENT +static time_t storeUfsDirExpiredReferenceAge(SwapDir *); +#endif + +/* + * These functions were ripped straight out of the heart of store_dir.c. + * They assume that the given filenum is on a ufs partiton, which may or + * may not be true.. + * XXX this evilness should be tidied up at a later date! + */ + +int +storeUfsDirMapBitTest(SwapDir *SD, int fn) +{ + sfileno filn = fn; + ufsinfo_t *ufsinfo; + ufsinfo = (ufsinfo_t *)SD->fsdata; + return file_map_bit_test(ufsinfo->map, filn); +} + +void +storeUfsDirMapBitSet(SwapDir *SD, int fn) +{ + sfileno filn = fn; + ufsinfo_t *ufsinfo; + ufsinfo = (ufsinfo_t *)SD->fsdata; + file_map_bit_set(ufsinfo->map, filn); +} + +void +storeUfsDirMapBitReset(SwapDir *SD, int fn) +{ + sfileno filn = fn; + ufsinfo_t *ufsinfo; + ufsinfo = (ufsinfo_t *)SD->fsdata; + file_map_bit_reset(ufsinfo->map, filn); +} + +int +storeUfsDirMapBitAllocate(SwapDir *SD) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + int fn; + fn = file_map_allocate(ufsinfo->map, ufsinfo->suggest); + file_map_bit_set(ufsinfo->map, fn); + ufsinfo->suggest = fn + 1; + return fn; +} + +/* + * Initialise the ufs bitmap + * + * If there already is a bitmap, and the numobjects is larger than currently + * configured, we allocate a new bitmap and 'grow' the old one into it. + */ +static void +storeUfsDirInitBitmap(SwapDir *sd) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + + if (ufsinfo->map == NULL) { + /* First time */ + ufsinfo->map = file_map_create(); + } else if (ufsinfo->map->max_n_files) { + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ + } + /* else it shrunk, and we leave the old one in place */ +} + +static char * +storeUfsDirSwapSubDir(SwapDir * sd, int subdirn) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + assert(0 <= subdirn && subdirn < ufsinfo->l1); + snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn); + return fullfilename; +} + +static int +storeUfsDirCreateDirectory(const char *path, int should_exist) +{ + int created = 0; + struct stat st; + getCurrentTime(); + if (0 == stat(path, &st)) { + if (S_ISDIR(st.st_mode)) { + debug(20, should_exist ? 3 : 1) ("%s exists\n", path); + } else { + fatalf("Swap directory %s is not a directory.", path); + } + } else if (0 == mkdir(path, 0755)) { + debug(20, should_exist ? 1 : 3) ("%s created\n", path); + created = 1; + } else { + fatalf("Failed to make swap directory %s: %s", + path, xstrerror()); + } + return created; +} + +static int +storeUfsDirVerifyDirectory(const char *path) +{ + struct stat sb; + if (stat(path, &sb) < 0) { + debug(20, 0) ("%s: %s\n", path, xstrerror()); + return -1; + } + if (S_ISDIR(sb.st_mode) == 0) { + debug(20, 0) ("%s is not a directory\n", path); + return -1; + } + return 0; +} + +/* + * This function is called by storeUfsDirInit(). If this returns < 0, + * then Squid exits, complains about swap directories not + * existing, and instructs the admin to run 'squid -z' + */ +static int +storeUfsDirVerifyCacheDirs(SwapDir * sd) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + int j; + const char *path = sd->path; + + if (storeUfsDirVerifyDirectory(path) < 0) + return -1; + for (j = 0; j < ufsinfo->l1; j++) { + path = storeUfsDirSwapSubDir(sd, j); + if (storeUfsDirVerifyDirectory(path) < 0) + return -1; + } + return 0; +} + +static void +storeUfsDirCreateSwapSubDirs(SwapDir * sd) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + int i, k; + int should_exist; + LOCAL_ARRAY(char, name, MAXPATHLEN); + for (i = 0; i < ufsinfo->l1; i++) { + snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i); + if (storeUfsDirCreateDirectory(name, 0)) + should_exist = 0; + else + should_exist = 1; + debug(47, 1) ("Making directories in %s\n", name); + for (k = 0; k < ufsinfo->l2; k++) { + snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k); + storeUfsDirCreateDirectory(name, should_exist); + } + } +} + +static char * +storeUfsDirSwapLogFile(SwapDir * sd, const char *ext) +{ + LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; + if (Config.Log.swap) { + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } + } else { + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); + } + if (ext) + strncat(path, ext, 16); + return path; +} + +static void +storeUfsDirOpenSwapLog(SwapDir * sd) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + char *path; + int fd; + path = storeUfsDirSwapLogFile(sd, NULL); + fd = file_open(path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", path, xstrerror()); + fatal("storeUfsDirOpenSwapLog: Failed to open swap log."); + } + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); + ufsinfo->swaplog_fd = fd; + if (0 == n_ufs_dirs) + assert(NULL == ufs_dir_index); + n_ufs_dirs++; + assert(n_ufs_dirs <= Config.cacheSwap.n_configured); +} + +static void +storeUfsDirCloseSwapLog(SwapDir * sd) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + if (ufsinfo->swaplog_fd < 0) /* not open */ + return; + file_close(ufsinfo->swaplog_fd); + debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", + sd->index, ufsinfo->swaplog_fd); + ufsinfo->swaplog_fd = -1; + n_ufs_dirs--; + assert(n_ufs_dirs >= 0); + if (0 == n_ufs_dirs) + safe_free(ufs_dir_index); +} + +static void +storeUfsDirInit(SwapDir * sd) +{ + static int started_clean_event = 0; + static const char *errmsg = + "\tFailed to verify one of the swap directories, Check cache.log\n" + "\tfor details. Run 'squid -z' to create swap directories\n" + "\tif needed, or if running Squid for the first time."; + storeUfsDirInitBitmap(sd); + if (storeUfsDirVerifyCacheDirs(sd) < 0) + fatal(errmsg); + storeUfsDirOpenSwapLog(sd); + storeUfsDirRebuild(sd); + if (!started_clean_event) { + eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL, 15.0, 1); + started_clean_event = 1; + } +} + +static void +storeUfsDirRebuildFromDirectory(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE); + StoreEntry *e = NULL; + StoreEntry tmpe; + cache_key key[MD5_DIGEST_CHARS]; + int sfileno = 0; + int count; + int size; + struct stat sb; + int swap_hdr_len; + int fd = -1; + tlv *tlv_list; + tlv *t; + assert(rb != NULL); + debug(20, 3) ("storeUfsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index); + for (count = 0; count < rb->speed; count++) { + assert(fd == -1); + fd = storeUfsDirGetNextFile(rb, &sfileno, &size); + if (fd == -2) { + debug(20, 1) ("Done scanning %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + store_dirs_rebuilding--; + storeUfsDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } else if (fd < 0) { + continue; + } + assert(fd > -1); + /* lets get file stats here */ + if (fstat(fd, &sb) < 0) { + debug(20, 1) ("storeUfsDirRebuildFromDirectory: fstat(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %s %7d files opened so far.\n", + rb->sd->path, rb->counts.scancount); + debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno); + Counter.syscalls.disk.reads++; + if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) { + debug(20, 1) ("storeUfsDirRebuildFromDirectory: read(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + file_close(fd); + store_open_disk_fd--; + fd = -1; + swap_hdr_len = 0; +#if USE_TRUNCATE + if (sb.st_size == 0) + continue; +#endif + tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); + if (tlv_list == NULL) { + debug(20, 1) ("storeUfsDirRebuildFromDirectory: failed to get meta data\n"); + /* XXX shouldn't this be a call to storeUfsUnlink ? */ + storeUfsDirUnlinkFile(SD, sfileno); + continue; + } + debug(20, 3) ("storeUfsDirRebuildFromDirectory: successful swap meta unpacking\n"); + memset(key, '\0', MD5_DIGEST_CHARS); + memset(&tmpe, '\0', sizeof(StoreEntry)); + for (t = tlv_list; t; t = t->next) { + switch (t->type) { + case STORE_META_KEY: + assert(t->length == MD5_DIGEST_CHARS); + xmemcpy(key, t->value, MD5_DIGEST_CHARS); + break; + case STORE_META_STD: + assert(t->length == STORE_HDR_METASIZE); + xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); + break; + default: + break; + } + } + storeSwapTLVFree(tlv_list); + tlv_list = NULL; + if (storeKeyNull(key)) { + debug(20, 1) ("storeUfsDirRebuildFromDirectory: NULL key\n"); + storeUfsDirUnlinkFile(SD, sfileno); + continue; + } + tmpe.key = key; + /* check sizes */ + if (tmpe.swap_file_sz == 0) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz != sb.st_size) { + debug(20, 1) ("storeUfsDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n", + tmpe.swap_file_sz, (int) sb.st_size); + storeUfsDirUnlinkFile(SD, sfileno); + continue; + } + if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { + storeUfsDirUnlinkFile(SD, sfileno); + rb->counts.badflags++; + continue; + } + e = storeGet(key); + if (e && e->lastref >= tmpe.lastref) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (NULL != e) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + rb->counts.dupcount++; + } + rb->counts.objcount++; + storeEntryDump(&tmpe, 5); + e = storeUfsDirAddDiskRestore(SD, key, + sfileno, + tmpe.swap_file_sz, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + tmpe.refcount, /* refcount */ + tmpe.flags, /* flags */ + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeUfsDirRebuildFromDirectory, rb, 0.0, 1); +} + +static void +storeUfsDirRebuildFromSwapLog(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + StoreEntry *e = NULL; + storeSwapLogData s; + size_t ss = sizeof(storeSwapLogData); + int count; + int used; /* is swapfile already in use? */ + int disk_entry_newer; /* is the log entry newer than current entry? */ + double x; + assert(rb != NULL); + /* load a number of objects per invocation */ + for (count = 0; count < rb->speed; count++) { + if (fread(&s, ss, 1, rb->log) != 1) { + debug(20, 1) ("Done reading %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + fclose(rb->log); + rb->log = NULL; + store_dirs_rebuilding--; + storeUfsDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } + rb->n_read++; + if (s.op <= SWAP_LOG_NOP) + continue; + if (s.op >= SWAP_LOG_MAX) + continue; + debug(20, 3) ("storeUfsDirRebuildFromSwapLog: %s %s %08X\n", + swap_log_op_str[(int) s.op], + storeKeyText(s.key), + s.swap_filen); + if (s.op == SWAP_LOG_ADD) { + (void) 0; + } else if (s.op == SWAP_LOG_DEL) { + if ((e = storeGet(s.key)) != NULL) { + /* + * Make sure we don't unlink the file, it might be + * in use by a subsequent entry. Also note that + * we don't have to subtract from store_swap_size + * because adding to store_swap_size happens in + * the cleanup procedure. + */ + storeExpireNow(e); + storeReleaseRequest(e); + storeUfsDirReplRemove(e); + if (e->swap_filen > -1) { + storeUfsDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.objcount--; + rb->counts.cancelcount++; + } + continue; + } else { + x = log(++rb->counts.bad_log_op) / log(10.0); + if (0.0 == x - (double) (int) x) + debug(20, 1) ("WARNING: %d invalid swap log entries found\n", + rb->counts.bad_log_op); + rb->counts.invalid++; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %7d %s Entries read so far.\n", + rb->counts.scancount, rb->sd->path); + if (!storeUfsDirValidFileno(SD, s.swap_filen)) { + rb->counts.invalid++; + continue; + } + if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + rb->counts.badflags++; + continue; + } + e = storeGet(s.key); + used = storeUfsDirMapBitTest(SD, s.swap_filen); + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; + if (used && !disk_entry_newer) { + /* log entry is old, ignore it */ + rb->counts.clashcount++; + continue; + } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) { + /* swapfile taken, same URL, newer, update meta */ + if (e->store_status == STORE_OK) { + e->lastref = s.timestamp; + e->timestamp = s.timestamp; + e->expires = s.expires; + e->lastmod = s.lastmod; + e->flags = s.flags; + e->refcount += s.refcount; +#if HEAP_REPLACEMENT + storeHeapPositionUpdate(e, SD); + storeUfsDirUnrefObj(SD, e); +#endif + } else { + debug_trap("storeUfsDirRebuildFromSwapLog: bad condition"); + debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); + } + continue; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", + SD->index, s.swap_filen); + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + assert(rb->flags.need_to_validate); + rb->counts.clashcount++; + continue; + } else if (e && !disk_entry_newer) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (e) { + /* key already exists, this swapfile not being used */ + /* junk old, load new */ + storeExpireNow(e); + storeReleaseRequest(e); + storeUfsDirReplRemove(e); + if (e->swap_filen > -1) { + /* Make sure we don't actually unlink the file */ + storeUfsDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.dupcount++; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + /* update store_swap_size */ + rb->counts.objcount++; + e = storeUfsDirAddDiskRestore(SD, s.key, + s.swap_filen, + s.swap_file_sz, + s.expires, + s.timestamp, + s.lastref, + s.lastmod, + s.refcount, + s.flags, + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLog, rb, 0.0, 1); +} + +static int +storeUfsDirGetNextFile(RebuildState * rb, int *sfileno, int *size) +{ + SwapDir *SD = rb->sd; + ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + int fd = -1; + int used = 0; + int dirs_opened = 0; + debug(20, 3) ("storeUfsDirGetNextFile: flag=%d, %d: /%02X/%02X\n", + rb->flags.init, + rb->sd->index, + rb->curlvl1, + rb->curlvl2); + if (rb->done) + return -2; + while (fd < 0 && rb->done == 0) { + fd = -1; + if (0 == rb->flags.init) { /* initialize, open first file */ + rb->done = 0; + rb->curlvl1 = 0; + rb->curlvl2 = 0; + rb->in_dir = 0; + rb->flags.init = 1; + assert(Config.cacheSwap.n_configured > 0); + } + if (0 == rb->in_dir) { /* we need to read in a new directory */ + snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", + rb->sd->path, + rb->curlvl1, rb->curlvl2); + if (rb->flags.init && rb->td != NULL) + closedir(rb->td); + rb->td = NULL; + if (dirs_opened) + return -1; + rb->td = opendir(rb->fullpath); + dirs_opened++; + if (rb->td == NULL) { + debug(50, 1) ("storeUfsDirGetNextFile: opendir: %s: %s\n", + rb->fullpath, xstrerror()); + } else { + rb->entry = readdir(rb->td); /* skip . and .. */ + rb->entry = readdir(rb->td); + if (rb->entry == NULL && errno == ENOENT) + debug(20, 1) ("storeUfsDirGetNextFile: directory does not exist!.\n"); + debug(20, 3) ("storeUfsDirGetNextFile: Directory %s\n", rb->fullpath); + } + } + if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) { + rb->in_dir++; + if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) { + debug(20, 3) ("storeUfsDirGetNextFile: invalid %s\n", + rb->entry->d_name); + continue; + } + if (!storeUfsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) { + debug(20, 3) ("storeUfsDirGetNextFile: %08X does not belong in %d/%d/%d\n", + rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2); + continue; + } + used = storeUfsDirMapBitTest(SD, rb->fn); + if (used) { + debug(20, 3) ("storeUfsDirGetNextFile: Locked, continuing with next.\n"); + continue; + } + snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s", + rb->fullpath, rb->entry->d_name); + debug(20, 3) ("storeUfsDirGetNextFile: Opening %s\n", rb->fullfilename); + fd = file_open(rb->fullfilename, O_RDONLY); + if (fd < 0) + debug(50, 1) ("storeUfsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror()); + else + store_open_disk_fd++; + continue; + } + rb->in_dir = 0; + if (++rb->curlvl2 < ufsinfo->l2) + continue; + rb->curlvl2 = 0; + if (++rb->curlvl1 < ufsinfo->l1) + continue; + rb->curlvl1 = 0; + rb->done = 1; + } + *sfileno = rb->fn; + return fd; +} + +/* Add a new object to the cache with empty memory copy and pointer to disk + * use to rebuild store from disk. */ +static StoreEntry * +storeUfsDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean) +{ + StoreEntry *e = NULL; + debug(20, 5) ("storeUfsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); + /* if you call this you'd better be sure file_number is not + * already in use! */ + e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + e->store_status = STORE_OK; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; + e->swap_filen = file_number; + e->swap_dirn = SD->index; + e->swap_file_sz = swap_file_sz; + e->lock_count = 0; +#if !HEAP_REPLACEMENT + e->refcount = 0; +#endif + e->lastref = lastref; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->refcount = refcount; + e->flags = flags; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + storeUfsDirMapBitSet(SD, e->swap_filen); + storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ + storeUfsDirReplAdd(SD, e); + return e; +} + +static void +storeUfsDirRebuild(SwapDir * sd) +{ + RebuildState *rb = xcalloc(1, sizeof(*rb)); + int clean = 0; + int zero = 0; + FILE *fp; + EVH *func = NULL; + rb->sd = sd; + rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; + /* + * If the swap.state file exists in the cache_dir, then + * we'll use storeUfsDirRebuildFromSwapLog(), otherwise we'll + * use storeUfsDirRebuildFromDirectory() to open up each file + * and suck in the meta data. + */ + fp = storeUfsDirOpenTmpSwapLog(sd, &clean, &zero); + if (fp == NULL || zero) { + if (fp != NULL) + fclose(fp); + func = storeUfsDirRebuildFromDirectory; + } else { + func = storeUfsDirRebuildFromSwapLog; + rb->log = fp; + rb->flags.clean = (unsigned int) clean; + } + if (!clean) + rb->flags.need_to_validate = 1; + debug(20, 1) ("Rebuilding storage in %s (%s)\n", + sd->path, clean ? "CLEAN" : "DIRTY"); + store_dirs_rebuilding++; + cbdataAdd(rb, cbdataXfree, 0); + eventAdd("storeRebuild", func, rb, 0.0, 1); +} + +static void +storeUfsDirCloseTmpSwapLog(SwapDir * sd) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); + char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new")); + int fd; + file_close(ufsinfo->swaplog_fd); +#ifdef _SQUID_OS2_ + if (unlink(swaplog_path) < 0) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeUfsDirCloseTmpSwapLog: unlink failed"); + } +#endif + if (xrename(new_path, swaplog_path) < 0) { + fatal("storeUfsDirCloseTmpSwapLog: rename failed"); + } + fd = file_open(swaplog_path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeUfsDirCloseTmpSwapLog: Failed to open swap log."); + } + safe_free(swaplog_path); + safe_free(new_path); + ufsinfo->swaplog_fd = fd; + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); +} + +static FILE * +storeUfsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); + char *clean_path = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean")); + char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new")); + struct stat log_sb; + struct stat clean_sb; + FILE *fp; + int fd; + if (stat(swaplog_path, &log_sb) < 0) { + debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return NULL; + } + *zero_flag = log_sb.st_size == 0 ? 1 : 0; + /* close the existing write-only FD */ + if (ufsinfo->swaplog_fd >= 0) + file_close(ufsinfo->swaplog_fd); + /* open a write-only FD for the new log */ + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", new_path, xstrerror()); + fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); + } + ufsinfo->swaplog_fd = fd; + /* open a read-only stream of the old log */ + fp = fopen(swaplog_path, "r"); + if (fp == NULL) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("Failed to open swap log for reading"); + } + memset(&clean_sb, '\0', sizeof(struct stat)); + if (stat(clean_path, &clean_sb) < 0) + *clean_flag = 0; + else if (clean_sb.st_mtime < log_sb.st_mtime) + *clean_flag = 0; + else + *clean_flag = 1; + safeunlink(clean_path, 1); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return fp; +} + +struct _clean_state { + char *cur; + char *new; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; +}; + +#define CLEAN_BUF_SZ 16384 +/* + * Begin the process to write clean cache state. For UFS this means + * opening some log files and allocating write buffers. Return 0 if + * we succeed, and assign the 'func' and 'data' return pointers. + */ +static int +storeUfsDirWriteCleanOpen(SwapDir * sd) +{ + struct _clean_state *state = xcalloc(1, sizeof(*state)); + struct stat sb; + sd->log.clean.write = NULL; + sd->log.clean.state = NULL; + state->cur = xstrdup(storeUfsDirSwapLogFile(sd, NULL)); + state->new = xstrdup(storeUfsDirSwapLogFile(sd, ".clean")); + state->cln = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean")); + state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); + state->outbuf_offset = 0; + unlink(state->new); + unlink(state->cln); + state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); + if (state->fd < 0) + return -1; + debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n", + state->new, state->fd); +#if HAVE_FCHMOD + if (stat(state->cur, &sb) == 0) + fchmod(state->fd, sb.st_mode); +#endif + sd->log.clean.write = storeUfsDirWriteCleanEntry; + sd->log.clean.state = state; + return 0; +} + +/* + * "write" an entry to the clean log file. + */ +static void +storeUfsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +{ + storeSwapLogData s; + static size_t ss = sizeof(storeSwapLogData); + struct _clean_state *state = sd->log.clean.state; + if (NULL == e) { + storeUfsDirWriteCleanClose(sd); + return; + } + memset(&s, '\0', ss); + s.op = (char) SWAP_LOG_ADD; + s.swap_filen = e->swap_filen; + s.timestamp = e->timestamp; + s.lastref = e->lastref; + s.expires = e->expires; + s.lastmod = e->lastmod; + s.swap_file_sz = e->swap_file_sz; + s.refcount = e->refcount; + s.flags = e->flags; + xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); + xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); + state->outbuf_offset += ss; + /* buffered write */ + if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; + } + state->outbuf_offset = 0; + } +} + +static void +storeUfsDirWriteCleanClose(SwapDir * sd) +{ + struct _clean_state *state = sd->log.clean.state; + if (state->fd < 0) + return; + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile " + "not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + } + safe_free(state->outbuf); + /* + * You can't rename open files on Microsoft "operating systems" + * so we have to close before renaming. + */ + storeUfsDirCloseSwapLog(sd); + /* rename */ + if (state->fd >= 0) { +#ifdef _SQUID_OS2_ + file_close(state->fd); + state->fd = -1; + if (unlink(cur) < 0) + debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n", + xstrerror(), cur); +#endif + xrename(state->new, state->cur); + } + /* touch a timestamp file if we're not still validating */ + if (store_dirs_rebuilding) + (void) 0; + else if (state->fd < 0) + (void) 0; + else + file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC)); + /* close */ + safe_free(state->cur); + safe_free(state->new); + safe_free(state->cln); + if (state->fd >= 0) + file_close(state->fd); + state->fd = -1; + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; +} + +static void +storeUfsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata; + storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); + s->op = (char) op; + s->swap_filen = e->swap_filen; + s->timestamp = e->timestamp; + s->lastref = e->lastref; + s->expires = e->expires; + s->lastmod = e->lastmod; + s->swap_file_sz = e->swap_file_sz; + s->refcount = e->refcount; + s->flags = e->flags; + xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); + file_write(ufsinfo->swaplog_fd, + -1, + s, + sizeof(storeSwapLogData), + NULL, + NULL, + xfree); +} + +static void +storeUfsDirNewfs(SwapDir * sd) +{ + debug(47, 3) ("Creating swap space in %s\n", sd->path); + storeUfsDirCreateDirectory(sd->path, 0); + storeUfsDirCreateSwapSubDirs(sd); +} + +static int +rev_int_sort(const void *A, const void *B) +{ + const int *i1 = A; + const int *i2 = B; + return *i2 - *i1; +} + +static int +storeUfsDirClean(int swap_index) +{ + DIR *dp = NULL; + struct dirent *de = NULL; + LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); + LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); +#if USE_TRUNCATE + struct stat sb; +#endif + int files[20]; + int swapfileno; + int fn; /* same as swapfileno, but with dirn bits set */ + int n = 0; + int k = 0; + int N0, N1, N2; + int D0, D1, D2; + SwapDir *SD; + ufsinfo_t *ufsinfo; + N0 = n_ufs_dirs; + D0 = ufs_dir_index[swap_index % N0]; + SD = &Config.cacheSwap.swapDirs[D0]; + ufsinfo = (ufsinfo_t *)SD->fsdata; + N1 = ufsinfo->l1; + D1 = (swap_index / N0) % N1; + N2 = ufsinfo->l2; + D2 = ((swap_index / N0) / N1) % N2; + snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X", + Config.cacheSwap.swapDirs[D0].path, D1, D2); + debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1); + dp = opendir(p1); + if (dp == NULL) { + if (errno == ENOENT) { + debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1); + if (mkdir(p1, 0777) == 0) + return 0; + } + debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror()); + safeunlink(p1, 1); + return 0; + } + while ((de = readdir(dp)) != NULL && k < 20) { + if (sscanf(de->d_name, "%X", &swapfileno) != 1) + continue; + fn = swapfileno; /* XXX should remove this cruft ! */ + if (storeUfsDirValidFileno(SD, fn)) + if (storeUfsDirMapBitTest(SD, fn)) + if (storeUfsFilenoBelongsHere(fn, D0, D1, D2)) + continue; +#if USE_TRUNCATE + if (!stat(de->d_name, &sb)) + if (sb.st_size == 0) + continue; +#endif + files[k++] = swapfileno; + } + closedir(dp); + if (k == 0) + return 0; + qsort(files, k, sizeof(int), rev_int_sort); + if (k > 10) + k = 10; + for (n = 0; n < k; n++) { + debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]); + snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); +#if USE_TRUNCATE + truncate(p2, 0); +#else + safeunlink(p2, 0); +#endif + Counter.swap_files_cleaned++; + } + debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1); + return k; +} + +static void +storeUfsDirCleanEvent(void *unused) +{ + static int swap_index = 0; + int i; + int j = 0; + int n = 0; + /* + * Assert that there are UFS cache_dirs configured, otherwise + * we should never be called. + */ + assert(n_ufs_dirs); + if (NULL == ufs_dir_index) { + SwapDir *sd; + ufsinfo_t *ufsinfo; + /* + * Initialize the little array that translates UFS cache_dir + * number into the Config.cacheSwap.swapDirs array index. + */ + ufs_dir_index = xcalloc(n_ufs_dirs, sizeof(*ufs_dir_index)); + for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { + sd = &Config.cacheSwap.swapDirs[i]; + if (!storeUfsDirIs(sd)) + continue; + ufs_dir_index[n++] = i; + ufsinfo = (ufsinfo_t *)sd->fsdata; + j += (ufsinfo->l1 * ufsinfo->l2); + } + assert(n == n_ufs_dirs); + /* + * Start the storeUfsDirClean() swap_index with a random + * value. j equals the total number of UFS level 2 + * swap directories + */ + swap_index = (int) (squid_random() % j); + } + if (0 == store_dirs_rebuilding) { + n = storeUfsDirClean(swap_index); + swap_index++; + } + eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL, + 15.0 * exp(-0.25 * n), 1); +} + +static int +storeUfsDirIs(SwapDir * sd) +{ + if (strncmp(sd->type, "ufs", 3) == 0) + return 1; + return 0; +} + +/* + * Does swapfile number 'fn' belong in cachedir #F0, + * level1 dir #F1, level2 dir #F2? + */ +static int +storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2) +{ + int D1, D2; + int L1, L2; + int filn = fn; + ufsinfo_t *ufsinfo; + assert(F0 < Config.cacheSwap.n_configured); + ufsinfo = (ufsinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata; + L1 = ufsinfo->l1; + L2 = ufsinfo->l2; + D1 = ((filn / L2) / L2) % L1; + if (F1 != D1) + return 0; + D2 = (filn / L2) % L2; + if (F2 != D2) + return 0; + return 1; +} + +int +storeUfsDirValidFileno(SwapDir *SD, sfileno filn) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + if (filn < 0) + return 0; + if (filn > ufsinfo->map->max_n_files) + return 0; + return 1; +} + +void +storeUfsDirMaintain(SwapDir *SD) +{ + StoreEntry *e = NULL; + int scanned = 0; + int locked = 0; + int expired = 0; + int max_scan; + int max_remove; + double f; + static time_t last_warn_time = 0; +#if !HEAP_REPLACEMENT + dlink_node *m; + dlink_node *prev = NULL; +#else + heap_key age; + heap_key min_age = 0.0; + link_list *locked_entries = NULL; +#if HEAP_REPLACEMENT_DEBUG + if (!verify_heap_property(SD->repl.heap.heap)) { + debug(20, 1) ("Heap property violated!\n"); + } +#endif +#endif + /* We can't delete objects while rebuilding swap */ + if (store_dirs_rebuilding) { + return; + } else { + f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + max_scan = (int) (f * 400.0 + 100.0); + max_remove = (int) (f * 70.0 + 10.0); + /* + * This is kinda cheap, but so we need this priority hack? + */ +#if 0 + eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); +#endif + } + debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove); +#if HEAP_REPLACEMENT + while (heap_nodes(SD->repl.heap.heap) > 0) { + if (store_swap_size < store_swap_low) + break; + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + age = heap_peepminkey(SD->repl.heap.heap); + e = heap_extractmin(SD->repl.heap.heap); + e->repl.node = NULL; /* no longer in the heap */ + scanned++; + if (storeEntryLocked(e)) { + /* + * Entry is in use ... put it in a linked list to ignore it. + */ + if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + /* + * If this was a "SPECIAL" do not add it back into the heap. + * It will always be "SPECIAL" and therefore never removed. + */ + debug(20, 4) ("storeUfsDirMaintain: locked url %s\n", + (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e-> +key)); + linklistPush(&locked_entries, e); + } + locked++; + continue; + } else if (storeUfsDirCheckExpired(SD, e)) { + /* + * Note: This will not check the reference age ifdef + * HEAP_REPLACEMENT, but it does some other useful + * checks... + */ + expired++; + debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", + age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); + min_age = age; + storeRelease(e); + } else { + /* + * Did not expire the object so we need to add it back + * into the heap! + */ + debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", + storeKeyText(e->key)); + linklistPush(&locked_entries, e); + continue; + } + if (store_swap_size < store_swap_low) + break; + else if (expired >= max_remove) + break; + else if (scanned >= max_scan) + break; + } + /* + * Bump the heap age factor. + */ + if (min_age > 0.0) + SD->repl.heap.heap->age = min_age; + /* + * Reinsert all bumped locked entries back into heap... + */ + while ((e = linklistShift(&locked_entries))) + e->repl.node = heap_insert(SD->repl.heap.heap, e); +#else + for (m = SD->repl.lru.list.tail; m; m = prev) { + prev = m->prev; + e = m->data; + scanned++; + if (storeEntryLocked(e)) { + /* + * If there is a locked entry at the tail of the LRU list, + * move it to the beginning to get it out of the way. + * Theoretically, we might have all locked objects at the + * tail, and then we'll never remove anything here and the + * LRU age will go to zero. + */ + if (memInUse(MEM_STOREENTRY) > max_scan) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } + locked++; + + } else if (storeUfsDirCheckExpired(SD, e)) { + expired++; + storeRelease(e); + } + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + } +#endif + debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l +ocked %d f=%.03f\n", + scanned, max_scan, expired, max_remove, locked, f); + debug(20, 3) ("storeMaintainSwapSpace stats:\n"); + debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); + debug(20, 3) (" %6d were scanned\n", scanned); + debug(20, 3) (" %6d were locked\n", locked); + debug(20, 3) (" %6d were expired\n", expired); + if (store_swap_size < Config.Swap.maxSize) + return; + if (squid_curtime - last_warn_time < 10) + return; + debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", + store_swap_size, Config.Swap.maxSize); + last_warn_time = squid_curtime; +} + +/* + * storeUfsDirCheckObj + * + * This routine is called by storeDirSelectSwapDir to see if the given + * object is able to be stored on this filesystem. UFS filesystems will + * happily store anything as long as the LRU time isn't too small. + */ +int +storeUfsDirCheckObj(SwapDir *SD, const StoreEntry *e) +{ +#if !HEAP_REPLACEMENT + if (storeUfsDirExpiredReferenceAge(SD) < 300) { + debug(20, 3) ("storeUfsDirCheckObj: NO: LRU Age = %d\n", + storeUfsDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; + } +#endif + /* Return 999 (99.9%) constant load */ + return 999; +} + +/* + * storeUfsDirRefObj + * + * This routine is called whenever an object is referenced, so we can + * maintain replacement information within the storage fs. + */ +void +storeUfsDirRefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeUfsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT + /* Nothing to do here */ +#else + /* Reference the object */ + if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && + !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } +#endif +} + +/* + * storeUfsDirUnrefObj + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ +void +storeUfsDirUnrefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeUfsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT + if (e->repl.node) + heap_update(SD->repl.heap.heap, e->repl.node, e); +#endif +} + +/* + * storeUfsDirUnlinkFile + * + * This routine unlinks a file and pulls it out of the bitmap. + * It used to be in storeUfsUnlink(), however an interface change + * forced this bit of code here. Eeek. + */ +void +storeUfsDirUnlinkFile(SwapDir *SD, sfileno f) +{ + debug(79, 3) ("storeUfsDirUnlinkFile: unlinking fileno %08X\n", f); + storeUfsDirMapBitReset(SD, f); + unlinkdUnlink(storeUfsDirFullPath(SD, f, NULL)); +} + +#if !HEAP_REPLACEMENT +/* + * storeUfsDirExpiredReferenceAge + * + * The LRU age is scaled exponentially between 1 minute and + * Config.referenceAge , when store_swap_low < store_swap_size < + * store_swap_high. This keeps store_swap_size within the low and high + * water marks. If the cache is very busy then store_swap_size stays + * closer to the low water mark, if it is not busy, then it will stay + * near the high water mark. The LRU age value can be examined on the + * cachemgr 'info' page. + */ +static time_t +storeUfsDirExpiredReferenceAge(SwapDir *SD) +{ + double x; + double z; + time_t age; + long store_high, store_low; + + store_high = (long) (((float) SD->max_size * + (float) Config.Swap.highWaterMark) / (float) 100); + store_low = (long) (((float) SD->max_size * + (float) Config.Swap.lowWaterMark) / (float) 100); + debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size); + + x = (double) (store_high - SD->cur_size) / + (store_high - store_low); + x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; + z = pow((double) (Config.referenceAge / 60), x); + age = (time_t) (z * 60.0); + if (age < 60) + age = 60; + else if (age > Config.referenceAge) + age = Config.referenceAge; + return age; +} +#endif + +/* + * storeUfsDirCheckExpired + * + * Check whether the given object is expired or not + * It breaks layering a little by calling the upper layers to find + * out whether the object is locked or not, but we can't help this + * right now. + */ +static int +storeUfsDirCheckExpired(SwapDir *SD, StoreEntry *e) +{ + if (storeEntryLocked(e)) + return 0; + if (EBIT_TEST(e->flags, RELEASE_REQUEST)) + return 1; + if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) + return 1; + +#if HEAP_REPLACEMENT + /* + * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap + * controls the replacement of objects. + */ + return 1; +#else + if (squid_curtime - e->lastref > storeUfsDirExpiredReferenceAge(SD)) + return 1; + return 0; +#endif +} + +/* + * Add and remove the given StoreEntry from the replacement policy in + * use. + */ + +void +storeUfsDirReplAdd(SwapDir *SD, StoreEntry *e) +{ + debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + (void) 0; + } else { + e->repl.node = heap_insert(SD->repl.heap.heap, e); + debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node); + } +#else + /* Shouldn't we not throw special objects into the lru ? */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); +#endif +} + + +void +storeUfsDirReplRemove(StoreEntry *e) +{ + SwapDir *SD = INDEXSD(e->swap_dirn); + debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + /* And now, release the object from the replacement policy */ + if (e->repl.node) { + debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n", + e->repl.node); + heap_delete(SD->repl.heap.heap, e->repl.node); + e->repl.node = NULL; + } +#else + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); +#endif +} + + + +/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ + +void +storeUfsDirStats(SwapDir *SD, StoreEntry * sentry) +{ + ufsinfo_t *ufsinfo; +#if HAVE_STATVFS + struct statvfs sfs; +#endif + ufsinfo = (ufsinfo_t *)SD->fsdata; + storeAppendPrintf(sentry, "First level subdirectories: %d\n", ufsinfo->l1); + storeAppendPrintf(sentry, "Second level subdirectories: %d\n", ufsinfo->l2); + storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); + storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); + storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", + 100.0 * SD->cur_size / SD->max_size); + storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", + ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files, + percent(ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files)); +#if HAVE_STATVFS +#define fsbtoblk(num, fsbs, bs) \ + (((fsbs) != 0 && (fsbs) < (bs)) ? \ + (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + } +#endif + storeAppendPrintf(sentry, "Flags:"); + if (SD->flags.selected) + storeAppendPrintf(sentry, " SELECTED"); + if (SD->flags.read_only) + storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, "\n"); +#if !HEAP_REPLACEMENT + storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", + (double) storeUfsDirExpiredReferenceAge(SD) / 86400.0); +#else +#if 0 + storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", + heap_peepminkey(sd.repl.heap.heap)); +#endif +#endif +} + +/* + * storeUfsDirReconfigure + * + * This routine is called when the given swapdir needs reconfiguring + */ +void +storeUfsDirReconfigure(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + unsigned int read_only = 0; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeUfsDirReconfigure: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeUfsDirReconfigure: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeUfsDirReconfigure: invalid level 2 directories value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + /* just reconfigure it */ + if (size == sd->max_size) + debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", + path, size); + else + debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", + path, size); + sd->max_size = size; + if (sd->flags.read_only != read_only) + debug(3, 1) ("Cache dir '%s' now %s\n", + path, read_only ? "Read-Only" : "Read-Write"); + sd->flags.read_only = read_only; + return; +} + +void +storeUfsDirDump(StoreEntry * entry, const char *name, SwapDir * s) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)s->fsdata; + storeAppendPrintf(entry, "%s %s %s %d %d %d\n", + name, + "ufs", + s->path, + s->max_size >> 10, + ufsinfo->l1, + ufsinfo->l2); +} + +/* + * Only "free" the filesystem specific stuff here + */ +static void +storeUfsDirFree(SwapDir * s) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)s->fsdata; + if (ufsinfo->swaplog_fd > -1) { + file_close(ufsinfo->swaplog_fd); + ufsinfo->swaplog_fd = -1; + } + filemapFreeMemory(ufsinfo->map); + xfree(ufsinfo); + s->fsdata = NULL; /* Will aid debugging... */ + +} + +char * +storeUfsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath) +{ + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + int L1 = ufsinfo->l1; + int L2 = ufsinfo->l2; + if (!fullpath) + fullpath = fullfilename; + fullpath[0] = '\0'; + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", + SD->path, + ((filn / L2) / L2) % L1, + (filn / L2) % L2, + filn); + return fullpath; +} + +/* + * storeUfsCleanupDoubleCheck + * + * This is called by storeCleanup() if -S was given on the command line. + */ +static int +storeUfsCleanupDoubleCheck(SwapDir *sd, StoreEntry *e) +{ + struct stat sb; + + if (stat(storeUfsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { + debug(20, 0) ("storeUfsCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n", + storeUfsDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } + if (e->swap_file_sz != sb.st_size) { + debug(20, 0) ("storeUfsCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n", + storeUfsDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeUfsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } + return 0; +} + +/* + * storeUfsDirParse + * + * Called when a *new* fs is being setup. + */ +void +storeUfsDirParse(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + unsigned int read_only = 0; + ufsinfo_t *ufsinfo; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeUfsDirParse: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeUfsDirParse: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeUfsDirParse: invalid level 2 directories value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + ufsinfo = xmalloc(sizeof(ufsinfo_t)); + if (ufsinfo == NULL) + fatal("storeUfsDirParse: couldn't xmalloc() ufsinfo_t!\n"); + + sd->index = index; + sd->path = xstrdup(path); + sd->max_size = size; + sd->fsdata = ufsinfo; + ufsinfo->l1 = l1; + ufsinfo->l2 = l2; + ufsinfo->swaplog_fd = -1; + ufsinfo->map = NULL; /* Debugging purposes */ + ufsinfo->suggest = 0; + sd->flags.read_only = read_only; + sd->init = storeUfsDirInit; + sd->newfs = storeUfsDirNewfs; + sd->dump = storeUfsDirDump; + sd->freefs = storeUfsDirFree; + sd->dblcheck = storeUfsCleanupDoubleCheck; + sd->statfs = storeUfsDirStats; + sd->maintainfs = storeUfsDirMaintain; + sd->checkobj = storeUfsDirCheckObj; + sd->refobj = storeUfsDirRefObj; + sd->unrefobj = storeUfsDirUnrefObj; + sd->callback = NULL; + sd->sync = NULL; + sd->obj.create = storeUfsCreate; + sd->obj.open = storeUfsOpen; + sd->obj.close = storeUfsClose; + sd->obj.read = storeUfsRead; + sd->obj.write = storeUfsWrite; + sd->obj.unlink = storeUfsUnlink; + sd->log.open = storeUfsDirOpenSwapLog; + sd->log.close = storeUfsDirCloseSwapLog; + sd->log.write = storeUfsDirSwapLog; + sd->log.clean.open = storeUfsDirWriteCleanOpen; + + /* Initialise replacement policy stuff */ +#if HEAP_REPLACEMENT + /* + * Create new heaps with cache replacement policies attached to them. + * The cache replacement policy is specified as either GDSF or LFUDA in + * the squid.conf configuration file. Note that the replacement policy + * applies only to the disk replacement algorithm. Memory replacement + * always uses GDSF since we want to maximize object hit rate. + */ + if (Config.replPolicy) { + if (tolower(Config.replPolicy[0]) == 'g') { + debug(20, 1) ("Using GDSF disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } else if (tolower(Config.replPolicy[0]) == 'l') { + if (tolower(Config.replPolicy[1]) == 'f') { + debug(20, 1) ("Using LFUDA disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); + } else if (tolower(Config.replPolicy[1]) == 'r') { + debug(20, 1) ("Using LRU heap disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); + } + } else { + debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } + } else { + debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } +#else + sd->repl.lru.list.head = NULL; + sd->repl.lru.list.tail = NULL; +#endif +} + +/* + * Initial setup / end destruction + */ +void +storeUfsDirDone(void) +{ + memPoolDestroy(ufs_state_pool); + ufs_initialised = 0; +} + +void +storeFsSetup_ufs(storefs_entry_t *storefs) +{ + assert(!ufs_initialised); + storefs->parsefunc = storeUfsDirParse; + storefs->reconfigurefunc = storeUfsDirReconfigure; + storefs->donefunc = storeUfsDirDone; + ufs_state_pool = memPoolCreate("UFS IO State data", sizeof(ufsstate_t)); + ufs_initialised = 1; +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/ufs/store_io_ufs.c Wed Feb 14 00:43:59 2007 @@ -0,0 +1,267 @@ + +/* + * $Id$ + * + * DEBUG: section 79 Storage Manager UFS Interface + * AUTHOR: Duane Wessels + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "store_ufs.h" + + +static DRCB storeUfsReadDone; +static DWCB storeUfsWriteDone; +static void storeUfsIOCallback(storeIOState * sio, int errflag); +static void storeUfsIOFreeEntry(void *, int); + +/* === PUBLIC =========================================================== */ + +storeIOState * +storeUfsOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) +{ + sfileno f = e->swap_filen; + char *path = storeUfsDirFullPath(SD, f, NULL); + storeIOState *sio; + struct stat sb; + int fd; + debug(79, 3) ("storeUfsOpen: fileno %08X\n", f); + fd = file_open(path, O_RDONLY); + if (fd < 0) { + debug(79, 3) ("storeUfsOpen: got failure (%d)\n", errno); + return NULL; + } + debug(79, 3) ("storeUfsOpen: opened FD %d\n", fd); + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeUfsIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(ufs_state_pool); + + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->mode = O_RDONLY; + sio->callback = callback; + sio->callback_data = callback_data; + cbdataLock(callback_data); + sio->e = e; + ((ufsstate_t *)(sio->fsstate))->fd = fd; + ((ufsstate_t *)(sio->fsstate))->flags.writing = 0; + ((ufsstate_t *)(sio->fsstate))->flags.reading = 0; + ((ufsstate_t *)(sio->fsstate))->flags.close_request = 0; + if (fstat(fd, &sb) == 0) + sio->st_size = sb.st_size; + store_open_disk_fd++; + + /* We should update the heap/dlink position here ! */ + return sio; +} + +storeIOState * +storeUfsCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data) +{ + storeIOState *sio; + int fd; + int mode = (O_WRONLY | O_CREAT | O_TRUNC); + char *path; + ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + sfileno filn; + sdirno dirn; + + /* Allocate a number */ + dirn = SD->index; + filn = storeUfsDirMapBitAllocate(SD); + ufsinfo->suggest = filn + 1; + /* Shouldn't we handle a 'bitmap full' error here? */ + path = storeUfsDirFullPath(SD, filn, NULL); + + debug(79, 3) ("storeUfsCreate: fileno %08X\n", filn); + fd = file_open(path, mode); + if (fd < 0) { + debug(79, 3) ("storeUfsCreate: got failure (%d)\n", errno); + return NULL; + } + debug(79, 3) ("storeUfsCreate: opened FD %d\n", fd); + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeUfsIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(ufs_state_pool); + + sio->swap_filen = filn; + sio->swap_dirn = dirn; + sio->mode = mode; + sio->callback = callback; + sio->callback_data = callback_data; + cbdataLock(callback_data); + sio->e = (StoreEntry *) e; + ((ufsstate_t *)(sio->fsstate))->fd = fd; + ((ufsstate_t *)(sio->fsstate))->flags.writing = 0; + ((ufsstate_t *)(sio->fsstate))->flags.reading = 0; + ((ufsstate_t *)(sio->fsstate))->flags.close_request = 0; + store_open_disk_fd++; + + /* now insert into the replacement policy */ + storeUfsDirReplAdd(SD, e); + return sio; +} + +void +storeUfsClose(SwapDir *SD, storeIOState * sio) +{ + ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + + debug(79, 3) ("storeUfsClose: dirno %d, fileno %08X, FD %d\n", + sio->swap_dirn, sio->swap_filen, ufsstate->fd); + if (ufsstate->flags.reading || ufsstate->flags.writing) { + ufsstate->flags.close_request = 1; + return; + } + storeUfsIOCallback(sio, 0); +} + +void +storeUfsRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +{ + ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + + assert(sio->read.callback == NULL); + assert(sio->read.callback_data == NULL); + sio->read.callback = callback; + sio->read.callback_data = callback_data; + cbdataLock(callback_data); + debug(79, 3) ("storeUfsRead: dirno %d, fileno %08X, FD %d\n", + sio->swap_dirn, sio->swap_filen, ufsstate->fd); + sio->offset = offset; + ufsstate->flags.reading = 1; + file_read(ufsstate->fd, + buf, + size, + offset, + storeUfsReadDone, + sio); +} + +void +storeUfsWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +{ + ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + debug(79, 3) ("storeUfsWrite: dirn %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, ufsstate->fd); + ufsstate->flags.writing = 1; + file_write(ufsstate->fd, + offset, + buf, + size, + storeUfsWriteDone, + sio, + free_func); +} + +void +storeUfsUnlink(SwapDir *SD, StoreEntry *e) +{ + debug(79, 3) ("storeUfsUnlink: fileno %08X\n", e->swap_filen); + storeUfsDirReplRemove(e); + storeUfsDirUnlinkFile(SD, e->swap_filen); +} + +/* === STATIC =========================================================== */ + +static void +storeUfsReadDone(int fd, const char *buf, int len, int errflag, void *my_data) +{ + storeIOState *sio = my_data; + ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + STRCB *callback = sio->read.callback; + void *their_data = sio->read.callback_data; + ssize_t rlen; + + debug(79, 3) ("storeUfsReadDone: dirno %d, fileno %08X, FD %d, len %d\n", + sio->swap_dirn, sio->swap_filen, fd, len); + ufsstate->flags.reading = 0; + if (errflag) { + debug(79, 3) ("storeUfsReadDone: got failure (%d)\n", errflag); + rlen = -1; + } else { + rlen = (ssize_t) len; + sio->offset += len; + } + assert(callback); + assert(their_data); + sio->read.callback = NULL; + sio->read.callback_data = NULL; + if (cbdataValid(their_data)) + callback(their_data, buf, (size_t) rlen); + cbdataUnlock(their_data); +} + +static void +storeUfsWriteDone(int fd, int errflag, size_t len, void *my_data) +{ + storeIOState *sio = my_data; + ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + debug(79, 3) ("storeUfsWriteDone: dirno %d, fileno %08X, FD %d, len %d\n", + sio->swap_dirn, sio->swap_filen, fd, len); + ufsstate->flags.writing = 0; + if (errflag) { + debug(79, 0) ("storeUfsWriteDone: got failure (%d)\n", errflag); + storeUfsIOCallback(sio, errflag); + return; + } + sio->offset += len; + if (ufsstate->flags.close_request) + storeUfsIOCallback(sio, errflag); +} + +static void +storeUfsIOCallback(storeIOState * sio, int errflag) +{ + ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate; + debug(79, 3) ("storeUfsIOCallback: errflag=%d\n", errflag); + if (ufsstate->fd > -1) { + file_close(ufsstate->fd); + store_open_disk_fd--; + } + if (cbdataValid(sio->callback_data)) + sio->callback(sio->callback_data, errflag, sio); + cbdataUnlock(sio->callback_data); + sio->callback_data = NULL; + sio->callback = NULL; + cbdataFree(sio); +} + + +/* + * We can't pass memFree() as a free function here, because we need to free + * the fsstate variable .. + */ +static void +storeUfsIOFreeEntry(void *sio, int foo) +{ + memPoolFree(ufs_state_pool, ((storeIOState *)sio)->fsstate); + memFree(sio, MEM_STORE_IO); +} --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/ufs/store_ufs.h Wed Feb 14 00:43:59 2007 @@ -0,0 +1,50 @@ +/* + * store_ufs.h + * + * Internal declarations for the ufs routines + */ + +#ifndef __STORE_UFS_H__ +#define __STORE_UFS_H__ + +struct _ufsinfo_t { + int swaplog_fd; + int l1; + int l2; + fileMap *map; + int suggest; +}; + +struct _ufsstate_t { + int fd; + struct { + unsigned int close_request:1; + unsigned int reading:1; + unsigned int writing:1; + } flags; +}; + +typedef struct _ufsinfo_t ufsinfo_t; +typedef struct _ufsstate_t ufsstate_t; + +/* The ufs_state memory pool */ +extern MemPool * ufs_state_pool; + +extern void storeUfsDirMapBitReset(SwapDir *, sfileno); +extern int storeUfsDirMapBitAllocate(SwapDir *); +extern char * storeUfsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); +extern void storeUfsDirUnlinkFile(SwapDir *, sfileno); +extern void storeUfsDirReplAdd(SwapDir *SD, StoreEntry *); +extern void storeUfsDirReplRemove(StoreEntry *); + +/* + * Store IO stuff + */ +extern STOBJCREATE storeUfsCreate; +extern STOBJOPEN storeUfsOpen; +extern STOBJCLOSE storeUfsClose; +extern STOBJREAD storeUfsRead; +extern STOBJWRITE storeUfsWrite; +extern STOBJUNLINK storeUfsUnlink; + +#endif