--------------------- PatchSet 248 Date: 2000/05/02 22:50:27 Author: hno Branch: hno-devel Tag: (none) Log: Syncronized with "devel" via "modio" Members: CONTRIBUTORS:1.1.1.3->1.1.1.3.6.1 ChangeLog:1.1.1.3.6.5->1.1.1.3.6.5.2.1 configure.in:1.1.1.3.6.7.2.2->1.1.1.3.6.7.2.3 makefile.in:1.1.1.3.6.1.2.1->1.1.1.3.6.1.2.2 auth_modules/SMB/Makefile.in:1.1.1.1.10.2->1.1.1.1.10.2.2.1 doc/Programming-Guide/prog-guide.sgml:1.1.1.2.6.1->1.1.1.2.6.1.2.1 src/Makefile.in:1.1.1.3.10.7.2.1->1.1.1.3.10.7.2.2 src/access_log.c:1.1.1.3.10.2->1.1.1.3.10.2.2.1 src/acl.c:1.1.1.3.10.5->1.1.1.3.10.5.2.1 src/cf.data.pre:1.1.1.3.10.6.2.1->1.1.1.3.10.6.2.2 src/cf_gen.c:1.1.1.3.10.2->1.1.1.3.10.2.2.1 src/client_side.c:1.1.1.3.10.7->1.1.1.3.10.7.2.1 src/comm_select.c:1.1.1.3.10.2.2.1->1.1.1.3.10.2.2.2 src/diskd.c:1.1.4.1.2.1->1.1.4.1.2.2(DEAD) src/dns_internal.c:1.1.1.1.10.4->1.1.1.1.10.4.2.1 src/enums.h:1.1.1.3.10.5->1.1.1.3.10.5.2.1 src/filemap.c:1.1.1.2.6.1->1.1.1.2.6.1.2.1 src/forward.c:1.1.1.3.10.3->1.1.1.3.10.3.2.1 src/mem.c:1.1.1.3.10.3->1.1.1.3.10.3.2.1 src/protos.h:1.1.1.3.10.8.2.1->1.1.1.3.10.8.2.2 src/pump.c:1.1.1.3.10.2->1.1.1.3.10.2.2.1 src/ssl.c:1.1.1.3.10.3->1.1.1.3.10.3.2.1 src/store.c:1.1.1.3.10.8->1.1.1.3.10.8.2.1 src/store_client.c:1.1.1.3.10.4->1.1.1.3.10.4.2.1 src/store_dir.c:1.1.1.3.10.6->1.1.1.3.10.6.2.1 src/structs.h:1.1.1.3.10.8.2.3->1.1.1.3.10.8.2.4 src/typedefs.h:1.1.1.3.10.6->1.1.1.3.10.6.2.1 src/unlinkd.c:1.1.1.3.10.3.2.1->1.1.1.3.10.3.2.2 src/fs/Makefile.in:1.1.6.2->1.1.6.2.2.1 src/fs/aufs/Makefile.in:1.1.6.3->1.1.6.3.2.1 src/fs/aufs/async_io.c:1.1.6.1.2.2->1.1.6.1.2.3 src/fs/aufs/store_asyncufs.h:1.1.6.3->1.1.6.3.2.1 src/fs/aufs/store_dir_aufs.c:1.1.6.6.2.1->1.1.6.6.2.2 src/fs/aufs/store_io_aufs.c:1.1.6.6.2.1->1.1.6.6.2.2 src/fs/coss/Makefile.in:1.1.4.2->1.1.4.2.2.1 src/fs/coss/store_dir_coss.c:1.1.4.6->1.1.4.6.2.1 src/fs/coss/store_io_coss.c:1.1.4.8.2.2->1.1.4.8.2.3 src/fs/diskd/Makefile.in:1.1->1.1.6.1 src/fs/diskd/diskd.c:1.1->1.1.6.1 src/fs/diskd/store_dir_diskd.c:1.1->1.1.6.1 src/fs/diskd/store_diskd.h:1.1->1.1.6.1 src/fs/diskd/store_io_diskd.c:1.1->1.1.6.1 src/fs/ufs/Makefile.in:1.1.6.3->1.1.6.3.2.1 src/fs/ufs/store_dir_ufs.c:1.1.6.6.2.1->1.1.6.6.2.2 src/fs/ufs/store_io_ufs.c:1.1.6.6.2.2->1.1.6.6.2.3 src/fs/ufs/store_ufs.h:1.1.6.3->1.1.6.3.2.1 Index: squid/CONTRIBUTORS =================================================================== RCS file: /cvsroot/squid-sf//squid/CONTRIBUTORS,v retrieving revision 1.1.1.3 retrieving revision 1.1.1.3.6.1 diff -u -r1.1.1.3 -r1.1.1.3.6.1 --- squid/CONTRIBUTORS 26 Jan 2000 03:25:00 -0000 1.1.1.3 +++ squid/CONTRIBUTORS 2 May 2000 22:50:27 -0000 1.1.1.3.6.1 @@ -70,6 +70,7 @@ Niall Doherty Pedro Ribeiro John Dilley + Adrian Chadd Development of this caching software is funded by the National Science Foundation (grants NCR-9616602 and NCR-9521745). Paid staff members on Index: squid/ChangeLog =================================================================== RCS file: /cvsroot/squid-sf//squid/ChangeLog,v retrieving revision 1.1.1.3.6.5 retrieving revision 1.1.1.3.6.5.2.1 diff -u -r1.1.1.3.6.5 -r1.1.1.3.6.5.2.1 --- squid/ChangeLog 20 Apr 2000 18:14:20 -0000 1.1.1.3.6.5 +++ squid/ChangeLog 2 May 2000 22:50:27 -0000 1.1.1.3.6.5.2.1 @@ -5,63 +5,72 @@ making life a lot easier for people playing around with new types of object stores. - Transfer-Encoding support by Patrick R. McManus/AppliedTheory - - Fixed a temporary memory leak on malformed server responses. - - Fixed a temporary memory leak on persistent POST requests. - myport ACL type now supported for redirector_access - - Content-Length is added on cache-hits if not sent by the origin - server. This is to improve the use of persistent connections. - - Squid now shows all FTP server messages in directory listings - - DNS retransmission tuning directives in squid.conf - - round-robin peer selection fixed to include previsously dead - peers after at most 5 minutes. - - httpd_accel_single_host for simplified accelerator setups - where a single server serves multiple domains/sites. - - Range request to servers not supporting Range could cause - bandwidth spikes. Squid now does not attempt to perform - Range processing on cache misses, only cache hits (on misses - it will be done by the server). - - Some minor issues with aclDomainComare fixed. - - HEAD on FTP objects is now supported without downloading - the whole file. - - PURGE now also purges the ipcache - - Added a cachemgr menu for inspecting what the helper processes - currently are processing. This is especially useful for the - dnsserver and redirector helpers. - - Helpers are now restarted when you rotate the logs. - - All unnedded filedescriptors are now properly closed when starting - helper processes. - The pid file is now delated late in the shutdown process, not when the shutdown stats. This is to allow scripts to monitor the pid file for Squid shutdown. - - Optimized to not block a lot of memory during cache rebuild. - - auth_regex and ident_regex ACL types - - CONNECT now honours miss_access. - - no_cache is now a full ACL check, so you safely can use any ACL type - - Log the destination IP in the hierarchy field when going direct. - More fine-grained control of the timeout for persistent connections. - - allow-miss cachee_peer option added - - Optimization to not start caching objects if it is known - that their size is larger than the allowed size. - Use UNIX domain sockets (if available) for inter-process communiation, rather than TCP/IP on the loopback interface. - - Adjustments of the ICP timeout selection algorithm, to make it - work better you have some far away parents and close-by siblings. - - Small bugfix where netdb selection could bypass never_direct. - - nonhierarchical_direct tag, for having TCP peerings support - non-hierarchical requests. The default is not to honour the - non-hierarchical types of requests. - - Corrected and improved TCP dead peer detection. Now dead-peer - detection can be done fine without using ICP, with only a small - (a few seconds) delay to the user in the event that a peer goes - dead and Squid hasn't detected this yet. - - Improved ICP dead peer detection to also work when the workload - is low, and waste less bandwidth on dead peers. Changes to Squid-2.4.DEVEL3 (): - Added Logfile module. - Added DISKD stats via cachemgr. - Added squid.conf options for DISKD magic constants. + - Added --enable-auth-modules=... configure option + - Improved ICP dead peer detection to also work when the workload + is low + - Improved TCP dead peer detection and recovery + - Squid is now a bit more persistent in trying to find a alive + parent when never_direct is used. + - nonhierarchical_direct squid.conf directive to make non-ICP + peer selection behave a bit more like ICP selection with respect + to hierarchy. + - Bugfix where netdb selection could override never_direct + - ICP timeout selection now prefers to use parents only when + calculating the dynamic timeout to compensate for common RTT + differences between parents and siblings. + - No longer starts to swap out objects which are known to be above + the maximum allowed size. + - allow-miss cache_peer option disabling the use of "only-if-cached". + Meant to be used in conjunction with icp_hit_stale. + - Delay pools tuned to allow large initial pool values + - cachemgr filesystem space information changed to show useable space + rather than raw space, and platform support somewhat extended. + - Logs destination IP in the hierarchy log tag when going direct. + (can be disabled by turning log_ip_on_direct off) + - Async-IO on linux now makes proper use of mutexes. This fixes some + odd pthread segfaults on SMP Linux machines, at a slight performance + penalty. + - %s can now be used in cache_swap_log and will be substituded with + the last path component of cache_dir. + - no_cache is now a full ACL check without, allowing most ACL types + to be used. + - The CONNECT method now obeys miss_access requirements + - proxy_auth_regex and ident_regex ACL types + - Fixed a StoreEntry memory leak during "dirty" rebuild + - Helper processes no longer hold unrelated filedescriptors open + - Helpers are now restarted when the logs are rotated + - Negatively cached DNS entries are now purged on "reload". + - PURGE now also purges the DNS cache + - HEAD on FTP objects no longer retreives the whole object + - More cleanups of the dstdomain ACL type + - Squid no longer tries to do Range internally if it is not supported + by the origin server. Doing so could cause bandwidth spikes and/or + negative hit ratio. + - httpd_accel_single_host squid.conf directive + - "round-robin" cache_peer counters are reset every 5 minutes to + compensate previously dead peers + - DNS retransmit parameters + - Show all FTP server messages + - squid.conf.default now indicates if a directive isn't enabled in + the installed binary, and what configure option to use for enabling it + - Fixed a temporary memory leak on persistent POSTs + - Fixed a temporary memory leak when the server response headers + includes NULL characters + - authenticate_ip_ttl_is_strict squid.conf option + - req_mime_type ACL type Changes to Squid-2.4.DEVEL2 (): Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.1.1.3.6.7.2.2 retrieving revision 1.1.1.3.6.7.2.3 diff -u -r1.1.1.3.6.7.2.2 -r1.1.1.3.6.7.2.3 --- squid/configure.in 23 Apr 2000 09:55:37 -0000 1.1.1.3.6.7.2.2 +++ squid/configure.in 2 May 2000 22:50:27 -0000 1.1.1.3.6.7.2.3 @@ -3,13 +3,13 @@ dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.1.1.3.6.7.2.2 2000/04/23 09:55:37 hno Exp $ +dnl $Id: configure.in,v 1.1.1.3.6.7.2.3 2000/05/02 22:50:27 hno Exp $ dnl dnl dnl AC_INIT(src/main.c) AC_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.1.1.3.6.7.2.2 $)dnl +AC_REVISION($Revision: 1.1.1.3.6.7.2.3 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AC_CONFIG_AUX_DIR(cfgaux) @@ -278,16 +278,15 @@ echo "With pthreads" SQUID_PTHREAD_LIB='$(PTHREADLIB)' CFLAGS="$CFLAGS -D_REENTRANT" -# Why is this done?? -# case "$host" in -# i386-unknown-freebsd*) -# if test "$GCC" = "yes" ; then -# if test -z "$PRESET_LDFLAGS"; then -# LDFLAGS="$LDFLAGS -pthread" -# fi -# fi -# ;; -# esac + case "$host" in + i386-unknown-freebsd*) + if test "$GCC" = "yes" ; then + if test -z "$PRESET_LDFLAGS"; then + LDFLAGS="$LDFLAGS -pthread" + fi + fi + ;; + esac fi AC_SUBST(SQUID_PTHREAD_LIB) @@ -643,46 +642,6 @@ fi ]) -dnl Enable "diskd" code -AC_ARG_ENABLE(diskd, -[ --enable-diskd Uses shared memory, message queues, and external - processes for disk I/O.], -[ if test "$enableval" = "yes" ; then - echo "DISKD enabled" - AC_DEFINE(USE_DISKD, 1) - OPT_DISKD_EXE='$(OPT_DISKD_EXE)' - AC_SUBST(OPT_DISKD_EXE) - DISKD_OBJS='diskd.o' - AC_SUBST(DISKD_OBJS) - fi -]) - -dnl Disable "unlinkd" cde -AC_ARG_ENABLE(unlinkd, -[ --disable-unlinkd Do not use unlinkd], -[ if test "$enableval" = "no" ; then - USE_UNLINKD=0 - else - USE_UNLINKD=1 - fi -],[ - # Here we should probably use some magic depending on the selected - # storage models - USE_UNLINKD=1 -]) -if test $USE_UNLINKD; then - echo "unlinkd enabled" - AC_DEFINE(USE_UNLINKD) - OPT_UNLINKD_EXE='$(OPT_UNLINKD_EXE)' - UNLINKD_OBJS='unlinkd.o' -else - echo "unlinkd disabled" - OPT_UNLINKD_EXE='' - UNLINKD_OBJS='' -fi -AC_SUBST(OPT_UNLINKD_EXE) -AC_SUBST(UNLINKD_OBJS) - # Force some compilers to use ANSI features # case "$host" in @@ -790,6 +749,7 @@ sys/socket.h \ sys/stat.h \ sys/statvfs.h \ + sys/vfs.h \ sys/syscall.h \ sys/time.h \ sys/types.h \ @@ -1593,6 +1553,7 @@ ./src/fs/ufs/Makefile \ ./src/fs/aufs/Makefile \ ./src/fs/coss/Makefile \ + ./src/fs/diskd/Makefile \ ./contrib/Makefile \ $SNMP_MAKEFILE \ ./icons/Makefile \ @@ -1600,8 +1561,13 @@ ./auth_modules/Makefile \ ./auth_modules/NCSA/Makefile \ ./auth_modules/PAM/Makefile \ +<<<<<<< configure.in ./auth_modules/SMB/Makefile ./auth_modules/MSNT/Makefile +======= + ./auth_modules/SMB/Makefile \ + ./auth_modules/MSNT/Makefile \ +>>>>>>> 1.1.1.3.2.12 ./auth_modules/getpwnam/Makefile \ ./auth_modules/LDAP/Makefile \ ) Index: squid/makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/Attic/makefile.in,v retrieving revision 1.1.1.3.6.1.2.1 retrieving revision 1.1.1.3.6.1.2.2 diff -u -r1.1.1.3.6.1.2.1 -r1.1.1.3.6.1.2.2 --- squid/makefile.in 23 Apr 2000 09:51:19 -0000 1.1.1.3.6.1.2.1 +++ squid/makefile.in 2 May 2000 22:50:27 -0000 1.1.1.3.6.1.2.2 @@ -1,4 +1,4 @@ -# $Id: makefile.in,v 1.1.1.3.6.1.2.1 2000/04/23 09:51:19 hno Exp $ +# $Id: makefile.in,v 1.1.1.3.6.1.2.2 2000/05/02 22:50:27 hno Exp $ # srcdir = @srcdir@ @@ -15,6 +15,7 @@ exec_prefix = @exec_prefix@ SUBDIRS = lib @makesnmplib@ scripts src icons errors auth_modules + noargs: all all install: makefile Index: squid/auth_modules/SMB/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/auth_modules/SMB/Attic/Makefile.in,v retrieving revision 1.1.1.1.10.2 retrieving revision 1.1.1.1.10.2.2.1 diff -u -r1.1.1.1.10.2 -r1.1.1.1.10.2.2.1 --- squid/auth_modules/SMB/Makefile.in 4 Feb 2000 18:28:13 -0000 1.1.1.1.10.2 +++ squid/auth_modules/SMB/Makefile.in 2 May 2000 22:50:28 -0000 1.1.1.1.10.2.2.1 @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.1.1.1.10.2 2000/02/04 18:28:13 hno Exp $ +# $Id: Makefile.in,v 1.1.1.1.10.2.2.1 2000/05/02 22:50:28 hno Exp $ # # Uncomment and customize the following to suit your needs: # @@ -62,7 +62,7 @@ LIBSCRIPTS = $(SMB_AUTH_HELPER) OBJS = smb_auth.o -all: $(SMB_AUTH_EXE) +all: $(LIBPROGS) $(SMB_AUTH_EXE): smb_auth.o $(CC) $(LDFLAGS) smb_auth.o -o $@ $(AUTH_LIBS) @@ -106,7 +106,7 @@ done clean: - -rm -rf *.o *pure_* core $(SMB_AUTH_EXE) + -rm -rf *.o *pure_* core $(LIBPROGS) distclean: clean -rm -f Makefile Index: squid/doc/Programming-Guide/prog-guide.sgml =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/Programming-Guide/prog-guide.sgml,v retrieving revision 1.1.1.2.6.1 retrieving revision 1.1.1.2.6.1.2.1 diff -u -r1.1.1.2.6.1 -r1.1.1.2.6.1.2.1 --- squid/doc/Programming-Guide/prog-guide.sgml 20 Apr 2000 18:14:20 -0000 1.1.1.2.6.1 +++ squid/doc/Programming-Guide/prog-guide.sgml 2 May 2000 22:50:28 -0000 1.1.1.2.6.1.2.1 @@ -2,7 +2,7 @@
Squid Programmers Guide Duane Wessels, Squid Developers -$Id: prog-guide.sgml,v 1.1.1.2.6.1 2000/04/20 18:14:20 hno Exp $ +$Id: prog-guide.sgml,v 1.1.1.2.6.1.2.1 2000/05/02 22:50:28 hno Exp $ Squid is a WWW Cache application developed by the National Laboratory @@ -54,7 +54,7 @@ Note that the Squid source changes rapidly, and some parts of this document may become out-of-date. If you find any inconsistencies, please feel free to notify . + url="mailto:squid-dev@squid-cache.org" name="the Squid Developers">. Conventions @@ -106,22 +106,22 @@ Squid can quickly locate cached objects because it keeps (in memory) a hash table of all - Objects are saved to disk in a two-level directory structure. - For each object the + For each object the Client-side requests register themselves with a Processing Client Requests +

+ To be written... + Storage Manager - -Filesystem Interface +Introduction + +

+ The Storage Manager is the glue between client and server + sides. Every object saved in the cache is allocated a + + Squid can quickly locate cached objects because it keeps + (in memory) a hash table of all + For each object the + Client-side requests register themselves with a Object storage + +

+ To be written... + +Object retreival + +

+ To be written... + + +Storage Interface Introduction @@ -673,134 +723,528 @@ We want to try out our own, customized filesystems with Squid. In order to do that, we need a well-defined interface for the bits of Squid that access the permanent storage - devices. + devices. We also require tigher control of the replacement + policy by each storage module, rather than a single global + replacement policy. -The Interface +Build structure -Data Structures +

+ The storage types live in squid/src/fs/ . Each subdirectory corresponds + to the name of the storage type. When a new storage type is implemented + configure.in must be updated to autogenerate a Makefile in + squid/src/fs/$type/ from a Makefile.in file. - + configure will take a list of storage types through the + + Each storage type must create an archive file + + Each storefs must export a function named - Every cache object that is ``opened'' for reading or writing - will have a - struct _storeIOState { - sfileno swap_file_number; - mode_t mode; - size_t st_size; /* do stat(2) after read open */ - off_t offset; /* current offset pointer */ - STIOCB *callback; - void *callback_data; - struct { - STRCB *callback; - void *callback_data; - } read; - struct { - unsigned int closing:1; /* debugging aid */ - } flags; - union { - struct { - int fd; - struct { - unsigned int close_request:1; - unsigned int reading:1; - unsigned int writing:1; - } flags; - } ufs; - } type; - }; - - - External Functions - -Object I/O - -

- These functions all relate to per-object I/O tasks: opening, - closing, reading, writing, and unlinking objects on disk. - These functions can all be found in - Note that the underlying storage system functions are - accessed through function pointers, kept in the - + + +Initialisation of a storage type + +

+ Each storage type initialises through the - struct _SwapDir { - .... - struct { - STOBJOPEN *open; - STOBJCLOSE *close; - STOBJREAD *read; - STOBJWRITE *write; - STOBJUNLINK *unlink; - } obj; - .... - }; + void + storeFsSetup_ufs(storefs_entry_t *storefs) + { + assert(!ufs_initialised); + storefs->parsefunc = storeUfsDirParse; + storefs->reconfigurefunc = storeUfsDirReconfigure; + storefs->donefunc = storeUfsDirDone; + ufs_state_pool = memPoolCreate("UFS IO State data", sizeof(ufsstate_t)); + ufs_initialised = 1; + } + + +

+ There are five function pointers in the storefs_entry which require + initialising. In this example, some protection is made against the + setup function being called twice, and a memory pool is initialised + for use inside the storage module. + +

+ Each function will be covered below. + + +done + +

+ + typedef void + STFSSHUTDOWN(void); + + +

+ This function is called whenever the storage system is to be shut down. + It should take care of deallocating any resources currently allocated. + + + + typedef void STFSPARSE(SwapDir *SD, int index, char *path); + typedef void STFSRECONFIGURE(SwapDir *SD, int index, char *path); + + +

+ These functions handle configuring and reconfiguring a storage + directory. Additional arguments from the cache_dir configuration + line can be retrieved through calls to strtok() and GetInteger(). + +

+ + + struct _SwapDir { + char *type; /* Pointer to the store dir type string */ + int cur_size; /* Current swapsize in kb */ + int low_size; /* ?? */ + int max_size; /* Maximum swapsize in kb */ + char *path; /* Path to store */ + int index; /* This entry's index into the swapDir array */ + int suggest; /* Suggestion for UFS style stores (??) */ + size_t max_objsize; /* Maximum object size for this store */ + union { /* Replacement policy-specific fields */ + #ifdef HEAP_REPLACEMENT + struct { + heap *heap; + } heap; + #endif + struct { + dlink_list list; + dlink_node *walker; + } lru; + } repl; + int removals; + int scanned; + struct { + unsigned int selected:1; /* Currently selected for write */ + unsigned int read_only:1; /* This store is read only */ + } flags; + STINIT *init; /* Initialise the fs */ + STNEWFS *newfs; /* Create a new fs */ + STDUMP *dump; /* Dump fs config snippet */ + STFREE *freefs; /* Free the fs data */ + STDBLCHECK *dblcheck; /* Double check the obj integrity */ + STSTATFS *statfs; /* Dump fs statistics */ + STMAINTAINFS *maintainfs; /* Replacement maintainence */ + STCHECKOBJ *checkob; /* Check if the fs will store an object, and get the FS load */ + /* These two are notifications */ + STREFOBJ *refobj; /* Reference this object */ + STUNREFOBJ *unrefobj; /* Unreference this object */ + STCALLBACK *callback; /* Handle pending callbacks */ + STSYNC *sync; /* Sync the directory */ + struct { + STOBJCREATE *create; /* Create a new object */ + STOBJOPEN *open; /* Open an existing object */ + STOBJCLOSE *close; /* Close an open object */ + STOBJREAD *read; /* Read from an open object */ + STOBJWRITE *write; /* Write to a created object */ + STOBJUNLINK *unlink; /* Remove the given object */ + } obj; + struct { + STLOGOPEN *open; /* Open the log */ + STLOGCLOSE *close; /* Close the log */ + STLOGWRITE *write; /* Write to the log */ + struct { + STLOGCLEANOPEN *open; /* Open a clean log */ + STLOGCLEANWRITE *write; /* Write to the log */ + void *state; /* Current state */ + } clean; + } log; + void *fsdata; /* FS-specific data */ + }; + + + + +Operation of a storage module + +

+ Squid understands the concept of multiple diverse storage directories. + Each storage directory provides a caching object store, with object + storage, retrieval, indexing and replacement. + +

+ Each open object has associated with it a + + struct _storeIOState { + sdirno swap_dirn; /* SwapDir index */ + sfileno swap_filen; /* Unique file index number */ + StoreEntry *e; /* Pointer to parent StoreEntry */ + mode_t mode; /* Mode - O_RDONLY or O_WRONLY */ + size_t st_size; /* Size of the object if known */ + off_t offset; /* current _on-disk_ offset pointer */ + STFNCB *file_callback; /* called on delayed sfileno assignments */ + STIOCB *callback; /* IO Error handler callback */ + void *callback_data; /* IO Error handler callback data */ + struct { + STRCB *callback; /* Read completion callback */ + void *callback_data; /* Read complation callback data */ + } read; + struct { + unsigned int closing:1; /* debugging aid */ + } flags; + void *fsstate; /* pointer to private fs state */ + }; + + +

+ Each + The specific filesystem operations listed in the SwapDir object are + covered below. + +initfs + +

+ + typedef void + STINIT(SwapDir *SD);

- Thus, a storage system must do something like this - when initializing its newfs + +

- SwapDir->obj.open = storeFooOpen; - SwapDir->obj.close = storeFooClose; - SwapDir->obj.read = storeFooRead; - SwapDir->obj.write = storeFooWrite; - SwapDir->obj.unlink = storeFooUnlink; + typedef void + STNEWFS(SwapDir *SD); - + Called for each configured dumpfs + +

+ + typedef void + STDUMP(StoreEntry *e, const char *path, SwapDir *SD); + + +

+ Dump the configuration of the current freefs + +

+ + typedef void + STFREE(SwapDir *SD); + + +

+ Free the fsdata/. + + +doublecheckfs + +

+ + typedef int + STDBLCHECK(SwapDir *SD, StoreEntry *e); + + +

+ Double-check the given object for validity. Called during rebuild if + the '-S' flag is given to squid on the command line. Returns 1 if the + object is indeed valid, and 0 if the object is found invalid. + + +statfs + +

+ + typedef void + STSTATFS(SwapDir *SD, StoreEntry *e); + + +

+ Called to retrieve filesystem statistics, such as usage, load and + errors. The information should be appended to the passed + maintainfs + +

+ + typedef void + STMAINTAINFS(SwapDir *SD); + + +

+ Called periodically to replace objects. The active replacement policy + should be used to timeout unused objects in order to make room for + new objects. + +callback + +

+ + typedef void + STCALLBACK(SwapDir *SD); + + +

+ This function is called inside the comm_select/comm_poll loop to handle + any callbacks pending. + + +sync + +

+ + typedef void + STSYNC(SwapDir *SD); + + +

+ This function is called whenever a sync to disk is required. This + function should not return until all pending data has been flushed to + disk. + + +parse/reconfigure + +

+ +checkobj + +

+ + typedef int + STCHECKOBJ(SwapDir *SD, const StoreEntry *e); + + +

+ Called by referenceobj + +

+ + typedef void + STREFOBJ(SwapDir *SD, StoreEntry *e); + + +

+ Called whenever an object is locked by unreferenceobj + +

+ + typedef void + STUNREFOBJ(SwapDir *SD, StoreEntry *e); + + +

+ Called whenever the object is unlocked by createobj + +

+ + typedef storeIOState * + STOBJCREATE(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *io_callback, void *io_callback_data); + + +

+ Create an object in the + The IO callback should be called when an error occurs and when the + object is closed. Once the IO callback is called, the + openobj + +

+ + typedef storeIOState * + STOBJOPEN(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *io_callback, void *io_callback_data); + + +

+ Open the closeobj + +

+ + typedef void + STOBJCLOSE(SwapDir *SD, storeIOState *sio); + + +

+ Close an opened object. The readobj + +

+ + typedef void + STOBJREAD(SwapDir *SD, storeIOState *sio, char *buf, size_t size, off_t offset, STRCB *read_callback, void *read_callback_data); + + +

+ Read part of the object of into + If a read operation fails, the filesystem layer notifies the + calling module by calling the writeobj + +

+ + typedef void + STOBJWRITE(SwapDir *SD, storeIOState *sio, char *buf, size_t size, off_t offset, FREE *freefunc); + + +

+ Write the given block of data to the given store object. + If a write operation fails, the filesystem layer notifies the + calling module by calling the unlinkobj + +

+ + typedef void STOBJUNLINK(SwapDir *, StoreEntry *); + + +

+ Remove the Store IO calls + +

+ These routines are used inside the storage manager to create and + retrieve objects from a storage directory. + +storeCreate()

storeIOState * - storeOpen(sfileno f, mode_t mode, STIOCB *callback, void *callback_data) + storeCreate(StoreEntry *e, STIOCB *file_callback, STIOCB *close_callback, void * callback_data)

- - storeOpen()

- void - storeClose(storeIOState *sio) + storeIOState * + storeOpen(StoreEntry *e, STFNCB * file_callback, STIOCB * callback, void *callback_data)

- + + storeRead() + +

void storeRead(storeIOState *sio, char *buf, size_t size, off_t offset, STRCB *callback, void *callback_data) @@ -842,7 +1297,7 @@

- The caller is responsible for allocating and freeing +storeWrite() + +

void storeWrite(storeIOState *sio, char *buf, size_t size, off_t offset, FREE *free_func) @@ -862,7 +1318,7 @@

+storeUnlink() + +

- void - storeUnlink(sfileno f) + void + storeUnlink(StoreEntry *e)

@@ -888,21 +1345,28 @@ does not need to be opened first. The filesystem layer will remove the object if it exists on the disk. - +storeOffset() + +

- off_t - storeOffset(storeIOState *sio) + off_t storeOffset(storeIOState *sio) + +

- Returns the current byte-offset of the cache object on - disk. For read-objects, this is the offset after the last - successful disk read operation. For write-objects, it is - the offset of the last successful disk write operation. + offset/ refers to the ondisk offset, or undefined + results will occur. For reads, this returns the current offset of + sucessfully read data, not including queued reads. + -Callbacks + + @@ -929,7 +1393,7 @@ Once the The @@ -946,43 +1410,11 @@ called if the read operation is successful. If it fails, then the Config file parsing - -

- There are three functions relating to the Squid configuration - file: parsing, dumping, and freeing. +State Logging

- The parse function is called at startup, and during a reconfigure, - for a - The ``dump'' function is used to output a configuration - file from the in-memory configuration structure. It is - called with a - The free function is called during a reconfigure (and at - exit) to free up (or un-initialize) any memory or structures - associated with the configuration line. The Filesystem Startup, Initialization, and State Logging - -

- These functions deal with initializing, state - logging, and related tasks for a squid storage system. + These functions deal with state + logging and related tasks for a squid storage system. These functions are used (called) in @@ -1010,36 +1442,7 @@ }; - - - void - STINIT(SwapDir *); - - -

- The - - void - STNEWFS(SwapDir *); - - -

- The @@ -1060,7 +1463,7 @@ the state log and then re-opens them. A @@ -1074,7 +1477,7 @@ the open state-holding log files (if any) for the storage system. - @@ -1089,7 +1492,7 @@ This feature may not be required by some storage systems and can be implemented as a null-function (no-op). - @@ -1111,7 +1514,7 @@ keep state information for the clean-writing process, but should not be accessed by upper layers. - @@ -1130,9 +1533,301 @@ to become the official state-holding log. +Replacement policy implementation + +

+The replacement policy can be updated during STOBJREAD/STOBJWRITE/STOBJOPEN/ +STOBJCLOSE as well as STREFOBJ and STUNREFOBJ. Care should be taken to +only modify the relevant replacement policy entries in the StoreEntry. +The responsibility of replacement policy maintainence has been moved into +each SwapDir so that the storage code can have tight control of the +replacement policy. Cyclic filesystems such as COSS require this tight +coupling between the storage layer and the replacement policy. + + +Future removal policy + +

+ (replaces the above Replace policy) + +

+ The removal policy is responsible for determining in which order + objects are deleted when Squid needs to reclaim space for new objects. + Such a policy is used by a object storage for maintaining the stored + objects and determining what to remove to reclaim space for new objects. + (together they implements a replacement policy) + +API +

+ It is implemented as a modular API where a storage directory or + memory creates a policy of choice for maintaining it's objects, + and modules registerering to be used by this API. + +createRemovalPolicy() + +

+ + RemovalPolicy policy = createRemovalPolicy(cons char *type, cons char *args) + + +

+ Creates a removal policy instance where object priority can be + maintained + +

+ The returned RemovalPolicy instance is cbdata registered + +policy.Free() + +

+ + policy->Free(RemovalPolicy *policy) + + +

+ Destroys the policy instance and frees all related memory. + +policy.Add() + +

+ + policy->Add(RemovalPolicy *policy, StoreEntry *, RemovalPolicyNode *node) + + +

+ Adds a StoreEntry to the policy instance. + +

+ datap is a pointer to where policy specific data can be stored + for the store entry, currently the size of one (void *) pointer. + +policy.Remove() + +

+ + policy->Remove(RemovalPolicy *policy, StoreEntry *, RemovalPolicyNode *node) + + +

+ Removes a StoreEntry from the policy instance out of + policy order. For example when an object is replaced + by a newer one or is manually purged from the store. + +

+ datap is a pointer to where policy specific data is stored + for the store entry, currently the size of one (void *) pointer. + +policy.Referenced() + +

+ + policy->Referenced(RemovalPolicy *policy, const StoreEntry *, RemovalPolicyNode *node) + + +

+ Tells the policy that a StoreEntry has been referenced. + +

+ datap is a pointer to where policy specific data is stored + for the store entry, currently the size of one (void *) pointer. + +policy.WalkInit() + +

+ + RemovalPolicyWalker walker = policy->WalkInit(RemovalPolicy *policy) + + +

+ Initiates a walk of all objects in the policy instance. + The objects is returned in an order suitable for using + as reinsertion order when rebuilding the policy. + +

+ The returned RemovalPolicyWalker instance is cbdata registered + +

+ Note: The walk must be performed as an atomic operation + with no other policy actions interveaning, or the outcome + will be undefined. + +walker.Next() + +

+ + const StoreEntry *entry = walker->Next(RemovalPolicyWalker *walker) + + +

+ Gets the next object in the walk chain + +

+ Return NULL when there is no further objects + +walker.Done() + +

+ + walker->Done(RemovalPolicyWalker *walker) + + +

+ Finishes a walk of the maintaned objects, destroys + walker. + +policy.PurgeInit() + +

+ + RemovalPurgeWalker purgewalker = policy->PurgeInit(RemovalPolicy *policy, int max_scan) + + +

+ Initiates a search for removal candidates. Seach depth is indicated + by max_scan. + +

+ The returned RemovalPurgeWalker instance is cbdata registered + +

+ Note: The walk must be performed as an atomic operation + with no other policy actions interveaning, or the outcome + will be undefined. + +purgewalker.Next() + +

+ + StoreEntry *entry = purgewalker->Next(RemovalPurgeWalker *purgewalker) + + +

+ Gets the next object to purge. The purgewalker will remove each + returned object from the policy. + +

It is the policys responsibility to verify that the object + isn't locked or otherwise prevented from being removed. What this + means is that the policy must not return objects where + storeEntryLocked() is true. + +

+ Return NULL when there is no further purgeable objects in the policy. + +purgewalker.Done() + +

+ + purgewalker->Done(RemovalPurgeWalker *purgewalker) + + +

+ Finishes a walk of the maintaned objects, destroys + walker and restores the policy to it's normal state. + +Future removal policy implementation + +Source layout + +

+ Policy implementations resides in src/repl/<name>/, and a make in + such a directory must result in a object archive src/repl/<name>.a + containing all the objects implementing the policy. + +Internal structures + +RemovalPolicy + +

+ + typedef struct _RemovalPolicy RemovalPolicy; + struct _RemovalPolicy { + char *_type; + void *_data; + void (*add)(RemovalPolicy *policy, StoreEntry *); + ... /* see the API definition above */ + }; + + +

+ The _type member is mainly for debugging and diagnostics purposes, and + should be a pointer to the name of the policy (same name as used for + creation) + +

+ The _data member is for storing policy specific information. + +RemovalPolicyWalker + +

+ + typedef struct _RemovalPolicyWalker RemovalPolicyWalker; + struct _RemovalPolicyWalker { + RemovalPolicy *_policy; + void *_data; + StoreEntry *(*next)(RemovalPolicyWalker *); + ... /* see the API definition above */ + }; + + +RemovalPolicyNode + +

+ + typedef struct _RemovalPolicyNode RemovalPolicyNode; + struct _RemovalPolicyNode { + void *data; + }; + + + Stores policy specific information about a entry. Currently + there is only space for a single pointer, but plans are to + maybe later provide more space here to allow simple policies + to store all their data "inline" to preserve some memory. + +Policy registration + +

+ Policies are automatically registered in the Squid binary from the + policy selection made by the user building Squid. In the future this + might get extended to support loadable modules. All registered + policies are available to object stores which wishes to use them. + +Policy instance creation + +

+ Each policy must implement a "create/new" function " + It should also populate the _data member with a pointer to policy + specific data. + +

+ Prior to returning the created instance must be registered as + callback-data by calling cbdataAdd(). + +Walker + +

+ When a walker is created the policy populates it with at least the API + methods supported. Currently all API calls are mandatory, but the + policy implementation must make sure to NULL fill the structure prior + to populating it in order to assure future API compability. + +

+ Prior to returning the created instance must be registered as + callback-data by calling cbdataAdd(). + Forwarding Selection +

+ To be written... + IP Cache and FQDN Cache @@ -1217,30 +1912,72 @@ Server Protocols HTTP + +

+ To be written... + FTP + +

+ To be written... + Gopher + +

+ To be written... + Wais + +

+ To be written... + SSL + +

+ To be written... + Passthrough +

+ To be written... + Timeouts +

+ To be written... + Events +

+ To be written... + Access Controls +

+ To be written... + ICP +

+ To be written... + Network Measurement Database +

+ To be written... + Error Pages +

+ To be written... + + Callback Data Database

@@ -1309,6 +2046,9 @@ Cache Manager +

+ To be written... + HTTP Headers Index: squid/src/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/Makefile.in,v retrieving revision 1.1.1.3.10.7.2.1 retrieving revision 1.1.1.3.10.7.2.2 diff -u -r1.1.1.3.10.7.2.1 -r1.1.1.3.10.7.2.2 --- squid/src/Makefile.in 23 Apr 2000 09:55:37 -0000 1.1.1.3.10.7.2.1 +++ squid/src/Makefile.in 2 May 2000 22:50:28 -0000 1.1.1.3.10.7.2.2 @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.1.1.3.10.7.2.1 2000/04/23 09:55:37 hno Exp $ +# $Id: Makefile.in,v 1.1.1.3.10.7.2.2 2000/05/02 22:50:28 hno Exp $ # # Uncomment and customize the following to suit your needs: # @@ -18,6 +18,8 @@ srcdir = @srcdir@ VPATH = @srcdir@ +SUBDIRS = fs + # Gotta love the DOS legacy # SQUID_EXE = squid$(exec_suffix) @@ -27,8 +29,6 @@ UNLINKD_EXE = @OPT_UNLINKD_EXE@ PINGER_EXE = pinger$(exec_suffix) CACHEMGR_EXE = cachemgr$(cgi_suffix) -OPT_DISKD_EXE = diskd$(exec_suffix) -DISKD_EXE = @OPT_DISKD_EXE@ DEFAULT_PREFIX = $(prefix) DEFAULT_CONFIG_FILE = $(sysconfdir)/squid.conf @@ -100,7 +100,6 @@ debug.o \ @DELAY_OBJS@ \ disk.o \ - diskd.o \ dns.o \ dns_internal.o \ errorpage.o \ @@ -159,7 +158,6 @@ stmem.o \ store.o \ store_io.o \ - @DISKD_OBJS@ \ store_client.o \ store_digest.o \ store_dir.o \ @@ -220,12 +218,6 @@ $(DNSSERVER_EXE): dnsserver.o $(CC) -o $@ $(LDFLAGS) dnsserver.o $(DNSSERVER_LIBS) -$(DISKD_EXE): diskd-daemon.o - $(CC) -o $@ $(LDFLAGS) diskd-daemon.o $(DISKD_LIBS) - -diskd-daemon.o: diskd.c - $(CC) -c $(CFLAGS) -DDISKD_DAEMON=1 $(srcdir)/diskd.c -o $@ - $(CACHEMGR_EXE): cachemgr.o $(CC) -o $@ $(LDFLAGS) cachemgr.o $(CLIENT_LIBS) @@ -254,7 +246,7 @@ cf_gen.o: cf_gen.c cf_gen_defines.h Makefile defines.h ../include/autoconf.h -cf_gen_defines.h: cf.data.pre cf_gen_defines +cf_gen_defines.h: $(srcdir)/cf_gen_defines $(srcdir)/cf.data.pre awk -f $(srcdir)/cf_gen_defines <$(srcdir)/cf.data.pre >cf_gen_defines.h cf.data: cf.data.pre Makefile @@ -318,6 +310,10 @@ # Michael Lupp wants to know about additions # to the install target. install: all install-mkdirs + @for dir in $(SUBDIRS); do \ + echo "Making $@ in $$dir..."; \ + (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \ + done @for f in $(PROGS); do \ if test -f $(bindir)/$$f; then \ echo $(MV) $(bindir)/$$f $(bindir)/-$$f; \ @@ -388,13 +384,19 @@ -rm -rf *.o *pure_* core $(PROGS) $(UTILS) $(CGIPROGS) $(SUID_UTILS) -rm -f cf_gen cf_gen_defines.h cf_parser.c cf.data globals.c string_arrays.c -rm -f store_modules.c - -sh -c "cd fs && $(MAKE) clean" + @for dir in $(SUBDIRS); do \ + echo "Making $@ in $$dir..."; \ + (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \ + done distclean: clean -rm -f Makefile squid.conf squid.conf.pre -rm -f Makefile.bak -rm -f tags - -sh -c "cd fs && $(MAKE) distclean" + @for dir in $(SUBDIRS); do \ + echo "Making $@ in $$dir..."; \ + (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \ + done tags: ctags *.[ch] ../include/*.h ../lib/*.[ch] Index: squid/src/access_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/access_log.c,v retrieving revision 1.1.1.3.10.2 retrieving revision 1.1.1.3.10.2.2.1 diff -u -r1.1.1.3.10.2 -r1.1.1.3.10.2.2.1 --- squid/src/access_log.c 20 Apr 2000 18:14:21 -0000 1.1.1.3.10.2 +++ squid/src/access_log.c 2 May 2000 22:50:28 -0000 1.1.1.3.10.2.2.1 @@ -1,7 +1,6 @@ - /* - * $Id: access_log.c,v 1.1.1.3.10.2 2000/04/20 18:14:21 hno Exp $ + * $Id: access_log.c,v 1.1.1.3.10.2.2.1 2000/05/02 22:50:28 hno Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -37,7 +36,6 @@ #include "squid.h" -char *log_quote(const char *header); static void accessLogSquid(AccessLogEntry * al); static void accessLogCommon(AccessLogEntry * al); static Logfile *logfile = NULL; Index: squid/src/acl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/acl.c,v retrieving revision 1.1.1.3.10.5 retrieving revision 1.1.1.3.10.5.2.1 diff -u -r1.1.1.3.10.5 -r1.1.1.3.10.5.2.1 --- squid/src/acl.c 22 Apr 2000 09:18:51 -0000 1.1.1.3.10.5 +++ squid/src/acl.c 2 May 2000 22:50:28 -0000 1.1.1.3.10.5.2.1 @@ -1,6 +1,6 @@ /* - * $Id: acl.c,v 1.1.1.3.10.5 2000/04/22 09:18:51 hno Exp $ + * $Id: acl.c,v 1.1.1.3.10.5.2.1 2000/05/02 22:50:28 hno Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Duane Wessels @@ -1152,17 +1152,20 @@ return 0; /* NOTREACHED */ } } else { - /* user has switched to another IP addr */ - debug(28, 1) ("aclMatchProxyAuth: user '%s' has changed IP address\n", user); - if (!Config.authenticateIpTTL > 0) { + if (Config.onoff.authenticateIpTTLStrict) { + /* Access from some other IP address than the one owning + * this user ID. Deny access + */ + debug(28, 1) ("aclMatchProxyAuth: user '%s' tries to use multple IP addresses!\n", user); + return 0; + } else { + /* user has switched to another IP addr */ + debug(28, 1) ("aclMatchProxyAuth: user '%s' has changed IP address\n", user); /* remove this user from the hash, making him unknown */ hash_remove_link(proxy_auth_cache, (hash_link *) auth_user); aclFreeProxyAuthUser(auth_user); /* require the user to reauthenticate */ return -2; - } else { - /* Strict TTL is set. Deny the "other" user access */ - return 0; } } } else { @@ -2305,9 +2308,14 @@ * * NOTE: Linux code by David Luyer . * Original (BSD-specific) code no longer works. + * Solaris code by R. Gancarz */ +#ifdef _SQUID_SOLARIS_ +#include +#else #include +#endif #ifdef _SQUID_LINUX_ #include #include @@ -2382,10 +2390,10 @@ /***************/ /* aclMatchArp */ /***************/ -#ifdef _SQUID_LINUX_ static int aclMatchArp(void *dataptr, struct in_addr c) { +#if defined(_SQUID_LINUX_) struct arpreq arpReq; struct sockaddr_in ipAddr; unsigned char ifbuffer[sizeof(struct ifreq) * 64]; @@ -2499,6 +2507,48 @@ * exist on multiple interfaces? */ } +#elif defined(_SQUID_SOLARIS_) + struct arpreq arpReq; + struct sockaddr_in ipAddr; + unsigned char ifbuffer[sizeof(struct ifreq) * 64]; + struct ifconf ifc; + struct ifreq *ifr; + int offset; + splayNode **Top = dataptr; + /* + * Set up structures for ARP lookup with blank interface name + */ + ipAddr.sin_family = AF_INET; + ipAddr.sin_port = 0; + ipAddr.sin_addr = c; + memset(&arpReq, '\0', sizeof(arpReq)); + memcpy(&arpReq.arp_pa, &ipAddr, sizeof(struct sockaddr_in)); + /* Query ARP table */ + if (ioctl(HttpSockets[0], SIOCGARP, &arpReq) != -1) { + /* + * Solaris (at least 2.6/x86) does not use arp_ha.sa_family - + * it returns 00:00:00:00:00:00 for non-ethernet media + */ + if (arpReq.arp_ha.sa_data[0] == 0 && + arpReq.arp_ha.sa_data[1] == 0 && + arpReq.arp_ha.sa_data[2] == 0 && + arpReq.arp_ha.sa_data[3] == 0 && + arpReq.arp_ha.sa_data[4] == 0 && + arpReq.arp_ha.sa_data[5] == 0) + return 0; + debug(28, 4) ("Got address %02x:%02x:%02x:%02x:%02x:%02x\n", + arpReq.arp_ha.sa_data[0] & 0xff, arpReq.arp_ha.sa_data[1] & 0xff, + arpReq.arp_ha.sa_data[2] & 0xff, arpReq.arp_ha.sa_data[3] & 0xff, + arpReq.arp_ha.sa_data[4] & 0xff, arpReq.arp_ha.sa_data[5] & 0xff); + /* Do lookup */ + *Top = splay_splay(&arpReq.arp_ha.sa_data, *Top, aclArpCompare); + debug(28, 3) ("aclMatchArp: '%s' %s\n", + inet_ntoa(c), splayLastResult ? "NOT found" : "found"); + return (0 == splayLastResult); + } +#else + WRITE ME; +#endif /* * Address was not found on any interface */ @@ -2509,6 +2559,7 @@ static int aclArpCompare(const void *a, const void *b) { +#if defined(_SQUID_LINUX_) const unsigned short *d1 = a; const unsigned short *d2 = b; if (d1[0] != d2[0]) @@ -2517,22 +2568,28 @@ return (d1[1] > d2[1]) ? 1 : -1; if (d1[2] != d2[2]) return (d1[2] > d2[2]) ? 1 : -1; - return 0; -} +#elif defined(_SQUID_SOLARIS_) + const unsigned char *d1 = a; + const unsigned char *d2 = b; + if (d1[0] != d2[0]) + return (d1[0] > d2[0]) ? 1 : -1; + if (d1[1] != d2[1]) + return (d1[1] > d2[1]) ? 1 : -1; + if (d1[2] != d2[2]) + return (d1[2] > d2[2]) ? 1 : -1; + if (d1[3] != d2[3]) + return (d1[3] > d2[3]) ? 1 : -1; + if (d1[4] != d2[4]) + return (d1[4] > d2[4]) ? 1 : -1; + if (d1[5] != d2[5]) + return (d1[5] > d2[5]) ? 1 : -1; #else - -static int -aclMatchArp(void *dataptr, struct in_addr c) -{ - WRITE ME; -} - -static int -aclArpCompare(const void *data, splayNode * n) -{ WRITE ME; +#endif + return 0; } +#if UNUSED_CODE /********************************************************************** * This is from the pre-splay-tree code for BSD * I suspect the Linux approach will work on most O/S and be much Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.1.1.3.10.6.2.1 retrieving revision 1.1.1.3.10.6.2.2 diff -u -r1.1.1.3.10.6.2.1 -r1.1.1.3.10.6.2.2 --- squid/src/cf.data.pre 23 Apr 2000 09:55:37 -0000 1.1.1.3.10.6.2.1 +++ squid/src/cf.data.pre 2 May 2000 22:50:28 -0000 1.1.1.3.10.6.2.2 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.1.1.3.10.6.2.1 2000/04/23 09:55:37 hno Exp $ +# $Id: cf.data.pre,v 1.1.1.3.10.6.2.2 2000/05/02 22:50:28 hno Exp $ # # # SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -712,6 +712,16 @@ information that Squid-specific log analyzers use. DOC_END +NAME: log_ip_on_direct +COMMENT: on|off +TYPE: onoff +DEFAULT: on +LOC: Config.onoff.log_ip_on_direct +DOC_START + Log the destination IP address in the hierarchy log tag when going + direct. Earlier Squid versions logged the hostname here. If you + prefer the old way set this to off. +DOC_END NAME: mime_table TYPE: string @@ -880,6 +890,7 @@ TYPE: onoff DEFAULT: off LOC: Config.onoff.res_defnames +IFDEF: USE_DNSSERVERS DOC_START Normally the 'dnsserver' disables the RES_DEFNAMES resolver option (see res_init(3)). This prevents caches in a hierarchy @@ -1027,9 +1038,22 @@ until the TTL has expired. The default is 0 to disable the check. Recommended value - if you have dialup users are no more than 60 seconds. If - all your users are stationary then higher values may be - used, or even strict checking. + if you have dialup users are no more than 60 seconds to allow + the user to redial without hassle. If all your users are + stationary then higher values may be used. + + See also authenticate_ip_ttl_is_strict +DOC_END + +NAME: authenticate_ip_ttl_is_strict +TYPE: onoff +LOC: Config.onoff.authenticateIpTTLStrict +DEFAULT: on +DOC_START + This option makes authenticate_ip_ttl a bit stricted. With this + enabled authenticate_ip_ttl will deny all access from other IP + addresses until the TTL has expired, and the IP address "owning" + the userid will not be forced to reauthenticate. DOC_END COMMENT_START @@ -1453,9 +1477,6 @@ DOC_START Timeout for idle persistent connections to clients, servers and other proxies. - - Persistent connections will be disabled if this is less than 10 - seconds. DOC_END NAME: ident_timeout @@ -3125,7 +3146,9 @@ TYPE: int LOC: Config.diskd.magic2 DEFAULT: 72 -DOC_NONE +DOC_START + Macic constants for the diskd daemon.. +DOC_END EOF Index: squid/src/cf_gen.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf_gen.c,v retrieving revision 1.1.1.3.10.2 retrieving revision 1.1.1.3.10.2.2.1 diff -u -r1.1.1.3.10.2 -r1.1.1.3.10.2.2.1 --- squid/src/cf_gen.c 20 Apr 2000 18:14:22 -0000 1.1.1.3.10.2 +++ squid/src/cf_gen.c 2 May 2000 22:50:28 -0000 1.1.1.3.10.2.2.1 @@ -1,5 +1,6 @@ + /* - * $Id: cf_gen.c,v 1.1.1.3.10.2 2000/04/20 18:14:22 hno Exp $ + * $Id: cf_gen.c,v 1.1.1.3.10.2.2.1 2000/05/02 22:50:28 hno Exp $ * * DEBUG: none * AUTHOR: Max Okumoto Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.1.1.3.10.7 retrieving revision 1.1.1.3.10.7.2.1 diff -u -r1.1.1.3.10.7 -r1.1.1.3.10.7.2.1 --- squid/src/client_side.c 21 Apr 2000 22:17:08 -0000 1.1.1.3.10.7 +++ squid/src/client_side.c 2 May 2000 22:50:28 -0000 1.1.1.3.10.7.2.1 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.1.1.3.10.7 2000/04/21 22:17:08 hno Exp $ + * $Id: client_side.c,v 1.1.1.3.10.7.2.1 2000/05/02 22:50:28 hno Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -200,7 +200,8 @@ #if DELAY_POOLS delaySetStoreClient(e, h, delayClient(h->request)); #endif - storeClientCopy(e, 0, 0, 0, NULL, clientSendMoreData, h); + storeClientCopy(e, 0, 0, CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, h); return e; } @@ -374,8 +375,8 @@ storeClientCopy(entry, http->out.offset, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), clientHandleIMSReply, http); } @@ -431,9 +432,11 @@ http_status status; debug(33, 3) ("clientHandleIMSReply: %s, %d bytes\n", url, (int) size); if (entry == NULL) { + memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + memFree(buf, MEM_CLIENT_SOCK_BUF); return; } mem = entry->mem_obj; @@ -448,7 +451,7 @@ entry = http->entry = http->old_entry; } else if (STORE_PENDING == entry->store_status && 0 == status) { debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); - if (size >= SM_PAGE_SIZE) { + if (size >= CLIENT_SOCK_SZ) { /* will not get any bigger than that */ debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); /* use old entry, this repeats the code abovez */ @@ -461,8 +464,8 @@ storeClientCopy(entry, http->out.offset + size, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + buf, clientHandleIMSReply, http); return; @@ -510,8 +513,8 @@ storeClientCopy(entry, http->out.offset, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + buf, clientSendMoreData, http); } else { @@ -2013,10 +2016,12 @@ request_t *r = http->request; debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); if (http->entry == NULL) { + memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: request aborted\n"); return; } else if (size < 0) { /* swap in failure */ + memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); http->log_type = LOG_TCP_SWAPFAIL_MISS; if ((e = http->entry)) { @@ -2036,16 +2041,18 @@ * punt to clientProcessMiss. */ if (e->mem_status == IN_MEMORY || e->store_status == STORE_OK) { + memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); - } else if (size == SM_PAGE_SIZE && http->out.offset == 0) { + } else if (size == CLIENT_SOCK_SZ && http->out.offset == 0) { + memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); } else { debug(33, 3) ("clientCacheHit: waiting for HTTP reply headers\n"); storeClientCopy(e, http->out.offset + size, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + buf, clientCacheHit, http); } @@ -2109,6 +2116,7 @@ http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } + memFree(buf, MEM_CLIENT_SOCK_BUF); } else if (r->flags.ims) { /* * Handle If-Modified-Since requests from the client @@ -2116,6 +2124,7 @@ if (mem->reply->sline.status != HTTP_OK) { debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", mem->reply->sline.status); + memFree(buf, MEM_CLIENT_SOCK_BUF); http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } else if (modifiedSince(e, http->request)) { @@ -2124,6 +2133,7 @@ } else { MemBuf mb = httpPacked304Reply(e->mem_obj->reply); http->log_type = LOG_TCP_IMS_HIT; + memFree(buf, MEM_CLIENT_SOCK_BUF); storeUnregister(e, http); storeUnlockObject(e); e = clientCreateStoreEntry(http, http->request->method, null_request_flags); @@ -2392,7 +2402,7 @@ MemBuf mb; ssize_t check_size = 0; debug(33, 5) ("clientSendMoreData: %s, %d bytes\n", http->uri, (int) size); - assert(size <= SM_PAGE_SIZE); /* We work in SM_PAGE_SIZE chunks now */ + assert(size <= CLIENT_SOCK_SZ); assert(http->request != NULL); dlinkDelete(&http->active, &ClientActiveRequests); dlinkAdd(http, &http->active, &ClientActiveRequests); @@ -2401,18 +2411,22 @@ if (conn->chr != http) { /* there is another object in progress, defer this one */ debug(33, 1) ("clientSendMoreData: Deferring %s\n", storeUrl(entry)); + memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); + memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size < 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); + memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size == 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); + memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (http->out.offset == 0) { @@ -2442,14 +2456,13 @@ http->range_iter.prefix_size = rep->hdr_sz; debug(33, 3) ("clientSendMoreData: Appending %d bytes after %d bytes of headers\n", body_size, rep->hdr_sz); - } else if (size < SM_PAGE_SIZE && entry->store_status == STORE_PENDING) { + } else if (size < CLIENT_SOCK_SZ && entry->store_status == STORE_PENDING) { /* wait for more to arrive */ - /* XXX With no copy :) */ storeClientCopy(entry, http->out.offset + size, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + buf, clientSendMoreData, http); return; @@ -2523,6 +2536,8 @@ assert(check_size == size); /* write */ comm_write_mbuf(fd, mb, clientWriteComplete, http); + /* if we don't do it, who will? */ + memFree(buf, MEM_CLIENT_SOCK_BUF); } static void @@ -2560,8 +2575,8 @@ storeClientCopy(entry, http->out.offset, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, http); } @@ -2651,8 +2666,8 @@ storeClientCopy(entry, http->out.offset, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, http); } @@ -2813,8 +2828,8 @@ storeClientCopy(http->entry, http->out.offset, http->out.offset, - 0, - NULL, + CLIENT_SOCK_SZ, + memAllocate(MEM_CLIENT_SOCK_BUF), clientCacheHit, http); } else { Index: squid/src/comm_select.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/comm_select.c,v retrieving revision 1.1.1.3.10.2.2.1 retrieving revision 1.1.1.3.10.2.2.2 diff -u -r1.1.1.3.10.2.2.1 -r1.1.1.3.10.2.2.2 --- squid/src/comm_select.c 22 Apr 2000 20:02:55 -0000 1.1.1.3.10.2.2.1 +++ squid/src/comm_select.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.2.2.2 @@ -1,6 +1,6 @@ /* - * $Id: comm_select.c,v 1.1.1.3.10.2.2.1 2000/04/22 20:02:55 hno Exp $ + * $Id: comm_select.c,v 1.1.1.3.10.2.2.2 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 5 Socket Functions * @@ -658,8 +658,6 @@ #if !ALARM_UPDATES_TIME getCurrentTime(); #endif - /* Handle any fs callbacks that need doing */ - storeDirCallback(); #if DELAY_POOLS FD_ZERO(&slowfds); #endif --- squid/src/diskd.c Wed Feb 14 00:43:54 2007 +++ /dev/null Wed Feb 14 00:43:41 2007 @@ -1,907 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 81 Async disk IO routines - * AUTHOR: Duane Wessels - * - * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from the - * Internet community. Development is led by Duane Wessels of the - * National Laboratory for Applied Network Research and funded by the - * National Science Foundation. Squid is Copyrighted (C) 1998 by - * the Regents of the University of California. Please see the - * COPYRIGHT file for full details. Squid incorporates software - * developed and/or copyrighted by other sources. Please see the - * CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - - -#include "config.h" -#include "squid.h" - - -#include -#include -#include - -#undef assert -#include - -enum { - _MQD_NOP, - _MQD_OPEN, - _MQD_CLOSE, - _MQD_READ, - _MQD_WRITE, - _MQD_UNLINK -}; - -typedef struct _diomsg { - mtyp_t mtype; - int id; - int seq_no; - void *callback_data; - int size; - int offset; - int status; - int shm_offset; -} diomsg; - -static const int msg_snd_rcv_sz = sizeof(diomsg) - sizeof(mtyp_t); - -#if DISKD_DAEMON - -#define STDERR_DEBUG 0 - -typedef struct _file_state file_state; - -struct _file_state { - void *key; - file_state *next; - int id; - int fd; - off_t offset; -}; - -static hash_table *hash = NULL; -static pid_t mypid; -static char *shmbuf; - -static int -do_open(diomsg * r, int len, const char *buf) -{ - int fd; - file_state *fs; - /* - * note r->offset holds open() flags - */ - fd = open(buf, r->offset, 0600); - if (fd < 0) { - fprintf(stderr, "%d %s: ", (int) mypid, buf); - perror("open"); - return -errno; - } - fs = xcalloc(1, sizeof(*fs)); - fs->id = r->id; - fs->key = &fs->id; /* gack */ - fs->fd = fd; - hash_join(hash, (hash_link *) fs); -#if STDERR_DEBUG - fprintf(stderr, "%d OPEN id %d, FD %d, fs %p\n", - (int) mypid, - fs->id, - fs->fd, - fs); -#endif - return fd; -} - -static int -do_close(diomsg * r, int len) -{ - int fd; - file_state *fs; - fs = (file_state *) hash_lookup(hash, &r->id); - if (NULL == fs) { - errno = EBADF; - fprintf(stderr, "%d CLOSE id %d: ", (int) mypid, r->id); - perror("do_close"); - return -EBADF; - } - fd = fs->fd; - hash_remove_link(hash, (hash_link *) fs); -#if STDERR_DEBUG - fprintf(stderr, "%d CLOSE id %d, FD %d, fs %p\n", - (int) mypid, - r->id, - fs->fd, - fs); -#endif - xfree(fs); - return close(fd); -} - -static int -do_read(diomsg * r, int len, char *buf) -{ - int x; - int readlen = r->size; - file_state *fs; - fs = (file_state *) hash_lookup(hash, &r->id); - if (NULL == fs) { - errno = EBADF; - fprintf(stderr, "%d READ id %d: ", (int) mypid, r->id); - perror("do_read"); - return -EBADF; - } - if (r->offset > -1 && r->offset != fs->offset) { -#if STDERR_DEBUG - fprintf(stderr, "seeking to %d\n", r->offset); -#endif - if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { - fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); - perror("lseek"); - } - } - x = read(fs->fd, buf, readlen); -#if STDERR_DEBUG - fprintf(stderr, "%d READ %d,%d,%d ret %d\n", (int) mypid, - fs->fd, readlen, r->offset, x); -#endif - if (x < 0) { - fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); - perror("read"); - return -errno; - } - fs->offset = r->offset + x; - return x; -} - -static int -do_write(diomsg * r, int len, const char *buf) -{ - int wrtlen = r->size; - int x; - file_state *fs; - fs = (file_state *) hash_lookup(hash, &r->id); - if (NULL == fs) { - errno = EBADF; - fprintf(stderr, "%d WRITE id %d: ", (int) mypid, r->id); - perror("do_write"); - return -EBADF; - } - if (r->offset > -1 && r->offset != fs->offset) { - if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { - fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); - perror("lseek"); - } - } -#if STDERR_DEBUG - fprintf(stderr, "%d WRITE %d,%d,%d\n", (int) mypid, - fs->fd, wrtlen, r->offset); -#endif - x = write(fs->fd, buf, wrtlen); - if (x < 0) { - fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); - perror("write"); - return -errno; - } - fs->offset = r->offset + x; - return x; -} - -static int -do_unlink(diomsg * r, int len, const char *buf) -{ - if (truncate(buf, 0) < 0) { - fprintf(stderr, "%d UNLNK id %d %s: ", (int) mypid, r->id, buf); - perror("truncate"); - return -errno; - } -#if STDERR_DEBUG - fprintf(stderr, "%d UNLNK %s\n", (int) mypid, buf); -#endif - return 0; -} - -static void -msg_handle(diomsg * r, int rl, diomsg * s) -{ - char *buf = NULL; - s->mtype = r->mtype; - s->callback_data = r->callback_data; - s->shm_offset = r->shm_offset; - s->id = r->id; - if (s->shm_offset > -1) - buf = shmbuf + s->shm_offset; - switch (r->mtype) { - case _MQD_OPEN: - s->status = do_open(r, rl, buf); - break; - case _MQD_CLOSE: - s->status = do_close(r, rl); - break; - case _MQD_READ: - s->status = do_read(r, rl, buf); - break; - case _MQD_WRITE: - s->status = do_write(r, rl, buf); - break; - case _MQD_UNLINK: - s->status = do_unlink(r, rl, buf); - break; - default: - assert(0); - break; - } -} - -int -fsCmp(const void *a, const void *b) -{ - const int *A = a; - const int *B = b; - return *A != *B; -} - -unsigned int -fsHash(const void *key, unsigned int n) -{ - /* note, n must be a power of 2! */ - const int *k = key; - return (*k & (--n)); -} - -static void -alarm_handler(int sig) -{ - (void) 0; -} - -int -main(int argc, char *argv[]) -{ - int key; - int rmsgid; - int smsgid; - int shmid; - diomsg rmsg; - diomsg smsg; - int rlen; - char rbuf[512]; - struct sigaction sa; - setbuf(stdout, NULL); - setbuf(stderr, NULL); - mypid = getpid(); - assert(4 == argc); - key = atoi(argv[1]); - rmsgid = msgget(key, 0600); - if (rmsgid < 0) { - perror("msgget"); - return 1; - } - key = atoi(argv[2]); - smsgid = msgget(key, 0600); - if (smsgid < 0) { - perror("msgget"); - return 1; - } - key = atoi(argv[3]); - shmid = shmget(key, 0, 0600); - if (shmid < 0) { - perror("shmget"); - return 1; - } - shmbuf = shmat(shmid, NULL, 0); - if (shmbuf == (void *) -1) { - perror("shmat"); - return 1; - } - hash = hash_create(fsCmp, 1 << 4, fsHash); - assert(hash); - fcntl(0, F_SETFL, SQUID_NONBLOCK); - memset(&sa, '\0', sizeof(sa)); - sa.sa_handler = alarm_handler; - sa.sa_flags = SA_RESTART; - sigaction(SIGALRM, &sa, NULL); - for (;;) { - alarm(1); - memset(&rmsg, '\0', sizeof(rmsg)); - rlen = msgrcv(rmsgid, &rmsg, msg_snd_rcv_sz, 0, 0); - if (rlen < 0) { - if (EINTR == errno) { - if (read(0, rbuf, 512) <= 0) { - if (EWOULDBLOCK == errno) - (void) 0; - else if (EAGAIN == errno) - (void) 0; - else - break; - } - } - if (EAGAIN == errno) { - continue; - } - perror("msgrcv"); - break; - } - alarm(0); - msg_handle(&rmsg, rlen, &smsg); - if (msgsnd(smsgid, &smsg, msg_snd_rcv_sz, 0) < 0) { - perror("msgsnd"); - break; - } - } -#if STDERR_DEBUG - fprintf(stderr, "%d diskd exiting\n", (int) mypid); -#endif - if (msgctl(rmsgid, IPC_RMID, 0) < 0) - perror("msgctl IPC_RMID"); - if (msgctl(smsgid, IPC_RMID, 0) < 0) - perror("msgctl IPC_RMID"); - if (shmdt(shmbuf) < 0) - perror("shmdt"); - if (shmctl(shmid, IPC_RMID, 0) < 0) - perror("shmctl IPC_RMID"); - return 0; -} - -#elif 0 /* turn this into the squid side of the diskd interface */ - -/* - * DEBUG: section 81 Diskd Interface functions - */ - -static int sent_count = 0; -static int recv_count = 0; -static int shmbuf_count = 0; -static int sio_id = 0; - -static int storeDiskdSend(int, SwapDir *, int, storeIOState *, int, int, int); -static void storeDiskdShmPut(SwapDir *, int); -static void *storeDiskdShmGet(SwapDir *, int *); -static void storeDiskdHandle(diomsg * M); -static void storeDiskdIOCallback(storeIOState * sio, int errflag); -static int storeDiskdReadIndividualQueue(SwapDir * sd); -static SwapDir *swapDirFromFileno(sfileno f); -static OBJH storeDiskdStats; - -/* - * SHMBUFS is the number of shared memory buffers to allocate for - * Each SwapDir. - */ -#define SHMBUFS 96 -#define SHMBUF_BLKSZ SM_PAGE_SIZE -/* - * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv. - * If a queue has MAGIC2 (or more) messages away, then we read the - * queue until the level falls below MAGIC2. Recommended value - * is 75% of SHMBUFS. - */ -#define MAGIC1 Config.diskd.magic1 -#define MAGIC2 Config.diskd.magic2 - -struct { - int open_fail_queue_len; - int block_queue_len; - int max_away; - int max_shmuse; -} diskd_stats; - - -/* === PUBLIC =========================================================== */ - -storeIOState * -storeDiskdOpen(sfileno f, mode_t mode, STIOCB * callback, void *callback_data) -{ - int x; - storeIOState *sio; - char *buf; - int shm_offset; - SwapDir *sd = swapDirFromFileno(f); - debug(81, 3) ("storeDiskdOpen: fileno %08X, mode %d\n", f, mode); - /* - * XXX Eventually there should be an option here to fail on open() - * If there are too many requests queued. - */ - if (sd->u.diskd.away > MAGIC1) { - diskd_stats.open_fail_queue_len++; - return NULL; - } - assert(mode == O_RDONLY || mode == O_WRONLY); - sio = memAllocate(MEM_STORE_IO); - cbdataAdd(sio, memFree, MEM_STORE_IO); - sio->swap_file_number = f; - sio->mode = mode; - sio->callback = callback; - sio->callback_data = callback_data; - cbdataLock(sio->callback_data); - sio->type.diskd.id = sio_id++; - if (mode == O_WRONLY) - mode |= (O_CREAT | O_TRUNC); - buf = storeDiskdShmGet(sd, &shm_offset); - storeUfsFullPath(f, buf); - x = storeDiskdSend(_MQD_OPEN, - sd, - sio->type.diskd.id, - sio, - strlen(buf) + 1, - mode, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror()); - storeDiskdShmPut(sd, shm_offset); - cbdataUnlock(sio->callback_data); - cbdataFree(sio); - return NULL; - } - return sio; -} - -void -storeDiskdClose(storeIOState * sio) -{ - int x; - debug(81, 3) ("storeDiskdClose: fileno %08X\n", sio->swap_file_number); - x = storeDiskdSend(_MQD_CLOSE, - swapDirFromFileno(sio->swap_file_number), - sio->type.diskd.id, - sio, - 0, - 0, - -1); - if (x < 0) { - debug(50, 1) ("storeDiskdSend CLOSE: %s\n", xstrerror()); - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -void -storeDiskdRead(storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *their_data) -{ - int x; - int shm_offset; - char *rbuf; - SwapDir *sd = swapDirFromFileno(sio->swap_file_number); - if (!cbdataValid(sio)) - return; - if (sio->type.diskd.flags.reading) { - debug(81, 1) ("storeDiskdRead: already reading!\n"); - return; - } - assert(sio->read.callback == NULL); - assert(sio->read.callback_data == NULL); - sio->read.callback = callback; - sio->read.callback_data = their_data; - sio->type.diskd.read_buf = buf; /* the one passed from above */ - cbdataLock(sio->read.callback_data); - debug(81, 3) ("storeDiskdRead: fileno %08X\n", sio->swap_file_number); - sio->offset = offset; - sio->type.diskd.flags.reading = 1; - rbuf = storeDiskdShmGet(sd, &shm_offset); - assert(rbuf); - x = storeDiskdSend(_MQD_READ, - sd, - sio->type.diskd.id, - sio, - (int) size, - (int) offset, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend READ: %s\n", xstrerror()); - storeDiskdShmPut(sd, shm_offset); - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -void -storeDiskdWrite(storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) -{ - int x; - char *sbuf; - int shm_offset; - SwapDir *sd = swapDirFromFileno(sio->swap_file_number); - debug(81, 3) ("storeDiskdWrite: fileno %08X\n", sio->swap_file_number); - if (!cbdataValid(sio)) { - free_func(buf); - return; - } - sio->type.diskd.flags.writing = 1; - sbuf = storeDiskdShmGet(sd, &shm_offset); - xmemcpy(sbuf, buf, size); - free_func(buf); - x = storeDiskdSend(_MQD_WRITE, - sd, - sio->type.diskd.id, - sio, - (int) size, - (int) offset, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend WRITE: %s\n", xstrerror()); - storeDiskdShmPut(sd, shm_offset); - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -void -storeDiskdUnlink(sfileno f) -{ - int x; - int shm_offset; - char *buf; - SwapDir *sd = swapDirFromFileno(f); - debug(81, 3) ("storeDiskdUnlink: fileno %08X\n", f); - buf = storeDiskdShmGet(sd, &shm_offset); - storeUfsFullPath(f, buf); - x = storeDiskdSend(_MQD_UNLINK, - sd, - f, - NULL, - 0, - 0, - shm_offset); - if (x < 0) { - debug(50, 1) ("storeDiskdSend UNLINK: %s\n", xstrerror()); - unlink(buf); - storeDiskdShmPut(sd, shm_offset); - } -} - -void -storeDiskdInit(SwapDir * sd) -{ - static int first = 0; - int x; - int i; - int rfd; - int ikey = (getpid() << 16) + (sd->index << 4); - char *args[5]; - char skey1[32]; - char skey2[32]; - char skey3[32]; - storeUfsDirInit(sd); - sd->u.diskd.smsgid = msgget((key_t) ikey, 0700 | IPC_CREAT); - if (sd->u.diskd.smsgid < 0) { - debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); - fatal("msgget failed"); - } - sd->u.diskd.rmsgid = msgget((key_t) (ikey + 1), 0700 | IPC_CREAT); - if (sd->u.diskd.rmsgid < 0) { - debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); - fatal("msgget failed"); - } - sd->u.diskd.shm.id = shmget((key_t) (ikey + 2), - SHMBUFS * SHMBUF_BLKSZ, 0600 | IPC_CREAT); - if (sd->u.diskd.shm.id < 0) { - debug(50, 0) ("storeDiskdInit: shmget: %s\n", xstrerror()); - fatal("shmget failed"); - } - sd->u.diskd.shm.buf = shmat(sd->u.diskd.shm.id, NULL, 0); - if (sd->u.diskd.shm.buf == (void *) -1) { - debug(50, 0) ("storeDiskdInit: shmat: %s\n", xstrerror()); - fatal("shmat failed"); - } - shmbuf_count += SHMBUFS; - for (i = 0; i < SHMBUFS; i++) - storeDiskdShmPut(sd, i * SHMBUF_BLKSZ); - snprintf(skey1, 32, "%d", ikey); - snprintf(skey2, 32, "%d", ikey + 1); - snprintf(skey3, 32, "%d", ikey + 2); - args[0] = "diskd"; - args[1] = skey1; - args[2] = skey2; - args[3] = skey3; - args[4] = NULL; -#if HAVE_POLL && defined(_SQUID_OSF_) - /* pipes and poll() don't get along on DUNIX -DW */ - x = ipcCreate(IPC_TCP_SOCKET, -#else - x = ipcCreate(IPC_FIFO, -#endif - "/usr/local/squid/bin/diskd", - args, - "diskd", - &rfd, - &sd->u.diskd.wfd); - if (x < 0) - fatal("execl /usr/local/squid/bin/diskd failed"); - if (rfd != sd->u.diskd.wfd) - comm_close(rfd); - fd_note(sd->u.diskd.wfd, "squid -> diskd"); - commSetTimeout(sd->u.diskd.wfd, -1, NULL, NULL); - commSetNonBlocking(sd->u.diskd.wfd); - debug(81, 1) ("diskd started\n"); - if (0 == first) { - first++; - memset(&diskd_stats, '\0', sizeof(diskd_stats)); - cachemgrRegister("diskd", "DISKD Stats", storeDiskdStats, 0, 1); - } - comm_quick_polls_required(); -} - -void -storeDiskdReadQueue(void) -{ - SwapDir *sd; - int i; - int j; - static int ndir = 0; - if (sent_count - recv_count > diskd_stats.max_away) { - diskd_stats.max_away = sent_count - recv_count; - diskd_stats.max_shmuse = shmbuf_count; - } - do { - j = 0; - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - if (ndir >= Config.cacheSwap.n_configured) - ndir = ndir % Config.cacheSwap.n_configured; - sd = &Config.cacheSwap.swapDirs[ndir++]; - if (sd->type != SWAPDIR_DISKD) - continue; - if (0 == sd->u.diskd.away) - continue; - /* stay in this loop if one queue is over the limit */ - if (MAGIC2 <= sd->u.diskd.away) - j++; - j += storeDiskdReadIndividualQueue(sd); - } - } while (j > 0); - ndir++; -} - - -/* === STATIC =========================================================== */ - -static void -storeDiskdOpenDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - Counter.syscalls.disk.opens++; - debug(81, 3) ("storeDiskdOpenDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - } -} - -static void -storeDiskdCloseDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - Counter.syscalls.disk.closes++; - debug(81, 3) ("storeDiskdCloseDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - return; - } - storeDiskdIOCallback(sio, DISK_OK); -} - -static void -storeDiskdReadDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - STRCB *callback = sio->read.callback; - void *their_data = sio->read.callback_data; - char *their_buf = sio->type.diskd.read_buf; - char *sbuf; - size_t len; - SwapDir *sd = swapDirFromFileno(sio->swap_file_number); - int valid; - Counter.syscalls.disk.reads++; - sio->type.diskd.flags.reading = 0; - valid = cbdataValid(sio->read.callback_data); - cbdataUnlock(sio->read.callback_data); - debug(81, 3) ("storeDiskdReadDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - return; - } - sbuf = sd->u.diskd.shm.buf + M->shm_offset; - len = M->status; - xmemcpy(their_buf, sbuf, len); /* yucky copy */ - sio->offset += len; - assert(callback); - assert(their_data); - sio->read.callback = NULL; - sio->read.callback_data = NULL; - if (valid) - callback(their_data, their_buf, len); -} - -static void -storeDiskdWriteDone(diomsg * M) -{ - storeIOState *sio = M->callback_data; - Counter.syscalls.disk.writes++; - sio->type.diskd.flags.writing = 0; - debug(81, 3) ("storeDiskdWriteDone: fileno %08x status %d\n", - sio->swap_file_number, M->status); - if (M->status < 0) { - storeDiskdIOCallback(sio, DISK_ERROR); - return; - } - sio->offset += M->status; -} - -static void -storeDiskdUnlinkDone(diomsg * M) -{ - debug(81, 3) ("storeDiskdUnlinkDone: fileno %08x status %d\n", - M->id, M->status); - Counter.syscalls.disk.unlinks++; -} - -static void -storeDiskdHandle(diomsg * M) -{ - int valid = M->callback_data ? cbdataValid(M->callback_data) : 1; - if (M->callback_data) - cbdataUnlock(M->callback_data); - if (!valid) { - debug(81, 3) ("storeDiskdHandle: Invalid callback_data %p\n", - M->callback_data); - /* - * The read operation has its own callback. If we don't - * call storeDiskdReadDone(), then we must make sure the - * callback_data gets unlocked! - */ - if (_MQD_READ == M->mtype) { - storeIOState *sio = M->callback_data; - cbdataUnlock(sio->read.callback_data); - } - return; - } - switch (M->mtype) { - case _MQD_OPEN: - storeDiskdOpenDone(M); - break; - case _MQD_CLOSE: - storeDiskdCloseDone(M); - break; - case _MQD_READ: - storeDiskdReadDone(M); - break; - case _MQD_WRITE: - storeDiskdWriteDone(M); - break; - case _MQD_UNLINK: - storeDiskdUnlinkDone(M); - break; - default: - assert(0); - break; - } -} - -static void -storeDiskdIOCallback(storeIOState * sio, int errflag) -{ - int valid = cbdataValid(sio->callback_data); - debug(81, 3) ("storeUfsIOCallback: errflag=%d\n", errflag); - cbdataUnlock(sio->callback_data); - if (valid) - sio->callback(sio->callback_data, errflag, sio); - cbdataFree(sio); -} - -static int -storeDiskdSend(int mtype, SwapDir * sd, int id, storeIOState * sio, int size, int offset, int shm_offset) -{ - int x; - diomsg M; - static int send_errors = 0; - static int last_seq_no = 0; - static int seq_no = 0; - M.mtype = mtype; - M.callback_data = sio; - M.size = size; - M.offset = offset; - M.status = -1; - M.shm_offset = shm_offset; - M.id = id; - M.seq_no = ++seq_no; - if (M.callback_data) - cbdataLock(M.callback_data); - if (M.seq_no < last_seq_no) - debug(81, 1) ("WARNING: sequencing out of order\n"); - x = msgsnd(sd->u.diskd.smsgid, &M, msg_snd_rcv_sz, IPC_NOWAIT); - last_seq_no = M.seq_no; - if (0 == x) { - sent_count++; - sd->u.diskd.away++; - } else { - debug(50, 1) ("storeDiskdSend: msgsnd: %s\n", xstrerror()); - if (M.callback_data) - cbdataUnlock(M.callback_data); - assert(++send_errors < 100); - } - if (sd->u.diskd.away > MAGIC2) { - diskd_stats.block_queue_len++; - storeDiskdReadQueue(); - } - return x; -} - -static void * -storeDiskdShmGet(SwapDir * sd, int *shm_offset) -{ - char *buf; - buf = linklistShift(&sd->u.diskd.shm.stack); - assert(buf); - *shm_offset = buf - sd->u.diskd.shm.buf; - assert(0 <= *shm_offset && *shm_offset < SHMBUFS * SHMBUF_BLKSZ); - shmbuf_count++; - return buf; -} - -static void -storeDiskdShmPut(SwapDir * sd, int offset) -{ - char *buf; - assert(offset >= 0); - assert(offset < SHMBUFS * SHMBUF_BLKSZ); - buf = sd->u.diskd.shm.buf + offset; - linklistPush(&sd->u.diskd.shm.stack, buf); - shmbuf_count--; -} - -static int -storeDiskdReadIndividualQueue(SwapDir * sd) -{ - diomsg M; - int x; - memset(&M, '\0', sizeof(M)); - x = msgrcv(sd->u.diskd.rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT); - if (x < 0) - return 0; - if (x != msg_snd_rcv_sz) { - debug(81, 1) ("storeDiskdReadIndividualQueue: msgget returns %d\n", x); - return 0; - } - recv_count++; - sd->u.diskd.away--; - storeDiskdHandle(&M); - if (M.shm_offset > -1) - storeDiskdShmPut(sd, M.shm_offset); - return 1; -} - -static SwapDir * -swapDirFromFileno(sfileno f) -{ - return &Config.cacheSwap.swapDirs[f >> SWAP_DIR_SHIFT]; -} - -static void -storeDiskdStats(StoreEntry * sentry) -{ - storeAppendPrintf(sentry, "max_away: %d\n", diskd_stats.max_away); - storeAppendPrintf(sentry, "max_shmuse: %d\n", diskd_stats.max_shmuse); - storeAppendPrintf(sentry, "open_fail_queue_len: %d\n", diskd_stats.open_fail_queue_len); - storeAppendPrintf(sentry, "block_queue_len: %d\n", diskd_stats.block_queue_len); - diskd_stats.max_away = diskd_stats.max_shmuse = 0; -} - -#endif Index: squid/src/dns_internal.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/dns_internal.c,v retrieving revision 1.1.1.1.10.4 retrieving revision 1.1.1.1.10.4.2.1 diff -u -r1.1.1.1.10.4 -r1.1.1.1.10.4.2.1 --- squid/src/dns_internal.c 20 Apr 2000 23:20:47 -0000 1.1.1.1.10.4 +++ squid/src/dns_internal.c 2 May 2000 22:50:29 -0000 1.1.1.1.10.4.2.1 @@ -1,6 +1,6 @@ /* - * $Id: dns_internal.c,v 1.1.1.1.10.4 2000/04/20 23:20:47 hno Exp $ + * $Id: dns_internal.c,v 1.1.1.1.10.4.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 78 DNS lookups; interacts with lib/rfc1035.c * AUTHOR: Duane Wessels @@ -146,7 +146,7 @@ t = strtok(NULL, w_space); if (t == NULL) continue;; - debug(78, 1) ("idnsParseResolvConf: nameserver %s\n", t); + debug(78, 1) ("Adding nameserver %s from %s\n", t, _PATH_RESOLV_CONF); idnsAddNameserver(t); } } Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.1.1.3.10.5 retrieving revision 1.1.1.3.10.5.2.1 diff -u -r1.1.1.3.10.5 -r1.1.1.3.10.5.2.1 --- squid/src/enums.h 22 Apr 2000 09:18:51 -0000 1.1.1.3.10.5 +++ squid/src/enums.h 2 May 2000 22:50:29 -0000 1.1.1.3.10.5.2.1 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.1.1.3.10.5 2000/04/22 09:18:51 hno Exp $ + * $Id: enums.h,v 1.1.1.3.10.5.2.1 2000/05/02 22:50:29 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -515,6 +515,7 @@ #endif MEM_CLIENTHTTPREQUEST, MEM_CLIENT_INFO, + MEM_CLIENT_SOCK_BUF, MEM_CLOSE_HANDLER, MEM_COMMWRITESTATEDATA, MEM_CONNSTATEDATA, Index: squid/src/filemap.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/filemap.c,v retrieving revision 1.1.1.2.6.1 retrieving revision 1.1.1.2.6.1.2.1 diff -u -r1.1.1.2.6.1 -r1.1.1.2.6.1.2.1 --- squid/src/filemap.c 20 Apr 2000 18:14:22 -0000 1.1.1.2.6.1 +++ squid/src/filemap.c 2 May 2000 22:50:29 -0000 1.1.1.2.6.1.2.1 @@ -1,6 +1,6 @@ /* - * $Id: filemap.c,v 1.1.1.2.6.1 2000/04/20 18:14:22 hno Exp $ + * $Id: filemap.c,v 1.1.1.2.6.1.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 8 Swap File Bitmap * AUTHOR: Harvest Derived @@ -133,7 +133,7 @@ for (bit = 0; bit < BITS_IN_A_LONG; bit++) { suggestion = ((unsigned long) word << LONG_BIT_SHIFT) | bit; if (!file_map_bit_test(fm, suggestion)) { - return file_map_bit_set(fm, suggestion); + return suggestion; } } debug(8, 3) ("growing from file_map_allocate\n"); Index: squid/src/forward.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/forward.c,v retrieving revision 1.1.1.3.10.3 retrieving revision 1.1.1.3.10.3.2.1 diff -u -r1.1.1.3.10.3 -r1.1.1.3.10.3.2.1 --- squid/src/forward.c 20 Apr 2000 18:14:22 -0000 1.1.1.3.10.3 +++ squid/src/forward.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.3.2.1 @@ -1,6 +1,6 @@ /* - * $Id: forward.c,v 1.1.1.3.10.3 2000/04/20 18:14:22 hno Exp $ + * $Id: forward.c,v 1.1.1.3.10.3.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 17 Request Forwarding * AUTHOR: Duane Wessels @@ -195,8 +195,10 @@ debug(17, 3) ("fwdConnectDone: FD %d: '%s'\n", server_fd, storeUrl(fwdState->entry)); if (fs->peer) hierarchyNote(&fwdState->request->hier, fs->code, fs->peer->host); - else + else if (Config.onoff.log_ip_on_direct) hierarchyNote(&fwdState->request->hier, fs->code, fd_table[server_fd].ipaddr); + else + hierarchyNote(&fwdState->request->hier, fs->code, request->host); fd_note(server_fd, storeUrl(fwdState->entry)); fd_table[server_fd].uses++; if (fs->peer) Index: squid/src/mem.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/mem.c,v retrieving revision 1.1.1.3.10.3 retrieving revision 1.1.1.3.10.3.2.1 diff -u -r1.1.1.3.10.3 -r1.1.1.3.10.3.2.1 --- squid/src/mem.c 20 Apr 2000 18:14:22 -0000 1.1.1.3.10.3 +++ squid/src/mem.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.3.2.1 @@ -1,6 +1,6 @@ /* - * $Id: mem.c,v 1.1.1.3.10.3 2000/04/20 18:14:22 hno Exp $ + * $Id: mem.c,v 1.1.1.3.10.3.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 13 High Level Memory Pool Management * AUTHOR: Harvest Derived @@ -189,6 +189,7 @@ memDataInit(MEM_2K_BUF, "2K Buffer", 2048, 10); memDataInit(MEM_4K_BUF, "4K Buffer", 4096, 10); memDataInit(MEM_8K_BUF, "8K Buffer", 8192, 10); + memDataInit(MEM_CLIENT_SOCK_BUF, "Client Socket Buffer", CLIENT_SOCK_SZ, 0); memDataInit(MEM_ACCESSLOGENTRY, "AccessLogEntry", sizeof(AccessLogEntry), 10); memDataInit(MEM_ACL, "acl", sizeof(acl), 0); Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.1.1.3.10.8.2.1 retrieving revision 1.1.1.3.10.8.2.2 diff -u -r1.1.1.3.10.8.2.1 -r1.1.1.3.10.8.2.2 --- squid/src/protos.h 22 Apr 2000 20:02:55 -0000 1.1.1.3.10.8.2.1 +++ squid/src/protos.h 2 May 2000 22:50:29 -0000 1.1.1.3.10.8.2.2 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.1.1.3.10.8.2.1 2000/04/22 20:02:55 hno Exp $ + * $Id: protos.h,v 1.1.1.3.10.8.2.2 2000/05/02 22:50:29 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -45,6 +45,7 @@ #if HEADERS_LOG extern void headersLog(int cs, int pq, method_t m, void *data); #endif +char *log_quote(const char *header); extern aclCheck_t *aclChecklistCreate(const struct _acl_access *, request_t *, Index: squid/src/pump.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Attic/pump.c,v retrieving revision 1.1.1.3.10.2 retrieving revision 1.1.1.3.10.2.2.1 diff -u -r1.1.1.3.10.2 -r1.1.1.3.10.2.2.1 --- squid/src/pump.c 20 Apr 2000 18:14:22 -0000 1.1.1.3.10.2 +++ squid/src/pump.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.2.2.1 @@ -1,5 +1,6 @@ + /* - * $Id: pump.c,v 1.1.1.3.10.2 2000/04/20 18:14:22 hno Exp $ + * $Id: pump.c,v 1.1.1.3.10.2.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 61 PUMP handler * AUTHOR: Kostas Anagnostakis Index: squid/src/ssl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ssl.c,v retrieving revision 1.1.1.3.10.3 retrieving revision 1.1.1.3.10.3.2.1 diff -u -r1.1.1.3.10.3 -r1.1.1.3.10.3.2.1 --- squid/src/ssl.c 20 Apr 2000 18:14:23 -0000 1.1.1.3.10.3 +++ squid/src/ssl.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.3.2.1 @@ -1,6 +1,6 @@ /* - * $Id: ssl.c,v 1.1.1.3.10.3 2000/04/20 18:14:23 hno Exp $ + * $Id: ssl.c,v 1.1.1.3.10.3.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 26 Secure Sockets Layer Proxy * AUTHOR: Duane Wessels @@ -377,13 +377,15 @@ SslStateData *sslState = data; request_t *request = sslState->request; ErrorState *err = NULL; - if (sslState->servers->peer) { + if (sslState->servers->peer) hierarchyNote(&sslState->request->hier, sslState->servers->code, sslState->servers->peer->host); - } else { + else if (Config.onoff.log_ip_on_direct) hierarchyNote(&sslState->request->hier, sslState->servers->code, fd_table[sslState->server.fd].ipaddr); - } + else + hierarchyNote(&sslState->request->hier, sslState->servers->code, + sslState->host); if (status == COMM_ERR_DNS) { debug(26, 4) ("sslConnect: Unknown host: %s\n", sslState->host); err = errorCon(ERR_DNS_FAIL, HTTP_NOT_FOUND); Index: squid/src/store.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store.c,v retrieving revision 1.1.1.3.10.8 retrieving revision 1.1.1.3.10.8.2.1 diff -u -r1.1.1.3.10.8 -r1.1.1.3.10.8.2.1 --- squid/src/store.c 20 Apr 2000 18:14:23 -0000 1.1.1.3.10.8 +++ squid/src/store.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.8.2.1 @@ -1,6 +1,6 @@ /* - * $Id: store.c,v 1.1.1.3.10.8 2000/04/20 18:14:23 hno Exp $ + * $Id: store.c,v 1.1.1.3.10.8.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -804,11 +804,12 @@ * we'll just call storeUnlockObject() on these. */ e->lock_count++; + EBIT_SET(e->flags, RELEASE_REQUEST); stackPush(&LateReleaseStack, e); + return; } else { destroy_StoreEntry(e); } - return; } storeLog(STORE_LOG_RELEASE, e); if (e->swap_filen > -1) { Index: squid/src/store_client.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_client.c,v retrieving revision 1.1.1.3.10.4 retrieving revision 1.1.1.3.10.4.2.1 diff -u -r1.1.1.3.10.4 -r1.1.1.3.10.4.2.1 --- squid/src/store_client.c 21 Apr 2000 22:18:06 -0000 1.1.1.3.10.4 +++ squid/src/store_client.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.4.2.1 @@ -1,6 +1,6 @@ /* - * $Id: store_client.c,v 1.1.1.3.10.4 2000/04/21 22:18:06 hno Exp $ + * $Id: store_client.c,v 1.1.1.3.10.4.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 20 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -258,9 +258,6 @@ { MemObject *mem = e->mem_obj; size_t sz; - mem_node *mp = mem->data_hdr.head; - off_t mp_ofs = mem->data_hdr.origin_offset; - off_t offset; if (storeClientNoMoreToSend(e, sc)) { /* There is no more to send! */ @@ -309,68 +306,10 @@ if (sc->copy_offset >= mem->inmem_lo && sc->copy_offset < mem->inmem_hi) { /* What the client wants is in memory */ debug(20, 3) ("storeClientCopy3: Copying from memory\n"); - /* - * We are now throwing data to clients in SM_PAGE_SIZE chunks, following - * the mem_node linked list just like src/store_swapout.c does. We send - * data to a client when we have at least a page to send, and we only - * send partial pages when we have the entire object in memory. We then - * throw the buffer to the callback, and let the callback routine use the - * buffer directly rather than copy and waste memory. Yes, evil, but thats - * life, right? - */ - - /* Fudge to figure out whether we're doing the old or new style call */ - if (sc->copy_buf != NULL) { - /* Old style */ - sz = stmemCopy(&mem->data_hdr, - sc->copy_offset, sc->copy_buf, sc->copy_size); - storeClientCallback(sc, sz); - return; - } else { - /* XXX Traverse the membuf list, find the memnode and offset, - * and use that buffer. We still end up walking the list :( .. - * its a temporary hack. When the code works, I'd like to do this: - * - * - if we don't have a 'hint' mem_node *, we traverse the list - * and look for it - * - if we do have a hint mem_node *, we use it as the basis - * for figuring out the next mem_node to look at, and if we - * can't figure it out, just fallback to the route search. - */ - - /* Whats the offset? :-) */ - offset = sc->copy_offset; - /* XXX THIS SHOULD BE IN STMEM.C!!!! */ - while ((mp_ofs + mp->len - 1) < offset) { - mp_ofs += mp->len; - if (!mp->next) { - /* No match */ - mp = NULL; - break; - } - assert(mp->next); - mp = mp->next; - } - - /* If we don't have a membuf pointer here, we are fucked */ - assert(mp); - - /* - * Ok, we have mp_ofs with the offset of this mem_node, and mp - * which is the current memnode. The offset inside the memnode - * we need to be is offset - mp_ofs, and the length of the data - * we are handing off is mp->len - sz. - */ - sz = offset - mp_ofs; - - /* Call the callback with the buffer */ - sc->flags.disk_io_pending = 0; - debug(20, 5) ("storeClientCopy2: copy_offset = %d, seen_offset = %d\n", sc->copy_offset, sc->seen_offset); - debug(20, 5) ("storeClientCopy2: copying %d bytes of data from offset %d in the membuf of size %d\n", mp->len - sz, sz, mp->len); - sc->copy_buf = mp->data + sz; - storeClientCallback(sc, mp->len - sz); - return; - } + sz = stmemCopy(&mem->data_hdr, + sc->copy_offset, sc->copy_buf, sc->copy_size); + storeClientCallback(sc, sz); + return; } /* What the client wants is not in memory. Schedule a disk read */ assert(STORE_DISK_CLIENT == sc->type); @@ -448,7 +387,7 @@ return; } if (tlv_list == NULL) { - debug(20, 1) ("WARNING: failed to unpack swapfile meta data\n"); + debug(20, 1) ("WARNING: failed to unpack meta data\n"); storeClientCallback(sc, -1); return; } Index: squid/src/store_dir.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_dir.c,v retrieving revision 1.1.1.3.10.6 retrieving revision 1.1.1.3.10.6.2.1 diff -u -r1.1.1.3.10.6 -r1.1.1.3.10.6.2.1 --- squid/src/store_dir.c 20 Apr 2000 18:14:23 -0000 1.1.1.3.10.6 +++ squid/src/store_dir.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.6.2.1 @@ -1,6 +1,6 @@ /* - * $Id: store_dir.c,v 1.1.1.3.10.6 2000/04/20 18:14:23 hno Exp $ + * $Id: store_dir.c,v 1.1.1.3.10.6.2.1 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -182,83 +182,50 @@ storeDirSelectSwapDir(const StoreEntry *e) { size_t objsize; - size_t least_size = -1; - size_t least_maxobjsize = -1; - double high = (double) Config.Swap.highWaterMark / 100.0; - double u; - int dirn; + size_t least_size; + size_t least_objsize; + int least_load = 1000; + int load; + int dirn = -1; int i; SwapDir *SD; - /* - * Handle simplest case of a single swap directory immediately - * XXX You'd hope to hell that its a valid one, or something is - * definitely wrong. - */ - if (Config.cacheSwap.n_configured == 1) - return 0; - /* Calculate the object size */ objsize = objectLen(e); if (objsize != -1) objsize += e->mem_obj->swap_hdr_sz; - - /* - * Go through the list of store dirs, doing the following: - * - if its read only, ignore it - * - if its over the high water mark, ignore it - * - if its not valid for the object size, ignore it - * - if the checkobj routine returns false, ignore it - * - if its got a smaller maxobjsize than the last match, use it - */ - dirn = -1; + /* Initial defaults */ + least_size = Config.cacheSwap.swapDirs[0].cur_size; + least_objsize = Config.cacheSwap.swapDirs[0].max_objsize; for (i = 0; i < Config.cacheSwap.n_configured; i++) { SD = &Config.cacheSwap.swapDirs[i]; - SD->flags.selected = 0; - - /* Read only check */ + SD->flags.selected = 0; if (SD->flags.read_only) continue; - - /* Over high water mark check */ - u = (double) SD->cur_size / SD->max_size; - if (u > high) - continue; - /* Valid for object size check */ if (!storeDirValidSwapDirSize(i, objsize)) continue; - - /* Will the filesystem actually store this object ? */ - if (!SD->checkobj(SD, e)) + load = SD->checkobj(SD, e); + if (load < 0) continue; - - /* Is this maxobjsize smaller? */ - if ((least_maxobjsize == -1) || (SD->max_objsize < least_maxobjsize)) { - least_maxobjsize = SD->max_objsize; - least_size = SD->cur_size; - dirn = i; + if (SD->cur_size > SD->max_size) + continue; + if (load > least_load) + continue; + if ((least_objsize > 0) && (objsize > least_objsize)) continue; - } - - /* Does this device have more free space? */ - if ((least_size == -1) || (SD->cur_size < least_size)) { - least_size = SD->cur_size; - dirn = i; + /* Only use leastsize if the load is equal */ + if ((load == least_load) && (SD->cur_size > least_size)) continue; - } + least_load = load; + least_size = SD->cur_size; + dirn = i; } - /* Now, if dirn isn't -1, we can use it */ - if (dirn != -1) { - Config.cacheSwap.swapDirs[dirn].flags.selected = 1; - return (dirn); - } + if (dirn >= 0) + Config.cacheSwap.swapDirs[dirn].flags.selected = 1; - /* - * If we didn't find anything valid, return so. - */ - return (-1); + return dirn; } @@ -331,9 +298,11 @@ /* Now go through each swapdir, calling its statfs routine */ for (i = 0; i < Config.cacheSwap.n_configured; i++) { - storeAppendPrintf(sentry, "\n"); - SD = &(Config.cacheSwap.swapDirs[i]); - SD->statfs(SD, sentry); + storeAppendPrintf(sentry, "\n"); + SD = &(Config.cacheSwap.swapDirs[i]); + storeAppendPrintf(sentry, "Store Directory #%d (%s): %s\n", i, SD->type, + storeSwapDir(i)); + SD->statfs(SD, sentry); } } @@ -473,12 +442,13 @@ void storeDirSync(void) { - int i = 0; + int i; + SwapDir *SD; - while (storefs_list[i].typestr != NULL) { - if (storefs_list[i].syncfunc != NULL) - storefs_list[i].syncfunc(); - i++; + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->sync != NULL) + SD->sync(SD); } } @@ -488,12 +458,13 @@ void storeDirCallback(void) { - int i = 0; + int i; + SwapDir *SD; - while (storefs_list[i].typestr != NULL) { - if (storefs_list[i].callbackfunc != NULL) - storefs_list[i].callbackfunc(); - i++; + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + SD = &Config.cacheSwap.swapDirs[i]; + if (SD->callback != NULL) + SD->callback(SD); } } Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.1.1.3.10.8.2.3 retrieving revision 1.1.1.3.10.8.2.4 diff -u -r1.1.1.3.10.8.2.3 -r1.1.1.3.10.8.2.4 --- squid/src/structs.h 23 Apr 2000 09:55:37 -0000 1.1.1.3.10.8.2.3 +++ squid/src/structs.h 2 May 2000 22:50:29 -0000 1.1.1.3.10.8.2.4 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.1.1.3.10.8.2.3 2000/04/23 09:55:37 hno Exp $ + * $Id: structs.h,v 1.1.1.3.10.8.2.4 2000/05/02 22:50:29 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -311,9 +311,7 @@ #if USE_ICMP char *pinger; #endif -#if USE_UNLINKD char *unlinkd; -#endif } Program; #if USE_DNSSERVERS int dnsChildren; @@ -416,6 +414,8 @@ #if USE_CACHE_DIGESTS int digest_generation; #endif + int log_ip_on_direct; + int authenticateIpTTLStrict; } onoff; acl *aclList; struct { @@ -1384,6 +1384,8 @@ /* These two are notifications */ STREFOBJ *refobj; /* Reference this object */ STUNREFOBJ *unrefobj; /* Unreference this object */ + STCALLBACK *callback; /* Handle pending callbacks */ + STSYNC *sync; /* Sync the directory */ struct { STOBJCREATE *create; STOBJOPEN *open; @@ -1875,8 +1877,6 @@ STFSPARSE *parsefunc; STFSRECONFIGURE *reconfigurefunc; STFSSHUTDOWN *donefunc; - STFSCALLBACK *callbackfunc; - STFSSYNC *syncfunc; }; typedef struct chunked_tag Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.1.1.3.10.6 retrieving revision 1.1.1.3.10.6.2.1 diff -u -r1.1.1.3.10.6 -r1.1.1.3.10.6.2.1 --- squid/src/typedefs.h 20 Apr 2000 18:14:23 -0000 1.1.1.3.10.6 +++ squid/src/typedefs.h 2 May 2000 22:50:29 -0000 1.1.1.3.10.6.2.1 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.1.1.3.10.6 2000/04/20 18:14:23 hno Exp $ + * $Id: typedefs.h,v 1.1.1.3.10.6.2.1 2000/05/02 22:50:29 hno Exp $ * * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -238,6 +238,8 @@ typedef void STUNREFOBJ(SwapDir *, StoreEntry *); typedef void STSETUP(storefs_entry_t *); typedef void STDONE(void); +typedef void STCALLBACK(SwapDir *); +typedef void STSYNC(SwapDir *); typedef storeIOState *STOBJCREATE(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *); typedef storeIOState *STOBJOPEN(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *); @@ -258,8 +260,6 @@ typedef void STFSRECONFIGURE(SwapDir *, int, char *); typedef void STFSSTARTUP(void); typedef void STFSSHUTDOWN(void); -typedef void STFSCALLBACK(void); -typedef void STFSSYNC(void); typedef double hbase_f(double); typedef void StatHistBinDumper(StoreEntry *, int idx, double val, double size, int count); Index: squid/src/unlinkd.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/unlinkd.c,v retrieving revision 1.1.1.3.10.3.2.1 retrieving revision 1.1.1.3.10.3.2.2 diff -u -r1.1.1.3.10.3.2.1 -r1.1.1.3.10.3.2.2 --- squid/src/unlinkd.c 22 Apr 2000 20:02:55 -0000 1.1.1.3.10.3.2.1 +++ squid/src/unlinkd.c 2 May 2000 22:50:29 -0000 1.1.1.3.10.3.2.2 @@ -1,5 +1,6 @@ + /* - * $Id: unlinkd.c,v 1.1.1.3.10.3.2.1 2000/04/22 20:02:55 hno Exp $ + * $Id: unlinkd.c,v 1.1.1.3.10.3.2.2 2000/05/02 22:50:29 hno Exp $ * * DEBUG: section 12 Unlink Daemon * AUTHOR: Duane Wessels Index: squid/src/fs/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/Attic/Makefile.in,v retrieving revision 1.1.6.2 retrieving revision 1.1.6.2.2.1 diff -u -r1.1.6.2 -r1.1.6.2.2.1 --- squid/src/fs/Makefile.in 8 Feb 2000 22:33:22 -0000 1.1.6.2 +++ squid/src/fs/Makefile.in 2 May 2000 22:50:29 -0000 1.1.6.2.2.1 @@ -15,7 +15,7 @@ clean: -rm -f *.a stamp - -for dir in *; do \ + @for dir in *; do \ if [ -f $$dir/Makefile ]; then \ sh -c "cd $$dir && $(MAKE) $@" || exit 1;\ fi; \ @@ -23,7 +23,7 @@ distclean: -rm -f *.a Makefile - -for dir in *; do \ + @for dir in *; do \ if [ -f $$dir/Makefile ]; then \ sh -c "cd $$dir && $(MAKE) distclean"; \ fi; \ Index: squid/src/fs/aufs/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/Attic/Makefile.in,v retrieving revision 1.1.6.3 retrieving revision 1.1.6.3.2.1 diff -u -r1.1.6.3 -r1.1.6.3.2.1 --- squid/src/fs/aufs/Makefile.in 5 Mar 2000 23:39:05 -0000 1.1.6.3 +++ squid/src/fs/aufs/Makefile.in 2 May 2000 22:50:29 -0000 1.1.6.3.2.1 @@ -49,6 +49,8 @@ -rm -f Makefile.bak -rm -f tags +install: + tags: ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] Index: squid/src/fs/aufs/async_io.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/async_io.c,v retrieving revision 1.1.6.1.2.2 retrieving revision 1.1.6.1.2.3 diff -u -r1.1.6.1.2.2 -r1.1.6.1.2.3 --- squid/src/fs/aufs/async_io.c 24 Apr 2000 22:13:02 -0000 1.1.6.1.2.2 +++ squid/src/fs/aufs/async_io.c 2 May 2000 22:50:29 -0000 1.1.6.1.2.3 @@ -220,7 +220,8 @@ errno = EWOULDBLOCK; if (callback) (callback) (fd, callback_data, -1, errno); - free_func(bufp); + if (free_func) + free_func(bufp); return; } ctrlp = memPoolAlloc(aio_ctrl_pool); @@ -350,7 +351,7 @@ void -aioCheckCallbacks(void) +aioCheckCallbacks(SwapDir *SD) { aio_result_t *resultp; aio_ctrl_t *ctrlp; @@ -408,14 +409,14 @@ /* Flush all pending I/O */ void -aioSync(void) +aioSync(SwapDir *SD) { if (!initialised) return; /* nothing to do then */ /* Flush all pending operations */ debug(32, 1) ("aioSync: flushing pending I/O operations\n"); do { - aioCheckCallbacks(); + aioCheckCallbacks(SD); } while (aio_sync()); debug(32, 1) ("aioSync: done\n"); } Index: squid/src/fs/aufs/store_asyncufs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_asyncufs.h,v retrieving revision 1.1.6.3 retrieving revision 1.1.6.3.2.1 diff -u -r1.1.6.3 -r1.1.6.3.2.1 --- squid/src/fs/aufs/store_asyncufs.h 11 Apr 2000 18:18:41 -0000 1.1.6.3 +++ squid/src/fs/aufs/store_asyncufs.h 2 May 2000 22:50:29 -0000 1.1.6.3.2.1 @@ -13,6 +13,8 @@ #define NUMTHREADS 16 #endif +#define MAGIC1 (NUMTHREADS/Config.cacheSwap.n_configured/2) + struct _aio_result_t { int aio_return; int aio_errno; @@ -45,8 +47,8 @@ void aioRead(int, int offset, char *, int size, AIOCB *, void *); void aioStat(char *, struct stat *, AIOCB *, void *); void aioUnlink(const char *, AIOCB *, void *); -void aioCheckCallbacks(void); -void aioSync(void); +void aioCheckCallbacks(SwapDir *); +void aioSync(SwapDir *); int aioQueueSize(void); struct _aioinfo_t { @@ -77,9 +79,12 @@ extern MemPool * aio_state_pool; extern void storeAufsDirMapBitReset(SwapDir *, sfileno); +extern int storeAufsDirMapBitAllocate(SwapDir *); extern char * storeAufsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); extern void storeAufsDirUnlinkFile(SwapDir *, sfileno); +extern void storeAufsDirReplAdd(SwapDir *SD, StoreEntry *); +extern void storeAufsDirReplRemove(StoreEntry *); /* * Store IO stuff Index: squid/src/fs/aufs/store_dir_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_dir_aufs.c,v retrieving revision 1.1.6.6.2.1 retrieving revision 1.1.6.6.2.2 diff -u -r1.1.6.6.2.1 -r1.1.6.6.2.2 --- squid/src/fs/aufs/store_dir_aufs.c 22 Apr 2000 20:02:55 -0000 1.1.6.6.2.1 +++ squid/src/fs/aufs/store_dir_aufs.c 2 May 2000 22:50:29 -0000 1.1.6.6.2.2 @@ -71,7 +71,6 @@ static int n_asyncufs_dirs = 0; static int *asyncufs_dir_index = NULL; -MemPool * aio_state_pool = NULL; static char *storeAufsDirSwapSubDir(SwapDir *, int subdirn); static int storeAufsDirCreateDirectory(const char *path, int); @@ -108,6 +107,7 @@ static STMAINTAINFS storeAufsDirMaintain; static STCHECKOBJ storeAufsDirCheckObj; static STREFOBJ storeAufsDirRefObj; +static STUNREFOBJ storeAufsDirUnrefObj; static QS rev_int_sort; static int storeAufsDirClean(int swap_index); static EVH storeAufsDirCleanEvent; @@ -155,6 +155,18 @@ aioinfo = (aioinfo_t *)SD->fsdata; file_map_bit_reset(aioinfo->map, filn); } + +int +storeAufsDirMapBitAllocate(SwapDir *SD) +{ + aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; + int fn; + fn = file_map_allocate(aioinfo->map, aioinfo->suggest); + file_map_bit_set(aioinfo->map, fn); + aioinfo->suggest = fn + 1; + return fn; +} + /* * Initialise the asyncufs bitmap @@ -272,18 +284,28 @@ storeAufsDirSwapLogFile(SwapDir * sd, const char *ext) { LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; if (Config.Log.swap) { - xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64); - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } } else { - xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); - strcat(path, "/swap.state"); + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); } if (ext) - strncat(path, ext, 16); + strncat(path, ext, 16); return path; } @@ -409,6 +431,7 @@ tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); if (tlv_list == NULL) { debug(20, 1) ("storeAufsDirRebuildFromDirectory: failed to get meta data\n"); + /* XXX shouldn't this be a call to storeAufsUnlink ? */ storeAufsDirUnlinkFile(SD, sfileno); continue; } @@ -528,10 +551,12 @@ * because adding to store_swap_size happens in * the cleanup procedure. */ - storeExpireNow(e); - storeReleaseRequest(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeAufsDirReplRemove(e); + storeRelease(e); if (e->swap_filen > -1) { - storeAufsDirMapBitReset(SD, e->swap_filen); + /* Fake a unlink here, this is a bad hack :( */ e->swap_filen = -1; e->swap_dirn = -1; } @@ -579,6 +604,7 @@ e->refcount += s.refcount; #if HEAP_REPLACEMENT storeHeapPositionUpdate(e, SD); + storeAufsDirUnrefObj(SD, e); #endif } else { debug_trap("storeAufsDirRebuildFromSwapLog: bad condition"); @@ -613,10 +639,12 @@ } else if (e) { /* key already exists, this swapfile not being used */ /* junk old, load new */ - storeExpireNow(e); - storeReleaseRequest(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeAufsDirReplRemove(e); + storeRelease(e); if (e->swap_filen > -1) { - storeAufsDirMapBitReset(SD, e->swap_filen); + /* Fake a unlink here, this is a bad hack :( */ e->swap_filen = -1; e->swap_dirn = -1; } @@ -772,16 +800,7 @@ EBIT_CLR(e->flags, ENTRY_VALIDATED); storeAufsDirMapBitSet(SD, e->swap_filen); storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ -#if HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->repl.node = heap_insert(SD->repl.heap.heap, e); - debug(20, 4) ("storeAufsAddDiskRestore: inserted node %p\n", e->repl.node); - } -#else - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); -#endif + storeAufsDirReplAdd(SD, e); return e; } @@ -1389,22 +1408,39 @@ last_warn_time = squid_curtime; } -/* +/* * storeAufsDirCheckObj * * This routine is called by storeDirSelectSwapDir to see if the given * object is able to be stored on this filesystem. AUFS filesystems will - * happily store anything. We don't check for maxobjsize here since its - * done by the upper layers. + * happily store anything as long as the LRU time isn't too small. */ int storeAufsDirCheckObj(SwapDir *SD, const StoreEntry *e) { - return 1; + int loadav; + int ql; + +#if !HEAP_REPLACEMENT + if (storeAufsDirExpiredReferenceAge(SD) < 300) { + debug(20, 3) ("storeAufsDirCheckObj: NO: LRU Age = %d\n", + storeAufsDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; + } +#endif + ql = aioQueueSize(); + if (ql == 0) + loadav = 0; + else if (ql >= MAGIC1) /* Queue is too long, don't even consider it */ + loadav = -1; + else + loadav = MAGIC1 * 1000 / ql; + return loadav; } /* - * storeAfsDirRefObj + * storeAufsDirRefObj * * This routine is called whenever an object is referenced, so we can * maintain replacement information within the storage fs. @@ -1412,27 +1448,47 @@ void storeAufsDirRefObj(SwapDir *SD, StoreEntry *e) { -#ifdef HEAP_REPLACEMENT + debug(1, 3) ("storeAufsDirRefObj: referencing %d/%d\n", e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT /* Nothing to do here */ #else /* Reference the object */ - debug(1, 3) ("storeAufsDirRefObj: referencing %d/%d\n", e->swap_dirn, e->swap_filen); - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && + !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } #endif } /* - * storeAufsDirUnlikFile + * storeAufsDirUnrefObj + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ +void +storeAufsDirUnrefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeAufsDirUnrefObj: referencing %d/%d\n", e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT + if (e->repl.node) + heap_update(SD->repl.heap.heap, e->repl.node, e); +#endif +} + +/* + * storeAufsDirUnlinkFile * * This routine unlinks a file and pulls it out of the bitmap. - * It used to be in storeUfsUnlink(), however an interface change + * It used to be in storeAufsUnlink(), however an interface change * forced this bit of code here. Eeek. */ void storeAufsDirUnlinkFile(SwapDir *SD, sfileno f) { - debug(79, 3) ("storeUfsDirUnlinkFile: unlinking fileno %08X\n", f); + debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f); storeAufsDirMapBitReset(SD, f); aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL); } @@ -1470,8 +1526,8 @@ age = (time_t) (z * 60.0); if (age < 60) age = 60; - else if (age > 31536000) - age = 31536000; + else if (age > Config.referenceAge) + age = Config.referenceAge; return age; } #endif @@ -1507,8 +1563,51 @@ #endif } +/* + * Add and remove the given StoreEntry from the replacement policy in + * use. + */ + +void +storeAufsDirReplAdd(SwapDir *SD, StoreEntry *e) +{ + debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + (void) 0; + } else { + e->repl.node = heap_insert(SD->repl.heap.heap, e); + debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node); + } +#else + /* Shouldn't we not throw special objects into the lru ? */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); +#endif +} + + +void +storeAufsDirReplRemove(StoreEntry *e) +{ + SwapDir *SD = INDEXSD(e->swap_dirn); + debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + /* And now, release the object from the replacement policy */ + if (e->repl.node) { + debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n", + e->repl.node); + heap_delete(SD->repl.heap.heap, e->repl.node); + e->repl.node = NULL; + } +#else + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); +#endif +} + -/* ========= LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ +/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ void storeAufsDirStats(SwapDir *SD, StoreEntry * sentry) @@ -1518,7 +1617,6 @@ struct statvfs sfs; #endif aioinfo = (aioinfo_t *)SD->fsdata; - storeAppendPrintf(sentry, "Store Directory #%d: %s\n", SD->index, SD->path); storeAppendPrintf(sentry, "First level subdirectories: %d\n", aioinfo->l1); storeAppendPrintf(sentry, "Second level subdirectories: %d\n", aioinfo->l2); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); @@ -1548,6 +1646,15 @@ if (SD->flags.read_only) storeAppendPrintf(sentry, " READ-ONLY"); storeAppendPrintf(sentry, "\n"); +#if !HEAP_REPLACEMENT + storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", + (double) storeAufsDirExpiredReferenceAge(SD) / 86400.0); +#else +#if 0 + storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", + heap_peepminkey(sd.repl.heap.heap)); +#endif +#endif } /* @@ -1730,6 +1837,9 @@ sd->maintainfs = storeAufsDirMaintain; sd->checkobj = storeAufsDirCheckObj; sd->refobj = storeAufsDirRefObj; + sd->unrefobj = storeAufsDirUnrefObj; + sd->callback = aioCheckCallbacks; + sd->sync = aioSync; sd->obj.create = storeAufsCreate; sd->obj.open = storeAufsOpen; sd->obj.close = storeAufsClose; @@ -1782,7 +1892,5 @@ storefs->parsefunc = storeAufsDirParse; storefs->reconfigurefunc = storeAufsDirReconfigure; storefs->donefunc = aioDone; - storefs->callbackfunc = aioCheckCallbacks; - storefs->syncfunc = aioSync; aioInit(); } Index: squid/src/fs/aufs/store_io_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_io_aufs.c,v retrieving revision 1.1.6.6.2.1 retrieving revision 1.1.6.6.2.2 diff -u -r1.1.6.6.2.1 -r1.1.6.6.2.2 --- squid/src/fs/aufs/store_io_aufs.c 22 Apr 2000 20:02:55 -0000 1.1.6.6.2.1 +++ squid/src/fs/aufs/store_io_aufs.c 2 May 2000 22:50:29 -0000 1.1.6.6.2.2 @@ -6,10 +6,6 @@ #include "squid.h" #include "store_asyncufs.h" -#define SWAP_DIR_SHIFT 24 -#define SWAP_FILE_MASK 0x00FFFFFF -#define MAGIC1 (NUMTHREADS/Config.cacheSwap.n_configured/2) - static AIOCB storeAufsReadDone; static AIOCB storeAufsWriteDone; static void storeAufsIOCallback(storeIOState * sio, int errflag); @@ -72,7 +68,6 @@ storeIOState * storeAufsCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data) { - aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata; char *path ; storeIOState *sio; sfileno filn; @@ -80,8 +75,7 @@ /* Allocate a number */ dirn = SD->index; - filn = file_map_allocate(aioinfo->map, aioinfo->suggest); - aioinfo->suggest = filn + 1; + filn = storeAufsDirMapBitAllocate(SD); path = storeAufsDirFullPath(SD, filn, NULL); debug(78, 3) ("storeAufsCreate: fileno %08X\n", filn); @@ -108,16 +102,7 @@ store_open_disk_fd++; /* now insert into the replacement policy */ -#ifdef HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->repl.node = heap_insert(SD->repl.heap.heap, e); - debug(20, 4) ("storeUfsCreate: inserted node 0x%x\n", e->repl.node); - } -#else - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); -#endif + storeAufsDirReplAdd(SD, e); return sio; } @@ -218,18 +203,8 @@ storeAufsUnlink(SwapDir *SD, StoreEntry *e) { debug(78, 3) ("storeAufsUnlink: dirno %d, fileno %08X\n", SD->index, e->swap_filen); + storeAufsDirReplRemove(e); storeAufsDirUnlinkFile(SD, e->swap_filen); - -#ifdef HEAP_REPLACEMENT - /* And now, release the object from the replacement policy */ - if (e->repl.node) { - debug(20, 4) ("storeUfsUnlink: deleting node 0x%x\n", e->repl.node); - heap_delete(SD->repl.heap.heap, e->repl.node); - e->repl.node = NULL; - } -#else - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); -#endif } /* === STATIC =========================================================== */ @@ -312,6 +287,13 @@ if (cbdataValid(their_data)) callback(their_data, aiostate->read_buf, rlen); cbdataUnlock(their_data); + /* + * XXX is this safe? The above callback may have caused sio + * to be freed/closed already? Philip Guenther + * says it fixes his FD leaks, with no side effects. + */ + if (aiostate->flags.close_request) + storeAufsIOCallback(sio, DISK_OK); } /* Index: squid/src/fs/coss/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/coss/Attic/Makefile.in,v retrieving revision 1.1.4.2 retrieving revision 1.1.4.2.2.1 diff -u -r1.1.4.2 -r1.1.4.2.2.1 --- squid/src/fs/coss/Makefile.in 5 Mar 2000 23:39:05 -0000 1.1.4.2 +++ squid/src/fs/coss/Makefile.in 2 May 2000 22:50:29 -0000 1.1.4.2.2.1 @@ -47,6 +47,8 @@ -rm -f Makefile.bak -rm -f tags +install: + tags: ctags *.[ch] ../../*.[ch] ../../../include/*.h ../lib/*.[ch] Index: squid/src/fs/coss/store_dir_coss.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/coss/store_dir_coss.c,v retrieving revision 1.1.4.6 retrieving revision 1.1.4.6.2.1 diff -u -r1.1.4.6 -r1.1.4.6.2.1 --- squid/src/fs/coss/store_dir_coss.c 20 Apr 2000 18:14:23 -0000 1.1.4.6 +++ squid/src/fs/coss/store_dir_coss.c 2 May 2000 22:50:29 -0000 1.1.4.6.2.1 @@ -102,18 +102,28 @@ storeCossDirSwapLogFile(SwapDir * sd, const char *ext) { LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; if (Config.Log.swap) { - xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64); - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } } else { - xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); - strcat(path, "/swap.state"); + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); } if (ext) - strncat(path, ext, 16); + strncat(path, ext, 16); return path; } @@ -203,6 +213,7 @@ if (e->swap_filen > -1) { e->swap_filen = -1; } + storeRelease(e); /* Fake an unlink here, this is a bad hack :( */ dlinkDelete(&e->repl.lru, &rb->sd->repl.lru.list); rb->counts.objcount--; @@ -320,7 +331,15 @@ if (fp != NULL) fclose(fp); storeCossDirCloseTmpSwapLog(rb->sd); - storeRebuildComplete(&rb->counts); + /* + * XXX Make sure we don't trigger an assertion if this is the first + * storedir, since if we are, this call will cause storeRebuildComplete + * to prematurely complete the rebuild process, and then some other + * storedir will try to rebuild and eventually die. + */ + store_dirs_rebuilding++; + storeRebuildComplete(&rb->counts); + store_dirs_rebuilding--; xfree(rb); return; } @@ -612,10 +631,11 @@ { /* Check if the object is a special object, we can't cache these */ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) - return 0; + return -1; /* Otherwise, we're ok */ - return 1; + /* Return 900 (90%) load */ + return 900; } /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ @@ -626,7 +646,6 @@ CossInfo *cs = (CossInfo *)SD->fsdata; storeAppendPrintf(sentry, "\n"); - storeAppendPrintf(sentry, "Store Directory #%d: %s\n", SD->index, SD->path); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", @@ -686,6 +705,8 @@ sd->checkobj = storeCossDirCheckObj; sd->refobj = NULL; /* LRU is done in storeCossRead */ sd->unrefobj = NULL; + sd->callback = NULL; + sd->sync = NULL; /* should we make it call the coss sync? */ sd->obj.create = storeCossCreate; sd->obj.open = storeCossOpen; @@ -818,8 +839,6 @@ storefs->parsefunc = storeCossDirParse; storefs->reconfigurefunc = storeCossDirReconfigure; storefs->donefunc = storeCossDirDone; - storefs->callbackfunc = NULL; - storefs->syncfunc = NULL; /* should we make it call the coss sync? */ coss_membuf_pool = memPoolCreate("COSS Membuf data", sizeof(CossMemBuf)); coss_state_pool = memPoolCreate("COSS IO State data", sizeof(CossState)); coss_initialised = 1; Index: squid/src/fs/coss/store_io_coss.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/coss/store_io_coss.c,v retrieving revision 1.1.4.8.2.2 retrieving revision 1.1.4.8.2.3 diff -u -r1.1.4.8.2.2 -r1.1.4.8.2.3 --- squid/src/fs/coss/store_io_coss.c 22 Apr 2000 20:02:55 -0000 1.1.4.8.2.2 +++ squid/src/fs/coss/store_io_coss.c 2 May 2000 22:50:29 -0000 1.1.4.8.2.3 @@ -139,7 +139,7 @@ sio->st_size = objectLen(e) + e->mem_obj->swap_hdr_sz; sio->swap_dirn = SD->index; sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_ALLOCATE); - debug(81, 3) ("storeCossCreate: offset %d\n", sio->swap_filen); + debug(81, 3) ("storeCossCreate: offset %d, size %d, end %d\n", sio->swap_filen, sio->st_size, sio->swap_filen + sio->st_size); sio->callback = callback; sio->callback_data = callback_data; --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/Makefile.in Wed Feb 14 00:43:55 2007 @@ -0,0 +1,93 @@ +# +# Makefile for the DISKD storage driver for the Squid Object Cache server +# +# $Id$ +# + +FS = diskd + +prefix = @prefix@ +exec_prefix = @exec_prefix@ +exec_suffix = @exec_suffix@ +cgi_suffix = @cgi_suffix@ +top_srcdir = @top_srcdir@ +bindir = @bindir@ +libexecdir = @libexecdir@ +sysconfdir = @sysconfdir@ +localstatedir = @localstatedir@ +srcdir = @srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh +LDFLAGS = @LDFLAGS@ +INSTALL = @INSTALL@ +INSTALL_BIN = @INSTALL_PROGRAM@ +MV = @MV@ +RM = @RM@ + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(FS).a +DISKD_EXE = diskd$(exec_suffix) + + +OBJS = \ + store_dir_diskd.o \ + store_io_diskd.o + +UTILS = \ + $(DISKD_EXE) + +all: $(OUT) $(UTILS) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) -DSQUID_PREFIX=\"$(prefix)\" $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a diskd + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c + +diskd.o: $(srcdir)/diskd.c + $(CC) $(CFLAGS) $(srcdir)/diskd.c -c -o diskd.o + +$(DISKD_EXE): diskd.o + $(CC) $(LDFLAGS) -L../../../lib/ $(CFLAGS) diskd.o -o $(DISKD_EXE) -lmiscutil -lm + +install: $(UTILS) + @for f in $(UTILS); do \ + if test -f $(libexecdir)/$$f; then \ + echo $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \ + fi; \ + echo $(INSTALL_BIN) $$f $(libexecdir); \ + $(INSTALL_BIN) $$f $(libexecdir); \ + if test -f $(libexecdir)/-$$f; then \ + echo $(RM) -f $(libexecdir)/-$$f; \ + $(RM) -f $(libexecdir)/-$$f; \ + fi; \ + done + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/diskd.c Wed Feb 14 00:43:55 2007 @@ -0,0 +1,310 @@ + +#include "config.h" +#include "squid.h" + + +#include "store_diskd.h" + +#include +#include +#include + +#undef assert +#include + + +#define STDERR_DEBUG 0 + +typedef struct _file_state file_state; + +struct _file_state { + void *key; + file_state *next; + int id; + int fd; + off_t offset; +}; + +static hash_table *hash = NULL; +static pid_t mypid; +static char *shmbuf; + +static int +do_open(diomsg * r, int len, const char *buf) +{ + int fd; + file_state *fs; + /* + * note r->offset holds open() flags + */ + fd = open(buf, r->offset, 0600); + if (fd < 0) { + fprintf(stderr, "%d %s: ", (int) mypid, buf); + perror("open"); + return -errno; + } + fs = xcalloc(1, sizeof(*fs)); + fs->id = r->id; + fs->key = &fs->id; /* gack */ + fs->fd = fd; + hash_join(hash, (hash_link *) fs); +#if STDERR_DEBUG + fprintf(stderr, "%d OPEN id %d, FD %d, fs %p\n", + (int) mypid, + fs->id, + fs->fd, + fs); +#endif + return fd; +} + +static int +do_close(diomsg * r, int len) +{ + int fd; + file_state *fs; + fs = (file_state *) hash_lookup(hash, &r->id); + if (NULL == fs) { + errno = EBADF; + fprintf(stderr, "%d CLOSE id %d: ", (int) mypid, r->id); + perror("do_close"); + return -EBADF; + } + fd = fs->fd; + hash_remove_link(hash, (hash_link *) fs); +#if STDERR_DEBUG + fprintf(stderr, "%d CLOSE id %d, FD %d, fs %p\n", + (int) mypid, + r->id, + fs->fd, + fs); +#endif + xfree(fs); + return close(fd); +} + +static int +do_read(diomsg * r, int len, char *buf) +{ + int x; + int readlen = r->size; + file_state *fs; + fs = (file_state *) hash_lookup(hash, &r->id); + if (NULL == fs) { + errno = EBADF; + fprintf(stderr, "%d READ id %d: ", (int) mypid, r->id); + perror("do_read"); + return -EBADF; + } + if (r->offset > -1 && r->offset != fs->offset) { +#if STDERR_DEBUG + fprintf(stderr, "seeking to %d\n", r->offset); +#endif + if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { + fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); + perror("lseek"); + } + } + x = read(fs->fd, buf, readlen); +#if STDERR_DEBUG + fprintf(stderr, "%d READ %d,%d,%d ret %d\n", (int) mypid, + fs->fd, readlen, r->offset, x); +#endif + if (x < 0) { + fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); + perror("read"); + return -errno; + } + fs->offset = r->offset + x; + return x; +} + +static int +do_write(diomsg * r, int len, const char *buf) +{ + int wrtlen = r->size; + int x; + file_state *fs; + fs = (file_state *) hash_lookup(hash, &r->id); + if (NULL == fs) { + errno = EBADF; + fprintf(stderr, "%d WRITE id %d: ", (int) mypid, r->id); + perror("do_write"); + return -EBADF; + } + if (r->offset > -1 && r->offset != fs->offset) { + if (lseek(fs->fd, r->offset, SEEK_SET) < 0) { + fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset); + perror("lseek"); + } + } +#if STDERR_DEBUG + fprintf(stderr, "%d WRITE %d,%d,%d\n", (int) mypid, + fs->fd, wrtlen, r->offset); +#endif + x = write(fs->fd, buf, wrtlen); + if (x < 0) { + fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd); + perror("write"); + return -errno; + } + fs->offset = r->offset + x; + return x; +} + +static int +do_unlink(diomsg * r, int len, const char *buf) +{ + if (truncate(buf, 0) < 0) { + fprintf(stderr, "%d UNLNK id %d %s: ", (int) mypid, r->id, buf); + perror("truncate"); + return -errno; + } +#if STDERR_DEBUG + fprintf(stderr, "%d UNLNK %s\n", (int) mypid, buf); +#endif + return 0; +} + +static void +msg_handle(diomsg * r, int rl, diomsg * s) +{ + char *buf = NULL; + s->mtype = r->mtype; + s->callback_data = r->callback_data; + s->shm_offset = r->shm_offset; + s->id = r->id; + if (s->shm_offset > -1) + buf = shmbuf + s->shm_offset; + switch (r->mtype) { + case _MQD_OPEN: + s->status = do_open(r, rl, buf); + break; + case _MQD_CLOSE: + s->status = do_close(r, rl); + break; + case _MQD_READ: + s->status = do_read(r, rl, buf); + break; + case _MQD_WRITE: + s->status = do_write(r, rl, buf); + break; + case _MQD_UNLINK: + s->status = do_unlink(r, rl, buf); + break; + default: + assert(0); + break; + } +} + +int +fsCmp(const void *a, const void *b) +{ + const int *A = a; + const int *B = b; + return *A != *B; +} + +unsigned int +fsHash(const void *key, unsigned int n) +{ + /* note, n must be a power of 2! */ + const int *k = key; + return (*k & (--n)); +} + +static void +alarm_handler(int sig) +{ + (void) 0; +} + +int +main(int argc, char *argv[]) +{ + int key; + int rmsgid; + int smsgid; + int shmid; + diomsg rmsg; + diomsg smsg; + int rlen; + char rbuf[512]; + struct sigaction sa; + setbuf(stdout, NULL); + setbuf(stderr, NULL); + mypid = getpid(); + assert(4 == argc); + key = atoi(argv[1]); + rmsgid = msgget(key, 0600); + if (rmsgid < 0) { + perror("msgget"); + return 1; + } + key = atoi(argv[2]); + smsgid = msgget(key, 0600); + if (smsgid < 0) { + perror("msgget"); + return 1; + } + key = atoi(argv[3]); + shmid = shmget(key, 0, 0600); + if (shmid < 0) { + perror("shmget"); + return 1; + } + shmbuf = shmat(shmid, NULL, 0); + if (shmbuf == (void *) -1) { + perror("shmat"); + return 1; + } + hash = hash_create(fsCmp, 1 << 4, fsHash); + assert(hash); + fcntl(0, F_SETFL, SQUID_NONBLOCK); + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = alarm_handler; + sa.sa_flags = SA_RESTART; + sigaction(SIGALRM, &sa, NULL); + for (;;) { + alarm(1); + memset(&rmsg, '\0', sizeof(rmsg)); + rlen = msgrcv(rmsgid, &rmsg, msg_snd_rcv_sz, 0, 0); + if (rlen < 0) { + if (EINTR == errno) { + if (read(0, rbuf, 512) <= 0) { + if (EWOULDBLOCK == errno) + (void) 0; + else if (EAGAIN == errno) + (void) 0; + else + break; + } + } + if (EAGAIN == errno) { + continue; + } + perror("msgrcv"); + break; + } + alarm(0); + msg_handle(&rmsg, rlen, &smsg); + if (msgsnd(smsgid, &smsg, msg_snd_rcv_sz, 0) < 0) { + perror("msgsnd"); + break; + } + } +#if STDERR_DEBUG + fprintf(stderr, "%d diskd exiting\n", (int) mypid); +#endif + if (msgctl(rmsgid, IPC_RMID, 0) < 0) + perror("msgctl IPC_RMID"); + if (msgctl(smsgid, IPC_RMID, 0) < 0) + perror("msgctl IPC_RMID"); + if (shmdt(shmbuf) < 0) + perror("shmdt"); + if (shmctl(shmid, IPC_RMID, 0) < 0) + perror("shmctl IPC_RMID"); + return 0; +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/store_dir_diskd.c Wed Feb 14 00:43:55 2007 @@ -0,0 +1,2125 @@ + +/* + * $Id$ + * + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Duane Wessels + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#if HAVE_STATVFS +#if HAVE_SYS_STATVFS_H +#include +#endif +#endif + +#include +#include +#include + +#include "store_diskd.h" + +#define DefaultLevelOneDirs 16 +#define DefaultLevelTwoDirs 256 +#define STORE_META_BDISKDZ 4096 + +#ifndef SQUID_PREFIX +#error "SQUID_PREFIX needs defining!" +#endif + +diskd_stats_t diskd_stats; + +typedef struct _RebuildState RebuildState; +struct _RebuildState { + SwapDir *sd; + int n_read; + FILE *log; + int speed; + int curlvl1; + int curlvl2; + struct { + unsigned int need_to_validate:1; + unsigned int clean:1; + unsigned int init:1; + } flags; + int done; + int in_dir; + int fn; + struct dirent *entry; + DIR *td; + char fullpath[SQUID_MAXPATHLEN]; + char fullfilename[SQUID_MAXPATHLEN]; + struct _store_rebuild_data counts; +}; + +static int n_diskd_dirs = 0; +static int *diskd_dir_index = NULL; +MemPool * diskd_state_pool = NULL; +static int diskd_initialised = 0; + +static char *storeDiskdDirSwapSubDir(SwapDir *, int subdirn); +static int storeDiskdDirCreateDirectory(const char *path, int); +static int storeDiskdDirVerifyCacheDirs(SwapDir *); +static int storeDiskdDirVerifyDirectory(const char *path); +static void storeDiskdDirCreateSwapSubDirs(SwapDir *); +static char *storeDiskdDirSwapLogFile(SwapDir *, const char *); +static EVH storeDiskdDirRebuildFromDirectory; +static EVH storeDiskdDirRebuildFromSwapLog; +static int storeDiskdDirGetNextFile(RebuildState *, int *sfileno, int *size); +static StoreEntry *storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean); +static void storeDiskdDirRebuild(SwapDir * sd); +static void storeDiskdDirCloseTmpSwapLog(SwapDir * sd); +static FILE *storeDiskdDirOpenTmpSwapLog(SwapDir *, int *, int *); +static STLOGOPEN storeDiskdDirOpenSwapLog; +static STINIT storeDiskdDirInit; +static STFREE storeDiskdDirFree; +static STLOGCLEANOPEN storeDiskdDirWriteCleanOpen; +static void storeDiskdDirWriteCleanClose(SwapDir * sd); +static STLOGCLEANWRITE storeDiskdDirWriteCleanEntry; +static STLOGCLOSE storeDiskdDirCloseSwapLog; +static STLOGWRITE storeDiskdDirSwapLog; +static STNEWFS storeDiskdDirNewfs; +static STDUMP storeDiskdDirDump; +static STMAINTAINFS storeDiskdDirMaintain; +static STCHECKOBJ storeDiskdDirCheckObj; +static STREFOBJ storeDiskdDirRefObj; +static STUNREFOBJ storeDiskdDirUnrefObj; +static QS rev_int_sort; +static int storeDiskdDirClean(int swap_index); +static EVH storeDiskdDirCleanEvent; +static int storeDiskdDirIs(SwapDir * sd); +static int storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2); +static int storeDiskdCleanupDoubleCheck(SwapDir *, StoreEntry *); +static void storeDiskdDirStats(SwapDir *, StoreEntry *); +static void storeDiskdDirInitBitmap(SwapDir *); +static int storeDiskdDirValidFileno(SwapDir *, sfileno); +static int storeDiskdDirCheckExpired(SwapDir *, StoreEntry *); +#if !HEAP_REPLACEMENT +static time_t storeDiskdDirExpiredReferenceAge(SwapDir *); +#endif +static void storeDiskdStats(StoreEntry * sentry); +static void storeDiskdDirSync(SwapDir *); +static void storeDiskdDirCallback(SwapDir *); + + +/* + * These functions were ripped straight out of the heart of store_dir.c. + * They assume that the given filenum is on a diskd partiton, which may or + * may not be true.. + * XXX this evilness should be tidied up at a later date! + */ + +int +storeDiskdDirMapBitTest(SwapDir *SD, int fn) +{ + sfileno filn = fn; + diskdinfo_t *diskdinfo; + diskdinfo = (diskdinfo_t *)SD->fsdata; + return file_map_bit_test(diskdinfo->map, filn); +} + +void +storeDiskdDirMapBitSet(SwapDir *SD, int fn) +{ + sfileno filn = fn; + diskdinfo_t *diskdinfo; + diskdinfo = (diskdinfo_t *)SD->fsdata; + file_map_bit_set(diskdinfo->map, filn); +} + +void +storeDiskdDirMapBitReset(SwapDir *SD, int fn) +{ + sfileno filn = fn; + diskdinfo_t *diskdinfo; + diskdinfo = (diskdinfo_t *)SD->fsdata; + file_map_bit_reset(diskdinfo->map, filn); +} + +int +storeDiskdDirMapBitAllocate(SwapDir *SD) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + int fn; + fn = file_map_allocate(diskdinfo->map, diskdinfo->suggest); + file_map_bit_set(diskdinfo->map, fn); + diskdinfo->suggest = fn + 1; + return fn; +} + +/* + * Initialise the diskd bitmap + * + * If there already is a bitmap, and the numobjects is larger than currently + * configured, we allocate a new bitmap and 'grow' the old one into it. + */ +static void +storeDiskdDirInitBitmap(SwapDir *sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + + if (diskdinfo->map == NULL) { + /* First time */ + diskdinfo->map = file_map_create(); + } else if (diskdinfo->map->max_n_files) { + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ + } + /* else it shrunk, and we leave the old one in place */ +} + +static char * +storeDiskdDirSwapSubDir(SwapDir * sd, int subdirn) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + assert(0 <= subdirn && subdirn < diskdinfo->l1); + snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn); + return fullfilename; +} + +static int +storeDiskdDirCreateDirectory(const char *path, int should_exist) +{ + int created = 0; + struct stat st; + getCurrentTime(); + if (0 == stat(path, &st)) { + if (S_ISDIR(st.st_mode)) { + debug(20, should_exist ? 3 : 1) ("%s exists\n", path); + } else { + fatalf("Swap directory %s is not a directory.", path); + } + } else if (0 == mkdir(path, 0755)) { + debug(20, should_exist ? 1 : 3) ("%s created\n", path); + created = 1; + } else { + fatalf("Failed to make swap directory %s: %s", + path, xstrerror()); + } + return created; +} + +static int +storeDiskdDirVerifyDirectory(const char *path) +{ + struct stat sb; + if (stat(path, &sb) < 0) { + debug(20, 0) ("%s: %s\n", path, xstrerror()); + return -1; + } + if (S_ISDIR(sb.st_mode) == 0) { + debug(20, 0) ("%s is not a directory\n", path); + return -1; + } + return 0; +} + +/* + * This function is called by storeDiskdDirInit(). If this returns < 0, + * then Squid exits, complains about swap directories not + * existing, and instructs the admin to run 'squid -z' + */ +static int +storeDiskdDirVerifyCacheDirs(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + int j; + const char *path = sd->path; + + if (storeDiskdDirVerifyDirectory(path) < 0) + return -1; + for (j = 0; j < diskdinfo->l1; j++) { + path = storeDiskdDirSwapSubDir(sd, j); + if (storeDiskdDirVerifyDirectory(path) < 0) + return -1; + } + return 0; +} + +static void +storeDiskdDirCreateSwapSubDirs(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + int i, k; + int should_exist; + LOCAL_ARRAY(char, name, MAXPATHLEN); + for (i = 0; i < diskdinfo->l1; i++) { + snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i); + if (storeDiskdDirCreateDirectory(name, 0)) + should_exist = 0; + else + should_exist = 1; + debug(47, 1) ("Making directories in %s\n", name); + for (k = 0; k < diskdinfo->l2; k++) { + snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k); + storeDiskdDirCreateDirectory(name, should_exist); + } + } +} + +static char * +storeDiskdDirSwapLogFile(SwapDir * sd, const char *ext) +{ + LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; + if (Config.Log.swap) { + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } + } else { + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); + } + if (ext) + strncat(path, ext, 16); + return path; +} + +static void +storeDiskdDirOpenSwapLog(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + char *path; + int fd; + path = storeDiskdDirSwapLogFile(sd, NULL); + fd = file_open(path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", path, xstrerror()); + fatal("storeDiskdDirOpenSwapLog: Failed to open swap log."); + } + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); + diskdinfo->swaplog_fd = fd; + if (0 == n_diskd_dirs) + assert(NULL == diskd_dir_index); + n_diskd_dirs++; + assert(n_diskd_dirs <= Config.cacheSwap.n_configured); +} + +static void +storeDiskdDirCloseSwapLog(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + if (diskdinfo->swaplog_fd < 0) /* not open */ + return; + file_close(diskdinfo->swaplog_fd); + debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", + sd->index, diskdinfo->swaplog_fd); + diskdinfo->swaplog_fd = -1; + n_diskd_dirs--; + assert(n_diskd_dirs >= 0); + if (0 == n_diskd_dirs) + safe_free(diskd_dir_index); +} + +static void +storeDiskdDirInit(SwapDir * sd) +{ + static int started_clean_event = 0; + int x; + int i; + int rfd; + int ikey = (getpid() << 16) + (sd->index << 4); + char *args[5]; + char skey1[32]; + char skey2[32]; + char skey3[32]; + diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata; + static const char *errmsg = + "\tFailed to verify one of the swap directories, Check cache.log\n" + "\tfor details. Run 'squid -z' to create swap directories\n" + "\tif needed, or if running Squid for the first time."; + + diskdinfo->smsgid = msgget((key_t) ikey, 0700 | IPC_CREAT); + if (diskdinfo->smsgid < 0) { + debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); + fatal("msgget failed"); + } + diskdinfo->rmsgid = msgget((key_t) (ikey + 1), 0700 | IPC_CREAT); + if (diskdinfo->rmsgid < 0) { + debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror()); + fatal("msgget failed"); + } + diskdinfo->shm.id = shmget((key_t) (ikey + 2), + SHMBUFS * SHMBUF_BLKSZ, 0600 | IPC_CREAT); + if (diskdinfo->shm.id < 0) { + debug(50, 0) ("storeDiskdInit: shmget: %s\n", xstrerror()); + fatal("shmget failed"); + } + diskdinfo->shm.buf = shmat(diskdinfo->shm.id, NULL, 0); + if (diskdinfo->shm.buf == (void *) -1) { + debug(50, 0) ("storeDiskdInit: shmat: %s\n", xstrerror()); + fatal("shmat failed"); + } + diskd_stats.shmbuf_count += SHMBUFS; + for (i = 0; i < SHMBUFS; i++) + storeDiskdShmPut(sd, i * SHMBUF_BLKSZ); + snprintf(skey1, 32, "%d", ikey); + snprintf(skey2, 32, "%d", ikey + 1); + snprintf(skey3, 32, "%d", ikey + 2); + args[0] = "diskd"; + args[1] = skey1; + args[2] = skey2; + args[3] = skey3; + args[4] = NULL; +#if HAVE_POLL && defined(_SQUID_OSF_) + /* pipes and poll() don't get along on DUNIX -DW */ + x = ipcCreate(IPC_TCP_SOCKET, +#else + x = ipcCreate(IPC_FIFO, +#endif + SQUID_PREFIX "/bin/diskd", + args, + "diskd", + &rfd, + &diskdinfo->wfd); + if (x < 0) + fatal("execl " SQUID_PREFIX "/bin/diskd failed"); + if (rfd != diskdinfo->wfd) + comm_close(rfd); + fd_note(diskdinfo->wfd, "squid -> diskd"); + commSetTimeout(diskdinfo->wfd, -1, NULL, NULL); + commSetNonBlocking(diskdinfo->wfd); + storeDiskdDirInitBitmap(sd); + if (storeDiskdDirVerifyCacheDirs(sd) < 0) + fatal(errmsg); + storeDiskdDirOpenSwapLog(sd); + storeDiskdDirRebuild(sd); + if (!started_clean_event) { + eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL, 15.0, 1); + started_clean_event = 1; + } +} + + +static void +storeDiskdStats(StoreEntry * sentry) +{ + storeAppendPrintf(sentry, "sent_count: %d\n", diskd_stats.sent_count); + storeAppendPrintf(sentry, "recv_count: %d\n", diskd_stats.recv_count); + storeAppendPrintf(sentry, "max_away: %d\n", diskd_stats.max_away); + storeAppendPrintf(sentry, "max_shmuse: %d\n", diskd_stats.max_shmuse); + storeAppendPrintf(sentry, "open_fail_queue_len: %d\n", diskd_stats.open_fail_queue_len); + storeAppendPrintf(sentry, "block_queue_len: %d\n", diskd_stats.block_queue_len); + diskd_stats.max_away = diskd_stats.max_shmuse = 0; +} + +/* + * storeDiskdDirSync + * + * Sync any pending data. We just sit around and read the queue + * until the data has finished writing. + */ +static void +storeDiskdDirSync(SwapDir *SD) +{ + /* XXX NOT DONE YET! */ +#warning "storeDiskdSync() needs to be written" +} + + +/* + * storeDiskdDirCallback + * + * Handle callbacks. If we have more than magic2 requests away, we block + * until the queue is below magic2. Otherwise, we simply return when we + * don't get a message. + */ +static void +storeDiskdDirCallback(SwapDir *SD) +{ + diomsg M; + int x; + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + + if (diskdinfo->away >= diskdinfo->magic2) + diskd_stats.block_queue_len++; + + if (diskd_stats.sent_count - diskd_stats.recv_count > + diskd_stats.max_away) { + diskd_stats.max_away = diskd_stats.sent_count - diskd_stats.recv_count; + diskd_stats.max_shmuse = diskd_stats.shmbuf_count; + } + + /* if we are above magic2, we do not break under any reason */ + while (1) { + memset(&M, '\0', sizeof(M)); + x = msgrcv(diskdinfo->rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT); + if (x < 0) { + if (diskdinfo->away >= diskdinfo->magic2) + continue; + else + break; + } else if (x != msg_snd_rcv_sz) { + debug(81, 1) ("storeDiskdReadIndividualQueue: msgget returns %d\n", + x); + break; + } + diskd_stats.recv_count++; + diskdinfo->away--; + storeDiskdHandle(&M); + if (M.shm_offset > -1) + storeDiskdShmPut(SD, M.shm_offset); + } +} + + + +static void +storeDiskdDirRebuildFromDirectory(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE); + StoreEntry *e = NULL; + StoreEntry tmpe; + cache_key key[MD5_DIGEST_CHARS]; + int sfileno = 0; + int count; + int size; + struct stat sb; + int swap_hdr_len; + int fd = -1; + tlv *tlv_list; + tlv *t; + assert(rb != NULL); + debug(20, 3) ("storeDiskdDirRebuildFromDirectory: DIR #%d\n", rb->sd->index); + for (count = 0; count < rb->speed; count++) { + assert(fd == -1); + fd = storeDiskdDirGetNextFile(rb, &sfileno, &size); + if (fd == -2) { + debug(20, 1) ("Done scanning %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + store_dirs_rebuilding--; + storeDiskdDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } else if (fd < 0) { + continue; + } + assert(fd > -1); + /* lets get file stats here */ + if (fstat(fd, &sb) < 0) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: fstat(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %s %7d files opened so far.\n", + rb->sd->path, rb->counts.scancount); + debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno); + Counter.syscalls.disk.reads++; + if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: read(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + store_open_disk_fd--; + fd = -1; + continue; + } + file_close(fd); + store_open_disk_fd--; + fd = -1; + swap_hdr_len = 0; +#if USE_TRUNCATE + if (sb.st_size == 0) + continue; +#endif + tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); + if (tlv_list == NULL) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: failed to get meta data\n"); + /* XXX shouldn't this be a call to storeDiskdUnlink ? */ + storeDiskdDirUnlinkFile(SD, sfileno); + continue; + } + debug(20, 3) ("storeDiskdDirRebuildFromDirectory: successful swap meta unpacking\n"); + memset(key, '\0', MD5_DIGEST_CHARS); + memset(&tmpe, '\0', sizeof(StoreEntry)); + for (t = tlv_list; t; t = t->next) { + switch (t->type) { + case STORE_META_KEY: + assert(t->length == MD5_DIGEST_CHARS); + xmemcpy(key, t->value, MD5_DIGEST_CHARS); + break; + case STORE_META_STD: + assert(t->length == STORE_HDR_METASIZE); + xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); + break; + default: + break; + } + } + storeSwapTLVFree(tlv_list); + tlv_list = NULL; + if (storeKeyNull(key)) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: NULL key\n"); + storeDiskdDirUnlinkFile(SD, sfileno); + continue; + } + tmpe.key = key; + /* check sizes */ + if (tmpe.swap_file_sz == 0) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { + tmpe.swap_file_sz = sb.st_size; + } else if (tmpe.swap_file_sz != sb.st_size) { + debug(20, 1) ("storeDiskdDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n", + tmpe.swap_file_sz, (int) sb.st_size); + storeDiskdDirUnlinkFile(SD, sfileno); + continue; + } + if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { + storeDiskdDirUnlinkFile(SD, sfileno); + rb->counts.badflags++; + continue; + } + e = storeGet(key); + if (e && e->lastref >= tmpe.lastref) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (NULL != e) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + rb->counts.dupcount++; + } + rb->counts.objcount++; + storeEntryDump(&tmpe, 5); + e = storeDiskdDirAddDiskRestore(SD, key, + sfileno, + tmpe.swap_file_sz, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + tmpe.refcount, /* refcount */ + tmpe.flags, /* flags */ + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeDiskdDirRebuildFromDirectory, rb, 0.0, 1); +} + +static void +storeDiskdDirRebuildFromSwapLog(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + StoreEntry *e = NULL; + storeSwapLogData s; + size_t ss = sizeof(storeSwapLogData); + int count; + int used; /* is swapfile already in use? */ + int disk_entry_newer; /* is the log entry newer than current entry? */ + double x; + assert(rb != NULL); + /* load a number of objects per invocation */ + for (count = 0; count < rb->speed; count++) { + if (fread(&s, ss, 1, rb->log) != 1) { + debug(20, 1) ("Done reading %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + fclose(rb->log); + rb->log = NULL; + store_dirs_rebuilding--; + storeDiskdDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); + return; + } + rb->n_read++; + if (s.op <= SWAP_LOG_NOP) + continue; + if (s.op >= SWAP_LOG_MAX) + continue; + debug(20, 3) ("storeDiskdDirRebuildFromSwapLog: %s %s %08X\n", + swap_log_op_str[(int) s.op], + storeKeyText(s.key), + s.swap_filen); + if (s.op == SWAP_LOG_ADD) { + (void) 0; + } else if (s.op == SWAP_LOG_DEL) { + if ((e = storeGet(s.key)) != NULL) { + /* + * Make sure we don't unlink the file, it might be + * in use by a subsequent entry. Also note that + * we don't have to subtract from store_swap_size + * because adding to store_swap_size happens in + * the cleanup procedure. + */ + storeExpireNow(e); + storeReleaseRequest(e); + storeDiskdDirReplRemove(e); + if (e->swap_filen > -1) { + storeDiskdDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.objcount--; + rb->counts.cancelcount++; + } + continue; + } else { + x = log(++rb->counts.bad_log_op) / log(10.0); + if (0.0 == x - (double) (int) x) + debug(20, 1) ("WARNING: %d invalid swap log entries found\n", + rb->counts.bad_log_op); + rb->counts.invalid++; + continue; + } + if ((++rb->counts.scancount & 0xFFFF) == 0) + debug(20, 3) (" %7d %s Entries read so far.\n", + rb->counts.scancount, rb->sd->path); + if (!storeDiskdDirValidFileno(SD, s.swap_filen)) { + rb->counts.invalid++; + continue; + } + if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + rb->counts.badflags++; + continue; + } + e = storeGet(s.key); + used = storeDiskdDirMapBitTest(SD, s.swap_filen); + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; + if (used && !disk_entry_newer) { + /* log entry is old, ignore it */ + rb->counts.clashcount++; + continue; + } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) { + /* swapfile taken, same URL, newer, update meta */ + if (e->store_status == STORE_OK) { + e->lastref = s.timestamp; + e->timestamp = s.timestamp; + e->expires = s.expires; + e->lastmod = s.lastmod; + e->flags = s.flags; + e->refcount += s.refcount; +#if HEAP_REPLACEMENT + storeHeapPositionUpdate(e, SD); + storeDiskdDirUnrefObj(SD, e); +#endif + } else { + debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition"); + debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); + } + continue; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", + SD->index, s.swap_filen); + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + assert(rb->flags.need_to_validate); + rb->counts.clashcount++; + continue; + } else if (e && !disk_entry_newer) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; + continue; + } else if (e) { + /* key already exists, this swapfile not being used */ + /* junk old, load new */ + storeExpireNow(e); + storeReleaseRequest(e); + storeDiskdDirReplRemove(e); + if (e->swap_filen > -1) { + /* Make sure we don't actually unlink the file */ + storeDiskdDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.dupcount++; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + /* update store_swap_size */ + rb->counts.objcount++; + e = storeDiskdDirAddDiskRestore(SD, s.key, + s.swap_filen, + s.swap_file_sz, + s.expires, + s.timestamp, + s.lastref, + s.lastmod, + s.refcount, + s.flags, + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1); +} + +static int +storeDiskdDirGetNextFile(RebuildState * rb, int *sfileno, int *size) +{ + SwapDir *SD = rb->sd; + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + int fd = -1; + int used = 0; + int dirs_opened = 0; + debug(20, 3) ("storeDiskdDirGetNextFile: flag=%d, %d: /%02X/%02X\n", + rb->flags.init, + rb->sd->index, + rb->curlvl1, + rb->curlvl2); + if (rb->done) + return -2; + while (fd < 0 && rb->done == 0) { + fd = -1; + if (0 == rb->flags.init) { /* initialize, open first file */ + rb->done = 0; + rb->curlvl1 = 0; + rb->curlvl2 = 0; + rb->in_dir = 0; + rb->flags.init = 1; + assert(Config.cacheSwap.n_configured > 0); + } + if (0 == rb->in_dir) { /* we need to read in a new directory */ + snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", + rb->sd->path, + rb->curlvl1, rb->curlvl2); + if (rb->flags.init && rb->td != NULL) + closedir(rb->td); + rb->td = NULL; + if (dirs_opened) + return -1; + rb->td = opendir(rb->fullpath); + dirs_opened++; + if (rb->td == NULL) { + debug(50, 1) ("storeDiskdDirGetNextFile: opendir: %s: %s\n", + rb->fullpath, xstrerror()); + } else { + rb->entry = readdir(rb->td); /* skip . and .. */ + rb->entry = readdir(rb->td); + if (rb->entry == NULL && errno == ENOENT) + debug(20, 1) ("storeDiskdDirGetNextFile: directory does not exist!.\n"); + debug(20, 3) ("storeDiskdDirGetNextFile: Directory %s\n", rb->fullpath); + } + } + if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) { + rb->in_dir++; + if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) { + debug(20, 3) ("storeDiskdDirGetNextFile: invalid %s\n", + rb->entry->d_name); + continue; + } + if (!storeDiskdFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) { + debug(20, 3) ("storeDiskdDirGetNextFile: %08X does not belong in %d/%d/%d\n", + rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2); + continue; + } + used = storeDiskdDirMapBitTest(SD, rb->fn); + if (used) { + debug(20, 3) ("storeDiskdDirGetNextFile: Locked, continuing with next.\n"); + continue; + } + snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s", + rb->fullpath, rb->entry->d_name); + debug(20, 3) ("storeDiskdDirGetNextFile: Opening %s\n", rb->fullfilename); + fd = file_open(rb->fullfilename, O_RDONLY); + if (fd < 0) + debug(50, 1) ("storeDiskdDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror()); + else + store_open_disk_fd++; + continue; + } + rb->in_dir = 0; + if (++rb->curlvl2 < diskdinfo->l2) + continue; + rb->curlvl2 = 0; + if (++rb->curlvl1 < diskdinfo->l1) + continue; + rb->curlvl1 = 0; + rb->done = 1; + } + *sfileno = rb->fn; + return fd; +} + +/* Add a new object to the cache with empty memory copy and pointer to disk + * use to rebuild store from disk. */ +static StoreEntry * +storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_short flags, + int clean) +{ + StoreEntry *e = NULL; + debug(20, 5) ("storeDiskdAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); + /* if you call this you'd better be sure file_number is not + * already in use! */ + e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + e->store_status = STORE_OK; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; + e->swap_filen = file_number; + e->swap_dirn = SD->index; + e->swap_file_sz = swap_file_sz; + e->lock_count = 0; +#if !HEAP_REPLACEMENT + e->refcount = 0; +#endif + e->lastref = lastref; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->refcount = refcount; + e->flags = flags; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + storeDiskdDirMapBitSet(SD, e->swap_filen); + storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ + storeDiskdDirReplAdd(SD, e); + return e; +} + +static void +storeDiskdDirRebuild(SwapDir * sd) +{ + RebuildState *rb = xcalloc(1, sizeof(*rb)); + int clean = 0; + int zero = 0; + FILE *fp; + EVH *func = NULL; + rb->sd = sd; + rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; + /* + * If the swap.state file exists in the cache_dir, then + * we'll use storeDiskdDirRebuildFromSwapLog(), otherwise we'll + * use storeDiskdDirRebuildFromDirectory() to open up each file + * and suck in the meta data. + */ + fp = storeDiskdDirOpenTmpSwapLog(sd, &clean, &zero); + if (fp == NULL || zero) { + if (fp != NULL) + fclose(fp); + func = storeDiskdDirRebuildFromDirectory; + } else { + func = storeDiskdDirRebuildFromSwapLog; + rb->log = fp; + rb->flags.clean = (unsigned int) clean; + } + if (!clean) + rb->flags.need_to_validate = 1; + debug(20, 1) ("Rebuilding storage in %s (%s)\n", + sd->path, clean ? "CLEAN" : "DIRTY"); + store_dirs_rebuilding++; + cbdataAdd(rb, cbdataXfree, 0); + eventAdd("storeRebuild", func, rb, 0.0, 1); +} + +static void +storeDiskdDirCloseTmpSwapLog(SwapDir * sd) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); + char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new")); + int fd; + file_close(diskdinfo->swaplog_fd); +#ifdef _SQUID_OS2_ + if (unlink(swaplog_path) < 0) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeDiskdDirCloseTmpSwapLog: unlink failed"); + } +#endif + if (xrename(new_path, swaplog_path) < 0) { + fatal("storeDiskdDirCloseTmpSwapLog: rename failed"); + } + fd = file_open(swaplog_path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("storeDiskdDirCloseTmpSwapLog: Failed to open swap log."); + } + safe_free(swaplog_path); + safe_free(new_path); + diskdinfo->swaplog_fd = fd; + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); +} + +static FILE * +storeDiskdDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); + char *clean_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean")); + char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new")); + struct stat log_sb; + struct stat clean_sb; + FILE *fp; + int fd; + if (stat(swaplog_path, &log_sb) < 0) { + debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return NULL; + } + *zero_flag = log_sb.st_size == 0 ? 1 : 0; + /* close the existing write-only FD */ + if (diskdinfo->swaplog_fd >= 0) + file_close(diskdinfo->swaplog_fd); + /* open a write-only FD for the new log */ + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", new_path, xstrerror()); + fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); + } + diskdinfo->swaplog_fd = fd; + /* open a read-only stream of the old log */ + fp = fopen(swaplog_path, "r"); + if (fp == NULL) { + debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); + fatal("Failed to open swap log for reading"); + } + memset(&clean_sb, '\0', sizeof(struct stat)); + if (stat(clean_path, &clean_sb) < 0) + *clean_flag = 0; + else if (clean_sb.st_mtime < log_sb.st_mtime) + *clean_flag = 0; + else + *clean_flag = 1; + safeunlink(clean_path, 1); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return fp; +} + +struct _clean_state { + char *cur; + char *new; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; +}; + +#define CLEAN_BUF_SZ 16384 +/* + * Begin the process to write clean cache state. For DISKD this means + * opening some log files and allocating write buffers. Return 0 if + * we succeed, and assign the 'func' and 'data' return pointers. + */ +static int +storeDiskdDirWriteCleanOpen(SwapDir * sd) +{ + struct _clean_state *state = xcalloc(1, sizeof(*state)); + struct stat sb; + sd->log.clean.write = NULL; + sd->log.clean.state = NULL; + state->cur = xstrdup(storeDiskdDirSwapLogFile(sd, NULL)); + state->new = xstrdup(storeDiskdDirSwapLogFile(sd, ".clean")); + state->cln = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean")); + state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); + state->outbuf_offset = 0; + unlink(state->new); + unlink(state->cln); + state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC); + if (state->fd < 0) + return -1; + debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n", + state->new, state->fd); +#if HAVE_FCHMOD + if (stat(state->cur, &sb) == 0) + fchmod(state->fd, sb.st_mode); +#endif + sd->log.clean.write = storeDiskdDirWriteCleanEntry; + sd->log.clean.state = state; + return 0; +} + +/* + * "write" an entry to the clean log file. + */ +static void +storeDiskdDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd) +{ + storeSwapLogData s; + static size_t ss = sizeof(storeSwapLogData); + struct _clean_state *state = sd->log.clean.state; + if (NULL == e) { + storeDiskdDirWriteCleanClose(sd); + return; + } + memset(&s, '\0', ss); + s.op = (char) SWAP_LOG_ADD; + s.swap_filen = e->swap_filen; + s.timestamp = e->timestamp; + s.lastref = e->lastref; + s.expires = e->expires; + s.lastmod = e->lastmod; + s.swap_file_sz = e->swap_file_sz; + s.refcount = e->refcount; + s.flags = e->flags; + xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); + xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); + state->outbuf_offset += ss; + /* buffered write */ + if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; + } + state->outbuf_offset = 0; + } +} + +static void +storeDiskdDirWriteCleanClose(SwapDir * sd) +{ + struct _clean_state *state = sd->log.clean.state; + if (state->fd < 0) + return; + if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", + state->new, xstrerror()); + debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile " + "not replaced.\n"); + file_close(state->fd); + state->fd = -1; + unlink(state->new); + } + safe_free(state->outbuf); + /* + * You can't rename open files on Microsoft "operating systems" + * so we have to close before renaming. + */ + storeDiskdDirCloseSwapLog(sd); + /* rename */ + if (state->fd >= 0) { +#ifdef _SQUID_OS2_ + file_close(state->fd); + state->fd = -1; + if (unlink(cur) < 0) + debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n", + xstrerror(), cur); +#endif + xrename(state->new, state->cur); + } + /* touch a timestamp file if we're not still validating */ + if (store_dirs_rebuilding) + (void) 0; + else if (state->fd < 0) + (void) 0; + else + file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC)); + /* close */ + safe_free(state->cur); + safe_free(state->new); + safe_free(state->cln); + if (state->fd >= 0) + file_close(state->fd); + state->fd = -1; + safe_free(state); + sd->log.clean.state = NULL; + sd->log.clean.write = NULL; +} + +static void +storeDiskdDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); + s->op = (char) op; + s->swap_filen = e->swap_filen; + s->timestamp = e->timestamp; + s->lastref = e->lastref; + s->expires = e->expires; + s->lastmod = e->lastmod; + s->swap_file_sz = e->swap_file_sz; + s->refcount = e->refcount; + s->flags = e->flags; + xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); + file_write(diskdinfo->swaplog_fd, + -1, + s, + sizeof(storeSwapLogData), + NULL, + NULL, + xfree); +} + +static void +storeDiskdDirNewfs(SwapDir * sd) +{ + debug(47, 3) ("Creating swap space in %s\n", sd->path); + storeDiskdDirCreateDirectory(sd->path, 0); + storeDiskdDirCreateSwapSubDirs(sd); +} + +static int +rev_int_sort(const void *A, const void *B) +{ + const int *i1 = A; + const int *i2 = B; + return *i2 - *i1; +} + +static int +storeDiskdDirClean(int swap_index) +{ + DIR *dp = NULL; + struct dirent *de = NULL; + LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); + LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); +#if USE_TRUNCATE + struct stat sb; +#endif + int files[20]; + int swapfileno; + int fn; /* same as swapfileno, but with dirn bits set */ + int n = 0; + int k = 0; + int N0, N1, N2; + int D0, D1, D2; + SwapDir *SD; + diskdinfo_t *diskdinfo; + N0 = n_diskd_dirs; + D0 = diskd_dir_index[swap_index % N0]; + SD = &Config.cacheSwap.swapDirs[D0]; + diskdinfo = (diskdinfo_t *)SD->fsdata; + N1 = diskdinfo->l1; + D1 = (swap_index / N0) % N1; + N2 = diskdinfo->l2; + D2 = ((swap_index / N0) / N1) % N2; + snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X", + Config.cacheSwap.swapDirs[D0].path, D1, D2); + debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1); + dp = opendir(p1); + if (dp == NULL) { + if (errno == ENOENT) { + debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1); + if (mkdir(p1, 0777) == 0) + return 0; + } + debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror()); + safeunlink(p1, 1); + return 0; + } + while ((de = readdir(dp)) != NULL && k < 20) { + if (sscanf(de->d_name, "%X", &swapfileno) != 1) + continue; + fn = swapfileno; /* XXX should remove this cruft ! */ + if (storeDiskdDirValidFileno(SD, fn)) + if (storeDiskdDirMapBitTest(SD, fn)) + if (storeDiskdFilenoBelongsHere(fn, D0, D1, D2)) + continue; +#if USE_TRUNCATE + if (!stat(de->d_name, &sb)) + if (sb.st_size == 0) + continue; +#endif + files[k++] = swapfileno; + } + closedir(dp); + if (k == 0) + return 0; + qsort(files, k, sizeof(int), rev_int_sort); + if (k > 10) + k = 10; + for (n = 0; n < k; n++) { + debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]); + snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); +#if USE_TRUNCATE + truncate(p2, 0); +#else + safeunlink(p2, 0); +#endif + Counter.swap_files_cleaned++; + } + debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1); + return k; +} + +static void +storeDiskdDirCleanEvent(void *unused) +{ + static int swap_index = 0; + int i; + int j = 0; + int n = 0; + /* + * Assert that there are DISKD cache_dirs configured, otherwise + * we should never be called. + */ + assert(n_diskd_dirs); + if (NULL == diskd_dir_index) { + SwapDir *sd; + diskdinfo_t *diskdinfo; + /* + * Initialize the little array that translates DISKD cache_dir + * number into the Config.cacheSwap.swapDirs array index. + */ + diskd_dir_index = xcalloc(n_diskd_dirs, sizeof(*diskd_dir_index)); + for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { + sd = &Config.cacheSwap.swapDirs[i]; + if (!storeDiskdDirIs(sd)) + continue; + diskd_dir_index[n++] = i; + diskdinfo = (diskdinfo_t *)sd->fsdata; + j += (diskdinfo->l1 * diskdinfo->l2); + } + assert(n == n_diskd_dirs); + /* + * Start the storeDiskdDirClean() swap_index with a random + * value. j equals the total number of DISKD level 2 + * swap directories + */ + swap_index = (int) (squid_random() % j); + } + if (0 == store_dirs_rebuilding) { + n = storeDiskdDirClean(swap_index); + swap_index++; + } + eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL, + 15.0 * exp(-0.25 * n), 1); +} + +static int +storeDiskdDirIs(SwapDir * sd) +{ + if (strncmp(sd->type, "diskd", 3) == 0) + return 1; + return 0; +} + +/* + * Does swapfile number 'fn' belong in cachedir #F0, + * level1 dir #F1, level2 dir #F2? + */ +static int +storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2) +{ + int D1, D2; + int L1, L2; + int filn = fn; + diskdinfo_t *diskdinfo; + assert(F0 < Config.cacheSwap.n_configured); + diskdinfo = (diskdinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata; + L1 = diskdinfo->l1; + L2 = diskdinfo->l2; + D1 = ((filn / L2) / L2) % L1; + if (F1 != D1) + return 0; + D2 = (filn / L2) % L2; + if (F2 != D2) + return 0; + return 1; +} + +int +storeDiskdDirValidFileno(SwapDir *SD, sfileno filn) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + if (filn < 0) + return 0; + if (filn > diskdinfo->map->max_n_files) + return 0; + return 1; +} + +void +storeDiskdDirMaintain(SwapDir *SD) +{ + StoreEntry *e = NULL; + int scanned = 0; + int locked = 0; + int expired = 0; + int max_scan; + int max_remove; + double f; + static time_t last_warn_time = 0; +#if !HEAP_REPLACEMENT + dlink_node *m; + dlink_node *prev = NULL; +#else + heap_key age; + heap_key min_age = 0.0; + link_list *locked_entries = NULL; +#if HEAP_REPLACEMENT_DEBUG + if (!verify_heap_property(SD->repl.heap.heap)) { + debug(20, 1) ("Heap property violated!\n"); + } +#endif +#endif + /* We can't delete objects while rebuilding swap */ + if (store_dirs_rebuilding) { + return; + } else { + f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low); + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + max_scan = (int) (f * 400.0 + 100.0); + max_remove = (int) (f * 70.0 + 10.0); + /* + * This is kinda cheap, but so we need this priority hack? + */ +#if 0 + eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1); +#endif + } + debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove); +#if HEAP_REPLACEMENT + while (heap_nodes(SD->repl.heap.heap) > 0) { + if (store_swap_size < store_swap_low) + break; + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + age = heap_peepminkey(SD->repl.heap.heap); + e = heap_extractmin(SD->repl.heap.heap); + e->repl.node = NULL; /* no longer in the heap */ + scanned++; + if (storeEntryLocked(e)) { + /* + * Entry is in use ... put it in a linked list to ignore it. + */ + if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + /* + * If this was a "SPECIAL" do not add it back into the heap. + * It will always be "SPECIAL" and therefore never removed. + */ + debug(20, 4) ("storeDiskdDirMaintain: locked url %s\n", + (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e-> +key)); + linklistPush(&locked_entries, e); + } + locked++; + continue; + } else if (storeDiskdDirCheckExpired(SD, e)) { + /* + * Note: This will not check the reference age ifdef + * HEAP_REPLACEMENT, but it does some other useful + * checks... + */ + expired++; + debug(20, 3) ("Released store object age %f size %d refs %d key %s\n", + age, e->swap_file_sz, e->refcount, storeKeyText(e->key)); + min_age = age; + storeRelease(e); + } else { + /* + * Did not expire the object so we need to add it back + * into the heap! + */ + debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n", + storeKeyText(e->key)); + linklistPush(&locked_entries, e); + continue; + } + if (store_swap_size < store_swap_low) + break; + else if (expired >= max_remove) + break; + else if (scanned >= max_scan) + break; + } + /* + * Bump the heap age factor. + */ + if (min_age > 0.0) + SD->repl.heap.heap->age = min_age; + /* + * Reinsert all bumped locked entries back into heap... + */ + while ((e = linklistShift(&locked_entries))) + e->repl.node = heap_insert(SD->repl.heap.heap, e); +#else + for (m = SD->repl.lru.list.tail; m; m = prev) { + prev = m->prev; + e = m->data; + scanned++; + if (storeEntryLocked(e)) { + /* + * If there is a locked entry at the tail of the LRU list, + * move it to the beginning to get it out of the way. + * Theoretically, we might have all locked objects at the + * tail, and then we'll never remove anything here and the + * LRU age will go to zero. + */ + if (memInUse(MEM_STOREENTRY) > max_scan) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } + locked++; + + } else if (storeDiskdDirCheckExpired(SD, e)) { + expired++; + storeRelease(e); + } + if (expired >= max_remove) + break; + if (scanned >= max_scan) + break; + } +#endif + debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l +ocked %d f=%.03f\n", + scanned, max_scan, expired, max_remove, locked, f); + debug(20, 3) ("storeMaintainSwapSpace stats:\n"); + debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); + debug(20, 3) (" %6d were scanned\n", scanned); + debug(20, 3) (" %6d were locked\n", locked); + debug(20, 3) (" %6d were expired\n", expired); + if (store_swap_size < Config.Swap.maxSize) + return; + if (squid_curtime - last_warn_time < 10) + return; + debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", + store_swap_size, Config.Swap.maxSize); + last_warn_time = squid_curtime; +} + +/* + * storeDiskdDirCheckObj + * + * This routine is called by storeDirSelectSwapDir to see if the given + * object is able to be stored on this filesystem. DISKD filesystems will + * happily store anything as long as the LRU time isn't too small. + */ +int +storeDiskdDirCheckObj(SwapDir *SD, const StoreEntry *e) +{ + int loadav; + + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; +#if !HEAP_REPLACEMENT + if (storeDiskdDirExpiredReferenceAge(SD) < 300) { + debug(20, 3) ("storeDiskdDirCheckObj: NO: LRU Age = %d\n", + storeDiskdDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; + } +#endif + + /* Check the queue length */ + if (diskdinfo->away >= diskdinfo->magic1) + return -1; + + /* Calculate the storedir load relative to magic2 on a scale of 0 .. 1000 */ + if (diskdinfo->away == 0) + loadav = 0; + else + loadav = diskdinfo->magic2 * 1000 / diskdinfo->away; + return loadav; +} + +/* + * storeDiskdDirRefObj + * + * This routine is called whenever an object is referenced, so we can + * maintain replacement information within the storage fs. + */ +void +storeDiskdDirRefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeDiskdDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); +#if HEAP_REPLACEMENT + /* Nothing to do here */ +#else + /* Reference the object */ + if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && + !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } +#endif +} + +/* + * storeDiskdDirUnrefObj + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ +void +storeDiskdDirUnrefObj(SwapDir *SD, StoreEntry *e) +{ + debug(1, 3) ("storeDiskdDirUnrefObj: referencing %p %d/%d\n", e, + e->swap_dirn, e->swap_filen); +#if HEAP_REPLACEMENT + if (e->repl.node) + heap_update(SD->repl.heap.heap, e->repl.node, e); +#endif +} + +/* + * storeDiskdDirUnlinkFile + * + * This is a *synchronous* unlink which is currently used in the rebuild + * process. This is bad, but it'll have to stay until the dir rebuild + * uses storeDiskdUnlink() .. + */ +void +storeDiskdDirUnlinkFile(SwapDir *SD, sfileno f) +{ + debug(79, 3) ("storeDiskdDirUnlinkFile: unlinking fileno %08X\n", f); + storeDiskdDirMapBitReset(SD, f); + unlinkdUnlink(storeDiskdDirFullPath(SD, f, NULL)); +} + +#if !HEAP_REPLACEMENT +/* + * storeDiskdDirExpiredReferenceAge + * + * The LRU age is scaled exponentially between 1 minute and + * Config.referenceAge , when store_swap_low < store_swap_size < + * store_swap_high. This keeps store_swap_size within the low and high + * water marks. If the cache is very busy then store_swap_size stays + * closer to the low water mark, if it is not busy, then it will stay + * near the high water mark. The LRU age value can be examined on the + * cachemgr 'info' page. + */ +static time_t +storeDiskdDirExpiredReferenceAge(SwapDir *SD) +{ + double x; + double z; + time_t age; + long store_high, store_low; + + store_high = (long) (((float) SD->max_size * + (float) Config.Swap.highWaterMark) / (float) 100); + store_low = (long) (((float) SD->max_size * + (float) Config.Swap.lowWaterMark) / (float) 100); + debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size); + + x = (double) (store_high - SD->cur_size) / + (store_high - store_low); + x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; + z = pow((double) (Config.referenceAge / 60), x); + age = (time_t) (z * 60.0); + if (age < 60) + age = 60; + else if (age > Config.referenceAge) + age = Config.referenceAge; + return age; +} +#endif + +/* + * storeDiskdDirCheckExpired + * + * Check whether the given object is expired or not + * It breaks layering a little by calling the upper layers to find + * out whether the object is locked or not, but we can't help this + * right now. + */ +static int +storeDiskdDirCheckExpired(SwapDir *SD, StoreEntry *e) +{ + if (storeEntryLocked(e)) + return 0; + if (EBIT_TEST(e->flags, RELEASE_REQUEST)) + return 1; + if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) + return 1; + +#if HEAP_REPLACEMENT + /* + * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap + * controls the replacement of objects. + */ + return 1; +#else + if (squid_curtime - e->lastref > storeDiskdDirExpiredReferenceAge(SD)) + return 1; + return 0; +#endif +} + +/* + * Add and remove the given StoreEntry from the replacement policy in + * use. + */ + +void +storeDiskdDirReplAdd(SwapDir *SD, StoreEntry *e) +{ + debug(20, 4) ("storeDiskdDirReplAdd: added node %p to dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + (void) 0; + } else { + e->repl.node = heap_insert(SD->repl.heap.heap, e); + debug(20, 4) ("storeDiskdDirReplAdd: inserted node 0x%x\n", e->repl.node); + } +#else + /* Shouldn't we not throw special objects into the lru ? */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); +#endif +} + + +void +storeDiskdDirReplRemove(StoreEntry *e) +{ + SwapDir *SD = INDEXSD(e->swap_dirn); + debug(20, 4) ("storeDiskdDirReplRemove: remove node %p from dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + /* And now, release the object from the replacement policy */ + if (e->repl.node) { + debug(20, 4) ("storeDiskdDirReplRemove: deleting node 0x%x\n", + e->repl.node); + heap_delete(SD->repl.heap.heap, e->repl.node); + e->repl.node = NULL; + } +#else + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); +#endif +} + + + +/* + * SHM manipulation routines + */ + +void * +storeDiskdShmGet(SwapDir * sd, int *shm_offset) +{ + char *buf; + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + buf = linklistShift(&diskdinfo->shm.stack); + assert(buf); + *shm_offset = buf - diskdinfo->shm.buf; + assert(0 <= *shm_offset && *shm_offset < SHMBUFS * SHMBUF_BLKSZ); + diskd_stats.shmbuf_count++; + return buf; +} + +void +storeDiskdShmPut(SwapDir * sd, int offset) +{ + char *buf; + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + assert(offset >= 0); + assert(offset < SHMBUFS * SHMBUF_BLKSZ); + buf = diskdinfo->shm.buf + offset; + linklistPush(&diskdinfo->shm.stack, buf); + diskd_stats.shmbuf_count--; +} + + + + +/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ + +void +storeDiskdDirStats(SwapDir *SD, StoreEntry * sentry) +{ + diskdinfo_t *diskdinfo; +#if HAVE_STATVFS + struct statvfs sfs; +#endif + diskdinfo = (diskdinfo_t *)SD->fsdata; + storeAppendPrintf(sentry, "First level subdirectories: %d\n", diskdinfo->l1); + storeAppendPrintf(sentry, "Second level subdirectories: %d\n", diskdinfo->l2); + storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); + storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); + storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", + 100.0 * SD->cur_size / SD->max_size); + storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", + diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files, + percent(diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files)); +#if HAVE_STATVFS +#define fsbtoblk(num, fsbs, bs) \ + (((fsbs) != 0 && (fsbs) < (bs)) ? \ + (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + } +#endif + storeAppendPrintf(sentry, "Flags:"); + if (SD->flags.selected) + storeAppendPrintf(sentry, " SELECTED"); + if (SD->flags.read_only) + storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, "\n"); +#if !HEAP_REPLACEMENT + storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", + (double) storeDiskdDirExpiredReferenceAge(SD) / 86400.0); +#else +#if 0 + storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", + heap_peepminkey(sd.repl.heap.heap)); +#endif +#endif + storeAppendPrintf(sentry, "Pending operations: %d\n", diskdinfo->away); +} + +/* + * storeDiskdDirReconfigure + * + * This routine is called when the given swapdir needs reconfiguring + */ +void +storeDiskdDirReconfigure(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + int magic1, magic2; + unsigned int read_only = 0; + diskdinfo_t *diskdinfo; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeDiskdDirReconfigure: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeDiskdDirReconfigure: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeDiskdDirReconfigure: invalid level 2 directories value"); + i = GetInteger(); + magic1 = i; + if (magic1 <= 0) + fatal("storeDiskdDirParse: invalid magic1 value"); + i = GetInteger(); + magic2 = i; + if (magic2 <= 0) + fatal("storeDiskdDirParse: invalid magic2 value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + /* just reconfigure it */ + if (size == sd->max_size) + debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", + path, size); + else + debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", + path, size); + sd->max_size = size; + if (sd->flags.read_only != read_only) + debug(3, 1) ("Cache dir '%s' now %s\n", + path, read_only ? "Read-Only" : "Read-Write"); + diskdinfo = (diskdinfo_t *)sd->fsdata; + diskdinfo->magic1 = magic1; + diskdinfo->magic2 = magic2; + sd->flags.read_only = read_only; + return; +} + +void +storeDiskdDirDump(StoreEntry * entry, const char *name, SwapDir * s) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata; + storeAppendPrintf(entry, "%s %s %s %d %d %d\n", + name, + "diskd", + s->path, + s->max_size >> 10, + diskdinfo->l1, + diskdinfo->l2); +} + +/* + * Only "free" the filesystem specific stuff here + */ +static void +storeDiskdDirFree(SwapDir * s) +{ + diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata; + if (diskdinfo->swaplog_fd > -1) { + file_close(diskdinfo->swaplog_fd); + diskdinfo->swaplog_fd = -1; + } + filemapFreeMemory(diskdinfo->map); + xfree(diskdinfo); + s->fsdata = NULL; /* Will aid debugging... */ + +} + +char * +storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath) +{ + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + int L1 = diskdinfo->l1; + int L2 = diskdinfo->l2; + if (!fullpath) + fullpath = fullfilename; + fullpath[0] = '\0'; + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", + SD->path, + ((filn / L2) / L2) % L1, + (filn / L2) % L2, + filn); + return fullpath; +} + +/* + * storeDiskdCleanupDoubleCheck + * + * This is called by storeCleanup() if -S was given on the command line. + */ +static int +storeDiskdCleanupDoubleCheck(SwapDir *sd, StoreEntry *e) +{ + struct stat sb; + + if (stat(storeDiskdDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { + debug(20, 0) ("storeDiskdCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", + storeDiskdDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } + if (e->swap_file_sz != sb.st_size) { + debug(20, 0) ("storeDiskdCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n", + storeDiskdDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeDiskdCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } + return 0; +} + +/* + * storeDiskdDirParse + * + * Called when a *new* fs is being setup. + */ +void +storeDiskdDirParse(SwapDir *sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + int magic1, magic2; + unsigned int read_only = 0; + diskdinfo_t *diskdinfo; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeDiskdDirParse: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeDiskdDirParse: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeDiskdDirParse: invalid level 2 directories value"); + i = GetInteger(); + magic1 = i; + if (magic1 <= 0) + fatal("storeDiskdDirParse: invalid magic1 value"); + i = GetInteger(); + magic2 = i; + if (magic2 <= 0) + fatal("storeDiskdDirParse: invalid magic2 value"); + + + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + diskdinfo = xmalloc(sizeof(diskdinfo_t)); + if (diskdinfo == NULL) + fatal("storeDiskdDirParse: couldn't xmalloc() diskdinfo_t!\n"); + + sd->index = index; + sd->path = xstrdup(path); + sd->max_size = size; + sd->fsdata = diskdinfo; + diskdinfo->l1 = l1; + diskdinfo->l2 = l2; + diskdinfo->swaplog_fd = -1; + diskdinfo->map = NULL; /* Debugging purposes */ + diskdinfo->suggest = 0; + diskdinfo->magic1 = magic1; + diskdinfo->magic2 = magic2; + sd->flags.read_only = read_only; + sd->init = storeDiskdDirInit; + sd->newfs = storeDiskdDirNewfs; + sd->dump = storeDiskdDirDump; + sd->freefs = storeDiskdDirFree; + sd->dblcheck = storeDiskdCleanupDoubleCheck; + sd->statfs = storeDiskdDirStats; + sd->maintainfs = storeDiskdDirMaintain; + sd->checkobj = storeDiskdDirCheckObj; + sd->refobj = storeDiskdDirRefObj; + sd->unrefobj = storeDiskdDirUnrefObj; + sd->callback = storeDiskdDirCallback; + sd->sync = storeDiskdDirSync; + sd->obj.create = storeDiskdCreate; + sd->obj.open = storeDiskdOpen; + sd->obj.close = storeDiskdClose; + sd->obj.read = storeDiskdRead; + sd->obj.write = storeDiskdWrite; + sd->obj.unlink = storeDiskdUnlink; + sd->log.open = storeDiskdDirOpenSwapLog; + sd->log.close = storeDiskdDirCloseSwapLog; + sd->log.write = storeDiskdDirSwapLog; + sd->log.clean.open = storeDiskdDirWriteCleanOpen; + + /* Initialise replacement policy stuff */ +#if HEAP_REPLACEMENT + /* + * Create new heaps with cache replacement policies attached to them. + * The cache replacement policy is specified as either GDSF or LFUDA in + * the squid.conf configuration file. Note that the replacement policy + * applies only to the disk replacement algorithm. Memory replacement + * always uses GDSF since we want to maximize object hit rate. + */ + if (Config.replPolicy) { + if (tolower(Config.replPolicy[0]) == 'g') { + debug(20, 1) ("Using GDSF disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } else if (tolower(Config.replPolicy[0]) == 'l') { + if (tolower(Config.replPolicy[1]) == 'f') { + debug(20, 1) ("Using LFUDA disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA); + } else if (tolower(Config.replPolicy[1]) == 'r') { + debug(20, 1) ("Using LRU heap disk replacement policy\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU); + } + } else { + debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } + } else { + debug(20, 1) ("Using default disk replacement policy (GDSF)\n"); + sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF); + } +#else + sd->repl.lru.list.head = NULL; + sd->repl.lru.list.tail = NULL; +#endif +} + +/* + * Initial setup / end destruction + */ +void +storeDiskdDirDone(void) +{ + memPoolDestroy(diskd_state_pool); + diskd_initialised = 0; +} + +void +storeFsSetup_diskd(storefs_entry_t *storefs) +{ + assert(!diskd_initialised); + storefs->parsefunc = storeDiskdDirParse; + storefs->reconfigurefunc = storeDiskdDirReconfigure; + storefs->donefunc = storeDiskdDirDone; + diskd_state_pool = memPoolCreate("DISKD IO State data", sizeof(diskdstate_t)); + memset(&diskd_stats, '\0', sizeof(diskd_stats)); + cachemgrRegister("diskd", "DISKD Stats", storeDiskdStats, 0, 1); + debug(81, 1) ("diskd started\n"); + diskd_initialised = 1; +} + --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/store_diskd.h Wed Feb 14 00:43:55 2007 @@ -0,0 +1,126 @@ +/* + * store_diskd.h + * + * Internal declarations for the diskd routines + */ + +#ifndef __STORE_DISKD_H__ +#define __STORE_DISKD_H__ + +/* + * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv. + * If a queue has MAGIC2 (or more) messages away, then we read the + * queue until the level falls below MAGIC2. Recommended value + * is 75% of SHMBUFS. MAGIC1 is the number of messages away which we + * stop allowing open/create for. + */ + +struct _diskdinfo_t { + int swaplog_fd; + int l1; + int l2; + fileMap *map; + int suggest; + int smsgid; + int rmsgid; + int wfd; + int away; + struct { + char *buf; + link_list *stack; + int id; + } shm; + int magic1; + int magic2; +}; + +struct _diskdstate_t { + int id; + struct { + unsigned int close_request:1; + unsigned int reading:1; + unsigned int writing:1; + } flags; + char *read_buf; +}; + +enum { + _MQD_NOP, + _MQD_OPEN, + _MQD_CLOSE, + _MQD_READ, + _MQD_WRITE, + _MQD_UNLINK +}; + +typedef struct _diomsg { + mtyp_t mtype; + int id; + int seq_no; + void *callback_data; + int size; + int offset; + int status; + int shm_offset; +} diomsg; + +struct _diskd_stats { + int open_fail_queue_len; + int block_queue_len; + int max_away; + int max_shmuse; + int shmbuf_count; + int sent_count; + int recv_count; + int sio_id; +}; + +typedef struct _diskd_stats diskd_stats_t; +typedef struct _diskdinfo_t diskdinfo_t; +typedef struct _diskdstate_t diskdstate_t; + +static const int msg_snd_rcv_sz = sizeof(diomsg) - sizeof(mtyp_t); + +/* The diskd_state memory pool */ +extern MemPool * diskd_state_pool; + +extern void storeDiskdDirMapBitReset(SwapDir *, sfileno); +extern int storeDiskdDirMapBitAllocate(SwapDir *); +extern char * storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); +extern void storeDiskdDirUnlinkFile(SwapDir *, sfileno); +extern void storeDiskdDirReplAdd(SwapDir *, StoreEntry *); +extern void storeDiskdDirReplRemove(StoreEntry *); +extern void storeDiskdShmPut(SwapDir *, int); +extern void *storeDiskdShmGet(SwapDir *, int *); +extern void storeDiskdHandle(diomsg *M); + + +/* + * Store IO stuff + */ +extern STOBJCREATE storeDiskdCreate; +extern STOBJOPEN storeDiskdOpen; +extern STOBJCLOSE storeDiskdClose; +extern STOBJREAD storeDiskdRead; +extern STOBJWRITE storeDiskdWrite; +extern STOBJUNLINK storeDiskdUnlink; + +/* + * SHMBUFS is the number of shared memory buffers to allocate for + * Each SwapDir. + */ +#define SHMBUFS 96 +#define SHMBUF_BLKSZ SM_PAGE_SIZE +/* + * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv. + * If a queue has MAGIC2 (or more) messages away, then we read the + * queue until the level falls below MAGIC2. Recommended value + * is 75% of SHMBUFS. + */ +#define MAGIC1 Config.diskd.magic1 +#define MAGIC2 Config.diskd.magic2 + + +extern diskd_stats_t diskd_stats; + +#endif --- /dev/null Wed Feb 14 00:43:41 2007 +++ squid/src/fs/diskd/store_io_diskd.c Wed Feb 14 00:43:55 2007 @@ -0,0 +1,464 @@ + +#include "config.h" +#include "squid.h" +#include "store_diskd.h" + +#include +#include +#include + +#undef assert +#include + + +/* + * DEBUG: section 81 Diskd Interface functions + */ + + +static int storeDiskdSend(int, SwapDir *, int, storeIOState *, int, int, int); +static void storeDiskdIOCallback(storeIOState * sio, int errflag); +static void storeDiskdIOFreeEntry(void *sio, int foo); + +/* + * SHMBUFS is the number of shared memory buffers to allocate for + * Each SwapDir. + */ +#define SHMBUFS 96 +#define SHMBUF_BLKSZ SM_PAGE_SIZE + + +/* === PUBLIC =========================================================== */ + +storeIOState * +storeDiskdOpen(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, + STIOCB *callback, void *callback_data) +{ + sfileno f = e->swap_filen; + int x; + storeIOState *sio; + char *buf; + int shm_offset; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; + debug(81, 3) ("storeDiskdOpen: fileno %08X\n", f); + /* + * XXX Eventually there should be an option here to fail on open() + * If there are too many requests queued. + */ + if (diskdinfo->away > diskdinfo->magic1) { + debug(81, 3) ("storeDiskdOpen: FAILING, too many requests away\n"); + diskd_stats.open_fail_queue_len++; + return NULL; + } + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeDiskdIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(diskd_state_pool); + + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->mode = O_RDONLY; + sio->callback = callback; + sio->callback_data = callback_data; + sio->e = e; + cbdataLock(callback_data); + + ((diskdstate_t *)(sio->fsstate))->flags.writing = 0; + ((diskdstate_t *)(sio->fsstate))->flags.reading = 0; + ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0; + ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++; + + buf = storeDiskdShmGet(SD, &shm_offset); + /* XXX WRONG!!! :) */ + strcpy(buf, storeDiskdDirFullPath(SD, f, NULL)); + x = storeDiskdSend(_MQD_OPEN, + SD, + ((diskdstate_t *)(sio->fsstate))->id, + sio, + strlen(buf) + 1, + O_RDONLY, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + cbdataUnlock(sio->callback_data); + cbdataFree(sio); + return NULL; + } + return sio; +} + +storeIOState * +storeDiskdCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, + STIOCB *callback, void *callback_data) +{ + sfileno f; + int x; + storeIOState *sio; + char *buf; + int shm_offset; + diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata; + /* + * XXX Eventually there should be an option here to fail on open() + * If there are too many requests queued. + */ + if (diskdinfo->away > diskdinfo->magic1) { + diskd_stats.open_fail_queue_len++; + return NULL; + } + + /* Allocate a number */ + f = storeDiskdDirMapBitAllocate(SD); + debug(81, 3) ("storeDiskdCreate: fileno %08X\n", f); + + sio = memAllocate(MEM_STORE_IO); + cbdataAdd(sio, storeDiskdIOFreeEntry, MEM_STORE_IO); + sio->fsstate = memPoolAlloc(diskd_state_pool); + + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->mode = O_WRONLY | O_CREAT | O_TRUNC; + sio->callback = callback; + sio->callback_data = callback_data; + sio->e = e; + cbdataLock(callback_data); + + ((diskdstate_t *)(sio->fsstate))->flags.writing = 0; + ((diskdstate_t *)(sio->fsstate))->flags.reading = 0; + ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0; + ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++; + + buf = storeDiskdShmGet(SD, &shm_offset); + /* XXX WRONG!!! :) */ + strcpy(buf, storeDiskdDirFullPath(SD, f, NULL)); + x = storeDiskdSend(_MQD_OPEN, + SD, + ((diskdstate_t *)(sio->fsstate))->id, + sio, + strlen(buf) + 1, + sio->mode, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + cbdataUnlock(sio->callback_data); + cbdataFree(sio); + return NULL; + } + storeDiskdDirReplAdd(SD, e); + return sio; +} + + +void +storeDiskdClose(SwapDir *SD, storeIOState * sio) +{ + int x; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + debug(81, 3) ("storeDiskdClose: dirno %d, fileno %08X\n", SD->index, + sio->swap_filen); + x = storeDiskdSend(_MQD_CLOSE, + SD, + diskdstate->id, + sio, + 0, + 0, + -1); + if (x < 0) { + debug(50, 1) ("storeDiskdSend CLOSE: %s\n", xstrerror()); + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +void +storeDiskdRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +{ + int x; + int shm_offset; + char *rbuf; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + if (!cbdataValid(sio)) + return; + if (diskdstate->flags.reading) { + debug(81, 1) ("storeDiskdRead: already reading!\n"); + return; + } + assert(sio->read.callback == NULL); + assert(sio->read.callback_data == NULL); + sio->read.callback = callback; + sio->read.callback_data = callback_data; + diskdstate->read_buf = buf; /* the one passed from above */ + cbdataLock(sio->read.callback_data); + debug(81, 3) ("storeDiskdRead: dirno %d, fileno %08X\n", sio->swap_dirn, sio->swap_filen); + sio->offset = offset; + diskdstate->flags.reading = 1; + rbuf = storeDiskdShmGet(SD, &shm_offset); + assert(rbuf); + x = storeDiskdSend(_MQD_READ, + SD, + diskdstate->id, + sio, + (int) size, + (int) offset, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend READ: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +void +storeDiskdWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +{ + int x; + char *sbuf; + int shm_offset; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + debug(81, 3) ("storeDiskdWrite: dirno %d, fileno %08X\n", SD->index, sio->swap_filen); + if (!cbdataValid(sio)) { + free_func(buf); + return; + } + diskdstate->flags.writing = 1; + sbuf = storeDiskdShmGet(SD, &shm_offset); + xmemcpy(sbuf, buf, size); + if (free_func) + free_func(buf); + x = storeDiskdSend(_MQD_WRITE, + SD, + diskdstate->id, + sio, + (int) size, + (int) offset, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend WRITE: %s\n", xstrerror()); + storeDiskdShmPut(SD, shm_offset); + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +void +storeDiskdUnlink(SwapDir *SD, StoreEntry *e) +{ + int x; + int shm_offset; + char *buf; + diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata; + + debug(81, 3) ("storeDiskdUnlink: dirno %d, fileno %08X\n", SD->index, + e->swap_filen); + storeDiskdDirReplRemove(e); + storeDiskdDirMapBitReset(SD, e->swap_filen); + if (diskdinfo->away >= diskdinfo->magic1) { + /* Damn, we need to issue a sync unlink here :( */ + debug(50, 2) ("storeDiskUnlink: Out of queue space, sync unlink\n"); + storeDiskdDirUnlinkFile(SD, e->swap_filen); + return; + } + + /* We can attempt a diskd unlink */ + buf = storeDiskdShmGet(SD, &shm_offset); + strcpy(buf, storeDiskdDirFullPath(SD, e->swap_filen, NULL)); + x = storeDiskdSend(_MQD_UNLINK, + SD, + e->swap_filen, + NULL, + 0, + 0, + shm_offset); + if (x < 0) { + debug(50, 1) ("storeDiskdSend UNLINK: %s\n", xstrerror()); + unlink(buf); /* XXX EWW! */ + storeDiskdShmPut(SD, shm_offset); + } +} + + +/* === STATIC =========================================================== */ + +static void +storeDiskdOpenDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + Counter.syscalls.disk.opens++; + debug(81, 3) ("storeDiskdOpenDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + } +} + +static void +storeDiskdCloseDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + Counter.syscalls.disk.closes++; + debug(81, 3) ("storeDiskdCloseDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + return; + } + storeDiskdIOCallback(sio, DISK_OK); +} + +static void +storeDiskdReadDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + STRCB *callback = sio->read.callback; + SwapDir *sd = INDEXSD(sio->swap_dirn); + diskdstate_t *diskdstate = sio->fsstate; + diskdinfo_t *diskdinfo = sd->fsdata; + void *their_data = sio->read.callback_data; + char *their_buf = diskdstate->read_buf; + char *sbuf; + size_t len; + int valid; + Counter.syscalls.disk.reads++; + diskdstate->flags.reading = 0; + valid = cbdataValid(sio->read.callback_data); + cbdataUnlock(sio->read.callback_data); + debug(81, 3) ("storeDiskdReadDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + return; + } + sbuf = diskdinfo->shm.buf + M->shm_offset; + len = M->status; + xmemcpy(their_buf, sbuf, len); /* yucky copy */ + sio->offset += len; + assert(callback); + assert(their_data); + sio->read.callback = NULL; + sio->read.callback_data = NULL; + if (valid) + callback(their_data, their_buf, len); +} + +static void +storeDiskdWriteDone(diomsg * M) +{ + storeIOState *sio = M->callback_data; + diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate; + Counter.syscalls.disk.writes++; + diskdstate->flags.writing = 0; + debug(81, 3) ("storeDiskdWriteDone: dirno %d, fileno %08x status %d\n", + sio->swap_dirn, sio->swap_filen, M->status); + if (M->status < 0) { + storeDiskdIOCallback(sio, DISK_ERROR); + return; + } + sio->offset += M->status; +} + +static void +storeDiskdUnlinkDone(diomsg * M) +{ + debug(81, 3) ("storeDiskdUnlinkDone: fileno %08x status %d\n", + M->id, M->status); + Counter.syscalls.disk.unlinks++; +} + +void +storeDiskdHandle(diomsg * M) +{ + int valid = M->callback_data ? cbdataValid(M->callback_data) : 1; + if (M->callback_data) + cbdataUnlock(M->callback_data); + if (!valid) { + debug(81, 3) ("storeDiskdHandle: Invalid callback_data %p\n", + M->callback_data); + /* + * The read operation has its own callback. If we don't + * call storeDiskdReadDone(), then we must make sure the + * callback_data gets unlocked! + */ + if (_MQD_READ == M->mtype) { + storeIOState *sio = M->callback_data; + cbdataUnlock(sio->read.callback_data); + } + return; + } + switch (M->mtype) { + case _MQD_OPEN: + storeDiskdOpenDone(M); + break; + case _MQD_CLOSE: + storeDiskdCloseDone(M); + break; + case _MQD_READ: + storeDiskdReadDone(M); + break; + case _MQD_WRITE: + storeDiskdWriteDone(M); + break; + case _MQD_UNLINK: + storeDiskdUnlinkDone(M); + break; + default: + assert(0); + break; + } +} + +static void +storeDiskdIOCallback(storeIOState * sio, int errflag) +{ + int valid = cbdataValid(sio->callback_data); + debug(81, 3) ("storeUfsIOCallback: errflag=%d\n", errflag); + cbdataUnlock(sio->callback_data); + if (valid) + sio->callback(sio->callback_data, errflag, sio); + cbdataFree(sio); +} + +static int +storeDiskdSend(int mtype, SwapDir * sd, int id, storeIOState * sio, int size, int offset, int shm_offset) +{ + int x; + diomsg M; + static int send_errors = 0; + static int last_seq_no = 0; + static int seq_no = 0; + diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata; + M.mtype = mtype; + M.callback_data = sio; + M.size = size; + M.offset = offset; + M.status = -1; + M.shm_offset = shm_offset; + M.id = id; + M.seq_no = ++seq_no; + if (M.callback_data) + cbdataLock(M.callback_data); + if (M.seq_no < last_seq_no) + debug(81, 1) ("WARNING: sequencing out of order\n"); + x = msgsnd(diskdinfo->smsgid, &M, msg_snd_rcv_sz, IPC_NOWAIT); + last_seq_no = M.seq_no; + if (0 == x) { + diskd_stats.sent_count++; + diskdinfo->away++; + } else { + debug(50, 1) ("storeDiskdSend: msgsnd: %s\n", xstrerror()); + if (M.callback_data) + cbdataUnlock(M.callback_data); + assert(++send_errors < 100); + } + return x; +} + + +/* + * We can't pass memFree() as a free function here, because we need to free + * the fsstate variable .. + */ +static void +storeDiskdIOFreeEntry(void *sio, int foo) +{ + memPoolFree(diskd_state_pool, ((storeIOState *)sio)->fsstate); + memFree(sio, MEM_STORE_IO); +} + Index: squid/src/fs/ufs/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/ufs/Attic/Makefile.in,v retrieving revision 1.1.6.3 retrieving revision 1.1.6.3.2.1 diff -u -r1.1.6.3 -r1.1.6.3.2.1 --- squid/src/fs/ufs/Makefile.in 5 Mar 2000 23:39:05 -0000 1.1.6.3 +++ squid/src/fs/ufs/Makefile.in 2 May 2000 22:50:30 -0000 1.1.6.3.2.1 @@ -47,6 +47,8 @@ -rm -f Makefile.bak -rm -f tags +install: + tags: ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] Index: squid/src/fs/ufs/store_dir_ufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/ufs/store_dir_ufs.c,v retrieving revision 1.1.6.6.2.1 retrieving revision 1.1.6.6.2.2 diff -u -r1.1.6.6.2.1 -r1.1.6.6.2.2 --- squid/src/fs/ufs/store_dir_ufs.c 22 Apr 2000 20:02:55 -0000 1.1.6.6.2.1 +++ squid/src/fs/ufs/store_dir_ufs.c 2 May 2000 22:50:30 -0000 1.1.6.6.2.2 @@ -157,6 +157,17 @@ ufsinfo = (ufsinfo_t *)SD->fsdata; file_map_bit_reset(ufsinfo->map, filn); } + +int +storeUfsDirMapBitAllocate(SwapDir *SD) +{ + ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata; + int fn; + fn = file_map_allocate(ufsinfo->map, ufsinfo->suggest); + file_map_bit_set(ufsinfo->map, fn); + ufsinfo->suggest = fn + 1; + return fn; +} /* * Initialise the ufs bitmap @@ -274,12 +285,22 @@ storeUfsDirSwapLogFile(SwapDir * sd, const char *ext) { LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; if (Config.Log.swap) { - xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64); - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp,'/')) + *index(pathtmp,'/')='.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.') + pathtmp[strlen(pathtmp)-1]= '\0'; + for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } } else { xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); strcat(path, "/swap.state"); @@ -531,22 +552,15 @@ * because adding to store_swap_size happens in * the cleanup procedure. */ - storeExpireNow(e); - storeReleaseRequest(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeUfsDirReplRemove(e); if (e->swap_filen > -1) { storeUfsDirMapBitReset(SD, e->swap_filen); e->swap_filen = -1; e->swap_dirn = -1; - /* Fake a unlink here, this is a bad hack :( */ -#if HEAP_REPLACEMENT - if (e->repl.node) { - heap_delete(SD->repl.heap.heap, e->repl.node); - e->repl.node = NULL; - } -#else - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); -#endif } + storeRelease(e); rb->counts.objcount--; rb->counts.cancelcount++; } @@ -626,22 +640,16 @@ } else if (e) { /* key already exists, this swapfile not being used */ /* junk old, load new */ - storeExpireNow(e); - storeReleaseRequest(e); + storeExpireNow(e); + storeReleaseRequest(e); + storeUfsDirReplRemove(e); if (e->swap_filen > -1) { + /* Make sure we don't actually unlink the file */ storeUfsDirMapBitReset(SD, e->swap_filen); e->swap_filen = -1; e->swap_dirn = -1; - /* Fake a unlink here, this is a bad hack :( */ -#if HEAP_REPLACEMENT - if (e->repl.node) { - heap_delete(SD->repl.heap.heap, e->repl.node); - e->repl.node = NULL; - } -#else - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); -#endif } + storeRelease(e); rb->counts.dupcount++; } else { /* URL doesnt exist, swapfile not in use */ @@ -794,16 +802,7 @@ EBIT_CLR(e->flags, ENTRY_VALIDATED); storeUfsDirMapBitSet(SD, e->swap_filen); storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ -#if HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->repl.node = heap_insert(SD->repl.heap.heap, e); - debug(20, 4) ("storeUfsAddDiskRestore: inserted node %p\n", e->repl.node); - } -#else - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); -#endif + storeUfsDirReplAdd(SD, e); return e; } @@ -1426,10 +1425,11 @@ debug(20, 3) ("storeUfsDirCheckObj: NO: LRU Age = %d\n", storeUfsDirExpiredReferenceAge(SD)); /* store_check_cachable_hist.no.lru_age_too_low++; */ - return 0; + return -1; } #endif - return 1; + /* Return 999 (99.9%) constant load */ + return 999; } /* @@ -1441,13 +1441,17 @@ void storeUfsDirRefObj(SwapDir *SD, StoreEntry *e) { + debug(1, 3) ("storeUfsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); #if HEAP_REPLACEMENT /* Nothing to do here */ #else /* Reference the object */ - debug(1, 3) ("storeUfsDirRefObj: referencing %d/%d\n", e->swap_dirn, e->swap_filen); - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && + !EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); + } #endif } @@ -1459,13 +1463,11 @@ void storeUfsDirUnrefObj(SwapDir *SD, StoreEntry *e) { + debug(1, 3) ("storeUfsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); #if HEAP_REPLACEMENT if (e->repl.node) heap_update(SD->repl.heap.heap, e->repl.node, e); -#else - debug(1, 3) ("storeUfsDirUnrefObj: referencing %d/%d\n", e->swap_dirn, e->swap_filen); - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); #endif } @@ -1517,8 +1519,8 @@ age = (time_t) (z * 60.0); if (age < 60) age = 60; - else if (age > 31536000) - age = 31536000; + else if (age > Config.referenceAge) + age = Config.referenceAge; return age; } #endif @@ -1554,6 +1556,49 @@ #endif } +/* + * Add and remove the given StoreEntry from the replacement policy in + * use. + */ + +void +storeUfsDirReplAdd(SwapDir *SD, StoreEntry *e) +{ + debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + (void) 0; + } else { + e->repl.node = heap_insert(SD->repl.heap.heap, e); + debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node); + } +#else + /* Shouldn't we not throw special objects into the lru ? */ + dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); +#endif +} + + +void +storeUfsDirReplRemove(StoreEntry *e) +{ + SwapDir *SD = INDEXSD(e->swap_dirn); + debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e, + SD->index); +#if HEAP_REPLACEMENT + /* And now, release the object from the replacement policy */ + if (e->repl.node) { + debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n", + e->repl.node); + heap_delete(SD->repl.heap.heap, e->repl.node); + e->repl.node = NULL; + } +#else + dlinkDelete(&e->repl.lru, &SD->repl.lru.list); +#endif +} + /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ @@ -1566,7 +1611,6 @@ struct statvfs sfs; #endif ufsinfo = (ufsinfo_t *)SD->fsdata; - storeAppendPrintf(sentry, "Store Directory #%d: %s\n", SD->index, SD->path); storeAppendPrintf(sentry, "First level subdirectories: %d\n", ufsinfo->l1); storeAppendPrintf(sentry, "Second level subdirectories: %d\n", ufsinfo->l2); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); @@ -1788,6 +1832,8 @@ sd->checkobj = storeUfsDirCheckObj; sd->refobj = storeUfsDirRefObj; sd->unrefobj = storeUfsDirUnrefObj; + sd->callback = NULL; + sd->sync = NULL; sd->obj.create = storeUfsCreate; sd->obj.open = storeUfsOpen; sd->obj.close = storeUfsClose; @@ -1851,8 +1897,6 @@ storefs->parsefunc = storeUfsDirParse; storefs->reconfigurefunc = storeUfsDirReconfigure; storefs->donefunc = storeUfsDirDone; - storefs->callbackfunc = NULL; - storefs->syncfunc = NULL; ufs_state_pool = memPoolCreate("UFS IO State data", sizeof(ufsstate_t)); ufs_initialised = 1; } Index: squid/src/fs/ufs/store_io_ufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/ufs/store_io_ufs.c,v retrieving revision 1.1.6.6.2.2 retrieving revision 1.1.6.6.2.3 diff -u -r1.1.6.6.2.2 -r1.1.6.6.2.3 --- squid/src/fs/ufs/store_io_ufs.c 22 Apr 2000 20:02:55 -0000 1.1.6.6.2.2 +++ squid/src/fs/ufs/store_io_ufs.c 2 May 2000 22:50:30 -0000 1.1.6.6.2.3 @@ -96,7 +96,7 @@ /* Allocate a number */ dirn = SD->index; - filn = file_map_allocate(ufsinfo->map, ufsinfo->suggest); + filn = storeUfsDirMapBitAllocate(SD); ufsinfo->suggest = filn + 1; /* Shouldn't we handle a 'bitmap full' error here? */ path = storeUfsDirFullPath(SD, filn, NULL); @@ -126,16 +126,7 @@ store_open_disk_fd++; /* now insert into the replacement policy */ -#if HEAP_REPLACEMENT - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - (void) 0; - } else { - e->repl.node = heap_insert(SD->repl.heap.heap, e); - debug(20, 4) ("storeUfsCreate: inserted node 0x%x\n", e->repl.node); - } -#else - dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list); -#endif + storeUfsDirReplAdd(SD, e); return sio; } @@ -194,18 +185,8 @@ storeUfsUnlink(SwapDir *SD, StoreEntry *e) { debug(79, 3) ("storeUfsUnlink: fileno %08X\n", e->swap_filen); + storeUfsDirReplRemove(e); storeUfsDirUnlinkFile(SD, e->swap_filen); - -#if HEAP_REPLACEMENT - /* And now, release the object from the replacement policy */ - if (e->repl.node) { - debug(20, 4) ("storeUfsUnlink: deleting node 0x%x\n", e->repl.node); - heap_delete(SD->repl.heap.heap, e->repl.node); - e->repl.node = NULL; - } -#else - dlinkDelete(&e->repl.lru, &SD->repl.lru.list); -#endif } /* === STATIC =========================================================== */ Index: squid/src/fs/ufs/store_ufs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/ufs/store_ufs.h,v retrieving revision 1.1.6.3 retrieving revision 1.1.6.3.2.1 diff -u -r1.1.6.3 -r1.1.6.3.2.1 --- squid/src/fs/ufs/store_ufs.h 11 Apr 2000 18:18:50 -0000 1.1.6.3 +++ squid/src/fs/ufs/store_ufs.h 2 May 2000 22:50:30 -0000 1.1.6.3.2.1 @@ -31,9 +31,11 @@ extern MemPool * ufs_state_pool; extern void storeUfsDirMapBitReset(SwapDir *, sfileno); - +extern int storeUfsDirMapBitAllocate(SwapDir *); extern char * storeUfsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath); extern void storeUfsDirUnlinkFile(SwapDir *, sfileno); +extern void storeUfsDirReplAdd(SwapDir *SD, StoreEntry *); +extern void storeUfsDirReplRemove(StoreEntry *); /* * Store IO stuff