Squid 2.6 release notes

Squid 2.6 release notes Squid Developers -$Id: release-2.6.sgml,v 1.2.4.25 2006/09/08 20:13:37 serassio Exp $ +$Id: release-2.6.sgml,v 1.2.4.26 2006/09/09 12:30:38 serassio Exp $ This document contains the release notes for version 2.6 of Squid. @@ -278,14 +278,6 @@ -Awin32 fs module - -awin32 is a native WIN32 Async I/O Squid fs module derived from aufs. Configuration -in squid.conf is identical to aufs, but You nedd to specify 'awin32' instead 'aufs' as -cache type. - - - Key changes from SquidNT 2.5 Changed the default Windows service name from "SquidNT" to "Squid" Index: squid/port/win32/squid_mswin.mak =================================================================== RCS file: /cvsroot/squid-sf//squid/port/win32/Attic/squid_mswin.mak,v retrieving revision 1.1.38.4 retrieving revision 1.1.38.5 diff -u -r1.1.38.4 -r1.1.38.5 --- squid/port/win32/squid_mswin.mak 19 Aug 2006 16:56:53 -0000 1.1.38.4 +++ squid/port/win32/squid_mswin.mak 9 Sep 2006 12:30:38 -0000 1.1.38.5 @@ -51,7 +51,7 @@ AUTH_MODULES = basic ntlm digest negotiate REPL_POLICIES = lru heap -STORE_MODULES = ufs awin32 null +STORE_MODULES = ufs aufs null SUBSTITUTE=sed "\ s%@DEFAULT_CONFIG_FILE@%$(DEFAULT_CONFIG_FILE)%g;\ Index: squid/port/win32/include/autoconf.h =================================================================== RCS file: /cvsroot/squid-sf//squid/port/win32/include/Attic/autoconf.h,v retrieving revision 1.1.38.12 retrieving revision 1.1.38.13 diff -u -r1.1.38.12 -r1.1.38.13 --- squid/port/win32/include/autoconf.h 1 Sep 2006 13:23:56 -0000 1.1.38.12 +++ squid/port/win32/include/autoconf.h 9 Sep 2006 12:30:38 -0000 1.1.38.13 @@ -929,7 +929,7 @@ #define STR_USE_ARP_ACL "" #endif -#define SQUID_CONFIGURE_OPTIONS "--enable-win32-service --enable-storeio='ufs awin32 null' " \ +#define SQUID_CONFIGURE_OPTIONS "--enable-win32-service --enable-storeio='ufs aufs null' " \ "--enable-removal-policies='heap lru' --enable-snmp --enable-htcp --disable-wccp --disable-wccpv2 --enable-useragent-log " \ "--enable-referer-log --enable-cache-digests --enable-auth='basic ntlm digest negotiate' " \ "--enable-basic-auth-helpers='LDAP NCSA mswin_sspi' --enable-negotiate-auth-helpers=mswin_sspi" \ Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.100.2.34 retrieving revision 1.100.2.35 diff -u -r1.100.2.34 -r1.100.2.35 --- squid/src/cf.data.pre 2 Sep 2006 20:30:49 -0000 1.100.2.34 +++ squid/src/cf.data.pre 9 Sep 2006 12:30:39 -0000 1.100.2.35 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.100.2.34 2006/09/02 20:30:49 serassio Exp $ +# $Id: cf.data.pre,v 1.100.2.35 2006/09/09 12:30:39 serassio Exp $ # # # SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1099,16 +1099,6 @@ see argument descriptions under ufs above - The awin32 store type: - - "awin32" uses the same storage format as "ufs", utilizing - WIN32-threads to avoid blocking the main Squid process on - disk-I/O. This was formerly known in Squid as async-io. - - cache_dir awin32 Directory-Name Mbytes L1 L2 [options] - - see argument descriptions under ufs above - The diskd store type: "diskd" uses the same storage format as "ufs", utilizing a Index: squid/src/fs/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/Makefile.am,v retrieving revision 1.3.30.8 retrieving revision 1.3.30.9 diff -u -r1.3.30.8 -r1.3.30.9 --- squid/src/fs/Makefile.am 12 Jun 2006 19:06:20 -0000 1.3.30.8 +++ squid/src/fs/Makefile.am 9 Sep 2006 12:30:41 -0000 1.3.30.9 @@ -5,13 +5,19 @@ AUTOMAKE_OPTIONS = subdir-objects -EXTRA_LIBRARIES = libaufs.a libawin32.a libcoss.a libdiskd.a libnull.a libufs.a +if USE_AIOPS_WIN32 +AIOPS_SOURCE = aufs/aiops_win32.c +else +AIOPS_SOURCE = aufs/aiops.c +endif + +EXTRA_LIBRARIES = libaufs.a libcoss.a libdiskd.a libnull.a libufs.a noinst_LIBRARIES = @STORE_LIBS@ -libaufs_a_SOURCES = aufs/aiops.c aufs/async_io.c aufs/store_asyncufs.h \ +EXTRA_libaufs_a_SOURCES = aufs/aiops.c aufs/aiops_win32.c + +libaufs_a_SOURCES = $(AIOPS_SOURCE) aufs/async_io.c aufs/store_asyncufs.h \ aufs/store_dir_aufs.c aufs/store_io_aufs.c aufs/async_io.h -libawin32_a_SOURCES = awin32/aiops.c awin32/async_io.c awin32/store_asyncufs.h \ - awin32/store_dir_aufs.c awin32/store_io_aufs.c awin32/async_io.h libcoss_a_SOURCES = coss/store_coss.h coss/store_io_coss.c coss/store_dir_coss.c \ coss/async_io.c coss/async_io.h libdiskd_a_SOURCES = diskd/diskd.c diskd/store_dir_diskd.c diskd/store_diskd.h \ @@ -40,8 +46,6 @@ ## targets below to emulate distributed makefiles aufs/all: libaufs.a aufs/clean: clean -awin32/all: libawin32.a -awin32/clean: clean coss/all: libcoss.a coss/clean: clean null/all: libnull.a Index: squid/src/fs/aufs/aiops.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/aiops.c,v retrieving revision 1.21.2.4 retrieving revision 1.21.2.5 diff -u -r1.21.2.4 -r1.21.2.5 --- squid/src/fs/aufs/aiops.c 29 Jul 2006 18:43:14 -0000 1.21.2.4 +++ squid/src/fs/aufs/aiops.c 9 Sep 2006 12:30:41 -0000 1.21.2.5 @@ -129,7 +129,9 @@ static void squidaio_do_close(squidaio_request_t *); static void squidaio_do_stat(squidaio_request_t *); static void squidaio_do_unlink(squidaio_request_t *); +#if USE_TRUNCATE static void squidaio_do_truncate(squidaio_request_t *); +#endif #if AIO_OPENDIR static void *squidaio_do_opendir(squidaio_request_t *); #endif @@ -448,9 +450,11 @@ case _AIO_OP_UNLINK: squidaio_do_unlink(request); break; +#if USE_TRUNCATE case _AIO_OP_TRUNCATE: squidaio_do_truncate(request); break; +#endif #if AIO_OPENDIR /* Opendir not implemented yet */ case _AIO_OP_OPENDIR: squidaio_do_opendir(request); @@ -801,6 +805,8 @@ requestp->err = errno; } + +#if USE_TRUNCATE int squidaio_truncate(const char *path, off_t length, squidaio_result_t * resultp) { @@ -824,6 +830,7 @@ requestp->ret = truncate(requestp->path, requestp->offset); requestp->err = errno; } +#endif #if AIO_OPENDIR --- /dev/null Wed Feb 14 01:16:58 2007 +++ squid/src/fs/aufs/aiops_win32.c Wed Feb 14 01:17:06 2007 @@ -0,0 +1,1075 @@ +/* + * $Id: aiops_win32.c,v 1.1.2.1 2006/09/09 12:30:41 serassio Exp $ + * + * DEBUG: section 43 AIOPS + * AUTHOR: Stewart Forster + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include +#include "async_io.h" + +#include +#include +#include +#include +#include +#include +#include + +#define RIDICULOUS_LENGTH 4096 + +#ifdef AUFS_IO_THREADS +int squidaio_nthreads = AUFS_IO_THREADS; +#else +int squidaio_nthreads = 0; +#endif +int squidaio_magic1 = 1; /* dummy initializer value */ +int squidaio_magic2 = 1; /* real value set in aiops.c */ + +enum _squidaio_thread_status { + _THREAD_STARTING = 0, + _THREAD_WAITING, + _THREAD_BUSY, + _THREAD_FAILED, + _THREAD_DONE +}; +typedef enum _squidaio_thread_status squidaio_thread_status; + +enum _squidaio_request_type { + _AIO_OP_NONE = 0, + _AIO_OP_OPEN, + _AIO_OP_READ, + _AIO_OP_WRITE, + _AIO_OP_CLOSE, + _AIO_OP_UNLINK, + _AIO_OP_TRUNCATE, + _AIO_OP_OPENDIR, + _AIO_OP_STAT +}; +typedef enum _squidaio_request_type squidaio_request_type; + +typedef struct squidaio_request_t { + struct squidaio_request_t *next; + squidaio_request_type request_type; + int cancelled; + char *path; + int oflag; + mode_t mode; + int fd; + char *bufferp; + int buflen; + off_t offset; + int whence; + int ret; + int err; + struct stat *tmpstatp; + struct stat *statp; + squidaio_result_t *resultp; +} squidaio_request_t; + +typedef struct squidaio_request_queue_t { + HANDLE mutex; + HANDLE cond; /* See Event objects */ + squidaio_request_t *volatile head; + squidaio_request_t *volatile *volatile tailp; + unsigned long requests; + unsigned long blocked; /* main failed to lock the queue */ +} squidaio_request_queue_t; + +typedef struct squidaio_thread_t squidaio_thread_t; +struct squidaio_thread_t { + squidaio_thread_t *next; + HANDLE thread; + DWORD dwThreadId; /* thread ID */ + squidaio_thread_status status; + struct squidaio_request_t *current_req; + unsigned long requests; + int volatile exit; +}; + +static void squidaio_queue_request(squidaio_request_t *); +static void squidaio_cleanup_request(squidaio_request_t *); +static DWORD WINAPI squidaio_thread_loop(LPVOID lpParam); +static void squidaio_do_open(squidaio_request_t *); +static void squidaio_do_read(squidaio_request_t *); +static void squidaio_do_write(squidaio_request_t *); +static void squidaio_do_close(squidaio_request_t *); +static void squidaio_do_stat(squidaio_request_t *); +static void squidaio_do_unlink(squidaio_request_t *); +#if USE_TRUNCATE +static void squidaio_do_truncate(squidaio_request_t *); +#endif +#if AIO_OPENDIR +static void *squidaio_do_opendir(squidaio_request_t *); +#endif +static void squidaio_debug(squidaio_request_t *); +static void squidaio_poll_queues(void); + +static squidaio_thread_t *threads = NULL; +static int squidaio_initialised = 0; + +#define AIO_LARGE_BUFS 16384 +#define AIO_MEDIUM_BUFS AIO_LARGE_BUFS >> 1 +#define AIO_SMALL_BUFS AIO_LARGE_BUFS >> 2 +#define AIO_TINY_BUFS AIO_LARGE_BUFS >> 3 +#define AIO_MICRO_BUFS 128 + +static MemPool *squidaio_large_bufs = NULL; /* 16K */ +static MemPool *squidaio_medium_bufs = NULL; /* 8K */ +static MemPool *squidaio_small_bufs = NULL; /* 4K */ +static MemPool *squidaio_tiny_bufs = NULL; /* 2K */ +static MemPool *squidaio_micro_bufs = NULL; /* 128K */ + +static int request_queue_len = 0; +static MemPool *squidaio_request_pool = NULL; +static MemPool *squidaio_thread_pool = NULL; +static squidaio_request_queue_t request_queue; +static struct { + squidaio_request_t *head, **tailp; +} request_queue2 = { + + NULL, &request_queue2.head +}; +static squidaio_request_queue_t done_queue; +static struct { + squidaio_request_t *head, **tailp; +} done_requests = { + + NULL, &done_requests.head +}; +static int done_fd = 0; +static int done_fd_read = 0; +static int done_signalled = 0; +static HANDLE main_thread; + +static MemPool * +squidaio_get_pool(int size) +{ + MemPool *p; + if (size <= AIO_LARGE_BUFS) { + if (size <= AIO_MICRO_BUFS) + p = squidaio_micro_bufs; + else if (size <= AIO_TINY_BUFS) + p = squidaio_tiny_bufs; + else if (size <= AIO_SMALL_BUFS) + p = squidaio_small_bufs; + else if (size <= AIO_MEDIUM_BUFS) + p = squidaio_medium_bufs; + else + p = squidaio_large_bufs; + } else + p = NULL; + return p; +} + +void * +squidaio_xmalloc(int size) +{ + void *p; + MemPool *pool; + + if ((pool = squidaio_get_pool(size)) != NULL) { + p = memPoolAlloc(pool); + } else + p = xmalloc(size); + + return p; +} + +static char * +squidaio_xstrdup(const char *str) +{ + char *p; + int len = strlen(str) + 1; + + p = squidaio_xmalloc(len); + strncpy(p, str, len); + + return p; +} + +void +squidaio_xfree(void *p, int size) +{ + MemPool *pool; + + if ((pool = squidaio_get_pool(size)) != NULL) { + memPoolFree(pool, p); + } else + xfree(p); +} + +static void +squidaio_xstrfree(char *str) +{ + MemPool *pool; + int len = strlen(str) + 1; + + if ((pool = squidaio_get_pool(len)) != NULL) { + memPoolFree(pool, str); + } else + xfree(str); +} + +static void +squidaio_fdhandler(int fd, void *data) +{ + char junk[256]; + int x; + fde *F = &fd_table[done_fd_read]; + if ((x = FD_READ_METHOD(done_fd_read, junk, sizeof(junk))) > 0) + F->bytes_read += x; + commSetSelect(fd, COMM_SELECT_READ, squidaio_fdhandler, NULL, 0); +} + +void +squidaio_init(void) +{ + int i; + int done_pipe[2]; + squidaio_thread_t *threadp; + + if (squidaio_initialised) + return; + + if (!DuplicateHandle(GetCurrentProcess(), /* pseudo handle, don't close */ + GetCurrentThread(), /* pseudo handle to copy */ + GetCurrentProcess(), /* pseudo handle, don't close */ + &main_thread, + 0, /* required access */ + FALSE, /* child process's don't inherit the handle */ + DUPLICATE_SAME_ACCESS)) { + /* spit errors */ + fatal("couldn't get current thread handle\n"); + } + /* Initialize request queue */ + if ((request_queue.mutex = CreateMutex(NULL, /* no inheritance */ + FALSE, /* start unowned (as per mutex_init) */ + NULL) /* no name */ + ) == NULL) { + fatal("failed to create mutex\n"); + } + if ((request_queue.cond = CreateEvent(NULL, /* no inheritance */ + FALSE, /* auto signal reset - which I think is pthreads like ? */ + FALSE, /* start non signaled */ + NULL) /* no name */ + ) == NULL) { + fatal("failed to create condition event variable.\n"); + } + request_queue.head = NULL; + request_queue.tailp = &request_queue.head; + request_queue.requests = 0; + request_queue.blocked = 0; + + /* Initialize done queue */ + if ((done_queue.mutex = CreateMutex(NULL, /* no inheritance */ + FALSE, /* start unowned (as per mutex_init) */ + NULL) /* no name */ + ) == NULL) { + fatal("failed to create mutex\n"); + } + if ((done_queue.cond = CreateEvent(NULL, /* no inheritance */ + TRUE, /* manually signaled - which I think is pthreads like ? */ + FALSE, /* start non signaled */ + NULL) /* no name */ + ) == NULL) { + fatal("failed to create condition event variable.\n"); + } + done_queue.head = NULL; + done_queue.tailp = &done_queue.head; + done_queue.requests = 0; + done_queue.blocked = 0; + + /* Initialize done pipe signal */ + pipe(done_pipe); + done_fd = done_pipe[1]; + done_fd_read = done_pipe[0]; + fd_open(done_fd_read, FD_PIPE, "async-io completion event: main"); + fd_open(done_fd, FD_PIPE, "async-io completion event: threads"); + commSetNonBlocking(done_pipe[0]); + commSetNonBlocking(done_pipe[1]); + commSetCloseOnExec(done_pipe[0]); + commSetCloseOnExec(done_pipe[1]); + commSetSelect(done_pipe[0], COMM_SELECT_READ, squidaio_fdhandler, NULL, 0); + + /* Create threads and get them to sit in their wait loop */ + squidaio_thread_pool = memPoolCreate("aio_thread", sizeof(squidaio_thread_t)); + if (squidaio_nthreads == 0) { + int j = 16; + for (i = 0; i < n_asyncufs_dirs; i++) { + squidaio_nthreads += j; + j = j * 2 / 3; + if (j < 4) + j = 4; + } +#if COSS_USE_AUFSOPS + j = 6; + for (i = 0; i < n_coss_dirs; i++) { + squidaio_nthreads += j; + j = 3; + } +#endif + } + if (squidaio_nthreads == 0) + squidaio_nthreads = 16; + squidaio_magic1 = squidaio_nthreads * MAGIC1_FACTOR; + squidaio_magic2 = squidaio_nthreads * MAGIC2_FACTOR; + for (i = 0; i < squidaio_nthreads; i++) { + threadp = memPoolAlloc(squidaio_thread_pool); + threadp->status = _THREAD_STARTING; + threadp->current_req = NULL; + threadp->requests = 0; + threadp->next = threads; + threads = threadp; + if ((threadp->thread = CreateThread(NULL, /* no security attributes */ + 0, /* use default stack size */ + squidaio_thread_loop, /* thread function */ + threadp, /* argument to thread function */ + 0, /* use default creation flags */ + &(threadp->dwThreadId)) /* returns the thread identifier */ + ) == NULL) { + fprintf(stderr, "Thread creation failed\n"); + threadp->status = _THREAD_FAILED; + continue; + } + /* Set the new thread priority above parent process */ + SetThreadPriority(threadp->thread, THREAD_PRIORITY_ABOVE_NORMAL); + } + + /* Create request pool */ + squidaio_request_pool = memPoolCreate("aio_request", sizeof(squidaio_request_t)); + squidaio_large_bufs = memPoolCreate("squidaio_large_bufs", AIO_LARGE_BUFS); + squidaio_medium_bufs = memPoolCreate("squidaio_medium_bufs", AIO_MEDIUM_BUFS); + squidaio_small_bufs = memPoolCreate("squidaio_small_bufs", AIO_SMALL_BUFS); + squidaio_tiny_bufs = memPoolCreate("squidaio_tiny_bufs", AIO_TINY_BUFS); + squidaio_micro_bufs = memPoolCreate("squidaio_micro_bufs", AIO_MICRO_BUFS); + + squidaio_initialised = 1; +} + +void +squidaio_shutdown(void) +{ + squidaio_thread_t *threadp; + int i; + HANDLE *hthreads; + + if (!squidaio_initialised) + return; + + /* This is the same as in squidaio_sync */ + do { + squidaio_poll_queues(); + } while (request_queue_len > 0); + + hthreads = (HANDLE *) xcalloc(squidaio_nthreads, sizeof(HANDLE)); + threadp = threads; + for (i = 0; i < squidaio_nthreads; i++) { + threadp->exit = 1; + hthreads[i] = threadp->thread; + threadp = threadp->next; + } + ReleaseMutex(request_queue.mutex); + ResetEvent(request_queue.cond); + ReleaseMutex(done_queue.mutex); + ResetEvent(done_queue.cond); + Sleep(0); + + WaitForMultipleObjects(squidaio_nthreads, hthreads, TRUE, 2000); + for (i = 0; i < squidaio_nthreads; i++) { + CloseHandle(hthreads[i]); + } + CloseHandle(main_thread); + xfree(hthreads); + + fd_close(done_fd); + fd_close(done_fd_read); + close(done_fd); + close(done_fd_read); +} + +static DWORD WINAPI +squidaio_thread_loop(LPVOID lpParam) +{ + squidaio_thread_t *threadp = lpParam; + squidaio_request_t *request; + HANDLE cond; /* local copy of the event queue because win32 event handles + * don't atomically release the mutex as cond variables do. */ + + /* lock the thread info */ + if (WAIT_FAILED == WaitForSingleObject(request_queue.mutex, INFINITE)) { + fatal("Can't get ownership of mutex\n"); + } + /* duplicate the handle */ + if (!DuplicateHandle(GetCurrentProcess(), /* pseudo handle, don't close */ + request_queue.cond, /* handle to copy */ + GetCurrentProcess(), /* pseudo handle, don't close */ + &cond, + 0, /* required access */ + FALSE, /* child process's don't inherit the handle */ + DUPLICATE_SAME_ACCESS)) + fatal("Can't duplicate mutex handle\n"); + if (!ReleaseMutex(request_queue.mutex)) { + CloseHandle(cond); + fatal("Can't release mutex\n"); + } + Sleep(0); + + while (1) { + DWORD rv; + threadp->current_req = request = NULL; + request = NULL; + /* Get a request to process */ + threadp->status = _THREAD_WAITING; + if (threadp->exit) { + CloseHandle(request_queue.mutex); + CloseHandle(cond); + return 0; + } + rv = WaitForSingleObject(request_queue.mutex, INFINITE); + if (rv == WAIT_FAILED) { + CloseHandle(cond); + return 1; + } + while (!request_queue.head) { + if (!ReleaseMutex(request_queue.mutex)) { + CloseHandle(cond); + threadp->status = _THREAD_FAILED; + return 1; + } + rv = WaitForSingleObject(cond, INFINITE); + if (rv == WAIT_FAILED) { + CloseHandle(cond); + return 1; + } + rv = WaitForSingleObject(request_queue.mutex, INFINITE); + if (rv == WAIT_FAILED) { + CloseHandle(cond); + return 1; + } + } + request = request_queue.head; + if (request) + request_queue.head = request->next; + if (!request_queue.head) + request_queue.tailp = &request_queue.head; + if (!ReleaseMutex(request_queue.mutex)) { + CloseHandle(cond); + return 1; + } + /* process the request */ + threadp->status = _THREAD_BUSY; + request->next = NULL; + threadp->current_req = request; + errno = 0; + if (!request->cancelled) { + switch (request->request_type) { + case _AIO_OP_OPEN: + squidaio_do_open(request); + break; + case _AIO_OP_READ: + squidaio_do_read(request); + break; + case _AIO_OP_WRITE: + squidaio_do_write(request); + break; + case _AIO_OP_CLOSE: + squidaio_do_close(request); + break; + case _AIO_OP_UNLINK: + squidaio_do_unlink(request); + break; +#if USE_TRUNCATE + case _AIO_OP_TRUNCATE: + squidaio_do_truncate(request); + break; +#endif +#if AIO_OPENDIR /* Opendir not implemented yet */ + case _AIO_OP_OPENDIR: + squidaio_do_opendir(request); + break; +#endif + case _AIO_OP_STAT: + squidaio_do_stat(request); + break; + default: + request->ret = -1; + request->err = EINVAL; + break; + } + } else { /* cancelled */ + request->ret = -1; + request->err = EINTR; + } + threadp->status = _THREAD_DONE; + /* put the request in the done queue */ + rv = WaitForSingleObject(done_queue.mutex, INFINITE); + if (rv == WAIT_FAILED) { + CloseHandle(cond); + return 1; + } + *done_queue.tailp = request; + done_queue.tailp = &request->next; + if (!ReleaseMutex(done_queue.mutex)) { + CloseHandle(cond); + return 1; + } + if (!done_signalled) { + fde *F = &fd_table[done_fd]; + int x; + done_signalled = 1; + if ((x = FD_WRITE_METHOD(done_fd, "!", 1)) > 0) + F->bytes_written += x; + } + threadp->requests++; +/* Relinquish the remainder of thread time slice to any other thread + * of equal priority that is ready to run. + */ + Sleep(0); + } /* while forever */ + CloseHandle(cond); + return 0; +} /* squidaio_thread_loop */ + +static void +squidaio_queue_request(squidaio_request_t * request) +{ + static int high_start = 0; + debug(43, 9) ("squidaio_queue_request: %p type=%d result=%p\n", + request, request->request_type, request->resultp); + /* Mark it as not executed (failing result, no error) */ + request->ret = -1; + request->err = 0; + /* Internal housekeeping */ + request_queue_len += 1; + request->resultp->_data = request; + /* Play some tricks with the request_queue2 queue */ + request->next = NULL; + if (WaitForSingleObject(request_queue.mutex, 0) == WAIT_OBJECT_0) { + if (request_queue2.head) { + /* Grab blocked requests */ + *request_queue.tailp = request_queue2.head; + request_queue.tailp = request_queue2.tailp; + } + /* Enqueue request */ + *request_queue.tailp = request; + request_queue.tailp = &request->next; + if (!SetEvent(request_queue.cond)) + fatal("couldn't push queue\n"); + if (!ReleaseMutex(request_queue.mutex)) { + /* unexpected error */ + fatal("couldn't push queue\n"); + } + if (request_queue2.head) { + /* Clear queue of blocked requests */ + request_queue2.head = NULL; + request_queue2.tailp = &request_queue2.head; + } + } else { + /* Oops, the request queue is blocked, use request_queue2 */ + *request_queue2.tailp = request; + request_queue2.tailp = &request->next; + } + if (request_queue2.head) { + static int filter = 0; + static int filter_limit = 8; + if (++filter >= filter_limit) { + filter_limit += filter; + filter = 0; + debug(43, 1) ("squidaio_queue_request: WARNING - Queue congestion\n"); + } + } + /* Warn if out of threads */ + if (request_queue_len > MAGIC1) { + static int last_warn = 0; + static int queue_high, queue_low; + if (high_start == 0) { + high_start = squid_curtime; + queue_high = request_queue_len; + queue_low = request_queue_len; + } + if (request_queue_len > queue_high) + queue_high = request_queue_len; + if (request_queue_len < queue_low) + queue_low = request_queue_len; + if (squid_curtime >= (last_warn + 15) && + squid_curtime >= (high_start + 5)) { + debug(43, 1) ("squidaio_queue_request: WARNING - Disk I/O overloading\n"); + if (squid_curtime >= (high_start + 15)) + debug(43, 1) ("squidaio_queue_request: Queue Length: current=%d, high=%d, low=%d, duration=%ld\n", + request_queue_len, queue_high, queue_low, (long int) (squid_curtime - high_start)); + last_warn = squid_curtime; + } + } else { + high_start = 0; + } + /* Warn if seriously overloaded */ + if (request_queue_len > RIDICULOUS_LENGTH) { + debug(43, 0) ("squidaio_queue_request: Async request queue growing uncontrollably!\n"); + debug(43, 0) ("squidaio_queue_request: Syncing pending I/O operations.. (blocking)\n"); + squidaio_sync(); + debug(43, 0) ("squidaio_queue_request: Synced\n"); + } +} /* squidaio_queue_request */ + +static void +squidaio_cleanup_request(squidaio_request_t * requestp) +{ + squidaio_result_t *resultp = requestp->resultp; + int cancelled = requestp->cancelled; + + /* Free allocated structures and copy data back to user space if the */ + /* request hasn't been cancelled */ + switch (requestp->request_type) { + case _AIO_OP_STAT: + if (!cancelled && requestp->ret == 0) + xmemcpy(requestp->statp, requestp->tmpstatp, sizeof(struct stat)); + squidaio_xfree(requestp->tmpstatp, sizeof(struct stat)); + squidaio_xstrfree(requestp->path); + break; + case _AIO_OP_OPEN: + if (cancelled && requestp->ret >= 0) + /* The open() was cancelled but completed */ + close(requestp->ret); + squidaio_xstrfree(requestp->path); + break; + case _AIO_OP_CLOSE: + if (cancelled && requestp->ret < 0) + /* The close() was cancelled and never got executed */ + close(requestp->fd); + break; + case _AIO_OP_UNLINK: + case _AIO_OP_TRUNCATE: + case _AIO_OP_OPENDIR: + squidaio_xstrfree(requestp->path); + break; + case _AIO_OP_READ: + break; + case _AIO_OP_WRITE: + break; + default: + break; + } + if (resultp != NULL && !cancelled) { + resultp->aio_return = requestp->ret; + resultp->aio_errno = requestp->err; + } + memPoolFree(squidaio_request_pool, requestp); +} /* squidaio_cleanup_request */ + + +int +squidaio_cancel(squidaio_result_t * resultp) +{ + squidaio_request_t *request = resultp->_data; + + if (request && request->resultp == resultp) { + debug(43, 9) ("squidaio_cancel: %p type=%d result=%p\n", + request, request->request_type, request->resultp); + request->cancelled = 1; + request->resultp = NULL; + resultp->_data = NULL; + return 0; + } + return 1; +} /* squidaio_cancel */ + + +int +squidaio_open(const char *path, int oflag, mode_t mode, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + + requestp = memPoolAlloc(squidaio_request_pool); + requestp->path = (char *) squidaio_xstrdup(path); + requestp->oflag = oflag; + requestp->mode = mode; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_OPEN; + requestp->cancelled = 0; + + squidaio_queue_request(requestp); + return 0; +} + + +static void +squidaio_do_open(squidaio_request_t * requestp) +{ + requestp->ret = open(requestp->path, requestp->oflag, requestp->mode); + requestp->err = errno; +} + + +int +squidaio_read(int fd, char *bufp, int bufs, off_t offset, int whence, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + + requestp = memPoolAlloc(squidaio_request_pool); + requestp->fd = fd; + requestp->bufferp = bufp; + requestp->buflen = bufs; + requestp->offset = offset; + requestp->whence = whence; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_READ; + requestp->cancelled = 0; + + squidaio_queue_request(requestp); + return 0; +} + + +static void +squidaio_do_read(squidaio_request_t * requestp) +{ +#ifdef _SQUID_MSWIN_ + lseek(requestp->fd, requestp->offset, requestp->whence); + if (!ReadFile((HANDLE) _get_osfhandle(requestp->fd), requestp->bufferp, + requestp->buflen, (LPDWORD) & requestp->ret, NULL)) { + WIN32_maperror(GetLastError()); + requestp->ret = -1; + } +#else + requestp->ret = pread(requestp->fd, requestp->bufferp, requestp->buflen, requestp->offset); +#endif + requestp->err = errno; +} + + +int +squidaio_write(int fd, char *bufp, int bufs, off_t offset, int whence, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + + requestp = memPoolAlloc(squidaio_request_pool); + requestp->fd = fd; + requestp->bufferp = bufp; + requestp->buflen = bufs; + requestp->offset = offset; + requestp->whence = whence; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_WRITE; + requestp->cancelled = 0; + + squidaio_queue_request(requestp); + return 0; +} + + +static void +squidaio_do_write(squidaio_request_t * requestp) +{ +#ifdef _SQUID_MSWIN_ + if (!WriteFile((HANDLE) _get_osfhandle(requestp->fd), requestp->bufferp, + requestp->buflen, (LPDWORD) & requestp->ret, NULL)) { + WIN32_maperror(GetLastError()); + requestp->ret = -1; + } +#else + requestp->ret = pwrite(requestp->fd, requestp->bufferp, requestp->buflen, requestp->offset); +#endif + requestp->err = errno; +} + + +int +squidaio_close(int fd, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + + requestp = memPoolAlloc(squidaio_request_pool); + requestp->fd = fd; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_CLOSE; + requestp->cancelled = 0; + + squidaio_queue_request(requestp); + return 0; +} + + +static void +squidaio_do_close(squidaio_request_t * requestp) +{ + if ((requestp->ret = close(requestp->fd)) < 0) { + debug(43, 0) ("squidaio_do_close: FD %d, errno %d\n", requestp->fd, errno); + close(requestp->fd); + } + requestp->err = errno; +} + + +int +squidaio_stat(const char *path, struct stat *sb, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + + requestp = memPoolAlloc(squidaio_request_pool); + requestp->path = (char *) squidaio_xstrdup(path); + requestp->statp = sb; + requestp->tmpstatp = (struct stat *) squidaio_xmalloc(sizeof(struct stat)); + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_STAT; + requestp->cancelled = 0; + + squidaio_queue_request(requestp); + return 0; +} + + +static void +squidaio_do_stat(squidaio_request_t * requestp) +{ + requestp->ret = stat(requestp->path, requestp->tmpstatp); + requestp->err = errno; +} + + +int +squidaio_unlink(const char *path, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + + requestp = memPoolAlloc(squidaio_request_pool); + requestp->path = squidaio_xstrdup(path); + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_UNLINK; + requestp->cancelled = 0; + + squidaio_queue_request(requestp); + return 0; +} + + +static void +squidaio_do_unlink(squidaio_request_t * requestp) +{ + requestp->ret = unlink(requestp->path); + requestp->err = errno; +} + + +#if USE_TRUNCATE +int +squidaio_truncate(const char *path, off_t length, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + + requestp = memPoolAlloc(squidaio_request_pool); + requestp->path = (char *) squidaio_xstrdup(path); + requestp->offset = length; + requestp->resultp = resultp; + requestp->request_type = _AIO_OP_TRUNCATE; + requestp->cancelled = 0; + + squidaio_queue_request(requestp); + return 0; +} + + +static void +squidaio_do_truncate(squidaio_request_t * requestp) +{ + requestp->ret = truncate(requestp->path, requestp->offset); + requestp->err = errno; +} +#endif + + +#if AIO_OPENDIR +/* XXX squidaio_opendir NOT implemented yet.. */ + +int +squidaio_opendir(const char *path, squidaio_result_t * resultp) +{ + squidaio_request_t *requestp; + int len; + + requestp = memPoolAlloc(squidaio_request_pool); + return -1; +} + +static void +squidaio_do_opendir(squidaio_request_t * requestp) +{ + /* NOT IMPLEMENTED */ +} + +#endif + +static void +squidaio_poll_queues(void) +{ + /* kick "overflow" request queue */ + if (request_queue2.head && + (WaitForSingleObject(request_queue.mutex, 0) == WAIT_OBJECT_0)) { + *request_queue.tailp = request_queue2.head; + request_queue.tailp = request_queue2.tailp; + if (!SetEvent(request_queue.cond)) + fatal("couldn't push queue\n"); + if (!ReleaseMutex(request_queue.mutex)) { + /* unexpected error */ + } + request_queue2.head = NULL; + request_queue2.tailp = &request_queue2.head; + } + /* Give up the CPU to allow the threads to do their work */ + if (done_queue.head || request_queue.head) + Sleep(0); + /* poll done queue */ + if (done_queue.head && + (WaitForSingleObject(done_queue.mutex, 0) == WAIT_OBJECT_0)) { + struct squidaio_request_t *requests = done_queue.head; + done_queue.head = NULL; + done_queue.tailp = &done_queue.head; + if (!ReleaseMutex(done_queue.mutex)) { + /* unexpected error */ + } + *done_requests.tailp = requests; + request_queue_len -= 1; + while (requests->next) { + requests = requests->next; + request_queue_len -= 1; + } + done_requests.tailp = &requests->next; + } +} + +squidaio_result_t * +squidaio_poll_done(void) +{ + squidaio_request_t *request; + squidaio_result_t *resultp; + int cancelled; + int polled = 0; + + AIO_REPOLL: + request = done_requests.head; + if (request == NULL && !polled) { + if (done_signalled) { + char junk[256]; + fde *F = &fd_table[done_fd_read]; + int x; + if ((x = FD_READ_METHOD(done_fd_read, junk, sizeof(junk))) > 0) + F->bytes_read += x; + done_signalled = 0; + } + squidaio_poll_queues(); + polled = 1; + request = done_requests.head; + } + if (!request) { + return NULL; + } + debug(43, 9) ("squidaio_poll_done: %p type=%d result=%p\n", + request, request->request_type, request->resultp); + done_requests.head = request->next; + if (!done_requests.head) + done_requests.tailp = &done_requests.head; + resultp = request->resultp; + cancelled = request->cancelled; + squidaio_debug(request); + debug(43, 5) ("DONE: %d -> %d\n", request->ret, request->err); + squidaio_cleanup_request(request); + if (cancelled) + goto AIO_REPOLL; + return resultp; +} /* squidaio_poll_done */ + +int +squidaio_operations_pending(void) +{ + return request_queue_len + (done_requests.head ? 1 : 0); +} + +int +squidaio_sync(void) +{ + /* XXX This might take a while if the queue is large.. */ + do { + squidaio_poll_queues(); + } while (request_queue_len > 0); + return squidaio_operations_pending(); +} + +int +squidaio_get_queue_len(void) +{ + return request_queue_len; +} + +static void +squidaio_debug(squidaio_request_t * request) +{ + switch (request->request_type) { + case _AIO_OP_OPEN: + debug(43, 5) ("OPEN of %s to FD %d\n", request->path, request->ret); + break; + case _AIO_OP_READ: + debug(43, 5) ("READ on fd: %d\n", request->fd); + break; + case _AIO_OP_WRITE: + debug(43, 5) ("WRITE on fd: %d\n", request->fd); + break; + case _AIO_OP_CLOSE: + debug(43, 5) ("CLOSE of fd: %d\n", request->fd); + break; + case _AIO_OP_UNLINK: + debug(43, 5) ("UNLINK of %s\n", request->path); + break; + case _AIO_OP_TRUNCATE: + debug(43, 5) ("UNLINK of %s\n", request->path); + break; + default: + break; + } +} + +void +squidaio_stats(StoreEntry * sentry) +{ + squidaio_thread_t *threadp; + int i; + + if (!squidaio_initialised) + return; + + storeAppendPrintf(sentry, "\n\nThreads Status:\n"); + storeAppendPrintf(sentry, "#\tID\t# Requests\n"); + + threadp = threads; + for (i = 0; i < squidaio_nthreads; i++) { + storeAppendPrintf(sentry, "%i\t0x%lx\t%ld\n", i + 1, threadp->dwThreadId, threadp->requests); + threadp = threadp->next; + } +} Index: squid/src/fs/aufs/async_io.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/async_io.c,v retrieving revision 1.15.8.1 retrieving revision 1.15.8.2 diff -u -r1.15.8.1 -r1.15.8.2 --- squid/src/fs/aufs/async_io.c 19 May 2006 16:00:19 -0000 1.15.8.1 +++ squid/src/fs/aufs/async_io.c 9 Sep 2006 12:30:41 -0000 1.15.8.2 @@ -268,6 +268,7 @@ dlinkAdd(ctrlp, &ctrlp->node, &used_list); } /* aioUnlink */ +#if USE_TRUNCATE void aioTruncate(const char *path, off_t length, AIOCB * callback, void *callback_data) { @@ -284,6 +285,7 @@ squidaio_truncate(path, length, &ctrlp->result); dlinkAdd(ctrlp, &ctrlp->node, &used_list); } /* aioTruncate */ +#endif int Index: squid/src/fs/aufs/store_dir_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_dir_aufs.c,v retrieving revision 1.35.8.12 retrieving revision 1.35.8.13 diff -u -r1.35.8.12 -r1.35.8.13 --- squid/src/fs/aufs/store_dir_aufs.c 3 Aug 2006 19:08:57 -0000 1.35.8.12 +++ squid/src/fs/aufs/store_dir_aufs.c 9 Sep 2006 12:30:42 -0000 1.35.8.13 @@ -216,7 +216,11 @@ } else { fatalf("Swap directory %s is not a directory.", path); } +#ifdef _SQUID_MSWIN_ + } else if (0 == mkdir(path)) { +#else } else if (0 == mkdir(path, 0755)) { +#endif debug(47, should_exist ? 1 : 3) ("%s created\n", path); created = 1; } else { @@ -1301,7 +1305,7 @@ fd = state->fd; /* rename */ if (state->fd >= 0) { -#if defined(_SQUID_OS2_) || defined (_SQUID_CYGWIN_) +#if defined(_SQUID_OS2_) || defined(_SQUID_WIN32_) file_close(state->fd); state->fd = -1; #endif @@ -1400,7 +1404,11 @@ if (dp == NULL) { if (errno == ENOENT) { debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1); +#ifdef _SQUID_MSWIN_ + if (mkdir(p1) == 0) +#else if (mkdir(p1, 0777) == 0) +#endif return 0; } debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror()); @@ -1625,7 +1633,7 @@ { debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f); /* storeAufsDirMapBitReset(SD, f); */ -#if USE_TRUNCATE_NOT_UNLINK +#if USE_TRUNCATE aioTruncate(storeAufsDirFullPath(SD, f, NULL), NULL, NULL); #else aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL); --- squid/src/fs/awin32/.cvsignore Wed Feb 14 01:17:06 2007 +++ /dev/null Wed Feb 14 01:16:58 2007 @@ -1,4 +0,0 @@ -.deps -.dirstamp -Makefile -Makefile.in --- squid/src/fs/awin32/aiops.c Wed Feb 14 01:17:06 2007 +++ /dev/null Wed Feb 14 01:16:58 2007 @@ -1,1076 +0,0 @@ -/* - * $Id: aiops.c,v 1.1.88.4 2006/07/31 10:48:39 serassio Exp $ - * - * DEBUG: section 43 AIOPS - * AUTHOR: Stewart Forster - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include -#include "async_io.h" - -#include -#include -#include -#include -#include -#include -#include - -#define RIDICULOUS_LENGTH 4096 - -#ifdef AUFS_IO_THREADS -int squidaio_nthreads = AUFS_IO_THREADS; -#else -int squidaio_nthreads = 0; -#endif -int squidaio_magic1 = 1; /* dummy initializer value */ -int squidaio_magic2 = 1; /* real value set in aiops.c */ - -enum _squidaio_thread_status { - _THREAD_STARTING = 0, - _THREAD_WAITING, - _THREAD_BUSY, - _THREAD_FAILED, - _THREAD_DONE -}; -typedef enum _squidaio_thread_status squidaio_thread_status; - -enum _squidaio_request_type { - _AIO_OP_NONE = 0, - _AIO_OP_OPEN, - _AIO_OP_READ, - _AIO_OP_WRITE, - _AIO_OP_CLOSE, - _AIO_OP_UNLINK, - _AIO_OP_TRUNCATE, - _AIO_OP_OPENDIR, - _AIO_OP_STAT -}; -typedef enum _squidaio_request_type squidaio_request_type; - -typedef struct squidaio_request_t { - struct squidaio_request_t *next; - squidaio_request_type request_type; - int cancelled; - char *path; - int oflag; - mode_t mode; - int fd; - char *bufferp; - int buflen; - off_t offset; - int whence; - int ret; - int err; - struct stat *tmpstatp; - struct stat *statp; - squidaio_result_t *resultp; -} squidaio_request_t; - -typedef struct squidaio_request_queue_t { - HANDLE mutex; - HANDLE cond; /* See Event objects */ - squidaio_request_t *volatile head; - squidaio_request_t *volatile *volatile tailp; - unsigned long requests; - unsigned long blocked; /* main failed to lock the queue */ -} squidaio_request_queue_t; - -typedef struct squidaio_thread_t squidaio_thread_t; -struct squidaio_thread_t { - squidaio_thread_t *next; - HANDLE thread; - DWORD dwThreadId; /* thread ID */ - squidaio_thread_status status; - struct squidaio_request_t *current_req; - unsigned long requests; - int volatile exit; -}; - -static void squidaio_queue_request(squidaio_request_t *); -static void squidaio_cleanup_request(squidaio_request_t *); -static DWORD WINAPI squidaio_thread_loop(LPVOID lpParam); -static void squidaio_do_open(squidaio_request_t *); -static void squidaio_do_read(squidaio_request_t *); -static void squidaio_do_write(squidaio_request_t *); -static void squidaio_do_close(squidaio_request_t *); -static void squidaio_do_stat(squidaio_request_t *); -static void squidaio_do_unlink(squidaio_request_t *); -#if USE_TRUNCATE -static void squidaio_do_truncate(squidaio_request_t *); -#endif -#if AIO_OPENDIR -static void *squidaio_do_opendir(squidaio_request_t *); -#endif -static void squidaio_debug(squidaio_request_t *); -static void squidaio_poll_queues(void); - -static squidaio_thread_t *threads = NULL; -static int squidaio_initialised = 0; - -#define AIO_LARGE_BUFS 16384 -#define AIO_MEDIUM_BUFS AIO_LARGE_BUFS >> 1 -#define AIO_SMALL_BUFS AIO_LARGE_BUFS >> 2 -#define AIO_TINY_BUFS AIO_LARGE_BUFS >> 3 -#define AIO_MICRO_BUFS 128 - -static MemPool *squidaio_large_bufs = NULL; /* 16K */ -static MemPool *squidaio_medium_bufs = NULL; /* 8K */ -static MemPool *squidaio_small_bufs = NULL; /* 4K */ -static MemPool *squidaio_tiny_bufs = NULL; /* 2K */ -static MemPool *squidaio_micro_bufs = NULL; /* 128K */ - -static int request_queue_len = 0; -static MemPool *squidaio_request_pool = NULL; -static MemPool *squidaio_thread_pool = NULL; -static squidaio_request_queue_t request_queue; -static struct { - squidaio_request_t *head, **tailp; -} request_queue2 = { - - NULL, &request_queue2.head -}; -static squidaio_request_queue_t done_queue; -static struct { - squidaio_request_t *head, **tailp; -} done_requests = { - - NULL, &done_requests.head -}; -static int done_fd = 0; -static int done_fd_read = 0; -static int done_signalled = 0; -static HANDLE main_thread; - -static MemPool * -squidaio_get_pool(int size) -{ - MemPool *p; - if (size <= AIO_LARGE_BUFS) { - if (size <= AIO_MICRO_BUFS) - p = squidaio_micro_bufs; - else if (size <= AIO_TINY_BUFS) - p = squidaio_tiny_bufs; - else if (size <= AIO_SMALL_BUFS) - p = squidaio_small_bufs; - else if (size <= AIO_MEDIUM_BUFS) - p = squidaio_medium_bufs; - else - p = squidaio_large_bufs; - } else - p = NULL; - return p; -} - -void * -squidaio_xmalloc(int size) -{ - void *p; - MemPool *pool; - - if ((pool = squidaio_get_pool(size)) != NULL) { - p = memPoolAlloc(pool); - } else - p = xmalloc(size); - - return p; -} - -static char * -squidaio_xstrdup(const char *str) -{ - char *p; - int len = strlen(str) + 1; - - p = squidaio_xmalloc(len); - strncpy(p, str, len); - - return p; -} - -void -squidaio_xfree(void *p, int size) -{ - MemPool *pool; - - if ((pool = squidaio_get_pool(size)) != NULL) { - memPoolFree(pool, p); - } else - xfree(p); -} - -static void -squidaio_xstrfree(char *str) -{ - MemPool *pool; - int len = strlen(str) + 1; - - if ((pool = squidaio_get_pool(len)) != NULL) { - memPoolFree(pool, str); - } else - xfree(str); -} - -static void -squidaio_fdhandler(int fd, void *data) -{ - char junk[256]; - int x; - fde *F = &fd_table[done_fd_read]; - if ((x = FD_READ_METHOD(done_fd_read, junk, sizeof(junk))) > 0) - F->bytes_read += x; - commSetSelect(fd, COMM_SELECT_READ, squidaio_fdhandler, NULL, 0); -} - -void -squidaio_init(void) -{ - int i; - int done_pipe[2]; - squidaio_thread_t *threadp; - - if (squidaio_initialised) - return; - - if (!DuplicateHandle(GetCurrentProcess(), /* pseudo handle, don't close */ - GetCurrentThread(), /* pseudo handle to copy */ - GetCurrentProcess(), /* pseudo handle, don't close */ - &main_thread, - 0, /* required access */ - FALSE, /* child process's don't inherit the handle */ - DUPLICATE_SAME_ACCESS)) { - /* spit errors */ - fatal("couldn't get current thread handle\n"); - } - /* Initialize request queue */ - if ((request_queue.mutex = CreateMutex(NULL, /* no inheritance */ - FALSE, /* start unowned (as per mutex_init) */ - NULL) /* no name */ - ) == NULL) { - fatal("failed to create mutex\n"); - } - if ((request_queue.cond = CreateEvent(NULL, /* no inheritance */ - FALSE, /* auto signal reset - which I think is pthreads like ? */ - FALSE, /* start non signaled */ - NULL) /* no name */ - ) == NULL) { - fatal("failed to create condition event variable.\n"); - } - request_queue.head = NULL; - request_queue.tailp = &request_queue.head; - request_queue.requests = 0; - request_queue.blocked = 0; - - /* Initialize done queue */ - if ((done_queue.mutex = CreateMutex(NULL, /* no inheritance */ - FALSE, /* start unowned (as per mutex_init) */ - NULL) /* no name */ - ) == NULL) { - fatal("failed to create mutex\n"); - } - if ((done_queue.cond = CreateEvent(NULL, /* no inheritance */ - TRUE, /* manually signaled - which I think is pthreads like ? */ - FALSE, /* start non signaled */ - NULL) /* no name */ - ) == NULL) { - fatal("failed to create condition event variable.\n"); - } - done_queue.head = NULL; - done_queue.tailp = &done_queue.head; - done_queue.requests = 0; - done_queue.blocked = 0; - - /* Initialize done pipe signal */ - pipe(done_pipe); - done_fd = done_pipe[1]; - done_fd_read = done_pipe[0]; - fd_open(done_fd_read, FD_PIPE, "async-io completion event: main"); - fd_open(done_fd, FD_PIPE, "async-io completion event: threads"); - commSetNonBlocking(done_pipe[0]); - commSetNonBlocking(done_pipe[1]); - commSetCloseOnExec(done_pipe[0]); - commSetCloseOnExec(done_pipe[1]); - commSetSelect(done_pipe[0], COMM_SELECT_READ, squidaio_fdhandler, NULL, 0); - - /* Create threads and get them to sit in their wait loop */ - squidaio_thread_pool = memPoolCreate("aio_thread", sizeof(squidaio_thread_t)); - if (squidaio_nthreads == 0) { - int j = 16; - for (i = 0; i < n_asyncufs_dirs; i++) { - squidaio_nthreads += j; - j = j * 2 / 3; - if (j < 4) - j = 4; - } -#if COSS_USE_AUFSOPS - j = 6; - for (i = 0; i < n_coss_dirs; i++) { - squidaio_nthreads += j; - j = 3; - } -#endif - } - if (squidaio_nthreads == 0) - squidaio_nthreads = 16; - squidaio_magic1 = squidaio_nthreads * MAGIC1_FACTOR; - squidaio_magic2 = squidaio_nthreads * MAGIC2_FACTOR; - for (i = 0; i < squidaio_nthreads; i++) { - threadp = memPoolAlloc(squidaio_thread_pool); - threadp->status = _THREAD_STARTING; - threadp->current_req = NULL; - threadp->requests = 0; - threadp->next = threads; - threads = threadp; - if ((threadp->thread = CreateThread(NULL, /* no security attributes */ - 0, /* use default stack size */ - squidaio_thread_loop, /* thread function */ - threadp, /* argument to thread function */ - 0, /* use default creation flags */ - &(threadp->dwThreadId)) /* returns the thread identifier */ - ) == NULL) { - fprintf(stderr, "Thread creation failed\n"); - threadp->status = _THREAD_FAILED; - continue; - } - /* Set the new thread priority above parent process */ - SetThreadPriority(threadp->thread, THREAD_PRIORITY_ABOVE_NORMAL); - } - - /* Create request pool */ - squidaio_request_pool = memPoolCreate("aio_request", sizeof(squidaio_request_t)); - squidaio_large_bufs = memPoolCreate("squidaio_large_bufs", AIO_LARGE_BUFS); - squidaio_medium_bufs = memPoolCreate("squidaio_medium_bufs", AIO_MEDIUM_BUFS); - squidaio_small_bufs = memPoolCreate("squidaio_small_bufs", AIO_SMALL_BUFS); - squidaio_tiny_bufs = memPoolCreate("squidaio_tiny_bufs", AIO_TINY_BUFS); - squidaio_micro_bufs = memPoolCreate("squidaio_micro_bufs", AIO_MICRO_BUFS); - - squidaio_initialised = 1; -} - -void -squidaio_shutdown(void) -{ - squidaio_thread_t *threadp; - int i; - HANDLE *hthreads; - - if (!squidaio_initialised) - return; - - /* This is the same as in squidaio_sync */ - do { - squidaio_poll_queues(); - } while (request_queue_len > 0); - - hthreads = (HANDLE *) xcalloc(squidaio_nthreads, sizeof(HANDLE)); - threadp = threads; - for (i = 0; i < squidaio_nthreads; i++) { - threadp->exit = 1; - hthreads[i] = threadp->thread; - threadp = threadp->next; - } - ReleaseMutex(request_queue.mutex); - ResetEvent(request_queue.cond); - ReleaseMutex(done_queue.mutex); - ResetEvent(done_queue.cond); - Sleep(0); - - WaitForMultipleObjects(squidaio_nthreads, hthreads, TRUE, 2000); - for (i = 0; i < squidaio_nthreads; i++) { - CloseHandle(hthreads[i]); - } - CloseHandle(main_thread); - xfree(hthreads); - - fd_close(done_fd); - fd_close(done_fd_read); - close(done_fd); - close(done_fd_read); -} - -static DWORD WINAPI -squidaio_thread_loop(LPVOID lpParam) -{ - squidaio_thread_t *threadp = lpParam; - squidaio_request_t *request; - HANDLE cond; /* local copy of the event queue because win32 event handles - * don't atomically release the mutex as cond variables do. */ - - /* lock the thread info */ - if (WAIT_FAILED == WaitForSingleObject(request_queue.mutex, INFINITE)) { - fatal("Can't get ownership of mutex\n"); - } - /* duplicate the handle */ - if (!DuplicateHandle(GetCurrentProcess(), /* pseudo handle, don't close */ - request_queue.cond, /* handle to copy */ - GetCurrentProcess(), /* pseudo handle, don't close */ - &cond, - 0, /* required access */ - FALSE, /* child process's don't inherit the handle */ - DUPLICATE_SAME_ACCESS)) - fatal("Can't duplicate mutex handle\n"); - if (!ReleaseMutex(request_queue.mutex)) { - CloseHandle(cond); - fatal("Can't release mutex\n"); - } - Sleep(0); - - while (1) { - DWORD rv; - threadp->current_req = request = NULL; - request = NULL; - /* Get a request to process */ - threadp->status = _THREAD_WAITING; - if (threadp->exit) { - CloseHandle(request_queue.mutex); - CloseHandle(cond); - return 0; - } - rv = WaitForSingleObject(request_queue.mutex, INFINITE); - if (rv == WAIT_FAILED) { - CloseHandle(cond); - return 1; - } - while (!request_queue.head) { - if (!ReleaseMutex(request_queue.mutex)) { - CloseHandle(cond); - threadp->status = _THREAD_FAILED; - return 1; - } - rv = WaitForSingleObject(cond, INFINITE); - if (rv == WAIT_FAILED) { - CloseHandle(cond); - return 1; - } - rv = WaitForSingleObject(request_queue.mutex, INFINITE); - if (rv == WAIT_FAILED) { - CloseHandle(cond); - return 1; - } - } - request = request_queue.head; - if (request) - request_queue.head = request->next; - if (!request_queue.head) - request_queue.tailp = &request_queue.head; - if (!ReleaseMutex(request_queue.mutex)) { - CloseHandle(cond); - return 1; - } - /* process the request */ - threadp->status = _THREAD_BUSY; - request->next = NULL; - threadp->current_req = request; - errno = 0; - if (!request->cancelled) { - switch (request->request_type) { - case _AIO_OP_OPEN: - squidaio_do_open(request); - break; - case _AIO_OP_READ: - squidaio_do_read(request); - break; - case _AIO_OP_WRITE: - squidaio_do_write(request); - break; - case _AIO_OP_CLOSE: - squidaio_do_close(request); - break; - case _AIO_OP_UNLINK: - squidaio_do_unlink(request); - break; -#if USE_TRUNCATE - case _AIO_OP_TRUNCATE: - squidaio_do_truncate(request); - break; -#endif -#if AIO_OPENDIR /* Opendir not implemented yet */ - case _AIO_OP_OPENDIR: - squidaio_do_opendir(request); - break; -#endif - case _AIO_OP_STAT: - squidaio_do_stat(request); - break; - default: - request->ret = -1; - request->err = EINVAL; - break; - } - } else { /* cancelled */ - request->ret = -1; - request->err = EINTR; - } - threadp->status = _THREAD_DONE; - /* put the request in the done queue */ - rv = WaitForSingleObject(done_queue.mutex, INFINITE); - if (rv == WAIT_FAILED) { - CloseHandle(cond); - return 1; - } - *done_queue.tailp = request; - done_queue.tailp = &request->next; - if (!ReleaseMutex(done_queue.mutex)) { - CloseHandle(cond); - return 1; - } - if (!done_signalled) { - fde *F = &fd_table[done_fd]; - int x; - done_signalled = 1; - if ((x = FD_WRITE_METHOD(done_fd, "!", 1)) > 0) - F->bytes_written += x; - } - threadp->requests++; -/* Relinquish the remainder of thread time slice to any other thread - * of equal priority that is ready to run. - */ - Sleep(0); - } /* while forever */ - CloseHandle(cond); - return 0; -} /* squidaio_thread_loop */ - -static void -squidaio_queue_request(squidaio_request_t * request) -{ - static int high_start = 0; - debug(43, 9) ("squidaio_queue_request: %p type=%d result=%p\n", - request, request->request_type, request->resultp); - /* Mark it as not executed (failing result, no error) */ - request->ret = -1; - request->err = 0; - /* Internal housekeeping */ - request_queue_len += 1; - request->resultp->_data = request; - /* Play some tricks with the request_queue2 queue */ - request->next = NULL; - if (WaitForSingleObject(request_queue.mutex, 0) == WAIT_OBJECT_0) { - if (request_queue2.head) { - /* Grab blocked requests */ - *request_queue.tailp = request_queue2.head; - request_queue.tailp = request_queue2.tailp; - } - /* Enqueue request */ - *request_queue.tailp = request; - request_queue.tailp = &request->next; - if (!SetEvent(request_queue.cond)) - fatal("couldn't push queue\n"); - if (!ReleaseMutex(request_queue.mutex)) { - /* unexpected error */ - fatal("couldn't push queue\n"); - } - if (request_queue2.head) { - /* Clear queue of blocked requests */ - request_queue2.head = NULL; - request_queue2.tailp = &request_queue2.head; - } - } else { - /* Oops, the request queue is blocked, use request_queue2 */ - *request_queue2.tailp = request; - request_queue2.tailp = &request->next; - } - if (request_queue2.head) { - static int filter = 0; - static int filter_limit = 8; - if (++filter >= filter_limit) { - filter_limit += filter; - filter = 0; - debug(43, 1) ("squidaio_queue_request: WARNING - Queue congestion\n"); - } - } - /* Warn if out of threads */ - if (request_queue_len > MAGIC1) { - static int last_warn = 0; - static int queue_high, queue_low; - if (high_start == 0) { - high_start = squid_curtime; - queue_high = request_queue_len; - queue_low = request_queue_len; - } - if (request_queue_len > queue_high) - queue_high = request_queue_len; - if (request_queue_len < queue_low) - queue_low = request_queue_len; - if (squid_curtime >= (last_warn + 15) && - squid_curtime >= (high_start + 5)) { - debug(43, 1) ("squidaio_queue_request: WARNING - Disk I/O overloading\n"); - if (squid_curtime >= (high_start + 15)) - debug(43, 1) ("squidaio_queue_request: Queue Length: current=%d, high=%d, low=%d, duration=%ld\n", - request_queue_len, queue_high, queue_low, (long int) (squid_curtime - high_start)); - last_warn = squid_curtime; - } - } else { - high_start = 0; - } - /* Warn if seriously overloaded */ - if (request_queue_len > RIDICULOUS_LENGTH) { - debug(43, 0) ("squidaio_queue_request: Async request queue growing uncontrollably!\n"); - debug(43, 0) ("squidaio_queue_request: Syncing pending I/O operations.. (blocking)\n"); - squidaio_sync(); - debug(43, 0) ("squidaio_queue_request: Synced\n"); - } -} /* squidaio_queue_request */ - -static void -squidaio_cleanup_request(squidaio_request_t * requestp) -{ - squidaio_result_t *resultp = requestp->resultp; - int cancelled = requestp->cancelled; - - /* Free allocated structures and copy data back to user space if the */ - /* request hasn't been cancelled */ - switch (requestp->request_type) { - case _AIO_OP_STAT: - if (!cancelled && requestp->ret == 0) - xmemcpy(requestp->statp, requestp->tmpstatp, sizeof(struct stat)); - squidaio_xfree(requestp->tmpstatp, sizeof(struct stat)); - squidaio_xstrfree(requestp->path); - break; - case _AIO_OP_OPEN: - if (cancelled && requestp->ret >= 0) - /* The open() was cancelled but completed */ - close(requestp->ret); - squidaio_xstrfree(requestp->path); - break; - case _AIO_OP_CLOSE: - if (cancelled && requestp->ret < 0) - /* The close() was cancelled and never got executed */ - close(requestp->fd); - break; - case _AIO_OP_UNLINK: - case _AIO_OP_TRUNCATE: - case _AIO_OP_OPENDIR: - squidaio_xstrfree(requestp->path); - break; - case _AIO_OP_READ: - break; - case _AIO_OP_WRITE: - break; - default: - break; - } - if (resultp != NULL && !cancelled) { - resultp->aio_return = requestp->ret; - resultp->aio_errno = requestp->err; - } - memPoolFree(squidaio_request_pool, requestp); -} /* squidaio_cleanup_request */ - - -int -squidaio_cancel(squidaio_result_t * resultp) -{ - squidaio_request_t *request = resultp->_data; - - if (request && request->resultp == resultp) { - debug(43, 9) ("squidaio_cancel: %p type=%d result=%p\n", - request, request->request_type, request->resultp); - request->cancelled = 1; - request->resultp = NULL; - resultp->_data = NULL; - return 0; - } - return 1; -} /* squidaio_cancel */ - - -int -squidaio_open(const char *path, int oflag, mode_t mode, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - - requestp = memPoolAlloc(squidaio_request_pool); - requestp->path = (char *) squidaio_xstrdup(path); - requestp->oflag = oflag; - requestp->mode = mode; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_OPEN; - requestp->cancelled = 0; - - squidaio_queue_request(requestp); - return 0; -} - - -static void -squidaio_do_open(squidaio_request_t * requestp) -{ - requestp->ret = open(requestp->path, requestp->oflag, requestp->mode); - requestp->err = errno; -} - - -int -squidaio_read(int fd, char *bufp, int bufs, off_t offset, int whence, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - - requestp = memPoolAlloc(squidaio_request_pool); - requestp->fd = fd; - requestp->bufferp = bufp; - requestp->buflen = bufs; - requestp->offset = offset; - requestp->whence = whence; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_READ; - requestp->cancelled = 0; - - squidaio_queue_request(requestp); - return 0; -} - - -static void -squidaio_do_read(squidaio_request_t * requestp) -{ -#ifdef _SQUID_MSWIN_ - lseek(requestp->fd, requestp->offset, requestp->whence); - if (!ReadFile((HANDLE) _get_osfhandle(requestp->fd), requestp->bufferp, - requestp->buflen, (LPDWORD) & requestp->ret, NULL)) { - WIN32_maperror(GetLastError()); - requestp->ret = -1; - } -#else - requestp->ret = pread(requestp->fd, requestp->bufferp, requestp->buflen, requestp->offset); -#endif - requestp->err = errno; -} - - -int -squidaio_write(int fd, char *bufp, int bufs, off_t offset, int whence, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - - requestp = memPoolAlloc(squidaio_request_pool); - requestp->fd = fd; - requestp->bufferp = bufp; - requestp->buflen = bufs; - requestp->offset = offset; - requestp->whence = whence; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_WRITE; - requestp->cancelled = 0; - - squidaio_queue_request(requestp); - return 0; -} - - -static void -squidaio_do_write(squidaio_request_t * requestp) -{ -#ifdef _SQUID_MSWIN_ - if (!WriteFile((HANDLE) _get_osfhandle(requestp->fd), requestp->bufferp, - requestp->buflen, (LPDWORD) & requestp->ret, NULL)) { - WIN32_maperror(GetLastError()); - requestp->ret = -1; - } -#else - requestp->ret = pwrite(requestp->fd, requestp->bufferp, requestp->buflen, requestp->offset); -#endif - requestp->err = errno; -} - - -int -squidaio_close(int fd, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - - requestp = memPoolAlloc(squidaio_request_pool); - requestp->fd = fd; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_CLOSE; - requestp->cancelled = 0; - - squidaio_queue_request(requestp); - return 0; -} - - -static void -squidaio_do_close(squidaio_request_t * requestp) -{ - if ((requestp->ret = close(requestp->fd)) < 0) { - debug(43, 0) ("squidaio_do_close: FD %d, errno %d\n", requestp->fd, errno); - close(requestp->fd); - } - requestp->err = errno; -} - - -int -squidaio_stat(const char *path, struct stat *sb, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - - requestp = memPoolAlloc(squidaio_request_pool); - requestp->path = (char *) squidaio_xstrdup(path); - requestp->statp = sb; - requestp->tmpstatp = (struct stat *) squidaio_xmalloc(sizeof(struct stat)); - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_STAT; - requestp->cancelled = 0; - - squidaio_queue_request(requestp); - return 0; -} - - -static void -squidaio_do_stat(squidaio_request_t * requestp) -{ - requestp->ret = stat(requestp->path, requestp->tmpstatp); - requestp->err = errno; -} - - -int -squidaio_unlink(const char *path, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - - requestp = memPoolAlloc(squidaio_request_pool); - requestp->path = squidaio_xstrdup(path); - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_UNLINK; - requestp->cancelled = 0; - - squidaio_queue_request(requestp); - return 0; -} - - -static void -squidaio_do_unlink(squidaio_request_t * requestp) -{ - requestp->ret = unlink(requestp->path); - requestp->err = errno; -} - - -#if USE_TRUNCATE -int -squidaio_truncate(const char *path, off_t length, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - - requestp = memPoolAlloc(squidaio_request_pool); - requestp->path = (char *) squidaio_xstrdup(path); - requestp->offset = length; - requestp->resultp = resultp; - requestp->request_type = _AIO_OP_TRUNCATE; - requestp->cancelled = 0; - - squidaio_queue_request(requestp); - return 0; -} - - -static void -squidaio_do_truncate(squidaio_request_t * requestp) -{ - requestp->ret = truncate(requestp->path, requestp->offset); - requestp->err = errno; -} - -#endif - - -#if AIO_OPENDIR -/* XXX squidaio_opendir NOT implemented yet.. */ - -int -squidaio_opendir(const char *path, squidaio_result_t * resultp) -{ - squidaio_request_t *requestp; - int len; - - requestp = memPoolAlloc(squidaio_request_pool); - return -1; -} - -static void -squidaio_do_opendir(squidaio_request_t * requestp) -{ - /* NOT IMPLEMENTED */ -} - -#endif - -static void -squidaio_poll_queues(void) -{ - /* kick "overflow" request queue */ - if (request_queue2.head && - (WaitForSingleObject(request_queue.mutex, 0) == WAIT_OBJECT_0)) { - *request_queue.tailp = request_queue2.head; - request_queue.tailp = request_queue2.tailp; - if (!SetEvent(request_queue.cond)) - fatal("couldn't push queue\n"); - if (!ReleaseMutex(request_queue.mutex)) { - /* unexpected error */ - } - request_queue2.head = NULL; - request_queue2.tailp = &request_queue2.head; - } - /* Give up the CPU to allow the threads to do their work */ - if (done_queue.head || request_queue.head) - Sleep(0); - /* poll done queue */ - if (done_queue.head && - (WaitForSingleObject(done_queue.mutex, 0) == WAIT_OBJECT_0)) { - struct squidaio_request_t *requests = done_queue.head; - done_queue.head = NULL; - done_queue.tailp = &done_queue.head; - if (!ReleaseMutex(done_queue.mutex)) { - /* unexpected error */ - } - *done_requests.tailp = requests; - request_queue_len -= 1; - while (requests->next) { - requests = requests->next; - request_queue_len -= 1; - } - done_requests.tailp = &requests->next; - } -} - -squidaio_result_t * -squidaio_poll_done(void) -{ - squidaio_request_t *request; - squidaio_result_t *resultp; - int cancelled; - int polled = 0; - - AIO_REPOLL: - request = done_requests.head; - if (request == NULL && !polled) { - if (done_signalled) { - char junk[256]; - fde *F = &fd_table[done_fd_read]; - int x; - if ((x = FD_READ_METHOD(done_fd_read, junk, sizeof(junk))) > 0) - F->bytes_read += x; - done_signalled = 0; - } - squidaio_poll_queues(); - polled = 1; - request = done_requests.head; - } - if (!request) { - return NULL; - } - debug(43, 9) ("squidaio_poll_done: %p type=%d result=%p\n", - request, request->request_type, request->resultp); - done_requests.head = request->next; - if (!done_requests.head) - done_requests.tailp = &done_requests.head; - resultp = request->resultp; - cancelled = request->cancelled; - squidaio_debug(request); - debug(43, 5) ("DONE: %d -> %d\n", request->ret, request->err); - squidaio_cleanup_request(request); - if (cancelled) - goto AIO_REPOLL; - return resultp; -} /* squidaio_poll_done */ - -int -squidaio_operations_pending(void) -{ - return request_queue_len + (done_requests.head ? 1 : 0); -} - -int -squidaio_sync(void) -{ - /* XXX This might take a while if the queue is large.. */ - do { - squidaio_poll_queues(); - } while (request_queue_len > 0); - return squidaio_operations_pending(); -} - -int -squidaio_get_queue_len(void) -{ - return request_queue_len; -} - -static void -squidaio_debug(squidaio_request_t * request) -{ - switch (request->request_type) { - case _AIO_OP_OPEN: - debug(43, 5) ("OPEN of %s to FD %d\n", request->path, request->ret); - break; - case _AIO_OP_READ: - debug(43, 5) ("READ on fd: %d\n", request->fd); - break; - case _AIO_OP_WRITE: - debug(43, 5) ("WRITE on fd: %d\n", request->fd); - break; - case _AIO_OP_CLOSE: - debug(43, 5) ("CLOSE of fd: %d\n", request->fd); - break; - case _AIO_OP_UNLINK: - debug(43, 5) ("UNLINK of %s\n", request->path); - break; - case _AIO_OP_TRUNCATE: - debug(43, 5) ("UNLINK of %s\n", request->path); - break; - default: - break; - } -} - -void -squidaio_stats(StoreEntry * sentry) -{ - squidaio_thread_t *threadp; - int i; - - if (!squidaio_initialised) - return; - - storeAppendPrintf(sentry, "\n\nThreads Status:\n"); - storeAppendPrintf(sentry, "#\tID\t# Requests\n"); - - threadp = threads; - for (i = 0; i < squidaio_nthreads; i++) { - storeAppendPrintf(sentry, "%i\t0x%lx\t%ld\n", i + 1, threadp->dwThreadId, threadp->requests); - threadp = threadp->next; - } -} --- squid/src/fs/awin32/async_io.c Wed Feb 14 01:17:06 2007 +++ /dev/null Wed Feb 14 01:16:58 2007 @@ -1,374 +0,0 @@ - -/* - * $Id: async_io.c,v 1.1.88.3 2006/07/05 16:45:18 serassio Exp $ - * - * DEBUG: section 32 Asynchronous Disk I/O - * AUTHOR: Pete Bentley - * AUTHOR: Stewart Forster - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "async_io.h" - -#define _AIO_OPEN 0 -#define _AIO_READ 1 -#define _AIO_WRITE 2 -#define _AIO_CLOSE 3 -#define _AIO_UNLINK 4 -#define _AIO_TRUNCATE 4 -#define _AIO_OPENDIR 5 -#define _AIO_STAT 6 - -typedef struct squidaio_ctrl_t { - struct squidaio_ctrl_t *next; - int fd; - int operation; - AIOCB *done_handler; - void *done_handler_data; - squidaio_result_t result; - int len; - char *bufp; - FREE *free_func; - dlink_node node; -} squidaio_ctrl_t; - -static struct { - int open; - int close; - int cancel; - int write; - int read; - int stat; - int unlink; - int check_callback; -} squidaio_counts; - -typedef struct squidaio_unlinkq_t { - char *path; - struct squidaio_unlinkq_t *next; -} squidaio_unlinkq_t; - -static dlink_list used_list; -static int initialised = 0; -static OBJH aioStats; -static MemPool *squidaio_ctrl_pool; - -void -aioInit(void) -{ - if (initialised) - return; - squidaio_ctrl_pool = memPoolCreate("aio_ctrl", sizeof(squidaio_ctrl_t)); - cachemgrRegister("squidaio_counts", "Async IO Function Counters", - aioStats, 0, 1); - initialised = 1; -} - -void -aioDone(void) -{ - squidaio_shutdown(); - memPoolDestroy(squidaio_ctrl_pool); - initialised = 0; -} - -void -aioOpen(const char *path, int oflag, mode_t mode, AIOCB * callback, void *callback_data) -{ - squidaio_ctrl_t *ctrlp; - - assert(initialised); - squidaio_counts.open++; - ctrlp = memPoolAlloc(squidaio_ctrl_pool); - ctrlp->fd = -2; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_OPEN; - cbdataLock(callback_data); - ctrlp->result.data = ctrlp; - squidaio_open(path, oflag, mode, &ctrlp->result); - dlinkAdd(ctrlp, &ctrlp->node, &used_list); - return; -} - -void -aioClose(int fd) -{ - squidaio_ctrl_t *ctrlp; - - assert(initialised); - squidaio_counts.close++; - aioCancel(fd); - ctrlp = memPoolAlloc(squidaio_ctrl_pool); - ctrlp->fd = fd; - ctrlp->done_handler = NULL; - ctrlp->done_handler_data = NULL; - ctrlp->operation = _AIO_CLOSE; - ctrlp->result.data = ctrlp; - squidaio_close(fd, &ctrlp->result); - dlinkAdd(ctrlp, &ctrlp->node, &used_list); - return; -} - -void -aioCancel(int fd) -{ - squidaio_ctrl_t *ctrlp; - AIOCB *done_handler; - void *their_data; - dlink_node *m, *next; - - assert(initialised); - squidaio_counts.cancel++; - for (m = used_list.head; m; m = next) { - next = m->next; - ctrlp = m->data; - if (ctrlp->fd != fd) - continue; - - squidaio_cancel(&ctrlp->result); - - if ((done_handler = ctrlp->done_handler)) { - their_data = ctrlp->done_handler_data; - ctrlp->done_handler = NULL; - ctrlp->done_handler_data = NULL; - debug(32, 0) ("this be aioCancel. Danger ahead!\n"); - if (cbdataValid(their_data)) - done_handler(fd, their_data, NULL, -2, -2); - cbdataUnlock(their_data); - /* free data if requested to aioWrite() */ - if (ctrlp->free_func) - ctrlp->free_func(ctrlp->bufp); - /* free temporary read buffer */ - if (ctrlp->operation == _AIO_READ) - squidaio_xfree(ctrlp->bufp, ctrlp->len); - } - dlinkDelete(m, &used_list); - memPoolFree(squidaio_ctrl_pool, ctrlp); - } -} - - -void -aioWrite(int fd, off_t offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func) -{ - squidaio_ctrl_t *ctrlp; - int seekmode; - - assert(initialised); - squidaio_counts.write++; - ctrlp = memPoolAlloc(squidaio_ctrl_pool); - ctrlp->fd = fd; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_WRITE; - ctrlp->bufp = bufp; - ctrlp->free_func = free_func; - if (offset >= 0) - seekmode = SEEK_SET; - else { - seekmode = SEEK_END; - offset = 0; - } - cbdataLock(callback_data); - ctrlp->result.data = ctrlp; - squidaio_write(fd, bufp, len, offset, seekmode, &ctrlp->result); - dlinkAdd(ctrlp, &ctrlp->node, &used_list); -} /* aioWrite */ - - -void -aioRead(int fd, off_t offset, int len, AIOCB * callback, void *callback_data) -{ - squidaio_ctrl_t *ctrlp; - int seekmode; - - assert(initialised); - squidaio_counts.read++; - ctrlp = memPoolAlloc(squidaio_ctrl_pool); - ctrlp->fd = fd; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_READ; - ctrlp->len = len; - ctrlp->bufp = squidaio_xmalloc(len); - if (offset >= 0) - seekmode = SEEK_SET; - else { - seekmode = SEEK_CUR; - offset = 0; - } - cbdataLock(callback_data); - ctrlp->result.data = ctrlp; - squidaio_read(fd, ctrlp->bufp, len, offset, seekmode, &ctrlp->result); - dlinkAdd(ctrlp, &ctrlp->node, &used_list); - return; -} /* aioRead */ - -void -aioStat(char *path, struct stat *sb, AIOCB * callback, void *callback_data) -{ - squidaio_ctrl_t *ctrlp; - - assert(initialised); - squidaio_counts.stat++; - ctrlp = memPoolAlloc(squidaio_ctrl_pool); - ctrlp->fd = -2; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_STAT; - cbdataLock(callback_data); - ctrlp->result.data = ctrlp; - squidaio_stat(path, sb, &ctrlp->result); - dlinkAdd(ctrlp, &ctrlp->node, &used_list); - return; -} /* aioStat */ - -void -aioUnlink(const char *path, AIOCB * callback, void *callback_data) -{ - squidaio_ctrl_t *ctrlp; - assert(initialised); - squidaio_counts.unlink++; - ctrlp = memPoolAlloc(squidaio_ctrl_pool); - ctrlp->fd = -2; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_UNLINK; - cbdataLock(callback_data); - ctrlp->result.data = ctrlp; - squidaio_unlink(path, &ctrlp->result); - dlinkAdd(ctrlp, &ctrlp->node, &used_list); -} /* aioUnlink */ - -#if USE_TRUNCATE -void -aioTruncate(const char *path, off_t length, AIOCB * callback, void *callback_data) -{ - squidaio_ctrl_t *ctrlp; - assert(initialised); - squidaio_counts.unlink++; - ctrlp = memPoolAlloc(squidaio_ctrl_pool); - ctrlp->fd = -2; - ctrlp->done_handler = callback; - ctrlp->done_handler_data = callback_data; - ctrlp->operation = _AIO_TRUNCATE; - cbdataLock(callback_data); - ctrlp->result.data = ctrlp; - squidaio_truncate(path, length, &ctrlp->result); - dlinkAdd(ctrlp, &ctrlp->node, &used_list); -} /* aioTruncate */ - -#endif - - -int -aioCheckCallbacks(SwapDir * SD) -{ - squidaio_result_t *resultp; - squidaio_ctrl_t *ctrlp; - AIOCB *done_handler; - void *their_data; - int retval = 0; - - assert(initialised); - squidaio_counts.check_callback++; - for (;;) { - if ((resultp = squidaio_poll_done()) == NULL) - break; - ctrlp = (squidaio_ctrl_t *) resultp->data; - if (ctrlp == NULL) - continue; /* XXX Should not happen */ - dlinkDelete(&ctrlp->node, &used_list); - if ((done_handler = ctrlp->done_handler)) { - their_data = ctrlp->done_handler_data; - ctrlp->done_handler = NULL; - ctrlp->done_handler_data = NULL; - if (cbdataValid(their_data)) { - retval = 1; /* Return that we've actually done some work */ - done_handler(ctrlp->fd, their_data, ctrlp->bufp, - ctrlp->result.aio_return, ctrlp->result.aio_errno); - } else { - if (ctrlp->operation == _AIO_OPEN) { - /* The open operation was aborted.. */ - int fd = ctrlp->result.aio_return; - if (fd >= 0) - aioClose(fd); - } - } - cbdataUnlock(their_data); - } - /* free data if requested to aioWrite() */ - if (ctrlp->free_func) - ctrlp->free_func(ctrlp->bufp); - /* free temporary read buffer */ - if (ctrlp->operation == _AIO_READ) - squidaio_xfree(ctrlp->bufp, ctrlp->len); - memPoolFree(squidaio_ctrl_pool, ctrlp); - } - return retval; -} - -void -aioStats(StoreEntry * sentry) -{ - storeAppendPrintf(sentry, "ASYNC IO Counters:\n"); - storeAppendPrintf(sentry, "Operation\t# Requests\n"); - storeAppendPrintf(sentry, "open\t%d\n", squidaio_counts.open); - storeAppendPrintf(sentry, "close\t%d\n", squidaio_counts.close); - storeAppendPrintf(sentry, "cancel\t%d\n", squidaio_counts.cancel); - storeAppendPrintf(sentry, "write\t%d\n", squidaio_counts.write); - storeAppendPrintf(sentry, "read\t%d\n", squidaio_counts.read); - storeAppendPrintf(sentry, "stat\t%d\n", squidaio_counts.stat); - storeAppendPrintf(sentry, "unlink\t%d\n", squidaio_counts.unlink); - storeAppendPrintf(sentry, "check_callback\t%d\n", squidaio_counts.check_callback); - storeAppendPrintf(sentry, "queue\t%d\n", squidaio_get_queue_len()); - squidaio_stats(sentry); -} - -/* Flush all pending I/O */ -void -aioSync(SwapDir * SD) -{ - if (!initialised) - return; /* nothing to do then */ - /* Flush all pending operations */ - debug(32, 1) ("aioSync: flushing pending I/O operations\n"); - do { - aioCheckCallbacks(SD); - } while (squidaio_sync()); - debug(32, 1) ("aioSync: done\n"); -} - -int -aioQueueSize(void) -{ - return memPoolInUseCount(squidaio_ctrl_pool); -} --- squid/src/fs/awin32/async_io.h Wed Feb 14 01:17:06 2007 +++ /dev/null Wed Feb 14 01:16:58 2007 @@ -1,71 +0,0 @@ -/* - * store_aufs.h - * - * Internal declarations for the aufs routines - */ - -#ifndef __ASYNC_IO_H__ -#define __ASYNC_IO_H__ - -extern int n_asyncufs_dirs; -extern int n_coss_dirs; -extern int squidaio_nthreads; -extern int squidaio_magic1; -extern int squidaio_magic2; - -/* Base number of threads if not specified to configure. - * Weighted by number of directories (see aiops.c) */ -#define THREAD_FACTOR 16 - -/* Queue limit where swapouts are deferred (load calculation) */ -#define MAGIC1_FACTOR 10 -#define MAGIC1 squidaio_magic1 -/* Queue limit where swapins are deferred (open/create fails) */ -#define MAGIC2_FACTOR 20 -#define MAGIC2 squidaio_magic2 - -struct _squidaio_result_t { - int aio_return; - int aio_errno; - void *_data; /* Internal housekeeping */ - void *data; /* Available to the caller */ -}; - -typedef struct _squidaio_result_t squidaio_result_t; - -typedef void AIOCB(int fd, void *cbdata, const char *buf, int aio_return, int aio_errno); - -void squidaio_init(void); -void squidaio_shutdown(void); -int squidaio_cancel(squidaio_result_t *); -int squidaio_open(const char *, int, mode_t, squidaio_result_t *); -int squidaio_read(int, char *, int, off_t, int, squidaio_result_t *); -int squidaio_write(int, char *, int, off_t, int, squidaio_result_t *); -int squidaio_close(int, squidaio_result_t *); -int squidaio_stat(const char *, struct stat *, squidaio_result_t *); -int squidaio_unlink(const char *, squidaio_result_t *); -int squidaio_truncate(const char *, off_t length, squidaio_result_t *); -int squidaio_opendir(const char *, squidaio_result_t *); -squidaio_result_t *squidaio_poll_done(void); -int squidaio_operations_pending(void); -int squidaio_sync(void); -int squidaio_get_queue_len(void); -void *squidaio_xmalloc(int size); -void squidaio_xfree(void *p, int size); -void squidaio_stats(StoreEntry *); - -void aioInit(void); -void aioDone(void); -void aioCancel(int); -void aioOpen(const char *, int, mode_t, AIOCB *, void *); -void aioClose(int); -void aioWrite(int, off_t offset, char *, int size, AIOCB *, void *, FREE *); -void aioRead(int, off_t offset, int size, AIOCB *, void *); -void aioStat(char *, struct stat *, AIOCB *, void *); -void aioUnlink(const char *, AIOCB *, void *); -void aioTruncate(const char *, off_t length, AIOCB *, void *); -int aioCheckCallbacks(SwapDir *); -void aioSync(SwapDir *); -int aioQueueSize(void); - -#endif --- squid/src/fs/awin32/store_asyncufs.h Wed Feb 14 01:17:06 2007 +++ /dev/null Wed Feb 14 01:16:58 2007 @@ -1,84 +0,0 @@ -/* - * store_aufs.h - * - * Internal declarations for the aufs routines - */ - -#ifndef __STORE_ASYNCUFS_H__ -#define __STORE_ASYNCUFS_H__ - -/* Which operations to run async */ -#define ASYNC_OPEN 1 -#define ASYNC_CLOSE 0 -#define ASYNC_CREATE 1 -#define ASYNC_WRITE 0 -#define ASYNC_READ 1 - -struct _squidaioinfo_t { - int swaplog_fd; - int l1; - int l2; - fileMap *map; - int suggest; -}; - -struct _squidaiostate_t { - int fd; - struct { - unsigned int close_request:1; - unsigned int reading:1; - unsigned int writing:1; - unsigned int opening:1; -#if !ASYNC_WRITE - unsigned int write_kicking:1; -#endif - unsigned int inreaddone:1; - } flags; - char *read_buf; - link_list *pending_writes; - link_list *pending_reads; -}; - -struct _queued_write { - char *buf; - size_t size; - off_t offset; - FREE *free_func; -}; - -struct _queued_read { - char *buf; - size_t size; - off_t offset; - STRCB *callback; - void *callback_data; -}; - -typedef struct _squidaioinfo_t squidaioinfo_t; -typedef struct _squidaiostate_t squidaiostate_t; - -/* The squidaio_state memory pools */ -extern MemPool *squidaio_state_pool; -extern MemPool *aufs_qread_pool; -extern MemPool *aufs_qwrite_pool; - -extern void storeAufsDirMapBitReset(SwapDir *, sfileno); -extern int storeAufsDirMapBitAllocate(SwapDir *); - -extern char *storeAufsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath); -extern void storeAufsDirUnlinkFile(SwapDir *, sfileno); -extern void storeAufsDirReplAdd(SwapDir * SD, StoreEntry *); -extern void storeAufsDirReplRemove(StoreEntry *); - -/* - * Store IO stuff - */ -extern STOBJCREATE storeAufsCreate; -extern STOBJOPEN storeAufsOpen; -extern STOBJCLOSE storeAufsClose; -extern STOBJREAD storeAufsRead; -extern STOBJWRITE storeAufsWrite; -extern STOBJUNLINK storeAufsUnlink; -extern STOBJRECYCLE storeAufsRecycle; - -#endif --- squid/src/fs/awin32/store_dir_aufs.c Wed Feb 14 01:17:06 2007 +++ /dev/null Wed Feb 14 01:16:58 2007 @@ -1,1936 +0,0 @@ - -/* - * $Id: store_dir_aufs.c,v 1.1.88.9 2006/08/06 14:38:57 serassio Exp $ - * - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Duane Wessels - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" - -#include "async_io.h" -#include "store_asyncufs.h" - -#define DefaultLevelOneDirs 16 -#define DefaultLevelTwoDirs 256 -#define STORE_META_BUFSZ 4096 - -typedef struct _RebuildState RebuildState; -struct _RebuildState { - SwapDir *sd; - int n_read; - FILE *log; - int speed; - int curlvl1; - int curlvl2; - struct { - unsigned int need_to_validate:1; - unsigned int clean:1; - unsigned int init:1; - } flags; - int done; - int in_dir; - int fn; - struct dirent *entry; - DIR *td; - char fullpath[SQUID_MAXPATHLEN]; - char fullfilename[SQUID_MAXPATHLEN]; - struct _store_rebuild_data counts; -}; - -int n_asyncufs_dirs = 0; -static int *asyncufs_dir_index = NULL; -MemPool *squidaio_state_pool = NULL; -MemPool *aufs_qread_pool = NULL; -MemPool *aufs_qwrite_pool = NULL; -static int asyncufs_initialised = 0; - -static char *storeAufsDirSwapSubDir(SwapDir *, int subdirn); -static int storeAufsDirCreateDirectory(const char *path, int); -static int storeAufsDirVerifyCacheDirs(SwapDir *); -static int storeAufsDirVerifyDirectory(const char *path); -static void storeAufsDirCreateSwapSubDirs(SwapDir *); -static char *storeAufsDirSwapLogFile(SwapDir *, const char *); -static EVH storeAufsDirRebuildFromDirectory; -static EVH storeAufsDirRebuildFromSwapLog; -static int storeAufsDirGetNextFile(RebuildState *, sfileno *, int *size); -static StoreEntry *storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key, - sfileno file_number, - squid_file_sz swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_num32 refcount, - u_short flags, - int clean); -static void storeAufsDirRebuild(SwapDir * sd); -static void storeAufsDirCloseTmpSwapLog(SwapDir * sd); -static FILE *storeAufsDirOpenTmpSwapLog(SwapDir *, int *, int *); -static STLOGOPEN storeAufsDirOpenSwapLog; -static STINIT storeAufsDirInit; -static STFREE storeAufsDirFree; -static STLOGCLEANSTART storeAufsDirWriteCleanStart; -static STLOGCLEANNEXTENTRY storeAufsDirCleanLogNextEntry; -static STLOGCLEANWRITE storeAufsDirWriteCleanEntry; -static STLOGCLEANDONE storeAufsDirWriteCleanDone; -static STLOGCLOSE storeAufsDirCloseSwapLog; -static STLOGWRITE storeAufsDirSwapLog; -static STNEWFS storeAufsDirNewfs; -static STDUMP storeAufsDirDump; -static STMAINTAINFS storeAufsDirMaintain; -static STCHECKOBJ storeAufsDirCheckObj; -static STCHECKLOADAV storeAufsDirCheckLoadAv; -static STREFOBJ storeAufsDirRefObj; -static STUNREFOBJ storeAufsDirUnrefObj; -static QS rev_int_sort; -static int storeAufsDirClean(int swap_index); -static EVH storeAufsDirCleanEvent; -static int storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2); -static int storeAufsCleanupDoubleCheck(SwapDir *, StoreEntry *); -static void storeAufsDirStats(SwapDir *, StoreEntry *); -static void storeAufsDirInitBitmap(SwapDir *); -static int storeAufsDirValidFileno(SwapDir *, sfileno, int); - -/* The MAIN externally visible function */ -STSETUP storeFsSetup_awin32; - -/* - * These functions were ripped straight out of the heart of store_dir.c. - * They assume that the given filenum is on a asyncufs partiton, which may or - * may not be true.. - * XXX this evilness should be tidied up at a later date! - */ - -static int -storeAufsDirMapBitTest(SwapDir * SD, sfileno filn) -{ - squidaioinfo_t *aioinfo; - aioinfo = (squidaioinfo_t *) SD->fsdata; - return file_map_bit_test(aioinfo->map, filn); -} - -static void -storeAufsDirMapBitSet(SwapDir * SD, sfileno filn) -{ - squidaioinfo_t *aioinfo; - aioinfo = (squidaioinfo_t *) SD->fsdata; - file_map_bit_set(aioinfo->map, filn); -} - -void -storeAufsDirMapBitReset(SwapDir * SD, sfileno filn) -{ - squidaioinfo_t *aioinfo; - aioinfo = (squidaioinfo_t *) SD->fsdata; - /* - * We have to test the bit before calling file_map_bit_reset. - * file_map_bit_reset doesn't do bounds checking. It assumes - * filn is a valid file number, but it might not be because - * the map is dynamic in size. Also clearing an already clear - * bit puts the map counter of-of-whack. - */ - if (file_map_bit_test(aioinfo->map, filn)) - file_map_bit_reset(aioinfo->map, filn); -} - -int -storeAufsDirMapBitAllocate(SwapDir * SD) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata; - int fn; - fn = file_map_allocate(aioinfo->map, aioinfo->suggest); - file_map_bit_set(aioinfo->map, fn); - aioinfo->suggest = fn + 1; - return fn; -} - -/* - * Initialise the asyncufs bitmap - * - * If there already is a bitmap, and the numobjects is larger than currently - * configured, we allocate a new bitmap and 'grow' the old one into it. - */ -static void -storeAufsDirInitBitmap(SwapDir * sd) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - - if (aioinfo->map == NULL) { - /* First time */ - aioinfo->map = file_map_create(); - } else if (aioinfo->map->max_n_files) { - /* it grew, need to expand */ - /* XXX We don't need it anymore .. */ - } - /* else it shrunk, and we leave the old one in place */ -} - -static char * -storeAufsDirSwapSubDir(SwapDir * sd, int subdirn) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - - LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - assert(0 <= subdirn && subdirn < aioinfo->l1); - snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn); - return fullfilename; -} - -static int -storeAufsDirCreateDirectory(const char *path, int should_exist) -{ - int created = 0; - struct stat st; - getCurrentTime(); - if (0 == stat(path, &st)) { - if (S_ISDIR(st.st_mode)) { - debug(47, should_exist ? 3 : 1) ("%s exists\n", path); - } else { - fatalf("Swap directory %s is not a directory.", path); - } -#ifdef _SQUID_MSWIN_ - } else if (0 == mkdir(path)) { -#else - } else if (0 == mkdir(path, 0755)) { -#endif - debug(47, should_exist ? 1 : 3) ("%s created\n", path); - created = 1; - } else { - fatalf("Failed to make swap directory %s: %s", - path, xstrerror()); - } - return created; -} - -static int -storeAufsDirVerifyDirectory(const char *path) -{ - struct stat sb; - if (stat(path, &sb) < 0) { - debug(47, 0) ("%s: %s\n", path, xstrerror()); - return -1; - } - if (S_ISDIR(sb.st_mode) == 0) { - debug(47, 0) ("%s is not a directory\n", path); - return -1; - } - return 0; -} - -/* - * This function is called by storeAufsDirInit(). If this returns < 0, - * then Squid exits, complains about swap directories not - * existing, and instructs the admin to run 'squid -z' - */ -static int -storeAufsDirVerifyCacheDirs(SwapDir * sd) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - int j; - const char *path = sd->path; - - if (storeAufsDirVerifyDirectory(path) < 0) - return -1; - for (j = 0; j < aioinfo->l1; j++) { - path = storeAufsDirSwapSubDir(sd, j); - if (storeAufsDirVerifyDirectory(path) < 0) - return -1; - } - return 0; -} - -static void -storeAufsDirCreateSwapSubDirs(SwapDir * sd) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - int i, k; - int should_exist; - LOCAL_ARRAY(char, name, MAXPATHLEN); - for (i = 0; i < aioinfo->l1; i++) { - snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i); - if (storeAufsDirCreateDirectory(name, 0)) - should_exist = 0; - else - should_exist = 1; - debug(47, 1) ("Making directories in %s\n", name); - for (k = 0; k < aioinfo->l2; k++) { - snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k); - storeAufsDirCreateDirectory(name, should_exist); - } - } -} - -static char * -storeAufsDirSwapLogFile(SwapDir * sd, const char *ext) -{ - LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); - LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); - LOCAL_ARRAY(char, digit, 32); - char *pathtmp2; - if (Config.Log.swap) { - xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); - pathtmp2 = pathtmp; - while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) - *pathtmp2 = '.'; - while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') - pathtmp[strlen(pathtmp) - 1] = '\0'; - for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); - snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); - if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); - } - } else { - xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); - strcat(path, "/swap.state"); - } - if (ext) - strncat(path, ext, 16); - return path; -} - -static void -storeAufsDirOpenSwapLog(SwapDir * sd) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - char *path; - int fd; - path = storeAufsDirSwapLogFile(sd, NULL); - if (aioinfo->swaplog_fd >= 0) { - debug(50, 1) ("storeAufsDirOpenSwapLog: %s already open\n", path); - return; - } - fd = file_open(path, O_WRONLY | O_CREAT | O_BINARY); - if (fd < 0) { - debug(50, 1) ("%s: %s\n", path, xstrerror()); - fatal("storeAufsDirOpenSwapLog: Failed to open swap log."); - } - debug(50, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); - aioinfo->swaplog_fd = fd; -} - -static void -storeAufsDirCloseSwapLog(SwapDir * sd) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - if (aioinfo->swaplog_fd < 0) /* not open */ - return; - file_close(aioinfo->swaplog_fd); - debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", - sd->index, aioinfo->swaplog_fd); - aioinfo->swaplog_fd = -1; -} - -static void -storeAufsCheckConfig(SwapDir * sd) -{ - if (!opt_create_swap_dirs) - requirePathnameExists("cache_dir", sd->path); -} - - -static void -storeAufsDirInit(SwapDir * sd) -{ - static int started_clean_event = 0; - static const char *errmsg = - "\tFailed to verify one of the swap directories, Check cache.log\n" - "\tfor details. Run 'squid -z' to create swap directories\n" - "\tif needed, or if running Squid for the first time."; - storeAufsDirInitBitmap(sd); - if (storeAufsDirVerifyCacheDirs(sd) < 0) - fatal(errmsg); - squidaio_init(); - storeAufsDirOpenSwapLog(sd); - storeAufsDirRebuild(sd); - if (!started_clean_event) { - eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL, 15.0, 1); - started_clean_event = 1; - } - (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize); -} - -static void -storeAufsDirRebuildComplete(RebuildState * rb) -{ - if (rb->log) { - debug(47, 1) ("Done reading %s swaplog (%d entries)\n", - rb->sd->path, rb->n_read); - fclose(rb->log); - rb->log = NULL; - } else { - debug(47, 1) ("Done scanning %s (%d entries)\n", - rb->sd->path, rb->counts.scancount); - } - store_dirs_rebuilding--; - storeAufsDirCloseTmpSwapLog(rb->sd); - storeRebuildComplete(&rb->counts); - cbdataFree(rb); -} - -static void -storeAufsDirRebuildFromDirectory(void *data) -{ - RebuildState *rb = data; - SwapDir *SD = rb->sd; - LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE); - StoreEntry *e = NULL; - StoreEntry tmpe; - cache_key key[MD5_DIGEST_CHARS]; - sfileno filn = 0; - int count; - int size; - struct stat sb; - int swap_hdr_len; - int fd = -1; - tlv *tlv_list; - tlv *t; - assert(rb != NULL); - debug(47, 3) ("storeAufsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index); - for (count = 0; count < rb->speed; count++) { - assert(fd == -1); - fd = storeAufsDirGetNextFile(rb, &filn, &size); - if (fd == -2) { - storeAufsDirRebuildComplete(rb); - return; - } else if (fd < 0) { - continue; - } - assert(fd > -1); - /* lets get file stats here */ - if (fstat(fd, &sb) < 0) { - debug(47, 1) ("storeAufsDirRebuildFromDirectory: fstat(FD %d): %s\n", - fd, xstrerror()); - file_close(fd); - store_open_disk_fd--; - fd = -1; - continue; - } - if ((++rb->counts.scancount & 0xFFFF) == 0) - debug(47, 3) (" %s %7d files opened so far.\n", - rb->sd->path, rb->counts.scancount); - debug(47, 9) ("file_in: fd=%d %08X\n", fd, filn); - statCounter.syscalls.disk.reads++; - if (FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE) < 0) { - debug(47, 1) ("storeAufsDirRebuildFromDirectory: read(FD %d): %s\n", - fd, xstrerror()); - file_close(fd); - store_open_disk_fd--; - fd = -1; - continue; - } - file_close(fd); - store_open_disk_fd--; - fd = -1; - swap_hdr_len = 0; -#if USE_TRUNCATE - if (sb.st_size == 0) - continue; -#endif - tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); - if (tlv_list == NULL) { - debug(47, 1) ("storeAufsDirRebuildFromDirectory: failed to get meta data\n"); - /* XXX shouldn't this be a call to storeAufsUnlink ? */ - storeAufsDirUnlinkFile(SD, filn); - continue; - } - debug(47, 3) ("storeAufsDirRebuildFromDirectory: successful swap meta unpacking\n"); - memset(key, '\0', MD5_DIGEST_CHARS); - memset(&tmpe, '\0', sizeof(StoreEntry)); - for (t = tlv_list; t; t = t->next) { - switch (t->type) { - case STORE_META_KEY: - assert(t->length == MD5_DIGEST_CHARS); - xmemcpy(key, t->value, MD5_DIGEST_CHARS); - break; -#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T - case STORE_META_STD: - assert(t->length == STORE_HDR_METASIZE); - xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); - break; -#else - case STORE_META_STD_LFS: - assert(t->length == STORE_HDR_METASIZE); - xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); - break; - case STORE_META_STD: - assert(t->length == STORE_HDR_METASIZE_OLD); - { - struct { - time_t timestamp; - time_t lastref; - time_t expires; - time_t lastmod; - size_t swap_file_sz; - u_short refcount; - u_short flags; - } *tmp = t->value; - assert(sizeof(*tmp) == STORE_HDR_METASIZE_OLD); - tmpe.timestamp = tmp->timestamp; - tmpe.lastref = tmp->lastref; - tmpe.expires = tmp->expires; - tmpe.lastmod = tmp->lastmod; - tmpe.swap_file_sz = tmp->swap_file_sz; - tmpe.refcount = tmp->refcount; - tmpe.flags = tmp->flags; - } - break; -#endif - default: - break; - } - } - storeSwapTLVFree(tlv_list); - tlv_list = NULL; - if (storeKeyNull(key)) { - debug(47, 1) ("storeAufsDirRebuildFromDirectory: NULL key\n"); - storeAufsDirUnlinkFile(SD, filn); - continue; - } - tmpe.hash.key = key; - /* check sizes */ - if (tmpe.swap_file_sz == 0) { - tmpe.swap_file_sz = sb.st_size; - } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { - tmpe.swap_file_sz = sb.st_size; - } else if (tmpe.swap_file_sz != sb.st_size) { - debug(47, 1) ("storeAufsDirRebuildFromDirectory: SIZE MISMATCH %ld!=%ld\n", - (long int) tmpe.swap_file_sz, (long int) sb.st_size); - storeAufsDirUnlinkFile(SD, filn); - continue; - } - if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { - storeAufsDirUnlinkFile(SD, filn); - rb->counts.badflags++; - continue; - } - e = storeGet(key); - if (e && e->lastref >= tmpe.lastref) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - rb->counts.dupcount++; - continue; - } else if (NULL != e) { - /* URL already exists, this swapfile not being used */ - /* junk old, load new */ - storeRelease(e); /* release old entry */ - rb->counts.dupcount++; - } - rb->counts.objcount++; - storeEntryDump(&tmpe, 5); - e = storeAufsDirAddDiskRestore(SD, key, - filn, - tmpe.swap_file_sz, - tmpe.expires, - tmpe.timestamp, - tmpe.lastref, - tmpe.lastmod, - tmpe.refcount, /* refcount */ - tmpe.flags, /* flags */ - (int) rb->flags.clean); - storeDirSwapLog(e, SWAP_LOG_ADD); - } - eventAdd("storeRebuild", storeAufsDirRebuildFromDirectory, rb, 0.0, 1); -} - -static void -storeAufsDirRebuildFromSwapLog(void *data) -{ - RebuildState *rb = data; - SwapDir *SD = rb->sd; - StoreEntry *e = NULL; - storeSwapLogData s; - size_t ss = sizeof(storeSwapLogData); - int count; - int used; /* is swapfile already in use? */ - int disk_entry_newer; /* is the log entry newer than current entry? */ - double x; - assert(rb != NULL); - /* load a number of objects per invocation */ - for (count = 0; count < rb->speed; count++) { - if (fread(&s, ss, 1, rb->log) != 1) { - storeAufsDirRebuildComplete(rb); - return; - } - rb->n_read++; - /* - * BC: during 2.4 development, we changed the way swap file - * numbers are assigned and stored. The high 16 bits used - * to encode the SD index number. There used to be a call - * to storeDirProperFileno here that re-assigned the index - * bits. Now, for backwards compatibility, we just need - * to mask it off. - */ - s.swap_filen &= 0x00FFFFFF; - debug(47, 3) ("storeAufsDirRebuildFromSwapLog: %s %s %08X\n", - swap_log_op_str[(int) s.op], - storeKeyText(s.key), - s.swap_filen); - if (s.op == SWAP_LOG_ADD) { - (void) 0; - } else if (s.op == SWAP_LOG_DEL) { - /* Delete unless we already have a newer copy */ - if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) { - /* - * Make sure we don't unlink the file, it might be - * in use by a subsequent entry. Also note that - * we don't have to subtract from store_swap_size - * because adding to store_swap_size happens in - * the cleanup procedure. - */ - storeRecycle(e); - rb->counts.objcount--; - rb->counts.cancelcount++; - } - continue; - } else { - x = log(++rb->counts.bad_log_op) / log(10.0); - if (0.0 == x - (double) (int) x) - debug(47, 1) ("WARNING: %d invalid swap log entries found\n", - rb->counts.bad_log_op); - rb->counts.invalid++; - continue; - } - if ((++rb->counts.scancount & 0xFFF) == 0) { - struct stat sb; - if (0 == fstat(fileno(rb->log), &sb)) - storeRebuildProgress(SD->index, - (int) sb.st_size / ss, rb->n_read); - } - if (!storeAufsDirValidFileno(SD, s.swap_filen, 0)) { - rb->counts.invalid++; - continue; - } - if (EBIT_TEST(s.flags, KEY_PRIVATE)) { - rb->counts.badflags++; - continue; - } - e = storeGet(s.key); - used = storeAufsDirMapBitTest(SD, s.swap_filen); - /* If this URL already exists in the cache, does the swap log - * appear to have a newer entry? Compare 'lastref' from the - * swap log to e->lastref. */ - disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; - if (used && !disk_entry_newer) { - /* log entry is old, ignore it */ - rb->counts.clashcount++; - continue; - } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) { - /* swapfile taken, same URL, newer, update meta */ - if (e->store_status == STORE_OK) { - e->lastref = s.timestamp; - e->timestamp = s.timestamp; - e->expires = s.expires; - e->lastmod = s.lastmod; - e->flags = s.flags; - e->refcount += s.refcount; - storeAufsDirUnrefObj(SD, e); - } else { - debug_trap("storeAufsDirRebuildFromSwapLog: bad condition"); - debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); - } - continue; - } else if (used) { - /* swapfile in use, not by this URL, log entry is newer */ - /* This is sorta bad: the log entry should NOT be newer at this - * point. If the log is dirty, the filesize check should have - * caught this. If the log is clean, there should never be a - * newer entry. */ - debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", - SD->index, s.swap_filen); - /* I'm tempted to remove the swapfile here just to be safe, - * but there is a bad race condition in the NOVM version if - * the swapfile has recently been opened for writing, but - * not yet opened for reading. Because we can't map - * swapfiles back to StoreEntrys, we don't know the state - * of the entry using that file. */ - /* We'll assume the existing entry is valid, probably because - * were in a slow rebuild and the the swap file number got taken - * and the validation procedure hasn't run. */ - assert(rb->flags.need_to_validate); - rb->counts.clashcount++; - continue; - } else if (e && !disk_entry_newer) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - rb->counts.dupcount++; - continue; - } else if (e) { - /* key already exists, this swapfile not being used */ - /* junk old, load new */ - storeRecycle(e); - rb->counts.dupcount++; - } else { - /* URL doesnt exist, swapfile not in use */ - /* load new */ - (void) 0; - } - /* update store_swap_size */ - rb->counts.objcount++; - e = storeAufsDirAddDiskRestore(SD, s.key, - s.swap_filen, - s.swap_file_sz, - s.expires, - s.timestamp, - s.lastref, - s.lastmod, - s.refcount, - s.flags, - (int) rb->flags.clean); - storeDirSwapLog(e, SWAP_LOG_ADD); - } - eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1); -} - -#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T -/* This is an exact copy of the above, but using storeSwapLogDataOld entry type */ -static void -storeAufsDirRebuildFromSwapLogOld(void *data) -{ - RebuildState *rb = data; - SwapDir *SD = rb->sd; - StoreEntry *e = NULL; - storeSwapLogDataOld s; - size_t ss = sizeof(storeSwapLogDataOld); - int count; - int used; /* is swapfile already in use? */ - int disk_entry_newer; /* is the log entry newer than current entry? */ - double x; - assert(rb != NULL); - /* load a number of objects per invocation */ - for (count = 0; count < rb->speed; count++) { - if (fread(&s, ss, 1, rb->log) != 1) { - storeAufsDirRebuildComplete(rb); - return; - } - rb->n_read++; - /* - * BC: during 2.4 development, we changed the way swap file - * numbers are assigned and stored. The high 16 bits used - * to encode the SD index number. There used to be a call - * to storeDirProperFileno here that re-assigned the index - * bits. Now, for backwards compatibility, we just need - * to mask it off. - */ - s.swap_filen &= 0x00FFFFFF; - debug(47, 3) ("storeAufsDirRebuildFromSwapLog: %s %s %08X\n", - swap_log_op_str[(int) s.op], - storeKeyText(s.key), - s.swap_filen); - if (s.op == SWAP_LOG_ADD) { - (void) 0; - } else if (s.op == SWAP_LOG_DEL) { - /* Delete unless we already have a newer copy */ - if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) { - /* - * Make sure we don't unlink the file, it might be - * in use by a subsequent entry. Also note that - * we don't have to subtract from store_swap_size - * because adding to store_swap_size happens in - * the cleanup procedure. - */ - storeRecycle(e); - rb->counts.objcount--; - rb->counts.cancelcount++; - } - continue; - } else { - x = log(++rb->counts.bad_log_op) / log(10.0); - if (0.0 == x - (double) (int) x) - debug(47, 1) ("WARNING: %d invalid swap log entries found\n", - rb->counts.bad_log_op); - rb->counts.invalid++; - continue; - } - if ((++rb->counts.scancount & 0xFFF) == 0) { - struct stat sb; - if (0 == fstat(fileno(rb->log), &sb)) - storeRebuildProgress(SD->index, - (int) sb.st_size / ss, rb->n_read); - } - if (!storeAufsDirValidFileno(SD, s.swap_filen, 0)) { - rb->counts.invalid++; - continue; - } - if (EBIT_TEST(s.flags, KEY_PRIVATE)) { - rb->counts.badflags++; - continue; - } - e = storeGet(s.key); - used = storeAufsDirMapBitTest(SD, s.swap_filen); - /* If this URL already exists in the cache, does the swap log - * appear to have a newer entry? Compare 'lastref' from the - * swap log to e->lastref. */ - disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; - if (used && !disk_entry_newer) { - /* log entry is old, ignore it */ - rb->counts.clashcount++; - continue; - } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) { - /* swapfile taken, same URL, newer, update meta */ - if (e->store_status == STORE_OK) { - e->lastref = s.timestamp; - e->timestamp = s.timestamp; - e->expires = s.expires; - e->lastmod = s.lastmod; - e->flags = s.flags; - e->refcount += s.refcount; - storeAufsDirUnrefObj(SD, e); - } else { - debug_trap("storeAufsDirRebuildFromSwapLog: bad condition"); - debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); - } - continue; - } else if (used) { - /* swapfile in use, not by this URL, log entry is newer */ - /* This is sorta bad: the log entry should NOT be newer at this - * point. If the log is dirty, the filesize check should have - * caught this. If the log is clean, there should never be a - * newer entry. */ - debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", - SD->index, s.swap_filen); - /* I'm tempted to remove the swapfile here just to be safe, - * but there is a bad race condition in the NOVM version if - * the swapfile has recently been opened for writing, but - * not yet opened for reading. Because we can't map - * swapfiles back to StoreEntrys, we don't know the state - * of the entry using that file. */ - /* We'll assume the existing entry is valid, probably because - * were in a slow rebuild and the the swap file number got taken - * and the validation procedure hasn't run. */ - assert(rb->flags.need_to_validate); - rb->counts.clashcount++; - continue; - } else if (e && !disk_entry_newer) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - rb->counts.dupcount++; - continue; - } else if (e) { - /* key already exists, this swapfile not being used */ - /* junk old, load new */ - storeRecycle(e); - rb->counts.dupcount++; - } else { - /* URL doesnt exist, swapfile not in use */ - /* load new */ - (void) 0; - } - /* update store_swap_size */ - rb->counts.objcount++; - e = storeAufsDirAddDiskRestore(SD, s.key, - s.swap_filen, - s.swap_file_sz, - s.expires, - s.timestamp, - s.lastref, - s.lastmod, - s.refcount, - s.flags, - (int) rb->flags.clean); - storeDirSwapLog(e, SWAP_LOG_ADD); - } - eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1); -} - -#endif - -static void -storeAufsDirRebuildFromSwapLogCheckVersion(void *data) -{ - RebuildState *rb = data; - storeSwapLogHeader hdr; - - if (fread(&hdr, sizeof(hdr), 1, rb->log) != 1) { - storeAufsDirRebuildComplete(rb); - return; - } - if (hdr.op == SWAP_LOG_VERSION) { - if (fseek(rb->log, hdr.record_size, SEEK_SET) != 0) { - storeAufsDirRebuildComplete(rb); - return; - } - if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogData)) { - eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1); - return; - } -#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T - if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogDataOld)) { - debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Found current version but without large file support. Upgrading\n"); - eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1); - return; - } -#endif - debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Unsupported swap.state version %d size %d\n", - hdr.version, hdr.record_size); - storeAufsDirRebuildComplete(rb); - return; - } - rewind(rb->log); - debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Old version detected. Upgrading\n"); -#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T - eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1); -#else - eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1); -#endif -} - -static int -storeAufsDirGetNextFile(RebuildState * rb, sfileno * filn_p, int *size) -{ - SwapDir *SD = rb->sd; - squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata; - int fd = -1; - int used = 0; - int dirs_opened = 0; - debug(47, 3) ("storeAufsDirGetNextFile: flag=%d, %d: /%02X/%02X\n", - rb->flags.init, - rb->sd->index, - rb->curlvl1, - rb->curlvl2); - if (rb->done) - return -2; - while (fd < 0 && rb->done == 0) { - fd = -1; - if (0 == rb->flags.init) { /* initialize, open first file */ - rb->done = 0; - rb->curlvl1 = 0; - rb->curlvl2 = 0; - rb->in_dir = 0; - rb->flags.init = 1; - assert(Config.cacheSwap.n_configured > 0); - } - if (0 == rb->in_dir) { /* we need to read in a new directory */ - snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", - rb->sd->path, - rb->curlvl1, rb->curlvl2); - if (dirs_opened) - return -1; - rb->td = opendir(rb->fullpath); - dirs_opened++; - if (rb->td == NULL) { - debug(47, 1) ("storeAufsDirGetNextFile: opendir: %s: %s\n", - rb->fullpath, xstrerror()); - } else { - rb->entry = readdir(rb->td); /* skip . and .. */ - rb->entry = readdir(rb->td); - if (rb->entry == NULL && errno == ENOENT) - debug(47, 1) ("storeAufsDirGetNextFile: directory does not exist!.\n"); - debug(47, 3) ("storeAufsDirGetNextFile: Directory %s\n", rb->fullpath); - } - } - if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) { - rb->in_dir++; - if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) { - debug(47, 3) ("storeAufsDirGetNextFile: invalid %s\n", - rb->entry->d_name); - continue; - } - if (!storeAufsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) { - debug(47, 3) ("storeAufsDirGetNextFile: %08X does not belong in %d/%d/%d\n", - rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2); - continue; - } - used = storeAufsDirMapBitTest(SD, rb->fn); - if (used) { - debug(47, 3) ("storeAufsDirGetNextFile: Locked, continuing with next.\n"); - continue; - } - snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s", - rb->fullpath, rb->entry->d_name); - debug(47, 3) ("storeAufsDirGetNextFile: Opening %s\n", rb->fullfilename); - fd = file_open(rb->fullfilename, O_RDONLY | O_BINARY); - if (fd < 0) - debug(47, 1) ("storeAufsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror()); - else - store_open_disk_fd++; - continue; - } - if (rb->td != NULL) - closedir(rb->td); - rb->td = NULL; - rb->in_dir = 0; - if (++rb->curlvl2 < aioinfo->l2) - continue; - rb->curlvl2 = 0; - if (++rb->curlvl1 < aioinfo->l1) - continue; - rb->curlvl1 = 0; - rb->done = 1; - } - *filn_p = rb->fn; - return fd; -} - -/* Add a new object to the cache with empty memory copy and pointer to disk - * use to rebuild store from disk. */ -static StoreEntry * -storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key, - sfileno file_number, - squid_file_sz swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_num32 refcount, - u_short flags, - int clean) -{ - StoreEntry *e = NULL; - debug(47, 5) ("storeAufsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); - /* if you call this you'd better be sure file_number is not - * already in use! */ - e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); - e->store_status = STORE_OK; - storeSetMemStatus(e, NOT_IN_MEMORY); - e->swap_status = SWAPOUT_DONE; - e->swap_filen = file_number; - e->swap_dirn = SD->index; - e->swap_file_sz = swap_file_sz; - e->lock_count = 0; - e->lastref = lastref; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->refcount = refcount; - e->flags = flags; - EBIT_SET(e->flags, ENTRY_CACHABLE); - EBIT_CLR(e->flags, RELEASE_REQUEST); - EBIT_CLR(e->flags, KEY_PRIVATE); - e->ping_status = PING_NONE; - EBIT_CLR(e->flags, ENTRY_VALIDATED); - storeAufsDirMapBitSet(SD, e->swap_filen); - storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ - storeAufsDirReplAdd(SD, e); - return e; -} - -CBDATA_TYPE(RebuildState); - -static void -storeAufsDirRebuild(SwapDir * sd) -{ - RebuildState *rb; - int clean = 0; - int zero = 0; - FILE *fp; - EVH *func = NULL; - CBDATA_INIT_TYPE(RebuildState); - rb = cbdataAlloc(RebuildState); - rb->sd = sd; - rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; - /* - * If the swap.state file exists in the cache_dir, then - * we'll use storeAufsDirRebuildFromSwapLog(), otherwise we'll - * use storeAufsDirRebuildFromDirectory() to open up each file - * and suck in the meta data. - */ - fp = storeAufsDirOpenTmpSwapLog(sd, &clean, &zero); - if (fp == NULL || zero) { - if (fp != NULL) - fclose(fp); - func = storeAufsDirRebuildFromDirectory; - } else { - func = storeAufsDirRebuildFromSwapLogCheckVersion; - rb->log = fp; - rb->flags.clean = (unsigned int) clean; - } - if (!clean) - rb->flags.need_to_validate = 1; - debug(47, 1) ("Rebuilding storage in %s (%s)\n", - sd->path, clean ? "CLEAN" : "DIRTY"); - store_dirs_rebuilding++; - eventAdd("storeRebuild", func, rb, 0.0, 1); -} - -static void -storeAufsDirCloseTmpSwapLog(SwapDir * sd) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); - char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new")); - int fd; - file_close(aioinfo->swaplog_fd); - if (xrename(new_path, swaplog_path) < 0) { - fatal("storeAufsDirCloseTmpSwapLog: rename failed"); - } - fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); - if (fd < 0) { - debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); - fatal("storeAufsDirCloseTmpSwapLog: Failed to open swap log."); - } - safe_free(swaplog_path); - safe_free(new_path); - aioinfo->swaplog_fd = fd; - debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); -} - -static void -storeSwapLogDataFree(void *s) -{ - memFree(s, MEM_SWAP_LOG_DATA); -} - -static void -storeAufsWriteSwapLogheader(int fd) -{ - storeSwapLogHeader *hdr = memAllocate(MEM_SWAP_LOG_DATA); - hdr->op = SWAP_LOG_VERSION; - hdr->version = 1; - hdr->record_size = sizeof(storeSwapLogData); - /* The header size is a full log record to keep some level of backward - * compatibility even if the actual header is smaller - */ - file_write(fd, - -1, - hdr, - sizeof(storeSwapLogData), - NULL, - NULL, - (FREE *) storeSwapLogDataFree); -} - -static FILE * -storeAufsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); - char *clean_path = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean")); - char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new")); - struct stat log_sb; - struct stat clean_sb; - FILE *fp; - int fd; - if (stat(swaplog_path, &log_sb) < 0) { - debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index); - safe_free(swaplog_path); - safe_free(clean_path); - safe_free(new_path); - return NULL; - } - *zero_flag = log_sb.st_size == 0 ? 1 : 0; - /* close the existing write-only FD */ - if (aioinfo->swaplog_fd >= 0) - file_close(aioinfo->swaplog_fd); - /* open a write-only FD for the new log */ - fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - if (fd < 0) { - debug(50, 1) ("%s: %s\n", new_path, xstrerror()); - fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); - } - aioinfo->swaplog_fd = fd; - storeAufsWriteSwapLogheader(fd); - /* open a read-only stream of the old log */ - fp = fopen(swaplog_path, "rb"); - if (fp == NULL) { - debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); - fatal("Failed to open swap log for reading"); - } - memset(&clean_sb, '\0', sizeof(struct stat)); - if (stat(clean_path, &clean_sb) < 0) - *clean_flag = 0; - else if (clean_sb.st_mtime < log_sb.st_mtime) - *clean_flag = 0; - else - *clean_flag = 1; - safeunlink(clean_path, 1); - safe_free(swaplog_path); - safe_free(clean_path); - safe_free(new_path); - return fp; -} - -struct _clean_state { - char *cur; - char *new; - char *cln; - char *outbuf; - int outbuf_offset; - int fd; - RemovalPolicyWalker *walker; -}; - -#define CLEAN_BUF_SZ 16384 -/* - * Begin the process to write clean cache state. For AUFS this means - * opening some log files and allocating write buffers. Return 0 if - * we succeed, and assign the 'func' and 'data' return pointers. - */ -static int -storeAufsDirWriteCleanStart(SwapDir * sd) -{ - struct _clean_state *state = xcalloc(1, sizeof(*state)); -#if HAVE_FCHMOD - struct stat sb; -#endif - sd->log.clean.write = NULL; - sd->log.clean.state = NULL; - state->new = xstrdup(storeAufsDirSwapLogFile(sd, ".clean")); - state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - if (state->fd < 0) { - debug(50, 0) ("storeDirWriteCleanStart: %s: open: %s\n", - state->new, xstrerror()); - debug(50, 0) ("storeDirWriteCleanStart: Current swap logfile " - "not replaced.\n"); - xfree(state->new); - xfree(state); - return -1; - } - storeAufsWriteSwapLogheader(state->fd); - state->cur = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); - state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean")); - state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); - state->outbuf_offset = 0; - state->walker = sd->repl->WalkInit(sd->repl); - unlink(state->cln); - debug(47, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n", - state->new, state->fd); -#if HAVE_FCHMOD - if (stat(state->cur, &sb) == 0) - fchmod(state->fd, sb.st_mode); -#endif - sd->log.clean.write = storeAufsDirWriteCleanEntry; - sd->log.clean.state = state; - return 0; -} - -/* - * Get the next entry that is a candidate for clean log writing - */ -const StoreEntry * -storeAufsDirCleanLogNextEntry(SwapDir * sd) -{ - const StoreEntry *entry = NULL; - struct _clean_state *state = sd->log.clean.state; - if (state->walker) - entry = state->walker->Next(state->walker); - return entry; -} - -/* - * "write" an entry to the clean log file. - */ -static void -storeAufsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e) -{ - storeSwapLogData s; - static size_t ss = sizeof(storeSwapLogData); - struct _clean_state *state = sd->log.clean.state; - memset(&s, '\0', ss); - s.op = (char) SWAP_LOG_ADD; - s.swap_filen = e->swap_filen; - s.timestamp = e->timestamp; - s.lastref = e->lastref; - s.expires = e->expires; - s.lastmod = e->lastmod; - s.swap_file_sz = e->swap_file_sz; - s.refcount = e->refcount; - s.flags = e->flags; - xmemcpy(&s.key, e->hash.key, MD5_DIGEST_CHARS); - xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); - state->outbuf_offset += ss; - /* buffered write */ - if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { - if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", - state->new, xstrerror()); - debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); - file_close(state->fd); - state->fd = -1; - unlink(state->new); - safe_free(state); - sd->log.clean.state = NULL; - sd->log.clean.write = NULL; - return; - } - state->outbuf_offset = 0; - } -} - -static void -storeAufsDirWriteCleanDone(SwapDir * sd) -{ - int fd; - struct _clean_state *state = sd->log.clean.state; - if (NULL == state) - return; - if (state->fd < 0) - return; - state->walker->Done(state->walker); - if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", - state->new, xstrerror()); - debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile " - "not replaced.\n"); - file_close(state->fd); - state->fd = -1; - unlink(state->new); - } - safe_free(state->outbuf); - /* - * You can't rename open files on Microsoft "operating systems" - * so we have to close before renaming. - */ - storeAufsDirCloseSwapLog(sd); - /* save the fd value for a later test */ - fd = state->fd; - /* rename */ - if (state->fd >= 0) { -#if defined(_SQUID_OS2_) || defined(_SQUID_WIN32_) - file_close(state->fd); - state->fd = -1; -#endif - xrename(state->new, state->cur); - } - /* touch a timestamp file if we're not still validating */ - if (store_dirs_rebuilding) - (void) 0; - else if (fd < 0) - (void) 0; - else - file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); - /* close */ - safe_free(state->cur); - safe_free(state->new); - safe_free(state->cln); - if (state->fd >= 0) - file_close(state->fd); - state->fd = -1; - safe_free(state); - sd->log.clean.state = NULL; - sd->log.clean.write = NULL; -} - -static void -storeAufsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata; - storeSwapLogData *s = memAllocate(MEM_SWAP_LOG_DATA); - s->op = (char) op; - s->swap_filen = e->swap_filen; - s->timestamp = e->timestamp; - s->lastref = e->lastref; - s->expires = e->expires; - s->lastmod = e->lastmod; - s->swap_file_sz = e->swap_file_sz; - s->refcount = e->refcount; - s->flags = e->flags; - xmemcpy(s->key, e->hash.key, MD5_DIGEST_CHARS); - file_write(aioinfo->swaplog_fd, - -1, - s, - sizeof(storeSwapLogData), - NULL, - NULL, - (FREE *) storeSwapLogDataFree); -} - -static void -storeAufsDirNewfs(SwapDir * sd) -{ - debug(47, 3) ("Creating swap space in %s\n", sd->path); - storeAufsDirCreateDirectory(sd->path, 0); - storeAufsDirCreateSwapSubDirs(sd); -} - -static int -rev_int_sort(const void *A, const void *B) -{ - const int *i1 = A; - const int *i2 = B; - return *i2 - *i1; -} - -static int -storeAufsDirClean(int swap_index) -{ - DIR *dp = NULL; - struct dirent *de = NULL; - LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); - LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); -#if USE_TRUNCATE - struct stat sb; -#endif - int files[20]; - int swapfileno; - int fn; /* same as swapfileno, but with dirn bits set */ - int n = 0; - int k = 0; - int N0, N1, N2; - int D0, D1, D2; - SwapDir *SD; - squidaioinfo_t *aioinfo; - N0 = n_asyncufs_dirs; - D0 = asyncufs_dir_index[swap_index % N0]; - SD = &Config.cacheSwap.swapDirs[D0]; - aioinfo = (squidaioinfo_t *) SD->fsdata; - N1 = aioinfo->l1; - D1 = (swap_index / N0) % N1; - N2 = aioinfo->l2; - D2 = ((swap_index / N0) / N1) % N2; - snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X", - Config.cacheSwap.swapDirs[D0].path, D1, D2); - debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1); - dp = opendir(p1); - if (dp == NULL) { - if (errno == ENOENT) { - debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1); -#ifdef _SQUID_MSWIN_ - if (mkdir(p1) == 0) -#else - if (mkdir(p1, 0777) == 0) -#endif - return 0; - } - debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror()); - safeunlink(p1, 1); - return 0; - } - while ((de = readdir(dp)) != NULL && k < 20) { - if (sscanf(de->d_name, "%X", &swapfileno) != 1) - continue; - fn = swapfileno; /* XXX should remove this cruft ! */ - if (storeAufsDirValidFileno(SD, fn, 1)) - if (storeAufsDirMapBitTest(SD, fn)) - if (storeAufsFilenoBelongsHere(fn, D0, D1, D2)) - continue; -#if USE_TRUNCATE - if (!stat(de->d_name, &sb)) - if (sb.st_size == 0) - continue; -#endif - files[k++] = swapfileno; - } - closedir(dp); - if (k == 0) - return 0; - qsort(files, k, sizeof(int), rev_int_sort); - if (k > 10) - k = 10; - for (n = 0; n < k; n++) { - debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]); - snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); -#if USE_TRUNCATE - truncate(p2, 0); -#else - safeunlink(p2, 0); -#endif - statCounter.swap.files_cleaned++; - } - debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1); - return k; -} - -static void -storeAufsDirCleanEvent(void *unused) -{ - static int swap_index = -1; - int j = 0; - int n = 0; - /* - * Assert that there are AUFS cache_dirs configured, otherwise - * we should never be called. - */ - if (swap_index == -1) { - SwapDir *sd; - squidaioinfo_t *aioinfo; - /* - * Start the storeAufsDirClean() swap_index with a random - * value. j equals the total number of AUFS level 2 - * swap directories - */ - for (n = 0; n < n_asyncufs_dirs; n++) { - sd = &Config.cacheSwap.swapDirs[asyncufs_dir_index[n]]; - aioinfo = (squidaioinfo_t *) sd->fsdata; - j += (aioinfo->l1 * aioinfo->l2); - } - swap_index = (int) (squid_random() % j); - } - if (0 == store_dirs_rebuilding) { - n = storeAufsDirClean(swap_index); - swap_index++; - if (swap_index < 0) - swap_index = 0; - } - eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL, - 15.0 * exp(-0.25 * n), 1); -} - -/* - * Does swapfile number 'fn' belong in cachedir #F0, - * level1 dir #F1, level2 dir #F2? - */ -static int -storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2) -{ - int D1, D2; - int L1, L2; - int filn = fn; - squidaioinfo_t *aioinfo; - assert(F0 < Config.cacheSwap.n_configured); - aioinfo = (squidaioinfo_t *) Config.cacheSwap.swapDirs[F0].fsdata; - L1 = aioinfo->l1; - L2 = aioinfo->l2; - D1 = ((filn / L2) / L2) % L1; - if (F1 != D1) - return 0; - D2 = (filn / L2) % L2; - if (F2 != D2) - return 0; - return 1; -} - -int -storeAufsDirValidFileno(SwapDir * SD, sfileno filn, int flag) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata; - if (filn < 0) - return 0; - /* - * If flag is set it means out-of-range file number should - * be considered invalid. - */ - if (flag) - if (filn > aioinfo->map->max_n_files) - return 0; - return 1; -} - -void -storeAufsDirMaintain(SwapDir * SD) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata; - StoreEntry *e = NULL; - int removed = 0; - int max_scan; - int max_remove; - double f; - RemovalPurgeWalker *walker; - /* We can't delete objects while rebuilding swap */ - if (store_dirs_rebuilding) { - return; - } else { - f = (double) (SD->cur_size - SD->low_size) / (SD->max_size - SD->low_size); - f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; - max_scan = (int) (f * 400.0 + 100.0); - max_remove = (int) (f * 70.0 + 10.0); - /* - * This is kinda cheap, but so we need this priority hack? - */ - } - debug(47, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", - f, max_scan, max_remove); - walker = SD->repl->PurgeInit(SD->repl, max_scan); - while (1) { - if (SD->cur_size < SD->low_size && aioinfo->map->n_files_in_map < FILEMAP_MAX) - break; - if (removed >= max_remove) - break; - e = walker->Next(walker); - if (!e) - break; /* no more objects */ - removed++; - storeRelease(e); - } - walker->Done(walker); - debug(47, (removed ? 2 : 3)) ("storeAufsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n", - SD->path, removed, max_remove, f, max_scan); -} - -/* - * storeAufsDirCheckObj - * - * This routine is called by storeDirSelectSwapDir to see if the given - * object is able to be stored on this filesystem. AUFS filesystems will - * happily store anything as long as the LRU time isn't too small. - */ -int -storeAufsDirCheckObj(SwapDir * SD, const StoreEntry * e) -{ - return 1; -} - -int -storeAufsDirCheckLoadAv(SwapDir * SD, store_op_t op) -{ - int loadav, ql; - - ql = aioQueueSize(); - if (ql == 0) { - return AUFS_LOAD_BASE; - } - loadav = AUFS_LOAD_BASE + (ql * AUFS_LOAD_QUEUE_WEIGHT / MAGIC1); - return loadav; -} - -/* - * storeAufsDirRefObj - * - * This routine is called whenever an object is referenced, so we can - * maintain replacement information within the storage fs. - */ -void -storeAufsDirRefObj(SwapDir * SD, StoreEntry * e) -{ - debug(47, 3) ("storeAufsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, - e->swap_filen); - if (SD->repl->Referenced) - SD->repl->Referenced(SD->repl, e, &e->repl); -} - -/* - * storeAufsDirUnrefObj - * This routine is called whenever the last reference to an object is - * removed, to maintain replacement information within the storage fs. - */ -void -storeAufsDirUnrefObj(SwapDir * SD, StoreEntry * e) -{ - debug(47, 3) ("storeAufsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn, - e->swap_filen); - if (SD->repl->Dereferenced) - SD->repl->Dereferenced(SD->repl, e, &e->repl); -} - -/* - * storeAufsDirUnlinkFile - * - * This routine unlinks a file and pulls it out of the bitmap. - * It used to be in storeAufsUnlink(), however an interface change - * forced this bit of code here. Eeek. - */ -void -storeAufsDirUnlinkFile(SwapDir * SD, sfileno f) -{ - debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f); - /* storeAufsDirMapBitReset(SD, f); */ -#if USE_TRUNCATE_NOT_UNLINK - aioTruncate(storeAufsDirFullPath(SD, f, NULL), NULL, NULL); -#else - aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL); -#endif -} - -/* - * Add and remove the given StoreEntry from the replacement policy in - * use. - */ - -void -storeAufsDirReplAdd(SwapDir * SD, StoreEntry * e) -{ - debug(47, 4) ("storeAufsDirReplAdd: added node %p to dir %d\n", e, - SD->index); - SD->repl->Add(SD->repl, e, &e->repl); -} - - -void -storeAufsDirReplRemove(StoreEntry * e) -{ - SwapDir *SD = INDEXSD(e->swap_dirn); - debug(47, 4) ("storeAufsDirReplRemove: remove node %p from dir %d\n", e, - SD->index); - SD->repl->Remove(SD->repl, e, &e->repl); -} - - - -/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ - -void -storeAufsDirStats(SwapDir * SD, StoreEntry * sentry) -{ - squidaioinfo_t *aioinfo = SD->fsdata; - int totl_kb = 0; - int free_kb = 0; - int totl_in = 0; - int free_in = 0; - int x; - storeAppendPrintf(sentry, "First level subdirectories: %d\n", aioinfo->l1); - storeAppendPrintf(sentry, "Second level subdirectories: %d\n", aioinfo->l2); - storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); - storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); - storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", - 100.0 * SD->cur_size / SD->max_size); - storeAppendPrintf(sentry, "Current load metric: %d / %d\n", storeAufsDirCheckLoadAv(SD, ST_OP_CREATE), MAX_LOAD_VALUE); - storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", - aioinfo->map->n_files_in_map, aioinfo->map->max_n_files, - percent(aioinfo->map->n_files_in_map, aioinfo->map->max_n_files)); - x = storeDirGetUFSStats(SD->path, &totl_kb, &free_kb, &totl_in, &free_in); - if (0 == x) { - storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", - totl_kb - free_kb, - totl_kb, - percent(totl_kb - free_kb, totl_kb)); - storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", - totl_in - free_in, - totl_in, - percent(totl_in - free_in, totl_in)); - } - storeAppendPrintf(sentry, "Flags:"); - if (SD->flags.selected) - storeAppendPrintf(sentry, " SELECTED"); - if (SD->flags.read_only) - storeAppendPrintf(sentry, " READ-ONLY"); - storeAppendPrintf(sentry, "\n"); -} - -static struct cache_dir_option options[] = -{ -#if NOT_YET_DONE - {"L1", storeAufsDirParseL1, storeAufsDirDumpL1}, - {"L2", storeAufsDirParseL2, storeAufsDirDumpL2}, -#endif - {NULL, NULL} -}; - -/* - * storeAufsDirReconfigure - * - * This routine is called when the given swapdir needs reconfiguring - */ -static void -storeAufsDirReconfigure(SwapDir * sd, int index, char *path) -{ - int i; - int size; - int l1; - int l2; - - i = GetInteger(); - size = i << 10; /* Mbytes to kbytes */ - if (size <= 0) - fatal("storeAufsDirReconfigure: invalid size value"); - i = GetInteger(); - l1 = i; - if (l1 <= 0) - fatal("storeAufsDirReconfigure: invalid level 1 directories value"); - i = GetInteger(); - l2 = i; - if (l2 <= 0) - fatal("storeAufsDirReconfigure: invalid level 2 directories value"); - - /* just reconfigure it */ - if (size == sd->max_size) - debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", - path, size); - else - debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", - path, size); - sd->max_size = size; - - parse_cachedir_options(sd, options, 0); - - return; -} - -void -storeAufsDirDump(StoreEntry * entry, SwapDir * s) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) s->fsdata; - storeAppendPrintf(entry, " %d %d %d", - s->max_size >> 10, - aioinfo->l1, - aioinfo->l2); - dump_cachedir_options(entry, options, s); -} - -/* - * Only "free" the filesystem specific stuff here - */ -static void -storeAufsDirFree(SwapDir * s) -{ - squidaioinfo_t *aioinfo = (squidaioinfo_t *) s->fsdata; - if (aioinfo->swaplog_fd > -1) { - file_close(aioinfo->swaplog_fd); - aioinfo->swaplog_fd = -1; - } - filemapFreeMemory(aioinfo->map); - xfree(aioinfo); - s->fsdata = NULL; /* Will aid debugging... */ -} - -char * -storeAufsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath) -{ - LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata; - int L1 = aioinfo->l1; - int L2 = aioinfo->l2; - if (!fullpath) - fullpath = fullfilename; - fullpath[0] = '\0'; - snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", - SD->path, - ((filn / L2) / L2) % L1, - (filn / L2) % L2, - filn); - return fullpath; -} - -/* - * storeAufsCleanupDoubleCheck - * - * This is called by storeCleanup() if -S was given on the command line. - */ -static int -storeAufsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e) -{ - struct stat sb; - if (stat(storeAufsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { - debug(47, 0) ("storeAufsCleanupDoubleCheck: MISSING SWAP FILE\n"); - debug(47, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(47, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", - storeAufsDirFullPath(sd, e->swap_filen, NULL)); - storeEntryDump(e, 0); - return -1; - } - if (e->swap_file_sz != sb.st_size) { - debug(47, 0) ("storeAufsCleanupDoubleCheck: SIZE MISMATCH\n"); - debug(47, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); - debug(47, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n", - storeAufsDirFullPath(sd, e->swap_filen, NULL)); - debug(47, 0) ("storeAufsCleanupDoubleCheck: ENTRY SIZE: %ld, FILE SIZE: %ld\n", - (long int) e->swap_file_sz, (long int) sb.st_size); - storeEntryDump(e, 0); - return -1; - } - return 0; -} - -/* - * storeAufsDirParse * - * Called when a *new* fs is being setup. - */ -static void -storeAufsDirParse(SwapDir * sd, int index, char *path) -{ - int i; - int size; - int l1; - int l2; - squidaioinfo_t *aioinfo; - - i = GetInteger(); - size = i << 10; /* Mbytes to kbytes */ - if (size <= 0) - fatal("storeAufsDirParse: invalid size value"); - i = GetInteger(); - l1 = i; - if (l1 <= 0) - fatal("storeAufsDirParse: invalid level 1 directories value"); - i = GetInteger(); - l2 = i; - if (l2 <= 0) - fatal("storeAufsDirParse: invalid level 2 directories value"); - - aioinfo = xmalloc(sizeof(squidaioinfo_t)); - if (aioinfo == NULL) - fatal("storeAufsDirParse: couldn't xmalloc() squidaioinfo_t!\n"); - - sd->index = index; - sd->path = xstrdup(path); - sd->max_size = size; - sd->fsdata = aioinfo; - aioinfo->l1 = l1; - aioinfo->l2 = l2; - aioinfo->swaplog_fd = -1; - aioinfo->map = NULL; /* Debugging purposes */ - aioinfo->suggest = 0; - sd->checkconfig = storeAufsCheckConfig; - sd->init = storeAufsDirInit; - sd->newfs = storeAufsDirNewfs; - sd->dump = storeAufsDirDump; - sd->freefs = storeAufsDirFree; - sd->dblcheck = storeAufsCleanupDoubleCheck; - sd->statfs = storeAufsDirStats; - sd->maintainfs = storeAufsDirMaintain; - sd->checkobj = storeAufsDirCheckObj; - sd->checkload = storeAufsDirCheckLoadAv; - sd->refobj = storeAufsDirRefObj; - sd->unrefobj = storeAufsDirUnrefObj; - sd->callback = aioCheckCallbacks; - sd->sync = aioSync; - sd->obj.create = storeAufsCreate; - sd->obj.open = storeAufsOpen; - sd->obj.close = storeAufsClose; - sd->obj.read = storeAufsRead; - sd->obj.write = storeAufsWrite; - sd->obj.unlink = storeAufsUnlink; - sd->obj.recycle = storeAufsRecycle; - sd->log.open = storeAufsDirOpenSwapLog; - sd->log.close = storeAufsDirCloseSwapLog; - sd->log.write = storeAufsDirSwapLog; - sd->log.clean.start = storeAufsDirWriteCleanStart; - sd->log.clean.nextentry = storeAufsDirCleanLogNextEntry; - sd->log.clean.done = storeAufsDirWriteCleanDone; - - parse_cachedir_options(sd, options, 0); - - /* Initialise replacement policy stuff */ - sd->repl = createRemovalPolicy(Config.replPolicy); - - asyncufs_dir_index = realloc(asyncufs_dir_index, (n_asyncufs_dirs + 1) * sizeof(*asyncufs_dir_index)); - asyncufs_dir_index[n_asyncufs_dirs++] = index; -} - -/* - * Initial setup / end destruction - */ -static void -storeAufsDirDone(void) -{ - aioDone(); - memPoolDestroy(squidaio_state_pool); - memPoolDestroy(aufs_qread_pool); - memPoolDestroy(aufs_qwrite_pool); - asyncufs_initialised = 0; -} - -void -storeFsSetup_awin32(storefs_entry_t * storefs) -{ - assert(!asyncufs_initialised); - storefs->parsefunc = storeAufsDirParse; - storefs->reconfigurefunc = storeAufsDirReconfigure; - storefs->donefunc = storeAufsDirDone; - squidaio_state_pool = memPoolCreate("AWIN32 IO State data", sizeof(squidaiostate_t)); - aufs_qread_pool = memPoolCreate("AWIN32 Queued read data", - sizeof(queued_read)); - aufs_qwrite_pool = memPoolCreate("AWIN32 Queued write data", - sizeof(queued_write)); - - asyncufs_initialised = 1; - aioInit(); -} --- squid/src/fs/awin32/store_io_aufs.c Wed Feb 14 01:17:06 2007 +++ /dev/null Wed Feb 14 01:16:58 2007 @@ -1,552 +0,0 @@ - -/* - * $Id: store_io_aufs.c,v 1.1.88.3 2006/07/05 16:34:12 serassio Exp $ - * - * DEBUG: section 79 Storage Manager awin32 Interface - * AUTHOR: Robert Collins - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "async_io.h" -#include "store_asyncufs.h" - -#if ASYNC_READ -static AIOCB storeAufsReadDone; -#else -static DRCB storeAufsReadDone; -#endif -#if ASYNC_WRITE -static AIOCB storeAufsWriteDone; -#else -static DWCB storeAufsWriteDone; -#endif -static void storeAufsIOCallback(storeIOState * sio, int errflag); -static AIOCB storeAufsOpenDone; -static int storeAufsNeedCompletetion(storeIOState *); -static int storeAufsKickWriteQueue(storeIOState * sio); -static CBDUNL storeAufsIOFreeEntry; - -CBDATA_TYPE(storeIOState); - -/* === PUBLIC =========================================================== */ - -/* open for reading */ -storeIOState * -storeAufsOpen(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, - STIOCB * callback, void *callback_data) -{ - sfileno f = e->swap_filen; - char *path = storeAufsDirFullPath(SD, f, NULL); - storeIOState *sio; -#if !ASYNC_OPEN - int fd; -#endif - debug(79, 3) ("storeAufsOpen: fileno %08X\n", f); - /* - * we should detect some 'too many files open' condition and return - * NULL here. - */ -#ifdef MAGIC2 - if (aioQueueSize() > MAGIC2) - return NULL; -#endif -#if !ASYNC_OPEN - fd = file_open(path, O_RDONLY | O_BINARY); - if (fd < 0) { - debug(79, 3) ("storeAufsOpen: got failure (%d)\n", errno); - return NULL; - } -#endif - CBDATA_INIT_TYPE_FREECB(storeIOState, storeAufsIOFreeEntry); - sio = cbdataAlloc(storeIOState); - sio->fsstate = memPoolAlloc(squidaio_state_pool); - ((squidaiostate_t *) (sio->fsstate))->fd = -1; - ((squidaiostate_t *) (sio->fsstate))->flags.opening = 1; - sio->swap_filen = f; - sio->swap_dirn = SD->index; - sio->mode = O_RDONLY | O_BINARY; - sio->callback = callback; - sio->callback_data = callback_data; - sio->e = e; - cbdataLock(callback_data); - Opening_FD++; - statCounter.syscalls.disk.opens++; -#if ASYNC_OPEN - aioOpen(path, O_RDONLY | O_BINARY, 0644, storeAufsOpenDone, sio); -#else - storeAufsOpenDone(fd, sio, fd, 0); -#endif - return sio; -} - -/* open for creating */ -storeIOState * -storeAufsCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data) -{ - char *path; - storeIOState *sio; - sfileno filn; - sdirno dirn; -#if !ASYNC_CREATE - int fd; -#endif - - /* Allocate a number */ - dirn = SD->index; - filn = storeAufsDirMapBitAllocate(SD); - path = storeAufsDirFullPath(SD, filn, NULL); - - debug(79, 3) ("storeAufsCreate: fileno %08X\n", filn); - /* - * we should detect some 'too many files open' condition and return - * NULL here. - */ -#ifdef MAGIC2 - if (aioQueueSize() > MAGIC2) - return NULL; -#endif -#if !ASYNC_CREATE - fd = file_open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - if (fd < 0) { - debug(79, 3) ("storeAufsCreate: got failure (%d)\n", errno); - return NULL; - } -#endif - CBDATA_INIT_TYPE_FREECB(storeIOState, storeAufsIOFreeEntry); - sio = cbdataAlloc(storeIOState); - sio->fsstate = memPoolAlloc(squidaio_state_pool); - ((squidaiostate_t *) (sio->fsstate))->fd = -1; - ((squidaiostate_t *) (sio->fsstate))->flags.opening = 1; - sio->swap_filen = filn; - sio->swap_dirn = dirn; - sio->mode = O_WRONLY | O_BINARY; - sio->callback = callback; - sio->callback_data = callback_data; - sio->e = (StoreEntry *) e; - cbdataLock(callback_data); - Opening_FD++; - statCounter.syscalls.disk.opens++; -#if ASYNC_CREATE - aioOpen(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644, storeAufsOpenDone, sio); -#else - storeAufsOpenDone(fd, sio, fd, 0); -#endif - - /* now insert into the replacement policy */ - storeAufsDirReplAdd(SD, e); - return sio; - -} - - - -/* Close */ -void -storeAufsClose(SwapDir * SD, storeIOState * sio) -{ - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - debug(79, 3) ("storeAufsClose: dirno %d, fileno %08X, FD %d\n", - sio->swap_dirn, sio->swap_filen, aiostate->fd); - if (storeAufsNeedCompletetion(sio)) { - aiostate->flags.close_request = 1; - return; - } - storeAufsIOCallback(sio, DISK_OK); -} - - -/* Read */ -void -storeAufsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, STRCB * callback, void *callback_data) -{ - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - assert(sio->read.callback == NULL); - assert(sio->read.callback_data == NULL); - assert(!aiostate->flags.reading); - if (aiostate->fd < 0) { - struct _queued_read *q; - debug(79, 3) ("storeAufsRead: queueing read because FD < 0\n"); - assert(aiostate->flags.opening); - assert(aiostate->pending_reads == NULL); - q = memPoolAlloc(aufs_qread_pool); - q->buf = buf; - q->size = size; - q->offset = (off_t) offset; - q->callback = callback; - q->callback_data = callback_data; - cbdataLock(q->callback_data); - linklistPush(&(aiostate->pending_reads), q); - return; - } - sio->read.callback = callback; - sio->read.callback_data = callback_data; - aiostate->read_buf = buf; - cbdataLock(callback_data); - debug(79, 3) ("storeAufsRead: dirno %d, fileno %08X, FD %d\n", - sio->swap_dirn, sio->swap_filen, aiostate->fd); - sio->offset = offset; - aiostate->flags.reading = 1; -#if ASYNC_READ - aioRead(aiostate->fd, (off_t) offset, size, storeAufsReadDone, sio); - statCounter.syscalls.disk.reads++; -#else - file_read(aiostate->fd, buf, size, (off_t) offset, storeAufsReadDone, sio); - /* file_read() increments syscalls.disk.reads */ -#endif -} - - -/* Write */ -void -storeAufsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, FREE * free_func) -{ - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - debug(79, 3) ("storeAufsWrite: dirno %d, fileno %08X, FD %d\n", - sio->swap_dirn, sio->swap_filen, aiostate->fd); - if (aiostate->fd < 0) { - /* disk file not opened yet */ - struct _queued_write *q; - assert(aiostate->flags.opening); - q = memPoolAlloc(aufs_qwrite_pool); - q->buf = buf; - q->size = size; - q->offset = (off_t) offset; - q->free_func = free_func; - linklistPush(&(aiostate->pending_writes), q); - return; - } -#if ASYNC_WRITE - if (aiostate->flags.writing) { - struct _queued_write *q; - debug(79, 3) ("storeAufsWrite: queuing write\n"); - q = memPoolAlloc(aufs_qwrite_pool); - q->buf = buf; - q->size = size; - q->offset = (off_t) offset; - q->free_func = free_func; - linklistPush(&(aiostate->pending_writes), q); - return; - } - aiostate->flags.writing = 1; - aioWrite(aiostate->fd, (off_t) offset, buf, size, storeAufsWriteDone, sio, - free_func); - statCounter.syscalls.disk.writes++; -#else - file_write(aiostate->fd, (off_t) offset, buf, size, storeAufsWriteDone, sio, - free_func); - /* file_write() increments syscalls.disk.writes */ -#endif -} - -/* Unlink */ -void -storeAufsUnlink(SwapDir * SD, StoreEntry * e) -{ - debug(79, 3) ("storeAufsUnlink: dirno %d, fileno %08X\n", SD->index, e->swap_filen); - storeAufsDirReplRemove(e); - storeAufsDirMapBitReset(SD, e->swap_filen); - storeAufsDirUnlinkFile(SD, e->swap_filen); - statCounter.syscalls.disk.unlinks++; -} - -void -storeAufsRecycle(SwapDir * SD, StoreEntry * e) -{ - debug(79, 3) ("storeAufsUnlink: fileno %08X\n", e->swap_filen); - - /* Release the object without releasing the underlying physical object */ - storeExpireNow(e); - storeReleaseRequest(e); - if (e->swap_filen > -1) { - storeAufsDirReplRemove(e); - storeAufsDirMapBitReset(SD, e->swap_filen); - e->swap_filen = -1; - e->swap_dirn = -1; - } - storeRelease(e); -} - -/* === STATIC =========================================================== */ - -static int -storeAufsKickWriteQueue(storeIOState * sio) -{ - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - struct _queued_write *q = linklistShift(&aiostate->pending_writes); - if (NULL == q) - return 0; - debug(79, 3) ("storeAufsKickWriteQueue: writing queued chunk of %ld bytes\n", - (long int) q->size); - storeAufsWrite(INDEXSD(sio->swap_dirn), sio, q->buf, q->size, q->offset, q->free_func); - memPoolFree(aufs_qwrite_pool, q); - return 1; -} - -static int -storeAufsKickReadQueue(storeIOState * sio) -{ - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - struct _queued_read *q = linklistShift(&(aiostate->pending_reads)); - if (NULL == q) - return 0; - debug(79, 3) ("storeAufsKickReadQueue: reading queued request of %ld bytes\n", - (long int) q->size); - if (cbdataValid(q->callback_data)) - storeAufsRead(INDEXSD(sio->swap_dirn), sio, q->buf, q->size, q->offset, q->callback, q->callback_data); - cbdataUnlock(q->callback_data); - memPoolFree(aufs_qread_pool, q); - return 1; -} - -static void -storeAufsOpenDone(int unused, void *my_data, const char *unused2, int fd, int errflag) -{ - storeIOState *sio = my_data; - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - debug(79, 3) ("storeAufsOpenDone: FD %d, errflag %d\n", fd, errflag); - Opening_FD--; - aiostate->flags.opening = 0; - if (errflag || fd < 0) { - errno = errflag; - debug(79, 0) ("storeAufsOpenDone: %s\n", xstrerror()); - debug(79, 1) ("\t%s\n", storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL)); - storeAufsIOCallback(sio, DISK_ERROR); - return; - } - store_open_disk_fd++; - aiostate->fd = fd; - commSetCloseOnExec(fd); - fd_open(fd, FD_FILE, storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL)); - if (FILE_MODE(sio->mode) == O_WRONLY) { - if (storeAufsKickWriteQueue(sio)) - return; - } else if ((FILE_MODE(sio->mode) == O_RDONLY) && !aiostate->flags.close_request) { - if (storeAufsKickReadQueue(sio)) - return; - } - if (aiostate->flags.close_request) - storeAufsIOCallback(sio, errflag); - debug(79, 3) ("storeAufsOpenDone: exiting\n"); -} - -#if ASYNC_READ -static void -storeAufsReadDone(int fd, void *my_data, const char *buf, int len, int errflag) -#else -static void -storeAufsReadDone(int fd, const char *buf, int len, int errflag, void *my_data) -#endif -{ - storeIOState *sio = my_data; - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - STRCB *callback = sio->read.callback; - void *their_data = sio->read.callback_data; - ssize_t rlen; - int inreaddone = aiostate->flags.inreaddone; /* Protect from callback loops */ - debug(79, 3) ("storeAufsReadDone: dirno %d, fileno %08X, FD %d, len %d\n", - sio->swap_dirn, sio->swap_filen, fd, len); - aiostate->flags.inreaddone = 1; - aiostate->flags.reading = 0; - if (errflag) { - debug(79, 3) ("storeAufsReadDone: got failure (%d)\n", errflag); - rlen = -1; - } else { - rlen = len; - sio->offset += len; - } -#if ASYNC_READ - /* translate errflag from errno to Squid disk error */ - errno = errflag; - if (errflag) - errflag = DISK_ERROR; - else - errflag = DISK_OK; -#else - if (errflag == DISK_EOF) - errflag = DISK_OK; /* EOF is signalled by len == 0, not errors... */ -#endif - assert(callback); - assert(their_data); - sio->read.callback = NULL; - sio->read.callback_data = NULL; - if (!aiostate->flags.close_request && cbdataValid(their_data)) { -#if ASYNC_READ - if (rlen > 0) - memcpy(aiostate->read_buf, buf, rlen); -#endif - callback(their_data, aiostate->read_buf, rlen); - } - cbdataUnlock(their_data); - aiostate->flags.inreaddone = 0; - if (aiostate->flags.close_request && !inreaddone) - storeAufsIOCallback(sio, errflag); -} - -#if ASYNC_WRITE -static void -storeAufsWriteDone(int fd, void *my_data, const char *buf, int aio_return, int aio_errno) -#else -static void -storeAufsWriteDone(int fd, int errflag, size_t len, void *my_data) -#endif -{ - static int loop_detect = 0; - storeIOState *sio = my_data; - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; -#if ASYNC_WRITE - int errflag; - int len = aio_return; - /* Translate from errno to Squid disk error */ - if (aio_errno) - errflag = aio_errno == ENOSPC ? DISK_NO_SPACE_LEFT : DISK_ERROR; - else - errflag = DISK_OK; -#endif - debug(79, 3) ("storeAufsWriteDone: dirno %d, fileno %08X, FD %d, len %ld, err=%d\n", - sio->swap_dirn, sio->swap_filen, fd, (long int) len, errflag); - assert(++loop_detect < 10); - aiostate->flags.writing = 0; - if (errflag) { - debug(79, 0) ("storeAufsWriteDone: got failure (%d)\n", errflag); - storeAufsIOCallback(sio, errflag); - loop_detect--; - return; - } - sio->offset += len; -#if ASYNC_WRITE - if (storeAufsKickWriteQueue(sio)) - (void) 0; - else if (aiostate->flags.close_request) - storeAufsIOCallback(sio, errflag); -#else - /* loop around storeAufsKickWriteQueue to break recursion stack - * overflow when large amounts of data has been queued for write. - * As writes are blocking here we immediately get called again - * without going via the I/O event loop.. - */ - if (!aiostate->flags.write_kicking) { - /* cbdataLock to protect us from the storeAufsIOCallback on error above */ - cbdataLock(sio); - aiostate->flags.write_kicking = 1; - while (storeAufsKickWriteQueue(sio)) - if (!cbdataValid(sio)) - break; - if (cbdataValid(sio)) { - aiostate->flags.write_kicking = 0; - if (aiostate->flags.close_request) - storeAufsIOCallback(sio, errflag); - } - cbdataUnlock(sio); - } -#endif - loop_detect--; -} - -static void -storeAufsIOCallback(storeIOState * sio, int errflag) -{ - STIOCB *callback = sio->callback; - void *their_data = sio->callback_data; - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - int fd = aiostate->fd; - debug(79, 3) ("storeAufsIOCallback: errflag=%d\n", errflag); - sio->callback = NULL; - sio->callback_data = NULL; - debug(79, 9) ("%s:%d\n", __FILE__, __LINE__); - if (callback) - if (NULL == their_data || cbdataValid(their_data)) - callback(their_data, errflag, sio); - debug(79, 9) ("%s:%d\n", __FILE__, __LINE__); - cbdataUnlock(their_data); - aiostate->fd = -1; - if (aiostate->flags.opening) - Opening_FD--; - cbdataFree(sio); - if (fd < 0) - return; - debug(79, 9) ("%s:%d\n", __FILE__, __LINE__); -#if ASYNC_CLOSE - fd_close(fd); - aioClose(fd); -#else - aioCancel(fd); - file_close(fd); -#endif - store_open_disk_fd--; - statCounter.syscalls.disk.closes++; - debug(79, 9) ("%s:%d\n", __FILE__, __LINE__); -} - - -static int -storeAufsNeedCompletetion(storeIOState * sio) -{ - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - - if (aiostate->flags.writing) - return 1; - if (aiostate->flags.opening && FILE_MODE(sio->mode) == O_WRONLY) - return 1; - if (aiostate->flags.reading) - return 1; - if (aiostate->flags.inreaddone) - return 1; - - return 0; -} - - -/* - * Clean up references from the SIO before it gets released. - * The actuall SIO is managed by cbdata so we do not need - * to bother with that. - */ -static void -storeAufsIOFreeEntry(void *siop) -{ - storeIOState *sio = (storeIOState *) siop; - squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; - struct _queued_write *qw; - struct _queued_read *qr; - while ((qw = linklistShift(&aiostate->pending_writes))) { - if (qw->free_func) - qw->free_func(qw->buf); - memPoolFree(aufs_qwrite_pool, qw); - } - while ((qr = linklistShift(&aiostate->pending_reads))) { - cbdataUnlock(qr->callback_data); - memPoolFree(aufs_qread_pool, qr); - } - if (sio->read.callback_data) - cbdataUnlock(sio->read.callback_data); - if (sio->callback_data) - cbdataUnlock(sio->callback_data); - memPoolFree(squidaio_state_pool, aiostate); -}