Squid-2.2.STABLE4: Some tuning of Async-IO code This patch makes some tuning of the Async-IO code to avoid wasting threads on operations which usually does not block. It also reverts to using unlinkd for unlinks even when using async-io due to some load balancing troubles. This can be tuned in include/config.h if you'd like to. Index: squid/acconfig.h diff -u squid/acconfig.h:1.1.1.22 squid/acconfig.h:1.1.1.22.2.1 --- squid/acconfig.h:1.1.1.22 Tue Jul 13 00:08:44 1999 +++ squid/acconfig.h Tue Jul 13 02:11:47 1999 @@ -48,6 +48,9 @@ /* Define to use async disk I/O operations */ #undef USE_ASYNC_IO +/* Number of worker theads for async-io */ +#undef NR_ASYNC_IO_THREADS + /* * If you want to use Squid's ICMP features (highly recommended!) then * define this. When USE_ICMP is defined, Squid will send ICMP pings @@ -227,4 +230,10 @@ * If your system has statvfs(), and if it actually works! */ #undef HAVE_STATVFS + +/* Default HTTP port */ +#undef CACHE_HTTP_PORT + +/* Default ICP port */ +#undef CACHE_ICP_PORT Index: squid/configure diff -u squid/configure:1.1.1.32.2.3 squid/configure:1.1.1.32.2.4 --- squid/configure:1.1.1.32.2.3 Tue Jul 13 01:49:43 1999 +++ squid/configure Tue Jul 13 02:11:47 1999 @@ -27,7 +27,11 @@ ac_help="$ac_help --enable-carp Enable CARP support" ac_help="$ac_help - --enable-async-io Do ASYNC disk I/O using threads" + --enable-async-io[=N_THREADS] + Do ASYNC disk I/O using threads. + N_THREADS is the number of worker threads + defaults to 16. See also src/squid.h for + some additional platform tuning" ac_help="$ac_help --enable-icmp Enable ICMP pinging" ac_help="$ac_help @@ -673,7 +677,13 @@ CACHE_ICP_PORT="3130" fi +cat >> confdefs.h <> confdefs.h <> confdefs.h <> confdefs.h <> confdefs.h <<\EOF #define USE_ASYNC_IO 1 EOF @@ -1134,13 +1162,10 @@ fi ;; esac - fi - fi - # Check whether --enable-icmp or --disable-icmp was given. if test "${enable_icmp+set}" = set; then enableval="$enable_icmp" Index: squid/configure.in diff -u squid/configure.in:1.1.1.34.2.3 squid/configure.in:1.1.1.34.2.4 --- squid/configure.in:1.1.1.34.2.3 Tue Jul 13 00:51:26 1999 +++ squid/configure.in Tue Jul 13 02:11:52 1999 @@ -48,8 +48,8 @@ fi dnl Subsitutions -AC_SUBST(CACHE_HTTP_PORT) -AC_SUBST(CACHE_ICP_PORT) +AC_DEFINE_UNQUOTED(CACHE_HTTP_PORT, $CACHE_HTTP_PORT) +AC_DEFINE_UNQUOTED(CACHE_ICP_PORT, $CACHE_ICP_PORT) AC_DEFINE_UNQUOTED(CONFIG_HOST_TYPE, "$host") @@ -239,10 +239,27 @@ ]) AC_ARG_ENABLE(async_io, -[ --enable-async-io Do ASYNC disk I/O using threads], -[ if test "$enableval" = "yes" ; then - echo "Async I/O enabled" +[ --enable-async-io[=N_THREADS] + Do ASYNC disk I/O using threads. + N_THREADS is the number of worker threads + defaults to 16. See also src/squid.h for + some additional platform tuning], +[ case "$enableval" in + yes) + async_io=yes + ;; + no) + async_io='' + ;; + *) async_io=yes + AC_DEFINE_UNQUOTED(NR_ASYNC_IO_THREADS,$enableval) + ;; + esac +]) + +if test -n "$async_io" ; then + echo "Async I/O enabled" AC_DEFINE(USE_ASYNC_IO) ASYNC_OBJS='$(ASYNC_OBJS)' SQUID_PTHREAD_LIB='$(PTHREADLIB)' @@ -256,8 +273,7 @@ fi ;; esac - fi -]) +fi AC_SUBST(ASYNC_OBJS) AC_SUBST(SQUID_PTHREAD_LIB) Index: squid/include/autoconf.h.in diff -u squid/include/autoconf.h.in:1.1.1.27.2.2 squid/include/autoconf.h.in:1.1.1.27.2.3 --- squid/include/autoconf.h.in:1.1.1.27.2.2 Tue Jul 13 01:49:56 1999 +++ squid/include/autoconf.h.in Tue Jul 13 02:11:54 1999 @@ -70,6 +70,9 @@ /* Define to use async disk I/O operations */ #undef USE_ASYNC_IO +/* Number of worker theads for async-io */ +#undef NR_ASYNC_IO_THREADS + /* * If you want to use Squid's ICMP features (highly recommended!) then * define this. When USE_ICMP is defined, Squid will send ICMP pings @@ -245,6 +248,12 @@ * If your system has statvfs(), and if it actually works! */ #undef HAVE_STATVFS + +/* Default HTTP port */ +#undef CACHE_HTTP_PORT + +/* Default ICP port */ +#undef CACHE_ICP_PORT /* The number of bytes in a int. */ #undef SIZEOF_INT Index: squid/include/config.h.in diff -u squid/include/config.h.in:1.1.1.14.2.1 squid/include/config.h.in:1.1.1.14.2.2 --- squid/include/config.h.in:1.1.1.14.2.1 Tue Jul 13 00:50:18 1999 +++ squid/include/config.h.in Tue Jul 13 02:11:55 1999 @@ -119,11 +119,11 @@ #endif #if !defined(CACHE_HTTP_PORT) -#define CACHE_HTTP_PORT @CACHE_HTTP_PORT@ +#define CACHE_HTTP_PORT 3128 #endif #if !defined(CACHE_ICP_PORT) -#define CACHE_ICP_PORT @CACHE_ICP_PORT@ +#define CACHE_ICP_PORT 3130 #endif #if !defined(CACHEMGR_HOSTNAME) @@ -185,5 +185,19 @@ typedef unsigned long u_num32; #endif #define NUM32LEN sizeof(num32) /* this should always be 4 */ + +/* Async-IO tuning */ +#if USE_ASYNC_IO +#ifdef _SQUID_LINUX_ +#define USE_ASYNC_IO_CLOSE 0 +#else +#define USE_ASYNC_IO_CLOSE 1 +#endif +#define USE_ASYNC_IO_OPEN 1 +#define USE_ASYNC_IO_UNLINK 0 +#define USE_ASYNC_IO_READ 1 +#define USE_ASYNC_IO_WRITE 0 +#define USE_ASYNC_IO_STAT 1 +#endif #endif /* _CONFIG_H_ */ Index: squid/src/aiops.c diff -u squid/src/aiops.c:1.1.1.16.2.2 squid/src/aiops.c:1.1.1.16.2.3 --- squid/src/aiops.c:1.1.1.16.2.2 Tue Jul 13 01:11:30 1999 +++ squid/src/aiops.c Tue Jul 13 02:11:55 1999 @@ -46,8 +46,8 @@ #include #endif -#ifndef NUMTHREADS -#define NUMTHREADS 16 +#ifndef NR_ASYNC_IO_THREADS +#define NR_ASYNC_IO_THREADS 16 #endif #define RIDICULOUS_LENGTH 4096 @@ -169,9 +169,9 @@ #endif /* Create threads and get them to sit in their wait loop */ - threads = xcalloc(NUMTHREADS, sizeof(aio_thread_t)); + threads = xcalloc(NR_ASYNC_IO_THREADS, sizeof(aio_thread_t)); - for (i = 0; i < NUMTHREADS; i++) { + for (i = 0; i < NR_ASYNC_IO_THREADS; i++) { threadp = &threads[i]; threadp->status = _THREAD_STARTING; if (pthread_mutex_init(&(threadp->mutex), NULL)) { @@ -334,7 +334,7 @@ /* Kick it rolling */ aio_process_request_queue(); /* Warn if out of threads */ - if (request_queue_len > (NUMTHREADS >> 1)) { + if (request_queue_len > (NR_ASYNC_IO_THREADS >> 1)) { if (high_start == 0) { high_start = squid_curtime; queue_high = request_queue_len; @@ -349,11 +349,11 @@ debug(43, 1) ("aio_queue_request: WARNING - Running out of I/O theads\n"); debug(43, 2) ("aio_queue_request: Queue Length: current=%d, high=%d, low=%d, duration=%d\n", request_queue_len, queue_high, queue_low, squid_curtime - high_start); - debug(43, 1) ("aio_queue_request: Perhaps you should increase NUMTHREADS\n"); + debug(43, 1) ("aio_queue_request: Perhaps you should increase NR_ASYNC_IO_THREADS\n"); debug(43, 1) ("aio_queue_request: Or install more disks to share the load\n"); - debug(43, 3) ("aio_queue_request: First %d items on request queue\n", NUMTHREADS); + debug(43, 3) ("aio_queue_request: First %d items on request queue\n", NR_ASYNC_IO_THREADS); rp = request_queue_head; - for (i = 1; i <= NUMTHREADS; i++) { + for (i = 1; i <= NR_ASYNC_IO_THREADS; i++) { switch (rp->request_type) { case _AIO_OP_OPEN: debug(43, 3) ("aio_queue_request: %d : open -> %s\n", i, rp->path); Index: squid/src/comm.c diff -u squid/src/comm.c:1.1.1.32 squid/src/comm.c:1.1.1.32.2.1 --- squid/src/comm.c:1.1.1.32 Tue Jul 13 00:09:25 1999 +++ squid/src/comm.c Tue Jul 13 02:11:55 1999 @@ -39,12 +39,6 @@ #include #endif -#if USE_ASYNC_IO -#define MAX_POLL_TIME 10 -#else -#define MAX_POLL_TIME 1000 -#endif - typedef struct { char *host; u_short port; Index: squid/src/disk.c diff -u squid/src/disk.c:1.1.1.22 squid/src/disk.c:1.1.1.22.2.1 --- squid/src/disk.c:1.1.1.22 Tue Jul 13 00:09:26 1999 +++ squid/src/disk.c Tue Jul 13 02:11:55 1999 @@ -76,7 +76,7 @@ /* Open file */ Opening_FD++; -#if USE_ASYNC_IO +#if USE_ASYNC_IO_OPEN if (callback != NULL) { aioOpen(path, mode, 0644, fileOpenComplete, ctrlp, tag); return DISK_OK; @@ -131,14 +131,7 @@ { fde *F = &fd_table[fd]; PF *callback; -#if USE_ASYNC_IO - if (fd < 0) { - debug(6, 0) ("file_close: FD less than zero: %d\n", fd); - return; - } -#else assert(fd >= 0); -#endif assert(F->flags.open); if ((callback = F->read_handler)) { F->read_handler = NULL; @@ -164,7 +157,7 @@ */ assert(F->write_handler == NULL); F->flags.closing = 1; -#if USE_ASYNC_IO +#if USE_ASYNC_IO_CLOSE aioClose(fd); #else #if CALL_FSYNC_BEFORE_CLOSE @@ -174,7 +167,7 @@ #endif debug(6, F->flags.close_request ? 2 : 5) ("file_close: FD %d, really closing\n", fd); -#if !USE_ASYNC_IO +#if !USE_ASYNC_IO_CLOSE fd_close(fd); #endif Counter.syscalls.disk.closes++; @@ -230,7 +223,7 @@ static void diskHandleWrite(int fd, void *notused) { -#if !USE_ASYNC_IO +#if !USE_ASYNC_IO_WRITE int len = 0; #endif fde *F = &fd_table[fd]; @@ -243,7 +236,7 @@ debug(6, 3) ("diskHandleWrite: FD %d\n", fd); assert(fdd->write_q != NULL); assert(fdd->write_q->len > fdd->write_q->buf_offset); -#if USE_ASYNC_IO +#if USE_ASYNC_IO_WRITE aioWrite(fd, -1, /* seek offset, -1 == append */ fdd->write_q->buf + fdd->write_q->buf_offset, @@ -434,7 +427,7 @@ } if (!F->flags.write_daemon) { cbdataLock(F->disk.wrt_handle_data); -#if USE_ASYNC_IO +#if USE_ASYNC_IO_WRITE diskHandleWrite(fd, NULL); #else #ifdef OPTIMISTIC_IO @@ -467,7 +460,7 @@ diskHandleRead(int fd, void *data) { dread_ctrl *ctrl_dat = data; -#if !USE_ASYNC_IO +#if !USE_ASYNC_IO_READ fde *F = &fd_table[fd]; int len; #endif @@ -482,7 +475,7 @@ memFree(ctrl_dat, MEM_DREAD_CTRL); return; } -#if USE_ASYNC_IO +#if USE_ASYNC_IO_READ aioRead(fd, ctrl_dat->offset, ctrl_dat->buf, @@ -566,7 +559,7 @@ ctrl_dat->handler = handler; ctrl_dat->client_data = client_data; cbdataLock(client_data); -#if USE_ASYNC_IO +#if USE_ASYNC_IO_READ diskHandleRead(fd, ctrl_dat); #else #ifndef OPTIMISTIC_IO Index: squid/src/squid.h diff -u squid/src/squid.h:1.1.1.23.2.1 squid/src/squid.h:1.1.1.23.2.2 --- squid/src/squid.h:1.1.1.23.2.1 Tue Jul 13 00:50:21 1999 +++ squid/src/squid.h Tue Jul 13 02:11:55 1999 @@ -320,7 +320,7 @@ #include #endif -#if USE_ASYNC_IO +#if USE_ASYNC_IO_UNLINK #undef USE_UNLINKD #else #define USE_UNLINKD 1 Index: squid/src/store.c diff -u squid/src/store.c:1.1.1.39.2.2 squid/src/store.c:1.1.1.39.2.3 --- squid/src/store.c:1.1.1.39.2.2 Tue Jul 13 00:44:24 1999 +++ squid/src/store.c Tue Jul 13 02:11:55 1999 @@ -1207,10 +1207,10 @@ storeUnlinkFileno(int fileno) { debug(20, 5) ("storeUnlinkFileno: %08X\n", fileno); -#if USE_ASYNC_IO - safeunlink(storeSwapFullPath(fileno, NULL), 1); -#else +#if USE_UNLINKD unlinkdUnlink(storeSwapFullPath(fileno, NULL)); +#else + safeunlink(storeSwapFullPath(fileno, NULL), 1); #endif } Index: squid/src/store_rebuild.c diff -u squid/src/store_rebuild.c:1.1.1.18 squid/src/store_rebuild.c:1.1.1.18.2.1 --- squid/src/store_rebuild.c:1.1.1.18 Tue Jul 13 00:09:38 1999 +++ squid/src/store_rebuild.c Tue Jul 13 02:11:56 1999 @@ -653,7 +653,7 @@ valid_ctrl_t *ctrlp; char *path; struct stat *sb; -#if !USE_ASYNC_IO +#if !USE_ASYNC_IO_STAT int x; #endif assert(!EBIT_TEST(e->flags, ENTRY_VALIDATED)); @@ -669,7 +669,7 @@ ctrlp->e = e; ctrlp->callback = callback; ctrlp->callback_data = callback_data; -#if USE_ASYNC_IO +#if USE_ASYNC_IO_STAT aioStat(path, sb, storeValidateComplete, ctrlp, tag); #else /*