--------------------- PatchSet 6565 Date: 2005/03/29 03:00:05 Author: swsf Branch: epoll-2_5 Tag: (none) Log: Merge s2_5 changes (aio and large file support) Members: src/forward.c:1.13.6.13.4.2->1.13.6.13.4.3 src/main.c:1.28.6.19.2.1->1.28.6.19.2.2 src/protos.h:1.41.6.22.4.3->1.41.6.22.4.4 src/ssl.c:1.13.6.7.10.1->1.13.6.7.10.2 src/stmem.c:1.7.68.1->1.7.68.2 src/store_swapout.c:1.11.2.1.32.2->1.11.2.1.32.3 src/structs.h:1.48.2.34.2.2->1.48.2.34.2.3 src/auth/basic/auth_basic.c:1.17.6.8->1.17.6.8.10.1 src/fs/aufs/aiops.c:1.11.6.7->1.11.6.7.18.1 src/fs/aufs/async_io.c:1.7.6.4.24.1->1.7.6.4.24.2 src/fs/aufs/store_asyncufs.h:1.6.14.2.18.1->1.6.14.2.18.2 src/fs/aufs/store_dir_aufs.c:1.23.6.9.10.1->1.23.6.9.10.2 src/fs/aufs/store_io_aufs.c:1.13.2.11.10.1->1.13.2.11.10.2 Index: squid/src/forward.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/forward.c,v retrieving revision 1.13.6.13.4.2 retrieving revision 1.13.6.13.4.3 diff -u -r1.13.6.13.4.2 -r1.13.6.13.4.3 --- squid/src/forward.c 29 Mar 2005 02:27:16 -0000 1.13.6.13.4.2 +++ squid/src/forward.c 29 Mar 2005 03:00:05 -0000 1.13.6.13.4.3 @@ -1,6 +1,6 @@ /* - * $Id: forward.c,v 1.13.6.13.4.2 2005/03/29 02:27:16 swsf Exp $ + * $Id: forward.c,v 1.13.6.13.4.3 2005/03/29 03:00:05 swsf Exp $ * * DEBUG: section 17 Request Forwarding * AUTHOR: Duane Wessels @@ -695,13 +695,6 @@ return 1; if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) return rc; -<<<<<<< forward.c - if (mem->inmem_hi - storeLowestMemReaderOffset(e) < READ_AHEAD_GAP) - return rc; - /* record that we're defering the read */ - EBIT_SET(e->flags, ENTRY_DEFER_READ); - return 1; -======= if (EBIT_TEST(e->flags, RELEASE_REQUEST)) { /* Just a small safety cap to defer storing more data into the object * if there already is way too much. This handles the case when there @@ -709,12 +702,13 @@ * few other corner cases. */ if (mem->inmem_hi - mem->inmem_lo > SM_PAGE_SIZE + Config.Store.maxInMemObjSize + READ_AHEAD_GAP) + EBIT_SET(e->flags, ENTRY_DEFER_READ); return 1; } if (mem->inmem_hi - storeLowestMemReaderOffset(e) > READ_AHEAD_GAP) + EBIT_SET(e->flags, ENTRY_DEFER_READ); return 1; return rc; ->>>>>>> 1.13.6.14 } void Index: squid/src/main.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/main.c,v retrieving revision 1.28.6.19.2.1 retrieving revision 1.28.6.19.2.2 diff -u -r1.28.6.19.2.1 -r1.28.6.19.2.2 --- squid/src/main.c 17 Mar 2005 03:57:44 -0000 1.28.6.19.2.1 +++ squid/src/main.c 29 Mar 2005 03:00:05 -0000 1.28.6.19.2.2 @@ -1,6 +1,6 @@ /* - * $Id: main.c,v 1.28.6.19.2.1 2005/03/17 03:57:44 swsf Exp $ + * $Id: main.c,v 1.28.6.19.2.2 2005/03/29 03:00:05 swsf Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -990,8 +990,8 @@ fwdUninit(); #endif storeDirSync(); /* Flush log close */ -#if PURIFY || XMALLOC_TRACE storeFsDone(); +#if PURIFY || XMALLOC_TRACE configFreeMemory(); storeFreeMemory(); /*stmemFreeMemory(); */ @@ -1008,9 +1008,9 @@ #endif #if !XMALLOC_TRACE if (opt_no_daemon) { - file_close(0); - file_close(1); - file_close(2); + fd_close(0); + fd_close(1); + fd_close(2); } #endif fdDumpOpen(); Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.41.6.22.4.3 retrieving revision 1.41.6.22.4.4 diff -u -r1.41.6.22.4.3 -r1.41.6.22.4.4 --- squid/src/protos.h 29 Mar 2005 02:27:20 -0000 1.41.6.22.4.3 +++ squid/src/protos.h 29 Mar 2005 03:00:06 -0000 1.41.6.22.4.4 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.41.6.22.4.3 2005/03/29 02:27:20 swsf Exp $ + * $Id: protos.h,v 1.41.6.22.4.4 2005/03/29 03:00:06 swsf Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -866,6 +866,8 @@ extern ssize_t stmemCopy(const mem_hdr *, squid_off_t, char *, size_t); extern void stmemFree(mem_hdr *); extern void stmemFreeData(mem_hdr *); +extern void stmemNodeFree(void *); +extern char *stmemNodeGet(mem_node *); /* ----------------------------------------------------------------- */ Index: squid/src/ssl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ssl.c,v retrieving revision 1.13.6.7.10.1 retrieving revision 1.13.6.7.10.2 diff -u -r1.13.6.7.10.1 -r1.13.6.7.10.2 --- squid/src/ssl.c 29 Mar 2005 02:27:26 -0000 1.13.6.7.10.1 +++ squid/src/ssl.c 29 Mar 2005 03:00:07 -0000 1.13.6.7.10.2 @@ -1,6 +1,6 @@ /* - * $Id: ssl.c,v 1.13.6.7.10.1 2005/03/29 02:27:26 swsf Exp $ + * $Id: ssl.c,v 1.13.6.7.10.2 2005/03/29 03:00:07 swsf Exp $ * * DEBUG: section 26 Secure Sockets Layer Proxy * AUTHOR: Duane Wessels @@ -75,6 +75,18 @@ #endif static void +sslAbort(SslStateData * sslState) +{ + debug(26, 3) ("sslAbort: FD %d/%d\n", sslState->client.fd, sslState->server.fd); + cbdataLock(sslState); + if (sslState->client.fd > -1) + comm_close(sslState->client.fd); + if (sslState->server.fd > -1) + comm_close(sslState->server.fd); + cbdataUnlock(sslState); +} + +static void sslServerClosed(int fd, void *data) { SslStateData *sslState = data; @@ -92,9 +104,7 @@ debug(26, 3) ("sslClientClosed: FD %d\n", fd); assert(fd == sslState->client.fd); sslState->client.fd = -1; - if (sslState->server.fd != -1) - comm_close(sslState->server.fd); - else + if (sslState->server.fd == -1) sslStateFree(sslState); } @@ -152,7 +162,7 @@ sslState, Config.Timeout.read); } - } else if (sslState->client.len == 0) { + } else if (sslState->client.len == 0 && sslState->server.fd > -1) { comm_close(sslState->server.fd); } if (!sslState->connected) { @@ -183,9 +193,7 @@ sslState, Config.Timeout.read); } - } else if (sslState->client.fd == -1) { - /* client already closed, nothing more to do */ - } else if (sslState->server.len == 0) { + } else if (sslState->server.len == 0 && sslState->client.fd > -1) { comm_close(sslState->client.fd); } } @@ -262,7 +270,7 @@ debug(50, level) ("sslReadClient: FD %d: read failure: %s\n", fd, xstrerror()); if (!ignoreErrno(errno)) - comm_close(fd); + sslAbort(sslState); } else if (len == 0) { comm_close(fd); } @@ -309,7 +317,7 @@ debug(50, ignoreErrno(errno) ? 3 : 1) ("sslWriteServer: FD %d: write failure: %s.\n", fd, xstrerror()); if (!ignoreErrno(errno)) - comm_close(fd); + sslAbort(sslState); } if (cbdataValid(sslState)) sslSetSelect(sslState); @@ -353,7 +361,7 @@ debug(50, ignoreErrno(errno) ? 3 : 1) ("sslWriteClient: FD %d: write failure: %s.\n", fd, xstrerror()); if (!ignoreErrno(errno)) - comm_close(fd); + sslAbort(sslState); } if (cbdataValid(sslState)) sslSetSelect(sslState); @@ -365,7 +373,7 @@ { SslStateData *sslState = data; debug(26, 3) ("sslTimeout: FD %d\n", fd); - comm_close(sslState->client.fd); + sslAbort(sslState); } static void @@ -389,7 +397,7 @@ static void -sslConnectDone(int fdnotused, int status, void *data) +sslConnectDone(int fd, int status, void *data) { SslStateData *sslState = data; request_t *request = sslState->request; @@ -405,6 +413,7 @@ sslState->host); if (status == COMM_ERR_DNS) { debug(26, 4) ("sslConnect: Unknown host: %s\n", sslState->host); + comm_close(fd); err = errorCon(ERR_DNS_FAIL, HTTP_NOT_FOUND); *sslState->status_ptr = HTTP_NOT_FOUND; err->request = requestLink(request); @@ -413,6 +422,7 @@ err->callback_data = sslState; errorSend(sslState->client.fd, err); } else if (status != COMM_OK) { + comm_close(fd); err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE); *sslState->status_ptr = HTTP_SERVICE_UNAVAILABLE; err->xerrno = errno; Index: squid/src/stmem.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stmem.c,v retrieving revision 1.7.68.1 retrieving revision 1.7.68.2 diff -u -r1.7.68.1 -r1.7.68.2 --- squid/src/stmem.c 29 Mar 2005 02:27:27 -0000 1.7.68.1 +++ squid/src/stmem.c 29 Mar 2005 03:00:08 -0000 1.7.68.2 @@ -1,6 +1,6 @@ /* - * $Id: stmem.c,v 1.7.68.1 2005/03/29 02:27:27 swsf Exp $ + * $Id: stmem.c,v 1.7.68.2 2005/03/29 03:00:08 swsf Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Harvest Derived @@ -36,16 +36,30 @@ #include "squid.h" void +stmemNodeFree(void *buf) +{ + mem_node *p = (mem_node *) buf; + if (!p->uses) + memFree(p, MEM_MEM_NODE); + else + p->uses--; +} + +char * +stmemNodeGet(mem_node * p) +{ + p->uses++; + return p->data; +} + +void stmemFree(mem_hdr * mem) { mem_node *p; while ((p = mem->head)) { mem->head = p->next; store_mem_size -= SM_PAGE_SIZE; - if (p) { - memFree(p, MEM_MEM_NODE); - p = NULL; - } + stmemNodeFree(p); } mem->head = mem->tail = NULL; mem->origin_offset = 0; @@ -54,12 +68,7 @@ squid_off_t stmemFreeDataUpto(mem_hdr * mem, squid_off_t target_offset) { -<<<<<<< stmem.c - int current_offset = mem->origin_offset; - mem_node *lastp; -======= squid_off_t current_offset = mem->origin_offset; ->>>>>>> 1.7.6.3 mem_node *p = mem->head; while (p && ((current_offset + p->len) <= target_offset)) { if (p == mem->tail) { @@ -68,14 +77,11 @@ mem->origin_offset = current_offset; return current_offset; } else { - lastp = p; + mem_node *lastp = p; p = p->next; current_offset += lastp->len; store_mem_size -= SM_PAGE_SIZE; - if (lastp) { - memFree(lastp, MEM_MEM_NODE); - lastp = NULL; - } + stmemNodeFree(lastp); } } mem->head = p; Index: squid/src/store_swapout.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_swapout.c,v retrieving revision 1.11.2.1.32.2 retrieving revision 1.11.2.1.32.3 diff -u -r1.11.2.1.32.2 -r1.11.2.1.32.3 --- squid/src/store_swapout.c 29 Mar 2005 02:27:29 -0000 1.11.2.1.32.2 +++ squid/src/store_swapout.c 29 Mar 2005 03:00:08 -0000 1.11.2.1.32.3 @@ -1,6 +1,6 @@ /* - * $Id: store_swapout.c,v 1.11.2.1.32.2 2005/03/29 02:27:29 swsf Exp $ + * $Id: store_swapout.c,v 1.11.2.1.32.3 2005/03/29 03:00:08 swsf Exp $ * * DEBUG: section 20 Storage Manager Swapout Functions * AUTHOR: Duane Wessels @@ -151,9 +151,13 @@ if (new_mem_lo == -1) new_mem_lo = 0; /* the above might become -1 */ } - if (mem->inmem_lo != new_mem_lo) + if (mem->inmem_lo != new_mem_lo) { mem->inmem_lo = stmemFreeDataUpto(&mem->data_hdr, new_mem_lo); + /* The read may not be deferred any longer */ + EBIT_CLR(e->flags, ENTRY_DEFER_READ); + } + return on_disk; } @@ -208,50 +212,8 @@ debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n"); return; } -<<<<<<< store_swapout.c - /* - * Careful. lowest_offset can be greater than inmem_hi, such - * as in the case of a range request. - */ - if (mem->inmem_hi < lowest_offset) - new_mem_lo = lowest_offset; - else if (mem->inmem_hi - mem->inmem_lo > Config.Store.maxInMemObjSize) - new_mem_lo = lowest_offset; - else - new_mem_lo = mem->inmem_lo; - assert(new_mem_lo >= mem->inmem_lo); - if (storeSwapOutAble(e)) { - /* - * We should only free up to what we know has been written - * to disk, not what has been queued for writing. Otherwise - * there will be a chunk of the data which is not in memory - * and is not yet on disk. - * The -1 makes sure the page isn't freed until storeSwapOut has - * walked to the next page. (mem->swapout.memnode) - */ - if ((on_disk = storeSwapOutObjectBytesOnDisk(mem)) - 1 < new_mem_lo) - new_mem_lo = on_disk - 1; - if (new_mem_lo == -1) - new_mem_lo = 0; /* the above might become -1 */ - } else if (new_mem_lo > 0) { - /* - * Its not swap-able, and we're about to delete a chunk, - * so we must make it PRIVATE. This is tricky/ugly because - * for the most part, we treat swapable == cachable here. - */ - storeReleaseRequest(e); - } - stmemFreeDataUpto(&mem->data_hdr, new_mem_lo); - mem->inmem_lo = new_mem_lo; - - /* The read may not be deferred any longer */ - EBIT_CLR(e->flags, ENTRY_DEFER_READ); - -#if SIZEOF_OFF_T == 4 -======= on_disk = storeSwapOutMaintainMemObject(e); #if SIZEOF_SQUID_OFF_T <= 4 ->>>>>>> 1.11.2.5 if (mem->inmem_hi > 0x7FFF0000) { debug(20, 0) ("WARNING: preventing squid_off_t overflow for %s\n", storeUrl(e)); storeAbort(e); @@ -322,7 +284,7 @@ debug(20, 3) ("storeSwapOut: swapping out %d bytes from %" PRINTF_OFF_T "\n", (int) swap_buf_len, mem->swapout.queue_offset); mem->swapout.queue_offset += swap_buf_len; - storeWrite(mem->swapout.sio, mem->swapout.memnode->data, swap_buf_len, -1, NULL); + storeWrite(mem->swapout.sio, stmemNodeGet(mem->swapout.memnode), swap_buf_len, -1, stmemNodeFree); /* the storeWrite() call might generate an error */ if (e->swap_status != SWAPOUT_WRITING) break; Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.48.2.34.2.2 retrieving revision 1.48.2.34.2.3 diff -u -r1.48.2.34.2.2 -r1.48.2.34.2.3 --- squid/src/structs.h 29 Mar 2005 02:27:29 -0000 1.48.2.34.2.2 +++ squid/src/structs.h 29 Mar 2005 03:00:08 -0000 1.48.2.34.2.3 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.48.2.34.2.2 2005/03/29 02:27:29 swsf Exp $ + * $Id: structs.h,v 1.48.2.34.2.3 2005/03/29 03:00:08 swsf Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1423,6 +1423,7 @@ struct _mem_node { char data[SM_PAGE_SIZE]; int len; + int uses; mem_node *next; }; Index: squid/src/auth/basic/auth_basic.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/auth/basic/auth_basic.c,v retrieving revision 1.17.6.8 retrieving revision 1.17.6.8.10.1 diff -u -r1.17.6.8 -r1.17.6.8.10.1 --- squid/src/auth/basic/auth_basic.c 26 Sep 2004 02:14:26 -0000 1.17.6.8 +++ squid/src/auth/basic/auth_basic.c 29 Mar 2005 03:00:31 -0000 1.17.6.8.10.1 @@ -442,10 +442,14 @@ * Don't allow NL or CR in the credentials. * Oezguer Kesim */ - strtok(cleartext, "\r\n"); debug(29, 9) ("authenticateBasicDecodeAuth: cleartext = '%s'\n", cleartext); - local_basic.username = xstrndup(cleartext, USER_IDENT_SZ); - xfree(cleartext); + if (strcspn(cleartext, "\r\n") != strlen(cleartext)) { + debug(29, 1) ("authenticateBasicDecodeAuth: bad characters in authorization header '%s'\n", + proxy_auth); + xfree(cleartext); + return; + } + local_basic.username = cleartext; if ((cleartext = strchr(local_basic.username, ':')) != NULL) *(cleartext)++ = '\0'; local_basic.passwd = cleartext; Index: squid/src/fs/aufs/aiops.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/aiops.c,v retrieving revision 1.11.6.7 retrieving revision 1.11.6.7.18.1 diff -u -r1.11.6.7 -r1.11.6.7.18.1 --- squid/src/fs/aufs/aiops.c 7 Aug 2003 02:13:45 -0000 1.11.6.7 +++ squid/src/fs/aufs/aiops.c 29 Mar 2005 03:00:32 -0000 1.11.6.7.18.1 @@ -92,7 +92,6 @@ mode_t mode; int fd; char *bufferp; - char *tmpbufp; int buflen; off_t offset; int whence; @@ -121,7 +120,6 @@ unsigned long requests; }; -static void squidaio_init(void); static void squidaio_queue_request(squidaio_request_t *); static void squidaio_cleanup_request(squidaio_request_t *); static void *squidaio_thread_loop(void *); @@ -256,7 +254,7 @@ commSetSelect(fd, COMM_SELECT_READ, squidaio_fdhandler, NULL, 0); } -static void +void squidaio_init(void) { int i; @@ -308,8 +306,8 @@ pipe(done_pipe); done_fd = done_pipe[1]; done_fd_read = done_pipe[0]; - fd_open(done_pipe[0], FD_PIPE, "async-io completetion event: main"); - fd_open(done_pipe[1], FD_PIPE, "async-io completetion event: threads"); + fd_open(done_fd_read, FD_PIPE, "async-io completion event: main"); + fd_open(done_fd, FD_PIPE, "async-io completion event: threads"); commSetNonBlocking(done_pipe[0]); commSetNonBlocking(done_pipe[1]); commSetSelect(done_pipe[0], COMM_SELECT_READ, squidaio_fdhandler, NULL, 0); @@ -352,6 +350,23 @@ squidaio_initialised = 1; } +void +squidaio_shutdown(void) +{ + if (!squidaio_initialised) + return; + + /* This is the same as in squidaio_sync */ + do { + squidaio_poll_queues(); + } while (request_queue_len > 0); + + close(done_fd); + close(done_fd_read); + fd_close(done_fd); + fd_close(done_fd_read); +} + static void * squidaio_thread_loop(void *ptr) @@ -573,7 +588,6 @@ case _AIO_OP_READ: break; case _AIO_OP_WRITE: - squidaio_xfree(requestp->tmpbufp, requestp->buflen); break; default: break; @@ -608,8 +622,6 @@ { squidaio_request_t *requestp; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); requestp->path = (char *) squidaio_xstrdup(path); requestp->oflag = oflag; @@ -636,8 +648,6 @@ { squidaio_request_t *requestp; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); requestp->fd = fd; requestp->bufferp = bufp; @@ -667,12 +677,9 @@ { squidaio_request_t *requestp; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); requestp->fd = fd; - requestp->tmpbufp = (char *) squidaio_xmalloc(bufs); - xmemcpy(requestp->tmpbufp, bufp, bufs); + requestp->bufferp = bufp; requestp->buflen = bufs; requestp->offset = offset; requestp->whence = whence; @@ -688,7 +695,7 @@ static void squidaio_do_write(squidaio_request_t * requestp) { - requestp->ret = write(requestp->fd, requestp->tmpbufp, requestp->buflen); + requestp->ret = write(requestp->fd, requestp->bufferp, requestp->buflen); requestp->err = errno; } @@ -698,8 +705,6 @@ { squidaio_request_t *requestp; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); requestp->fd = fd; requestp->resultp = resultp; @@ -724,8 +729,6 @@ { squidaio_request_t *requestp; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); requestp->path = (char *) squidaio_xstrdup(path); requestp->statp = sb; @@ -752,8 +755,6 @@ { squidaio_request_t *requestp; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); requestp->path = squidaio_xstrdup(path); requestp->resultp = resultp; @@ -777,8 +778,6 @@ { squidaio_request_t *requestp; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); requestp->path = (char *) squidaio_xstrdup(path); requestp->offset = length; @@ -808,8 +807,6 @@ squidaio_request_t *requestp; int len; - if (!squidaio_initialised) - squidaio_init(); requestp = memPoolAlloc(squidaio_request_pool); return -1; } Index: squid/src/fs/aufs/async_io.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/async_io.c,v retrieving revision 1.7.6.4.24.1 retrieving revision 1.7.6.4.24.2 diff -u -r1.7.6.4.24.1 -r1.7.6.4.24.2 --- squid/src/fs/aufs/async_io.c 29 Mar 2005 02:27:31 -0000 1.7.6.4.24.1 +++ squid/src/fs/aufs/async_io.c 29 Mar 2005 03:00:35 -0000 1.7.6.4.24.2 @@ -103,6 +103,7 @@ void aioDone(void) { + squidaio_shutdown(); memPoolDestroy(squidaio_ctrl_pool); initialised = 0; } Index: squid/src/fs/aufs/store_asyncufs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_asyncufs.h,v retrieving revision 1.6.14.2.18.1 retrieving revision 1.6.14.2.18.2 diff -u -r1.6.14.2.18.1 -r1.6.14.2.18.2 --- squid/src/fs/aufs/store_asyncufs.h 29 Mar 2005 02:27:32 -0000 1.6.14.2.18.1 +++ squid/src/fs/aufs/store_asyncufs.h 29 Mar 2005 03:00:37 -0000 1.6.14.2.18.2 @@ -41,6 +41,8 @@ typedef void AIOCB(int fd, void *cbdata, const char *buf, int aio_return, int aio_errno); +void squidaio_init(void); +void squidaio_shutdown(void); int squidaio_cancel(squidaio_result_t *); int squidaio_open(const char *, int, mode_t, squidaio_result_t *); int squidaio_read(int, char *, int, off_t, int, squidaio_result_t *); @@ -86,8 +88,9 @@ unsigned int reading:1; unsigned int writing:1; unsigned int opening:1; +#if !ASYNC_WRITE unsigned int write_kicking:1; - unsigned int read_kicking:1; +#endif unsigned int inreaddone:1; } flags; char *read_buf; Index: squid/src/fs/aufs/store_dir_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_dir_aufs.c,v retrieving revision 1.23.6.9.10.1 retrieving revision 1.23.6.9.10.2 diff -u -r1.23.6.9.10.1 -r1.23.6.9.10.2 --- squid/src/fs/aufs/store_dir_aufs.c 29 Mar 2005 02:27:32 -0000 1.23.6.9.10.1 +++ squid/src/fs/aufs/store_dir_aufs.c 29 Mar 2005 03:00:38 -0000 1.23.6.9.10.2 @@ -111,7 +111,6 @@ static QS rev_int_sort; static int storeAufsDirClean(int swap_index); static EVH storeAufsDirCleanEvent; -static int storeAufsDirIs(SwapDir * sd); static int storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2); static int storeAufsCleanupDoubleCheck(SwapDir *, StoreEntry *); static void storeAufsDirStats(SwapDir *, StoreEntry *); @@ -331,10 +330,6 @@ } debug(50, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); aioinfo->swaplog_fd = fd; - if (0 == n_asyncufs_dirs) - assert(NULL == asyncufs_dir_index); - n_asyncufs_dirs++; - assert(n_asyncufs_dirs <= Config.cacheSwap.n_configured); } static void @@ -347,10 +342,6 @@ debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", sd->index, aioinfo->swaplog_fd); aioinfo->swaplog_fd = -1; - n_asyncufs_dirs--; - assert(n_asyncufs_dirs >= 0); - if (0 == n_asyncufs_dirs) - safe_free(asyncufs_dir_index); } static void @@ -364,6 +355,7 @@ storeAufsDirInitBitmap(sd); if (storeAufsDirVerifyCacheDirs(sd) < 0) fatal(errmsg); + squidaio_init(); storeAufsDirOpenSwapLog(sd); storeAufsDirRebuild(sd); if (!started_clean_event) { @@ -1480,55 +1472,38 @@ static void storeAufsDirCleanEvent(void *unused) { - static int swap_index = 0; - int i; + static int swap_index = -1; int j = 0; int n = 0; /* * Assert that there are AUFS cache_dirs configured, otherwise * we should never be called. */ - assert(n_asyncufs_dirs); - if (NULL == asyncufs_dir_index) { + if (swap_index == -1) { SwapDir *sd; squidaioinfo_t *aioinfo; /* - * Initialize the little array that translates AUFS cache_dir - * number into the Config.cacheSwap.swapDirs array index. - */ - asyncufs_dir_index = xcalloc(n_asyncufs_dirs, sizeof(*asyncufs_dir_index)); - for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { - sd = &Config.cacheSwap.swapDirs[i]; - if (!storeAufsDirIs(sd)) - continue; - asyncufs_dir_index[n++] = i; - aioinfo = (squidaioinfo_t *) sd->fsdata; - j += (aioinfo->l1 * aioinfo->l2); - } - assert(n == n_asyncufs_dirs); - /* * Start the storeAufsDirClean() swap_index with a random * value. j equals the total number of AUFS level 2 * swap directories */ + for (n = 0; n < n_asyncufs_dirs; n++) { + sd = &Config.cacheSwap.swapDirs[asyncufs_dir_index[n]]; + aioinfo = (squidaioinfo_t *) sd->fsdata; + j += (aioinfo->l1 * aioinfo->l2); + } swap_index = (int) (squid_random() % j); } if (0 == store_dirs_rebuilding) { n = storeAufsDirClean(swap_index); swap_index++; + if (swap_index < 0) + swap_index = 0; } eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL, 15.0 * exp(-0.25 * n), 1); } -static int -storeAufsDirIs(SwapDir * sd) -{ - if (strncmp(sd->type, "aufs", 4) == 0) - return 1; - return 0; -} - /* * Does swapfile number 'fn' belong in cachedir #F0, * level1 dir #F1, level2 dir #F2? @@ -1942,6 +1917,9 @@ /* Initialise replacement policy stuff */ sd->repl = createRemovalPolicy(Config.replPolicy); + + asyncufs_dir_index = realloc(asyncufs_dir_index, (n_asyncufs_dirs + 1) * sizeof(*asyncufs_dir_index)); + asyncufs_dir_index[n_asyncufs_dirs++] = index; } /* Index: squid/src/fs/aufs/store_io_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/aufs/store_io_aufs.c,v retrieving revision 1.13.2.11.10.1 retrieving revision 1.13.2.11.10.2 diff -u -r1.13.2.11.10.1 -r1.13.2.11.10.2 --- squid/src/fs/aufs/store_io_aufs.c 29 Mar 2005 02:27:52 -0000 1.13.2.11.10.1 +++ squid/src/fs/aufs/store_io_aufs.c 29 Mar 2005 03:00:51 -0000 1.13.2.11.10.2 @@ -393,18 +393,29 @@ } sio->offset += len; #if ASYNC_WRITE - if (!storeAufsKickWriteQueue(sio)) + if (storeAufsKickWriteQueue(sio)) (void) 0; else if (aiostate->flags.close_request) storeAufsIOCallback(sio, errflag); #else + /* loop around storeAufsKickWriteQueue to break recursion stack + * overflow when large amounts of data has been queued for write. + * As writes are blocking here we immediately get called again + * without going via the I/O event loop.. + */ if (!aiostate->flags.write_kicking) { + /* cbdataLock to protect us from the storeAufsIOCallback on error above */ + cbdataLock(sio); aiostate->flags.write_kicking = 1; while (storeAufsKickWriteQueue(sio)) - (void) 0; - aiostate->flags.write_kicking = 0; - if (aiostate->flags.close_request) - storeAufsIOCallback(sio, errflag); + if (!cbdataValid(sio)) + break; + if (cbdataValid(sio)) { + aiostate->flags.write_kicking = 0; + if (aiostate->flags.close_request) + storeAufsIOCallback(sio, errflag); + } + cbdataUnlock(sio); } #endif loop_detect--; @@ -433,8 +444,13 @@ if (fd < 0) return; debug(79, 9) ("%s:%d\n", __FILE__, __LINE__); +#if ASYNC_CLOSE aioClose(fd); fd_close(fd); +#else + aioCancel(fd); + file_close(fd); +#endif store_open_disk_fd--; statCounter.syscalls.disk.closes++; debug(79, 9) ("%s:%d\n", __FILE__, __LINE__);