--------------------- PatchSet 6550 Date: 2005/03/26 11:11:18 Author: serassio Branch: nt-2_5 Tag: (none) Log: Updated awin32 sources with lfs changes Members: src/fs/awin32/async_io.c:1.1.62.7->1.1.62.8 src/fs/awin32/store_asyncufs.h:1.1.62.8->1.1.62.9 src/fs/awin32/store_dir_aufs.c:1.1.62.17->1.1.62.18 src/fs/awin32/store_io_aufs.c:1.1.62.17->1.1.62.18 Index: squid/src/fs/awin32/async_io.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/awin32/Attic/async_io.c,v retrieving revision 1.1.62.7 retrieving revision 1.1.62.8 diff -u -r1.1.62.7 -r1.1.62.8 --- squid/src/fs/awin32/async_io.c 20 Mar 2005 10:24:27 -0000 1.1.62.7 +++ squid/src/fs/awin32/async_io.c 26 Mar 2005 11:11:18 -0000 1.1.62.8 @@ -1,6 +1,6 @@ /* - * $Id: async_io.c,v 1.1.62.7 2005/03/20 10:24:27 serassio Exp $ + * $Id: async_io.c,v 1.1.62.8 2005/03/26 11:11:18 serassio Exp $ * * DEBUG: section 32 Asynchronous Disk I/O * AUTHOR: Pete Bentley @@ -186,7 +186,7 @@ void -aioWrite(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func) +aioWrite(int fd, off_t offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func) { squidaio_ctrl_t *ctrlp; int seekmode; @@ -214,7 +214,7 @@ void -aioRead(int fd, int offset, int len, AIOCB * callback, void *callback_data) +aioRead(int fd, off_t offset, int len, AIOCB * callback, void *callback_data) { squidaio_ctrl_t *ctrlp; int seekmode; Index: squid/src/fs/awin32/store_asyncufs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/awin32/Attic/store_asyncufs.h,v retrieving revision 1.1.62.8 retrieving revision 1.1.62.9 diff -u -r1.1.62.8 -r1.1.62.9 --- squid/src/fs/awin32/store_asyncufs.h 20 Mar 2005 10:24:27 -0000 1.1.62.8 +++ squid/src/fs/awin32/store_asyncufs.h 26 Mar 2005 11:11:20 -0000 1.1.62.9 @@ -65,8 +65,8 @@ void aioCancel(int); void aioOpen(const char *, int, mode_t, AIOCB *, void *); void aioClose(int); -void aioWrite(int, int offset, char *, int size, AIOCB *, void *, FREE *); -void aioRead(int, int offset, int size, AIOCB *, void *); +void aioWrite(int, off_t offset, char *, int size, AIOCB *, void *, FREE *); +void aioRead(int, off_t offset, int size, AIOCB *, void *); void aioStat(char *, struct stat *, AIOCB *, void *); void aioUnlink(const char *, AIOCB *, void *); void aioTruncate(const char *, off_t length, AIOCB *, void *); Index: squid/src/fs/awin32/store_dir_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/awin32/Attic/store_dir_aufs.c,v retrieving revision 1.1.62.17 retrieving revision 1.1.62.18 diff -u -r1.1.62.17 -r1.1.62.18 --- squid/src/fs/awin32/store_dir_aufs.c 20 Mar 2005 10:24:27 -0000 1.1.62.17 +++ squid/src/fs/awin32/store_dir_aufs.c 26 Mar 2005 11:11:20 -0000 1.1.62.18 @@ -1,6 +1,6 @@ /* - * $Id: store_dir_aufs.c,v 1.1.62.17 2005/03/20 10:24:27 serassio Exp $ + * $Id: store_dir_aufs.c,v 1.1.62.18 2005/03/26 11:11:20 serassio Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -82,7 +82,7 @@ static int storeAufsDirGetNextFile(RebuildState *, sfileno *, int *size); static StoreEntry *storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key, sfileno file_number, - size_t swap_file_sz, + squid_file_sz swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -370,6 +370,24 @@ } static void +storeAufsDirRebuildComplete(RebuildState * rb) +{ + if (rb->log) { + debug(47, 1) ("Done reading %s swaplog (%d entries)\n", + rb->sd->path, rb->n_read); + fclose(rb->log); + rb->log = NULL; + } else { + debug(47, 1) ("Done scanning %s (%d entries)\n", + rb->sd->path, rb->counts.scancount); + } + store_dirs_rebuilding--; + storeAufsDirCloseTmpSwapLog(rb->sd); + storeRebuildComplete(&rb->counts); + cbdataFree(rb); +} + +static void storeAufsDirRebuildFromDirectory(void *data) { RebuildState *rb = data; @@ -392,12 +410,7 @@ assert(fd == -1); fd = storeAufsDirGetNextFile(rb, &filn, &size); if (fd == -2) { - debug(47, 1) ("Done scanning %s swaplog (%d entries)\n", - rb->sd->path, rb->n_read); - store_dirs_rebuilding--; - storeAufsDirCloseTmpSwapLog(rb->sd); - storeRebuildComplete(&rb->counts); - cbdataFree(rb); + storeAufsDirRebuildComplete(rb); return; } else if (fd < 0) { continue; @@ -449,10 +462,39 @@ assert(t->length == MD5_DIGEST_CHARS); xmemcpy(key, t->value, MD5_DIGEST_CHARS); break; +#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T case STORE_META_STD: assert(t->length == STORE_HDR_METASIZE); xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); break; +#else + case STORE_META_STD_LFS: + assert(t->length == STORE_HDR_METASIZE); + xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); + break; + case STORE_META_STD: + assert(t->length == STORE_HDR_METASIZE_OLD); + { + struct { + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + size_t swap_file_sz; + u_short refcount; + u_short flags; + } *tmp = t->value; + assert(sizeof(*tmp) == STORE_HDR_METASIZE_OLD); + tmpe.timestamp = tmp->timestamp; + tmpe.lastref = tmp->lastref; + tmpe.expires = tmp->expires; + tmpe.lastmod = tmp->lastmod; + tmpe.swap_file_sz = tmp->swap_file_sz; + tmpe.refcount = tmp->refcount; + tmpe.flags = tmp->flags; + } + break; +#endif default: break; } @@ -467,9 +509,9 @@ tmpe.hash.key = key; /* check sizes */ if (tmpe.swap_file_sz == 0) { - tmpe.swap_file_sz = (size_t) sb.st_size; + tmpe.swap_file_sz = sb.st_size; } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { - tmpe.swap_file_sz = (size_t) sb.st_size; + tmpe.swap_file_sz = sb.st_size; } else if (tmpe.swap_file_sz != sb.st_size) { debug(47, 1) ("storeAufsDirRebuildFromDirectory: SIZE MISMATCH %ld!=%ld\n", (long int) tmpe.swap_file_sz, (long int) sb.st_size); @@ -526,21 +568,178 @@ /* load a number of objects per invocation */ for (count = 0; count < rb->speed; count++) { if (fread(&s, ss, 1, rb->log) != 1) { - debug(47, 1) ("Done reading %s swaplog (%d entries)\n", - rb->sd->path, rb->n_read); - fclose(rb->log); - rb->log = NULL; - store_dirs_rebuilding--; - storeAufsDirCloseTmpSwapLog(rb->sd); - storeRebuildComplete(&rb->counts); - cbdataFree(rb); + storeAufsDirRebuildComplete(rb); return; } rb->n_read++; - if (s.op <= SWAP_LOG_NOP) + /* + * BC: during 2.4 development, we changed the way swap file + * numbers are assigned and stored. The high 16 bits used + * to encode the SD index number. There used to be a call + * to storeDirProperFileno here that re-assigned the index + * bits. Now, for backwards compatibility, we just need + * to mask it off. + */ + s.swap_filen &= 0x00FFFFFF; + debug(47, 3) ("storeAufsDirRebuildFromSwapLog: %s %s %08X\n", + swap_log_op_str[(int) s.op], + storeKeyText(s.key), + s.swap_filen); + if (s.op == SWAP_LOG_ADD) { + (void) 0; + } else if (s.op == SWAP_LOG_DEL) { + /* Delete unless we already have a newer copy */ + if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) { + /* + * Make sure we don't unlink the file, it might be + * in use by a subsequent entry. Also note that + * we don't have to subtract from store_swap_size + * because adding to store_swap_size happens in + * the cleanup procedure. + */ + storeExpireNow(e); + storeReleaseRequest(e); + if (e->swap_filen > -1) { + storeAufsDirReplRemove(e); + storeAufsDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.objcount--; + rb->counts.cancelcount++; + } + continue; + } else { + x = log(++rb->counts.bad_log_op) / log(10.0); + if (0.0 == x - (double) (int) x) + debug(47, 1) ("WARNING: %d invalid swap log entries found\n", + rb->counts.bad_log_op); + rb->counts.invalid++; + continue; + } + if ((++rb->counts.scancount & 0xFFF) == 0) { + struct stat sb; + if (0 == fstat(fileno(rb->log), &sb)) + storeRebuildProgress(SD->index, + (int) sb.st_size / ss, rb->n_read); + } + if (!storeAufsDirValidFileno(SD, s.swap_filen, 0)) { + rb->counts.invalid++; + continue; + } + if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + rb->counts.badflags++; + continue; + } + e = storeGet(s.key); + used = storeAufsDirMapBitTest(SD, s.swap_filen); + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; + if (used && !disk_entry_newer) { + /* log entry is old, ignore it */ + rb->counts.clashcount++; + continue; + } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) { + /* swapfile taken, same URL, newer, update meta */ + if (e->store_status == STORE_OK) { + e->lastref = s.timestamp; + e->timestamp = s.timestamp; + e->expires = s.expires; + e->lastmod = s.lastmod; + e->flags = s.flags; + e->refcount += s.refcount; + storeAufsDirUnrefObj(SD, e); + } else { + debug_trap("storeAufsDirRebuildFromSwapLog: bad condition"); + debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__); + } + continue; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", + SD->index, s.swap_filen); + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + assert(rb->flags.need_to_validate); + rb->counts.clashcount++; continue; - if (s.op >= SWAP_LOG_MAX) + } else if (e && !disk_entry_newer) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + rb->counts.dupcount++; continue; + } else if (e) { + /* key already exists, this swapfile not being used */ + /* junk old, load new */ + storeExpireNow(e); + storeReleaseRequest(e); + if (e->swap_filen > -1) { + storeAufsDirReplRemove(e); + /* Make sure we don't actually unlink the file */ + storeAufsDirMapBitReset(SD, e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + } + storeRelease(e); + rb->counts.dupcount++; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + /* update store_swap_size */ + rb->counts.objcount++; + e = storeAufsDirAddDiskRestore(SD, s.key, + s.swap_filen, + s.swap_file_sz, + s.expires, + s.timestamp, + s.lastref, + s.lastmod, + s.refcount, + s.flags, + (int) rb->flags.clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1); +} + +#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T +/* This is an exact copy of the above, but using storeSwapLogDataOld entry type */ +static void +storeAufsDirRebuildFromSwapLogOld(void *data) +{ + RebuildState *rb = data; + SwapDir *SD = rb->sd; + StoreEntry *e = NULL; + storeSwapLogDataOld s; + size_t ss = sizeof(storeSwapLogDataOld); + int count; + int used; /* is swapfile already in use? */ + int disk_entry_newer; /* is the log entry newer than current entry? */ + double x; + assert(rb != NULL); + /* load a number of objects per invocation */ + for (count = 0; count < rb->speed; count++) { + if (fread(&s, ss, 1, rb->log) != 1) { + storeAufsDirRebuildComplete(rb); + return; + } + rb->n_read++; /* * BC: during 2.4 development, we changed the way swap file * numbers are assigned and stored. The high 16 bits used @@ -684,7 +883,49 @@ (int) rb->flags.clean); storeDirSwapLog(e, SWAP_LOG_ADD); } + eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1); +} + +#endif + +static void +storeAufsDirRebuildFromSwapLogCheckVersion(void *data) +{ + RebuildState *rb = data; + storeSwapLogHeader hdr; + + if (fread(&hdr, sizeof(hdr), 1, rb->log) != 1) { + storeAufsDirRebuildComplete(rb); + return; + } + if (hdr.op == SWAP_LOG_VERSION) { + if (fseek(rb->log, hdr.record_size, SEEK_SET) != 0) { + storeAufsDirRebuildComplete(rb); + return; + } + if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogData)) { + eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1); + return; + } +#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T + if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogDataOld)) { + debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Found current version but without large file support. Upgrading\n"); + eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1); + return; + } +#endif + debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Unsupported swap.state version %d size %d\n", + hdr.version, hdr.record_size); + storeAufsDirRebuildComplete(rb); + return; + } + rewind(rb->log); + debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Old version detected. Upgrading\n"); +#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1); +#else + eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1); +#endif } static int @@ -779,7 +1020,7 @@ static StoreEntry * storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key, sfileno file_number, - size_t swap_file_sz, + squid_file_sz swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -843,7 +1084,7 @@ fclose(fp); func = storeAufsDirRebuildFromDirectory; } else { - func = storeAufsDirRebuildFromSwapLog; + func = storeAufsDirRebuildFromSwapLogCheckVersion; rb->log = fp; rb->flags.clean = (unsigned int) clean; } @@ -888,6 +1129,10 @@ struct stat clean_sb; FILE *fp; int fd; + union { + storeSwapLogHeader hdr; + storeSwapLogData _data; + } hd; if (stat(swaplog_path, &log_sb) < 0) { debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index); safe_free(swaplog_path); @@ -905,6 +1150,14 @@ debug(50, 1) ("%s: %s\n", new_path, xstrerror()); fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); } + memset(&hd, 0, sizeof(hd)); + hd.hdr.op = SWAP_LOG_VERSION; + hd.hdr.version = 1; + hd.hdr.record_size = sizeof(storeSwapLogData); + if (write(fd, &hd, sizeof(hd)) != sizeof(hd)) { + debug(50, 1) ("%s: %s\n", new_path, xstrerror()); + fatal("storeDirOpenTmpSwapLog: Failed to write swap log header."); + } aioinfo->swaplog_fd = fd; /* open a read-only stream of the old log */ fp = fopen(swaplog_path, "rb"); @@ -931,7 +1184,7 @@ char *new; char *cln; char *outbuf; - off_t outbuf_offset; + int outbuf_offset; int fd; RemovalPolicyWalker *walker; }; @@ -946,6 +1199,10 @@ storeAufsDirWriteCleanStart(SwapDir * sd) { struct _clean_state *state = xcalloc(1, sizeof(*state)); + union { + storeSwapLogHeader hdr; + storeSwapLogData _data; + } hd; #if HAVE_FCHMOD struct stat sb; #endif @@ -962,6 +1219,14 @@ xfree(state); return -1; } + memset(&hd, 0, sizeof(hd)); + hd.hdr.op = SWAP_LOG_VERSION; + hd.hdr.version = 1; + hd.hdr.record_size = sizeof(storeSwapLogData); + if (write(state->fd, &hd, sizeof(hd)) != sizeof(hd)) { + debug(50, 1) ("%s: %s\n", state->new, xstrerror()); + fatal("storeAufsDirWriteCleanStart: Failed to write swap log header."); + } state->cur = xstrdup(storeAufsDirSwapLogFile(sd, NULL)); state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean")); state->outbuf = xcalloc(CLEAN_BUF_SZ, 1); Index: squid/src/fs/awin32/store_io_aufs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/awin32/Attic/store_io_aufs.c,v retrieving revision 1.1.62.17 retrieving revision 1.1.62.18 diff -u -r1.1.62.17 -r1.1.62.18 --- squid/src/fs/awin32/store_io_aufs.c 20 Mar 2005 10:24:28 -0000 1.1.62.17 +++ squid/src/fs/awin32/store_io_aufs.c 26 Mar 2005 11:11:20 -0000 1.1.62.18 @@ -1,6 +1,6 @@ /* - * $Id: store_io_aufs.c,v 1.1.62.17 2005/03/20 10:24:28 serassio Exp $ + * $Id: store_io_aufs.c,v 1.1.62.18 2005/03/26 11:11:20 serassio Exp $ * * DEBUG: section 79 Storage Manager awin32 Interface * AUTHOR: Robert Collins @@ -183,7 +183,7 @@ /* Read */ void -storeAufsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +storeAufsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, STRCB * callback, void *callback_data) { squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; assert(sio->read.callback == NULL); @@ -197,7 +197,7 @@ q = memPoolAlloc(aufs_qread_pool); q->buf = buf; q->size = size; - q->offset = offset; + q->offset = (off_t) offset; q->callback = callback; q->callback_data = callback_data; cbdataLock(q->callback_data); @@ -213,10 +213,10 @@ sio->offset = offset; aiostate->flags.reading = 1; #if ASYNC_READ - aioRead(aiostate->fd, offset, size, storeAufsReadDone, sio); + aioRead(aiostate->fd, (off_t) offset, size, storeAufsReadDone, sio); statCounter.syscalls.disk.reads++; #else - file_read(aiostate->fd, buf, size, offset, storeAufsReadDone, sio); + file_read(aiostate->fd, buf, size, (off_t) offset, storeAufsReadDone, sio); /* file_read() increments syscalls.disk.reads */ #endif } @@ -224,7 +224,7 @@ /* Write */ void -storeAufsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +storeAufsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, FREE * free_func) { squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate; debug(79, 3) ("storeAufsWrite: dirno %d, fileno %08X, FD %d\n", @@ -236,7 +236,7 @@ q = memPoolAlloc(aufs_qwrite_pool); q->buf = buf; q->size = size; - q->offset = offset; + q->offset = (off_t) offset; q->free_func = free_func; linklistPush(&(aiostate->pending_writes), q); return; @@ -248,17 +248,17 @@ q = memPoolAlloc(aufs_qwrite_pool); q->buf = buf; q->size = size; - q->offset = offset; + q->offset = (off_t) offset; q->free_func = free_func; linklistPush(&(aiostate->pending_writes), q); return; } aiostate->flags.writing = 1; - aioWrite(aiostate->fd, offset, buf, size, storeAufsWriteDone, sio, + aioWrite(aiostate->fd, (off_t) offset, buf, size, storeAufsWriteDone, sio, free_func); statCounter.syscalls.disk.writes++; #else - file_write(aiostate->fd, offset, buf, size, storeAufsWriteDone, sio, + file_write(aiostate->fd, (off_t) offset, buf, size, storeAufsWriteDone, sio, free_func); /* file_write() increments syscalls.disk.writes */ #endif @@ -360,7 +360,7 @@ debug(79, 3) ("storeAufsReadDone: got failure (%d)\n", errflag); rlen = -1; } else { - rlen = (ssize_t) len; + rlen = len; sio->offset += len; } #if ASYNC_READ