This patch is generated from the coss branch of HEAD in squid Tue Aug 17 18:58:13 2004 GMT See http://devel.squid-cache.org/ Index: squid/configure.in diff -u squid/configure.in:1.76 squid/configure.in:1.30.2.15 --- squid/configure.in:1.76 Tue Oct 22 01:35:46 2002 +++ squid/configure.in Sat Oct 26 00:45:39 2002 @@ -370,6 +370,8 @@ fi ;; coss) + echo "coss store used, additional large file support enabled" + CFLAGS="$CFLAGS -D_LARGEFILE64_SOURCE" if test -z "$with_aio"; then echo "coss store used, aio support automatically enabled" with_aio=yes Index: squid/src/client_side_request.c diff -u squid/src/client_side_request.c:1.6 squid/src/client_side_request.c:1.3.8.2 --- squid/src/client_side_request.c:1.6 Thu Oct 3 05:55:28 2002 +++ squid/src/client_side_request.c Sun Oct 13 02:53:36 2002 @@ -1,10 +1,10 @@ /* * $Id: squid-coss-HEAD,v 1.2 2004/09/29 00:22:50 hno Exp $ - * - * DEBUG: section 85 Client-side Request Routines AUTHOR: Robert Collins - * (Originally Duane Wessels in client_side.c) - * + * + * DEBUG: section 85 Client-side Request Routines + * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c) + * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- * Index: squid/src/main.c diff -u squid/src/main.c:1.38 squid/src/main.c:1.27.2.9 --- squid/src/main.c:1.38 Sun Oct 13 14:46:16 2002 +++ squid/src/main.c Sat Oct 26 00:45:43 2002 @@ -956,8 +956,8 @@ fwdUninit(); #endif storeDirSync(); /* Flush log close */ -#if PURIFY || XMALLOC_TRACE storeFsDone(); +#if PURIFY || XMALLOC_TRACE configFreeMemory(); storeFreeMemory(); /*stmemFreeMemory(); */ Index: squid/src/fs/coss/async_io.c diff -u squid/src/fs/coss/async_io.c:1.6 squid/src/fs/coss/async_io.c:1.3.22.5 --- squid/src/fs/coss/async_io.c:1.6 Sat Jul 20 18:06:08 2002 +++ squid/src/fs/coss/async_io.c Fri Jul 26 03:35:26 2002 @@ -18,7 +18,7 @@ #include #include -#include "async_io.h" +#include "store_coss.h" /* * For the time being, we kinda don't need to have our own @@ -49,12 +49,10 @@ } - - /* Exported routines */ void -a_file_read(async_queue_t * q, int fd, void *buf, int req_len, off_t offset, +a_file_read(async_queue_t * q, int fd, void *buf, size_t req_len, coss_off_t offset, DRCB * callback, void *data) { int slot; @@ -62,9 +60,6 @@ assert(q->aq_state == AQ_STATE_SETUP); -#if 0 - file_read(fd, buf, req_len, offset, callback, data); -#endif /* Find a free slot */ slot = a_file_findslot(q); if (slot < 0) { @@ -90,14 +85,14 @@ q->aq_numpending++; /* Initiate aio */ - if (aio_read(&qe->aq_e_aiocb) < 0) { + if (COSS_AIO_READ(&qe->aq_e_aiocb) < 0) { fatalf("Aiee! aio_read() returned error (%d)!\n", errno); } } void -a_file_write(async_queue_t * q, int fd, off_t offset, void *buf, int len, +a_file_write(async_queue_t * q, int fd, coss_off_t offset, void *buf, size_t len, DWCB * callback, void *data, FREE * freefunc) { int slot; @@ -105,9 +100,6 @@ assert(q->aq_state == AQ_STATE_SETUP); -#if 0 - file_write(fd, offset, buf, len, callback, data, freefunc); -#endif /* Find a free slot */ slot = a_file_findslot(q); if (slot < 0) { @@ -133,9 +125,8 @@ q->aq_numpending++; /* Initiate aio */ - if (aio_write(&qe->aq_e_aiocb) < 0) { + if (COSS_AIO_WRITE(&qe->aq_e_aiocb) < 0) { fatalf("Aiee! aio_read() returned error (%d)!\n", errno); - assert(1 == 0); } } @@ -171,13 +162,13 @@ if (q->aq_queue[i].aq_e_state == AQ_ENTRY_USED) { aqe = &q->aq_queue[i]; /* Active, get status */ - reterr = aio_error(&aqe->aq_e_aiocb); + reterr = COSS_AIO_ERROR(&aqe->aq_e_aiocb); if (reterr < 0) { fatal("aio_error returned an error!\n"); } if (reterr != EINPROGRESS) { /* Get the return code */ - retval = aio_return(&aqe->aq_e_aiocb); + retval = COSS_AIO_RETURN(&aqe->aq_e_aiocb); /* Get the callback parameters */ freefunc = aqe->aq_e_free; Index: squid/src/fs/coss/async_io.h diff -u squid/src/fs/coss/async_io.h:1.2 squid/src/fs/coss/async_io.h:1.2.32.4 --- squid/src/fs/coss/async_io.h:1.2 Thu Aug 16 00:39:04 2001 +++ squid/src/fs/coss/async_io.h Fri Jul 26 04:38:14 2002 @@ -27,7 +27,7 @@ struct _async_queue_entry { async_queue_entry_state_t aq_e_state; async_queue_entry_type_t aq_e_type; - struct aiocb aq_e_aiocb; + struct COSS_AIOCB aq_e_aiocb; union { DRCB *read; DWCB *write; @@ -47,10 +47,8 @@ /* Functions */ -extern void a_file_read(async_queue_t * q, int fd, void *buf, int req_len, - off_t offset, DRCB * callback, void *data); -extern void a_file_write(async_queue_t * q, int fd, off_t offset, void *buf, - int len, DWCB * callback, void *data, FREE * freefunc); +extern void a_file_read(async_queue_t *, int, void *, size_t, coss_off_t, DRCB *, void *); +extern void a_file_write(async_queue_t *, int, coss_off_t, void *, size_t, DWCB *, void *, FREE *); extern int a_file_callback(async_queue_t * q); extern void a_file_setupqueue(async_queue_t * q); extern void a_file_syncqueue(async_queue_t * q); Index: squid/src/fs/coss/store_coss.h diff -u squid/src/fs/coss/store_coss.h:1.4 squid/src/fs/coss/store_coss.h:1.4.12.7 --- squid/src/fs/coss/store_coss.h:1.4 Sun Aug 12 08:20:29 2001 +++ squid/src/fs/coss/store_coss.h Fri Jul 26 04:38:14 2002 @@ -1,17 +1,79 @@ #ifndef __COSS_H__ #define __COSS_H__ +/* Define COSS_OFFSET_BITS if not already defined according to _FILE_OFFSET_BITS + Needed for large file support +*/ + +#if (_FILE_OFFSET_BITS == 64) || defined(_LARGEFILE64_SOURCE) +#define COSS_OFFSET_BITS 64 +#endif + +#ifndef COSS_OFFSET_BITS +#define COSS_OFFSET_BITS 32 +#endif + +#if COSS_OFFSET_BITS == 64 /* We are in 64 bit I/O mode */ + +typedef int64_t size64_t; + +#ifndef off64_t /* define off64_t if not already defined */ +typedef int64_t off64_t; +#endif + +typedef off64_t coss_off_t; +typedef size64_t coss_size_t; + +#ifdef _SQUID_MSWIN_ +#define COSSFMT64 "I64" +#else +#define COSSFMT64 "ll" +#endif /* _SQUID_MSWIN_ */ + +#ifdef _SQUID_FREEBSD_ /* FreeBSD doesn't have aio_*64 */ +#define COSS_AIO_ERROR aio_error +#define COSS_AIO_READ aio_read +#define COSS_AIO_RETURN aio_return +#define COSS_AIO_WRITE aio_write + +#define COSS_AIOCB aiocb +#else +#define COSS_AIO_ERROR aio_error64 +#define COSS_AIO_READ aio_read64 +#define COSS_AIO_RETURN aio_return64 +#define COSS_AIO_WRITE aio_write64 + +#define COSS_AIOCB aiocb64 +#endif + +#else /* We are in 32 bit I/O mode */ + +typedef off_t off64_t; +typedef size_t size64_t; +typedef off_t coss_off_t; +typedef size_t coss_size_t; + +#define COSSFMT64 "l" + +#define COSS_AIO_ERROR aio_error +#define COSS_AIO_READ aio_read +#define COSS_AIO_RETURN aio_return +#define COSS_AIO_WRITE aio_write + +#define COSS_AIOCB aiocb +#endif /* COSS_OFFSET_BITS == 64 */ + #ifndef COSS_MEMBUF_SZ #define COSS_MEMBUF_SZ 1048576 #endif #ifndef COSS_BLOCK_SZ -#define COSS_BLOCK_SZ 512 +#define COSS_BLOCK_SZ 1024 #endif /* Macros to help block<->offset transiting */ -#define COSS_OFS_TO_BLK(ofs) ((ofs) / COSS_BLOCK_SZ) -#define COSS_BLK_TO_OFS(ofs) ((ofs) * COSS_BLOCK_SZ) +#define COSS_OFS_TO_BLK(ofs) (sfileno)((ofs) / COSS_BLOCK_SZ) +#define COSS_BLK_TO_OFS(ofs) ((coss_off_t)(ofs) * COSS_BLOCK_SZ) /* Note that swap_filen in sio/e are actually disk offsets too! */ @@ -20,10 +82,14 @@ #define COSS_ALLOC_ALLOCATE 1 #define COSS_ALLOC_REALLOC 2 +#define SWAPDIR_COSS "coss" + +#include "async_io.h" + struct _cossmembuf { dlink_node node; - size_t diskstart; /* in blocks */ - size_t diskend; /* in blocks */ + coss_off_t diskstart; /* actual offset */ + coss_off_t diskend; /* actual offset */ SwapDir *SD; int lockcount; char buffer[COSS_MEMBUF_SZ]; @@ -38,7 +104,7 @@ struct _cossinfo { dlink_list membufs; struct _cossmembuf *current_membuf; - size_t current_offset; /* in Blocks */ + coss_off_t current_offset; /* in Blocks */ int fd; int swaplog_fd; int numcollisions; @@ -61,8 +127,8 @@ char *readbuffer; char *requestbuf; size_t requestlen; - size_t requestoffset; /* in blocks */ - sfileno reqdiskoffset; /* in blocks */ + size_t requestoffset; /* in bytes */ + coss_off_t reqdiskoffset; /* in bytes */ struct { unsigned int reading:1; unsigned int writing:1; @@ -91,7 +157,7 @@ extern STOBJUNLINK storeCossUnlink; extern STSYNC storeCossSync; -extern off_t storeCossAllocate(SwapDir * SD, const StoreEntry * e, int which); +extern coss_off_t storeCossAllocate(SwapDir *, const StoreEntry *, int); extern void storeCossAdd(SwapDir *, StoreEntry *); extern void storeCossRemove(SwapDir *, StoreEntry *); extern void storeCossStartMembuf(SwapDir * SD); Index: squid/src/fs/coss/store_dir_coss.c diff -u squid/src/fs/coss/store_dir_coss.c:1.24 squid/src/fs/coss/store_dir_coss.c:1.16.10.7 --- squid/src/fs/coss/store_dir_coss.c:1.24 Fri Aug 9 14:46:02 2002 +++ squid/src/fs/coss/store_dir_coss.c Sat Aug 10 08:43:48 2002 @@ -36,13 +36,12 @@ #include "squid.h" #include -#include "async_io.h" #include "store_coss.h" #define STORE_META_BUFSZ 4096 int n_coss_dirs = 0; -/* static int last_coss_pick_index = -1; */ +static int last_coss_pick_index = -1; int coss_initialised = 0; MemPool *coss_state_pool = NULL; MemPool *coss_index_pool = NULL; @@ -100,6 +99,7 @@ LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); LOCAL_ARRAY(char, digit, 32); char *pathtmp2; + int i; if (Config.Log.swap) { xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); pathtmp2 = pathtmp; @@ -116,6 +116,12 @@ } } else { xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + for (i = strlen(path); i >= 0; i--) { + if (path[i] == '/') { + path[i] = '\0'; + break; + } + } strcat(path, "/swap.state"); } if (ext) @@ -158,13 +164,18 @@ a_file_setupqueue(&cs->aq); storeCossDirOpenSwapLog(sd); storeCossDirRebuild(sd); +#if defined(_SQUID_MSWIN_) || defined(_SQUID_CYGWIN_) + cs->fd = aio_open(sd->path, O_RDWR | O_CREAT); +#else cs->fd = file_open(sd->path, O_RDWR | O_CREAT); +#endif if (cs->fd < 0) { debug(47, 1) ("%s: %s\n", sd->path, xstrerror()); fatal("storeCossDirInit: Failed to open a COSS directory."); } n_coss_dirs++; - (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize); + sd->fs.blksize = COSS_BLOCK_SZ; + // (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize); } void @@ -211,6 +222,13 @@ int count; double x; assert(rb != NULL); + + /* This rebuild currently doesn't work! -- ac */ + fclose(rb->log); + rb->log = NULL; + storeCossRebuildComplete(rb); + return; + /* load a number of objects per invocation */ for (count = 0; count < rb->speed; count++) { if (fread(&s, ss, 1, rb->log) != 1) { @@ -334,7 +352,7 @@ EBIT_CLR(e->flags, ENTRY_VALIDATED); storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ storeCossAdd(SD, e); - e->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY); + e->swap_filen = (sfileno) storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY); return e; } @@ -390,7 +408,7 @@ char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); int fd; file_close(cs->swaplog_fd); -#if defined (_SQUID_OS2_) || defined (_SQUID_CYGWIN_) +#if defined (_SQUID_OS2_) || defined (_SQUID_CYGWIN_) || defined (_SQUID_MSWIN_) if (unlink(swaplog_path) < 0) { debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); fatal("storeCossDirCloseTmpSwapLog: unlink failed"); @@ -592,7 +610,7 @@ fd = state->fd; /* rename */ if (state->fd >= 0) { -#if defined(_SQUID_OS2_) || defined (_SQUID_CYGWIN_) +#if defined(_SQUID_OS2_) || defined (_SQUID_CYGWIN_) || defined (_SQUID_MSWIN_) file_close(state->fd); state->fd = -1; if (unlink(state->cur) < 0) @@ -664,13 +682,18 @@ storeCossSync(SD); /* This'll call a_file_syncqueue() */ a_file_closequeue(&cs->aq); +#if defined(_SQUID_MSWIN_) || (_SQUID_CYGWIN_) + aio_close(cs->fd); +#else file_close(cs->fd); +#endif cs->fd = -1; if (cs->swaplog_fd > -1) { file_close(cs->swaplog_fd); cs->swaplog_fd = -1; } + xfree(cs); n_coss_dirs--; } @@ -716,12 +739,19 @@ storeCossDirStats(SwapDir * SD, StoreEntry * sentry) { CossInfo *cs = (CossInfo *) SD->fsdata; + double size; storeAppendPrintf(sentry, "\n"); storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", 100.0 * SD->cur_size / SD->max_size); + size = (cs->count ? SD->cur_size / cs->count : 0); + storeAppendPrintf(sentry, "Maximum Object Size: %7d\n", SD->max_objsize); + storeAppendPrintf(sentry, "Average Object Size: %7.0f\n", size * 1024); + storeAppendPrintf(sentry, "Current #of Objects: %7d\n", cs->count); + storeAppendPrintf(sentry, "Estimate maxObjects: %7.0f\n", (size ? SD->max_size / size : 0)); + storeAppendPrintf(sentry, "Number of object collisions: %d\n", (int) cs->numcollisions); #if 0 /* is this applicable? I Hope not .. */ @@ -750,9 +780,9 @@ if (size <= 0) fatal("storeCossDirParse: invalid size value"); - cs = xmalloc(sizeof(CossInfo)); + cs = xcalloc(1, sizeof(CossInfo)); if (cs == NULL) - fatal("storeCossDirParse: couldn't xmalloc() CossInfo!\n"); + fatal("storeCossDirParse: couldn't xcalloc() CossInfo!\n"); sd->index = index; sd->path = xstrdup(path); @@ -837,7 +867,6 @@ dump_cachedir_options(entry, NULL, s); } -#if OLD_UNUSED_CODE SwapDir * storeCossDirPick(void) { @@ -848,7 +877,7 @@ return NULL; for (i = 0; i < Config.cacheSwap.n_configured; i++) { SD = &Config.cacheSwap.swapDirs[i]; - if (SD->type == SWAPDIR_COSS) { + if (strcmp(SD->type, SWAPDIR_COSS) == 0) { if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { last_coss_pick_index = i; return SD; @@ -862,7 +891,7 @@ } for (i = 0; i < Config.cacheSwap.n_configured; i++) { SD = &Config.cacheSwap.swapDirs[i]; - if (SD->type == SWAPDIR_COSS) { + if (strcmp(SD->type, SWAPDIR_COSS) == 0) { if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { last_coss_pick_index = i; return SD; @@ -876,7 +905,6 @@ } return NULL; } -#endif /* * initial setup/done code @@ -884,6 +912,10 @@ static void storeCossDirDone(void) { + int i; + + for (i=0; i < n_coss_dirs; i++) + storeCossDirShutdown(storeCossDirPick()); memPoolDestroy(&coss_state_pool); /* memPoolDestroy(&coss_index_pool); XXX Should be here? */ coss_initialised = 0; Index: squid/src/fs/coss/store_io_coss.c diff -u squid/src/fs/coss/store_io_coss.c:1.14 squid/src/fs/coss/store_io_coss.c:1.8.12.10 --- squid/src/fs/coss/store_io_coss.c:1.14 Thu Aug 8 13:15:20 2002 +++ squid/src/fs/coss/store_io_coss.c Sat Aug 10 08:43:48 2002 @@ -35,24 +35,27 @@ #include "squid.h" #include -#include "async_io.h" #include "store_coss.h" static DWCB storeCossWriteMemBufDone; static DRCB storeCossReadDone; static void storeCossIOCallback(storeIOState * sio, int errflag); -static char *storeCossMemPointerFromDiskOffset(SwapDir * SD, size_t offset, CossMemBuf ** mb); +static char *storeCossMemPointerFromDiskOffset(SwapDir *, coss_size_t, CossMemBuf **); static void storeCossMemBufLock(SwapDir * SD, storeIOState * e); static void storeCossMemBufUnlock(SwapDir * SD, storeIOState * e); static void storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t); static void storeCossWriteMemBufDone(int fd, int errflag, size_t len, void *my_data); -static CossMemBuf *storeCossCreateMemBuf(SwapDir * SD, size_t start, - sfileno curfn, int *collision); +static CossMemBuf *storeCossCreateMemBuf(SwapDir *, coss_size_t, sfileno, int *); static CBDUNL storeCossIOFreeEntry; CBDATA_TYPE(storeIOState); CBDATA_TYPE(CossMemBuf); +#if 0 +#undef xmemcpy +#define xmemcpy(dst,src,len) { debug(81,1) ("Line %d: memcpy(%08X, %08X, %d)\n", __LINE__,dst,src,len); memcpy(dst,src,len); } while(0); +#endif + /* === PUBLIC =========================================================== */ /* @@ -62,13 +65,13 @@ * to work.. * -- Adrian */ -off_t +coss_off_t storeCossAllocate(SwapDir * SD, const StoreEntry * e, int which) { CossInfo *cs = (CossInfo *) SD->fsdata; CossMemBuf *newmb; - off_t retofs; - size_t allocsize; + coss_off_t retofs; + coss_size_t allocsize; int coll = 0; sfileno checkf; @@ -78,18 +81,21 @@ else checkf = -1; - retofs = e->swap_filen; /* Just for defaults, or while rebuilding */ + retofs = COSS_BLK_TO_OFS(e->swap_filen); /* Just for defaults, or while rebuilding */ if (e->swap_file_sz > 0) allocsize = e->swap_file_sz; else allocsize = objectLen(e) + e->mem_obj->swap_hdr_sz; + /* Round our data to the nearest blocksize */ + allocsize = (allocsize | (COSS_BLOCK_SZ -1)) + 1; + /* Since we're not supporting NOTIFY anymore, lets fail */ assert(which != COSS_ALLOC_NOTIFY); /* Check if we have overflowed the disk .. */ - if ((cs->current_offset + allocsize) > (SD->max_size << 10)) { + if ((cs->current_offset + allocsize) > (((coss_off_t)SD->max_size) << 10)) { /* * tried to allocate past the end of the disk, so wrap * back to the beginning @@ -99,7 +105,7 @@ cs->current_offset = 0; /* wrap back to beginning */ debug(79, 2) ("storeCossAllocate: wrap to 0\n"); - newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll); + newmb = storeCossCreateMemBuf(SD, (coss_size_t)0, checkf, &coll); cs->current_membuf = newmb; /* Check if we have overflowed the MemBuf */ @@ -109,8 +115,13 @@ */ cs->current_membuf->flags.full = 1; cs->current_offset = cs->current_membuf->diskend + 1; - debug(79, 2) ("storeCossAllocate: New offset - %ld\n", - (long int) cs->current_offset); + + /* Update Swap Dir cur_size and total store_swap_size according to stripe overhead */ + store_swap_size += ((size_t)(cs->current_offset >> 10) - SD->cur_size); + SD->cur_size = (size_t)(cs->current_offset >> 10); + + debug(79, 2) ("storeCossAllocate: New offset - %"COSSFMT64"d\n", + cs->current_offset); newmb = storeCossCreateMemBuf(SD, cs->current_offset, checkf, &coll); cs->current_membuf = newmb; } @@ -128,7 +139,7 @@ void storeCossUnlink(SwapDir * SD, StoreEntry * e) { - debug(79, 3) ("storeCossUnlink: offset %d\n", e->swap_filen); + debug(79, 3) ("storeCossUnlink: block %d\n", e->swap_filen); storeCossRemove(SD, e); } @@ -158,8 +169,13 @@ */ sio->st_size = objectLen(e) + e->mem_obj->swap_hdr_sz; sio->swap_dirn = SD->index; - sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_ALLOCATE); - debug(79, 3) ("storeCossCreate: offset %d, size %ld, end %ld\n", sio->swap_filen, (long int) sio->st_size, (long int) (sio->swap_filen + sio->st_size)); + + sio->swap_filen = COSS_OFS_TO_BLK(storeCossAllocate(SD, e, COSS_ALLOC_ALLOCATE)); + debug(79, 3) ("storeCossCreate: block %ld offset %"COSSFMT64"d, size %ld, end %"COSSFMT64"d\n", + sio->swap_filen, + COSS_BLK_TO_OFS(sio->swap_filen), + (size_t) sio->st_size, + COSS_BLK_TO_OFS(sio->swap_filen) + sio->st_size); sio->callback = callback; sio->file_callback = file_callback; @@ -188,7 +204,7 @@ sfileno f = e->swap_filen; CossInfo *cs = (CossInfo *) SD->fsdata; - debug(79, 3) ("storeCossOpen: offset %d\n", f); + debug(79, 3) ("storeCossOpen: block %d\n", f); CBDATA_INIT_TYPE_FREECB(storeIOState, storeCossIOFreeEntry); sio = cbdataAlloc(storeIOState); @@ -209,7 +225,7 @@ cstate->flags.reading = 0; cstate->readbuffer = NULL; cstate->reqdiskoffset = -1; - p = storeCossMemPointerFromDiskOffset(SD, f, NULL); + p = storeCossMemPointerFromDiskOffset(SD, COSS_BLK_TO_OFS(f), NULL); /* make local copy so we don't have to lock membuf */ if (p) { cstate->readbuffer = xmalloc(sio->st_size); @@ -225,9 +241,9 @@ * a place for the object here, and the file_read() reads the object * into the cossmembuf for later writing .. */ - cstate->reqdiskoffset = sio->swap_filen; + cstate->reqdiskoffset = COSS_BLK_TO_OFS(sio->swap_filen); sio->swap_filen = -1; - sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC); + sio->swap_filen = COSS_OFS_TO_BLK(storeCossAllocate(SD, e, COSS_ALLOC_REALLOC)); if (sio->swap_filen == -1) { /* We have to clean up neatly .. */ cbdataFree(sio); @@ -263,7 +279,7 @@ void storeCossClose(SwapDir * SD, storeIOState * sio) { - debug(79, 3) ("storeCossClose: offset %d\n", sio->swap_filen); + debug(79, 3) ("storeCossClose: block %d\n", sio->swap_filen); if (FILE_MODE(sio->mode) == O_WRONLY) storeCossMemBufUnlock(SD, sio); storeCossIOCallback(sio, 0); @@ -280,7 +296,8 @@ assert(sio->read.callback_data == NULL); sio->read.callback = callback; sio->read.callback_data = cbdataReference(callback_data); - debug(79, 3) ("storeCossRead: offset %ld\n", (long int) offset); + debug(79, 3) ("storeCossRead: block %ld, offset %ld, size %ld\n", + sio->swap_filen, (long int) offset, size); sio->offset = offset; cstate->flags.reading = 1; if ((offset + size) > sio->st_size) @@ -289,7 +306,7 @@ cstate->requestbuf = buf; cstate->requestoffset = offset; if (cstate->readbuffer == NULL) { - p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); + p = storeCossMemPointerFromDiskOffset(SD, COSS_BLK_TO_OFS(sio->swap_filen), NULL); /* Remember we need to translate the block offset to a disk offset! */ a_file_read(&cs->aq, cs->fd, p, @@ -312,7 +329,7 @@ { char *dest; CossMemBuf *membuf; - off_t diskoffset; + coss_off_t diskoffset; /* * If we get handed an object with a size of -1, @@ -320,8 +337,18 @@ */ assert(sio->e->mem_obj->object_sz != -1); - debug(79, 3) ("storeCossWrite: offset %ld, len %lu\n", (long int) sio->offset, (unsigned long int) size); - diskoffset = sio->swap_filen + sio->offset; + /* + * If someone for some reason supplies an offset, make sure they're + * streaming the data over + */ + if (offset > -1) { + assert(offset == sio->offset); + } + diskoffset = COSS_BLK_TO_OFS(sio->swap_filen) + sio->offset; + + debug(79, 3) ("storeCossWrite: SD Offset %"COSSFMT64"d block %ld, disk offset %"COSSFMT64"d, offset %ld, len %lu\n", + (((coss_off_t) SD->cur_size) << 10), + sio->swap_filen, diskoffset, (long int) sio->offset, (size_t) size); dest = storeCossMemPointerFromDiskOffset(SD, diskoffset, &membuf); assert(dest != NULL); xmemcpy(dest, buf, size); @@ -342,9 +369,9 @@ void *cbdata; SwapDir *SD = INDEXSD(sio->swap_dirn); CossState *cstate = (CossState *) sio->fsstate; - size_t rlen; + ssize_t rlen; - debug(79, 3) ("storeCossReadDone: fileno %d, FD %d, len %d\n", + debug(79, 3) ("storeCossReadDone: block %d, FD %d, len %d\n", sio->swap_filen, fd, len); cstate->flags.reading = 0; if (errflag) { @@ -353,14 +380,14 @@ } else { if (cstate->readbuffer == NULL) { cstate->readbuffer = xmalloc(sio->st_size); - p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL); + p = storeCossMemPointerFromDiskOffset(SD, COSS_BLK_TO_OFS(sio->swap_filen), NULL); xmemcpy(cstate->readbuffer, p, sio->st_size); storeCossMemBufUnlock(SD, sio); } sio->offset += len; xmemcpy(cstate->requestbuf, &cstate->readbuffer[cstate->requestoffset], cstate->requestlen); - rlen = (size_t) cstate->requestlen; + rlen = (ssize_t) cstate->requestlen; } assert(callback); sio->read.callback = NULL; @@ -382,8 +409,12 @@ cbdataFree(sio); } +/* + * storeCossMemPointerFromDiskOffset - takes a disk offset, checks to see + * whether its in memory or not + */ static char * -storeCossMemPointerFromDiskOffset(SwapDir * SD, size_t offset, CossMemBuf ** mb) +storeCossMemPointerFromDiskOffset(SwapDir * SD, coss_size_t offset, CossMemBuf ** mb) { CossMemBuf *t; dlink_node *m; @@ -412,7 +443,7 @@ for (m = cs->membufs.head; m; m = m->next) { t = m->data; - if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { + if ((COSS_BLK_TO_OFS(e->swap_filen) >= t->diskstart) && (COSS_BLK_TO_OFS(e->swap_filen) <= t->diskend)) { debug(79, 3) ("storeCossMemBufLock: locking %p, lockcount %d\n", t, t->lockcount); t->lockcount++; return; @@ -436,7 +467,7 @@ */ n = m->next; t = m->data; - if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) { + if ((COSS_BLK_TO_OFS(e->swap_filen) >= t->diskstart) && (COSS_BLK_TO_OFS(e->swap_filen) <= t->diskend)) { t->lockcount--; debug(79, 3) ("storeCossMemBufUnlock: unlocking %p, lockcount %d\n", t, t->lockcount); } @@ -451,7 +482,10 @@ CossInfo *cs = (CossInfo *) SD->fsdata; CossMemBuf *t; dlink_node *m; - int end; + coss_off_t end; +#if defined(_SQUID_MSWIN_) || defined(_SQUID_CYGWIN_) + LPOVERLAPPED Overlapped; +#endif /* First, flush pending IO ops */ a_file_syncqueue(&cs->aq); @@ -463,9 +497,27 @@ t = m->data; if (t->flags.writing) sleep(5); /* XXX EEEWWW! */ - lseek(cs->fd, t->diskstart, SEEK_SET); end = (t == cs->current_membuf) ? cs->current_offset : t->diskend; - FD_WRITE_METHOD(cs->fd, t->buffer, end - t->diskstart); +#if defined(_SQUID_MSWIN_) || defined(_SQUID_CYGWIN_) + /* Allocate an overlapped structure. */ + Overlapped = xcalloc(1, sizeof(OVERLAPPED)); +#if COSS_OFFSET_BITS==64 + Overlapped->Offset = (DWORD)(t->diskstart % 0x100000000); + Overlapped->OffsetHigh = (DWORD)(t->diskstart / 0x100000000); +#else + Overlapped->Offset = t->diskstart; + Overlapped->OffsetHigh = 0; +#endif + Overlapped->hEvent = NULL; + WriteFile((HANDLE)_get_osfhandle(cs->fd), /* handle to file */ + t->buffer, /* data buffer */ + (size_t)(end - t->diskstart + 1), /* number of bytes to write */ + NULL, /* number of bytes written */ + Overlapped); /* overlapped buffer */ +#else + lseek(cs->fd, t->diskstart, SEEK_SET); + FD_WRITE_METHOD(cs->fd, t->buffer, (size_t)(end - t->diskstart + 1)); +#endif } } @@ -473,12 +525,12 @@ storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t) { CossInfo *cs = (CossInfo *) SD->fsdata; - debug(79, 3) ("storeCossWriteMemBuf: offset %ld, len %ld\n", - (long int) t->diskstart, (long int) (t->diskend - t->diskstart)); + debug(79, 3) ("storeCossWriteMemBuf: disk offset %"COSSFMT64"d, SD offset %"COSSFMT64"d, len %ld\n", + t->diskstart, (((coss_off_t) SD->cur_size) << 10), (size_t) (t->diskend - t->diskstart + 1)); t->flags.writing = 1; /* Remember that diskstart/diskend are block offsets! */ a_file_write(&cs->aq, cs->fd, t->diskstart, &t->buffer, - t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL); + (size_t)(t->diskend - t->diskstart + 1), storeCossWriteMemBufDone, t, NULL); } @@ -488,7 +540,7 @@ CossMemBuf *t = my_data; CossInfo *cs = (CossInfo *) t->SD->fsdata; - debug(79, 3) ("storeCossWriteMemBufDone: buf %p, len %ld\n", t, (long int) len); + debug(79, 3) ("storeCossWriteMemBufDone: fd %d, buf %p, len %ld\n", fd, t, (long int) len); if (errflag) debug(79, 0) ("storeCossMemBufWriteDone: got failure (%d)\n", errflag); @@ -497,7 +549,7 @@ } static CossMemBuf * -storeCossCreateMemBuf(SwapDir * SD, size_t start, +storeCossCreateMemBuf(SwapDir * SD, coss_size_t start, sfileno curfn, int *collision) { CossMemBuf *newmb, *t; @@ -509,7 +561,7 @@ CBDATA_INIT_TYPE_FREECB(CossMemBuf, NULL); newmb = cbdataAlloc(CossMemBuf); newmb->diskstart = start; - debug(79, 3) ("storeCossCreateMemBuf: creating new membuf at %ld\n", (long int) newmb->diskstart); + debug(79, 3) ("storeCossCreateMemBuf: creating new membuf at %"COSSFMT64"d\n", newmb->diskstart); debug(79, 3) ("storeCossCreateMemBuf: at %p\n", newmb); newmb->diskend = newmb->diskstart + COSS_MEMBUF_SZ - 1; newmb->flags.full = 0; @@ -522,7 +574,7 @@ /* Print out the list of membufs */ for (m = cs->membufs.head; m; m = m->next) { t = m->data; - debug(79, 3) ("storeCossCreateMemBuf: membuflist %ld lockcount %d\n", (long int) t->diskstart, t->lockcount); + debug(79, 3) ("storeCossCreateMemBuf: membuflist %"COSSFMT64"d lockcount %d\n", t->diskstart, t->lockcount); } /* @@ -533,8 +585,8 @@ e = m->data; if (curfn == e->swap_filen) *collision = 1; /* Mark an object alloc collision */ - if ((e->swap_filen >= newmb->diskstart) && - (e->swap_filen <= newmb->diskend)) { + if ((COSS_BLK_TO_OFS(e->swap_filen) >= newmb->diskstart) && + (COSS_BLK_TO_OFS(e->swap_filen) <= newmb->diskend)) { storeRelease(e); numreleased++; } else