--------------------- PatchSet 1484 Date: 2001/02/04 01:16:04 Author: darius Branch: sfs Tag: (none) Log: A few changes at once: * sfs_seek now queues up with everything else - no sense in changing the position we're reading from out of order, that just confuses things. * sfs_close now works properly - before the callbacks were occuring in strange orders and not properly closing files. * Addition of SFS_TEST symbol to define whether we're testing or actually running code - used for compiling sfs_test.c I've got a read bug to sort out next. The store_dir_* and store_io_* files there are a mixed blessing - I have a feeling a lot of what they do could be stripped out and replaced in sfs_interface.c, but I want to reach functional before I go back and really clean out the interface (and build a shared set of tools for storeDirRebuild and friends across multiple fs'es). Oh, and the mimeIcon loading stuff seems to take place before storeDirRebuild - which means it's currently creating the mime icons anew every execution. Most strange - I'm obviously missing something. KL Members: src/fs/sfs/DESIGN:1.1.2.1->1.1.2.2 src/fs/sfs/Makefile.in:1.1.2.6->1.1.2.7 src/fs/sfs/sfs.h:1.1.2.1->1.1.2.2 src/fs/sfs/sfs_defines.h:1.1.2.9->1.1.2.10 src/fs/sfs/sfs_interface.c:1.1.2.16->1.1.2.17 src/fs/sfs/sfs_lib.h:1.1.2.9->1.1.2.10 src/fs/sfs/sfs_llo.c:1.1.2.12->1.1.2.13 src/fs/sfs/sfs_test.c:1.1.2.6->1.1.2.7 src/fs/sfs/sfs_util.c:1.1.2.12->1.1.2.13 src/fs/sfs/store_dir_sfs.c:1.1.2.8->1.1.2.9 src/fs/sfs/store_io_sfs.c:1.1.2.6->1.1.2.7 Index: squid/src/fs/sfs/DESIGN =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/DESIGN,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/fs/sfs/DESIGN 24 Jan 2001 14:11:54 -0000 1.1.2.1 +++ squid/src/fs/sfs/DESIGN 4 Feb 2001 01:16:04 -0000 1.1.2.2 @@ -1,4 +1,4 @@ -$Id: DESIGN,v 1.1.2.1 2001/01/24 14:11:54 adri Exp $ +$Id: DESIGN,v 1.1.2.2 2001/02/04 01:16:04 darius Exp $ SQUIDFS README @@ -34,14 +34,14 @@ The filesystem will have 3 bitmaps that index all the fragments on a filesystem. -BM1 - (on disk) Indexes fragments with valid data in them. -BM2 - (on disk) Indexes inode fragments -IMB - (memory) Indexes blocks that are allocated but not necessarily written +FBM - (on disk) Indexes fragments with valid data in them. +IBM - (on disk) Indexes inode fragments +MHB - (memory) Indexes blocks that are allocated but not necessarily written to disk. States of the on-disk bitmaps are: -BM1 BM2 +FBM IBM 0 0 Fragment free 1 0 Fragment contains valid file data 0 1 Fragment contains complete inode data that references blocks @@ -52,14 +52,14 @@ The first block (inode) is never written until a close is issued. -Whenever a subsequent block is written IMB is set - block is flushed +Whenever a subsequent block is written MHB is set - block is flushed on demand. Whenever a file close occurs the current block and the completed inode block is -flushed to disk immediately and BM2 and IMB is set for the inode block. -When BM2 flush is complete BM1 is set for all non-inode blocks -When BM1 flush is complete BM1 is then set for inode block and flushed to -disk. Once inode BM1 flush is complete the file is valid +flushed to disk immediately and IBM and MHB is set for the inode block. +When IBM flush is complete FBM is set for all non-inode blocks +When FBM flush is complete FBM is then set for inode block and flushed to +disk. Once inode FBM flush is complete the file is valid Bitmap flushes should be scheduled events an should not occur on demand. A file that is waiting for a bitmap flush to occur should register itself @@ -76,7 +76,7 @@ power failure or system crash. In this event the following algorithms are used to return the filesystem to a consistent state: -BM1 BM2 +FBM IBM 0 0 Fragment is free - ignore 1 0 Fragment contains valid data - ignore 0 1 Fragment is an inode that references data blocks that have not Index: squid/src/fs/sfs/Makefile.in =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/Makefile.in,v retrieving revision 1.1.2.6 retrieving revision 1.1.2.7 diff -u -r1.1.2.6 -r1.1.2.7 --- squid/src/fs/sfs/Makefile.in 27 Jan 2001 01:22:23 -0000 1.1.2.6 +++ squid/src/fs/sfs/Makefile.in 4 Feb 2001 01:16:04 -0000 1.1.2.7 @@ -1,7 +1,7 @@ # # Makefile for the sfs storage driver for the Squid Object Cache server # -# $Id: Makefile.in,v 1.1.2.6 2001/01/27 01:22:23 adri Exp $ +# $Id: Makefile.in,v 1.1.2.7 2001/02/04 01:16:04 darius Exp $ # FS = sfs @@ -36,7 +36,7 @@ all: $(OUT) test: $(SFSOBJS) sfs_test.o - $(CC) $(CFLAGS) $(LDFLAGS) -L../../../lib $(SFSOBJS) sfs_test.o sfs_shim.c ../../globals.o -lmiscutil -lm -o sfs_test + $(CC) $(CFLAGS) $(LDFLAGS) -DSFS_TEST -L../../../lib $(SFSOBJS) sfs_test.o sfs_shim.c ../../globals.o -lmiscutil -lm -o sfs_test $(OUT): $(OBJS) @rm -f ../stamp Index: squid/src/fs/sfs/sfs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/sfs.h,v retrieving revision 1.1.2.1 retrieving revision 1.1.2.2 diff -u -r1.1.2.1 -r1.1.2.2 --- squid/src/fs/sfs/sfs.h 24 Jan 2001 14:11:54 -0000 1.1.2.1 +++ squid/src/fs/sfs/sfs.h 4 Feb 2001 01:16:04 -0000 1.1.2.2 @@ -1,4 +1,6 @@ -/* $Id: sfs.h,v 1.1.2.1 2001/01/24 14:11:54 adri Exp $ */ +/* $Id: sfs.h,v 1.1.2.2 2001/02/04 01:16:04 darius Exp $ */ + +/* NOT USED */ #ifndef SFS_H #define SFS_H Index: squid/src/fs/sfs/sfs_defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/sfs_defines.h,v retrieving revision 1.1.2.9 retrieving revision 1.1.2.10 diff -u -r1.1.2.9 -r1.1.2.10 --- squid/src/fs/sfs/sfs_defines.h 28 Jan 2001 18:48:48 -0000 1.1.2.9 +++ squid/src/fs/sfs/sfs_defines.h 4 Feb 2001 01:16:04 -0000 1.1.2.10 @@ -1,4 +1,4 @@ -/* $Id: sfs_defines.h,v 1.1.2.9 2001/01/28 18:48:48 adri Exp $ */ +/* $Id: sfs_defines.h,v 1.1.2.10 2001/02/04 01:16:04 darius Exp $ */ #ifndef SFS_DEFINES_H #define SFS_DEFINES_H @@ -55,7 +55,8 @@ _SFS_OP_CLOSE, _SFS_OP_UNLINK, _SFS_OP_SYNC, - _SFS_OP_UMOUNT + _SFS_OP_UMOUNT, + _SFS_OP_SEEK, }; enum sfs_request_state { @@ -179,11 +180,10 @@ sfs_blockbuf_t *tail[2]; /* list of blocks */ int accepting_requests; int pending_requests; - dlink_list request_queue; + dlink_list *request_queue; pthread_mutex_t req_lock; pthread_cond_t req_signal; pthread_mutex_t req_signal_lock; - pthread_mutex_t openfiles_lock; /* Lock for the 'other' array */ pthread_t thread_id; dlink_list done_queue; Index: squid/src/fs/sfs/sfs_interface.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/sfs_interface.c,v retrieving revision 1.1.2.16 retrieving revision 1.1.2.17 diff -u -r1.1.2.16 -r1.1.2.17 --- squid/src/fs/sfs/sfs_interface.c 31 Jan 2001 15:09:34 -0000 1.1.2.16 +++ squid/src/fs/sfs/sfs_interface.c 4 Feb 2001 01:16:04 -0000 1.1.2.17 @@ -17,6 +17,11 @@ #include "store_sfs.h" +/* Define this is you want to compile sfs_test, the test program + * - it will remove the references to cbdata* functions, which cause + * linking problems otherwise. */ +#undef SFS_TEST + /* Public interfaces - the ones squid requires us to provide */ sfsfd_t @@ -45,8 +50,10 @@ return -1; } assert((io_type == _SFS_IO_SYNC) || dataptr); +#ifndef SFS_TEST if (dataptr) cbdataLock(dataptr); +#endif req->sfsinode = sfsinode; req->dataptr = dataptr; _sfs_submit_request(req); @@ -55,8 +62,10 @@ return 0; } _sfs_waitfor_request(req); +#ifndef SFS_TEST if (dataptr) cbdataUnlock(dataptr); +#endif ret = req->ret; _sfs_remove_request(req); return ret; @@ -72,8 +81,10 @@ if(!(req = _sfs_create_requestor(sfsfd >> 24, _SFS_OP_CLOSE, io_type))) return -1; assert((io_type == _SFS_IO_SYNC) || dataptr); +#ifndef SFS_TEST if (dataptr) cbdataLock(dataptr); +#endif req->sfsfd = sfsfd; req->dataptr = dataptr; _sfs_submit_request(req); @@ -81,8 +92,10 @@ return 0; } _sfs_waitfor_request(req); +#ifndef SFS_TEST if (dataptr) cbdataUnlock(dataptr); +#endif ret = req->ret; _sfs_remove_request(req); return ret; @@ -106,8 +119,10 @@ return -1; } assert((io_type == _SFS_IO_SYNC) || dataptr); +#ifndef SFS_TEST if (dataptr) cbdataLock(dataptr); +#endif req->sfsfd = sfsfd; req->offset = -1; req->buflen = buflen; @@ -116,8 +131,10 @@ if (io_type != _SFS_IO_SYNC) return 0; _sfs_waitfor_request(req); +#ifndef SFS_TEST if (dataptr) cbdataUnlock(dataptr); +#endif if ((!buf) || (req->ret == 0)) return 0; if (req->ret < 0) @@ -132,7 +149,7 @@ } ssize_t -sfs_write(sfsfd_t sfsfd, const void *buf, size_t buflen, +sfs_write(sfsfd_t sfsfd, const void *buf, ssize_t buflen, enum sfs_io_type io_type, void *dataptr) { sfs_requestor *req; @@ -144,12 +161,16 @@ return -1; } assert((io_type == _SFS_IO_SYNC) || dataptr); +#ifndef SFS_TEST if (dataptr) cbdataLock(dataptr); +#endif req->sfsfd = sfsfd; if (!(req->buf = xstrdup(buf))) { +#ifndef SFS_TEST if (dataptr) cbdataUnlock(dataptr); +#endif return -1; } req->buflen = buflen; @@ -158,8 +179,10 @@ if (io_type != _SFS_IO_SYNC) return 0; _sfs_waitfor_request(req); +#ifndef SFS_TEST if (dataptr) cbdataUnlock(dataptr); +#endif ret = req->ret; if (req->buf) xfree(req->buf); @@ -182,16 +205,20 @@ return -1; } assert((io_type == _SFS_IO_SYNC) || dataptr); +#ifndef SFS_TEST if (dataptr) cbdataLock(dataptr); +#endif req->sfsinode = sfsinode; req->dataptr = dataptr; _sfs_submit_request(req); if (io_type != _SFS_IO_SYNC) return 0; _sfs_waitfor_request(req); +#ifndef SFS_TEST if (dataptr) cbdataUnlock(dataptr); +#endif ret = req->ret; _sfs_remove_request(req); return ret; @@ -202,37 +229,6 @@ void sfs_thread_loop(sfs_mountfs_t *mount_point); -sfsblock_t -sfs_get_inode(sfsfd_t sfsfd) -{ -/* This function returns the inode position for a given open fd. The theory - is when you open a new file, you don't know what it's 'called'. Given the - filedescriptor, this will return the canonical 'name' (number) of the file - - which is also the block location on disk. */ -/* Note also, this one employs thread-level locks without going into the thread - - I judge it faster this way. */ - sfsid_t sfsid; - sfs_openfile_t *tmp; - sfsblock_t sfsinode; - - sfsid = sfsfd >> 24; - pthread_mutex_lock(&(_sfs_mounted[sfsid].openfiles_lock)); - tmp = _sfs_openfiles[sfsid]; -/* This while loop could be optimised */ - while (tmp) { - if (tmp->sfsfd == sfsfd) { - sfsinode = tmp->sfsinode; - pthread_mutex_unlock(&(_sfs_mounted[sfsid].openfiles_lock)); - return sfsinode; - } - tmp = tmp->next; - } - pthread_mutex_unlock(&(_sfs_mounted[sfsid].openfiles_lock)); -/* block 0 should never be an inode - therefore can be used as an error value, - in place of negative values (unsigned :(. */ - return 0; -} - int sfs_umount(sfsid_t sfsid, enum sfs_io_type io_type) /* As noted below, mount and umount need to be called only from a single @@ -277,10 +273,10 @@ sfsblock_t ibmpos, fbmpos; sfsblock_t magic; -/* This hunt is not thread-safe - assume only one thread doing these -things (initialising/mounting) - otherwise bad things happen(tm). -Fixing this assumption would mean adding a lock over the _sfs_mounted -array */ + /* This hunt is not thread-safe - assume only one thread doing these + * things (initialising/mounting) - otherwise bad things happen(tm). + * Fixing this assumption would mean adding a lock over the _sfs_mounted + * array */ for(i = 1; (_sfs_mounted[i].rootblock != NULL) && (i < MAXFILESYS); i++); if (i == MAXFILESYS) return -1; @@ -357,13 +353,11 @@ CBIT_SET(_sfs_mounted[i].mhb, j); } _sfs_mounted[i].dirty = NULL; - _sfs_mounted[i].request_queue.head = NULL; - _sfs_mounted[i].request_queue.tail = NULL; + _sfs_mounted[i].request_queue = xcalloc(1,sizeof(dlink_list)); _sfs_mounted[i].pending_requests = 0; pthread_mutex_init(&(_sfs_mounted[i].req_lock), NULL); pthread_mutex_init(&(_sfs_mounted[i].req_signal_lock), NULL); pthread_cond_init(&(_sfs_mounted[i].req_signal), NULL); - pthread_mutex_init(&(_sfs_mounted[i].openfiles_lock), NULL); pthread_mutex_init(&(_sfs_mounted[i].done_lock), NULL); _sfs_mounted[i].done_queue.head = NULL; _sfs_mounted[i].done_queue.tail = NULL; @@ -377,40 +371,40 @@ } off_t -sfs_seek(sfsfd_t sfsfd, off_t pos, int whence) +sfs_seek(sfsfd_t sfsfd, off_t pos, enum sfs_io_type io_type, void *dataptr) /* Takes: sfsid, fd, and position to seek to. Returns: 0 for success, -1 for failure. - - This can all be done inside or outside the thread - I chose outside - atm, for no particularly good reason (less requests == good?) - - whence is probably broken atm - I don't like the idea of supporting - seeks from the end. squid doesn't need it either, AFAICS. */ { - sfs_openfile_t *tmp; - unsigned char sfsid; + sfs_requestor *req; + sfsid_t sfsid; + int ret; sfsid = sfsfd >> 24; - pthread_mutex_lock(&(_sfs_mounted[sfsid].openfiles_lock)); - tmp = _sfs_openfiles[sfsid]; - while (tmp) { - if (tmp->sfsfd == sfsfd) - break; - tmp = tmp->next; - } - if (!tmp) { - printf("DEBUG: Can't find an openfile for fd %d!\n", sfsfd); - pthread_mutex_unlock(&(_sfs_mounted[sfsid].openfiles_lock)); - return -1; - } - if (pos > tmp->inode->len) { - pthread_mutex_unlock(&(_sfs_mounted[sfsid].openfiles_lock)); + if(!(req = _sfs_create_requestor(sfsid, _SFS_OP_SEEK, io_type))) return -1; + + assert((io_type == _SFS_IO_SYNC) || dataptr); +#ifndef SFS_TEST + if (dataptr) + cbdataLock(dataptr); +#endif + req->sfsfd = sfsfd; + req->dataptr = dataptr; + req->offset = pos; + _sfs_submit_request(req); + _sfs_print_request(req); + if (io_type != _SFS_IO_SYNC) { + return 0; } - tmp->pos = pos; - pthread_mutex_unlock(&(_sfs_mounted[sfsid].openfiles_lock)); - return 0; + _sfs_waitfor_request(req); +#ifndef SFS_TEST + if (dataptr) + cbdataUnlock(dataptr); +#endif + ret = req->ret; + _sfs_remove_request(req); + return ret; } /* @@ -446,3 +440,36 @@ else return NULL; } + +int +sfs_filesize(sfsid_t sfsid, sfsblock_t sfsinode) +{ + /* This function returns the size of the given file - file given as + * sfsid and inode. It leans on the block cache to find the info. */ + sfs_blockbuf_t *inode; + + inode = _sfs_read_block(sfsid,sfsinode); + return ((sfs_inode_t *)inode->buf)->len; +} + +int +sfs_openNextInode(sfsid_t sfsid, sfsblock_t *cur) +{ + sfsblock_t i; + int found, fd; + /* This function walks through a mounted filesystem, returning the + * next inode that's in-use. Used by rebuildDir stuff. + * First block is in use for storing rootblock. */ + for(i=max(3,*cur), found=0; i<_sfs_mounted[sfsid].rootblock->numfrags; i += 1) { + if (CBIT_TEST(_sfs_mounted[sfsid].ibm, i)) { + *cur = i; + printf("DEBUG: next inode %d\n",*cur); + /* Note, should make an sio, but I'm too lazy - SYNC doesn't + * _really_ need one... */ + fd = sfs_open(sfsid,i,0,_SFS_OP_OPEN_READ,_SFS_IO_SYNC,NULL); + return fd; + } + } + *cur = 0; + return -2; +} Index: squid/src/fs/sfs/sfs_lib.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/sfs_lib.h,v retrieving revision 1.1.2.9 retrieving revision 1.1.2.10 diff -u -r1.1.2.9 -r1.1.2.10 --- squid/src/fs/sfs/sfs_lib.h 31 Jan 2001 15:09:35 -0000 1.1.2.9 +++ squid/src/fs/sfs/sfs_lib.h 4 Feb 2001 01:16:04 -0000 1.1.2.10 @@ -57,11 +57,10 @@ extern int sfs_close(sfsfd_t, enum sfs_io_type, void *); extern ssize_t sfs_read(sfsfd_t , void * , ssize_t, enum sfs_io_type, void *); -extern off_t sfs_seek(sfsfd_t , off_t , int ); +extern off_t sfs_seek(sfsfd_t , off_t , enum sfs_io_type, void *); extern int sfs_unlink(sfsid_t , sfsblock_t, enum sfs_io_type, void *); -extern ssize_t sfs_write(sfsfd_t , const void * , size_t, enum sfs_io_type, - void *); -extern sfsblock_t sfs_get_inode(sfsblock_t ); +extern ssize_t sfs_write(sfsfd_t , const void * , ssize_t, + enum sfs_io_type, void *); extern sfs_requestor * sfs_getcompleted(sfsid_t); int sfs_openNextInode(sfsid_t sfsid, sfsblock_t *cur); Index: squid/src/fs/sfs/sfs_llo.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/sfs_llo.c,v retrieving revision 1.1.2.12 retrieving revision 1.1.2.13 diff -u -r1.1.2.12 -r1.1.2.13 --- squid/src/fs/sfs/sfs_llo.c 31 Jan 2001 15:12:01 -0000 1.1.2.12 +++ squid/src/fs/sfs/sfs_llo.c 4 Feb 2001 01:16:04 -0000 1.1.2.13 @@ -26,6 +26,7 @@ void sfs_do_write(sfs_requestor *req); void sfs_do_unlink(sfs_requestor *req); void sfs_do_close(sfs_requestor *req); +void sfs_do_seek(sfs_requestor *req); void sfs_initialise() @@ -77,17 +78,17 @@ i = 0; while (1) { pthread_cond_wait(&(mount_point->req_signal), &(mount_point->req_signal_lock)); - - /* Should I lock the request queue while I cycle through these? - * Probably wise, but slow... */ + pthread_mutex_lock(&(mount_point->req_lock)); while (mount_point->pending_requests > 0) { - tnode = mount_point->request_queue.head; + printf("Pending Requests: %d\n",mount_point->pending_requests); + tnode = mount_point->request_queue->head; assert(tnode); req = tnode->data; - if (req->request_state == _SFS_PENDING) { + if (req && (req->request_state == _SFS_PENDING)) { /* If we're not accepting requests, return fail for each * request. Note, we can't just lock the request queue, as * things are still being removed from it by other threads. */ + printf("dealing with pending request, %d\n",mount_point->pending_requests); if (!(mount_point->accepting_requests)) { _sfs_done_request(req,-1); } else { @@ -95,6 +96,7 @@ * to do - open, read, write, close, sync, unlink. */ req->request_state = _SFS_IN_PROGRESS; mount_point->pending_requests--; + printf("pending requests now %d\n",mount_point->pending_requests); switch (req->request_type) { case _SFS_OP_OPEN_READ: case _SFS_OP_OPEN_WRITE: @@ -115,14 +117,18 @@ case _SFS_OP_CLOSE: sfs_do_close(req); break; + case _SFS_OP_SEEK: + sfs_do_seek(req); + break; default: _sfs_done_request(req,-1); } } } - printf("%p %p\n",tnode->prev,tnode->next); + printf("tnode prev: %p next: %p\n",tnode->prev,tnode->next); tnode = tnode->next; } + pthread_mutex_unlock(&(mount_point->req_lock)); /* Flush the bitmaps every 10 seconds */ i = (i + 1) % 10; if (i == 0) { @@ -215,6 +221,8 @@ else offset = openfd->pos = req->offset; + diskpos = 0; + /* written tracks how much we've written so far in total. */ /* offset tells us where we start writing, minus the inode. */ /* bytes_left gets set to the number of bytes left to write. */ @@ -253,7 +261,6 @@ _sfs_done_request(req,written); return; } - /* This sets the appropriate inode block pointer */ if (type != _SFS_INODE) { if (openfd->pos < direct_pointer_threshold) { openfd->inode->dip[(openfd->pos - inode_data_size) / FRAGSIZE] = diskpos; @@ -268,10 +275,8 @@ /* How much more to write? */ if (current) { - printf("DEBUG: current block found\n"); buf = current->buf; } else { - printf("DEBUG: sfs_do_write new buf created\n"); buf = (char *)xcalloc(1, FRAGSIZE); } memcpy(((char *)buf)+inblock,((char *)req->buf)+written,fragsize); @@ -291,7 +296,7 @@ /* $DEITY forbid two people try to write at once - maybe I need some * locking to prevent that... */ openfd->inode->len = max(openfd->inode->len,openfd->pos); - printf("DEBUG: written %d bytes to fd %d (%s)\n",req->buflen,req->sfsfd,current->buf+sizeof(sfs_inode_t)); + printf("DEBUG: written %d bytes to fd %d (block %d)\n",req->buflen,req->sfsfd,diskpos); _sfs_done_request(req,written); } @@ -334,13 +339,13 @@ while (mnt->clean) { mnt->clean = sfs_splay_delete(req->sfsid, mnt->clean); } - if (mnt->request_queue.head != NULL) { + if (mnt->request_queue->head != NULL) { /* Make doubly sure we've cleared any pending requests - shouldn't need to, * but we _are_ umounting... This is actually bodgy code, if it ever does * anything, then something's gone wrong. Probably shouldn't do this, but * it saves deadlock, and we'd rather see corruption than hang * indefinately.*/ - lnode = mnt->request_queue.head; + lnode = mnt->request_queue->head; while (lnode) { req = lnode->data; if (req->request_state == _SFS_PENDING) { @@ -351,7 +356,7 @@ i = 0; /* Waiting for the request queue to be empty of all bar the umount * request */ - while ((mnt->request_queue.head->next) && (i < 5)) { + while ((mnt->request_queue->head->next) && (i < 5)) { i++; sleep(1); } @@ -369,25 +374,6 @@ pthread_exit(NULL); } -int -sfs_openNextInode(sfsid_t sfsid, sfsblock_t *cur) -{ - sfsblock_t i; - int found; - /* This function walks through a mounted filesystem, returning the - * next inode that's in-use. Used by rebuildDir stuff */ - for(i=*cur, found=0; i<_sfs_mounted[sfsid].rootblock->numfrags; i += 2) { - if (CBIT_TEST(_sfs_mounted[sfsid].mhb, i)) { - *cur = i; - /* Note, should make an sio, but I'm too lazy - SYNC doesn't - * _really_ need one... */ - return sfs_open(sfsid,i,0,_SFS_OP_OPEN_READ,_SFS_IO_SYNC,NULL); - } - } - *cur = 0; - return -2; -} - void sfs_do_open(sfs_requestor *req) { @@ -477,6 +463,7 @@ { sfs_openfile_t *ptr; + printf("DEBUG: closing file %d\n",req->sfsfd); if (!(ptr = _sfs_find_fd(req->sfsfd))) { printf("DEBUG: couldn't find fd %d\n",req->sfsfd); _sfs_done_request(req,-1); @@ -492,3 +479,31 @@ _sfs_done_request(req,0); return; } + +void +sfs_do_seek(sfs_requestor *req) +{ + sfs_openfile_t *ptr; + unsigned char sfsid; + + sfsid = req->sfsid; + ptr = _sfs_openfiles[sfsid]; + while (ptr) { + if (ptr->sfsfd == req->sfsfd) + break; + ptr = ptr->next; + } + if (!ptr) { + printf("DEBUG: Can't find an openfile for fd %d!\n", req->sfsfd); + _sfs_done_request(req,-1); + return; + } + if (req->offset > ptr->inode->len) { + printf("DEBUG: seek beyond EOF for fd %d\n",req->sfsfd); + _sfs_done_request(req,-1); + return; + } + ptr->pos = req->offset; + _sfs_done_request(req,0); + return; +} Index: squid/src/fs/sfs/sfs_test.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/sfs_test.c,v retrieving revision 1.1.2.6 retrieving revision 1.1.2.7 diff -u -r1.1.2.6 -r1.1.2.7 --- squid/src/fs/sfs/sfs_test.c 28 Jan 2001 15:37:55 -0000 1.1.2.6 +++ squid/src/fs/sfs/sfs_test.c 4 Feb 2001 01:16:04 -0000 1.1.2.7 @@ -1,4 +1,4 @@ -/* $Id: sfs_test.c,v 1.1.2.6 2001/01/28 15:37:55 adri Exp $ */ +/* $Id: sfs_test.c,v 1.1.2.7 2001/02/04 01:16:04 darius Exp $ */ #include #include @@ -32,13 +32,11 @@ snprintf(filename,20,"%d/0",sfsid); printf("opening %s",filename); - sfsfd = sfs_open(sfsid, 0, O_CREAT, 0, _SFS_IO_SYNC); + sfsfd = sfs_open(sfsid, 0, O_CREAT, 0, _SFS_IO_SYNC, NULL); - sfsinode = sfs_get_inode(sfsfd); - printf("sfsfd = %d, sfsinode = %d\n",sfsfd,sfsinode); - sfs_write(sfsfd,"Hello...\n",strlen("Hello...\n"), _SFS_IO_SYNC); - sfs_write(sfsfd,"Hello, again!\n",strlen("Hello, again!\n"), _SFS_IO_SYNC); - printf("close result = %d\n",sfs_close(sfsfd, _SFS_IO_SYNC)); + sfs_write(sfsfd,"Hello...\n",strlen("Hello...\n"), _SFS_IO_SYNC, NULL); + sfs_write(sfsfd,"Hello, again!\n",strlen("Hello, again!\n"), _SFS_IO_SYNC, NULL); + printf("close result = %d\n",sfs_close(sfsfd, _SFS_IO_SYNC, NULL)); printf("umount result = %d\n",sfs_umount(sfsid, _SFS_IO_SYNC)); printf("About to remount and read...\n"); sfsid = sfs_mount("test.drv"); @@ -47,14 +45,14 @@ snprintf(filename,20,"%d/%d",sfsid,sfsinode); printf("DEBUG: %s\n",filename); - sfsfd = sfs_open(sfsid, sfsinode, O_RDONLY, 0, _SFS_IO_SYNC); + sfsfd = sfs_open(sfsid, sfsinode, O_RDONLY, 0, _SFS_IO_SYNC, NULL); printf("sfsfd = %d, sfsinode = %d\n",sfsfd,sfsinode); if (sfsfd >= 0) { - printf("read result = %d\n",sfs_read(sfsfd,buf,80, _SFS_IO_SYNC)); + printf("read result = %d\n",sfs_read(sfsfd,buf,80, _SFS_IO_SYNC, NULL)); printf("\n***** %s *****\n",buf); printf("strlen buf = %d\n",strlen(buf)); - printf("close result = %d\n",sfs_close(sfsfd, _SFS_IO_SYNC)); + printf("close result = %d\n",sfs_close(sfsfd, _SFS_IO_SYNC, NULL)); } printf("umount result = %d\n",sfs_umount(sfsid, _SFS_IO_SYNC)); exit(0); Index: squid/src/fs/sfs/sfs_util.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/sfs_util.c,v retrieving revision 1.1.2.12 retrieving revision 1.1.2.13 diff -u -r1.1.2.12 -r1.1.2.13 --- squid/src/fs/sfs/sfs_util.c 31 Jan 2001 15:09:35 -0000 1.1.2.12 +++ squid/src/fs/sfs/sfs_util.c 4 Feb 2001 01:16:04 -0000 1.1.2.13 @@ -30,7 +30,7 @@ * by various commloops/modio changes - hopefully, we won't even be supplying * the buffer. */ { - printf("DEBUG: Removing %d request from queue\n",req->request_type); + printf("DEBUG: Removing %d request from done queue\n",req->request_type); pthread_mutex_lock(&(_sfs_mounted[req->sfsid].done_lock)); dlinkDelete(&req->node, &(_sfs_mounted[req->sfsid].done_queue)); _sfs_mounted[req->sfsid].done_requests--; @@ -52,10 +52,10 @@ req->ret = retval; req->request_state = _SFS_DONE; - /* Take it off the mount point request queue */ - pthread_mutex_lock(&(_sfs_mounted[req->sfsid].req_lock)); - dlinkDelete(&req->node, &(_sfs_mounted[req->sfsid].request_queue)); - pthread_mutex_unlock(&(_sfs_mounted[req->sfsid].req_lock)); + /* Take it off the mount point request queue. Note, requests are + * always marked done from within the thread, within the lock that's + * already in place from the main sfs_thread_loop. Hence, no locking. */ + dlinkDelete(&req->node, _sfs_mounted[req->sfsid].request_queue); /* Add it to the squid done queue */ pthread_mutex_lock(&(_sfs_mounted[req->sfsid].done_lock)); @@ -99,7 +99,7 @@ { /* add the request to the end of the list rather than the start */ pthread_mutex_lock(&(_sfs_mounted[req->sfsid].req_lock)); - dlinkAddTail(req, &req->node, &_sfs_mounted[req->sfsid].request_queue); + dlinkAddTail(req, &req->node, _sfs_mounted[req->sfsid].request_queue); _sfs_mounted[req->sfsid].pending_requests++; pthread_mutex_unlock(&(_sfs_mounted[req->sfsid].req_lock)); @@ -297,7 +297,7 @@ blocks = 2; else blocks = 1; -/* First two blocks are always already used? */ +/* First block is always already used - rootblock */ for(i=1, found=0; i<_sfs_mounted[sfsid].rootblock->numfrags; i += blocks) { if (!(CBIT_TEST(_sfs_mounted[sfsid].mhb, i))) { found = 1; @@ -341,7 +341,6 @@ sfs_openfile_t *tmpfile; uint diskpos; - pthread_mutex_lock(&(_sfs_mounted[sfsid].openfiles_lock)); tmpfile = _sfs_openfiles[sfsid]; while (tmpfile) { if (tmpfile->sfsfd == fd->sfsfd) { Index: squid/src/fs/sfs/store_dir_sfs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/store_dir_sfs.c,v retrieving revision 1.1.2.8 retrieving revision 1.1.2.9 diff -u -r1.1.2.8 -r1.1.2.9 --- squid/src/fs/sfs/store_dir_sfs.c 31 Jan 2001 15:12:01 -0000 1.1.2.8 +++ squid/src/fs/sfs/store_dir_sfs.c 4 Feb 2001 01:16:04 -0000 1.1.2.9 @@ -1,6 +1,6 @@ /* - * $Id: store_dir_sfs.c,v 1.1.2.8 2001/01/31 15:12:01 darius Exp $ + * $Id: store_dir_sfs.c,v 1.1.2.9 2001/02/04 01:16:04 darius Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -355,16 +355,18 @@ sfsblock_t currentEntry; sfsinfo_t *sfsinfo = rb->sd->fsdata; + int filesize; assert(rb != NULL); debug(20, 3) ("storeSfsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index); - /* We don't do anything right now */ + /* We don't do anything right now store_dirs_rebuilding--; storeSfsDirCloseTmpSwapLog(rb->sd); storeRebuildComplete(&rb->counts); cbdataFree(rb); return; + */ currentEntry = 0; @@ -383,44 +385,30 @@ continue; } assert(fd > -1); + filesize = sfs_filesize(sfsinfo->sfsid,currentEntry); - /* XXX Darius up to here on converting this function */ - - /* lets get file stats here */ - if (fstat(fd, &sb) < 0) { - debug(20, 1) ("storeSfsDirRebuildFromDirectory: fstat(FD %d): %s\n", - fd, xstrerror()); - file_close(fd); - store_open_disk_fd--; - fd = -1; - continue; - } if ((++rb->counts.scancount & 0xFFFF) == 0) debug(20, 3) (" %s %7d files opened so far.\n", rb->sd->path, rb->counts.scancount); - debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno); + debug(20, 9) ("file_in: fd=%d %08X\n", fd, currentEntry); statCounter.syscalls.disk.reads++; - if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) { - debug(20, 1) ("storeSfsDirRebuildFromDirectory: read(FD %d): %s\n", + if (sfs_read(fd, hdr_buf, SM_PAGE_SIZE, _SFS_IO_SYNC, NULL) < 0) { + debug(20, 1) ("storeSfsDirRebuildFromDirectory: sfs_read(FD %d): %s\n", fd, xstrerror()); - file_close(fd); + sfs_close(fd,_SFS_IO_SYNC,NULL); store_open_disk_fd--; fd = -1; continue; } - file_close(fd); + sfs_close(fd,_SFS_IO_SYNC,NULL); store_open_disk_fd--; fd = -1; swap_hdr_len = 0; -#if USE_TRUNCATE - if (sb.st_size == 0) - continue; -#endif tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); if (tlv_list == NULL) { - debug(20, 1) ("storeSfsDirRebuildFromDirectory: failed to get meta data\n"); - /* XXX shouldn't this be a call to storeSfsUnlink ? */ - storeSfsDirUnlinkFile(SD, sfileno); + debug(20, 1) ("storeSfsDirRebuildFromDirectory: failed to get meta data for file %d\n",currentEntry); + sfs_close(fd,_SFS_IO_SYNC,NULL); + sfs_unlink(sfsinfo->sfsid,currentEntry,_SFS_IO_SYNC,NULL); continue; } debug(20, 3) ("storeSfsDirRebuildFromDirectory: successful swap meta unpacking\n"); @@ -444,23 +432,25 @@ tlv_list = NULL; if (storeKeyNull(key)) { debug(20, 1) ("storeSfsDirRebuildFromDirectory: NULL key\n"); - storeSfsDirUnlinkFile(SD, sfileno); + sfs_close(fd,_SFS_IO_SYNC,NULL); + sfs_unlink(sfsinfo->sfsid,currentEntry,_SFS_IO_SYNC,NULL); continue; } tmpe.hash.key = key; - /* check sizes */ if (tmpe.swap_file_sz == 0) { - tmpe.swap_file_sz = sb.st_size; - } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) { - tmpe.swap_file_sz = sb.st_size; - } else if (tmpe.swap_file_sz != sb.st_size) { + tmpe.swap_file_sz = filesize; + } else if (tmpe.swap_file_sz == filesize - swap_hdr_len) { + tmpe.swap_file_sz = filesize; + } else if (tmpe.swap_file_sz != filesize) { debug(20, 1) ("storeSfsDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n", - tmpe.swap_file_sz, (int) sb.st_size); - storeSfsDirUnlinkFile(SD, sfileno); + tmpe.swap_file_sz, filesize); + sfs_close(fd,_SFS_IO_SYNC,NULL); + sfs_unlink(sfsinfo->sfsid,currentEntry,_SFS_IO_SYNC,NULL); continue; } if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { - storeSfsDirUnlinkFile(SD, sfileno); + sfs_close(fd,_SFS_IO_SYNC,NULL); + sfs_unlink(sfsinfo->sfsid,currentEntry,_SFS_IO_SYNC,NULL); rb->counts.badflags++; continue; } Index: squid/src/fs/sfs/store_io_sfs.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fs/sfs/Attic/store_io_sfs.c,v retrieving revision 1.1.2.6 retrieving revision 1.1.2.7 diff -u -r1.1.2.6 -r1.1.2.7 --- squid/src/fs/sfs/store_io_sfs.c 31 Jan 2001 04:17:18 -0000 1.1.2.6 +++ squid/src/fs/sfs/store_io_sfs.c 4 Feb 2001 01:16:04 -0000 1.1.2.7 @@ -1,6 +1,6 @@ /* - * $Id: store_io_sfs.c,v 1.1.2.6 2001/01/31 04:17:18 darius Exp $ + * $Id: store_io_sfs.c,v 1.1.2.7 2001/02/04 01:16:04 darius Exp $ * * DEBUG: section 79 Storage Manager SFS Interface * AUTHOR: Duane Wessels @@ -137,10 +137,8 @@ debug(79, 3) ("storeSfsClose: dirno %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, sfsstate->fd); - if (sfsstate->flags.reading || sfsstate->flags.writing) { - sfsstate->flags.close_request = 1; - return; - } + /* storeSfsIOCallback calls sfs_close as part of it's normal operation + * - who said this interface was untidy? :( */ storeSfsIOCallback(sio, 0); } @@ -164,7 +162,7 @@ if (offset > -1) { debug(79, 3) ("storeSfsRead: seeking to %d\n", offset); - retval = sfs_seek(sfsstate->fd, offset, SEEK_SET); + retval = sfs_seek(sfsstate->fd, offset, _SFS_IO_SYNC, NULL); if (retval < 0) { debug(79, 2) ("storeSfsRead: sfs_seek on %d failed!\n", sfsstate->fd); @@ -189,7 +187,7 @@ if (offset > -1) { debug(79, 3) ("storeSfsWrite: seeking to %d\n", offset); - retval = sfs_seek(sfsstate->fd, offset, SEEK_SET); + retval = sfs_seek(sfsstate->fd, offset, _SFS_IO_SYNC, NULL); if (retval < 0) { debug(79, 2) ("storeSfsWrite: sfs_seek on %d failed!\n", sfsstate->fd); @@ -258,6 +256,7 @@ sio->offset += retval; } + /* === STATIC =========================================================== */ static void