diff -N -c -r -X exclude_files squid-1.0.alpha15/src/Makefile.in squid-1.0.alpha15.henrik/src/Makefile.in *** squid-1.0.alpha15/src/Makefile.in Fri Apr 12 02:11:02 1996 --- squid-1.0.alpha15.henrik/src/Makefile.in Sun Apr 14 20:04:49 1996 *************** *** 57,63 **** main.o mime.o neighbors.o objcache.o \ proto.o send-announce.o stack.o stat.o stmem.o \ store.o storetoString.o tools.o ttl.o \ ! url.o wais.o $(XTRA_OBJS) DEFAULTS = \ -DDEFAULT_CONFIG_FILE=\"$(sysconfdir)/cached.conf\" \ --- 57,63 ---- main.o mime.o neighbors.o objcache.o \ proto.o send-announce.o stack.o stat.o stmem.o \ store.o storetoString.o tools.o ttl.o \ ! url.o wais.o background.o $(XTRA_OBJS) DEFAULTS = \ -DDEFAULT_CONFIG_FILE=\"$(sysconfdir)/cached.conf\" \ diff -N -c -r -X exclude_files squid-1.0.alpha15/src/background.c squid-1.0.alpha15.henrik/src/background.c *** squid-1.0.alpha15/src/background.c Thu Jan 1 01:00:00 1970 --- squid-1.0.alpha15.henrik/src/background.c Sun Apr 14 22:40:20 1996 *************** *** 0 **** --- 1,85 ---- + /* A small package for cooperative background processing + * This package polls functions until they return true. + */ + + #include "squid.h" + + /* The list of background processes */ + static struct bg_entry { + int (*func) _PARAMS((void *arg)); + void (*done) _PARAMS((void *arg)); + void *arg; + char *name; + struct bg_entry *next; + } *tasks=NULL; + + /* Last called process */ + static struct bg_entry *last_called=NULL; + + /* runInBackground(func,arg) + * int (*func)(void *arg) + * + * Add a function to the list of background processes + * doBackgroundProcessing calls func until func returns true + * when func returns true, done is called. + */ + void runInBackground(name,func,arg,done) + char *name; + int (*func) _PARAMS((void *arg)); + void *arg; + void (*done) _PARAMS((void *arg)); + { + struct bg_entry *entry; + + entry=xmalloc(sizeof(*entry)); + entry->func=func; + entry->arg=arg; + entry->name=name; + entry->done=done; + entry->next=tasks; + tasks=entry; + } + + + /* int doBackgroundProcessing() + * Call one background processing function + * returns true if there is more background processing to do + */ + extern int doBackgroundProcessing() + { + struct bg_entry *this; + + if(!tasks) { + last_called=NULL; + return 0; + } + + if(last_called && last_called->next) { + this=last_called->next; + } else { + this=tasks; + last_called=NULL; + } + + if(this->func(this->arg)) { + /* Call this again later */ + /* Remember that this one is called, to call the next one on next call */ + last_called=this; + } else { + /* We are done */ + if(this->done) + this->done(this->arg); + /* Delete this entry */ + if(last_called) + last_called->next=this->next; + else + tasks=this->next; + safe_free(this); + } + + if(tasks) + return 1; + else + return 0; + } + diff -N -c -r -X exclude_files squid-1.0.alpha15/src/background.h squid-1.0.alpha15.henrik/src/background.h *** squid-1.0.alpha15/src/background.h Thu Jan 1 01:00:00 1970 --- squid-1.0.alpha15.henrik/src/background.h Sun Apr 14 22:18:31 1996 *************** *** 0 **** --- 1,19 ---- + /* A small package for cooperative background processing + * This package polls functions until they return true. + */ + + /* runInBackground(name,func,arg,done) + * char *name; + * int (*func)(char *name,void *arg) + * void (*done)(char *name,void *arg) + * + * Add func to the list of background processes + */ + extern void runInBackground _PARAMS((char *name, int (*func)(void *), void *arg, void (*done)(void *))); + + /* int doBackgroundProcessing() + * Call one background processing function + * returns true if there is more background processing to do + */ + extern int doBackgroundProcessing _PARAMS((void)); + diff -N -c -r -X exclude_files squid-1.0.alpha15/src/main.c squid-1.0.alpha15.henrik/src/main.c *** squid-1.0.alpha15/src/main.c Thu Apr 11 06:47:24 1996 --- squid-1.0.alpha15.henrik/src/main.c Sun Apr 14 22:35:16 1996 *************** *** 298,309 **** if (getCleanRate() > 0) next_cleaning = time(0L) + getCleanRate(); while (1) { /* maintain cache storage */ if (cached_curtime > last_maintain) { storeMaintainSwapSpace(); last_maintain = cached_curtime; } ! switch (comm_select((time_t) 60, next_cleaning)) { case COMM_OK: /* do nothing */ break; --- 298,313 ---- if (getCleanRate() > 0) next_cleaning = time(0L) + getCleanRate(); while (1) { + time_t loop_delay=(time_t)60; /* maintain cache storage */ if (cached_curtime > last_maintain) { storeMaintainSwapSpace(); last_maintain = cached_curtime; } ! /* do background processing */ ! if(doBackgroundProcessing()) ! loop_delay=(time_t)0; ! switch (comm_select(loop_delay, next_cleaning)) { case COMM_OK: /* do nothing */ break; diff -N -c -r -X exclude_files squid-1.0.alpha15/src/squid.h squid-1.0.alpha15.henrik/src/squid.h *** squid-1.0.alpha15/src/squid.h Fri Apr 12 00:52:30 1996 --- squid-1.0.alpha15.henrik/src/squid.h Sun Apr 14 19:58:26 1996 *************** *** 119,124 **** --- 119,125 ---- #include "send-announce.h" #include "acl.h" #include "util.h" + #include "background.h" extern time_t cached_starttime; /* main.c */ extern time_t next_cleaning; /* main.c */ diff -N -c -r -X exclude_files squid-1.0.alpha15/src/store.c squid-1.0.alpha15.henrik/src/store.c *** squid-1.0.alpha15/src/store.c Sun Apr 14 05:03:46 1996 --- squid-1.0.alpha15.henrik/src/store.c Mon Apr 15 18:08:39 1996 *************** *** 1205,1390 **** } /* recreate meta data from disk image in swap directory */ ! void storeRebuildFromDisk() { - int objcount = 0; /* # objects successfully reloaded */ - int expcount = 0; /* # objects expired */ - int linecount = 0; /* # lines parsed from cache logfile */ - int clashcount = 0; /* # swapfile clashes avoided */ - int dupcount = 0; /* # duplicates purged */ - static char line_in[4096]; static char log_swapfile[1024]; static char swapfile[1024]; static char url[MAX_URL]; char *t = NULL; StoreEntry *e = NULL; - struct stat sb; - time_t start, stop, r; time_t expires; time_t timestamp; - time_t last_clean; int scan1, scan2, scan3; int delta; - int i; int sfileno = 0; ! off_t size; ! int fast_mode = 0; ! FILE *old_log = NULL; ! FILE *new_log = NULL; ! char *new_log_name = NULL; ! ! if (stat(swaplog_file, &sb) < 0) ! return; ! ! for (i = 0; i < ncache_dirs; ++i) ! debug(20, 1, "Rebuilding storage from disk image in %s\n", swappath(i)); ! start = getCurrentTime(); ! ! sprintf(line_in, "%s/log-last-clean", swappath(0)); ! if (stat(line_in, &sb) >= 0) { ! last_clean = sb.st_mtime; ! sprintf(line_in, "%s/log", swappath(0)); ! if (stat(line_in, &sb) >= 0) { ! fast_mode = (sb.st_mtime <= last_clean) ? 1 : 0; ! } ! } ! /* Open the existing swap log for reading */ ! if ((old_log = fopen(swaplog_file, "r")) == (FILE *) NULL) { ! sprintf(tmp_error_buf, "storeRebuildFromDisk: %s: %s", ! swaplog_file, xstrerror()); ! fatal(tmp_error_buf); ! } ! /* Open a new log for writing */ ! sprintf(line_in, "%s.new", swaplog_file); ! new_log_name = xstrdup(line_in); ! new_log = fopen(new_log_name, "w"); ! if (new_log == (FILE *) NULL) { ! sprintf(tmp_error_buf, "storeRebuildFromDisk: %s: %s", ! new_log_name, xstrerror()); ! fatal(tmp_error_buf); ! } ! if (fast_mode) ! debug(20, 1, "Rebuilding in FAST MODE.\n"); ! ! memset(line_in, '\0', 4096); ! while (fgets(line_in, 4096, old_log)) { ! if ((linecount++ & 0x7F) == 0) /* update current time */ ! getCurrentTime(); ! if ((linecount & 0xFFF) == 0) ! debug(20, 1, " %7d Lines read so far.\n", linecount); ! debug(20, 10, "line_in: %s", line_in); ! if ((line_in[0] == '\0') || (line_in[0] == '\n') || ! (line_in[0] == '#')) ! continue; /* skip bad lines */ ! ! url[0] = log_swapfile[0] = '\0'; ! expires = cached_curtime; ! ! scan3 = 0; ! size = 0; ! if (sscanf(line_in, "%s %s %d %d %d", ! log_swapfile, url, &scan1, &scan2, &scan3) != 5) { ! if (opt_unlink_on_reload && log_swapfile[0]) ! safeunlink(log_swapfile, 0); ! continue; ! } ! expires = (time_t) scan1; ! timestamp = (time_t) scan2; ! size = (off_t) scan3; ! if ((t = strrchr(log_swapfile, '/'))) ! sfileno = atoi(t + 1); ! else ! sfileno = atoi(log_swapfile); ! storeSwapFullPath(sfileno, swapfile); ! ! /* ! * Note that swapfile may be different than log_swapfile if ! * another cache_dir is added. ! */ ! ! if (!fast_mode) { ! if (stat(swapfile, &sb) < 0) { ! if (expires < cached_curtime) { ! debug(20, 3, "storeRebuildFromDisk: Expired: \n", url); ! if (opt_unlink_on_reload) ! safeunlink(swapfile, 1); ! expcount++; ! } else { ! debug(20, 3, "storeRebuildFromDisk: Swap file missing: : %s: %s.\n", url, swapfile, xstrerror()); ! if (opt_unlink_on_reload) ! safeunlink(log_swapfile, 1); ! } ! continue; ! } ! if ((size = sb.st_size) == 0) { if (opt_unlink_on_reload) safeunlink(log_swapfile, 1); - continue; - } - /* timestamp might be a little bigger than sb.st_mtime */ - delta = abs((int) (timestamp - sb.st_mtime)); - if (delta > REBUILD_TIMESTAMP_DELTA_MAX) { - /* this log entry doesn't correspond to this file */ - clashcount++; - continue; } ! timestamp = sb.st_mtime; ! debug(20, 10, "storeRebuildFromDisk: Cached file exists: : %s\n", ! url, swapfile); } ! if ((e = storeGet(url))) { ! debug(20, 6, "storeRebuildFromDisk: Duplicate: \n", url); ! storeRelease(e); ! objcount--; ! dupcount++; } ! if (expires < cached_curtime) { ! debug(20, 3, "storeRebuildFromDisk: Expired: \n", url); if (opt_unlink_on_reload) ! safeunlink(swapfile, 1); ! expcount++; ! continue; } ! /* update store_swap_size */ ! store_swap_size += (int) ((size + 1023) >> 10); ! objcount++; ! ! fprintf(new_log, "%s %s %d %d %d\n", ! swapfile, url, (int) expires, (int) timestamp, (int) size); ! storeAddDiskRestore(url, sfileno, (int) size, expires, timestamp); ! CacheInfo->proto_newobject(CacheInfo, ! CacheInfo->proto_id(url), ! (int) size, TRUE); ! } ! ! fclose(old_log); ! fclose(new_log); ! if (rename(new_log_name, swaplog_file) < 0) { ! sprintf(tmp_error_buf, "storeRebuildFromDisk: rename: %s", ! xstrerror()); ! fatal(tmp_error_buf); } ! xfree(new_log_name); stop = getCurrentTime(); ! r = stop - start; debug(20, 1, "Finished rebuilding storage from disk image.\n"); ! debug(20, 1, " %7d Lines read from previous logfile.\n", linecount); ! debug(20, 1, " %7d Objects loaded.\n", objcount); ! debug(20, 1, " %7d Objects expired.\n", expcount); ! debug(20, 1, " %7d Duplicate URLs purged.\n", dupcount); ! debug(20, 1, " %7d Swapfile clashes avoided.\n", clashcount); debug(20, 1, " Took %d seconds (%6.1lf objects/sec).\n", ! r > 0 ? r : 0, (double) objcount / (r > 0 ? r : 1)); debug(20, 1, " store_swap_size = %dk\n", store_swap_size); ! /* touch a timestamp file */ ! sprintf(line_in, "%s/log-last-clean", swappath(0)); ! file_close(file_open(line_in, NULL, O_WRONLY | O_CREAT | O_TRUNC)); } /* return current swap size in kilo-bytes */ int storeGetSwapSize() --- 1205,1435 ---- } /* recreate meta data from disk image in swap directory */ ! ! struct storeRebuild_data { ! FILE *log; ! int objcount; /* # objects successfully reloaded */ ! int expcount; /* # objects expired */ ! int linecount; /* # lines parsed from cache logfile */ ! int clashcount; /* # swapfile clashes avoided */ ! int dupcount; /* # duplicates purged */ ! time_t start,stop; ! int fast_mode; ! char line_in[4096]; ! }; ! ! /* Add one swap file at a time from disk storage */ ! static int storeDoRebuildFromDisk(data) ! struct storeRebuild_data *data; { static char log_swapfile[1024]; static char swapfile[1024]; static char url[MAX_URL]; char *t = NULL; StoreEntry *e = NULL; time_t expires; time_t timestamp; int scan1, scan2, scan3; + struct stat sb; + off_t size; int delta; int sfileno = 0; ! ! if (!fgets(data->line_in, 4095, data->log)) ! return 0; ! ! if ((data->linecount++ & 0x7F) == 0) /* update current time */ ! getCurrentTime(); ! ! if ((data->linecount & 0xFFF) == 0) ! debug(20, 1, " %7d Lines read so far.\n", data->linecount); ! ! debug(20, 10, "line_in: %s", data->line_in); ! if ((data->line_in[0] == '\0') || (data->line_in[0] == '\n') || ! (data->line_in[0] == '#')) ! goto skip; /* skip bad lines */ ! ! url[0] = log_swapfile[0] = '\0'; ! expires = cached_curtime; ! ! scan3 = 0; ! size = 0; ! if (sscanf(data->line_in, "%s %s %d %d %d", ! log_swapfile, url, &scan1, &scan2, &scan3) != 5) { ! if (opt_unlink_on_reload && log_swapfile[0]) ! safeunlink(log_swapfile, 0); ! goto skip; ! } ! expires = (time_t) scan1; ! timestamp = (time_t) scan2; ! size = (off_t) scan3; ! if ((t = strrchr(log_swapfile, '/'))) ! sfileno = atoi(t + 1); ! else ! sfileno = atoi(log_swapfile); ! storeSwapFullPath(sfileno, swapfile); ! /* ! * Note that swapfile may be different than log_swapfile if ! * another cache_dir is added. ! */ ! if (!data->fast_mode) { ! if (stat(swapfile, &sb) < 0) { ! if (expires < cached_curtime) { ! debug(20, 3, "storeRebuildFromDisk: Expired: \n", url); ! if (opt_unlink_on_reload) ! safeunlink(swapfile, 1); ! data->expcount++; ! } else { ! debug(20, 3, "storeRebuildFromDisk: Swap file missing: : %s: %s.\n", url, swapfile, xstrerror()); if (opt_unlink_on_reload) safeunlink(log_swapfile, 1); } ! goto skip; } ! /* Empty swap file? */ ! if (sb.st_size == 0) { ! if (opt_unlink_on_reload) ! safeunlink(log_swapfile, 1); ! goto skip; } ! /* timestamp might be a little bigger than sb.st_mtime */ ! delta = (int) (timestamp - sb.st_mtime); ! if (delta > REBUILD_TIMESTAMP_DELTA_MAX || delta < 0) { ! /* this log entry doesn't correspond to this file */ ! data->clashcount++; ! goto skip; ! } ! /* Wrong size? */ ! if (sb.st_size != size) { ! /* this log entry doesn't correspond to this file */ ! data->clashcount++; ! goto skip; ! } ! ! timestamp = sb.st_mtime; ! debug(20, 10, "storeRebuildFromDisk: Cached file exists: : %s\n", ! url, swapfile); ! } ! if ((e = storeGet(url))) { ! if(e->timestamp==timestamp && e->swap_file_number==sfileno) { ! /* This line is already in memory metadata, we are done */ ! debug(20, 4, "storeRebuildFromDisk: Swapfile %d already in memory:\n", sfileno, url); ! return 0; ! } else if (e->timestamp > timestamp) { ! /* We already have a newer object in memory, throw this old one away */ ! debug(20, 3, "storeRebuildFromDisk: Replaced: %s\n", url); if (opt_unlink_on_reload) ! safeunlink(swapfile,1); ! data->dupcount++; ! goto skip; } ! debug(20, 6, "storeRebuildFromDisk: Duplicate: \n", url); ! storeRelease(e); ! data->objcount--; ! data->dupcount++; } ! if (expires < cached_curtime) { ! debug(20, 3, "storeRebuildFromDisk: Expired: \n", url); ! if (opt_unlink_on_reload) ! safeunlink(swapfile, 1); ! data->expcount++; ! goto skip; ! } ! /* Is the swap file number taken by a active request? */ ! if(file_map_bit_test(sfileno)) { ! /* Yes is is, we can't use this swapfile */ ! debug(20, 4, "storeRebuildFromDisk: Active clash: ",sfileno,url); ! if (opt_unlink_on_reload) ! safeunlink(swapfile, 1); ! data->clashcount++; ! goto skip; ! } ! ! /* update store_swap_size */ ! store_swap_size += (int) ((size + 1023) >> 10); ! data->objcount++; ! ! storeAddDiskRestore(url, sfileno, (int) size, expires, timestamp); ! CacheInfo->proto_newobject(CacheInfo, ! CacheInfo->proto_id(url), ! (int) size, TRUE); ! ! skip: ! return 1; ! } ! ! /* meta data recreated from disk image in swap directory */ ! static void storeRebuiltFromDisk(data) ! struct storeRebuild_data *data; ! { ! time_t r; ! time_t stop; stop = getCurrentTime(); ! r = stop - data->start; debug(20, 1, "Finished rebuilding storage from disk image.\n"); ! debug(20, 1, " %7d Lines read from previous logfile.\n", data->linecount); ! debug(20, 1, " %7d Objects loaded.\n", data->objcount); ! debug(20, 1, " %7d Objects expired.\n", data->expcount); ! debug(20, 1, " %7d Duplicate URLs purged.\n", data->dupcount); ! debug(20, 1, " %7d Swapfile clashes avoided.\n", data->clashcount); debug(20, 1, " Took %d seconds (%6.1lf objects/sec).\n", ! r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1)); debug(20, 1, " store_swap_size = %dk\n", store_swap_size); ! ok_write_clean_log=1; ! ! fclose(data->log); ! safe_free(data); } + void storeStartRebuildFromDisk() + { + char filename[4096]; + struct stat sb; + int i; + struct storeRebuild_data *data; + time_t last_clean; + + if (stat(swaplog_file, &sb) < 0) { + debug(20, 1, "storeRebuildFromDisk: No log file\n"); + ok_write_clean_log=1; + return; + } + + data=xmalloc(sizeof(*data)); + memset(data,0,sizeof(*data)); + + for (i = 0; i < ncache_dirs; ++i) + debug(20, 1, "Rebuilding storage from disk image in %s\n", swappath(i)); + data->start = getCurrentTime(); + + /* Check if log is clean */ + sprintf(filename, "%s/log-last-clean", swappath(0)); + if (stat(filename, &sb) >= 0) { + last_clean = sb.st_mtime; + sprintf(filename, "%s/log", swappath(0)); + if (stat(filename, &sb) >= 0) { + data->fast_mode = (sb.st_mtime <= last_clean) ? 1 : 0; + } + } + + /* Open the existing swap log for reading */ + if ((data->log = fopen(swaplog_file, "r")) == (FILE *) NULL) { + sprintf(tmp_error_buf, "storeRebuildFromDisk: %s: %s", + swaplog_file, xstrerror()); + fatal(tmp_error_buf); + } + if (data->fast_mode) + debug(20, 1, "Rebuilding in FAST MODE.\n"); + + memset(data->line_in, '\0', 4096); + + /* Start reading the log file */ + runInBackground("storeRebuild",storeDoRebuildFromDisk,data,storeRebuiltFromDisk); + } /* return current swap size in kilo-bytes */ int storeGetSwapSize() *************** *** 2382,2392 **** sprintf(swaplog_file, "%s/log", swappath(0)); - if (!zap_disk_store) { - ok_write_clean_log = 0; - storeRebuildFromDisk(); - ok_write_clean_log = 1; - } swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT | O_APPEND); if (swaplog_fd < 0) { sprintf(tmp_error_buf, "Cannot open swap logfile: %s", swaplog_file); --- 2427,2432 ---- *************** *** 2403,2408 **** --- 2443,2455 ---- fatal(tmp_error_buf); } swaplog_lock = file_write_lock(swaplog_fd); + + if (!zap_disk_store) { + ok_write_clean_log = 0; + storeStartRebuildFromDisk(); + } else { + ok_write_clean_log = 1; + } if (dir_created || zap_disk_store) storeCreateSwapSubDirs();