This patch is generated from the lfs-2_5 branch of s2_5 in squid
Tue Apr 5 10:25:49 2005 GMT
See http://devel.squid-cache.org/
Index: squid/configure.in
diff -u squid/configure.in:1.42.2.60 squid/configure.in:1.42.2.59.2.9
--- squid/configure.in:1.42.2.60 Fri Mar 18 17:42:53 2005
+++ squid/configure.in Thu Mar 24 19:39:56 2005
@@ -747,6 +747,75 @@
fi
])
+dnl Enable Large file support
+buildmodel=""
+needlargefiles=
+
+AC_ARG_WITH(large-files,
+[ --with-large-files Enable support for large files (logs etc).],
+[ if test "$withval" = yes; then
+ needlargefiles=1
+ fi
+])
+
+AC_ARG_ENABLE(large-cache-files,
+[ --enable-cache-large-files
+ Enable support for large cache files (>2GB).
+ WARNING: on-disk cache format is changed by this option],
+[ if test "$enableval" = "yes" ; then
+ echo "Large cache file support enabled"
+ AC_DEFINE(LARGE_CACHE_FILES, 1, [Support large cache files > 2GB])
+dnl needlargefiles=1
+ fi
+])
+
+dnl UNIX Build environment
+AC_ARG_WITH(build-environment,
+[ --with-build-environment=model
+ The build environment to use. Normally one of
+ POSIX_V6_ILP32_OFF32 32 bits
+ POSIX_V6_ILP32_OFFBIG 32 bits with large file support
+ POSIX_V6_LP64_OFF64 64 bits
+ POSIX_V6_LPBIG_OFFBIG large pointers and files
+ XBS5_ILP32_OFF32 32 bits (legacy)
+ XBS5_ILP32_OFFBIG 32 bits with large file support (legacy)
+ XBS5_LP64_OFF64 64 bits (legacy)
+ XBS5_LPBIG_OFFBIG large pointers and files (legacy)
+ default The default for your OS],
+[ case "$enableval" in
+ yes|no)
+ echo "--with-build-environment expects a build environment string as used by getconf"
+ exit 1
+ ;;
+ _*)
+ buildmodel="$enableval"
+ ;;
+ *)
+ buildmodel="_$enableval"
+ ;;
+ esac
+])
+
+if test $needlargefiles && test -z "$buildmodel"; then
+ for model in POSIX_V6_ILP32_OFFBIG XBS5_ILP32_OFFBIG POSIX_V6_LP64_OFF64 XBS5_LP64_OFF64 POSIX_V6_LPBIG_OFFBIG XBS5_LPBIG_OFFBIG; do
+ if test "`getconf _$model 2>/dev/null || true`" = 1 || test "`getconf $model 2>/dev/null || true`" ; then
+ buildmodel=$model
+ break
+ fi
+ done
+ if test -z "$buildmodel"; then
+ echo "WARNING: No suitable build environment found for large files. Trying to use _FILE_OFFSET_BITS=64"
+ sleep 1
+ CFLAGS="-D_FILE_OFFSET_BITS=64 $CFLAGS"
+ fi
+fi
+if test -n "$buildmodel" && test "$buildmodel" != "_default"; then
+ echo "Using $buildmodel build environment"
+ CFLAGS="`getconf ${buildmodel}_CFLAGS` $CFLAGS"
+ LIBS="`getconf ${buildmodel}_LIBS` $LIBS"
+ LDFLAGS="`getconf ${buildmodel}_LDFLAGS` $LDFLAGS"
+fi
+
AM_CONDITIONAL(MAKE_LEAKFINDER, false)
dnl Enable Leak Finding Functions
AC_ARG_ENABLE(leakfinder,
@@ -1743,6 +1812,7 @@
srand48 \
srandom \
statfs \
+ strtoll \
sysconf \
syslog \
timegm \
Index: squid/helpers/basic_auth/MSNT/smblib-util.c
diff -u squid/helpers/basic_auth/MSNT/smblib-util.c:1.1.2.2 squid/helpers/basic_auth/MSNT/smblib-util.c:1.1.2.1.26.2
--- squid/helpers/basic_auth/MSNT/smblib-util.c:1.1.2.2 Thu Mar 10 06:11:45 2005
+++ squid/helpers/basic_auth/MSNT/smblib-util.c Thu Mar 10 06:18:14 2005
@@ -255,7 +255,6 @@
Con_Handle->Encrypt_Key_Len = SVAL(SMB_Hdr(pkt), SMB_negrLM_ekl_offset);
p = (SMB_Hdr(pkt) + SMB_negrLM_buf_offset);
- fprintf(stderr, "%d", (int) (SMB_Hdr(pkt) + SMB_negrLM_buf_offset));
memcpy(Con_Handle->Encrypt_Key, p, 8);
p = (SMB_Hdr(pkt) + SMB_negrLM_buf_offset + Con_Handle->Encrypt_Key_Len);
Index: squid/helpers/external_acl/winbind_group/wb_check_group.c
diff -u squid/helpers/external_acl/winbind_group/wb_check_group.c:1.2.2.11 squid/helpers/external_acl/winbind_group/wb_check_group.c:1.2.2.11.16.1
--- squid/helpers/external_acl/winbind_group/wb_check_group.c:1.2.2.11 Fri Sep 12 19:14:23 2003
+++ squid/helpers/external_acl/winbind_group/wb_check_group.c Sat Mar 5 12:58:56 2005
@@ -364,7 +364,7 @@
if ((p = strchr(buf, '\r')) != NULL)
*p = '\0'; /* strip \r */
- debug("Got '%s' from Squid (length: %d).\n",buf,strlen(buf));
+ debug("Got '%s' from Squid (length: %d).\n",buf,(int)strlen(buf));
if (buf[0] == '\0') {
warn("Invalid Request\n");
Index: squid/lib/rfc1035.c
diff -u squid/lib/rfc1035.c:1.7.6.6 squid/lib/rfc1035.c:1.7.6.6.6.3
--- squid/lib/rfc1035.c:1.7.6.6 Sun Jan 16 19:14:23 2005
+++ squid/lib/rfc1035.c Fri Mar 25 17:52:22 2005
@@ -113,10 +113,10 @@
* Packs a rfc1035_header structure into a buffer.
* Returns number of octets packed (should always be 12)
*/
-static off_t
+static int
rfc1035HeaderPack(char *buf, size_t sz, rfc1035_header * hdr)
{
- off_t off = 0;
+ int off = 0;
unsigned short s;
unsigned short t;
assert(sz >= 12);
@@ -158,10 +158,10 @@
* bytes to follow. Labels must be smaller than 64 octets.
* Returns number of octets packed.
*/
-static off_t
+static int
rfc1035LabelPack(char *buf, size_t sz, const char *label)
{
- off_t off = 0;
+ int off = 0;
size_t len = label ? strlen(label) : 0;
if (label)
assert(!strchr(label, '.'));
@@ -183,10 +183,10 @@
* Note message compression is not supported here.
* Returns number of octets packed.
*/
-static off_t
+static int
rfc1035NamePack(char *buf, size_t sz, const char *name)
{
- off_t off = 0;
+ int off = 0;
char *copy = strdup(name);
char *t;
/*
@@ -206,14 +206,14 @@
* Packs a QUESTION section of a message.
* Returns number of octets packed.
*/
-static off_t
+static int
rfc1035QuestionPack(char *buf,
size_t sz,
const char *name,
unsigned short type,
unsigned short class)
{
- off_t off = 0;
+ int off = 0;
unsigned short s;
off += rfc1035NamePack(buf + off, sz - off, name);
s = htons(type);
@@ -238,7 +238,7 @@
* Returns 0 (success) or 1 (error)
*/
static int
-rfc1035HeaderUnpack(const char *buf, size_t sz, off_t * off, rfc1035_header * h)
+rfc1035HeaderUnpack(const char *buf, size_t sz, int *off, rfc1035_header * h)
{
unsigned short s;
unsigned short t;
@@ -299,9 +299,9 @@
* Returns 0 (success) or 1 (error)
*/
static int
-rfc1035NameUnpack(const char *buf, size_t sz, off_t * off, unsigned short *rdlength, char *name, size_t ns, int rdepth)
+rfc1035NameUnpack(const char *buf, size_t sz, int *off, unsigned short *rdlength, char *name, size_t ns, int rdepth)
{
- off_t no = 0;
+ int no = 0;
unsigned char c;
size_t len;
assert(ns > 0);
@@ -311,8 +311,8 @@
if (c > 191) {
/* blasted compression */
unsigned short s;
- off_t ptr;
- if (rdepth > 64) /* infinite pointer loop */
+ int ptr;
+ if (rdepth > 64) /* infinite pointer loop */
return 1;
memcpy(&s, buf + (*off), sizeof(s));
s = ntohs(s);
@@ -367,12 +367,12 @@
* Returns 0 (success) or 1 (error)
*/
static int
-rfc1035RRUnpack(const char *buf, size_t sz, off_t * off, rfc1035_rr * RR)
+rfc1035RRUnpack(const char *buf, size_t sz, int *off, rfc1035_rr * RR)
{
unsigned short s;
unsigned int i;
unsigned short rdlength;
- off_t rdata_off;
+ int rdata_off;
if (rfc1035NameUnpack(buf, sz, off, NULL, RR->name, RFC1035_MAXHOSTNAMESZ, 0)) {
RFC1035_UNPACK_DEBUG;
memset(RR, '\0', sizeof(*RR));
@@ -413,7 +413,7 @@
case RFC1035_TYPE_PTR:
RR->rdata = malloc(RFC1035_MAXHOSTNAMESZ);
rdata_off = *off;
- RR->rdlength = 0; /* Filled in by rfc1035NameUnpack */
+ RR->rdlength = 0; /* Filled in by rfc1035NameUnpack */
if (rfc1035NameUnpack(buf, sz, &rdata_off, &RR->rdlength, RR->rdata, RFC1035_MAXHOSTNAMESZ, 0))
return 1;
if (rdata_off != ((*off) + rdlength)) {
@@ -515,7 +515,7 @@
rfc1035_rr ** records,
unsigned short *id)
{
- off_t off = 0;
+ int off = 0;
int l;
int i;
int nr = 0;
@@ -562,8 +562,8 @@
}
if (hdr.ancount == 0)
return 0;
- recs = calloc((int)hdr.ancount, sizeof(*recs));
- for (i = 0; i < (int)hdr.ancount; i++) {
+ recs = calloc((int) hdr.ancount, sizeof(*recs));
+ for (i = 0; i < (int) hdr.ancount; i++) {
if (off >= sz) { /* corrupt packet */
RFC1035_UNPACK_DEBUG;
break;
Index: squid/src/HttpHdrContRange.c
diff -u squid/src/HttpHdrContRange.c:1.5 squid/src/HttpHdrContRange.c:1.5.74.3
--- squid/src/HttpHdrContRange.c:1.5 Wed Oct 24 02:42:11 2001
+++ squid/src/HttpHdrContRange.c Fri Mar 25 17:52:22 2005
@@ -81,7 +81,7 @@
p++;
/* do we have last-pos ? */
if (p - field < flen) {
- ssize_t last_pos;
+ squid_off_t last_pos;
if (!httpHeaderParseSize(p, &last_pos))
return 0;
spec->length = size_diff(last_pos + 1, spec->offset);
@@ -101,8 +101,8 @@
if (!known_spec(spec->offset) || !known_spec(spec->length))
packerPrintf(p, "*");
else
- packerPrintf(p, "bytes %ld-%ld",
- (long int) spec->offset, (long int) spec->offset + spec->length - 1);
+ packerPrintf(p, "bytes %" PRINTF_OFF_T "-%" PRINTF_OFF_T "",
+ spec->offset, spec->offset + spec->length - 1);
}
/*
@@ -152,9 +152,9 @@
range->elength = range_spec_unknown;
else if (!httpHeaderParseSize(p, &range->elength))
return 0;
- debug(68, 8) ("parsed content-range field: %ld-%ld / %ld\n",
- (long int) range->spec.offset, (long int) range->spec.offset + range->spec.length - 1,
- (long int) range->elength);
+ debug(68, 8) ("parsed content-range field: %" PRINTF_OFF_T "-%" PRINTF_OFF_T " / %" PRINTF_OFF_T "\n",
+ range->spec.offset, range->spec.offset + range->spec.length - 1,
+ range->elength);
return 1;
}
@@ -187,7 +187,7 @@
}
void
-httpHdrContRangeSet(HttpHdrContRange * cr, HttpHdrRangeSpec spec, ssize_t ent_len)
+httpHdrContRangeSet(HttpHdrContRange * cr, HttpHdrRangeSpec spec, squid_off_t ent_len)
{
assert(cr && ent_len >= 0);
cr->spec = spec;
Index: squid/src/HttpHdrRange.c
diff -u squid/src/HttpHdrRange.c:1.7 squid/src/HttpHdrRange.c:1.7.74.3
--- squid/src/HttpHdrRange.c:1.7 Wed Oct 24 02:42:11 2001
+++ squid/src/HttpHdrRange.c Fri Mar 25 17:52:22 2005
@@ -55,7 +55,7 @@
/* local constants */
-#define range_spec_unknown ((ssize_t)-1)
+#define range_spec_unknown ((squid_off_t)-1)
/* local routines */
#define known_spec(s) ((s) != range_spec_unknown)
@@ -102,7 +102,7 @@
p++;
/* do we have last-pos ? */
if (p - field < flen) {
- ssize_t last_pos;
+ squid_off_t last_pos;
if (!httpHeaderParseSize(p, &last_pos))
return NULL;
spec.length = size_diff(last_pos + 1, spec.offset);
@@ -136,12 +136,12 @@
httpHdrRangeSpecPackInto(const HttpHdrRangeSpec * spec, Packer * p)
{
if (!known_spec(spec->offset)) /* suffix */
- packerPrintf(p, "-%ld", (long int) spec->length);
+ packerPrintf(p, "-%" PRINTF_OFF_T, spec->length);
else if (!known_spec(spec->length)) /* trailer */
- packerPrintf(p, "%ld-", (long int) spec->offset);
+ packerPrintf(p, "%" PRINTF_OFF_T "-", spec->offset);
else /* range */
- packerPrintf(p, "%ld-%ld",
- (long int) spec->offset, (long int) spec->offset + spec->length - 1);
+ packerPrintf(p, "%" PRINTF_OFF_T "-%" PRINTF_OFF_T,
+ spec->offset, spec->offset + spec->length - 1);
}
/* fills "absent" positions in range specification based on response body size
@@ -151,8 +151,8 @@
static int
httpHdrRangeSpecCanonize(HttpHdrRangeSpec * spec, size_t clen)
{
- debug(64, 5) ("httpHdrRangeSpecCanonize: have: [%ld, %ld) len: %ld\n",
- (long int) spec->offset, (long int) spec->offset + spec->length, (long int) spec->length);
+ debug(64, 5) ("httpHdrRangeSpecCanonize: have: [%" PRINTF_OFF_T ", %" PRINTF_OFF_T ") len: %" PRINTF_OFF_T "\n",
+ spec->offset, spec->offset + spec->length, spec->length);
if (!known_spec(spec->offset)) /* suffix */
spec->offset = size_diff(clen, spec->length);
else if (!known_spec(spec->length)) /* trailer */
@@ -162,8 +162,8 @@
assert(known_spec(spec->offset));
spec->length = size_min(size_diff(clen, spec->offset), spec->length);
/* check range validity */
- debug(64, 5) ("httpHdrRangeSpecCanonize: done: [%ld, %ld) len: %ld\n",
- (long int) spec->offset, (long int) spec->offset + (long int) spec->length, (long int) spec->length);
+ debug(64, 5) ("httpHdrRangeSpecCanonize: done: [%" PRINTF_OFF_T ", %" PRINTF_OFF_T ") len: %" PRINTF_OFF_T "\n",
+ spec->offset, spec->offset + spec->length, spec->length);
return spec->length > 0;
}
@@ -303,7 +303,7 @@
* - there is at least one range spec
*/
int
-httpHdrRangeCanonize(HttpHdrRange * range, ssize_t clen)
+httpHdrRangeCanonize(HttpHdrRange * range, squid_off_t clen)
{
int i;
HttpHdrRangeSpec *spec;
@@ -415,10 +415,10 @@
* Returns lowest known offset in range spec(s), or range_spec_unknown
* this is used for size limiting
*/
-ssize_t
+squid_off_t
httpHdrRangeFirstOffset(const HttpHdrRange * range)
{
- ssize_t offset = range_spec_unknown;
+ squid_off_t offset = range_spec_unknown;
HttpHdrRangePos pos = HttpHdrRangeInitPos;
const HttpHdrRangeSpec *spec;
assert(range);
@@ -435,11 +435,11 @@
* ranges are combined into one, for example FTP REST.
* Use 0 for size if unknown
*/
-ssize_t
-httpHdrRangeLowestOffset(const HttpHdrRange * range, ssize_t size)
+squid_off_t
+httpHdrRangeLowestOffset(const HttpHdrRange * range, squid_off_t size)
{
- ssize_t offset = range_spec_unknown;
- ssize_t current;
+ squid_off_t offset = range_spec_unknown;
+ squid_off_t current;
HttpHdrRangePos pos = HttpHdrRangeInitPos;
const HttpHdrRangeSpec *spec;
assert(range);
Index: squid/src/HttpHeader.c
diff -u squid/src/HttpHeader.c:1.10.6.20 squid/src/HttpHeader.c:1.10.6.19.2.5
--- squid/src/HttpHeader.c:1.10.6.20 Thu Mar 10 06:11:51 2005
+++ squid/src/HttpHeader.c Fri Mar 25 17:52:22 2005
@@ -80,7 +80,7 @@
{"Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr},
{"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
{"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
- {"Content-Length", HDR_CONTENT_LENGTH, ftInt},
+ {"Content-Length", HDR_CONTENT_LENGTH, ftSize},
{"Content-Location", HDR_CONTENT_LOCATION, ftStr},
{"Content-MD5", HDR_CONTENT_MD5, ftStr}, /* for now */
{"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
@@ -814,6 +814,17 @@
}
void
+httpHeaderPutSize(HttpHeader * hdr, http_hdr_type id, squid_off_t number)
+{
+ char size[64];
+ assert_eid(id);
+ assert(Headers[id].type == ftSize); /* must be of an appropriate type */
+ assert(number >= 0);
+ snprintf(size, sizeof(size), "%" PRINTF_OFF_T, number);
+ httpHeaderAddEntry(hdr, httpHeaderEntryCreate(id, NULL, size));
+}
+
+void
httpHeaderPutTime(HttpHeader * hdr, http_hdr_type id, time_t htime)
{
assert_eid(id);
@@ -919,6 +930,21 @@
return value;
}
+squid_off_t
+httpHeaderGetSize(const HttpHeader * hdr, http_hdr_type id)
+{
+ HttpHeaderEntry *e;
+ squid_off_t value = -1;
+ int ok;
+ assert_eid(id);
+ assert(Headers[id].type == ftSize); /* must be of an appropriate type */
+ if ((e = httpHeaderFindEntry(hdr, id))) {
+ ok = httpHeaderParseSize(strBuf(e->value), &value);
+ httpHeaderNoteParsedEntry(e->id, e->value, !ok);
+ }
+ return value;
+}
+
time_t
httpHeaderGetTime(const HttpHeader * hdr, http_hdr_type id)
{
Index: squid/src/HttpHeaderTools.c
diff -u squid/src/HttpHeaderTools.c:1.7.12.3 squid/src/HttpHeaderTools.c:1.7.12.3.8.3
--- squid/src/HttpHeaderTools.c:1.7.12.3 Sat Sep 25 19:14:22 2004
+++ squid/src/HttpHeaderTools.c Wed Mar 2 19:44:01 2005
@@ -138,7 +138,7 @@
/* wrapper arrounf PutContRange */
void
-httpHeaderAddContRange(HttpHeader * hdr, HttpHdrRangeSpec spec, ssize_t ent_len)
+httpHeaderAddContRange(HttpHeader * hdr, HttpHdrRangeSpec spec, squid_off_t ent_len)
{
HttpHdrContRange *cr = httpHdrContRangeCreate();
assert(hdr && ent_len >= 0);
@@ -316,11 +316,14 @@
return 1;
}
-int
-httpHeaderParseSize(const char *start, ssize_t * value)
+squid_off_t
+httpHeaderParseSize(const char *start, squid_off_t * value)
{
- int v;
- const int res = httpHeaderParseInt(start, &v);
+ squid_off_t v;
+ char *end;
+ int res;
+ v = strto_off_t(start, &end, 10);
+ res = start != end;
assert(value);
*value = res ? v : 0;
return res;
Index: squid/src/HttpReply.c
diff -u squid/src/HttpReply.c:1.10.6.4 squid/src/HttpReply.c:1.10.6.4.6.3
--- squid/src/HttpReply.c:1.10.6.4 Fri Jan 21 19:15:02 2005
+++ squid/src/HttpReply.c Wed Mar 2 20:43:41 2005
@@ -135,7 +135,7 @@
* returns true on success.
*/
int
-httpReplyParse(HttpReply * rep, const char *buf, ssize_t end)
+httpReplyParse(HttpReply * rep, const char *buf, size_t end)
{
/*
* this extra buffer/copy will be eliminated when headers become
@@ -194,9 +194,10 @@
packerClean(&p);
}
+#if UNUSED_CODE
MemBuf
httpPackedReply(http_version_t ver, http_status status, const char *ctype,
- int clen, time_t lmt, time_t expires)
+ squid_off_t clen, time_t lmt, time_t expires)
{
HttpReply *rep = httpReplyCreate();
MemBuf mb;
@@ -205,6 +206,7 @@
httpReplyDestroy(rep);
return mb;
}
+#endif
MemBuf
httpPacked304Reply(const HttpReply * rep)
@@ -230,7 +232,7 @@
void
httpReplySetHeaders(HttpReply * reply, http_version_t ver, http_status status, const char *reason,
- const char *ctype, int clen, time_t lmt, time_t expires)
+ const char *ctype, squid_off_t clen, time_t lmt, time_t expires)
{
HttpHeader *hdr;
assert(reply);
@@ -245,7 +247,7 @@
} else
reply->content_type = StringNull;
if (clen >= 0)
- httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, clen);
+ httpHeaderPutSize(hdr, HDR_CONTENT_LENGTH, clen);
if (expires >= 0)
httpHeaderPutTime(hdr, HDR_EXPIRES, expires);
if (lmt > 0) /* this used to be lmt != 0 @?@ */
@@ -267,7 +269,7 @@
hdr = &reply->header;
httpHeaderPutStr(hdr, HDR_SERVER, full_appname_string);
httpHeaderPutTime(hdr, HDR_DATE, squid_curtime);
- httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, 0);
+ httpHeaderPutSize(hdr, HDR_CONTENT_LENGTH, 0);
httpHeaderPutStr(hdr, HDR_LOCATION, loc);
reply->date = squid_curtime;
reply->content_length = 0;
@@ -342,7 +344,7 @@
{
const HttpHeader *hdr = &rep->header;
const char *str;
- rep->content_length = httpHeaderGetInt(hdr, HDR_CONTENT_LENGTH);
+ rep->content_length = httpHeaderGetSize(hdr, HDR_CONTENT_LENGTH);
rep->date = httpHeaderGetTime(hdr, HDR_DATE);
rep->last_modified = httpHeaderGetTime(hdr, HDR_LAST_MODIFIED);
str = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE);
@@ -449,7 +451,7 @@
/*
* Returns the body size of a HTTP response
*/
-int
+squid_off_t
httpReplyBodySize(method_t method, const HttpReply * reply)
{
if (reply->sline.version.major < 1)
Index: squid/src/MemBuf.c
diff -u squid/src/MemBuf.c:1.5.30.2 squid/src/MemBuf.c:1.5.30.2.8.1
--- squid/src/MemBuf.c:1.5.30.2 Tue Oct 5 19:15:11 2004
+++ squid/src/MemBuf.c Wed Mar 2 16:11:55 2005
@@ -192,7 +192,7 @@
/* calls memcpy, appends exactly size bytes, extends buffer if needed */
void
-memBufAppend(MemBuf * mb, const char *buf, mb_size_t sz)
+memBufAppend(MemBuf * mb, const char *buf, int sz)
{
assert(mb && buf && sz >= 0);
assert(mb->buf);
Index: squid/src/MemPool.c
diff -u squid/src/MemPool.c:1.6.6.1 squid/src/MemPool.c:1.6.6.1.34.3
--- squid/src/MemPool.c:1.6.6.1 Sun Mar 10 21:06:56 2002
+++ squid/src/MemPool.c Wed Mar 2 20:43:41 2005
@@ -46,7 +46,7 @@
/* module globals */
/* huge constant to set mem_idle_limit to "unlimited" */
-static const size_t mem_unlimited_size = 2 * 1024 * MB;
+static const size_t mem_unlimited_size = 2 * 1024 * MB - 1;
/* we cannot keep idle more than this limit */
static size_t mem_idle_limit = 0;
@@ -58,9 +58,9 @@
static Stack Pools;
/* local prototypes */
-static void memShrink(ssize_t new_limit);
+static void memShrink(size_t new_limit);
static void memPoolDescribe(const MemPool * pool);
-static void memPoolShrink(MemPool * pool, ssize_t new_limit);
+static void memPoolShrink(MemPool * pool, size_t new_limit);
static double
@@ -135,9 +135,9 @@
static void
-memShrink(ssize_t new_limit)
+memShrink(size_t new_limit)
{
- ssize_t start_limit = TheMeter.idle.level;
+ size_t start_limit = TheMeter.idle.level;
int i;
assert(start_limit >= 0 && new_limit >= 0);
debug(63, 1) ("memShrink: started with %ld KB goal: %ld KB\n",
@@ -145,7 +145,7 @@
/* first phase: cut proportionally to the pool idle size */
for (i = 0; i < Pools.count && TheMeter.idle.level > new_limit; ++i) {
MemPool *pool = Pools.items[i];
- const ssize_t target_pool_size = (size_t) ((double) pool->meter.idle.level * new_limit) / start_limit;
+ const size_t target_pool_size = (size_t) ((double) pool->meter.idle.level * new_limit) / start_limit;
memPoolShrink(pool, target_pool_size);
}
debug(63, 1) ("memShrink: 1st phase done with %ld KB left\n", (long int) toKB(TheMeter.idle.level));
@@ -276,7 +276,7 @@
}
static void
-memPoolShrink(MemPool * pool, ssize_t new_limit)
+memPoolShrink(MemPool * pool, size_t new_limit)
{
assert(pool);
assert(new_limit >= 0);
@@ -308,7 +308,7 @@
memPoolInUseSize(const MemPool * pool)
{
assert(pool);
- return pool->obj_size * pool->meter.inuse.level;
+ return (pool->obj_size * pool->meter.inuse.level);
}
/* to-do: make debug level a parameter? */
@@ -321,7 +321,7 @@
(long int) toKB(memPoolInUseSize(pool)));
}
-size_t
+int
memTotalAllocated(void)
{
return TheMeter.alloc.level;
Index: squid/src/Packer.c
diff -u squid/src/Packer.c:1.4 squid/src/Packer.c:1.4.126.1
--- squid/src/Packer.c:1.4 Fri Jan 12 00:20:32 2001
+++ squid/src/Packer.c Wed Mar 2 16:11:55 2005
@@ -93,7 +93,7 @@
/* append()'s */
static void (*const store_append) (StoreEntry *, const char *, int) = &storeAppend;
-static void (*const memBuf_append) (MemBuf *, const char *, mb_size_t) = &memBufAppend;
+static void (*const memBuf_append) (MemBuf *, const char *, int) = &memBufAppend;
/* vprintf()'s */
static void (*const store_vprintf) (StoreEntry *, const char *, va_list ap) = &storeAppendVPrintf;
@@ -109,7 +109,7 @@
assert(p && e);
p->append = (append_f) store_append;
p->vprintf = (vprintf_f) store_vprintf;
- p->real_handler = e;
+ p->real_handle = e;
}
/* init with this to accumulate data in MemBuf */
@@ -119,7 +119,7 @@
assert(p && mb);
p->append = (append_f) memBuf_append;
p->vprintf = (vprintf_f) memBuf_vprintf;
- p->real_handler = mb;
+ p->real_handle = mb;
}
/* call this when you are done */
@@ -130,15 +130,15 @@
/* it is not really necessary to do this, but, just in case... */
p->append = NULL;
p->vprintf = NULL;
- p->real_handler = NULL;
+ p->real_handle = NULL;
}
void
packerAppend(Packer * p, const char *buf, int sz)
{
assert(p);
- assert(p->real_handler && p->append);
- p->append(p->real_handler, buf, sz);
+ assert(p->real_handle && p->append);
+ p->append(p->real_handle, buf, sz);
}
#if STDC_HEADERS
@@ -161,7 +161,7 @@
fmt = va_arg(args, char *);
#endif
assert(p);
- assert(p->real_handler && p->vprintf);
- p->vprintf(p->real_handler, fmt, args);
+ assert(p->real_handle && p->vprintf);
+ p->vprintf(p->real_handle, fmt, args);
va_end(args);
}
Index: squid/src/access_log.c
diff -u squid/src/access_log.c:1.15.6.6 squid/src/access_log.c:1.15.6.6.8.1
--- squid/src/access_log.c:1.15.6.6 Mon Sep 27 19:14:07 2004
+++ squid/src/access_log.c Fri Mar 11 17:31:51 2005
@@ -245,8 +245,8 @@
client = inet_ntoa(al->cache.caddr);
user = accessLogFormatName(al->cache.authuser ?
al->cache.authuser : al->cache.rfc931);
- logfilePrintf(logfile, "%9d.%03d %6d %s %s/%03d %lu %s %s %s %s%s/%s %s",
- (int) current_time.tv_sec,
+ logfilePrintf(logfile, "%9ld.%03d %6d %s %s/%03d %lu %s %s %s %s%s/%s %s",
+ (long int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
al->cache.msec,
client,
Index: squid/src/asn.c
diff -u squid/src/asn.c:1.15.6.1 squid/src/asn.c:1.15.6.1.20.1
--- squid/src/asn.c:1.15.6.1 Wed Jun 18 19:13:57 2003
+++ squid/src/asn.c Wed Mar 2 19:44:01 2005
@@ -73,8 +73,8 @@
store_client *sc;
request_t *request;
int as_number;
- off_t seen;
- off_t offset;
+ squid_off_t seen;
+ squid_off_t offset;
};
typedef struct _ASState ASState;
Index: squid/src/authenticate.c
diff -u squid/src/authenticate.c:1.23.2.13 squid/src/authenticate.c:1.23.2.13.8.1
--- squid/src/authenticate.c:1.23.2.13 Mon Jan 10 19:17:25 2005
+++ squid/src/authenticate.c Fri Feb 25 02:48:45 2005
@@ -352,7 +352,7 @@
authenticateAuthUserClearIp(auth_user_request->auth_user);
}
-size_t
+int
authenticateAuthUserRequestIPCount(auth_user_request_t * auth_user_request)
{
assert(auth_user_request);
Index: squid/src/cache_cf.c
diff -u squid/src/cache_cf.c:1.38.6.20 squid/src/cache_cf.c:1.38.6.18.2.7
--- squid/src/cache_cf.c:1.38.6.20 Thu Mar 10 06:11:52 2005
+++ squid/src/cache_cf.c Fri Mar 25 17:52:22 2005
@@ -78,7 +78,7 @@
static void default_all(void);
static void defaults_if_none(void);
static int parse_line(char *);
-static void parseBytesLine(size_t * bptr, const char *units);
+static void parseBytesLine(squid_off_t * bptr, const char *units);
static size_t parseBytesUnits(const char *unit);
static void free_all(void);
void requirePathnameExists(const char *name, const char *path);
@@ -211,10 +211,35 @@
GetInteger(void)
{
char *token = strtok(NULL, w_space);
+ char *end;
int i;
+ double d;
if (token == NULL)
self_destruct();
- if (sscanf(token, "%d", &i) != 1)
+ i = strtol(token, &end, 0);
+ d = strtod(token, NULL);
+ if (d > INT_MAX || end == token)
+ self_destruct();
+ return i;
+}
+
+static squid_off_t
+GetOffT(void)
+{
+ char *token = strtok(NULL, w_space);
+ char *end;
+ squid_off_t i;
+ if (token == NULL)
+ self_destruct();
+ i = strto_off_t(token, &end, 0);
+#if SIZEOF_SQUID_OFF_T <= 4
+ {
+ double d = strtod(token, NULL);
+ if (d > INT_MAX)
+ end = token;
+ }
+#endif
+ if (end == token)
self_destruct();
return i;
}
@@ -223,7 +248,7 @@
update_maxobjsize(void)
{
int i;
- ssize_t ms = -1;
+ squid_off_t ms = -1;
for (i = 0; i < Config.cacheSwap.n_configured; i++) {
if (Config.cacheSwap.swapDirs[i].max_objsize > ms)
@@ -440,6 +465,26 @@
debug(22, 0) ("NOTICE: positive_dns_ttl must be larger than negative_dns_ttl. Resetting negative_dns_ttl to match\n");
Config.positiveDnsTtl = Config.negativeDnsTtl;
}
+#if SIZEOF_SQUID_FILE_SZ <= 4
+#if SIZEOF_SQUID_OFF_T <= 4
+ if (Config.Store.maxObjectSize > 0x7FFF0000) {
+ debug(22, 0) ("NOTICE: maximum_object_size limited to %d KB due to hardware limitations\n", 0x7FFF0000 / 1024);
+ Config.Store.maxObjectSize = 0x7FFF0000;
+ }
+#elif SIZEOF_OFF_T <= 4
+ if (Config.Store.maxObjectSize > 0xFFFF0000) {
+ debug(22, 0) ("NOTICE: maximum_object_size limited to %d KB due to OS limitations\n", 0xFFFF0000 / 1024);
+ Config.Store.maxObjectSize = 0xFFFF0000;
+ }
+#else
+ if (Config.Store.maxObjectSize > 0xFFFF0000) {
+ debug(22, 0) ("NOTICE: maximum_object_size limited to %d KB to keep compatibility with existing cache\n", 0xFFFF0000 / 1024);
+ Config.Store.maxObjectSize = 0xFFFF0000;
+ }
+#endif
+#endif
+ if (Config.Store.maxInMemObjSize > 8 * 1024 * 1024)
+ debug(22, 0) ("WARNING: Very large maximum_object_size_in_memory settings can have negative impact on performance\n");
}
/* Parse a time specification from the config file. Store the
@@ -493,18 +538,18 @@
}
static void
-parseBytesLine(size_t * bptr, const char *units)
+parseBytesLine(squid_off_t * bptr, const char *units)
{
char *token;
double d;
- size_t m;
- size_t u;
+ squid_off_t m;
+ squid_off_t u;
if ((u = parseBytesUnits(units)) == 0)
self_destruct();
if ((token = strtok(NULL, w_space)) == NULL)
self_destruct();
if (strcmp(token, "none") == 0 || strcmp(token, "-1") == 0) {
- *bptr = (size_t) - 1;
+ *bptr = (squid_off_t) - 1;
return;
}
d = atof(token);
@@ -1324,12 +1369,12 @@
static void
parse_cachedir_option_maxsize(SwapDir * sd, const char *option, const char *value, int reconfiguring)
{
- ssize_t size;
+ squid_off_t size;
if (!value)
self_destruct();
- size = atoi(value);
+ size = strto_off_t(value, NULL, 10);
if (reconfiguring && sd->max_objsize != size)
debug(3, 1) ("Cache dir '%s' max object size now %ld\n", sd->path, (long int) size);
@@ -2074,34 +2119,29 @@
*var = 0;
}
+#if UNUSED_CODE
static void
-dump_size_t(StoreEntry * entry, const char *name, size_t var)
+dump_size_t(StoreEntry * entry, const char *name, squid_off_t var)
{
- storeAppendPrintf(entry, "%s %d\n", name, (int) var);
+ storeAppendPrintf(entry, "%s %" PRINTF_OFF_T "\n", name, var);
}
-static void
-dump_b_size_t(StoreEntry * entry, const char *name, size_t var)
-{
- storeAppendPrintf(entry, "%s %d %s\n", name, (int) var, B_BYTES_STR);
-}
+#endif
static void
-dump_kb_size_t(StoreEntry * entry, const char *name, size_t var)
+dump_b_size_t(StoreEntry * entry, const char *name, squid_off_t var)
{
- storeAppendPrintf(entry, "%s %d %s\n", name, (int) var, B_KBYTES_STR);
+ storeAppendPrintf(entry, "%s %" PRINTF_OFF_T " %s\n", name, var, B_BYTES_STR);
}
static void
-parse_size_t(size_t * var)
+dump_kb_size_t(StoreEntry * entry, const char *name, squid_off_t var)
{
- int i;
- i = GetInteger();
- *var = (size_t) i;
+ storeAppendPrintf(entry, "%s %" PRINTF_OFF_T " %s\n", name, var, B_KBYTES_STR);
}
static void
-parse_b_size_t(size_t * var)
+parse_b_size_t(squid_off_t * var)
{
parseBytesLine(var, B_BYTES_STR);
}
@@ -2114,7 +2154,7 @@
body_size *bs;
CBDATA_INIT_TYPE(body_size);
bs = cbdataAlloc(body_size);
- parse_size_t(&bs->maxsize);
+ bs->maxsize = GetOffT();
aclParseAccessLine(&bs->access_list);
dlinkAddTail(bs, &bs->node, bodylist);
@@ -2129,7 +2169,7 @@
acl_list *l;
acl_access *head = bs->access_list;
while (head != NULL) {
- storeAppendPrintf(entry, "%s %ld %s", name, (long int) bs->maxsize,
+ storeAppendPrintf(entry, "%s %" PRINTF_OFF_T " %s", name, bs->maxsize,
head->allow ? "Allow" : "Deny");
for (l = head->acl_list; l != NULL; l = l->next) {
storeAppendPrintf(entry, " %s%s",
@@ -2166,13 +2206,13 @@
static void
-parse_kb_size_t(size_t * var)
+parse_kb_size_t(squid_off_t * var)
{
parseBytesLine(var, B_KBYTES_STR);
}
static void
-free_size_t(size_t * var)
+free_size_t(squid_off_t * var)
{
*var = 0;
}
Index: squid/src/cachemgr.c
diff -u squid/src/cachemgr.c:1.6.6.5 squid/src/cachemgr.c:1.6.6.5.26.1
--- squid/src/cachemgr.c:1.6.6.5 Sun Nov 10 14:44:12 2002
+++ squid/src/cachemgr.c Wed Mar 2 19:44:02 2005
@@ -735,7 +735,7 @@
make_auth_header(const cachemgr_request * req)
{
static char buf[1024];
- off_t l = 0;
+ int l = 0;
const char *str64;
if (!req->passwd)
return "";
Index: squid/src/cf.data.pre
diff -u squid/src/cf.data.pre:1.49.2.71 squid/src/cf.data.pre:1.49.2.69.2.2
--- squid/src/cf.data.pre:1.49.2.71 Thu Mar 10 06:11:53 2005
+++ squid/src/cf.data.pre Thu Mar 10 06:18:20 2005
@@ -1046,7 +1046,7 @@
DOC_END
NAME: ftp_list_width
-TYPE: size_t
+TYPE: int
DEFAULT: 32
LOC: Config.Ftp.list_width
DOC_START
Index: squid/src/client.c
diff -u squid/src/client.c:1.8.6.5 squid/src/client.c:1.8.6.5.10.6
--- squid/src/client.c:1.8.6.5 Wed Feb 18 19:08:38 2004
+++ squid/src/client.c Fri Mar 25 17:52:23 2005
@@ -311,7 +311,7 @@
}
loops = ping ? pcount : 1;
for (i = 0; loops == 0 || i < loops; i++) {
- int fsize = 0;
+ squid_off_t fsize = 0;
/* Connect to the server */
if ((conn = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
perror("client: socket");
@@ -321,7 +321,7 @@
perror("client: bind");
exit(1);
}
- if (client_comm_connect(conn, hostname, port, ping ? &tv1 : NULL) < 0) {
+ if (client_comm_connect(conn, hostname, port, (ping || opt_verbose) ? &tv1 : NULL) < 0) {
if (errno == 0) {
fprintf(stderr, "client: ERROR: Cannot connect to %s:%d: Host unknown.\n", hostname, port);
} else {
@@ -365,7 +365,7 @@
if (interrupted)
break;
- if (ping) {
+ if (ping || opt_verbose) {
struct tm *tmp;
time_t t2s;
long elapsed_msec;
@@ -374,18 +374,19 @@
elapsed_msec = tvSubMsec(tv1, tv2);
t2s = tv2.tv_sec;
tmp = localtime(&t2s);
- fprintf(stderr, "%d-%02d-%02d %02d:%02d:%02d [%d]: %ld.%03ld secs, %f KB/s\n",
+ fprintf(stderr, "%d-%02d-%02d %02d:%02d:%02d [%d]: %ld.%03ld secs, %f KB/s (%lldKB)\n",
tmp->tm_year + 1900, tmp->tm_mon + 1, tmp->tm_mday,
tmp->tm_hour, tmp->tm_min, tmp->tm_sec, i + 1,
elapsed_msec / 1000, elapsed_msec % 1000,
- elapsed_msec ? (double) fsize / elapsed_msec : -1.0);
+ elapsed_msec ? (double) fsize / elapsed_msec * 1000 / 1024 : -1.0,
+ (long long) (fsize + 1023) / 1024);
if (i == 0 || elapsed_msec < ping_min)
ping_min = elapsed_msec;
if (i == 0 || elapsed_msec > ping_max)
ping_max = elapsed_msec;
ping_sum += elapsed_msec;
/* Delay until next "ping_int" boundary */
- if ((loops == 0 || i + 1 < loops) && elapsed_msec < ping_int) {
+ if (ping && (loops == 0 || i + 1 < loops) && elapsed_msec < ping_int) {
struct timeval tvs;
long msec_left = ping_int - elapsed_msec;
Index: squid/src/client_db.c
diff -u squid/src/client_db.c:1.6.54.4 squid/src/client_db.c:1.6.54.4.8.2
--- squid/src/client_db.c:1.6.54.4 Mon Oct 11 19:13:35 2004
+++ squid/src/client_db.c Wed Mar 2 19:44:03 2005
@@ -77,7 +77,7 @@
}
void
-clientdbUpdate(struct in_addr addr, log_type ltype, protocol_t p, size_t size)
+clientdbUpdate(struct in_addr addr, log_type ltype, protocol_t p, squid_off_t size)
{
char *key;
ClientInfo *c;
Index: squid/src/client_side.c
diff -u squid/src/client_side.c:1.47.2.59 squid/src/client_side.c:1.47.2.56.2.8
--- squid/src/client_side.c:1.47.2.59 Thu Mar 10 06:11:55 2005
+++ squid/src/client_side.c Fri Mar 25 17:52:23 2005
@@ -125,8 +125,8 @@
static int clientCheckContentLength(request_t * r);
static DEFER httpAcceptDefer;
static log_type clientProcessRequest2(clientHttpRequest * http);
-static int clientReplyBodyTooLarge(clientHttpRequest *, ssize_t clen);
-static int clientRequestBodyTooLarge(int clen);
+static int clientReplyBodyTooLarge(clientHttpRequest *, squid_off_t clen);
+static int clientRequestBodyTooLarge(squid_off_t clen);
static void clientProcessBody(ConnStateData * conn);
static void clientEatRequestBody(clientHttpRequest *);
static BODY_HANDLER clientReadBody;
@@ -603,7 +603,7 @@
int
modifiedSince(StoreEntry * entry, request_t * request)
{
- int object_length;
+ squid_off_t object_length;
MemObject *mem = entry->mem_obj;
time_t mod_time = entry->lastmod;
debug(33, 3) ("modifiedSince: '%s'\n", storeUrl(entry));
@@ -1225,10 +1225,10 @@
* warning: assumes that HTTP headers for individual ranges at the
* time of the actuall assembly will be exactly the same as
* the headers when clientMRangeCLen() is called */
-static int
+static squid_off_t
clientMRangeCLen(clientHttpRequest * http)
{
- int clen = 0;
+ squid_off_t clen = 0;
HttpHdrRangePos pos = HttpHdrRangeInitPos;
const HttpHdrRangeSpec *spec;
MemBuf mb;
@@ -1247,8 +1247,8 @@
/* account for range content */
clen += spec->length;
- debug(33, 6) ("clientMRangeCLen: (clen += %ld + %ld) == %d\n",
- (long int) mb.size, (long int) spec->length, clen);
+ debug(33, 6) ("clientMRangeCLen: (clen += %ld + %" PRINTF_OFF_T ") == %" PRINTF_OFF_T "\n",
+ (long int) mb.size, spec->length, clen);
}
/* account for the terminating boundary */
memBufReset(&mb);
@@ -1295,9 +1295,9 @@
http->request->range = NULL;
} else {
const int spec_count = http->request->range->specs.count;
- int actual_clen = -1;
+ squid_off_t actual_clen = -1;
- debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %d\n",
+ debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %" PRINTF_OFF_T "\n",
spec_count, rep->content_length);
assert(spec_count > 0);
/* ETags should not be returned with Partial Content replies? */
@@ -1329,8 +1329,8 @@
/* replace Content-Length header */
assert(actual_clen >= 0);
httpHeaderDelById(hdr, HDR_CONTENT_LENGTH);
- httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, actual_clen);
- debug(33, 3) ("clientBuildRangeHeader: actual content length: %d\n", actual_clen);
+ httpHeaderPutSize(hdr, HDR_CONTENT_LENGTH, actual_clen);
+ debug(33, 3) ("clientBuildRangeHeader: actual content length: %" PRINTF_OFF_T "\n", actual_clen);
}
}
@@ -1734,11 +1734,11 @@
clientPackRange(clientHttpRequest * http,
HttpHdrRangeIter * i,
const char **buf,
- ssize_t * size,
+ size_t * size,
MemBuf * mb)
{
- const ssize_t copy_sz = i->debt_size <= *size ? i->debt_size : *size;
- off_t body_off = http->out.offset - i->prefix_size;
+ const size_t copy_sz = i->debt_size <= *size ? i->debt_size : *size;
+ squid_off_t body_off = http->out.offset - i->prefix_size;
assert(*size > 0);
assert(i->spec);
/*
@@ -1782,7 +1782,7 @@
* increments iterator "i"
* used by clientPackMoreRanges */
static int
-clientCanPackMoreRanges(const clientHttpRequest * http, HttpHdrRangeIter * i, ssize_t size)
+clientCanPackMoreRanges(const clientHttpRequest * http, HttpHdrRangeIter * i, size_t size)
{
/* first update "i" if needed */
if (!i->debt_size) {
@@ -1797,17 +1797,17 @@
/* extracts "ranges" from buf and appends them to mb, updating all offsets and such */
/* returns true if we need more data */
static int
-clientPackMoreRanges(clientHttpRequest * http, const char *buf, ssize_t size, MemBuf * mb)
+clientPackMoreRanges(clientHttpRequest * http, const char *buf, size_t size, MemBuf * mb)
{
HttpHdrRangeIter *i = &http->range_iter;
/* offset in range specs does not count the prefix of an http msg */
- off_t body_off = http->out.offset - i->prefix_size;
+ squid_off_t body_off = http->out.offset - i->prefix_size;
assert(size >= 0);
/* check: reply was parsed and range iterator was initialized */
assert(i->prefix_size > 0);
/* filter out data according to range specs */
while (clientCanPackMoreRanges(http, i, size)) {
- off_t start; /* offset of still missing data */
+ squid_off_t start; /* offset of still missing data */
assert(i->spec);
start = i->spec->offset + i->spec->length - i->debt_size;
debug(33, 3) ("clientPackMoreRanges: in: offset: %ld size: %ld\n",
@@ -1881,7 +1881,7 @@
}
static int
-clientReplyBodyTooLarge(clientHttpRequest * http, ssize_t clen)
+clientReplyBodyTooLarge(clientHttpRequest * http, squid_off_t clen)
{
if (0 == http->maxBodySize)
return 0; /* disabled */
@@ -1893,7 +1893,7 @@
}
static int
-clientRequestBodyTooLarge(int clen)
+clientRequestBodyTooLarge(squid_off_t clen)
{
if (0 == Config.maxRequestBodySize)
return 0; /* disabled */
@@ -1944,9 +1944,9 @@
int fd = conn->fd;
HttpReply *rep = NULL;
const char *body_buf = buf;
- ssize_t body_size = size;
+ squid_off_t body_size = size;
MemBuf mb;
- ssize_t check_size = 0;
+ squid_off_t check_size = 0;
debug(33, 5) ("clientSendMoreData: %s, %d bytes\n", http->uri, (int) size);
assert(size <= CLIENT_SOCK_SZ);
assert(http->request != NULL);
@@ -2189,14 +2189,14 @@
StoreEntry *entry = http->entry;
int done;
http->out.size += size;
- debug(33, 5) ("clientWriteComplete: FD %d, sz %ld, err %d, off %ld, len %d\n",
- fd, (long int) size, errflag, (long int) http->out.offset, entry ? objectLen(entry) : 0);
+ debug(33, 5) ("clientWriteComplete: FD %d, sz %d, err %d, off %" PRINTF_OFF_T ", len %" PRINTF_OFF_T "\n",
+ fd, (int) size, errflag, http->out.offset, entry ? objectLen(entry) : (squid_off_t) 0);
if (size > 0) {
kb_incr(&statCounter.client_http.kbytes_out, size);
if (isTcpHit(http->log_type))
kb_incr(&statCounter.client_http.hit_kbytes_out, size);
}
-#if SIZEOF_SIZE_T == 4
+#if SIZEOF_SQUID_OFF_T <= 4
if (http->out.size > 0x7FFF0000) {
debug(33, 1) ("WARNING: closing FD %d to prevent counter overflow\n", fd);
debug(33, 1) ("\tclient %s\n", inet_ntoa(http->conn->peer.sin_addr));
@@ -2205,7 +2205,7 @@
comm_close(fd);
} else
#endif
-#if SIZEOF_OFF_T == 4
+#if SIZEOF_SQUID_OFF_T <= 4
if (http->out.offset > 0x7FFF0000) {
debug(33, 1) ("WARNING: closing FD %d to prevent counter overflow\n", fd);
debug(33, 1) ("\tclient %s\n", inet_ntoa(http->conn->peer.sin_addr));
@@ -2595,7 +2595,7 @@
static int pffd = -1;
#endif
#if LINUX_NETFILTER
- size_t sock_sz = sizeof(conn->me);
+ socklen_t sock_sz = sizeof(conn->me);
#endif
/* pre-set these values to make aborting simpler */
@@ -3130,7 +3130,7 @@
/*
* cache the Content-length value in request_t.
*/
- request->content_length = httpHeaderGetInt(&request->header,
+ request->content_length = httpHeaderGetSize(&request->header,
HDR_CONTENT_LENGTH);
request->flags.internal = http->flags.internal;
safe_free(prefix);
@@ -3642,7 +3642,7 @@
StoreEntry *entry = http->entry;
MemObject *mem;
http_reply *reply;
- int sendlen;
+ squid_off_t sendlen;
if (entry == NULL)
return 0;
/*
@@ -3705,7 +3705,7 @@
static int
clientGotNotEnough(clientHttpRequest * http)
{
- int cl = httpReplyBodySize(http->request->method, http->entry->mem_obj->reply);
+ squid_off_t cl = httpReplyBodySize(http->request->method, http->entry->mem_obj->reply);
int hs = http->entry->mem_obj->reply->hdr_sz;
assert(cl >= 0);
if (http->out.offset < cl + hs)
Index: squid/src/defines.h
diff -u squid/src/defines.h:1.15.6.3 squid/src/defines.h:1.15.6.3.26.4
--- squid/src/defines.h:1.15.6.3 Thu Aug 8 13:18:40 2002
+++ squid/src/defines.h Thu Mar 24 19:34:31 2005
@@ -200,7 +200,8 @@
#define SwapMetaType(x) (char)x[0]
#define SwapMetaSize(x) &x[sizeof(char)]
#define SwapMetaData(x) &x[STORE_META_TLD_START]
-#define STORE_HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(size_t))
+#define STORE_HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(squid_file_sz))
+#define STORE_HDR_METASIZE_OLD (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(size_t))
#define STORE_ENTRY_WITH_MEMOBJ 1
#define STORE_ENTRY_WITHOUT_MEMOBJ 0
Index: squid/src/disk.c
diff -u squid/src/disk.c:1.8.14.2 squid/src/disk.c:1.8.14.2.8.5
--- squid/src/disk.c:1.8.14.2 Sat Nov 6 19:14:27 2004
+++ squid/src/disk.c Fri Mar 25 17:52:23 2005
@@ -239,7 +239,7 @@
q->buf_offset += len;
if (q->buf_offset > q->len)
debug(50, 1) ("diskHandleWriteComplete: q->buf_offset > q->len (%p,%d, %d, %d FD %d)\n",
- q, (int) q->buf_offset, q->len, len, fd);
+ q, (int) q->buf_offset, (int) q->len, len, fd);
assert(q->buf_offset <= q->len);
if (q->buf_offset == q->len) {
/* complete write */
@@ -293,7 +293,7 @@
file_write(int fd,
off_t file_offset,
void *ptr_to_buf,
- int len,
+ size_t len,
DWCB * handle,
void *handle_data,
FREE * free_func)
@@ -331,9 +331,9 @@
* in a snap
*/
void
-file_write_mbuf(int fd, off_t off, MemBuf mb, DWCB * handler, void *handler_data)
+file_write_mbuf(int fd, off_t file_offset, MemBuf mb, DWCB * handler, void *handler_data)
{
- file_write(fd, off, mb.buf, mb.size, handler, handler_data, memBufFreeFunc(&mb));
+ file_write(fd, file_offset, mb.buf, mb.size, handler, handler_data, memBufFreeFunc(&mb));
}
/* Read from FD */
@@ -352,12 +352,12 @@
memFree(ctrl_dat, MEM_DREAD_CTRL);
return;
}
- if (F->disk.offset != ctrl_dat->offset) {
+ if (F->disk.offset != ctrl_dat->file_offset) {
debug(6, 3) ("diskHandleRead: FD %d seeking to offset %d\n",
- fd, (int) ctrl_dat->offset);
- lseek(fd, ctrl_dat->offset, SEEK_SET); /* XXX ignore return? */
+ fd, (int) ctrl_dat->file_offset);
+ lseek(fd, ctrl_dat->file_offset, SEEK_SET); /* XXX ignore return? */
statCounter.syscalls.disk.seeks++;
- F->disk.offset = ctrl_dat->offset;
+ F->disk.offset = ctrl_dat->file_offset;
}
errno = 0;
len = FD_READ_METHOD(fd, ctrl_dat->buf, ctrl_dat->req_len);
@@ -388,13 +388,13 @@
* It must have at least req_len space in there.
* call handler when a reading is complete. */
void
-file_read(int fd, char *buf, int req_len, off_t offset, DRCB * handler, void *client_data)
+file_read(int fd, char *buf, size_t req_len, off_t file_offset, DRCB * handler, void *client_data)
{
dread_ctrl *ctrl_dat;
assert(fd >= 0);
ctrl_dat = memAllocate(MEM_DREAD_CTRL);
ctrl_dat->fd = fd;
- ctrl_dat->offset = offset;
+ ctrl_dat->file_offset = file_offset;
ctrl_dat->req_len = req_len;
ctrl_dat->buf = buf;
ctrl_dat->end_of_file = 0;
Index: squid/src/enums.h
diff -u squid/src/enums.h:1.29.2.14 squid/src/enums.h:1.29.2.14.6.2
--- squid/src/enums.h:1.29.2.14 Mon Jan 17 19:16:49 2005
+++ squid/src/enums.h Thu Mar 24 19:34:31 2005
@@ -274,7 +274,8 @@
ftPCc,
ftPContRange,
ftPRange,
- ftDate_1123_or_ETag
+ ftDate_1123_or_ETag,
+ ftSize
} field_type;
/* possible owners of http header */
@@ -623,6 +624,7 @@
STORE_META_HITMETERING, /* reserved for hit metering */
STORE_META_VALID,
STORE_META_VARY_HEADERS, /* Stores Vary request headers */
+ STORE_META_STD_LFS, /* standard metadata in lfs format */
STORE_META_END
};
@@ -638,6 +640,7 @@
SWAP_LOG_NOP,
SWAP_LOG_ADD,
SWAP_LOG_DEL,
+ SWAP_LOG_VERSION,
SWAP_LOG_MAX
} swap_log_op;
Index: squid/src/forward.c
diff -u squid/src/forward.c:1.13.6.13 squid/src/forward.c:1.13.6.13.2.1
--- squid/src/forward.c:1.13.6.13 Tue Feb 22 19:15:08 2005
+++ squid/src/forward.c Fri Mar 11 12:12:11 2005
@@ -693,9 +693,18 @@
#endif
if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT))
return rc;
- if (mem->inmem_hi - storeLowestMemReaderOffset(e) < READ_AHEAD_GAP)
- return rc;
- return 1;
+ if (EBIT_TEST(e->flags, RELEASE_REQUEST)) {
+ /* Just a small safety cap to defer storing more data into the object
+ * if there already is way too much. This handles the case when there
+ * is disk clients pending on a too large object being fetched and a
+ * few other corner cases.
+ */
+ if (mem->inmem_hi - mem->inmem_lo > SM_PAGE_SIZE + Config.Store.maxInMemObjSize + READ_AHEAD_GAP)
+ return 1;
+ }
+ if (mem->inmem_hi - storeLowestMemReaderOffset(e) > READ_AHEAD_GAP)
+ return 1;
+ return rc;
}
void
Index: squid/src/ftp.c
diff -u squid/src/ftp.c:1.18.6.20 squid/src/ftp.c:1.18.6.18.2.6
--- squid/src/ftp.c:1.18.6.20 Mon Mar 7 19:15:09 2005
+++ squid/src/ftp.c Fri Mar 25 17:52:23 2005
@@ -93,14 +93,14 @@
int login_att;
ftp_state_t state;
time_t mdtm;
- int size;
+ squid_off_t size;
wordlist *pathcomps;
char *filepath;
- int restart_offset;
- int restarted_offset;
+ squid_off_t restart_offset;
+ squid_off_t restarted_offset;
int rest_att;
char *proxy_host;
- size_t list_width;
+ int list_width;
wordlist *cwd_message;
char *old_request;
char *old_reply;
@@ -110,7 +110,7 @@
int fd;
char *buf;
size_t size;
- off_t offset;
+ size_t offset;
FREE *freefunc;
wordlist *message;
char *last_command;
@@ -121,7 +121,7 @@
int fd;
char *buf;
size_t size;
- off_t offset;
+ size_t offset;
FREE *freefunc;
char *host;
u_short port;
@@ -132,7 +132,7 @@
typedef struct {
char type;
- int size;
+ squid_off_t size;
char *date;
char *name;
char *showname;
@@ -537,7 +537,7 @@
month, day, year);
if ((t = strstr(buf, tbuf))) {
p->type = *tokens[0];
- p->size = atoi(size);
+ p->size = strto_off_t(size, NULL, 10);
p->date = xstrdup(tbuf);
if (flags.skip_whitespace) {
t += strlen(tbuf);
@@ -566,7 +566,7 @@
p->type = 'd';
} else {
p->type = '-';
- p->size = atoi(tokens[2]);
+ p->size = strto_off_t(tokens[2], NULL, 10);
}
snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]);
p->date = xstrdup(tbuf);
@@ -604,10 +604,10 @@
p->name = xstrndup(ct + 1, l + 1);
break;
case 's':
- p->size = atoi(ct + 1);
+ p->size = strto_off_t(ct + 1, NULL, 10);
break;
case 'm':
- t = (time_t) strtol(ct + 1, &tmp, 0);
+ t = (time_t) strto_off_t(ct + 1, &tmp, 0);
if (tmp != ct + l)
break; /* not a valid integer */
p->date = xstrdup(ctime(&t));
@@ -728,8 +728,6 @@
ftpListPartsFree(&parts);
return html;
}
- parts->size += 1023;
- parts->size >>= 10;
parts->showname = xstrdup(parts->name);
if (!Config.Ftp.list_wrap) {
if (strlen(parts->showname) > width - 1) {
@@ -771,10 +769,26 @@
break;
case '-':
default:
- snprintf(icon, 2048, "![\"%-6s\"](\"%s\")
",
- mimeGetIconURL(parts->name),
- "[FILE]");
- snprintf(size, 2048, " %6dk", parts->size);
+ {
+ squid_off_t sz = parts->size;
+ char units = ' ';
+ snprintf(icon, 2048, "",
+ mimeGetIconURL(parts->name),
+ "[FILE]");
+ if (sz > 10 * 1024) {
+ sz = (sz + 1023) >> 10;
+ units = 'K';
+ }
+ if (sz > 10 * 1024) {
+ sz = (sz + 1023) >> 10;
+ units = 'M';
+ }
+ if (sz > 10 * 1024) {
+ sz = (sz + 1023) >> 10;
+ units = 'G';
+ }
+ snprintf(size, 2048, " %6" PRINTF_OFF_T "%c", sz, units);
+ }
break;
}
if (parts->type != 'd') {
@@ -818,7 +832,7 @@
size_t linelen;
size_t usable;
StoreEntry *e = ftpState->entry;
- int len = ftpState->data.offset;
+ size_t len = ftpState->data.offset;
/*
* We need a NULL-terminated buffer for scanning, ick
*/
@@ -834,7 +848,7 @@
xfree(sbuf);
return;
}
- debug(9, 3) ("ftpParseListing: %d bytes to play with\n", len);
+ debug(9, 3) ("ftpParseListing: %d bytes to play with\n", (int) len);
line = memAllocate(MEM_4K_BUF);
end++;
storeBuffer(e);
@@ -1224,8 +1238,8 @@
wordlist *head = NULL;
wordlist *list;
wordlist **tail = &head;
- off_t offset;
- size_t linelen;
+ int offset;
+ int linelen;
int code = -1;
debug(9, 5) ("ftpParseControlReply\n");
/*
@@ -1699,7 +1713,7 @@
debug(9, 3) ("This is ftpReadSize\n");
if (code == 213) {
ftpUnhack(ftpState);
- ftpState->size = atoi(ftpState->ctrl.last_reply);
+ ftpState->size = strto_off_t(ftpState->ctrl.last_reply, NULL, 10);
if (ftpState->size == 0) {
debug(9, 2) ("ftpReadSize: SIZE reported %s on %s\n",
ftpState->ctrl.last_reply,
@@ -2029,7 +2043,7 @@
snprintf(cbuf, 1024, "STOR %s\r\n", ftpState->filepath);
ftpWriteCommand(cbuf, ftpState);
ftpState->state = SENT_STOR;
- } else if (httpHeaderGetInt(&ftpState->request->header, HDR_CONTENT_LENGTH) > 0) {
+ } else if (httpHeaderGetSize(&ftpState->request->header, HDR_CONTENT_LENGTH) > 0) {
/* File upload without a filename. use STOU to generate one */
snprintf(cbuf, 1024, "STOU\r\n");
ftpWriteCommand(cbuf, ftpState);
@@ -2079,7 +2093,7 @@
static void
ftpSendRest(FtpStateData * ftpState)
{
- snprintf(cbuf, 1024, "REST %d\r\n", ftpState->restart_offset);
+ snprintf(cbuf, 1024, "REST %" PRINTF_OFF_T "\r\n", ftpState->restart_offset);
ftpWriteCommand(cbuf, ftpState);
ftpState->state = SENT_REST;
}
Index: squid/src/globals.h
diff -u squid/src/globals.h:1.14.6.4 squid/src/globals.h:1.14.6.4.6.2
--- squid/src/globals.h:1.14.6.4 Fri Jan 21 19:15:06 2005
+++ squid/src/globals.h Wed Mar 2 19:44:04 2005
@@ -154,7 +154,7 @@
extern int store_swap_low; /* 0 */
extern int store_swap_high; /* 0 */
extern int store_pages_max; /* 0 */
-extern ssize_t store_maxobjsize; /* -1 */
+extern squid_off_t store_maxobjsize; /* -1 */
extern RemovalPolicy *mem_policy;
extern hash_table *proxy_auth_username_cache; /* NULL */
extern int incoming_sockets_accepted;
Index: squid/src/helper.c
diff -u squid/src/helper.c:1.16.2.14 squid/src/helper.c:1.16.2.14.8.1
--- squid/src/helper.c:1.16.2.14 Sat Nov 6 19:14:27 2004
+++ squid/src/helper.c Wed Mar 2 20:43:43 2005
@@ -374,7 +374,7 @@
srv->flags.closing ? 'C' : ' ',
srv->flags.shutdown ? 'S' : ' ',
tt < 0.0 ? 0.0 : tt,
- (int) srv->offset,
+ srv->offset,
srv->request ? log_quote(srv->request->buf) : "(none)");
}
storeAppendPrintf(sentry, "\nFlags key:\n\n");
@@ -423,7 +423,7 @@
srv->flags.reserved ? 'R' : ' ',
srv->flags.shutdown ? 'S' : ' ',
tt < 0.0 ? 0.0 : tt,
- (int) srv->offset,
+ srv->offset,
srv->request ? log_quote(srv->request->buf) : "(none)");
}
storeAppendPrintf(sentry, "\nFlags key:\n\n");
Index: squid/src/htcp.c
diff -u squid/src/htcp.c:1.9.2.5 squid/src/htcp.c:1.9.2.5.2.6
--- squid/src/htcp.c:1.9.2.5 Sun Feb 20 11:42:04 2005
+++ squid/src/htcp.c Fri Mar 25 17:52:23 2005
@@ -227,14 +227,14 @@
{
u_short length;
size_t len;
- off_t off = 0;
+ int off = 0;
if (buflen - off < 2)
return -1;
if (s)
len = strlen(s);
else
len = 0;
- debug(31, 3) ("htcpBuildCountstr: LENGTH = %d\n", len);
+ debug(31, 3) ("htcpBuildCountstr: LENGTH = %d\n", (int) len);
debug(31, 3) ("htcpBuildCountstr: TEXT = {%s}\n", s ? s : "");
length = htons((u_short) len);
xmemcpy(buf + off, &length, 2);
Index: squid/src/http.c
diff -u squid/src/http.c:1.17.6.27 squid/src/http.c:1.17.6.26.2.5
--- squid/src/http.c:1.17.6.27 Thu Mar 10 06:12:05 2005
+++ squid/src/http.c Fri Mar 25 17:52:24 2005
@@ -528,9 +528,9 @@
/* return 1 if we got the last of the data on a persistent connection */
MemObject *mem = httpState->entry->mem_obj;
HttpReply *reply = mem->reply;
- int clen;
+ squid_off_t clen;
debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd);
- debug(11, 5) ("httpPconnTransferDone: content_length=%d\n",
+ debug(11, 5) ("httpPconnTransferDone: content_length=%" PRINTF_OFF_T "\n",
reply->content_length);
/* If we haven't seen the end of reply headers, we are not done */
if (httpState->reply_hdr_state < 2)
@@ -1061,7 +1061,7 @@
/* build request prefix and append it to a given MemBuf;
* return the length of the prefix */
-mb_size_t
+int
httpBuildRequestPrefix(request_t * request,
request_t * orig_request,
StoreEntry * entry,
Index: squid/src/mime.c
diff -u squid/src/mime.c:1.11.6.4 squid/src/mime.c:1.11.6.4.6.1
--- squid/src/mime.c:1.11.6.4 Fri Jan 21 19:15:06 2005
+++ squid/src/mime.c Fri Feb 25 02:48:50 2005
@@ -442,7 +442,7 @@
httpReplyReset(reply = e->mem_obj->reply);
httpBuildVersion(&version, 1, 0);
httpReplySetHeaders(reply, version, HTTP_OK, NULL,
- type, (int) sb.st_size, sb.st_mtime, -1);
+ type, sb.st_size, sb.st_mtime, -1);
reply->cache_control = httpHdrCcCreate();
httpHdrCcSetMaxAge(reply->cache_control, 86400);
httpHeaderPutCc(&reply->header, reply->cache_control);
Index: squid/src/net_db.c
diff -u squid/src/net_db.c:1.11.22.5 squid/src/net_db.c:1.11.22.3.2.5
--- squid/src/net_db.c:1.11.22.5 Wed Mar 16 19:13:38 2005
+++ squid/src/net_db.c Fri Mar 18 17:47:36 2005
@@ -42,8 +42,8 @@
StoreEntry *e;
store_client *sc;
request_t *r;
- off_t seen;
- off_t used;
+ squid_off_t seen;
+ squid_off_t used;
size_t buf_sz;
char *buf;
} netdbExchangeState;
@@ -531,7 +531,7 @@
{
netdbExchangeState *ex = data;
int rec_sz = 0;
- off_t o;
+ ssize_t o;
struct in_addr addr;
double rtt;
double hops;
Index: squid/src/peer_digest.c
diff -u squid/src/peer_digest.c:1.10.6.1 squid/src/peer_digest.c:1.10.6.1.16.4
--- squid/src/peer_digest.c:1.10.6.1 Wed Jul 16 19:13:28 2003
+++ squid/src/peer_digest.c Fri Mar 25 17:52:24 2005
@@ -453,7 +453,7 @@
if (size >= StoreDigestCBlockSize) {
PeerDigest *pd = fetch->pd;
HttpReply *rep = fetch->entry->mem_obj->reply;
- const int seen = fetch->offset + size;
+ const squid_off_t seen = fetch->offset + size;
assert(pd && rep);
if (peerDigestSetCBlock(pd, buf)) {
@@ -498,7 +498,7 @@
fetch->offset += size;
fetch->mask_offset += size;
if (fetch->mask_offset >= pd->cd->mask_size) {
- debug(72, 2) ("peerDigestSwapInMask: Done! Got %d, expected %d\n",
+ debug(72, 2) ("peerDigestSwapInMask: Done! Got %" PRINTF_OFF_T ", expected %d\n",
fetch->mask_offset, pd->cd->mask_size);
assert(fetch->mask_offset == pd->cd->mask_size);
assert(peerDigestFetchedEnough(fetch, NULL, 0, "peerDigestSwapInMask"));
@@ -540,8 +540,8 @@
else
host = strBuf(pd->host);
}
- debug(72, 6) ("%s: peer %s, offset: %d size: %d.\n",
- step_name, host, fcb_valid ? fetch->offset : -1, size);
+ debug(72, 6) ("%s: peer %s, offset: %" PRINTF_OFF_T " size: %d.\n",
+ step_name, host, fcb_valid ? fetch->offset : (squid_off_t) - 1, (int) size);
/* continue checking (with pd and host known and valid) */
if (!reason) {
@@ -778,7 +778,7 @@
/* check consistency further */
if (cblock.mask_size != cacheDigestCalcMaskSize(cblock.capacity, cblock.bits_per_entry)) {
debug(72, 0) ("%s digest cblock is corrupted (mask size mismatch: %d ? %d).\n",
- host, cblock.mask_size, cacheDigestCalcMaskSize(cblock.capacity, cblock.bits_per_entry));
+ host, cblock.mask_size, (int) cacheDigestCalcMaskSize(cblock.capacity, cblock.bits_per_entry));
return 0;
}
/* there are some things we cannot do yet */
Index: squid/src/protos.h
diff -u squid/src/protos.h:1.41.6.23 squid/src/protos.h:1.41.6.22.2.9
--- squid/src/protos.h:1.41.6.23 Fri Mar 18 17:42:55 2005
+++ squid/src/protos.h Fri Mar 18 17:47:36 2005
@@ -125,7 +125,7 @@
extern int cbdataLocked(const void *p);
extern void clientdbInit(void);
-extern void clientdbUpdate(struct in_addr, log_type, protocol_t, size_t);
+extern void clientdbUpdate(struct in_addr, log_type, protocol_t, squid_off_t);
extern int clientdbCutoffDenied(struct in_addr);
extern void clientdbDump(StoreEntry *);
extern void clientdbFreeMemory(void);
@@ -223,24 +223,12 @@
/* disk.c */
extern int file_open(const char *path, int mode);
extern void file_close(int fd);
-extern void file_write(int, off_t, void *, int len, DWCB *, void *, FREE *);
-extern void file_write_mbuf(int fd, off_t, MemBuf mb, DWCB * handler, void *handler_data);
-extern void file_read(int, char *, int, off_t, DRCB *, void *);
+extern void file_write(int, squid_off_t, void *, size_t len, DWCB *, void *, FREE *);
+extern void file_write_mbuf(int fd, squid_off_t, MemBuf mb, DWCB * handler, void *handler_data);
+extern void file_read(int, char *, size_t, squid_off_t, DRCB *, void *);
extern void disk_init(void);
-/* diskd.c */
-extern diskd_queue *afile_create_queue(void);
-extern void afile_destroy_queue(diskd_queue *);
-extern void afile_sync_queue(diskd_queue *);
-extern void afile_sync(void);
-extern void afile_open(const char *path, int mode, DOCB *, void *);
-extern void afile_close(int fd, DCCB * callback, void *data);
-extern void afile_write(int, off_t, void *, int len, DWCB *, void *, FREE *);
-extern void afile_write_mbuf(int fd, off_t, MemBuf, DWCB *, void *);
-extern void afile_read(int, char *, int, off_t, DRCB *, void *);
-extern void afile_unlink(const char *path, DUCB *, void *);
-extern void afile_truncate(const char *path, DTCB *, void *);
-
+/* dns.s */
extern void dnsShutdown(void);
extern void dnsInit(void);
extern void dnsSubmit(const char *lookup, HLPCB * callback, void *data);
@@ -303,7 +291,7 @@
/* http.c */
extern int httpCachable(method_t);
extern void httpStart(FwdState *);
-extern mb_size_t httpBuildRequestPrefix(request_t * request,
+extern int httpBuildRequestPrefix(request_t * request,
request_t * orig_request,
StoreEntry * entry,
MemBuf * mb,
@@ -371,13 +359,13 @@
/* iterate through specs */
extern HttpHdrRangeSpec *httpHdrRangeGetSpec(const HttpHdrRange * range, HttpHdrRangePos * pos);
/* adjust specs after the length is known */
-extern int httpHdrRangeCanonize(HttpHdrRange *, ssize_t);
+extern int httpHdrRangeCanonize(HttpHdrRange *, squid_off_t);
/* other */
extern String httpHdrRangeBoundaryStr(clientHttpRequest * http);
extern int httpHdrRangeIsComplex(const HttpHdrRange * range);
extern int httpHdrRangeWillBeComplex(const HttpHdrRange * range);
-extern ssize_t httpHdrRangeFirstOffset(const HttpHdrRange * range);
-extern ssize_t httpHdrRangeLowestOffset(const HttpHdrRange * range, ssize_t);
+extern squid_off_t httpHdrRangeFirstOffset(const HttpHdrRange * range);
+extern squid_off_t httpHdrRangeLowestOffset(const HttpHdrRange * range, squid_off_t);
extern int httpHdrRangeOffsetLimit(HttpHdrRange *);
@@ -390,7 +378,7 @@
extern HttpHdrContRange *httpHdrContRangeDup(const HttpHdrContRange * crange);
extern void httpHdrContRangePackInto(const HttpHdrContRange * crange, Packer * p);
/* inits with given spec */
-extern void httpHdrContRangeSet(HttpHdrContRange *, HttpHdrRangeSpec, ssize_t);
+extern void httpHdrContRangeSet(HttpHdrContRange *, HttpHdrRangeSpec, squid_off_t);
/* Http Header Tools */
extern HttpHeaderFieldInfo *httpHeaderBuildFieldsInfo(const HttpHeaderFieldAttrs * attrs, int count);
@@ -401,14 +389,14 @@
extern void httpHeaderMaskInit(HttpHeaderMask * mask, int value);
extern void httpHeaderCalcMask(HttpHeaderMask * mask, const int *enums, int count);
extern int httpHeaderHasConnDir(const HttpHeader * hdr, const char *directive);
-extern void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, ssize_t);
+extern void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, squid_off_t);
extern void strListAdd(String * str, const char *item, char del);
extern int strListIsMember(const String * str, const char *item, char del);
extern int strListIsSubstr(const String * list, const char *s, char del);
extern int strListGetItem(const String * str, char del, const char **item, int *ilen, const char **pos);
extern const char *getStringPrefix(const char *str, const char *end);
extern int httpHeaderParseInt(const char *start, int *val);
-extern int httpHeaderParseSize(const char *start, ssize_t * sz);
+extern squid_off_t httpHeaderParseSize(const char *start, squid_off_t * sz);
extern int httpHeaderReset(HttpHeader * hdr);
#if STDC_HEADERS
extern void
@@ -433,6 +421,7 @@
/* field manipulation */
extern int httpHeaderHas(const HttpHeader * hdr, http_hdr_type type);
extern void httpHeaderPutInt(HttpHeader * hdr, http_hdr_type type, int number);
+extern void httpHeaderPutSize(HttpHeader * hdr, http_hdr_type type, squid_off_t number);
extern void httpHeaderPutTime(HttpHeader * hdr, http_hdr_type type, time_t htime);
extern void httpHeaderPutStr(HttpHeader * hdr, http_hdr_type type, const char *str);
extern void httpHeaderPutAuth(HttpHeader * hdr, const char *auth_scheme, const char *realm);
@@ -441,6 +430,7 @@
extern void httpHeaderPutRange(HttpHeader * hdr, const HttpHdrRange * range);
extern void httpHeaderPutExt(HttpHeader * hdr, const char *name, const char *value);
extern int httpHeaderGetInt(const HttpHeader * hdr, http_hdr_type id);
+extern squid_off_t httpHeaderGetSize(const HttpHeader * hdr, http_hdr_type id);
extern time_t httpHeaderGetTime(const HttpHeader * hdr, http_hdr_type id);
extern TimeOrTag httpHeaderGetTimeOrTag(const HttpHeader * hdr, http_hdr_type id);
extern HttpHdrCc *httpHeaderGetCc(const HttpHeader * hdr);
@@ -482,7 +472,7 @@
/* absorb: copy the contents of a new reply to the old one, destroy new one */
extern void httpReplyAbsorb(HttpReply * rep, HttpReply * new_rep);
/* parse returns -1,0,+1 on error,need-more-data,success */
-extern int httpReplyParse(HttpReply * rep, const char *buf, ssize_t);
+extern int httpReplyParse(HttpReply * rep, const char *buf, size_t);
extern void httpReplyPackInto(const HttpReply * rep, Packer * p);
/* ez-routines */
/* mem-pack: returns a ready to use mem buffer with a packed reply */
@@ -491,10 +481,10 @@
extern void httpReplySwapOut(const HttpReply * rep, StoreEntry * e);
/* set commonly used info with one call */
extern void httpReplySetHeaders(HttpReply * rep, http_version_t ver, http_status status,
- const char *reason, const char *ctype, int clen, time_t lmt, time_t expires);
+ const char *reason, const char *ctype, squid_off_t clen, time_t lmt, time_t expires);
/* do everything in one call: init, set, pack, clean, return MemBuf */
extern MemBuf httpPackedReply(http_version_t ver, http_status status, const char *ctype,
- int clen, time_t lmt, time_t expires);
+ squid_off_t clen, time_t lmt, time_t expires);
/* construct 304 reply and pack it into MemBuf, return MemBuf */
extern MemBuf httpPacked304Reply(const HttpReply * rep);
/* update when 304 reply is received for a cached object */
@@ -505,7 +495,7 @@
extern time_t httpReplyExpires(const HttpReply * rep);
extern int httpReplyHasCc(const HttpReply * rep, http_hdr_cc_type type);
extern void httpRedirectReply(HttpReply *, http_status, const char *);
-extern int httpReplyBodySize(method_t, const HttpReply *);
+extern squid_off_t httpReplyBodySize(method_t, const HttpReply *);
/* Http Request */
extern request_t *requestCreate(method_t, protocol_t, const char *urlpath);
@@ -607,7 +597,7 @@
/* unfirtunate hack to test if the buffer has been Init()ialized */
extern int memBufIsNull(MemBuf * mb);
/* calls memcpy, appends exactly size bytes, extends buffer if needed */
-extern void memBufAppend(MemBuf * mb, const char *buf, mb_size_t size);
+extern void memBufAppend(MemBuf * mb, const char *buf, int size);
/* calls snprintf, extends buffer if needed */
#if STDC_HEADERS
extern void
@@ -756,7 +746,7 @@
extern int authenticateAuthUserInuse(auth_user_t * auth_user);
extern void authenticateAuthUserRequestRemoveIp(auth_user_request_t *, struct in_addr);
extern void authenticateAuthUserRequestClearIp(auth_user_request_t *);
-extern size_t authenticateAuthUserRequestIPCount(auth_user_request_t *);
+extern int authenticateAuthUserRequestIPCount(auth_user_request_t *);
extern int authenticateDirection(auth_user_request_t *);
extern FREE authenticateFreeProxyAuthUser;
extern void authenticateFreeProxyAuthUserACLResults(void *data);
@@ -788,7 +778,7 @@
extern void start_announce(void *unused);
-extern void sslStart(clientHttpRequest *, size_t *, int *);
+extern void sslStart(clientHttpRequest *, squid_off_t *, int *);
extern void waisStart(FwdState *);
/* ident.c */
@@ -851,7 +841,7 @@
extern void memFree32K(void *);
extern void memFree64K(void *);
extern int memInUse(mem_type);
-extern size_t memTotalAllocated(void);
+extern int memTotalAllocated(void);
extern void memDataInit(mem_type, const char *, size_t, int);
extern void memCheckInit(void);
@@ -869,9 +859,9 @@
/* Mem */
extern void memReport(StoreEntry * e);
-extern int stmemFreeDataUpto(mem_hdr *, int);
+extern squid_off_t stmemFreeDataUpto(mem_hdr *, squid_off_t);
extern void stmemAppend(mem_hdr *, const char *, int);
-extern ssize_t stmemCopy(const mem_hdr *, off_t, char *, size_t);
+extern ssize_t stmemCopy(const mem_hdr *, squid_off_t, char *, size_t);
extern void stmemFree(mem_hdr *);
extern void stmemFreeData(mem_hdr *);
extern void stmemNodeFree(void *);
@@ -926,8 +916,8 @@
extern void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
extern int storeCheckCachable(StoreEntry * e);
extern void storeSetPrivateKey(StoreEntry *);
-extern int objectLen(const StoreEntry * e);
-extern int contentLen(const StoreEntry * e);
+extern squid_off_t objectLen(const StoreEntry * e);
+extern squid_off_t contentLen(const StoreEntry * e);
extern HttpReply *storeEntryReply(StoreEntry *);
extern int storeTooManyDiskFilesOpen(void);
extern void storeEntryReset(StoreEntry *);
@@ -948,10 +938,10 @@
extern storeIOState *storeCreate(StoreEntry *, STFNCB *, STIOCB *, void *);
extern storeIOState *storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *);
extern void storeClose(storeIOState *);
-extern void storeRead(storeIOState *, char *, size_t, off_t, STRCB *, void *);
-extern void storeWrite(storeIOState *, char *, size_t, off_t, FREE *);
+extern void storeRead(storeIOState *, char *, size_t, squid_off_t, STRCB *, void *);
+extern void storeWrite(storeIOState *, char *, size_t, squid_off_t, FREE *);
extern void storeUnlink(StoreEntry *);
-extern off_t storeOffset(storeIOState *);
+extern squid_off_t storeOffset(storeIOState *);
/*
* store_log.c
@@ -1009,7 +999,7 @@
extern void storeDirInit(void);
extern void storeDirOpenSwapLogs(void);
extern void storeDirSwapLog(const StoreEntry *, int op);
-extern void storeDirUpdateSwapSize(SwapDir *, size_t size, int sign);
+extern void storeDirUpdateSwapSize(SwapDir *, squid_off_t size, int sign);
extern void storeDirSync(void);
extern void storeDirCallback(void);
extern void storeDirLRUDelete(StoreEntry *);
@@ -1042,7 +1032,7 @@
*/
extern void storeSwapOut(StoreEntry * e);
extern void storeSwapOutFileClose(StoreEntry * e);
-extern int storeSwapOutAble(const StoreEntry * e);
+extern squid_off_t storeSwapOutMaintainMemObject(StoreEntry * e);
/*
* store_client.c
@@ -1051,10 +1041,10 @@
extern store_client *storeClientListSearch(const MemObject * mem, void *data);
#endif
extern store_client *storeClientListAdd(StoreEntry * e, void *data);
-extern void storeClientCopy(store_client *, StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *);
+extern void storeClientCopy(store_client *, StoreEntry *, squid_off_t, squid_off_t, size_t, char *, STCB *, void *);
extern int storeClientCopyPending(store_client *, StoreEntry * e, void *data);
extern int storeUnregister(store_client * sc, StoreEntry * e, void *data);
-extern off_t storeLowestMemReaderOffset(const StoreEntry * entry);
+extern squid_off_t storeLowestMemReaderOffset(const StoreEntry * entry);
extern void InvokeHandlers(StoreEntry * e);
extern int storePendingNClients(const StoreEntry * e);
@@ -1154,7 +1144,7 @@
extern void dlinkNodeDelete(dlink_node * m);
extern dlink_node *dlinkNodeNew(void);
-extern void kb_incr(kb_t *, size_t);
+extern void kb_incr(kb_t *, squid_off_t);
extern double gb_to_double(const gb_t *);
extern const char *gb_to_str(const gb_t *);
extern void gb_flush(gb_t *); /* internal, do not use this */
Index: squid/src/squid.h
diff -u squid/src/squid.h:1.13.6.7 squid/src/squid.h:1.13.6.7.8.6
--- squid/src/squid.h:1.13.6.7 Mon Aug 9 19:13:32 2004
+++ squid/src/squid.h Fri Mar 25 17:52:25 2005
@@ -481,4 +481,12 @@
#define FD_READ_METHOD(fd, buf, len) (*fd_table[fd].read_method)(fd, buf, len)
#define FD_WRITE_METHOD(fd, buf, len) (*fd_table[fd].write_method)(fd, buf, len)
+/*
+ * Trap attempts to build large file cache support without support for
+ * large objects
+ */
+#if LARGE_CACHE_FILES && SIZEOF_SQUID_OFF_T <= 4
+#error Your platform does not support large integers. Can not build with --enable-large-cache-files
+#endif
+
#endif /* SQUID_H */
Index: squid/src/ssl.c
diff -u squid/src/ssl.c:1.13.6.9 squid/src/ssl.c:1.13.6.7.8.6
--- squid/src/ssl.c:1.13.6.9 Mon Mar 21 19:18:35 2005
+++ squid/src/ssl.c Fri Mar 25 17:52:25 2005
@@ -46,7 +46,7 @@
int len;
char *buf;
} client, server;
- size_t *size_ptr; /* pointer to size in an ConnStateData for logging */
+ squid_off_t *size_ptr; /* pointer to size in an ConnStateData for logging */
int *status_ptr; /* pointer to status for logging */
#if DELAY_POOLS
delay_id delay_id;
@@ -297,6 +297,12 @@
fd_bytes(fd, len, FD_WRITE);
kb_incr(&statCounter.server.all.kbytes_out, len);
kb_incr(&statCounter.server.other.kbytes_out, len);
+ /* increment total object size */
+ if (sslState->size_ptr)
+#if SIZEOF_SQUID_OFF_T <= 4
+ if (*sslState->size_ptr < 0x7FFF0000)
+#endif
+ *sslState->size_ptr += len;
assert(len <= sslState->client.len);
sslState->client.len -= len;
if (sslState->client.len > 0) {
@@ -339,7 +345,7 @@
sslState->server.len -= len;
/* increment total object size */
if (sslState->size_ptr)
-#if SIZEOF_SIZE_T == 4
+#if SIZEOF_SQUID_OFF_T <= 4
if (*sslState->size_ptr < 0x7FFF0000)
#endif
*sslState->size_ptr += len;
@@ -471,7 +477,7 @@
CBDATA_TYPE(SslStateData);
void
-sslStart(clientHttpRequest * http, size_t * size_ptr, int *status_ptr)
+sslStart(clientHttpRequest * http, squid_off_t * size_ptr, int *status_ptr)
{
/* Create state structure. */
SslStateData *sslState = NULL;
Index: squid/src/stat.c
diff -u squid/src/stat.c:1.13.6.11 squid/src/stat.c:1.13.6.11.2.4
--- squid/src/stat.c:1.13.6.11 Sun Feb 13 19:15:35 2005
+++ squid/src/stat.c Fri Mar 25 17:52:25 2005
@@ -432,7 +432,7 @@
f = &fd_table[i];
if (!f->flags.open)
continue;
- storeAppendPrintf(sentry, "%4d %-6.6s %4d %7d%c %7d%c %-21s %s\n",
+ storeAppendPrintf(sentry, "%4d %-6.6s %4d %7" PRINTF_OFF_T "%c %7" PRINTF_OFF_T "%c %-21s %s\n",
i,
fdTypeStr[f->type],
f->timeout_handler ? (int) (f->timeout - squid_curtime) / 60 : 0,
@@ -1397,7 +1397,7 @@
*/
cd = CountHist[0].cd.kbytes_recv.kb - CountHist[minutes].cd.kbytes_recv.kb;
if (s < cd)
- debug(18, 1) ("STRANGE: srv_kbytes=%d, cd_kbytes=%d\n", s, cd);
+ debug(18, 1) ("STRANGE: srv_kbytes=%d, cd_kbytes=%d\n", (int) s, (int) cd);
s -= cd;
#endif
if (c > s)
@@ -1421,7 +1421,7 @@
storeAppendPrintf(s, "Connection: %p\n", conn);
if (conn) {
fd = conn->fd;
- storeAppendPrintf(s, "\tFD %d, read %d, wrote %d\n", fd,
+ storeAppendPrintf(s, "\tFD %d, read %" PRINTF_OFF_T ", wrote %" PRINTF_OFF_T "\n", fd,
fd_table[fd].bytes_read, fd_table[fd].bytes_written);
storeAppendPrintf(s, "\tFD desc: %s\n", fd_table[fd].desc);
storeAppendPrintf(s, "\tin: buf %p, offset %ld, size %ld\n",
Index: squid/src/stmem.c
diff -u squid/src/stmem.c:1.7.6.2 squid/src/stmem.c:1.7.66.7
--- squid/src/stmem.c:1.7.6.2 Sat Mar 19 14:24:33 2005
+++ squid/src/stmem.c Fri Mar 25 17:52:25 2005
@@ -65,10 +65,10 @@
mem->origin_offset = 0;
}
-int
-stmemFreeDataUpto(mem_hdr * mem, int target_offset)
+squid_off_t
+stmemFreeDataUpto(mem_hdr * mem, squid_off_t target_offset)
{
- int current_offset = mem->origin_offset;
+ squid_off_t current_offset = mem->origin_offset;
mem_node *p = mem->head;
while (p && ((current_offset + p->len) <= target_offset)) {
if (p == mem->tail) {
@@ -135,15 +135,15 @@
}
ssize_t
-stmemCopy(const mem_hdr * mem, off_t offset, char *buf, size_t size)
+stmemCopy(const mem_hdr * mem, squid_off_t offset, char *buf, size_t size)
{
mem_node *p = mem->head;
- off_t t_off = mem->origin_offset;
+ squid_off_t t_off = mem->origin_offset;
size_t bytes_to_go = size;
char *ptr_to_buf = NULL;
int bytes_from_this_packet = 0;
int bytes_into_this_packet = 0;
- debug(19, 6) ("memCopy: offset %ld: size %d\n", (long int) offset, (int) size);
+ debug(19, 6) ("memCopy: offset %" PRINTF_OFF_T ": size %d\n", offset, (int) size);
if (p == NULL)
return 0;
assert(size > 0);
Index: squid/src/store.c
diff -u squid/src/store.c:1.16.6.7 squid/src/store.c:1.16.6.7.6.8
--- squid/src/store.c:1.16.6.7 Fri Jan 21 19:15:07 2005
+++ squid/src/store.c Fri Mar 25 17:52:25 2005
@@ -606,7 +606,7 @@
if (mem->object_sz < Config.Store.minObjectSize)
return 1;
if (mem->reply->content_length > -1)
- if (mem->reply->content_length < (int) Config.Store.minObjectSize)
+ if (mem->reply->content_length < Config.Store.minObjectSize)
return 1;
return 0;
}
@@ -638,9 +638,6 @@
e->mem_obj->inmem_hi > Config.Store.maxObjectSize) {
debug(20, 2) ("storeCheckCachable: NO: too big\n");
store_check_cachable_hist.no.too_big++;
- } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) {
- debug(20, 2) ("storeCheckCachable: NO: too big\n");
- store_check_cachable_hist.no.too_big++;
} else if (storeCheckTooSmall(e)) {
debug(20, 2) ("storeCheckCachable: NO: too small\n");
store_check_cachable_hist.no.too_small++;
@@ -939,18 +936,18 @@
static int
storeEntryValidLength(const StoreEntry * e)
{
- int diff;
- int clen;
+ squid_off_t diff;
+ squid_off_t clen;
const HttpReply *reply;
assert(e->mem_obj != NULL);
reply = e->mem_obj->reply;
debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->hash.key));
- debug(20, 5) ("storeEntryValidLength: object_len = %d\n",
+ debug(20, 5) ("storeEntryValidLength: object_len = %" PRINTF_OFF_T "\n",
objectLen(e));
debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n",
reply->hdr_sz);
clen = httpReplyBodySize(e->mem_obj->method, reply);
- debug(20, 5) ("storeEntryValidLength: content_length = %d\n",
+ debug(20, 5) ("storeEntryValidLength: content_length = %" PRINTF_OFF_T "\n",
clen);
if (clen < 0) {
debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n",
@@ -960,7 +957,7 @@
diff = reply->hdr_sz + clen - objectLen(e);
if (diff == 0)
return 1;
- debug(20, 2) ("storeEntryValidLength: %d bytes too %s; '%s'\n",
+ debug(20, 2) ("storeEntryValidLength: %" PRINTF_OFF_T " bytes too %s; '%s'\n",
diff < 0 ? -diff : diff,
diff < 0 ? "big" : "small",
storeKeyText(e->hash.key));
@@ -1128,15 +1125,15 @@
mem->data_hdr.head);
debug(20, 1) ("MemObject->data.tail: %p\n",
mem->data_hdr.tail);
- debug(20, 1) ("MemObject->data.origin_offset: %d\n",
+ debug(20, 1) ("MemObject->data.origin_offset: %" PRINTF_OFF_T "\n",
mem->data_hdr.origin_offset);
- debug(20, 1) ("MemObject->start_ping: %d.%06d\n",
- (int) mem->start_ping.tv_sec,
+ debug(20, 1) ("MemObject->start_ping: %ld.%06d\n",
+ (long int) mem->start_ping.tv_sec,
(int) mem->start_ping.tv_usec);
- debug(20, 1) ("MemObject->inmem_hi: %d\n",
- (int) mem->inmem_hi);
- debug(20, 1) ("MemObject->inmem_lo: %d\n",
- (int) mem->inmem_lo);
+ debug(20, 1) ("MemObject->inmem_hi: %" PRINTF_OFF_T "\n",
+ mem->inmem_hi);
+ debug(20, 1) ("MemObject->inmem_lo: %" PRINTF_OFF_T "\n",
+ mem->inmem_lo);
debug(20, 1) ("MemObject->nclients: %d\n",
mem->nclients);
debug(20, 1) ("MemObject->reply: %p\n",
@@ -1154,11 +1151,11 @@
debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->hash.key));
debug(20, l) ("StoreEntry->next: %p\n", e->hash.next);
debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj);
- debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp);
- debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref);
- debug(20, l) ("StoreEntry->expires: %d\n", (int) e->expires);
- debug(20, l) ("StoreEntry->lastmod: %d\n", (int) e->lastmod);
- debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz);
+ debug(20, l) ("StoreEntry->timestamp: %ld\n", (long int) e->timestamp);
+ debug(20, l) ("StoreEntry->lastref: %ld\n", (long int) e->lastref);
+ debug(20, l) ("StoreEntry->expires: %ld\n", (long int) e->expires);
+ debug(20, l) ("StoreEntry->lastmod: %ld\n", (long int) e->lastmod);
+ debug(20, l) ("StoreEntry->swap_file_sz: %" PRINTF_OFF_T "\n", (squid_off_t) e->swap_file_sz);
debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount);
debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e));
debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn);
@@ -1240,14 +1237,14 @@
storeSwapOut(e);
}
-int
+squid_off_t
objectLen(const StoreEntry * e)
{
assert(e->mem_obj != NULL);
return e->mem_obj->object_sz;
}
-int
+squid_off_t
contentLen(const StoreEntry * e)
{
assert(e->mem_obj != NULL);
Index: squid/src/store_client.c
diff -u squid/src/store_client.c:1.9.6.2 squid/src/store_client.c:1.9.6.2.16.9
--- squid/src/store_client.c:1.9.6.2 Wed Aug 6 19:13:44 2003
+++ squid/src/store_client.c Fri Mar 25 17:52:26 2005
@@ -141,10 +141,6 @@
sc->flags.disk_io_pending = 0;
sc->entry = e;
sc->type = storeClientType(e);
- if (sc->type == STORE_DISK_CLIENT)
- /* assert we'll be able to get the data we want */
- /* maybe we should open swapin_fd here */
- assert(e->swap_filen > -1 || storeSwapOutAble(e));
dlinkAdd(sc, &sc->node, &mem->clients);
#if DELAY_POOLS
sc->delay_id = 0;
@@ -179,18 +175,18 @@
void
storeClientCopy(store_client * sc,
StoreEntry * e,
- off_t seen_offset,
- off_t copy_offset,
+ squid_off_t seen_offset,
+ squid_off_t copy_offset,
size_t size,
char *buf,
STCB * callback,
void *data)
{
assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
- debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n",
+ debug(20, 3) ("storeClientCopy: %s, seen %" PRINTF_OFF_T ", want %" PRINTF_OFF_T ", size %d, cb %p, cbdata %p\n",
storeKeyText(e->hash.key),
- (int) seen_offset,
- (int) copy_offset,
+ seen_offset,
+ copy_offset,
(int) size,
callback,
data);
@@ -220,7 +216,7 @@
static int
storeClientNoMoreToSend(StoreEntry * e, store_client * sc)
{
- ssize_t len;
+ squid_off_t len;
if (e->store_status == STORE_PENDING)
return 0;
if ((len = objectLen(e)) < 0)
@@ -265,7 +261,7 @@
storeClientCopy3(StoreEntry * e, store_client * sc)
{
MemObject *mem = e->mem_obj;
- size_t sz;
+ ssize_t sz;
if (storeClientNoMoreToSend(e, sc)) {
/* There is no more to send! */
@@ -316,6 +312,8 @@
debug(20, 3) ("storeClientCopy3: Copying from memory\n");
sz = stmemCopy(&mem->data_hdr,
sc->copy_offset, sc->copy_buf, sc->copy_size);
+ if (EBIT_TEST(e->flags, RELEASE_REQUEST))
+ storeSwapOutMaintainMemObject(e);
storeClientCallback(sc, sz);
return;
}
@@ -431,6 +429,7 @@
}
break;
case STORE_META_STD:
+ case STORE_META_STD_LFS:
break;
case STORE_META_VARY_HEADERS:
if (mem->vary_headers) {
@@ -534,16 +533,18 @@
/*assert(!sc->flags.disk_io_pending); */
cbdataFree(sc);
assert(e->lock_count > 0);
+ storeSwapOutMaintainMemObject(e);
if (mem->nclients == 0)
CheckQuickAbort(e);
return 1;
}
-off_t
+squid_off_t
storeLowestMemReaderOffset(const StoreEntry * entry)
{
const MemObject *mem = entry->mem_obj;
- off_t lowest = mem->inmem_hi + 1;
+ squid_off_t lowest = mem->inmem_hi + 1;
+ squid_off_t highest = -1;
store_client *sc;
dlink_node *nx = NULL;
dlink_node *node;
@@ -553,14 +554,15 @@
nx = node->next;
if (sc->callback_data == NULL) /* open slot */
continue;
- if (sc->type != STORE_MEM_CLIENT)
+ if (sc->copy_offset > highest)
+ highest = sc->copy_offset;
+ if (mem->swapout.sio != NULL && sc->type != STORE_MEM_CLIENT)
continue;
- if (sc->type == STORE_DISK_CLIENT)
- if (NULL != sc->swapin_sio)
- continue;
if (sc->copy_offset < lowest)
lowest = sc->copy_offset;
}
+ if (highest < lowest && highest >= 0)
+ return highest;
return lowest;
}
@@ -603,9 +605,9 @@
static int
CheckQuickAbort2(StoreEntry * entry)
{
- int curlen;
- int minlen;
- int expectlen;
+ squid_off_t curlen;
+ squid_off_t minlen;
+ squid_off_t expectlen;
MemObject *mem = entry->mem_obj;
assert(mem);
debug(20, 3) ("CheckQuickAbort2: entry=%p, mem=%p\n", entry, mem);
@@ -618,8 +620,8 @@
return 1;
}
expectlen = mem->reply->content_length + mem->reply->hdr_sz;
- curlen = (int) mem->inmem_hi;
- minlen = (int) Config.quickAbort.min << 10;
+ curlen = mem->inmem_hi;
+ minlen = Config.quickAbort.min << 10;
if (minlen < 0) {
debug(20, 3) ("CheckQuickAbort2: NO disabled\n");
return 0;
Index: squid/src/store_dir.c
diff -u squid/src/store_dir.c:1.19.6.2 squid/src/store_dir.c:1.19.6.2.16.3
--- squid/src/store_dir.c:1.19.6.2 Thu Jul 17 19:13:40 2003
+++ squid/src/store_dir.c Wed Mar 2 20:43:44 2005
@@ -45,7 +45,7 @@
#include
#endif
-static int storeDirValidSwapDirSize(int, ssize_t);
+static int storeDirValidSwapDirSize(int, squid_off_t);
static STDIRSELECT storeDirSelectSwapDirRoundRobin;
static STDIRSELECT storeDirSelectSwapDirLeastLoad;
@@ -106,7 +106,7 @@
* ie any-sized-object swapdirs. This is a good thing.
*/
static int
-storeDirValidSwapDirSize(int swapdir, ssize_t objsize)
+storeDirValidSwapDirSize(int swapdir, squid_off_t objsize)
{
/*
* If the swapdir's max_obj_size is -1, then it definitely can
@@ -144,7 +144,7 @@
int i;
int load;
SwapDir *sd;
- ssize_t objsize = (ssize_t) objectLen(e);
+ squid_off_t objsize = objectLen(e);
for (i = 0; i <= Config.cacheSwap.n_configured; i++) {
if (++dirn >= Config.cacheSwap.n_configured)
dirn = 0;
@@ -181,9 +181,9 @@
static int
storeDirSelectSwapDirLeastLoad(const StoreEntry * e)
{
- ssize_t objsize;
- ssize_t most_free = 0, cur_free;
- ssize_t least_objsize = -1;
+ squid_off_t objsize;
+ int most_free = 0, cur_free;
+ squid_off_t least_objsize = -1;
int least_load = INT_MAX;
int load;
int dirn = -1;
@@ -191,7 +191,7 @@
SwapDir *SD;
/* Calculate the object size */
- objsize = (ssize_t) objectLen(e);
+ objsize = objectLen(e);
if (objsize != -1)
objsize += e->mem_obj->swap_hdr_sz;
for (i = 0; i < Config.cacheSwap.n_configured; i++) {
@@ -270,7 +270,7 @@
}
void
-storeDirUpdateSwapSize(SwapDir * SD, size_t size, int sign)
+storeDirUpdateSwapSize(SwapDir * SD, squid_off_t size, int sign)
{
int blks = (size + SD->fs.blksize - 1) / SD->fs.blksize;
int k = (blks * SD->fs.blksize >> 10) * sign;
Index: squid/src/store_io.c
diff -u squid/src/store_io.c:1.3 squid/src/store_io.c:1.3.74.1
--- squid/src/store_io.c:1.3 Wed Oct 24 02:42:14 2001
+++ squid/src/store_io.c Wed Mar 2 19:44:07 2005
@@ -76,14 +76,14 @@
}
void
-storeRead(storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+storeRead(storeIOState * sio, char *buf, size_t size, squid_off_t offset, STRCB * callback, void *callback_data)
{
SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn];
SD->obj.read(SD, sio, buf, size, offset, callback, callback_data);
}
void
-storeWrite(storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+storeWrite(storeIOState * sio, char *buf, size_t size, squid_off_t offset, FREE * free_func)
{
SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn];
SD->obj.write(SD, sio, buf, size, offset, free_func);
@@ -96,7 +96,7 @@
SD->obj.unlink(SD, e);
}
-off_t
+squid_off_t
storeOffset(storeIOState * sio)
{
return sio->offset;
Index: squid/src/store_log.c
diff -u squid/src/store_log.c:1.7 squid/src/store_log.c:1.7.74.3
--- squid/src/store_log.c:1.7 Thu Oct 18 13:52:11 2001
+++ squid/src/store_log.c Fri Mar 25 17:52:26 2005
@@ -69,26 +69,26 @@
* Because if we print it before the swap file number, it'll break
* the existing log format.
*/
- logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s %4d %9d %9d %9d %s %d/%d %s %s\n",
- (int) current_time.tv_sec,
+ logfilePrintf(storelog, "%9ld.%03d %-7s %02d %08X %s %4d %9ld %9ld %9ld %s %" PRINTF_OFF_T "/%" PRINTF_OFF_T " %s %s\n",
+ (long int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_dirn,
e->swap_filen,
storeKeyText(e->hash.key),
reply->sline.status,
- (int) reply->date,
- (int) reply->last_modified,
- (int) reply->expires,
+ (long int) reply->date,
+ (long int) reply->last_modified,
+ (long int) reply->expires,
strLen(reply->content_type) ? strBuf(reply->content_type) : "unknown",
reply->content_length,
- (int) (mem->inmem_hi - mem->reply->hdr_sz),
+ mem->inmem_hi - mem->reply->hdr_sz,
RequestMethodStr[mem->method],
mem->log_url);
} else {
/* no mem object. Most RELEASE cases */
- logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s ? ? ? ? ?/? ?/? ? ?\n",
- (int) current_time.tv_sec,
+ logfilePrintf(storelog, "%9ld.%03d %-7s %02d %08X %s ? ? ? ? ?/? ?/? ? ?\n",
+ (long int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_dirn,
Index: squid/src/store_swapmeta.c
diff -u squid/src/store_swapmeta.c:1.8 squid/src/store_swapmeta.c:1.8.74.3
--- squid/src/store_swapmeta.c:1.8 Wed Oct 24 02:42:14 2001
+++ squid/src/store_swapmeta.c Thu Mar 24 19:34:32 2005
@@ -73,7 +73,11 @@
url = storeUrl(e);
debug(20, 3) ("storeSwapMetaBuild: %s\n", url);
T = storeSwapTLVAdd(STORE_META_KEY, e->hash.key, MD5_DIGEST_CHARS, T);
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
T = storeSwapTLVAdd(STORE_META_STD, &e->timestamp, STORE_HDR_METASIZE, T);
+#else
+ T = storeSwapTLVAdd(STORE_META_STD_LFS, &e->timestamp, STORE_HDR_METASIZE, T);
+#endif
T = storeSwapTLVAdd(STORE_META_URL, url, strlen(url) + 1, T);
vary = e->mem_obj->vary_headers;
if (vary)
@@ -86,7 +90,7 @@
{
int buflen = 0;
tlv *t;
- off_t j = 0;
+ int j = 0;
char *buf;
assert(length != NULL);
buflen++; /* STORE_META_OK */
@@ -119,7 +123,7 @@
char type;
int length;
int buflen;
- off_t j = 0;
+ int j = 0;
assert(buf != NULL);
assert(hdr_len != NULL);
if (buf[j++] != (char) STORE_META_OK)
Index: squid/src/store_swapout.c
diff -u squid/src/store_swapout.c:1.11.2.2 squid/src/store_swapout.c:1.11.2.1.30.10
--- squid/src/store_swapout.c:1.11.2.2 Fri Mar 18 17:42:55 2005
+++ squid/src/store_swapout.c Fri Mar 25 17:52:26 2005
@@ -35,10 +35,11 @@
#include "squid.h"
-static off_t storeSwapOutObjectBytesOnDisk(const MemObject *);
+static squid_off_t storeSwapOutObjectBytesOnDisk(const MemObject *);
static void storeSwapOutStart(StoreEntry * e);
static STIOCB storeSwapOutFileClosed;
static STIOCB storeSwapOutFileNotify;
+static int storeSwapOutAble(const StoreEntry * e);
/* start swapping object to disk */
static void
@@ -95,15 +96,70 @@
e->swap_dirn = mem->swapout.sio->swap_dirn;
}
+/* as sideeffect returns the on disk offset for use by storeSwapOut() */
+squid_off_t
+storeSwapOutMaintainMemObject(StoreEntry * e)
+{
+ MemObject *mem = e->mem_obj;
+ squid_off_t lowest_offset;
+ squid_off_t new_mem_lo;
+ squid_off_t on_disk = -1;
+ int swapout_able = storeSwapOutAble(e);
+
+ if (!swapout_able) {
+ /* Stop writing to disk */
+ storeReleaseRequest(e);
+ if (e->mem_obj->swapout.sio != NULL)
+ storeSwapOutFileClose(e);
+ }
+ /* storeLowestMemReaderOffset must be after the storeSwapOutFileClose
+ * call above as storeLowestMemReaderOffset needs to know if we are
+ * still writing to disk or not
+ */
+ lowest_offset = storeLowestMemReaderOffset(e);
+ debug(20, 7) ("storeSwapOut: lowest_offset = %" PRINTF_OFF_T "\n",
+ lowest_offset);
+ /*
+ * Careful. lowest_offset can be greater than inmem_hi, such
+ * as in the case of a range request.
+ */
+ if (mem->inmem_hi < lowest_offset)
+ new_mem_lo = lowest_offset;
+ else if (mem->inmem_hi > Config.Store.maxInMemObjSize)
+ new_mem_lo = lowest_offset;
+ else if (mem->reply->content_length > Config.Store.maxInMemObjSize)
+ new_mem_lo = lowest_offset;
+ else
+ new_mem_lo = mem->inmem_lo;
+ if (new_mem_lo < mem->inmem_lo)
+ new_mem_lo = mem->inmem_lo;
+ if (swapout_able) {
+ /*
+ * We should only free up to what we know has been written
+ * to disk, not what has been queued for writing. Otherwise
+ * there will be a chunk of the data which is not in memory
+ * and is not yet on disk.
+ * The -1 makes sure the page isn't freed until storeSwapOut has
+ * walked to the next page. (mem->swapout.memnode)
+ */
+ if ((on_disk = storeSwapOutObjectBytesOnDisk(mem)) - 1 < new_mem_lo)
+ new_mem_lo = on_disk - 1;
+ if (new_mem_lo == -1)
+ new_mem_lo = 0; /* the above might become -1 */
+ }
+ if (mem->inmem_lo != new_mem_lo)
+ mem->inmem_lo = stmemFreeDataUpto(&mem->data_hdr, new_mem_lo);
+
+ return on_disk;
+}
+
void
storeSwapOut(StoreEntry * e)
{
MemObject *mem = e->mem_obj;
- off_t lowest_offset;
- off_t new_mem_lo;
- off_t on_disk = 0;
- ssize_t swapout_size;
- ssize_t swap_buf_len;
+ squid_off_t on_disk;
+ squid_off_t swapout_size;
+ size_t swap_buf_len;
if (mem == NULL)
return;
/* should we swap something out to disk? */
@@ -119,24 +175,21 @@
debug(20, 3) ("storeSwapOut: %s SPECIAL\n", storeUrl(e));
return;
}
- debug(20, 7) ("storeSwapOut: mem->inmem_lo = %d\n",
- (int) mem->inmem_lo);
- debug(20, 7) ("storeSwapOut: mem->inmem_hi = %d\n",
- (int) mem->inmem_hi);
- debug(20, 7) ("storeSwapOut: swapout.queue_offset = %d\n",
- (int) mem->swapout.queue_offset);
+ debug(20, 7) ("storeSwapOut: mem->inmem_lo = %" PRINTF_OFF_T "\n",
+ mem->inmem_lo);
+ debug(20, 7) ("storeSwapOut: mem->inmem_hi = %" PRINTF_OFF_T "\n",
+ mem->inmem_hi);
+ debug(20, 7) ("storeSwapOut: swapout.queue_offset = %" PRINTF_OFF_T "\n",
+ mem->swapout.queue_offset);
if (mem->swapout.sio)
- debug(20, 7) ("storeSwapOut: storeOffset() = %d\n",
- (int) storeOffset(mem->swapout.sio));
+ debug(20, 7) ("storeSwapOut: storeOffset() = %" PRINTF_OFF_T "\n",
+ storeOffset(mem->swapout.sio));
assert(mem->inmem_hi >= mem->swapout.queue_offset);
- lowest_offset = storeLowestMemReaderOffset(e);
- debug(20, 7) ("storeSwapOut: lowest_offset = %d\n",
- (int) lowest_offset);
/*
* Grab the swapout_size and check to see whether we're going to defer
* the swapout based upon size
*/
- swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset);
+ swapout_size = mem->inmem_hi - mem->swapout.queue_offset;
if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) {
/*
* NOTE: the store_maxobjsize here is the max of optional
@@ -151,53 +204,20 @@
debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n");
return;
}
- /*
- * Careful. lowest_offset can be greater than inmem_hi, such
- * as in the case of a range request.
- */
- if (mem->inmem_hi < lowest_offset)
- new_mem_lo = lowest_offset;
- else if (mem->inmem_hi - mem->inmem_lo > Config.Store.maxInMemObjSize)
- new_mem_lo = lowest_offset;
- else
- new_mem_lo = mem->inmem_lo;
- assert(new_mem_lo >= mem->inmem_lo);
- if (storeSwapOutAble(e)) {
- /*
- * We should only free up to what we know has been written
- * to disk, not what has been queued for writing. Otherwise
- * there will be a chunk of the data which is not in memory
- * and is not yet on disk.
- * The -1 makes sure the page isn't freed until storeSwapOut has
- * walked to the next page. (mem->swapout.memnode)
- */
- if ((on_disk = storeSwapOutObjectBytesOnDisk(mem)) - 1 < new_mem_lo)
- new_mem_lo = on_disk - 1;
- if (new_mem_lo == -1)
- new_mem_lo = 0; /* the above might become -1 */
- } else if (new_mem_lo > 0) {
- /*
- * Its not swap-able, and we're about to delete a chunk,
- * so we must make it PRIVATE. This is tricky/ugly because
- * for the most part, we treat swapable == cachable here.
- */
- storeReleaseRequest(e);
- }
- stmemFreeDataUpto(&mem->data_hdr, new_mem_lo);
- mem->inmem_lo = new_mem_lo;
-#if SIZEOF_OFF_T == 4
+ on_disk = storeSwapOutMaintainMemObject(e);
+#if SIZEOF_SQUID_OFF_T <= 4
if (mem->inmem_hi > 0x7FFF0000) {
- debug(20, 0) ("WARNING: preventing off_t overflow for %s\n", storeUrl(e));
+ debug(20, 0) ("WARNING: preventing squid_off_t overflow for %s\n", storeUrl(e));
storeAbort(e);
return;
}
#endif
+ if (on_disk < 0)
+ return;
if (e->swap_status == SWAPOUT_WRITING)
assert(mem->inmem_lo <= on_disk);
- if (!storeSwapOutAble(e))
- return;
- debug(20, 7) ("storeSwapOut: swapout_size = %d\n",
- (int) swapout_size);
+ debug(20, 7) ("storeSwapOut: swapout_size = %" PRINTF_OFF_T "\n",
+ swapout_size);
if (swapout_size == 0) {
if (e->store_status == STORE_OK)
storeSwapOutFileClose(e);
@@ -253,14 +273,14 @@
debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len);
assert(swap_buf_len > 0);
- debug(20, 3) ("storeSwapOut: swapping out %ld bytes from %ld\n",
- (long int) swap_buf_len, (long int) mem->swapout.queue_offset);
+ debug(20, 3) ("storeSwapOut: swapping out %d bytes from %" PRINTF_OFF_T "\n",
+ (int) swap_buf_len, mem->swapout.queue_offset);
mem->swapout.queue_offset += swap_buf_len;
storeWrite(mem->swapout.sio, stmemNodeGet(mem->swapout.memnode), swap_buf_len, -1, stmemNodeFree);
/* the storeWrite() call might generate an error */
if (e->swap_status != SWAPOUT_WRITING)
break;
- swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset);
+ swapout_size = mem->inmem_hi - mem->swapout.queue_offset;
if (e->store_status == STORE_PENDING)
if (swapout_size < SM_PAGE_SIZE)
break;
@@ -283,12 +303,14 @@
storeSwapOutFileClose(StoreEntry * e)
{
MemObject *mem = e->mem_obj;
+ storeIOState *sio = mem->swapout.sio;
assert(mem != NULL);
debug(20, 3) ("storeSwapOutFileClose: %s\n", storeKeyText(e->hash.key));
debug(20, 3) ("storeSwapOutFileClose: sio = %p\n", mem->swapout.sio);
- if (mem->swapout.sio == NULL)
+ if (sio == NULL)
return;
- storeClose(mem->swapout.sio);
+ mem->swapout.sio = NULL;
+ storeClose(sio);
}
static void
@@ -335,7 +357,7 @@
/*
* How much of the object data is on the disk?
*/
-static off_t
+static squid_off_t
storeSwapOutObjectBytesOnDisk(const MemObject * mem)
{
/*
@@ -349,7 +371,7 @@
* meaning we haven't even opened the swapout file
* yet.
*/
- off_t nwritten;
+ squid_off_t nwritten;
if (mem->swapout.sio == NULL)
return 0;
nwritten = storeOffset(mem->swapout.sio);
@@ -361,24 +383,19 @@
/*
* Is this entry a candidate for writing to disk?
*/
-int
+static int
storeSwapOutAble(const StoreEntry * e)
{
- dlink_node *node;
+ if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize)
+ return 0;
+ if (!EBIT_TEST(e->flags, ENTRY_CACHABLE))
+ return 0;
if (e->mem_obj->swapout.sio != NULL)
return 1;
if (e->mem_obj->inmem_lo > 0)
return 0;
- /*
- * If there are DISK clients, we must write to disk
- * even if its not cachable
- */
- for (node = e->mem_obj->clients.head; node; node = node->next) {
- if (((store_client *) node->data)->type == STORE_DISK_CLIENT)
- return 1;
- }
/* Don't pollute the disk with icons and other special entries */
if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
return 0;
- return EBIT_TEST(e->flags, ENTRY_CACHABLE);
+ return 1;
}
Index: squid/src/structs.h
diff -u squid/src/structs.h:1.48.2.35 squid/src/structs.h:1.48.2.33.2.13
--- squid/src/structs.h:1.48.2.35 Fri Mar 18 17:42:56 2005
+++ squid/src/structs.h Fri Mar 25 17:52:26 2005
@@ -57,7 +57,7 @@
};
struct _acl_user_ip_data {
- size_t max;
+ int max;
struct {
unsigned int strict:1;
} flags;
@@ -122,10 +122,10 @@
dlink_list proxy_match_cache;
/* what ip addresses has this user been seen at?, plus a list length cache */
dlink_list ip_list;
- size_t ipcount;
+ int ipcount;
long expiretime;
/* how many references are outstanding to this instance */
- size_t references;
+ int references;
/* the auth scheme has it's own private data area */
void *scheme_data;
/* the auth_user_request structures that link to this. Yes it could be a splaytree
@@ -143,7 +143,7 @@
/* any scheme specific request related data */
void *scheme_data;
/* how many 'processes' are working on this data */
- size_t references;
+ int references;
/* We only attempt authentication once per http request. This
* is to allow multiple auth acl references from different _access areas
* when using connection based authentication
@@ -228,7 +228,7 @@
struct _body_size {
dlink_node node;
acl_access *access_list;
- size_t maxsize;
+ squid_off_t maxsize;
};
struct _http_version_t {
@@ -385,20 +385,20 @@
struct _SquidConfig {
struct {
- size_t maxSize;
+ squid_off_t maxSize;
int highWaterMark;
int lowWaterMark;
} Swap;
- size_t memMaxSize;
+ squid_off_t memMaxSize;
struct {
char *relayHost;
u_short relayPort;
peer *peer;
} Wais;
struct {
- size_t min;
+ squid_off_t min;
int pct;
- size_t max;
+ squid_off_t max;
} quickAbort;
RemovalPolicySettings *replPolicy;
RemovalPolicySettings *memPolicy;
@@ -428,9 +428,9 @@
time_t idns_query;
#endif
} Timeout;
- size_t maxRequestHeaderSize;
- size_t maxRequestBodySize;
- size_t maxReplyHeaderSize;
+ squid_off_t maxRequestHeaderSize;
+ squid_off_t maxRequestBodySize;
+ squid_off_t maxReplyHeaderSize;
dlink_list ReplyBodySize;
struct {
u_short icp;
@@ -507,7 +507,7 @@
u_short port;
} Accel;
char *appendDomain;
- size_t appendDomainLen;
+ int appendDomainLen;
char *debugOptions;
char *pidFilename;
char *mimeTablePathname;
@@ -531,8 +531,8 @@
#endif
struct in_addr client_netmask;
} Addrs;
- size_t tcpRcvBufsz;
- size_t udpMaxHitObjsz;
+ squid_off_t tcpRcvBufsz;
+ squid_off_t udpMaxHitObjsz;
wordlist *hierarchy_stoplist;
wordlist *mcast_group_list;
wordlist *dns_testname_list;
@@ -552,10 +552,10 @@
cachemgr_passwd *passwd_list;
struct {
int objectsPerBucket;
- size_t avgObjectSize;
- size_t maxObjectSize;
- size_t minObjectSize;
- size_t maxInMemObjSize;
+ squid_off_t avgObjectSize;
+ squid_off_t maxObjectSize;
+ squid_off_t minObjectSize;
+ squid_off_t maxInMemObjSize;
} Store;
struct {
int high;
@@ -637,7 +637,7 @@
int n_configured;
} authConfig;
struct {
- size_t list_width;
+ int list_width;
int list_wrap;
char *anon_user;
int passive;
@@ -660,7 +660,7 @@
int onerror;
} retry;
struct {
- size_t limit;
+ squid_off_t limit;
} MemPools;
#if DELAY_POOLS
delayConfig Delay;
@@ -675,7 +675,7 @@
} comm_incoming;
int max_open_disk_fds;
int uri_whitespace;
- size_t rangeOffsetLimit;
+ squid_off_t rangeOffsetLimit;
#if MULTICAST_MISS_STREAM
struct {
struct in_addr addr;
@@ -692,7 +692,7 @@
int bits_per_entry;
time_t rebuild_period;
time_t rewrite_period;
- size_t swapout_chunk_size;
+ squid_off_t swapout_chunk_size;
int rebuild_chunk_percentage;
} digest;
#endif
@@ -705,7 +705,7 @@
struct {
int high_rptm;
int high_pf;
- size_t high_memory;
+ squid_off_t high_memory;
} warnings;
char *store_dir_select_algorithm;
int sleep_after_fork; /* microseconds */
@@ -732,31 +732,19 @@
struct _dread_ctrl {
int fd;
- off_t offset;
- int req_len;
+ squid_off_t file_offset;
+ size_t req_len;
char *buf;
int end_of_file;
DRCB *handler;
void *client_data;
};
-struct _dnsserver_t {
- int id;
- int inpipe;
- int outpipe;
- time_t answer;
- off_t offset;
- size_t size;
- char ip_inbuf[DNS_INBUF_SZ];
- struct timeval dispatch_time;
- void *data;
-};
-
struct _dwrite_q {
- off_t file_offset;
+ squid_off_t file_offset;
char *buf;
- int len;
- off_t buf_offset;
+ size_t len;
+ size_t buf_offset;
dwrite_q *next;
FREE *free_func;
};
@@ -793,15 +781,15 @@
unsigned int close_on_exec:1;
unsigned int read_pending:1;
} flags;
- int bytes_read;
- int bytes_written;
+ squid_off_t bytes_read;
+ squid_off_t bytes_written;
int uses; /* ie # req's over persistent conn */
struct _fde_disk {
DWCB *wrt_handle;
void *wrt_handle_data;
dwrite_q *write_q;
dwrite_q *write_q_tail;
- off_t offset;
+ squid_off_t offset;
} disk;
PF *read_handler;
void *read_data;
@@ -849,7 +837,7 @@
/* protected, use interface functions instead */
append_f append;
vprintf_f vprintf;
- void *real_handler; /* first parameter to real append and vprintf */
+ void *real_handle; /* first parameter to real append and vprintf */
};
/* http status line */
@@ -885,8 +873,8 @@
/* http byte-range-spec */
struct _HttpHdrRangeSpec {
- ssize_t offset;
- ssize_t length;
+ squid_off_t offset;
+ squid_off_t length;
};
/* There may be more than one byte range specified in the request.
@@ -900,7 +888,7 @@
/* http content-range header field */
struct _HttpHdrContRange {
HttpHdrRangeSpec spec;
- ssize_t elength; /* entity length, not content length */
+ squid_off_t elength; /* entity length, not content length */
};
/* some fields can hold either time or etag specs (e.g. If-Range) */
@@ -914,8 +902,8 @@
struct _HttpHdrRangeIter {
HttpHdrRangePos pos;
const HttpHdrRangeSpec *spec; /* current spec at pos */
- ssize_t debt_size; /* bytes left to send from the current spec */
- ssize_t prefix_size; /* the size of the incoming HTTP msg prefix */
+ squid_off_t debt_size; /* bytes left to send from the current spec */
+ squid_off_t prefix_size; /* the size of the incoming HTTP msg prefix */
String boundary; /* boundary for multipart responses */
};
@@ -962,7 +950,7 @@
int hdr_sz; /* sums _stored_ status-line, headers, and */
/* public, readable; never update these or their .hdr equivalents directly */
- int content_length;
+ squid_off_t content_length;
time_t date;
time_t last_modified;
time_t expires;
@@ -1053,7 +1041,7 @@
} icp;
struct {
struct in_addr caddr;
- size_t size;
+ squid_off_t size;
log_type code;
int msec;
const char *rfc931;
@@ -1077,8 +1065,8 @@
char *uri;
char *log_uri;
struct {
- off_t offset;
- size_t size;
+ squid_off_t offset;
+ squid_off_t size;
} out;
HttpHdrRangeIter range_iter; /* data for iterating thru range specs */
size_t req_sz; /* raw request size on input, not current request size */
@@ -1106,18 +1094,18 @@
char *location;
} redirect;
dlink_node active;
- size_t maxBodySize;
+ squid_off_t maxBodySize;
};
struct _ConnStateData {
int fd;
struct {
char *buf;
- off_t offset;
+ size_t offset;
size_t size;
} in;
struct {
- size_t size_left; /* How much body left to process */
+ squid_off_t size_left; /* How much body left to process */
request_t *request; /* Parameters passed to clientReadBody */
char *buf;
size_t bufsize;
@@ -1187,8 +1175,8 @@
store_client *sc;
store_client *old_sc;
request_t *request;
- int offset;
- int mask_offset;
+ squid_off_t offset;
+ squid_off_t mask_offset;
time_t start_time;
time_t resp_time;
time_t expires;
@@ -1438,14 +1426,14 @@
struct _mem_hdr {
mem_node *head;
mem_node *tail;
- int origin_offset;
+ squid_off_t origin_offset;
};
/* keep track each client receiving data from that particular StoreEntry */
struct _store_client {
int type;
- off_t copy_offset;
- off_t seen_offset;
+ squid_off_t copy_offset;
+ squid_off_t seen_offset;
size_t copy_size;
char *copy_buf;
STCB *callback;
@@ -1503,12 +1491,12 @@
method_t method;
char *url;
mem_hdr data_hdr;
- off_t inmem_hi;
- off_t inmem_lo;
+ squid_off_t inmem_hi;
+ squid_off_t inmem_lo;
dlink_list clients;
int nclients;
struct {
- off_t queue_offset; /* relative to in-mem data */
+ squid_off_t queue_offset; /* relative to in-mem data */
mem_node *memnode; /* which node we're currently paging out */
storeIOState *sio;
} swapout;
@@ -1524,7 +1512,7 @@
char *log_url;
RemovalPolicyNode repl;
int id;
- ssize_t object_sz;
+ squid_off_t object_sz;
size_t swap_hdr_sz;
#if URL_CHECKSUM_DEBUG
unsigned int chksum;
@@ -1541,7 +1529,7 @@
time_t lastref;
time_t expires;
time_t lastmod;
- size_t swap_file_sz;
+ squid_file_sz swap_file_sz;
u_short refcount;
u_short flags;
/* END OF ON-DISK STORE_META_STD */
@@ -1561,7 +1549,7 @@
int max_size;
char *path;
int index; /* This entry's index into the swapDirs array */
- ssize_t max_objsize;
+ squid_off_t max_objsize;
RemovalPolicy *repl;
int removals;
int scanned;
@@ -1641,8 +1629,8 @@
sfileno swap_filen;
StoreEntry *e; /* Need this so the FS layers can play god */
mode_t mode;
- size_t st_size; /* do stat(2) after read open */
- off_t offset; /* current on-disk offset pointer */
+ squid_off_t st_size; /* do stat(2) after read open */
+ squid_off_t offset; /* current on-disk offset pointer */
STFNCB *file_callback; /* called on delayed sfileno assignments */
STIOCB *callback;
void *callback_data;
@@ -1678,7 +1666,7 @@
struct in_addr my_addr;
unsigned short my_port;
HttpHeader header;
- int content_length;
+ squid_off_t content_length;
HierarchyLogEntry hier;
err_type err_type;
char *peer_login; /* Configured peer login:password */
@@ -1715,7 +1703,7 @@
struct _CommWriteStateData {
char *buf;
size_t size;
- off_t offset;
+ squid_off_t offset;
CWCB *handler;
void *handler_data;
FREE *free_func;
@@ -1912,12 +1900,35 @@
time_t lastref;
time_t expires;
time_t lastmod;
+ squid_file_sz swap_file_sz;
+ u_short refcount;
+ u_short flags;
+ unsigned char key[MD5_DIGEST_CHARS];
+};
+
+struct _storeSwapLogHeader {
+ char op;
+ int version;
+ int record_size;
+};
+
+#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
+struct _storeSwapLogDataOld {
+ char op;
+ sfileno swap_filen;
+ time_t timestamp;
+ time_t lastref;
+ time_t expires;
+ time_t lastmod;
size_t swap_file_sz;
u_short refcount;
u_short flags;
unsigned char key[MD5_DIGEST_CHARS];
};
+#endif
+
+
/* object to track per-action memory usage (e.g. #idle objects) */
struct _MemMeter {
ssize_t level; /* current level (count or volume) */
@@ -1964,7 +1975,7 @@
struct _CacheDigest {
/* public, read-only */
char *mask; /* bit mask */
- size_t mask_size; /* mask size in bytes */
+ int mask_size; /* mask size in bytes */
int capacity; /* expected maximum for .count, not a hard limit */
int bits_per_entry; /* number of bits allocated for each entry from capacity */
int count; /* number of digested entries */
@@ -2076,7 +2087,7 @@
int wfd;
char *buf;
size_t buf_sz;
- off_t offset;
+ int offset;
struct timeval dispatch_time;
struct timeval answer_time;
dlink_node link;
@@ -2100,7 +2111,7 @@
int wfd;
char *buf;
size_t buf_sz;
- off_t offset;
+ int offset;
struct timeval dispatch_time;
struct timeval answer_time;
dlink_node link;
@@ -2185,7 +2196,7 @@
char path[MAXPATHLEN];
char *buf;
size_t bufsz;
- off_t offset;
+ ssize_t offset;
struct {
unsigned int fatal:1;
} flags;
Index: squid/src/tools.c
diff -u squid/src/tools.c:1.19.6.11 squid/src/tools.c:1.19.6.10.2.3
--- squid/src/tools.c:1.19.6.11 Fri Mar 4 19:15:54 2005
+++ squid/src/tools.c Thu Mar 10 06:18:23 2005
@@ -841,7 +841,7 @@
}
void
-kb_incr(kb_t * k, size_t v)
+kb_incr(kb_t * k, squid_off_t v)
{
k->bytes += v;
k->kb += (k->bytes >> 10);
Index: squid/src/typedefs.h
diff -u squid/src/typedefs.h:1.25.6.6 squid/src/typedefs.h:1.25.6.6.8.9
--- squid/src/typedefs.h:1.25.6.6 Tue Oct 5 19:15:18 2004
+++ squid/src/typedefs.h Fri Mar 25 17:52:26 2005
@@ -41,9 +41,29 @@
typedef signed int sfileno;
typedef signed int sdirno;
+#if SIZEOF_INT64_T > SIZEOF_LONG && defined(PRId64) && defined(INT64_MAX) && HAVE_STRTOLL
+typedef int64_t squid_off_t;
+#define SIZEOF_SQUID_OFF_T SIZEOF_INT64_T
+#define PRINTF_OFF_T PRId64
+#define strto_off_t (int64_t)strtoll
+#else
+typedef long squid_off_t;
+#define SIZEOF_SQUID_OFF_T SIZEOF_LONG
+#define PRINTF_OFF_T "ld"
+#define strto_off_t strtol
+#endif
+
+#if LARGE_CACHE_FILES
+typedef squid_off_t squid_file_sz;
+#define SIZEOF_SQUID_FILE_SZ SQUIDOF_SQUID_OFF_T
+#else
+typedef size_t squid_file_sz;
+#define SIZEOF_SQUID_FILE_SZ SIZEOF_SIZE_T
+#endif
+
typedef struct {
- size_t bytes;
- size_t kb;
+ squid_off_t bytes;
+ squid_off_t kb;
} kb_t;
typedef struct {
@@ -90,7 +110,6 @@
typedef struct _SquidConfig2 SquidConfig2;
typedef struct _close_handler close_handler;
typedef struct _dread_ctrl dread_ctrl;
-typedef struct _dnsserver_t dnsserver_t;
typedef struct _dwrite_q dwrite_q;
typedef struct _ETag ETag;
typedef struct _fde fde;
@@ -160,6 +179,8 @@
typedef struct _StatCounters StatCounters;
typedef struct _tlv tlv;
typedef struct _storeSwapLogData storeSwapLogData;
+typedef struct _storeSwapLogDataOld storeSwapLogDataOld;
+typedef struct _storeSwapLogHeader storeSwapLogHeader;
typedef struct _authConfig authConfig;
typedef struct _cacheSwap cacheSwap;
typedef struct _StatHist StatHist;
@@ -275,8 +296,8 @@
typedef storeIOState *STOBJCREATE(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *);
typedef storeIOState *STOBJOPEN(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *);
typedef void STOBJCLOSE(SwapDir *, storeIOState *);
-typedef void STOBJREAD(SwapDir *, storeIOState *, char *, size_t, off_t, STRCB *, void *);
-typedef void STOBJWRITE(SwapDir *, storeIOState *, char *, size_t, off_t, FREE *);
+typedef void STOBJREAD(SwapDir *, storeIOState *, char *, size_t, squid_off_t, STRCB *, void *);
+typedef void STOBJWRITE(SwapDir *, storeIOState *, char *, size_t, squid_off_t, FREE *);
typedef void STOBJUNLINK(SwapDir *, StoreEntry *);
typedef void STLOGOPEN(SwapDir *);
@@ -322,7 +343,7 @@
typedef const char *AUTHSCONNLASTHEADER(auth_user_request_t *);
/* append/vprintf's for Packer */
-typedef void (*append_f) (void *, const char *buf, int size);
+typedef void (*append_f) (void *, const char *buf, size_t size);
#if STDC_HEADERS
typedef void (*vprintf_f) (void *, const char *fmt, va_list args);
#else
@@ -336,13 +357,13 @@
typedef int Ctx;
/* in case we want to change it later */
-typedef ssize_t mb_size_t;
+typedef int mb_size_t;
/* iteration for HttpHdrRange */
typedef int HttpHdrRangePos;
/*iteration for headers; use HttpHeaderPos as opaque type, do not interpret */
-typedef ssize_t HttpHeaderPos;
+typedef int HttpHeaderPos;
/* big mask for http headers */
typedef char HttpHeaderMask[8];
Index: squid/src/fs/aufs/async_io.c
diff -u squid/src/fs/aufs/async_io.c:1.7.6.5 squid/src/fs/aufs/async_io.c:1.7.6.4.22.3
--- squid/src/fs/aufs/async_io.c:1.7.6.5 Sat Mar 19 15:11:00 2005
+++ squid/src/fs/aufs/async_io.c Sat Mar 19 15:19:14 2005
@@ -186,7 +186,7 @@
void
-aioWrite(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func)
+aioWrite(int fd, off_t offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func)
{
squidaio_ctrl_t *ctrlp;
int seekmode;
@@ -214,7 +214,7 @@
void
-aioRead(int fd, int offset, int len, AIOCB * callback, void *callback_data)
+aioRead(int fd, off_t offset, int len, AIOCB * callback, void *callback_data)
{
squidaio_ctrl_t *ctrlp;
int seekmode;
Index: squid/src/fs/aufs/store_asyncufs.h
diff -u squid/src/fs/aufs/store_asyncufs.h:1.6.14.4 squid/src/fs/aufs/store_asyncufs.h:1.6.14.2.16.4
--- squid/src/fs/aufs/store_asyncufs.h:1.6.14.4 Sat Mar 19 15:11:01 2005
+++ squid/src/fs/aufs/store_asyncufs.h Sat Mar 19 15:19:14 2005
@@ -64,8 +64,8 @@
void aioCancel(int);
void aioOpen(const char *, int, mode_t, AIOCB *, void *);
void aioClose(int);
-void aioWrite(int, int offset, char *, int size, AIOCB *, void *, FREE *);
-void aioRead(int, int offset, int size, AIOCB *, void *);
+void aioWrite(int, off_t offset, char *, int size, AIOCB *, void *, FREE *);
+void aioRead(int, off_t offset, int size, AIOCB *, void *);
void aioStat(char *, struct stat *, AIOCB *, void *);
void aioUnlink(const char *, AIOCB *, void *);
void aioTruncate(const char *, off_t length, AIOCB *, void *);
Index: squid/src/fs/aufs/store_dir_aufs.c
diff -u squid/src/fs/aufs/store_dir_aufs.c:1.23.6.11 squid/src/fs/aufs/store_dir_aufs.c:1.23.6.9.8.9
--- squid/src/fs/aufs/store_dir_aufs.c:1.23.6.11 Sat Mar 19 15:45:55 2005
+++ squid/src/fs/aufs/store_dir_aufs.c Fri Mar 25 17:52:26 2005
@@ -82,7 +82,7 @@
static int storeAufsDirGetNextFile(RebuildState *, sfileno *, int *size);
static StoreEntry *storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
sfileno file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -366,6 +366,24 @@
}
static void
+storeAufsDirRebuildComplete(RebuildState * rb)
+{
+ if (rb->log) {
+ debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ } else {
+ debug(47, 1) ("Done scanning %s (%d entries)\n",
+ rb->sd->path, rb->counts.scancount);
+ }
+ store_dirs_rebuilding--;
+ storeAufsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+}
+
+static void
storeAufsDirRebuildFromDirectory(void *data)
{
RebuildState *rb = data;
@@ -388,12 +406,7 @@
assert(fd == -1);
fd = storeAufsDirGetNextFile(rb, &filn, &size);
if (fd == -2) {
- debug(47, 1) ("Done scanning %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- store_dirs_rebuilding--;
- storeAufsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
+ storeAufsDirRebuildComplete(rb);
return;
} else if (fd < 0) {
continue;
@@ -445,10 +458,39 @@
assert(t->length == MD5_DIGEST_CHARS);
xmemcpy(key, t->value, MD5_DIGEST_CHARS);
break;
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
case STORE_META_STD:
assert(t->length == STORE_HDR_METASIZE);
xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
break;
+#else
+ case STORE_META_STD_LFS:
+ assert(t->length == STORE_HDR_METASIZE);
+ xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
+ break;
+ case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE_OLD);
+ {
+ struct {
+ time_t timestamp;
+ time_t lastref;
+ time_t expires;
+ time_t lastmod;
+ size_t swap_file_sz;
+ u_short refcount;
+ u_short flags;
+ } *tmp = t->value;
+ assert(sizeof(*tmp) == STORE_HDR_METASIZE_OLD);
+ tmpe.timestamp = tmp->timestamp;
+ tmpe.lastref = tmp->lastref;
+ tmpe.expires = tmp->expires;
+ tmpe.lastmod = tmp->lastmod;
+ tmpe.swap_file_sz = tmp->swap_file_sz;
+ tmpe.refcount = tmp->refcount;
+ tmpe.flags = tmp->flags;
+ }
+ break;
+#endif
default:
break;
}
@@ -522,21 +564,178 @@
/* load a number of objects per invocation */
for (count = 0; count < rb->speed; count++) {
if (fread(&s, ss, 1, rb->log) != 1) {
- debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- fclose(rb->log);
- rb->log = NULL;
- store_dirs_rebuilding--;
- storeAufsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
+ storeAufsDirRebuildComplete(rb);
return;
}
rb->n_read++;
- if (s.op <= SWAP_LOG_NOP)
+ /*
+ * BC: during 2.4 development, we changed the way swap file
+ * numbers are assigned and stored. The high 16 bits used
+ * to encode the SD index number. There used to be a call
+ * to storeDirProperFileno here that re-assigned the index
+ * bits. Now, for backwards compatibility, we just need
+ * to mask it off.
+ */
+ s.swap_filen &= 0x00FFFFFF;
+ debug(47, 3) ("storeAufsDirRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ (void) 0;
+ } else if (s.op == SWAP_LOG_DEL) {
+ /* Delete unless we already have a newer copy */
+ if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeAufsDirReplRemove(e);
+ storeAufsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(47, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFF) == 0) {
+ struct stat sb;
+ if (0 == fstat(fileno(rb->log), &sb))
+ storeRebuildProgress(SD->index,
+ (int) sb.st_size / ss, rb->n_read);
+ }
+ if (!storeAufsDirValidFileno(SD, s.swap_filen, 0)) {
+ rb->counts.invalid++;
+ continue;
+ }
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ used = storeAufsDirMapBitTest(SD, s.swap_filen);
+ /* If this URL already exists in the cache, does the swap log
+ * appear to have a newer entry? Compare 'lastref' from the
+ * swap log to e->lastref. */
+ disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
+ if (used && !disk_entry_newer) {
+ /* log entry is old, ignore it */
+ rb->counts.clashcount++;
+ continue;
+ } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
+ /* swapfile taken, same URL, newer, update meta */
+ if (e->store_status == STORE_OK) {
+ e->lastref = s.timestamp;
+ e->timestamp = s.timestamp;
+ e->expires = s.expires;
+ e->lastmod = s.lastmod;
+ e->flags = s.flags;
+ e->refcount += s.refcount;
+ storeAufsDirUnrefObj(SD, e);
+ } else {
+ debug_trap("storeAufsDirRebuildFromSwapLog: bad condition");
+ debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
+ }
+ continue;
+ } else if (used) {
+ /* swapfile in use, not by this URL, log entry is newer */
+ /* This is sorta bad: the log entry should NOT be newer at this
+ * point. If the log is dirty, the filesize check should have
+ * caught this. If the log is clean, there should never be a
+ * newer entry. */
+ debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
+ SD->index, s.swap_filen);
+ /* I'm tempted to remove the swapfile here just to be safe,
+ * but there is a bad race condition in the NOVM version if
+ * the swapfile has recently been opened for writing, but
+ * not yet opened for reading. Because we can't map
+ * swapfiles back to StoreEntrys, we don't know the state
+ * of the entry using that file. */
+ /* We'll assume the existing entry is valid, probably because
+ * were in a slow rebuild and the the swap file number got taken
+ * and the validation procedure hasn't run. */
+ assert(rb->flags.need_to_validate);
+ rb->counts.clashcount++;
continue;
- if (s.op >= SWAP_LOG_MAX)
+ } else if (e && !disk_entry_newer) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
continue;
+ } else if (e) {
+ /* key already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeAufsDirReplRemove(e);
+ /* Make sure we don't actually unlink the file */
+ storeAufsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.dupcount++;
+ } else {
+ /* URL doesnt exist, swapfile not in use */
+ /* load new */
+ (void) 0;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeAufsDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
+/* This is an exact copy of the above, but using storeSwapLogDataOld entry type */
+static void
+storeAufsDirRebuildFromSwapLogOld(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ StoreEntry *e = NULL;
+ storeSwapLogDataOld s;
+ size_t ss = sizeof(storeSwapLogDataOld);
+ int count;
+ int used; /* is swapfile already in use? */
+ int disk_entry_newer; /* is the log entry newer than current entry? */
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ storeAufsDirRebuildComplete(rb);
+ return;
+ }
+ rb->n_read++;
/*
* BC: during 2.4 development, we changed the way swap file
* numbers are assigned and stored. The high 16 bits used
@@ -680,7 +879,49 @@
(int) rb->flags.clean);
storeDirSwapLog(e, SWAP_LOG_ADD);
}
+ eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1);
+}
+
+#endif
+
+static void
+storeAufsDirRebuildFromSwapLogCheckVersion(void *data)
+{
+ RebuildState *rb = data;
+ storeSwapLogHeader hdr;
+
+ if (fread(&hdr, sizeof(hdr), 1, rb->log) != 1) {
+ storeAufsDirRebuildComplete(rb);
+ return;
+ }
+ if (hdr.op == SWAP_LOG_VERSION) {
+ if (fseek(rb->log, hdr.record_size, SEEK_SET) != 0) {
+ storeAufsDirRebuildComplete(rb);
+ return;
+ }
+ if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogData)) {
+ eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1);
+ return;
+ }
+#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
+ if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogDataOld)) {
+ debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Found current version but without large file support. Upgrading\n");
+ eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1);
+ return;
+ }
+#endif
+ debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Unsupported swap.state version %d size %d\n",
+ hdr.version, hdr.record_size);
+ storeAufsDirRebuildComplete(rb);
+ return;
+ }
+ rewind(rb->log);
+ debug(47, 1) ("storeAufsDirRebuildFromSwapLog: Old version detected. Upgrading\n");
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1);
+#else
+ eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLogOld, rb, 0.0, 1);
+#endif
}
static int
@@ -775,7 +1016,7 @@
static StoreEntry *
storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
sfileno file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -839,7 +1080,7 @@
fclose(fp);
func = storeAufsDirRebuildFromDirectory;
} else {
- func = storeAufsDirRebuildFromSwapLog;
+ func = storeAufsDirRebuildFromSwapLogCheckVersion;
rb->log = fp;
rb->flags.clean = (unsigned int) clean;
}
@@ -890,6 +1131,10 @@
struct stat clean_sb;
FILE *fp;
int fd;
+ union {
+ storeSwapLogHeader hdr;
+ storeSwapLogData _data;
+ } hd;
if (stat(swaplog_path, &log_sb) < 0) {
debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
safe_free(swaplog_path);
@@ -907,6 +1152,14 @@
debug(50, 1) ("%s: %s\n", new_path, xstrerror());
fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
}
+ memset(&hd, 0, sizeof(hd));
+ hd.hdr.op = SWAP_LOG_VERSION;
+ hd.hdr.version = 1;
+ hd.hdr.record_size = sizeof(storeSwapLogData);
+ if (write(fd, &hd, sizeof(hd)) != sizeof(hd)) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to write swap log header.");
+ }
aioinfo->swaplog_fd = fd;
/* open a read-only stream of the old log */
fp = fopen(swaplog_path, "r");
@@ -936,7 +1189,7 @@
char *new;
char *cln;
char *outbuf;
- off_t outbuf_offset;
+ int outbuf_offset;
int fd;
RemovalPolicyWalker *walker;
};
@@ -951,6 +1204,10 @@
storeAufsDirWriteCleanStart(SwapDir * sd)
{
struct _clean_state *state = xcalloc(1, sizeof(*state));
+ union {
+ storeSwapLogHeader hdr;
+ storeSwapLogData _data;
+ } hd;
#if HAVE_FCHMOD
struct stat sb;
#endif
@@ -967,6 +1224,14 @@
xfree(state);
return -1;
}
+ memset(&hd, 0, sizeof(hd));
+ hd.hdr.op = SWAP_LOG_VERSION;
+ hd.hdr.version = 1;
+ hd.hdr.record_size = sizeof(storeSwapLogData);
+ if (write(state->fd, &hd, sizeof(hd)) != sizeof(hd)) {
+ debug(50, 1) ("%s: %s\n", state->new, xstrerror());
+ fatal("storeAufsDirWriteCleanStart: Failed to write swap log header.");
+ }
state->cur = xstrdup(storeAufsDirSwapLogFile(sd, NULL));
state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean"));
state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
Index: squid/src/fs/aufs/store_io_aufs.c
diff -u squid/src/fs/aufs/store_io_aufs.c:1.13.2.12 squid/src/fs/aufs/store_io_aufs.c:1.13.2.11.8.5
--- squid/src/fs/aufs/store_io_aufs.c:1.13.2.12 Fri Mar 18 17:42:57 2005
+++ squid/src/fs/aufs/store_io_aufs.c Fri Mar 25 17:52:26 2005
@@ -153,7 +153,7 @@
/* Read */
void
-storeAufsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+storeAufsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, STRCB * callback, void *callback_data)
{
squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate;
assert(sio->read.callback == NULL);
@@ -167,7 +167,7 @@
q = memPoolAlloc(aufs_qread_pool);
q->buf = buf;
q->size = size;
- q->offset = offset;
+ q->offset = (off_t) offset;
q->callback = callback;
q->callback_data = callback_data;
cbdataLock(q->callback_data);
@@ -183,10 +183,10 @@
sio->offset = offset;
aiostate->flags.reading = 1;
#if ASYNC_READ
- aioRead(aiostate->fd, offset, size, storeAufsReadDone, sio);
+ aioRead(aiostate->fd, (off_t) offset, size, storeAufsReadDone, sio);
statCounter.syscalls.disk.reads++;
#else
- file_read(aiostate->fd, buf, size, offset, storeAufsReadDone, sio);
+ file_read(aiostate->fd, buf, size, (off_t) offset, storeAufsReadDone, sio);
/* file_read() increments syscalls.disk.reads */
#endif
}
@@ -194,7 +194,7 @@
/* Write */
void
-storeAufsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+storeAufsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, FREE * free_func)
{
squidaiostate_t *aiostate = (squidaiostate_t *) sio->fsstate;
debug(79, 3) ("storeAufsWrite: dirno %d, fileno %08X, FD %d\n",
@@ -206,7 +206,7 @@
q = memPoolAlloc(aufs_qwrite_pool);
q->buf = buf;
q->size = size;
- q->offset = offset;
+ q->offset = (off_t) offset;
q->free_func = free_func;
linklistPush(&(aiostate->pending_writes), q);
return;
@@ -218,17 +218,17 @@
q = memPoolAlloc(aufs_qwrite_pool);
q->buf = buf;
q->size = size;
- q->offset = offset;
+ q->offset = (off_t) offset;
q->free_func = free_func;
linklistPush(&(aiostate->pending_writes), q);
return;
}
aiostate->flags.writing = 1;
- aioWrite(aiostate->fd, offset, buf, size, storeAufsWriteDone, sio,
+ aioWrite(aiostate->fd, (off_t) offset, buf, size, storeAufsWriteDone, sio,
free_func);
statCounter.syscalls.disk.writes++;
#else
- file_write(aiostate->fd, offset, buf, size, storeAufsWriteDone, sio,
+ file_write(aiostate->fd, (off_t) offset, buf, size, storeAufsWriteDone, sio,
free_func);
/* file_write() increments syscalls.disk.writes */
#endif
@@ -330,7 +330,7 @@
debug(79, 3) ("storeAufsReadDone: got failure (%d)\n", errflag);
rlen = -1;
} else {
- rlen = (ssize_t) len;
+ rlen = len;
sio->offset += len;
}
#if ASYNC_READ
Index: squid/src/fs/coss/async_io.c
diff -u squid/src/fs/coss/async_io.c:1.3.28.4 squid/src/fs/coss/async_io.c:1.3.28.3.16.2
--- squid/src/fs/coss/async_io.c:1.3.28.4 Thu Mar 10 06:12:06 2005
+++ squid/src/fs/coss/async_io.c Thu Mar 10 06:18:24 2005
@@ -54,7 +54,7 @@
/* Exported routines */
void
-a_file_read(async_queue_t * q, int fd, void *buf, int req_len, off_t offset,
+a_file_read(async_queue_t * q, int fd, void *buf, int req_len, squid_off_t offset,
DRCB * callback, void *data)
{
int slot;
@@ -102,7 +102,7 @@
void
-a_file_write(async_queue_t * q, int fd, off_t offset, void *buf, int len,
+a_file_write(async_queue_t * q, int fd, squid_off_t offset, void *buf, int len,
DWCB * callback, void *data, FREE * freefunc)
{
int slot;
Index: squid/src/fs/coss/async_io.h
diff -u squid/src/fs/coss/async_io.h:1.2 squid/src/fs/coss/async_io.h:1.2.110.1
--- squid/src/fs/coss/async_io.h:1.2 Thu Aug 16 00:39:04 2001
+++ squid/src/fs/coss/async_io.h Wed Mar 2 19:44:27 2005
@@ -48,8 +48,8 @@
/* Functions */
extern void a_file_read(async_queue_t * q, int fd, void *buf, int req_len,
- off_t offset, DRCB * callback, void *data);
-extern void a_file_write(async_queue_t * q, int fd, off_t offset, void *buf,
+ squid_off_t offset, DRCB * callback, void *data);
+extern void a_file_write(async_queue_t * q, int fd, squid_off_t offset, void *buf,
int len, DWCB * callback, void *data, FREE * freefunc);
extern int a_file_callback(async_queue_t * q);
extern void a_file_setupqueue(async_queue_t * q);
Index: squid/src/fs/coss/store_dir_coss.c
diff -u squid/src/fs/coss/store_dir_coss.c:1.17.6.9 squid/src/fs/coss/store_dir_coss.c:1.17.6.9.16.2
--- squid/src/fs/coss/store_dir_coss.c:1.17.6.9 Mon Aug 4 19:14:39 2003
+++ squid/src/fs/coss/store_dir_coss.c Fri Mar 11 17:32:23 2005
@@ -63,7 +63,7 @@
static EVH storeCossRebuildFromSwapLog;
static StoreEntry *storeCossAddDiskRestore(SwapDir * SD, const cache_key * key,
int file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -318,7 +318,7 @@
static StoreEntry *
storeCossAddDiskRestore(SwapDir * SD, const cache_key * key,
int file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -485,7 +485,7 @@
char *new;
char *cln;
char *outbuf;
- off_t outbuf_offset;
+ squid_off_t outbuf_offset;
int fd;
dlink_node *current;
};
@@ -762,7 +762,7 @@
unsigned int i;
unsigned int size;
CossInfo *cs;
- off_t max_offset;
+ squid_off_t max_offset;
i = GetInteger();
size = i << 10; /* Mbytes to Kbytes */
@@ -831,7 +831,7 @@
* largest possible sfileno, assuming sfileno is a 25-bit
* signed integer, as defined in structs.h.
*/
- max_offset = (off_t) 0xFFFFFF << cs->blksz_bits;
+ max_offset = (squid_off_t) 0xFFFFFF << cs->blksz_bits;
if (sd->max_size > (unsigned long) (max_offset >> 10)) {
debug(47, 0) ("COSS block-size = %d bytes\n", 1 << cs->blksz_bits);
debug(47, 0) ("COSS largest file offset = %lu KB\n", (unsigned long) max_offset >> 10);
Index: squid/src/fs/coss/store_io_coss.c
diff -u squid/src/fs/coss/store_io_coss.c:1.9.6.6 squid/src/fs/coss/store_io_coss.c:1.9.6.6.16.3
--- squid/src/fs/coss/store_io_coss.c:1.9.6.6 Tue Jul 29 19:13:44 2003
+++ squid/src/fs/coss/store_io_coss.c Fri Mar 25 17:52:28 2005
@@ -49,8 +49,8 @@
static CossMemBuf *storeCossCreateMemBuf(SwapDir * SD, size_t start,
sfileno curfn, int *collision);
static CBDUNL storeCossIOFreeEntry;
-static off_t storeCossFilenoToDiskOffset(sfileno f, CossInfo *);
-static sfileno storeCossDiskOffsetToFileno(off_t o, CossInfo *);
+static squid_off_t storeCossFilenoToDiskOffset(sfileno f, CossInfo *);
+static sfileno storeCossDiskOffsetToFileno(squid_off_t o, CossInfo *);
static void storeCossMaybeWriteMemBuf(SwapDir * SD, CossMemBuf * t);
static void membuf_describe(CossMemBuf * t, int level, int line);
@@ -72,7 +72,7 @@
{
CossInfo *cs = (CossInfo *) SD->fsdata;
CossMemBuf *newmb;
- off_t retofs;
+ squid_off_t retofs;
size_t allocsize;
int coll = 0;
sfileno checkf;
@@ -95,7 +95,7 @@
assert(which != COSS_ALLOC_NOTIFY);
/* Check if we have overflowed the disk .. */
- if ((cs->current_offset + allocsize) > ((off_t) SD->max_size << 10)) {
+ if ((cs->current_offset + allocsize) > ((squid_off_t) SD->max_size << 10)) {
/*
* tried to allocate past the end of the disk, so wrap
* back to the beginning
@@ -296,7 +296,7 @@
}
void
-storeCossRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+storeCossRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, STRCB * callback, void *callback_data)
{
char *p;
CossState *cstate = (CossState *) sio->fsstate;
@@ -337,11 +337,11 @@
}
void
-storeCossWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+storeCossWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, FREE * free_func)
{
char *dest;
CossMemBuf *membuf;
- off_t diskoffset;
+ squid_off_t diskoffset;
/*
* If we get handed an object with a size of -1,
@@ -450,7 +450,7 @@
CossMemBuf *t = NULL;
dlink_node *m;
CossInfo *cs = (CossInfo *) SD->fsdata;
- off_t o = storeCossFilenoToDiskOffset(s, cs);
+ squid_off_t o = storeCossFilenoToDiskOffset(s, cs);
for (m = cs->membufs.head; m; m = m->next) {
t = m->data;
if ((o >= t->diskstart) && (o < t->diskend))
@@ -547,7 +547,7 @@
if (errflag) {
coss_stats.stripe_write.fail++;
debug(79, 1) ("storeCossWriteMemBufDone: got failure (%d)\n", errflag);
- debug(79, 1) ("FD %d, size=%x\n", fd, t->diskend - t->diskstart);
+ debug(79, 1) ("FD %d, size=%x\n", fd, (int) (t->diskend - t->diskstart));
} else {
coss_stats.stripe_write.success++;
}
@@ -590,7 +590,7 @@
* Kill objects from the tail to make space for a new chunk
*/
for (m = cs->index.tail; m; m = prev) {
- off_t o;
+ squid_off_t o;
prev = m->prev;
e = m->data;
o = storeCossFilenoToDiskOffset(e->swap_filen, cs);
@@ -633,14 +633,14 @@
memPoolFree(coss_state_pool, ((storeIOState *) sio)->fsstate);
}
-static off_t
+static squid_off_t
storeCossFilenoToDiskOffset(sfileno f, CossInfo * cs)
{
- return (off_t) f << cs->blksz_bits;
+ return (squid_off_t) f << cs->blksz_bits;
}
static sfileno
-storeCossDiskOffsetToFileno(off_t o, CossInfo * cs)
+storeCossDiskOffsetToFileno(squid_off_t o, CossInfo * cs)
{
assert(0 == (o & cs->blksz_mask));
return o >> cs->blksz_bits;
Index: squid/src/fs/diskd/diskd.c
diff -u squid/src/fs/diskd/diskd.c:1.6 squid/src/fs/diskd/diskd.c:1.6.122.3
--- squid/src/fs/diskd/diskd.c:1.6 Wed Feb 7 11:11:49 2001
+++ squid/src/fs/diskd/diskd.c Fri Mar 25 17:52:29 2005
@@ -54,7 +54,7 @@
file_state *next;
int id;
int fd;
- off_t offset;
+ squid_off_t offset;
};
static hash_table *hash = NULL;
@@ -135,18 +135,18 @@
}
if (r->offset > -1 && r->offset != fs->offset) {
DEBUG(2)
- fprintf(stderr, "seeking to %d\n", r->offset);
+ fprintf(stderr, "seeking to %" PRINTF_OFF_T "\n", (squid_off_t) r->offset);
if (lseek(fs->fd, r->offset, SEEK_SET) < 0) {
DEBUG(1) {
- fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset);
+ fprintf(stderr, "%d FD %d, offset %" PRINTF_OFF_T ": ", (int) mypid, fs->fd, (squid_off_t) r->offset);
perror("lseek");
}
}
}
x = read(fs->fd, buf, readlen);
DEBUG(2)
- fprintf(stderr, "%d READ %d,%d,%d ret %d\n", (int) mypid,
- fs->fd, readlen, r->offset, x);
+ fprintf(stderr, "%d READ %d,%d,%" PRINTF_OFF_T " ret %d\n", (int) mypid,
+ fs->fd, readlen, (squid_off_t) r->offset, x);
if (x < 0) {
DEBUG(1) {
fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd);
@@ -176,14 +176,14 @@
if (r->offset > -1 && r->offset != fs->offset) {
if (lseek(fs->fd, r->offset, SEEK_SET) < 0) {
DEBUG(1) {
- fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset);
+ fprintf(stderr, "%d FD %d, offset %" PRINTF_OFF_T ": ", (int) mypid, fs->fd, (squid_off_t) r->offset);
perror("lseek");
}
}
}
DEBUG(2)
- fprintf(stderr, "%d WRITE %d,%d,%d\n", (int) mypid,
- fs->fd, wrtlen, r->offset);
+ fprintf(stderr, "%d WRITE %d,%d,%" PRINTF_OFF_T "\n", (int) mypid,
+ fs->fd, wrtlen, (squid_off_t) r->offset);
x = write(fs->fd, buf, wrtlen);
if (x < 0) {
DEBUG(1) {
Index: squid/src/fs/diskd/store_dir_diskd.c
diff -u squid/src/fs/diskd/store_dir_diskd.c:1.32.6.8 squid/src/fs/diskd/store_dir_diskd.c:1.32.6.8.16.8
--- squid/src/fs/diskd/store_dir_diskd.c:1.32.6.8 Wed Jul 23 19:14:09 2003
+++ squid/src/fs/diskd/store_dir_diskd.c Fri Mar 25 17:52:29 2005
@@ -86,7 +86,7 @@
static int storeDiskdDirGetNextFile(RebuildState *, sfileno *, int *size);
static StoreEntry *storeDiskdDirAddDiskRestore(SwapDir * SD, const cache_key * key,
sfileno file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -528,7 +528,7 @@
storeDiskdHandle(&M);
retval = 1; /* Return that we've actually done some work */
if (M.shm_offset > -1)
- storeDiskdShmPut(SD, (off_t) M.shm_offset);
+ storeDiskdShmPut(SD, M.shm_offset);
}
return retval;
}
@@ -536,6 +536,24 @@
static void
+storeDiskdDirRebuildComplete(RebuildState * rb)
+{
+ if (rb->log) {
+ debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ } else {
+ debug(47, 1) ("Done scanning %s (%d entries)\n",
+ rb->sd->path, rb->counts.scancount);
+ }
+ store_dirs_rebuilding--;
+ storeDiskdDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+}
+
+static void
storeDiskdDirRebuildFromDirectory(void *data)
{
RebuildState *rb = data;
@@ -558,12 +576,7 @@
assert(fd == -1);
fd = storeDiskdDirGetNextFile(rb, &filn, &size);
if (fd == -2) {
- debug(20, 1) ("Done scanning %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- store_dirs_rebuilding--;
- storeDiskdDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
+ storeDiskdDirRebuildComplete(rb);
return;
} else if (fd < 0) {
continue;
@@ -615,10 +628,39 @@
assert(t->length == MD5_DIGEST_CHARS);
xmemcpy(key, t->value, MD5_DIGEST_CHARS);
break;
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
case STORE_META_STD:
assert(t->length == STORE_HDR_METASIZE);
xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
break;
+#else
+ case STORE_META_STD_LFS:
+ assert(t->length == STORE_HDR_METASIZE);
+ xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
+ break;
+ case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE_OLD);
+ {
+ struct {
+ time_t timestamp;
+ time_t lastref;
+ time_t expires;
+ time_t lastmod;
+ size_t swap_file_sz;
+ u_short refcount;
+ u_short flags;
+ } *tmp = t->value;
+ assert(sizeof(*tmp) == STORE_HDR_METASIZE_OLD);
+ tmpe.timestamp = tmp->timestamp;
+ tmpe.lastref = tmp->lastref;
+ tmpe.expires = tmp->expires;
+ tmpe.lastmod = tmp->lastmod;
+ tmpe.swap_file_sz = tmp->swap_file_sz;
+ tmpe.refcount = tmp->refcount;
+ tmpe.flags = tmp->flags;
+ }
+ break;
+#endif
default:
break;
}
@@ -692,21 +734,203 @@
/* load a number of objects per invocation */
for (count = 0; count < rb->speed; count++) {
if (fread(&s, ss, 1, rb->log) != 1) {
- debug(20, 1) ("Done reading %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- fclose(rb->log);
- rb->log = NULL;
- store_dirs_rebuilding--;
- storeDiskdDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
+ storeDiskdDirRebuildComplete(rb);
return;
}
rb->n_read++;
- if (s.op <= SWAP_LOG_NOP)
+ /*
+ * BC: during 2.4 development, we changed the way swap file
+ * numbers are assigned and stored. The high 16 bits used
+ * to encode the SD index number. There used to be a call
+ * to storeDirProperFileno here that re-assigned the index
+ * bits. Now, for backwards compatibility, we just need
+ * to mask it off.
+ */
+ s.swap_filen &= 0x00FFFFFF;
+ debug(20, 3) ("storeDiskdDirRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ /*
+ * Here we have some special checks for large files.
+ * I've been seeing a system crash followed by a reboot
+ * that seems to corrupt the swap log. Squid believes
+ * that the disk holds some really large files. It
+ * complains about using being over the high water mark
+ * and proceeds to delete files as fast as it can. To
+ * prevent that, we call stat() on sufficiently large
+ * files (>128KB) and reject those that are missing or
+ * have the wrong size.
+ */
+ struct stat sb;
+ char *p = storeDiskdDirFullPath(SD, s.swap_filen, NULL);
+ if (s.swap_file_sz < (1 << 17)) {
+ (void) 0;
+ } else if (stat(p, &sb) < 0) {
+ debug(47, 2) ("its missing!: %s\n", p);
+ continue;
+ } else if (sb.st_size != s.swap_file_sz) {
+ debug(47, 2) ("size mismatch!: stat=%d, log=%d\n",
+ (int) sb.st_size, (int) s.swap_file_sz);
+ continue;
+ } else {
+ debug(47, 2) ("big file (%d bytes) checks out\n",
+ (int) s.swap_file_sz);
+ }
+ } else if (s.op == SWAP_LOG_DEL) {
+ /* Delete unless we already have a newer copy */
+ if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeDiskdDirReplRemove(e);
+ storeDiskdDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFF) == 0) {
+ struct stat sb;
+ if (0 == fstat(fileno(rb->log), &sb))
+ storeRebuildProgress(SD->index,
+ (int) sb.st_size / ss, rb->n_read);
+ }
+ if (!storeDiskdDirValidFileno(SD, s.swap_filen, 0)) {
+ rb->counts.invalid++;
continue;
- if (s.op >= SWAP_LOG_MAX)
+ }
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ used = storeDiskdDirMapBitTest(SD, s.swap_filen);
+ /* If this URL already exists in the cache, does the swap log
+ * appear to have a newer entry? Compare 'lastref' from the
+ * swap log to e->lastref. */
+ disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
+ if (used && !disk_entry_newer) {
+ /* log entry is old, ignore it */
+ rb->counts.clashcount++;
continue;
+ } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
+ /* swapfile taken, same URL, newer, update meta */
+ if (e->store_status == STORE_OK) {
+ e->lastref = s.timestamp;
+ e->timestamp = s.timestamp;
+ e->expires = s.expires;
+ e->lastmod = s.lastmod;
+ e->flags = s.flags;
+ e->refcount += s.refcount;
+ storeDiskdDirUnrefObj(SD, e);
+ } else {
+ debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition");
+ debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
+ }
+ continue;
+ } else if (used) {
+ /* swapfile in use, not by this URL, log entry is newer */
+ /* This is sorta bad: the log entry should NOT be newer at this
+ * point. If the log is dirty, the filesize check should have
+ * caught this. If the log is clean, there should never be a
+ * newer entry. */
+ debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
+ SD->index, s.swap_filen);
+ /* I'm tempted to remove the swapfile here just to be safe,
+ * but there is a bad race condition in the NOVM version if
+ * the swapfile has recently been opened for writing, but
+ * not yet opened for reading. Because we can't map
+ * swapfiles back to StoreEntrys, we don't know the state
+ * of the entry using that file. */
+ /* We'll assume the existing entry is valid, probably because
+ * were in a slow rebuild and the the swap file number got taken
+ * and the validation procedure hasn't run. */
+ assert(rb->flags.need_to_validate);
+ rb->counts.clashcount++;
+ continue;
+ } else if (e && !disk_entry_newer) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (e) {
+ /* key already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeDiskdDirReplRemove(e);
+ /* Make sure we don't actually unlink the file */
+ storeDiskdDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.dupcount++;
+ } else {
+ /* URL doesnt exist, swapfile not in use */
+ /* load new */
+ (void) 0;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeDiskdDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
+/* This is an exact copy of the above, but using storeSwapLogDataOld entry type */
+static void
+storeDiskdDirRebuildFromSwapLogOld(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ StoreEntry *e = NULL;
+ storeSwapLogDataOld s;
+ size_t ss = sizeof(storeSwapLogDataOld);
+ int count;
+ int used; /* is swapfile already in use? */
+ int disk_entry_newer; /* is the log entry newer than current entry? */
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ storeDiskdDirRebuildComplete(rb);
+ return;
+ }
+ rb->n_read++;
/*
* BC: during 2.4 development, we changed the way swap file
* numbers are assigned and stored. The high 16 bits used
@@ -875,7 +1099,49 @@
(int) rb->flags.clean);
storeDirSwapLog(e, SWAP_LOG_ADD);
}
+ eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLogOld, rb, 0.0, 1);
+}
+
+#endif
+
+static void
+storeDiskdDirRebuildFromSwapLogCheckVersion(void *data)
+{
+ RebuildState *rb = data;
+ storeSwapLogHeader hdr;
+
+ if (fread(&hdr, sizeof(hdr), 1, rb->log) != 1) {
+ storeDiskdDirRebuildComplete(rb);
+ return;
+ }
+ if (hdr.op == SWAP_LOG_VERSION) {
+ if (fseek(rb->log, hdr.record_size, SEEK_SET) != 0) {
+ storeDiskdDirRebuildComplete(rb);
+ return;
+ }
+ if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogData)) {
+ eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
+ return;
+ }
+#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
+ if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogDataOld)) {
+ debug(47, 1) ("storeDiskdDirRebuildFromSwapLog: Found current version but without large file support. Upgrading\n");
+ eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLogOld, rb, 0.0, 1);
+ return;
+ }
+#endif
+ debug(47, 1) ("storeDiskdDirRebuildFromSwapLog: Unsupported swap.state version %d size %d\n",
+ hdr.version, hdr.record_size);
+ storeDiskdDirRebuildComplete(rb);
+ return;
+ }
+ rewind(rb->log);
+ debug(47, 1) ("storeDiskdDirRebuildFromSwapLog: Old version detected. Upgrading\n");
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
+#else
+ eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLogOld, rb, 0.0, 1);
+#endif
}
static int
@@ -970,7 +1236,7 @@
static StoreEntry *
storeDiskdDirAddDiskRestore(SwapDir * SD, const cache_key * key,
int file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -1034,7 +1300,7 @@
fclose(fp);
func = storeDiskdDirRebuildFromDirectory;
} else {
- func = storeDiskdDirRebuildFromSwapLog;
+ func = storeDiskdDirRebuildFromSwapLogCheckVersion;
rb->log = fp;
rb->flags.clean = (unsigned int) clean;
}
@@ -1085,6 +1351,10 @@
struct stat clean_sb;
FILE *fp;
int fd;
+ union {
+ storeSwapLogHeader hdr;
+ storeSwapLogData _data;
+ } hd;
if (stat(swaplog_path, &log_sb) < 0) {
debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
safe_free(swaplog_path);
@@ -1102,6 +1372,14 @@
debug(50, 1) ("%s: %s\n", new_path, xstrerror());
fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
}
+ memset(&hd, 0, sizeof(hd));
+ hd.hdr.op = SWAP_LOG_VERSION;
+ hd.hdr.version = 1;
+ hd.hdr.record_size = sizeof(storeSwapLogData);
+ if (write(fd, &hd, sizeof(hd)) != sizeof(hd)) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to write swap log header.");
+ }
diskdinfo->swaplog_fd = fd;
/* open a read-only stream of the old log */
fp = fopen(swaplog_path, "r");
@@ -1128,7 +1406,7 @@
char *new;
char *cln;
char *outbuf;
- off_t outbuf_offset;
+ int outbuf_offset;
int fd;
RemovalPolicyWalker *walker;
};
@@ -1143,6 +1421,10 @@
storeDiskdDirWriteCleanStart(SwapDir * sd)
{
struct _clean_state *state = xcalloc(1, sizeof(*state));
+ union {
+ storeSwapLogHeader hdr;
+ storeSwapLogData _data;
+ } hd;
#if HAVE_FCHMOD
struct stat sb;
#endif
@@ -1165,6 +1447,14 @@
}
debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
state->new, state->fd);
+ memset(&hd, 0, sizeof(hd));
+ hd.hdr.op = SWAP_LOG_VERSION;
+ hd.hdr.version = 1;
+ hd.hdr.record_size = sizeof(storeSwapLogData);
+ if (write(state->fd, &hd, sizeof(hd)) != sizeof(hd)) {
+ debug(50, 1) ("%s: %s\n", state->new, xstrerror());
+ fatal("storeDiskdDirWriteCleanStart: Failed to write swap log header.");
+ }
#if HAVE_FCHMOD
if (stat(state->cur, &sb) == 0)
fchmod(state->fd, sb.st_mode);
@@ -1636,7 +1926,7 @@
*/
void *
-storeDiskdShmGet(SwapDir * sd, off_t * shm_offset)
+storeDiskdShmGet(SwapDir * sd, int *shm_offset)
{
char *buf = NULL;
diskdinfo_t *diskdinfo = sd->fsdata;
@@ -1659,7 +1949,7 @@
}
void
-storeDiskdShmPut(SwapDir * sd, off_t offset)
+storeDiskdShmPut(SwapDir * sd, int offset)
{
int i;
diskdinfo_t *diskdinfo = sd->fsdata;
Index: squid/src/fs/diskd/store_diskd.h
diff -u squid/src/fs/diskd/store_diskd.h:1.5 squid/src/fs/diskd/store_diskd.h:1.5.64.2
--- squid/src/fs/diskd/store_diskd.h:1.5 Tue Jul 10 21:35:15 2001
+++ squid/src/fs/diskd/store_diskd.h Fri Mar 11 17:32:25 2005
@@ -59,7 +59,7 @@
int seq_no;
void *callback_data;
int size;
- int offset;
+ squid_off_t offset;
int status;
int shm_offset;
} diomsg;
@@ -95,8 +95,8 @@
extern void storeDiskdDirUnlinkFile(SwapDir *, sfileno);
extern void storeDiskdDirReplAdd(SwapDir *, StoreEntry *);
extern void storeDiskdDirReplRemove(StoreEntry *);
-extern void storeDiskdShmPut(SwapDir *, off_t);
-extern void *storeDiskdShmGet(SwapDir *, off_t *);
+extern void storeDiskdShmPut(SwapDir *, int);
+extern void *storeDiskdShmGet(SwapDir *, int *);
extern void storeDiskdHandle(diomsg * M);
extern int storeDiskdDirCallback(SwapDir *);
Index: squid/src/fs/diskd/store_io_diskd.c
diff -u squid/src/fs/diskd/store_io_diskd.c:1.8.22.3 squid/src/fs/diskd/store_io_diskd.c:1.8.22.3.26.3
--- squid/src/fs/diskd/store_io_diskd.c:1.8.22.3 Thu Aug 8 13:18:41 2002
+++ squid/src/fs/diskd/store_io_diskd.c Fri Mar 25 17:52:29 2005
@@ -42,7 +42,7 @@
#include "store_diskd.h"
-static int storeDiskdSend(int, SwapDir *, int, storeIOState *, int, int, off_t);
+static int storeDiskdSend(int, SwapDir *, int, storeIOState *, int, off_t, int);
static void storeDiskdIOCallback(storeIOState * sio, int errflag);
static CBDUNL storeDiskdIOFreeEntry;
@@ -59,7 +59,7 @@
storeIOState *sio;
char *buf;
diskdstate_t *diskdstate;
- off_t shm_offset;
+ int shm_offset;
diskdinfo_t *diskdinfo = SD->fsdata;
debug(79, 3) ("storeDiskdOpen: fileno %08X\n", f);
/*
@@ -115,7 +115,7 @@
int x;
storeIOState *sio;
char *buf;
- off_t shm_offset;
+ int shm_offset;
diskdinfo_t *diskdinfo = SD->fsdata;
diskdstate_t *diskdstate;
/*
@@ -191,10 +191,10 @@
}
void
-storeDiskdRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+storeDiskdRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, STRCB * callback, void *callback_data)
{
int x;
- off_t shm_offset;
+ int shm_offset;
char *rbuf;
diskdstate_t *diskdstate = sio->fsstate;
debug(79, 3) ("storeDiskdRead: dirno %d, fileno %08X\n", sio->swap_dirn, sio->swap_filen);
@@ -219,8 +219,8 @@
SD,
diskdstate->id,
sio,
- (int) size,
- (int) offset,
+ size,
+ (off_t) offset,
shm_offset);
if (x < 0) {
debug(79, 1) ("storeDiskdSend READ: %s\n", xstrerror());
@@ -231,11 +231,11 @@
}
void
-storeDiskdWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+storeDiskdWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, FREE * free_func)
{
int x;
char *sbuf;
- off_t shm_offset;
+ int shm_offset;
diskdstate_t *diskdstate = sio->fsstate;
debug(79, 3) ("storeDiskdWrite: dirno %d, fileno %08X\n", SD->index, sio->swap_filen);
assert(!diskdstate->flags.close_request);
@@ -252,8 +252,8 @@
SD,
diskdstate->id,
sio,
- (int) size,
- (int) offset,
+ size,
+ (off_t) offset,
shm_offset);
if (x < 0) {
debug(79, 1) ("storeDiskdSend WRITE: %s\n", xstrerror());
@@ -267,7 +267,7 @@
storeDiskdUnlink(SwapDir * SD, StoreEntry * e)
{
int x;
- off_t shm_offset;
+ int shm_offset;
char *buf;
diskdinfo_t *diskdinfo = SD->fsdata;
@@ -462,7 +462,7 @@
}
static int
-storeDiskdSend(int mtype, SwapDir * sd, int id, storeIOState * sio, int size, int offset, off_t shm_offset)
+storeDiskdSend(int mtype, SwapDir * sd, int id, storeIOState * sio, int size, off_t offset, int shm_offset)
{
int x;
diomsg M;
Index: squid/src/fs/ufs/store_dir_ufs.c
diff -u squid/src/fs/ufs/store_dir_ufs.c:1.21.6.10 squid/src/fs/ufs/store_dir_ufs.c:1.21.6.10.8.8
--- squid/src/fs/ufs/store_dir_ufs.c:1.21.6.10 Wed Aug 25 19:13:45 2004
+++ squid/src/fs/ufs/store_dir_ufs.c Fri Mar 25 17:52:29 2005
@@ -80,7 +80,7 @@
static int storeUfsDirGetNextFile(RebuildState *, sfileno *, int *size);
static StoreEntry *storeUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
int file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -373,6 +373,24 @@
}
static void
+storeUfsDirRebuildComplete(RebuildState * rb)
+{
+ if (rb->log) {
+ debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ } else {
+ debug(47, 1) ("Done scanning %s (%d entries)\n",
+ rb->sd->path, rb->counts.scancount);
+ }
+ store_dirs_rebuilding--;
+ storeUfsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+}
+
+static void
storeUfsDirRebuildFromDirectory(void *data)
{
RebuildState *rb = data;
@@ -395,12 +413,7 @@
assert(fd == -1);
fd = storeUfsDirGetNextFile(rb, &filn, &size);
if (fd == -2) {
- debug(47, 1) ("Done scanning %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- store_dirs_rebuilding--;
- storeUfsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
+ storeUfsDirRebuildComplete(rb);
return;
} else if (fd < 0) {
continue;
@@ -452,10 +465,39 @@
assert(t->length == MD5_DIGEST_CHARS);
xmemcpy(key, t->value, MD5_DIGEST_CHARS);
break;
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
case STORE_META_STD:
assert(t->length == STORE_HDR_METASIZE);
xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
break;
+#else
+ case STORE_META_STD_LFS:
+ assert(t->length == STORE_HDR_METASIZE);
+ xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
+ break;
+ case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE_OLD);
+ {
+ struct {
+ time_t timestamp;
+ time_t lastref;
+ time_t expires;
+ time_t lastmod;
+ size_t swap_file_sz;
+ u_short refcount;
+ u_short flags;
+ } *tmp = t->value;
+ assert(sizeof(*tmp) == STORE_HDR_METASIZE_OLD);
+ tmpe.timestamp = tmp->timestamp;
+ tmpe.lastref = tmp->lastref;
+ tmpe.expires = tmp->expires;
+ tmpe.lastmod = tmp->lastmod;
+ tmpe.swap_file_sz = tmp->swap_file_sz;
+ tmpe.refcount = tmp->refcount;
+ tmpe.flags = tmp->flags;
+ }
+ break;
+#endif
default:
break;
}
@@ -529,21 +571,178 @@
/* load a number of objects per invocation */
for (count = 0; count < rb->speed; count++) {
if (fread(&s, ss, 1, rb->log) != 1) {
- debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- fclose(rb->log);
- rb->log = NULL;
- store_dirs_rebuilding--;
- storeUfsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
+ storeUfsDirRebuildComplete(rb);
return;
}
rb->n_read++;
- if (s.op <= SWAP_LOG_NOP)
+ /*
+ * BC: during 2.4 development, we changed the way swap file
+ * numbers are assigned and stored. The high 16 bits used
+ * to encode the SD index number. There used to be a call
+ * to storeDirProperFileno here that re-assigned the index
+ * bits. Now, for backwards compatibility, we just need
+ * to mask it off.
+ */
+ s.swap_filen &= 0x00FFFFFF;
+ debug(47, 3) ("storeUfsDirRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ (void) 0;
+ } else if (s.op == SWAP_LOG_DEL) {
+ /* Delete unless we already have a newer copy */
+ if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeUfsDirReplRemove(e);
+ storeUfsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(47, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFF) == 0) {
+ struct stat sb;
+ if (0 == fstat(fileno(rb->log), &sb))
+ storeRebuildProgress(SD->index,
+ (int) sb.st_size / ss, rb->n_read);
+ }
+ if (!storeUfsDirValidFileno(SD, s.swap_filen, 0)) {
+ rb->counts.invalid++;
+ continue;
+ }
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ used = storeUfsDirMapBitTest(SD, s.swap_filen);
+ /* If this URL already exists in the cache, does the swap log
+ * appear to have a newer entry? Compare 'lastref' from the
+ * swap log to e->lastref. */
+ disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
+ if (used && !disk_entry_newer) {
+ /* log entry is old, ignore it */
+ rb->counts.clashcount++;
+ continue;
+ } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
+ /* swapfile taken, same URL, newer, update meta */
+ if (e->store_status == STORE_OK) {
+ e->lastref = s.timestamp;
+ e->timestamp = s.timestamp;
+ e->expires = s.expires;
+ e->lastmod = s.lastmod;
+ e->flags = s.flags;
+ e->refcount += s.refcount;
+ storeUfsDirUnrefObj(SD, e);
+ } else {
+ debug_trap("storeUfsDirRebuildFromSwapLog: bad condition");
+ debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
+ }
+ continue;
+ } else if (used) {
+ /* swapfile in use, not by this URL, log entry is newer */
+ /* This is sorta bad: the log entry should NOT be newer at this
+ * point. If the log is dirty, the filesize check should have
+ * caught this. If the log is clean, there should never be a
+ * newer entry. */
+ debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
+ SD->index, s.swap_filen);
+ /* I'm tempted to remove the swapfile here just to be safe,
+ * but there is a bad race condition in the NOVM version if
+ * the swapfile has recently been opened for writing, but
+ * not yet opened for reading. Because we can't map
+ * swapfiles back to StoreEntrys, we don't know the state
+ * of the entry using that file. */
+ /* We'll assume the existing entry is valid, probably because
+ * were in a slow rebuild and the the swap file number got taken
+ * and the validation procedure hasn't run. */
+ assert(rb->flags.need_to_validate);
+ rb->counts.clashcount++;
continue;
- if (s.op >= SWAP_LOG_MAX)
+ } else if (e && !disk_entry_newer) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
continue;
+ } else if (e) {
+ /* key already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ storeUfsDirReplRemove(e);
+ /* Make sure we don't actually unlink the file */
+ storeUfsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.dupcount++;
+ } else {
+ /* URL doesnt exist, swapfile not in use */
+ /* load new */
+ (void) 0;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeUfsDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
+/* This is an exact copy of the above, but using storeSwapLogDataOld entry type */
+static void
+storeUfsDirRebuildFromSwapLogOld(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ StoreEntry *e = NULL;
+ storeSwapLogDataOld s;
+ size_t ss = sizeof(storeSwapLogDataOld);
+ int count;
+ int used; /* is swapfile already in use? */
+ int disk_entry_newer; /* is the log entry newer than current entry? */
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ storeUfsDirRebuildComplete(rb);
+ return;
+ }
+ rb->n_read++;
/*
* BC: during 2.4 development, we changed the way swap file
* numbers are assigned and stored. The high 16 bits used
@@ -687,7 +886,49 @@
(int) rb->flags.clean);
storeDirSwapLog(e, SWAP_LOG_ADD);
}
+ eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLogOld, rb, 0.0, 1);
+}
+
+#endif
+
+static void
+storeUfsDirRebuildFromSwapLogCheckVersion(void *data)
+{
+ RebuildState *rb = data;
+ storeSwapLogHeader hdr;
+
+ if (fread(&hdr, sizeof(hdr), 1, rb->log) != 1) {
+ storeUfsDirRebuildComplete(rb);
+ return;
+ }
+ if (hdr.op == SWAP_LOG_VERSION) {
+ if (fseek(rb->log, hdr.record_size, SEEK_SET) != 0) {
+ storeUfsDirRebuildComplete(rb);
+ return;
+ }
+ if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogData)) {
+ eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLog, rb, 0.0, 1);
+ return;
+ }
+#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
+ if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogDataOld)) {
+ debug(47, 1) ("storeUfsDirRebuildFromSwapLog: Found current version but without large file support. Upgrading\n");
+ eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLogOld, rb, 0.0, 1);
+ return;
+ }
+#endif
+ debug(47, 1) ("storeUfsDirRebuildFromSwapLog: Unsupported swap.state version %d size %d\n",
+ hdr.version, hdr.record_size);
+ storeUfsDirRebuildComplete(rb);
+ return;
+ }
+ rewind(rb->log);
+ debug(47, 1) ("storeUfsDirRebuildFromSwapLog: Old version detected. Upgrading\n");
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLog, rb, 0.0, 1);
+#else
+ eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLogOld, rb, 0.0, 1);
+#endif
}
static int
@@ -782,7 +1023,7 @@
static StoreEntry *
storeUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
int file_number,
- size_t swap_file_sz,
+ squid_file_sz swap_file_sz,
time_t expires,
time_t timestamp,
time_t lastref,
@@ -845,7 +1086,7 @@
fclose(fp);
func = storeUfsDirRebuildFromDirectory;
} else {
- func = storeUfsDirRebuildFromSwapLog;
+ func = storeUfsDirRebuildFromSwapLogCheckVersion;
rb->log = fp;
rb->flags.clean = (unsigned int) clean;
}
@@ -896,6 +1137,10 @@
struct stat clean_sb;
FILE *fp;
int fd;
+ union {
+ storeSwapLogHeader hdr;
+ storeSwapLogData _data;
+ } hd;
if (stat(swaplog_path, &log_sb) < 0) {
debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
safe_free(swaplog_path);
@@ -913,6 +1158,14 @@
debug(50, 1) ("%s: %s\n", new_path, xstrerror());
fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
}
+ memset(&hd, 0, sizeof(hd));
+ hd.hdr.op = SWAP_LOG_VERSION;
+ hd.hdr.version = 1;
+ hd.hdr.record_size = sizeof(storeSwapLogData);
+ if (write(fd, &hd, sizeof(hd)) != sizeof(hd)) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to write swap log header.");
+ }
ufsinfo->swaplog_fd = fd;
/* open a read-only stream of the old log */
fp = fopen(swaplog_path, "r");
@@ -942,7 +1195,7 @@
char *new;
char *cln;
char *outbuf;
- off_t outbuf_offset;
+ int outbuf_offset;
int fd;
RemovalPolicyWalker *walker;
};
@@ -957,6 +1210,10 @@
storeUfsDirWriteCleanStart(SwapDir * sd)
{
struct _clean_state *state = xcalloc(1, sizeof(*state));
+ union {
+ storeSwapLogHeader hdr;
+ storeSwapLogData _data;
+ } hd;
#if HAVE_FCHMOD
struct stat sb;
#endif
@@ -973,6 +1230,14 @@
xfree(state);
return -1;
}
+ memset(&hd, 0, sizeof(hd));
+ hd.hdr.op = SWAP_LOG_VERSION;
+ hd.hdr.version = 1;
+ hd.hdr.record_size = sizeof(storeSwapLogData);
+ if (write(state->fd, &hd, sizeof(hd)) != sizeof(hd)) {
+ debug(50, 1) ("%s: %s\n", state->new, xstrerror());
+ fatal("storeUfsDirWriteCleanStart: Failed to write swap log header.");
+ }
state->cur = xstrdup(storeUfsDirSwapLogFile(sd, NULL));
state->cln = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean"));
state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
Index: squid/src/fs/ufs/store_io_ufs.c
diff -u squid/src/fs/ufs/store_io_ufs.c:1.7.6.3 squid/src/fs/ufs/store_io_ufs.c:1.7.6.3.8.4
--- squid/src/fs/ufs/store_io_ufs.c:1.7.6.3 Sat Jul 17 19:13:13 2004
+++ squid/src/fs/ufs/store_io_ufs.c Fri Mar 25 17:52:29 2005
@@ -150,7 +150,7 @@
}
void
-storeUfsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+storeUfsRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, STRCB * callback, void *callback_data)
{
ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate;
@@ -166,19 +166,19 @@
file_read(ufsstate->fd,
buf,
size,
- offset,
+ (off_t) offset,
storeUfsReadDone,
sio);
}
void
-storeUfsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+storeUfsWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, squid_off_t offset, FREE * free_func)
{
ufsstate_t *ufsstate = (ufsstate_t *) sio->fsstate;
debug(79, 3) ("storeUfsWrite: dirn %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, ufsstate->fd);
ufsstate->flags.writing = 1;
file_write(ufsstate->fd,
- offset,
+ (off_t) offset,
buf,
size,
storeUfsWriteDone,
@@ -213,7 +213,7 @@
debug(79, 3) ("storeUfsReadDone: got failure (%d)\n", errflag);
rlen = -1;
} else {
- rlen = (ssize_t) len;
+ rlen = len;
sio->offset += len;
}
assert(callback);