This patch is generated from the modio branch of HEAD in squid Sun Jan 25 14:36:32 2004 GMT See http://devel.squid-cache.org/ Index: squid/include/version.h diff -u squid/include/version.h:1.4 squid/include/version.h:1.2.2.2 --- squid/include/version.h:1.4 Tue Nov 14 05:03:47 2000 +++ squid/include/version.h Wed Dec 20 07:27:39 2000 @@ -4,7 +4,7 @@ * SQUID_VERSION - String for version id of this distribution */ #ifndef SQUID_VERSION -#define SQUID_VERSION "2.5.DEVEL" +#define SQUID_VERSION "2.5.DEVEL-modio" #endif #ifndef SQUID_RELEASE_TIME Index: squid/src/HttpHdrRange.c diff -u squid/src/HttpHdrRange.c:1.6 squid/src/HttpHdrRange.c:1.2.2.4 --- squid/src/HttpHdrRange.c:1.6 Wed Feb 7 11:11:47 2001 +++ squid/src/HttpHdrRange.c Tue Feb 13 12:54:28 2001 @@ -467,7 +467,7 @@ assert(http); stringAppend(&b, full_appname_string, strlen(full_appname_string)); stringAppend(&b, ":", 1); - key = storeKeyText(http->entry->hash.key); + key = storeKeyPublic(storeKeyUrl(http->entry), METHOD_GET); stringAppend(&b, key, strlen(key)); return b; } Index: squid/src/HttpRequest.c diff -u squid/src/HttpRequest.c:1.6 squid/src/HttpRequest.c:1.2.2.5 --- squid/src/HttpRequest.c:1.6 Fri Jan 12 00:20:32 2001 +++ squid/src/HttpRequest.c Sun Mar 18 14:51:21 2001 @@ -35,10 +35,24 @@ #include "squid.h" +CBDATA_TYPE(request_t); + +static int request_initialised = 0; + +void +requestInit(void) +{ + if (request_initialised) + return; + + request_initialised = 1; + CBDATA_INIT_TYPE(request_t); +} + request_t * requestCreate(method_t method, protocol_t protocol, const char *urlpath) { - request_t *req = memAllocate(MEM_REQUEST_T); + request_t *req = cbdataAlloc(request_t); req->method = method; req->protocol = protocol; if (urlpath) @@ -51,6 +65,7 @@ return req; } + void requestDestroy(request_t * req) { @@ -64,9 +79,7 @@ httpHeaderClean(&req->header); if (req->cache_control) httpHdrCcDestroy(req->cache_control); - if (req->range) - httpHdrRangeDestroy(req->range); - memFree(req, MEM_REQUEST_T); + cbdataFree(req); } request_t * Index: squid/src/Makefile.in diff -u squid/src/Makefile.in:1.7 squid/src/Makefile.in:1.2.2.6 --- squid/src/Makefile.in:1.7 Sat Feb 10 08:49:03 2001 +++ squid/src/Makefile.in Tue Feb 13 12:54:29 2001 @@ -183,6 +183,8 @@ wais.o \ wccp.o \ whois.o \ + reply_internal.o \ + reply_network.o \ $(XTRA_OBJS) SNMP_OBJS = \ Index: squid/src/asn.c diff -u squid/src/asn.c:1.10 squid/src/asn.c:1.2.2.12 --- squid/src/asn.c:1.10 Sat Mar 3 02:44:31 2001 +++ squid/src/asn.c Sun Mar 18 14:33:31 2001 @@ -195,18 +195,19 @@ assert(NULL != req); asState->request = requestLink(req); if ((e = storeGetPublic(asres, METHOD_GET)) == NULL) { - e = storeCreateEntry(asres, asres, null_request_flags, METHOD_GET); - asState->sc = storeClientListAdd(e, asState); + /* I'm guessing that this should be private .. */ + e = storeCreateEntry(asres, asres, null_request_flags, METHOD_GET, + REPLY_OBJ_INTERNAL); + storeClientRegister(e, asState); fwdStart(-1, e, asState->request); } else { storeLockObject(e); - asState->sc = storeClientListAdd(e, asState); + storeClientRegister(e, asState); } asState->entry = e; asState->seen = 0; asState->offset = 0; - storeClientCopy(asState->sc, - e, + storeClientCopy(e, asState->seen, asState->offset, 4096, @@ -228,7 +229,7 @@ asStateFree(asState); return; } - if (size == 0 && e->mem_obj->inmem_hi > 0) { + if (size == 0 && storeMemHiOffset(e) > 0) { memFree(buf, MEM_4K_BUF); asStateFree(asState); return; @@ -267,18 +268,16 @@ asState->seen, asState->offset); if (e->store_status == STORE_PENDING) { debug(53, 3) ("asHandleReply: store_status == STORE_PENDING: %s\n", storeUrl(e)); - storeClientCopy(asState->sc, - e, + storeClientCopy(e, asState->seen, asState->offset, 4096, buf, asHandleReply, asState); - } else if (asState->seen < e->mem_obj->inmem_hi) { - debug(53, 3) ("asHandleReply: asState->seen < e->mem_obj->inmem_hi %s\n", storeUrl(e)); - storeClientCopy(asState->sc, - e, + } else if (asState->seen < storeMemHiOffset(e)) { + debug(53, 3) ("asHandleReply: asState->seen < storeMemHiOffset(e) %s\n", storeUrl(e)); + storeClientCopy(e, asState->seen, asState->offset, 4096, @@ -297,7 +296,7 @@ { ASState *asState = data; debug(53, 3) ("asnStateFree: %s\n", storeUrl(asState->entry)); - storeUnregister(asState->sc, asState->entry, asState); + storeClientUnregister(asState->entry, asState); storeUnlockObject(asState->entry); requestUnlink(asState->request); cbdataFree(asState); Index: squid/src/cbdata.c diff -u squid/src/cbdata.c:1.10 squid/src/cbdata.c:1.2.2.6 --- squid/src/cbdata.c:1.10 Sat Mar 3 02:44:31 2001 +++ squid/src/cbdata.c Sun Mar 18 14:51:21 2001 @@ -134,7 +134,6 @@ #define CREATE_CBDATA_FREE(type, free_func) cbdataInitType(CBDATA_##type, #type, sizeof(type), free_func) CREATE_CBDATA(acl_access); CREATE_CBDATA(aclCheck_t); - CREATE_CBDATA(clientHttpRequest); CREATE_CBDATA(ConnStateData); CREATE_CBDATA(ErrorState); CREATE_CBDATA(FwdState); Index: squid/src/client_side.c diff -u squid/src/client_side.c:1.24 squid/src/client_side.c:1.2.2.25 --- squid/src/client_side.c:1.24 Sun Mar 4 05:45:04 2001 +++ squid/src/client_side.c Sun Mar 18 14:51:21 2001 @@ -78,7 +78,6 @@ /* Local functions */ static CWCB clientWriteComplete; -static CWCB clientWriteBodyComplete; static PF clientReadRequest; static PF connStateFree; static PF requestTimeout; @@ -103,8 +102,6 @@ static STCB clientSendMoreData; static STCB clientCacheHit; static void clientSetKeepaliveFlag(clientHttpRequest *); -static void clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb); -static void clientPackTermBound(String boundary, MemBuf * mb); static void clientInterpretRequestHeaders(clientHttpRequest *); static void clientProcessRequest(clientHttpRequest *); static void clientProcessExpired(void *data); @@ -113,11 +110,27 @@ static int clientHierarchical(clientHttpRequest * http); static int clientCheckContentLength(request_t * r); static DEFER httpAcceptDefer; -static log_type clientProcessRequest2(clientHttpRequest * http); +static log_type clientProcessRequest2(clientHttpRequest * http, StoreEntry *e); static int clientReplyBodyTooLarge(int clen); static int clientRequestBodyTooLarge(int clen); +static void httpMemFree(void *); +static STGETDONE clientPurgeRequest; +static STGETDONE clientLookupDone; +static STGETDONE clientLookupHeadDone; +static void clientProcessRequestDone(clientHttpRequest *http); static void clientProcessBody(ConnStateData * conn); + +CBDATA_TYPE(clientHttpRequest); + +static void httpMemFree(void *data) +{ + clientHttpRequest *h = data; + + memFree(h->buf, MEM_CLIENT_SOCK_BUF); +} + + static int checkAccelOnly(clientHttpRequest * http) { @@ -192,7 +205,8 @@ } StoreEntry * -clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags) +clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags, + reply_obj_t reply_obj) { StoreEntry *e; /* @@ -201,13 +215,13 @@ */ if (h->request == NULL) h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); - e = storeCreateEntry(h->uri, h->log_uri, flags, m); - h->sc = storeClientListAdd(e, h); + e = storeCreateEntry(h->uri, h->log_uri, flags, m, reply_obj); + storeClientRegister(e, h); #if DELAY_POOLS delaySetStoreClient(h->sc, delayClient(h->request)); #endif - storeClientCopy(h->sc, e, 0, 0, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, h); + h->bufofs = 0; + storeClientCopy(e, 0, 0, CLIENT_SOCK_SZ, h->buf, clientSendMoreData, h); return e; } @@ -246,7 +260,7 @@ page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName); http->log_type = LOG_TCP_DENIED; http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); + null_request_flags, REPLY_OBJ_INTERNAL); if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) { if (!http->flags.accel) { /* Proxy authorisation needed */ @@ -367,19 +381,19 @@ } http->request->flags.refresh = 1; http->old_entry = http->entry; - http->old_sc = http->sc; /* * Assert that 'http' is already a client of old_entry. If * it is not, then the beginning of the object data might get * freed from memory before we need to access it. */ - assert(http->sc->callback_data == http); + assert(http->entry->sc.callback_data == http); entry = storeCreateEntry(url, http->log_uri, http->request->flags, - http->request->method); + http->request->method, + REPLY_OBJ_NETWORK); /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ - http->sc = storeClientListAdd(entry, http); + storeClientRegister(entry, http); #if DELAY_POOLS /* delay_id is already set on original store client */ delaySetStoreClient(http->sc, delayClient(http->request)); @@ -392,11 +406,12 @@ /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientHandleIMSReply, http); } @@ -452,11 +467,9 @@ http_status status; debug(33, 3) ("clientHandleIMSReply: %s, %d bytes\n", url, (int) size); if (entry == NULL) { - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } mem = entry->mem_obj; @@ -466,10 +479,9 @@ /* We have an existing entry, but failed to validate it */ /* Its okay to send the old one anyway */ http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeUnregister(http->sc, entry, http); + storeClientUnregister(entry, http); storeUnlockObject(entry); entry = http->entry = http->old_entry; - http->sc = http->old_sc; } else if (STORE_PENDING == entry->store_status && 0 == status) { debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); if (size >= CLIENT_SOCK_SZ) { @@ -477,17 +489,18 @@ debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); /* use old entry, this repeats the code abovez */ http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeUnregister(http->sc, entry, http); + storeClientUnregister(entry, http); storeUnlockObject(entry); entry = http->entry = http->old_entry; - http->sc = http->old_sc; /* continue */ } else { - storeClientCopy(http->sc, entry, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->bufofs += size; + assert(http->bufofs <= CLIENT_SOCK_SZ); + storeClientCopy(entry, + http->out.offset + http->bufofs, + http->out.offset + http->bufofs, + CLIENT_SOCK_SZ - http->bufofs, + buf + http->bufofs, clientHandleIMSReply, http); return; @@ -508,8 +521,7 @@ * not the body they refer to. */ httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); storeTimestampsSet(oldentry); - storeUnregister(http->sc, entry, http); - http->sc = http->old_sc; + storeClientUnregister(entry, http); storeUnlockObject(entry); entry = http->entry = oldentry; entry->timestamp = squid_curtime; @@ -526,15 +538,15 @@ storeTimestampsSet(http->old_entry); http->log_type = LOG_TCP_REFRESH_HIT; } - storeUnregister(http->old_sc, http->old_entry, http); + storeClientUnregister(http->old_entry, http); storeUnlockObject(http->old_entry); recopy = 0; } http->old_entry = NULL; /* done with old_entry */ - http->old_sc = NULL; assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); if (recopy) { - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, @@ -581,27 +593,38 @@ } void -clientPurgeRequest(clientHttpRequest * http) +clientStartPurgeRequest(clientHttpRequest * http) { - StoreEntry *entry; ErrorState *err = NULL; - HttpReply *r; - http_status status; - http_version_t version; + debug(33, 3) ("Config2.onoff.enable_purge = %d\n", Config2.onoff.enable_purge); if (!Config2.onoff.enable_purge) { http->log_type = LOG_TCP_DENIED; err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); err->request = requestLink(http->request); err->src_addr = http->conn->peer.sin_addr; - http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); return; } http->log_type = LOG_TCP_MISS; /* Release both IP and object cache entries */ ipcacheInvalidate(http->request->host); - if ((entry = storeGetPublic(http->uri, METHOD_GET)) == NULL) { + /* Initiate a lookup to try and get said object */ + storeDoubleCheckRequest(http->request, http->uri, METHOD_GET); + storeLookup(http->request, clientPurgeRequest, http); +} + +static void +clientPurgeRequest(void *data, StoreEntry *entry) +{ + clientHttpRequest *http = data; + HttpReply *r; + http_status status; + http_version_t version; + + if (entry == NULL) { status = HTTP_NOT_FOUND; } else { storeRelease(entry); @@ -613,7 +636,8 @@ * Make a new entry to hold the reply to be written * to the client. */ - http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); httpReplyReset(r = http->entry->mem_obj->reply); httpBuildVersion(&version, 1, 0); httpReplySetHeaders(r, version, status, NULL, NULL, 0, 0, -1); @@ -718,7 +742,7 @@ */ if ((e = http->entry)) { http->entry = NULL; - storeUnregister(http->sc, e, http); + storeClientUnregister(e, http); storeUnlockObject(e); } #endif @@ -773,19 +797,16 @@ safe_free(http->al.headers.request); safe_free(http->al.headers.reply); safe_free(http->redirect.location); - stringClean(&http->range_iter.boundary); if ((e = http->entry)) { http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); } /* old_entry might still be set if we didn't yet get the reply * code in clientHandleIMSReply() */ if ((e = http->old_entry)) { http->old_entry = NULL; - storeUnregister(http->old_sc, e, http); - http->old_sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); } requestUnlink(http->request); @@ -888,12 +909,15 @@ #endif request->flags.nocache = 1; } + /* ignore range header in non-GETs */ if (request->method == METHOD_GET) { - request->range = httpHeaderGetRange(req_hdr); - if (request->range) - request->flags.range = 1; + if (httpHeaderHas(req_hdr, HDR_RANGE)) + request->flags.range = 1; + if (httpHeaderHas(req_hdr, HDR_REQUEST_RANGE)) + request->flags.range = 1; } + if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) request->flags.auth = 1; if (request->login[0] != '\0') @@ -1069,149 +1093,6 @@ return 0; } -/* - * returns true if If-Range specs match reply, false otherwise - */ -static int -clientIfRangeMatch(clientHttpRequest * http, HttpReply * rep) -{ - const TimeOrTag spec = httpHeaderGetTimeOrTag(&http->request->header, HDR_IF_RANGE); - /* check for parsing falure */ - if (!spec.valid) - return 0; - /* got an ETag? */ - if (spec.tag.str) { - ETag rep_tag = httpHeaderGetETag(&rep->header, HDR_ETAG); - debug(33, 3) ("clientIfRangeMatch: ETags: %s and %s\n", - spec.tag.str, rep_tag.str ? rep_tag.str : ""); - if (!rep_tag.str) - return 0; /* entity has no etag to compare with! */ - if (spec.tag.weak || rep_tag.weak) { - debug(33, 1) ("clientIfRangeMatch: Weak ETags are not allowed in If-Range: %s ? %s\n", - spec.tag.str, rep_tag.str); - return 0; /* must use strong validator for sub-range requests */ - } - return etagIsEqual(&rep_tag, &spec.tag); - } - /* got modification time? */ - if (spec.time >= 0) { - return http->entry->lastmod <= spec.time; - } - assert(0); /* should not happen */ - return 0; -} - -/* returns expected content length for multi-range replies - * note: assumes that httpHdrRangeCanonize has already been called - * warning: assumes that HTTP headers for individual ranges at the - * time of the actuall assembly will be exactly the same as - * the headers when clientMRangeCLen() is called */ -static int -clientMRangeCLen(clientHttpRequest * http) -{ - int clen = 0; - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec; - MemBuf mb; - - assert(http->entry->mem_obj); - - memBufDefInit(&mb); - while ((spec = httpHdrRangeGetSpec(http->request->range, &pos))) { - - /* account for headers for this range */ - memBufReset(&mb); - clientPackRangeHdr(http->entry->mem_obj->reply, - spec, http->range_iter.boundary, &mb); - clen += mb.size; - - /* account for range content */ - clen += spec->length; - - debug(33, 6) ("clientMRangeCLen: (clen += %d + %d) == %d\n", - mb.size, spec->length, clen); - } - /* account for the terminating boundary */ - memBufReset(&mb); - clientPackTermBound(http->range_iter.boundary, &mb); - clen += mb.size; - - memBufClean(&mb); - return clen; -} - -/* adds appropriate Range headers if needed */ -static void -clientBuildRangeHeader(clientHttpRequest * http, HttpReply * rep) -{ - HttpHeader *hdr = rep ? &rep->header : 0; - const char *range_err = NULL; - request_t *request = http->request; - assert(request->range); - /* check if we still want to do ranges */ - if (!rep) - range_err = "no [parse-able] reply"; - else if (rep->sline.status != HTTP_OK) - range_err = "wrong status code"; - else if (httpHeaderHas(hdr, HDR_CONTENT_RANGE)) - range_err = "origin server does ranges"; - else if (rep->content_length < 0) - range_err = "unknown length"; - else if (rep->content_length != http->entry->mem_obj->reply->content_length) - range_err = "INCONSISTENT length"; /* a bug? */ - else if (httpHeaderHas(&http->request->header, HDR_IF_RANGE) && !clientIfRangeMatch(http, rep)) - range_err = "If-Range match failed"; - else if (!httpHdrRangeCanonize(http->request->range, rep->content_length)) - range_err = "canonization failed"; - else if (httpHdrRangeIsComplex(http->request->range)) - range_err = "too complex range header"; - else if (!request->flags.cachable) /* from we_do_ranges in http.c */ - range_err = "non-cachable request"; - /* get rid of our range specs on error */ - if (range_err) { - debug(33, 3) ("clientBuildRangeHeader: will not do ranges: %s.\n", range_err); - httpHdrRangeDestroy(http->request->range); - http->request->range = NULL; - } else { - const int spec_count = http->request->range->specs.count; - int actual_clen = -1; - - debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %d\n", - spec_count, rep->content_length); - assert(spec_count > 0); - /* ETags should not be returned with Partial Content replies? */ - httpHeaderDelById(hdr, HDR_ETAG); - /* append appropriate header(s) */ - if (spec_count == 1) { - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec = httpHdrRangeGetSpec(http->request->range, &pos); - assert(spec); - /* append Content-Range */ - httpHeaderAddContRange(hdr, *spec, rep->content_length); - /* set new Content-Length to the actual number of bytes - * transmitted in the message-body */ - actual_clen = spec->length; - } else { - /* multipart! */ - /* generate boundary string */ - http->range_iter.boundary = httpHdrRangeBoundaryStr(http); - /* delete old Content-Type, add ours */ - httpHeaderDelById(hdr, HDR_CONTENT_TYPE); - httpHeaderPutStrf(hdr, HDR_CONTENT_TYPE, - "multipart/byteranges; boundary=\"%s\"", - strBuf(http->range_iter.boundary)); - /* Content-Length is not required in multipart responses - * but it is always nice to have one */ - actual_clen = clientMRangeCLen(http); - } - - /* replace Content-Length header */ - assert(actual_clen >= 0); - httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); - httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, actual_clen); - debug(33, 3) ("clientBuildRangeHeader: actual content length: %d\n", actual_clen); - } -} /* * filters out unwanted entries from original reply header @@ -1256,9 +1137,6 @@ httpHeaderDelById(hdr, HDR_CONNECTION); stringClean(&strConnection); } - /* Handle Ranges */ - if (request->range) - clientBuildRangeHeader(http, rep); /* * Add a estimated Age header on cache hits. */ @@ -1331,19 +1209,15 @@ httpBuildVersion(&rep->sline.version, 1, 0); /* do header conversions */ clientBuildReplyHeader(http, rep); + /* if we do ranges, change status to "Partial Content" */ - if (http->request->range) + if (http->request->flags.range) httpStatusLineSet(&rep->sline, rep->sline.version, HTTP_PARTIAL_CONTENT, NULL); } else { /* parsing failure, get rid of the invalid reply */ httpReplyDestroy(rep); rep = NULL; - /* if we were going to do ranges, backoff */ - if (http->request->range) { - /* this will fail and destroy request->range */ - clientBuildRangeHeader(http, rep); - } } return rep; } @@ -1364,18 +1238,15 @@ request_t *r = http->request; debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); if (http->entry == NULL) { - memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: request aborted\n"); return; } else if (size < 0) { /* swap in failure */ - memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); http->log_type = LOG_TCP_SWAPFAIL_MISS; if ((e = http->entry)) { http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); } clientProcessMiss(http); @@ -1390,18 +1261,18 @@ * punt to clientProcessMiss. */ if (e->mem_status == IN_MEMORY || e->store_status == STORE_OK) { - memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); } else if (size == CLIENT_SOCK_SZ && http->out.offset == 0) { - memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); } else { debug(33, 3) ("clientCacheHit: waiting for HTTP reply headers\n"); - storeClientCopy(http->sc, e, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->bufofs += size; + assert(http->bufofs <= CLIENT_SOCK_SZ); + storeClientCopy(e, + http->out.offset + http->bufofs, + http->out.offset + http->bufofs, + CLIENT_SOCK_SZ - http->bufofs, + buf + http->bufofs, clientCacheHit, http); } @@ -1465,7 +1336,6 @@ http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } - memFree(buf, MEM_CLIENT_SOCK_BUF); } else if (r->flags.ims) { /* * Handle If-Modified-Since requests from the client @@ -1473,7 +1343,6 @@ if (mem->reply->sline.status != HTTP_OK) { debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", mem->reply->sline.status); - memFree(buf, MEM_CLIENT_SOCK_BUF); http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } else if (modifiedSince(e, http->request)) { @@ -1483,11 +1352,10 @@ time_t timestamp = e->timestamp; MemBuf mb = httpPacked304Reply(e->mem_obj->reply); http->log_type = LOG_TCP_IMS_HIT; - memFree(buf, MEM_CLIENT_SOCK_BUF); - storeUnregister(http->sc, e, http); - http->sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); - e = clientCreateStoreEntry(http, http->request->method, null_request_flags); + e = clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); /* * Copy timestamp from the original entry so the 304 * reply has a meaningful Age: header. @@ -1511,168 +1379,6 @@ } } -/* put terminating boundary for multiparts */ -static void -clientPackTermBound(String boundary, MemBuf * mb) -{ - memBufPrintf(mb, "\r\n--%s--\r\n", strBuf(boundary)); - debug(33, 6) ("clientPackTermBound: buf offset: %d\n", mb->size); -} - -/* appends a "part" HTTP header (as in a multi-part/range reply) to the buffer */ -static void -clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb) -{ - HttpHeader hdr; - Packer p; - assert(rep); - assert(spec); - - /* put boundary */ - debug(33, 5) ("clientPackRangeHdr: appending boundary: %s\n", strBuf(boundary)); - /* rfc2046 requires to _prepend_ boundary with ! */ - memBufPrintf(mb, "\r\n--%s\r\n", strBuf(boundary)); - - /* stuff the header with required entries and pack it */ - httpHeaderInit(&hdr, hoReply); - if (httpHeaderHas(&rep->header, HDR_CONTENT_TYPE)) - httpHeaderPutStr(&hdr, HDR_CONTENT_TYPE, httpHeaderGetStr(&rep->header, HDR_CONTENT_TYPE)); - httpHeaderAddContRange(&hdr, *spec, rep->content_length); - packerToMemInit(&p, mb); - httpHeaderPackInto(&hdr, &p); - packerClean(&p); - httpHeaderClean(&hdr); - - /* append (we packed a header, not a reply) */ - memBufPrintf(mb, crlf); -} - -/* - * extracts a "range" from *buf and appends them to mb, updating - * all offsets and such. - */ -static void -clientPackRange(clientHttpRequest * http, - HttpHdrRangeIter * i, - const char **buf, - ssize_t * size, - MemBuf * mb) -{ - const ssize_t copy_sz = i->debt_size <= *size ? i->debt_size : *size; - off_t body_off = http->out.offset - i->prefix_size; - assert(*size > 0); - assert(i->spec); - /* - * intersection of "have" and "need" ranges must not be empty - */ - assert(body_off < i->spec->offset + i->spec->length); - assert(body_off + *size > i->spec->offset); - /* - * put boundary and headers at the beginning of a range in a - * multi-range - */ - if (http->request->range->specs.count > 1 && i->debt_size == i->spec->length) { - assert(http->entry->mem_obj); - clientPackRangeHdr( - http->entry->mem_obj->reply, /* original reply */ - i->spec, /* current range */ - i->boundary, /* boundary, the same for all */ - mb - ); - } - /* - * append content - */ - debug(33, 3) ("clientPackRange: appending %d bytes\n", copy_sz); - memBufAppend(mb, *buf, copy_sz); - /* - * update offsets - */ - *size -= copy_sz; - i->debt_size -= copy_sz; - body_off += copy_sz; - *buf += copy_sz; - http->out.offset = body_off + i->prefix_size; /* sync */ - /* - * paranoid check - */ - assert(*size >= 0 && i->debt_size >= 0); -} - -/* returns true if there is still data available to pack more ranges - * increments iterator "i" - * used by clientPackMoreRanges */ -static int -clientCanPackMoreRanges(const clientHttpRequest * http, HttpHdrRangeIter * i, ssize_t size) -{ - /* first update "i" if needed */ - if (!i->debt_size) { - if ((i->spec = httpHdrRangeGetSpec(http->request->range, &i->pos))) - i->debt_size = i->spec->length; - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - /* continue condition: need_more_data && have_more_data */ - return i->spec && size > 0; -} - -/* extracts "ranges" from buf and appends them to mb, updating all offsets and such */ -/* returns true if we need more data */ -static int -clientPackMoreRanges(clientHttpRequest * http, const char *buf, ssize_t size, MemBuf * mb) -{ - HttpHdrRangeIter *i = &http->range_iter; - /* offset in range specs does not count the prefix of an http msg */ - off_t body_off = http->out.offset - i->prefix_size; - assert(size >= 0); - /* check: reply was parsed and range iterator was initialized */ - assert(i->prefix_size > 0); - /* filter out data according to range specs */ - while (clientCanPackMoreRanges(http, i, size)) { - off_t start; /* offset of still missing data */ - assert(i->spec); - start = i->spec->offset + i->spec->length - i->debt_size; - debug(33, 3) ("clientPackMoreRanges: in: offset: %d size: %d\n", - (int) body_off, size); - debug(33, 3) ("clientPackMoreRanges: out: start: %d spec[%d]: [%d, %d), len: %d debt: %d\n", - (int) start, (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length, i->debt_size); - assert(body_off <= start); /* we did not miss it */ - /* skip up to start */ - if (body_off + size > start) { - const size_t skip_size = start - body_off; - body_off = start; - size -= skip_size; - buf += skip_size; - } else { - /* has not reached start yet */ - body_off += size; - size = 0; - buf = NULL; - } - /* put next chunk if any */ - if (size) { - http->out.offset = body_off + i->prefix_size; /* sync */ - clientPackRange(http, i, &buf, &size, mb); - body_off = http->out.offset - i->prefix_size; /* sync */ - } - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - debug(33, 3) ("clientPackMoreRanges: buf exhausted: in: offset: %d size: %d need_more: %d\n", - (int) body_off, size, i->debt_size); - if (i->debt_size) { - debug(33, 3) ("clientPackMoreRanges: need more: spec[%d]: [%d, %d), len: %d\n", - (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length); - /* skip the data we do not need if possible */ - if (i->debt_size == i->spec->length) /* at the start of the cur. spec */ - body_off = i->spec->offset; - else - assert(body_off == i->spec->offset + i->spec->length - i->debt_size); - } else if (http->request->range->specs.count > 1) { - /* put terminating boundary for multiparts */ - clientPackTermBound(i->boundary, mb); - } - http->out.offset = body_off + i->prefix_size; /* sync */ - return i->debt_size > 0; -} static int clientReplyBodyTooLarge(int clen) @@ -1752,22 +1458,18 @@ if (conn->chr != http) { /* there is another object in progress, defer this one */ debug(33, 1) ("clientSendMoreData: Deferring %s\n", storeUrl(entry)); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size < 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size == 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (http->out.offset == 0) { @@ -1783,11 +1485,10 @@ if (rep && clientReplyBodyTooLarge(rep->content_length)) { ErrorState *err = errorCon(ERR_TOO_BIG, HTTP_FORBIDDEN); err->request = requestLink(http->request); - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); httpReplyDestroy(rep); return; @@ -1815,11 +1516,11 @@ ErrorState *err; err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); err->request = requestLink(http->request); - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); - http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); + http->entry = + clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); httpReplyDestroy(rep); return; @@ -1827,24 +1528,17 @@ aclChecklistFree(ch); } else if (size < CLIENT_SOCK_SZ && entry->store_status == STORE_PENDING) { /* wait for more to arrive */ - storeClientCopy(http->sc, entry, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->bufofs += size; + assert(http->bufofs <= CLIENT_SOCK_SZ); + storeClientCopy(entry, + http->out.offset + http->bufofs, + http->out.offset + http->bufofs, + CLIENT_SOCK_SZ - http->bufofs, + buf + http->bufofs, clientSendMoreData, http); return; } - /* reset range iterator */ - http->range_iter.pos = HttpHdrRangeInitPos; - } else if (!http->request->range) { - /* Avoid copying to MemBuf for non-range requests */ - /* Note, if we're here, then 'rep' is known to be NULL */ - http->out.offset += body_size; - comm_write(fd, buf, size, clientWriteBodyComplete, http, NULL); - /* NULL because clientWriteBodyComplete frees it */ - return; } if (http->request->method == METHOD_HEAD) { if (rep) { @@ -1879,42 +1573,18 @@ memBufDefInit(&mb); } /* append body if any */ - if (http->request->range) { - /* Only GET requests should have ranges */ - assert(http->request->method == METHOD_GET); - /* clientPackMoreRanges() updates http->out.offset */ - /* force the end of the transfer if we are done */ - if (!clientPackMoreRanges(http, body_buf, body_size, &mb)) - http->flags.done_copying = 1; - } else if (body_buf && body_size) { + if (body_buf && body_size) { http->out.offset += body_size; check_size += body_size; memBufAppend(&mb, body_buf, body_size); } - if (!http->request->range && http->request->method == METHOD_GET) + if (http->request->method == METHOD_GET) assert(check_size == size); /* write */ comm_write_mbuf(fd, mb, clientWriteComplete, http); /* if we don't do it, who will? */ - memFree(buf, MEM_CLIENT_SOCK_BUF); } -/* - * clientWriteBodyComplete is called for MEM_CLIENT_SOCK_BUF's - * written directly to the client socket, versus copying to a MemBuf - * and going through comm_write_mbuf. Most non-range responses after - * the headers probably go through here. - */ -static void -clientWriteBodyComplete(int fd, char *buf, size_t size, int errflag, void *data) -{ - /* - * NOTE: clientWriteComplete doesn't currently use its "buf" - * (second) argument, so we pass in NULL. - */ - clientWriteComplete(fd, NULL, size, errflag, data); - memFree(buf, MEM_CLIENT_SOCK_BUF); -} static void clientKeepaliveNextRequest(clientHttpRequest * http) @@ -1955,14 +1625,15 @@ debug(33, 1) ("clientKeepaliveNextRequest: FD %d Sending next\n", conn->fd); assert(entry); - if (0 == storeClientCopyPending(http->sc, entry, http)) { + if (0 == storeClientCopyPending(entry, http)) { if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientSendMoreData, http); } @@ -2017,11 +1688,12 @@ * storage manager. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientWriteComplete 2: ENTRY_ABORTED\n"); - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientSendMoreData, http); } @@ -2046,60 +1718,68 @@ err->request = requestLink(r); err->src_addr = http->conn->peer.sin_addr; if (http->entry) { - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); } - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, r->method, null_request_flags, + REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); } -/* - * Return true if we should force a cache miss on this range request. - * entry must be non-NULL. + +/* + * clientLookupDone - handle the result of the first lookup + * + * If the result of the first lookup is a MISS and the method is METHOD_HEAD, + * we can generate a HEAD reply from a cached GET object, so initiate a + * second lookup. Otherwise, call clientProcessRequest2() to get the + * cache hit/miss information, and punt to clientProcessRequestDone(). */ -static int -clientCheckRangeForceMiss(StoreEntry * entry, HttpHdrRange * range) +static void +clientLookupDone(void *data, StoreEntry *e) { - /* - * If the range_offset_limit is NOT in effect, there - * is no reason to force a miss. - */ - if (0 == httpHdrRangeOffsetLimit(range)) - return 0; - /* - * Here, we know it's possibly a hit. If we already have the - * whole object cached, we won't force a miss. - */ - if (STORE_OK == entry->store_status) - return 0; /* we have the whole object */ - /* - * Now we have a hit on a PENDING object. We need to see - * if the part we want is already cached. If so, we don't - * force a miss. - */ - assert(NULL != entry->mem_obj); - if (httpHdrRangeFirstOffset(range) <= entry->mem_obj->inmem_hi) - return 0; - /* - * Even though we have a PENDING copy of the object, we - * don't want to wait to reach the first range offset, - * so we force a miss for a new range request to the - * origin. - */ - return 1; + clientHttpRequest *http = data; + + if (http->request->method == METHOD_HEAD && e == NULL) { + /* + * Start off the GET lookup, and change the request to be a + * HEAD afterwards + */ + http->request->method = METHOD_GET; + storeDoubleCheckRequest(http->request, http->uri, METHOD_GET); + storeLookup(http->request, clientLookupHeadDone, http); + http->request->method = METHOD_HEAD; + } else { + /* We have to continue with our result, NULL or not */ + http->log_type = clientProcessRequest2(http, e); + clientProcessRequestDone(http); + } + /* NOTREACHED */ } +/* + * clientLookupHeadDone - handle the HEAD -> GET request + * + * Similar to clientLookupDone(), we simply call clientProcessRequest2 + */ +static void +clientLookupHeadDone(void *data, StoreEntry *e) +{ + clientHttpRequest *http = data; + + http->log_type = clientProcessRequest2(http, e); + clientProcessRequestDone(http); +} + +/* + * decide whether we've got a cache HIT or MISS, and then punt across + * to clientProcessRequestDone() + */ static log_type -clientProcessRequest2(clientHttpRequest * http) +clientProcessRequest2(clientHttpRequest * http, StoreEntry *e) { request_t *r = http->request; - StoreEntry *e; - e = http->entry = storeGetPublic(http->uri, r->method); - if (r->method == METHOD_HEAD && e == NULL) { - /* We can generate a HEAD reply from a cached GET object */ - e = http->entry = storeGetPublic(http->uri, METHOD_GET); - } + /* Release negatively cached IP-cache entries on reload */ if (r->flags.nocache) ipcacheInvalidate(r->host); @@ -2112,7 +1792,7 @@ #endif if (NULL == e) { /* this object isn't in the cache */ - debug(33, 3) ("clientProcessRequest2: storeGet() MISS\n"); + debug(33, 3) ("clientProcessRequest2: storeLookup() MISS\n"); return LOG_TCP_MISS; } if (Config.onoff.offline) { @@ -2151,21 +1831,14 @@ http->entry = NULL; ipcacheInvalidate(r->host); return LOG_TCP_CLIENT_REFRESH_MISS; - } - if (NULL == r->range) { - (void) 0; - } else if (httpHdrRangeWillBeComplex(r->range)) { + } else if (r->flags.range) { /* * Some clients break if we return "200 OK" for a Range * request. We would have to return "200 OK" for a _complex_ * Range request that is also a HIT. Thus, let's prevent HITs * on complex Range requests */ - debug(33, 3) ("clientProcessRequest2: complex range MISS\n"); - http->entry = NULL; - return LOG_TCP_MISS; - } else if (clientCheckRangeForceMiss(e, r->range)) { - debug(33, 3) ("clientProcessRequest2: forcing miss due to range_offset_limit\n"); + debug(33, 3) ("clientProcessRequest2: range MISS\n"); http->entry = NULL; return LOG_TCP_MISS; } @@ -2190,11 +1863,12 @@ sslStart(fd, url, r, &http->out.size, &http->al.http.code); return; } else if (r->method == METHOD_PURGE) { - clientPurgeRequest(http); + clientStartPurgeRequest(http); return; } else if (r->method == METHOD_TRACE) { if (r->max_forwards == 0) { - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, r->method, + null_request_flags, REPLY_OBJ_INTERNAL); storeReleaseRequest(http->entry); storeBuffer(http->entry); rep = httpReplyCreate(); @@ -2210,25 +1884,44 @@ /* yes, continue */ http->log_type = LOG_TCP_MISS; } else { - http->log_type = clientProcessRequest2(http); + /* We need to initiate a lookup, so defer completing the request */ + debug(33, 3) ("clientProcessRequest: looking up %s\n", http->uri); + storeDoubleCheckRequest(r, http->uri, r->method); + storeLookup(r, clientLookupDone, http); + return; } - debug(33, 4) ("clientProcessRequest: %s for '%s'\n", + /* Continue processing */ + clientProcessRequestDone(http); +} + + +/* + * clientProcessRequestDone - the completion of the request chain + * + * Either start getting data from a cached object, or punt across + * to clientProcessMiss(). + */ +static void +clientProcessRequestDone(clientHttpRequest *http) +{ + debug(33, 4) ("clientProcessRequestDone: %s for '%s'\n", log_tags[http->log_type], http->uri); http->out.offset = 0; if (NULL != http->entry) { storeLockObject(http->entry); storeCreateMemObject(http->entry, http->uri, http->log_uri); - http->entry->mem_obj->method = r->method; - http->sc = storeClientListAdd(http->entry, http); + http->entry->mem_obj->method = http->request->method; + storeClientRegister(http->entry, http); #if DELAY_POOLS - delaySetStoreClient(http->sc, delayClient(r)); + delaySetStoreClient(http->sc, delayClient(http->request)); #endif - storeClientCopy(http->sc, http->entry, + http->bufofs = 0; + storeClientCopy(http->entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientCacheHit, http); } else { @@ -2258,8 +1951,7 @@ debug(33, 0) ("\tlog_type = %s\n", log_tags[http->log_type]); storeEntryDump(http->entry, 1); } - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); http->entry = NULL; } @@ -2275,12 +1967,14 @@ err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); err->request = requestLink(r); err->src_addr = http->conn->peer.sin_addr; - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, r->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); return; } assert(http->out.offset == 0); - http->entry = clientCreateStoreEntry(http, r->method, r->flags); + http->entry = clientCreateStoreEntry(http, r->method, r->flags, + REPLY_OBJ_NETWORK); if (http->redirect.status) { HttpReply *rep = httpReplyCreate(); #if LOG_TCP_REDIRECTS @@ -2302,6 +1996,7 @@ parseHttpRequestAbort(ConnStateData * conn, const char *uri) { clientHttpRequest *http; + CBDATA_INIT_TYPE_FREECB(clientHttpRequest, httpMemFree); http = cbdataAlloc(clientHttpRequest); http->conn = conn; http->start = current_time; @@ -2309,6 +2004,7 @@ http->uri = xstrdup(uri); http->log_uri = xstrndup(uri, MAX_URL); http->range_iter.boundary = StringNull; + http->buf = memAllocate(MEM_CLIENT_SOCK_BUF); dlinkAdd(http, &http->active, &ClientActiveRequests); return http; } @@ -2441,12 +2137,14 @@ assert(prefix_sz <= conn->in.offset); /* Ok, all headers are received */ + CBDATA_INIT_TYPE_FREECB(clientHttpRequest, httpMemFree); http = cbdataAlloc(clientHttpRequest); http->http_ver = http_ver; http->conn = conn; http->start = current_time; http->req_sz = prefix_sz; http->range_iter.boundary = StringNull; + http->buf = memAllocate(MEM_CLIENT_SOCK_BUF); *prefix_p = xmalloc(prefix_sz + 1); xmemcpy(*prefix_p, conn->in.buf, prefix_sz); *(*prefix_p + prefix_sz) = '\0'; @@ -2719,7 +2417,8 @@ debug(33, 1) ("clientReadRequest: FD %d Invalid Request\n", fd); err = errorCon(ERR_INVALID_REQ, HTTP_BAD_REQUEST); err->request_hdrs = xstrdup(conn->in.buf); - http->entry = clientCreateStoreEntry(http, method, null_request_flags); + http->entry = clientCreateStoreEntry(http, method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); safe_free(prefix); break; @@ -2730,7 +2429,8 @@ err->src_addr = conn->peer.sin_addr; err->url = xstrdup(http->uri); http->al.http.code = err->http_status; - http->entry = clientCreateStoreEntry(http, method, null_request_flags); + http->entry = clientCreateStoreEntry(http, method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); safe_free(prefix); break; @@ -2774,7 +2474,8 @@ err->request = requestLink(request); request->flags.proxy_keepalive = 0; http->al.http.code = err->http_status; - http->entry = clientCreateStoreEntry(http, request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); break; } @@ -2783,7 +2484,8 @@ err->src_addr = conn->peer.sin_addr; err->request = requestLink(request); http->al.http.code = err->http_status; - http->entry = clientCreateStoreEntry(http, request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); break; } @@ -2798,7 +2500,7 @@ err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); err->request = requestLink(request); http->entry = clientCreateStoreEntry(http, - METHOD_NONE, null_request_flags); + METHOD_NONE, null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); break; } @@ -2823,7 +2525,8 @@ /* add to the client request queue */ for (H = &conn->chr; *H; H = &(*H)->next); *H = http; - http->entry = clientCreateStoreEntry(http, METHOD_NONE, null_request_flags); + http->entry = clientCreateStoreEntry(http, METHOD_NONE, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); return; } Index: squid/src/enums.h diff -u squid/src/enums.h:1.17 squid/src/enums.h:1.2.2.13 --- squid/src/enums.h:1.17 Sat Mar 3 02:44:32 2001 +++ squid/src/enums.h Sun Mar 18 14:51:21 2001 @@ -487,7 +487,6 @@ KEY_PRIVATE, ENTRY_FWD_HDR_WAIT, ENTRY_NEGCACHED, - ENTRY_VALIDATED, ENTRY_BAD_LENGTH, ENTRY_ABORTED #if UNUSED_CODE @@ -634,7 +633,6 @@ MEM_PS_STATE, MEM_REFRESH_T, MEM_RELIST, - MEM_REQUEST_T, MEM_SQUIDCONFIG, MEM_SQUIDCONFIG2, MEM_STATCOUNTERS, @@ -726,6 +724,15 @@ }; /* + * reply object types + */ +typedef enum { + REPLY_OBJ_NONE, + REPLY_OBJ_INTERNAL, + REPLY_OBJ_NETWORK +} reply_obj_t; + +/* * cbdata types. similar to the MEM_* types above, but managed * in cbdata.c. A big difference is that these types are dynamically * allocated. This list is only a list of predefined types. Other types @@ -736,7 +743,6 @@ CBDATA_UNDEF = 0, CBDATA_acl_access, CBDATA_aclCheck_t, - CBDATA_clientHttpRequest, CBDATA_ConnStateData, CBDATA_DigestFetchState, CBDATA_ErrorState, Index: squid/src/errorpage.c diff -u squid/src/errorpage.c:1.11 squid/src/errorpage.c:1.2.2.9 --- squid/src/errorpage.c:1.11 Sat Mar 3 02:44:32 2001 +++ squid/src/errorpage.c Sun Mar 18 14:33:31 2001 @@ -262,7 +262,7 @@ HttpReply *rep; MemObject *mem = entry->mem_obj; assert(mem != NULL); - assert(mem->inmem_hi == 0); + assert(storeMemHiOffset(entry) == 0); if (entry->store_status != STORE_PENDING) { /* * If the entry is not STORE_PENDING, then no clients @@ -270,7 +270,7 @@ * error message */ assert(EBIT_TEST(entry->flags, ENTRY_ABORTED)); - assert(mem->nclients == 0); + assert(storePendingNClients(entry) == 0); errorStateFree(err); return; } Index: squid/src/forward.c diff -u squid/src/forward.c:1.10 squid/src/forward.c:1.2.2.7 --- squid/src/forward.c:1.10 Sat Mar 3 02:44:32 2001 +++ squid/src/forward.c Sun Mar 18 14:33:31 2001 @@ -92,7 +92,7 @@ fwdLog(fwdState); #endif if (e->store_status == STORE_PENDING) { - if (e->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(e) == 0) { assert(fwdState->err); errorAppendEntry(e, fwdState->err); fwdState->err = NULL; @@ -130,7 +130,7 @@ { if (fwdState->entry->store_status != STORE_PENDING) return 0; - if (fwdState->entry->mem_obj->inmem_hi > 0) + if (storeMemHiOffset(fwdState->entry) > 0) return 0; if (fwdState->n_tries > 10) return 0; @@ -249,7 +249,7 @@ peer *p = fwdStateServerPeer(fwdState); debug(17, 2) ("fwdConnectTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); assert(fd == fwdState->server_fd); - if (entry->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(entry) == 0) { err = errorCon(ERR_CONNECT_FAIL, HTTP_GATEWAY_TIMEOUT); err->request = requestLink(fwdState->request); err->xerrno = ETIMEDOUT; @@ -585,7 +585,7 @@ #endif if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) return rc; - if (mem->inmem_hi - storeLowestMemReaderOffset(e) < READ_AHEAD_GAP) + if (storeMemHiOffset(e) - storeLowestMemReaderOffset(e) < READ_AHEAD_GAP) return rc; return 1; } Index: squid/src/ftp.c diff -u squid/src/ftp.c:1.12 squid/src/ftp.c:1.2.2.8 --- squid/src/ftp.c:1.12 Sat Mar 3 02:44:32 2001 +++ squid/src/ftp.c Sun Mar 18 14:33:31 2001 @@ -925,11 +925,11 @@ storeAppend(entry, ftpState->data.buf, len); ftpState->data.offset = 0; } - commSetSelect(fd, - COMM_SELECT_READ, - ftpDataRead, - data, - Config.Timeout.read); + commSetSelect(fd, + COMM_SELECT_READ, + ftpDataRead, + data, + Config.Timeout.read); } } @@ -1045,7 +1045,6 @@ const char *url = storeUrl(entry); FtpStateData *ftpState; HttpReply *reply; - CBDATA_INIT_TYPE(FtpStateData); ftpState = cbdataAlloc(FtpStateData); debug(9, 3) ("ftpStart: '%s'\n", url); @@ -2002,6 +2001,11 @@ static int ftpRestartable(FtpStateData * ftpState) { + + /* With the advent of the range header in the request_range going away.. */ + return 0; + +#if 0 if (ftpState->restart_offset > 0) return 1; if (!ftpState->request->range) @@ -2015,6 +2019,7 @@ if (ftpState->restart_offset <= 0) return 0; return 1; +#endif } static void @@ -2364,7 +2369,7 @@ ftpFailed(FtpStateData * ftpState, err_type error) { StoreEntry *entry = ftpState->entry; - if (entry->mem_obj->inmem_hi == 0) + if (storeMemHiOffset(entry) == 0) ftpFailedErrorMessage(ftpState, error); if (ftpState->data.fd > -1) { comm_close(ftpState->data.fd); @@ -2480,7 +2485,7 @@ if (ftpState->flags.http_header_sent) return; ftpState->flags.http_header_sent = 1; - assert(e->mem_obj->inmem_hi == 0); + assert(storeMemHiOffset(e) == 0); EBIT_CLR(e->flags, ENTRY_FWD_HDR_WAIT); filename = (t = strRChr(urlpath, '/')) ? t + 1 : strBuf(urlpath); if (ftpState->flags.isdir) { @@ -2523,7 +2528,7 @@ httpHeaderPutStr(&reply->header, HDR_CONTENT_ENCODING, mime_enc); httpReplySwapOut(reply, e); storeBufferFlush(e); - reply->hdr_sz = e->mem_obj->inmem_hi; + reply->hdr_sz = storeMemHiOffset(e); storeTimestampsSet(e); if (ftpState->flags.authenticated) { /* Index: squid/src/globals.h diff -u squid/src/globals.h:1.7 squid/src/globals.h:1.2.2.5 --- squid/src/globals.h:1.7 Fri Jan 12 00:20:33 2001 +++ squid/src/globals.h Fri Jan 12 00:46:07 2001 @@ -125,6 +125,7 @@ extern double current_dtime; extern int store_hash_buckets; /* 0 */ extern hash_table *store_table; /* NULL */ +extern dlink_list store_list; /* { NULL, NULL } */ extern dlink_list ClientActiveRequests; extern const String StringNull; /* { 0, 0, NULL } */ extern const MemBuf MemBufNull; /* MemBufNULL */ Index: squid/src/gopher.c diff -u squid/src/gopher.c:1.8 squid/src/gopher.c:1.2.2.6 --- squid/src/gopher.c:1.8 Sat Mar 3 02:44:32 2001 +++ squid/src/gopher.c Sun Mar 18 14:33:31 2001 @@ -577,7 +577,7 @@ StoreEntry *entry = gopherState->entry; debug(10, 4) ("gopherTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); if (entry->store_status == STORE_PENDING) { - if (entry->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(entry) == 0) { fwdFail(gopherState->fwdState, errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); } @@ -633,7 +633,7 @@ debug(50, 1) ("gopherReadReply: error reading: %s\n", xstrerror()); if (ignoreErrno(errno)) { commSetSelect(fd, COMM_SELECT_READ, gopherReadReply, data, 0); - } else if (entry->mem_obj->inmem_hi == 0) { + } else if (storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); err->xerrno = errno; @@ -643,7 +643,7 @@ } else { comm_close(fd); } - } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { + } else if (len == 0 && storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); err->xerrno = errno; Index: squid/src/http.c diff -u squid/src/http.c:1.12 squid/src/http.c:1.2.2.8 --- squid/src/http.c:1.12 Sat Mar 3 02:44:32 2001 +++ squid/src/http.c Sun Mar 18 14:33:31 2001 @@ -93,7 +93,7 @@ StoreEntry *entry = httpState->entry; debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); if (entry->store_status == STORE_PENDING) { - if (entry->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(entry) == 0) { fwdFail(httpState->fwd, errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); } @@ -229,6 +229,13 @@ */ if (httpHeaderHas(hdr, HDR_VARY)) return 0; + + /* We also don't deal with range requests */ + if (httpHeaderHas(hdr, HDR_RANGE)) + return 0; + if (httpHeaderHas(hdr, HDR_REQUEST_RANGE)) + return 0; + /* Pragma: no-cache in _replies_ is not documented in HTTP, * but servers like "Active Imaging Webcast/2.0" sure do use it */ if (httpHeaderHas(hdr, HDR_PRAGMA)) { @@ -324,7 +331,7 @@ HttpReply *reply = entry->mem_obj->reply; Ctx ctx; debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", - storeKeyText(entry->hash.key)); + storeKeyUrl(entry)); if (httpState->reply_hdr == NULL) httpState->reply_hdr = memAllocate(MEM_8K_BUF); assert(httpState->reply_hdr_state == 0); @@ -405,8 +412,8 @@ httpPconnTransferDone(HttpStateData * httpState) { /* return 1 if we got the last of the data on a persistent connection */ - MemObject *mem = httpState->entry->mem_obj; - HttpReply *reply = mem->reply; + StoreEntry *entry = httpState->entry; + HttpReply *reply = entry->mem_obj->reply; int clen; debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); /* @@ -442,7 +449,7 @@ if (clen < 0) return 0; /* If the body size is known, we must wait until we've gotten all of it. */ - if (mem->inmem_hi < reply->content_length + reply->hdr_sz) + if (storeMemHiOffset(entry) < reply->content_length + reply->hdr_sz) return 0; /* We got it all */ return 1; @@ -512,7 +519,7 @@ fd, xstrerror()); if (ignoreErrno(errno)) { commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - } else if (entry->mem_obj->inmem_hi == 0) { + } else if (storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); err->xerrno = errno; @@ -521,7 +528,7 @@ } else { comm_close(fd); } - } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { + } else if (len == 0 && storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); err->xerrno = errno; @@ -647,7 +654,6 @@ LOCAL_ARRAY(char, bbuf, BBUF_SZ); String strConnection = StringNull; const HttpHeader *hdr_in = &orig_request->header; - int we_do_ranges; const HttpHeaderEntry *e; String strVia; String strFwd; @@ -657,26 +663,8 @@ if (request->lastmod > -1 && request->method == METHOD_GET) httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); - /* decide if we want to do Ranges ourselves - * (and fetch the whole object now) - * We want to handle Ranges ourselves iff - * - we can actually parse client Range specs - * - the specs are expected to be simple enough (e.g. no out-of-order ranges) - * - reply will be cachable - * (If the reply will be uncachable we have to throw it away after - * serving this request, so it is better to forward ranges to - * the server and fetch only the requested content) - */ - if (NULL == orig_request->range) - we_do_ranges = 0; - else if (!orig_request->flags.cachable) - we_do_ranges = 0; - else if (httpHdrRangeOffsetLimit(orig_request->range)) - we_do_ranges = 0; - else - we_do_ranges = 1; - debug(11, 8) ("httpBuildRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d\n", - orig_request->range, orig_request->flags.cachable, we_do_ranges); + /* We are not doing ranges at all now - we're just passing them through */ + debug(11, 8) ("httpBuildRequestHeader: cachable: %d\n", orig_request->flags.cachable); strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); while ((e = httpHeaderGetEntry(hdr_in, &pos))) { @@ -737,12 +725,6 @@ httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); } break; - case HDR_RANGE: - case HDR_IF_RANGE: - case HDR_REQUEST_RANGE: - if (!we_do_ranges) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - break; case HDR_PROXY_CONNECTION: case HDR_CONNECTION: case HDR_VIA: Index: squid/src/main.c diff -u squid/src/main.c:1.19 squid/src/main.c:1.2.2.10 --- squid/src/main.c:1.19 Tue Jan 30 16:27:46 2001 +++ squid/src/main.c Tue Feb 6 02:08:04 2001 @@ -479,6 +479,7 @@ #else idnsInit(); #endif + requestInit(); redirectInit(); authenticateInit(&Config.authConfig); useragentOpenLog(); @@ -501,6 +502,9 @@ #if USE_UNLINKD unlinkdInit(); #endif + reply_internal_init(); + reply_network_init(); + urlInitialize(); cachemgrInit(); statInit(); Index: squid/src/mem.c diff -u squid/src/mem.c:1.9 squid/src/mem.c:1.2.2.4 --- squid/src/mem.c:1.9 Sat Feb 17 12:56:22 2001 +++ squid/src/mem.c Tue Feb 20 07:39:42 2001 @@ -275,8 +275,6 @@ memDataInit(MEM_PS_STATE, "ps_state", sizeof(ps_state), 0); memDataInit(MEM_REFRESH_T, "refresh_t", sizeof(refresh_t), 0); memDataInit(MEM_RELIST, "relist", sizeof(relist), 0); - memDataInit(MEM_REQUEST_T, "request_t", sizeof(request_t), - Squid_MaxFD >> 3); memDataInit(MEM_SQUIDCONFIG, "SquidConfig", sizeof(SquidConfig), 0); memDataInit(MEM_SQUIDCONFIG2, "SquidConfig2", sizeof(SquidConfig2), 0); memDataInit(MEM_STATCOUNTERS, "StatCounters", sizeof(StatCounters), 0); Index: squid/src/mime.c diff -u squid/src/mime.c:1.8 squid/src/mime.c:1.2.2.9 --- squid/src/mime.c:1.8 Fri Jan 12 00:20:33 2001 +++ squid/src/mime.c Wed Feb 7 05:54:38 2001 @@ -364,12 +364,18 @@ debug(25, 5) ("mimeInit: added '%s'\n", buf); } fclose(fp); + /* * Create Icon StoreEntry's */ + + /* Remember, we're not doing icons in this fashion.. -- adrian */ +#if 0 for (m = MimeTable; m != NULL; m = m->next) mimeLoadIconFile(m->icon); debug(25, 1) ("Loaded Icons.\n"); +#endif + debug(25, 1) ("Not loading icons.\n"); } void @@ -391,62 +397,5 @@ static void mimeLoadIconFile(const char *icon) { - int fd; - int n; - request_flags flags; - struct stat sb; - StoreEntry *e; - LOCAL_ARRAY(char, path, MAXPATHLEN); - LOCAL_ARRAY(char, url, MAX_URL); - char *buf; - const char *type = mimeGetContentType(icon); - HttpReply *reply; - http_version_t version; - if (type == NULL) - fatal("Unknown icon format while reading mime.conf\n"); - buf = internalLocalUri("/squid-internal-static/icons/", icon); - xstrncpy(url, buf, MAX_URL); - if (storeGetPublic(url, METHOD_GET)) - return; - snprintf(path, MAXPATHLEN, "%s/%s", Config.icons.directory, icon); - fd = file_open(path, O_RDONLY | O_BINARY); - if (fd < 0) { - debug(25, 0) ("mimeLoadIconFile: %s: %s\n", path, xstrerror()); - return; - } - if (fstat(fd, &sb) < 0) { - debug(50, 0) ("mimeLoadIconFile: FD %d: fstat: %s\n", fd, xstrerror()); - return; - } - flags = null_request_flags; - flags.cachable = 1; - e = storeCreateEntry(url, - url, - flags, - METHOD_GET); - assert(e != NULL); - storeSetPublicKey(e); - storeBuffer(e); - e->mem_obj->request = requestLink(urlParse(METHOD_GET, url)); - httpReplyReset(reply = e->mem_obj->reply); - httpBuildVersion(&version, 1, 0); - httpReplySetHeaders(reply, version, HTTP_OK, NULL, - type, (int) sb.st_size, sb.st_mtime, -1); - reply->cache_control = httpHdrCcCreate(); - httpHdrCcSetMaxAge(reply->cache_control, 86400); - httpHeaderPutCc(&reply->header, reply->cache_control); - httpReplySwapOut(reply, e); - reply->hdr_sz = e->mem_obj->inmem_hi; /* yuk */ - /* read the file into the buffer and append it to store */ - buf = memAllocate(MEM_4K_BUF); - while ((n = read(fd, buf, 4096)) > 0) - storeAppend(e, buf, n); - file_close(fd); - EBIT_SET(e->flags, ENTRY_SPECIAL); - storeBufferFlush(e); - storeComplete(e); - storeTimestampsSet(e); - debug(25, 3) ("Loaded icon %s\n", url); - storeUnlockObject(e); - memFree(buf, MEM_4K_BUF); + fatal("I shouldn't get here yet!\n"); } Index: squid/src/neighbors.c diff -u squid/src/neighbors.c:1.9 squid/src/neighbors.c:1.2.2.7 --- squid/src/neighbors.c:1.9 Sat Mar 3 02:44:32 2001 +++ squid/src/neighbors.c Sun Mar 18 14:51:21 2001 @@ -1122,7 +1122,8 @@ assert(p->type == PEER_MULTICAST); p->mcast.flags.count_event_pending = 0; snprintf(url, MAX_URL, "http://%s/", inet_ntoa(p->in_addr.sin_addr)); - fake = storeCreateEntry(url, url, null_request_flags, METHOD_GET); + fake = storeCreateEntry(url, url, null_request_flags, METHOD_GET, + REPLY_OBJ_INTERNAL); psstate = cbdataAlloc(ps_state); psstate->request = requestLink(urlParse(METHOD_GET, url)); psstate->entry = fake; Index: squid/src/net_db.c diff -u squid/src/net_db.c:1.10 squid/src/net_db.c:1.2.2.8 --- squid/src/net_db.c:1.10 Sat Mar 3 02:44:32 2001 +++ squid/src/net_db.c Sun Mar 18 14:33:32 2001 @@ -627,8 +627,8 @@ debug(38, 3) ("netdbExchangeHandleReply: STORE_PENDING\n"); storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, ex->buf, netdbExchangeHandleReply, ex); - } else if (ex->seen < ex->e->mem_obj->inmem_hi) { - debug(38, 3) ("netdbExchangeHandleReply: ex->e->mem_obj->inmem_hi\n"); + } else if (ex->seen < storeMemHiOffset(ex->e)) { + debug(38, 3) ("netdbExchangeHandleReply: storeMemHiOffset(ex->e)\n"); storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, ex->buf, netdbExchangeHandleReply, ex); } else { @@ -644,7 +644,7 @@ debug(38, 3) ("netdbExchangeDone: %s\n", storeUrl(ex->e)); memFree(ex->buf, MEM_4K_BUF); requestUnlink(ex->r); - storeUnregister(ex->sc, ex->e, ex); + storeClientUnregister(ex->sc, ex->e, ex); storeUnlockObject(ex->e); cbdataUnlock(ex->p); cbdataFree(ex); @@ -1005,11 +1005,12 @@ requestLink(ex->r); assert(NULL != ex->r); httpBuildVersion(&ex->r->http_ver, 1, 0); - ex->e = storeCreateEntry(uri, uri, null_request_flags, METHOD_GET); + ex->e = storeCreateEntry(uri, uri, null_request_flags, METHOD_GET, + REPLY_OBJ_INTERNAL); ex->buf_sz = 4096; ex->buf = memAllocate(MEM_4K_BUF); assert(NULL != ex->e); - ex->sc = storeClientListAdd(ex->e, ex); + ex->sc = storeClientRegister(ex->e, ex); storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, ex->buf, netdbExchangeHandleReply, ex); ex->r->flags.loopdetect = 1; /* cheat! -- force direct */ Index: squid/src/peer_digest.c diff -u squid/src/peer_digest.c:1.7 squid/src/peer_digest.c:1.2.2.8 --- squid/src/peer_digest.c:1.7 Sat Mar 3 02:44:32 2001 +++ squid/src/peer_digest.c Sun Mar 18 14:33:32 2001 @@ -269,7 +269,6 @@ peer *p = pd->peer; StoreEntry *e, *old_e; char *url; - const cache_key *key; request_t *req; DigestFetchState *fetch = NULL; @@ -283,8 +282,7 @@ url = internalRemoteUri(p->host, p->http_port, "/squid-internal-periodic/", StoreDigestFileName); - key = storeKeyPublic(url, METHOD_GET); - debug(72, 2) ("peerDigestRequest: %s key: %s\n", url, storeKeyText(key)); + debug(72, 2) ("peerDigestRequest: %s\n", url); req = urlParse(METHOD_GET, url); assert(req); @@ -308,16 +306,17 @@ req->flags.cachable = 1; /* the rest is based on clientProcessExpired() */ req->flags.refresh = 1; - old_e = fetch->old_entry = storeGet(key); + old_e = fetch->old_entry = storeGetPublic(url, METHOD_GET); if (old_e) { debug(72, 5) ("peerDigestRequest: found old entry\n"); storeLockObject(old_e); storeCreateMemObject(old_e, url, url); - fetch->old_sc = storeClientListAdd(old_e, fetch); + fetch->old_sc = storeClientRegister(old_e, fetch); } - e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method); + e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method, + REPLY_OBJ_NETWORK); assert(EBIT_TEST(e->flags, KEY_PRIVATE)); - fetch->sc = storeClientListAdd(e, fetch); + fetch->sc = storeClientRegister(e, fetch); /* set lastmod to trigger IMS request if possible */ if (old_e) e->lastmod = old_e->lastmod; @@ -366,7 +365,7 @@ httpReplyUpdateOnNotModified(fetch->old_entry->mem_obj->reply, reply); storeTimestampsSet(fetch->old_entry); /* get rid of 304 reply */ - storeUnregister(fetch->sc, fetch->entry, fetch); + storeClientUnregister(fetch->sc, fetch->entry, fetch); storeUnlockObject(fetch->entry); fetch->entry = fetch->old_entry; fetch->old_entry = NULL; @@ -377,7 +376,7 @@ /* get rid of old entry if any */ if (fetch->old_entry) { debug(72, 3) ("peerDigestFetchReply: got new digest, releasing old one\n"); - storeUnregister(fetch->old_sc, fetch->old_entry, fetch); + storeClientUnregister(fetch->old_sc, fetch->old_entry, fetch); storeReleaseRequest(fetch->old_entry); storeUnlockObject(fetch->old_entry); fetch->old_entry = NULL; @@ -685,7 +684,7 @@ if (fetch->old_entry) { debug(72, 2) ("peerDigestFetchFinish: deleting old entry\n"); - storeUnregister(fetch->sc, fetch->old_entry, fetch); + storeClientUnregister(fetch->sc, fetch->old_entry, fetch); storeReleaseRequest(fetch->old_entry); storeUnlockObject(fetch->old_entry); fetch->old_entry = NULL; @@ -697,7 +696,7 @@ statCounter.cd.msgs_recv += fetch->recv.msg; /* unlock everything */ - storeUnregister(fetch->sc, fetch->entry, fetch); + storeClientUnregister(fetch->sc, fetch->entry, fetch); storeUnlockObject(fetch->entry); requestUnlink(fetch->request); fetch->entry = NULL; @@ -721,7 +720,7 @@ /* XXX: we must distinguish between 304 hits and misses here */ fetch->sent.bytes = httpRequestPrefixLen(fetch->request); fetch->recv.bytes = fetch->entry->store_status == STORE_PENDING ? - mem->inmem_hi : mem->object_sz; + storeMemHiOffset(fetch->entry) : mem->object_sz; fetch->sent.msg = fetch->recv.msg = 1; fetch->expires = fetch->entry->expires; fetch->resp_time = squid_curtime - fetch->start_time; Index: squid/src/protos.h diff -u squid/src/protos.h:1.20 squid/src/protos.h:1.2.2.19 --- squid/src/protos.h:1.20 Sat Mar 3 02:44:32 2001 +++ squid/src/protos.h Sun Mar 18 14:33:32 2001 @@ -127,11 +127,12 @@ extern void clientAccessCheckDone(int, void *); extern int modifiedSince(StoreEntry *, request_t *); extern char *clientConstructTraceEcho(clientHttpRequest *); -extern void clientPurgeRequest(clientHttpRequest *); +extern void clientStartPurgeRequest(clientHttpRequest *); extern int checkNegativeHit(StoreEntry *); extern void clientHttpConnectionsOpen(void); extern void clientHttpConnectionsClose(void); -extern StoreEntry *clientCreateStoreEntry(clientHttpRequest *, method_t, request_flags); +extern StoreEntry *clientCreateStoreEntry(clientHttpRequest *, method_t, + request_flags, reply_obj_t); extern int isTcpHit(log_type); extern void clientReadBody(request_t * req, char *buf, size_t size, CBCB * callback, void *data); extern int clientAbortBody(request_t * req); @@ -493,6 +494,7 @@ extern int httpReplyBodySize(method_t, HttpReply *); /* Http Request */ +extern void requestInit(void); extern request_t *requestCreate(method_t, protocol_t, const char *urlpath); extern void requestDestroy(request_t *); extern request_t *requestLink(request_t *); @@ -866,7 +868,8 @@ extern StoreEntry *new_StoreEntry(int, const char *, const char *); extern StoreEntry *storeGet(const cache_key *); extern StoreEntry *storeGetPublic(const char *uri, const method_t method); -extern StoreEntry *storeCreateEntry(const char *, const char *, request_flags, method_t); +extern StoreEntry *storeCreateEntry(const char *, const char *, request_flags, + method_t, reply_obj_t); extern void storeSetPublicKey(StoreEntry *); extern void storeComplete(StoreEntry *); extern void storeInit(void); @@ -893,6 +896,7 @@ extern void storeMemObjectDump(MemObject * mem); extern void storeEntryDump(const StoreEntry * e, int debug_lvl); extern const char *storeUrl(const StoreEntry *); +extern const char *storeKeyUrl(const StoreEntry *); extern void storeCreateMemObject(StoreEntry *, const char *, const char *); extern void storeCopyNotModifiedReplyHeaders(MemObject * O, MemObject * N); extern void storeBuffer(StoreEntry *); @@ -918,6 +922,10 @@ extern void storeFsDone(void); extern void storeFsAdd(char *, STSETUP *); extern void storeReplAdd(char *, REMOVALPOLICYCREATE *); +extern off_t storeMemHiOffset(const StoreEntry *); +extern off_t storeMemLoOffset(const StoreEntry *); +extern void storeDetachReply(StoreEntry *); +extern void storeDoubleCheckRequest(request_t *, char *, method_t); /* store_modules.c */ extern void storeFsSetup(void); @@ -926,6 +934,9 @@ extern void storeReplSetup(void); /* store_io.c */ +extern void storeLookup(request_t * , STGETDONE *, void *); +extern void storeLookupComplete(request_t *, STGETDONE *, void *, + StoreEntry *); extern storeIOState *storeCreate(StoreEntry *, STFNCB *, STIOCB *, void *); extern storeIOState *storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *); extern void storeClose(storeIOState *); @@ -1029,10 +1040,10 @@ #if STORE_CLIENT_LIST_DEBUG extern store_client *storeClientListSearch(const MemObject * mem, void *data); #endif -extern store_client *storeClientListAdd(StoreEntry * e, void *data); -extern void storeClientCopy(store_client *, StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); -extern int storeClientCopyPending(store_client *, StoreEntry * e, void *data); -extern int storeUnregister(store_client * sc, StoreEntry * e, void *data); +extern void storeClientRegister(StoreEntry * e, void *data); +extern void storeClientCopy(StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); +extern int storeClientCopyPending(StoreEntry * e, void *data); +extern int storeClientUnregister(StoreEntry * e, void *data); extern off_t storeLowestMemReaderOffset(const StoreEntry * entry); extern void InvokeHandlers(StoreEntry * e); extern int storePendingNClients(const StoreEntry * e); @@ -1289,6 +1300,16 @@ extern unsigned int url_checksum(const char *url); #endif + +/* + * reply objects + */ +extern void reply_internal_init(void); +extern void reply_internal_create(StoreEntry *); + +extern void reply_network_init(void); +extern void reply_network_create(StoreEntry *); + /* * hack to allow snmp access to the statistics counters */ Index: squid/src/reply_internal.c diff -u /dev/null squid/src/reply_internal.c:1.1.2.6 --- /dev/null Sun Jan 25 06:36:42 2004 +++ squid/src/reply_internal.c Wed Jan 3 04:11:33 2001 @@ -0,0 +1,290 @@ +/* + * reply_internal.c + * + * Handles "internal" replies, such as error pages, internal gifs, etc.. + * + * Adrian Chadd + */ + + +#include "squid.h" + +struct _reply_internal_state { + mem_hdr data_hdr; + off_t inmem_hi; + off_t inmem_lo; +}; + + + +typedef struct _reply_internal_state reply_internal_state_t; + +MemPool *reply_internal_state_pool = NULL; + +/* Prototypes */ +static void reply_internal_clientcopy(StoreEntry * e); +static void reply_internal_clientcopy3(StoreEntry * e); + + + +void +reply_internal_init(void) +{ + reply_internal_state_pool = memPoolCreate("Internal reply pool", + sizeof(reply_internal_state_t)); +} + + +/* + * reply_internal_done + */ +static void +reply_internal_done(StoreEntry *e) +{ + reply_internal_state_t *state = e->stdata; + + /* Make sure noone is left ! */ + + /* Kill the state struct */ + stmemFree(&state->data_hdr); + memPoolFree(reply_internal_state_pool, state); + + /* Done! */ +} + + +/* + * reply_internal_memhi + */ +static off_t +reply_internal_memhi(const StoreEntry *e) +{ + reply_internal_state_t *state = e->stdata; + + return state->inmem_hi; +} + + +/* + * reply_internal_memlo + */ +static off_t +reply_internal_memlo(const StoreEntry *e) +{ + reply_internal_state_t *state = e->stdata; + + return state->inmem_lo; +} + + +/* + * reply_internal_append + */ +static void +reply_internal_append(StoreEntry *e, const char *buf, int len) +{ + MemObject *mem = e->mem_obj; + reply_internal_state_t *state = e->stdata; + assert(mem != NULL); + assert(len >= 0); + assert(e->store_status == STORE_PENDING); + if (len) { + debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", + len, + storeKeyUrl(e)); + /* Used to call storeGetMemSpace here .. */ + /* storeGetMemSpace(len); */ + stmemAppend(&state->data_hdr, buf, len); + state->inmem_hi += len; + } + if (EBIT_TEST(e->flags, DELAY_SENDING)) + return; + e->stflush(e); +} + + +/* + * reply_internal_flush + */ +static void +reply_internal_flush(StoreEntry *e) +{ + debug(20, 3) ("reply_internal_flush: %s\n", storeKeyUrl(e)); + /* walk the entire list looking for valid callbacks */ + if (e->sc.callback_data == NULL) + return; + if (e->sc.callback == NULL) + return; + if (e->sc.flags.disk_io_pending) + return; + reply_internal_clientcopy3(e); +} + + +/* store client copy evilness */ + +static void +reply_internal_clientcallback(store_client * sc, ssize_t sz) +{ + STCB *callback = sc->callback; + char *buf = sc->copy_buf; + assert(sc->callback); + sc->callback = NULL; + sc->copy_buf = NULL; + if (cbdataValid(sc->callback_data)) + callback(sc->callback_data, buf, sz); +} + +static void +reply_internal_clientcopyevent(void *data) +{ + StoreEntry *e = data; + debug(20, 3) ("reply_internal_clientcopyevent: Running\n"); + e->sc.flags.copy_event_pending = 0; + if (!e->sc.valid) + return; + reply_internal_clientcopy3(e); +} + + +/* + * This function is used below to decide if we have any more data to + * send to the client. If the store_status is STORE_PENDING, then we + * do have more data to send. If its STORE_OK, then + * we continue checking. + * If the length is >= 0, then we compare it to the requested copy + * offset. + */ +static int +reply_internal_clientnomoretosend(StoreEntry * e, store_client * sc) +{ + if (e->store_status == STORE_PENDING) + return 0; + assert(objectLen(e) > -1); /* Internal objects will never be swap backed */ + if (sc->copy_offset < objectLen(e)) + return 0; + return 1; +} + +/* copy bytes requested by the client */ +static void +reply_internal_clientcopy(StoreEntry * e) +{ + store_client *sc = &e->sc; + if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) { + debug(20, 5) ("reply_internal_clientcopy: returning because ENTRY_FWD_HDR_WAIT set\n"); + return; + } + debug(20, 3) ("reply_internal_clientcopy: %s\n", storeKeyUrl(e)); + assert(sc->callback != NULL); + /* + * We used to check for ENTRY_ABORTED here. But there were some + * problems. For example, we might have a slow client (or two) and + * the server-side is reading far ahead and swapping to disk. Even + * if the server-side aborts, we want to give the client(s) + * everything we got before the abort condition occurred. + */ + reply_internal_clientcopy3(e); +} + +static void +reply_internal_clientcopy3(StoreEntry * e) +{ + store_client *sc = &e->sc; + reply_internal_state_t *state = e->stdata; + size_t sz; + + sc->flags.store_copying = 1; + if (reply_internal_clientnomoretosend(e, sc)) { + /* There is no more to send! */ + reply_internal_clientcallback(sc, 0); + goto finish; + } + if ((e->store_status == STORE_PENDING) && + (sc->copy_offset >= state->inmem_hi)) { + /* client has already seen this, wait for more */ + debug(20, 3) ("reply_internal_clientcopy3: Waiting for more\n"); + goto finish; + } + + if ((sc->copy_offset >= state->inmem_lo) && + (sc->copy_offset < state->inmem_hi)) { + /* What the client wants is in memory */ + debug(20, 3) ("reply_internal_clientcopy3: Copying from memory\n"); + sz = stmemCopy(&state->data_hdr, + sc->copy_offset, sc->copy_buf, sc->copy_size); + reply_internal_clientcallback(sc, sz); + goto finish; + } + /* We shouldn't ever get here! */ + assert(1 == 0); + +finish: + sc->flags.store_copying = 0; +} + + +/* + * reply_makepublic - make this object public + */ +static void +reply_internal_makepublic(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_makeprivate - make this object private + */ +static void +reply_internal_makeprivate(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_internal_release - object is being deleted + * + * Yes, its a no-op for us because all objects are private + */ +static void +reply_internal_release(StoreEntry *e) +{ +} + + +/* + * reply_internal_create - create an internal reply .. + */ +void +reply_internal_create(StoreEntry *e) +{ + reply_internal_state_t *state; + MemObject *mem; + + assert(e != NULL); + assert(e->mem_obj != NULL); + + mem = e->mem_obj; + + /* Create a new state struct */ + state = memPoolAlloc(reply_internal_state_pool); + + /* Initialise it */ + e->stdata = state; + + /* Assign our functions .. */ + e->stappend = reply_internal_append; + e->stflush = reply_internal_flush; + e->stmemhi = reply_internal_memhi; + e->stmemlo = reply_internal_memlo; + e->stclientcopy = reply_internal_clientcopy; + e->stmakeprivate = reply_internal_makeprivate; + e->stmakepublic = reply_internal_makepublic; + e->strelease = reply_internal_release; + e->stdone = reply_internal_done; + + /* Done! */ +} Index: squid/src/reply_network.c diff -u /dev/null squid/src/reply_network.c:1.1.2.2 --- /dev/null Sun Jan 25 06:36:42 2004 +++ squid/src/reply_network.c Sun Jan 7 15:22:47 2001 @@ -0,0 +1,296 @@ +/* + * reply_network.c + * + * Handles replies which have a network as a data server. + * This module handles limiting the amount of data which is requested + * from a server so as to not flood the client, and also writing incoming + * data to a backing object if needed. + * + * Adrian Chadd + */ + + +#include "squid.h" + +struct _reply_network_state { + mem_hdr data_hdr; + off_t inmem_hi; + off_t inmem_lo; +}; + + + +typedef struct _reply_network_state reply_network_state_t; + +MemPool *reply_network_state_pool = NULL; + +/* Prototypes */ +static void reply_network_clientcopy(StoreEntry * e); +static void reply_network_clientcopy3(StoreEntry * e); + + + +void +reply_network_init(void) +{ + reply_network_state_pool = memPoolCreate("Internal reply pool", + sizeof(reply_network_state_t)); +} + + +/* + * reply_network_done + */ +static void +reply_network_done(StoreEntry *e) +{ + reply_network_state_t *state = e->stdata; + + /* Make sure noone is left ! */ + + /* Kill the state struct */ + stmemFree(&state->data_hdr); + memPoolFree(reply_network_state_pool, state); + + /* Done! */ +} + + +/* + * reply_network_memhi + */ +static off_t +reply_network_memhi(const StoreEntry *e) +{ + reply_network_state_t *state = e->stdata; + + return state->inmem_hi; +} + + +/* + * reply_network_memlo + */ +static off_t +reply_network_memlo(const StoreEntry *e) +{ + reply_network_state_t *state = e->stdata; + + return state->inmem_lo; +} + + +/* + * reply_network_append + */ +static void +reply_network_append(StoreEntry *e, const char *buf, int len) +{ + MemObject *mem = e->mem_obj; + reply_network_state_t *state = e->stdata; + assert(mem != NULL); + assert(len >= 0); + assert(e->store_status == STORE_PENDING); + if (len) { + debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", + len, + storeKeyUrl(e)); + /* Used to call storeGetMemSpace here .. */ + /* storeGetMemSpace(len); */ + stmemAppend(&state->data_hdr, buf, len); + state->inmem_hi += len; + } + if (EBIT_TEST(e->flags, DELAY_SENDING)) + return; + e->stflush(e); +} + + +/* + * reply_network_flush + */ +static void +reply_network_flush(StoreEntry *e) +{ + debug(20, 3) ("reply_network_flush: %s\n", storeKeyUrl(e)); + /* walk the entire list looking for valid callbacks */ + if (e->sc.callback_data == NULL) + return; + if (e->sc.callback == NULL) + return; + if (e->sc.flags.disk_io_pending) + return; + reply_network_clientcopy3(e); +} + + +/* store client copy evilness */ + +static void +reply_network_clientcallback(store_client * sc, ssize_t sz) +{ + STCB *callback = sc->callback; + char *buf = sc->copy_buf; + assert(sc->callback); + sc->callback = NULL; + sc->copy_buf = NULL; + if (cbdataValid(sc->callback_data)) + callback(sc->callback_data, buf, sz); +} + +static void +reply_network_clientcopyevent(void *data) +{ + StoreEntry *e = data; + debug(20, 3) ("reply_network_clientcopyevent: Running\n"); + e->sc.flags.copy_event_pending = 0; + if (!e->sc.valid) + return; + reply_network_clientcopy3(e); +} + + +/* + * This function is used below to decide if we have any more data to + * send to the client. If the store_status is STORE_PENDING, then we + * do have more data to send. If its STORE_OK, then + * we continue checking. + * If the length is >= 0, then we compare it to the requested copy + * offset. + */ +static int +reply_network_clientnomoretosend(StoreEntry * e, store_client * sc) +{ + if (e->store_status == STORE_PENDING) + return 0; + assert(objectLen(e) > -1); /* Internal objects will never be swap backed */ + if (sc->copy_offset < objectLen(e)) + return 0; + return 1; +} + +/* copy bytes requested by the client */ +static void +reply_network_clientcopy(StoreEntry * e) +{ + store_client *sc = &e->sc; + if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) { + debug(20, 5) ("reply_network_clientcopy: returning because ENTRY_FWD_HDR_WAIT set\n"); + return; + } + debug(20, 3) ("reply_network_clientcopy: %s\n", storeKeyUrl(e)); + assert(sc->callback != NULL); + /* + * We used to check for ENTRY_ABORTED here. But there were some + * problems. For example, we might have a slow client (or two) and + * the server-side is reading far ahead and swapping to disk. Even + * if the server-side aborts, we want to give the client(s) + * everything we got before the abort condition occurred. + */ + reply_network_clientcopy3(e); +} + +static void +reply_network_clientcopy3(StoreEntry * e) +{ + store_client *sc = &e->sc; + reply_network_state_t *state = e->stdata; + size_t sz; + + sc->flags.store_copying = 1; + if (reply_network_clientnomoretosend(e, sc)) { + /* There is no more to send! */ + reply_network_clientcallback(sc, 0); + goto finish; + } + if ((e->store_status == STORE_PENDING) && + (sc->copy_offset >= state->inmem_hi)) { + /* client has already seen this, wait for more */ + debug(20, 3) ("reply_network_clientcopy3: Waiting for more\n"); + goto finish; + } + + if ((sc->copy_offset >= state->inmem_lo) && + (sc->copy_offset < state->inmem_hi)) { + /* What the client wants is in memory */ + debug(20, 3) ("reply_network_clientcopy3: Copying from memory\n"); + sz = stmemCopy(&state->data_hdr, + sc->copy_offset, sc->copy_buf, sc->copy_size); + state->inmem_lo = sc->copy_offset + sz; + assert(state->inmem_lo <= state->inmem_hi); + stmemFreeDataUpto(&state->data_hdr, state->inmem_lo - 1); + reply_network_clientcallback(sc, sz); + goto finish; + } + /* We shouldn't ever get here! */ + assert(1 == 0); + +finish: + sc->flags.store_copying = 0; +} + + +/* + * reply_makepublic - make this object public + */ +static void +reply_network_makepublic(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_makeprivate - make this object private + */ +static void +reply_network_makeprivate(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_network_release - object is being deleted + * + * Yes, its a no-op for us because all objects are private + */ +static void +reply_network_release(StoreEntry *e) +{ +} + + +/* + * reply_network_create - create an internal reply .. + */ +void +reply_network_create(StoreEntry *e) +{ + reply_network_state_t *state; + MemObject *mem; + + assert(e != NULL); + assert(e->mem_obj != NULL); + + mem = e->mem_obj; + + /* Create a new state struct */ + state = memPoolAlloc(reply_network_state_pool); + + /* Initialise it */ + e->stdata = state; + + /* Assign our functions .. */ + e->stappend = reply_network_append; + e->stflush = reply_network_flush; + e->stmemhi = reply_network_memhi; + e->stmemlo = reply_network_memlo; + e->stclientcopy = reply_network_clientcopy; + e->stmakeprivate = reply_network_makeprivate; + e->stmakepublic = reply_network_makepublic; + e->strelease = reply_network_release; + e->stdone = reply_network_done; + + /* Done! */ +} Index: squid/src/stat.c diff -u squid/src/stat.c:1.9 squid/src/stat.c:1.2.2.10 --- squid/src/stat.c:1.9 Sat Mar 3 02:44:32 2001 +++ squid/src/stat.c Sun Mar 18 14:33:32 2001 @@ -40,9 +40,9 @@ typedef int STOBJFLT(const StoreEntry *); typedef struct { - StoreEntry *sentry; - int bucket; - STOBJFLT *filter; + StoreEntry *sentry; /* Where we're outputting to */ + STOBJFLT *filter; /* Filter function for output */ + StoreEntry *cur_entry; /* Current StoreEntry being checked .. */ } StatObjectsState; /* LOCALS */ @@ -230,8 +230,6 @@ strcat(buf, "FWD_HDR_WAIT,"); if (EBIT_TEST(flags, ENTRY_NEGCACHED)) strcat(buf, "NEGCACHED,"); - if (EBIT_TEST(flags, ENTRY_VALIDATED)) - strcat(buf, "VALIDATED,"); if (EBIT_TEST(flags, ENTRY_BAD_LENGTH)) strcat(buf, "BAD_LENGTH,"); if (EBIT_TEST(flags, ENTRY_ABORTED)) @@ -257,10 +255,8 @@ statStoreEntry(StoreEntry * s, StoreEntry * e) { MemObject *mem = e->mem_obj; - int i; struct _store_client *sc; - dlink_node *node; - storeAppendPrintf(s, "KEY %s\n", storeKeyText(e->hash.key)); + storeAppendPrintf(s, "KEY %s\n", storeKeyUrl(e)); if (mem) storeAppendPrintf(s, "\t%s %s\n", RequestMethodStr[mem->method], mem->log_url); @@ -274,18 +270,12 @@ storeAppendPrintf(s, "\tSwap Dir %d, File %#08X\n", e->swap_dirn, e->swap_filen); if (mem != NULL) { - storeAppendPrintf(s, "\tinmem_lo: %d\n", (int) mem->inmem_lo); - storeAppendPrintf(s, "\tinmem_hi: %d\n", (int) mem->inmem_hi); - storeAppendPrintf(s, "\tswapout: %d bytes queued\n", - (int) mem->swapout.queue_offset); - if (mem->swapout.sio) - storeAppendPrintf(s, "\tswapout: %d bytes written\n", - (int) storeOffset(mem->swapout.sio)); - for (i = 0, node = mem->clients.head; node; node = node->next, i++) { - sc = (store_client *) node->data; - if (sc->callback_data == NULL) - continue; - storeAppendPrintf(s, "\tClient #%d, %p\n", i, sc->callback_data); + storeAppendPrintf(s, "\tInMem_LoOffset: %d\n", (int) storeMemLoOffset(e)); + storeAppendPrintf(s, "\tInMem_HiOffset: %d\n", (int) storeMemHiOffset(e)); + + sc = &e->sc; + if (sc->valid) { + storeAppendPrintf(s, "\tClient #%d, %p\n", 1, sc->callback_data); storeAppendPrintf(s, "\t\tcopy_offset: %d\n", (int) sc->copy_offset); storeAppendPrintf(s, "\t\tseen_offset: %d\n", @@ -305,20 +295,18 @@ storeAppendPrintf(s, "\n"); } +#define MAX_NUMOBJECTS_TO_SCAN 1000 + /* process objects list */ static void statObjects(void *data) { StatObjectsState *state = data; StoreEntry *e; - hash_link *link_ptr = NULL; - hash_link *link_next = NULL; - if (state->bucket >= store_hash_buckets) { - storeComplete(state->sentry); - storeUnlockObject(state->sentry); - cbdataFree(state); - return; - } else if (EBIT_TEST(state->sentry->flags, ENTRY_ABORTED)) { + int i; + + + if (EBIT_TEST(state->sentry->flags, ENTRY_ABORTED)) { storeUnlockObject(state->sentry); cbdataFree(state); return; @@ -326,17 +314,33 @@ eventAdd("statObjects", statObjects, state, 0.1, 1); return; } + + /* Don't return the data to the client immediately */ storeBuffer(state->sentry); - debug(49, 3) ("statObjects: Bucket #%d\n", state->bucket); - link_next = hash_get_bucket(store_table, state->bucket); - while (NULL != (link_ptr = link_next)) { - link_next = link_ptr->next; - e = (StoreEntry *) link_ptr; - if (state->filter && 0 == state->filter(e)) - continue; - statStoreEntry(state->sentry, e); + + for (i = 0; i < MAX_NUMOBJECTS_TO_SCAN; i++) { + /* Assume we were given a non-NULL object .. */ + e = state->cur_entry; + assert(e != NULL); + if (state->filter) { + if (state->filter(e)) + statStoreEntry(state->sentry, e); + } else + statStoreEntry(state->sentry, e); + storeUnlockObject(e); + + /* Make sure we have a next object .. */ + if (e->node.next == NULL) { + storeComplete(state->sentry); + storeUnlockObject(state->sentry); + cbdataFree(state); + return; + } + /* e becomes this if we get to the top of the loop again! */ + state->cur_entry = (StoreEntry *) e->node.next->data; + storeLockObject(state->cur_entry); } - state->bucket++; + eventAdd("statObjects", statObjects, state, 0.0, 1); storeBufferFlush(state->sentry); } @@ -348,6 +352,9 @@ state = cbdataAlloc(StatObjectsState); state->sentry = sentry; state->filter = filter; + state->cur_entry = (StoreEntry *) store_list.head->data; + /* This is so it doesn't get riped out from beneath us! */ + storeLockObject(state->cur_entry); storeLockObject(sentry); eventAdd("statObjects", statObjects, state, 0.0, 1); } @@ -376,8 +383,10 @@ { if (e->mem_obj == NULL) return 0; +#if 0 if (e->mem_obj->swapout.sio == NULL) return 0; +#endif return 1; } Index: squid/src/store.c diff -u squid/src/store.c:1.10 squid/src/store.c:1.2.2.21 --- squid/src/store.c:1.10 Thu Feb 22 21:51:08 2001 +++ squid/src/store.c Sun Mar 18 14:33:32 2001 @@ -77,23 +77,20 @@ * local function prototypes */ static int storeEntryValidLength(const StoreEntry *); +#if 0 static void storeGetMemSpace(int); -static void storeHashDelete(StoreEntry *); +#endif static MemObject *new_MemObject(const char *, const char *); static void destroy_MemObject(StoreEntry *); static FREE destroy_StoreEntry; +#if 0 static void storePurgeMem(StoreEntry *); -static void storeEntryReferenced(StoreEntry *); -static void storeEntryDereferenced(StoreEntry *); -static int getKeyCounter(void); -static int storeKeepInMemory(const StoreEntry *); +#endif static OBJH storeCheckCachableStats; -static EVH storeLateRelease; /* * local variables */ -static Stack LateReleaseStack; #if URL_CHECKSUM_DEBUG unsigned int @@ -132,8 +129,8 @@ { StoreEntry *e = NULL; e = memAllocate(MEM_STOREENTRY); - if (mem_obj_flag) - e->mem_obj = new_MemObject(url, log_url); + /* We're always creating MemObjects now! */ + e->mem_obj = new_MemObject(url, log_url); debug(20, 3) ("new_StoreEntry: returning %p\n", e); e->expires = e->lastmod = e->lastref = e->timestamp = -1; e->swap_filen = -1; @@ -150,16 +147,15 @@ #if URL_CHECKSUM_DEBUG assert(mem->chksum == url_checksum(mem->url)); #endif + + e->mem_obj = NULL; - if (!shutting_down) - assert(mem->swapout.sio == NULL); - stmemFree(&mem->data_hdr); - mem->inmem_hi = 0; + /* * There is no way to abort FD-less clients, so they might * still have mem->clients set if mem->fd == -1 */ - assert(mem->fd == -1 || mem->clients.head == NULL); + assert(mem->fd == -1 || e->sc.valid == 0); httpReplyDestroy(mem->reply); requestUnlink(mem->request); mem->request = NULL; @@ -175,102 +171,55 @@ StoreEntry *e = data; debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e); assert(e != NULL); - if (e->mem_obj) - destroy_MemObject(e); - storeHashDelete(e); - assert(e->hash.key == NULL); - memFree(e, MEM_STOREENTRY); -} + assert(e->mem_obj != NULL); -/* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ + dlinkDelete(&e->node, &store_list); -void -storeHashInsert(StoreEntry * e, const cache_key * key) -{ - debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", - e, storeKeyText(key)); - e->hash.key = storeKeyDup(key); - hash_join(store_table, &e->hash); -} + /* Free the reply .. */ + storeDetachReply(e); -static void -storeHashDelete(StoreEntry * e) -{ - hash_remove_link(store_table, &e->hash); - storeKeyFree(e->hash.key); - e->hash.key = NULL; + /* Scrape it clean .. */ + destroy_MemObject(e); + + memFree(e, MEM_STOREENTRY); } /* -------------------------------------------------------------------------- */ +#if 0 /* get rid of memory copy of the object */ /* Only call this if storeCheckPurgeMem(e) returns 1 */ static void storePurgeMem(StoreEntry * e) { - if (e->mem_obj == NULL) - return; + assert(e->mem_obj != NULL); debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", - storeKeyText(e->hash.key)); - storeSetMemStatus(e, NOT_IN_MEMORY); + storeKeyUrl(e)); destroy_MemObject(e); if (e->swap_status != SWAPOUT_DONE) storeRelease(e); } +#endif -static void -storeEntryReferenced(StoreEntry * e) -{ - SwapDir *SD; - - /* Notify the fs that we're referencing this object again */ - if (e->swap_dirn > -1) { - SD = INDEXSD(e->swap_dirn); - if (SD->refobj) - SD->refobj(SD, e); - } - /* Notify the memory cache that we're referencing this object again */ - if (e->mem_obj) { - if (mem_policy->Referenced) - mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl); - } -} - -static void -storeEntryDereferenced(StoreEntry * e) -{ - SwapDir *SD; - - /* Notify the fs that we're not referencing this object any more */ - if (e->swap_filen > -1) { - SD = INDEXSD(e->swap_dirn); - if (SD->unrefobj != NULL) - SD->unrefobj(SD, e); - } - /* Notify the memory cache that we're not referencing this object any more */ - if (e->mem_obj) { - if (mem_policy->Dereferenced) - mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl); - } -} void storeLockObject(StoreEntry * e) { + assert(e->mem_obj != NULL); e->lock_count++; debug(20, 3) ("storeLockObject: key '%s' count=%d\n", - storeKeyText(e->hash.key), (int) e->lock_count); + storeKeyUrl(e), (int) e->lock_count); e->lastref = squid_curtime; - storeEntryReferenced(e); } void storeReleaseRequest(StoreEntry * e) { + assert(e->mem_obj != NULL); if (EBIT_TEST(e->flags, RELEASE_REQUEST)) return; - debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyUrl(e)); EBIT_SET(e->flags, RELEASE_REQUEST); /* * Clear cachable flag here because we might get called before @@ -278,6 +227,15 @@ * prevents httpMakePublic from really setting a public key. */ EBIT_CLR(e->flags, ENTRY_CACHABLE); + /* + * Notify the reply layer that we want this object released.. + * XXX an evil hack - this can be called before the reply object + * is attached. If this happens, don't try to notify the reply + * object. When the reply object is created by createNewStoreEntry + * we'll be ok. -- adrian + */ + if (e->strelease != NULL) + e->strelease(e); storeSetPrivateKey(e); } @@ -286,28 +244,30 @@ int storeUnlockObject(StoreEntry * e) { + assert(e->mem_obj != NULL); e->lock_count--; debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", - storeKeyText(e->hash.key), e->lock_count); + storeKeyUrl(e), e->lock_count); if (e->lock_count) return (int) e->lock_count; - if (e->store_status == STORE_PENDING) + + /* If the object is still pending data, we set RELEASE_REQUEST and wait */ + if (e->store_status == STORE_PENDING) { EBIT_SET(e->flags, RELEASE_REQUEST); + return 0; + } + + /* The object is not pending data, so kill it .. */ assert(storePendingNClients(e) == 0); + assert(e->store_status == STORE_OK); + + /* Only release an object if we've been requested to.. */ if (EBIT_TEST(e->flags, RELEASE_REQUEST)) - storeRelease(e); - else if (storeKeepInMemory(e)) { - storeEntryDereferenced(e); - storeSetMemStatus(e, IN_MEMORY); - requestUnlink(e->mem_obj->request); - e->mem_obj->request = NULL; - } else { - storePurgeMem(e); - storeEntryDereferenced(e); - if (EBIT_TEST(e->flags, KEY_PRIVATE)) - debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__); - } - return 0; + storeRelease(e); + else + destroy_StoreEntry(e); + + return -1; } /* Lookup an object in the cache. @@ -316,7 +276,12 @@ storeGet(const cache_key * key) { debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); + return NULL; + + /* XXX we will perform a FS lookup here .. */ +#if 0 return (StoreEntry *) hash_lookup(store_table, key); +#endif } StoreEntry * @@ -325,43 +290,23 @@ return storeGet(storeKeyPublic(uri, method)); } -static int -getKeyCounter(void) -{ - static int key_counter = 0; - if (++key_counter < 0) - key_counter = 1; - return key_counter; -} - void storeSetPrivateKey(StoreEntry * e) { - const cache_key *newkey; - MemObject *mem = e->mem_obj; - if (e->hash.key && EBIT_TEST(e->flags, KEY_PRIVATE)) + if (EBIT_TEST(e->flags, KEY_PRIVATE)) return; /* is already private */ - if (e->hash.key) { - if (e->swap_filen > -1) - storeDirSwapLog(e, SWAP_LOG_DEL); - storeHashDelete(e); - } - if (mem != NULL) { - mem->id = getKeyCounter(); - newkey = storeKeyPrivate(mem->url, mem->method, mem->id); - } else { - newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter()); - } - assert(hash_lookup(store_table, newkey) == NULL); EBIT_SET(e->flags, KEY_PRIVATE); - storeHashInsert(e, newkey); + /* XXX this lame check can go !!! -- adrian */ + if (e->stmakeprivate) + e->stmakeprivate(e); } void storeSetPublicKey(StoreEntry * e) { +#if 0 StoreEntry *e2 = NULL; - const cache_key *newkey; +#endif MemObject *mem = e->mem_obj; if (e->hash.key && !EBIT_TEST(e->flags, KEY_PRIVATE)) return; /* is already public */ @@ -382,7 +327,12 @@ e->hash.key, mem->url); #endif assert(!EBIT_TEST(e->flags, RELEASE_REQUEST)); - newkey = storeKeyPublic(mem->url, mem->method); +#if 0 + /* + * We don't need this anymore, key collisions should be handled in + * the object store(s) + * -- adrian + */ if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); storeSetPrivateKey(e2); @@ -391,14 +341,18 @@ } if (e->hash.key) storeHashDelete(e); +#endif EBIT_CLR(e->flags, KEY_PRIVATE); - storeHashInsert(e, newkey); + /* XXX this lame check can go !!! -- adrian */ + if (e->stmakepublic) + e->stmakepublic(e); if (e->swap_filen > -1) storeDirSwapLog(e, SWAP_LOG_ADD); } StoreEntry * -storeCreateEntry(const char *url, const char *log_url, request_flags flags, method_t method) +storeCreateEntry(const char *url, const char *log_url, request_flags flags, + method_t method, reply_obj_t reply_obj) { StoreEntry *e = NULL; MemObject *mem = NULL; @@ -408,6 +362,20 @@ e->lock_count = 1; /* Note lock here w/o calling storeLock() */ mem = e->mem_obj; mem->method = method; + dlinkAdd(e, &e->node, &store_list); + + switch (reply_obj) { + case REPLY_OBJ_INTERNAL: + reply_internal_create(e); + break; + case REPLY_OBJ_NETWORK: + reply_network_create(e); + break; + default: + fatal("No valid reply_obj given to storeCreateEntry!\n"); + /* NOTREACHED */ + } + if (neighbors_do_private_keys || !flags.hierarchical) storeSetPrivateKey(e); else @@ -419,8 +387,8 @@ EBIT_CLR(e->flags, ENTRY_CACHABLE); storeReleaseRequest(e); } + e->store_status = STORE_PENDING; - storeSetMemStatus(e, NOT_IN_MEMORY); e->swap_status = SWAPOUT_NONE; e->swap_filen = -1; e->swap_dirn = -1; @@ -428,7 +396,6 @@ e->lastref = squid_curtime; e->timestamp = -1; /* set in storeTimestampsSet() */ e->ping_status = PING_NONE; - EBIT_SET(e->flags, ENTRY_VALIDATED); return e; } @@ -436,7 +403,7 @@ void storeExpireNow(StoreEntry * e) { - debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyUrl(e)); e->expires = squid_curtime; } @@ -444,22 +411,7 @@ void storeAppend(StoreEntry * e, const char *buf, int len) { - MemObject *mem = e->mem_obj; - assert(mem != NULL); - assert(len >= 0); - assert(e->store_status == STORE_PENDING); - if (len) { - debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", - len, - storeKeyText(e->hash.key)); - storeGetMemSpace(len); - stmemAppend(&mem->data_hdr, buf, len); - mem->inmem_hi += len; - } - if (EBIT_TEST(e->flags, DELAY_SENDING)) - return; - InvokeHandlers(e); - storeSwapOut(e); + e->stappend(e, buf, len); } void @@ -562,7 +514,7 @@ return 0; /* avoid release call below */ } else if ((e->mem_obj->reply->content_length > 0 && e->mem_obj->reply->content_length > Config.Store.maxObjectSize) || - e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { + storeMemHiOffset(e) > Config.Store.maxObjectSize) { debug(20, 2) ("storeCheckCachable: NO: too big\n"); store_check_cachable_hist.no.too_big++; } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) { @@ -629,17 +581,17 @@ void storeComplete(StoreEntry * e) { - debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeComplete: '%s'\n", storeKeyUrl(e)); if (e->store_status != STORE_PENDING) { /* * if we're not STORE_PENDING, then probably we got aborted * and there should be NO clients on this entry */ assert(EBIT_TEST(e->flags, ENTRY_ABORTED)); - assert(e->mem_obj->nclients == 0); + assert(e->sc.valid == 0); return; } - e->mem_obj->object_sz = e->mem_obj->inmem_hi; + e->mem_obj->object_sz = storeMemHiOffset(e); e->store_status = STORE_OK; assert(e->mem_status == NOT_IN_MEMORY); if (!storeEntryValidLength(e)) { @@ -656,7 +608,6 @@ * responses without content length would sometimes get released * in client_side, thinking that the response is incomplete. */ - storeSwapOut(e); InvokeHandlers(e); } @@ -671,18 +622,17 @@ MemObject *mem = e->mem_obj; assert(e->store_status == STORE_PENDING); assert(mem != NULL); - debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->hash.key)); + debug(20, 6) ("storeAbort: %s\n", storeKeyUrl(e)); storeLockObject(e); /* lock while aborting */ storeNegativeCache(e); storeReleaseRequest(e); EBIT_SET(e->flags, ENTRY_ABORTED); - storeSetMemStatus(e, NOT_IN_MEMORY); e->store_status = STORE_OK; /* * We assign an object length here. The only other place we assign * the object length is in storeComplete() */ - mem->object_sz = mem->inmem_hi; + mem->object_sz = storeMemHiOffset(e); /* Notify the server side */ if (mem->abort.callback) { eventAdd("mem->abort.callback", @@ -696,10 +646,11 @@ /* Notify the client side */ InvokeHandlers(e); /* Close any swapout file */ - storeSwapOutFileClose(e); storeUnlockObject(e); /* unlock */ } +/* This function isn't used right now .. -- adrian */ +#if 0 /* Clear Memory storage to accommodate the given object len */ static void storeGetMemSpace(int size) @@ -729,10 +680,8 @@ debug(20, 3) (" %6d HOT objects\n", hot_obj_count); debug(20, 3) (" %6d were released\n", released); } +#endif -/* The maximum objects to scan for maintain storage space */ -#define MAINTAIN_MAX_SCAN 1024 -#define MAINTAIN_MAX_REMOVE 64 /* * This routine is to be called by main loop in main.c. @@ -774,7 +723,8 @@ void storeRelease(StoreEntry * e) { - debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyUrl(e)); + /* If, for any reason we can't discard this object because of an * outstanding request, mark it for pending release */ if (storeEntryLocked(e)) { @@ -783,81 +733,24 @@ storeReleaseRequest(e); return; } - if (store_dirs_rebuilding && e->swap_filen > -1) { - storeSetPrivateKey(e); - if (e->mem_obj) { - storeSetMemStatus(e, NOT_IN_MEMORY); - destroy_MemObject(e); - } - if (e->swap_filen > -1) { - /* - * Fake a call to storeLockObject(). When rebuilding is done, - * we'll just call storeUnlockObject() on these. - */ - e->lock_count++; - EBIT_SET(e->flags, RELEASE_REQUEST); - stackPush(&LateReleaseStack, e); - return; - } else { - destroy_StoreEntry(e); - } - } + + /* Notify the reply layer to release this object */ + if (!EBIT_TEST(e->flags, RELEASE_REQUEST)) + e->strelease(e); + + /* Time to deallocate the storeentry .. */ storeLog(STORE_LOG_RELEASE, e); - if (e->swap_filen > -1) { - storeUnlink(e); - if (e->swap_status == SWAPOUT_DONE) - if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) - storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, -1); - if (!EBIT_TEST(e->flags, KEY_PRIVATE)) - storeDirSwapLog(e, SWAP_LOG_DEL); -#if 0 - /* From 2.4. I think we do this in storeUnlink? */ - storeSwapFileNumberSet(e, -1); -#endif - } - storeSetMemStatus(e, NOT_IN_MEMORY); + + /* Here we'd log a swap log delete, but thats done at the FS layer .. */ destroy_StoreEntry(e); } -static void -storeLateRelease(void *unused) -{ - StoreEntry *e; - int i; - static int n = 0; - if (store_dirs_rebuilding) { - eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); - return; - } - for (i = 0; i < 10; i++) { - e = stackPop(&LateReleaseStack); - if (e == NULL) { - /* done! */ - debug(20, 1) ("storeLateRelease: released %d objects\n", n); - return; - } - storeUnlockObject(e); - n++; - } - eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1); -} - /* return 1 if a store entry is locked */ int storeEntryLocked(const StoreEntry * e) { if (e->lock_count) return 1; - if (e->swap_status == SWAPOUT_WRITING) - return 1; - if (e->store_status == STORE_PENDING) - return 1; - /* - * SPECIAL, PUBLIC entries should be "locked" - */ - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) - if (!EBIT_TEST(e->flags, KEY_PRIVATE)) - return 1; return 0; } @@ -868,7 +761,7 @@ const HttpReply *reply; assert(e->mem_obj != NULL); reply = e->mem_obj->reply; - debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyUrl(e)); debug(20, 5) ("storeEntryValidLength: object_len = %d\n", objectLen(e)); debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", @@ -877,17 +770,17 @@ reply->content_length); if (reply->content_length < 0) { debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 1; } if (reply->hdr_sz == 0) { debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 1; } if (e->mem_obj->method == METHOD_HEAD) { debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 1; } if (reply->sline.status == HTTP_NOT_MODIFIED) @@ -900,7 +793,7 @@ debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", diff < 0 ? -diff : diff, diff < 0 ? "big" : "small", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 0; } @@ -925,15 +818,13 @@ void storeInit(void) { + store_list.head = NULL; + store_list.tail = NULL; storeKeyInit(); storeInitHashValues(); - store_table = hash_create(storeKeyHashCmp, - store_hash_buckets, storeKeyHashHash); mem_policy = createRemovalPolicy(Config.memPolicy); storeDigestInit(); storeLogOpen(); - stackInit(&LateReleaseStack); - eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); storeDirInit(); storeRebuildStart(); cachemgrRegister("storedir", @@ -957,17 +848,6 @@ store_pages_max = Config.memMaxSize / SM_PAGE_SIZE; } -static int -storeKeepInMemory(const StoreEntry * e) -{ - MemObject *mem = e->mem_obj; - if (mem == NULL) - return 0; - if (mem->data_hdr.head == NULL) - return 0; - return mem->inmem_lo == 0; -} - void storeNegativeCache(StoreEntry * e) { @@ -978,8 +858,6 @@ void storeFreeMemory(void) { - hashFreeItems(store_table, destroy_StoreEntry); - hashFreeMemory(store_table); store_table = NULL; #if USE_CACHE_DIGESTS if (store_digest) @@ -1061,23 +939,9 @@ void storeMemObjectDump(MemObject * mem) { - debug(20, 1) ("MemObject->data.head: %p\n", - mem->data_hdr.head); - debug(20, 1) ("MemObject->data.tail: %p\n", - mem->data_hdr.tail); - debug(20, 1) ("MemObject->data.origin_offset: %d\n", - mem->data_hdr.origin_offset); debug(20, 1) ("MemObject->start_ping: %d.%06d\n", (int) mem->start_ping.tv_sec, (int) mem->start_ping.tv_usec); - debug(20, 1) ("MemObject->inmem_hi: %d\n", - (int) mem->inmem_hi); - debug(20, 1) ("MemObject->inmem_lo: %d\n", - (int) mem->inmem_lo); - debug(20, 1) ("MemObject->clients: %p\n", - mem->clients); - debug(20, 1) ("MemObject->nclients: %d\n", - mem->nclients); debug(20, 1) ("MemObject->reply: %p\n", mem->reply); debug(20, 1) ("MemObject->request: %p\n", @@ -1090,8 +954,7 @@ void storeEntryDump(const StoreEntry * e, int l) { - debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->hash.key)); - debug(20, l) ("StoreEntry->next: %p\n", e->hash.next); + debug(20, l) ("StoreEntry->key: %s\n", storeKeyUrl(e)); debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj); debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp); debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref); @@ -1109,40 +972,6 @@ debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status); } -/* - * NOTE, this function assumes only two mem states - */ -void -storeSetMemStatus(StoreEntry * e, int new_status) -{ - MemObject *mem = e->mem_obj; - if (new_status == e->mem_status) - return; - assert(mem != NULL); - if (new_status == IN_MEMORY) { - assert(mem->inmem_lo == 0); - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n", - mem->url); - } else { - mem_policy->Add(mem_policy, e, &mem->repl); - debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n", - mem->url); - } - hot_obj_count++; - } else { - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - debug(20, 4) ("storeSetMemStatus: special entry %s\n", - mem->url); - } else { - mem_policy->Remove(mem_policy, e, &mem->repl); - debug(20, 4) ("storeSetMemStatus: removed mem node %s\n", - mem->url); - } - hot_obj_count--; - } - e->mem_status = new_status; -} const char * storeUrl(const StoreEntry * e) @@ -1155,6 +984,13 @@ return e->mem_obj->url; } + +const char * +storeKeyUrl(const StoreEntry *e) +{ + return storeUrl(e); +} + void storeCreateMemObject(StoreEntry * e, const char *url, const char *log_url) { @@ -1176,7 +1012,6 @@ { EBIT_CLR(e->flags, DELAY_SENDING); InvokeHandlers(e); - storeSwapOut(e); } int @@ -1209,9 +1044,7 @@ { MemObject *mem = e->mem_obj; debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e)); - assert(mem->swapout.sio == NULL); - stmemFree(&mem->data_hdr); - mem->inmem_hi = mem->inmem_lo = 0; + /* XXX hrm, we have to reset the reply object? -- adrian */ httpReplyDestroy(mem->reply); mem->reply = httpReplyCreate(); e->expires = e->lastmod = e->timestamp = -1; @@ -1299,7 +1132,24 @@ debug(20, 1) ("ERROR: Be sure to have set cache_replacement_policy\n"); debug(20, 1) ("ERROR: and memory_replacement_policy in squid.conf!\n"); fatalf("ERROR: Unknown policy %s\n", settings->type); - return NULL; /* NOTREACHED */ + return NULL; /* NOTREACHED */ +} + +/* + * Peek into the amount of an object that is in memory + */ +off_t +storeMemHiOffset(const StoreEntry *e) +{ + assert(e->mem_obj != NULL); + return e->stmemhi(e); +} + +off_t +storeMemLoOffset(const StoreEntry *e) +{ + assert(e->mem_obj != NULL); + return e->stmemlo(e); } #if 0 @@ -1320,3 +1170,49 @@ } } #endif + + + +/* + * storeDetachReply() - detach a reply from a MemObject + */ +void +storeDetachReply(StoreEntry *e) +{ + STRDONE *stdone; + + assert (e != NULL); + + stdone = e->stdone; + if (stdone == NULL) + return; + + stdone(e); + + e->stappend = NULL; + e->stflush = NULL; + e->stmemhi = NULL; + e->stmemlo = NULL; + e->stclientcopy = NULL; + e->strelease = NULL; + e->stdone = NULL; + e->stmakeprivate = NULL; + e->stmakepublic = NULL; + e->stdata = NULL; +} + + +/* + * double check the request matches what we are after + * this should be used *before* calling storeLookup to make sure I + * have things 'right'. + */ +void +storeDoubleCheckRequest(request_t *request, char *uri, method_t method) +{ + assert(request->method == method); + /* + * ignore the URI for the time being, as it kinda has to be set + */ + /* assert(strncmp(request->urlpath.buf, uri, request->urlpath.len) == 0); */ +} Index: squid/src/store_client.c diff -u squid/src/store_client.c:1.7 squid/src/store_client.c:1.2.2.13 --- squid/src/store_client.c:1.7 Sat Mar 3 02:44:32 2001 +++ squid/src/store_client.c Sun Mar 18 14:33:32 2001 @@ -40,13 +40,11 @@ * 'Body' refers to the swapfile body, which is the full * HTTP reply (including HTTP headers and body). */ +#if 0 static STRCB storeClientReadBody; static STRCB storeClientReadHeader; -static void storeClientCopy2(StoreEntry * e, store_client * sc); -static void storeClientCopy3(StoreEntry * e, store_client * sc); static void storeClientFileRead(store_client * sc); -static EVH storeClientCopyEvent; -static store_client_t storeClientType(StoreEntry *); +#endif static int CheckQuickAbort2(StoreEntry * entry); static void CheckQuickAbort(StoreEntry * entry); @@ -55,278 +53,92 @@ int storeClientWaiting(const StoreEntry * e) { - MemObject *mem = e->mem_obj; - dlink_node *node; - store_client *sc; - for (node = mem->clients.head; node; node = node->next) { - sc = node->data; - if (sc->callback_data != NULL) + if (e->sc.callback_data != NULL) return 1; - } return 0; } -#if STORE_CLIENT_LIST_DEBUG -store_client * -storeClientListSearch(const MemObject * mem, void *data) -{ - dlink_node *node; - store_client *sc = NULL; - for (node = mem->clients.head; node; node = node->next) { - sc = node->data; - if (sc->callback_data == data) - return sc; - } - return NULL; -} -#endif - -static store_client_t -storeClientType(StoreEntry * e) -{ - MemObject *mem = e->mem_obj; - if (mem->inmem_lo) - return STORE_DISK_CLIENT; - if (EBIT_TEST(e->flags, ENTRY_ABORTED)) { - /* I don't think we should be adding clients to aborted entries */ - debug(20, 1) ("storeClientType: adding to ENTRY_ABORTED entry\n"); - return STORE_MEM_CLIENT; - } - if (e->store_status == STORE_OK) { - if (mem->inmem_lo == 0 && mem->inmem_hi > 0) - return STORE_MEM_CLIENT; - else - return STORE_DISK_CLIENT; - } - /* here and past, entry is STORE_PENDING */ - /* - * If this is the first client, let it be the mem client - */ - else if (mem->nclients == 1) - return STORE_MEM_CLIENT; - /* - * If there is no disk file to open yet, we must make this a - * mem client. If we can't open the swapin file before writing - * to the client, there is no guarantee that we will be able - * to open it later when we really need it. - */ - else if (e->swap_status == SWAPOUT_NONE) - return STORE_MEM_CLIENT; - /* - * otherwise, make subsequent clients read from disk so they - * can not delay the first, and vice-versa. - */ - else - return STORE_DISK_CLIENT; -} /* add client with fd to client list */ -store_client * -storeClientListAdd(StoreEntry * e, void *data) +void +storeClientRegister(StoreEntry * e, void *data) { - MemObject *mem = e->mem_obj; - store_client *sc; - assert(mem); -#if STORE_CLIENT_LIST_DEBUG - if (storeClientListSearch(mem, data) != NULL) - assert(1 == 0); /* XXX die! */ -#endif e->refcount++; - mem->nclients++; - sc = cbdataAlloc(store_client); - cbdataLock(data); /* locked while we point to it */ - sc->callback_data = data; - sc->seen_offset = 0; - sc->copy_offset = 0; - sc->flags.disk_io_pending = 0; - sc->entry = e; - sc->type = storeClientType(e); - if (sc->type == STORE_DISK_CLIENT) - /* assert we'll be able to get the data we want */ - /* maybe we should open swapin_fd here */ - assert(e->swap_filen > -1 || storeSwapOutAble(e)); - dlinkAdd(sc, &sc->node, &mem->clients); + + assert(e->sc.valid == 0); + e->sc.valid = 1; + e->sc.callback_data = data; + e->sc.seen_offset = 0; + e->sc.copy_offset = 0; + e->sc.flags.disk_io_pending = 0; + e->sc.entry = e; + + /* Lock the data that we're using .. */ + cbdataLock(data); #if DELAY_POOLS - sc->delay_id = 0; + e->sc.delay_id = 0; #endif - return sc; } static void -storeClientCallback(store_client * sc, ssize_t sz) +storeClientCallback(StoreEntry *e, ssize_t sz) { - STCB *callback = sc->callback; - char *buf = sc->copy_buf; - assert(sc->callback); - sc->callback = NULL; - sc->copy_buf = NULL; - if (cbdataValid(sc->callback_data)) - callback(sc->callback_data, buf, sz); -} - -static void -storeClientCopyEvent(void *data) -{ - store_client *sc = data; - debug(20, 3) ("storeClientCopyEvent: Running\n"); - sc->flags.copy_event_pending = 0; - if (!sc->callback) - return; - storeClientCopy2(sc->entry, sc); + STCB *callback = e->sc.callback; + char *buf = e->sc.copy_buf; + assert(e->sc.valid == 1); + assert(e->sc.callback); + e->sc.callback = NULL; + e->sc.copy_buf = NULL; + if (cbdataValid(e->sc.callback_data)) + callback(e->sc.callback_data, buf, sz); } /* copy bytes requested by the client */ void -storeClientCopy(store_client * sc, - StoreEntry * e, +storeClientCopy(StoreEntry * e, off_t seen_offset, off_t copy_offset, size_t size, char *buf, STCB * callback, void *data) -{ +{ + store_client *sc = &e->sc; assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); - debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", - storeKeyText(e->hash.key), - (int) seen_offset, - (int) copy_offset, - (int) size, - callback, - data); - assert(sc != NULL); -#if STORE_CLIENT_LIST_DEBUG - assert(sc == storeClientListSearch(e->mem_obj, data)); -#endif + + assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); + assert(sc->valid == 1); + debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", storeKeyUrl(e), + (int) seen_offset, + (int) copy_offset, + (int) size, + callback, + data); + assert(sc->callback == NULL); assert(sc->entry == e); + assert(sc->copy_offset == sc->seen_offset); /* Just until everything has been changed */ sc->copy_offset = copy_offset; sc->seen_offset = seen_offset; sc->callback = callback; sc->copy_buf = buf; sc->copy_size = size; sc->copy_offset = copy_offset; - storeClientCopy2(e, sc); -} -/* - * This function is used below to decide if we have any more data to - * send to the client. If the store_status is STORE_PENDING, then we - * do have more data to send. If its STORE_OK, then - * we continue checking. If the object length is negative, then we - * don't know the real length and must open the swap file to find out. - * If the length is >= 0, then we compare it to the requested copy - * offset. - */ -static int -storeClientNoMoreToSend(StoreEntry * e, store_client * sc) -{ - ssize_t len; - if (e->store_status == STORE_PENDING) - return 0; - if ((len = objectLen(e)) < 0) - return 0; - if (sc->copy_offset < len) - return 0; - return 1; + e->stclientcopy(e); } -static void -storeClientCopy2(StoreEntry * e, store_client * sc) -{ - if (sc->flags.copy_event_pending) - return; - if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) { - debug(20, 5) ("storeClientCopy2: returning because ENTRY_FWD_HDR_WAIT set\n"); - return; - } - if (sc->flags.store_copying) { - sc->flags.copy_event_pending = 1; - debug(20, 3) ("storeClientCopy2: Queueing storeClientCopyEvent()\n"); - eventAdd("storeClientCopyEvent", storeClientCopyEvent, sc, 0.0, 0); - return; - } - cbdataLock(sc); /* ick, prevent sc from getting freed */ - sc->flags.store_copying = 1; - debug(20, 3) ("storeClientCopy2: %s\n", storeKeyText(e->hash.key)); - assert(sc->callback != NULL); - /* - * We used to check for ENTRY_ABORTED here. But there were some - * problems. For example, we might have a slow client (or two) and - * the server-side is reading far ahead and swapping to disk. Even - * if the server-side aborts, we want to give the client(s) - * everything we got before the abort condition occurred. - */ - storeClientCopy3(e, sc); - sc->flags.store_copying = 0; - cbdataUnlock(sc); /* ick, allow sc to be freed */ -} -static void -storeClientCopy3(StoreEntry * e, store_client * sc) +int +storeClientCopyPending(StoreEntry * e, void *data) { - MemObject *mem = e->mem_obj; - size_t sz; - - if (storeClientNoMoreToSend(e, sc)) { - /* There is no more to send! */ - storeClientCallback(sc, 0); - return; - } - if (e->store_status == STORE_PENDING && sc->seen_offset >= mem->inmem_hi) { - /* client has already seen this, wait for more */ - debug(20, 3) ("storeClientCopy3: Waiting for more\n"); - return; - } - /* - * Slight weirdness here. We open a swapin file for any - * STORE_DISK_CLIENT, even if we can copy the requested chunk - * from memory in the next block. We must try to open the - * swapin file before sending any data to the client side. If - * we postpone the open, and then can not open the file later - * on, the client loses big time. Its transfer just gets cut - * off. Better to open it early (while the client side handler - * is clientCacheHit) so that we can fall back to a cache miss - * if needed. - */ - if (STORE_DISK_CLIENT == sc->type && NULL == sc->swapin_sio) { - debug(20, 3) ("storeClientCopy3: Need to open swap in file\n"); - /* gotta open the swapin file */ - if (storeTooManyDiskFilesOpen()) { - /* yuck -- this causes a TCP_SWAPFAIL_MISS on the client side */ - storeClientCallback(sc, -1); - return; - } else if (!sc->flags.disk_io_pending) { - /* Don't set store_io_pending here */ - storeSwapInStart(sc); - if (NULL == sc->swapin_sio) { - storeClientCallback(sc, -1); - return; - } - /* - * If the open succeeds we either copy from memory, or - * schedule a disk read in the next block. - */ - } else { - debug(20, 1) ("WARNING: Averted multiple fd operation (1)\n"); - return; - } - } - if (sc->copy_offset >= mem->inmem_lo && sc->copy_offset < mem->inmem_hi) { - /* What the client wants is in memory */ - debug(20, 3) ("storeClientCopy3: Copying from memory\n"); - sz = stmemCopy(&mem->data_hdr, - sc->copy_offset, sc->copy_buf, sc->copy_size); - storeClientCallback(sc, sz); - return; - } - /* What the client wants is not in memory. Schedule a disk read */ - assert(STORE_DISK_CLIENT == sc->type); - assert(!sc->flags.disk_io_pending); - debug(20, 3) ("storeClientCopy3: reading from STORE\n"); - storeClientFileRead(sc); + if (e->sc.callback == NULL) + return 0; + return 1; } + +#if 0 static void storeClientFileRead(store_client * sc) { @@ -343,7 +155,9 @@ sc); } else { if (sc->entry->swap_status == SWAPOUT_WRITING) +#if 0 assert(storeOffset(mem->swapout.sio) > sc->copy_offset + mem->swap_hdr_sz); +#endif storeRead(sc->swapin_sio, sc->copy_buf, sc->copy_size, @@ -471,129 +285,68 @@ storeClientFileRead(sc); } -int -storeClientCopyPending(store_client * sc, StoreEntry * e, void *data) -{ -#if STORE_CLIENT_LIST_DEBUG - assert(sc == storeClientListSearch(e->mem_obj, data)); #endif - assert(sc->entry == e); - if (sc == NULL) - return 0; - if (sc->callback == NULL) - return 0; - return 1; -} + /* - * This routine hasn't been optimised to take advantage of the - * passed sc. Yet. + * Disassociate a client from a storeentry */ int -storeUnregister(store_client * sc, StoreEntry * e, void *data) +storeClientUnregister(StoreEntry * e, void *data) { + store_client *sc = &e->sc; MemObject *mem = e->mem_obj; -#if STORE_CLIENT_LIST_DEBUG - assert(sc == storeClientListSearch(e->mem_obj, data)); -#endif - if (mem == NULL) - return 0; - debug(20, 3) ("storeUnregister: called for '%s'\n", storeKeyText(e->hash.key)); + + debug(20, 3) ("storeUnregister: called for '%s'\n", storeKeyUrl(e)); + if (sc == NULL) return 0; - if (mem->clients.head == NULL) - return 0; - if (sc == mem->clients.head->data) { - /* - * If we are unregistering the _first_ client for this - * entry, then we have to reset the client FD to -1. - */ - mem->fd = -1; - } - dlinkDelete(&sc->node, &mem->clients); - mem->nclients--; - if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) - storeSwapOut(e); - if (sc->swapin_sio) { - storeClose(sc->swapin_sio); - cbdataUnlock(sc->swapin_sio); - sc->swapin_sio = NULL; - statCounter.swap.ins++; - } + + assert(sc == &e->sc); + assert(mem != NULL); + assert(e->sc.valid == 1); + + mem->fd = -1; + if (NULL != sc->callback) { /* callback with ssize = -1 to indicate unexpected termination */ debug(20, 3) ("storeUnregister: store_client for %s has a callback\n", mem->url); - storeClientCallback(sc, -1); + storeClientCallback(e, -1); } #if DELAY_POOLS delayUnregisterDelayIdPtr(&sc->delay_id); #endif + e->sc.valid = 0; cbdataUnlock(sc->callback_data); /* we're done with it now */ - /*assert(!sc->flags.disk_io_pending); */ - cbdataFree(sc); assert(e->lock_count > 0); - if (mem->nclients == 0) - CheckQuickAbort(e); + CheckQuickAbort(e); return 1; } + +/* .. and this routine becomes meaningless.. :) */ off_t storeLowestMemReaderOffset(const StoreEntry * entry) { - const MemObject *mem = entry->mem_obj; - off_t lowest = mem->inmem_hi + 1; - store_client *sc; - dlink_node *nx = NULL; - dlink_node *node; - - for (node = mem->clients.head; node; node = nx) { - sc = node->data; - nx = node->next; - if (sc->callback_data == NULL) /* open slot */ - continue; - if (sc->type != STORE_MEM_CLIENT) - continue; - if (sc->type == STORE_DISK_CLIENT) - if (NULL != sc->swapin_sio) - continue; - if (sc->copy_offset < lowest) - lowest = sc->copy_offset; - } - return lowest; + return entry->sc.copy_offset; } /* Call handlers waiting for data to be appended to E. */ void InvokeHandlers(StoreEntry * e) { - int i = 0; - MemObject *mem = e->mem_obj; - store_client *sc; - dlink_node *nx = NULL; - dlink_node *node; - - debug(20, 3) ("InvokeHandlers: %s\n", storeKeyText(e->hash.key)); - /* walk the entire list looking for valid callbacks */ - for (node = mem->clients.head; node; node = nx) { - sc = node->data; - nx = node->next; - debug(20, 3) ("InvokeHandlers: checking client #%d\n", i++); - if (sc->callback_data == NULL) - continue; - if (sc->callback == NULL) - continue; - if (sc->flags.disk_io_pending) - continue; - storeClientCopy2(e, sc); - } + assert(e->mem_obj != NULL); + debug(20, 3) ("InvokeHandlers: %s\n", storeKeyUrl(e)); + e->stflush(e); } int storePendingNClients(const StoreEntry * e) { - MemObject *mem = e->mem_obj; - int npend = NULL == mem ? 0 : mem->nclients; + int npend = 0; + if (e->sc.valid) + npend = 1; debug(20, 3) ("storePendingNClients: returning %d\n", npend); return npend; } @@ -617,7 +370,7 @@ return 1; } expectlen = mem->reply->content_length + mem->reply->hdr_sz; - curlen = (int) mem->inmem_hi; + curlen = (int) storeMemHiOffset(entry); minlen = (int) Config.quickAbort.min << 10; if (minlen < 0) { debug(20, 3) ("CheckQuickAbort2: NO disabled\n"); Index: squid/src/store_digest.c diff -u squid/src/store_digest.c:1.9 squid/src/store_digest.c:1.2.2.7 --- squid/src/store_digest.c:1.9 Sat Mar 3 02:44:32 2001 +++ squid/src/store_digest.c Sun Mar 18 14:33:32 2001 @@ -350,10 +350,14 @@ url = internalLocalUri("/squid-internal-periodic/", StoreDigestFileName); flags = null_request_flags; flags.cachable = 1; - e = storeCreateEntry(url, url, flags, METHOD_GET); + sd_state.rewrite_lock = CBDATA_ALLOC(generic_cbdata, NULL); assert(e); sd_state.rewrite_lock = cbdataAlloc(generic_cbdata); sd_state.rewrite_lock->data = e; + e = storeCreateEntry(url, url, flags, METHOD_GET, + REPLY_OBJ_NETWORK); + assert(sd_state.rewrite_lock); + cbdataAdd(sd_state.rewrite_lock, NULL, 0); debug(71, 3) ("storeDigestRewrite: url: %s key: %s\n", url, storeKeyText(e->hash.key)); e->mem_obj->request = requestLink(urlParse(METHOD_GET, url)); /* wait for rebuild (if any) to finish */ Index: squid/src/store_dir.c diff -u squid/src/store_dir.c:1.14 squid/src/store_dir.c:1.2.2.7 --- squid/src/store_dir.c:1.14 Tue Mar 13 14:49:49 2001 +++ squid/src/store_dir.c Sun Mar 18 14:33:32 2001 @@ -197,6 +197,7 @@ continue; if (load > least_load) continue; + /* Only use leastsize if the load is equal */ cur_free = SD->max_size - SD->cur_size; /* If the load is equal, then look in more details */ if (load == least_load) { @@ -252,7 +253,7 @@ assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX); debug(20, 3) ("storeDirSwapLog: %s %s %d %08X\n", swap_log_op_str[op], - storeKeyText(e->hash.key), + storeKeyUrl(e), e->swap_dirn, e->swap_filen); sd = &Config.cacheSwap.swapDirs[e->swap_dirn]; @@ -371,11 +372,11 @@ SwapDir *sd; int dirn; int notdone = 1; - if (store_dirs_rebuilding) { - debug(20, 1) ("Not currently OK to rewrite swap log.\n"); - debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n"); - return 0; - } + /* + * We used to check here whether we were rebuilding the store dirs. + * If we were, we didn't attempt to write clean logs. Now, we let the + * filesystems decide that for themselves in sd->log.clean.start(). + */ debug(20, 1) ("storeDirWriteCleanLogs: Starting...\n"); getCurrentTime(); start = current_time; Index: squid/src/store_io.c diff -u squid/src/store_io.c:1.2 squid/src/store_io.c:1.2.2.2 --- squid/src/store_io.c:1.2 Sat Oct 21 08:16:13 2000 +++ squid/src/store_io.c Tue Feb 6 02:08:04 2001 @@ -18,7 +18,8 @@ * to select different polices depending on object size or type. */ storeIOState * -storeCreate(StoreEntry * e, STIOCB * file_callback, STIOCB * close_callback, void *callback_data) +storeCreate(StoreEntry * e, STIOCB * file_callback, STIOCB * close_callback, + void *callback_data) { size_t objsize; sdirno dirn; @@ -29,7 +30,7 @@ /* This is just done for logging purposes */ objsize = objectLen(e); if (objsize != -1) - objsize += e->mem_obj->swap_hdr_sz; + objsize += e->mem_obj->swap_hdr_sz; /* * Pick the swapdir @@ -37,32 +38,65 @@ */ dirn = storeDirSelectSwapDir(e); if (dirn == -1) { - debug(20, 2) ("storeCreate: no valid swapdirs for this object\n"); - store_io_stats.create.select_fail++; - return NULL; + debug(20, 2) ("storeCreate: no valid swapdirs for this object\n"); + store_io_stats.create.select_fail++; + return NULL; } - debug(20, 2) ("storeCreate: Selected dir '%d' for obj size '%d'\n", dirn, objsize); + debug(20, 2) ("storeCreate: Selected dir '%d' for obj size '%d'\n", dirn, + objsize); SD = &Config.cacheSwap.swapDirs[dirn]; /* Now that we have a fs to use, call its storeCreate function */ sio = SD->obj.create(SD, e, file_callback, close_callback, callback_data); if (NULL == sio) - store_io_stats.create.create_fail++; + store_io_stats.create.create_fail++; else - store_io_stats.create.success++; + store_io_stats.create.success++; return sio; } + +/* + * storeLookup - the eventual replacement for storeGet()/storeGetPublic() + * + * This function handles issuing lookups to each of the object stores. + * A callback/data pair will be called with the StoreEntry referencing + * the object if one is found, or NULL is returned. + * + * Yes, its inefficient right now, but there isn't much we can do about + * it until we've got *something* that works.. + */ +void +storeLookup(request_t *request, STGETDONE *callback, void *callback_data) +{ + /* Only begin a lookup in StoreDir 0 for now */ + SwapDir *sd = INDEXSD(0); + + /* Lock the request and callback */ + cbdataLock(request); + cbdataLock(callback_data); + + sd->obj.get(sd, request, callback, callback_data); +} + /* - * storeOpen() is purely for reading .. + * storeLookupComplete - called at the completion (sucess/failure!) of + * a given lookup. + * + * This function unlocks the request and then issues the callback. */ -storeIOState * -storeOpen(StoreEntry * e, STFNCB * file_callback, STIOCB * callback, - void *callback_data) +void +storeLookupComplete(request_t *request, STGETDONE *callback, + void *callback_data, StoreEntry *e) { - SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn]; - return SD->obj.open(SD, e, file_callback, callback, callback_data); + /* Only callback if the request_t and the callback are valid! */ + if (cbdataValid(request) && cbdataValid(callback_data)) + callback(callback_data, e); + + /* Unlock, which can possibly free these two entries now */ + cbdataUnlock(request); + cbdataUnlock(callback_data); } void Index: squid/src/store_log.c diff -u squid/src/store_log.c:1.6 squid/src/store_log.c:1.2.2.5 --- squid/src/store_log.c:1.6 Sat Feb 17 12:56:22 2001 +++ squid/src/store_log.c Tue Feb 20 07:39:42 2001 @@ -75,14 +75,14 @@ storeLogTags[tag], e->swap_dirn, e->swap_filen, - storeKeyText(e->hash.key), + storeKeyUrl(e), reply->sline.status, (int) reply->date, (int) reply->last_modified, (int) reply->expires, strLen(reply->content_type) ? strBuf(reply->content_type) : "unknown", reply->content_length, - (int) (mem->inmem_hi - mem->reply->hdr_sz), + (int) (storeMemHiOffset(e) - mem->reply->hdr_sz), RequestMethodStr[mem->method], mem->log_url); } else { @@ -93,7 +93,7 @@ storeLogTags[tag], e->swap_dirn, e->swap_filen, - storeKeyText(e->hash.key)); + storeKeyUrl(e)); } } Index: squid/src/store_rebuild.c diff -u squid/src/store_rebuild.c:1.7 squid/src/store_rebuild.c:1.2.2.6 --- squid/src/store_rebuild.c:1.7 Fri Jan 12 00:20:33 2001 +++ squid/src/store_rebuild.c Fri Jan 12 00:46:08 2001 @@ -37,7 +37,6 @@ static struct _store_rebuild_data counts; static struct timeval rebuild_start; -static void storeCleanup(void *); typedef struct { /* total number of "swap.state" entries that will be read */ @@ -55,58 +54,6 @@ return (SD->dblcheck(SD, e)); } -static void -storeCleanup(void *datanotused) -{ - static int bucketnum = -1; - static int validnum = 0; - static int store_errors = 0; - int validnum_start; - StoreEntry *e; - hash_link *link_ptr = NULL; - hash_link *link_next = NULL; - validnum_start = validnum; - while (validnum - validnum_start < 500) { - if (++bucketnum >= store_hash_buckets) { - debug(20, 1) (" Completed Validation Procedure\n"); - debug(20, 1) (" Validated %d Entries\n", validnum); - debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); - store_dirs_rebuilding--; - assert(0 == store_dirs_rebuilding); - if (opt_store_doublecheck) - assert(store_errors == 0); - if (store_digest) - storeDigestNoteStoreReady(); - return; - } - link_next = hash_get_bucket(store_table, bucketnum); - while (NULL != (link_ptr = link_next)) { - link_next = link_ptr->next; - e = (StoreEntry *) link_ptr; - if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) - continue; - /* - * Calling storeRelease() has no effect because we're - * still in 'store_rebuilding' state - */ - if (e->swap_filen < 0) - continue; - if (opt_store_doublecheck) - if (storeCleanupDoubleCheck(e)) - store_errors++; - EBIT_SET(e->flags, ENTRY_VALIDATED); - /* - * Only set the file bit if we know its a valid entry - * otherwise, set it in the validation procedure - */ - storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1); - if ((++validnum & 0x3FFFF) == 0) - debug(20, 1) (" %7d Entries Validated so far.\n", validnum); - } - } - eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1); -} - /* meta data recreated from disk image in swap directory */ void storeRebuildComplete(struct _store_rebuild_data *dc) @@ -122,6 +69,8 @@ counts.badflags += dc->badflags; counts.bad_log_op += dc->bad_log_op; counts.zero_object_sz += dc->zero_object_sz; + store_dirs_rebuilding--; + /* * When store_dirs_rebuilding == 1, it means we are done reading * or scanning all cache_dirs. Now report the stats and start @@ -141,9 +90,8 @@ debug(20, 1) (" %7d Swapfile clashes avoided.\n", counts.clashcount); debug(20, 1) (" Took %3.1f seconds (%6.1f objects/sec).\n", dt, (double) counts.objcount / (dt > 0.0 ? dt : 1.0)); - debug(20, 1) ("Beginning Validation Procedure\n"); - eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1); xfree(RebuildProgress); + store_dirs_rebuilding = 0; RebuildProgress = NULL; } Index: squid/src/store_swapin.c diff -u squid/src/store_swapin.c:1.5 squid/src/store_swapin.c:1.2.2.3 --- squid/src/store_swapin.c:1.5 Fri Jan 12 00:20:33 2001 +++ squid/src/store_swapin.c Fri Jan 12 00:46:08 2001 @@ -38,6 +38,7 @@ static STIOCB storeSwapInFileClosed; static STFNCB storeSwapInFileNotify; +#if 0 void storeSwapInStart(store_client * sc) { @@ -94,3 +95,4 @@ e->swap_filen = sio->swap_filen; e->swap_dirn = sio->swap_dirn; } +#endif Index: squid/src/store_swapout.c diff -u squid/src/store_swapout.c:1.7 squid/src/store_swapout.c:1.2.2.6 --- squid/src/store_swapout.c:1.7 Sat Mar 3 02:44:32 2001 +++ squid/src/store_swapout.c Sun Mar 18 14:33:32 2001 @@ -40,6 +40,12 @@ static STIOCB storeSwapOutFileClosed; static STIOCB storeSwapOutFileNotify; + + +#if 0 + + + /* start swapping object to disk */ static void storeSwapOutStart(StoreEntry * e) @@ -115,16 +121,16 @@ storeSwapOutFileClose(e); return; } - debug(20, 7) ("storeSwapOut: mem->inmem_lo = %d\n", - (int) mem->inmem_lo); - debug(20, 7) ("storeSwapOut: mem->inmem_hi = %d\n", - (int) mem->inmem_hi); + debug(20, 7) ("storeSwapOut: storeMemLoOffset = %d\n", + (int) storeMemLoOffset(e)); + debug(20, 7) ("storeSwapOut: storeMemHiOffset = %d\n", + (int) storeMemHiOffset(e)); debug(20, 7) ("storeSwapOut: swapout.queue_offset = %d\n", (int) mem->swapout.queue_offset); if (mem->swapout.sio) debug(20, 7) ("storeSwapOut: storeOffset() = %d\n", (int) storeOffset(mem->swapout.sio)); - assert(mem->inmem_hi >= mem->swapout.queue_offset); + assert(storeMemHiOffset(e) >= mem->swapout.queue_offset); lowest_offset = storeLowestMemReaderOffset(e); debug(20, 7) ("storeSwapOut: lowest_offset = %d\n", (int) lowest_offset); @@ -132,22 +138,22 @@ * Grab the swapout_size and check to see whether we're going to defer * the swapout based upon size */ - swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset); + swapout_size = (ssize_t) (storeMemHiOffset(e) - mem->swapout.queue_offset); if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) { debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n"); return; } /* - * Careful. lowest_offset can be greater than inmem_hi, such + * Careful. lowest_offset can be greater than storeMemHiOffset, such * as in the case of a range request. */ - if (mem->inmem_hi < lowest_offset) + if (storeMemHiOffset(e) < lowest_offset) new_mem_lo = lowest_offset; - else if (mem->inmem_hi - lowest_offset > Config.Store.maxInMemObjSize) + else if (storeMemHiOffset(e) - lowest_offset > Config.Store.maxInMemObjSize) new_mem_lo = lowest_offset; else - new_mem_lo = mem->inmem_lo; - assert(new_mem_lo >= mem->inmem_lo); + new_mem_lo = storeMemLoOffset(e); + assert(new_mem_lo >= storeMemLoOffset(e)); if (storeSwapOutAble(e)) { /* * We should only free up to what we know has been written @@ -172,7 +178,7 @@ stmemFreeDataUpto(&mem->data_hdr, new_mem_lo); mem->inmem_lo = new_mem_lo; if (e->swap_status == SWAPOUT_WRITING) - assert(mem->inmem_lo <= on_disk); + assert(storeMemLoOffset(e) <= on_disk); if (!storeSwapOutAble(e)) return; debug(20, 7) ("storeSwapOut: swapout_size = %d\n", @@ -196,7 +202,7 @@ /* Ok, we have stuff to swap out. Is there a swapout.sio open? */ if (e->swap_status == SWAPOUT_NONE) { assert(mem->swapout.sio == NULL); - assert(mem->inmem_lo == 0); + assert(storeMemLoOffset(e) == 0); if (storeCheckCachable(e)) storeSwapOutStart(e); else @@ -239,7 +245,7 @@ /* the storeWrite() call might generate an error */ if (e->swap_status != SWAPOUT_WRITING) break; - swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset); + swapout_size = (ssize_t) (storeMemHiOffset(e) - mem->swapout.queue_offset); if (e->store_status == STORE_PENDING) if (swapout_size < SM_PAGE_SIZE) break; @@ -253,7 +259,7 @@ * to the filesystem at this point because storeSwapOut() is * not going to be called again for this entry. */ - assert(mem->inmem_hi == mem->swapout.queue_offset); + assert(storeMemHiOffset(e) == mem->swapout.queue_offset); storeSwapOutFileClose(e); } } @@ -346,7 +352,7 @@ dlink_node *node; if (e->mem_obj->swapout.sio != NULL) return 1; - if (e->mem_obj->inmem_lo > 0) + if (storeMemLoOffset(e) > 0) return 0; /* * If there are DISK clients, we must write to disk @@ -361,3 +367,7 @@ return 0; return EBIT_TEST(e->flags, ENTRY_CACHABLE); } + + + +#endif Index: squid/src/structs.h diff -u squid/src/structs.h:1.26 squid/src/structs.h:1.2.2.20 --- squid/src/structs.h:1.26 Tue Feb 20 16:10:13 2001 +++ squid/src/structs.h Sun Mar 18 14:33:32 2001 @@ -981,8 +981,6 @@ struct _clientHttpRequest { ConnStateData *conn; request_t *request; /* Parsed URL ... */ - store_client *sc; /* The store_client we're using */ - store_client *old_sc; /* ... for entry to be validated */ char *uri; char *log_uri; struct { @@ -1013,6 +1011,8 @@ char *location; } redirect; dlink_node active; + char *buf; /* transient buffer... eeeew! */ + int bufofs; /* offset in the current buffer .. */ }; struct _ConnStateData { @@ -1090,8 +1090,6 @@ PeerDigest *pd; StoreEntry *entry; StoreEntry *old_entry; - store_client *sc; - store_client *old_sc; request_t *request; int offset; int mask_offset; @@ -1348,6 +1346,7 @@ /* keep track each client receiving data from that particular StoreEntry */ struct _store_client { + int valid; int type; off_t copy_offset; off_t seen_offset; @@ -1406,16 +1405,6 @@ struct _MemObject { method_t method; char *url; - mem_hdr data_hdr; - off_t inmem_hi; - off_t inmem_lo; - dlink_list clients; - int nclients; - struct { - off_t queue_offset; /* relative to in-mem data */ - mem_node *memnode; /* which node we're currently paging out */ - storeIOState *sio; - } swapout; HttpReply *reply; request_t *request; struct timeval start_ping; @@ -1438,6 +1427,7 @@ struct _StoreEntry { hash_link hash; /* must be first */ + dlink_node node; /* to track current objects .. */ MemObject *mem_obj; RemovalPolicyNode repl; time_t timestamp; @@ -1454,6 +1444,21 @@ ping_status_t ping_status:3; store_status_t store_status:3; swap_status_t swap_status:3; + + /* The request callback information */ + store_client sc; + + /* The reply callback functions */ + STRAPPEND *stappend; + STRFLUSH *stflush; + STRMEMHI *stmemhi; + STRMEMLO *stmemlo; + STRCLIENTCOPY *stclientcopy; + STRPRIVATE *stmakeprivate; + STRPUBLIC *stmakepublic; + STRDONE *stdone; + STRRELEASE *strelease; + void *stdata; }; struct _SwapDir { @@ -1486,8 +1491,8 @@ STCALLBACK *callback; /* Handle pending callbacks */ STSYNC *sync; /* Sync the directory */ struct { - STOBJCREATE *create; - STOBJOPEN *open; + STOBJCREATE *create; + STOBJGET *get; STOBJCLOSE *close; STOBJREAD *read; STOBJWRITE *write; @@ -1570,7 +1575,6 @@ int link_count; /* free when zero */ request_flags flags; HttpHdrCc *cache_control; - HttpHdrRange *range; http_version_t http_ver; time_t ims; int imslen; Index: squid/src/tools.c diff -u squid/src/tools.c:1.11 squid/src/tools.c:1.2.2.7 --- squid/src/tools.c:1.11 Thu Feb 15 13:09:17 2001 +++ squid/src/tools.c Tue Feb 20 07:39:42 2001 @@ -348,11 +348,11 @@ fatal(const char *message) { releaseServerSockets(); - /* check for store_dirs_rebuilding because fatal() is often - * used in early initialization phases, long before we ever - * get to the store log. */ - if (0 == store_dirs_rebuilding) - storeDirWriteCleanLogs(0); + /* + * Attempt to write clean store log information. If any of the filestores + * are rebuilding, they will fail in their own time. + */ + storeDirWriteCleanLogs(0); fatal_common(message); exit(shutting_down ? 0 : 1); } Index: squid/src/typedefs.h diff -u squid/src/typedefs.h:1.17 squid/src/typedefs.h:1.2.2.13 --- squid/src/typedefs.h:1.17 Wed Feb 28 20:04:19 2001 +++ squid/src/typedefs.h Sun Mar 18 14:33:32 2001 @@ -263,13 +263,19 @@ typedef int STCALLBACK(SwapDir *); typedef void STSYNC(SwapDir *); -typedef storeIOState *STOBJCREATE(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *); -typedef storeIOState *STOBJOPEN(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *); +/* store lookup callback */ +typedef void STGETDONE(void *, StoreEntry *); + +/* store IO functions */ +typedef storeIOState *STOBJCREATE(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, + void *); typedef void STOBJCLOSE(SwapDir *, storeIOState *); typedef void STOBJREAD(SwapDir *, storeIOState *, char *, size_t, off_t, STRCB *, void *); typedef void STOBJWRITE(SwapDir *, storeIOState *, char *, size_t, off_t, FREE *); typedef void STOBJUNLINK(SwapDir *, StoreEntry *); +typedef void STOBJGET(SwapDir *, request_t *, STGETDONE *, void *); + typedef void STLOGOPEN(SwapDir *); typedef void STLOGCLOSE(SwapDir *); typedef void STLOGWRITE(const SwapDir *, const StoreEntry *, int); @@ -285,6 +291,18 @@ typedef void STFSSTARTUP(void); typedef void STFSSHUTDOWN(void); +/* store reply routines */ +typedef void STRAPPEND(StoreEntry *, const char *, int); +typedef void STRFLUSH(StoreEntry *); +typedef off_t STRMEMHI(const StoreEntry *); +typedef off_t STRMEMLO(const StoreEntry *); +typedef void STRCLIENTCOPY(StoreEntry *); +typedef void STRRELEASE(StoreEntry *); +typedef void STRDONE(StoreEntry *); +typedef void STRPUBLIC(StoreEntry *); +typedef void STRPRIVATE(StoreEntry *); + + typedef double hbase_f(double); typedef void StatHistBinDumper(StoreEntry *, int idx, double val, double size, int count); Index: squid/src/urn.c diff -u squid/src/urn.c:1.10 squid/src/urn.c:1.2.2.9 --- squid/src/urn.c:1.10 Sat Mar 3 02:44:32 2001 +++ squid/src/urn.c Sun Mar 18 14:33:32 2001 @@ -137,16 +137,17 @@ } httpHeaderPutStr(&urlres_r->header, HDR_ACCEPT, "text/plain"); if ((urlres_e = storeGetPublic(urlres, METHOD_GET)) == NULL) { - urlres_e = storeCreateEntry(urlres, urlres, null_request_flags, METHOD_GET); - urnState->sc = storeClientListAdd(urlres_e, urnState); + urlres_e = storeCreateEntry(urlres, urlres, null_request_flags, + METHOD_GET, REPLY_OBJ_INTERNAL); + storeClientRegister(urlres_e, urnState); fwdStart(-1, urlres_e, urlres_r); } else { storeLockObject(urlres_e); - urnState->sc = storeClientListAdd(urlres_e, urnState); + storeClientRegister(urlres_e, urnState); } urnState->urlres_e = urlres_e; urnState->urlres_r = requestLink(urlres_r); - storeClientCopy(urnState->sc, urlres_e, + storeClientCopy(urlres_e, 0, 0, 4096, @@ -201,7 +202,7 @@ return; } if (urlres_e->store_status == STORE_PENDING && size < SM_PAGE_SIZE) { - storeClientCopy(urnState->sc, urlres_e, + storeClientCopy(urlres_e, size, 0, SM_PAGE_SIZE, @@ -292,7 +293,7 @@ } safe_free(urls); /* mb was absorbed in httpBodySet call, so we must not clean it */ - storeUnregister(urnState->sc, urlres_e, urnState); + storeClientUnregister(urlres_e, urnState); storeUnlockObject(urlres_e); storeUnlockObject(urnState->entry); requestUnlink(urnState->request); Index: squid/src/wais.c diff -u squid/src/wais.c:1.6 squid/src/wais.c:1.2.2.4 --- squid/src/wais.c:1.6 Sat Mar 3 02:44:32 2001 +++ squid/src/wais.c Sun Mar 18 14:33:32 2001 @@ -70,7 +70,7 @@ StoreEntry *entry = waisState->entry; debug(24, 4) ("waisTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); if (entry->store_status == STORE_PENDING) { - if (entry->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(entry) == 0) { fwdFail(waisState->fwd, errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); } @@ -138,7 +138,7 @@ errorAppendEntry(entry, err); comm_close(fd); } - } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { + } else if (len == 0 && storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); err->xerrno = errno; Index: squid/src/whois.c diff -u squid/src/whois.c:1.6 squid/src/whois.c:1.2.2.6 --- squid/src/whois.c:1.6 Sat Mar 3 02:44:32 2001 +++ squid/src/whois.c Sun Mar 18 14:33:32 2001 @@ -97,7 +97,7 @@ debug(75, 3) ("whoisReadReply: FD %d read %d bytes\n", fd, len); debug(75, 5) ("{%s}\n", buf); if (len > 0) { - if (0 == mem->inmem_hi) + if (0 == storeMemHiOffset(entry)) mem->reply->sline.status = HTTP_OK; fd_bytes(fd, len, FD_READ); kb_incr(&statCounter.server.all.kbytes_in, len); @@ -109,7 +109,7 @@ fd, xstrerror()); if (ignoreErrno(errno)) { commSetSelect(fd, COMM_SELECT_READ, whoisReadReply, p, Config.Timeout.read); - } else if (mem->inmem_hi == 0) { + } else if (storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); err->xerrno = errno; Index: squid/src/fs/hash/Makefile.in diff -u /dev/null squid/src/fs/hash/Makefile.in:1.1.2.1 --- /dev/null Sun Jan 25 06:37:04 2004 +++ squid/src/fs/hash/Makefile.in Sat Dec 16 01:29:03 2000 @@ -0,0 +1,56 @@ +# +# Makefile for the hash storage driver for the Squid Object Cache server +# +# $Id: squid-modio-HEAD,v 1.1 2004/08/17 20:55:12 hno Exp $ +# + +FS = hash + +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ + +CC = @CC@ +MAKEDEPEND = @MAKEDEPEND@ +AR_R = @AR_R@ +RANLIB = @RANLIB@ +AC_CFLAGS = @CFLAGS@ +SHELL = /bin/sh + +INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/ +CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES) + +OUT = ../$(FS).a + +OBJS = \ + store_dir_hash.o \ + store_io_hash.o + + +all: $(OUT) + +$(OUT): $(OBJS) + @rm -f ../stamp + $(AR_R) $(OUT) $(OBJS) + $(RANLIB) $(OUT) + +$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h + +.c.o: + @rm -f ../stamp + $(CC) $(CFLAGS) -c $< + +clean: + -rm -rf *.o *pure_* core ../$(FS).a + +distclean: clean + -rm -f Makefile + -rm -f Makefile.bak + -rm -f tags + +install: + +tags: + ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch] + +depend: + $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c Index: squid/src/fs/hash/store_dir_hash.c diff -u /dev/null squid/src/fs/hash/store_dir_hash.c:1.1.2.5 --- /dev/null Sun Jan 25 06:37:04 2004 +++ squid/src/fs/hash/store_dir_hash.c Wed Jan 10 12:41:14 2001 @@ -0,0 +1,621 @@ + +/* + * $Id: squid-modio-HEAD,v 1.1 2004/08/17 20:55:12 hno Exp $ + * + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Duane Wessels + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#if HAVE_STATVFS +#if HAVE_SYS_STATVFS_H +#include +#endif +#endif + +#include "store_hash.h" + +#define DefaultLevelOneDirs 16 +#define DefaultLevelTwoDirs 256 +#define STORE_META_BHASHZ 4096 + +static int n_hash_dirs = 0; +static int *hash_dir_index = NULL; +MemPool *hash_state_pool = NULL; +static int hash_initialised = 0; + +static int storeHashDirCreateDirectory(const char *path, int); +static int storeHashDirVerifyDirectory(const char *path); +static char *storeHashDirSwapLogFile(SwapDir *, const char *); +static STLOGOPEN storeHashDirOpenSwapLog; +static STINIT storeHashDirInit; +static STFREE storeHashDirFree; +static STNEWFS storeHashDirNewfs; +static STDUMP storeHashDirDump; +static STMAINTAINFS storeHashDirMaintain; +static STCHECKOBJ storeHashDirCheckObj; +static STREFOBJ storeHashDirRefObj; +static STUNREFOBJ storeHashDirUnrefObj; +static EVH storeHashDirCleanEvent; +static EVH storeHashDirRebuildEvent; +static int storeHashCleanupDoubleCheck(SwapDir *, StoreEntry *); +static void storeHashDirStats(SwapDir *, StoreEntry *); +static void storeHashDirInitBitmap(SwapDir *); + +/* + * These functions were ripped straight out of the heart of store_dir.c. + * They assume that the given filenum is on a hash partiton, which may or + * may not be true.. + * XXX this evilness should be tidied up at a later date! + */ + +int +storeHashDirMapBitTest(SwapDir * SD, int fn) +{ + sfileno filn = fn; + hashinfo_t *hashinfo; + hashinfo = (hashinfo_t *) SD->fsdata; + return file_map_bit_test(hashinfo->map, filn); +} + +void +storeHashDirMapBitSet(SwapDir * SD, int fn) +{ + sfileno filn = fn; + hashinfo_t *hashinfo; + hashinfo = (hashinfo_t *) SD->fsdata; + file_map_bit_set(hashinfo->map, filn); +} + +void +storeHashDirMapBitReset(SwapDir * SD, int fn) +{ + sfileno filn = fn; + hashinfo_t *hashinfo; + hashinfo = (hashinfo_t *) SD->fsdata; + /* + * We have to test the bit before calling file_map_bit_reset. + * file_map_bit_reset doesn't do bounds checking. It assumes + * filn is a valid file number, but it might not be because + * the map is dynamic in size. Also clearing an already clear + * bit puts the map counter of-of-whack. + */ + if (file_map_bit_test(hashinfo->map, filn)) + file_map_bit_reset(hashinfo->map, filn); +} + +int +storeHashDirMapBitAllocate(SwapDir * SD) +{ + hashinfo_t *hashinfo = (hashinfo_t *) SD->fsdata; + int fn; + fn = file_map_allocate(hashinfo->map, hashinfo->suggest); + file_map_bit_set(hashinfo->map, fn); + hashinfo->suggest = fn + 1; + return fn; +} + +/* + * Initialise the hash bitmap + * + * If there already is a bitmap, and the numobjects is larger than currently + * configured, we allocate a new bitmap and 'grow' the old one into it. + */ +static void +storeHashDirInitBitmap(SwapDir * sd) +{ + hashinfo_t *hashinfo = (hashinfo_t *) sd->fsdata; + + if (hashinfo->map == NULL) { + /* First time */ + hashinfo->map = file_map_create(); + } else if (hashinfo->map->max_n_files) { + /* it grew, need to expand */ + /* XXX We don't need it anymore .. */ + } + /* else it shrunk, and we leave the old one in place */ +} + +static int +storeHashDirCreateDirectory(const char *path, int should_exist) +{ + int created = 0; + struct stat st; + getCurrentTime(); + if (0 == stat(path, &st)) { + if (S_ISDIR(st.st_mode)) { + debug(20, should_exist ? 3 : 1) ("%s exists\n", path); + } else { + fatalf("Swap directory %s is not a directory.", path); + } + } else if (0 == mkdir(path, 0755)) { + debug(20, should_exist ? 1 : 3) ("%s created\n", path); + created = 1; + } else { + fatalf("Failed to make swap directory %s: %s", + path, xstrerror()); + } + return created; +} + +static int +storeHashDirVerifyDirectory(const char *path) +{ + struct stat sb; + if (stat(path, &sb) < 0) { + debug(20, 0) ("%s: %s\n", path, xstrerror()); + return -1; + } + if (S_ISDIR(sb.st_mode) == 0) { + debug(20, 0) ("%s is not a directory\n", path); + return -1; + } + return 0; +} + +static char * +storeHashDirSwapLogFile(SwapDir * sd, const char *ext) +{ + LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; + if (Config.Log.swap) { + xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); + while (index(pathtmp, '/')) + *index(pathtmp, '/') = '.'; + while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') + pathtmp[strlen(pathtmp) - 1] = '\0'; + for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); + snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); + if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { + strcat(path, "."); + snprintf(digit, 32, "%02d", sd->index); + strncat(path, digit, 3); + } + } else { + xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); + strcat(path, "/swap.state"); + } + if (ext) + strncat(path, ext, 16); + return path; +} + +static void +storeHashDirOpenSwapLog(SwapDir * sd) +{ + hashinfo_t *hashinfo = (hashinfo_t *) sd->fsdata; + char *path; + int fd; + path = storeHashDirSwapLogFile(sd, NULL); + fd = file_open(path, O_WRONLY | O_CREAT); + if (fd < 0) { + debug(50, 1) ("%s: %s\n", path, xstrerror()); + fatal("storeHashDirOpenSwapLog: Failed to open swap log."); + } + debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd); + hashinfo->swaplog_fd = fd; + if (0 == n_hash_dirs) + assert(NULL == hash_dir_index); + n_hash_dirs++; + assert(n_hash_dirs <= Config.cacheSwap.n_configured); +} + +static void +storeHashDirInit(SwapDir * sd) +{ + static int started_clean_event = 0; + storeHashDirInitBitmap(sd); + storeHashDirOpenSwapLog(sd); + if (!started_clean_event) { + eventAdd("storeHashDirClean", storeHashDirCleanEvent, NULL, 15.0, 1); + /* we can't call the rebuild completion routine until the next loop */ + eventAdd("storeHashRebuild", storeHashDirRebuildEvent, NULL, 0.0, 1); + started_clean_event = 1; + } +} + + +struct _clean_state { + char *cur; + char *new; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; + RemovalPolicyWalker *walker; +}; + +static void +storeHashDirNewfs(SwapDir * sd) +{ + debug(47, 3) ("Creating swap space in %s\n", sd->path); + storeHashDirCreateDirectory(sd->path, 0); +} + +static void +storeHashDirCleanEvent(void *unused) +{ + /* Empty for now */ +#if 0 + eventAdd("storeDirClean", storeHashDirCleanEvent, NULL, + 15.0 * exp(-0.25 * n), 1); +#endif +} + +static void +storeHashDirRebuildEvent(void *data) +{ + SwapDir *sd = data; + + struct _store_rebuild_data sr; + + /* feed it an empty rebuild, cause we're empty! */ + bzero(&sr, sizeof(struct _store_rebuild_data)); + storeRebuildComplete(&sr); +} + +void +storeHashDirMaintain(SwapDir * SD) +{ + /* Do nothing for now */ + return; +} + +/* + * storeHashDirCheckObj + * + * This routine is called by storeDirSelectSwapDir to see if the given + * object is able to be stored on this filesystem. HASH filesystems will + * happily store anything as long as the LRU time isn't too small. + */ +int +storeHashDirCheckObj(SwapDir * SD, const StoreEntry * e) +{ +#if OLD_UNUSED_CODE + if (storeHashDirExpiredReferenceAge(SD) < 300) { + debug(20, 3) ("storeHashDirCheckObj: NO: LRU Age = %d\n", + storeHashDirExpiredReferenceAge(SD)); + /* store_check_cachable_hist.no.lru_age_too_low++; */ + return -1; + } +#endif + /* Return 999 (99.9%) constant load */ + return 999; +} + +/* + * storeHashDirRefObj + * + * This routine is called whenever an object is referenced, so we can + * maintain replacement information within the storage fs. + */ +void +storeHashDirRefObj(SwapDir * SD, StoreEntry * e) +{ + debug(1, 3) ("storeHashDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); + if (SD->repl->Referenced) + SD->repl->Referenced(SD->repl, e, &e->repl); +} + +/* + * storeHashDirUnrefObj + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ +void +storeHashDirUnrefObj(SwapDir * SD, StoreEntry * e) +{ + debug(1, 3) ("storeHashDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn, + e->swap_filen); + if (SD->repl->Dereferenced) + SD->repl->Dereferenced(SD->repl, e, &e->repl); +} + +/* + * storeHashDirUnlinkFile + * + * This routine unlinks a file and pulls it out of the bitmap. + * It used to be in storeHashUnlink(), however an interface change + * forced this bit of code here. Eeek. + */ +void +storeHashDirUnlinkFile(SwapDir * SD, sfileno f) +{ + debug(79, 3) ("storeHashDirUnlinkFile: unlinking fileno %08X\n", f); + /* storeHashDirMapBitReset(SD, f); */ +#if 0 + unlinkdUnlink(storeHashDirFullPath(SD, f, NULL)); +#endif +} + +/* + * Add and remove the given StoreEntry from the replacement policy in + * use. + */ + +void +storeHashDirReplAdd(SwapDir * SD, StoreEntry * e) +{ + debug(20, 4) ("storeHashDirReplAdd: added node %p to dir %d\n", e, + SD->index); + SD->repl->Add(SD->repl, e, &e->repl); +} + + +void +storeHashDirReplRemove(StoreEntry * e) +{ + SwapDir *SD = INDEXSD(e->swap_dirn); + debug(20, 4) ("storeHashDirReplRemove: remove node %p from dir %d\n", e, + SD->index); + SD->repl->Remove(SD->repl, e, &e->repl); +} + + + +/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ + +void +storeHashDirStats(SwapDir * SD, StoreEntry * sentry) +{ + hashinfo_t *hashinfo; +#if HAVE_STATVFS + struct statvfs sfs; +#endif + hashinfo = (hashinfo_t *) SD->fsdata; + storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); + storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); + storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", + 100.0 * SD->cur_size / SD->max_size); + storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", + hashinfo->map->n_files_in_map, hashinfo->map->max_n_files, + percent(hashinfo->map->n_files_in_map, hashinfo->map->max_n_files)); +#if HAVE_STATVFS +#define fsbtoblk(num, fsbs, bs) \ + (((fsbs) != 0 && (fsbs) < (bs)) ? \ + (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs))) + if (!statvfs(SD->path, &sfs)) { + storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024), + fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024), + percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks)); + storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + sfs.f_files - sfs.f_ffree, sfs.f_files, + percent(sfs.f_files - sfs.f_ffree, sfs.f_files)); + } +#endif + storeAppendPrintf(sentry, "Flags:"); + if (SD->flags.selected) + storeAppendPrintf(sentry, " SELECTED"); + if (SD->flags.read_only) + storeAppendPrintf(sentry, " READ-ONLY"); + storeAppendPrintf(sentry, "\n"); +#if OLD_UNUSED_CODE +#if !HEAP_REPLACEMENT + storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n", + (double) storeHashDirExpiredReferenceAge(SD) / 86400.0); +#else + storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n", + heap_peepminkey(sd.repl.heap.heap)); +#endif +#endif /* OLD_UNUSED_CODE */ +} + +/* + * storeHashDirReconfigure + * + * This routine is called when the given swapdir needs reconfiguring + */ +void +storeHashDirReconfigure(SwapDir * sd, int index, char *path) +{ + char *token; + int i; + int size; + int l1; + int l2; + unsigned int read_only = 0; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeHashDirReconfigure: invalid size value"); + i = GetInteger(); + l1 = i; + if (l1 <= 0) + fatal("storeHashDirReconfigure: invalid level 1 directories value"); + i = GetInteger(); + l2 = i; + if (l2 <= 0) + fatal("storeHashDirReconfigure: invalid level 2 directories value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + /* just reconfigure it */ + if (size == sd->max_size) + debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n", + path, size); + else + debug(3, 1) ("Cache dir '%s' size changed to %d KB\n", + path, size); + sd->max_size = size; + if (sd->flags.read_only != read_only) + debug(3, 1) ("Cache dir '%s' now %s\n", + path, read_only ? "Read-Only" : "Read-Write"); + sd->flags.read_only = read_only; + return; +} + +void +storeHashDirDump(StoreEntry * entry, const char *name, SwapDir * s) +{ + storeAppendPrintf(entry, "%s %s %s %d\n", + name, + "hash", + s->path, + s->max_size >> 10); +} + +/* + * Only "free" the filesystem specific stuff here + */ +static void +storeHashDirFree(SwapDir * s) +{ + hashinfo_t *hashinfo = (hashinfo_t *) s->fsdata; + if (hashinfo->swaplog_fd > -1) { + file_close(hashinfo->swaplog_fd); + hashinfo->swaplog_fd = -1; + } + filemapFreeMemory(hashinfo->map); + xfree(hashinfo); + s->fsdata = NULL; /* Will aid debugging... */ + +} + +/* + * storeHashCleanupDoubleCheck + * + * This is called by storeCleanup() if -S was given on the command line. + */ +static int +storeHashCleanupDoubleCheck(SwapDir * sd, StoreEntry * e) +{ + struct stat sb; +#if 0 + if (stat(storeHashDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) { + debug(20, 0) ("storeHashCleanupDoubleCheck: MISSING SWAP FILE\n"); + debug(20, 0) ("storeHashCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeHashCleanupDoubleCheck: PATH %s\n", + storeHashDirFullPath(sd, e->swap_filen, NULL)); + storeEntryDump(e, 0); + return -1; + } + if (e->swap_file_sz != sb.st_size) { + debug(20, 0) ("storeHashCleanupDoubleCheck: SIZE MISMATCH\n"); + debug(20, 0) ("storeHashCleanupDoubleCheck: FILENO %08X\n", e->swap_filen); + debug(20, 0) ("storeHashCleanupDoubleCheck: PATH %s\n", + storeHashDirFullPath(sd, e->swap_filen, NULL)); + debug(20, 0) ("storeHashCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n", + e->swap_file_sz, (int) sb.st_size); + storeEntryDump(e, 0); + return -1; + } +#endif + return 0; +} + +/* + * storeHashDirParse + * + * Called when a *new* fs is being setup. + */ +void +storeHashDirParse(SwapDir * sd, int index, char *path) +{ + char *token; + int i; + int size; + unsigned int read_only = 0; + hashinfo_t *hashinfo; + + i = GetInteger(); + size = i << 10; /* Mbytes to kbytes */ + if (size <= 0) + fatal("storeHashDirParse: invalid size value"); + if ((token = strtok(NULL, w_space))) + if (!strcasecmp(token, "read-only")) + read_only = 1; + + hashinfo = xmalloc(sizeof(hashinfo_t)); + if (hashinfo == NULL) + fatal("storeHashDirParse: couldn't xmalloc() hashinfo_t!\n"); + + sd->index = index; + sd->path = xstrdup(path); + sd->max_size = size; + sd->fsdata = hashinfo; + hashinfo->swaplog_fd = -1; + hashinfo->map = NULL; /* Debugging purposes */ + hashinfo->suggest = 0; + sd->flags.read_only = read_only; + sd->init = storeHashDirInit; + sd->newfs = storeHashDirNewfs; + sd->dump = storeHashDirDump; + sd->freefs = storeHashDirFree; + sd->dblcheck = storeHashCleanupDoubleCheck; + sd->statfs = storeHashDirStats; + sd->maintainfs = storeHashDirMaintain; + sd->checkobj = storeHashDirCheckObj; + sd->refobj = storeHashDirRefObj; + sd->unrefobj = storeHashDirUnrefObj; + sd->callback = NULL; + sd->sync = NULL; + + sd->obj.get = storeHashGet; + sd->obj.close = storeHashClose; + sd->obj.read = storeHashRead; + sd->obj.write = storeHashWrite; + sd->obj.unlink = storeHashUnlink; + sd->log.open = NULL; + sd->log.close = NULL; + sd->log.write = NULL; + sd->log.clean.start = NULL; + sd->log.clean.nextentry = NULL; + sd->log.clean.done = NULL; + + /* Initialise replacement policy stuff */ + sd->repl = createRemovalPolicy(Config.replPolicy); +} + +/* + * Initial setup / end destruction + */ +void +storeHashDirDone(void) +{ + memPoolDestroy(hash_state_pool); + hash_initialised = 0; +} + +void +storeFsSetup_hash(storefs_entry_t * storefs) +{ + assert(!hash_initialised); + storefs->parsefunc = storeHashDirParse; + storefs->reconfigurefunc = storeHashDirReconfigure; + storefs->donefunc = storeHashDirDone; + hash_state_pool = memPoolCreate("HASH IO State data", sizeof(hashstate_t)); + hash_initialised = 1; +} Index: squid/src/fs/hash/store_hash.h diff -u /dev/null squid/src/fs/hash/store_hash.h:1.1.2.4 --- /dev/null Sun Jan 25 06:37:04 2004 +++ squid/src/fs/hash/store_hash.h Sat Jan 6 06:35:57 2001 @@ -0,0 +1,48 @@ +/* + * store_hash.h + * + * Internal declarations for the hash routines + */ + +#ifndef __STORE_HASH_H__ +#define __STORE_HASH_H__ + +struct _hashinfo_t { + int swaplog_fd; + fileMap *map; + int suggest; +}; + +struct _hashstate_t { + int fd; + struct { + unsigned int close_request:1; + unsigned int reading:1; + unsigned int writing:1; + } flags; +}; + +typedef struct _hashinfo_t hashinfo_t; +typedef struct _hashstate_t hashstate_t; + +/* The hash_state memory pool */ +extern MemPool *hash_state_pool; + +extern void storeHashDirMapBitReset(SwapDir *, sfileno); +extern int storeHashDirMapBitAllocate(SwapDir *); +extern char *storeHashDirFullPath(SwapDir * SD, const char *uri, + const method_t method, char *fullpath); +extern void storeHashDirUnlinkFile(SwapDir *, sfileno); +extern void storeHashDirReplAdd(SwapDir * SD, StoreEntry *); +extern void storeHashDirReplRemove(StoreEntry *); + +/* + * Store IO stuff + */ +extern STOBJGET storeHashGet; +extern STOBJCLOSE storeHashClose; +extern STOBJREAD storeHashRead; +extern STOBJWRITE storeHashWrite; +extern STOBJUNLINK storeHashUnlink; + +#endif Index: squid/src/fs/hash/store_io_hash.c diff -u /dev/null squid/src/fs/hash/store_io_hash.c:1.1.2.6 --- /dev/null Sun Jan 25 06:37:04 2004 +++ squid/src/fs/hash/store_io_hash.c Sun Mar 18 14:51:22 2001 @@ -0,0 +1,283 @@ + +/* + * $Id: squid-modio-HEAD,v 1.1 2004/08/17 20:55:12 hno Exp $ + * + * DEBUG: section 79 Storage Manager HASH Interface + * AUTHOR: Duane Wessels + * + * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from the + * Internet community. Development is led by Duane Wessels of the + * National Laboratory for Applied Network Research and funded by the + * National Science Foundation. Squid is Copyrighted (C) 1998 by + * Duane Wessels and the University of California San Diego. Please + * see the COPYRIGHT file for full details. Squid incorporates + * software developed and/or copyrighted by other sources. Please see + * the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "store_hash.h" + + +static DRCB storeHashReadDone; +static DWCB storeHashWriteDone; +static void storeHashIOCallback(storeIOState * sio, int errflag); +static void storeHashIOFreeEntry(void *); + +CBDATA_TYPE(storeIOState); + +/* === PUBLIC =========================================================== */ + +void +storeHashGet(SwapDir *sd, request_t *request, STGETDONE *callback, + void *callback_data) +{ + /* Empty for now , return NULL */ + storeLookupComplete(request, callback, callback_data, NULL); +} + +storeIOState * +storeHashOpen(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, + STIOCB * callback, void *callback_data) +{ + sfileno f = e->swap_filen; +#if 0 + char *path = storeHashDirFullPath(SD, f, NULL); +#endif + char *path; + storeIOState *sio; + struct stat sb; + int fd; + debug(79, 3) ("storeHashOpen: fileno %08X\n", f); + fd = file_open(path, O_RDONLY); + if (fd < 0) { + debug(79, 3) ("storeHashOpen: got failure (%d)\n", errno); + return NULL; + } + debug(79, 3) ("storeHashOpen: opened FD %d\n", fd); + CBDATA_INIT_TYPE_FREECB(storeIOState, storeHashIOFreeEntry); + sio = cbdataAlloc(storeIOState); + sio->fsstate = memPoolAlloc(hash_state_pool); + + sio->swap_filen = f; + sio->swap_dirn = SD->index; + sio->mode = O_RDONLY; + sio->callback = callback; + sio->callback_data = callback_data; + cbdataLock(callback_data); + sio->e = e; + ((hashstate_t *) (sio->fsstate))->fd = fd; + ((hashstate_t *) (sio->fsstate))->flags.writing = 0; + ((hashstate_t *) (sio->fsstate))->flags.reading = 0; + ((hashstate_t *) (sio->fsstate))->flags.close_request = 0; + if (fstat(fd, &sb) == 0) + sio->st_size = sb.st_size; + store_open_disk_fd++; + + /* We should update the heap/dlink position here ! */ + return sio; +} + +storeIOState * +storeHashCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data) +{ + storeIOState *sio; + int fd; + int mode = (O_WRONLY | O_CREAT | O_TRUNC); + char *path; + hashinfo_t *hashinfo = (hashinfo_t *) SD->fsdata; + sfileno filn; + sdirno dirn; + + /* Allocate a number */ + dirn = SD->index; + filn = storeHashDirMapBitAllocate(SD); + hashinfo->suggest = filn + 1; + /* Shouldn't we handle a 'bitmap full' error here? */ +#if 0 + path = storeHashDirFullPath(SD, filn, NULL); +#endif + + debug(79, 3) ("storeHashCreate: fileno %08X\n", filn); + fd = file_open(path, mode); + if (fd < 0) { + debug(79, 3) ("storeHashCreate: got failure (%d)\n", errno); + return NULL; + } + debug(79, 3) ("storeHashCreate: opened FD %d\n", fd); + CBDATA_INIT_TYPE_FREECB(storeIOState, storeHashIOFreeEntry); + sio = cbdataAlloc(storeIOState); + sio->fsstate = memPoolAlloc(hash_state_pool); + + sio->swap_filen = filn; + sio->swap_dirn = dirn; + sio->mode = mode; + sio->callback = callback; + sio->callback_data = callback_data; + cbdataLock(callback_data); + sio->e = (StoreEntry *) e; + ((hashstate_t *) (sio->fsstate))->fd = fd; + ((hashstate_t *) (sio->fsstate))->flags.writing = 0; + ((hashstate_t *) (sio->fsstate))->flags.reading = 0; + ((hashstate_t *) (sio->fsstate))->flags.close_request = 0; + store_open_disk_fd++; + + /* now insert into the replacement policy */ + storeHashDirReplAdd(SD, e); + return sio; +} + +void +storeHashClose(SwapDir * SD, storeIOState * sio) +{ + hashstate_t *hashstate = (hashstate_t *) sio->fsstate; + + debug(79, 3) ("storeHashClose: dirno %d, fileno %08X, FD %d\n", + sio->swap_dirn, sio->swap_filen, hashstate->fd); + if (hashstate->flags.reading || hashstate->flags.writing) { + hashstate->flags.close_request = 1; + return; + } + storeHashIOCallback(sio, 0); +} + +void +storeHashRead(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) +{ + hashstate_t *hashstate = (hashstate_t *) sio->fsstate; + + assert(sio->read.callback == NULL); + assert(sio->read.callback_data == NULL); + sio->read.callback = callback; + sio->read.callback_data = callback_data; + cbdataLock(callback_data); + debug(79, 3) ("storeHashRead: dirno %d, fileno %08X, FD %d\n", + sio->swap_dirn, sio->swap_filen, hashstate->fd); + sio->offset = offset; + hashstate->flags.reading = 1; + file_read(hashstate->fd, + buf, + size, + offset, + storeHashReadDone, + sio); +} + +void +storeHashWrite(SwapDir * SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func) +{ + hashstate_t *hashstate = (hashstate_t *) sio->fsstate; + debug(79, 3) ("storeHashWrite: dirn %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, hashstate->fd); + hashstate->flags.writing = 1; + file_write(hashstate->fd, + offset, + buf, + size, + storeHashWriteDone, + sio, + free_func); +} + +void +storeHashUnlink(SwapDir * SD, StoreEntry * e) +{ + debug(79, 3) ("storeHashUnlink: fileno %08X\n", e->swap_filen); + storeHashDirReplRemove(e); + storeHashDirMapBitReset(SD, e->swap_filen); + storeHashDirUnlinkFile(SD, e->swap_filen); +} + +/* === STATIC =========================================================== */ + +static void +storeHashReadDone(int fd, const char *buf, int len, int errflag, void *my_data) +{ + storeIOState *sio = my_data; + hashstate_t *hashstate = (hashstate_t *) sio->fsstate; + STRCB *callback = sio->read.callback; + void *their_data = sio->read.callback_data; + ssize_t rlen; + + debug(79, 3) ("storeHashReadDone: dirno %d, fileno %08X, FD %d, len %d\n", + sio->swap_dirn, sio->swap_filen, fd, len); + hashstate->flags.reading = 0; + if (errflag) { + debug(79, 3) ("storeHashReadDone: got failure (%d)\n", errflag); + rlen = -1; + } else { + rlen = (ssize_t) len; + sio->offset += len; + } + assert(callback); + assert(their_data); + sio->read.callback = NULL; + sio->read.callback_data = NULL; + if (cbdataValid(their_data)) + callback(their_data, buf, (size_t) rlen); + cbdataUnlock(their_data); +} + +static void +storeHashWriteDone(int fd, int errflag, size_t len, void *my_data) +{ + storeIOState *sio = my_data; + hashstate_t *hashstate = (hashstate_t *) sio->fsstate; + debug(79, 3) ("storeHashWriteDone: dirno %d, fileno %08X, FD %d, len %d\n", + sio->swap_dirn, sio->swap_filen, fd, len); + hashstate->flags.writing = 0; + if (errflag) { + debug(79, 0) ("storeHashWriteDone: got failure (%d)\n", errflag); + storeHashIOCallback(sio, errflag); + return; + } + sio->offset += len; + if (hashstate->flags.close_request) + storeHashIOCallback(sio, errflag); +} + +static void +storeHashIOCallback(storeIOState * sio, int errflag) +{ + hashstate_t *hashstate = (hashstate_t *) sio->fsstate; + debug(79, 3) ("storeHashIOCallback: errflag=%d\n", errflag); + if (hashstate->fd > -1) { + file_close(hashstate->fd); + store_open_disk_fd--; + } + if (cbdataValid(sio->callback_data)) + sio->callback(sio->callback_data, errflag, sio); + cbdataUnlock(sio->callback_data); + sio->callback_data = NULL; + sio->callback = NULL; + cbdataFree(sio); +} + + +/* + * We can't pass memFree() as a free function here, because we need to free + * the fsstate variable .. + */ +static void +storeHashIOFreeEntry(void *sio) +{ + memPoolFree(hash_state_pool, ((storeIOState *) sio)->fsstate); + memFree(sio, MEM_STORE_IO); +} squid-modio-HEAD.new squid-modio-HEAD differ: char 65, line 2