--------------------- PatchSet 5392 Date: 2007/08/20 03:07:44 Author: amosjeffries Branch: docs Tag: (none) Log: Move swap.state file format docs to source Move some Store API docs to soruce. Members: doc/Programming-Guide/27_MiscOther.dox:1.1.2.6->1.1.2.7 src/Store.h:1.32.4.1->1.32.4.2 src/StoreSwapLogData.h:1.3.26.3->1.3.26.4 src/enums.h:1.37.2.1->1.37.2.2 src/fs/FileSystems.dox:1.1.2.5->1.1.2.6 Index: squid3/doc/Programming-Guide/27_MiscOther.dox =================================================================== RCS file: /cvsroot/squid-sf//squid3/doc/Programming-Guide/Attic/27_MiscOther.dox,v retrieving revision 1.1.2.6 retrieving revision 1.1.2.7 diff -u -r1.1.2.6 -r1.1.2.7 --- squid3/doc/Programming-Guide/27_MiscOther.dox 20 Aug 2007 01:11:41 -0000 1.1.2.6 +++ squid3/doc/Programming-Guide/27_MiscOther.dox 20 Aug 2007 03:07:44 -0000 1.1.2.7 @@ -1,102 +1,6 @@ /** /page 27_MiscOther Miscellaneous Other Details -\section FileFormats File Formats - -\section StoreSwapMeta Store "swap meta" Description -\par - "swap meta" refers to a section of meta data stored at the beginning - of an object that is stored on disk. This meta data includes information - such as the object's cache key (MD5), URL, and part of the StoreEntry - structure. - -\par - The meta data is stored using a TYPE-LENGTH-VALUE format. That is, - each chunk of meta information consists of a TYPE identifier, a - LENGTH field, and then the VALUE (which is LENGTH octets long). - -\subsection Types Types -\par - As of Squid-2.3, the following TYPES are defined (from enums.h): - -\par STORE_META_VOID - Just a placeholder for the zeroth value. It is never used - on disk. - -\par STORE_META_KEY_URL - This represents the case when we use the URL as the cache - key, as Squid-1.1 does. Currently we don't support using - a URL as a cache key, so this is not used. - -\par STORE_META_KEY_SHA - For a brief time we considered supporting SHA (secure - hash algorithm) as a cache key. Nobody liked it, and - this type is not currently used. - -\par STORE_META_KEY_MD5 - This represents the MD5 cache key that Squid currently uses. - When Squid opens a disk file for reading, it can check that - this MD5 matches the MD5 of the user's request. If not, then - something went wrong and this is probably the wrong object. - -\par STORE_META_URL - The object's URL. This also may be matched against a user's - request for cache hits to make sure we got the right object. - -\par STORE_META_STD - This is the "standard metadata" for an object. Really - its just this middle chunk of the StoreEntry structure: -\code - time_t timestamp; - time_t lastref; - time_t expires; - time_t lastmod; - size_t swap_file_sz; - u_short refcount; - u_short flags; -\endcode - -\par STORE_META_STD_LFS - Updated version of STORE_META_STD, with support for - >2GB objects. As STORE_META_STD except that the swap_file_sz - is a squid_file_sz (64-bit integer) instead of size_t. - -\par STORE_META_HITMETERING - Reserved for future hit-metering (RFC 2227) stuff. - -\par STORE_META_VALID -\todo Document STORE_META_VALID - -\par STORE_META_VARY_HEADERS - Information about the Vary header relation on this object - -\par STORE_META_OBJSIZE - object size, if its known - -\section ImplementationNotes Implementation Notes -\par - When writing an object to disk, we must first write the meta data. - This is done with a couple of functions. First, storeSwapMetaPack() - takes a StoreEntry as a parameter and returns a tlv linked - list. Second, storeSwapMetaPack() converts the tlv list - into a character buffer that we can write. - -\note MemObject has a member called swap_hdr_sz. - This value is the size of that character buffer; the size of the - swap file meta data. The StoreEntry has a member named - swap_file_sz that represents the size of the disk file. - Thus, the size of the object "content" is -\code - StoreEntry->swap_file_sz - MemObject->swap_hdr_sz; -\endcode -\note The swap file content includes the HTTP reply headers and the HTTP reply body (if any). - -\par - When reading a swap file, there is a similar process to extract - the swap meta data. First, storeSwapMetaUnpack() converts a - character buffer into a tlv linked list. It also tells us - the value for MemObject->swap_hdr_sz. - \section leakFinder leakFinder \par src/leakfinder.c contains some routines useful for debugging Index: squid3/src/Store.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/Store.h,v retrieving revision 1.32.4.1 retrieving revision 1.32.4.2 diff -u -r1.32.4.1 -r1.32.4.2 --- squid3/src/Store.h 15 Aug 2007 01:00:00 -0000 1.32.4.1 +++ squid3/src/Store.h 20 Aug 2007 03:07:44 -0000 1.32.4.2 @@ -1,6 +1,5 @@ - /* - * $Id: Store.h,v 1.32.4.1 2007/08/15 01:00:00 amosjeffries Exp $ + * $Id: Store.h,v 1.32.4.2 2007/08/20 03:07:44 amosjeffries Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -34,6 +33,11 @@ #ifndef SQUID_STORE_H #define SQUID_STORE_H +/** + \defgroup StoreAPI Store API + \ingroup FileSystems + */ + #include "squid.h" #include @@ -57,6 +61,9 @@ class StoreSearch; +/** + \ingroup StoreAPI + */ class StoreEntry : public hash_link { @@ -192,6 +199,7 @@ bool validLength() const; }; +/// \ingroup StoreAPI class NullStoreEntry:public StoreEntry { @@ -225,11 +233,14 @@ static NullStoreEntry _instance; }; +/// \ingroup StoreAPI typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata); -/* Abstract base class that will replace the whole store and swapdir interface. */ - +/** + \ingroup StoreAPI + * Abstract base class that will replace the whole store and swapdir interface. + */ class Store : public RefCountable { @@ -298,41 +309,70 @@ static RefCount CurrentRoot; }; +/// \ingroup StoreAPI typedef RefCount StorePointer; +/// \ingroup StoreAPI SQUIDCEXTERN size_t storeEntryInUse(); #if UNUSED_CODE_20070420 SQUIDCEXTERN off_t storeLowestMemReaderOffset(const StoreEntry * entry); #endif + +/// \ingroup StoreAPI SQUIDCEXTERN const char *storeEntryFlags(const StoreEntry *); +/// \ingroup StoreAPI extern void storeEntryReplaceObject(StoreEntry *, HttpReply *); + +/// \ingroup StoreAPI SQUIDCEXTERN StoreEntry *storeGetPublic(const char *uri, const method_t method); +/// \ingroup StoreAPI SQUIDCEXTERN StoreEntry *storeGetPublicByRequest(HttpRequest * request); +/// \ingroup StoreAPI SQUIDCEXTERN StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const method_t method); +/// \ingroup StoreAPI SQUIDCEXTERN StoreEntry *storeCreateEntry(const char *, const char *, request_flags, method_t); +/// \ingroup StoreAPI SQUIDCEXTERN void storeInit(void); +/// \ingroup StoreAPI extern void storeRegisterWithCacheManager(CacheManager & manager); +/// \ingroup StoreAPI SQUIDCEXTERN void storeConfigure(void); +/// \ingroup StoreAPI SQUIDCEXTERN void storeFreeMemory(void); +/// \ingroup StoreAPI SQUIDCEXTERN int expiresMoreThan(time_t, time_t); #if STDC_HEADERS +/// \ingroup StoreAPI SQUIDCEXTERN void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2; #else +/// \ingroup StoreAPI SQUIDCEXTERN void storeAppendPrintf(); #endif +/// \ingroup StoreAPI SQUIDCEXTERN void storeAppendVPrintf(StoreEntry *, const char *, va_list ap); +/// \ingroup StoreAPI SQUIDCEXTERN ssize_t objectLen(const StoreEntry * e); +/// \ingroup StoreAPI SQUIDCEXTERN int storeTooManyDiskFilesOpen(void); +/// \ingroup StoreAPI SQUIDCEXTERN void storeHeapPositionUpdate(StoreEntry *, SwapDir *); +/// \ingroup StoreAPI SQUIDCEXTERN void storeSwapFileNumberSet(StoreEntry * e, sfileno filn); +/// \ingroup StoreAPI SQUIDCEXTERN void storeFsInit(void); +/// \ingroup StoreAPI SQUIDCEXTERN void storeFsDone(void); +/// \ingroup StoreAPI SQUIDCEXTERN void storeReplAdd(const char *, REMOVALPOLICYCREATE *); +/// \ingroup StoreAPI extern FREE destroyStoreEntry; -/* should be a subclass of Packer perhaps ? */ +/** + \ingroup StoreAPI + \todo should be a subclass of Packer perhaps ? + */ SQUIDCEXTERN void packerToStoreInit(Packer * p, StoreEntry * e); #ifdef _USE_INLINE_ Index: squid3/src/StoreSwapLogData.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/StoreSwapLogData.h,v retrieving revision 1.3.26.3 retrieving revision 1.3.26.4 diff -u -r1.3.26.3 -r1.3.26.4 --- squid3/src/StoreSwapLogData.h 20 Aug 2007 01:11:41 -0000 1.3.26.3 +++ squid3/src/StoreSwapLogData.h 20 Aug 2007 03:07:44 -0000 1.3.26.4 @@ -1,5 +1,5 @@ /* - * $Id: StoreSwapLogData.h,v 1.3.26.3 2007/08/20 01:11:41 amosjeffries Exp $ + * $Id: StoreSwapLogData.h,v 1.3.26.4 2007/08/20 03:07:44 amosjeffries Exp $ * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -34,8 +34,29 @@ #define SQUID_STORESWAPLOGDATA_H /** - \defgroup SwapStateAPI swap.state file Structure + \defgroup FileFormatSwapStateAPI swap.state File Structure \ingroup FileSystems + \section ImplementationNotes Implementation Notes + \par + * When writing an object to disk, we must first write the meta data. + * This is done with a couple of functions. First, storeSwapMetaPack() + * takes a StoreEntry as a parameter and returns a tlv linked + * list. Second, storeSwapMetaPack() converts the tlv list + * into a character buffer that we can write. + * + \note MemObject has a MemObject::swap_hdr_sz. + * This value is the size of that character buffer; the size of the + * swap file meta data. The StoreEntry has a member + * StoreEntry::swap_file_sz that represents the size of the disk file. + * Thus, the size of the object "content" is + \fn StoreEntry->swap_file_sz - MemObject->swap_hdr_sz; \endfn + \note The swap file content includes the HTTP reply headers and the HTTP reply body (if any). + * + \par + * When reading a swap file, there is a similar process to extract + * the swap meta data. First, storeSwapMetaUnpack() converts a + * character buffer into a tlv linked list. It also tells us + * the value for MemObject->swap_hdr_sz. */ #include "squid.h" @@ -45,7 +66,7 @@ * know the dirn .. */ /** - \ingroup SwapStateAPI + \ingroup FielFormatSwapStateAPI \note This information is current as of version 2.2.STABLE4 * \li Binary format on disk. @@ -142,7 +163,7 @@ MEMPROXY_CLASS_INLINE(StoreSwapLogData) /**DOCS_NOSEMI*/ -/// \ingroup SwapStateAPI +/// \ingroup FileFormatSwapStateAPI class StoreSwapLogHeader { public: Index: squid3/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/enums.h,v retrieving revision 1.37.2.1 retrieving revision 1.37.2.2 diff -u -r1.37.2.1 -r1.37.2.2 --- squid3/src/enums.h 15 Aug 2007 01:00:03 -0000 1.37.2.1 +++ squid3/src/enums.h 20 Aug 2007 03:07:44 -0000 1.37.2.2 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.37.2.1 2007/08/15 01:00:03 amosjeffries Exp $ + * $Id: enums.h,v 1.37.2.2 2007/08/20 03:07:44 amosjeffries Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -427,21 +427,99 @@ MEM_MAX } mem_type; -/* - * NOTE! We must preserve the order of this list! +/** + \ingroup SwapStoreAPI + \todo AYJ: for critical lists like this we should use A=64,B=65 etc to enforce and reserve values. + \note NOTE! We must preserve the order of this list! + * + \section StoreSwapMeta Store "swap meta" Description + \par + * "swap meta" refers to a section of meta data stored at the beginning + * of an object that is stored on disk. This meta data includes information + * such as the object's cache key (MD5), URL, and part of the StoreEntry + * structure. + * + \par + * The meta data is stored using a TYPE-LENGTH-VALUE format. That is, + * each chunk of meta information consists of a TYPE identifier, a + * LENGTH field, and then the VALUE (which is LENGTH octets long). */ enum { - STORE_META_VOID, /* should not come up */ - STORE_META_KEY_URL, /* key w/ keytype */ + /** + * Just a placeholder for the zeroth value. It is never used on disk. + */ + STORE_META_VOID, + + /** + \deprecated + * This represents the case when we use the URL as the cache + * key, as Squid-1.1 does. Currently we don't support using + * a URL as a cache key, so this is not used. + */ + STORE_META_KEY_URL, + + /** + \deprecated + * For a brief time we considered supporting SHA (secure + * hash algorithm) as a cache key. Nobody liked it, and + * this type is not currently used. + */ STORE_META_KEY_SHA, + + /** + * This represents the MD5 cache key that Squid currently uses. + * When Squid opens a disk file for reading, it can check that + * this MD5 matches the MD5 of the user's request. If not, then + * something went wrong and this is probably the wrong object. + */ STORE_META_KEY_MD5, - STORE_META_URL, /* the url , if not in the header */ - STORE_META_STD, /* standard metadata */ - STORE_META_HITMETERING, /* reserved for hit metering */ + + /** + * The object's URL. This also may be matched against a user's + * request for cache hits to make sure we got the right object. + */ + STORE_META_URL, + + /** + * This is the "standard metadata" for an object. + * Really its just this middle chunk of the StoreEntry structure: + \code + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + size_t swap_file_sz; + u_short refcount; + u_short flags; + \endcode + */ + STORE_META_STD, + + /** + * Reserved for future hit-metering (RFC 2227) stuff + */ + STORE_META_HITMETERING, + + /// \todo DOCS: document. STORE_META_VALID, - STORE_META_VARY_HEADERS, /* Stores Vary request headers */ - STORE_META_STD_LFS, /* standard metadata in lfs format */ - STORE_META_OBJSIZE, /* object size, not impleemented, squid26 compatibility */ + + /** + * Stores Vary request headers + */ + STORE_META_VARY_HEADERS, + + /** + * Updated version of STORE_META_STD, with support for >2GB objects. + * As STORE_META_STD except that the swap_file_sz is a 64-bit integer instead of 32-bit. + */ + STORE_META_STD_LFS, + + /** + \deprecated + * Object size, not implemented, squid26 compatibility + */ + STORE_META_OBJSIZE, + STORE_META_END }; Index: squid3/src/fs/FileSystems.dox =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/fs/Attic/FileSystems.dox,v retrieving revision 1.1.2.5 retrieving revision 1.1.2.6 diff -u -r1.1.2.5 -r1.1.2.6 --- squid3/src/fs/FileSystems.dox 31 Jul 2007 04:29:07 -0000 1.1.2.5 +++ squid3/src/fs/FileSystems.dox 20 Aug 2007 03:07:44 -0000 1.1.2.6 @@ -1,6 +1,5 @@ /** \defgroup FileSystems Storage Filesystems -\ingroup Components \section Introduction Introduction \par