--------------------- PatchSet 3750 Date: 2002/03/19 23:54:50 Author: kinkie Branch: customlog Tag: (none) Log: Initial implementation of the custom logformat engine. The configuration file parser is halfway done, and #warnings have been littered here and there where more works needs to be done. Members: src/access_log.c:1.16->1.16.4.1 src/acl.c:1.44->1.44.6.1 src/cache_cf.c:1.41->1.41.4.1 src/cf.data.pre:1.54->1.54.4.1 src/enums.h:1.31->1.31.4.1 src/protos.h:1.48->1.48.4.1 src/structs.h:1.50->1.50.4.1 src/typedefs.h:1.25->1.25.16.1 Index: squid/src/access_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/access_log.c,v retrieving revision 1.16 retrieving revision 1.16.4.1 diff -u -r1.16 -r1.16.4.1 --- squid/src/access_log.c 27 Feb 2002 09:16:51 -0000 1.16 +++ squid/src/access_log.c 19 Mar 2002 23:54:50 -0000 1.16.4.1 @@ -1,6 +1,6 @@ /* - * $Id: access_log.c,v 1.16 2002/02/27 09:16:51 squidadm Exp $ + * $Id: access_log.c,v 1.16.4.1 2002/03/19 23:54:50 kinkie Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -38,6 +38,7 @@ static void accessLogSquid(AccessLogEntry * al); static void accessLogCommon(AccessLogEntry * al); +static void accessLogCustom(AccessLogEntry *al); static Logfile *logfile = NULL; #if HEADERS_LOG static Logfile *headerslog = NULL; @@ -261,6 +262,175 @@ } static void +accessLogCustom(AccessLogEntry * al) +{ +#warning "TODO: missing implementation" +} + +struct logformat_token_table_entry { + char *config; + logformat_bcode_t token_type; +}; + +struct logformat_token_table_entry logformat_token_table[] = { + { "a", LFT_SERVER_IP_ADDRESS }, + { ">A", LFT_SERVER_IP_OR_PEER_NAME }, + { "h{", LFT_REPLY_HEADER }, /* requires lots of extra handling */ + { "H", LFT_REPLY_ALL_HEADERS }, + { "un", LFT_USER_NAME }, + { "ur", LFT_USER_REALM }, + { "us", LFT_USER_SCHEME }, + { "ui", LFT_USER_IDENT }, + { "Sh", LFT_HTTP_CODE }, + { "Ss", LFT_SQUID_STATUS_CODE }, + { NULL, LFT_NONE } /* this must be last */ +#warning "TODO: complete this" +}; + +/* parses a single token. Returns the token length in characters, + * and fills in the lt item with the token information. + * def is for sure null-terminated +*/ +static int +accessLogGetNewLogFormatToken(logformat_token *lt, char *def) +{ + char *cp, *cp2, *cur=def; + struct logformat_token_table_entry *lte; + int j, l; + if (def[0]!='%') { + /* it's a string for sure, until \0 or the next % */ + cp=strchr(def,'%'); + if (cp==NULL) { + l=strlen(def); + } else { + l=cp-def; + } + debug(46,0)("Got a string of length %d, '%*.*s'\n",l,l,l,def); + cp2=xmalloc(strlen(def)+1); + xstrncpy(cp2,def,l); + lt->type=LFT_STRING; + lt->strarg=cp; + return l; + } + cur++; + lte=logformat_token_table; + lt->type=LFT_NONE; + while (lte->config!=NULL) { + if (0==strncmp(lte->config,cur,strlen(lte->config))) { + debug(46,0)("Got configuration token '%s'\n",lte->config); + lt->type=lte->token_type; + cur+=strlen(lte->config); + break; + } + lte++; + } + if (lt->type==LFT_NONE) { + char buf[1024]; + snprintf(buf,1024,"Can't parse configuration token: '%s'\n", + def); + fatal(buf); + } + switch (lt->type) { + case LFT_TIME_STRFTIME: + if (NULL==(cp=strchr(cur,'}'))) { + char buf[1024]; + snprintf(buf,1024,"Can't find end of strftime definition in '%s'\n", + def); + fatal(buf); + } + lt->strarg=xmalloc(cp-cur+1); + xstrncpy(lt->strarg,cur,cp-cur); + cur=cp+1; + break; + case LFT_TIME_SUBSECOND: + j=0; + while (isdigit(cur[j])) + j++; + /* we have j+1 digits now */ + if (j==0) { + fatal("No digits in sub-second resolution timer\n"); + } + lt->strarg=xmalloc(j+2); + xstrncpy(lt->strarg,cur,j+1); + cur+=j+1; + break; + case LFT_REQUEST_HEADER: + case LFT_REPLY_HEADER: + if (NULL==(cp=strchr(cur,'}'))) { + char buf[1024]; + snprintf(buf,1024,"Can't find end of header definition in '%s'\n", + def); + fatal(buf); + } + if (NULL==(cp2=strchr(cur,':'))) { /* simple header */ + lt->strarg=xmalloc(cp-cur+1); + xstrncpy(lt->strarg,cur,cp-cur); + cur=cp+1; + break; + } + fatal("Ranged headers are not yet implemented. Bug Kinkie about it"); + break; + default: + break; /* no fixup needed */ + } +#warning("TODO: complete with the missing heaeder types"); + return (cur-def); +} + +void +accessLogParseLogFormat (logformat_token **fmt, char *def) +{ + char *cp, *cur, *eos; + logformat_token *new_lt, *last_lt; + + debug(46,1)("accessLogParseLogFormat: got definition '%s'\n",def); + + cp=strchr(def,'"'); + if (cp) { + debug(46,1)("Stripping leading quotes\n"); + def=cp+1; + } else { + debug(46,0)("No quotes in definition?"); + } + cp=strrchr(def,'"'); + if (cp) { + debug(46,1)("Stripping tailing quotes\n"); + *cp='\0'; + } else { + debug(46,0)("No tailing quotes?"); + } + if (strlen(cp+1)) + debug(46,5)("Warning: stripping some characters, might be legitimate\n", + cp+1); + + /* very inefficent parser, but who cares, this needs to be simple */ + /* First off, let's tokenize, we'll optimize in a second pass. + A token can either be a %-prefixed sequence (usually a dynamic + token but it can be an escaped sequence), or a string.*/ + cur=def; + eos=def+strlen(def); + *fmt=new_lt=last_lt=xmalloc(sizeof(logformat_token)); + cur += accessLogGetNewLogFormatToken(new_lt,cur); + while (cur < eos) { + new_lt=xmalloc(sizeof(logformat_token)); + last_lt->next=new_lt; + last_lt=new_lt; + cur += accessLogGetNewLogFormatToken(new_lt,cur); + } + /* TODO: optimize merging consecutive strings together */ + /* crossing fingers */ +} + +static void accessLogCommon(AccessLogEntry * al) { const char *client = NULL; @@ -303,6 +473,7 @@ if (al->hier.host[0] == '\0') xstrncpy(al->hier.host, dash_str, SQUIDHOSTNAMELEN); +#warning "TODO: fix implementation for accesslogcustom" if (Config.onoff.common_log) accessLogCommon(al); else Index: squid/src/acl.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/acl.c,v retrieving revision 1.44 retrieving revision 1.44.6.1 diff -u -r1.44 -r1.44.6.1 --- squid/src/acl.c 13 Jan 2002 03:21:39 -0000 1.44 +++ squid/src/acl.c 19 Mar 2002 23:54:50 -0000 1.44.6.1 @@ -1,6 +1,6 @@ /* - * $Id: acl.c,v 1.44 2002/01/13 03:21:39 squidadm Exp $ + * $Id: acl.c,v 1.44.6.1 2002/03/19 23:54:50 kinkie Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Duane Wessels @@ -55,7 +55,7 @@ static void aclDestroyIntRange(intrange *); static void aclLookupProxyAuthStart(aclCheck_t * checklist); static void aclLookupProxyAuthDone(void *data, char *result); -static struct _acl *aclFindByName(const char *name); +static acl *aclFindByName(const char *name); static int aclMatchAcl(struct _acl *, aclCheck_t *); static int aclMatchTime(acl_time_data * data, time_t when); static int aclMatchUser(void *proxyauth_acl, char *user); Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.41 retrieving revision 1.41.4.1 diff -u -r1.41 -r1.41.4.1 --- squid/src/cache_cf.c 11 Mar 2002 05:05:40 -0000 1.41 +++ squid/src/cache_cf.c 19 Mar 2002 23:54:51 -0000 1.41.4.1 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.41 2002/03/11 05:05:40 squidadm Exp $ + * $Id: cache_cf.c,v 1.41.4.1 2002/03/19 23:54:51 kinkie Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -60,6 +60,14 @@ static void dump_cachedir_option_readonly(StoreEntry * e, const char *option, SwapDir * sd); static void parse_cachedir_option_maxsize(SwapDir * sd, const char *option, const char *value, int reconfiguring); static void dump_cachedir_option_maxsize(StoreEntry * e, const char *option, SwapDir * sd); +static void parse_logformat_def(logformat ** logformat_definitions); +static void parse_access_log_def(customlog **customlog_definitions); +static void dump_logformat_def(StoreEntry * entry, const char *name, logformat * definitions); +static void dump_access_log_def(StoreEntry * entry, const char *name, customlog * definitions); +static void free_logformat_def(logformat **definitions); +static void free_access_log_def (customlog **definitions); + + static struct cache_dir_option common_cachedir_options[] = { {"read-only", parse_cachedir_option_readonly, dump_cachedir_option_readonly}, @@ -2390,3 +2398,74 @@ if (stat(path, &sb) < 0) fatalf("%s: %s", path, xstrerror()); } + +static void +parse_logformat (logformat_token ** fmt, char *def) { + accessLogParseLogFormat(fmt,def); +} + + +static void +parse_logformat_def(logformat ** logformat_definitions) { + logformat *nlf, *lf; + char *name, *def; + + lf=*logformat_definitions; + if (lf) + while (lf->next) + lf=lf->next; /* go to the end or NULL*/ + if (NULL==(name=strtok(NULL,w_space))) { /* the name */ + debug(3,0)("Can't find out accesslog definition name\n"); + return; + } + if (NULL==(def=strtok(NULL,"\r\n"))) { /* the definition, until EOL */ + debug(3,0)("Can't find out accesslog definition format\n"); + return; + } + + debug(3,1)("Logformat for '%s' is '%s'\n",name,def); + + nlf=xmalloc(sizeof(logformat)); + nlf->name=xmalloc(strlen(name)+1); + strcpy(nlf->name,name); + nlf->format=NULL; + parse_logformat(&nlf->format,def); + if (NULL==nlf->format) { /* error while parsing */ + xfree(nlf->name); + xfree(nlf); + return; + } + nlf->next=NULL; + if (lf==NULL) + *logformat_definitions=nlf; + else + lf->next=nlf; +} + +static void +parse_access_log_def(customlog **customlog_definitions) { +#warning "TODO: implement this" +} + +static void +dump_logformat_def(StoreEntry * entry, const char *name, logformat * definitions) +{ +#warning "TODO: implement this" +} + +static void +dump_access_log_def(StoreEntry * entry, const char *name, customlog * definitions) +{ +#warning "TODO: implement this" +} + +static void +free_logformat_def(logformat **definitions) +{ +#warning "TODO: implement this" +} + +static void free_access_log_def (customlog **definitions) +{ +#warning "TODO: implement this" +} Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.54 retrieving revision 1.54.4.1 diff -u -r1.54 -r1.54.4.1 --- squid/src/cf.data.pre 25 Feb 2002 03:15:33 -0000 1.54 +++ squid/src/cf.data.pre 19 Mar 2002 23:54:51 -0000 1.54.4.1 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.54 2002/02/25 03:15:33 squidadm Exp $ +# $Id: cf.data.pre,v 1.54.4.1 2002/03/19 23:54:51 kinkie Exp $ # # # SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -803,6 +803,36 @@ ones with no max-size specification last. DOC_END +NAME: logformat +TYPE: logformat_def +LOC: Config.Log.logformatsList +DEFAULT: none +DOC_START + Defnies an access logfile format. Documentation on the allowed fields + is to be finalized yet. Bug Kinkie about it. + Format is: + logformat "definition string" + +NOCOMMENT_START +logformat squid "%ts.%t#03 %T %A %m" +logformat common "% [acl acl ...] + + Will log to the specified file using the specified format (which + must be defined in a logformat directive) those entries which match + ALL the acl's specified (which must be defined in acl clauses). + If no acl is specified, all requests will be logged by this clause. +DOC_END NAME: cache_access_log TYPE: string Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.31 retrieving revision 1.31.4.1 diff -u -r1.31 -r1.31.4.1 --- squid/src/enums.h 27 Feb 2002 09:16:52 -0000 1.31 +++ squid/src/enums.h 19 Mar 2002 23:54:51 -0000 1.31.4.1 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.31 2002/02/27 09:16:52 squidadm Exp $ + * $Id: enums.h,v 1.31.4.1 2002/03/19 23:54:51 kinkie Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -732,6 +732,40 @@ DIGEST_READ_DONE } digest_read_state_t; +/* + * Bytecodes for the configureable logformat stuff + */ +typedef enum { + LFT_EOL, /* end-of-line */ + LFT_NONE, /* dummy */ + LFT_STRING, /* a static string. uses arg.strarg */ + LFT_CLIENT_IP_ADDRESS, + LFT_SERVER_IP_ADDRESS, + LFT_SERVER_IP_OR_PEER_NAME, + LFT_LOCAL_IP, + LFT_LOCAL_PORT, + LFT_TIME_SECONDS_SINCE_EPOCH, + LFT_TIME_STRFTIME, /* uses strarg for the strftime string */ + LFT_TIME_SUBSECOND, /* uses intarg for the width length */ + LFT_TIME_TO_HANDLE, + LFT_REQUEST_HEADER, /* uses strarg for the header name*/ + LFT_REQUEST_HEADER_ELEM, /* uses strarg, iarg for the elem #*/ + LFT_REQUEST_HEADER_ELEM_SEP, /* uses strarg, iarg and carg for the sep. */ + LFT_REQUEST_ALL_HEADERS, + LFT_REPLY_HEADER, /* uses strarg for the header name*/ + LFT_REPLY_HEADER_ELEM, /* uses strarg, iarg for the elem #*/ + LFT_REPLY_HEADER_ELEM_SEP, /* uses strarg, iarg and carg for the sep. */ + LFT_REPLY_ALL_HEADERS, + LFT_USER_NAME, + LFT_USER_REALM, + LFT_USER_SCHEME, + LFT_USER_IDENT, + LFT_HTTP_CODE, + LFT_SQUID_STATUS_CODE, + + +} logformat_bcode_t; + /* CygWin & Windows NT Port */ #if defined(_SQUID_MSWIN_) || defined(_SQUID_CYGWIN_) /* Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.48 retrieving revision 1.48.4.1 diff -u -r1.48 -r1.48.4.1 --- squid/src/protos.h 27 Feb 2002 09:16:52 -0000 1.48 +++ squid/src/protos.h 19 Mar 2002 23:54:51 -0000 1.48.4.1 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.48 2002/02/27 09:16:52 squidadm Exp $ + * $Id: protos.h,v 1.48.4.1 2002/03/19 23:54:51 kinkie Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -39,6 +39,7 @@ extern void accessLogClose(void); extern void accessLogInit(void); extern const char *accessLogTime(time_t); +extern void accessLogParseLogFormat (logformat_token **fmt, char *def); extern void hierarchyNote(HierarchyLogEntry *, hier_code, const char *); #if FORW_VIA_DB extern void fvdbCountVia(const char *key); @@ -1325,4 +1326,5 @@ extern void WIN32_Exit(void); #endif + #endif /* SQUID_PROTOS_H */ Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.50 retrieving revision 1.50.4.1 diff -u -r1.50 -r1.50.4.1 --- squid/src/structs.h 27 Feb 2002 09:16:53 -0000 1.50 +++ squid/src/structs.h 19 Mar 2002 23:54:51 -0000 1.50.4.1 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.50 2002/02/27 09:16:53 squidadm Exp $ + * $Id: structs.h,v 1.50.4.1 2002/03/19 23:54:51 kinkie Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -460,6 +460,8 @@ char *forward; #endif int rotateNumber; + logformat *logformatsList; + customlog *logsList; } Log; char *adminEmail; char *effectiveUser; @@ -2166,6 +2168,26 @@ } flags; }; +struct _logformat_token { + logformat_bcode_t type; + char *strarg; /* for strings */ + char carg; /* for characters */ + int32_t iarg; /* for integers */ + logformat_token *next; /* todo: move from linked list to array */ +}; + +struct _logformat { + char *name; + logformat_token *format; + logformat *next; +}; + +struct _customlog { + char *filename; + acl **aclList; + logformat *logFormat; +}; + struct cache_dir_option { const char *name; void (*parse) (SwapDir * sd, const char *option, const char *value, int reconfiguring); Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/typedefs.h,v retrieving revision 1.25 retrieving revision 1.25.16.1 diff -u -r1.25 -r1.25.16.1 --- squid/src/typedefs.h 10 Oct 2001 18:07:43 -0000 1.25 +++ squid/src/typedefs.h 19 Mar 2002 23:54:51 -0000 1.25.16.1 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.25 2001/10/10 18:07:43 squidadm Exp $ + * $Id: typedefs.h,v 1.25.16.1 2002/03/19 23:54:51 kinkie Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -187,6 +187,9 @@ typedef struct _storerepl_entry storerepl_entry_t; typedef struct _diskd_queue diskd_queue; typedef struct _Logfile Logfile; +typedef struct _logformat_token logformat_token; +typedef struct _logformat logformat; +typedef struct _customlog customlog; typedef struct _RemovalPolicy RemovalPolicy; typedef struct _RemovalPolicyWalker RemovalPolicyWalker; typedef struct _RemovalPurgeWalker RemovalPurgeWalker;