--------------------- PatchSet 4805 Date: 2007/06/24 11:26:11 Author: amosjeffries Branch: squid3-ipv6 Tag: (none) Log: Make Squid CNAME-aware. Recursing to lookup IPA if a CNAME is given. - Merges any given implicitly in the DNS packet with the CNAME results. - Currently merges without removing duplicate IPA from the ipcache list. (Also fixes a fde compile error introduced by last update). Members: src/dns_internal.cc:1.15.6.25->1.15.6.26 src/fde.cc:1.3.10.3->1.3.10.4 src/fde.h:1.7.8.6->1.7.8.7 src/ipcache.cc:1.9.2.29->1.9.2.30 Index: squid3/src/dns_internal.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/dns_internal.cc,v retrieving revision 1.15.6.25 retrieving revision 1.15.6.26 diff -u -r1.15.6.25 -r1.15.6.26 --- squid3/src/dns_internal.cc 21 Jun 2007 12:29:39 -0000 1.15.6.25 +++ squid3/src/dns_internal.cc 24 Jun 2007 11:26:11 -0000 1.15.6.26 @@ -1,6 +1,6 @@ /* - * $Id: dns_internal.cc,v 1.15.6.25 2007/06/21 12:29:39 amosjeffries Exp $ + * $Id: dns_internal.cc,v 1.15.6.26 2007/06/24 11:26:11 amosjeffries Exp $ * * DEBUG: section 78 DNS lookups; interacts with lib/rfc1035.c * AUTHOR: Duane Wessels @@ -882,6 +882,19 @@ } } +void +idnsDropMessage(rfc1035_message *message, idns_query *q) +{ + rfc1035MessageDestroy(message); + + /* BUG 1968: Unlink previously linked q->hash */ + /* previously susceptable to infinite loop testing dead hash entries */ + if (q->hash.key) { + hash_remove_link(idns_lookup_hash, &q->hash); + q->hash.key = NULL; + } +} + static void idnsGrokReply(const char *buf, size_t sz) { @@ -965,14 +978,7 @@ q->attempt++; } - rfc1035MessageDestroy(message); - - /* BUG 1968: Unlink previously linked q->hash */ - /* previously susceptable to infinite loop testing dead hash entries */ - if (q->hash.key) { - hash_remove_link(idns_lookup_hash, &q->hash); - q->hash.key = NULL; - } + idnsDropMessage(message, q); q->start_t = current_time; q->id = idnsQueryID(); @@ -997,20 +1003,13 @@ #if USE_IPV6 if(n <= 0 && q->need_A) { - /* ERROR or NO AAA exist. Failover to A records. */ + /* ERROR or NO AAAA exist. Failover to A records. */ if(n == 0) debugs(78, 3, "idnsGrokReply: " << q->name << " has no AAAA records. Looking up A record instead."); else debugs(78, 3, "idnsGrokReply: " << q->name << " AAAA query failed. Trying A now instead."); - rfc1035MessageDestroy(message); - - /* BUG 1968: Unlink previously linked q->hash */ - /* previously susceptable to infinite loop testing dead hash entries */ - if (q->hash.key) { - hash_remove_link(idns_lookup_hash, &q->hash); - q->hash.key = NULL; - } + idnsDropMessage(message, q); q->start_t = current_time; q->id = idnsQueryID(); Index: squid3/src/fde.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/fde.cc,v retrieving revision 1.3.10.3 retrieving revision 1.3.10.4 diff -u -r1.3.10.3 -r1.3.10.4 --- squid3/src/fde.cc 22 Jun 2007 06:09:45 -0000 1.3.10.3 +++ squid3/src/fde.cc 24 Jun 2007 11:26:11 -0000 1.3.10.4 @@ -1,6 +1,6 @@ /* - * $Id: fde.cc,v 1.3.10.3 2007/06/22 06:09:45 amosjeffries Exp $ + * $Id: fde.cc,v 1.3.10.4 2007/06/24 11:26:11 amosjeffries Exp $ * * DEBUG: none FDE * AUTHOR: Robert Collins @@ -39,18 +39,6 @@ #include "Store.h" #include "comm.h" -fde::fde() -{ - clear(); -} - -void -fde::clear() -{ - memset(this, 0, sizeof(fde)); - local_addr.SetEmpty(); // IPAddress likes to be setup nicely. -} - bool fde::readPending(int fdNumber) { Index: squid3/src/fde.h =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/fde.h,v retrieving revision 1.7.8.6 retrieving revision 1.7.8.7 diff -u -r1.7.8.6 -r1.7.8.7 --- squid3/src/fde.h 22 Jun 2007 06:09:45 -0000 1.7.8.6 +++ squid3/src/fde.h 24 Jun 2007 11:26:11 -0000 1.7.8.7 @@ -1,6 +1,6 @@ /* - * $Id: fde.h,v 1.7.8.6 2007/06/22 06:09:45 amosjeffries Exp $ + * $Id: fde.h,v 1.7.8.7 2007/06/24 11:26:11 amosjeffries Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -43,8 +43,12 @@ { public: + fde() { clear(); }; /** Clear the fde class properly back to NULL equivalent. */ - void clear(); + inline void clear() { + memset(this, 0, sizeof(fde)); + local_addr.SetEmpty(); // IPAddress likes to be setup nicely. + } /* NOTE: memset is used on fdes today. 20030715 RBC */ static void DumpStats (StoreEntry *); Index: squid3/src/ipcache.cc =================================================================== RCS file: /cvsroot/squid-sf//squid3/src/ipcache.cc,v retrieving revision 1.9.2.29 retrieving revision 1.9.2.30 diff -u -r1.9.2.29 -r1.9.2.30 --- squid3/src/ipcache.cc 10 Jun 2007 11:01:24 -0000 1.9.2.29 +++ squid3/src/ipcache.cc 24 Jun 2007 11:26:12 -0000 1.9.2.30 @@ -1,6 +1,6 @@ /* - * $Id: ipcache.cc,v 1.9.2.29 2007/06/10 11:01:24 amosjeffries Exp $ + * $Id: ipcache.cc,v 1.9.2.30 2007/06/24 11:26:12 amosjeffries Exp $ * * DEBUG: section 14 IP Cache * AUTHOR: Harvest Derived @@ -90,7 +90,7 @@ #else static IDNSCB ipcacheHandleReply; #endif -static IPH dummy_handler; +static IPH ipcacheHandleCnameRecurse; static int ipcacheExpiredEntry(ipcache_entry *); static int ipcache_testname(void); #if USE_DNSSERVERS @@ -344,7 +344,10 @@ int j, k; i->addrs.in_addrs = (IPAddress *)xcalloc(ipcount, sizeof(IPAddress)); + for(int l = 0; l < ipcount; l++) + i->addrs.in_addrs[l].SetEmpty(); // perform same init actions as constructor would. i->addrs.bad_mask = (unsigned char *)xcalloc(ipcount, sizeof(unsigned char)); + memset(i->addrs.bad_mask, 0 sizeof(unsigned char) * ipcount); for (j = 0, k = 0; k < ipcount; k++) { if ( A[k] = i->addrs.in_addrs[j] ) @@ -379,15 +382,19 @@ ipcacheParse(ipcache_entry *i, rfc1035_rr * answers, int nr, const char *error_message) { int k; - int j; + int j = 0; int na = 0; int ttl = 0; + bool cnames = false; const char *name = (const char *)i->hash.key; i->expires = squid_curtime + Config.negativeDnsTtl; i->flags.negcached = 1; safe_free(i->addrs.in_addrs); + assert(i->addrs.in_addrs == NULL); safe_free(i->addrs.bad_mask); + assert(i->addrs.bad_mask == NULL); safe_free(i->error_message); + assert(i->error_message == NULL); i->addrs.count = 0; if (nr < 0) { @@ -399,14 +406,12 @@ if (nr == 0) { debugs(14, 3, "ipcacheParse: No DNS records in response to '" << name << "'"); i->error_message = xstrdup("No DNS records"); - return 0; + return -1; } assert(answers); for (k = 0; k < nr; k++) { - if (answers[k]._class != RFC1035_CLASS_IN) - continue; #if USE_IPV6 if (answers[k].type == RFC1035_TYPE_AAAA) { @@ -427,6 +432,31 @@ na++; continue; } + + /* With A and AAAA, the CNAME does not necessarily come with additional records to use. */ + if (answers[k].type == RFC1035_TYPE_CNAME) { + debugs(14, 5, "ipcacheParse: " << name << " CNAME " << answers[k].rdata << " (checking destination: " << i << ")."); + const ipcache_addrs *res = ipcache_gethostbyname(answers[k].rdata, 0); + if(res) { + na += res->count; + debugs(14, 5, "ipcacheParse: CNAME " << answers[k].rdata << " already has " << res->count << " IPs cached."); + } + else { + /* lock this object for changes until CNAME loop releases it. */ + ipcacheLockEntry(i); + + /* keep going on this, but flag the fact that we need to wait for more IPs */ + debugs(14, 5, "ipcacheParse: CNAME " << answers[k].rdata << " has no IPs! Recursing."); + ipcache_nbgethostbyname(answers[k].rdata, ipcacheHandleCnameRecurse, new generic_cbdata(i) ); + cnames = true; + } + continue; + } + } + + if(na == 0 && cnames) { + /* don't set any error message (yet). Allow recursion to do its work first. */ + return 0; } if (na == 0) { @@ -436,12 +466,11 @@ } i->addrs.in_addrs = (IPAddress *)xcalloc(na, sizeof(IPAddress)); - i->addrs.in_addrs->SetEmpty(); // perform same init actions as constructor would. + for(int l = 0; l < na; l++) + i->addrs.in_addrs[l].SetEmpty(); // perform same init actions as constructor would. i->addrs.bad_mask = (unsigned char *)xcalloc(na, sizeof(unsigned char)); for (j = 0, k = 0; k < nr; k++) { - if (answers[k]._class != RFC1035_CLASS_IN) - continue; if (answers[k].type == RFC1035_TYPE_A) { if (answers[k].rdlength != sizeof(struct in_addr)) @@ -451,7 +480,7 @@ xmemcpy(&temp, answers[k].rdata, sizeof(struct in_addr)); i->addrs.in_addrs[j] = temp; - debugs(14, 3, "ipcacheParse: #" << j << " " << i->addrs.in_addrs[j]); + debugs(14, 3, "ipcacheParse: " << name << " #" << j << " " << i->addrs.in_addrs[j]); j++; #if USE_IPV6 @@ -463,17 +492,27 @@ xmemcpy(&temp, answers[k].rdata, sizeof(struct in6_addr)); i->addrs.in_addrs[j] = temp; - debugs(14, 3, "ipcacheParse: #" << j << " " << i->addrs.in_addrs[j] ); + debugs(14, 3, "ipcacheParse: " << name << " #" << j << " " << i->addrs.in_addrs[j] ); j++; #endif - } else if (answers[k].type != RFC1035_TYPE_CNAME) { - debugs(14, 3, "ipcacheParse: #x CNAME (dropped)."); - continue; + } else if (answers[k].type == RFC1035_TYPE_CNAME) { + debugs(14, 3, "ipcacheParse: " << name << " #x CNAME " << answers[k].rdata); + const ipcache_addrs *res = ipcache_gethostbyname(answers[k].rdata, 0); + if(res) { + /* NP: the results of *that* query need to be integrated in place of the CNAME */ + /* Ideally we should also integrate the min TTL of the above IPA's into ttl. */ + for(int l = 0; l < res->count; l++, j++) { + i->addrs.in_addrs[j] = res->in_addrs[l]; + debugs(14, 3, "ipcacheParse: " << name << " #" << j << " " << i->addrs.in_addrs[j] ); + } + } + else { + debugs(14, 9, "ipcacheParse: " << answers[k].rdata << " (CNAME) waiting on A/AAAA records."); + } } if (ttl == 0 || (int) answers[k].ttl < ttl) ttl = answers[k].ttl; - } assert(j == na); @@ -505,6 +544,7 @@ ipcacheHandleReply(void *data, rfc1035_rr * answers, int na, const char *error_message) #endif { + int done; ipcache_entry *i; static_cast(data)->unwrap(&i); IpcacheStats.replies++; @@ -512,14 +552,17 @@ tvSubMsec(i->request_time, current_time)); #if USE_DNSSERVERS - ipcacheParse(i, reply); + done = ipcacheParse(i, reply); #else - ipcacheParse(i, answers, na, error_message); + done = ipcacheParse(i, answers, na, error_message); #endif - ipcacheAddEntry(i); - ipcacheCallback(i); + /* If we have not produced either IPs or Error immediately, wait for recursion to finish. */ + if(done != 0 || error_message != NULL) { + ipcacheAddEntry(i); + ipcacheCallback(i); + } } void @@ -662,7 +705,7 @@ IpcacheStats.misses++; if (flags & IP_LOOKUP_IF_MISS) - ipcache_nbgethostbyname(name, dummy_handler, NULL); + ipcache_nbgethostbyname(name, ipcacheHandleCnameRecurse, NULL); return NULL; } @@ -722,9 +765,107 @@ } static void -dummy_handler(const ipcache_addrs * addrsnotused, void *datanotused) +ipcacheHandleCnameRecurse(const ipcache_addrs *addrs, void *cbdata) { - return; + ipcache_entry *i = NULL; + char *pname = NULL; + int j = 0; + int fc = 0; + void *tmpbuf = NULL; + int ttl = 0; + generic_cbdata* gcb = (generic_cbdata*)cbdata; + // count of addrs at parent and child (REQ as .count is a char type!) + int ccount = 0, pcount = 0; + + debugs(14, 5, "ipcacheHandleCnameRecurse: Handling basic A/AAAA response."); + + /* IFF no CNAME recursion being processed. do nothing. */ + if(cbdata == NULL) + return; + + debugs(14, 5, "ipcacheHandleCnameRecurse: Handling CNAME recursion. CBDATA('" << gcb->data << "')"); + + gcb->unwrap(&i); + assert(i != NULL); + + pname = (char*)i->hash.key; + assert(pname != NULL); + + ccount = (0+ addrs->count); + pcount = (0+ i->addrs.count); + ttl = i->expires; + fc = pcount; + + /* IFF no CNAME results. do none of the processing BUT finish anyway. */ + if(addrs) { + fc += ccount; + + debugs(14, 5, "ipcacheHandleCnameRecurse: Merge IP Lists for " << pname << " (" << pcount << "+" << ccount << ")"); + + if( pcount > 0) { + /* IFF the parent initial lookup was given Additional records with A */ + /* we need to merge the two lists (dropping duplicates) */ + + debugs(14, 5, "ipcacheHandleCnameRecurse: Add " << pcount << " IPs from Initial Lookup."); + + // copy the old IPs into the new list buffer. + tmpbuf = i->addrs.in_addrs; + i->addrs.in_addrs = (IPAddress *)xcalloc(fc, sizeof(IPAddress)); + for(int l = 0; j < fc; l++, j++) { + i->addrs.in_addrs[j] = ((IPAddress*)tmpbuf)[l]; + debugs(14, 5, "ipcacheHandleCnameRecurse: CNAME (" << pname << ") #" << j << " " << i->addrs.in_addrs[j] ); + } + + // copy the 'bad IP mask' + safe_free(i->addrs.bad_mask); + i->addrs.bad_mask = (unsigned char*)xcalloc(fc, sizeof(unsigned char)); + memset(i->addrs.bad_mask, 0, sizeof(unsigned char)*fc); + } + else { + i->addrs.in_addrs = (IPAddress *)xcalloc(ccount, sizeof(IPAddress)); + i->addrs.bad_mask = (unsigned char *)xcalloc(ccount, sizeof(unsigned char)); + memset(i->addrs.bad_mask, 0, sizeof(unsigned char) * ccount); + } + + /* add new IP records to entry */ + debugs(14, 5, "ipcacheHandleCnameRecurse: Add " << ccount << " IPs from CNAME Lookup."); + for(int l = 0; j < fc; l++, j++) { + i->addrs.in_addrs[j] = addrs->in_addrs[l]; + debugs(14, 3, "ipcacheHandleCnameRecurse: CNAME (" << pname << ") #" << j << " " << i->addrs.in_addrs[j] ); + } + + assert(j == fc); + + if (fc < 256) + i->addrs.count = (unsigned char) fc; + else + i->addrs.count = 255; + + if (ttl == 0 || ttl > Config.positiveDnsTtl) + ttl = Config.positiveDnsTtl; + + if (ttl < Config.negativeDnsTtl) + ttl = Config.negativeDnsTtl; + + i->expires = squid_curtime + ttl; + + i->flags.negcached = 0; + + i->addrs.cur = 0; + + i->addrs.badcount = 0; + } + + if(fc == 0) { + i->error_message = xstrdup("No DNS Records"); + } + + /* unlock the parent object locked to prevent corruption during the recursion delay */ + ipcacheUnlockEntry(i); + + /* finish the lookup we were doing on parent when we got side-tracked for CNAME loop */ + ipcacheAddEntry(i); + ipcacheCallback(i); } void @@ -839,7 +980,7 @@ ia->cur = 0; } - debugs(14, 3, "ipcacheCycleAddr: " << name << " now at " << ia->in_addrs[ia->cur]); + debugs(14, 3, "ipcacheCycleAddr: " << name << " now at " << ia->in_addrs[ia->cur] << " (" << ia->cur << " of " << ia->count << ")"); } /*