---------------------
PatchSet 1342 
Date: 2005/05/05 18:24:13
Author: nlewycky
Branch: prefetching
Tag: (none) 
Log:
Added recursion limits:
 - Don't recurse beyond three fetches (one original, two indirections)
 - Don't follow prefetchables found in a page referenced by an img element.

Members: 
	src/HTMLAnalysisStream.cc:1.1.2.4->1.1.2.5 
	src/HTMLAnalysisStream.h:1.1.2.2->1.1.2.3 
	src/HttpRequest.h:1.10->1.10.4.1 
	src/PrefetchStream.cc:1.1.2.6->1.1.2.7 
	src/PrefetchStream.h:1.1.2.1->1.1.2.2 
	src/client_side_reply.cc:1.57.2.1->1.57.2.2 
	src/client_side_request.cc:1.33.2.2->1.33.2.3 
	src/client_side_request.h:1.17.6.2->1.17.6.3 

Index: squid3/src/HTMLAnalysisStream.cc
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/Attic/HTMLAnalysisStream.cc,v
retrieving revision 1.1.2.4
retrieving revision 1.1.2.5
diff -u -r1.1.2.4 -r1.1.2.5
--- squid3/src/HTMLAnalysisStream.cc	26 Mar 2005 23:43:53 -0000	1.1.2.4
+++ squid3/src/HTMLAnalysisStream.cc	5 May 2005 18:24:13 -0000	1.1.2.5
@@ -1,6 +1,6 @@
 
 /*
- * $Id: HTMLAnalysisStream.cc,v 1.1.2.4 2005/03/26 23:43:53 nlewycky Exp $
+ * $Id: HTMLAnalysisStream.cc,v 1.1.2.5 2005/05/05 18:24:13 nlewycky Exp $
  *
  * DEBUG: section 93    HTML parsing and fetching
  * AUTHOR: Nick Lewycky
@@ -119,6 +119,8 @@
 
     relative_url = urlCanonicalClean(request);
 
+    debugs(93, 3, "depth: " << request->recursion_depth);
+
     debugs(93, 4, "analyzing " << relative_url);
 }
 
@@ -127,9 +129,9 @@
 }
 
 inline void
-HTMLAnalysisStream::prefetch(const string &url)
+HTMLAnalysisStream::prefetch(const string &url, bool allow_recursion)
 {
-    PrefetchStream::prefetch(url, relative_url, request);
+    PrefetchStream::prefetch(url, relative_url, request, allow_recursion);
 }
 
 void
@@ -154,13 +156,26 @@
 
     // Example:
     // <img src="https://squid.sourceforge.net/changesets/squid3/patches/foo.png">
+    if (strcasecmp(name, "img") == 0) {
+        while (*attr) {
+            if (strcasecmp(*attr, "src") == 0) {
+              self->prefetch(*(attr + 1), false);
+                return;
+            }
+
+            attr += 2;
+        }
+
+        return;
+    }
+
+    // Example:
     // <frame src="https://squid.sourceforge.net/changesets/squid3/patches/foo.html">
     // <iframe src="https://squid.sourceforge.net/changesets/squid3/patches/foo.html">
-    if (strcasecmp(name, "img") == 0 || strcasecmp(name, "frame") == 0 ||
-        strcasecmp(name, "iframe") == 0) {
+    if (strcasecmp(name, "frame") == 0 || strcasecmp(name, "iframe") == 0) {
         while (*attr) {
             if (strcasecmp(*attr, "src") == 0) {
-                self->prefetch(*(attr + 1));
+              self->prefetch(*(attr + 1), true);
                 return;
             }
 
@@ -179,7 +194,7 @@
         while (*attr) {
             if (strcasecmp(*attr, "href") == 0) {
                 if (is_fetchable) {
-                    self->prefetch(*(attr + 1));
+                    self->prefetch(*(attr + 1), true);
                     return;
                 } else
                     href = *(attr + 1);
@@ -189,7 +204,7 @@
                     return;
 
                 if (!href.empty()) {
-                    self->prefetch(*(attr + 1));
+                    self->prefetch(*(attr + 1), true);
                     return;
                 } else
                     is_fetchable = true;
@@ -288,7 +303,7 @@
                                 return;
                             else {
                                 if (is_httpequiv) {
-                                    self->prefetch(href);
+                                    self->prefetch(href, true);
                                     return;
                                 } else
                                     is_fetchable = true;
@@ -323,7 +338,7 @@
             } else if (strcasecmp(*attr, "http-equiv") == 0 &&
                        strcasecmp(*(attr + 1), "link") == 0) {
                 if (!href.empty()) {
-                    self->prefetch(href);
+                    self->prefetch(href, true);
                     return;
                 } else
                     is_httpequiv = true;
Index: squid3/src/HTMLAnalysisStream.h
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/Attic/HTMLAnalysisStream.h,v
retrieving revision 1.1.2.2
retrieving revision 1.1.2.3
diff -u -r1.1.2.2 -r1.1.2.3
--- squid3/src/HTMLAnalysisStream.h	25 Feb 2005 05:43:31 -0000	1.1.2.2
+++ squid3/src/HTMLAnalysisStream.h	5 May 2005 18:24:14 -0000	1.1.2.3
@@ -44,7 +44,7 @@
     ~HTMLAnalysisStream();
 
 private:
-    void prefetch(const std::string &);
+    void prefetch(const std::string &, bool);
 
     void parse(const char *document, size_t len, bool partial);
 
Index: squid3/src/HttpRequest.h
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/HttpRequest.h,v
retrieving revision 1.10
retrieving revision 1.10.4.1
diff -u -r1.10 -r1.10.4.1
--- squid3/src/HttpRequest.h	31 Aug 2004 02:14:26 -0000	1.10
+++ squid3/src/HttpRequest.h	5 May 2005 18:24:15 -0000	1.10.4.1
@@ -1,6 +1,6 @@
 
 /*
- * $Id: HttpRequest.h,v 1.10 2004/08/31 02:14:26 squidadm Exp $
+ * $Id: HttpRequest.h,v 1.10.4.1 2005/05/05 18:24:15 nlewycky Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -80,6 +80,10 @@
     time_t ims;
     int imslen;
     int max_forwards;
+#if USE_HTMLPREFETCH
+    int recursion_depth;
+#endif
+
     /* these in_addr's could probably be sockaddr_in's */
 
     struct in_addr client_addr;
Index: squid3/src/PrefetchStream.cc
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/Attic/PrefetchStream.cc,v
retrieving revision 1.1.2.6
retrieving revision 1.1.2.7
diff -u -r1.1.2.6 -r1.1.2.7
--- squid3/src/PrefetchStream.cc	26 Mar 2005 23:43:53 -0000	1.1.2.6
+++ squid3/src/PrefetchStream.cc	5 May 2005 18:24:15 -0000	1.1.2.7
@@ -1,6 +1,6 @@
 
 /*
- * $Id: PrefetchStream.cc,v 1.1.2.6 2005/03/26 23:43:53 nlewycky Exp $
+ * $Id: PrefetchStream.cc,v 1.1.2.7 2005/05/05 18:24:15 nlewycky Exp $
  *
  * DEBUG: section 93    HTML parsing and fetching
  * AUTHOR: Nick Lewycky
@@ -35,6 +35,7 @@
  *
  */
 
+#include <limits>
 #include <set>
 
 #include "squid.h"
@@ -50,8 +51,10 @@
 
 void
 PrefetchStream::prefetch(const string &href, const string &relative_url,
-                         const HttpRequest *request)
+                         const HttpRequest *request, bool recurse)
 {
+    static int no_recursion = numeric_limits<int>::max();
+
     char *url = urlResolveRelative(href.c_str(), relative_url.c_str());
 
     if (!url) {
@@ -123,7 +126,8 @@
     if (clientBeginRequest(METHOD_GET, url, PrefetchStream::BufferData,
                            PrefetchStream::Detach, stream.getRaw(), &tempheaders,
                            stream->discard_buffer, /*256*/HTTP_REQBUF_SZ,
-                           (ClientHttpRequest::flags_type){0,0,0,0,0}, no_addr)) {
+                           (ClientHttpRequest::flags_type){0,0,0,0,0}, no_addr,
+                           recurse?request->recursion_depth+1:no_recursion)) {
         debugs(93, 4, "(PS) Failed to prefetch " << url);
         return;
     }
Index: squid3/src/PrefetchStream.h
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/Attic/PrefetchStream.h,v
retrieving revision 1.1.2.1
retrieving revision 1.1.2.2
diff -u -r1.1.2.1 -r1.1.2.2
--- squid3/src/PrefetchStream.h	13 Jan 2005 03:53:05 -0000	1.1.2.1
+++ squid3/src/PrefetchStream.h	5 May 2005 18:24:15 -0000	1.1.2.2
@@ -25,7 +25,7 @@
     std::set<std::string>::iterator pending_element;
 
     static void prefetch(const std::string &, const std::string &,
-                         const HttpRequest *);
+                         const HttpRequest *, bool);
 
     // used for coalescing identical requests
     static std::set<std::string> pending;
Index: squid3/src/client_side_reply.cc
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/client_side_reply.cc,v
retrieving revision 1.57.2.1
retrieving revision 1.57.2.2
diff -u -r1.57.2.1 -r1.57.2.2
--- squid3/src/client_side_reply.cc	26 Jan 2005 19:51:34 -0000	1.57.2.1
+++ squid3/src/client_side_reply.cc	5 May 2005 18:24:15 -0000	1.57.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side_reply.cc,v 1.57.2.1 2005/01/26 19:51:34 nlewycky Exp $
+ * $Id: client_side_reply.cc,v 1.57.2.2 2005/05/05 18:24:15 nlewycky Exp $
  *
  * DEBUG: section 88    Client-side Reply Routines
  * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
@@ -1933,7 +1933,7 @@
              content_type.caseCmp("application/xhtml+xml") == 0) &&
             !httpHeaderHas(&http->storeEntry()->getReply()->header,
                            HDR_CONTENT_ENCODING) &&
-            !http->request->range) {
+            !http->request->range && http->request->recursion_depth < 3) {
             HTMLAnalysisStream *htmlas = new HTMLAnalysisStream(http->request);
             clientStreamInsertHead(&http->client_stream, htmlStreamRead,
                                    htmlBufferData, htmlStreamDetach,
Index: squid3/src/client_side_request.cc
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/client_side_request.cc,v
retrieving revision 1.33.2.2
retrieving revision 1.33.2.3
diff -u -r1.33.2.2 -r1.33.2.3
--- squid3/src/client_side_request.cc	25 Feb 2005 05:43:31 -0000	1.33.2.2
+++ squid3/src/client_side_request.cc	5 May 2005 18:24:25 -0000	1.33.2.3
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side_request.cc,v 1.33.2.2 2005/02/25 05:43:31 nlewycky Exp $
+ * $Id: client_side_request.cc,v 1.33.2.3 2005/05/05 18:24:25 nlewycky Exp $
  * 
  * DEBUG: section 85    Client-side Request Routines
  * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
@@ -264,7 +264,7 @@
 int				/* returns nonzero on failure */
 clientBeginRequest(method_t method, char const *url, CSCB * streamcallback,
                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
-                   char *tailbuf, size_t taillen, ClientHttpRequest::flags_type flags, in_addr addr)
+                   char *tailbuf, size_t taillen, ClientHttpRequest::flags_type flags, in_addr addr, int prefetch_recursion_depth)
 {
     size_t url_sz;
     HttpVersion http_ver (1, 0);
@@ -334,6 +334,10 @@
 
     request->http_ver = http_ver;
 
+#if USE_HTMLPREFETCH
+    request->recursion_depth = prefetch_recursion_depth;
+#endif
+
     http->request = requestLink(request);
 
     /* optional - skip the access check ? */
Index: squid3/src/client_side_request.h
===================================================================
RCS file: /cvsroot/squid-sf//squid3/src/client_side_request.h,v
retrieving revision 1.17.6.2
retrieving revision 1.17.6.3
diff -u -r1.17.6.2 -r1.17.6.3
--- squid3/src/client_side_request.h	25 Feb 2005 05:43:31 -0000	1.17.6.2
+++ squid3/src/client_side_request.h	5 May 2005 18:24:25 -0000	1.17.6.3
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side_request.h,v 1.17.6.2 2005/02/25 05:43:31 nlewycky Exp $
+ * $Id: client_side_request.h,v 1.17.6.3 2005/05/05 18:24:25 nlewycky Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -142,7 +142,7 @@
     ConnStateData::Pointer conn_;
 };
 
-extern int clientBeginRequest(method_t, char const *, CSCB *, CSD *, ClientStreamData, HttpHeader const *, char *, size_t, ClientHttpRequest::flags_type, in_addr = no_addr);
+extern int clientBeginRequest(method_t, char const *, CSCB *, CSD *, ClientStreamData, HttpHeader const *, char *, size_t, ClientHttpRequest::flags_type, in_addr = no_addr, int prefetch_recursion_depth = 0);
 
 /* client http based routines */
 SQUIDCEXTERN char *clientConstructTraceEcho(clientHttpRequest *);