---------------------
PatchSet 8697 
Date: 2006/10/01 00:59:26
Author: hno
Branch: commloops-2_6
Tag: (none) 
Log:
Randomize comm callbacks on delayed filedescriptors

Members: 
	src/comm.c:1.41.2.1->1.41.2.2 
	src/comm_epoll.c:1.17.2.1->1.17.2.2 
	src/comm_generic.c:1.2.2.1->1.2.2.2 
	src/comm_kqueue.c:1.6.2.1->1.6.2.2 
	src/comm_poll.c:1.14.2.2->1.14.2.3 
	src/comm_select.c:1.29.2.2->1.29.2.3 
	src/fd.c:1.15.2.1->1.15.2.2 
	src/protos.h:1.107.2.1->1.107.2.2 
	src/structs.h:1.112.2.1->1.112.2.2 

Index: squid/src/comm.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/comm.c,v
retrieving revision 1.41.2.1
retrieving revision 1.41.2.2
diff -u -r1.41.2.1 -r1.41.2.2
--- squid/src/comm.c	30 Sep 2006 23:45:27 -0000	1.41.2.1
+++ squid/src/comm.c	1 Oct 2006 00:59:26 -0000	1.41.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: comm.c,v 1.41.2.1 2006/09/30 23:45:27 hno Exp $
+ * $Id: comm.c,v 1.41.2.2 2006/10/01 00:59:26 hno Exp $
  *
  * DEBUG: section 5     Socket Functions
  * AUTHOR: Harvest Derived
@@ -1036,8 +1036,7 @@
 void
 comm_init(void)
 {
-    fd_table = xcalloc(Squid_MaxFD, sizeof(fde));
-    /* XXX account fd_table */
+    fd_init();
     /* Keep a few file descriptors free so that we don't run out of FD's
      * after accepting a client but before it opens a socket or a file.
      * Since Squid_MaxFD can be as high as several thousand, don't waste them */
Index: squid/src/comm_epoll.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/comm_epoll.c,v
retrieving revision 1.17.2.1
retrieving revision 1.17.2.2
diff -u -r1.17.2.1 -r1.17.2.2
--- squid/src/comm_epoll.c	30 Sep 2006 23:45:27 -0000	1.17.2.1
+++ squid/src/comm_epoll.c	1 Oct 2006 00:59:26 -0000	1.17.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: comm_epoll.c,v 1.17.2.1 2006/09/30 23:45:27 hno Exp $
+ * $Id: comm_epoll.c,v 1.17.2.2 2006/10/01 00:59:26 hno Exp $
  *
  * DEBUG: section 5     Socket Functions
  *
@@ -68,6 +68,7 @@
 void
 comm_select_init()
 {
+    comm_generic_init();
     kdpfd = epoll_create(Squid_MaxFD);
     if (kdpfd < 0)
 	fatalf("comm_select_init: epoll_create(): %s\n", xstrerror());
@@ -90,6 +91,7 @@
     close(kdpfd);
     kdpfd = -1;
     safe_free(epoll_state);
+    comm_generic_shutdown();
 }
 
 void
@@ -160,6 +162,7 @@
     int i;
     int num;
     int fd;
+    int rc;
     struct epoll_event *cevents;
     double start = current_dtime;
 
@@ -180,6 +183,13 @@
 	last_timeout = squid_curtime;
 	checkTimeouts();
     }
+
+#if DELAY_POOLS
+    /* We have delayed fds in queue? */
+    if (n_slow_fds)
+	msec = 0;
+#endif
+
     statCounter.syscalls.polls++;
     num = epoll_wait(kdpfd, events, MAX_EVENTS, msec);
     statCounter.select_loops++;
@@ -199,12 +209,16 @@
 	    fd = cevents->data.fd;
 	    comm_call_handlers(fd, cevents->events & ~EPOLLOUT, cevents->events & ~EPOLLIN);
 	}
-	getCurrentTime();
-	statCounter.select_time += (current_dtime - start);
-	return COMM_OK;
+	rc = COMM_OK;
     } else {
-	getCurrentTime();
-	debug(5, 8) ("comm_select: time out: %ld.\n", (long int) squid_curtime);
-	return COMM_TIMEOUT;
+	debug(5, 8) ("comm_select: time out\n");
+	rc = COMM_TIMEOUT;
     }
+#if DELAY_POOLS
+    if (num < MAX_EVENTS)
+	comm_call_slowfds();
+#endif
+    getCurrentTime();
+    statCounter.select_time += (current_dtime - start);
+    return COMM_OK;
 }
Index: squid/src/comm_generic.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/comm_generic.c,v
retrieving revision 1.2.2.1
retrieving revision 1.2.2.2
diff -u -r1.2.2.1 -r1.2.2.2
--- squid/src/comm_generic.c	30 Sep 2006 23:45:27 -0000	1.2.2.1
+++ squid/src/comm_generic.c	1 Oct 2006 00:59:26 -0000	1.2.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: comm_generic.c,v 1.2.2.1 2006/09/30 23:45:27 hno Exp $
+ * $Id: comm_generic.c,v 1.2.2.2 2006/10/01 00:59:26 hno Exp $
  *
  * DEBUG: section 5     Socket Functions
  *
@@ -39,6 +39,27 @@
 
 static int MAX_POLL_TIME = 1000;	/* see also comm_quick_poll_required() */
 
+#if DELAY_POOLS
+static int *slow_fds = NULL;
+static int n_slow_fds = 0;
+#endif
+
+static void
+comm_generic_init(void)
+{
+#if DELAY_POOLS
+    slow_fds = xmalloc(sizeof(int) * Squid_MaxFD);
+#endif
+}
+
+static void
+comm_generic_shutdown(void)
+{
+#if DELAY_POOLS
+    safe_free(slow_fds);
+#endif
+}
+
 /* Defer reads from this fd */
 void
 commDeferFD(int fd)
@@ -84,6 +105,66 @@
     return F->defer_check(fd, F->defer_data);
 }
 
+#if DELAY_POOLS
+static void
+commAddSlow(int fd)
+{
+    fde *F = &fd_table[fd];
+    if (F->slow_id)
+	return;
+    F->slow_id = ++n_slow_fds;
+    assert(n_slow_fds < Squid_MaxFD);
+    slow_fds[n_slow_fds] = fd;
+}
+
+void
+commRemoveSlow(int fd)
+{
+    int fd2;
+    fde *F = &fd_table[fd];
+    if (!F->slow_id)
+	return;
+    fd2 = slow_fds[n_slow_fds--];
+    if (F->slow_id <= n_slow_fds) {
+	slow_fds[F->slow_id] = fd2;
+	fde *F2 = &fd_table[fd2];
+	F2->slow_id = F->slow_id;
+    }
+    F->slow_id = 0;
+}
+#endif
+
+#if DELAY_POOLS
+static void
+comm_call_slowfds(void)
+{
+    while (n_slow_fds) {
+	int i = (squid_random() % n_slow_fds) + 1;
+	int fd = slow_fds[i];
+	fde *F = &fd_table[fd];
+	commRemoveSlow(fd);
+	if (F->read_handler) {
+	    PF *hdl = F->read_handler;
+	    void *hdl_data = F->read_data;
+	    debug(5, 8) ("comm_call_handlers(): Calling read handler on fd=%d\n", fd);
+#if SIMPLE_COMM_HANDLER
+	    commUpdateReadHandler(fd, NULL, NULL);
+	    hdl(fd, hdl_data);
+#else
+	    /* Optimized version to avoid the fd bouncing in/out of the waited set */
+	    F->read_handler = NULL;
+	    F->read_data = NULL;
+	    F->read_pending = COMM_PENDING_NORMAL;
+	    hdl(fd, hdl_data);
+	    if (F->flags.open && !F->read_handler)
+		commUpdateEvents(fd);
+#endif
+	    statCounter.select_fds++;
+	}
+    }
+}
+#endif
+
 static void
 comm_call_handlers(int fd, int read_event, int write_event)
 {
@@ -115,6 +196,10 @@
 		    commDeferFD(fd);
 		}
 		break;
+	    case -1:
+		commDeferFD(fd);
+		commAddSlow(fd);
+		break;
 	    default:
 		debug(5, 8) ("comm_call_handlers(): Calling read handler on fd=%d\n", fd);
 #if SIMPLE_COMM_HANDLER
@@ -130,6 +215,7 @@
 		    commUpdateEvents(fd);
 #endif
 		statCounter.select_fds++;
+		break;
 	    }
 	}
     }
@@ -177,6 +263,10 @@
 	F = &fd_table[fd];
 	if (!F->flags.open)
 	    continue;
+#if DELAYPOOLS
+	if (F->slow_id)
+	    continue;
+#endif
 	if (F->flags.backoff)
 	    commResumeFD(fd);
 	if (F->timeout == 0)
Index: squid/src/comm_kqueue.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/comm_kqueue.c,v
retrieving revision 1.6.2.1
retrieving revision 1.6.2.2
diff -u -r1.6.2.1 -r1.6.2.2
--- squid/src/comm_kqueue.c	30 Sep 2006 23:45:27 -0000	1.6.2.1
+++ squid/src/comm_kqueue.c	1 Oct 2006 00:59:26 -0000	1.6.2.2
@@ -53,6 +53,7 @@
 void
 comm_select_init()
 {
+    comm_generic_init();
     kq = kqueue();
     if (kq < 0)
 	fatalf("comm_select_init: kqueue(): %s\n", xstrerror());
@@ -78,6 +79,7 @@
     close(kq);
     kq = -1;
     safe_free(kqueue_state);
+    comm_generic_shutdown();
 }
 
 void
@@ -129,6 +131,7 @@
     static time_t last_timeout = 0;
     int i;
     int num;
+    int rc = COMM_OK;
     double start = current_dtime;
     static struct kevent ke[KE_LENGTH];
     struct timespec timeout;
@@ -136,11 +139,6 @@
     if (msec > MAX_POLL_TIME)
 	msec = MAX_POLL_TIME;
 
-    timeout.tv_sec = msec / 1000;
-    timeout.tv_nsec = (msec % 1000) * 1000000;
-
-    debug(5, 3) ("comm_select: timeout %d\n", msec);
-
     /* Check for disk io callbacks */
     storeDirCallback();
 
@@ -149,6 +147,18 @@
 	last_timeout = squid_curtime;
 	checkTimeouts();
     }
+
+#if DELAY_POOLS
+    /* We have delayed fds in queue? */
+    if (n_slow_fds)
+	msec = 0;
+#endif
+
+    timeout.tv_sec = msec / 1000;
+    timeout.tv_nsec = (msec % 1000) * 1000000;
+
+    debug(5, 3) ("comm_select: timeout %d\n", msec);
+
     statCounter.syscalls.polls++;
     num = kevent(kq, kqlst, kqoff, ke, KE_LENGTH, &timeout);
     statCounter.select_loops++;
@@ -162,10 +172,8 @@
 	debug(5, 1) ("comm_select: kevent failure: %s\n", xstrerror());
 	return COMM_ERROR;
     }
-    if (num == 0) {
-	getCurrentTime();
-	return COMM_TIMEOUT;
-    }
+    if (num == 0)
+	rc = COMM_TIMEOUT;
     statHistCount(&statCounter.select_fds_hist, num);
 
     for (i = 0; i < num; i++) {
@@ -190,7 +198,12 @@
 	}
     }
 
+#if DELAY_POOLS
+    if (num < MAX_EVENTS)
+	comm_call_slowfds();
+#endif
+
     getCurrentTime();
     statCounter.select_time += (current_dtime - start);
-    return COMM_OK;
+    return rc;
 }
Index: squid/src/comm_poll.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/comm_poll.c,v
retrieving revision 1.14.2.2
retrieving revision 1.14.2.3
diff -u -r1.14.2.2 -r1.14.2.3
--- squid/src/comm_poll.c	30 Sep 2006 23:45:28 -0000	1.14.2.2
+++ squid/src/comm_poll.c	1 Oct 2006 00:59:26 -0000	1.14.2.3
@@ -49,6 +49,7 @@
 comm_select_init()
 {
     int i;
+    comm_generic_init();
     pfds = xcalloc(sizeof(*pfds), Squid_MaxFD);
     pfd_map = xcalloc(sizeof(*pfd_map), Squid_MaxFD);
     for (i = 0; i < Squid_MaxFD; i++) {
@@ -66,6 +67,7 @@
 comm_select_shutdown()
 {
     safe_free(pfds);
+    comm_generic_shutdown();
 }
 
 void
@@ -165,6 +167,8 @@
 	num--;
     }
 
+    comm_call_slowfds();
+
     getCurrentTime();
     statCounter.select_time += (current_dtime - start);
     return COMM_OK;
Index: squid/src/comm_select.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/comm_select.c,v
retrieving revision 1.29.2.2
retrieving revision 1.29.2.3
diff -u -r1.29.2.2 -r1.29.2.3
--- squid/src/comm_select.c	30 Sep 2006 23:45:29 -0000	1.29.2.2
+++ squid/src/comm_select.c	1 Oct 2006 00:59:26 -0000	1.29.2.3
@@ -1,6 +1,6 @@
 
 /*
- * $Id: comm_select.c,v 1.29.2.2 2006/09/30 23:45:29 hno Exp $
+ * $Id: comm_select.c,v 1.29.2.3 2006/10/01 00:59:26 hno Exp $
  *
  * DEBUG: section 5     Socket Functions
  *
@@ -58,6 +58,7 @@
 void
 comm_select_init()
 {
+    comm_generic_init();
     global_readfds = xcalloc(FD_MASK_BYTES, howmany(Squid_MaxFD, FD_MASK_BITS));
     global_writefds = xcalloc(FD_MASK_BYTES, howmany(Squid_MaxFD, FD_MASK_BITS));
     current_readfds = xcalloc(FD_MASK_BYTES, howmany(Squid_MaxFD, FD_MASK_BITS));
@@ -78,6 +79,7 @@
     safe_free(global_writefds);
     safe_free(current_readfds);
     safe_free(current_writefds);
+    comm_generic_shutdown();
 }
 
 void
@@ -95,8 +97,7 @@
     } else if (!need_read && FD_ISSET(fd, global_readfds)) {
 	FD_CLR(fd, global_readfds);
 	nreadfds--;
-    }
-    if (need_write && !FD_ISSET(fd, global_writefds)) {
+    } if (need_write && !FD_ISSET(fd, global_writefds)) {
 	FD_SET(fd, global_writefds);
 	nwritefds++;
     } else if (!need_write && FD_ISSET(fd, global_writefds)) {
@@ -172,6 +173,8 @@
 	}
     }
 
+    comm_call_slowfds();
+
     getCurrentTime();
     statCounter.select_time += (current_dtime - start);
     return COMM_OK;
Index: squid/src/fd.c
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/fd.c,v
retrieving revision 1.15.2.1
retrieving revision 1.15.2.2
diff -u -r1.15.2.1 -r1.15.2.2
--- squid/src/fd.c	30 Sep 2006 23:45:29 -0000	1.15.2.1
+++ squid/src/fd.c	1 Oct 2006 00:59:27 -0000	1.15.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: fd.c,v 1.15.2.1 2006/09/30 23:45:29 hno Exp $
+ * $Id: fd.c,v 1.15.2.2 2006/10/01 00:59:27 hno Exp $
  *
  * DEBUG: section 51    Filedescriptor Functions
  * AUTHOR: Duane Wessels
@@ -92,6 +92,10 @@
     debug(51, 3) ("fd_close FD %d %s\n", fd, F->desc);
     commSetEvents(fd, 0, 0);
     F->flags.open = 0;
+#if DELAY_POOLS
+    if (F->slow_id)
+	commRemoveSlow(fd);
+#endif
     fdUpdateBiggest(fd, 0);
     Number_FD--;
     memset(F, '\0', sizeof(fde));
@@ -197,6 +201,13 @@
 }
 
 void
+fd_init(void)
+{
+    fd_table = xcalloc(Squid_MaxFD, sizeof(fde));
+    /* XXX account fd_table */
+}
+
+void
 fdFreeMemory(void)
 {
     safe_free(fd_table);
Index: squid/src/protos.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/protos.h,v
retrieving revision 1.107.2.1
retrieving revision 1.107.2.2
diff -u -r1.107.2.1 -r1.107.2.2
--- squid/src/protos.h	30 Sep 2006 23:45:31 -0000	1.107.2.1
+++ squid/src/protos.h	1 Oct 2006 00:59:27 -0000	1.107.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: protos.h,v 1.107.2.1 2006/09/30 23:45:31 hno Exp $
+ * $Id: protos.h,v 1.107.2.2 2006/10/01 00:59:27 hno Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -166,6 +166,7 @@
 extern void commDeferFD(int fd);
 extern void commResumeFD(int fd);
 extern void commSetSelect(int, unsigned int, PF *, void *, time_t);
+extern void commRemoveSlow(int fd);
 extern void comm_add_close_handler(int fd, PF *, void *);
 extern void comm_remove_close_handler(int fd, PF *, void *);
 extern int comm_udp_sendto(int, const struct sockaddr_in *, int, const void *, int);
@@ -258,6 +259,7 @@
 extern void eventFreeMemory(void);
 extern int eventFind(EVH *, void *);
 
+extern void fd_init(void);
 extern void fd_close(int fd);
 extern void fd_open(int fd, unsigned int type, const char *);
 extern void fd_note(int fd, const char *);
Index: squid/src/structs.h
===================================================================
RCS file: /cvsroot/squid-sf//squid/src/structs.h,v
retrieving revision 1.112.2.1
retrieving revision 1.112.2.2
diff -u -r1.112.2.1 -r1.112.2.2
--- squid/src/structs.h	30 Sep 2006 23:45:31 -0000	1.112.2.1
+++ squid/src/structs.h	1 Oct 2006 00:59:27 -0000	1.112.2.2
@@ -1,6 +1,6 @@
 
 /*
- * $Id: structs.h,v 1.112.2.1 2006/09/30 23:45:31 hno Exp $
+ * $Id: structs.h,v 1.112.2.2 2006/10/01 00:59:27 hno Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -921,6 +921,9 @@
 	long handle;
     } win32;
 #endif
+#if DELAY_POOLS
+    int slow_id;
+#endif
 };
 
 struct _fileMap {