[master] ae3e5a3 Push allocation of request into the assigned worker thread when we come back from the waiter, to lessen amount of work waiter does.

[Poul-Henning Kamp]

commit ae3e5a3a9fcf0214c0d0cfc75f7ae909cf439f8a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jul 17 20:10:09 2012 +0000

    Push allocation of request into the assigned worker thread
    when we come back from the waiter, to lessen amount of work
    waiter does.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index bbec54f..da2cdf8 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -172,6 +172,8 @@ vca_pace_good(void)
 
 /*--------------------------------------------------------------------
  * Accept on a listen socket, and handle error returns.
+ *
+ * Called from a worker thread from a pool
  */
 
 int
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 59dccf6..d2cca4b 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -28,9 +28,14 @@
  *
  * Session management
  *
- * This is a little bit of a mixed back, containing both memory management
+ * This is a little bit of a mixed bag, containing both memory management
  * and various state-change functions.
  *
+ * The overall picture is complicated by the fact that requests can
+ * disembark their worker-threads if they hit a busy object, then come
+ * back later in a different worker thread to continue.
+ * XXX: I wonder if that complexity pays of any more ?
+ *
  */
 
 #include "config.h"
@@ -121,11 +126,11 @@ ses_new(struct sesspool *pp)
 }
 
 /*--------------------------------------------------------------------
- * The pool-task function for sessions
+ * Process new/existing request on this session.
  */
 
 static void
-ses_pool_task(struct worker *wrk, void *arg)
+ses_req_pool_task(struct worker *wrk, void *arg)
 {
 	struct req *req;
 	struct sess *sp;
@@ -149,6 +154,28 @@ ses_pool_task(struct worker *wrk, void *arg)
 }
 
 /*--------------------------------------------------------------------
+ * Allocate a request + vxid, call ses_req_pool_task()
+ */
+
+static void
+ses_sess_pool_task(struct worker *wrk, void *arg)
+{
+	struct req *req;
+	struct sess *sp;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC);
+
+	req = ses_GetReq(sp);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	req->vxid = VXID_Get(&wrk->vxid_pool);
+
+	sp->sess_step = S_STP_NEWREQ;
+	ses_req_pool_task(wrk, req);
+}
+
+/*--------------------------------------------------------------------
  * VSL log the endpoints of the TCP connection.
  *
  * We use VSL() to get the sessions vxid and to make sure tha this
@@ -189,7 +216,6 @@ void
 SES_pool_accept_task(struct worker *wrk, void *arg)
 {
 	struct sesspool *pp;
-	struct req *req;
 	struct sess *sp;
 	const char *lsockname;
 
@@ -213,17 +239,13 @@ SES_pool_accept_task(struct worker *wrk, void *arg)
 	lsockname = VCA_SetupSess(wrk, sp);
 	ses_vsl_socket(sp, lsockname);
 
-	req = ses_GetReq(sp);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	req->vxid = VXID_Get(&wrk->vxid_pool);
-
-	sp->sess_step = S_STP_NEWREQ;
-	ses_pool_task(wrk, req);
+	ses_sess_pool_task(wrk, sp);
 }
 
 /*--------------------------------------------------------------------
  * Schedule a request back on a work-thread from its sessions pool
+ *
+ * This is used to reschedule requests waiting on busy objects
  */
 
 int
@@ -239,7 +261,7 @@ SES_ScheduleReq(struct req *req)
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	AN(pp->pool);
 
-	sp->task.func = ses_pool_task;
+	sp->task.func = ses_req_pool_task;
 	sp->task.priv = req;
 
 	if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) {
@@ -260,22 +282,14 @@ SES_ScheduleReq(struct req *req)
 void
 SES_Handle(struct sess *sp, double now)
 {
-	struct req *req;
 	struct sesspool *pp;
 
-	/* NB: This only works with single-threaded waiters */
-	static struct vxid_pool vxid_pool;
-
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	pp = sp->sesspool;
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	AN(pp->pool);
-	req = ses_GetReq(sp);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	req->vxid = VXID_Get(&vxid_pool);
-	sp->task.func = ses_pool_task;
-	sp->task.priv = req;
-	sp->sess_step = S_STP_NEWREQ;
+	sp->task.func = ses_sess_pool_task;
+	sp->task.priv = sp;
 	sp->t_rx = now;
 	if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) {
 		VSC_C_main->client_drop_late++;