[4.1] 4e8f14c Having researched it rather comprehensively, I have decided that there is no sane way to refcount (dynamic) backends in relation to VCL use at this point in the 4.1 release cycle.

[Poul-Henning Kamp]

commit 4e8f14c495a34728f75856b070a09cfa3c87e64f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jul 12 09:07:02 2015 +0000

    Having researched it rather comprehensively, I have decided that there
    is no sane way to refcount (dynamic) backends in relation to VCL
    use at this point in the 4.1 release cycle.
    
    Solve the problem, for now, by cooling dynamic backends off for a
    minute before actually removing them.
    
    Varnish 5 will have to look at proper refcounting of (all) VCL objects.

diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 49fc110..17b90f0 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -50,6 +50,8 @@ struct backend {
 	unsigned		magic;
 #define BACKEND_MAGIC		0x64c4c7c6
 
+	unsigned		n_conn;
+
 	VTAILQ_ENTRY(backend)	list;
 	VTAILQ_ENTRY(backend)	vcl_list;
 	struct lock		mtx;
@@ -59,7 +61,6 @@ struct backend {
 	struct vcl		*vcl;
 	char			*display_name;
 
-	unsigned		n_conn;
 
 	struct vbp_target	*probe;
 	unsigned		healthy;
@@ -71,6 +72,8 @@ struct backend {
 	struct tcp_pool		*tcp_pool;
 
 	struct director		director[1];
+
+	double			cooled;
 };
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 8b89625..3db5dbd 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -48,6 +48,8 @@
 #include "cache_backend.h"
 
 static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
+static VTAILQ_HEAD(, backend) cool_backends =
+    VTAILQ_HEAD_INITIALIZER(cool_backends);
 static struct lock backends_mtx;
 
 static const char * const vbe_ah_healthy	= "healthy";
@@ -130,7 +132,6 @@ VRT_delete_backend(VRT_CTX, struct director **dp)
 {
 	struct director *d;
 	struct backend *be;
-	int r;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(dp);
@@ -141,17 +142,10 @@ VRT_delete_backend(VRT_CTX, struct director **dp)
 	Lck_Lock(&be->mtx);
 	be->admin_health = vbe_ah_deleted;
 	be->health_changed = VTIM_real();
-	r = be->n_conn;
-	if (r > 0) {
-		/* move to front of list for fast access */
-		VTAILQ_REMOVE(&backends, be, list);
-		VTAILQ_INSERT_HEAD(&backends, be, list);
-	}
+	be->cooled = VTIM_real() + 60.;
+	VTAILQ_REMOVE(&backends, be, list);
+	VTAILQ_INSERT_TAIL(&cool_backends, be, list);
 	Lck_Unlock(&be->mtx);
-	if (r == 0) {
-		VCL_DelBackend(be);
-		VBE_Delete(be);
-	}
 }
 
 /*---------------------------------------------------------------------
@@ -191,7 +185,10 @@ VBE_Delete(struct backend *be)
 		VBP_Remove(be);
 
 	Lck_Lock(&backends_mtx);
-	VTAILQ_REMOVE(&backends, be, list);
+	if (be->cooled > 0)
+		VTAILQ_REMOVE(&cool_backends, be, list);
+	else
+		VTAILQ_REMOVE(&backends, be, list);
 	VSC_C_main->n_backend--;
 	VBT_Rel(&be->tcp_pool);
 	Lck_Unlock(&backends_mtx);
@@ -425,18 +422,21 @@ void
 VBE_Poll(void)
 {
 	struct backend *be;
+	double now = VTIM_real();
+
+	Lck_Lock(&backends_mtx);
 	while (1) {
-		Lck_Lock(&backends_mtx);
-		be = VTAILQ_FIRST(&backends);
+		be = VTAILQ_FIRST(&cool_backends);
 		if (be == NULL)
 			break;
-		if (be->admin_health != vbe_ah_deleted)
+		if (be->cooled > now)
 			break;
 		if (be->n_conn > 0)
-			break;
+			continue;
 		Lck_Unlock(&backends_mtx);
 		VCL_DelBackend(be);
 		VBE_Delete(be);
+		Lck_Lock(&backends_mtx);
 	}
 	Lck_Unlock(&backends_mtx);
 }