[master] 5ed9aece4 Teardown straggler backends by sending them DISCARD event

[Poul-Henning Kamp]

commit 5ed9aece431e5f0d3a6c4cf337fc90ca2a52c1be
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 24 11:57:35 2021 +0000

    Teardown straggler backends by sending them DISCARD event
    
    (3rd part of #3599)

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 063048931..dada7f2b7 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -408,6 +408,8 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev)
 		if (bp->probe != NULL)
 			VBP_Control(bp, 0);
 		VRT_VSC_Hide(bp->vsc_seg);
+	} else if (ev == VCL_EVENT_DISCARD) {
+		VRT_Assign_Backend(&bp->director, NULL);
 	}
 }
 
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index c69e1e5c7..48ee357ba 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -447,24 +447,19 @@ vcl_BackendEvent(const struct vcl *vcl, enum vcl_event_e e)
 }
 
 static void
-vcl_KillBackends(struct vcl *vcl)
+vcl_KillBackends(const struct vcl *vcl)
 {
-	struct vcldir *vdir;
+	struct vcldir *vdir, *vdir2;
 
 	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
-	AZ(vcl->busy);
-	assert(VTAILQ_EMPTY(&vcl->ref_list));
-	Lck_Lock(&vcl_mtx);
-	while (1) {
-		vdir = VTAILQ_FIRST(&vcl->director_list);
-		if (vdir == NULL)
-			break;
-		VTAILQ_REMOVE(&vcl->director_list, vdir, list);
-		AN(vdir->methods->destroy);
-		vdir->methods->destroy(vdir->dir);
-		vcldir_free(vdir);
-	}
-	Lck_Unlock(&vcl_mtx);
+	assert(vcl->temp == VCL_TEMP_COLD || vcl->temp == VCL_TEMP_INIT);
+	/*
+	 * Unlocked because no further directors can be added, and the
+	 * remaining ones need to be able to remove themselves.
+	 */
+	VTAILQ_FOREACH_SAFE(vdir, &vcl->director_list, list, vdir2)
+		VDI_Event(vdir->dir, VCL_EVENT_DISCARD);
+	assert(VTAILQ_EMPTY(&vcl->director_list));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c
index 4144c13f2..6e4592c06 100644
--- a/bin/varnishd/cache/cache_vrt_vcl.c
+++ b/bin/varnishd/cache/cache_vrt_vcl.c
@@ -146,6 +146,7 @@ vcldir_free(struct vcldir *vdir)
 
 	CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC);
 	CHECK_OBJ_NOTNULL(vdir->dir, DIRECTOR_MAGIC);
+	AZ(vdir->refcnt);
 	Lck_Delete(&vdir->dlck);
 	free(vdir->cli_name);
 	FREE_OBJ(vdir->dir);
@@ -225,10 +226,9 @@ VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv,
 	return (vdir->dir);
 }
 
-void
-VRT_DelDirector(VCL_BACKEND *bp)
+static void
+retire_backend(VCL_BACKEND *bp)
 {
-	struct vcl *vcl;
 	struct vcldir *vdir;
 	const struct vcltemp *temp;
 	VCL_BACKEND d;
@@ -236,17 +236,13 @@ VRT_DelDirector(VCL_BACKEND *bp)
 	TAKE_OBJ_NOTNULL(d, bp, DIRECTOR_MAGIC);
 	vdir = d->vdir;
 	CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC);
-	vcl = vdir->vcl;
-	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
-
-	Lck_Lock(d->mtx);
-	assert(vdir->refcnt == 1);
-	--vdir->refcnt;
-	Lck_Unlock(d->mtx);
+	assert(vdir->refcnt == 0);
+	assert (d == vdir->dir);
+	CHECK_OBJ_NOTNULL(vdir->vcl, VCL_MAGIC);
 
 	Lck_Lock(&vcl_mtx);
-	temp = vcl->temp;
-	VTAILQ_REMOVE(&vcl->director_list, vdir, list);
+	temp = vdir->vcl->temp;
+	VTAILQ_REMOVE(&vdir->vcl->director_list, vdir, list);
 	Lck_Unlock(&vcl_mtx);
 
 	if (temp->is_warm)
@@ -257,9 +253,25 @@ VRT_DelDirector(VCL_BACKEND *bp)
 	vcldir_free(vdir);
 }
 
+void
+VRT_DelDirector(VCL_BACKEND *bp)
+{
+	struct vcldir *vdir;
+
+	AN(bp);
+	vdir = (*bp)->vdir;
+	CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC);
+	Lck_Lock(&vdir->dlck);
+	assert(vdir->refcnt == 1);
+	vdir->refcnt = 0;
+	Lck_Unlock(&vdir->dlck);
+	retire_backend(bp);
+}
+
 void
 VRT_Assign_Backend(VCL_BACKEND *dst, VCL_BACKEND src)
 {
+	int busy;
 
 	AN(dst);
 	CHECK_OBJ_ORNULL((*dst), DIRECTOR_MAGIC);
@@ -268,8 +280,10 @@ VRT_Assign_Backend(VCL_BACKEND *dst, VCL_BACKEND src)
 		CHECK_OBJ_NOTNULL((*dst)->vdir, VCLDIR_MAGIC);
 		Lck_Lock((*dst)->mtx);
 		assert((*dst)->vdir->refcnt > 0);
-		--(*dst)->vdir->refcnt;
+		busy = --(*dst)->vdir->refcnt;
 		Lck_Unlock((*dst)->mtx);
+		if (!busy)
+			retire_backend(dst);
 	}
 	if (src != NULL) {
 		CHECK_OBJ_NOTNULL(src->vdir, VCLDIR_MAGIC);