From fgsch at lodoss.net  Tue Oct  1 13:13:07 2019
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue,  1 Oct 2019 13:13:07 +0000 (UTC)
Subject: [master] 63c23cc15 Ignore empty blocks
Message-ID: <20191001131307.5BCCD98924@lists.varnish-cache.org>


commit 63c23cc15e8af05db8d67b83f943256f37268c7c
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Oct 1 14:12:10 2019 +0100

    Ignore empty blocks

diff --git a/.lgtm.yml b/.lgtm.yml
index 269898eef..c851b5631 100644
--- a/.lgtm.yml
+++ b/.lgtm.yml
@@ -1,4 +1,5 @@
 queries:
+  - exclude: cpp/empty-block
   - exclude: cpp/missing-header-guard
   - exclude: cpp/short-global-name
 extraction:

From nils.goroll at uplex.de  Wed Oct  2 09:28:02 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed,  2 Oct 2019 09:28:02 +0000 (UTC)
Subject: [6.0] 42b138ad3 fix regression: ban-lurker skips objects
Message-ID: <20191002092802.4B0F660AA1@lists.varnish-cache.org>


commit 42b138ad3466e17ab64c5ec8bbec7c1e32206941
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed May 29 18:54:06 2019 +0200

    fix regression: ban-lurker skips objects
    
    93d805014df919f9be761c6d8ad4ed86eb90c2cb made execution of
    ban_lurker_test_ban() conditional on bd != b, which effectively caused
    objects hanging off bans below request bans to not get tested against
    relevant bans.
    
    Because object bans (from the obans list) are being marked completed,
    the objects which were skipped would also be missed to get evaluated
    against the relevant bans at lookup time unless they were evaluated in
    request context. So, in effect, we would simply miss to test bans.
    
    Fixes #3007
    
    Maybe related to #3006

diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index 1729e04a9..abfbbacdd 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -289,7 +289,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 			VSC_C_main->bans_lurker_obj_killed += lok;
 			VSC_C_main->bans_lurker_obj_killed_cutoff += lokc;
 			tested = tested_tests = lok = lokc = 0;
-			if (oc->ban == bt) {
+			if (oc->ban == bt && bt != bd) {
 				bt->refcount--;
 				VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
 				oc->ban = bd;
@@ -340,7 +340,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	bd = NULL;
 	VTAILQ_INIT(&obans);
 	for (; b != NULL; b = VTAILQ_NEXT(b, list)) {
-		if (bd != NULL && bd != b)
+		if (bd != NULL)
 			ban_lurker_test_ban(wrk, vsl, b, &obans, bd,
 			    count > cutoff);
 		if (b->flags & BANS_FLAG_COMPLETED)
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index a442833fa..3e6ac286b 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -33,6 +33,15 @@ server s1 {
 	expect req.url == /4
 	txresp -hdr "Foo: bar4.1"
 
+	rxreq
+	expect req.url == /r1
+	txresp
+	rxreq
+	expect req.url == /r2
+	txresp
+	rxreq
+	expect req.url == /r3
+	txresp
 } -start
 
 varnish v1 -vcl+backend {} -start
@@ -132,8 +141,8 @@ varnish v1 -expect bans_deleted == 2
 varnish v1 -expect bans_tested == 0
 varnish v1 -expect bans_tests_tested == 0
 varnish v1 -expect bans_obj_killed == 0
-varnish v1 -expect bans_lurker_tested == 8
-varnish v1 -expect bans_lurker_tests_tested == 9
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
 varnish v1 -expect bans_lurker_obj_killed == 4
 varnish v1 -expect bans_dups == 0
 
@@ -157,8 +166,8 @@ varnish v1 -expect bans_deleted == 2
 varnish v1 -expect bans_tested == 1
 varnish v1 -expect bans_tests_tested == 1
 varnish v1 -expect bans_obj_killed == 0
-varnish v1 -expect bans_lurker_tested == 8
-varnish v1 -expect bans_lurker_tests_tested == 9
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
 varnish v1 -expect bans_lurker_obj_killed == 4
 varnish v1 -expect bans_dups == 0
 
@@ -182,8 +191,8 @@ varnish v1 -expect bans_deleted == 5
 varnish v1 -expect bans_tested == 2
 varnish v1 -expect bans_tests_tested == 2
 varnish v1 -expect bans_obj_killed == 1
-varnish v1 -expect bans_lurker_tested == 8
-varnish v1 -expect bans_lurker_tests_tested == 9
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
 varnish v1 -expect bans_lurker_obj_killed == 4
 varnish v1 -expect bans_dups == 0
 
@@ -217,10 +226,41 @@ varnish v1 -expect bans_deleted == 10
 varnish v1 -expect bans_tested == 2
 varnish v1 -expect bans_tests_tested == 2
 varnish v1 -expect bans_obj_killed == 1
-varnish v1 -expect bans_lurker_tested == 8
-varnish v1 -expect bans_lurker_tests_tested == 9
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
 varnish v1 -expect bans_lurker_obj_killed == 4
 varnish v1 -expect bans_lurker_obj_killed_cutoff == 3
 varnish v1 -expect bans_dups == 0
 
 varnish v1 -expect n_object == 0
+
+client c1 {
+	txreq -url /r1
+	rxresp
+} -run
+
+varnish v1 -cliok "ban req.http.nevermatch == 1"
+
+client c1 {
+	txreq -url /r2
+	rxresp
+} -run
+
+varnish v1 -cliok "ban req.http.nevermatch == 2"
+
+client c1 {
+	txreq -url /r3
+	rxresp
+} -run
+
+varnish v1 -cliok "ban req.http.nevermatch == 3"
+
+varnish v1 -cliok "ban.list"
+
+varnish v1 -cliok "ban obj.http.status != 0"
+
+delay 1
+
+varnish v1 -cliok "ban.list"
+
+varnish v1 -expect n_object == 0

From fgsch at lodoss.net  Wed Oct  2 19:33:06 2019
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed,  2 Oct 2019 19:33:06 +0000 (UTC)
Subject: [master] de908d309 Fix asan build in travis
Message-ID: <20191002193306.EDBC997CE1@lists.varnish-cache.org>


commit de908d3095563d4f6440897feaea105b6c74d0ce
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Oct 2 20:31:48 2019 +0100

    Fix asan build in travis

diff --git a/.travis.yml b/.travis.yml
index b68503157..16a24b6aa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,6 +39,7 @@ jobs:
             - llvm-toolchain-xenial-8
           packages:
             - clang-8
+            - libunwind-dev
             - llvm-8
             - nghttp2
             - python3-docutils

From phk at FreeBSD.org  Thu Oct  3 09:06:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu,  3 Oct 2019 09:06:07 +0000 (UTC)
Subject: [master] 15f3ecff3 Flexelinting: Avoid stuff which looks like
 trigraphs
Message-ID: <20191003090607.61DD2B03D9@lists.varnish-cache.org>


commit 15f3ecff31f59d2893a80434601d1196d5ca5607
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 09:04:12 2019 +0000

    Flexelinting:  Avoid stuff which looks like trigraphs

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index aa9d914b7..37ea6b8fd 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -647,7 +647,7 @@ pan_backtrace(struct vsb *vsb)
 		unw_get_reg(&cursor, UNW_REG_SP, &sp);
 		unw_get_proc_name(&cursor, fname, sizeof(fname), &offp);
 		VSB_printf(vsb, "ip=0x%lx, sp=0x%lx <%s+0x%lx>\n", (long) ip,
-		    (long) sp, fname[0] ? fname : "???", offp);
+		    (long) sp, fname[0] ? fname : "<unknown>", offp);
 	}
 
 	VSB_indent(vsb, -2);

From phk at FreeBSD.org  Thu Oct  3 09:06:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu,  3 Oct 2019 09:06:07 +0000 (UTC)
Subject: [master] fb70da351 Change the ID of TCP pools from void * to
 uintmax_t.
Message-ID: <20191003090607.7B3F4B03DC@lists.varnish-cache.org>


commit fb70da351bed1c9d3e51f688c3dc98b11e017ed5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 09:05:03 2019 +0000

    Change the ID of TCP pools from void * to uintmax_t.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index f1fd423e6..a3062176d 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -563,7 +563,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 	AN(be->vsc);
 
 	be->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr,
-	    vrt->path, vbe_proto_ident);
+	    vrt->path, (uintptr_t)vbe_proto_ident);
 	AN(be->tcp_pool);
 
 	vbp = vrt->probe;
diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index 63691b91b..49a2836c8 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -85,7 +85,7 @@ struct conn_pool {
 
 	const struct cp_methods			*methods;
 
-	const void				*id;
+	uintmax_t				id;
 	void					*priv;
 
 	VTAILQ_ENTRY(conn_pool)			list;
@@ -201,7 +201,7 @@ vcp_handle(struct waited *w, enum wait_event ev, vtim_real now)
  */
 
 static struct conn_pool *
-VCP_Ref(const void *id, const void *priv)
+VCP_Ref(uintmax_t id, const void *priv)
 {
 	struct conn_pool *cp;
 
@@ -224,7 +224,7 @@ VCP_Ref(const void *id, const void *priv)
  */
 
 static void *
-VCP_New(struct conn_pool *cp, const void *id, void *priv,
+VCP_New(struct conn_pool *cp, uintmax_t id, void *priv,
     const struct cp_methods *cm)
 {
 
@@ -728,7 +728,7 @@ static const struct cp_methods vus_methods = {
 
 struct tcp_pool *
 VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds,
-	const void *id)
+	uintmax_t id)
 {
 	struct tcp_pool *tp;
 	struct conn_pool *cp;
diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h
index 3c872336f..d7608da85 100644
--- a/bin/varnishd/cache/cache_tcp_pool.h
+++ b/bin/varnishd/cache/cache_tcp_pool.h
@@ -53,7 +53,7 @@ void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned);
 struct VSC_vbe;
 
 struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6,
-    const char *uds, const void *id);
+    const char *uds, uintmax_t id);
 	/*
 	 * Get a reference to a TCP pool. Either one or both of ip4 or
 	 * ip6 arg must be non-NULL, or uds must be non-NULL. If recycling

From dridi.boukelmoune at gmail.com  Fri Oct  4 08:22:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  4 Oct 2019 08:22:06 +0000 (UTC)
Subject: [master] d32906f86 Check IPv6 in varnishtest dns feature
Message-ID: <20191004082206.CD953A414F@lists.varnish-cache.org>


commit d32906f86aeee1fe462ced9897a31c6d40110ea8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 4 10:17:49 2019 +0200

    Check IPv6 in varnishtest dns feature
    
    On a network where you can't trust the DNS resolver because it filters
    AAAA records out v00016.vtc fails.

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index d2c897f46..d49388352 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -352,13 +352,20 @@ dns_works(void)
 	char abuf[VTCP_ADDRBUFSIZE];
 	char pbuf[VTCP_PORTBUFSIZE];
 
-	sa = VSS_ResolveOne(NULL, "dns-canary.freebsd.dk", NULL, AF_UNSPEC, SOCK_STREAM, 0);
+	sa = VSS_ResolveOne(NULL, "dns-canary.freebsd.dk", NULL,
+	    AF_UNSPEC, SOCK_STREAM, 0);
 	if (sa == NULL)
 		return (0);
 	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
 	free(sa);
 	if (strcmp(abuf, "192.0.2.255"))
 		return (0);
+
+	sa = VSS_ResolveOne(NULL, "varnish.org", NULL,
+	    AF_INET6, SOCK_STREAM, 0);
+	if (sa == NULL)
+		return (0);
+	free(sa);
 	return (1);
 }
 

From phk at FreeBSD.org  Mon Oct  7 13:23:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  7 Oct 2019 13:23:06 +0000 (UTC)
Subject: [master] 0ba4d2941 Update docs for supported VCL versions
Message-ID: <20191007132306.4F332700D@lists.varnish-cache.org>


commit 0ba4d2941404f90cfeb0abae26cb9200b2931ad0
Author: Jordan Christiansen <jordan.christiansen at target.com>
Date:   Thu Oct 3 10:32:51 2019 -0500

    Update docs for supported VCL versions

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 03382272a..1fc2395fb 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -573,7 +573,7 @@ not have surprises in the future.  The syntax version set in an
 included file only applies to that file and any files it includes -
 unless these set their own VCL syntax version.
 
-The version of Varnish this file belongs to supports syntax 4.0 only.
+The version of Varnish this file belongs to supports syntax 4.0 and 4.1.
 
 
 EXAMPLES

From nils.goroll at uplex.de  Mon Oct  7 14:27:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  7 Oct 2019 14:27:06 +0000 (UTC)
Subject: [master] ecfd1eb02 make VFP_Close() idempotent
Message-ID: <20191007142706.77B4597D4@lists.varnish-cache.org>


commit ecfd1eb02daa8d90c87d33c3866a79fea80691c3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:42:46 2019 +0200

    make VFP_Close() idempotent
    
    required for #3009

diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index d9db19f57..e92c8a685 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -118,13 +118,14 @@ VFP_Setup(struct vfp_ctx *vc, struct worker *wrk)
 void
 VFP_Close(struct vfp_ctx *vc)
 {
-	struct vfp_entry *vfe;
+	struct vfp_entry *vfe, *tmp;
 
-	VTAILQ_FOREACH(vfe, &vc->vfp, list) {
+	VTAILQ_FOREACH_SAFE(vfe, &vc->vfp, list, tmp) {
 		if (vfe->vfp->fini != NULL)
 			vfe->vfp->fini(vc, vfe);
 		VSLb(vc->wrk->vsl, SLT_VfpAcct, "%s %ju %ju", vfe->vfp->name,
 		    (uintmax_t)vfe->calls, (uintmax_t)vfe->bytes_out);
+		VTAILQ_REMOVE(&vc->vfp, vfe, list);
 	}
 }
 

From nils.goroll at uplex.de  Mon Oct  7 14:27:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  7 Oct 2019 14:27:06 +0000 (UTC)
Subject: [master] bbd4c476b centralize cleanup after fetch errors
Message-ID: <20191007142706.9117197D6@lists.varnish-cache.org>


commit bbd4c476bf0e2d9b7282f4dab59fa390d304a470
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:43:05 2019 +0200

    centralize cleanup after fetch errors
    
    imples the following changes:
    
    * VDI_Finish() is now always conditional on bo->director_state !=
      DIR_S_NULL, making it idempotent
    
    * introduces additional calls to VFP_Close() from startfetch and
      for the filter_list / VCL_StackVFP error in vbf_stp_fetch(),
      but VFP_Close() is idempotent.
    
    * adds VFP_Close() for VFP_Open() failure in vbf_stp_fetch() which
      I think is actually missing (for the case that some VFPs could
      get opened before the open failure)
    
    * calls VDI_Finish() earlier in vbf_stp_fetchend: I checked the
      code and can not see any issue with this.
    
    motivated by #3009

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f9cab4055..ec393a092 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -84,6 +84,20 @@ vbf_allocobj(struct busyobj *bo, unsigned l)
 	return (STV_NewObject(bo->wrk, bo->fetch_objcore, stv_transient, l));
 }
 
+static void
+vbf_cleanup(struct busyobj *bo)
+{
+	struct vfp_ctx *vfc;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	vfc = bo->vfc;
+	CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC);
+
+	VFP_Close(vfc);
+	if (bo->director_state != DIR_S_NULL)
+		VDI_Finish(bo);
+}
+
 /*--------------------------------------------------------------------
  * Turn the beresp into a obj
  */
@@ -232,7 +246,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 
 	VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk));
 
-	/* VDI_Finish must have been called before */
+	/* VDI_Finish (via vbf_cleanup) must have been called before */
 	assert(bo->director_state == DIR_S_NULL);
 
 	/* reset other bo attributes - See VBO_GetBusyObj */
@@ -307,7 +321,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->htc->body_status == BS_ERROR) {
 		bo->htc->doclose = SC_RX_BODY;
-		VDI_Finish(bo);
+		vbf_cleanup(bo);
 		VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
 		assert(bo->director_state == DIR_S_NULL);
 		return (F_STP_ERROR);
@@ -380,7 +394,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 			VSLb(bo->vsl, SLT_Error,
 			    "304 response but not conditional fetch");
 			bo->htc->doclose = SC_RX_BAD;
-			VDI_Finish(bo);
+			vbf_cleanup(bo);
 			return (F_STP_ERROR);
 		}
 	}
@@ -390,7 +404,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL ||
 	    wrk->handling == VCL_RET_ERROR) {
 		bo->htc->doclose = SC_RESP_CLOSE;
-		VDI_Finish(bo);
+		vbf_cleanup(bo);
 		if (wrk->handling == VCL_RET_ERROR)
 			return (F_STP_ERROR);
 		else
@@ -400,8 +414,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_RETRY) {
 		if (bo->htc->body_status != BS_NONE)
 			bo->htc->doclose = SC_RESP_CLOSE;
-		if (bo->director_state != DIR_S_NULL)
-			VDI_Finish(bo);
+		vbf_cleanup(bo);
 
 		if (bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
@@ -491,8 +504,7 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo)
 	if (vfc->failed) {
 		(void)VFP_Error(vfc, "Fetch pipeline failed to process");
 		bo->htc->doclose = SC_RX_BODY;
-		VFP_Close(vfc);
-		VDI_Finish(bo);
+		vbf_cleanup(bo);
 		if (!bo->do_stream) {
 			assert(bo->fetch_objcore->boc->state < BOS_STREAM);
 			// XXX: doclose = ?
@@ -529,7 +541,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->filter_list == NULL ||
 	    VCL_StackVFP(bo->vfc, bo->vcl, bo->filter_list)) {
 		(bo)->htc->doclose = SC_OVERLOAD;
-		VDI_Finish(bo);
+		vbf_cleanup(bo);
 		return (F_STP_ERROR);
 	}
 
@@ -541,15 +553,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (VFP_Open(bo->vfc)) {
 		(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
 		bo->htc->doclose = SC_RX_BODY;
-		VDI_Finish(bo);
+		vbf_cleanup(bo);
 		return (F_STP_ERROR);
 	}
 
 	if (vbf_beresp2obj(bo)) {
 		(void)VFP_Error(bo->vfc, "Could not get storage");
 		bo->htc->doclose = SC_RX_BODY;
-		VFP_Close(bo->vfc);
-		VDI_Finish(bo);
+		vbf_cleanup(bo);
 		return (F_STP_ERROR);
 	}
 
@@ -591,7 +602,10 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
 {
 
 	AZ(bo->vfc->failed);
-	VFP_Close(bo->vfc);
+
+	/* Recycle the backend connection before setting BOS_FINISHED to
+	   give predictable backend reuse behavior for varnishtest */
+	vbf_cleanup(bo);
 
 	AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN,
 	    bo->fetch_objcore->boc->len_so_far));
@@ -604,10 +618,6 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
 		HSH_Unbusy(wrk, bo->fetch_objcore);
 	}
 
-	/* Recycle the backend connection before setting BOS_FINISHED to
-	   give predictable backend reuse behavior for varnishtest */
-	VDI_Finish(bo);
-
 	ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	if (bo->stale_oc != NULL)
@@ -673,7 +683,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->stale_oc->flags & OC_F_FAILED)
 		(void)VFP_Error(bo->vfc, "Template object failed");
 	if (bo->vfc->failed) {
-		VDI_Finish(bo);
+		vbf_cleanup(bo);
 		wrk->stats->fetch_failed++;
 		return (F_STP_FAIL);
 	}

From nils.goroll at uplex.de  Mon Oct  7 14:27:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  7 Oct 2019 14:27:06 +0000 (UTC)
Subject: [master] e2b43f86f also reset filter_list upon cleanup
Message-ID: <20191007142706.AEAFA97DA@lists.varnish-cache.org>


commit e2b43f86fdf719c2f9cf1f6f976e5bef064ea565
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:43:09 2019 +0200

    also reset filter_list upon cleanup
    
    in particular because it may live on the workspace.
    
    Required for #3009

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ec393a092..130278b65 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -94,6 +94,8 @@ vbf_cleanup(struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC);
 
 	VFP_Close(vfc);
+	bo->filter_list = NULL;
+
 	if (bo->director_state != DIR_S_NULL)
 		VDI_Finish(bo);
 }
@@ -253,7 +255,6 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 	bo->storage = NULL;
 	bo->do_esi = 0;
 	bo->do_stream = 1;
-	bo->filter_list = NULL;
 	bo->was_304 = 0;
 
 	// XXX: BereqEnd + BereqAcct ?

From nils.goroll at uplex.de  Mon Oct  7 14:27:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  7 Oct 2019 14:27:06 +0000 (UTC)
Subject: [master] 0ed38379a add test case by @rezan
Message-ID: <20191007142706.CC32B97DE@lists.varnish-cache.org>


commit 0ed38379ae750e02765fd3880741a66d03a06dab
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:43:13 2019 +0200

    add test case by @rezan

diff --git a/bin/varnishtest/tests/r03009.vtc b/bin/varnishtest/tests/r03009.vtc
new file mode 100644
index 000000000..406a40d02
--- /dev/null
+++ b/bin/varnishtest/tests/r03009.vtc
@@ -0,0 +1,33 @@
+varnishtest "Rollback without restart/retry is unsafe"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+
+	sub vcl_recv {
+		set req.http.test = "1";
+	}
+
+	sub vcl_backend_fetch {
+		unset bereq.http.test;
+	}
+
+	sub vcl_backend_response {
+		std.rollback(bereq);
+		set beresp.http.test = bereq.http.test;
+		set beresp.http.workspace = "start overwriting active workspace";
+		set beresp.http.workspace = "0123456789012345678901234567890123456789";
+		# panic...
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.test == "1"
+} -run

From nils.goroll at uplex.de  Mon Oct  7 14:27:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  7 Oct 2019 14:27:06 +0000 (UTC)
Subject: [master] 7d25b83b8 fix bereq rollback
Message-ID: <20191007142706.EF3E097EC@lists.varnish-cache.org>


commit 7d25b83b855e650dde9aa5af59d52a4f0be302e2
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:49:28 2019 +0200

    fix bereq rollback
    
    by properly cleaning up the busyobj
    
    Also move the relevant code from cache_vrt.c to cache_fetch.c
    
    As we fini the director during cleanup, we now also need to handle the
    backend connection gone missing in vbf_stp_fetch(). The hypothetical
    alternative would be to not fini the director, but I believe this is not
    safe in case it also used some workspace.
    
    Fixes #3009

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 130278b65..38a25d6dd 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -100,6 +100,18 @@ vbf_cleanup(struct busyobj *bo)
 		VDI_Finish(bo);
 }
 
+void Bereq_Rollback(struct busyobj *bo)
+{
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	vbf_cleanup(bo);
+	VCL_TaskLeave(bo->vcl, bo->privs);
+	VCL_TaskEnter(bo->vcl, bo->privs);
+	HTTP_Clone(bo->bereq, bo->bereq0);
+	WS_Reset(bo->bereq->ws, bo->ws_bo);
+	WS_Reset(bo->ws, bo->ws_bo);
+}
+
 /*--------------------------------------------------------------------
  * Turn the beresp into a obj
  */
@@ -404,7 +416,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL ||
 	    wrk->handling == VCL_RET_ERROR) {
-		bo->htc->doclose = SC_RESP_CLOSE;
+		if (bo->htc)
+			bo->htc->doclose = SC_RESP_CLOSE;
 		vbf_cleanup(bo);
 		if (wrk->handling == VCL_RET_ERROR)
 			return (F_STP_ERROR);
@@ -413,7 +426,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	}
 
 	if (wrk->handling == VCL_RET_RETRY) {
-		if (bo->htc->body_status != BS_NONE)
+		if (bo->htc && bo->htc->body_status != BS_NONE)
 			bo->htc->doclose = SC_RESP_CLOSE;
 		vbf_cleanup(bo);
 
@@ -529,6 +542,12 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(wrk->handling == VCL_RET_DELIVER);
 
+	if (bo->htc == NULL) {
+		(void)VFP_Error(bo->vfc, "No backend connection (rollback?)");
+		vbf_cleanup(bo);
+		return (F_STP_ERROR);
+	}
+
 	/* No body -> done */
 	if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) {
 		http_Unset(bo->beresp, H_Content_Encoding);
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index b3f766080..d32c8dde8 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -238,6 +238,7 @@ enum vbf_fetch_mode_e {
 void VBF_Fetch(struct worker *wrk, struct req *req,
     struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e);
 const char *VBF_Get_Filter_List(struct busyobj *);
+void Bereq_Rollback(struct busyobj *);
 
 /* cache_fetch_proc.c */
 void VFP_Init(void);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index e25866be1..2838e9190 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -719,12 +719,7 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp)
 		Req_Rollback(ctx->req);
 	} else if (hp == ctx->http_bereq) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-		// -> VBO_Rollback ?
-		VCL_TaskLeave(ctx->bo->vcl, ctx->bo->privs);
-		VCL_TaskEnter(ctx->bo->vcl, ctx->bo->privs);
-		HTTP_Clone(ctx->bo->bereq, ctx->bo->bereq0);
-		WS_Reset(ctx->bo->bereq->ws, ctx->bo->ws_bo);
-		WS_Reset(ctx->bo->ws, ctx->bo->ws_bo);
+		Bereq_Rollback(ctx->bo);
 	} else
 		WRONG("VRT_Rollback 'hp' invalid");
 }
diff --git a/bin/varnishtest/tests/r03009.vtc b/bin/varnishtest/tests/r03009.vtc
index 406a40d02..1bc44521f 100644
--- a/bin/varnishtest/tests/r03009.vtc
+++ b/bin/varnishtest/tests/r03009.vtc
@@ -28,6 +28,5 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq
 	rxresp
-	expect resp.status == 200
-	expect resp.http.test == "1"
+	expect resp.status == 503
 } -run

From reza at naghibi.com  Mon Oct  7 15:46:06 2019
From: reza at naghibi.com (Reza Naghibi)
Date: Mon,  7 Oct 2019 15:46:06 +0000 (UTC)
Subject: [6.0] e4f08065f Make waitinglist rushes propagate on streaming
 delivery
Message-ID: <20191007154606.E49C861AE3@lists.varnish-cache.org>


commit e4f08065f4e03d6596d9d15d8d130453354cd6d4
Author: Andrew Wiik <andrew at varnish-software.com>
Date:   Tue Sep 10 15:08:18 2019 -0400

    Make waitinglist rushes propagate on streaming delivery
    
    This makes waitinglist rushes happen also in HSH_Lookup when encountering
    cache hits. This helps to get the requests on the waitinglist restarted
    when doing streaming delivery. Fixes #2977. Backported from @mbgrydeland
    master commits.
    
    Based on: 3736849608cb8907b0c3ef93b01691ea3281f48a

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9101cc568..d224e6c6b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -658,6 +658,7 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
 		assert(bo->fetch_objcore->boc->state == BOS_STREAM);
 	else {
 		assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE);
+		ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
 		HSH_Unbusy(wrk, bo->fetch_objcore);
 	}
 
@@ -839,6 +840,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	}
 	AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o));
 	VSB_destroy(&synth_body);
+	ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
 	HSH_Unbusy(wrk, bo->fetch_objcore);
 	ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
 	return (F_STP_DONE);
@@ -1027,11 +1029,10 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 			(void)VRB_Ignore(req);
 		} else {
 			ObjWaitState(oc, BOS_STREAM);
-			if (oc->boc->state == BOS_FAILED) {
+			if (oc->boc->state == BOS_FAILED)
 				AN((oc->flags & OC_F_FAILED));
-			} else {
+			else
 				AZ(oc->flags & OC_F_BUSY);
-			}
 		}
 	}
 	AZ(bo);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index f4d2fecce..0bd31f2ca 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -77,7 +77,8 @@ static void hsh_rush1(const struct worker *, struct objhead *,
     struct rush *, int);
 static void hsh_rush2(struct worker *, struct rush *);
 static int hsh_deref_objhead(struct worker *wrk, struct objhead **poh);
-static int hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh);
+static int hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh,
+	int);
 
 /*---------------------------------------------------------------------*/
 
@@ -408,11 +409,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 			continue;
 
 		CHECK_OBJ_ORNULL(oc->boc, BOC_MAGIC);
-		if (oc->boc != NULL && oc->boc->state < BOS_STREAM) {
+		if (oc->flags & OC_F_BUSY) {
 			if (req->hash_ignore_busy)
 				continue;
 
-			if (oc->boc->vary != NULL &&
+			if (oc->boc && oc->boc->vary != NULL &&
 			    !VRY_Match(req, oc->boc->vary))
 				continue;
 
@@ -455,7 +456,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 	if (oc != NULL && oc->flags & OC_F_HFP) {
 		xid = ObjGetXID(wrk, oc);
 		dttl = EXP_Dttl(req, oc);
-		AN(hsh_deref_objhead_unlock(wrk, &oh));
+		AN(hsh_deref_objhead_unlock(wrk, &oh, HSH_RUSH_POLICY));
 		wrk->stats->cache_hitpass++;
 		VSLb(req->vsl, SLT_HitPass, "%u %.6f", xid, dttl);
 		return (HSH_HITPASS);
@@ -475,7 +476,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 		}
 		if (oc->hits < LONG_MAX)
 			oc->hits++;
-		AN(hsh_deref_objhead_unlock(wrk, &oh));
+		AN(hsh_deref_objhead_unlock(wrk, &oh, HSH_RUSH_POLICY));
 		return (HSH_HIT);
 	}
 
@@ -518,7 +519,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 		*ocp = exp_oc;
 		if (exp_oc->hits < LONG_MAX)
 			exp_oc->hits++;
-		AN(hsh_deref_objhead_unlock(wrk, &oh));
+		AN(hsh_deref_objhead_unlock(wrk, &oh, 0));
 		return (HSH_GRACE);
 	}
 
@@ -959,9 +960,11 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax)
 }
 
 static int
-hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh)
+hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh, int max)
 {
 	struct objhead *oh;
+	struct rush rush;
+	int r;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC);
@@ -976,11 +979,19 @@ hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh)
 		return(1);
 	}
 
+	INIT_OBJ(&rush, RUSH_MAGIC);
+	if (!VTAILQ_EMPTY(&oh->waitinglist)) {
+		assert(oh->refcnt > 1);
+		hsh_rush1(wrk, oh, &rush, max);
+	}
+
 	if (oh->refcnt == 1)
 		assert(VTAILQ_EMPTY(&oh->waitinglist));
 
 	assert(oh->refcnt > 0);
-	return (hash->deref(wrk, oh));
+	r = hash->deref(wrk, oh); /* Unlocks oh->mtx */
+	hsh_rush2(wrk, &rush);
+	return (r);
 }
 
 static int
@@ -992,7 +1003,7 @@ hsh_deref_objhead(struct worker *wrk, struct objhead **poh)
 	TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC);
 
 	Lck_Lock(&oh->mtx);
-	return (hsh_deref_objhead_unlock(wrk, &oh));
+	return (hsh_deref_objhead_unlock(wrk, &oh, 0));
 }
 
 void
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 78395d7e5..c01d45715 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -354,6 +354,8 @@ cnt_transmit(struct worker *wrk, struct req *req)
 
 	/* Grab a ref to the bo if there is one (=streaming) */
 	boc = HSH_RefBoc(req->objcore);
+	if (boc && boc->state < BOS_STREAM)
+		ObjWaitState(req->objcore, BOS_STREAM);
 	clval = http_GetContentLength(req->resp);
 	/* RFC 7230, 3.3.3 */
 	status = http_GetStatus(req->resp);
diff --git a/bin/varnishtest/tests/c00097.vtc b/bin/varnishtest/tests/c00097.vtc
new file mode 100644
index 000000000..afc77fbd9
--- /dev/null
+++ b/bin/varnishtest/tests/c00097.vtc
@@ -0,0 +1,65 @@
+varnishtest "Streaming delivery and waitinglist rushing"
+
+# Barrier to make sure that c1 connects to s1
+barrier b1 cond 2
+
+# Barrier to make sure that all requests are on waitinglist before
+# HSH_Unbusy is called
+barrier b2 cond 2
+
+# Barrier to control that all requests start streaming before the object
+# finishes. This tests that waitinglists are rushed before
+# HSH_DerefObjCore().
+barrier b3 sock 4
+
+server s1 {
+	rxreq
+	barrier b1 sync
+	barrier b2 sync
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend {
+	import vtc;
+	sub vcl_hit {
+		vtc.barrier_sync("${b3_sock}");
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+} -start
+
+barrier b1 sync
+
+client c2 {
+	txreq
+	rxresp
+} -start
+
+client c3 {
+	txreq
+	rxresp
+} -start
+
+client c4 {
+	txreq
+	rxresp
+} -start
+
+# Wait until c2-c4 are on the waitinglist
+delay 1
+varnish v1 -expect busy_sleep == 3
+
+# Open up the response headers from s1, and as a result HSH_Unbusy
+barrier b2 sync
+
+client c1 -wait
+client c2 -wait
+client c3 -wait
+client c4 -wait
diff --git a/bin/varnishtest/tests/c00098.vtc b/bin/varnishtest/tests/c00098.vtc
new file mode 100644
index 000000000..5b34d3a75
--- /dev/null
+++ b/bin/varnishtest/tests/c00098.vtc
@@ -0,0 +1,137 @@
+varnishtest "Hit-for-pass and waitinglist rushing"
+
+# Barrier to make sure that s1 is run first
+barrier b1 cond 2
+
+# Barrier to make sure that all requests are on waitinglist before
+# HSH_Unbusy is called
+barrier b2 cond 2
+
+# Barrier to control that all backends are reached before any request
+# finishes. This tests that waitinglists are rushed before
+# HSH_DerefObjCore().
+barrier b3 cond 6
+
+server s1 {
+	rxreq
+	barrier b1 sync
+	barrier b2 sync
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s2 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s3 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s4 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s5 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s6 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend {
+	sub vcl_backend_fetch {
+		if (bereq.http.client == "1") {
+			set bereq.backend = s1;
+		} else if (bereq.http.client == "2") {
+			set bereq.backend = s2;
+		} else if (bereq.http.client == "3") {
+			set bereq.backend = s3;
+		} else if (bereq.http.client == "4") {
+			set bereq.backend = s4;
+		} else if (bereq.http.client == "5") {
+			set bereq.backend = s5;
+		} else if (bereq.http.client == "6") {
+			set bereq.backend = s6;
+		}
+	}
+	sub vcl_backend_response {
+		return (pass(1m));
+	}
+} -start
+
+client c1 {
+	txreq -url /hfp -hdr "Client: 1"
+	rxresp
+} -start
+
+# This makes sure that c1->s1 is done first
+barrier b1 sync
+
+client c2 {
+	txreq -url /hfp -hdr "Client: 2"
+	rxresp
+} -start
+
+client c3 {
+	txreq -url /hfp -hdr "Client: 3"
+	rxresp
+} -start
+
+client c4 {
+	txreq -url /hfp -hdr "Client: 4"
+	rxresp
+} -start
+
+client c5 {
+	txreq -url /hfp -hdr "Client: 5"
+	rxresp
+} -start
+
+client c6 {
+	txreq -url /hfp -hdr "Client: 6"
+	rxresp
+} -start
+
+# Wait until c2-c6 are on the waitinglist
+delay 1
+varnish v1 -expect busy_sleep == 5
+
+# Open up the response headers from s1, and as a result HSH_Unbusy
+barrier b2 sync
+
+client c1 -wait
+client c2 -wait
+client c3 -wait
+client c4 -wait
+client c5 -wait
+client c6 -wait
diff --git a/bin/varnishtest/tests/c00099.vtc b/bin/varnishtest/tests/c00099.vtc
new file mode 100644
index 000000000..4bbd904a0
--- /dev/null
+++ b/bin/varnishtest/tests/c00099.vtc
@@ -0,0 +1,137 @@
+varnishtest "Hit-for-miss and waitinglist rushing"
+
+# Barrier to make sure that s1 is run first
+barrier b1 cond 2
+
+# Barrier to make sure that all requests are on waitinglist before
+# HSH_Unbusy is called
+barrier b2 cond 2
+
+# Barrier to control that all backends are reached before any request
+# finishes. This tests that waitinglists are rushed before
+# HSH_DerefObjCore().
+barrier b3 cond 6
+
+server s1 {
+	rxreq
+	barrier b1 sync
+	barrier b2 sync
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s2 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s3 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s4 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s5 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+server s6 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	barrier b3 sync
+	chunkedlen 10
+	chunkedlen 0
+} -start
+
+varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend {
+	sub vcl_backend_fetch {
+		if (bereq.http.client == "1") {
+			set bereq.backend = s1;
+		} else if (bereq.http.client == "2") {
+			set bereq.backend = s2;
+		} else if (bereq.http.client == "3") {
+			set bereq.backend = s3;
+		} else if (bereq.http.client == "4") {
+			set bereq.backend = s4;
+		} else if (bereq.http.client == "5") {
+			set bereq.backend = s5;
+		} else if (bereq.http.client == "6") {
+			set bereq.backend = s6;
+		}
+	}
+	sub vcl_backend_response {
+		set beresp.uncacheable = true;
+	}
+} -start
+
+client c1 {
+	txreq -url /hfm -hdr "Client: 1"
+	rxresp
+} -start
+
+# This makes sure that c1->s1 is done first
+barrier b1 sync
+
+client c2 {
+	txreq -url /hfm -hdr "Client: 2"
+	rxresp
+} -start
+
+client c3 {
+	txreq -url /hfm -hdr "Client: 3"
+	rxresp
+} -start
+
+client c4 {
+	txreq -url /hfm -hdr "Client: 4"
+	rxresp
+} -start
+
+client c5 {
+	txreq -url /hfm -hdr "Client: 5"
+	rxresp
+} -start
+
+client c6 {
+	txreq -url /hfm -hdr "Client: 6"
+	rxresp
+} -start
+
+# Wait until c2-c6 are on the waitinglist
+delay 1
+varnish v1 -expect busy_sleep == 5
+
+# Open up the response headers from s1, and as a result HSH_Unbusy
+barrier b2 sync
+
+client c1 -wait
+client c2 -wait
+client c3 -wait
+client c4 -wait
+client c5 -wait
+client c6 -wait

From phk at FreeBSD.org  Tue Oct  8 07:55:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  8 Oct 2019 07:55:06 +0000 (UTC)
Subject: [master] ec70dbc75 Make requestor responsible for providing unique
 TCP-pool ident.
Message-ID: <20191008075506.533F7A6764@lists.varnish-cache.org>


commit ec70dbc7502f8fbe6c8eb5af6108df4168500f9a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 8 07:52:49 2019 +0000

    Make requestor responsible for providing unique TCP-pool ident.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index a3062176d..80134b35d 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -36,6 +36,9 @@
 
 #include "cache_varnishd.h"
 
+#include "vend.h"
+#include "vsa.h"
+#include "vsha256.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -524,6 +527,24 @@ VRT_backend_vsm_need(VRT_CTX)
 	return (VRT_VSC_Overhead(VSC_vbe_size));
 }
 
+static uint64_t
+vrt_hash_be(const struct vrt_backend *vrt)
+{
+	struct VSHA256Context cx[1];
+	unsigned char ident[VSHA256_DIGEST_LENGTH];
+
+	VSHA256_Init(cx);
+	VSHA256_Update(cx, vbe_proto_ident, strlen(vbe_proto_ident));
+	if (vrt->ipv4_suckaddr != NULL)
+		VSHA256_Update(cx, vrt->ipv4_suckaddr, vsa_suckaddr_len);
+	if (vrt->ipv6_suckaddr != NULL)
+		VSHA256_Update(cx, vrt->ipv6_suckaddr, vsa_suckaddr_len);
+	if (vrt->path != NULL)
+		VSHA256_Update(cx, vrt->path, strlen(vrt->path));
+	VSHA256_Final(ident, cx);
+	return (vbe64dec(ident));
+}
+
 VCL_BACKEND v_matchproto_()
 VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
     const struct vrt_backend *vrt)
@@ -563,7 +584,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 	AN(be->vsc);
 
 	be->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr,
-	    vrt->path, (uintptr_t)vbe_proto_ident);
+	    vrt->path, vrt_hash_be(vrt));
 	AN(be->tcp_pool);
 
 	vbp = vrt->probe;
diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index 49a2836c8..5aa207f84 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -68,13 +68,11 @@ struct pfd {
 typedef int cp_open_f(const struct conn_pool *, vtim_dur tmo,
     const void **privp);
 typedef void cp_close_f(struct pfd *);
-typedef int cp_cmp_f(const struct conn_pool *, const void *priv);
 typedef void cp_name_f(const struct pfd *, char *, unsigned, char *, unsigned);
 
 struct cp_methods {
 	cp_open_f				*open;
 	cp_close_f				*close;
-	cp_cmp_f				*cmp;
 	cp_name_f				*local_name;
 	cp_name_f				*remote_name;
 };
@@ -201,7 +199,7 @@ vcp_handle(struct waited *w, enum wait_event ev, vtim_real now)
  */
 
 static struct conn_pool *
-VCP_Ref(uintmax_t id, const void *priv)
+VCP_Ref(uintmax_t id)
 {
 	struct conn_pool *cp;
 
@@ -210,8 +208,6 @@ VCP_Ref(uintmax_t id, const void *priv)
 		assert(cp->refcnt > 0);
 		if (cp->id != id)
 			continue;
-		if (cp->methods->cmp(cp, priv))
-			continue;
 		cp->refcnt++;
 		Lck_Unlock(&conn_pools_mtx);
 		return (cp);
@@ -232,7 +228,6 @@ VCP_New(struct conn_pool *cp, uintmax_t id, void *priv,
 	AN(cm);
 	AN(cm->open);
 	AN(cm->close);
-	AN(cm->cmp);
 
 	INIT_OBJ(cp, CONN_POOL_MAGIC);
 	cp->id = id;
@@ -620,31 +615,6 @@ vtp_close(struct pfd *pfd)
 	VTCP_close(&pfd->fd);
 }
 
-static int v_matchproto_(cp_cmp_f)
-vtp_cmp(const struct conn_pool *cp, const void *priv)
-{
-	const struct vtp_cs *vcs;
-	const struct tcp_pool *tp;
-
-	CAST_OBJ_NOTNULL(vcs, priv, VTP_CS_MAGIC);
-	CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC);
-	if (tp->ip4 == NULL && vcs->ip4 != NULL)
-		return (1);
-	if (tp->ip4 != NULL && vcs->ip4 == NULL)
-		return (1);
-	if (tp->ip6 == NULL && vcs->ip6 != NULL)
-		return (1);
-	if (tp->ip6 != NULL && vcs->ip6 == NULL)
-		return (1);
-	if (tp->ip4 != NULL && vcs->ip4 != NULL &&
-	    VSA_Compare(tp->ip4, vcs->ip4))
-		return (1);
-	if (tp->ip6 != NULL && vcs->ip6 != NULL &&
-	    VSA_Compare(tp->ip6, vcs->ip6))
-		return (1);
-	return (0);
-}
-
 static void v_matchproto_(cp_name_f)
 vtp_local_name(const struct pfd *pfd, char *addr, unsigned alen, char *pbuf,
 	       unsigned plen)
@@ -664,7 +634,6 @@ vtp_remote_name(const struct pfd *pfd, char *addr, unsigned alen, char *pbuf,
 static const struct cp_methods vtp_methods = {
 	.open = vtp_open,
 	.close = vtp_close,
-	.cmp = vtp_cmp,
 	.local_name = vtp_local_name,
 	.remote_name = vtp_remote_name,
 };
@@ -689,19 +658,6 @@ vus_open(const struct conn_pool *cp, vtim_dur tmo, const void **privp)
 	return (s);
 }
 
-static int v_matchproto_(cp_cmp_f)
-vus_cmp(const struct conn_pool *cp, const void *priv)
-{
-	const struct vtp_cs *vcs;
-	const struct tcp_pool *tp;
-
-	CAST_OBJ_NOTNULL(vcs, priv, VTP_CS_MAGIC);
-	CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC);
-	if (tp->uds != NULL && vcs->uds != NULL)
-		return (strcmp(tp->uds, vcs->uds));
-	return (1);
-}
-
 static void v_matchproto_(cp_name_f)
 vus_name(const struct pfd *pfd, char *addr, unsigned alen, char *pbuf,
 	 unsigned plen)
@@ -716,7 +672,6 @@ vus_name(const struct pfd *pfd, char *addr, unsigned alen, char *pbuf,
 static const struct cp_methods vus_methods = {
 	.open = vus_open,
 	.close = vtp_close,
-	.cmp = vus_cmp,
 	.local_name = vus_name,
 	.remote_name = vus_name,
 };
@@ -742,7 +697,7 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds,
 	vcs.ip6 = ip6;
 	vcs.uds = uds;
 
-	cp = VCP_Ref(id, &vcs);
+	cp = VCP_Ref(id);
 	if (cp != NULL)
 		return (cp->priv);
 

From phk at FreeBSD.org  Tue Oct  8 08:06:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  8 Oct 2019 08:06:06 +0000 (UTC)
Subject: [master] bba1499f5 Extend vrt_backend with a connection prefix ptr+len
Message-ID: <20191008080606.903B0A6CE5@lists.varnish-cache.org>


commit bba1499f586dd7d75343d7869497d991ded2607e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 8 08:04:26 2019 +0000

    Extend vrt_backend with a connection prefix ptr+len

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 80134b35d..328e9078b 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -541,6 +541,8 @@ vrt_hash_be(const struct vrt_backend *vrt)
 		VSHA256_Update(cx, vrt->ipv6_suckaddr, vsa_suckaddr_len);
 	if (vrt->path != NULL)
 		VSHA256_Update(cx, vrt->path, strlen(vrt->path));
+	if (vrt->prefix_ptr != NULL)
+		VSHA256_Update(cx, vrt->prefix_ptr, vrt->prefix_len);
 	VSHA256_Final(ident, cx);
 	return (vbe64dec(ident));
 }
diff --git a/include/vrt.h b/include/vrt.h
index f5128fda2..556425298 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -52,7 +52,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * unreleased (planned for 2020-03-15)
- *	(nothing yet)
+ *	New prefix_{ptr|len} fields in vrt_backend
  * 10.0 (2019-09-15)
  *	VRT_UpperLowerStrands added.
  *	VRT_synth_page now takes STRANDS argument
@@ -344,7 +344,9 @@ extern const void * const vrt_magic_string_unset;
 	vtim_dur			first_byte_timeout;	\
 	vtim_dur			between_bytes_timeout;	\
 	unsigned			max_connections;	\
-	unsigned			proxy_header;
+	unsigned			proxy_header;		\
+	void				*prefix_ptr;		\
+	unsigned			prefix_len;
 
 #define VRT_BACKEND_HANDLE()			\
 	do {					\
@@ -363,7 +365,7 @@ extern const void * const vrt_magic_string_unset;
 
 struct vrt_backend {
 	unsigned			magic;
-#define VRT_BACKEND_MAGIC		0x4799ce6b
+#define VRT_BACKEND_MAGIC		0x4799ce6c
 	VRT_BACKEND_FIELDS(const)
 	VCL_IP				ipv4_suckaddr;
 	VCL_IP				ipv6_suckaddr;

From phk at FreeBSD.org  Wed Oct  9 10:33:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  9 Oct 2019 10:33:06 +0000 (UTC)
Subject: [master] 1d6d96060 Add Backend None Documentation
Message-ID: <20191009103306.7C178ACD16@lists.varnish-cache.org>


commit 1d6d96060f1d8434199ff4b1dd738878267806c7
Author: Andrew Wiik <andrew at varnish-software.com>
Date:   Mon Oct 7 20:28:19 2019 -0400

    Add Backend None Documentation

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 1fc2395fb..df8399c61 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -297,6 +297,12 @@ parameters. The following attributes are available:
     Varnish reaches the maximum Varnish it will start failing
     connections.
 
+Empty backends can also be defined using the following syntax.::
+
+  backend name none;
+
+An empty backend will always return status code 503 as if it is sick.
+
 Backends can be used with *directors*. Please see the
 :ref:`vmod_directors(3)` man page for more information.
 
diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 24d0dec7d..d4fe87763 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -31,6 +31,9 @@ Varnish can have several backends defined you can even join
 several backends together into clusters of backends for load balancing
 purposes.
 
+backends can also be empty or 'none' with the following syntax.::
+
+    backend default none;
 
 Multiple backends
 -----------------

From nils.goroll at uplex.de  Wed Oct  9 11:04:05 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed,  9 Oct 2019 11:04:05 +0000 (UTC)
Subject: [master] a0a3dfcff use backend None instead of bad_ip backend
Message-ID: <20191009110406.0388EAD951@lists.varnish-cache.org>


commit a0a3dfcff4390545c02475266a7ae75d9cebc251
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 9 12:52:40 2019 +0200

    use backend None instead of bad_ip backend
    
    ... for those cases which can easily be changed (most of them).
    
    This avoids issues with failing tests when bad_ip is actually reachable
    via tcp on the respective port, as seen for example with b00068.vtc

diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc
index adcec5974..e8cc9ab99 100644
--- a/bin/varnishtest/tests/b00053.vtc
+++ b/bin/varnishtest/tests/b00053.vtc
@@ -38,7 +38,7 @@ varnish v1 -expect s_resp_hdrbytes == 178
 varnish v2 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
 
 varnish v2 -syntax 4.0 -errvcl {Compiled VCL version (4.0) not supported.} {
-	backend default { .host = "${bad_ip}"; }
+	backend default None;
 }
 
 varnish v2 -syntax 4.0 -errvcl \
@@ -46,7 +46,7 @@ varnish v2 -syntax 4.0 -errvcl \
 	{backend default { .path = "${tmpdir}/v1.sock"; }}
 
 varnish v3 -vcl {
-	backend default { .host = "${bad_ip}"; }
+	backend default None;
 
 	sub vcl_recv {
 		return(synth(200));
diff --git a/bin/varnishtest/tests/b00068.vtc b/bin/varnishtest/tests/b00068.vtc
index e42b1a4e7..d5f6910b1 100644
--- a/bin/varnishtest/tests/b00068.vtc
+++ b/bin/varnishtest/tests/b00068.vtc
@@ -15,7 +15,7 @@ varnish v1 -arg "-p timeout_linger=1"	\
 	import vtc;
 	import blob;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_recv {
 		std.log(blob.encode(encoding=HEX,
diff --git a/bin/varnishtest/tests/c00083.vtc b/bin/varnishtest/tests/c00083.vtc
index 6b5ff7e69..78fb14a53 100644
--- a/bin/varnishtest/tests/c00083.vtc
+++ b/bin/varnishtest/tests/c00083.vtc
@@ -1,7 +1,7 @@
 varnishtest "Test VSM _.index rewrite when too many deletes"
 
 varnish v1 -vcl {
-	backend default { .host = "${bad_ip}"; }
+	backend default None;
 } -start
 
 delay 1
@@ -17,10 +17,10 @@ process p1 {
 #process p3 {tail -F ${tmpdir}/v1/_.vsm_mgt/_.index} -dump -start
 
 varnish v1 -vcl {
-	backend b00 { .host = "${bad_ip}"; }
-	backend b01 { .host = "${bad_ip}"; }
-	backend b02 { .host = "${bad_ip}"; }
-	backend b03 { .host = "${bad_ip}"; }
+	backend b00 None;
+	backend b01 None;
+	backend b02 None;
+	backend b03 None;
 
 	sub vcl_recv {
 		set req.backend_hint = b00;
@@ -46,26 +46,26 @@ process p1 -run
 # cause a _.index rewrite.
 # Make it 20 to be on the safe side.
 varnish v1 -vcl {
-	backend b00 { .host = "${bad_ip}"; }
-	backend b01 { .host = "${bad_ip}"; }
-	backend b02 { .host = "${bad_ip}"; }
-	backend b03 { .host = "${bad_ip}"; }
-	backend b04 { .host = "${bad_ip}"; }
-	backend b05 { .host = "${bad_ip}"; }
-	backend b06 { .host = "${bad_ip}"; }
-	backend b07 { .host = "${bad_ip}"; }
-	backend b08 { .host = "${bad_ip}"; }
-	backend b09 { .host = "${bad_ip}"; }
-	backend b10 { .host = "${bad_ip}"; }
-	backend b11 { .host = "${bad_ip}"; }
-	backend b12 { .host = "${bad_ip}"; }
-	backend b13 { .host = "${bad_ip}"; }
-	backend b14 { .host = "${bad_ip}"; }
-	backend b15 { .host = "${bad_ip}"; }
-	backend b16 { .host = "${bad_ip}"; }
-	backend b17 { .host = "${bad_ip}"; }
-	backend b18 { .host = "${bad_ip}"; }
-	backend b19 { .host = "${bad_ip}"; }
+	backend b00 None;
+	backend b01 None;
+	backend b02 None;
+	backend b03 None;
+	backend b04 None;
+	backend b05 None;
+	backend b06 None;
+	backend b07 None;
+	backend b08 None;
+	backend b09 None;
+	backend b10 None;
+	backend b11 None;
+	backend b12 None;
+	backend b13 None;
+	backend b14 None;
+	backend b15 None;
+	backend b16 None;
+	backend b17 None;
+	backend b18 None;
+	backend b19 None;
 
 	sub vcl_recv {
 		set req.backend_hint = b00;
diff --git a/bin/varnishtest/tests/c00086.vtc b/bin/varnishtest/tests/c00086.vtc
index 0e6979ef3..0991ee23f 100644
--- a/bin/varnishtest/tests/c00086.vtc
+++ b/bin/varnishtest/tests/c00086.vtc
@@ -60,7 +60,7 @@ shell { chmod go-rx ${tmpdir}/dir }
 
 varnish v8 \
 	-jail "-junix,user=varnish,ccgroup=varnish,workuser=vcache" \
-	-vcl { backend b {.host="${bad_ip}";}
+	-vcl { backend b None;
 }
 
 varnish v8 -cliexpect "(?s)Cannot stat:.+That was just a warning" \
diff --git a/bin/varnishtest/tests/c00087.vtc b/bin/varnishtest/tests/c00087.vtc
index 443cea561..d84862860 100644
--- a/bin/varnishtest/tests/c00087.vtc
+++ b/bin/varnishtest/tests/c00087.vtc
@@ -69,7 +69,7 @@ logexpect l1 -v v1 -d 1 -g vxid -q "VCL_acl" {
 } -run
 
 varnish v1 -vcl {
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	acl acl1 {
 		"0.0.0.0";
@@ -97,7 +97,7 @@ client c1 -connect "${tmpdir}/v1.sock" {
 } -run
 
 varnish v1 -errvcl {.../mask is not numeric.} {
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	acl acl1 {
 		"${tmpdir}/v1.sock";
diff --git a/bin/varnishtest/tests/d00032.vtc b/bin/varnishtest/tests/d00032.vtc
index 5e307675c..52039132d 100644
--- a/bin/varnishtest/tests/d00032.vtc
+++ b/bin/varnishtest/tests/d00032.vtc
@@ -8,7 +8,7 @@ server s1 -listen "${tmpdir}/s1.sock" {
 varnish v1 -vcl {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${s1_sock}");
@@ -19,7 +19,7 @@ varnish v1 -vcl {
 	}
 } -start
 
-varnish v1 -expect MAIN.n_backend == 2
+varnish v1 -expect MAIN.n_backend == 1
 
 client c1 {
 	txreq
@@ -30,7 +30,7 @@ client c1 {
 varnish v1 -errvcl {path must be an absolute path} {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("");
@@ -40,7 +40,7 @@ varnish v1 -errvcl {path must be an absolute path} {
 varnish v1 -errvcl {path must be an absolute path} {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("s1.sock");
@@ -52,7 +52,7 @@ shell { rm -f ${tmpdir}/foo }
 varnish v1 -errvcl {Cannot stat path} {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${tmpdir}/foo");
@@ -62,7 +62,7 @@ varnish v1 -errvcl {Cannot stat path} {
 varnish v1 -errvcl {is not a socket} {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${tmpdir}");
diff --git a/bin/varnishtest/tests/d00033.vtc b/bin/varnishtest/tests/d00033.vtc
index 1d837aba6..877005ea8 100644
--- a/bin/varnishtest/tests/d00033.vtc
+++ b/bin/varnishtest/tests/d00033.vtc
@@ -15,7 +15,7 @@ server s2 -listen "${tmpdir}/s2.sock" {
 varnish v1 -vcl {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${s1_sock}");
@@ -30,7 +30,7 @@ varnish v1 -vcl {
 	}
 } -start
 
-varnish v1 -expect MAIN.n_backend == 2
+varnish v1 -expect MAIN.n_backend == 1
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/d00034.vtc b/bin/varnishtest/tests/d00034.vtc
index 314673fee..7e99eca11 100644
--- a/bin/varnishtest/tests/d00034.vtc
+++ b/bin/varnishtest/tests/d00034.vtc
@@ -21,7 +21,7 @@ server s2 -listen "${tmpdir}/s2.sock" {
 varnish v1 -vcl {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${s1_sock}");
@@ -36,7 +36,7 @@ varnish v1 -vcl {
 	}
 } -start
 
-varnish v1 -expect MAIN.n_backend == 2
+varnish v1 -expect MAIN.n_backend == 1
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/d00035.vtc b/bin/varnishtest/tests/d00035.vtc
index e47c6703a..338dd5888 100644
--- a/bin/varnishtest/tests/d00035.vtc
+++ b/bin/varnishtest/tests/d00035.vtc
@@ -21,7 +21,7 @@ server s2 -listen "${tmpdir}/s2.sock" {
 varnish v1 -vcl {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${s1_sock}");
@@ -37,7 +37,7 @@ varnish v1 -vcl {
 	}
 } -start
 
-varnish v1 -expect MAIN.n_backend == 2
+varnish v1 -expect MAIN.n_backend == 1
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/d00036.vtc b/bin/varnishtest/tests/d00036.vtc
index 3b739c755..b75278b44 100644
--- a/bin/varnishtest/tests/d00036.vtc
+++ b/bin/varnishtest/tests/d00036.vtc
@@ -15,7 +15,7 @@ varnish v1 -vcl {
 	import debug;
 	import vtc;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${s1_sock}");
@@ -40,7 +40,7 @@ varnish v1 -vcl {
 	}
 } -start
 
-varnish v1 -expect MAIN.n_backend == 2
+varnish v1 -expect MAIN.n_backend == 1
 
 client c1 {
 	txreq
diff --git a/bin/varnishtest/tests/d00037.vtc b/bin/varnishtest/tests/d00037.vtc
index 51c6e8f84..66c9b7258 100644
--- a/bin/varnishtest/tests/d00037.vtc
+++ b/bin/varnishtest/tests/d00037.vtc
@@ -14,7 +14,7 @@ server s1 -listen "${tmpdir}/s1.sock" {
 varnish v1 -arg "-p thread_pools=1" -vcl {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${s1_sock}");
@@ -31,7 +31,7 @@ client c1 {
 	expect resp.status == 200
 } -start
 
-varnish v1 -expect MAIN.n_backend == 2
+varnish v1 -expect MAIN.n_backend == 1
 
 server s2 -listen "${tmpdir}/s2.sock" {
 	rxreq
@@ -44,7 +44,7 @@ barrier b1 sync
 varnish v1 -vcl {
 	import debug;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s2 = debug.dyn_uds("${s2_sock}");
@@ -61,7 +61,7 @@ barrier b2 sync
 client c1 -wait
 delay 2
 
-varnish v1 -expect MAIN.n_backend == 4
+varnish v1 -expect MAIN.n_backend == 2
 
 varnish v1 -expect n_vcl_avail == 1
 varnish v1 -expect n_vcl_discard == 1
diff --git a/bin/varnishtest/tests/d00038.vtc b/bin/varnishtest/tests/d00038.vtc
index 1b8b70405..c3ce7cbb9 100644
--- a/bin/varnishtest/tests/d00038.vtc
+++ b/bin/varnishtest/tests/d00038.vtc
@@ -15,7 +15,7 @@ varnish v1 -vcl {
 	import debug;
 	import vtc;
 
-	backend dummy { .host = "${bad_ip}"; }
+	backend dummy None;
 
 	sub vcl_init {
 		new s1 = debug.dyn_uds("${s1_sock}");
@@ -37,7 +37,7 @@ varnish v1 -vcl {
 	}
 } -start
 
-varnish v1 -expect MAIN.n_backend == 2
+varnish v1 -expect MAIN.n_backend == 1
 
 client c1 {
 	txreq
diff --git a/bin/varnishtest/tests/m00033.vtc b/bin/varnishtest/tests/m00033.vtc
index 64ab20850..fc5d4ddde 100644
--- a/bin/varnishtest/tests/m00033.vtc
+++ b/bin/varnishtest/tests/m00033.vtc
@@ -2,7 +2,7 @@ varnishtest "VMOD blob IDENTITY encode and decode"
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00034.vtc b/bin/varnishtest/tests/m00034.vtc
index 9d7144e0b..ef094124e 100644
--- a/bin/varnishtest/tests/m00034.vtc
+++ b/bin/varnishtest/tests/m00034.vtc
@@ -2,7 +2,7 @@ varnishtest "VMOD blob IDENTITY decode() n chars"
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00035.vtc b/bin/varnishtest/tests/m00035.vtc
index 9f7df333c..0af6184f0 100644
--- a/bin/varnishtest/tests/m00035.vtc
+++ b/bin/varnishtest/tests/m00035.vtc
@@ -4,7 +4,7 @@ varnishtest "VMOD blob hex encode and decode"
 varnish v1 -arg "-p workspace_client=256k" -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00036.vtc b/bin/varnishtest/tests/m00036.vtc
index 12e1c24cc..ee3ea5246 100644
--- a/bin/varnishtest/tests/m00036.vtc
+++ b/bin/varnishtest/tests/m00036.vtc
@@ -3,7 +3,7 @@ varnishtest "VMOD blob hex decode() n chars"
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00037.vtc b/bin/varnishtest/tests/m00037.vtc
index e545f1c92..3d463ec0e 100644
--- a/bin/varnishtest/tests/m00037.vtc
+++ b/bin/varnishtest/tests/m00037.vtc
@@ -2,7 +2,7 @@ varnishtest "VMOD blob base64 encode and decode"
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
@@ -126,7 +126,7 @@ client c1 {
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00038.vtc b/bin/varnishtest/tests/m00038.vtc
index a5859817d..31a729ea9 100644
--- a/bin/varnishtest/tests/m00038.vtc
+++ b/bin/varnishtest/tests/m00038.vtc
@@ -2,7 +2,7 @@ varnishtest "VMOD blob base64 decode() n chars"
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00039.vtc b/bin/varnishtest/tests/m00039.vtc
index bfcd67ead..413152b6f 100644
--- a/bin/varnishtest/tests/m00039.vtc
+++ b/bin/varnishtest/tests/m00039.vtc
@@ -3,7 +3,7 @@ varnishtest "VMOD blob url encode and decode"
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
@@ -143,7 +143,7 @@ client c1 {
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00040.vtc b/bin/varnishtest/tests/m00040.vtc
index 2177f8a73..35e4c18a1 100644
--- a/bin/varnishtest/tests/m00040.vtc
+++ b/bin/varnishtest/tests/m00040.vtc
@@ -3,7 +3,7 @@ varnishtest "VMOD blob decode() n chars with URL"
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00041.vtc b/bin/varnishtest/tests/m00041.vtc
index aeda981f9..15e2e3c72 100644
--- a/bin/varnishtest/tests/m00041.vtc
+++ b/bin/varnishtest/tests/m00041.vtc
@@ -4,7 +4,7 @@ varnishtest "VMOD blob test transcode()"
 varnish v1 -arg "-p workspace_client=256k" -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
@@ -129,7 +129,7 @@ client c1 {
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
@@ -274,7 +274,7 @@ client c1 {
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00042.vtc b/bin/varnishtest/tests/m00042.vtc
index 6e6004f63..a03e46a3d 100644
--- a/bin/varnishtest/tests/m00042.vtc
+++ b/bin/varnishtest/tests/m00042.vtc
@@ -3,7 +3,7 @@ varnishtest "VMOD blob test transcode() n chars"
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
@@ -134,7 +134,7 @@ client c1 {
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
@@ -270,7 +270,7 @@ client c1 {
 varnish v1 -vcl {
 	import blob;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 	  return(synth(200));
diff --git a/bin/varnishtest/tests/m00043.vtc b/bin/varnishtest/tests/m00043.vtc
index 9062314ef..e58cfbdc3 100644
--- a/bin/varnishtest/tests/m00043.vtc
+++ b/bin/varnishtest/tests/m00043.vtc
@@ -2,7 +2,7 @@ varnishtest "VMOD blob blob object interface"
 
 varnish v1 -arg "-i serverid" -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 	  new id = blob.blob(IDENTITY,
@@ -94,7 +94,7 @@ client c1 -run
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 	  new idempty = blob.blob(IDENTITY, "");
@@ -167,7 +167,7 @@ client c1 -run
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 	  new b64 = blob.blob(BASE64, "L0hlbGxvIHdvcmxkLw==");
@@ -257,7 +257,7 @@ client c2 -run
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 	  new id = blob.blob(IDENTITY,
@@ -455,7 +455,7 @@ logexpect l1 -wait
 
 varnish v1 -errvcl {vmod blob error: cannot create blob err, illegal encoding beginning with "g"} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new err = blob.blob(HEX, "g");
 	}
@@ -463,7 +463,7 @@ varnish v1 -errvcl {vmod blob error: cannot create blob err, illegal encoding be
 
 varnish v1 -errvcl {vmod blob error: cannot create blob bad64, illegal encoding beginning with "-_-_"} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new bad64 = blob.blob(BASE64, "-_-_");
 	}
@@ -471,7 +471,7 @@ varnish v1 -errvcl {vmod blob error: cannot create blob bad64, illegal encoding
 
 varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding beginning with "+/+/"} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new badurl = blob.blob(BASE64URL, "+/+/");
 	}
@@ -479,7 +479,7 @@ varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding
 
 varnish v1 -errvcl {vmod blob error: cannot create blob badpad, illegal encoding beginning with "YWI="} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new badpad = blob.blob(BASE64URLNOPAD, "YWI=");
 	}
@@ -487,7 +487,7 @@ varnish v1 -errvcl {vmod blob error: cannot create blob badpad, illegal encoding
 
 varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding beginning with "%"} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new badurl = blob.blob(URL, "%");
 	}
@@ -495,7 +495,7 @@ varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding
 
 varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding beginning with "%2"} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new badurl = blob.blob(URL, "%2");
 	}
@@ -503,7 +503,7 @@ varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding
 
 varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding beginning with "%q"} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new badurl = blob.blob(URL, "%q");
 	}
@@ -511,7 +511,7 @@ varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding
 
 varnish v1 -errvcl {vmod blob error: cannot create blob badurl, illegal encoding beginning with "%2q"} {
 	import blob;
-	backend b { .host="${bad_ip}"; }
+	backend b None;
 	sub vcl_init {
 	  new badurl = blob.blob(URL, "%2q");
 	}
diff --git a/bin/varnishtest/tests/m00045.vtc b/bin/varnishtest/tests/m00045.vtc
index 010671179..1bf37fd94 100644
--- a/bin/varnishtest/tests/m00045.vtc
+++ b/bin/varnishtest/tests/m00045.vtc
@@ -2,7 +2,7 @@ varnishtest "VMOD blob same(), equal(), length() and sub()"
 
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		new foo1 = blob.blob(IDENTITY, "foobarbazquux");
@@ -168,7 +168,7 @@ client c1 {
 # sub()
 varnish v1 -vcl {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		# Byte values 0 up to 7
@@ -291,7 +291,7 @@ logexpect l1 -v v1 -d 1 -g vxid -q "VCL_Error" {
 # VCL load failures from sub()
 varnish v1 -errvcl {vmod blob error: blob is empty in blob.sub()} {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		new empty = blob.blob(IDENTITY, "");
@@ -303,7 +303,7 @@ varnish v1 -errvcl {vmod blob error: blob is empty in blob.sub()} {
 
 varnish v1 -errvcl {vmod blob error: size 9 from offset 0 requires more bytes than blob length 8 in blob.sub()} {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		new up07 = blob.blob(BASE64, "AAECAwQFBgc=");
@@ -315,7 +315,7 @@ varnish v1 -errvcl {vmod blob error: size 9 from offset 0 requires more bytes th
 
 varnish v1 -errvcl {vmod blob error: size 4 from offset 5 requires more bytes than blob length 8 in blob.sub()} {
 	import blob;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		new up07 = blob.blob(BASE64, "AAECAwQFBgc=");
diff --git a/bin/varnishtest/tests/m00046.vtc b/bin/varnishtest/tests/m00046.vtc
index dca4fba25..8d2e2fd61 100644
--- a/bin/varnishtest/tests/m00046.vtc
+++ b/bin/varnishtest/tests/m00046.vtc
@@ -4,7 +4,7 @@ varnish v1 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
 
 varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl {
 	import std;
-	backend default { .host = "${bad_ip}"; }
+	backend default None;
 
 	sub vcl_recv {
 		return(synth(200));
@@ -31,7 +31,7 @@ varnish v2 -errvcl {Cannot convert to an IP address: '"${v1_addr}"'} {
 
 varnish v2 -vcl {
 	import std;
-	backend default { .host = "${bad_ip}"; }
+	backend default None;
 
 	sub vcl_recv {
 	    # Silently ignored for a UDS
diff --git a/bin/varnishtest/tests/m00047.vtc b/bin/varnishtest/tests/m00047.vtc
index 3eb3857b4..e9c05b032 100644
--- a/bin/varnishtest/tests/m00047.vtc
+++ b/bin/varnishtest/tests/m00047.vtc
@@ -61,7 +61,7 @@ logexpect l1 -v v1 -d 1 -c {
 varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} {
 	import unix;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		std.log(unix.uid());
@@ -71,7 +71,7 @@ varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini
 varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} {
 	import unix;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		std.log(unix.gid());
@@ -81,7 +81,7 @@ varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini
 varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} {
 	import unix;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		std.log(unix.user());
@@ -91,7 +91,7 @@ varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini
 varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} {
 	import unix;
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		std.log(unix.group());
diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc
index 17ea49e84..b92968dea 100644
--- a/bin/varnishtest/tests/m00049.vtc
+++ b/bin/varnishtest/tests/m00049.vtc
@@ -3,7 +3,7 @@ varnishtest "VMOD blob workspace overflow conditions"
 varnish v1 -vcl {
 	import blob;
 	import vtc;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 		if (req.url == "/1") {
@@ -72,7 +72,7 @@ client c4 {
 varnish v1 -vcl {
 	import blob;
 	import vtc;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 		if (req.url == "/1") {
@@ -104,7 +104,7 @@ client c3 -run
 varnish v1 -vcl {
 	import blob;
 	import vtc;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 		if (req.url == "/1") {
@@ -135,7 +135,7 @@ client c3 -run
 varnish v1 -vcl {
 	import blob;
 	import vtc;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 		if (req.url == "/1") {
@@ -166,7 +166,7 @@ client c3 -run
 varnish v1 -vcl {
 	import blob;
 	import vtc;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_miss {
 		if (req.url == "/1") {
diff --git a/bin/varnishtest/tests/m00050.vtc b/bin/varnishtest/tests/m00050.vtc
index a6578a6bd..c7192be4f 100644
--- a/bin/varnishtest/tests/m00050.vtc
+++ b/bin/varnishtest/tests/m00050.vtc
@@ -2,7 +2,7 @@ varnishtest "std.fnmatch()"
 
 varnish v1 -vcl {
 	import std;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 		return (synth(200));
diff --git a/bin/varnishtest/tests/r02418.vtc b/bin/varnishtest/tests/r02418.vtc
index a03ca546c..01e8feee0 100644
--- a/bin/varnishtest/tests/r02418.vtc
+++ b/bin/varnishtest/tests/r02418.vtc
@@ -15,7 +15,7 @@ varnish v1 -cliok "param.set feature +http2"
 varnish v1 -cliok "param.set feature +no_coredump"
 
 varnish v1 -vcl {
-	backend b1 { .host = "${bad_ip}"; }
+	backend b1 None;
 	sub vcl_recv {
 		return (synth(200));
 	}
diff --git a/bin/varnishtest/tests/r02880.vtc b/bin/varnishtest/tests/r02880.vtc
index 1d554230b..9a33ac6b8 100644
--- a/bin/varnishtest/tests/r02880.vtc
+++ b/bin/varnishtest/tests/r02880.vtc
@@ -2,7 +2,7 @@ varnishtest "Long VMOD object names"
 
 varnish v1 -vcl {
 	import debug;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		new l234567890123456789012345678901234567890123456789012345678
diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc
index d408896b8..6662eefb5 100644
--- a/bin/varnishtest/tests/v00038.vtc
+++ b/bin/varnishtest/tests/v00038.vtc
@@ -132,7 +132,7 @@ varnish v1 -errvcl "Not a socket:" {
 # VCC warns, but does not fail, if stat(UDS) fails with ENOENT.
 shell { rm -f ${tmpdir}/foo }
 
-varnish v1 -vcl { backend b {.host="${bad_ip}";} }
+varnish v1 -vcl { backend b None; }
 
 varnish v1 -cliexpect "(?s)Cannot stat:.+That was just a warning" \
 	{vcl.inline test "vcl 4.1; backend b {.path=\"${tmpdir}/foo\";}"}
diff --git a/bin/varnishtest/tests/v00054.vtc b/bin/varnishtest/tests/v00054.vtc
index c92e8cce6..f298c4fb1 100644
--- a/bin/varnishtest/tests/v00054.vtc
+++ b/bin/varnishtest/tests/v00054.vtc
@@ -35,7 +35,7 @@ client c1 -connect "${tmpdir}/v2.sock" {
 } -run
 
 varnish v3 -arg "-a ${tmpdir}/v3.sock" -vcl {
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_recv {
 		if (req.url == "/nobody") {
diff --git a/bin/varnishtest/tests/v00058.vtc b/bin/varnishtest/tests/v00058.vtc
index dda187f6d..2aa5ee2b2 100644
--- a/bin/varnishtest/tests/v00058.vtc
+++ b/bin/varnishtest/tests/v00058.vtc
@@ -2,7 +2,7 @@ varnishtest "Test VRT STRANDS functions"
 
 varnish v1 -arg "-i foobar" -vcl {
 	import debug;
-	backend b { .host = "${bad_ip}"; }
+	backend b None;
 
 	sub vcl_init {
 		# tests VRT_Strands()

From nils.goroll at uplex.de  Wed Oct  9 11:14:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed,  9 Oct 2019 11:14:06 +0000 (UTC)
Subject: [master] 5f039a4a4 revert backend None cases where I acted
 over-zealously
Message-ID: <20191009111406.AFE0AADE83@lists.varnish-cache.org>


commit 5f039a4a464949cb378fe49a8101c273b84cb92e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 9 13:09:39 2019 +0200

    revert backend None cases where I acted over-zealously
    
    backend None does not actually create a backend, so tests which require
    one need to use the bad_ip method.

diff --git a/bin/varnishtest/tests/c00083.vtc b/bin/varnishtest/tests/c00083.vtc
index 78fb14a53..d991201be 100644
--- a/bin/varnishtest/tests/c00083.vtc
+++ b/bin/varnishtest/tests/c00083.vtc
@@ -1,7 +1,7 @@
 varnishtest "Test VSM _.index rewrite when too many deletes"
 
 varnish v1 -vcl {
-	backend default None;
+	backend default { .host = "${bad_ip}"; }
 } -start
 
 delay 1
@@ -17,10 +17,10 @@ process p1 {
 #process p3 {tail -F ${tmpdir}/v1/_.vsm_mgt/_.index} -dump -start
 
 varnish v1 -vcl {
-	backend b00 None;
-	backend b01 None;
-	backend b02 None;
-	backend b03 None;
+	backend b00 { .host = "${bad_ip}"; }
+	backend b01 { .host = "${bad_ip}"; }
+	backend b02 { .host = "${bad_ip}"; }
+	backend b03 { .host = "${bad_ip}"; }
 
 	sub vcl_recv {
 		set req.backend_hint = b00;
@@ -46,26 +46,26 @@ process p1 -run
 # cause a _.index rewrite.
 # Make it 20 to be on the safe side.
 varnish v1 -vcl {
-	backend b00 None;
-	backend b01 None;
-	backend b02 None;
-	backend b03 None;
-	backend b04 None;
-	backend b05 None;
-	backend b06 None;
-	backend b07 None;
-	backend b08 None;
-	backend b09 None;
-	backend b10 None;
-	backend b11 None;
-	backend b12 None;
-	backend b13 None;
-	backend b14 None;
-	backend b15 None;
-	backend b16 None;
-	backend b17 None;
-	backend b18 None;
-	backend b19 None;
+	backend b00 { .host = "${bad_ip}"; }
+	backend b01 { .host = "${bad_ip}"; }
+	backend b02 { .host = "${bad_ip}"; }
+	backend b03 { .host = "${bad_ip}"; }
+	backend b04 { .host = "${bad_ip}"; }
+	backend b05 { .host = "${bad_ip}"; }
+	backend b06 { .host = "${bad_ip}"; }
+	backend b07 { .host = "${bad_ip}"; }
+	backend b08 { .host = "${bad_ip}"; }
+	backend b09 { .host = "${bad_ip}"; }
+	backend b10 { .host = "${bad_ip}"; }
+	backend b11 { .host = "${bad_ip}"; }
+	backend b12 { .host = "${bad_ip}"; }
+	backend b13 { .host = "${bad_ip}"; }
+	backend b14 { .host = "${bad_ip}"; }
+	backend b15 { .host = "${bad_ip}"; }
+	backend b16 { .host = "${bad_ip}"; }
+	backend b17 { .host = "${bad_ip}"; }
+	backend b18 { .host = "${bad_ip}"; }
+	backend b19 { .host = "${bad_ip}"; }
 
 	sub vcl_recv {
 		set req.backend_hint = b00;
@@ -92,6 +92,7 @@ varnish v1 -vcl {
 }
 
 varnish v1 -cliok vcl.list
+varnish v1 -cliok backend.list
 
 delay 1
 

From dridi.boukelmoune at gmail.com  Wed Oct  9 15:23:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed,  9 Oct 2019 15:23:06 +0000 (UTC)
Subject: [master] a791188e2 genhufdec: Free top table before exit
Message-ID: <20191009152306.55F0FB2CAE@lists.varnish-cache.org>


commit a791188e2e8b7782d81d93b465f32cf53e0509f9
Author: Alf-Andr? Walla <fwsgonzo at hotmail.com>
Date:   Wed Oct 9 11:13:33 2019 +0200

    genhufdec: Free top table before exit

diff --git a/bin/varnishd/hpack/vhp_gen_hufdec.c b/bin/varnishd/hpack/vhp_gen_hufdec.c
index 5f864402a..da0593a94 100644
--- a/bin/varnishd/hpack/vhp_gen_hufdec.c
+++ b/bin/varnishd/hpack/vhp_gen_hufdec.c
@@ -92,6 +92,16 @@ tbl_new(unsigned mask)
 	return (tbl);
 }
 
+static void
+tbl_free(struct tbl* table)
+{
+	for (unsigned i = 0; i < table->n; i++) {
+		if (table->e[i].next != NULL)
+			tbl_free(table->e[i].next);
+	}
+	free(table);
+}
+
 static void
 tbl_add(struct tbl *tbl, uint32_t code, unsigned codelen,
     uint32_t bits, unsigned len, char chr)
@@ -250,5 +260,6 @@ main(int argc, const char **argv)
 	tbl_print(top);
 	printf("};\n");
 
+	tbl_free(top);
 	return (0);
 }

From dridi.boukelmoune at gmail.com  Wed Oct  9 15:23:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed,  9 Oct 2019 15:23:06 +0000 (UTC)
Subject: [master] 8a5a9ab6e GC stale lsan suppression
Message-ID: <20191009152306.6D939B2CB1@lists.varnish-cache.org>


commit 8a5a9ab6e7b853c50616baa5f8ebb7d28947f22f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 9 17:22:04 2019 +0200

    GC stale lsan suppression
    
    Refs a791188e2e8b7782d81d93b465f32cf53e0509f9

diff --git a/tools/lsan.suppr b/tools/lsan.suppr
index b6d916a30..03abb5f9c 100644
--- a/tools/lsan.suppr
+++ b/tools/lsan.suppr
@@ -13,8 +13,6 @@ leak:vcc_
 leak:VSL_Setup
 leak:WRK_BgThread
 #
-leak:hpack/vhp_gen_hufdec.c
-#
 leak:binheap_new
 # ev
 leak:mct_callback

From nils.goroll at uplex.de  Wed Oct  9 17:52:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed,  9 Oct 2019 17:52:06 +0000 (UTC)
Subject: [master] 1f226e6d8 Add a fixed-buffer strcpy analogous to bprintf
Message-ID: <20191009175206.8440C6093D@lists.varnish-cache.org>


commit 1f226e6d8c1c5443d9df127797c62b1e1228bfd1
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 9 13:18:48 2019 +0200

    Add a fixed-buffer strcpy analogous to bprintf

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 68e61524e..0b7c5cede 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -501,8 +501,8 @@ vca_accept_task(struct worker *wrk, void *arg)
 			wrk->stats->sess_fail++;
 
 			if (wa.acceptlsock->uds) {
-				strcpy(laddr, "0.0.0.0");
-				strcpy(lport, "0");
+				bstrcpy(laddr, "0.0.0.0");
+				bstrcpy(lport, "0");
 			} else {
 				VTCP_myname(ls->sock, laddr, VTCP_ADDRBUFSIZE,
 				    lport, VTCP_PORTBUFSIZE);
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index a8e035ccf..7b1b6d8f8 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -104,8 +104,7 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 	*ws->e = 0x15;
 	ws->f = ws->s;
 	assert(id[0] & 0x20);
-	assert(strlen(id) < sizeof ws->id);
-	strcpy(ws->id, id);
+	bstrcpy(ws->id, id);
 	WS_Assert(ws);
 }
 
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index 9f8d202af..32144f374 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -278,7 +278,6 @@ smp_newsilo(struct smp_sc *sc)
 	struct smp_ident	*si;
 
 	ASSERT_MGT();
-	assert(strlen(SMP_IDENT_STRING) < sizeof si->ident);
 
 	/* Choose a new random number */
 	AZ(VRND_RandomCrypto(&sc->unique, sizeof sc->unique));
@@ -287,7 +286,7 @@ smp_newsilo(struct smp_sc *sc)
 	si = sc->ident;
 
 	memset(si, 0, sizeof *si);
-	strcpy(si->ident, SMP_IDENT_STRING);
+	bstrcpy(si->ident, SMP_IDENT_STRING);
 	si->byte_order = 0x12345678;
 	si->size = sizeof *si;
 	si->major_version = 2;
diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c
index ace90356e..5ce3ce3d7 100644
--- a/bin/varnishd/storage/storage_umem.c
+++ b/bin/varnishd/storage/storage_umem.c
@@ -406,7 +406,7 @@ smu_open(struct stevedore *st)
 
 	smu_open_init();
 
-	AN(strcpy(ident, st->ident));
+	bstrcpy(ident, st->ident);
 	smu_sc->smu_cache = umem_cache_createf(ident,
 					  sizeof(struct smu),
 					  0,		// align
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 160e165d7..3ff49705e 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -343,7 +343,7 @@ server_dispatch_thread(void *priv)
 		vtc_log(vl, 3, "dispatch fd %d -> %s", fd, snbuf);
 		s2 = server_new(snbuf, vl);
 		s2->spec = s->spec;
-		strcpy(s2->listen, s->listen);
+		bstrcpy(s2->listen, s->listen);
 		s2->fd = fd;
 		s2->run = 1;
 		AZ(pthread_create(&s2->tp, NULL, server_dispatch_wrk, s2));
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 010c2d915..f0d6d7bdc 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -508,7 +508,7 @@ varnish_launch(struct varnish *v)
 	assert(nfd >= 0);
 
 	assert(sizeof abuf >= CLI_AUTH_RESPONSE_LEN + 7);
-	strcpy(abuf, "auth ");
+	bstrcpy(abuf, "auth ");
 	VCLI_AuthResponse(nfd, r, abuf + 5);
 	closefd(&nfd);
 	free(r);
diff --git a/include/vdef.h b/include/vdef.h
index 2c78fa041..dc622e273 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -56,6 +56,16 @@
 		assert(ivbprintf >= 0 && ivbprintf < (int)sizeof buf);	\
 	} while (0)
 
+/* Safe strcpy into a fixed-size buffer */
+#define bstrcpy(dst, src)						\
+	do {								\
+		size_t lbstrcpy = strlen(src) + 1;			\
+		assert(lbstrcpy <= sizeof dst);				\
+		memcpy(dst, src, lbstrcpy);				\
+	} while (0)
+
+// TODO #define strcpy BANNED
+
 /* Close and discard filedescriptor */
 #define closefd(fdp)				\
 	do {					\

From nils.goroll at uplex.de  Fri Oct 11 10:27:07 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 11 Oct 2019 10:27:07 +0000 (UTC)
Subject: [master] 6c510ef79 signal handling for varnishstat in curses mode
Message-ID: <20191011102707.A21F0963E3@lists.varnish-cache.org>


commit 6c510ef79dd684d23c136d065b40ee6f355cd595
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 11 11:13:56 2019 +0200

    signal handling for varnishstat in curses mode
    
    Fixes #3088 for varnishstat
    
    note that besides handling SIGHUP, this also adds handling of SIGTERM
    and SIGINT, which were also missing.

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 1038f0200..a97fe0c62 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -45,6 +45,7 @@
 #include "miniobj.h"
 #include "vqueue.h"
 #include "vtim.h"
+#include "vapi/vsig.h"
 
 #include "varnishstat.h"
 #include "vcurses.h"
@@ -1032,7 +1033,7 @@ do_curses(struct vsm *vsm, struct vsc *vsc)
 	build_pt_array();
 	init_hitrate();
 
-	while (keep_running) {
+	while (keep_running && !VSIG_int && !VSIG_term && !VSIG_hup) {
 		(void)VSC_Iter(vsc, vsm, NULL, NULL);
 		vsm_status = VSM_Status(vsm);
 		if (vsm_status & (VSM_MGT_RESTARTED|VSM_WRK_RESTARTED))

From nils.goroll at uplex.de  Fri Oct 11 10:27:07 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 11 Oct 2019 10:27:07 +0000 (UTC)
Subject: [master] 475798715 fix signal handling in varnishtop and varnishhist
Message-ID: <20191011102707.B675B963E6@lists.varnish-cache.org>


commit 475798715cf4f3ce3ef0d129447cf2f342984652
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 11 12:06:10 2019 +0200

    fix signal handling in varnishtop and varnishhist
    
    Using vut->last_sighup was not adequate, because the VUT_Main loop might
    have terminated for some other signal or because of EOF.
    
    On the other hand, we do not want to end the curses loop just because
    the VUT_Main loop has ended in order to continue to display the last
    state (and maybe the EOF in the top right corner).
    
    So while I see that checking just one flag has some beauty to it, I do
    think that directly checking the signal counters is both simple and
    robust.
    
    Fixes #3088 for varnishtop and varnishhist

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 6d441ca6f..8f5dd795d 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -53,6 +53,7 @@
 #include "vas.h"
 #include "vut.h"
 #include "vtim.h"
+#include "vapi/vsig.h"
 
 #define HIST_N 2000		/* how far back we remember */
 #define HIST_RES 100		/* bucket resolution */
@@ -131,8 +132,6 @@ static const struct profile profiles[] = {
 
 static const struct profile *active_profile;
 
-static volatile sig_atomic_t quit = 0;
-
 static void
 update(void)
 {
@@ -384,7 +383,7 @@ do_curses(void *arg)
 	intrflush(stdscr, FALSE);
 	curs_set(0);
 	erase();
-	while (!quit && !vut->last_sighup) {
+	while (!VSIG_int && !VSIG_term && !VSIG_hup) {
 		AZ(pthread_mutex_lock(&mtx));
 		update();
 		AZ(pthread_mutex_unlock(&mtx));
@@ -413,7 +412,6 @@ do_curses(void *arg)
 		case 'Q':
 		case 'q':
 			AZ(raise(SIGINT));
-			quit = 1;
 			break;
 		case '0':
 		case '1':
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 3bfb9ec56..070b71bd7 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -54,6 +54,7 @@
 #include "vas.h"
 #include "vtree.h"
 #include "vut.h"
+#include "vapi/vsig.h"
 
 #if 0
 #define AC(x) assert((x) != ERR)
@@ -82,8 +83,6 @@ static int f_flag = 0;
 static unsigned maxfieldlen = 0;
 static const char *ident;
 
-static volatile sig_atomic_t quit = 0;
-
 static VRBT_HEAD(t_order, top) h_order = VRBT_INITIALIZER(&h_order);
 static VRBT_HEAD(t_key, top) h_key = VRBT_INITIALIZER(&h_key);
 
@@ -257,7 +256,7 @@ do_curses(void *arg)
 	(void)curs_set(0);
 	AC(erase());
 	timeout(1000);
-	while (!quit && !vut->last_sighup) {
+	while (!VSIG_int && !VSIG_term && !VSIG_hup) {
 		AZ(pthread_mutex_lock(&mtx));
 		update(period);
 		AZ(pthread_mutex_unlock(&mtx));
@@ -284,7 +283,6 @@ do_curses(void *arg)
 		case 'Q':
 		case 'q':
 			AZ(raise(SIGINT));
-			quit = 1;
 			break;
 		default:
 			AC(beep());

From nils.goroll at uplex.de  Fri Oct 11 10:27:07 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 11 Oct 2019 10:27:07 +0000 (UTC)
Subject: [master] 5362e7295 test that varnish{top, stat,
 hist} properly handle signals
Message-ID: <20191011102707.CEAB1963EB@lists.varnish-cache.org>


commit 5362e7295d191107dbbcee8dfc333b49670e7cd8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 11 12:21:03 2019 +0200

    test that varnish{top,stat,hist} properly handle signals
    
    Tests #3088

diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc
index 9953fccad..86503c972 100644
--- a/bin/varnishtest/tests/u00008.vtc
+++ b/bin/varnishtest/tests/u00008.vtc
@@ -12,6 +12,8 @@ varnish v1 -vcl+backend {
 } -start
 
 process p1 -dump {varnishstat -n ${v1_name}} -start
+process p2 -dump {varnishstat -n ${v1_name}} -start
+process p3 -dump {varnishstat -n ${v1_name}} -start
 
 process p1 -expect-text 0 0 "VBE.vcl1.s1.happy"
 process p1 -screen_dump
@@ -46,4 +48,5 @@ process p1 -expect-text 4 124 "AVG_1000"
 process p1 -expect-text 22 108 "UNSEEN  DIAG"
 
 process p1 -screen_dump -write {q} -wait
-
+process p2 -screen_dump -kill TERM -wait
+process p3 -screen_dump -kill HUP -wait
diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index 3f62e34ab..07743d128 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -52,4 +52,4 @@ process p1 -writehex 0c -need-bytes +10
 
 process p1 -screen_dump -write {q} -wait
 process p2 -screen_dump -kill HUP -wait
-process p3 -screen_dump -write {q} -wait
+process p3 -screen_dump -kill TERM -wait
diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index 9b92f0774..b4f255191 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -8,6 +8,8 @@ server s1 {
 varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishtop -n ${v1_name}} -start
+process p2 -dump {varnishtop -n ${v1_name}} -start
+process p3 -dump {varnishtop -n ${v1_name}} -start
 
 process p1 -expect-text 1 1 {list length}
 
@@ -35,3 +37,5 @@ process p1 -winsz 30 80
 process p1 -need-bytes +1
 
 process p1 -screen_dump -write {q} -wait
+process p2 -screen_dump -kill TERM -wait
+process p3 -screen_dump -kill HUP {q} -wait

From nils.goroll at uplex.de  Fri Oct 11 10:30:07 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 11 Oct 2019 10:30:07 +0000 (UTC)
Subject: [master] 667025750 fix glitch from previous
Message-ID: <20191011103007.1E7E59694E@lists.varnish-cache.org>


commit 667025750af1447e68886d00741f283b564aa3e4
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 11 12:29:06 2019 +0200

    fix glitch from previous
    
    Ref #3088
    
    apparently the additional argument was just silently tolerated by
    varnishtest

diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index b4f255191..6490b4e93 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -38,4 +38,4 @@ process p1 -need-bytes +1
 
 process p1 -screen_dump -write {q} -wait
 process p2 -screen_dump -kill TERM -wait
-process p3 -screen_dump -kill HUP {q} -wait
+process p3 -screen_dump -kill HUP -wait

From martin at varnish-software.com  Fri Oct 11 14:59:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 11 Oct 2019 14:59:06 +0000 (UTC)
Subject: [master] d22dbc0cf Fix error message on chunked insufficient bytes
Message-ID: <20191011145906.429E7A27F0@lists.varnish-cache.org>


commit d22dbc0cfc3ef1a00a5ffe8269fbccda442297c4
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Oct 11 16:55:42 2019 +0200

    Fix error message on chunked insufficient bytes
    
    Failure to read the right number of bytes (typically due to remote HUP)
    would log "straight insufficient bytes", which is the same as we log for
    C-L based fetches. Change this to "chunked insufficient bytes". (Assuming
    this was a copy-paste error).

diff --git a/bin/varnishd/http1/cache_http1_vfp.c b/bin/varnishd/http1/cache_http1_vfp.c
index 4171c6151..a6cda3caf 100644
--- a/bin/varnishd/http1/cache_http1_vfp.c
+++ b/bin/varnishd/http1/cache_http1_vfp.c
@@ -161,7 +161,7 @@ v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr,
 			l = vfe->priv2;
 		lr = v1f_read(vc, htc, ptr, l);
 		if (lr <= 0)
-			return (VFP_Error(vc, "straight insufficient bytes"));
+			return (VFP_Error(vc, "chunked insufficient bytes"));
 		*lp = lr;
 		vfe->priv2 -= lr;
 		if (vfe->priv2 == 0)

From guillaume at varnish-software.com  Fri Oct 11 17:11:06 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 11 Oct 2019 17:11:06 +0000 (UTC)
Subject: [master] e1321f36a [cci] don't distcheck centos:7 for the moment
Message-ID: <20191011171106.1E99EA5737@lists.varnish-cache.org>


commit e1321f36a1a5a258bce4c385c88ff6cb5014b031
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Fri Oct 11 10:08:30 2019 -0700

    [cci] don't distcheck centos:7 for the moment

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e1e9b7c7a..b419b8610 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -380,12 +380,12 @@ workflows:
       - push_packages:
           requires:
             - hold
-      - distcheck:
-          name: distcheck_centos_7
-          dist: centos
-          release: "7"
-          requires:
-            - dist_ubuntu
+#      - distcheck:
+#          name: distcheck_centos_7
+#          dist: centos
+#          release: "7"
+#          requires:
+#            - dist_ubuntu
       - distcheck:
           name: distcheck_debian_buster
           dist: debian

From guillaume at varnish-software.com  Fri Oct 11 17:32:05 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 11 Oct 2019 17:32:05 +0000 (UTC)
Subject: [master] 76c7d40b4 [cci] VERBOSE=1 for distchecks
Message-ID: <20191011173205.9814DA5F97@lists.varnish-cache.org>


commit 76c7d40b4114901a5519df4f93830f03aec85e8c
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Fri Oct 11 10:31:03 2019 -0700

    [cci] VERBOSE=1 for distchecks

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b419b8610..d0d93f752 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -236,7 +236,7 @@ jobs:
             	<< parameters.extra_conf >>
             sudo -u varnish \
             	--preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
-            	make distcheck -j 12 -k
+            	make distcheck VERBOSE=1 -j 12 -k
   push_packages:
     docker:
       - image: centos:7

From dridi at varni.sh  Fri Oct 11 17:50:59 2019
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Fri, 11 Oct 2019 17:50:59 +0000
Subject: [master] 76c7d40b4 [cci] VERBOSE=1 for distchecks
In-Reply-To: <20191011173205.9814DA5F97@lists.varnish-cache.org>
References: <20191011173205.9814DA5F97@lists.varnish-cache.org>
Message-ID: <CABoVN9ARRCc3fa1ca7LSzmss=5E2OY0k54=8hYanw3SdZvcToQ@mail.gmail.com>

On Fri, Oct 11, 2019 at 5:32 PM Guillaume Quintard
<guillaume at varnish-software.com> wrote:
>
>
> commit 76c7d40b4114901a5519df4f93830f03aec85e8c
> Author: Guillaume Quintard <guillaume at varnish-software.com>
> Date:   Fri Oct 11 10:31:03 2019 -0700
>
>     [cci] VERBOSE=1 for distchecks
>
> diff --git a/.circleci/config.yml b/.circleci/config.yml
> index b419b8610..d0d93f752 100644
> --- a/.circleci/config.yml
> +++ b/.circleci/config.yml
> @@ -236,7 +236,7 @@ jobs:
>                 << parameters.extra_conf >>
>              sudo -u varnish \
>                 --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
> -               make distcheck -j 12 -k
> +               make distcheck VERBOSE=1 -j 12 -k

Can we make this global instead?

https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-project

This way all `make *check` commands will print their failing test logs
and we don't have to worry about it next time we add a new
incantation.

If we can't do that at the project level, please make that a job
configuration instead for each job:

https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-job

Thanks,
Dridi

From guillaume at varnish-software.com  Fri Oct 11 17:59:07 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 11 Oct 2019 17:59:07 +0000 (UTC)
Subject: [master] 762fda9af [cci] VERBOSE=1 in the whole job,
 not just the command
Message-ID: <20191011175907.6CD9BA6B6C@lists.varnish-cache.org>


commit 762fda9afc74592efad93f9f09ca82c3bb4cf4cb
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Fri Oct 11 10:58:28 2019 -0700

    [cci] VERBOSE=1 in the whole job, not just the command

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d0d93f752..404567aed 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -205,6 +205,8 @@ jobs:
         type: string
     docker:
       - image: << parameters.dist >>:<< parameters.release >>
+    environment:
+      VERBOSE: "1"
     working_directory: /workspace
     steps:
       - << parameters.dist >>_install_build_deps
@@ -236,7 +238,7 @@ jobs:
             	<< parameters.extra_conf >>
             sudo -u varnish \
             	--preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
-            	make distcheck VERBOSE=1 -j 12 -k
+            	make distcheck -j 12 -k
   push_packages:
     docker:
       - image: centos:7

From guillaume at varnish-software.com  Fri Oct 11 18:00:23 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 11 Oct 2019 11:00:23 -0700
Subject: [master] 76c7d40b4 [cci] VERBOSE=1 for distchecks
In-Reply-To: <CABoVN9ARRCc3fa1ca7LSzmss=5E2OY0k54=8hYanw3SdZvcToQ@mail.gmail.com>
References: <20191011173205.9814DA5F97@lists.varnish-cache.org>
 <CABoVN9ARRCc3fa1ca7LSzmss=5E2OY0k54=8hYanw3SdZvcToQ@mail.gmail.com>
Message-ID: <CAJ6ZYQxzD+r7m48YPbngPvu64+o2moLt7X5SYttYMKLyn_8n+A@mail.gmail.com>

Done.
Can't be done globally, only at the job level, which only has that one make
command, so I don't feel the same appeal you do.
-- 
Guillaume Quintard


On Fri, Oct 11, 2019 at 10:51 AM Dridi Boukelmoune <dridi at varni.sh> wrote:

> On Fri, Oct 11, 2019 at 5:32 PM Guillaume Quintard
> <guillaume at varnish-software.com> wrote:
> >
> >
> > commit 76c7d40b4114901a5519df4f93830f03aec85e8c
> > Author: Guillaume Quintard <guillaume at varnish-software.com>
> > Date:   Fri Oct 11 10:31:03 2019 -0700
> >
> >     [cci] VERBOSE=1 for distchecks
> >
> > diff --git a/.circleci/config.yml b/.circleci/config.yml
> > index b419b8610..d0d93f752 100644
> > --- a/.circleci/config.yml
> > +++ b/.circleci/config.yml
> > @@ -236,7 +236,7 @@ jobs:
> >                 << parameters.extra_conf >>
> >              sudo -u varnish \
> >
>  --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
> > -               make distcheck -j 12 -k
> > +               make distcheck VERBOSE=1 -j 12 -k
>
> Can we make this global instead?
>
>
> https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-project
>
> This way all `make *check` commands will print their failing test logs
> and we don't have to worry about it next time we add a new
> incantation.
>
> If we can't do that at the project level, please make that a job
> configuration instead for each job:
>
>
> https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-job
>
> Thanks,
> Dridi
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20191011/04b73b10/attachment.html>

From guillaume at varnish-software.com  Fri Oct 11 18:43:08 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 11 Oct 2019 11:43:08 -0700
Subject: [master] 76c7d40b4 [cci] VERBOSE=1 for distchecks
In-Reply-To: <CAJ6ZYQxzD+r7m48YPbngPvu64+o2moLt7X5SYttYMKLyn_8n+A@mail.gmail.com>
References: <20191011173205.9814DA5F97@lists.varnish-cache.org>
 <CABoVN9ARRCc3fa1ca7LSzmss=5E2OY0k54=8hYanw3SdZvcToQ@mail.gmail.com>
 <CAJ6ZYQxzD+r7m48YPbngPvu64+o2moLt7X5SYttYMKLyn_8n+A@mail.gmail.com>
Message-ID: <CAJ6ZYQwUdKVz96FDk1YaJeyKdSXdmi+LuUV2Eq=uGTZToVfoug@mail.gmail.com>

hum, interestingly, it doesn't appear to work:
https://circleci.com/gh/varnishcache/varnish-cache/186 vs
https://circleci.com/gh/varnishcache/varnish-cache/196

I'm wondering if the sudo command doesn't screw things up here...

-- 
Guillaume Quintard


On Fri, Oct 11, 2019 at 11:00 AM Guillaume Quintard <
guillaume at varnish-software.com> wrote:

> Done.
> Can't be done globally, only at the job level, which only has that one
> make command, so I don't feel the same appeal you do.
> --
> Guillaume Quintard
>
>
> On Fri, Oct 11, 2019 at 10:51 AM Dridi Boukelmoune <dridi at varni.sh> wrote:
>
>> On Fri, Oct 11, 2019 at 5:32 PM Guillaume Quintard
>> <guillaume at varnish-software.com> wrote:
>> >
>> >
>> > commit 76c7d40b4114901a5519df4f93830f03aec85e8c
>> > Author: Guillaume Quintard <guillaume at varnish-software.com>
>> > Date:   Fri Oct 11 10:31:03 2019 -0700
>> >
>> >     [cci] VERBOSE=1 for distchecks
>> >
>> > diff --git a/.circleci/config.yml b/.circleci/config.yml
>> > index b419b8610..d0d93f752 100644
>> > --- a/.circleci/config.yml
>> > +++ b/.circleci/config.yml
>> > @@ -236,7 +236,7 @@ jobs:
>> >                 << parameters.extra_conf >>
>> >              sudo -u varnish \
>> >
>>  --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
>> > -               make distcheck -j 12 -k
>> > +               make distcheck VERBOSE=1 -j 12 -k
>>
>> Can we make this global instead?
>>
>>
>> https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-project
>>
>> This way all `make *check` commands will print their failing test logs
>> and we don't have to worry about it next time we add a new
>> incantation.
>>
>> If we can't do that at the project level, please make that a job
>> configuration instead for each job:
>>
>>
>> https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-job
>>
>> Thanks,
>> Dridi
>>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20191011/abbbfd60/attachment.html>

From dridi at varni.sh  Fri Oct 11 18:52:05 2019
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Fri, 11 Oct 2019 18:52:05 +0000
Subject: [master] 76c7d40b4 [cci] VERBOSE=1 for distchecks
In-Reply-To: <CAJ6ZYQwUdKVz96FDk1YaJeyKdSXdmi+LuUV2Eq=uGTZToVfoug@mail.gmail.com>
References: <20191011173205.9814DA5F97@lists.varnish-cache.org>
 <CABoVN9ARRCc3fa1ca7LSzmss=5E2OY0k54=8hYanw3SdZvcToQ@mail.gmail.com>
 <CAJ6ZYQxzD+r7m48YPbngPvu64+o2moLt7X5SYttYMKLyn_8n+A@mail.gmail.com>
 <CAJ6ZYQwUdKVz96FDk1YaJeyKdSXdmi+LuUV2Eq=uGTZToVfoug@mail.gmail.com>
Message-ID: <CABoVN9DHKejO=VYNHWx+_bjjbnG54_jZ6khsi19v1nLmxX5dFA@mail.gmail.com>

On Fri, Oct 11, 2019 at 6:43 PM Guillaume Quintard
<guillaume at varnish-software.com> wrote:
>
> hum, interestingly, it doesn't appear to work:
> https://circleci.com/gh/varnishcache/varnish-cache/186 vs https://circleci.com/gh/varnishcache/varnish-cache/196
>
> I'm wondering if the sudo command doesn't screw things up here...

*sigh* why does ci have to suck?

From dridi at varni.sh  Mon Oct 14 12:46:14 2019
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Mon, 14 Oct 2019 12:46:14 +0000
Subject: [master] a0a3dfcff use backend None instead of bad_ip backend
In-Reply-To: <20191009110406.0388EAD951@lists.varnish-cache.org>
References: <20191009110406.0388EAD951@lists.varnish-cache.org>
Message-ID: <CABoVN9BOGPz=dqgboPytrCWHzSJ5qR0uFDGbrt2ckHpsFcf-5w@mail.gmail.com>

On Wed, Oct 9, 2019 at 11:04 AM Nils Goroll <nils.goroll at uplex.de> wrote:
>
>
> commit a0a3dfcff4390545c02475266a7ae75d9cebc251
> Author: Nils Goroll <nils.goroll at uplex.de>
> Date:   Wed Oct 9 12:52:40 2019 +0200
>
>     use backend None instead of bad_ip backend
>
>     ... for those cases which can easily be changed (most of them).

Otherwise ${bad_backend} would have done the trick too.

>     This avoids issues with failing tests when bad_ip is actually reachable
>     via tcp on the respective port, as seen for example with b00068.vtc

I thought I had taken care of this last time I ran into this problem
but I suppose only a subset of the test cases failed me for this
reason. Should we evaluate the remaining uses of ${bad_ip} and
consider retiring it? Especially since it is purely arbitrary and not
really "bad" per-se.

Dridi

From dridi.boukelmoune at gmail.com  Mon Oct 14 15:00:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 14 Oct 2019 15:00:06 +0000 (UTC)
Subject: [master] 652f59194 Use the VCL_HTTP type in VRT code
Message-ID: <20191014150006.CDDF4116624@lists.varnish-cache.org>


commit 652f591945eb9bc3367e44aa6cf8f88554eaea65
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Oct 14 16:46:10 2019 +0200

    Use the VCL_HTTP type in VRT code

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 2838e9190..d19e85b90 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -128,7 +128,7 @@ VRT_hit_for_pass(VRT_CTX, VCL_DURATION d)
 VCL_HTTP
 VRT_selecthttp(VRT_CTX, enum gethdr_e where)
 {
-	struct http *hp;
+	VCL_HTTP hp;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	switch (where) {
@@ -158,8 +158,8 @@ VRT_selecthttp(VRT_CTX, enum gethdr_e where)
 VCL_STRING
 VRT_GetHdr(VRT_CTX, const struct gethdr_s *hs)
 {
+	VCL_HTTP hp;
 	const char *p;
-	struct http *hp;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (hs->where == HDR_OBJ) {
@@ -543,7 +543,7 @@ VCL_VOID
 VRT_SetHdr(VRT_CTX , const struct gethdr_s *hs,
     const char *p, ...)
 {
-	struct http *hp;
+	VCL_HTTP hp;
 	va_list ap;
 	const char *b;
 

From dridi.boukelmoune at gmail.com  Mon Oct 14 18:30:08 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 14 Oct 2019 18:30:08 +0000 (UTC)
Subject: [master] cacbaa6d0 Whitespace
Message-ID: <20191014183008.DD83941BA@lists.varnish-cache.org>


commit cacbaa6d075512c726019e94ad351fc9ebc989d3
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Oct 14 20:29:00 2019 +0200

    Whitespace

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index f7f47dceb..905042cbe 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -229,7 +229,7 @@ SLTM(FetchError, 0, "Error while fetching object",
 #undef SLTH
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
-	SLTM(Obj##tag, (resp ? 0 : SLT_F_UNUSED), "Object  " sdesc, ldesc)
+	SLTM(Obj##tag, (resp ? 0 : SLT_F_UNUSED), "Object " sdesc, ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 

From guillaume at varnish-software.com  Mon Oct 14 20:01:06 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Mon, 14 Oct 2019 20:01:06 +0000 (UTC)
Subject: [master] 9bf34a20a if --with-unwind, use it for distcheck too
Message-ID: <20191014200106.C3D5B6260@lists.varnish-cache.org>


commit 9bf34a20a7f05a277bbd9042e35aeba3de292da9
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Oct 14 12:38:11 2019 -0700

    if --with-unwind, use it for distcheck too

diff --git a/Makefile.am b/Makefile.am
index f019082a2..add21a8ca 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -24,6 +24,10 @@ AM_DISTCHECK_CONFIGURE_FLAGS = \
     --enable-dependency-tracking \
     CFLAGS="$(EXTCFLAGS)"
 
+if WITH_UNWIND
+AM_DISTCHECK_CONFIGURE_FLAGS += --with-unwind
+endif
+
 install-data-local:
 	$(install_sh) -d -m 0755 $(DESTDIR)$(localstatedir)/varnish
 

From guillaume at varnish-software.com  Mon Oct 14 20:01:06 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Mon, 14 Oct 2019 20:01:06 +0000 (UTC)
Subject: [master] f457cea3e Revert "[cci] VERBOSE=1 in the whole job,
 not just the command"
Message-ID: <20191014200106.D9D5C6263@lists.varnish-cache.org>


commit f457cea3e5c65d147ae668c15e7ac1b8a16456d1
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Oct 14 12:59:31 2019 -0700

    Revert "[cci] VERBOSE=1 in the whole job, not just the command"
    
    This reverts commit 762fda9afc74592efad93f9f09ca82c3bb4cf4cb.

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 404567aed..d0d93f752 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -205,8 +205,6 @@ jobs:
         type: string
     docker:
       - image: << parameters.dist >>:<< parameters.release >>
-    environment:
-      VERBOSE: "1"
     working_directory: /workspace
     steps:
       - << parameters.dist >>_install_build_deps
@@ -238,7 +236,7 @@ jobs:
             	<< parameters.extra_conf >>
             sudo -u varnish \
             	--preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
-            	make distcheck -j 12 -k
+            	make distcheck VERBOSE=1 -j 12 -k
   push_packages:
     docker:
       - image: centos:7

From dridi.boukelmoune at gmail.com  Tue Oct 15 15:33:05 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 15 Oct 2019 15:33:05 +0000 (UTC)
Subject: [master] 3a9681dec Revert "More work on refactoring req-cleanup"
Message-ID: <20191015153305.637AA10ECBA@lists.varnish-cache.org>


commit 3a9681dec62144e31822a6b9aa776260d7e67677
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 15 14:26:33 2019 +0200

    Revert "More work on refactoring req-cleanup"
    
    This reverts commit 6233b08842d8ea48e78b9fc5381a26a2c753862e.
    
    Conflicts:
            bin/varnishd/cache/cache_req.c
    
    The conflict was introduced with the handling of h2 stream 0 pseudo
    request: 51127b4600f095fb086c5115905b90ea91731d4f. This fixes a
    regression where timeout_idle could be circumvented by a periodic
    CRLF, except that it has no real consequence like holding onto a
    worker thread thanks to timeout_linger kicking in as expected.
    
    In addition to reverting the change, test coverage is added.

diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index def146b04..b13440e47 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -43,8 +43,8 @@
 
 #include "vtim.h"
 
-static void
-req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req)
+void
+Req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req)
 {
 	struct acct_req *a;
 
@@ -165,7 +165,8 @@ Req_Release(struct req *req)
 #include "tbl/acct_fields_req.h"
 
 	AZ(req->vcl);
-	AZ(req->vsl->wid);
+	if (req->vsl->wid)
+		VSL_End(req->vsl);
 	sp = req->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	pp = sp->pool;
@@ -219,8 +220,8 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 		VCL_Recache(wrk, &req->vcl);
 
 	/* Charge and log byte counters */
-	req_AcctLogCharge(wrk->stats, req);
 	if (req->vsl->wid) {
+		Req_AcctLogCharge(wrk->stats, req);
 		if (req->vsl->wid != sp->vxid)
 			VSL_End(req->vsl);
 		else
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index d32c8dde8..0fb461c61 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -350,6 +350,7 @@ void Req_Release(struct req *);
 void Req_Rollback(struct req *req);
 void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req);
 void Req_Fail(struct req *req, enum sess_close reason);
+void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *);
 
 /* cache_req_body.c */
 int VRB_Ignore(struct req *);
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index c289bbbf1..fcd914ba4 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -338,7 +338,7 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			if (hs < HTC_S_EMPTY) {
 				req->acct.req_hdrbytes +=
 				    req->htc->rxbuf_e - req->htc->rxbuf_b;
-				Req_Cleanup(sp, wrk, req);
+				Req_AcctLogCharge(wrk->stats, req);
 				Req_Release(req);
 				switch (hs) {
 				case HTC_S_CLOSE:
@@ -359,7 +359,6 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			}
 			if (hs == HTC_S_IDLE) {
 				wrk->stats->sess_herd++;
-				Req_Cleanup(sp, wrk, req);
 				Req_Release(req);
 				SES_Wait(sp, &HTTP1_transport);
 				return;
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 4fd379cd9..8eb5c0506 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -240,7 +240,7 @@ h2_ou_rel(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	AZ(req->vcl);
-	Req_Cleanup(req->sp, wrk, req);
+	Req_AcctLogCharge(wrk->stats, req);
 	Req_Release(req);
 	return (0);
 }
diff --git a/bin/varnishtest/tests/b00067.vtc b/bin/varnishtest/tests/b00067.vtc
index 6eb38996c..612f5fcd5 100644
--- a/bin/varnishtest/tests/b00067.vtc
+++ b/bin/varnishtest/tests/b00067.vtc
@@ -37,6 +37,28 @@ client c2 {
 	expect_close
 } -start
 
+client c3 {
+	loop 3 {
+		# send a periodic CRLF
+		delay 0.3
+		sendhex 0d0a
+	}
+	delay 0.3
+	expect_close
+} -start
+
+client c4 {
+	txreq
+	rxresp
+	loop 3 {
+		# send a periodic CRLF
+		delay 0.3
+		sendhex 0d0a
+	}
+	delay 0.3
+	expect_close
+} -start
+
 client c1u -connect "${tmpdir}/v1.sock" {
 	txreq
 	rxresp
@@ -57,7 +79,33 @@ client c2u -connect "${tmpdir}/v1.sock" {
 	expect_close
 } -start
 
+client c3u -connect "${tmpdir}/v1.sock" {
+	loop 3 {
+		# send a periodic CRLF
+		delay 0.3
+		sendhex 0d0a
+	}
+	delay 0.3
+	expect_close
+} -start
+
+client c4u -connect "${tmpdir}/v1.sock" {
+	txreq
+	rxresp
+	loop 3 {
+		# send a periodic CRLF
+		delay 0.3
+		sendhex 0d0a
+	}
+	delay 0.3
+	expect_close
+} -start
+
 client c1 -wait
 client c2 -wait
+client c3 -wait
+client c4 -wait
 client c1u -wait
 client c2u -wait
+client c3u -wait
+client c4u -wait

From dridi.boukelmoune at gmail.com  Tue Oct 15 15:33:05 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 15 Oct 2019 15:33:05 +0000 (UTC)
Subject: [master] e31cc250a Polish
Message-ID: <20191015153305.79C1110ECBD@lists.varnish-cache.org>


commit e31cc250a16190762e257bdd8df8a02062e55e34
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 15 14:28:49 2019 +0200

    Polish
    
    I was in the neighborhood...

diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index b13440e47..8fb6ec7f2 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -167,14 +167,12 @@ Req_Release(struct req *req)
 	AZ(req->vcl);
 	if (req->vsl->wid)
 		VSL_End(req->vsl);
-	sp = req->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	TAKE_OBJ_NOTNULL(sp, &req->sp, SESS_MAGIC);
 	pp = sp->pool;
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	MPL_AssertSane(req);
 	VSL_Flush(req->vsl, 0);
-	req->sp = NULL;
 	req->topreq = NULL;
 	MPL_Free(pp->mpl_req, req);
 }

From dridi.boukelmoune at gmail.com  Wed Oct 16 08:13:07 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 16 Oct 2019 08:13:07 +0000 (UTC)
Subject: [master] c5adaa3b6 Polish
Message-ID: <20191016081307.4DB1F65907@lists.varnish-cache.org>


commit c5adaa3b6ad51bab88dc21fca45210b84d7315f3
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 16 10:11:18 2019 +0200

    Polish

diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 98ad42c3c..da2eab1ca 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -121,7 +121,7 @@ struct parspec WRK_parspec[] = {
 		DELAYED_EFFECT,
 		"100", "threads" },
 	{ "thread_pool_reserve", tweak_uint, &mgt_param.wthread_reserve,
-		0, NULL,
+		NULL, NULL,
 		"The number of worker threads reserved for vital tasks "
 		"in each pool.\n"
 		"\n"

From nils.goroll at uplex.de  Thu Oct 17 13:06:41 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 17 Oct 2019 15:06:41 +0200
Subject: [master] a0a3dfcff use backend None instead of bad_ip backend
In-Reply-To: <CABoVN9BOGPz=dqgboPytrCWHzSJ5qR0uFDGbrt2ckHpsFcf-5w@mail.gmail.com>
References: <20191009110406.0388EAD951@lists.varnish-cache.org>
 <CABoVN9BOGPz=dqgboPytrCWHzSJ5qR0uFDGbrt2ckHpsFcf-5w@mail.gmail.com>
Message-ID: <a80bda18-38ea-7541-f5df-3778079422c2@uplex.de>

On 14/10/2019 14:46, Dridi Boukelmoune wrote:
>>     This avoids issues with failing tests when bad_ip is actually reachable
>>     via tcp on the respective port, as seen for example with b00068.vtc
> I thought I had taken care of this last time I ran into this problem
> but I suppose only a subset of the test cases failed me for this
> reason. Should we evaluate the remaining uses of ${bad_ip} and
> consider retiring it? Especially since it is purely arbitrary and not
> really "bad" per-se.

I do not think we should try to catch cases caused by "security software" which
some enterprises may or may not use

Nils

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20191017/5b33eb41/attachment-0001.bin>

From reza at naghibi.com  Thu Oct 17 16:33:07 2019
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 17 Oct 2019 16:33:07 +0000 (UTC)
Subject: [6.0] e95234225 Add param.reset command to varnishadm
Message-ID: <20191017163307.D09C6102045@lists.varnish-cache.org>


commit e95234225ec7093fc10d489414d0ea39c446a7e1
Author: Andrew Wiik <andrew at varnish-software.com>
Date:   Mon Oct 7 11:23:04 2019 -0400

    Add param.reset command to varnishadm
    
    b23dddd3ee02cd5cf093ae7fd534ace34ef90656
    
    Co-authored-by: Guillaume Quintard <guillaume at varnish-software.com>

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 9c6a74685..2bf69a0e5 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -505,6 +505,8 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 		VCLI_Out(cli, "parameter \"%s\" is protected.", param);
 		return;
 	}
+	if (!val)
+		val = pp->def;
 	if (pp->func(cli->sb, pp, val))
 		VCLI_SetResult(cli, CLIS_PARAM);
 
@@ -534,6 +536,16 @@ mcf_param_set(struct cli *cli, const char * const *av, void *priv)
 	MCF_ParamSet(cli, av[2], av[3]);
 }
 
+/*--------------------------------------------------------------------*/
+
+static void v_matchproto_(cli_func_t)
+mcf_param_reset(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)priv;
+	MCF_ParamSet(cli, av[2], NULL);
+}
+
 /*--------------------------------------------------------------------
  * Add a group of parameters to the global set and sort by name.
  */
@@ -595,6 +607,7 @@ static struct cli_proto cli_params[] = {
 	{ CLICMD_PARAM_SHOW,		"", mcf_param_show,
 	  mcf_param_show_json },
 	{ CLICMD_PARAM_SET,		"", mcf_param_set },
+	{ CLICMD_PARAM_RESET,		"", mcf_param_reset },
 	{ NULL }
 };
 
diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc
index 071e8c358..4b35d6434 100644
--- a/bin/varnishtest/tests/b00008.vtc
+++ b/bin/varnishtest/tests/b00008.vtc
@@ -36,6 +36,12 @@ varnish v1 -clijson "ping -j"
 
 varnish v1 -clierr 106 "param.set waiter HASH(0x8839c4c)"
 
+varnish v1 -cliexpect 60 "param.show first_byte_timeout"
+varnish v1 -cliok "param.set first_byte_timeout 120"
+varnish v1 -cliexpect 120 "param.show first_byte_timeout"
+varnish v1 -cliok "param.reset first_byte_timeout"
+varnish v1 -cliexpect 60 "param.show first_byte_timeout"
+
 varnish v1 -cliok "param.set cli_limit 128"
 
 varnish v1 -clierr 201 "param.show"
diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h
index 25f13d4ac..57efeaab6 100644
--- a/include/tbl/cli_cmds.h
+++ b/include/tbl/cli_cmds.h
@@ -133,6 +133,14 @@ CLI_CMD(VCL_LABEL,
 	2, 2
 )
 
+CLI_CMD(PARAM_RESET,
+	"param.reset",
+	"param.reset <param>",
+	"Reset parameter to default value.",
+	"",
+	1,1
+)
+
 CLI_CMD(PARAM_SHOW,
 	"param.show",
 	"param.show [-l|-j] [<param>|changed]",

From reza at naghibi.com  Thu Oct 17 16:33:07 2019
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 17 Oct 2019 16:33:07 +0000 (UTC)
Subject: [6.0] 9fe43bdf8 Add a 'debug.reqpool.fail F__F____F_FFF' command
Message-ID: <20191017163308.0216B102048@lists.varnish-cache.org>


commit 9fe43bdf89fb02344ead9b8fa2bad939f51d8169
Author: Andrew Wiik <andrew at varnish-software.com>
Date:   Mon Oct 7 11:25:08 2019 -0400

    Add a 'debug.reqpool.fail F__F____F_FFF' command
    
    0ad32622710c1787d85333faea844d4ab4dd760d
    
    Co-authored-by: Poul-Henning Kamp <phk at FreeBSD.org>

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index c86db2887..da5d1befd 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -391,6 +391,8 @@ child_main(int sigmagic, size_t altstksz)
 
 	VMOD_Init();
 
+	WRK_Init();
+
 	BAN_Compile();
 
 	VRND_SeedAll();
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 45aba733a..ad5e0be6d 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -414,6 +414,9 @@ extern struct vrt_privs cli_task_privs[1];
 void VMOD_Init(void);
 void VMOD_Panic(struct vsb *);
 
+/* cache_wrk.c */
+void WRK_Init(void);
+
 /* http1/cache_http1_pipe.c */
 void V1P_Init(void);
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 4d546ffd3..14bbfd4fd 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -58,12 +58,15 @@
 #include "cache_varnishd.h"
 #include "cache_pool.h"
 
+#include "vcli_serve.h"
 #include "vtim.h"
 
 #include "hash/hash_slinger.h"
 
 static void Pool_Work_Thread(struct pool *pp, struct worker *wrk);
 
+static uintmax_t reqpoolfail;
+
 /*--------------------------------------------------------------------
  * Create and start a back-ground thread which as its own worker and
  * session data structures;
@@ -261,6 +264,17 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio)
 	AN(task->func);
 	assert(prio < TASK_QUEUE_END);
 
+	if (prio == TASK_QUEUE_REQ && reqpoolfail) {
+		retval = reqpoolfail & 1;
+		reqpoolfail >>= 1;
+		if (retval) {
+			VSL(SLT_Debug, 0,
+			    "Failing due to reqpoolfail (next= 0x%jx)",
+			    reqpoolfail);
+			return(retval);
+		}
+	}
+
 	Lck_Lock(&pp->mtx);
 
 	/* The common case first:  Take an idle thread, do it. */
@@ -610,3 +624,39 @@ pool_herder(void *priv)
 	}
 	return (NULL);
 }
+
+/*--------------------------------------------------------------------
+ * Debugging aids
+ */
+
+static void v_matchproto_(cli_func_t)
+debug_reqpoolfail(struct cli *cli, const char * const *av, void *priv)
+{
+	uintmax_t u = 1;
+	const char *p;
+
+        (void)priv;
+        (void)cli;
+	reqpoolfail = 0;
+	for (p = av[2]; *p != '\0'; p++) {
+		if (*p == 'F' || *p == 'f')
+			reqpoolfail |= u;
+		u <<= 1;
+	}
+}
+
+static struct cli_proto debug_cmds[] = {
+	{ CLICMD_DEBUG_REQPOOLFAIL,		"d", debug_reqpoolfail },
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------
+ *
+ */
+
+void
+WRK_Init(void)
+{
+
+        CLI_AddFuncs(debug_cmds);
+}
diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h
index 57efeaab6..732522736 100644
--- a/include/tbl/cli_cmds.h
+++ b/include/tbl/cli_cmds.h
@@ -278,6 +278,16 @@ CLI_CMD(DEBUG_FRAGFETCH,
 	1, 1
 )
 
+CLI_CMD(DEBUG_REQPOOLFAIL,
+	"debug.reqpool.fail",
+	"debug.reqpool.fail",
+	"Schedule req-pool failures.",
+	"The argument is read L-R and 'f' means fail:\n\n"
+	"\tparam.set debug.reqpoolfail F__F\n\n"
+	"Means that the frist and the third attempted allocation will fail",
+	1, 1
+)
+
 CLI_CMD(DEBUG_XID,
 	"debug.xid",
 	"debug.xid",

From reza at naghibi.com  Thu Oct 17 16:33:08 2019
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 17 Oct 2019 16:33:08 +0000 (UTC)
Subject: [6.0] c3902d57f H2 Backports
Message-ID: <20191017163308.6C36A10204D@lists.varnish-cache.org>


commit c3902d57ff253eb7bf90f067c98e40f3bc5cad70
Author: Andrew Wiik <andrew at varnish-software.com>
Date:   Mon Oct 7 11:26:32 2019 -0400

    H2 Backports
    
    6233b08842d8ea48e78b9fc5381a26a2c753862e
    4d8e4b9cb960039d08814034d65d3ece1e78d042
    c726746621f7b87a069fdc0ca83bef82b4cefd5b
    cda1921004f10d3a56e6e044426473d99c88fa56
    dee47cc47b52dd8af11cb073661e9a757944aa7c
    e34de5c17515d9975588becde2577db60858fcc9
    a0ad902537bd0d14e663a7208f398ad55aed29e1
    54be487be4863decea34f4cf9600789b0c51f838
    1dc891a5aad0e20f45726ea1c2d923d520917691
    0582753d3708bd8cd546b30e6d6096e91a721b39
    78694521f5b41677e0c7f45907eb20ff48e7bb46
    3c04f5c715a078cec448ffc14a48fec38596d4e7
    f2357c88fa396b3fe911e9f194527e1fbe0dd978
    740ee39c8d6ffd36631a136cb9be23cdb13893d0
    cb1b5899b13f0240631f54794dd7c26660245cb0
    458306fa8ea4391b4567e52a5b869267b0f76ffc
    0e32f1667ea55156fc2a7fa84e3f1080a05bcb2e
    9e1e0972c47c5ea03350c39c76370ccafd678802
    a82d8552fc9f612b2295c357a256f896c6168368
    4bbf6df68722dbe3ea77508d160c1dd258ab1d5e
    d8421dcd59a8cf271e4587a0228a3a24954636f2
    3c8ecb4cc0447e0149715e2fbb773a8fa9f433ce
    a0b8a73441511e3ee90479f11f927d0600e9ae06
    8c5118d17cba2f4d78a672f42f16f1cbff2ca766
    3d170c0d8fd029eb8604854b9a758404f19008c2
    d869640ba8b4978c890610a1a6dcd1bf82a600d3
    1506c5cffc9b7882eb059ce128dfdf824c41445f
    5ecbde3812f9d17b47faf3b6c44567bcb7060fac
    e1a1fdc7688de5f37e35fc528639019d5bd3efbf
    64c0e1aec5145dd1c977f6e53e84e54288447855
    93e6dd706074d5f5815bd03e76d63fa30f6666c9
    f3e9ca6abc4a03e48df4e9894323cad25472793f
    07ad2fb863b7db12e94449e11d47a1eefb2dd359
    
    Co-authored-by: Dag Haavi Finstad <daghf at varnish-software.com>
    Co-authored-by: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
    Co-authored-by: Federico G. Schwindt <fgsch at lodoss.net>
    Co-authored-by: Nils Goroll <nils.goroll at uplex.de>
    Co-authored-by: Poul-Henning Kamp <phk at FreeBSD.org>
    Co-authored-by: Shohei Tanaka(@xcir) <kokoniimasu+git at gmail.com>

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a6d4f514c..2d6f1ccbc 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2015 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -567,8 +567,12 @@ struct sess {
 	vtim_real		t_open;		/* fd accepted */
 	vtim_real		t_idle;		/* fd accepted or resp sent */
 
+	vtim_dur		timeout_idle;
 };
 
+#define SESS_TMO(sp, tmo)					\
+	(isnan((sp)->tmo) ? cache_param->tmo : (sp)->tmo)
+
 /* Prototypes etc ----------------------------------------------------*/
 
 
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 95bc5f2ed..2f3f8e86f 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -208,7 +208,7 @@ ved_include(struct req *preq, const char *src, const char *host,
 	req->wrk = NULL;
 	THR_SetRequest(preq);
 
-	Req_AcctLogCharge(wrk->stats, req);
+	Req_Cleanup(sp, wrk, req);
 	Req_Release(req);
 	SES_Rel(sp);
 }
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 5ab13a2cf..eb7bea2a5 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -43,8 +43,8 @@
 
 #include "vtim.h"
 
-void
-Req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req)
+static void
+req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req)
 {
 	struct acct_req *a;
 
@@ -63,7 +63,11 @@ Req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req)
 		    (uintmax_t)(a->resp_hdrbytes + a->resp_bodybytes));
 	}
 
-	/* Charge to main byte counters (except for ESI subrequests) */
+	/*
+	 * Charge to main byte counters, except for ESI subrequests
+	 * which are charged as they pass through the topreq.
+	 * XXX: make this test req->top instead
+	 */
 #define ACCT(foo)			\
 	if (req->esi_level == 0)	\
 		ds->s_##foo += a->foo;	\
@@ -166,8 +170,7 @@ Req_Release(struct req *req)
 #include "tbl/acct_fields_req.h"
 
 	AZ(req->vcl);
-	if (req->vsl->wid)
-		VSL_End(req->vsl);
+	AZ(req->vsl->wid);
 	sp = req->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	pp = sp->pool;
@@ -212,9 +215,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->director_hint = NULL;
 	req->restarts = 0;
 
-	AZ(req->esi_level);
 	AZ(req->privs->magic);
-	assert(req->top == req);
 
 	if (req->vcl != NULL) {
 		if (wrk->vcl != NULL)
@@ -224,10 +225,9 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 
 	/* Charge and log byte counters */
-	if (req->vsl->wid) {
-		Req_AcctLogCharge(wrk->stats, req);
+	req_AcctLogCharge(wrk->stats, req);
+	if (req->vsl->wid)
 		VSL_End(req->vsl);
-	}
 	req->req_bodybytes = 0;
 
 	if (!isnan(req->t_prev) && req->t_prev > 0. && req->t_prev > sp->t_idle)
@@ -243,6 +243,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->hash_always_miss = 0;
 	req->hash_ignore_busy = 0;
 	req->is_hit = 0;
+	req->esi_level = 0;
 
 	if (WS_Overflowed(req->ws))
 		wrk->stats->ws_client_overflow++;
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 2402b10b5..a8a75b7e7 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -248,12 +248,14 @@ HTC_RxPipeline(struct http_conn *htc, void *p)
  * *t1 becomes time of first non-idle rx
  * *t2 becomes time of complete rx
  * ti is when we return IDLE if nothing has arrived
- * tn is when we timeout on non-complete
+ * tn is when we timeout on non-complete (total timeout)
+ * td is max timeout between reads
  */
 
 enum htc_status_e
 HTC_RxStuff(struct http_conn *htc, htc_complete_f *func,
-    vtim_real *t1, vtim_real *t2, vtim_real ti, vtim_real tn, int maxbytes)
+    vtim_real *t1, vtim_real *t2, vtim_real ti, vtim_real tn, vtim_dur td,
+    int maxbytes)
 {
 	vtim_dur tmo;
 	vtim_real now;
@@ -268,7 +270,7 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func,
 	assert(htc->rxbuf_b <= htc->rxbuf_e);
 	assert(htc->rxbuf_e <= htc->ws->r);
 
-	AZ(isnan(tn));
+	AZ(isnan(tn) && isnan(td));
 	if (t1 != NULL)
 		assert(isnan(*t1));
 
@@ -310,9 +312,18 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func,
 		else
 			WRONG("htc_status_e");
 
-		tmo = tn - now;
-		if (!isnan(ti) && ti < tn && hs == HTC_S_EMPTY)
+		if (hs == HTC_S_EMPTY && !isnan(ti) && (isnan(tn) || ti < tn))
 			tmo = ti - now;
+		else if (isnan(tn))
+			tmo = td;
+		else if (isnan(td))
+			tmo = tn - now;
+		else if (td < tn - now)
+			tmo = td;
+		else
+			tmo = tn - now;
+
+		AZ(isnan(tmo));
 		z = maxbytes - (htc->rxbuf_e - htc->rxbuf_b);
 		if (z <= 0) {
 			/* maxbytes reached but not HTC_S_COMPLETE. Return
@@ -329,14 +340,11 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func,
 		} else if (z > 0)
 			htc->rxbuf_e += z;
 		else if (z == -2) {
-			if (hs == HTC_S_EMPTY && ti <= now) {
-				WS_ReleaseP(htc->ws, htc->rxbuf_b);
+			WS_ReleaseP(htc->ws, htc->rxbuf_b);
+			if (hs == HTC_S_EMPTY)
 				return (HTC_S_IDLE);
-			}
-			if (tn <= now) {
-				WS_ReleaseP(htc->ws, htc->rxbuf_b);
+			else
 				return (HTC_S_TIMEOUT);
-			}
 		}
 	}
 }
@@ -371,6 +379,7 @@ SES_New(struct pool *pp)
 
 	sp->t_open = NAN;
 	sp->t_idle = NAN;
+	sp->timeout_idle = NAN;
 	Lck_New(&sp->mtx, lck_sess);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	return (sp);
@@ -458,7 +467,7 @@ SES_Wait(struct sess *sp, const struct transport *xp)
 	wp->priv2 = (uintptr_t)xp;
 	wp->idle = sp->t_idle;
 	wp->func = ses_handle;
-	wp->tmo = &cache_param->timeout_idle;
+	wp->tmo = SESS_TMO(sp, timeout_idle);
 	if (Wait_Enter(pp->waiter, wp))
 		SES_Delete(sp, SC_PIPE_OVERFLOW, NAN);
 }
diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index 9d623ef9c..7ea5942ee 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Varnish Software AS
+ * Copyright (c) 2015-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -339,7 +339,7 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp)
 	pfd->waited->idle = VTIM_real();
 	pfd->state = PFD_STATE_AVAIL;
 	pfd->waited->func = vcp_handle;
-	pfd->waited->tmo = &cache_param->backend_idle_timeout;
+	pfd->waited->tmo = cache_param->backend_idle_timeout;
 	if (Wait_Enter(wrk->pool->waiter, pfd->waited)) {
 		cp->methods->close(pfd);
 		memset(pfd, 0x33, sizeof *pfd);
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index ad5e0be6d..5d036f6da 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2015 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -318,7 +318,6 @@ void Req_Release(struct req *);
 void Req_Rollback(struct req *req);
 void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req);
 void Req_Fail(struct req *req, enum sess_close reason);
-void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *);
 
 /* cache_req_body.c */
 int VRB_Ignore(struct req *);
@@ -347,7 +346,8 @@ const char * HTC_Status(enum htc_status_e);
 void HTC_RxInit(struct http_conn *htc, struct ws *ws);
 void HTC_RxPipeline(struct http_conn *htc, void *);
 enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *,
-    vtim_real *t1, vtim_real *t2, vtim_real ti, vtim_real tn, int maxbytes);
+    vtim_real *t1, vtim_real *t2, vtim_real ti, vtim_real tn, vtim_dur td,
+    int maxbytes);
 
 #define SESS_ATTR(UP, low, typ, len)					\
 	int SES_Set_##low(const struct sess *sp, const typ *src);	\
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index d3787b193..b7557edf5 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2015 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -923,3 +923,27 @@ HTTP_VAR(req)
 HTTP_VAR(resp)
 HTTP_VAR(bereq)
 HTTP_VAR(beresp)
+
+/*--------------------------------------------------------------------*/
+
+VCL_VOID
+VRT_l_sess_timeout_idle(VRT_CTX, VCL_DURATION d)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->sp, SESS_MAGIC);
+	ctx->sp->timeout_idle = d > 0.0 ? d : 0.0;
+}
+
+VCL_DURATION
+VRT_r_sess_timeout_idle(VRT_CTX)
+{
+	VCL_DURATION d;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->sp, SESS_MAGIC);
+
+	d = ctx->sp->timeout_idle;
+	if (isnan(d))
+		return (cache_param->timeout_idle);
+	return (d);
+}
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index c937d741e..879115b63 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2015 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -170,7 +170,7 @@ V1F_FetchRespHdr(struct busyobj *bo)
 
 	t = VTIM_real() + htc->first_byte_timeout;
 	hs = HTC_RxStuff(htc, HTTP1_Complete, NULL, NULL,
-	    t, t + htc->between_bytes_timeout, cache_param->http_resp_size);
+	     t, NAN, htc->between_bytes_timeout, cache_param->http_resp_size);
 	if (hs != HTC_S_COMPLETE) {
 		bo->acct.beresp_hdrbytes +=
 		    htc->rxbuf_e - htc->rxbuf_b;
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index fad043e84..1c216dc78 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2015 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -337,13 +337,14 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			hs = HTC_RxStuff(req->htc, HTTP1_Complete,
 			    &req->t_first, &req->t_req,
 			    sp->t_idle + cache_param->timeout_linger,
-			    sp->t_idle + cache_param->timeout_idle,
+			    sp->t_idle + SESS_TMO(sp, timeout_idle),
+			    NAN,
 			    cache_param->http_req_size);
 			AZ(req->htc->ws->r);
 			if (hs < HTC_S_EMPTY) {
 				req->acct.req_hdrbytes +=
 				    req->htc->rxbuf_e - req->htc->rxbuf_b;
-				Req_AcctLogCharge(wrk->stats, req);
+				Req_Cleanup(sp, wrk, req);
 				Req_Release(req);
 				switch (hs) {
 				case HTC_S_CLOSE:
@@ -364,6 +365,7 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			}
 			if (hs == HTC_S_IDLE) {
 				wrk->stats->sess_herd++;
+				Req_Cleanup(sp, wrk, req);
 				Req_Release(req);
 				SES_Wait(sp, &HTTP1_transport);
 				return;
diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index 70e31d14e..c377d03aa 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 Varnish Software AS
+ * Copyright (c) 2016-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -133,6 +133,8 @@ struct h2_req {
 
 	VTAILQ_ENTRY(h2_req)		tx_list;
 	h2_error			error;
+
+	int				counted;
 };
 
 VTAILQ_HEAD(h2_req_s, h2_req);
@@ -178,6 +180,8 @@ struct h2_sess {
 	VTAILQ_HEAD(,h2_req)		txqueue;
 
 	h2_error			error;
+
+	int				open_streams;
 };
 
 #define ASSERT_RXTHR(h2) do {assert(h2->rxthr == pthread_self());} while(0)
@@ -221,15 +225,18 @@ void H2_Send_Frame(struct worker *, struct h2_sess *,
     h2_frame type, uint8_t flags, uint32_t len, uint32_t stream,
     const void *);
 
-void H2_Send(struct worker *, struct h2_req *,
-    h2_frame type, uint8_t flags, uint32_t len, const void *);
+void H2_Send_RST(struct worker *wrk, struct h2_sess *h2,
+    const struct h2_req *r2, uint32_t stream, h2_error h2e);
+
+void H2_Send(struct worker *, struct h2_req *, h2_frame type, uint8_t flags,
+    uint32_t len, const void *, uint64_t *acct);
 
 /* cache_http2_proto.c */
 struct h2_req * h2_new_req(const struct worker *, struct h2_sess *,
     unsigned stream, struct req *);
+int h2_stream_tmo(struct h2_sess *, const struct h2_req *, vtim_real);
 void h2_del_req(struct worker *, const struct h2_req *);
-void h2_kill_req(struct worker *, const struct h2_sess *,
-    struct h2_req *, h2_error);
+void h2_kill_req(struct worker *, struct h2_sess *, struct h2_req *, h2_error);
 int h2_rxframe(struct worker *, struct h2_sess *);
 h2_error h2_set_setting(struct h2_sess *, const uint8_t *);
 void h2_req_body(struct req*);
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 1a1de05be..46f7654d9 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 Varnish Software AS
+ * Copyright (c) 2016-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -87,8 +87,12 @@ h2_fini(struct req *req, void **priv)
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	TAKE_OBJ_NOTNULL(r2, priv, H2_REQ_MAGIC);
+
+	if (r2->error)
+		return (0);
+
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
-	H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, "");
+	H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, "", NULL);
 	H2_Send_Rel(r2->h2sess, r2);
 	return (0);
 }
@@ -106,9 +110,9 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv,
 	if ((r2->h2sess->error || r2->error))
 		return (-1);
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
-	H2_Send(req->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr);
+	H2_Send(req->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr,
+	    &req->acct.resp_bodybytes);
 	H2_Send_Rel(r2->h2sess, r2);
-	req->acct.resp_bodybytes += len;
 	return (0);
 }
 
@@ -173,7 +177,7 @@ h2_minimal_response(struct req *req, uint16_t status)
 	    H2_F_HEADERS,
 	    H2FF_HEADERS_END_HEADERS |
 		(status < 200 ? 0 : H2FF_HEADERS_END_STREAM),
-	    l, buf);
+	    l, buf, NULL);
 	H2_Send_Rel(r2->h2sess, r2);
 	return (0);
 }
@@ -214,35 +218,31 @@ static const uint8_t h2_500_resp[] = {
 	0x1f, 0x27, 0x07, 'V', 'a', 'r', 'n', 'i', 's', 'h',
 };
 
-void v_matchproto_(vtr_deliver_f)
-h2_deliver(struct req *req, struct boc *boc, int sendbody)
+static int
+h2_build_headers(struct vsb *resp, struct req *req)
 {
-	ssize_t sz, sz1;
 	unsigned u, l;
-	uint8_t buf[6];
-	const char *r;
-	struct http *hp;
-	struct sess *sp;
-	struct h2_req *r2;
-	struct vsb resp;
 	int i;
+	struct http *hp;
+	const char *r;
 	const struct hpack_static *hps;
-
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_ORNULL(boc, BOC_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC);
-	sp = req->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	uint8_t buf[6];
+	ssize_t sz, sz1;
 
 	l = WS_Reserve(req->ws, 0);
-	AN(VSB_new(&resp, req->ws->f, l, VSB_FIXEDLEN));
+	AN(VSB_new(resp, req->ws->f, l, VSB_FIXEDLEN));
+	if (l < 10) {
+		WS_Release(req->ws, 0);
+		return (-1);
+	}
+
+	AN(VSB_new(resp, req->ws->f, l, VSB_FIXEDLEN));
 
 	l = h2_status(buf, req->resp->status);
-	VSB_bcat(&resp, buf, l);
+	VSB_bcat(resp, buf, l);
 
 	hp = req->resp;
-	for (u = HTTP_HDR_FIRST; u < hp->nhd && !VSB_error(&resp); u++) {
+	for (u = HTTP_HDR_FIRST; u < hp->nhd && !VSB_error(resp); u++) {
 		r = strchr(hp->hd[u].b, ':');
 		AN(r);
 
@@ -263,25 +263,46 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 			VSLb(req->vsl, SLT_Debug,
 			    "HP {%d, \"%s\", \"%s\"} <%s>",
 			    hps->idx, hps->name, hps->val, hp->hd[u].b);
-			h2_enc_len(&resp, 4, hps->idx, 0x10);
+			h2_enc_len(resp, 4, hps->idx, 0x10);
 		} else {
-			VSB_putc(&resp, 0x10);
+			VSB_putc(resp, 0x10);
 			sz--;
-			h2_enc_len(&resp, 7, sz, 0);
+			h2_enc_len(resp, 7, sz, 0);
 			for (sz1 = 0; sz1 < sz; sz1++)
-				VSB_putc(&resp, tolower(hp->hd[u].b[sz1]));
+				VSB_putc(resp, tolower(hp->hd[u].b[sz1]));
 
 		}
 
 		while (vct_islws(*++r))
 			continue;
 		sz = hp->hd[u].e - r;
-		h2_enc_len(&resp, 7, sz, 0);
-		VSB_bcat(&resp, r, sz);
+		h2_enc_len(resp, 7, sz, 0);
+		VSB_bcat(resp, r, sz);
 	}
-	if (VSB_finish(&resp)) {
-		WS_MarkOverflow(req->ws);
+	return (VSB_finish(resp));
+}
+
+void v_matchproto_(vtr_deliver_f)
+h2_deliver(struct req *req, struct boc *boc, int sendbody)
+{
+	ssize_t sz;
+	const char *r;
+	struct sess *sp;
+	struct h2_req *r2;
+	struct vsb resp;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(boc, BOC_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+	CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC);
+	sp = req->sp;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	VSLb(req->vsl, SLT_RespProtocol, "HTTP/2.0");
+
+	if (h2_build_headers(&resp, req)) {
 		// We ran out of workspace, return minimal 500
+		WS_MarkOverflow(req->ws);
 		VSLb(req->vsl, SLT_Error, "workspace_client overflow");
 		VSLb(req->vsl, SLT_RespStatus, "500");
 		VSLb(req->vsl, SLT_RespReason, "Internal Server Error");
@@ -305,9 +326,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
 	H2_Send(req->wrk, r2, H2_F_HEADERS,
 	    (sendbody ? 0 : H2FF_HEADERS_END_STREAM) | H2FF_HEADERS_END_HEADERS,
-	    sz, r);
+	    sz, r, &req->acct.resp_hdrbytes);
 	H2_Send_Rel(r2->h2sess, r2);
-	req->acct.resp_hdrbytes += sz;
 
 	WS_Release(req->ws, 0);
 
diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c
index 0524f3e32..f2ef11cf7 100644
--- a/bin/varnishd/http2/cache_http2_hpack.c
+++ b/bin/varnishd/http2/cache_http2_hpack.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 Varnish Software AS
+ * Copyright (c) 2016-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Martin Blix Grydeland <martin at varnish-software.com>
@@ -141,7 +141,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len)
 
 	if (n < HTTP_HDR_FIRST) {
 		/* Check for duplicate pseudo-header */
-		if (n < HTTP_HDR_FIRST && hp->hd[n].b != NULL) {
+		if (hp->hd[n].b != NULL) {
 			VSLb(hp->vsl, SLT_BogoHeader,
 			    "Duplicate pseudo-header: %.*s",
 			    (int)(len > 20 ? 20 : len), b);
@@ -176,8 +176,11 @@ h2h_decode_init(const struct h2_sess *h2)
 	INIT_OBJ(d, H2H_DECODE_MAGIC);
 	VHD_Init(d->vhd);
 	d->out_l = WS_Reserve(h2->new_req->http->ws, 0);
-	assert(d->out_l > 0);	/* Can't do any work without any buffer
-				   space. Require non-zero size. */
+	/*
+	 * Can't do any work without any buffer
+	 * space. Require non-zero size.
+	 */
+	XXXAN(d->out_l);
 	d->out = h2->new_req->http->ws->f;
 	d->reset = d->out;
 }
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 194f225c5..6a98a1c4b 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 Varnish Software AS
+ * Copyright (c) 2016-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -132,28 +132,6 @@ h2_connectionerror(uint32_t u)
 
 /**********************************************************************/
 
-static void
-h2_tx_rst(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2,
-    uint32_t stream, h2_error h2e)
-{
-	char b[4];
-
-	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
-
-	Lck_Lock(&h2->sess->mtx);
-	VSLb(h2->vsl, SLT_Debug, "H2: stream %u: %s", stream, h2e->txt);
-	Lck_Unlock(&h2->sess->mtx);
-	vbe32enc(b, h2e->val);
-
-	H2_Send_Get(wrk, h2, r2);
-	H2_Send_Frame(wrk, h2, H2_F_RST_STREAM, 0, sizeof b, stream, b);
-	H2_Send_Rel(h2, r2);
-}
-
-/**********************************************************************
- */
-
 struct h2_req *
 h2_new_req(const struct worker *wrk, struct h2_sess *h2,
     unsigned stream, struct req *req)
@@ -172,10 +150,14 @@ h2_new_req(const struct worker *wrk, struct h2_sess *h2,
 	r2->h2sess = h2;
 	r2->stream = stream;
 	r2->req = req;
+	if (stream)
+		r2->counted = 1;
 	r2->r_window = h2->local_settings.initial_window_size;
 	r2->t_window = h2->remote_settings.initial_window_size;
 	req->transport_priv = r2;
 	Lck_Lock(&h2->sess->mtx);
+	if (stream)
+		h2->open_streams++;
 	VTAILQ_INSERT_TAIL(&h2->streams, r2, list);
 	Lck_Unlock(&h2->sess->mtx);
 	h2->refcnt++;
@@ -206,14 +188,20 @@ h2_del_req(struct worker *wrk, const struct h2_req *r2)
 }
 
 void
-h2_kill_req(struct worker *wrk, const struct h2_sess *h2,
+h2_kill_req(struct worker *wrk, struct h2_sess *h2,
     struct h2_req *r2, h2_error h2e)
 {
 
 	ASSERT_RXTHR(h2);
 	AN(h2e);
 	Lck_Lock(&h2->sess->mtx);
-	VSLb(h2->vsl, SLT_Debug, "KILL st=%u state=%d", r2->stream, r2->state);
+	VSLb(h2->vsl, SLT_Debug, "KILL st=%u state=%d sched=%d",
+	    r2->stream, r2->state, r2->scheduled);
+	if (r2->counted) {
+		assert(h2->open_streams > 0);
+		h2->open_streams--;
+		r2->counted = 0;
+	}
 	if (r2->error == NULL)
 		r2->error = h2e;
 	if (r2->scheduled) {
@@ -631,7 +619,12 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	if (r2 == NULL) {
 		if (h2->rxf_stream <= h2->highest_stream)
 			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
-		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
+		/* NB: we don't need to guard the read of h2->open_streams
+		 * because headers are handled sequentially so it cannot
+		 * increase under our feet.
+		 */
+		if (h2->open_streams >=
+		    h2->local_settings.max_concurrent_streams) {
 			VSLb(h2->vsl, SLT_Debug,
 			     "H2: stream %u: Hit maximum number of "
 			     "concurrent streams", h2->rxf_stream);
@@ -755,6 +748,10 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		return (H2SE_STREAM_CLOSED); // rfc7540,l,1766,1769
 	}
 	Lck_Lock(&h2->sess->mtx);
+	while (h2->mailcall != NULL && h2->error == 0 && r2->error == 0)
+		AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0));
+	if (h2->error || r2->error)
+		return (h2->error ? h2->error : r2->error);
 	AZ(h2->mailcall);
 	h2->mailcall = r2;
 	h2->req0->r_window -= h2->rxf_len;
@@ -858,8 +855,10 @@ h2_vfp_body_fini(struct vfp_ctx *vc, struct vfp_entry *vfe)
 
 	if (vc->failed) {
 		CHECK_OBJ_NOTNULL(r2->req->wrk, WORKER_MAGIC);
-		h2_tx_rst(r2->req->wrk, h2, r2, r2->stream,
+		H2_Send_Get(r2->req->wrk, h2, r2);
+		H2_Send_RST(r2->req->wrk, h2, r2, r2->stream,
 		    H2SE_REFUSED_STREAM);
+		H2_Send_Rel(h2, r2);
 		Lck_Lock(&h2->sess->mtx);
 		r2->error = H2SE_REFUSED_STREAM;
 		if (h2->mailcall == r2) {
@@ -963,41 +962,60 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 	if (h2->rxf_stream == 0 || h2e->connection)
 		return (h2e);	// Connection errors one level up
 
-	h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream, h2e);
+	H2_Send_Get(wrk, h2, h2->req0);
+	H2_Send_RST(wrk, h2, h2->req0, h2->rxf_stream, h2e);
+	H2_Send_Rel(h2, h2->req0);
 	return (0);
 }
 
-static int
-h2_stream_tmo(struct h2_sess *h2, const struct h2_req *r2)
+int
+h2_stream_tmo(struct h2_sess *h2, const struct h2_req *r2, vtim_real now)
 {
 	int r = 0;
 
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	Lck_AssertHeld(&h2->sess->mtx);
 
-	if (r2->t_winupd != 0 || r2->t_send != 0) {
-		Lck_Lock(&h2->sess->mtx);
-		if (r2->t_winupd != 0 &&
-		    h2->sess->t_idle - r2->t_winupd >
-		    cache_param->idle_send_timeout) {
-			VSLb(h2->vsl, SLT_Debug,
-			     "H2: stream %u: Hit idle_send_timeout waiting "
-			     "for WINDOW_UPDATE", r2->stream);
-			r = 1;
-		}
+	/* NB: when now is NAN, it means that idle_send_timeout was hit
+	 * on a lock condwait operation.
+	 */
+	if (isnan(now))
+		AN(r2->t_winupd);
 
-		if (r == 0 && r2->t_send != 0 &&
-		    h2->sess->t_idle - r2->t_send > cache_param->send_timeout) {
-			VSLb(h2->vsl, SLT_Debug,
-			     "H2: stream %u: Hit send_timeout", r2->stream);
-			r = 1;
-		}
-		Lck_Unlock(&h2->sess->mtx);
+	if (r2->t_winupd == 0 && r2->t_send == 0)
+		return (0);
+
+	if (isnan(now) || (r2->t_winupd != 0 &&
+	    now - r2->t_winupd > cache_param->idle_send_timeout)) {
+		VSLb(h2->vsl, SLT_Debug,
+		     "H2: stream %u: Hit idle_send_timeout waiting for"
+		     " WINDOW_UPDATE", r2->stream);
+		r = 1;
+	}
+
+	if (r == 0 && r2->t_send != 0 &&
+	    now - r2->t_send > cache_param->send_timeout) {
+		VSLb(h2->vsl, SLT_Debug,
+		     "H2: stream %u: Hit send_timeout", r2->stream);
+		r = 1;
 	}
 
 	return (r);
 }
 
+static int
+h2_stream_tmo_unlocked(struct h2_sess *h2, const struct h2_req *r2)
+{
+	int r;
+
+	Lck_Lock(&h2->sess->mtx);
+	r = h2_stream_tmo(h2, r2, h2->sess->t_idle);
+	Lck_Unlock(&h2->sess->mtx);
+
+	return (r);
+}
+
 /*
  * This is the janitorial task of cleaning up any closed & refused
  * streams, and checking if the session is timed out.
@@ -1023,15 +1041,17 @@ h2_sweep(struct worker *wrk, struct h2_sess *h2)
 			break;
 		case H2_S_CLOS_REM:
 			if (!r2->scheduled) {
-				h2_tx_rst(wrk, h2, h2->req0, r2->stream,
+				H2_Send_Get(wrk, h2, h2->req0);
+				H2_Send_RST(wrk, h2, h2->req0, r2->stream,
 				    H2SE_REFUSED_STREAM);
+				H2_Send_Rel(h2, h2->req0);
 				h2_del_req(wrk, r2);
 				continue;
 			}
 			/* FALLTHROUGH */
 		case H2_S_CLOS_LOC:
 		case H2_S_OPEN:
-			if (h2_stream_tmo(h2, r2)) {
+			if (h2_stream_tmo_unlocked(h2, r2)) {
 				tmo = 1;
 				continue;
 			}
@@ -1080,8 +1100,8 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 	h2->sess->t_idle = VTIM_real();
 	hs = HTC_RxStuff(h2->htc, h2_frame_complete,
 	    NULL, NULL, NAN,
-	    h2->sess->t_idle + cache_param->timeout_idle,
-	    h2->local_settings.max_frame_size + 9);
+	    h2->sess->t_idle + SESS_TMO(h2->sess, timeout_idle),
+	    NAN, h2->local_settings.max_frame_size + 9);
 	switch (hs) {
 	case HTC_S_COMPLETE:
 		break;
diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c
index 638fea632..f0a8b96cd 100644
--- a/bin/varnishd/http2/cache_http2_send.c
+++ b/bin/varnishd/http2/cache_http2_send.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 Varnish Software AS
+ * Copyright (c) 2016-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -30,6 +30,7 @@
 #include "config.h"
 
 #include <sys/uio.h>
+#include <errno.h>
 
 #include "cache/cache_varnishd.h"
 
@@ -39,8 +40,52 @@
 #include "vend.h"
 #include "vtim.h"
 
+#define H2_SEND_HELD(h2, r2) (VTAILQ_FIRST(&(h2)->txqueue) == (r2))
+
+static int
+h2_cond_wait(pthread_cond_t *cond, struct h2_sess *h2, struct h2_req *r2)
+{
+	vtim_real now, when = 0.;
+	int r;
+
+	AN(cond);
+	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+
+	Lck_AssertHeld(&h2->sess->mtx);
+
+	now = VTIM_real();
+	if (cache_param->idle_send_timeout > 0.)
+		when = now + cache_param->idle_send_timeout;
+
+	r = Lck_CondWait(cond, &h2->sess->mtx, when);
+	assert(r == 0 || r == ETIMEDOUT);
+
+	now = VTIM_real();
+	/* NB: when we grab idle_send_timeout before acquiring the session
+	 * lock we may time out, but once we wake up both send_timeout and
+	 * idle_send_timeout may have changed meanwhile. For this reason
+	 * h2_stream_tmo() may not log what timed out and we need to call
+	 * again with a magic NAN "now" that indicates to h2_stream_tmo()
+	 * that the stream reached the idle_send_timeout via the lock and
+	 * force it to log it.
+	 */
+	if (h2_stream_tmo(h2, r2, now))
+		r = ETIMEDOUT;
+	else if (r == ETIMEDOUT)
+		AN(h2_stream_tmo(h2, r2, NAN));
+
+	if (r == ETIMEDOUT) {
+		if (r2->error == NULL)
+			r2->error = H2SE_CANCEL;
+		return (-1);
+	}
+
+	return (0);
+}
+
 static void
-h2_send_get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
+h2_send_get_locked(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
@@ -51,7 +96,7 @@ h2_send_get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		ASSERT_RXTHR(h2);
 	r2->wrk = wrk;
 	VTAILQ_INSERT_TAIL(&h2->txqueue, r2, tx_list);
-	while (VTAILQ_FIRST(&h2->txqueue) != r2)
+	while (!H2_SEND_HELD(h2, r2))
 		AZ(Lck_CondWait(&wrk->cond, &h2->sess->mtx, 0));
 	r2->wrk = NULL;
 }
@@ -64,18 +109,18 @@ H2_Send_Get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	Lck_Lock(&h2->sess->mtx);
-	h2_send_get(wrk, h2, r2);
+	h2_send_get_locked(wrk, h2, r2);
 	Lck_Unlock(&h2->sess->mtx);
 }
 
 static void
-h2_send_rel(struct h2_sess *h2, const struct h2_req *r2)
+h2_send_rel_locked(struct h2_sess *h2, const struct h2_req *r2)
 {
 	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
 
 	Lck_AssertHeld(&h2->sess->mtx);
-	assert(VTAILQ_FIRST(&h2->txqueue) == r2);
+	AN(H2_SEND_HELD(h2, r2));
 	VTAILQ_REMOVE(&h2->txqueue, r2, tx_list);
 	r2 = VTAILQ_FIRST(&h2->txqueue);
 	if (r2 != NULL) {
@@ -91,7 +136,7 @@ H2_Send_Rel(struct h2_sess *h2, const struct h2_req *r2)
 	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 
 	Lck_Lock(&h2->sess->mtx);
-	h2_send_rel(h2, r2);
+	h2_send_rel_locked(h2, r2);
 	Lck_Unlock(&h2->sess->mtx);
 }
 
@@ -109,7 +154,7 @@ h2_mk_hdr(uint8_t *hdr, h2_frame ftyp, uint8_t flags,
 }
 
 /*
- * This is the "raw" frame sender, all per stream accounting and
+ * This is the "raw" frame sender, all per-stream accounting and
  * prioritization must have happened before this is called, and
  * the session mtx must be held.
  */
@@ -217,15 +262,14 @@ h2_do_window(struct worker *wrk, struct h2_req *r2,
 	Lck_Lock(&h2->sess->mtx);
 	if (r2->t_window <= 0 || h2->req0->t_window <= 0) {
 		r2->t_winupd = VTIM_real();
-		h2_send_rel(h2, r2);
+		h2_send_rel_locked(h2, r2);
 		while (r2->t_window <= 0 && h2_errcheck(r2, h2) == 0) {
 			r2->cond = &wrk->cond;
-			AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0));
+			(void)h2_cond_wait(r2->cond, h2, r2);
 			r2->cond = NULL;
 		}
-		while (h2->req0->t_window <= 0 && h2_errcheck(r2, h2) == 0) {
-			AZ(Lck_CondWait(h2->winupd_cond, &h2->sess->mtx, 0));
-		}
+		while (h2->req0->t_window <= 0 && h2_errcheck(r2, h2) == 0)
+			(void)h2_cond_wait(h2->winupd_cond, h2, r2);
 
 		if (h2_errcheck(r2, h2) == 0) {
 			w = h2_win_limit(r2, h2);
@@ -234,7 +278,7 @@ h2_do_window(struct worker *wrk, struct h2_req *r2,
 			h2_win_charge(r2, h2, w);
 			assert (w > 0);
 		}
-		h2_send_get(wrk, h2, r2);
+		h2_send_get_locked(wrk, h2, r2);
 	}
 
 	if (w == 0 && h2_errcheck(r2, h2) == 0) {
@@ -256,9 +300,9 @@ h2_do_window(struct worker *wrk, struct h2_req *r2,
  * XXX: priority
  */
 
-void
-H2_Send(struct worker *wrk, struct h2_req *r2,
-    h2_frame ftyp, uint8_t flags, uint32_t len, const void *ptr)
+static void
+h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags,
+    uint32_t len, const void *ptr, uint64_t *counter)
 {
 	struct h2_sess *h2;
 	uint32_t mfs, tf;
@@ -270,8 +314,9 @@ H2_Send(struct worker *wrk, struct h2_req *r2,
 	h2 = r2->h2sess;
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
 	assert(len == 0 || ptr != NULL);
+	AN(counter);
 
-	assert(VTAILQ_FIRST(&h2->txqueue) == r2);
+	AN(H2_SEND_HELD(h2, r2));
 
 	if (h2_errcheck(r2, h2))
 		return;
@@ -285,19 +330,28 @@ H2_Send(struct worker *wrk, struct h2_req *r2,
 
 	Lck_Lock(&h2->sess->mtx);
 	mfs = h2->remote_settings.max_frame_size;
+	if (r2->counted && (
+	    (ftyp == H2_F_HEADERS && (flags & H2FF_HEADERS_END_STREAM)) ||
+	    (ftyp == H2_F_DATA && (flags & H2FF_DATA_END_STREAM)) ||
+	    ftyp == H2_F_RST_STREAM
+	    )) {
+		assert(h2->open_streams > 0);
+		h2->open_streams--;
+		r2->counted = 0;
+	}
 	Lck_Unlock(&h2->sess->mtx);
 
 	if (ftyp->respect_window) {
-		tf = h2_do_window(wrk, r2, h2,
-				  (len > mfs) ? mfs : len);
+		tf = h2_do_window(wrk, r2, h2, (len > mfs) ? mfs : len);
 		if (h2_errcheck(r2, h2))
 			return;
-		assert(VTAILQ_FIRST(&h2->txqueue) == r2);
+		AN(H2_SEND_HELD(h2, r2));
 	} else
 		tf = mfs;
 
 	if (len <= tf) {
 		H2_Send_Frame(wrk, h2, ftyp, flags, len, r2->stream, ptr);
+		*counter += len;
 	} else {
 		AN(ptr);
 		p = ptr;
@@ -309,10 +363,10 @@ H2_Send(struct worker *wrk, struct h2_req *r2,
 				tf = mfs;
 			if (ftyp->respect_window && p != ptr) {
 				tf = h2_do_window(wrk, r2, h2,
-						  (len > mfs) ? mfs : len);
+					(len > mfs) ? mfs : len);
 				if (h2_errcheck(r2, h2))
 					return;
-				assert(VTAILQ_FIRST(&h2->txqueue) == r2);
+				AN(H2_SEND_HELD(h2, r2));
 			}
 			if (tf < len) {
 				H2_Send_Frame(wrk, h2, ftyp,
@@ -321,13 +375,49 @@ H2_Send(struct worker *wrk, struct h2_req *r2,
 				if (ftyp->respect_window)
 					assert(tf == len);
 				tf = len;
-				H2_Send_Frame(wrk, h2, ftyp,
-				    final_flags, tf, r2->stream, p);
+				H2_Send_Frame(wrk, h2, ftyp, final_flags, tf,
+					r2->stream, p);
 				flags = 0;
 			}
 			p += tf;
 			len -= tf;
+			*counter += tf;
 			ftyp = ftyp->continuation;
+			flags &= ftyp->flags;
+			final_flags &= ftyp->flags;
 		} while (!h2->error && len > 0);
 	}
 }
+
+void
+H2_Send_RST(struct worker *wrk, struct h2_sess *h2, const struct h2_req *r2,
+    uint32_t stream, h2_error h2e)
+{
+	char b[4];
+
+	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	AN(H2_SEND_HELD(h2, r2));
+
+	Lck_Lock(&h2->sess->mtx);
+	VSLb(h2->vsl, SLT_Debug, "H2: stream %u: %s", stream, h2e->txt);
+	Lck_Unlock(&h2->sess->mtx);
+	vbe32enc(b, h2e->val);
+
+	H2_Send_Frame(wrk, h2, H2_F_RST_STREAM, 0, sizeof b, stream, b);
+}
+
+void
+H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags,
+    uint32_t len, const void *ptr, uint64_t *counter)
+{
+	uint64_t dummy_counter;
+
+	if (counter == NULL)
+		counter = &dummy_counter;
+
+	h2_send(wrk, r2, ftyp, flags, len, ptr, counter);
+
+	if (h2_errcheck(r2, r2->h2sess) == H2SE_CANCEL)
+		H2_Send_RST(wrk, r2->h2sess, r2, r2->stream, H2SE_CANCEL);
+}
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 246400de7..140007ffc 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 Varnish Software AS
+ * Copyright (c) 2016-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -53,21 +53,21 @@ static const char H2_prism[24] = {
 };
 
 static size_t
-h2_enc_settings(const struct h2_settings *h2s, uint8_t *buf, size_t n)
+h2_enc_settings(const struct h2_settings *h2s, uint8_t *buf, ssize_t n)
 {
-	size_t len = 0;
+	uint8_t *p = buf;
 
 #define H2_SETTING(U,l,v,d,...)				\
 	if (h2s->l != d) {				\
-		len += 6;				\
-		assert(len <= n);			\
-		vbe16enc(buf, v);			\
-		buf += 2;				\
-		vbe32enc(buf, h2s->l);			\
-		buf += 4;				\
+		n -= 6;					\
+		assert(n >= 0);				\
+		vbe16enc(p, v);				\
+		p += 2;					\
+		vbe32enc(p, h2s->l);			\
+		p += 4;					\
 	}
 #include "tbl/h2_settings.h"
-	return (len);
+	return (p - buf);
 }
 
 static const struct h2_settings H2_proto_settings = {
@@ -151,7 +151,7 @@ h2_del_sess(struct worker *wrk, struct h2_sess *h2, enum sess_close reason)
 
 	VHT_Fini(h2->dectbl);
 	AZ(pthread_cond_destroy(h2->winupd_cond));
-	req = h2->srq;
+	TAKE_OBJ_NOTNULL(req, &h2->srq, REQ_MAGIC);
 	AZ(req->ws->r);
 	sp = h2->sess;
 	Req_Cleanup(sp, wrk, req);
@@ -242,7 +242,7 @@ h2_ou_rel(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	AN (req->vcl);
 	VCL_Rel(&req->vcl);
-	Req_AcctLogCharge(wrk->stats, req);
+	Req_Cleanup(req->sp, wrk, req);
 	Req_Release(req);
 	return (0);
 }
@@ -293,7 +293,7 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2,
 
 	/* Wait for PRISM response */
 	hs = HTC_RxStuff(h2->htc, H2_prism_complete,
-	    NULL, NULL, NAN, h2->sess->t_idle + cache_param->timeout_idle,
+	    NULL, NULL, NAN, h2->sess->t_idle + cache_param->timeout_idle, NAN,
 	    sizeof H2_prism);
 	if (hs != HTC_S_COMPLETE) {
 		VSLb(h2->vsl, SLT_Debug, "H2: No/Bad OU PRISM (hs=%d)", hs);
@@ -301,7 +301,12 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2,
 		h2_del_req(wrk, r2);
 		return (0);
 	}
-	XXXAZ(Pool_Task(wrk->pool, &req->task, TASK_QUEUE_REQ));
+	if (Pool_Task(wrk->pool, &req->task, TASK_QUEUE_REQ)) {
+		r2->scheduled = 0;
+		h2_del_req(wrk, r2);
+		VSLb(h2->vsl, SLT_Debug, "H2: No Worker-threads");
+		return (0);
+	}
 	return (1);
 }
 
diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index 1def50ebe..9232254d3 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Varnish Software AS
+ * Copyright (c) 2015-2019 Varnish Software AS
  * Copyright (c) 2018 GANDI SAS
  * All rights reserved.
  *
@@ -530,7 +530,7 @@ vpx_new_session(struct worker *wrk, void *arg)
 
 	HTC_RxInit(req->htc, req->ws);
 	hs = HTC_RxStuff(req->htc, vpx_complete,
-	    NULL, NULL, NAN, sp->t_idle + cache_param->timeout_idle,
+	    NULL, NULL, NAN, sp->t_idle + cache_param->timeout_idle, NAN,
 	    1024);			// XXX ?
 	if (hs != HTC_S_COMPLETE) {
 		Req_Release(req);
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index fc316424b..06cc22bfc 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2015 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -133,7 +133,6 @@ Wait_Enter(const struct waiter *w, struct waited *wp)
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	assert(wp->fd > 0);			// stdin never comes here
 	AN(wp->func);
-	AN(wp->tmo);
 	wp->idx = BINHEAP_NOIDX;
 	return (w->impl->enter(w->priv, wp));
 }
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index 4a0d8a7d5..81b1f3923 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -62,7 +62,7 @@ struct waited {
 	void			*priv1;
 	uintptr_t		priv2;
 	waiter_handle_f		*func;
-	volatile vtim_real	*tmo;
+	vtim_dur		tmo;
 	vtim_real		idle;
 };
 
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index 502d34961..dccbc7a71 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2019 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -63,15 +63,14 @@ Wait_Tmo(const struct waited *wp)
 {
 	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
 	AN(wp->tmo);
-	return (*wp->tmo);
+	return (wp->tmo);
 }
 
 static inline double
 Wait_When(const struct waited *wp)
 {
 	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
-	AN(wp->tmo);
-	return (wp->idle + *wp->tmo);
+	return (wp->idle + wp->tmo);
 }
 
 void Wait_Call(const struct waiter *, struct waited *,
diff --git a/bin/varnishtest/tests/b00068.vtc b/bin/varnishtest/tests/b00068.vtc
new file mode 100644
index 000000000..5bb1fee4e
--- /dev/null
+++ b/bin/varnishtest/tests/b00068.vtc
@@ -0,0 +1,35 @@
+varnishtest "Check timeout_idle"
+
+varnish v1 -arg "-p timeout_idle=1" -vcl {
+	backend dummy { .host = "${bad_ip}"; }
+
+	sub vcl_deliver {
+		if (req.url == "/sess") {
+			set sess.timeout_idle = 2s;
+		}
+	}
+	sub vcl_backend_error {
+		set beresp.status = 200;
+		set beresp.ttl = 1h;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	delay 0.2
+	txreq
+	rxresp
+	delay 1.2
+	expect_close
+} -run
+
+client c1 {
+	txreq -url "/sess"
+	rxresp
+	delay 1.2
+	txreq
+	rxresp
+	delay 2.2
+	expect_close
+} -run
diff --git a/bin/varnishtest/tests/r02395.vtc b/bin/varnishtest/tests/r02395.vtc
new file mode 100644
index 000000000..99461e16c
--- /dev/null
+++ b/bin/varnishtest/tests/r02395.vtc
@@ -0,0 +1,22 @@
+varnishtest "Test between_bytes_timeout works fetching headers"
+
+server s1 {
+        rxreq
+        send "HTTP/1.0 "
+        delay 2
+} -start
+
+varnish v1 -vcl+backend {
+        sub vcl_recv { return (pass); }
+        sub vcl_backend_fetch { set bereq.between_bytes_timeout = 1s; }
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
+
+client c1 {
+        txreq
+        rxresp
+        expect resp.status == 503
+} -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/r02923.vtc b/bin/varnishtest/tests/r02923.vtc
new file mode 100644
index 000000000..2fa9f65d1
--- /dev/null
+++ b/bin/varnishtest/tests/r02923.vtc
@@ -0,0 +1,99 @@
+varnishtest "race cond in max_concurrent_streams when request after receiving RST_STREAM"
+
+barrier b1 sock 2
+barrier b2 sock 2
+barrier b3 sock 2
+barrier b4 sock 2
+barrier b5 sock 3
+
+server s1 {
+	rxreq
+	expect req.url == /nosync
+	txresp
+	rxreq
+	expect req.url == /sync
+	txresp
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set h2_max_concurrent_streams 3"
+
+varnish v1 -vcl+backend {
+	import vtc;
+
+	sub vcl_recv {
+	}
+
+	sub vcl_backend_fetch {
+		if(bereq.url ~ "/sync"){
+			vtc.barrier_sync("${b1_sock}");
+			vtc.barrier_sync("${b5_sock}");
+		}
+	}
+} -start
+
+client c1 {
+	txpri
+	stream 0 rxsettings -run
+
+	stream 1 {
+		txreq -url /sync
+		rxresp
+		expect resp.status == 200
+	} -start
+
+	stream 3 {
+		barrier b1 sync
+		delay .5
+		# Goes on waiting list
+		txreq -url /sync
+		delay .5
+		txrst -err 0x8
+		delay .5
+		barrier b2 sync
+	} -start
+
+	stream 5 {
+		barrier b2 sync
+		txreq -url /nosync
+		delay .5
+		rxresp
+		delay .5
+		expect resp.status == 200
+		barrier b3 sync
+	} -start
+
+	stream 7 {
+		barrier b3 sync
+		# Goes on waiting list
+		txreq -url /sync
+		delay .5
+		txrst -err 0x8
+		delay .5
+		barrier b4 sync
+	} -start
+
+	stream 9 {
+		barrier b4 sync
+		# Goes on waiting list
+		txreq -url /sync
+		delay .5
+		barrier b5 sync
+		delay .5
+		rxresp
+		delay .5
+		expect resp.status == 200
+	} -start
+
+	stream 11 {
+		barrier b5 sync
+		txreq -url /sync
+		delay .5
+		rxresp
+		delay .5
+		expect resp.status == 200
+	} -start
+
+
+} -run
diff --git a/bin/varnishtest/tests/r02934.vtc b/bin/varnishtest/tests/r02934.vtc
new file mode 100644
index 000000000..923baf47c
--- /dev/null
+++ b/bin/varnishtest/tests/r02934.vtc
@@ -0,0 +1,27 @@
+varnishtest "Bug in CONTINUATION flags when no sendbody"
+
+server s1 {
+	rxreq
+	expect req.proto == HTTP/1.1
+	txresp -hdr "Content-Type: text/plain" -hdrlen Foo 100
+
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set debug +h2_nocheck"
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	stream 0 {
+		txsettings -framesize 64
+		rxsettings
+	} -run
+	stream 1 {
+		txreq -req HEAD
+		rxresp
+		expect resp.status == 200
+		expect resp.http.content-Type == "text/plain"
+	} -run
+} -run
diff --git a/bin/varnishtest/tests/r02937.vtc b/bin/varnishtest/tests/r02937.vtc
new file mode 100644
index 000000000..8a2d00d58
--- /dev/null
+++ b/bin/varnishtest/tests/r02937.vtc
@@ -0,0 +1,25 @@
+varnishtest "#2937: Panic on OU pool failure"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "debug.reqpool.fail F"
+
+client c1 {
+	send "GET / HTTP/1.1\r\n"
+	send "Host: foo\r\n"
+	send "Upgrade: h2c\r\n"
+	send "HTTP2-Settings: AAMAAABkAAQAAP__\r\n"
+	send "\r\n"
+	rxresp
+	expect resp.status == 101
+	expect resp.http.upgrade == h2c
+	expect resp.http.connection == Upgrade
+	txpri
+	expect_close
+} -run
diff --git a/bin/varnishtest/tests/t02012.vtc b/bin/varnishtest/tests/t02012.vtc
index 5319453e7..3f7ec8092 100644
--- a/bin/varnishtest/tests/t02012.vtc
+++ b/bin/varnishtest/tests/t02012.vtc
@@ -12,7 +12,7 @@ server s1 {
 
 varnish v1 -cliok "param.set feature +http2"
 varnish v1 -cliok "param.set debug +syncvsl"
-varnish v1 -cliok "param.set h2_max_concurrent_streams 3"
+varnish v1 -cliok "param.set h2_max_concurrent_streams 2"
 
 varnish v1 -vcl+backend {
 	import vtc;
diff --git a/bin/varnishtest/tests/t02015.vtc b/bin/varnishtest/tests/t02015.vtc
new file mode 100644
index 000000000..3da5c24d4
--- /dev/null
+++ b/bin/varnishtest/tests/t02015.vtc
@@ -0,0 +1,49 @@
+varnishtest "h2 ReqAcct"
+
+server s1 {
+	rxreq
+	txresp -bodylen 12345
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -vcl+backend "" -start
+
+client c1 {
+	txpri
+
+	stream 0 {
+		rxsettings
+		expect settings.ack == false
+		txsettings -ack
+		txsettings -winsize 1000
+		rxsettings
+		expect settings.ack == true
+	} -run
+
+	stream 1 {
+		txreq -hdr stream 1
+		rxhdrs
+		rxdata
+		txwinup -size 11345
+		rxdata
+	} -run
+
+	stream 3 {
+		txreq -hdr stream 3
+		rxhdrs
+		rxdata
+		txrst
+	} -run
+} -run
+
+shell {
+	varnishncsa -n ${v1_name} -F '%{VSL:ReqAcct[5]}x' -d \
+		-q 'ReqHeader:stream == 1' |
+	grep 12345
+}
+
+shell {
+	varnishncsa -n ${v1_name} -F '%{VSL:ReqAcct[5]}x' -d \
+		-q 'ReqHeader:stream == 3' |
+	grep 1000
+}
diff --git a/bin/varnishtest/tests/t02016.vtc b/bin/varnishtest/tests/t02016.vtc
new file mode 100644
index 000000000..29ffea354
--- /dev/null
+++ b/bin/varnishtest/tests/t02016.vtc
@@ -0,0 +1,119 @@
+varnishtest "client h2 send timeouts"
+
+server s1 {
+	rxreq
+	txresp -bodylen 12345
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -vcl+backend "" -start
+
+# coverage for send_timeout with c1
+
+varnish v1 -cliok "param.set send_timeout 1"
+
+logexpect l1 -v v1 {
+	expect * * Debug "Hit send_timeout"
+} -start
+
+client c1 {
+	txpri
+
+	stream 0 {
+		rxsettings
+		expect settings.ack == false
+		txsettings -ack
+		txsettings -winsize 1000
+		rxsettings
+		expect settings.ack == true
+	} -run
+
+	stream 1 {
+		txreq
+		rxhdrs
+		rxdata
+		# keep the stream idle for 2 seconds
+		delay 2
+		txwinup -size 256
+		# too late
+		rxrst
+		expect rst.err == CANCEL
+	} -run
+
+} -run
+
+logexpect l1 -wait
+
+# coverage for idle_send_timeout with c2
+
+varnish v1 -cliok "param.set idle_send_timeout 1"
+varnish v1 -cliok "param.reset send_timeout"
+
+logexpect l2 -v v1 {
+	expect * * Debug "Hit idle_send_timeout"
+} -start
+
+client c2 {
+	txpri
+
+	stream 0 {
+		rxsettings
+		expect settings.ack == false
+		txsettings -ack
+		txsettings -winsize 1000
+		rxsettings
+		expect settings.ack == true
+	} -run
+
+	stream 1 {
+		txreq
+		rxhdrs
+		rxdata
+		# keep the stream idle for 2 seconds
+		delay 2
+		rxrst
+		expect rst.err == CANCEL
+	} -run
+
+} -run
+
+logexpect l2 -wait
+
+# coverage for idle_send_timeout change with c3
+
+barrier b3 cond 2
+
+logexpect l3 -v v1 {
+	expect * * Debug "Hit idle_send_timeout"
+} -start
+
+client c3 {
+	txpri
+
+	stream 0 {
+		rxsettings
+		expect settings.ack == false
+		txsettings -ack
+		txsettings -winsize 1000
+		rxsettings
+		expect settings.ack == true
+	} -run
+
+	stream 1 {
+		txreq
+		rxhdrs
+		rxdata
+		# keep the stream idle for 2 seconds
+		barrier b3 sync
+		delay 2
+		rxrst
+		expect rst.err == CANCEL
+	} -run
+
+} -start
+
+barrier b3 sync
+varnish v1 -cliok "param.reset idle_send_timeout"
+
+client c3 -wait
+logexpect l3 -wait
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index e8c04d928..16a35106a 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -1185,6 +1185,17 @@ sess.xid	``VCL >= 4.1``
 
 	Unique ID of this session.
 
+sess.timeout_idle
+
+	Type: DURATION
+
+	Readable from: client
+
+	Writable from: client
+
+	Idle timeout for this session, defaults to the
+	``timeout_idle`` parameter, see :ref:`varnishd(1)`
+
 storage
 ~~~~~~~
 

From guillaume at varnish-software.com  Fri Oct 18 00:37:09 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 18 Oct 2019 00:37:09 +0000 (UTC)
Subject: [master] 7ee8f6c74 [cci] start packaging alpine
Message-ID: <20191018003709.946BD10AFCF@lists.varnish-cache.org>


commit 7ee8f6c743cd51fb5a3453796914294d7c8adca3
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Oct 14 14:26:13 2019 -0700

    [cci] start packaging alpine

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d0d93f752..461c6fb7b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -143,6 +143,66 @@ jobs:
           root: .
           paths:
             - debs/varnish*.deb
+  build_alpine:
+    description: Build alpine apks
+    docker:
+      - image: alpine
+    working_directory: /workspace
+    steps:
+      - run:
+          name: Install certificates to mount the workspace, and tar
+          command: |
+            apk update
+            apk add -q ca-certificates tar
+      - attach_workspace:
+          at: /workspace
+      - run:
+          name: Untar alpine
+          command: |
+            tar xavf alpine.tar.gz --strip 1
+      - run:
+          name: Install sdk, add user
+          command: |
+            apk add alpine-sdk
+            adduser -D builder
+            echo "builder ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
+            addgroup builder abuild
+            mkdir -p /var/cache/distfiles
+            chmod a+w /var/cache/distfiles
+      - run:
+          name: Generate key
+          command: |
+            su builder -c "abuild-keygen -nai"
+      - run:
+          name: Fix APKBUILD's variables
+          command: |
+            tar xavf varnish-*.tar.gz
+            VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}')
+            if [ "$VERSION" = trunk ]; then
+            	WEEKLYVERSION=`date "+%Y%m%d"`
+            	sed -i \
+                    -e "s@^pkgver=.*@pkgver=$WEEKLYVERSION@" \
+                    -e "s@^builddir=.*@builddir=\"\$srcdir/varnish-$VERSION\"@" \
+                    -e "s@^source=.*@source=varnish-trunk.tar.gz@" APKBUILD
+            else
+            	sed -i "s@^pkgver=.*@pkgver=$VERSION@" APKBUILD
+            fi
+            rm -rf varnish-$VERSION
+      - run:
+          name: Fix checksums, build
+          command: |
+            chown builder -R /workspace
+            su builder -c "abuild checksum"
+            su builder -c "abuild -r"
+      - run:
+          name: Fix the APKBUILD's version
+          command: |
+            mkdir apks
+            cp /home/builder/packages/x86_64/*.apk apks
+      - persist_to_workspace:
+          root: .
+          paths:
+            - apks/*.apk
   dist_ubuntu:
     docker:
       - image: ubuntu:bionic
@@ -181,14 +241,17 @@ jobs:
             mkdir -p ~/.ssh
             ssh-keyscan -H github.com >> ~/.ssh/known_hosts
             echo ${CIRCLE_REPOSITORY_URL}
-            git clone --branch=weekly git at github.com:varnishcache/pkg-varnish-cache.git .
+            git clone --branch=alpine https://github.com/gquintard/pkg-varnish-cache.git .
+            git checkout alpine
             tar cvzf debian.tar.gz debian --dereference
             tar cvzf redhat.tar.gz redhat --dereference 
+            tar cvzf alpine.tar.gz alpine --dereference
       - persist_to_workspace:
           root: .
           paths:
             - debian.tar.gz
             - redhat.tar.gz
+            - alpine.tar.gz
   distcheck:
     parameters:
       release:
@@ -248,7 +311,7 @@ jobs:
           command: |
               rm rpms/varnish*.src.rpm
               mv rpms/*/*.rpm rpms/
-              tar cvzf packages.tar.gz rpms/*.rpm debs/*.deb
+              tar cvzf packages.tar.gz rpms/*.rpm debs/*.deb apks/*.apk
       - store_artifacts:
           destination: packages.tar.gz
           path: packages.tar.gz
@@ -369,6 +432,8 @@ workflows:
           <<: *pkg_req
       - build_centos_7:
           <<: *pkg_req
+      - build_alpine:
+          <<: *pkg_req
       - hold:
           type: approval
           requires:
@@ -377,6 +442,7 @@ workflows:
             - build_ubuntu_xenial
             - build_ubuntu_bionic
             - build_centos_7
+            - build_alpine
       - push_packages:
           requires:
             - hold
@@ -390,7 +456,7 @@ workflows:
           name: distcheck_debian_buster
           dist: debian
           release: buster
-          extra_conf: --enable-asan --enable-ubsan  
+          extra_conf: --enable-asan --enable-ubsan
           requires:
             - dist_ubuntu
       - distcheck:

From guillaume at varnish-software.com  Fri Oct 18 00:37:09 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 18 Oct 2019 00:37:09 +0000 (UTC)
Subject: [master] 29b1c8880 [cci] remove the manual approval job
Message-ID: <20191018003709.A52E310AFD1@lists.varnish-cache.org>


commit 29b1c888031a101b7dac8d66ff541af861981fb2
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Tue Oct 15 22:39:53 2019 -0700

    [cci] remove the manual approval job

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 461c6fb7b..6a33b3a65 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -434,8 +434,7 @@ workflows:
           <<: *pkg_req
       - build_alpine:
           <<: *pkg_req
-      - hold:
-          type: approval
+      - push_packages:
           requires:
             - build_debian_stretch
             - build_debian_buster
@@ -443,9 +442,6 @@ workflows:
             - build_ubuntu_bionic
             - build_centos_7
             - build_alpine
-      - push_packages:
-          requires:
-            - hold
 #      - distcheck:
 #          name: distcheck_centos_7
 #          dist: centos

From guillaume at varnish-software.com  Fri Oct 18 00:37:09 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 18 Oct 2019 00:37:09 +0000 (UTC)
Subject: [master] 072597248 Add CI links to README.rst
Message-ID: <20191018003709.C6C6E10AFD4@lists.varnish-cache.org>


commit 072597248184cbe884228c45feaf5cc276987aee
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Oct 16 10:31:17 2019 -0700

    Add CI links to README.rst

diff --git a/README.rst b/README.rst
index 966da29e6..f18de281c 100644
--- a/README.rst
+++ b/README.rst
@@ -13,3 +13,10 @@ Please see CONTRIBUTING for how to contribute patches and report bugs.
 
 Questions about commercial support and services related to Varnish
 should be addressed to <sales at varnish-software.com>.
+
+.. |ccibadge| image:: https://circleci.com/gh/varnishcache/varnish-cache/tree/master.svg?style=svg
+    :target: https://circleci.com/gh/varnishcache/varnish-cache/tree/master
+.. _vtest: https://varnish-cache.org/vtest/
+CircleCI tests: |ccibadge|
+
+More platforms are tested via vtest_

From hermunn at varnish-software.com  Fri Oct 18 12:14:06 2019
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Fri, 18 Oct 2019 12:14:06 +0000 (UTC)
Subject: [6.0] f0160a118 Update changelog to current state of 6.0.5 development
Message-ID: <20191018121406.B13324C0E@lists.varnish-cache.org>


commit f0160a118caa518fa1813bd4036ef9ef1a2a5162
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Fri Oct 18 14:05:38 2019 +0200

    Update changelog to current state of 6.0.5 development

diff --git a/doc/changes.rst b/doc/changes.rst
index 8626f4284..b6110cf59 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -26,6 +26,37 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+================================
+Varnish Cache 6.0.5 (unreleased)
+================================
+
+* Various H2 bug fixes and improvements have been back ported from the
+  master branch. (2395_, 2572_, 2905_, 2923_, 2930_, 2931_, 2933_,
+  2934_, 2937_, 2938_, 2967_)
+
+* The idle timeout can now be set per session, by setting the
+  `sess.timeout_idle` variable in VCL.
+
+* A `param.reset` command has been added to `varnishadm`.
+
+* Make waitinglist rushes propagate on streaming delivery. (2977_)
+
+* Fix a problem where the ban lurker would skip objects. (3007_)
+
+.. _2395: https://github.com/varnishcache/varnish-cache/issues/2395
+.. _2572: https://github.com/varnishcache/varnish-cache/issues/2572
+.. _2905: https://github.com/varnishcache/varnish-cache/issues/2905
+.. _2923: https://github.com/varnishcache/varnish-cache/issues/2923
+.. _2923: https://github.com/varnishcache/varnish-cache/issues/2930
+.. _2931: https://github.com/varnishcache/varnish-cache/issues/2931
+.. _2933: https://github.com/varnishcache/varnish-cache/pull/2933
+.. _2934: https://github.com/varnishcache/varnish-cache/issues/2934
+.. _2937: https://github.com/varnishcache/varnish-cache/issues/2937
+.. _2938: https://github.com/varnishcache/varnish-cache/issues/2938
+.. _2967: https://github.com/varnishcache/varnish-cache/issues/2967
+.. _2977: https://github.com/varnishcache/varnish-cache/issues/2977
+.. _3007: https://github.com/varnishcache/varnish-cache/issues/3007
+
 ================================
 Varnish Cache 6.0.4 (2019-09-03)
 ================================

From hermunn at varnish-software.com  Fri Oct 18 12:22:06 2019
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Fri, 18 Oct 2019 12:22:06 +0000 (UTC)
Subject: [6.0] c2dc84f20 Fixup: Forgot to ammend the previous commit
Message-ID: <20191018122206.1B6175055@lists.varnish-cache.org>


commit c2dc84f20f75a64f5c246007a04e09abaa89301f
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Fri Oct 18 14:20:53 2019 +0200

    Fixup: Forgot to ammend the previous commit

diff --git a/doc/changes.rst b/doc/changes.rst
index b6110cf59..3ace7977f 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -47,7 +47,7 @@ Varnish Cache 6.0.5 (unreleased)
 .. _2572: https://github.com/varnishcache/varnish-cache/issues/2572
 .. _2905: https://github.com/varnishcache/varnish-cache/issues/2905
 .. _2923: https://github.com/varnishcache/varnish-cache/issues/2923
-.. _2923: https://github.com/varnishcache/varnish-cache/issues/2930
+.. _2930: https://github.com/varnishcache/varnish-cache/issues/2930
 .. _2931: https://github.com/varnishcache/varnish-cache/issues/2931
 .. _2933: https://github.com/varnishcache/varnish-cache/pull/2933
 .. _2934: https://github.com/varnishcache/varnish-cache/issues/2934

From martin at varnish-software.com  Fri Oct 18 13:23:05 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:05 +0000 (UTC)
Subject: [6.0] 45691139e Use VLU to parse the VSM _.index file
Message-ID: <20191018132305.94F0A676D@lists.varnish-cache.org>


commit 45691139e2a73fa17644fa79f88f88709a5978ea
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jul 31 08:59:13 2019 +0000

    Use VLU to parse the VSM _.index file

diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index bc32df9e0..a120c4df6 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -123,7 +123,9 @@ VLU_Fd(struct vlu *l, int fd)
 
 	CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC);
 	i = read(fd, l->buf + l->bufp, l->bufl - l->bufp);
-	if (i <= 0)
+	if (i == 0)
+		return (-2);
+	if (i < 0)
 		return (-1);
 	l->bufp += i;
 	return (LineUpProcess(l));
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index e679de3ac..30935a8d7 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -24,6 +24,7 @@ libvarnishapi_la_SOURCES = \
 	../libvarnish/vfl.c \
 	../libvarnish/vin.c \
 	../libvarnish/vjsn.c \
+	../libvarnish/vlu.c \
 	../libvarnish/vmb.c \
 	../libvarnish/vpf.c \
 	../libvarnish/vre.c \
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 36d737775..17699fc9f 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -51,6 +51,7 @@
 
 #include "vav.h"
 #include "vin.h"
+#include "vlu.h"
 #include "vsb.h"
 #include "vsm_priv.h"
 #include "vqueue.h"
@@ -96,6 +97,7 @@ struct vsm_set {
 	unsigned		magic;
 #define VSM_SET_MAGIC		0xdee401b8
 	const char		*dname;
+	struct vsm		*vsm;
 	VTAILQ_HEAD(,vsm_seg)	segs;
 	VTAILQ_HEAD(,vsm_seg)	stale;
 	VTAILQ_HEAD(,vsm_seg)	clusters;
@@ -107,6 +109,10 @@ struct vsm_set {
 	struct stat		fst;
 
 	uintmax_t		id1, id2;
+
+	// _.index reading state
+	unsigned		retval;
+	struct vsm_seg		*vg;
 };
 
 struct vsm {
@@ -291,6 +297,8 @@ VSM_New(void)
 
 	vd->mgt = vsm_newset(VSM_MGT_DIRNAME);
 	vd->child = vsm_newset(VSM_CHILD_DIRNAME);
+	vd->mgt->vsm = vd;
+	vd->child->vsm = vd;
 	vd->dfd = -1;
 	vd->patience = 5;
 	if (getenv("VSM_NOPID") != NULL)
@@ -418,21 +426,110 @@ vsm_findcluster(const struct vsm_seg *vga)
 	return (NULL);
 }
 
-static unsigned
-vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
+static int
+vsm_vlu_hash(struct vsm *vd, struct vsm_set *vs, const char *line)
 {
-	unsigned retval = 0;
-	struct stat st;
-	char buf[BUFSIZ];
-	ssize_t sz;
-	int i, ac;
-	char *p, *e;
+	int i;
 	uintmax_t id1, id2;
+
+	i = sscanf(line, "# %ju %ju", &id1, &id2);
+	if (i != 2) {
+		vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
+		return (0);
+	}
+	if (vd->couldkill >= 0 && !kill(id1, 0)) {
+		vd->couldkill = 1;
+	} else if (vd->couldkill > 0 && errno == ESRCH) {
+		vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
+		return (0);
+	}
+	vs->retval |= VSM_MGT_RUNNING;
+	if (id1 != vs->id1 || id2 != vs->id2) {
+		vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
+		vs->id1 = id1;
+		vs->id2 = id2;
+	}
+	return (0);
+}
+
+static int
+vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
+{
 	char **av;
-	struct vsm_seg *vg, *vg2;
+	int ac;
+	struct vsm_seg *vg2;
+
+	av = VAV_Parse(line, &ac, 0);
+
+	if (av[0] != NULL || ac < 4 || ac > 6) {
+		(void)(vsm_diag(vd,
+		    "vsm_refresh_set2: bad index (%d/%s)",
+		    ac, av[0]));
+		VAV_Free(av);
+		return(-1);
+	}
+
+	if (vs->vg == NULL) {
+		ALLOC_OBJ(vg2, VSM_SEG_MAGIC);
+		AN(vg2);
+		vg2->av = av;
+		vg2->set = vs;
+		vg2->flags = VSM_FLAG_MARKSCAN;
+		vg2->serial = ++vd->serial;
+		if (ac == 4) {
+			vg2->flags |= VSM_FLAG_CLUSTER;
+			VTAILQ_INSERT_TAIL(&vs->clusters, vg2, list);
+		} else {
+			VTAILQ_INSERT_TAIL(&vs->segs, vg2, list);
+			vg2->cluster = vsm_findcluster(vg2);
+		}
+	} else {
+		while (vs->vg != NULL && vsm_cmp_av(&vs->vg->av[1], &av[1]))
+			vs->vg = VTAILQ_NEXT(vs->vg, list);
+		VAV_Free(av);
+		if (vs->vg != NULL) {
+			/* entry compared equal, so it survives */
+			vs->vg->flags |= VSM_FLAG_MARKSCAN;
+			vs->vg = VTAILQ_NEXT(vs->vg, list);
+		}
+	}
+	return (0);
+}
+
+static int v_matchproto_(vlu_f)
+vsm_vlu_func(void *priv, const char *line)
+{
+	struct vsm *vd;
+	struct vsm_set *vs;
+	int i = -1;
+
+	CAST_OBJ_NOTNULL(vs, priv, VSM_SET_MAGIC);
+	vd = vs->vsm;
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	AN(line);
+
+	if (line[0] == '#') {
+		i = vsm_vlu_hash(vd, vs, line);
+		VTAILQ_FOREACH(vs->vg, &vs->segs, list)
+			vs->vg->flags &= ~VSM_FLAG_MARKSCAN;
+		if (!(vs->retval & VSM_MGT_RESTARTED))
+			vs->vg = VTAILQ_FIRST(&vs->segs);
+	} else {
+		i = vsm_vlu_plus(vd, vs, line);
+	}
+	return (i);
+}
+
+static unsigned
+vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs)
+{
+	struct stat st;
+	int i;
+	struct vlu *vlu;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	CHECK_OBJ_NOTNULL(vs, VSM_SET_MAGIC);
+	vs->retval = 0;
 	if (vs->dfd >= 0) {
 		if (fstatat(vd->dfd, vs->dname, &st, AT_SYMLINK_NOFOLLOW)) {
 			closefd(&vs->dfd);
@@ -451,10 +548,10 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 		if (vs->fd >= 0)
 			closefd(&vs->fd);
 		vs->dfd = openat(vd->dfd, vs->dname, O_RDONLY);
-		retval |= VSM_MGT_RESTARTED;
+		vs->retval |= VSM_MGT_RESTARTED;
 		if (vs->dfd < 0) {
 			vs->id1 = vs->id2 = 0;
-			return (retval|VSM_NUKE_ALL);
+			return (vs->retval|VSM_NUKE_ALL);
 		}
 		AZ(fstat(vs->dfd, &vs->dst));
 	}
@@ -472,123 +569,36 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 
 	if (vs->fd >= 0) {
 		if (vd->couldkill < 1 || !kill(vs->id1, 0))
-			retval |= VSM_MGT_RUNNING;
-		return (retval);
+			vs->retval |= VSM_MGT_RUNNING;
+		return (vs->retval);
 	}
 
-	retval |= VSM_MGT_CHANGED;
+	vs->retval |= VSM_MGT_CHANGED;
 	vs->fd = openat(vs->dfd, "_.index", O_RDONLY);
 	if (vs->fd < 0)
-		return (retval|VSM_MGT_RESTARTED);
+		return (vs->retval|VSM_MGT_RESTARTED);
 
 	AZ(fstat(vs->fd, &vs->fst));
 
-	VSB_clear(vsb);
-	do {
-		sz = read(vs->fd, buf, sizeof buf);
-		if (sz > 0)
-			VSB_bcat(vsb, buf, sz);
-	} while (sz > 0);
-	AZ(VSB_finish(vsb));
-
-	vs->fst.st_size = VSB_len(vsb);
-
-	if (VSB_len(vsb) == 0)
-		return (retval|VSM_NUKE_ALL);
-
-	/*
-	 * First line is ident comment
-	 */
-	i = sscanf(VSB_data(vsb), "# %ju %ju\n%n", &id1, &id2, &ac);
-	if (i != 2) {
-		retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
-		return (retval);
-	}
-	if (vd->couldkill >= 0 && !kill(id1, 0)) {
-		vd->couldkill = 1;
-	} else if (vd->couldkill > 0 && errno == ESRCH) {
-		retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
-		return (retval);
-	}
-	retval |= VSM_MGT_RUNNING;
-	if (id1 != vs->id1 || id2 != vs->id2) {
-		retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
-		vs->id1 = id1;
-		vs->id2 = id2;
-	}
-	p = VSB_data(vsb) + ac;
+	vlu = VLU_New(vsm_vlu_func, vs, 0);
+	AN(vlu);
 
-	VTAILQ_FOREACH(vg, &vs->segs, list)
-		vg->flags &= ~VSM_FLAG_MARKSCAN;
-
-	/*
-	 * Efficient comparison by walking the two lists side-by-side because
-	 * segment inserts always happen at the tail (VSMW_Allocv()). So, as
-	 * soon as vg is exhausted, we only insert.
-	 *
-	 * For restarts, we require a tabula rasa
-	 */
-
-	if (retval & VSM_MGT_RESTARTED)
-		vg = NULL;
-	else
-		vg = VTAILQ_FIRST(&vs->segs);
-
-	while (p != NULL && *p != '\0') {
-		e = strchr(p, '\n');
-		if (e == NULL)
-			break;
-		*e = '\0';
-		av = VAV_Parse(p, &ac, 0);
-		p = e + 1;
-
-		if (av[0] != NULL || ac < 4 || ac > 6) {
-			(void)(vsm_diag(vd,
-			    "vsm_refresh_set2: bad index (%d/%s)",
-			    ac, av[0]));
-			VAV_Free(av);
-			break;
-		}
-
-		if (vg == NULL) {
-			ALLOC_OBJ(vg2, VSM_SEG_MAGIC);
-			AN(vg2);
-			vg2->av = av;
-			vg2->set = vs;
-			vg2->flags = VSM_FLAG_MARKSCAN;
-			vg2->serial = ++vd->serial;
-			if (ac == 4) {
-				vg2->flags |= VSM_FLAG_CLUSTER;
-				VTAILQ_INSERT_TAIL(&vs->clusters, vg2, list);
-			} else {
-				VTAILQ_INSERT_TAIL(&vs->segs, vg2, list);
-				vg2->cluster = vsm_findcluster(vg2);
-			}
-			continue;
-		}
-
-		while (vg != NULL && vsm_cmp_av(&vg->av[1], &av[1]))
-			vg = VTAILQ_NEXT(vg, list);
-
-		VAV_Free(av);
-
-		if (vg == NULL)
-			continue;
-
-		/* entry compared equal, so it survives */
-		vg->flags |= VSM_FLAG_MARKSCAN;
-		vg = VTAILQ_NEXT(vg, list);
-	}
-	return (retval);
+	vs->vg = NULL;
+	do {
+		i = VLU_Fd(vlu, vs->fd);
+	} while (!i);
+	assert(i == -2);
+	VLU_Destroy(&vlu);
+	return (vs->retval);
 }
 
 static unsigned
-vsm_refresh_set(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
+vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 {
 	unsigned retval;
 	struct vsm_seg *vg, *vg2;
 
-	retval = vsm_refresh_set2(vd, vs, vsb);
+	retval = vsm_refresh_set2(vd, vs);
 	if (retval & VSM_NUKE_ALL)
 		retval |= VSM_MGT_CHANGED;
 	VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
@@ -614,7 +624,6 @@ VSM_Status(struct vsm *vd)
 {
 	unsigned retval = 0, u;
 	struct stat st;
-	struct vsb *vsb;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
@@ -641,14 +650,10 @@ VSM_Status(struct vsm *vd)
 			AZ(fstat(vd->dfd, &vd->dst));
 	}
 
-	vsb = VSB_new_auto();
-	AN(vsb);
-
-	u = vsm_refresh_set(vd, vd->mgt, vsb);
+	u = vsm_refresh_set(vd, vd->mgt);
 	retval |= u;
 	if (u & VSM_MGT_RUNNING)
-		retval |= vsm_refresh_set(vd, vd->child, vsb) << 8;
-	VSB_destroy(&vsb);
+		retval |= vsm_refresh_set(vd, vd->child) << 8;
 	return (retval);
 }
 

From martin at varnish-software.com  Fri Oct 18 13:23:05 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:05 +0000 (UTC)
Subject: [6.0] f4059aa68 Prefix VSM lines which add segments with a '+'
Message-ID: <20191018132305.A98B96770@lists.varnish-cache.org>


commit f4059aa68a0df829d24e383b7408ddff5ca49434
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jul 31 19:55:42 2019 +0000

    Prefix VSM lines which add segments with a '+'

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 339394b58..a71d3e307 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -132,7 +132,7 @@ vsmw_fmt_index(const struct vsmw *vsmw, const struct vsmwseg *seg)
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
-	VSB_printf(vsmw->vsb, "%s %zu %zu %s %s\n",
+	VSB_printf(vsmw->vsb, "+ %s %zu %zu %s %s\n",
 	    seg->cluster->fn,
 	    seg->off,
 	    seg->len,
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 17699fc9f..1e200e75f 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -459,7 +459,7 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 	int ac;
 	struct vsm_seg *vg2;
 
-	av = VAV_Parse(line, &ac, 0);
+	av = VAV_Parse(line + 1, &ac, 0);
 
 	if (av[0] != NULL || ac < 4 || ac > 6) {
 		(void)(vsm_diag(vd,
@@ -508,14 +508,19 @@ vsm_vlu_func(void *priv, const char *line)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(line);
 
-	if (line[0] == '#') {
+	switch (line[0]) {
+	case '#':
 		i = vsm_vlu_hash(vd, vs, line);
 		VTAILQ_FOREACH(vs->vg, &vs->segs, list)
 			vs->vg->flags &= ~VSM_FLAG_MARKSCAN;
 		if (!(vs->retval & VSM_MGT_RESTARTED))
 			vs->vg = VTAILQ_FIRST(&vs->segs);
-	} else {
+		break;
+	case '+':
 		i = vsm_vlu_plus(vd, vs, line);
+		break;
+	default:
+		break;
 	}
 	return (i);
 }

From martin at varnish-software.com  Fri Oct 18 13:23:05 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:05 +0000 (UTC)
Subject: [6.0] 3f436c0cf Avoid lock/unlock/lock/unlock for cluster delete
Message-ID: <20191018132305.C36056773@lists.varnish-cache.org>


commit 3f436c0cf7ca3684a9989ba4605b20bc77b59d04
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 1 07:33:40 2019 +0000

    Avoid lock/unlock/lock/unlock for cluster delete

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index a71d3e307..c865fb233 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -285,7 +285,6 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
 {
 	struct vsmw_cluster *vc;
 
-	vsmw_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	AN(vsmcp);
 	vc = *vsmcp;
@@ -301,10 +300,8 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
 		 */
 		vsmw_delseg(vsmw, vc->cseg, 1);
 		vc->cseg = NULL;
-		if (vc->refs > 0) {
-			vsmw_unlock();
+		if (vc->refs > 0)
 			return;
-		}
 	}
 	AZ(munmap(vc->ptr, vc->len));
 
@@ -314,7 +311,6 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
 		assert (errno == ENOENT);
 	REPLACE(vc->fn, NULL);
 	FREE_OBJ(vc);
-	vsmw_unlock();
 }
 
 /*--------------------------------------------------------------------*/
@@ -376,26 +372,24 @@ void
 VSMW_Free(struct vsmw *vsmw, void **pp)
 {
 	struct vsmwseg *seg;
-	void *p;
+	struct vsmw_cluster *cp;
 
 	vsmw_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	AN(pp);
-	p = *pp;
-	AN(p);
-	*pp = NULL;
 	VTAILQ_FOREACH(seg, &vsmw->segs, list)
-		if (seg->ptr == p)
+		if (seg->ptr == *pp)
 			break;
 	AN(seg);
+	*pp = NULL;
 
-	if (!--seg->cluster->refs && seg->cluster->cseg == NULL) {
-		vsmw_unlock();
-		VSMW_DestroyCluster(vsmw, &seg->cluster);
-		vsmw_lock();
-	}
+	cp = seg->cluster;
 
 	vsmw_delseg(vsmw, seg, 1);
+
+	if (!--cp->refs && cp->cseg == NULL)
+		VSMW_DestroyCluster(vsmw, &cp);
+
 	vsmw_unlock();
 }
 

From martin at varnish-software.com  Fri Oct 18 13:23:05 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:05 +0000 (UTC)
Subject: [6.0] d801eb11d Introduce '-' records in VSM _.index files,
 to reduce VFS workload.
Message-ID: <20191018132305.E01BF6779@lists.varnish-cache.org>


commit d801eb11d033154c8c483c12b1bfd7af2ef64ede
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 1 08:00:56 2019 +0000

    Introduce '-' records in VSM _.index files, to reduce VFS workload.

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index c865fb233..3fa89c1ce 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -113,6 +113,8 @@ struct vsmw {
 	struct vsb			*vsb;
 	pid_t				pid;
 	time_t				birth;
+	uint64_t			nsegs;
+	uint64_t			nsubs;
 };
 
 /*--------------------------------------------------------------------*/
@@ -127,12 +129,14 @@ vsmw_idx_head(const struct vsmw *vsmw, int fd)
 }
 
 static void
-vsmw_fmt_index(const struct vsmw *vsmw, const struct vsmwseg *seg)
+vsmw_fmt_index(const struct vsmw *vsmw, const struct vsmwseg *seg, char act)
 {
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
-	VSB_printf(vsmw->vsb, "+ %s %zu %zu %s %s\n",
+	AN(seg->cluster);
+	VSB_printf(vsmw->vsb, "%c %s %zu %zu %s %s\n",
+	    act,
 	    seg->cluster->fn,
 	    seg->off,
 	    seg->len,
@@ -166,20 +170,33 @@ vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
 /*--------------------------------------------------------------------*/
 
 static void
-vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg)
+vsmw_append_record(struct vsmw *vsmw, struct vsmwseg *seg, char act)
 {
 	int fd;
 	ssize_t s;
 
-	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
 	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
 	assert(fd >= 0);
 	VSB_clear(vsmw->vsb);
-	vsmw_fmt_index(vsmw, seg);
+	vsmw_fmt_index(vsmw, seg, act);
 	AZ(VSB_finish(vsmw->vsb));
 	s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
-	assert(s == VSB_len(vsmw->vsb));
+	assert(s == VSB_len(vsmw->vsb)); // XXX handle ENOSPC? #2764
 	AZ(close(fd));
+	vsmw->nsegs++;
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg)
+{
+
+	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
+	vsmw_append_record(vsmw, seg, '+');
+	vsmw->nsegs++;
 }
 
 /*--------------------------------------------------------------------*/
@@ -190,16 +207,19 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 	char *t = NULL;
 	ssize_t s;
 	int fd;
+	struct vsmwseg *s2;
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
 
 	VTAILQ_REMOVE(&vsmw->segs, seg, list);
-	REPLACE(seg->class, NULL);
-	REPLACE(seg->id, NULL);
-	FREE_OBJ(seg);
 
-	if (fixidx) {
+	vsmw->nsegs--;
+	if (vsmw->nsubs < vsmw->nsegs || !fixidx) {
+		vsmw_append_record(vsmw, seg, '-');
+		vsmw->nsubs++;
+	} else {
+		vsmw->nsubs = 0;
 		vsmw_mkent(vsmw, vsmw->idx);
 		REPLACE(t, VSB_data(vsmw->vsb));
 		AN(t);
@@ -208,8 +228,8 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 		assert(fd >= 0);
 		vsmw_idx_head(vsmw, fd);
 		VSB_clear(vsmw->vsb);
-		VTAILQ_FOREACH(seg, &vsmw->segs, list)
-			vsmw_fmt_index(vsmw, seg);
+		VTAILQ_FOREACH(s2, &vsmw->segs, list)
+			vsmw_fmt_index(vsmw, s2, '+');
 		AZ(VSB_finish(vsmw->vsb));
 		s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
 		assert(s == VSB_len(vsmw->vsb));
@@ -217,6 +237,9 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
 		REPLACE(t, NULL);
 	}
+	REPLACE(seg->class, NULL);
+	REPLACE(seg->id, NULL);
+	FREE_OBJ(seg);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 1e200e75f..43f3394d8 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -463,7 +463,7 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 
 	if (av[0] != NULL || ac < 4 || ac > 6) {
 		(void)(vsm_diag(vd,
-		    "vsm_refresh_set2: bad index (%d/%s)",
+		    "vsm_vlu_plus: bad index (%d/%s)",
 		    ac, av[0]));
 		VAV_Free(av);
 		return(-1);
@@ -496,6 +496,39 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 	return (0);
 }
 
+static int
+vsm_vlu_minus(struct vsm *vd, struct vsm_set *vs, const char *line)
+{
+	char **av;
+	int ac;
+	struct vsm_seg *vg, *vg2;
+
+	av = VAV_Parse(line + 1, &ac, 0);
+
+	if (av[0] != NULL || ac < 4 || ac > 6) {
+		(void)(vsm_diag(vd,
+		    "vsm_vlu_minus: bad index (%d/%s)",
+		    ac, av[0]));
+		VAV_Free(av);
+		return(-1);
+	}
+
+	VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
+		if (vsm_cmp_av(&vg->av[1], &av[1]))
+			continue;
+		VTAILQ_REMOVE(&vs->segs, vg, list);
+		if (vg->refs) {
+			vg->flags |= VSM_FLAG_STALE;
+			VTAILQ_INSERT_TAIL(&vs->stale, vg, list);
+		} else {
+			VAV_Free(vg->av);
+			FREE_OBJ(vg);
+		}
+		break;
+	}
+	return (0);
+}
+
 static int v_matchproto_(vlu_f)
 vsm_vlu_func(void *priv, const char *line)
 {
@@ -519,6 +552,9 @@ vsm_vlu_func(void *priv, const char *line)
 	case '+':
 		i = vsm_vlu_plus(vd, vs, line);
 		break;
+	case '-':
+		i = vsm_vlu_minus(vd, vs, line);
+		break;
 	default:
 		break;
 	}

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] d3be9d0ec Fix locking mistake introduced in 5f97b89
Message-ID: <20191018132306.077226780@lists.varnish-cache.org>


commit d3be9d0ec9ee24a8a7615e00d0af9a1f353ecf79
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 5 08:27:34 2019 +0000

    Fix locking mistake introduced in 5f97b89

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 3fa89c1ce..a98cfeae9 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -71,9 +71,29 @@ vsmw_dummy_lock(void)
 {
 }
 
+static int vsmw_haslock;
 vsm_lock_f *vsmw_lock = vsmw_dummy_lock;
 vsm_lock_f *vsmw_unlock = vsmw_dummy_lock;
 
+#define vsmw_assert_lock()	AN(vsmw_haslock)
+
+#define vsmw_do_lock() vsmw_do_lock_(__func__, __LINE__)
+
+#define vsmw_do_lock_(f, l) 				\
+	do {					\
+		vsmw_lock();			\
+		AZ(vsmw_haslock);		\
+		vsmw_haslock = 1;		\
+	} while(0)
+
+#define vsmw_do_unlock() vsmw_do_unlock_(__func__, __LINE__)
+#define vsmw_do_unlock_(f, l) 				\
+	do {					\
+		AN(vsmw_haslock);		\
+		vsmw_haslock = 0;		\
+		vsmw_unlock();			\
+	} while(0)
+
 /*--------------------------------------------------------------------*/
 
 struct vsmw_cluster {
@@ -153,6 +173,7 @@ vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
 	uint64_t rn;
 
 	AN(pfx);
+	vsmw_assert_lock();
 	while (1) {
 		VSB_clear(vsmw->vsb);
 		VSB_printf(vsmw->vsb, "_.%s", pfx);
@@ -194,6 +215,7 @@ static void
 vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg)
 {
 
+	vsmw_assert_lock();
 	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
 	vsmw_append_record(vsmw, seg, '+');
 	vsmw->nsegs++;
@@ -209,6 +231,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 	int fd;
 	struct vsmwseg *s2;
 
+	vsmw_assert_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
 
@@ -287,7 +310,7 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	struct vsmw_cluster *vc;
 	struct vsmwseg *seg;
 
-	vsmw_lock();
+	vsmw_do_lock();
 	vc = vsmw_newcluster(vsmw, len, pfx);
 
 	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
@@ -299,12 +322,12 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	REPLACE(seg->id, "");
 	vsmw_addseg(vsmw, seg);
 
-	vsmw_unlock();
+	vsmw_do_unlock();
 	return (vc);
 }
 
-void
-VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
+static void
+vsmw_DestroyCluster_locked(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
 {
 	struct vsmw_cluster *vc;
 
@@ -336,6 +359,15 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
 	FREE_OBJ(vc);
 }
 
+void
+VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
+{
+
+	vsmw_do_lock();
+	vsmw_DestroyCluster_locked(vsmw, vsmcp);
+	vsmw_do_unlock();
+}
+
 /*--------------------------------------------------------------------*/
 
 void *
@@ -345,7 +377,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 {
 	struct vsmwseg *seg;
 
-	vsmw_lock();
+	vsmw_do_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	(void)vc;
 
@@ -372,7 +404,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 
 	vsmw_addseg(vsmw, seg);
 
-	vsmw_unlock();
+	vsmw_do_unlock();
 	return (seg->ptr);
 }
 
@@ -397,7 +429,7 @@ VSMW_Free(struct vsmw *vsmw, void **pp)
 	struct vsmwseg *seg;
 	struct vsmw_cluster *cp;
 
-	vsmw_lock();
+	vsmw_do_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	AN(pp);
 	VTAILQ_FOREACH(seg, &vsmw->segs, list)
@@ -411,9 +443,8 @@ VSMW_Free(struct vsmw *vsmw, void **pp)
 	vsmw_delseg(vsmw, seg, 1);
 
 	if (!--cp->refs && cp->cseg == NULL)
-		VSMW_DestroyCluster(vsmw, &cp);
-
-	vsmw_unlock();
+		vsmw_DestroyCluster_locked(vsmw, &cp);
+	vsmw_do_unlock();
 }
 
 /*--------------------------------------------------------------------*/
@@ -428,7 +459,7 @@ VSMW_New(int vdirfd, int mode, const char *idxname)
 	assert(mode > 0);
 	AN(idxname);
 
-	vsmw_lock();
+	vsmw_do_lock();
 	ALLOC_OBJ(vsmw, VSMW_MAGIC);
 	AN(vsmw);
 
@@ -450,7 +481,7 @@ VSMW_New(int vdirfd, int mode, const char *idxname)
 	vsmw_idx_head(vsmw, fd);
 	AZ(close(fd));
 
-	vsmw_unlock();
+	vsmw_do_unlock();
 	return (vsmw);
 }
 
@@ -460,7 +491,7 @@ VSMW_Destroy(struct vsmw **pp)
 	struct vsmw *vsmw;
 	struct vsmwseg *seg, *s2;
 
-	vsmw_lock();
+	vsmw_do_lock();
 	TAKE_OBJ_NOTNULL(vsmw, pp, VSMW_MAGIC);
 	VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
 		vsmw_delseg(vsmw, seg, 0);
@@ -470,5 +501,5 @@ VSMW_Destroy(struct vsmw **pp)
 	VSB_destroy(&vsmw->vsb);
 	AZ(close(vsmw->vdirfd));
 	FREE_OBJ(vsmw);
-	vsmw_unlock();
+	vsmw_do_unlock();
 }

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] 3c338fcb3 Properly count vsm segments and rebuild _.index when
 retired segments dominate the _.index file.
Message-ID: <20191018132306.26A546785@lists.varnish-cache.org>


commit 3c338fcb35e37c9d0875600f5a6615853cc6c8e8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 5 10:25:36 2019 +0000

    Properly count vsm segments and rebuild _.index when retired
    segments dominate the _.index file.

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index a98cfeae9..0439084df 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -206,7 +206,6 @@ vsmw_append_record(struct vsmw *vsmw, struct vsmwseg *seg, char act)
 	s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
 	assert(s == VSB_len(vsmw->vsb)); // XXX handle ENOSPC? #2764
 	AZ(close(fd));
-	vsmw->nsegs++;
 }
 
 /*--------------------------------------------------------------------*/
@@ -238,7 +237,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 	VTAILQ_REMOVE(&vsmw->segs, seg, list);
 
 	vsmw->nsegs--;
-	if (vsmw->nsubs < vsmw->nsegs || !fixidx) {
+	if (vsmw->nsubs * 2 < vsmw->nsegs || !fixidx) {
 		vsmw_append_record(vsmw, seg, '-');
 		vsmw->nsubs++;
 	} else {

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] 8053650b0 Test VSM _.index rewrites when more than half of the
 space is inert
Message-ID: <20191018132306.40D926789@lists.varnish-cache.org>


commit 8053650b09f5b375ee38a97b551fe663d2118b77
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 5 10:36:04 2019 +0000

    Test VSM _.index rewrites when more than half of the space is inert

diff --git a/bin/varnishtest/tests/c00083.vtc b/bin/varnishtest/tests/c00083.vtc
new file mode 100644
index 000000000..67f52b612
--- /dev/null
+++ b/bin/varnishtest/tests/c00083.vtc
@@ -0,0 +1,85 @@
+varnishtest "Test VSM _.index rewrite when too many deletes"
+
+varnish v1 -vcl {
+	backend default { .host = "${bad_ip}"; }
+} -start
+
+process p1 {
+	nlines=`wc -l < ${tmpdir}/v1/_.vsm_child/_.index`
+	nminus=`grep -c '^-' ${tmpdir}/v1/_.vsm_child/_.index`
+	echo NLINES $nlines NMINUS $nminus
+} -dump -run
+
+# The child process starts out with approx 37 VSM segments
+# so it takes 20 backends to cause a _.index rewrite.
+# Make it 25 to be safe.
+varnish v1 -vcl {
+	backend b00 { .host = "${bad_ip}"; }
+	backend b01 { .host = "${bad_ip}"; }
+	backend b02 { .host = "${bad_ip}"; }
+	backend b03 { .host = "${bad_ip}"; }
+	backend b04 { .host = "${bad_ip}"; }
+	backend b05 { .host = "${bad_ip}"; }
+	backend b06 { .host = "${bad_ip}"; }
+	backend b07 { .host = "${bad_ip}"; }
+	backend b08 { .host = "${bad_ip}"; }
+	backend b09 { .host = "${bad_ip}"; }
+	backend b10 { .host = "${bad_ip}"; }
+	backend b11 { .host = "${bad_ip}"; }
+	backend b12 { .host = "${bad_ip}"; }
+	backend b13 { .host = "${bad_ip}"; }
+	backend b14 { .host = "${bad_ip}"; }
+	backend b15 { .host = "${bad_ip}"; }
+	backend b16 { .host = "${bad_ip}"; }
+	backend b17 { .host = "${bad_ip}"; }
+	backend b18 { .host = "${bad_ip}"; }
+	backend b19 { .host = "${bad_ip}"; }
+	backend b20 { .host = "${bad_ip}"; }
+	backend b21 { .host = "${bad_ip}"; }
+	backend b22 { .host = "${bad_ip}"; }
+	backend b23 { .host = "${bad_ip}"; }
+	backend b24 { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		set req.backend_hint = b00;
+		set req.backend_hint = b01;
+		set req.backend_hint = b02;
+		set req.backend_hint = b03;
+		set req.backend_hint = b04;
+		set req.backend_hint = b05;
+		set req.backend_hint = b06;
+		set req.backend_hint = b07;
+		set req.backend_hint = b08;
+		set req.backend_hint = b09;
+		set req.backend_hint = b10;
+		set req.backend_hint = b11;
+		set req.backend_hint = b12;
+		set req.backend_hint = b13;
+		set req.backend_hint = b14;
+		set req.backend_hint = b15;
+		set req.backend_hint = b16;
+		set req.backend_hint = b17;
+		set req.backend_hint = b18;
+		set req.backend_hint = b19;
+		set req.backend_hint = b20;
+		set req.backend_hint = b21;
+		set req.backend_hint = b22;
+		set req.backend_hint = b23;
+		set req.backend_hint = b24;
+	}
+}
+
+varnish v1 -cliok vcl.list
+
+process p1 -run
+
+varnish v1 -cliok "vcl.use vcl1"
+varnish v1 -cliok "vcl.discard vcl2"
+
+# Check that the _.index rewrite did happen
+process p1 {
+	nlines=`wc -l < ${tmpdir}/v1/_.vsm_child/_.index`
+	nminus=`grep -c '^-' ${tmpdir}/v1/_.vsm_child/_.index`
+	echo NLINES $nlines NMINUS $nminus
+	test $nminus -lt 25
+} -run

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] db7f56cd6 Refactor code a bit
Message-ID: <20191018132306.5AF326790@lists.varnish-cache.org>


commit db7f56cd62e539379d791d53ac755a091a64e70b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 7 12:16:40 2019 +0000

    Refactor code a bit

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 43f3394d8..c3c75ee88 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -285,6 +285,25 @@ vsm_delset(struct vsm_set **p)
 	FREE_OBJ(vs);
 }
 
+static void
+vsm_wash_set(struct vsm_set *vs, int all)
+{
+	struct vsm_seg *vg, *vg2;
+
+	VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
+		if (all || (vg->flags & VSM_FLAG_MARKSCAN) == 0) {
+			VTAILQ_REMOVE(&vs->segs, vg, list);
+			if (vg->refs) {
+				vg->flags |= VSM_FLAG_STALE;
+				VTAILQ_INSERT_TAIL(&vs->stale, vg, list);
+			} else {
+				VAV_Free(vg->av);
+				FREE_OBJ(vg);
+			}
+		}
+	}
+}
+
 /*--------------------------------------------------------------------*/
 
 struct vsm *
@@ -393,8 +412,6 @@ VSM_ResetError(struct vsm *vd)
 /*--------------------------------------------------------------------
  */
 
-#define VSM_NUKE_ALL	(1U << 16)
-
 static int
 vsm_cmp_av(char * const *a1, char * const *a2)
 {
@@ -562,7 +579,7 @@ vsm_vlu_func(void *priv, const char *line)
 }
 
 static unsigned
-vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs)
+vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 {
 	struct stat st;
 	int i;
@@ -575,7 +592,8 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs)
 		if (fstatat(vd->dfd, vs->dname, &st, AT_SYMLINK_NOFOLLOW)) {
 			closefd(&vs->dfd);
 			vs->id1 = vs->id2 = 0;
-			return (VSM_MGT_RESTARTED|VSM_NUKE_ALL);
+			vsm_wash_set(vs, 1);
+			return (VSM_MGT_RESTARTED|VSM_MGT_CHANGED);
 		}
 		if (st.st_ino != vs->dst.st_ino ||
 		    st.st_dev != vs->dst.st_dev ||
@@ -592,7 +610,8 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs)
 		vs->retval |= VSM_MGT_RESTARTED;
 		if (vs->dfd < 0) {
 			vs->id1 = vs->id2 = 0;
-			return (vs->retval|VSM_NUKE_ALL);
+			vsm_wash_set(vs, 1);
+			return (vs->retval|VSM_MGT_CHANGED);
 		}
 		AZ(fstat(vs->dfd, &vs->dst));
 	}
@@ -630,34 +649,10 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs)
 	} while (!i);
 	assert(i == -2);
 	VLU_Destroy(&vlu);
+	vsm_wash_set(vs, 0);
 	return (vs->retval);
 }
 
-static unsigned
-vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
-{
-	unsigned retval;
-	struct vsm_seg *vg, *vg2;
-
-	retval = vsm_refresh_set2(vd, vs);
-	if (retval & VSM_NUKE_ALL)
-		retval |= VSM_MGT_CHANGED;
-	VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
-		if ((vg->flags & VSM_FLAG_MARKSCAN) == 0 ||
-		    (retval & VSM_NUKE_ALL)) {
-			VTAILQ_REMOVE(&vs->segs, vg, list);
-			if (vg->refs) {
-				vg->flags |= VSM_FLAG_STALE;
-				VTAILQ_INSERT_TAIL(&vs->stale, vg, list);
-			} else {
-				VAV_Free(vg->av);
-				FREE_OBJ(vg);
-			}
-		}
-	}
-	return (retval & ~VSM_NUKE_ALL);
-}
-
 /*--------------------------------------------------------------------*/
 
 unsigned

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] 418e3365b Refactor & Polish
Message-ID: <20191018132306.713866797@lists.varnish-cache.org>


commit 418e3365b3b717a96577c8480499fa659490102b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 7 13:34:22 2019 +0000

    Refactor & Polish

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index c3c75ee88..2ef04edec 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -113,6 +113,10 @@ struct vsm_set {
 	// _.index reading state
 	unsigned		retval;
 	struct vsm_seg		*vg;
+
+	unsigned		flag_running;
+	unsigned		flag_changed;
+	unsigned		flag_restarted;
 };
 
 struct vsm {
@@ -122,9 +126,9 @@ struct vsm {
 	struct vsb		*diag;
 	uintptr_t		serial;
 
-	int			dfd;
-	struct stat		dst;
-	char			*dname;
+	int			wdfd;
+	struct stat		wdst;
+	char			*wdname;
 
 	struct vsm_set		*mgt;
 	struct vsm_set		*child;
@@ -186,7 +190,7 @@ vsm_mapseg(struct vsm *vd, struct vsm_seg *vg)
 
 	vsb = VSB_new_auto();
 	AN(vsb);
-	VSB_printf(vsb, "%s/%s/%s", vd->dname, vg->set->dname, vg->av[1]);
+	VSB_printf(vsb, "%s/%s/%s", vd->wdname, vg->set->dname, vg->av[1]);
 	AZ(VSB_finish(vsb));
 
 	fd = open(VSB_data(vsb), O_RDONLY);	// XXX: openat
@@ -315,10 +319,18 @@ VSM_New(void)
 	AN(vd);
 
 	vd->mgt = vsm_newset(VSM_MGT_DIRNAME);
+	vd->mgt->flag_running = VSM_MGT_RUNNING;
+	vd->mgt->flag_changed = VSM_MGT_CHANGED;
+	vd->mgt->flag_restarted = VSM_MGT_RESTARTED;
+
 	vd->child = vsm_newset(VSM_CHILD_DIRNAME);
+	vd->child->flag_running = VSM_WRK_RUNNING;
+	vd->child->flag_changed = VSM_WRK_CHANGED;
+	vd->child->flag_restarted = VSM_WRK_RESTARTED;
+
 	vd->mgt->vsm = vd;
 	vd->child->vsm = vd;
-	vd->dfd = -1;
+	vd->wdfd = -1;
 	vd->patience = 5;
 	if (getenv("VSM_NOPID") != NULL)
 		vd->couldkill = -1;
@@ -353,7 +365,7 @@ VSM_Arg(struct vsm *vd, char flag, const char *arg)
 			return (vsm_diag(vd, "Invalid instance name: %s",
 			    strerror(errno)));
 		AN(p);
-		REPLACE(vd->dname, p);
+		REPLACE(vd->wdname, p);
 		free(p);
 		break;
 	default:
@@ -372,11 +384,11 @@ VSM_Destroy(struct vsm **vdp)
 	TAKE_OBJ_NOTNULL(vd, vdp, VSM_MAGIC);
 
 	VSM_ResetError(vd);
-	REPLACE(vd->dname, NULL);
+	REPLACE(vd->wdname, NULL);
 	if (vd->diag != NULL)
 		VSB_destroy(&vd->diag);
-	if (vd->dfd >= 0)
-		closefd(&vd->dfd);
+	if (vd->wdfd >= 0)
+		closefd(&vd->wdfd);
 	vsm_delset(&vd->mgt);
 	vsm_delset(&vd->child);
 	FREE_OBJ(vd);
@@ -563,7 +575,7 @@ vsm_vlu_func(void *priv, const char *line)
 		i = vsm_vlu_hash(vd, vs, line);
 		VTAILQ_FOREACH(vs->vg, &vs->segs, list)
 			vs->vg->flags &= ~VSM_FLAG_MARKSCAN;
-		if (!(vs->retval & VSM_MGT_RESTARTED))
+		if (!(vs->retval & vs->flag_restarted))
 			vs->vg = VTAILQ_FIRST(&vs->segs);
 		break;
 	case '+':
@@ -589,29 +601,27 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 	CHECK_OBJ_NOTNULL(vs, VSM_SET_MAGIC);
 	vs->retval = 0;
 	if (vs->dfd >= 0) {
-		if (fstatat(vd->dfd, vs->dname, &st, AT_SYMLINK_NOFOLLOW)) {
-			closefd(&vs->dfd);
-			vs->id1 = vs->id2 = 0;
-			vsm_wash_set(vs, 1);
-			return (VSM_MGT_RESTARTED|VSM_MGT_CHANGED);
-		}
-		if (st.st_ino != vs->dst.st_ino ||
+		if (fstatat(vd->wdfd, vs->dname, &st, AT_SYMLINK_NOFOLLOW) ||
+		    st.st_ino != vs->dst.st_ino ||
 		    st.st_dev != vs->dst.st_dev ||
-		    st.st_mode != vs->dst.st_mode) {
+		    st.st_mode != vs->dst.st_mode ||
+		    st.st_nlink == 0) {
 			closefd(&vs->dfd);
 			vs->id1 = vs->id2 = 0;
+			vsm_wash_set(vs, 1);
+			vs->retval |= vs->flag_restarted;
 		}
 	}
 
 	if (vs->dfd < 0) {
 		if (vs->fd >= 0)
 			closefd(&vs->fd);
-		vs->dfd = openat(vd->dfd, vs->dname, O_RDONLY);
-		vs->retval |= VSM_MGT_RESTARTED;
+		vs->dfd = openat(vd->wdfd, vs->dname, O_RDONLY);
 		if (vs->dfd < 0) {
 			vs->id1 = vs->id2 = 0;
 			vsm_wash_set(vs, 1);
-			return (vs->retval|VSM_MGT_CHANGED);
+			vs->retval |= vs->flag_restarted;
+			return (vs->retval);
 		}
 		AZ(fstat(vs->dfd, &vs->dst));
 	}
@@ -629,14 +639,14 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 
 	if (vs->fd >= 0) {
 		if (vd->couldkill < 1 || !kill(vs->id1, 0))
-			vs->retval |= VSM_MGT_RUNNING;
+			vs->retval |= vs->flag_running;
 		return (vs->retval);
 	}
 
-	vs->retval |= VSM_MGT_CHANGED;
+	vs->retval |= vs->flag_changed;
 	vs->fd = openat(vs->dfd, "_.index", O_RDONLY);
 	if (vs->fd < 0)
-		return (vs->retval|VSM_MGT_RESTARTED);
+		return (vs->retval|vs->flag_restarted);
 
 	AZ(fstat(vs->fd, &vs->fst));
 
@@ -658,38 +668,41 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 unsigned
 VSM_Status(struct vsm *vd)
 {
-	unsigned retval = 0, u;
+	unsigned retval = 0;
 	struct stat st;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
 	/* See if the -n workdir changed */
-	if (vd->dfd >= 0) {
-		AZ(fstat(vd->dfd, &st));
-		if (st.st_ino != vd->dst.st_ino ||
-		    st.st_dev != vd->dst.st_dev ||
-		    st.st_mode != vd->dst.st_mode ||
+	if (vd->wdfd >= 0) {
+		AZ(fstat(vd->wdfd, &st));
+		if (st.st_ino != vd->wdst.st_ino ||
+		    st.st_dev != vd->wdst.st_dev ||
+		    st.st_mode != vd->wdst.st_mode ||
 		    st.st_nlink == 0) {
-			closefd(&vd->dfd);
-			retval |= VSM_MGT_CHANGED;
-			retval |= VSM_WRK_CHANGED;
+			closefd(&vd->wdfd);
+			vsm_wash_set(vd->mgt, 1);
+			vsm_wash_set(vd->child, 1);
 		}
 	}
 
 	/* Open workdir */
-	if (vd->dfd < 0) {
-		vd->dfd = open(vd->dname, O_RDONLY);
-		if (vd->dfd < 0)
+	if (vd->wdfd < 0) {
+		retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
+		retval |= VSM_WRK_RESTARTED | VSM_MGT_CHANGED;
+		vd->wdfd = open(vd->wdname, O_RDONLY);
+		if (vd->wdfd < 0)
 			(void)vsm_diag(vd,
 			    "VSM_Status: Cannot open workdir");
 		else
-			AZ(fstat(vd->dfd, &vd->dst));
+			AZ(fstat(vd->wdfd, &vd->wdst));
 	}
 
-	u = vsm_refresh_set(vd, vd->mgt);
-	retval |= u;
-	if (u & VSM_MGT_RUNNING)
-		retval |= vsm_refresh_set(vd, vd->child) << 8;
+	if (vd->wdfd >= 0) {
+		retval |= vsm_refresh_set(vd, vd->mgt);
+		if (retval & VSM_MGT_RUNNING)
+			retval |= vsm_refresh_set(vd, vd->child);
+	}
 	return (retval);
 }
 
@@ -709,12 +722,12 @@ VSM_Attach(struct vsm *vd, int progress)
 	else
 		t0 = VTIM_mono() + vd->patience;
 
-	if (vd->dname == NULL) {
+	if (vd->wdname == NULL) {
 		/* Use default (hostname) */
 		i = VSM_Arg(vd, 'n', "");
 		if (i < 0)
 			return (i);
-		AN(vd->dname);
+		AN(vd->wdname);
 	}
 
 	AZ(vd->attached);

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] 05c168aea Add a VLU_Reset() function
Message-ID: <20191018132306.9276A67A6@lists.varnish-cache.org>


commit 05c168aea3181fb0c610c9d88e8801b58198fdd6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 8 08:24:13 2019 +0000

    Add a VLU_Reset() function

diff --git a/include/vlu.h b/include/vlu.h
index 47cf705f5..8ce98dd3f 100644
--- a/include/vlu.h
+++ b/include/vlu.h
@@ -34,6 +34,7 @@
 
 typedef int (vlu_f)(void *, const char *);
 struct vlu *VLU_New(vlu_f *, void *, unsigned);
+void VLU_Reset(struct vlu *);
 int VLU_Fd(struct vlu *, int);
 void VLU_Destroy(struct vlu **);
 int VLU_File(int, vlu_f *, void *, unsigned);
diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index a120c4df6..cdcb6eeee 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -72,6 +72,13 @@ VLU_New(vlu_f *func, void *priv, unsigned bufsize)
 	return (l);
 }
 
+void
+VLU_Reset(struct vlu *l)
+{
+	CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC);
+	l->bufp = 0;
+}
+
 void
 VLU_Destroy(struct vlu **lp)
 {

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] 631b0f747 Destroy the childs Param and Panicstr VSMs properly
Message-ID: <20191018132306.ACF9267BB@lists.varnish-cache.org>


commit 631b0f74787029f64f4f11c4559941769e0e07ba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 8 11:08:34 2019 +0000

    Destroy the childs Param and Panicstr VSMs properly

diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index ddfa6504a..5fdcfc95c 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -143,6 +143,6 @@ mgt_SHM_ChildDestroy(void)
 		AZ(system("rm -rf " VSM_CHILD_DIRNAME));
 		VJ_master(JAIL_MASTER_LOW);
 	}
-	heritage.panic_str = NULL;
-	heritage.param = NULL;
+	VSMW_Free(mgt_vsmw, (void**)&heritage.panic_str);
+	VSMW_Free(mgt_vsmw, (void**)&heritage.param);
 }

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] ab0821e28 Only emit the '-' line when the cluster is empty.
Message-ID: <20191018132306.C572967CA@lists.varnish-cache.org>


commit ab0821e28710440f4fa0c59ea674a5104ee05524
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 8 14:12:21 2019 +0000

    Only emit the '-' line when the cluster is empty.

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 0439084df..328999e42 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -107,6 +107,7 @@ struct vsmw_cluster {
 	void				*ptr;
 	size_t				next;
 	int				refs;
+	int				named;
 };
 
 struct vsmwseg {
@@ -223,7 +224,7 @@ vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg)
 /*--------------------------------------------------------------------*/
 
 static void
-vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
+vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg)
 {
 	char *t = NULL;
 	ssize_t s;
@@ -237,11 +238,10 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 	VTAILQ_REMOVE(&vsmw->segs, seg, list);
 
 	vsmw->nsegs--;
-	if (vsmw->nsubs * 2 < vsmw->nsegs || !fixidx) {
+	if (vsmw->nsubs < 10 || vsmw->nsubs * 2 < vsmw->nsegs) {
 		vsmw_append_record(vsmw, seg, '-');
 		vsmw->nsubs++;
 	} else {
-		vsmw->nsubs = 0;
 		vsmw_mkent(vsmw, vsmw->idx);
 		REPLACE(t, VSB_data(vsmw->vsb));
 		AN(t);
@@ -258,6 +258,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 		AZ(close(fd));
 		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
 		REPLACE(t, NULL);
+		vsmw->nsubs = 0;
 	}
 	REPLACE(seg->class, NULL);
 	REPLACE(seg->id, NULL);
@@ -319,6 +320,8 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	seg->cluster = vc;
 	REPLACE(seg->class, "");
 	REPLACE(seg->id, "");
+	vc->refs++;
+	vc->named = 1;
 	vsmw_addseg(vsmw, seg);
 
 	vsmw_do_unlock();
@@ -326,31 +329,19 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 }
 
 static void
-vsmw_DestroyCluster_locked(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
+vsmw_DestroyCluster_locked(struct vsmw *vsmw, struct vsmw_cluster *vc)
 {
-	struct vsmw_cluster *vc;
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
-	AN(vsmcp);
-	vc = *vsmcp;
-	*vsmcp = NULL;
 	CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC);
 
-	if (vc->cseg != NULL) {
-		/*
-		 * Backends go on the cool list, so the VGC cluster is
-		 * destroyed before they are.  Solve this by turning the
-		 * cluster into an anonymous cluster which dies with the
-		 * refcount on it.
-		 */
-		vsmw_delseg(vsmw, vc->cseg, 1);
-		vc->cseg = NULL;
-		if (vc->refs > 0)
-			return;
-	}
+	AZ(vc->refs);
+
 	AZ(munmap(vc->ptr, vc->len));
+	if (vc->named)
+		vsmw_delseg(vsmw, vc->cseg);
+	vc->cseg = 0;
 
-	AZ(vc->refs);
 	VTAILQ_REMOVE(&vsmw->clusters, vc, list);
 	if (unlinkat(vsmw->vdirfd, vc->fn, 0))
 		assert (errno == ENOENT);
@@ -361,9 +352,13 @@ vsmw_DestroyCluster_locked(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
 void
 VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
 {
+	struct vsmw_cluster *vc;
+
+	TAKE_OBJ_NOTNULL(vc, vsmcp, VSMW_CLUSTER_MAGIC);
 
 	vsmw_do_lock();
-	vsmw_DestroyCluster_locked(vsmw, vsmcp);
+	if (--vc->refs == 0)
+		vsmw_DestroyCluster_locked(vsmw, vc);
 	vsmw_do_unlock();
 }
 
@@ -378,7 +373,6 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 
 	vsmw_do_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
-	(void)vc;
 
 	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
 	AN(seg);
@@ -438,11 +432,13 @@ VSMW_Free(struct vsmw *vsmw, void **pp)
 	*pp = NULL;
 
 	cp = seg->cluster;
+	CHECK_OBJ_NOTNULL(cp, VSMW_CLUSTER_MAGIC);
+	assert(cp->refs > 0);
 
-	vsmw_delseg(vsmw, seg, 1);
+	vsmw_delseg(vsmw, seg);
 
-	if (!--cp->refs && cp->cseg == NULL)
-		vsmw_DestroyCluster_locked(vsmw, &cp);
+	if (!--cp->refs)
+		vsmw_DestroyCluster_locked(vsmw, cp);
 	vsmw_do_unlock();
 }
 
@@ -493,7 +489,7 @@ VSMW_Destroy(struct vsmw **pp)
 	vsmw_do_lock();
 	TAKE_OBJ_NOTNULL(vsmw, pp, VSMW_MAGIC);
 	VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
-		vsmw_delseg(vsmw, seg, 0);
+		vsmw_delseg(vsmw, seg);
 	if (unlinkat(vsmw->vdirfd, vsmw->idx, 0))
 		assert (errno == ENOENT);
 	REPLACE(vsmw->idx, NULL);

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] 2963be64c Never allocate at offset zero in a cluster,
 this makes it faster for clients what the _index entry is about.
Message-ID: <20191018132306.DCEC967D8@lists.varnish-cache.org>


commit 2963be64cf6c0179c48876377030d9b9b36dd8ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 13 07:38:51 2019 +0000

    Never allocate at offset zero in a cluster, this makes it faster
    for clients what the _index entry is about.

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 328999e42..9fab01d7d 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -138,6 +138,9 @@ struct vsmw {
 	uint64_t			nsubs;
 };
 
+/* Allocations in clusters never start at offset zero */
+#define VSM_CLUSTER_OFFSET 16
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -311,7 +314,9 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	struct vsmwseg *seg;
 
 	vsmw_do_lock();
-	vc = vsmw_newcluster(vsmw, len, pfx);
+	vc = vsmw_newcluster(vsmw, len + VSM_CLUSTER_OFFSET, pfx);
+	AN(vc);
+	vc->next += VSM_CLUSTER_OFFSET;
 
 	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
 	AN(seg);

From martin at varnish-software.com  Fri Oct 18 13:23:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:06 +0000 (UTC)
Subject: [6.0] 7b61ac998 Polishing/Refactoring
Message-ID: <20191018132307.0549B67E6@lists.varnish-cache.org>


commit 7b61ac998cfd16be9d4cddcf211be18aec9b3f66
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 13 09:47:10 2019 +0000

    Polishing/Refactoring

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 2ef04edec..59c03f068 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -82,6 +82,7 @@ struct vsm_seg {
 #define VSM_FLAG_STALE		(1U<<2)
 #define VSM_FLAG_CLUSTER	(1U<<3)
 	VTAILQ_ENTRY(vsm_seg)	list;
+	VTAILQ_ENTRY(vsm_seg)	clist;
 	struct vsm_set		*set;
 	struct vsm_seg		*cluster;
 	char			**av;
@@ -233,18 +234,26 @@ vsm_unmapseg(struct vsm_seg *vg)
 /*--------------------------------------------------------------------*/
 
 static void
-vsm_delseg(struct vsm_seg *vg)
+vsm_delseg(struct vsm_seg *vg, int refsok)
 {
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 
+	if (refsok && vg->refs) {
+		AZ(vg->flags & VSM_FLAG_STALE);
+		vg->flags |= VSM_FLAG_STALE;
+		VTAILQ_REMOVE(&vg->set->segs, vg, list);
+		VTAILQ_INSERT_TAIL(&vg->set->stale, vg, list);
+		return;
+	}
+
 	if (vg->s != NULL)
 		vsm_unmapseg(vg);
 
 	if (vg->flags & VSM_FLAG_STALE)
 		VTAILQ_REMOVE(&vg->set->stale, vg, list);
 	else if (vg->flags & VSM_FLAG_CLUSTER)
-		VTAILQ_REMOVE(&vg->set->clusters, vg, list);
+		VTAILQ_REMOVE(&vg->set->clusters, vg, clist);
 	else
 		VTAILQ_REMOVE(&vg->set->segs, vg, list);
 	VAV_Free(vg->av);
@@ -281,11 +290,11 @@ vsm_delset(struct vsm_set **p)
 	if (vs->dfd >= 0)
 		closefd(&vs->dfd);
 	while (!VTAILQ_EMPTY(&vs->stale))
-		vsm_delseg(VTAILQ_FIRST(&vs->stale));
+		vsm_delseg(VTAILQ_FIRST(&vs->stale), 0);
 	while (!VTAILQ_EMPTY(&vs->segs))
-		vsm_delseg(VTAILQ_FIRST(&vs->segs));
+		vsm_delseg(VTAILQ_FIRST(&vs->segs), 0);
 	while (!VTAILQ_EMPTY(&vs->clusters))
-		vsm_delseg(VTAILQ_FIRST(&vs->clusters));
+		vsm_delseg(VTAILQ_FIRST(&vs->clusters), 0);
 	FREE_OBJ(vs);
 }
 
@@ -295,16 +304,8 @@ vsm_wash_set(struct vsm_set *vs, int all)
 	struct vsm_seg *vg, *vg2;
 
 	VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
-		if (all || (vg->flags & VSM_FLAG_MARKSCAN) == 0) {
-			VTAILQ_REMOVE(&vs->segs, vg, list);
-			if (vg->refs) {
-				vg->flags |= VSM_FLAG_STALE;
-				VTAILQ_INSERT_TAIL(&vs->stale, vg, list);
-			} else {
-				VAV_Free(vg->av);
-				FREE_OBJ(vg);
-			}
-		}
+		if (all || (vg->flags & VSM_FLAG_MARKSCAN) == 0)
+			vsm_delseg(vg, 1);
 	}
 }
 
@@ -441,15 +442,14 @@ vsm_cmp_av(char * const *a1, char * const *a2)
 }
 
 static struct vsm_seg *
-vsm_findcluster(const struct vsm_seg *vga)
+vsm_findcluster(const struct vsm_set *vs, const char *cnam)
 {
-	const struct vsm_set *vs = vga->set;
 	struct vsm_seg *vg;
 	AN(vs);
-	AN(vga->av[1]);
-	VTAILQ_FOREACH(vg, &vs->clusters, list) {
+	AN(cnam);
+	VTAILQ_FOREACH(vg, &vs->clusters, clist) {
 		AN(vg->av[1]);
-		if (!strcmp(vga->av[1], vg->av[1]))
+		if (!strcmp(cnam, vg->av[1]))
 			return (vg);
 	}
 	return (NULL);
@@ -498,7 +498,14 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 		return(-1);
 	}
 
-	if (vs->vg == NULL) {
+	while (vs->vg != NULL && vsm_cmp_av(&vs->vg->av[1], &av[1]))
+		vs->vg = VTAILQ_NEXT(vs->vg, list);
+	if (vs->vg != NULL) {
+		VAV_Free(av);
+		/* entry compared equal, so it survives */
+		vs->vg->flags |= VSM_FLAG_MARKSCAN;
+		vs->vg = VTAILQ_NEXT(vs->vg, list);
+	} else {
 		ALLOC_OBJ(vg2, VSM_SEG_MAGIC);
 		AN(vg2);
 		vg2->av = av;
@@ -507,19 +514,10 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 		vg2->serial = ++vd->serial;
 		if (ac == 4) {
 			vg2->flags |= VSM_FLAG_CLUSTER;
-			VTAILQ_INSERT_TAIL(&vs->clusters, vg2, list);
+			VTAILQ_INSERT_TAIL(&vs->clusters, vg2, clist);
 		} else {
 			VTAILQ_INSERT_TAIL(&vs->segs, vg2, list);
-			vg2->cluster = vsm_findcluster(vg2);
-		}
-	} else {
-		while (vs->vg != NULL && vsm_cmp_av(&vs->vg->av[1], &av[1]))
-			vs->vg = VTAILQ_NEXT(vs->vg, list);
-		VAV_Free(av);
-		if (vs->vg != NULL) {
-			/* entry compared equal, so it survives */
-			vs->vg->flags |= VSM_FLAG_MARKSCAN;
-			vs->vg = VTAILQ_NEXT(vs->vg, list);
+			vg2->cluster = vsm_findcluster(vs, vg2->av[1]);
 		}
 	}
 	return (0);
@@ -530,7 +528,7 @@ vsm_vlu_minus(struct vsm *vd, struct vsm_set *vs, const char *line)
 {
 	char **av;
 	int ac;
-	struct vsm_seg *vg, *vg2;
+	struct vsm_seg *vg;
 
 	av = VAV_Parse(line + 1, &ac, 0);
 
@@ -542,18 +540,11 @@ vsm_vlu_minus(struct vsm *vd, struct vsm_set *vs, const char *line)
 		return(-1);
 	}
 
-	VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
-		if (vsm_cmp_av(&vg->av[1], &av[1]))
-			continue;
-		VTAILQ_REMOVE(&vs->segs, vg, list);
-		if (vg->refs) {
-			vg->flags |= VSM_FLAG_STALE;
-			VTAILQ_INSERT_TAIL(&vs->stale, vg, list);
-		} else {
-			VAV_Free(vg->av);
-			FREE_OBJ(vg);
+	VTAILQ_FOREACH(vg, &vs->segs, list) {
+		if (!vsm_cmp_av(&vg->av[1], &av[1])) {
+			vsm_delseg(vg, 1);
+			break;
 		}
-		break;
 	}
 	return (0);
 }
@@ -923,7 +914,7 @@ VSM_Unmap(struct vsm *vd, struct vsm_fantom *vf)
 		vsm_unmapseg(vg);
 	}
 	if (vg->flags & VSM_FLAG_STALE)
-		vsm_delseg(vg);
+		vsm_delseg(vg, 0);
 	return (0);
 }
 

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] 08eceda3f Try to get the VSM cluster inconsistency fixed for
 good.
Message-ID: <20191018132307.2664D67F0@lists.varnish-cache.org>


commit 08eceda3f20bcdf6c79228489c88f9d8464edbc9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 13 11:08:18 2019 +0000

    Try to get the VSM cluster inconsistency fixed for good.

diff --git a/bin/varnishtest/tests/c00083.vtc b/bin/varnishtest/tests/c00083.vtc
index 67f52b612..3f79baae2 100644
--- a/bin/varnishtest/tests/c00083.vtc
+++ b/bin/varnishtest/tests/c00083.vtc
@@ -4,15 +4,47 @@ varnish v1 -vcl {
 	backend default { .host = "${bad_ip}"; }
 } -start
 
+delay 1
+
 process p1 {
 	nlines=`wc -l < ${tmpdir}/v1/_.vsm_child/_.index`
 	nminus=`grep -c '^-' ${tmpdir}/v1/_.vsm_child/_.index`
-	echo NLINES $nlines NMINUS $nminus
+	echo CHILD NLINES $nlines NMINUS $nminus
 } -dump -run
 
+# Useful for debugging
+process p2 {tail -F ${tmpdir}/v1/_.vsm_child/_.index} -dump -start
+process p3 {tail -F ${tmpdir}/v1/_.vsm_mgt/_.index} -dump -start
+
+varnish v1 -vcl {
+	backend b00 { .host = "${bad_ip}"; }
+	backend b01 { .host = "${bad_ip}"; }
+	backend b02 { .host = "${bad_ip}"; }
+	backend b03 { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		set req.backend_hint = b00;
+		set req.backend_hint = b01;
+		set req.backend_hint = b02;
+		set req.backend_hint = b03;
+	}
+}
+
+varnish v1 -cliok vcl.list
+
+process p1 -run
+
+varnish v1 -cliok "vcl.use vcl1"
+varnish v1 -cliok "vcl.discard vcl2"
+
+delay 1
+
+process p1 -run
+
 # The child process starts out with approx 37 VSM segments
-# so it takes 20 backends to cause a _.index rewrite.
-# Make it 25 to be safe.
+# and spent 10 lines on vcl2, so it takes ~ 15 backends to
+# cause a _.index rewrite.
+# Make it 20 to be on the safe side.
 varnish v1 -vcl {
 	backend b00 { .host = "${bad_ip}"; }
 	backend b01 { .host = "${bad_ip}"; }
@@ -34,11 +66,6 @@ varnish v1 -vcl {
 	backend b17 { .host = "${bad_ip}"; }
 	backend b18 { .host = "${bad_ip}"; }
 	backend b19 { .host = "${bad_ip}"; }
-	backend b20 { .host = "${bad_ip}"; }
-	backend b21 { .host = "${bad_ip}"; }
-	backend b22 { .host = "${bad_ip}"; }
-	backend b23 { .host = "${bad_ip}"; }
-	backend b24 { .host = "${bad_ip}"; }
 
 	sub vcl_recv {
 		set req.backend_hint = b00;
@@ -61,25 +88,54 @@ varnish v1 -vcl {
 		set req.backend_hint = b17;
 		set req.backend_hint = b18;
 		set req.backend_hint = b19;
-		set req.backend_hint = b20;
-		set req.backend_hint = b21;
-		set req.backend_hint = b22;
-		set req.backend_hint = b23;
-		set req.backend_hint = b24;
 	}
 }
 
 varnish v1 -cliok vcl.list
 
+delay 1
+
 process p1 -run
 
 varnish v1 -cliok "vcl.use vcl1"
-varnish v1 -cliok "vcl.discard vcl2"
+varnish v1 -cliok "vcl.discard vcl3"
+
+delay 1
 
 # Check that the _.index rewrite did happen
+
 process p1 {
 	nlines=`wc -l < ${tmpdir}/v1/_.vsm_child/_.index`
 	nminus=`grep -c '^-' ${tmpdir}/v1/_.vsm_child/_.index`
-	echo NLINES $nlines NMINUS $nminus
-	test $nminus -lt 25
+	echo CHILD NLINES $nlines NMINUS $nminus
+	# cat ${tmpdir}/v1/_.vsm_child/_.index
+	test $nminus -lt 20
 } -run
+
+# Now check the management process VSM
+
+process p1 {
+	nlines=`wc -l < ${tmpdir}/v1/_.vsm_mgt/_.index`
+	nminus=`grep -c '^-' ${tmpdir}/v1/_.vsm_mgt/_.index`
+	echo MGT NLINES $nlines NMINUS $nminus
+	# cat ${tmpdir}/v1/_.vsm_child/_.index
+	test $nminus -eq 0
+} -run
+
+varnish v1 -cliok "stop"
+
+delay 1
+
+process p1 {
+	nlines=`wc -l < ${tmpdir}/v1/_.vsm_mgt/_.index`
+	nminus=`grep -c '^-' ${tmpdir}/v1/_.vsm_mgt/_.index`
+	echo MGT NLINES $nlines NMINUS $nminus
+	# cat ${tmpdir}/v1/_.vsm_child/_.index
+	test $nminus -eq 2
+} -run
+
+varnish v1 -cliok "start"
+
+delay 1
+
+process p1 -run
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 78246d6fa..bde52a019 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -385,6 +385,7 @@ main(int argc, char **argv)
 		ident = VSM_Dup(vut->vsm, "Arg", "-i");
 	else
 		ident = strdup("");
+	AN(ident);
 	vut->dispatch_f = accumulate;
 	vut->dispatch_priv = NULL;
 	vut->sighup_f = sighup;
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 0c243fecd..ce63dcb7e 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -422,6 +422,7 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv)
 	AN(vsm);
 	sp = VTAILQ_FIRST(&vsc->segs);
 	VSM_FOREACH(&ifantom, vsm) {
+		AN(ifantom.class);
 		if (strcmp(ifantom.class, VSC_CLASS) &&
 		    strcmp(ifantom.class, VSC_DOC_CLASS))
 			continue;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 59c03f068..175f5f441 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -239,6 +239,11 @@ vsm_delseg(struct vsm_seg *vg, int refsok)
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 
+	if (vg->flags & VSM_FLAG_CLUSTER) {
+		vg->flags &= ~VSM_FLAG_CLUSTER;
+		VTAILQ_REMOVE(&vg->set->clusters, vg, clist);
+	}
+
 	if (refsok && vg->refs) {
 		AZ(vg->flags & VSM_FLAG_STALE);
 		vg->flags |= VSM_FLAG_STALE;
@@ -252,8 +257,6 @@ vsm_delseg(struct vsm_seg *vg, int refsok)
 
 	if (vg->flags & VSM_FLAG_STALE)
 		VTAILQ_REMOVE(&vg->set->stale, vg, list);
-	else if (vg->flags & VSM_FLAG_CLUSTER)
-		VTAILQ_REMOVE(&vg->set->clusters, vg, clist);
 	else
 		VTAILQ_REMOVE(&vg->set->segs, vg, list);
 	VAV_Free(vg->av);
@@ -293,13 +296,12 @@ vsm_delset(struct vsm_set **p)
 		vsm_delseg(VTAILQ_FIRST(&vs->stale), 0);
 	while (!VTAILQ_EMPTY(&vs->segs))
 		vsm_delseg(VTAILQ_FIRST(&vs->segs), 0);
-	while (!VTAILQ_EMPTY(&vs->clusters))
-		vsm_delseg(VTAILQ_FIRST(&vs->clusters), 0);
+	assert(VTAILQ_EMPTY(&vs->clusters));
 	FREE_OBJ(vs);
 }
 
 static void
-vsm_wash_set(struct vsm_set *vs, int all)
+vsm_wash_set(const struct vsm_set *vs, int all)
 {
 	struct vsm_seg *vg, *vg2;
 
@@ -486,13 +488,12 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 {
 	char **av;
 	int ac;
-	struct vsm_seg *vg2;
+	struct vsm_seg *vg;
 
 	av = VAV_Parse(line + 1, &ac, 0);
 
 	if (av[0] != NULL || ac < 4 || ac > 6) {
-		(void)(vsm_diag(vd,
-		    "vsm_vlu_plus: bad index (%d/%s)",
+		(void)(vsm_diag(vd, "vsm_vlu_plus: bad index (%d/%s)",
 		    ac, av[0]));
 		VAV_Free(av);
 		return(-1);
@@ -506,25 +507,26 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 		vs->vg->flags |= VSM_FLAG_MARKSCAN;
 		vs->vg = VTAILQ_NEXT(vs->vg, list);
 	} else {
-		ALLOC_OBJ(vg2, VSM_SEG_MAGIC);
-		AN(vg2);
-		vg2->av = av;
-		vg2->set = vs;
-		vg2->flags = VSM_FLAG_MARKSCAN;
-		vg2->serial = ++vd->serial;
+		ALLOC_OBJ(vg, VSM_SEG_MAGIC);
+		AN(vg);
+		vg->av = av;
+		vg->set = vs;
+		vg->flags = VSM_FLAG_MARKSCAN;
+		vg->serial = ++vd->serial;
+		VTAILQ_INSERT_TAIL(&vs->segs, vg, list);
 		if (ac == 4) {
-			vg2->flags |= VSM_FLAG_CLUSTER;
-			VTAILQ_INSERT_TAIL(&vs->clusters, vg2, clist);
-		} else {
-			VTAILQ_INSERT_TAIL(&vs->segs, vg2, list);
-			vg2->cluster = vsm_findcluster(vs, vg2->av[1]);
+			vg->flags |= VSM_FLAG_CLUSTER;
+			VTAILQ_INSERT_TAIL(&vs->clusters, vg, clist);
+		} else if (*vg->av[2] != '0') {
+			vg->cluster = vsm_findcluster(vs, vg->av[1]);
+			AN(vg->cluster);
 		}
 	}
 	return (0);
 }
 
 static int
-vsm_vlu_minus(struct vsm *vd, struct vsm_set *vs, const char *line)
+vsm_vlu_minus(struct vsm *vd, const struct vsm_set *vs, const char *line)
 {
 	char **av;
 	int ac;
@@ -533,19 +535,26 @@ vsm_vlu_minus(struct vsm *vd, struct vsm_set *vs, const char *line)
 	av = VAV_Parse(line + 1, &ac, 0);
 
 	if (av[0] != NULL || ac < 4 || ac > 6) {
-		(void)(vsm_diag(vd,
-		    "vsm_vlu_minus: bad index (%d/%s)",
+		(void)(vsm_diag(vd, "vsm_vlu_minus: bad index (%d/%s)",
 		    ac, av[0]));
 		VAV_Free(av);
 		return(-1);
 	}
 
-	VTAILQ_FOREACH(vg, &vs->segs, list) {
+	/* Clustered segments cannot come before their cluster */
+	if (*av[2] != '0')
+		vg = vsm_findcluster(vs, av[1]);
+	else
+		vg = VTAILQ_FIRST(&vs->segs);
+
+	for (;vg != NULL; vg = VTAILQ_NEXT(vg, list)) {
 		if (!vsm_cmp_av(&vg->av[1], &av[1])) {
 			vsm_delseg(vg, 1);
 			break;
 		}
 	}
+	AN(vg);
+	VAV_Free(av);
 	return (0);
 }
 
@@ -786,28 +795,36 @@ VSM__iter0(const struct vsm *vd, struct vsm_fantom *vf)
 int
 VSM__itern(struct vsm *vd, struct vsm_fantom *vf)
 {
-	struct vsm_seg *vg, *vg2;
+	struct vsm_seg *vg;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(vd->attached);
 	AN(vf);
 
 	if (vf->priv == 0) {
-		vg2 = VTAILQ_FIRST(&vd->mgt->segs);
+		vg = VTAILQ_FIRST(&vd->mgt->segs);
+		if (vg == NULL)
+			return (0);
 	} else {
 		vg = vsm_findseg(vd, vf);
 		if (vg == NULL)
 			return (vsm_diag(vd, "VSM_FOREACH: inconsistency"));
-		vg2 = VTAILQ_NEXT(vg, list);
-		if (vg2 == NULL && vg->set == vd->mgt)
-			vg2 = VTAILQ_FIRST(&vd->child->segs);
+		while (1) {
+			if (vg->set == vd->mgt && VTAILQ_NEXT(vg, list) == NULL)
+				vg = VTAILQ_FIRST(&vd->child->segs);
+			else
+				vg = VTAILQ_NEXT(vg, list);
+			if (vg == NULL)
+				return (0);
+			if (!(vg->flags & VSM_FLAG_CLUSTER))
+				break;
+		}
 	}
-	if (vg2 == NULL)
-		return (0);
 	memset(vf, 0, sizeof *vf);
-	vf->priv = vg2->serial;
-	vf->class = vg2->av[4];
-	vf->ident = vg2->av[5];
+	vf->priv = vg->serial;
+	vf->class = vg->av[4];
+	vf->ident = vg->av[5];
+	AN(vf->class);
 	return (1);
 }
 

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] 053979745 Comment out 'tail -F', SunOS does not support it.
Message-ID: <20191018132307.461FD680E@lists.varnish-cache.org>


commit 053979745595a355d8f373984f96057b8d299ad2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 13 11:36:44 2019 +0000

    Comment out 'tail -F', SunOS does not support it.

diff --git a/bin/varnishtest/tests/c00083.vtc b/bin/varnishtest/tests/c00083.vtc
index 3f79baae2..6b5ff7e69 100644
--- a/bin/varnishtest/tests/c00083.vtc
+++ b/bin/varnishtest/tests/c00083.vtc
@@ -13,8 +13,8 @@ process p1 {
 } -dump -run
 
 # Useful for debugging
-process p2 {tail -F ${tmpdir}/v1/_.vsm_child/_.index} -dump -start
-process p3 {tail -F ${tmpdir}/v1/_.vsm_mgt/_.index} -dump -start
+#process p2 {tail -F ${tmpdir}/v1/_.vsm_child/_.index} -dump -start
+#process p3 {tail -F ${tmpdir}/v1/_.vsm_mgt/_.index} -dump -start
 
 varnish v1 -vcl {
 	backend b00 { .host = "${bad_ip}"; }

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] 514a9de19 Don't close the _.index fd,
 but continue reading until the file is recreated by varnishd.
Message-ID: <20191018132307.643EA681D@lists.varnish-cache.org>


commit 514a9de19e990411180e971ebb00216cc62e51e7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 13 13:09:14 2019 +0000

    Don't close the _.index fd, but continue reading until the
    file is recreated by varnishd.
    
    Martin spotted that I lost this in the previous commit.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 175f5f441..c7bded025 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -70,6 +70,8 @@
 const struct vsm_valid VSM_invalid[1] = {{"invalid"}};
 const struct vsm_valid VSM_valid[1] = {{"valid"}};
 
+static vlu_f vsm_vlu_func;
+
 /*--------------------------------------------------------------------*/
 
 struct vsm_set;
@@ -112,6 +114,7 @@ struct vsm_set {
 	uintmax_t		id1, id2;
 
 	// _.index reading state
+	struct vlu		*vlu;
 	unsigned		retval;
 	struct vsm_seg		*vg;
 
@@ -277,6 +280,8 @@ vsm_newset(const char *dirname)
 	VTAILQ_INIT(&vs->clusters);
 	vs->dname = dirname;
 	vs->dfd = vs->fd = -1;
+	vs->vlu = VLU_New(vsm_vlu_func, vs, 0);
+	AN(vs->vlu);
 	return (vs);
 }
 
@@ -297,6 +302,7 @@ vsm_delset(struct vsm_set **p)
 	while (!VTAILQ_EMPTY(&vs->segs))
 		vsm_delseg(VTAILQ_FIRST(&vs->segs), 0);
 	assert(VTAILQ_EMPTY(&vs->clusters));
+	VLU_Destroy(&vs->vlu);
 	FREE_OBJ(vs);
 }
 
@@ -590,76 +596,79 @@ vsm_vlu_func(void *priv, const char *line)
 	return (i);
 }
 
+static void
+vsm_readlines(struct vsm_set *vs)
+{
+	int i;
+
+	do {
+		i = VLU_Fd(vs->vlu, vs->fd);
+	} while (!i);
+	assert(i == -2);
+}
+
 static unsigned
 vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 {
 	struct stat st;
-	int i;
-	struct vlu *vlu;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	CHECK_OBJ_NOTNULL(vs, VSM_SET_MAGIC);
 	vs->retval = 0;
-	if (vs->dfd >= 0) {
-		if (fstatat(vd->wdfd, vs->dname, &st, AT_SYMLINK_NOFOLLOW) ||
-		    st.st_ino != vs->dst.st_ino ||
-		    st.st_dev != vs->dst.st_dev ||
-		    st.st_mode != vs->dst.st_mode ||
-		    st.st_nlink == 0) {
-			closefd(&vs->dfd);
-			vs->id1 = vs->id2 = 0;
-			vsm_wash_set(vs, 1);
-			vs->retval |= vs->flag_restarted;
-		}
+	if (vs->dfd >= 0 && (
+	    fstatat(vd->wdfd, vs->dname, &st, AT_SYMLINK_NOFOLLOW) ||
+	    st.st_ino != vs->dst.st_ino ||
+	    st.st_dev != vs->dst.st_dev ||
+	    st.st_mode != vs->dst.st_mode ||
+	    st.st_nlink == 0)) {
+		closefd(&vs->dfd);
 	}
 
 	if (vs->dfd < 0) {
 		if (vs->fd >= 0)
 			closefd(&vs->fd);
 		vs->dfd = openat(vd->wdfd, vs->dname, O_RDONLY);
-		if (vs->dfd < 0) {
-			vs->id1 = vs->id2 = 0;
-			vsm_wash_set(vs, 1);
-			vs->retval |= vs->flag_restarted;
-			return (vs->retval);
-		}
-		AZ(fstat(vs->dfd, &vs->dst));
 	}
 
+	if (vs->dfd < 0) {
+		vs->id1 = vs->id2 = 0;
+		vsm_wash_set(vs, 1);
+		return (vs->retval | vs->flag_restarted);
+	}
+
+	AZ(fstat(vs->dfd, &vs->dst));
+
 	if (vs->fd >= 0 && (
 	    fstatat(vs->dfd, "_.index", &st, AT_SYMLINK_NOFOLLOW) ||
 	    st.st_ino != vs->fst.st_ino ||
 	    st.st_dev != vs->fst.st_dev ||
 	    st.st_mode != vs->fst.st_mode ||
-	    st.st_size != vs->fst.st_size ||
+	    st.st_size < vs->fst.st_size ||
 	    st.st_nlink < 1 ||
 	    memcmp(&st.st_mtime, &vs->fst.st_mtime, sizeof st.st_mtime))) {
 		closefd(&vs->fd);
 	}
 
 	if (vs->fd >= 0) {
-		if (vd->couldkill < 1 || !kill(vs->id1, 0))
-			vs->retval |= vs->flag_running;
-		return (vs->retval);
+		vs->vg = NULL;
+		vsm_readlines(vs);
+	} else {
+		VTAILQ_FOREACH(vs->vg, &vs->segs, list)
+			vs->vg->flags &= ~VSM_FLAG_MARKSCAN;
+		vs->vg = VTAILQ_FIRST(&vs->segs);
+		vs->fd = openat(vs->dfd, "_.index", O_RDONLY);
+		if (vs->fd < 0)
+			return (vs->retval|vs->flag_restarted);
+		VLU_Reset(vs->vlu);
+		AZ(fstat(vs->fd, &vs->fst));
+		vsm_readlines(vs);
+		vsm_wash_set(vs, 0);
 	}
 
-	vs->retval |= vs->flag_changed;
-	vs->fd = openat(vs->dfd, "_.index", O_RDONLY);
-	if (vs->fd < 0)
-		return (vs->retval|vs->flag_restarted);
-
-	AZ(fstat(vs->fd, &vs->fst));
-
-	vlu = VLU_New(vsm_vlu_func, vs, 0);
-	AN(vlu);
+	vs->fst.st_size = lseek(vs->fd, 0L, SEEK_CUR);
 
-	vs->vg = NULL;
-	do {
-		i = VLU_Fd(vlu, vs->fd);
-	} while (!i);
-	assert(i == -2);
-	VLU_Destroy(&vlu);
-	vsm_wash_set(vs, 0);
+	if (vd->couldkill < 1 || !kill(vs->id1, 0))
+		vs->retval |= vs->flag_running;
 	return (vs->retval);
 }
 

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] 28dc4b07c Don't remove VSM_FLAG_CLUSTER too early
Message-ID: <20191018132307.80FA5682D@lists.varnish-cache.org>


commit 28dc4b07c3bbf39a8a19cd00757fa7eeab48365e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Aug 30 15:18:46 2019 +0200

    Don't remove VSM_FLAG_CLUSTER too early
    
    If a segment will be marked stale, it should not have its cluster flag
    removed until its actually deleted.
    
    This problem was observed in varnishtest as an assert. What happened was
    that a VSM_Map() was executed on a stale segment inside a cluster, which
    caused an assert when the cluster segment it pointed to was no longer
    marked VSM_FLAG_CLUSTER.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index c7bded025..06f13af28 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -242,11 +242,6 @@ vsm_delseg(struct vsm_seg *vg, int refsok)
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 
-	if (vg->flags & VSM_FLAG_CLUSTER) {
-		vg->flags &= ~VSM_FLAG_CLUSTER;
-		VTAILQ_REMOVE(&vg->set->clusters, vg, clist);
-	}
-
 	if (refsok && vg->refs) {
 		AZ(vg->flags & VSM_FLAG_STALE);
 		vg->flags |= VSM_FLAG_STALE;
@@ -258,6 +253,11 @@ vsm_delseg(struct vsm_seg *vg, int refsok)
 	if (vg->s != NULL)
 		vsm_unmapseg(vg);
 
+	if (vg->flags & VSM_FLAG_CLUSTER) {
+		vg->flags &= ~VSM_FLAG_CLUSTER;
+		VTAILQ_REMOVE(&vg->set->clusters, vg, clist);
+	}
+
 	if (vg->flags & VSM_FLAG_STALE)
 		VTAILQ_REMOVE(&vg->set->stale, vg, list);
 	else

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] a4c8da94f Remove stale cluster segments when their refcount
 goes to zero
Message-ID: <20191018132307.9E3C06838@lists.varnish-cache.org>


commit a4c8da94fdd4f1d772eeddcbff109d7dcf85fd64
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Aug 30 15:24:32 2019 +0200

    Remove stale cluster segments when their refcount goes to zero
    
    When a cluster is marked stale, there is no mechanism to actually remove
    it once all its containing segments (which would also be marked stale)
    goes away.
    
    Fix this by executing vsm_delseg on the cluster in VSM_Unmap if it is marked
    stale.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 06f13af28..5c2d6efe1 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -933,8 +933,13 @@ VSM_Unmap(struct vsm *vd, struct vsm_fantom *vf)
 		assert(vg->s == NULL);
 		assert(vg->sz == 0);
 		assert(vg->cluster->refs > 0);
-		if (--vg->cluster->refs == 0)
+		if (--vg->cluster->refs == 0) {
 			vsm_unmapseg(vg->cluster);
+			if (vg->cluster->flags & VSM_FLAG_STALE) {
+				AN(vg->flags & VSM_FLAG_STALE);
+				vsm_delseg(vg->cluster, 0);
+			}
+		}
 		vg->b = vg->e = NULL;
 	} else {
 		vsm_unmapseg(vg);

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] d78477d21 Sprinkle CHECK_OBJ_NOTNULL on struct vsm_seg
Message-ID: <20191018132307.B9CFB6842@lists.varnish-cache.org>


commit d78477d21a3cc36b8733ce8a90e5151cdbddcfed
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sat Aug 31 12:43:39 2019 +0200

    Sprinkle CHECK_OBJ_NOTNULL on struct vsm_seg
    
    Several places in the code lacked checking that these were still valid
    objects (ie not free'd).

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 5c2d6efe1..8f7d469e2 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -525,7 +525,7 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 			VTAILQ_INSERT_TAIL(&vs->clusters, vg, clist);
 		} else if (*vg->av[2] != '0') {
 			vg->cluster = vsm_findcluster(vs, vg->av[1]);
-			AN(vg->cluster);
+			CHECK_OBJ_NOTNULL(vg->cluster, VSM_SEG_MAGIC);
 		}
 	}
 	return (0);
@@ -882,6 +882,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 		return (0);
 	}
 
+	CHECK_OBJ_NOTNULL(vgc, VSM_SEG_MAGIC);
 	assert(vgc->flags & VSM_FLAG_CLUSTER);
 	assert(vg->s == NULL);
 	assert(vg->sz == 0);
@@ -922,6 +923,7 @@ VSM_Unmap(struct vsm *vd, struct vsm_fantom *vf)
 	vg = vsm_findseg(vd, vf);
 	if (vg == NULL)
 		return (vsm_diag(vd, "VSM_Unmap: bad fantom"));
+	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 	assert(vg->refs > 0);
 	vg->refs--;
 	vf->b = NULL;
@@ -930,6 +932,7 @@ VSM_Unmap(struct vsm *vd, struct vsm_fantom *vf)
 		return(0);
 
 	if (vg->cluster) {
+		CHECK_OBJ_NOTNULL(vg->cluster, VSM_SEG_MAGIC);
 		assert(vg->s == NULL);
 		assert(vg->sz == 0);
 		assert(vg->cluster->refs > 0);

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] 888b7234e Remove the st_mtime check in vsm_refresh_set
Message-ID: <20191018132307.D1815684F@lists.varnish-cache.org>


commit 888b7234eb5969e1574434e2fd5906064771c104
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sat Aug 31 12:46:40 2019 +0200

    Remove the st_mtime check in vsm_refresh_set
    
    Now that we incrementally read the file, the check if the mtime of the
    index file has checked has to go away. If not, the index file will always
    be marked as new, forcing a reread of the entire index.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 8f7d469e2..92423ac5b 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -644,8 +644,7 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 	    st.st_dev != vs->fst.st_dev ||
 	    st.st_mode != vs->fst.st_mode ||
 	    st.st_size < vs->fst.st_size ||
-	    st.st_nlink < 1 ||
-	    memcmp(&st.st_mtime, &vs->fst.st_mtime, sizeof st.st_mtime))) {
+	    st.st_nlink < 1)) {
 		closefd(&vs->fd);
 	}
 

From martin at varnish-software.com  Fri Oct 18 13:23:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:07 +0000 (UTC)
Subject: [6.0] 8bd323591 Do not advance the vs->vg pointer unless there was a
 match
Message-ID: <20191018132307.EE2956862@lists.varnish-cache.org>


commit 8bd323591db8d555cf43be48528349f20b6d597c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sat Aug 31 13:22:23 2019 +0200

    Do not advance the vs->vg pointer unless there was a match
    
    The vsm_set vg pointer points to the last matched segment insertion, and
    is used as an optimization when looking for existing entries in the
    list. varnishd guarantees that insertions are always ordered, so there is
    no need to search the preceeding entries on a successful match. When
    parsing an index containg elements that included entries that are added
    and then subsequently removed in a later entry, this pointer would be
    advanced to the very end, failing to match the entries in between when
    parsing the rest of the file.
    
    Fix this mechanism by only updating the pointer on a successful match, and
    also advancing it one step it if the entry it is pointing to is removed.
    
    Note that these types of events are not supposed to be possible, because
    varnishd will when rewriting the index only include the active ones,
    removing any add-then-remove pairs. But if for whatever reason an attempt
    is made to reread a list, with this patch it should not cause problems.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 92423ac5b..3fca11fb8 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -242,6 +242,11 @@ vsm_delseg(struct vsm_seg *vg, int refsok)
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 
+	if (vg->set->vg == vg) {
+		AZ(vg->flags & VSM_FLAG_STALE);
+		vg->set->vg = VTAILQ_NEXT(vg, list);
+	}
+
 	if (refsok && vg->refs) {
 		AZ(vg->flags & VSM_FLAG_STALE);
 		vg->flags |= VSM_FLAG_STALE;
@@ -505,13 +510,18 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 		return(-1);
 	}
 
-	while (vs->vg != NULL && vsm_cmp_av(&vs->vg->av[1], &av[1]))
-		vs->vg = VTAILQ_NEXT(vs->vg, list);
-	if (vs->vg != NULL) {
-		VAV_Free(av);
+	vg = vs->vg;
+	CHECK_OBJ_ORNULL(vg, VSM_SEG_MAGIC);
+	if (vg != NULL)
+		AZ(vg->flags & VSM_FLAG_STALE);
+	while (vg != NULL && vsm_cmp_av(&vg->av[1], &av[1]))
+		vg = VTAILQ_NEXT(vg, list);
+	if (vg != NULL) {
 		/* entry compared equal, so it survives */
-		vs->vg->flags |= VSM_FLAG_MARKSCAN;
-		vs->vg = VTAILQ_NEXT(vs->vg, list);
+		CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
+		VAV_Free(av);
+		vg->flags |= VSM_FLAG_MARKSCAN;
+		vs->vg = VTAILQ_NEXT(vg, list);
 	} else {
 		ALLOC_OBJ(vg, VSM_SEG_MAGIC);
 		AN(vg);

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] e4534f792 Store VSC exposed state per segment
Message-ID: <20191018132308.1517C686D@lists.varnish-cache.org>


commit e4534f792b576b377e8a743877c896b0f4ff1ce6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Aug 2 14:26:03 2019 +0200

    Store VSC exposed state per segment
    
    This moves the exposed flag handling to the segment level, rather than the
    point level. This enables not having to iterate over each point when there
    is no change from the previous.

diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index ce63dcb7e..83c83136d 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -62,7 +62,6 @@ VTAILQ_HEAD(vsc_sf_head, vsc_sf);
 struct vsc_pt {
 	struct VSC_point	point;
 	char			*name;
-	int			exposed;
 };
 
 struct vsc_seg {
@@ -77,6 +76,7 @@ struct vsc_seg {
 
 	unsigned		npoints;
 	struct vsc_pt		*points;
+	int			exposed;
 };
 
 struct vsc {
@@ -367,26 +367,31 @@ vsc_add_seg(const struct vsc *vsc, struct vsm *vsm, const struct vsm_fantom *fp)
  */
 
 static void
-vsc_expose(const struct vsc *vsc, const struct vsc_seg *sp, int del)
+vsc_expose(const struct vsc *vsc, struct vsc_seg *sp, int del)
 {
 	struct vsc_pt *pp;
 	unsigned u;
+	int expose;
+
+	if (vsc->fnew != NULL && !sp->exposed &&
+	    !del && sp->head->ready == 1)
+		expose = 1;
+	else if (vsc->fdestroy != NULL && sp->exposed &&
+	    (del || sp->head->ready == 2))
+		expose = 0;
+	else
+		return;
 
 	pp = sp->points;
 	for (u = 0; u < sp->npoints; u++, pp++) {
 		if (pp->name == NULL)
 			continue;
-		if (vsc->fdestroy != NULL && pp->exposed &&
-		    (del || sp->head->ready == 2)) {
-			vsc->fdestroy(vsc->priv, &pp->point);
-			pp->exposed = 0;
-		}
-		if (vsc->fnew != NULL && !pp->exposed &&
-		    !del && sp->head->ready == 1) {
+		if (expose)
 			pp->point.priv = vsc->fnew(vsc->priv, &pp->point);
-			pp->exposed = 1;
-		}
+		else
+			vsc->fdestroy(vsc->priv, &pp->point);
 	}
+	sp->exposed = expose;
 }
 
 /*--------------------------------------------------------------------

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] 2bed7b05d Ignore index lines without either a '+' or '-'
Message-ID: <20191018132308.2F8B66884@lists.varnish-cache.org>


commit 2bed7b05d36299ec0d929faebf4010d2a1ea6f0b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Aug 29 17:43:12 2019 +0200

    Ignore index lines without either a '+' or '-'
    
    When upgrading from when a previous release that doesn't have the latest
    VSM index file changes, the utilities will assert when trying to parse the
    index file from the previous version. Fix that by ignoring lines not
    starting with either '#', '+' or '-'.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 3fca11fb8..3f26ca516 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -579,7 +579,7 @@ vsm_vlu_func(void *priv, const char *line)
 {
 	struct vsm *vd;
 	struct vsm_set *vs;
-	int i = -1;
+	int i = 0;
 
 	CAST_OBJ_NOTNULL(vs, priv, VSM_SET_MAGIC);
 	vd = vs->vsm;

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] 4675b78a8 Speed up vsm_findseg
Message-ID: <20191018132308.49F8E6892@lists.varnish-cache.org>


commit 4675b78a887c696e51e4053b7f2b13d6751452f6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Aug 30 11:18:12 2019 +0200

    Speed up vsm_findseg
    
    This speeds up vsm_findseg by keeping a direct pointer to the struct
    vsm_seg it points to in the vsm_fantom.
    
    To prevent stale vsm_fantoms in an application from causing segfaults, the
    last serial number seen is also kept on the fantom. If this serial number
    does not match, the slow path of iterating all segs to find the right
    match is done, and the fantom is updated to make any subsequent calls on
    the same fantom take the fast path.
    
    With this patch the fantoms store 2 serial numbers and a direct pointer to
    the seg. To fit this in struct vsm_fantom without breaking the ABI, the
    unused chunk pointer value has been repurposed, giving us two uintptr_t
    priv variables to work with. The direct segment pointer occupies one, and
    the two serial numbers occupy one half each of the other. This limits the
    bits for each serial to only 16 bits on 32 bit systems. But only direct
    matches is done, and it is unlikely that a stale fantom should have the
    exact right value to be accepted as valid.

diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index cd7387032..a852e4c77 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -44,7 +44,7 @@ struct vsm;
 
 struct vsm_fantom {
 	uintptr_t		priv;		/* VSM private */
-	struct VSM_chunk	*chunk;
+	uintptr_t		priv2;		/* VSM private */
 	void			*b;		/* first byte of payload */
 	void			*e;		/* first byte past payload */
 	char			*class;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 3f26ca516..edb6ab4e5 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -72,6 +72,17 @@ const struct vsm_valid VSM_valid[1] = {{"valid"}};
 
 static vlu_f vsm_vlu_func;
 
+#define VSM_PRIV_SHIFT							\
+	(sizeof (uintptr_t) * 4)
+#define VSM_PRIV_MASK							\
+	(~((uintptr_t)UINTPTR_MAX << VSM_PRIV_SHIFT))
+#define VSM_PRIV_LOW(u)							\
+	((uintptr_t)(u) & VSM_PRIV_MASK)
+#define VSM_PRIV_HIGH(u)						\
+	(((uintptr_t)(u) >> VSM_PRIV_SHIFT) & VSM_PRIV_MASK)
+#define VSM_PRIV_MERGE(low, high)					\
+	(VSM_PRIV_LOW(low) | (VSM_PRIV_LOW(high) << VSM_PRIV_SHIFT))
+
 /*--------------------------------------------------------------------*/
 
 struct vsm_set;
@@ -528,7 +539,8 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line)
 		vg->av = av;
 		vg->set = vs;
 		vg->flags = VSM_FLAG_MARKSCAN;
-		vg->serial = ++vd->serial;
+		vg->serial = vd->serial;
+
 		VTAILQ_INSERT_TAIL(&vs->segs, vg, list);
 		if (ac == 4) {
 			vg->flags |= VSM_FLAG_CLUSTER;
@@ -586,6 +598,11 @@ vsm_vlu_func(void *priv, const char *line)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(line);
 
+	/* Up the serial counter. This wraps at UINTPTR_MAX/2
+	 * because thats the highest value we can store in struct
+	 * vsm_fantom. */
+	vd->serial = VSM_PRIV_LOW(vd->serial + 1);
+
 	switch (line[0]) {
 	case '#':
 		i = vsm_vlu_hash(vd, vs, line);
@@ -612,6 +629,7 @@ vsm_readlines(struct vsm_set *vs)
 	int i;
 
 	do {
+		assert(vs->fd >= 0);
 		i = VLU_Fd(vs->vlu, vs->fd);
 	} while (!i);
 	assert(i == -2);
@@ -779,22 +797,44 @@ vsm_findseg(const struct vsm *vd, const struct vsm_fantom *vf)
 	struct vsm_seg *vg;
 	uintptr_t x;
 
-	x = vf->priv;
+	x = VSM_PRIV_HIGH(vf->priv);
+	if (x == vd->serial) {
+		vg = (struct vsm_seg *)vf->priv2;
+		if (!VALID_OBJ(vg, VSM_SEG_MAGIC) ||
+		    vg->serial != VSM_PRIV_LOW(vf->priv))
+			WRONG("Corrupt fantom");
+		return (vg);
+	}
+
+	x = VSM_PRIV_LOW(vf->priv);
 	vs = vd->mgt;
 	VTAILQ_FOREACH(vg, &vs->segs, list)
 		if (vg->serial == x)
-			return (vg);
-	VTAILQ_FOREACH(vg, &vs->stale, list)
-		if (vg->serial == x)
-			return (vg);
+			break;
+	if (vg == NULL) {
+		VTAILQ_FOREACH(vg, &vs->stale, list)
+			if (vg->serial == x)
+				break;
+	}
 	vs = vd->child;
-	VTAILQ_FOREACH(vg, &vs->segs, list)
-		if (vg->serial == x)
-			return (vg);
-	VTAILQ_FOREACH(vg, &vs->stale, list)
-		if (vg->serial == x)
-			return (vg);
-	return (NULL);
+	if (vg == NULL) {
+		VTAILQ_FOREACH(vg, &vs->segs, list)
+			if (vg->serial == x)
+				break;
+	}
+	if (vg == NULL) {
+		VTAILQ_FOREACH(vg, &vs->stale, list)
+			if (vg->serial == x)
+				break;
+	}
+	if (vg == NULL)
+		return (NULL);
+
+	/* Update the fantom with the new priv so that lookups will be
+	 * fast on the next call. Note that this casts away the const. */
+	((struct vsm_fantom *)TRUST_ME(vf))->priv =
+	    VSM_PRIV_MERGE(vg->serial, vd->serial);
+	return (vg);
 }
 
 /*--------------------------------------------------------------------*/
@@ -839,7 +879,8 @@ VSM__itern(struct vsm *vd, struct vsm_fantom *vf)
 		}
 	}
 	memset(vf, 0, sizeof *vf);
-	vf->priv = vg->serial;
+	vf->priv = VSM_PRIV_MERGE(vg->serial, vd->serial);
+	vf->priv2 = (uintptr_t)vg;
 	vf->class = vg->av[4];
 	vf->ident = vg->av[5];
 	AN(vf->class);
@@ -862,7 +903,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 	if (vg == NULL)
 		return (vsm_diag(vd, "VSM_Map: bad fantom"));
 
-	assert(vg->serial == vf->priv);
+	assert(vg->serial == VSM_PRIV_LOW(vf->priv));
 	assert(vg->av[4] == vf->class);
 	assert(vg->av[5] == vf->ident);
 

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] 9d558eb65 Whitespace fixes
Message-ID: <20191018132308.65C5E68A0@lists.varnish-cache.org>


commit 9d558eb65b2d6568b4c306ab2c12d8e5797e494f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Sep 1 14:44:09 2019 +0200

    Whitespace fixes

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 9fab01d7d..9742f8b25 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -79,7 +79,7 @@ vsm_lock_f *vsmw_unlock = vsmw_dummy_lock;
 
 #define vsmw_do_lock() vsmw_do_lock_(__func__, __LINE__)
 
-#define vsmw_do_lock_(f, l) 				\
+#define vsmw_do_lock_(f, l)			\
 	do {					\
 		vsmw_lock();			\
 		AZ(vsmw_haslock);		\
@@ -87,7 +87,7 @@ vsm_lock_f *vsmw_unlock = vsmw_dummy_lock;
 	} while(0)
 
 #define vsmw_do_unlock() vsmw_do_unlock_(__func__, __LINE__)
-#define vsmw_do_unlock_(f, l) 				\
+#define vsmw_do_unlock_(f, l)			\
 	do {					\
 		AN(vsmw_haslock);		\
 		vsmw_haslock = 0;		\

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] 79231c306 Adjust the segment length for the cluster allocation
 offset
Message-ID: <20191018132308.7CDBB68B1@lists.varnish-cache.org>


commit 79231c3062811ad1b916e0bb55559a38cfb6ae60
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 4 17:54:54 2019 +0200

    Adjust the segment length for the cluster allocation offset
    
    Clusters VSM allocations start at offset 16. Make sure that the published
    segment includes that adjustment.

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 9742f8b25..5149b8bfb 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -321,7 +321,7 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
 	AN(seg);
 	vc->cseg = seg;
-	seg->len = len;
+	seg->len = vc->len;
 	seg->cluster = vc;
 	REPLACE(seg->class, "");
 	REPLACE(seg->id, "");

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] e35abaedf Retire the kill(SIGNULL) test in VSM
Message-ID: <20191018132308.98B4C68BB@lists.varnish-cache.org>


commit e35abaedfbcc26c0559f1d811f1b54fedb997b61
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 10 16:31:04 2019 +0200

    Retire the kill(SIGNULL) test in VSM
    
    A bug was uncovered in the VSM code that checks if kill(SIGNULL) would
    work as a test of liveness of the master and worker processes. If the user
    running the utility has the required permissions to send signal to the
    worker process, but not the management process, the code would wrongly
    assume it could do kill on both. It would then end up in a connect loop
    never succeeding.
    
    Because kill() can not always be successfully run (a common scenario when
    the user running varnishlog is not the same UID as the cache processes),
    there is a fall back to using fstat to check for Varnish restarts. Since
    this fallback is active for most use cases anyways, it was decided to
    retire the kill() mechanism rather than to fix it. This way the behaviour
    does not change depending on what user the utility is run as.
    
    This change was OK'd by PHK after discussing it on IRC.

diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc
index 0c35fb3fc..9953fccad 100644
--- a/bin/varnishtest/tests/u00008.vtc
+++ b/bin/varnishtest/tests/u00008.vtc
@@ -37,10 +37,6 @@ process p1 -expect-text 0 0 "VBE.vcl1.s1.req"
 process p1 -expect-text 0 0 "DIAG"
 process p1 -screen_dump
 
-varnish v1 -stop
-process p1 -expect-text 2 1 "Uptime child:   Not Running"
-process p1 -screen_dump
-
 process p1 -write {dek}
 process p1 -expect-text 0 1 "Concurrent connections to backend:"
 process p1 -screen_dump
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index edb6ab4e5..adb4f8aef 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -150,8 +150,6 @@ struct vsm {
 
 	int			attached;
 	double			patience;
-
-	int			couldkill;
 };
 
 /*--------------------------------------------------------------------*/
@@ -357,8 +355,6 @@ VSM_New(void)
 	vd->child->vsm = vd;
 	vd->wdfd = -1;
 	vd->patience = 5;
-	if (getenv("VSM_NOPID") != NULL)
-		vd->couldkill = -1;
 	return (vd);
 }
 
@@ -485,17 +481,13 @@ vsm_vlu_hash(struct vsm *vd, struct vsm_set *vs, const char *line)
 	int i;
 	uintmax_t id1, id2;
 
+	(void)vd;
+
 	i = sscanf(line, "# %ju %ju", &id1, &id2);
 	if (i != 2) {
 		vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
 		return (0);
 	}
-	if (vd->couldkill >= 0 && !kill(id1, 0)) {
-		vd->couldkill = 1;
-	} else if (vd->couldkill > 0 && errno == ESRCH) {
-		vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
-		return (0);
-	}
 	vs->retval |= VSM_MGT_RUNNING;
 	if (id1 != vs->id1 || id2 != vs->id2) {
 		vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED;
@@ -694,8 +686,7 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs)
 
 	vs->fst.st_size = lseek(vs->fd, 0L, SEEK_CUR);
 
-	if (vd->couldkill < 1 || !kill(vs->id1, 0))
-		vs->retval |= vs->flag_running;
+	vs->retval |= vs->flag_running;
 	return (vs->retval);
 }
 

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] 73e498df7 Retire also the reference to VSM_NOPID in docs
Message-ID: <20191018132308.B25F268D2@lists.varnish-cache.org>


commit 73e498df73280cba50a85dc3390c55845bbd8f04
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 11 13:04:13 2019 +0200

    Retire also the reference to VSM_NOPID in docs
    
    Ref previous commit to remove the use of kill(signull) on the pids of the
    Varnish processes to test liveness.

diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst
index 7771b0110..d9a689c55 100644
--- a/doc/sphinx/reference/vsm.rst
+++ b/doc/sphinx/reference/vsm.rst
@@ -97,22 +97,3 @@ a chance to discover the deallocation.
 
 The include file <vapi/vsm.h> provides the supported API for accessing
 VSM files.
-
-VSM and Containers
-------------------
-
-The varnish way works great with single purpose containers. By sharing
-the varnish working directory read-only, vsm readers can be run in
-containers separate from those running varnishd instances on the same
-host.
-
-When running varnishd and vsm readers in the same process namespace,
-pid information can be used by vsm readers to determine if varnishd
-processes are alive.
-
-But, when running varnishd and vsm readers in different containers,
-the pid information has no relevance and may even be ambiguous across
-name spaces.
-
-Thus, with such setups, the environment variable VSM_NOPID needs to be
-set for vsm readers to disable use of pid information.

From martin at varnish-software.com  Fri Oct 18 13:23:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 18 Oct 2019 13:23:08 +0000 (UTC)
Subject: [6.0] f887e0db4 Update the changelog with the VSM changes
Message-ID: <20191018132308.CCA0868E1@lists.varnish-cache.org>


commit f887e0db4b4be09b6fdc523ca6568d115eea5026
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 5 10:41:17 2019 +0200

    Update the changelog with the VSM changes

diff --git a/doc/changes.rst b/doc/changes.rst
index 3ace7977f..3dddc3952 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -43,6 +43,25 @@ Varnish Cache 6.0.5 (unreleased)
 
 * Fix a problem where the ban lurker would skip objects. (3007_)
 
+* Incremental VSM updates. With this change, added or removed VSM segments
+  (ie varnishstat counters) will be done incrementally instead of complete
+  republishments of the entire set of VSM segments. This reduces the load
+  in the utilities (varnishncsa, varnishstat etc.) when there are frequent
+  changes to the set.
+
+* Optimize the VSM and VSC subsystems to handle large sets of counters
+  more gracefully.
+
+* Fix several resource leaks in libvarnishapi that would cause the
+  utilities to incrementally go slower and use CPU cycles after many
+  changes to the set of VSM segments.
+
+* Fixed a VSM bug that would cause varnishlog like utilities to not
+  produce log data. This could trigger when the varnish management process
+  is running root, the cache worker as a non-privileged user, and the log
+  utility run as the same user as the cache worker. This retires the
+  VSM_NOPID environment variable.
+
 .. _2395: https://github.com/varnishcache/varnish-cache/issues/2395
 .. _2572: https://github.com/varnishcache/varnish-cache/issues/2572
 .. _2905: https://github.com/varnishcache/varnish-cache/issues/2905

From dridi.boukelmoune at gmail.com  Fri Oct 18 15:20:05 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 18 Oct 2019 15:20:05 +0000 (UTC)
Subject: [master] fca5b0e4e Add a VRT_HashStrands32 utility
Message-ID: <20191018152005.DA46462DED@lists.varnish-cache.org>


commit fca5b0e4edcd3e56da2b210817e4eb258845c056
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jul 3 19:18:44 2019 +0200

    Add a VRT_HashStrands32 utility
    
    This de-duplicates logic between the two hashing directors, and in
    the case of vmod-shard adds a missing check for empty components in
    the strands that should be skipped.
    
    This breaks the hashing of the hash director, but unlike the shard
    director it offers no stability and results may vary depending on the
    health of individual clusters. For this reason only d00003.vtc needed
    a bit of reshuffling.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index d19e85b90..07587008b 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -37,8 +37,10 @@
 #include "vav.h"
 #include "vcl.h"
 #include "vct.h"
+#include "vend.h"
 #include "vrt_obj.h"
 #include "vsa.h"
+#include "vsha256.h"
 #include "vtcp.h"
 #include "vtim.h"
 #include "vcc_interface.h"
@@ -250,6 +252,35 @@ VRT_Strands2Bool(VCL_STRANDS s)
 	return (0);
 }
 
+/*--------------------------------------------------------------------
+ * Hash a STRANDS
+ */
+
+uint32_t
+VRT_HashStrands32(VCL_STRANDS s)
+{
+	struct VSHA256Context sha_ctx;
+	unsigned char sha256[VSHA256_LEN];
+	const char *p;
+	int i;
+
+	AN(s);
+	VSHA256_Init(&sha_ctx);
+	for (i = 0; i < s->n; i++) {
+		p = s->p[i];
+		if (p != NULL && *p != '\0')
+			VSHA256_Update(&sha_ctx, p, strlen(p));
+	}
+	VSHA256_Final(sha256, &sha_ctx);
+
+	/* NB: for some reason vmod_director's shard director specifically
+	 * relied on little-endian decoding of the last 4 octets. In order
+	 * to maintain a stable hash function to share across consumers we
+	 * need to stick to that.
+	 */
+	return (vle32dec(sha256 + VSHA256_LEN - 4));
+}
+
 /*--------------------------------------------------------------------
  * Collapse a STRING_LIST in the space provided, or return NULL
  */
diff --git a/bin/varnishtest/tests/d00003.vtc b/bin/varnishtest/tests/d00003.vtc
index 7f12096b2..30c73cfb5 100644
--- a/bin/varnishtest/tests/d00003.vtc
+++ b/bin/varnishtest/tests/d00003.vtc
@@ -2,22 +2,22 @@ varnishtest "Test hash director"
 
 server s1 {
 	rxreq
-	txresp -hdr "Foo: 1" -body "1"
+	txresp -hdr "Foo: 2" -body "2"
 	rxreq
 	txresp -hdr "Foo: 3" -body "3"
 	rxreq
-	txresp -hdr "Foo: 9" -body "9"
+	txresp -hdr "Foo: 6" -body "6"
+	rxreq
+	txresp -hdr "Foo: 8" -body "8"
 } -start
 
 server s2 {
 	rxreq
-	txresp -hdr "Foo: 2" -body "2"
+	txresp -hdr "Foo: 1" -body "1"
 	rxreq
 	txresp -hdr "Foo: 4" -body "4"
 	rxreq
-	txresp -hdr "Foo: 6" -body "6"
-	rxreq
-	txresp -hdr "Foo: 8" -body "8"
+	txresp -hdr "Foo: 9" -body "9"
 } -start
 
 varnish v1 -vcl+backend {
@@ -67,15 +67,15 @@ client c1 {
 	rxresp
 	expect resp.http.foo == "4"
 
-	txreq -url /emptystring
+	txreq -url "/emptystring"
 	rxresp
 	expect resp.http.foo == "6"
 
-	txreq -url /nohdr
+	txreq -url "/nohdr"
 	rxresp
 	expect resp.http.foo == "8"
 
-	txreq -url /ip
+	txreq -url "/ip"
 	rxresp
 	expect resp.http.foo == "9"
 } -run
@@ -89,7 +89,7 @@ client c1 {
 
 	txreq
 	rxresp
-	expect resp.http.foo == "2"
+	expect resp.http.foo == "1"
 
 	txreq
 	rxresp
@@ -97,9 +97,5 @@ client c1 {
 
 	txreq
 	rxresp
-	expect resp.http.foo == "6"
-
-	txreq
-	rxresp
-	expect resp.http.foo == "8"
+	expect resp.http.foo == "9"
 } -run
diff --git a/include/vrt.h b/include/vrt.h
index 556425298..6eaa34690 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -53,6 +53,7 @@
  *
  * unreleased (planned for 2020-03-15)
  *	New prefix_{ptr|len} fields in vrt_backend
+ *	VRT_HashStrands32() added
  * 10.0 (2019-09-15)
  *	VRT_UpperLowerStrands added.
  *	VRT_synth_page now takes STRANDS argument
@@ -546,6 +547,7 @@ VCL_STEVEDORE VRT_stevedore(const char *nm);
 
 int VRT_CompareStrands(VCL_STRANDS a, VCL_STRANDS b);
 VCL_BOOL VRT_Strands2Bool(VCL_STRANDS);
+uint32_t VRT_HashStrands32(VCL_STRANDS);
 char *VRT_Strands(char *, size_t, VCL_STRANDS);
 VCL_STRING VRT_StrandsWS(struct ws *, const char *, VCL_STRANDS);
 VCL_STRING VRT_CollectStrands(VRT_CTX, VCL_STRANDS);
diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c
index 897c5a567..d1f7fd6b3 100644
--- a/lib/libvmod_directors/hash.c
+++ b/lib/libvmod_directors/hash.c
@@ -33,9 +33,6 @@
 
 #include "cache/cache.h"
 
-#include "vend.h"
-#include "vsha256.h"
-
 #include "vdir.h"
 
 #include "vcc_if.h"
@@ -110,26 +107,15 @@ vmod_hash_remove_backend(VRT_CTX,
 VCL_BACKEND v_matchproto_()
 vmod_hash_backend(VRT_CTX, struct vmod_directors_hash *rr, VCL_STRANDS s)
 {
-	struct VSHA256Context sha_ctx;
-	const char *p;
-	unsigned char sha256[VSHA256_LEN];
 	VCL_BACKEND be;
 	double r;
-	int i;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_ORNULL(ctx->bo, BUSYOBJ_MAGIC);
-
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC);
-	VSHA256_Init(&sha_ctx);
-	for (i = 0; i < s->n; i++) {
-		p = s->p[i];
-		if (p != NULL && *p != '\0')
-			VSHA256_Update(&sha_ctx, p, strlen(p));
-	}
-	VSHA256_Final(sha256, &sha_ctx);
-
-	r = vbe32dec(sha256);
+	AN(s);
+
+	r = VRT_HashStrands32(s);
 	r = scalbn(r, -32);
 	assert(r >= 0 && r <= 1.0);
 	be = vdir_pick_be(ctx, rr->vd, r);
diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 532b0d9e8..cb67915c6 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -268,7 +268,7 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
 			ssp[1] = s;
 			ss->p = ssp;
 			shardd->hashcircle[i * replicas + j].point =
-				sharddir_sha256(ss);
+			    VRT_HashStrands32(ss);
 			shardd->hashcircle[i * replicas + j].host = i;
 		}
 		/* not used in current interface */
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index db2c5be78..40472e677 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -40,8 +40,6 @@
 
 #include "vbm.h"
 #include "vrnd.h"
-#include "vsha256.h"
-#include "vend.h"
 
 #include "vcc_if.h"
 #include "shard_dir.h"
@@ -89,25 +87,6 @@ sharddir_err(VRT_CTX, enum VSL_tag_e tag,  const char *fmt, ...)
 	va_end(ap);
 }
 
-uint32_t
-sharddir_sha256(VCL_STRANDS s)
-{
-	struct VSHA256Context sha256;
-	unsigned char digest[VSHA256_LEN];
-	int i;
-
-	AN(s);
-	VSHA256_Init(&sha256);
-	for (i = 0; i < s->n; i++) {
-		if (s->p[i] != NULL)
-			VSHA256_Update(&sha256, s->p[i], strlen(s->p[i]));
-	}
-	VSHA256_Final(digest, &sha256);
-
-	/* The low 32 bits are as good as any. */
-	return (vle32dec(&digest[VSHA256_LEN - 4]));
-}
-
 static int
 shard_lookup(const struct sharddir *shardd, const uint32_t key)
 {
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 6d70a7603..1888aaf4b 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -98,7 +98,6 @@ sharddir_backend(const struct sharddir *shardd, int id)
 
 void sharddir_debug(struct sharddir *shardd, const uint32_t flags);
 void sharddir_err(VRT_CTX, enum VSL_tag_e tag,  const char *fmt, ...);
-uint32_t sharddir_sha256(VCL_STRANDS s);
 void sharddir_new(struct sharddir **sharddp, const char *vcl_name,
     const struct vmod_directors_shard_param *param);
 void sharddir_set_param(struct sharddir *shardd,
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index a5e07e14e..09a547faa 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -252,7 +252,7 @@ vmod_shard_key(VRT_CTX, struct vmod_directors_shard *vshard, VCL_STRANDS s)
 	(void)ctx;
 	(void)vshard;
 
-	return ((VCL_INT)sharddir_sha256(s));
+	return ((VCL_INT)VRT_HashStrands32(s));
 }
 
 VCL_VOID v_matchproto_(td_directors_set_warmup)
@@ -376,7 +376,7 @@ shard_get_key(VRT_CTX, const struct vmod_directors_shard_param *p)
 		sp[0] = http->hd[HTTP_HDR_URL].b;
 		s->n = 1;
 		s->p = sp;
-		return (sharddir_sha256(s));
+		return (VRT_HashStrands32(s));
 	}
 	WRONG("by enum");
 }

From phk at FreeBSD.org  Mon Oct 21 08:34:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 08:34:07 +0000 (UTC)
Subject: [master] 0fa4baead Using memcpy() in bstrcpy() confused FlexeLint,
 use strcpy() instead.
Message-ID: <20191021083407.384D91029DF@lists.varnish-cache.org>


commit 0fa4baead49f0a45f68d3db0b7743c5e4e93ad4d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 21 08:14:20 2019 +0000

    Using memcpy() in bstrcpy() confused FlexeLint, use strcpy() instead.

diff --git a/include/vdef.h b/include/vdef.h
index dc622e273..6781026ef 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -59,9 +59,8 @@
 /* Safe strcpy into a fixed-size buffer */
 #define bstrcpy(dst, src)						\
 	do {								\
-		size_t lbstrcpy = strlen(src) + 1;			\
-		assert(lbstrcpy <= sizeof dst);				\
-		memcpy(dst, src, lbstrcpy);				\
+		assert(strlen(src) + 1 <= sizeof (dst));		\
+		strcpy((dst), (src));					\
 	} while (0)
 
 // TODO #define strcpy BANNED

From phk at FreeBSD.org  Mon Oct 21 08:34:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 08:34:07 +0000 (UTC)
Subject: [master] e9a460f86 Ensure that ban timestamps have a defined
 byte-order.
Message-ID: <20191021083407.1F9D01029DC@lists.varnish-cache.org>


commit e9a460f86c20cc6f9efa2add6e61e0657450c665
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 21 08:01:50 2019 +0000

    Ensure that ban timestamps have a defined byte-order.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index dc4751d2c..354e9fc08 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -134,9 +134,12 @@ vtim_real
 ban_time(const uint8_t *banspec)
 {
 	vtim_real t;
+	uint64_t u;
 
+	assert(sizeof t == sizeof u);
 	assert(sizeof t == (BANS_LENGTH - BANS_TIMESTAMP));
-	memcpy(&t, banspec, sizeof t);
+	u = vbe64dec(banspec + BANS_TIMESTAMP);
+	memcpy(&t, &u, sizeof t);
 	return (t);
 }
 
@@ -157,8 +160,8 @@ ban_equal(const uint8_t *bs1, const uint8_t *bs2)
 	/*
 	 * Compare two ban-strings.
 	 */
-	u = vbe32dec(bs1 + BANS_LENGTH);
-	if (u != vbe32dec(bs2 + BANS_LENGTH))
+	u = ban_len(bs1);
+	if (u != ban_len(bs2))
 		return (0);
 	if (bs1[BANS_FLAGS] & BANS_FLAG_NODEDUP)
 		return (0);
diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c
index 441f71b93..55fcc4bb1 100644
--- a/bin/varnishd/cache/cache_ban_build.c
+++ b/bin/varnishd/cache/cache_ban_build.c
@@ -306,9 +306,11 @@ BAN_Commit(struct ban_proto *bp)
 	struct ban  *b, *bi;
 	ssize_t ln;
 	vtim_real t0;
+	uint64_t u;
 
 	CHECK_OBJ_NOTNULL(bp, BAN_PROTO_MAGIC);
 	AN(bp->vsb);
+	assert(sizeof u == sizeof t0);
 
 	if (ban_shutdown)
 		return (ban_error(bp, "Shutting down"));
@@ -330,10 +332,10 @@ BAN_Commit(struct ban_proto *bp)
 
 	b->flags = bp->flags;
 
-	// XXX why don't we vbe*enc timestamp and flags?
 	memset(b->spec, 0, BANS_HEAD_LEN);
 	t0 = VTIM_real();
-	memcpy(b->spec + BANS_TIMESTAMP, &t0, sizeof t0);
+	memcpy(&u, &t0, sizeof u);
+	vbe64enc(b->spec + BANS_TIMESTAMP, u);
 	b->spec[BANS_FLAGS] = b->flags & 0xff;
 	memcpy(b->spec + BANS_HEAD_LEN, VSB_data(bp->vsb), ln);
 	ln += BANS_HEAD_LEN;

From nils.goroll at uplex.de  Mon Oct 21 08:45:30 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 21 Oct 2019 10:45:30 +0200
Subject: [master] 0fa4baead Using memcpy() in bstrcpy() confused
 FlexeLint, use strcpy() instead.
In-Reply-To: <20191021083407.384D91029DF@lists.varnish-cache.org>
References: <20191021083407.384D91029DF@lists.varnish-cache.org>
Message-ID: <1ed5d96b-c6f3-5665-f311-23a6e369d1c5@uplex.de>

On 21/10/2019 10:34, Poul-Henning Kamp wrote:
> -		size_t lbstrcpy = strlen(src) + 1;			\
> -		assert(lbstrcpy <= sizeof dst);				\
> -		memcpy(dst, src, lbstrcpy);				\
> +		assert(strlen(src) + 1 <= sizeof (dst));		\
> +		strcpy((dst), (src));					\
>  	} while (0)
>  
>  // TODO #define strcpy BANNED

This introduces the conflict with the plan to #define ban strcpy.

Actually this variant of the code had been discussed
https://github.com/varnishcache/varnish-cache/pull/3080

Can't we find a different way to appease flexelint?

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20191021/7fcc8ac2/attachment.bin>

From phk at phk.freebsd.dk  Mon Oct 21 09:11:45 2019
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 09:11:45 +0000
Subject: [master] 0fa4baead Using memcpy() in bstrcpy() confused FlexeLint,
 use strcpy() instead.
In-Reply-To: <1ed5d96b-c6f3-5665-f311-23a6e369d1c5@uplex.de>
References: <20191021083407.384D91029DF@lists.varnish-cache.org>
 <1ed5d96b-c6f3-5665-f311-23a6e369d1c5@uplex.de>
Message-ID: <93219.1571649105@critter.freebsd.dk>

--------
In message <1ed5d96b-c6f3-5665-f311-23a6e369d1c5 at uplex.de>, Nils Goroll writes:

>On 21/10/2019 10:34, Poul-Henning Kamp wrote:
>> -		size_t lbstrcpy =3D strlen(src) + 1;			\
>> -		assert(lbstrcpy <=3D sizeof dst);				\
>> -		memcpy(dst, src, lbstrcpy);				\
>> +		assert(strlen(src) + 1 <=3D sizeof (dst));		\
>> +		strcpy((dst), (src));					\
>>  	} while (0)
>> =20
>>  // TODO #define strcpy BANNED
>
>This introduces the conflict with the plan to #define ban strcpy.

Yes, but we can live with this strcpy() as protected by the assert
for now.  Once it is the last one in the tree, not so much.

>Can't we find a different way to appease flexelint?

I tried, but it seems to use some heuristic for memcpy which
ignores what it learned from strlen() right above.

I didnt want to spend a lot of time on it now.

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From dridi at varni.sh  Mon Oct 21 09:20:36 2019
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Mon, 21 Oct 2019 09:20:36 +0000
Subject: [master] 0fa4baead Using memcpy() in bstrcpy() confused
 FlexeLint, use strcpy() instead.
In-Reply-To: <93219.1571649105@critter.freebsd.dk>
References: <20191021083407.384D91029DF@lists.varnish-cache.org>
 <1ed5d96b-c6f3-5665-f311-23a6e369d1c5@uplex.de>
 <93219.1571649105@critter.freebsd.dk>
Message-ID: <CABoVN9DeAZmA7XKbBA-KaxDKCBUP3yFiQ9MPPAEp0cmg5V3=Zg@mail.gmail.com>

On Mon, Oct 21, 2019 at 9:11 AM Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:
>
> --------
> In message <1ed5d96b-c6f3-5665-f311-23a6e369d1c5 at uplex.de>, Nils Goroll writes:
>
> >On 21/10/2019 10:34, Poul-Henning Kamp wrote:
> >> -            size_t lbstrcpy =3D strlen(src) + 1;                    \
> >> -            assert(lbstrcpy <=3D sizeof dst);                               \
> >> -            memcpy(dst, src, lbstrcpy);                             \
> >> +            assert(strlen(src) + 1 <=3D sizeof (dst));              \
> >> +            strcpy((dst), (src));                                   \
> >>      } while (0)
> >> =20
> >>  // TODO #define strcpy BANNED
> >
> >This introduces the conflict with the plan to #define ban strcpy.
>
> Yes, but we can live with this strcpy() as protected by the assert
> for now.  Once it is the last one in the tree, not so much.
>
> >Can't we find a different way to appease flexelint?
>
> I tried, but it seems to use some heuristic for memcpy which
> ignores what it learned from strlen() right above.
>
> I didnt want to spend a lot of time on it now.

When we get to the point of banning strcpy we can maybe disable the
offending lint before the offending line and restore it afterwards?

We have some margin anyway until we get there.

Dridi

From phk at phk.freebsd.dk  Mon Oct 21 09:26:46 2019
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 09:26:46 +0000
Subject: [master] 0fa4baead Using memcpy() in bstrcpy() confused FlexeLint,
 use strcpy() instead.
In-Reply-To: <CABoVN9DeAZmA7XKbBA-KaxDKCBUP3yFiQ9MPPAEp0cmg5V3=Zg@mail.gmail.com>
References: <20191021083407.384D91029DF@lists.varnish-cache.org>
 <1ed5d96b-c6f3-5665-f311-23a6e369d1c5@uplex.de>
 <93219.1571649105@critter.freebsd.dk>
 <CABoVN9DeAZmA7XKbBA-KaxDKCBUP3yFiQ9MPPAEp0cmg5V3=Zg@mail.gmail.com>
Message-ID: <93316.1571650006@critter.freebsd.dk>

--------
In message <CABoVN9DeAZmA7XKbBA-KaxDKCBUP3yFiQ9MPPAEp0cmg5V3=Zg at mail.gmail.com>
, Dridi Boukelmoune writes:

>When we get to the point of banning strcpy we can maybe disable the
>offending lint before the offending line and restore it afterwards?

Yes, if all else fails.


-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From martin at varnish-software.com  Mon Oct 21 10:05:06 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 10:05:06 +0000 (UTC)
Subject: [master] bd7b3d6d4 Clear err_code and err_reason at start of request
 handling
Message-ID: <20191021100506.8A45710593B@lists.varnish-cache.org>


commit bd7b3d6d47ccbb5e1747126f8e2a297f38e56b8c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 11:17:17 2019 +0200

    Clear err_code and err_reason at start of request handling
    
    req->err_code and req->err_reason are set when going to synthetic
    handling. From there the resp.reason HTTP field is set from
    req->err_reason if set, or the generic code based on req->err_code is used
    if it was NULL. This patch clears these members so that a value from the
    handling of a previous request doesn't linger.
    
    Fixes: VSV00004

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c04eb9bf7..6fdce85a2 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -806,6 +806,8 @@ cnt_recv_prep(struct req *req, const char *ci)
 	req->is_hit = 0;
 	req->is_hitmiss = 0;
 	req->is_hitpass = 0;
+	req->err_code = 0;
+	req->err_reason = NULL;
 }
 
 /*--------------------------------------------------------------------

From martin at varnish-software.com  Mon Oct 21 10:13:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 10:13:07 +0000 (UTC)
Subject: [6.2] 8998a1b20 Clear err_code and err_reason at start of request
 handling
Message-ID: <20191021101307.AAE0D106B23@lists.varnish-cache.org>


commit 8998a1b20eb78750463b2cbe4937f056228c6f33
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 11:17:17 2019 +0200

    Clear err_code and err_reason at start of request handling
    
    req->err_code and req->err_reason are set when going to synthetic
    handling. From there the resp.reason HTTP field is set from
    req->err_reason if set, or the generic code based on req->err_code is used
    if it was NULL. This patch clears these members so that a value from the
    handling of a previous request doesn't linger.
    
    Fixes: VSV00004

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index a310cd32b..ba503fa62 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -802,6 +802,8 @@ cnt_recv_prep(struct req *req, const char *ci)
 	req->is_hit = 0;
 	req->is_hitmiss = 0;
 	req->is_hitpass = 0;
+	req->err_code = 0;
+	req->err_reason = NULL;
 }
 
 /*--------------------------------------------------------------------

From martin at varnish-software.com  Mon Oct 21 10:13:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 10:13:07 +0000 (UTC)
Subject: [6.2] 3ed150689 Prepare for 6.2.2
Message-ID: <20191021101307.C716D106B27@lists.varnish-cache.org>


commit 3ed1506895ecaddb91f658bee11742f0b0b982b5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 11:22:35 2019 +0200

    Prepare for 6.2.2

diff --git a/configure.ac b/configure.ac
index e80c8f179..5586e5f04 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2019 Varnish Software])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [6.2.1], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [6.2.2], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index 3c0e400db..d421bed9b 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -24,6 +24,18 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+================================
+Varnish Cache 6.2.2 (2019-10-21)
+================================
+
+Bugs fixed
+----------
+
+* Fixed clearing of a state variable that could cause an information leak
+  (VSV00004_)
+
+.. _VSV00004: https://varnish-cache.org/security/VSV00004.html
+
 ================================
 Varnish Cache 6.2.1 (2019-09-03)
 ================================

From martin at varnish-software.com  Mon Oct 21 10:15:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 10:15:08 +0000 (UTC)
Subject: [6.3] 99da6e40d Clear err_code and err_reason at start of request
 handling
Message-ID: <20191021101508.5735B107B96@lists.varnish-cache.org>


commit 99da6e40df70824bb72854be96ea649659cd1920
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 11:17:17 2019 +0200

    Clear err_code and err_reason at start of request handling
    
    req->err_code and req->err_reason are set when going to synthetic
    handling. From there the resp.reason HTTP field is set from
    req->err_reason if set, or the generic code based on req->err_code is used
    if it was NULL. This patch clears these members so that a value from the
    handling of a previous request doesn't linger.
    
    Fixes: VSV00004

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index a8ee9c61b..c6f4e7f3e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -807,6 +807,8 @@ cnt_recv_prep(struct req *req, const char *ci)
 	req->is_hit = 0;
 	req->is_hitmiss = 0;
 	req->is_hitpass = 0;
+	req->err_code = 0;
+	req->err_reason = NULL;
 }
 
 /*--------------------------------------------------------------------

From martin at varnish-software.com  Mon Oct 21 10:15:08 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 10:15:08 +0000 (UTC)
Subject: [6.3] 6e96ff048 Prepare for 6.3.1
Message-ID: <20191021101508.876C1107B9B@lists.varnish-cache.org>


commit 6e96ff048692235e64565211a38c41432a26c055
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 11:31:04 2019 +0200

    Prepare for 6.3.1

diff --git a/configure.ac b/configure.ac
index 74d84aa26..fd180c170 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2019 Varnish Software])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [6.3.0], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [6.3.1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index 081ab68a8..dfb9b9636 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -26,6 +26,18 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+================================
+Varnish Cache 6.3.1 (2019-10-21)
+================================
+
+Bugs fixed
+----------
+
+* Fixed clearing of a state variable that could cause an information leak
+  (VSV00004_)
+
+.. _VSV00004: https://varnish-cache.org/security/VSV00004.html
+
 ================================
 Varnish Cache 6.3.0 (2019-09-15)
 ================================

From martin at varnish-software.com  Mon Oct 21 11:29:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 11:29:07 +0000 (UTC)
Subject: [6.0] 83849e018 Clear err_code and err_reason at start of request
 handling
Message-ID: <20191021112907.29A7C10D866@lists.varnish-cache.org>


commit 83849e018f22fe0acbf6fc17bc513afaf60749bd
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 11:17:17 2019 +0200

    Clear err_code and err_reason at start of request handling
    
    req->err_code and req->err_reason are set when going to synthetic
    handling. From there the resp.reason HTTP field is set from
    req->err_reason if set, or the generic code based on req->err_code is used
    if it was NULL. This patch clears these members so that a value from the
    handling of a previous request doesn't linger.
    
    Fixes: VSV00004

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c01d45715..0f32f38c3 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -835,6 +835,8 @@ cnt_recv_prep(struct req *req, const char *ci)
 	req->is_hit = 0;
 	req->is_hitmiss = 0;
 	req->is_hitpass = 0;
+	req->err_code = 0;
+	req->err_reason = NULL;
 }
 /*--------------------------------------------------------------------
  * We have a complete request, set everything up and start it.

From martin at varnish-software.com  Mon Oct 21 11:29:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 11:29:07 +0000 (UTC)
Subject: [6.0] 1896280a3 Add VSV00004 to changes.rst
Message-ID: <20191021112907.44FE910D869@lists.varnish-cache.org>


commit 1896280a3ed76599d85dc07f75799a0271b5512f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Oct 18 13:40:43 2019 +0200

    Add VSV00004 to changes.rst

diff --git a/doc/changes.rst b/doc/changes.rst
index 3dddc3952..c5260b2a0 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -62,6 +62,9 @@ Varnish Cache 6.0.5 (unreleased)
   utility run as the same user as the cache worker. This retires the
   VSM_NOPID environment variable.
 
+* Fixed clearing of a state variable that could case an information leak
+  (VSV00004_)
+
 .. _2395: https://github.com/varnishcache/varnish-cache/issues/2395
 .. _2572: https://github.com/varnishcache/varnish-cache/issues/2572
 .. _2905: https://github.com/varnishcache/varnish-cache/issues/2905
@@ -75,6 +78,7 @@ Varnish Cache 6.0.5 (unreleased)
 .. _2967: https://github.com/varnishcache/varnish-cache/issues/2967
 .. _2977: https://github.com/varnishcache/varnish-cache/issues/2977
 .. _3007: https://github.com/varnishcache/varnish-cache/issues/3007
+.. _VSV00004: https://varnish-cache.org/security/VSV00004.html
 
 ================================
 Varnish Cache 6.0.4 (2019-09-03)

From martin at varnish-software.com  Mon Oct 21 11:29:07 2019
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 21 Oct 2019 11:29:07 +0000 (UTC)
Subject: [6.0] 3065ccaac Prepare for 6.0.5
Message-ID: <20191021112907.6AE7110D86D@lists.varnish-cache.org>


commit 3065ccaacc4bb537fb976a524bd808db42c5fe40
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Oct 18 14:22:54 2019 +0200

    Prepare for 6.0.5

diff --git a/configure.ac b/configure.ac
index b444ed72d..5348d9fd4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2019 Varnish Software])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [6.0.4], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [6.0.5], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index c5260b2a0..b2eeba60e 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -27,7 +27,7 @@ individual releases. These documents are updated as part of the
 release process.
 
 ================================
-Varnish Cache 6.0.5 (unreleased)
+Varnish Cache 6.0.5 (2019-10-21)
 ================================
 
 * Various H2 bug fixes and improvements have been back ported from the

From phk at FreeBSD.org  Mon Oct 21 11:56:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:06 +0000 (UTC)
Subject: [master] a5d2273fa Keep track of dynamic param bounds
Message-ID: <20191021115606.6B5F110E444@lists.varnish-cache.org>


commit a5d2273fa8cb892f4e98f4b0094f291263d9baf0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 16 10:48:04 2019 +0200

    Keep track of dynamic param bounds

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 3a0b57f61..b65884eae 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -708,16 +708,22 @@ MCF_ParamConf(enum mcf_which_e which, const char * const param,
 	AZ(VSB_finish(vsb));
 	switch (which) {
 	case MCF_DEFAULT:
-		pp->def = strdup(VSB_data(vsb));
-		AN(pp->def);
+		free(pp->dyn_def);
+		pp->dyn_def = strdup(VSB_data(vsb));
+		AN(pp->dyn_def);
+		pp->def = pp->dyn_def;
 		break;
 	case MCF_MINIMUM:
-		pp->min = strdup(VSB_data(vsb));
-		AN(pp->min);
+		free(pp->dyn_min);
+		pp->dyn_min = strdup(VSB_data(vsb));
+		AN(pp->dyn_min);
+		pp->min = pp->dyn_min;
 		break;
 	case MCF_MAXIMUM:
-		pp->max = strdup(VSB_data(vsb));
-		AN(pp->max);
+		free(pp->dyn_max);
+		pp->dyn_max = strdup(VSB_data(vsb));
+		AN(pp->dyn_max);
+		pp->max = pp->dyn_max;
 		break;
 	default:
 		WRONG("bad 'which'");
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 7188b64e1..0e5258728 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -55,6 +55,10 @@ struct parspec {
 
 	const char	*def;
 	const char	*units;
+
+	char		*dyn_min;
+	char		*dyn_max;
+	char		*dyn_def;
 };
 
 tweak_t tweak_bool;

From phk at FreeBSD.org  Mon Oct 21 11:56:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:06 +0000 (UTC)
Subject: [master] 0dcf8360c Teach tweak_generic_uint more error reporting
Message-ID: <20191021115606.8690B10E447@lists.varnish-cache.org>


commit 0dcf8360c5b3d21d928ab03589479d280c07d44f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 16 11:49:00 2019 +0200

    Teach tweak_generic_uint more error reporting

diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 0e5258728..2d4936c94 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -72,7 +72,14 @@ tweak_t tweak_uint;
 tweak_t tweak_vsl_buffer;
 tweak_t tweak_vsl_reclen;
 
-int tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest,
+enum tweak_e {
+	TWEAK_OK,
+	TWEAK_ERR,
+	TWEAK_BELOW_MIN,
+	TWEAK_ABOVE_MAX,
+};
+
+enum tweak_e tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest,
     const char *arg, const char *min, const char *max);
 
 extern struct parspec mgt_parspec[]; /* mgt_param_tbl.c */
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index 6900f4e5f..fbb6933dc 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -156,7 +156,7 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 
 /*--------------------------------------------------------------------*/
 
-int
+enum tweak_e
 tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
     const char *min, const char *max)
 {
@@ -169,7 +169,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			minv = strtoul(min, &p, 0);
 			if (*arg == '\0' || *p != '\0') {
 				VSB_printf(vsb, "Illegal Min: %s\n", min);
-				return (-1);
+				return (TWEAK_ERR);
 			}
 		}
 		if (max != NULL) {
@@ -177,7 +177,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			maxv = strtoul(max, &p, 0);
 			if (*arg == '\0' || *p != '\0') {
 				VSB_printf(vsb, "Illegal Max: %s\n", max);
-				return (-1);
+				return (TWEAK_ERR);
 			}
 		}
 		p = NULL;
@@ -187,16 +187,16 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			u = strtoul(arg, &p, 0);
 			if (*arg == '\0' || *p != '\0') {
 				VSB_printf(vsb, "Not a number (%s)\n", arg);
-				return (-1);
+				return (TWEAK_ERR);
 			}
 		}
 		if (min != NULL && u < minv) {
 			VSB_printf(vsb, "Must be at least %s\n", min);
-			return (-1);
+			return (TWEAK_BELOW_MIN);
 		}
 		if (max != NULL && u > maxv) {
 			VSB_printf(vsb, "Must be no more than %s\n", max);
-			return (-1);
+			return (TWEAK_ABOVE_MAX);
 		}
 		*dest = u;
 	} else if (*dest == UINT_MAX && arg != JSON_FMT) {
@@ -204,7 +204,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 	} else {
 		VSB_printf(vsb, "%u", *dest);
 	}
-	return (0);
+	return (TWEAK_OK);
 }
 
 /*--------------------------------------------------------------------*/

From phk at FreeBSD.org  Mon Oct 21 11:56:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:06 +0000 (UTC)
Subject: [master] 52237ffc6 Improve thread pool parameters error messages
Message-ID: <20191021115606.A3F0310E44A@lists.varnish-cache.org>


commit 52237ffc68b36de94c56693ffe2d61460b32e4af
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 16 12:03:12 2019 +0200

    Improve thread pool parameters error messages
    
    Inform users when the parameter we try to change depends on another
    parameter.
    
    This is not meant to be merged as-is, we could have a new VSB function
    to trim the end of the string if it matches the one passed as the second
    argument. In this case, we would trim the trailing newline only of the
    CLI response didn't overflow in the first place, adding the extra
    information on the same line. Alternatively we could also simply add the
    explanation for the dynamic bounds on the next line.
    
    Feedback welcome.
    
    Fixes #3098

diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index da2eab1ca..5184ea147 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -57,26 +57,64 @@ static int
 tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
+	enum tweak_e tweak;
 
-	if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max))
-		return (-1);
-	MCF_ParamConf(MCF_MINIMUM, "thread_pool_max",
-	    "%u", mgt_param.wthread_min);
-	MCF_ParamConf(MCF_MAXIMUM, "thread_pool_reserve",
-	    "%u", mgt_param.wthread_min * 950 / 1000);
-	return (0);
+	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max);
+
+	if (tweak == TWEAK_OK) {
+		MCF_ParamConf(MCF_MINIMUM, "thread_pool_max",
+		    "%u", mgt_param.wthread_min);
+		MCF_ParamConf(MCF_MAXIMUM, "thread_pool_reserve",
+		    "%u", mgt_param.wthread_min * 950 / 1000);
+		return (0);
+	}
+
+	if (arg != JSON_FMT && tweak == TWEAK_ABOVE_MAX) {
+		vsb->s_len--; /* XXX: VSB_trim(vsb, "\n"); instead? */
+		VSB_cat(vsb, " (thread_pool_max)\n");
+	}
+
+	return (-1);
 }
 
 static int
 tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
+	enum tweak_e tweak;
+
+	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max);
+
+	if (tweak == TWEAK_OK) {
+		MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min",
+		    "%u", mgt_param.wthread_max);
+		return (0);
+	}
+
+	if (arg != JSON_FMT && tweak == TWEAK_BELOW_MIN) {
+		vsb->s_len--; /* XXX: VSB_trim(vsb, "\n"); instead? */
+		VSB_cat(vsb, " (thread_pool_min)\n");
+	}
+
+	return (-1);
+}
+
+static int
+tweak_thread_pool_reserve(struct vsb *vsb, const struct parspec *par,
+    const char *arg)
+{
+	enum tweak_e tweak;
+
+	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max);
+	if (tweak == TWEAK_OK)
+		return (0);
+
+	if (arg != JSON_FMT && tweak == TWEAK_ABOVE_MAX) {
+		vsb->s_len--; /* XXX: VSB_trim(vsb, "\n"); instead? */
+		VSB_cat(vsb, " (95% of thread_pool_min)\n");
+	}
 
-	if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max))
-		return (-1);
-	MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min",
-	    "%u", mgt_param.wthread_max);
-	return (0);
+	return (-1);
 }
 
 /*--------------------------------------------------------------------*/
@@ -120,7 +158,8 @@ struct parspec WRK_parspec[] = {
 		"Minimum is 10 threads.",
 		DELAYED_EFFECT,
 		"100", "threads" },
-	{ "thread_pool_reserve", tweak_uint, &mgt_param.wthread_reserve,
+	{ "thread_pool_reserve", tweak_thread_pool_reserve,
+		&mgt_param.wthread_reserve,
 		NULL, NULL,
 		"The number of worker threads reserved for vital tasks "
 		"in each pool.\n"
diff --git a/bin/varnishtest/tests/r03098.vtc b/bin/varnishtest/tests/r03098.vtc
new file mode 100644
index 000000000..a4d3fee67
--- /dev/null
+++ b/bin/varnishtest/tests/r03098.vtc
@@ -0,0 +1,25 @@
+varnishtest "Explain how param.(re)?set may fail"
+
+# NB: we don't need or want to start the cache process
+
+varnish v1 -cliok "param.set thread_pool_max 10000"
+varnish v1 -cliok "param.set thread_pool_min  8000"
+
+# NB: "varnish v1 -cliexpect" wouldn't work with a non-200 status
+shell -err -expect "Must be at least 8000 (thread_pool_min)" {
+	exec varnishadm -n ${v1_name} param.reset thread_pool_max
+}
+
+varnish v1 -cliok "param.set thread_pool_min  8"
+varnish v1 -cliok "param.set thread_pool_max 10"
+
+shell -err -expect "Must be no more than 10 (thread_pool_max)" {
+	exec varnishadm -n ${v1_name} param.reset thread_pool_min
+}
+
+varnish v1 -cliok "param.reset thread_pool_max"
+varnish v1 -cliok "param.reset thread_pool_min"
+
+shell -err -expect "Must be no more than 95 (95% of thread_pool_min)" {
+	exec varnishadm -n ${v1_name} param.set thread_pool_reserve 96
+}

From phk at FreeBSD.org  Mon Oct 21 11:56:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:06 +0000 (UTC)
Subject: [master] b8ee449f9 New replace.cocci semantic patch
Message-ID: <20191021115606.C1CBD10E44E@lists.varnish-cache.org>


commit b8ee449f9331814bea23db78b79f33cecbe03178
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Oct 17 12:04:55 2019 +0200

    New replace.cocci semantic patch

diff --git a/tools/coccinelle/replace.cocci b/tools/coccinelle/replace.cocci
new file mode 100644
index 000000000..f70b7aeb8
--- /dev/null
+++ b/tools/coccinelle/replace.cocci
@@ -0,0 +1,12 @@
+/*
+ * This patch simplifies code using the REPLACE() macro.
+ */
+
+@@
+expression ptr, val;
+@@
+
+- free(ptr);
+- ptr = strdup(val);
+- AN(ptr);
++ REPLACE(ptr, val);

From phk at FreeBSD.org  Mon Oct 21 11:56:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:06 +0000 (UTC)
Subject: [master] 779459acd Apply replace.cocci patch for @fgsch's suggestion
Message-ID: <20191021115606.DEE0810E453@lists.varnish-cache.org>


commit 779459acd1de98e59f22e81caa90a39e7aac3c27
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Oct 17 12:05:17 2019 +0200

    Apply replace.cocci patch for @fgsch's suggestion

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index b65884eae..b15540f32 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -708,21 +708,15 @@ MCF_ParamConf(enum mcf_which_e which, const char * const param,
 	AZ(VSB_finish(vsb));
 	switch (which) {
 	case MCF_DEFAULT:
-		free(pp->dyn_def);
-		pp->dyn_def = strdup(VSB_data(vsb));
-		AN(pp->dyn_def);
+		REPLACE(pp->dyn_def, VSB_data(vsb));
 		pp->def = pp->dyn_def;
 		break;
 	case MCF_MINIMUM:
-		free(pp->dyn_min);
-		pp->dyn_min = strdup(VSB_data(vsb));
-		AN(pp->dyn_min);
+		REPLACE(pp->dyn_min, VSB_data(vsb));
 		pp->min = pp->dyn_min;
 		break;
 	case MCF_MAXIMUM:
-		free(pp->dyn_max);
-		pp->dyn_max = strdup(VSB_data(vsb));
-		AN(pp->dyn_max);
+		REPLACE(pp->dyn_max, VSB_data(vsb));
 		pp->max = pp->dyn_max;
 		break;
 	default:

From phk at FreeBSD.org  Mon Oct 21 11:56:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:07 +0000 (UTC)
Subject: [master] 06bb24099 Add the reason for dynamic bounds in param specs
Message-ID: <20191021115607.0687510E457@lists.varnish-cache.org>


commit 06bb24099deefa80013b419754c9492424119358
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 18 16:01:35 2019 +0200

    Add the reason for dynamic bounds in param specs

diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 2d4936c94..1ce2bdc5f 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -56,6 +56,8 @@ struct parspec {
 	const char	*def;
 	const char	*units;
 
+	const char	*dyn_min_reason;
+	const char	*dyn_max_reason;
 	char		*dyn_min;
 	char		*dyn_max;
 	char		*dyn_def;
@@ -80,7 +82,8 @@ enum tweak_e {
 };
 
 enum tweak_e tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest,
-    const char *arg, const char *min, const char *max);
+    const char *arg, const char *min, const char *max,
+    const char *min_reason, const char *max_reason);
 
 extern struct parspec mgt_parspec[]; /* mgt_param_tbl.c */
 extern struct parspec VSL_parspec[]; /* mgt_param_vsl.c */
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index fbb6933dc..3ae97aa95 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -158,7 +158,8 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 
 enum tweak_e
 tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
-    const char *min, const char *max)
+    const char *min, const char *max,
+    const char *min_reason, const char *max_reason)
 {
 	unsigned u, minv = 0, maxv = 0;
 	char *p;
@@ -191,11 +192,17 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			}
 		}
 		if (min != NULL && u < minv) {
-			VSB_printf(vsb, "Must be at least %s\n", min);
+			VSB_printf(vsb, "Must be at least %s", min);
+			if (min_reason != NULL)
+				VSB_printf(vsb, " %s", min_reason);
+			VSB_putc(vsb, '\n');
 			return (TWEAK_BELOW_MIN);
 		}
 		if (max != NULL && u > maxv) {
-			VSB_printf(vsb, "Must be no more than %s\n", max);
+			VSB_printf(vsb, "Must be no more than %s", max);
+			if (max_reason != NULL)
+				VSB_printf(vsb, " %s", max_reason);
+			VSB_putc(vsb, '\n');
 			return (TWEAK_ABOVE_MAX);
 		}
 		*dest = u;
@@ -215,7 +222,8 @@ tweak_uint(struct vsb *vsb, const struct parspec *par, const char *arg)
 	volatile unsigned *dest;
 
 	dest = par->priv;
-	return (tweak_generic_uint(vsb, dest, arg, par->min, par->max));
+	return (tweak_generic_uint(vsb, dest, arg, par->min, par->max,
+	    par->dyn_min_reason, par->dyn_max_reason));
 }
 
 /*--------------------------------------------------------------------*/
@@ -415,11 +423,13 @@ tweak_poolparam(struct vsb *vsb, const struct parspec *par, const char *arg)
 			}
 			px = *pp;
 			retval = tweak_generic_uint(vsb, &px.min_pool, av[1],
-			    par->min, par->max);
+			    par->min, par->max, par->dyn_min_reason,
+			    par->dyn_max_reason);
 			if (retval)
 				break;
 			retval = tweak_generic_uint(vsb, &px.max_pool, av[2],
-			    par->min, par->max);
+			    par->min, par->max, par->dyn_min_reason,
+			    par->dyn_max_reason);
 			if (retval)
 				break;
 			retval = tweak_generic_double(vsb,
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 5184ea147..b2728de6d 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -59,7 +59,8 @@ tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par,
 {
 	enum tweak_e tweak;
 
-	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max);
+	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
+	    par->dyn_min_reason, par->dyn_max_reason);
 
 	if (tweak == TWEAK_OK) {
 		MCF_ParamConf(MCF_MINIMUM, "thread_pool_max",
@@ -69,11 +70,6 @@ tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par,
 		return (0);
 	}
 
-	if (arg != JSON_FMT && tweak == TWEAK_ABOVE_MAX) {
-		vsb->s_len--; /* XXX: VSB_trim(vsb, "\n"); instead? */
-		VSB_cat(vsb, " (thread_pool_max)\n");
-	}
-
 	return (-1);
 }
 
@@ -83,7 +79,8 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
 {
 	enum tweak_e tweak;
 
-	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max);
+	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
+	    par->dyn_min_reason, par->dyn_max_reason);
 
 	if (tweak == TWEAK_OK) {
 		MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min",
@@ -91,29 +88,6 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
 		return (0);
 	}
 
-	if (arg != JSON_FMT && tweak == TWEAK_BELOW_MIN) {
-		vsb->s_len--; /* XXX: VSB_trim(vsb, "\n"); instead? */
-		VSB_cat(vsb, " (thread_pool_min)\n");
-	}
-
-	return (-1);
-}
-
-static int
-tweak_thread_pool_reserve(struct vsb *vsb, const struct parspec *par,
-    const char *arg)
-{
-	enum tweak_e tweak;
-
-	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max);
-	if (tweak == TWEAK_OK)
-		return (0);
-
-	if (arg != JSON_FMT && tweak == TWEAK_ABOVE_MAX) {
-		vsb->s_len--; /* XXX: VSB_trim(vsb, "\n"); instead? */
-		VSB_cat(vsb, " (95% of thread_pool_min)\n");
-	}
-
 	return (-1);
 }
 
@@ -146,7 +120,8 @@ struct parspec WRK_parspec[] = {
 		"worker threads soak up RAM and CPU and generally just get "
 		"in the way of getting work done.",
 		DELAYED_EFFECT,
-		"5000", "threads" },
+		"5000", "threads",
+		"(thread_pool_min)" },
 	{ "thread_pool_min", tweak_thread_pool_min, &mgt_param.wthread_min,
 		NULL, NULL,
 		"The minimum number of worker threads in each pool. The "
@@ -157,8 +132,9 @@ struct parspec WRK_parspec[] = {
 		"\n"
 		"Minimum is 10 threads.",
 		DELAYED_EFFECT,
-		"100", "threads" },
-	{ "thread_pool_reserve", tweak_thread_pool_reserve,
+		"100", "threads",
+		NULL, "(thread_pool_max)" },
+	{ "thread_pool_reserve", tweak_uint,
 		&mgt_param.wthread_reserve,
 		NULL, NULL,
 		"The number of worker threads reserved for vital tasks "
@@ -177,7 +153,8 @@ struct parspec WRK_parspec[] = {
 		"Default is 0 to auto-tune (currently 5% of thread_pool_min).\n"
 		"Minimum is 1 otherwise, maximum is 95% of thread_pool_min.",
 		DELAYED_EFFECT,
-		"0", "threads" },
+		"0", "threads",
+		NULL, "(95% of thread_pool_min)" },
 	{ "thread_pool_timeout",
 		tweak_timeout, &mgt_param.wthread_timeout,
 		"10", NULL,

From phk at FreeBSD.org  Mon Oct 21 11:56:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:07 +0000 (UTC)
Subject: [master] aebdba1ab Undo the enum tweak change
Message-ID: <20191021115607.21E2210E45B@lists.varnish-cache.org>


commit aebdba1ab6cb99c1916b7d3d19d6c056d04777ca
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 18 16:04:52 2019 +0200

    Undo the enum tweak change

diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 1ce2bdc5f..82eda8671 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -74,14 +74,7 @@ tweak_t tweak_uint;
 tweak_t tweak_vsl_buffer;
 tweak_t tweak_vsl_reclen;
 
-enum tweak_e {
-	TWEAK_OK,
-	TWEAK_ERR,
-	TWEAK_BELOW_MIN,
-	TWEAK_ABOVE_MAX,
-};
-
-enum tweak_e tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest,
+int tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest,
     const char *arg, const char *min, const char *max,
     const char *min_reason, const char *max_reason);
 
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index 3ae97aa95..ac557fd58 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -156,7 +156,7 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 
 /*--------------------------------------------------------------------*/
 
-enum tweak_e
+int
 tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
     const char *min, const char *max,
     const char *min_reason, const char *max_reason)
@@ -170,7 +170,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			minv = strtoul(min, &p, 0);
 			if (*arg == '\0' || *p != '\0') {
 				VSB_printf(vsb, "Illegal Min: %s\n", min);
-				return (TWEAK_ERR);
+				return (-1);
 			}
 		}
 		if (max != NULL) {
@@ -178,7 +178,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			maxv = strtoul(max, &p, 0);
 			if (*arg == '\0' || *p != '\0') {
 				VSB_printf(vsb, "Illegal Max: %s\n", max);
-				return (TWEAK_ERR);
+				return (-1);
 			}
 		}
 		p = NULL;
@@ -188,7 +188,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			u = strtoul(arg, &p, 0);
 			if (*arg == '\0' || *p != '\0') {
 				VSB_printf(vsb, "Not a number (%s)\n", arg);
-				return (TWEAK_ERR);
+				return (-1);
 			}
 		}
 		if (min != NULL && u < minv) {
@@ -196,14 +196,14 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			if (min_reason != NULL)
 				VSB_printf(vsb, " %s", min_reason);
 			VSB_putc(vsb, '\n');
-			return (TWEAK_BELOW_MIN);
+			return (-1);
 		}
 		if (max != NULL && u > maxv) {
 			VSB_printf(vsb, "Must be no more than %s", max);
 			if (max_reason != NULL)
 				VSB_printf(vsb, " %s", max_reason);
 			VSB_putc(vsb, '\n');
-			return (TWEAK_ABOVE_MAX);
+			return (-1);
 		}
 		*dest = u;
 	} else if (*dest == UINT_MAX && arg != JSON_FMT) {
@@ -211,7 +211,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 	} else {
 		VSB_printf(vsb, "%u", *dest);
 	}
-	return (TWEAK_OK);
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index b2728de6d..3ec53f743 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -57,38 +57,30 @@ static int
 tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
-	enum tweak_e tweak;
 
-	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
-	    par->dyn_min_reason, par->dyn_max_reason);
+	if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
+	    par->dyn_min_reason, par->dyn_max_reason))
+		return (-1);
 
-	if (tweak == TWEAK_OK) {
-		MCF_ParamConf(MCF_MINIMUM, "thread_pool_max",
-		    "%u", mgt_param.wthread_min);
-		MCF_ParamConf(MCF_MAXIMUM, "thread_pool_reserve",
-		    "%u", mgt_param.wthread_min * 950 / 1000);
-		return (0);
-	}
-
-	return (-1);
+	MCF_ParamConf(MCF_MINIMUM, "thread_pool_max",
+	    "%u", mgt_param.wthread_min);
+	MCF_ParamConf(MCF_MAXIMUM, "thread_pool_reserve",
+	    "%u", mgt_param.wthread_min * 950 / 1000);
+	return (0);
 }
 
 static int
 tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
-	enum tweak_e tweak;
-
-	tweak = tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
-	    par->dyn_min_reason, par->dyn_max_reason);
 
-	if (tweak == TWEAK_OK) {
-		MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min",
-		    "%u", mgt_param.wthread_max);
-		return (0);
-	}
+	if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
+	    par->dyn_min_reason, par->dyn_max_reason))
+		return (-1);
 
-	return (-1);
+	MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min",
+	    "%u", mgt_param.wthread_max);
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/

From phk at FreeBSD.org  Mon Oct 21 11:56:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:07 +0000 (UTC)
Subject: [master] dd3d865c6 Hide tweak_generic_uint()
Message-ID: <20191021115607.3B9C710E45F@lists.varnish-cache.org>


commit dd3d865c668106a15d94c23e80d578894160dbbd
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Oct 21 12:09:20 2019 +0200

    Hide tweak_generic_uint()
    
    It is only really needed for one last use case.

diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 82eda8671..c1e799874 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -74,10 +74,6 @@ tweak_t tweak_uint;
 tweak_t tweak_vsl_buffer;
 tweak_t tweak_vsl_reclen;
 
-int tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest,
-    const char *arg, const char *min, const char *max,
-    const char *min_reason, const char *max_reason);
-
 extern struct parspec mgt_parspec[]; /* mgt_param_tbl.c */
 extern struct parspec VSL_parspec[]; /* mgt_param_vsl.c */
 extern struct parspec WRK_parspec[]; /* mgt_pool.c */
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index ac557fd58..fcb9a3bd5 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -156,7 +156,7 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 
 /*--------------------------------------------------------------------*/
 
-int
+static int
 tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
     const char *min, const char *max,
     const char *min_reason, const char *max_reason)
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 3ec53f743..8fb30ede5 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -58,8 +58,7 @@ tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
 
-	if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
-	    par->dyn_min_reason, par->dyn_max_reason))
+	if (tweak_uint(vsb, par, arg))
 		return (-1);
 
 	MCF_ParamConf(MCF_MINIMUM, "thread_pool_max",
@@ -74,8 +73,7 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
 
-	if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max,
-	    par->dyn_min_reason, par->dyn_max_reason))
+	if (tweak_uint(vsb, par, arg))
 		return (-1);
 
 	MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min",

From phk at FreeBSD.org  Mon Oct 21 11:56:07 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 11:56:07 +0000 (UTC)
Subject: [master] 2e7544c9a Add a note regarding RST generation
Message-ID: <20191021115607.5450010E463@lists.varnish-cache.org>


commit 2e7544c9a5caafdc1b59b34c4016316b85ff21ec
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Oct 21 12:17:17 2019 +0200

    Add a note regarding RST generation

diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 8fb30ede5..7bbc1f959 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -81,7 +81,10 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
 	return (0);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * The thread pool parameter definitions used to generate the varnishd
+ * manual. Check the generated RST after updating.
+ */
 
 struct parspec WRK_parspec[] = {
 	{ "thread_pools", tweak_uint, &mgt_param.wthread_pools,

From nils.goroll at uplex.de  Mon Oct 21 12:57:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 21 Oct 2019 12:57:06 +0000 (UTC)
Subject: [master] 035d982b0 Use a logexpect to check for the substance of
 c00036.vtc
Message-ID: <20191021125706.A817C110252@lists.varnish-cache.org>


commit 035d982b0a98611f70a68bd3536d68da22a1ae57
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 21 14:55:45 2019 +0200

    Use a logexpect to check for the substance of c00036.vtc
    
    brought up on IRC by @scn

diff --git a/bin/varnishtest/tests/c00036.vtc b/bin/varnishtest/tests/c00036.vtc
index 4e26467c7..df0574138 100644
--- a/bin/varnishtest/tests/c00036.vtc
+++ b/bin/varnishtest/tests/c00036.vtc
@@ -18,6 +18,25 @@ varnish v1 -vcl+backend {
 	}
 } -start
 
+logexpect l1 -v v1 -q "vxid == 1004" {
+	expect * 1004 VCL_return      {^fetch}
+	expect 0 1004 BackendOpen     {^\d+ s1}
+	expect 0 1004 Timestamp       {^Bereq:}
+
+	# purpose of this vtc: test the internal retry when the
+	# backend goes away on a keepalive TCP connection:
+	expect 0 1004 FetchError      {^HTC eof .-1.}
+	expect 0 1004 BackendClose    {^\d+ s1}
+	expect 0 1004 BackendOpen     {^\d+ s1}
+
+	expect 0 1004 Timestamp       {^Bereq:}
+	expect 0 1004 Timestamp       {^Beresp:}
+	expect 0 1004 BerespProtocol  {^HTTP/1.1}
+	expect 0 1004 BerespStatus    {^200}
+	expect 0 1004 BerespReason    {^OK}
+	expect 0 1004 BerespHeader    {^Content-Length: 6}
+} -start
+
 client c1 {
 	txreq
 	rxresp
@@ -31,3 +50,4 @@ client c1 {
 } -run
 
 varnish v1 -expect backend_retry == 1
+logexpect l1 -wait

From phk at FreeBSD.org  Mon Oct 21 16:19:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 21 Oct 2019 16:19:06 +0000 (UTC)
Subject: [master] f28417d18 Minor FlexeLinting
Message-ID: <20191021161906.3821B1157AB@lists.varnish-cache.org>


commit f28417d1803fca8d35db2c3bf7590109644ae048
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 21 12:17:47 2019 +0000

    Minor FlexeLinting

diff --git a/lib/libvmod_blob/hex.c b/lib/libvmod_blob/hex.c
index a06e3da22..6476308ae 100644
--- a/lib/libvmod_blob/hex.c
+++ b/lib/libvmod_blob/hex.c
@@ -82,7 +82,7 @@ hex_encode(const enum encoding enc, const enum case_e kase,
 {
 	char *p = buf;
 	const char *alphabet = hex_alphabet[0];
-	int i;
+	size_t i;
 
 	AN(buf);
 	assert(enc == HEX);
diff --git a/lib/libvmod_blob/url.c b/lib/libvmod_blob/url.c
index b038a556d..0134d3859 100644
--- a/lib/libvmod_blob/url.c
+++ b/lib/libvmod_blob/url.c
@@ -86,7 +86,7 @@ url_encode(const enum encoding enc, const enum case_e kase,
 	char *p = buf;
 	const char * const end = buf + buflen;
 	const char *alphabet = hex_alphabet[0];
-	int i;
+	size_t i;
 
 	AN(buf);
 	assert(enc == URL);

From nils.goroll at uplex.de  Mon Oct 21 17:48:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 21 Oct 2019 17:48:06 +0000 (UTC)
Subject: [master] 57ba8f3ef save some time later
Message-ID: <20191021174806.9817611746E@lists.varnish-cache.org>


commit 57ba8f3ef931011b01bc6cf384649b75e2ad0892
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 21 19:46:47 2019 +0200

    save some time later

diff --git a/include/vdef.h b/include/vdef.h
index 6781026ef..250ff2bfe 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -64,6 +64,8 @@
 	} while (0)
 
 // TODO #define strcpy BANNED
+// TODO then revert 0fa4baead49f0a45f68d3db0b7743c5e4e93ad4d
+// TODO and replace with flexelint exception
 
 /* Close and discard filedescriptor */
 #define closefd(fdp)				\

From dridi.boukelmoune at gmail.com  Tue Oct 22 08:32:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 22 Oct 2019 08:32:06 +0000 (UTC)
Subject: [master] ddd5df3d4 Use the VCL_HEADER type in VRT code
Message-ID: <20191022083206.2E618105868@lists.varnish-cache.org>


commit ddd5df3d40938e899404e33b1b353ff480dbd781
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 22 10:27:42 2019 +0200

    Use the VCL_HEADER type in VRT code

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 07587008b..4ec470ea0 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -158,7 +158,7 @@ VRT_selecthttp(VRT_CTX, enum gethdr_e where)
 /*--------------------------------------------------------------------*/
 
 VCL_STRING
-VRT_GetHdr(VRT_CTX, const struct gethdr_s *hs)
+VRT_GetHdr(VRT_CTX, VCL_HEADER hs)
 {
 	VCL_HTTP hp;
 	const char *p;
@@ -571,8 +571,7 @@ VRT_UpperLowerStrands(VRT_CTX, VCL_STRANDS s, int up)
 /*--------------------------------------------------------------------*/
 
 VCL_VOID
-VRT_SetHdr(VRT_CTX , const struct gethdr_s *hs,
-    const char *p, ...)
+VRT_SetHdr(VRT_CTX , VCL_HEADER hs, const char *p, ...)
 {
 	VCL_HTTP hp;
 	va_list ap;

From guillaume at varnish-software.com  Wed Oct 23 19:13:09 2019
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Wed, 23 Oct 2019 19:13:09 +0000 (UTC)
Subject: [master] 5ff94556f [cci] Various fixes and improvements
Message-ID: <20191023191309.5F4B410D7CB@lists.varnish-cache.org>


commit 5ff94556fc195b6bed54f49515db8e206d090d33
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Thu Oct 17 21:04:48 2019 -0700

    [cci] Various fixes and improvements
    
    - fix centos:7 distcheck
    - use centos to build the dist tarball
    - simplify git cloning with builtin command
    - no need for ssh when building
    - tar the dsc files with the deb ones
    - use the regular pkg repo now that the alpine PR has been merged

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6a33b3a65..d9ebaed2f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -47,10 +47,9 @@ commands:
                 make \
                 openssh-clients \
                 pcre-devel \
-                python-docutils \
-                python3-sphinx \
+                python3 \
+                python-sphinx \
                 rst2pdf \
-                ssh \
                 sudo
   alpine_install_build_deps:
     description: Install build dependencies
@@ -138,11 +137,12 @@ jobs:
           name: Import the packages into the workspace
           command: |
             mkdir debs
-            mv ../*.deb debs/
+            mv ../*.deb ../*.dsc debs/
       - persist_to_workspace:
           root: .
           paths:
             - debs/varnish*.deb
+            - debs/varnish*.dsc
   build_alpine:
     description: Build alpine apks
     docker:
@@ -203,23 +203,15 @@ jobs:
           root: .
           paths:
             - apks/*.apk
-  dist_ubuntu:
+  dist:
     docker:
-      - image: ubuntu:bionic
+      - image: centos:7
     steps:
-      - run:
-          command: |
-            export DEBIAN_FRONTEND=noninteractive
-            export DEBCONF_NONINTERACTIVE_SEEN=true
-            apt-get update
-            apt-get install -y python3-sphinx autoconf automake libedit-dev make libtool pkg-config git libconfig-dev libpcre3-dev
+      - centos_install_build_deps
+      - checkout
       - run:
           name: Create the dist tarball
           command: |
-            mkdir -p ~/.ssh
-            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
-            git clone --branch=${CIRCLE_BRANCH} ${CIRCLE_REPOSITORY_URL} .
-            git checkout ${CIRCLE_SHA1}
             ./autogen.des --quiet
             make dist -j 16
       - persist_to_workspace:
@@ -241,7 +233,7 @@ jobs:
             mkdir -p ~/.ssh
             ssh-keyscan -H github.com >> ~/.ssh/known_hosts
             echo ${CIRCLE_REPOSITORY_URL}
-            git clone --branch=alpine https://github.com/gquintard/pkg-varnish-cache.git .
+            git clone --branch=weekly https://github.com/varnishcache/pkg-varnish-cache.git .
             git checkout alpine
             tar cvzf debian.tar.gz debian --dereference
             tar cvzf redhat.tar.gz redhat --dereference 
@@ -309,9 +301,7 @@ jobs:
       - run:
           name: Tar the packages
           command: |
-              rm rpms/varnish*.src.rpm
-              mv rpms/*/*.rpm rpms/
-              tar cvzf packages.tar.gz rpms/*.rpm debs/*.deb apks/*.apk
+              tar cvzf packages.tar.gz rpms/*.rpm debs/*.deb debs/*.dsc apks/*.apk
       - store_artifacts:
           destination: packages.tar.gz
           path: packages.tar.gz
@@ -401,14 +391,14 @@ jobs:
 
 pkg_req: &pkg_req
   requires:
-    - dist_ubuntu
+    - dist
     - tar_pkg_tools
 
 workflows:
   version: 2
   build:
     jobs:
-      - dist_ubuntu
+      - dist
       - tar_pkg_tools
       - build_debs:
           name: build_debian_stretch
@@ -442,23 +432,23 @@ workflows:
             - build_ubuntu_bionic
             - build_centos_7
             - build_alpine
-#      - distcheck:
-#          name: distcheck_centos_7
-#          dist: centos
-#          release: "7"
-#          requires:
-#            - dist_ubuntu
+      - distcheck:
+          name: distcheck_centos_7
+          dist: centos
+          release: "7"
+          requires:
+            - dist
       - distcheck:
           name: distcheck_debian_buster
           dist: debian
           release: buster
           extra_conf: --enable-asan --enable-ubsan
           requires:
-            - dist_ubuntu
+            - dist
       - distcheck:
           name: distcheck_alpine_3.10
           dist: alpine
           release: "latest"
           #extra_conf: --without-jemalloc
           requires:
-            - dist_ubuntu
+            - dist

From dridi.boukelmoune at gmail.com  Thu Oct 24 16:58:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 24 Oct 2019 16:58:06 +0000 (UTC)
Subject: [master] e6193fb46 Support dquote-delimited fields in VSL records
Message-ID: <20191024165806.10DEE106626@lists.varnish-cache.org>


commit e6193fb46dd225c3f68317af82b69c94561f5766
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Dec 21 11:05:31 2018 +0100

    Support dquote-delimited fields in VSL records
    
    When we output a VSL field containing spaces, all bets are off for VSL
    queries relying on that field or subsequent fields in the same record.
    The solution is to allow a quoted-string format for such fields.

diff --git a/bin/varnishtest/tests/r02872.vtc b/bin/varnishtest/tests/r02872.vtc
new file mode 100644
index 000000000..e43af59cb
--- /dev/null
+++ b/bin/varnishtest/tests/r02872.vtc
@@ -0,0 +1,26 @@
+varnishtest "VSL quoted fields"
+
+varnish v1 -vcl {
+	import std;
+        backend be none;
+	sub vcl_recv {
+		# a series of 3-fields log records
+		std.log({"  custom   log     "ok" "});
+		std.log({" "valid"  "fields"  ok "});
+		std.log({" "missing""blank"   ko "});
+		std.log({"  missing  dquote  "ko "});
+		# "
+		return (synth(200));
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+# records with malformed fields don't show up
+shell -expect "2" {
+        varnishlog -d -n ${v1_name} -g raw -q 'VCL_Log[3]' | wc -l
+}
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index a60bc6017..227591190 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -122,7 +122,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	double lhs_float = 0.;
 	const char *b, *e, *q;
 	char *p;
-	int i;
+	int i, dq;
 
 	AN(vex);
 	AN(rec);
@@ -144,15 +144,40 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 
 	/* Field */
 	if (vex->lhs->field > 0) {
-		for (e = b, i = 0; *e && i < vex->lhs->field; i++) {
+		for (e = b, i = 0, dq = 0; *e && i < vex->lhs->field; i++) {
+			/* Skip end of previous field */
+			if (dq) {
+				assert(e > b);
+				assert(*e == '"');
+				dq = 0;
+				e++;
+				if (*e == '\0')
+					break;
+				if (!isspace(*e))
+					return (0);
+			}
+
 			b = e;
 			/* Skip ws */
 			while (*b && isspace(*b))
 				b++;
+
+			dq = (*b == '"');
+			if (dq)
+				b++;
 			e = b;
-			/* Skip non-ws */
-			while (*e && !isspace(*e))
-				e++;
+
+			if (dq) {
+				/* Find end of string */
+				while (*e && *e != '"')
+					e++;
+				if (*e != '"')
+					return (0);
+			} else {
+				/* Skip non-ws */
+				while (*e && !isspace(*e))
+					e++;
+			}
 		}
 		assert(b <= e);
 		if (*b == '\0' || i < vex->lhs->field)

From dridi.boukelmoune at gmail.com  Thu Oct 24 16:58:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 24 Oct 2019 16:58:06 +0000 (UTC)
Subject: [master] 49a2600e7 Quote VSL fields that are known to contain spaces
Message-ID: <20191024165806.29E47106629@lists.varnish-cache.org>


commit 49a2600e71959b7989197013d16c7560d1fb0fb8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Dec 21 11:09:03 2018 +0100

    Quote VSL fields that are known to contain spaces
    
    The test case covers the last field of Backend_health records.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 0b7c5cede..9e86c9b62 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -508,7 +508,7 @@ vca_accept_task(struct worker *wrk, void *arg)
 				    lport, VTCP_PORTBUFSIZE);
 			}
 
-			VSL(SLT_SessError, 0, "%s %s %s %d %d %s",
+			VSL(SLT_SessError, 0, "%s %s %s %d %d \"%s\"",
 			    wa.acceptlsock->name, laddr, lport,
 			    ls->sock, i, vstrerror(i));
 			(void)Pool_TrySumstat(wrk);
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index 571154859..d51da01a2 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -181,7 +181,7 @@ VBP_Update_Backend(struct vbp_target *vt)
 	vt->backend->sick = i;
 
 	AN(dir->vcl_name);
-	VSL(SLT_Backend_health, 0, "%s %s %s %s %u %u %u %.6f %.6f %s",
+	VSL(SLT_Backend_health, 0, "%s %s %s %s %u %u %u %.6f %.6f \"%s\"",
 	    dir->vcl_name, chg ? "Went" : "Still",
 	    i ? "sick" : "healthy", bits,
 	    vt->good, vt->threshold, vt->window,
diff --git a/bin/varnishtest/tests/r02872.vtc b/bin/varnishtest/tests/r02872.vtc
index e43af59cb..657cd6c7f 100644
--- a/bin/varnishtest/tests/r02872.vtc
+++ b/bin/varnishtest/tests/r02872.vtc
@@ -1,8 +1,16 @@
 varnishtest "VSL quoted fields"
 
+server s1 {
+        rxreq
+        txresp
+} -start
+
 varnish v1 -vcl {
 	import std;
-        backend be none;
+        backend be {
+                .host = "${s1_sock}";
+                .probe = { .interval = 1m; }
+        }
 	sub vcl_recv {
 		# a series of 3-fields log records
 		std.log({"  custom   log     "ok" "});
@@ -24,3 +32,10 @@ client c1 {
 shell -expect "2" {
         varnishlog -d -n ${v1_name} -g raw -q 'VCL_Log[3]' | wc -l
 }
+
+server s1 -wait
+
+shell -expect "Went healthy" {
+        varnishlog -d -n ${v1_name} -g raw \
+		-q 'Backend_health[10] eq "HTTP/1.1 200 OK"'
+}
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 905042cbe..f38f2f3c2 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -191,7 +191,7 @@ SLTM(Length, 0, "Size of object body",
 
 SLTM(FetchError, 0, "Error while fetching object",
 	"Logs the error message of a failed fetch operation.\n\n"
-	"Error messages should be self-explanatory, yet the http connection"
+	"Error messages should be self-explanatory, yet the http connection\n"
 	"(HTC) class of errors is reported with these symbols:\n\n"
 	"\t* junk (-5): Received unexpected data\n"
 	"\t* close (-4): Connection closed\n"
@@ -272,11 +272,11 @@ SLTM(TTL, 0, "TTL set on object",
 SLTM(Fetch_Body, 0, "Body fetched from backend",
 	"Ready to fetch body from backend.\n\n"
 	"The format is::\n\n"
-	"\t%d (%s) %s\n"
-	"\t|   |    |\n"
-	"\t|   |    +---- 'stream' or '-'\n"
-	"\t|   +--------- Text description of body fetch mode\n"
-	"\t+------------- Body fetch mode\n"
+	"\t%d %s %s\n"
+	"\t|  |  |\n"
+	"\t|  |  +---- 'stream' or '-'\n"
+	"\t|  +------- Text description of body fetch mode\n"
+	"\t+---------- Body fetch mode\n"
 	"\n"
 )
 

From dridi.boukelmoune at gmail.com  Thu Oct 24 16:58:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 24 Oct 2019 16:58:06 +0000 (UTC)
Subject: [master] 28ba64820 It is now possible to match empty fields
Message-ID: <20191024165806.445B810662D@lists.varnish-cache.org>


commit 28ba64820ac71a832e3169da3a706ba610b5b1dc
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Dec 24 12:14:02 2018 +0100

    It is now possible to match empty fields
    
    The VSL query could in theory do that, but I don't think it was
    actionable without double quotes to delimit empty fields or fields
    containing spaces.

diff --git a/bin/varnishtest/tests/r02872.vtc b/bin/varnishtest/tests/r02872.vtc
index 657cd6c7f..e7d1174fe 100644
--- a/bin/varnishtest/tests/r02872.vtc
+++ b/bin/varnishtest/tests/r02872.vtc
@@ -39,3 +39,8 @@ shell -expect "Went healthy" {
         varnishlog -d -n ${v1_name} -g raw \
 		-q 'Backend_health[10] eq "HTTP/1.1 200 OK"'
 }
+
+# s1 starts sick before the first probe request is made
+shell -expect "Went sick" {
+        varnishlog -d -n ${v1_name} -g raw -q 'Backend_health[10] eq ""'
+}

From dridi.boukelmoune at gmail.com  Thu Oct 24 16:58:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 24 Oct 2019 16:58:06 +0000 (UTC)
Subject: [master] 58e8d1341 Un-indent the VSL NOSUP_NOTICE
Message-ID: <20191024165806.63C45106631@lists.varnish-cache.org>


commit 58e8d1341e7e5adf3b987c2fb3a335a374d09ea1
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 16 10:21:59 2019 +0200

    Un-indent the VSL NOSUP_NOTICE
    
    Otherwise it is considered a code block instead of a paragraph where it
    is used.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index f38f2f3c2..0252d0743 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -51,9 +51,9 @@
  * kept for now for VSL binary compatibility
  */
 #define NOSUP_NOTICE \
-    "\tNOTE: This tag is currently not in use in the Varnish log.\n" \
-    "\tIt is mentioned here to document legacy versions of the log,\n" \
-    "\tor reserved for possible use in future versions.\n\n"
+    "NOTE: This tag is currently not in use in the Varnish log.\n" \
+    "It is mentioned here to document legacy versions of the log,\n" \
+    "or reserved for possible use in future versions.\n\n"
 
 SLTM(Debug, SLT_F_UNSAFE, "Debug messages",
 	"Debug messages can normally be ignored, but are sometimes"

From dridi.boukelmoune at gmail.com  Thu Oct 24 16:58:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 24 Oct 2019 16:58:06 +0000 (UTC)
Subject: [master] 97cebdec2 Add a notice for *Header VSL records too
Message-ID: <20191024165806.7F53A106635@lists.varnish-cache.org>


commit 97cebdec2fe5bc0fc51f51e4c6667864af4be776
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 16 11:02:31 2019 +0200

    Add a notice for *Header VSL records too

diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index 5e38ef2da..249e35c98 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -43,6 +43,11 @@
  *
  */
 
+#define HEADER_NOTICE \
+    "NOTE: HTTP header fields are free form records and not strictly\n" \
+    "made of 2 fields. Accessing a specific header with the prefix\n" \
+    "notation helps treating the header value as a single string.\n\n"
+
 /*lint -save -e525 -e539 -e835 */
 
 SLTH(Method,	HTTP_HDR_METHOD,	1, 0, "method",
@@ -73,6 +78,7 @@ SLTH(Header,	HTTP_HDR_FIRST,		1, 1, "header",
 	"\t|   +- Header value\n"
 	"\t+----- Header name\n"
 	"\n"
+	HEADER_NOTICE
 )
 
 SLTH(Unset,	HTTP_HDR_UNSET,		0, 0, "unset header",
@@ -83,12 +89,14 @@ SLTH(Unset,	HTTP_HDR_UNSET,		0, 0, "unset header",
 	"\t|   +- Header value\n"
 	"\t+----- Header name\n"
 	"\n"
+	HEADER_NOTICE
 )
 
 SLTH(Lost,	HTTP_HDR_LOST,		0, 0, "lost header",
 	""
 )
 
+#undef HEADER_NOTICE
 #undef SLTH
 
 /*lint -restore */

From dridi.boukelmoune at gmail.com  Fri Oct 25 07:33:10 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 25 Oct 2019 07:33:10 +0000 (UTC)
Subject: [master] e66abb742 Document quoted VSL fields
Message-ID: <20191025073310.49F96117CE0@lists.varnish-cache.org>


commit e66abb7428160bed712d13757cb4e9d59e87936a
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 25 09:31:16 2019 +0200

    Document quoted VSL fields
    
    Refs #2872

diff --git a/doc/changes.rst b/doc/changes.rst
index e55a4857c..44873d77c 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -38,6 +38,10 @@ NEXT (2020-03-15)
 
 * backend ``none`` was added for "no backend"
 
+* Log records can safely have empty fields or fields containing blanks if
+  they are delimited by "double quotes". This was applied to ``SessError``
+  and ``Backend_health``.
+
 ================================
 Varnish Cache 6.3.0 (2019-09-15)
 ================================

From daghf at varnish-software.com  Mon Oct 28 11:16:08 2019
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 28 Oct 2019 11:16:08 +0000 (UTC)
Subject: [master] 799f68e91 h2_rx_data: Remember to drop the lock before
 returning
Message-ID: <20191028111608.F30B31048DA@lists.varnish-cache.org>


commit 799f68e918fd3fb8a373338c7886042317e1910c
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Oct 28 12:13:43 2019 +0100

    h2_rx_data: Remember to drop the lock before returning
    
    Fixes: #3086

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 902c1e08c..0f2a21230 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -750,8 +750,10 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	Lck_Lock(&h2->sess->mtx);
 	while (h2->mailcall != NULL && h2->error == 0 && r2->error == 0)
 		AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0));
-	if (h2->error || r2->error)
+	if (h2->error || r2->error) {
+		Lck_Unlock(&h2->sess->mtx);
 		return (h2->error ? h2->error : r2->error);
+	}
 	AZ(h2->mailcall);
 	h2->mailcall = r2;
 	h2->req0->r_window -= h2->rxf_len;

From daghf at varnish-software.com  Mon Oct 28 11:19:06 2019
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 28 Oct 2019 11:19:06 +0000 (UTC)
Subject: [6.0] a1786b0db h2_rx_data: Remember to drop the lock before returning
Message-ID: <20191028111906.C96E6104AFE@lists.varnish-cache.org>


commit a1786b0db403e3e3772ca7c28500a962070449b4
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Oct 28 12:13:43 2019 +0100

    h2_rx_data: Remember to drop the lock before returning
    
    Fixes: #3086

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 6a98a1c4b..cb35bb487 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -750,8 +750,10 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	Lck_Lock(&h2->sess->mtx);
 	while (h2->mailcall != NULL && h2->error == 0 && r2->error == 0)
 		AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0));
-	if (h2->error || r2->error)
+	if (h2->error || r2->error) {
+		Lck_Unlock(&h2->sess->mtx);
 		return (h2->error ? h2->error : r2->error);
+	}
 	AZ(h2->mailcall);
 	h2->mailcall = r2;
 	h2->req0->r_window -= h2->rxf_len;

From yiyiba0801 at hotmail.com  Tue Oct 29 03:43:28 2019
From: yiyiba0801 at hotmail.com (li yiyi)
Date: Tue, 29 Oct 2019 03:43:28 +0000
Subject: varnish don't set malloc size,varnish cache data only 200M size
Message-ID: <HK0PR02MB3681ED56B05FA8E4680979A8C5610@HK0PR02MB3681.apcprd02.prod.outlook.com>

hi, i use varnish6.2.0 to cache image, my machine memory is 16g, varnish malloc do not set size and use the default?
i see the documentation from varnish-cache.org, the default size is unlimited. my test images size about 6g,when i send http request to
varnish,only a little images can cache from varnish, about 200m size. most of them is miss.
my some varnish settings as follows:
/usr/local/varnish/sbin/varnishd -a 0.0.0.0:80,HTTP -f /usr/local/varnish/etc/default.vcl
when i set malloc size(8g),varnish is working ok, malloc must be seted specify size?how i can do if use unlimited ?

thank you?regards
________________________________

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20191029/ae21cbc5/attachment.html>

From nils.goroll at uplex.de  Tue Oct 29 09:24:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 29 Oct 2019 09:24:06 +0000 (UTC)
Subject: [master] 175655234 improve varnishd -s documentation
Message-ID: <20191029092406.8EDA762515@lists.varnish-cache.org>


commit 1756552343a6d62192de8f7306e71cd7468b3f11
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 29 10:21:56 2019 +0100

    improve varnishd -s documentation
    
    * clarify defaults
    * document Transient
    
    Fixes #3108

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 53488b175..44e2caf8f 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -304,7 +304,36 @@ The following hash algorithms are available:
 Storage Backend
 ---------------
 
-The following storage types are available:
+The argument format to define storage backends is:
+
+-s <[name]=kind[,options]>
+
+  If *name* is omitted, Varnish will name storages ``s``\ *N*,
+  starting with ``s0`` and incrementing *N* for every new storage.
+
+  For *kind* and *options* see details below.
+
+Storages can be used in vcl as ``storage.``\ *name*, so, for
+example if ``myStorage`` was defined by ``-s myStorage=malloc,5G``, it
+could be used in VCL like so::
+
+  set beresp.storage = storage.myStorage;
+
+A special *name* is ``Transient`` which is the default storage for
+uncacheable objects as resulting from a pass, hit-for-miss or
+hit-for-pass.
+
+If no ``-s`` options are given, the default is::
+
+	-s malloc=100m
+
+If no ``Transient`` storage is defined, the default is an unbound
+``malloc`` storage as if defined as::
+
+	-s Transient,malloc
+
+
+The following storage types and options are available:
 
 -s <default[,size]>
 
@@ -354,18 +383,6 @@ The following storage types are available:
   storage backend has multiple issues with it and will likely be
   removed from a future version of Varnish.
 
-
-You can also prefix the type with ``NAME=`` to explicitly name a storage::
-
-  -s myStorage=malloc,5G
-
-This allows to address it more easily in VCL::
-
-  set beresp.storage = storage.myStorage;
-
-If the name is omitted, Varnish will name storages ``sN``, starting with ``s0``
-and incrementing N for every new storage.
-
 .. _ref-varnishd-opt_j:
 
 Jail

From dridi.boukelmoune at gmail.com  Tue Oct 29 09:59:05 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 29 Oct 2019 09:59:05 +0000 (UTC)
Subject: [master] c59b80661 Typo
Message-ID: <20191029095905.BF22464478@lists.varnish-cache.org>


commit c59b80661bf4f331e908ce4e9f1e85038c929829
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 29 10:58:08 2019 +0100

    Typo
    
    Refs #3108

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 44e2caf8f..98c5ff0db 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -330,7 +330,7 @@ If no ``-s`` options are given, the default is::
 If no ``Transient`` storage is defined, the default is an unbound
 ``malloc`` storage as if defined as::
 
-	-s Transient,malloc
+	-s Transient=malloc
 
 
 The following storage types and options are available:

From dridi.boukelmoune at gmail.com  Tue Oct 29 15:21:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 29 Oct 2019 15:21:06 +0000 (UTC)
Subject: [master] 89428d077 Remove brackets from dynamic param bounds reasons
Message-ID: <20191029152106.5A6FF103FFB@lists.varnish-cache.org>


commit 89428d0775d063022d3425d260541e39e11aeaf7
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 29 16:11:16 2019 +0100

    Remove brackets from dynamic param bounds reasons
    
    Instead, they are added where they are needed: only in error messages
    involving them.
    
    Refs #3099

diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index fcb9a3bd5..30e58e2fc 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -194,14 +194,14 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 		if (min != NULL && u < minv) {
 			VSB_printf(vsb, "Must be at least %s", min);
 			if (min_reason != NULL)
-				VSB_printf(vsb, " %s", min_reason);
+				VSB_printf(vsb, " (%s)", min_reason);
 			VSB_putc(vsb, '\n');
 			return (-1);
 		}
 		if (max != NULL && u > maxv) {
 			VSB_printf(vsb, "Must be no more than %s", max);
 			if (max_reason != NULL)
-				VSB_printf(vsb, " %s", max_reason);
+				VSB_printf(vsb, " (%s)", max_reason);
 			VSB_putc(vsb, '\n');
 			return (-1);
 		}
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 7bbc1f959..8367453b6 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -114,7 +114,7 @@ struct parspec WRK_parspec[] = {
 		"in the way of getting work done.",
 		DELAYED_EFFECT,
 		"5000", "threads",
-		"(thread_pool_min)" },
+		"thread_pool_min" },
 	{ "thread_pool_min", tweak_thread_pool_min, &mgt_param.wthread_min,
 		NULL, NULL,
 		"The minimum number of worker threads in each pool. The "
@@ -126,7 +126,7 @@ struct parspec WRK_parspec[] = {
 		"Minimum is 10 threads.",
 		DELAYED_EFFECT,
 		"100", "threads",
-		NULL, "(thread_pool_max)" },
+		NULL, "thread_pool_max" },
 	{ "thread_pool_reserve", tweak_uint,
 		&mgt_param.wthread_reserve,
 		NULL, NULL,
@@ -147,7 +147,7 @@ struct parspec WRK_parspec[] = {
 		"Minimum is 1 otherwise, maximum is 95% of thread_pool_min.",
 		DELAYED_EFFECT,
 		"0", "threads",
-		NULL, "(95% of thread_pool_min)" },
+		NULL, "95% of thread_pool_min" },
 	{ "thread_pool_timeout",
 		tweak_timeout, &mgt_param.wthread_timeout,
 		"10", NULL,

From dridi.boukelmoune at gmail.com  Tue Oct 29 15:21:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 29 Oct 2019 15:21:06 +0000 (UTC)
Subject: [master] 345548166 Use the dynamic param bounds in the manual
Message-ID: <20191029152106.6F212103FFE@lists.varnish-cache.org>


commit 3455481663565dc43cbff92b47a15492515075ef
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 29 16:12:43 2019 +0100

    Use the dynamic param bounds in the manual
    
    Instead of having to manually document them, we can now let the RST dump
    figure everything out.
    
    Refs #3099

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index b15540f32..67bacd946 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -751,9 +751,13 @@ MCF_DumpRstParam(void)
 		if (pp->units != NULL && *pp->units != '\0')
 			printf("\t* Units: %s\n", pp->units);
 		printf("\t* Default: %s\n", pp->def);
-		if (pp->min != NULL)
+		if (pp->dyn_min_reason != NULL)
+			printf("\t* Minimum: %s\n", pp->dyn_min_reason);
+		else if (pp->min != NULL)
 			printf("\t* Minimum: %s\n", pp->min);
-		if (pp->max != NULL)
+		if (pp->dyn_max_reason != NULL)
+			printf("\t* Maximum: %s\n", pp->dyn_max_reason);
+		else if (pp->max != NULL)
 			printf("\t* Maximum: %s\n", pp->max);
 		/*
 		 * XXX: we should mark the params with one/two flags
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 8367453b6..b32224d72 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -106,8 +106,7 @@ struct parspec WRK_parspec[] = {
 		"2", "pools" },
 	{ "thread_pool_max", tweak_thread_pool_max, &mgt_param.wthread_max,
 		NULL, NULL,
-		"The maximum number of worker threads in each pool. The "
-		"minimum value depends on thread_pool_min.\n"
+		"The maximum number of worker threads in each pool.\n"
 		"\n"
 		"Do not set this higher than you have to, since excess "
 		"worker threads soak up RAM and CPU and generally just get "
@@ -117,8 +116,7 @@ struct parspec WRK_parspec[] = {
 		"thread_pool_min" },
 	{ "thread_pool_min", tweak_thread_pool_min, &mgt_param.wthread_min,
 		NULL, NULL,
-		"The minimum number of worker threads in each pool. The "
-		"maximum value depends on thread_pool_max.\n"
+		"The minimum number of worker threads in each pool.\n"
 		"\n"
 		"Increasing this may help ramp up faster from low load "
 		"situations or when threads have expired.\n"
@@ -144,7 +142,7 @@ struct parspec WRK_parspec[] = {
 		"unused.\n"
 		"\n"
 		"Default is 0 to auto-tune (currently 5% of thread_pool_min).\n"
-		"Minimum is 1 otherwise, maximum is 95% of thread_pool_min.",
+		"Minimum is 1 otherwise.",
 		DELAYED_EFFECT,
 		"0", "threads",
 		NULL, "95% of thread_pool_min" },

From dridi.boukelmoune at gmail.com  Tue Oct 29 17:19:07 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 29 Oct 2019 17:19:07 +0000 (UTC)
Subject: [master] 049ce0e73 Fix h2 error handling in varnishtest
Message-ID: <20191029171907.A51141069E5@lists.varnish-cache.org>


commit 049ce0e73250ee12e2d4e3e3370f4dacce4c07cd
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 29 17:32:17 2019 +0100

    Fix h2 error handling in varnishtest
    
    If varnish closes the connection while a client is waiting for an
    rxsomething command you may run into this kind of scenario:
    
        ***  c1   HTTP2 rx failed (fd:20 read: Connection reset by peer)
        ***  c1   rx: stream: 0, type: DATA (0), flags: 0x00, size: 0
        **** c1   s0 - no data
        ---- c1   Wrong frame type DATA (0) wanted SETTINGS
    
    This was already done correctly for the frame body.

diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index 0e7bdea7f..049d7dd1b 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -710,7 +710,7 @@ receive_frame(void *priv)
 		}
 		AZ(pthread_mutex_unlock(&hp->mtx));
 
-		if (!get_bytes(hp, hdr, 9)) {
+		if (get_bytes(hp, hdr, sizeof hdr) <= 0) {
 			AZ(pthread_mutex_lock(&hp->mtx));
 			VTAILQ_FOREACH(s, &hp->streams, list)
 				AZ(pthread_cond_signal(&s->cond));

From nils.goroll at uplex.de  Wed Oct 30 06:12:08 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 30 Oct 2019 06:12:08 +0000 (UTC)
Subject: [master] b0858dde8 gc after connection pool rework
Message-ID: <20191030061209.07691115FD0@lists.varnish-cache.org>


commit b0858dde8b57334ac7e21d311c8d675c8d985e1f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 30 07:09:24 2019 +0100

    gc after connection pool rework
    
    Ref: ec70dbc7502f8fbe6c8eb5af6108df4168500f9a
    
    struct vtp_cs was only used for comparisons during pool lookup

diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index 5aa207f84..bb854540c 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -565,14 +565,6 @@ VCP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo)
 /*--------------------------------------------------------------------
  */
 
-struct vtp_cs {
-	unsigned			magic;
-#define VTP_CS_MAGIC			0xc1e40447
-	const struct suckaddr		*ip4;
-	const struct suckaddr		*ip6;
-	const char			*uds;
-};
-
 static inline int
 tmo2msec(vtim_dur tmo)
 {
@@ -687,15 +679,10 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds,
 {
 	struct tcp_pool *tp;
 	struct conn_pool *cp;
-	struct vtp_cs vcs;
 	const struct cp_methods *methods;
 
 	assert((uds != NULL && ip4 == NULL && ip6 == NULL) ||
 	    (uds == NULL && (ip4 != NULL || ip6 != NULL)));
-	INIT_OBJ(&vcs, VTP_CS_MAGIC);
-	vcs.ip4 = ip4;
-	vcs.ip6 = ip6;
-	vcs.uds = uds;
 
 	cp = VCP_Ref(id);
 	if (cp != NULL)

From nils.goroll at uplex.de  Wed Oct 30 06:27:06 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 30 Oct 2019 06:27:06 +0000 (UTC)
Subject: [master] 685a57869 Guard against hash collisions in the vtp code
Message-ID: <20191030062706.6C2CE116582@lists.varnish-cache.org>


commit 685a578691b6986fde95d1d01abbd1575a97f022
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 30 07:19:27 2019 +0100

    Guard against hash collisions in the vtp code
    
    We use the first 64bit of a sha256 as our pool id, assuming that those
    are safe enough against collisions. Ensure we do not fail on that
    assumption.
    
    This also makes it a caller error to deliberately use the same pool id
    for different endpoints. As this was not possible before the id change,
    I do not consider it a regression.
    
    Ref: ec70dbc7502f8fbe6c8eb5af6108df4168500f9a

diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index bb854540c..14c9577cd 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -685,12 +685,24 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds,
 	    (uds == NULL && (ip4 != NULL || ip6 != NULL)));
 
 	cp = VCP_Ref(id);
-	if (cp != NULL)
+	if (cp != NULL) {
+		tp = cp->priv;
+		CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+
+		if (uds != NULL) {
+			AN(tp->uds);
+			AZ(strcmp(tp->uds, uds));
+		}
+		if (ip4 != NULL)
+			AZ(VSA_Compare(tp->ip4, ip4));
+		if (ip6 != NULL)
+			AZ(VSA_Compare(tp->ip6, ip6));
 		return (cp->priv);
+	}
 
 	/*
-	 * this is racy - we could end up with additional pools on the same id /
-	 * destination address with just a single connection
+	 * this is racy - we could end up with additional pools on the same id
+	 * with just a single connection
 	 */
 	ALLOC_OBJ(tp, TCP_POOL_MAGIC);
 	AN(tp);

From nils.goroll at uplex.de  Wed Oct 30 14:21:05 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 30 Oct 2019 14:21:05 +0000 (UTC)
Subject: [master] d235b3c90 fix an infinite loop in the gunzip VDP with junk
 after GZ_END
Message-ID: <20191030142105.E099B64253@lists.varnish-cache.org>


commit d235b3c90a631ef39fdf0a8103e44ebfb0ddbacb
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 30 15:12:08 2019 +0100

    fix an infinite loop in the gunzip VDP with junk after GZ_END
    
    The gunzip vdp failed to handle junk after end of gzip data. This
    basically mirrors #942 on the client side, also the fix is basically the
    same as 41f7a356e2be38f03428589710d163bd4110d9fd
    
    The impact of this bug is likely to be low, because the built-in
    beresp.filters logic will push the testgunzip VFP for gzip content
    received from backends, so, unless VCL is forced to pass backend
    responses unchecked or vmods generate body data, it can be considered
    unlikely that this issue will be hit.
    
    Fixes #3109

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 46146beb3..758c990cf 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -364,6 +364,11 @@ vdp_gunzip_bytes(struct req *req, enum vdp_action act, void **priv,
 	VGZ_Ibuf(vg, ptr, len);
 	do {
 		vr = VGZ_Gunzip(vg, &dp, &dl);
+		if (vr == VGZ_END && !VGZ_IbufEmpty(vg)) {
+			VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
+			     vr, "junk after VGZ_END");
+			return (-1);
+		}
 		vg->m_len += dl;
 		if (vr < VGZ_OK)
 			return (-1);
diff --git a/bin/varnishtest/tests/r03109.vtc b/bin/varnishtest/tests/r03109.vtc
new file mode 100644
index 000000000..f3763a742
--- /dev/null
+++ b/bin/varnishtest/tests/r03109.vtc
@@ -0,0 +1,34 @@
+varnishtest "Test garbage after gzip end reaching gunzip vdp"
+
+server s1 {
+	rxreq
+	txresp -hdr "content-encoding: gzip" -nolen
+	# (date | gzip -9f ; echo bad)  | od -t x1|
+	# sed -e 's:^[0-9a-f]* :sendhex ":' -e 's:$:":' -e '/^[0-9a-f]*"/ d'
+	sendhex "1f 8b 08 00 f5 8a b9 5d 02 03 0b 4f 4d 51 30 36"
+	sendhex "50 f0 4f 2e 51 30 34 b1 32 30 b7 32 30 54 70 76"
+	sendhex "0d 51 30 32 30 b4 e4 02 00 fa 76 79 ba 1d 00 00"
+	sendhex "00 62 61 64 0a"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		# no gunzip check
+		set beresp.filters = "";
+	}
+	sub vcl_deliver {
+		set resp.filters = "gunzip";
+	}
+} -start
+
+logexpect l1 -v v1 -q "vxid == 1001" {
+	expect * 1001	Gzip {^G.un.zip error: 1 .junk after VGZ_END.$}
+} -start
+
+client c1 {
+	txreq
+	rxresphdrs
+	expect_close
+} -run
+
+logexpect l1 -wait

From dridi.boukelmoune at gmail.com  Wed Oct 30 14:54:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 30 Oct 2019 14:54:06 +0000 (UTC)
Subject: [master] f8817b8b6 Fix failing tests in witness mode
Message-ID: <20191030145406.43D0A6509F@lists.varnish-cache.org>


commit f8817b8b6cfc504912436ffc74b1b7d1e851d35a
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Sep 30 07:53:29 2019 +0200

    Fix failing tests in witness mode
    
    The simple fact that Witness records might show up in the log might
    break logexpect commands. There's no reason why we'd want to expect
    Witness records since their purpose is to be checked after the test
    finishes.

diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index 957436d5d..7369a8642 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -281,7 +281,7 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			data = VSL_CDATA(t->c->rec.ptr);
 			len = VSL_LEN(t->c->rec.ptr) - 1;
 
-			if (tag == SLT__Batch)
+			if (tag == SLT__Batch || tag == SLT_Witness)
 				continue;
 
 			ok = 1;

From dridi.boukelmoune at gmail.com  Wed Oct 30 14:54:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 30 Oct 2019 14:54:06 +0000 (UTC)
Subject: [master] 7e3e7f904 Make witness mode a first-class citizen
Message-ID: <20191030145406.7006B650A2@lists.varnish-cache.org>


commit 7e3e7f9040ce9527332d928967fc34f8dfcc2843
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Sep 30 08:02:20 2019 +0200

    Make witness mode a first-class citizen
    
    This change introduces a top-level make witness target that builds a dot
    graph and if graphviz is available, an SVG file as well. A shell script
    replaces the previous python script that no longer works. Instead of
    fixing witness.py, which is probably trivial, the shell script does an
    intermediate pass and programmatically looks for cycles using tsort(1).
    
    Checking lock dependencies becomes actionable in a CI context.
    
    The script also takes explicit test directories on purpose, to have the
    ability to aggregate test results from multiple executions. For example
    when the test suite is run on various operating systems or with varying
    privileges to cover feature-conditional tests.

diff --git a/.gitignore b/.gitignore
index 07e77ffa9..f2b991fc5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -144,6 +144,10 @@ cscope.*out
 /cov-int
 /myproject.tgz
 
+# Witness droppings
+witness.dot
+witness.svg
+
 # Flexelint droppings
 _.fl
 _.fl.old
diff --git a/Makefile.am b/Makefile.am
index add21a8ca..c2cbd1c13 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -8,7 +8,13 @@ pkgconfig_DATA = varnishapi.pc
 m4dir = $(datadir)/aclocal
 m4_DATA = varnish.m4 varnish-legacy.m4
 
-CLEANFILES = cscope.in.out cscope.out cscope.po.out
+CLEANFILES = \
+	cscope.in.out \
+	cscope.out \
+	cscope.po.out \
+	witness.dot \
+	witness.svg
+
 EXTRA_DIST = \
 	README.rst \
 	README.Packaging \
@@ -52,4 +58,24 @@ cscope:
 gcov_digest:
 	${PYTHON} tools/gcov_digest.py -o _gcov
 
-.PHONY: cscope
+witness.dot: all
+	$(MAKE) -C bin/varnishtest check AM_VTC_LOG_FLAGS=-pdebug=+witness
+	$(AM_V_GEN) $(srcdir)/tools/witness.sh witness.dot bin/varnishtest
+
+.dot.svg:
+if ! HAVE_DOT
+	@echo ==================================================
+	@echo You need graphviz installed to generate svg output
+	@echo ==================================================
+	@false
+else
+	$(AM_V_GEN) $(DOT) -Tsvg $< >$@
+endif
+
+if HAVE_DOT
+witness: witness.svg
+else
+witness: witness.dot
+endif
+
+.PHONY: cscope witness.dot
diff --git a/bin/varnishtest/witness.py b/bin/varnishtest/witness.py
deleted file mode 100644
index f5b516ee0..000000000
--- a/bin/varnishtest/witness.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-#
-# This script is in the public domain
-#
-# Run instructions:
-#       varnishtest -W -iv -j <pick_a_number> > _.w
-#       python witness.py
-#       dot -Tpng /tmp/_.dot > /tmp/_.png
-
-d = dict()
-a = dict()
-
-fi = open("_.w")
-fo = open("/tmp/_.dot", "w")
-
-fo.write('''digraph {
-    #rotate="90"
-    #page="8.2,11.7"
-    size="8.2,11.7"
-    rankdir="LR"
-    node [fontname="Inconsolata", fontsize="10"]
-    edge [fontname="Inconsolata", fontsize="10"]
-''')
-
-for i in fi:
-    l = "ROOT"
-    j = i.split()
-    if len(j) < 8:
-        continue
-    if j[1][0] != 'v':
-        continue
-    if j[3] != "vsl|":
-        continue
-    if j[5] != "Witness":
-        continue
-    t = j[7:]
-    tt = str(t)
-    if tt in d:
-        continue
-    d[tt] = True
-    for e in t:
-        f = e.split(",")
-        x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2])
-        if not x in a:
-            a[x] = True
-            fo.write(x)
-        l = f[0]
-
-fo.write("}\n")
-
diff --git a/tools/witness.sh b/tools/witness.sh
new file mode 100755
index 000000000..9b8d0bfaa
--- /dev/null
+++ b/tools/witness.sh
@@ -0,0 +1,82 @@
+#!/bin/sh
+
+set -e
+set -u
+
+readonly work_dir=$(mktemp -d)
+trap 'rm -rf "$work_dir"' EXIT
+
+witness_full_paths() {
+	find "$@" -name '*.log' -print0 |
+	xargs -0 awk '$3 == "vsl|" && $5 == "Witness" {
+		printf "%s", "ROOT"
+		for (i = 7; i <= NF; i++) {
+			printf " %s", $i
+		}
+		printf "\n"
+	}' |
+	sort |
+	uniq
+}
+
+witness_edges() {
+	awk '{
+		for (i = 1; i < NF; i++) {
+			printf "%s %s\n", $i, $(i + 1)
+		}
+	}' |
+	sort |
+	uniq
+}
+
+witness_cycles() {
+	! awk -F '[ ,]' '{print $1 " " $(NF - 2)}' |
+	tsort >/dev/null 2>&1
+}
+
+witness_graph() {
+	cat <<-EOF
+	digraph {
+	    size="8.2,11.7"
+	    rankdir="LR"
+	    node [fontname="Inconsolata", fontsize="10"]
+	    edge [fontname="Inconsolata", fontsize="10"]
+	EOF
+
+	awk -F '[ ,]' '{
+		printf "    \"%s\" -> \"%s\" [label=\"%s(%s)\"]\n",
+		    $1, $(NF - 2), $(NF - 1), $NF
+	}' |
+	sort |
+	uniq
+
+	echo '}'
+}
+
+if [ $# -lt 2 ]
+then
+	cat >&2 <<-EOF
+	usage: $0 dot_file test_dirs...
+	EOF
+	exit 1
+fi
+
+dest_file=$1
+shift
+
+witness_full_paths "$@" |
+witness_edges >"$work_dir/witness-edges.txt"
+
+tsort_err=
+
+if witness_cycles <"$work_dir/witness-edges.txt"
+then
+	echo "Error: lock cycle witnessed" >&2
+	tsort_err=1
+fi
+
+witness_graph <"$work_dir/witness-edges.txt" >"$work_dir/witness.dot"
+
+mv "$work_dir/witness.dot" "$dest_file"
+
+exit $tsort_err

From dridi.boukelmoune at gmail.com  Wed Oct 30 14:54:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 30 Oct 2019 14:54:06 +0000 (UTC)
Subject: [master] 79bf4f812 Retire the varnishtest -W option
Message-ID: <20191030145406.9B2EF650A6@lists.varnish-cache.org>


commit 79bf4f812d29d78a4e9c592e625df974821963a4
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Sep 30 07:52:49 2019 +0200

    Retire the varnishtest -W option
    
    The same can be achieved with `varnishtest -p debug=+witness`, already
    used by `make witness`.

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index e1f167daa..ab218abce 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -81,7 +81,6 @@ extern unsigned vtc_maxdur;
 extern char *vmod_path;
 extern struct vsb *params_vsb;
 extern int leave_temp;
-extern int vtc_witness;
 extern int ign_unknown_macro;
 
 void init_server(void);
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index ac21b1000..a2c346067 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -106,7 +106,6 @@ static char *cwd = NULL;
 char *vmod_path = NULL;
 struct vsb *params_vsb = NULL;
 int leave_temp;
-int vtc_witness = 0;
 static struct vsb *cbvsb;
 static int bad_backend_fd;
 
@@ -189,7 +188,6 @@ usage(void)
 	fprintf(stderr, FMT, "-q", "Quiet mode: report only failures");
 	fprintf(stderr, FMT, "-t duration", "Time tests out after this long");
 	fprintf(stderr, FMT, "-v", "Verbose mode: always report test log");
-	fprintf(stderr, FMT, "-W", "Enable the witness facility for locking");
 	exit(1);
 }
 
@@ -748,9 +746,6 @@ main(int argc, char * const *argv)
 			if (vtc_verbosity < 2)
 				vtc_verbosity++;
 			break;
-		case 'W':
-			vtc_witness++;
-			break;
 		default:
 			usage();
 		}
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index f0d6d7bdc..24ffab467 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -411,8 +411,6 @@ varnish_launch(struct varnish *v)
 	VSB_printf(vsb, " exec varnishd %s -d -n %s",
 	    v->jail, v->workdir);
 	VSB_cat(vsb, VSB_data(params_vsb));
-	if (vtc_witness)
-		VSB_cat(vsb, " -p debug=+witness");
 	if (leave_temp) {
 		VSB_cat(vsb, " -p debug=+vcl_keep");
 		VSB_cat(vsb, " -p debug=+vmod_so_keep");
diff --git a/doc/changes.rst b/doc/changes.rst
index 44873d77c..fdc03d521 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -42,6 +42,9 @@ NEXT (2020-03-15)
   they are delimited by "double quotes". This was applied to ``SessError``
   and ``Backend_health``.
 
+* The option ``varnishtest -W`` is gone, the same can be achieved with
+  ``varnishtest -p debug=+witness``.
+
 ================================
 Varnish Cache 6.3.0 (2019-09-15)
 ================================
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index b6133d70c..5cd180949 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -15,7 +15,7 @@ Test program for Varnish
 SYNOPSIS
 ========
 
-varnishtest [-hikLlqvW] [-b size] [-D name=val] [-j jobs] [-n iter] [-t duration] file [file ...]
+varnishtest [-hikLlqv] [-b size] [-D name=val] [-j jobs] [-n iter] [-t duration] file [file ...]
 
 DESCRIPTION
 ===========
@@ -56,8 +56,6 @@ The following options are available:
 
 -v               Verbose mode: always report test log
 
--W               Enable the witness facility for locking
-
 file             File to use as a script
 
 

From nils.goroll at uplex.de  Wed Oct 30 15:07:08 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 30 Oct 2019 15:07:08 +0000 (UTC)
Subject: [master] ab7a5174d Re-Setup the response after rollback
Message-ID: <20191030150708.2289065BD9@lists.varnish-cache.org>


commit ab7a5174d5be93c60a1bd1dd84efc9459af5f11a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 7 18:09:19 2019 +0200

    Re-Setup the response after rollback
    
    Fixes #3083
    
    to get there, also centralize req->resp setup for deliver and synth:
    
    No semantic changes other than some reordering, which also fixes an odd
    log line ordering as shown by the change to c00018.vtc

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 6fdce85a2..977f57e39 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -104,6 +104,96 @@ cnt_transport(struct worker *wrk, struct req *req)
  * Deliver an object to client
  */
 
+static int
+resp_setup_deliver(struct req *req)
+{
+	struct http *h;
+	struct objcore *oc;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	oc = req->objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	h = req->resp;
+
+	HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
+
+	if (HTTP_Decode(h, ObjGetAttr(req->wrk, oc, OA_HEADERS, NULL)))
+		return (-1);
+
+	http_ForceField(h, HTTP_HDR_PROTO, "HTTP/1.1");
+
+	if (req->is_hit)
+		http_PrintfHeader(h, "X-Varnish: %u %u", VXID(req->vsl->wid),
+		    ObjGetXID(req->wrk, oc));
+	else
+		http_PrintfHeader(h, "X-Varnish: %u", VXID(req->vsl->wid));
+
+	/*
+	 * We base Age calculation upon the last timestamp taken during client
+	 * request processing. This gives some inaccuracy, but since Age is only
+	 * full second resolution that shouldn't matter. (Last request timestamp
+	 * could be a Start timestamp taken before the object entered into cache
+	 * leading to negative age. Truncate to zero in that case).
+	 */
+	http_PrintfHeader(h, "Age: %.0f",
+	    floor(fmax(0., req->t_prev - oc->t_origin)));
+
+	http_SetHeader(h, "Via: 1.1 varnish (Varnish/6.3)");
+
+	if (cache_param->http_gzip_support &&
+	    ObjCheckFlag(req->wrk, oc, OF_GZIPED) &&
+	    !RFC2616_Req_Gzip(req->http))
+		RFC2616_Weaken_Etag(h);
+
+	return (0);
+}
+
+static int
+resp_setup_synth(struct req *req)
+{
+	struct http *h;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	h = req->resp;
+
+	HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
+
+	AZ(req->objcore);
+	http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
+
+	http_TimeHeader(h, "Date: ", W_TIM_real(req->wrk));
+	http_SetHeader(h, "Server: Varnish");
+	http_PrintfHeader(h, "X-Varnish: %u",
+	    VXID(req->vsl->wid));
+
+	/*
+	 * For late 100-continue, we suggest to VCL to close the connection to
+	 * neither send a 100-continue nor drain-read the request. But VCL has
+	 * the option to veto by removing Connection: close
+	 */
+	if (req->want100cont)
+		http_SetHeader(h, "Connection: close");
+
+	return (0);
+}
+
+int
+Resp_Setup(struct req *req, unsigned method)
+{
+	switch (method) {
+	case VCL_MET_DELIVER:
+		return (resp_setup_deliver(req));
+	case VCL_MET_SYNTH:
+		return (resp_setup_synth(req));
+	default:
+		WRONG("vcl method");
+	}
+
+	return (0);
+}
+
 static enum req_fsm_nxt
 cnt_deliver(struct worker *wrk, struct req *req)
 {
@@ -119,41 +209,12 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	ObjTouch(req->wrk, req->objcore, req->t_prev);
 
-	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-	if (HTTP_Decode(req->resp,
-	    ObjGetAttr(req->wrk, req->objcore, OA_HEADERS, NULL))) {
+	if (resp_setup_deliver(req)) {
 		(void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY);
 		req->err_code = 500;
 		req->req_step = R_STP_SYNTH;
 		return (REQ_FSM_MORE);
 	}
-	http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1");
-
-	if (req->is_hit)
-		http_PrintfHeader(req->resp,
-		    "X-Varnish: %u %u", VXID(req->vsl->wid),
-		    ObjGetXID(wrk, req->objcore));
-	else
-		http_PrintfHeader(req->resp,
-		    "X-Varnish: %u", VXID(req->vsl->wid));
-
-	/*
-	 * We base Age calculation upon the last timestamp taken during
-	 * client request processing. This gives some inaccuracy, but
-	 * since Age is only full second resolution that shouldn't
-	 * matter. (Last request timestamp could be a Start timestamp
-	 * taken before the object entered into cache leading to negative
-	 * age. Truncate to zero in that case).
-	 */
-	http_PrintfHeader(req->resp, "Age: %.0f",
-	    floor(fmax(0., req->t_prev - req->objcore->t_origin)));
-
-	http_SetHeader(req->resp, "Via: 1.1 varnish (Varnish/6.3)");
-
-	if (cache_param->http_gzip_support &&
-	    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
-	    !RFC2616_Req_Gzip(req->http))
-		RFC2616_Weaken_Etag(req->resp);
 
 	VCL_deliver_method(req->vcl, wrk, req, NULL, NULL);
 	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
@@ -221,8 +282,6 @@ cnt_vclfail(const struct worker *wrk, struct req *req)
 static enum req_fsm_nxt
 cnt_synth(struct worker *wrk, struct req *req)
 {
-	struct http *h;
-	double now;
 	struct vsb *synth_body;
 	ssize_t sz, szl;
 	uint8_t *ptr;
@@ -235,27 +294,12 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	wrk->stats->s_synth++;
 
-	now = W_TIM_real(wrk);
-	VSLb_ts_req(req, "Process", now);
+	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
 
 	if (req->err_code < 100)
 		req->err_code = 501;
 
-	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-	h = req->resp;
-	http_TimeHeader(h, "Date: ", now);
-	http_SetHeader(h, "Server: Varnish");
-	http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
-	http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
-
-	/*
-	 * For late 100-continue, we suggest to VCL to close the connection to
-	 * neither send a 100-continue nor drain-read the request. But VCL has
-	 * the option to veto by removing Connection: close
-	 */
-	if (req->want100cont) {
-		http_SetHeader(h, "Connection: close");
-	}
+	(void) resp_setup_synth(req);
 
 	synth_body = VSB_new_auto();
 	AN(synth_body);
@@ -281,14 +325,14 @@ cnt_synth(struct worker *wrk, struct req *req)
 		 * XXX: Should we reset req->doclose = SC_VCL_FAILURE
 		 * XXX: If so, to what ?
 		 */
-		HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
+		HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 		VSB_destroy(&synth_body);
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
 	}
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	http_Unset(h, H_Content_Length);
+	http_Unset(req->resp, H_Content_Length);
 	http_PrintfHeader(req->resp, "Content-Length: %zd",
 	    VSB_len(synth_body));
 
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 0fb461c61..6a1ace8aa 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -360,6 +360,8 @@ void VRB_Free(struct req *);
 
 /* cache_req_fsm.c [CNT] */
 
+int Resp_Setup(struct req *, unsigned);
+
 enum req_fsm_nxt {
 	REQ_FSM_MORE,
 	REQ_FSM_DONE,
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 4ec470ea0..0fa600f2a 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -747,6 +747,8 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp)
 	if (hp == ctx->http_req) {
 		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 		Req_Rollback(ctx->req);
+		if ((ctx->method & (VCL_MET_DELIVER | VCL_MET_SYNTH)) != 0)
+			Resp_Setup(ctx->req, ctx->method);
 	} else if (hp == ctx->http_bereq) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 		Bereq_Rollback(ctx->bo);
diff --git a/bin/varnishtest/tests/c00018.vtc b/bin/varnishtest/tests/c00018.vtc
index 7a27755ff..10359f758 100644
--- a/bin/varnishtest/tests/c00018.vtc
+++ b/bin/varnishtest/tests/c00018.vtc
@@ -123,11 +123,13 @@ logexpect l1 -v v1 -g raw {
 	expect 0 1011	VCL_call        {^HASH$}
 	expect 0 1011	VCL_return      {^lookup$}
 	expect 0 1011	Timestamp       {^Process:}
-	expect 0 1011	RespHeader      {^Date:}
-	expect 0 1011	RespHeader      {^Server: Varnish$}
-	expect 0 1011	RespHeader      {^X-Varnish: 1011$}
 	expect 0 1011	RespProtocol    {^HTTP/1.1$}
 	expect 0 1011	RespStatus      {^405$}
+	expect 0 1011	RespReason      {^Method Not Allowed$}
+	# XXX dup RespReason
+	expect 1 1011	RespHeader      {^Date:}
+	expect 0 1011	RespHeader      {^Server: Varnish$}
+	expect 0 1011	RespHeader      {^X-Varnish: 1011$}
 
 } -start
 

From nils.goroll at uplex.de  Wed Oct 30 15:07:08 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 30 Oct 2019 15:07:08 +0000 (UTC)
Subject: [master] ee51cc96b integrate @rezan 's regression tests into m17.vtc
Message-ID: <20191030150708.3E29F65BDE@lists.varnish-cache.org>


commit ee51cc96b07cc7a5350265d83aa945ec4b608b4e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 7 18:25:01 2019 +0200

    integrate @rezan 's regression tests into m17.vtc
    
    I had previously overlooked that there exists a more complete regression
    test by him in #3010
    
    Integrate that vtc, polished and modified to account for the backend 503
    for no restart after rollback, into m17, making r3009 obsolete again

diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc
index 8fa38f37a..c31322d50 100644
--- a/bin/varnishtest/tests/m00017.vtc
+++ b/bin/varnishtest/tests/m00017.vtc
@@ -1,5 +1,9 @@
 varnishtest "Test std.rollback"
 
+# bug regressions:
+# - #3009
+# - #3083
+
 server s1 {
 	rxreq
 	expect req.url == "/foo"
@@ -67,3 +71,253 @@ client c1 {
 	txreq -url "/baz" -hdr "foobar: baz"
 	rxresp
 } -run
+
+server s1 -wait
+
+varnish v1 -cliok "param.set workspace_client 12000"
+varnish v1 -cliok "param.set workspace_backend 12000"
+
+# Dont panic
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		set req.http.test = "1";
+	}
+
+	sub vcl_backend_fetch {
+		unset bereq.http.test;
+	}
+
+	sub vcl_backend_response {
+		std.rollback(bereq);
+		set beresp.http.test = bereq.http.test;
+		vtc.workspace_alloc(backend, -10);
+	}
+}
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	txresp
+} -start
+
+client c1 {
+	txreq -url /1
+	rxresp
+	expect resp.status == 503
+	expect resp.http.test == <undef>
+} -run
+
+server s1 -wait
+
+# Dont run out of workspace
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_backend_fetch {
+		vtc.workspace_alloc(backend, 1000);
+	}
+
+	sub vcl_backend_response {
+		if (bereq.retries == 0) {
+			vtc.workspace_alloc(backend, -10);
+			std.rollback(bereq);
+			return (retry);
+		}
+	}
+}
+
+server s1 -repeat 2 {
+	rxreq
+	expect req.url == "/2"
+	txresp
+} -start
+
+client c2 {
+	txreq -url /2
+	rxresp
+	expect resp.status == 200
+} -run
+
+server s1 -wait
+
+# Keep workspace intact (and possibly overflow)
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_backend_fetch {
+		set bereq.http.fetch = "Fetch value " + bereq.retries;
+	}
+
+	sub vcl_backend_response {
+		if (bereq.retries == 0) {
+			std.rollback(bereq);
+			set bereq.http.response = "123";
+			set bereq.http.response2 = "Another response";
+			if (bereq.url == "/4") {
+				vtc.workspace_alloc(backend, -10);
+			} else if (bereq.url == "/5") {
+				vtc.workspace_alloc(backend, -10);
+				std.rollback(bereq);
+			}
+			return (retry);
+		}
+		set beresp.http.fetch = bereq.http.fetch;
+		set beresp.http.response = bereq.http.response;
+		set beresp.http.response2 = bereq.http.response2;
+	}
+}
+
+server s1 -repeat 5 {
+	rxreq
+	txresp
+} -start
+
+client c3 {
+	txreq -url /3
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == "123"
+	expect resp.http.response2 == "Another response"
+
+	txreq -url /4
+	rxresp
+	expect resp.status == 503
+
+	txreq -url /5
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == ""
+	expect resp.http.response2 == ""
+} -run
+
+server s1 -wait
+
+# CLIENT
+
+# Dont panic
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		unset req.http.test;
+	}
+
+	sub vcl_deliver {
+		std.rollback(req);
+		set resp.http.test = req.http.test;
+		vtc.workspace_alloc(client, -200);
+	}
+}
+
+# XXX server keeps params, so we need to reset previous -repeat 5
+server s1 -repeat 1 {
+	rxreq
+	expect req.url == "/6"
+	txresp
+} -start
+
+client c4 {
+	txreq -url /6 -hdr "test: 1"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.test == "1"
+} -run
+
+server s1 -wait
+
+# Dont run out of workspace
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		vtc.workspace_alloc(client, 1000);
+	}
+
+	sub vcl_deliver {
+		if (req.restarts == 0) {
+			vtc.workspace_alloc(client, -10);
+			std.rollback(req);
+			return (restart);
+		}
+	}
+}
+
+server s1 {
+	rxreq
+	expect req.url == "/7"
+	txresp
+} -start
+
+client c5 {
+	txreq -url /7
+	rxresp
+	expect resp.status == 200
+} -run
+
+server s1 -wait
+
+# Keep workspace intact (and possibly overflow)
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		set req.http.fetch = "Fetch value " + req.restarts;
+	}
+
+	sub vcl_deliver {
+		if (req.restarts == 0) {
+			std.rollback(req);
+			set req.http.response = "123";
+			set req.http.response2 = "Another response";
+			if (req.url == "/9") {
+				vtc.workspace_alloc(client, -200);
+			} else if (req.url == "/10") {
+				vtc.workspace_alloc(client, -10);
+				std.rollback(req);
+			}
+			return (restart);
+		}
+		set resp.http.fetch = req.http.fetch;
+		set resp.http.response = req.http.response;
+		set resp.http.response2 = req.http.response2;
+	}
+}
+
+server s1 -repeat 3 {
+	rxreq
+	txresp
+} -start
+
+client c6 {
+	txreq -url /8
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == "123"
+	expect resp.http.response2 == "Another response"
+
+	txreq -url /9
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == "123"
+	expect resp.http.response2 == "Another response"
+
+	txreq -url /10
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == ""
+} -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/r03009.vtc b/bin/varnishtest/tests/r03009.vtc
deleted file mode 100644
index 1bc44521f..000000000
--- a/bin/varnishtest/tests/r03009.vtc
+++ /dev/null
@@ -1,32 +0,0 @@
-varnishtest "Rollback without restart/retry is unsafe"
-
-server s1 {
-	rxreq
-	txresp
-} -start
-
-varnish v1 -vcl+backend {
-	import std;
-
-	sub vcl_recv {
-		set req.http.test = "1";
-	}
-
-	sub vcl_backend_fetch {
-		unset bereq.http.test;
-	}
-
-	sub vcl_backend_response {
-		std.rollback(bereq);
-		set beresp.http.test = bereq.http.test;
-		set beresp.http.workspace = "start overwriting active workspace";
-		set beresp.http.workspace = "0123456789012345678901234567890123456789";
-		# panic...
-	}
-} -start
-
-client c1 {
-	txreq
-	rxresp
-	expect resp.status == 503
-} -run

From nils.goroll at uplex.de  Wed Oct 30 15:07:08 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 30 Oct 2019 15:07:08 +0000 (UTC)
Subject: [master] 084cd13ed fix when the vcl_synth {} Process timestamp gets
 emitted
Message-ID: <20191030150708.59D6A65BE2@lists.varnish-cache.org>


commit 084cd13eda6063ca761dfda7f45694ca6ef1eb47
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 11 09:38:12 2019 +0200

    fix when the vcl_synth {} Process timestamp gets emitted
    
    Found by @Dridi

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 977f57e39..e13de2250 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -294,8 +294,6 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	wrk->stats->s_synth++;
 
-	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
-
 	if (req->err_code < 100)
 		req->err_code = 501;
 
@@ -308,6 +306,8 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	AZ(VSB_finish(synth_body));
 
+	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
+
 	if (wrk->handling == VCL_RET_FAIL) {
 		VSB_destroy(&synth_body);
 		req->doclose = SC_VCL_FAILURE;
diff --git a/bin/varnishtest/tests/c00018.vtc b/bin/varnishtest/tests/c00018.vtc
index 10359f758..cff0bebde 100644
--- a/bin/varnishtest/tests/c00018.vtc
+++ b/bin/varnishtest/tests/c00018.vtc
@@ -122,7 +122,6 @@ logexpect l1 -v v1 -g raw {
 	expect 0 1011	VCL_return      {^synth$}
 	expect 0 1011	VCL_call        {^HASH$}
 	expect 0 1011	VCL_return      {^lookup$}
-	expect 0 1011	Timestamp       {^Process:}
 	expect 0 1011	RespProtocol    {^HTTP/1.1$}
 	expect 0 1011	RespStatus      {^405$}
 	expect 0 1011	RespReason      {^Method Not Allowed$}
@@ -130,6 +129,7 @@ logexpect l1 -v v1 -g raw {
 	expect 1 1011	RespHeader      {^Date:}
 	expect 0 1011	RespHeader      {^Server: Varnish$}
 	expect 0 1011	RespHeader      {^X-Varnish: 1011$}
+	expect * 1011	Timestamp       {^Process:}
 
 } -start
 
diff --git a/doc/changes.rst b/doc/changes.rst
index fdc03d521..37c638da8 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -45,6 +45,10 @@ NEXT (2020-03-15)
 * The option ``varnishtest -W`` is gone, the same can be achieved with
   ``varnishtest -p debug=+witness``.
 
+* The ``Process`` timestamp for ``vcl_synth {}`` was wrongly issued
+  before the VCL callback, now it gets emitted after VCL returns for
+  consistency with ``vcl_deliver {}``
+
 ================================
 Varnish Cache 6.3.0 (2019-09-15)
 ================================

From nils.goroll at uplex.de  Wed Oct 30 15:07:08 2019
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 30 Oct 2019 15:07:08 +0000 (UTC)
Subject: [master] cbd7675a2 Avoid setting the http response twice via
 http_PutResponse()
Message-ID: <20191030150708.9BADE65BE7@lists.varnish-cache.org>


commit cbd7675a263195ae62207d438d54e1855bfd68b8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 21 11:31:54 2019 +0200

    Avoid setting the http response twice via http_PutResponse()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 778db6861..40f1f9185 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -628,7 +628,7 @@ double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
 ssize_t http_GetContentLength(const struct http *hp);
 uint16_t http_GetStatus(const struct http *hp);
 int http_IsStatus(const struct http *hp, int);
-void http_SetStatus(struct http *to, uint16_t status);
+void http_SetStatus(struct http *to, uint16_t status, const char *reason);
 const char *http_GetMethod(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
 void http_CopyHome(const struct http *hp);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index ec65a9cb6..5bf32db7b 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -798,10 +798,9 @@ http_IsStatus(const struct http *hp, int val)
  */
 
 void
-http_SetStatus(struct http *to, uint16_t status)
+http_SetStatus(struct http *to, uint16_t status, const char *reason)
 {
 	char buf[4];
-	const char *reason;
 	const char *sstr = NULL;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
@@ -813,7 +812,11 @@ http_SetStatus(struct http *to, uint16_t status)
 	status %= 1000;
 	assert(status >= 100);
 
-	reason = http_Status2Reason(status, &sstr);
+	if (reason == NULL)
+		reason = http_Status2Reason(status, &sstr);
+	else
+		(void)http_Status2Reason(status, &sstr);
+
 	if (sstr) {
 		http_SetH(to, HTTP_HDR_STATUS, sstr);
 	} else {
@@ -865,9 +868,7 @@ http_PutResponse(struct http *to, const char *proto, uint16_t status,
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	if (proto != NULL)
 		http_SetH(to, HTTP_HDR_PROTO, proto);
-	http_SetStatus(to, status);
-	if (reason != NULL)
-		http_SetH(to, HTTP_HDR_REASON, reason);
+	http_SetStatus(to, status, reason);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 2b471930e..7ed80cfd6 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -108,7 +108,7 @@ VRT_l_##obj##_status(VRT_CTX, VCL_INT num)				\
 		VRT_fail(ctx, "illegal %s.status (%jd) (..0##)",	\
 		    #obj, (intmax_t)num);				\
 	else								\
-		http_SetStatus(ctx->http_##obj, (uint16_t)num);		\
+		http_SetStatus(ctx->http_##obj, (uint16_t)num, NULL);	\
 }
 
 #define VRT_STATUS_R(obj)						\
diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c
index 31c75ed88..7fe4422db 100644
--- a/bin/varnishd/http1/cache_http1_proto.c
+++ b/bin/varnishd/http1/cache_http1_proto.c
@@ -453,7 +453,7 @@ HTTP1_DissectResponse(struct http_conn *htc, struct http *hp,
 		    (int)(htc->rxbuf_e - htc->rxbuf_b), htc->rxbuf_b);
 		assert(retval >= 100 && retval <= 999);
 		assert(retval == 503);
-		http_SetStatus(hp, 503);
+		http_SetStatus(hp, 503, NULL);
 	}
 
 	if (hp->hd[HTTP_HDR_REASON].b == NULL ||
diff --git a/bin/varnishtest/tests/c00018.vtc b/bin/varnishtest/tests/c00018.vtc
index cff0bebde..bb542b48f 100644
--- a/bin/varnishtest/tests/c00018.vtc
+++ b/bin/varnishtest/tests/c00018.vtc
@@ -125,8 +125,7 @@ logexpect l1 -v v1 -g raw {
 	expect 0 1011	RespProtocol    {^HTTP/1.1$}
 	expect 0 1011	RespStatus      {^405$}
 	expect 0 1011	RespReason      {^Method Not Allowed$}
-	# XXX dup RespReason
-	expect 1 1011	RespHeader      {^Date:}
+	expect 0 1011	RespHeader      {^Date:}
 	expect 0 1011	RespHeader      {^Server: Varnish$}
 	expect 0 1011	RespHeader      {^X-Varnish: 1011$}
 	expect * 1011	Timestamp       {^Process:}

From fgsch at lodoss.net  Thu Oct 31 07:14:08 2019
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 31 Oct 2019 07:14:08 +0000 (UTC)
Subject: [master] 84786afdf Make unwind work under macos
Message-ID: <20191031071408.5E77E11246B@lists.varnish-cache.org>


commit 84786afdf3d8afe08432907b87df86d77f8ee679
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Oct 31 11:07:30 2019 +0900

    Make unwind work under macos

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 37ea6b8fd..26be29cf7 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -647,7 +647,7 @@ pan_backtrace(struct vsb *vsb)
 		unw_get_reg(&cursor, UNW_REG_SP, &sp);
 		unw_get_proc_name(&cursor, fname, sizeof(fname), &offp);
 		VSB_printf(vsb, "ip=0x%lx, sp=0x%lx <%s+0x%lx>\n", (long) ip,
-		    (long) sp, fname[0] ? fname : "<unknown>", offp);
+		    (long) sp, fname[0] ? fname : "<unknown>", (long)offp);
 	}
 
 	VSB_indent(vsb, -2);
diff --git a/configure.ac b/configure.ac
index c5b5930cc..297e8223e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -349,7 +349,14 @@ AC_ARG_WITH([unwind],
               [use libunwind to print stacktraces (use libexecinfo otherwise). Recommended on alpine linux. Defaults to no.])])
 
 if test "$with_unwind" = yes; then
-    PKG_CHECK_MODULES([LIBUNWIND], [libunwind])
+    case $target in
+    *-*-darwin*)
+        # Always present but .pc is not installed
+        ;;
+    *)
+        PKG_CHECK_MODULES([LIBUNWIND], [libunwind])
+        ;;
+    esac
     AC_DEFINE([WITH_UNWIND], [1],
               [Define to 1 to use libunwind instead of libexecinfo])
 else

From phk at FreeBSD.org  Thu Oct 31 09:23:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 31 Oct 2019 09:23:06 +0000 (UTC)
Subject: [master] f9ca634fe Zero in first/between byte params do not wait
 forever.
Message-ID: <20191031092306.C0517114ADF@lists.varnish-cache.org>


commit f9ca634fe911b1e2e1f823c6191f0f9c18f6c8fa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 31 09:21:36 2019 +0000

    Zero in first/between byte params do not wait forever.
    
    Fixes #3045

diff --git a/include/tbl/params.h b/include/tbl/params.h
index 9e52e598b..fe1589371 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -245,10 +245,10 @@ PARAM(
 	/* flags */	0,
 	/* s-text */
 	"Default timeout for receiving first byte from backend. We only "
-	"wait for this many seconds for the first byte before giving up. A "
-	"value of 0 means it will never time out. VCL can override this "
-	"default value for each backend and backend request. This "
-	"parameter does not apply to pipe.",
+	"wait for this many seconds for the first byte before giving up.\n"
+	"VCL can override this default value for each backend and backend "
+	"request.\n"
+	"This parameter does not apply to pipe'ed requests.",
 	/* l-text */	"",
 	/* func */	NULL
 )
@@ -264,7 +264,6 @@ PARAM(
 	/* s-text */
 	"We only wait for this many seconds between bytes received from "
 	"the backend before giving up the fetch.\n"
-	"A value of zero means never give up.\n"
 	"VCL values, per backend or per backend request take precedence.\n"
 	"This parameter does not apply to pipe'ed requests.",
 	/* l-text */	"",

From dridi.boukelmoune at gmail.com  Thu Oct 31 09:27:06 2019
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 31 Oct 2019 09:27:06 +0000 (UTC)
Subject: [master] 6e47cd6ab Yield to newly bred threads
Message-ID: <20191031092706.5B218114D4B@lists.varnish-cache.org>


commit 6e47cd6abfa624552f72c7888ba9810b34a142f6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 28 17:06:35 2019 +0100

    Yield to newly bred threads
    
    When we breed threads without a delay, we should make an effort to let
    them run as soon as possible and not have the cpu occupied with the
    herder code.
    
    This will increase the overhead due to additional context switching for
    the benefit of reducing latencies until new threads get to run.
    
    Effect seen with this vtc by @Dridi:
    
        varnishtest "over-breeding"
    
        varnish v1 -arg "-p thread_pools=1 -p thread_pool_min=10"
        varnish v1 -vcl {
                backend be none;
        } -start
    
        varnish v1 -expect MAIN.threads == 10
    
    Tested with the following script:
    
        ./varnishtest -i -k -j40 -n1000 tests/as_above.vtc |
        awk '/^#/ { print $5; }'|
        sort |
        uniq -c
    
    Multiple runs show a consistent drop of failed tests:
    
        before | after
        -------+------
            76 |    18
            67 |    24
            66 |    18

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index a0662c221..01a379ddd 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -53,6 +53,7 @@
 #include "config.h"
 
 #include <stdlib.h>
+#include <sched.h>
 
 #include "cache_varnishd.h"
 #include "cache_pool.h"
@@ -478,7 +479,10 @@ pool_breed(struct pool *qp)
 		VSC_C_main->threads++;
 		VSC_C_main->threads_created++;
 		Lck_Unlock(&pool_mtx);
-		VTIM_sleep(cache_param->wthread_add_delay);
+		if (cache_param->wthread_add_delay > 0.0)
+			VTIM_sleep(cache_param->wthread_add_delay);
+		else
+			sched_yield();
 	}
 
 	AZ(pthread_attr_destroy(&tp_attr));

From daghf at varnish-software.com  Thu Oct 31 09:47:06 2019
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 31 Oct 2019 09:47:06 +0000 (UTC)
Subject: [master] 38526b68a Limit travis irc notifications to master branch
Message-ID: <20191031094706.E3D81115446@lists.varnish-cache.org>


commit 38526b68a2be966cfab8063bef2052609c4b42fa
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Oct 31 10:46:41 2019 +0100

    Limit travis irc notifications to master branch

diff --git a/.travis.yml b/.travis.yml
index 16a24b6aa..c5d157204 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -104,6 +104,7 @@ stages:
 
 notifications:
   irc:
+    if: branch = master
     channels:
       - "irc.linpro.no#varnish-hacking"
     on_success: change

From phk at FreeBSD.org  Thu Oct 31 13:24:06 2019
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 31 Oct 2019 13:24:06 +0000 (UTC)
Subject: [master] 2e125484e Give VRB_Iterate() explicit wrk+vsl_log arguments,
 since it is called from both client and backend side.
Message-ID: <20191031132406.C2540119597@lists.varnish-cache.org>


commit 2e125484ef5285628794dc19ac9093d3520ce333
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 31 13:17:20 2019 +0000

    Give VRB_Iterate() explicit wrk+vsl_log arguments, since it is called
    from both client and backend side.

diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index 13c4ed07f..ddb2fdc6e 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -179,7 +179,8 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv)
  */
 
 ssize_t
-VRB_Iterate(struct req *req, objiterate_f *func, void *priv)
+VRB_Iterate(struct worker *wrk, struct vsl_log *vsl,
+    struct req *req, objiterate_f *func, void *priv)
 {
 	int i;
 
@@ -189,7 +190,7 @@ VRB_Iterate(struct req *req, objiterate_f *func, void *priv)
 	switch (req->req_body_status) {
 	case REQ_BODY_CACHED:
 		AN(req->body_oc);
-		if (ObjIterate(req->wrk, req->body_oc, priv, func, 0))
+		if (ObjIterate(wrk, req->body_oc, priv, func, 0))
 			return (-1);
 		return (0);
 	case REQ_BODY_NONE:
@@ -198,11 +199,11 @@ VRB_Iterate(struct req *req, objiterate_f *func, void *priv)
 	case REQ_BODY_WITHOUT_LEN:
 		break;
 	case REQ_BODY_TAKEN:
-		VSLb(req->vsl, SLT_VCL_Error,
+		VSLb(vsl, SLT_VCL_Error,
 		    "Uncached req.body can only be consumed once.");
 		return (-1);
 	case REQ_BODY_FAIL:
-		VSLb(req->vsl, SLT_FetchError,
+		VSLb(vsl, SLT_FetchError,
 		    "Had failed reading req.body before.");
 		return (-1);
 	default:
@@ -217,7 +218,7 @@ VRB_Iterate(struct req *req, objiterate_f *func, void *priv)
 		i = -1;
 	Lck_Unlock(&req->sp->mtx);
 	if (i) {
-		VSLb(req->vsl, SLT_VCL_Error,
+		VSLb(vsl, SLT_VCL_Error,
 		    "Multiple attempts to access non-cached req.body");
 		return (i);
 	}
@@ -253,7 +254,7 @@ VRB_Ignore(struct req *req)
 		return (0);
 	if (req->req_body_status == REQ_BODY_WITH_LEN ||
 	    req->req_body_status == REQ_BODY_WITHOUT_LEN)
-		(void)VRB_Iterate(req, httpq_req_body_discard, NULL);
+		(void)VRB_Iterate(req->wrk, req->vsl, req, httpq_req_body_discard, NULL);
 	return (0);
 }
 
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 6a1ace8aa..f242b229c 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -355,7 +355,8 @@ void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *);
 /* cache_req_body.c */
 int VRB_Ignore(struct req *);
 ssize_t VRB_Cache(struct req *, ssize_t maxsize);
-ssize_t VRB_Iterate(struct req *, objiterate_f *func, void *priv);
+ssize_t VRB_Iterate(struct worker *, struct vsl_log *, struct req *,
+    objiterate_f *func, void *priv);
 void VRB_Free(struct req *);
 
 /* cache_req_fsm.c [CNT] */
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index dc1515ef1..12d5e7c2b 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -108,7 +108,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
 	    (bo->req->req_body_status == REQ_BODY_CACHED || !onlycached)) {
 		if (do_chunked)
 			V1L_Chunked(wrk);
-		i = VRB_Iterate(bo->req, vbf_iter_req_body, bo);
+		i = VRB_Iterate(wrk, bo->vsl, bo->req, vbf_iter_req_body, bo);
 
 		if (bo->req->req_body_status == REQ_BODY_FAIL) {
 			/*